michael@0: /* michael@0: * Copyright 2011 The LibYuv Project Authors. All rights reserved. michael@0: * michael@0: * Use of this source code is governed by a BSD-style license michael@0: * that can be found in the LICENSE file in the root of the source michael@0: * tree. An additional intellectual property rights grant can be found michael@0: * in the file PATENTS. All contributing project authors may michael@0: * be found in the AUTHORS file in the root of the source tree. michael@0: */ michael@0: michael@0: #include "libyuv/row.h" michael@0: michael@0: #ifdef __cplusplus michael@0: namespace libyuv { michael@0: extern "C" { michael@0: #endif michael@0: michael@0: // This module is for GCC x86 and x64. michael@0: #if !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || defined(__i386__)) michael@0: michael@0: #if defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3) michael@0: michael@0: // Constants for ARGB michael@0: static vec8 kARGBToY = { michael@0: 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33, 0 michael@0: }; michael@0: michael@0: // JPeg full range. michael@0: static vec8 kARGBToYJ = { michael@0: 15, 75, 38, 0, 15, 75, 38, 0, 15, 75, 38, 0, 15, 75, 38, 0 michael@0: }; michael@0: #endif // defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3) michael@0: michael@0: #if defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3) michael@0: michael@0: static vec8 kARGBToU = { michael@0: 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38, 0 michael@0: }; michael@0: michael@0: static vec8 kARGBToUJ = { michael@0: 127, -84, -43, 0, 127, -84, -43, 0, 127, -84, -43, 0, 127, -84, -43, 0 michael@0: }; michael@0: michael@0: static vec8 kARGBToV = { michael@0: -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, michael@0: }; michael@0: michael@0: static vec8 kARGBToVJ = { michael@0: -20, -107, 127, 0, -20, -107, 127, 0, -20, -107, 127, 0, -20, -107, 127, 0 michael@0: }; michael@0: michael@0: // Constants for BGRA michael@0: static vec8 kBGRAToY = { michael@0: 0, 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13 michael@0: }; michael@0: michael@0: static vec8 kBGRAToU = { michael@0: 0, -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112 michael@0: }; michael@0: michael@0: static vec8 kBGRAToV = { michael@0: 0, 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18 michael@0: }; michael@0: michael@0: // Constants for ABGR michael@0: static vec8 kABGRToY = { michael@0: 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13, 0, 33, 65, 13, 0 michael@0: }; michael@0: michael@0: static vec8 kABGRToU = { michael@0: -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112, 0, -38, -74, 112, 0 michael@0: }; michael@0: michael@0: static vec8 kABGRToV = { michael@0: 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18, 0, 112, -94, -18, 0 michael@0: }; michael@0: michael@0: // Constants for RGBA. michael@0: static vec8 kRGBAToY = { michael@0: 0, 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33, 0, 13, 65, 33 michael@0: }; michael@0: michael@0: static vec8 kRGBAToU = { michael@0: 0, 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38, 0, 112, -74, -38 michael@0: }; michael@0: michael@0: static vec8 kRGBAToV = { michael@0: 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112, 0, -18, -94, 112 michael@0: }; michael@0: michael@0: static uvec8 kAddY16 = { michael@0: 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u, 16u michael@0: }; michael@0: michael@0: static vec16 kAddYJ64 = { michael@0: 64, 64, 64, 64, 64, 64, 64, 64 michael@0: }; michael@0: michael@0: static uvec8 kAddUV128 = { michael@0: 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u, michael@0: 128u, 128u, 128u, 128u, 128u, 128u, 128u, 128u michael@0: }; michael@0: michael@0: static uvec16 kAddUVJ128 = { michael@0: 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u, 0x8080u michael@0: }; michael@0: #endif // defined(HAS_ARGBTOYROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3) michael@0: michael@0: #ifdef HAS_RGB24TOARGBROW_SSSE3 michael@0: michael@0: // Shuffle table for converting RGB24 to ARGB. michael@0: static uvec8 kShuffleMaskRGB24ToARGB = { michael@0: 0u, 1u, 2u, 12u, 3u, 4u, 5u, 13u, 6u, 7u, 8u, 14u, 9u, 10u, 11u, 15u michael@0: }; michael@0: michael@0: // Shuffle table for converting RAW to ARGB. michael@0: static uvec8 kShuffleMaskRAWToARGB = { michael@0: 2u, 1u, 0u, 12u, 5u, 4u, 3u, 13u, 8u, 7u, 6u, 14u, 11u, 10u, 9u, 15u michael@0: }; michael@0: michael@0: // Shuffle table for converting ARGB to RGB24. michael@0: static uvec8 kShuffleMaskARGBToRGB24 = { michael@0: 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u michael@0: }; michael@0: michael@0: // Shuffle table for converting ARGB to RAW. michael@0: static uvec8 kShuffleMaskARGBToRAW = { michael@0: 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, 128u michael@0: }; michael@0: michael@0: // Shuffle table for converting ARGBToRGB24 for I422ToRGB24. First 8 + next 4 michael@0: static uvec8 kShuffleMaskARGBToRGB24_0 = { michael@0: 0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u michael@0: }; michael@0: michael@0: // Shuffle table for converting ARGB to RAW. michael@0: static uvec8 kShuffleMaskARGBToRAW_0 = { michael@0: 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 128u, 128u, 128u, 128u, 8u, 14u, 13u, 12u michael@0: }; michael@0: #endif // HAS_RGB24TOARGBROW_SSSE3 michael@0: michael@0: #if defined(TESTING) && defined(__x86_64__) michael@0: void TestRow_SSE2(const uint8* src_y, uint8* dst_argb, int pix) { michael@0: asm volatile ( michael@0: ".p2align 5 \n" michael@0: "mov %%eax,%%eax \n" michael@0: "mov %%ebx,%%ebx \n" michael@0: "mov %%ecx,%%ecx \n" michael@0: "mov %%edx,%%edx \n" michael@0: "mov %%esi,%%esi \n" michael@0: "mov %%edi,%%edi \n" michael@0: "mov %%ebp,%%ebp \n" michael@0: "mov %%esp,%%esp \n" michael@0: ".p2align 5 \n" michael@0: "mov %%r8d,%%r8d \n" michael@0: "mov %%r9d,%%r9d \n" michael@0: "mov %%r10d,%%r10d \n" michael@0: "mov %%r11d,%%r11d \n" michael@0: "mov %%r12d,%%r12d \n" michael@0: "mov %%r13d,%%r13d \n" michael@0: "mov %%r14d,%%r14d \n" michael@0: "mov %%r15d,%%r15d \n" michael@0: ".p2align 5 \n" michael@0: "lea (%%rax),%%eax \n" michael@0: "lea (%%rbx),%%ebx \n" michael@0: "lea (%%rcx),%%ecx \n" michael@0: "lea (%%rdx),%%edx \n" michael@0: "lea (%%rsi),%%esi \n" michael@0: "lea (%%rdi),%%edi \n" michael@0: "lea (%%rbp),%%ebp \n" michael@0: "lea (%%rsp),%%esp \n" michael@0: ".p2align 5 \n" michael@0: "lea (%%r8),%%r8d \n" michael@0: "lea (%%r9),%%r9d \n" michael@0: "lea (%%r10),%%r10d \n" michael@0: "lea (%%r11),%%r11d \n" michael@0: "lea (%%r12),%%r12d \n" michael@0: "lea (%%r13),%%r13d \n" michael@0: "lea (%%r14),%%r14d \n" michael@0: "lea (%%r15),%%r15d \n" michael@0: michael@0: ".p2align 5 \n" michael@0: "lea 0x10(%%rax),%%eax \n" michael@0: "lea 0x10(%%rbx),%%ebx \n" michael@0: "lea 0x10(%%rcx),%%ecx \n" michael@0: "lea 0x10(%%rdx),%%edx \n" michael@0: "lea 0x10(%%rsi),%%esi \n" michael@0: "lea 0x10(%%rdi),%%edi \n" michael@0: "lea 0x10(%%rbp),%%ebp \n" michael@0: "lea 0x10(%%rsp),%%esp \n" michael@0: ".p2align 5 \n" michael@0: "lea 0x10(%%r8),%%r8d \n" michael@0: "lea 0x10(%%r9),%%r9d \n" michael@0: "lea 0x10(%%r10),%%r10d \n" michael@0: "lea 0x10(%%r11),%%r11d \n" michael@0: "lea 0x10(%%r12),%%r12d \n" michael@0: "lea 0x10(%%r13),%%r13d \n" michael@0: "lea 0x10(%%r14),%%r14d \n" michael@0: "lea 0x10(%%r15),%%r15d \n" michael@0: michael@0: ".p2align 5 \n" michael@0: "add 0x10,%%eax \n" michael@0: "add 0x10,%%ebx \n" michael@0: "add 0x10,%%ecx \n" michael@0: "add 0x10,%%edx \n" michael@0: "add 0x10,%%esi \n" michael@0: "add 0x10,%%edi \n" michael@0: "add 0x10,%%ebp \n" michael@0: "add 0x10,%%esp \n" michael@0: ".p2align 5 \n" michael@0: "add 0x10,%%r8d \n" michael@0: "add 0x10,%%r9d \n" michael@0: "add 0x10,%%r10d \n" michael@0: "add 0x10,%%r11d \n" michael@0: "add 0x10,%%r12d \n" michael@0: "add 0x10,%%r13d \n" michael@0: "add 0x10,%%r14d \n" michael@0: "add 0x10,%%r15d \n" michael@0: michael@0: ".p2align 2 \n" michael@0: "1: \n" michael@0: "movq " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x8,0) ",%0 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "sub $0x8,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_y), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // TESTING michael@0: michael@0: #ifdef HAS_I400TOARGBROW_SSE2 michael@0: void I400ToARGBRow_SSE2(const uint8* src_y, uint8* dst_argb, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pslld $0x18,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movq " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x8,0) ",%0 \n" michael@0: "punpcklbw %%xmm0,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm0,%%xmm0 \n" michael@0: "punpckhwd %%xmm1,%%xmm1 \n" michael@0: "por %%xmm5,%%xmm0 \n" michael@0: "por %%xmm5,%%xmm1 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "sub $0x8,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_y), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void I400ToARGBRow_Unaligned_SSE2(const uint8* src_y, uint8* dst_argb, michael@0: int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pslld $0x18,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movq " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x8,0) ",%0 \n" michael@0: "punpcklbw %%xmm0,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm0,%%xmm0 \n" michael@0: "punpckhwd %%xmm1,%%xmm1 \n" michael@0: "por %%xmm5,%%xmm0 \n" michael@0: "por %%xmm5,%%xmm1 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "sub $0x8,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_y), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_I400TOARGBROW_SSE2 michael@0: michael@0: #ifdef HAS_RGB24TOARGBROW_SSSE3 michael@0: void RGB24ToARGBRow_SSSE3(const uint8* src_rgb24, uint8* dst_argb, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" // generate mask 0xff000000 michael@0: "pslld $0x18,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x30,0) ",%0 \n" michael@0: "movdqa %%xmm3,%%xmm2 \n" michael@0: "palignr $0x8,%%xmm1,%%xmm2 \n" michael@0: "pshufb %%xmm4,%%xmm2 \n" michael@0: "por %%xmm5,%%xmm2 \n" michael@0: "palignr $0xc,%%xmm0,%%xmm1 \n" michael@0: "pshufb %%xmm4,%%xmm0 \n" michael@0: "movdqa %%xmm2," MEMACCESS2(0x20,1) " \n" michael@0: "por %%xmm5,%%xmm0 \n" michael@0: "pshufb %%xmm4,%%xmm1 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "por %%xmm5,%%xmm1 \n" michael@0: "palignr $0x4,%%xmm3,%%xmm3 \n" michael@0: "pshufb %%xmm4,%%xmm3 \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "por %%xmm5,%%xmm3 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqa %%xmm3," MEMACCESS2(0x30,1) " \n" michael@0: "lea " MEMLEA(0x40,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_rgb24), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kShuffleMaskRGB24ToARGB) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void RAWToARGBRow_SSSE3(const uint8* src_raw, uint8* dst_argb, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" // generate mask 0xff000000 michael@0: "pslld $0x18,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x30,0) ",%0 \n" michael@0: "movdqa %%xmm3,%%xmm2 \n" michael@0: "palignr $0x8,%%xmm1,%%xmm2 \n" michael@0: "pshufb %%xmm4,%%xmm2 \n" michael@0: "por %%xmm5,%%xmm2 \n" michael@0: "palignr $0xc,%%xmm0,%%xmm1 \n" michael@0: "pshufb %%xmm4,%%xmm0 \n" michael@0: "movdqa %%xmm2," MEMACCESS2(0x20,1) " \n" michael@0: "por %%xmm5,%%xmm0 \n" michael@0: "pshufb %%xmm4,%%xmm1 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "por %%xmm5,%%xmm1 \n" michael@0: "palignr $0x4,%%xmm3,%%xmm3 \n" michael@0: "pshufb %%xmm4,%%xmm3 \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "por %%xmm5,%%xmm3 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqa %%xmm3," MEMACCESS2(0x30,1) " \n" michael@0: "lea " MEMLEA(0x40,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_raw), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kShuffleMaskRAWToARGB) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void RGB565ToARGBRow_SSE2(const uint8* src, uint8* dst, int pix) { michael@0: asm volatile ( michael@0: "mov $0x1080108,%%eax \n" michael@0: "movd %%eax,%%xmm5 \n" michael@0: "pshufd $0x0,%%xmm5,%%xmm5 \n" michael@0: "mov $0x20802080,%%eax \n" michael@0: "movd %%eax,%%xmm6 \n" michael@0: "pshufd $0x0,%%xmm6,%%xmm6 \n" michael@0: "pcmpeqb %%xmm3,%%xmm3 \n" michael@0: "psllw $0xb,%%xmm3 \n" michael@0: "pcmpeqb %%xmm4,%%xmm4 \n" michael@0: "psllw $0xa,%%xmm4 \n" michael@0: "psrlw $0x5,%%xmm4 \n" michael@0: "pcmpeqb %%xmm7,%%xmm7 \n" michael@0: "psllw $0x8,%%xmm7 \n" michael@0: "sub %0,%1 \n" michael@0: "sub %0,%1 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "pand %%xmm3,%%xmm1 \n" michael@0: "psllw $0xb,%%xmm2 \n" michael@0: "pmulhuw %%xmm5,%%xmm1 \n" michael@0: "pmulhuw %%xmm5,%%xmm2 \n" michael@0: "psllw $0x8,%%xmm1 \n" michael@0: "por %%xmm2,%%xmm1 \n" michael@0: "pand %%xmm4,%%xmm0 \n" michael@0: "pmulhuw %%xmm6,%%xmm0 \n" michael@0: "por %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm1,%%xmm2 \n" michael@0: "punpcklbw %%xmm0,%%xmm1 \n" michael@0: "punpckhbw %%xmm0,%%xmm2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm1,0x00,1,0,2) // movdqa %%xmm1,(%1,%0,2) michael@0: MEMOPMEM(movdqa,xmm2,0x10,1,0,2) // movdqa %%xmm2,0x10(%1,%0,2) michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "sub $0x8,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc", "eax" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGB1555ToARGBRow_SSE2(const uint8* src, uint8* dst, int pix) { michael@0: asm volatile ( michael@0: "mov $0x1080108,%%eax \n" michael@0: "movd %%eax,%%xmm5 \n" michael@0: "pshufd $0x0,%%xmm5,%%xmm5 \n" michael@0: "mov $0x42004200,%%eax \n" michael@0: "movd %%eax,%%xmm6 \n" michael@0: "pshufd $0x0,%%xmm6,%%xmm6 \n" michael@0: "pcmpeqb %%xmm3,%%xmm3 \n" michael@0: "psllw $0xb,%%xmm3 \n" michael@0: "movdqa %%xmm3,%%xmm4 \n" michael@0: "psrlw $0x6,%%xmm4 \n" michael@0: "pcmpeqb %%xmm7,%%xmm7 \n" michael@0: "psllw $0x8,%%xmm7 \n" michael@0: "sub %0,%1 \n" michael@0: "sub %0,%1 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "psllw $0x1,%%xmm1 \n" michael@0: "psllw $0xb,%%xmm2 \n" michael@0: "pand %%xmm3,%%xmm1 \n" michael@0: "pmulhuw %%xmm5,%%xmm2 \n" michael@0: "pmulhuw %%xmm5,%%xmm1 \n" michael@0: "psllw $0x8,%%xmm1 \n" michael@0: "por %%xmm2,%%xmm1 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "pand %%xmm4,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm2 \n" michael@0: "pmulhuw %%xmm6,%%xmm0 \n" michael@0: "pand %%xmm7,%%xmm2 \n" michael@0: "por %%xmm2,%%xmm0 \n" michael@0: "movdqa %%xmm1,%%xmm2 \n" michael@0: "punpcklbw %%xmm0,%%xmm1 \n" michael@0: "punpckhbw %%xmm0,%%xmm2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm1,0x00,1,0,2) // movdqa %%xmm1,(%1,%0,2) michael@0: MEMOPMEM(movdqa,xmm2,0x10,1,0,2) // movdqa %%xmm2,0x10(%1,%0,2) michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "sub $0x8,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc", "eax" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGB4444ToARGBRow_SSE2(const uint8* src, uint8* dst, int pix) { michael@0: asm volatile ( michael@0: "mov $0xf0f0f0f,%%eax \n" michael@0: "movd %%eax,%%xmm4 \n" michael@0: "pshufd $0x0,%%xmm4,%%xmm4 \n" michael@0: "movdqa %%xmm4,%%xmm5 \n" michael@0: "pslld $0x4,%%xmm5 \n" michael@0: "sub %0,%1 \n" michael@0: "sub %0,%1 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "pand %%xmm4,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm3 \n" michael@0: "psllw $0x4,%%xmm1 \n" michael@0: "psrlw $0x4,%%xmm3 \n" michael@0: "por %%xmm1,%%xmm0 \n" michael@0: "por %%xmm3,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm2,%%xmm0 \n" michael@0: "punpckhbw %%xmm2,%%xmm1 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,2) // movdqa %%xmm0,(%1,%0,2) michael@0: MEMOPMEM(movdqa,xmm1,0x10,1,0,2) // movdqa %%xmm1,0x10(%1,%0,2) michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "sub $0x8,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc", "eax" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToRGB24Row_SSSE3(const uint8* src, uint8* dst, int pix) { michael@0: asm volatile ( michael@0: "movdqa %3,%%xmm6 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "pshufb %%xmm6,%%xmm0 \n" michael@0: "pshufb %%xmm6,%%xmm1 \n" michael@0: "pshufb %%xmm6,%%xmm2 \n" michael@0: "pshufb %%xmm6,%%xmm3 \n" michael@0: "movdqa %%xmm1,%%xmm4 \n" michael@0: "psrldq $0x4,%%xmm1 \n" michael@0: "pslldq $0xc,%%xmm4 \n" michael@0: "movdqa %%xmm2,%%xmm5 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "pslldq $0x8,%%xmm5 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "por %%xmm5,%%xmm1 \n" michael@0: "psrldq $0x8,%%xmm2 \n" michael@0: "pslldq $0x4,%%xmm3 \n" michael@0: "por %%xmm3,%%xmm2 \n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "movdqu %%xmm2," MEMACCESS2(0x20,1) " \n" michael@0: "lea " MEMLEA(0x30,1) ",%1 \n" michael@0: "sub $0x10,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kShuffleMaskARGBToRGB24) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToRAWRow_SSSE3(const uint8* src, uint8* dst, int pix) { michael@0: asm volatile ( michael@0: "movdqa %3,%%xmm6 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "pshufb %%xmm6,%%xmm0 \n" michael@0: "pshufb %%xmm6,%%xmm1 \n" michael@0: "pshufb %%xmm6,%%xmm2 \n" michael@0: "pshufb %%xmm6,%%xmm3 \n" michael@0: "movdqa %%xmm1,%%xmm4 \n" michael@0: "psrldq $0x4,%%xmm1 \n" michael@0: "pslldq $0xc,%%xmm4 \n" michael@0: "movdqa %%xmm2,%%xmm5 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "pslldq $0x8,%%xmm5 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "por %%xmm5,%%xmm1 \n" michael@0: "psrldq $0x8,%%xmm2 \n" michael@0: "pslldq $0x4,%%xmm3 \n" michael@0: "por %%xmm3,%%xmm2 \n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "movdqu %%xmm2," MEMACCESS2(0x20,1) " \n" michael@0: "lea " MEMLEA(0x30,1) ",%1 \n" michael@0: "sub $0x10,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kShuffleMaskARGBToRAW) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToRGB565Row_SSE2(const uint8* src, uint8* dst, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm3,%%xmm3 \n" michael@0: "psrld $0x1b,%%xmm3 \n" michael@0: "pcmpeqb %%xmm4,%%xmm4 \n" michael@0: "psrld $0x1a,%%xmm4 \n" michael@0: "pslld $0x5,%%xmm4 \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pslld $0xb,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "pslld $0x8,%%xmm0 \n" michael@0: "psrld $0x3,%%xmm1 \n" michael@0: "psrld $0x5,%%xmm2 \n" michael@0: "psrad $0x10,%%xmm0 \n" michael@0: "pand %%xmm3,%%xmm1 \n" michael@0: "pand %%xmm4,%%xmm2 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "por %%xmm2,%%xmm1 \n" michael@0: "por %%xmm1,%%xmm0 \n" michael@0: "packssdw %%xmm0,%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x4,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToARGB1555Row_SSE2(const uint8* src, uint8* dst, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm4,%%xmm4 \n" michael@0: "psrld $0x1b,%%xmm4 \n" michael@0: "movdqa %%xmm4,%%xmm5 \n" michael@0: "pslld $0x5,%%xmm5 \n" michael@0: "movdqa %%xmm4,%%xmm6 \n" michael@0: "pslld $0xa,%%xmm6 \n" michael@0: "pcmpeqb %%xmm7,%%xmm7 \n" michael@0: "pslld $0xf,%%xmm7 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm3 \n" michael@0: "psrad $0x10,%%xmm0 \n" michael@0: "psrld $0x3,%%xmm1 \n" michael@0: "psrld $0x6,%%xmm2 \n" michael@0: "psrld $0x9,%%xmm3 \n" michael@0: "pand %%xmm7,%%xmm0 \n" michael@0: "pand %%xmm4,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm2 \n" michael@0: "pand %%xmm6,%%xmm3 \n" michael@0: "por %%xmm1,%%xmm0 \n" michael@0: "por %%xmm3,%%xmm2 \n" michael@0: "por %%xmm2,%%xmm0 \n" michael@0: "packssdw %%xmm0,%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMACCESS2(0x8,1) ",%1 \n" michael@0: "sub $0x4,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToARGB4444Row_SSE2(const uint8* src, uint8* dst, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm4,%%xmm4 \n" michael@0: "psllw $0xc,%%xmm4 \n" michael@0: "movdqa %%xmm4,%%xmm3 \n" michael@0: "psrlw $0x8,%%xmm3 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "pand %%xmm3,%%xmm0 \n" michael@0: "pand %%xmm4,%%xmm1 \n" michael@0: "psrlq $0x4,%%xmm0 \n" michael@0: "psrlq $0x8,%%xmm1 \n" michael@0: "por %%xmm1,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x4,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_RGB24TOARGBROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBTOYROW_SSSE3 michael@0: void ARGBToYRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %4,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kARGBToY), // %3 michael@0: "m"(kAddY16) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToYRow_Unaligned_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %4,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kARGBToY), // %3 michael@0: "m"(kAddY16) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBTOYROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBTOYJROW_SSSE3 michael@0: void ARGBToYJRow_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %3,%%xmm4 \n" michael@0: "movdqa %4,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "paddw %%xmm5,%%xmm0 \n" michael@0: "paddw %%xmm5,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kARGBToYJ), // %3 michael@0: "m"(kAddYJ64) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToYJRow_Unaligned_SSSE3(const uint8* src_argb, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %3,%%xmm4 \n" michael@0: "movdqa %4,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "paddw %%xmm5,%%xmm0 \n" michael@0: "paddw %%xmm5,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kARGBToYJ), // %3 michael@0: "m"(kAddYJ64) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBTOYJROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBTOUVROW_SSSE3 michael@0: // TODO(fbarchard): pass xmm constants to single block of assembly. michael@0: // fpic on GCC 4.2 for OSX runs out of GPR registers. "m" effectively takes michael@0: // 3 registers - ebx, ebp and eax. "m" can be passed with 3 normal registers, michael@0: // or 4 if stack frame is disabled. Doing 2 assembly blocks is a work around michael@0: // and considered unsafe. michael@0: void ARGBToUVRow_SSSE3(const uint8* src_argb0, int src_stride_argb, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kARGBToU), // %0 michael@0: "m"(kARGBToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(pavgb,0x00,0,4,1,xmm0) // pavgb (%0,%4,1),%%xmm0 michael@0: MEMOPREG(pavgb,0x10,0,4,1,xmm1) // pavgb 0x10(%0,%4,1),%%xmm1 michael@0: MEMOPREG(pavgb,0x20,0,4,1,xmm2) // pavgb 0x20(%0,%4,1),%%xmm2 michael@0: MEMOPREG(pavgb,0x30,0,4,1,xmm6) // pavgb 0x30(%0,%4,1),%%xmm6 michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_argb)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: // TODO(fbarchard): Share code with ARGBToUVRow_SSSE3. michael@0: void ARGBToUVJRow_SSSE3(const uint8* src_argb0, int src_stride_argb, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kARGBToUJ), // %0 michael@0: "m"(kARGBToVJ), // %1 michael@0: "m"(kAddUVJ128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(pavgb,0x00,0,4,1,xmm0) // pavgb (%0,%4,1),%%xmm0 michael@0: MEMOPREG(pavgb,0x10,0,4,1,xmm1) // pavgb 0x10(%0,%4,1),%%xmm1 michael@0: MEMOPREG(pavgb,0x20,0,4,1,xmm2) // pavgb 0x20(%0,%4,1),%%xmm2 michael@0: MEMOPREG(pavgb,0x30,0,4,1,xmm6) // pavgb 0x30(%0,%4,1),%%xmm6 michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "paddw %%xmm5,%%xmm0 \n" michael@0: "paddw %%xmm5,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_argb)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToUVRow_Unaligned_SSSE3(const uint8* src_argb0, int src_stride_argb, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kARGBToU), // %0 michael@0: "m"(kARGBToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm1 \n" michael@0: MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm6 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_argb)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToUVJRow_Unaligned_SSSE3(const uint8* src_argb0, int src_stride_argb, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kARGBToUJ), // %0 michael@0: "m"(kARGBToVJ), // %1 michael@0: "m"(kAddUVJ128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm1 \n" michael@0: MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm6 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "paddw %%xmm5,%%xmm0 \n" michael@0: "paddw %%xmm5,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_argb)) michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToUV444Row_SSSE3(const uint8* src_argb, uint8* dst_u, uint8* dst_v, michael@0: int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kARGBToU), // %0 michael@0: "m"(kARGBToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm6 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm2 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm2 \n" michael@0: "packsswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: "pmaddubsw %%xmm3,%%xmm0 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm2 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm2 \n" michael@0: "packsswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,2,1) // movdqa %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToUV444Row_Unaligned_SSSE3(const uint8* src_argb, uint8* dst_u, michael@0: uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kARGBToU), // %0 michael@0: "m"(kARGBToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm6 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm2 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm2 \n" michael@0: "packsswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: "pmaddubsw %%xmm3,%%xmm0 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm2 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm2 \n" michael@0: "packsswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,2,1) // movdqu %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToUV422Row_SSSE3(const uint8* src_argb0, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kARGBToU), // %0 michael@0: "m"(kARGBToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBToUV422Row_Unaligned_SSSE3(const uint8* src_argb0, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kARGBToU), // %0 michael@0: "m"(kARGBToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void BGRAToYRow_SSSE3(const uint8* src_bgra, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %4,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_bgra), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kBGRAToY), // %3 michael@0: "m"(kAddY16) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void BGRAToYRow_Unaligned_SSSE3(const uint8* src_bgra, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %4,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_bgra), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kBGRAToY), // %3 michael@0: "m"(kAddY16) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void BGRAToUVRow_SSSE3(const uint8* src_bgra0, int src_stride_bgra, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kBGRAToU), // %0 michael@0: "m"(kBGRAToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(pavgb,0x00,0,4,1,xmm0) // pavgb (%0,%4,1),%%xmm0 michael@0: MEMOPREG(pavgb,0x10,0,4,1,xmm1) // pavgb 0x10(%0,%4,1),%%xmm1 michael@0: MEMOPREG(pavgb,0x20,0,4,1,xmm2) // pavgb 0x20(%0,%4,1),%%xmm2 michael@0: MEMOPREG(pavgb,0x30,0,4,1,xmm6) // pavgb 0x30(%0,%4,1),%%xmm6 michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_bgra0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_bgra)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void BGRAToUVRow_Unaligned_SSSE3(const uint8* src_bgra0, int src_stride_bgra, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kBGRAToU), // %0 michael@0: "m"(kBGRAToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm1 \n" michael@0: MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm6 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_bgra0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_bgra)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ABGRToYRow_SSSE3(const uint8* src_abgr, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %4,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_abgr), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kABGRToY), // %3 michael@0: "m"(kAddY16) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ABGRToYRow_Unaligned_SSSE3(const uint8* src_abgr, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %4,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_abgr), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kABGRToY), // %3 michael@0: "m"(kAddY16) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void RGBAToYRow_SSSE3(const uint8* src_rgba, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %4,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_rgba), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kRGBAToY), // %3 michael@0: "m"(kAddY16) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void RGBAToYRow_Unaligned_SSSE3(const uint8* src_rgba, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "movdqa %4,%%xmm5 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm4,%%xmm3 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "phaddw %%xmm3,%%xmm2 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_rgba), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : "m"(kRGBAToY), // %3 michael@0: "m"(kAddY16) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ABGRToUVRow_SSSE3(const uint8* src_abgr0, int src_stride_abgr, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kABGRToU), // %0 michael@0: "m"(kABGRToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(pavgb,0x00,0,4,1,xmm0) // pavgb (%0,%4,1),%%xmm0 michael@0: MEMOPREG(pavgb,0x10,0,4,1,xmm1) // pavgb 0x10(%0,%4,1),%%xmm1 michael@0: MEMOPREG(pavgb,0x20,0,4,1,xmm2) // pavgb 0x20(%0,%4,1),%%xmm2 michael@0: MEMOPREG(pavgb,0x30,0,4,1,xmm6) // pavgb 0x30(%0,%4,1),%%xmm6 michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_abgr0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_abgr)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ABGRToUVRow_Unaligned_SSSE3(const uint8* src_abgr0, int src_stride_abgr, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kABGRToU), // %0 michael@0: "m"(kABGRToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm1 \n" michael@0: MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm6 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_abgr0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_abgr)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void RGBAToUVRow_SSSE3(const uint8* src_rgba0, int src_stride_rgba, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kRGBAToU), // %0 michael@0: "m"(kRGBAToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(pavgb,0x00,0,4,1,xmm0) // pavgb (%0,%4,1),%%xmm0 michael@0: MEMOPREG(pavgb,0x10,0,4,1,xmm1) // pavgb 0x10(%0,%4,1),%%xmm1 michael@0: MEMOPREG(pavgb,0x20,0,4,1,xmm2) // pavgb 0x20(%0,%4,1),%%xmm2 michael@0: MEMOPREG(pavgb,0x30,0,4,1,xmm6) // pavgb 0x30(%0,%4,1),%%xmm6 michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_rgba0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_rgba)) michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void RGBAToUVRow_Unaligned_SSSE3(const uint8* src_rgba0, int src_stride_rgba, michael@0: uint8* dst_u, uint8* dst_v, int width) { michael@0: asm volatile ( michael@0: "movdqa %0,%%xmm4 \n" michael@0: "movdqa %1,%%xmm3 \n" michael@0: "movdqa %2,%%xmm5 \n" michael@0: : michael@0: : "m"(kRGBAToU), // %0 michael@0: "m"(kRGBAToV), // %1 michael@0: "m"(kAddUV128) // %2 michael@0: ); michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqu " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqu " MEMACCESS2(0x30,0) ",%%xmm6 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movdqu,0x00,0,4,1,xmm7) // movdqu (%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x10,0,4,1,xmm7) // movdqu 0x10(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm1 \n" michael@0: MEMOPREG(movdqu,0x20,0,4,1,xmm7) // movdqu 0x20(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: MEMOPREG(movdqu,0x30,0,4,1,xmm7) // movdqu 0x30(%0,%4,1),%%xmm7 michael@0: "pavgb %%xmm7,%%xmm6 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm1,%%xmm0 \n" michael@0: "shufps $0xdd,%%xmm1,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm7 \n" michael@0: "shufps $0x88,%%xmm6,%%xmm2 \n" michael@0: "shufps $0xdd,%%xmm6,%%xmm7 \n" michael@0: "pavgb %%xmm7,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm6 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm2 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "phaddw %%xmm2,%%xmm0 \n" michael@0: "phaddw %%xmm6,%%xmm1 \n" michael@0: "psraw $0x8,%%xmm0 \n" michael@0: "psraw $0x8,%%xmm1 \n" michael@0: "packsswb %%xmm1,%%xmm0 \n" michael@0: "paddb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movlps %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhps,xmm0,0x00,1,2,1) // movhps %%xmm0,(%1,%2,1) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_rgba0), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+rm"(width) // %3 michael@0: : "r"((intptr_t)(src_stride_rgba)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBTOUVROW_SSSE3 michael@0: michael@0: #ifdef HAS_I422TOARGBROW_SSSE3 michael@0: #define UB 127 /* min(63,(int8)(2.018 * 64)) */ michael@0: #define UG -25 /* (int8)(-0.391 * 64 - 0.5) */ michael@0: #define UR 0 michael@0: michael@0: #define VB 0 michael@0: #define VG -52 /* (int8)(-0.813 * 64 - 0.5) */ michael@0: #define VR 102 /* (int8)(1.596 * 64 + 0.5) */ michael@0: michael@0: // Bias michael@0: #define BB UB * 128 + VB * 128 michael@0: #define BG UG * 128 + VG * 128 michael@0: #define BR UR * 128 + VR * 128 michael@0: michael@0: #define YG 74 /* (int8)(1.164 * 64 + 0.5) */ michael@0: michael@0: struct { michael@0: vec8 kUVToB; // 0 michael@0: vec8 kUVToG; // 16 michael@0: vec8 kUVToR; // 32 michael@0: vec16 kUVBiasB; // 48 michael@0: vec16 kUVBiasG; // 64 michael@0: vec16 kUVBiasR; // 80 michael@0: vec16 kYSub16; // 96 michael@0: vec16 kYToRgb; // 112 michael@0: vec8 kVUToB; // 128 michael@0: vec8 kVUToG; // 144 michael@0: vec8 kVUToR; // 160 michael@0: } static SIMD_ALIGNED(kYuvConstants) = { michael@0: { UB, VB, UB, VB, UB, VB, UB, VB, UB, VB, UB, VB, UB, VB, UB, VB }, michael@0: { UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG }, michael@0: { UR, VR, UR, VR, UR, VR, UR, VR, UR, VR, UR, VR, UR, VR, UR, VR }, michael@0: { BB, BB, BB, BB, BB, BB, BB, BB }, michael@0: { BG, BG, BG, BG, BG, BG, BG, BG }, michael@0: { BR, BR, BR, BR, BR, BR, BR, BR }, michael@0: { 16, 16, 16, 16, 16, 16, 16, 16 }, michael@0: { YG, YG, YG, YG, YG, YG, YG, YG }, michael@0: { VB, UB, VB, UB, VB, UB, VB, UB, VB, UB, VB, UB, VB, UB, VB, UB }, michael@0: { VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG, VG, UG }, michael@0: { VR, UR, VR, UR, VR, UR, VR, UR, VR, UR, VR, UR, VR, UR, VR, UR } michael@0: }; michael@0: michael@0: michael@0: // Read 8 UV from 411 michael@0: #define READYUV444 \ michael@0: "movq " MEMACCESS([u_buf]) ",%%xmm0 \n" \ michael@0: BUNDLEALIGN \ michael@0: MEMOPREG(movq, 0x00, [u_buf], [v_buf], 1, xmm1) \ michael@0: "lea " MEMLEA(0x8, [u_buf]) ",%[u_buf] \n" \ michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: michael@0: // Read 4 UV from 422, upsample to 8 UV michael@0: #define READYUV422 \ michael@0: "movd " MEMACCESS([u_buf]) ",%%xmm0 \n" \ michael@0: BUNDLEALIGN \ michael@0: MEMOPREG(movd, 0x00, [u_buf], [v_buf], 1, xmm1) \ michael@0: "lea " MEMLEA(0x4, [u_buf]) ",%[u_buf] \n" \ michael@0: "punpcklbw %%xmm1,%%xmm0 \n" \ michael@0: "punpcklwd %%xmm0,%%xmm0 \n" michael@0: michael@0: // Read 2 UV from 411, upsample to 8 UV michael@0: #define READYUV411 \ michael@0: "movd " MEMACCESS([u_buf]) ",%%xmm0 \n" \ michael@0: BUNDLEALIGN \ michael@0: MEMOPREG(movd, 0x00, [u_buf], [v_buf], 1, xmm1) \ michael@0: "lea " MEMLEA(0x2, [u_buf]) ",%[u_buf] \n" \ michael@0: "punpcklbw %%xmm1,%%xmm0 \n" \ michael@0: "punpcklwd %%xmm0,%%xmm0 \n" \ michael@0: "punpckldq %%xmm0,%%xmm0 \n" michael@0: michael@0: // Read 4 UV from NV12, upsample to 8 UV michael@0: #define READNV12 \ michael@0: "movq " MEMACCESS([uv_buf]) ",%%xmm0 \n" \ michael@0: "lea " MEMLEA(0x8, [uv_buf]) ",%[uv_buf] \n" \ michael@0: "punpcklwd %%xmm0,%%xmm0 \n" michael@0: michael@0: // Convert 8 pixels: 8 UV and 8 Y michael@0: #define YUVTORGB \ michael@0: "movdqa %%xmm0,%%xmm1 \n" \ michael@0: "movdqa %%xmm0,%%xmm2 \n" \ michael@0: "pmaddubsw " MEMACCESS([kYuvConstants]) ",%%xmm0 \n" \ michael@0: "pmaddubsw " MEMACCESS2(16, [kYuvConstants]) ",%%xmm1 \n" \ michael@0: "pmaddubsw " MEMACCESS2(32, [kYuvConstants]) ",%%xmm2 \n" \ michael@0: "psubw " MEMACCESS2(48, [kYuvConstants]) ",%%xmm0 \n" \ michael@0: "psubw " MEMACCESS2(64, [kYuvConstants]) ",%%xmm1 \n" \ michael@0: "psubw " MEMACCESS2(80, [kYuvConstants]) ",%%xmm2 \n" \ michael@0: "movq " MEMACCESS([y_buf]) ",%%xmm3 \n" \ michael@0: "lea " MEMLEA(0x8, [y_buf]) ",%[y_buf] \n" \ michael@0: "punpcklbw %%xmm4,%%xmm3 \n" \ michael@0: "psubsw " MEMACCESS2(96, [kYuvConstants]) ",%%xmm3 \n" \ michael@0: "pmullw " MEMACCESS2(112, [kYuvConstants]) ",%%xmm3 \n" \ michael@0: "paddsw %%xmm3,%%xmm0 \n" \ michael@0: "paddsw %%xmm3,%%xmm1 \n" \ michael@0: "paddsw %%xmm3,%%xmm2 \n" \ michael@0: "psraw $0x6,%%xmm0 \n" \ michael@0: "psraw $0x6,%%xmm1 \n" \ michael@0: "psraw $0x6,%%xmm2 \n" \ michael@0: "packuswb %%xmm0,%%xmm0 \n" \ michael@0: "packuswb %%xmm1,%%xmm1 \n" \ michael@0: "packuswb %%xmm2,%%xmm2 \n" michael@0: michael@0: // Convert 8 pixels: 8 VU and 8 Y michael@0: #define YVUTORGB \ michael@0: "movdqa %%xmm0,%%xmm1 \n" \ michael@0: "movdqa %%xmm0,%%xmm2 \n" \ michael@0: "pmaddubsw " MEMACCESS2(128, [kYuvConstants]) ",%%xmm0 \n" \ michael@0: "pmaddubsw " MEMACCESS2(144, [kYuvConstants]) ",%%xmm1 \n" \ michael@0: "pmaddubsw " MEMACCESS2(160, [kYuvConstants]) ",%%xmm2 \n" \ michael@0: "psubw " MEMACCESS2(48, [kYuvConstants]) ",%%xmm0 \n" \ michael@0: "psubw " MEMACCESS2(64, [kYuvConstants]) ",%%xmm1 \n" \ michael@0: "psubw " MEMACCESS2(80, [kYuvConstants]) ",%%xmm2 \n" \ michael@0: "movq " MEMACCESS([y_buf]) ",%%xmm3 \n" \ michael@0: "lea " MEMLEA(0x8, [y_buf]) ",%[y_buf] \n" \ michael@0: "punpcklbw %%xmm4,%%xmm3 \n" \ michael@0: "psubsw " MEMACCESS2(96, [kYuvConstants]) ",%%xmm3 \n" \ michael@0: "pmullw " MEMACCESS2(112, [kYuvConstants]) ",%%xmm3 \n" \ michael@0: "paddsw %%xmm3,%%xmm0 \n" \ michael@0: "paddsw %%xmm3,%%xmm1 \n" \ michael@0: "paddsw %%xmm3,%%xmm2 \n" \ michael@0: "psraw $0x6,%%xmm0 \n" \ michael@0: "psraw $0x6,%%xmm1 \n" \ michael@0: "psraw $0x6,%%xmm2 \n" \ michael@0: "packuswb %%xmm0,%%xmm0 \n" \ michael@0: "packuswb %%xmm1,%%xmm1 \n" \ michael@0: "packuswb %%xmm2,%%xmm2 \n" michael@0: michael@0: void OMITFP I444ToARGBRow_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV444 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqa %%xmm0," MEMACCESS([dst_argb]) " \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,[dst_argb]) " \n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToRGB24Row_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_rgb24, michael@0: int width) { michael@0: // fpic 32 bit gcc 4.2 on OSX runs out of GPR regs. michael@0: #if defined(__i386__) michael@0: asm volatile ( michael@0: "movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm5 \n" michael@0: "movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n" michael@0: :: [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24_0), michael@0: [kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24)); michael@0: #endif michael@0: michael@0: asm volatile ( michael@0: #if !defined(__i386__) michael@0: "movdqa %[kShuffleMaskARGBToRGB24_0],%%xmm5 \n" michael@0: "movdqa %[kShuffleMaskARGBToRGB24],%%xmm6 \n" michael@0: #endif michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm2,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "pshufb %%xmm5,%%xmm0 \n" michael@0: "pshufb %%xmm6,%%xmm1 \n" michael@0: "palignr $0xc,%%xmm0,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS([dst_rgb24]) "\n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x8,[dst_rgb24]) "\n" michael@0: "lea " MEMLEA(0x18,[dst_rgb24]) ",%[dst_rgb24] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_rgb24]"+r"(dst_rgb24), // %[dst_rgb24] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) michael@0: #if !defined(__i386__) michael@0: , [kShuffleMaskARGBToRGB24_0]"m"(kShuffleMaskARGBToRGB24_0), michael@0: [kShuffleMaskARGBToRGB24]"m"(kShuffleMaskARGBToRGB24) michael@0: #endif michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToRAWRow_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_raw, michael@0: int width) { michael@0: // fpic 32 bit gcc 4.2 on OSX runs out of GPR regs. michael@0: #if defined(__i386__) michael@0: asm volatile ( michael@0: "movdqa %[kShuffleMaskARGBToRAW_0],%%xmm5 \n" michael@0: "movdqa %[kShuffleMaskARGBToRAW],%%xmm6 \n" michael@0: :: [kShuffleMaskARGBToRAW_0]"m"(kShuffleMaskARGBToRAW_0), michael@0: [kShuffleMaskARGBToRAW]"m"(kShuffleMaskARGBToRAW)); michael@0: #endif michael@0: michael@0: asm volatile ( michael@0: #if !defined(__i386__) michael@0: "movdqa %[kShuffleMaskARGBToRAW_0],%%xmm5 \n" michael@0: "movdqa %[kShuffleMaskARGBToRAW],%%xmm6 \n" michael@0: #endif michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm2,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "pshufb %%xmm5,%%xmm0 \n" michael@0: "pshufb %%xmm6,%%xmm1 \n" michael@0: "palignr $0xc,%%xmm0,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS([dst_raw]) " \n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x8,[dst_raw]) "\n" michael@0: "lea " MEMLEA(0x18,[dst_raw]) ",%[dst_raw] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_raw]"+r"(dst_raw), // %[dst_raw] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) michael@0: #if !defined(__i386__) michael@0: , [kShuffleMaskARGBToRAW_0]"m"(kShuffleMaskARGBToRAW_0), michael@0: [kShuffleMaskARGBToRAW]"m"(kShuffleMaskARGBToRAW) michael@0: #endif michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToARGBRow_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqa %%xmm0," MEMACCESS([dst_argb]) "\n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,[dst_argb]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I411ToARGBRow_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV411 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqa %%xmm0," MEMACCESS([dst_argb]) "\n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,[dst_argb]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP NV12ToARGBRow_SSSE3(const uint8* y_buf, michael@0: const uint8* uv_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READNV12 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqa %%xmm0," MEMACCESS([dst_argb]) "\n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,[dst_argb]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [uv_buf]"+r"(uv_buf), // %[uv_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: // Does not use r14. michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP NV21ToARGBRow_SSSE3(const uint8* y_buf, michael@0: const uint8* uv_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READNV12 michael@0: YVUTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqa %%xmm0," MEMACCESS([dst_argb]) "\n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,[dst_argb]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [uv_buf]"+r"(uv_buf), // %[uv_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: // Does not use r14. michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I444ToARGBRow_Unaligned_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV444 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqu %%xmm0," MEMACCESS([dst_argb]) "\n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,[dst_argb]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToARGBRow_Unaligned_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqu %%xmm0," MEMACCESS([dst_argb]) "\n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,[dst_argb]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I411ToARGBRow_Unaligned_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV411 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqu %%xmm0," MEMACCESS([dst_argb]) "\n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,[dst_argb]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP NV12ToARGBRow_Unaligned_SSSE3(const uint8* y_buf, michael@0: const uint8* uv_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READNV12 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqu %%xmm0," MEMACCESS([dst_argb]) "\n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,[dst_argb]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [uv_buf]"+r"(uv_buf), // %[uv_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: // Does not use r14. michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP NV21ToARGBRow_Unaligned_SSSE3(const uint8* y_buf, michael@0: const uint8* uv_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READNV12 michael@0: YVUTORGB michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm0 \n" michael@0: "punpckhwd %%xmm2,%%xmm1 \n" michael@0: "movdqu %%xmm0," MEMACCESS([dst_argb]) "\n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,[dst_argb]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_argb]) ",%[dst_argb] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [uv_buf]"+r"(uv_buf), // %[uv_buf] michael@0: [dst_argb]"+r"(dst_argb), // %[dst_argb] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: // Does not use r14. michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToBGRARow_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_bgra, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "punpcklbw %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm2,%%xmm5 \n" michael@0: "movdqa %%xmm5,%%xmm0 \n" michael@0: "punpcklwd %%xmm1,%%xmm5 \n" michael@0: "punpckhwd %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm5," MEMACCESS([dst_bgra]) "\n" michael@0: "movdqa %%xmm0," MEMACCESS2(0x10,[dst_bgra]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_bgra]) ",%[dst_bgra] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_bgra]"+r"(dst_bgra), // %[dst_bgra] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToABGRRow_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_abgr, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm2 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm1 \n" michael@0: "punpcklwd %%xmm0,%%xmm2 \n" michael@0: "punpckhwd %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2," MEMACCESS([dst_abgr]) "\n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,[dst_abgr]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_abgr]) ",%[dst_abgr] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_abgr]"+r"(dst_abgr), // %[dst_abgr] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToRGBARow_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_rgba, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "punpcklbw %%xmm2,%%xmm1 \n" michael@0: "punpcklbw %%xmm0,%%xmm5 \n" michael@0: "movdqa %%xmm5,%%xmm0 \n" michael@0: "punpcklwd %%xmm1,%%xmm5 \n" michael@0: "punpckhwd %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm5," MEMACCESS([dst_rgba]) "\n" michael@0: "movdqa %%xmm0," MEMACCESS2(0x10,[dst_rgba]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_rgba]) ",%[dst_rgba] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_rgba]"+r"(dst_rgba), // %[dst_rgba] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToBGRARow_Unaligned_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_bgra, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "punpcklbw %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm2,%%xmm5 \n" michael@0: "movdqa %%xmm5,%%xmm0 \n" michael@0: "punpcklwd %%xmm1,%%xmm5 \n" michael@0: "punpckhwd %%xmm1,%%xmm0 \n" michael@0: "movdqu %%xmm5," MEMACCESS([dst_bgra]) "\n" michael@0: "movdqu %%xmm0," MEMACCESS2(0x10,[dst_bgra]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_bgra]) ",%[dst_bgra] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_bgra]"+r"(dst_bgra), // %[dst_bgra] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToABGRRow_Unaligned_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_abgr, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "punpcklbw %%xmm1,%%xmm2 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm1 \n" michael@0: "punpcklwd %%xmm0,%%xmm2 \n" michael@0: "punpckhwd %%xmm0,%%xmm1 \n" michael@0: "movdqu %%xmm2," MEMACCESS([dst_abgr]) "\n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,[dst_abgr]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_abgr]) ",%[dst_abgr] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_abgr]"+r"(dst_abgr), // %[dst_abgr] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void OMITFP I422ToRGBARow_Unaligned_SSSE3(const uint8* y_buf, michael@0: const uint8* u_buf, michael@0: const uint8* v_buf, michael@0: uint8* dst_rgba, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %[u_buf],%[v_buf] \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: READYUV422 michael@0: YUVTORGB michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "punpcklbw %%xmm2,%%xmm1 \n" michael@0: "punpcklbw %%xmm0,%%xmm5 \n" michael@0: "movdqa %%xmm5,%%xmm0 \n" michael@0: "punpcklwd %%xmm1,%%xmm5 \n" michael@0: "punpckhwd %%xmm1,%%xmm0 \n" michael@0: "movdqu %%xmm5," MEMACCESS([dst_rgba]) "\n" michael@0: "movdqu %%xmm0," MEMACCESS2(0x10,[dst_rgba]) "\n" michael@0: "lea " MEMLEA(0x20,[dst_rgba]) ",%[dst_rgba] \n" michael@0: "sub $0x8,%[width] \n" michael@0: "jg 1b \n" michael@0: : [y_buf]"+r"(y_buf), // %[y_buf] michael@0: [u_buf]"+r"(u_buf), // %[u_buf] michael@0: [v_buf]"+r"(v_buf), // %[v_buf] michael@0: [dst_rgba]"+r"(dst_rgba), // %[dst_rgba] michael@0: [width]"+rm"(width) // %[width] michael@0: : [kYuvConstants]"r"(&kYuvConstants.kUVToB) // %[kYuvConstants] michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: #endif // HAS_I422TOARGBROW_SSSE3 michael@0: michael@0: #ifdef HAS_YTOARGBROW_SSE2 michael@0: void YToARGBRow_SSE2(const uint8* y_buf, michael@0: uint8* dst_argb, michael@0: int width) { michael@0: asm volatile ( michael@0: "pxor %%xmm5,%%xmm5 \n" michael@0: "pcmpeqb %%xmm4,%%xmm4 \n" michael@0: "pslld $0x18,%%xmm4 \n" michael@0: "mov $0x00100010,%%eax \n" michael@0: "movd %%eax,%%xmm3 \n" michael@0: "pshufd $0x0,%%xmm3,%%xmm3 \n" michael@0: "mov $0x004a004a,%%eax \n" michael@0: "movd %%eax,%%xmm2 \n" michael@0: "pshufd $0x0,%%xmm2,%%xmm2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: // Step 1: Scale Y contribution to 8 G values. G = (y - 16) * 1.164 michael@0: "movq " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x8,0) ",%0 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "psubusw %%xmm3,%%xmm0 \n" michael@0: "pmullw %%xmm2,%%xmm0 \n" michael@0: "psrlw $6, %%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: michael@0: // Step 2: Weave into ARGB michael@0: "punpcklbw %%xmm0,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm0,%%xmm0 \n" michael@0: "punpckhwd %%xmm1,%%xmm1 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "por %%xmm4,%%xmm1 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: michael@0: "sub $0x8,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(y_buf), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+rm"(width) // %2 michael@0: : michael@0: : "memory", "cc", "eax" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_YTOARGBROW_SSE2 michael@0: michael@0: #ifdef HAS_MIRRORROW_SSSE3 michael@0: // Shuffle table for reversing the bytes. michael@0: static uvec8 kShuffleMirror = { michael@0: 15u, 14u, 13u, 12u, 11u, 10u, 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u michael@0: }; michael@0: michael@0: void MirrorRow_SSSE3(const uint8* src, uint8* dst, int width) { michael@0: intptr_t temp_width = (intptr_t)(width); michael@0: asm volatile ( michael@0: "movdqa %3,%%xmm5 \n" michael@0: "lea " MEMLEA(-0x10,0) ",%0 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: MEMOPREG(movdqa,0x00,0,2,1,xmm0) // movdqa (%0,%2),%%xmm0 michael@0: "pshufb %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(temp_width) // %2 michael@0: : "m"(kShuffleMirror) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_MIRRORROW_SSSE3 michael@0: michael@0: #ifdef HAS_MIRRORROW_SSE2 michael@0: void MirrorRow_SSE2(const uint8* src, uint8* dst, int width) { michael@0: intptr_t temp_width = (intptr_t)(width); michael@0: asm volatile ( michael@0: "lea " MEMLEA(-0x10,0) ",%0 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: MEMOPREG(movdqu,0x00,0,2,1,xmm0) // movdqu (%0,%2),%%xmm0 michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "psllw $0x8,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "por %%xmm1,%%xmm0 \n" michael@0: "pshuflw $0x1b,%%xmm0,%%xmm0 \n" michael@0: "pshufhw $0x1b,%%xmm0,%%xmm0 \n" michael@0: "pshufd $0x4e,%%xmm0,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1)",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(temp_width) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_MIRRORROW_SSE2 michael@0: michael@0: #ifdef HAS_MIRRORROW_UV_SSSE3 michael@0: // Shuffle table for reversing the bytes of UV channels. michael@0: static uvec8 kShuffleMirrorUV = { michael@0: 14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u, 15u, 13u, 11u, 9u, 7u, 5u, 3u, 1u michael@0: }; michael@0: void MirrorUVRow_SSSE3(const uint8* src, uint8* dst_u, uint8* dst_v, michael@0: int width) { michael@0: intptr_t temp_width = (intptr_t)(width); michael@0: asm volatile ( michael@0: "movdqa %4,%%xmm1 \n" michael@0: "lea " MEMLEA4(-0x10,0,3,2) ",%0 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(-0x10,0) ",%0 \n" michael@0: "pshufb %%xmm1,%%xmm0 \n" michael@0: "sub $8,%3 \n" michael@0: "movlpd %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movhpd,xmm0,0x00,1,2,1) // movhpd %%xmm0,(%1,%2) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(temp_width) // %3 michael@0: : "m"(kShuffleMirrorUV) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_MIRRORROW_UV_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBMIRRORROW_SSSE3 michael@0: // Shuffle table for reversing the bytes. michael@0: static uvec8 kARGBShuffleMirror = { michael@0: 12u, 13u, 14u, 15u, 8u, 9u, 10u, 11u, 4u, 5u, 6u, 7u, 0u, 1u, 2u, 3u michael@0: }; michael@0: michael@0: void ARGBMirrorRow_SSSE3(const uint8* src, uint8* dst, int width) { michael@0: intptr_t temp_width = (intptr_t)(width); michael@0: asm volatile ( michael@0: "lea " MEMLEA4(-0x10,0,2,4) ",%0 \n" michael@0: "movdqa %3,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "pshufb %%xmm5,%%xmm0 \n" michael@0: "lea " MEMLEA(-0x10,0) ",%0 \n" michael@0: "sub $0x4,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(temp_width) // %2 michael@0: : "m"(kARGBShuffleMirror) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBMIRRORROW_SSSE3 michael@0: michael@0: #ifdef HAS_SPLITUVROW_SSE2 michael@0: void SplitUVRow_SSE2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "movdqa %%xmm1,%%xmm3 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm2 \n" michael@0: "psrlw $0x8,%%xmm3 \n" michael@0: "packuswb %%xmm3,%%xmm2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: MEMOPMEM(movdqa,xmm2,0x00,1,2,1) // movdqa %%xmm2,(%1,%2) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_uv), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void SplitUVRow_Unaligned_SSE2(const uint8* src_uv, uint8* dst_u, uint8* dst_v, michael@0: int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "movdqa %%xmm1,%%xmm3 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm2 \n" michael@0: "psrlw $0x8,%%xmm3 \n" michael@0: "packuswb %%xmm3,%%xmm2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: MEMOPMEM(movdqu,xmm2,0x00,1,2,1) // movdqu %%xmm2,(%1,%2) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_uv), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_SPLITUVROW_SSE2 michael@0: michael@0: #ifdef HAS_MERGEUVROW_SSE2 michael@0: void MergeUVRow_SSE2(const uint8* src_u, const uint8* src_v, uint8* dst_uv, michael@0: int width) { michael@0: asm volatile ( michael@0: "sub %0,%1 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,0,1,1,xmm1) // movdqa (%0,%1,1),%%xmm1 michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpckhbw %%xmm1,%%xmm2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(2) " \n" michael@0: "movdqa %%xmm2," MEMACCESS2(0x10,2) " \n" michael@0: "lea " MEMLEA(0x20,2) ",%2 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_u), // %0 michael@0: "+r"(src_v), // %1 michael@0: "+r"(dst_uv), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void MergeUVRow_Unaligned_SSE2(const uint8* src_u, const uint8* src_v, michael@0: uint8* dst_uv, int width) { michael@0: asm volatile ( michael@0: "sub %0,%1 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x00,0,1,1,xmm1) // movdqu (%0,%1,1),%%xmm1 michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "punpcklbw %%xmm1,%%xmm0 \n" michael@0: "punpckhbw %%xmm1,%%xmm2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(2) " \n" michael@0: "movdqu %%xmm2," MEMACCESS2(0x10,2) " \n" michael@0: "lea " MEMLEA(0x20,2) ",%2 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_u), // %0 michael@0: "+r"(src_v), // %1 michael@0: "+r"(dst_uv), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_MERGEUVROW_SSE2 michael@0: michael@0: #ifdef HAS_COPYROW_SSE2 michael@0: void CopyRow_SSE2(const uint8* src, uint8* dst, int count) { michael@0: asm volatile ( michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "sub $0x20,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(count) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_COPYROW_SSE2 michael@0: michael@0: #ifdef HAS_COPYROW_X86 michael@0: void CopyRow_X86(const uint8* src, uint8* dst, int width) { michael@0: size_t width_tmp = (size_t)(width); michael@0: asm volatile ( michael@0: "shr $0x2,%2 \n" michael@0: "rep movsl " MEMMOVESTRING(0,1) " \n" michael@0: : "+S"(src), // %0 michael@0: "+D"(dst), // %1 michael@0: "+c"(width_tmp) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: ); michael@0: } michael@0: #endif // HAS_COPYROW_X86 michael@0: michael@0: #ifdef HAS_COPYROW_ERMS michael@0: // Unaligned Multiple of 1. michael@0: void CopyRow_ERMS(const uint8* src, uint8* dst, int width) { michael@0: size_t width_tmp = (size_t)(width); michael@0: asm volatile ( michael@0: "rep movsb " MEMMOVESTRING(0,1) " \n" michael@0: : "+S"(src), // %0 michael@0: "+D"(dst), // %1 michael@0: "+c"(width_tmp) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: ); michael@0: } michael@0: #endif // HAS_COPYROW_ERMS michael@0: michael@0: #ifdef HAS_ARGBCOPYALPHAROW_SSE2 michael@0: // width in pixels michael@0: void ARGBCopyAlphaRow_SSE2(const uint8* src, uint8* dst, int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm0,%%xmm0 \n" michael@0: "pslld $0x18,%%xmm0 \n" michael@0: "pcmpeqb %%xmm1,%%xmm1 \n" michael@0: "psrld $0x8,%%xmm1 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm4 \n" michael@0: "movdqa " MEMACCESS2(0x10,1) ",%%xmm5 \n" michael@0: "pand %%xmm0,%%xmm2 \n" michael@0: "pand %%xmm0,%%xmm3 \n" michael@0: "pand %%xmm1,%%xmm4 \n" michael@0: "pand %%xmm1,%%xmm5 \n" michael@0: "por %%xmm4,%%xmm2 \n" michael@0: "por %%xmm5,%%xmm3 \n" michael@0: "movdqa %%xmm2," MEMACCESS(1) " \n" michael@0: "movdqa %%xmm3," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "sub $0x8,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(width) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBCOPYALPHAROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBCOPYALPHAROW_AVX2 michael@0: // width in pixels michael@0: void ARGBCopyAlphaRow_AVX2(const uint8* src, uint8* dst, int width) { michael@0: asm volatile ( michael@0: "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" michael@0: "vpsrld $0x8,%%ymm0,%%ymm0 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "vmovdqu " MEMACCESS(0) ",%%ymm1 \n" michael@0: "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm2 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "vpblendvb %%ymm0," MEMACCESS(1) ",%%ymm1,%%ymm1 \n" michael@0: "vpblendvb %%ymm0," MEMACCESS2(0x20,1) ",%%ymm2,%%ymm2 \n" michael@0: "vmovdqu %%ymm1," MEMACCESS(1) " \n" michael@0: "vmovdqu %%ymm2," MEMACCESS2(0x20,1) " \n" michael@0: "lea " MEMLEA(0x40,1) ",%1 \n" michael@0: "sub $0x10,%2 \n" michael@0: "jg 1b \n" michael@0: "vzeroupper \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(width) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBCOPYALPHAROW_AVX2 michael@0: michael@0: #ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2 michael@0: // width in pixels michael@0: void ARGBCopyYToAlphaRow_SSE2(const uint8* src, uint8* dst, int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm0,%%xmm0 \n" michael@0: "pslld $0x18,%%xmm0 \n" michael@0: "pcmpeqb %%xmm1,%%xmm1 \n" michael@0: "psrld $0x8,%%xmm1 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movq " MEMACCESS(0) ",%%xmm2 \n" michael@0: "lea " MEMLEA(0x8,0) ",%0 \n" michael@0: "punpcklbw %%xmm2,%%xmm2 \n" michael@0: "punpckhwd %%xmm2,%%xmm3 \n" michael@0: "punpcklwd %%xmm2,%%xmm2 \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm4 \n" michael@0: "movdqa " MEMACCESS2(0x10,1) ",%%xmm5 \n" michael@0: "pand %%xmm0,%%xmm2 \n" michael@0: "pand %%xmm0,%%xmm3 \n" michael@0: "pand %%xmm1,%%xmm4 \n" michael@0: "pand %%xmm1,%%xmm5 \n" michael@0: "por %%xmm4,%%xmm2 \n" michael@0: "por %%xmm5,%%xmm3 \n" michael@0: "movdqa %%xmm2," MEMACCESS(1) " \n" michael@0: "movdqa %%xmm3," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "sub $0x8,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(width) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBCOPYYTOALPHAROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2 michael@0: // width in pixels michael@0: void ARGBCopyYToAlphaRow_AVX2(const uint8* src, uint8* dst, int width) { michael@0: asm volatile ( michael@0: "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" michael@0: "vpsrld $0x8,%%ymm0,%%ymm0 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "vpmovzxbd " MEMACCESS(0) ",%%ymm1 \n" michael@0: "vpmovzxbd " MEMACCESS2(0x8,0) ",%%ymm2 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "vpslld $0x18,%%ymm1,%%ymm1 \n" michael@0: "vpslld $0x18,%%ymm2,%%ymm2 \n" michael@0: "vpblendvb %%ymm0," MEMACCESS(1) ",%%ymm1,%%ymm1 \n" michael@0: "vpblendvb %%ymm0," MEMACCESS2(0x20,1) ",%%ymm2,%%ymm2 \n" michael@0: "vmovdqu %%ymm1," MEMACCESS(1) " \n" michael@0: "vmovdqu %%ymm2," MEMACCESS2(0x20,1) " \n" michael@0: "lea " MEMLEA(0x40,1) ",%1 \n" michael@0: "sub $0x10,%2 \n" michael@0: "jg 1b \n" michael@0: "vzeroupper \n" michael@0: : "+r"(src), // %0 michael@0: "+r"(dst), // %1 michael@0: "+r"(width) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBCOPYYTOALPHAROW_AVX2 michael@0: michael@0: #ifdef HAS_SETROW_X86 michael@0: void SetRow_X86(uint8* dst, uint32 v32, int width) { michael@0: size_t width_tmp = (size_t)(width); michael@0: asm volatile ( michael@0: "shr $0x2,%1 \n" michael@0: "rep stosl " MEMSTORESTRING(eax,0) " \n" michael@0: : "+D"(dst), // %0 michael@0: "+c"(width_tmp) // %1 michael@0: : "a"(v32) // %2 michael@0: : "memory", "cc"); michael@0: } michael@0: michael@0: void ARGBSetRows_X86(uint8* dst, uint32 v32, int width, michael@0: int dst_stride, int height) { michael@0: for (int y = 0; y < height; ++y) { michael@0: size_t width_tmp = (size_t)(width); michael@0: uint32* d = (uint32*)(dst); michael@0: asm volatile ( michael@0: "rep stosl " MEMSTORESTRING(eax,0) " \n" michael@0: : "+D"(d), // %0 michael@0: "+c"(width_tmp) // %1 michael@0: : "a"(v32) // %2 michael@0: : "memory", "cc"); michael@0: dst += dst_stride; michael@0: } michael@0: } michael@0: #endif // HAS_SETROW_X86 michael@0: michael@0: #ifdef HAS_YUY2TOYROW_SSE2 michael@0: void YUY2ToYRow_SSE2(const uint8* src_yuy2, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "sub $0x10,%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_yuy2), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void YUY2ToUVRow_SSE2(const uint8* src_yuy2, int stride_yuy2, michael@0: uint8* dst_u, uint8* dst_v, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movdqa,0x00,0,4,1,xmm2) // movdqa (%0,%4,1),%%xmm2 michael@0: MEMOPREG(movdqa,0x10,0,4,1,xmm3) // movdqa 0x10(%0,%4,1),%%xmm3 michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pavgb %%xmm2,%%xmm0 \n" michael@0: "pavgb %%xmm3,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_yuy2), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : "r"((intptr_t)(stride_yuy2)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void YUY2ToUV422Row_SSE2(const uint8* src_yuy2, michael@0: uint8* dst_u, uint8* dst_v, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "psrlw $0x8,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_yuy2), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void YUY2ToYRow_Unaligned_SSE2(const uint8* src_yuy2, michael@0: uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_yuy2), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void YUY2ToUVRow_Unaligned_SSE2(const uint8* src_yuy2, michael@0: int stride_yuy2, michael@0: uint8* dst_u, uint8* dst_v, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movdqu,0x00,0,4,1,xmm2) // movdqu (%0,%4,1),%%xmm2 michael@0: MEMOPREG(movdqu,0x10,0,4,1,xmm3) // movdqu 0x10(%0,%4,1),%%xmm3 michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pavgb %%xmm2,%%xmm0 \n" michael@0: "pavgb %%xmm3,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_yuy2), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : "r"((intptr_t)(stride_yuy2)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void YUY2ToUV422Row_Unaligned_SSE2(const uint8* src_yuy2, michael@0: uint8* dst_u, uint8* dst_v, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "psrlw $0x8,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_yuy2), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void UYVYToYRow_SSE2(const uint8* src_uyvy, uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "psrlw $0x8,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_uyvy), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void UYVYToUVRow_SSE2(const uint8* src_uyvy, int stride_uyvy, michael@0: uint8* dst_u, uint8* dst_v, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movdqa,0x00,0,4,1,xmm2) // movdqa (%0,%4,1),%%xmm2 michael@0: MEMOPREG(movdqa,0x10,0,4,1,xmm3) // movdqa 0x10(%0,%4,1),%%xmm3 michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pavgb %%xmm2,%%xmm0 \n" michael@0: "pavgb %%xmm3,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_uyvy), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : "r"((intptr_t)(stride_uyvy)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void UYVYToUV422Row_SSE2(const uint8* src_uyvy, michael@0: uint8* dst_u, uint8* dst_v, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_uyvy), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void UYVYToYRow_Unaligned_SSE2(const uint8* src_uyvy, michael@0: uint8* dst_y, int pix) { michael@0: asm volatile ( michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "psrlw $0x8,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_uyvy), // %0 michael@0: "+r"(dst_y), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void UYVYToUVRow_Unaligned_SSE2(const uint8* src_uyvy, int stride_uyvy, michael@0: uint8* dst_u, uint8* dst_v, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movdqu,0x00,0,4,1,xmm2) // movdqu (%0,%4,1),%%xmm2 michael@0: MEMOPREG(movdqu,0x10,0,4,1,xmm3) // movdqu 0x10(%0,%4,1),%%xmm3 michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pavgb %%xmm2,%%xmm0 \n" michael@0: "pavgb %%xmm3,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_uyvy), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : "r"((intptr_t)(stride_uyvy)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void UYVYToUV422Row_Unaligned_SSE2(const uint8* src_uyvy, michael@0: uint8* dst_u, uint8* dst_v, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrlw $0x8,%%xmm5 \n" michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm1 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm1,0x00,1,2,1) // movq %%xmm1,(%1,%2) michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_uyvy), // %0 michael@0: "+r"(dst_u), // %1 michael@0: "+r"(dst_v), // %2 michael@0: "+r"(pix) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_YUY2TOYROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBBLENDROW_SSE2 michael@0: // Blend 8 pixels at a time. michael@0: void ARGBBlendRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, michael@0: uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm7,%%xmm7 \n" michael@0: "psrlw $0xf,%%xmm7 \n" michael@0: "pcmpeqb %%xmm6,%%xmm6 \n" michael@0: "psrlw $0x8,%%xmm6 \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psllw $0x8,%%xmm5 \n" michael@0: "pcmpeqb %%xmm4,%%xmm4 \n" michael@0: "pslld $0x18,%%xmm4 \n" michael@0: "sub $0x1,%3 \n" michael@0: "je 91f \n" michael@0: "jl 99f \n" michael@0: michael@0: // 1 pixel loop until destination pointer is aligned. michael@0: "10: \n" michael@0: "test $0xf,%2 \n" michael@0: "je 19f \n" michael@0: "movd " MEMACCESS(0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x4,0) ",%0 \n" michael@0: "movdqa %%xmm3,%%xmm0 \n" michael@0: "pxor %%xmm4,%%xmm3 \n" michael@0: "movd " MEMACCESS(1) ",%%xmm2 \n" michael@0: "psrlw $0x8,%%xmm3 \n" michael@0: "pshufhw $0xf5,%%xmm3,%%xmm3 \n" michael@0: "pshuflw $0xf5,%%xmm3,%%xmm3 \n" michael@0: "pand %%xmm6,%%xmm2 \n" michael@0: "paddw %%xmm7,%%xmm3 \n" michael@0: "pmullw %%xmm3,%%xmm2 \n" michael@0: "movd " MEMACCESS(1) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x4,1) ",%1 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "pmullw %%xmm3,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm2 \n" michael@0: "paddusb %%xmm2,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x1,%3 \n" michael@0: "movd %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x4,2) ",%2 \n" michael@0: "jge 10b \n" michael@0: michael@0: "19: \n" michael@0: "add $1-4,%3 \n" michael@0: "jl 49f \n" michael@0: michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "41: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm3,%%xmm0 \n" michael@0: "pxor %%xmm4,%%xmm3 \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm2 \n" michael@0: "psrlw $0x8,%%xmm3 \n" michael@0: "pshufhw $0xf5,%%xmm3,%%xmm3 \n" michael@0: "pshuflw $0xf5,%%xmm3,%%xmm3 \n" michael@0: "pand %%xmm6,%%xmm2 \n" michael@0: "paddw %%xmm7,%%xmm3 \n" michael@0: "pmullw %%xmm3,%%xmm2 \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "pmullw %%xmm3,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm2 \n" michael@0: "paddusb %%xmm2,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqa %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "jge 41b \n" michael@0: michael@0: "49: \n" michael@0: "add $0x3,%3 \n" michael@0: "jl 99f \n" michael@0: michael@0: // 1 pixel loop. michael@0: "91: \n" michael@0: "movd " MEMACCESS(0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x4,0) ",%0 \n" michael@0: "movdqa %%xmm3,%%xmm0 \n" michael@0: "pxor %%xmm4,%%xmm3 \n" michael@0: "movd " MEMACCESS(1) ",%%xmm2 \n" michael@0: "psrlw $0x8,%%xmm3 \n" michael@0: "pshufhw $0xf5,%%xmm3,%%xmm3 \n" michael@0: "pshuflw $0xf5,%%xmm3,%%xmm3 \n" michael@0: "pand %%xmm6,%%xmm2 \n" michael@0: "paddw %%xmm7,%%xmm3 \n" michael@0: "pmullw %%xmm3,%%xmm2 \n" michael@0: "movd " MEMACCESS(1) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x4,1) ",%1 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "pmullw %%xmm3,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm2 \n" michael@0: "paddusb %%xmm2,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x1,%3 \n" michael@0: "movd %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x4,2) ",%2 \n" michael@0: "jge 91b \n" michael@0: "99: \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(src_argb1), // %1 michael@0: "+r"(dst_argb), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBBLENDROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBBLENDROW_SSSE3 michael@0: // Shuffle table for isolating alpha. michael@0: static uvec8 kShuffleAlpha = { michael@0: 3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, michael@0: 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80 michael@0: }; michael@0: michael@0: // Blend 8 pixels at a time michael@0: // Shuffle table for reversing the bytes. michael@0: michael@0: // Same as SSE2, but replaces michael@0: // psrlw xmm3, 8 // alpha michael@0: // pshufhw xmm3, xmm3,0F5h // 8 alpha words michael@0: // pshuflw xmm3, xmm3,0F5h michael@0: // with.. michael@0: // pshufb xmm3, kShuffleAlpha // alpha michael@0: michael@0: void ARGBBlendRow_SSSE3(const uint8* src_argb0, const uint8* src_argb1, michael@0: uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm7,%%xmm7 \n" michael@0: "psrlw $0xf,%%xmm7 \n" michael@0: "pcmpeqb %%xmm6,%%xmm6 \n" michael@0: "psrlw $0x8,%%xmm6 \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psllw $0x8,%%xmm5 \n" michael@0: "pcmpeqb %%xmm4,%%xmm4 \n" michael@0: "pslld $0x18,%%xmm4 \n" michael@0: "sub $0x1,%3 \n" michael@0: "je 91f \n" michael@0: "jl 99f \n" michael@0: michael@0: // 1 pixel loop until destination pointer is aligned. michael@0: "10: \n" michael@0: "test $0xf,%2 \n" michael@0: "je 19f \n" michael@0: "movd " MEMACCESS(0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x4,0) ",%0 \n" michael@0: "movdqa %%xmm3,%%xmm0 \n" michael@0: "pxor %%xmm4,%%xmm3 \n" michael@0: "movd " MEMACCESS(1) ",%%xmm2 \n" michael@0: "pshufb %4,%%xmm3 \n" michael@0: "pand %%xmm6,%%xmm2 \n" michael@0: "paddw %%xmm7,%%xmm3 \n" michael@0: "pmullw %%xmm3,%%xmm2 \n" michael@0: "movd " MEMACCESS(1) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x4,1) ",%1 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "pmullw %%xmm3,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm2 \n" michael@0: "paddusb %%xmm2,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x1,%3 \n" michael@0: "movd %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x4,2) ",%2 \n" michael@0: "jge 10b \n" michael@0: michael@0: "19: \n" michael@0: "add $1-4,%3 \n" michael@0: "jl 49f \n" michael@0: "test $0xf,%0 \n" michael@0: "jne 41f \n" michael@0: "test $0xf,%1 \n" michael@0: "jne 41f \n" michael@0: michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "40: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm3,%%xmm0 \n" michael@0: "pxor %%xmm4,%%xmm3 \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm2 \n" michael@0: "pshufb %4,%%xmm3 \n" michael@0: "pand %%xmm6,%%xmm2 \n" michael@0: "paddw %%xmm7,%%xmm3 \n" michael@0: "pmullw %%xmm3,%%xmm2 \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "pmullw %%xmm3,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm2 \n" michael@0: "paddusb %%xmm2,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqa %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "jge 40b \n" michael@0: "jmp 49f \n" michael@0: michael@0: // 4 pixel unaligned loop. michael@0: LABELALIGN michael@0: "41: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm3,%%xmm0 \n" michael@0: "pxor %%xmm4,%%xmm3 \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm2 \n" michael@0: "pshufb %4,%%xmm3 \n" michael@0: "pand %%xmm6,%%xmm2 \n" michael@0: "paddw %%xmm7,%%xmm3 \n" michael@0: "pmullw %%xmm3,%%xmm2 \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "pmullw %%xmm3,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm2 \n" michael@0: "paddusb %%xmm2,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqa %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "jge 41b \n" michael@0: michael@0: "49: \n" michael@0: "add $0x3,%3 \n" michael@0: "jl 99f \n" michael@0: michael@0: // 1 pixel loop. michael@0: "91: \n" michael@0: "movd " MEMACCESS(0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x4,0) ",%0 \n" michael@0: "movdqa %%xmm3,%%xmm0 \n" michael@0: "pxor %%xmm4,%%xmm3 \n" michael@0: "movd " MEMACCESS(1) ",%%xmm2 \n" michael@0: "pshufb %4,%%xmm3 \n" michael@0: "pand %%xmm6,%%xmm2 \n" michael@0: "paddw %%xmm7,%%xmm3 \n" michael@0: "pmullw %%xmm3,%%xmm2 \n" michael@0: "movd " MEMACCESS(1) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x4,1) ",%1 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "por %%xmm4,%%xmm0 \n" michael@0: "pmullw %%xmm3,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm2 \n" michael@0: "paddusb %%xmm2,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x1,%3 \n" michael@0: "movd %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x4,2) ",%2 \n" michael@0: "jge 91b \n" michael@0: "99: \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(src_argb1), // %1 michael@0: "+r"(dst_argb), // %2 michael@0: "+r"(width) // %3 michael@0: : "m"(kShuffleAlpha) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBBLENDROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBATTENUATEROW_SSE2 michael@0: // Attenuate 4 pixels at a time. michael@0: // aligned to 16 bytes michael@0: void ARGBAttenuateRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm4,%%xmm4 \n" michael@0: "pslld $0x18,%%xmm4 \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrld $0x8,%%xmm5 \n" michael@0: michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "punpcklbw %%xmm0,%%xmm0 \n" michael@0: "pshufhw $0xff,%%xmm0,%%xmm2 \n" michael@0: "pshuflw $0xff,%%xmm2,%%xmm2 \n" michael@0: "pmulhuw %%xmm2,%%xmm0 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm1 \n" michael@0: "punpckhbw %%xmm1,%%xmm1 \n" michael@0: "pshufhw $0xff,%%xmm1,%%xmm2 \n" michael@0: "pshuflw $0xff,%%xmm2,%%xmm2 \n" michael@0: "pmulhuw %%xmm2,%%xmm1 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm2 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "psrlw $0x8,%%xmm0 \n" michael@0: "pand %%xmm4,%%xmm2 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "por %%xmm2,%%xmm0 \n" michael@0: "sub $0x4,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(width) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBATTENUATEROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBATTENUATEROW_SSSE3 michael@0: // Shuffle table duplicating alpha michael@0: static uvec8 kShuffleAlpha0 = { michael@0: 3u, 3u, 3u, 3u, 3u, 3u, 128u, 128u, 7u, 7u, 7u, 7u, 7u, 7u, 128u, 128u, michael@0: }; michael@0: static uvec8 kShuffleAlpha1 = { michael@0: 11u, 11u, 11u, 11u, 11u, 11u, 128u, 128u, michael@0: 15u, 15u, 15u, 15u, 15u, 15u, 128u, 128u, michael@0: }; michael@0: // Attenuate 4 pixels at a time. michael@0: // aligned to 16 bytes michael@0: void ARGBAttenuateRow_SSSE3(const uint8* src_argb, uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm3,%%xmm3 \n" michael@0: "pslld $0x18,%%xmm3 \n" michael@0: "movdqa %3,%%xmm4 \n" michael@0: "movdqa %4,%%xmm5 \n" michael@0: michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "pshufb %%xmm4,%%xmm0 \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm1 \n" michael@0: "punpcklbw %%xmm1,%%xmm1 \n" michael@0: "pmulhuw %%xmm1,%%xmm0 \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm1 \n" michael@0: "pshufb %%xmm5,%%xmm1 \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm2 \n" michael@0: "punpckhbw %%xmm2,%%xmm2 \n" michael@0: "pmulhuw %%xmm2,%%xmm1 \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm2 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "pand %%xmm3,%%xmm2 \n" michael@0: "psrlw $0x8,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "por %%xmm2,%%xmm0 \n" michael@0: "sub $0x4,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(width) // %2 michael@0: : "m"(kShuffleAlpha0), // %3 michael@0: "m"(kShuffleAlpha1) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBATTENUATEROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBUNATTENUATEROW_SSE2 michael@0: // Unattenuate 4 pixels at a time. michael@0: // aligned to 16 bytes michael@0: void ARGBUnattenuateRow_SSE2(const uint8* src_argb, uint8* dst_argb, michael@0: int width) { michael@0: uintptr_t alpha = 0; michael@0: asm volatile ( michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movzb " MEMACCESS2(0x03,0) ",%3 \n" michael@0: "punpcklbw %%xmm0,%%xmm0 \n" michael@0: MEMOPREG(movd,0x00,4,3,4,xmm2) // movd 0x0(%4,%3,4),%%xmm2 michael@0: "movzb " MEMACCESS2(0x07,0) ",%3 \n" michael@0: MEMOPREG(movd,0x00,4,3,4,xmm3) // movd 0x0(%4,%3,4),%%xmm3 michael@0: "pshuflw $0x40,%%xmm2,%%xmm2 \n" michael@0: "pshuflw $0x40,%%xmm3,%%xmm3 \n" michael@0: "movlhps %%xmm3,%%xmm2 \n" michael@0: "pmulhuw %%xmm2,%%xmm0 \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm1 \n" michael@0: "movzb " MEMACCESS2(0x0b,0) ",%3 \n" michael@0: "punpckhbw %%xmm1,%%xmm1 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movd,0x00,4,3,4,xmm2) // movd 0x0(%4,%3,4),%%xmm2 michael@0: "movzb " MEMACCESS2(0x0f,0) ",%3 \n" michael@0: MEMOPREG(movd,0x00,4,3,4,xmm3) // movd 0x0(%4,%3,4),%%xmm3 michael@0: "pshuflw $0x40,%%xmm2,%%xmm2 \n" michael@0: "pshuflw $0x40,%%xmm3,%%xmm3 \n" michael@0: "movlhps %%xmm3,%%xmm2 \n" michael@0: "pmulhuw %%xmm2,%%xmm1 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(width), // %2 michael@0: "+r"(alpha) // %3 michael@0: : "r"(fixed_invtbl8) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBUNATTENUATEROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBGRAYROW_SSSE3 michael@0: // Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels michael@0: void ARGBGrayRow_SSSE3(const uint8* src_argb, uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: "movdqa %3,%%xmm4 \n" michael@0: "movdqa %4,%%xmm5 \n" michael@0: michael@0: // 8 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm0 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "phaddw %%xmm1,%%xmm0 \n" michael@0: "paddw %%xmm5,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm3 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "psrld $0x18,%%xmm2 \n" michael@0: "psrld $0x18,%%xmm3 \n" michael@0: "packuswb %%xmm3,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm3 \n" michael@0: "punpcklbw %%xmm0,%%xmm0 \n" michael@0: "punpcklbw %%xmm2,%%xmm3 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm3,%%xmm0 \n" michael@0: "punpckhwd %%xmm3,%%xmm1 \n" michael@0: "sub $0x8,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(width) // %2 michael@0: : "m"(kARGBToYJ), // %3 michael@0: "m"(kAddYJ64) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBGRAYROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBSEPIAROW_SSSE3 michael@0: // b = (r * 35 + g * 68 + b * 17) >> 7 michael@0: // g = (r * 45 + g * 88 + b * 22) >> 7 michael@0: // r = (r * 50 + g * 98 + b * 24) >> 7 michael@0: // Constant for ARGB color to sepia tone michael@0: static vec8 kARGBToSepiaB = { michael@0: 17, 68, 35, 0, 17, 68, 35, 0, 17, 68, 35, 0, 17, 68, 35, 0 michael@0: }; michael@0: michael@0: static vec8 kARGBToSepiaG = { michael@0: 22, 88, 45, 0, 22, 88, 45, 0, 22, 88, 45, 0, 22, 88, 45, 0 michael@0: }; michael@0: michael@0: static vec8 kARGBToSepiaR = { michael@0: 24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0 michael@0: }; michael@0: michael@0: // Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels. michael@0: void ARGBSepiaRow_SSSE3(uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: "movdqa %2,%%xmm2 \n" michael@0: "movdqa %3,%%xmm3 \n" michael@0: "movdqa %4,%%xmm4 \n" michael@0: michael@0: // 8 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm6 \n" michael@0: "pmaddubsw %%xmm2,%%xmm0 \n" michael@0: "pmaddubsw %%xmm2,%%xmm6 \n" michael@0: "phaddw %%xmm6,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm5 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm5 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "phaddw %%xmm1,%%xmm5 \n" michael@0: "psrlw $0x7,%%xmm5 \n" michael@0: "packuswb %%xmm5,%%xmm5 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm5 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm5 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "phaddw %%xmm1,%%xmm5 \n" michael@0: "psrlw $0x7,%%xmm5 \n" michael@0: "packuswb %%xmm5,%%xmm5 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm6 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "psrld $0x18,%%xmm6 \n" michael@0: "psrld $0x18,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm6 \n" michael@0: "packuswb %%xmm6,%%xmm6 \n" michael@0: "punpcklbw %%xmm6,%%xmm5 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklwd %%xmm5,%%xmm0 \n" michael@0: "punpckhwd %%xmm5,%%xmm1 \n" michael@0: "sub $0x8,%1 \n" michael@0: "movdqa %%xmm0," MEMACCESS(0) " \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,0) " \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "jg 1b \n" michael@0: : "+r"(dst_argb), // %0 michael@0: "+r"(width) // %1 michael@0: : "m"(kARGBToSepiaB), // %2 michael@0: "m"(kARGBToSepiaG), // %3 michael@0: "m"(kARGBToSepiaR) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBSEPIAROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBCOLORMATRIXROW_SSSE3 michael@0: // Tranform 8 ARGB pixels (32 bytes) with color matrix. michael@0: // Same as Sepia except matrix is provided. michael@0: void ARGBColorMatrixRow_SSSE3(const uint8* src_argb, uint8* dst_argb, michael@0: const int8* matrix_argb, int width) { michael@0: asm volatile ( michael@0: "movdqu " MEMACCESS(3) ",%%xmm5 \n" michael@0: "pshufd $0x00,%%xmm5,%%xmm2 \n" michael@0: "pshufd $0x55,%%xmm5,%%xmm3 \n" michael@0: "pshufd $0xaa,%%xmm5,%%xmm4 \n" michael@0: "pshufd $0xff,%%xmm5,%%xmm5 \n" michael@0: michael@0: // 8 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm7 \n" michael@0: "pmaddubsw %%xmm2,%%xmm0 \n" michael@0: "pmaddubsw %%xmm2,%%xmm7 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm6 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "pmaddubsw %%xmm3,%%xmm6 \n" michael@0: "pmaddubsw %%xmm3,%%xmm1 \n" michael@0: "phaddsw %%xmm7,%%xmm0 \n" michael@0: "phaddsw %%xmm1,%%xmm6 \n" michael@0: "psraw $0x6,%%xmm0 \n" michael@0: "psraw $0x6,%%xmm6 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "packuswb %%xmm6,%%xmm6 \n" michael@0: "punpcklbw %%xmm6,%%xmm0 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm7 \n" michael@0: "pmaddubsw %%xmm4,%%xmm1 \n" michael@0: "pmaddubsw %%xmm4,%%xmm7 \n" michael@0: "phaddsw %%xmm7,%%xmm1 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm6 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm7 \n" michael@0: "pmaddubsw %%xmm5,%%xmm6 \n" michael@0: "pmaddubsw %%xmm5,%%xmm7 \n" michael@0: "phaddsw %%xmm7,%%xmm6 \n" michael@0: "psraw $0x6,%%xmm1 \n" michael@0: "psraw $0x6,%%xmm6 \n" michael@0: "packuswb %%xmm1,%%xmm1 \n" michael@0: "packuswb %%xmm6,%%xmm6 \n" michael@0: "punpcklbw %%xmm6,%%xmm1 \n" michael@0: "movdqa %%xmm0,%%xmm6 \n" michael@0: "punpcklwd %%xmm1,%%xmm0 \n" michael@0: "punpckhwd %%xmm1,%%xmm6 \n" michael@0: "sub $0x8,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqa %%xmm6," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(width) // %2 michael@0: : "r"(matrix_argb) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBCOLORMATRIXROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBQUANTIZEROW_SSE2 michael@0: // Quantize 4 ARGB pixels (16 bytes). michael@0: // aligned to 16 bytes michael@0: void ARGBQuantizeRow_SSE2(uint8* dst_argb, int scale, int interval_size, michael@0: int interval_offset, int width) { michael@0: asm volatile ( michael@0: "movd %2,%%xmm2 \n" michael@0: "movd %3,%%xmm3 \n" michael@0: "movd %4,%%xmm4 \n" michael@0: "pshuflw $0x40,%%xmm2,%%xmm2 \n" michael@0: "pshufd $0x44,%%xmm2,%%xmm2 \n" michael@0: "pshuflw $0x40,%%xmm3,%%xmm3 \n" michael@0: "pshufd $0x44,%%xmm3,%%xmm3 \n" michael@0: "pshuflw $0x40,%%xmm4,%%xmm4 \n" michael@0: "pshufd $0x44,%%xmm4,%%xmm4 \n" michael@0: "pxor %%xmm5,%%xmm5 \n" michael@0: "pcmpeqb %%xmm6,%%xmm6 \n" michael@0: "pslld $0x18,%%xmm6 \n" michael@0: michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "pmulhuw %%xmm2,%%xmm0 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm1 \n" michael@0: "punpckhbw %%xmm5,%%xmm1 \n" michael@0: "pmulhuw %%xmm2,%%xmm1 \n" michael@0: "pmullw %%xmm3,%%xmm0 \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm7 \n" michael@0: "pmullw %%xmm3,%%xmm1 \n" michael@0: "pand %%xmm6,%%xmm7 \n" michael@0: "paddw %%xmm4,%%xmm0 \n" michael@0: "paddw %%xmm4,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "por %%xmm7,%%xmm0 \n" michael@0: "sub $0x4,%1 \n" michael@0: "movdqa %%xmm0," MEMACCESS(0) " \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "jg 1b \n" michael@0: : "+r"(dst_argb), // %0 michael@0: "+r"(width) // %1 michael@0: : "r"(scale), // %2 michael@0: "r"(interval_size), // %3 michael@0: "r"(interval_offset) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBQUANTIZEROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBSHADEROW_SSE2 michael@0: // Shade 4 pixels at a time by specified value. michael@0: // Aligned to 16 bytes. michael@0: void ARGBShadeRow_SSE2(const uint8* src_argb, uint8* dst_argb, int width, michael@0: uint32 value) { michael@0: asm volatile ( michael@0: "movd %3,%%xmm2 \n" michael@0: "punpcklbw %%xmm2,%%xmm2 \n" michael@0: "punpcklqdq %%xmm2,%%xmm2 \n" michael@0: michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm0,%%xmm0 \n" michael@0: "punpckhbw %%xmm1,%%xmm1 \n" michael@0: "pmulhuw %%xmm2,%%xmm0 \n" michael@0: "pmulhuw %%xmm2,%%xmm1 \n" michael@0: "psrlw $0x8,%%xmm0 \n" michael@0: "psrlw $0x8,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(width) // %2 michael@0: : "r"(value) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBSHADEROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBMULTIPLYROW_SSE2 michael@0: // Multiply 2 rows of ARGB pixels together, 4 pixels at a time. michael@0: void ARGBMultiplyRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, michael@0: uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: "pxor %%xmm5,%%xmm5 \n" michael@0: michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm2 \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "movdqu %%xmm0,%%xmm1 \n" michael@0: "movdqu %%xmm2,%%xmm3 \n" michael@0: "punpcklbw %%xmm0,%%xmm0 \n" michael@0: "punpckhbw %%xmm1,%%xmm1 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "punpckhbw %%xmm5,%%xmm3 \n" michael@0: "pmulhuw %%xmm2,%%xmm0 \n" michael@0: "pmulhuw %%xmm3,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqu %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(src_argb1), // %1 michael@0: "+r"(dst_argb), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBMULTIPLYROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBADDROW_SSE2 michael@0: // Add 2 rows of ARGB pixels together, 4 pixels at a time. michael@0: void ARGBAddRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, michael@0: uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqu %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(src_argb1), // %1 michael@0: "+r"(dst_argb), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBADDROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBSUBTRACTROW_SSE2 michael@0: // Subtract 2 rows of ARGB pixels, 4 pixels at a time. michael@0: void ARGBSubtractRow_SSE2(const uint8* src_argb0, const uint8* src_argb1, michael@0: uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "psubusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqu %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb0), // %0 michael@0: "+r"(src_argb1), // %1 michael@0: "+r"(dst_argb), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBSUBTRACTROW_SSE2 michael@0: michael@0: #ifdef HAS_SOBELXROW_SSE2 michael@0: // SobelX as a matrix is michael@0: // -1 0 1 michael@0: // -2 0 2 michael@0: // -1 0 1 michael@0: void SobelXRow_SSE2(const uint8* src_y0, const uint8* src_y1, michael@0: const uint8* src_y2, uint8* dst_sobelx, int width) { michael@0: asm volatile ( michael@0: "sub %0,%1 \n" michael@0: "sub %0,%2 \n" michael@0: "sub %0,%3 \n" michael@0: "pxor %%xmm5,%%xmm5 \n" michael@0: michael@0: // 8 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movq " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movq " MEMACCESS2(0x2,0) ",%%xmm1 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm1 \n" michael@0: "psubw %%xmm1,%%xmm0 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movq,0x00,0,1,1,xmm1) // movq (%0,%1,1),%%xmm1 michael@0: MEMOPREG(movq,0x02,0,1,1,xmm2) // movq 0x2(%0,%1,1),%%xmm2 michael@0: "punpcklbw %%xmm5,%%xmm1 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "psubw %%xmm2,%%xmm1 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movq,0x00,0,2,1,xmm2) // movq (%0,%2,1),%%xmm2 michael@0: MEMOPREG(movq,0x02,0,2,1,xmm3) // movq 0x2(%0,%2,1),%%xmm3 michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "punpcklbw %%xmm5,%%xmm3 \n" michael@0: "psubw %%xmm3,%%xmm2 \n" michael@0: "paddw %%xmm2,%%xmm0 \n" michael@0: "paddw %%xmm1,%%xmm0 \n" michael@0: "paddw %%xmm1,%%xmm0 \n" michael@0: "pxor %%xmm1,%%xmm1 \n" michael@0: "psubw %%xmm0,%%xmm1 \n" michael@0: "pmaxsw %%xmm1,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "sub $0x8,%4 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm0,0x00,0,3,1) // movq %%xmm0,(%0,%3,1) michael@0: "lea " MEMLEA(0x8,0) ",%0 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_y0), // %0 michael@0: "+r"(src_y1), // %1 michael@0: "+r"(src_y2), // %2 michael@0: "+r"(dst_sobelx), // %3 michael@0: "+r"(width) // %4 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_SOBELXROW_SSE2 michael@0: michael@0: #ifdef HAS_SOBELYROW_SSE2 michael@0: // SobelY as a matrix is michael@0: // -1 -2 -1 michael@0: // 0 0 0 michael@0: // 1 2 1 michael@0: void SobelYRow_SSE2(const uint8* src_y0, const uint8* src_y1, michael@0: uint8* dst_sobely, int width) { michael@0: asm volatile ( michael@0: "sub %0,%1 \n" michael@0: "sub %0,%2 \n" michael@0: "pxor %%xmm5,%%xmm5 \n" michael@0: michael@0: // 8 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movq " MEMACCESS(0) ",%%xmm0 \n" michael@0: MEMOPREG(movq,0x00,0,1,1,xmm1) // movq (%0,%1,1),%%xmm1 michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "punpcklbw %%xmm5,%%xmm1 \n" michael@0: "psubw %%xmm1,%%xmm0 \n" michael@0: BUNDLEALIGN michael@0: "movq " MEMACCESS2(0x1,0) ",%%xmm1 \n" michael@0: MEMOPREG(movq,0x01,0,1,1,xmm2) // movq 0x1(%0,%1,1),%%xmm2 michael@0: "punpcklbw %%xmm5,%%xmm1 \n" michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "psubw %%xmm2,%%xmm1 \n" michael@0: BUNDLEALIGN michael@0: "movq " MEMACCESS2(0x2,0) ",%%xmm2 \n" michael@0: MEMOPREG(movq,0x02,0,1,1,xmm3) // movq 0x2(%0,%1,1),%%xmm3 michael@0: "punpcklbw %%xmm5,%%xmm2 \n" michael@0: "punpcklbw %%xmm5,%%xmm3 \n" michael@0: "psubw %%xmm3,%%xmm2 \n" michael@0: "paddw %%xmm2,%%xmm0 \n" michael@0: "paddw %%xmm1,%%xmm0 \n" michael@0: "paddw %%xmm1,%%xmm0 \n" michael@0: "pxor %%xmm1,%%xmm1 \n" michael@0: "psubw %%xmm0,%%xmm1 \n" michael@0: "pmaxsw %%xmm1,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "sub $0x8,%3 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movq,xmm0,0x00,0,2,1) // movq %%xmm0,(%0,%2,1) michael@0: "lea " MEMLEA(0x8,0) ",%0 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_y0), // %0 michael@0: "+r"(src_y1), // %1 michael@0: "+r"(dst_sobely), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_SOBELYROW_SSE2 michael@0: michael@0: #ifdef HAS_SOBELROW_SSE2 michael@0: // Adds Sobel X and Sobel Y and stores Sobel into ARGB. michael@0: // A = 255 michael@0: // R = Sobel michael@0: // G = Sobel michael@0: // B = Sobel michael@0: void SobelRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, michael@0: uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: "sub %0,%1 \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pslld $0x18,%%xmm5 \n" michael@0: michael@0: // 8 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,0,1,1,xmm1) // movdqa (%0,%1,1),%%xmm1 michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "punpcklbw %%xmm0,%%xmm2 \n" michael@0: "punpckhbw %%xmm0,%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm1 \n" michael@0: "punpcklwd %%xmm2,%%xmm1 \n" michael@0: "punpckhwd %%xmm2,%%xmm2 \n" michael@0: "por %%xmm5,%%xmm1 \n" michael@0: "por %%xmm5,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm3 \n" michael@0: "punpcklwd %%xmm0,%%xmm3 \n" michael@0: "punpckhwd %%xmm0,%%xmm0 \n" michael@0: "por %%xmm5,%%xmm3 \n" michael@0: "por %%xmm5,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movdqa %%xmm1," MEMACCESS(2) " \n" michael@0: "movdqa %%xmm2," MEMACCESS2(0x10,2) " \n" michael@0: "movdqa %%xmm3," MEMACCESS2(0x20,2) " \n" michael@0: "movdqa %%xmm0," MEMACCESS2(0x30,2) " \n" michael@0: "lea " MEMLEA(0x40,2) ",%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_sobelx), // %0 michael@0: "+r"(src_sobely), // %1 michael@0: "+r"(dst_argb), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_SOBELROW_SSE2 michael@0: michael@0: #ifdef HAS_SOBELTOPLANEROW_SSE2 michael@0: // Adds Sobel X and Sobel Y and stores Sobel into a plane. michael@0: void SobelToPlaneRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, michael@0: uint8* dst_y, int width) { michael@0: asm volatile ( michael@0: "sub %0,%1 \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "pslld $0x18,%%xmm5 \n" michael@0: michael@0: // 8 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,0,1,1,xmm1) // movdqa (%0,%1,1),%%xmm1 michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "paddusb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movdqa %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_sobelx), // %0 michael@0: "+r"(src_sobely), // %1 michael@0: "+r"(dst_y), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_SOBELTOPLANEROW_SSE2 michael@0: michael@0: #ifdef HAS_SOBELXYROW_SSE2 michael@0: // Mixes Sobel X, Sobel Y and Sobel into ARGB. michael@0: // A = 255 michael@0: // R = Sobel X michael@0: // G = Sobel michael@0: // B = Sobel Y michael@0: void SobelXYRow_SSE2(const uint8* src_sobelx, const uint8* src_sobely, michael@0: uint8* dst_argb, int width) { michael@0: asm volatile ( michael@0: "sub %0,%1 \n" michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: michael@0: // 8 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,0,1,1,xmm1) // movdqa (%0,%1,1),%%xmm1 michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm2 \n" michael@0: "paddusb %%xmm1,%%xmm2 \n" michael@0: "movdqa %%xmm0,%%xmm3 \n" michael@0: "punpcklbw %%xmm5,%%xmm3 \n" michael@0: "punpckhbw %%xmm5,%%xmm0 \n" michael@0: "movdqa %%xmm1,%%xmm4 \n" michael@0: "punpcklbw %%xmm2,%%xmm4 \n" michael@0: "punpckhbw %%xmm2,%%xmm1 \n" michael@0: "movdqa %%xmm4,%%xmm6 \n" michael@0: "punpcklwd %%xmm3,%%xmm6 \n" michael@0: "punpckhwd %%xmm3,%%xmm4 \n" michael@0: "movdqa %%xmm1,%%xmm7 \n" michael@0: "punpcklwd %%xmm0,%%xmm7 \n" michael@0: "punpckhwd %%xmm0,%%xmm1 \n" michael@0: "sub $0x10,%3 \n" michael@0: "movdqa %%xmm6," MEMACCESS(2) " \n" michael@0: "movdqa %%xmm4," MEMACCESS2(0x10,2) " \n" michael@0: "movdqa %%xmm7," MEMACCESS2(0x20,2) " \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x30,2) " \n" michael@0: "lea " MEMLEA(0x40,2) ",%2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_sobelx), // %0 michael@0: "+r"(src_sobely), // %1 michael@0: "+r"(dst_argb), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_SOBELXYROW_SSE2 michael@0: michael@0: #ifdef HAS_COMPUTECUMULATIVESUMROW_SSE2 michael@0: // Creates a table of cumulative sums where each value is a sum of all values michael@0: // above and to the left of the value, inclusive of the value. michael@0: void ComputeCumulativeSumRow_SSE2(const uint8* row, int32* cumsum, michael@0: const int32* previous_cumsum, int width) { michael@0: asm volatile ( michael@0: "pxor %%xmm0,%%xmm0 \n" michael@0: "pxor %%xmm1,%%xmm1 \n" michael@0: "sub $0x4,%3 \n" michael@0: "jl 49f \n" michael@0: "test $0xf,%1 \n" michael@0: "jne 49f \n" michael@0: michael@0: // 4 pixel loop \n" michael@0: LABELALIGN michael@0: "40: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm2 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm2,%%xmm4 \n" michael@0: "punpcklbw %%xmm1,%%xmm2 \n" michael@0: "movdqa %%xmm2,%%xmm3 \n" michael@0: "punpcklwd %%xmm1,%%xmm2 \n" michael@0: "punpckhwd %%xmm1,%%xmm3 \n" michael@0: "punpckhbw %%xmm1,%%xmm4 \n" michael@0: "movdqa %%xmm4,%%xmm5 \n" michael@0: "punpcklwd %%xmm1,%%xmm4 \n" michael@0: "punpckhwd %%xmm1,%%xmm5 \n" michael@0: "paddd %%xmm2,%%xmm0 \n" michael@0: "movdqa " MEMACCESS(2) ",%%xmm2 \n" michael@0: "paddd %%xmm0,%%xmm2 \n" michael@0: "paddd %%xmm3,%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,2) ",%%xmm3 \n" michael@0: "paddd %%xmm0,%%xmm3 \n" michael@0: "paddd %%xmm4,%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x20,2) ",%%xmm4 \n" michael@0: "paddd %%xmm0,%%xmm4 \n" michael@0: "paddd %%xmm5,%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x30,2) ",%%xmm5 \n" michael@0: "lea " MEMLEA(0x40,2) ",%2 \n" michael@0: "paddd %%xmm0,%%xmm5 \n" michael@0: "movdqa %%xmm2," MEMACCESS(1) " \n" michael@0: "movdqa %%xmm3," MEMACCESS2(0x10,1) " \n" michael@0: "movdqa %%xmm4," MEMACCESS2(0x20,1) " \n" michael@0: "movdqa %%xmm5," MEMACCESS2(0x30,1) " \n" michael@0: "lea " MEMLEA(0x40,1) ",%1 \n" michael@0: "sub $0x4,%3 \n" michael@0: "jge 40b \n" michael@0: michael@0: "49: \n" michael@0: "add $0x3,%3 \n" michael@0: "jl 19f \n" michael@0: michael@0: // 1 pixel loop \n" michael@0: LABELALIGN michael@0: "10: \n" michael@0: "movd " MEMACCESS(0) ",%%xmm2 \n" michael@0: "lea " MEMLEA(0x4,0) ",%0 \n" michael@0: "punpcklbw %%xmm1,%%xmm2 \n" michael@0: "punpcklwd %%xmm1,%%xmm2 \n" michael@0: "paddd %%xmm2,%%xmm0 \n" michael@0: "movdqu " MEMACCESS(2) ",%%xmm2 \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "paddd %%xmm0,%%xmm2 \n" michael@0: "movdqu %%xmm2," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "sub $0x1,%3 \n" michael@0: "jge 10b \n" michael@0: michael@0: "19: \n" michael@0: : "+r"(row), // %0 michael@0: "+r"(cumsum), // %1 michael@0: "+r"(previous_cumsum), // %2 michael@0: "+r"(width) // %3 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_COMPUTECUMULATIVESUMROW_SSE2 michael@0: michael@0: #ifdef HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 michael@0: void CumulativeSumToAverageRow_SSE2(const int32* topleft, const int32* botleft, michael@0: int width, int area, uint8* dst, michael@0: int count) { michael@0: asm volatile ( michael@0: "movd %5,%%xmm5 \n" michael@0: "cvtdq2ps %%xmm5,%%xmm5 \n" michael@0: "rcpss %%xmm5,%%xmm4 \n" michael@0: "pshufd $0x0,%%xmm4,%%xmm4 \n" michael@0: "sub $0x4,%3 \n" michael@0: "jl 49f \n" michael@0: "cmpl $0x80,%5 \n" michael@0: "ja 40f \n" michael@0: michael@0: "pshufd $0x0,%%xmm5,%%xmm5 \n" michael@0: "pcmpeqb %%xmm6,%%xmm6 \n" michael@0: "psrld $0x10,%%xmm6 \n" michael@0: "cvtdq2ps %%xmm6,%%xmm6 \n" michael@0: "addps %%xmm6,%%xmm5 \n" michael@0: "mulps %%xmm4,%%xmm5 \n" michael@0: "cvtps2dq %%xmm5,%%xmm5 \n" michael@0: "packssdw %%xmm5,%%xmm5 \n" michael@0: michael@0: // 4 pixel small loop \n" michael@0: LABELALIGN michael@0: "4: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(psubd,0x00,0,4,4,xmm0) // psubd 0x00(%0,%4,4),%%xmm0 michael@0: MEMOPREG(psubd,0x10,0,4,4,xmm1) // psubd 0x10(%0,%4,4),%%xmm1 michael@0: MEMOPREG(psubd,0x20,0,4,4,xmm2) // psubd 0x20(%0,%4,4),%%xmm2 michael@0: MEMOPREG(psubd,0x30,0,4,4,xmm3) // psubd 0x30(%0,%4,4),%%xmm3 michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "psubd " MEMACCESS(1) ",%%xmm0 \n" michael@0: "psubd " MEMACCESS2(0x10,1) ",%%xmm1 \n" michael@0: "psubd " MEMACCESS2(0x20,1) ",%%xmm2 \n" michael@0: "psubd " MEMACCESS2(0x30,1) ",%%xmm3 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(paddd,0x00,1,4,4,xmm0) // paddd 0x00(%1,%4,4),%%xmm0 michael@0: MEMOPREG(paddd,0x10,1,4,4,xmm1) // paddd 0x10(%1,%4,4),%%xmm1 michael@0: MEMOPREG(paddd,0x20,1,4,4,xmm2) // paddd 0x20(%1,%4,4),%%xmm2 michael@0: MEMOPREG(paddd,0x30,1,4,4,xmm3) // paddd 0x30(%1,%4,4),%%xmm3 michael@0: "lea " MEMLEA(0x40,1) ",%1 \n" michael@0: "packssdw %%xmm1,%%xmm0 \n" michael@0: "packssdw %%xmm3,%%xmm2 \n" michael@0: "pmulhuw %%xmm5,%%xmm0 \n" michael@0: "pmulhuw %%xmm5,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "movdqu %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "sub $0x4,%3 \n" michael@0: "jge 4b \n" michael@0: "jmp 49f \n" michael@0: michael@0: // 4 pixel loop \n" michael@0: LABELALIGN michael@0: "40: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "movdqa " MEMACCESS2(0x20,0) ",%%xmm2 \n" michael@0: "movdqa " MEMACCESS2(0x30,0) ",%%xmm3 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(psubd,0x00,0,4,4,xmm0) // psubd 0x00(%0,%4,4),%%xmm0 michael@0: MEMOPREG(psubd,0x10,0,4,4,xmm1) // psubd 0x10(%0,%4,4),%%xmm1 michael@0: MEMOPREG(psubd,0x20,0,4,4,xmm2) // psubd 0x20(%0,%4,4),%%xmm2 michael@0: MEMOPREG(psubd,0x30,0,4,4,xmm3) // psubd 0x30(%0,%4,4),%%xmm3 michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "psubd " MEMACCESS(1) ",%%xmm0 \n" michael@0: "psubd " MEMACCESS2(0x10,1) ",%%xmm1 \n" michael@0: "psubd " MEMACCESS2(0x20,1) ",%%xmm2 \n" michael@0: "psubd " MEMACCESS2(0x30,1) ",%%xmm3 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(paddd,0x00,1,4,4,xmm0) // paddd 0x00(%1,%4,4),%%xmm0 michael@0: MEMOPREG(paddd,0x10,1,4,4,xmm1) // paddd 0x10(%1,%4,4),%%xmm1 michael@0: MEMOPREG(paddd,0x20,1,4,4,xmm2) // paddd 0x20(%1,%4,4),%%xmm2 michael@0: MEMOPREG(paddd,0x30,1,4,4,xmm3) // paddd 0x30(%1,%4,4),%%xmm3 michael@0: "lea " MEMLEA(0x40,1) ",%1 \n" michael@0: "cvtdq2ps %%xmm0,%%xmm0 \n" michael@0: "cvtdq2ps %%xmm1,%%xmm1 \n" michael@0: "mulps %%xmm4,%%xmm0 \n" michael@0: "mulps %%xmm4,%%xmm1 \n" michael@0: "cvtdq2ps %%xmm2,%%xmm2 \n" michael@0: "cvtdq2ps %%xmm3,%%xmm3 \n" michael@0: "mulps %%xmm4,%%xmm2 \n" michael@0: "mulps %%xmm4,%%xmm3 \n" michael@0: "cvtps2dq %%xmm0,%%xmm0 \n" michael@0: "cvtps2dq %%xmm1,%%xmm1 \n" michael@0: "cvtps2dq %%xmm2,%%xmm2 \n" michael@0: "cvtps2dq %%xmm3,%%xmm3 \n" michael@0: "packssdw %%xmm1,%%xmm0 \n" michael@0: "packssdw %%xmm3,%%xmm2 \n" michael@0: "packuswb %%xmm2,%%xmm0 \n" michael@0: "movdqu %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "sub $0x4,%3 \n" michael@0: "jge 40b \n" michael@0: michael@0: "49: \n" michael@0: "add $0x3,%3 \n" michael@0: "jl 19f \n" michael@0: michael@0: // 1 pixel loop \n" michael@0: LABELALIGN michael@0: "10: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: MEMOPREG(psubd,0x00,0,4,4,xmm0) // psubd 0x00(%0,%4,4),%%xmm0 michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "psubd " MEMACCESS(1) ",%%xmm0 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(paddd,0x00,1,4,4,xmm0) // paddd 0x00(%1,%4,4),%%xmm0 michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "cvtdq2ps %%xmm0,%%xmm0 \n" michael@0: "mulps %%xmm4,%%xmm0 \n" michael@0: "cvtps2dq %%xmm0,%%xmm0 \n" michael@0: "packssdw %%xmm0,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "movd %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x4,2) ",%2 \n" michael@0: "sub $0x1,%3 \n" michael@0: "jge 10b \n" michael@0: "19: \n" michael@0: : "+r"(topleft), // %0 michael@0: "+r"(botleft), // %1 michael@0: "+r"(dst), // %2 michael@0: "+rm"(count) // %3 michael@0: : "r"((intptr_t)(width)), // %4 michael@0: "rm"(area) // %5 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBAFFINEROW_SSE2 michael@0: // Copy ARGB pixels from source image with slope to a row of destination. michael@0: LIBYUV_API michael@0: void ARGBAffineRow_SSE2(const uint8* src_argb, int src_argb_stride, michael@0: uint8* dst_argb, const float* src_dudv, int width) { michael@0: intptr_t src_argb_stride_temp = src_argb_stride; michael@0: intptr_t temp = 0; michael@0: asm volatile ( michael@0: "movq " MEMACCESS(3) ",%%xmm2 \n" michael@0: "movq " MEMACCESS2(0x08,3) ",%%xmm7 \n" michael@0: "shl $0x10,%1 \n" michael@0: "add $0x4,%1 \n" michael@0: "movd %1,%%xmm5 \n" michael@0: "sub $0x4,%4 \n" michael@0: "jl 49f \n" michael@0: michael@0: "pshufd $0x44,%%xmm7,%%xmm7 \n" michael@0: "pshufd $0x0,%%xmm5,%%xmm5 \n" michael@0: "movdqa %%xmm2,%%xmm0 \n" michael@0: "addps %%xmm7,%%xmm0 \n" michael@0: "movlhps %%xmm0,%%xmm2 \n" michael@0: "movdqa %%xmm7,%%xmm4 \n" michael@0: "addps %%xmm4,%%xmm4 \n" michael@0: "movdqa %%xmm2,%%xmm3 \n" michael@0: "addps %%xmm4,%%xmm3 \n" michael@0: "addps %%xmm4,%%xmm4 \n" michael@0: michael@0: // 4 pixel loop \n" michael@0: LABELALIGN michael@0: "40: \n" michael@0: "cvttps2dq %%xmm2,%%xmm0 \n" // x, y float to int first 2 michael@0: "cvttps2dq %%xmm3,%%xmm1 \n" // x, y float to int next 2 michael@0: "packssdw %%xmm1,%%xmm0 \n" // x, y as 8 shorts michael@0: "pmaddwd %%xmm5,%%xmm0 \n" // off = x * 4 + y * stride michael@0: "movd %%xmm0,%k1 \n" michael@0: "pshufd $0x39,%%xmm0,%%xmm0 \n" michael@0: "movd %%xmm0,%k5 \n" michael@0: "pshufd $0x39,%%xmm0,%%xmm0 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movd,0x00,0,1,1,xmm1) // movd (%0,%1,1),%%xmm1 michael@0: MEMOPREG(movd,0x00,0,5,1,xmm6) // movd (%0,%5,1),%%xmm6 michael@0: "punpckldq %%xmm6,%%xmm1 \n" michael@0: "addps %%xmm4,%%xmm2 \n" michael@0: "movq %%xmm1," MEMACCESS(2) " \n" michael@0: "movd %%xmm0,%k1 \n" michael@0: "pshufd $0x39,%%xmm0,%%xmm0 \n" michael@0: "movd %%xmm0,%k5 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movd,0x00,0,1,1,xmm0) // movd (%0,%1,1),%%xmm0 michael@0: MEMOPREG(movd,0x00,0,5,1,xmm6) // movd (%0,%5,1),%%xmm6 michael@0: "punpckldq %%xmm6,%%xmm0 \n" michael@0: "addps %%xmm4,%%xmm3 \n" michael@0: "sub $0x4,%4 \n" michael@0: "movq %%xmm0," MEMACCESS2(0x08,2) " \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "jge 40b \n" michael@0: michael@0: "49: \n" michael@0: "add $0x3,%4 \n" michael@0: "jl 19f \n" michael@0: michael@0: // 1 pixel loop \n" michael@0: LABELALIGN michael@0: "10: \n" michael@0: "cvttps2dq %%xmm2,%%xmm0 \n" michael@0: "packssdw %%xmm0,%%xmm0 \n" michael@0: "pmaddwd %%xmm5,%%xmm0 \n" michael@0: "addps %%xmm7,%%xmm2 \n" michael@0: "movd %%xmm0,%k1 \n" michael@0: BUNDLEALIGN michael@0: MEMOPREG(movd,0x00,0,1,1,xmm0) // movd (%0,%1,1),%%xmm0 michael@0: "sub $0x1,%4 \n" michael@0: "movd %%xmm0," MEMACCESS(2) " \n" michael@0: "lea " MEMLEA(0x04,2) ",%2 \n" michael@0: "jge 10b \n" michael@0: "19: \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(src_argb_stride_temp), // %1 michael@0: "+r"(dst_argb), // %2 michael@0: "+r"(src_dudv), // %3 michael@0: "+rm"(width), // %4 michael@0: "+r"(temp) // %5 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBAFFINEROW_SSE2 michael@0: michael@0: #ifdef HAS_INTERPOLATEROW_SSSE3 michael@0: // Bilinear filter 16x2 -> 16x1 michael@0: void InterpolateRow_SSSE3(uint8* dst_ptr, const uint8* src_ptr, michael@0: ptrdiff_t src_stride, int dst_width, michael@0: int source_y_fraction) { michael@0: asm volatile ( michael@0: "sub %1,%0 \n" michael@0: "shr %3 \n" michael@0: "cmp $0x0,%3 \n" michael@0: "je 100f \n" michael@0: "cmp $0x20,%3 \n" michael@0: "je 75f \n" michael@0: "cmp $0x40,%3 \n" michael@0: "je 50f \n" michael@0: "cmp $0x60,%3 \n" michael@0: "je 25f \n" michael@0: michael@0: "movd %3,%%xmm0 \n" michael@0: "neg %3 \n" michael@0: "add $0x80,%3 \n" michael@0: "movd %3,%%xmm5 \n" michael@0: "punpcklbw %%xmm0,%%xmm5 \n" michael@0: "punpcklwd %%xmm5,%%xmm5 \n" michael@0: "pshufd $0x0,%%xmm5,%%xmm5 \n" michael@0: michael@0: // General purpose row blend. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,1,4,1,xmm2) michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm2,%%xmm0 \n" michael@0: "punpckhbw %%xmm2,%%xmm1 \n" michael@0: "pmaddubsw %%xmm5,%%xmm0 \n" michael@0: "pmaddubsw %%xmm5,%%xmm1 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 25 / 75. michael@0: LABELALIGN michael@0: "25: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,1,4,1,xmm1) michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 25b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 50 / 50. michael@0: LABELALIGN michael@0: "50: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,1,4,1,xmm1) michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 50b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 75 / 25. michael@0: LABELALIGN michael@0: "75: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm1 \n" michael@0: MEMOPREG(movdqa,0x00,1,4,1,xmm0) michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 75b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 100 / 0 - Copy row unchanged. michael@0: LABELALIGN michael@0: "100: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 100b \n" michael@0: michael@0: "99: \n" michael@0: : "+r"(dst_ptr), // %0 michael@0: "+r"(src_ptr), // %1 michael@0: "+r"(dst_width), // %2 michael@0: "+r"(source_y_fraction) // %3 michael@0: : "r"((intptr_t)(src_stride)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_INTERPOLATEROW_SSSE3 michael@0: michael@0: #ifdef HAS_INTERPOLATEROW_SSE2 michael@0: // Bilinear filter 16x2 -> 16x1 michael@0: void InterpolateRow_SSE2(uint8* dst_ptr, const uint8* src_ptr, michael@0: ptrdiff_t src_stride, int dst_width, michael@0: int source_y_fraction) { michael@0: asm volatile ( michael@0: "sub %1,%0 \n" michael@0: "shr %3 \n" michael@0: "cmp $0x0,%3 \n" michael@0: "je 100f \n" michael@0: "cmp $0x20,%3 \n" michael@0: "je 75f \n" michael@0: "cmp $0x40,%3 \n" michael@0: "je 50f \n" michael@0: "cmp $0x60,%3 \n" michael@0: "je 25f \n" michael@0: michael@0: "movd %3,%%xmm0 \n" michael@0: "neg %3 \n" michael@0: "add $0x80,%3 \n" michael@0: "movd %3,%%xmm5 \n" michael@0: "punpcklbw %%xmm0,%%xmm5 \n" michael@0: "punpcklwd %%xmm5,%%xmm5 \n" michael@0: "pshufd $0x0,%%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: michael@0: // General purpose row blend. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,1,4,1,xmm2) // movdqa (%1,%4,1),%%xmm2 michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm2,%%xmm3 \n" michael@0: "punpcklbw %%xmm4,%%xmm2 \n" michael@0: "punpckhbw %%xmm4,%%xmm3 \n" michael@0: "punpcklbw %%xmm4,%%xmm0 \n" michael@0: "punpckhbw %%xmm4,%%xmm1 \n" michael@0: "psubw %%xmm0,%%xmm2 \n" michael@0: "psubw %%xmm1,%%xmm3 \n" michael@0: "paddw %%xmm2,%%xmm2 \n" michael@0: "paddw %%xmm3,%%xmm3 \n" michael@0: "pmulhw %%xmm5,%%xmm2 \n" michael@0: "pmulhw %%xmm5,%%xmm3 \n" michael@0: "paddw %%xmm2,%%xmm0 \n" michael@0: "paddw %%xmm3,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) // movdqa %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 25 / 75. michael@0: LABELALIGN michael@0: "25: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,1,4,1,xmm1) // movdqa (%1,%4,1),%%xmm1 michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) // movdqa %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 25b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 50 / 50. michael@0: LABELALIGN michael@0: "50: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqa,0x00,1,4,1,xmm1) // movdqa (%1,%4,1),%%xmm1 michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) // movdqa %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 50b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 75 / 25. michael@0: LABELALIGN michael@0: "75: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm1 \n" michael@0: MEMOPREG(movdqa,0x00,1,4,1,xmm0) // movdqa (%1,%4,1),%%xmm0 michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) // movdqa %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 75b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 100 / 0 - Copy row unchanged. michael@0: LABELALIGN michael@0: "100: \n" michael@0: "movdqa " MEMACCESS(1) ",%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: MEMOPMEM(movdqa,xmm0,0x00,1,0,1) // movdqa %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 100b \n" michael@0: michael@0: "99: \n" michael@0: : "+r"(dst_ptr), // %0 michael@0: "+r"(src_ptr), // %1 michael@0: "+r"(dst_width), // %2 michael@0: "+r"(source_y_fraction) // %3 michael@0: : "r"((intptr_t)(src_stride)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_INTERPOLATEROW_SSE2 michael@0: michael@0: #ifdef HAS_INTERPOLATEROW_SSSE3 michael@0: // Bilinear filter 16x2 -> 16x1 michael@0: void InterpolateRow_Unaligned_SSSE3(uint8* dst_ptr, const uint8* src_ptr, michael@0: ptrdiff_t src_stride, int dst_width, michael@0: int source_y_fraction) { michael@0: asm volatile ( michael@0: "sub %1,%0 \n" michael@0: "shr %3 \n" michael@0: "cmp $0x0,%3 \n" michael@0: "je 100f \n" michael@0: "cmp $0x20,%3 \n" michael@0: "je 75f \n" michael@0: "cmp $0x40,%3 \n" michael@0: "je 50f \n" michael@0: "cmp $0x60,%3 \n" michael@0: "je 25f \n" michael@0: michael@0: "movd %3,%%xmm0 \n" michael@0: "neg %3 \n" michael@0: "add $0x80,%3 \n" michael@0: "movd %3,%%xmm5 \n" michael@0: "punpcklbw %%xmm0,%%xmm5 \n" michael@0: "punpcklwd %%xmm5,%%xmm5 \n" michael@0: "pshufd $0x0,%%xmm5,%%xmm5 \n" michael@0: michael@0: // General purpose row blend. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x00,1,4,1,xmm2) michael@0: "movdqu %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm2,%%xmm0 \n" michael@0: "punpckhbw %%xmm2,%%xmm1 \n" michael@0: "pmaddubsw %%xmm5,%%xmm0 \n" michael@0: "pmaddubsw %%xmm5,%%xmm1 \n" michael@0: "psrlw $0x7,%%xmm0 \n" michael@0: "psrlw $0x7,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 25 / 75. michael@0: LABELALIGN michael@0: "25: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x00,1,4,1,xmm1) michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 25b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 50 / 50. michael@0: LABELALIGN michael@0: "50: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x00,1,4,1,xmm1) michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 50b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 75 / 25. michael@0: LABELALIGN michael@0: "75: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm1 \n" michael@0: MEMOPREG(movdqu,0x00,1,4,1,xmm0) michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 75b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 100 / 0 - Copy row unchanged. michael@0: LABELALIGN michael@0: "100: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 100b \n" michael@0: michael@0: "99: \n" michael@0: : "+r"(dst_ptr), // %0 michael@0: "+r"(src_ptr), // %1 michael@0: "+r"(dst_width), // %2 michael@0: "+r"(source_y_fraction) // %3 michael@0: : "r"((intptr_t)(src_stride)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_INTERPOLATEROW_SSSE3 michael@0: michael@0: #ifdef HAS_INTERPOLATEROW_SSE2 michael@0: // Bilinear filter 16x2 -> 16x1 michael@0: void InterpolateRow_Unaligned_SSE2(uint8* dst_ptr, const uint8* src_ptr, michael@0: ptrdiff_t src_stride, int dst_width, michael@0: int source_y_fraction) { michael@0: asm volatile ( michael@0: "sub %1,%0 \n" michael@0: "shr %3 \n" michael@0: "cmp $0x0,%3 \n" michael@0: "je 100f \n" michael@0: "cmp $0x20,%3 \n" michael@0: "je 75f \n" michael@0: "cmp $0x40,%3 \n" michael@0: "je 50f \n" michael@0: "cmp $0x60,%3 \n" michael@0: "je 25f \n" michael@0: michael@0: "movd %3,%%xmm0 \n" michael@0: "neg %3 \n" michael@0: "add $0x80,%3 \n" michael@0: "movd %3,%%xmm5 \n" michael@0: "punpcklbw %%xmm0,%%xmm5 \n" michael@0: "punpcklwd %%xmm5,%%xmm5 \n" michael@0: "pshufd $0x0,%%xmm5,%%xmm5 \n" michael@0: "pxor %%xmm4,%%xmm4 \n" michael@0: michael@0: // General purpose row blend. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x00,1,4,1,xmm2) // movdqu (%1,%4,1),%%xmm2 michael@0: "movdqu %%xmm0,%%xmm1 \n" michael@0: "movdqu %%xmm2,%%xmm3 \n" michael@0: "punpcklbw %%xmm4,%%xmm2 \n" michael@0: "punpckhbw %%xmm4,%%xmm3 \n" michael@0: "punpcklbw %%xmm4,%%xmm0 \n" michael@0: "punpckhbw %%xmm4,%%xmm1 \n" michael@0: "psubw %%xmm0,%%xmm2 \n" michael@0: "psubw %%xmm1,%%xmm3 \n" michael@0: "paddw %%xmm2,%%xmm2 \n" michael@0: "paddw %%xmm3,%%xmm3 \n" michael@0: "pmulhw %%xmm5,%%xmm2 \n" michael@0: "pmulhw %%xmm5,%%xmm3 \n" michael@0: "paddw %%xmm2,%%xmm0 \n" michael@0: "paddw %%xmm3,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 25 / 75. michael@0: LABELALIGN michael@0: "25: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x00,1,4,1,xmm1) // movdqu (%1,%4,1),%%xmm1 michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 25b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 50 / 50. michael@0: LABELALIGN michael@0: "50: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm0 \n" michael@0: MEMOPREG(movdqu,0x00,1,4,1,xmm1) // movdqu (%1,%4,1),%%xmm1 michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 50b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 75 / 25. michael@0: LABELALIGN michael@0: "75: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm1 \n" michael@0: MEMOPREG(movdqu,0x00,1,4,1,xmm0) // movdqu (%1,%4,1),%%xmm0 michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "pavgb %%xmm1,%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: BUNDLEALIGN michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 75b \n" michael@0: "jmp 99f \n" michael@0: michael@0: // Blend 100 / 0 - Copy row unchanged. michael@0: LABELALIGN michael@0: "100: \n" michael@0: "movdqu " MEMACCESS(1) ",%%xmm0 \n" michael@0: "sub $0x10,%2 \n" michael@0: MEMOPMEM(movdqu,xmm0,0x00,1,0,1) // movdqu %%xmm0,(%1,%0,1) michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 100b \n" michael@0: michael@0: "99: \n" michael@0: : "+r"(dst_ptr), // %0 michael@0: "+r"(src_ptr), // %1 michael@0: "+r"(dst_width), // %2 michael@0: "+r"(source_y_fraction) // %3 michael@0: : "r"((intptr_t)(src_stride)) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_INTERPOLATEROW_SSE2 michael@0: michael@0: #ifdef HAS_HALFROW_SSE2 michael@0: void HalfRow_SSE2(const uint8* src_uv, int src_uv_stride, michael@0: uint8* dst_uv, int pix) { michael@0: asm volatile ( michael@0: "sub %0,%1 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: MEMOPREG(pavgb,0x00,0,3,1,xmm0) // pavgb (%0,%3),%%xmm0 michael@0: "sub $0x10,%2 \n" michael@0: MEMOPMEM(movdqa,xmm0,0x00,0,1,1) // movdqa %%xmm0,(%0,%1) michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_uv), // %0 michael@0: "+r"(dst_uv), // %1 michael@0: "+r"(pix) // %2 michael@0: : "r"((intptr_t)(src_uv_stride)) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_HALFROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBTOBAYERROW_SSSE3 michael@0: void ARGBToBayerRow_SSSE3(const uint8* src_argb, uint8* dst_bayer, michael@0: uint32 selector, int pix) { michael@0: asm volatile ( michael@0: // NaCL caveat - assumes movd is from GPR michael@0: "movd %3,%%xmm5 \n" michael@0: "pshufd $0x0,%%xmm5,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pshufb %%xmm5,%%xmm0 \n" michael@0: "pshufb %%xmm5,%%xmm1 \n" michael@0: "punpckldq %%xmm1,%%xmm0 \n" michael@0: "sub $0x8,%2 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_bayer), // %1 michael@0: "+r"(pix) // %2 michael@0: : "g"(selector) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBTOBAYERROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBTOBAYERGGROW_SSE2 michael@0: void ARGBToBayerGGRow_SSE2(const uint8* src_argb, uint8* dst_bayer, michael@0: uint32 selector, int pix) { michael@0: asm volatile ( michael@0: "pcmpeqb %%xmm5,%%xmm5 \n" michael@0: "psrld $0x18,%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "psrld $0x8,%%xmm0 \n" michael@0: "psrld $0x8,%%xmm1 \n" michael@0: "pand %%xmm5,%%xmm0 \n" michael@0: "pand %%xmm5,%%xmm1 \n" michael@0: "packssdw %%xmm1,%%xmm0 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x8,%2 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_bayer), // %1 michael@0: "+r"(pix) // %2 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBTOBAYERGGROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBSHUFFLEROW_SSSE3 michael@0: // For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. michael@0: void ARGBShuffleRow_SSSE3(const uint8* src_argb, uint8* dst_argb, michael@0: const uint8* shuffler, int pix) { michael@0: asm volatile ( michael@0: "movdqa " MEMACCESS(3) ",%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqa " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pshufb %%xmm5,%%xmm0 \n" michael@0: "pshufb %%xmm5,%%xmm1 \n" michael@0: "sub $0x8,%2 \n" michael@0: "movdqa %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqa %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(pix) // %2 michael@0: : "r"(shuffler) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: michael@0: void ARGBShuffleRow_Unaligned_SSSE3(const uint8* src_argb, uint8* dst_argb, michael@0: const uint8* shuffler, int pix) { michael@0: asm volatile ( michael@0: "movdqa " MEMACCESS(3) ",%%xmm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqu " MEMACCESS2(0x10,0) ",%%xmm1 \n" michael@0: "lea " MEMLEA(0x20,0) ",%0 \n" michael@0: "pshufb %%xmm5,%%xmm0 \n" michael@0: "pshufb %%xmm5,%%xmm1 \n" michael@0: "sub $0x8,%2 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,1) " \n" michael@0: "lea " MEMLEA(0x20,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(pix) // %2 michael@0: : "r"(shuffler) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBSHUFFLEROW_SSSE3 michael@0: michael@0: #ifdef HAS_ARGBSHUFFLEROW_AVX2 michael@0: // For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. michael@0: void ARGBShuffleRow_AVX2(const uint8* src_argb, uint8* dst_argb, michael@0: const uint8* shuffler, int pix) { michael@0: asm volatile ( michael@0: "vbroadcastf128 " MEMACCESS(3) ",%%ymm5 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "vmovdqu " MEMACCESS(0) ",%%ymm0 \n" michael@0: "vmovdqu " MEMACCESS2(0x20,0) ",%%ymm1 \n" michael@0: "lea " MEMLEA(0x40,0) ",%0 \n" michael@0: "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" michael@0: "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" michael@0: "sub $0x10,%2 \n" michael@0: "vmovdqu %%ymm0," MEMACCESS(1) " \n" michael@0: "vmovdqu %%ymm1," MEMACCESS2(0x20,1) " \n" michael@0: "lea " MEMLEA(0x40,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(pix) // %2 michael@0: : "r"(shuffler) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBSHUFFLEROW_AVX2 michael@0: michael@0: #ifdef HAS_ARGBSHUFFLEROW_SSE2 michael@0: // For BGRAToARGB, ABGRToARGB, RGBAToARGB, and ARGBToRGBA. michael@0: void ARGBShuffleRow_SSE2(const uint8* src_argb, uint8* dst_argb, michael@0: const uint8* shuffler, int pix) { michael@0: uintptr_t pixel_temp = 0u; michael@0: asm volatile ( michael@0: "pxor %%xmm5,%%xmm5 \n" michael@0: "mov " MEMACCESS(4) ",%k2 \n" michael@0: "cmp $0x3000102,%k2 \n" michael@0: "je 3012f \n" michael@0: "cmp $0x10203,%k2 \n" michael@0: "je 123f \n" michael@0: "cmp $0x30201,%k2 \n" michael@0: "je 321f \n" michael@0: "cmp $0x2010003,%k2 \n" michael@0: "je 2103f \n" michael@0: michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movzb " MEMACCESS(4) ",%2 \n" michael@0: MEMOPARG(movzb,0x00,0,2,1,2) " \n" // movzb (%0,%2,1),%2 michael@0: "mov %b2," MEMACCESS(1) " \n" michael@0: "movzb " MEMACCESS2(0x1,4) ",%2 \n" michael@0: MEMOPARG(movzb,0x00,0,2,1,2) " \n" // movzb (%0,%2,1),%2 michael@0: "mov %b2," MEMACCESS2(0x1,1) " \n" michael@0: BUNDLEALIGN michael@0: "movzb " MEMACCESS2(0x2,4) ",%2 \n" michael@0: MEMOPARG(movzb,0x00,0,2,1,2) " \n" // movzb (%0,%2,1),%2 michael@0: "mov %b2," MEMACCESS2(0x2,1) " \n" michael@0: "movzb " MEMACCESS2(0x3,4) ",%2 \n" michael@0: MEMOPARG(movzb,0x00,0,2,1,2) " \n" // movzb (%0,%2,1),%2 michael@0: "mov %b2," MEMACCESS2(0x3,1) " \n" michael@0: "lea " MEMLEA(0x4,0) ",%0 \n" michael@0: "lea " MEMLEA(0x4,1) ",%1 \n" michael@0: "sub $0x1,%3 \n" michael@0: "jg 1b \n" michael@0: "jmp 99f \n" michael@0: michael@0: LABELALIGN michael@0: "123: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "punpckhbw %%xmm5,%%xmm1 \n" michael@0: "pshufhw $0x1b,%%xmm0,%%xmm0 \n" michael@0: "pshuflw $0x1b,%%xmm0,%%xmm0 \n" michael@0: "pshufhw $0x1b,%%xmm1,%%xmm1 \n" michael@0: "pshuflw $0x1b,%%xmm1,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 123b \n" michael@0: "jmp 99f \n" michael@0: michael@0: LABELALIGN michael@0: "321: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "punpckhbw %%xmm5,%%xmm1 \n" michael@0: "pshufhw $0x39,%%xmm0,%%xmm0 \n" michael@0: "pshuflw $0x39,%%xmm0,%%xmm0 \n" michael@0: "pshufhw $0x39,%%xmm1,%%xmm1 \n" michael@0: "pshuflw $0x39,%%xmm1,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 321b \n" michael@0: "jmp 99f \n" michael@0: michael@0: LABELALIGN michael@0: "2103: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "punpckhbw %%xmm5,%%xmm1 \n" michael@0: "pshufhw $0x93,%%xmm0,%%xmm0 \n" michael@0: "pshuflw $0x93,%%xmm0,%%xmm0 \n" michael@0: "pshufhw $0x93,%%xmm1,%%xmm1 \n" michael@0: "pshuflw $0x93,%%xmm1,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 2103b \n" michael@0: "jmp 99f \n" michael@0: michael@0: LABELALIGN michael@0: "3012: \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm5,%%xmm0 \n" michael@0: "punpckhbw %%xmm5,%%xmm1 \n" michael@0: "pshufhw $0xc6,%%xmm0,%%xmm0 \n" michael@0: "pshuflw $0xc6,%%xmm0,%%xmm0 \n" michael@0: "pshufhw $0xc6,%%xmm1,%%xmm1 \n" michael@0: "pshuflw $0xc6,%%xmm1,%%xmm1 \n" michael@0: "packuswb %%xmm1,%%xmm0 \n" michael@0: "sub $0x4,%3 \n" michael@0: "movdqu %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x10,1) ",%1 \n" michael@0: "jg 3012b \n" michael@0: michael@0: "99: \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+d"(pixel_temp), // %2 michael@0: "+r"(pix) // %3 michael@0: : "r"(shuffler) // %4 michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBSHUFFLEROW_SSE2 michael@0: michael@0: #ifdef HAS_I422TOYUY2ROW_SSE2 michael@0: void I422ToYUY2Row_SSE2(const uint8* src_y, michael@0: const uint8* src_u, michael@0: const uint8* src_v, michael@0: uint8* dst_frame, int width) { michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movq " MEMACCESS(1) ",%%xmm2 \n" michael@0: MEMOPREG(movq,0x00,1,2,1,xmm3) // movq (%1,%2,1),%%xmm3 michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "punpcklbw %%xmm3,%%xmm2 \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "punpcklbw %%xmm2,%%xmm0 \n" michael@0: "punpckhbw %%xmm2,%%xmm1 \n" michael@0: "movdqu %%xmm0," MEMACCESS(3) " \n" michael@0: "movdqu %%xmm1," MEMACCESS2(0x10,3) " \n" michael@0: "lea " MEMLEA(0x20,3) ",%3 \n" michael@0: "sub $0x10,%4 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_y), // %0 michael@0: "+r"(src_u), // %1 michael@0: "+r"(src_v), // %2 michael@0: "+r"(dst_frame), // %3 michael@0: "+rm"(width) // %4 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_I422TOYUY2ROW_SSE2 michael@0: michael@0: #ifdef HAS_I422TOUYVYROW_SSE2 michael@0: void I422ToUYVYRow_SSE2(const uint8* src_y, michael@0: const uint8* src_u, michael@0: const uint8* src_v, michael@0: uint8* dst_frame, int width) { michael@0: asm volatile ( michael@0: "sub %1,%2 \n" michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movq " MEMACCESS(1) ",%%xmm2 \n" michael@0: MEMOPREG(movq,0x00,1,2,1,xmm3) // movq (%1,%2,1),%%xmm3 michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "punpcklbw %%xmm3,%%xmm2 \n" michael@0: "movdqu " MEMACCESS(0) ",%%xmm0 \n" michael@0: "movdqa %%xmm2,%%xmm1 \n" michael@0: "lea " MEMLEA(0x10,0) ",%0 \n" michael@0: "punpcklbw %%xmm0,%%xmm1 \n" michael@0: "punpckhbw %%xmm0,%%xmm2 \n" michael@0: "movdqu %%xmm1," MEMACCESS(3) " \n" michael@0: "movdqu %%xmm2," MEMACCESS2(0x10,3) " \n" michael@0: "lea " MEMLEA(0x20,3) ",%3 \n" michael@0: "sub $0x10,%4 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_y), // %0 michael@0: "+r"(src_u), // %1 michael@0: "+r"(src_v), // %2 michael@0: "+r"(dst_frame), // %3 michael@0: "+rm"(width) // %4 michael@0: : michael@0: : "memory", "cc" michael@0: #if defined(__native_client__) && defined(__x86_64__) michael@0: , "r14" michael@0: #endif michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_I422TOUYVYROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBPOLYNOMIALROW_SSE2 michael@0: void ARGBPolynomialRow_SSE2(const uint8* src_argb, michael@0: uint8* dst_argb, const float* poly, michael@0: int width) { michael@0: asm volatile ( michael@0: "pxor %%xmm3,%%xmm3 \n" michael@0: michael@0: // 2 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movq " MEMACCESS(0) ",%%xmm0 \n" michael@0: "lea " MEMLEA(0x8,0) ",%0 \n" michael@0: "punpcklbw %%xmm3,%%xmm0 \n" michael@0: "movdqa %%xmm0,%%xmm4 \n" michael@0: "punpcklwd %%xmm3,%%xmm0 \n" michael@0: "punpckhwd %%xmm3,%%xmm4 \n" michael@0: "cvtdq2ps %%xmm0,%%xmm0 \n" michael@0: "cvtdq2ps %%xmm4,%%xmm4 \n" michael@0: "movdqa %%xmm0,%%xmm1 \n" michael@0: "movdqa %%xmm4,%%xmm5 \n" michael@0: "mulps " MEMACCESS2(0x10,3) ",%%xmm0 \n" michael@0: "mulps " MEMACCESS2(0x10,3) ",%%xmm4 \n" michael@0: "addps " MEMACCESS(3) ",%%xmm0 \n" michael@0: "addps " MEMACCESS(3) ",%%xmm4 \n" michael@0: "movdqa %%xmm1,%%xmm2 \n" michael@0: "movdqa %%xmm5,%%xmm6 \n" michael@0: "mulps %%xmm1,%%xmm2 \n" michael@0: "mulps %%xmm5,%%xmm6 \n" michael@0: "mulps %%xmm2,%%xmm1 \n" michael@0: "mulps %%xmm6,%%xmm5 \n" michael@0: "mulps " MEMACCESS2(0x20,3) ",%%xmm2 \n" michael@0: "mulps " MEMACCESS2(0x20,3) ",%%xmm6 \n" michael@0: "mulps " MEMACCESS2(0x30,3) ",%%xmm1 \n" michael@0: "mulps " MEMACCESS2(0x30,3) ",%%xmm5 \n" michael@0: "addps %%xmm2,%%xmm0 \n" michael@0: "addps %%xmm6,%%xmm4 \n" michael@0: "addps %%xmm1,%%xmm0 \n" michael@0: "addps %%xmm5,%%xmm4 \n" michael@0: "cvttps2dq %%xmm0,%%xmm0 \n" michael@0: "cvttps2dq %%xmm4,%%xmm4 \n" michael@0: "packuswb %%xmm4,%%xmm0 \n" michael@0: "packuswb %%xmm0,%%xmm0 \n" michael@0: "sub $0x2,%2 \n" michael@0: "movq %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(width) // %2 michael@0: : "r"(poly) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBPOLYNOMIALROW_SSE2 michael@0: michael@0: #ifdef HAS_ARGBPOLYNOMIALROW_AVX2 michael@0: void ARGBPolynomialRow_AVX2(const uint8* src_argb, michael@0: uint8* dst_argb, const float* poly, michael@0: int width) { michael@0: asm volatile ( michael@0: "vbroadcastf128 " MEMACCESS(3) ",%%ymm4 \n" michael@0: "vbroadcastf128 " MEMACCESS2(0x10,3) ",%%ymm5 \n" michael@0: "vbroadcastf128 " MEMACCESS2(0x20,3) ",%%ymm6 \n" michael@0: "vbroadcastf128 " MEMACCESS2(0x30,3) ",%%ymm7 \n" michael@0: michael@0: // 2 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "vpmovzxbd " MEMACCESS(0) ",%%ymm0 \n" // 2 ARGB pixels michael@0: "lea " MEMLEA(0x8,0) ",%0 \n" michael@0: "vcvtdq2ps %%ymm0,%%ymm0 \n" // X 8 floats michael@0: "vmulps %%ymm0,%%ymm0,%%ymm2 \n" // X * X michael@0: "vmulps %%ymm7,%%ymm0,%%ymm3 \n" // C3 * X michael@0: "vfmadd132ps %%ymm5,%%ymm4,%%ymm0 \n" // result = C0 + C1 * X michael@0: "vfmadd231ps %%ymm6,%%ymm2,%%ymm0 \n" // result += C2 * X * X michael@0: "vfmadd231ps %%ymm3,%%ymm2,%%ymm0 \n" // result += C3 * X * X * X michael@0: "vcvttps2dq %%ymm0,%%ymm0 \n" michael@0: "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n" michael@0: "vpermq $0xd8,%%ymm0,%%ymm0 \n" michael@0: "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" michael@0: "sub $0x2,%2 \n" michael@0: "vmovq %%xmm0," MEMACCESS(1) " \n" michael@0: "lea " MEMLEA(0x8,1) ",%1 \n" michael@0: "jg 1b \n" michael@0: "vzeroupper \n" michael@0: : "+r"(src_argb), // %0 michael@0: "+r"(dst_argb), // %1 michael@0: "+r"(width) // %2 michael@0: : "r"(poly) // %3 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: // TODO(fbarchard): declare ymm usage when applicable. michael@0: , "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBPOLYNOMIALROW_AVX2 michael@0: michael@0: #ifdef HAS_ARGBCOLORTABLEROW_X86 michael@0: // Tranform ARGB pixels with color table. michael@0: void ARGBColorTableRow_X86(uint8* dst_argb, const uint8* table_argb, michael@0: int width) { michael@0: uintptr_t pixel_temp = 0u; michael@0: asm volatile ( michael@0: // 1 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movzb " MEMACCESS(0) ",%1 \n" michael@0: "lea " MEMLEA(0x4,0) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,3,1,4,1) " \n" // movzb (%3,%1,4),%1 michael@0: "mov %b1," MEMACCESS2(-0x4,0) " \n" michael@0: "movzb " MEMACCESS2(-0x3,0) ",%1 \n" michael@0: MEMOPARG(movzb,0x01,3,1,4,1) " \n" // movzb 0x1(%3,%1,4),%1 michael@0: "mov %b1," MEMACCESS2(-0x3,0) " \n" michael@0: "movzb " MEMACCESS2(-0x2,0) ",%1 \n" michael@0: MEMOPARG(movzb,0x02,3,1,4,1) " \n" // movzb 0x2(%3,%1,4),%1 michael@0: "mov %b1," MEMACCESS2(-0x2,0) " \n" michael@0: "movzb " MEMACCESS2(-0x1,0) ",%1 \n" michael@0: MEMOPARG(movzb,0x03,3,1,4,1) " \n" // movzb 0x3(%3,%1,4),%1 michael@0: "mov %b1," MEMACCESS2(-0x1,0) " \n" michael@0: "dec %2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(dst_argb), // %0 michael@0: "+d"(pixel_temp), // %1 michael@0: "+r"(width) // %2 michael@0: : "r"(table_argb) // %3 michael@0: : "memory", "cc"); michael@0: } michael@0: #endif // HAS_ARGBCOLORTABLEROW_X86 michael@0: michael@0: #ifdef HAS_RGBCOLORTABLEROW_X86 michael@0: // Tranform RGB pixels with color table. michael@0: void RGBColorTableRow_X86(uint8* dst_argb, const uint8* table_argb, int width) { michael@0: uintptr_t pixel_temp = 0u; michael@0: asm volatile ( michael@0: // 1 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movzb " MEMACCESS(0) ",%1 \n" michael@0: "lea " MEMLEA(0x4,0) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,3,1,4,1) " \n" // movzb (%3,%1,4),%1 michael@0: "mov %b1," MEMACCESS2(-0x4,0) " \n" michael@0: "movzb " MEMACCESS2(-0x3,0) ",%1 \n" michael@0: MEMOPARG(movzb,0x01,3,1,4,1) " \n" // movzb 0x1(%3,%1,4),%1 michael@0: "mov %b1," MEMACCESS2(-0x3,0) " \n" michael@0: "movzb " MEMACCESS2(-0x2,0) ",%1 \n" michael@0: MEMOPARG(movzb,0x02,3,1,4,1) " \n" // movzb 0x2(%3,%1,4),%1 michael@0: "mov %b1," MEMACCESS2(-0x2,0) " \n" michael@0: "dec %2 \n" michael@0: "jg 1b \n" michael@0: : "+r"(dst_argb), // %0 michael@0: "+d"(pixel_temp), // %1 michael@0: "+r"(width) // %2 michael@0: : "r"(table_argb) // %3 michael@0: : "memory", "cc"); michael@0: } michael@0: #endif // HAS_RGBCOLORTABLEROW_X86 michael@0: michael@0: #ifdef HAS_ARGBLUMACOLORTABLEROW_SSSE3 michael@0: // Tranform RGB pixels with luma table. michael@0: void ARGBLumaColorTableRow_SSSE3(const uint8* src_argb, uint8* dst_argb, michael@0: int width, michael@0: const uint8* luma, uint32 lumacoeff) { michael@0: uintptr_t pixel_temp = 0u; michael@0: uintptr_t table_temp = 0u; michael@0: asm volatile ( michael@0: "movd %6,%%xmm3 \n" michael@0: "pshufd $0x0,%%xmm3,%%xmm3 \n" michael@0: "pcmpeqb %%xmm4,%%xmm4 \n" michael@0: "psllw $0x8,%%xmm4 \n" michael@0: "pxor %%xmm5,%%xmm5 \n" michael@0: michael@0: // 4 pixel loop. michael@0: LABELALIGN michael@0: "1: \n" michael@0: "movdqu " MEMACCESS(2) ",%%xmm0 \n" michael@0: "pmaddubsw %%xmm3,%%xmm0 \n" michael@0: "phaddw %%xmm0,%%xmm0 \n" michael@0: "pand %%xmm4,%%xmm0 \n" michael@0: "punpcklwd %%xmm5,%%xmm0 \n" michael@0: "movd %%xmm0,%k1 \n" // 32 bit offset michael@0: "add %5,%1 \n" michael@0: "pshufd $0x39,%%xmm0,%%xmm0 \n" michael@0: michael@0: "movzb " MEMACCESS(2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS(3) " \n" michael@0: "movzb " MEMACCESS2(0x1,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0x1,3) " \n" michael@0: "movzb " MEMACCESS2(0x2,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0x2,3) " \n" michael@0: "movzb " MEMACCESS2(0x3,2) ",%0 \n" michael@0: "mov %b0," MEMACCESS2(0x3,3) " \n" michael@0: michael@0: "movd %%xmm0,%k1 \n" // 32 bit offset michael@0: "add %5,%1 \n" michael@0: "pshufd $0x39,%%xmm0,%%xmm0 \n" michael@0: michael@0: "movzb " MEMACCESS2(0x4,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0x4,3) " \n" michael@0: BUNDLEALIGN michael@0: "movzb " MEMACCESS2(0x5,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0x5,3) " \n" michael@0: "movzb " MEMACCESS2(0x6,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0x6,3) " \n" michael@0: "movzb " MEMACCESS2(0x7,2) ",%0 \n" michael@0: "mov %b0," MEMACCESS2(0x7,3) " \n" michael@0: michael@0: "movd %%xmm0,%k1 \n" // 32 bit offset michael@0: "add %5,%1 \n" michael@0: "pshufd $0x39,%%xmm0,%%xmm0 \n" michael@0: michael@0: "movzb " MEMACCESS2(0x8,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0x8,3) " \n" michael@0: "movzb " MEMACCESS2(0x9,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0x9,3) " \n" michael@0: "movzb " MEMACCESS2(0xa,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0xa,3) " \n" michael@0: "movzb " MEMACCESS2(0xb,2) ",%0 \n" michael@0: "mov %b0," MEMACCESS2(0xb,3) " \n" michael@0: michael@0: "movd %%xmm0,%k1 \n" // 32 bit offset michael@0: "add %5,%1 \n" michael@0: michael@0: "movzb " MEMACCESS2(0xc,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0xc,3) " \n" michael@0: "movzb " MEMACCESS2(0xd,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0xd,3) " \n" michael@0: "movzb " MEMACCESS2(0xe,2) ",%0 \n" michael@0: MEMOPARG(movzb,0x00,1,0,1,0) " \n" // movzb (%1,%0,1),%0 michael@0: "mov %b0," MEMACCESS2(0xe,3) " \n" michael@0: "movzb " MEMACCESS2(0xf,2) ",%0 \n" michael@0: "mov %b0," MEMACCESS2(0xf,3) " \n" michael@0: "sub $0x4,%4 \n" michael@0: "lea " MEMLEA(0x10,2) ",%2 \n" michael@0: "lea " MEMLEA(0x10,3) ",%3 \n" michael@0: "jg 1b \n" michael@0: : "+d"(pixel_temp), // %0 michael@0: "+a"(table_temp), // %1 michael@0: "+r"(src_argb), // %2 michael@0: "+r"(dst_argb), // %3 michael@0: "+rm"(width) // %4 michael@0: : "r"(luma), // %5 michael@0: "rm"(lumacoeff) // %6 michael@0: : "memory", "cc" michael@0: #if defined(__SSE2__) michael@0: , "xmm0", "xmm3", "xmm4", "xmm5" michael@0: #endif michael@0: ); michael@0: } michael@0: #endif // HAS_ARGBLUMACOLORTABLEROW_SSSE3 michael@0: michael@0: #endif // defined(__x86_64__) || defined(__i386__) michael@0: michael@0: #ifdef __cplusplus michael@0: } // extern "C" michael@0: } // namespace libyuv michael@0: #endif