michael@0: /*
michael@0:  *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
michael@0:  *
michael@0:  *  Use of this source code is governed by a BSD-style license
michael@0:  *  that can be found in the LICENSE file in the root of the source
michael@0:  *  tree. An additional intellectual property rights grant can be found
michael@0:  *  in the file PATENTS.  All contributing project authors may
michael@0:  *  be found in the AUTHORS file in the root of the source tree.
michael@0:  */
michael@0: 
michael@0: #include <assert.h>
michael@0: 
michael@0: #include "./vpx_config.h"
michael@0: #include "./vp9_rtcd.h"
michael@0: #include "vpx_ports/mem.h"
michael@0: ///////////////////////////////////////////////////////////////////////////
michael@0: // the mmx function that does the bilinear filtering and var calculation //
michael@0: // int one pass                                                          //
michael@0: ///////////////////////////////////////////////////////////////////////////
michael@0: DECLARE_ALIGNED(16, const short, vp9_bilinear_filters_mmx[16][8]) = {
michael@0:   { 128, 128, 128, 128,  0,  0,  0,  0 },
michael@0:   { 120, 120, 120, 120,  8,  8,  8,  8 },
michael@0:   { 112, 112, 112, 112, 16, 16, 16, 16 },
michael@0:   { 104, 104, 104, 104, 24, 24, 24, 24 },
michael@0:   {  96, 96, 96, 96, 32, 32, 32, 32 },
michael@0:   {  88, 88, 88, 88, 40, 40, 40, 40 },
michael@0:   {  80, 80, 80, 80, 48, 48, 48, 48 },
michael@0:   {  72, 72, 72, 72, 56, 56, 56, 56 },
michael@0:   {  64, 64, 64, 64, 64, 64, 64, 64 },
michael@0:   {  56, 56, 56, 56, 72, 72, 72, 72 },
michael@0:   {  48, 48, 48, 48, 80, 80, 80, 80 },
michael@0:   {  40, 40, 40, 40, 88, 88, 88, 88 },
michael@0:   {  32, 32, 32, 32, 96, 96, 96, 96 },
michael@0:   {  24, 24, 24, 24, 104, 104, 104, 104 },
michael@0:   {  16, 16, 16, 16, 112, 112, 112, 112 },
michael@0:   {   8,  8,  8,  8, 120, 120, 120, 120 }
michael@0: };
michael@0: 
michael@0: typedef void filter8_1dfunction (
michael@0:   const unsigned char *src_ptr,
michael@0:   const unsigned int src_pitch,
michael@0:   unsigned char *output_ptr,
michael@0:   unsigned int out_pitch,
michael@0:   unsigned int output_height,
michael@0:   const short *filter
michael@0: );
michael@0: 
michael@0: #if HAVE_SSSE3
michael@0: filter8_1dfunction vp9_filter_block1d16_v8_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d16_h8_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d8_v8_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d8_h8_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d4_v8_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d4_h8_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d16_v8_avg_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d16_h8_avg_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d8_v8_avg_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d8_h8_avg_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d4_v8_avg_ssse3;
michael@0: filter8_1dfunction vp9_filter_block1d4_h8_avg_ssse3;
michael@0: 
michael@0: void vp9_convolve8_horiz_ssse3(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                                uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                                const int16_t *filter_x, int x_step_q4,
michael@0:                                const int16_t *filter_y, int y_step_q4,
michael@0:                                int w, int h) {
michael@0:   /* Ensure the filter can be compressed to int16_t. */
michael@0:   if (x_step_q4 == 16 && filter_x[3] != 128) {
michael@0:     while (w >= 16) {
michael@0:       vp9_filter_block1d16_h8_ssse3(src, src_stride,
michael@0:                                     dst, dst_stride,
michael@0:                                     h, filter_x);
michael@0:       src += 16;
michael@0:       dst += 16;
michael@0:       w -= 16;
michael@0:     }
michael@0:     while (w >= 8) {
michael@0:       vp9_filter_block1d8_h8_ssse3(src, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_x);
michael@0:       src += 8;
michael@0:       dst += 8;
michael@0:       w -= 8;
michael@0:     }
michael@0:     while (w >= 4) {
michael@0:       vp9_filter_block1d4_h8_ssse3(src, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_x);
michael@0:       src += 4;
michael@0:       dst += 4;
michael@0:       w -= 4;
michael@0:     }
michael@0:   }
michael@0:   if (w) {
michael@0:     vp9_convolve8_horiz_c(src, src_stride, dst, dst_stride,
michael@0:                           filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                           w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_vert_ssse3(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                               uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                               const int16_t *filter_x, int x_step_q4,
michael@0:                               const int16_t *filter_y, int y_step_q4,
michael@0:                               int w, int h) {
michael@0:   if (y_step_q4 == 16 && filter_y[3] != 128) {
michael@0:     while (w >= 16) {
michael@0:       vp9_filter_block1d16_v8_ssse3(src - src_stride * 3, src_stride,
michael@0:                                     dst, dst_stride,
michael@0:                                     h, filter_y);
michael@0:       src += 16;
michael@0:       dst += 16;
michael@0:       w -= 16;
michael@0:     }
michael@0:     while (w >= 8) {
michael@0:       vp9_filter_block1d8_v8_ssse3(src - src_stride * 3, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_y);
michael@0:       src += 8;
michael@0:       dst += 8;
michael@0:       w -= 8;
michael@0:     }
michael@0:     while (w >= 4) {
michael@0:       vp9_filter_block1d4_v8_ssse3(src - src_stride * 3, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_y);
michael@0:       src += 4;
michael@0:       dst += 4;
michael@0:       w -= 4;
michael@0:     }
michael@0:   }
michael@0:   if (w) {
michael@0:     vp9_convolve8_vert_c(src, src_stride, dst, dst_stride,
michael@0:                          filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                          w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_avg_horiz_ssse3(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                                uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                                const int16_t *filter_x, int x_step_q4,
michael@0:                                const int16_t *filter_y, int y_step_q4,
michael@0:                                int w, int h) {
michael@0:   if (x_step_q4 == 16 && filter_x[3] != 128) {
michael@0:     while (w >= 16) {
michael@0:       vp9_filter_block1d16_h8_avg_ssse3(src, src_stride,
michael@0:                                     dst, dst_stride,
michael@0:                                     h, filter_x);
michael@0:       src += 16;
michael@0:       dst += 16;
michael@0:       w -= 16;
michael@0:     }
michael@0:     while (w >= 8) {
michael@0:       vp9_filter_block1d8_h8_avg_ssse3(src, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_x);
michael@0:       src += 8;
michael@0:       dst += 8;
michael@0:       w -= 8;
michael@0:     }
michael@0:     while (w >= 4) {
michael@0:       vp9_filter_block1d4_h8_avg_ssse3(src, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_x);
michael@0:       src += 4;
michael@0:       dst += 4;
michael@0:       w -= 4;
michael@0:     }
michael@0:   }
michael@0:   if (w) {
michael@0:     vp9_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride,
michael@0:                               filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                               w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_avg_vert_ssse3(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                               uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                               const int16_t *filter_x, int x_step_q4,
michael@0:                               const int16_t *filter_y, int y_step_q4,
michael@0:                               int w, int h) {
michael@0:   if (y_step_q4 == 16 && filter_y[3] != 128) {
michael@0:     while (w >= 16) {
michael@0:       vp9_filter_block1d16_v8_avg_ssse3(src - src_stride * 3, src_stride,
michael@0:                                     dst, dst_stride,
michael@0:                                     h, filter_y);
michael@0:       src += 16;
michael@0:       dst += 16;
michael@0:       w -= 16;
michael@0:     }
michael@0:     while (w >= 8) {
michael@0:       vp9_filter_block1d8_v8_avg_ssse3(src - src_stride * 3, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_y);
michael@0:       src += 8;
michael@0:       dst += 8;
michael@0:       w -= 8;
michael@0:     }
michael@0:     while (w >= 4) {
michael@0:       vp9_filter_block1d4_v8_avg_ssse3(src - src_stride * 3, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_y);
michael@0:       src += 4;
michael@0:       dst += 4;
michael@0:       w -= 4;
michael@0:     }
michael@0:   }
michael@0:   if (w) {
michael@0:     vp9_convolve8_avg_vert_c(src, src_stride, dst, dst_stride,
michael@0:                              filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                              w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_ssse3(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                          uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                          const int16_t *filter_x, int x_step_q4,
michael@0:                          const int16_t *filter_y, int y_step_q4,
michael@0:                          int w, int h) {
michael@0:   DECLARE_ALIGNED_ARRAY(16, unsigned char, fdata2, 64 * 71);
michael@0: 
michael@0:   assert(w <= 64);
michael@0:   assert(h <= 64);
michael@0:   if (x_step_q4 == 16 && y_step_q4 == 16) {
michael@0:     vp9_convolve8_horiz_ssse3(src - 3 * src_stride, src_stride, fdata2, 64,
michael@0:                               filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                               w, h + 7);
michael@0:     vp9_convolve8_vert_ssse3(fdata2 + 3 * 64, 64, dst, dst_stride,
michael@0:                              filter_x, x_step_q4, filter_y, y_step_q4, w, h);
michael@0:   } else {
michael@0:     vp9_convolve8_c(src, src_stride, dst, dst_stride,
michael@0:                     filter_x, x_step_q4, filter_y, y_step_q4, w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_avg_ssse3(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                          uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                          const int16_t *filter_x, int x_step_q4,
michael@0:                          const int16_t *filter_y, int y_step_q4,
michael@0:                          int w, int h) {
michael@0:   DECLARE_ALIGNED_ARRAY(16, unsigned char, fdata2, 64 * 71);
michael@0: 
michael@0:   assert(w <= 64);
michael@0:   assert(h <= 64);
michael@0:   if (x_step_q4 == 16 && y_step_q4 == 16) {
michael@0:     vp9_convolve8_horiz_ssse3(src - 3 * src_stride, src_stride, fdata2, 64,
michael@0:                               filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                               w, h + 7);
michael@0:     vp9_convolve8_avg_vert_ssse3(fdata2 + 3 * 64, 64, dst, dst_stride,
michael@0:                                  filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                                  w, h);
michael@0:   } else {
michael@0:     vp9_convolve8_avg_c(src, src_stride, dst, dst_stride,
michael@0:                         filter_x, x_step_q4, filter_y, y_step_q4, w, h);
michael@0:   }
michael@0: }
michael@0: #endif
michael@0: 
michael@0: #if HAVE_SSE2
michael@0: filter8_1dfunction vp9_filter_block1d16_v8_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d16_h8_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d8_v8_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d8_h8_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d4_v8_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d4_h8_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d16_v8_avg_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d16_h8_avg_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d8_v8_avg_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d8_h8_avg_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d4_v8_avg_sse2;
michael@0: filter8_1dfunction vp9_filter_block1d4_h8_avg_sse2;
michael@0: 
michael@0: void vp9_convolve8_horiz_sse2(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                                uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                                const int16_t *filter_x, int x_step_q4,
michael@0:                                const int16_t *filter_y, int y_step_q4,
michael@0:                                int w, int h) {
michael@0:   /* Ensure the filter can be compressed to int16_t. */
michael@0:   if (x_step_q4 == 16 && filter_x[3] != 128) {
michael@0:     while (w >= 16) {
michael@0:       vp9_filter_block1d16_h8_sse2(src, src_stride,
michael@0:                                     dst, dst_stride,
michael@0:                                     h, filter_x);
michael@0:       src += 16;
michael@0:       dst += 16;
michael@0:       w -= 16;
michael@0:     }
michael@0:     while (w >= 8) {
michael@0:       vp9_filter_block1d8_h8_sse2(src, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_x);
michael@0:       src += 8;
michael@0:       dst += 8;
michael@0:       w -= 8;
michael@0:     }
michael@0:     while (w >= 4) {
michael@0:       vp9_filter_block1d4_h8_sse2(src, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_x);
michael@0:       src += 4;
michael@0:       dst += 4;
michael@0:       w -= 4;
michael@0:     }
michael@0:   }
michael@0:   if (w) {
michael@0:     vp9_convolve8_horiz_c(src, src_stride, dst, dst_stride,
michael@0:                           filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                           w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_vert_sse2(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                               uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                               const int16_t *filter_x, int x_step_q4,
michael@0:                               const int16_t *filter_y, int y_step_q4,
michael@0:                               int w, int h) {
michael@0:   if (y_step_q4 == 16 && filter_y[3] != 128) {
michael@0:     while (w >= 16) {
michael@0:       vp9_filter_block1d16_v8_sse2(src - src_stride * 3, src_stride,
michael@0:                                     dst, dst_stride,
michael@0:                                     h, filter_y);
michael@0:       src += 16;
michael@0:       dst += 16;
michael@0:       w -= 16;
michael@0:     }
michael@0:     while (w >= 8) {
michael@0:       vp9_filter_block1d8_v8_sse2(src - src_stride * 3, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_y);
michael@0:       src += 8;
michael@0:       dst += 8;
michael@0:       w -= 8;
michael@0:     }
michael@0:     while (w >= 4) {
michael@0:       vp9_filter_block1d4_v8_sse2(src - src_stride * 3, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_y);
michael@0:       src += 4;
michael@0:       dst += 4;
michael@0:       w -= 4;
michael@0:     }
michael@0:   }
michael@0:   if (w) {
michael@0:     vp9_convolve8_vert_c(src, src_stride, dst, dst_stride,
michael@0:                          filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                          w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_avg_horiz_sse2(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                                uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                                const int16_t *filter_x, int x_step_q4,
michael@0:                                const int16_t *filter_y, int y_step_q4,
michael@0:                                int w, int h) {
michael@0:   if (x_step_q4 == 16 && filter_x[3] != 128) {
michael@0:     while (w >= 16) {
michael@0:       vp9_filter_block1d16_h8_avg_sse2(src, src_stride,
michael@0:                                     dst, dst_stride,
michael@0:                                     h, filter_x);
michael@0:       src += 16;
michael@0:       dst += 16;
michael@0:       w -= 16;
michael@0:     }
michael@0:     while (w >= 8) {
michael@0:       vp9_filter_block1d8_h8_avg_sse2(src, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_x);
michael@0:       src += 8;
michael@0:       dst += 8;
michael@0:       w -= 8;
michael@0:     }
michael@0:     while (w >= 4) {
michael@0:       vp9_filter_block1d4_h8_avg_sse2(src, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_x);
michael@0:       src += 4;
michael@0:       dst += 4;
michael@0:       w -= 4;
michael@0:     }
michael@0:   }
michael@0:   if (w) {
michael@0:     vp9_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride,
michael@0:                               filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                               w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_avg_vert_sse2(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                               uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                               const int16_t *filter_x, int x_step_q4,
michael@0:                               const int16_t *filter_y, int y_step_q4,
michael@0:                               int w, int h) {
michael@0:   if (y_step_q4 == 16 && filter_y[3] != 128) {
michael@0:     while (w >= 16) {
michael@0:       vp9_filter_block1d16_v8_avg_sse2(src - src_stride * 3, src_stride,
michael@0:                                     dst, dst_stride,
michael@0:                                     h, filter_y);
michael@0:       src += 16;
michael@0:       dst += 16;
michael@0:       w -= 16;
michael@0:     }
michael@0:     while (w >= 8) {
michael@0:       vp9_filter_block1d8_v8_avg_sse2(src - src_stride * 3, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_y);
michael@0:       src += 8;
michael@0:       dst += 8;
michael@0:       w -= 8;
michael@0:     }
michael@0:     while (w >= 4) {
michael@0:       vp9_filter_block1d4_v8_avg_sse2(src - src_stride * 3, src_stride,
michael@0:                                    dst, dst_stride,
michael@0:                                    h, filter_y);
michael@0:       src += 4;
michael@0:       dst += 4;
michael@0:       w -= 4;
michael@0:     }
michael@0:   }
michael@0:   if (w) {
michael@0:     vp9_convolve8_avg_vert_c(src, src_stride, dst, dst_stride,
michael@0:                              filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                              w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_sse2(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                          uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                          const int16_t *filter_x, int x_step_q4,
michael@0:                          const int16_t *filter_y, int y_step_q4,
michael@0:                          int w, int h) {
michael@0:   DECLARE_ALIGNED_ARRAY(16, unsigned char, fdata2, 64 * 71);
michael@0: 
michael@0:   assert(w <= 64);
michael@0:   assert(h <= 64);
michael@0:   if (x_step_q4 == 16 && y_step_q4 == 16) {
michael@0:     vp9_convolve8_horiz_sse2(src - 3 * src_stride, src_stride, fdata2, 64,
michael@0:                               filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                               w, h + 7);
michael@0:     vp9_convolve8_vert_sse2(fdata2 + 3 * 64, 64, dst, dst_stride,
michael@0:                              filter_x, x_step_q4, filter_y, y_step_q4, w, h);
michael@0:   } else {
michael@0:     vp9_convolve8_c(src, src_stride, dst, dst_stride,
michael@0:                     filter_x, x_step_q4, filter_y, y_step_q4, w, h);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void vp9_convolve8_avg_sse2(const uint8_t *src, ptrdiff_t src_stride,
michael@0:                          uint8_t *dst, ptrdiff_t dst_stride,
michael@0:                          const int16_t *filter_x, int x_step_q4,
michael@0:                          const int16_t *filter_y, int y_step_q4,
michael@0:                          int w, int h) {
michael@0:   DECLARE_ALIGNED_ARRAY(16, unsigned char, fdata2, 64 * 71);
michael@0: 
michael@0:   assert(w <= 64);
michael@0:   assert(h <= 64);
michael@0:   if (x_step_q4 == 16 && y_step_q4 == 16) {
michael@0:     vp9_convolve8_horiz_sse2(src - 3 * src_stride, src_stride, fdata2, 64,
michael@0:                               filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                               w, h + 7);
michael@0:     vp9_convolve8_avg_vert_sse2(fdata2 + 3 * 64, 64, dst, dst_stride,
michael@0:                                  filter_x, x_step_q4, filter_y, y_step_q4,
michael@0:                                  w, h);
michael@0:   } else {
michael@0:     vp9_convolve8_avg_c(src, src_stride, dst, dst_stride,
michael@0:                         filter_x, x_step_q4, filter_y, y_step_q4, w, h);
michael@0:   }
michael@0: }
michael@0: #endif