michael@0: /*
michael@0:  *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
michael@0:  *
michael@0:  *  Use of this source code is governed by a BSD-style license
michael@0:  *  that can be found in the LICENSE file in the root of the source
michael@0:  *  tree. An additional intellectual property rights grant can be found
michael@0:  *  in the file PATENTS.  All contributing project authors may
michael@0:  *  be found in the AUTHORS file in the root of the source tree.
michael@0:  */
michael@0: 
michael@0: 
michael@0: #include <limits.h>
michael@0: #include "vpx_config.h"
michael@0: #include "onyx_int.h"
michael@0: #include "modecosts.h"
michael@0: #include "encodeintra.h"
michael@0: #include "vp8/common/entropymode.h"
michael@0: #include "pickinter.h"
michael@0: #include "vp8/common/findnearmv.h"
michael@0: #include "encodemb.h"
michael@0: #include "vp8/common/reconinter.h"
michael@0: #include "vp8/common/reconintra4x4.h"
michael@0: #include "vp8/common/variance.h"
michael@0: #include "mcomp.h"
michael@0: #include "rdopt.h"
michael@0: #include "vpx_mem/vpx_mem.h"
michael@0: #if CONFIG_TEMPORAL_DENOISING
michael@0: #include "denoising.h"
michael@0: #endif
michael@0: 
michael@0: extern int VP8_UVSSE(MACROBLOCK *x);
michael@0: 
michael@0: #ifdef SPEEDSTATS
michael@0: extern unsigned int cnt_pm;
michael@0: #endif
michael@0: 
michael@0: extern const int vp8_ref_frame_order[MAX_MODES];
michael@0: extern const MB_PREDICTION_MODE vp8_mode_order[MAX_MODES];
michael@0: 
michael@0: extern int vp8_cost_mv_ref(MB_PREDICTION_MODE m, const int near_mv_ref_ct[4]);
michael@0: 
michael@0: 
michael@0: int vp8_skip_fractional_mv_step(MACROBLOCK *mb, BLOCK *b, BLOCKD *d,
michael@0:                                 int_mv *bestmv, int_mv *ref_mv,
michael@0:                                 int error_per_bit,
michael@0:                                 const vp8_variance_fn_ptr_t *vfp,
michael@0:                                 int *mvcost[2], int *distortion,
michael@0:                                 unsigned int *sse)
michael@0: {
michael@0:     (void) b;
michael@0:     (void) d;
michael@0:     (void) ref_mv;
michael@0:     (void) error_per_bit;
michael@0:     (void) vfp;
michael@0:     (void) mvcost;
michael@0:     (void) distortion;
michael@0:     (void) sse;
michael@0:     bestmv->as_mv.row <<= 3;
michael@0:     bestmv->as_mv.col <<= 3;
michael@0:     return 0;
michael@0: }
michael@0: 
michael@0: 
michael@0: int vp8_get_inter_mbpred_error(MACROBLOCK *mb,
michael@0:                                   const vp8_variance_fn_ptr_t *vfp,
michael@0:                                   unsigned int *sse,
michael@0:                                   int_mv this_mv)
michael@0: {
michael@0: 
michael@0:     BLOCK *b = &mb->block[0];
michael@0:     BLOCKD *d = &mb->e_mbd.block[0];
michael@0:     unsigned char *what = (*(b->base_src) + b->src);
michael@0:     int what_stride = b->src_stride;
michael@0:     int pre_stride = mb->e_mbd.pre.y_stride;
michael@0:     unsigned char *in_what = mb->e_mbd.pre.y_buffer + d->offset ;
michael@0:     int in_what_stride = pre_stride;
michael@0:     int xoffset = this_mv.as_mv.col & 7;
michael@0:     int yoffset = this_mv.as_mv.row & 7;
michael@0: 
michael@0:     in_what += (this_mv.as_mv.row >> 3) * pre_stride + (this_mv.as_mv.col >> 3);
michael@0: 
michael@0:     if (xoffset | yoffset)
michael@0:     {
michael@0:         return vfp->svf(in_what, in_what_stride, xoffset, yoffset, what, what_stride, sse);
michael@0:     }
michael@0:     else
michael@0:     {
michael@0:         return vfp->vf(what, what_stride, in_what, in_what_stride, sse);
michael@0:     }
michael@0: 
michael@0: }
michael@0: 
michael@0: 
michael@0: unsigned int vp8_get4x4sse_cs_c
michael@0: (
michael@0:     const unsigned char *src_ptr,
michael@0:     int  source_stride,
michael@0:     const unsigned char *ref_ptr,
michael@0:     int  recon_stride
michael@0: )
michael@0: {
michael@0:     int distortion = 0;
michael@0:     int r, c;
michael@0: 
michael@0:     for (r = 0; r < 4; r++)
michael@0:     {
michael@0:         for (c = 0; c < 4; c++)
michael@0:         {
michael@0:             int diff = src_ptr[c] - ref_ptr[c];
michael@0:             distortion += diff * diff;
michael@0:         }
michael@0: 
michael@0:         src_ptr += source_stride;
michael@0:         ref_ptr += recon_stride;
michael@0:     }
michael@0: 
michael@0:     return distortion;
michael@0: }
michael@0: 
michael@0: static int get_prediction_error(BLOCK *be, BLOCKD *b)
michael@0: {
michael@0:     unsigned char *sptr;
michael@0:     unsigned char *dptr;
michael@0:     sptr = (*(be->base_src) + be->src);
michael@0:     dptr = b->predictor;
michael@0: 
michael@0:     return vp8_get4x4sse_cs(sptr, be->src_stride, dptr, 16);
michael@0: 
michael@0: }
michael@0: 
michael@0: static int pick_intra4x4block(
michael@0:     MACROBLOCK *x,
michael@0:     int ib,
michael@0:     B_PREDICTION_MODE *best_mode,
michael@0:     const int *mode_costs,
michael@0: 
michael@0:     int *bestrate,
michael@0:     int *bestdistortion)
michael@0: {
michael@0: 
michael@0:     BLOCKD *b = &x->e_mbd.block[ib];
michael@0:     BLOCK *be = &x->block[ib];
michael@0:     int dst_stride = x->e_mbd.dst.y_stride;
michael@0:     unsigned char *dst = x->e_mbd.dst.y_buffer + b->offset;
michael@0:     B_PREDICTION_MODE mode;
michael@0:     int best_rd = INT_MAX;
michael@0:     int rate;
michael@0:     int distortion;
michael@0: 
michael@0:     unsigned char *Above = dst - dst_stride;
michael@0:     unsigned char *yleft = dst - 1;
michael@0:     unsigned char top_left = Above[-1];
michael@0: 
michael@0:     for (mode = B_DC_PRED; mode <= B_HE_PRED; mode++)
michael@0:     {
michael@0:         int this_rd;
michael@0: 
michael@0:         rate = mode_costs[mode];
michael@0: 
michael@0:         vp8_intra4x4_predict(Above, yleft, dst_stride, mode,
michael@0:                              b->predictor, 16, top_left);
michael@0:         distortion = get_prediction_error(be, b);
michael@0:         this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
michael@0: 
michael@0:         if (this_rd < best_rd)
michael@0:         {
michael@0:             *bestrate = rate;
michael@0:             *bestdistortion = distortion;
michael@0:             best_rd = this_rd;
michael@0:             *best_mode = mode;
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     b->bmi.as_mode = *best_mode;
michael@0:     vp8_encode_intra4x4block(x, ib);
michael@0:     return best_rd;
michael@0: }
michael@0: 
michael@0: 
michael@0: static int pick_intra4x4mby_modes
michael@0: (
michael@0:     MACROBLOCK *mb,
michael@0:     int *Rate,
michael@0:     int *best_dist
michael@0: )
michael@0: {
michael@0:     MACROBLOCKD *const xd = &mb->e_mbd;
michael@0:     int i;
michael@0:     int cost = mb->mbmode_cost [xd->frame_type] [B_PRED];
michael@0:     int error;
michael@0:     int distortion = 0;
michael@0:     const int *bmode_costs;
michael@0: 
michael@0:     intra_prediction_down_copy(xd, xd->dst.y_buffer - xd->dst.y_stride + 16);
michael@0: 
michael@0:     bmode_costs = mb->inter_bmode_costs;
michael@0: 
michael@0:     for (i = 0; i < 16; i++)
michael@0:     {
michael@0:         MODE_INFO *const mic = xd->mode_info_context;
michael@0:         const int mis = xd->mode_info_stride;
michael@0: 
michael@0:         B_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
michael@0:         int UNINITIALIZED_IS_SAFE(r), UNINITIALIZED_IS_SAFE(d);
michael@0: 
michael@0:         if (mb->e_mbd.frame_type == KEY_FRAME)
michael@0:         {
michael@0:             const B_PREDICTION_MODE A = above_block_mode(mic, i, mis);
michael@0:             const B_PREDICTION_MODE L = left_block_mode(mic, i);
michael@0: 
michael@0:             bmode_costs  = mb->bmode_costs[A][L];
michael@0:         }
michael@0: 
michael@0: 
michael@0:         pick_intra4x4block(mb, i, &best_mode, bmode_costs, &r, &d);
michael@0: 
michael@0:         cost += r;
michael@0:         distortion += d;
michael@0:         mic->bmi[i].as_mode = best_mode;
michael@0: 
michael@0:         /* Break out case where we have already exceeded best so far value
michael@0:          * that was passed in
michael@0:          */
michael@0:         if (distortion > *best_dist)
michael@0:             break;
michael@0:     }
michael@0: 
michael@0:     *Rate = cost;
michael@0: 
michael@0:     if (i == 16)
michael@0:     {
michael@0:         *best_dist = distortion;
michael@0:         error = RDCOST(mb->rdmult, mb->rddiv, cost, distortion);
michael@0:     }
michael@0:     else
michael@0:     {
michael@0:         *best_dist = INT_MAX;
michael@0:         error = INT_MAX;
michael@0:     }
michael@0: 
michael@0:     return error;
michael@0: }
michael@0: 
michael@0: static void pick_intra_mbuv_mode(MACROBLOCK *mb)
michael@0: {
michael@0: 
michael@0:     MACROBLOCKD *x = &mb->e_mbd;
michael@0:     unsigned char *uabove_row = x->dst.u_buffer - x->dst.uv_stride;
michael@0:     unsigned char *vabove_row = x->dst.v_buffer - x->dst.uv_stride;
michael@0:     unsigned char *usrc_ptr = (mb->block[16].src + *mb->block[16].base_src);
michael@0:     unsigned char *vsrc_ptr = (mb->block[20].src + *mb->block[20].base_src);
michael@0:     int uvsrc_stride = mb->block[16].src_stride;
michael@0:     unsigned char uleft_col[8];
michael@0:     unsigned char vleft_col[8];
michael@0:     unsigned char utop_left = uabove_row[-1];
michael@0:     unsigned char vtop_left = vabove_row[-1];
michael@0:     int i, j;
michael@0:     int expected_udc;
michael@0:     int expected_vdc;
michael@0:     int shift;
michael@0:     int Uaverage = 0;
michael@0:     int Vaverage = 0;
michael@0:     int diff;
michael@0:     int pred_error[4] = {0, 0, 0, 0}, best_error = INT_MAX;
michael@0:     MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
michael@0: 
michael@0: 
michael@0:     for (i = 0; i < 8; i++)
michael@0:     {
michael@0:         uleft_col[i] = x->dst.u_buffer [i* x->dst.uv_stride -1];
michael@0:         vleft_col[i] = x->dst.v_buffer [i* x->dst.uv_stride -1];
michael@0:     }
michael@0: 
michael@0:     if (!x->up_available && !x->left_available)
michael@0:     {
michael@0:         expected_udc = 128;
michael@0:         expected_vdc = 128;
michael@0:     }
michael@0:     else
michael@0:     {
michael@0:         shift = 2;
michael@0: 
michael@0:         if (x->up_available)
michael@0:         {
michael@0: 
michael@0:             for (i = 0; i < 8; i++)
michael@0:             {
michael@0:                 Uaverage += uabove_row[i];
michael@0:                 Vaverage += vabove_row[i];
michael@0:             }
michael@0: 
michael@0:             shift ++;
michael@0: 
michael@0:         }
michael@0: 
michael@0:         if (x->left_available)
michael@0:         {
michael@0:             for (i = 0; i < 8; i++)
michael@0:             {
michael@0:                 Uaverage += uleft_col[i];
michael@0:                 Vaverage += vleft_col[i];
michael@0:             }
michael@0: 
michael@0:             shift ++;
michael@0: 
michael@0:         }
michael@0: 
michael@0:         expected_udc = (Uaverage + (1 << (shift - 1))) >> shift;
michael@0:         expected_vdc = (Vaverage + (1 << (shift - 1))) >> shift;
michael@0:     }
michael@0: 
michael@0: 
michael@0:     for (i = 0; i < 8; i++)
michael@0:     {
michael@0:         for (j = 0; j < 8; j++)
michael@0:         {
michael@0: 
michael@0:             int predu = uleft_col[i] + uabove_row[j] - utop_left;
michael@0:             int predv = vleft_col[i] + vabove_row[j] - vtop_left;
michael@0:             int u_p, v_p;
michael@0: 
michael@0:             u_p = usrc_ptr[j];
michael@0:             v_p = vsrc_ptr[j];
michael@0: 
michael@0:             if (predu < 0)
michael@0:                 predu = 0;
michael@0: 
michael@0:             if (predu > 255)
michael@0:                 predu = 255;
michael@0: 
michael@0:             if (predv < 0)
michael@0:                 predv = 0;
michael@0: 
michael@0:             if (predv > 255)
michael@0:                 predv = 255;
michael@0: 
michael@0: 
michael@0:             diff = u_p - expected_udc;
michael@0:             pred_error[DC_PRED] += diff * diff;
michael@0:             diff = v_p - expected_vdc;
michael@0:             pred_error[DC_PRED] += diff * diff;
michael@0: 
michael@0: 
michael@0:             diff = u_p - uabove_row[j];
michael@0:             pred_error[V_PRED] += diff * diff;
michael@0:             diff = v_p - vabove_row[j];
michael@0:             pred_error[V_PRED] += diff * diff;
michael@0: 
michael@0: 
michael@0:             diff = u_p - uleft_col[i];
michael@0:             pred_error[H_PRED] += diff * diff;
michael@0:             diff = v_p - vleft_col[i];
michael@0:             pred_error[H_PRED] += diff * diff;
michael@0: 
michael@0: 
michael@0:             diff = u_p - predu;
michael@0:             pred_error[TM_PRED] += diff * diff;
michael@0:             diff = v_p - predv;
michael@0:             pred_error[TM_PRED] += diff * diff;
michael@0: 
michael@0: 
michael@0:         }
michael@0: 
michael@0:         usrc_ptr += uvsrc_stride;
michael@0:         vsrc_ptr += uvsrc_stride;
michael@0: 
michael@0:         if (i == 3)
michael@0:         {
michael@0:             usrc_ptr = (mb->block[18].src + *mb->block[18].base_src);
michael@0:             vsrc_ptr = (mb->block[22].src + *mb->block[22].base_src);
michael@0:         }
michael@0: 
michael@0: 
michael@0: 
michael@0:     }
michael@0: 
michael@0: 
michael@0:     for (i = DC_PRED; i <= TM_PRED; i++)
michael@0:     {
michael@0:         if (best_error > pred_error[i])
michael@0:         {
michael@0:             best_error = pred_error[i];
michael@0:             best_mode = (MB_PREDICTION_MODE)i;
michael@0:         }
michael@0:     }
michael@0: 
michael@0: 
michael@0:     mb->e_mbd.mode_info_context->mbmi.uv_mode = best_mode;
michael@0: 
michael@0: }
michael@0: 
michael@0: static void update_mvcount(MACROBLOCK *x, int_mv *best_ref_mv)
michael@0: {
michael@0:     MACROBLOCKD *xd = &x->e_mbd;
michael@0:     /* Split MV modes currently not supported when RD is nopt enabled,
michael@0:      * therefore, only need to modify MVcount in NEWMV mode. */
michael@0:     if (xd->mode_info_context->mbmi.mode == NEWMV)
michael@0:     {
michael@0:         x->MVcount[0][mv_max+((xd->mode_info_context->mbmi.mv.as_mv.row -
michael@0:                                       best_ref_mv->as_mv.row) >> 1)]++;
michael@0:         x->MVcount[1][mv_max+((xd->mode_info_context->mbmi.mv.as_mv.col -
michael@0:                                       best_ref_mv->as_mv.col) >> 1)]++;
michael@0:     }
michael@0: }
michael@0: 
michael@0: 
michael@0: #if CONFIG_MULTI_RES_ENCODING
michael@0: static
michael@0: void get_lower_res_motion_info(VP8_COMP *cpi, MACROBLOCKD *xd, int *dissim,
michael@0:                                int *parent_ref_frame,
michael@0:                                MB_PREDICTION_MODE *parent_mode,
michael@0:                                int_mv *parent_ref_mv, int mb_row, int mb_col)
michael@0: {
michael@0:     LOWER_RES_MB_INFO* store_mode_info
michael@0:                           = ((LOWER_RES_FRAME_INFO*)cpi->oxcf.mr_low_res_mode_info)->mb_info;
michael@0:     unsigned int parent_mb_index;
michael@0: 
michael@0:     /* Consider different down_sampling_factor.  */
michael@0:     {
michael@0:         /* TODO: Removed the loop that supports special down_sampling_factor
michael@0:          * such as 2, 4, 8. Will revisit it if needed.
michael@0:          * Should also try using a look-up table to see if it helps
michael@0:          * performance. */
michael@0:         int parent_mb_row, parent_mb_col;
michael@0: 
michael@0:         parent_mb_row = mb_row*cpi->oxcf.mr_down_sampling_factor.den
michael@0:                     /cpi->oxcf.mr_down_sampling_factor.num;
michael@0:         parent_mb_col = mb_col*cpi->oxcf.mr_down_sampling_factor.den
michael@0:                     /cpi->oxcf.mr_down_sampling_factor.num;
michael@0:         parent_mb_index = parent_mb_row*cpi->mr_low_res_mb_cols + parent_mb_col;
michael@0:     }
michael@0: 
michael@0:     /* Read lower-resolution mode & motion result from memory.*/
michael@0:     *parent_ref_frame = store_mode_info[parent_mb_index].ref_frame;
michael@0:     *parent_mode =  store_mode_info[parent_mb_index].mode;
michael@0:     *dissim = store_mode_info[parent_mb_index].dissim;
michael@0: 
michael@0:     /* For highest-resolution encoder, adjust dissim value. Lower its quality
michael@0:      * for good performance. */
michael@0:     if (cpi->oxcf.mr_encoder_id == (cpi->oxcf.mr_total_resolutions - 1))
michael@0:         *dissim>>=1;
michael@0: 
michael@0:     if(*parent_ref_frame != INTRA_FRAME)
michael@0:     {
michael@0:         /* Consider different down_sampling_factor.
michael@0:          * The result can be rounded to be more precise, but it takes more time.
michael@0:          */
michael@0:         (*parent_ref_mv).as_mv.row = store_mode_info[parent_mb_index].mv.as_mv.row
michael@0:                                   *cpi->oxcf.mr_down_sampling_factor.num
michael@0:                                   /cpi->oxcf.mr_down_sampling_factor.den;
michael@0:         (*parent_ref_mv).as_mv.col = store_mode_info[parent_mb_index].mv.as_mv.col
michael@0:                                   *cpi->oxcf.mr_down_sampling_factor.num
michael@0:                                   /cpi->oxcf.mr_down_sampling_factor.den;
michael@0: 
michael@0:         vp8_clamp_mv2(parent_ref_mv, xd);
michael@0:     }
michael@0: }
michael@0: #endif
michael@0: 
michael@0: static void check_for_encode_breakout(unsigned int sse, MACROBLOCK* x)
michael@0: {
michael@0:     MACROBLOCKD *xd = &x->e_mbd;
michael@0: 
michael@0:     unsigned int threshold = (xd->block[0].dequant[1]
michael@0:         * xd->block[0].dequant[1] >>4);
michael@0: 
michael@0:     if(threshold < x->encode_breakout)
michael@0:         threshold = x->encode_breakout;
michael@0: 
michael@0:     if (sse < threshold )
michael@0:     {
michael@0:         /* Check u and v to make sure skip is ok */
michael@0:         unsigned int sse2 = 0;
michael@0: 
michael@0:         sse2 = VP8_UVSSE(x);
michael@0: 
michael@0:         if (sse2 * 2 < x->encode_breakout)
michael@0:             x->skip = 1;
michael@0:         else
michael@0:             x->skip = 0;
michael@0:     }
michael@0: }
michael@0: 
michael@0: static int evaluate_inter_mode(unsigned int* sse, int rate2, int* distortion2,
michael@0:                                VP8_COMP *cpi, MACROBLOCK *x, int rd_adj)
michael@0: {
michael@0:     MB_PREDICTION_MODE this_mode = x->e_mbd.mode_info_context->mbmi.mode;
michael@0:     int_mv mv = x->e_mbd.mode_info_context->mbmi.mv;
michael@0:     int this_rd;
michael@0:     /* Exit early and don't compute the distortion if this macroblock
michael@0:      * is marked inactive. */
michael@0:     if (cpi->active_map_enabled && x->active_ptr[0] == 0)
michael@0:     {
michael@0:         *sse = 0;
michael@0:         *distortion2 = 0;
michael@0:         x->skip = 1;
michael@0:         return INT_MAX;
michael@0:     }
michael@0: 
michael@0:     if((this_mode != NEWMV) ||
michael@0:         !(cpi->sf.half_pixel_search) || cpi->common.full_pixel==1)
michael@0:         *distortion2 = vp8_get_inter_mbpred_error(x,
michael@0:                                               &cpi->fn_ptr[BLOCK_16X16],
michael@0:                                               sse, mv);
michael@0: 
michael@0:     this_rd = RDCOST(x->rdmult, x->rddiv, rate2, *distortion2);
michael@0: 
michael@0:     /* Adjust rd to bias to ZEROMV */
michael@0:     if(this_mode == ZEROMV)
michael@0:     {
michael@0:         /* Bias to ZEROMV on LAST_FRAME reference when it is available. */
michael@0:         if ((cpi->ref_frame_flags & VP8_LAST_FRAME &
michael@0:             cpi->common.refresh_last_frame)
michael@0:             && x->e_mbd.mode_info_context->mbmi.ref_frame != LAST_FRAME)
michael@0:             rd_adj = 100;
michael@0: 
michael@0:         // rd_adj <= 100
michael@0:         this_rd = ((int64_t)this_rd) * rd_adj / 100;
michael@0:     }
michael@0: 
michael@0:     check_for_encode_breakout(*sse, x);
michael@0:     return this_rd;
michael@0: }
michael@0: 
michael@0: static void calculate_zeromv_rd_adjustment(VP8_COMP *cpi, MACROBLOCK *x,
michael@0:                                     int *rd_adjustment)
michael@0: {
michael@0:     MODE_INFO *mic = x->e_mbd.mode_info_context;
michael@0:     int_mv mv_l, mv_a, mv_al;
michael@0:     int local_motion_check = 0;
michael@0: 
michael@0:     if (cpi->lf_zeromv_pct > 40)
michael@0:     {
michael@0:         /* left mb */
michael@0:         mic -= 1;
michael@0:         mv_l = mic->mbmi.mv;
michael@0: 
michael@0:         if (mic->mbmi.ref_frame != INTRA_FRAME)
michael@0:             if( abs(mv_l.as_mv.row) < 8 && abs(mv_l.as_mv.col) < 8)
michael@0:                 local_motion_check++;
michael@0: 
michael@0:         /* above-left mb */
michael@0:         mic -= x->e_mbd.mode_info_stride;
michael@0:         mv_al = mic->mbmi.mv;
michael@0: 
michael@0:         if (mic->mbmi.ref_frame != INTRA_FRAME)
michael@0:             if( abs(mv_al.as_mv.row) < 8 && abs(mv_al.as_mv.col) < 8)
michael@0:                 local_motion_check++;
michael@0: 
michael@0:         /* above mb */
michael@0:         mic += 1;
michael@0:         mv_a = mic->mbmi.mv;
michael@0: 
michael@0:         if (mic->mbmi.ref_frame != INTRA_FRAME)
michael@0:             if( abs(mv_a.as_mv.row) < 8 && abs(mv_a.as_mv.col) < 8)
michael@0:                 local_motion_check++;
michael@0: 
michael@0:         if (((!x->e_mbd.mb_to_top_edge || !x->e_mbd.mb_to_left_edge)
michael@0:             && local_motion_check >0) ||  local_motion_check >2 )
michael@0:             *rd_adjustment = 80;
michael@0:         else if (local_motion_check > 0)
michael@0:             *rd_adjustment = 90;
michael@0:     }
michael@0: }
michael@0: 
michael@0: void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset,
michael@0:                          int recon_uvoffset, int *returnrate,
michael@0:                          int *returndistortion, int *returnintra, int mb_row,
michael@0:                          int mb_col)
michael@0: {
michael@0:     BLOCK *b = &x->block[0];
michael@0:     BLOCKD *d = &x->e_mbd.block[0];
michael@0:     MACROBLOCKD *xd = &x->e_mbd;
michael@0:     MB_MODE_INFO best_mbmode;
michael@0: 
michael@0:     int_mv best_ref_mv_sb[2];
michael@0:     int_mv mode_mv_sb[2][MB_MODE_COUNT];
michael@0:     int_mv best_ref_mv;
michael@0:     int_mv *mode_mv;
michael@0:     MB_PREDICTION_MODE this_mode;
michael@0:     int num00;
michael@0:     int mdcounts[4];
michael@0:     int best_rd = INT_MAX;
michael@0:     int rd_adjustment = 100;
michael@0:     int best_intra_rd = INT_MAX;
michael@0:     int mode_index;
michael@0:     int rate;
michael@0:     int rate2;
michael@0:     int distortion2;
michael@0:     int bestsme = INT_MAX;
michael@0:     int best_mode_index = 0;
michael@0:     unsigned int sse = INT_MAX, best_rd_sse = INT_MAX;
michael@0: #if CONFIG_TEMPORAL_DENOISING
michael@0:     unsigned int zero_mv_sse = INT_MAX, best_sse = INT_MAX;
michael@0: #endif
michael@0: 
michael@0:     int sf_improved_mv_pred = cpi->sf.improved_mv_pred;
michael@0:     int_mv mvp;
michael@0: 
michael@0:     int near_sadidx[8] = {0, 1, 2, 3, 4, 5, 6, 7};
michael@0:     int saddone=0;
michael@0:     /* search range got from mv_pred(). It uses step_param levels. (0-7) */
michael@0:     int sr=0;
michael@0: 
michael@0:     unsigned char *plane[4][3];
michael@0:     int ref_frame_map[4];
michael@0:     int sign_bias = 0;
michael@0: 
michael@0: #if CONFIG_MULTI_RES_ENCODING
michael@0:     int dissim = INT_MAX;
michael@0:     int parent_ref_frame = 0;
michael@0:     int parent_ref_valid = cpi->oxcf.mr_encoder_id && cpi->mr_low_res_mv_avail;
michael@0:     int_mv parent_ref_mv;
michael@0:     MB_PREDICTION_MODE parent_mode = 0;
michael@0: 
michael@0:     if (parent_ref_valid)
michael@0:     {
michael@0:         int parent_ref_flag;
michael@0: 
michael@0:         get_lower_res_motion_info(cpi, xd, &dissim, &parent_ref_frame,
michael@0:                                   &parent_mode, &parent_ref_mv, mb_row, mb_col);
michael@0: 
michael@0:         /* TODO(jkoleszar): The references available (ref_frame_flags) to the
michael@0:          * lower res encoder should match those available to this encoder, but
michael@0:          * there seems to be a situation where this mismatch can happen in the
michael@0:          * case of frame dropping and temporal layers. For example,
michael@0:          * GOLD being disallowed in ref_frame_flags, but being returned as
michael@0:          * parent_ref_frame.
michael@0:          *
michael@0:          * In this event, take the conservative approach of disabling the
michael@0:          * lower res info for this MB.
michael@0:          */
michael@0:         parent_ref_flag = 0;
michael@0:         if (parent_ref_frame == LAST_FRAME)
michael@0:             parent_ref_flag = (cpi->ref_frame_flags & VP8_LAST_FRAME);
michael@0:         else if (parent_ref_frame == GOLDEN_FRAME)
michael@0:             parent_ref_flag = (cpi->ref_frame_flags & VP8_GOLD_FRAME);
michael@0:         else if (parent_ref_frame == ALTREF_FRAME)
michael@0:             parent_ref_flag = (cpi->ref_frame_flags & VP8_ALTR_FRAME);
michael@0: 
michael@0:         //assert(!parent_ref_frame || parent_ref_flag);
michael@0:         if (parent_ref_frame && !parent_ref_flag)
michael@0:             parent_ref_valid = 0;
michael@0:     }
michael@0: #endif
michael@0: 
michael@0:     mode_mv = mode_mv_sb[sign_bias];
michael@0:     best_ref_mv.as_int = 0;
michael@0:     vpx_memset(mode_mv_sb, 0, sizeof(mode_mv_sb));
michael@0:     vpx_memset(&best_mbmode, 0, sizeof(best_mbmode));
michael@0: 
michael@0:     /* Setup search priorities */
michael@0: #if CONFIG_MULTI_RES_ENCODING
michael@0:     if (parent_ref_valid && parent_ref_frame && dissim < 8)
michael@0:     {
michael@0:         ref_frame_map[0] = -1;
michael@0:         ref_frame_map[1] = parent_ref_frame;
michael@0:         ref_frame_map[2] = -1;
michael@0:         ref_frame_map[3] = -1;
michael@0:     } else
michael@0: #endif
michael@0:     get_reference_search_order(cpi, ref_frame_map);
michael@0: 
michael@0:     /* Check to see if there is at least 1 valid reference frame that we need
michael@0:      * to calculate near_mvs.
michael@0:      */
michael@0:     if (ref_frame_map[1] > 0)
michael@0:     {
michael@0:         sign_bias = vp8_find_near_mvs_bias(&x->e_mbd,
michael@0:                                            x->e_mbd.mode_info_context,
michael@0:                                            mode_mv_sb,
michael@0:                                            best_ref_mv_sb,
michael@0:                                            mdcounts,
michael@0:                                            ref_frame_map[1],
michael@0:                                            cpi->common.ref_frame_sign_bias);
michael@0: 
michael@0:         mode_mv = mode_mv_sb[sign_bias];
michael@0:         best_ref_mv.as_int = best_ref_mv_sb[sign_bias].as_int;
michael@0:     }
michael@0: 
michael@0:     get_predictor_pointers(cpi, plane, recon_yoffset, recon_uvoffset);
michael@0: 
michael@0:     /* Count of the number of MBs tested so far this frame */
michael@0:     x->mbs_tested_so_far++;
michael@0: 
michael@0:     *returnintra = INT_MAX;
michael@0:     x->skip = 0;
michael@0: 
michael@0:     x->e_mbd.mode_info_context->mbmi.ref_frame = INTRA_FRAME;
michael@0: 
michael@0:     /* If the frame has big static background and current MB is in low
michael@0:      * motion area, its mode decision is biased to ZEROMV mode.
michael@0:      */
michael@0:     calculate_zeromv_rd_adjustment(cpi, x, &rd_adjustment);
michael@0: 
michael@0:     /* if we encode a new mv this is important
michael@0:      * find the best new motion vector
michael@0:      */
michael@0:     for (mode_index = 0; mode_index < MAX_MODES; mode_index++)
michael@0:     {
michael@0:         int frame_cost;
michael@0:         int this_rd = INT_MAX;
michael@0:         int this_ref_frame = ref_frame_map[vp8_ref_frame_order[mode_index]];
michael@0: 
michael@0:         if (best_rd <= x->rd_threshes[mode_index])
michael@0:             continue;
michael@0: 
michael@0:         if (this_ref_frame < 0)
michael@0:             continue;
michael@0: 
michael@0:         x->e_mbd.mode_info_context->mbmi.ref_frame = this_ref_frame;
michael@0: 
michael@0:         /* everything but intra */
michael@0:         if (x->e_mbd.mode_info_context->mbmi.ref_frame)
michael@0:         {
michael@0:             x->e_mbd.pre.y_buffer = plane[this_ref_frame][0];
michael@0:             x->e_mbd.pre.u_buffer = plane[this_ref_frame][1];
michael@0:             x->e_mbd.pre.v_buffer = plane[this_ref_frame][2];
michael@0: 
michael@0:             if (sign_bias != cpi->common.ref_frame_sign_bias[this_ref_frame])
michael@0:             {
michael@0:                 sign_bias = cpi->common.ref_frame_sign_bias[this_ref_frame];
michael@0:                 mode_mv = mode_mv_sb[sign_bias];
michael@0:                 best_ref_mv.as_int = best_ref_mv_sb[sign_bias].as_int;
michael@0:             }
michael@0: 
michael@0: #if CONFIG_MULTI_RES_ENCODING
michael@0:             if (parent_ref_valid)
michael@0:             {
michael@0:                 if (vp8_mode_order[mode_index] == NEARESTMV &&
michael@0:                     mode_mv[NEARESTMV].as_int ==0)
michael@0:                     continue;
michael@0:                 if (vp8_mode_order[mode_index] == NEARMV &&
michael@0:                     mode_mv[NEARMV].as_int ==0)
michael@0:                     continue;
michael@0: 
michael@0:                 if (vp8_mode_order[mode_index] == NEWMV && parent_mode == ZEROMV
michael@0:                     && best_ref_mv.as_int==0)
michael@0:                     continue;
michael@0:                 else if(vp8_mode_order[mode_index] == NEWMV && dissim==0
michael@0:                     && best_ref_mv.as_int==parent_ref_mv.as_int)
michael@0:                     continue;
michael@0:             }
michael@0: #endif
michael@0:         }
michael@0: 
michael@0:         /* Check to see if the testing frequency for this mode is at its max
michael@0:          * If so then prevent it from being tested and increase the threshold
michael@0:          * for its testing */
michael@0:         if (x->mode_test_hit_counts[mode_index] &&
michael@0:                                          (cpi->mode_check_freq[mode_index] > 1))
michael@0:         {
michael@0:             if (x->mbs_tested_so_far <= (cpi->mode_check_freq[mode_index] *
michael@0:                                          x->mode_test_hit_counts[mode_index]))
michael@0:             {
michael@0:                 /* Increase the threshold for coding this mode to make it less
michael@0:                  * likely to be chosen */
michael@0:                 x->rd_thresh_mult[mode_index] += 4;
michael@0: 
michael@0:                 if (x->rd_thresh_mult[mode_index] > MAX_THRESHMULT)
michael@0:                     x->rd_thresh_mult[mode_index] = MAX_THRESHMULT;
michael@0: 
michael@0:                 x->rd_threshes[mode_index] =
michael@0:                                  (cpi->rd_baseline_thresh[mode_index] >> 7) *
michael@0:                                  x->rd_thresh_mult[mode_index];
michael@0:                 continue;
michael@0:             }
michael@0:         }
michael@0: 
michael@0:         /* We have now reached the point where we are going to test the current
michael@0:          * mode so increment the counter for the number of times it has been
michael@0:          * tested */
michael@0:         x->mode_test_hit_counts[mode_index] ++;
michael@0: 
michael@0:         rate2 = 0;
michael@0:         distortion2 = 0;
michael@0: 
michael@0:         this_mode = vp8_mode_order[mode_index];
michael@0: 
michael@0:         x->e_mbd.mode_info_context->mbmi.mode = this_mode;
michael@0:         x->e_mbd.mode_info_context->mbmi.uv_mode = DC_PRED;
michael@0: 
michael@0:         /* Work out the cost assosciated with selecting the reference frame */
michael@0:         frame_cost =
michael@0:             x->ref_frame_cost[x->e_mbd.mode_info_context->mbmi.ref_frame];
michael@0:         rate2 += frame_cost;
michael@0: 
michael@0:         /* Only consider ZEROMV/ALTREF_FRAME for alt ref frame,
michael@0:          * unless ARNR filtering is enabled in which case we want
michael@0:          * an unfiltered alternative */
michael@0:         if (cpi->is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0))
michael@0:         {
michael@0:             if (this_mode != ZEROMV ||
michael@0:                 x->e_mbd.mode_info_context->mbmi.ref_frame != ALTREF_FRAME)
michael@0:                 continue;
michael@0:         }
michael@0: 
michael@0:         switch (this_mode)
michael@0:         {
michael@0:         case B_PRED:
michael@0:             /* Pass best so far to pick_intra4x4mby_modes to use as breakout */
michael@0:             distortion2 = best_rd_sse;
michael@0:             pick_intra4x4mby_modes(x, &rate, &distortion2);
michael@0: 
michael@0:             if (distortion2 == INT_MAX)
michael@0:             {
michael@0:                 this_rd = INT_MAX;
michael@0:             }
michael@0:             else
michael@0:             {
michael@0:                 rate2 += rate;
michael@0:                 distortion2 = vp8_variance16x16(
michael@0:                                     *(b->base_src), b->src_stride,
michael@0:                                     x->e_mbd.predictor, 16, &sse);
michael@0:                 this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
michael@0: 
michael@0:                 if (this_rd < best_intra_rd)
michael@0:                 {
michael@0:                     best_intra_rd = this_rd;
michael@0:                     *returnintra = distortion2;
michael@0:                 }
michael@0:             }
michael@0: 
michael@0:             break;
michael@0: 
michael@0:         case SPLITMV:
michael@0: 
michael@0:             /* Split MV modes currently not supported when RD is not enabled. */
michael@0:             break;
michael@0: 
michael@0:         case DC_PRED:
michael@0:         case V_PRED:
michael@0:         case H_PRED:
michael@0:         case TM_PRED:
michael@0:             vp8_build_intra_predictors_mby_s(xd,
michael@0:                                              xd->dst.y_buffer - xd->dst.y_stride,
michael@0:                                              xd->dst.y_buffer - 1,
michael@0:                                              xd->dst.y_stride,
michael@0:                                              xd->predictor,
michael@0:                                              16);
michael@0:             distortion2 = vp8_variance16x16
michael@0:                                           (*(b->base_src), b->src_stride,
michael@0:                                           x->e_mbd.predictor, 16, &sse);
michael@0:             rate2 += x->mbmode_cost[x->e_mbd.frame_type][x->e_mbd.mode_info_context->mbmi.mode];
michael@0:             this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
michael@0: 
michael@0:             if (this_rd < best_intra_rd)
michael@0:             {
michael@0:                 best_intra_rd = this_rd;
michael@0:                 *returnintra = distortion2;
michael@0:             }
michael@0:             break;
michael@0: 
michael@0:         case NEWMV:
michael@0:         {
michael@0:             int thissme;
michael@0:             int step_param;
michael@0:             int further_steps;
michael@0:             int n = 0;
michael@0:             int sadpb = x->sadperbit16;
michael@0:             int_mv mvp_full;
michael@0: 
michael@0:             int col_min = ((best_ref_mv.as_mv.col+7)>>3) - MAX_FULL_PEL_VAL;
michael@0:             int row_min = ((best_ref_mv.as_mv.row+7)>>3) - MAX_FULL_PEL_VAL;
michael@0:             int col_max = (best_ref_mv.as_mv.col>>3)
michael@0:                          + MAX_FULL_PEL_VAL;
michael@0:             int row_max = (best_ref_mv.as_mv.row>>3)
michael@0:                          + MAX_FULL_PEL_VAL;
michael@0: 
michael@0:             int tmp_col_min = x->mv_col_min;
michael@0:             int tmp_col_max = x->mv_col_max;
michael@0:             int tmp_row_min = x->mv_row_min;
michael@0:             int tmp_row_max = x->mv_row_max;
michael@0: 
michael@0:             int speed_adjust = (cpi->Speed > 5) ? ((cpi->Speed >= 8)? 3 : 2) : 1;
michael@0: 
michael@0:             /* Further step/diamond searches as necessary */
michael@0:             step_param = cpi->sf.first_step + speed_adjust;
michael@0: 
michael@0: #if CONFIG_MULTI_RES_ENCODING
michael@0:             /* If lower-res drops this frame, then higher-res encoder does
michael@0:                motion search without any previous knowledge. Also, since
michael@0:                last frame motion info is not stored, then we can not
michael@0:                use improved_mv_pred. */
michael@0:             if (cpi->oxcf.mr_encoder_id && !parent_ref_valid)
michael@0:                 sf_improved_mv_pred = 0;
michael@0: 
michael@0:             if (parent_ref_valid && parent_ref_frame)
michael@0:             {
michael@0:                 /* Use parent MV as predictor. Adjust search range
michael@0:                  * accordingly.
michael@0:                  */
michael@0:                 mvp.as_int = parent_ref_mv.as_int;
michael@0:                 mvp_full.as_mv.col = parent_ref_mv.as_mv.col>>3;
michael@0:                 mvp_full.as_mv.row = parent_ref_mv.as_mv.row>>3;
michael@0: 
michael@0:                 if(dissim <=32) step_param += 3;
michael@0:                 else if(dissim <=128) step_param += 2;
michael@0:                 else step_param += 1;
michael@0:             }else
michael@0: #endif
michael@0:             {
michael@0:                 if(sf_improved_mv_pred)
michael@0:                 {
michael@0:                     if(!saddone)
michael@0:                     {
michael@0:                         vp8_cal_sad(cpi,xd,x, recon_yoffset ,&near_sadidx[0] );
michael@0:                         saddone = 1;
michael@0:                     }
michael@0: 
michael@0:                     vp8_mv_pred(cpi, &x->e_mbd, x->e_mbd.mode_info_context,
michael@0:                                 &mvp,x->e_mbd.mode_info_context->mbmi.ref_frame,
michael@0:                                 cpi->common.ref_frame_sign_bias, &sr,
michael@0:                                 &near_sadidx[0]);
michael@0: 
michael@0:                     sr += speed_adjust;
michael@0:                     /* adjust search range according to sr from mv prediction */
michael@0:                     if(sr > step_param)
michael@0:                         step_param = sr;
michael@0: 
michael@0:                     mvp_full.as_mv.col = mvp.as_mv.col>>3;
michael@0:                     mvp_full.as_mv.row = mvp.as_mv.row>>3;
michael@0:                 }else
michael@0:                 {
michael@0:                     mvp.as_int = best_ref_mv.as_int;
michael@0:                     mvp_full.as_mv.col = best_ref_mv.as_mv.col>>3;
michael@0:                     mvp_full.as_mv.row = best_ref_mv.as_mv.row>>3;
michael@0:                 }
michael@0:             }
michael@0: 
michael@0: #if CONFIG_MULTI_RES_ENCODING
michael@0:             if (parent_ref_valid && parent_ref_frame && dissim <= 2 &&
michael@0:                 MAX(abs(best_ref_mv.as_mv.row - parent_ref_mv.as_mv.row),
michael@0:                     abs(best_ref_mv.as_mv.col - parent_ref_mv.as_mv.col)) <= 4)
michael@0:             {
michael@0:                 d->bmi.mv.as_int = mvp_full.as_int;
michael@0:                 mode_mv[NEWMV].as_int = mvp_full.as_int;
michael@0: 
michael@0:                 cpi->find_fractional_mv_step(x, b, d, &d->bmi.mv, &best_ref_mv,
michael@0:                                              x->errorperbit,
michael@0:                                              &cpi->fn_ptr[BLOCK_16X16],
michael@0:                                              cpi->mb.mvcost,
michael@0:                                              &distortion2,&sse);
michael@0:             }else
michael@0: #endif
michael@0:             {
michael@0:                 /* Get intersection of UMV window and valid MV window to
michael@0:                  * reduce # of checks in diamond search. */
michael@0:                 if (x->mv_col_min < col_min )
michael@0:                     x->mv_col_min = col_min;
michael@0:                 if (x->mv_col_max > col_max )
michael@0:                     x->mv_col_max = col_max;
michael@0:                 if (x->mv_row_min < row_min )
michael@0:                     x->mv_row_min = row_min;
michael@0:                 if (x->mv_row_max > row_max )
michael@0:                     x->mv_row_max = row_max;
michael@0: 
michael@0:                 further_steps = (cpi->Speed >= 8)?
michael@0:                            0: (cpi->sf.max_step_search_steps - 1 - step_param);
michael@0: 
michael@0:                 if (cpi->sf.search_method == HEX)
michael@0:                 {
michael@0: #if CONFIG_MULTI_RES_ENCODING
michael@0:                 /* TODO: In higher-res pick_inter_mode, step_param is used to
michael@0:                  * modify hex search range. Here, set step_param to 0 not to
michael@0:                  * change the behavior in lowest-resolution encoder.
michael@0:                  * Will improve it later.
michael@0:                  */
michael@0:                  /* Set step_param to 0 to ensure large-range motion search
michael@0:                     when encoder drops this frame at lower-resolution.
michael@0:                   */
michael@0:                 if (!parent_ref_valid)
michael@0:                     step_param = 0;
michael@0: #endif
michael@0:                     bestsme = vp8_hex_search(x, b, d, &mvp_full, &d->bmi.mv,
michael@0:                                           step_param, sadpb,
michael@0:                                           &cpi->fn_ptr[BLOCK_16X16],
michael@0:                                           x->mvsadcost, x->mvcost, &best_ref_mv);
michael@0:                     mode_mv[NEWMV].as_int = d->bmi.mv.as_int;
michael@0:                 }
michael@0:                 else
michael@0:                 {
michael@0:                     bestsme = cpi->diamond_search_sad(x, b, d, &mvp_full,
michael@0:                                           &d->bmi.mv, step_param, sadpb, &num00,
michael@0:                                           &cpi->fn_ptr[BLOCK_16X16],
michael@0:                                           x->mvcost, &best_ref_mv);
michael@0:                     mode_mv[NEWMV].as_int = d->bmi.mv.as_int;
michael@0: 
michael@0:                     /* Further step/diamond searches as necessary */
michael@0:                     n = num00;
michael@0:                     num00 = 0;
michael@0: 
michael@0:                     while (n < further_steps)
michael@0:                     {
michael@0:                         n++;
michael@0: 
michael@0:                         if (num00)
michael@0:                             num00--;
michael@0:                         else
michael@0:                         {
michael@0:                             thissme =
michael@0:                             cpi->diamond_search_sad(x, b, d, &mvp_full,
michael@0:                                                     &d->bmi.mv,
michael@0:                                                     step_param + n,
michael@0:                                                     sadpb, &num00,
michael@0:                                                     &cpi->fn_ptr[BLOCK_16X16],
michael@0:                                                     x->mvcost, &best_ref_mv);
michael@0:                             if (thissme < bestsme)
michael@0:                             {
michael@0:                                 bestsme = thissme;
michael@0:                                 mode_mv[NEWMV].as_int = d->bmi.mv.as_int;
michael@0:                             }
michael@0:                             else
michael@0:                             {
michael@0:                                 d->bmi.mv.as_int = mode_mv[NEWMV].as_int;
michael@0:                             }
michael@0:                         }
michael@0:                     }
michael@0:                 }
michael@0: 
michael@0:                 x->mv_col_min = tmp_col_min;
michael@0:                 x->mv_col_max = tmp_col_max;
michael@0:                 x->mv_row_min = tmp_row_min;
michael@0:                 x->mv_row_max = tmp_row_max;
michael@0: 
michael@0:                 if (bestsme < INT_MAX)
michael@0:                     cpi->find_fractional_mv_step(x, b, d, &d->bmi.mv,
michael@0:                                              &best_ref_mv, x->errorperbit,
michael@0:                                              &cpi->fn_ptr[BLOCK_16X16],
michael@0:                                              cpi->mb.mvcost,
michael@0:                                              &distortion2,&sse);
michael@0:             }
michael@0: 
michael@0:             mode_mv[NEWMV].as_int = d->bmi.mv.as_int;
michael@0: 
michael@0:             /* mv cost; */
michael@0:             rate2 += vp8_mv_bit_cost(&mode_mv[NEWMV], &best_ref_mv,
michael@0:                                      cpi->mb.mvcost, 128);
michael@0:         }
michael@0: 
michael@0:         case NEARESTMV:
michael@0:         case NEARMV:
michael@0: 
michael@0:             if (mode_mv[this_mode].as_int == 0)
michael@0:                 continue;
michael@0: 
michael@0:         case ZEROMV:
michael@0: 
michael@0:             /* Trap vectors that reach beyond the UMV borders
michael@0:              * Note that ALL New MV, Nearest MV Near MV and Zero MV code drops
michael@0:              * through to this point because of the lack of break statements
michael@0:              * in the previous two cases.
michael@0:              */
michael@0:             if (((mode_mv[this_mode].as_mv.row >> 3) < x->mv_row_min) ||
michael@0:                 ((mode_mv[this_mode].as_mv.row >> 3) > x->mv_row_max) ||
michael@0:                 ((mode_mv[this_mode].as_mv.col >> 3) < x->mv_col_min) ||
michael@0:                 ((mode_mv[this_mode].as_mv.col >> 3) > x->mv_col_max))
michael@0:                 continue;
michael@0: 
michael@0:             rate2 += vp8_cost_mv_ref(this_mode, mdcounts);
michael@0:             x->e_mbd.mode_info_context->mbmi.mv.as_int =
michael@0:                                                     mode_mv[this_mode].as_int;
michael@0:             this_rd = evaluate_inter_mode(&sse, rate2, &distortion2, cpi, x,
michael@0:                                           rd_adjustment);
michael@0: 
michael@0:             break;
michael@0:         default:
michael@0:             break;
michael@0:         }
michael@0: 
michael@0: #if CONFIG_TEMPORAL_DENOISING
michael@0:         if (cpi->oxcf.noise_sensitivity)
michael@0:         {
michael@0: 
michael@0:             /* Store for later use by denoiser. */
michael@0:             if (this_mode == ZEROMV && sse < zero_mv_sse )
michael@0:             {
michael@0:                 zero_mv_sse = sse;
michael@0:                 x->best_zeromv_reference_frame =
michael@0:                         x->e_mbd.mode_info_context->mbmi.ref_frame;
michael@0:             }
michael@0: 
michael@0:             /* Store the best NEWMV in x for later use in the denoiser. */
michael@0:             if (x->e_mbd.mode_info_context->mbmi.mode == NEWMV &&
michael@0:                     sse < best_sse)
michael@0:             {
michael@0:                 best_sse = sse;
michael@0:                 x->best_sse_inter_mode = NEWMV;
michael@0:                 x->best_sse_mv = x->e_mbd.mode_info_context->mbmi.mv;
michael@0:                 x->need_to_clamp_best_mvs =
michael@0:                     x->e_mbd.mode_info_context->mbmi.need_to_clamp_mvs;
michael@0:                 x->best_reference_frame =
michael@0:                     x->e_mbd.mode_info_context->mbmi.ref_frame;
michael@0:             }
michael@0:         }
michael@0: #endif
michael@0: 
michael@0:         if (this_rd < best_rd || x->skip)
michael@0:         {
michael@0:             /* Note index of best mode */
michael@0:             best_mode_index = mode_index;
michael@0: 
michael@0:             *returnrate = rate2;
michael@0:             *returndistortion = distortion2;
michael@0:             best_rd_sse = sse;
michael@0:             best_rd = this_rd;
michael@0:             vpx_memcpy(&best_mbmode, &x->e_mbd.mode_info_context->mbmi,
michael@0:                        sizeof(MB_MODE_INFO));
michael@0: 
michael@0:             /* Testing this mode gave rise to an improvement in best error
michael@0:              * score. Lower threshold a bit for next time
michael@0:              */
michael@0:             x->rd_thresh_mult[mode_index] =
michael@0:                      (x->rd_thresh_mult[mode_index] >= (MIN_THRESHMULT + 2)) ?
michael@0:                      x->rd_thresh_mult[mode_index] - 2 : MIN_THRESHMULT;
michael@0:             x->rd_threshes[mode_index] =
michael@0:                                    (cpi->rd_baseline_thresh[mode_index] >> 7) *
michael@0:                                    x->rd_thresh_mult[mode_index];
michael@0:         }
michael@0: 
michael@0:         /* If the mode did not help improve the best error case then raise the
michael@0:          * threshold for testing that mode next time around.
michael@0:          */
michael@0:         else
michael@0:         {
michael@0:             x->rd_thresh_mult[mode_index] += 4;
michael@0: 
michael@0:             if (x->rd_thresh_mult[mode_index] > MAX_THRESHMULT)
michael@0:                 x->rd_thresh_mult[mode_index] = MAX_THRESHMULT;
michael@0: 
michael@0:             x->rd_threshes[mode_index] =
michael@0:                          (cpi->rd_baseline_thresh[mode_index] >> 7) *
michael@0:                          x->rd_thresh_mult[mode_index];
michael@0:         }
michael@0: 
michael@0:         if (x->skip)
michael@0:             break;
michael@0:     }
michael@0: 
michael@0:     /* Reduce the activation RD thresholds for the best choice mode */
michael@0:     if ((cpi->rd_baseline_thresh[best_mode_index] > 0) && (cpi->rd_baseline_thresh[best_mode_index] < (INT_MAX >> 2)))
michael@0:     {
michael@0:         int best_adjustment = (x->rd_thresh_mult[best_mode_index] >> 3);
michael@0: 
michael@0:         x->rd_thresh_mult[best_mode_index] =
michael@0:                         (x->rd_thresh_mult[best_mode_index]
michael@0:                         >= (MIN_THRESHMULT + best_adjustment)) ?
michael@0:                         x->rd_thresh_mult[best_mode_index] - best_adjustment :
michael@0:                         MIN_THRESHMULT;
michael@0:         x->rd_threshes[best_mode_index] =
michael@0:                         (cpi->rd_baseline_thresh[best_mode_index] >> 7) *
michael@0:                         x->rd_thresh_mult[best_mode_index];
michael@0:     }
michael@0: 
michael@0: 
michael@0:     {
michael@0:         int this_rdbin = (*returndistortion >> 7);
michael@0: 
michael@0:         if (this_rdbin >= 1024)
michael@0:         {
michael@0:             this_rdbin = 1023;
michael@0:         }
michael@0: 
michael@0:         x->error_bins[this_rdbin] ++;
michael@0:     }
michael@0: 
michael@0: #if CONFIG_TEMPORAL_DENOISING
michael@0:     if (cpi->oxcf.noise_sensitivity)
michael@0:     {
michael@0:         if (x->best_sse_inter_mode == DC_PRED)
michael@0:         {
michael@0:             /* No best MV found. */
michael@0:             x->best_sse_inter_mode = best_mbmode.mode;
michael@0:             x->best_sse_mv = best_mbmode.mv;
michael@0:             x->need_to_clamp_best_mvs = best_mbmode.need_to_clamp_mvs;
michael@0:             x->best_reference_frame = best_mbmode.ref_frame;
michael@0:             best_sse = best_rd_sse;
michael@0:         }
michael@0:         vp8_denoiser_denoise_mb(&cpi->denoiser, x, best_sse, zero_mv_sse,
michael@0:                                 recon_yoffset, recon_uvoffset);
michael@0: 
michael@0: 
michael@0:         /* Reevaluate ZEROMV after denoising. */
michael@0:         if (best_mbmode.ref_frame == INTRA_FRAME &&
michael@0:             x->best_zeromv_reference_frame != INTRA_FRAME)
michael@0:         {
michael@0:             int this_rd = 0;
michael@0:             int this_ref_frame = x->best_zeromv_reference_frame;
michael@0:             rate2 = x->ref_frame_cost[this_ref_frame] +
michael@0:                     vp8_cost_mv_ref(ZEROMV, mdcounts);
michael@0:             distortion2 = 0;
michael@0: 
michael@0:             /* set up the proper prediction buffers for the frame */
michael@0:             x->e_mbd.mode_info_context->mbmi.ref_frame = this_ref_frame;
michael@0:             x->e_mbd.pre.y_buffer = plane[this_ref_frame][0];
michael@0:             x->e_mbd.pre.u_buffer = plane[this_ref_frame][1];
michael@0:             x->e_mbd.pre.v_buffer = plane[this_ref_frame][2];
michael@0: 
michael@0:             x->e_mbd.mode_info_context->mbmi.mode = ZEROMV;
michael@0:             x->e_mbd.mode_info_context->mbmi.uv_mode = DC_PRED;
michael@0:             x->e_mbd.mode_info_context->mbmi.mv.as_int = 0;
michael@0:             this_rd = evaluate_inter_mode(&sse, rate2, &distortion2, cpi, x,
michael@0:                                           rd_adjustment);
michael@0: 
michael@0:             if (this_rd < best_rd)
michael@0:             {
michael@0:                 vpx_memcpy(&best_mbmode, &x->e_mbd.mode_info_context->mbmi,
michael@0:                            sizeof(MB_MODE_INFO));
michael@0:             }
michael@0:         }
michael@0: 
michael@0:     }
michael@0: #endif
michael@0: 
michael@0:     if (cpi->is_src_frame_alt_ref &&
michael@0:         (best_mbmode.mode != ZEROMV || best_mbmode.ref_frame != ALTREF_FRAME))
michael@0:     {
michael@0:         x->e_mbd.mode_info_context->mbmi.mode = ZEROMV;
michael@0:         x->e_mbd.mode_info_context->mbmi.ref_frame = ALTREF_FRAME;
michael@0:         x->e_mbd.mode_info_context->mbmi.mv.as_int = 0;
michael@0:         x->e_mbd.mode_info_context->mbmi.uv_mode = DC_PRED;
michael@0:         x->e_mbd.mode_info_context->mbmi.mb_skip_coeff =
michael@0:                                         (cpi->common.mb_no_coeff_skip);
michael@0:         x->e_mbd.mode_info_context->mbmi.partitioning = 0;
michael@0: 
michael@0:         return;
michael@0:     }
michael@0: 
michael@0:     /* set to the best mb mode, this copy can be skip if x->skip since it
michael@0:      * already has the right content */
michael@0:     if (!x->skip)
michael@0:         vpx_memcpy(&x->e_mbd.mode_info_context->mbmi, &best_mbmode,
michael@0:                    sizeof(MB_MODE_INFO));
michael@0: 
michael@0:     if (best_mbmode.mode <= B_PRED)
michael@0:     {
michael@0:         /* set mode_info_context->mbmi.uv_mode */
michael@0:         pick_intra_mbuv_mode(x);
michael@0:     }
michael@0: 
michael@0:     if (sign_bias
michael@0:       != cpi->common.ref_frame_sign_bias[xd->mode_info_context->mbmi.ref_frame])
michael@0:         best_ref_mv.as_int = best_ref_mv_sb[!sign_bias].as_int;
michael@0: 
michael@0:     update_mvcount(x, &best_ref_mv);
michael@0: }
michael@0: 
michael@0: 
michael@0: void vp8_pick_intra_mode(MACROBLOCK *x, int *rate_)
michael@0: {
michael@0:     int error4x4, error16x16 = INT_MAX;
michael@0:     int rate, best_rate = 0, distortion, best_sse;
michael@0:     MB_PREDICTION_MODE mode, best_mode = DC_PRED;
michael@0:     int this_rd;
michael@0:     unsigned int sse;
michael@0:     BLOCK *b = &x->block[0];
michael@0:     MACROBLOCKD *xd = &x->e_mbd;
michael@0: 
michael@0:     xd->mode_info_context->mbmi.ref_frame = INTRA_FRAME;
michael@0: 
michael@0:     pick_intra_mbuv_mode(x);
michael@0: 
michael@0:     for (mode = DC_PRED; mode <= TM_PRED; mode ++)
michael@0:     {
michael@0:         xd->mode_info_context->mbmi.mode = mode;
michael@0:         vp8_build_intra_predictors_mby_s(xd,
michael@0:                                          xd->dst.y_buffer - xd->dst.y_stride,
michael@0:                                          xd->dst.y_buffer - 1,
michael@0:                                          xd->dst.y_stride,
michael@0:                                          xd->predictor,
michael@0:                                          16);
michael@0:         distortion = vp8_variance16x16
michael@0:             (*(b->base_src), b->src_stride, xd->predictor, 16, &sse);
michael@0:         rate = x->mbmode_cost[xd->frame_type][mode];
michael@0:         this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
michael@0: 
michael@0:         if (error16x16 > this_rd)
michael@0:         {
michael@0:             error16x16 = this_rd;
michael@0:             best_mode = mode;
michael@0:             best_sse = sse;
michael@0:             best_rate = rate;
michael@0:         }
michael@0:     }
michael@0:     xd->mode_info_context->mbmi.mode = best_mode;
michael@0: 
michael@0:     error4x4 = pick_intra4x4mby_modes(x, &rate,
michael@0:                                       &best_sse);
michael@0:     if (error4x4 < error16x16)
michael@0:     {
michael@0:         xd->mode_info_context->mbmi.mode = B_PRED;
michael@0:         best_rate = rate;
michael@0:     }
michael@0: 
michael@0:     *rate_ = best_rate;
michael@0: }