michael@0: ; michael@0: ; Copyright (c) 2010 The WebM project authors. All Rights Reserved. michael@0: ; michael@0: ; Use of this source code is governed by a BSD-style license and patent michael@0: ; grant that can be found in the LICENSE file in the root of the source michael@0: ; tree. All contributing project authors may be found in the AUTHORS michael@0: ; file in the root of the source tree. michael@0: ; michael@0: michael@0: michael@0: %include "vpx_ports/x86_abi_support.asm" michael@0: %include "vp8_asm_enc_offsets.asm" michael@0: michael@0: michael@0: ; void vp8_regular_quantize_b_sse4 | arg michael@0: ; (BLOCK *b, | 0 michael@0: ; BLOCKD *d) | 1 michael@0: michael@0: global sym(vp8_regular_quantize_b_sse4) PRIVATE michael@0: sym(vp8_regular_quantize_b_sse4): michael@0: michael@0: %if ABI_IS_32BIT michael@0: push rbp michael@0: mov rbp, rsp michael@0: GET_GOT rbx michael@0: push rdi michael@0: push rsi michael@0: michael@0: ALIGN_STACK 16, rax michael@0: %define qcoeff 0 ; 32 michael@0: %define stack_size 32 michael@0: sub rsp, stack_size michael@0: %else michael@0: %if LIBVPX_YASM_WIN64 michael@0: SAVE_XMM 8, u michael@0: push rdi michael@0: push rsi michael@0: %endif michael@0: %endif michael@0: ; end prolog michael@0: michael@0: %if ABI_IS_32BIT michael@0: mov rdi, arg(0) ; BLOCK *b michael@0: mov rsi, arg(1) ; BLOCKD *d michael@0: %else michael@0: %if LIBVPX_YASM_WIN64 michael@0: mov rdi, rcx ; BLOCK *b michael@0: mov rsi, rdx ; BLOCKD *d michael@0: %else michael@0: ;mov rdi, rdi ; BLOCK *b michael@0: ;mov rsi, rsi ; BLOCKD *d michael@0: %endif michael@0: %endif michael@0: michael@0: mov rax, [rdi + vp8_block_coeff] michael@0: mov rcx, [rdi + vp8_block_zbin] michael@0: mov rdx, [rdi + vp8_block_round] michael@0: movd xmm7, [rdi + vp8_block_zbin_extra] michael@0: michael@0: ; z michael@0: movdqa xmm0, [rax] michael@0: movdqa xmm1, [rax + 16] michael@0: michael@0: ; duplicate zbin_oq_value michael@0: pshuflw xmm7, xmm7, 0 michael@0: punpcklwd xmm7, xmm7 michael@0: michael@0: movdqa xmm2, xmm0 michael@0: movdqa xmm3, xmm1 michael@0: michael@0: ; sz michael@0: psraw xmm0, 15 michael@0: psraw xmm1, 15 michael@0: michael@0: ; (z ^ sz) michael@0: pxor xmm2, xmm0 michael@0: pxor xmm3, xmm1 michael@0: michael@0: ; x = abs(z) michael@0: psubw xmm2, xmm0 michael@0: psubw xmm3, xmm1 michael@0: michael@0: ; zbin michael@0: movdqa xmm4, [rcx] michael@0: movdqa xmm5, [rcx + 16] michael@0: michael@0: ; *zbin_ptr + zbin_oq_value michael@0: paddw xmm4, xmm7 michael@0: paddw xmm5, xmm7 michael@0: michael@0: movdqa xmm6, xmm2 michael@0: movdqa xmm7, xmm3 michael@0: michael@0: ; x - (*zbin_ptr + zbin_oq_value) michael@0: psubw xmm6, xmm4 michael@0: psubw xmm7, xmm5 michael@0: michael@0: ; round michael@0: movdqa xmm4, [rdx] michael@0: movdqa xmm5, [rdx + 16] michael@0: michael@0: mov rax, [rdi + vp8_block_quant_shift] michael@0: mov rcx, [rdi + vp8_block_quant] michael@0: mov rdx, [rdi + vp8_block_zrun_zbin_boost] michael@0: michael@0: ; x + round michael@0: paddw xmm2, xmm4 michael@0: paddw xmm3, xmm5 michael@0: michael@0: ; quant michael@0: movdqa xmm4, [rcx] michael@0: movdqa xmm5, [rcx + 16] michael@0: michael@0: ; y = x * quant_ptr >> 16 michael@0: pmulhw xmm4, xmm2 michael@0: pmulhw xmm5, xmm3 michael@0: michael@0: ; y += x michael@0: paddw xmm2, xmm4 michael@0: paddw xmm3, xmm5 michael@0: michael@0: pxor xmm4, xmm4 michael@0: %if ABI_IS_32BIT michael@0: movdqa [rsp + qcoeff], xmm4 michael@0: movdqa [rsp + qcoeff + 16], xmm4 michael@0: %else michael@0: pxor xmm8, xmm8 michael@0: %endif michael@0: michael@0: ; quant_shift michael@0: movdqa xmm5, [rax] michael@0: michael@0: ; zrun_zbin_boost michael@0: mov rax, rdx michael@0: michael@0: %macro ZIGZAG_LOOP 5 michael@0: ; x michael@0: pextrw ecx, %4, %2 michael@0: michael@0: ; if (x >= zbin) michael@0: sub cx, WORD PTR[rdx] ; x - zbin michael@0: lea rdx, [rdx + 2] ; zbin_boost_ptr++ michael@0: jl .rq_zigzag_loop_%1 ; x < zbin michael@0: michael@0: pextrw edi, %3, %2 ; y michael@0: michael@0: ; downshift by quant_shift[rc] michael@0: pextrb ecx, xmm5, %1 ; quant_shift[rc] michael@0: sar edi, cl ; also sets Z bit michael@0: je .rq_zigzag_loop_%1 ; !y michael@0: %if ABI_IS_32BIT michael@0: mov WORD PTR[rsp + qcoeff + %1 *2], di michael@0: %else michael@0: pinsrw %5, edi, %2 ; qcoeff[rc] michael@0: %endif michael@0: mov rdx, rax ; reset to b->zrun_zbin_boost michael@0: .rq_zigzag_loop_%1: michael@0: %endmacro michael@0: ; in vp8_default_zig_zag1d order: see vp8/common/entropy.c michael@0: ZIGZAG_LOOP 0, 0, xmm2, xmm6, xmm4 michael@0: ZIGZAG_LOOP 1, 1, xmm2, xmm6, xmm4 michael@0: ZIGZAG_LOOP 4, 4, xmm2, xmm6, xmm4 michael@0: ZIGZAG_LOOP 8, 0, xmm3, xmm7, xmm8 michael@0: ZIGZAG_LOOP 5, 5, xmm2, xmm6, xmm4 michael@0: ZIGZAG_LOOP 2, 2, xmm2, xmm6, xmm4 michael@0: ZIGZAG_LOOP 3, 3, xmm2, xmm6, xmm4 michael@0: ZIGZAG_LOOP 6, 6, xmm2, xmm6, xmm4 michael@0: ZIGZAG_LOOP 9, 1, xmm3, xmm7, xmm8 michael@0: ZIGZAG_LOOP 12, 4, xmm3, xmm7, xmm8 michael@0: ZIGZAG_LOOP 13, 5, xmm3, xmm7, xmm8 michael@0: ZIGZAG_LOOP 10, 2, xmm3, xmm7, xmm8 michael@0: ZIGZAG_LOOP 7, 7, xmm2, xmm6, xmm4 michael@0: ZIGZAG_LOOP 11, 3, xmm3, xmm7, xmm8 michael@0: ZIGZAG_LOOP 14, 6, xmm3, xmm7, xmm8 michael@0: ZIGZAG_LOOP 15, 7, xmm3, xmm7, xmm8 michael@0: michael@0: mov rcx, [rsi + vp8_blockd_dequant] michael@0: mov rdi, [rsi + vp8_blockd_dqcoeff] michael@0: michael@0: %if ABI_IS_32BIT michael@0: movdqa xmm4, [rsp + qcoeff] michael@0: movdqa xmm5, [rsp + qcoeff + 16] michael@0: %else michael@0: %define xmm5 xmm8 michael@0: %endif michael@0: michael@0: ; y ^ sz michael@0: pxor xmm4, xmm0 michael@0: pxor xmm5, xmm1 michael@0: ; x = (y ^ sz) - sz michael@0: psubw xmm4, xmm0 michael@0: psubw xmm5, xmm1 michael@0: michael@0: ; dequant michael@0: movdqa xmm0, [rcx] michael@0: movdqa xmm1, [rcx + 16] michael@0: michael@0: mov rcx, [rsi + vp8_blockd_qcoeff] michael@0: michael@0: pmullw xmm0, xmm4 michael@0: pmullw xmm1, xmm5 michael@0: michael@0: ; store qcoeff michael@0: movdqa [rcx], xmm4 michael@0: movdqa [rcx + 16], xmm5 michael@0: michael@0: ; store dqcoeff michael@0: movdqa [rdi], xmm0 michael@0: movdqa [rdi + 16], xmm1 michael@0: michael@0: mov rcx, [rsi + vp8_blockd_eob] michael@0: michael@0: ; select the last value (in zig_zag order) for EOB michael@0: pxor xmm6, xmm6 michael@0: pcmpeqw xmm4, xmm6 michael@0: pcmpeqw xmm5, xmm6 michael@0: michael@0: packsswb xmm4, xmm5 michael@0: pshufb xmm4, [GLOBAL(zig_zag1d)] michael@0: pmovmskb edx, xmm4 michael@0: xor rdi, rdi michael@0: mov eax, -1 michael@0: xor dx, ax michael@0: bsr eax, edx michael@0: sub edi, edx michael@0: sar edi, 31 michael@0: add eax, 1 michael@0: and eax, edi michael@0: michael@0: mov BYTE PTR [rcx], al ; store eob michael@0: michael@0: ; begin epilog michael@0: %if ABI_IS_32BIT michael@0: add rsp, stack_size michael@0: pop rsp michael@0: michael@0: pop rsi michael@0: pop rdi michael@0: RESTORE_GOT michael@0: pop rbp michael@0: %else michael@0: %undef xmm5 michael@0: %if LIBVPX_YASM_WIN64 michael@0: pop rsi michael@0: pop rdi michael@0: RESTORE_XMM michael@0: %endif michael@0: %endif michael@0: michael@0: ret michael@0: michael@0: SECTION_RODATA michael@0: align 16 michael@0: ; vp8/common/entropy.c: vp8_default_zig_zag1d michael@0: zig_zag1d: michael@0: db 0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15