michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
michael@0:  * vim: set ts=8 sts=4 et sw=4 tw=99:
michael@0:  * This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #ifndef jit_x64_Assembler_x64_h
michael@0: #define jit_x64_Assembler_x64_h
michael@0: 
michael@0: #include "mozilla/ArrayUtils.h"
michael@0: 
michael@0: #include "jit/IonCode.h"
michael@0: #include "jit/shared/Assembler-shared.h"
michael@0: 
michael@0: namespace js {
michael@0: namespace jit {
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR Register rax = { JSC::X86Registers::eax };
michael@0: static MOZ_CONSTEXPR_VAR Register rbx = { JSC::X86Registers::ebx };
michael@0: static MOZ_CONSTEXPR_VAR Register rcx = { JSC::X86Registers::ecx };
michael@0: static MOZ_CONSTEXPR_VAR Register rdx = { JSC::X86Registers::edx };
michael@0: static MOZ_CONSTEXPR_VAR Register rsi = { JSC::X86Registers::esi };
michael@0: static MOZ_CONSTEXPR_VAR Register rdi = { JSC::X86Registers::edi };
michael@0: static MOZ_CONSTEXPR_VAR Register rbp = { JSC::X86Registers::ebp };
michael@0: static MOZ_CONSTEXPR_VAR Register r8  = { JSC::X86Registers::r8  };
michael@0: static MOZ_CONSTEXPR_VAR Register r9  = { JSC::X86Registers::r9  };
michael@0: static MOZ_CONSTEXPR_VAR Register r10 = { JSC::X86Registers::r10 };
michael@0: static MOZ_CONSTEXPR_VAR Register r11 = { JSC::X86Registers::r11 };
michael@0: static MOZ_CONSTEXPR_VAR Register r12 = { JSC::X86Registers::r12 };
michael@0: static MOZ_CONSTEXPR_VAR Register r13 = { JSC::X86Registers::r13 };
michael@0: static MOZ_CONSTEXPR_VAR Register r14 = { JSC::X86Registers::r14 };
michael@0: static MOZ_CONSTEXPR_VAR Register r15 = { JSC::X86Registers::r15 };
michael@0: static MOZ_CONSTEXPR_VAR Register rsp = { JSC::X86Registers::esp };
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm0 = { JSC::X86Registers::xmm0 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm1 = { JSC::X86Registers::xmm1 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm2 = { JSC::X86Registers::xmm2 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm3 = { JSC::X86Registers::xmm3 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm4 = { JSC::X86Registers::xmm4 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm5 = { JSC::X86Registers::xmm5 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm6 = { JSC::X86Registers::xmm6 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm7 = { JSC::X86Registers::xmm7 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm8 = { JSC::X86Registers::xmm8 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm9 = { JSC::X86Registers::xmm9 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm10 = { JSC::X86Registers::xmm10 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm11 = { JSC::X86Registers::xmm11 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm12 = { JSC::X86Registers::xmm12 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm13 = { JSC::X86Registers::xmm13 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm14 = { JSC::X86Registers::xmm14 };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister xmm15 = { JSC::X86Registers::xmm15 };
michael@0: 
michael@0: // X86-common synonyms.
michael@0: static MOZ_CONSTEXPR_VAR Register eax = rax;
michael@0: static MOZ_CONSTEXPR_VAR Register ebx = rbx;
michael@0: static MOZ_CONSTEXPR_VAR Register ecx = rcx;
michael@0: static MOZ_CONSTEXPR_VAR Register edx = rdx;
michael@0: static MOZ_CONSTEXPR_VAR Register esi = rsi;
michael@0: static MOZ_CONSTEXPR_VAR Register edi = rdi;
michael@0: static MOZ_CONSTEXPR_VAR Register ebp = rbp;
michael@0: static MOZ_CONSTEXPR_VAR Register esp = rsp;
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR Register InvalidReg = { JSC::X86Registers::invalid_reg };
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister InvalidFloatReg = { JSC::X86Registers::invalid_xmm };
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR Register StackPointer = rsp;
michael@0: static MOZ_CONSTEXPR_VAR Register FramePointer = rbp;
michael@0: static MOZ_CONSTEXPR_VAR Register JSReturnReg = rcx;
michael@0: // Avoid, except for assertions.
michael@0: static MOZ_CONSTEXPR_VAR Register JSReturnReg_Type = JSReturnReg;
michael@0: static MOZ_CONSTEXPR_VAR Register JSReturnReg_Data = JSReturnReg;
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR Register ReturnReg = rax;
michael@0: static MOZ_CONSTEXPR_VAR Register ScratchReg = r11;
michael@0: static MOZ_CONSTEXPR_VAR Register HeapReg = r15;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister ReturnFloatReg = xmm0;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister ScratchFloatReg = xmm15;
michael@0: 
michael@0: // Avoid rbp, which is the FramePointer, which is unavailable in some modes.
michael@0: static MOZ_CONSTEXPR_VAR Register ArgumentsRectifierReg = r8;
michael@0: static MOZ_CONSTEXPR_VAR Register CallTempReg0 = rax;
michael@0: static MOZ_CONSTEXPR_VAR Register CallTempReg1 = rdi;
michael@0: static MOZ_CONSTEXPR_VAR Register CallTempReg2 = rbx;
michael@0: static MOZ_CONSTEXPR_VAR Register CallTempReg3 = rcx;
michael@0: static MOZ_CONSTEXPR_VAR Register CallTempReg4 = rsi;
michael@0: static MOZ_CONSTEXPR_VAR Register CallTempReg5 = rdx;
michael@0: 
michael@0: // Different argument registers for WIN64
michael@0: #if defined(_WIN64)
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg0 = rcx;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg1 = rdx;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg2 = r8;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg3 = r9;
michael@0: static MOZ_CONSTEXPR_VAR uint32_t NumIntArgRegs = 4;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgRegs[NumIntArgRegs] = { rcx, rdx, r8, r9 };
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR Register CallTempNonArgRegs[] = { rax, rdi, rbx, rsi };
michael@0: static const uint32_t NumCallTempNonArgRegs =
michael@0:     mozilla::ArrayLength(CallTempNonArgRegs);
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg0 = xmm0;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg1 = xmm1;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg2 = xmm2;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg3 = xmm3;
michael@0: static const uint32_t NumFloatArgRegs = 4;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgRegs[NumFloatArgRegs] = { xmm0, xmm1, xmm2, xmm3 };
michael@0: #else
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg0 = rdi;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg1 = rsi;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg2 = rdx;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg3 = rcx;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg4 = r8;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgReg5 = r9;
michael@0: static MOZ_CONSTEXPR_VAR uint32_t NumIntArgRegs = 6;
michael@0: static MOZ_CONSTEXPR_VAR Register IntArgRegs[NumIntArgRegs] = { rdi, rsi, rdx, rcx, r8, r9 };
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR Register CallTempNonArgRegs[] = { rax, rbx };
michael@0: static const uint32_t NumCallTempNonArgRegs =
michael@0:     mozilla::ArrayLength(CallTempNonArgRegs);
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg0 = xmm0;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg1 = xmm1;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg2 = xmm2;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg3 = xmm3;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg4 = xmm4;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg5 = xmm5;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg6 = xmm6;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgReg7 = xmm7;
michael@0: static MOZ_CONSTEXPR_VAR uint32_t NumFloatArgRegs = 8;
michael@0: static MOZ_CONSTEXPR_VAR FloatRegister FloatArgRegs[NumFloatArgRegs] = { xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 };
michael@0: #endif
michael@0: 
michael@0: // The convention used by the ForkJoinGetSlice stub. None of these can be rax
michael@0: // or rdx, which the stub also needs for cmpxchg and div, respectively.
michael@0: static MOZ_CONSTEXPR_VAR Register ForkJoinGetSliceReg_cx = rdi;
michael@0: static MOZ_CONSTEXPR_VAR Register ForkJoinGetSliceReg_temp0 = rbx;
michael@0: static MOZ_CONSTEXPR_VAR Register ForkJoinGetSliceReg_temp1 = rcx;
michael@0: static MOZ_CONSTEXPR_VAR Register ForkJoinGetSliceReg_output = rsi;
michael@0: 
michael@0: // Registers used in the GenerateFFIIonExit Enable Activation block.
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegCallee = r10;
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegE0 = rax;
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegE1 = rdi;
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegE2 = rbx;
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegE3 = rsi;
michael@0: 
michael@0: // Registers used in the GenerateFFIIonExit Disable Activation block.
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegReturnData = ecx;
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegReturnType = ecx;
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegD0 = rax;
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegD1 = rdi;
michael@0: static MOZ_CONSTEXPR_VAR Register AsmJSIonExitRegD2 = rbx;
michael@0: 
michael@0: class ABIArgGenerator
michael@0: {
michael@0: #if defined(XP_WIN)
michael@0:     unsigned regIndex_;
michael@0: #else
michael@0:     unsigned intRegIndex_;
michael@0:     unsigned floatRegIndex_;
michael@0: #endif
michael@0:     uint32_t stackOffset_;
michael@0:     ABIArg current_;
michael@0: 
michael@0:   public:
michael@0:     ABIArgGenerator();
michael@0:     ABIArg next(MIRType argType);
michael@0:     ABIArg &current() { return current_; }
michael@0:     uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
michael@0: 
michael@0:     // Note: these registers are all guaranteed to be different
michael@0:     static const Register NonArgReturnVolatileReg0;
michael@0:     static const Register NonArgReturnVolatileReg1;
michael@0:     static const Register NonVolatileReg;
michael@0: };
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR Register OsrFrameReg = IntArgReg3;
michael@0: 
michael@0: static MOZ_CONSTEXPR_VAR Register PreBarrierReg = rdx;
michael@0: 
michael@0: // GCC stack is aligned on 16 bytes, but we don't maintain the invariant in
michael@0: // jitted code.
michael@0: static const uint32_t StackAlignment = 16;
michael@0: static const bool StackKeptAligned = false;
michael@0: static const uint32_t CodeAlignment = 8;
michael@0: static const uint32_t NativeFrameSize = sizeof(void*);
michael@0: static const uint32_t AlignmentAtPrologue = sizeof(void*);
michael@0: static const uint32_t AlignmentMidPrologue = AlignmentAtPrologue;
michael@0: 
michael@0: static const Scale ScalePointer = TimesEight;
michael@0: 
michael@0: } // namespace jit
michael@0: } // namespace js
michael@0: 
michael@0: #include "jit/shared/Assembler-x86-shared.h"
michael@0: 
michael@0: namespace js {
michael@0: namespace jit {
michael@0: 
michael@0: // Return operand from a JS -> JS call.
michael@0: static MOZ_CONSTEXPR_VAR ValueOperand JSReturnOperand = ValueOperand(JSReturnReg);
michael@0: 
michael@0: class Assembler : public AssemblerX86Shared
michael@0: {
michael@0:     // x64 jumps may need extra bits of relocation, because a jump may extend
michael@0:     // beyond the signed 32-bit range. To account for this we add an extended
michael@0:     // jump table at the bottom of the instruction stream, and if a jump
michael@0:     // overflows its range, it will redirect here.
michael@0:     //
michael@0:     // In our relocation table, we store two offsets instead of one: the offset
michael@0:     // to the original jump, and an offset to the extended jump if we will need
michael@0:     // to use it instead. The offsets are stored as:
michael@0:     //    [unsigned] Unsigned offset to short jump, from the start of the code.
michael@0:     //    [unsigned] Unsigned offset to the extended jump, from the start of
michael@0:     //               the jump table, in units of SizeOfJumpTableEntry.
michael@0:     //
michael@0:     // The start of the relocation table contains the offset from the code
michael@0:     // buffer to the start of the extended jump table.
michael@0:     //
michael@0:     // Each entry in this table is a jmp [rip], followed by a ud2 to hint to the
michael@0:     // hardware branch predictor that there is no fallthrough, followed by the
michael@0:     // eight bytes containing an immediate address. This comes out to 16 bytes.
michael@0:     //    +1 byte for opcode
michael@0:     //    +1 byte for mod r/m
michael@0:     //    +4 bytes for rip-relative offset (2)
michael@0:     //    +2 bytes for ud2 instruction
michael@0:     //    +8 bytes for 64-bit address
michael@0:     //
michael@0:     static const uint32_t SizeOfExtendedJump = 1 + 1 + 4 + 2 + 8;
michael@0:     static const uint32_t SizeOfJumpTableEntry = 16;
michael@0: 
michael@0:     uint32_t extendedJumpTable_;
michael@0: 
michael@0:     static JitCode *CodeFromJump(JitCode *code, uint8_t *jump);
michael@0: 
michael@0:   private:
michael@0:     void writeRelocation(JmpSrc src, Relocation::Kind reloc);
michael@0:     void addPendingJump(JmpSrc src, ImmPtr target, Relocation::Kind reloc);
michael@0: 
michael@0:   protected:
michael@0:     size_t addPatchableJump(JmpSrc src, Relocation::Kind reloc);
michael@0: 
michael@0:   public:
michael@0:     using AssemblerX86Shared::j;
michael@0:     using AssemblerX86Shared::jmp;
michael@0:     using AssemblerX86Shared::push;
michael@0:     using AssemblerX86Shared::pop;
michael@0: 
michael@0:     static uint8_t *PatchableJumpAddress(JitCode *code, size_t index);
michael@0:     static void PatchJumpEntry(uint8_t *entry, uint8_t *target);
michael@0: 
michael@0:     Assembler()
michael@0:       : extendedJumpTable_(0)
michael@0:     {
michael@0:     }
michael@0: 
michael@0:     static void TraceJumpRelocations(JSTracer *trc, JitCode *code, CompactBufferReader &reader);
michael@0: 
michael@0:     // The buffer is about to be linked, make sure any constant pools or excess
michael@0:     // bookkeeping has been flushed to the instruction stream.
michael@0:     void finish();
michael@0: 
michael@0:     // Copy the assembly code to the given buffer, and perform any pending
michael@0:     // relocations relying on the target address.
michael@0:     void executableCopy(uint8_t *buffer);
michael@0: 
michael@0:     // Actual assembly emitting functions.
michael@0: 
michael@0:     void push(const ImmGCPtr ptr) {
michael@0:         movq(ptr, ScratchReg);
michael@0:         push(ScratchReg);
michael@0:     }
michael@0:     void push(const ImmWord ptr) {
michael@0:         // We often end up with ImmWords that actually fit into int32.
michael@0:         // Be aware of the sign extension behavior.
michael@0:         if (ptr.value <= INT32_MAX) {
michael@0:             push(Imm32(ptr.value));
michael@0:         } else {
michael@0:             movq(ptr, ScratchReg);
michael@0:             push(ScratchReg);
michael@0:         }
michael@0:     }
michael@0:     void push(const ImmPtr &imm) {
michael@0:         push(ImmWord(uintptr_t(imm.value)));
michael@0:     }
michael@0:     void push(const FloatRegister &src) {
michael@0:         subq(Imm32(sizeof(double)), StackPointer);
michael@0:         movsd(src, Address(StackPointer, 0));
michael@0:     }
michael@0:     CodeOffsetLabel pushWithPatch(const ImmWord &word) {
michael@0:         CodeOffsetLabel label = movWithPatch(word, ScratchReg);
michael@0:         push(ScratchReg);
michael@0:         return label;
michael@0:     }
michael@0: 
michael@0:     void pop(const FloatRegister &src) {
michael@0:         movsd(Address(StackPointer, 0), src);
michael@0:         addq(Imm32(sizeof(double)), StackPointer);
michael@0:     }
michael@0: 
michael@0:     CodeOffsetLabel movWithPatch(const ImmWord &word, const Register &dest) {
michael@0:         masm.movq_i64r(word.value, dest.code());
michael@0:         return masm.currentOffset();
michael@0:     }
michael@0:     CodeOffsetLabel movWithPatch(const ImmPtr &imm, const Register &dest) {
michael@0:         return movWithPatch(ImmWord(uintptr_t(imm.value)), dest);
michael@0:     }
michael@0: 
michael@0:     // Load an ImmWord value into a register. Note that this instruction will
michael@0:     // attempt to optimize its immediate field size. When a full 64-bit
michael@0:     // immediate is needed for a relocation, use movWithPatch.
michael@0:     void movq(ImmWord word, const Register &dest) {
michael@0:         // Load a 64-bit immediate into a register. If the value falls into
michael@0:         // certain ranges, we can use specialized instructions which have
michael@0:         // smaller encodings.
michael@0:         if (word.value <= UINT32_MAX) {
michael@0:             // movl has a 32-bit unsigned (effectively) immediate field.
michael@0:             masm.movl_i32r((uint32_t)word.value, dest.code());
michael@0:         } else if ((intptr_t)word.value >= INT32_MIN && (intptr_t)word.value <= INT32_MAX) {
michael@0:             // movq has a 32-bit signed immediate field.
michael@0:             masm.movq_i32r((int32_t)(intptr_t)word.value, dest.code());
michael@0:         } else {
michael@0:             // Otherwise use movabs.
michael@0:             masm.movq_i64r(word.value, dest.code());
michael@0:         }
michael@0:     }
michael@0:     void movq(ImmPtr imm, const Register &dest) {
michael@0:         movq(ImmWord(uintptr_t(imm.value)), dest);
michael@0:     }
michael@0:     void movq(ImmGCPtr ptr, const Register &dest) {
michael@0:         masm.movq_i64r(ptr.value, dest.code());
michael@0:         writeDataRelocation(ptr);
michael@0:     }
michael@0:     void movq(const Operand &src, const Register &dest) {
michael@0:         switch (src.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.movq_rr(src.reg(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.movq_mr(src.disp(), src.base(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_SCALE:
michael@0:             masm.movq_mr(src.disp(), src.base(), src.index(), src.scale(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_ADDRESS32:
michael@0:             masm.movq_mr(src.address(), dest.code());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void movq(const Register &src, const Operand &dest) {
michael@0:         switch (dest.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.movq_rr(src.code(), dest.reg());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.movq_rm(src.code(), dest.disp(), dest.base());
michael@0:             break;
michael@0:           case Operand::MEM_SCALE:
michael@0:             masm.movq_rm(src.code(), dest.disp(), dest.base(), dest.index(), dest.scale());
michael@0:             break;
michael@0:           case Operand::MEM_ADDRESS32:
michael@0:             masm.movq_rm(src.code(), dest.address());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void movq(Imm32 imm32, const Operand &dest) {
michael@0:         switch (dest.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.movl_i32r(imm32.value, dest.reg());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.movq_i32m(imm32.value, dest.disp(), dest.base());
michael@0:             break;
michael@0:           case Operand::MEM_SCALE:
michael@0:             masm.movq_i32m(imm32.value, dest.disp(), dest.base(), dest.index(), dest.scale());
michael@0:             break;
michael@0:           case Operand::MEM_ADDRESS32:
michael@0:             masm.movq_i32m(imm32.value, dest.address());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void movq(const Register &src, const FloatRegister &dest) {
michael@0:         masm.movq_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void movq(const FloatRegister &src, const Register &dest) {
michael@0:         masm.movq_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void movq(const Register &src, const Register &dest) {
michael@0:         masm.movq_rr(src.code(), dest.code());
michael@0:     }
michael@0: 
michael@0:     void xchgq(const Register &src, const Register &dest) {
michael@0:         masm.xchgq_rr(src.code(), dest.code());
michael@0:     }
michael@0: 
michael@0:     void andq(const Register &src, const Register &dest) {
michael@0:         masm.andq_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void andq(Imm32 imm, const Register &dest) {
michael@0:         masm.andq_ir(imm.value, dest.code());
michael@0:     }
michael@0: 
michael@0:     void addq(Imm32 imm, const Register &dest) {
michael@0:         masm.addq_ir(imm.value, dest.code());
michael@0:     }
michael@0:     void addq(Imm32 imm, const Operand &dest) {
michael@0:         switch (dest.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.addq_ir(imm.value, dest.reg());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.addq_im(imm.value, dest.disp(), dest.base());
michael@0:             break;
michael@0:           case Operand::MEM_ADDRESS32:
michael@0:             masm.addq_im(imm.value, dest.address());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void addq(const Register &src, const Register &dest) {
michael@0:         masm.addq_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void addq(const Operand &src, const Register &dest) {
michael@0:         switch (src.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.addq_rr(src.reg(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.addq_mr(src.disp(), src.base(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_ADDRESS32:
michael@0:             masm.addq_mr(src.address(), dest.code());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     void subq(Imm32 imm, const Register &dest) {
michael@0:         masm.subq_ir(imm.value, dest.code());
michael@0:     }
michael@0:     void subq(const Register &src, const Register &dest) {
michael@0:         masm.subq_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void subq(const Operand &src, const Register &dest) {
michael@0:         switch (src.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.subq_rr(src.reg(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.subq_mr(src.disp(), src.base(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_ADDRESS32:
michael@0:             masm.subq_mr(src.address(), dest.code());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void subq(const Register &src, const Operand &dest) {
michael@0:         switch (dest.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.subq_rr(src.code(), dest.reg());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.subq_rm(src.code(), dest.disp(), dest.base());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void shlq(Imm32 imm, const Register &dest) {
michael@0:         masm.shlq_i8r(imm.value, dest.code());
michael@0:     }
michael@0:     void shrq(Imm32 imm, const Register &dest) {
michael@0:         masm.shrq_i8r(imm.value, dest.code());
michael@0:     }
michael@0:     void sarq(Imm32 imm, const Register &dest) {
michael@0:         masm.sarq_i8r(imm.value, dest.code());
michael@0:     }
michael@0:     void orq(Imm32 imm, const Register &dest) {
michael@0:         masm.orq_ir(imm.value, dest.code());
michael@0:     }
michael@0:     void orq(const Register &src, const Register &dest) {
michael@0:         masm.orq_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void orq(const Operand &src, const Register &dest) {
michael@0:         switch (src.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.orq_rr(src.reg(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.orq_mr(src.disp(), src.base(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_ADDRESS32:
michael@0:             masm.orq_mr(src.address(), dest.code());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void xorq(const Register &src, const Register &dest) {
michael@0:         masm.xorq_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void xorq(Imm32 imm, const Register &dest) {
michael@0:         masm.xorq_ir(imm.value, dest.code());
michael@0:     }
michael@0: 
michael@0:     void mov(ImmWord word, const Register &dest) {
michael@0:         // Use xor for setting registers to zero, as it is specially optimized
michael@0:         // for this purpose on modern hardware. Note that it does clobber FLAGS
michael@0:         // though. Use xorl instead of xorq since they are functionally
michael@0:         // equivalent (32-bit instructions zero-extend their results to 64 bits)
michael@0:         // and xorl has a smaller encoding.
michael@0:         if (word.value == 0)
michael@0:             xorl(dest, dest);
michael@0:         else
michael@0:             movq(word, dest);
michael@0:     }
michael@0:     void mov(ImmPtr imm, const Register &dest) {
michael@0:         movq(imm, dest);
michael@0:     }
michael@0:     void mov(AsmJSImmPtr imm, const Register &dest) {
michael@0:         masm.movq_i64r(-1, dest.code());
michael@0:         enoughMemory_ &= append(AsmJSAbsoluteLink(masm.currentOffset(), imm.kind()));
michael@0:     }
michael@0:     void mov(const Operand &src, const Register &dest) {
michael@0:         movq(src, dest);
michael@0:     }
michael@0:     void mov(const Register &src, const Operand &dest) {
michael@0:         movq(src, dest);
michael@0:     }
michael@0:     void mov(const Imm32 &imm32, const Operand &dest) {
michael@0:         movq(imm32, dest);
michael@0:     }
michael@0:     void mov(const Register &src, const Register &dest) {
michael@0:         movq(src, dest);
michael@0:     }
michael@0:     void mov(AbsoluteLabel *label, const Register &dest) {
michael@0:         JS_ASSERT(!label->bound());
michael@0:         // Thread the patch list through the unpatched address word in the
michael@0:         // instruction stream.
michael@0:         masm.movq_i64r(label->prev(), dest.code());
michael@0:         label->setPrev(masm.size());
michael@0:     }
michael@0:     void xchg(const Register &src, const Register &dest) {
michael@0:         xchgq(src, dest);
michael@0:     }
michael@0:     void lea(const Operand &src, const Register &dest) {
michael@0:         switch (src.kind()) {
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.leaq_mr(src.disp(), src.base(), dest.code());
michael@0:             break;
michael@0:           case Operand::MEM_SCALE:
michael@0:             masm.leaq_mr(src.disp(), src.base(), src.index(), src.scale(), dest.code());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexepcted operand kind");
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     CodeOffsetLabel loadRipRelativeInt32(const Register &dest) {
michael@0:         return CodeOffsetLabel(masm.movl_ripr(dest.code()).offset());
michael@0:     }
michael@0:     CodeOffsetLabel loadRipRelativeInt64(const Register &dest) {
michael@0:         return CodeOffsetLabel(masm.movq_ripr(dest.code()).offset());
michael@0:     }
michael@0:     CodeOffsetLabel loadRipRelativeDouble(const FloatRegister &dest) {
michael@0:         return CodeOffsetLabel(masm.movsd_ripr(dest.code()).offset());
michael@0:     }
michael@0:     CodeOffsetLabel storeRipRelativeInt32(const Register &dest) {
michael@0:         return CodeOffsetLabel(masm.movl_rrip(dest.code()).offset());
michael@0:     }
michael@0:     CodeOffsetLabel storeRipRelativeDouble(const FloatRegister &dest) {
michael@0:         return CodeOffsetLabel(masm.movsd_rrip(dest.code()).offset());
michael@0:     }
michael@0:     CodeOffsetLabel leaRipRelative(const Register &dest) {
michael@0:         return CodeOffsetLabel(masm.leaq_rip(dest.code()).offset());
michael@0:     }
michael@0: 
michael@0:     // The below cmpq methods switch the lhs and rhs when it invokes the
michael@0:     // macroassembler to conform with intel standard.  When calling this
michael@0:     // function put the left operand on the left as you would expect.
michael@0:     void cmpq(const Operand &lhs, const Register &rhs) {
michael@0:         switch (lhs.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.cmpq_rr(rhs.code(), lhs.reg());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.cmpq_rm(rhs.code(), lhs.disp(), lhs.base());
michael@0:             break;
michael@0:           case Operand::MEM_ADDRESS32:
michael@0:             masm.cmpq_rm(rhs.code(), lhs.address());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void cmpq(const Operand &lhs, Imm32 rhs) {
michael@0:         switch (lhs.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.cmpq_ir(rhs.value, lhs.reg());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.cmpq_im(rhs.value, lhs.disp(), lhs.base());
michael@0:             break;
michael@0:           case Operand::MEM_ADDRESS32:
michael@0:             masm.cmpq_im(rhs.value, lhs.address());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void cmpq(const Register &lhs, const Operand &rhs) {
michael@0:         switch (rhs.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.cmpq_rr(rhs.reg(), lhs.code());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.cmpq_mr(rhs.disp(), rhs.base(), lhs.code());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     void cmpq(const Register &lhs, const Register &rhs) {
michael@0:         masm.cmpq_rr(rhs.code(), lhs.code());
michael@0:     }
michael@0:     void cmpq(const Register &lhs, Imm32 rhs) {
michael@0:         masm.cmpq_ir(rhs.value, lhs.code());
michael@0:     }
michael@0: 
michael@0:     void testq(const Register &lhs, Imm32 rhs) {
michael@0:         masm.testq_i32r(rhs.value, lhs.code());
michael@0:     }
michael@0:     void testq(const Register &lhs, const Register &rhs) {
michael@0:         masm.testq_rr(rhs.code(), lhs.code());
michael@0:     }
michael@0:     void testq(const Operand &lhs, Imm32 rhs) {
michael@0:         switch (lhs.kind()) {
michael@0:           case Operand::REG:
michael@0:             masm.testq_i32r(rhs.value, lhs.reg());
michael@0:             break;
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             masm.testq_i32m(rhs.value, lhs.disp(), lhs.base());
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:             break;
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     void jmp(ImmPtr target, Relocation::Kind reloc = Relocation::HARDCODED) {
michael@0:         JmpSrc src = masm.jmp();
michael@0:         addPendingJump(src, target, reloc);
michael@0:     }
michael@0:     void j(Condition cond, ImmPtr target,
michael@0:            Relocation::Kind reloc = Relocation::HARDCODED) {
michael@0:         JmpSrc src = masm.jCC(static_cast<JSC::X86Assembler::Condition>(cond));
michael@0:         addPendingJump(src, target, reloc);
michael@0:     }
michael@0: 
michael@0:     void jmp(JitCode *target) {
michael@0:         jmp(ImmPtr(target->raw()), Relocation::JITCODE);
michael@0:     }
michael@0:     void j(Condition cond, JitCode *target) {
michael@0:         j(cond, ImmPtr(target->raw()), Relocation::JITCODE);
michael@0:     }
michael@0:     void call(JitCode *target) {
michael@0:         JmpSrc src = masm.call();
michael@0:         addPendingJump(src, ImmPtr(target->raw()), Relocation::JITCODE);
michael@0:     }
michael@0: 
michael@0:     // Emit a CALL or CMP (nop) instruction. ToggleCall can be used to patch
michael@0:     // this instruction.
michael@0:     CodeOffsetLabel toggledCall(JitCode *target, bool enabled) {
michael@0:         CodeOffsetLabel offset(size());
michael@0:         JmpSrc src = enabled ? masm.call() : masm.cmp_eax();
michael@0:         addPendingJump(src, ImmPtr(target->raw()), Relocation::JITCODE);
michael@0:         JS_ASSERT(size() - offset.offset() == ToggledCallSize());
michael@0:         return offset;
michael@0:     }
michael@0: 
michael@0:     static size_t ToggledCallSize() {
michael@0:         // Size of a call instruction.
michael@0:         return 5;
michael@0:     }
michael@0: 
michael@0:     // Do not mask shared implementations.
michael@0:     using AssemblerX86Shared::call;
michael@0: 
michael@0:     void cvttsd2sq(const FloatRegister &src, const Register &dest) {
michael@0:         masm.cvttsd2sq_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void cvttss2sq(const FloatRegister &src, const Register &dest) {
michael@0:         masm.cvttss2sq_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void cvtsq2sd(const Register &src, const FloatRegister &dest) {
michael@0:         masm.cvtsq2sd_rr(src.code(), dest.code());
michael@0:     }
michael@0:     void cvtsq2ss(const Register &src, const FloatRegister &dest) {
michael@0:         masm.cvtsq2ss_rr(src.code(), dest.code());
michael@0:     }
michael@0: };
michael@0: 
michael@0: static inline void
michael@0: PatchJump(CodeLocationJump jump, CodeLocationLabel label)
michael@0: {
michael@0:     if (JSC::X86Assembler::canRelinkJump(jump.raw(), label.raw())) {
michael@0:         JSC::X86Assembler::setRel32(jump.raw(), label.raw());
michael@0:     } else {
michael@0:         JSC::X86Assembler::setRel32(jump.raw(), jump.jumpTableEntry());
michael@0:         Assembler::PatchJumpEntry(jump.jumpTableEntry(), label.raw());
michael@0:     }
michael@0: }
michael@0: 
michael@0: static inline bool
michael@0: GetIntArgReg(uint32_t intArg, uint32_t floatArg, Register *out)
michael@0: {
michael@0: #if defined(_WIN64)
michael@0:     uint32_t arg = intArg + floatArg;
michael@0: #else
michael@0:     uint32_t arg = intArg;
michael@0: #endif
michael@0:     if (arg >= NumIntArgRegs)
michael@0:         return false;
michael@0:     *out = IntArgRegs[arg];
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: // Get a register in which we plan to put a quantity that will be used as an
michael@0: // integer argument.  This differs from GetIntArgReg in that if we have no more
michael@0: // actual argument registers to use we will fall back on using whatever
michael@0: // CallTempReg* don't overlap the argument registers, and only fail once those
michael@0: // run out too.
michael@0: static inline bool
michael@0: GetTempRegForIntArg(uint32_t usedIntArgs, uint32_t usedFloatArgs, Register *out)
michael@0: {
michael@0:     if (GetIntArgReg(usedIntArgs, usedFloatArgs, out))
michael@0:         return true;
michael@0:     // Unfortunately, we have to assume things about the point at which
michael@0:     // GetIntArgReg returns false, because we need to know how many registers it
michael@0:     // can allocate.
michael@0: #if defined(_WIN64)
michael@0:     uint32_t arg = usedIntArgs + usedFloatArgs;
michael@0: #else
michael@0:     uint32_t arg = usedIntArgs;
michael@0: #endif
michael@0:     arg -= NumIntArgRegs;
michael@0:     if (arg >= NumCallTempNonArgRegs)
michael@0:         return false;
michael@0:     *out = CallTempNonArgRegs[arg];
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: static inline bool
michael@0: GetFloatArgReg(uint32_t intArg, uint32_t floatArg, FloatRegister *out)
michael@0: {
michael@0: #if defined(_WIN64)
michael@0:     uint32_t arg = intArg + floatArg;
michael@0: #else
michael@0:     uint32_t arg = floatArg;
michael@0: #endif
michael@0:     if (floatArg >= NumFloatArgRegs)
michael@0:         return false;
michael@0:     *out = FloatArgRegs[arg];
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: } // namespace jit
michael@0: } // namespace js
michael@0: 
michael@0: #endif /* jit_x64_Assembler_x64_h */