michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
michael@0:  * vim: set ts=8 sts=4 et sw=4 tw=99:
michael@0:  * This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #ifndef jit_x64_MacroAssembler_x64_h
michael@0: #define jit_x64_MacroAssembler_x64_h
michael@0: 
michael@0: #include "jit/IonFrames.h"
michael@0: #include "jit/MoveResolver.h"
michael@0: #include "jit/shared/MacroAssembler-x86-shared.h"
michael@0: 
michael@0: namespace js {
michael@0: namespace jit {
michael@0: 
michael@0: struct ImmShiftedTag : public ImmWord
michael@0: {
michael@0:     ImmShiftedTag(JSValueShiftedTag shtag)
michael@0:       : ImmWord((uintptr_t)shtag)
michael@0:     { }
michael@0: 
michael@0:     ImmShiftedTag(JSValueType type)
michael@0:       : ImmWord(uintptr_t(JSValueShiftedTag(JSVAL_TYPE_TO_SHIFTED_TAG(type))))
michael@0:     { }
michael@0: };
michael@0: 
michael@0: struct ImmTag : public Imm32
michael@0: {
michael@0:     ImmTag(JSValueTag tag)
michael@0:       : Imm32(tag)
michael@0:     { }
michael@0: };
michael@0: 
michael@0: class MacroAssemblerX64 : public MacroAssemblerX86Shared
michael@0: {
michael@0:     // Number of bytes the stack is adjusted inside a call to C. Calls to C may
michael@0:     // not be nested.
michael@0:     bool inCall_;
michael@0:     uint32_t args_;
michael@0:     uint32_t passedIntArgs_;
michael@0:     uint32_t passedFloatArgs_;
michael@0:     uint32_t stackForCall_;
michael@0:     bool dynamicAlignment_;
michael@0:     bool enoughMemory_;
michael@0: 
michael@0:     // These use SystemAllocPolicy since asm.js releases memory after each
michael@0:     // function is compiled, and these need to live until after all functions
michael@0:     // are compiled.
michael@0:     struct Double {
michael@0:         double value;
michael@0:         NonAssertingLabel uses;
michael@0:         Double(double value) : value(value) {}
michael@0:     };
michael@0:     Vector<Double, 0, SystemAllocPolicy> doubles_;
michael@0: 
michael@0:     typedef HashMap<double, size_t, DefaultHasher<double>, SystemAllocPolicy> DoubleMap;
michael@0:     DoubleMap doubleMap_;
michael@0: 
michael@0:     struct Float {
michael@0:         float value;
michael@0:         NonAssertingLabel uses;
michael@0:         Float(float value) : value(value) {}
michael@0:     };
michael@0:     Vector<Float, 0, SystemAllocPolicy> floats_;
michael@0: 
michael@0:     typedef HashMap<float, size_t, DefaultHasher<float>, SystemAllocPolicy> FloatMap;
michael@0:     FloatMap floatMap_;
michael@0: 
michael@0:     void setupABICall(uint32_t arg);
michael@0: 
michael@0:   protected:
michael@0:     MoveResolver moveResolver_;
michael@0: 
michael@0:   public:
michael@0:     using MacroAssemblerX86Shared::call;
michael@0:     using MacroAssemblerX86Shared::Push;
michael@0:     using MacroAssemblerX86Shared::Pop;
michael@0:     using MacroAssemblerX86Shared::callWithExitFrame;
michael@0:     using MacroAssemblerX86Shared::branch32;
michael@0:     using MacroAssemblerX86Shared::load32;
michael@0:     using MacroAssemblerX86Shared::store32;
michael@0: 
michael@0:     MacroAssemblerX64()
michael@0:       : inCall_(false),
michael@0:         enoughMemory_(true)
michael@0:     {
michael@0:     }
michael@0: 
michael@0:     // The buffer is about to be linked, make sure any constant pools or excess
michael@0:     // bookkeeping has been flushed to the instruction stream.
michael@0:     void finish();
michael@0: 
michael@0:     bool oom() const {
michael@0:         return MacroAssemblerX86Shared::oom() || !enoughMemory_;
michael@0:     }
michael@0: 
michael@0:     /////////////////////////////////////////////////////////////////
michael@0:     // X64 helpers.
michael@0:     /////////////////////////////////////////////////////////////////
michael@0:     void call(ImmWord target) {
michael@0:         mov(target, rax);
michael@0:         call(rax);
michael@0:     }
michael@0:     void call(ImmPtr target) {
michael@0:         call(ImmWord(uintptr_t(target.value)));
michael@0:     }
michael@0:     void call(AsmJSImmPtr target) {
michael@0:         mov(target, rax);
michael@0:         call(rax);
michael@0:     }
michael@0: 
michael@0:     void call(const CallSiteDesc &desc, AsmJSImmPtr target) {
michael@0:         call(target);
michael@0:         appendCallSite(desc);
michael@0:     }
michael@0:     void callExit(AsmJSImmPtr target, uint32_t stackArgBytes) {
michael@0:         call(CallSiteDesc::Exit(), target);
michael@0:     }
michael@0: 
michael@0:     // Refers to the upper 32 bits of a 64-bit Value operand.
michael@0:     // On x86_64, the upper 32 bits do not necessarily only contain the type.
michael@0:     Operand ToUpper32(Operand base) {
michael@0:         switch (base.kind()) {
michael@0:           case Operand::MEM_REG_DISP:
michael@0:             return Operand(Register::FromCode(base.base()), base.disp() + 4);
michael@0: 
michael@0:           case Operand::MEM_SCALE:
michael@0:             return Operand(Register::FromCode(base.base()), Register::FromCode(base.index()),
michael@0:                            base.scale(), base.disp() + 4);
michael@0: 
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("unexpected operand kind");
michael@0:         }
michael@0:     }
michael@0:     static inline Operand ToUpper32(const Address &address) {
michael@0:         return Operand(address.base, address.offset + 4);
michael@0:     }
michael@0:     static inline Operand ToUpper32(const BaseIndex &address) {
michael@0:         return Operand(address.base, address.index, address.scale, address.offset + 4);
michael@0:     }
michael@0: 
michael@0:     uint32_t Upper32Of(JSValueShiftedTag tag) {
michael@0:         union { // Implemented in this way to appease MSVC++.
michael@0:             uint64_t tag;
michael@0:             struct {
michael@0:                 uint32_t lo32;
michael@0:                 uint32_t hi32;
michael@0:             } s;
michael@0:         } e;
michael@0:         e.tag = tag;
michael@0:         return e.s.hi32;
michael@0:     }
michael@0: 
michael@0:     JSValueShiftedTag GetShiftedTag(JSValueType type) {
michael@0:         return (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
michael@0:     }
michael@0: 
michael@0:     /////////////////////////////////////////////////////////////////
michael@0:     // X86/X64-common interface.
michael@0:     /////////////////////////////////////////////////////////////////
michael@0:     void storeValue(ValueOperand val, Operand dest) {
michael@0:         movq(val.valueReg(), dest);
michael@0:     }
michael@0:     void storeValue(ValueOperand val, const Address &dest) {
michael@0:         storeValue(val, Operand(dest));
michael@0:     }
michael@0:     template <typename T>
michael@0:     void storeValue(JSValueType type, Register reg, const T &dest) {
michael@0:         // Value types with 32-bit payloads can be emitted as two 32-bit moves.
michael@0:         if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
michael@0:             movl(reg, Operand(dest));
michael@0:             movl(Imm32(Upper32Of(GetShiftedTag(type))), ToUpper32(Operand(dest)));
michael@0:         } else {
michael@0:             boxValue(type, reg, ScratchReg);
michael@0:             movq(ScratchReg, Operand(dest));
michael@0:         }
michael@0:     }
michael@0:     template <typename T>
michael@0:     void storeValue(const Value &val, const T &dest) {
michael@0:         jsval_layout jv = JSVAL_TO_IMPL(val);
michael@0:         if (val.isMarkable()) {
michael@0:             movWithPatch(ImmWord(jv.asBits), ScratchReg);
michael@0:             writeDataRelocation(val);
michael@0:         } else {
michael@0:             mov(ImmWord(jv.asBits), ScratchReg);
michael@0:         }
michael@0:         movq(ScratchReg, Operand(dest));
michael@0:     }
michael@0:     void storeValue(ValueOperand val, BaseIndex dest) {
michael@0:         storeValue(val, Operand(dest));
michael@0:     }
michael@0:     void loadValue(Operand src, ValueOperand val) {
michael@0:         movq(src, val.valueReg());
michael@0:     }
michael@0:     void loadValue(Address src, ValueOperand val) {
michael@0:         loadValue(Operand(src), val);
michael@0:     }
michael@0:     void loadValue(const BaseIndex &src, ValueOperand val) {
michael@0:         loadValue(Operand(src), val);
michael@0:     }
michael@0:     void tagValue(JSValueType type, Register payload, ValueOperand dest) {
michael@0:         JS_ASSERT(dest.valueReg() != ScratchReg);
michael@0:         if (payload != dest.valueReg())
michael@0:             movq(payload, dest.valueReg());
michael@0:         mov(ImmShiftedTag(type), ScratchReg);
michael@0:         orq(ScratchReg, dest.valueReg());
michael@0:     }
michael@0:     void pushValue(ValueOperand val) {
michael@0:         push(val.valueReg());
michael@0:     }
michael@0:     void Push(const ValueOperand &val) {
michael@0:         pushValue(val);
michael@0:         framePushed_ += sizeof(Value);
michael@0:     }
michael@0:     void popValue(ValueOperand val) {
michael@0:         pop(val.valueReg());
michael@0:     }
michael@0:     void pushValue(const Value &val) {
michael@0:         jsval_layout jv = JSVAL_TO_IMPL(val);
michael@0:         push(ImmWord(jv.asBits));
michael@0:     }
michael@0:     void pushValue(JSValueType type, Register reg) {
michael@0:         boxValue(type, reg, ScratchReg);
michael@0:         push(ScratchReg);
michael@0:     }
michael@0:     void pushValue(const Address &addr) {
michael@0:         push(Operand(addr));
michael@0:     }
michael@0:     void Pop(const ValueOperand &val) {
michael@0:         popValue(val);
michael@0:         framePushed_ -= sizeof(Value);
michael@0:     }
michael@0: 
michael@0:     void moveValue(const Value &val, const Register &dest) {
michael@0:         jsval_layout jv = JSVAL_TO_IMPL(val);
michael@0:         movWithPatch(ImmWord(jv.asBits), dest);
michael@0:         writeDataRelocation(val);
michael@0:     }
michael@0:     void moveValue(const Value &src, const ValueOperand &dest) {
michael@0:         moveValue(src, dest.valueReg());
michael@0:     }
michael@0:     void moveValue(const ValueOperand &src, const ValueOperand &dest) {
michael@0:         if (src.valueReg() != dest.valueReg())
michael@0:             movq(src.valueReg(), dest.valueReg());
michael@0:     }
michael@0:     void boxValue(JSValueType type, Register src, Register dest) {
michael@0:         JS_ASSERT(src != dest);
michael@0: 
michael@0:         JSValueShiftedTag tag = (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
michael@0: #ifdef DEBUG
michael@0:         if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
michael@0:             Label upper32BitsZeroed;
michael@0:             movePtr(ImmWord(UINT32_MAX), dest);
michael@0:             branchPtr(Assembler::BelowOrEqual, src, dest, &upper32BitsZeroed);
michael@0:             breakpoint();
michael@0:             bind(&upper32BitsZeroed);
michael@0:         }
michael@0: #endif
michael@0:         mov(ImmShiftedTag(tag), dest);
michael@0:         orq(src, dest);
michael@0:     }
michael@0: 
michael@0:     Condition testUndefined(Condition cond, Register tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, ImmTag(JSVAL_TAG_UNDEFINED));
michael@0:         return cond;
michael@0:     }
michael@0:     Condition testInt32(Condition cond, Register tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, ImmTag(JSVAL_TAG_INT32));
michael@0:         return cond;
michael@0:     }
michael@0:     Condition testBoolean(Condition cond, Register tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, ImmTag(JSVAL_TAG_BOOLEAN));
michael@0:         return cond;
michael@0:     }
michael@0:     Condition testNull(Condition cond, Register tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, ImmTag(JSVAL_TAG_NULL));
michael@0:         return cond;
michael@0:     }
michael@0:     Condition testString(Condition cond, Register tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, ImmTag(JSVAL_TAG_STRING));
michael@0:         return cond;
michael@0:     }
michael@0:     Condition testObject(Condition cond, Register tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, ImmTag(JSVAL_TAG_OBJECT));
michael@0:         return cond;
michael@0:     }
michael@0:     Condition testDouble(Condition cond, Register tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, Imm32(JSVAL_TAG_MAX_DOUBLE));
michael@0:         return cond == Equal ? BelowOrEqual : Above;
michael@0:     }
michael@0:     Condition testNumber(Condition cond, Register tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, Imm32(JSVAL_UPPER_INCL_TAG_OF_NUMBER_SET));
michael@0:         return cond == Equal ? BelowOrEqual : Above;
michael@0:     }
michael@0:     Condition testGCThing(Condition cond, Register tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, Imm32(JSVAL_LOWER_INCL_TAG_OF_GCTHING_SET));
michael@0:         return cond == Equal ? AboveOrEqual : Below;
michael@0:     }
michael@0: 
michael@0:     Condition testMagic(Condition cond, const Register &tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, ImmTag(JSVAL_TAG_MAGIC));
michael@0:         return cond;
michael@0:     }
michael@0:     Condition testError(Condition cond, const Register &tag) {
michael@0:         return testMagic(cond, tag);
michael@0:     }
michael@0:     Condition testPrimitive(Condition cond, const Register &tag) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(tag, ImmTag(JSVAL_UPPER_EXCL_TAG_OF_PRIMITIVE_SET));
michael@0:         return cond == Equal ? Below : AboveOrEqual;
michael@0:     }
michael@0: 
michael@0:     Condition testUndefined(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testUndefined(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testInt32(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testInt32(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testBoolean(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testBoolean(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testDouble(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testDouble(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testNumber(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testNumber(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testNull(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testNull(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testString(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testString(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testObject(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testObject(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testGCThing(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testGCThing(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testPrimitive(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testPrimitive(cond, ScratchReg);
michael@0:     }
michael@0: 
michael@0: 
michael@0:     Condition testUndefined(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testUndefined(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testInt32(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testInt32(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testBoolean(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testBoolean(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testDouble(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testDouble(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testNumber(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testNumber(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testNull(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testNull(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testString(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testString(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testObject(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testObject(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testPrimitive(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testPrimitive(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testGCThing(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testGCThing(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testMagic(Condition cond, const Address &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testMagic(cond, ScratchReg);
michael@0:     }
michael@0: 
michael@0: 
michael@0:     Condition testUndefined(Condition cond, const BaseIndex &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testUndefined(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testNull(Condition cond, const BaseIndex &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testNull(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testBoolean(Condition cond, const BaseIndex &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testBoolean(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testString(Condition cond, const BaseIndex &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testString(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testInt32(Condition cond, const BaseIndex &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testInt32(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testObject(Condition cond, const BaseIndex &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testObject(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testDouble(Condition cond, const BaseIndex &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testDouble(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testMagic(Condition cond, const BaseIndex &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testMagic(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testGCThing(Condition cond, const BaseIndex &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testGCThing(cond, ScratchReg);
michael@0:     }
michael@0: 
michael@0:     Condition isMagic(Condition cond, const ValueOperand &src, JSWhyMagic why) {
michael@0:         uint64_t magic = MagicValue(why).asRawBits();
michael@0:         cmpPtr(src.valueReg(), ImmWord(magic));
michael@0:         return cond;
michael@0:     }
michael@0: 
michael@0:     void cmpPtr(const Register &lhs, const ImmWord rhs) {
michael@0:         JS_ASSERT(lhs != ScratchReg);
michael@0:         mov(rhs, ScratchReg);
michael@0:         cmpq(lhs, ScratchReg);
michael@0:     }
michael@0:     void cmpPtr(const Register &lhs, const ImmPtr rhs) {
michael@0:         cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
michael@0:     }
michael@0:     void cmpPtr(const Register &lhs, const ImmGCPtr rhs) {
michael@0:         JS_ASSERT(lhs != ScratchReg);
michael@0:         movq(rhs, ScratchReg);
michael@0:         cmpq(lhs, ScratchReg);
michael@0:     }
michael@0:     void cmpPtr(const Register &lhs, const Imm32 rhs) {
michael@0:         cmpq(lhs, rhs);
michael@0:     }
michael@0:     void cmpPtr(const Operand &lhs, const ImmGCPtr rhs) {
michael@0:         movq(rhs, ScratchReg);
michael@0:         cmpq(lhs, ScratchReg);
michael@0:     }
michael@0:     void cmpPtr(const Operand &lhs, const ImmWord rhs) {
michael@0:         if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
michael@0:             cmpq(lhs, Imm32((int32_t)rhs.value));
michael@0:         } else {
michael@0:             mov(rhs, ScratchReg);
michael@0:             cmpq(lhs, ScratchReg);
michael@0:         }
michael@0:     }
michael@0:     void cmpPtr(const Operand &lhs, const ImmPtr rhs) {
michael@0:         cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
michael@0:     }
michael@0:     void cmpPtr(const Address &lhs, const ImmGCPtr rhs) {
michael@0:         cmpPtr(Operand(lhs), rhs);
michael@0:     }
michael@0:     void cmpPtr(const Address &lhs, const ImmWord rhs) {
michael@0:         cmpPtr(Operand(lhs), rhs);
michael@0:     }
michael@0:     void cmpPtr(const Address &lhs, const ImmPtr rhs) {
michael@0:         cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
michael@0:     }
michael@0:     void cmpPtr(const Operand &lhs, const Register &rhs) {
michael@0:         cmpq(lhs, rhs);
michael@0:     }
michael@0:     void cmpPtr(const Operand &lhs, const Imm32 rhs) {
michael@0:         cmpq(lhs, rhs);
michael@0:     }
michael@0:     void cmpPtr(const Address &lhs, const Register &rhs) {
michael@0:         cmpPtr(Operand(lhs), rhs);
michael@0:     }
michael@0:     void cmpPtr(const Register &lhs, const Register &rhs) {
michael@0:         return cmpq(lhs, rhs);
michael@0:     }
michael@0:     void testPtr(const Register &lhs, const Register &rhs) {
michael@0:         testq(lhs, rhs);
michael@0:     }
michael@0: 
michael@0:     template <typename T1, typename T2>
michael@0:     void cmpPtrSet(Assembler::Condition cond, T1 lhs, T2 rhs, const Register &dest)
michael@0:     {
michael@0:         cmpPtr(lhs, rhs);
michael@0:         emitSet(cond, dest);
michael@0:     }
michael@0: 
michael@0:     /////////////////////////////////////////////////////////////////
michael@0:     // Common interface.
michael@0:     /////////////////////////////////////////////////////////////////
michael@0:     void reserveStack(uint32_t amount) {
michael@0:         if (amount)
michael@0:             subq(Imm32(amount), StackPointer);
michael@0:         framePushed_ += amount;
michael@0:     }
michael@0:     void freeStack(uint32_t amount) {
michael@0:         JS_ASSERT(amount <= framePushed_);
michael@0:         if (amount)
michael@0:             addq(Imm32(amount), StackPointer);
michael@0:         framePushed_ -= amount;
michael@0:     }
michael@0:     void freeStack(Register amount) {
michael@0:         addq(amount, StackPointer);
michael@0:     }
michael@0: 
michael@0:     void addPtr(const Register &src, const Register &dest) {
michael@0:         addq(src, dest);
michael@0:     }
michael@0:     void addPtr(Imm32 imm, const Register &dest) {
michael@0:         addq(imm, dest);
michael@0:     }
michael@0:     void addPtr(Imm32 imm, const Address &dest) {
michael@0:         addq(imm, Operand(dest));
michael@0:     }
michael@0:     void addPtr(Imm32 imm, const Operand &dest) {
michael@0:         addq(imm, dest);
michael@0:     }
michael@0:     void addPtr(ImmWord imm, const Register &dest) {
michael@0:         JS_ASSERT(dest != ScratchReg);
michael@0:         if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
michael@0:             addq(Imm32((int32_t)imm.value), dest);
michael@0:         } else {
michael@0:             mov(imm, ScratchReg);
michael@0:             addq(ScratchReg, dest);
michael@0:         }
michael@0:     }
michael@0:     void addPtr(ImmPtr imm, const Register &dest) {
michael@0:         addPtr(ImmWord(uintptr_t(imm.value)), dest);
michael@0:     }
michael@0:     void addPtr(const Address &src, const Register &dest) {
michael@0:         addq(Operand(src), dest);
michael@0:     }
michael@0:     void subPtr(Imm32 imm, const Register &dest) {
michael@0:         subq(imm, dest);
michael@0:     }
michael@0:     void subPtr(const Register &src, const Register &dest) {
michael@0:         subq(src, dest);
michael@0:     }
michael@0:     void subPtr(const Address &addr, const Register &dest) {
michael@0:         subq(Operand(addr), dest);
michael@0:     }
michael@0:     void subPtr(const Register &src, const Address &dest) {
michael@0:         subq(src, Operand(dest));
michael@0:     }
michael@0: 
michael@0:     void branch32(Condition cond, const AbsoluteAddress &lhs, Imm32 rhs, Label *label) {
michael@0:         if (JSC::X86Assembler::isAddressImmediate(lhs.addr)) {
michael@0:             branch32(cond, Operand(lhs), rhs, label);
michael@0:         } else {
michael@0:             mov(ImmPtr(lhs.addr), ScratchReg);
michael@0:             branch32(cond, Address(ScratchReg, 0), rhs, label);
michael@0:         }
michael@0:     }
michael@0:     void branch32(Condition cond, const AbsoluteAddress &lhs, Register rhs, Label *label) {
michael@0:         if (JSC::X86Assembler::isAddressImmediate(lhs.addr)) {
michael@0:             branch32(cond, Operand(lhs), rhs, label);
michael@0:         } else {
michael@0:             mov(ImmPtr(lhs.addr), ScratchReg);
michael@0:             branch32(cond, Address(ScratchReg, 0), rhs, label);
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     // Specialization for AbsoluteAddress.
michael@0:     void branchPtr(Condition cond, const AbsoluteAddress &addr, const Register &ptr, Label *label) {
michael@0:         JS_ASSERT(ptr != ScratchReg);
michael@0:         if (JSC::X86Assembler::isAddressImmediate(addr.addr)) {
michael@0:             branchPtr(cond, Operand(addr), ptr, label);
michael@0:         } else {
michael@0:             mov(ImmPtr(addr.addr), ScratchReg);
michael@0:             branchPtr(cond, Operand(ScratchReg, 0x0), ptr, label);
michael@0:         }
michael@0:     }
michael@0:     void branchPtr(Condition cond, const AsmJSAbsoluteAddress &addr, const Register &ptr, Label *label) {
michael@0:         JS_ASSERT(ptr != ScratchReg);
michael@0:         mov(AsmJSImmPtr(addr.kind()), ScratchReg);
michael@0:         branchPtr(cond, Operand(ScratchReg, 0x0), ptr, label);
michael@0:     }
michael@0: 
michael@0:     void branchPrivatePtr(Condition cond, Address lhs, ImmPtr ptr, Label *label) {
michael@0:         branchPtr(cond, lhs, ImmWord(uintptr_t(ptr.value) >> 1), label);
michael@0:     }
michael@0: 
michael@0:     void branchPrivatePtr(Condition cond, Address lhs, Register ptr, Label *label) {
michael@0:         if (ptr != ScratchReg)
michael@0:             movePtr(ptr, ScratchReg);
michael@0:         rshiftPtr(Imm32(1), ScratchReg);
michael@0:         branchPtr(cond, lhs, ScratchReg, label);
michael@0:     }
michael@0: 
michael@0:     template <typename T, typename S>
michael@0:     void branchPtr(Condition cond, T lhs, S ptr, Label *label) {
michael@0:         cmpPtr(Operand(lhs), ptr);
michael@0:         j(cond, label);
michael@0:     }
michael@0: 
michael@0:     CodeOffsetJump jumpWithPatch(RepatchLabel *label) {
michael@0:         JmpSrc src = jmpSrc(label);
michael@0:         return CodeOffsetJump(size(), addPatchableJump(src, Relocation::HARDCODED));
michael@0:     }
michael@0: 
michael@0:     CodeOffsetJump jumpWithPatch(RepatchLabel *label, Condition cond) {
michael@0:         JmpSrc src = jSrc(cond, label);
michael@0:         return CodeOffsetJump(size(), addPatchableJump(src, Relocation::HARDCODED));
michael@0:     }
michael@0: 
michael@0:     template <typename S, typename T>
michael@0:     CodeOffsetJump branchPtrWithPatch(Condition cond, S lhs, T ptr, RepatchLabel *label) {
michael@0:         cmpPtr(lhs, ptr);
michael@0:         return jumpWithPatch(label, cond);
michael@0:     }
michael@0:     void branchPtr(Condition cond, Register lhs, Register rhs, Label *label) {
michael@0:         cmpPtr(lhs, rhs);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestPtr(Condition cond, Register lhs, Register rhs, Label *label) {
michael@0:         testq(lhs, rhs);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestPtr(Condition cond, Register lhs, Imm32 imm, Label *label) {
michael@0:         testq(lhs, imm);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestPtr(Condition cond, const Address &lhs, Imm32 imm, Label *label) {
michael@0:         testq(Operand(lhs), imm);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void decBranchPtr(Condition cond, const Register &lhs, Imm32 imm, Label *label) {
michael@0:         subPtr(imm, lhs);
michael@0:         j(cond, label);
michael@0:     }
michael@0: 
michael@0:     void movePtr(const Register &src, const Register &dest) {
michael@0:         movq(src, dest);
michael@0:     }
michael@0:     void movePtr(const Register &src, const Operand &dest) {
michael@0:         movq(src, dest);
michael@0:     }
michael@0:     void movePtr(ImmWord imm, Register dest) {
michael@0:         mov(imm, dest);
michael@0:     }
michael@0:     void movePtr(ImmPtr imm, Register dest) {
michael@0:         mov(imm, dest);
michael@0:     }
michael@0:     void movePtr(AsmJSImmPtr imm, const Register &dest) {
michael@0:         mov(imm, dest);
michael@0:     }
michael@0:     void movePtr(ImmGCPtr imm, Register dest) {
michael@0:         movq(imm, dest);
michael@0:     }
michael@0:     void loadPtr(const AbsoluteAddress &address, Register dest) {
michael@0:         if (JSC::X86Assembler::isAddressImmediate(address.addr)) {
michael@0:             movq(Operand(address), dest);
michael@0:         } else {
michael@0:             mov(ImmPtr(address.addr), ScratchReg);
michael@0:             loadPtr(Address(ScratchReg, 0x0), dest);
michael@0:         }
michael@0:     }
michael@0:     void loadPtr(const Address &address, Register dest) {
michael@0:         movq(Operand(address), dest);
michael@0:     }
michael@0:     void loadPtr(const Operand &src, Register dest) {
michael@0:         movq(src, dest);
michael@0:     }
michael@0:     void loadPtr(const BaseIndex &src, Register dest) {
michael@0:         movq(Operand(src), dest);
michael@0:     }
michael@0:     void loadPrivate(const Address &src, Register dest) {
michael@0:         loadPtr(src, dest);
michael@0:         shlq(Imm32(1), dest);
michael@0:     }
michael@0:     void load32(const AbsoluteAddress &address, Register dest) {
michael@0:         if (JSC::X86Assembler::isAddressImmediate(address.addr)) {
michael@0:             movl(Operand(address), dest);
michael@0:         } else {
michael@0:             mov(ImmPtr(address.addr), ScratchReg);
michael@0:             load32(Address(ScratchReg, 0x0), dest);
michael@0:         }
michael@0:     }
michael@0:     void storePtr(ImmWord imm, const Address &address) {
michael@0:         if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
michael@0:             movq(Imm32((int32_t)imm.value), Operand(address));
michael@0:         } else {
michael@0:             mov(imm, ScratchReg);
michael@0:             movq(ScratchReg, Operand(address));
michael@0:         }
michael@0:     }
michael@0:     void storePtr(ImmPtr imm, const Address &address) {
michael@0:         storePtr(ImmWord(uintptr_t(imm.value)), address);
michael@0:     }
michael@0:     void storePtr(ImmGCPtr imm, const Address &address) {
michael@0:         movq(imm, ScratchReg);
michael@0:         movq(ScratchReg, Operand(address));
michael@0:     }
michael@0:     void storePtr(Register src, const Address &address) {
michael@0:         movq(src, Operand(address));
michael@0:     }
michael@0:     void storePtr(Register src, const Operand &dest) {
michael@0:         movq(src, dest);
michael@0:     }
michael@0:     void storePtr(const Register &src, const AbsoluteAddress &address) {
michael@0:         if (JSC::X86Assembler::isAddressImmediate(address.addr)) {
michael@0:             movq(src, Operand(address));
michael@0:         } else {
michael@0:             mov(ImmPtr(address.addr), ScratchReg);
michael@0:             storePtr(src, Address(ScratchReg, 0x0));
michael@0:         }
michael@0:     }
michael@0:     void store32(const Register &src, const AbsoluteAddress &address) {
michael@0:         if (JSC::X86Assembler::isAddressImmediate(address.addr)) {
michael@0:             movl(src, Operand(address));
michael@0:         } else {
michael@0:             mov(ImmPtr(address.addr), ScratchReg);
michael@0:             store32(src, Address(ScratchReg, 0x0));
michael@0:         }
michael@0:     }
michael@0:     void rshiftPtr(Imm32 imm, Register dest) {
michael@0:         shrq(imm, dest);
michael@0:     }
michael@0:     void lshiftPtr(Imm32 imm, Register dest) {
michael@0:         shlq(imm, dest);
michael@0:     }
michael@0:     void xorPtr(Imm32 imm, Register dest) {
michael@0:         xorq(imm, dest);
michael@0:     }
michael@0:     void xorPtr(Register src, Register dest) {
michael@0:         xorq(src, dest);
michael@0:     }
michael@0:     void orPtr(Imm32 imm, Register dest) {
michael@0:         orq(imm, dest);
michael@0:     }
michael@0:     void orPtr(Register src, Register dest) {
michael@0:         orq(src, dest);
michael@0:     }
michael@0:     void andPtr(Imm32 imm, Register dest) {
michael@0:         andq(imm, dest);
michael@0:     }
michael@0:     void andPtr(Register src, Register dest) {
michael@0:         andq(src, dest);
michael@0:     }
michael@0: 
michael@0:     void splitTag(Register src, Register dest) {
michael@0:         if (src != dest)
michael@0:             movq(src, dest);
michael@0:         shrq(Imm32(JSVAL_TAG_SHIFT), dest);
michael@0:     }
michael@0:     void splitTag(const ValueOperand &operand, const Register &dest) {
michael@0:         splitTag(operand.valueReg(), dest);
michael@0:     }
michael@0:     void splitTag(const Operand &operand, const Register &dest) {
michael@0:         movq(operand, dest);
michael@0:         shrq(Imm32(JSVAL_TAG_SHIFT), dest);
michael@0:     }
michael@0:     void splitTag(const Address &operand, const Register &dest) {
michael@0:         splitTag(Operand(operand), dest);
michael@0:     }
michael@0:     void splitTag(const BaseIndex &operand, const Register &dest) {
michael@0:         splitTag(Operand(operand), dest);
michael@0:     }
michael@0: 
michael@0:     // Extracts the tag of a value and places it in ScratchReg.
michael@0:     Register splitTagForTest(const ValueOperand &value) {
michael@0:         splitTag(value, ScratchReg);
michael@0:         return ScratchReg;
michael@0:     }
michael@0:     void cmpTag(const ValueOperand &operand, ImmTag tag) {
michael@0:         Register reg = splitTagForTest(operand);
michael@0:         cmpl(reg, tag);
michael@0:     }
michael@0: 
michael@0:     void branchTestUndefined(Condition cond, Register tag, Label *label) {
michael@0:         cond = testUndefined(cond, tag);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestInt32(Condition cond, Register tag, Label *label) {
michael@0:         cond = testInt32(cond, tag);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestDouble(Condition cond, Register tag, Label *label) {
michael@0:         cond = testDouble(cond, tag);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestBoolean(Condition cond, Register tag, Label *label) {
michael@0:         cond = testBoolean(cond, tag);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestNull(Condition cond, Register tag, Label *label) {
michael@0:         cond = testNull(cond, tag);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestString(Condition cond, Register tag, Label *label) {
michael@0:         cond = testString(cond, tag);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestObject(Condition cond, Register tag, Label *label) {
michael@0:         cond = testObject(cond, tag);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestNumber(Condition cond, Register tag, Label *label) {
michael@0:         cond = testNumber(cond, tag);
michael@0:         j(cond, label);
michael@0:     }
michael@0: 
michael@0:     // x64 can test for certain types directly from memory when the payload
michael@0:     // of the type is limited to 32 bits. This avoids loading into a register,
michael@0:     // accesses half as much memory, and removes a right-shift.
michael@0:     void branchTestUndefined(Condition cond, const Operand &operand, Label *label) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(ToUpper32(operand), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_UNDEFINED))));
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestUndefined(Condition cond, const Address &address, Label *label) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         branchTestUndefined(cond, Operand(address), label);
michael@0:     }
michael@0:     void branchTestInt32(Condition cond, const Operand &operand, Label *label) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(ToUpper32(operand), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_INT32))));
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestInt32(Condition cond, const Address &address, Label *label) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         branchTestInt32(cond, Operand(address), label);
michael@0:     }
michael@0:     void branchTestDouble(Condition cond, const Operand &operand, Label *label) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         splitTag(operand, ScratchReg);
michael@0:         branchTestDouble(cond, ScratchReg, label);
michael@0:     }
michael@0:     void branchTestDouble(Condition cond, const Address &address, Label *label) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         branchTestDouble(cond, Operand(address), label);
michael@0:     }
michael@0:     void branchTestBoolean(Condition cond, const Operand &operand, Label *label) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(ToUpper32(operand), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_BOOLEAN))));
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestNull(Condition cond, const Operand &operand, Label *label) {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         cmpl(ToUpper32(operand), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_NULL))));
michael@0:         j(cond, label);
michael@0:     }
michael@0: 
michael@0:     // Perform a type-test on a full Value loaded into a register.
michael@0:     // Clobbers the ScratchReg.
michael@0:     void branchTestUndefined(Condition cond, const ValueOperand &src, Label *label) {
michael@0:         cond = testUndefined(cond, src);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestInt32(Condition cond, const ValueOperand &src, Label *label) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         branchTestInt32(cond, ScratchReg, label);
michael@0:     }
michael@0:     void branchTestBoolean(Condition cond, const ValueOperand &src, Label *label) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         branchTestBoolean(cond, ScratchReg, label);
michael@0:     }
michael@0:     void branchTestDouble(Condition cond, const ValueOperand &src, Label *label) {
michael@0:         cond = testDouble(cond, src);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestNull(Condition cond, const ValueOperand &src, Label *label) {
michael@0:         cond = testNull(cond, src);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestString(Condition cond, const ValueOperand &src, Label *label) {
michael@0:         cond = testString(cond, src);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestObject(Condition cond, const ValueOperand &src, Label *label) {
michael@0:         cond = testObject(cond, src);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestNumber(Condition cond, const ValueOperand &src, Label *label) {
michael@0:         cond = testNumber(cond, src);
michael@0:         j(cond, label);
michael@0:     }
michael@0: 
michael@0:     // Perform a type-test on a Value addressed by BaseIndex.
michael@0:     // Clobbers the ScratchReg.
michael@0:     void branchTestUndefined(Condition cond, const BaseIndex &address, Label *label) {
michael@0:         cond = testUndefined(cond, address);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestInt32(Condition cond, const BaseIndex &address, Label *label) {
michael@0:         splitTag(address, ScratchReg);
michael@0:         branchTestInt32(cond, ScratchReg, label);
michael@0:     }
michael@0:     void branchTestBoolean(Condition cond, const BaseIndex &address, Label *label) {
michael@0:         splitTag(address, ScratchReg);
michael@0:         branchTestBoolean(cond, ScratchReg, label);
michael@0:     }
michael@0:     void branchTestDouble(Condition cond, const BaseIndex &address, Label *label) {
michael@0:         cond = testDouble(cond, address);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestNull(Condition cond, const BaseIndex &address, Label *label) {
michael@0:         cond = testNull(cond, address);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestString(Condition cond, const BaseIndex &address, Label *label) {
michael@0:         cond = testString(cond, address);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestObject(Condition cond, const BaseIndex &address, Label *label) {
michael@0:         cond = testObject(cond, address);
michael@0:         j(cond, label);
michael@0:     }
michael@0: 
michael@0:     template <typename T>
michael@0:     void branchTestGCThing(Condition cond, const T &src, Label *label) {
michael@0:         cond = testGCThing(cond, src);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     template <typename T>
michael@0:     void branchTestPrimitive(Condition cond, const T &t, Label *label) {
michael@0:         cond = testPrimitive(cond, t);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     template <typename T>
michael@0:     void branchTestMagic(Condition cond, const T &t, Label *label) {
michael@0:         cond = testMagic(cond, t);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestMagicValue(Condition cond, const ValueOperand &val, JSWhyMagic why,
michael@0:                               Label *label)
michael@0:     {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         // Test for magic
michael@0:         Label notmagic;
michael@0:         Condition testCond = testMagic(cond, val);
michael@0:         j(InvertCondition(testCond), &notmagic);
michael@0:         // Test magic value
michael@0:         unboxMagic(val, ScratchReg);
michael@0:         branch32(cond, ScratchReg, Imm32(static_cast<int32_t>(why)), label);
michael@0:         bind(&notmagic);
michael@0:     }
michael@0:     Condition testMagic(Condition cond, const ValueOperand &src) {
michael@0:         splitTag(src, ScratchReg);
michael@0:         return testMagic(cond, ScratchReg);
michael@0:     }
michael@0:     Condition testError(Condition cond, const ValueOperand &src) {
michael@0:         return testMagic(cond, src);
michael@0:     }
michael@0:     void branchTestValue(Condition cond, const ValueOperand &value, const Value &v, Label *label) {
michael@0:         JS_ASSERT(value.valueReg() != ScratchReg);
michael@0:         moveValue(v, ScratchReg);
michael@0:         cmpq(value.valueReg(), ScratchReg);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestValue(Condition cond, const Address &valaddr, const ValueOperand &value,
michael@0:                          Label *label)
michael@0:     {
michael@0:         JS_ASSERT(cond == Equal || cond == NotEqual);
michael@0:         branchPtr(cond, valaddr, value.valueReg(), label);
michael@0:     }
michael@0: 
michael@0:     void testNullSet(Condition cond, const ValueOperand &value, Register dest) {
michael@0:         cond = testNull(cond, value);
michael@0:         emitSet(cond, dest);
michael@0:     }
michael@0:     void testUndefinedSet(Condition cond, const ValueOperand &value, Register dest) {
michael@0:         cond = testUndefined(cond, value);
michael@0:         emitSet(cond, dest);
michael@0:     }
michael@0: 
michael@0:     void boxDouble(const FloatRegister &src, const ValueOperand &dest) {
michael@0:         movq(src, dest.valueReg());
michael@0:     }
michael@0:     void boxNonDouble(JSValueType type, const Register &src, const ValueOperand &dest) {
michael@0:         JS_ASSERT(src != dest.valueReg());
michael@0:         boxValue(type, src, dest.valueReg());
michael@0:     }
michael@0: 
michael@0:     // Note that the |dest| register here may be ScratchReg, so we shouldn't
michael@0:     // use it.
michael@0:     void unboxInt32(const ValueOperand &src, const Register &dest) {
michael@0:         movl(src.valueReg(), dest);
michael@0:     }
michael@0:     void unboxInt32(const Operand &src, const Register &dest) {
michael@0:         movl(src, dest);
michael@0:     }
michael@0:     void unboxInt32(const Address &src, const Register &dest) {
michael@0:         unboxInt32(Operand(src), dest);
michael@0:     }
michael@0:     void unboxDouble(const Address &src, const FloatRegister &dest) {
michael@0:         loadDouble(Operand(src), dest);
michael@0:     }
michael@0: 
michael@0:     void unboxArgObjMagic(const ValueOperand &src, const Register &dest) {
michael@0:         unboxArgObjMagic(Operand(src.valueReg()), dest);
michael@0:     }
michael@0:     void unboxArgObjMagic(const Operand &src, const Register &dest) {
michael@0:         mov(ImmWord(0), dest);
michael@0:     }
michael@0:     void unboxArgObjMagic(const Address &src, const Register &dest) {
michael@0:         unboxArgObjMagic(Operand(src), dest);
michael@0:     }
michael@0: 
michael@0:     void unboxBoolean(const ValueOperand &src, const Register &dest) {
michael@0:         movl(src.valueReg(), dest);
michael@0:     }
michael@0:     void unboxBoolean(const Operand &src, const Register &dest) {
michael@0:         movl(src, dest);
michael@0:     }
michael@0:     void unboxBoolean(const Address &src, const Register &dest) {
michael@0:         unboxBoolean(Operand(src), dest);
michael@0:     }
michael@0: 
michael@0:     void unboxMagic(const ValueOperand &src, const Register &dest) {
michael@0:         movl(src.valueReg(), dest);
michael@0:     }
michael@0: 
michael@0:     void unboxDouble(const ValueOperand &src, const FloatRegister &dest) {
michael@0:         movq(src.valueReg(), dest);
michael@0:     }
michael@0:     void unboxPrivate(const ValueOperand &src, const Register dest) {
michael@0:         movq(src.valueReg(), dest);
michael@0:         shlq(Imm32(1), dest);
michael@0:     }
michael@0: 
michael@0:     void notBoolean(const ValueOperand &val) {
michael@0:         xorq(Imm32(1), val.valueReg());
michael@0:     }
michael@0: 
michael@0:     // Unbox any non-double value into dest. Prefer unboxInt32 or unboxBoolean
michael@0:     // instead if the source type is known.
michael@0:     void unboxNonDouble(const ValueOperand &src, const Register &dest) {
michael@0:         // In a non-trivial coupling, we're not permitted to use ScratchReg when
michael@0:         // src and dest are different registers, because of how extractObject is
michael@0:         // implemented.
michael@0:         if (src.valueReg() == dest) {
michael@0:             mov(ImmWord(JSVAL_PAYLOAD_MASK), ScratchReg);
michael@0:             andq(ScratchReg, dest);
michael@0:         } else {
michael@0:             mov(ImmWord(JSVAL_PAYLOAD_MASK), dest);
michael@0:             andq(src.valueReg(), dest);
michael@0:         }
michael@0:     }
michael@0:     void unboxNonDouble(const Operand &src, const Register &dest) {
michael@0:         // Explicitly permits |dest| to be used in |src|.
michael@0:         JS_ASSERT(dest != ScratchReg);
michael@0:         mov(ImmWord(JSVAL_PAYLOAD_MASK), ScratchReg);
michael@0:         movq(src, dest);
michael@0:         andq(ScratchReg, dest);
michael@0:     }
michael@0: 
michael@0:     void unboxString(const ValueOperand &src, const Register &dest) { unboxNonDouble(src, dest); }
michael@0:     void unboxString(const Operand &src, const Register &dest) { unboxNonDouble(src, dest); }
michael@0: 
michael@0:     void unboxObject(const ValueOperand &src, const Register &dest) { unboxNonDouble(src, dest); }
michael@0:     void unboxObject(const Operand &src, const Register &dest) { unboxNonDouble(src, dest); }
michael@0: 
michael@0:     // Extended unboxing API. If the payload is already in a register, returns
michael@0:     // that register. Otherwise, provides a move to the given scratch register,
michael@0:     // and returns that.
michael@0:     Register extractObject(const Address &address, Register scratch) {
michael@0:         JS_ASSERT(scratch != ScratchReg);
michael@0:         loadPtr(address, ScratchReg);
michael@0:         // We have a special coupling with unboxObject. As long as the registers
michael@0:         // aren't equal, it doesn't use ScratchReg.
michael@0:         unboxObject(ValueOperand(ScratchReg), scratch);
michael@0:         return scratch;
michael@0:     }
michael@0:     Register extractObject(const ValueOperand &value, Register scratch) {
michael@0:         JS_ASSERT(scratch != ScratchReg);
michael@0:         unboxObject(value, scratch);
michael@0:         return scratch;
michael@0:     }
michael@0:     Register extractInt32(const ValueOperand &value, Register scratch) {
michael@0:         JS_ASSERT(scratch != ScratchReg);
michael@0:         unboxInt32(value, scratch);
michael@0:         return scratch;
michael@0:     }
michael@0:     Register extractBoolean(const ValueOperand &value, Register scratch) {
michael@0:         JS_ASSERT(scratch != ScratchReg);
michael@0:         unboxBoolean(value, scratch);
michael@0:         return scratch;
michael@0:     }
michael@0:     Register extractTag(const Address &address, Register scratch) {
michael@0:         JS_ASSERT(scratch != ScratchReg);
michael@0:         loadPtr(address, scratch);
michael@0:         splitTag(scratch, scratch);
michael@0:         return scratch;
michael@0:     }
michael@0:     Register extractTag(const ValueOperand &value, Register scratch) {
michael@0:         JS_ASSERT(scratch != ScratchReg);
michael@0:         splitTag(value, scratch);
michael@0:         return scratch;
michael@0:     }
michael@0: 
michael@0:     void unboxValue(const ValueOperand &src, AnyRegister dest) {
michael@0:         if (dest.isFloat()) {
michael@0:             Label notInt32, end;
michael@0:             branchTestInt32(Assembler::NotEqual, src, &notInt32);
michael@0:             convertInt32ToDouble(src.valueReg(), dest.fpu());
michael@0:             jump(&end);
michael@0:             bind(&notInt32);
michael@0:             unboxDouble(src, dest.fpu());
michael@0:             bind(&end);
michael@0:         } else {
michael@0:             unboxNonDouble(src, dest.gpr());
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     // These two functions use the low 32-bits of the full value register.
michael@0:     void boolValueToDouble(const ValueOperand &operand, const FloatRegister &dest) {
michael@0:         convertInt32ToDouble(operand.valueReg(), dest);
michael@0:     }
michael@0:     void int32ValueToDouble(const ValueOperand &operand, const FloatRegister &dest) {
michael@0:         convertInt32ToDouble(operand.valueReg(), dest);
michael@0:     }
michael@0: 
michael@0:     void boolValueToFloat32(const ValueOperand &operand, const FloatRegister &dest) {
michael@0:         convertInt32ToFloat32(operand.valueReg(), dest);
michael@0:     }
michael@0:     void int32ValueToFloat32(const ValueOperand &operand, const FloatRegister &dest) {
michael@0:         convertInt32ToFloat32(operand.valueReg(), dest);
michael@0:     }
michael@0: 
michael@0:     void loadConstantDouble(double d, const FloatRegister &dest);
michael@0:     void loadConstantFloat32(float f, const FloatRegister &dest);
michael@0: 
michael@0:     void branchTruncateDouble(const FloatRegister &src, const Register &dest, Label *fail) {
michael@0:         cvttsd2sq(src, dest);
michael@0: 
michael@0:         // cvttsd2sq returns 0x8000000000000000 on failure. Test for it by
michael@0:         // subtracting 1 and testing overflow (this avoids the need to
michael@0:         // materialize that value in a register).
michael@0:         cmpq(dest, Imm32(1));
michael@0:         j(Assembler::Overflow, fail);
michael@0: 
michael@0:         movl(dest, dest); // Zero upper 32-bits.
michael@0:     }
michael@0:     void branchTruncateFloat32(const FloatRegister &src, const Register &dest, Label *fail) {
michael@0:         cvttss2sq(src, dest);
michael@0: 
michael@0:         // Same trick as for Doubles
michael@0:         cmpq(dest, Imm32(1));
michael@0:         j(Assembler::Overflow, fail);
michael@0: 
michael@0:         movl(dest, dest); // Zero upper 32-bits.
michael@0:     }
michael@0: 
michael@0:     Condition testInt32Truthy(bool truthy, const ValueOperand &operand) {
michael@0:         testl(operand.valueReg(), operand.valueReg());
michael@0:         return truthy ? NonZero : Zero;
michael@0:     }
michael@0:     void branchTestInt32Truthy(bool truthy, const ValueOperand &operand, Label *label) {
michael@0:         Condition cond = testInt32Truthy(truthy, operand);
michael@0:         j(cond, label);
michael@0:     }
michael@0:     void branchTestBooleanTruthy(bool truthy, const ValueOperand &operand, Label *label) {
michael@0:         testl(operand.valueReg(), operand.valueReg());
michael@0:         j(truthy ? NonZero : Zero, label);
michael@0:     }
michael@0:     Condition testStringTruthy(bool truthy, const ValueOperand &value) {
michael@0:         unboxString(value, ScratchReg);
michael@0: 
michael@0:         Operand lengthAndFlags(ScratchReg, JSString::offsetOfLengthAndFlags());
michael@0:         testq(lengthAndFlags, Imm32(-1 << JSString::LENGTH_SHIFT));
michael@0:         return truthy ? Assembler::NonZero : Assembler::Zero;
michael@0:     }
michael@0:     void branchTestStringTruthy(bool truthy, const ValueOperand &value, Label *label) {
michael@0:         Condition cond = testStringTruthy(truthy, value);
michael@0:         j(cond, label);
michael@0:     }
michael@0: 
michael@0:     void loadInt32OrDouble(const Operand &operand, const FloatRegister &dest) {
michael@0:         Label notInt32, end;
michael@0:         branchTestInt32(Assembler::NotEqual, operand, &notInt32);
michael@0:         convertInt32ToDouble(operand, dest);
michael@0:         jump(&end);
michael@0:         bind(&notInt32);
michael@0:         loadDouble(operand, dest);
michael@0:         bind(&end);
michael@0:     }
michael@0: 
michael@0:     template <typename T>
michael@0:     void loadUnboxedValue(const T &src, MIRType type, AnyRegister dest) {
michael@0:         if (dest.isFloat())
michael@0:             loadInt32OrDouble(Operand(src), dest.fpu());
michael@0:         else if (type == MIRType_Int32 || type == MIRType_Boolean)
michael@0:             movl(Operand(src), dest.gpr());
michael@0:         else
michael@0:             unboxNonDouble(Operand(src), dest.gpr());
michael@0:     }
michael@0: 
michael@0:     void loadInstructionPointerAfterCall(const Register &dest) {
michael@0:         loadPtr(Address(StackPointer, 0x0), dest);
michael@0:     }
michael@0: 
michael@0:     void convertUInt32ToDouble(const Register &src, const FloatRegister &dest) {
michael@0:         cvtsq2sd(src, dest);
michael@0:     }
michael@0: 
michael@0:     void convertUInt32ToFloat32(const Register &src, const FloatRegister &dest) {
michael@0:         cvtsq2ss(src, dest);
michael@0:     }
michael@0: 
michael@0:     void inc64(AbsoluteAddress dest) {
michael@0:         if (JSC::X86Assembler::isAddressImmediate(dest.addr)) {
michael@0:             addPtr(Imm32(1), Operand(dest));
michael@0:         } else {
michael@0:             mov(ImmPtr(dest.addr), ScratchReg);
michael@0:             addPtr(Imm32(1), Address(ScratchReg, 0));
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     // If source is a double, load it into dest. If source is int32,
michael@0:     // convert it to double. Else, branch to failure.
michael@0:     void ensureDouble(const ValueOperand &source, FloatRegister dest, Label *failure) {
michael@0:         Label isDouble, done;
michael@0:         Register tag = splitTagForTest(source);
michael@0:         branchTestDouble(Assembler::Equal, tag, &isDouble);
michael@0:         branchTestInt32(Assembler::NotEqual, tag, failure);
michael@0: 
michael@0:         unboxInt32(source, ScratchReg);
michael@0:         convertInt32ToDouble(ScratchReg, dest);
michael@0:         jump(&done);
michael@0: 
michael@0:         bind(&isDouble);
michael@0:         unboxDouble(source, dest);
michael@0: 
michael@0:         bind(&done);
michael@0:     }
michael@0: 
michael@0:     // Setup a call to C/C++ code, given the number of general arguments it
michael@0:     // takes. Note that this only supports cdecl.
michael@0:     //
michael@0:     // In order for alignment to work correctly, the MacroAssembler must have a
michael@0:     // consistent view of the stack displacement. It is okay to call "push"
michael@0:     // manually, however, if the stack alignment were to change, the macro
michael@0:     // assembler should be notified before starting a call.
michael@0:     void setupAlignedABICall(uint32_t args);
michael@0: 
michael@0:     // Sets up an ABI call for when the alignment is not known. This may need a
michael@0:     // scratch register.
michael@0:     void setupUnalignedABICall(uint32_t args, const Register &scratch);
michael@0: 
michael@0:     // Arguments must be assigned to a C/C++ call in order. They are moved
michael@0:     // in parallel immediately before performing the call. This process may
michael@0:     // temporarily use more stack, in which case esp-relative addresses will be
michael@0:     // automatically adjusted. It is extremely important that esp-relative
michael@0:     // addresses are computed *after* setupABICall(). Furthermore, no
michael@0:     // operations should be emitted while setting arguments.
michael@0:     void passABIArg(const MoveOperand &from, MoveOp::Type type);
michael@0:     void passABIArg(const Register &reg);
michael@0:     void passABIArg(const FloatRegister &reg, MoveOp::Type type);
michael@0: 
michael@0:   private:
michael@0:     void callWithABIPre(uint32_t *stackAdjust);
michael@0:     void callWithABIPost(uint32_t stackAdjust, MoveOp::Type result);
michael@0: 
michael@0:   public:
michael@0:     // Emits a call to a C/C++ function, resolving all argument moves.
michael@0:     void callWithABI(void *fun, MoveOp::Type result = MoveOp::GENERAL);
michael@0:     void callWithABI(AsmJSImmPtr imm, MoveOp::Type result = MoveOp::GENERAL);
michael@0:     void callWithABI(Address fun, MoveOp::Type result = MoveOp::GENERAL);
michael@0: 
michael@0:     void handleFailureWithHandler(void *handler);
michael@0:     void handleFailureWithHandlerTail();
michael@0: 
michael@0:     void makeFrameDescriptor(Register frameSizeReg, FrameType type) {
michael@0:         shlq(Imm32(FRAMESIZE_SHIFT), frameSizeReg);
michael@0:         orq(Imm32(type), frameSizeReg);
michael@0:     }
michael@0: 
michael@0:     // Save an exit frame (which must be aligned to the stack pointer) to
michael@0:     // ThreadData::ionTop of the main thread.
michael@0:     void linkExitFrame() {
michael@0:         storePtr(StackPointer,
michael@0:                  AbsoluteAddress(GetIonContext()->runtime->addressOfIonTop()));
michael@0:     }
michael@0: 
michael@0:     void callWithExitFrame(JitCode *target, Register dynStack) {
michael@0:         addPtr(Imm32(framePushed()), dynStack);
michael@0:         makeFrameDescriptor(dynStack, JitFrame_IonJS);
michael@0:         Push(dynStack);
michael@0:         call(target);
michael@0:     }
michael@0: 
michael@0:     // Save an exit frame to the thread data of the current thread, given a
michael@0:     // register that holds a PerThreadData *.
michael@0:     void linkParallelExitFrame(const Register &pt) {
michael@0:         storePtr(StackPointer, Address(pt, offsetof(PerThreadData, ionTop)));
michael@0:     }
michael@0: 
michael@0:     // See CodeGeneratorX64 calls to noteAsmJSGlobalAccess.
michael@0:     void patchAsmJSGlobalAccess(CodeOffsetLabel patchAt, uint8_t *code, uint8_t *globalData,
michael@0:                                 unsigned globalDataOffset)
michael@0:     {
michael@0:         uint8_t *nextInsn = code + patchAt.offset();
michael@0:         JS_ASSERT(nextInsn <= globalData);
michael@0:         uint8_t *target = globalData + globalDataOffset;
michael@0:         ((int32_t *)nextInsn)[-1] = target - nextInsn;
michael@0:     }
michael@0:     void memIntToValue(Address Source, Address Dest) {
michael@0:         load32(Source, ScratchReg);
michael@0:         storeValue(JSVAL_TYPE_INT32, ScratchReg, Dest);
michael@0:     }
michael@0: 
michael@0: #ifdef JSGC_GENERATIONAL
michael@0:     void branchPtrInNurseryRange(Register ptr, Register temp, Label *label);
michael@0:     void branchValueIsNurseryObject(ValueOperand value, Register temp, Label *label);
michael@0: #endif
michael@0: };
michael@0: 
michael@0: typedef MacroAssemblerX64 MacroAssemblerSpecific;
michael@0: 
michael@0: } // namespace jit
michael@0: } // namespace js
michael@0: 
michael@0: #endif /* jit_x64_MacroAssembler_x64_h */