The Tor Browser / file revision

 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-

  * vim: set ts=8 sts=4 et sw=4 tw=99:

  * This Source Code Form is subject to the terms of the Mozilla Public

  * License, v. 2.0. If a copy of the MPL was not distributed with this

  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

 #include "jit/CodeGenerator.h"

 #include "mozilla/Assertions.h"

 #include "mozilla/Attributes.h"

 #include "mozilla/DebugOnly.h"

 #include "jslibmath.h"

 #include "jsmath.h"

 #include "jsnum.h"

 #include "jsprf.h"

 #include "builtin/Eval.h"

 #include "builtin/TypedObject.h"

 #ifdef JSGC_GENERATIONAL

 # include "gc/Nursery.h"

 #endif

 #include "jit/IonCaches.h"

 #include "jit/IonLinker.h"

 #include "jit/IonOptimizationLevels.h"

 #include "jit/IonSpewer.h"

 #include "jit/MIRGenerator.h"

 #include "jit/MoveEmitter.h"

 #include "jit/ParallelFunctions.h"

 #include "jit/ParallelSafetyAnalysis.h"

 #include "jit/RangeAnalysis.h"

 #include "vm/ForkJoin.h"

 #include "vm/TraceLogging.h"

 #include "jsboolinlines.h"

 #include "jit/ExecutionMode-inl.h"

 #include "jit/shared/CodeGenerator-shared-inl.h"

 #include "vm/Interpreter-inl.h"

 using namespace js;

 using namespace js::jit;

 using mozilla::DebugOnly;

 using mozilla::FloatingPoint;

 using mozilla::Maybe;

 using mozilla::NegativeInfinity;

 using mozilla::PositiveInfinity;

 using JS::GenericNaN;

 namespace js {

 namespace jit {

 // This out-of-line cache is used to do a double dispatch including it-self and

 // the wrapped IonCache.

 class OutOfLineUpdateCache :

   public OutOfLineCodeBase<CodeGenerator>,

   public IonCacheVisitor

 {

   private:

     LInstruction *lir_;

     size_t cacheIndex_;

     AddCacheState state_;

   public:

     OutOfLineUpdateCache(LInstruction *lir, size_t cacheIndex)

       : lir_(lir),

         cacheIndex_(cacheIndex)

     { }

     void bind(MacroAssembler *masm) {

         // The binding of the initial jump is done in

         // CodeGenerator::visitOutOfLineCache.

     }

     size_t getCacheIndex() const {

         return cacheIndex_;

     }

     LInstruction *lir() const {

         return lir_;

     }

     AddCacheState &state() {

         return state_;

     }

     bool accept(CodeGenerator *codegen) {

         return codegen->visitOutOfLineCache(this);

     }

     // ICs' visit functions delegating the work to the CodeGen visit funtions.

 #define VISIT_CACHE_FUNCTION(op)                                        \

     bool visit##op##IC(CodeGenerator *codegen) {                        \

         CodeGenerator::DataPtr<op##IC> ic(codegen, getCacheIndex());    \

         return codegen->visit##op##IC(this, ic);                        \

     }

     IONCACHE_KIND_LIST(VISIT_CACHE_FUNCTION)

 #undef VISIT_CACHE_FUNCTION

 };

 // This function is declared here because it needs to instantiate an

 // OutOfLineUpdateCache, but we want to keep it visible inside the

 // CodeGeneratorShared such as we can specialize inline caches in function of

 // the architecture.

 bool

 CodeGeneratorShared::addCache(LInstruction *lir, size_t cacheIndex)

 {

     if (cacheIndex == SIZE_MAX)

         return false;

     DataPtr<IonCache> cache(this, cacheIndex);

     MInstruction *mir = lir->mirRaw()->toInstruction();

     if (mir->resumePoint())

         cache->setScriptedLocation(mir->block()->info().script(),

                                    mir->resumePoint()->pc());

     else

         cache->setIdempotent();

     OutOfLineUpdateCache *ool = new(alloc()) OutOfLineUpdateCache(lir, cacheIndex);

     if (!addOutOfLineCode(ool))

         return false;

     // OOL-specific state depends on the type of cache.

     cache->initializeAddCacheState(lir, &ool->state());

     cache->emitInitialJump(masm, ool->state());

     masm.bind(ool->rejoin());

     return true;

 }

 bool

 CodeGenerator::visitOutOfLineCache(OutOfLineUpdateCache *ool)

 {

     DataPtr<IonCache> cache(this, ool->getCacheIndex());

     // Register the location of the OOL path in the IC.

     cache->setFallbackLabel(masm.labelForPatch());

     cache->bindInitialJump(masm, ool->state());

     // Dispatch to ICs' accept functions.

     return cache->accept(this, ool);

 }

 StringObject *

 MNewStringObject::templateObj() const {

     return &templateObj_->as<StringObject>();

 }

 CodeGenerator::CodeGenerator(MIRGenerator *gen, LIRGraph *graph, MacroAssembler *masm)

   : CodeGeneratorSpecific(gen, graph, masm)

   , ionScriptLabels_(gen->alloc())

 {

 }

 CodeGenerator::~CodeGenerator()

 {

     JS_ASSERT_IF(!gen->compilingAsmJS(), masm.numAsmJSAbsoluteLinks() == 0);

 }

 typedef bool (*StringToNumberFn)(ThreadSafeContext *, JSString *, double *);

 typedef bool (*StringToNumberParFn)(ForkJoinContext *, JSString *, double *);

 static const VMFunctionsModal StringToNumberInfo = VMFunctionsModal(

     FunctionInfo<StringToNumberFn>(StringToNumber),

     FunctionInfo<StringToNumberParFn>(StringToNumberPar));

 bool

 CodeGenerator::visitValueToInt32(LValueToInt32 *lir)

 {

     ValueOperand operand = ToValue(lir, LValueToInt32::Input);

     Register output = ToRegister(lir->output());

     FloatRegister temp = ToFloatRegister(lir->tempFloat());

     MDefinition *input;

     if (lir->mode() == LValueToInt32::NORMAL)

         input = lir->mirNormal()->input();

     else

         input = lir->mirTruncate()->input();

     Label fails;

     if (lir->mode() == LValueToInt32::TRUNCATE) {

         OutOfLineCode *oolDouble = oolTruncateDouble(temp, output);

         if (!oolDouble)

             return false;

         // We can only handle strings in truncation contexts, like bitwise

         // operations.

         Label *stringEntry, *stringRejoin;

         Register stringReg;

         if (input->mightBeType(MIRType_String)) {

             stringReg = ToRegister(lir->temp());

             OutOfLineCode *oolString = oolCallVM(StringToNumberInfo, lir, (ArgList(), stringReg),

                                                  StoreFloatRegisterTo(temp));

             if (!oolString)

                 return false;

             stringEntry = oolString->entry();

             stringRejoin = oolString->rejoin();

         } else {

             stringReg = InvalidReg;

             stringEntry = nullptr;

             stringRejoin = nullptr;

         }

         masm.truncateValueToInt32(operand, input, stringEntry, stringRejoin, oolDouble->entry(),

                                   stringReg, temp, output, &fails);

         masm.bind(oolDouble->rejoin());

     } else {

         masm.convertValueToInt32(operand, input, temp, output, &fails,

                                  lir->mirNormal()->canBeNegativeZero(),

                                  lir->mirNormal()->conversion());

     }

     return bailoutFrom(&fails, lir->snapshot());

 }

 bool

 CodeGenerator::visitValueToDouble(LValueToDouble *lir)

 {

     MToDouble *mir = lir->mir();

     ValueOperand operand = ToValue(lir, LValueToDouble::Input);

     FloatRegister output = ToFloatRegister(lir->output());

     Register tag = masm.splitTagForTest(operand);

     Label isDouble, isInt32, isBool, isNull, isUndefined, done;

     bool hasBoolean = false, hasNull = false, hasUndefined = false;

     masm.branchTestDouble(Assembler::Equal, tag, &isDouble);

     masm.branchTestInt32(Assembler::Equal, tag, &isInt32);

     if (mir->conversion() != MToDouble::NumbersOnly) {

         masm.branchTestBoolean(Assembler::Equal, tag, &isBool);

         masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);

         hasBoolean = true;

         hasUndefined = true;

         if (mir->conversion() != MToDouble::NonNullNonStringPrimitives) {

             masm.branchTestNull(Assembler::Equal, tag, &isNull);

             hasNull = true;

         }

     }

     if (!bailout(lir->snapshot()))

         return false;

     if (hasNull) {

         masm.bind(&isNull);

         masm.loadConstantDouble(0.0, output);

         masm.jump(&done);

     }

     if (hasUndefined) {

         masm.bind(&isUndefined);

         masm.loadConstantDouble(GenericNaN(), output);

         masm.jump(&done);

     }

     if (hasBoolean) {

         masm.bind(&isBool);

         masm.boolValueToDouble(operand, output);

         masm.jump(&done);

     }

     masm.bind(&isInt32);

     masm.int32ValueToDouble(operand, output);

     masm.jump(&done);

     masm.bind(&isDouble);

     masm.unboxDouble(operand, output);

     masm.bind(&done);

     return true;

 }

 bool

 CodeGenerator::visitValueToFloat32(LValueToFloat32 *lir)

 {

     MToFloat32 *mir = lir->mir();

     ValueOperand operand = ToValue(lir, LValueToFloat32::Input);

     FloatRegister output = ToFloatRegister(lir->output());

     Register tag = masm.splitTagForTest(operand);

     Label isDouble, isInt32, isBool, isNull, isUndefined, done;

     bool hasBoolean = false, hasNull = false, hasUndefined = false;

     masm.branchTestDouble(Assembler::Equal, tag, &isDouble);

     masm.branchTestInt32(Assembler::Equal, tag, &isInt32);

     if (mir->conversion() != MToFloat32::NumbersOnly) {

         masm.branchTestBoolean(Assembler::Equal, tag, &isBool);

         masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);

         hasBoolean = true;

         hasUndefined = true;

         if (mir->conversion() != MToFloat32::NonNullNonStringPrimitives) {

             masm.branchTestNull(Assembler::Equal, tag, &isNull);

             hasNull = true;

         }

     }

     if (!bailout(lir->snapshot()))

         return false;

     if (hasNull) {

         masm.bind(&isNull);

         masm.loadConstantFloat32(0.0f, output);

         masm.jump(&done);

     }

     if (hasUndefined) {

         masm.bind(&isUndefined);

         masm.loadConstantFloat32(float(GenericNaN()), output);

         masm.jump(&done);

     }

     if (hasBoolean) {

         masm.bind(&isBool);

         masm.boolValueToFloat32(operand, output);

         masm.jump(&done);

     }

     masm.bind(&isInt32);

     masm.int32ValueToFloat32(operand, output);

     masm.jump(&done);

     masm.bind(&isDouble);

     masm.unboxDouble(operand, output);

     masm.convertDoubleToFloat32(output, output);

     masm.bind(&done);

     return true;

 }

 bool

 CodeGenerator::visitInt32ToDouble(LInt32ToDouble *lir)

 {

     masm.convertInt32ToDouble(ToRegister(lir->input()), ToFloatRegister(lir->output()));

     return true;

 }

 bool

 CodeGenerator::visitFloat32ToDouble(LFloat32ToDouble *lir)

 {

     masm.convertFloat32ToDouble(ToFloatRegister(lir->input()), ToFloatRegister(lir->output()));

     return true;

 }

 bool

 CodeGenerator::visitDoubleToFloat32(LDoubleToFloat32 *lir)

 {

     masm.convertDoubleToFloat32(ToFloatRegister(lir->input()), ToFloatRegister(lir->output()));

     return true;

 }

 bool

 CodeGenerator::visitInt32ToFloat32(LInt32ToFloat32 *lir)

 {

     masm.convertInt32ToFloat32(ToRegister(lir->input()), ToFloatRegister(lir->output()));

     return true;

 }

 bool

 CodeGenerator::visitDoubleToInt32(LDoubleToInt32 *lir)

 {

     Label fail;

     FloatRegister input = ToFloatRegister(lir->input());

     Register output = ToRegister(lir->output());

     masm.convertDoubleToInt32(input, output, &fail, lir->mir()->canBeNegativeZero());

     if (!bailoutFrom(&fail, lir->snapshot()))

         return false;

     return true;

 }

 bool

 CodeGenerator::visitFloat32ToInt32(LFloat32ToInt32 *lir)

 {

     Label fail;

     FloatRegister input = ToFloatRegister(lir->input());

     Register output = ToRegister(lir->output());

     masm.convertFloat32ToInt32(input, output, &fail, lir->mir()->canBeNegativeZero());

     if (!bailoutFrom(&fail, lir->snapshot()))

         return false;

     return true;

 }

 void

 CodeGenerator::emitOOLTestObject(Register objreg,

                                  Label *ifEmulatesUndefined,

                                  Label *ifDoesntEmulateUndefined,

                                  Register scratch)

 {

     saveVolatile(scratch);

     masm.setupUnalignedABICall(1, scratch);

     masm.passABIArg(objreg);

     masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, js::EmulatesUndefined));

     masm.storeCallResult(scratch);

     restoreVolatile(scratch);

     masm.branchIfTrueBool(scratch, ifEmulatesUndefined);

     masm.jump(ifDoesntEmulateUndefined);

 }

 // Base out-of-line code generator for all tests of the truthiness of an

 // object, where the object might not be truthy.  (Recall that per spec all

 // objects are truthy, but we implement the JSCLASS_EMULATES_UNDEFINED class

 // flag to permit objects to look like |undefined| in certain contexts,

 // including in object truthiness testing.)  We check truthiness inline except

 // when we're testing it on a proxy (or if TI guarantees us that the specified

 // object will never emulate |undefined|), in which case out-of-line code will

 // call EmulatesUndefined for a conclusive answer.

 class OutOfLineTestObject : public OutOfLineCodeBase<CodeGenerator>

 {

     Register objreg_;

     Register scratch_;

     Label *ifEmulatesUndefined_;

     Label *ifDoesntEmulateUndefined_;

 #ifdef DEBUG

     bool initialized() { return ifEmulatesUndefined_ != nullptr; }

 #endif

   public:

     OutOfLineTestObject()

 #ifdef DEBUG

       : ifEmulatesUndefined_(nullptr), ifDoesntEmulateUndefined_(nullptr)

 #endif

     { }

     bool accept(CodeGenerator *codegen) MOZ_FINAL MOZ_OVERRIDE {

         MOZ_ASSERT(initialized());

         codegen->emitOOLTestObject(objreg_, ifEmulatesUndefined_, ifDoesntEmulateUndefined_,

                                    scratch_);

         return true;

     }

     // Specify the register where the object to be tested is found, labels to

     // jump to if the object is truthy or falsy, and a scratch register for

     // use in the out-of-line path.

     void setInputAndTargets(Register objreg, Label *ifEmulatesUndefined, Label *ifDoesntEmulateUndefined,

                             Register scratch)

     {

         MOZ_ASSERT(!initialized());

         MOZ_ASSERT(ifEmulatesUndefined);

         objreg_ = objreg;

         scratch_ = scratch;

         ifEmulatesUndefined_ = ifEmulatesUndefined;

         ifDoesntEmulateUndefined_ = ifDoesntEmulateUndefined;

     }

 };

 // A subclass of OutOfLineTestObject containing two extra labels, for use when

 // the ifTruthy/ifFalsy labels are needed in inline code as well as out-of-line

 // code.  The user should bind these labels in inline code, and specify them as

 // targets via setInputAndTargets, as appropriate.

 class OutOfLineTestObjectWithLabels : public OutOfLineTestObject

 {

     Label label1_;

     Label label2_;

   public:

     OutOfLineTestObjectWithLabels() { }

     Label *label1() { return &label1_; }

     Label *label2() { return &label2_; }

 };

 void

 CodeGenerator::testObjectEmulatesUndefinedKernel(Register objreg,

                                                  Label *ifEmulatesUndefined,

                                                  Label *ifDoesntEmulateUndefined,

                                                  Register scratch, OutOfLineTestObject *ool)

 {

     ool->setInputAndTargets(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined, scratch);

     // Perform a fast-path check of the object's class flags if the object's

     // not a proxy.  Let out-of-line code handle the slow cases that require

     // saving registers, making a function call, and restoring registers.

     masm.branchTestObjectTruthy(false, objreg, scratch, ool->entry(), ifEmulatesUndefined);

 }

 void

 CodeGenerator::branchTestObjectEmulatesUndefined(Register objreg,

                                                  Label *ifEmulatesUndefined,

                                                  Label *ifDoesntEmulateUndefined,

                                                  Register scratch, OutOfLineTestObject *ool)

 {

     MOZ_ASSERT(!ifDoesntEmulateUndefined->bound(),

                "ifDoesntEmulateUndefined will be bound to the fallthrough path");

     testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,

                                       scratch, ool);

     masm.bind(ifDoesntEmulateUndefined);

 }

 void

 CodeGenerator::testObjectEmulatesUndefined(Register objreg,

                                            Label *ifEmulatesUndefined,

                                            Label *ifDoesntEmulateUndefined,

                                            Register scratch, OutOfLineTestObject *ool)

 {

     testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,

                                       scratch, ool);

     masm.jump(ifDoesntEmulateUndefined);

 }

 void

 CodeGenerator::testValueTruthyKernel(const ValueOperand &value,

                                      const LDefinition *scratch1, const LDefinition *scratch2,

                                      FloatRegister fr,

                                      Label *ifTruthy, Label *ifFalsy,

                                      OutOfLineTestObject *ool)

 {

     Register tag = masm.splitTagForTest(value);

     // Eventually we will want some sort of type filter here. For now, just

     // emit all easy cases. For speed we use the cached tag for all comparison,

     // except for doubles, which we test last (as the operation can clobber the

     // tag, which may be in ScratchReg).

     masm.branchTestUndefined(Assembler::Equal, tag, ifFalsy);

     masm.branchTestNull(Assembler::Equal, tag, ifFalsy);

     Label notBoolean;

     masm.branchTestBoolean(Assembler::NotEqual, tag, &notBoolean);

     masm.branchTestBooleanTruthy(false, value, ifFalsy);

     masm.jump(ifTruthy);

     masm.bind(&notBoolean);

     Label notInt32;

     masm.branchTestInt32(Assembler::NotEqual, tag, &notInt32);

     masm.branchTestInt32Truthy(false, value, ifFalsy);

     masm.jump(ifTruthy);

     masm.bind(&notInt32);

     if (ool) {

         Label notObject;

         masm.branchTestObject(Assembler::NotEqual, tag, &notObject);

         Register objreg = masm.extractObject(value, ToRegister(scratch1));

         testObjectEmulatesUndefined(objreg, ifFalsy, ifTruthy, ToRegister(scratch2), ool);

         masm.bind(&notObject);

     } else {

         masm.branchTestObject(Assembler::Equal, tag, ifTruthy);

     }

     // Test if a string is non-empty.

     Label notString;

     masm.branchTestString(Assembler::NotEqual, tag, &notString);

     masm.branchTestStringTruthy(false, value, ifFalsy);

     masm.jump(ifTruthy);

     masm.bind(&notString);

     // If we reach here the value is a double.

     masm.unboxDouble(value, fr);

     masm.branchTestDoubleTruthy(false, fr, ifFalsy);

     // Fall through for truthy.

 }

 void

 CodeGenerator::testValueTruthy(const ValueOperand &value,

                                const LDefinition *scratch1, const LDefinition *scratch2,

                                FloatRegister fr,

                                Label *ifTruthy, Label *ifFalsy,

                                OutOfLineTestObject *ool)

 {

     testValueTruthyKernel(value, scratch1, scratch2, fr, ifTruthy, ifFalsy, ool);

     masm.jump(ifTruthy);

 }

 Label *

 CodeGenerator::getJumpLabelForBranch(MBasicBlock *block)

 {

     if (!labelForBackedgeWithImplicitCheck(block))

         return block->lir()->label();

     // We need to use a patchable jump for this backedge, but want to treat

     // this as a normal label target to simplify codegen. Efficiency isn't so

     // important here as these tests are extremely unlikely to be used in loop

     // backedges, so emit inline code for the patchable jump. Heap allocating

     // the label allows it to be used by out of line blocks.

     Label *res = GetIonContext()->temp->lifoAlloc()->new_<Label>();

     Label after;

     masm.jump(&after);

     masm.bind(res);

     jumpToBlock(block);

     masm.bind(&after);

     return res;

 }

 bool

 CodeGenerator::visitTestOAndBranch(LTestOAndBranch *lir)

 {

     MOZ_ASSERT(lir->mir()->operandMightEmulateUndefined(),

                "Objects which can't emulate undefined should have been constant-folded");

     OutOfLineTestObject *ool = new(alloc()) OutOfLineTestObject();

     if (!addOutOfLineCode(ool))

         return false;

     Label *truthy = getJumpLabelForBranch(lir->ifTruthy());

     Label *falsy = getJumpLabelForBranch(lir->ifFalsy());

     testObjectEmulatesUndefined(ToRegister(lir->input()), falsy, truthy,

                                 ToRegister(lir->temp()), ool);

     return true;

 }

 bool

 CodeGenerator::visitTestVAndBranch(LTestVAndBranch *lir)

 {

     OutOfLineTestObject *ool = nullptr;

     if (lir->mir()->operandMightEmulateUndefined()) {

         ool = new(alloc()) OutOfLineTestObject();

         if (!addOutOfLineCode(ool))

             return false;

     }

     Label *truthy = getJumpLabelForBranch(lir->ifTruthy());

     Label *falsy = getJumpLabelForBranch(lir->ifFalsy());

     testValueTruthy(ToValue(lir, LTestVAndBranch::Input),

                     lir->temp1(), lir->temp2(),

                     ToFloatRegister(lir->tempFloat()),

                     truthy, falsy, ool);

     return true;

 }

 bool

 CodeGenerator::visitFunctionDispatch(LFunctionDispatch *lir)

 {

     MFunctionDispatch *mir = lir->mir();

     Register input = ToRegister(lir->input());

     Label *lastLabel;

     size_t casesWithFallback;

     // Determine if the last case is fallback or an ordinary case.

     if (!mir->hasFallback()) {

         JS_ASSERT(mir->numCases() > 0);

         casesWithFallback = mir->numCases();

         lastLabel = mir->getCaseBlock(mir->numCases() - 1)->lir()->label();

     } else {

         casesWithFallback = mir->numCases() + 1;

         lastLabel = mir->getFallback()->lir()->label();

     }

     // Compare function pointers, except for the last case.

     for (size_t i = 0; i < casesWithFallback - 1; i++) {

         JS_ASSERT(i < mir->numCases());

         JSFunction *func = mir->getCase(i);

         LBlock *target = mir->getCaseBlock(i)->lir();

         masm.branchPtr(Assembler::Equal, input, ImmGCPtr(func), target->label());

     }

     // Jump to the last case.

     masm.jump(lastLabel);

     return true;

 }

 bool

 CodeGenerator::visitTypeObjectDispatch(LTypeObjectDispatch *lir)

 {

     MTypeObjectDispatch *mir = lir->mir();

     Register input = ToRegister(lir->input());

     Register temp = ToRegister(lir->temp());

     // Hold the incoming TypeObject.

     masm.loadPtr(Address(input, JSObject::offsetOfType()), temp);

     // Compare TypeObjects.

     InlinePropertyTable *propTable = mir->propTable();

     for (size_t i = 0; i < mir->numCases(); i++) {

         JSFunction *func = mir->getCase(i);

         LBlock *target = mir->getCaseBlock(i)->lir();

         DebugOnly<bool> found = false;

         for (size_t j = 0; j < propTable->numEntries(); j++) {

             if (propTable->getFunction(j) != func)

                 continue;

             types::TypeObject *typeObj = propTable->getTypeObject(j);

             masm.branchPtr(Assembler::Equal, temp, ImmGCPtr(typeObj), target->label());

             found = true;

         }

         JS_ASSERT(found);

     }

     // Unknown function: jump to fallback block.

     LBlock *fallback = mir->getFallback()->lir();

     masm.jump(fallback->label());

     return true;

 }

 bool

 CodeGenerator::visitBooleanToString(LBooleanToString *lir)

 {

     Register input = ToRegister(lir->input());

     Register output = ToRegister(lir->output());

     const JSAtomState &names = GetIonContext()->runtime->names();

     Label true_, done;

     masm.branchTest32(Assembler::NonZero, input, input, &true_);

     masm.movePtr(ImmGCPtr(names.false_), output);

     masm.jump(&done);

     masm.bind(&true_);

     masm.movePtr(ImmGCPtr(names.true_), output);

     masm.bind(&done);

     return true;

 }

 void

 CodeGenerator::emitIntToString(Register input, Register output, Label *ool)

 {

     masm.branch32(Assembler::AboveOrEqual, input, Imm32(StaticStrings::INT_STATIC_LIMIT), ool);

     // Fast path for small integers.

     masm.movePtr(ImmPtr(&GetIonContext()->runtime->staticStrings().intStaticTable), output);

     masm.loadPtr(BaseIndex(output, input, ScalePointer), output);

 }

 typedef JSFlatString *(*IntToStringFn)(ThreadSafeContext *, int);

 typedef JSFlatString *(*IntToStringParFn)(ForkJoinContext *, int);

 static const VMFunctionsModal IntToStringInfo = VMFunctionsModal(

     FunctionInfo<IntToStringFn>(Int32ToString<CanGC>),

     FunctionInfo<IntToStringParFn>(IntToStringPar));

 bool

 CodeGenerator::visitIntToString(LIntToString *lir)

 {

     Register input = ToRegister(lir->input());

     Register output = ToRegister(lir->output());

     OutOfLineCode *ool = oolCallVM(IntToStringInfo, lir, (ArgList(), input),

                                    StoreRegisterTo(output));

     if (!ool)

         return false;

     emitIntToString(input, output, ool->entry());

     masm.bind(ool->rejoin());

     return true;

 }

 typedef JSString *(*DoubleToStringFn)(ThreadSafeContext *, double);

 typedef JSString *(*DoubleToStringParFn)(ForkJoinContext *, double);

 static const VMFunctionsModal DoubleToStringInfo = VMFunctionsModal(

     FunctionInfo<DoubleToStringFn>(NumberToString<CanGC>),

     FunctionInfo<DoubleToStringParFn>(DoubleToStringPar));

 bool

 CodeGenerator::visitDoubleToString(LDoubleToString *lir)

 {

     FloatRegister input = ToFloatRegister(lir->input());

     Register temp = ToRegister(lir->tempInt());

     Register output = ToRegister(lir->output());

     OutOfLineCode *ool = oolCallVM(DoubleToStringInfo, lir, (ArgList(), input),

                                    StoreRegisterTo(output));

     if (!ool)

         return false;

     // Try double to integer conversion and run integer to string code.

     masm.convertDoubleToInt32(input, temp, ool->entry(), true);

     emitIntToString(temp, output, ool->entry());

     masm.bind(ool->rejoin());

     return true;

 }

 typedef JSString *(*PrimitiveToStringFn)(JSContext *, HandleValue);

 typedef JSString *(*PrimitiveToStringParFn)(ForkJoinContext *, HandleValue);

 static const VMFunctionsModal PrimitiveToStringInfo = VMFunctionsModal(

     FunctionInfo<PrimitiveToStringFn>(ToStringSlow),

     FunctionInfo<PrimitiveToStringParFn>(PrimitiveToStringPar));

 bool

 CodeGenerator::visitPrimitiveToString(LPrimitiveToString *lir)

 {

     ValueOperand input = ToValue(lir, LPrimitiveToString::Input);

     Register output = ToRegister(lir->output());

     OutOfLineCode *ool = oolCallVM(PrimitiveToStringInfo, lir, (ArgList(), input),

                                    StoreRegisterTo(output));

     if (!ool)

         return false;

     Label done;

     Register tag = masm.splitTagForTest(input);

     const JSAtomState &names = GetIonContext()->runtime->names();

     // String

     if (lir->mir()->input()->mightBeType(MIRType_String)) {

         Label notString;

         masm.branchTestString(Assembler::NotEqual, tag, &notString);

         masm.unboxString(input, output);

         masm.jump(&done);

         masm.bind(&notString);

     }

     // Integer

     if (lir->mir()->input()->mightBeType(MIRType_Int32)) {

         Label notInteger;

         masm.branchTestInt32(Assembler::NotEqual, tag, &notInteger);

         Register unboxed = ToTempUnboxRegister(lir->tempToUnbox());

         unboxed = masm.extractInt32(input, unboxed);

         emitIntToString(unboxed, output, ool->entry());

         masm.jump(&done);

         masm.bind(&notInteger);

     }

     // Double

     if (lir->mir()->input()->mightBeType(MIRType_Double)) {

         // Note: no fastpath. Need two extra registers and can only convert doubles

         // that fit integers and are smaller than StaticStrings::INT_STATIC_LIMIT.

         masm.branchTestDouble(Assembler::Equal, tag, ool->entry());

     }

     // Undefined

     if (lir->mir()->input()->mightBeType(MIRType_Undefined)) {

         Label notUndefined;

         masm.branchTestUndefined(Assembler::NotEqual, tag, &notUndefined);

         masm.movePtr(ImmGCPtr(names.undefined), output);

         masm.jump(&done);

         masm.bind(&notUndefined);

     }

     // Null

     if (lir->mir()->input()->mightBeType(MIRType_Null)) {

         Label notNull;

         masm.branchTestNull(Assembler::NotEqual, tag, &notNull);

         masm.movePtr(ImmGCPtr(names.null), output);

         masm.jump(&done);

         masm.bind(&notNull);

     }

     // Boolean

     if (lir->mir()->input()->mightBeType(MIRType_Boolean)) {

         Label notBoolean, true_;

         masm.branchTestBoolean(Assembler::NotEqual, tag, &notBoolean);

         masm.branchTestBooleanTruthy(true, input, &true_);

         masm.movePtr(ImmGCPtr(names.false_), output);

         masm.jump(&done);

         masm.bind(&true_);

         masm.movePtr(ImmGCPtr(names.true_), output);

         masm.jump(&done);

         masm.bind(&notBoolean);

     }

 #ifdef DEBUG

     // Objects are not supported or we see a type that wasn't accounted for.

     masm.assumeUnreachable("Unexpected type for MPrimitiveToString.");

 #endif

     masm.bind(&done);

     masm.bind(ool->rejoin());

     return true;

 }

 typedef JSObject *(*CloneRegExpObjectFn)(JSContext *, JSObject *);

 static const VMFunction CloneRegExpObjectInfo =

     FunctionInfo<CloneRegExpObjectFn>(CloneRegExpObject);

 bool

 CodeGenerator::visitRegExp(LRegExp *lir)

 {

     pushArg(ImmGCPtr(lir->mir()->source()));

     return callVM(CloneRegExpObjectInfo, lir);

 }

 typedef bool (*RegExpExecRawFn)(JSContext *cx, HandleObject regexp,

                                 HandleString input, MutableHandleValue output);

 static const VMFunction RegExpExecRawInfo = FunctionInfo<RegExpExecRawFn>(regexp_exec_raw);

 bool

 CodeGenerator::visitRegExpExec(LRegExpExec *lir)

 {

     pushArg(ToRegister(lir->string()));

     pushArg(ToRegister(lir->regexp()));

     return callVM(RegExpExecRawInfo, lir);

 }

 typedef bool (*RegExpTestRawFn)(JSContext *cx, HandleObject regexp,

                                 HandleString input, bool *result);

 static const VMFunction RegExpTestRawInfo = FunctionInfo<RegExpTestRawFn>(regexp_test_raw);

 bool

 CodeGenerator::visitRegExpTest(LRegExpTest *lir)

 {

     pushArg(ToRegister(lir->string()));

     pushArg(ToRegister(lir->regexp()));

     return callVM(RegExpTestRawInfo, lir);

 }

 typedef JSString *(*RegExpReplaceFn)(JSContext *, HandleString, HandleObject, HandleString);

 static const VMFunction RegExpReplaceInfo = FunctionInfo<RegExpReplaceFn>(RegExpReplace);

 bool

 CodeGenerator::visitRegExpReplace(LRegExpReplace *lir)

 {

     if (lir->replacement()->isConstant())

         pushArg(ImmGCPtr(lir->replacement()->toConstant()->toString()));

     else

         pushArg(ToRegister(lir->replacement()));

     pushArg(ToRegister(lir->pattern()));

     if (lir->string()->isConstant())

         pushArg(ImmGCPtr(lir->string()->toConstant()->toString()));

     else

         pushArg(ToRegister(lir->string()));

     return callVM(RegExpReplaceInfo, lir);

 }

 typedef JSString *(*StringReplaceFn)(JSContext *, HandleString, HandleString, HandleString);

 static const VMFunction StringReplaceInfo = FunctionInfo<StringReplaceFn>(StringReplace);

 bool

 CodeGenerator::visitStringReplace(LStringReplace *lir)

 {

     if (lir->replacement()->isConstant())

         pushArg(ImmGCPtr(lir->replacement()->toConstant()->toString()));

     else

         pushArg(ToRegister(lir->replacement()));

     if (lir->pattern()->isConstant())

         pushArg(ImmGCPtr(lir->pattern()->toConstant()->toString()));

     else

         pushArg(ToRegister(lir->pattern()));

     if (lir->string()->isConstant())

         pushArg(ImmGCPtr(lir->string()->toConstant()->toString()));

     else

         pushArg(ToRegister(lir->string()));

     return callVM(StringReplaceInfo, lir);

 }

 typedef JSObject *(*LambdaFn)(JSContext *, HandleFunction, HandleObject);

 static const VMFunction LambdaInfo = FunctionInfo<LambdaFn>(js::Lambda);

 bool

 CodeGenerator::visitLambdaForSingleton(LLambdaForSingleton *lir)

 {

     pushArg(ToRegister(lir->scopeChain()));

     pushArg(ImmGCPtr(lir->mir()->info().fun));

     return callVM(LambdaInfo, lir);

 }

 bool

 CodeGenerator::visitLambda(LLambda *lir)

 {

     Register scopeChain = ToRegister(lir->scopeChain());

     Register output = ToRegister(lir->output());

     Register tempReg = ToRegister(lir->temp());

     const LambdaFunctionInfo &info = lir->mir()->info();

     OutOfLineCode *ool = oolCallVM(LambdaInfo, lir, (ArgList(), ImmGCPtr(info.fun), scopeChain),

                                    StoreRegisterTo(output));

     if (!ool)

         return false;

     JS_ASSERT(!info.singletonType);

     masm.newGCThing(output, tempReg, info.fun, ool->entry(), gc::DefaultHeap);

     masm.initGCThing(output, tempReg, info.fun);

     emitLambdaInit(output, scopeChain, info);

     masm.bind(ool->rejoin());

     return true;

 }

 typedef JSObject *(*LambdaArrowFn)(JSContext *, HandleFunction, HandleObject, HandleValue);

 static const VMFunction LambdaArrowInfo = FunctionInfo<LambdaArrowFn>(js::LambdaArrow);

 bool

 CodeGenerator::visitLambdaArrow(LLambdaArrow *lir)

 {

     Register scopeChain = ToRegister(lir->scopeChain());

     ValueOperand thisv = ToValue(lir, LLambdaArrow::ThisValue);

     Register output = ToRegister(lir->output());

     Register tempReg = ToRegister(lir->temp());

     const LambdaFunctionInfo &info = lir->mir()->info();

     OutOfLineCode *ool = oolCallVM(LambdaArrowInfo, lir,

                                    (ArgList(), ImmGCPtr(info.fun), scopeChain, thisv),

                                    StoreRegisterTo(output));

     if (!ool)

         return false;

     MOZ_ASSERT(!info.useNewTypeForClone);

     if (info.singletonType) {

         // If the function has a singleton type, this instruction will only be

         // executed once so we don't bother inlining it.

         masm.jump(ool->entry());

         masm.bind(ool->rejoin());

         return true;

     }

     masm.newGCThing(output, tempReg, info.fun, ool->entry(), gc::DefaultHeap);

     masm.initGCThing(output, tempReg, info.fun);

     emitLambdaInit(output, scopeChain, info);

     // Initialize extended slots. Lexical |this| is stored in the first one.

     MOZ_ASSERT(info.flags & JSFunction::EXTENDED);

     static_assert(FunctionExtended::NUM_EXTENDED_SLOTS == 2, "All slots must be initialized");

     static_assert(FunctionExtended::ARROW_THIS_SLOT == 0, "|this| must be stored in first slot");

     masm.storeValue(thisv, Address(output, FunctionExtended::offsetOfExtendedSlot(0)));

     masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(1)));

     masm.bind(ool->rejoin());

     return true;

 }

 void

 CodeGenerator::emitLambdaInit(Register output, Register scopeChain,

                               const LambdaFunctionInfo &info)

 {

     MOZ_ASSERT(!!(info.flags & JSFunction::ARROW) == !!(info.flags & JSFunction::EXTENDED));

     // Initialize nargs and flags. We do this with a single uint32 to avoid

     // 16-bit writes.

     union {

         struct S {

             uint16_t nargs;

             uint16_t flags;

         } s;

         uint32_t word;

     } u;

     u.s.nargs = info.fun->nargs();

     u.s.flags = info.flags;

     JS_ASSERT(JSFunction::offsetOfFlags() == JSFunction::offsetOfNargs() + 2);

     masm.store32(Imm32(u.word), Address(output, JSFunction::offsetOfNargs()));

     masm.storePtr(ImmGCPtr(info.scriptOrLazyScript),

                   Address(output, JSFunction::offsetOfNativeOrScript()));

     masm.storePtr(scopeChain, Address(output, JSFunction::offsetOfEnvironment()));

     masm.storePtr(ImmGCPtr(info.fun->displayAtom()), Address(output, JSFunction::offsetOfAtom()));

 }

 bool

 CodeGenerator::visitLambdaPar(LLambdaPar *lir)

 {

     Register resultReg = ToRegister(lir->output());

     Register cxReg = ToRegister(lir->forkJoinContext());

     Register scopeChainReg = ToRegister(lir->scopeChain());

     Register tempReg1 = ToRegister(lir->getTemp0());

     Register tempReg2 = ToRegister(lir->getTemp1());

     const LambdaFunctionInfo &info = lir->mir()->info();

     JS_ASSERT(scopeChainReg != resultReg);

     emitAllocateGCThingPar(lir, resultReg, cxReg, tempReg1, tempReg2, info.fun);

     emitLambdaInit(resultReg, scopeChainReg, info);

     return true;

 }

 bool

 CodeGenerator::visitLabel(LLabel *lir)

 {

     return true;

 }

 bool

 CodeGenerator::visitNop(LNop *lir)

 {

     return true;

 }

 bool

 CodeGenerator::visitOsiPoint(LOsiPoint *lir)

 {

     // Note: markOsiPoint ensures enough space exists between the last

     // LOsiPoint and this one to patch adjacent call instructions.

     JS_ASSERT(masm.framePushed() == frameSize());

     uint32_t osiCallPointOffset;

     if (!markOsiPoint(lir, &osiCallPointOffset))

         return false;

     LSafepoint *safepoint = lir->associatedSafepoint();

     JS_ASSERT(!safepoint->osiCallPointOffset());

     safepoint->setOsiCallPointOffset(osiCallPointOffset);

 #ifdef DEBUG

     // There should be no movegroups or other instructions between

     // an instruction and its OsiPoint. This is necessary because

     // we use the OsiPoint's snapshot from within VM calls.

     for (LInstructionReverseIterator iter(current->rbegin(lir)); iter != current->rend(); iter++) {

         if (*iter == lir || iter->isNop())

             continue;

         JS_ASSERT(!iter->isMoveGroup());

         JS_ASSERT(iter->safepoint() == safepoint);

         break;

     }

 #endif

 #ifdef CHECK_OSIPOINT_REGISTERS

     if (shouldVerifyOsiPointRegs(safepoint))

         verifyOsiPointRegs(safepoint);

 #endif

     return true;

 }

 bool

 CodeGenerator::visitGoto(LGoto *lir)

 {

     jumpToBlock(lir->target());

     return true;

 }

 // Out-of-line path to execute any move groups between the start of a loop

 // header and its interrupt check, then invoke the interrupt handler.

 class OutOfLineInterruptCheckImplicit : public OutOfLineCodeBase<CodeGenerator>

 {

   public:

     LBlock *block;

     LInterruptCheckImplicit *lir;

     OutOfLineInterruptCheckImplicit(LBlock *block, LInterruptCheckImplicit *lir)

       : block(block), lir(lir)

     { }

     bool accept(CodeGenerator *codegen) {

         return codegen->visitOutOfLineInterruptCheckImplicit(this);

     }

 };

 bool

 CodeGenerator::visitOutOfLineInterruptCheckImplicit(OutOfLineInterruptCheckImplicit *ool)

 {

 #ifdef CHECK_OSIPOINT_REGISTERS

     // This is path is entered from the patched back-edge of the loop. This

     // means that the JitAtivation flags used for checking the validity of the

     // OSI points are not reseted by the path generated by generateBody, so we

     // have to reset it here.

     resetOsiPointRegs(ool->lir->safepoint());

 #endif

     LInstructionIterator iter = ool->block->begin();

     for (; iter != ool->block->end(); iter++) {

         if (iter->isLabel()) {

             // Nothing to do.

         } else if (iter->isMoveGroup()) {

             // Replay this move group that preceds the interrupt check at the

             // start of the loop header. Any incoming jumps here will be from

             // the backedge and will skip over the move group emitted inline.

             visitMoveGroup(iter->toMoveGroup());

         } else {

             break;

         }

     }

     JS_ASSERT(*iter == ool->lir);

     saveLive(ool->lir);

     if (!callVM(InterruptCheckInfo, ool->lir))

         return false;

     restoreLive(ool->lir);

     masm.jump(ool->rejoin());

     return true;

 }

 bool

 CodeGenerator::visitInterruptCheckImplicit(LInterruptCheckImplicit *lir)

 {

     OutOfLineInterruptCheckImplicit *ool = new(alloc()) OutOfLineInterruptCheckImplicit(current, lir);

     if (!addOutOfLineCode(ool))

         return false;

     lir->setOolEntry(ool->entry());

     masm.bind(ool->rejoin());

     return true;

 }

 bool

 CodeGenerator::visitTableSwitch(LTableSwitch *ins)

 {

     MTableSwitch *mir = ins->mir();

     Label *defaultcase = mir->getDefault()->lir()->label();

     const LAllocation *temp;

     if (mir->getOperand(0)->type() != MIRType_Int32) {

         temp = ins->tempInt()->output();

         // The input is a double, so try and convert it to an integer.

         // If it does not fit in an integer, take the default case.

         masm.convertDoubleToInt32(ToFloatRegister(ins->index()), ToRegister(temp), defaultcase, false);

     } else {

         temp = ins->index();

     }

     return emitTableSwitchDispatch(mir, ToRegister(temp), ToRegisterOrInvalid(ins->tempPointer()));

 }

 bool

 CodeGenerator::visitTableSwitchV(LTableSwitchV *ins)

 {

     MTableSwitch *mir = ins->mir();

     Label *defaultcase = mir->getDefault()->lir()->label();

     Register index = ToRegister(ins->tempInt());

     ValueOperand value = ToValue(ins, LTableSwitchV::InputValue);

     Register tag = masm.extractTag(value, index);

     masm.branchTestNumber(Assembler::NotEqual, tag, defaultcase);

     Label unboxInt, isInt;

     masm.branchTestInt32(Assembler::Equal, tag, &unboxInt);

     {

         FloatRegister floatIndex = ToFloatRegister(ins->tempFloat());

         masm.unboxDouble(value, floatIndex);

         masm.convertDoubleToInt32(floatIndex, index, defaultcase, false);

         masm.jump(&isInt);

     }

     masm.bind(&unboxInt);

     masm.unboxInt32(value, index);

     masm.bind(&isInt);

     return emitTableSwitchDispatch(mir, index, ToRegisterOrInvalid(ins->tempPointer()));

 }

 typedef JSObject *(*DeepCloneObjectLiteralFn)(JSContext *, HandleObject, NewObjectKind);

 static const VMFunction DeepCloneObjectLiteralInfo =

     FunctionInfo<DeepCloneObjectLiteralFn>(DeepCloneObjectLiteral);

 bool

 CodeGenerator::visitCloneLiteral(LCloneLiteral *lir)

 {

     pushArg(ImmWord(js::MaybeSingletonObject));

     pushArg(ToRegister(lir->output()));

     return callVM(DeepCloneObjectLiteralInfo, lir);

 }

 bool

 CodeGenerator::visitParameter(LParameter *lir)

 {

     return true;

 }

 bool

 CodeGenerator::visitCallee(LCallee *lir)

 {

     // read number of actual arguments from the JS frame.

     Register callee = ToRegister(lir->output());

     Address ptr(StackPointer, frameSize() + IonJSFrameLayout::offsetOfCalleeToken());

     masm.loadPtr(ptr, callee);

     return true;

 }

 bool

 CodeGenerator::visitStart(LStart *lir)

 {

     return true;

 }

 bool

 CodeGenerator::visitReturn(LReturn *lir)

 {

 #if defined(JS_NUNBOX32)

     DebugOnly<LAllocation *> type    = lir->getOperand(TYPE_INDEX);

     DebugOnly<LAllocation *> payload = lir->getOperand(PAYLOAD_INDEX);

     JS_ASSERT(ToRegister(type)    == JSReturnReg_Type);

     JS_ASSERT(ToRegister(payload) == JSReturnReg_Data);

 #elif defined(JS_PUNBOX64)

     DebugOnly<LAllocation *> result = lir->getOperand(0);

     JS_ASSERT(ToRegister(result) == JSReturnReg);

 #endif

     // Don't emit a jump to the return label if this is the last block.

     if (current->mir() != *gen->graph().poBegin())

         masm.jump(&returnLabel_);

     return true;

 }

 bool

 CodeGenerator::visitOsrEntry(LOsrEntry *lir)

 {

     // Remember the OSR entry offset into the code buffer.

     masm.flushBuffer();

     setOsrEntryOffset(masm.size());

 #ifdef JS_TRACE_LOGGING

     if (gen->info().executionMode() == SequentialExecution) {

         if (!emitTracelogStopEvent(TraceLogger::Baseline))

             return false;

         if (!emitTracelogStartEvent(TraceLogger::IonMonkey))

             return false;

     }

 #endif

     // Allocate the full frame for this function.

     uint32_t size = frameSize();

     if (size != 0)

         masm.subPtr(Imm32(size), StackPointer);

     return true;

 }

 bool

 CodeGenerator::visitOsrScopeChain(LOsrScopeChain *lir)

 {

     const LAllocation *frame   = lir->getOperand(0);

     const LDefinition *object  = lir->getDef(0);

     const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfScopeChain();

     masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));

     return true;

 }

 bool

 CodeGenerator::visitOsrArgumentsObject(LOsrArgumentsObject *lir)

 {

     const LAllocation *frame   = lir->getOperand(0);

     const LDefinition *object  = lir->getDef(0);

     const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfArgsObj();

     masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));

     return true;

 }

 bool

 CodeGenerator::visitOsrValue(LOsrValue *value)

 {

     const LAllocation *frame   = value->getOperand(0);

     const ValueOperand out     = ToOutValue(value);

     const ptrdiff_t frameOffset = value->mir()->frameOffset();

     masm.loadValue(Address(ToRegister(frame), frameOffset), out);

     return true;

 }

 bool

 CodeGenerator::visitOsrReturnValue(LOsrReturnValue *lir)

 {

     const LAllocation *frame   = lir->getOperand(0);

     const ValueOperand out     = ToOutValue(lir);

     Address flags = Address(ToRegister(frame), BaselineFrame::reverseOffsetOfFlags());

     Address retval = Address(ToRegister(frame), BaselineFrame::reverseOffsetOfReturnValue());

     masm.moveValue(UndefinedValue(), out);

     Label done;

     masm.branchTest32(Assembler::Zero, flags, Imm32(BaselineFrame::HAS_RVAL), &done);

     masm.loadValue(retval, out);

     masm.bind(&done);

     return true;

 }

 bool

 CodeGenerator::visitStackArgT(LStackArgT *lir)

 {

     const LAllocation *arg = lir->getArgument();

     MIRType argType = lir->type();

     uint32_t argslot = lir->argslot();

     JS_ASSERT(argslot - 1u < graph.argumentSlotCount());

     int32_t stack_offset = StackOffsetOfPassedArg(argslot);

     Address dest(StackPointer, stack_offset);

     if (arg->isFloatReg())

         masm.storeDouble(ToFloatRegister(arg), dest);

     else if (arg->isRegister())

         masm.storeValue(ValueTypeFromMIRType(argType), ToRegister(arg), dest);

     else

         masm.storeValue(*(arg->toConstant()), dest);

     uint32_t slot = StackOffsetToSlot(stack_offset);

     JS_ASSERT(slot - 1u < graph.totalSlotCount());

     return pushedArgumentSlots_.append(slot);

 }

 bool

 CodeGenerator::visitStackArgV(LStackArgV *lir)

 {

     ValueOperand val = ToValue(lir, 0);

     uint32_t argslot = lir->argslot();

     JS_ASSERT(argslot - 1u < graph.argumentSlotCount());

     int32_t stack_offset = StackOffsetOfPassedArg(argslot);

     masm.storeValue(val, Address(StackPointer, stack_offset));

     uint32_t slot = StackOffsetToSlot(stack_offset);

     JS_ASSERT(slot - 1u < graph.totalSlotCount());

     return pushedArgumentSlots_.append(slot);

 }

 bool

 CodeGenerator::visitMoveGroup(LMoveGroup *group)

 {

     if (!group->numMoves())

         return true;

     MoveResolver &resolver = masm.moveResolver();

     for (size_t i = 0; i < group->numMoves(); i++) {

         const LMove &move = group->getMove(i);

         const LAllocation *from = move.from();

         const LAllocation *to = move.to();

         LDefinition::Type type = move.type();

         // No bogus moves.

         JS_ASSERT(*from != *to);

         JS_ASSERT(!from->isConstant());

         MoveOp::Type moveType;

         switch (type) {

           case LDefinition::OBJECT:

           case LDefinition::SLOTS:

 #ifdef JS_NUNBOX32

           case LDefinition::TYPE:

           case LDefinition::PAYLOAD:

 #else

           case LDefinition::BOX:

 #endif

           case LDefinition::GENERAL: moveType = MoveOp::GENERAL; break;

           case LDefinition::INT32:   moveType = MoveOp::INT32;   break;

           case LDefinition::FLOAT32: moveType = MoveOp::FLOAT32; break;

           case LDefinition::DOUBLE:  moveType = MoveOp::DOUBLE;  break;

           default: MOZ_ASSUME_UNREACHABLE("Unexpected move type");

         }

         if (!resolver.addMove(toMoveOperand(from), toMoveOperand(to), moveType))

             return false;

     }

     if (!resolver.resolve())

         return false;

     MoveEmitter emitter(masm);

     emitter.emit(resolver);

     emitter.finish();

     return true;

 }

 bool

 CodeGenerator::visitInteger(LInteger *lir)

 {

     masm.move32(Imm32(lir->getValue()), ToRegister(lir->output()));

     return true;

 }

 bool

 CodeGenerator::visitPointer(LPointer *lir)

 {

     if (lir->kind() == LPointer::GC_THING)

         masm.movePtr(ImmGCPtr(lir->gcptr()), ToRegister(lir->output()));

     else

         masm.movePtr(ImmPtr(lir->ptr()), ToRegister(lir->output()));

     return true;

 }

 bool

 CodeGenerator::visitSlots(LSlots *lir)

 {

     Address slots(ToRegister(lir->object()), JSObject::offsetOfSlots());

     masm.loadPtr(slots, ToRegister(lir->output()));

     return true;

 }

 bool

 CodeGenerator::visitStoreSlotV(LStoreSlotV *store)

 {

     Register base = ToRegister(store->slots());

     int32_t offset = store->mir()->slot() * sizeof(Value);

     const ValueOperand value = ToValue(store, LStoreSlotV::Value);

     if (store->mir()->needsBarrier())

        emitPreBarrier(Address(base, offset), MIRType_Value);

     masm.storeValue(value, Address(base, offset));

     return true;

 }

 bool

 CodeGenerator::emitGetPropertyPolymorphic(LInstruction *ins, Register obj, Register scratch,

                                           const TypedOrValueRegister &output)

 {

     MGetPropertyPolymorphic *mir = ins->mirRaw()->toGetPropertyPolymorphic();

     JS_ASSERT(mir->numShapes() > 1);

     masm.loadObjShape(obj, scratch);

     Label done;

     for (size_t i = 0; i < mir->numShapes(); i++) {

         Label next;

         masm.branchPtr(Assembler::NotEqual, scratch, ImmGCPtr(mir->objShape(i)), &next);

         Shape *shape = mir->shape(i);

         if (shape->slot() < shape->numFixedSlots()) {

             // Fixed slot.

             masm.loadTypedOrValue(Address(obj, JSObject::getFixedSlotOffset(shape->slot())),

                                   output);

         } else {

             // Dynamic slot.

             uint32_t offset = (shape->slot() - shape->numFixedSlots()) * sizeof(js::Value);

             masm.loadPtr(Address(obj, JSObject::offsetOfSlots()), scratch);

             masm.loadTypedOrValue(Address(scratch, offset), output);

         }

         masm.jump(&done);

         masm.bind(&next);

     }

     // Bailout if no shape matches.

     if (!bailout(ins->snapshot()))

         return false;

     masm.bind(&done);

     return true;

 }

 bool

 CodeGenerator::visitGetPropertyPolymorphicV(LGetPropertyPolymorphicV *ins)

 {

     Register obj = ToRegister(ins->obj());

     ValueOperand output = GetValueOutput(ins);

     return emitGetPropertyPolymorphic(ins, obj, output.scratchReg(), output);

 }

 bool

 CodeGenerator::visitGetPropertyPolymorphicT(LGetPropertyPolymorphicT *ins)

 {

     Register obj = ToRegister(ins->obj());

     TypedOrValueRegister output(ins->mir()->type(), ToAnyRegister(ins->output()));

     Register temp = (output.type() == MIRType_Double)

                     ? ToRegister(ins->temp())

                     : output.typedReg().gpr();

     return emitGetPropertyPolymorphic(ins, obj, temp, output);

 }

 bool

 CodeGenerator::emitSetPropertyPolymorphic(LInstruction *ins, Register obj, Register scratch,

                                           const ConstantOrRegister &value)

 {

     MSetPropertyPolymorphic *mir = ins->mirRaw()->toSetPropertyPolymorphic();

     JS_ASSERT(mir->numShapes() > 1);

     masm.loadObjShape(obj, scratch);

     Label done;

     for (size_t i = 0; i < mir->numShapes(); i++) {

         Label next;

         masm.branchPtr(Assembler::NotEqual, scratch, ImmGCPtr(mir->objShape(i)), &next);

         Shape *shape = mir->shape(i);

         if (shape->slot() < shape->numFixedSlots()) {

             // Fixed slot.

             Address addr(obj, JSObject::getFixedSlotOffset(shape->slot()));

             if (mir->needsBarrier())

                 emitPreBarrier(addr, MIRType_Value);

             masm.storeConstantOrRegister(value, addr);

         } else {

             // Dynamic slot.

             masm.loadPtr(Address(obj, JSObject::offsetOfSlots()), scratch);

             Address addr(scratch, (shape->slot() - shape->numFixedSlots()) * sizeof(js::Value));

             if (mir->needsBarrier())

                 emitPreBarrier(addr, MIRType_Value);

             masm.storeConstantOrRegister(value, addr);

         }

         masm.jump(&done);

         masm.bind(&next);

     }

     // Bailout if no shape matches.

     if (!bailout(ins->snapshot()))

         return false;

     masm.bind(&done);

     return true;

 }

 bool

 CodeGenerator::visitSetPropertyPolymorphicV(LSetPropertyPolymorphicV *ins)

 {

     Register obj = ToRegister(ins->obj());

     Register temp = ToRegister(ins->temp());

     ValueOperand value = ToValue(ins, LSetPropertyPolymorphicV::Value);

     return emitSetPropertyPolymorphic(ins, obj, temp, TypedOrValueRegister(value));

 }

 bool

 CodeGenerator::visitSetPropertyPolymorphicT(LSetPropertyPolymorphicT *ins)

 {

     Register obj = ToRegister(ins->obj());

     Register temp = ToRegister(ins->temp());

     ConstantOrRegister value;

     if (ins->mir()->value()->isConstant())

         value = ConstantOrRegister(ins->mir()->value()->toConstant()->value());

     else

         value = TypedOrValueRegister(ins->mir()->value()->type(), ToAnyRegister(ins->value()));

     return emitSetPropertyPolymorphic(ins, obj, temp, value);

 }

 bool

 CodeGenerator::visitElements(LElements *lir)

 {

     Address elements(ToRegister(lir->object()), JSObject::offsetOfElements());

     masm.loadPtr(elements, ToRegister(lir->output()));

     return true;

 }

 typedef bool (*ConvertElementsToDoublesFn)(JSContext *, uintptr_t);

 static const VMFunction ConvertElementsToDoublesInfo =

     FunctionInfo<ConvertElementsToDoublesFn>(ObjectElements::ConvertElementsToDoubles);

 bool

 CodeGenerator::visitConvertElementsToDoubles(LConvertElementsToDoubles *lir)

 {

     Register elements = ToRegister(lir->elements());

     OutOfLineCode *ool = oolCallVM(ConvertElementsToDoublesInfo, lir,

                                    (ArgList(), elements), StoreNothing());

     if (!ool)

         return false;

     Address convertedAddress(elements, ObjectElements::offsetOfFlags());

     Imm32 bit(ObjectElements::CONVERT_DOUBLE_ELEMENTS);

     masm.branchTest32(Assembler::Zero, convertedAddress, bit, ool->entry());

     masm.bind(ool->rejoin());

     return true;

 }

 bool

 CodeGenerator::visitMaybeToDoubleElement(LMaybeToDoubleElement *lir)

 {

     Register elements = ToRegister(lir->elements());

     Register value = ToRegister(lir->value());

     ValueOperand out = ToOutValue(lir);

     FloatRegister temp = ToFloatRegister(lir->tempFloat());

     Label convert, done;

     // If the CONVERT_DOUBLE_ELEMENTS flag is set, convert the int32

     // value to double. Else, just box it.

     masm.branchTest32(Assembler::NonZero,

                       Address(elements, ObjectElements::offsetOfFlags()),

                       Imm32(ObjectElements::CONVERT_DOUBLE_ELEMENTS),

                       &convert);

     masm.tagValue(JSVAL_TYPE_INT32, value, out);

     masm.jump(&done);

     masm.bind(&convert);

     masm.convertInt32ToDouble(value, temp);

     masm.boxDouble(temp, out);

     masm.bind(&done);

     return true;

 }

 bool

 CodeGenerator::visitFunctionEnvironment(LFunctionEnvironment *lir)

 {

     Address environment(ToRegister(lir->function()), JSFunction::offsetOfEnvironment());

     masm.loadPtr(environment, ToRegister(lir->output()));

     return true;

 }

 bool

 CodeGenerator::visitForkJoinContext(LForkJoinContext *lir)

 {

     const Register tempReg = ToRegister(lir->getTempReg());

     masm.setupUnalignedABICall(0, tempReg);

     masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, ForkJoinContextPar));

     JS_ASSERT(ToRegister(lir->output()) == ReturnReg);

     return true;

 }

 bool

 CodeGenerator::visitGuardThreadExclusive(LGuardThreadExclusive *lir)

 {

     JS_ASSERT(gen->info().executionMode() == ParallelExecution);

     const Register tempReg = ToRegister(lir->getTempReg());

     masm.setupUnalignedABICall(2, tempReg);

     masm.passABIArg(ToRegister(lir->forkJoinContext()));

     masm.passABIArg(ToRegister(lir->object()));

     masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, ParallelWriteGuard));

     OutOfLineAbortPar *bail = oolAbortPar(ParallelBailoutIllegalWrite, lir);

     if (!bail)

         return false;

     // branch to the OOL failure code if false is returned

     masm.branchIfFalseBool(ReturnReg, bail->entry());

     return true;

 }

 bool

 CodeGenerator::visitGuardObjectIdentity(LGuardObjectIdentity *guard)

 {

     Register obj = ToRegister(guard->input());

     Assembler::Condition cond =

         guard->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual;

     return bailoutCmpPtr(cond, obj, ImmGCPtr(guard->mir()->singleObject()), guard->snapshot());

 }

 bool

 CodeGenerator::visitTypeBarrierV(LTypeBarrierV *lir)

 {

     ValueOperand operand = ToValue(lir, LTypeBarrierV::Input);

     Register scratch = ToTempRegisterOrInvalid(lir->temp());

     Label miss;

     masm.guardTypeSet(operand, lir->mir()->resultTypeSet(), scratch, &miss);

     if (!bailoutFrom(&miss, lir->snapshot()))

         return false;

     return true;

 }

 bool

 CodeGenerator::visitTypeBarrierO(LTypeBarrierO *lir)

 {

     Register obj = ToRegister(lir->object());

     Register scratch = ToTempRegisterOrInvalid(lir->temp());

     Label miss;

     masm.guardObjectType(obj, lir->mir()->resultTypeSet(), scratch, &miss);

     if (!bailoutFrom(&miss, lir->snapshot()))

         return false;

     return true;

 }

 bool

 CodeGenerator::visitMonitorTypes(LMonitorTypes *lir)

 {

     ValueOperand operand = ToValue(lir, LMonitorTypes::Input);

     Register scratch = ToTempUnboxRegister(lir->temp());

     Label matched, miss;

     masm.guardTypeSet(operand, lir->mir()->typeSet(), scratch, &miss);

     if (!bailoutFrom(&miss, lir->snapshot()))

         return false;

     return true;

 }

 #ifdef JSGC_GENERATIONAL

 // Out-of-line path to update the store buffer.

 class OutOfLineCallPostWriteBarrier : public OutOfLineCodeBase<CodeGenerator>

 {

     LInstruction *lir_;

     const LAllocation *object_;

   public:

     OutOfLineCallPostWriteBarrier(LInstruction *lir, const LAllocation *object)

       : lir_(lir), object_(object)

     { }

     bool accept(CodeGenerator *codegen) {

         return codegen->visitOutOfLineCallPostWriteBarrier(this);

     }

     LInstruction *lir() const {

         return lir_;

     }

     const LAllocation *object() const {

         return object_;

     }

 };

 bool

 CodeGenerator::visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier *ool)

 {

     saveLiveVolatile(ool->lir());

     const LAllocation *obj = ool->object();

     GeneralRegisterSet regs = GeneralRegisterSet::Volatile();

     Register objreg;

     bool isGlobal = false;

     if (obj->isConstant()) {

         JSObject *object = &obj->toConstant()->toObject();

         isGlobal = object->is<GlobalObject>();

         objreg = regs.takeAny();

         masm.movePtr(ImmGCPtr(object), objreg);

     } else {

         objreg = ToRegister(obj);

         regs.takeUnchecked(objreg);

     }

     Register runtimereg = regs.takeAny();

     masm.mov(ImmPtr(GetIonContext()->runtime), runtimereg);

     void (*fun)(JSRuntime*, JSObject*) = isGlobal ? PostGlobalWriteBarrier : PostWriteBarrier;

     masm.setupUnalignedABICall(2, regs.takeAny());

     masm.passABIArg(runtimereg);

     masm.passABIArg(objreg);

     masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, fun));

     restoreLiveVolatile(ool->lir());

     masm.jump(ool->rejoin());

     return true;

 }

 #endif

 bool

 CodeGenerator::visitPostWriteBarrierO(LPostWriteBarrierO *lir)

 {

 #ifdef JSGC_GENERATIONAL

     OutOfLineCallPostWriteBarrier *ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());

     if (!addOutOfLineCode(ool))

         return false;

     Register temp = ToTempRegisterOrInvalid(lir->temp());

     if (lir->object()->isConstant()) {

 #ifdef DEBUG

         const Nursery &nursery = GetIonContext()->runtime->gcNursery();

         JS_ASSERT(!nursery.isInside(&lir->object()->toConstant()->toObject()));

 #endif

     } else {

         masm.branchPtrInNurseryRange(ToRegister(lir->object()), temp, ool->rejoin());

     }

     masm.branchPtrInNurseryRange(ToRegister(lir->value()), temp, ool->entry());

     masm.bind(ool->rejoin());

 #endif

     return true;

 }

 bool

 CodeGenerator::visitPostWriteBarrierV(LPostWriteBarrierV *lir)

 {

 #ifdef JSGC_GENERATIONAL

     OutOfLineCallPostWriteBarrier *ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());

     if (!addOutOfLineCode(ool))

         return false;

     Register temp = ToTempRegisterOrInvalid(lir->temp());

     if (lir->object()->isConstant()) {

 #ifdef DEBUG

         const Nursery &nursery = GetIonContext()->runtime->gcNursery();

         JS_ASSERT(!nursery.isInside(&lir->object()->toConstant()->toObject()));

 #endif

     } else {

         masm.branchPtrInNurseryRange(ToRegister(lir->object()), temp, ool->rejoin());

     }

     ValueOperand value = ToValue(lir, LPostWriteBarrierV::Input);

     masm.branchValueIsNurseryObject(value, temp, ool->entry());

     masm.bind(ool->rejoin());

 #endif

     return true;

 }

 bool

 CodeGenerator::visitCallNative(LCallNative *call)

 {

     JSFunction *target = call->getSingleTarget();

     JS_ASSERT(target);

     JS_ASSERT(target->isNative());

     int callargslot = call->argslot();

     int unusedStack = StackOffsetOfPassedArg(callargslot);

     // Registers used for callWithABI() argument-passing.

     const Register argContextReg   = ToRegister(call->getArgContextReg());

     const Register argUintNReg     = ToRegister(call->getArgUintNReg());

     const Register argVpReg        = ToRegister(call->getArgVpReg());

     // Misc. temporary registers.

     const Register tempReg = ToRegister(call->getTempReg());

     DebugOnly<uint32_t> initialStack = masm.framePushed();

     masm.checkStackAlignment();

     // Sequential native functions have the signature:

     //  bool (*)(JSContext *, unsigned, Value *vp)

     // and parallel native functions have the signature:

     //  ParallelResult (*)(ForkJoinContext *, unsigned, Value *vp)

     // Where vp[0] is space for an outparam, vp[1] is |this|, and vp[2] onward

     // are the function arguments.

     // Allocate space for the outparam, moving the StackPointer to what will be &vp[1].

     masm.adjustStack(unusedStack);

     // Push a Value containing the callee object: natives are allowed to access their callee before

     // setitng the return value. The StackPointer is moved to &vp[0].

     masm.Push(ObjectValue(*target));

     // Preload arguments into registers.

     //

     // Note that for parallel execution, loadContext does an ABI call, so we

     // need to do this before we load the other argument registers, otherwise

     // we'll hose them.

     ExecutionMode executionMode = gen->info().executionMode();

     masm.loadContext(argContextReg, tempReg, executionMode);

     masm.move32(Imm32(call->numStackArgs()), argUintNReg);

     masm.movePtr(StackPointer, argVpReg);

     masm.Push(argUintNReg);

     // Construct native exit frame.

     uint32_t safepointOffset;

     if (!masm.buildFakeExitFrame(tempReg, &safepointOffset))

         return false;

     masm.enterFakeExitFrame(argContextReg, tempReg, executionMode);

     if (!markSafepointAt(safepointOffset, call))

         return false;

     // Construct and execute call.

     masm.setupUnalignedABICall(3, tempReg);

     masm.passABIArg(argContextReg);

     masm.passABIArg(argUintNReg);

     masm.passABIArg(argVpReg);

     switch (executionMode) {

       case SequentialExecution:

         masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, target->native()));

         break;

       case ParallelExecution:

         masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, target->parallelNative()));

         break;

       default:

         MOZ_ASSUME_UNREACHABLE("No such execution mode");

     }

     // Test for failure.

     masm.branchIfFalseBool(ReturnReg, masm.failureLabel(executionMode));

     // Load the outparam vp[0] into output register(s).

     masm.loadValue(Address(StackPointer, IonNativeExitFrameLayout::offsetOfResult()), JSReturnOperand);

     // The next instruction is removing the footer of the exit frame, so there

     // is no need for leaveFakeExitFrame.

     // Move the StackPointer back to its original location, unwinding the native exit frame.

     masm.adjustStack(IonNativeExitFrameLayout::Size() - unusedStack);

     JS_ASSERT(masm.framePushed() == initialStack);

     dropArguments(call->numStackArgs() + 1);

     return true;

 }

 bool

 CodeGenerator::visitCallDOMNative(LCallDOMNative *call)

 {

     JSFunction *target = call->getSingleTarget();

     JS_ASSERT(target);

     JS_ASSERT(target->isNative());

     JS_ASSERT(target->jitInfo());

     JS_ASSERT(call->mir()->isCallDOMNative());

     int callargslot = call->argslot();

     int unusedStack = StackOffsetOfPassedArg(callargslot);

     // Registers used for callWithABI() argument-passing.

     const Register argJSContext = ToRegister(call->getArgJSContext());

     const Register argObj       = ToRegister(call->getArgObj());

     const Register argPrivate   = ToRegister(call->getArgPrivate());

     const Register argArgs      = ToRegister(call->getArgArgs());

     DebugOnly<uint32_t> initialStack = masm.framePushed();

     masm.checkStackAlignment();

     // DOM methods have the signature:

     //  bool (*)(JSContext *, HandleObject, void *private, const JSJitMethodCallArgs& args)

     // Where args is initialized from an argc and a vp, vp[0] is space for an

     // outparam and the callee, vp[1] is |this|, and vp[2] onward are the

     // function arguments.  Note that args stores the argv, not the vp, and

     // argv == vp + 2.

     // Nestle the stack up against the pushed arguments, leaving StackPointer at

     // &vp[1]

     masm.adjustStack(unusedStack);

     // argObj is filled with the extracted object, then returned.

     Register obj = masm.extractObject(Address(StackPointer, 0), argObj);

     JS_ASSERT(obj == argObj);

     // Push a Value containing the callee object: natives are allowed to access their callee before

     // setitng the return value. After this the StackPointer points to &vp[0].

     masm.Push(ObjectValue(*target));

     // Now compute the argv value.  Since StackPointer is pointing to &vp[0] and

     // argv is &vp[2] we just need to add 2*sizeof(Value) to the current

     // StackPointer.

     JS_STATIC_ASSERT(JSJitMethodCallArgsTraits::offsetOfArgv == 0);

     JS_STATIC_ASSERT(JSJitMethodCallArgsTraits::offsetOfArgc ==

                      IonDOMMethodExitFrameLayoutTraits::offsetOfArgcFromArgv);

     masm.computeEffectiveAddress(Address(StackPointer, 2 * sizeof(Value)), argArgs);

     // GetReservedSlot(obj, DOM_OBJECT_SLOT).toPrivate()

     masm.loadPrivate(Address(obj, JSObject::getFixedSlotOffset(0)), argPrivate);

     // Push argc from the call instruction into what will become the IonExitFrame

     masm.Push(Imm32(call->numStackArgs()));

     // Push our argv onto the stack

     masm.Push(argArgs);

     // And store our JSJitMethodCallArgs* in argArgs.

     masm.movePtr(StackPointer, argArgs);

     // Push |this| object for passing HandleObject. We push after argc to

     // maintain the same sp-relative location of the object pointer with other

     // DOMExitFrames.

     masm.Push(argObj);

     masm.movePtr(StackPointer, argObj);

     // Construct native exit frame.

     uint32_t safepointOffset;

     if (!masm.buildFakeExitFrame(argJSContext, &safepointOffset))

         return false;

     masm.enterFakeExitFrame(ION_FRAME_DOMMETHOD);

     if (!markSafepointAt(safepointOffset, call))

         return false;

     // Construct and execute call.

     masm.setupUnalignedABICall(4, argJSContext);

     masm.loadJSContext(argJSContext);

     masm.passABIArg(argJSContext);

     masm.passABIArg(argObj);

     masm.passABIArg(argPrivate);

     masm.passABIArg(argArgs);

     masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, target->jitInfo()->method));

     if (target->jitInfo()->isInfallible) {

         masm.loadValue(Address(StackPointer, IonDOMMethodExitFrameLayout::offsetOfResult()),

                        JSReturnOperand);

     } else {

         // Test for failure.

         masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());

         // Load the outparam vp[0] into output register(s).

         masm.loadValue(Address(StackPointer, IonDOMMethodExitFrameLayout::offsetOfResult()),

                        JSReturnOperand);

     }

     // The next instruction is removing the footer of the exit frame, so there

     // is no need for leaveFakeExitFrame.

     // Move the StackPointer back to its original location, unwinding the native exit frame.

     masm.adjustStack(IonDOMMethodExitFrameLayout::Size() - unusedStack);

     JS_ASSERT(masm.framePushed() == initialStack);

     dropArguments(call->numStackArgs() + 1);

     return true;

 }

 typedef bool (*GetIntrinsicValueFn)(JSContext *cx, HandlePropertyName, MutableHandleValue);

 static const VMFunction GetIntrinsicValueInfo =

     FunctionInfo<GetIntrinsicValueFn>(GetIntrinsicValue);

 bool

 CodeGenerator::visitCallGetIntrinsicValue(LCallGetIntrinsicValue *lir)

 {

     pushArg(ImmGCPtr(lir->mir()->name()));

     return callVM(GetIntrinsicValueInfo, lir);

 }

 typedef bool (*InvokeFunctionFn)(JSContext *, HandleObject, uint32_t, Value *, Value *);

 static const VMFunction InvokeFunctionInfo = FunctionInfo<InvokeFunctionFn>(InvokeFunction);

 bool

 CodeGenerator::emitCallInvokeFunction(LInstruction *call, Register calleereg,

                                       uint32_t argc, uint32_t unusedStack)

 {

     // Nestle %esp up to the argument vector.

     // Each path must account for framePushed_ separately, for callVM to be valid.

     masm.freeStack(unusedStack);

     pushArg(StackPointer); // argv.

     pushArg(Imm32(argc));  // argc.

     pushArg(calleereg);    // JSFunction *.

     if (!callVM(InvokeFunctionInfo, call))

         return false;

     // Un-nestle %esp from the argument vector. No prefix was pushed.

     masm.reserveStack(unusedStack);

     return true;

 }

 bool

 CodeGenerator::visitCallGeneric(LCallGeneric *call)

 {

     Register calleereg = ToRegister(call->getFunction());

     Register objreg    = ToRegister(call->getTempObject());

     Register nargsreg  = ToRegister(call->getNargsReg());

     uint32_t unusedStack = StackOffsetOfPassedArg(call->argslot());

     ExecutionMode executionMode = gen->info().executionMode();

     Label invoke, thunk, makeCall, end;

     // Known-target case is handled by LCallKnown.

     JS_ASSERT(!call->hasSingleTarget());

     // Generate an ArgumentsRectifier.

     JitCode *argumentsRectifier = gen->jitRuntime()->getArgumentsRectifier(executionMode);

     masm.checkStackAlignment();

     // Guard that calleereg is actually a function object.

     masm.loadObjClass(calleereg, nargsreg);

     masm.branchPtr(Assembler::NotEqual, nargsreg, ImmPtr(&JSFunction::class_), &invoke);

     // Guard that calleereg is an interpreted function with a JSScript.

     // If we are constructing, also ensure the callee is a constructor.

     if (call->mir()->isConstructing())

         masm.branchIfNotInterpretedConstructor(calleereg, nargsreg, &invoke);

     else

         masm.branchIfFunctionHasNoScript(calleereg, &invoke);

     // Knowing that calleereg is a non-native function, load the JSScript.

     masm.loadPtr(Address(calleereg, JSFunction::offsetOfNativeOrScript()), objreg);

     // Load script jitcode.

     masm.loadBaselineOrIonRaw(objreg, objreg, executionMode, &invoke);

     // Nestle the StackPointer up to the argument vector.

     masm.freeStack(unusedStack);

     // Construct the IonFramePrefix.

     uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS);

     masm.Push(Imm32(call->numActualArgs()));

     masm.Push(calleereg);

     masm.Push(Imm32(descriptor));

     // Check whether the provided arguments satisfy target argc.

     masm.load16ZeroExtend(Address(calleereg, JSFunction::offsetOfNargs()), nargsreg);

     masm.branch32(Assembler::Above, nargsreg, Imm32(call->numStackArgs()), &thunk);

     masm.jump(&makeCall);

     // Argument fixed needed. Load the ArgumentsRectifier.

     masm.bind(&thunk);

     {

         JS_ASSERT(ArgumentsRectifierReg != objreg);

         masm.movePtr(ImmGCPtr(argumentsRectifier), objreg); // Necessary for GC marking.

         masm.loadPtr(Address(objreg, JitCode::offsetOfCode()), objreg);

         masm.move32(Imm32(call->numStackArgs()), ArgumentsRectifierReg);

     }

     // Finally call the function in objreg.

     masm.bind(&makeCall);

     uint32_t callOffset = masm.callIon(objreg);

     if (!markSafepointAt(callOffset, call))

         return false;

     // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass.

     // The return address has already been removed from the Ion frame.

     int prefixGarbage = sizeof(IonJSFrameLayout) - sizeof(void *);

     masm.adjustStack(prefixGarbage - unusedStack);

     masm.jump(&end);

     // Handle uncompiled or native functions.

     masm.bind(&invoke);

     switch (executionMode) {

       case SequentialExecution:

         if (!emitCallInvokeFunction(call, calleereg, call->numActualArgs(), unusedStack))

             return false;

         break;

       case ParallelExecution:

         if (!emitCallToUncompiledScriptPar(call, calleereg))

             return false;

         break;

       default:

         MOZ_ASSUME_UNREACHABLE("No such execution mode");

     }

     masm.bind(&end);

     // If the return value of the constructing function is Primitive,

     // replace the return value with the Object from CreateThis.

     if (call->mir()->isConstructing()) {

         Label notPrimitive;

         masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, &notPrimitive);

         masm.loadValue(Address(StackPointer, unusedStack), JSReturnOperand);

         masm.bind(&notPrimitive);

     }

     if (!checkForAbortPar(call))

         return false;

     dropArguments(call->numStackArgs() + 1);

     return true;

 }

 // Generates a call to CallToUncompiledScriptPar() and then bails out.

 // |calleeReg| should contain the JSFunction*.

 bool

 CodeGenerator::emitCallToUncompiledScriptPar(LInstruction *lir, Register calleeReg)

 {

     OutOfLineCode *bail = oolAbortPar(ParallelBailoutCalledToUncompiledScript, lir);

     if (!bail)

         return false;

     masm.movePtr(calleeReg, CallTempReg0);

     masm.setupUnalignedABICall(1, CallTempReg1);

     masm.passABIArg(CallTempReg0);

     masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, CallToUncompiledScriptPar));

     masm.jump(bail->entry());

     return true;

 }

 bool

 CodeGenerator::visitCallKnown(LCallKnown *call)

 {

     Register calleereg = ToRegister(call->getFunction());

     Register objreg    = ToRegister(call->getTempObject());

     uint32_t unusedStack = StackOffsetOfPassedArg(call->argslot());

     DebugOnly<JSFunction *> target = call->getSingleTarget();

     ExecutionMode executionMode = gen->info().executionMode();

     Label end, uncompiled;

     // Native single targets are handled by LCallNative.

     JS_ASSERT(!target->isNative());

     // Missing arguments must have been explicitly appended by the IonBuilder.

     JS_ASSERT(target->nargs() <= call->numStackArgs());

     JS_ASSERT_IF(call->mir()->isConstructing(), target->isInterpretedConstructor());

     masm.checkStackAlignment();

     // The calleereg is known to be a non-native function, but might point to

     // a LazyScript instead of a JSScript.

     masm.branchIfFunctionHasNoScript(calleereg, &uncompiled);

     // Knowing that calleereg is a non-native function, load the JSScript.

     masm.loadPtr(Address(calleereg, JSFunction::offsetOfNativeOrScript()), objreg);

     // Load script jitcode.

     if (call->mir()->needsArgCheck())

         masm.loadBaselineOrIonRaw(objreg, objreg, executionMode, &uncompiled);

     else

         masm.loadBaselineOrIonNoArgCheck(objreg, objreg, executionMode, &uncompiled);

     // Nestle the StackPointer up to the argument vector.

     masm.freeStack(unusedStack);

     // Construct the IonFramePrefix.

     uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS);

     masm.Push(Imm32(call->numActualArgs()));

     masm.Push(calleereg);

     masm.Push(Imm32(descriptor));

     // Finally call the function in objreg.

     uint32_t callOffset = masm.callIon(objreg);

     if (!markSafepointAt(callOffset, call))

         return false;

     // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass.

     // The return address has already been removed from the Ion frame.

     int prefixGarbage = sizeof(IonJSFrameLayout) - sizeof(void *);

     masm.adjustStack(prefixGarbage - unusedStack);

     masm.jump(&end);

     // Handle uncompiled functions.

     masm.bind(&uncompiled);

     switch (executionMode) {

       case SequentialExecution:

         if (!emitCallInvokeFunction(call, calleereg, call->numActualArgs(), unusedStack))

             return false;

         break;

       case ParallelExecution:

         if (!emitCallToUncompiledScriptPar(call, calleereg))

             return false;

         break;

       default:

         MOZ_ASSUME_UNREACHABLE("No such execution mode");

     }

     masm.bind(&end);

     if (!checkForAbortPar(call))

         return false;

     // If the return value of the constructing function is Primitive,

     // replace the return value with the Object from CreateThis.

     if (call->mir()->isConstructing()) {

         Label notPrimitive;

         masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, &notPrimitive);

         masm.loadValue(Address(StackPointer, unusedStack), JSReturnOperand);

         masm.bind(&notPrimitive);

     }

     dropArguments(call->numStackArgs() + 1);

     return true;

 }

 bool

 CodeGenerator::checkForAbortPar(LInstruction *lir)

 {

     // In parallel mode, if we call another ion-compiled function and

     // it returns JS_ION_ERROR, that indicates a bailout that we have

     // to propagate up the stack.

     ExecutionMode executionMode = gen->info().executionMode();

     if (executionMode == ParallelExecution) {

         OutOfLinePropagateAbortPar *bail = oolPropagateAbortPar(lir);

         if (!bail)

             return false;

         masm.branchTestMagic(Assembler::Equal, JSReturnOperand, bail->entry());

     }

     return true;

 }

 bool

 CodeGenerator::emitCallInvokeFunction(LApplyArgsGeneric *apply, Register extraStackSize)

 {

     Register objreg = ToRegister(apply->getTempObject());

     JS_ASSERT(objreg != extraStackSize);

     // Push the space used by the arguments.

     masm.movePtr(StackPointer, objreg);

     masm.Push(extraStackSize);

     pushArg(objreg);                           // argv.

     pushArg(ToRegister(apply->getArgc()));     // argc.

     pushArg(ToRegister(apply->getFunction())); // JSFunction *.

     // This specialization og callVM restore the extraStackSize after the call.

     if (!callVM(InvokeFunctionInfo, apply, &extraStackSize))

         return false;

     masm.Pop(extraStackSize);

     return true;

 }

 // Do not bailout after the execution of this function since the stack no longer

 // correspond to what is expected by the snapshots.

 void

 CodeGenerator::emitPushArguments(LApplyArgsGeneric *apply, Register extraStackSpace)

 {

     // Holds the function nargs. Initially undefined.

     Register argcreg = ToRegister(apply->getArgc());

     Register copyreg = ToRegister(apply->getTempObject());

     size_t argvOffset = frameSize() + IonJSFrameLayout::offsetOfActualArgs();

     Label end;

     // Initialize the loop counter AND Compute the stack usage (if == 0)

     masm.movePtr(argcreg, extraStackSpace);

     masm.branchTestPtr(Assembler::Zero, argcreg, argcreg, &end);

     // Copy arguments.

     {

         Register count = extraStackSpace; // <- argcreg

         Label loop;

         masm.bind(&loop);

         // We remove sizeof(void*) from argvOffset because withtout it we target

         // the address after the memory area that we want to copy.

         BaseIndex disp(StackPointer, argcreg, ScaleFromElemWidth(sizeof(Value)), argvOffset - sizeof(void*));

         // Do not use Push here because other this account to 1 in the framePushed

         // instead of 0.  These push are only counted by argcreg.

         masm.loadPtr(disp, copyreg);

         masm.push(copyreg);

         // Handle 32 bits architectures.

         if (sizeof(Value) == 2 * sizeof(void*)) {

             masm.loadPtr(disp, copyreg);

             masm.push(copyreg);

         }

         masm.decBranchPtr(Assembler::NonZero, count, Imm32(1), &loop);

     }

     // Compute the stack usage.

     masm.movePtr(argcreg, extraStackSpace);

     masm.lshiftPtr(Imm32::ShiftOf(ScaleFromElemWidth(sizeof(Value))), extraStackSpace);

     // Join with all arguments copied and the extra stack usage computed.

     masm.bind(&end);

     // Push |this|.

     masm.addPtr(Imm32(sizeof(Value)), extraStackSpace);

     masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex));

 }

 void

 CodeGenerator::emitPopArguments(LApplyArgsGeneric *apply, Register extraStackSpace)

 {

     // Pop |this| and Arguments.

     masm.freeStack(extraStackSpace);

 }

 bool

 CodeGenerator::visitApplyArgsGeneric(LApplyArgsGeneric *apply)

 {

     // Holds the function object.

     Register calleereg = ToRegister(apply->getFunction());

     // Temporary register for modifying the function object.

     Register objreg = ToRegister(apply->getTempObject());

     Register copyreg = ToRegister(apply->getTempCopy());

     // Holds the function nargs. Initially undefined.

     Register argcreg = ToRegister(apply->getArgc());

     // Unless already known, guard that calleereg is actually a function object.

     if (!apply->hasSingleTarget()) {

         masm.loadObjClass(calleereg, objreg);

         ImmPtr ptr = ImmPtr(&JSFunction::class_);

         if (!bailoutCmpPtr(Assembler::NotEqual, objreg, ptr, apply->snapshot()))

             return false;

     }

     // Copy the arguments of the current function.

     emitPushArguments(apply, copyreg);

     masm.checkStackAlignment();

     // If the function is known to be uncompilable, only emit the call to InvokeFunction.

     ExecutionMode executionMode = gen->info().executionMode();

     if (apply->hasSingleTarget()) {

         JSFunction *target = apply->getSingleTarget();

         if (target->isNative()) {

             if (!emitCallInvokeFunction(apply, copyreg))

                 return false;

             emitPopArguments(apply, copyreg);

             return true;

         }

     }

     Label end, invoke;

     // Guard that calleereg is an interpreted function with a JSScript:

     if (!apply->hasSingleTarget()) {

         masm.branchIfFunctionHasNoScript(calleereg, &invoke);

     } else {

         // Native single targets are handled by LCallNative.

         JS_ASSERT(!apply->getSingleTarget()->isNative());

     }

     // Knowing that calleereg is a non-native function, load the JSScript.

     masm.loadPtr(Address(calleereg, JSFunction::offsetOfNativeOrScript()), objreg);

     // Load script jitcode.

     masm.loadBaselineOrIonRaw(objreg, objreg, executionMode, &invoke);

     // Call with an Ion frame or a rectifier frame.

     {

         // Create the frame descriptor.

         unsigned pushed = masm.framePushed();

         masm.addPtr(Imm32(pushed), copyreg);

         masm.makeFrameDescriptor(copyreg, JitFrame_IonJS);

         masm.Push(argcreg);

         masm.Push(calleereg);

         masm.Push(copyreg); // descriptor

         Label underflow, rejoin;

         // Check whether the provided arguments satisfy target argc.

         if (!apply->hasSingleTarget()) {

             masm.load16ZeroExtend(Address(calleereg, JSFunction::offsetOfNargs()), copyreg);

             masm.branch32(Assembler::Below, argcreg, copyreg, &underflow);

         } else {

             masm.branch32(Assembler::Below, argcreg, Imm32(apply->getSingleTarget()->nargs()),

                           &underflow);

         }

         // Skip the construction of the rectifier frame because we have no

         // underflow.

         masm.jump(&rejoin);

         // Argument fixup needed. Get ready to call the argumentsRectifier.

         {

             masm.bind(&underflow);

             // Hardcode the address of the argumentsRectifier code.

             JitCode *argumentsRectifier = gen->jitRuntime()->getArgumentsRectifier(executionMode);

             JS_ASSERT(ArgumentsRectifierReg != objreg);

             masm.movePtr(ImmGCPtr(argumentsRectifier), objreg); // Necessary for GC marking.

             masm.loadPtr(Address(objreg, JitCode::offsetOfCode()), objreg);

             masm.movePtr(argcreg, ArgumentsRectifierReg);

         }

         masm.bind(&rejoin);

         // Finally call the function in objreg, as assigned by one of the paths above.

         uint32_t callOffset = masm.callIon(objreg);

         if (!markSafepointAt(callOffset, apply))

             return false;

         // Recover the number of arguments from the frame descriptor.

         masm.loadPtr(Address(StackPointer, 0), copyreg);

         masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), copyreg);

         masm.subPtr(Imm32(pushed), copyreg);

         // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass.

         // The return address has already been removed from the Ion frame.

         int prefixGarbage = sizeof(IonJSFrameLayout) - sizeof(void *);

         masm.adjustStack(prefixGarbage);

         masm.jump(&end);

     }

     // Handle uncompiled or native functions.

     {

         masm.bind(&invoke);

         if (!emitCallInvokeFunction(apply, copyreg))

             return false;

     }

     // Pop arguments and continue.

     masm.bind(&end);

     emitPopArguments(apply, copyreg);

     return true;

 }

 typedef bool (*ArraySpliceDenseFn)(JSContext *, HandleObject, uint32_t, uint32_t);

 static const VMFunction ArraySpliceDenseInfo = FunctionInfo<ArraySpliceDenseFn>(ArraySpliceDense);

 bool

 CodeGenerator::visitArraySplice(LArraySplice *lir)

 {

     pushArg(ToRegister(lir->getDeleteCount()));

     pushArg(ToRegister(lir->getStart()));

     pushArg(ToRegister(lir->getObject()));

     return callVM(ArraySpliceDenseInfo, lir);

 }

 bool

 CodeGenerator::visitBail(LBail *lir)

 {

     return bailout(lir->snapshot());

 }

 bool

 CodeGenerator::visitGetDynamicName(LGetDynamicName *lir)

 {

     Register scopeChain = ToRegister(lir->getScopeChain());

     Register name = ToRegister(lir->getName());

     Register temp1 = ToRegister(lir->temp1());

     Register temp2 = ToRegister(lir->temp2());

     Register temp3 = ToRegister(lir->temp3());

     masm.loadJSContext(temp3);

     /* Make space for the outparam. */

     masm.adjustStack(-int32_t(sizeof(Value)));

     masm.movePtr(StackPointer, temp2);

     masm.setupUnalignedABICall(4, temp1);

     masm.passABIArg(temp3);

     masm.passABIArg(scopeChain);

     masm.passABIArg(name);

     masm.passABIArg(temp2);

     masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, GetDynamicName));

     const ValueOperand out = ToOutValue(lir);

     masm.loadValue(Address(StackPointer, 0), out);

     masm.adjustStack(sizeof(Value));

     Label undefined;

     masm.branchTestUndefined(Assembler::Equal, out, &undefined);

     return bailoutFrom(&undefined, lir->snapshot());

 }

 bool

 CodeGenerator::emitFilterArgumentsOrEval(LInstruction *lir, Register string,

                                          Register temp1, Register temp2)

 {

     masm.loadJSContext(temp2);

     masm.setupUnalignedABICall(2, temp1);

     masm.passABIArg(temp2);

     masm.passABIArg(string);

     masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, FilterArgumentsOrEval));

     Label bail;

     masm.branchIfFalseBool(ReturnReg, &bail);

     return bailoutFrom(&bail, lir->snapshot());

 }

 bool

 CodeGenerator::visitFilterArgumentsOrEvalS(LFilterArgumentsOrEvalS *lir)

 {

     return emitFilterArgumentsOrEval(lir, ToRegister(lir->getString()),

                                      ToRegister(lir->temp1()),

                                      ToRegister(lir->temp2()));

 }

 bool

 CodeGenerator::visitFilterArgumentsOrEvalV(LFilterArgumentsOrEvalV *lir)

 {

     ValueOperand input = ToValue(lir, LFilterArgumentsOrEvalV::Input);

     // Act as nop on non-strings.

     Label done;

     masm.branchTestString(Assembler::NotEqual, input, &done);

     if (!emitFilterArgumentsOrEval(lir, masm.extractString(input, ToRegister(lir->temp3())),

                                    ToRegister(lir->temp1()), ToRegister(lir->temp2())))

     {

         return false;

     }

     masm.bind(&done);

     return true;

 }

 typedef bool (*DirectEvalSFn)(JSContext *, HandleObject, HandleScript, HandleValue, HandleString,

                               jsbytecode *, MutableHandleValue);

 static const VMFunction DirectEvalStringInfo = FunctionInfo<DirectEvalSFn>(DirectEvalStringFromIon);

 bool

 CodeGenerator::visitCallDirectEvalS(LCallDirectEvalS *lir)

 {

     Register scopeChain = ToRegister(lir->getScopeChain());