diff -r 000000000000 -r 6474c204b198 js/src/jit/Ion.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/js/src/jit/Ion.cpp Wed Dec 31 06:09:35 2014 +0100 @@ -0,0 +1,3157 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/Ion.h" + +#include "mozilla/MemoryReporting.h" +#include "mozilla/ThreadLocal.h" + +#include "jscompartment.h" +#include "jsprf.h" +#include "jsworkers.h" + +#include "gc/Marking.h" +#include "jit/AliasAnalysis.h" +#include "jit/AsmJSModule.h" +#include "jit/BacktrackingAllocator.h" +#include "jit/BaselineDebugModeOSR.h" +#include "jit/BaselineFrame.h" +#include "jit/BaselineInspector.h" +#include "jit/BaselineJIT.h" +#include "jit/CodeGenerator.h" +#include "jit/EdgeCaseAnalysis.h" +#include "jit/EffectiveAddressAnalysis.h" +#include "jit/IonAnalysis.h" +#include "jit/IonBuilder.h" +#include "jit/IonOptimizationLevels.h" +#include "jit/IonSpewer.h" +#include "jit/JitCommon.h" +#include "jit/JitCompartment.h" +#include "jit/LICM.h" +#include "jit/LinearScan.h" +#include "jit/LIR.h" +#include "jit/Lowering.h" +#include "jit/ParallelSafetyAnalysis.h" +#include "jit/PerfSpewer.h" +#include "jit/RangeAnalysis.h" +#include "jit/StupidAllocator.h" +#include "jit/UnreachableCodeElimination.h" +#include "jit/ValueNumbering.h" +#include "vm/ForkJoin.h" +#include "vm/TraceLogging.h" + +#include "jscompartmentinlines.h" +#include "jsgcinlines.h" +#include "jsinferinlines.h" +#include "jsobjinlines.h" + +#include "jit/ExecutionMode-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::ThreadLocal; + +// Assert that JitCode is gc::Cell aligned. +JS_STATIC_ASSERT(sizeof(JitCode) % gc::CellSize == 0); + +static ThreadLocal TlsIonContext; + +static IonContext * +CurrentIonContext() +{ + if (!TlsIonContext.initialized()) + return nullptr; + return TlsIonContext.get(); +} + +void +jit::SetIonContext(IonContext *ctx) +{ + TlsIonContext.set(ctx); +} + +IonContext * +jit::GetIonContext() +{ + MOZ_ASSERT(CurrentIonContext()); + return CurrentIonContext(); +} + +IonContext * +jit::MaybeGetIonContext() +{ + return CurrentIonContext(); +} + +IonContext::IonContext(JSContext *cx, TempAllocator *temp) + : cx(cx), + temp(temp), + runtime(CompileRuntime::get(cx->runtime())), + compartment(CompileCompartment::get(cx->compartment())), + prev_(CurrentIonContext()), + assemblerCount_(0) +{ + SetIonContext(this); +} + +IonContext::IonContext(ExclusiveContext *cx, TempAllocator *temp) + : cx(nullptr), + temp(temp), + runtime(CompileRuntime::get(cx->runtime_)), + compartment(nullptr), + prev_(CurrentIonContext()), + assemblerCount_(0) +{ + SetIonContext(this); +} + +IonContext::IonContext(CompileRuntime *rt, CompileCompartment *comp, TempAllocator *temp) + : cx(nullptr), + temp(temp), + runtime(rt), + compartment(comp), + prev_(CurrentIonContext()), + assemblerCount_(0) +{ + SetIonContext(this); +} + +IonContext::IonContext(CompileRuntime *rt) + : cx(nullptr), + temp(nullptr), + runtime(rt), + compartment(nullptr), + prev_(CurrentIonContext()), + assemblerCount_(0) +{ + SetIonContext(this); +} + +IonContext::~IonContext() +{ + SetIonContext(prev_); +} + +bool +jit::InitializeIon() +{ + if (!TlsIonContext.initialized() && !TlsIonContext.init()) + return false; + CheckLogging(); + CheckPerf(); + return true; +} + +JitRuntime::JitRuntime() + : execAlloc_(nullptr), + ionAlloc_(nullptr), + exceptionTail_(nullptr), + bailoutTail_(nullptr), + enterJIT_(nullptr), + bailoutHandler_(nullptr), + argumentsRectifier_(nullptr), + argumentsRectifierReturnAddr_(nullptr), + parallelArgumentsRectifier_(nullptr), + invalidator_(nullptr), + debugTrapHandler_(nullptr), + forkJoinGetSliceStub_(nullptr), + baselineDebugModeOSRHandler_(nullptr), + functionWrappers_(nullptr), + osrTempData_(nullptr), + ionCodeProtected_(false) +{ +} + +JitRuntime::~JitRuntime() +{ + js_delete(functionWrappers_); + freeOsrTempData(); + + // Note: The interrupt lock is not taken here, as JitRuntime is only + // destroyed along with its containing JSRuntime. + js_delete(ionAlloc_); +} + +bool +JitRuntime::initialize(JSContext *cx) +{ + JS_ASSERT(cx->runtime()->currentThreadHasExclusiveAccess()); + JS_ASSERT(cx->runtime()->currentThreadOwnsInterruptLock()); + + AutoCompartment ac(cx, cx->atomsCompartment()); + + IonContext ictx(cx, nullptr); + + execAlloc_ = cx->runtime()->getExecAlloc(cx); + if (!execAlloc_) + return false; + + if (!cx->compartment()->ensureJitCompartmentExists(cx)) + return false; + + functionWrappers_ = cx->new_(cx); + if (!functionWrappers_ || !functionWrappers_->init()) + return false; + + IonSpew(IonSpew_Codegen, "# Emitting exception tail stub"); + exceptionTail_ = generateExceptionTailStub(cx); + if (!exceptionTail_) + return false; + + IonSpew(IonSpew_Codegen, "# Emitting bailout tail stub"); + bailoutTail_ = generateBailoutTailStub(cx); + if (!bailoutTail_) + return false; + + if (cx->runtime()->jitSupportsFloatingPoint) { + IonSpew(IonSpew_Codegen, "# Emitting bailout tables"); + + // Initialize some Ion-only stubs that require floating-point support. + if (!bailoutTables_.reserve(FrameSizeClass::ClassLimit().classId())) + return false; + + for (uint32_t id = 0;; id++) { + FrameSizeClass class_ = FrameSizeClass::FromClass(id); + if (class_ == FrameSizeClass::ClassLimit()) + break; + bailoutTables_.infallibleAppend((JitCode *)nullptr); + bailoutTables_[id] = generateBailoutTable(cx, id); + if (!bailoutTables_[id]) + return false; + } + + IonSpew(IonSpew_Codegen, "# Emitting bailout handler"); + bailoutHandler_ = generateBailoutHandler(cx); + if (!bailoutHandler_) + return false; + + IonSpew(IonSpew_Codegen, "# Emitting invalidator"); + invalidator_ = generateInvalidator(cx); + if (!invalidator_) + return false; + } + + IonSpew(IonSpew_Codegen, "# Emitting sequential arguments rectifier"); + argumentsRectifier_ = generateArgumentsRectifier(cx, SequentialExecution, &argumentsRectifierReturnAddr_); + if (!argumentsRectifier_) + return false; + +#ifdef JS_THREADSAFE + IonSpew(IonSpew_Codegen, "# Emitting parallel arguments rectifier"); + parallelArgumentsRectifier_ = generateArgumentsRectifier(cx, ParallelExecution, nullptr); + if (!parallelArgumentsRectifier_) + return false; +#endif + + IonSpew(IonSpew_Codegen, "# Emitting EnterJIT sequence"); + enterJIT_ = generateEnterJIT(cx, EnterJitOptimized); + if (!enterJIT_) + return false; + + IonSpew(IonSpew_Codegen, "# Emitting EnterBaselineJIT sequence"); + enterBaselineJIT_ = generateEnterJIT(cx, EnterJitBaseline); + if (!enterBaselineJIT_) + return false; + + IonSpew(IonSpew_Codegen, "# Emitting Pre Barrier for Value"); + valuePreBarrier_ = generatePreBarrier(cx, MIRType_Value); + if (!valuePreBarrier_) + return false; + + IonSpew(IonSpew_Codegen, "# Emitting Pre Barrier for Shape"); + shapePreBarrier_ = generatePreBarrier(cx, MIRType_Shape); + if (!shapePreBarrier_) + return false; + + IonSpew(IonSpew_Codegen, "# Emitting VM function wrappers"); + for (VMFunction *fun = VMFunction::functions; fun; fun = fun->next) { + if (!generateVMWrapper(cx, *fun)) + return false; + } + + return true; +} + +JitCode * +JitRuntime::debugTrapHandler(JSContext *cx) +{ + if (!debugTrapHandler_) { + // JitRuntime code stubs are shared across compartments and have to + // be allocated in the atoms compartment. + AutoLockForExclusiveAccess lock(cx); + AutoCompartment ac(cx, cx->runtime()->atomsCompartment()); + debugTrapHandler_ = generateDebugTrapHandler(cx); + } + return debugTrapHandler_; +} + +bool +JitRuntime::ensureForkJoinGetSliceStubExists(JSContext *cx) +{ + if (!forkJoinGetSliceStub_) { + IonSpew(IonSpew_Codegen, "# Emitting ForkJoinGetSlice stub"); + AutoLockForExclusiveAccess lock(cx); + AutoCompartment ac(cx, cx->runtime()->atomsCompartment()); + forkJoinGetSliceStub_ = generateForkJoinGetSliceStub(cx); + } + return !!forkJoinGetSliceStub_; +} + +uint8_t * +JitRuntime::allocateOsrTempData(size_t size) +{ + osrTempData_ = (uint8_t *)js_realloc(osrTempData_, size); + return osrTempData_; +} + +void +JitRuntime::freeOsrTempData() +{ + js_free(osrTempData_); + osrTempData_ = nullptr; +} + +JSC::ExecutableAllocator * +JitRuntime::createIonAlloc(JSContext *cx) +{ + JS_ASSERT(cx->runtime()->currentThreadOwnsInterruptLock()); + + ionAlloc_ = js_new(); + if (!ionAlloc_) + js_ReportOutOfMemory(cx); + return ionAlloc_; +} + +void +JitRuntime::ensureIonCodeProtected(JSRuntime *rt) +{ + JS_ASSERT(rt->currentThreadOwnsInterruptLock()); + + if (!rt->signalHandlersInstalled() || ionCodeProtected_ || !ionAlloc_) + return; + + // Protect all Ion code in the runtime to trigger an access violation the + // next time any of it runs on the main thread. + ionAlloc_->toggleAllCodeAsAccessible(false); + ionCodeProtected_ = true; +} + +bool +JitRuntime::handleAccessViolation(JSRuntime *rt, void *faultingAddress) +{ + if (!rt->signalHandlersInstalled() || !ionAlloc_ || !ionAlloc_->codeContains((char *) faultingAddress)) + return false; + +#ifdef JS_THREADSAFE + // All places where the interrupt lock is taken must either ensure that Ion + // code memory won't be accessed within, or call ensureIonCodeAccessible to + // render the memory safe for accessing. Otherwise taking the lock below + // will deadlock the process. + JS_ASSERT(!rt->currentThreadOwnsInterruptLock()); +#endif + + // Taking this lock is necessary to prevent the interrupting thread from marking + // the memory as inaccessible while we are patching backedges. This will cause us + // to SEGV while still inside the signal handler, and the process will terminate. + JSRuntime::AutoLockForInterrupt lock(rt); + + // Ion code in the runtime faulted after it was made inaccessible. Reset + // the code privileges and patch all loop backedges to perform an interrupt + // check instead. + ensureIonCodeAccessible(rt); + return true; +} + +void +JitRuntime::ensureIonCodeAccessible(JSRuntime *rt) +{ + JS_ASSERT(rt->currentThreadOwnsInterruptLock()); + + // This can only be called on the main thread and while handling signals, + // which happens on a separate thread in OS X. +#ifndef XP_MACOSX + JS_ASSERT(CurrentThreadCanAccessRuntime(rt)); +#endif + + if (ionCodeProtected_) { + ionAlloc_->toggleAllCodeAsAccessible(true); + ionCodeProtected_ = false; + } + + if (rt->interrupt) { + // The interrupt handler needs to be invoked by this thread, but we may + // be inside a signal handler and have no idea what is above us on the + // stack (probably we are executing Ion code at an arbitrary point, but + // we could be elsewhere, say repatching a jump for an IonCache). + // Patch all backedges in the runtime so they will invoke the interrupt + // handler the next time they execute. + patchIonBackedges(rt, BackedgeInterruptCheck); + } +} + +void +JitRuntime::patchIonBackedges(JSRuntime *rt, BackedgeTarget target) +{ +#ifndef XP_MACOSX + JS_ASSERT(CurrentThreadCanAccessRuntime(rt)); +#endif + + // Patch all loop backedges in Ion code so that they either jump to the + // normal loop header or to an interrupt handler each time they run. + for (InlineListIterator iter(backedgeList_.begin()); + iter != backedgeList_.end(); + iter++) + { + PatchableBackedge *patchableBackedge = *iter; + PatchJump(patchableBackedge->backedge, target == BackedgeLoopHeader + ? patchableBackedge->loopHeader + : patchableBackedge->interruptCheck); + } +} + +void +jit::RequestInterruptForIonCode(JSRuntime *rt, JSRuntime::InterruptMode mode) +{ + JitRuntime *jitRuntime = rt->jitRuntime(); + if (!jitRuntime) + return; + + JS_ASSERT(rt->currentThreadOwnsInterruptLock()); + + // The mechanism for interrupting normal ion code varies depending on how + // the interrupt is being requested. + switch (mode) { + case JSRuntime::RequestInterruptMainThread: + // When requesting an interrupt from the main thread, Ion loop + // backedges can be patched directly. Make sure we don't segv while + // patching the backedges, to avoid deadlocking inside the signal + // handler. + JS_ASSERT(CurrentThreadCanAccessRuntime(rt)); + jitRuntime->ensureIonCodeAccessible(rt); + break; + + case JSRuntime::RequestInterruptAnyThread: + // When requesting an interrupt from off the main thread, protect + // Ion code memory so that the main thread will fault and enter a + // signal handler when trying to execute the code. The signal + // handler will unprotect the code and patch loop backedges so + // that the interrupt handler is invoked afterwards. + jitRuntime->ensureIonCodeProtected(rt); + break; + + case JSRuntime::RequestInterruptAnyThreadDontStopIon: + case JSRuntime::RequestInterruptAnyThreadForkJoin: + // The caller does not require Ion code to be interrupted. + // Nothing more needs to be done. + break; + + default: + MOZ_ASSUME_UNREACHABLE("Bad interrupt mode"); + } +} + +JitCompartment::JitCompartment() + : stubCodes_(nullptr), + baselineCallReturnFromIonAddr_(nullptr), + baselineGetPropReturnFromIonAddr_(nullptr), + baselineSetPropReturnFromIonAddr_(nullptr), + baselineCallReturnFromStubAddr_(nullptr), + baselineGetPropReturnFromStubAddr_(nullptr), + baselineSetPropReturnFromStubAddr_(nullptr), + stringConcatStub_(nullptr), + parallelStringConcatStub_(nullptr), + activeParallelEntryScripts_(nullptr) +{ +} + +JitCompartment::~JitCompartment() +{ + js_delete(stubCodes_); + js_delete(activeParallelEntryScripts_); +} + +bool +JitCompartment::initialize(JSContext *cx) +{ + stubCodes_ = cx->new_(cx); + if (!stubCodes_ || !stubCodes_->init()) + return false; + + return true; +} + +bool +JitCompartment::ensureIonStubsExist(JSContext *cx) +{ + if (!stringConcatStub_) { + stringConcatStub_ = generateStringConcatStub(cx, SequentialExecution); + if (!stringConcatStub_) + return false; + } + +#ifdef JS_THREADSAFE + if (!parallelStringConcatStub_) { + parallelStringConcatStub_ = generateStringConcatStub(cx, ParallelExecution); + if (!parallelStringConcatStub_) + return false; + } +#endif + + return true; +} + +bool +JitCompartment::notifyOfActiveParallelEntryScript(JSContext *cx, HandleScript script) +{ + // Fast path. The isParallelEntryScript bit guarantees that the script is + // already in the set. + if (script->parallelIonScript()->isParallelEntryScript()) { + MOZ_ASSERT(activeParallelEntryScripts_ && activeParallelEntryScripts_->has(script)); + script->parallelIonScript()->resetParallelAge(); + return true; + } + + if (!activeParallelEntryScripts_) { + activeParallelEntryScripts_ = cx->new_(cx); + if (!activeParallelEntryScripts_ || !activeParallelEntryScripts_->init()) + return false; + } + + script->parallelIonScript()->setIsParallelEntryScript(); + ScriptSet::AddPtr p = activeParallelEntryScripts_->lookupForAdd(script); + return p || activeParallelEntryScripts_->add(p, script); +} + +void +jit::FinishOffThreadBuilder(IonBuilder *builder) +{ + ExecutionMode executionMode = builder->info().executionMode(); + + // Clear the recompiling flag of the old ionScript, since we continue to + // use the old ionScript if recompiling fails. + if (executionMode == SequentialExecution && builder->script()->hasIonScript()) + builder->script()->ionScript()->clearRecompiling(); + + // Clean up if compilation did not succeed. + if (CompilingOffThread(builder->script(), executionMode)) + SetIonScript(builder->script(), executionMode, nullptr); + + // The builder is allocated into its LifoAlloc, so destroying that will + // destroy the builder and all other data accumulated during compilation, + // except any final codegen (which includes an assembler and needs to be + // explicitly destroyed). + js_delete(builder->backgroundCodegen()); + js_delete(builder->alloc().lifoAlloc()); +} + +static inline void +FinishAllOffThreadCompilations(JSCompartment *comp) +{ +#ifdef JS_THREADSAFE + AutoLockWorkerThreadState lock; + GlobalWorkerThreadState::IonBuilderVector &finished = WorkerThreadState().ionFinishedList(); + + for (size_t i = 0; i < finished.length(); i++) { + IonBuilder *builder = finished[i]; + if (builder->compartment == CompileCompartment::get(comp)) { + FinishOffThreadBuilder(builder); + WorkerThreadState().remove(finished, &i); + } + } +#endif +} + +/* static */ void +JitRuntime::Mark(JSTracer *trc) +{ + JS_ASSERT(!trc->runtime()->isHeapMinorCollecting()); + Zone *zone = trc->runtime()->atomsCompartment()->zone(); + for (gc::CellIterUnderGC i(zone, gc::FINALIZE_JITCODE); !i.done(); i.next()) { + JitCode *code = i.get(); + MarkJitCodeRoot(trc, &code, "wrapper"); + } +} + +void +JitCompartment::mark(JSTracer *trc, JSCompartment *compartment) +{ + // Cancel any active or pending off thread compilations. Note that the + // MIR graph does not hold any nursery pointers, so there's no need to + // do this for minor GCs. + JS_ASSERT(!trc->runtime()->isHeapMinorCollecting()); + CancelOffThreadIonCompile(compartment, nullptr); + FinishAllOffThreadCompilations(compartment); + + // Free temporary OSR buffer. + trc->runtime()->jitRuntime()->freeOsrTempData(); + + // Mark scripts with parallel IonScripts if we should preserve them. + if (activeParallelEntryScripts_) { + for (ScriptSet::Enum e(*activeParallelEntryScripts_); !e.empty(); e.popFront()) { + JSScript *script = e.front(); + + // If the script has since been invalidated or was attached by an + // off-thread worker too late (i.e., the ForkJoin finished with + // warmup doing all the work), remove it. + if (!script->hasParallelIonScript() || + !script->parallelIonScript()->isParallelEntryScript()) + { + e.removeFront(); + continue; + } + + // Check and increment the age. If the script is below the max + // age, mark it. + // + // Subtlety: We depend on the tracing of the parallel IonScript's + // callTargetEntries to propagate the parallel age to the entire + // call graph. + if (ShouldPreserveParallelJITCode(trc->runtime(), script, /* increase = */ true)) { + MarkScript(trc, const_cast(&e.front()), "par-script"); + MOZ_ASSERT(script == e.front()); + } + } + } +} + +void +JitCompartment::sweep(FreeOp *fop) +{ + stubCodes_->sweep(fop); + + // If the sweep removed the ICCall_Fallback stub, nullptr the baselineCallReturnAddr_ field. + if (!stubCodes_->lookup(static_cast(ICStub::Call_Fallback))) { + baselineCallReturnFromIonAddr_ = nullptr; + baselineCallReturnFromStubAddr_ = nullptr; + } + // Similarly for the ICGetProp_Fallback stub. + if (!stubCodes_->lookup(static_cast(ICStub::GetProp_Fallback))) { + baselineGetPropReturnFromIonAddr_ = nullptr; + baselineGetPropReturnFromStubAddr_ = nullptr; + } + if (!stubCodes_->lookup(static_cast(ICStub::SetProp_Fallback))) { + baselineSetPropReturnFromIonAddr_ = nullptr; + baselineSetPropReturnFromStubAddr_ = nullptr; + } + + if (stringConcatStub_ && !IsJitCodeMarked(stringConcatStub_.unsafeGet())) + stringConcatStub_ = nullptr; + + if (parallelStringConcatStub_ && !IsJitCodeMarked(parallelStringConcatStub_.unsafeGet())) + parallelStringConcatStub_ = nullptr; + + if (activeParallelEntryScripts_) { + for (ScriptSet::Enum e(*activeParallelEntryScripts_); !e.empty(); e.popFront()) { + JSScript *script = e.front(); + if (!IsScriptMarked(&script)) + e.removeFront(); + else + MOZ_ASSERT(script == e.front()); + } + } +} + +JitCode * +JitRuntime::getBailoutTable(const FrameSizeClass &frameClass) const +{ + JS_ASSERT(frameClass != FrameSizeClass::None()); + return bailoutTables_[frameClass.classId()]; +} + +JitCode * +JitRuntime::getVMWrapper(const VMFunction &f) const +{ + JS_ASSERT(functionWrappers_); + JS_ASSERT(functionWrappers_->initialized()); + JitRuntime::VMWrapperMap::Ptr p = functionWrappers_->readonlyThreadsafeLookup(&f); + JS_ASSERT(p); + + return p->value(); +} + +template +JitCode * +JitCode::New(JSContext *cx, uint8_t *code, uint32_t bufferSize, uint32_t headerSize, + JSC::ExecutablePool *pool, JSC::CodeKind kind) +{ + JitCode *codeObj = js::NewJitCode(cx); + if (!codeObj) { + pool->release(headerSize + bufferSize, kind); + return nullptr; + } + + new (codeObj) JitCode(code, bufferSize, headerSize, pool, kind); + return codeObj; +} + +template +JitCode * +JitCode::New(JSContext *cx, uint8_t *code, uint32_t bufferSize, uint32_t headerSize, + JSC::ExecutablePool *pool, JSC::CodeKind kind); + +template +JitCode * +JitCode::New(JSContext *cx, uint8_t *code, uint32_t bufferSize, uint32_t headerSize, + JSC::ExecutablePool *pool, JSC::CodeKind kind); + +void +JitCode::copyFrom(MacroAssembler &masm) +{ + // Store the JitCode pointer right before the code buffer, so we can + // recover the gcthing from relocation tables. + *(JitCode **)(code_ - sizeof(JitCode *)) = this; + insnSize_ = masm.instructionsSize(); + masm.executableCopy(code_); + + jumpRelocTableBytes_ = masm.jumpRelocationTableBytes(); + masm.copyJumpRelocationTable(code_ + jumpRelocTableOffset()); + + dataRelocTableBytes_ = masm.dataRelocationTableBytes(); + masm.copyDataRelocationTable(code_ + dataRelocTableOffset()); + + preBarrierTableBytes_ = masm.preBarrierTableBytes(); + masm.copyPreBarrierTable(code_ + preBarrierTableOffset()); + + masm.processCodeLabels(code_); +} + +void +JitCode::trace(JSTracer *trc) +{ + // Note that we cannot mark invalidated scripts, since we've basically + // corrupted the code stream by injecting bailouts. + if (invalidated()) + return; + + if (jumpRelocTableBytes_) { + uint8_t *start = code_ + jumpRelocTableOffset(); + CompactBufferReader reader(start, start + jumpRelocTableBytes_); + MacroAssembler::TraceJumpRelocations(trc, this, reader); + } + if (dataRelocTableBytes_) { + uint8_t *start = code_ + dataRelocTableOffset(); + CompactBufferReader reader(start, start + dataRelocTableBytes_); + MacroAssembler::TraceDataRelocations(trc, this, reader); + } +} + +void +JitCode::finalize(FreeOp *fop) +{ + // Make sure this can't race with an interrupting thread, which may try + // to read the contents of the pool we are releasing references in. + JS_ASSERT(fop->runtime()->currentThreadOwnsInterruptLock()); + + // Buffer can be freed at any time hereafter. Catch use-after-free bugs. + // Don't do this if the Ion code is protected, as the signal handler will + // deadlock trying to reacquire the interrupt lock. + if (fop->runtime()->jitRuntime() && !fop->runtime()->jitRuntime()->ionCodeProtected()) + memset(code_, JS_SWEPT_CODE_PATTERN, bufferSize_); + code_ = nullptr; + + // Code buffers are stored inside JSC pools. + // Pools are refcounted. Releasing the pool may free it. + if (pool_) { + // Horrible hack: if we are using perf integration, we don't + // want to reuse code addresses, so we just leak the memory instead. + if (!PerfEnabled()) + pool_->release(headerSize_ + bufferSize_, JSC::CodeKind(kind_)); + pool_ = nullptr; + } +} + +void +JitCode::togglePreBarriers(bool enabled) +{ + uint8_t *start = code_ + preBarrierTableOffset(); + CompactBufferReader reader(start, start + preBarrierTableBytes_); + + while (reader.more()) { + size_t offset = reader.readUnsigned(); + CodeLocationLabel loc(this, offset); + if (enabled) + Assembler::ToggleToCmp(loc); + else + Assembler::ToggleToJmp(loc); + } +} + +IonScript::IonScript() + : method_(nullptr), + deoptTable_(nullptr), + osrPc_(nullptr), + osrEntryOffset_(0), + skipArgCheckEntryOffset_(0), + invalidateEpilogueOffset_(0), + invalidateEpilogueDataOffset_(0), + numBailouts_(0), + hasUncompiledCallTarget_(false), + isParallelEntryScript_(false), + hasSPSInstrumentation_(false), + recompiling_(false), + runtimeData_(0), + runtimeSize_(0), + cacheIndex_(0), + cacheEntries_(0), + safepointIndexOffset_(0), + safepointIndexEntries_(0), + safepointsStart_(0), + safepointsSize_(0), + frameSlots_(0), + frameSize_(0), + bailoutTable_(0), + bailoutEntries_(0), + osiIndexOffset_(0), + osiIndexEntries_(0), + snapshots_(0), + snapshotsListSize_(0), + snapshotsRVATableSize_(0), + constantTable_(0), + constantEntries_(0), + callTargetList_(0), + callTargetEntries_(0), + backedgeList_(0), + backedgeEntries_(0), + refcount_(0), + parallelAge_(0), + recompileInfo_(), + osrPcMismatchCounter_(0), + dependentAsmJSModules(nullptr) +{ +} + +IonScript * +IonScript::New(JSContext *cx, types::RecompileInfo recompileInfo, + uint32_t frameSlots, uint32_t frameSize, + size_t snapshotsListSize, size_t snapshotsRVATableSize, + size_t recoversSize, size_t bailoutEntries, + size_t constants, size_t safepointIndices, + size_t osiIndices, size_t cacheEntries, + size_t runtimeSize, size_t safepointsSize, + size_t callTargetEntries, size_t backedgeEntries, + OptimizationLevel optimizationLevel) +{ + static const int DataAlignment = sizeof(void *); + + if (snapshotsListSize >= MAX_BUFFER_SIZE || + (bailoutEntries >= MAX_BUFFER_SIZE / sizeof(uint32_t))) + { + js_ReportOutOfMemory(cx); + return nullptr; + } + + // This should not overflow on x86, because the memory is already allocated + // *somewhere* and if their total overflowed there would be no memory left + // at all. + size_t paddedSnapshotsSize = AlignBytes(snapshotsListSize + snapshotsRVATableSize, DataAlignment); + size_t paddedRecoversSize = AlignBytes(recoversSize, DataAlignment); + size_t paddedBailoutSize = AlignBytes(bailoutEntries * sizeof(uint32_t), DataAlignment); + size_t paddedConstantsSize = AlignBytes(constants * sizeof(Value), DataAlignment); + size_t paddedSafepointIndicesSize = AlignBytes(safepointIndices * sizeof(SafepointIndex), DataAlignment); + size_t paddedOsiIndicesSize = AlignBytes(osiIndices * sizeof(OsiIndex), DataAlignment); + size_t paddedCacheEntriesSize = AlignBytes(cacheEntries * sizeof(uint32_t), DataAlignment); + size_t paddedRuntimeSize = AlignBytes(runtimeSize, DataAlignment); + size_t paddedSafepointSize = AlignBytes(safepointsSize, DataAlignment); + size_t paddedCallTargetSize = AlignBytes(callTargetEntries * sizeof(JSScript *), DataAlignment); + size_t paddedBackedgeSize = AlignBytes(backedgeEntries * sizeof(PatchableBackedge), DataAlignment); + size_t bytes = paddedSnapshotsSize + + paddedRecoversSize + + paddedBailoutSize + + paddedConstantsSize + + paddedSafepointIndicesSize+ + paddedOsiIndicesSize + + paddedCacheEntriesSize + + paddedRuntimeSize + + paddedSafepointSize + + paddedCallTargetSize + + paddedBackedgeSize; + uint8_t *buffer = (uint8_t *)cx->malloc_(sizeof(IonScript) + bytes); + if (!buffer) + return nullptr; + + IonScript *script = reinterpret_cast(buffer); + new (script) IonScript(); + + uint32_t offsetCursor = sizeof(IonScript); + + script->runtimeData_ = offsetCursor; + script->runtimeSize_ = runtimeSize; + offsetCursor += paddedRuntimeSize; + + script->cacheIndex_ = offsetCursor; + script->cacheEntries_ = cacheEntries; + offsetCursor += paddedCacheEntriesSize; + + script->safepointIndexOffset_ = offsetCursor; + script->safepointIndexEntries_ = safepointIndices; + offsetCursor += paddedSafepointIndicesSize; + + script->safepointsStart_ = offsetCursor; + script->safepointsSize_ = safepointsSize; + offsetCursor += paddedSafepointSize; + + script->bailoutTable_ = offsetCursor; + script->bailoutEntries_ = bailoutEntries; + offsetCursor += paddedBailoutSize; + + script->osiIndexOffset_ = offsetCursor; + script->osiIndexEntries_ = osiIndices; + offsetCursor += paddedOsiIndicesSize; + + script->snapshots_ = offsetCursor; + script->snapshotsListSize_ = snapshotsListSize; + script->snapshotsRVATableSize_ = snapshotsRVATableSize; + offsetCursor += paddedSnapshotsSize; + + script->recovers_ = offsetCursor; + script->recoversSize_ = recoversSize; + offsetCursor += paddedRecoversSize; + + script->constantTable_ = offsetCursor; + script->constantEntries_ = constants; + offsetCursor += paddedConstantsSize; + + script->callTargetList_ = offsetCursor; + script->callTargetEntries_ = callTargetEntries; + offsetCursor += paddedCallTargetSize; + + script->backedgeList_ = offsetCursor; + script->backedgeEntries_ = backedgeEntries; + offsetCursor += paddedBackedgeSize; + + script->frameSlots_ = frameSlots; + script->frameSize_ = frameSize; + + script->recompileInfo_ = recompileInfo; + script->optimizationLevel_ = optimizationLevel; + + return script; +} + +void +IonScript::trace(JSTracer *trc) +{ + if (method_) + MarkJitCode(trc, &method_, "method"); + + if (deoptTable_) + MarkJitCode(trc, &deoptTable_, "deoptimizationTable"); + + for (size_t i = 0; i < numConstants(); i++) + gc::MarkValue(trc, &getConstant(i), "constant"); + + // No write barrier is needed for the call target list, as it's attached + // at compilation time and is read only. + for (size_t i = 0; i < callTargetEntries(); i++) { + // Propagate the parallelAge to the call targets. + if (callTargetList()[i]->hasParallelIonScript()) + callTargetList()[i]->parallelIonScript()->parallelAge_ = parallelAge_; + + gc::MarkScriptUnbarriered(trc, &callTargetList()[i], "callTarget"); + } +} + +/* static */ void +IonScript::writeBarrierPre(Zone *zone, IonScript *ionScript) +{ +#ifdef JSGC_INCREMENTAL + if (zone->needsBarrier()) + ionScript->trace(zone->barrierTracer()); +#endif +} + +void +IonScript::copySnapshots(const SnapshotWriter *writer) +{ + MOZ_ASSERT(writer->listSize() == snapshotsListSize_); + memcpy((uint8_t *)this + snapshots_, + writer->listBuffer(), snapshotsListSize_); + + MOZ_ASSERT(snapshotsRVATableSize_); + MOZ_ASSERT(writer->RVATableSize() == snapshotsRVATableSize_); + memcpy((uint8_t *)this + snapshots_ + snapshotsListSize_, + writer->RVATableBuffer(), snapshotsRVATableSize_); +} + +void +IonScript::copyRecovers(const RecoverWriter *writer) +{ + MOZ_ASSERT(writer->size() == recoversSize_); + memcpy((uint8_t *)this + recovers_, writer->buffer(), recoversSize_); +} + +void +IonScript::copySafepoints(const SafepointWriter *writer) +{ + JS_ASSERT(writer->size() == safepointsSize_); + memcpy((uint8_t *)this + safepointsStart_, writer->buffer(), safepointsSize_); +} + +void +IonScript::copyBailoutTable(const SnapshotOffset *table) +{ + memcpy(bailoutTable(), table, bailoutEntries_ * sizeof(uint32_t)); +} + +void +IonScript::copyConstants(const Value *vp) +{ + for (size_t i = 0; i < constantEntries_; i++) + constants()[i].init(vp[i]); +} + +void +IonScript::copyCallTargetEntries(JSScript **callTargets) +{ + for (size_t i = 0; i < callTargetEntries_; i++) + callTargetList()[i] = callTargets[i]; +} + +void +IonScript::copyPatchableBackedges(JSContext *cx, JitCode *code, + PatchableBackedgeInfo *backedges) +{ + for (size_t i = 0; i < backedgeEntries_; i++) { + const PatchableBackedgeInfo &info = backedges[i]; + PatchableBackedge *patchableBackedge = &backedgeList()[i]; + + CodeLocationJump backedge(code, info.backedge); + CodeLocationLabel loopHeader(code, CodeOffsetLabel(info.loopHeader->offset())); + CodeLocationLabel interruptCheck(code, CodeOffsetLabel(info.interruptCheck->offset())); + new(patchableBackedge) PatchableBackedge(backedge, loopHeader, interruptCheck); + + // Point the backedge to either of its possible targets, according to + // whether an interrupt is currently desired, matching the targets + // established by ensureIonCodeAccessible() above. We don't handle the + // interrupt immediately as the interrupt lock is held here. + PatchJump(backedge, cx->runtime()->interrupt ? interruptCheck : loopHeader); + + cx->runtime()->jitRuntime()->addPatchableBackedge(patchableBackedge); + } +} + +void +IonScript::copySafepointIndices(const SafepointIndex *si, MacroAssembler &masm) +{ + // Jumps in the caches reflect the offset of those jumps in the compiled + // code, not the absolute positions of the jumps. Update according to the + // final code address now. + SafepointIndex *table = safepointIndices(); + memcpy(table, si, safepointIndexEntries_ * sizeof(SafepointIndex)); + for (size_t i = 0; i < safepointIndexEntries_; i++) + table[i].adjustDisplacement(masm.actualOffset(table[i].displacement())); +} + +void +IonScript::copyOsiIndices(const OsiIndex *oi, MacroAssembler &masm) +{ + memcpy(osiIndices(), oi, osiIndexEntries_ * sizeof(OsiIndex)); + for (unsigned i = 0; i < osiIndexEntries_; i++) + osiIndices()[i].fixUpOffset(masm); +} + +void +IonScript::copyRuntimeData(const uint8_t *data) +{ + memcpy(runtimeData(), data, runtimeSize()); +} + +void +IonScript::copyCacheEntries(const uint32_t *caches, MacroAssembler &masm) +{ + memcpy(cacheIndex(), caches, numCaches() * sizeof(uint32_t)); + + // Jumps in the caches reflect the offset of those jumps in the compiled + // code, not the absolute positions of the jumps. Update according to the + // final code address now. + for (size_t i = 0; i < numCaches(); i++) + getCacheFromIndex(i).updateBaseAddress(method_, masm); +} + +const SafepointIndex * +IonScript::getSafepointIndex(uint32_t disp) const +{ + JS_ASSERT(safepointIndexEntries_ > 0); + + const SafepointIndex *table = safepointIndices(); + if (safepointIndexEntries_ == 1) { + JS_ASSERT(disp == table[0].displacement()); + return &table[0]; + } + + size_t minEntry = 0; + size_t maxEntry = safepointIndexEntries_ - 1; + uint32_t min = table[minEntry].displacement(); + uint32_t max = table[maxEntry].displacement(); + + // Raise if the element is not in the list. + JS_ASSERT(min <= disp && disp <= max); + + // Approximate the location of the FrameInfo. + size_t guess = (disp - min) * (maxEntry - minEntry) / (max - min) + minEntry; + uint32_t guessDisp = table[guess].displacement(); + + if (table[guess].displacement() == disp) + return &table[guess]; + + // Doing a linear scan from the guess should be more efficient in case of + // small group which are equally distributed on the code. + // + // such as: <... ... ... ... . ... ...> + if (guessDisp > disp) { + while (--guess >= minEntry) { + guessDisp = table[guess].displacement(); + JS_ASSERT(guessDisp >= disp); + if (guessDisp == disp) + return &table[guess]; + } + } else { + while (++guess <= maxEntry) { + guessDisp = table[guess].displacement(); + JS_ASSERT(guessDisp <= disp); + if (guessDisp == disp) + return &table[guess]; + } + } + + MOZ_ASSUME_UNREACHABLE("displacement not found."); +} + +const OsiIndex * +IonScript::getOsiIndex(uint32_t disp) const +{ + for (const OsiIndex *it = osiIndices(), *end = osiIndices() + osiIndexEntries_; + it != end; + ++it) + { + if (it->returnPointDisplacement() == disp) + return it; + } + + MOZ_ASSUME_UNREACHABLE("Failed to find OSI point return address"); +} + +const OsiIndex * +IonScript::getOsiIndex(uint8_t *retAddr) const +{ + IonSpew(IonSpew_Invalidate, "IonScript %p has method %p raw %p", (void *) this, (void *) + method(), method()->raw()); + + JS_ASSERT(containsCodeAddress(retAddr)); + uint32_t disp = retAddr - method()->raw(); + return getOsiIndex(disp); +} + +void +IonScript::Trace(JSTracer *trc, IonScript *script) +{ + if (script != ION_DISABLED_SCRIPT) + script->trace(trc); +} + +void +IonScript::Destroy(FreeOp *fop, IonScript *script) +{ + script->destroyCaches(); + script->unlinkFromRuntime(fop); + fop->free_(script); +} + +void +IonScript::toggleBarriers(bool enabled) +{ + method()->togglePreBarriers(enabled); +} + +void +IonScript::purgeCaches() +{ + // Don't reset any ICs if we're invalidated, otherwise, repointing the + // inline jump could overwrite an invalidation marker. These ICs can + // no longer run, however, the IC slow paths may be active on the stack. + // ICs therefore are required to check for invalidation before patching, + // to ensure the same invariant. + if (invalidated()) + return; + + for (size_t i = 0; i < numCaches(); i++) + getCacheFromIndex(i).reset(); +} + +void +IonScript::destroyCaches() +{ + for (size_t i = 0; i < numCaches(); i++) + getCacheFromIndex(i).destroy(); +} + +bool +IonScript::addDependentAsmJSModule(JSContext *cx, DependentAsmJSModuleExit exit) +{ + if (!dependentAsmJSModules) { + dependentAsmJSModules = cx->new_ >(cx); + if (!dependentAsmJSModules) + return false; + } + return dependentAsmJSModules->append(exit); +} + +void +IonScript::unlinkFromRuntime(FreeOp *fop) +{ + // Remove any links from AsmJSModules that contain optimized FFI calls into + // this IonScript. + if (dependentAsmJSModules) { + for (size_t i = 0; i < dependentAsmJSModules->length(); i++) { + DependentAsmJSModuleExit exit = dependentAsmJSModules->begin()[i]; + exit.module->detachIonCompilation(exit.exitIndex); + } + + fop->delete_(dependentAsmJSModules); + dependentAsmJSModules = nullptr; + } + + // The writes to the executable buffer below may clobber backedge jumps, so + // make sure that those backedges are unlinked from the runtime and not + // reclobbered with garbage if an interrupt is requested. + JSRuntime *rt = fop->runtime(); + for (size_t i = 0; i < backedgeEntries_; i++) { + PatchableBackedge *backedge = &backedgeList()[i]; + rt->jitRuntime()->removePatchableBackedge(backedge); + } + + // Clear the list of backedges, so that this method is idempotent. It is + // called during destruction, and may be additionally called when the + // script is invalidated. + backedgeEntries_ = 0; +} + +void +jit::ToggleBarriers(JS::Zone *zone, bool needs) +{ + JSRuntime *rt = zone->runtimeFromMainThread(); + if (!rt->hasJitRuntime()) + return; + + for (gc::CellIterUnderGC i(zone, gc::FINALIZE_SCRIPT); !i.done(); i.next()) { + JSScript *script = i.get(); + if (script->hasIonScript()) + script->ionScript()->toggleBarriers(needs); + if (script->hasBaselineScript()) + script->baselineScript()->toggleBarriers(needs); + } + + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + if (comp->jitCompartment()) + comp->jitCompartment()->toggleBaselineStubBarriers(needs); + } +} + +namespace js { +namespace jit { + +bool +OptimizeMIR(MIRGenerator *mir) +{ + MIRGraph &graph = mir->graph(); + TraceLogger *logger; + if (GetIonContext()->runtime->onMainThread()) + logger = TraceLoggerForMainThread(GetIonContext()->runtime); + else + logger = TraceLoggerForCurrentThread(); + + if (!mir->compilingAsmJS()) { + if (!MakeMRegExpHoistable(graph)) + return false; + } + + IonSpewPass("BuildSSA"); + AssertBasicGraphCoherency(graph); + + if (mir->shouldCancel("Start")) + return false; + + { + AutoTraceLog log(logger, TraceLogger::SplitCriticalEdges); + if (!SplitCriticalEdges(graph)) + return false; + IonSpewPass("Split Critical Edges"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Split Critical Edges")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger::RenumberBlocks); + if (!RenumberBlocks(graph)) + return false; + IonSpewPass("Renumber Blocks"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Renumber Blocks")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger::DominatorTree); + if (!BuildDominatorTree(graph)) + return false; + // No spew: graph not changed. + + if (mir->shouldCancel("Dominator Tree")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger::PhiAnalysis); + // Aggressive phi elimination must occur before any code elimination. If the + // script contains a try-statement, we only compiled the try block and not + // the catch or finally blocks, so in this case it's also invalid to use + // aggressive phi elimination. + Observability observability = graph.hasTryBlock() + ? ConservativeObservability + : AggressiveObservability; + if (!EliminatePhis(mir, graph, observability)) + return false; + IonSpewPass("Eliminate phis"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Eliminate phis")) + return false; + + if (!BuildPhiReverseMapping(graph)) + return false; + AssertExtendedGraphCoherency(graph); + // No spew: graph not changed. + + if (mir->shouldCancel("Phi reverse mapping")) + return false; + } + + if (!mir->compilingAsmJS()) { + AutoTraceLog log(logger, TraceLogger::ApplyTypes); + if (!ApplyTypeInformation(mir, graph)) + return false; + IonSpewPass("Apply types"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Apply types")) + return false; + } + + if (graph.entryBlock()->info().executionMode() == ParallelExecution) { + AutoTraceLog log(logger, TraceLogger::ParallelSafetyAnalysis); + ParallelSafetyAnalysis analysis(mir, graph); + if (!analysis.analyze()) + return false; + } + + // Alias analysis is required for LICM and GVN so that we don't move + // loads across stores. + if (mir->optimizationInfo().licmEnabled() || + mir->optimizationInfo().gvnEnabled()) + { + AutoTraceLog log(logger, TraceLogger::AliasAnalysis); + AliasAnalysis analysis(mir, graph); + if (!analysis.analyze()) + return false; + IonSpewPass("Alias analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Alias analysis")) + return false; + + // Eliminating dead resume point operands requires basic block + // instructions to be numbered. Reuse the numbering computed during + // alias analysis. + if (!EliminateDeadResumePointOperands(mir, graph)) + return false; + + if (mir->shouldCancel("Eliminate dead resume point operands")) + return false; + } + + if (mir->optimizationInfo().gvnEnabled()) { + AutoTraceLog log(logger, TraceLogger::GVN); + ValueNumberer gvn(mir, graph, mir->optimizationInfo().gvnKind() == GVN_Optimistic); + if (!gvn.analyze()) + return false; + IonSpewPass("GVN"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("GVN")) + return false; + } + + if (mir->optimizationInfo().uceEnabled()) { + AutoTraceLog log(logger, TraceLogger::UCE); + UnreachableCodeElimination uce(mir, graph); + if (!uce.analyze()) + return false; + IonSpewPass("UCE"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("UCE")) + return false; + } + + if (mir->optimizationInfo().licmEnabled()) { + AutoTraceLog log(logger, TraceLogger::LICM); + // LICM can hoist instructions from conditional branches and trigger + // repeated bailouts. Disable it if this script is known to bailout + // frequently. + JSScript *script = mir->info().script(); + if (!script || !script->hadFrequentBailouts()) { + LICM licm(mir, graph); + if (!licm.analyze()) + return false; + IonSpewPass("LICM"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("LICM")) + return false; + } + } + + if (mir->optimizationInfo().rangeAnalysisEnabled()) { + AutoTraceLog log(logger, TraceLogger::RangeAnalysis); + RangeAnalysis r(mir, graph); + if (!r.addBetaNodes()) + return false; + IonSpewPass("Beta"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("RA Beta")) + return false; + + if (!r.analyze() || !r.addRangeAssertions()) + return false; + IonSpewPass("Range Analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Range Analysis")) + return false; + + if (!r.removeBetaNodes()) + return false; + IonSpewPass("De-Beta"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("RA De-Beta")) + return false; + + if (mir->optimizationInfo().uceEnabled()) { + bool shouldRunUCE = false; + if (!r.prepareForUCE(&shouldRunUCE)) + return false; + IonSpewPass("RA check UCE"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("RA check UCE")) + return false; + + if (shouldRunUCE) { + UnreachableCodeElimination uce(mir, graph); + uce.disableAliasAnalysis(); + if (!uce.analyze()) + return false; + IonSpewPass("UCE After RA"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("UCE After RA")) + return false; + } + } + + if (!r.truncate()) + return false; + IonSpewPass("Truncate Doubles"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Truncate Doubles")) + return false; + } + + if (mir->optimizationInfo().eaaEnabled()) { + AutoTraceLog log(logger, TraceLogger::EffectiveAddressAnalysis); + EffectiveAddressAnalysis eaa(graph); + if (!eaa.analyze()) + return false; + IonSpewPass("Effective Address Analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Effective Address Analysis")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger::EliminateDeadCode); + if (!EliminateDeadCode(mir, graph)) + return false; + IonSpewPass("DCE"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("DCE")) + return false; + } + + // Passes after this point must not move instructions; these analyses + // depend on knowing the final order in which instructions will execute. + + if (mir->optimizationInfo().edgeCaseAnalysisEnabled()) { + AutoTraceLog log(logger, TraceLogger::EdgeCaseAnalysis); + EdgeCaseAnalysis edgeCaseAnalysis(mir, graph); + if (!edgeCaseAnalysis.analyzeLate()) + return false; + IonSpewPass("Edge Case Analysis (Late)"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Edge Case Analysis (Late)")) + return false; + } + + if (mir->optimizationInfo().eliminateRedundantChecksEnabled()) { + AutoTraceLog log(logger, TraceLogger::EliminateRedundantChecks); + // Note: check elimination has to run after all other passes that move + // instructions. Since check uses are replaced with the actual index, + // code motion after this pass could incorrectly move a load or store + // before its bounds check. + if (!EliminateRedundantChecks(graph)) + return false; + IonSpewPass("Bounds Check Elimination"); + AssertGraphCoherency(graph); + } + + return true; +} + +LIRGraph * +GenerateLIR(MIRGenerator *mir) +{ + MIRGraph &graph = mir->graph(); + + LIRGraph *lir = mir->alloc().lifoAlloc()->new_(&graph); + if (!lir || !lir->init()) + return nullptr; + + LIRGenerator lirgen(mir, graph, *lir); + if (!lirgen.generate()) + return nullptr; + IonSpewPass("Generate LIR"); + + if (mir->shouldCancel("Generate LIR")) + return nullptr; + + AllocationIntegrityState integrity(*lir); + + switch (mir->optimizationInfo().registerAllocator()) { + case RegisterAllocator_LSRA: { +#ifdef DEBUG + if (!integrity.record()) + return nullptr; +#endif + + LinearScanAllocator regalloc(mir, &lirgen, *lir); + if (!regalloc.go()) + return nullptr; + +#ifdef DEBUG + if (!integrity.check(false)) + return nullptr; +#endif + + IonSpewPass("Allocate Registers [LSRA]", ®alloc); + break; + } + + case RegisterAllocator_Backtracking: { +#ifdef DEBUG + if (!integrity.record()) + return nullptr; +#endif + + BacktrackingAllocator regalloc(mir, &lirgen, *lir); + if (!regalloc.go()) + return nullptr; + +#ifdef DEBUG + if (!integrity.check(false)) + return nullptr; +#endif + + IonSpewPass("Allocate Registers [Backtracking]"); + break; + } + + case RegisterAllocator_Stupid: { + // Use the integrity checker to populate safepoint information, so + // run it in all builds. + if (!integrity.record()) + return nullptr; + + StupidAllocator regalloc(mir, &lirgen, *lir); + if (!regalloc.go()) + return nullptr; + if (!integrity.check(true)) + return nullptr; + IonSpewPass("Allocate Registers [Stupid]"); + break; + } + + default: + MOZ_ASSUME_UNREACHABLE("Bad regalloc"); + } + + if (mir->shouldCancel("Allocate Registers")) + return nullptr; + + // Now that all optimization and register allocation is done, re-introduce + // critical edges to avoid unnecessary jumps. + if (!UnsplitEdges(lir)) + return nullptr; + IonSpewPass("Unsplit Critical Edges"); + AssertBasicGraphCoherency(graph); + + return lir; +} + +CodeGenerator * +GenerateCode(MIRGenerator *mir, LIRGraph *lir) +{ + CodeGenerator *codegen = js_new(mir, lir); + if (!codegen) + return nullptr; + + if (!codegen->generate()) { + js_delete(codegen); + return nullptr; + } + + return codegen; +} + +CodeGenerator * +CompileBackEnd(MIRGenerator *mir) +{ + if (!OptimizeMIR(mir)) + return nullptr; + + LIRGraph *lir = GenerateLIR(mir); + if (!lir) + return nullptr; + + return GenerateCode(mir, lir); +} + +void +AttachFinishedCompilations(JSContext *cx) +{ +#ifdef JS_THREADSAFE + JitCompartment *ion = cx->compartment()->jitCompartment(); + if (!ion) + return; + + types::AutoEnterAnalysis enterTypes(cx); + AutoLockWorkerThreadState lock; + + GlobalWorkerThreadState::IonBuilderVector &finished = WorkerThreadState().ionFinishedList(); + + TraceLogger *logger = TraceLoggerForMainThread(cx->runtime()); + + // Incorporate any off thread compilations for the compartment which have + // finished, failed or have been cancelled. + while (true) { + IonBuilder *builder = nullptr; + + // Find a finished builder for the compartment. + for (size_t i = 0; i < finished.length(); i++) { + IonBuilder *testBuilder = finished[i]; + if (testBuilder->compartment == CompileCompartment::get(cx->compartment())) { + builder = testBuilder; + WorkerThreadState().remove(finished, &i); + break; + } + } + if (!builder) + break; + + if (CodeGenerator *codegen = builder->backgroundCodegen()) { + RootedScript script(cx, builder->script()); + IonContext ictx(cx, &builder->alloc()); + AutoTraceLog logScript(logger, TraceLogCreateTextId(logger, script)); + AutoTraceLog logLink(logger, TraceLogger::IonLinking); + + // Root the assembler until the builder is finished below. As it + // was constructed off thread, the assembler has not been rooted + // previously, though any GC activity would discard the builder. + codegen->masm.constructRoot(cx); + + bool success; + { + // Release the worker thread lock and root the compiler for GC. + AutoTempAllocatorRooter root(cx, &builder->alloc()); + AutoUnlockWorkerThreadState unlock; + success = codegen->link(cx, builder->constraints()); + } + + if (!success) { + // Silently ignore OOM during code generation. The caller is + // InvokeInterruptCallback, which always runs at a + // nondeterministic time. It's not OK to throw a catchable + // exception from there. + cx->clearPendingException(); + } + } + + FinishOffThreadBuilder(builder); + } +#endif +} + +static const size_t BUILDER_LIFO_ALLOC_PRIMARY_CHUNK_SIZE = 1 << 12; + +static inline bool +OffThreadCompilationAvailable(JSContext *cx) +{ +#ifdef JS_THREADSAFE + // Even if off thread compilation is enabled, compilation must still occur + // on the main thread in some cases. Do not compile off thread during an + // incremental GC, as this may trip incremental read barriers. + // + // Require cpuCount > 1 so that Ion compilation jobs and main-thread + // execution are not competing for the same resources. + // + // Skip off thread compilation if PC count profiling is enabled, as + // CodeGenerator::maybeCreateScriptCounts will not attach script profiles + // when running off thread. + return cx->runtime()->canUseParallelIonCompilation() + && WorkerThreadState().cpuCount > 1 + && cx->runtime()->gcIncrementalState == gc::NO_INCREMENTAL + && !cx->runtime()->profilingScripts; +#else + return false; +#endif +} + +static void +TrackAllProperties(JSContext *cx, JSObject *obj) +{ + JS_ASSERT(obj->hasSingletonType()); + + for (Shape::Range range(obj->lastProperty()); !range.empty(); range.popFront()) + types::EnsureTrackPropertyTypes(cx, obj, range.front().propid()); +} + +static void +TrackPropertiesForSingletonScopes(JSContext *cx, JSScript *script, BaselineFrame *baselineFrame) +{ + // Ensure that all properties of singleton call objects which the script + // could access are tracked. These are generally accessed through + // ALIASEDVAR operations in baseline and will not be tracked even if they + // have been accessed in baseline code. + JSObject *environment = script->functionNonDelazifying() + ? script->functionNonDelazifying()->environment() + : nullptr; + + while (environment && !environment->is()) { + if (environment->is() && environment->hasSingletonType()) + TrackAllProperties(cx, environment); + environment = environment->enclosingScope(); + } + + if (baselineFrame) { + JSObject *scope = baselineFrame->scopeChain(); + if (scope->is() && scope->hasSingletonType()) + TrackAllProperties(cx, scope); + } +} + +static AbortReason +IonCompile(JSContext *cx, JSScript *script, + BaselineFrame *baselineFrame, jsbytecode *osrPc, bool constructing, + ExecutionMode executionMode, bool recompile, + OptimizationLevel optimizationLevel) +{ + TraceLogger *logger = TraceLoggerForMainThread(cx->runtime()); + AutoTraceLog logScript(logger, TraceLogCreateTextId(logger, script)); + AutoTraceLog logCompile(logger, TraceLogger::IonCompilation); + + JS_ASSERT(optimizationLevel > Optimization_DontCompile); + + // Make sure the script's canonical function isn't lazy. We can't de-lazify + // it in a worker thread. + script->ensureNonLazyCanonicalFunction(cx); + + TrackPropertiesForSingletonScopes(cx, script, baselineFrame); + + LifoAlloc *alloc = cx->new_(BUILDER_LIFO_ALLOC_PRIMARY_CHUNK_SIZE); + if (!alloc) + return AbortReason_Alloc; + + ScopedJSDeletePtr autoDelete(alloc); + + TempAllocator *temp = alloc->new_(alloc); + if (!temp) + return AbortReason_Alloc; + + IonContext ictx(cx, temp); + + types::AutoEnterAnalysis enter(cx); + + if (!cx->compartment()->ensureJitCompartmentExists(cx)) + return AbortReason_Alloc; + + if (!cx->compartment()->jitCompartment()->ensureIonStubsExist(cx)) + return AbortReason_Alloc; + + if (executionMode == ParallelExecution && + LIRGenerator::allowInlineForkJoinGetSlice() && + !cx->runtime()->jitRuntime()->ensureForkJoinGetSliceStubExists(cx)) + { + return AbortReason_Alloc; + } + + MIRGraph *graph = alloc->new_(temp); + if (!graph) + return AbortReason_Alloc; + + CompileInfo *info = alloc->new_(script, script->functionNonDelazifying(), osrPc, + constructing, executionMode, + script->needsArgsObj()); + if (!info) + return AbortReason_Alloc; + + BaselineInspector *inspector = alloc->new_(script); + if (!inspector) + return AbortReason_Alloc; + + BaselineFrameInspector *baselineFrameInspector = nullptr; + if (baselineFrame) { + baselineFrameInspector = NewBaselineFrameInspector(temp, baselineFrame, info); + if (!baselineFrameInspector) + return AbortReason_Alloc; + } + + AutoTempAllocatorRooter root(cx, temp); + types::CompilerConstraintList *constraints = types::NewCompilerConstraintList(*temp); + if (!constraints) + return AbortReason_Alloc; + + const OptimizationInfo *optimizationInfo = js_IonOptimizations.get(optimizationLevel); + const JitCompileOptions options(cx); + + IonBuilder *builder = alloc->new_((JSContext *) nullptr, + CompileCompartment::get(cx->compartment()), + options, temp, graph, constraints, + inspector, info, optimizationInfo, + baselineFrameInspector); + if (!builder) + return AbortReason_Alloc; + + JS_ASSERT(recompile == HasIonScript(builder->script(), executionMode)); + JS_ASSERT(CanIonCompile(builder->script(), executionMode)); + + RootedScript builderScript(cx, builder->script()); + + if (recompile) { + JS_ASSERT(executionMode == SequentialExecution); + builderScript->ionScript()->setRecompiling(); + } + + IonSpewNewFunction(graph, builderScript); + + bool succeeded = builder->build(); + builder->clearForBackEnd(); + + if (!succeeded) + return builder->abortReason(); + + // If possible, compile the script off thread. + if (OffThreadCompilationAvailable(cx)) { + if (!recompile) + SetIonScript(builderScript, executionMode, ION_COMPILING_SCRIPT); + + IonSpew(IonSpew_Logs, "Can't log script %s:%d. (Compiled on background thread.)", + builderScript->filename(), builderScript->lineno()); + + if (!StartOffThreadIonCompile(cx, builder)) { + IonSpew(IonSpew_Abort, "Unable to start off-thread ion compilation."); + return AbortReason_Alloc; + } + + // The allocator and associated data will be destroyed after being + // processed in the finishedOffThreadCompilations list. + autoDelete.forget(); + + return AbortReason_NoAbort; + } + + ScopedJSDeletePtr codegen(CompileBackEnd(builder)); + if (!codegen) { + IonSpew(IonSpew_Abort, "Failed during back-end compilation."); + return AbortReason_Disable; + } + + bool success = codegen->link(cx, builder->constraints()); + + IonSpewEndFunction(); + + return success ? AbortReason_NoAbort : AbortReason_Disable; +} + +static bool +CheckFrame(BaselineFrame *frame) +{ + JS_ASSERT(!frame->isGeneratorFrame()); + JS_ASSERT(!frame->isDebuggerFrame()); + + // This check is to not overrun the stack. + if (frame->isFunctionFrame() && TooManyArguments(frame->numActualArgs())) { + IonSpew(IonSpew_Abort, "too many actual args"); + return false; + } + + return true; +} + +static bool +CheckScript(JSContext *cx, JSScript *script, bool osr) +{ + if (script->isForEval()) { + // Eval frames are not yet supported. Supporting this will require new + // logic in pushBailoutFrame to deal with linking prev. + // Additionally, JSOP_DEFVAR support will require baking in isEvalFrame(). + IonSpew(IonSpew_Abort, "eval script"); + return false; + } + + if (!script->compileAndGo()) { + IonSpew(IonSpew_Abort, "not compile-and-go"); + return false; + } + + return true; +} + +static MethodStatus +CheckScriptSize(JSContext *cx, JSScript* script) +{ + if (!js_JitOptions.limitScriptSize) + return Method_Compiled; + + if (script->length() > MAX_OFF_THREAD_SCRIPT_SIZE) { + // Some scripts are so large we never try to Ion compile them. + IonSpew(IonSpew_Abort, "Script too large (%u bytes)", script->length()); + return Method_CantCompile; + } + + uint32_t numLocalsAndArgs = analyze::TotalSlots(script); + if (cx->runtime()->isWorkerRuntime()) { + // DOM Workers don't have off thread compilation enabled. Since workers + // don't block the browser's event loop, allow them to compile larger + // scripts. + JS_ASSERT(!cx->runtime()->canUseParallelIonCompilation()); + + if (script->length() > MAX_DOM_WORKER_SCRIPT_SIZE || + numLocalsAndArgs > MAX_DOM_WORKER_LOCALS_AND_ARGS) + { + return Method_CantCompile; + } + + return Method_Compiled; + } + + if (script->length() > MAX_MAIN_THREAD_SCRIPT_SIZE || + numLocalsAndArgs > MAX_MAIN_THREAD_LOCALS_AND_ARGS) + { +#ifdef JS_THREADSAFE + size_t cpuCount = WorkerThreadState().cpuCount; +#else + size_t cpuCount = 1; +#endif + if (cx->runtime()->canUseParallelIonCompilation() && cpuCount > 1) { + // Even if off thread compilation is enabled, there are cases where + // compilation must still occur on the main thread. Don't compile + // in these cases (except when profiling scripts, as compilations + // occurring with profiling should reflect those without), but do + // not forbid compilation so that the script may be compiled later. + if (!OffThreadCompilationAvailable(cx) && !cx->runtime()->profilingScripts) { + IonSpew(IonSpew_Abort, + "Script too large for main thread, skipping (%u bytes) (%u locals/args)", + script->length(), numLocalsAndArgs); + return Method_Skipped; + } + } else { + IonSpew(IonSpew_Abort, "Script too large (%u bytes) (%u locals/args)", + script->length(), numLocalsAndArgs); + return Method_CantCompile; + } + } + + return Method_Compiled; +} + +bool +CanIonCompileScript(JSContext *cx, JSScript *script, bool osr) +{ + if (!script->canIonCompile() || !CheckScript(cx, script, osr)) + return false; + + return CheckScriptSize(cx, script) == Method_Compiled; +} + +static OptimizationLevel +GetOptimizationLevel(HandleScript script, jsbytecode *pc, ExecutionMode executionMode) +{ + if (executionMode == ParallelExecution) + return Optimization_Normal; + + JS_ASSERT(executionMode == SequentialExecution); + + return js_IonOptimizations.levelForScript(script, pc); +} + +static MethodStatus +Compile(JSContext *cx, HandleScript script, BaselineFrame *osrFrame, jsbytecode *osrPc, + bool constructing, ExecutionMode executionMode) +{ + JS_ASSERT(jit::IsIonEnabled(cx)); + JS_ASSERT(jit::IsBaselineEnabled(cx)); + JS_ASSERT_IF(osrPc != nullptr, LoopEntryCanIonOsr(osrPc)); + JS_ASSERT_IF(executionMode == ParallelExecution, !osrFrame && !osrPc); + JS_ASSERT_IF(executionMode == ParallelExecution, !HasIonScript(script, executionMode)); + + if (!script->hasBaselineScript()) + return Method_Skipped; + + if (cx->compartment()->debugMode()) { + IonSpew(IonSpew_Abort, "debugging"); + return Method_CantCompile; + } + + if (!CheckScript(cx, script, bool(osrPc))) { + IonSpew(IonSpew_Abort, "Aborted compilation of %s:%d", script->filename(), script->lineno()); + return Method_CantCompile; + } + + MethodStatus status = CheckScriptSize(cx, script); + if (status != Method_Compiled) { + IonSpew(IonSpew_Abort, "Aborted compilation of %s:%d", script->filename(), script->lineno()); + return status; + } + + bool recompile = false; + OptimizationLevel optimizationLevel = GetOptimizationLevel(script, osrPc, executionMode); + if (optimizationLevel == Optimization_DontCompile) + return Method_Skipped; + + IonScript *scriptIon = GetIonScript(script, executionMode); + if (scriptIon) { + if (!scriptIon->method()) + return Method_CantCompile; + + MethodStatus failedState = Method_Compiled; + + // If we keep failing to enter the script due to an OSR pc mismatch, + // recompile with the right pc. + if (osrPc && script->ionScript()->osrPc() != osrPc) { + uint32_t count = script->ionScript()->incrOsrPcMismatchCounter(); + if (count <= js_JitOptions.osrPcMismatchesBeforeRecompile) + return Method_Skipped; + + failedState = Method_Skipped; + } + + // Don't recompile/overwrite higher optimized code, + // with a lower optimization level. + if (optimizationLevel < scriptIon->optimizationLevel()) + return failedState; + + if (optimizationLevel == scriptIon->optimizationLevel() && + (!osrPc || script->ionScript()->osrPc() == osrPc)) + { + return failedState; + } + + // Don't start compiling if already compiling + if (scriptIon->isRecompiling()) + return failedState; + + if (osrPc) + script->ionScript()->resetOsrPcMismatchCounter(); + + recompile = true; + } + + AbortReason reason = IonCompile(cx, script, osrFrame, osrPc, constructing, executionMode, + recompile, optimizationLevel); + if (reason == AbortReason_Error) + return Method_Error; + + if (reason == AbortReason_Disable) + return Method_CantCompile; + + if (reason == AbortReason_Alloc) { + js_ReportOutOfMemory(cx); + return Method_Error; + } + + // Compilation succeeded or we invalidated right away or an inlining/alloc abort + if (HasIonScript(script, executionMode)) { + if (osrPc && script->ionScript()->osrPc() != osrPc) + return Method_Skipped; + return Method_Compiled; + } + return Method_Skipped; +} + +} // namespace jit +} // namespace js + +// Decide if a transition from interpreter execution to Ion code should occur. +// May compile or recompile the target JSScript. +MethodStatus +jit::CanEnterAtBranch(JSContext *cx, JSScript *script, BaselineFrame *osrFrame, + jsbytecode *pc, bool isConstructing) +{ + JS_ASSERT(jit::IsIonEnabled(cx)); + JS_ASSERT((JSOp)*pc == JSOP_LOOPENTRY); + JS_ASSERT(LoopEntryCanIonOsr(pc)); + + // Skip if the script has been disabled. + if (!script->canIonCompile()) + return Method_Skipped; + + // Skip if the script is being compiled off thread. + if (script->isIonCompilingOffThread()) + return Method_Skipped; + + // Skip if the code is expected to result in a bailout. + if (script->hasIonScript() && script->ionScript()->bailoutExpected()) + return Method_Skipped; + + // Optionally ignore on user request. + if (!js_JitOptions.osr) + return Method_Skipped; + + // Mark as forbidden if frame can't be handled. + if (!CheckFrame(osrFrame)) { + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + // Attempt compilation. + // - Returns Method_Compiled if the right ionscript is present + // (Meaning it was present or a sequantial compile finished) + // - Returns Method_Skipped if pc doesn't match + // (This means a background thread compilation with that pc could have started or not.) + RootedScript rscript(cx, script); + MethodStatus status = Compile(cx, rscript, osrFrame, pc, isConstructing, SequentialExecution); + if (status != Method_Compiled) { + if (status == Method_CantCompile) + ForbidCompilation(cx, script); + return status; + } + + return Method_Compiled; +} + +MethodStatus +jit::CanEnter(JSContext *cx, RunState &state) +{ + JS_ASSERT(jit::IsIonEnabled(cx)); + + JSScript *script = state.script(); + + // Skip if the script has been disabled. + if (!script->canIonCompile()) + return Method_Skipped; + + // Skip if the script is being compiled off thread. + if (script->isIonCompilingOffThread()) + return Method_Skipped; + + // Skip if the code is expected to result in a bailout. + if (script->hasIonScript() && script->ionScript()->bailoutExpected()) + return Method_Skipped; + + // If constructing, allocate a new |this| object before building Ion. + // Creating |this| is done before building Ion because it may change the + // type information and invalidate compilation results. + if (state.isInvoke()) { + InvokeState &invoke = *state.asInvoke(); + + if (TooManyArguments(invoke.args().length())) { + IonSpew(IonSpew_Abort, "too many actual args"); + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + if (TooManyArguments(invoke.args().callee().as().nargs())) { + IonSpew(IonSpew_Abort, "too many args"); + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + if (invoke.constructing() && invoke.args().thisv().isPrimitive()) { + RootedScript scriptRoot(cx, script); + RootedObject callee(cx, &invoke.args().callee()); + RootedObject obj(cx, CreateThisForFunction(cx, callee, + invoke.useNewType() + ? SingletonObject + : GenericObject)); + if (!obj || !jit::IsIonEnabled(cx)) // Note: OOM under CreateThis can disable TI. + return Method_Skipped; + invoke.args().setThis(ObjectValue(*obj)); + script = scriptRoot; + } + } else if (state.isGenerator()) { + IonSpew(IonSpew_Abort, "generator frame"); + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + // If --ion-eager is used, compile with Baseline first, so that we + // can directly enter IonMonkey. + RootedScript rscript(cx, script); + if (js_JitOptions.eagerCompilation && !rscript->hasBaselineScript()) { + MethodStatus status = CanEnterBaselineMethod(cx, state); + if (status != Method_Compiled) + return status; + } + + // Attempt compilation. Returns Method_Compiled if already compiled. + bool constructing = state.isInvoke() && state.asInvoke()->constructing(); + MethodStatus status = + Compile(cx, rscript, nullptr, nullptr, constructing, SequentialExecution); + if (status != Method_Compiled) { + if (status == Method_CantCompile) + ForbidCompilation(cx, rscript); + return status; + } + + return Method_Compiled; +} + +MethodStatus +jit::CompileFunctionForBaseline(JSContext *cx, HandleScript script, BaselineFrame *frame, + bool isConstructing) +{ + JS_ASSERT(jit::IsIonEnabled(cx)); + JS_ASSERT(frame->fun()->nonLazyScript()->canIonCompile()); + JS_ASSERT(!frame->fun()->nonLazyScript()->isIonCompilingOffThread()); + JS_ASSERT(!frame->fun()->nonLazyScript()->hasIonScript()); + JS_ASSERT(frame->isFunctionFrame()); + + // Mark as forbidden if frame can't be handled. + if (!CheckFrame(frame)) { + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + // Attempt compilation. Returns Method_Compiled if already compiled. + MethodStatus status = + Compile(cx, script, frame, nullptr, isConstructing, SequentialExecution); + if (status != Method_Compiled) { + if (status == Method_CantCompile) + ForbidCompilation(cx, script); + return status; + } + + return Method_Compiled; +} + +MethodStatus +jit::Recompile(JSContext *cx, HandleScript script, BaselineFrame *osrFrame, jsbytecode *osrPc, + bool constructing) +{ + JS_ASSERT(script->hasIonScript()); + if (script->ionScript()->isRecompiling()) + return Method_Compiled; + + MethodStatus status = + Compile(cx, script, osrFrame, osrPc, constructing, SequentialExecution); + if (status != Method_Compiled) { + if (status == Method_CantCompile) + ForbidCompilation(cx, script); + return status; + } + + return Method_Compiled; +} + +MethodStatus +jit::CanEnterInParallel(JSContext *cx, HandleScript script) +{ + // Skip if the script has been disabled. + // + // Note: We return Method_Skipped in this case because the other + // CanEnter() methods do so. However, ForkJoin.cpp detects this + // condition differently treats it more like an error. + if (!script->canParallelIonCompile()) + return Method_Skipped; + + // Skip if the script is being compiled off thread. + if (script->isParallelIonCompilingOffThread()) + return Method_Skipped; + + MethodStatus status = Compile(cx, script, nullptr, nullptr, false, ParallelExecution); + if (status != Method_Compiled) { + if (status == Method_CantCompile) + ForbidCompilation(cx, script, ParallelExecution); + return status; + } + + // This can GC, so afterward, script->parallelIon is + // not guaranteed to be valid. + if (!cx->runtime()->jitRuntime()->enterIon()) + return Method_Error; + + // Subtle: it is possible for GC to occur during + // compilation of one of the invoked functions, which + // would cause the earlier functions (such as the + // kernel itself) to be collected. In this event, we + // give up and fallback to sequential for now. + if (!script->hasParallelIonScript()) { + parallel::Spew( + parallel::SpewCompile, + "Script %p:%s:%u was garbage-collected or invalidated", + script.get(), script->filename(), script->lineno()); + return Method_Skipped; + } + + return Method_Compiled; +} + +MethodStatus +jit::CanEnterUsingFastInvoke(JSContext *cx, HandleScript script, uint32_t numActualArgs) +{ + JS_ASSERT(jit::IsIonEnabled(cx)); + + // Skip if the code is expected to result in a bailout. + if (!script->hasIonScript() || script->ionScript()->bailoutExpected()) + return Method_Skipped; + + // Don't handle arguments underflow, to make this work we would have to pad + // missing arguments with |undefined|. + if (numActualArgs < script->functionNonDelazifying()->nargs()) + return Method_Skipped; + + if (!cx->compartment()->ensureJitCompartmentExists(cx)) + return Method_Error; + + // This can GC, so afterward, script->ion is not guaranteed to be valid. + if (!cx->runtime()->jitRuntime()->enterIon()) + return Method_Error; + + if (!script->hasIonScript()) + return Method_Skipped; + + return Method_Compiled; +} + +static IonExecStatus +EnterIon(JSContext *cx, EnterJitData &data) +{ + JS_CHECK_RECURSION(cx, return IonExec_Aborted); + JS_ASSERT(jit::IsIonEnabled(cx)); + JS_ASSERT(!data.osrFrame); + + EnterJitCode enter = cx->runtime()->jitRuntime()->enterIon(); + + // Caller must construct |this| before invoking the Ion function. + JS_ASSERT_IF(data.constructing, data.maxArgv[0].isObject()); + + data.result.setInt32(data.numActualArgs); + { + AssertCompartmentUnchanged pcc(cx); + JitActivation activation(cx, data.constructing); + + CALL_GENERATED_CODE(enter, data.jitcode, data.maxArgc, data.maxArgv, /* osrFrame = */nullptr, data.calleeToken, + /* scopeChain = */ nullptr, 0, data.result.address()); + } + + JS_ASSERT(!cx->runtime()->hasIonReturnOverride()); + + // Jit callers wrap primitive constructor return. + if (!data.result.isMagic() && data.constructing && data.result.isPrimitive()) + data.result = data.maxArgv[0]; + + // Release temporary buffer used for OSR into Ion. + cx->runtime()->getJitRuntime(cx)->freeOsrTempData(); + + JS_ASSERT_IF(data.result.isMagic(), data.result.isMagic(JS_ION_ERROR)); + return data.result.isMagic() ? IonExec_Error : IonExec_Ok; +} + +bool +jit::SetEnterJitData(JSContext *cx, EnterJitData &data, RunState &state, AutoValueVector &vals) +{ + data.osrFrame = nullptr; + + if (state.isInvoke()) { + CallArgs &args = state.asInvoke()->args(); + unsigned numFormals = state.script()->functionNonDelazifying()->nargs(); + data.constructing = state.asInvoke()->constructing(); + data.numActualArgs = args.length(); + data.maxArgc = Max(args.length(), numFormals) + 1; + data.scopeChain = nullptr; + data.calleeToken = CalleeToToken(&args.callee().as()); + + if (data.numActualArgs >= numFormals) { + data.maxArgv = args.base() + 1; + } else { + // Pad missing arguments with |undefined|. + for (size_t i = 1; i < args.length() + 2; i++) { + if (!vals.append(args.base()[i])) + return false; + } + + while (vals.length() < numFormals + 1) { + if (!vals.append(UndefinedValue())) + return false; + } + + JS_ASSERT(vals.length() >= numFormals + 1); + data.maxArgv = vals.begin(); + } + } else { + data.constructing = false; + data.numActualArgs = 0; + data.maxArgc = 1; + data.maxArgv = state.asExecute()->addressOfThisv(); + data.scopeChain = state.asExecute()->scopeChain(); + + data.calleeToken = CalleeToToken(state.script()); + + if (state.script()->isForEval() && + !(state.asExecute()->type() & InterpreterFrame::GLOBAL)) + { + ScriptFrameIter iter(cx); + if (iter.isFunctionFrame()) + data.calleeToken = CalleeToToken(iter.callee()); + } + } + + return true; +} + +IonExecStatus +jit::IonCannon(JSContext *cx, RunState &state) +{ + IonScript *ion = state.script()->ionScript(); + + EnterJitData data(cx); + data.jitcode = ion->method()->raw(); + + AutoValueVector vals(cx); + if (!SetEnterJitData(cx, data, state, vals)) + return IonExec_Error; + + IonExecStatus status = EnterIon(cx, data); + + if (status == IonExec_Ok) + state.setReturnValue(data.result); + + return status; +} + +IonExecStatus +jit::FastInvoke(JSContext *cx, HandleFunction fun, CallArgs &args) +{ + JS_CHECK_RECURSION(cx, return IonExec_Error); + + IonScript *ion = fun->nonLazyScript()->ionScript(); + JitCode *code = ion->method(); + void *jitcode = code->raw(); + + JS_ASSERT(jit::IsIonEnabled(cx)); + JS_ASSERT(!ion->bailoutExpected()); + + JitActivation activation(cx, /* firstFrameIsConstructing = */false); + + EnterJitCode enter = cx->runtime()->jitRuntime()->enterIon(); + void *calleeToken = CalleeToToken(fun); + + RootedValue result(cx, Int32Value(args.length())); + JS_ASSERT(args.length() >= fun->nargs()); + + CALL_GENERATED_CODE(enter, jitcode, args.length() + 1, args.array() - 1, /* osrFrame = */nullptr, + calleeToken, /* scopeChain = */ nullptr, 0, result.address()); + + JS_ASSERT(!cx->runtime()->hasIonReturnOverride()); + + args.rval().set(result); + + JS_ASSERT_IF(result.isMagic(), result.isMagic(JS_ION_ERROR)); + return result.isMagic() ? IonExec_Error : IonExec_Ok; +} + +static void +InvalidateActivation(FreeOp *fop, uint8_t *ionTop, bool invalidateAll) +{ + IonSpew(IonSpew_Invalidate, "BEGIN invalidating activation"); + + size_t frameno = 1; + + for (JitFrameIterator it(ionTop, SequentialExecution); !it.done(); ++it, ++frameno) { + JS_ASSERT_IF(frameno == 1, it.type() == JitFrame_Exit); + +#ifdef DEBUG + switch (it.type()) { + case JitFrame_Exit: + IonSpew(IonSpew_Invalidate, "#%d exit frame @ %p", frameno, it.fp()); + break; + case JitFrame_BaselineJS: + case JitFrame_IonJS: + { + JS_ASSERT(it.isScripted()); + const char *type = it.isIonJS() ? "Optimized" : "Baseline"; + IonSpew(IonSpew_Invalidate, "#%d %s JS frame @ %p, %s:%d (fun: %p, script: %p, pc %p)", + frameno, type, it.fp(), it.script()->filename(), it.script()->lineno(), + it.maybeCallee(), (JSScript *)it.script(), it.returnAddressToFp()); + break; + } + case JitFrame_BaselineStub: + IonSpew(IonSpew_Invalidate, "#%d baseline stub frame @ %p", frameno, it.fp()); + break; + case JitFrame_Rectifier: + IonSpew(IonSpew_Invalidate, "#%d rectifier frame @ %p", frameno, it.fp()); + break; + case JitFrame_Unwound_IonJS: + case JitFrame_Unwound_BaselineStub: + MOZ_ASSUME_UNREACHABLE("invalid"); + case JitFrame_Unwound_Rectifier: + IonSpew(IonSpew_Invalidate, "#%d unwound rectifier frame @ %p", frameno, it.fp()); + break; + case JitFrame_Entry: + IonSpew(IonSpew_Invalidate, "#%d entry frame @ %p", frameno, it.fp()); + break; + } +#endif + + if (!it.isIonJS()) + continue; + + // See if the frame has already been invalidated. + if (it.checkInvalidation()) + continue; + + JSScript *script = it.script(); + if (!script->hasIonScript()) + continue; + + if (!invalidateAll && !script->ionScript()->invalidated()) + continue; + + IonScript *ionScript = script->ionScript(); + + // Purge ICs before we mark this script as invalidated. This will + // prevent lastJump_ from appearing to be a bogus pointer, just + // in case anyone tries to read it. + ionScript->purgeCaches(); + + // Clean up any pointers from elsewhere in the runtime to this IonScript + // which is about to become disconnected from its JSScript. + ionScript->unlinkFromRuntime(fop); + + // This frame needs to be invalidated. We do the following: + // + // 1. Increment the reference counter to keep the ionScript alive + // for the invalidation bailout or for the exception handler. + // 2. Determine safepoint that corresponds to the current call. + // 3. From safepoint, get distance to the OSI-patchable offset. + // 4. From the IonScript, determine the distance between the + // call-patchable offset and the invalidation epilogue. + // 5. Patch the OSI point with a call-relative to the + // invalidation epilogue. + // + // The code generator ensures that there's enough space for us + // to patch in a call-relative operation at each invalidation + // point. + // + // Note: you can't simplify this mechanism to "just patch the + // instruction immediately after the call" because things may + // need to move into a well-defined register state (using move + // instructions after the call) in to capture an appropriate + // snapshot after the call occurs. + + ionScript->incref(); + + const SafepointIndex *si = ionScript->getSafepointIndex(it.returnAddressToFp()); + JitCode *ionCode = ionScript->method(); + + JS::Zone *zone = script->zone(); + if (zone->needsBarrier()) { + // We're about to remove edges from the JSScript to gcthings + // embedded in the JitCode. Perform one final trace of the + // JitCode for the incremental GC, as it must know about + // those edges. + ionCode->trace(zone->barrierTracer()); + } + ionCode->setInvalidated(); + + // Write the delta (from the return address offset to the + // IonScript pointer embedded into the invalidation epilogue) + // where the safepointed call instruction used to be. We rely on + // the call sequence causing the safepoint being >= the size of + // a uint32, which is checked during safepoint index + // construction. + CodeLocationLabel dataLabelToMunge(it.returnAddressToFp()); + ptrdiff_t delta = ionScript->invalidateEpilogueDataOffset() - + (it.returnAddressToFp() - ionCode->raw()); + Assembler::patchWrite_Imm32(dataLabelToMunge, Imm32(delta)); + + CodeLocationLabel osiPatchPoint = SafepointReader::InvalidationPatchPoint(ionScript, si); + CodeLocationLabel invalidateEpilogue(ionCode, ionScript->invalidateEpilogueOffset()); + + IonSpew(IonSpew_Invalidate, " ! Invalidate ionScript %p (ref %u) -> patching osipoint %p", + ionScript, ionScript->refcount(), (void *) osiPatchPoint.raw()); + Assembler::patchWrite_NearCall(osiPatchPoint, invalidateEpilogue); + } + + IonSpew(IonSpew_Invalidate, "END invalidating activation"); +} + +void +jit::StopAllOffThreadCompilations(JSCompartment *comp) +{ + if (!comp->jitCompartment()) + return; + CancelOffThreadIonCompile(comp, nullptr); + FinishAllOffThreadCompilations(comp); +} + +void +jit::InvalidateAll(FreeOp *fop, Zone *zone) +{ + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) + StopAllOffThreadCompilations(comp); + + for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) { + if (iter->compartment()->zone() == zone) { + IonSpew(IonSpew_Invalidate, "Invalidating all frames for GC"); + InvalidateActivation(fop, iter.jitTop(), true); + } + } +} + + +void +jit::Invalidate(types::TypeZone &types, FreeOp *fop, + const Vector &invalid, bool resetUses, + bool cancelOffThread) +{ + IonSpew(IonSpew_Invalidate, "Start invalidation."); + + // Add an invalidation reference to all invalidated IonScripts to indicate + // to the traversal which frames have been invalidated. + size_t numInvalidations = 0; + for (size_t i = 0; i < invalid.length(); i++) { + const types::CompilerOutput &co = *invalid[i].compilerOutput(types); + if (!co.isValid()) + continue; + + if (cancelOffThread) + CancelOffThreadIonCompile(co.script()->compartment(), co.script()); + + if (!co.ion()) + continue; + + IonSpew(IonSpew_Invalidate, " Invalidate %s:%u, IonScript %p", + co.script()->filename(), co.script()->lineno(), co.ion()); + + // Keep the ion script alive during the invalidation and flag this + // ionScript as being invalidated. This increment is removed by the + // loop after the calls to InvalidateActivation. + co.ion()->incref(); + numInvalidations++; + } + + if (!numInvalidations) { + IonSpew(IonSpew_Invalidate, " No IonScript invalidation."); + return; + } + + for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) + InvalidateActivation(fop, iter.jitTop(), false); + + // Drop the references added above. If a script was never active, its + // IonScript will be immediately destroyed. Otherwise, it will be held live + // until its last invalidated frame is destroyed. + for (size_t i = 0; i < invalid.length(); i++) { + types::CompilerOutput &co = *invalid[i].compilerOutput(types); + if (!co.isValid()) + continue; + + ExecutionMode executionMode = co.mode(); + JSScript *script = co.script(); + IonScript *ionScript = co.ion(); + if (!ionScript) + continue; + + SetIonScript(script, executionMode, nullptr); + ionScript->decref(fop); + co.invalidate(); + numInvalidations--; + + // Wait for the scripts to get warm again before doing another + // compile, unless either: + // (1) we are recompiling *because* a script got hot; + // (resetUses is false); or, + // (2) we are invalidating a parallel script. This is because + // the useCount only applies to sequential uses. Parallel + // execution *requires* ion, and so we don't limit it to + // methods with a high usage count (though we do check that + // the useCount is at least 1 when compiling the transitive + // closure of potential callees, to avoid compiling things + // that are never run at all). + if (resetUses && executionMode != ParallelExecution) + script->resetUseCount(); + } + + // Make sure we didn't leak references by invalidating the same IonScript + // multiple times in the above loop. + JS_ASSERT(!numInvalidations); +} + +void +jit::Invalidate(JSContext *cx, const Vector &invalid, bool resetUses, + bool cancelOffThread) +{ + jit::Invalidate(cx->zone()->types, cx->runtime()->defaultFreeOp(), invalid, resetUses, + cancelOffThread); +} + +bool +jit::Invalidate(JSContext *cx, JSScript *script, ExecutionMode mode, bool resetUses, + bool cancelOffThread) +{ + JS_ASSERT(script->hasIonScript()); + + if (cx->runtime()->spsProfiler.enabled()) { + // Register invalidation with profiler. + // Format of event payload string: + // ":" + + // Get the script filename, if any, and its length. + const char *filename = script->filename(); + if (filename == nullptr) + filename = ""; + + size_t len = strlen(filename) + 20; + char *buf = js_pod_malloc(len); + if (!buf) + return false; + + // Construct the descriptive string. + JS_snprintf(buf, len, "Invalidate %s:%llu", filename, script->lineno()); + cx->runtime()->spsProfiler.markEvent(buf); + js_free(buf); + } + + Vector scripts(cx); + + switch (mode) { + case SequentialExecution: + JS_ASSERT(script->hasIonScript()); + if (!scripts.append(script->ionScript()->recompileInfo())) + return false; + break; + case ParallelExecution: + JS_ASSERT(script->hasParallelIonScript()); + if (!scripts.append(script->parallelIonScript()->recompileInfo())) + return false; + break; + default: + MOZ_ASSUME_UNREACHABLE("No such execution mode"); + } + + Invalidate(cx, scripts, resetUses, cancelOffThread); + return true; +} + +bool +jit::Invalidate(JSContext *cx, JSScript *script, bool resetUses, bool cancelOffThread) +{ + return Invalidate(cx, script, SequentialExecution, resetUses, cancelOffThread); +} + +static void +FinishInvalidationOf(FreeOp *fop, JSScript *script, IonScript *ionScript) +{ + types::TypeZone &types = script->zone()->types; + + // Note: If the script is about to be swept, the compiler output may have + // already been destroyed. + if (types::CompilerOutput *output = ionScript->recompileInfo().compilerOutput(types)) + output->invalidate(); + + // If this script has Ion code on the stack, invalidated() will return + // true. In this case we have to wait until destroying it. + if (!ionScript->invalidated()) + jit::IonScript::Destroy(fop, ionScript); +} + +template +void +jit::FinishInvalidation(FreeOp *fop, JSScript *script) +{ + // In all cases, nullptr out script->ion or script->parallelIon to avoid + // re-entry. + switch (mode) { + case SequentialExecution: + if (script->hasIonScript()) { + IonScript *ion = script->ionScript(); + script->setIonScript(nullptr); + FinishInvalidationOf(fop, script, ion); + } + return; + + case ParallelExecution: + if (script->hasParallelIonScript()) { + IonScript *parallelIon = script->parallelIonScript(); + script->setParallelIonScript(nullptr); + FinishInvalidationOf(fop, script, parallelIon); + } + return; + + default: + MOZ_ASSUME_UNREACHABLE("bad execution mode"); + } +} + +template void +jit::FinishInvalidation(FreeOp *fop, JSScript *script); + +template void +jit::FinishInvalidation(FreeOp *fop, JSScript *script); + +void +jit::MarkValueFromIon(JSRuntime *rt, Value *vp) +{ + gc::MarkValueUnbarriered(&rt->gcMarker, vp, "write barrier"); +} + +void +jit::MarkShapeFromIon(JSRuntime *rt, Shape **shapep) +{ + gc::MarkShapeUnbarriered(&rt->gcMarker, shapep, "write barrier"); +} + +void +jit::ForbidCompilation(JSContext *cx, JSScript *script) +{ + ForbidCompilation(cx, script, SequentialExecution); +} + +void +jit::ForbidCompilation(JSContext *cx, JSScript *script, ExecutionMode mode) +{ + IonSpew(IonSpew_Abort, "Disabling Ion mode %d compilation of script %s:%d", + mode, script->filename(), script->lineno()); + + CancelOffThreadIonCompile(cx->compartment(), script); + + switch (mode) { + case SequentialExecution: + if (script->hasIonScript()) { + // It is only safe to modify script->ion if the script is not currently + // running, because JitFrameIterator needs to tell what ionScript to + // use (either the one on the JSScript, or the one hidden in the + // breadcrumbs Invalidation() leaves). Therefore, if invalidation + // fails, we cannot disable the script. + if (!Invalidate(cx, script, mode, false)) + return; + } + + script->setIonScript(ION_DISABLED_SCRIPT); + return; + + case ParallelExecution: + if (script->hasParallelIonScript()) { + if (!Invalidate(cx, script, mode, false)) + return; + } + + script->setParallelIonScript(ION_DISABLED_SCRIPT); + return; + + default: + MOZ_ASSUME_UNREACHABLE("No such execution mode"); + } + + MOZ_ASSUME_UNREACHABLE("No such execution mode"); +} + +AutoFlushICache * +PerThreadData::autoFlushICache() const +{ + return autoFlushICache_; +} + +void +PerThreadData::setAutoFlushICache(AutoFlushICache *afc) +{ + autoFlushICache_ = afc; +} + +// Set the range for the merging of flushes. The flushing is deferred until the end of +// the AutoFlushICache context. Subsequent flushing within this range will is also +// deferred. This is only expected to be defined once for each AutoFlushICache +// context. It assumes the range will be flushed is required to be within an +// AutoFlushICache context. +void +AutoFlushICache::setRange(uintptr_t start, size_t len) +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS) + AutoFlushICache *afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache(); + JS_ASSERT(afc); + JS_ASSERT(!afc->start_); + IonSpewCont(IonSpew_CacheFlush, "(%x %x):", start, len); + + uintptr_t stop = start + len; + afc->start_ = start; + afc->stop_ = stop; +#endif +} + +// Flush the instruction cache. +// +// If called within a dynamic AutoFlushICache context and if the range is already pending +// flushing for this AutoFlushICache context then the request is ignored with the +// understanding that it will be flushed on exit from the AutoFlushICache context. +// Otherwise the range is flushed immediately. +// +// Updates outside the current code object are typically the exception so they are flushed +// immediately rather than attempting to merge them. +// +// For efficiency it is expected that all large ranges will be flushed within an +// AutoFlushICache, so check. If this assertion is hit then it does not necessarily +// indicate a progam fault but it might indicate a lost opportunity to merge cache +// flushing. It can be corrected by wrapping the call in an AutoFlushICache to context. +void +AutoFlushICache::flush(uintptr_t start, size_t len) +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS) + AutoFlushICache *afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache(); + if (!afc) { + IonSpewCont(IonSpew_CacheFlush, "#"); + JSC::ExecutableAllocator::cacheFlush((void*)start, len); + JS_ASSERT(len <= 16); + return; + } + + uintptr_t stop = start + len; + if (start >= afc->start_ && stop <= afc->stop_) { + // Update is within the pending flush range, so defer to the end of the context. + IonSpewCont(IonSpew_CacheFlush, afc->inhibit_ ? "-" : "="); + return; + } + + IonSpewCont(IonSpew_CacheFlush, afc->inhibit_ ? "x" : "*"); + JSC::ExecutableAllocator::cacheFlush((void *)start, len); +#endif +} + +// Flag the current dynamic AutoFlushICache as inhibiting flushing. Useful in error paths +// where the changes are being abandoned. +void +AutoFlushICache::setInhibit() +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS) + AutoFlushICache *afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache(); + JS_ASSERT(afc); + JS_ASSERT(afc->start_); + IonSpewCont(IonSpew_CacheFlush, "I"); + afc->inhibit_ = true; +#endif +} + +// The common use case is merging cache flushes when preparing a code object. In this +// case the entire range of the code object is being flushed and as the code is patched +// smaller redundant flushes could occur. The design allows an AutoFlushICache dynamic +// thread local context to be declared in which the range of the code object can be set +// which defers flushing until the end of this dynamic context. The redundant flushing +// within this code range is also deferred avoiding redundant flushing. Flushing outside +// this code range is not affected and proceeds immediately. +// +// In some cases flushing is not necessary, such as when compiling an asm.js module which +// is flushed again when dynamically linked, and also in error paths that abandon the +// code. Flushing within the set code range can be inhibited within the AutoFlushICache +// dynamic context by setting an inhibit flag. +// +// The JS compiler can be re-entered while within an AutoFlushICache dynamic context and +// it is assumed that code being assembled or patched is not executed before the exit of +// the respective AutoFlushICache dynamic context. +// +AutoFlushICache::AutoFlushICache(const char *nonce, bool inhibit) + : start_(0), + stop_(0), + name_(nonce), + inhibit_(inhibit) +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS) + PerThreadData *pt = TlsPerThreadData.get(); + AutoFlushICache *afc = pt->PerThreadData::autoFlushICache(); + if (afc) + IonSpew(IonSpew_CacheFlush, "<%s,%s%s ", nonce, afc->name_, inhibit ? " I" : ""); + else + IonSpewCont(IonSpew_CacheFlush, "<%s%s ", nonce, inhibit ? " I" : ""); + + prev_ = afc; + pt->PerThreadData::setAutoFlushICache(this); +#endif +} + +AutoFlushICache::~AutoFlushICache() +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS) + PerThreadData *pt = TlsPerThreadData.get(); + JS_ASSERT(pt->PerThreadData::autoFlushICache() == this); + + if (!inhibit_ && start_) + JSC::ExecutableAllocator::cacheFlush((void *)start_, size_t(stop_ - start_)); + + IonSpewCont(IonSpew_CacheFlush, "%s%s>", name_, start_ ? "" : " U"); + IonSpewFin(IonSpew_CacheFlush); + pt->PerThreadData::setAutoFlushICache(prev_); +#endif +} + +void +jit::PurgeCaches(JSScript *script) +{ + if (script->hasIonScript()) + script->ionScript()->purgeCaches(); + + if (script->hasParallelIonScript()) + script->parallelIonScript()->purgeCaches(); +} + +size_t +jit::SizeOfIonData(JSScript *script, mozilla::MallocSizeOf mallocSizeOf) +{ + size_t result = 0; + + if (script->hasIonScript()) + result += script->ionScript()->sizeOfIncludingThis(mallocSizeOf); + + if (script->hasParallelIonScript()) + result += script->parallelIonScript()->sizeOfIncludingThis(mallocSizeOf); + + return result; +} + +void +jit::DestroyIonScripts(FreeOp *fop, JSScript *script) +{ + if (script->hasIonScript()) + jit::IonScript::Destroy(fop, script->ionScript()); + + if (script->hasParallelIonScript()) + jit::IonScript::Destroy(fop, script->parallelIonScript()); + + if (script->hasBaselineScript()) + jit::BaselineScript::Destroy(fop, script->baselineScript()); +} + +void +jit::TraceIonScripts(JSTracer* trc, JSScript *script) +{ + if (script->hasIonScript()) + jit::IonScript::Trace(trc, script->ionScript()); + + if (script->hasParallelIonScript()) + jit::IonScript::Trace(trc, script->parallelIonScript()); + + if (script->hasBaselineScript()) + jit::BaselineScript::Trace(trc, script->baselineScript()); +} + +bool +jit::RematerializeAllFrames(JSContext *cx, JSCompartment *comp) +{ + for (JitActivationIterator iter(comp->runtimeFromMainThread()); !iter.done(); ++iter) { + if (iter.activation()->compartment() == comp) { + for (JitFrameIterator frameIter(iter); !frameIter.done(); ++frameIter) { + if (!frameIter.isIonJS()) + continue; + if (!iter.activation()->asJit()->getRematerializedFrame(cx, frameIter)) + return false; + } + } + } + return true; +} + +bool +jit::UpdateForDebugMode(JSContext *maybecx, JSCompartment *comp, + AutoDebugModeInvalidation &invalidate) +{ + MOZ_ASSERT(invalidate.isFor(comp)); + + // Schedule invalidation of all optimized JIT code since debug mode + // invalidates assumptions. + invalidate.scheduleInvalidation(comp->debugMode()); + + // Recompile on-stack baseline scripts if we have a cx. + if (maybecx) { + IonContext ictx(maybecx, nullptr); + if (!RecompileOnStackBaselineScriptsForDebugMode(maybecx, comp)) { + js_ReportOutOfMemory(maybecx); + return false; + } + } + + return true; +} + +AutoDebugModeInvalidation::~AutoDebugModeInvalidation() +{ + MOZ_ASSERT(!!comp_ != !!zone_); + + if (needInvalidation_ == NoNeed) + return; + + Zone *zone = zone_ ? zone_ : comp_->zone(); + JSRuntime *rt = zone->runtimeFromMainThread(); + FreeOp *fop = rt->defaultFreeOp(); + + if (comp_) { + StopAllOffThreadCompilations(comp_); + } else { + for (CompartmentsInZoneIter comp(zone_); !comp.done(); comp.next()) + StopAllOffThreadCompilations(comp); + } + + // Don't discard active baseline scripts. They are recompiled for debug + // mode. + jit::MarkActiveBaselineScripts(zone); + + for (JitActivationIterator iter(rt); !iter.done(); ++iter) { + JSCompartment *comp = iter->compartment(); + if (comp_ == comp || zone_ == comp->zone()) { + IonContext ictx(CompileRuntime::get(rt)); + IonSpew(IonSpew_Invalidate, "Invalidating frames for debug mode toggle"); + InvalidateActivation(fop, iter.jitTop(), true); + } + } + + for (gc::CellIter i(zone, gc::FINALIZE_SCRIPT); !i.done(); i.next()) { + JSScript *script = i.get(); + if (script->compartment() == comp_ || zone_) { + FinishInvalidation(fop, script); + FinishInvalidation(fop, script); + FinishDiscardBaselineScript(fop, script); + script->resetUseCount(); + } else if (script->hasBaselineScript()) { + script->baselineScript()->resetActive(); + } + } +}