michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- michael@0: * vim: set ts=8 sts=4 et sw=4 tw=99: michael@0: * This Source Code Form is subject to the terms of the Mozilla Public michael@0: * License, v. 2.0. If a copy of the MPL was not distributed with this michael@0: * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ michael@0: michael@0: #ifndef jsgcinlines_h michael@0: #define jsgcinlines_h michael@0: michael@0: #include "jsgc.h" michael@0: michael@0: #include "gc/Zone.h" michael@0: michael@0: namespace js { michael@0: michael@0: class Shape; michael@0: michael@0: /* michael@0: * This auto class should be used around any code that might cause a mark bit to michael@0: * be set on an object in a dead zone. See AutoMaybeTouchDeadZones michael@0: * for more details. michael@0: */ michael@0: struct AutoMarkInDeadZone michael@0: { michael@0: AutoMarkInDeadZone(JS::Zone *zone) michael@0: : zone(zone), michael@0: scheduled(zone->scheduledForDestruction) michael@0: { michael@0: JSRuntime *rt = zone->runtimeFromMainThread(); michael@0: if (rt->gcManipulatingDeadZones && zone->scheduledForDestruction) { michael@0: rt->gcObjectsMarkedInDeadZones++; michael@0: zone->scheduledForDestruction = false; michael@0: } michael@0: } michael@0: michael@0: ~AutoMarkInDeadZone() { michael@0: zone->scheduledForDestruction = scheduled; michael@0: } michael@0: michael@0: private: michael@0: JS::Zone *zone; michael@0: bool scheduled; michael@0: }; michael@0: michael@0: inline Allocator *const michael@0: ThreadSafeContext::allocator() michael@0: { michael@0: JS_ASSERT_IF(isJSContext(), &asJSContext()->zone()->allocator == allocator_); michael@0: return allocator_; michael@0: } michael@0: michael@0: template michael@0: inline bool michael@0: ThreadSafeContext::isThreadLocal(T thing) const michael@0: { michael@0: if (!isForkJoinContext()) michael@0: return true; michael@0: michael@0: if (!IsInsideNursery(runtime_, thing) && michael@0: allocator_->arenas.containsArena(runtime_, thing->arenaHeader())) michael@0: { michael@0: // GC should be suppressed in preparation for mutating thread local michael@0: // objects, as we don't want to trip any barriers. michael@0: JS_ASSERT(!thing->zoneFromAnyThread()->needsBarrier()); michael@0: JS_ASSERT(!thing->runtimeFromAnyThread()->needsBarrier()); michael@0: michael@0: return true; michael@0: } michael@0: michael@0: return false; michael@0: } michael@0: michael@0: namespace gc { michael@0: michael@0: static inline AllocKind michael@0: GetGCObjectKind(const Class *clasp) michael@0: { michael@0: if (clasp == FunctionClassPtr) michael@0: return JSFunction::FinalizeKind; michael@0: uint32_t nslots = JSCLASS_RESERVED_SLOTS(clasp); michael@0: if (clasp->flags & JSCLASS_HAS_PRIVATE) michael@0: nslots++; michael@0: return GetGCObjectKind(nslots); michael@0: } michael@0: michael@0: #ifdef JSGC_GENERATIONAL michael@0: inline bool michael@0: ShouldNurseryAllocate(const Nursery &nursery, AllocKind kind, InitialHeap heap) michael@0: { michael@0: return nursery.isEnabled() && IsNurseryAllocable(kind) && heap != TenuredHeap; michael@0: } michael@0: #endif michael@0: michael@0: inline JSGCTraceKind michael@0: GetGCThingTraceKind(const void *thing) michael@0: { michael@0: JS_ASSERT(thing); michael@0: const Cell *cell = static_cast(thing); michael@0: #ifdef JSGC_GENERATIONAL michael@0: if (IsInsideNursery(cell->runtimeFromAnyThread(), cell)) michael@0: return JSTRACE_OBJECT; michael@0: #endif michael@0: return MapAllocToTraceKind(cell->tenuredGetAllocKind()); michael@0: } michael@0: michael@0: static inline void michael@0: GCPoke(JSRuntime *rt) michael@0: { michael@0: rt->gcPoke = true; michael@0: michael@0: #ifdef JS_GC_ZEAL michael@0: /* Schedule a GC to happen "soon" after a GC poke. */ michael@0: if (rt->gcZeal() == js::gc::ZealPokeValue) michael@0: rt->gcNextScheduled = 1; michael@0: #endif michael@0: } michael@0: michael@0: class ArenaIter michael@0: { michael@0: ArenaHeader *aheader; michael@0: ArenaHeader *remainingHeader; michael@0: michael@0: public: michael@0: ArenaIter() { michael@0: init(); michael@0: } michael@0: michael@0: ArenaIter(JS::Zone *zone, AllocKind kind) { michael@0: init(zone, kind); michael@0: } michael@0: michael@0: void init() { michael@0: aheader = nullptr; michael@0: remainingHeader = nullptr; michael@0: } michael@0: michael@0: void init(ArenaHeader *aheaderArg) { michael@0: aheader = aheaderArg; michael@0: remainingHeader = nullptr; michael@0: } michael@0: michael@0: void init(JS::Zone *zone, AllocKind kind) { michael@0: aheader = zone->allocator.arenas.getFirstArena(kind); michael@0: remainingHeader = zone->allocator.arenas.getFirstArenaToSweep(kind); michael@0: if (!aheader) { michael@0: aheader = remainingHeader; michael@0: remainingHeader = nullptr; michael@0: } michael@0: } michael@0: michael@0: bool done() { michael@0: return !aheader; michael@0: } michael@0: michael@0: ArenaHeader *get() { michael@0: return aheader; michael@0: } michael@0: michael@0: void next() { michael@0: aheader = aheader->next; michael@0: if (!aheader) { michael@0: aheader = remainingHeader; michael@0: remainingHeader = nullptr; michael@0: } michael@0: } michael@0: }; michael@0: michael@0: class CellIterImpl michael@0: { michael@0: size_t firstThingOffset; michael@0: size_t thingSize; michael@0: ArenaIter aiter; michael@0: FreeSpan firstSpan; michael@0: const FreeSpan *span; michael@0: uintptr_t thing; michael@0: Cell *cell; michael@0: michael@0: protected: michael@0: CellIterImpl() { michael@0: } michael@0: michael@0: void initSpan(JS::Zone *zone, AllocKind kind) { michael@0: JS_ASSERT(zone->allocator.arenas.isSynchronizedFreeList(kind)); michael@0: firstThingOffset = Arena::firstThingOffset(kind); michael@0: thingSize = Arena::thingSize(kind); michael@0: firstSpan.initAsEmpty(); michael@0: span = &firstSpan; michael@0: thing = span->first; michael@0: } michael@0: michael@0: void init(ArenaHeader *singleAheader) { michael@0: initSpan(singleAheader->zone, singleAheader->getAllocKind()); michael@0: aiter.init(singleAheader); michael@0: next(); michael@0: aiter.init(); michael@0: } michael@0: michael@0: void init(JS::Zone *zone, AllocKind kind) { michael@0: initSpan(zone, kind); michael@0: aiter.init(zone, kind); michael@0: next(); michael@0: } michael@0: michael@0: public: michael@0: bool done() const { michael@0: return !cell; michael@0: } michael@0: michael@0: template T *get() const { michael@0: JS_ASSERT(!done()); michael@0: return static_cast(cell); michael@0: } michael@0: michael@0: Cell *getCell() const { michael@0: JS_ASSERT(!done()); michael@0: return cell; michael@0: } michael@0: michael@0: void next() { michael@0: for (;;) { michael@0: if (thing != span->first) michael@0: break; michael@0: if (MOZ_LIKELY(span->hasNext())) { michael@0: thing = span->last + thingSize; michael@0: span = span->nextSpan(); michael@0: break; michael@0: } michael@0: if (aiter.done()) { michael@0: cell = nullptr; michael@0: return; michael@0: } michael@0: ArenaHeader *aheader = aiter.get(); michael@0: firstSpan = aheader->getFirstFreeSpan(); michael@0: span = &firstSpan; michael@0: thing = aheader->arenaAddress() | firstThingOffset; michael@0: aiter.next(); michael@0: } michael@0: cell = reinterpret_cast(thing); michael@0: thing += thingSize; michael@0: } michael@0: }; michael@0: michael@0: class CellIterUnderGC : public CellIterImpl michael@0: { michael@0: public: michael@0: CellIterUnderGC(JS::Zone *zone, AllocKind kind) { michael@0: #ifdef JSGC_GENERATIONAL michael@0: JS_ASSERT(zone->runtimeFromAnyThread()->gcNursery.isEmpty()); michael@0: #endif michael@0: JS_ASSERT(zone->runtimeFromAnyThread()->isHeapBusy()); michael@0: init(zone, kind); michael@0: } michael@0: michael@0: CellIterUnderGC(ArenaHeader *aheader) { michael@0: JS_ASSERT(aheader->zone->runtimeFromAnyThread()->isHeapBusy()); michael@0: init(aheader); michael@0: } michael@0: }; michael@0: michael@0: class CellIter : public CellIterImpl michael@0: { michael@0: ArenaLists *lists; michael@0: AllocKind kind; michael@0: #ifdef DEBUG michael@0: size_t *counter; michael@0: #endif michael@0: public: michael@0: CellIter(JS::Zone *zone, AllocKind kind) michael@0: : lists(&zone->allocator.arenas), michael@0: kind(kind) michael@0: { michael@0: /* michael@0: * We have a single-threaded runtime, so there's no need to protect michael@0: * against other threads iterating or allocating. However, we do have michael@0: * background finalization; we have to wait for this to finish if it's michael@0: * currently active. michael@0: */ michael@0: if (IsBackgroundFinalized(kind) && michael@0: zone->allocator.arenas.needBackgroundFinalizeWait(kind)) michael@0: { michael@0: gc::FinishBackgroundFinalize(zone->runtimeFromMainThread()); michael@0: } michael@0: michael@0: #ifdef JSGC_GENERATIONAL michael@0: /* Evict the nursery before iterating so we can see all things. */ michael@0: JSRuntime *rt = zone->runtimeFromMainThread(); michael@0: if (!rt->gcNursery.isEmpty()) michael@0: MinorGC(rt, JS::gcreason::EVICT_NURSERY); michael@0: #endif michael@0: michael@0: if (lists->isSynchronizedFreeList(kind)) { michael@0: lists = nullptr; michael@0: } else { michael@0: JS_ASSERT(!zone->runtimeFromMainThread()->isHeapBusy()); michael@0: lists->copyFreeListToArena(kind); michael@0: } michael@0: michael@0: #ifdef DEBUG michael@0: /* Assert that no GCs can occur while a CellIter is live. */ michael@0: counter = &zone->runtimeFromAnyThread()->noGCOrAllocationCheck; michael@0: ++*counter; michael@0: #endif michael@0: michael@0: init(zone, kind); michael@0: } michael@0: michael@0: ~CellIter() { michael@0: #ifdef DEBUG michael@0: JS_ASSERT(*counter > 0); michael@0: --*counter; michael@0: #endif michael@0: if (lists) michael@0: lists->clearFreeListInArena(kind); michael@0: } michael@0: }; michael@0: michael@0: class GCZonesIter michael@0: { michael@0: private: michael@0: ZonesIter zone; michael@0: michael@0: public: michael@0: GCZonesIter(JSRuntime *rt) : zone(rt, WithAtoms) { michael@0: if (!zone->isCollecting()) michael@0: next(); michael@0: } michael@0: michael@0: bool done() const { return zone.done(); } michael@0: michael@0: void next() { michael@0: JS_ASSERT(!done()); michael@0: do { michael@0: zone.next(); michael@0: } while (!zone.done() && !zone->isCollecting()); michael@0: } michael@0: michael@0: JS::Zone *get() const { michael@0: JS_ASSERT(!done()); michael@0: return zone; michael@0: } michael@0: michael@0: operator JS::Zone *() const { return get(); } michael@0: JS::Zone *operator->() const { return get(); } michael@0: }; michael@0: michael@0: typedef CompartmentsIterT GCCompartmentsIter; michael@0: michael@0: /* Iterates over all zones in the current zone group. */ michael@0: class GCZoneGroupIter { michael@0: private: michael@0: JS::Zone *current; michael@0: michael@0: public: michael@0: GCZoneGroupIter(JSRuntime *rt) { michael@0: JS_ASSERT(rt->isHeapBusy()); michael@0: current = rt->gcCurrentZoneGroup; michael@0: } michael@0: michael@0: bool done() const { return !current; } michael@0: michael@0: void next() { michael@0: JS_ASSERT(!done()); michael@0: current = current->nextNodeInGroup(); michael@0: } michael@0: michael@0: JS::Zone *get() const { michael@0: JS_ASSERT(!done()); michael@0: return current; michael@0: } michael@0: michael@0: operator JS::Zone *() const { return get(); } michael@0: JS::Zone *operator->() const { return get(); } michael@0: }; michael@0: michael@0: typedef CompartmentsIterT GCCompartmentGroupIter; michael@0: michael@0: #ifdef JSGC_GENERATIONAL michael@0: /* michael@0: * Attempt to allocate a new GC thing out of the nursery. If there is not enough michael@0: * room in the nursery or there is an OOM, this method will return nullptr. michael@0: */ michael@0: template michael@0: inline JSObject * michael@0: TryNewNurseryObject(ThreadSafeContext *cxArg, size_t thingSize, size_t nDynamicSlots) michael@0: { michael@0: JSContext *cx = cxArg->asJSContext(); michael@0: michael@0: JS_ASSERT(!IsAtomsCompartment(cx->compartment())); michael@0: JSRuntime *rt = cx->runtime(); michael@0: Nursery &nursery = rt->gcNursery; michael@0: JSObject *obj = nursery.allocateObject(cx, thingSize, nDynamicSlots); michael@0: if (obj) michael@0: return obj; michael@0: if (allowGC && !rt->mainThread.suppressGC) { michael@0: MinorGC(cx, JS::gcreason::OUT_OF_NURSERY); michael@0: michael@0: /* Exceeding gcMaxBytes while tenuring can disable the Nursery. */ michael@0: if (nursery.isEnabled()) { michael@0: JSObject *obj = nursery.allocateObject(cx, thingSize, nDynamicSlots); michael@0: JS_ASSERT(obj); michael@0: return obj; michael@0: } michael@0: } michael@0: return nullptr; michael@0: } michael@0: #endif /* JSGC_GENERATIONAL */ michael@0: michael@0: static inline bool michael@0: PossiblyFail() michael@0: { michael@0: JS_OOM_POSSIBLY_FAIL(); michael@0: return true; michael@0: } michael@0: michael@0: template michael@0: static inline bool michael@0: CheckAllocatorState(ThreadSafeContext *cx, AllocKind kind) michael@0: { michael@0: if (!cx->isJSContext()) michael@0: return true; michael@0: michael@0: JSContext *ncx = cx->asJSContext(); michael@0: JSRuntime *rt = ncx->runtime(); michael@0: #if defined(JS_GC_ZEAL) || defined(DEBUG) michael@0: JS_ASSERT_IF(rt->isAtomsCompartment(ncx->compartment()), michael@0: kind == FINALIZE_STRING || michael@0: kind == FINALIZE_FAT_INLINE_STRING || michael@0: kind == FINALIZE_JITCODE); michael@0: JS_ASSERT(!rt->isHeapBusy()); michael@0: JS_ASSERT(!rt->noGCOrAllocationCheck); michael@0: #endif michael@0: michael@0: // For testing out of memory conditions michael@0: if (!PossiblyFail()) { michael@0: js_ReportOutOfMemory(cx); michael@0: return false; michael@0: } michael@0: michael@0: if (allowGC) { michael@0: #ifdef JS_GC_ZEAL michael@0: if (rt->needZealousGC()) michael@0: js::gc::RunDebugGC(ncx); michael@0: #endif michael@0: michael@0: if (rt->interrupt) { michael@0: // Invoking the interrupt callback can fail and we can't usefully michael@0: // handle that here. Just check in case we need to collect instead. michael@0: js::gc::GCIfNeeded(ncx); michael@0: } michael@0: } michael@0: michael@0: return true; michael@0: } michael@0: michael@0: template michael@0: static inline void michael@0: CheckIncrementalZoneState(ThreadSafeContext *cx, T *t) michael@0: { michael@0: #ifdef DEBUG michael@0: if (!cx->isJSContext()) michael@0: return; michael@0: michael@0: Zone *zone = cx->asJSContext()->zone(); michael@0: JS_ASSERT_IF(t && zone->wasGCStarted() && (zone->isGCMarking() || zone->isGCSweeping()), michael@0: t->arenaHeader()->allocatedDuringIncremental); michael@0: #endif michael@0: } michael@0: michael@0: /* michael@0: * Allocate a new GC thing. After a successful allocation the caller must michael@0: * fully initialize the thing before calling any function that can potentially michael@0: * trigger GC. This will ensure that GC tracing never sees junk values stored michael@0: * in the partially initialized thing. michael@0: */ michael@0: michael@0: template michael@0: inline JSObject * michael@0: AllocateObject(ThreadSafeContext *cx, AllocKind kind, size_t nDynamicSlots, InitialHeap heap) michael@0: { michael@0: size_t thingSize = Arena::thingSize(kind); michael@0: michael@0: JS_ASSERT(thingSize == Arena::thingSize(kind)); michael@0: if (!CheckAllocatorState(cx, kind)) michael@0: return nullptr; michael@0: michael@0: #ifdef JSGC_GENERATIONAL michael@0: if (cx->hasNursery() && ShouldNurseryAllocate(cx->nursery(), kind, heap)) { michael@0: JSObject *obj = TryNewNurseryObject(cx, thingSize, nDynamicSlots); michael@0: if (obj) michael@0: return obj; michael@0: } michael@0: #endif michael@0: michael@0: HeapSlot *slots = nullptr; michael@0: if (nDynamicSlots) { michael@0: slots = cx->pod_malloc(nDynamicSlots); michael@0: if (MOZ_UNLIKELY(!slots)) michael@0: return nullptr; michael@0: js::Debug_SetSlotRangeToCrashOnTouch(slots, nDynamicSlots); michael@0: } michael@0: michael@0: JSObject *obj = static_cast(cx->allocator()->arenas.allocateFromFreeList(kind, thingSize)); michael@0: if (!obj) michael@0: obj = static_cast(js::gc::ArenaLists::refillFreeList(cx, kind)); michael@0: michael@0: if (obj) michael@0: obj->setInitialSlots(slots); michael@0: else michael@0: js_free(slots); michael@0: michael@0: CheckIncrementalZoneState(cx, obj); michael@0: return obj; michael@0: } michael@0: michael@0: template michael@0: inline T * michael@0: AllocateNonObject(ThreadSafeContext *cx) michael@0: { michael@0: AllocKind kind = MapTypeToFinalizeKind::kind; michael@0: size_t thingSize = sizeof(T); michael@0: michael@0: JS_ASSERT(thingSize == Arena::thingSize(kind)); michael@0: if (!CheckAllocatorState(cx, kind)) michael@0: return nullptr; michael@0: michael@0: T *t = static_cast(cx->allocator()->arenas.allocateFromFreeList(kind, thingSize)); michael@0: if (!t) michael@0: t = static_cast(js::gc::ArenaLists::refillFreeList(cx, kind)); michael@0: michael@0: CheckIncrementalZoneState(cx, t); michael@0: return t; michael@0: } michael@0: michael@0: /* michael@0: * When allocating for initialization from a cached object copy, we will michael@0: * potentially destroy the cache entry we want to copy if we allow GC. On the michael@0: * other hand, since these allocations are extremely common, we don't want to michael@0: * delay GC from these allocation sites. Instead we allow the GC, but still michael@0: * fail the allocation, forcing the non-cached path. michael@0: */ michael@0: template michael@0: inline JSObject * michael@0: AllocateObjectForCacheHit(JSContext *cx, AllocKind kind, InitialHeap heap) michael@0: { michael@0: #ifdef JSGC_GENERATIONAL michael@0: if (ShouldNurseryAllocate(cx->nursery(), kind, heap)) { michael@0: size_t thingSize = Arena::thingSize(kind); michael@0: michael@0: JS_ASSERT(thingSize == Arena::thingSize(kind)); michael@0: if (!CheckAllocatorState(cx, kind)) michael@0: return nullptr; michael@0: michael@0: JSObject *obj = TryNewNurseryObject(cx, thingSize, 0); michael@0: if (!obj && allowGC) { michael@0: MinorGC(cx, JS::gcreason::OUT_OF_NURSERY); michael@0: return nullptr; michael@0: } michael@0: return obj; michael@0: } michael@0: #endif michael@0: michael@0: JSObject *obj = AllocateObject(cx, kind, 0, heap); michael@0: if (!obj && allowGC) { michael@0: MaybeGC(cx); michael@0: return nullptr; michael@0: } michael@0: michael@0: return obj; michael@0: } michael@0: michael@0: } /* namespace gc */ michael@0: michael@0: template michael@0: inline JSObject * michael@0: NewGCObject(js::ThreadSafeContext *cx, js::gc::AllocKind kind, size_t nDynamicSlots, js::gc::InitialHeap heap) michael@0: { michael@0: JS_ASSERT(kind >= js::gc::FINALIZE_OBJECT0 && kind <= js::gc::FINALIZE_OBJECT_LAST); michael@0: return js::gc::AllocateObject(cx, kind, nDynamicSlots, heap); michael@0: } michael@0: michael@0: template michael@0: inline jit::JitCode * michael@0: NewJitCode(js::ThreadSafeContext *cx) michael@0: { michael@0: return gc::AllocateNonObject(cx); michael@0: } michael@0: michael@0: inline michael@0: types::TypeObject * michael@0: NewTypeObject(js::ThreadSafeContext *cx) michael@0: { michael@0: return gc::AllocateNonObject(cx); michael@0: } michael@0: michael@0: } /* namespace js */ michael@0: michael@0: template michael@0: inline JSString * michael@0: js_NewGCString(js::ThreadSafeContext *cx) michael@0: { michael@0: return js::gc::AllocateNonObject(cx); michael@0: } michael@0: michael@0: template michael@0: inline JSFatInlineString * michael@0: js_NewGCFatInlineString(js::ThreadSafeContext *cx) michael@0: { michael@0: return js::gc::AllocateNonObject(cx); michael@0: } michael@0: michael@0: inline JSExternalString * michael@0: js_NewGCExternalString(js::ThreadSafeContext *cx) michael@0: { michael@0: return js::gc::AllocateNonObject(cx); michael@0: } michael@0: michael@0: inline JSScript * michael@0: js_NewGCScript(js::ThreadSafeContext *cx) michael@0: { michael@0: return js::gc::AllocateNonObject(cx); michael@0: } michael@0: michael@0: inline js::LazyScript * michael@0: js_NewGCLazyScript(js::ThreadSafeContext *cx) michael@0: { michael@0: return js::gc::AllocateNonObject(cx); michael@0: } michael@0: michael@0: inline js::Shape * michael@0: js_NewGCShape(js::ThreadSafeContext *cx) michael@0: { michael@0: return js::gc::AllocateNonObject(cx); michael@0: } michael@0: michael@0: template michael@0: inline js::BaseShape * michael@0: js_NewGCBaseShape(js::ThreadSafeContext *cx) michael@0: { michael@0: return js::gc::AllocateNonObject(cx); michael@0: } michael@0: michael@0: #endif /* jsgcinlines_h */