1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/gc/Nursery.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,950 @@
1.4 +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
1.5 + * vim: set ts=8 sw=4 et tw=78:
1.6 + *
1.7 + * This Source Code Form is subject to the terms of the Mozilla Public
1.8 + * License, v. 2.0. If a copy of the MPL was not distributed with this file,
1.9 + * You can obtain one at http://mozilla.org/MPL/2.0/. */
1.10 +
1.11 +#ifdef JSGC_GENERATIONAL
1.12 +
1.13 +#include "gc/Nursery-inl.h"
1.14 +
1.15 +#include "jscompartment.h"
1.16 +#include "jsgc.h"
1.17 +#include "jsinfer.h"
1.18 +#include "jsutil.h"
1.19 +#include "prmjtime.h"
1.20 +
1.21 +#include "gc/GCInternals.h"
1.22 +#include "gc/Memory.h"
1.23 +#ifdef JS_ION
1.24 +#include "jit/IonFrames.h"
1.25 +#endif
1.26 +#include "mozilla/IntegerPrintfMacros.h"
1.27 +#include "vm/ArrayObject.h"
1.28 +#include "vm/Debugger.h"
1.29 +#if defined(DEBUG)
1.30 +#include "vm/ScopeObject.h"
1.31 +#endif
1.32 +#include "vm/TypedArrayObject.h"
1.33 +
1.34 +#include "jsgcinlines.h"
1.35 +
1.36 +#include "vm/ObjectImpl-inl.h"
1.37 +
1.38 +using namespace js;
1.39 +using namespace gc;
1.40 +using namespace mozilla;
1.41 +
1.42 +//#define PROFILE_NURSERY
1.43 +
1.44 +#ifdef PROFILE_NURSERY
1.45 +/*
1.46 + * Print timing information for minor GCs that take longer than this time in microseconds.
1.47 + */
1.48 +static int64_t GCReportThreshold = INT64_MAX;
1.49 +#endif
1.50 +
1.51 +bool
1.52 +js::Nursery::init()
1.53 +{
1.54 + JS_ASSERT(start() == 0);
1.55 +
1.56 + if (!hugeSlots.init())
1.57 + return false;
1.58 +
1.59 + void *heap = MapAlignedPages(runtime(), NurserySize, Alignment);
1.60 + if (!heap)
1.61 + return false;
1.62 +
1.63 + JSRuntime *rt = runtime();
1.64 + rt->gcNurseryStart_ = uintptr_t(heap);
1.65 + currentStart_ = start();
1.66 + rt->gcNurseryEnd_ = chunk(LastNurseryChunk).end();
1.67 + numActiveChunks_ = 1;
1.68 + JS_POISON(heap, JS_FRESH_NURSERY_PATTERN, NurserySize);
1.69 + setCurrentChunk(0);
1.70 + updateDecommittedRegion();
1.71 +
1.72 +#ifdef PROFILE_NURSERY
1.73 + char *env = getenv("JS_MINORGC_TIME");
1.74 + if (env)
1.75 + GCReportThreshold = atoi(env);
1.76 +#endif
1.77 +
1.78 + JS_ASSERT(isEnabled());
1.79 + return true;
1.80 +}
1.81 +
1.82 +js::Nursery::~Nursery()
1.83 +{
1.84 + if (start())
1.85 + UnmapPages(runtime(), (void *)start(), NurserySize);
1.86 +}
1.87 +
1.88 +void
1.89 +js::Nursery::enable()
1.90 +{
1.91 + JS_ASSERT(isEmpty());
1.92 + if (isEnabled())
1.93 + return;
1.94 + numActiveChunks_ = 1;
1.95 + setCurrentChunk(0);
1.96 + currentStart_ = position();
1.97 +#ifdef JS_GC_ZEAL
1.98 + if (runtime()->gcZeal_ == ZealGenerationalGCValue)
1.99 + enterZealMode();
1.100 +#endif
1.101 +}
1.102 +
1.103 +void
1.104 +js::Nursery::disable()
1.105 +{
1.106 + JS_ASSERT(isEmpty());
1.107 + if (!isEnabled())
1.108 + return;
1.109 + numActiveChunks_ = 0;
1.110 + currentEnd_ = 0;
1.111 + updateDecommittedRegion();
1.112 +}
1.113 +
1.114 +bool
1.115 +js::Nursery::isEmpty() const
1.116 +{
1.117 + JS_ASSERT(runtime_);
1.118 + if (!isEnabled())
1.119 + return true;
1.120 + JS_ASSERT_IF(runtime_->gcZeal_ != ZealGenerationalGCValue, currentStart_ == start());
1.121 + return position() == currentStart_;
1.122 +}
1.123 +
1.124 +JSObject *
1.125 +js::Nursery::allocateObject(JSContext *cx, size_t size, size_t numDynamic)
1.126 +{
1.127 + /* Ensure there's enough space to replace the contents with a RelocationOverlay. */
1.128 + JS_ASSERT(size >= sizeof(RelocationOverlay));
1.129 +
1.130 + /* Attempt to allocate slots contiguously after object, if possible. */
1.131 + if (numDynamic && numDynamic <= MaxNurserySlots) {
1.132 + size_t totalSize = size + sizeof(HeapSlot) * numDynamic;
1.133 + JSObject *obj = static_cast<JSObject *>(allocate(totalSize));
1.134 + if (obj) {
1.135 + obj->setInitialSlots(reinterpret_cast<HeapSlot *>(size_t(obj) + size));
1.136 + return obj;
1.137 + }
1.138 + /* If we failed to allocate as a block, retry with out-of-line slots. */
1.139 + }
1.140 +
1.141 + HeapSlot *slots = nullptr;
1.142 + if (numDynamic) {
1.143 + slots = allocateHugeSlots(cx, numDynamic);
1.144 + if (MOZ_UNLIKELY(!slots))
1.145 + return nullptr;
1.146 + }
1.147 +
1.148 + JSObject *obj = static_cast<JSObject *>(allocate(size));
1.149 +
1.150 + if (obj)
1.151 + obj->setInitialSlots(slots);
1.152 + else
1.153 + freeSlots(cx, slots);
1.154 +
1.155 + return obj;
1.156 +}
1.157 +
1.158 +void *
1.159 +js::Nursery::allocate(size_t size)
1.160 +{
1.161 + JS_ASSERT(isEnabled());
1.162 + JS_ASSERT(!runtime()->isHeapBusy());
1.163 + JS_ASSERT(position() >= currentStart_);
1.164 +
1.165 + if (position() + size > currentEnd()) {
1.166 + if (currentChunk_ + 1 == numActiveChunks_)
1.167 + return nullptr;
1.168 + setCurrentChunk(currentChunk_ + 1);
1.169 + }
1.170 +
1.171 + void *thing = (void *)position();
1.172 + position_ = position() + size;
1.173 +
1.174 + JS_EXTRA_POISON(thing, JS_ALLOCATED_NURSERY_PATTERN, size);
1.175 + return thing;
1.176 +}
1.177 +
1.178 +/* Internally, this function is used to allocate elements as well as slots. */
1.179 +HeapSlot *
1.180 +js::Nursery::allocateSlots(JSContext *cx, JSObject *obj, uint32_t nslots)
1.181 +{
1.182 + JS_ASSERT(obj);
1.183 + JS_ASSERT(nslots > 0);
1.184 +
1.185 + if (!isInside(obj))
1.186 + return cx->pod_malloc<HeapSlot>(nslots);
1.187 +
1.188 + if (nslots > MaxNurserySlots)
1.189 + return allocateHugeSlots(cx, nslots);
1.190 +
1.191 + size_t size = sizeof(HeapSlot) * nslots;
1.192 + HeapSlot *slots = static_cast<HeapSlot *>(allocate(size));
1.193 + if (slots)
1.194 + return slots;
1.195 +
1.196 + return allocateHugeSlots(cx, nslots);
1.197 +}
1.198 +
1.199 +ObjectElements *
1.200 +js::Nursery::allocateElements(JSContext *cx, JSObject *obj, uint32_t nelems)
1.201 +{
1.202 + JS_ASSERT(nelems >= ObjectElements::VALUES_PER_HEADER);
1.203 + return reinterpret_cast<ObjectElements *>(allocateSlots(cx, obj, nelems));
1.204 +}
1.205 +
1.206 +HeapSlot *
1.207 +js::Nursery::reallocateSlots(JSContext *cx, JSObject *obj, HeapSlot *oldSlots,
1.208 + uint32_t oldCount, uint32_t newCount)
1.209 +{
1.210 + size_t oldSize = oldCount * sizeof(HeapSlot);
1.211 + size_t newSize = newCount * sizeof(HeapSlot);
1.212 +
1.213 + if (!isInside(obj))
1.214 + return static_cast<HeapSlot *>(cx->realloc_(oldSlots, oldSize, newSize));
1.215 +
1.216 + if (!isInside(oldSlots)) {
1.217 + HeapSlot *newSlots = static_cast<HeapSlot *>(cx->realloc_(oldSlots, oldSize, newSize));
1.218 + if (oldSlots != newSlots) {
1.219 + hugeSlots.remove(oldSlots);
1.220 + /* If this put fails, we will only leak the slots. */
1.221 + (void)hugeSlots.put(newSlots);
1.222 + }
1.223 + return newSlots;
1.224 + }
1.225 +
1.226 + /* The nursery cannot make use of the returned slots data. */
1.227 + if (newCount < oldCount)
1.228 + return oldSlots;
1.229 +
1.230 + HeapSlot *newSlots = allocateSlots(cx, obj, newCount);
1.231 + PodCopy(newSlots, oldSlots, oldCount);
1.232 + return newSlots;
1.233 +}
1.234 +
1.235 +ObjectElements *
1.236 +js::Nursery::reallocateElements(JSContext *cx, JSObject *obj, ObjectElements *oldHeader,
1.237 + uint32_t oldCount, uint32_t newCount)
1.238 +{
1.239 + HeapSlot *slots = reallocateSlots(cx, obj, reinterpret_cast<HeapSlot *>(oldHeader),
1.240 + oldCount, newCount);
1.241 + return reinterpret_cast<ObjectElements *>(slots);
1.242 +}
1.243 +
1.244 +void
1.245 +js::Nursery::freeSlots(JSContext *cx, HeapSlot *slots)
1.246 +{
1.247 + if (!isInside(slots)) {
1.248 + hugeSlots.remove(slots);
1.249 + js_free(slots);
1.250 + }
1.251 +}
1.252 +
1.253 +HeapSlot *
1.254 +js::Nursery::allocateHugeSlots(JSContext *cx, size_t nslots)
1.255 +{
1.256 + HeapSlot *slots = cx->pod_malloc<HeapSlot>(nslots);
1.257 + /* If this put fails, we will only leak the slots. */
1.258 + (void)hugeSlots.put(slots);
1.259 + return slots;
1.260 +}
1.261 +
1.262 +void
1.263 +js::Nursery::notifyInitialSlots(Cell *cell, HeapSlot *slots)
1.264 +{
1.265 + if (isInside(cell) && !isInside(slots)) {
1.266 + /* If this put fails, we will only leak the slots. */
1.267 + (void)hugeSlots.put(slots);
1.268 + }
1.269 +}
1.270 +
1.271 +namespace js {
1.272 +namespace gc {
1.273 +
1.274 +class MinorCollectionTracer : public JSTracer
1.275 +{
1.276 + public:
1.277 + Nursery *nursery;
1.278 + AutoTraceSession session;
1.279 +
1.280 + /* Amount of data moved to the tenured generation during collection. */
1.281 + size_t tenuredSize;
1.282 +
1.283 + /*
1.284 + * This list is threaded through the Nursery using the space from already
1.285 + * moved things. The list is used to fix up the moved things and to find
1.286 + * things held live by intra-Nursery pointers.
1.287 + */
1.288 + RelocationOverlay *head;
1.289 + RelocationOverlay **tail;
1.290 +
1.291 + /* Save and restore all of the runtime state we use during MinorGC. */
1.292 + bool savedRuntimeNeedBarrier;
1.293 + AutoDisableProxyCheck disableStrictProxyChecking;
1.294 + AutoEnterOOMUnsafeRegion oomUnsafeRegion;
1.295 + ArrayBufferVector liveArrayBuffers;
1.296 +
1.297 + /* Insert the given relocation entry into the list of things to visit. */
1.298 + MOZ_ALWAYS_INLINE void insertIntoFixupList(RelocationOverlay *entry) {
1.299 + *tail = entry;
1.300 + tail = &entry->next_;
1.301 + *tail = nullptr;
1.302 + }
1.303 +
1.304 + MinorCollectionTracer(JSRuntime *rt, Nursery *nursery)
1.305 + : JSTracer(rt, Nursery::MinorGCCallback, TraceWeakMapKeysValues),
1.306 + nursery(nursery),
1.307 + session(rt, MinorCollecting),
1.308 + tenuredSize(0),
1.309 + head(nullptr),
1.310 + tail(&head),
1.311 + savedRuntimeNeedBarrier(rt->needsBarrier()),
1.312 + disableStrictProxyChecking(rt)
1.313 + {
1.314 + rt->gcNumber++;
1.315 +
1.316 + /*
1.317 + * We disable the runtime needsBarrier() check so that pre-barriers do
1.318 + * not fire on objects that have been relocated. The pre-barrier's
1.319 + * call to obj->zone() will try to look through shape_, which is now
1.320 + * the relocation magic and will crash. However, zone->needsBarrier()
1.321 + * must still be set correctly so that allocations we make in minor
1.322 + * GCs between incremental slices will allocate their objects marked.
1.323 + */
1.324 + rt->setNeedsBarrier(false);
1.325 +
1.326 + /*
1.327 + * We use the live array buffer lists to track traced buffers so we can
1.328 + * sweep their dead views. Incremental collection also use these lists,
1.329 + * so we may need to save and restore their contents here.
1.330 + */
1.331 + if (rt->gcIncrementalState != NO_INCREMENTAL) {
1.332 + for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
1.333 + if (!ArrayBufferObject::saveArrayBufferList(c, liveArrayBuffers))
1.334 + CrashAtUnhandlableOOM("OOM while saving live array buffers");
1.335 + ArrayBufferObject::resetArrayBufferList(c);
1.336 + }
1.337 + }
1.338 + }
1.339 +
1.340 + ~MinorCollectionTracer() {
1.341 + runtime()->setNeedsBarrier(savedRuntimeNeedBarrier);
1.342 + if (runtime()->gcIncrementalState != NO_INCREMENTAL)
1.343 + ArrayBufferObject::restoreArrayBufferLists(liveArrayBuffers);
1.344 + }
1.345 +};
1.346 +
1.347 +} /* namespace gc */
1.348 +} /* namespace js */
1.349 +
1.350 +static AllocKind
1.351 +GetObjectAllocKindForCopy(JSRuntime *rt, JSObject *obj)
1.352 +{
1.353 + if (obj->is<ArrayObject>()) {
1.354 + JS_ASSERT(obj->numFixedSlots() == 0);
1.355 +
1.356 + /* Use minimal size object if we are just going to copy the pointer. */
1.357 + if (!IsInsideNursery(rt, (void *)obj->getElementsHeader()))
1.358 + return FINALIZE_OBJECT0_BACKGROUND;
1.359 +
1.360 + size_t nelements = obj->getDenseCapacity();
1.361 + return GetBackgroundAllocKind(GetGCArrayKind(nelements));
1.362 + }
1.363 +
1.364 + if (obj->is<JSFunction>())
1.365 + return obj->as<JSFunction>().getAllocKind();
1.366 +
1.367 + /*
1.368 + * Typed arrays in the nursery may have a lazily allocated buffer, make
1.369 + * sure there is room for the array's fixed data when moving the array.
1.370 + */
1.371 + if (obj->is<TypedArrayObject>() && !obj->as<TypedArrayObject>().buffer()) {
1.372 + size_t nbytes = obj->as<TypedArrayObject>().byteLength();
1.373 + return GetBackgroundAllocKind(TypedArrayObject::AllocKindForLazyBuffer(nbytes));
1.374 + }
1.375 +
1.376 + AllocKind kind = GetGCObjectFixedSlotsKind(obj->numFixedSlots());
1.377 + JS_ASSERT(!IsBackgroundFinalized(kind));
1.378 + JS_ASSERT(CanBeFinalizedInBackground(kind, obj->getClass()));
1.379 + return GetBackgroundAllocKind(kind);
1.380 +}
1.381 +
1.382 +void *
1.383 +js::Nursery::allocateFromTenured(Zone *zone, AllocKind thingKind)
1.384 +{
1.385 + void *t = zone->allocator.arenas.allocateFromFreeList(thingKind, Arena::thingSize(thingKind));
1.386 + if (t)
1.387 + return t;
1.388 + zone->allocator.arenas.checkEmptyFreeList(thingKind);
1.389 + return zone->allocator.arenas.allocateFromArena(zone, thingKind);
1.390 +}
1.391 +
1.392 +void
1.393 +js::Nursery::setSlotsForwardingPointer(HeapSlot *oldSlots, HeapSlot *newSlots, uint32_t nslots)
1.394 +{
1.395 + JS_ASSERT(nslots > 0);
1.396 + JS_ASSERT(isInside(oldSlots));
1.397 + JS_ASSERT(!isInside(newSlots));
1.398 + *reinterpret_cast<HeapSlot **>(oldSlots) = newSlots;
1.399 +}
1.400 +
1.401 +void
1.402 +js::Nursery::setElementsForwardingPointer(ObjectElements *oldHeader, ObjectElements *newHeader,
1.403 + uint32_t nelems)
1.404 +{
1.405 + /*
1.406 + * If the JIT has hoisted a zero length pointer, then we do not need to
1.407 + * relocate it because reads and writes to/from this pointer are invalid.
1.408 + */
1.409 + if (nelems - ObjectElements::VALUES_PER_HEADER < 1)
1.410 + return;
1.411 + JS_ASSERT(isInside(oldHeader));
1.412 + JS_ASSERT(!isInside(newHeader));
1.413 + *reinterpret_cast<HeapSlot **>(oldHeader->elements()) = newHeader->elements();
1.414 +}
1.415 +
1.416 +#ifdef DEBUG
1.417 +static bool IsWriteableAddress(void *ptr)
1.418 +{
1.419 + volatile uint64_t *vPtr = reinterpret_cast<volatile uint64_t *>(ptr);
1.420 + *vPtr = *vPtr;
1.421 + return true;
1.422 +}
1.423 +#endif
1.424 +
1.425 +void
1.426 +js::Nursery::forwardBufferPointer(HeapSlot **pSlotsElems)
1.427 +{
1.428 + HeapSlot *old = *pSlotsElems;
1.429 +
1.430 + if (!isInside(old))
1.431 + return;
1.432 +
1.433 + /*
1.434 + * If the elements buffer is zero length, the "first" item could be inside
1.435 + * of the next object or past the end of the allocable area. However,
1.436 + * since we always store the runtime as the last word in the nursery,
1.437 + * isInside will still be true, even if this zero-size allocation abuts the
1.438 + * end of the allocable area. Thus, it is always safe to read the first
1.439 + * word of |old| here.
1.440 + */
1.441 + *pSlotsElems = *reinterpret_cast<HeapSlot **>(old);
1.442 + JS_ASSERT(!isInside(*pSlotsElems));
1.443 + JS_ASSERT(IsWriteableAddress(*pSlotsElems));
1.444 +}
1.445 +
1.446 +// Structure for counting how many times objects of a particular type have been
1.447 +// tenured during a minor collection.
1.448 +struct TenureCount
1.449 +{
1.450 + types::TypeObject *type;
1.451 + int count;
1.452 +};
1.453 +
1.454 +// Keep rough track of how many times we tenure objects of particular types
1.455 +// during minor collections, using a fixed size hash for efficiency at the cost
1.456 +// of potential collisions.
1.457 +struct Nursery::TenureCountCache
1.458 +{
1.459 + TenureCount entries[16];
1.460 +
1.461 + TenureCountCache() { PodZero(this); }
1.462 +
1.463 + TenureCount &findEntry(types::TypeObject *type) {
1.464 + return entries[PointerHasher<types::TypeObject *, 3>::hash(type) % ArrayLength(entries)];
1.465 + }
1.466 +};
1.467 +
1.468 +void
1.469 +js::Nursery::collectToFixedPoint(MinorCollectionTracer *trc, TenureCountCache &tenureCounts)
1.470 +{
1.471 + for (RelocationOverlay *p = trc->head; p; p = p->next()) {
1.472 + JSObject *obj = static_cast<JSObject*>(p->forwardingAddress());
1.473 + traceObject(trc, obj);
1.474 +
1.475 + TenureCount &entry = tenureCounts.findEntry(obj->type());
1.476 + if (entry.type == obj->type()) {
1.477 + entry.count++;
1.478 + } else if (!entry.type) {
1.479 + entry.type = obj->type();
1.480 + entry.count = 1;
1.481 + }
1.482 + }
1.483 +}
1.484 +
1.485 +MOZ_ALWAYS_INLINE void
1.486 +js::Nursery::traceObject(MinorCollectionTracer *trc, JSObject *obj)
1.487 +{
1.488 + const Class *clasp = obj->getClass();
1.489 + if (clasp->trace)
1.490 + clasp->trace(trc, obj);
1.491 +
1.492 + if (!obj->isNative())
1.493 + return;
1.494 +
1.495 + if (!obj->hasEmptyElements())
1.496 + markSlots(trc, obj->getDenseElements(), obj->getDenseInitializedLength());
1.497 +
1.498 + HeapSlot *fixedStart, *fixedEnd, *dynStart, *dynEnd;
1.499 + obj->getSlotRange(0, obj->slotSpan(), &fixedStart, &fixedEnd, &dynStart, &dynEnd);
1.500 + markSlots(trc, fixedStart, fixedEnd);
1.501 + markSlots(trc, dynStart, dynEnd);
1.502 +}
1.503 +
1.504 +MOZ_ALWAYS_INLINE void
1.505 +js::Nursery::markSlots(MinorCollectionTracer *trc, HeapSlot *vp, uint32_t nslots)
1.506 +{
1.507 + markSlots(trc, vp, vp + nslots);
1.508 +}
1.509 +
1.510 +MOZ_ALWAYS_INLINE void
1.511 +js::Nursery::markSlots(MinorCollectionTracer *trc, HeapSlot *vp, HeapSlot *end)
1.512 +{
1.513 + for (; vp != end; ++vp)
1.514 + markSlot(trc, vp);
1.515 +}
1.516 +
1.517 +MOZ_ALWAYS_INLINE void
1.518 +js::Nursery::markSlot(MinorCollectionTracer *trc, HeapSlot *slotp)
1.519 +{
1.520 + if (!slotp->isObject())
1.521 + return;
1.522 +
1.523 + JSObject *obj = &slotp->toObject();
1.524 + if (!isInside(obj))
1.525 + return;
1.526 +
1.527 + if (getForwardedPointer(&obj)) {
1.528 + slotp->unsafeGet()->setObject(*obj);
1.529 + return;
1.530 + }
1.531 +
1.532 + JSObject *tenured = static_cast<JSObject*>(moveToTenured(trc, obj));
1.533 + slotp->unsafeGet()->setObject(*tenured);
1.534 +}
1.535 +
1.536 +void *
1.537 +js::Nursery::moveToTenured(MinorCollectionTracer *trc, JSObject *src)
1.538 +{
1.539 + Zone *zone = src->zone();
1.540 + AllocKind dstKind = GetObjectAllocKindForCopy(trc->runtime(), src);
1.541 + JSObject *dst = static_cast<JSObject *>(allocateFromTenured(zone, dstKind));
1.542 + if (!dst)
1.543 + CrashAtUnhandlableOOM("Failed to allocate object while tenuring.");
1.544 +
1.545 + trc->tenuredSize += moveObjectToTenured(dst, src, dstKind);
1.546 +
1.547 + RelocationOverlay *overlay = reinterpret_cast<RelocationOverlay *>(src);
1.548 + overlay->forwardTo(dst);
1.549 + trc->insertIntoFixupList(overlay);
1.550 +
1.551 + return static_cast<void *>(dst);
1.552 +}
1.553 +
1.554 +size_t
1.555 +js::Nursery::moveObjectToTenured(JSObject *dst, JSObject *src, AllocKind dstKind)
1.556 +{
1.557 + size_t srcSize = Arena::thingSize(dstKind);
1.558 + size_t tenuredSize = srcSize;
1.559 +
1.560 + /*
1.561 + * Arrays do not necessarily have the same AllocKind between src and dst.
1.562 + * We deal with this by copying elements manually, possibly re-inlining
1.563 + * them if there is adequate room inline in dst.
1.564 + */
1.565 + if (src->is<ArrayObject>())
1.566 + srcSize = sizeof(ObjectImpl);
1.567 +
1.568 + js_memcpy(dst, src, srcSize);
1.569 + tenuredSize += moveSlotsToTenured(dst, src, dstKind);
1.570 + tenuredSize += moveElementsToTenured(dst, src, dstKind);
1.571 +
1.572 + if (src->is<TypedArrayObject>())
1.573 + forwardTypedArrayPointers(dst, src);
1.574 +
1.575 + /* The shape's list head may point into the old object. */
1.576 + if (&src->shape_ == dst->shape_->listp)
1.577 + dst->shape_->listp = &dst->shape_;
1.578 +
1.579 + return tenuredSize;
1.580 +}
1.581 +
1.582 +void
1.583 +js::Nursery::forwardTypedArrayPointers(JSObject *dst, JSObject *src)
1.584 +{
1.585 + /*
1.586 + * Typed array data may be stored inline inside the object's fixed slots. If
1.587 + * so, we need update the private pointer and leave a forwarding pointer at
1.588 + * the start of the data.
1.589 + */
1.590 + TypedArrayObject &typedArray = src->as<TypedArrayObject>();
1.591 + JS_ASSERT_IF(typedArray.buffer(), !isInside(src->getPrivate()));
1.592 + if (typedArray.buffer())
1.593 + return;
1.594 +
1.595 + void *srcData = src->fixedData(TypedArrayObject::FIXED_DATA_START);
1.596 + void *dstData = dst->fixedData(TypedArrayObject::FIXED_DATA_START);
1.597 + JS_ASSERT(src->getPrivate() == srcData);
1.598 + dst->setPrivate(dstData);
1.599 +
1.600 + /*
1.601 + * We don't know the number of slots here, but
1.602 + * TypedArrayObject::AllocKindForLazyBuffer ensures that it's always at
1.603 + * least one.
1.604 + */
1.605 + size_t nslots = 1;
1.606 + setSlotsForwardingPointer(reinterpret_cast<HeapSlot*>(srcData),
1.607 + reinterpret_cast<HeapSlot*>(dstData),
1.608 + nslots);
1.609 +}
1.610 +
1.611 +size_t
1.612 +js::Nursery::moveSlotsToTenured(JSObject *dst, JSObject *src, AllocKind dstKind)
1.613 +{
1.614 + /* Fixed slots have already been copied over. */
1.615 + if (!src->hasDynamicSlots())
1.616 + return 0;
1.617 +
1.618 + if (!isInside(src->slots)) {
1.619 + hugeSlots.remove(src->slots);
1.620 + return 0;
1.621 + }
1.622 +
1.623 + Zone *zone = src->zone();
1.624 + size_t count = src->numDynamicSlots();
1.625 + dst->slots = zone->pod_malloc<HeapSlot>(count);
1.626 + if (!dst->slots)
1.627 + CrashAtUnhandlableOOM("Failed to allocate slots while tenuring.");
1.628 + PodCopy(dst->slots, src->slots, count);
1.629 + setSlotsForwardingPointer(src->slots, dst->slots, count);
1.630 + return count * sizeof(HeapSlot);
1.631 +}
1.632 +
1.633 +size_t
1.634 +js::Nursery::moveElementsToTenured(JSObject *dst, JSObject *src, AllocKind dstKind)
1.635 +{
1.636 + if (src->hasEmptyElements())
1.637 + return 0;
1.638 +
1.639 + Zone *zone = src->zone();
1.640 + ObjectElements *srcHeader = src->getElementsHeader();
1.641 + ObjectElements *dstHeader;
1.642 +
1.643 + /* TODO Bug 874151: Prefer to put element data inline if we have space. */
1.644 + if (!isInside(srcHeader)) {
1.645 + JS_ASSERT(src->elements == dst->elements);
1.646 + hugeSlots.remove(reinterpret_cast<HeapSlot*>(srcHeader));
1.647 + return 0;
1.648 + }
1.649 +
1.650 + size_t nslots = ObjectElements::VALUES_PER_HEADER + srcHeader->capacity;
1.651 +
1.652 + /* Unlike other objects, Arrays can have fixed elements. */
1.653 + if (src->is<ArrayObject>() && nslots <= GetGCKindSlots(dstKind)) {
1.654 + dst->setFixedElements();
1.655 + dstHeader = dst->getElementsHeader();
1.656 + js_memcpy(dstHeader, srcHeader, nslots * sizeof(HeapSlot));
1.657 + setElementsForwardingPointer(srcHeader, dstHeader, nslots);
1.658 + return nslots * sizeof(HeapSlot);
1.659 + }
1.660 +
1.661 + JS_ASSERT(nslots >= 2);
1.662 + size_t nbytes = nslots * sizeof(HeapValue);
1.663 + dstHeader = static_cast<ObjectElements *>(zone->malloc_(nbytes));
1.664 + if (!dstHeader)
1.665 + CrashAtUnhandlableOOM("Failed to allocate elements while tenuring.");
1.666 + js_memcpy(dstHeader, srcHeader, nslots * sizeof(HeapSlot));
1.667 + setElementsForwardingPointer(srcHeader, dstHeader, nslots);
1.668 + dst->elements = dstHeader->elements();
1.669 + return nslots * sizeof(HeapSlot);
1.670 +}
1.671 +
1.672 +static bool
1.673 +ShouldMoveToTenured(MinorCollectionTracer *trc, void **thingp)
1.674 +{
1.675 + Cell *cell = static_cast<Cell *>(*thingp);
1.676 + Nursery &nursery = *trc->nursery;
1.677 + return !nursery.isInside(thingp) && nursery.isInside(cell) &&
1.678 + !nursery.getForwardedPointer(thingp);
1.679 +}
1.680 +
1.681 +/* static */ void
1.682 +js::Nursery::MinorGCCallback(JSTracer *jstrc, void **thingp, JSGCTraceKind kind)
1.683 +{
1.684 + MinorCollectionTracer *trc = static_cast<MinorCollectionTracer *>(jstrc);
1.685 + if (ShouldMoveToTenured(trc, thingp))
1.686 + *thingp = trc->nursery->moveToTenured(trc, static_cast<JSObject *>(*thingp));
1.687 +}
1.688 +
1.689 +static void
1.690 +CheckHashTablesAfterMovingGC(JSRuntime *rt)
1.691 +{
1.692 +#ifdef JS_GC_ZEAL
1.693 + if (rt->gcZeal() == ZealCheckHashTablesOnMinorGC) {
1.694 + /* Check that internal hash tables no longer have any pointers into the nursery. */
1.695 + for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
1.696 + c->checkNewTypeObjectTableAfterMovingGC();
1.697 + c->checkInitialShapesTableAfterMovingGC();
1.698 + c->checkWrapperMapAfterMovingGC();
1.699 + if (c->debugScopes)
1.700 + c->debugScopes->checkHashTablesAfterMovingGC(rt);
1.701 + }
1.702 + }
1.703 +#endif
1.704 +}
1.705 +
1.706 +#ifdef PROFILE_NURSERY
1.707 +#define TIME_START(name) int64_t timstampStart_##name = PRMJ_Now()
1.708 +#define TIME_END(name) int64_t timstampEnd_##name = PRMJ_Now()
1.709 +#define TIME_TOTAL(name) (timstampEnd_##name - timstampStart_##name)
1.710 +#else
1.711 +#define TIME_START(name)
1.712 +#define TIME_END(name)
1.713 +#define TIME_TOTAL(name)
1.714 +#endif
1.715 +
1.716 +void
1.717 +js::Nursery::collect(JSRuntime *rt, JS::gcreason::Reason reason, TypeObjectList *pretenureTypes)
1.718 +{
1.719 + JS_AbortIfWrongThread(rt);
1.720 +
1.721 + if (rt->mainThread.suppressGC)
1.722 + return;
1.723 +
1.724 + if (!isEnabled())
1.725 + return;
1.726 +
1.727 + if (isEmpty())
1.728 + return;
1.729 +
1.730 + rt->gcStats.count(gcstats::STAT_MINOR_GC);
1.731 +
1.732 + TIME_START(total);
1.733 +
1.734 + AutoStopVerifyingBarriers av(rt, false);
1.735 +
1.736 + // Move objects pointed to by roots from the nursery to the major heap.
1.737 + MinorCollectionTracer trc(rt, this);
1.738 +
1.739 + // Mark the store buffer. This must happen first.
1.740 + StoreBuffer &sb = rt->gcStoreBuffer;
1.741 + TIME_START(markValues);
1.742 + sb.markValues(&trc);
1.743 + TIME_END(markValues);
1.744 +
1.745 + TIME_START(markCells);
1.746 + sb.markCells(&trc);
1.747 + TIME_END(markCells);
1.748 +
1.749 + TIME_START(markSlots);
1.750 + sb.markSlots(&trc);
1.751 + TIME_END(markSlots);
1.752 +
1.753 + TIME_START(markWholeCells);
1.754 + sb.markWholeCells(&trc);
1.755 + TIME_END(markWholeCells);
1.756 +
1.757 + TIME_START(markRelocatableValues);
1.758 + sb.markRelocatableValues(&trc);
1.759 + TIME_END(markRelocatableValues);
1.760 +
1.761 + TIME_START(markRelocatableCells);
1.762 + sb.markRelocatableCells(&trc);
1.763 + TIME_END(markRelocatableCells);
1.764 +
1.765 + TIME_START(markGenericEntries);
1.766 + sb.markGenericEntries(&trc);
1.767 + TIME_END(markGenericEntries);
1.768 +
1.769 + TIME_START(checkHashTables);
1.770 + CheckHashTablesAfterMovingGC(rt);
1.771 + TIME_END(checkHashTables);
1.772 +
1.773 + TIME_START(markRuntime);
1.774 + MarkRuntime(&trc);
1.775 + TIME_END(markRuntime);
1.776 +
1.777 + TIME_START(markDebugger);
1.778 + Debugger::markAll(&trc);
1.779 + TIME_END(markDebugger);
1.780 +
1.781 + TIME_START(clearNewObjectCache);
1.782 + rt->newObjectCache.clearNurseryObjects(rt);
1.783 + TIME_END(clearNewObjectCache);
1.784 +
1.785 + // Most of the work is done here. This loop iterates over objects that have
1.786 + // been moved to the major heap. If these objects have any outgoing pointers
1.787 + // to the nursery, then those nursery objects get moved as well, until no
1.788 + // objects are left to move. That is, we iterate to a fixed point.
1.789 + TIME_START(collectToFP);
1.790 + TenureCountCache tenureCounts;
1.791 + collectToFixedPoint(&trc, tenureCounts);
1.792 + TIME_END(collectToFP);
1.793 +
1.794 + // Update the array buffer object's view lists.
1.795 + TIME_START(sweepArrayBufferViewList);
1.796 + for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
1.797 + if (!c->gcLiveArrayBuffers.empty())
1.798 + ArrayBufferObject::sweep(c);
1.799 + }
1.800 + TIME_END(sweepArrayBufferViewList);
1.801 +
1.802 + // Update any slot or element pointers whose destination has been tenured.
1.803 + TIME_START(updateJitActivations);
1.804 +#ifdef JS_ION
1.805 + js::jit::UpdateJitActivationsForMinorGC(rt, &trc);
1.806 +#endif
1.807 + TIME_END(updateJitActivations);
1.808 +
1.809 + // Resize the nursery.
1.810 + TIME_START(resize);
1.811 + double promotionRate = trc.tenuredSize / double(allocationEnd() - start());
1.812 + if (promotionRate > 0.05)
1.813 + growAllocableSpace();
1.814 + else if (promotionRate < 0.01)
1.815 + shrinkAllocableSpace();
1.816 + TIME_END(resize);
1.817 +
1.818 + // If we are promoting the nursery, or exhausted the store buffer with
1.819 + // pointers to nursery things, which will force a collection well before
1.820 + // the nursery is full, look for object types that are getting promoted
1.821 + // excessively and try to pretenure them.
1.822 + TIME_START(pretenure);
1.823 + if (pretenureTypes && (promotionRate > 0.8 || reason == JS::gcreason::FULL_STORE_BUFFER)) {
1.824 + for (size_t i = 0; i < ArrayLength(tenureCounts.entries); i++) {
1.825 + const TenureCount &entry = tenureCounts.entries[i];
1.826 + if (entry.count >= 3000)
1.827 + pretenureTypes->append(entry.type); // ignore alloc failure
1.828 + }
1.829 + }
1.830 + TIME_END(pretenure);
1.831 +
1.832 + // Sweep.
1.833 + TIME_START(freeHugeSlots);
1.834 + freeHugeSlots(rt);
1.835 + TIME_END(freeHugeSlots);
1.836 +
1.837 + TIME_START(sweep);
1.838 + sweep(rt);
1.839 + TIME_END(sweep);
1.840 +
1.841 + TIME_START(clearStoreBuffer);
1.842 + rt->gcStoreBuffer.clear();
1.843 + TIME_END(clearStoreBuffer);
1.844 +
1.845 + // We ignore gcMaxBytes when allocating for minor collection. However, if we
1.846 + // overflowed, we disable the nursery. The next time we allocate, we'll fail
1.847 + // because gcBytes >= gcMaxBytes.
1.848 + if (rt->gcBytes >= rt->gcMaxBytes)
1.849 + disable();
1.850 +
1.851 + TIME_END(total);
1.852 +
1.853 +#ifdef PROFILE_NURSERY
1.854 + int64_t totalTime = TIME_TOTAL(total);
1.855 +
1.856 + if (totalTime >= GCReportThreshold) {
1.857 + static bool printedHeader = false;
1.858 + if (!printedHeader) {
1.859 + fprintf(stderr,
1.860 + "MinorGC: Reason PRate Size Time mkVals mkClls mkSlts mkWCll mkRVal mkRCll mkGnrc ckTbls mkRntm mkDbgr clrNOC collct swpABO updtIn resize pretnr frSlts clrSB sweep\n");
1.861 + printedHeader = true;
1.862 + }
1.863 +
1.864 +#define FMT " %6" PRIu64
1.865 + fprintf(stderr,
1.866 + "MinorGC: %20s %5.1f%% %4d" FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT FMT "\n",
1.867 + js::gcstats::ExplainReason(reason),
1.868 + promotionRate * 100,
1.869 + numActiveChunks_,
1.870 + totalTime,
1.871 + TIME_TOTAL(markValues),
1.872 + TIME_TOTAL(markCells),
1.873 + TIME_TOTAL(markSlots),
1.874 + TIME_TOTAL(markWholeCells),
1.875 + TIME_TOTAL(markRelocatableValues),
1.876 + TIME_TOTAL(markRelocatableCells),
1.877 + TIME_TOTAL(markGenericEntries),
1.878 + TIME_TOTAL(checkHashTables),
1.879 + TIME_TOTAL(markRuntime),
1.880 + TIME_TOTAL(markDebugger),
1.881 + TIME_TOTAL(clearNewObjectCache),
1.882 + TIME_TOTAL(collectToFP),
1.883 + TIME_TOTAL(sweepArrayBufferViewList),
1.884 + TIME_TOTAL(updateJitActivations),
1.885 + TIME_TOTAL(resize),
1.886 + TIME_TOTAL(pretenure),
1.887 + TIME_TOTAL(freeHugeSlots),
1.888 + TIME_TOTAL(clearStoreBuffer),
1.889 + TIME_TOTAL(sweep));
1.890 +#undef FMT
1.891 + }
1.892 +#endif
1.893 +}
1.894 +
1.895 +void
1.896 +js::Nursery::freeHugeSlots(JSRuntime *rt)
1.897 +{
1.898 + for (HugeSlotsSet::Range r = hugeSlots.all(); !r.empty(); r.popFront())
1.899 + rt->defaultFreeOp()->free_(r.front());
1.900 + hugeSlots.clear();
1.901 +}
1.902 +
1.903 +void
1.904 +js::Nursery::sweep(JSRuntime *rt)
1.905 +{
1.906 +#ifdef JS_GC_ZEAL
1.907 + /* Poison the nursery contents so touching a freed object will crash. */
1.908 + JS_POISON((void *)start(), JS_SWEPT_NURSERY_PATTERN, NurserySize);
1.909 + for (int i = 0; i < NumNurseryChunks; ++i)
1.910 + initChunk(i);
1.911 +
1.912 + if (rt->gcZeal_ == ZealGenerationalGCValue) {
1.913 + MOZ_ASSERT(numActiveChunks_ == NumNurseryChunks);
1.914 +
1.915 + /* Only reset the alloc point when we are close to the end. */
1.916 + if (currentChunk_ + 1 == NumNurseryChunks)
1.917 + setCurrentChunk(0);
1.918 + } else
1.919 +#endif
1.920 + {
1.921 +#ifdef JS_CRASH_DIAGNOSTICS
1.922 + JS_POISON((void *)start(), JS_SWEPT_NURSERY_PATTERN, allocationEnd() - start());
1.923 + for (int i = 0; i < numActiveChunks_; ++i)
1.924 + chunk(i).trailer.runtime = runtime();
1.925 +#endif
1.926 + setCurrentChunk(0);
1.927 + }
1.928 +
1.929 + /* Set current start position for isEmpty checks. */
1.930 + currentStart_ = position();
1.931 +}
1.932 +
1.933 +void
1.934 +js::Nursery::growAllocableSpace()
1.935 +{
1.936 +#ifdef JS_GC_ZEAL
1.937 + MOZ_ASSERT_IF(runtime()->gcZeal_ == ZealGenerationalGCValue, numActiveChunks_ == NumNurseryChunks);
1.938 +#endif
1.939 + numActiveChunks_ = Min(numActiveChunks_ * 2, NumNurseryChunks);
1.940 +}
1.941 +
1.942 +void
1.943 +js::Nursery::shrinkAllocableSpace()
1.944 +{
1.945 +#ifdef JS_GC_ZEAL
1.946 + if (runtime()->gcZeal_ == ZealGenerationalGCValue)
1.947 + return;
1.948 +#endif
1.949 + numActiveChunks_ = Max(numActiveChunks_ - 1, 1);
1.950 + updateDecommittedRegion();
1.951 +}
1.952 +
1.953 +#endif /* JSGC_GENERATIONAL */
