michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
michael@0:  * vim: set ts=8 sts=4 et sw=4 tw=99:
michael@0:  * This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #ifdef JSGC_GENERATIONAL
michael@0: 
michael@0: #include "gc/StoreBuffer.h"
michael@0: 
michael@0: #include "mozilla/Assertions.h"
michael@0: 
michael@0: #include "vm/ArgumentsObject.h"
michael@0: #include "vm/ForkJoin.h"
michael@0: 
michael@0: #include "jsgcinlines.h"
michael@0: 
michael@0: using namespace js;
michael@0: using namespace js::gc;
michael@0: using mozilla::ReentrancyGuard;
michael@0: 
michael@0: /*** Edges ***/
michael@0: 
michael@0: void
michael@0: StoreBuffer::SlotsEdge::mark(JSTracer *trc)
michael@0: {
michael@0:     JSObject *obj = object();
michael@0: 
michael@0:     if (trc->runtime()->gcNursery.isInside(obj))
michael@0:         return;
michael@0: 
michael@0:     if (!obj->isNative()) {
michael@0:         const Class *clasp = obj->getClass();
michael@0:         if (clasp)
michael@0:             clasp->trace(trc, obj);
michael@0:         return;
michael@0:     }
michael@0: 
michael@0:     if (kind() == ElementKind) {
michael@0:         int32_t initLen = obj->getDenseInitializedLength();
michael@0:         int32_t clampedStart = Min(start_, initLen);
michael@0:         int32_t clampedEnd = Min(start_ + count_, initLen);
michael@0:         gc::MarkArraySlots(trc, clampedEnd - clampedStart,
michael@0:                            obj->getDenseElements() + clampedStart, "element");
michael@0:     } else {
michael@0:         int32_t start = Min(uint32_t(start_), obj->slotSpan());
michael@0:         int32_t end = Min(uint32_t(start_) + count_, obj->slotSpan());
michael@0:         MOZ_ASSERT(end >= start);
michael@0:         MarkObjectSlots(trc, obj, start, end - start);
michael@0:     }
michael@0: }
michael@0: 
michael@0: void
michael@0: StoreBuffer::WholeCellEdges::mark(JSTracer *trc)
michael@0: {
michael@0:     JS_ASSERT(edge->isTenured());
michael@0:     JSGCTraceKind kind = GetGCThingTraceKind(edge);
michael@0:     if (kind <= JSTRACE_OBJECT) {
michael@0:         JSObject *object = static_cast<JSObject *>(edge);
michael@0:         if (object->is<ArgumentsObject>())
michael@0:             ArgumentsObject::trace(trc, object);
michael@0:         MarkChildren(trc, object);
michael@0:         return;
michael@0:     }
michael@0: #ifdef JS_ION
michael@0:     JS_ASSERT(kind == JSTRACE_JITCODE);
michael@0:     static_cast<jit::JitCode *>(edge)->trace(trc);
michael@0: #else
michael@0:     MOZ_ASSUME_UNREACHABLE("Only objects can be in the wholeCellBuffer if IonMonkey is disabled.");
michael@0: #endif
michael@0: }
michael@0: 
michael@0: void
michael@0: StoreBuffer::CellPtrEdge::mark(JSTracer *trc)
michael@0: {
michael@0:     if (!*edge)
michael@0:         return;
michael@0: 
michael@0:     JS_ASSERT(GetGCThingTraceKind(*edge) == JSTRACE_OBJECT);
michael@0:     MarkObjectRoot(trc, reinterpret_cast<JSObject**>(edge), "store buffer edge");
michael@0: }
michael@0: 
michael@0: void
michael@0: StoreBuffer::ValueEdge::mark(JSTracer *trc)
michael@0: {
michael@0:     if (!deref())
michael@0:         return;
michael@0: 
michael@0:     MarkValueRoot(trc, edge, "store buffer edge");
michael@0: }
michael@0: 
michael@0: /*** MonoTypeBuffer ***/
michael@0: 
michael@0: template <typename T>
michael@0: void
michael@0: StoreBuffer::MonoTypeBuffer<T>::handleOverflow(StoreBuffer *owner)
michael@0: {
michael@0:     if (!owner->isAboutToOverflow()) {
michael@0:         /*
michael@0:          * Compact the buffer now, and if that fails to free enough space then
michael@0:          * trigger a minor collection.
michael@0:          */
michael@0:         compact(owner);
michael@0:         if (isAboutToOverflow())
michael@0:             owner->setAboutToOverflow();
michael@0:     } else {
michael@0:          /*
michael@0:           * A minor GC has already been triggered, so there's no point
michael@0:           * compacting unless the buffer is totally full.
michael@0:           */
michael@0:         if (storage_->availableInCurrentChunk() < sizeof(T))
michael@0:             maybeCompact(owner);
michael@0:     }
michael@0: }
michael@0: 
michael@0: template <typename T>
michael@0: void
michael@0: StoreBuffer::MonoTypeBuffer<T>::compactRemoveDuplicates(StoreBuffer *owner)
michael@0: {
michael@0:     typedef HashSet<T, typename T::Hasher, SystemAllocPolicy> DedupSet;
michael@0: 
michael@0:     DedupSet duplicates;
michael@0:     if (!duplicates.init())
michael@0:         return; /* Failure to de-dup is acceptable. */
michael@0: 
michael@0:     LifoAlloc::Enum insert(*storage_);
michael@0:     for (LifoAlloc::Enum e(*storage_); !e.empty(); e.popFront<T>()) {
michael@0:         T *edge = e.get<T>();
michael@0:         if (!duplicates.has(*edge)) {
michael@0:             insert.updateFront<T>(*edge);
michael@0:             insert.popFront<T>();
michael@0: 
michael@0:             /* Failure to insert will leave the set with duplicates. Oh well. */
michael@0:             duplicates.put(*edge);
michael@0:         }
michael@0:     }
michael@0:     storage_->release(insert.mark());
michael@0: 
michael@0:     duplicates.clear();
michael@0: }
michael@0: 
michael@0: template <typename T>
michael@0: void
michael@0: StoreBuffer::MonoTypeBuffer<T>::compact(StoreBuffer *owner)
michael@0: {
michael@0:     JS_ASSERT(storage_);
michael@0:     compactRemoveDuplicates(owner);
michael@0:     usedAtLastCompact_ = storage_->used();
michael@0: }
michael@0: 
michael@0: template <typename T>
michael@0: void
michael@0: StoreBuffer::MonoTypeBuffer<T>::maybeCompact(StoreBuffer *owner)
michael@0: {
michael@0:     JS_ASSERT(storage_);
michael@0:     if (storage_->used() != usedAtLastCompact_)
michael@0:         compact(owner);
michael@0: }
michael@0: 
michael@0: template <typename T>
michael@0: void
michael@0: StoreBuffer::MonoTypeBuffer<T>::mark(StoreBuffer *owner, JSTracer *trc)
michael@0: {
michael@0:     JS_ASSERT(owner->isEnabled());
michael@0:     ReentrancyGuard g(*owner);
michael@0:     if (!storage_)
michael@0:         return;
michael@0: 
michael@0:     maybeCompact(owner);
michael@0:     for (LifoAlloc::Enum e(*storage_); !e.empty(); e.popFront<T>()) {
michael@0:         T *edge = e.get<T>();
michael@0:         edge->mark(trc);
michael@0:     }
michael@0: }
michael@0: 
michael@0: /*** RelocatableMonoTypeBuffer ***/
michael@0: 
michael@0: template <typename T>
michael@0: void
michael@0: StoreBuffer::RelocatableMonoTypeBuffer<T>::compactMoved(StoreBuffer *owner)
michael@0: {
michael@0:     LifoAlloc &storage = *this->storage_;
michael@0:     EdgeSet invalidated;
michael@0:     if (!invalidated.init())
michael@0:         CrashAtUnhandlableOOM("RelocatableMonoTypeBuffer::compactMoved: Failed to init table.");
michael@0: 
michael@0:     /* Collect the set of entries which are currently invalid. */
michael@0:     for (LifoAlloc::Enum e(storage); !e.empty(); e.popFront<T>()) {
michael@0:         T *edge = e.get<T>();
michael@0:         if (edge->isTagged()) {
michael@0:             if (!invalidated.put(edge->untagged().edge))
michael@0:                 CrashAtUnhandlableOOM("RelocatableMonoTypeBuffer::compactMoved: Failed to put removal.");
michael@0:         } else {
michael@0:             invalidated.remove(edge->untagged().edge);
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     /* Remove all entries which are in the invalidated set. */
michael@0:     LifoAlloc::Enum insert(storage);
michael@0:     for (LifoAlloc::Enum e(storage); !e.empty(); e.popFront<T>()) {
michael@0:         T *edge = e.get<T>();
michael@0:         if (!edge->isTagged() && !invalidated.has(edge->untagged().edge)) {
michael@0:             insert.updateFront<T>(*edge);
michael@0:             insert.popFront<T>();
michael@0:         }
michael@0:     }
michael@0:     storage.release(insert.mark());
michael@0: 
michael@0:     invalidated.clear();
michael@0: 
michael@0: #ifdef DEBUG
michael@0:     for (LifoAlloc::Enum e(storage); !e.empty(); e.popFront<T>())
michael@0:         JS_ASSERT(!e.get<T>()->isTagged());
michael@0: #endif
michael@0: }
michael@0: 
michael@0: template <typename T>
michael@0: void
michael@0: StoreBuffer::RelocatableMonoTypeBuffer<T>::compact(StoreBuffer *owner)
michael@0: {
michael@0:     compactMoved(owner);
michael@0:     StoreBuffer::MonoTypeBuffer<T>::compact(owner);
michael@0: }
michael@0: 
michael@0: /*** GenericBuffer ***/
michael@0: 
michael@0: void
michael@0: StoreBuffer::GenericBuffer::mark(StoreBuffer *owner, JSTracer *trc)
michael@0: {
michael@0:     JS_ASSERT(owner->isEnabled());
michael@0:     ReentrancyGuard g(*owner);
michael@0:     if (!storage_)
michael@0:         return;
michael@0: 
michael@0:     for (LifoAlloc::Enum e(*storage_); !e.empty();) {
michael@0:         unsigned size = *e.get<unsigned>();
michael@0:         e.popFront<unsigned>();
michael@0:         BufferableRef *edge = e.get<BufferableRef>(size);
michael@0:         edge->mark(trc);
michael@0:         e.popFront(size);
michael@0:     }
michael@0: }
michael@0: 
michael@0: /*** StoreBuffer ***/
michael@0: 
michael@0: bool
michael@0: StoreBuffer::enable()
michael@0: {
michael@0:     if (enabled_)
michael@0:         return true;
michael@0: 
michael@0:     if (!bufferVal.init() ||
michael@0:         !bufferCell.init() ||
michael@0:         !bufferSlot.init() ||
michael@0:         !bufferWholeCell.init() ||
michael@0:         !bufferRelocVal.init() ||
michael@0:         !bufferRelocCell.init() ||
michael@0:         !bufferGeneric.init())
michael@0:     {
michael@0:         return false;
michael@0:     }
michael@0: 
michael@0:     enabled_ = true;
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: void
michael@0: StoreBuffer::disable()
michael@0: {
michael@0:     if (!enabled_)
michael@0:         return;
michael@0: 
michael@0:     aboutToOverflow_ = false;
michael@0: 
michael@0:     enabled_ = false;
michael@0: }
michael@0: 
michael@0: bool
michael@0: StoreBuffer::clear()
michael@0: {
michael@0:     if (!enabled_)
michael@0:         return true;
michael@0: 
michael@0:     aboutToOverflow_ = false;
michael@0: 
michael@0:     bufferVal.clear();
michael@0:     bufferCell.clear();
michael@0:     bufferSlot.clear();
michael@0:     bufferWholeCell.clear();
michael@0:     bufferRelocVal.clear();
michael@0:     bufferRelocCell.clear();
michael@0:     bufferGeneric.clear();
michael@0: 
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: void
michael@0: StoreBuffer::markAll(JSTracer *trc)
michael@0: {
michael@0:     bufferVal.mark(this, trc);
michael@0:     bufferCell.mark(this, trc);
michael@0:     bufferSlot.mark(this, trc);
michael@0:     bufferWholeCell.mark(this, trc);
michael@0:     bufferRelocVal.mark(this, trc);
michael@0:     bufferRelocCell.mark(this, trc);
michael@0:     bufferGeneric.mark(this, trc);
michael@0: }
michael@0: 
michael@0: void
michael@0: StoreBuffer::setAboutToOverflow()
michael@0: {
michael@0:     aboutToOverflow_ = true;
michael@0:     runtime_->requestInterrupt(JSRuntime::RequestInterruptMainThread);
michael@0: }
michael@0: 
michael@0: bool
michael@0: StoreBuffer::inParallelSection() const
michael@0: {
michael@0:     return InParallelSection();
michael@0: }
michael@0: 
michael@0: void
michael@0: StoreBuffer::addSizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf, JS::GCSizes
michael@0: *sizes)
michael@0: {
michael@0:     sizes->storeBufferVals       += bufferVal.sizeOfExcludingThis(mallocSizeOf);
michael@0:     sizes->storeBufferCells      += bufferCell.sizeOfExcludingThis(mallocSizeOf);
michael@0:     sizes->storeBufferSlots      += bufferSlot.sizeOfExcludingThis(mallocSizeOf);
michael@0:     sizes->storeBufferWholeCells += bufferWholeCell.sizeOfExcludingThis(mallocSizeOf);
michael@0:     sizes->storeBufferRelocVals  += bufferRelocVal.sizeOfExcludingThis(mallocSizeOf);
michael@0:     sizes->storeBufferRelocCells += bufferRelocCell.sizeOfExcludingThis(mallocSizeOf);
michael@0:     sizes->storeBufferGenerics   += bufferGeneric.sizeOfExcludingThis(mallocSizeOf);
michael@0: }
michael@0: 
michael@0: JS_PUBLIC_API(void)
michael@0: JS::HeapCellPostBarrier(js::gc::Cell **cellp)
michael@0: {
michael@0:     JS_ASSERT(*cellp);
michael@0:     JSRuntime *runtime = (*cellp)->runtimeFromMainThread();
michael@0:     runtime->gcStoreBuffer.putRelocatableCell(cellp);
michael@0: }
michael@0: 
michael@0: JS_PUBLIC_API(void)
michael@0: JS::HeapCellRelocate(js::gc::Cell **cellp)
michael@0: {
michael@0:     /* Called with old contents of *pp before overwriting. */
michael@0:     JS_ASSERT(*cellp);
michael@0:     JSRuntime *runtime = (*cellp)->runtimeFromMainThread();
michael@0:     runtime->gcStoreBuffer.removeRelocatableCell(cellp);
michael@0: }
michael@0: 
michael@0: JS_PUBLIC_API(void)
michael@0: JS::HeapValuePostBarrier(JS::Value *valuep)
michael@0: {
michael@0:     JS_ASSERT(valuep->isMarkable());
michael@0:     if (valuep->isString() && StringIsPermanentAtom(valuep->toString()))
michael@0:         return;
michael@0:     JSRuntime *runtime = static_cast<js::gc::Cell *>(valuep->toGCThing())->runtimeFromMainThread();
michael@0:     runtime->gcStoreBuffer.putRelocatableValue(valuep);
michael@0: }
michael@0: 
michael@0: JS_PUBLIC_API(void)
michael@0: JS::HeapValueRelocate(JS::Value *valuep)
michael@0: {
michael@0:     /* Called with old contents of *valuep before overwriting. */
michael@0:     JS_ASSERT(valuep->isMarkable());
michael@0:     if (valuep->isString() && StringIsPermanentAtom(valuep->toString()))
michael@0:         return;
michael@0:     JSRuntime *runtime = static_cast<js::gc::Cell *>(valuep->toGCThing())->runtimeFromMainThread();
michael@0:     runtime->gcStoreBuffer.removeRelocatableValue(valuep);
michael@0: }
michael@0: 
michael@0: template class StoreBuffer::MonoTypeBuffer<StoreBuffer::ValueEdge>;
michael@0: template class StoreBuffer::MonoTypeBuffer<StoreBuffer::CellPtrEdge>;
michael@0: template class StoreBuffer::MonoTypeBuffer<StoreBuffer::SlotsEdge>;
michael@0: template class StoreBuffer::MonoTypeBuffer<StoreBuffer::WholeCellEdges>;
michael@0: template class StoreBuffer::RelocatableMonoTypeBuffer<StoreBuffer::ValueEdge>;
michael@0: template class StoreBuffer::RelocatableMonoTypeBuffer<StoreBuffer::CellPtrEdge>;
michael@0: 
michael@0: #endif /* JSGC_GENERATIONAL */