michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
michael@0:  * vim: set ts=8 sts=4 et sw=4 tw=99:
michael@0:  * This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "jit/Safepoints.h"
michael@0: 
michael@0: #include "mozilla/MathAlgorithms.h"
michael@0: 
michael@0: #include "jit/BitSet.h"
michael@0: #include "jit/IonSpewer.h"
michael@0: #include "jit/LIR.h"
michael@0: 
michael@0: using namespace js;
michael@0: using namespace jit;
michael@0: 
michael@0: using mozilla::FloorLog2;
michael@0: 
michael@0: bool
michael@0: SafepointWriter::init(TempAllocator &alloc, uint32_t slotCount)
michael@0: {
michael@0:     frameSlots_ = BitSet::New(alloc, slotCount / sizeof(intptr_t));
michael@0:     if (!frameSlots_)
michael@0:         return false;
michael@0: 
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: uint32_t
michael@0: SafepointWriter::startEntry()
michael@0: {
michael@0:     IonSpew(IonSpew_Safepoints, "Encoding safepoint (position %d):", stream_.length());
michael@0:     return uint32_t(stream_.length());
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointWriter::writeOsiCallPointOffset(uint32_t osiCallPointOffset)
michael@0: {
michael@0:     stream_.writeUnsigned(osiCallPointOffset);
michael@0: }
michael@0: 
michael@0: static void
michael@0: WriteRegisterMask(CompactBufferWriter &stream, uint32_t bits)
michael@0: {
michael@0:     if (sizeof(PackedRegisterMask) == 1)
michael@0:         stream.writeByte(bits);
michael@0:     else
michael@0:         stream.writeUnsigned(bits);
michael@0: }
michael@0: 
michael@0: static int32_t
michael@0: ReadRegisterMask(CompactBufferReader &stream)
michael@0: {
michael@0:     if (sizeof(PackedRegisterMask) == 1)
michael@0:         return stream.readByte();
michael@0:     return stream.readUnsigned();
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointWriter::writeGcRegs(LSafepoint *safepoint)
michael@0: {
michael@0:     GeneralRegisterSet gc = safepoint->gcRegs();
michael@0:     GeneralRegisterSet spilledGpr = safepoint->liveRegs().gprs();
michael@0:     FloatRegisterSet spilledFloat = safepoint->liveRegs().fpus();
michael@0:     GeneralRegisterSet slots = safepoint->slotsOrElementsRegs();
michael@0:     GeneralRegisterSet valueRegs;
michael@0: 
michael@0:     WriteRegisterMask(stream_, spilledGpr.bits());
michael@0:     if (!spilledGpr.empty()) {
michael@0:         WriteRegisterMask(stream_, gc.bits());
michael@0:         WriteRegisterMask(stream_, slots.bits());
michael@0: 
michael@0: #ifdef JS_PUNBOX64
michael@0:         valueRegs = safepoint->valueRegs();
michael@0:         WriteRegisterMask(stream_, valueRegs.bits());
michael@0: #endif
michael@0:     }
michael@0: 
michael@0:     // GC registers are a subset of the spilled registers.
michael@0:     JS_ASSERT((valueRegs.bits() & ~spilledGpr.bits()) == 0);
michael@0:     JS_ASSERT((gc.bits() & ~spilledGpr.bits()) == 0);
michael@0: 
michael@0:     WriteRegisterMask(stream_, spilledFloat.bits());
michael@0: 
michael@0: #ifdef DEBUG
michael@0:     if (IonSpewEnabled(IonSpew_Safepoints)) {
michael@0:         for (GeneralRegisterForwardIterator iter(spilledGpr); iter.more(); iter++) {
michael@0:             const char *type = gc.has(*iter)
michael@0:                                ? "gc"
michael@0:                                : slots.has(*iter)
michael@0:                                  ? "slots"
michael@0:                                  : valueRegs.has(*iter)
michael@0:                                    ? "value"
michael@0:                                    : "any";
michael@0:             IonSpew(IonSpew_Safepoints, "    %s reg: %s", type, (*iter).name());
michael@0:         }
michael@0:         for (FloatRegisterForwardIterator iter(spilledFloat); iter.more(); iter++)
michael@0:             IonSpew(IonSpew_Safepoints, "    float reg: %s", (*iter).name());
michael@0:     }
michael@0: #endif
michael@0: }
michael@0: 
michael@0: static void
michael@0: MapSlotsToBitset(BitSet *set, CompactBufferWriter &stream, uint32_t nslots, uint32_t *slots)
michael@0: {
michael@0:     set->clear();
michael@0: 
michael@0:     for (uint32_t i = 0; i < nslots; i++) {
michael@0:         // Slots are represented at a distance from |fp|. We divide by the
michael@0:         // pointer size, since we only care about pointer-sized/aligned slots
michael@0:         // here. Since the stack grows down, this means slots start at index 1,
michael@0:         // so we subtract 1 to pack the bitset.
michael@0:         JS_ASSERT(slots[i] % sizeof(intptr_t) == 0);
michael@0:         JS_ASSERT(slots[i] / sizeof(intptr_t) > 0);
michael@0:         set->insert(slots[i] / sizeof(intptr_t) - 1);
michael@0:     }
michael@0: 
michael@0:     size_t count = set->rawLength();
michael@0:     const uint32_t *words = set->raw();
michael@0:     for (size_t i = 0; i < count; i++)
michael@0:         stream.writeUnsigned(words[i]);
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointWriter::writeGcSlots(LSafepoint *safepoint)
michael@0: {
michael@0:     LSafepoint::SlotList &slots = safepoint->gcSlots();
michael@0: 
michael@0: #ifdef DEBUG
michael@0:     for (uint32_t i = 0; i < slots.length(); i++)
michael@0:         IonSpew(IonSpew_Safepoints, "    gc slot: %d", slots[i]);
michael@0: #endif
michael@0: 
michael@0:     MapSlotsToBitset(frameSlots_,
michael@0:                      stream_,
michael@0:                      slots.length(),
michael@0:                      slots.begin());
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointWriter::writeSlotsOrElementsSlots(LSafepoint *safepoint)
michael@0: {
michael@0:     LSafepoint::SlotList &slots = safepoint->slotsOrElementsSlots();
michael@0: 
michael@0:     stream_.writeUnsigned(slots.length());
michael@0: 
michael@0:     for (uint32_t i = 0; i < slots.length(); i++) {
michael@0: #ifdef DEBUG
michael@0:         IonSpew(IonSpew_Safepoints, "    slots/elements slot: %d", slots[i]);
michael@0: #endif
michael@0:         stream_.writeUnsigned(slots[i]);
michael@0:     }
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointWriter::writeValueSlots(LSafepoint *safepoint)
michael@0: {
michael@0:     LSafepoint::SlotList &slots = safepoint->valueSlots();
michael@0: 
michael@0: #ifdef DEBUG
michael@0:     for (uint32_t i = 0; i < slots.length(); i++)
michael@0:         IonSpew(IonSpew_Safepoints, "    gc value: %d", slots[i]);
michael@0: #endif
michael@0: 
michael@0:     MapSlotsToBitset(frameSlots_, stream_, slots.length(), slots.begin());
michael@0: }
michael@0: 
michael@0: #if defined(DEBUG) && defined(JS_NUNBOX32)
michael@0: static void
michael@0: DumpNunboxPart(const LAllocation &a)
michael@0: {
michael@0:     if (a.isStackSlot()) {
michael@0:         fprintf(IonSpewFile, "stack %d", a.toStackSlot()->slot());
michael@0:     } else if (a.isArgument()) {
michael@0:         fprintf(IonSpewFile, "arg %d", a.toArgument()->index());
michael@0:     } else {
michael@0:         fprintf(IonSpewFile, "reg %s", a.toGeneralReg()->reg().name());
michael@0:     }
michael@0: }
michael@0: #endif // DEBUG
michael@0: 
michael@0: // Nunbox part encoding:
michael@0: //
michael@0: // Reg = 000
michael@0: // Stack = 001
michael@0: // Arg = 010
michael@0: //
michael@0: // [vwu] nentries:
michael@0: //    uint16_t:  tttp ppXX XXXY YYYY
michael@0: //
michael@0: //     If ttt = Reg, type is reg XXXXX
michael@0: //     If ppp = Reg, payload is reg YYYYY
michael@0: //
michael@0: //     If ttt != Reg, type is:
michael@0: //          XXXXX if not 11111, otherwise followed by [vwu]
michael@0: //     If ppp != Reg, payload is:
michael@0: //          YYYYY if not 11111, otherwise followed by [vwu]
michael@0: //
michael@0: enum NunboxPartKind {
michael@0:     Part_Reg,
michael@0:     Part_Stack,
michael@0:     Part_Arg
michael@0: };
michael@0: 
michael@0: static const uint32_t PART_KIND_BITS = 3;
michael@0: static const uint32_t PART_KIND_MASK = (1 << PART_KIND_BITS) - 1;
michael@0: static const uint32_t PART_INFO_BITS = 5;
michael@0: static const uint32_t PART_INFO_MASK = (1 << PART_INFO_BITS) - 1;
michael@0: 
michael@0: static const uint32_t MAX_INFO_VALUE = (1 << PART_INFO_BITS) - 1;
michael@0: static const uint32_t TYPE_KIND_SHIFT = 16 - PART_KIND_BITS;
michael@0: static const uint32_t PAYLOAD_KIND_SHIFT = TYPE_KIND_SHIFT - PART_KIND_BITS;
michael@0: static const uint32_t TYPE_INFO_SHIFT = PAYLOAD_KIND_SHIFT - PART_INFO_BITS;
michael@0: static const uint32_t PAYLOAD_INFO_SHIFT = TYPE_INFO_SHIFT - PART_INFO_BITS;
michael@0: 
michael@0: JS_STATIC_ASSERT(PAYLOAD_INFO_SHIFT == 0);
michael@0: 
michael@0: #ifdef JS_NUNBOX32
michael@0: static inline NunboxPartKind
michael@0: AllocationToPartKind(const LAllocation &a)
michael@0: {
michael@0:     if (a.isRegister())
michael@0:         return Part_Reg;
michael@0:     if (a.isStackSlot())
michael@0:         return Part_Stack;
michael@0:     JS_ASSERT(a.isArgument());
michael@0:     return Part_Arg;
michael@0: }
michael@0: 
michael@0: // gcc 4.5 doesn't actually inline CanEncodeInfoInHeader when only
michael@0: // using the "inline" keyword, and miscompiles the function as well
michael@0: // when doing block reordering with branch prediction information.
michael@0: // See bug 799295 comment 71.
michael@0: static MOZ_ALWAYS_INLINE bool
michael@0: CanEncodeInfoInHeader(const LAllocation &a, uint32_t *out)
michael@0: {
michael@0:     if (a.isGeneralReg()) {
michael@0:         *out = a.toGeneralReg()->reg().code();
michael@0:         return true;
michael@0:     }
michael@0: 
michael@0:     if (a.isStackSlot())
michael@0:         *out = a.toStackSlot()->slot();
michael@0:     else
michael@0:         *out = a.toArgument()->index();
michael@0: 
michael@0:     return *out < MAX_INFO_VALUE;
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointWriter::writeNunboxParts(LSafepoint *safepoint)
michael@0: {
michael@0:     LSafepoint::NunboxList &entries = safepoint->nunboxParts();
michael@0: 
michael@0: # ifdef DEBUG
michael@0:     if (IonSpewEnabled(IonSpew_Safepoints)) {
michael@0:         for (uint32_t i = 0; i < entries.length(); i++) {
michael@0:             SafepointNunboxEntry &entry = entries[i];
michael@0:             if (entry.type.isUse() || entry.payload.isUse())
michael@0:                 continue;
michael@0:             IonSpewHeader(IonSpew_Safepoints);
michael@0:             fprintf(IonSpewFile, "    nunbox (type in ");
michael@0:             DumpNunboxPart(entry.type);
michael@0:             fprintf(IonSpewFile, ", payload in ");
michael@0:             DumpNunboxPart(entry.payload);
michael@0:             fprintf(IonSpewFile, ")\n");
michael@0:         }
michael@0:     }
michael@0: # endif
michael@0: 
michael@0:     // Safepoints are permitted to have partially filled in entries for nunboxes,
michael@0:     // provided that only the type is live and not the payload. Omit these from
michael@0:     // the written safepoint.
michael@0:     uint32_t partials = safepoint->partialNunboxes();
michael@0: 
michael@0:     stream_.writeUnsigned(entries.length() - partials);
michael@0: 
michael@0:     for (size_t i = 0; i < entries.length(); i++) {
michael@0:         SafepointNunboxEntry &entry = entries[i];
michael@0: 
michael@0:         if (entry.type.isUse() || entry.payload.isUse()) {
michael@0:             partials--;
michael@0:             continue;
michael@0:         }
michael@0: 
michael@0:         uint16_t header = 0;
michael@0: 
michael@0:         header |= (AllocationToPartKind(entry.type) << TYPE_KIND_SHIFT);
michael@0:         header |= (AllocationToPartKind(entry.payload) << PAYLOAD_KIND_SHIFT);
michael@0: 
michael@0:         uint32_t typeVal;
michael@0:         bool typeExtra = !CanEncodeInfoInHeader(entry.type, &typeVal);
michael@0:         if (!typeExtra)
michael@0:             header |= (typeVal << TYPE_INFO_SHIFT);
michael@0:         else
michael@0:             header |= (MAX_INFO_VALUE << TYPE_INFO_SHIFT);
michael@0: 
michael@0:         uint32_t payloadVal;
michael@0:         bool payloadExtra = !CanEncodeInfoInHeader(entry.payload, &payloadVal);
michael@0:         if (!payloadExtra)
michael@0:             header |= (payloadVal << PAYLOAD_INFO_SHIFT);
michael@0:         else
michael@0:             header |= (MAX_INFO_VALUE << PAYLOAD_INFO_SHIFT);
michael@0: 
michael@0:         stream_.writeFixedUint16_t(header);
michael@0:         if (typeExtra)
michael@0:             stream_.writeUnsigned(typeVal);
michael@0:         if (payloadExtra)
michael@0:             stream_.writeUnsigned(payloadVal);
michael@0:     }
michael@0: 
michael@0:     JS_ASSERT(partials == 0);
michael@0: }
michael@0: #endif
michael@0: 
michael@0: void
michael@0: SafepointWriter::encode(LSafepoint *safepoint)
michael@0: {
michael@0:     uint32_t safepointOffset = startEntry();
michael@0: 
michael@0:     JS_ASSERT(safepoint->osiCallPointOffset());
michael@0: 
michael@0:     writeOsiCallPointOffset(safepoint->osiCallPointOffset());
michael@0:     writeGcRegs(safepoint);
michael@0:     writeGcSlots(safepoint);
michael@0:     writeValueSlots(safepoint);
michael@0: 
michael@0: #ifdef JS_NUNBOX32
michael@0:     writeNunboxParts(safepoint);
michael@0: #endif
michael@0: 
michael@0:     writeSlotsOrElementsSlots(safepoint);
michael@0: 
michael@0:     endEntry();
michael@0:     safepoint->setOffset(safepointOffset);
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointWriter::endEntry()
michael@0: {
michael@0:     IonSpew(IonSpew_Safepoints, "    -- entry ended at %d", uint32_t(stream_.length()));
michael@0: }
michael@0: 
michael@0: SafepointReader::SafepointReader(IonScript *script, const SafepointIndex *si)
michael@0:   : stream_(script->safepoints() + si->safepointOffset(),
michael@0:             script->safepoints() + script->safepointsSize()),
michael@0:     frameSlots_(script->frameSlots() / sizeof(intptr_t))
michael@0: {
michael@0:     osiCallPointOffset_ = stream_.readUnsigned();
michael@0: 
michael@0:     // gcSpills is a subset of allGprSpills.
michael@0:     allGprSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
michael@0:     if (allGprSpills_.empty()) {
michael@0:         gcSpills_ = allGprSpills_;
michael@0:         valueSpills_ = allGprSpills_;
michael@0:         slotsOrElementsSpills_ = allGprSpills_;
michael@0:     } else {
michael@0:         gcSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
michael@0:         slotsOrElementsSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
michael@0: #ifdef JS_PUNBOX64
michael@0:         valueSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
michael@0: #endif
michael@0:     }
michael@0: 
michael@0:     allFloatSpills_ = FloatRegisterSet(ReadRegisterMask(stream_));
michael@0: 
michael@0:     advanceFromGcRegs();
michael@0: }
michael@0: 
michael@0: uint32_t
michael@0: SafepointReader::osiReturnPointOffset() const
michael@0: {
michael@0:     return osiCallPointOffset_ + Assembler::patchWrite_NearCallSize();
michael@0: }
michael@0: 
michael@0: CodeLocationLabel
michael@0: SafepointReader::InvalidationPatchPoint(IonScript *script, const SafepointIndex *si)
michael@0: {
michael@0:     SafepointReader reader(script, si);
michael@0: 
michael@0:     return CodeLocationLabel(script->method(), reader.osiCallPointOffset());
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointReader::advanceFromGcRegs()
michael@0: {
michael@0:     currentSlotChunk_ = 0;
michael@0:     nextSlotChunkNumber_ = 0;
michael@0: }
michael@0: 
michael@0: bool
michael@0: SafepointReader::getSlotFromBitmap(uint32_t *slot)
michael@0: {
michael@0:     while (currentSlotChunk_ == 0) {
michael@0:         // Are there any more chunks to read?
michael@0:         if (nextSlotChunkNumber_ == BitSet::RawLengthForBits(frameSlots_))
michael@0:             return false;
michael@0: 
michael@0:         // Yes, read the next chunk.
michael@0:         currentSlotChunk_ = stream_.readUnsigned();
michael@0:         nextSlotChunkNumber_++;
michael@0:     }
michael@0: 
michael@0:     // The current chunk still has bits in it, so get the next bit, then mask
michael@0:     // it out of the slot chunk.
michael@0:     uint32_t bit = FloorLog2(currentSlotChunk_);
michael@0:     currentSlotChunk_ &= ~(1 << bit);
michael@0: 
michael@0:     // Return the slot, taking care to add 1 back in since it was subtracted
michael@0:     // when added in the original bitset, and re-scale it by the pointer size,
michael@0:     // reversing the transformation in MapSlotsToBitset.
michael@0:     *slot = (((nextSlotChunkNumber_ - 1) * BitSet::BitsPerWord) + bit + 1) * sizeof(intptr_t);
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: bool
michael@0: SafepointReader::getGcSlot(uint32_t *slot)
michael@0: {
michael@0:     if (getSlotFromBitmap(slot))
michael@0:         return true;
michael@0:     advanceFromGcSlots();
michael@0:     return false;
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointReader::advanceFromGcSlots()
michael@0: {
michael@0:     // No, reset the counter.
michael@0:     currentSlotChunk_ = 0;
michael@0:     nextSlotChunkNumber_ = 0;
michael@0: }
michael@0: 
michael@0: bool
michael@0: SafepointReader::getValueSlot(uint32_t *slot)
michael@0: {
michael@0:     if (getSlotFromBitmap(slot))
michael@0:         return true;
michael@0:     advanceFromValueSlots();
michael@0:     return false;
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointReader::advanceFromValueSlots()
michael@0: {
michael@0: #ifdef JS_NUNBOX32
michael@0:     nunboxSlotsRemaining_ = stream_.readUnsigned();
michael@0: #else
michael@0:     nunboxSlotsRemaining_ = 0;
michael@0:     advanceFromNunboxSlots();
michael@0: #endif
michael@0: }
michael@0: 
michael@0: static inline LAllocation
michael@0: PartFromStream(CompactBufferReader &stream, NunboxPartKind kind, uint32_t info)
michael@0: {
michael@0:     if (kind == Part_Reg)
michael@0:         return LGeneralReg(Register::FromCode(info));
michael@0: 
michael@0:     if (info == MAX_INFO_VALUE)
michael@0:         info = stream.readUnsigned();
michael@0: 
michael@0:     if (kind == Part_Stack)
michael@0:         return LStackSlot(info);
michael@0: 
michael@0:     JS_ASSERT(kind == Part_Arg);
michael@0:     return LArgument(info);
michael@0: }
michael@0: 
michael@0: bool
michael@0: SafepointReader::getNunboxSlot(LAllocation *type, LAllocation *payload)
michael@0: {
michael@0:     if (!nunboxSlotsRemaining_--) {
michael@0:         advanceFromNunboxSlots();
michael@0:         return false;
michael@0:     }
michael@0: 
michael@0:     uint16_t header = stream_.readFixedUint16_t();
michael@0:     NunboxPartKind typeKind = (NunboxPartKind)((header >> TYPE_KIND_SHIFT) & PART_KIND_MASK);
michael@0:     NunboxPartKind payloadKind = (NunboxPartKind)((header >> PAYLOAD_KIND_SHIFT) & PART_KIND_MASK);
michael@0:     uint32_t typeInfo = (header >> TYPE_INFO_SHIFT) & PART_INFO_MASK;
michael@0:     uint32_t payloadInfo = (header >> PAYLOAD_INFO_SHIFT) & PART_INFO_MASK;
michael@0: 
michael@0:     *type = PartFromStream(stream_, typeKind, typeInfo);
michael@0:     *payload = PartFromStream(stream_, payloadKind, payloadInfo);
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: void
michael@0: SafepointReader::advanceFromNunboxSlots()
michael@0: {
michael@0:     slotsOrElementsSlotsRemaining_ = stream_.readUnsigned();
michael@0: }
michael@0: 
michael@0: bool
michael@0: SafepointReader::getSlotsOrElementsSlot(uint32_t *slot)
michael@0: {
michael@0:     if (!slotsOrElementsSlotsRemaining_--)
michael@0:         return false;
michael@0:     *slot = stream_.readUnsigned();
michael@0:     return true;
michael@0: }