michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
michael@0:  * vim: set ts=8 sts=4 et sw=4 tw=99:
michael@0:  * This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "jit/shared/MoveEmitter-x86-shared.h"
michael@0: 
michael@0: using namespace js;
michael@0: using namespace js::jit;
michael@0: 
michael@0: MoveEmitterX86::MoveEmitterX86(MacroAssemblerSpecific &masm)
michael@0:   : inCycle_(false),
michael@0:     masm(masm),
michael@0:     pushedAtCycle_(-1)
michael@0: {
michael@0:     pushedAtStart_ = masm.framePushed();
michael@0: }
michael@0: 
michael@0: // Examine the cycle in moves starting at position i. Determine if it's a
michael@0: // simple cycle consisting of all register-to-register moves in a single class,
michael@0: // and whether it can be implemented entirely by swaps.
michael@0: size_t
michael@0: MoveEmitterX86::characterizeCycle(const MoveResolver &moves, size_t i,
michael@0:                                   bool *allGeneralRegs, bool *allFloatRegs)
michael@0: {
michael@0:     size_t swapCount = 0;
michael@0: 
michael@0:     for (size_t j = i; ; j++) {
michael@0:         const MoveOp &move = moves.getMove(j);
michael@0: 
michael@0:         // If it isn't a cycle of registers of the same kind, we won't be able
michael@0:         // to optimize it.
michael@0:         if (!move.to().isGeneralReg())
michael@0:             *allGeneralRegs = false;
michael@0:         if (!move.to().isFloatReg())
michael@0:             *allFloatRegs = false;
michael@0:         if (!*allGeneralRegs && !*allFloatRegs)
michael@0:             return -1;
michael@0: 
michael@0:         // Stop iterating when we see the last one.
michael@0:         if (j != i && move.isCycleEnd())
michael@0:             break;
michael@0: 
michael@0:         // Check that this move is actually part of the cycle. This is
michael@0:         // over-conservative when there are multiple reads from the same source,
michael@0:         // but that's expected to be rare.
michael@0:         if (move.from() != moves.getMove(j + 1).to()) {
michael@0:             *allGeneralRegs = false;
michael@0:             *allFloatRegs = false;
michael@0:             return -1;
michael@0:         }
michael@0: 
michael@0:         swapCount++;
michael@0:     }
michael@0: 
michael@0:     // Check that the last move cycles back to the first move.
michael@0:     const MoveOp &move = moves.getMove(i + swapCount);
michael@0:     if (move.from() != moves.getMove(i).to()) {
michael@0:         *allGeneralRegs = false;
michael@0:         *allFloatRegs = false;
michael@0:         return -1;
michael@0:     }
michael@0: 
michael@0:     return swapCount;
michael@0: }
michael@0: 
michael@0: // If we can emit optimized code for the cycle in moves starting at position i,
michael@0: // do so, and return true.
michael@0: bool
michael@0: MoveEmitterX86::maybeEmitOptimizedCycle(const MoveResolver &moves, size_t i,
michael@0:                                         bool allGeneralRegs, bool allFloatRegs, size_t swapCount)
michael@0: {
michael@0:     if (allGeneralRegs && swapCount <= 2) {
michael@0:         // Use x86's swap-integer-registers instruction if we only have a few
michael@0:         // swaps. (x86 also has a swap between registers and memory but it's
michael@0:         // slow.)
michael@0:         for (size_t k = 0; k < swapCount; k++)
michael@0:             masm.xchg(moves.getMove(i + k).to().reg(), moves.getMove(i + k + 1).to().reg());
michael@0:         return true;
michael@0:     }
michael@0: 
michael@0:     if (allFloatRegs && swapCount == 1) {
michael@0:         // There's no xchg for xmm registers, but if we only need a single swap,
michael@0:         // it's cheap to do an XOR swap.
michael@0:         FloatRegister a = moves.getMove(i).to().floatReg();
michael@0:         FloatRegister b = moves.getMove(i + 1).to().floatReg();
michael@0:         masm.xorpd(a, b);
michael@0:         masm.xorpd(b, a);
michael@0:         masm.xorpd(a, b);
michael@0:         return true;
michael@0:     }
michael@0: 
michael@0:     return false;
michael@0: }
michael@0: 
michael@0: void
michael@0: MoveEmitterX86::emit(const MoveResolver &moves)
michael@0: {
michael@0:     for (size_t i = 0; i < moves.numMoves(); i++) {
michael@0:         const MoveOp &move = moves.getMove(i);
michael@0:         const MoveOperand &from = move.from();
michael@0:         const MoveOperand &to = move.to();
michael@0: 
michael@0:         if (move.isCycleEnd()) {
michael@0:             JS_ASSERT(inCycle_);
michael@0:             completeCycle(to, move.type());
michael@0:             inCycle_ = false;
michael@0:             continue;
michael@0:         }
michael@0: 
michael@0:         if (move.isCycleBegin()) {
michael@0:             JS_ASSERT(!inCycle_);
michael@0: 
michael@0:             // Characterize the cycle.
michael@0:             bool allGeneralRegs = true, allFloatRegs = true;
michael@0:             size_t swapCount = characterizeCycle(moves, i, &allGeneralRegs, &allFloatRegs);
michael@0: 
michael@0:             // Attempt to optimize it to avoid using the stack.
michael@0:             if (maybeEmitOptimizedCycle(moves, i, allGeneralRegs, allFloatRegs, swapCount)) {
michael@0:                 i += swapCount;
michael@0:                 continue;
michael@0:             }
michael@0: 
michael@0:             // Otherwise use the stack.
michael@0:             breakCycle(to, move.endCycleType());
michael@0:             inCycle_ = true;
michael@0:         }
michael@0: 
michael@0:         // A normal move which is not part of a cycle.
michael@0:         switch (move.type()) {
michael@0:           case MoveOp::FLOAT32:
michael@0:             emitFloat32Move(from, to);
michael@0:             break;
michael@0:           case MoveOp::DOUBLE:
michael@0:             emitDoubleMove(from, to);
michael@0:             break;
michael@0:           case MoveOp::INT32:
michael@0:             emitInt32Move(from, to);
michael@0:             break;
michael@0:           case MoveOp::GENERAL:
michael@0:             emitGeneralMove(from, to);
michael@0:             break;
michael@0:           default:
michael@0:             MOZ_ASSUME_UNREACHABLE("Unexpected move type");
michael@0:         }
michael@0:     }
michael@0: }
michael@0: 
michael@0: MoveEmitterX86::~MoveEmitterX86()
michael@0: {
michael@0:     assertDone();
michael@0: }
michael@0: 
michael@0: Address
michael@0: MoveEmitterX86::cycleSlot()
michael@0: {
michael@0:     if (pushedAtCycle_ == -1) {
michael@0:         // Reserve stack for cycle resolution
michael@0:         masm.reserveStack(sizeof(double));
michael@0:         pushedAtCycle_ = masm.framePushed();
michael@0:     }
michael@0: 
michael@0:     return Address(StackPointer, masm.framePushed() - pushedAtCycle_);
michael@0: }
michael@0: 
michael@0: Address
michael@0: MoveEmitterX86::toAddress(const MoveOperand &operand) const
michael@0: {
michael@0:     if (operand.base() != StackPointer)
michael@0:         return Address(operand.base(), operand.disp());
michael@0: 
michael@0:     JS_ASSERT(operand.disp() >= 0);
michael@0: 
michael@0:     // Otherwise, the stack offset may need to be adjusted.
michael@0:     return Address(StackPointer, operand.disp() + (masm.framePushed() - pushedAtStart_));
michael@0: }
michael@0: 
michael@0: // Warning, do not use the resulting operand with pop instructions, since they
michael@0: // compute the effective destination address after altering the stack pointer.
michael@0: // Use toPopOperand if an Operand is needed for a pop.
michael@0: Operand
michael@0: MoveEmitterX86::toOperand(const MoveOperand &operand) const
michael@0: {
michael@0:     if (operand.isMemoryOrEffectiveAddress())
michael@0:         return Operand(toAddress(operand));
michael@0:     if (operand.isGeneralReg())
michael@0:         return Operand(operand.reg());
michael@0: 
michael@0:     JS_ASSERT(operand.isFloatReg());
michael@0:     return Operand(operand.floatReg());
michael@0: }
michael@0: 
michael@0: // This is the same as toOperand except that it computes an Operand suitable for
michael@0: // use in a pop.
michael@0: Operand
michael@0: MoveEmitterX86::toPopOperand(const MoveOperand &operand) const
michael@0: {
michael@0:     if (operand.isMemory()) {
michael@0:         if (operand.base() != StackPointer)
michael@0:             return Operand(operand.base(), operand.disp());
michael@0: 
michael@0:         JS_ASSERT(operand.disp() >= 0);
michael@0: 
michael@0:         // Otherwise, the stack offset may need to be adjusted.
michael@0:         // Note the adjustment by the stack slot here, to offset for the fact that pop
michael@0:         // computes its effective address after incrementing the stack pointer.
michael@0:         return Operand(StackPointer,
michael@0:                        operand.disp() + (masm.framePushed() - sizeof(void *) - pushedAtStart_));
michael@0:     }
michael@0:     if (operand.isGeneralReg())
michael@0:         return Operand(operand.reg());
michael@0: 
michael@0:     JS_ASSERT(operand.isFloatReg());
michael@0:     return Operand(operand.floatReg());
michael@0: }
michael@0: 
michael@0: void
michael@0: MoveEmitterX86::breakCycle(const MoveOperand &to, MoveOp::Type type)
michael@0: {
michael@0:     // There is some pattern:
michael@0:     //   (A -> B)
michael@0:     //   (B -> A)
michael@0:     //
michael@0:     // This case handles (A -> B), which we reach first. We save B, then allow
michael@0:     // the original move to continue.
michael@0:     switch (type) {
michael@0:       case MoveOp::FLOAT32:
michael@0:         if (to.isMemory()) {
michael@0:             masm.loadFloat32(toAddress(to), ScratchFloatReg);
michael@0:             masm.storeFloat32(ScratchFloatReg, cycleSlot());
michael@0:         } else {
michael@0:             masm.storeFloat32(to.floatReg(), cycleSlot());
michael@0:         }
michael@0:         break;
michael@0:       case MoveOp::DOUBLE:
michael@0:         if (to.isMemory()) {
michael@0:             masm.loadDouble(toAddress(to), ScratchFloatReg);
michael@0:             masm.storeDouble(ScratchFloatReg, cycleSlot());
michael@0:         } else {
michael@0:             masm.storeDouble(to.floatReg(), cycleSlot());
michael@0:         }
michael@0:         break;
michael@0: #ifdef JS_CODEGEN_X64
michael@0:       case MoveOp::INT32:
michael@0:         // x64 can't pop to a 32-bit destination, so don't push.
michael@0:         if (to.isMemory()) {
michael@0:             masm.load32(toAddress(to), ScratchReg);
michael@0:             masm.store32(ScratchReg, cycleSlot());
michael@0:         } else {
michael@0:             masm.store32(to.reg(), cycleSlot());
michael@0:         }
michael@0:         break;
michael@0: #endif
michael@0: #ifndef JS_CODEGEN_X64
michael@0:       case MoveOp::INT32:
michael@0: #endif
michael@0:       case MoveOp::GENERAL:
michael@0:         masm.Push(toOperand(to));
michael@0:         break;
michael@0:       default:
michael@0:         MOZ_ASSUME_UNREACHABLE("Unexpected move type");
michael@0:     }
michael@0: }
michael@0: 
michael@0: void
michael@0: MoveEmitterX86::completeCycle(const MoveOperand &to, MoveOp::Type type)
michael@0: {
michael@0:     // There is some pattern:
michael@0:     //   (A -> B)
michael@0:     //   (B -> A)
michael@0:     //
michael@0:     // This case handles (B -> A), which we reach last. We emit a move from the
michael@0:     // saved value of B, to A.
michael@0:     switch (type) {
michael@0:       case MoveOp::FLOAT32:
michael@0:         JS_ASSERT(pushedAtCycle_ != -1);
michael@0:         JS_ASSERT(pushedAtCycle_ - pushedAtStart_ >= sizeof(float));
michael@0:         if (to.isMemory()) {
michael@0:             masm.loadFloat32(cycleSlot(), ScratchFloatReg);
michael@0:             masm.storeFloat32(ScratchFloatReg, toAddress(to));
michael@0:         } else {
michael@0:             masm.loadFloat32(cycleSlot(), to.floatReg());
michael@0:         }
michael@0:         break;
michael@0:       case MoveOp::DOUBLE:
michael@0:         JS_ASSERT(pushedAtCycle_ != -1);
michael@0:         JS_ASSERT(pushedAtCycle_ - pushedAtStart_ >= sizeof(double));
michael@0:         if (to.isMemory()) {
michael@0:             masm.loadDouble(cycleSlot(), ScratchFloatReg);
michael@0:             masm.storeDouble(ScratchFloatReg, toAddress(to));
michael@0:         } else {
michael@0:             masm.loadDouble(cycleSlot(), to.floatReg());
michael@0:         }
michael@0:         break;
michael@0: #ifdef JS_CODEGEN_X64
michael@0:       case MoveOp::INT32:
michael@0:         JS_ASSERT(pushedAtCycle_ != -1);
michael@0:         JS_ASSERT(pushedAtCycle_ - pushedAtStart_ >= sizeof(int32_t));
michael@0:         // x64 can't pop to a 32-bit destination.
michael@0:         if (to.isMemory()) {
michael@0:             masm.load32(cycleSlot(), ScratchReg);
michael@0:             masm.store32(ScratchReg, toAddress(to));
michael@0:         } else {
michael@0:             masm.load32(cycleSlot(), to.reg());
michael@0:         }
michael@0:         break;
michael@0: #endif
michael@0: #ifndef JS_CODEGEN_X64
michael@0:       case MoveOp::INT32:
michael@0: #endif
michael@0:       case MoveOp::GENERAL:
michael@0:         JS_ASSERT(masm.framePushed() - pushedAtStart_ >= sizeof(intptr_t));
michael@0:         masm.Pop(toPopOperand(to));
michael@0:         break;
michael@0:       default:
michael@0:         MOZ_ASSUME_UNREACHABLE("Unexpected move type");
michael@0:     }
michael@0: }
michael@0: 
michael@0: void
michael@0: MoveEmitterX86::emitInt32Move(const MoveOperand &from, const MoveOperand &to)
michael@0: {
michael@0:     if (from.isGeneralReg()) {
michael@0:         masm.move32(from.reg(), toOperand(to));
michael@0:     } else if (to.isGeneralReg()) {
michael@0:         JS_ASSERT(from.isMemory());
michael@0:         masm.load32(toAddress(from), to.reg());
michael@0:     } else {
michael@0:         // Memory to memory gpr move.
michael@0:         JS_ASSERT(from.isMemory());
michael@0: #ifdef JS_CODEGEN_X64
michael@0:         // x64 has a ScratchReg. Use it.
michael@0:         masm.load32(toAddress(from), ScratchReg);
michael@0:         masm.move32(ScratchReg, toOperand(to));
michael@0: #else
michael@0:         // No ScratchReg; bounce it off the stack.
michael@0:         masm.Push(toOperand(from));
michael@0:         masm.Pop(toPopOperand(to));
michael@0: #endif
michael@0:     }
michael@0: }
michael@0: 
michael@0: void
michael@0: MoveEmitterX86::emitGeneralMove(const MoveOperand &from, const MoveOperand &to)
michael@0: {
michael@0:     if (from.isGeneralReg()) {
michael@0:         masm.mov(from.reg(), toOperand(to));
michael@0:     } else if (to.isGeneralReg()) {
michael@0:         JS_ASSERT(from.isMemoryOrEffectiveAddress());
michael@0:         if (from.isMemory())
michael@0:             masm.loadPtr(toAddress(from), to.reg());
michael@0:         else
michael@0:             masm.lea(toOperand(from), to.reg());
michael@0:     } else if (from.isMemory()) {
michael@0:         // Memory to memory gpr move.
michael@0: #ifdef JS_CODEGEN_X64
michael@0:         // x64 has a ScratchReg. Use it.
michael@0:         masm.loadPtr(toAddress(from), ScratchReg);
michael@0:         masm.mov(ScratchReg, toOperand(to));
michael@0: #else
michael@0:         // No ScratchReg; bounce it off the stack.
michael@0:         masm.Push(toOperand(from));
michael@0:         masm.Pop(toPopOperand(to));
michael@0: #endif
michael@0:     } else {
michael@0:         // Effective address to memory move.
michael@0:         JS_ASSERT(from.isEffectiveAddress());
michael@0: #ifdef JS_CODEGEN_X64
michael@0:         // x64 has a ScratchReg. Use it.
michael@0:         masm.lea(toOperand(from), ScratchReg);
michael@0:         masm.mov(ScratchReg, toOperand(to));
michael@0: #else
michael@0:         // This is tricky without a ScratchReg. We can't do an lea. Bounce the
michael@0:         // base register off the stack, then add the offset in place. Note that
michael@0:         // this clobbers FLAGS!
michael@0:         masm.Push(from.base());
michael@0:         masm.Pop(toPopOperand(to));
michael@0:         masm.addPtr(Imm32(from.disp()), toOperand(to));
michael@0: #endif
michael@0:     }
michael@0: }
michael@0: 
michael@0: void
michael@0: MoveEmitterX86::emitFloat32Move(const MoveOperand &from, const MoveOperand &to)
michael@0: {
michael@0:     if (from.isFloatReg()) {
michael@0:         if (to.isFloatReg())
michael@0:             masm.moveFloat32(from.floatReg(), to.floatReg());
michael@0:         else
michael@0:             masm.storeFloat32(from.floatReg(), toAddress(to));
michael@0:     } else if (to.isFloatReg()) {
michael@0:         masm.loadFloat32(toAddress(from), to.floatReg());
michael@0:     } else {
michael@0:         // Memory to memory move.
michael@0:         JS_ASSERT(from.isMemory());
michael@0:         masm.loadFloat32(toAddress(from), ScratchFloatReg);
michael@0:         masm.storeFloat32(ScratchFloatReg, toAddress(to));
michael@0:     }
michael@0: }
michael@0: 
michael@0: void
michael@0: MoveEmitterX86::emitDoubleMove(const MoveOperand &from, const MoveOperand &to)
michael@0: {
michael@0:     if (from.isFloatReg()) {
michael@0:         if (to.isFloatReg())
michael@0:             masm.moveDouble(from.floatReg(), to.floatReg());
michael@0:         else
michael@0:             masm.storeDouble(from.floatReg(), toAddress(to));
michael@0:     } else if (to.isFloatReg()) {
michael@0:         masm.loadDouble(toAddress(from), to.floatReg());
michael@0:     } else {
michael@0:         // Memory to memory move.
michael@0:         JS_ASSERT(from.isMemory());
michael@0:         masm.loadDouble(toAddress(from), ScratchFloatReg);
michael@0:         masm.storeDouble(ScratchFloatReg, toAddress(to));
michael@0:     }
michael@0: }
michael@0: 
michael@0: void
michael@0: MoveEmitterX86::assertDone()
michael@0: {
michael@0:     JS_ASSERT(!inCycle_);
michael@0: }
michael@0: 
michael@0: void
michael@0: MoveEmitterX86::finish()
michael@0: {
michael@0:     assertDone();
michael@0: 
michael@0:     masm.freeStack(masm.framePushed() - pushedAtStart_);
michael@0: }
michael@0: