1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/jit/arm/Trampoline-arm.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,964 @@
1.4 +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
1.5 + * vim: set ts=8 sts=4 et sw=4 tw=99:
1.6 + * This Source Code Form is subject to the terms of the Mozilla Public
1.7 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.8 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.9 +
1.10 +#include "jscompartment.h"
1.11 +
1.12 +#include "assembler/assembler/MacroAssembler.h"
1.13 +#include "jit/arm/BaselineHelpers-arm.h"
1.14 +#include "jit/Bailouts.h"
1.15 +#include "jit/IonFrames.h"
1.16 +#include "jit/IonLinker.h"
1.17 +#include "jit/IonSpewer.h"
1.18 +#include "jit/JitCompartment.h"
1.19 +#ifdef JS_ION_PERF
1.20 +# include "jit/PerfSpewer.h"
1.21 +#endif
1.22 +#include "jit/VMFunctions.h"
1.23 +
1.24 +#include "jit/ExecutionMode-inl.h"
1.25 +
1.26 +using namespace js;
1.27 +using namespace js::jit;
1.28 +
1.29 +static const FloatRegisterSet NonVolatileFloatRegs =
1.30 + FloatRegisterSet((1 << FloatRegisters::d8) |
1.31 + (1 << FloatRegisters::d9) |
1.32 + (1 << FloatRegisters::d10) |
1.33 + (1 << FloatRegisters::d11) |
1.34 + (1 << FloatRegisters::d12) |
1.35 + (1 << FloatRegisters::d13) |
1.36 + (1 << FloatRegisters::d14) |
1.37 + (1 << FloatRegisters::d15));
1.38 +
1.39 +static void
1.40 +GenerateReturn(MacroAssembler &masm, int returnCode, SPSProfiler *prof)
1.41 +{
1.42 + // Restore non-volatile floating point registers
1.43 + masm.transferMultipleByRuns(NonVolatileFloatRegs, IsLoad, StackPointer, IA);
1.44 +
1.45 + // Unwind the sps mark.
1.46 + masm.spsUnmarkJit(prof, r8);
1.47 +
1.48 + // Set up return value
1.49 + masm.ma_mov(Imm32(returnCode), r0);
1.50 +
1.51 + // Pop and return
1.52 + masm.startDataTransferM(IsLoad, sp, IA, WriteBack);
1.53 + masm.transferReg(r4);
1.54 + masm.transferReg(r5);
1.55 + masm.transferReg(r6);
1.56 + masm.transferReg(r7);
1.57 + masm.transferReg(r8);
1.58 + masm.transferReg(r9);
1.59 + masm.transferReg(r10);
1.60 + masm.transferReg(r11);
1.61 + // r12 isn't saved, so it shouldn't be restored.
1.62 + masm.transferReg(pc);
1.63 + masm.finishDataTransfer();
1.64 + masm.dumpPool();
1.65 +}
1.66 +
1.67 +struct EnterJITStack
1.68 +{
1.69 + double d8;
1.70 + double d9;
1.71 + double d10;
1.72 + double d11;
1.73 + double d12;
1.74 + double d13;
1.75 + double d14;
1.76 + double d15;
1.77 +
1.78 + size_t hasSPSMark;
1.79 +
1.80 + // non-volatile registers.
1.81 + void *r4;
1.82 + void *r5;
1.83 + void *r6;
1.84 + void *r7;
1.85 + void *r8;
1.86 + void *r9;
1.87 + void *r10;
1.88 + void *r11;
1.89 + // The abi does not expect r12 (ip) to be preserved
1.90 + void *lr;
1.91 +
1.92 + // Arguments.
1.93 + // code == r0
1.94 + // argc == r1
1.95 + // argv == r2
1.96 + // frame == r3
1.97 + CalleeToken token;
1.98 + JSObject *scopeChain;
1.99 + size_t numStackValues;
1.100 + Value *vp;
1.101 +};
1.102 +
1.103 +/*
1.104 + * This method generates a trampoline for a c++ function with the following
1.105 + * signature:
1.106 + * void enter(void *code, int argc, Value *argv, InterpreterFrame *fp, CalleeToken
1.107 + * calleeToken, JSObject *scopeChain, Value *vp)
1.108 + * ...using standard EABI calling convention
1.109 + */
1.110 +JitCode *
1.111 +JitRuntime::generateEnterJIT(JSContext *cx, EnterJitType type)
1.112 +{
1.113 + const Address slot_token(sp, offsetof(EnterJITStack, token));
1.114 + const Address slot_vp(sp, offsetof(EnterJITStack, vp));
1.115 +
1.116 + JS_ASSERT(OsrFrameReg == r3);
1.117 +
1.118 + MacroAssembler masm(cx);
1.119 + Assembler *aasm = &masm;
1.120 +
1.121 + // Save non-volatile registers. These must be saved by the trampoline,
1.122 + // rather than the JIT'd code, because they are scanned by the conservative
1.123 + // scanner.
1.124 + masm.startDataTransferM(IsStore, sp, DB, WriteBack);
1.125 + masm.transferReg(r4); // [sp,0]
1.126 + masm.transferReg(r5); // [sp,4]
1.127 + masm.transferReg(r6); // [sp,8]
1.128 + masm.transferReg(r7); // [sp,12]
1.129 + masm.transferReg(r8); // [sp,16]
1.130 + masm.transferReg(r9); // [sp,20]
1.131 + masm.transferReg(r10); // [sp,24]
1.132 + masm.transferReg(r11); // [sp,28]
1.133 + // The abi does not expect r12 (ip) to be preserved
1.134 + masm.transferReg(lr); // [sp,32]
1.135 + // The 5th argument is located at [sp, 36]
1.136 + masm.finishDataTransfer();
1.137 +
1.138 + // Push the EnterJIT sps mark. "Frame pointer" = start of saved core regs.
1.139 + masm.movePtr(sp, r8);
1.140 + masm.spsMarkJit(&cx->runtime()->spsProfiler, r8, r9);
1.141 +
1.142 + // Push the float registers.
1.143 + masm.transferMultipleByRuns(NonVolatileFloatRegs, IsStore, sp, DB);
1.144 +
1.145 + // Save stack pointer into r8
1.146 + masm.movePtr(sp, r8);
1.147 +
1.148 + // Load calleeToken into r9.
1.149 + masm.loadPtr(slot_token, r9);
1.150 +
1.151 + // Save stack pointer.
1.152 + if (type == EnterJitBaseline)
1.153 + masm.movePtr(sp, r11);
1.154 +
1.155 + // Load the number of actual arguments into r10.
1.156 + masm.loadPtr(slot_vp, r10);
1.157 + masm.unboxInt32(Address(r10, 0), r10);
1.158 +
1.159 + // Subtract off the size of the arguments from the stack pointer, store elsewhere
1.160 + aasm->as_sub(r4, sp, O2RegImmShift(r1, LSL, 3)); //r4 = sp - argc*8
1.161 + // Get the final position of the stack pointer into the stack pointer
1.162 + aasm->as_sub(sp, r4, Imm8(16)); // sp' = sp - argc*8 - 16
1.163 + // Get a copy of the number of args to use as a decrement counter, also
1.164 + // Set the zero condition code
1.165 + aasm->as_mov(r5, O2Reg(r1), SetCond);
1.166 +
1.167 + // Loop over arguments, copying them from an unknown buffer onto the Ion
1.168 + // stack so they can be accessed from JIT'ed code.
1.169 + {
1.170 + Label header, footer;
1.171 + // If there aren't any arguments, don't do anything
1.172 + aasm->as_b(&footer, Assembler::Zero);
1.173 + // Get the top of the loop
1.174 + masm.bind(&header);
1.175 + aasm->as_sub(r5, r5, Imm8(1), SetCond);
1.176 + // We could be more awesome, and unroll this, using a loadm
1.177 + // (particularly since the offset is effectively 0)
1.178 + // but that seems more error prone, and complex.
1.179 + // BIG FAT WARNING: this loads both r6 and r7.
1.180 + aasm->as_extdtr(IsLoad, 64, true, PostIndex, r6, EDtrAddr(r2, EDtrOffImm(8)));
1.181 + aasm->as_extdtr(IsStore, 64, true, PostIndex, r6, EDtrAddr(r4, EDtrOffImm(8)));
1.182 + aasm->as_b(&header, Assembler::NonZero);
1.183 + masm.bind(&footer);
1.184 + }
1.185 +
1.186 + masm.ma_sub(r8, sp, r8);
1.187 + masm.makeFrameDescriptor(r8, JitFrame_Entry);
1.188 +
1.189 + masm.startDataTransferM(IsStore, sp, IB, NoWriteBack);
1.190 + // [sp] = return address (written later)
1.191 + masm.transferReg(r8); // [sp',4] = descriptor, argc*8+20
1.192 + masm.transferReg(r9); // [sp',8] = callee token
1.193 + masm.transferReg(r10); // [sp',12] = actual arguments
1.194 + masm.finishDataTransfer();
1.195 +
1.196 + Label returnLabel;
1.197 + if (type == EnterJitBaseline) {
1.198 + // Handle OSR.
1.199 + GeneralRegisterSet regs(GeneralRegisterSet::All());
1.200 + regs.take(JSReturnOperand);
1.201 + regs.takeUnchecked(OsrFrameReg);
1.202 + regs.take(r11);
1.203 + regs.take(ReturnReg);
1.204 +
1.205 + const Address slot_numStackValues(r11, offsetof(EnterJITStack, numStackValues));
1.206 +
1.207 + Label notOsr;
1.208 + masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr);
1.209 +
1.210 + Register scratch = regs.takeAny();
1.211 +
1.212 + Register numStackValues = regs.takeAny();
1.213 + masm.load32(slot_numStackValues, numStackValues);
1.214 +
1.215 + // Write return address. On ARM, CodeLabel is only used for tableswitch,
1.216 + // so we can't use it here to get the return address. Instead, we use
1.217 + // pc + a fixed offset to a jump to returnLabel. The pc register holds
1.218 + // pc + 8, so we add the size of 2 instructions to skip the instructions
1.219 + // emitted by storePtr and jump(&skipJump).
1.220 + {
1.221 + AutoForbidPools afp(&masm);
1.222 + Label skipJump;
1.223 + masm.mov(pc, scratch);
1.224 + masm.addPtr(Imm32(2 * sizeof(uint32_t)), scratch);
1.225 + masm.storePtr(scratch, Address(sp, 0));
1.226 + masm.jump(&skipJump);
1.227 + masm.jump(&returnLabel);
1.228 + masm.bind(&skipJump);
1.229 + }
1.230 +
1.231 + // Push previous frame pointer.
1.232 + masm.push(r11);
1.233 +
1.234 + // Reserve frame.
1.235 + Register framePtr = r11;
1.236 + masm.subPtr(Imm32(BaselineFrame::Size()), sp);
1.237 + masm.mov(sp, framePtr);
1.238 +
1.239 +#ifdef XP_WIN
1.240 + // Can't push large frames blindly on windows. Touch frame memory incrementally.
1.241 + masm.ma_lsl(Imm32(3), numStackValues, scratch);
1.242 + masm.subPtr(scratch, framePtr);
1.243 + {
1.244 + masm.ma_sub(sp, Imm32(WINDOWS_BIG_FRAME_TOUCH_INCREMENT), scratch);
1.245 +
1.246 + Label touchFrameLoop;
1.247 + Label touchFrameLoopEnd;
1.248 + masm.bind(&touchFrameLoop);
1.249 + masm.branchPtr(Assembler::Below, scratch, framePtr, &touchFrameLoopEnd);
1.250 + masm.store32(Imm32(0), Address(scratch, 0));
1.251 + masm.subPtr(Imm32(WINDOWS_BIG_FRAME_TOUCH_INCREMENT), scratch);
1.252 + masm.jump(&touchFrameLoop);
1.253 + masm.bind(&touchFrameLoopEnd);
1.254 + }
1.255 + masm.mov(sp, framePtr);
1.256 +#endif
1.257 +
1.258 + // Reserve space for locals and stack values.
1.259 + masm.ma_lsl(Imm32(3), numStackValues, scratch);
1.260 + masm.ma_sub(sp, scratch, sp);
1.261 +
1.262 + // Enter exit frame.
1.263 + masm.addPtr(Imm32(BaselineFrame::Size() + BaselineFrame::FramePointerOffset), scratch);
1.264 + masm.makeFrameDescriptor(scratch, JitFrame_BaselineJS);
1.265 + masm.push(scratch);
1.266 + masm.push(Imm32(0)); // Fake return address.
1.267 + masm.enterFakeExitFrame();
1.268 +
1.269 + masm.push(framePtr); // BaselineFrame
1.270 + masm.push(r0); // jitcode
1.271 +
1.272 + masm.setupUnalignedABICall(3, scratch);
1.273 + masm.passABIArg(r11); // BaselineFrame
1.274 + masm.passABIArg(OsrFrameReg); // InterpreterFrame
1.275 + masm.passABIArg(numStackValues);
1.276 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, jit::InitBaselineFrameForOsr));
1.277 +
1.278 + Register jitcode = regs.takeAny();
1.279 + masm.pop(jitcode);
1.280 + masm.pop(framePtr);
1.281 +
1.282 + JS_ASSERT(jitcode != ReturnReg);
1.283 +
1.284 + Label error;
1.285 + masm.addPtr(Imm32(IonExitFrameLayout::SizeWithFooter()), sp);
1.286 + masm.addPtr(Imm32(BaselineFrame::Size()), framePtr);
1.287 + masm.branchIfFalseBool(ReturnReg, &error);
1.288 +
1.289 + masm.jump(jitcode);
1.290 +
1.291 + // OOM: load error value, discard return address and previous frame
1.292 + // pointer and return.
1.293 + masm.bind(&error);
1.294 + masm.mov(framePtr, sp);
1.295 + masm.addPtr(Imm32(2 * sizeof(uintptr_t)), sp);
1.296 + masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
1.297 + masm.jump(&returnLabel);
1.298 +
1.299 + masm.bind(¬Osr);
1.300 + // Load the scope chain in R1.
1.301 + JS_ASSERT(R1.scratchReg() != r0);
1.302 + masm.loadPtr(Address(r11, offsetof(EnterJITStack, scopeChain)), R1.scratchReg());
1.303 + }
1.304 +
1.305 + // Call the function.
1.306 + masm.ma_callIonNoPush(r0);
1.307 +
1.308 + if (type == EnterJitBaseline) {
1.309 + // Baseline OSR will return here.
1.310 + masm.bind(&returnLabel);
1.311 + }
1.312 +
1.313 + // The top of the stack now points to the address of the field following
1.314 + // the return address because the return address is popped for the
1.315 + // return, so we need to remove the size of the return address field.
1.316 + aasm->as_sub(sp, sp, Imm8(4));
1.317 +
1.318 + // Load off of the stack the size of our local stack
1.319 + masm.loadPtr(Address(sp, IonJSFrameLayout::offsetOfDescriptor()), r5);
1.320 + aasm->as_add(sp, sp, lsr(r5, FRAMESIZE_SHIFT));
1.321 +
1.322 + // Store the returned value into the slot_vp
1.323 + masm.loadPtr(slot_vp, r5);
1.324 + masm.storeValue(JSReturnOperand, Address(r5, 0));
1.325 +
1.326 + // :TODO: Optimize storeValue with:
1.327 + // We're using a load-double here. In order for that to work,
1.328 + // the data needs to be stored in two consecutive registers,
1.329 + // make sure this is the case
1.330 + // ASSERT(JSReturnReg_Type.code() == JSReturnReg_Data.code()+1);
1.331 + // aasm->as_extdtr(IsStore, 64, true, Offset,
1.332 + // JSReturnReg_Data, EDtrAddr(r5, EDtrOffImm(0)));
1.333 +
1.334 + // Restore non-volatile registers and return.
1.335 + GenerateReturn(masm, true, &cx->runtime()->spsProfiler);
1.336 +
1.337 + Linker linker(masm);
1.338 + AutoFlushICache afc("EnterJIT");
1.339 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.340 +
1.341 +#ifdef JS_ION_PERF
1.342 + writePerfSpewerJitCodeProfile(code, "EnterJIT");
1.343 +#endif
1.344 +
1.345 + return code;
1.346 +}
1.347 +
1.348 +JitCode *
1.349 +JitRuntime::generateInvalidator(JSContext *cx)
1.350 +{
1.351 + // See large comment in x86's JitRuntime::generateInvalidator.
1.352 + MacroAssembler masm(cx);
1.353 + //masm.as_bkpt();
1.354 + // At this point, one of two things has happened.
1.355 + // 1) Execution has just returned from C code, which left the stack aligned
1.356 + // 2) Execution has just returned from Ion code, which left the stack unaligned.
1.357 + // The old return address should not matter, but we still want the
1.358 + // stack to be aligned, and there is no good reason to automatically align it with
1.359 + // a call to setupUnalignedABICall.
1.360 + masm.ma_and(Imm32(~7), sp, sp);
1.361 + masm.startDataTransferM(IsStore, sp, DB, WriteBack);
1.362 + // We don't have to push everything, but this is likely easier.
1.363 + // setting regs_
1.364 + for (uint32_t i = 0; i < Registers::Total; i++)
1.365 + masm.transferReg(Register::FromCode(i));
1.366 + masm.finishDataTransfer();
1.367 +
1.368 + masm.startFloatTransferM(IsStore, sp, DB, WriteBack);
1.369 + for (uint32_t i = 0; i < FloatRegisters::Total; i++)
1.370 + masm.transferFloatReg(FloatRegister::FromCode(i));
1.371 + masm.finishFloatTransfer();
1.372 +
1.373 + masm.ma_mov(sp, r0);
1.374 + const int sizeOfRetval = sizeof(size_t)*2;
1.375 + masm.reserveStack(sizeOfRetval);
1.376 + masm.mov(sp, r1);
1.377 + const int sizeOfBailoutInfo = sizeof(void *)*2;
1.378 + masm.reserveStack(sizeOfBailoutInfo);
1.379 + masm.mov(sp, r2);
1.380 + masm.setupAlignedABICall(3);
1.381 + masm.passABIArg(r0);
1.382 + masm.passABIArg(r1);
1.383 + masm.passABIArg(r2);
1.384 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, InvalidationBailout));
1.385 +
1.386 + masm.ma_ldr(Address(sp, 0), r2);
1.387 + masm.ma_ldr(Address(sp, sizeOfBailoutInfo), r1);
1.388 + // Remove the return address, the IonScript, the register state
1.389 + // (InvaliationBailoutStack) and the space that was allocated for the return value
1.390 + masm.ma_add(sp, Imm32(sizeof(InvalidationBailoutStack) + sizeOfRetval + sizeOfBailoutInfo), sp);
1.391 + // remove the space that this frame was using before the bailout
1.392 + // (computed by InvalidationBailout)
1.393 + masm.ma_add(sp, r1, sp);
1.394 +
1.395 + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
1.396 + JitCode *bailoutTail = cx->runtime()->jitRuntime()->getBailoutTail();
1.397 + masm.branch(bailoutTail);
1.398 +
1.399 + Linker linker(masm);
1.400 + AutoFlushICache afc("Invalidator");
1.401 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.402 + IonSpew(IonSpew_Invalidate, " invalidation thunk created at %p", (void *) code->raw());
1.403 +
1.404 +#ifdef JS_ION_PERF
1.405 + writePerfSpewerJitCodeProfile(code, "Invalidator");
1.406 +#endif
1.407 +
1.408 + return code;
1.409 +}
1.410 +
1.411 +JitCode *
1.412 +JitRuntime::generateArgumentsRectifier(JSContext *cx, ExecutionMode mode, void **returnAddrOut)
1.413 +{
1.414 + MacroAssembler masm(cx);
1.415 + // ArgumentsRectifierReg contains the |nargs| pushed onto the current frame.
1.416 + // Including |this|, there are (|nargs| + 1) arguments to copy.
1.417 + JS_ASSERT(ArgumentsRectifierReg == r8);
1.418 +
1.419 + // Copy number of actual arguments into r0
1.420 + masm.ma_ldr(DTRAddr(sp, DtrOffImm(IonRectifierFrameLayout::offsetOfNumActualArgs())), r0);
1.421 +
1.422 + // Load the number of |undefined|s to push into r6.
1.423 + masm.ma_ldr(DTRAddr(sp, DtrOffImm(IonRectifierFrameLayout::offsetOfCalleeToken())), r1);
1.424 + masm.ma_ldrh(EDtrAddr(r1, EDtrOffImm(JSFunction::offsetOfNargs())), r6);
1.425 +
1.426 + masm.ma_sub(r6, r8, r2);
1.427 +
1.428 + masm.moveValue(UndefinedValue(), r5, r4);
1.429 +
1.430 + masm.ma_mov(sp, r3); // Save %sp.
1.431 + masm.ma_mov(sp, r7); // Save %sp again.
1.432 +
1.433 + // Push undefined.
1.434 + {
1.435 + Label undefLoopTop;
1.436 + masm.bind(&undefLoopTop);
1.437 + masm.ma_dataTransferN(IsStore, 64, true, sp, Imm32(-8), r4, PreIndex);
1.438 + masm.ma_sub(r2, Imm32(1), r2, SetCond);
1.439 +
1.440 + masm.ma_b(&undefLoopTop, Assembler::NonZero);
1.441 + }
1.442 +
1.443 + // Get the topmost argument.
1.444 +
1.445 + masm.ma_alu(r3, lsl(r8, 3), r3, op_add); // r3 <- r3 + nargs * 8
1.446 + masm.ma_add(r3, Imm32(sizeof(IonRectifierFrameLayout)), r3);
1.447 +
1.448 + // Push arguments, |nargs| + 1 times (to include |this|).
1.449 + {
1.450 + Label copyLoopTop;
1.451 + masm.bind(©LoopTop);
1.452 + masm.ma_dataTransferN(IsLoad, 64, true, r3, Imm32(-8), r4, PostIndex);
1.453 + masm.ma_dataTransferN(IsStore, 64, true, sp, Imm32(-8), r4, PreIndex);
1.454 +
1.455 + masm.ma_sub(r8, Imm32(1), r8, SetCond);
1.456 + masm.ma_b(©LoopTop, Assembler::NotSigned);
1.457 + }
1.458 +
1.459 + // translate the framesize from values into bytes
1.460 + masm.ma_add(r6, Imm32(1), r6);
1.461 + masm.ma_lsl(Imm32(3), r6, r6);
1.462 +
1.463 + // Construct sizeDescriptor.
1.464 + masm.makeFrameDescriptor(r6, JitFrame_Rectifier);
1.465 +
1.466 + // Construct IonJSFrameLayout.
1.467 + masm.ma_push(r0); // actual arguments.
1.468 + masm.ma_push(r1); // callee token
1.469 + masm.ma_push(r6); // frame descriptor.
1.470 +
1.471 + // Call the target function.
1.472 + // Note that this code assumes the function is JITted.
1.473 + masm.ma_ldr(DTRAddr(r1, DtrOffImm(JSFunction::offsetOfNativeOrScript())), r3);
1.474 + masm.loadBaselineOrIonRaw(r3, r3, mode, nullptr);
1.475 + masm.ma_callIonHalfPush(r3);
1.476 +
1.477 + uint32_t returnOffset = masm.currentOffset();
1.478 +
1.479 + // arg1
1.480 + // ...
1.481 + // argN
1.482 + // num actual args
1.483 + // callee token
1.484 + // sizeDescriptor <- sp now
1.485 + // return address
1.486 +
1.487 + // Remove the rectifier frame.
1.488 + masm.ma_dtr(IsLoad, sp, Imm32(12), r4, PostIndex);
1.489 +
1.490 + // arg1
1.491 + // ...
1.492 + // argN <- sp now; r4 <- frame descriptor
1.493 + // num actual args
1.494 + // callee token
1.495 + // sizeDescriptor
1.496 + // return address
1.497 +
1.498 + // Discard pushed arguments.
1.499 + masm.ma_alu(sp, lsr(r4, FRAMESIZE_SHIFT), sp, op_add);
1.500 +
1.501 + masm.ret();
1.502 + Linker linker(masm);
1.503 + AutoFlushICache afc("ArgumentsRectifier");
1.504 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.505 +
1.506 + CodeOffsetLabel returnLabel(returnOffset);
1.507 + returnLabel.fixup(&masm);
1.508 + if (returnAddrOut)
1.509 + *returnAddrOut = (void *) (code->raw() + returnLabel.offset());
1.510 +
1.511 +#ifdef JS_ION_PERF
1.512 + writePerfSpewerJitCodeProfile(code, "ArgumentsRectifier");
1.513 +#endif
1.514 +
1.515 + return code;
1.516 +}
1.517 +
1.518 +static void
1.519 +GenerateBailoutThunk(JSContext *cx, MacroAssembler &masm, uint32_t frameClass)
1.520 +{
1.521 + // the stack should look like:
1.522 + // [IonFrame]
1.523 + // bailoutFrame.registersnapshot
1.524 + // bailoutFrame.fpsnapshot
1.525 + // bailoutFrame.snapshotOffset
1.526 + // bailoutFrame.frameSize
1.527 +
1.528 + // STEP 1a: save our register sets to the stack so Bailout() can
1.529 + // read everything.
1.530 + // sp % 8 == 0
1.531 + masm.startDataTransferM(IsStore, sp, DB, WriteBack);
1.532 + // We don't have to push everything, but this is likely easier.
1.533 + // setting regs_
1.534 + for (uint32_t i = 0; i < Registers::Total; i++)
1.535 + masm.transferReg(Register::FromCode(i));
1.536 + masm.finishDataTransfer();
1.537 +
1.538 + masm.startFloatTransferM(IsStore, sp, DB, WriteBack);
1.539 + for (uint32_t i = 0; i < FloatRegisters::Total; i++)
1.540 + masm.transferFloatReg(FloatRegister::FromCode(i));
1.541 + masm.finishFloatTransfer();
1.542 +
1.543 + // STEP 1b: Push both the "return address" of the function call (the
1.544 + // address of the instruction after the call that we used to get
1.545 + // here) as well as the callee token onto the stack. The return
1.546 + // address is currently in r14. We will proceed by loading the
1.547 + // callee token into a sacrificial register <= r14, then pushing
1.548 + // both onto the stack
1.549 +
1.550 + // now place the frameClass onto the stack, via a register
1.551 + masm.ma_mov(Imm32(frameClass), r4);
1.552 + // And onto the stack. Since the stack is full, we need to put this
1.553 + // one past the end of the current stack. Sadly, the ABI says that we need
1.554 + // to always point to the lowest place that has been written. the OS is
1.555 + // free to do whatever it wants below sp.
1.556 + masm.startDataTransferM(IsStore, sp, DB, WriteBack);
1.557 + // set frameClassId_
1.558 + masm.transferReg(r4);
1.559 + // Set tableOffset_; higher registers are stored at higher locations on
1.560 + // the stack.
1.561 + masm.transferReg(lr);
1.562 + masm.finishDataTransfer();
1.563 +
1.564 + // SP % 8 == 4
1.565 + // STEP 1c: Call the bailout function, giving a pointer to the
1.566 + // structure we just blitted onto the stack
1.567 + masm.ma_mov(sp, r0);
1.568 + const int sizeOfBailoutInfo = sizeof(void *)*2;
1.569 + masm.reserveStack(sizeOfBailoutInfo);
1.570 + masm.mov(sp, r1);
1.571 + masm.setupAlignedABICall(2);
1.572 +
1.573 + // Decrement sp by another 4, so we keep alignment
1.574 + // Not Anymore! pushing both the snapshotoffset as well as the
1.575 + // masm.as_sub(sp, sp, Imm8(4));
1.576 +
1.577 + // Set the old (4-byte aligned) value of the sp as the first argument
1.578 + masm.passABIArg(r0);
1.579 + masm.passABIArg(r1);
1.580 +
1.581 + // Sp % 8 == 0
1.582 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, Bailout));
1.583 + masm.ma_ldr(Address(sp, 0), r2);
1.584 + masm.ma_add(sp, Imm32(sizeOfBailoutInfo), sp);
1.585 + // Common size of a bailout frame.
1.586 + uint32_t bailoutFrameSize = sizeof(void *) + // frameClass
1.587 + sizeof(double) * FloatRegisters::Total +
1.588 + sizeof(void *) * Registers::Total;
1.589 +
1.590 + if (frameClass == NO_FRAME_SIZE_CLASS_ID) {
1.591 + // Make sure the bailout frame size fits into the offset for a load
1.592 + masm.as_dtr(IsLoad, 32, Offset,
1.593 + r4, DTRAddr(sp, DtrOffImm(4)));
1.594 + // used to be: offsetof(BailoutStack, frameSize_)
1.595 + // this structure is no longer available to us :(
1.596 + // We add 12 to the bailoutFrameSize because:
1.597 + // sizeof(uint32_t) for the tableOffset that was pushed onto the stack
1.598 + // sizeof(uintptr_t) for the snapshotOffset;
1.599 + // alignment to round the uintptr_t up to a multiple of 8 bytes.
1.600 + masm.ma_add(sp, Imm32(bailoutFrameSize+12), sp);
1.601 + masm.as_add(sp, sp, O2Reg(r4));
1.602 + } else {
1.603 + uint32_t frameSize = FrameSizeClass::FromClass(frameClass).frameSize();
1.604 + masm.ma_add(Imm32(frameSize // the frame that was added when we entered the most recent function
1.605 + + sizeof(void*) // the size of the "return address" that was dumped on the stack
1.606 + + bailoutFrameSize) // everything else that was pushed on the stack
1.607 + , sp);
1.608 + }
1.609 +
1.610 + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
1.611 + JitCode *bailoutTail = cx->runtime()->jitRuntime()->getBailoutTail();
1.612 + masm.branch(bailoutTail);
1.613 +}
1.614 +
1.615 +JitCode *
1.616 +JitRuntime::generateBailoutTable(JSContext *cx, uint32_t frameClass)
1.617 +{
1.618 + MacroAssembler masm(cx);
1.619 +
1.620 + Label bailout;
1.621 + for (size_t i = 0; i < BAILOUT_TABLE_SIZE; i++)
1.622 + masm.ma_bl(&bailout);
1.623 + masm.bind(&bailout);
1.624 +
1.625 + GenerateBailoutThunk(cx, masm, frameClass);
1.626 +
1.627 + Linker linker(masm);
1.628 + AutoFlushICache afc("BailoutTable");
1.629 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.630 +
1.631 +#ifdef JS_ION_PERF
1.632 + writePerfSpewerJitCodeProfile(code, "BailoutTable");
1.633 +#endif
1.634 +
1.635 + return code;
1.636 +}
1.637 +
1.638 +JitCode *
1.639 +JitRuntime::generateBailoutHandler(JSContext *cx)
1.640 +{
1.641 + MacroAssembler masm(cx);
1.642 + GenerateBailoutThunk(cx, masm, NO_FRAME_SIZE_CLASS_ID);
1.643 +
1.644 + Linker linker(masm);
1.645 + AutoFlushICache afc("BailoutHandler");
1.646 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.647 +
1.648 +#ifdef JS_ION_PERF
1.649 + writePerfSpewerJitCodeProfile(code, "BailoutHandler");
1.650 +#endif
1.651 +
1.652 + return code;
1.653 +}
1.654 +
1.655 +JitCode *
1.656 +JitRuntime::generateVMWrapper(JSContext *cx, const VMFunction &f)
1.657 +{
1.658 + JS_ASSERT(functionWrappers_);
1.659 + JS_ASSERT(functionWrappers_->initialized());
1.660 + VMWrapperMap::AddPtr p = functionWrappers_->lookupForAdd(&f);
1.661 + if (p)
1.662 + return p->value();
1.663 +
1.664 + // Generate a separated code for the wrapper.
1.665 + MacroAssembler masm(cx);
1.666 + GeneralRegisterSet regs = GeneralRegisterSet(Register::Codes::WrapperMask);
1.667 +
1.668 + // Wrapper register set is a superset of Volatile register set.
1.669 + JS_STATIC_ASSERT((Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0);
1.670 +
1.671 + // The context is the first argument; r0 is the first argument register.
1.672 + Register cxreg = r0;
1.673 + regs.take(cxreg);
1.674 +
1.675 + // Stack is:
1.676 + // ... frame ...
1.677 + // +8 [args] + argPadding
1.678 + // +0 ExitFrame
1.679 + //
1.680 + // We're aligned to an exit frame, so link it up.
1.681 + masm.enterExitFrameAndLoadContext(&f, cxreg, regs.getAny(), f.executionMode);
1.682 +
1.683 + // Save the base of the argument set stored on the stack.
1.684 + Register argsBase = InvalidReg;
1.685 + if (f.explicitArgs) {
1.686 + argsBase = r5;
1.687 + regs.take(argsBase);
1.688 + masm.ma_add(sp, Imm32(IonExitFrameLayout::SizeWithFooter()), argsBase);
1.689 + }
1.690 +
1.691 + // Reserve space for the outparameter.
1.692 + Register outReg = InvalidReg;
1.693 + switch (f.outParam) {
1.694 + case Type_Value:
1.695 + outReg = r4;
1.696 + regs.take(outReg);
1.697 + masm.reserveStack(sizeof(Value));
1.698 + masm.ma_mov(sp, outReg);
1.699 + break;
1.700 +
1.701 + case Type_Handle:
1.702 + outReg = r4;
1.703 + regs.take(outReg);
1.704 + masm.PushEmptyRooted(f.outParamRootType);
1.705 + masm.ma_mov(sp, outReg);
1.706 + break;
1.707 +
1.708 + case Type_Int32:
1.709 + case Type_Pointer:
1.710 + case Type_Bool:
1.711 + outReg = r4;
1.712 + regs.take(outReg);
1.713 + masm.reserveStack(sizeof(int32_t));
1.714 + masm.ma_mov(sp, outReg);
1.715 + break;
1.716 +
1.717 + case Type_Double:
1.718 + outReg = r4;
1.719 + regs.take(outReg);
1.720 + masm.reserveStack(sizeof(double));
1.721 + masm.ma_mov(sp, outReg);
1.722 + break;
1.723 +
1.724 + default:
1.725 + JS_ASSERT(f.outParam == Type_Void);
1.726 + break;
1.727 + }
1.728 +
1.729 + masm.setupUnalignedABICall(f.argc(), regs.getAny());
1.730 + masm.passABIArg(cxreg);
1.731 +
1.732 + size_t argDisp = 0;
1.733 +
1.734 + // Copy any arguments.
1.735 + for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
1.736 + MoveOperand from;
1.737 + switch (f.argProperties(explicitArg)) {
1.738 + case VMFunction::WordByValue:
1.739 + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
1.740 + argDisp += sizeof(void *);
1.741 + break;
1.742 + case VMFunction::DoubleByValue:
1.743 + // Values should be passed by reference, not by value, so we
1.744 + // assert that the argument is a double-precision float.
1.745 + JS_ASSERT(f.argPassedInFloatReg(explicitArg));
1.746 + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
1.747 + argDisp += sizeof(double);
1.748 + break;
1.749 + case VMFunction::WordByRef:
1.750 + masm.passABIArg(MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS), MoveOp::GENERAL);
1.751 + argDisp += sizeof(void *);
1.752 + break;
1.753 + case VMFunction::DoubleByRef:
1.754 + masm.passABIArg(MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS), MoveOp::GENERAL);
1.755 + argDisp += 2 * sizeof(void *);
1.756 + break;
1.757 + }
1.758 + }
1.759 +
1.760 + // Copy the implicit outparam, if any.
1.761 + if (outReg != InvalidReg)
1.762 + masm.passABIArg(outReg);
1.763 +
1.764 + masm.callWithABI(f.wrapped);
1.765 +
1.766 + // Test for failure.
1.767 + switch (f.failType()) {
1.768 + case Type_Object:
1.769 + masm.branchTestPtr(Assembler::Zero, r0, r0, masm.failureLabel(f.executionMode));
1.770 + break;
1.771 + case Type_Bool:
1.772 + masm.branchIfFalseBool(r0, masm.failureLabel(f.executionMode));
1.773 + break;
1.774 + default:
1.775 + MOZ_ASSUME_UNREACHABLE("unknown failure kind");
1.776 + }
1.777 +
1.778 + // Load the outparam and free any allocated stack.
1.779 + switch (f.outParam) {
1.780 + case Type_Handle:
1.781 + masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
1.782 + break;
1.783 +
1.784 + case Type_Value:
1.785 + masm.loadValue(Address(sp, 0), JSReturnOperand);
1.786 + masm.freeStack(sizeof(Value));
1.787 + break;
1.788 +
1.789 + case Type_Int32:
1.790 + case Type_Pointer:
1.791 + masm.load32(Address(sp, 0), ReturnReg);
1.792 + masm.freeStack(sizeof(int32_t));
1.793 + break;
1.794 +
1.795 + case Type_Bool:
1.796 + masm.load8ZeroExtend(Address(sp, 0), ReturnReg);
1.797 + masm.freeStack(sizeof(int32_t));
1.798 + break;
1.799 +
1.800 + case Type_Double:
1.801 + if (cx->runtime()->jitSupportsFloatingPoint)
1.802 + masm.loadDouble(Address(sp, 0), ReturnFloatReg);
1.803 + else
1.804 + masm.assumeUnreachable("Unable to load into float reg, with no FP support.");
1.805 + masm.freeStack(sizeof(double));
1.806 + break;
1.807 +
1.808 + default:
1.809 + JS_ASSERT(f.outParam == Type_Void);
1.810 + break;
1.811 + }
1.812 + masm.leaveExitFrame();
1.813 + masm.retn(Imm32(sizeof(IonExitFrameLayout) +
1.814 + f.explicitStackSlots() * sizeof(void *) +
1.815 + f.extraValuesToPop * sizeof(Value)));
1.816 +
1.817 + Linker linker(masm);
1.818 + AutoFlushICache afc("VMWrapper");
1.819 + JitCode *wrapper = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.820 + if (!wrapper)
1.821 + return nullptr;
1.822 +
1.823 + // linker.newCode may trigger a GC and sweep functionWrappers_ so we have to
1.824 + // use relookupOrAdd instead of add.
1.825 + if (!functionWrappers_->relookupOrAdd(p, &f, wrapper))
1.826 + return nullptr;
1.827 +
1.828 +#ifdef JS_ION_PERF
1.829 + writePerfSpewerJitCodeProfile(wrapper, "VMWrapper");
1.830 +#endif
1.831 +
1.832 + return wrapper;
1.833 +}
1.834 +
1.835 +JitCode *
1.836 +JitRuntime::generatePreBarrier(JSContext *cx, MIRType type)
1.837 +{
1.838 + MacroAssembler masm(cx);
1.839 +
1.840 + RegisterSet save;
1.841 + if (cx->runtime()->jitSupportsFloatingPoint) {
1.842 + save = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
1.843 + FloatRegisterSet(FloatRegisters::VolatileMask));
1.844 + } else {
1.845 + save = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
1.846 + FloatRegisterSet());
1.847 + }
1.848 + masm.PushRegsInMask(save);
1.849 +
1.850 + JS_ASSERT(PreBarrierReg == r1);
1.851 + masm.movePtr(ImmPtr(cx->runtime()), r0);
1.852 +
1.853 + masm.setupUnalignedABICall(2, r2);
1.854 + masm.passABIArg(r0);
1.855 + masm.passABIArg(r1);
1.856 + if (type == MIRType_Value) {
1.857 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, MarkValueFromIon));
1.858 + } else {
1.859 + JS_ASSERT(type == MIRType_Shape);
1.860 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, MarkShapeFromIon));
1.861 + }
1.862 +
1.863 + masm.PopRegsInMask(save);
1.864 + masm.ret();
1.865 +
1.866 + Linker linker(masm);
1.867 + AutoFlushICache afc("PreBarrier");
1.868 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.869 +
1.870 +#ifdef JS_ION_PERF
1.871 + writePerfSpewerJitCodeProfile(code, "PreBarrier");
1.872 +#endif
1.873 +
1.874 + return code;
1.875 +}
1.876 +
1.877 +typedef bool (*HandleDebugTrapFn)(JSContext *, BaselineFrame *, uint8_t *, bool *);
1.878 +static const VMFunction HandleDebugTrapInfo = FunctionInfo<HandleDebugTrapFn>(HandleDebugTrap);
1.879 +
1.880 +JitCode *
1.881 +JitRuntime::generateDebugTrapHandler(JSContext *cx)
1.882 +{
1.883 + MacroAssembler masm;
1.884 +
1.885 + Register scratch1 = r0;
1.886 + Register scratch2 = r1;
1.887 +
1.888 + // Load BaselineFrame pointer in scratch1.
1.889 + masm.mov(r11, scratch1);
1.890 + masm.subPtr(Imm32(BaselineFrame::Size()), scratch1);
1.891 +
1.892 + // Enter a stub frame and call the HandleDebugTrap VM function. Ensure
1.893 + // the stub frame has a nullptr ICStub pointer, since this pointer is
1.894 + // marked during GC.
1.895 + masm.movePtr(ImmPtr(nullptr), BaselineStubReg);
1.896 + EmitEnterStubFrame(masm, scratch2);
1.897 +
1.898 + JitCode *code = cx->runtime()->jitRuntime()->getVMWrapper(HandleDebugTrapInfo);
1.899 + if (!code)
1.900 + return nullptr;
1.901 +
1.902 + masm.push(lr);
1.903 + masm.push(scratch1);
1.904 + EmitCallVM(code, masm);
1.905 +
1.906 + EmitLeaveStubFrame(masm);
1.907 +
1.908 + // If the stub returns |true|, we have to perform a forced return
1.909 + // (return from the JS frame). If the stub returns |false|, just return
1.910 + // from the trap stub so that execution continues at the current pc.
1.911 + Label forcedReturn;
1.912 + masm.branchTest32(Assembler::NonZero, ReturnReg, ReturnReg, &forcedReturn);
1.913 + masm.mov(lr, pc);
1.914 +
1.915 + masm.bind(&forcedReturn);
1.916 + masm.loadValue(Address(r11, BaselineFrame::reverseOffsetOfReturnValue()),
1.917 + JSReturnOperand);
1.918 + masm.mov(r11, sp);
1.919 + masm.pop(r11);
1.920 + masm.ret();
1.921 +
1.922 + Linker linker(masm);
1.923 + AutoFlushICache afc("DebugTrapHandler");
1.924 + JitCode *codeDbg = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.925 +
1.926 +#ifdef JS_ION_PERF
1.927 + writePerfSpewerJitCodeProfile(codeDbg, "DebugTrapHandler");
1.928 +#endif
1.929 +
1.930 + return codeDbg;
1.931 +}
1.932 +
1.933 +JitCode *
1.934 +JitRuntime::generateExceptionTailStub(JSContext *cx)
1.935 +{
1.936 + MacroAssembler masm;
1.937 +
1.938 + masm.handleFailureWithHandlerTail();
1.939 +
1.940 + Linker linker(masm);
1.941 + AutoFlushICache afc("ExceptionTailStub");
1.942 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.943 +
1.944 +#ifdef JS_ION_PERF
1.945 + writePerfSpewerJitCodeProfile(code, "ExceptionTailStub");
1.946 +#endif
1.947 +
1.948 + return code;
1.949 +}
1.950 +
1.951 +JitCode *
1.952 +JitRuntime::generateBailoutTailStub(JSContext *cx)
1.953 +{
1.954 + MacroAssembler masm;
1.955 +
1.956 + masm.generateBailoutTail(r1, r2);
1.957 +
1.958 + Linker linker(masm);
1.959 + AutoFlushICache afc("BailoutTailStub");
1.960 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.961 +
1.962 +#ifdef JS_ION_PERF
1.963 + writePerfSpewerJitCodeProfile(code, "BailoutTailStub");
1.964 +#endif
1.965 +
1.966 + return code;
1.967 +}
