1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/jit/x64/Trampoline-x64.cpp Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,819 @@
1.4 +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
1.5 + * vim: set ts=8 sts=4 et sw=4 tw=99:
1.6 + * This Source Code Form is subject to the terms of the Mozilla Public
1.7 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.8 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.9 +
1.10 +#include "jit/Bailouts.h"
1.11 +#include "jit/IonFrames.h"
1.12 +#include "jit/IonLinker.h"
1.13 +#include "jit/JitCompartment.h"
1.14 +#ifdef JS_ION_PERF
1.15 +# include "jit/PerfSpewer.h"
1.16 +#endif
1.17 +#include "jit/VMFunctions.h"
1.18 +#include "jit/x64/BaselineHelpers-x64.h"
1.19 +
1.20 +using namespace js;
1.21 +using namespace js::jit;
1.22 +
1.23 +// All registers to save and restore. This includes the stack pointer, since we
1.24 +// use the ability to reference register values on the stack by index.
1.25 +static const RegisterSet AllRegs =
1.26 + RegisterSet(GeneralRegisterSet(Registers::AllMask),
1.27 + FloatRegisterSet(FloatRegisters::AllMask));
1.28 +
1.29 +/* This method generates a trampoline on x64 for a c++ function with
1.30 + * the following signature:
1.31 + * bool blah(void *code, int argc, Value *argv, JSObject *scopeChain,
1.32 + * Value *vp)
1.33 + * ...using standard x64 fastcall calling convention
1.34 + */
1.35 +JitCode *
1.36 +JitRuntime::generateEnterJIT(JSContext *cx, EnterJitType type)
1.37 +{
1.38 + MacroAssembler masm(cx);
1.39 +
1.40 + const Register reg_code = IntArgReg0;
1.41 + const Register reg_argc = IntArgReg1;
1.42 + const Register reg_argv = IntArgReg2;
1.43 + JS_ASSERT(OsrFrameReg == IntArgReg3);
1.44 +
1.45 +#if defined(_WIN64)
1.46 + const Operand token = Operand(rbp, 16 + ShadowStackSpace);
1.47 + const Operand scopeChain = Operand(rbp, 24 + ShadowStackSpace);
1.48 + const Operand numStackValuesAddr = Operand(rbp, 32 + ShadowStackSpace);
1.49 + const Operand result = Operand(rbp, 40 + ShadowStackSpace);
1.50 +#else
1.51 + const Register token = IntArgReg4;
1.52 + const Register scopeChain = IntArgReg5;
1.53 + const Operand numStackValuesAddr = Operand(rbp, 16 + ShadowStackSpace);
1.54 + const Operand result = Operand(rbp, 24 + ShadowStackSpace);
1.55 +#endif
1.56 +
1.57 + // Save old stack frame pointer, set new stack frame pointer.
1.58 + masm.push(rbp);
1.59 + masm.mov(rsp, rbp);
1.60 +
1.61 + // Save non-volatile registers. These must be saved by the trampoline, rather
1.62 + // than by the JIT'd code, because they are scanned by the conservative scanner.
1.63 + masm.push(rbx);
1.64 + masm.push(r12);
1.65 + masm.push(r13);
1.66 + masm.push(r14);
1.67 + masm.push(r15);
1.68 +#if defined(_WIN64)
1.69 + masm.push(rdi);
1.70 + masm.push(rsi);
1.71 +
1.72 + // 16-byte aligment for movdqa
1.73 + masm.subq(Imm32(16 * 10 + 8), rsp);
1.74 +
1.75 + masm.movdqa(xmm6, Operand(rsp, 16 * 0));
1.76 + masm.movdqa(xmm7, Operand(rsp, 16 * 1));
1.77 + masm.movdqa(xmm8, Operand(rsp, 16 * 2));
1.78 + masm.movdqa(xmm9, Operand(rsp, 16 * 3));
1.79 + masm.movdqa(xmm10, Operand(rsp, 16 * 4));
1.80 + masm.movdqa(xmm11, Operand(rsp, 16 * 5));
1.81 + masm.movdqa(xmm12, Operand(rsp, 16 * 6));
1.82 + masm.movdqa(xmm13, Operand(rsp, 16 * 7));
1.83 + masm.movdqa(xmm14, Operand(rsp, 16 * 8));
1.84 + masm.movdqa(xmm15, Operand(rsp, 16 * 9));
1.85 +#endif
1.86 +
1.87 + // Push the EnterJIT sps mark.
1.88 + masm.spsMarkJit(&cx->runtime()->spsProfiler, rbp, rbx);
1.89 +
1.90 + // Save arguments passed in registers needed after function call.
1.91 + masm.push(result);
1.92 +
1.93 + // Remember stack depth without padding and arguments.
1.94 + masm.mov(rsp, r14);
1.95 +
1.96 + // Remember number of bytes occupied by argument vector
1.97 + masm.mov(reg_argc, r13);
1.98 + masm.shll(Imm32(3), r13);
1.99 +
1.100 + // Guarantee 16-byte alignment.
1.101 + // We push argc, callee token, frame size, and return address.
1.102 + // The latter two are 16 bytes together, so we only consider argc and the
1.103 + // token.
1.104 + masm.mov(rsp, r12);
1.105 + masm.subq(r13, r12);
1.106 + masm.subq(Imm32(8), r12);
1.107 + masm.andl(Imm32(0xf), r12);
1.108 + masm.subq(r12, rsp);
1.109 +
1.110 + /***************************************************************
1.111 + Loop over argv vector, push arguments onto stack in reverse order
1.112 + ***************************************************************/
1.113 +
1.114 + // r13 still stores the number of bytes in the argument vector.
1.115 + masm.addq(reg_argv, r13); // r13 points above last argument.
1.116 +
1.117 + // while r13 > rdx, push arguments.
1.118 + {
1.119 + Label header, footer;
1.120 + masm.bind(&header);
1.121 +
1.122 + masm.cmpq(r13, reg_argv);
1.123 + masm.j(AssemblerX86Shared::BelowOrEqual, &footer);
1.124 +
1.125 + masm.subq(Imm32(8), r13);
1.126 + masm.push(Operand(r13, 0));
1.127 + masm.jmp(&header);
1.128 +
1.129 + masm.bind(&footer);
1.130 + }
1.131 +
1.132 + // Push the number of actual arguments. |result| is used to store the
1.133 + // actual number of arguments without adding an extra argument to the enter
1.134 + // JIT.
1.135 + masm.movq(result, reg_argc);
1.136 + masm.unboxInt32(Operand(reg_argc, 0), reg_argc);
1.137 + masm.push(reg_argc);
1.138 +
1.139 + // Push the callee token.
1.140 + masm.push(token);
1.141 +
1.142 + /*****************************************************************
1.143 + Push the number of bytes we've pushed so far on the stack and call
1.144 + *****************************************************************/
1.145 + masm.subq(rsp, r14);
1.146 +
1.147 + // Create a frame descriptor.
1.148 + masm.makeFrameDescriptor(r14, JitFrame_Entry);
1.149 + masm.push(r14);
1.150 +
1.151 + CodeLabel returnLabel;
1.152 + if (type == EnterJitBaseline) {
1.153 + // Handle OSR.
1.154 + GeneralRegisterSet regs(GeneralRegisterSet::All());
1.155 + regs.takeUnchecked(OsrFrameReg);
1.156 + regs.take(rbp);
1.157 + regs.take(reg_code);
1.158 +
1.159 + // Ensure that |scratch| does not end up being JSReturnOperand.
1.160 + // Do takeUnchecked because on Win64/x64, reg_code (IntArgReg0) and JSReturnOperand are
1.161 + // the same (rcx). See bug 849398.
1.162 + regs.takeUnchecked(JSReturnOperand);
1.163 + Register scratch = regs.takeAny();
1.164 +
1.165 + Label notOsr;
1.166 + masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr);
1.167 +
1.168 + Register numStackValues = regs.takeAny();
1.169 + masm.movq(numStackValuesAddr, numStackValues);
1.170 +
1.171 + // Push return address, previous frame pointer.
1.172 + masm.mov(returnLabel.dest(), scratch);
1.173 + masm.push(scratch);
1.174 + masm.push(rbp);
1.175 +
1.176 + // Reserve frame.
1.177 + Register framePtr = rbp;
1.178 + masm.subPtr(Imm32(BaselineFrame::Size()), rsp);
1.179 + masm.mov(rsp, framePtr);
1.180 +
1.181 +#ifdef XP_WIN
1.182 + // Can't push large frames blindly on windows. Touch frame memory incrementally.
1.183 + masm.mov(numStackValues, scratch);
1.184 + masm.lshiftPtr(Imm32(3), scratch);
1.185 + masm.subPtr(scratch, framePtr);
1.186 + {
1.187 + masm.movePtr(rsp, scratch);
1.188 + masm.subPtr(Imm32(WINDOWS_BIG_FRAME_TOUCH_INCREMENT), scratch);
1.189 +
1.190 + Label touchFrameLoop;
1.191 + Label touchFrameLoopEnd;
1.192 + masm.bind(&touchFrameLoop);
1.193 + masm.branchPtr(Assembler::Below, scratch, framePtr, &touchFrameLoopEnd);
1.194 + masm.store32(Imm32(0), Address(scratch, 0));
1.195 + masm.subPtr(Imm32(WINDOWS_BIG_FRAME_TOUCH_INCREMENT), scratch);
1.196 + masm.jump(&touchFrameLoop);
1.197 + masm.bind(&touchFrameLoopEnd);
1.198 + }
1.199 + masm.mov(rsp, framePtr);
1.200 +#endif
1.201 +
1.202 + // Reserve space for locals and stack values.
1.203 + Register valuesSize = regs.takeAny();
1.204 + masm.mov(numStackValues, valuesSize);
1.205 + masm.shll(Imm32(3), valuesSize);
1.206 + masm.subPtr(valuesSize, rsp);
1.207 +
1.208 + // Enter exit frame.
1.209 + masm.addPtr(Imm32(BaselineFrame::Size() + BaselineFrame::FramePointerOffset), valuesSize);
1.210 + masm.makeFrameDescriptor(valuesSize, JitFrame_BaselineJS);
1.211 + masm.push(valuesSize);
1.212 + masm.push(Imm32(0)); // Fake return address.
1.213 + masm.enterFakeExitFrame();
1.214 +
1.215 + regs.add(valuesSize);
1.216 +
1.217 + masm.push(framePtr);
1.218 + masm.push(reg_code);
1.219 +
1.220 + masm.setupUnalignedABICall(3, scratch);
1.221 + masm.passABIArg(framePtr); // BaselineFrame
1.222 + masm.passABIArg(OsrFrameReg); // InterpreterFrame
1.223 + masm.passABIArg(numStackValues);
1.224 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, jit::InitBaselineFrameForOsr));
1.225 +
1.226 + masm.pop(reg_code);
1.227 + masm.pop(framePtr);
1.228 +
1.229 + JS_ASSERT(reg_code != ReturnReg);
1.230 +
1.231 + Label error;
1.232 + masm.addPtr(Imm32(IonExitFrameLayout::SizeWithFooter()), rsp);
1.233 + masm.addPtr(Imm32(BaselineFrame::Size()), framePtr);
1.234 + masm.branchIfFalseBool(ReturnReg, &error);
1.235 +
1.236 + masm.jump(reg_code);
1.237 +
1.238 + // OOM: load error value, discard return address and previous frame
1.239 + // pointer and return.
1.240 + masm.bind(&error);
1.241 + masm.mov(framePtr, rsp);
1.242 + masm.addPtr(Imm32(2 * sizeof(uintptr_t)), rsp);
1.243 + masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
1.244 + masm.mov(returnLabel.dest(), scratch);
1.245 + masm.jump(scratch);
1.246 +
1.247 + masm.bind(¬Osr);
1.248 + masm.movq(scopeChain, R1.scratchReg());
1.249 + }
1.250 +
1.251 + // Call function.
1.252 + masm.call(reg_code);
1.253 +
1.254 + if (type == EnterJitBaseline) {
1.255 + // Baseline OSR will return here.
1.256 + masm.bind(returnLabel.src());
1.257 + if (!masm.addCodeLabel(returnLabel))
1.258 + return nullptr;
1.259 + }
1.260 +
1.261 + // Pop arguments and padding from stack.
1.262 + masm.pop(r14); // Pop and decode descriptor.
1.263 + masm.shrq(Imm32(FRAMESIZE_SHIFT), r14);
1.264 + masm.addq(r14, rsp); // Remove arguments.
1.265 +
1.266 + /*****************************************************************
1.267 + Place return value where it belongs, pop all saved registers
1.268 + *****************************************************************/
1.269 + masm.pop(r12); // vp
1.270 + masm.storeValue(JSReturnOperand, Operand(r12, 0));
1.271 +
1.272 + // Unwind the sps mark.
1.273 + masm.spsUnmarkJit(&cx->runtime()->spsProfiler, rbx);
1.274 +
1.275 + // Restore non-volatile registers.
1.276 +#if defined(_WIN64)
1.277 + masm.movdqa(Operand(rsp, 16 * 0), xmm6);
1.278 + masm.movdqa(Operand(rsp, 16 * 1), xmm7);
1.279 + masm.movdqa(Operand(rsp, 16 * 2), xmm8);
1.280 + masm.movdqa(Operand(rsp, 16 * 3), xmm9);
1.281 + masm.movdqa(Operand(rsp, 16 * 4), xmm10);
1.282 + masm.movdqa(Operand(rsp, 16 * 5), xmm11);
1.283 + masm.movdqa(Operand(rsp, 16 * 6), xmm12);
1.284 + masm.movdqa(Operand(rsp, 16 * 7), xmm13);
1.285 + masm.movdqa(Operand(rsp, 16 * 8), xmm14);
1.286 + masm.movdqa(Operand(rsp, 16 * 9), xmm15);
1.287 +
1.288 + masm.addq(Imm32(16 * 10 + 8), rsp);
1.289 +
1.290 + masm.pop(rsi);
1.291 + masm.pop(rdi);
1.292 +#endif
1.293 + masm.pop(r15);
1.294 + masm.pop(r14);
1.295 + masm.pop(r13);
1.296 + masm.pop(r12);
1.297 + masm.pop(rbx);
1.298 +
1.299 + // Restore frame pointer and return.
1.300 + masm.pop(rbp);
1.301 + masm.ret();
1.302 +
1.303 + Linker linker(masm);
1.304 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.305 +
1.306 +#ifdef JS_ION_PERF
1.307 + writePerfSpewerJitCodeProfile(code, "EnterJIT");
1.308 +#endif
1.309 +
1.310 + return code;
1.311 +}
1.312 +
1.313 +JitCode *
1.314 +JitRuntime::generateInvalidator(JSContext *cx)
1.315 +{
1.316 + AutoIonContextAlloc aica(cx);
1.317 + MacroAssembler masm(cx);
1.318 +
1.319 + // See explanatory comment in x86's JitRuntime::generateInvalidator.
1.320 +
1.321 + masm.addq(Imm32(sizeof(uintptr_t)), rsp);
1.322 +
1.323 + // Push registers such that we can access them from [base + code].
1.324 + masm.PushRegsInMask(AllRegs);
1.325 +
1.326 + masm.movq(rsp, rax); // Argument to jit::InvalidationBailout.
1.327 +
1.328 + // Make space for InvalidationBailout's frameSize outparam.
1.329 + masm.reserveStack(sizeof(size_t));
1.330 + masm.movq(rsp, rbx);
1.331 +
1.332 + // Make space for InvalidationBailout's bailoutInfo outparam.
1.333 + masm.reserveStack(sizeof(void *));
1.334 + masm.movq(rsp, r9);
1.335 +
1.336 + masm.setupUnalignedABICall(3, rdx);
1.337 + masm.passABIArg(rax);
1.338 + masm.passABIArg(rbx);
1.339 + masm.passABIArg(r9);
1.340 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, InvalidationBailout));
1.341 +
1.342 + masm.pop(r9); // Get the bailoutInfo outparam.
1.343 + masm.pop(rbx); // Get the frameSize outparam.
1.344 +
1.345 + // Pop the machine state and the dead frame.
1.346 + masm.lea(Operand(rsp, rbx, TimesOne, sizeof(InvalidationBailoutStack)), rsp);
1.347 +
1.348 + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
1.349 + JitCode *bailoutTail = cx->runtime()->jitRuntime()->getBailoutTail();
1.350 + masm.jmp(bailoutTail);
1.351 +
1.352 + Linker linker(masm);
1.353 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.354 +
1.355 +#ifdef JS_ION_PERF
1.356 + writePerfSpewerJitCodeProfile(code, "Invalidator");
1.357 +#endif
1.358 +
1.359 + return code;
1.360 +}
1.361 +
1.362 +JitCode *
1.363 +JitRuntime::generateArgumentsRectifier(JSContext *cx, ExecutionMode mode, void **returnAddrOut)
1.364 +{
1.365 + // Do not erase the frame pointer in this function.
1.366 +
1.367 + MacroAssembler masm(cx);
1.368 +
1.369 + // ArgumentsRectifierReg contains the |nargs| pushed onto the current frame.
1.370 + // Including |this|, there are (|nargs| + 1) arguments to copy.
1.371 + JS_ASSERT(ArgumentsRectifierReg == r8);
1.372 +
1.373 + // Load the number of |undefined|s to push into %rcx.
1.374 + masm.loadPtr(Address(rsp, IonRectifierFrameLayout::offsetOfCalleeToken()), rax);
1.375 + masm.movzwl(Operand(rax, JSFunction::offsetOfNargs()), rcx);
1.376 + masm.subq(r8, rcx);
1.377 +
1.378 + // Copy the number of actual arguments
1.379 + masm.loadPtr(Address(rsp, IonRectifierFrameLayout::offsetOfNumActualArgs()), rdx);
1.380 +
1.381 + masm.moveValue(UndefinedValue(), r10);
1.382 +
1.383 + masm.movq(rsp, r9); // Save %rsp.
1.384 +
1.385 + // Push undefined.
1.386 + {
1.387 + Label undefLoopTop;
1.388 + masm.bind(&undefLoopTop);
1.389 +
1.390 + masm.push(r10);
1.391 + masm.subl(Imm32(1), rcx);
1.392 + masm.j(Assembler::NonZero, &undefLoopTop);
1.393 + }
1.394 +
1.395 + // Get the topmost argument.
1.396 + BaseIndex b = BaseIndex(r9, r8, TimesEight, sizeof(IonRectifierFrameLayout));
1.397 + masm.lea(Operand(b), rcx);
1.398 +
1.399 + // Push arguments, |nargs| + 1 times (to include |this|).
1.400 + masm.addl(Imm32(1), r8);
1.401 + {
1.402 + Label copyLoopTop;
1.403 +
1.404 + masm.bind(©LoopTop);
1.405 + masm.push(Operand(rcx, 0x0));
1.406 + masm.subq(Imm32(sizeof(Value)), rcx);
1.407 + masm.subl(Imm32(1), r8);
1.408 + masm.j(Assembler::NonZero, ©LoopTop);
1.409 + }
1.410 +
1.411 + // Construct descriptor.
1.412 + masm.subq(rsp, r9);
1.413 + masm.makeFrameDescriptor(r9, JitFrame_Rectifier);
1.414 +
1.415 + // Construct IonJSFrameLayout.
1.416 + masm.push(rdx); // numActualArgs
1.417 + masm.push(rax); // callee token
1.418 + masm.push(r9); // descriptor
1.419 +
1.420 + // Call the target function.
1.421 + // Note that this code assumes the function is JITted.
1.422 + masm.loadPtr(Address(rax, JSFunction::offsetOfNativeOrScript()), rax);
1.423 + masm.loadBaselineOrIonRaw(rax, rax, mode, nullptr);
1.424 + masm.call(rax);
1.425 + uint32_t returnOffset = masm.currentOffset();
1.426 +
1.427 + // Remove the rectifier frame.
1.428 + masm.pop(r9); // r9 <- descriptor with FrameType.
1.429 + masm.shrq(Imm32(FRAMESIZE_SHIFT), r9);
1.430 + masm.pop(r11); // Discard calleeToken.
1.431 + masm.pop(r11); // Discard numActualArgs.
1.432 + masm.addq(r9, rsp); // Discard pushed arguments.
1.433 +
1.434 + masm.ret();
1.435 +
1.436 + Linker linker(masm);
1.437 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.438 +
1.439 +#ifdef JS_ION_PERF
1.440 + writePerfSpewerJitCodeProfile(code, "ArgumentsRectifier");
1.441 +#endif
1.442 +
1.443 + CodeOffsetLabel returnLabel(returnOffset);
1.444 + returnLabel.fixup(&masm);
1.445 + if (returnAddrOut)
1.446 + *returnAddrOut = (void *) (code->raw() + returnLabel.offset());
1.447 + return code;
1.448 +}
1.449 +
1.450 +static void
1.451 +GenerateBailoutThunk(JSContext *cx, MacroAssembler &masm, uint32_t frameClass)
1.452 +{
1.453 + // Push registers such that we can access them from [base + code].
1.454 + masm.PushRegsInMask(AllRegs);
1.455 +
1.456 + // Get the stack pointer into a register, pre-alignment.
1.457 + masm.movq(rsp, r8);
1.458 +
1.459 + // Make space for Bailout's bailoutInfo outparam.
1.460 + masm.reserveStack(sizeof(void *));
1.461 + masm.movq(rsp, r9);
1.462 +
1.463 + // Call the bailout function.
1.464 + masm.setupUnalignedABICall(2, rax);
1.465 + masm.passABIArg(r8);
1.466 + masm.passABIArg(r9);
1.467 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, Bailout));
1.468 +
1.469 + masm.pop(r9); // Get the bailoutInfo outparam.
1.470 +
1.471 + // Stack is:
1.472 + // [frame]
1.473 + // snapshotOffset
1.474 + // frameSize
1.475 + // [bailoutFrame]
1.476 + //
1.477 + // Remove both the bailout frame and the topmost Ion frame's stack.
1.478 + static const uint32_t BailoutDataSize = sizeof(void *) * Registers::Total +
1.479 + sizeof(double) * FloatRegisters::Total;
1.480 + masm.addq(Imm32(BailoutDataSize), rsp);
1.481 + masm.pop(rcx);
1.482 + masm.lea(Operand(rsp, rcx, TimesOne, sizeof(void *)), rsp);
1.483 +
1.484 + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
1.485 + JitCode *bailoutTail = cx->runtime()->jitRuntime()->getBailoutTail();
1.486 + masm.jmp(bailoutTail);
1.487 +}
1.488 +
1.489 +JitCode *
1.490 +JitRuntime::generateBailoutTable(JSContext *cx, uint32_t frameClass)
1.491 +{
1.492 + MOZ_ASSUME_UNREACHABLE("x64 does not use bailout tables");
1.493 +}
1.494 +
1.495 +JitCode *
1.496 +JitRuntime::generateBailoutHandler(JSContext *cx)
1.497 +{
1.498 + MacroAssembler masm;
1.499 +
1.500 + GenerateBailoutThunk(cx, masm, NO_FRAME_SIZE_CLASS_ID);
1.501 +
1.502 + Linker linker(masm);
1.503 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.504 +
1.505 +#ifdef JS_ION_PERF
1.506 + writePerfSpewerJitCodeProfile(code, "BailoutHandler");
1.507 +#endif
1.508 +
1.509 + return code;
1.510 +}
1.511 +
1.512 +JitCode *
1.513 +JitRuntime::generateVMWrapper(JSContext *cx, const VMFunction &f)
1.514 +{
1.515 + JS_ASSERT(!StackKeptAligned);
1.516 + JS_ASSERT(functionWrappers_);
1.517 + JS_ASSERT(functionWrappers_->initialized());
1.518 + VMWrapperMap::AddPtr p = functionWrappers_->lookupForAdd(&f);
1.519 + if (p)
1.520 + return p->value();
1.521 +
1.522 + // Generate a separated code for the wrapper.
1.523 + MacroAssembler masm;
1.524 +
1.525 + // Avoid conflicts with argument registers while discarding the result after
1.526 + // the function call.
1.527 + GeneralRegisterSet regs = GeneralRegisterSet(Register::Codes::WrapperMask);
1.528 +
1.529 + // Wrapper register set is a superset of Volatile register set.
1.530 + JS_STATIC_ASSERT((Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0);
1.531 +
1.532 + // The context is the first argument.
1.533 + Register cxreg = IntArgReg0;
1.534 + regs.take(cxreg);
1.535 +
1.536 + // Stack is:
1.537 + // ... frame ...
1.538 + // +12 [args]
1.539 + // +8 descriptor
1.540 + // +0 returnAddress
1.541 + //
1.542 + // We're aligned to an exit frame, so link it up.
1.543 + masm.enterExitFrameAndLoadContext(&f, cxreg, regs.getAny(), f.executionMode);
1.544 +
1.545 + // Save the current stack pointer as the base for copying arguments.
1.546 + Register argsBase = InvalidReg;
1.547 + if (f.explicitArgs) {
1.548 + argsBase = r10;
1.549 + regs.take(argsBase);
1.550 + masm.lea(Operand(rsp,IonExitFrameLayout::SizeWithFooter()), argsBase);
1.551 + }
1.552 +
1.553 + // Reserve space for the outparameter.
1.554 + Register outReg = InvalidReg;
1.555 + switch (f.outParam) {
1.556 + case Type_Value:
1.557 + outReg = regs.takeAny();
1.558 + masm.reserveStack(sizeof(Value));
1.559 + masm.movq(esp, outReg);
1.560 + break;
1.561 +
1.562 + case Type_Handle:
1.563 + outReg = regs.takeAny();
1.564 + masm.PushEmptyRooted(f.outParamRootType);
1.565 + masm.movq(esp, outReg);
1.566 + break;
1.567 +
1.568 + case Type_Int32:
1.569 + case Type_Bool:
1.570 + outReg = regs.takeAny();
1.571 + masm.reserveStack(sizeof(int32_t));
1.572 + masm.movq(esp, outReg);
1.573 + break;
1.574 +
1.575 + case Type_Double:
1.576 + outReg = regs.takeAny();
1.577 + masm.reserveStack(sizeof(double));
1.578 + masm.movq(esp, outReg);
1.579 + break;
1.580 +
1.581 + case Type_Pointer:
1.582 + outReg = regs.takeAny();
1.583 + masm.reserveStack(sizeof(uintptr_t));
1.584 + masm.movq(esp, outReg);
1.585 + break;
1.586 +
1.587 + default:
1.588 + JS_ASSERT(f.outParam == Type_Void);
1.589 + break;
1.590 + }
1.591 +
1.592 + masm.setupUnalignedABICall(f.argc(), regs.getAny());
1.593 + masm.passABIArg(cxreg);
1.594 +
1.595 + size_t argDisp = 0;
1.596 +
1.597 + // Copy arguments.
1.598 + for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
1.599 + MoveOperand from;
1.600 + switch (f.argProperties(explicitArg)) {
1.601 + case VMFunction::WordByValue:
1.602 + if (f.argPassedInFloatReg(explicitArg))
1.603 + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
1.604 + else
1.605 + masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
1.606 + argDisp += sizeof(void *);
1.607 + break;
1.608 + case VMFunction::WordByRef:
1.609 + masm.passABIArg(MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
1.610 + MoveOp::GENERAL);
1.611 + argDisp += sizeof(void *);
1.612 + break;
1.613 + case VMFunction::DoubleByValue:
1.614 + case VMFunction::DoubleByRef:
1.615 + MOZ_ASSUME_UNREACHABLE("NYI: x64 callVM should not be used with 128bits values.");
1.616 + }
1.617 + }
1.618 +
1.619 + // Copy the implicit outparam, if any.
1.620 + if (outReg != InvalidReg)
1.621 + masm.passABIArg(outReg);
1.622 +
1.623 + masm.callWithABI(f.wrapped);
1.624 +
1.625 + // Test for failure.
1.626 + switch (f.failType()) {
1.627 + case Type_Object:
1.628 + masm.branchTestPtr(Assembler::Zero, rax, rax, masm.failureLabel(f.executionMode));
1.629 + break;
1.630 + case Type_Bool:
1.631 + masm.testb(rax, rax);
1.632 + masm.j(Assembler::Zero, masm.failureLabel(f.executionMode));
1.633 + break;
1.634 + default:
1.635 + MOZ_ASSUME_UNREACHABLE("unknown failure kind");
1.636 + }
1.637 +
1.638 + // Load the outparam and free any allocated stack.
1.639 + switch (f.outParam) {
1.640 + case Type_Handle:
1.641 + masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
1.642 + break;
1.643 +
1.644 + case Type_Value:
1.645 + masm.loadValue(Address(esp, 0), JSReturnOperand);
1.646 + masm.freeStack(sizeof(Value));
1.647 + break;
1.648 +
1.649 + case Type_Int32:
1.650 + masm.load32(Address(esp, 0), ReturnReg);
1.651 + masm.freeStack(sizeof(int32_t));
1.652 + break;
1.653 +
1.654 + case Type_Bool:
1.655 + masm.load8ZeroExtend(Address(esp, 0), ReturnReg);
1.656 + masm.freeStack(sizeof(int32_t));
1.657 + break;
1.658 +
1.659 + case Type_Double:
1.660 + JS_ASSERT(cx->runtime()->jitSupportsFloatingPoint);
1.661 + masm.loadDouble(Address(esp, 0), ReturnFloatReg);
1.662 + masm.freeStack(sizeof(double));
1.663 + break;
1.664 +
1.665 + case Type_Pointer:
1.666 + masm.loadPtr(Address(esp, 0), ReturnReg);
1.667 + masm.freeStack(sizeof(uintptr_t));
1.668 + break;
1.669 +
1.670 + default:
1.671 + JS_ASSERT(f.outParam == Type_Void);
1.672 + break;
1.673 + }
1.674 + masm.leaveExitFrame();
1.675 + masm.retn(Imm32(sizeof(IonExitFrameLayout) +
1.676 + f.explicitStackSlots() * sizeof(void *) +
1.677 + f.extraValuesToPop * sizeof(Value)));
1.678 +
1.679 + Linker linker(masm);
1.680 + JitCode *wrapper = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.681 + if (!wrapper)
1.682 + return nullptr;
1.683 +
1.684 +#ifdef JS_ION_PERF
1.685 + writePerfSpewerJitCodeProfile(wrapper, "VMWrapper");
1.686 +#endif
1.687 +
1.688 + // linker.newCode may trigger a GC and sweep functionWrappers_ so we have to
1.689 + // use relookupOrAdd instead of add.
1.690 + if (!functionWrappers_->relookupOrAdd(p, &f, wrapper))
1.691 + return nullptr;
1.692 +
1.693 + return wrapper;
1.694 +}
1.695 +
1.696 +JitCode *
1.697 +JitRuntime::generatePreBarrier(JSContext *cx, MIRType type)
1.698 +{
1.699 + MacroAssembler masm;
1.700 +
1.701 + RegisterSet regs = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
1.702 + FloatRegisterSet(FloatRegisters::VolatileMask));
1.703 + masm.PushRegsInMask(regs);
1.704 +
1.705 + JS_ASSERT(PreBarrierReg == rdx);
1.706 + masm.mov(ImmPtr(cx->runtime()), rcx);
1.707 +
1.708 + masm.setupUnalignedABICall(2, rax);
1.709 + masm.passABIArg(rcx);
1.710 + masm.passABIArg(rdx);
1.711 + if (type == MIRType_Value) {
1.712 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, MarkValueFromIon));
1.713 + } else {
1.714 + JS_ASSERT(type == MIRType_Shape);
1.715 + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void *, MarkShapeFromIon));
1.716 + }
1.717 +
1.718 + masm.PopRegsInMask(regs);
1.719 + masm.ret();
1.720 +
1.721 + Linker linker(masm);
1.722 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.723 +
1.724 +#ifdef JS_ION_PERF
1.725 + writePerfSpewerJitCodeProfile(code, "PreBarrier");
1.726 +#endif
1.727 +
1.728 + return code;
1.729 +}
1.730 +
1.731 +typedef bool (*HandleDebugTrapFn)(JSContext *, BaselineFrame *, uint8_t *, bool *);
1.732 +static const VMFunction HandleDebugTrapInfo = FunctionInfo<HandleDebugTrapFn>(HandleDebugTrap);
1.733 +
1.734 +JitCode *
1.735 +JitRuntime::generateDebugTrapHandler(JSContext *cx)
1.736 +{
1.737 + MacroAssembler masm;
1.738 +
1.739 + Register scratch1 = rax;
1.740 + Register scratch2 = rcx;
1.741 + Register scratch3 = rdx;
1.742 +
1.743 + // Load the return address in scratch1.
1.744 + masm.loadPtr(Address(rsp, 0), scratch1);
1.745 +
1.746 + // Load BaselineFrame pointer in scratch2.
1.747 + masm.mov(rbp, scratch2);
1.748 + masm.subPtr(Imm32(BaselineFrame::Size()), scratch2);
1.749 +
1.750 + // Enter a stub frame and call the HandleDebugTrap VM function. Ensure
1.751 + // the stub frame has a nullptr ICStub pointer, since this pointer is marked
1.752 + // during GC.
1.753 + masm.movePtr(ImmPtr(nullptr), BaselineStubReg);
1.754 + EmitEnterStubFrame(masm, scratch3);
1.755 +
1.756 + JitCode *code = cx->runtime()->jitRuntime()->getVMWrapper(HandleDebugTrapInfo);
1.757 + if (!code)
1.758 + return nullptr;
1.759 +
1.760 + masm.push(scratch1);
1.761 + masm.push(scratch2);
1.762 + EmitCallVM(code, masm);
1.763 +
1.764 + EmitLeaveStubFrame(masm);
1.765 +
1.766 + // If the stub returns |true|, we have to perform a forced return
1.767 + // (return from the JS frame). If the stub returns |false|, just return
1.768 + // from the trap stub so that execution continues at the current pc.
1.769 + Label forcedReturn;
1.770 + masm.branchTest32(Assembler::NonZero, ReturnReg, ReturnReg, &forcedReturn);
1.771 + masm.ret();
1.772 +
1.773 + masm.bind(&forcedReturn);
1.774 + masm.loadValue(Address(ebp, BaselineFrame::reverseOffsetOfReturnValue()),
1.775 + JSReturnOperand);
1.776 + masm.mov(rbp, rsp);
1.777 + masm.pop(rbp);
1.778 + masm.ret();
1.779 +
1.780 + Linker linker(masm);
1.781 + JitCode *codeDbg = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.782 +
1.783 +#ifdef JS_ION_PERF
1.784 + writePerfSpewerJitCodeProfile(codeDbg, "DebugTrapHandler");
1.785 +#endif
1.786 +
1.787 + return codeDbg;
1.788 +}
1.789 +
1.790 +JitCode *
1.791 +JitRuntime::generateExceptionTailStub(JSContext *cx)
1.792 +{
1.793 + MacroAssembler masm;
1.794 +
1.795 + masm.handleFailureWithHandlerTail();
1.796 +
1.797 + Linker linker(masm);
1.798 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.799 +
1.800 +#ifdef JS_ION_PERF
1.801 + writePerfSpewerJitCodeProfile(code, "ExceptionTailStub");
1.802 +#endif
1.803 +
1.804 + return code;
1.805 +}
1.806 +
1.807 +JitCode *
1.808 +JitRuntime::generateBailoutTailStub(JSContext *cx)
1.809 +{
1.810 + MacroAssembler masm;
1.811 +
1.812 + masm.generateBailoutTail(rdx, r9);
1.813 +
1.814 + Linker linker(masm);
1.815 + JitCode *code = linker.newCode<NoGC>(cx, JSC::OTHER_CODE);
1.816 +
1.817 +#ifdef JS_ION_PERF
1.818 + writePerfSpewerJitCodeProfile(code, "BailoutTailStub");
1.819 +#endif
1.820 +
1.821 + return code;
1.822 +}
