michael@0: /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
michael@0:  * vim: set ts=8 sts=4 et sw=4 tw=99:
michael@0:  * This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "gc/Memory.h"
michael@0: 
michael@0: #include "js/HeapAPI.h"
michael@0: #include "vm/Runtime.h"
michael@0: 
michael@0: using namespace js;
michael@0: using namespace js::gc;
michael@0: 
michael@0: static bool
michael@0: DecommitEnabled(JSRuntime *rt)
michael@0: {
michael@0:     return rt->gcSystemPageSize == ArenaSize;
michael@0: }
michael@0: 
michael@0: #if defined(XP_WIN)
michael@0: #include "jswin.h"
michael@0: #include <psapi.h>
michael@0: 
michael@0: void
michael@0: gc::InitMemorySubsystem(JSRuntime *rt)
michael@0: {
michael@0:     SYSTEM_INFO sysinfo;
michael@0:     GetSystemInfo(&sysinfo);
michael@0:     rt->gcSystemPageSize = sysinfo.dwPageSize;
michael@0:     rt->gcSystemAllocGranularity = sysinfo.dwAllocationGranularity;
michael@0: }
michael@0: 
michael@0: void *
michael@0: gc::MapAlignedPages(JSRuntime *rt, size_t size, size_t alignment)
michael@0: {
michael@0:     JS_ASSERT(size >= alignment);
michael@0:     JS_ASSERT(size % alignment == 0);
michael@0:     JS_ASSERT(size % rt->gcSystemPageSize == 0);
michael@0:     JS_ASSERT(alignment % rt->gcSystemAllocGranularity == 0);
michael@0: 
michael@0:     /* Special case: If we want allocation alignment, no further work is needed. */
michael@0:     if (alignment == rt->gcSystemAllocGranularity) {
michael@0:         return VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
michael@0:     }
michael@0: 
michael@0:     /*
michael@0:      * Windows requires that there be a 1:1 mapping between VM allocation
michael@0:      * and deallocation operations.  Therefore, take care here to acquire the
michael@0:      * final result via one mapping operation.  This means unmapping any
michael@0:      * preliminary result that is not correctly aligned.
michael@0:      */
michael@0:     void *p = nullptr;
michael@0:     while (!p) {
michael@0:         /*
michael@0:          * Over-allocate in order to map a memory region that is definitely
michael@0:          * large enough, then deallocate and allocate again the correct size,
michael@0:          * within the over-sized mapping.
michael@0:          *
michael@0:          * Since we're going to unmap the whole thing anyway, the first
michael@0:          * mapping doesn't have to commit pages.
michael@0:          */
michael@0:         size_t reserveSize = size + alignment - rt->gcSystemPageSize;
michael@0:         p = VirtualAlloc(nullptr, reserveSize, MEM_RESERVE, PAGE_READWRITE);
michael@0:         if (!p)
michael@0:             return nullptr;
michael@0:         void *chunkStart = (void *)AlignBytes(uintptr_t(p), alignment);
michael@0:         UnmapPages(rt, p, reserveSize);
michael@0:         p = VirtualAlloc(chunkStart, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
michael@0: 
michael@0:         /* Failure here indicates a race with another thread, so try again. */
michael@0:     }
michael@0: 
michael@0:     JS_ASSERT(uintptr_t(p) % alignment == 0);
michael@0:     return p;
michael@0: }
michael@0: 
michael@0: void
michael@0: gc::UnmapPages(JSRuntime *rt, void *p, size_t size)
michael@0: {
michael@0:     JS_ALWAYS_TRUE(VirtualFree(p, 0, MEM_RELEASE));
michael@0: }
michael@0: 
michael@0: bool
michael@0: gc::MarkPagesUnused(JSRuntime *rt, void *p, size_t size)
michael@0: {
michael@0:     if (!DecommitEnabled(rt))
michael@0:         return true;
michael@0: 
michael@0:     JS_ASSERT(uintptr_t(p) % rt->gcSystemPageSize == 0);
michael@0:     LPVOID p2 = VirtualAlloc(p, size, MEM_RESET, PAGE_READWRITE);
michael@0:     return p2 == p;
michael@0: }
michael@0: 
michael@0: bool
michael@0: gc::MarkPagesInUse(JSRuntime *rt, void *p, size_t size)
michael@0: {
michael@0:     JS_ASSERT(uintptr_t(p) % rt->gcSystemPageSize == 0);
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: size_t
michael@0: gc::GetPageFaultCount()
michael@0: {
michael@0:     PROCESS_MEMORY_COUNTERS pmc;
michael@0:     if (!GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)))
michael@0:         return 0;
michael@0:     return pmc.PageFaultCount;
michael@0: }
michael@0: 
michael@0: void *
michael@0: gc::AllocateMappedContent(int fd, size_t offset, size_t length, size_t alignment)
michael@0: {
michael@0:     // TODO: Bug 988813 - Support memory mapped array buffer for Windows platform.
michael@0:     return nullptr;
michael@0: }
michael@0: 
michael@0: // Deallocate mapped memory for object.
michael@0: void
michael@0: gc::DeallocateMappedContent(void *p, size_t length)
michael@0: {
michael@0:     // TODO: Bug 988813 - Support memory mapped array buffer for Windows platform.
michael@0: }
michael@0: 
michael@0: #elif defined(SOLARIS)
michael@0: 
michael@0: #include <sys/mman.h>
michael@0: #include <unistd.h>
michael@0: 
michael@0: #ifndef MAP_NOSYNC
michael@0: # define MAP_NOSYNC 0
michael@0: #endif
michael@0: 
michael@0: void
michael@0: gc::InitMemorySubsystem(JSRuntime *rt)
michael@0: {
michael@0:     rt->gcSystemPageSize = rt->gcSystemAllocGranularity = size_t(sysconf(_SC_PAGESIZE));
michael@0: }
michael@0: 
michael@0: void *
michael@0: gc::MapAlignedPages(JSRuntime *rt, size_t size, size_t alignment)
michael@0: {
michael@0:     JS_ASSERT(size >= alignment);
michael@0:     JS_ASSERT(size % alignment == 0);
michael@0:     JS_ASSERT(size % rt->gcSystemPageSize == 0);
michael@0:     JS_ASSERT(alignment % rt->gcSystemAllocGranularity == 0);
michael@0: 
michael@0:     int prot = PROT_READ | PROT_WRITE;
michael@0:     int flags = MAP_PRIVATE | MAP_ANON | MAP_ALIGN | MAP_NOSYNC;
michael@0: 
michael@0:     void *p = mmap((caddr_t)alignment, size, prot, flags, -1, 0);
michael@0:     if (p == MAP_FAILED)
michael@0:         return nullptr;
michael@0:     return p;
michael@0: }
michael@0: 
michael@0: void
michael@0: gc::UnmapPages(JSRuntime *rt, void *p, size_t size)
michael@0: {
michael@0:     JS_ALWAYS_TRUE(0 == munmap((caddr_t)p, size));
michael@0: }
michael@0: 
michael@0: bool
michael@0: gc::MarkPagesUnused(JSRuntime *rt, void *p, size_t size)
michael@0: {
michael@0:     JS_ASSERT(uintptr_t(p) % rt->gcSystemPageSize == 0);
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: bool
michael@0: gc::MarkPagesInUse(JSRuntime *rt, void *p, size_t size)
michael@0: {
michael@0:     JS_ASSERT(uintptr_t(p) % rt->gcSystemPageSize == 0);
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: size_t
michael@0: gc::GetPageFaultCount()
michael@0: {
michael@0:     return 0;
michael@0: }
michael@0: 
michael@0: void *
michael@0: gc::AllocateMappedContent(int fd, size_t offset, size_t length, size_t alignment)
michael@0: {
michael@0:     // Not implemented.
michael@0:     return nullptr;
michael@0: }
michael@0: 
michael@0: // Deallocate mapped memory for object.
michael@0: void
michael@0: gc::DeallocateMappedContent(void *p, size_t length)
michael@0: {
michael@0:     // Not implemented.
michael@0: }
michael@0: 
michael@0: #elif defined(XP_UNIX)
michael@0: 
michael@0: #include <algorithm>
michael@0: #include <sys/mman.h>
michael@0: #include <sys/resource.h>
michael@0: #include <sys/stat.h>
michael@0: #include <sys/types.h>
michael@0: #include <unistd.h>
michael@0: 
michael@0: void
michael@0: gc::InitMemorySubsystem(JSRuntime *rt)
michael@0: {
michael@0:     rt->gcSystemPageSize = rt->gcSystemAllocGranularity = size_t(sysconf(_SC_PAGESIZE));
michael@0: }
michael@0: 
michael@0: static inline void *
michael@0: MapMemory(size_t length, int prot, int flags, int fd, off_t offset)
michael@0: {
michael@0: #if defined(__ia64__)
michael@0:     /*
michael@0:      * The JS engine assumes that all allocated pointers have their high 17 bits clear,
michael@0:      * which ia64's mmap doesn't support directly. However, we can emulate it by passing
michael@0:      * mmap an "addr" parameter with those bits clear. The mmap will return that address,
michael@0:      * or the nearest available memory above that address, providing a near-guarantee
michael@0:      * that those bits are clear. If they are not, we return nullptr below to indicate
michael@0:      * out-of-memory.
michael@0:      *
michael@0:      * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual
michael@0:      * address space.
michael@0:      *
michael@0:      * See Bug 589735 for more information.
michael@0:      */
michael@0:     void *region = mmap((void*)0x0000070000000000, length, prot, flags, fd, offset);
michael@0:     if (region == MAP_FAILED)
michael@0:         return MAP_FAILED;
michael@0:     /*
michael@0:      * If the allocated memory doesn't have its upper 17 bits clear, consider it
michael@0:      * as out of memory.
michael@0:      */
michael@0:     if ((uintptr_t(region) + (length - 1)) & 0xffff800000000000) {
michael@0:         JS_ALWAYS_TRUE(0 == munmap(region, length));
michael@0:         return MAP_FAILED;
michael@0:     }
michael@0:     return region;
michael@0: #else
michael@0:     return mmap(nullptr, length, prot, flags, fd, offset);
michael@0: #endif
michael@0: }
michael@0: 
michael@0: void *
michael@0: gc::MapAlignedPages(JSRuntime *rt, size_t size, size_t alignment)
michael@0: {
michael@0:     JS_ASSERT(size >= alignment);
michael@0:     JS_ASSERT(size % alignment == 0);
michael@0:     JS_ASSERT(size % rt->gcSystemPageSize == 0);
michael@0:     JS_ASSERT(alignment % rt->gcSystemAllocGranularity == 0);
michael@0: 
michael@0:     int prot = PROT_READ | PROT_WRITE;
michael@0:     int flags = MAP_PRIVATE | MAP_ANON;
michael@0: 
michael@0:     /* Special case: If we want page alignment, no further work is needed. */
michael@0:     if (alignment == rt->gcSystemAllocGranularity) {
michael@0:         void *region = MapMemory(size, prot, flags, -1, 0);
michael@0:         if (region == MAP_FAILED)
michael@0:             return nullptr;
michael@0:         return region;
michael@0:     }
michael@0: 
michael@0:     /* Overallocate and unmap the region's edges. */
michael@0:     size_t reqSize = Min(size + 2 * alignment, 2 * size);
michael@0:     void *region = MapMemory(reqSize, prot, flags, -1, 0);
michael@0:     if (region == MAP_FAILED)
michael@0:         return nullptr;
michael@0: 
michael@0:     uintptr_t regionEnd = uintptr_t(region) + reqSize;
michael@0:     uintptr_t offset = uintptr_t(region) % alignment;
michael@0:     JS_ASSERT(offset < reqSize - size);
michael@0: 
michael@0:     void *front = (void *)AlignBytes(uintptr_t(region), alignment);
michael@0:     void *end = (void *)(uintptr_t(front) + size);
michael@0:     if (front != region)
michael@0:         JS_ALWAYS_TRUE(0 == munmap(region, alignment - offset));
michael@0:     if (uintptr_t(end) != regionEnd)
michael@0:         JS_ALWAYS_TRUE(0 == munmap(end, regionEnd - uintptr_t(end)));
michael@0: 
michael@0:     JS_ASSERT(uintptr_t(front) % alignment == 0);
michael@0:     return front;
michael@0: }
michael@0: 
michael@0: void
michael@0: gc::UnmapPages(JSRuntime *rt, void *p, size_t size)
michael@0: {
michael@0:     JS_ALWAYS_TRUE(0 == munmap(p, size));
michael@0: }
michael@0: 
michael@0: bool
michael@0: gc::MarkPagesUnused(JSRuntime *rt, void *p, size_t size)
michael@0: {
michael@0:     if (!DecommitEnabled(rt))
michael@0:         return false;
michael@0: 
michael@0:     JS_ASSERT(uintptr_t(p) % rt->gcSystemPageSize == 0);
michael@0:     int result = madvise(p, size, MADV_DONTNEED);
michael@0:     return result != -1;
michael@0: }
michael@0: 
michael@0: bool
michael@0: gc::MarkPagesInUse(JSRuntime *rt, void *p, size_t size)
michael@0: {
michael@0:     JS_ASSERT(uintptr_t(p) % rt->gcSystemPageSize == 0);
michael@0:     return true;
michael@0: }
michael@0: 
michael@0: size_t
michael@0: gc::GetPageFaultCount()
michael@0: {
michael@0:     struct rusage usage;
michael@0:     int err = getrusage(RUSAGE_SELF, &usage);
michael@0:     if (err)
michael@0:         return 0;
michael@0:     return usage.ru_majflt;
michael@0: }
michael@0: 
michael@0: void *
michael@0: gc::AllocateMappedContent(int fd, size_t offset, size_t length, size_t alignment)
michael@0: {
michael@0: #define NEED_PAGE_ALIGNED 0
michael@0:     size_t pa_start; // Page aligned starting
michael@0:     size_t pa_end; // Page aligned ending
michael@0:     size_t pa_size; // Total page aligned size
michael@0:     size_t page_size = sysconf(_SC_PAGESIZE); // Page size
michael@0:     struct stat st;
michael@0:     uint8_t *buf;
michael@0: 
michael@0:     // Make sure file exists and do sanity check for offset and size.
michael@0:     if (fstat(fd, &st) < 0 || offset >= (size_t) st.st_size ||
michael@0:         length == 0 || length > (size_t) st.st_size - offset)
michael@0:         return nullptr;
michael@0: 
michael@0:     // Check for minimal alignment requirement.
michael@0: #if NEED_PAGE_ALIGNED
michael@0:     alignment = std::max(alignment, page_size);
michael@0: #endif
michael@0:     if (offset & (alignment - 1))
michael@0:         return nullptr;
michael@0: 
michael@0:     // Page aligned starting of the offset.
michael@0:     pa_start = offset & ~(page_size - 1);
michael@0:     // Calculate page aligned ending by adding one page to the page aligned
michael@0:     // starting of data end position(offset + length - 1).
michael@0:     pa_end = ((offset + length - 1) & ~(page_size - 1)) + page_size;
michael@0:     pa_size = pa_end - pa_start;
michael@0: 
michael@0:     // Ask for a continuous memory location.
michael@0:     buf = (uint8_t *) MapMemory(pa_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
michael@0:     if (buf == MAP_FAILED)
michael@0:         return nullptr;
michael@0: 
michael@0:     buf = (uint8_t *) mmap(buf, pa_size, PROT_READ | PROT_WRITE,
michael@0:                            MAP_PRIVATE | MAP_FIXED, fd, pa_start);
michael@0:     if (buf == MAP_FAILED)
michael@0:         return nullptr;
michael@0: 
michael@0:     // Reset the data before target file, which we don't need to see.
michael@0:     memset(buf, 0, offset - pa_start);
michael@0: 
michael@0:     // Reset the data after target file, which we don't need to see.
michael@0:     memset(buf + (offset - pa_start) + length, 0, pa_end - (offset + length));
michael@0: 
michael@0:     return buf + (offset - pa_start);
michael@0: }
michael@0: 
michael@0: void
michael@0: gc::DeallocateMappedContent(void *p, size_t length)
michael@0: {
michael@0:     void *pa_start; // Page aligned starting
michael@0:     size_t page_size = sysconf(_SC_PAGESIZE); // Page size
michael@0:     size_t total_size; // Total allocated size
michael@0: 
michael@0:     pa_start = (void *)(uintptr_t(p) & ~(page_size - 1));
michael@0:     total_size = ((uintptr_t(p) + length) & ~(page_size - 1)) + page_size - uintptr_t(pa_start);
michael@0:     munmap(pa_start, total_size);
michael@0: }
michael@0: 
michael@0: #else
michael@0: #error "Memory mapping functions are not defined for your OS."
michael@0: #endif