michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #ifndef MOZ_MEMORY
michael@0: #  error Should not compile this file when MOZ_MEMORY is not set
michael@0: #endif
michael@0: 
michael@0: #ifndef MOZ_REPLACE_MALLOC
michael@0: #  error Should not compile this file when replace-malloc is disabled
michael@0: #endif
michael@0: 
michael@0: #ifdef MOZ_NATIVE_JEMALLOC
michael@0: #  error Should not compile this file when we want to use native jemalloc
michael@0: #endif
michael@0: 
michael@0: #include "mozmemory_wrap.h"
michael@0: 
michael@0: /* Declare all je_* functions */
michael@0: #define MALLOC_DECL(name, return_type, ...) \
michael@0:   return_type je_ ## name(__VA_ARGS__);
michael@0: #include "malloc_decls.h"
michael@0: 
michael@0: #include "mozilla/Likely.h"
michael@0: /*
michael@0:  * Windows doesn't come with weak imports as they are possible with
michael@0:  * LD_PRELOAD or DYLD_INSERT_LIBRARIES on Linux/OSX. On this platform,
michael@0:  * the replacement functions are defined as variable pointers to the
michael@0:  * function resolved with GetProcAddress() instead of weak definitions
michael@0:  * of functions. On Android, the same needs to happen as well, because
michael@0:  * the Android linker doesn't handle weak linking with non LD_PRELOADed
michael@0:  * libraries, but LD_PRELOADing is not very convenient on Android, with
michael@0:  * the zygote.
michael@0:  */
michael@0: #ifdef XP_DARWIN
michael@0: #  define MOZ_REPLACE_WEAK __attribute__((weak_import))
michael@0: #elif defined(XP_WIN) || defined(MOZ_WIDGET_ANDROID)
michael@0: #  define MOZ_NO_REPLACE_FUNC_DECL
michael@0: #elif defined(__GNUC__)
michael@0: #  define MOZ_REPLACE_WEAK __attribute__((weak))
michael@0: #endif
michael@0: 
michael@0: #include "replace_malloc.h"
michael@0: 
michael@0: #define MALLOC_DECL(name, return_type, ...) \
michael@0:     je_ ## name,
michael@0: 
michael@0: static const malloc_table_t malloc_table = {
michael@0: #include "malloc_decls.h"
michael@0: };
michael@0: 
michael@0: #ifdef MOZ_NO_REPLACE_FUNC_DECL
michael@0: #  define MALLOC_DECL(name, return_type, ...) \
michael@0:     typedef return_type (replace_ ## name ## _impl_t)(__VA_ARGS__); \
michael@0:     replace_ ## name ## _impl_t *replace_ ## name = NULL;
michael@0: #  define MALLOC_FUNCS MALLOC_FUNCS_ALL
michael@0: #  include "malloc_decls.h"
michael@0: 
michael@0: #  ifdef XP_WIN
michael@0: #    include <windows.h>
michael@0: static void
michael@0: replace_malloc_init_funcs()
michael@0: {
michael@0:   char replace_malloc_lib[1024];
michael@0:   if (GetEnvironmentVariableA("MOZ_REPLACE_MALLOC_LIB", (LPSTR)&replace_malloc_lib,
michael@0:                               sizeof(replace_malloc_lib)) > 0) {
michael@0:     HMODULE handle = LoadLibraryA(replace_malloc_lib);
michael@0:     if (handle) {
michael@0: #define MALLOC_DECL(name, ...) \
michael@0:   replace_ ## name = (replace_ ## name ## _impl_t *) GetProcAddress(handle, "replace_" # name);
michael@0: 
michael@0: #  define MALLOC_FUNCS MALLOC_FUNCS_ALL
michael@0: #include "malloc_decls.h"
michael@0:     }
michael@0:   }
michael@0: }
michael@0: #  elif defined(MOZ_WIDGET_ANDROID)
michael@0: #    include <dlfcn.h>
michael@0: static void
michael@0: replace_malloc_init_funcs()
michael@0: {
michael@0:   char *replace_malloc_lib = getenv("MOZ_REPLACE_MALLOC_LIB");
michael@0:   if (replace_malloc_lib && *replace_malloc_lib) {
michael@0:     void *handle = dlopen(replace_malloc_lib, RTLD_LAZY);
michael@0:     if (handle) {
michael@0: #define MALLOC_DECL(name, ...) \
michael@0:   replace_ ## name = (replace_ ## name ## _impl_t *) dlsym(handle, "replace_" # name);
michael@0: 
michael@0: #  define MALLOC_FUNCS MALLOC_FUNCS_ALL
michael@0: #include "malloc_decls.h"
michael@0:     }
michael@0:   }
michael@0: }
michael@0: #  else
michael@0: #    error No implementation for replace_malloc_init_funcs()
michael@0: #  endif
michael@0: 
michael@0: #endif /* MOZ_NO_REPLACE_FUNC_DECL */
michael@0: 
michael@0: /*
michael@0:  * Below is the malloc implementation overriding jemalloc and calling the
michael@0:  * replacement functions if they exist.
michael@0:  */
michael@0: 
michael@0: /*
michael@0:  * On OSX, MOZ_MEMORY_API is defined to nothing, because malloc functions
michael@0:  * are meant to have hidden visibility. But since the functions are only
michael@0:  * used locally in the zone allocator further below, we can allow the
michael@0:  * compiler to optimize more by switching to static.
michael@0:  */
michael@0: #ifdef XP_DARWIN
michael@0: #undef MOZ_MEMORY_API
michael@0: #define MOZ_MEMORY_API static
michael@0: #endif
michael@0: 
michael@0: /*
michael@0:  * Malloc implementation functions are MOZ_MEMORY_API, and jemalloc
michael@0:  * specific functions MOZ_JEMALLOC_API; see mozmemory_wrap.h
michael@0:  */
michael@0: #define MALLOC_DECL(name, return_type, ...) \
michael@0:   MOZ_MEMORY_API return_type name ## _impl(__VA_ARGS__);
michael@0: #define MALLOC_FUNCS MALLOC_FUNCS_MALLOC
michael@0: #include "malloc_decls.h"
michael@0: 
michael@0: #define MALLOC_DECL(name, return_type, ...) \
michael@0:   MOZ_JEMALLOC_API return_type name ## _impl(__VA_ARGS__);
michael@0: #define MALLOC_FUNCS MALLOC_FUNCS_JEMALLOC
michael@0: #include "malloc_decls.h"
michael@0: 
michael@0: static int replace_malloc_initialized = 0;
michael@0: static void
michael@0: init()
michael@0: {
michael@0: #ifdef MOZ_NO_REPLACE_FUNC_DECL
michael@0:   replace_malloc_init_funcs();
michael@0: #endif
michael@0:   // Set this *before* calling replace_init, otherwise if replace_init calls
michael@0:   // malloc() we'll get an infinite loop.
michael@0:   replace_malloc_initialized = 1;
michael@0:   if (replace_init)
michael@0:     replace_init(&malloc_table);
michael@0: }
michael@0: 
michael@0: void*
michael@0: malloc_impl(size_t size)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_malloc))
michael@0:     return je_malloc(size);
michael@0:   return replace_malloc(size);
michael@0: }
michael@0: 
michael@0: int
michael@0: posix_memalign_impl(void **memptr, size_t alignment, size_t size)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_posix_memalign))
michael@0:     return je_posix_memalign(memptr, alignment, size);
michael@0:   return replace_posix_memalign(memptr, alignment, size);
michael@0: }
michael@0: 
michael@0: void*
michael@0: aligned_alloc_impl(size_t alignment, size_t size)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_aligned_alloc))
michael@0:     return je_aligned_alloc(alignment, size);
michael@0:   return replace_aligned_alloc(alignment, size);
michael@0: }
michael@0: 
michael@0: void*
michael@0: calloc_impl(size_t num, size_t size)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_calloc))
michael@0:     return je_calloc(num, size);
michael@0:   return replace_calloc(num, size);
michael@0: }
michael@0: 
michael@0: void*
michael@0: realloc_impl(void *ptr, size_t size)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_realloc))
michael@0:     return je_realloc(ptr, size);
michael@0:   return replace_realloc(ptr, size);
michael@0: }
michael@0: 
michael@0: void
michael@0: free_impl(void *ptr)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_free))
michael@0:     je_free(ptr);
michael@0:   else
michael@0:     replace_free(ptr);
michael@0: }
michael@0: 
michael@0: void*
michael@0: memalign_impl(size_t alignment, size_t size)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_memalign))
michael@0:     return je_memalign(alignment, size);
michael@0:   return replace_memalign(alignment, size);
michael@0: }
michael@0: 
michael@0: void*
michael@0: valloc_impl(size_t size)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_valloc))
michael@0:     return je_valloc(size);
michael@0:   return replace_valloc(size);
michael@0: }
michael@0: 
michael@0: size_t
michael@0: malloc_usable_size_impl(usable_ptr_t ptr)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_malloc_usable_size))
michael@0:     return je_malloc_usable_size(ptr);
michael@0:   return replace_malloc_usable_size(ptr);
michael@0: }
michael@0: 
michael@0: size_t
michael@0: malloc_good_size_impl(size_t size)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_malloc_good_size))
michael@0:     return je_malloc_good_size(size);
michael@0:   return replace_malloc_good_size(size);
michael@0: }
michael@0: 
michael@0: void
michael@0: jemalloc_stats_impl(jemalloc_stats_t *stats)
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_jemalloc_stats))
michael@0:     je_jemalloc_stats(stats);
michael@0:   else
michael@0:     replace_jemalloc_stats(stats);
michael@0: }
michael@0: 
michael@0: void
michael@0: jemalloc_purge_freed_pages_impl()
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_jemalloc_purge_freed_pages))
michael@0:     je_jemalloc_purge_freed_pages();
michael@0:   else
michael@0:     replace_jemalloc_purge_freed_pages();
michael@0: }
michael@0: 
michael@0: void
michael@0: jemalloc_free_dirty_pages_impl()
michael@0: {
michael@0:   if (MOZ_UNLIKELY(!replace_malloc_initialized))
michael@0:     init();
michael@0:   if (MOZ_LIKELY(!replace_jemalloc_free_dirty_pages))
michael@0:     je_jemalloc_free_dirty_pages();
michael@0:   else
michael@0:     replace_jemalloc_free_dirty_pages();
michael@0: }
michael@0: 
michael@0: /* The following comment and definitions are from jemalloc.c: */
michael@0: #if defined(__GLIBC__) && !defined(__UCLIBC__)
michael@0: 
michael@0: /*
michael@0:  * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
michael@0:  * to inconsistently reference libc's malloc(3)-compatible functions
michael@0:  * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
michael@0:  *
michael@0:  * These definitions interpose hooks in glibc.  The functions are actually
michael@0:  * passed an extra argument for the caller return address, which will be
michael@0:  * ignored.
michael@0:  */
michael@0: 
michael@0: typedef void (* __free_hook_type)(void *ptr);
michael@0: typedef void *(* __malloc_hook_type)(size_t size);
michael@0: typedef void *(* __realloc_hook_type)(void *ptr, size_t size);
michael@0: typedef void *(* __memalign_hook_type)(size_t alignment, size_t size);
michael@0: 
michael@0: MOZ_MEMORY_API __free_hook_type __free_hook = free_impl;
michael@0: MOZ_MEMORY_API __malloc_hook_type __malloc_hook = malloc_impl;
michael@0: MOZ_MEMORY_API __realloc_hook_type __realloc_hook = realloc_impl;
michael@0: MOZ_MEMORY_API __memalign_hook_type __memalign_hook = memalign_impl;
michael@0: 
michael@0: #endif
michael@0: 
michael@0: /*
michael@0:  * The following is a OSX zone allocator implementation.
michael@0:  * /!\ WARNING. It assumes the underlying malloc implementation's
michael@0:  * malloc_usable_size returns 0 when the given pointer is not owned by
michael@0:  * the allocator. Sadly, OSX does call zone_size with pointers not
michael@0:  * owned by the allocator.
michael@0:  */
michael@0: 
michael@0: #ifdef XP_DARWIN
michael@0: #include <stdlib.h>
michael@0: #include <malloc/malloc.h>
michael@0: #include "mozilla/Assertions.h"
michael@0: 
michael@0: static size_t
michael@0: zone_size(malloc_zone_t *zone, void *ptr)
michael@0: {
michael@0:   return malloc_usable_size_impl(ptr);
michael@0: }
michael@0: 
michael@0: static void *
michael@0: zone_malloc(malloc_zone_t *zone, size_t size)
michael@0: {
michael@0:   return malloc_impl(size);
michael@0: }
michael@0: 
michael@0: static void *
michael@0: zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
michael@0: {
michael@0:   return calloc_impl(num, size);
michael@0: }
michael@0: 
michael@0: static void *
michael@0: zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
michael@0: {
michael@0:   if (malloc_usable_size_impl(ptr))
michael@0:     return realloc_impl(ptr, size);
michael@0:   return realloc(ptr, size);
michael@0: }
michael@0: 
michael@0: static void
michael@0: zone_free(malloc_zone_t *zone, void *ptr)
michael@0: {
michael@0:   if (malloc_usable_size_impl(ptr)) {
michael@0:     free_impl(ptr);
michael@0:     return;
michael@0:   }
michael@0:   free(ptr);
michael@0: }
michael@0: 
michael@0: static void
michael@0: zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
michael@0: {
michael@0:   size_t current_size = malloc_usable_size_impl(ptr);
michael@0:   if (current_size) {
michael@0:     MOZ_ASSERT(current_size == size);
michael@0:     free_impl(ptr);
michael@0:     return;
michael@0:   }
michael@0:   free(ptr);
michael@0: }
michael@0: 
michael@0: static void *
michael@0: zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
michael@0: {
michael@0:   void *ptr;
michael@0:   if (posix_memalign_impl(&ptr, alignment, size) == 0)
michael@0:     return ptr;
michael@0:   return NULL;
michael@0: }
michael@0: 
michael@0: static void *
michael@0: zone_valloc(malloc_zone_t *zone, size_t size)
michael@0: {
michael@0:   return valloc_impl(size);
michael@0: }
michael@0: 
michael@0: static void *
michael@0: zone_destroy(malloc_zone_t *zone)
michael@0: {
michael@0:   /* This function should never be called. */
michael@0:   MOZ_CRASH();
michael@0: }
michael@0: 
michael@0: static size_t
michael@0: zone_good_size(malloc_zone_t *zone, size_t size)
michael@0: {
michael@0:   return malloc_good_size_impl(size);
michael@0: }
michael@0: 
michael@0: #ifdef MOZ_JEMALLOC
michael@0: 
michael@0: #include "jemalloc/internal/jemalloc_internal.h"
michael@0: 
michael@0: static void
michael@0: zone_force_lock(malloc_zone_t *zone)
michael@0: {
michael@0:   /* /!\ This calls into jemalloc. It works because we're linked in the
michael@0:    * same library. Stolen from jemalloc's zone.c. */
michael@0:   if (isthreaded)
michael@0:     jemalloc_prefork();
michael@0: }
michael@0: 
michael@0: static void
michael@0: zone_force_unlock(malloc_zone_t *zone)
michael@0: {
michael@0:   /* /!\ This calls into jemalloc. It works because we're linked in the
michael@0:    * same library. Stolen from jemalloc's zone.c. */
michael@0:   if (isthreaded)
michael@0:     jemalloc_postfork_parent();
michael@0: }
michael@0: 
michael@0: #else
michael@0: 
michael@0: #define JEMALLOC_ZONE_VERSION 6
michael@0: 
michael@0: /* Empty implementations are needed, because fork() calls zone->force_(un)lock
michael@0:  * unconditionally. */
michael@0: static void
michael@0: zone_force_lock(malloc_zone_t *zone)
michael@0: {
michael@0: }
michael@0: 
michael@0: static void
michael@0: zone_force_unlock(malloc_zone_t *zone)
michael@0: {
michael@0: }
michael@0: 
michael@0: #endif
michael@0: 
michael@0: static malloc_zone_t zone;
michael@0: static struct malloc_introspection_t zone_introspect;
michael@0: 
michael@0: __attribute__((constructor)) void
michael@0: register_zone(void)
michael@0: {
michael@0:   zone.size = (void *)zone_size;
michael@0:   zone.malloc = (void *)zone_malloc;
michael@0:   zone.calloc = (void *)zone_calloc;
michael@0:   zone.valloc = (void *)zone_valloc;
michael@0:   zone.free = (void *)zone_free;
michael@0:   zone.realloc = (void *)zone_realloc;
michael@0:   zone.destroy = (void *)zone_destroy;
michael@0:   zone.zone_name = "replace_malloc_zone";
michael@0:   zone.batch_malloc = NULL;
michael@0:   zone.batch_free = NULL;
michael@0:   zone.introspect = &zone_introspect;
michael@0:   zone.version = JEMALLOC_ZONE_VERSION;
michael@0:   zone.memalign = zone_memalign;
michael@0:   zone.free_definite_size = zone_free_definite_size;
michael@0: #if (JEMALLOC_ZONE_VERSION >= 8)
michael@0:   zone.pressure_relief = NULL;
michael@0: #endif
michael@0:   zone_introspect.enumerator = NULL;
michael@0:   zone_introspect.good_size = (void *)zone_good_size;
michael@0:   zone_introspect.check = NULL;
michael@0:   zone_introspect.print = NULL;
michael@0:   zone_introspect.log = NULL;
michael@0:   zone_introspect.force_lock = (void *)zone_force_lock;
michael@0:   zone_introspect.force_unlock = (void *)zone_force_unlock;
michael@0:   zone_introspect.statistics = NULL;
michael@0:   zone_introspect.zone_locked = NULL;
michael@0: #if (JEMALLOC_ZONE_VERSION >= 7)
michael@0:   zone_introspect.enable_discharge_checking = NULL;
michael@0:   zone_introspect.disable_discharge_checking = NULL;
michael@0:   zone_introspect.discharge = NULL;
michael@0: #ifdef __BLOCKS__
michael@0:   zone_introspect.enumerate_discharged_pointers = NULL;
michael@0: #else
michael@0:   zone_introspect.enumerate_unavailable_without_blocks = NULL;
michael@0: #endif
michael@0: #endif
michael@0: 
michael@0:   /*
michael@0:    * The default purgeable zone is created lazily by OSX's libc.  It uses
michael@0:    * the default zone when it is created for "small" allocations
michael@0:    * (< 15 KiB), but assumes the default zone is a scalable_zone.  This
michael@0:    * obviously fails when the default zone is the jemalloc zone, so
michael@0:    * malloc_default_purgeable_zone is called beforehand so that the
michael@0:    * default purgeable zone is created when the default zone is still
michael@0:    * a scalable_zone.  As purgeable zones only exist on >= 10.6, we need
michael@0:    * to check for the existence of malloc_default_purgeable_zone() at
michael@0:    * run time.
michael@0:    */
michael@0:   malloc_default_purgeable_zone();
michael@0: 
michael@0:   /* Register the custom zone.  At this point it won't be the default. */
michael@0:   malloc_zone_register(&zone);
michael@0: 
michael@0:   /*
michael@0:    * Unregister and reregister the default zone.  On OSX >= 10.6,
michael@0:    * unregistering takes the last registered zone and places it at the
michael@0:    * location of the specified zone.  Unregistering the default zone thus
michael@0:    * makes the last registered one the default.  On OSX < 10.6,
michael@0:    * unregistering shifts all registered zones.  The first registered zone
michael@0:    * then becomes the default.
michael@0:    */
michael@0:   do {
michael@0:     malloc_zone_t *default_zone = malloc_default_zone();
michael@0:     malloc_zone_unregister(default_zone);
michael@0:     malloc_zone_register(default_zone);
michael@0:   } while (malloc_default_zone() != &zone);
michael@0: }
michael@0: #endif