The Tor Browser / file revision

 /* This Source Code Form is subject to the terms of the Mozilla Public

  * License, v. 2.0. If a copy of the MPL was not distributed with this file,

  * You can obtain one at http://mozilla.org/MPL/2.0/. */

 #ifndef MOZ_MEMORY

 #  error Should not compile this file when MOZ_MEMORY is not set

 #endif

 #ifndef MOZ_REPLACE_MALLOC

 #  error Should not compile this file when replace-malloc is disabled

 #endif

 #ifdef MOZ_NATIVE_JEMALLOC

 #  error Should not compile this file when we want to use native jemalloc

 #endif

 #include "mozmemory_wrap.h"

 /* Declare all je_* functions */

 #define MALLOC_DECL(name, return_type, ...) \

   return_type je_ ## name(__VA_ARGS__);

 #include "malloc_decls.h"

 #include "mozilla/Likely.h"

 /*

  * Windows doesn't come with weak imports as they are possible with

  * LD_PRELOAD or DYLD_INSERT_LIBRARIES on Linux/OSX. On this platform,

  * the replacement functions are defined as variable pointers to the

  * function resolved with GetProcAddress() instead of weak definitions

  * of functions. On Android, the same needs to happen as well, because

  * the Android linker doesn't handle weak linking with non LD_PRELOADed

  * libraries, but LD_PRELOADing is not very convenient on Android, with

  * the zygote.

  */

 #ifdef XP_DARWIN

 #  define MOZ_REPLACE_WEAK __attribute__((weak_import))

 #elif defined(XP_WIN) || defined(MOZ_WIDGET_ANDROID)

 #  define MOZ_NO_REPLACE_FUNC_DECL

 #elif defined(__GNUC__)

 #  define MOZ_REPLACE_WEAK __attribute__((weak))

 #endif

 #include "replace_malloc.h"

 #define MALLOC_DECL(name, return_type, ...) \

     je_ ## name,

 static const malloc_table_t malloc_table = {

 #include "malloc_decls.h"

 };

 #ifdef MOZ_NO_REPLACE_FUNC_DECL

 #  define MALLOC_DECL(name, return_type, ...) \

     typedef return_type (replace_ ## name ## _impl_t)(__VA_ARGS__); \

     replace_ ## name ## _impl_t *replace_ ## name = NULL;

 #  define MALLOC_FUNCS MALLOC_FUNCS_ALL

 #  include "malloc_decls.h"

 #  ifdef XP_WIN

 #    include <windows.h>

 static void

 replace_malloc_init_funcs()

 {

   char replace_malloc_lib[1024];

   if (GetEnvironmentVariableA("MOZ_REPLACE_MALLOC_LIB", (LPSTR)&replace_malloc_lib,

                               sizeof(replace_malloc_lib)) > 0) {

     HMODULE handle = LoadLibraryA(replace_malloc_lib);

     if (handle) {

 #define MALLOC_DECL(name, ...) \

   replace_ ## name = (replace_ ## name ## _impl_t *) GetProcAddress(handle, "replace_" # name);

 #  define MALLOC_FUNCS MALLOC_FUNCS_ALL

 #include "malloc_decls.h"

     }

   }

 }

 #  elif defined(MOZ_WIDGET_ANDROID)

 #    include <dlfcn.h>

 static void

 replace_malloc_init_funcs()

 {

   char *replace_malloc_lib = getenv("MOZ_REPLACE_MALLOC_LIB");

   if (replace_malloc_lib && *replace_malloc_lib) {

     void *handle = dlopen(replace_malloc_lib, RTLD_LAZY);

     if (handle) {

 #define MALLOC_DECL(name, ...) \

   replace_ ## name = (replace_ ## name ## _impl_t *) dlsym(handle, "replace_" # name);

 #  define MALLOC_FUNCS MALLOC_FUNCS_ALL

 #include "malloc_decls.h"

     }

   }

 }

 #  else

 #    error No implementation for replace_malloc_init_funcs()

 #  endif

 #endif /* MOZ_NO_REPLACE_FUNC_DECL */

 /*

  * Below is the malloc implementation overriding jemalloc and calling the

  * replacement functions if they exist.

  */

 /*

  * On OSX, MOZ_MEMORY_API is defined to nothing, because malloc functions

  * are meant to have hidden visibility. But since the functions are only

  * used locally in the zone allocator further below, we can allow the

  * compiler to optimize more by switching to static.

  */

 #ifdef XP_DARWIN

 #undef MOZ_MEMORY_API

 #define MOZ_MEMORY_API static

 #endif

 /*

  * Malloc implementation functions are MOZ_MEMORY_API, and jemalloc

  * specific functions MOZ_JEMALLOC_API; see mozmemory_wrap.h

  */

 #define MALLOC_DECL(name, return_type, ...) \

   MOZ_MEMORY_API return_type name ## _impl(__VA_ARGS__);

 #define MALLOC_FUNCS MALLOC_FUNCS_MALLOC

 #include "malloc_decls.h"

 #define MALLOC_DECL(name, return_type, ...) \

   MOZ_JEMALLOC_API return_type name ## _impl(__VA_ARGS__);

 #define MALLOC_FUNCS MALLOC_FUNCS_JEMALLOC

 #include "malloc_decls.h"

 static int replace_malloc_initialized = 0;

 static void

 init()

 {

 #ifdef MOZ_NO_REPLACE_FUNC_DECL

   replace_malloc_init_funcs();

 #endif

   // Set this *before* calling replace_init, otherwise if replace_init calls

   // malloc() we'll get an infinite loop.

   replace_malloc_initialized = 1;

   if (replace_init)

     replace_init(&malloc_table);

 }

 void*

 malloc_impl(size_t size)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_malloc))

     return je_malloc(size);

   return replace_malloc(size);

 }

 int

 posix_memalign_impl(void **memptr, size_t alignment, size_t size)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_posix_memalign))

     return je_posix_memalign(memptr, alignment, size);

   return replace_posix_memalign(memptr, alignment, size);

 }

 void*

 aligned_alloc_impl(size_t alignment, size_t size)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_aligned_alloc))

     return je_aligned_alloc(alignment, size);

   return replace_aligned_alloc(alignment, size);

 }

 void*

 calloc_impl(size_t num, size_t size)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_calloc))

     return je_calloc(num, size);

   return replace_calloc(num, size);

 }

 void*

 realloc_impl(void *ptr, size_t size)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_realloc))

     return je_realloc(ptr, size);

   return replace_realloc(ptr, size);

 }

 void

 free_impl(void *ptr)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_free))

     je_free(ptr);

   else

     replace_free(ptr);

 }

 void*

 memalign_impl(size_t alignment, size_t size)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_memalign))

     return je_memalign(alignment, size);

   return replace_memalign(alignment, size);

 }

 void*

 valloc_impl(size_t size)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_valloc))

     return je_valloc(size);

   return replace_valloc(size);

 }

 size_t

 malloc_usable_size_impl(usable_ptr_t ptr)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_malloc_usable_size))

     return je_malloc_usable_size(ptr);

   return replace_malloc_usable_size(ptr);

 }

 size_t

 malloc_good_size_impl(size_t size)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_malloc_good_size))

     return je_malloc_good_size(size);

   return replace_malloc_good_size(size);

 }

 void

 jemalloc_stats_impl(jemalloc_stats_t *stats)

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_jemalloc_stats))

     je_jemalloc_stats(stats);

   else

     replace_jemalloc_stats(stats);

 }

 void

 jemalloc_purge_freed_pages_impl()

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_jemalloc_purge_freed_pages))

     je_jemalloc_purge_freed_pages();

   else

     replace_jemalloc_purge_freed_pages();

 }

 void

 jemalloc_free_dirty_pages_impl()

 {

   if (MOZ_UNLIKELY(!replace_malloc_initialized))

     init();

   if (MOZ_LIKELY(!replace_jemalloc_free_dirty_pages))

     je_jemalloc_free_dirty_pages();

   else

     replace_jemalloc_free_dirty_pages();

 }

 /* The following comment and definitions are from jemalloc.c: */

 #if defined(__GLIBC__) && !defined(__UCLIBC__)

 /*

  * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible

  * to inconsistently reference libc's malloc(3)-compatible functions

  * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).

  *

  * These definitions interpose hooks in glibc.  The functions are actually

  * passed an extra argument for the caller return address, which will be

  * ignored.

  */

 typedef void (* __free_hook_type)(void *ptr);

 typedef void *(* __malloc_hook_type)(size_t size);

 typedef void *(* __realloc_hook_type)(void *ptr, size_t size);

 typedef void *(* __memalign_hook_type)(size_t alignment, size_t size);

 MOZ_MEMORY_API __free_hook_type __free_hook = free_impl;

 MOZ_MEMORY_API __malloc_hook_type __malloc_hook = malloc_impl;

 MOZ_MEMORY_API __realloc_hook_type __realloc_hook = realloc_impl;

 MOZ_MEMORY_API __memalign_hook_type __memalign_hook = memalign_impl;

 #endif

 /*

  * The following is a OSX zone allocator implementation.

  * /!\ WARNING. It assumes the underlying malloc implementation's

  * malloc_usable_size returns 0 when the given pointer is not owned by

  * the allocator. Sadly, OSX does call zone_size with pointers not

  * owned by the allocator.

  */

 #ifdef XP_DARWIN

 #include <stdlib.h>

 #include <malloc/malloc.h>

 #include "mozilla/Assertions.h"

 static size_t

 zone_size(malloc_zone_t *zone, void *ptr)

 {

   return malloc_usable_size_impl(ptr);

 }

 static void *

 zone_malloc(malloc_zone_t *zone, size_t size)

 {

   return malloc_impl(size);

 }

 static void *

 zone_calloc(malloc_zone_t *zone, size_t num, size_t size)

 {

   return calloc_impl(num, size);

 }

 static void *

 zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)

 {

   if (malloc_usable_size_impl(ptr))

     return realloc_impl(ptr, size);

   return realloc(ptr, size);

 }

 static void

 zone_free(malloc_zone_t *zone, void *ptr)

 {

   if (malloc_usable_size_impl(ptr)) {

     free_impl(ptr);

     return;

   }

   free(ptr);

 }

 static void

 zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)

 {

   size_t current_size = malloc_usable_size_impl(ptr);

   if (current_size) {

     MOZ_ASSERT(current_size == size);

     free_impl(ptr);

     return;

   }

   free(ptr);

 }

 static void *

 zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)

 {

   void *ptr;

   if (posix_memalign_impl(&ptr, alignment, size) == 0)

     return ptr;

   return NULL;

 }

 static void *

 zone_valloc(malloc_zone_t *zone, size_t size)

 {

   return valloc_impl(size);

 }

 static void *

 zone_destroy(malloc_zone_t *zone)

 {

   /* This function should never be called. */

   MOZ_CRASH();

 }

 static size_t

 zone_good_size(malloc_zone_t *zone, size_t size)

 {

   return malloc_good_size_impl(size);

 }

 #ifdef MOZ_JEMALLOC

 #include "jemalloc/internal/jemalloc_internal.h"

 static void

 zone_force_lock(malloc_zone_t *zone)

 {

   /* /!\ This calls into jemalloc. It works because we're linked in the

    * same library. Stolen from jemalloc's zone.c. */

   if (isthreaded)

     jemalloc_prefork();

 }

 static void

 zone_force_unlock(malloc_zone_t *zone)

 {

   /* /!\ This calls into jemalloc. It works because we're linked in the

    * same library. Stolen from jemalloc's zone.c. */

   if (isthreaded)

     jemalloc_postfork_parent();

 }

 #else

 #define JEMALLOC_ZONE_VERSION 6

 /* Empty implementations are needed, because fork() calls zone->force_(un)lock

  * unconditionally. */

 static void

 zone_force_lock(malloc_zone_t *zone)

 {

 }

 static void

 zone_force_unlock(malloc_zone_t *zone)

 {

 }

 #endif

 static malloc_zone_t zone;

 static struct malloc_introspection_t zone_introspect;

 __attribute__((constructor)) void

 register_zone(void)

 {

   zone.size = (void *)zone_size;

   zone.malloc = (void *)zone_malloc;

   zone.calloc = (void *)zone_calloc;

   zone.valloc = (void *)zone_valloc;

   zone.free = (void *)zone_free;

   zone.realloc = (void *)zone_realloc;

   zone.destroy = (void *)zone_destroy;

   zone.zone_name = "replace_malloc_zone";

   zone.batch_malloc = NULL;

   zone.batch_free = NULL;

   zone.introspect = &zone_introspect;

   zone.version = JEMALLOC_ZONE_VERSION;

   zone.memalign = zone_memalign;

   zone.free_definite_size = zone_free_definite_size;

 #if (JEMALLOC_ZONE_VERSION >= 8)

   zone.pressure_relief = NULL;

 #endif

   zone_introspect.enumerator = NULL;

   zone_introspect.good_size = (void *)zone_good_size;

   zone_introspect.check = NULL;

   zone_introspect.print = NULL;

   zone_introspect.log = NULL;

   zone_introspect.force_lock = (void *)zone_force_lock;

   zone_introspect.force_unlock = (void *)zone_force_unlock;

   zone_introspect.statistics = NULL;

   zone_introspect.zone_locked = NULL;

 #if (JEMALLOC_ZONE_VERSION >= 7)

   zone_introspect.enable_discharge_checking = NULL;

   zone_introspect.disable_discharge_checking = NULL;

   zone_introspect.discharge = NULL;

 #ifdef __BLOCKS__

   zone_introspect.enumerate_discharged_pointers = NULL;

 #else

   zone_introspect.enumerate_unavailable_without_blocks = NULL;

 #endif

 #endif

   /*

    * The default purgeable zone is created lazily by OSX's libc.  It uses

    * the default zone when it is created for "small" allocations

    * (< 15 KiB), but assumes the default zone is a scalable_zone.  This

    * obviously fails when the default zone is the jemalloc zone, so

    * malloc_default_purgeable_zone is called beforehand so that the

    * default purgeable zone is created when the default zone is still

    * a scalable_zone.  As purgeable zones only exist on >= 10.6, we need

    * to check for the existence of malloc_default_purgeable_zone() at

    * run time.

    */

   malloc_default_purgeable_zone();

   /* Register the custom zone.  At this point it won't be the default. */

   malloc_zone_register(&zone);

   /*

    * Unregister and reregister the default zone.  On OSX >= 10.6,

    * unregistering takes the last registered zone and places it at the

    * location of the specified zone.  Unregistering the default zone thus

    * makes the last registered one the default.  On OSX < 10.6,

    * unregistering shifts all registered zones.  The first registered zone

    * then becomes the default.

    */

   do {

     malloc_zone_t *default_zone = malloc_default_zone();

     malloc_zone_unregister(default_zone);

     malloc_zone_register(default_zone);

   } while (malloc_default_zone() != &zone);

 }

 #endif