1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/memory/build/replace_malloc.c Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,505 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this file,
1.6 + * You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +#ifndef MOZ_MEMORY
1.9 +# error Should not compile this file when MOZ_MEMORY is not set
1.10 +#endif
1.11 +
1.12 +#ifndef MOZ_REPLACE_MALLOC
1.13 +# error Should not compile this file when replace-malloc is disabled
1.14 +#endif
1.15 +
1.16 +#ifdef MOZ_NATIVE_JEMALLOC
1.17 +# error Should not compile this file when we want to use native jemalloc
1.18 +#endif
1.19 +
1.20 +#include "mozmemory_wrap.h"
1.21 +
1.22 +/* Declare all je_* functions */
1.23 +#define MALLOC_DECL(name, return_type, ...) \
1.24 + return_type je_ ## name(__VA_ARGS__);
1.25 +#include "malloc_decls.h"
1.26 +
1.27 +#include "mozilla/Likely.h"
1.28 +/*
1.29 + * Windows doesn't come with weak imports as they are possible with
1.30 + * LD_PRELOAD or DYLD_INSERT_LIBRARIES on Linux/OSX. On this platform,
1.31 + * the replacement functions are defined as variable pointers to the
1.32 + * function resolved with GetProcAddress() instead of weak definitions
1.33 + * of functions. On Android, the same needs to happen as well, because
1.34 + * the Android linker doesn't handle weak linking with non LD_PRELOADed
1.35 + * libraries, but LD_PRELOADing is not very convenient on Android, with
1.36 + * the zygote.
1.37 + */
1.38 +#ifdef XP_DARWIN
1.39 +# define MOZ_REPLACE_WEAK __attribute__((weak_import))
1.40 +#elif defined(XP_WIN) || defined(MOZ_WIDGET_ANDROID)
1.41 +# define MOZ_NO_REPLACE_FUNC_DECL
1.42 +#elif defined(__GNUC__)
1.43 +# define MOZ_REPLACE_WEAK __attribute__((weak))
1.44 +#endif
1.45 +
1.46 +#include "replace_malloc.h"
1.47 +
1.48 +#define MALLOC_DECL(name, return_type, ...) \
1.49 + je_ ## name,
1.50 +
1.51 +static const malloc_table_t malloc_table = {
1.52 +#include "malloc_decls.h"
1.53 +};
1.54 +
1.55 +#ifdef MOZ_NO_REPLACE_FUNC_DECL
1.56 +# define MALLOC_DECL(name, return_type, ...) \
1.57 + typedef return_type (replace_ ## name ## _impl_t)(__VA_ARGS__); \
1.58 + replace_ ## name ## _impl_t *replace_ ## name = NULL;
1.59 +# define MALLOC_FUNCS MALLOC_FUNCS_ALL
1.60 +# include "malloc_decls.h"
1.61 +
1.62 +# ifdef XP_WIN
1.63 +# include <windows.h>
1.64 +static void
1.65 +replace_malloc_init_funcs()
1.66 +{
1.67 + char replace_malloc_lib[1024];
1.68 + if (GetEnvironmentVariableA("MOZ_REPLACE_MALLOC_LIB", (LPSTR)&replace_malloc_lib,
1.69 + sizeof(replace_malloc_lib)) > 0) {
1.70 + HMODULE handle = LoadLibraryA(replace_malloc_lib);
1.71 + if (handle) {
1.72 +#define MALLOC_DECL(name, ...) \
1.73 + replace_ ## name = (replace_ ## name ## _impl_t *) GetProcAddress(handle, "replace_" # name);
1.74 +
1.75 +# define MALLOC_FUNCS MALLOC_FUNCS_ALL
1.76 +#include "malloc_decls.h"
1.77 + }
1.78 + }
1.79 +}
1.80 +# elif defined(MOZ_WIDGET_ANDROID)
1.81 +# include <dlfcn.h>
1.82 +static void
1.83 +replace_malloc_init_funcs()
1.84 +{
1.85 + char *replace_malloc_lib = getenv("MOZ_REPLACE_MALLOC_LIB");
1.86 + if (replace_malloc_lib && *replace_malloc_lib) {
1.87 + void *handle = dlopen(replace_malloc_lib, RTLD_LAZY);
1.88 + if (handle) {
1.89 +#define MALLOC_DECL(name, ...) \
1.90 + replace_ ## name = (replace_ ## name ## _impl_t *) dlsym(handle, "replace_" # name);
1.91 +
1.92 +# define MALLOC_FUNCS MALLOC_FUNCS_ALL
1.93 +#include "malloc_decls.h"
1.94 + }
1.95 + }
1.96 +}
1.97 +# else
1.98 +# error No implementation for replace_malloc_init_funcs()
1.99 +# endif
1.100 +
1.101 +#endif /* MOZ_NO_REPLACE_FUNC_DECL */
1.102 +
1.103 +/*
1.104 + * Below is the malloc implementation overriding jemalloc and calling the
1.105 + * replacement functions if they exist.
1.106 + */
1.107 +
1.108 +/*
1.109 + * On OSX, MOZ_MEMORY_API is defined to nothing, because malloc functions
1.110 + * are meant to have hidden visibility. But since the functions are only
1.111 + * used locally in the zone allocator further below, we can allow the
1.112 + * compiler to optimize more by switching to static.
1.113 + */
1.114 +#ifdef XP_DARWIN
1.115 +#undef MOZ_MEMORY_API
1.116 +#define MOZ_MEMORY_API static
1.117 +#endif
1.118 +
1.119 +/*
1.120 + * Malloc implementation functions are MOZ_MEMORY_API, and jemalloc
1.121 + * specific functions MOZ_JEMALLOC_API; see mozmemory_wrap.h
1.122 + */
1.123 +#define MALLOC_DECL(name, return_type, ...) \
1.124 + MOZ_MEMORY_API return_type name ## _impl(__VA_ARGS__);
1.125 +#define MALLOC_FUNCS MALLOC_FUNCS_MALLOC
1.126 +#include "malloc_decls.h"
1.127 +
1.128 +#define MALLOC_DECL(name, return_type, ...) \
1.129 + MOZ_JEMALLOC_API return_type name ## _impl(__VA_ARGS__);
1.130 +#define MALLOC_FUNCS MALLOC_FUNCS_JEMALLOC
1.131 +#include "malloc_decls.h"
1.132 +
1.133 +static int replace_malloc_initialized = 0;
1.134 +static void
1.135 +init()
1.136 +{
1.137 +#ifdef MOZ_NO_REPLACE_FUNC_DECL
1.138 + replace_malloc_init_funcs();
1.139 +#endif
1.140 + // Set this *before* calling replace_init, otherwise if replace_init calls
1.141 + // malloc() we'll get an infinite loop.
1.142 + replace_malloc_initialized = 1;
1.143 + if (replace_init)
1.144 + replace_init(&malloc_table);
1.145 +}
1.146 +
1.147 +void*
1.148 +malloc_impl(size_t size)
1.149 +{
1.150 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.151 + init();
1.152 + if (MOZ_LIKELY(!replace_malloc))
1.153 + return je_malloc(size);
1.154 + return replace_malloc(size);
1.155 +}
1.156 +
1.157 +int
1.158 +posix_memalign_impl(void **memptr, size_t alignment, size_t size)
1.159 +{
1.160 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.161 + init();
1.162 + if (MOZ_LIKELY(!replace_posix_memalign))
1.163 + return je_posix_memalign(memptr, alignment, size);
1.164 + return replace_posix_memalign(memptr, alignment, size);
1.165 +}
1.166 +
1.167 +void*
1.168 +aligned_alloc_impl(size_t alignment, size_t size)
1.169 +{
1.170 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.171 + init();
1.172 + if (MOZ_LIKELY(!replace_aligned_alloc))
1.173 + return je_aligned_alloc(alignment, size);
1.174 + return replace_aligned_alloc(alignment, size);
1.175 +}
1.176 +
1.177 +void*
1.178 +calloc_impl(size_t num, size_t size)
1.179 +{
1.180 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.181 + init();
1.182 + if (MOZ_LIKELY(!replace_calloc))
1.183 + return je_calloc(num, size);
1.184 + return replace_calloc(num, size);
1.185 +}
1.186 +
1.187 +void*
1.188 +realloc_impl(void *ptr, size_t size)
1.189 +{
1.190 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.191 + init();
1.192 + if (MOZ_LIKELY(!replace_realloc))
1.193 + return je_realloc(ptr, size);
1.194 + return replace_realloc(ptr, size);
1.195 +}
1.196 +
1.197 +void
1.198 +free_impl(void *ptr)
1.199 +{
1.200 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.201 + init();
1.202 + if (MOZ_LIKELY(!replace_free))
1.203 + je_free(ptr);
1.204 + else
1.205 + replace_free(ptr);
1.206 +}
1.207 +
1.208 +void*
1.209 +memalign_impl(size_t alignment, size_t size)
1.210 +{
1.211 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.212 + init();
1.213 + if (MOZ_LIKELY(!replace_memalign))
1.214 + return je_memalign(alignment, size);
1.215 + return replace_memalign(alignment, size);
1.216 +}
1.217 +
1.218 +void*
1.219 +valloc_impl(size_t size)
1.220 +{
1.221 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.222 + init();
1.223 + if (MOZ_LIKELY(!replace_valloc))
1.224 + return je_valloc(size);
1.225 + return replace_valloc(size);
1.226 +}
1.227 +
1.228 +size_t
1.229 +malloc_usable_size_impl(usable_ptr_t ptr)
1.230 +{
1.231 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.232 + init();
1.233 + if (MOZ_LIKELY(!replace_malloc_usable_size))
1.234 + return je_malloc_usable_size(ptr);
1.235 + return replace_malloc_usable_size(ptr);
1.236 +}
1.237 +
1.238 +size_t
1.239 +malloc_good_size_impl(size_t size)
1.240 +{
1.241 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.242 + init();
1.243 + if (MOZ_LIKELY(!replace_malloc_good_size))
1.244 + return je_malloc_good_size(size);
1.245 + return replace_malloc_good_size(size);
1.246 +}
1.247 +
1.248 +void
1.249 +jemalloc_stats_impl(jemalloc_stats_t *stats)
1.250 +{
1.251 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.252 + init();
1.253 + if (MOZ_LIKELY(!replace_jemalloc_stats))
1.254 + je_jemalloc_stats(stats);
1.255 + else
1.256 + replace_jemalloc_stats(stats);
1.257 +}
1.258 +
1.259 +void
1.260 +jemalloc_purge_freed_pages_impl()
1.261 +{
1.262 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.263 + init();
1.264 + if (MOZ_LIKELY(!replace_jemalloc_purge_freed_pages))
1.265 + je_jemalloc_purge_freed_pages();
1.266 + else
1.267 + replace_jemalloc_purge_freed_pages();
1.268 +}
1.269 +
1.270 +void
1.271 +jemalloc_free_dirty_pages_impl()
1.272 +{
1.273 + if (MOZ_UNLIKELY(!replace_malloc_initialized))
1.274 + init();
1.275 + if (MOZ_LIKELY(!replace_jemalloc_free_dirty_pages))
1.276 + je_jemalloc_free_dirty_pages();
1.277 + else
1.278 + replace_jemalloc_free_dirty_pages();
1.279 +}
1.280 +
1.281 +/* The following comment and definitions are from jemalloc.c: */
1.282 +#if defined(__GLIBC__) && !defined(__UCLIBC__)
1.283 +
1.284 +/*
1.285 + * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
1.286 + * to inconsistently reference libc's malloc(3)-compatible functions
1.287 + * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
1.288 + *
1.289 + * These definitions interpose hooks in glibc. The functions are actually
1.290 + * passed an extra argument for the caller return address, which will be
1.291 + * ignored.
1.292 + */
1.293 +
1.294 +typedef void (* __free_hook_type)(void *ptr);
1.295 +typedef void *(* __malloc_hook_type)(size_t size);
1.296 +typedef void *(* __realloc_hook_type)(void *ptr, size_t size);
1.297 +typedef void *(* __memalign_hook_type)(size_t alignment, size_t size);
1.298 +
1.299 +MOZ_MEMORY_API __free_hook_type __free_hook = free_impl;
1.300 +MOZ_MEMORY_API __malloc_hook_type __malloc_hook = malloc_impl;
1.301 +MOZ_MEMORY_API __realloc_hook_type __realloc_hook = realloc_impl;
1.302 +MOZ_MEMORY_API __memalign_hook_type __memalign_hook = memalign_impl;
1.303 +
1.304 +#endif
1.305 +
1.306 +/*
1.307 + * The following is a OSX zone allocator implementation.
1.308 + * /!\ WARNING. It assumes the underlying malloc implementation's
1.309 + * malloc_usable_size returns 0 when the given pointer is not owned by
1.310 + * the allocator. Sadly, OSX does call zone_size with pointers not
1.311 + * owned by the allocator.
1.312 + */
1.313 +
1.314 +#ifdef XP_DARWIN
1.315 +#include <stdlib.h>
1.316 +#include <malloc/malloc.h>
1.317 +#include "mozilla/Assertions.h"
1.318 +
1.319 +static size_t
1.320 +zone_size(malloc_zone_t *zone, void *ptr)
1.321 +{
1.322 + return malloc_usable_size_impl(ptr);
1.323 +}
1.324 +
1.325 +static void *
1.326 +zone_malloc(malloc_zone_t *zone, size_t size)
1.327 +{
1.328 + return malloc_impl(size);
1.329 +}
1.330 +
1.331 +static void *
1.332 +zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
1.333 +{
1.334 + return calloc_impl(num, size);
1.335 +}
1.336 +
1.337 +static void *
1.338 +zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
1.339 +{
1.340 + if (malloc_usable_size_impl(ptr))
1.341 + return realloc_impl(ptr, size);
1.342 + return realloc(ptr, size);
1.343 +}
1.344 +
1.345 +static void
1.346 +zone_free(malloc_zone_t *zone, void *ptr)
1.347 +{
1.348 + if (malloc_usable_size_impl(ptr)) {
1.349 + free_impl(ptr);
1.350 + return;
1.351 + }
1.352 + free(ptr);
1.353 +}
1.354 +
1.355 +static void
1.356 +zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
1.357 +{
1.358 + size_t current_size = malloc_usable_size_impl(ptr);
1.359 + if (current_size) {
1.360 + MOZ_ASSERT(current_size == size);
1.361 + free_impl(ptr);
1.362 + return;
1.363 + }
1.364 + free(ptr);
1.365 +}
1.366 +
1.367 +static void *
1.368 +zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
1.369 +{
1.370 + void *ptr;
1.371 + if (posix_memalign_impl(&ptr, alignment, size) == 0)
1.372 + return ptr;
1.373 + return NULL;
1.374 +}
1.375 +
1.376 +static void *
1.377 +zone_valloc(malloc_zone_t *zone, size_t size)
1.378 +{
1.379 + return valloc_impl(size);
1.380 +}
1.381 +
1.382 +static void *
1.383 +zone_destroy(malloc_zone_t *zone)
1.384 +{
1.385 + /* This function should never be called. */
1.386 + MOZ_CRASH();
1.387 +}
1.388 +
1.389 +static size_t
1.390 +zone_good_size(malloc_zone_t *zone, size_t size)
1.391 +{
1.392 + return malloc_good_size_impl(size);
1.393 +}
1.394 +
1.395 +#ifdef MOZ_JEMALLOC
1.396 +
1.397 +#include "jemalloc/internal/jemalloc_internal.h"
1.398 +
1.399 +static void
1.400 +zone_force_lock(malloc_zone_t *zone)
1.401 +{
1.402 + /* /!\ This calls into jemalloc. It works because we're linked in the
1.403 + * same library. Stolen from jemalloc's zone.c. */
1.404 + if (isthreaded)
1.405 + jemalloc_prefork();
1.406 +}
1.407 +
1.408 +static void
1.409 +zone_force_unlock(malloc_zone_t *zone)
1.410 +{
1.411 + /* /!\ This calls into jemalloc. It works because we're linked in the
1.412 + * same library. Stolen from jemalloc's zone.c. */
1.413 + if (isthreaded)
1.414 + jemalloc_postfork_parent();
1.415 +}
1.416 +
1.417 +#else
1.418 +
1.419 +#define JEMALLOC_ZONE_VERSION 6
1.420 +
1.421 +/* Empty implementations are needed, because fork() calls zone->force_(un)lock
1.422 + * unconditionally. */
1.423 +static void
1.424 +zone_force_lock(malloc_zone_t *zone)
1.425 +{
1.426 +}
1.427 +
1.428 +static void
1.429 +zone_force_unlock(malloc_zone_t *zone)
1.430 +{
1.431 +}
1.432 +
1.433 +#endif
1.434 +
1.435 +static malloc_zone_t zone;
1.436 +static struct malloc_introspection_t zone_introspect;
1.437 +
1.438 +__attribute__((constructor)) void
1.439 +register_zone(void)
1.440 +{
1.441 + zone.size = (void *)zone_size;
1.442 + zone.malloc = (void *)zone_malloc;
1.443 + zone.calloc = (void *)zone_calloc;
1.444 + zone.valloc = (void *)zone_valloc;
1.445 + zone.free = (void *)zone_free;
1.446 + zone.realloc = (void *)zone_realloc;
1.447 + zone.destroy = (void *)zone_destroy;
1.448 + zone.zone_name = "replace_malloc_zone";
1.449 + zone.batch_malloc = NULL;
1.450 + zone.batch_free = NULL;
1.451 + zone.introspect = &zone_introspect;
1.452 + zone.version = JEMALLOC_ZONE_VERSION;
1.453 + zone.memalign = zone_memalign;
1.454 + zone.free_definite_size = zone_free_definite_size;
1.455 +#if (JEMALLOC_ZONE_VERSION >= 8)
1.456 + zone.pressure_relief = NULL;
1.457 +#endif
1.458 + zone_introspect.enumerator = NULL;
1.459 + zone_introspect.good_size = (void *)zone_good_size;
1.460 + zone_introspect.check = NULL;
1.461 + zone_introspect.print = NULL;
1.462 + zone_introspect.log = NULL;
1.463 + zone_introspect.force_lock = (void *)zone_force_lock;
1.464 + zone_introspect.force_unlock = (void *)zone_force_unlock;
1.465 + zone_introspect.statistics = NULL;
1.466 + zone_introspect.zone_locked = NULL;
1.467 +#if (JEMALLOC_ZONE_VERSION >= 7)
1.468 + zone_introspect.enable_discharge_checking = NULL;
1.469 + zone_introspect.disable_discharge_checking = NULL;
1.470 + zone_introspect.discharge = NULL;
1.471 +#ifdef __BLOCKS__
1.472 + zone_introspect.enumerate_discharged_pointers = NULL;
1.473 +#else
1.474 + zone_introspect.enumerate_unavailable_without_blocks = NULL;
1.475 +#endif
1.476 +#endif
1.477 +
1.478 + /*
1.479 + * The default purgeable zone is created lazily by OSX's libc. It uses
1.480 + * the default zone when it is created for "small" allocations
1.481 + * (< 15 KiB), but assumes the default zone is a scalable_zone. This
1.482 + * obviously fails when the default zone is the jemalloc zone, so
1.483 + * malloc_default_purgeable_zone is called beforehand so that the
1.484 + * default purgeable zone is created when the default zone is still
1.485 + * a scalable_zone. As purgeable zones only exist on >= 10.6, we need
1.486 + * to check for the existence of malloc_default_purgeable_zone() at
1.487 + * run time.
1.488 + */
1.489 + malloc_default_purgeable_zone();
1.490 +
1.491 + /* Register the custom zone. At this point it won't be the default. */
1.492 + malloc_zone_register(&zone);
1.493 +
1.494 + /*
1.495 + * Unregister and reregister the default zone. On OSX >= 10.6,
1.496 + * unregistering takes the last registered zone and places it at the
1.497 + * location of the specified zone. Unregistering the default zone thus
1.498 + * makes the last registered one the default. On OSX < 10.6,
1.499 + * unregistering shifts all registered zones. The first registered zone
1.500 + * then becomes the default.
1.501 + */
1.502 + do {
1.503 + malloc_zone_t *default_zone = malloc_default_zone();
1.504 + malloc_zone_unregister(default_zone);
1.505 + malloc_zone_register(default_zone);
1.506 + } while (malloc_default_zone() != &zone);
1.507 +}
1.508 +#endif
