1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/memory/mozjemalloc/jemalloc.c Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,7187 @@
1.4 +/* -*- Mode: C; tab-width: 8; c-basic-offset: 8; indent-tabs-mode: t -*- */
1.5 +/* vim:set softtabstop=8 shiftwidth=8 noet: */
1.6 +/*-
1.7 + * Copyright (C) 2006-2008 Jason Evans <jasone@FreeBSD.org>.
1.8 + * All rights reserved.
1.9 + *
1.10 + * Redistribution and use in source and binary forms, with or without
1.11 + * modification, are permitted provided that the following conditions
1.12 + * are met:
1.13 + * 1. Redistributions of source code must retain the above copyright
1.14 + * notice(s), this list of conditions and the following disclaimer as
1.15 + * the first lines of this file unmodified other than the possible
1.16 + * addition of one or more copyright notices.
1.17 + * 2. Redistributions in binary form must reproduce the above copyright
1.18 + * notice(s), this list of conditions and the following disclaimer in
1.19 + * the documentation and/or other materials provided with the
1.20 + * distribution.
1.21 + *
1.22 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
1.23 + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1.24 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
1.25 + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
1.26 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
1.27 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
1.28 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
1.29 + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
1.30 + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
1.31 + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
1.32 + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1.33 + *
1.34 + *******************************************************************************
1.35 + *
1.36 + * This allocator implementation is designed to provide scalable performance
1.37 + * for multi-threaded programs on multi-processor systems. The following
1.38 + * features are included for this purpose:
1.39 + *
1.40 + * + Multiple arenas are used if there are multiple CPUs, which reduces lock
1.41 + * contention and cache sloshing.
1.42 + *
1.43 + * + Cache line sharing between arenas is avoided for internal data
1.44 + * structures.
1.45 + *
1.46 + * + Memory is managed in chunks and runs (chunks can be split into runs),
1.47 + * rather than as individual pages. This provides a constant-time
1.48 + * mechanism for associating allocations with particular arenas.
1.49 + *
1.50 + * Allocation requests are rounded up to the nearest size class, and no record
1.51 + * of the original request size is maintained. Allocations are broken into
1.52 + * categories according to size class. Assuming runtime defaults, 4 kB pages
1.53 + * and a 16 byte quantum on a 32-bit system, the size classes in each category
1.54 + * are as follows:
1.55 + *
1.56 + * |=====================================|
1.57 + * | Category | Subcategory | Size |
1.58 + * |=====================================|
1.59 + * | Small | Tiny | 2 |
1.60 + * | | | 4 |
1.61 + * | | | 8 |
1.62 + * | |----------------+---------|
1.63 + * | | Quantum-spaced | 16 |
1.64 + * | | | 32 |
1.65 + * | | | 48 |
1.66 + * | | | ... |
1.67 + * | | | 480 |
1.68 + * | | | 496 |
1.69 + * | | | 512 |
1.70 + * | |----------------+---------|
1.71 + * | | Sub-page | 1 kB |
1.72 + * | | | 2 kB |
1.73 + * |=====================================|
1.74 + * | Large | 4 kB |
1.75 + * | | 8 kB |
1.76 + * | | 12 kB |
1.77 + * | | ... |
1.78 + * | | 1012 kB |
1.79 + * | | 1016 kB |
1.80 + * | | 1020 kB |
1.81 + * |=====================================|
1.82 + * | Huge | 1 MB |
1.83 + * | | 2 MB |
1.84 + * | | 3 MB |
1.85 + * | | ... |
1.86 + * |=====================================|
1.87 + *
1.88 + * NOTE: Due to Mozilla bug 691003, we cannot reserve less than one word for an
1.89 + * allocation on Linux or Mac. So on 32-bit *nix, the smallest bucket size is
1.90 + * 4 bytes, and on 64-bit, the smallest bucket size is 8 bytes.
1.91 + *
1.92 + * A different mechanism is used for each category:
1.93 + *
1.94 + * Small : Each size class is segregated into its own set of runs. Each run
1.95 + * maintains a bitmap of which regions are free/allocated.
1.96 + *
1.97 + * Large : Each allocation is backed by a dedicated run. Metadata are stored
1.98 + * in the associated arena chunk header maps.
1.99 + *
1.100 + * Huge : Each allocation is backed by a dedicated contiguous set of chunks.
1.101 + * Metadata are stored in a separate red-black tree.
1.102 + *
1.103 + *******************************************************************************
1.104 + */
1.105 +
1.106 +#ifdef MOZ_MEMORY_ANDROID
1.107 +#define NO_TLS
1.108 +#define _pthread_self() pthread_self()
1.109 +#endif
1.110 +
1.111 +/*
1.112 + * On Linux, we use madvise(MADV_DONTNEED) to release memory back to the
1.113 + * operating system. If we release 1MB of live pages with MADV_DONTNEED, our
1.114 + * RSS will decrease by 1MB (almost) immediately.
1.115 + *
1.116 + * On Mac, we use madvise(MADV_FREE). Unlike MADV_DONTNEED on Linux, MADV_FREE
1.117 + * on Mac doesn't cause the OS to release the specified pages immediately; the
1.118 + * OS keeps them in our process until the machine comes under memory pressure.
1.119 + *
1.120 + * It's therefore difficult to measure the process's RSS on Mac, since, in the
1.121 + * absence of memory pressure, the contribution from the heap to RSS will not
1.122 + * decrease due to our madvise calls.
1.123 + *
1.124 + * We therefore define MALLOC_DOUBLE_PURGE on Mac. This causes jemalloc to
1.125 + * track which pages have been MADV_FREE'd. You can then call
1.126 + * jemalloc_purge_freed_pages(), which will force the OS to release those
1.127 + * MADV_FREE'd pages, making the process's RSS reflect its true memory usage.
1.128 + *
1.129 + * The jemalloc_purge_freed_pages definition in memory/build/mozmemory.h needs
1.130 + * to be adjusted if MALLOC_DOUBLE_PURGE is ever enabled on Linux.
1.131 + */
1.132 +#ifdef MOZ_MEMORY_DARWIN
1.133 +#define MALLOC_DOUBLE_PURGE
1.134 +#endif
1.135 +
1.136 +/*
1.137 + * MALLOC_PRODUCTION disables assertions and statistics gathering. It also
1.138 + * defaults the A and J runtime options to off. These settings are appropriate
1.139 + * for production systems.
1.140 + */
1.141 +#ifndef MOZ_MEMORY_DEBUG
1.142 +# define MALLOC_PRODUCTION
1.143 +#endif
1.144 +
1.145 +/*
1.146 + * Use only one arena by default. Mozilla does not currently make extensive
1.147 + * use of concurrent allocation, so the increased fragmentation associated with
1.148 + * multiple arenas is not warranted.
1.149 + */
1.150 +#define MOZ_MEMORY_NARENAS_DEFAULT_ONE
1.151 +
1.152 +/*
1.153 + * Pass this set of options to jemalloc as its default. It does not override
1.154 + * the options passed via the MALLOC_OPTIONS environment variable but is
1.155 + * applied in addition to them.
1.156 + */
1.157 +#ifdef MOZ_B2G
1.158 + /* Reduce the amount of unused dirty pages to 1MiB on B2G */
1.159 +# define MOZ_MALLOC_OPTIONS "ff"
1.160 +#else
1.161 +# define MOZ_MALLOC_OPTIONS ""
1.162 +#endif
1.163 +
1.164 +/*
1.165 + * MALLOC_STATS enables statistics calculation, and is required for
1.166 + * jemalloc_stats().
1.167 + */
1.168 +#define MALLOC_STATS
1.169 +
1.170 +/* Memory filling (junk/poison/zero). */
1.171 +#define MALLOC_FILL
1.172 +
1.173 +#ifndef MALLOC_PRODUCTION
1.174 + /*
1.175 + * MALLOC_DEBUG enables assertions and other sanity checks, and disables
1.176 + * inline functions.
1.177 + */
1.178 +# define MALLOC_DEBUG
1.179 +
1.180 + /* Allocation tracing. */
1.181 +# ifndef MOZ_MEMORY_WINDOWS
1.182 +# define MALLOC_UTRACE
1.183 +# endif
1.184 +
1.185 + /* Support optional abort() on OOM. */
1.186 +# define MALLOC_XMALLOC
1.187 +
1.188 + /* Support SYSV semantics. */
1.189 +# define MALLOC_SYSV
1.190 +#endif
1.191 +
1.192 +/*
1.193 + * MALLOC_VALIDATE causes malloc_usable_size() to perform some pointer
1.194 + * validation. There are many possible errors that validation does not even
1.195 + * attempt to detect.
1.196 + */
1.197 +#define MALLOC_VALIDATE
1.198 +
1.199 +/* Embed no-op macros that support memory allocation tracking via valgrind. */
1.200 +#ifdef MOZ_VALGRIND
1.201 +# define MALLOC_VALGRIND
1.202 +#endif
1.203 +#ifdef MALLOC_VALGRIND
1.204 +# include <valgrind/valgrind.h>
1.205 +#else
1.206 +# define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
1.207 +# define VALGRIND_FREELIKE_BLOCK(addr, rzB)
1.208 +#endif
1.209 +
1.210 +/*
1.211 + * MALLOC_BALANCE enables monitoring of arena lock contention and dynamically
1.212 + * re-balances arena load if exponentially averaged contention exceeds a
1.213 + * certain threshold.
1.214 + */
1.215 +/* #define MALLOC_BALANCE */
1.216 +
1.217 +/*
1.218 + * MALLOC_PAGEFILE causes all mmap()ed memory to be backed by temporary
1.219 + * files, so that if a chunk is mapped, it is guaranteed to be swappable.
1.220 + * This avoids asynchronous OOM failures that are due to VM over-commit.
1.221 + */
1.222 +/* #define MALLOC_PAGEFILE */
1.223 +
1.224 +#ifdef MALLOC_PAGEFILE
1.225 +/* Write size when initializing a page file. */
1.226 +# define MALLOC_PAGEFILE_WRITE_SIZE 512
1.227 +#endif
1.228 +
1.229 +#if defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1.230 +#define _GNU_SOURCE /* For mremap(2). */
1.231 +#define issetugid() 0
1.232 +#if 0 /* Enable in order to test decommit code on Linux. */
1.233 +# define MALLOC_DECOMMIT
1.234 +#endif
1.235 +#endif
1.236 +
1.237 +#include <sys/types.h>
1.238 +
1.239 +#include <errno.h>
1.240 +#include <stdlib.h>
1.241 +#include <limits.h>
1.242 +#include <stdarg.h>
1.243 +#include <stdio.h>
1.244 +#include <string.h>
1.245 +
1.246 +#ifdef MOZ_MEMORY_WINDOWS
1.247 +
1.248 +/* Some defines from the CRT internal headers that we need here. */
1.249 +#define _CRT_SPINCOUNT 5000
1.250 +#define __crtInitCritSecAndSpinCount InitializeCriticalSectionAndSpinCount
1.251 +#include <io.h>
1.252 +#include <windows.h>
1.253 +
1.254 +#pragma warning( disable: 4267 4996 4146 )
1.255 +
1.256 +#define bool BOOL
1.257 +#define false FALSE
1.258 +#define true TRUE
1.259 +#define inline __inline
1.260 +#define SIZE_T_MAX SIZE_MAX
1.261 +#define STDERR_FILENO 2
1.262 +#define PATH_MAX MAX_PATH
1.263 +#define vsnprintf _vsnprintf
1.264 +
1.265 +#ifndef NO_TLS
1.266 +static unsigned long tlsIndex = 0xffffffff;
1.267 +#endif
1.268 +
1.269 +#define __thread
1.270 +#define _pthread_self() __threadid()
1.271 +#define issetugid() 0
1.272 +
1.273 +/* use MSVC intrinsics */
1.274 +#pragma intrinsic(_BitScanForward)
1.275 +static __forceinline int
1.276 +ffs(int x)
1.277 +{
1.278 + unsigned long i;
1.279 +
1.280 + if (_BitScanForward(&i, x) != 0)
1.281 + return (i + 1);
1.282 +
1.283 + return (0);
1.284 +}
1.285 +
1.286 +/* Implement getenv without using malloc */
1.287 +static char mozillaMallocOptionsBuf[64];
1.288 +
1.289 +#define getenv xgetenv
1.290 +static char *
1.291 +getenv(const char *name)
1.292 +{
1.293 +
1.294 + if (GetEnvironmentVariableA(name, (LPSTR)&mozillaMallocOptionsBuf,
1.295 + sizeof(mozillaMallocOptionsBuf)) > 0)
1.296 + return (mozillaMallocOptionsBuf);
1.297 +
1.298 + return (NULL);
1.299 +}
1.300 +
1.301 +typedef unsigned char uint8_t;
1.302 +typedef unsigned uint32_t;
1.303 +typedef unsigned long long uint64_t;
1.304 +typedef unsigned long long uintmax_t;
1.305 +#if defined(_WIN64)
1.306 +typedef long long ssize_t;
1.307 +#else
1.308 +typedef long ssize_t;
1.309 +#endif
1.310 +
1.311 +#define MALLOC_DECOMMIT
1.312 +#endif
1.313 +
1.314 +#ifndef MOZ_MEMORY_WINDOWS
1.315 +#ifndef MOZ_MEMORY_SOLARIS
1.316 +#include <sys/cdefs.h>
1.317 +#endif
1.318 +#ifndef __DECONST
1.319 +# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
1.320 +#endif
1.321 +#ifndef MOZ_MEMORY
1.322 +__FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 180599 2008-07-18 19:35:44Z jasone $");
1.323 +#include "libc_private.h"
1.324 +#ifdef MALLOC_DEBUG
1.325 +# define _LOCK_DEBUG
1.326 +#endif
1.327 +#include "spinlock.h"
1.328 +#include "namespace.h"
1.329 +#endif
1.330 +#include <sys/mman.h>
1.331 +#ifndef MADV_FREE
1.332 +# define MADV_FREE MADV_DONTNEED
1.333 +#endif
1.334 +#ifndef MAP_NOSYNC
1.335 +# define MAP_NOSYNC 0
1.336 +#endif
1.337 +#include <sys/param.h>
1.338 +#ifndef MOZ_MEMORY
1.339 +#include <sys/stddef.h>
1.340 +#endif
1.341 +#include <sys/time.h>
1.342 +#include <sys/types.h>
1.343 +#if !defined(MOZ_MEMORY_SOLARIS) && !defined(MOZ_MEMORY_ANDROID)
1.344 +#include <sys/sysctl.h>
1.345 +#endif
1.346 +#include <sys/uio.h>
1.347 +#ifndef MOZ_MEMORY
1.348 +#include <sys/ktrace.h> /* Must come after several other sys/ includes. */
1.349 +
1.350 +#include <machine/atomic.h>
1.351 +#include <machine/cpufunc.h>
1.352 +#include <machine/vmparam.h>
1.353 +#endif
1.354 +
1.355 +#include <errno.h>
1.356 +#include <limits.h>
1.357 +#ifndef SIZE_T_MAX
1.358 +# define SIZE_T_MAX SIZE_MAX
1.359 +#endif
1.360 +#include <pthread.h>
1.361 +#ifdef MOZ_MEMORY_DARWIN
1.362 +#define _pthread_self pthread_self
1.363 +#define _pthread_mutex_init pthread_mutex_init
1.364 +#define _pthread_mutex_trylock pthread_mutex_trylock
1.365 +#define _pthread_mutex_lock pthread_mutex_lock
1.366 +#define _pthread_mutex_unlock pthread_mutex_unlock
1.367 +#endif
1.368 +#include <sched.h>
1.369 +#include <stdarg.h>
1.370 +#include <stdio.h>
1.371 +#include <stdbool.h>
1.372 +#include <stdint.h>
1.373 +#include <stdlib.h>
1.374 +#include <string.h>
1.375 +#ifndef MOZ_MEMORY_DARWIN
1.376 +#include <strings.h>
1.377 +#endif
1.378 +#include <unistd.h>
1.379 +
1.380 +#ifdef MOZ_MEMORY_DARWIN
1.381 +#include <libkern/OSAtomic.h>
1.382 +#include <mach/mach_error.h>
1.383 +#include <mach/mach_init.h>
1.384 +#include <mach/vm_map.h>
1.385 +#include <malloc/malloc.h>
1.386 +#endif
1.387 +
1.388 +#ifndef MOZ_MEMORY
1.389 +#include "un-namespace.h"
1.390 +#endif
1.391 +
1.392 +#endif
1.393 +
1.394 +#include "jemalloc_types.h"
1.395 +#include "linkedlist.h"
1.396 +#include "mozmemory_wrap.h"
1.397 +
1.398 +/* Some tools, such as /dev/dsp wrappers, LD_PRELOAD libraries that
1.399 + * happen to override mmap() and call dlsym() from their overridden
1.400 + * mmap(). The problem is that dlsym() calls malloc(), and this ends
1.401 + * up in a dead lock in jemalloc.
1.402 + * On these systems, we prefer to directly use the system call.
1.403 + * We do that for Linux systems and kfreebsd with GNU userland.
1.404 + * Note sanity checks are not done (alignment of offset, ...) because
1.405 + * the uses of mmap are pretty limited, in jemalloc.
1.406 + *
1.407 + * On Alpha, glibc has a bug that prevents syscall() to work for system
1.408 + * calls with 6 arguments
1.409 + */
1.410 +#if (defined(MOZ_MEMORY_LINUX) && !defined(__alpha__)) || \
1.411 + (defined(MOZ_MEMORY_BSD) && defined(__GLIBC__))
1.412 +#include <sys/syscall.h>
1.413 +#if defined(SYS_mmap) || defined(SYS_mmap2)
1.414 +static inline
1.415 +void *_mmap(void *addr, size_t length, int prot, int flags,
1.416 + int fd, off_t offset)
1.417 +{
1.418 +/* S390 only passes one argument to the mmap system call, which is a
1.419 + * pointer to a structure containing the arguments */
1.420 +#ifdef __s390__
1.421 + struct {
1.422 + void *addr;
1.423 + size_t length;
1.424 + long prot;
1.425 + long flags;
1.426 + long fd;
1.427 + off_t offset;
1.428 + } args = { addr, length, prot, flags, fd, offset };
1.429 + return (void *) syscall(SYS_mmap, &args);
1.430 +#else
1.431 +#ifdef SYS_mmap2
1.432 + return (void *) syscall(SYS_mmap2, addr, length, prot, flags,
1.433 + fd, offset >> 12);
1.434 +#else
1.435 + return (void *) syscall(SYS_mmap, addr, length, prot, flags,
1.436 + fd, offset);
1.437 +#endif
1.438 +#endif
1.439 +}
1.440 +#define mmap _mmap
1.441 +#define munmap(a, l) syscall(SYS_munmap, a, l)
1.442 +#endif
1.443 +#endif
1.444 +
1.445 +#ifdef MOZ_MEMORY_DARWIN
1.446 +static const bool isthreaded = true;
1.447 +#endif
1.448 +
1.449 +#if defined(MOZ_MEMORY_SOLARIS) && defined(MAP_ALIGN) && !defined(JEMALLOC_NEVER_USES_MAP_ALIGN)
1.450 +#define JEMALLOC_USES_MAP_ALIGN /* Required on Solaris 10. Might improve performance elsewhere. */
1.451 +#endif
1.452 +
1.453 +#define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
1.454 +
1.455 +#ifdef MOZ_MEMORY_WINDOWS
1.456 + /* MSVC++ does not support C99 variable-length arrays. */
1.457 +# define RB_NO_C99_VARARRAYS
1.458 +#endif
1.459 +#include "rb.h"
1.460 +
1.461 +#ifdef MALLOC_DEBUG
1.462 + /* Disable inlining to make debugging easier. */
1.463 +#ifdef inline
1.464 +#undef inline
1.465 +#endif
1.466 +
1.467 +# define inline
1.468 +#endif
1.469 +
1.470 +/* Size of stack-allocated buffer passed to strerror_r(). */
1.471 +#define STRERROR_BUF 64
1.472 +
1.473 +/* Minimum alignment of non-tiny allocations is 2^QUANTUM_2POW_MIN bytes. */
1.474 +# define QUANTUM_2POW_MIN 4
1.475 +#if defined(_WIN64) || defined(__LP64__)
1.476 +# define SIZEOF_PTR_2POW 3
1.477 +#else
1.478 +# define SIZEOF_PTR_2POW 2
1.479 +#endif
1.480 +#define PIC
1.481 +#ifndef MOZ_MEMORY_DARWIN
1.482 +static const bool isthreaded = true;
1.483 +#else
1.484 +# define NO_TLS
1.485 +#endif
1.486 +#if 0
1.487 +#ifdef __i386__
1.488 +# define QUANTUM_2POW_MIN 4
1.489 +# define SIZEOF_PTR_2POW 2
1.490 +# define CPU_SPINWAIT __asm__ volatile("pause")
1.491 +#endif
1.492 +#ifdef __ia64__
1.493 +# define QUANTUM_2POW_MIN 4
1.494 +# define SIZEOF_PTR_2POW 3
1.495 +#endif
1.496 +#ifdef __alpha__
1.497 +# define QUANTUM_2POW_MIN 4
1.498 +# define SIZEOF_PTR_2POW 3
1.499 +# define NO_TLS
1.500 +#endif
1.501 +#ifdef __sparc64__
1.502 +# define QUANTUM_2POW_MIN 4
1.503 +# define SIZEOF_PTR_2POW 3
1.504 +# define NO_TLS
1.505 +#endif
1.506 +#ifdef __amd64__
1.507 +# define QUANTUM_2POW_MIN 4
1.508 +# define SIZEOF_PTR_2POW 3
1.509 +# define CPU_SPINWAIT __asm__ volatile("pause")
1.510 +#endif
1.511 +#ifdef __arm__
1.512 +# define QUANTUM_2POW_MIN 3
1.513 +# define SIZEOF_PTR_2POW 2
1.514 +# define NO_TLS
1.515 +#endif
1.516 +#ifdef __mips__
1.517 +# define QUANTUM_2POW_MIN 3
1.518 +# define SIZEOF_PTR_2POW 2
1.519 +# define NO_TLS
1.520 +#endif
1.521 +#ifdef __powerpc__
1.522 +# define QUANTUM_2POW_MIN 4
1.523 +# define SIZEOF_PTR_2POW 2
1.524 +#endif
1.525 +#endif
1.526 +
1.527 +#define SIZEOF_PTR (1U << SIZEOF_PTR_2POW)
1.528 +
1.529 +/* sizeof(int) == (1U << SIZEOF_INT_2POW). */
1.530 +#ifndef SIZEOF_INT_2POW
1.531 +# define SIZEOF_INT_2POW 2
1.532 +#endif
1.533 +
1.534 +/* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
1.535 +#if (!defined(PIC) && !defined(NO_TLS))
1.536 +# define NO_TLS
1.537 +#endif
1.538 +
1.539 +#ifdef NO_TLS
1.540 + /* MALLOC_BALANCE requires TLS. */
1.541 +# ifdef MALLOC_BALANCE
1.542 +# undef MALLOC_BALANCE
1.543 +# endif
1.544 +#endif
1.545 +
1.546 +/*
1.547 + * Size and alignment of memory chunks that are allocated by the OS's virtual
1.548 + * memory system.
1.549 + */
1.550 +#define CHUNK_2POW_DEFAULT 20
1.551 +/* Maximum number of dirty pages per arena. */
1.552 +#define DIRTY_MAX_DEFAULT (1U << 10)
1.553 +
1.554 +/*
1.555 + * Maximum size of L1 cache line. This is used to avoid cache line aliasing,
1.556 + * so over-estimates are okay (up to a point), but under-estimates will
1.557 + * negatively affect performance.
1.558 + */
1.559 +#define CACHELINE_2POW 6
1.560 +#define CACHELINE ((size_t)(1U << CACHELINE_2POW))
1.561 +
1.562 +/*
1.563 + * Smallest size class to support. On Linux and Mac, even malloc(1) must
1.564 + * reserve a word's worth of memory (see Mozilla bug 691003).
1.565 + */
1.566 +#ifdef MOZ_MEMORY_WINDOWS
1.567 +#define TINY_MIN_2POW 1
1.568 +#else
1.569 +#define TINY_MIN_2POW (sizeof(void*) == 8 ? 3 : 2)
1.570 +#endif
1.571 +
1.572 +/*
1.573 + * Maximum size class that is a multiple of the quantum, but not (necessarily)
1.574 + * a power of 2. Above this size, allocations are rounded up to the nearest
1.575 + * power of 2.
1.576 + */
1.577 +#define SMALL_MAX_2POW_DEFAULT 9
1.578 +#define SMALL_MAX_DEFAULT (1U << SMALL_MAX_2POW_DEFAULT)
1.579 +
1.580 +/*
1.581 + * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized
1.582 + * as small as possible such that this setting is still honored, without
1.583 + * violating other constraints. The goal is to make runs as small as possible
1.584 + * without exceeding a per run external fragmentation threshold.
1.585 + *
1.586 + * We use binary fixed point math for overhead computations, where the binary
1.587 + * point is implicitly RUN_BFP bits to the left.
1.588 + *
1.589 + * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
1.590 + * honored for some/all object sizes, since there is one bit of header overhead
1.591 + * per object (plus a constant). This constraint is relaxed (ignored) for runs
1.592 + * that are so small that the per-region overhead is greater than:
1.593 + *
1.594 + * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
1.595 + */
1.596 +#define RUN_BFP 12
1.597 +/* \/ Implicit binary fixed point. */
1.598 +#define RUN_MAX_OVRHD 0x0000003dU
1.599 +#define RUN_MAX_OVRHD_RELAX 0x00001800U
1.600 +
1.601 +/* Put a cap on small object run size. This overrides RUN_MAX_OVRHD. */
1.602 +#define RUN_MAX_SMALL_2POW 15
1.603 +#define RUN_MAX_SMALL (1U << RUN_MAX_SMALL_2POW)
1.604 +
1.605 +/*
1.606 + * Hyper-threaded CPUs may need a special instruction inside spin loops in
1.607 + * order to yield to another virtual CPU. If no such instruction is defined
1.608 + * above, make CPU_SPINWAIT a no-op.
1.609 + */
1.610 +#ifndef CPU_SPINWAIT
1.611 +# define CPU_SPINWAIT
1.612 +#endif
1.613 +
1.614 +/*
1.615 + * Adaptive spinning must eventually switch to blocking, in order to avoid the
1.616 + * potential for priority inversion deadlock. Backing off past a certain point
1.617 + * can actually waste time.
1.618 + */
1.619 +#define SPIN_LIMIT_2POW 11
1.620 +
1.621 +/*
1.622 + * Conversion from spinning to blocking is expensive; we use (1U <<
1.623 + * BLOCK_COST_2POW) to estimate how many more times costly blocking is than
1.624 + * worst-case spinning.
1.625 + */
1.626 +#define BLOCK_COST_2POW 4
1.627 +
1.628 +#ifdef MALLOC_BALANCE
1.629 + /*
1.630 + * We use an exponential moving average to track recent lock contention,
1.631 + * where the size of the history window is N, and alpha=2/(N+1).
1.632 + *
1.633 + * Due to integer math rounding, very small values here can cause
1.634 + * substantial degradation in accuracy, thus making the moving average decay
1.635 + * faster than it would with precise calculation.
1.636 + */
1.637 +# define BALANCE_ALPHA_INV_2POW 9
1.638 +
1.639 + /*
1.640 + * Threshold value for the exponential moving contention average at which to
1.641 + * re-assign a thread.
1.642 + */
1.643 +# define BALANCE_THRESHOLD_DEFAULT (1U << (SPIN_LIMIT_2POW-4))
1.644 +#endif
1.645 +
1.646 +/******************************************************************************/
1.647 +
1.648 +/* MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. */
1.649 +#if defined(MALLOC_DECOMMIT) && defined(MALLOC_DOUBLE_PURGE)
1.650 +#error MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive.
1.651 +#endif
1.652 +
1.653 +/*
1.654 + * Mutexes based on spinlocks. We can't use normal pthread spinlocks in all
1.655 + * places, because they require malloc()ed memory, which causes bootstrapping
1.656 + * issues in some cases.
1.657 + */
1.658 +#if defined(MOZ_MEMORY_WINDOWS)
1.659 +#define malloc_mutex_t CRITICAL_SECTION
1.660 +#define malloc_spinlock_t CRITICAL_SECTION
1.661 +#elif defined(MOZ_MEMORY_DARWIN)
1.662 +typedef struct {
1.663 + OSSpinLock lock;
1.664 +} malloc_mutex_t;
1.665 +typedef struct {
1.666 + OSSpinLock lock;
1.667 +} malloc_spinlock_t;
1.668 +#elif defined(MOZ_MEMORY)
1.669 +typedef pthread_mutex_t malloc_mutex_t;
1.670 +typedef pthread_mutex_t malloc_spinlock_t;
1.671 +#else
1.672 +/* XXX these should #ifdef these for freebsd (and linux?) only */
1.673 +typedef struct {
1.674 + spinlock_t lock;
1.675 +} malloc_mutex_t;
1.676 +typedef malloc_spinlock_t malloc_mutex_t;
1.677 +#endif
1.678 +
1.679 +/* Set to true once the allocator has been initialized. */
1.680 +static bool malloc_initialized = false;
1.681 +
1.682 +#if defined(MOZ_MEMORY_WINDOWS)
1.683 +/* No init lock for Windows. */
1.684 +#elif defined(MOZ_MEMORY_DARWIN)
1.685 +static malloc_mutex_t init_lock = {OS_SPINLOCK_INIT};
1.686 +#elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1.687 +static malloc_mutex_t init_lock = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP;
1.688 +#elif defined(MOZ_MEMORY)
1.689 +static malloc_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
1.690 +#else
1.691 +static malloc_mutex_t init_lock = {_SPINLOCK_INITIALIZER};
1.692 +#endif
1.693 +
1.694 +/******************************************************************************/
1.695 +/*
1.696 + * Statistics data structures.
1.697 + */
1.698 +
1.699 +#ifdef MALLOC_STATS
1.700 +
1.701 +typedef struct malloc_bin_stats_s malloc_bin_stats_t;
1.702 +struct malloc_bin_stats_s {
1.703 + /*
1.704 + * Number of allocation requests that corresponded to the size of this
1.705 + * bin.
1.706 + */
1.707 + uint64_t nrequests;
1.708 +
1.709 + /* Total number of runs created for this bin's size class. */
1.710 + uint64_t nruns;
1.711 +
1.712 + /*
1.713 + * Total number of runs reused by extracting them from the runs tree for
1.714 + * this bin's size class.
1.715 + */
1.716 + uint64_t reruns;
1.717 +
1.718 + /* High-water mark for this bin. */
1.719 + unsigned long highruns;
1.720 +
1.721 + /* Current number of runs in this bin. */
1.722 + unsigned long curruns;
1.723 +};
1.724 +
1.725 +typedef struct arena_stats_s arena_stats_t;
1.726 +struct arena_stats_s {
1.727 + /* Number of bytes currently mapped. */
1.728 + size_t mapped;
1.729 +
1.730 + /*
1.731 + * Total number of purge sweeps, total number of madvise calls made,
1.732 + * and total pages purged in order to keep dirty unused memory under
1.733 + * control.
1.734 + */
1.735 + uint64_t npurge;
1.736 + uint64_t nmadvise;
1.737 + uint64_t purged;
1.738 +#ifdef MALLOC_DECOMMIT
1.739 + /*
1.740 + * Total number of decommit/commit operations, and total number of
1.741 + * pages decommitted.
1.742 + */
1.743 + uint64_t ndecommit;
1.744 + uint64_t ncommit;
1.745 + uint64_t decommitted;
1.746 +#endif
1.747 +
1.748 + /* Current number of committed pages. */
1.749 + size_t committed;
1.750 +
1.751 + /* Per-size-category statistics. */
1.752 + size_t allocated_small;
1.753 + uint64_t nmalloc_small;
1.754 + uint64_t ndalloc_small;
1.755 +
1.756 + size_t allocated_large;
1.757 + uint64_t nmalloc_large;
1.758 + uint64_t ndalloc_large;
1.759 +
1.760 +#ifdef MALLOC_BALANCE
1.761 + /* Number of times this arena reassigned a thread due to contention. */
1.762 + uint64_t nbalance;
1.763 +#endif
1.764 +};
1.765 +
1.766 +#endif /* #ifdef MALLOC_STATS */
1.767 +
1.768 +/******************************************************************************/
1.769 +/*
1.770 + * Extent data structures.
1.771 + */
1.772 +
1.773 +/* Tree of extents. */
1.774 +typedef struct extent_node_s extent_node_t;
1.775 +struct extent_node_s {
1.776 + /* Linkage for the size/address-ordered tree. */
1.777 + rb_node(extent_node_t) link_szad;
1.778 +
1.779 + /* Linkage for the address-ordered tree. */
1.780 + rb_node(extent_node_t) link_ad;
1.781 +
1.782 + /* Pointer to the extent that this tree node is responsible for. */
1.783 + void *addr;
1.784 +
1.785 + /* Total region size. */
1.786 + size_t size;
1.787 +};
1.788 +typedef rb_tree(extent_node_t) extent_tree_t;
1.789 +
1.790 +/******************************************************************************/
1.791 +/*
1.792 + * Radix tree data structures.
1.793 + */
1.794 +
1.795 +#ifdef MALLOC_VALIDATE
1.796 + /*
1.797 + * Size of each radix tree node (must be a power of 2). This impacts tree
1.798 + * depth.
1.799 + */
1.800 +# if (SIZEOF_PTR == 4)
1.801 +# define MALLOC_RTREE_NODESIZE (1U << 14)
1.802 +# else
1.803 +# define MALLOC_RTREE_NODESIZE CACHELINE
1.804 +# endif
1.805 +
1.806 +typedef struct malloc_rtree_s malloc_rtree_t;
1.807 +struct malloc_rtree_s {
1.808 + malloc_spinlock_t lock;
1.809 + void **root;
1.810 + unsigned height;
1.811 + unsigned level2bits[1]; /* Dynamically sized. */
1.812 +};
1.813 +#endif
1.814 +
1.815 +/******************************************************************************/
1.816 +/*
1.817 + * Arena data structures.
1.818 + */
1.819 +
1.820 +typedef struct arena_s arena_t;
1.821 +typedef struct arena_bin_s arena_bin_t;
1.822 +
1.823 +/* Each element of the chunk map corresponds to one page within the chunk. */
1.824 +typedef struct arena_chunk_map_s arena_chunk_map_t;
1.825 +struct arena_chunk_map_s {
1.826 + /*
1.827 + * Linkage for run trees. There are two disjoint uses:
1.828 + *
1.829 + * 1) arena_t's runs_avail tree.
1.830 + * 2) arena_run_t conceptually uses this linkage for in-use non-full
1.831 + * runs, rather than directly embedding linkage.
1.832 + */
1.833 + rb_node(arena_chunk_map_t) link;
1.834 +
1.835 + /*
1.836 + * Run address (or size) and various flags are stored together. The bit
1.837 + * layout looks like (assuming 32-bit system):
1.838 + *
1.839 + * ???????? ???????? ????---- -mckdzla
1.840 + *
1.841 + * ? : Unallocated: Run address for first/last pages, unset for internal
1.842 + * pages.
1.843 + * Small: Run address.
1.844 + * Large: Run size for first page, unset for trailing pages.
1.845 + * - : Unused.
1.846 + * m : MADV_FREE/MADV_DONTNEED'ed?
1.847 + * c : decommitted?
1.848 + * k : key?
1.849 + * d : dirty?
1.850 + * z : zeroed?
1.851 + * l : large?
1.852 + * a : allocated?
1.853 + *
1.854 + * Following are example bit patterns for the three types of runs.
1.855 + *
1.856 + * r : run address
1.857 + * s : run size
1.858 + * x : don't care
1.859 + * - : 0
1.860 + * [cdzla] : bit set
1.861 + *
1.862 + * Unallocated:
1.863 + * ssssssss ssssssss ssss---- --c-----
1.864 + * xxxxxxxx xxxxxxxx xxxx---- ----d---
1.865 + * ssssssss ssssssss ssss---- -----z--
1.866 + *
1.867 + * Small:
1.868 + * rrrrrrrr rrrrrrrr rrrr---- -------a
1.869 + * rrrrrrrr rrrrrrrr rrrr---- -------a
1.870 + * rrrrrrrr rrrrrrrr rrrr---- -------a
1.871 + *
1.872 + * Large:
1.873 + * ssssssss ssssssss ssss---- ------la
1.874 + * -------- -------- -------- ------la
1.875 + * -------- -------- -------- ------la
1.876 + */
1.877 + size_t bits;
1.878 +
1.879 +/* Note that CHUNK_MAP_DECOMMITTED's meaning varies depending on whether
1.880 + * MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are defined.
1.881 + *
1.882 + * If MALLOC_DECOMMIT is defined, a page which is CHUNK_MAP_DECOMMITTED must be
1.883 + * re-committed with pages_commit() before it may be touched. If
1.884 + * MALLOC_DECOMMIT is defined, MALLOC_DOUBLE_PURGE may not be defined.
1.885 + *
1.886 + * If neither MALLOC_DECOMMIT nor MALLOC_DOUBLE_PURGE is defined, pages which
1.887 + * are madvised (with either MADV_DONTNEED or MADV_FREE) are marked with
1.888 + * CHUNK_MAP_MADVISED.
1.889 + *
1.890 + * Otherwise, if MALLOC_DECOMMIT is not defined and MALLOC_DOUBLE_PURGE is
1.891 + * defined, then a page which is madvised is marked as CHUNK_MAP_MADVISED.
1.892 + * When it's finally freed with jemalloc_purge_freed_pages, the page is marked
1.893 + * as CHUNK_MAP_DECOMMITTED.
1.894 + */
1.895 +#if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
1.896 +#define CHUNK_MAP_MADVISED ((size_t)0x40U)
1.897 +#define CHUNK_MAP_DECOMMITTED ((size_t)0x20U)
1.898 +#define CHUNK_MAP_MADVISED_OR_DECOMMITTED (CHUNK_MAP_MADVISED | CHUNK_MAP_DECOMMITTED)
1.899 +#endif
1.900 +#define CHUNK_MAP_KEY ((size_t)0x10U)
1.901 +#define CHUNK_MAP_DIRTY ((size_t)0x08U)
1.902 +#define CHUNK_MAP_ZEROED ((size_t)0x04U)
1.903 +#define CHUNK_MAP_LARGE ((size_t)0x02U)
1.904 +#define CHUNK_MAP_ALLOCATED ((size_t)0x01U)
1.905 +};
1.906 +typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
1.907 +typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
1.908 +
1.909 +/* Arena chunk header. */
1.910 +typedef struct arena_chunk_s arena_chunk_t;
1.911 +struct arena_chunk_s {
1.912 + /* Arena that owns the chunk. */
1.913 + arena_t *arena;
1.914 +
1.915 + /* Linkage for the arena's chunks_dirty tree. */
1.916 + rb_node(arena_chunk_t) link_dirty;
1.917 +
1.918 +#ifdef MALLOC_DOUBLE_PURGE
1.919 + /* If we're double-purging, we maintain a linked list of chunks which
1.920 + * have pages which have been madvise(MADV_FREE)'d but not explicitly
1.921 + * purged.
1.922 + *
1.923 + * We're currently lazy and don't remove a chunk from this list when
1.924 + * all its madvised pages are recommitted. */
1.925 + LinkedList chunks_madvised_elem;
1.926 +#endif
1.927 +
1.928 + /* Number of dirty pages. */
1.929 + size_t ndirty;
1.930 +
1.931 + /* Map of pages within chunk that keeps track of free/large/small. */
1.932 + arena_chunk_map_t map[1]; /* Dynamically sized. */
1.933 +};
1.934 +typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
1.935 +
1.936 +typedef struct arena_run_s arena_run_t;
1.937 +struct arena_run_s {
1.938 +#if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
1.939 + uint32_t magic;
1.940 +# define ARENA_RUN_MAGIC 0x384adf93
1.941 +#endif
1.942 +
1.943 + /* Bin this run is associated with. */
1.944 + arena_bin_t *bin;
1.945 +
1.946 + /* Index of first element that might have a free region. */
1.947 + unsigned regs_minelm;
1.948 +
1.949 + /* Number of free regions in run. */
1.950 + unsigned nfree;
1.951 +
1.952 + /* Bitmask of in-use regions (0: in use, 1: free). */
1.953 + unsigned regs_mask[1]; /* Dynamically sized. */
1.954 +};
1.955 +
1.956 +struct arena_bin_s {
1.957 + /*
1.958 + * Current run being used to service allocations of this bin's size
1.959 + * class.
1.960 + */
1.961 + arena_run_t *runcur;
1.962 +
1.963 + /*
1.964 + * Tree of non-full runs. This tree is used when looking for an
1.965 + * existing run when runcur is no longer usable. We choose the
1.966 + * non-full run that is lowest in memory; this policy tends to keep
1.967 + * objects packed well, and it can also help reduce the number of
1.968 + * almost-empty chunks.
1.969 + */
1.970 + arena_run_tree_t runs;
1.971 +
1.972 + /* Size of regions in a run for this bin's size class. */
1.973 + size_t reg_size;
1.974 +
1.975 + /* Total size of a run for this bin's size class. */
1.976 + size_t run_size;
1.977 +
1.978 + /* Total number of regions in a run for this bin's size class. */
1.979 + uint32_t nregs;
1.980 +
1.981 + /* Number of elements in a run's regs_mask for this bin's size class. */
1.982 + uint32_t regs_mask_nelms;
1.983 +
1.984 + /* Offset of first region in a run for this bin's size class. */
1.985 + uint32_t reg0_offset;
1.986 +
1.987 +#ifdef MALLOC_STATS
1.988 + /* Bin statistics. */
1.989 + malloc_bin_stats_t stats;
1.990 +#endif
1.991 +};
1.992 +
1.993 +struct arena_s {
1.994 +#if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
1.995 + uint32_t magic;
1.996 +# define ARENA_MAGIC 0x947d3d24
1.997 +#endif
1.998 +
1.999 + /* All operations on this arena require that lock be locked. */
1.1000 +#ifdef MOZ_MEMORY
1.1001 + malloc_spinlock_t lock;
1.1002 +#else
1.1003 + pthread_mutex_t lock;
1.1004 +#endif
1.1005 +
1.1006 +#ifdef MALLOC_STATS
1.1007 + arena_stats_t stats;
1.1008 +#endif
1.1009 +
1.1010 + /* Tree of dirty-page-containing chunks this arena manages. */
1.1011 + arena_chunk_tree_t chunks_dirty;
1.1012 +
1.1013 +#ifdef MALLOC_DOUBLE_PURGE
1.1014 + /* Head of a linked list of MADV_FREE'd-page-containing chunks this
1.1015 + * arena manages. */
1.1016 + LinkedList chunks_madvised;
1.1017 +#endif
1.1018 +
1.1019 + /*
1.1020 + * In order to avoid rapid chunk allocation/deallocation when an arena
1.1021 + * oscillates right on the cusp of needing a new chunk, cache the most
1.1022 + * recently freed chunk. The spare is left in the arena's chunk trees
1.1023 + * until it is deleted.
1.1024 + *
1.1025 + * There is one spare chunk per arena, rather than one spare total, in
1.1026 + * order to avoid interactions between multiple threads that could make
1.1027 + * a single spare inadequate.
1.1028 + */
1.1029 + arena_chunk_t *spare;
1.1030 +
1.1031 + /*
1.1032 + * Current count of pages within unused runs that are potentially
1.1033 + * dirty, and for which madvise(... MADV_FREE) has not been called. By
1.1034 + * tracking this, we can institute a limit on how much dirty unused
1.1035 + * memory is mapped for each arena.
1.1036 + */
1.1037 + size_t ndirty;
1.1038 +
1.1039 + /*
1.1040 + * Size/address-ordered tree of this arena's available runs. This tree
1.1041 + * is used for first-best-fit run allocation.
1.1042 + */
1.1043 + arena_avail_tree_t runs_avail;
1.1044 +
1.1045 +#ifdef MALLOC_BALANCE
1.1046 + /*
1.1047 + * The arena load balancing machinery needs to keep track of how much
1.1048 + * lock contention there is. This value is exponentially averaged.
1.1049 + */
1.1050 + uint32_t contention;
1.1051 +#endif
1.1052 +
1.1053 + /*
1.1054 + * bins is used to store rings of free regions of the following sizes,
1.1055 + * assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS.
1.1056 + *
1.1057 + * bins[i] | size |
1.1058 + * --------+------+
1.1059 + * 0 | 2 |
1.1060 + * 1 | 4 |
1.1061 + * 2 | 8 |
1.1062 + * --------+------+
1.1063 + * 3 | 16 |
1.1064 + * 4 | 32 |
1.1065 + * 5 | 48 |
1.1066 + * 6 | 64 |
1.1067 + * : :
1.1068 + * : :
1.1069 + * 33 | 496 |
1.1070 + * 34 | 512 |
1.1071 + * --------+------+
1.1072 + * 35 | 1024 |
1.1073 + * 36 | 2048 |
1.1074 + * --------+------+
1.1075 + */
1.1076 + arena_bin_t bins[1]; /* Dynamically sized. */
1.1077 +};
1.1078 +
1.1079 +/******************************************************************************/
1.1080 +/*
1.1081 + * Data.
1.1082 + */
1.1083 +
1.1084 +#ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
1.1085 +/* Number of CPUs. */
1.1086 +static unsigned ncpus;
1.1087 +#endif
1.1088 +
1.1089 +/*
1.1090 + * When MALLOC_STATIC_SIZES is defined most of the parameters
1.1091 + * controlling the malloc behavior are defined as compile-time constants
1.1092 + * for best performance and cannot be altered at runtime.
1.1093 + */
1.1094 +#if !defined(__ia64__) && !defined(__sparc__) && !defined(__mips__)
1.1095 +#define MALLOC_STATIC_SIZES 1
1.1096 +#endif
1.1097 +
1.1098 +#ifdef MALLOC_STATIC_SIZES
1.1099 +
1.1100 +/*
1.1101 + * VM page size. It must divide the runtime CPU page size or the code
1.1102 + * will abort.
1.1103 + * Platform specific page size conditions copied from js/public/HeapAPI.h
1.1104 + */
1.1105 +#if (defined(SOLARIS) || defined(__FreeBSD__)) && \
1.1106 + (defined(__sparc) || defined(__sparcv9) || defined(__ia64))
1.1107 +#define pagesize_2pow ((size_t) 13)
1.1108 +#elif defined(__powerpc64__) || defined(__aarch64__)
1.1109 +#define pagesize_2pow ((size_t) 16)
1.1110 +#else
1.1111 +#define pagesize_2pow ((size_t) 12)
1.1112 +#endif
1.1113 +#define pagesize ((size_t) 1 << pagesize_2pow)
1.1114 +#define pagesize_mask (pagesize - 1)
1.1115 +
1.1116 +/* Various quantum-related settings. */
1.1117 +
1.1118 +#define QUANTUM_DEFAULT ((size_t) 1 << QUANTUM_2POW_MIN)
1.1119 +static const size_t quantum = QUANTUM_DEFAULT;
1.1120 +static const size_t quantum_mask = QUANTUM_DEFAULT - 1;
1.1121 +
1.1122 +/* Various bin-related settings. */
1.1123 +
1.1124 +static const size_t small_min = (QUANTUM_DEFAULT >> 1) + 1;
1.1125 +static const size_t small_max = (size_t) SMALL_MAX_DEFAULT;
1.1126 +
1.1127 +/* Max size class for bins. */
1.1128 +static const size_t bin_maxclass = pagesize >> 1;
1.1129 +
1.1130 + /* Number of (2^n)-spaced tiny bins. */
1.1131 +static const unsigned ntbins = (unsigned)
1.1132 + (QUANTUM_2POW_MIN - TINY_MIN_2POW);
1.1133 +
1.1134 + /* Number of quantum-spaced bins. */
1.1135 +static const unsigned nqbins = (unsigned)
1.1136 + (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN);
1.1137 +
1.1138 +/* Number of (2^n)-spaced sub-page bins. */
1.1139 +static const unsigned nsbins = (unsigned)
1.1140 + (pagesize_2pow -
1.1141 + SMALL_MAX_2POW_DEFAULT - 1);
1.1142 +
1.1143 +#else /* !MALLOC_STATIC_SIZES */
1.1144 +
1.1145 +/* VM page size. */
1.1146 +static size_t pagesize;
1.1147 +static size_t pagesize_mask;
1.1148 +static size_t pagesize_2pow;
1.1149 +
1.1150 +/* Various bin-related settings. */
1.1151 +static size_t bin_maxclass; /* Max size class for bins. */
1.1152 +static unsigned ntbins; /* Number of (2^n)-spaced tiny bins. */
1.1153 +static unsigned nqbins; /* Number of quantum-spaced bins. */
1.1154 +static unsigned nsbins; /* Number of (2^n)-spaced sub-page bins. */
1.1155 +static size_t small_min;
1.1156 +static size_t small_max;
1.1157 +
1.1158 +/* Various quantum-related settings. */
1.1159 +static size_t quantum;
1.1160 +static size_t quantum_mask; /* (quantum - 1). */
1.1161 +
1.1162 +#endif
1.1163 +
1.1164 +/* Various chunk-related settings. */
1.1165 +
1.1166 +/*
1.1167 + * Compute the header size such that it is large enough to contain the page map
1.1168 + * and enough nodes for the worst case: one node per non-header page plus one
1.1169 + * extra for situations where we briefly have one more node allocated than we
1.1170 + * will need.
1.1171 + */
1.1172 +#define calculate_arena_header_size() \
1.1173 + (sizeof(arena_chunk_t) + sizeof(arena_chunk_map_t) * (chunk_npages - 1))
1.1174 +
1.1175 +#define calculate_arena_header_pages() \
1.1176 + ((calculate_arena_header_size() >> pagesize_2pow) + \
1.1177 + ((calculate_arena_header_size() & pagesize_mask) ? 1 : 0))
1.1178 +
1.1179 +/* Max size class for arenas. */
1.1180 +#define calculate_arena_maxclass() \
1.1181 + (chunksize - (arena_chunk_header_npages << pagesize_2pow))
1.1182 +
1.1183 +#ifdef MALLOC_STATIC_SIZES
1.1184 +#define CHUNKSIZE_DEFAULT ((size_t) 1 << CHUNK_2POW_DEFAULT)
1.1185 +static const size_t chunksize = CHUNKSIZE_DEFAULT;
1.1186 +static const size_t chunksize_mask =CHUNKSIZE_DEFAULT - 1;
1.1187 +static const size_t chunk_npages = CHUNKSIZE_DEFAULT >> pagesize_2pow;
1.1188 +#define arena_chunk_header_npages calculate_arena_header_pages()
1.1189 +#define arena_maxclass calculate_arena_maxclass()
1.1190 +#else
1.1191 +static size_t chunksize;
1.1192 +static size_t chunksize_mask; /* (chunksize - 1). */
1.1193 +static size_t chunk_npages;
1.1194 +static size_t arena_chunk_header_npages;
1.1195 +static size_t arena_maxclass; /* Max size class for arenas. */
1.1196 +#endif
1.1197 +
1.1198 +/********/
1.1199 +/*
1.1200 + * Chunks.
1.1201 + */
1.1202 +
1.1203 +#ifdef MALLOC_VALIDATE
1.1204 +static malloc_rtree_t *chunk_rtree;
1.1205 +#endif
1.1206 +
1.1207 +/* Protects chunk-related data structures. */
1.1208 +static malloc_mutex_t huge_mtx;
1.1209 +
1.1210 +/* Tree of chunks that are stand-alone huge allocations. */
1.1211 +static extent_tree_t huge;
1.1212 +
1.1213 +#ifdef MALLOC_STATS
1.1214 +/* Huge allocation statistics. */
1.1215 +static uint64_t huge_nmalloc;
1.1216 +static uint64_t huge_ndalloc;
1.1217 +static size_t huge_allocated;
1.1218 +static size_t huge_mapped;
1.1219 +#endif
1.1220 +
1.1221 +#ifdef MALLOC_PAGEFILE
1.1222 +static char pagefile_templ[PATH_MAX];
1.1223 +#endif
1.1224 +
1.1225 +/****************************/
1.1226 +/*
1.1227 + * base (internal allocation).
1.1228 + */
1.1229 +
1.1230 +/*
1.1231 + * Current pages that are being used for internal memory allocations. These
1.1232 + * pages are carved up in cacheline-size quanta, so that there is no chance of
1.1233 + * false cache line sharing.
1.1234 + */
1.1235 +static void *base_pages;
1.1236 +static void *base_next_addr;
1.1237 +#if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1.1238 +static void *base_next_decommitted;
1.1239 +#endif
1.1240 +static void *base_past_addr; /* Addr immediately past base_pages. */
1.1241 +static extent_node_t *base_nodes;
1.1242 +static malloc_mutex_t base_mtx;
1.1243 +#ifdef MALLOC_STATS
1.1244 +static size_t base_mapped;
1.1245 +static size_t base_committed;
1.1246 +#endif
1.1247 +
1.1248 +/********/
1.1249 +/*
1.1250 + * Arenas.
1.1251 + */
1.1252 +
1.1253 +/*
1.1254 + * Arenas that are used to service external requests. Not all elements of the
1.1255 + * arenas array are necessarily used; arenas are created lazily as needed.
1.1256 + */
1.1257 +static arena_t **arenas;
1.1258 +static unsigned narenas;
1.1259 +#ifndef NO_TLS
1.1260 +# ifdef MALLOC_BALANCE
1.1261 +static unsigned narenas_2pow;
1.1262 +# else
1.1263 +static unsigned next_arena;
1.1264 +# endif
1.1265 +#endif
1.1266 +#ifdef MOZ_MEMORY
1.1267 +static malloc_spinlock_t arenas_lock; /* Protects arenas initialization. */
1.1268 +#else
1.1269 +static pthread_mutex_t arenas_lock; /* Protects arenas initialization. */
1.1270 +#endif
1.1271 +
1.1272 +#ifndef NO_TLS
1.1273 +/*
1.1274 + * Map of pthread_self() --> arenas[???], used for selecting an arena to use
1.1275 + * for allocations.
1.1276 + */
1.1277 +#ifndef MOZ_MEMORY_WINDOWS
1.1278 +static __thread arena_t *arenas_map;
1.1279 +#endif
1.1280 +#endif
1.1281 +
1.1282 +/*******************************/
1.1283 +/*
1.1284 + * Runtime configuration options.
1.1285 + */
1.1286 +MOZ_JEMALLOC_API
1.1287 +const char *_malloc_options = MOZ_MALLOC_OPTIONS;
1.1288 +
1.1289 +#ifndef MALLOC_PRODUCTION
1.1290 +static bool opt_abort = true;
1.1291 +#ifdef MALLOC_FILL
1.1292 +static bool opt_junk = true;
1.1293 +static bool opt_poison = true;
1.1294 +static bool opt_zero = false;
1.1295 +#endif
1.1296 +#else
1.1297 +static bool opt_abort = false;
1.1298 +#ifdef MALLOC_FILL
1.1299 +static const bool opt_junk = false;
1.1300 +static const bool opt_poison = true;
1.1301 +static const bool opt_zero = false;
1.1302 +#endif
1.1303 +#endif
1.1304 +
1.1305 +static size_t opt_dirty_max = DIRTY_MAX_DEFAULT;
1.1306 +#ifdef MALLOC_BALANCE
1.1307 +static uint64_t opt_balance_threshold = BALANCE_THRESHOLD_DEFAULT;
1.1308 +#endif
1.1309 +static bool opt_print_stats = false;
1.1310 +#ifdef MALLOC_STATIC_SIZES
1.1311 +#define opt_quantum_2pow QUANTUM_2POW_MIN
1.1312 +#define opt_small_max_2pow SMALL_MAX_2POW_DEFAULT
1.1313 +#define opt_chunk_2pow CHUNK_2POW_DEFAULT
1.1314 +#else
1.1315 +static size_t opt_quantum_2pow = QUANTUM_2POW_MIN;
1.1316 +static size_t opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT;
1.1317 +static size_t opt_chunk_2pow = CHUNK_2POW_DEFAULT;
1.1318 +#endif
1.1319 +#ifdef MALLOC_PAGEFILE
1.1320 +static bool opt_pagefile = false;
1.1321 +#endif
1.1322 +#ifdef MALLOC_UTRACE
1.1323 +static bool opt_utrace = false;
1.1324 +#endif
1.1325 +#ifdef MALLOC_SYSV
1.1326 +static bool opt_sysv = false;
1.1327 +#endif
1.1328 +#ifdef MALLOC_XMALLOC
1.1329 +static bool opt_xmalloc = false;
1.1330 +#endif
1.1331 +static int opt_narenas_lshift = 0;
1.1332 +
1.1333 +#ifdef MALLOC_UTRACE
1.1334 +typedef struct {
1.1335 + void *p;
1.1336 + size_t s;
1.1337 + void *r;
1.1338 +} malloc_utrace_t;
1.1339 +
1.1340 +#define UTRACE(a, b, c) \
1.1341 + if (opt_utrace) { \
1.1342 + malloc_utrace_t ut; \
1.1343 + ut.p = (a); \
1.1344 + ut.s = (b); \
1.1345 + ut.r = (c); \
1.1346 + utrace(&ut, sizeof(ut)); \
1.1347 + }
1.1348 +#else
1.1349 +#define UTRACE(a, b, c)
1.1350 +#endif
1.1351 +
1.1352 +/******************************************************************************/
1.1353 +/*
1.1354 + * Begin function prototypes for non-inline static functions.
1.1355 + */
1.1356 +
1.1357 +static char *umax2s(uintmax_t x, unsigned base, char *s);
1.1358 +static bool malloc_mutex_init(malloc_mutex_t *mutex);
1.1359 +static bool malloc_spin_init(malloc_spinlock_t *lock);
1.1360 +static void wrtmessage(const char *p1, const char *p2, const char *p3,
1.1361 + const char *p4);
1.1362 +#ifdef MALLOC_STATS
1.1363 +#ifdef MOZ_MEMORY_DARWIN
1.1364 +/* Avoid namespace collision with OS X's malloc APIs. */
1.1365 +#define malloc_printf moz_malloc_printf
1.1366 +#endif
1.1367 +static void malloc_printf(const char *format, ...);
1.1368 +#endif
1.1369 +static bool base_pages_alloc_mmap(size_t minsize);
1.1370 +static bool base_pages_alloc(size_t minsize);
1.1371 +static void *base_alloc(size_t size);
1.1372 +static void *base_calloc(size_t number, size_t size);
1.1373 +static extent_node_t *base_node_alloc(void);
1.1374 +static void base_node_dealloc(extent_node_t *node);
1.1375 +#ifdef MALLOC_STATS
1.1376 +static void stats_print(arena_t *arena);
1.1377 +#endif
1.1378 +static void *pages_map(void *addr, size_t size, int pfd);
1.1379 +static void pages_unmap(void *addr, size_t size);
1.1380 +static void *chunk_alloc_mmap(size_t size, bool pagefile);
1.1381 +#ifdef MALLOC_PAGEFILE
1.1382 +static int pagefile_init(size_t size);
1.1383 +static void pagefile_close(int pfd);
1.1384 +#endif
1.1385 +static void *chunk_alloc(size_t size, bool zero, bool pagefile);
1.1386 +static void chunk_dealloc_mmap(void *chunk, size_t size);
1.1387 +static void chunk_dealloc(void *chunk, size_t size);
1.1388 +#ifndef NO_TLS
1.1389 +static arena_t *choose_arena_hard(void);
1.1390 +#endif
1.1391 +static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
1.1392 + bool large, bool zero);
1.1393 +static void arena_chunk_init(arena_t *arena, arena_chunk_t *chunk);
1.1394 +static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk);
1.1395 +static arena_run_t *arena_run_alloc(arena_t *arena, arena_bin_t *bin,
1.1396 + size_t size, bool large, bool zero);
1.1397 +static void arena_purge(arena_t *arena, bool all);
1.1398 +static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
1.1399 +static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
1.1400 + arena_run_t *run, size_t oldsize, size_t newsize);
1.1401 +static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
1.1402 + arena_run_t *run, size_t oldsize, size_t newsize, bool dirty);
1.1403 +static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
1.1404 +static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
1.1405 +static size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size);
1.1406 +#ifdef MALLOC_BALANCE
1.1407 +static void arena_lock_balance_hard(arena_t *arena);
1.1408 +#endif
1.1409 +static void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
1.1410 +static void *arena_palloc(arena_t *arena, size_t alignment, size_t size,
1.1411 + size_t alloc_size);
1.1412 +static size_t arena_salloc(const void *ptr);
1.1413 +static void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk,
1.1414 + void *ptr);
1.1415 +static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk,
1.1416 + void *ptr, size_t size, size_t oldsize);
1.1417 +static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk,
1.1418 + void *ptr, size_t size, size_t oldsize);
1.1419 +static bool arena_ralloc_large(void *ptr, size_t size, size_t oldsize);
1.1420 +static void *arena_ralloc(void *ptr, size_t size, size_t oldsize);
1.1421 +static bool arena_new(arena_t *arena);
1.1422 +static arena_t *arenas_extend(unsigned ind);
1.1423 +static void *huge_malloc(size_t size, bool zero);
1.1424 +static void *huge_palloc(size_t alignment, size_t size);
1.1425 +static void *huge_ralloc(void *ptr, size_t size, size_t oldsize);
1.1426 +static void huge_dalloc(void *ptr);
1.1427 +static void malloc_print_stats(void);
1.1428 +#ifndef MOZ_MEMORY_WINDOWS
1.1429 +static
1.1430 +#endif
1.1431 +bool malloc_init_hard(void);
1.1432 +
1.1433 +static void _malloc_prefork(void);
1.1434 +static void _malloc_postfork(void);
1.1435 +
1.1436 +#ifdef MOZ_MEMORY_DARWIN
1.1437 +/*
1.1438 + * MALLOC_ZONE_T_NOTE
1.1439 + *
1.1440 + * On Darwin, we hook into the memory allocator using a malloc_zone_t struct.
1.1441 + * We must be very careful around this struct because of different behaviour on
1.1442 + * different versions of OSX.
1.1443 + *
1.1444 + * Each of OSX 10.5, 10.6 and 10.7 use different versions of the struct
1.1445 + * (with version numbers 3, 6 and 8 respectively). The binary we use on each of
1.1446 + * these platforms will not necessarily be built using the correct SDK [1].
1.1447 + * This means we need to statically know the correct struct size to use on all
1.1448 + * OSX releases, and have a fallback for unknown future versions. The struct
1.1449 + * sizes defined in osx_zone_types.h.
1.1450 + *
1.1451 + * For OSX 10.8 and later, we may expect the malloc_zone_t struct to change
1.1452 + * again, and need to dynamically account for this. By simply leaving
1.1453 + * malloc_zone_t alone, we don't quite deal with the problem, because there
1.1454 + * remain calls to jemalloc through the mozalloc interface. We check this
1.1455 + * dynamically on each allocation, using the CHECK_DARWIN macro and
1.1456 + * osx_use_jemalloc.
1.1457 + *
1.1458 + *
1.1459 + * [1] Mozilla is built as a universal binary on Mac, supporting i386 and
1.1460 + * x86_64. The i386 target is built using the 10.5 SDK, even if it runs on
1.1461 + * 10.6. The x86_64 target is built using the 10.6 SDK, even if it runs on
1.1462 + * 10.7 or later, or 10.5.
1.1463 + *
1.1464 + * FIXME:
1.1465 + * When later versions of OSX come out (10.8 and up), we need to check their
1.1466 + * malloc_zone_t versions. If they're greater than 8, we need a new version
1.1467 + * of malloc_zone_t adapted into osx_zone_types.h.
1.1468 + */
1.1469 +
1.1470 +#ifndef MOZ_REPLACE_MALLOC
1.1471 +#include "osx_zone_types.h"
1.1472 +
1.1473 +#define LEOPARD_MALLOC_ZONE_T_VERSION 3
1.1474 +#define SNOW_LEOPARD_MALLOC_ZONE_T_VERSION 6
1.1475 +#define LION_MALLOC_ZONE_T_VERSION 8
1.1476 +
1.1477 +static bool osx_use_jemalloc = false;
1.1478 +
1.1479 +
1.1480 +static lion_malloc_zone l_szone;
1.1481 +static malloc_zone_t * szone = (malloc_zone_t*)(&l_szone);
1.1482 +
1.1483 +static lion_malloc_introspection l_ozone_introspect;
1.1484 +static malloc_introspection_t * const ozone_introspect =
1.1485 + (malloc_introspection_t*)(&l_ozone_introspect);
1.1486 +static void szone2ozone(malloc_zone_t *zone, size_t size);
1.1487 +static size_t zone_version_size(int version);
1.1488 +#else
1.1489 +static const bool osx_use_jemalloc = true;
1.1490 +#endif
1.1491 +
1.1492 +#endif
1.1493 +
1.1494 +/*
1.1495 + * End function prototypes.
1.1496 + */
1.1497 +/******************************************************************************/
1.1498 +
1.1499 +/*
1.1500 + * umax2s() provides minimal integer printing functionality, which is
1.1501 + * especially useful for situations where allocation in vsnprintf() calls would
1.1502 + * potentially cause deadlock.
1.1503 + */
1.1504 +#define UMAX2S_BUFSIZE 65
1.1505 +char *
1.1506 +umax2s(uintmax_t x, unsigned base, char *s)
1.1507 +{
1.1508 + unsigned i;
1.1509 +
1.1510 + i = UMAX2S_BUFSIZE - 1;
1.1511 + s[i] = '\0';
1.1512 + switch (base) {
1.1513 + case 10:
1.1514 + do {
1.1515 + i--;
1.1516 + s[i] = "0123456789"[x % 10];
1.1517 + x /= 10;
1.1518 + } while (x > 0);
1.1519 + break;
1.1520 + case 16:
1.1521 + do {
1.1522 + i--;
1.1523 + s[i] = "0123456789abcdef"[x & 0xf];
1.1524 + x >>= 4;
1.1525 + } while (x > 0);
1.1526 + break;
1.1527 + default:
1.1528 + do {
1.1529 + i--;
1.1530 + s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % base];
1.1531 + x /= base;
1.1532 + } while (x > 0);
1.1533 + }
1.1534 +
1.1535 + return (&s[i]);
1.1536 +}
1.1537 +
1.1538 +static void
1.1539 +wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
1.1540 +{
1.1541 +#if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_WINDOWS)
1.1542 +#define _write write
1.1543 +#endif
1.1544 + _write(STDERR_FILENO, p1, (unsigned int) strlen(p1));
1.1545 + _write(STDERR_FILENO, p2, (unsigned int) strlen(p2));
1.1546 + _write(STDERR_FILENO, p3, (unsigned int) strlen(p3));
1.1547 + _write(STDERR_FILENO, p4, (unsigned int) strlen(p4));
1.1548 +}
1.1549 +
1.1550 +MOZ_JEMALLOC_API
1.1551 +void (*_malloc_message)(const char *p1, const char *p2, const char *p3,
1.1552 + const char *p4) = wrtmessage;
1.1553 +
1.1554 +#ifdef MALLOC_DEBUG
1.1555 +# define assert(e) do { \
1.1556 + if (!(e)) { \
1.1557 + char line_buf[UMAX2S_BUFSIZE]; \
1.1558 + _malloc_message(__FILE__, ":", umax2s(__LINE__, 10, \
1.1559 + line_buf), ": Failed assertion: "); \
1.1560 + _malloc_message("\"", #e, "\"\n", ""); \
1.1561 + abort(); \
1.1562 + } \
1.1563 +} while (0)
1.1564 +#else
1.1565 +#define assert(e)
1.1566 +#endif
1.1567 +
1.1568 +#include <mozilla/Assertions.h>
1.1569 +#include <mozilla/Attributes.h>
1.1570 +
1.1571 +/* RELEASE_ASSERT calls jemalloc_crash() instead of calling MOZ_CRASH()
1.1572 + * directly because we want crashing to add a frame to the stack. This makes
1.1573 + * it easier to find the failing assertion in crash stacks. */
1.1574 +MOZ_NEVER_INLINE static void
1.1575 +jemalloc_crash()
1.1576 +{
1.1577 + MOZ_CRASH();
1.1578 +}
1.1579 +
1.1580 +#if defined(MOZ_JEMALLOC_HARD_ASSERTS)
1.1581 +# define RELEASE_ASSERT(assertion) do { \
1.1582 + if (!(assertion)) { \
1.1583 + jemalloc_crash(); \
1.1584 + } \
1.1585 +} while (0)
1.1586 +#else
1.1587 +# define RELEASE_ASSERT(assertion) assert(assertion)
1.1588 +#endif
1.1589 +
1.1590 +/******************************************************************************/
1.1591 +/*
1.1592 + * Begin mutex. We can't use normal pthread mutexes in all places, because
1.1593 + * they require malloc()ed memory, which causes bootstrapping issues in some
1.1594 + * cases.
1.1595 + */
1.1596 +
1.1597 +static bool
1.1598 +malloc_mutex_init(malloc_mutex_t *mutex)
1.1599 +{
1.1600 +#if defined(MOZ_MEMORY_WINDOWS)
1.1601 + if (isthreaded)
1.1602 + if (! __crtInitCritSecAndSpinCount(mutex, _CRT_SPINCOUNT))
1.1603 + return (true);
1.1604 +#elif defined(MOZ_MEMORY_DARWIN)
1.1605 + mutex->lock = OS_SPINLOCK_INIT;
1.1606 +#elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1.1607 + pthread_mutexattr_t attr;
1.1608 + if (pthread_mutexattr_init(&attr) != 0)
1.1609 + return (true);
1.1610 + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
1.1611 + if (pthread_mutex_init(mutex, &attr) != 0) {
1.1612 + pthread_mutexattr_destroy(&attr);
1.1613 + return (true);
1.1614 + }
1.1615 + pthread_mutexattr_destroy(&attr);
1.1616 +#elif defined(MOZ_MEMORY)
1.1617 + if (pthread_mutex_init(mutex, NULL) != 0)
1.1618 + return (true);
1.1619 +#else
1.1620 + static const spinlock_t lock = _SPINLOCK_INITIALIZER;
1.1621 +
1.1622 + mutex->lock = lock;
1.1623 +#endif
1.1624 + return (false);
1.1625 +}
1.1626 +
1.1627 +static inline void
1.1628 +malloc_mutex_lock(malloc_mutex_t *mutex)
1.1629 +{
1.1630 +
1.1631 +#if defined(MOZ_MEMORY_WINDOWS)
1.1632 + EnterCriticalSection(mutex);
1.1633 +#elif defined(MOZ_MEMORY_DARWIN)
1.1634 + OSSpinLockLock(&mutex->lock);
1.1635 +#elif defined(MOZ_MEMORY)
1.1636 + pthread_mutex_lock(mutex);
1.1637 +#else
1.1638 + if (isthreaded)
1.1639 + _SPINLOCK(&mutex->lock);
1.1640 +#endif
1.1641 +}
1.1642 +
1.1643 +static inline void
1.1644 +malloc_mutex_unlock(malloc_mutex_t *mutex)
1.1645 +{
1.1646 +
1.1647 +#if defined(MOZ_MEMORY_WINDOWS)
1.1648 + LeaveCriticalSection(mutex);
1.1649 +#elif defined(MOZ_MEMORY_DARWIN)
1.1650 + OSSpinLockUnlock(&mutex->lock);
1.1651 +#elif defined(MOZ_MEMORY)
1.1652 + pthread_mutex_unlock(mutex);
1.1653 +#else
1.1654 + if (isthreaded)
1.1655 + _SPINUNLOCK(&mutex->lock);
1.1656 +#endif
1.1657 +}
1.1658 +
1.1659 +static bool
1.1660 +malloc_spin_init(malloc_spinlock_t *lock)
1.1661 +{
1.1662 +#if defined(MOZ_MEMORY_WINDOWS)
1.1663 + if (isthreaded)
1.1664 + if (! __crtInitCritSecAndSpinCount(lock, _CRT_SPINCOUNT))
1.1665 + return (true);
1.1666 +#elif defined(MOZ_MEMORY_DARWIN)
1.1667 + lock->lock = OS_SPINLOCK_INIT;
1.1668 +#elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
1.1669 + pthread_mutexattr_t attr;
1.1670 + if (pthread_mutexattr_init(&attr) != 0)
1.1671 + return (true);
1.1672 + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
1.1673 + if (pthread_mutex_init(lock, &attr) != 0) {
1.1674 + pthread_mutexattr_destroy(&attr);
1.1675 + return (true);
1.1676 + }
1.1677 + pthread_mutexattr_destroy(&attr);
1.1678 +#elif defined(MOZ_MEMORY)
1.1679 + if (pthread_mutex_init(lock, NULL) != 0)
1.1680 + return (true);
1.1681 +#else
1.1682 + lock->lock = _SPINLOCK_INITIALIZER;
1.1683 +#endif
1.1684 + return (false);
1.1685 +}
1.1686 +
1.1687 +static inline void
1.1688 +malloc_spin_lock(malloc_spinlock_t *lock)
1.1689 +{
1.1690 +
1.1691 +#if defined(MOZ_MEMORY_WINDOWS)
1.1692 + EnterCriticalSection(lock);
1.1693 +#elif defined(MOZ_MEMORY_DARWIN)
1.1694 + OSSpinLockLock(&lock->lock);
1.1695 +#elif defined(MOZ_MEMORY)
1.1696 + pthread_mutex_lock(lock);
1.1697 +#else
1.1698 + if (isthreaded)
1.1699 + _SPINLOCK(&lock->lock);
1.1700 +#endif
1.1701 +}
1.1702 +
1.1703 +static inline void
1.1704 +malloc_spin_unlock(malloc_spinlock_t *lock)
1.1705 +{
1.1706 +#if defined(MOZ_MEMORY_WINDOWS)
1.1707 + LeaveCriticalSection(lock);
1.1708 +#elif defined(MOZ_MEMORY_DARWIN)
1.1709 + OSSpinLockUnlock(&lock->lock);
1.1710 +#elif defined(MOZ_MEMORY)
1.1711 + pthread_mutex_unlock(lock);
1.1712 +#else
1.1713 + if (isthreaded)
1.1714 + _SPINUNLOCK(&lock->lock);
1.1715 +#endif
1.1716 +}
1.1717 +
1.1718 +/*
1.1719 + * End mutex.
1.1720 + */
1.1721 +/******************************************************************************/
1.1722 +/*
1.1723 + * Begin spin lock. Spin locks here are actually adaptive mutexes that block
1.1724 + * after a period of spinning, because unbounded spinning would allow for
1.1725 + * priority inversion.
1.1726 + */
1.1727 +
1.1728 +#if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_DARWIN)
1.1729 +# define malloc_spin_init malloc_mutex_init
1.1730 +# define malloc_spin_lock malloc_mutex_lock
1.1731 +# define malloc_spin_unlock malloc_mutex_unlock
1.1732 +#endif
1.1733 +
1.1734 +#ifndef MOZ_MEMORY
1.1735 +/*
1.1736 + * We use an unpublished interface to initialize pthread mutexes with an
1.1737 + * allocation callback, in order to avoid infinite recursion.
1.1738 + */
1.1739 +int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
1.1740 + void *(calloc_cb)(size_t, size_t));
1.1741 +
1.1742 +__weak_reference(_pthread_mutex_init_calloc_cb_stub,
1.1743 + _pthread_mutex_init_calloc_cb);
1.1744 +
1.1745 +int
1.1746 +_pthread_mutex_init_calloc_cb_stub(pthread_mutex_t *mutex,
1.1747 + void *(calloc_cb)(size_t, size_t))
1.1748 +{
1.1749 +
1.1750 + return (0);
1.1751 +}
1.1752 +
1.1753 +static bool
1.1754 +malloc_spin_init(pthread_mutex_t *lock)
1.1755 +{
1.1756 +
1.1757 + if (_pthread_mutex_init_calloc_cb(lock, base_calloc) != 0)
1.1758 + return (true);
1.1759 +
1.1760 + return (false);
1.1761 +}
1.1762 +
1.1763 +static inline unsigned
1.1764 +malloc_spin_lock(pthread_mutex_t *lock)
1.1765 +{
1.1766 + unsigned ret = 0;
1.1767 +
1.1768 + if (isthreaded) {
1.1769 + if (_pthread_mutex_trylock(lock) != 0) {
1.1770 + unsigned i;
1.1771 + volatile unsigned j;
1.1772 +
1.1773 + /* Exponentially back off. */
1.1774 + for (i = 1; i <= SPIN_LIMIT_2POW; i++) {
1.1775 + for (j = 0; j < (1U << i); j++)
1.1776 + ret++;
1.1777 +
1.1778 + CPU_SPINWAIT;
1.1779 + if (_pthread_mutex_trylock(lock) == 0)
1.1780 + return (ret);
1.1781 + }
1.1782 +
1.1783 + /*
1.1784 + * Spinning failed. Block until the lock becomes
1.1785 + * available, in order to avoid indefinite priority
1.1786 + * inversion.
1.1787 + */
1.1788 + _pthread_mutex_lock(lock);
1.1789 + assert((ret << BLOCK_COST_2POW) != 0);
1.1790 + return (ret << BLOCK_COST_2POW);
1.1791 + }
1.1792 + }
1.1793 +
1.1794 + return (ret);
1.1795 +}
1.1796 +
1.1797 +static inline void
1.1798 +malloc_spin_unlock(pthread_mutex_t *lock)
1.1799 +{
1.1800 +
1.1801 + if (isthreaded)
1.1802 + _pthread_mutex_unlock(lock);
1.1803 +}
1.1804 +#endif
1.1805 +
1.1806 +/*
1.1807 + * End spin lock.
1.1808 + */
1.1809 +/******************************************************************************/
1.1810 +/*
1.1811 + * Begin Utility functions/macros.
1.1812 + */
1.1813 +
1.1814 +/* Return the chunk address for allocation address a. */
1.1815 +#define CHUNK_ADDR2BASE(a) \
1.1816 + ((void *)((uintptr_t)(a) & ~chunksize_mask))
1.1817 +
1.1818 +/* Return the chunk offset of address a. */
1.1819 +#define CHUNK_ADDR2OFFSET(a) \
1.1820 + ((size_t)((uintptr_t)(a) & chunksize_mask))
1.1821 +
1.1822 +/* Return the smallest chunk multiple that is >= s. */
1.1823 +#define CHUNK_CEILING(s) \
1.1824 + (((s) + chunksize_mask) & ~chunksize_mask)
1.1825 +
1.1826 +/* Return the smallest cacheline multiple that is >= s. */
1.1827 +#define CACHELINE_CEILING(s) \
1.1828 + (((s) + (CACHELINE - 1)) & ~(CACHELINE - 1))
1.1829 +
1.1830 +/* Return the smallest quantum multiple that is >= a. */
1.1831 +#define QUANTUM_CEILING(a) \
1.1832 + (((a) + quantum_mask) & ~quantum_mask)
1.1833 +
1.1834 +/* Return the smallest pagesize multiple that is >= s. */
1.1835 +#define PAGE_CEILING(s) \
1.1836 + (((s) + pagesize_mask) & ~pagesize_mask)
1.1837 +
1.1838 +/* Compute the smallest power of 2 that is >= x. */
1.1839 +static inline size_t
1.1840 +pow2_ceil(size_t x)
1.1841 +{
1.1842 +
1.1843 + x--;
1.1844 + x |= x >> 1;
1.1845 + x |= x >> 2;
1.1846 + x |= x >> 4;
1.1847 + x |= x >> 8;
1.1848 + x |= x >> 16;
1.1849 +#if (SIZEOF_PTR == 8)
1.1850 + x |= x >> 32;
1.1851 +#endif
1.1852 + x++;
1.1853 + return (x);
1.1854 +}
1.1855 +
1.1856 +#ifdef MALLOC_BALANCE
1.1857 +/*
1.1858 + * Use a simple linear congruential pseudo-random number generator:
1.1859 + *
1.1860 + * prn(y) = (a*x + c) % m
1.1861 + *
1.1862 + * where the following constants ensure maximal period:
1.1863 + *
1.1864 + * a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4.
1.1865 + * c == Odd number (relatively prime to 2^n).
1.1866 + * m == 2^32
1.1867 + *
1.1868 + * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints.
1.1869 + *
1.1870 + * This choice of m has the disadvantage that the quality of the bits is
1.1871 + * proportional to bit position. For example. the lowest bit has a cycle of 2,
1.1872 + * the next has a cycle of 4, etc. For this reason, we prefer to use the upper
1.1873 + * bits.
1.1874 + */
1.1875 +# define PRN_DEFINE(suffix, var, a, c) \
1.1876 +static inline void \
1.1877 +sprn_##suffix(uint32_t seed) \
1.1878 +{ \
1.1879 + var = seed; \
1.1880 +} \
1.1881 + \
1.1882 +static inline uint32_t \
1.1883 +prn_##suffix(uint32_t lg_range) \
1.1884 +{ \
1.1885 + uint32_t ret, x; \
1.1886 + \
1.1887 + assert(lg_range > 0); \
1.1888 + assert(lg_range <= 32); \
1.1889 + \
1.1890 + x = (var * (a)) + (c); \
1.1891 + var = x; \
1.1892 + ret = x >> (32 - lg_range); \
1.1893 + \
1.1894 + return (ret); \
1.1895 +}
1.1896 +# define SPRN(suffix, seed) sprn_##suffix(seed)
1.1897 +# define PRN(suffix, lg_range) prn_##suffix(lg_range)
1.1898 +#endif
1.1899 +
1.1900 +#ifdef MALLOC_BALANCE
1.1901 +/* Define the PRNG used for arena assignment. */
1.1902 +static __thread uint32_t balance_x;
1.1903 +PRN_DEFINE(balance, balance_x, 1297, 1301)
1.1904 +#endif
1.1905 +
1.1906 +#ifdef MALLOC_UTRACE
1.1907 +static int
1.1908 +utrace(const void *addr, size_t len)
1.1909 +{
1.1910 + malloc_utrace_t *ut = (malloc_utrace_t *)addr;
1.1911 + char buf_a[UMAX2S_BUFSIZE];
1.1912 + char buf_b[UMAX2S_BUFSIZE];
1.1913 +
1.1914 + assert(len == sizeof(malloc_utrace_t));
1.1915 +
1.1916 + if (ut->p == NULL && ut->s == 0 && ut->r == NULL) {
1.1917 + _malloc_message(
1.1918 + umax2s(getpid(), 10, buf_a),
1.1919 + " x USER malloc_init()\n", "", "");
1.1920 + } else if (ut->p == NULL && ut->r != NULL) {
1.1921 + _malloc_message(
1.1922 + umax2s(getpid(), 10, buf_a),
1.1923 + " x USER 0x",
1.1924 + umax2s((uintptr_t)ut->r, 16, buf_b),
1.1925 + " = malloc(");
1.1926 + _malloc_message(
1.1927 + umax2s(ut->s, 10, buf_a),
1.1928 + ")\n", "", "");
1.1929 + } else if (ut->p != NULL && ut->r != NULL) {
1.1930 + _malloc_message(
1.1931 + umax2s(getpid(), 10, buf_a),
1.1932 + " x USER 0x",
1.1933 + umax2s((uintptr_t)ut->r, 16, buf_b),
1.1934 + " = realloc(0x");
1.1935 + _malloc_message(
1.1936 + umax2s((uintptr_t)ut->p, 16, buf_a),
1.1937 + ", ",
1.1938 + umax2s(ut->s, 10, buf_b),
1.1939 + ")\n");
1.1940 + } else {
1.1941 + _malloc_message(
1.1942 + umax2s(getpid(), 10, buf_a),
1.1943 + " x USER free(0x",
1.1944 + umax2s((uintptr_t)ut->p, 16, buf_b),
1.1945 + ")\n");
1.1946 + }
1.1947 +
1.1948 + return (0);
1.1949 +}
1.1950 +#endif
1.1951 +
1.1952 +static inline const char *
1.1953 +_getprogname(void)
1.1954 +{
1.1955 +
1.1956 + return ("<jemalloc>");
1.1957 +}
1.1958 +
1.1959 +#ifdef MALLOC_STATS
1.1960 +/*
1.1961 + * Print to stderr in such a way as to (hopefully) avoid memory allocation.
1.1962 + */
1.1963 +static void
1.1964 +malloc_printf(const char *format, ...)
1.1965 +{
1.1966 + char buf[4096];
1.1967 + va_list ap;
1.1968 +
1.1969 + va_start(ap, format);
1.1970 + vsnprintf(buf, sizeof(buf), format, ap);
1.1971 + va_end(ap);
1.1972 + _malloc_message(buf, "", "", "");
1.1973 +}
1.1974 +#endif
1.1975 +
1.1976 +/******************************************************************************/
1.1977 +
1.1978 +static inline void
1.1979 +pages_decommit(void *addr, size_t size)
1.1980 +{
1.1981 +
1.1982 +#ifdef MOZ_MEMORY_WINDOWS
1.1983 + VirtualFree(addr, size, MEM_DECOMMIT);
1.1984 +#else
1.1985 + if (mmap(addr, size, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1,
1.1986 + 0) == MAP_FAILED)
1.1987 + abort();
1.1988 +#endif
1.1989 +}
1.1990 +
1.1991 +static inline void
1.1992 +pages_commit(void *addr, size_t size)
1.1993 +{
1.1994 +
1.1995 +# ifdef MOZ_MEMORY_WINDOWS
1.1996 + if (!VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE))
1.1997 + abort();
1.1998 +# else
1.1999 + if (mmap(addr, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE |
1.2000 + MAP_ANON, -1, 0) == MAP_FAILED)
1.2001 + abort();
1.2002 +# endif
1.2003 +}
1.2004 +
1.2005 +static bool
1.2006 +base_pages_alloc_mmap(size_t minsize)
1.2007 +{
1.2008 + bool ret;
1.2009 + size_t csize;
1.2010 +#if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1.2011 + size_t pminsize;
1.2012 +#endif
1.2013 + int pfd;
1.2014 +
1.2015 + assert(minsize != 0);
1.2016 + csize = CHUNK_CEILING(minsize);
1.2017 +#ifdef MALLOC_PAGEFILE
1.2018 + if (opt_pagefile) {
1.2019 + pfd = pagefile_init(csize);
1.2020 + if (pfd == -1)
1.2021 + return (true);
1.2022 + } else
1.2023 +#endif
1.2024 + pfd = -1;
1.2025 + base_pages = pages_map(NULL, csize, pfd);
1.2026 + if (base_pages == NULL) {
1.2027 + ret = true;
1.2028 + goto RETURN;
1.2029 + }
1.2030 + base_next_addr = base_pages;
1.2031 + base_past_addr = (void *)((uintptr_t)base_pages + csize);
1.2032 +#if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1.2033 + /*
1.2034 + * Leave enough pages for minsize committed, since otherwise they would
1.2035 + * have to be immediately recommitted.
1.2036 + */
1.2037 + pminsize = PAGE_CEILING(minsize);
1.2038 + base_next_decommitted = (void *)((uintptr_t)base_pages + pminsize);
1.2039 +# if defined(MALLOC_DECOMMIT)
1.2040 + if (pminsize < csize)
1.2041 + pages_decommit(base_next_decommitted, csize - pminsize);
1.2042 +# endif
1.2043 +# ifdef MALLOC_STATS
1.2044 + base_mapped += csize;
1.2045 + base_committed += pminsize;
1.2046 +# endif
1.2047 +#endif
1.2048 +
1.2049 + ret = false;
1.2050 +RETURN:
1.2051 +#ifdef MALLOC_PAGEFILE
1.2052 + if (pfd != -1)
1.2053 + pagefile_close(pfd);
1.2054 +#endif
1.2055 + return (false);
1.2056 +}
1.2057 +
1.2058 +static bool
1.2059 +base_pages_alloc(size_t minsize)
1.2060 +{
1.2061 +
1.2062 + if (base_pages_alloc_mmap(minsize) == false)
1.2063 + return (false);
1.2064 +
1.2065 + return (true);
1.2066 +}
1.2067 +
1.2068 +static void *
1.2069 +base_alloc(size_t size)
1.2070 +{
1.2071 + void *ret;
1.2072 + size_t csize;
1.2073 +
1.2074 + /* Round size up to nearest multiple of the cacheline size. */
1.2075 + csize = CACHELINE_CEILING(size);
1.2076 +
1.2077 + malloc_mutex_lock(&base_mtx);
1.2078 + /* Make sure there's enough space for the allocation. */
1.2079 + if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) {
1.2080 + if (base_pages_alloc(csize)) {
1.2081 + malloc_mutex_unlock(&base_mtx);
1.2082 + return (NULL);
1.2083 + }
1.2084 + }
1.2085 + /* Allocate. */
1.2086 + ret = base_next_addr;
1.2087 + base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
1.2088 +#if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
1.2089 + /* Make sure enough pages are committed for the new allocation. */
1.2090 + if ((uintptr_t)base_next_addr > (uintptr_t)base_next_decommitted) {
1.2091 + void *pbase_next_addr =
1.2092 + (void *)(PAGE_CEILING((uintptr_t)base_next_addr));
1.2093 +
1.2094 +# ifdef MALLOC_DECOMMIT
1.2095 + pages_commit(base_next_decommitted, (uintptr_t)pbase_next_addr -
1.2096 + (uintptr_t)base_next_decommitted);
1.2097 +# endif
1.2098 + base_next_decommitted = pbase_next_addr;
1.2099 +# ifdef MALLOC_STATS
1.2100 + base_committed += (uintptr_t)pbase_next_addr -
1.2101 + (uintptr_t)base_next_decommitted;
1.2102 +# endif
1.2103 + }
1.2104 +#endif
1.2105 + malloc_mutex_unlock(&base_mtx);
1.2106 + VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
1.2107 +
1.2108 + return (ret);
1.2109 +}
1.2110 +
1.2111 +static void *
1.2112 +base_calloc(size_t number, size_t size)
1.2113 +{
1.2114 + void *ret;
1.2115 +
1.2116 + ret = base_alloc(number * size);
1.2117 +#ifdef MALLOC_VALGRIND
1.2118 + if (ret != NULL) {
1.2119 + VALGRIND_FREELIKE_BLOCK(ret, 0);
1.2120 + VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, true);
1.2121 + }
1.2122 +#endif
1.2123 + memset(ret, 0, number * size);
1.2124 +
1.2125 + return (ret);
1.2126 +}
1.2127 +
1.2128 +static extent_node_t *
1.2129 +base_node_alloc(void)
1.2130 +{
1.2131 + extent_node_t *ret;
1.2132 +
1.2133 + malloc_mutex_lock(&base_mtx);
1.2134 + if (base_nodes != NULL) {
1.2135 + ret = base_nodes;
1.2136 + base_nodes = *(extent_node_t **)ret;
1.2137 + VALGRIND_FREELIKE_BLOCK(ret, 0);
1.2138 + VALGRIND_MALLOCLIKE_BLOCK(ret, sizeof(extent_node_t), 0, false);
1.2139 + malloc_mutex_unlock(&base_mtx);
1.2140 + } else {
1.2141 + malloc_mutex_unlock(&base_mtx);
1.2142 + ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
1.2143 + }
1.2144 +
1.2145 + return (ret);
1.2146 +}
1.2147 +
1.2148 +static void
1.2149 +base_node_dealloc(extent_node_t *node)
1.2150 +{
1.2151 +
1.2152 + malloc_mutex_lock(&base_mtx);
1.2153 + VALGRIND_FREELIKE_BLOCK(node, 0);
1.2154 + VALGRIND_MALLOCLIKE_BLOCK(node, sizeof(extent_node_t *), 0, false);
1.2155 + *(extent_node_t **)node = base_nodes;
1.2156 + base_nodes = node;
1.2157 + malloc_mutex_unlock(&base_mtx);
1.2158 +}
1.2159 +
1.2160 +/******************************************************************************/
1.2161 +
1.2162 +#ifdef MALLOC_STATS
1.2163 +static void
1.2164 +stats_print(arena_t *arena)
1.2165 +{
1.2166 + unsigned i, gap_start;
1.2167 +
1.2168 +#ifdef MOZ_MEMORY_WINDOWS
1.2169 + malloc_printf("dirty: %Iu page%s dirty, %I64u sweep%s,"
1.2170 + " %I64u madvise%s, %I64u page%s purged\n",
1.2171 + arena->ndirty, arena->ndirty == 1 ? "" : "s",
1.2172 + arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
1.2173 + arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
1.2174 + arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
1.2175 +# ifdef MALLOC_DECOMMIT
1.2176 + malloc_printf("decommit: %I64u decommit%s, %I64u commit%s,"
1.2177 + " %I64u page%s decommitted\n",
1.2178 + arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
1.2179 + arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
1.2180 + arena->stats.decommitted,
1.2181 + (arena->stats.decommitted == 1) ? "" : "s");
1.2182 +# endif
1.2183 +
1.2184 + malloc_printf(" allocated nmalloc ndalloc\n");
1.2185 + malloc_printf("small: %12Iu %12I64u %12I64u\n",
1.2186 + arena->stats.allocated_small, arena->stats.nmalloc_small,
1.2187 + arena->stats.ndalloc_small);
1.2188 + malloc_printf("large: %12Iu %12I64u %12I64u\n",
1.2189 + arena->stats.allocated_large, arena->stats.nmalloc_large,
1.2190 + arena->stats.ndalloc_large);
1.2191 + malloc_printf("total: %12Iu %12I64u %12I64u\n",
1.2192 + arena->stats.allocated_small + arena->stats.allocated_large,
1.2193 + arena->stats.nmalloc_small + arena->stats.nmalloc_large,
1.2194 + arena->stats.ndalloc_small + arena->stats.ndalloc_large);
1.2195 + malloc_printf("mapped: %12Iu\n", arena->stats.mapped);
1.2196 +#else
1.2197 + malloc_printf("dirty: %zu page%s dirty, %llu sweep%s,"
1.2198 + " %llu madvise%s, %llu page%s purged\n",
1.2199 + arena->ndirty, arena->ndirty == 1 ? "" : "s",
1.2200 + arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
1.2201 + arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
1.2202 + arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
1.2203 +# ifdef MALLOC_DECOMMIT
1.2204 + malloc_printf("decommit: %llu decommit%s, %llu commit%s,"
1.2205 + " %llu page%s decommitted\n",
1.2206 + arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
1.2207 + arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
1.2208 + arena->stats.decommitted,
1.2209 + (arena->stats.decommitted == 1) ? "" : "s");
1.2210 +# endif
1.2211 +
1.2212 + malloc_printf(" allocated nmalloc ndalloc\n");
1.2213 + malloc_printf("small: %12zu %12llu %12llu\n",
1.2214 + arena->stats.allocated_small, arena->stats.nmalloc_small,
1.2215 + arena->stats.ndalloc_small);
1.2216 + malloc_printf("large: %12zu %12llu %12llu\n",
1.2217 + arena->stats.allocated_large, arena->stats.nmalloc_large,
1.2218 + arena->stats.ndalloc_large);
1.2219 + malloc_printf("total: %12zu %12llu %12llu\n",
1.2220 + arena->stats.allocated_small + arena->stats.allocated_large,
1.2221 + arena->stats.nmalloc_small + arena->stats.nmalloc_large,
1.2222 + arena->stats.ndalloc_small + arena->stats.ndalloc_large);
1.2223 + malloc_printf("mapped: %12zu\n", arena->stats.mapped);
1.2224 +#endif
1.2225 + malloc_printf("bins: bin size regs pgs requests newruns"
1.2226 + " reruns maxruns curruns\n");
1.2227 + for (i = 0, gap_start = UINT_MAX; i < ntbins + nqbins + nsbins; i++) {
1.2228 + if (arena->bins[i].stats.nrequests == 0) {
1.2229 + if (gap_start == UINT_MAX)
1.2230 + gap_start = i;
1.2231 + } else {
1.2232 + if (gap_start != UINT_MAX) {
1.2233 + if (i > gap_start + 1) {
1.2234 + /* Gap of more than one size class. */
1.2235 + malloc_printf("[%u..%u]\n",
1.2236 + gap_start, i - 1);
1.2237 + } else {
1.2238 + /* Gap of one size class. */
1.2239 + malloc_printf("[%u]\n", gap_start);
1.2240 + }
1.2241 + gap_start = UINT_MAX;
1.2242 + }
1.2243 + malloc_printf(
1.2244 +#if defined(MOZ_MEMORY_WINDOWS)
1.2245 + "%13u %1s %4u %4u %3u %9I64u %9I64u"
1.2246 + " %9I64u %7u %7u\n",
1.2247 +#else
1.2248 + "%13u %1s %4u %4u %3u %9llu %9llu"
1.2249 + " %9llu %7lu %7lu\n",
1.2250 +#endif
1.2251 + i,
1.2252 + i < ntbins ? "T" : i < ntbins + nqbins ? "Q" : "S",
1.2253 + arena->bins[i].reg_size,
1.2254 + arena->bins[i].nregs,
1.2255 + arena->bins[i].run_size >> pagesize_2pow,
1.2256 + arena->bins[i].stats.nrequests,
1.2257 + arena->bins[i].stats.nruns,
1.2258 + arena->bins[i].stats.reruns,
1.2259 + arena->bins[i].stats.highruns,
1.2260 + arena->bins[i].stats.curruns);
1.2261 + }
1.2262 + }
1.2263 + if (gap_start != UINT_MAX) {
1.2264 + if (i > gap_start + 1) {
1.2265 + /* Gap of more than one size class. */
1.2266 + malloc_printf("[%u..%u]\n", gap_start, i - 1);
1.2267 + } else {
1.2268 + /* Gap of one size class. */
1.2269 + malloc_printf("[%u]\n", gap_start);
1.2270 + }
1.2271 + }
1.2272 +}
1.2273 +#endif
1.2274 +
1.2275 +/*
1.2276 + * End Utility functions/macros.
1.2277 + */
1.2278 +/******************************************************************************/
1.2279 +/*
1.2280 + * Begin extent tree code.
1.2281 + */
1.2282 +
1.2283 +static inline int
1.2284 +extent_szad_comp(extent_node_t *a, extent_node_t *b)
1.2285 +{
1.2286 + int ret;
1.2287 + size_t a_size = a->size;
1.2288 + size_t b_size = b->size;
1.2289 +
1.2290 + ret = (a_size > b_size) - (a_size < b_size);
1.2291 + if (ret == 0) {
1.2292 + uintptr_t a_addr = (uintptr_t)a->addr;
1.2293 + uintptr_t b_addr = (uintptr_t)b->addr;
1.2294 +
1.2295 + ret = (a_addr > b_addr) - (a_addr < b_addr);
1.2296 + }
1.2297 +
1.2298 + return (ret);
1.2299 +}
1.2300 +
1.2301 +/* Wrap red-black tree macros in functions. */
1.2302 +rb_wrap(static, extent_tree_szad_, extent_tree_t, extent_node_t,
1.2303 + link_szad, extent_szad_comp)
1.2304 +
1.2305 +static inline int
1.2306 +extent_ad_comp(extent_node_t *a, extent_node_t *b)
1.2307 +{
1.2308 + uintptr_t a_addr = (uintptr_t)a->addr;
1.2309 + uintptr_t b_addr = (uintptr_t)b->addr;
1.2310 +
1.2311 + return ((a_addr > b_addr) - (a_addr < b_addr));
1.2312 +}
1.2313 +
1.2314 +/* Wrap red-black tree macros in functions. */
1.2315 +rb_wrap(static, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
1.2316 + extent_ad_comp)
1.2317 +
1.2318 +/*
1.2319 + * End extent tree code.
1.2320 + */
1.2321 +/******************************************************************************/
1.2322 +/*
1.2323 + * Begin chunk management functions.
1.2324 + */
1.2325 +
1.2326 +#ifdef MOZ_MEMORY_WINDOWS
1.2327 +
1.2328 +static void *
1.2329 +pages_map(void *addr, size_t size, int pfd)
1.2330 +{
1.2331 + void *ret = NULL;
1.2332 + ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
1.2333 + PAGE_READWRITE);
1.2334 + return (ret);
1.2335 +}
1.2336 +
1.2337 +static void
1.2338 +pages_unmap(void *addr, size_t size)
1.2339 +{
1.2340 + if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
1.2341 + _malloc_message(_getprogname(),
1.2342 + ": (malloc) Error in VirtualFree()\n", "", "");
1.2343 + if (opt_abort)
1.2344 + abort();
1.2345 + }
1.2346 +}
1.2347 +#else
1.2348 +#ifdef JEMALLOC_USES_MAP_ALIGN
1.2349 +static void *
1.2350 +pages_map_align(size_t size, int pfd, size_t alignment)
1.2351 +{
1.2352 + void *ret;
1.2353 +
1.2354 + /*
1.2355 + * We don't use MAP_FIXED here, because it can cause the *replacement*
1.2356 + * of existing mappings, and we only want to create new mappings.
1.2357 + */
1.2358 +#ifdef MALLOC_PAGEFILE
1.2359 + if (pfd != -1) {
1.2360 + ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
1.2361 + MAP_NOSYNC | MAP_ALIGN, pfd, 0);
1.2362 + } else
1.2363 +#endif
1.2364 + {
1.2365 + ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
1.2366 + MAP_NOSYNC | MAP_ALIGN | MAP_ANON, -1, 0);
1.2367 + }
1.2368 + assert(ret != NULL);
1.2369 +
1.2370 + if (ret == MAP_FAILED)
1.2371 + ret = NULL;
1.2372 + return (ret);
1.2373 +}
1.2374 +#endif
1.2375 +
1.2376 +static void *
1.2377 +pages_map(void *addr, size_t size, int pfd)
1.2378 +{
1.2379 + void *ret;
1.2380 +#if defined(__ia64__)
1.2381 + /*
1.2382 + * The JS engine assumes that all allocated pointers have their high 17 bits clear,
1.2383 + * which ia64's mmap doesn't support directly. However, we can emulate it by passing
1.2384 + * mmap an "addr" parameter with those bits clear. The mmap will return that address,
1.2385 + * or the nearest available memory above that address, providing a near-guarantee
1.2386 + * that those bits are clear. If they are not, we return NULL below to indicate
1.2387 + * out-of-memory.
1.2388 + *
1.2389 + * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual
1.2390 + * address space.
1.2391 + *
1.2392 + * See Bug 589735 for more information.
1.2393 + */
1.2394 + bool check_placement = true;
1.2395 + if (addr == NULL) {
1.2396 + addr = (void*)0x0000070000000000;
1.2397 + check_placement = false;
1.2398 + }
1.2399 +#endif
1.2400 +
1.2401 + /*
1.2402 + * We don't use MAP_FIXED here, because it can cause the *replacement*
1.2403 + * of existing mappings, and we only want to create new mappings.
1.2404 + */
1.2405 +#ifdef MALLOC_PAGEFILE
1.2406 + if (pfd != -1) {
1.2407 + ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
1.2408 + MAP_NOSYNC, pfd, 0);
1.2409 + } else
1.2410 +#endif
1.2411 + {
1.2412 + ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
1.2413 + MAP_ANON, -1, 0);
1.2414 + }
1.2415 + assert(ret != NULL);
1.2416 +
1.2417 + if (ret == MAP_FAILED) {
1.2418 + ret = NULL;
1.2419 + }
1.2420 +#if defined(__ia64__)
1.2421 + /*
1.2422 + * If the allocated memory doesn't have its upper 17 bits clear, consider it
1.2423 + * as out of memory.
1.2424 + */
1.2425 + else if ((long long)ret & 0xffff800000000000) {
1.2426 + munmap(ret, size);
1.2427 + ret = NULL;
1.2428 + }
1.2429 + /* If the caller requested a specific memory location, verify that's what mmap returned. */
1.2430 + else if (check_placement && ret != addr) {
1.2431 +#else
1.2432 + else if (addr != NULL && ret != addr) {
1.2433 +#endif
1.2434 + /*
1.2435 + * We succeeded in mapping memory, but not in the right place.
1.2436 + */
1.2437 + if (munmap(ret, size) == -1) {
1.2438 + char buf[STRERROR_BUF];
1.2439 +
1.2440 + strerror_r(errno, buf, sizeof(buf));
1.2441 + _malloc_message(_getprogname(),
1.2442 + ": (malloc) Error in munmap(): ", buf, "\n");
1.2443 + if (opt_abort)
1.2444 + abort();
1.2445 + }
1.2446 + ret = NULL;
1.2447 + }
1.2448 +
1.2449 +#if defined(__ia64__)
1.2450 + assert(ret == NULL || (!check_placement && ret != NULL)
1.2451 + || (check_placement && ret == addr));
1.2452 +#else
1.2453 + assert(ret == NULL || (addr == NULL && ret != addr)
1.2454 + || (addr != NULL && ret == addr));
1.2455 +#endif
1.2456 + return (ret);
1.2457 +}
1.2458 +
1.2459 +static void
1.2460 +pages_unmap(void *addr, size_t size)
1.2461 +{
1.2462 +
1.2463 + if (munmap(addr, size) == -1) {
1.2464 + char buf[STRERROR_BUF];
1.2465 +
1.2466 + strerror_r(errno, buf, sizeof(buf));
1.2467 + _malloc_message(_getprogname(),
1.2468 + ": (malloc) Error in munmap(): ", buf, "\n");
1.2469 + if (opt_abort)
1.2470 + abort();
1.2471 + }
1.2472 +}
1.2473 +#endif
1.2474 +
1.2475 +#ifdef MOZ_MEMORY_DARWIN
1.2476 +#define VM_COPY_MIN (pagesize << 5)
1.2477 +static inline void
1.2478 +pages_copy(void *dest, const void *src, size_t n)
1.2479 +{
1.2480 +
1.2481 + assert((void *)((uintptr_t)dest & ~pagesize_mask) == dest);
1.2482 + assert(n >= VM_COPY_MIN);
1.2483 + assert((void *)((uintptr_t)src & ~pagesize_mask) == src);
1.2484 +
1.2485 + vm_copy(mach_task_self(), (vm_address_t)src, (vm_size_t)n,
1.2486 + (vm_address_t)dest);
1.2487 +}
1.2488 +#endif
1.2489 +
1.2490 +#ifdef MALLOC_VALIDATE
1.2491 +static inline malloc_rtree_t *
1.2492 +malloc_rtree_new(unsigned bits)
1.2493 +{
1.2494 + malloc_rtree_t *ret;
1.2495 + unsigned bits_per_level, height, i;
1.2496 +
1.2497 + bits_per_level = ffs(pow2_ceil((MALLOC_RTREE_NODESIZE /
1.2498 + sizeof(void *)))) - 1;
1.2499 + height = bits / bits_per_level;
1.2500 + if (height * bits_per_level != bits)
1.2501 + height++;
1.2502 + RELEASE_ASSERT(height * bits_per_level >= bits);
1.2503 +
1.2504 + ret = (malloc_rtree_t*)base_calloc(1, sizeof(malloc_rtree_t) +
1.2505 + (sizeof(unsigned) * (height - 1)));
1.2506 + if (ret == NULL)
1.2507 + return (NULL);
1.2508 +
1.2509 + malloc_spin_init(&ret->lock);
1.2510 + ret->height = height;
1.2511 + if (bits_per_level * height > bits)
1.2512 + ret->level2bits[0] = bits % bits_per_level;
1.2513 + else
1.2514 + ret->level2bits[0] = bits_per_level;
1.2515 + for (i = 1; i < height; i++)
1.2516 + ret->level2bits[i] = bits_per_level;
1.2517 +
1.2518 + ret->root = (void**)base_calloc(1, sizeof(void *) << ret->level2bits[0]);
1.2519 + if (ret->root == NULL) {
1.2520 + /*
1.2521 + * We leak the rtree here, since there's no generic base
1.2522 + * deallocation.
1.2523 + */
1.2524 + return (NULL);
1.2525 + }
1.2526 +
1.2527 + return (ret);
1.2528 +}
1.2529 +
1.2530 +#define MALLOC_RTREE_GET_GENERATE(f) \
1.2531 +/* The least significant bits of the key are ignored. */ \
1.2532 +static inline void * \
1.2533 +f(malloc_rtree_t *rtree, uintptr_t key) \
1.2534 +{ \
1.2535 + void *ret; \
1.2536 + uintptr_t subkey; \
1.2537 + unsigned i, lshift, height, bits; \
1.2538 + void **node, **child; \
1.2539 + \
1.2540 + MALLOC_RTREE_LOCK(&rtree->lock); \
1.2541 + for (i = lshift = 0, height = rtree->height, node = rtree->root;\
1.2542 + i < height - 1; \
1.2543 + i++, lshift += bits, node = child) { \
1.2544 + bits = rtree->level2bits[i]; \
1.2545 + subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits); \
1.2546 + child = (void**)node[subkey]; \
1.2547 + if (child == NULL) { \
1.2548 + MALLOC_RTREE_UNLOCK(&rtree->lock); \
1.2549 + return (NULL); \
1.2550 + } \
1.2551 + } \
1.2552 + \
1.2553 + /* \
1.2554 + * node is a leaf, so it contains values rather than node \
1.2555 + * pointers. \
1.2556 + */ \
1.2557 + bits = rtree->level2bits[i]; \
1.2558 + subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits); \
1.2559 + ret = node[subkey]; \
1.2560 + MALLOC_RTREE_UNLOCK(&rtree->lock); \
1.2561 + \
1.2562 + MALLOC_RTREE_GET_VALIDATE \
1.2563 + return (ret); \
1.2564 +}
1.2565 +
1.2566 +#ifdef MALLOC_DEBUG
1.2567 +# define MALLOC_RTREE_LOCK(l) malloc_spin_lock(l)
1.2568 +# define MALLOC_RTREE_UNLOCK(l) malloc_spin_unlock(l)
1.2569 +# define MALLOC_RTREE_GET_VALIDATE
1.2570 +MALLOC_RTREE_GET_GENERATE(malloc_rtree_get_locked)
1.2571 +# undef MALLOC_RTREE_LOCK
1.2572 +# undef MALLOC_RTREE_UNLOCK
1.2573 +# undef MALLOC_RTREE_GET_VALIDATE
1.2574 +#endif
1.2575 +
1.2576 +#define MALLOC_RTREE_LOCK(l)
1.2577 +#define MALLOC_RTREE_UNLOCK(l)
1.2578 +#ifdef MALLOC_DEBUG
1.2579 + /*
1.2580 + * Suppose that it were possible for a jemalloc-allocated chunk to be
1.2581 + * munmap()ped, followed by a different allocator in another thread re-using
1.2582 + * overlapping virtual memory, all without invalidating the cached rtree
1.2583 + * value. The result would be a false positive (the rtree would claim that
1.2584 + * jemalloc owns memory that it had actually discarded). I don't think this
1.2585 + * scenario is possible, but the following assertion is a prudent sanity
1.2586 + * check.
1.2587 + */
1.2588 +# define MALLOC_RTREE_GET_VALIDATE \
1.2589 + assert(malloc_rtree_get_locked(rtree, key) == ret);
1.2590 +#else
1.2591 +# define MALLOC_RTREE_GET_VALIDATE
1.2592 +#endif
1.2593 +MALLOC_RTREE_GET_GENERATE(malloc_rtree_get)
1.2594 +#undef MALLOC_RTREE_LOCK
1.2595 +#undef MALLOC_RTREE_UNLOCK
1.2596 +#undef MALLOC_RTREE_GET_VALIDATE
1.2597 +
1.2598 +static inline bool
1.2599 +malloc_rtree_set(malloc_rtree_t *rtree, uintptr_t key, void *val)
1.2600 +{
1.2601 + uintptr_t subkey;
1.2602 + unsigned i, lshift, height, bits;
1.2603 + void **node, **child;
1.2604 +
1.2605 + malloc_spin_lock(&rtree->lock);
1.2606 + for (i = lshift = 0, height = rtree->height, node = rtree->root;
1.2607 + i < height - 1;
1.2608 + i++, lshift += bits, node = child) {
1.2609 + bits = rtree->level2bits[i];
1.2610 + subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
1.2611 + child = (void**)node[subkey];
1.2612 + if (child == NULL) {
1.2613 + child = (void**)base_calloc(1, sizeof(void *) <<
1.2614 + rtree->level2bits[i+1]);
1.2615 + if (child == NULL) {
1.2616 + malloc_spin_unlock(&rtree->lock);
1.2617 + return (true);
1.2618 + }
1.2619 + node[subkey] = child;
1.2620 + }
1.2621 + }
1.2622 +
1.2623 + /* node is a leaf, so it contains values rather than node pointers. */
1.2624 + bits = rtree->level2bits[i];
1.2625 + subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
1.2626 + node[subkey] = val;
1.2627 + malloc_spin_unlock(&rtree->lock);
1.2628 +
1.2629 + return (false);
1.2630 +}
1.2631 +#endif
1.2632 +
1.2633 +#if defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)
1.2634 +
1.2635 +/* Allocate an aligned chunk while maintaining a 1:1 correspondence between
1.2636 + * mmap and unmap calls. This is important on Windows, but not elsewhere. */
1.2637 +static void *
1.2638 +chunk_alloc_mmap(size_t size, bool pagefile)
1.2639 +{
1.2640 + void *ret;
1.2641 +#ifndef JEMALLOC_USES_MAP_ALIGN
1.2642 + size_t offset;
1.2643 +#endif
1.2644 + int pfd;
1.2645 +
1.2646 +#ifdef MALLOC_PAGEFILE
1.2647 + if (opt_pagefile && pagefile) {
1.2648 + pfd = pagefile_init(size);
1.2649 + if (pfd == -1)
1.2650 + return (NULL);
1.2651 + } else
1.2652 +#endif
1.2653 + pfd = -1;
1.2654 +
1.2655 +#ifdef JEMALLOC_USES_MAP_ALIGN
1.2656 + ret = pages_map_align(size, pfd, chunksize);
1.2657 +#else
1.2658 + ret = pages_map(NULL, size, pfd);
1.2659 + if (ret == NULL)
1.2660 + goto RETURN;
1.2661 +
1.2662 + offset = CHUNK_ADDR2OFFSET(ret);
1.2663 + if (offset != 0) {
1.2664 + /* Deallocate, then try to allocate at (ret + size - offset). */
1.2665 + pages_unmap(ret, size);
1.2666 + ret = pages_map((void *)((uintptr_t)ret + size - offset), size,
1.2667 + pfd);
1.2668 + while (ret == NULL) {
1.2669 + /*
1.2670 + * Over-allocate in order to map a memory region that
1.2671 + * is definitely large enough.
1.2672 + */
1.2673 + ret = pages_map(NULL, size + chunksize, -1);
1.2674 + if (ret == NULL)
1.2675 + goto RETURN;
1.2676 + /*
1.2677 + * Deallocate, then allocate the correct size, within
1.2678 + * the over-sized mapping.
1.2679 + */
1.2680 + offset = CHUNK_ADDR2OFFSET(ret);
1.2681 + pages_unmap(ret, size + chunksize);
1.2682 + if (offset == 0)
1.2683 + ret = pages_map(ret, size, pfd);
1.2684 + else {
1.2685 + ret = pages_map((void *)((uintptr_t)ret +
1.2686 + chunksize - offset), size, pfd);
1.2687 + }
1.2688 + /*
1.2689 + * Failure here indicates a race with another thread, so
1.2690 + * try again.
1.2691 + */
1.2692 + }
1.2693 + }
1.2694 +RETURN:
1.2695 +#endif
1.2696 +#ifdef MALLOC_PAGEFILE
1.2697 + if (pfd != -1)
1.2698 + pagefile_close(pfd);
1.2699 +#endif
1.2700 +
1.2701 + return (ret);
1.2702 +}
1.2703 +
1.2704 +#else /* ! (defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)) */
1.2705 +
1.2706 +/* pages_trim, chunk_alloc_mmap_slow and chunk_alloc_mmap were cherry-picked
1.2707 + * from upstream jemalloc 3.4.1 to fix Mozilla bug 956501. */
1.2708 +
1.2709 +/* Return the offset between a and the nearest aligned address at or below a. */
1.2710 +#define ALIGNMENT_ADDR2OFFSET(a, alignment) \
1.2711 + ((size_t)((uintptr_t)(a) & (alignment - 1)))
1.2712 +
1.2713 +/* Return the smallest alignment multiple that is >= s. */
1.2714 +#define ALIGNMENT_CEILING(s, alignment) \
1.2715 + (((s) + (alignment - 1)) & (-(alignment)))
1.2716 +
1.2717 +static void *
1.2718 +pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
1.2719 +{
1.2720 + size_t trailsize;
1.2721 + void *ret = (void *)((uintptr_t)addr + leadsize);
1.2722 +
1.2723 + assert(alloc_size >= leadsize + size);
1.2724 + trailsize = alloc_size - leadsize - size;
1.2725 +
1.2726 + if (leadsize != 0)
1.2727 + pages_unmap(addr, leadsize);
1.2728 + if (trailsize != 0)
1.2729 + pages_unmap((void *)((uintptr_t)ret + size), trailsize);
1.2730 + return (ret);
1.2731 +}
1.2732 +
1.2733 +static void *
1.2734 +chunk_alloc_mmap_slow(size_t size, size_t alignment)
1.2735 +{
1.2736 + void *ret, *pages;
1.2737 + size_t alloc_size, leadsize;
1.2738 +
1.2739 + alloc_size = size + alignment - pagesize;
1.2740 + /* Beware size_t wrap-around. */
1.2741 + if (alloc_size < size)
1.2742 + return (NULL);
1.2743 + do {
1.2744 + pages = pages_map(NULL, alloc_size, -1);
1.2745 + if (pages == NULL)
1.2746 + return (NULL);
1.2747 + leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
1.2748 + (uintptr_t)pages;
1.2749 + ret = pages_trim(pages, alloc_size, leadsize, size);
1.2750 + } while (ret == NULL);
1.2751 +
1.2752 + assert(ret != NULL);
1.2753 + return (ret);
1.2754 +}
1.2755 +
1.2756 +static void *
1.2757 +chunk_alloc_mmap(size_t size, bool pagefile)
1.2758 +{
1.2759 + void *ret;
1.2760 + size_t offset;
1.2761 +
1.2762 + /*
1.2763 + * Ideally, there would be a way to specify alignment to mmap() (like
1.2764 + * NetBSD has), but in the absence of such a feature, we have to work
1.2765 + * hard to efficiently create aligned mappings. The reliable, but
1.2766 + * slow method is to create a mapping that is over-sized, then trim the
1.2767 + * excess. However, that always results in one or two calls to
1.2768 + * pages_unmap().
1.2769 + *
1.2770 + * Optimistically try mapping precisely the right amount before falling
1.2771 + * back to the slow method, with the expectation that the optimistic
1.2772 + * approach works most of the time.
1.2773 + */
1.2774 +
1.2775 + ret = pages_map(NULL, size, -1);
1.2776 + if (ret == NULL)
1.2777 + return (NULL);
1.2778 + offset = ALIGNMENT_ADDR2OFFSET(ret, chunksize);
1.2779 + if (offset != 0) {
1.2780 + pages_unmap(ret, size);
1.2781 + return (chunk_alloc_mmap_slow(size, chunksize));
1.2782 + }
1.2783 +
1.2784 + assert(ret != NULL);
1.2785 + return (ret);
1.2786 +}
1.2787 +
1.2788 +#endif /* defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE) */
1.2789 +
1.2790 +#ifdef MALLOC_PAGEFILE
1.2791 +static int
1.2792 +pagefile_init(size_t size)
1.2793 +{
1.2794 + int ret;
1.2795 + size_t i;
1.2796 + char pagefile_path[PATH_MAX];
1.2797 + char zbuf[MALLOC_PAGEFILE_WRITE_SIZE];
1.2798 +
1.2799 + /*
1.2800 + * Create a temporary file, then immediately unlink it so that it will
1.2801 + * not persist.
1.2802 + */
1.2803 + strcpy(pagefile_path, pagefile_templ);
1.2804 + ret = mkstemp(pagefile_path);
1.2805 + if (ret == -1)
1.2806 + return (ret);
1.2807 + if (unlink(pagefile_path)) {
1.2808 + char buf[STRERROR_BUF];
1.2809 +
1.2810 + strerror_r(errno, buf, sizeof(buf));
1.2811 + _malloc_message(_getprogname(), ": (malloc) Error in unlink(\"",
1.2812 + pagefile_path, "\"):");
1.2813 + _malloc_message(buf, "\n", "", "");
1.2814 + if (opt_abort)
1.2815 + abort();
1.2816 + }
1.2817 +
1.2818 + /*
1.2819 + * Write sequential zeroes to the file in order to assure that disk
1.2820 + * space is committed, with minimal fragmentation. It would be
1.2821 + * sufficient to write one zero per disk block, but that potentially
1.2822 + * results in more system calls, for no real gain.
1.2823 + */
1.2824 + memset(zbuf, 0, sizeof(zbuf));
1.2825 + for (i = 0; i < size; i += sizeof(zbuf)) {
1.2826 + if (write(ret, zbuf, sizeof(zbuf)) != sizeof(zbuf)) {
1.2827 + if (errno != ENOSPC) {
1.2828 + char buf[STRERROR_BUF];
1.2829 +
1.2830 + strerror_r(errno, buf, sizeof(buf));
1.2831 + _malloc_message(_getprogname(),
1.2832 + ": (malloc) Error in write(): ", buf, "\n");
1.2833 + if (opt_abort)
1.2834 + abort();
1.2835 + }
1.2836 + pagefile_close(ret);
1.2837 + return (-1);
1.2838 + }
1.2839 + }
1.2840 +
1.2841 + return (ret);
1.2842 +}
1.2843 +
1.2844 +static void
1.2845 +pagefile_close(int pfd)
1.2846 +{
1.2847 +
1.2848 + if (close(pfd)) {
1.2849 + char buf[STRERROR_BUF];
1.2850 +
1.2851 + strerror_r(errno, buf, sizeof(buf));
1.2852 + _malloc_message(_getprogname(),
1.2853 + ": (malloc) Error in close(): ", buf, "\n");
1.2854 + if (opt_abort)
1.2855 + abort();
1.2856 + }
1.2857 +}
1.2858 +#endif
1.2859 +
1.2860 +static void *
1.2861 +chunk_alloc(size_t size, bool zero, bool pagefile)
1.2862 +{
1.2863 + void *ret;
1.2864 +
1.2865 + assert(size != 0);
1.2866 + assert((size & chunksize_mask) == 0);
1.2867 +
1.2868 + ret = chunk_alloc_mmap(size, pagefile);
1.2869 + if (ret != NULL) {
1.2870 + goto RETURN;
1.2871 + }
1.2872 +
1.2873 + /* All strategies for allocation failed. */
1.2874 + ret = NULL;
1.2875 +RETURN:
1.2876 +
1.2877 +#ifdef MALLOC_VALIDATE
1.2878 + if (ret != NULL) {
1.2879 + if (malloc_rtree_set(chunk_rtree, (uintptr_t)ret, ret)) {
1.2880 + chunk_dealloc(ret, size);
1.2881 + return (NULL);
1.2882 + }
1.2883 + }
1.2884 +#endif
1.2885 +
1.2886 + assert(CHUNK_ADDR2BASE(ret) == ret);
1.2887 + return (ret);
1.2888 +}
1.2889 +
1.2890 +static void
1.2891 +chunk_dealloc_mmap(void *chunk, size_t size)
1.2892 +{
1.2893 +
1.2894 + pages_unmap(chunk, size);
1.2895 +}
1.2896 +
1.2897 +static void
1.2898 +chunk_dealloc(void *chunk, size_t size)
1.2899 +{
1.2900 +
1.2901 + assert(chunk != NULL);
1.2902 + assert(CHUNK_ADDR2BASE(chunk) == chunk);
1.2903 + assert(size != 0);
1.2904 + assert((size & chunksize_mask) == 0);
1.2905 +
1.2906 +#ifdef MALLOC_VALIDATE
1.2907 + malloc_rtree_set(chunk_rtree, (uintptr_t)chunk, NULL);
1.2908 +#endif
1.2909 +
1.2910 + chunk_dealloc_mmap(chunk, size);
1.2911 +}
1.2912 +
1.2913 +/*
1.2914 + * End chunk management functions.
1.2915 + */
1.2916 +/******************************************************************************/
1.2917 +/*
1.2918 + * Begin arena.
1.2919 + */
1.2920 +
1.2921 +/*
1.2922 + * Choose an arena based on a per-thread value (fast-path code, calls slow-path
1.2923 + * code if necessary).
1.2924 + */
1.2925 +static inline arena_t *
1.2926 +choose_arena(void)
1.2927 +{
1.2928 + arena_t *ret;
1.2929 +
1.2930 + /*
1.2931 + * We can only use TLS if this is a PIC library, since for the static
1.2932 + * library version, libc's malloc is used by TLS allocation, which
1.2933 + * introduces a bootstrapping issue.
1.2934 + */
1.2935 +#ifndef NO_TLS
1.2936 + if (isthreaded == false) {
1.2937 + /* Avoid the overhead of TLS for single-threaded operation. */
1.2938 + return (arenas[0]);
1.2939 + }
1.2940 +
1.2941 +# ifdef MOZ_MEMORY_WINDOWS
1.2942 + ret = (arena_t*)TlsGetValue(tlsIndex);
1.2943 +# else
1.2944 + ret = arenas_map;
1.2945 +# endif
1.2946 +
1.2947 + if (ret == NULL) {
1.2948 + ret = choose_arena_hard();
1.2949 + RELEASE_ASSERT(ret != NULL);
1.2950 + }
1.2951 +#else
1.2952 + if (isthreaded && narenas > 1) {
1.2953 + unsigned long ind;
1.2954 +
1.2955 + /*
1.2956 + * Hash _pthread_self() to one of the arenas. There is a prime
1.2957 + * number of arenas, so this has a reasonable chance of
1.2958 + * working. Even so, the hashing can be easily thwarted by
1.2959 + * inconvenient _pthread_self() values. Without specific
1.2960 + * knowledge of how _pthread_self() calculates values, we can't
1.2961 + * easily do much better than this.
1.2962 + */
1.2963 + ind = (unsigned long) _pthread_self() % narenas;
1.2964 +
1.2965 + /*
1.2966 + * Optimistially assume that arenas[ind] has been initialized.
1.2967 + * At worst, we find out that some other thread has already
1.2968 + * done so, after acquiring the lock in preparation. Note that
1.2969 + * this lazy locking also has the effect of lazily forcing
1.2970 + * cache coherency; without the lock acquisition, there's no
1.2971 + * guarantee that modification of arenas[ind] by another thread
1.2972 + * would be seen on this CPU for an arbitrary amount of time.
1.2973 + *
1.2974 + * In general, this approach to modifying a synchronized value
1.2975 + * isn't a good idea, but in this case we only ever modify the
1.2976 + * value once, so things work out well.
1.2977 + */
1.2978 + ret = arenas[ind];
1.2979 + if (ret == NULL) {
1.2980 + /*
1.2981 + * Avoid races with another thread that may have already
1.2982 + * initialized arenas[ind].
1.2983 + */
1.2984 + malloc_spin_lock(&arenas_lock);
1.2985 + if (arenas[ind] == NULL)
1.2986 + ret = arenas_extend((unsigned)ind);
1.2987 + else
1.2988 + ret = arenas[ind];
1.2989 + malloc_spin_unlock(&arenas_lock);
1.2990 + }
1.2991 + } else
1.2992 + ret = arenas[0];
1.2993 +#endif
1.2994 +
1.2995 + RELEASE_ASSERT(ret != NULL);
1.2996 + return (ret);
1.2997 +}
1.2998 +
1.2999 +#ifndef NO_TLS
1.3000 +/*
1.3001 + * Choose an arena based on a per-thread value (slow-path code only, called
1.3002 + * only by choose_arena()).
1.3003 + */
1.3004 +static arena_t *
1.3005 +choose_arena_hard(void)
1.3006 +{
1.3007 + arena_t *ret;
1.3008 +
1.3009 + assert(isthreaded);
1.3010 +
1.3011 +#ifdef MALLOC_BALANCE
1.3012 + /* Seed the PRNG used for arena load balancing. */
1.3013 + SPRN(balance, (uint32_t)(uintptr_t)(_pthread_self()));
1.3014 +#endif
1.3015 +
1.3016 + if (narenas > 1) {
1.3017 +#ifdef MALLOC_BALANCE
1.3018 + unsigned ind;
1.3019 +
1.3020 + ind = PRN(balance, narenas_2pow);
1.3021 + if ((ret = arenas[ind]) == NULL) {
1.3022 + malloc_spin_lock(&arenas_lock);
1.3023 + if ((ret = arenas[ind]) == NULL)
1.3024 + ret = arenas_extend(ind);
1.3025 + malloc_spin_unlock(&arenas_lock);
1.3026 + }
1.3027 +#else
1.3028 + malloc_spin_lock(&arenas_lock);
1.3029 + if ((ret = arenas[next_arena]) == NULL)
1.3030 + ret = arenas_extend(next_arena);
1.3031 + next_arena = (next_arena + 1) % narenas;
1.3032 + malloc_spin_unlock(&arenas_lock);
1.3033 +#endif
1.3034 + } else
1.3035 + ret = arenas[0];
1.3036 +
1.3037 +#ifdef MOZ_MEMORY_WINDOWS
1.3038 + TlsSetValue(tlsIndex, ret);
1.3039 +#else
1.3040 + arenas_map = ret;
1.3041 +#endif
1.3042 +
1.3043 + return (ret);
1.3044 +}
1.3045 +#endif
1.3046 +
1.3047 +static inline int
1.3048 +arena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b)
1.3049 +{
1.3050 + uintptr_t a_chunk = (uintptr_t)a;
1.3051 + uintptr_t b_chunk = (uintptr_t)b;
1.3052 +
1.3053 + assert(a != NULL);
1.3054 + assert(b != NULL);
1.3055 +
1.3056 + return ((a_chunk > b_chunk) - (a_chunk < b_chunk));
1.3057 +}
1.3058 +
1.3059 +/* Wrap red-black tree macros in functions. */
1.3060 +rb_wrap(static, arena_chunk_tree_dirty_, arena_chunk_tree_t,
1.3061 + arena_chunk_t, link_dirty, arena_chunk_comp)
1.3062 +
1.3063 +static inline int
1.3064 +arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
1.3065 +{
1.3066 + uintptr_t a_mapelm = (uintptr_t)a;
1.3067 + uintptr_t b_mapelm = (uintptr_t)b;
1.3068 +
1.3069 + assert(a != NULL);
1.3070 + assert(b != NULL);
1.3071 +
1.3072 + return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm));
1.3073 +}
1.3074 +
1.3075 +/* Wrap red-black tree macros in functions. */
1.3076 +rb_wrap(static, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, link,
1.3077 + arena_run_comp)
1.3078 +
1.3079 +static inline int
1.3080 +arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
1.3081 +{
1.3082 + int ret;
1.3083 + size_t a_size = a->bits & ~pagesize_mask;
1.3084 + size_t b_size = b->bits & ~pagesize_mask;
1.3085 +
1.3086 + ret = (a_size > b_size) - (a_size < b_size);
1.3087 + if (ret == 0) {
1.3088 + uintptr_t a_mapelm, b_mapelm;
1.3089 +
1.3090 + if ((a->bits & CHUNK_MAP_KEY) == 0)
1.3091 + a_mapelm = (uintptr_t)a;
1.3092 + else {
1.3093 + /*
1.3094 + * Treat keys as though they are lower than anything
1.3095 + * else.
1.3096 + */
1.3097 + a_mapelm = 0;
1.3098 + }
1.3099 + b_mapelm = (uintptr_t)b;
1.3100 +
1.3101 + ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm);
1.3102 + }
1.3103 +
1.3104 + return (ret);
1.3105 +}
1.3106 +
1.3107 +/* Wrap red-black tree macros in functions. */
1.3108 +rb_wrap(static, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, link,
1.3109 + arena_avail_comp)
1.3110 +
1.3111 +static inline void *
1.3112 +arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
1.3113 +{
1.3114 + void *ret;
1.3115 + unsigned i, mask, bit, regind;
1.3116 +
1.3117 + assert(run->magic == ARENA_RUN_MAGIC);
1.3118 + assert(run->regs_minelm < bin->regs_mask_nelms);
1.3119 +
1.3120 + /*
1.3121 + * Move the first check outside the loop, so that run->regs_minelm can
1.3122 + * be updated unconditionally, without the possibility of updating it
1.3123 + * multiple times.
1.3124 + */
1.3125 + i = run->regs_minelm;
1.3126 + mask = run->regs_mask[i];
1.3127 + if (mask != 0) {
1.3128 + /* Usable allocation found. */
1.3129 + bit = ffs((int)mask) - 1;
1.3130 +
1.3131 + regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
1.3132 + assert(regind < bin->nregs);
1.3133 + ret = (void *)(((uintptr_t)run) + bin->reg0_offset
1.3134 + + (bin->reg_size * regind));
1.3135 +
1.3136 + /* Clear bit. */
1.3137 + mask ^= (1U << bit);
1.3138 + run->regs_mask[i] = mask;
1.3139 +
1.3140 + return (ret);
1.3141 + }
1.3142 +
1.3143 + for (i++; i < bin->regs_mask_nelms; i++) {
1.3144 + mask = run->regs_mask[i];
1.3145 + if (mask != 0) {
1.3146 + /* Usable allocation found. */
1.3147 + bit = ffs((int)mask) - 1;
1.3148 +
1.3149 + regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
1.3150 + assert(regind < bin->nregs);
1.3151 + ret = (void *)(((uintptr_t)run) + bin->reg0_offset
1.3152 + + (bin->reg_size * regind));
1.3153 +
1.3154 + /* Clear bit. */
1.3155 + mask ^= (1U << bit);
1.3156 + run->regs_mask[i] = mask;
1.3157 +
1.3158 + /*
1.3159 + * Make a note that nothing before this element
1.3160 + * contains a free region.
1.3161 + */
1.3162 + run->regs_minelm = i; /* Low payoff: + (mask == 0); */
1.3163 +
1.3164 + return (ret);
1.3165 + }
1.3166 + }
1.3167 + /* Not reached. */
1.3168 + RELEASE_ASSERT(0);
1.3169 + return (NULL);
1.3170 +}
1.3171 +
1.3172 +static inline void
1.3173 +arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size)
1.3174 +{
1.3175 + /*
1.3176 + * To divide by a number D that is not a power of two we multiply
1.3177 + * by (2^21 / D) and then right shift by 21 positions.
1.3178 + *
1.3179 + * X / D
1.3180 + *
1.3181 + * becomes
1.3182 + *
1.3183 + * (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT
1.3184 + */
1.3185 +#define SIZE_INV_SHIFT 21
1.3186 +#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s << QUANTUM_2POW_MIN)) + 1)
1.3187 + static const unsigned size_invs[] = {
1.3188 + SIZE_INV(3),
1.3189 + SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
1.3190 + SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
1.3191 + SIZE_INV(12),SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
1.3192 + SIZE_INV(16),SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
1.3193 + SIZE_INV(20),SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
1.3194 + SIZE_INV(24),SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
1.3195 + SIZE_INV(28),SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
1.3196 +#if (QUANTUM_2POW_MIN < 4)
1.3197 + ,
1.3198 + SIZE_INV(32), SIZE_INV(33), SIZE_INV(34), SIZE_INV(35),
1.3199 + SIZE_INV(36), SIZE_INV(37), SIZE_INV(38), SIZE_INV(39),
1.3200 + SIZE_INV(40), SIZE_INV(41), SIZE_INV(42), SIZE_INV(43),
1.3201 + SIZE_INV(44), SIZE_INV(45), SIZE_INV(46), SIZE_INV(47),
1.3202 + SIZE_INV(48), SIZE_INV(49), SIZE_INV(50), SIZE_INV(51),
1.3203 + SIZE_INV(52), SIZE_INV(53), SIZE_INV(54), SIZE_INV(55),
1.3204 + SIZE_INV(56), SIZE_INV(57), SIZE_INV(58), SIZE_INV(59),
1.3205 + SIZE_INV(60), SIZE_INV(61), SIZE_INV(62), SIZE_INV(63)
1.3206 +#endif
1.3207 + };
1.3208 + unsigned diff, regind, elm, bit;
1.3209 +
1.3210 + assert(run->magic == ARENA_RUN_MAGIC);
1.3211 + assert(((sizeof(size_invs)) / sizeof(unsigned)) + 3
1.3212 + >= (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN));
1.3213 +
1.3214 + /*
1.3215 + * Avoid doing division with a variable divisor if possible. Using
1.3216 + * actual division here can reduce allocator throughput by over 20%!
1.3217 + */
1.3218 + diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset);
1.3219 + if ((size & (size - 1)) == 0) {
1.3220 + /*
1.3221 + * log2_table allows fast division of a power of two in the
1.3222 + * [1..128] range.
1.3223 + *
1.3224 + * (x / divisor) becomes (x >> log2_table[divisor - 1]).
1.3225 + */
1.3226 + static const unsigned char log2_table[] = {
1.3227 + 0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4,
1.3228 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
1.3229 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1.3230 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
1.3231 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1.3232 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1.3233 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1.3234 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
1.3235 + };
1.3236 +
1.3237 + if (size <= 128)
1.3238 + regind = (diff >> log2_table[size - 1]);
1.3239 + else if (size <= 32768)
1.3240 + regind = diff >> (8 + log2_table[(size >> 8) - 1]);
1.3241 + else {
1.3242 + /*
1.3243 + * The run size is too large for us to use the lookup
1.3244 + * table. Use real division.
1.3245 + */
1.3246 + regind = diff / size;
1.3247 + }
1.3248 + } else if (size <= ((sizeof(size_invs) / sizeof(unsigned))
1.3249 + << QUANTUM_2POW_MIN) + 2) {
1.3250 + regind = size_invs[(size >> QUANTUM_2POW_MIN) - 3] * diff;
1.3251 + regind >>= SIZE_INV_SHIFT;
1.3252 + } else {
1.3253 + /*
1.3254 + * size_invs isn't large enough to handle this size class, so
1.3255 + * calculate regind using actual division. This only happens
1.3256 + * if the user increases small_max via the 'S' runtime
1.3257 + * configuration option.
1.3258 + */
1.3259 + regind = diff / size;
1.3260 + };
1.3261 + RELEASE_ASSERT(diff == regind * size);
1.3262 + RELEASE_ASSERT(regind < bin->nregs);
1.3263 +
1.3264 + elm = regind >> (SIZEOF_INT_2POW + 3);
1.3265 + if (elm < run->regs_minelm)
1.3266 + run->regs_minelm = elm;
1.3267 + bit = regind - (elm << (SIZEOF_INT_2POW + 3));
1.3268 + RELEASE_ASSERT((run->regs_mask[elm] & (1U << bit)) == 0);
1.3269 + run->regs_mask[elm] |= (1U << bit);
1.3270 +#undef SIZE_INV
1.3271 +#undef SIZE_INV_SHIFT
1.3272 +}
1.3273 +
1.3274 +static void
1.3275 +arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
1.3276 + bool zero)
1.3277 +{
1.3278 + arena_chunk_t *chunk;
1.3279 + size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i;
1.3280 +
1.3281 + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
1.3282 + old_ndirty = chunk->ndirty;
1.3283 + run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk)
1.3284 + >> pagesize_2pow);
1.3285 + total_pages = (chunk->map[run_ind].bits & ~pagesize_mask) >>
1.3286 + pagesize_2pow;
1.3287 + need_pages = (size >> pagesize_2pow);
1.3288 + assert(need_pages > 0);
1.3289 + assert(need_pages <= total_pages);
1.3290 + rem_pages = total_pages - need_pages;
1.3291 +
1.3292 + arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]);
1.3293 +
1.3294 + /* Keep track of trailing unused pages for later use. */
1.3295 + if (rem_pages > 0) {
1.3296 + chunk->map[run_ind+need_pages].bits = (rem_pages <<
1.3297 + pagesize_2pow) | (chunk->map[run_ind+need_pages].bits &
1.3298 + pagesize_mask);
1.3299 + chunk->map[run_ind+total_pages-1].bits = (rem_pages <<
1.3300 + pagesize_2pow) | (chunk->map[run_ind+total_pages-1].bits &
1.3301 + pagesize_mask);
1.3302 + arena_avail_tree_insert(&arena->runs_avail,
1.3303 + &chunk->map[run_ind+need_pages]);
1.3304 + }
1.3305 +
1.3306 + for (i = 0; i < need_pages; i++) {
1.3307 +#if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
1.3308 + /*
1.3309 + * Commit decommitted pages if necessary. If a decommitted
1.3310 + * page is encountered, commit all needed adjacent decommitted
1.3311 + * pages in one operation, in order to reduce system call
1.3312 + * overhead.
1.3313 + */
1.3314 + if (chunk->map[run_ind + i].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED) {
1.3315 + size_t j;
1.3316 +
1.3317 + /*
1.3318 + * Advance i+j to just past the index of the last page
1.3319 + * to commit. Clear CHUNK_MAP_DECOMMITTED and
1.3320 + * CHUNK_MAP_MADVISED along the way.
1.3321 + */
1.3322 + for (j = 0; i + j < need_pages && (chunk->map[run_ind +
1.3323 + i + j].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED); j++) {
1.3324 + /* DECOMMITTED and MADVISED are mutually exclusive. */
1.3325 + assert(!(chunk->map[run_ind + i + j].bits & CHUNK_MAP_DECOMMITTED &&
1.3326 + chunk->map[run_ind + i + j].bits & CHUNK_MAP_MADVISED));
1.3327 +
1.3328 + chunk->map[run_ind + i + j].bits &=
1.3329 + ~CHUNK_MAP_MADVISED_OR_DECOMMITTED;
1.3330 + }
1.3331 +
1.3332 +# ifdef MALLOC_DECOMMIT
1.3333 + pages_commit((void *)((uintptr_t)chunk + ((run_ind + i)
1.3334 + << pagesize_2pow)), (j << pagesize_2pow));
1.3335 +# ifdef MALLOC_STATS
1.3336 + arena->stats.ncommit++;
1.3337 +# endif
1.3338 +# endif
1.3339 +
1.3340 +# ifdef MALLOC_STATS
1.3341 + arena->stats.committed += j;
1.3342 +# endif
1.3343 +
1.3344 +# ifndef MALLOC_DECOMMIT
1.3345 + }
1.3346 +# else
1.3347 + } else /* No need to zero since commit zeros. */
1.3348 +# endif
1.3349 +
1.3350 +#endif
1.3351 +
1.3352 + /* Zero if necessary. */
1.3353 + if (zero) {
1.3354 + if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED)
1.3355 + == 0) {
1.3356 + VALGRIND_MALLOCLIKE_BLOCK((void *)((uintptr_t)
1.3357 + chunk + ((run_ind + i) << pagesize_2pow)),
1.3358 + pagesize, 0, false);
1.3359 + memset((void *)((uintptr_t)chunk + ((run_ind
1.3360 + + i) << pagesize_2pow)), 0, pagesize);
1.3361 + VALGRIND_FREELIKE_BLOCK((void *)((uintptr_t)
1.3362 + chunk + ((run_ind + i) << pagesize_2pow)),
1.3363 + 0);
1.3364 + /* CHUNK_MAP_ZEROED is cleared below. */
1.3365 + }
1.3366 + }
1.3367 +
1.3368 + /* Update dirty page accounting. */
1.3369 + if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) {
1.3370 + chunk->ndirty--;
1.3371 + arena->ndirty--;
1.3372 + /* CHUNK_MAP_DIRTY is cleared below. */
1.3373 + }
1.3374 +
1.3375 + /* Initialize the chunk map. */
1.3376 + if (large) {
1.3377 + chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE
1.3378 + | CHUNK_MAP_ALLOCATED;
1.3379 + } else {
1.3380 + chunk->map[run_ind + i].bits = (size_t)run
1.3381 + | CHUNK_MAP_ALLOCATED;
1.3382 + }
1.3383 + }
1.3384 +
1.3385 + /*
1.3386 + * Set the run size only in the first element for large runs. This is
1.3387 + * primarily a debugging aid, since the lack of size info for trailing
1.3388 + * pages only matters if the application tries to operate on an
1.3389 + * interior pointer.
1.3390 + */
1.3391 + if (large)
1.3392 + chunk->map[run_ind].bits |= size;
1.3393 +
1.3394 + if (chunk->ndirty == 0 && old_ndirty > 0)
1.3395 + arena_chunk_tree_dirty_remove(&arena->chunks_dirty, chunk);
1.3396 +}
1.3397 +
1.3398 +static void
1.3399 +arena_chunk_init(arena_t *arena, arena_chunk_t *chunk)
1.3400 +{
1.3401 + arena_run_t *run;
1.3402 + size_t i;
1.3403 +
1.3404 + VALGRIND_MALLOCLIKE_BLOCK(chunk, (arena_chunk_header_npages <<
1.3405 + pagesize_2pow), 0, false);
1.3406 +#ifdef MALLOC_STATS
1.3407 + arena->stats.mapped += chunksize;
1.3408 +#endif
1.3409 +
1.3410 + chunk->arena = arena;
1.3411 +
1.3412 + /*
1.3413 + * Claim that no pages are in use, since the header is merely overhead.
1.3414 + */
1.3415 + chunk->ndirty = 0;
1.3416 +
1.3417 + /* Initialize the map to contain one maximal free untouched run. */
1.3418 + run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages <<
1.3419 + pagesize_2pow));
1.3420 + for (i = 0; i < arena_chunk_header_npages; i++)
1.3421 + chunk->map[i].bits = 0;
1.3422 + chunk->map[i].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
1.3423 + for (i++; i < chunk_npages-1; i++) {
1.3424 + chunk->map[i].bits = CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
1.3425 + }
1.3426 + chunk->map[chunk_npages-1].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
1.3427 +
1.3428 +#ifdef MALLOC_DECOMMIT
1.3429 + /*
1.3430 + * Start out decommitted, in order to force a closer correspondence
1.3431 + * between dirty pages and committed untouched pages.
1.3432 + */
1.3433 + pages_decommit(run, arena_maxclass);
1.3434 +# ifdef MALLOC_STATS
1.3435 + arena->stats.ndecommit++;
1.3436 + arena->stats.decommitted += (chunk_npages - arena_chunk_header_npages);
1.3437 +# endif
1.3438 +#endif
1.3439 +#ifdef MALLOC_STATS
1.3440 + arena->stats.committed += arena_chunk_header_npages;
1.3441 +#endif
1.3442 +
1.3443 + /* Insert the run into the runs_avail tree. */
1.3444 + arena_avail_tree_insert(&arena->runs_avail,
1.3445 + &chunk->map[arena_chunk_header_npages]);
1.3446 +
1.3447 +#ifdef MALLOC_DOUBLE_PURGE
1.3448 + LinkedList_Init(&chunk->chunks_madvised_elem);
1.3449 +#endif
1.3450 +}
1.3451 +
1.3452 +static void
1.3453 +arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
1.3454 +{
1.3455 +
1.3456 + if (arena->spare != NULL) {
1.3457 + if (arena->spare->ndirty > 0) {
1.3458 + arena_chunk_tree_dirty_remove(
1.3459 + &chunk->arena->chunks_dirty, arena->spare);
1.3460 + arena->ndirty -= arena->spare->ndirty;
1.3461 +#ifdef MALLOC_STATS
1.3462 + arena->stats.committed -= arena->spare->ndirty;
1.3463 +#endif
1.3464 + }
1.3465 +
1.3466 +#ifdef MALLOC_DOUBLE_PURGE
1.3467 + /* This is safe to do even if arena->spare is not in the list. */
1.3468 + LinkedList_Remove(&arena->spare->chunks_madvised_elem);
1.3469 +#endif
1.3470 +
1.3471 + VALGRIND_FREELIKE_BLOCK(arena->spare, 0);
1.3472 + chunk_dealloc((void *)arena->spare, chunksize);
1.3473 +#ifdef MALLOC_STATS
1.3474 + arena->stats.mapped -= chunksize;
1.3475 + arena->stats.committed -= arena_chunk_header_npages;
1.3476 +#endif
1.3477 + }
1.3478 +
1.3479 + /*
1.3480 + * Remove run from runs_avail, so that the arena does not use it.
1.3481 + * Dirty page flushing only uses the chunks_dirty tree, so leaving this
1.3482 + * chunk in the chunks_* trees is sufficient for that purpose.
1.3483 + */
1.3484 + arena_avail_tree_remove(&arena->runs_avail,
1.3485 + &chunk->map[arena_chunk_header_npages]);
1.3486 +
1.3487 + arena->spare = chunk;
1.3488 +}
1.3489 +
1.3490 +static arena_run_t *
1.3491 +arena_run_alloc(arena_t *arena, arena_bin_t *bin, size_t size, bool large,
1.3492 + bool zero)
1.3493 +{
1.3494 + arena_run_t *run;
1.3495 + arena_chunk_map_t *mapelm, key;
1.3496 +
1.3497 + assert(size <= arena_maxclass);
1.3498 + assert((size & pagesize_mask) == 0);
1.3499 +
1.3500 + /* Search the arena's chunks for the lowest best fit. */
1.3501 + key.bits = size | CHUNK_MAP_KEY;
1.3502 + mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key);
1.3503 + if (mapelm != NULL) {
1.3504 + arena_chunk_t *chunk =
1.3505 + (arena_chunk_t*)CHUNK_ADDR2BASE(mapelm);
1.3506 + size_t pageind = ((uintptr_t)mapelm -
1.3507 + (uintptr_t)chunk->map) /
1.3508 + sizeof(arena_chunk_map_t);
1.3509 +
1.3510 + run = (arena_run_t *)((uintptr_t)chunk + (pageind
1.3511 + << pagesize_2pow));
1.3512 + arena_run_split(arena, run, size, large, zero);
1.3513 + return (run);
1.3514 + }
1.3515 +
1.3516 + if (arena->spare != NULL) {
1.3517 + /* Use the spare. */
1.3518 + arena_chunk_t *chunk = arena->spare;
1.3519 + arena->spare = NULL;
1.3520 + run = (arena_run_t *)((uintptr_t)chunk +
1.3521 + (arena_chunk_header_npages << pagesize_2pow));
1.3522 + /* Insert the run into the runs_avail tree. */
1.3523 + arena_avail_tree_insert(&arena->runs_avail,
1.3524 + &chunk->map[arena_chunk_header_npages]);
1.3525 + arena_run_split(arena, run, size, large, zero);
1.3526 + return (run);
1.3527 + }
1.3528 +
1.3529 + /*
1.3530 + * No usable runs. Create a new chunk from which to allocate
1.3531 + * the run.
1.3532 + */
1.3533 + {
1.3534 + arena_chunk_t *chunk = (arena_chunk_t *)
1.3535 + chunk_alloc(chunksize, true, true);
1.3536 + if (chunk == NULL)
1.3537 + return (NULL);
1.3538 +
1.3539 + arena_chunk_init(arena, chunk);
1.3540 + run = (arena_run_t *)((uintptr_t)chunk +
1.3541 + (arena_chunk_header_npages << pagesize_2pow));
1.3542 + }
1.3543 + /* Update page map. */
1.3544 + arena_run_split(arena, run, size, large, zero);
1.3545 + return (run);
1.3546 +}
1.3547 +
1.3548 +static void
1.3549 +arena_purge(arena_t *arena, bool all)
1.3550 +{
1.3551 + arena_chunk_t *chunk;
1.3552 + size_t i, npages;
1.3553 + /* If all is set purge all dirty pages. */
1.3554 + size_t dirty_max = all ? 1 : opt_dirty_max;
1.3555 +#ifdef MALLOC_DEBUG
1.3556 + size_t ndirty = 0;
1.3557 + rb_foreach_begin(arena_chunk_t, link_dirty, &arena->chunks_dirty,
1.3558 + chunk) {
1.3559 + ndirty += chunk->ndirty;
1.3560 + } rb_foreach_end(arena_chunk_t, link_dirty, &arena->chunks_dirty, chunk)
1.3561 + assert(ndirty == arena->ndirty);
1.3562 +#endif
1.3563 + RELEASE_ASSERT(all || (arena->ndirty > opt_dirty_max));
1.3564 +
1.3565 +#ifdef MALLOC_STATS
1.3566 + arena->stats.npurge++;
1.3567 +#endif
1.3568 +
1.3569 + /*
1.3570 + * Iterate downward through chunks until enough dirty memory has been
1.3571 + * purged. Terminate as soon as possible in order to minimize the
1.3572 + * number of system calls, even if a chunk has only been partially
1.3573 + * purged.
1.3574 + */
1.3575 + while (arena->ndirty > (dirty_max >> 1)) {
1.3576 +#ifdef MALLOC_DOUBLE_PURGE
1.3577 + bool madvised = false;
1.3578 +#endif
1.3579 + chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty);
1.3580 + RELEASE_ASSERT(chunk != NULL);
1.3581 +
1.3582 + for (i = chunk_npages - 1; chunk->ndirty > 0; i--) {
1.3583 + RELEASE_ASSERT(i >= arena_chunk_header_npages);
1.3584 +
1.3585 + if (chunk->map[i].bits & CHUNK_MAP_DIRTY) {
1.3586 +#ifdef MALLOC_DECOMMIT
1.3587 + const size_t free_operation = CHUNK_MAP_DECOMMITTED;
1.3588 +#else
1.3589 + const size_t free_operation = CHUNK_MAP_MADVISED;
1.3590 +#endif
1.3591 + assert((chunk->map[i].bits &
1.3592 + CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
1.3593 + chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
1.3594 + /* Find adjacent dirty run(s). */
1.3595 + for (npages = 1;
1.3596 + i > arena_chunk_header_npages &&
1.3597 + (chunk->map[i - 1].bits & CHUNK_MAP_DIRTY);
1.3598 + npages++) {
1.3599 + i--;
1.3600 + assert((chunk->map[i].bits &
1.3601 + CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
1.3602 + chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
1.3603 + }
1.3604 + chunk->ndirty -= npages;
1.3605 + arena->ndirty -= npages;
1.3606 +
1.3607 +#ifdef MALLOC_DECOMMIT
1.3608 + pages_decommit((void *)((uintptr_t)
1.3609 + chunk + (i << pagesize_2pow)),
1.3610 + (npages << pagesize_2pow));
1.3611 +# ifdef MALLOC_STATS
1.3612 + arena->stats.ndecommit++;
1.3613 + arena->stats.decommitted += npages;
1.3614 +# endif
1.3615 +#endif
1.3616 +#ifdef MALLOC_STATS
1.3617 + arena->stats.committed -= npages;
1.3618 +#endif
1.3619 +
1.3620 +#ifndef MALLOC_DECOMMIT
1.3621 + madvise((void *)((uintptr_t)chunk + (i <<
1.3622 + pagesize_2pow)), (npages << pagesize_2pow),
1.3623 + MADV_FREE);
1.3624 +# ifdef MALLOC_DOUBLE_PURGE
1.3625 + madvised = true;
1.3626 +# endif
1.3627 +#endif
1.3628 +#ifdef MALLOC_STATS
1.3629 + arena->stats.nmadvise++;
1.3630 + arena->stats.purged += npages;
1.3631 +#endif
1.3632 + if (arena->ndirty <= (dirty_max >> 1))
1.3633 + break;
1.3634 + }
1.3635 + }
1.3636 +
1.3637 + if (chunk->ndirty == 0) {
1.3638 + arena_chunk_tree_dirty_remove(&arena->chunks_dirty,
1.3639 + chunk);
1.3640 + }
1.3641 +#ifdef MALLOC_DOUBLE_PURGE
1.3642 + if (madvised) {
1.3643 + /* The chunk might already be in the list, but this
1.3644 + * makes sure it's at the front. */
1.3645 + LinkedList_Remove(&chunk->chunks_madvised_elem);
1.3646 + LinkedList_InsertHead(&arena->chunks_madvised, &chunk->chunks_madvised_elem);
1.3647 + }
1.3648 +#endif
1.3649 + }
1.3650 +}
1.3651 +
1.3652 +static void
1.3653 +arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
1.3654 +{
1.3655 + arena_chunk_t *chunk;
1.3656 + size_t size, run_ind, run_pages;
1.3657 +
1.3658 + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
1.3659 + run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk)
1.3660 + >> pagesize_2pow);
1.3661 + RELEASE_ASSERT(run_ind >= arena_chunk_header_npages);
1.3662 + RELEASE_ASSERT(run_ind < chunk_npages);
1.3663 + if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0)
1.3664 + size = chunk->map[run_ind].bits & ~pagesize_mask;
1.3665 + else
1.3666 + size = run->bin->run_size;
1.3667 + run_pages = (size >> pagesize_2pow);
1.3668 +
1.3669 + /* Mark pages as unallocated in the chunk map. */
1.3670 + if (dirty) {
1.3671 + size_t i;
1.3672 +
1.3673 + for (i = 0; i < run_pages; i++) {
1.3674 + RELEASE_ASSERT((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY)
1.3675 + == 0);
1.3676 + chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY;
1.3677 + }
1.3678 +
1.3679 + if (chunk->ndirty == 0) {
1.3680 + arena_chunk_tree_dirty_insert(&arena->chunks_dirty,
1.3681 + chunk);
1.3682 + }
1.3683 + chunk->ndirty += run_pages;
1.3684 + arena->ndirty += run_pages;
1.3685 + } else {
1.3686 + size_t i;
1.3687 +
1.3688 + for (i = 0; i < run_pages; i++) {
1.3689 + chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE |
1.3690 + CHUNK_MAP_ALLOCATED);
1.3691 + }
1.3692 + }
1.3693 + chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
1.3694 + pagesize_mask);
1.3695 + chunk->map[run_ind+run_pages-1].bits = size |
1.3696 + (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
1.3697 +
1.3698 + /* Try to coalesce forward. */
1.3699 + if (run_ind + run_pages < chunk_npages &&
1.3700 + (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) {
1.3701 + size_t nrun_size = chunk->map[run_ind+run_pages].bits &
1.3702 + ~pagesize_mask;
1.3703 +
1.3704 + /*
1.3705 + * Remove successor from runs_avail; the coalesced run is
1.3706 + * inserted later.
1.3707 + */
1.3708 + arena_avail_tree_remove(&arena->runs_avail,
1.3709 + &chunk->map[run_ind+run_pages]);
1.3710 +
1.3711 + size += nrun_size;
1.3712 + run_pages = size >> pagesize_2pow;
1.3713 +
1.3714 + RELEASE_ASSERT((chunk->map[run_ind+run_pages-1].bits & ~pagesize_mask)
1.3715 + == nrun_size);
1.3716 + chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
1.3717 + pagesize_mask);
1.3718 + chunk->map[run_ind+run_pages-1].bits = size |
1.3719 + (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
1.3720 + }
1.3721 +
1.3722 + /* Try to coalesce backward. */
1.3723 + if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits &
1.3724 + CHUNK_MAP_ALLOCATED) == 0) {
1.3725 + size_t prun_size = chunk->map[run_ind-1].bits & ~pagesize_mask;
1.3726 +
1.3727 + run_ind -= prun_size >> pagesize_2pow;
1.3728 +
1.3729 + /*
1.3730 + * Remove predecessor from runs_avail; the coalesced run is
1.3731 + * inserted later.
1.3732 + */
1.3733 + arena_avail_tree_remove(&arena->runs_avail,
1.3734 + &chunk->map[run_ind]);
1.3735 +
1.3736 + size += prun_size;
1.3737 + run_pages = size >> pagesize_2pow;
1.3738 +
1.3739 + RELEASE_ASSERT((chunk->map[run_ind].bits & ~pagesize_mask) ==
1.3740 + prun_size);
1.3741 + chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
1.3742 + pagesize_mask);
1.3743 + chunk->map[run_ind+run_pages-1].bits = size |
1.3744 + (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
1.3745 + }
1.3746 +
1.3747 + /* Insert into runs_avail, now that coalescing is complete. */
1.3748 + arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]);
1.3749 +
1.3750 + /* Deallocate chunk if it is now completely unused. */
1.3751 + if ((chunk->map[arena_chunk_header_npages].bits & (~pagesize_mask |
1.3752 + CHUNK_MAP_ALLOCATED)) == arena_maxclass)
1.3753 + arena_chunk_dealloc(arena, chunk);
1.3754 +
1.3755 + /* Enforce opt_dirty_max. */
1.3756 + if (arena->ndirty > opt_dirty_max)
1.3757 + arena_purge(arena, false);
1.3758 +}
1.3759 +
1.3760 +static void
1.3761 +arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
1.3762 + size_t oldsize, size_t newsize)
1.3763 +{
1.3764 + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
1.3765 + size_t head_npages = (oldsize - newsize) >> pagesize_2pow;
1.3766 +
1.3767 + assert(oldsize > newsize);
1.3768 +
1.3769 + /*
1.3770 + * Update the chunk map so that arena_run_dalloc() can treat the
1.3771 + * leading run as separately allocated.
1.3772 + */
1.3773 + chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE |
1.3774 + CHUNK_MAP_ALLOCATED;
1.3775 + chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE |
1.3776 + CHUNK_MAP_ALLOCATED;
1.3777 +
1.3778 + arena_run_dalloc(arena, run, false);
1.3779 +}
1.3780 +
1.3781 +static void
1.3782 +arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
1.3783 + size_t oldsize, size_t newsize, bool dirty)
1.3784 +{
1.3785 + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
1.3786 + size_t npages = newsize >> pagesize_2pow;
1.3787 +
1.3788 + assert(oldsize > newsize);
1.3789 +
1.3790 + /*
1.3791 + * Update the chunk map so that arena_run_dalloc() can treat the
1.3792 + * trailing run as separately allocated.
1.3793 + */
1.3794 + chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE |
1.3795 + CHUNK_MAP_ALLOCATED;
1.3796 + chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE
1.3797 + | CHUNK_MAP_ALLOCATED;
1.3798 +
1.3799 + arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
1.3800 + dirty);
1.3801 +}
1.3802 +
1.3803 +static arena_run_t *
1.3804 +arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
1.3805 +{
1.3806 + arena_chunk_map_t *mapelm;
1.3807 + arena_run_t *run;
1.3808 + unsigned i, remainder;
1.3809 +
1.3810 + /* Look for a usable run. */
1.3811 + mapelm = arena_run_tree_first(&bin->runs);
1.3812 + if (mapelm != NULL) {
1.3813 + /* run is guaranteed to have available space. */
1.3814 + arena_run_tree_remove(&bin->runs, mapelm);
1.3815 + run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
1.3816 +#ifdef MALLOC_STATS
1.3817 + bin->stats.reruns++;
1.3818 +#endif
1.3819 + return (run);
1.3820 + }
1.3821 + /* No existing runs have any space available. */
1.3822 +
1.3823 + /* Allocate a new run. */
1.3824 + run = arena_run_alloc(arena, bin, bin->run_size, false, false);
1.3825 + if (run == NULL)
1.3826 + return (NULL);
1.3827 + /*
1.3828 + * Don't initialize if a race in arena_run_alloc() allowed an existing
1.3829 + * run to become usable.
1.3830 + */
1.3831 + if (run == bin->runcur)
1.3832 + return (run);
1.3833 +
1.3834 + VALGRIND_MALLOCLIKE_BLOCK(run, sizeof(arena_run_t) + (sizeof(unsigned) *
1.3835 + (bin->regs_mask_nelms - 1)), 0, false);
1.3836 +
1.3837 + /* Initialize run internals. */
1.3838 + run->bin = bin;
1.3839 +
1.3840 + for (i = 0; i < bin->regs_mask_nelms - 1; i++)
1.3841 + run->regs_mask[i] = UINT_MAX;
1.3842 + remainder = bin->nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1);
1.3843 + if (remainder == 0)
1.3844 + run->regs_mask[i] = UINT_MAX;
1.3845 + else {
1.3846 + /* The last element has spare bits that need to be unset. */
1.3847 + run->regs_mask[i] = (UINT_MAX >> ((1U << (SIZEOF_INT_2POW + 3))
1.3848 + - remainder));
1.3849 + }
1.3850 +
1.3851 + run->regs_minelm = 0;
1.3852 +
1.3853 + run->nfree = bin->nregs;
1.3854 +#if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
1.3855 + run->magic = ARENA_RUN_MAGIC;
1.3856 +#endif
1.3857 +
1.3858 +#ifdef MALLOC_STATS
1.3859 + bin->stats.nruns++;
1.3860 + bin->stats.curruns++;
1.3861 + if (bin->stats.curruns > bin->stats.highruns)
1.3862 + bin->stats.highruns = bin->stats.curruns;
1.3863 +#endif
1.3864 + return (run);
1.3865 +}
1.3866 +
1.3867 +/* bin->runcur must have space available before this function is called. */
1.3868 +static inline void *
1.3869 +arena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run)
1.3870 +{
1.3871 + void *ret;
1.3872 +
1.3873 + RELEASE_ASSERT(run->magic == ARENA_RUN_MAGIC);
1.3874 + RELEASE_ASSERT(run->nfree > 0);
1.3875 +
1.3876 + ret = arena_run_reg_alloc(run, bin);
1.3877 + RELEASE_ASSERT(ret != NULL);
1.3878 + run->nfree--;
1.3879 +
1.3880 + return (ret);
1.3881 +}
1.3882 +
1.3883 +/* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */
1.3884 +static void *
1.3885 +arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
1.3886 +{
1.3887 +
1.3888 + bin->runcur = arena_bin_nonfull_run_get(arena, bin);
1.3889 + if (bin->runcur == NULL)
1.3890 + return (NULL);
1.3891 + RELEASE_ASSERT(bin->runcur->magic == ARENA_RUN_MAGIC);
1.3892 + RELEASE_ASSERT(bin->runcur->nfree > 0);
1.3893 +
1.3894 + return (arena_bin_malloc_easy(arena, bin, bin->runcur));
1.3895 +}
1.3896 +
1.3897 +/*
1.3898 + * Calculate bin->run_size such that it meets the following constraints:
1.3899 + *
1.3900 + * *) bin->run_size >= min_run_size
1.3901 + * *) bin->run_size <= arena_maxclass
1.3902 + * *) bin->run_size <= RUN_MAX_SMALL
1.3903 + * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
1.3904 + *
1.3905 + * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are
1.3906 + * also calculated here, since these settings are all interdependent.
1.3907 + */
1.3908 +static size_t
1.3909 +arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
1.3910 +{
1.3911 + size_t try_run_size, good_run_size;
1.3912 + unsigned good_nregs, good_mask_nelms, good_reg0_offset;
1.3913 + unsigned try_nregs, try_mask_nelms, try_reg0_offset;
1.3914 +
1.3915 + assert(min_run_size >= pagesize);
1.3916 + assert(min_run_size <= arena_maxclass);
1.3917 + assert(min_run_size <= RUN_MAX_SMALL);
1.3918 +
1.3919 + /*
1.3920 + * Calculate known-valid settings before entering the run_size
1.3921 + * expansion loop, so that the first part of the loop always copies
1.3922 + * valid settings.
1.3923 + *
1.3924 + * The do..while loop iteratively reduces the number of regions until
1.3925 + * the run header and the regions no longer overlap. A closed formula
1.3926 + * would be quite messy, since there is an interdependency between the
1.3927 + * header's mask length and the number of regions.
1.3928 + */
1.3929 + try_run_size = min_run_size;
1.3930 + try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size)
1.3931 + + 1; /* Counter-act try_nregs-- in loop. */
1.3932 + do {
1.3933 + try_nregs--;
1.3934 + try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
1.3935 + ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0);
1.3936 + try_reg0_offset = try_run_size - (try_nregs * bin->reg_size);
1.3937 + } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1))
1.3938 + > try_reg0_offset);
1.3939 +
1.3940 + /* run_size expansion loop. */
1.3941 + do {
1.3942 + /*
1.3943 + * Copy valid settings before trying more aggressive settings.
1.3944 + */
1.3945 + good_run_size = try_run_size;
1.3946 + good_nregs = try_nregs;
1.3947 + good_mask_nelms = try_mask_nelms;
1.3948 + good_reg0_offset = try_reg0_offset;
1.3949 +
1.3950 + /* Try more aggressive settings. */
1.3951 + try_run_size += pagesize;
1.3952 + try_nregs = ((try_run_size - sizeof(arena_run_t)) /
1.3953 + bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */
1.3954 + do {
1.3955 + try_nregs--;
1.3956 + try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
1.3957 + ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ?
1.3958 + 1 : 0);
1.3959 + try_reg0_offset = try_run_size - (try_nregs *
1.3960 + bin->reg_size);
1.3961 + } while (sizeof(arena_run_t) + (sizeof(unsigned) *
1.3962 + (try_mask_nelms - 1)) > try_reg0_offset);
1.3963 + } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL
1.3964 + && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
1.3965 + && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
1.3966 +
1.3967 + assert(sizeof(arena_run_t) + (sizeof(unsigned) * (good_mask_nelms - 1))
1.3968 + <= good_reg0_offset);
1.3969 + assert((good_mask_nelms << (SIZEOF_INT_2POW + 3)) >= good_nregs);
1.3970 +
1.3971 + /* Copy final settings. */
1.3972 + bin->run_size = good_run_size;
1.3973 + bin->nregs = good_nregs;
1.3974 + bin->regs_mask_nelms = good_mask_nelms;
1.3975 + bin->reg0_offset = good_reg0_offset;
1.3976 +
1.3977 + return (good_run_size);
1.3978 +}
1.3979 +
1.3980 +#ifdef MALLOC_BALANCE
1.3981 +static inline void
1.3982 +arena_lock_balance(arena_t *arena)
1.3983 +{
1.3984 + unsigned contention;
1.3985 +
1.3986 + contention = malloc_spin_lock(&arena->lock);
1.3987 + if (narenas > 1) {
1.3988 + /*
1.3989 + * Calculate the exponentially averaged contention for this
1.3990 + * arena. Due to integer math always rounding down, this value
1.3991 + * decays somewhat faster then normal.
1.3992 + */
1.3993 + arena->contention = (((uint64_t)arena->contention
1.3994 + * (uint64_t)((1U << BALANCE_ALPHA_INV_2POW)-1))
1.3995 + + (uint64_t)contention) >> BALANCE_ALPHA_INV_2POW;
1.3996 + if (arena->contention >= opt_balance_threshold)
1.3997 + arena_lock_balance_hard(arena);
1.3998 + }
1.3999 +}
1.4000 +
1.4001 +static void
1.4002 +arena_lock_balance_hard(arena_t *arena)
1.4003 +{
1.4004 + uint32_t ind;
1.4005 +
1.4006 + arena->contention = 0;
1.4007 +#ifdef MALLOC_STATS
1.4008 + arena->stats.nbalance++;
1.4009 +#endif
1.4010 + ind = PRN(balance, narenas_2pow);
1.4011 + if (arenas[ind] != NULL) {
1.4012 +#ifdef MOZ_MEMORY_WINDOWS
1.4013 + TlsSetValue(tlsIndex, arenas[ind]);
1.4014 +#else
1.4015 + arenas_map = arenas[ind];
1.4016 +#endif
1.4017 + } else {
1.4018 + malloc_spin_lock(&arenas_lock);
1.4019 + if (arenas[ind] != NULL) {
1.4020 +#ifdef MOZ_MEMORY_WINDOWS
1.4021 + TlsSetValue(tlsIndex, arenas[ind]);
1.4022 +#else
1.4023 + arenas_map = arenas[ind];
1.4024 +#endif
1.4025 + } else {
1.4026 +#ifdef MOZ_MEMORY_WINDOWS
1.4027 + TlsSetValue(tlsIndex, arenas_extend(ind));
1.4028 +#else
1.4029 + arenas_map = arenas_extend(ind);
1.4030 +#endif
1.4031 + }
1.4032 + malloc_spin_unlock(&arenas_lock);
1.4033 + }
1.4034 +}
1.4035 +#endif
1.4036 +
1.4037 +static inline void *
1.4038 +arena_malloc_small(arena_t *arena, size_t size, bool zero)
1.4039 +{
1.4040 + void *ret;
1.4041 + arena_bin_t *bin;
1.4042 + arena_run_t *run;
1.4043 +
1.4044 + if (size < small_min) {
1.4045 + /* Tiny. */
1.4046 + size = pow2_ceil(size);
1.4047 + bin = &arena->bins[ffs((int)(size >> (TINY_MIN_2POW +
1.4048 + 1)))];
1.4049 +#if (!defined(NDEBUG) || defined(MALLOC_STATS))
1.4050 + /*
1.4051 + * Bin calculation is always correct, but we may need
1.4052 + * to fix size for the purposes of assertions and/or
1.4053 + * stats accuracy.
1.4054 + */
1.4055 + if (size < (1U << TINY_MIN_2POW))
1.4056 + size = (1U << TINY_MIN_2POW);
1.4057 +#endif
1.4058 + } else if (size <= small_max) {
1.4059 + /* Quantum-spaced. */
1.4060 + size = QUANTUM_CEILING(size);
1.4061 + bin = &arena->bins[ntbins + (size >> opt_quantum_2pow)
1.4062 + - 1];
1.4063 + } else {
1.4064 + /* Sub-page. */
1.4065 + size = pow2_ceil(size);
1.4066 + bin = &arena->bins[ntbins + nqbins
1.4067 + + (ffs((int)(size >> opt_small_max_2pow)) - 2)];
1.4068 + }
1.4069 + RELEASE_ASSERT(size == bin->reg_size);
1.4070 +
1.4071 +#ifdef MALLOC_BALANCE
1.4072 + arena_lock_balance(arena);
1.4073 +#else
1.4074 + malloc_spin_lock(&arena->lock);
1.4075 +#endif
1.4076 + if ((run = bin->runcur) != NULL && run->nfree > 0)
1.4077 + ret = arena_bin_malloc_easy(arena, bin, run);
1.4078 + else
1.4079 + ret = arena_bin_malloc_hard(arena, bin);
1.4080 +
1.4081 + if (ret == NULL) {
1.4082 + malloc_spin_unlock(&arena->lock);
1.4083 + return (NULL);
1.4084 + }
1.4085 +
1.4086 +#ifdef MALLOC_STATS
1.4087 + bin->stats.nrequests++;
1.4088 + arena->stats.nmalloc_small++;
1.4089 + arena->stats.allocated_small += size;
1.4090 +#endif
1.4091 + malloc_spin_unlock(&arena->lock);
1.4092 +
1.4093 + VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
1.4094 + if (zero == false) {
1.4095 +#ifdef MALLOC_FILL
1.4096 + if (opt_junk)
1.4097 + memset(ret, 0xa5, size);
1.4098 + else if (opt_zero)
1.4099 + memset(ret, 0, size);
1.4100 +#endif
1.4101 + } else
1.4102 + memset(ret, 0, size);
1.4103 +
1.4104 + return (ret);
1.4105 +}
1.4106 +
1.4107 +static void *
1.4108 +arena_malloc_large(arena_t *arena, size_t size, bool zero)
1.4109 +{
1.4110 + void *ret;
1.4111 +
1.4112 + /* Large allocation. */
1.4113 + size = PAGE_CEILING(size);
1.4114 +#ifdef MALLOC_BALANCE
1.4115 + arena_lock_balance(arena);
1.4116 +#else
1.4117 + malloc_spin_lock(&arena->lock);
1.4118 +#endif
1.4119 + ret = (void *)arena_run_alloc(arena, NULL, size, true, zero);
1.4120 + if (ret == NULL) {
1.4121 + malloc_spin_unlock(&arena->lock);
1.4122 + return (NULL);
1.4123 + }
1.4124 +#ifdef MALLOC_STATS
1.4125 + arena->stats.nmalloc_large++;
1.4126 + arena->stats.allocated_large += size;
1.4127 +#endif
1.4128 + malloc_spin_unlock(&arena->lock);
1.4129 +
1.4130 + VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
1.4131 + if (zero == false) {
1.4132 +#ifdef MALLOC_FILL
1.4133 + if (opt_junk)
1.4134 + memset(ret, 0xa5, size);
1.4135 + else if (opt_zero)
1.4136 + memset(ret, 0, size);
1.4137 +#endif
1.4138 + }
1.4139 +
1.4140 + return (ret);
1.4141 +}
1.4142 +
1.4143 +static inline void *
1.4144 +arena_malloc(arena_t *arena, size_t size, bool zero)
1.4145 +{
1.4146 +
1.4147 + assert(arena != NULL);
1.4148 + RELEASE_ASSERT(arena->magic == ARENA_MAGIC);
1.4149 + assert(size != 0);
1.4150 + assert(QUANTUM_CEILING(size) <= arena_maxclass);
1.4151 +
1.4152 + if (size <= bin_maxclass) {
1.4153 + return (arena_malloc_small(arena, size, zero));
1.4154 + } else
1.4155 + return (arena_malloc_large(arena, size, zero));
1.4156 +}
1.4157 +
1.4158 +static inline void *
1.4159 +imalloc(size_t size)
1.4160 +{
1.4161 +
1.4162 + assert(size != 0);
1.4163 +
1.4164 + if (size <= arena_maxclass)
1.4165 + return (arena_malloc(choose_arena(), size, false));
1.4166 + else
1.4167 + return (huge_malloc(size, false));
1.4168 +}
1.4169 +
1.4170 +static inline void *
1.4171 +icalloc(size_t size)
1.4172 +{
1.4173 +
1.4174 + if (size <= arena_maxclass)
1.4175 + return (arena_malloc(choose_arena(), size, true));
1.4176 + else
1.4177 + return (huge_malloc(size, true));
1.4178 +}
1.4179 +
1.4180 +/* Only handles large allocations that require more than page alignment. */
1.4181 +static void *
1.4182 +arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
1.4183 +{
1.4184 + void *ret;
1.4185 + size_t offset;
1.4186 + arena_chunk_t *chunk;
1.4187 +
1.4188 + assert((size & pagesize_mask) == 0);
1.4189 + assert((alignment & pagesize_mask) == 0);
1.4190 +
1.4191 +#ifdef MALLOC_BALANCE
1.4192 + arena_lock_balance(arena);
1.4193 +#else
1.4194 + malloc_spin_lock(&arena->lock);
1.4195 +#endif
1.4196 + ret = (void *)arena_run_alloc(arena, NULL, alloc_size, true, false);
1.4197 + if (ret == NULL) {
1.4198 + malloc_spin_unlock(&arena->lock);
1.4199 + return (NULL);
1.4200 + }
1.4201 +
1.4202 + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
1.4203 +
1.4204 + offset = (uintptr_t)ret & (alignment - 1);
1.4205 + assert((offset & pagesize_mask) == 0);
1.4206 + assert(offset < alloc_size);
1.4207 + if (offset == 0)
1.4208 + arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, alloc_size, size, false);
1.4209 + else {
1.4210 + size_t leadsize, trailsize;
1.4211 +
1.4212 + leadsize = alignment - offset;
1.4213 + if (leadsize > 0) {
1.4214 + arena_run_trim_head(arena, chunk, (arena_run_t*)ret, alloc_size,
1.4215 + alloc_size - leadsize);
1.4216 + ret = (void *)((uintptr_t)ret + leadsize);
1.4217 + }
1.4218 +
1.4219 + trailsize = alloc_size - leadsize - size;
1.4220 + if (trailsize != 0) {
1.4221 + /* Trim trailing space. */
1.4222 + assert(trailsize < alloc_size);
1.4223 + arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, size + trailsize,
1.4224 + size, false);
1.4225 + }
1.4226 + }
1.4227 +
1.4228 +#ifdef MALLOC_STATS
1.4229 + arena->stats.nmalloc_large++;
1.4230 + arena->stats.allocated_large += size;
1.4231 +#endif
1.4232 + malloc_spin_unlock(&arena->lock);
1.4233 +
1.4234 + VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
1.4235 +#ifdef MALLOC_FILL
1.4236 + if (opt_junk)
1.4237 + memset(ret, 0xa5, size);
1.4238 + else if (opt_zero)
1.4239 + memset(ret, 0, size);
1.4240 +#endif
1.4241 + return (ret);
1.4242 +}
1.4243 +
1.4244 +static inline void *
1.4245 +ipalloc(size_t alignment, size_t size)
1.4246 +{
1.4247 + void *ret;
1.4248 + size_t ceil_size;
1.4249 +
1.4250 + /*
1.4251 + * Round size up to the nearest multiple of alignment.
1.4252 + *
1.4253 + * This done, we can take advantage of the fact that for each small
1.4254 + * size class, every object is aligned at the smallest power of two
1.4255 + * that is non-zero in the base two representation of the size. For
1.4256 + * example:
1.4257 + *
1.4258 + * Size | Base 2 | Minimum alignment
1.4259 + * -----+----------+------------------
1.4260 + * 96 | 1100000 | 32
1.4261 + * 144 | 10100000 | 32
1.4262 + * 192 | 11000000 | 64
1.4263 + *
1.4264 + * Depending on runtime settings, it is possible that arena_malloc()
1.4265 + * will further round up to a power of two, but that never causes
1.4266 + * correctness issues.
1.4267 + */
1.4268 + ceil_size = (size + (alignment - 1)) & (-alignment);
1.4269 + /*
1.4270 + * (ceil_size < size) protects against the combination of maximal
1.4271 + * alignment and size greater than maximal alignment.
1.4272 + */
1.4273 + if (ceil_size < size) {
1.4274 + /* size_t overflow. */
1.4275 + return (NULL);
1.4276 + }
1.4277 +
1.4278 + if (ceil_size <= pagesize || (alignment <= pagesize
1.4279 + && ceil_size <= arena_maxclass))
1.4280 + ret = arena_malloc(choose_arena(), ceil_size, false);
1.4281 + else {
1.4282 + size_t run_size;
1.4283 +
1.4284 + /*
1.4285 + * We can't achieve sub-page alignment, so round up alignment
1.4286 + * permanently; it makes later calculations simpler.
1.4287 + */
1.4288 + alignment = PAGE_CEILING(alignment);
1.4289 + ceil_size = PAGE_CEILING(size);
1.4290 + /*
1.4291 + * (ceil_size < size) protects against very large sizes within
1.4292 + * pagesize of SIZE_T_MAX.
1.4293 + *
1.4294 + * (ceil_size + alignment < ceil_size) protects against the
1.4295 + * combination of maximal alignment and ceil_size large enough
1.4296 + * to cause overflow. This is similar to the first overflow
1.4297 + * check above, but it needs to be repeated due to the new
1.4298 + * ceil_size value, which may now be *equal* to maximal
1.4299 + * alignment, whereas before we only detected overflow if the
1.4300 + * original size was *greater* than maximal alignment.
1.4301 + */
1.4302 + if (ceil_size < size || ceil_size + alignment < ceil_size) {
1.4303 + /* size_t overflow. */
1.4304 + return (NULL);
1.4305 + }
1.4306 +
1.4307 + /*
1.4308 + * Calculate the size of the over-size run that arena_palloc()
1.4309 + * would need to allocate in order to guarantee the alignment.
1.4310 + */
1.4311 + if (ceil_size >= alignment)
1.4312 + run_size = ceil_size + alignment - pagesize;
1.4313 + else {
1.4314 + /*
1.4315 + * It is possible that (alignment << 1) will cause
1.4316 + * overflow, but it doesn't matter because we also
1.4317 + * subtract pagesize, which in the case of overflow
1.4318 + * leaves us with a very large run_size. That causes
1.4319 + * the first conditional below to fail, which means
1.4320 + * that the bogus run_size value never gets used for
1.4321 + * anything important.
1.4322 + */
1.4323 + run_size = (alignment << 1) - pagesize;
1.4324 + }
1.4325 +
1.4326 + if (run_size <= arena_maxclass) {
1.4327 + ret = arena_palloc(choose_arena(), alignment, ceil_size,
1.4328 + run_size);
1.4329 + } else if (alignment <= chunksize)
1.4330 + ret = huge_malloc(ceil_size, false);
1.4331 + else
1.4332 + ret = huge_palloc(alignment, ceil_size);
1.4333 + }
1.4334 +
1.4335 + assert(((uintptr_t)ret & (alignment - 1)) == 0);
1.4336 + return (ret);
1.4337 +}
1.4338 +
1.4339 +/* Return the size of the allocation pointed to by ptr. */
1.4340 +static size_t
1.4341 +arena_salloc(const void *ptr)
1.4342 +{
1.4343 + size_t ret;
1.4344 + arena_chunk_t *chunk;
1.4345 + size_t pageind, mapbits;
1.4346 +
1.4347 + assert(ptr != NULL);
1.4348 + assert(CHUNK_ADDR2BASE(ptr) != ptr);
1.4349 +
1.4350 + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1.4351 + pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow);
1.4352 + mapbits = chunk->map[pageind].bits;
1.4353 + RELEASE_ASSERT((mapbits & CHUNK_MAP_ALLOCATED) != 0);
1.4354 + if ((mapbits & CHUNK_MAP_LARGE) == 0) {
1.4355 + arena_run_t *run = (arena_run_t *)(mapbits & ~pagesize_mask);
1.4356 + RELEASE_ASSERT(run->magic == ARENA_RUN_MAGIC);
1.4357 + ret = run->bin->reg_size;
1.4358 + } else {
1.4359 + ret = mapbits & ~pagesize_mask;
1.4360 + RELEASE_ASSERT(ret != 0);
1.4361 + }
1.4362 +
1.4363 + return (ret);
1.4364 +}
1.4365 +
1.4366 +#if (defined(MALLOC_VALIDATE) || defined(MOZ_MEMORY_DARWIN))
1.4367 +/*
1.4368 + * Validate ptr before assuming that it points to an allocation. Currently,
1.4369 + * the following validation is performed:
1.4370 + *
1.4371 + * + Check that ptr is not NULL.
1.4372 + *
1.4373 + * + Check that ptr lies within a mapped chunk.
1.4374 + */
1.4375 +static inline size_t
1.4376 +isalloc_validate(const void *ptr)
1.4377 +{
1.4378 + arena_chunk_t *chunk;
1.4379 +
1.4380 + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1.4381 + if (chunk == NULL)
1.4382 + return (0);
1.4383 +
1.4384 + if (malloc_rtree_get(chunk_rtree, (uintptr_t)chunk) == NULL)
1.4385 + return (0);
1.4386 +
1.4387 + if (chunk != ptr) {
1.4388 + RELEASE_ASSERT(chunk->arena->magic == ARENA_MAGIC);
1.4389 + return (arena_salloc(ptr));
1.4390 + } else {
1.4391 + size_t ret;
1.4392 + extent_node_t *node;
1.4393 + extent_node_t key;
1.4394 +
1.4395 + /* Chunk. */
1.4396 + key.addr = (void *)chunk;
1.4397 + malloc_mutex_lock(&huge_mtx);
1.4398 + node = extent_tree_ad_search(&huge, &key);
1.4399 + if (node != NULL)
1.4400 + ret = node->size;
1.4401 + else
1.4402 + ret = 0;
1.4403 + malloc_mutex_unlock(&huge_mtx);
1.4404 + return (ret);
1.4405 + }
1.4406 +}
1.4407 +#endif
1.4408 +
1.4409 +static inline size_t
1.4410 +isalloc(const void *ptr)
1.4411 +{
1.4412 + size_t ret;
1.4413 + arena_chunk_t *chunk;
1.4414 +
1.4415 + assert(ptr != NULL);
1.4416 +
1.4417 + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1.4418 + if (chunk != ptr) {
1.4419 + /* Region. */
1.4420 + assert(chunk->arena->magic == ARENA_MAGIC);
1.4421 +
1.4422 + ret = arena_salloc(ptr);
1.4423 + } else {
1.4424 + extent_node_t *node, key;
1.4425 +
1.4426 + /* Chunk (huge allocation). */
1.4427 +
1.4428 + malloc_mutex_lock(&huge_mtx);
1.4429 +
1.4430 + /* Extract from tree of huge allocations. */
1.4431 + key.addr = __DECONST(void *, ptr);
1.4432 + node = extent_tree_ad_search(&huge, &key);
1.4433 + RELEASE_ASSERT(node != NULL);
1.4434 +
1.4435 + ret = node->size;
1.4436 +
1.4437 + malloc_mutex_unlock(&huge_mtx);
1.4438 + }
1.4439 +
1.4440 + return (ret);
1.4441 +}
1.4442 +
1.4443 +static inline void
1.4444 +arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
1.4445 + arena_chunk_map_t *mapelm)
1.4446 +{
1.4447 + arena_run_t *run;
1.4448 + arena_bin_t *bin;
1.4449 + size_t size;
1.4450 +
1.4451 + run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
1.4452 + RELEASE_ASSERT(run->magic == ARENA_RUN_MAGIC);
1.4453 + bin = run->bin;
1.4454 + size = bin->reg_size;
1.4455 +
1.4456 +#ifdef MALLOC_FILL
1.4457 + if (opt_poison)
1.4458 + memset(ptr, 0x5a, size);
1.4459 +#endif
1.4460 +
1.4461 + arena_run_reg_dalloc(run, bin, ptr, size);
1.4462 + run->nfree++;
1.4463 +
1.4464 + if (run->nfree == bin->nregs) {
1.4465 + /* Deallocate run. */
1.4466 + if (run == bin->runcur)
1.4467 + bin->runcur = NULL;
1.4468 + else if (bin->nregs != 1) {
1.4469 + size_t run_pageind = (((uintptr_t)run -
1.4470 + (uintptr_t)chunk)) >> pagesize_2pow;
1.4471 + arena_chunk_map_t *run_mapelm =
1.4472 + &chunk->map[run_pageind];
1.4473 + /*
1.4474 + * This block's conditional is necessary because if the
1.4475 + * run only contains one region, then it never gets
1.4476 + * inserted into the non-full runs tree.
1.4477 + */
1.4478 + RELEASE_ASSERT(arena_run_tree_search(&bin->runs, run_mapelm) ==
1.4479 + run_mapelm);
1.4480 + arena_run_tree_remove(&bin->runs, run_mapelm);
1.4481 + }
1.4482 +#if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
1.4483 + run->magic = 0;
1.4484 +#endif
1.4485 + VALGRIND_FREELIKE_BLOCK(run, 0);
1.4486 + arena_run_dalloc(arena, run, true);
1.4487 +#ifdef MALLOC_STATS
1.4488 + bin->stats.curruns--;
1.4489 +#endif
1.4490 + } else if (run->nfree == 1 && run != bin->runcur) {
1.4491 + /*
1.4492 + * Make sure that bin->runcur always refers to the lowest
1.4493 + * non-full run, if one exists.
1.4494 + */
1.4495 + if (bin->runcur == NULL)
1.4496 + bin->runcur = run;
1.4497 + else if ((uintptr_t)run < (uintptr_t)bin->runcur) {
1.4498 + /* Switch runcur. */
1.4499 + if (bin->runcur->nfree > 0) {
1.4500 + arena_chunk_t *runcur_chunk =
1.4501 + (arena_chunk_t*)CHUNK_ADDR2BASE(bin->runcur);
1.4502 + size_t runcur_pageind =
1.4503 + (((uintptr_t)bin->runcur -
1.4504 + (uintptr_t)runcur_chunk)) >> pagesize_2pow;
1.4505 + arena_chunk_map_t *runcur_mapelm =
1.4506 + &runcur_chunk->map[runcur_pageind];
1.4507 +
1.4508 + /* Insert runcur. */
1.4509 + RELEASE_ASSERT(arena_run_tree_search(&bin->runs,
1.4510 + runcur_mapelm) == NULL);
1.4511 + arena_run_tree_insert(&bin->runs,
1.4512 + runcur_mapelm);
1.4513 + }
1.4514 + bin->runcur = run;
1.4515 + } else {
1.4516 + size_t run_pageind = (((uintptr_t)run -
1.4517 + (uintptr_t)chunk)) >> pagesize_2pow;
1.4518 + arena_chunk_map_t *run_mapelm =
1.4519 + &chunk->map[run_pageind];
1.4520 +
1.4521 + RELEASE_ASSERT(arena_run_tree_search(&bin->runs, run_mapelm) ==
1.4522 + NULL);
1.4523 + arena_run_tree_insert(&bin->runs, run_mapelm);
1.4524 + }
1.4525 + }
1.4526 +#ifdef MALLOC_STATS
1.4527 + arena->stats.allocated_small -= size;
1.4528 + arena->stats.ndalloc_small++;
1.4529 +#endif
1.4530 +}
1.4531 +
1.4532 +static void
1.4533 +arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
1.4534 +{
1.4535 + /* Large allocation. */
1.4536 + malloc_spin_lock(&arena->lock);
1.4537 +
1.4538 +#ifdef MALLOC_FILL
1.4539 +#ifndef MALLOC_STATS
1.4540 + if (opt_poison)
1.4541 +#endif
1.4542 +#endif
1.4543 + {
1.4544 + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
1.4545 + pagesize_2pow;
1.4546 + size_t size = chunk->map[pageind].bits & ~pagesize_mask;
1.4547 +
1.4548 +#ifdef MALLOC_FILL
1.4549 +#ifdef MALLOC_STATS
1.4550 + if (opt_poison)
1.4551 +#endif
1.4552 + memset(ptr, 0x5a, size);
1.4553 +#endif
1.4554 +#ifdef MALLOC_STATS
1.4555 + arena->stats.allocated_large -= size;
1.4556 +#endif
1.4557 + }
1.4558 +#ifdef MALLOC_STATS
1.4559 + arena->stats.ndalloc_large++;
1.4560 +#endif
1.4561 +
1.4562 + arena_run_dalloc(arena, (arena_run_t *)ptr, true);
1.4563 + malloc_spin_unlock(&arena->lock);
1.4564 +}
1.4565 +
1.4566 +static inline void
1.4567 +arena_dalloc(void *ptr, size_t offset)
1.4568 +{
1.4569 + arena_chunk_t *chunk;
1.4570 + arena_t *arena;
1.4571 + size_t pageind;
1.4572 + arena_chunk_map_t *mapelm;
1.4573 +
1.4574 + assert(ptr != NULL);
1.4575 + assert(offset != 0);
1.4576 + assert(CHUNK_ADDR2OFFSET(ptr) == offset);
1.4577 +
1.4578 + chunk = (arena_chunk_t *) ((uintptr_t)ptr - offset);
1.4579 + arena = chunk->arena;
1.4580 + assert(arena != NULL);
1.4581 + RELEASE_ASSERT(arena->magic == ARENA_MAGIC);
1.4582 +
1.4583 + pageind = offset >> pagesize_2pow;
1.4584 + mapelm = &chunk->map[pageind];
1.4585 + RELEASE_ASSERT((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
1.4586 + if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
1.4587 + /* Small allocation. */
1.4588 + malloc_spin_lock(&arena->lock);
1.4589 + arena_dalloc_small(arena, chunk, ptr, mapelm);
1.4590 + malloc_spin_unlock(&arena->lock);
1.4591 + } else
1.4592 + arena_dalloc_large(arena, chunk, ptr);
1.4593 + VALGRIND_FREELIKE_BLOCK(ptr, 0);
1.4594 +}
1.4595 +
1.4596 +static inline void
1.4597 +idalloc(void *ptr)
1.4598 +{
1.4599 + size_t offset;
1.4600 +
1.4601 + assert(ptr != NULL);
1.4602 +
1.4603 + offset = CHUNK_ADDR2OFFSET(ptr);
1.4604 + if (offset != 0)
1.4605 + arena_dalloc(ptr, offset);
1.4606 + else
1.4607 + huge_dalloc(ptr);
1.4608 +}
1.4609 +
1.4610 +static void
1.4611 +arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
1.4612 + size_t size, size_t oldsize)
1.4613 +{
1.4614 +
1.4615 + assert(size < oldsize);
1.4616 +
1.4617 + /*
1.4618 + * Shrink the run, and make trailing pages available for other
1.4619 + * allocations.
1.4620 + */
1.4621 +#ifdef MALLOC_BALANCE
1.4622 + arena_lock_balance(arena);
1.4623 +#else
1.4624 + malloc_spin_lock(&arena->lock);
1.4625 +#endif
1.4626 + arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size,
1.4627 + true);
1.4628 +#ifdef MALLOC_STATS
1.4629 + arena->stats.allocated_large -= oldsize - size;
1.4630 +#endif
1.4631 + malloc_spin_unlock(&arena->lock);
1.4632 +}
1.4633 +
1.4634 +static bool
1.4635 +arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
1.4636 + size_t size, size_t oldsize)
1.4637 +{
1.4638 + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow;
1.4639 + size_t npages = oldsize >> pagesize_2pow;
1.4640 +
1.4641 + RELEASE_ASSERT(oldsize == (chunk->map[pageind].bits & ~pagesize_mask));
1.4642 +
1.4643 + /* Try to extend the run. */
1.4644 + assert(size > oldsize);
1.4645 +#ifdef MALLOC_BALANCE
1.4646 + arena_lock_balance(arena);
1.4647 +#else
1.4648 + malloc_spin_lock(&arena->lock);
1.4649 +#endif
1.4650 + if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits
1.4651 + & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits &
1.4652 + ~pagesize_mask) >= size - oldsize) {
1.4653 + /*
1.4654 + * The next run is available and sufficiently large. Split the
1.4655 + * following run, then merge the first part with the existing
1.4656 + * allocation.
1.4657 + */
1.4658 + arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk +
1.4659 + ((pageind+npages) << pagesize_2pow)), size - oldsize, true,
1.4660 + false);
1.4661 +
1.4662 + chunk->map[pageind].bits = size | CHUNK_MAP_LARGE |
1.4663 + CHUNK_MAP_ALLOCATED;
1.4664 + chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE |
1.4665 + CHUNK_MAP_ALLOCATED;
1.4666 +
1.4667 +#ifdef MALLOC_STATS
1.4668 + arena->stats.allocated_large += size - oldsize;
1.4669 +#endif
1.4670 + malloc_spin_unlock(&arena->lock);
1.4671 + return (false);
1.4672 + }
1.4673 + malloc_spin_unlock(&arena->lock);
1.4674 +
1.4675 + return (true);
1.4676 +}
1.4677 +
1.4678 +/*
1.4679 + * Try to resize a large allocation, in order to avoid copying. This will
1.4680 + * always fail if growing an object, and the following run is already in use.
1.4681 + */
1.4682 +static bool
1.4683 +arena_ralloc_large(void *ptr, size_t size, size_t oldsize)
1.4684 +{
1.4685 + size_t psize;
1.4686 +
1.4687 + psize = PAGE_CEILING(size);
1.4688 + if (psize == oldsize) {
1.4689 + /* Same size class. */
1.4690 +#ifdef MALLOC_FILL
1.4691 + if (opt_poison && size < oldsize) {
1.4692 + memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize -
1.4693 + size);
1.4694 + }
1.4695 +#endif
1.4696 + return (false);
1.4697 + } else {
1.4698 + arena_chunk_t *chunk;
1.4699 + arena_t *arena;
1.4700 +
1.4701 + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1.4702 + arena = chunk->arena;
1.4703 + RELEASE_ASSERT(arena->magic == ARENA_MAGIC);
1.4704 +
1.4705 + if (psize < oldsize) {
1.4706 +#ifdef MALLOC_FILL
1.4707 + /* Fill before shrinking in order avoid a race. */
1.4708 + if (opt_poison) {
1.4709 + memset((void *)((uintptr_t)ptr + size), 0x5a,
1.4710 + oldsize - size);
1.4711 + }
1.4712 +#endif
1.4713 + arena_ralloc_large_shrink(arena, chunk, ptr, psize,
1.4714 + oldsize);
1.4715 + return (false);
1.4716 + } else {
1.4717 + bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
1.4718 + psize, oldsize);
1.4719 +#ifdef MALLOC_FILL
1.4720 + if (ret == false && opt_zero) {
1.4721 + memset((void *)((uintptr_t)ptr + oldsize), 0,
1.4722 + size - oldsize);
1.4723 + }
1.4724 +#endif
1.4725 + return (ret);
1.4726 + }
1.4727 + }
1.4728 +}
1.4729 +
1.4730 +static void *
1.4731 +arena_ralloc(void *ptr, size_t size, size_t oldsize)
1.4732 +{
1.4733 + void *ret;
1.4734 + size_t copysize;
1.4735 +
1.4736 + /* Try to avoid moving the allocation. */
1.4737 + if (size < small_min) {
1.4738 + if (oldsize < small_min &&
1.4739 + ffs((int)(pow2_ceil(size) >> (TINY_MIN_2POW + 1)))
1.4740 + == ffs((int)(pow2_ceil(oldsize) >> (TINY_MIN_2POW + 1))))
1.4741 + goto IN_PLACE; /* Same size class. */
1.4742 + } else if (size <= small_max) {
1.4743 + if (oldsize >= small_min && oldsize <= small_max &&
1.4744 + (QUANTUM_CEILING(size) >> opt_quantum_2pow)
1.4745 + == (QUANTUM_CEILING(oldsize) >> opt_quantum_2pow))
1.4746 + goto IN_PLACE; /* Same size class. */
1.4747 + } else if (size <= bin_maxclass) {
1.4748 + if (oldsize > small_max && oldsize <= bin_maxclass &&
1.4749 + pow2_ceil(size) == pow2_ceil(oldsize))
1.4750 + goto IN_PLACE; /* Same size class. */
1.4751 + } else if (oldsize > bin_maxclass && oldsize <= arena_maxclass) {
1.4752 + assert(size > bin_maxclass);
1.4753 + if (arena_ralloc_large(ptr, size, oldsize) == false)
1.4754 + return (ptr);
1.4755 + }
1.4756 +
1.4757 + /*
1.4758 + * If we get here, then size and oldsize are different enough that we
1.4759 + * need to move the object. In that case, fall back to allocating new
1.4760 + * space and copying.
1.4761 + */
1.4762 + ret = arena_malloc(choose_arena(), size, false);
1.4763 + if (ret == NULL)
1.4764 + return (NULL);
1.4765 +
1.4766 + /* Junk/zero-filling were already done by arena_malloc(). */
1.4767 + copysize = (size < oldsize) ? size : oldsize;
1.4768 +#ifdef VM_COPY_MIN
1.4769 + if (copysize >= VM_COPY_MIN)
1.4770 + pages_copy(ret, ptr, copysize);
1.4771 + else
1.4772 +#endif
1.4773 + memcpy(ret, ptr, copysize);
1.4774 + idalloc(ptr);
1.4775 + return (ret);
1.4776 +IN_PLACE:
1.4777 +#ifdef MALLOC_FILL
1.4778 + if (opt_poison && size < oldsize)
1.4779 + memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size);
1.4780 + else if (opt_zero && size > oldsize)
1.4781 + memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize);
1.4782 +#endif
1.4783 + return (ptr);
1.4784 +}
1.4785 +
1.4786 +static inline void *
1.4787 +iralloc(void *ptr, size_t size)
1.4788 +{
1.4789 + size_t oldsize;
1.4790 +
1.4791 + assert(ptr != NULL);
1.4792 + assert(size != 0);
1.4793 +
1.4794 + oldsize = isalloc(ptr);
1.4795 +
1.4796 +#ifndef MALLOC_VALGRIND
1.4797 + if (size <= arena_maxclass)
1.4798 + return (arena_ralloc(ptr, size, oldsize));
1.4799 + else
1.4800 + return (huge_ralloc(ptr, size, oldsize));
1.4801 +#else
1.4802 + /*
1.4803 + * Valgrind does not provide a public interface for modifying an
1.4804 + * existing allocation, so use malloc/memcpy/free instead.
1.4805 + */
1.4806 + {
1.4807 + void *ret = imalloc(size);
1.4808 + if (ret != NULL) {
1.4809 + if (oldsize < size)
1.4810 + memcpy(ret, ptr, oldsize);
1.4811 + else
1.4812 + memcpy(ret, ptr, size);
1.4813 + idalloc(ptr);
1.4814 + }
1.4815 + return (ret);
1.4816 + }
1.4817 +#endif
1.4818 +}
1.4819 +
1.4820 +static bool
1.4821 +arena_new(arena_t *arena)
1.4822 +{
1.4823 + unsigned i;
1.4824 + arena_bin_t *bin;
1.4825 + size_t pow2_size, prev_run_size;
1.4826 +
1.4827 + if (malloc_spin_init(&arena->lock))
1.4828 + return (true);
1.4829 +
1.4830 +#ifdef MALLOC_STATS
1.4831 + memset(&arena->stats, 0, sizeof(arena_stats_t));
1.4832 +#endif
1.4833 +
1.4834 + /* Initialize chunks. */
1.4835 + arena_chunk_tree_dirty_new(&arena->chunks_dirty);
1.4836 +#ifdef MALLOC_DOUBLE_PURGE
1.4837 + LinkedList_Init(&arena->chunks_madvised);
1.4838 +#endif
1.4839 + arena->spare = NULL;
1.4840 +
1.4841 + arena->ndirty = 0;
1.4842 +
1.4843 + arena_avail_tree_new(&arena->runs_avail);
1.4844 +
1.4845 +#ifdef MALLOC_BALANCE
1.4846 + arena->contention = 0;
1.4847 +#endif
1.4848 +
1.4849 + /* Initialize bins. */
1.4850 + prev_run_size = pagesize;
1.4851 +
1.4852 + /* (2^n)-spaced tiny bins. */
1.4853 + for (i = 0; i < ntbins; i++) {
1.4854 + bin = &arena->bins[i];
1.4855 + bin->runcur = NULL;
1.4856 + arena_run_tree_new(&bin->runs);
1.4857 +
1.4858 + bin->reg_size = (1U << (TINY_MIN_2POW + i));
1.4859 +
1.4860 + prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
1.4861 +
1.4862 +#ifdef MALLOC_STATS
1.4863 + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
1.4864 +#endif
1.4865 + }
1.4866 +
1.4867 + /* Quantum-spaced bins. */
1.4868 + for (; i < ntbins + nqbins; i++) {
1.4869 + bin = &arena->bins[i];
1.4870 + bin->runcur = NULL;
1.4871 + arena_run_tree_new(&bin->runs);
1.4872 +
1.4873 + bin->reg_size = quantum * (i - ntbins + 1);
1.4874 +
1.4875 + pow2_size = pow2_ceil(quantum * (i - ntbins + 1));
1.4876 + prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
1.4877 +
1.4878 +#ifdef MALLOC_STATS
1.4879 + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
1.4880 +#endif
1.4881 + }
1.4882 +
1.4883 + /* (2^n)-spaced sub-page bins. */
1.4884 + for (; i < ntbins + nqbins + nsbins; i++) {
1.4885 + bin = &arena->bins[i];
1.4886 + bin->runcur = NULL;
1.4887 + arena_run_tree_new(&bin->runs);
1.4888 +
1.4889 + bin->reg_size = (small_max << (i - (ntbins + nqbins) + 1));
1.4890 +
1.4891 + prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
1.4892 +
1.4893 +#ifdef MALLOC_STATS
1.4894 + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
1.4895 +#endif
1.4896 + }
1.4897 +
1.4898 +#if defined(MALLOC_DEBUG) || defined(MOZ_JEMALLOC_HARD_ASSERTS)
1.4899 + arena->magic = ARENA_MAGIC;
1.4900 +#endif
1.4901 +
1.4902 + return (false);
1.4903 +}
1.4904 +
1.4905 +/* Create a new arena and insert it into the arenas array at index ind. */
1.4906 +static arena_t *
1.4907 +arenas_extend(unsigned ind)
1.4908 +{
1.4909 + arena_t *ret;
1.4910 +
1.4911 + /* Allocate enough space for trailing bins. */
1.4912 + ret = (arena_t *)base_alloc(sizeof(arena_t)
1.4913 + + (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1)));
1.4914 + if (ret != NULL && arena_new(ret) == false) {
1.4915 + arenas[ind] = ret;
1.4916 + return (ret);
1.4917 + }
1.4918 + /* Only reached if there is an OOM error. */
1.4919 +
1.4920 + /*
1.4921 + * OOM here is quite inconvenient to propagate, since dealing with it
1.4922 + * would require a check for failure in the fast path. Instead, punt
1.4923 + * by using arenas[0]. In practice, this is an extremely unlikely
1.4924 + * failure.
1.4925 + */
1.4926 + _malloc_message(_getprogname(),
1.4927 + ": (malloc) Error initializing arena\n", "", "");
1.4928 + if (opt_abort)
1.4929 + abort();
1.4930 +
1.4931 + return (arenas[0]);
1.4932 +}
1.4933 +
1.4934 +/*
1.4935 + * End arena.
1.4936 + */
1.4937 +/******************************************************************************/
1.4938 +/*
1.4939 + * Begin general internal functions.
1.4940 + */
1.4941 +
1.4942 +static void *
1.4943 +huge_malloc(size_t size, bool zero)
1.4944 +{
1.4945 + void *ret;
1.4946 + size_t csize;
1.4947 + size_t psize;
1.4948 + extent_node_t *node;
1.4949 +
1.4950 + /* Allocate one or more contiguous chunks for this request. */
1.4951 +
1.4952 + csize = CHUNK_CEILING(size);
1.4953 + if (csize == 0) {
1.4954 + /* size is large enough to cause size_t wrap-around. */
1.4955 + return (NULL);
1.4956 + }
1.4957 +
1.4958 + /* Allocate an extent node with which to track the chunk. */
1.4959 + node = base_node_alloc();
1.4960 + if (node == NULL)
1.4961 + return (NULL);
1.4962 +
1.4963 + ret = chunk_alloc(csize, zero, true);
1.4964 + if (ret == NULL) {
1.4965 + base_node_dealloc(node);
1.4966 + return (NULL);
1.4967 + }
1.4968 +
1.4969 + /* Insert node into huge. */
1.4970 + node->addr = ret;
1.4971 + psize = PAGE_CEILING(size);
1.4972 + node->size = psize;
1.4973 +
1.4974 + malloc_mutex_lock(&huge_mtx);
1.4975 + extent_tree_ad_insert(&huge, node);
1.4976 +#ifdef MALLOC_STATS
1.4977 + huge_nmalloc++;
1.4978 +
1.4979 + /* Although we allocated space for csize bytes, we indicate that we've
1.4980 + * allocated only psize bytes.
1.4981 + *
1.4982 + * If DECOMMIT is defined, this is a reasonable thing to do, since
1.4983 + * we'll explicitly decommit the bytes in excess of psize.
1.4984 + *
1.4985 + * If DECOMMIT is not defined, then we're relying on the OS to be lazy
1.4986 + * about how it allocates physical pages to mappings. If we never
1.4987 + * touch the pages in excess of psize, the OS won't allocate a physical
1.4988 + * page, and we won't use more than psize bytes of physical memory.
1.4989 + *
1.4990 + * A correct program will only touch memory in excess of how much it
1.4991 + * requested if it first calls malloc_usable_size and finds out how
1.4992 + * much space it has to play with. But because we set node->size =
1.4993 + * psize above, malloc_usable_size will return psize, not csize, and
1.4994 + * the program will (hopefully) never touch bytes in excess of psize.
1.4995 + * Thus those bytes won't take up space in physical memory, and we can
1.4996 + * reasonably claim we never "allocated" them in the first place. */
1.4997 + huge_allocated += psize;
1.4998 + huge_mapped += csize;
1.4999 +#endif
1.5000 + malloc_mutex_unlock(&huge_mtx);
1.5001 +
1.5002 +#ifdef MALLOC_DECOMMIT
1.5003 + if (csize - psize > 0)
1.5004 + pages_decommit((void *)((uintptr_t)ret + psize), csize - psize);
1.5005 +#endif
1.5006 +
1.5007 +#ifdef MALLOC_DECOMMIT
1.5008 + VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, zero);
1.5009 +#else
1.5010 + VALGRIND_MALLOCLIKE_BLOCK(ret, csize, 0, zero);
1.5011 +#endif
1.5012 +
1.5013 +#ifdef MALLOC_FILL
1.5014 + if (zero == false) {
1.5015 + if (opt_junk)
1.5016 +# ifdef MALLOC_DECOMMIT
1.5017 + memset(ret, 0xa5, psize);
1.5018 +# else
1.5019 + memset(ret, 0xa5, csize);
1.5020 +# endif
1.5021 + else if (opt_zero)
1.5022 +# ifdef MALLOC_DECOMMIT
1.5023 + memset(ret, 0, psize);
1.5024 +# else
1.5025 + memset(ret, 0, csize);
1.5026 +# endif
1.5027 + }
1.5028 +#endif
1.5029 +
1.5030 + return (ret);
1.5031 +}
1.5032 +
1.5033 +/* Only handles large allocations that require more than chunk alignment. */
1.5034 +static void *
1.5035 +huge_palloc(size_t alignment, size_t size)
1.5036 +{
1.5037 + void *ret;
1.5038 + size_t alloc_size, chunk_size, offset;
1.5039 + size_t psize;
1.5040 + extent_node_t *node;
1.5041 + int pfd;
1.5042 +
1.5043 + /*
1.5044 + * This allocation requires alignment that is even larger than chunk
1.5045 + * alignment. This means that huge_malloc() isn't good enough.
1.5046 + *
1.5047 + * Allocate almost twice as many chunks as are demanded by the size or
1.5048 + * alignment, in order to assure the alignment can be achieved, then
1.5049 + * unmap leading and trailing chunks.
1.5050 + */
1.5051 + assert(alignment >= chunksize);
1.5052 +
1.5053 + chunk_size = CHUNK_CEILING(size);
1.5054 +
1.5055 + if (size >= alignment)
1.5056 + alloc_size = chunk_size + alignment - chunksize;
1.5057 + else
1.5058 + alloc_size = (alignment << 1) - chunksize;
1.5059 +
1.5060 + /* Allocate an extent node with which to track the chunk. */
1.5061 + node = base_node_alloc();
1.5062 + if (node == NULL)
1.5063 + return (NULL);
1.5064 +
1.5065 + /*
1.5066 + * Windows requires that there be a 1:1 mapping between VM
1.5067 + * allocation/deallocation operations. Therefore, take care here to
1.5068 + * acquire the final result via one mapping operation.
1.5069 + *
1.5070 + * The MALLOC_PAGEFILE code also benefits from this mapping algorithm,
1.5071 + * since it reduces the number of page files.
1.5072 + */
1.5073 +#ifdef MALLOC_PAGEFILE
1.5074 + if (opt_pagefile) {
1.5075 + pfd = pagefile_init(size);
1.5076 + if (pfd == -1)
1.5077 + return (NULL);
1.5078 + } else
1.5079 +#endif
1.5080 + pfd = -1;
1.5081 +#ifdef JEMALLOC_USES_MAP_ALIGN
1.5082 + ret = pages_map_align(chunk_size, pfd, alignment);
1.5083 +#else
1.5084 + do {
1.5085 + void *over;
1.5086 +
1.5087 + over = chunk_alloc(alloc_size, false, false);
1.5088 + if (over == NULL) {
1.5089 + base_node_dealloc(node);
1.5090 + ret = NULL;
1.5091 + goto RETURN;
1.5092 + }
1.5093 +
1.5094 + offset = (uintptr_t)over & (alignment - 1);
1.5095 + assert((offset & chunksize_mask) == 0);
1.5096 + assert(offset < alloc_size);
1.5097 + ret = (void *)((uintptr_t)over + offset);
1.5098 + chunk_dealloc(over, alloc_size);
1.5099 + ret = pages_map(ret, chunk_size, pfd);
1.5100 + /*
1.5101 + * Failure here indicates a race with another thread, so try
1.5102 + * again.
1.5103 + */
1.5104 + } while (ret == NULL);
1.5105 +#endif
1.5106 + /* Insert node into huge. */
1.5107 + node->addr = ret;
1.5108 + psize = PAGE_CEILING(size);
1.5109 + node->size = psize;
1.5110 +
1.5111 + malloc_mutex_lock(&huge_mtx);
1.5112 + extent_tree_ad_insert(&huge, node);
1.5113 +#ifdef MALLOC_STATS
1.5114 + huge_nmalloc++;
1.5115 + /* See note in huge_alloc() for why huge_allocated += psize is correct
1.5116 + * here even when DECOMMIT is not defined. */
1.5117 + huge_allocated += psize;
1.5118 + huge_mapped += chunk_size;
1.5119 +#endif
1.5120 + malloc_mutex_unlock(&huge_mtx);
1.5121 +
1.5122 +#ifdef MALLOC_DECOMMIT
1.5123 + if (chunk_size - psize > 0) {
1.5124 + pages_decommit((void *)((uintptr_t)ret + psize),
1.5125 + chunk_size - psize);
1.5126 + }
1.5127 +#endif
1.5128 +
1.5129 +#ifdef MALLOC_DECOMMIT
1.5130 + VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, false);
1.5131 +#else
1.5132 + VALGRIND_MALLOCLIKE_BLOCK(ret, chunk_size, 0, false);
1.5133 +#endif
1.5134 +
1.5135 +#ifdef MALLOC_FILL
1.5136 + if (opt_junk)
1.5137 +# ifdef MALLOC_DECOMMIT
1.5138 + memset(ret, 0xa5, psize);
1.5139 +# else
1.5140 + memset(ret, 0xa5, chunk_size);
1.5141 +# endif
1.5142 + else if (opt_zero)
1.5143 +# ifdef MALLOC_DECOMMIT
1.5144 + memset(ret, 0, psize);
1.5145 +# else
1.5146 + memset(ret, 0, chunk_size);
1.5147 +# endif
1.5148 +#endif
1.5149 +
1.5150 +RETURN:
1.5151 +#ifdef MALLOC_PAGEFILE
1.5152 + if (pfd != -1)
1.5153 + pagefile_close(pfd);
1.5154 +#endif
1.5155 + return (ret);
1.5156 +}
1.5157 +
1.5158 +static void *
1.5159 +huge_ralloc(void *ptr, size_t size, size_t oldsize)
1.5160 +{
1.5161 + void *ret;
1.5162 + size_t copysize;
1.5163 +
1.5164 + /* Avoid moving the allocation if the size class would not change. */
1.5165 +
1.5166 + if (oldsize > arena_maxclass &&
1.5167 + CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
1.5168 + size_t psize = PAGE_CEILING(size);
1.5169 +#ifdef MALLOC_FILL
1.5170 + if (opt_poison && size < oldsize) {
1.5171 + memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize
1.5172 + - size);
1.5173 + }
1.5174 +#endif
1.5175 +#ifdef MALLOC_DECOMMIT
1.5176 + if (psize < oldsize) {
1.5177 + extent_node_t *node, key;
1.5178 +
1.5179 + pages_decommit((void *)((uintptr_t)ptr + psize),
1.5180 + oldsize - psize);
1.5181 +
1.5182 + /* Update recorded size. */
1.5183 + malloc_mutex_lock(&huge_mtx);
1.5184 + key.addr = __DECONST(void *, ptr);
1.5185 + node = extent_tree_ad_search(&huge, &key);
1.5186 + assert(node != NULL);
1.5187 + assert(node->size == oldsize);
1.5188 +# ifdef MALLOC_STATS
1.5189 + huge_allocated -= oldsize - psize;
1.5190 + /* No need to change huge_mapped, because we didn't
1.5191 + * (un)map anything. */
1.5192 +# endif
1.5193 + node->size = psize;
1.5194 + malloc_mutex_unlock(&huge_mtx);
1.5195 + } else if (psize > oldsize) {
1.5196 + pages_commit((void *)((uintptr_t)ptr + oldsize),
1.5197 + psize - oldsize);
1.5198 + }
1.5199 +#endif
1.5200 +
1.5201 + /* Although we don't have to commit or decommit anything if
1.5202 + * DECOMMIT is not defined and the size class didn't change, we
1.5203 + * do need to update the recorded size if the size increased,
1.5204 + * so malloc_usable_size doesn't return a value smaller than
1.5205 + * what was requested via realloc(). */
1.5206 +
1.5207 + if (psize > oldsize) {
1.5208 + /* Update recorded size. */
1.5209 + extent_node_t *node, key;
1.5210 + malloc_mutex_lock(&huge_mtx);
1.5211 + key.addr = __DECONST(void *, ptr);
1.5212 + node = extent_tree_ad_search(&huge, &key);
1.5213 + assert(node != NULL);
1.5214 + assert(node->size == oldsize);
1.5215 +# ifdef MALLOC_STATS
1.5216 + huge_allocated += psize - oldsize;
1.5217 + /* No need to change huge_mapped, because we didn't
1.5218 + * (un)map anything. */
1.5219 +# endif
1.5220 + node->size = psize;
1.5221 + malloc_mutex_unlock(&huge_mtx);
1.5222 + }
1.5223 +
1.5224 +#ifdef MALLOC_FILL
1.5225 + if (opt_zero && size > oldsize) {
1.5226 + memset((void *)((uintptr_t)ptr + oldsize), 0, size
1.5227 + - oldsize);
1.5228 + }
1.5229 +#endif
1.5230 + return (ptr);
1.5231 + }
1.5232 +
1.5233 + /*
1.5234 + * If we get here, then size and oldsize are different enough that we
1.5235 + * need to use a different size class. In that case, fall back to
1.5236 + * allocating new space and copying.
1.5237 + */
1.5238 + ret = huge_malloc(size, false);
1.5239 + if (ret == NULL)
1.5240 + return (NULL);
1.5241 +
1.5242 + copysize = (size < oldsize) ? size : oldsize;
1.5243 +#ifdef VM_COPY_MIN
1.5244 + if (copysize >= VM_COPY_MIN)
1.5245 + pages_copy(ret, ptr, copysize);
1.5246 + else
1.5247 +#endif
1.5248 + memcpy(ret, ptr, copysize);
1.5249 + idalloc(ptr);
1.5250 + return (ret);
1.5251 +}
1.5252 +
1.5253 +static void
1.5254 +huge_dalloc(void *ptr)
1.5255 +{
1.5256 + extent_node_t *node, key;
1.5257 +
1.5258 + malloc_mutex_lock(&huge_mtx);
1.5259 +
1.5260 + /* Extract from tree of huge allocations. */
1.5261 + key.addr = ptr;
1.5262 + node = extent_tree_ad_search(&huge, &key);
1.5263 + assert(node != NULL);
1.5264 + assert(node->addr == ptr);
1.5265 + extent_tree_ad_remove(&huge, node);
1.5266 +
1.5267 +#ifdef MALLOC_STATS
1.5268 + huge_ndalloc++;
1.5269 + huge_allocated -= node->size;
1.5270 + huge_mapped -= CHUNK_CEILING(node->size);
1.5271 +#endif
1.5272 +
1.5273 + malloc_mutex_unlock(&huge_mtx);
1.5274 +
1.5275 + /* Unmap chunk. */
1.5276 +#ifdef MALLOC_FILL
1.5277 + if (opt_junk)
1.5278 + memset(node->addr, 0x5a, node->size);
1.5279 +#endif
1.5280 + chunk_dealloc(node->addr, CHUNK_CEILING(node->size));
1.5281 + VALGRIND_FREELIKE_BLOCK(node->addr, 0);
1.5282 +
1.5283 + base_node_dealloc(node);
1.5284 +}
1.5285 +
1.5286 +#ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
1.5287 +#ifdef MOZ_MEMORY_BSD
1.5288 +static inline unsigned
1.5289 +malloc_ncpus(void)
1.5290 +{
1.5291 + unsigned ret;
1.5292 + int mib[2];
1.5293 + size_t len;
1.5294 +
1.5295 + mib[0] = CTL_HW;
1.5296 + mib[1] = HW_NCPU;
1.5297 + len = sizeof(ret);
1.5298 + if (sysctl(mib, 2, &ret, &len, (void *) 0, 0) == -1) {
1.5299 + /* Error. */
1.5300 + return (1);
1.5301 + }
1.5302 +
1.5303 + return (ret);
1.5304 +}
1.5305 +#elif (defined(MOZ_MEMORY_LINUX))
1.5306 +#include <fcntl.h>
1.5307 +
1.5308 +static inline unsigned
1.5309 +malloc_ncpus(void)
1.5310 +{
1.5311 + unsigned ret;
1.5312 + int fd, nread, column;
1.5313 + char buf[1024];
1.5314 + static const char matchstr[] = "processor\t:";
1.5315 + int i;
1.5316 +
1.5317 + /*
1.5318 + * sysconf(3) would be the preferred method for determining the number
1.5319 + * of CPUs, but it uses malloc internally, which causes untennable
1.5320 + * recursion during malloc initialization.
1.5321 + */
1.5322 + fd = open("/proc/cpuinfo", O_RDONLY);
1.5323 + if (fd == -1)
1.5324 + return (1); /* Error. */
1.5325 + /*
1.5326 + * Count the number of occurrences of matchstr at the beginnings of
1.5327 + * lines. This treats hyperthreaded CPUs as multiple processors.
1.5328 + */
1.5329 + column = 0;
1.5330 + ret = 0;
1.5331 + while (true) {
1.5332 + nread = read(fd, &buf, sizeof(buf));
1.5333 + if (nread <= 0)
1.5334 + break; /* EOF or error. */
1.5335 + for (i = 0;i < nread;i++) {
1.5336 + char c = buf[i];
1.5337 + if (c == '\n')
1.5338 + column = 0;
1.5339 + else if (column != -1) {
1.5340 + if (c == matchstr[column]) {
1.5341 + column++;
1.5342 + if (column == sizeof(matchstr) - 1) {
1.5343 + column = -1;
1.5344 + ret++;
1.5345 + }
1.5346 + } else
1.5347 + column = -1;
1.5348 + }
1.5349 + }
1.5350 + }
1.5351 +
1.5352 + if (ret == 0)
1.5353 + ret = 1; /* Something went wrong in the parser. */
1.5354 + close(fd);
1.5355 +
1.5356 + return (ret);
1.5357 +}
1.5358 +#elif (defined(MOZ_MEMORY_DARWIN))
1.5359 +#include <mach/mach_init.h>
1.5360 +#include <mach/mach_host.h>
1.5361 +
1.5362 +static inline unsigned
1.5363 +malloc_ncpus(void)
1.5364 +{
1.5365 + kern_return_t error;
1.5366 + natural_t n;
1.5367 + processor_info_array_t pinfo;
1.5368 + mach_msg_type_number_t pinfocnt;
1.5369 +
1.5370 + error = host_processor_info(mach_host_self(), PROCESSOR_BASIC_INFO,
1.5371 + &n, &pinfo, &pinfocnt);
1.5372 + if (error != KERN_SUCCESS)
1.5373 + return (1); /* Error. */
1.5374 + else
1.5375 + return (n);
1.5376 +}
1.5377 +#elif (defined(MOZ_MEMORY_SOLARIS))
1.5378 +
1.5379 +static inline unsigned
1.5380 +malloc_ncpus(void)
1.5381 +{
1.5382 + return sysconf(_SC_NPROCESSORS_ONLN);
1.5383 +}
1.5384 +#else
1.5385 +static inline unsigned
1.5386 +malloc_ncpus(void)
1.5387 +{
1.5388 +
1.5389 + /*
1.5390 + * We lack a way to determine the number of CPUs on this platform, so
1.5391 + * assume 1 CPU.
1.5392 + */
1.5393 + return (1);
1.5394 +}
1.5395 +#endif
1.5396 +#endif
1.5397 +
1.5398 +static void
1.5399 +malloc_print_stats(void)
1.5400 +{
1.5401 +
1.5402 + if (opt_print_stats) {
1.5403 + char s[UMAX2S_BUFSIZE];
1.5404 + _malloc_message("___ Begin malloc statistics ___\n", "", "",
1.5405 + "");
1.5406 + _malloc_message("Assertions ",
1.5407 +#ifdef NDEBUG
1.5408 + "disabled",
1.5409 +#else
1.5410 + "enabled",
1.5411 +#endif
1.5412 + "\n", "");
1.5413 + _malloc_message("Boolean MALLOC_OPTIONS: ",
1.5414 + opt_abort ? "A" : "a", "", "");
1.5415 +#ifdef MALLOC_FILL
1.5416 + _malloc_message(opt_poison ? "C" : "c", "", "", "");
1.5417 + _malloc_message(opt_junk ? "J" : "j", "", "", "");
1.5418 +#endif
1.5419 +#ifdef MALLOC_PAGEFILE
1.5420 + _malloc_message(opt_pagefile ? "o" : "O", "", "", "");
1.5421 +#endif
1.5422 + _malloc_message("P", "", "", "");
1.5423 +#ifdef MALLOC_UTRACE
1.5424 + _malloc_message(opt_utrace ? "U" : "u", "", "", "");
1.5425 +#endif
1.5426 +#ifdef MALLOC_SYSV
1.5427 + _malloc_message(opt_sysv ? "V" : "v", "", "", "");
1.5428 +#endif
1.5429 +#ifdef MALLOC_XMALLOC
1.5430 + _malloc_message(opt_xmalloc ? "X" : "x", "", "", "");
1.5431 +#endif
1.5432 +#ifdef MALLOC_FILL
1.5433 + _malloc_message(opt_zero ? "Z" : "z", "", "", "");
1.5434 +#endif
1.5435 + _malloc_message("\n", "", "", "");
1.5436 +
1.5437 +#ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
1.5438 + _malloc_message("CPUs: ", umax2s(ncpus, 10, s), "\n", "");
1.5439 +#endif
1.5440 + _malloc_message("Max arenas: ", umax2s(narenas, 10, s), "\n",
1.5441 + "");
1.5442 +#ifdef MALLOC_BALANCE
1.5443 + _malloc_message("Arena balance threshold: ",
1.5444 + umax2s(opt_balance_threshold, 10, s), "\n", "");
1.5445 +#endif
1.5446 + _malloc_message("Pointer size: ", umax2s(sizeof(void *), 10, s),
1.5447 + "\n", "");
1.5448 + _malloc_message("Quantum size: ", umax2s(quantum, 10, s), "\n",
1.5449 + "");
1.5450 + _malloc_message("Max small size: ", umax2s(small_max, 10, s),
1.5451 + "\n", "");
1.5452 + _malloc_message("Max dirty pages per arena: ",
1.5453 + umax2s(opt_dirty_max, 10, s), "\n", "");
1.5454 +
1.5455 + _malloc_message("Chunk size: ", umax2s(chunksize, 10, s), "",
1.5456 + "");
1.5457 + _malloc_message(" (2^", umax2s(opt_chunk_2pow, 10, s), ")\n",
1.5458 + "");
1.5459 +
1.5460 +#ifdef MALLOC_STATS
1.5461 + {
1.5462 + size_t allocated, mapped = 0;
1.5463 +#ifdef MALLOC_BALANCE
1.5464 + uint64_t nbalance = 0;
1.5465 +#endif
1.5466 + unsigned i;
1.5467 + arena_t *arena;
1.5468 +
1.5469 + /* Calculate and print allocated/mapped stats. */
1.5470 +
1.5471 + /* arenas. */
1.5472 + for (i = 0, allocated = 0; i < narenas; i++) {
1.5473 + if (arenas[i] != NULL) {
1.5474 + malloc_spin_lock(&arenas[i]->lock);
1.5475 + allocated +=
1.5476 + arenas[i]->stats.allocated_small;
1.5477 + allocated +=
1.5478 + arenas[i]->stats.allocated_large;
1.5479 + mapped += arenas[i]->stats.mapped;
1.5480 +#ifdef MALLOC_BALANCE
1.5481 + nbalance += arenas[i]->stats.nbalance;
1.5482 +#endif
1.5483 + malloc_spin_unlock(&arenas[i]->lock);
1.5484 + }
1.5485 + }
1.5486 +
1.5487 + /* huge/base. */
1.5488 + malloc_mutex_lock(&huge_mtx);
1.5489 + allocated += huge_allocated;
1.5490 + mapped += huge_mapped;
1.5491 + malloc_mutex_unlock(&huge_mtx);
1.5492 +
1.5493 + malloc_mutex_lock(&base_mtx);
1.5494 + mapped += base_mapped;
1.5495 + malloc_mutex_unlock(&base_mtx);
1.5496 +
1.5497 +#ifdef MOZ_MEMORY_WINDOWS
1.5498 + malloc_printf("Allocated: %lu, mapped: %lu\n",
1.5499 + allocated, mapped);
1.5500 +#else
1.5501 + malloc_printf("Allocated: %zu, mapped: %zu\n",
1.5502 + allocated, mapped);
1.5503 +#endif
1.5504 +
1.5505 +#ifdef MALLOC_BALANCE
1.5506 + malloc_printf("Arena balance reassignments: %llu\n",
1.5507 + nbalance);
1.5508 +#endif
1.5509 +
1.5510 + /* Print chunk stats. */
1.5511 + malloc_printf(
1.5512 + "huge: nmalloc ndalloc allocated\n");
1.5513 +#ifdef MOZ_MEMORY_WINDOWS
1.5514 + malloc_printf(" %12llu %12llu %12lu\n",
1.5515 + huge_nmalloc, huge_ndalloc, huge_allocated);
1.5516 +#else
1.5517 + malloc_printf(" %12llu %12llu %12zu\n",
1.5518 + huge_nmalloc, huge_ndalloc, huge_allocated);
1.5519 +#endif
1.5520 + /* Print stats for each arena. */
1.5521 + for (i = 0; i < narenas; i++) {
1.5522 + arena = arenas[i];
1.5523 + if (arena != NULL) {
1.5524 + malloc_printf(
1.5525 + "\narenas[%u]:\n", i);
1.5526 + malloc_spin_lock(&arena->lock);
1.5527 + stats_print(arena);
1.5528 + malloc_spin_unlock(&arena->lock);
1.5529 + }
1.5530 + }
1.5531 + }
1.5532 +#endif /* #ifdef MALLOC_STATS */
1.5533 + _malloc_message("--- End malloc statistics ---\n", "", "", "");
1.5534 + }
1.5535 +}
1.5536 +
1.5537 +/*
1.5538 + * FreeBSD's pthreads implementation calls malloc(3), so the malloc
1.5539 + * implementation has to take pains to avoid infinite recursion during
1.5540 + * initialization.
1.5541 + */
1.5542 +#if (defined(MOZ_MEMORY_WINDOWS) || defined(MOZ_MEMORY_DARWIN))
1.5543 +#define malloc_init() false
1.5544 +#else
1.5545 +static inline bool
1.5546 +malloc_init(void)
1.5547 +{
1.5548 +
1.5549 + if (malloc_initialized == false)
1.5550 + return (malloc_init_hard());
1.5551 +
1.5552 + return (false);
1.5553 +}
1.5554 +#endif
1.5555 +
1.5556 +#if !defined(MOZ_MEMORY_WINDOWS)
1.5557 +static
1.5558 +#endif
1.5559 +bool
1.5560 +malloc_init_hard(void)
1.5561 +{
1.5562 + unsigned i;
1.5563 + char buf[PATH_MAX + 1];
1.5564 + const char *opts;
1.5565 + long result;
1.5566 +#ifndef MOZ_MEMORY_WINDOWS
1.5567 + int linklen;
1.5568 +#endif
1.5569 +#ifdef MOZ_MEMORY_DARWIN
1.5570 + malloc_zone_t* default_zone;
1.5571 +#endif
1.5572 +
1.5573 +#ifndef MOZ_MEMORY_WINDOWS
1.5574 + malloc_mutex_lock(&init_lock);
1.5575 +#endif
1.5576 +
1.5577 + if (malloc_initialized) {
1.5578 + /*
1.5579 + * Another thread initialized the allocator before this one
1.5580 + * acquired init_lock.
1.5581 + */
1.5582 +#ifndef MOZ_MEMORY_WINDOWS
1.5583 + malloc_mutex_unlock(&init_lock);
1.5584 +#endif
1.5585 + return (false);
1.5586 + }
1.5587 +
1.5588 +#ifdef MOZ_MEMORY_WINDOWS
1.5589 + /* get a thread local storage index */
1.5590 + tlsIndex = TlsAlloc();
1.5591 +#endif
1.5592 +
1.5593 + /* Get page size and number of CPUs */
1.5594 +#ifdef MOZ_MEMORY_WINDOWS
1.5595 + {
1.5596 + SYSTEM_INFO info;
1.5597 +
1.5598 + GetSystemInfo(&info);
1.5599 + result = info.dwPageSize;
1.5600 +
1.5601 +#ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
1.5602 + ncpus = info.dwNumberOfProcessors;
1.5603 +#endif
1.5604 + }
1.5605 +#else
1.5606 +#ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
1.5607 + ncpus = malloc_ncpus();
1.5608 +#endif
1.5609 +
1.5610 + result = sysconf(_SC_PAGESIZE);
1.5611 + assert(result != -1);
1.5612 +#endif
1.5613 +
1.5614 + /* We assume that the page size is a power of 2. */
1.5615 + assert(((result - 1) & result) == 0);
1.5616 +#ifdef MALLOC_STATIC_SIZES
1.5617 + if (pagesize % (size_t) result) {
1.5618 + _malloc_message(_getprogname(),
1.5619 + "Compile-time page size does not divide the runtime one.\n",
1.5620 + "", "");
1.5621 + abort();
1.5622 + }
1.5623 +#else
1.5624 + pagesize = (size_t) result;
1.5625 + pagesize_mask = (size_t) result - 1;
1.5626 + pagesize_2pow = ffs((int)result) - 1;
1.5627 +#endif
1.5628 +
1.5629 +#ifdef MALLOC_PAGEFILE
1.5630 + /*
1.5631 + * Determine where to create page files. It is insufficient to
1.5632 + * unconditionally use P_tmpdir (typically "/tmp"), since for some
1.5633 + * operating systems /tmp is a separate filesystem that is rather small.
1.5634 + * Therefore prefer, in order, the following locations:
1.5635 + *
1.5636 + * 1) MALLOC_TMPDIR
1.5637 + * 2) TMPDIR
1.5638 + * 3) P_tmpdir
1.5639 + */
1.5640 + {
1.5641 + char *s;
1.5642 + size_t slen;
1.5643 + static const char suffix[] = "/jemalloc.XXXXXX";
1.5644 +
1.5645 + if ((s = getenv("MALLOC_TMPDIR")) == NULL && (s =
1.5646 + getenv("TMPDIR")) == NULL)
1.5647 + s = P_tmpdir;
1.5648 + slen = strlen(s);
1.5649 + if (slen + sizeof(suffix) > sizeof(pagefile_templ)) {
1.5650 + _malloc_message(_getprogname(),
1.5651 + ": (malloc) Page file path too long\n",
1.5652 + "", "");
1.5653 + abort();
1.5654 + }
1.5655 + memcpy(pagefile_templ, s, slen);
1.5656 + memcpy(&pagefile_templ[slen], suffix, sizeof(suffix));
1.5657 + }
1.5658 +#endif
1.5659 +
1.5660 + for (i = 0; i < 3; i++) {
1.5661 + unsigned j;
1.5662 +
1.5663 + /* Get runtime configuration. */
1.5664 + switch (i) {
1.5665 + case 0:
1.5666 +#ifndef MOZ_MEMORY_WINDOWS
1.5667 + if ((linklen = readlink("/etc/malloc.conf", buf,
1.5668 + sizeof(buf) - 1)) != -1) {
1.5669 + /*
1.5670 + * Use the contents of the "/etc/malloc.conf"
1.5671 + * symbolic link's name.
1.5672 + */
1.5673 + buf[linklen] = '\0';
1.5674 + opts = buf;
1.5675 + } else
1.5676 +#endif
1.5677 + {
1.5678 + /* No configuration specified. */
1.5679 + buf[0] = '\0';
1.5680 + opts = buf;
1.5681 + }
1.5682 + break;
1.5683 + case 1:
1.5684 + if (issetugid() == 0 && (opts =
1.5685 + getenv("MALLOC_OPTIONS")) != NULL) {
1.5686 + /*
1.5687 + * Do nothing; opts is already initialized to
1.5688 + * the value of the MALLOC_OPTIONS environment
1.5689 + * variable.
1.5690 + */
1.5691 + } else {
1.5692 + /* No configuration specified. */
1.5693 + buf[0] = '\0';
1.5694 + opts = buf;
1.5695 + }
1.5696 + break;
1.5697 + case 2:
1.5698 + if (_malloc_options != NULL) {
1.5699 + /*
1.5700 + * Use options that were compiled into the
1.5701 + * program.
1.5702 + */
1.5703 + opts = _malloc_options;
1.5704 + } else {
1.5705 + /* No configuration specified. */
1.5706 + buf[0] = '\0';
1.5707 + opts = buf;
1.5708 + }
1.5709 + break;
1.5710 + default:
1.5711 + /* NOTREACHED */
1.5712 + buf[0] = '\0';
1.5713 + opts = buf;
1.5714 + assert(false);
1.5715 + }
1.5716 +
1.5717 + for (j = 0; opts[j] != '\0'; j++) {
1.5718 + unsigned k, nreps;
1.5719 + bool nseen;
1.5720 +
1.5721 + /* Parse repetition count, if any. */
1.5722 + for (nreps = 0, nseen = false;; j++, nseen = true) {
1.5723 + switch (opts[j]) {
1.5724 + case '0': case '1': case '2': case '3':
1.5725 + case '4': case '5': case '6': case '7':
1.5726 + case '8': case '9':
1.5727 + nreps *= 10;
1.5728 + nreps += opts[j] - '0';
1.5729 + break;
1.5730 + default:
1.5731 + goto MALLOC_OUT;
1.5732 + }
1.5733 + }
1.5734 +MALLOC_OUT:
1.5735 + if (nseen == false)
1.5736 + nreps = 1;
1.5737 +
1.5738 + for (k = 0; k < nreps; k++) {
1.5739 + switch (opts[j]) {
1.5740 + case 'a':
1.5741 + opt_abort = false;
1.5742 + break;
1.5743 + case 'A':
1.5744 + opt_abort = true;
1.5745 + break;
1.5746 + case 'b':
1.5747 +#ifdef MALLOC_BALANCE
1.5748 + opt_balance_threshold >>= 1;
1.5749 +#endif
1.5750 + break;
1.5751 + case 'B':
1.5752 +#ifdef MALLOC_BALANCE
1.5753 + if (opt_balance_threshold == 0)
1.5754 + opt_balance_threshold = 1;
1.5755 + else if ((opt_balance_threshold << 1)
1.5756 + > opt_balance_threshold)
1.5757 + opt_balance_threshold <<= 1;
1.5758 +#endif
1.5759 + break;
1.5760 +#ifdef MALLOC_FILL
1.5761 +#ifndef MALLOC_PRODUCTION
1.5762 + case 'c':
1.5763 + opt_poison = false;
1.5764 + break;
1.5765 + case 'C':
1.5766 + opt_poison = true;
1.5767 + break;
1.5768 +#endif
1.5769 +#endif
1.5770 + case 'f':
1.5771 + opt_dirty_max >>= 1;
1.5772 + break;
1.5773 + case 'F':
1.5774 + if (opt_dirty_max == 0)
1.5775 + opt_dirty_max = 1;
1.5776 + else if ((opt_dirty_max << 1) != 0)
1.5777 + opt_dirty_max <<= 1;
1.5778 + break;
1.5779 +#ifdef MALLOC_FILL
1.5780 +#ifndef MALLOC_PRODUCTION
1.5781 + case 'j':
1.5782 + opt_junk = false;
1.5783 + break;
1.5784 + case 'J':
1.5785 + opt_junk = true;
1.5786 + break;
1.5787 +#endif
1.5788 +#endif
1.5789 +#ifndef MALLOC_STATIC_SIZES
1.5790 + case 'k':
1.5791 + /*
1.5792 + * Chunks always require at least one
1.5793 + * header page, so chunks can never be
1.5794 + * smaller than two pages.
1.5795 + */
1.5796 + if (opt_chunk_2pow > pagesize_2pow + 1)
1.5797 + opt_chunk_2pow--;
1.5798 + break;
1.5799 + case 'K':
1.5800 + if (opt_chunk_2pow + 1 <
1.5801 + (sizeof(size_t) << 3))
1.5802 + opt_chunk_2pow++;
1.5803 + break;
1.5804 +#endif
1.5805 + case 'n':
1.5806 + opt_narenas_lshift--;
1.5807 + break;
1.5808 + case 'N':
1.5809 + opt_narenas_lshift++;
1.5810 + break;
1.5811 +#ifdef MALLOC_PAGEFILE
1.5812 + case 'o':
1.5813 + /* Do not over-commit. */
1.5814 + opt_pagefile = true;
1.5815 + break;
1.5816 + case 'O':
1.5817 + /* Allow over-commit. */
1.5818 + opt_pagefile = false;
1.5819 + break;
1.5820 +#endif
1.5821 + case 'p':
1.5822 + opt_print_stats = false;
1.5823 + break;
1.5824 + case 'P':
1.5825 + opt_print_stats = true;
1.5826 + break;
1.5827 +#ifndef MALLOC_STATIC_SIZES
1.5828 + case 'q':
1.5829 + if (opt_quantum_2pow > QUANTUM_2POW_MIN)
1.5830 + opt_quantum_2pow--;
1.5831 + break;
1.5832 + case 'Q':
1.5833 + if (opt_quantum_2pow < pagesize_2pow -
1.5834 + 1)
1.5835 + opt_quantum_2pow++;
1.5836 + break;
1.5837 + case 's':
1.5838 + if (opt_small_max_2pow >
1.5839 + QUANTUM_2POW_MIN)
1.5840 + opt_small_max_2pow--;
1.5841 + break;
1.5842 + case 'S':
1.5843 + if (opt_small_max_2pow < pagesize_2pow
1.5844 + - 1)
1.5845 + opt_small_max_2pow++;
1.5846 + break;
1.5847 +#endif
1.5848 +#ifdef MALLOC_UTRACE
1.5849 + case 'u':
1.5850 + opt_utrace = false;
1.5851 + break;
1.5852 + case 'U':
1.5853 + opt_utrace = true;
1.5854 + break;
1.5855 +#endif
1.5856 +#ifdef MALLOC_SYSV
1.5857 + case 'v':
1.5858 + opt_sysv = false;
1.5859 + break;
1.5860 + case 'V':
1.5861 + opt_sysv = true;
1.5862 + break;
1.5863 +#endif
1.5864 +#ifdef MALLOC_XMALLOC
1.5865 + case 'x':
1.5866 + opt_xmalloc = false;
1.5867 + break;
1.5868 + case 'X':
1.5869 + opt_xmalloc = true;
1.5870 + break;
1.5871 +#endif
1.5872 +#ifdef MALLOC_FILL
1.5873 +#ifndef MALLOC_PRODUCTION
1.5874 + case 'z':
1.5875 + opt_zero = false;
1.5876 + break;
1.5877 + case 'Z':
1.5878 + opt_zero = true;
1.5879 + break;
1.5880 +#endif
1.5881 +#endif
1.5882 + default: {
1.5883 + char cbuf[2];
1.5884 +
1.5885 + cbuf[0] = opts[j];
1.5886 + cbuf[1] = '\0';
1.5887 + _malloc_message(_getprogname(),
1.5888 + ": (malloc) Unsupported character "
1.5889 + "in malloc options: '", cbuf,
1.5890 + "'\n");
1.5891 + }
1.5892 + }
1.5893 + }
1.5894 + }
1.5895 + }
1.5896 +
1.5897 + /* Take care to call atexit() only once. */
1.5898 + if (opt_print_stats) {
1.5899 +#ifndef MOZ_MEMORY_WINDOWS
1.5900 + /* Print statistics at exit. */
1.5901 + atexit(malloc_print_stats);
1.5902 +#endif
1.5903 + }
1.5904 +
1.5905 +#if !defined(MOZ_MEMORY_WINDOWS) && !defined(MOZ_MEMORY_DARWIN)
1.5906 + /* Prevent potential deadlock on malloc locks after fork. */
1.5907 + pthread_atfork(_malloc_prefork, _malloc_postfork, _malloc_postfork);
1.5908 +#endif
1.5909 +
1.5910 +#ifndef MALLOC_STATIC_SIZES
1.5911 + /* Set variables according to the value of opt_small_max_2pow. */
1.5912 + if (opt_small_max_2pow < opt_quantum_2pow)
1.5913 + opt_small_max_2pow = opt_quantum_2pow;
1.5914 + small_max = (1U << opt_small_max_2pow);
1.5915 +
1.5916 + /* Set bin-related variables. */
1.5917 + bin_maxclass = (pagesize >> 1);
1.5918 + assert(opt_quantum_2pow >= TINY_MIN_2POW);
1.5919 + ntbins = opt_quantum_2pow - TINY_MIN_2POW;
1.5920 + assert(ntbins <= opt_quantum_2pow);
1.5921 + nqbins = (small_max >> opt_quantum_2pow);
1.5922 + nsbins = pagesize_2pow - opt_small_max_2pow - 1;
1.5923 +
1.5924 + /* Set variables according to the value of opt_quantum_2pow. */
1.5925 + quantum = (1U << opt_quantum_2pow);
1.5926 + quantum_mask = quantum - 1;
1.5927 + if (ntbins > 0)
1.5928 + small_min = (quantum >> 1) + 1;
1.5929 + else
1.5930 + small_min = 1;
1.5931 + assert(small_min <= quantum);
1.5932 +
1.5933 + /* Set variables according to the value of opt_chunk_2pow. */
1.5934 + chunksize = (1LU << opt_chunk_2pow);
1.5935 + chunksize_mask = chunksize - 1;
1.5936 + chunk_npages = (chunksize >> pagesize_2pow);
1.5937 +
1.5938 + arena_chunk_header_npages = calculate_arena_header_pages();
1.5939 + arena_maxclass = calculate_arena_maxclass();
1.5940 +#endif
1.5941 +
1.5942 +#ifdef JEMALLOC_USES_MAP_ALIGN
1.5943 + /*
1.5944 + * When using MAP_ALIGN, the alignment parameter must be a power of two
1.5945 + * multiple of the system pagesize, or mmap will fail.
1.5946 + */
1.5947 + assert((chunksize % pagesize) == 0);
1.5948 + assert((1 << (ffs(chunksize / pagesize) - 1)) == (chunksize/pagesize));
1.5949 +#endif
1.5950 +
1.5951 + UTRACE(0, 0, 0);
1.5952 +
1.5953 + /* Various sanity checks that regard configuration. */
1.5954 + assert(quantum >= sizeof(void *));
1.5955 + assert(quantum <= pagesize);
1.5956 + assert(chunksize >= pagesize);
1.5957 + assert(quantum * 4 <= chunksize);
1.5958 +
1.5959 + /* Initialize chunks data. */
1.5960 + malloc_mutex_init(&huge_mtx);
1.5961 + extent_tree_ad_new(&huge);
1.5962 +#ifdef MALLOC_STATS
1.5963 + huge_nmalloc = 0;
1.5964 + huge_ndalloc = 0;
1.5965 + huge_allocated = 0;
1.5966 + huge_mapped = 0;
1.5967 +#endif
1.5968 +
1.5969 + /* Initialize base allocation data structures. */
1.5970 +#ifdef MALLOC_STATS
1.5971 + base_mapped = 0;
1.5972 + base_committed = 0;
1.5973 +#endif
1.5974 + base_nodes = NULL;
1.5975 + malloc_mutex_init(&base_mtx);
1.5976 +
1.5977 +#ifdef MOZ_MEMORY_NARENAS_DEFAULT_ONE
1.5978 + narenas = 1;
1.5979 +#else
1.5980 + if (ncpus > 1) {
1.5981 + /*
1.5982 + * For SMP systems, create four times as many arenas as there
1.5983 + * are CPUs by default.
1.5984 + */
1.5985 + opt_narenas_lshift += 2;
1.5986 + }
1.5987 +
1.5988 + /* Determine how many arenas to use. */
1.5989 + narenas = ncpus;
1.5990 +#endif
1.5991 + if (opt_narenas_lshift > 0) {
1.5992 + if ((narenas << opt_narenas_lshift) > narenas)
1.5993 + narenas <<= opt_narenas_lshift;
1.5994 + /*
1.5995 + * Make sure not to exceed the limits of what base_alloc() can
1.5996 + * handle.
1.5997 + */
1.5998 + if (narenas * sizeof(arena_t *) > chunksize)
1.5999 + narenas = chunksize / sizeof(arena_t *);
1.6000 + } else if (opt_narenas_lshift < 0) {
1.6001 + if ((narenas >> -opt_narenas_lshift) < narenas)
1.6002 + narenas >>= -opt_narenas_lshift;
1.6003 + /* Make sure there is at least one arena. */
1.6004 + if (narenas == 0)
1.6005 + narenas = 1;
1.6006 + }
1.6007 +#ifdef MALLOC_BALANCE
1.6008 + assert(narenas != 0);
1.6009 + for (narenas_2pow = 0;
1.6010 + (narenas >> (narenas_2pow + 1)) != 0;
1.6011 + narenas_2pow++);
1.6012 +#endif
1.6013 +
1.6014 +#ifdef NO_TLS
1.6015 + if (narenas > 1) {
1.6016 + static const unsigned primes[] = {1, 3, 5, 7, 11, 13, 17, 19,
1.6017 + 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,
1.6018 + 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149,
1.6019 + 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211,
1.6020 + 223, 227, 229, 233, 239, 241, 251, 257, 263};
1.6021 + unsigned nprimes, parenas;
1.6022 +
1.6023 + /*
1.6024 + * Pick a prime number of hash arenas that is more than narenas
1.6025 + * so that direct hashing of pthread_self() pointers tends to
1.6026 + * spread allocations evenly among the arenas.
1.6027 + */
1.6028 + assert((narenas & 1) == 0); /* narenas must be even. */
1.6029 + nprimes = (sizeof(primes) >> SIZEOF_INT_2POW);
1.6030 + parenas = primes[nprimes - 1]; /* In case not enough primes. */
1.6031 + for (i = 1; i < nprimes; i++) {
1.6032 + if (primes[i] > narenas) {
1.6033 + parenas = primes[i];
1.6034 + break;
1.6035 + }
1.6036 + }
1.6037 + narenas = parenas;
1.6038 + }
1.6039 +#endif
1.6040 +
1.6041 +#ifndef NO_TLS
1.6042 +# ifndef MALLOC_BALANCE
1.6043 + next_arena = 0;
1.6044 +# endif
1.6045 +#endif
1.6046 +
1.6047 + /* Allocate and initialize arenas. */
1.6048 + arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
1.6049 + if (arenas == NULL) {
1.6050 +#ifndef MOZ_MEMORY_WINDOWS
1.6051 + malloc_mutex_unlock(&init_lock);
1.6052 +#endif
1.6053 + return (true);
1.6054 + }
1.6055 + /*
1.6056 + * Zero the array. In practice, this should always be pre-zeroed,
1.6057 + * since it was just mmap()ed, but let's be sure.
1.6058 + */
1.6059 + memset(arenas, 0, sizeof(arena_t *) * narenas);
1.6060 +
1.6061 + /*
1.6062 + * Initialize one arena here. The rest are lazily created in
1.6063 + * choose_arena_hard().
1.6064 + */
1.6065 + arenas_extend(0);
1.6066 + if (arenas[0] == NULL) {
1.6067 +#ifndef MOZ_MEMORY_WINDOWS
1.6068 + malloc_mutex_unlock(&init_lock);
1.6069 +#endif
1.6070 + return (true);
1.6071 + }
1.6072 +#ifndef NO_TLS
1.6073 + /*
1.6074 + * Assign the initial arena to the initial thread, in order to avoid
1.6075 + * spurious creation of an extra arena if the application switches to
1.6076 + * threaded mode.
1.6077 + */
1.6078 +#ifdef MOZ_MEMORY_WINDOWS
1.6079 + TlsSetValue(tlsIndex, arenas[0]);
1.6080 +#else
1.6081 + arenas_map = arenas[0];
1.6082 +#endif
1.6083 +#endif
1.6084 +
1.6085 + /*
1.6086 + * Seed here for the initial thread, since choose_arena_hard() is only
1.6087 + * called for other threads. The seed value doesn't really matter.
1.6088 + */
1.6089 +#ifdef MALLOC_BALANCE
1.6090 + SPRN(balance, 42);
1.6091 +#endif
1.6092 +
1.6093 + malloc_spin_init(&arenas_lock);
1.6094 +
1.6095 +#ifdef MALLOC_VALIDATE
1.6096 + chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - opt_chunk_2pow);
1.6097 + if (chunk_rtree == NULL)
1.6098 + return (true);
1.6099 +#endif
1.6100 +
1.6101 + malloc_initialized = true;
1.6102 +
1.6103 +#if defined(NEEDS_PTHREAD_MMAP_UNALIGNED_TSD)
1.6104 + if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
1.6105 + malloc_printf("<jemalloc>: Error in pthread_key_create()\n");
1.6106 + }
1.6107 +#endif
1.6108 +
1.6109 +#if defined(MOZ_MEMORY_DARWIN) && !defined(MOZ_REPLACE_MALLOC)
1.6110 + /*
1.6111 + * Overwrite the default memory allocator to use jemalloc everywhere.
1.6112 + */
1.6113 + default_zone = malloc_default_zone();
1.6114 +
1.6115 + /*
1.6116 + * We only use jemalloc with MacOS 10.6 and 10.7. jemalloc is disabled
1.6117 + * on 32-bit builds (10.5 and 32-bit 10.6) due to bug 702250, an
1.6118 + * apparent MacOS bug. In fact, this code isn't even compiled on
1.6119 + * 32-bit builds.
1.6120 + *
1.6121 + * We'll have to update our code to work with newer versions, because
1.6122 + * the malloc zone layout is likely to change.
1.6123 + */
1.6124 +
1.6125 + osx_use_jemalloc = (default_zone->version == SNOW_LEOPARD_MALLOC_ZONE_T_VERSION ||
1.6126 + default_zone->version == LION_MALLOC_ZONE_T_VERSION);
1.6127 +
1.6128 + /* Allow us dynamically turn off jemalloc for testing. */
1.6129 + if (getenv("NO_MAC_JEMALLOC")) {
1.6130 + osx_use_jemalloc = false;
1.6131 +#ifdef __i386__
1.6132 + malloc_printf("Warning: NO_MAC_JEMALLOC has no effect on "
1.6133 + "i386 machines (such as this one).\n");
1.6134 +#endif
1.6135 + }
1.6136 +
1.6137 + if (osx_use_jemalloc) {
1.6138 + /*
1.6139 + * Convert the default szone to an "overlay zone" that is capable
1.6140 + * of deallocating szone-allocated objects, but allocating new
1.6141 + * objects from jemalloc.
1.6142 + */
1.6143 + size_t size = zone_version_size(default_zone->version);
1.6144 + szone2ozone(default_zone, size);
1.6145 + }
1.6146 + else {
1.6147 + szone = default_zone;
1.6148 + }
1.6149 +#endif
1.6150 +
1.6151 +#ifndef MOZ_MEMORY_WINDOWS
1.6152 + malloc_mutex_unlock(&init_lock);
1.6153 +#endif
1.6154 + return (false);
1.6155 +}
1.6156 +
1.6157 +/* XXX Why not just expose malloc_print_stats()? */
1.6158 +#ifdef MOZ_MEMORY_WINDOWS
1.6159 +void
1.6160 +malloc_shutdown()
1.6161 +{
1.6162 +
1.6163 + malloc_print_stats();
1.6164 +}
1.6165 +#endif
1.6166 +
1.6167 +/*
1.6168 + * End general internal functions.
1.6169 + */
1.6170 +/******************************************************************************/
1.6171 +/*
1.6172 + * Begin malloc(3)-compatible functions.
1.6173 + */
1.6174 +
1.6175 +/*
1.6176 + * Even though we compile with MOZ_MEMORY, we may have to dynamically decide
1.6177 + * not to use jemalloc, as discussed above. However, we call jemalloc
1.6178 + * functions directly from mozalloc. Since it's pretty dangerous to mix the
1.6179 + * allocators, we need to call the OSX allocators from the functions below,
1.6180 + * when osx_use_jemalloc is not (dynamically) set.
1.6181 + *
1.6182 + * Note that we assume jemalloc is enabled on i386. This is safe because the
1.6183 + * only i386 versions of MacOS are 10.5 and 10.6, which we support. We have to
1.6184 + * do this because madvise isn't in the malloc zone struct for 10.5.
1.6185 + *
1.6186 + * This means that NO_MAC_JEMALLOC doesn't work on i386.
1.6187 + */
1.6188 +#if defined(MOZ_MEMORY_DARWIN) && !defined(__i386__) && !defined(MOZ_REPLACE_MALLOC)
1.6189 +#define DARWIN_ONLY(A) if (!osx_use_jemalloc) { A; }
1.6190 +#else
1.6191 +#define DARWIN_ONLY(A)
1.6192 +#endif
1.6193 +
1.6194 +MOZ_MEMORY_API void *
1.6195 +malloc_impl(size_t size)
1.6196 +{
1.6197 + void *ret;
1.6198 +
1.6199 + DARWIN_ONLY(return (szone->malloc)(szone, size));
1.6200 +
1.6201 + if (malloc_init()) {
1.6202 + ret = NULL;
1.6203 + goto RETURN;
1.6204 + }
1.6205 +
1.6206 + if (size == 0) {
1.6207 +#ifdef MALLOC_SYSV
1.6208 + if (opt_sysv == false)
1.6209 +#endif
1.6210 + size = 1;
1.6211 +#ifdef MALLOC_SYSV
1.6212 + else {
1.6213 + ret = NULL;
1.6214 + goto RETURN;
1.6215 + }
1.6216 +#endif
1.6217 + }
1.6218 +
1.6219 + ret = imalloc(size);
1.6220 +
1.6221 +RETURN:
1.6222 + if (ret == NULL) {
1.6223 +#ifdef MALLOC_XMALLOC
1.6224 + if (opt_xmalloc) {
1.6225 + _malloc_message(_getprogname(),
1.6226 + ": (malloc) Error in malloc(): out of memory\n", "",
1.6227 + "");
1.6228 + abort();
1.6229 + }
1.6230 +#endif
1.6231 + errno = ENOMEM;
1.6232 + }
1.6233 +
1.6234 + UTRACE(0, size, ret);
1.6235 + return (ret);
1.6236 +}
1.6237 +
1.6238 +/*
1.6239 + * In ELF systems the default visibility allows symbols to be preempted at
1.6240 + * runtime. This in turn prevents the uses of memalign in this file from being
1.6241 + * optimized. What we do in here is define two aliasing symbols (they point to
1.6242 + * the same code): memalign and memalign_internal. The internal version has
1.6243 + * hidden visibility and is used in every reference from this file.
1.6244 + *
1.6245 + * For more information on this technique, see section 2.2.7 (Avoid Using
1.6246 + * Exported Symbols) in http://www.akkadia.org/drepper/dsohowto.pdf.
1.6247 + */
1.6248 +
1.6249 +#ifndef MOZ_REPLACE_MALLOC
1.6250 +#if defined(__GNUC__) && !defined(MOZ_MEMORY_DARWIN)
1.6251 +#define MOZ_MEMORY_ELF
1.6252 +#endif
1.6253 +
1.6254 +#ifdef MOZ_MEMORY_SOLARIS
1.6255 +# ifdef __SUNPRO_C
1.6256 +void *
1.6257 +memalign_impl(size_t alignment, size_t size);
1.6258 +#pragma no_inline(memalign_impl)
1.6259 +# elif (defined(__GNUC__))
1.6260 +__attribute__((noinline))
1.6261 +# endif
1.6262 +#else
1.6263 +#if (defined(MOZ_MEMORY_ELF))
1.6264 +__attribute__((visibility ("hidden")))
1.6265 +#endif
1.6266 +#endif
1.6267 +#endif /* MOZ_REPLACE_MALLOC */
1.6268 +
1.6269 +#ifdef MOZ_MEMORY_ELF
1.6270 +#define MEMALIGN memalign_internal
1.6271 +#else
1.6272 +#define MEMALIGN memalign_impl
1.6273 +#endif
1.6274 +
1.6275 +#ifndef MOZ_MEMORY_ELF
1.6276 +MOZ_MEMORY_API
1.6277 +#endif
1.6278 +void *
1.6279 +MEMALIGN(size_t alignment, size_t size)
1.6280 +{
1.6281 + void *ret;
1.6282 +
1.6283 + DARWIN_ONLY(return (szone->memalign)(szone, alignment, size));
1.6284 +
1.6285 + assert(((alignment - 1) & alignment) == 0);
1.6286 +
1.6287 + if (malloc_init()) {
1.6288 + ret = NULL;
1.6289 + goto RETURN;
1.6290 + }
1.6291 +
1.6292 + if (size == 0) {
1.6293 +#ifdef MALLOC_SYSV
1.6294 + if (opt_sysv == false)
1.6295 +#endif
1.6296 + size = 1;
1.6297 +#ifdef MALLOC_SYSV
1.6298 + else {
1.6299 + ret = NULL;
1.6300 + goto RETURN;
1.6301 + }
1.6302 +#endif
1.6303 + }
1.6304 +
1.6305 + alignment = alignment < sizeof(void*) ? sizeof(void*) : alignment;
1.6306 + ret = ipalloc(alignment, size);
1.6307 +
1.6308 +RETURN:
1.6309 +#ifdef MALLOC_XMALLOC
1.6310 + if (opt_xmalloc && ret == NULL) {
1.6311 + _malloc_message(_getprogname(),
1.6312 + ": (malloc) Error in memalign(): out of memory\n", "", "");
1.6313 + abort();
1.6314 + }
1.6315 +#endif
1.6316 + UTRACE(0, size, ret);
1.6317 + return (ret);
1.6318 +}
1.6319 +
1.6320 +#ifdef MOZ_MEMORY_ELF
1.6321 +extern void *
1.6322 +memalign_impl(size_t alignment, size_t size) __attribute__((alias ("memalign_internal"), visibility ("default")));
1.6323 +#endif
1.6324 +
1.6325 +MOZ_MEMORY_API int
1.6326 +posix_memalign_impl(void **memptr, size_t alignment, size_t size)
1.6327 +{
1.6328 + void *result;
1.6329 +
1.6330 + /* Make sure that alignment is a large enough power of 2. */
1.6331 + if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) {
1.6332 +#ifdef MALLOC_XMALLOC
1.6333 + if (opt_xmalloc) {
1.6334 + _malloc_message(_getprogname(),
1.6335 + ": (malloc) Error in posix_memalign(): "
1.6336 + "invalid alignment\n", "", "");
1.6337 + abort();
1.6338 + }
1.6339 +#endif
1.6340 + return (EINVAL);
1.6341 + }
1.6342 +
1.6343 + /* The 0-->1 size promotion is done in the memalign() call below */
1.6344 +
1.6345 + result = MEMALIGN(alignment, size);
1.6346 +
1.6347 + if (result == NULL)
1.6348 + return (ENOMEM);
1.6349 +
1.6350 + *memptr = result;
1.6351 + return (0);
1.6352 +}
1.6353 +
1.6354 +MOZ_MEMORY_API void *
1.6355 +aligned_alloc_impl(size_t alignment, size_t size)
1.6356 +{
1.6357 + if (size % alignment) {
1.6358 +#ifdef MALLOC_XMALLOC
1.6359 + if (opt_xmalloc) {
1.6360 + _malloc_message(_getprogname(),
1.6361 + ": (malloc) Error in aligned_alloc(): "
1.6362 + "size is not multiple of alignment\n", "", "");
1.6363 + abort();
1.6364 + }
1.6365 +#endif
1.6366 + return (NULL);
1.6367 + }
1.6368 + return MEMALIGN(alignment, size);
1.6369 +}
1.6370 +
1.6371 +MOZ_MEMORY_API void *
1.6372 +valloc_impl(size_t size)
1.6373 +{
1.6374 + return (MEMALIGN(pagesize, size));
1.6375 +}
1.6376 +
1.6377 +MOZ_MEMORY_API void *
1.6378 +calloc_impl(size_t num, size_t size)
1.6379 +{
1.6380 + void *ret;
1.6381 + size_t num_size;
1.6382 +
1.6383 + DARWIN_ONLY(return (szone->calloc)(szone, num, size));
1.6384 +
1.6385 + if (malloc_init()) {
1.6386 + num_size = 0;
1.6387 + ret = NULL;
1.6388 + goto RETURN;
1.6389 + }
1.6390 +
1.6391 + num_size = num * size;
1.6392 + if (num_size == 0) {
1.6393 +#ifdef MALLOC_SYSV
1.6394 + if ((opt_sysv == false) && ((num == 0) || (size == 0)))
1.6395 +#endif
1.6396 + num_size = 1;
1.6397 +#ifdef MALLOC_SYSV
1.6398 + else {
1.6399 + ret = NULL;
1.6400 + goto RETURN;
1.6401 + }
1.6402 +#endif
1.6403 + /*
1.6404 + * Try to avoid division here. We know that it isn't possible to
1.6405 + * overflow during multiplication if neither operand uses any of the
1.6406 + * most significant half of the bits in a size_t.
1.6407 + */
1.6408 + } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
1.6409 + && (num_size / size != num)) {
1.6410 + /* size_t overflow. */
1.6411 + ret = NULL;
1.6412 + goto RETURN;
1.6413 + }
1.6414 +
1.6415 + ret = icalloc(num_size);
1.6416 +
1.6417 +RETURN:
1.6418 + if (ret == NULL) {
1.6419 +#ifdef MALLOC_XMALLOC
1.6420 + if (opt_xmalloc) {
1.6421 + _malloc_message(_getprogname(),
1.6422 + ": (malloc) Error in calloc(): out of memory\n", "",
1.6423 + "");
1.6424 + abort();
1.6425 + }
1.6426 +#endif
1.6427 + errno = ENOMEM;
1.6428 + }
1.6429 +
1.6430 + UTRACE(0, num_size, ret);
1.6431 + return (ret);
1.6432 +}
1.6433 +
1.6434 +MOZ_MEMORY_API void *
1.6435 +realloc_impl(void *ptr, size_t size)
1.6436 +{
1.6437 + void *ret;
1.6438 +
1.6439 + DARWIN_ONLY(return (szone->realloc)(szone, ptr, size));
1.6440 +
1.6441 + if (size == 0) {
1.6442 +#ifdef MALLOC_SYSV
1.6443 + if (opt_sysv == false)
1.6444 +#endif
1.6445 + size = 1;
1.6446 +#ifdef MALLOC_SYSV
1.6447 + else {
1.6448 + if (ptr != NULL)
1.6449 + idalloc(ptr);
1.6450 + ret = NULL;
1.6451 + goto RETURN;
1.6452 + }
1.6453 +#endif
1.6454 + }
1.6455 +
1.6456 + if (ptr != NULL) {
1.6457 + assert(malloc_initialized);
1.6458 +
1.6459 + ret = iralloc(ptr, size);
1.6460 +
1.6461 + if (ret == NULL) {
1.6462 +#ifdef MALLOC_XMALLOC
1.6463 + if (opt_xmalloc) {
1.6464 + _malloc_message(_getprogname(),
1.6465 + ": (malloc) Error in realloc(): out of "
1.6466 + "memory\n", "", "");
1.6467 + abort();
1.6468 + }
1.6469 +#endif
1.6470 + errno = ENOMEM;
1.6471 + }
1.6472 + } else {
1.6473 + if (malloc_init())
1.6474 + ret = NULL;
1.6475 + else
1.6476 + ret = imalloc(size);
1.6477 +
1.6478 + if (ret == NULL) {
1.6479 +#ifdef MALLOC_XMALLOC
1.6480 + if (opt_xmalloc) {
1.6481 + _malloc_message(_getprogname(),
1.6482 + ": (malloc) Error in realloc(): out of "
1.6483 + "memory\n", "", "");
1.6484 + abort();
1.6485 + }
1.6486 +#endif
1.6487 + errno = ENOMEM;
1.6488 + }
1.6489 + }
1.6490 +
1.6491 +#ifdef MALLOC_SYSV
1.6492 +RETURN:
1.6493 +#endif
1.6494 + UTRACE(ptr, size, ret);
1.6495 + return (ret);
1.6496 +}
1.6497 +
1.6498 +MOZ_MEMORY_API void
1.6499 +free_impl(void *ptr)
1.6500 +{
1.6501 + size_t offset;
1.6502 +
1.6503 + DARWIN_ONLY((szone->free)(szone, ptr); return);
1.6504 +
1.6505 + UTRACE(ptr, 0, 0);
1.6506 +
1.6507 + /*
1.6508 + * A version of idalloc that checks for NULL pointer but only for
1.6509 + * huge allocations assuming that CHUNK_ADDR2OFFSET(NULL) == 0.
1.6510 + */
1.6511 + assert(CHUNK_ADDR2OFFSET(NULL) == 0);
1.6512 + offset = CHUNK_ADDR2OFFSET(ptr);
1.6513 + if (offset != 0)
1.6514 + arena_dalloc(ptr, offset);
1.6515 + else if (ptr != NULL)
1.6516 + huge_dalloc(ptr);
1.6517 +}
1.6518 +
1.6519 +/*
1.6520 + * End malloc(3)-compatible functions.
1.6521 + */
1.6522 +/******************************************************************************/
1.6523 +/*
1.6524 + * Begin non-standard functions.
1.6525 + */
1.6526 +
1.6527 +/* This was added by Mozilla for use by SQLite. */
1.6528 +#if defined(MOZ_MEMORY_DARWIN) && !defined(MOZ_REPLACE_MALLOC)
1.6529 +static
1.6530 +#else
1.6531 +MOZ_MEMORY_API
1.6532 +#endif
1.6533 +size_t
1.6534 +malloc_good_size_impl(size_t size)
1.6535 +{
1.6536 + /*
1.6537 + * This duplicates the logic in imalloc(), arena_malloc() and
1.6538 + * arena_malloc_small().
1.6539 + */
1.6540 + if (size < small_min) {
1.6541 + /* Small (tiny). */
1.6542 + size = pow2_ceil(size);
1.6543 + /*
1.6544 + * We omit the #ifdefs from arena_malloc_small() --
1.6545 + * it can be inaccurate with its size in some cases, but this
1.6546 + * function must be accurate.
1.6547 + */
1.6548 + if (size < (1U << TINY_MIN_2POW))
1.6549 + size = (1U << TINY_MIN_2POW);
1.6550 + } else if (size <= small_max) {
1.6551 + /* Small (quantum-spaced). */
1.6552 + size = QUANTUM_CEILING(size);
1.6553 + } else if (size <= bin_maxclass) {
1.6554 + /* Small (sub-page). */
1.6555 + size = pow2_ceil(size);
1.6556 + } else if (size <= arena_maxclass) {
1.6557 + /* Large. */
1.6558 + size = PAGE_CEILING(size);
1.6559 + } else {
1.6560 + /*
1.6561 + * Huge. We use PAGE_CEILING to get psize, instead of using
1.6562 + * CHUNK_CEILING to get csize. This ensures that this
1.6563 + * malloc_usable_size(malloc(n)) always matches
1.6564 + * malloc_good_size(n).
1.6565 + */
1.6566 + size = PAGE_CEILING(size);
1.6567 + }
1.6568 + return size;
1.6569 +}
1.6570 +
1.6571 +
1.6572 +#if defined(MOZ_MEMORY_ANDROID) && (ANDROID_VERSION < 19)
1.6573 +MOZ_MEMORY_API size_t
1.6574 +malloc_usable_size_impl(void *ptr)
1.6575 +#else
1.6576 +MOZ_MEMORY_API size_t
1.6577 +malloc_usable_size_impl(const void *ptr)
1.6578 +#endif
1.6579 +{
1.6580 + DARWIN_ONLY(return (szone->size)(szone, ptr));
1.6581 +
1.6582 +#ifdef MALLOC_VALIDATE
1.6583 + return (isalloc_validate(ptr));
1.6584 +#else
1.6585 + assert(ptr != NULL);
1.6586 +
1.6587 + return (isalloc(ptr));
1.6588 +#endif
1.6589 +}
1.6590 +
1.6591 +MOZ_JEMALLOC_API void
1.6592 +jemalloc_stats_impl(jemalloc_stats_t *stats)
1.6593 +{
1.6594 + size_t i;
1.6595 +
1.6596 + assert(stats != NULL);
1.6597 +
1.6598 + /*
1.6599 + * Gather runtime settings.
1.6600 + */
1.6601 + stats->opt_abort = opt_abort;
1.6602 + stats->opt_junk =
1.6603 +#ifdef MALLOC_FILL
1.6604 + opt_junk ? true :
1.6605 +#endif
1.6606 + false;
1.6607 + stats->opt_poison =
1.6608 +#ifdef MALLOC_FILL
1.6609 + opt_poison ? true :
1.6610 +#endif
1.6611 + false;
1.6612 + stats->opt_utrace =
1.6613 +#ifdef MALLOC_UTRACE
1.6614 + opt_utrace ? true :
1.6615 +#endif
1.6616 + false;
1.6617 + stats->opt_sysv =
1.6618 +#ifdef MALLOC_SYSV
1.6619 + opt_sysv ? true :
1.6620 +#endif
1.6621 + false;
1.6622 + stats->opt_xmalloc =
1.6623 +#ifdef MALLOC_XMALLOC
1.6624 + opt_xmalloc ? true :
1.6625 +#endif
1.6626 + false;
1.6627 + stats->opt_zero =
1.6628 +#ifdef MALLOC_FILL
1.6629 + opt_zero ? true :
1.6630 +#endif
1.6631 + false;
1.6632 + stats->narenas = narenas;
1.6633 + stats->balance_threshold =
1.6634 +#ifdef MALLOC_BALANCE
1.6635 + opt_balance_threshold
1.6636 +#else
1.6637 + SIZE_T_MAX
1.6638 +#endif
1.6639 + ;
1.6640 + stats->quantum = quantum;
1.6641 + stats->small_max = small_max;
1.6642 + stats->large_max = arena_maxclass;
1.6643 + stats->chunksize = chunksize;
1.6644 + stats->dirty_max = opt_dirty_max;
1.6645 +
1.6646 + /*
1.6647 + * Gather current memory usage statistics.
1.6648 + */
1.6649 + stats->mapped = 0;
1.6650 + stats->allocated = 0;
1.6651 + stats->waste = 0;
1.6652 + stats->page_cache = 0;
1.6653 + stats->bookkeeping = 0;
1.6654 +
1.6655 + /* Get huge mapped/allocated. */
1.6656 + malloc_mutex_lock(&huge_mtx);
1.6657 + stats->mapped += huge_mapped;
1.6658 + stats->allocated += huge_allocated;
1.6659 + assert(huge_mapped >= huge_allocated);
1.6660 + malloc_mutex_unlock(&huge_mtx);
1.6661 +
1.6662 + /* Get base mapped/allocated. */
1.6663 + malloc_mutex_lock(&base_mtx);
1.6664 + stats->mapped += base_mapped;
1.6665 + stats->bookkeeping += base_committed;
1.6666 + assert(base_mapped >= base_committed);
1.6667 + malloc_mutex_unlock(&base_mtx);
1.6668 +
1.6669 + /* Iterate over arenas. */
1.6670 + for (i = 0; i < narenas; i++) {
1.6671 + arena_t *arena = arenas[i];
1.6672 + size_t arena_mapped, arena_allocated, arena_committed, arena_dirty;
1.6673 +
1.6674 + if (arena == NULL) {
1.6675 + continue;
1.6676 + }
1.6677 +
1.6678 + malloc_spin_lock(&arena->lock);
1.6679 +
1.6680 + arena_mapped = arena->stats.mapped;
1.6681 +
1.6682 + /* "committed" counts dirty and allocated memory. */
1.6683 + arena_committed = arena->stats.committed << pagesize_2pow;
1.6684 +
1.6685 + arena_allocated = arena->stats.allocated_small +
1.6686 + arena->stats.allocated_large;
1.6687 +
1.6688 + arena_dirty = arena->ndirty << pagesize_2pow;
1.6689 +
1.6690 + malloc_spin_unlock(&arena->lock);
1.6691 +
1.6692 + assert(arena_mapped >= arena_committed);
1.6693 + assert(arena_committed >= arena_allocated + arena_dirty);
1.6694 +
1.6695 + /* "waste" is committed memory that is neither dirty nor
1.6696 + * allocated. */
1.6697 + stats->mapped += arena_mapped;
1.6698 + stats->allocated += arena_allocated;
1.6699 + stats->page_cache += arena_dirty;
1.6700 + stats->waste += arena_committed - arena_allocated - arena_dirty;
1.6701 + }
1.6702 +
1.6703 + assert(stats->mapped >= stats->allocated + stats->waste +
1.6704 + stats->page_cache + stats->bookkeeping);
1.6705 +}
1.6706 +
1.6707 +#ifdef MALLOC_DOUBLE_PURGE
1.6708 +
1.6709 +/* Explicitly remove all of this chunk's MADV_FREE'd pages from memory. */
1.6710 +static void
1.6711 +hard_purge_chunk(arena_chunk_t *chunk)
1.6712 +{
1.6713 + /* See similar logic in arena_purge(). */
1.6714 +
1.6715 + size_t i;
1.6716 + for (i = arena_chunk_header_npages; i < chunk_npages; i++) {
1.6717 + /* Find all adjacent pages with CHUNK_MAP_MADVISED set. */
1.6718 + size_t npages;
1.6719 + for (npages = 0;
1.6720 + chunk->map[i + npages].bits & CHUNK_MAP_MADVISED && i + npages < chunk_npages;
1.6721 + npages++) {
1.6722 + /* Turn off the chunk's MADV_FREED bit and turn on its
1.6723 + * DECOMMITTED bit. */
1.6724 + RELEASE_ASSERT(!(chunk->map[i + npages].bits & CHUNK_MAP_DECOMMITTED));
1.6725 + chunk->map[i + npages].bits ^= CHUNK_MAP_MADVISED_OR_DECOMMITTED;
1.6726 + }
1.6727 +
1.6728 + /* We could use mincore to find out which pages are actually
1.6729 + * present, but it's not clear that's better. */
1.6730 + if (npages > 0) {
1.6731 + pages_decommit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
1.6732 + pages_commit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
1.6733 + }
1.6734 + i += npages;
1.6735 + }
1.6736 +}
1.6737 +
1.6738 +/* Explicitly remove all of this arena's MADV_FREE'd pages from memory. */
1.6739 +static void
1.6740 +hard_purge_arena(arena_t *arena)
1.6741 +{
1.6742 + malloc_spin_lock(&arena->lock);
1.6743 +
1.6744 + while (!LinkedList_IsEmpty(&arena->chunks_madvised)) {
1.6745 + LinkedList* next = arena->chunks_madvised.next;
1.6746 + arena_chunk_t *chunk =
1.6747 + LinkedList_Get(arena->chunks_madvised.next,
1.6748 + arena_chunk_t, chunks_madvised_elem);
1.6749 + hard_purge_chunk(chunk);
1.6750 + LinkedList_Remove(&chunk->chunks_madvised_elem);
1.6751 + }
1.6752 +
1.6753 + malloc_spin_unlock(&arena->lock);
1.6754 +}
1.6755 +
1.6756 +MOZ_JEMALLOC_API void
1.6757 +jemalloc_purge_freed_pages_impl()
1.6758 +{
1.6759 + size_t i;
1.6760 + for (i = 0; i < narenas; i++) {
1.6761 + arena_t *arena = arenas[i];
1.6762 + if (arena != NULL)
1.6763 + hard_purge_arena(arena);
1.6764 + }
1.6765 +}
1.6766 +
1.6767 +#else /* !defined MALLOC_DOUBLE_PURGE */
1.6768 +
1.6769 +MOZ_JEMALLOC_API void
1.6770 +jemalloc_purge_freed_pages_impl()
1.6771 +{
1.6772 + /* Do nothing. */
1.6773 +}
1.6774 +
1.6775 +#endif /* defined MALLOC_DOUBLE_PURGE */
1.6776 +
1.6777 +
1.6778 +
1.6779 +#ifdef MOZ_MEMORY_WINDOWS
1.6780 +void*
1.6781 +_recalloc(void *ptr, size_t count, size_t size)
1.6782 +{
1.6783 + size_t oldsize = (ptr != NULL) ? isalloc(ptr) : 0;
1.6784 + size_t newsize = count * size;
1.6785 +
1.6786 + /*
1.6787 + * In order for all trailing bytes to be zeroed, the caller needs to
1.6788 + * use calloc(), followed by recalloc(). However, the current calloc()
1.6789 + * implementation only zeros the bytes requested, so if recalloc() is
1.6790 + * to work 100% correctly, calloc() will need to change to zero
1.6791 + * trailing bytes.
1.6792 + */
1.6793 +
1.6794 + ptr = realloc(ptr, newsize);
1.6795 + if (ptr != NULL && oldsize < newsize) {
1.6796 + memset((void *)((uintptr_t)ptr + oldsize), 0, newsize -
1.6797 + oldsize);
1.6798 + }
1.6799 +
1.6800 + return ptr;
1.6801 +}
1.6802 +
1.6803 +/*
1.6804 + * This impl of _expand doesn't ever actually expand or shrink blocks: it
1.6805 + * simply replies that you may continue using a shrunk block.
1.6806 + */
1.6807 +void*
1.6808 +_expand(void *ptr, size_t newsize)
1.6809 +{
1.6810 + if (isalloc(ptr) >= newsize)
1.6811 + return ptr;
1.6812 +
1.6813 + return NULL;
1.6814 +}
1.6815 +
1.6816 +size_t
1.6817 +_msize(const void *ptr)
1.6818 +{
1.6819 +
1.6820 + return malloc_usable_size_impl(ptr);
1.6821 +}
1.6822 +#endif
1.6823 +
1.6824 +MOZ_JEMALLOC_API void
1.6825 +jemalloc_free_dirty_pages_impl(void)
1.6826 +{
1.6827 + size_t i;
1.6828 + for (i = 0; i < narenas; i++) {
1.6829 + arena_t *arena = arenas[i];
1.6830 +
1.6831 + if (arena != NULL) {
1.6832 + malloc_spin_lock(&arena->lock);
1.6833 + arena_purge(arena, true);
1.6834 + malloc_spin_unlock(&arena->lock);
1.6835 + }
1.6836 + }
1.6837 +}
1.6838 +
1.6839 +/*
1.6840 + * End non-standard functions.
1.6841 + */
1.6842 +/******************************************************************************/
1.6843 +/*
1.6844 + * Begin library-private functions, used by threading libraries for protection
1.6845 + * of malloc during fork(). These functions are only called if the program is
1.6846 + * running in threaded mode, so there is no need to check whether the program
1.6847 + * is threaded here.
1.6848 + */
1.6849 +
1.6850 +static void
1.6851 +_malloc_prefork(void)
1.6852 +{
1.6853 + unsigned i;
1.6854 +
1.6855 + /* Acquire all mutexes in a safe order. */
1.6856 +
1.6857 + malloc_spin_lock(&arenas_lock);
1.6858 + for (i = 0; i < narenas; i++) {
1.6859 + if (arenas[i] != NULL)
1.6860 + malloc_spin_lock(&arenas[i]->lock);
1.6861 + }
1.6862 +
1.6863 + malloc_mutex_lock(&base_mtx);
1.6864 +
1.6865 + malloc_mutex_lock(&huge_mtx);
1.6866 +}
1.6867 +
1.6868 +static void
1.6869 +_malloc_postfork(void)
1.6870 +{
1.6871 + unsigned i;
1.6872 +
1.6873 + /* Release all mutexes, now that fork() has completed. */
1.6874 +
1.6875 + malloc_mutex_unlock(&huge_mtx);
1.6876 +
1.6877 + malloc_mutex_unlock(&base_mtx);
1.6878 +
1.6879 + for (i = 0; i < narenas; i++) {
1.6880 + if (arenas[i] != NULL)
1.6881 + malloc_spin_unlock(&arenas[i]->lock);
1.6882 + }
1.6883 + malloc_spin_unlock(&arenas_lock);
1.6884 +}
1.6885 +
1.6886 +/*
1.6887 + * End library-private functions.
1.6888 + */
1.6889 +/******************************************************************************/
1.6890 +
1.6891 +#ifdef HAVE_DLOPEN
1.6892 +# include <dlfcn.h>
1.6893 +#endif
1.6894 +
1.6895 +#if defined(MOZ_MEMORY_DARWIN)
1.6896 +
1.6897 +#if !defined(MOZ_REPLACE_MALLOC)
1.6898 +static void *
1.6899 +zone_malloc(malloc_zone_t *zone, size_t size)
1.6900 +{
1.6901 +
1.6902 + return (malloc_impl(size));
1.6903 +}
1.6904 +
1.6905 +static void *
1.6906 +zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
1.6907 +{
1.6908 +
1.6909 + return (calloc_impl(num, size));
1.6910 +}
1.6911 +
1.6912 +static void *
1.6913 +zone_valloc(malloc_zone_t *zone, size_t size)
1.6914 +{
1.6915 + void *ret = NULL; /* Assignment avoids useless compiler warning. */
1.6916 +
1.6917 + posix_memalign_impl(&ret, pagesize, size);
1.6918 +
1.6919 + return (ret);
1.6920 +}
1.6921 +
1.6922 +static void *
1.6923 +zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
1.6924 +{
1.6925 + return (memalign_impl(alignment, size));
1.6926 +}
1.6927 +
1.6928 +static void *
1.6929 +zone_destroy(malloc_zone_t *zone)
1.6930 +{
1.6931 +
1.6932 + /* This function should never be called. */
1.6933 + assert(false);
1.6934 + return (NULL);
1.6935 +}
1.6936 +
1.6937 +static size_t
1.6938 +zone_good_size(malloc_zone_t *zone, size_t size)
1.6939 +{
1.6940 + return malloc_good_size_impl(size);
1.6941 +}
1.6942 +
1.6943 +static size_t
1.6944 +ozone_size(malloc_zone_t *zone, void *ptr)
1.6945 +{
1.6946 + size_t ret = isalloc_validate(ptr);
1.6947 + if (ret == 0)
1.6948 + ret = szone->size(zone, ptr);
1.6949 +
1.6950 + return ret;
1.6951 +}
1.6952 +
1.6953 +static void
1.6954 +ozone_free(malloc_zone_t *zone, void *ptr)
1.6955 +{
1.6956 + if (isalloc_validate(ptr) != 0)
1.6957 + free_impl(ptr);
1.6958 + else {
1.6959 + size_t size = szone->size(zone, ptr);
1.6960 + if (size != 0)
1.6961 + (szone->free)(zone, ptr);
1.6962 + /* Otherwise we leak. */
1.6963 + }
1.6964 +}
1.6965 +
1.6966 +static void *
1.6967 +ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
1.6968 +{
1.6969 + size_t oldsize;
1.6970 + if (ptr == NULL)
1.6971 + return (malloc_impl(size));
1.6972 +
1.6973 + oldsize = isalloc_validate(ptr);
1.6974 + if (oldsize != 0)
1.6975 + return (realloc_impl(ptr, size));
1.6976 + else {
1.6977 + oldsize = szone->size(zone, ptr);
1.6978 + if (oldsize == 0)
1.6979 + return (malloc_impl(size));
1.6980 + else {
1.6981 + void *ret = malloc_impl(size);
1.6982 + if (ret != NULL) {
1.6983 + memcpy(ret, ptr, (oldsize < size) ? oldsize :
1.6984 + size);
1.6985 + (szone->free)(zone, ptr);
1.6986 + }
1.6987 + return (ret);
1.6988 + }
1.6989 + }
1.6990 +}
1.6991 +
1.6992 +static unsigned
1.6993 +ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results,
1.6994 + unsigned num_requested)
1.6995 +{
1.6996 + /* Don't bother implementing this interface, since it isn't required. */
1.6997 + return 0;
1.6998 +}
1.6999 +
1.7000 +static void
1.7001 +ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num)
1.7002 +{
1.7003 + unsigned i;
1.7004 +
1.7005 + for (i = 0; i < num; i++)
1.7006 + ozone_free(zone, to_be_freed[i]);
1.7007 +}
1.7008 +
1.7009 +static void
1.7010 +ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
1.7011 +{
1.7012 + if (isalloc_validate(ptr) != 0) {
1.7013 + assert(isalloc_validate(ptr) == size);
1.7014 + free_impl(ptr);
1.7015 + } else {
1.7016 + assert(size == szone->size(zone, ptr));
1.7017 + l_szone.m16(zone, ptr, size);
1.7018 + }
1.7019 +}
1.7020 +
1.7021 +static void
1.7022 +ozone_force_lock(malloc_zone_t *zone)
1.7023 +{
1.7024 + _malloc_prefork();
1.7025 + szone->introspect->force_lock(zone);
1.7026 +}
1.7027 +
1.7028 +static void
1.7029 +ozone_force_unlock(malloc_zone_t *zone)
1.7030 +{
1.7031 + szone->introspect->force_unlock(zone);
1.7032 + _malloc_postfork();
1.7033 +}
1.7034 +
1.7035 +static size_t
1.7036 +zone_version_size(int version)
1.7037 +{
1.7038 + switch (version)
1.7039 + {
1.7040 + case SNOW_LEOPARD_MALLOC_ZONE_T_VERSION:
1.7041 + return sizeof(snow_leopard_malloc_zone);
1.7042 + case LEOPARD_MALLOC_ZONE_T_VERSION:
1.7043 + return sizeof(leopard_malloc_zone);
1.7044 + default:
1.7045 + case LION_MALLOC_ZONE_T_VERSION:
1.7046 + return sizeof(lion_malloc_zone);
1.7047 + }
1.7048 +}
1.7049 +
1.7050 +/*
1.7051 + * Overlay the default scalable zone (szone) such that existing allocations are
1.7052 + * drained, and further allocations come from jemalloc. This is necessary
1.7053 + * because Core Foundation directly accesses and uses the szone before the
1.7054 + * jemalloc library is even loaded.
1.7055 + */
1.7056 +static void
1.7057 +szone2ozone(malloc_zone_t *default_zone, size_t size)
1.7058 +{
1.7059 + lion_malloc_zone *l_zone;
1.7060 + assert(malloc_initialized);
1.7061 +
1.7062 + /*
1.7063 + * Stash a copy of the original szone so that we can call its
1.7064 + * functions as needed. Note that internally, the szone stores its
1.7065 + * bookkeeping data structures immediately following the malloc_zone_t
1.7066 + * header, so when calling szone functions, we need to pass a pointer to
1.7067 + * the original zone structure.
1.7068 + */
1.7069 + memcpy(szone, default_zone, size);
1.7070 +
1.7071 + /* OSX 10.7 allocates the default zone in protected memory. */
1.7072 + if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
1.7073 + void* start_of_page = (void*)((size_t)(default_zone) & ~pagesize_mask);
1.7074 + mprotect (start_of_page, size, PROT_READ | PROT_WRITE);
1.7075 + }
1.7076 +
1.7077 + default_zone->size = (void *)ozone_size;
1.7078 + default_zone->malloc = (void *)zone_malloc;
1.7079 + default_zone->calloc = (void *)zone_calloc;
1.7080 + default_zone->valloc = (void *)zone_valloc;
1.7081 + default_zone->free = (void *)ozone_free;
1.7082 + default_zone->realloc = (void *)ozone_realloc;
1.7083 + default_zone->destroy = (void *)zone_destroy;
1.7084 + default_zone->batch_malloc = NULL;
1.7085 + default_zone->batch_free = ozone_batch_free;
1.7086 + default_zone->introspect = ozone_introspect;
1.7087 +
1.7088 + /* Don't modify default_zone->zone_name; Mac libc may rely on the name
1.7089 + * being unchanged. See Mozilla bug 694896. */
1.7090 +
1.7091 + ozone_introspect->enumerator = NULL;
1.7092 + ozone_introspect->good_size = (void *)zone_good_size;
1.7093 + ozone_introspect->check = NULL;
1.7094 + ozone_introspect->print = NULL;
1.7095 + ozone_introspect->log = NULL;
1.7096 + ozone_introspect->force_lock = (void *)ozone_force_lock;
1.7097 + ozone_introspect->force_unlock = (void *)ozone_force_unlock;
1.7098 + ozone_introspect->statistics = NULL;
1.7099 +
1.7100 + /* Platform-dependent structs */
1.7101 + l_zone = (lion_malloc_zone*)(default_zone);
1.7102 +
1.7103 + if (default_zone->version >= SNOW_LEOPARD_MALLOC_ZONE_T_VERSION) {
1.7104 + l_zone->m15 = (void (*)())zone_memalign;
1.7105 + l_zone->m16 = (void (*)())ozone_free_definite_size;
1.7106 + l_ozone_introspect.m9 = NULL;
1.7107 + }
1.7108 +
1.7109 + if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
1.7110 + l_zone->m17 = NULL;
1.7111 + l_ozone_introspect.m10 = NULL;
1.7112 + l_ozone_introspect.m11 = NULL;
1.7113 + l_ozone_introspect.m12 = NULL;
1.7114 + l_ozone_introspect.m13 = NULL;
1.7115 + }
1.7116 +}
1.7117 +#endif
1.7118 +
1.7119 +__attribute__((constructor))
1.7120 +void
1.7121 +jemalloc_darwin_init(void)
1.7122 +{
1.7123 + if (malloc_init_hard())
1.7124 + abort();
1.7125 +}
1.7126 +
1.7127 +#endif
1.7128 +
1.7129 +/*
1.7130 + * is_malloc(malloc_impl) is some macro magic to detect if malloc_impl is
1.7131 + * defined as "malloc" in mozmemory_wrap.h
1.7132 + */
1.7133 +#define malloc_is_malloc 1
1.7134 +#define is_malloc_(a) malloc_is_ ## a
1.7135 +#define is_malloc(a) is_malloc_(a)
1.7136 +
1.7137 +#if !defined(MOZ_MEMORY_DARWIN) && (is_malloc(malloc_impl) == 1)
1.7138 +# if defined(__GLIBC__) && !defined(__UCLIBC__)
1.7139 +/*
1.7140 + * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
1.7141 + * to inconsistently reference libc's malloc(3)-compatible functions
1.7142 + * (bug 493541).
1.7143 + *
1.7144 + * These definitions interpose hooks in glibc. The functions are actually
1.7145 + * passed an extra argument for the caller return address, which will be
1.7146 + * ignored.
1.7147 + */
1.7148 +MOZ_MEMORY_API void (*__free_hook)(void *ptr) = free_impl;
1.7149 +MOZ_MEMORY_API void *(*__malloc_hook)(size_t size) = malloc_impl;
1.7150 +MOZ_MEMORY_API void *(*__realloc_hook)(void *ptr, size_t size) = realloc_impl;
1.7151 +MOZ_MEMORY_API void *(*__memalign_hook)(size_t alignment, size_t size) = MEMALIGN;
1.7152 +
1.7153 +# elif defined(RTLD_DEEPBIND)
1.7154 +/*
1.7155 + * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their
1.7156 + * implementations permit similar inconsistencies? Should STV_SINGLETON
1.7157 + * visibility be used for interposition where available?
1.7158 + */
1.7159 +# error "Interposing malloc is unsafe on this system without libc malloc hooks."
1.7160 +# endif
1.7161 +#endif
1.7162 +
1.7163 +#ifdef MOZ_MEMORY_WINDOWS
1.7164 +/*
1.7165 + * In the new style jemalloc integration jemalloc is built as a separate
1.7166 + * shared library. Since we're no longer hooking into the CRT binary,
1.7167 + * we need to initialize the heap at the first opportunity we get.
1.7168 + * DLL_PROCESS_ATTACH in DllMain is that opportunity.
1.7169 + */
1.7170 +BOOL APIENTRY DllMain(HINSTANCE hModule,
1.7171 + DWORD reason,
1.7172 + LPVOID lpReserved)
1.7173 +{
1.7174 + switch (reason) {
1.7175 + case DLL_PROCESS_ATTACH:
1.7176 + /* Don't force the system to page DllMain back in every time
1.7177 + * we create/destroy a thread */
1.7178 + DisableThreadLibraryCalls(hModule);
1.7179 + /* Initialize the heap */
1.7180 + malloc_init_hard();
1.7181 + break;
1.7182 +
1.7183 + case DLL_PROCESS_DETACH:
1.7184 + break;
1.7185 +
1.7186 + }
1.7187 +
1.7188 + return TRUE;
1.7189 +}
1.7190 +#endif
