Differential D24522 Diff 70932 head/contrib/jemalloc/src/extent.c

Changeset View

Standalone View

head/contrib/jemalloc/src/extent.c

Show All 14 Lines

rtree_t extents_rtree;		rtree_t extents_rtree;
/* Keyed by the address of the extent_t being protected. */		/* Keyed by the address of the extent_t being protected. */
mutex_pool_t extent_mutex_pool;		mutex_pool_t extent_mutex_pool;

size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT;		size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT;

static const bitmap_info_t extents_bitmap_info =		static const bitmap_info_t extents_bitmap_info =
BITMAP_INFO_INITIALIZER(NPSIZES+1);		BITMAP_INFO_INITIALIZER(SC_NPSIZES+1);

static void extent_alloc_default(extent_hooks_t extent_hooks, void *new_addr,		static void extent_alloc_default(extent_hooks_t extent_hooks, void *new_addr,
size_t size, size_t alignment, bool zero, bool commit,		size_t size, size_t alignment, bool zero, bool commit,
unsigned arena_ind);		unsigned arena_ind);
static bool extent_dalloc_default(extent_hooks_t extent_hooks, void addr,		static bool extent_dalloc_default(extent_hooks_t extent_hooks, void addr,
size_t size, bool committed, unsigned arena_ind);		size_t size, bool committed, unsigned arena_ind);
static void extent_destroy_default(extent_hooks_t extent_hooks, void addr,		static void extent_destroy_default(extent_hooks_t extent_hooks, void addr,
size_t size, bool committed, unsigned arena_ind);		size_t size, bool committed, unsigned arena_ind);
Show All 13 Lines	static bool extent_purge_lazy_impl(tsdn_t tsdn, arena_t arena,
size_t length, bool growing_retained);		size_t length, bool growing_retained);
#ifdef PAGES_CAN_PURGE_FORCED		#ifdef PAGES_CAN_PURGE_FORCED
static bool extent_purge_forced_default(extent_hooks_t *extent_hooks,		static bool extent_purge_forced_default(extent_hooks_t *extent_hooks,
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);		void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
#endif		#endif
static bool extent_purge_forced_impl(tsdn_t tsdn, arena_t arena,		static bool extent_purge_forced_impl(tsdn_t tsdn, arena_t arena,
extent_hooks_t *r_extent_hooks, extent_t extent, size_t offset,		extent_hooks_t *r_extent_hooks, extent_t extent, size_t offset,
size_t length, bool growing_retained);		size_t length, bool growing_retained);
#ifdef JEMALLOC_MAPS_COALESCE
static bool extent_split_default(extent_hooks_t extent_hooks, void addr,		static bool extent_split_default(extent_hooks_t extent_hooks, void addr,
size_t size, size_t size_a, size_t size_b, bool committed,		size_t size, size_t size_a, size_t size_b, bool committed,
unsigned arena_ind);		unsigned arena_ind);
#endif
static extent_t extent_split_impl(tsdn_t tsdn, arena_t *arena,		static extent_t extent_split_impl(tsdn_t tsdn, arena_t *arena,
extent_hooks_t *r_extent_hooks, extent_t extent, size_t size_a,		extent_hooks_t *r_extent_hooks, extent_t extent, size_t size_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,		szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,
bool growing_retained);		bool growing_retained);
#ifdef JEMALLOC_MAPS_COALESCE
static bool extent_merge_default(extent_hooks_t extent_hooks, void addr_a,		static bool extent_merge_default(extent_hooks_t extent_hooks, void addr_a,
size_t size_a, void *addr_b, size_t size_b, bool committed,		size_t size_a, void *addr_b, size_t size_b, bool committed,
unsigned arena_ind);		unsigned arena_ind);
#endif
static bool extent_merge_impl(tsdn_t tsdn, arena_t arena,		static bool extent_merge_impl(tsdn_t tsdn, arena_t arena,
extent_hooks_t *r_extent_hooks, extent_t a, extent_t *b,		extent_hooks_t *r_extent_hooks, extent_t a, extent_t *b,
bool growing_retained);		bool growing_retained);

const extent_hooks_t extent_hooks_default = {		const extent_hooks_t extent_hooks_default = {
extent_alloc_default,		extent_alloc_default,
extent_dalloc_default,		extent_dalloc_default,
extent_destroy_default,		extent_destroy_default,
extent_commit_default,		extent_commit_default,
extent_decommit_default		extent_decommit_default
#ifdef PAGES_CAN_PURGE_LAZY		#ifdef PAGES_CAN_PURGE_LAZY
,		,
extent_purge_lazy_default		extent_purge_lazy_default
#else		#else
,		,
NULL		NULL
#endif		#endif
#ifdef PAGES_CAN_PURGE_FORCED		#ifdef PAGES_CAN_PURGE_FORCED
,		,
extent_purge_forced_default		extent_purge_forced_default
#else		#else
,		,
NULL		NULL
#endif		#endif
#ifdef JEMALLOC_MAPS_COALESCE
,		,
extent_split_default,		extent_split_default,
extent_merge_default		extent_merge_default
#endif
};		};

/* Used exclusively for gdump triggering. */		/* Used exclusively for gdump triggering. */
static atomic_zu_t curpages;		static atomic_zu_t curpages;
static atomic_zu_t highpages;		static atomic_zu_t highpages;

/******************************************************************************/		/******************************************************************************/
/*		/*
Show All 10 Lines	static extent_t extent_try_coalesce(tsdn_t tsdn, arena_t *arena,
extent_hooks_t *r_extent_hooks, rtree_ctx_t rtree_ctx, extents_t *extents,		extent_hooks_t *r_extent_hooks, rtree_ctx_t rtree_ctx, extents_t *extents,
extent_t extent, bool coalesced, bool growing_retained);		extent_t extent, bool coalesced, bool growing_retained);
static void extent_record(tsdn_t tsdn, arena_t arena,		static void extent_record(tsdn_t tsdn, arena_t arena,
extent_hooks_t *r_extent_hooks, extents_t extents, extent_t *extent,		extent_hooks_t *r_extent_hooks, extents_t extents, extent_t *extent,
bool growing_retained);		bool growing_retained);

/******************************************************************************/		/******************************************************************************/

ph_gen(UNUSED, extent_avail_, extent_tree_t, extent_t, ph_link,		#define ATTR_NONE /* does nothing */

		ph_gen(ATTR_NONE, extent_avail_, extent_tree_t, extent_t, ph_link,
extent_esnead_comp)		extent_esnead_comp)

		#undef ATTR_NONE

typedef enum {		typedef enum {
lock_result_success,		lock_result_success,
lock_result_failure,		lock_result_failure,
lock_result_no_extent		lock_result_no_extent
} lock_result_t;		} lock_result_t;

static lock_result_t		static lock_result_t
extent_rtree_leaf_elm_try_lock(tsdn_t tsdn, rtree_leaf_elm_t elm,		extent_rtree_leaf_elm_try_lock(tsdn_t tsdn, rtree_leaf_elm_t elm,
extent_t **result) {		extent_t **result, bool inactive_only) {
extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree,		extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree,
elm, true);		elm, true);

if (extent1 == NULL) {		/* Slab implies active extents and should be skipped. */
		if (extent1 == NULL \|\| (inactive_only && rtree_leaf_elm_slab_read(tsdn,
		&extents_rtree, elm, true))) {
return lock_result_no_extent;		return lock_result_no_extent;
}		}

/*		/*
* It's possible that the extent changed out from under us, and with it		* It's possible that the extent changed out from under us, and with it
* the leaf->extent mapping. We have to recheck while holding the lock.		* the leaf->extent mapping. We have to recheck while holding the lock.
*/		*/
extent_lock(tsdn, extent1);		extent_lock(tsdn, extent1);
extent_t *extent2 = rtree_leaf_elm_extent_read(tsdn,		extent_t *extent2 = rtree_leaf_elm_extent_read(tsdn,
&extents_rtree, elm, true);		&extents_rtree, elm, true);

if (extent1 == extent2) {		if (extent1 == extent2) {
*result = extent1;		*result = extent1;
return lock_result_success;		return lock_result_success;
} else {		} else {
extent_unlock(tsdn, extent1);		extent_unlock(tsdn, extent1);
return lock_result_failure;		return lock_result_failure;
}		}
}		}

/*		/*
* Returns a pool-locked extent_t * if there's one associated with the given		* Returns a pool-locked extent_t * if there's one associated with the given
* address, and NULL otherwise.		* address, and NULL otherwise.
*/		*/
static extent_t *		static extent_t *
extent_lock_from_addr(tsdn_t tsdn, rtree_ctx_t rtree_ctx, void *addr) {		extent_lock_from_addr(tsdn_t tsdn, rtree_ctx_t rtree_ctx, void *addr,
		bool inactive_only) {
extent_t *ret = NULL;		extent_t *ret = NULL;
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree,		rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree,
rtree_ctx, (uintptr_t)addr, false, false);		rtree_ctx, (uintptr_t)addr, false, false);
if (elm == NULL) {		if (elm == NULL) {
return NULL;		return NULL;
}		}
lock_result_t lock_result;		lock_result_t lock_result;
do {		do {
lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret);		lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret,
		inactive_only);
} while (lock_result == lock_result_failure);		} while (lock_result == lock_result_failure);
return ret;		return ret;
}		}

extent_t *		extent_t *
extent_alloc(tsdn_t tsdn, arena_t arena) {		extent_alloc(tsdn_t tsdn, arena_t arena) {
malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);		malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
extent_t *extent = extent_avail_first(&arena->extent_avail);		extent_t *extent = extent_avail_first(&arena->extent_avail);
if (extent == NULL) {		if (extent == NULL) {
malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);		malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
return base_alloc_extent(tsdn, arena->base);		return base_alloc_extent(tsdn, arena->base);
}		}
extent_avail_remove(&arena->extent_avail, extent);		extent_avail_remove(&arena->extent_avail, extent);
		atomic_fetch_sub_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED);
malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);		malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
return extent;		return extent;
}		}

void		void
extent_dalloc(tsdn_t tsdn, arena_t arena, extent_t *extent) {		extent_dalloc(tsdn_t tsdn, arena_t arena, extent_t *extent) {
malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);		malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
extent_avail_insert(&arena->extent_avail, extent);		extent_avail_insert(&arena->extent_avail, extent);
		atomic_fetch_add_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED);
malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);		malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
}		}

extent_hooks_t *		extent_hooks_t *
extent_hooks_get(arena_t *arena) {		extent_hooks_get(arena_t *arena) {
return base_extent_hooks_get(arena->base);		return base_extent_hooks_get(arena->base);
}		}

▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines
#ifndef JEMALLOC_JET		#ifndef JEMALLOC_JET
static		static
#endif		#endif
size_t		size_t
extent_size_quantize_ceil(size_t size) {		extent_size_quantize_ceil(size_t size) {
size_t ret;		size_t ret;

assert(size > 0);		assert(size > 0);
assert(size - sz_large_pad <= LARGE_MAXCLASS);		assert(size - sz_large_pad <= SC_LARGE_MAXCLASS);
assert((size & PAGE_MASK) == 0);		assert((size & PAGE_MASK) == 0);

ret = extent_size_quantize_floor(size);		ret = extent_size_quantize_floor(size);
if (ret < size) {		if (ret < size) {
/*		/*
* Skip a quantization that may have an adequately large extent,		* Skip a quantization that may have an adequately large extent,
* because under-sized extents may be mixed in. This only		* because under-sized extents may be mixed in. This only
* happens when an unusual size is requested, i.e. for aligned		* happens when an unusual size is requested, i.e. for aligned
Show All 12 Lines

bool		bool
extents_init(tsdn_t tsdn, extents_t extents, extent_state_t state,		extents_init(tsdn_t tsdn, extents_t extents, extent_state_t state,
bool delay_coalesce) {		bool delay_coalesce) {
if (malloc_mutex_init(&extents->mtx, "extents", WITNESS_RANK_EXTENTS,		if (malloc_mutex_init(&extents->mtx, "extents", WITNESS_RANK_EXTENTS,
malloc_mutex_rank_exclusive)) {		malloc_mutex_rank_exclusive)) {
return true;		return true;
}		}
for (unsigned i = 0; i < NPSIZES+1; i++) {		for (unsigned i = 0; i < SC_NPSIZES + 1; i++) {
extent_heap_new(&extents->heaps[i]);		extent_heap_new(&extents->heaps[i]);
}		}
bitmap_init(extents->bitmap, &extents_bitmap_info, true);		bitmap_init(extents->bitmap, &extents_bitmap_info, true);
extent_list_init(&extents->lru);		extent_list_init(&extents->lru);
atomic_store_zu(&extents->npages, 0, ATOMIC_RELAXED);		atomic_store_zu(&extents->npages, 0, ATOMIC_RELAXED);
extents->state = state;		extents->state = state;
extents->delay_coalesce = delay_coalesce;		extents->delay_coalesce = delay_coalesce;
return false;		return false;
}		}

extent_state_t		extent_state_t
extents_state_get(const extents_t *extents) {		extents_state_get(const extents_t *extents) {
return extents->state;		return extents->state;
}		}

size_t		size_t
extents_npages_get(extents_t *extents) {		extents_npages_get(extents_t *extents) {
return atomic_load_zu(&extents->npages, ATOMIC_RELAXED);		return atomic_load_zu(&extents->npages, ATOMIC_RELAXED);
}		}

		size_t
		extents_nextents_get(extents_t *extents, pszind_t pind) {
		return atomic_load_zu(&extents->nextents[pind], ATOMIC_RELAXED);
		}

		size_t
		extents_nbytes_get(extents_t *extents, pszind_t pind) {
		return atomic_load_zu(&extents->nbytes[pind], ATOMIC_RELAXED);
		}

static void		static void
		extents_stats_add(extents_t *extent, pszind_t pind, size_t sz) {
		size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED);
		atomic_store_zu(&extent->nextents[pind], cur + 1, ATOMIC_RELAXED);
		cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED);
		atomic_store_zu(&extent->nbytes[pind], cur + sz, ATOMIC_RELAXED);
		}

		static void
		extents_stats_sub(extents_t *extent, pszind_t pind, size_t sz) {
		size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED);
		atomic_store_zu(&extent->nextents[pind], cur - 1, ATOMIC_RELAXED);
		cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED);
		atomic_store_zu(&extent->nbytes[pind], cur - sz, ATOMIC_RELAXED);
		}

		static void
extents_insert_locked(tsdn_t tsdn, extents_t extents, extent_t *extent) {		extents_insert_locked(tsdn_t tsdn, extents_t extents, extent_t *extent) {
malloc_mutex_assert_owner(tsdn, &extents->mtx);		malloc_mutex_assert_owner(tsdn, &extents->mtx);
assert(extent_state_get(extent) == extents->state);		assert(extent_state_get(extent) == extents->state);

size_t size = extent_size_get(extent);		size_t size = extent_size_get(extent);
size_t psz = extent_size_quantize_floor(size);		size_t psz = extent_size_quantize_floor(size);
pszind_t pind = sz_psz2ind(psz);		pszind_t pind = sz_psz2ind(psz);
if (extent_heap_empty(&extents->heaps[pind])) {		if (extent_heap_empty(&extents->heaps[pind])) {
bitmap_unset(extents->bitmap, &extents_bitmap_info,		bitmap_unset(extents->bitmap, &extents_bitmap_info,
(size_t)pind);		(size_t)pind);
}		}
extent_heap_insert(&extents->heaps[pind], extent);		extent_heap_insert(&extents->heaps[pind], extent);

		if (config_stats) {
		extents_stats_add(extents, pind, size);
		}

extent_list_append(&extents->lru, extent);		extent_list_append(&extents->lru, extent);
size_t npages = size >> LG_PAGE;		size_t npages = size >> LG_PAGE;
/*		/*
* All modifications to npages hold the mutex (as asserted above), so we		* All modifications to npages hold the mutex (as asserted above), so we
* don't need an atomic fetch-add; we can get by with a load followed by		* don't need an atomic fetch-add; we can get by with a load followed by
* a store.		* a store.
*/		*/
size_t cur_extents_npages =		size_t cur_extents_npages =
atomic_load_zu(&extents->npages, ATOMIC_RELAXED);		atomic_load_zu(&extents->npages, ATOMIC_RELAXED);
atomic_store_zu(&extents->npages, cur_extents_npages + npages,		atomic_store_zu(&extents->npages, cur_extents_npages + npages,
ATOMIC_RELAXED);		ATOMIC_RELAXED);
}		}

static void		static void
extents_remove_locked(tsdn_t tsdn, extents_t extents, extent_t *extent) {		extents_remove_locked(tsdn_t tsdn, extents_t extents, extent_t *extent) {
malloc_mutex_assert_owner(tsdn, &extents->mtx);		malloc_mutex_assert_owner(tsdn, &extents->mtx);
assert(extent_state_get(extent) == extents->state);		assert(extent_state_get(extent) == extents->state);

size_t size = extent_size_get(extent);		size_t size = extent_size_get(extent);
size_t psz = extent_size_quantize_floor(size);		size_t psz = extent_size_quantize_floor(size);
pszind_t pind = sz_psz2ind(psz);		pszind_t pind = sz_psz2ind(psz);
extent_heap_remove(&extents->heaps[pind], extent);		extent_heap_remove(&extents->heaps[pind], extent);

		if (config_stats) {
		extents_stats_sub(extents, pind, size);
		}

if (extent_heap_empty(&extents->heaps[pind])) {		if (extent_heap_empty(&extents->heaps[pind])) {
bitmap_set(extents->bitmap, &extents_bitmap_info,		bitmap_set(extents->bitmap, &extents_bitmap_info,
(size_t)pind);		(size_t)pind);
}		}
extent_list_remove(&extents->lru, extent);		extent_list_remove(&extents->lru, extent);
size_t npages = size >> LG_PAGE;		size_t npages = size >> LG_PAGE;
/*		/*
* As in extents_insert_locked, we hold extents->mtx and so don't need		* As in extents_insert_locked, we hold extents->mtx and so don't need
Show All 15 Lines	extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size,
size_t alignment) {		size_t alignment) {
pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(min_size));		pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(min_size));
pszind_t pind_max = sz_psz2ind(extent_size_quantize_ceil(max_size));		pszind_t pind_max = sz_psz2ind(extent_size_quantize_ceil(max_size));

for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap,		for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap,
&extents_bitmap_info, (size_t)pind); i < pind_max; i =		&extents_bitmap_info, (size_t)pind); i < pind_max; i =
(pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,		(pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
(size_t)i+1)) {		(size_t)i+1)) {
assert(i < NPSIZES);		assert(i < SC_NPSIZES);
assert(!extent_heap_empty(&extents->heaps[i]));		assert(!extent_heap_empty(&extents->heaps[i]));
extent_t *extent = extent_heap_first(&extents->heaps[i]);		extent_t *extent = extent_heap_first(&extents->heaps[i]);
uintptr_t base = (uintptr_t)extent_base_get(extent);		uintptr_t base = (uintptr_t)extent_base_get(extent);
size_t candidate_size = extent_size_get(extent);		size_t candidate_size = extent_size_get(extent);
assert(candidate_size >= min_size);		assert(candidate_size >= min_size);

uintptr_t next_align = ALIGNMENT_CEILING((uintptr_t)base,		uintptr_t next_align = ALIGNMENT_CEILING((uintptr_t)base,
PAGE_CEILING(alignment));		PAGE_CEILING(alignment));
if (base > next_align \|\| base + candidate_size <= next_align) {		if (base > next_align \|\| base + candidate_size <= next_align) {
/* Overflow or not crossing the next alignment. */		/* Overflow or not crossing the next alignment. */
continue;		continue;
}		}

size_t leadsize = next_align - base;		size_t leadsize = next_align - base;
if (candidate_size - leadsize >= min_size) {		if (candidate_size - leadsize >= min_size) {
return extent;		return extent;
}		}
}		}

return NULL;		return NULL;
}		}

/* Do any-best-fit extent selection, i.e. select any extent that best fits. */
static extent_t *
extents_best_fit_locked(tsdn_t tsdn, arena_t arena, extents_t *extents,
size_t size) {
pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size));
pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
(size_t)pind);
if (i < NPSIZES+1) {
/*		/*
* In order to reduce fragmentation, avoid reusing and splitting
* large extents for much smaller sizes.
*/
if ((sz_pind2sz(i) >> opt_lg_extent_max_active_fit) > size) {
return NULL;
}
assert(!extent_heap_empty(&extents->heaps[i]));
extent_t *extent = extent_heap_first(&extents->heaps[i]);
assert(extent_size_get(extent) >= size);
return extent;
}

return NULL;
}

/*
* Do first-fit extent selection, i.e. select the oldest/lowest extent that is		* Do first-fit extent selection, i.e. select the oldest/lowest extent that is
* large enough.		* large enough.
*/		*/
static extent_t *		static extent_t *
extents_first_fit_locked(tsdn_t tsdn, arena_t arena, extents_t *extents,		extents_first_fit_locked(tsdn_t tsdn, arena_t arena, extents_t *extents,
size_t size) {		size_t size) {
extent_t *ret = NULL;		extent_t *ret = NULL;

pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size));		pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size));

		if (!maps_coalesce && !opt_retain) {
		/*
		* No split / merge allowed (Windows w/o retain). Try exact fit
		* only.
		*/
		return extent_heap_empty(&extents->heaps[pind]) ? NULL :
		extent_heap_first(&extents->heaps[pind]);
		}

for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap,		for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap,
&extents_bitmap_info, (size_t)pind); i < NPSIZES+1; i =		&extents_bitmap_info, (size_t)pind);
(pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,		i < SC_NPSIZES + 1;
		i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
(size_t)i+1)) {		(size_t)i+1)) {
assert(!extent_heap_empty(&extents->heaps[i]));		assert(!extent_heap_empty(&extents->heaps[i]));
extent_t *extent = extent_heap_first(&extents->heaps[i]);		extent_t *extent = extent_heap_first(&extents->heaps[i]);
assert(extent_size_get(extent) >= size);		assert(extent_size_get(extent) >= size);
		/*
		* In order to reduce fragmentation, avoid reusing and splitting
		* large extents for much smaller sizes.
		*
		* Only do check for dirty extents (delay_coalesce).
		*/
		if (extents->delay_coalesce &&
		(sz_pind2sz(i) >> opt_lg_extent_max_active_fit) > size) {
		break;
		}
if (ret == NULL \|\| extent_snad_comp(extent, ret) < 0) {		if (ret == NULL \|\| extent_snad_comp(extent, ret) < 0) {
ret = extent;		ret = extent;
}		}
if (i == NPSIZES) {		if (i == SC_NPSIZES) {
break;		break;
}		}
assert(i < NPSIZES);		assert(i < SC_NPSIZES);
}		}

return ret;		return ret;
}		}

/*		/*
* Do {best,first}-fit extent selection, where the selection policy choice is		* Do first-fit extent selection, where the selection policy choice is
* based on extents->delay_coalesce. Best-fit selection requires less		* based on extents->delay_coalesce.
* searching, but its layout policy is less stable and may cause higher virtual
* memory fragmentation as a side effect.
*/		*/
static extent_t *		static extent_t *
extents_fit_locked(tsdn_t tsdn, arena_t arena, extents_t *extents,		extents_fit_locked(tsdn_t tsdn, arena_t arena, extents_t *extents,
size_t esize, size_t alignment) {		size_t esize, size_t alignment) {
malloc_mutex_assert_owner(tsdn, &extents->mtx);		malloc_mutex_assert_owner(tsdn, &extents->mtx);

size_t max_size = esize + PAGE_CEILING(alignment) - PAGE;		size_t max_size = esize + PAGE_CEILING(alignment) - PAGE;
/* Beware size_t wrap-around. */		/* Beware size_t wrap-around. */
if (max_size < esize) {		if (max_size < esize) {
return NULL;		return NULL;
}		}

extent_t *extent = extents->delay_coalesce ?		extent_t *extent =
extents_best_fit_locked(tsdn, arena, extents, max_size) :
extents_first_fit_locked(tsdn, arena, extents, max_size);		extents_first_fit_locked(tsdn, arena, extents, max_size);

if (alignment > PAGE && extent == NULL) {		if (alignment > PAGE && extent == NULL) {
/*		/*
* max_size guarantees the alignment requirement but is rather		* max_size guarantees the alignment requirement but is rather
* pessimistic. Next we try to satisfy the aligned allocation		* pessimistic. Next we try to satisfy the aligned allocation
* with sizes in [esize, max_size).		* with sizes in [esize, max_size).
*/		*/
▲ Show 20 Lines • Show All 110 Lines • ▼ Show 20 Lines	default:
not_reached();		not_reached();
}		}

label_return:		label_return:
malloc_mutex_unlock(tsdn, &extents->mtx);		malloc_mutex_unlock(tsdn, &extents->mtx);
return extent;		return extent;
}		}

		/*
		* This can only happen when we fail to allocate a new extent struct (which
		* indicates OOM), e.g. when trying to split an existing extent.
		*/
static void		static void
extents_leak(tsdn_t tsdn, arena_t arena, extent_hooks_t **r_extent_hooks,		extents_abandon_vm(tsdn_t tsdn, arena_t arena, extent_hooks_t **r_extent_hooks,
extents_t extents, extent_t extent, bool growing_retained) {		extents_t extents, extent_t extent, bool growing_retained) {
		size_t sz = extent_size_get(extent);
		if (config_stats) {
		arena_stats_accum_zu(&arena->stats.abandoned_vm, sz);
		}
/*		/*
* Leak extent after making sure its pages have already been purged, so		* Leak extent after making sure its pages have already been purged, so
* that this is only a virtual memory leak.		* that this is only a virtual memory leak.
*/		*/
if (extents_state_get(extents) == extent_state_dirty) {		if (extents_state_get(extents) == extent_state_dirty) {
if (extent_purge_lazy_impl(tsdn, arena, r_extent_hooks,		if (extent_purge_lazy_impl(tsdn, arena, r_extent_hooks,
extent, 0, extent_size_get(extent), growing_retained)) {		extent, 0, sz, growing_retained)) {
extent_purge_forced_impl(tsdn, arena, r_extent_hooks,		extent_purge_forced_impl(tsdn, arena, r_extent_hooks,
extent, 0, extent_size_get(extent),		extent, 0, extent_size_get(extent),
growing_retained);		growing_retained);
}		}
}		}
extent_dalloc(tsdn, arena, extent);		extent_dalloc(tsdn, arena, extent);
}		}

▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines	extent_register_impl(tsdn_t tsdn, extent_t extent, bool gdump_add) {
/*		/*
* We need to hold the lock to protect against a concurrent coalesce		* We need to hold the lock to protect against a concurrent coalesce
* operation that sees us in a partial state.		* operation that sees us in a partial state.
*/		*/
extent_lock(tsdn, extent);		extent_lock(tsdn, extent);

if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false, true,		if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false, true,
&elm_a, &elm_b)) {		&elm_a, &elm_b)) {
		extent_unlock(tsdn, extent);
return true;		return true;
}		}

szind_t szind = extent_szind_get_maybe_invalid(extent);		szind_t szind = extent_szind_get_maybe_invalid(extent);
bool slab = extent_slab_get(extent);		bool slab = extent_slab_get(extent);
extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent, szind, slab);		extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent, szind, slab);
if (slab) {		if (slab) {
extent_interior_register(tsdn, rtree_ctx, extent, szind);		extent_interior_register(tsdn, rtree_ctx, extent, szind);
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines	extent_deregister_impl(tsdn_t tsdn, extent_t extent, bool gdump) {
rtree_ctx_t rtree_ctx_fallback;		rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);		rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_leaf_elm_t elm_a, elm_b;		rtree_leaf_elm_t elm_a, elm_b;
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, true, false,		extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, true, false,
&elm_a, &elm_b);		&elm_a, &elm_b);

extent_lock(tsdn, extent);		extent_lock(tsdn, extent);

extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, NSIZES, false);		extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false);
if (extent_slab_get(extent)) {		if (extent_slab_get(extent)) {
extent_interior_deregister(tsdn, rtree_ctx, extent);		extent_interior_deregister(tsdn, rtree_ctx, extent);
extent_slab_set(extent, false);		extent_slab_set(extent, false);
}		}

extent_unlock(tsdn, extent);		extent_unlock(tsdn, extent);

if (config_prof && gdump) {		if (config_prof && gdump) {
Show All 40 Lines	if (config_debug && new_addr != NULL) {
assert(alignment <= PAGE);		assert(alignment <= PAGE);
}		}

size_t esize = size + pad;		size_t esize = size + pad;
malloc_mutex_lock(tsdn, &extents->mtx);		malloc_mutex_lock(tsdn, &extents->mtx);
extent_hooks_assure_initialized(arena, r_extent_hooks);		extent_hooks_assure_initialized(arena, r_extent_hooks);
extent_t *extent;		extent_t *extent;
if (new_addr != NULL) {		if (new_addr != NULL) {
extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr);		extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr,
		false);
if (extent != NULL) {		if (extent != NULL) {
/*		/*
* We might null-out extent to report an error, but we		* We might null-out extent to report an error, but we
* still need to unlock the associated mutex after.		* still need to unlock the associated mutex after.
*/		*/
extent_t *unlock_extent = extent;		extent_t *unlock_extent = extent;
assert(extent_base_get(extent) == new_addr);		assert(extent_base_get(extent) == new_addr);
if (extent_arena_get(extent) != arena \|\|		if (extent_arena_get(extent) != arena \|\|
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines	extent_split_interior(tsdn_t tsdn, arena_t arena,
*trail = NULL;		*trail = NULL;
*to_leak = NULL;		*to_leak = NULL;
*to_salvage = NULL;		*to_salvage = NULL;

/* Split the lead. */		/* Split the lead. */
if (leadsize != 0) {		if (leadsize != 0) {
lead = extent;		lead = extent;
*extent = extent_split_impl(tsdn, arena, r_extent_hooks,		*extent = extent_split_impl(tsdn, arena, r_extent_hooks,
*lead, leadsize, NSIZES, false, esize + trailsize, szind,		*lead, leadsize, SC_NSIZES, false, esize + trailsize, szind,
slab, growing_retained);		slab, growing_retained);
if (*extent == NULL) {		if (*extent == NULL) {
to_leak = lead;		to_leak = lead;
*lead = NULL;		*lead = NULL;
return extent_split_interior_error;		return extent_split_interior_error;
}		}
}		}

/* Split the trail. */		/* Split the trail. */
if (trailsize != 0) {		if (trailsize != 0) {
trail = extent_split_impl(tsdn, arena, r_extent_hooks, extent,		trail = extent_split_impl(tsdn, arena, r_extent_hooks, extent,
esize, szind, slab, trailsize, NSIZES, false,		esize, szind, slab, trailsize, SC_NSIZES, false,
growing_retained);		growing_retained);
if (*trail == NULL) {		if (*trail == NULL) {
to_leak = extent;		to_leak = extent;
to_salvage = lead;		to_salvage = lead;
*lead = NULL;		*lead = NULL;
*extent = NULL;		*extent = NULL;
return extent_split_interior_error;		return extent_split_interior_error;
}		}
}		}

if (leadsize == 0 && trailsize == 0) {		if (leadsize == 0 && trailsize == 0) {
/*		/*
* Splitting causes szind to be set as a side effect, but no		* Splitting causes szind to be set as a side effect, but no
* splitting occurred.		* splitting occurred.
*/		*/
extent_szind_set(*extent, szind);		extent_szind_set(*extent, szind);
if (szind != NSIZES) {		if (szind != SC_NSIZES) {
rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx,		rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)extent_addr_get(*extent), szind, slab);		(uintptr_t)extent_addr_get(*extent), szind, slab);
if (slab && extent_size_get(*extent) > PAGE) {		if (slab && extent_size_get(*extent) > PAGE) {
rtree_szind_slab_update(tsdn, &extents_rtree,		rtree_szind_slab_update(tsdn, &extents_rtree,
rtree_ctx,		rtree_ctx,
(uintptr_t)extent_past_get(*extent) -		(uintptr_t)extent_past_get(*extent) -
(uintptr_t)PAGE, szind, slab);		(uintptr_t)PAGE, szind, slab);
}		}
Show All 19 Lines	extent_recycle_split(tsdn_t tsdn, arena_t arena,
extent_t *to_leak;		extent_t *to_leak;
extent_t *to_salvage;		extent_t *to_salvage;

extent_split_interior_result_t result = extent_split_interior(		extent_split_interior_result_t result = extent_split_interior(
tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail,		tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail,
&to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind,		&to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind,
growing_retained);		growing_retained);

		if (!maps_coalesce && result != extent_split_interior_ok
		&& !opt_retain) {
		/*
		* Split isn't supported (implies Windows w/o retain). Avoid
		* leaking the extents.
		*/
		assert(to_leak != NULL && lead == NULL && trail == NULL);
		extent_deactivate(tsdn, arena, extents, to_leak);
		return NULL;
		}

if (result == extent_split_interior_ok) {		if (result == extent_split_interior_ok) {
if (lead != NULL) {		if (lead != NULL) {
extent_deactivate(tsdn, arena, extents, lead);		extent_deactivate(tsdn, arena, extents, lead);
}		}
if (trail != NULL) {		if (trail != NULL) {
extent_deactivate(tsdn, arena, extents, trail);		extent_deactivate(tsdn, arena, extents, trail);
}		}
return extent;		return extent;
} else {		} else {
/*		/*
* We should have picked an extent that was large enough to		* We should have picked an extent that was large enough to
* fulfill our allocation request.		* fulfill our allocation request.
*/		*/
assert(result == extent_split_interior_error);		assert(result == extent_split_interior_error);
if (to_salvage != NULL) {		if (to_salvage != NULL) {
extent_deregister(tsdn, to_salvage);		extent_deregister(tsdn, to_salvage);
}		}
if (to_leak != NULL) {		if (to_leak != NULL) {
void *leak = extent_base_get(to_leak);		void *leak = extent_base_get(to_leak);
extent_deregister_no_gdump_sub(tsdn, to_leak);		extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_leak(tsdn, arena, r_extent_hooks, extents,		extents_abandon_vm(tsdn, arena, r_extent_hooks, extents,
to_leak, growing_retained);		to_leak, growing_retained);
assert(extent_lock_from_addr(tsdn, rtree_ctx, leak)		assert(extent_lock_from_addr(tsdn, rtree_ctx, leak,
== NULL);		false) == NULL);
}		}
return NULL;		return NULL;
}		}
unreachable();		unreachable();
}		}

		static bool
		extent_need_manual_zero(arena_t *arena) {
/*		/*
		* Need to manually zero the extent on repopulating if either; 1) non
		* default extent hooks installed (in which case the purge semantics may
		* change); or 2) transparent huge pages enabled.
		*/
		return (!arena_has_default_hooks(arena) \|\|
		(opt_thp == thp_mode_always));
		}

		/*
* Tries to satisfy the given allocation request by reusing one of the extents		* Tries to satisfy the given allocation request by reusing one of the extents
* in the given extents_t.		* in the given extents_t.
*/		*/
static extent_t *		static extent_t *
extent_recycle(tsdn_t tsdn, arena_t arena, extent_hooks_t **r_extent_hooks,		extent_recycle(tsdn_t tsdn, arena_t arena, extent_hooks_t **r_extent_hooks,
extents_t extents, void new_addr, size_t size, size_t pad,		extents_t extents, void new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool zero, bool commit,		size_t alignment, bool slab, szind_t szind, bool zero, bool commit,
bool growing_retained) {		bool growing_retained) {
Show All 22 Lines	extent_recycle(tsdn_t tsdn, arena_t arena, extent_hooks_t **r_extent_hooks,

if (*commit && !extent_committed_get(extent)) {		if (*commit && !extent_committed_get(extent)) {
if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent,		if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent,
0, extent_size_get(extent), growing_retained)) {		0, extent_size_get(extent), growing_retained)) {
extent_record(tsdn, arena, r_extent_hooks, extents,		extent_record(tsdn, arena, r_extent_hooks, extents,
extent, growing_retained);		extent, growing_retained);
return NULL;		return NULL;
}		}
		if (!extent_need_manual_zero(arena)) {
extent_zeroed_set(extent, true);		extent_zeroed_set(extent, true);
}		}
		}

if (extent_committed_get(extent)) {		if (extent_committed_get(extent)) {
*commit = true;		*commit = true;
}		}
if (extent_zeroed_get(extent)) {		if (extent_zeroed_get(extent)) {
*zero = true;		*zero = true;
}		}

if (pad != 0) {		if (pad != 0) {
extent_addr_randomize(tsdn, extent, alignment);		extent_addr_randomize(tsdn, extent, alignment);
}		}
assert(extent_state_get(extent) == extent_state_active);		assert(extent_state_get(extent) == extent_state_active);
if (slab) {		if (slab) {
extent_slab_set(extent, slab);		extent_slab_set(extent, slab);
extent_interior_register(tsdn, rtree_ctx, extent, szind);		extent_interior_register(tsdn, rtree_ctx, extent, szind);
}		}

if (*zero) {		if (*zero) {
void *addr = extent_base_get(extent);		void *addr = extent_base_get(extent);
if (!extent_zeroed_get(extent)) {		if (!extent_zeroed_get(extent)) {
size_t size = extent_size_get(extent);		size_t size = extent_size_get(extent);
if (pages_purge_forced(addr, size)) {		if (extent_need_manual_zero(arena) \|\|
		pages_purge_forced(addr, size)) {
memset(addr, 0, size);		memset(addr, 0, size);
}		}
} else if (config_debug) {		} else if (config_debug) {
size_t p = (size_t )(uintptr_t)addr;		size_t p = (size_t )(uintptr_t)addr;
/* Check the first page only. */		/* Check the first page only. */
for (size_t i = 0; i < PAGE / sizeof(size_t); i++) {		for (size_t i = 0; i < PAGE / sizeof(size_t); i++) {
assert(p[i] == 0);		assert(p[i] == 0);
}		}
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines	extent_alloc_default(extent_hooks_t extent_hooks, void new_addr, size_t size,
arena = arena_get(tsdn, arena_ind, false);		arena = arena_get(tsdn, arena_ind, false);
/*		/*
* The arena we're allocating on behalf of must have been initialized		* The arena we're allocating on behalf of must have been initialized
* already.		* already.
*/		*/
assert(arena != NULL);		assert(arena != NULL);

return extent_alloc_default_impl(tsdn, arena, new_addr, size,		return extent_alloc_default_impl(tsdn, arena, new_addr, size,
alignment, zero, commit);		ALIGNMENT_CEILING(alignment, PAGE), zero, commit);
}		}

static void		static void
extent_hook_pre_reentrancy(tsdn_t tsdn, arena_t arena) {		extent_hook_pre_reentrancy(tsdn_t tsdn, arena_t arena) {
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);		tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
if (arena == arena_get(tsd_tsdn(tsd), 0, false)) {		if (arena == arena_get(tsd_tsdn(tsd), 0, false)) {
/*		/*
* The only legitimate case of customized extent hooks for a0 is		* The only legitimate case of customized extent hooks for a0 is
Show All 36 Lines	extent_grow_retained(tsdn_t tsdn, arena_t arena,
/*		/*
* Find the next extent size in the series that would be large enough to		* Find the next extent size in the series that would be large enough to
* satisfy this request.		* satisfy this request.
*/		*/
pszind_t egn_skip = 0;		pszind_t egn_skip = 0;
size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);		size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
while (alloc_size < alloc_size_min) {		while (alloc_size < alloc_size_min) {
egn_skip++;		egn_skip++;
if (arena->extent_grow_next + egn_skip == NPSIZES) {		if (arena->extent_grow_next + egn_skip >=
		sz_psz2ind(SC_LARGE_MAXCLASS)) {
/* Outside legal range. */		/* Outside legal range. */
goto label_err;		goto label_err;
}		}
assert(arena->extent_grow_next + egn_skip < NPSIZES);
alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);		alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
}		}

extent_t *extent = extent_alloc(tsdn, arena);		extent_t *extent = extent_alloc(tsdn, arena);
if (extent == NULL) {		if (extent == NULL) {
goto label_err;		goto label_err;
}		}
bool zeroed = false;		bool zeroed = false;
bool committed = false;		bool committed = false;

void *ptr;		void *ptr;
if (*r_extent_hooks == &extent_hooks_default) {		if (*r_extent_hooks == &extent_hooks_default) {
ptr = extent_alloc_default_impl(tsdn, arena, NULL,		ptr = extent_alloc_default_impl(tsdn, arena, NULL,
alloc_size, PAGE, &zeroed, &committed);		alloc_size, PAGE, &zeroed, &committed);
} else {		} else {
extent_hook_pre_reentrancy(tsdn, arena);		extent_hook_pre_reentrancy(tsdn, arena);
ptr = (r_extent_hooks)->alloc(r_extent_hooks, NULL,		ptr = (r_extent_hooks)->alloc(r_extent_hooks, NULL,
alloc_size, PAGE, &zeroed, &committed,		alloc_size, PAGE, &zeroed, &committed,
arena_ind_get(arena));		arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn);		extent_hook_post_reentrancy(tsdn);
}		}

extent_init(extent, arena, ptr, alloc_size, false, NSIZES,		extent_init(extent, arena, ptr, alloc_size, false, SC_NSIZES,
arena_extent_sn_next(arena), extent_state_active, zeroed,		arena_extent_sn_next(arena), extent_state_active, zeroed,
committed, true);		committed, true, EXTENT_IS_HEAD);
if (ptr == NULL) {		if (ptr == NULL) {
extent_dalloc(tsdn, arena, extent);		extent_dalloc(tsdn, arena, extent);
goto label_err;		goto label_err;
}		}

if (extent_register_no_gdump_add(tsdn, extent)) {		if (extent_register_no_gdump_add(tsdn, extent)) {
extents_leak(tsdn, arena, r_extent_hooks,		extent_dalloc(tsdn, arena, extent);
&arena->extents_retained, extent, true);
goto label_err;		goto label_err;
}		}

if (extent_zeroed_get(extent) && extent_committed_get(extent)) {		if (extent_zeroed_get(extent) && extent_committed_get(extent)) {
*zero = true;		*zero = true;
}		}
if (extent_committed_get(extent)) {		if (extent_committed_get(extent)) {
*commit = true;		*commit = true;
Show All 30 Lines	if (to_salvage != NULL) {
if (config_prof) {		if (config_prof) {
extent_gdump_add(tsdn, to_salvage);		extent_gdump_add(tsdn, to_salvage);
}		}
extent_record(tsdn, arena, r_extent_hooks,		extent_record(tsdn, arena, r_extent_hooks,
&arena->extents_retained, to_salvage, true);		&arena->extents_retained, to_salvage, true);
}		}
if (to_leak != NULL) {		if (to_leak != NULL) {
extent_deregister_no_gdump_sub(tsdn, to_leak);		extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_leak(tsdn, arena, r_extent_hooks,		extents_abandon_vm(tsdn, arena, r_extent_hooks,
&arena->extents_retained, to_leak, true);		&arena->extents_retained, to_leak, true);
}		}
goto label_err;		goto label_err;
}		}

if (*commit && !extent_committed_get(extent)) {		if (*commit && !extent_committed_get(extent)) {
if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, 0,		if (extent_commit_impl(tsdn, arena, r_extent_hooks, extent, 0,
extent_size_get(extent), true)) {		extent_size_get(extent), true)) {
extent_record(tsdn, arena, r_extent_hooks,		extent_record(tsdn, arena, r_extent_hooks,
&arena->extents_retained, extent, true);		&arena->extents_retained, extent, true);
goto label_err;		goto label_err;
}		}
		if (!extent_need_manual_zero(arena)) {
extent_zeroed_set(extent, true);		extent_zeroed_set(extent, true);
}		}
		}

/*		/*
* Increment extent_grow_next if doing so wouldn't exceed the allowed		* Increment extent_grow_next if doing so wouldn't exceed the allowed
* range.		* range.
*/		*/
if (arena->extent_grow_next + egn_skip + 1 <=		if (arena->extent_grow_next + egn_skip + 1 <=
arena->retain_grow_limit) {		arena->retain_grow_limit) {
arena->extent_grow_next += egn_skip + 1;		arena->extent_grow_next += egn_skip + 1;
Show All 16 Lines	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
&rtree_ctx_fallback);		&rtree_ctx_fallback);

extent_slab_set(extent, true);		extent_slab_set(extent, true);
extent_interior_register(tsdn, rtree_ctx, extent, szind);		extent_interior_register(tsdn, rtree_ctx, extent, szind);
}		}
if (*zero && !extent_zeroed_get(extent)) {		if (*zero && !extent_zeroed_get(extent)) {
void *addr = extent_base_get(extent);		void *addr = extent_base_get(extent);
size_t size = extent_size_get(extent);		size_t size = extent_size_get(extent);
if (pages_purge_forced(addr, size)) {		if (extent_need_manual_zero(arena) \|\|
		pages_purge_forced(addr, size)) {
memset(addr, 0, size);		memset(addr, 0, size);
}		}
}		}

return extent;		return extent;
label_err:		label_err:
malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);		malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
return NULL;		return NULL;
Show All 33 Lines	extent_alloc_wrapper_hard(tsdn_t tsdn, arena_t arena,
extent_hooks_t *r_extent_hooks, void new_addr, size_t size, size_t pad,		extent_hooks_t *r_extent_hooks, void new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool zero, bool commit) {		size_t alignment, bool slab, szind_t szind, bool zero, bool commit) {
size_t esize = size + pad;		size_t esize = size + pad;
extent_t *extent = extent_alloc(tsdn, arena);		extent_t *extent = extent_alloc(tsdn, arena);
if (extent == NULL) {		if (extent == NULL) {
return NULL;		return NULL;
}		}
void *addr;		void *addr;
		size_t palignment = ALIGNMENT_CEILING(alignment, PAGE);
if (*r_extent_hooks == &extent_hooks_default) {		if (*r_extent_hooks == &extent_hooks_default) {
/* Call directly to propagate tsdn. */		/* Call directly to propagate tsdn. */
addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize,		addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize,
alignment, zero, commit);		palignment, zero, commit);
} else {		} else {
extent_hook_pre_reentrancy(tsdn, arena);		extent_hook_pre_reentrancy(tsdn, arena);
addr = (r_extent_hooks)->alloc(r_extent_hooks, new_addr,		addr = (r_extent_hooks)->alloc(r_extent_hooks, new_addr,
esize, alignment, zero, commit, arena_ind_get(arena));		esize, palignment, zero, commit, arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn);		extent_hook_post_reentrancy(tsdn);
}		}
if (addr == NULL) {		if (addr == NULL) {
extent_dalloc(tsdn, arena, extent);		extent_dalloc(tsdn, arena, extent);
return NULL;		return NULL;
}		}
extent_init(extent, arena, addr, esize, slab, szind,		extent_init(extent, arena, addr, esize, slab, szind,
arena_extent_sn_next(arena), extent_state_active, zero, commit,		arena_extent_sn_next(arena), extent_state_active, zero, commit,
true);		true, EXTENT_NOT_HEAD);
if (pad != 0) {		if (pad != 0) {
extent_addr_randomize(tsdn, extent, alignment);		extent_addr_randomize(tsdn, extent, alignment);
}		}
if (extent_register(tsdn, extent)) {		if (extent_register(tsdn, extent)) {
extents_leak(tsdn, arena, r_extent_hooks,		extent_dalloc(tsdn, arena, extent);
&arena->extents_retained, extent, false);
return NULL;		return NULL;
}		}

return extent;		return extent;
}		}

extent_t *		extent_t *
extent_alloc_wrapper(tsdn_t tsdn, arena_t arena,		extent_alloc_wrapper(tsdn_t tsdn, arena_t arena,
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines	extent_coalesce(tsdn_t tsdn, arena_t arena, extent_hooks_t **r_extent_hooks,
if (err) {		if (err) {
extent_deactivate_locked(tsdn, arena, extents, outer);		extent_deactivate_locked(tsdn, arena, extents, outer);
}		}

return err;		return err;
}		}

static extent_t *		static extent_t *
extent_try_coalesce(tsdn_t tsdn, arena_t arena,		extent_try_coalesce_impl(tsdn_t tsdn, arena_t arena,
extent_hooks_t *r_extent_hooks, rtree_ctx_t rtree_ctx, extents_t *extents,		extent_hooks_t *r_extent_hooks, rtree_ctx_t rtree_ctx, extents_t *extents,
extent_t extent, bool coalesced, bool growing_retained) {		extent_t extent, bool coalesced, bool growing_retained,
		bool inactive_only) {
/*		/*
		* We avoid checking / locking inactive neighbors for large size
		* classes, since they are eagerly coalesced on deallocation which can
		* cause lock contention.
		*/
		/*
* Continue attempting to coalesce until failure, to protect against		* Continue attempting to coalesce until failure, to protect against
* races with other threads that are thwarted by this one.		* races with other threads that are thwarted by this one.
*/		*/
bool again;		bool again;
do {		do {
again = false;		again = false;

/* Try to coalesce forward. */		/* Try to coalesce forward. */
extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx,		extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx,
extent_past_get(extent));		extent_past_get(extent), inactive_only);
if (next != NULL) {		if (next != NULL) {
/*		/*
* extents->mtx only protects against races for		* extents->mtx only protects against races for
* like-state extents, so call extent_can_coalesce()		* like-state extents, so call extent_can_coalesce()
* before releasing next's pool lock.		* before releasing next's pool lock.
*/		*/
bool can_coalesce = extent_can_coalesce(arena, extents,		bool can_coalesce = extent_can_coalesce(arena, extents,
extent, next);		extent, next);
Show All 9 Lines	if (next != NULL) {
return extent;		return extent;
}		}
again = true;		again = true;
}		}
}		}

/* Try to coalesce backward. */		/* Try to coalesce backward. */
extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx,		extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx,
extent_before_get(extent));		extent_before_get(extent), inactive_only);
if (prev != NULL) {		if (prev != NULL) {
bool can_coalesce = extent_can_coalesce(arena, extents,		bool can_coalesce = extent_can_coalesce(arena, extents,
extent, prev);		extent, prev);
extent_unlock(tsdn, prev);		extent_unlock(tsdn, prev);

if (can_coalesce && !extent_coalesce(tsdn, arena,		if (can_coalesce && !extent_coalesce(tsdn, arena,
r_extent_hooks, extents, extent, prev, false,		r_extent_hooks, extents, extent, prev, false,
growing_retained)) {		growing_retained)) {
Show All 9 Lines	extent_try_coalesce_impl(tsdn_t tsdn, arena_t arena,
} while (again);		} while (again);

if (extents->delay_coalesce) {		if (extents->delay_coalesce) {
*coalesced = false;		*coalesced = false;
}		}
return extent;		return extent;
}		}

		static extent_t *
		extent_try_coalesce(tsdn_t tsdn, arena_t arena,
		extent_hooks_t *r_extent_hooks, rtree_ctx_t rtree_ctx, extents_t *extents,
		extent_t extent, bool coalesced, bool growing_retained) {
		return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx,
		extents, extent, coalesced, growing_retained, false);
		}

		static extent_t *
		extent_try_coalesce_large(tsdn_t tsdn, arena_t arena,
		extent_hooks_t *r_extent_hooks, rtree_ctx_t rtree_ctx, extents_t *extents,
		extent_t extent, bool coalesced, bool growing_retained) {
		return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx,
		extents, extent, coalesced, growing_retained, true);
		}

/*		/*
* Does the metadata management portions of putting an unused extent into the		* Does the metadata management portions of putting an unused extent into the
* given extents_t (coalesces, deregisters slab interiors, the heap operations).		* given extents_t (coalesces, deregisters slab interiors, the heap operations).
*/		*/
static void		static void
extent_record(tsdn_t tsdn, arena_t arena, extent_hooks_t **r_extent_hooks,		extent_record(tsdn_t tsdn, arena_t arena, extent_hooks_t **r_extent_hooks,
extents_t extents, extent_t extent, bool growing_retained) {		extents_t extents, extent_t extent, bool growing_retained) {
rtree_ctx_t rtree_ctx_fallback;		rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);		rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);

assert((extents_state_get(extents) != extent_state_dirty &&		assert((extents_state_get(extents) != extent_state_dirty &&
extents_state_get(extents) != extent_state_muzzy) \|\|		extents_state_get(extents) != extent_state_muzzy) \|\|
!extent_zeroed_get(extent));		!extent_zeroed_get(extent));

malloc_mutex_lock(tsdn, &extents->mtx);		malloc_mutex_lock(tsdn, &extents->mtx);
extent_hooks_assure_initialized(arena, r_extent_hooks);		extent_hooks_assure_initialized(arena, r_extent_hooks);

extent_szind_set(extent, NSIZES);		extent_szind_set(extent, SC_NSIZES);
if (extent_slab_get(extent)) {		if (extent_slab_get(extent)) {
extent_interior_deregister(tsdn, rtree_ctx, extent);		extent_interior_deregister(tsdn, rtree_ctx, extent);
extent_slab_set(extent, false);		extent_slab_set(extent, false);
}		}

assert(rtree_extent_read(tsdn, &extents_rtree, rtree_ctx,		assert(rtree_extent_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)extent_base_get(extent), true) == extent);		(uintptr_t)extent_base_get(extent), true) == extent);

if (!extents->delay_coalesce) {		if (!extents->delay_coalesce) {
extent = extent_try_coalesce(tsdn, arena, r_extent_hooks,		extent = extent_try_coalesce(tsdn, arena, r_extent_hooks,
rtree_ctx, extents, extent, NULL, growing_retained);		rtree_ctx, extents, extent, NULL, growing_retained);
} else if (extent_size_get(extent) >= LARGE_MINCLASS) {		} else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) {
		assert(extents == &arena->extents_dirty);
/* Always coalesce large extents eagerly. */		/* Always coalesce large extents eagerly. */
bool coalesced;		bool coalesced;
size_t prev_size;
do {		do {
prev_size = extent_size_get(extent);
assert(extent_state_get(extent) == extent_state_active);		assert(extent_state_get(extent) == extent_state_active);
extent = extent_try_coalesce(tsdn, arena,		extent = extent_try_coalesce_large(tsdn, arena,
r_extent_hooks, rtree_ctx, extents, extent,		r_extent_hooks, rtree_ctx, extents, extent,
&coalesced, growing_retained);		&coalesced, growing_retained);
} while (coalesced &&		} while (coalesced);
extent_size_get(extent) >= prev_size + LARGE_MINCLASS);		if (extent_size_get(extent) >= oversize_threshold) {
		/* Shortcut to purge the oversize extent eagerly. */
		malloc_mutex_unlock(tsdn, &extents->mtx);
		arena_decay_extent(tsdn, arena, r_extent_hooks, extent);
		return;
}		}
		}
extent_deactivate_locked(tsdn, arena, extents, extent);		extent_deactivate_locked(tsdn, arena, extents, extent);

malloc_mutex_unlock(tsdn, &extents->mtx);		malloc_mutex_unlock(tsdn, &extents->mtx);
}		}

void		void
extent_dalloc_gap(tsdn_t tsdn, arena_t arena, extent_t *extent) {		extent_dalloc_gap(tsdn_t tsdn, arena_t arena, extent_t *extent) {
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;		extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;

witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),		witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);		WITNESS_RANK_CORE, 0);

if (extent_register(tsdn, extent)) {		if (extent_register(tsdn, extent)) {
extents_leak(tsdn, arena, &extent_hooks,		extent_dalloc(tsdn, arena, extent);
&arena->extents_retained, extent, false);
return;		return;
}		}
extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent);		extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent);
}		}

static bool		static bool
		extent_may_dalloc(void) {
		/* With retain enabled, the default dalloc always fails. */
		return !opt_retain;
		}

		static bool
extent_dalloc_default_impl(void *addr, size_t size) {		extent_dalloc_default_impl(void *addr, size_t size) {
if (!have_dss \|\| !extent_in_dss(addr)) {		if (!have_dss \|\| !extent_in_dss(addr)) {
return extent_dalloc_mmap(addr, size);		return extent_dalloc_mmap(addr, size);
}		}
return true;		return true;
}		}

static bool		static bool
Show All 38 Lines

void		void
extent_dalloc_wrapper(tsdn_t tsdn, arena_t arena,		extent_dalloc_wrapper(tsdn_t tsdn, arena_t arena,
extent_hooks_t *r_extent_hooks, extent_t extent) {		extent_hooks_t *r_extent_hooks, extent_t extent) {
assert(extent_dumpable_get(extent));		assert(extent_dumpable_get(extent));
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),		witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);		WITNESS_RANK_CORE, 0);

		/* Avoid calling the default extent_dalloc unless have to. */
		if (*r_extent_hooks != &extent_hooks_default \|\| extent_may_dalloc()) {
/*		/*
* Deregister first to avoid a race with other allocating threads, and		* Deregister first to avoid a race with other allocating
* reregister if deallocation fails.		* threads, and reregister if deallocation fails.
*/		*/
extent_deregister(tsdn, extent);		extent_deregister(tsdn, extent);
if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) {		if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks,
		extent)) {
return;		return;
}		}

extent_reregister(tsdn, extent);		extent_reregister(tsdn, extent);
		}

if (*r_extent_hooks != &extent_hooks_default) {		if (*r_extent_hooks != &extent_hooks_default) {
extent_hook_pre_reentrancy(tsdn, arena);		extent_hook_pre_reentrancy(tsdn, arena);
}		}
/* Try to decommit; purge if that fails. */		/* Try to decommit; purge if that fails. */
bool zeroed;		bool zeroed;
if (!extent_committed_get(extent)) {		if (!extent_committed_get(extent)) {
zeroed = true;		zeroed = true;
} else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,		} else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
▲ Show 20 Lines • Show All 223 Lines • ▼ Show 20 Lines
bool		bool
extent_purge_forced_wrapper(tsdn_t tsdn, arena_t arena,		extent_purge_forced_wrapper(tsdn_t tsdn, arena_t arena,
extent_hooks_t *r_extent_hooks, extent_t extent, size_t offset,		extent_hooks_t *r_extent_hooks, extent_t extent, size_t offset,
size_t length) {		size_t length) {
return extent_purge_forced_impl(tsdn, arena, r_extent_hooks, extent,		return extent_purge_forced_impl(tsdn, arena, r_extent_hooks, extent,
offset, length, false);		offset, length, false);
}		}

#ifdef JEMALLOC_MAPS_COALESCE
static bool		static bool
extent_split_default(extent_hooks_t extent_hooks, void addr, size_t size,		extent_split_default(extent_hooks_t extent_hooks, void addr, size_t size,
size_t size_a, size_t size_b, bool committed, unsigned arena_ind) {		size_t size_a, size_t size_b, bool committed, unsigned arena_ind) {
return !maps_coalesce;		if (!maps_coalesce) {
		/*
		* Without retain, only whole regions can be purged (required by
		* MEM_RELEASE on Windows) -- therefore disallow splitting. See
		* comments in extent_head_no_merge().
		*/
		return !opt_retain;
}		}
#endif

		return false;
		}

/*		/*
* Accepts the extent to split, and the characteristics of each side of the		* Accepts the extent to split, and the characteristics of each side of the
* split. The 'a' parameters go with the 'lead' of the resulting pair of		* split. The 'a' parameters go with the 'lead' of the resulting pair of
* extents (the lower addressed portion of the split), and the 'b' parameters go		* extents (the lower addressed portion of the split), and the 'b' parameters go
* with the trail (the higher addressed portion). This makes 'extent' the lead,		* with the trail (the higher addressed portion). This makes 'extent' the lead,
* and returns the trail (except in case of error).		* and returns the trail (except in case of error).
*/		*/
static extent_t *		static extent_t *
Show All 14 Lines	extent_split_impl(tsdn_t tsdn, arena_t arena,
extent_t *trail = extent_alloc(tsdn, arena);		extent_t *trail = extent_alloc(tsdn, arena);
if (trail == NULL) {		if (trail == NULL) {
goto label_error_a;		goto label_error_a;
}		}

extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) +		extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) +
size_a), size_b, slab_b, szind_b, extent_sn_get(extent),		size_a), size_b, slab_b, szind_b, extent_sn_get(extent),
extent_state_get(extent), extent_zeroed_get(extent),		extent_state_get(extent), extent_zeroed_get(extent),
extent_committed_get(extent), extent_dumpable_get(extent));		extent_committed_get(extent), extent_dumpable_get(extent),
		EXTENT_NOT_HEAD);

rtree_ctx_t rtree_ctx_fallback;		rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);		rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_leaf_elm_t lead_elm_a, lead_elm_b;		rtree_leaf_elm_t lead_elm_a, lead_elm_b;
{		{
extent_t lead;		extent_t lead;

extent_init(&lead, arena, extent_addr_get(extent), size_a,		extent_init(&lead, arena, extent_addr_get(extent), size_a,
slab_a, szind_a, extent_sn_get(extent),		slab_a, szind_a, extent_sn_get(extent),
extent_state_get(extent), extent_zeroed_get(extent),		extent_state_get(extent), extent_zeroed_get(extent),
extent_committed_get(extent), extent_dumpable_get(extent));		extent_committed_get(extent), extent_dumpable_get(extent),
		EXTENT_NOT_HEAD);

extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false,		extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false,
true, &lead_elm_a, &lead_elm_b);		true, &lead_elm_a, &lead_elm_b);
}		}
rtree_leaf_elm_t trail_elm_a, trail_elm_b;		rtree_leaf_elm_t trail_elm_a, trail_elm_b;
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, trail, false, true,		extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, trail, false, true,
&trail_elm_a, &trail_elm_b);		&trail_elm_a, &trail_elm_b);

▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines	extent_split_wrapper(tsdn_t tsdn, arena_t arena,
extent_hooks_t *r_extent_hooks, extent_t extent, size_t size_a,		extent_hooks_t *r_extent_hooks, extent_t extent, size_t size_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) {		szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) {
return extent_split_impl(tsdn, arena, r_extent_hooks, extent, size_a,		return extent_split_impl(tsdn, arena, r_extent_hooks, extent, size_a,
szind_a, slab_a, size_b, szind_b, slab_b, false);		szind_a, slab_a, size_b, szind_b, slab_b, false);
}		}

static bool		static bool
extent_merge_default_impl(void addr_a, void addr_b) {		extent_merge_default_impl(void addr_a, void addr_b) {
if (!maps_coalesce) {		if (!maps_coalesce && !opt_retain) {
return true;		return true;
}		}
if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) {		if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) {
return true;		return true;
}		}

return false;		return false;
}		}

#ifdef JEMALLOC_MAPS_COALESCE		/*
		* Returns true if the given extents can't be merged because of their head bit
		* settings. Assumes the second extent has the higher address.
		*/
static bool		static bool
		extent_head_no_merge(extent_t a, extent_t b) {
		assert(extent_base_get(a) < extent_base_get(b));
		/*
		* When coalesce is not always allowed (Windows), only merge extents
		* from the same VirtualAlloc region under opt.retain (in which case
		* MEM_DECOMMIT is utilized for purging).
		*/
		if (maps_coalesce) {
		return false;
		}
		if (!opt_retain) {
		return true;
		}
		/* If b is a head extent, disallow the cross-region merge. */
		if (extent_is_head_get(b)) {
		/*
		* Additionally, sn should not overflow with retain; sanity
		* check that different regions have unique sn.
		*/
		assert(extent_sn_comp(a, b) != 0);
		return true;
		}
		assert(extent_sn_comp(a, b) == 0);

		return false;
		}

		static bool
extent_merge_default(extent_hooks_t extent_hooks, void addr_a, size_t size_a,		extent_merge_default(extent_hooks_t extent_hooks, void addr_a, size_t size_a,
void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {		void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
		if (!maps_coalesce) {
		tsdn_t *tsdn = tsdn_fetch();
		extent_t *a = iealloc(tsdn, addr_a);
		extent_t *b = iealloc(tsdn, addr_b);
		if (extent_head_no_merge(a, b)) {
		return true;
		}
		}
return extent_merge_default_impl(addr_a, addr_b);		return extent_merge_default_impl(addr_a, addr_b);
}		}
#endif

static bool		static bool
extent_merge_impl(tsdn_t tsdn, arena_t arena,		extent_merge_impl(tsdn_t tsdn, arena_t arena,
extent_hooks_t *r_extent_hooks, extent_t a, extent_t *b,		extent_hooks_t *r_extent_hooks, extent_t a, extent_t *b,
bool growing_retained) {		bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),		witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);		WITNESS_RANK_CORE, growing_retained ? 1 : 0);
		assert(extent_base_get(a) < extent_base_get(b));

extent_hooks_assure_initialized(arena, r_extent_hooks);		extent_hooks_assure_initialized(arena, r_extent_hooks);

if ((*r_extent_hooks)->merge == NULL) {		if ((*r_extent_hooks)->merge == NULL \|\| extent_head_no_merge(a, b)) {
return true;		return true;
}		}

bool err;		bool err;
if (*r_extent_hooks == &extent_hooks_default) {		if (*r_extent_hooks == &extent_hooks_default) {
/* Call directly to propagate tsdn. */		/* Call directly to propagate tsdn. */
err = extent_merge_default_impl(extent_base_get(a),		err = extent_merge_default_impl(extent_base_get(a),
extent_base_get(b));		extent_base_get(b));
Show All 22 Lines	extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, a, true, false, &a_elm_a,
&a_elm_b);		&a_elm_b);
extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, b, true, false, &b_elm_a,		extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, b, true, false, &b_elm_a,
&b_elm_b);		&b_elm_b);

extent_lock2(tsdn, a, b);		extent_lock2(tsdn, a, b);

if (a_elm_b != NULL) {		if (a_elm_b != NULL) {
rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL,		rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL,
NSIZES, false);		SC_NSIZES, false);
}		}
if (b_elm_b != NULL) {		if (b_elm_b != NULL) {
rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL,		rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL,
NSIZES, false);		SC_NSIZES, false);
} else {		} else {
b_elm_b = b_elm_a;		b_elm_b = b_elm_a;
}		}

extent_size_set(a, extent_size_get(a) + extent_size_get(b));		extent_size_set(a, extent_size_get(a) + extent_size_get(b));
extent_szind_set(a, NSIZES);		extent_szind_set(a, SC_NSIZES);
extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ?		extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ?
extent_sn_get(a) : extent_sn_get(b));		extent_sn_get(a) : extent_sn_get(b));
extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b));		extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b));

extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, NSIZES, false);		extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES,
		false);

extent_unlock2(tsdn, a, b);		extent_unlock2(tsdn, a, b);

extent_dalloc(tsdn, extent_arena_get(b), b);		extent_dalloc(tsdn, extent_arena_get(b), b);

return false;		return false;
}		}

Show All 14 Lines	if (mutex_pool_init(&extent_mutex_pool, "extent_mutex_pool",
return true;		return true;
}		}

if (have_dss) {		if (have_dss) {
extent_dss_boot();		extent_dss_boot();
}		}

return false;		return false;
		}

		void
		extent_util_stats_get(tsdn_t tsdn, const void ptr,
		size_t nfree, size_t nregs, size_t *size) {
		assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL);

		const extent_t *extent = iealloc(tsdn, ptr);
		if (unlikely(extent == NULL)) {
		nfree = nregs = *size = 0;
		return;
		}

		*size = extent_size_get(extent);
		if (!extent_slab_get(extent)) {
		*nfree = 0;
		*nregs = 1;
		} else {
		*nfree = extent_nfree_get(extent);
		*nregs = bin_infos[extent_szind_get(extent)].nregs;
		assert(nfree <= nregs);
		assert(nfree extent_usize_get(extent) <= *size);
		}
		}

		void
		extent_util_stats_verbose_get(tsdn_t tsdn, const void ptr,
		size_t nfree, size_t nregs, size_t *size,
		size_t bin_nfree, size_t bin_nregs, void **slabcur_addr) {
		assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL
		&& bin_nfree != NULL && bin_nregs != NULL && slabcur_addr != NULL);

		const extent_t *extent = iealloc(tsdn, ptr);
		if (unlikely(extent == NULL)) {
		nfree = nregs = size = bin_nfree = *bin_nregs = 0;
		*slabcur_addr = NULL;
		return;
		}

		*size = extent_size_get(extent);
		if (!extent_slab_get(extent)) {
		nfree = bin_nfree = *bin_nregs = 0;
		*nregs = 1;
		*slabcur_addr = NULL;
		return;
		}

		*nfree = extent_nfree_get(extent);
		const szind_t szind = extent_szind_get(extent);
		*nregs = bin_infos[szind].nregs;
		assert(nfree <= nregs);
		assert(nfree extent_usize_get(extent) <= *size);

		const arena_t *arena = extent_arena_get(extent);
		assert(arena != NULL);
		const unsigned binshard = extent_binshard_get(extent);
		bin_t *bin = &arena->bins[szind].bin_shards[binshard];

		malloc_mutex_lock(tsdn, &bin->lock);
		if (config_stats) {
		bin_nregs = nregs * bin->stats.curslabs;
		assert(*bin_nregs >= bin->stats.curregs);
		bin_nfree = bin_nregs - bin->stats.curregs;
		} else {
		bin_nfree = bin_nregs = 0;
		}
		*slabcur_addr = extent_addr_get(bin->slabcur);
		assert(*slabcur_addr != NULL);
		malloc_mutex_unlock(tsdn, &bin->lock);
}		}