Differential D24522 Diff 70932 head/contrib/jemalloc/src/large.c

Changeset View

Standalone View

head/contrib/jemalloc/src/large.c

Show All 22 Lines	large_palloc(tsdn_t tsdn, arena_t arena, size_t usize, size_t alignment,
size_t ausize;		size_t ausize;
extent_t *extent;		extent_t *extent;
bool is_zeroed;		bool is_zeroed;
UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false);		UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false);

assert(!tsdn_null(tsdn) \|\| arena != NULL);		assert(!tsdn_null(tsdn) \|\| arena != NULL);

ausize = sz_sa2u(usize, alignment);		ausize = sz_sa2u(usize, alignment);
if (unlikely(ausize == 0 \|\| ausize > LARGE_MAXCLASS)) {		if (unlikely(ausize == 0 \|\| ausize > SC_LARGE_MAXCLASS)) {
return NULL;		return NULL;
}		}

if (config_fill && unlikely(opt_zero)) {		if (config_fill && unlikely(opt_zero)) {
zero = true;		zero = true;
}		}
/*		/*
* Copy zero into is_zeroed and pass the copy when allocating the		* Copy zero into is_zeroed and pass the copy when allocating the
* extent, so that it is possible to make correct junk/zero fill		* extent, so that it is possible to make correct junk/zero fill
* decisions below, even if is_zeroed ends up true when zero is false.		* decisions below, even if is_zeroed ends up true when zero is false.
*/		*/
is_zeroed = zero;		is_zeroed = zero;
if (likely(!tsdn_null(tsdn))) {		if (likely(!tsdn_null(tsdn))) {
arena = arena_choose(tsdn_tsd(tsdn), arena);		arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize);
}		}
if (unlikely(arena == NULL) \|\| (extent = arena_extent_alloc_large(tsdn,		if (unlikely(arena == NULL) \|\| (extent = arena_extent_alloc_large(tsdn,
arena, usize, alignment, &is_zeroed)) == NULL) {		arena, usize, alignment, &is_zeroed)) == NULL) {
return NULL;		return NULL;
}		}

/* See comments in arena_bin_slabs_full_insert(). */		/* See comments in arena_bin_slabs_full_insert(). */
if (!arena_is_auto(arena)) {		if (!arena_is_auto(arena)) {
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines	large_ralloc_no_move_shrink(tsdn_t tsdn, extent_t extent, size_t usize) {
if (extent_hooks->split == NULL) {		if (extent_hooks->split == NULL) {
return true;		return true;
}		}

/* Split excess pages. */		/* Split excess pages. */
if (diff != 0) {		if (diff != 0) {
extent_t *trail = extent_split_wrapper(tsdn, arena,		extent_t *trail = extent_split_wrapper(tsdn, arena,
&extent_hooks, extent, usize + sz_large_pad,		&extent_hooks, extent, usize + sz_large_pad,
sz_size2index(usize), false, diff, NSIZES, false);		sz_size2index(usize), false, diff, SC_NSIZES, false);
if (trail == NULL) {		if (trail == NULL) {
return true;		return true;
}		}

if (config_fill && unlikely(opt_junk_free)) {		if (config_fill && unlikely(opt_junk_free)) {
large_dalloc_maybe_junk(extent_addr_get(trail),		large_dalloc_maybe_junk(extent_addr_get(trail),
extent_size_get(trail));		extent_size_get(trail));
}		}
Show All 28 Lines	large_ralloc_no_move_expand(tsdn_t tsdn, extent_t extent, size_t usize,
* false.		* false.
*/		*/
bool is_zeroed_trail = zero;		bool is_zeroed_trail = zero;
bool commit = true;		bool commit = true;
extent_t *trail;		extent_t *trail;
bool new_mapping;		bool new_mapping;
if ((trail = extents_alloc(tsdn, arena, &extent_hooks,		if ((trail = extents_alloc(tsdn, arena, &extent_hooks,
&arena->extents_dirty, extent_past_get(extent), trailsize, 0,		&arena->extents_dirty, extent_past_get(extent), trailsize, 0,
CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL		CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL
\|\| (trail = extents_alloc(tsdn, arena, &extent_hooks,		\|\| (trail = extents_alloc(tsdn, arena, &extent_hooks,
&arena->extents_muzzy, extent_past_get(extent), trailsize, 0,		&arena->extents_muzzy, extent_past_get(extent), trailsize, 0,
CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL) {		CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) {
if (config_stats) {		if (config_stats) {
new_mapping = false;		new_mapping = false;
}		}
} else {		} else {
if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks,		if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks,
extent_past_get(extent), trailsize, 0, CACHELINE, false,		extent_past_get(extent), trailsize, 0, CACHELINE, false,
NSIZES, &is_zeroed_trail, &commit)) == NULL) {		SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) {
return true;		return true;
}		}
if (config_stats) {		if (config_stats) {
new_mapping = true;		new_mapping = true;
}		}
}		}

if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) {		if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) {
Show All 40 Lines
}		}

bool		bool
large_ralloc_no_move(tsdn_t tsdn, extent_t extent, size_t usize_min,		large_ralloc_no_move(tsdn_t tsdn, extent_t extent, size_t usize_min,
size_t usize_max, bool zero) {		size_t usize_max, bool zero) {
size_t oldusize = extent_usize_get(extent);		size_t oldusize = extent_usize_get(extent);

/* The following should have been caught by callers. */		/* The following should have been caught by callers. */
assert(usize_min > 0 && usize_max <= LARGE_MAXCLASS);		assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS);
/* Both allocation sizes must be large to avoid a move. */		/* Both allocation sizes must be large to avoid a move. */
assert(oldusize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS);		assert(oldusize >= SC_LARGE_MINCLASS
		&& usize_max >= SC_LARGE_MINCLASS);

if (usize_max > oldusize) {		if (usize_max > oldusize) {
/* Attempt to expand the allocation in-place. */		/* Attempt to expand the allocation in-place. */
if (!large_ralloc_no_move_expand(tsdn, extent, usize_max,		if (!large_ralloc_no_move_expand(tsdn, extent, usize_max,
zero)) {		zero)) {
arena_decay_tick(tsdn, extent_arena_get(extent));		arena_decay_tick(tsdn, extent_arena_get(extent));
return false;		return false;
}		}
Show All 30 Lines	large_ralloc_move_helper(tsdn_t tsdn, arena_t arena, size_t usize,
size_t alignment, bool zero) {		size_t alignment, bool zero) {
if (alignment <= CACHELINE) {		if (alignment <= CACHELINE) {
return large_malloc(tsdn, arena, usize, zero);		return large_malloc(tsdn, arena, usize, zero);
}		}
return large_palloc(tsdn, arena, usize, alignment, zero);		return large_palloc(tsdn, arena, usize, alignment, zero);
}		}

void *		void *
large_ralloc(tsdn_t tsdn, arena_t arena, extent_t *extent, size_t usize,		large_ralloc(tsdn_t tsdn, arena_t arena, void *ptr, size_t usize,
size_t alignment, bool zero, tcache_t *tcache) {		size_t alignment, bool zero, tcache_t *tcache,
size_t oldusize = extent_usize_get(extent);		hook_ralloc_args_t *hook_args) {
		extent_t *extent = iealloc(tsdn, ptr);

		size_t oldusize = extent_usize_get(extent);
/* The following should have been caught by callers. */		/* The following should have been caught by callers. */
assert(usize > 0 && usize <= LARGE_MAXCLASS);		assert(usize > 0 && usize <= SC_LARGE_MAXCLASS);
/* Both allocation sizes must be large to avoid a move. */		/* Both allocation sizes must be large to avoid a move. */
assert(oldusize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS);		assert(oldusize >= SC_LARGE_MINCLASS
		&& usize >= SC_LARGE_MINCLASS);

/* Try to avoid moving the allocation. */		/* Try to avoid moving the allocation. */
if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) {		if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) {
		hook_invoke_expand(hook_args->is_realloc
		? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize,
		usize, (uintptr_t)ptr, hook_args->args);
return extent_addr_get(extent);		return extent_addr_get(extent);
}		}

/*		/*
* usize and old size are different enough that we need to use a		* usize and old size are different enough that we need to use a
* different size class. In that case, fall back to allocating new		* different size class. In that case, fall back to allocating new
* space and copying.		* space and copying.
*/		*/
void *ret = large_ralloc_move_helper(tsdn, arena, usize, alignment,		void *ret = large_ralloc_move_helper(tsdn, arena, usize, alignment,
zero);		zero);
if (ret == NULL) {		if (ret == NULL) {
return NULL;		return NULL;
}		}

		hook_invoke_alloc(hook_args->is_realloc
		? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret,
		hook_args->args);
		hook_invoke_dalloc(hook_args->is_realloc
		? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args);

size_t copysize = (usize < oldusize) ? usize : oldusize;		size_t copysize = (usize < oldusize) ? usize : oldusize;
memcpy(ret, extent_addr_get(extent), copysize);		memcpy(ret, extent_addr_get(extent), copysize);
isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true);		isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true);
return ret;		return ret;
}		}

/*		/*
* junked_locked indicates whether the extent's data have been junk-filled, and		* junked_locked indicates whether the extent's data have been junk-filled, and
* whether the arena's large_mtx is currently held.		* whether the arena's large_mtx is currently held.
*/		*/
static void		static void
large_dalloc_prep_impl(tsdn_t tsdn, arena_t arena, extent_t *extent,		large_dalloc_prep_impl(tsdn_t tsdn, arena_t arena, extent_t *extent,
bool junked_locked) {		bool junked_locked) {
if (!junked_locked) {		if (!junked_locked) {
/* See comments in arena_bin_slabs_full_insert(). */		/* See comments in arena_bin_slabs_full_insert(). */
if (!arena_is_auto(arena)) {		if (!arena_is_auto(arena)) {
malloc_mutex_lock(tsdn, &arena->large_mtx);		malloc_mutex_lock(tsdn, &arena->large_mtx);
extent_list_remove(&arena->large, extent);		extent_list_remove(&arena->large, extent);
malloc_mutex_unlock(tsdn, &arena->large_mtx);		malloc_mutex_unlock(tsdn, &arena->large_mtx);
}		}
large_dalloc_maybe_junk(extent_addr_get(extent),		large_dalloc_maybe_junk(extent_addr_get(extent),
extent_usize_get(extent));		extent_usize_get(extent));
} else {		} else {
malloc_mutex_assert_owner(tsdn, &arena->large_mtx);		/* Only hold the large_mtx if necessary. */
if (!arena_is_auto(arena)) {		if (!arena_is_auto(arena)) {
		malloc_mutex_assert_owner(tsdn, &arena->large_mtx);
extent_list_remove(&arena->large, extent);		extent_list_remove(&arena->large, extent);
}		}
}		}
arena_extent_dalloc_large_prep(tsdn, arena, extent);		arena_extent_dalloc_large_prep(tsdn, arena, extent);
}		}

static void		static void
large_dalloc_finish_impl(tsdn_t tsdn, arena_t arena, extent_t *extent) {		large_dalloc_finish_impl(tsdn_t tsdn, arena_t arena, extent_t *extent) {
Show All 32 Lines
void		void
large_prof_tctx_set(tsdn_t tsdn, extent_t extent, prof_tctx_t *tctx) {		large_prof_tctx_set(tsdn_t tsdn, extent_t extent, prof_tctx_t *tctx) {
extent_prof_tctx_set(extent, tctx);		extent_prof_tctx_set(extent, tctx);
}		}

void		void
large_prof_tctx_reset(tsdn_t tsdn, extent_t extent) {		large_prof_tctx_reset(tsdn_t tsdn, extent_t extent) {
large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);		large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);
		}

		nstime_t
		large_prof_alloc_time_get(const extent_t *extent) {
		return extent_prof_alloc_time_get(extent);
		}

		void
		large_prof_alloc_time_set(extent_t *extent, nstime_t t) {
		extent_prof_alloc_time_set(extent, t);
}		}