Changeset View
Changeset View
Standalone View
Standalone View
head/contrib/jemalloc/include/jemalloc/internal/extent_inlines.h
#ifndef JEMALLOC_INTERNAL_EXTENT_INLINES_H | #ifndef JEMALLOC_INTERNAL_EXTENT_INLINES_H | ||||
#define JEMALLOC_INTERNAL_EXTENT_INLINES_H | #define JEMALLOC_INTERNAL_EXTENT_INLINES_H | ||||
#include "jemalloc/internal/mutex.h" | #include "jemalloc/internal/mutex.h" | ||||
#include "jemalloc/internal/mutex_pool.h" | #include "jemalloc/internal/mutex_pool.h" | ||||
#include "jemalloc/internal/pages.h" | #include "jemalloc/internal/pages.h" | ||||
#include "jemalloc/internal/prng.h" | #include "jemalloc/internal/prng.h" | ||||
#include "jemalloc/internal/ql.h" | #include "jemalloc/internal/ql.h" | ||||
#include "jemalloc/internal/sc.h" | |||||
#include "jemalloc/internal/sz.h" | #include "jemalloc/internal/sz.h" | ||||
static inline void | static inline void | ||||
extent_lock(tsdn_t *tsdn, extent_t *extent) { | extent_lock(tsdn_t *tsdn, extent_t *extent) { | ||||
assert(extent != NULL); | assert(extent != NULL); | ||||
mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent); | mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent); | ||||
} | } | ||||
Show All 12 Lines | |||||
static inline void | static inline void | ||||
extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { | extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { | ||||
assert(extent1 != NULL && extent2 != NULL); | assert(extent1 != NULL && extent2 != NULL); | ||||
mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, | mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1, | ||||
(uintptr_t)extent2); | (uintptr_t)extent2); | ||||
} | } | ||||
static inline arena_t * | static inline unsigned | ||||
extent_arena_get(const extent_t *extent) { | extent_arena_ind_get(const extent_t *extent) { | ||||
unsigned arena_ind = (unsigned)((extent->e_bits & | unsigned arena_ind = (unsigned)((extent->e_bits & | ||||
EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); | EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); | ||||
/* | |||||
* The following check is omitted because we should never actually read | |||||
* a NULL arena pointer. | |||||
*/ | |||||
if (false && arena_ind >= MALLOCX_ARENA_LIMIT) { | |||||
return NULL; | |||||
} | |||||
assert(arena_ind < MALLOCX_ARENA_LIMIT); | assert(arena_ind < MALLOCX_ARENA_LIMIT); | ||||
return arena_ind; | |||||
} | |||||
static inline arena_t * | |||||
extent_arena_get(const extent_t *extent) { | |||||
unsigned arena_ind = extent_arena_ind_get(extent); | |||||
return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); | return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); | ||||
} | } | ||||
static inline szind_t | static inline szind_t | ||||
extent_szind_get_maybe_invalid(const extent_t *extent) { | extent_szind_get_maybe_invalid(const extent_t *extent) { | ||||
szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> | szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> | ||||
EXTENT_BITS_SZIND_SHIFT); | EXTENT_BITS_SZIND_SHIFT); | ||||
assert(szind <= NSIZES); | assert(szind <= SC_NSIZES); | ||||
return szind; | return szind; | ||||
} | } | ||||
static inline szind_t | static inline szind_t | ||||
extent_szind_get(const extent_t *extent) { | extent_szind_get(const extent_t *extent) { | ||||
szind_t szind = extent_szind_get_maybe_invalid(extent); | szind_t szind = extent_szind_get_maybe_invalid(extent); | ||||
assert(szind < NSIZES); /* Never call when "invalid". */ | assert(szind < SC_NSIZES); /* Never call when "invalid". */ | ||||
return szind; | return szind; | ||||
} | } | ||||
static inline size_t | static inline size_t | ||||
extent_usize_get(const extent_t *extent) { | extent_usize_get(const extent_t *extent) { | ||||
return sz_index2size(extent_szind_get(extent)); | return sz_index2size(extent_szind_get(extent)); | ||||
} | } | ||||
static inline unsigned | |||||
extent_binshard_get(const extent_t *extent) { | |||||
unsigned binshard = (unsigned)((extent->e_bits & | |||||
EXTENT_BITS_BINSHARD_MASK) >> EXTENT_BITS_BINSHARD_SHIFT); | |||||
assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); | |||||
return binshard; | |||||
} | |||||
static inline size_t | static inline size_t | ||||
extent_sn_get(const extent_t *extent) { | extent_sn_get(const extent_t *extent) { | ||||
return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> | return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> | ||||
EXTENT_BITS_SN_SHIFT); | EXTENT_BITS_SN_SHIFT); | ||||
} | } | ||||
static inline extent_state_t | static inline extent_state_t | ||||
extent_state_get(const extent_t *extent) { | extent_state_get(const extent_t *extent) { | ||||
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static inline prof_tctx_t * | static inline prof_tctx_t * | ||||
extent_prof_tctx_get(const extent_t *extent) { | extent_prof_tctx_get(const extent_t *extent) { | ||||
return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx, | return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx, | ||||
ATOMIC_ACQUIRE); | ATOMIC_ACQUIRE); | ||||
} | } | ||||
static inline nstime_t | |||||
extent_prof_alloc_time_get(const extent_t *extent) { | |||||
return extent->e_alloc_time; | |||||
} | |||||
static inline void | static inline void | ||||
extent_arena_set(extent_t *extent, arena_t *arena) { | extent_arena_set(extent_t *extent, arena_t *arena) { | ||||
unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << | unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << | ||||
MALLOCX_ARENA_BITS) - 1); | MALLOCX_ARENA_BITS) - 1); | ||||
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ARENA_MASK) | | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ARENA_MASK) | | ||||
((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT); | ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT); | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_binshard_set(extent_t *extent, unsigned binshard) { | |||||
/* The assertion assumes szind is set already. */ | |||||
assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); | |||||
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_BINSHARD_MASK) | | |||||
((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT); | |||||
} | |||||
static inline void | |||||
extent_addr_set(extent_t *extent, void *addr) { | extent_addr_set(extent_t *extent, void *addr) { | ||||
extent->e_addr = addr; | extent->e_addr = addr; | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_addr_randomize(UNUSED tsdn_t *tsdn, extent_t *extent, size_t alignment) { | extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) { | ||||
assert(extent_base_get(extent) == extent_addr_get(extent)); | assert(extent_base_get(extent) == extent_addr_get(extent)); | ||||
if (alignment < PAGE) { | if (alignment < PAGE) { | ||||
unsigned lg_range = LG_PAGE - | unsigned lg_range = LG_PAGE - | ||||
lg_floor(CACHELINE_CEILING(alignment)); | lg_floor(CACHELINE_CEILING(alignment)); | ||||
size_t r; | size_t r; | ||||
if (!tsdn_null(tsdn)) { | if (!tsdn_null(tsdn)) { | ||||
tsd_t *tsd = tsdn_tsd(tsdn); | tsd_t *tsd = tsdn_tsd(tsdn); | ||||
Show All 27 Lines | |||||
static inline void | static inline void | ||||
extent_bsize_set(extent_t *extent, size_t bsize) { | extent_bsize_set(extent_t *extent, size_t bsize) { | ||||
extent->e_bsize = bsize; | extent->e_bsize = bsize; | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_szind_set(extent_t *extent, szind_t szind) { | extent_szind_set(extent_t *extent, szind_t szind) { | ||||
assert(szind <= NSIZES); /* NSIZES means "invalid". */ | assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */ | ||||
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | | ||||
((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); | ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_nfree_set(extent_t *extent, unsigned nfree) { | extent_nfree_set(extent_t *extent, unsigned nfree) { | ||||
assert(extent_slab_get(extent)); | assert(extent_slab_get(extent)); | ||||
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) | | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) | | ||||
((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); | ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_nfree_binshard_set(extent_t *extent, unsigned nfree, unsigned binshard) { | |||||
/* The assertion assumes szind is set already. */ | |||||
assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); | |||||
extent->e_bits = (extent->e_bits & | |||||
(~EXTENT_BITS_NFREE_MASK & ~EXTENT_BITS_BINSHARD_MASK)) | | |||||
((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT) | | |||||
((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); | |||||
} | |||||
static inline void | |||||
extent_nfree_inc(extent_t *extent) { | extent_nfree_inc(extent_t *extent) { | ||||
assert(extent_slab_get(extent)); | assert(extent_slab_get(extent)); | ||||
extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); | extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_nfree_dec(extent_t *extent) { | extent_nfree_dec(extent_t *extent) { | ||||
assert(extent_slab_get(extent)); | assert(extent_slab_get(extent)); | ||||
extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); | extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_nfree_sub(extent_t *extent, uint64_t n) { | |||||
assert(extent_slab_get(extent)); | |||||
extent->e_bits -= (n << EXTENT_BITS_NFREE_SHIFT); | |||||
} | |||||
static inline void | |||||
extent_sn_set(extent_t *extent, size_t sn) { | extent_sn_set(extent_t *extent, size_t sn) { | ||||
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | | ||||
((uint64_t)sn << EXTENT_BITS_SN_SHIFT); | ((uint64_t)sn << EXTENT_BITS_SN_SHIFT); | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_state_set(extent_t *extent, extent_state_t state) { | extent_state_set(extent_t *extent, extent_state_t state) { | ||||
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK) | | extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK) | | ||||
Show All 25 Lines | |||||
} | } | ||||
static inline void | static inline void | ||||
extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { | extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { | ||||
atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE); | atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE); | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_prof_alloc_time_set(extent_t *extent, nstime_t t) { | |||||
nstime_copy(&extent->e_alloc_time, &t); | |||||
} | |||||
static inline bool | |||||
extent_is_head_get(extent_t *extent) { | |||||
if (maps_coalesce) { | |||||
not_reached(); | |||||
} | |||||
return (bool)((extent->e_bits & EXTENT_BITS_IS_HEAD_MASK) >> | |||||
EXTENT_BITS_IS_HEAD_SHIFT); | |||||
} | |||||
static inline void | |||||
extent_is_head_set(extent_t *extent, bool is_head) { | |||||
if (maps_coalesce) { | |||||
not_reached(); | |||||
} | |||||
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_IS_HEAD_MASK) | | |||||
((uint64_t)is_head << EXTENT_BITS_IS_HEAD_SHIFT); | |||||
} | |||||
static inline void | |||||
extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, | extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, | ||||
bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, | bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, | ||||
bool committed, bool dumpable) { | bool committed, bool dumpable, extent_head_state_t is_head) { | ||||
assert(addr == PAGE_ADDR2BASE(addr) || !slab); | assert(addr == PAGE_ADDR2BASE(addr) || !slab); | ||||
extent_arena_set(extent, arena); | extent_arena_set(extent, arena); | ||||
extent_addr_set(extent, addr); | extent_addr_set(extent, addr); | ||||
extent_size_set(extent, size); | extent_size_set(extent, size); | ||||
extent_slab_set(extent, slab); | extent_slab_set(extent, slab); | ||||
extent_szind_set(extent, szind); | extent_szind_set(extent, szind); | ||||
extent_sn_set(extent, sn); | extent_sn_set(extent, sn); | ||||
extent_state_set(extent, state); | extent_state_set(extent, state); | ||||
extent_zeroed_set(extent, zeroed); | extent_zeroed_set(extent, zeroed); | ||||
extent_committed_set(extent, committed); | extent_committed_set(extent, committed); | ||||
extent_dumpable_set(extent, dumpable); | extent_dumpable_set(extent, dumpable); | ||||
ql_elm_new(extent, ql_link); | ql_elm_new(extent, ql_link); | ||||
if (!maps_coalesce) { | |||||
extent_is_head_set(extent, (is_head == EXTENT_IS_HEAD) ? true : | |||||
false); | |||||
} | |||||
if (config_prof) { | if (config_prof) { | ||||
extent_prof_tctx_set(extent, NULL); | extent_prof_tctx_set(extent, NULL); | ||||
} | } | ||||
} | } | ||||
static inline void | static inline void | ||||
extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { | extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { | ||||
extent_arena_set(extent, NULL); | extent_arena_set(extent, NULL); | ||||
extent_addr_set(extent, addr); | extent_addr_set(extent, addr); | ||||
extent_bsize_set(extent, bsize); | extent_bsize_set(extent, bsize); | ||||
extent_slab_set(extent, false); | extent_slab_set(extent, false); | ||||
extent_szind_set(extent, NSIZES); | extent_szind_set(extent, SC_NSIZES); | ||||
extent_sn_set(extent, sn); | extent_sn_set(extent, sn); | ||||
extent_state_set(extent, extent_state_active); | extent_state_set(extent, extent_state_active); | ||||
extent_zeroed_set(extent, true); | extent_zeroed_set(extent, true); | ||||
extent_committed_set(extent, true); | extent_committed_set(extent, true); | ||||
extent_dumpable_set(extent, true); | extent_dumpable_set(extent, true); | ||||
} | } | ||||
static inline void | static inline void | ||||
▲ Show 20 Lines • Show All 95 Lines • Show Last 20 Lines |