Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 1,835 Lines • ▼ Show 20 Lines | keg_layout(uma_keg_t keg) | ||||
*/ | */ | ||||
eff = UMA_FRAC_FIXPT(ipers * rsize, slabsize); | eff = UMA_FRAC_FIXPT(ipers * rsize, slabsize); | ||||
ipers_offpage = slab_ipers_hdr(keg->uk_size, rsize, slabsize, false); | ipers_offpage = slab_ipers_hdr(keg->uk_size, rsize, slabsize, false); | ||||
eff_offpage = UMA_FRAC_FIXPT(ipers_offpage * rsize, | eff_offpage = UMA_FRAC_FIXPT(ipers_offpage * rsize, | ||||
slabsize + slab_sizeof(SLAB_MAX_SETSIZE)); | slabsize + slab_sizeof(SLAB_MAX_SETSIZE)); | ||||
if (ipers == 0 || (eff < UMA_MIN_EFF && eff < eff_offpage)) { | if (ipers == 0 || (eff < UMA_MIN_EFF && eff < eff_offpage)) { | ||||
CTR5(KTR_UMA, "UMA decided we need offpage slab headers for " | CTR5(KTR_UMA, "UMA decided we need offpage slab headers for " | ||||
"keg: %s(%p), minimum efficiency allowed = %u%%, " | "keg: %s(%p), minimum efficiency allowed = %u%%, " | ||||
"old efficiency = %u%%, offpage efficiency = %u%%\n", | "old efficiency = %u%%, offpage efficiency = %u%%", | ||||
keg->uk_name, keg, UMA_FIXPT_PCT(UMA_MIN_EFF), | keg->uk_name, keg, UMA_FIXPT_PCT(UMA_MIN_EFF), | ||||
UMA_FIXPT_PCT(eff), UMA_FIXPT_PCT(eff_offpage)); | UMA_FIXPT_PCT(eff), UMA_FIXPT_PCT(eff_offpage)); | ||||
format = UMA_ZFLAG_OFFPAGE; | format = UMA_ZFLAG_OFFPAGE; | ||||
ipers = ipers_offpage; | ipers = ipers_offpage; | ||||
} | } | ||||
out: | out: | ||||
/* | /* | ||||
* How do we find the slab header if it is offpage or if not all item | * How do we find the slab header if it is offpage or if not all item | ||||
* start addresses are in the same page? We could solve the latter | * start addresses are in the same page? We could solve the latter | ||||
* case with vaddr alignment, but we don't. | * case with vaddr alignment, but we don't. | ||||
*/ | */ | ||||
if ((format & UMA_ZFLAG_OFFPAGE) != 0 || | if ((format & UMA_ZFLAG_OFFPAGE) != 0 || | ||||
(ipers - 1) * rsize >= PAGE_SIZE) { | (ipers - 1) * rsize >= PAGE_SIZE) { | ||||
if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0) | if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0) | ||||
format |= UMA_ZFLAG_HASH; | format |= UMA_ZFLAG_HASH; | ||||
else | else | ||||
format |= UMA_ZFLAG_VTOSLAB; | format |= UMA_ZFLAG_VTOSLAB; | ||||
} | } | ||||
keg->uk_ipers = ipers; | keg->uk_ipers = ipers; | ||||
keg->uk_rsize = rsize; | keg->uk_rsize = rsize; | ||||
keg->uk_flags |= format; | keg->uk_flags |= format; | ||||
keg->uk_ppera = pages; | keg->uk_ppera = pages; | ||||
CTR6(KTR_UMA, "%s: keg=%s, flags=%#x, rsize=%u, ipers=%u, ppera=%u\n", | CTR6(KTR_UMA, "%s: keg=%s, flags=%#x, rsize=%u, ipers=%u, ppera=%u", | ||||
__func__, keg->uk_name, keg->uk_flags, rsize, ipers, pages); | __func__, keg->uk_name, keg->uk_flags, rsize, ipers, pages); | ||||
KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_MAX_SETSIZE, | KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_MAX_SETSIZE, | ||||
("%s: keg=%s, flags=0x%b, rsize=%u, ipers=%u, ppera=%u", __func__, | ("%s: keg=%s, flags=0x%b, rsize=%u, ipers=%u, ppera=%u", __func__, | ||||
keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize, ipers, | keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize, ipers, | ||||
pages)); | pages)); | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 117 Lines • ▼ Show 20 Lines | if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) { | ||||
KASSERT(keg->uk_pgoff + shsize <= PAGE_SIZE * keg->uk_ppera, | KASSERT(keg->uk_pgoff + shsize <= PAGE_SIZE * keg->uk_ppera, | ||||
("zone %s ipers %d rsize %d size %d slab won't fit", | ("zone %s ipers %d rsize %d size %d slab won't fit", | ||||
zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); | zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); | ||||
} | } | ||||
if (keg->uk_flags & UMA_ZFLAG_HASH) | if (keg->uk_flags & UMA_ZFLAG_HASH) | ||||
hash_alloc(&keg->uk_hash, 0); | hash_alloc(&keg->uk_hash, 0); | ||||
CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)\n", keg, zone->uz_name, zone); | CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone); | ||||
LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); | LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); | ||||
rw_wlock(&uma_rwlock); | rw_wlock(&uma_rwlock); | ||||
LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); | LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); | ||||
rw_wunlock(&uma_rwlock); | rw_wunlock(&uma_rwlock); | ||||
return (0); | return (0); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 919 Lines • ▼ Show 20 Lines | uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
void *item; | void *item; | ||||
int domain, size, uz_flags; | int domain, size, uz_flags; | ||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
/* This is the fast path allocation */ | /* This is the fast path allocation */ | ||||
CTR4(KTR_UMA, "uma_zalloc_arg thread %x zone %s(%p) flags %d", | CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, | ||||
curthread, zone->uz_name, zone, flags); | zone, flags); | ||||
#ifdef WITNESS | #ifdef WITNESS | ||||
if (flags & M_WAITOK) { | if (flags & M_WAITOK) { | ||||
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, | WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, | ||||
"uma_zalloc_arg: zone \"%s\"", zone->uz_name); | "uma_zalloc_arg: zone \"%s\"", zone->uz_name); | ||||
} | } | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 198 Lines • ▼ Show 20 Lines | |||||
void * | void * | ||||
uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) | uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) | ||||
{ | { | ||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
/* This is the fast path allocation */ | /* This is the fast path allocation */ | ||||
CTR5(KTR_UMA, | CTR4(KTR_UMA, "uma_zalloc_domain zone %s(%p) domain %d flags %d", | ||||
"uma_zalloc_domain thread %x zone %s(%p) domain %d flags %d", | zone->uz_name, zone, domain, flags); | ||||
curthread, zone->uz_name, zone, domain, flags); | |||||
if (flags & M_WAITOK) { | if (flags & M_WAITOK) { | ||||
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, | WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, | ||||
"uma_zalloc_domain: zone \"%s\"", zone->uz_name); | "uma_zalloc_domain: zone \"%s\"", zone->uz_name); | ||||
} | } | ||||
KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | ||||
("uma_zalloc_domain: called with spinlock or critical section held")); | ("uma_zalloc_domain: called with spinlock or critical section held")); | ||||
▲ Show 20 Lines • Show All 353 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static uma_bucket_t | static uma_bucket_t | ||||
zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) | zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) | ||||
{ | { | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
int maxbucket, cnt; | int maxbucket, cnt; | ||||
CTR1(KTR_UMA, "zone_alloc:_bucket domain %d)", domain); | CTR3(KTR_UMA, "zone_alloc_bucket zone %s(%p) domain %d", zone->uz_name, | ||||
zone, domain); | |||||
/* Avoid allocs targeting empty domains. */ | /* Avoid allocs targeting empty domains. */ | ||||
if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) | if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) | ||||
domain = UMA_ANYDOMAIN; | domain = UMA_ANYDOMAIN; | ||||
if (zone->uz_max_items > 0) | if (zone->uz_max_items > 0) | ||||
maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size, | maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size, | ||||
M_NOWAIT); | M_NOWAIT); | ||||
▲ Show 20 Lines • Show All 118 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
uma_cache_bucket_t bucket; | uma_cache_bucket_t bucket; | ||||
int domain, itemdomain, uz_flags; | int domain, itemdomain, uz_flags; | ||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, | CTR2(KTR_UMA, "uma_zfree_arg zone %s(%p)", zone->uz_name, zone); | ||||
zone->uz_name); | |||||
KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | ||||
("uma_zfree_arg: called with spinlock or critical section held")); | ("uma_zfree_arg: called with spinlock or critical section held")); | ||||
/* uma_zfree(..., NULL) does nothing, to match free(9). */ | /* uma_zfree(..., NULL) does nothing, to match free(9). */ | ||||
if (item == NULL) | if (item == NULL) | ||||
return; | return; | ||||
#ifdef DEBUG_MEMGUARD | #ifdef DEBUG_MEMGUARD | ||||
▲ Show 20 Lines • Show All 284 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
uma_zfree_domain(uma_zone_t zone, void *item, void *udata) | uma_zfree_domain(uma_zone_t zone, void *item, void *udata) | ||||
{ | { | ||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
CTR2(KTR_UMA, "uma_zfree_domain thread %x zone %s", curthread, | CTR2(KTR_UMA, "uma_zfree_domain zone %s(%p)", zone->uz_name, zone); | ||||
zone->uz_name); | |||||
KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | ||||
("uma_zfree_domain: called with spinlock or critical section held")); | ("uma_zfree_domain: called with spinlock or critical section held")); | ||||
/* uma_zfree(..., NULL) does nothing, to match free(9). */ | /* uma_zfree(..., NULL) does nothing, to match free(9). */ | ||||
if (item == NULL) | if (item == NULL) | ||||
return; | return; | ||||
zone_free_item(zone, item, udata, SKIP_NONE); | zone_free_item(zone, item, udata, SKIP_NONE); | ||||
▲ Show 20 Lines • Show All 1,065 Lines • Show Last 20 Lines |