Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 1,863 Lines • ▼ Show 20 Lines | else { | ||||
zone->uz_allocs = EARLY_COUNTER; | zone->uz_allocs = EARLY_COUNTER; | ||||
zone->uz_frees = EARLY_COUNTER; | zone->uz_frees = EARLY_COUNTER; | ||||
zone->uz_fails = EARLY_COUNTER; | zone->uz_fails = EARLY_COUNTER; | ||||
} | } | ||||
for (i = 0; i < vm_ndomains; i++) | for (i = 0; i < vm_ndomains; i++) | ||||
TAILQ_INIT(&zone->uz_domain[i].uzd_buckets); | TAILQ_INIT(&zone->uz_domain[i].uzd_buckets); | ||||
#ifdef INVARIANTS | |||||
if (arg->uminit == trash_init && arg->fini == trash_fini) | |||||
zone->uz_flags |= UMA_ZFLAG_TRASH; | |||||
#endif | |||||
/* | /* | ||||
* This is a pure cache zone, no kegs. | * This is a pure cache zone, no kegs. | ||||
*/ | */ | ||||
if (arg->import) { | if (arg->import) { | ||||
if (arg->flags & UMA_ZONE_VM) | if (arg->flags & UMA_ZONE_VM) | ||||
arg->flags |= UMA_ZFLAG_CACHEONLY; | arg->flags |= UMA_ZFLAG_CACHEONLY; | ||||
zone->uz_flags = arg->flags; | zone->uz_flags = arg->flags; | ||||
zone->uz_size = arg->size; | zone->uz_size = arg->size; | ||||
▲ Show 20 Lines • Show All 417 Lines • ▼ Show 20 Lines | #endif | ||||
args.name = name; | args.name = name; | ||||
args.size = size; | args.size = size; | ||||
args.ctor = ctor; | args.ctor = ctor; | ||||
args.dtor = dtor; | args.dtor = dtor; | ||||
args.uminit = uminit; | args.uminit = uminit; | ||||
args.fini = fini; | args.fini = fini; | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
/* | /* | ||||
* If a zone is being created with an empty constructor and | * Inject procedures which check for memory use after free if we are | ||||
* destructor, pass UMA constructor/destructor which checks for | * allowed to scramble the memory while it is not allocated. This | ||||
* memory use after free. | * requires that: UMA is actually able to access the memory, no init | ||||
* or fini procedures, no dependency on the initial value of the | |||||
* memory, and no (legitimate) use of the memory after free. Note, | |||||
* the ctor and dtor do not need to be empty. | |||||
* | |||||
* XXX UMA_ZONE_OFFPAGE. | |||||
*/ | */ | ||||
if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOFREE))) && | if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOFREE))) && | ||||
ctor == NULL && dtor == NULL && uminit == NULL && fini == NULL) { | uminit == NULL && fini == NULL) { | ||||
args.ctor = trash_ctor; | |||||
args.dtor = trash_dtor; | |||||
args.uminit = trash_init; | args.uminit = trash_init; | ||||
args.fini = trash_fini; | args.fini = trash_fini; | ||||
} | } | ||||
#endif | #endif | ||||
args.align = align; | args.align = align; | ||||
args.flags = flags; | args.flags = flags; | ||||
args.keg = NULL; | args.keg = NULL; | ||||
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines | #endif | ||||
return (item); | return (item); | ||||
} | } | ||||
static void * | static void * | ||||
item_ctor(uma_zone_t zone, void *udata, int flags, void *item) | item_ctor(uma_zone_t zone, void *udata, int flags, void *item) | ||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
int skipdbg; | bool skipdbg; | ||||
skipdbg = uma_dbg_zskip(zone, item); | skipdbg = uma_dbg_zskip(zone, item); | ||||
if (zone->uz_ctor != NULL && | if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && | ||||
(!skipdbg || zone->uz_ctor != trash_ctor || | zone->uz_ctor != trash_ctor) | ||||
zone->uz_dtor != trash_dtor) && | trash_ctor(item, zone->uz_size, udata, flags); | ||||
#else | |||||
if (__predict_false(zone->uz_ctor != NULL) && | |||||
#endif | #endif | ||||
if (__predict_false(zone->uz_ctor != NULL) && | |||||
zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { | zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { | ||||
counter_u64_add(zone->uz_fails, 1); | counter_u64_add(zone->uz_fails, 1); | ||||
zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); | zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
if (!skipdbg) | if (!skipdbg) | ||||
uma_dbg_alloc(zone, NULL, item); | uma_dbg_alloc(zone, NULL, item); | ||||
#endif | #endif | ||||
if (flags & M_ZERO) | if (flags & M_ZERO) | ||||
uma_zero_item(item, zone); | uma_zero_item(item, zone); | ||||
return (item); | return (item); | ||||
} | } | ||||
static inline void | |||||
item_dtor(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | |||||
{ | |||||
#ifdef INVARIANTS | |||||
bool skipdbg; | |||||
skipdbg = uma_dbg_zskip(zone, item); | |||||
if (skip == SKIP_NONE && !skipdbg) { | |||||
if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0) | |||||
uma_dbg_free(zone, udata, item); | |||||
else | |||||
uma_dbg_free(zone, NULL, item); | |||||
} | |||||
#endif | |||||
if (skip < SKIP_DTOR) { | |||||
if (zone->uz_dtor != NULL) | |||||
zone->uz_dtor(item, zone->uz_size, udata); | |||||
#ifdef INVARIANTS | |||||
if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && | |||||
zone->uz_dtor != trash_dtor) | |||||
trash_dtor(item, zone->uz_size, udata); | |||||
#endif | |||||
} | |||||
} | |||||
/* See uma.h */ | /* See uma.h */ | ||||
void * | void * | ||||
uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) | uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) | ||||
{ | { | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
void *item; | void *item; | ||||
int cpu, domain; | int cpu, domain; | ||||
Show All 21 Lines | if (memguard_cmp_zone(zone)) { | ||||
item = memguard_alloc(zone->uz_size, flags); | item = memguard_alloc(zone->uz_size, flags); | ||||
if (item != NULL) { | if (item != NULL) { | ||||
if (zone->uz_init != NULL && | if (zone->uz_init != NULL && | ||||
zone->uz_init(item, zone->uz_size, flags) != 0) | zone->uz_init(item, zone->uz_size, flags) != 0) | ||||
return (NULL); | return (NULL); | ||||
if (zone->uz_ctor != NULL && | if (zone->uz_ctor != NULL && | ||||
zone->uz_ctor(item, zone->uz_size, udata, | zone->uz_ctor(item, zone->uz_size, udata, | ||||
flags) != 0) { | flags) != 0) { | ||||
counter_u64_add(zone->uz_fails, 1); | |||||
zone->uz_fini(item, zone->uz_size); | zone->uz_fini(item, zone->uz_size); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
return (item); | return (item); | ||||
} | } | ||||
/* This is unfortunate but should not be fatal. */ | /* This is unfortunate but should not be fatal. */ | ||||
} | } | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 592 Lines • ▼ Show 20 Lines | uma_zfree_arg(uma_zone_t zone, void *item, void *udata) | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
uma_zone_domain_t zdom; | uma_zone_domain_t zdom; | ||||
int cpu, domain; | int cpu, domain; | ||||
#ifdef UMA_XDOMAIN | #ifdef UMA_XDOMAIN | ||||
int itemdomain; | int itemdomain; | ||||
#endif | #endif | ||||
bool lockfail; | bool lockfail; | ||||
#ifdef INVARIANTS | |||||
bool skipdbg; | |||||
#endif | |||||
/* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ | ||||
random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); | ||||
CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, | CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, | ||||
zone->uz_name); | zone->uz_name); | ||||
KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), | ||||
("uma_zfree_arg: called with spinlock or critical section held")); | ("uma_zfree_arg: called with spinlock or critical section held")); | ||||
/* uma_zfree(..., NULL) does nothing, to match free(9). */ | /* uma_zfree(..., NULL) does nothing, to match free(9). */ | ||||
if (item == NULL) | if (item == NULL) | ||||
return; | return; | ||||
#ifdef DEBUG_MEMGUARD | #ifdef DEBUG_MEMGUARD | ||||
if (is_memguard_addr(item)) { | if (is_memguard_addr(item)) { | ||||
if (zone->uz_dtor != NULL) | if (zone->uz_dtor != NULL) | ||||
zone->uz_dtor(item, zone->uz_size, udata); | zone->uz_dtor(item, zone->uz_size, udata); | ||||
if (zone->uz_fini != NULL) | if (zone->uz_fini != NULL) | ||||
zone->uz_fini(item, zone->uz_size); | zone->uz_fini(item, zone->uz_size); | ||||
memguard_free(item); | memguard_free(item); | ||||
return; | return; | ||||
} | } | ||||
#endif | #endif | ||||
#ifdef INVARIANTS | item_dtor(zone, item, udata, SKIP_NONE); | ||||
skipdbg = uma_dbg_zskip(zone, item); | |||||
if (skipdbg == false) { | |||||
if (zone->uz_flags & UMA_ZONE_MALLOC) | |||||
uma_dbg_free(zone, udata, item); | |||||
else | |||||
uma_dbg_free(zone, NULL, item); | |||||
} | |||||
if (zone->uz_dtor != NULL && (!skipdbg || | |||||
zone->uz_dtor != trash_dtor || zone->uz_ctor != trash_ctor)) | |||||
#else | |||||
if (zone->uz_dtor != NULL) | |||||
#endif | |||||
zone->uz_dtor(item, zone->uz_size, udata); | |||||
/* | /* | ||||
* The race here is acceptable. If we miss it we'll just have to wait | * The race here is acceptable. If we miss it we'll just have to wait | ||||
* a little longer for the limits to be reset. | * a little longer for the limits to be reset. | ||||
*/ | */ | ||||
if (zone->uz_sleepers > 0) | if (zone->uz_sleepers > 0) | ||||
goto zfree_item; | goto zfree_item; | ||||
▲ Show 20 Lines • Show All 272 Lines • ▼ Show 20 Lines | |||||
* zone The zone to free to | * zone The zone to free to | ||||
* item The item we're freeing | * item The item we're freeing | ||||
* udata User supplied data for the dtor | * udata User supplied data for the dtor | ||||
* skip Skip dtors and finis | * skip Skip dtors and finis | ||||
*/ | */ | ||||
static void | static void | ||||
zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) | ||||
{ | { | ||||
#ifdef INVARIANTS | |||||
bool skipdbg; | |||||
skipdbg = uma_dbg_zskip(zone, item); | item_dtor(zone, item, udata, skip); | ||||
if (skip == SKIP_NONE && !skipdbg) { | |||||
if (zone->uz_flags & UMA_ZONE_MALLOC) | |||||
uma_dbg_free(zone, udata, item); | |||||
else | |||||
uma_dbg_free(zone, NULL, item); | |||||
} | |||||
if (skip < SKIP_DTOR && zone->uz_dtor != NULL && | |||||
(!skipdbg || zone->uz_dtor != trash_dtor || | |||||
zone->uz_ctor != trash_ctor)) | |||||
#else | |||||
if (skip < SKIP_DTOR && zone->uz_dtor != NULL) | |||||
#endif | |||||
zone->uz_dtor(item, zone->uz_size, udata); | |||||
if (skip < SKIP_FINI && zone->uz_fini) | if (skip < SKIP_FINI && zone->uz_fini) | ||||
zone->uz_fini(item, zone->uz_size); | zone->uz_fini(item, zone->uz_size); | ||||
zone->uz_release(zone->uz_arg, &item, 1); | zone->uz_release(zone->uz_arg, &item, 1); | ||||
if (skip & SKIP_CNT) | if (skip & SKIP_CNT) | ||||
return; | return; | ||||
▲ Show 20 Lines • Show All 1,031 Lines • Show Last 20 Lines |