Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/uma_core.c
Show First 20 Lines • Show All 2,222 Lines • ▼ Show 20 Lines | |||||
#ifdef NUMA | #ifdef NUMA | ||||
if ((keg->uk_flags & | if ((keg->uk_flags & | ||||
(UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0) | (UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0) | ||||
keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; | keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; | ||||
else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) | else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) | ||||
keg->uk_flags |= UMA_ZONE_ROUNDROBIN; | keg->uk_flags |= UMA_ZONE_ROUNDROBIN; | ||||
#endif | #endif | ||||
KASSERT(booted >= BOOT_KVA || (keg->uk_flags & UMA_ZONE_PCPU) == 0, | |||||
("%s: early pcpu keg %s", __func__, keg->uk_name)); | |||||
/* | /* | ||||
* If we haven't booted yet we need allocations to go through the | * If we haven't booted yet we need allocations to go through the | ||||
* startup cache until the vm is ready. | * startup cache until the vm is ready. | ||||
*/ | */ | ||||
#ifdef UMA_MD_SMALL_ALLOC | #ifdef UMA_MD_SMALL_ALLOC | ||||
if (keg->uk_ppera == 1) | if (keg->uk_ppera == 1) | ||||
keg->uk_allocf = uma_small_alloc; | keg->uk_allocf = uma_small_alloc; | ||||
else | else | ||||
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | zone_kva_available(uma_zone_t zone, void *unused) | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) | if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) | ||||
return; | return; | ||||
KEG_GET(zone, keg); | KEG_GET(zone, keg); | ||||
if (keg->uk_allocf == startup_alloc) { | if (keg->uk_allocf == startup_alloc) { | ||||
/* Switch to the real allocator. */ | /* Switch to the real allocator. */ | ||||
if (keg->uk_flags & UMA_ZONE_PCPU) | if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1) | ||||
keg->uk_allocf = pcpu_page_alloc; | |||||
else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && | |||||
keg->uk_ppera > 1) | |||||
keg->uk_allocf = contig_alloc; | keg->uk_allocf = contig_alloc; | ||||
else | else | ||||
keg->uk_allocf = page_alloc; | keg->uk_allocf = page_alloc; | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
zone_alloc_counters(uma_zone_t zone, void *unused) | zone_alloc_counters(uma_zone_t zone, void *unused) | ||||
▲ Show 20 Lines • Show All 542 Lines • ▼ Show 20 Lines | uma_startup1(vm_offset_t virtual_avail) | ||||
slabzones[1] = uma_zcreate("UMA Slabs 1", SLABZONE1_SIZE, | slabzones[1] = uma_zcreate("UMA Slabs 1", SLABZONE1_SIZE, | ||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | ||||
hashzone = uma_zcreate("UMA Hash", | hashzone = uma_zcreate("UMA Hash", | ||||
sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, | sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, | ||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | ||||
bucket_init(); | bucket_init(); | ||||
smr_init(); | |||||
} | } | ||||
#ifndef UMA_MD_SMALL_ALLOC | #ifndef UMA_MD_SMALL_ALLOC | ||||
extern void vm_radix_reserve_kva(void); | extern void vm_radix_reserve_kva(void); | ||||
#endif | #endif | ||||
/* | /* | ||||
* Advertise the availability of normal kva allocations and switch to | * Advertise the availability of normal kva allocations and switch to | ||||
▲ Show 20 Lines • Show All 426 Lines • ▼ Show 20 Lines | |||||
uma_zalloc_smr(uma_zone_t zone, int flags) | uma_zalloc_smr(uma_zone_t zone, int flags) | ||||
{ | { | ||||
uma_cache_bucket_t bucket; | uma_cache_bucket_t bucket; | ||||
uma_cache_t cache; | uma_cache_t cache; | ||||
#ifdef UMA_ZALLOC_DEBUG | #ifdef UMA_ZALLOC_DEBUG | ||||
void *item; | void *item; | ||||
KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, | KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0 || !smp_started, | ||||
("uma_zalloc_arg: called with non-SMR zone.\n")); | ("uma_zalloc_arg: called with non-SMR zone.\n")); | ||||
if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) | if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) | ||||
return (item); | return (item); | ||||
#endif | #endif | ||||
critical_enter(); | critical_enter(); | ||||
cache = &zone->uz_cpu[curcpu]; | cache = &zone->uz_cpu[curcpu]; | ||||
bucket = &cache->uc_allocbucket; | bucket = &cache->uc_allocbucket; | ||||
▲ Show 20 Lines • Show All 1,326 Lines • ▼ Show 20 Lines | uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) | ||||
keg->uk_allocf = allocf; | keg->uk_allocf = allocf; | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void | void | ||||
uma_zone_set_smr(uma_zone_t zone, smr_t smr) | uma_zone_set_smr(uma_zone_t zone, smr_t smr) | ||||
{ | { | ||||
if (smp_started) | |||||
ZONE_ASSERT_COLD(zone); | ZONE_ASSERT_COLD(zone); | ||||
KASSERT(smr != NULL, ("Got NULL smr")); | KASSERT(smr != NULL, ("Got NULL smr")); | ||||
KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, | KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, | ||||
("zone %p (%s) already uses SMR", zone, zone->uz_name)); | ("zone %p (%s) already uses SMR", zone, zone->uz_name)); | ||||
zone->uz_flags |= UMA_ZONE_SMR; | zone->uz_flags |= UMA_ZONE_SMR; | ||||
zone->uz_smr = smr; | zone->uz_smr = smr; | ||||
zone_update_caches(zone); | zone_update_caches(zone); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 780 Lines • Show Last 20 Lines |