Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 738 Lines • ▼ Show 20 Lines | |||||
* Returns nothing. | * Returns nothing. | ||||
*/ | */ | ||||
static void | static void | ||||
zone_timeout(uma_zone_t zone, void *unused) | zone_timeout(uma_zone_t zone, void *unused) | ||||
{ | { | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
u_int slabs, pages; | u_int slabs, pages; | ||||
if ((zone->uz_flags & UMA_ZONE_HASH) == 0) | if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) | ||||
goto update_wss; | goto update_wss; | ||||
keg = zone->uz_keg; | keg = zone->uz_keg; | ||||
/* | /* | ||||
* Hash zones are non-numa by definition so the first domain | * Hash zones are non-numa by definition so the first domain | ||||
* is the only one present. | * is the only one present. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 402 Lines • ▼ Show 20 Lines | #ifdef INVARIANTS | ||||
* albeit we don't make skip check for other init/fini | * albeit we don't make skip check for other init/fini | ||||
* invocations. | * invocations. | ||||
*/ | */ | ||||
if (!uma_dbg_kskip(keg, slab_item(slab, keg, i)) || | if (!uma_dbg_kskip(keg, slab_item(slab, keg, i)) || | ||||
keg->uk_fini != trash_fini) | keg->uk_fini != trash_fini) | ||||
#endif | #endif | ||||
keg->uk_fini(slab_item(slab, keg, i), keg->uk_size); | keg->uk_fini(slab_item(slab, keg, i), keg->uk_size); | ||||
} | } | ||||
if (keg->uk_flags & UMA_ZONE_OFFPAGE) | if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) | ||||
zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); | zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); | ||||
keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); | keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); | ||||
uma_total_dec(PAGE_SIZE * keg->uk_ppera); | uma_total_dec(PAGE_SIZE * keg->uk_ppera); | ||||
} | } | ||||
/* | /* | ||||
* Frees pages from a keg back to the system. This is done on demand from | * Frees pages from a keg back to the system. This is done on demand from | ||||
* the pageout daemon. | * the pageout daemon. | ||||
Show All 20 Lines | CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", | ||||
keg->uk_name, keg, i, dom->ud_free); | keg->uk_name, keg, i, dom->ud_free); | ||||
n = 0; | n = 0; | ||||
dom = &keg->uk_domain[i]; | dom = &keg->uk_domain[i]; | ||||
KEG_LOCK(keg, i); | KEG_LOCK(keg, i); | ||||
LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) { | LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) { | ||||
/* We have nowhere to free these to. */ | /* We have nowhere to free these to. */ | ||||
if (slab->us_flags & UMA_SLAB_BOOT) | if (slab->us_flags & UMA_SLAB_BOOT) | ||||
continue; | continue; | ||||
if (keg->uk_flags & UMA_ZONE_HASH) | if (keg->uk_flags & UMA_ZFLAG_HASH) | ||||
UMA_HASH_REMOVE(&keg->uk_hash, slab); | UMA_HASH_REMOVE(&keg->uk_hash, slab); | ||||
n++; | n++; | ||||
LIST_REMOVE(slab, us_link); | LIST_REMOVE(slab, us_link); | ||||
LIST_INSERT_HEAD(&freeslabs, slab, us_link); | LIST_INSERT_HEAD(&freeslabs, slab, us_link); | ||||
} | } | ||||
dom->ud_pages -= n * keg->uk_ppera; | dom->ud_pages -= n * keg->uk_ppera; | ||||
dom->ud_free -= n * keg->uk_ipers; | dom->ud_free -= n * keg->uk_ipers; | ||||
KEG_UNLOCK(keg, i); | KEG_UNLOCK(keg, i); | ||||
▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines | keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, | ||||
int i; | int i; | ||||
KASSERT(domain >= 0 && domain < vm_ndomains, | KASSERT(domain >= 0 && domain < vm_ndomains, | ||||
("keg_alloc_slab: domain %d out of range", domain)); | ("keg_alloc_slab: domain %d out of range", domain)); | ||||
allocf = keg->uk_allocf; | allocf = keg->uk_allocf; | ||||
slab = NULL; | slab = NULL; | ||||
mem = NULL; | mem = NULL; | ||||
if (keg->uk_flags & UMA_ZONE_OFFPAGE) { | if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { | ||||
slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, aflags); | slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, aflags); | ||||
if (slab == NULL) | if (slab == NULL) | ||||
goto fail; | goto fail; | ||||
} | } | ||||
/* | /* | ||||
* This reproduces the old vm_zone behavior of zero filling pages the | * This reproduces the old vm_zone behavior of zero filling pages the | ||||
* first time they are added to a zone. | * first time they are added to a zone. | ||||
* | * | ||||
* Malloced items are zeroed in uma_zalloc. | * Malloced items are zeroed in uma_zalloc. | ||||
*/ | */ | ||||
if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) | if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) | ||||
aflags |= M_ZERO; | aflags |= M_ZERO; | ||||
else | else | ||||
aflags &= ~M_ZERO; | aflags &= ~M_ZERO; | ||||
if (keg->uk_flags & UMA_ZONE_NODUMP) | if (keg->uk_flags & UMA_ZONE_NODUMP) | ||||
aflags |= M_NODUMP; | aflags |= M_NODUMP; | ||||
/* zone is passed for legacy reasons. */ | /* zone is passed for legacy reasons. */ | ||||
size = keg->uk_ppera * PAGE_SIZE; | size = keg->uk_ppera * PAGE_SIZE; | ||||
mem = allocf(zone, size, domain, &sflags, aflags); | mem = allocf(zone, size, domain, &sflags, aflags); | ||||
if (mem == NULL) { | if (mem == NULL) { | ||||
if (keg->uk_flags & UMA_ZONE_OFFPAGE) | if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) | ||||
zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); | zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
uma_total_inc(size); | uma_total_inc(size); | ||||
/* For HASH zones all pages go to the same uma_domain. */ | /* For HASH zones all pages go to the same uma_domain. */ | ||||
if ((keg->uk_flags & UMA_ZONE_HASH) != 0) | if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) | ||||
domain = 0; | domain = 0; | ||||
/* Point the slab into the allocated memory */ | /* Point the slab into the allocated memory */ | ||||
if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) | if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) | ||||
slab = (uma_slab_t )(mem + keg->uk_pgoff); | slab = (uma_slab_t )(mem + keg->uk_pgoff); | ||||
else | else | ||||
((uma_hash_slab_t)slab)->uhs_data = mem; | ((uma_hash_slab_t)slab)->uhs_data = mem; | ||||
if (keg->uk_flags & UMA_ZONE_VTOSLAB) | if (keg->uk_flags & UMA_ZFLAG_VTOSLAB) | ||||
for (i = 0; i < keg->uk_ppera; i++) | for (i = 0; i < keg->uk_ppera; i++) | ||||
vsetzoneslab((vm_offset_t)mem + (i * PAGE_SIZE), | vsetzoneslab((vm_offset_t)mem + (i * PAGE_SIZE), | ||||
zone, slab); | zone, slab); | ||||
slab->us_freecount = keg->uk_ipers; | slab->us_freecount = keg->uk_ipers; | ||||
slab->us_flags = sflags; | slab->us_flags = sflags; | ||||
slab->us_domain = domain; | slab->us_domain = domain; | ||||
Show All 12 Lines | if (i != keg->uk_ipers) { | ||||
goto fail; | goto fail; | ||||
} | } | ||||
} | } | ||||
KEG_LOCK(keg, domain); | KEG_LOCK(keg, domain); | ||||
CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)", | CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)", | ||||
slab, keg->uk_name, keg); | slab, keg->uk_name, keg); | ||||
if (keg->uk_flags & UMA_ZONE_HASH) | if (keg->uk_flags & UMA_ZFLAG_HASH) | ||||
UMA_HASH_INSERT(&keg->uk_hash, slab, mem); | UMA_HASH_INSERT(&keg->uk_hash, slab, mem); | ||||
/* | /* | ||||
* If we got a slab here it's safe to mark it partially used | * If we got a slab here it's safe to mark it partially used | ||||
* and return. We assume that the caller is going to remove | * and return. We assume that the caller is going to remove | ||||
* at least one item. | * at least one item. | ||||
*/ | */ | ||||
dom = &keg->uk_domain[domain]; | dom = &keg->uk_domain[domain]; | ||||
▲ Show 20 Lines • Show All 354 Lines • ▼ Show 20 Lines | KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || | ||||
keg->uk_rsize < UMA_PCPU_ALLOC_SIZE, | keg->uk_rsize < UMA_PCPU_ALLOC_SIZE, | ||||
("%s: size %u too large", __func__, keg->uk_rsize)); | ("%s: size %u too large", __func__, keg->uk_rsize)); | ||||
/* | /* | ||||
* Use a pessimistic bit count for shsize. It may be possible to | * Use a pessimistic bit count for shsize. It may be possible to | ||||
* squeeze one more item in for very particular sizes if we were | * squeeze one more item in for very particular sizes if we were | ||||
* to loop and reduce the bitsize if there is waste. | * to loop and reduce the bitsize if there is waste. | ||||
*/ | */ | ||||
if (keg->uk_flags & UMA_ZONE_OFFPAGE) | if (keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) { | ||||
keg->uk_flags |= UMA_ZFLAG_OFFPAGE; | |||||
shsize = 0; | shsize = 0; | ||||
else | } else | ||||
shsize = slab_sizeof(slabsize / rsize); | shsize = slab_sizeof(slabsize / rsize); | ||||
if (rsize <= slabsize - shsize) | if (rsize <= slabsize - shsize) | ||||
keg->uk_ipers = (slabsize - shsize) / rsize; | keg->uk_ipers = (slabsize - shsize) / rsize; | ||||
else { | else { | ||||
/* Handle special case when we have 1 item per slab, so | /* Handle special case when we have 1 item per slab, so | ||||
* alignment requirement can be relaxed. */ | * alignment requirement can be relaxed. */ | ||||
KASSERT(keg->uk_size <= slabsize - shsize, | KASSERT(keg->uk_size <= slabsize - shsize, | ||||
Show All 9 Lines | keg_small_init(uma_keg_t keg) | ||||
/* | /* | ||||
* We can't do OFFPAGE if we're internal or if we've been | * We can't do OFFPAGE if we're internal or if we've been | ||||
* asked to not go to the VM for buckets. If we do this we | * asked to not go to the VM for buckets. If we do this we | ||||
* may end up going to the VM for slabs which we do not | * may end up going to the VM for slabs which we do not | ||||
* want to do if we're UMA_ZFLAG_CACHEONLY as a result | * want to do if we're UMA_ZFLAG_CACHEONLY as a result | ||||
* of UMA_ZONE_VM, which clearly forbids it. | * of UMA_ZONE_VM, which clearly forbids it. | ||||
*/ | */ | ||||
if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) || | if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) || | ||||
(keg->uk_flags & UMA_ZFLAG_CACHEONLY)) | (keg->uk_flags & UMA_ZFLAG_CACHEONLY)) { | ||||
KASSERT((keg->uk_flags & UMA_ZFLAG_OFFPAGE) == 0, | |||||
("%s: incompatible flags 0x%b", __func__, keg->uk_flags, | |||||
PRINT_UMA_ZFLAGS)); | |||||
return; | return; | ||||
} | |||||
/* | /* | ||||
* See if using an OFFPAGE slab will limit our waste. Only do | * See if using an OFFPAGE slab will limit our waste. Only do | ||||
* this if it permits more items per-slab. | * this if it permits more items per-slab. | ||||
* | * | ||||
* XXX We could try growing slabsize to limit max waste as well. | * XXX We could try growing slabsize to limit max waste as well. | ||||
* Historically this was not done because the VM could not | * Historically this was not done because the VM could not | ||||
* efficiently handle contiguous allocations. | * efficiently handle contiguous allocations. | ||||
Show All 11 Lines | CTR6(KTR_UMA, "UMA decided we need offpage slab headers for " | ||||
slabsize / UMA_MAX_WASTE, keg->uk_ipers, | slabsize / UMA_MAX_WASTE, keg->uk_ipers, | ||||
slabsize - keg->uk_ipers * keg->uk_rsize); | slabsize - keg->uk_ipers * keg->uk_rsize); | ||||
/* | /* | ||||
* If we had access to memory to embed a slab header we | * If we had access to memory to embed a slab header we | ||||
* also have a page structure to use vtoslab() instead of | * also have a page structure to use vtoslab() instead of | ||||
* hash to find slabs. If the zone was explicitly created | * hash to find slabs. If the zone was explicitly created | ||||
* OFFPAGE we can't necessarily touch the memory. | * OFFPAGE we can't necessarily touch the memory. | ||||
*/ | */ | ||||
if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0) | keg->uk_flags |= UMA_ZFLAG_OFFPAGE; | ||||
keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB; | |||||
} | } | ||||
if ((keg->uk_flags & UMA_ZONE_OFFPAGE) && | if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0) { | ||||
(keg->uk_flags & UMA_ZONE_VTOSLAB) == 0) | if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0) | ||||
keg->uk_flags |= UMA_ZONE_HASH; | keg->uk_flags |= UMA_ZFLAG_HASH; | ||||
else | |||||
keg->uk_flags |= UMA_ZFLAG_VTOSLAB; | |||||
} | } | ||||
} | |||||
/* | /* | ||||
* Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do | * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do | ||||
* OFFPAGE for now. When I can allow for more dynamic slab sizes this will be | * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be | ||||
* more complicated. | * more complicated. | ||||
* | * | ||||
* Arguments | * Arguments | ||||
* keg The keg we should initialize | * keg The keg we should initialize | ||||
Show All 9 Lines | keg_large_init(uma_keg_t keg) | ||||
KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0, | KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0, | ||||
("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__)); | ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__)); | ||||
keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE); | keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE); | ||||
keg->uk_ipers = 1; | keg->uk_ipers = 1; | ||||
keg->uk_rsize = keg->uk_size; | keg->uk_rsize = keg->uk_size; | ||||
/* Check whether we have enough space to not do OFFPAGE. */ | /* Check whether we have enough space to not do OFFPAGE. */ | ||||
if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0 && | if ((keg->uk_flags & UMA_ZONE_NOTOUCH) == 0 && | ||||
PAGE_SIZE * keg->uk_ppera - keg->uk_rsize < | PAGE_SIZE * keg->uk_ppera - keg->uk_rsize < | ||||
slab_sizeof(SLAB_MIN_SETSIZE)) { | slab_sizeof(SLAB_MIN_SETSIZE)) { | ||||
/* | /* | ||||
* We can't do OFFPAGE if we're internal, in which case | * We can't do OFFPAGE if we're internal, in which case | ||||
* we need an extra page per allocation to contain the | * we need an extra page per allocation to contain the | ||||
* slab header. | * slab header. | ||||
*/ | */ | ||||
if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) == 0) | if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) == 0) | ||||
keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB; | keg->uk_flags |= UMA_ZFLAG_OFFPAGE; | ||||
else | else | ||||
keg->uk_ppera++; | keg->uk_ppera++; | ||||
} | } | ||||
if ((keg->uk_flags & UMA_ZONE_OFFPAGE) && | if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0) { | ||||
(keg->uk_flags & UMA_ZONE_VTOSLAB) == 0) | if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0) | ||||
keg->uk_flags |= UMA_ZONE_HASH; | keg->uk_flags |= UMA_ZFLAG_HASH; | ||||
else | |||||
keg->uk_flags |= UMA_ZFLAG_VTOSLAB; | |||||
} | } | ||||
} | |||||
static void | static void | ||||
keg_cachespread_init(uma_keg_t keg) | keg_cachespread_init(uma_keg_t keg) | ||||
{ | { | ||||
int alignsize; | int alignsize; | ||||
int trailer; | int trailer; | ||||
int pages; | int pages; | ||||
int rsize; | int rsize; | ||||
Show All 14 Lines | keg_cachespread_init(uma_keg_t keg) | ||||
if ((rsize & alignsize) == 0) | if ((rsize & alignsize) == 0) | ||||
rsize += alignsize; | rsize += alignsize; | ||||
trailer = rsize - keg->uk_size; | trailer = rsize - keg->uk_size; | ||||
pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE; | pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE; | ||||
pages = MIN(pages, (128 * 1024) / PAGE_SIZE); | pages = MIN(pages, (128 * 1024) / PAGE_SIZE); | ||||
keg->uk_rsize = rsize; | keg->uk_rsize = rsize; | ||||
keg->uk_ppera = pages; | keg->uk_ppera = pages; | ||||
keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize; | keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize; | ||||
keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB; | keg->uk_flags |= UMA_ZFLAG_OFFPAGE | UMA_ZFLAG_VTOSLAB; | ||||
KASSERT(keg->uk_ipers <= SLAB_MAX_SETSIZE, | KASSERT(keg->uk_ipers <= SLAB_MAX_SETSIZE, | ||||
("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__, | ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__, | ||||
keg->uk_ipers)); | keg->uk_ipers)); | ||||
} | } | ||||
/* | /* | ||||
* Keg header ctor. This initializes all fields, locks, etc. And inserts | * Keg header ctor. This initializes all fields, locks, etc. And inserts | ||||
* the keg onto the global keg list. | * the keg onto the global keg list. | ||||
Show All 34 Lines | keg_ctor(void *mem, int size, void *udata, int flags) | ||||
if (arg->flags & UMA_ZONE_VM) | if (arg->flags & UMA_ZONE_VM) | ||||
keg->uk_flags |= UMA_ZFLAG_CACHEONLY; | keg->uk_flags |= UMA_ZFLAG_CACHEONLY; | ||||
if (arg->flags & UMA_ZONE_ZINIT) | if (arg->flags & UMA_ZONE_ZINIT) | ||||
keg->uk_init = zero_init; | keg->uk_init = zero_init; | ||||
if (arg->flags & UMA_ZONE_MALLOC) | if (arg->flags & UMA_ZONE_MALLOC) | ||||
keg->uk_flags |= UMA_ZONE_VTOSLAB; | keg->uk_flags |= UMA_ZFLAG_VTOSLAB; | ||||
if (arg->flags & UMA_ZONE_PCPU) | #ifndef SMP | ||||
#ifdef SMP | |||||
keg->uk_flags |= UMA_ZONE_OFFPAGE; | |||||
#else | |||||
keg->uk_flags &= ~UMA_ZONE_PCPU; | keg->uk_flags &= ~UMA_ZONE_PCPU; | ||||
#endif | #endif | ||||
if (keg->uk_flags & UMA_ZONE_CACHESPREAD) { | if (keg->uk_flags & UMA_ZONE_CACHESPREAD) { | ||||
keg_cachespread_init(keg); | keg_cachespread_init(keg); | ||||
} else { | } else { | ||||
if (keg->uk_size > slab_space(SLAB_MIN_SETSIZE)) | if (keg->uk_size > slab_space(SLAB_MIN_SETSIZE)) | ||||
keg_large_init(keg); | keg_large_init(keg); | ||||
else | else | ||||
keg_small_init(keg); | keg_small_init(keg); | ||||
} | } | ||||
/* | /* | ||||
* Use a first-touch NUMA policy for all kegs that pmap_extract() | * Use a first-touch NUMA policy for all kegs that pmap_extract() | ||||
* will work on with the exception of critical VM structures | * will work on with the exception of critical VM structures | ||||
* necessary for paging. | * necessary for paging. | ||||
* | * | ||||
* Zones may override the default by specifying either. | * Zones may override the default by specifying either. | ||||
*/ | */ | ||||
#ifdef NUMA | #ifdef NUMA | ||||
if ((keg->uk_flags & | if ((keg->uk_flags & | ||||
(UMA_ZONE_HASH | UMA_ZONE_VM | UMA_ZONE_ROUNDROBIN)) == 0) | (UMA_ZFLAG_HASH | UMA_ZONE_VM | UMA_ZONE_ROUNDROBIN)) == 0) | ||||
keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; | keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; | ||||
else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) | else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) | ||||
keg->uk_flags |= UMA_ZONE_ROUNDROBIN; | keg->uk_flags |= UMA_ZONE_ROUNDROBIN; | ||||
#endif | #endif | ||||
if (keg->uk_flags & UMA_ZONE_OFFPAGE) | if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) | ||||
keg->uk_slabzone = slabzone; | keg->uk_slabzone = slabzone; | ||||
/* | /* | ||||
* If we haven't booted yet we need allocations to go through the | * If we haven't booted yet we need allocations to go through the | ||||
* startup cache until the vm is ready. | * startup cache until the vm is ready. | ||||
*/ | */ | ||||
if (booted < BOOT_PAGEALLOC) | if (booted < BOOT_PAGEALLOC) | ||||
keg->uk_allocf = startup_alloc; | keg->uk_allocf = startup_alloc; | ||||
Show All 21 Lines | #endif | ||||
for (i = 0; i < vm_ndomains; i++) | for (i = 0; i < vm_ndomains; i++) | ||||
KEG_LOCK_INIT(keg, i, (arg->flags & UMA_ZONE_MTXCLASS)); | KEG_LOCK_INIT(keg, i, (arg->flags & UMA_ZONE_MTXCLASS)); | ||||
/* | /* | ||||
* If we're putting the slab header in the actual page we need to | * If we're putting the slab header in the actual page we need to | ||||
* figure out where in each page it goes. See slab_sizeof | * figure out where in each page it goes. See slab_sizeof | ||||
* definition. | * definition. | ||||
*/ | */ | ||||
if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) { | if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) { | ||||
size_t shsize; | size_t shsize; | ||||
shsize = slab_sizeof(keg->uk_ipers); | shsize = slab_sizeof(keg->uk_ipers); | ||||
keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - shsize; | keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - shsize; | ||||
/* | /* | ||||
* The only way the following is possible is if with our | * The only way the following is possible is if with our | ||||
* UMA_ALIGN_PTR adjustments we are now bigger than | * UMA_ALIGN_PTR adjustments we are now bigger than | ||||
* UMA_SLAB_SIZE. I haven't checked whether this is | * UMA_SLAB_SIZE. I haven't checked whether this is | ||||
* mathematically possible for all cases, so we make | * mathematically possible for all cases, so we make | ||||
* sure here anyway. | * sure here anyway. | ||||
*/ | */ | ||||
KASSERT(keg->uk_pgoff + shsize <= PAGE_SIZE * keg->uk_ppera, | KASSERT(keg->uk_pgoff + shsize <= PAGE_SIZE * keg->uk_ppera, | ||||
("zone %s ipers %d rsize %d size %d slab won't fit", | ("zone %s ipers %d rsize %d size %d slab won't fit", | ||||
zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); | zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); | ||||
} | } | ||||
if (keg->uk_flags & UMA_ZONE_HASH) | if (keg->uk_flags & UMA_ZFLAG_HASH) | ||||
hash_alloc(&keg->uk_hash, 0); | hash_alloc(&keg->uk_hash, 0); | ||||
CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)\n", keg, zone->uz_name, zone); | CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)\n", keg, zone->uz_name, zone); | ||||
LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); | LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); | ||||
rw_wlock(&uma_rwlock); | rw_wlock(&uma_rwlock); | ||||
LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); | LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); | ||||
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"Desired per-cpu cache size"); | "Desired per-cpu cache size"); | ||||
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0, | "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0, | ||||
"Maximum allowed per-cpu cache size"); | "Maximum allowed per-cpu cache size"); | ||||
/* | /* | ||||
* keg if present. | * keg if present. | ||||
*/ | */ | ||||
if ((zone->uz_flags & UMA_ZONE_HASH) == 0) | if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) | ||||
domains = vm_ndomains; | domains = vm_ndomains; | ||||
else | else | ||||
domains = 1; | domains = 1; | ||||
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, | oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, | ||||
"keg", CTLFLAG_RD, NULL, ""); | "keg", CTLFLAG_RD, NULL, ""); | ||||
keg = zone->uz_keg; | keg = zone->uz_keg; | ||||
if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) { | if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) { | ||||
SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
▲ Show 20 Lines • Show All 628 Lines • ▼ Show 20 Lines | |||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
/* | /* | ||||
* Inject procedures which check for memory use after free if we are | * Inject procedures which check for memory use after free if we are | ||||
* allowed to scramble the memory while it is not allocated. This | * allowed to scramble the memory while it is not allocated. This | ||||
* requires that: UMA is actually able to access the memory, no init | * requires that: UMA is actually able to access the memory, no init | ||||
* or fini procedures, no dependency on the initial value of the | * or fini procedures, no dependency on the initial value of the | ||||
* memory, and no (legitimate) use of the memory after free. Note, | * memory, and no (legitimate) use of the memory after free. Note, | ||||
* the ctor and dtor do not need to be empty. | * the ctor and dtor do not need to be empty. | ||||
* | |||||
* XXX UMA_ZONE_OFFPAGE. | |||||
*/ | */ | ||||
if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOFREE))) && | if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOTOUCH | | ||||
uminit == NULL && fini == NULL) { | UMA_ZONE_NOFREE))) && uminit == NULL && fini == NULL) { | ||||
args.uminit = trash_init; | args.uminit = trash_init; | ||||
args.fini = trash_fini; | args.fini = trash_fini; | ||||
} | } | ||||
#endif | #endif | ||||
args.align = align; | args.align = align; | ||||
args.flags = flags; | args.flags = flags; | ||||
args.keg = NULL; | args.keg = NULL; | ||||
▲ Show 20 Lines • Show All 466 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
static uma_slab_t | static uma_slab_t | ||||
keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags) | keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags) | ||||
{ | { | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
uint32_t reserve; | uint32_t reserve; | ||||
/* HASH has a single free list. */ | /* HASH has a single free list. */ | ||||
if ((keg->uk_flags & UMA_ZONE_HASH) != 0) | if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) | ||||
domain = 0; | domain = 0; | ||||
KEG_LOCK(keg, domain); | KEG_LOCK(keg, domain); | ||||
reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; | reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; | ||||
if (keg->uk_domain[domain].ud_free <= reserve || | if (keg->uk_domain[domain].ud_free <= reserve || | ||||
(slab = keg_first_slab(keg, domain, rr)) == NULL) { | (slab = keg_first_slab(keg, domain, rr)) == NULL) { | ||||
KEG_UNLOCK(keg, domain); | KEG_UNLOCK(keg, domain); | ||||
return (NULL); | return (NULL); | ||||
▲ Show 20 Lines • Show All 785 Lines • ▼ Show 20 Lines | zone_release(void *arg, void **bucket, int cnt) | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
uint8_t *mem; | uint8_t *mem; | ||||
void *item; | void *item; | ||||
int i; | int i; | ||||
zone = arg; | zone = arg; | ||||
keg = zone->uz_keg; | keg = zone->uz_keg; | ||||
lock = NULL; | lock = NULL; | ||||
if (__predict_false((zone->uz_flags & UMA_ZONE_HASH) != 0)) | if (__predict_false((zone->uz_flags & UMA_ZFLAG_HASH) != 0)) | ||||
lock = KEG_LOCK(keg, 0); | lock = KEG_LOCK(keg, 0); | ||||
for (i = 0; i < cnt; i++) { | for (i = 0; i < cnt; i++) { | ||||
item = bucket[i]; | item = bucket[i]; | ||||
if (__predict_true((zone->uz_flags & UMA_ZONE_VTOSLAB) != 0)) { | if (__predict_true((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0)) { | ||||
slab = vtoslab((vm_offset_t)item); | slab = vtoslab((vm_offset_t)item); | ||||
} else { | } else { | ||||
mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); | mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); | ||||
if ((zone->uz_flags & UMA_ZONE_HASH) != 0) | if ((zone->uz_flags & UMA_ZFLAG_HASH) != 0) | ||||
slab = hash_sfind(&keg->uk_hash, mem); | slab = hash_sfind(&keg->uk_hash, mem); | ||||
else | else | ||||
slab = (uma_slab_t)(mem + keg->uk_pgoff); | slab = (uma_slab_t)(mem + keg->uk_pgoff); | ||||
} | } | ||||
if (lock != KEG_LOCKPTR(keg, slab->us_domain)) { | if (lock != KEG_LOCKPTR(keg, slab->us_domain)) { | ||||
if (lock != NULL) | if (lock != NULL) | ||||
mtx_unlock(lock); | mtx_unlock(lock); | ||||
lock = KEG_LOCK(keg, slab->us_domain); | lock = KEG_LOCK(keg, slab->us_domain); | ||||
▲ Show 20 Lines • Show All 724 Lines • ▼ Show 20 Lines | |||||
static int | static int | ||||
sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS) | sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
uma_keg_t keg = arg1; | uma_keg_t keg = arg1; | ||||
int avail, effpct, total; | int avail, effpct, total; | ||||
total = keg->uk_ppera * PAGE_SIZE; | total = keg->uk_ppera * PAGE_SIZE; | ||||
if ((keg->uk_flags & UMA_ZONE_OFFPAGE) != 0) | if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0) | ||||
total += slab_sizeof(SLAB_MAX_SETSIZE); | total += slab_sizeof(SLAB_MAX_SETSIZE); | ||||
/* | /* | ||||
* We consider the client's requested size and alignment here, not the | * We consider the client's requested size and alignment here, not the | ||||
* real size determination uk_rsize, because we also adjust the real | * real size determination uk_rsize, because we also adjust the real | ||||
* size for internal implementation reasons (max bitset size). | * size for internal implementation reasons (max bitset size). | ||||
*/ | */ | ||||
avail = keg->uk_ipers * roundup2(keg->uk_size, keg->uk_align + 1); | avail = keg->uk_ipers * roundup2(keg->uk_size, keg->uk_align + 1); | ||||
if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) | if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) | ||||
Show All 23 Lines | uma_dbg_getslab(uma_zone_t zone, void *item) | ||||
/* | /* | ||||
* It is safe to return the slab here even though the | * It is safe to return the slab here even though the | ||||
* zone is unlocked because the item's allocation state | * zone is unlocked because the item's allocation state | ||||
* essentially holds a reference. | * essentially holds a reference. | ||||
*/ | */ | ||||
mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); | mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); | ||||
if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) | if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) | ||||
return (NULL); | return (NULL); | ||||
if (zone->uz_flags & UMA_ZONE_VTOSLAB) | if (zone->uz_flags & UMA_ZFLAG_VTOSLAB) | ||||
return (vtoslab((vm_offset_t)mem)); | return (vtoslab((vm_offset_t)mem)); | ||||
keg = zone->uz_keg; | keg = zone->uz_keg; | ||||
if ((keg->uk_flags & UMA_ZONE_HASH) == 0) | if ((keg->uk_flags & UMA_ZFLAG_HASH) == 0) | ||||
return ((uma_slab_t)(mem + keg->uk_pgoff)); | return ((uma_slab_t)(mem + keg->uk_pgoff)); | ||||
KEG_LOCK(keg, 0); | KEG_LOCK(keg, 0); | ||||
slab = hash_sfind(&keg->uk_hash, mem); | slab = hash_sfind(&keg->uk_hash, mem); | ||||
KEG_UNLOCK(keg, 0); | KEG_UNLOCK(keg, 0); | ||||
return (slab); | return (slab); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 219 Lines • Show Last 20 Lines |