Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 1,252 Lines • ▼ Show 20 Lines | |||||
* Frees pages from a keg back to the system. This is done on demand from | * Frees pages from a keg back to the system. This is done on demand from | ||||
* the pageout daemon. | * the pageout daemon. | ||||
* | * | ||||
* Returns nothing. | * Returns nothing. | ||||
*/ | */ | ||||
static void | static void | ||||
keg_drain(uma_keg_t keg) | keg_drain(uma_keg_t keg) | ||||
{ | { | ||||
struct slabhead freeslabs = { 0 }; | struct slabhead freeslabs; | ||||
uma_domain_t dom; | uma_domain_t dom; | ||||
uma_slab_t slab, tmp; | uma_slab_t slab, tmp; | ||||
int i, n; | int i, n; | ||||
/* | |||||
* We don't want to take pages from statically allocated kegs at this | |||||
* time | |||||
*/ | |||||
if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) | if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) | ||||
return; | return; | ||||
for (i = 0; i < vm_ndomains; i++) { | for (i = 0; i < vm_ndomains; i++) { | ||||
CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", | CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", | ||||
keg->uk_name, keg, i, dom->ud_free); | keg->uk_name, keg, i, dom->ud_free_items); | ||||
n = 0; | |||||
dom = &keg->uk_domain[i]; | dom = &keg->uk_domain[i]; | ||||
LIST_INIT(&freeslabs); | |||||
KEG_LOCK(keg, i); | KEG_LOCK(keg, i); | ||||
LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) { | if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) { | ||||
if (keg->uk_flags & UMA_ZFLAG_HASH) | LIST_FOREACH(slab, &dom->ud_free_slab, us_link) | ||||
UMA_HASH_REMOVE(&keg->uk_hash, slab); | UMA_HASH_REMOVE(&keg->uk_hash, slab); | ||||
n++; | |||||
LIST_REMOVE(slab, us_link); | |||||
LIST_INSERT_HEAD(&freeslabs, slab, us_link); | |||||
} | } | ||||
n = dom->ud_free_slabs; | |||||
LIST_SWAP(&freeslabs, &dom->ud_free_slab, uma_slab, us_link); | |||||
dom->ud_free_slabs = 0; | |||||
dom->ud_free_items -= n * keg->uk_ipers; | |||||
dom->ud_pages -= n * keg->uk_ppera; | dom->ud_pages -= n * keg->uk_ppera; | ||||
dom->ud_free -= n * keg->uk_ipers; | |||||
KEG_UNLOCK(keg, i); | KEG_UNLOCK(keg, i); | ||||
} | |||||
while ((slab = LIST_FIRST(&freeslabs)) != NULL) { | LIST_FOREACH_SAFE(slab, &freeslabs, us_link, tmp) | ||||
LIST_REMOVE(slab, us_link); | |||||
keg_free_slab(keg, slab, keg->uk_ipers); | keg_free_slab(keg, slab, keg->uk_ipers); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
zone_reclaim(uma_zone_t zone, int waitok, bool drain) | zone_reclaim(uma_zone_t zone, int waitok, bool drain) | ||||
{ | { | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines | #endif | ||||
/* | /* | ||||
* If we got a slab here it's safe to mark it partially used | * If we got a slab here it's safe to mark it partially used | ||||
* and return. We assume that the caller is going to remove | * and return. We assume that the caller is going to remove | ||||
* at least one item. | * at least one item. | ||||
*/ | */ | ||||
dom = &keg->uk_domain[domain]; | dom = &keg->uk_domain[domain]; | ||||
LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | ||||
dom->ud_pages += keg->uk_ppera; | dom->ud_pages += keg->uk_ppera; | ||||
dom->ud_free += keg->uk_ipers; | dom->ud_free_items += keg->uk_ipers; | ||||
return (slab); | return (slab); | ||||
fail: | fail: | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 811 Lines • ▼ Show 20 Lines | for (i = 0; i < domains; i++) { | ||||
dom = &keg->uk_domain[i]; | dom = &keg->uk_domain[i]; | ||||
oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid), | oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid), | ||||
OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, | OID_AUTO, VM_DOMAIN(i)->vmd_name, CTLFLAG_RD, | ||||
NULL, ""); | NULL, ""); | ||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"pages", CTLFLAG_RD, &dom->ud_pages, 0, | "pages", CTLFLAG_RD, &dom->ud_pages, 0, | ||||
"Total pages currently allocated from VM"); | "Total pages currently allocated from VM"); | ||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"free", CTLFLAG_RD, &dom->ud_free, 0, | "free_items", CTLFLAG_RD, &dom->ud_free_items, 0, | ||||
"items free in the slab layer"); | "items free in the slab layer"); | ||||
} | } | ||||
} else | } else | ||||
SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"name", CTLFLAG_RD, nokeg, "Keg name"); | "name", CTLFLAG_RD, nokeg, "Keg name"); | ||||
/* | /* | ||||
* Information about zone limits. | * Information about zone limits. | ||||
▲ Show 20 Lines • Show All 269 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
uint32_t free, pages; | uint32_t free, pages; | ||||
int i; | int i; | ||||
keg = (uma_keg_t)arg; | keg = (uma_keg_t)arg; | ||||
free = pages = 0; | free = pages = 0; | ||||
for (i = 0; i < vm_ndomains; i++) { | for (i = 0; i < vm_ndomains; i++) { | ||||
free += keg->uk_domain[i].ud_free; | free += keg->uk_domain[i].ud_free_items; | ||||
pages += keg->uk_domain[i].ud_pages; | pages += keg->uk_domain[i].ud_pages; | ||||
KEG_LOCK_FINI(keg, i); | KEG_LOCK_FINI(keg, i); | ||||
} | } | ||||
if (pages != 0) | if (pages != 0) | ||||
printf("Freed UMA keg (%s) was not empty (%u items). " | printf("Freed UMA keg (%s) was not empty (%u items). " | ||||
" Lost %u pages of memory.\n", | " Lost %u pages of memory.\n", | ||||
keg->uk_name ? keg->uk_name : "", | keg->uk_name ? keg->uk_name : "", | ||||
pages / keg->uk_ppera * keg->uk_ipers - free, pages); | pages / keg->uk_ppera * keg->uk_ipers - free, pages); | ||||
▲ Show 20 Lines • Show All 797 Lines • ▼ Show 20 Lines | keg_first_slab(uma_keg_t keg, int domain, bool rr) | ||||
KASSERT(domain >= 0 && domain < vm_ndomains, | KASSERT(domain >= 0 && domain < vm_ndomains, | ||||
("keg_first_slab: domain %d out of range", domain)); | ("keg_first_slab: domain %d out of range", domain)); | ||||
KEG_LOCK_ASSERT(keg, domain); | KEG_LOCK_ASSERT(keg, domain); | ||||
slab = NULL; | slab = NULL; | ||||
start = domain; | start = domain; | ||||
do { | do { | ||||
dom = &keg->uk_domain[domain]; | dom = &keg->uk_domain[domain]; | ||||
if (!LIST_EMPTY(&dom->ud_part_slab)) | if ((slab = LIST_FIRST(&dom->ud_part_slab)) != NULL) | ||||
return (LIST_FIRST(&dom->ud_part_slab)); | return (slab); | ||||
if (!LIST_EMPTY(&dom->ud_free_slab)) { | if ((slab = LIST_FIRST(&dom->ud_free_slab)) != NULL) { | ||||
slab = LIST_FIRST(&dom->ud_free_slab); | |||||
LIST_REMOVE(slab, us_link); | LIST_REMOVE(slab, us_link); | ||||
dom->ud_free_slabs--; | |||||
LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | ||||
return (slab); | return (slab); | ||||
} | } | ||||
if (rr) | if (rr) | ||||
domain = (domain + 1) % vm_ndomains; | domain = (domain + 1) % vm_ndomains; | ||||
} while (domain != start); | } while (domain != start); | ||||
return (NULL); | return (NULL); | ||||
Show All 10 Lines | keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags) | ||||
uint32_t reserve; | uint32_t reserve; | ||||
/* HASH has a single free list. */ | /* HASH has a single free list. */ | ||||
if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) | if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) | ||||
domain = 0; | domain = 0; | ||||
KEG_LOCK(keg, domain); | KEG_LOCK(keg, domain); | ||||
reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; | reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; | ||||
if (keg->uk_domain[domain].ud_free <= reserve || | if (keg->uk_domain[domain].ud_free_items <= reserve || | ||||
(slab = keg_first_slab(keg, domain, rr)) == NULL) { | (slab = keg_first_slab(keg, domain, rr)) == NULL) { | ||||
KEG_UNLOCK(keg, domain); | KEG_UNLOCK(keg, domain); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
return (slab); | return (slab); | ||||
} | } | ||||
static uma_slab_t | static uma_slab_t | ||||
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines | slab_alloc_item(uma_keg_t keg, uma_slab_t slab) | ||||
KEG_LOCK_ASSERT(keg, slab->us_domain); | KEG_LOCK_ASSERT(keg, slab->us_domain); | ||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1; | freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1; | ||||
BIT_CLR(keg->uk_ipers, freei, &slab->us_free); | BIT_CLR(keg->uk_ipers, freei, &slab->us_free); | ||||
item = slab_item(slab, keg, freei); | item = slab_item(slab, keg, freei); | ||||
slab->us_freecount--; | slab->us_freecount--; | ||||
dom->ud_free--; | dom->ud_free_items--; | ||||
/* Move this slab to the full list */ | /* | ||||
* Move this slab to the full list. It must be on the partial list, so | |||||
* we do not need to update the free slab count. In particular, | |||||
* keg_fetch_slab() always returns slabs on the partial list. | |||||
*/ | |||||
if (slab->us_freecount == 0) { | if (slab->us_freecount == 0) { | ||||
LIST_REMOVE(slab, us_link); | LIST_REMOVE(slab, us_link); | ||||
LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link); | ||||
} | } | ||||
return (item); | return (item); | ||||
} | } | ||||
Show All 17 Lines | for (i = 0; i < max; ) { | ||||
if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) | if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) | ||||
break; | break; | ||||
#ifdef NUMA | #ifdef NUMA | ||||
stripe = howmany(max, vm_ndomains); | stripe = howmany(max, vm_ndomains); | ||||
#endif | #endif | ||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
while (slab->us_freecount && i < max) { | while (slab->us_freecount && i < max) { | ||||
bucket[i++] = slab_alloc_item(keg, slab); | bucket[i++] = slab_alloc_item(keg, slab); | ||||
if (dom->ud_free <= keg->uk_reserve) | if (dom->ud_free_items <= keg->uk_reserve) | ||||
break; | break; | ||||
#ifdef NUMA | #ifdef NUMA | ||||
/* | /* | ||||
* If the zone is striped we pick a new slab for every | * If the zone is striped we pick a new slab for every | ||||
* N allocations. Eliminating this conditional will | * N allocations. Eliminating this conditional will | ||||
* instead pick a new domain for each bucket rather | * instead pick a new domain for each bucket rather | ||||
* than stripe within each bucket. The current option | * than stripe within each bucket. The current option | ||||
* produces more fragmentation and requires more cpu | * produces more fragmentation and requires more cpu | ||||
▲ Show 20 Lines • Show All 685 Lines • ▼ Show 20 Lines | slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item) | ||||
uma_domain_t dom; | uma_domain_t dom; | ||||
int freei; | int freei; | ||||
keg = zone->uz_keg; | keg = zone->uz_keg; | ||||
KEG_LOCK_ASSERT(keg, slab->us_domain); | KEG_LOCK_ASSERT(keg, slab->us_domain); | ||||
/* Do we need to remove from any lists? */ | /* Do we need to remove from any lists? */ | ||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
if (slab->us_freecount+1 == keg->uk_ipers) { | if (slab->us_freecount + 1 == keg->uk_ipers) { | ||||
LIST_REMOVE(slab, us_link); | LIST_REMOVE(slab, us_link); | ||||
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); | ||||
dom->ud_free_slabs++; | |||||
} else if (slab->us_freecount == 0) { | } else if (slab->us_freecount == 0) { | ||||
LIST_REMOVE(slab, us_link); | LIST_REMOVE(slab, us_link); | ||||
LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | ||||
} | } | ||||
/* Slab management. */ | /* Slab management. */ | ||||
freei = slab_item_index(slab, keg, item); | freei = slab_item_index(slab, keg, item); | ||||
BIT_SET(keg->uk_ipers, freei, &slab->us_free); | BIT_SET(keg->uk_ipers, freei, &slab->us_free); | ||||
slab->us_freecount++; | slab->us_freecount++; | ||||
/* Keg statistics. */ | /* Keg statistics. */ | ||||
dom->ud_free++; | dom->ud_free_items++; | ||||
} | } | ||||
static void | static void | ||||
zone_release(void *arg, void **bucket, int cnt) | zone_release(void *arg, void **bucket, int cnt) | ||||
{ | { | ||||
struct mtx *lock; | struct mtx *lock; | ||||
uma_zone_t zone; | uma_zone_t zone; | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
▲ Show 20 Lines • Show All 364 Lines • ▼ Show 20 Lines | while (slabs-- > 0) { | ||||
aflags = M_NOWAIT; | aflags = M_NOWAIT; | ||||
vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, | vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, | ||||
&aflags); | &aflags); | ||||
for (;;) { | for (;;) { | ||||
slab = keg_alloc_slab(keg, zone, domain, M_WAITOK, | slab = keg_alloc_slab(keg, zone, domain, M_WAITOK, | ||||
aflags); | aflags); | ||||
if (slab != NULL) { | if (slab != NULL) { | ||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
/* | |||||
* keg_alloc_slab() always returns a slab on the | |||||
* partial list. | |||||
*/ | |||||
LIST_REMOVE(slab, us_link); | LIST_REMOVE(slab, us_link); | ||||
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, | LIST_INSERT_HEAD(&dom->ud_free_slab, slab, | ||||
us_link); | us_link); | ||||
dom->ud_free_slabs++; | |||||
KEG_UNLOCK(keg, slab->us_domain); | KEG_UNLOCK(keg, slab->us_domain); | ||||
break; | break; | ||||
} | } | ||||
if (vm_domainset_iter_policy(&di, &domain) != 0) | if (vm_domainset_iter_policy(&di, &domain) != 0) | ||||
vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); | vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 261 Lines • ▼ Show 20 Lines | sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) | ||||
ush.ush_version = UMA_STREAM_VERSION; | ush.ush_version = UMA_STREAM_VERSION; | ||||
ush.ush_maxcpus = (mp_maxid + 1); | ush.ush_maxcpus = (mp_maxid + 1); | ||||
ush.ush_count = count; | ush.ush_count = count; | ||||
(void)sbuf_bcat(&sbuf, &ush, sizeof(ush)); | (void)sbuf_bcat(&sbuf, &ush, sizeof(ush)); | ||||
LIST_FOREACH(kz, &uma_kegs, uk_link) { | LIST_FOREACH(kz, &uma_kegs, uk_link) { | ||||
kfree = pages = 0; | kfree = pages = 0; | ||||
for (i = 0; i < vm_ndomains; i++) { | for (i = 0; i < vm_ndomains; i++) { | ||||
kfree += kz->uk_domain[i].ud_free; | kfree += kz->uk_domain[i].ud_free_items; | ||||
pages += kz->uk_domain[i].ud_pages; | pages += kz->uk_domain[i].ud_pages; | ||||
} | } | ||||
LIST_FOREACH(z, &kz->uk_zones, uz_link) { | LIST_FOREACH(z, &kz->uk_zones, uz_link) { | ||||
bzero(&uth, sizeof(uth)); | bzero(&uth, sizeof(uth)); | ||||
ZONE_LOCK(z); | ZONE_LOCK(z); | ||||
strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); | strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); | ||||
uth.uth_align = kz->uk_align; | uth.uth_align = kz->uk_align; | ||||
uth.uth_size = kz->uk_size; | uth.uth_size = kz->uk_size; | ||||
▲ Show 20 Lines • Show All 287 Lines • ▼ Show 20 Lines | if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { | ||||
*xdomain = 0; | *xdomain = 0; | ||||
} else | } else | ||||
uma_zone_sumstat(z, cachefree, allocs, &frees, sleeps, | uma_zone_sumstat(z, cachefree, allocs, &frees, sleeps, | ||||
xdomain); | xdomain); | ||||
for (i = 0; i < vm_ndomains; i++) { | for (i = 0; i < vm_ndomains; i++) { | ||||
*cachefree += z->uz_domain[i].uzd_nitems; | *cachefree += z->uz_domain[i].uzd_nitems; | ||||
if (!((z->uz_flags & UMA_ZONE_SECONDARY) && | if (!((z->uz_flags & UMA_ZONE_SECONDARY) && | ||||
(LIST_FIRST(&kz->uk_zones) != z))) | (LIST_FIRST(&kz->uk_zones) != z))) | ||||
*cachefree += kz->uk_domain[i].ud_free; | *cachefree += kz->uk_domain[i].ud_free_items; | ||||
} | } | ||||
*used = *allocs - frees; | *used = *allocs - frees; | ||||
return (((int64_t)*used + *cachefree) * kz->uk_size); | return (((int64_t)*used + *cachefree) * kz->uk_size); | ||||
} | } | ||||
DB_SHOW_COMMAND(uma, db_show_uma) | DB_SHOW_COMMAND(uma, db_show_uma) | ||||
{ | { | ||||
const char *fmt_hdr, *fmt_entry; | const char *fmt_hdr, *fmt_entry; | ||||
▲ Show 20 Lines • Show All 92 Lines • Show Last 20 Lines |