Changeset View
Standalone View
sys/vm/uma_core.c
Show First 20 Lines • Show All 1,409 Lines • ▼ Show 20 Lines | #endif | ||||
} | } | ||||
if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) | if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) | ||||
zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab), | zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab), | ||||
NULL, SKIP_NONE); | NULL, SKIP_NONE); | ||||
keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); | keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); | ||||
uma_total_dec(PAGE_SIZE * keg->uk_ppera); | uma_total_dec(PAGE_SIZE * keg->uk_ppera); | ||||
} | } | ||||
/* | |||||
* Frees pages from a keg back to the system. This is done on demand from | |||||
* the pageout daemon. | |||||
* | |||||
* Returns nothing. | |||||
*/ | |||||
static void | static void | ||||
keg_drain(uma_keg_t keg) | keg_drain_domain(uma_keg_t keg, int domain) | ||||
{ | { | ||||
struct slabhead freeslabs; | struct slabhead freeslabs; | ||||
uma_domain_t dom; | uma_domain_t dom; | ||||
uma_slab_t slab, tmp; | uma_slab_t slab, tmp; | ||||
int i, n; | uint32_t ipers, itofree, partial, reserve; | ||||
if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) | dom = &keg->uk_domain[domain]; | ||||
return; | LIST_INIT(&freeslabs); | ||||
for (i = 0; i < vm_ndomains; i++) { | |||||
CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", | CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", | ||||
keg->uk_name, keg, i, dom->ud_free_items); | keg->uk_name, keg, i, dom->ud_free_items); | ||||
dom = &keg->uk_domain[i]; | |||||
LIST_INIT(&freeslabs); | |||||
KEG_LOCK(keg, i); | KEG_LOCK(keg, domain); | ||||
alc: Suggested comment to describe what follows: Are the free items in partially allocated slabs… | |||||
ipers = keg->uk_ipers; | |||||
itofree = dom->ud_free_slabs * ipers; | |||||
partial = dom->ud_free_items - itofree; | |||||
reserve = keg->uk_reserve; | |||||
Not Done Inline ActionsIf it might be the case that the reserve could be large, this could prevent some pointless list surgery when we can't reclaim anything: n = dom->ud_free_slabs; if (n == 0 || dom->ud_free_items - keg->uk_ipers < keg->uk_reserve) goto unlock; LIST_SWAP(&freeslabs, &dom->ud_free_slab, uma_slab, us_link); rlibby: If it might be the case that the reserve could be large, this could prevent some pointless list… | |||||
Done Inline ActionsI think we can go a bit further and determine whether we need to keep more slabs than we'd free, and in that case pull individual slabs off of the domain's free slab list. markj: I think we can go a bit further and determine whether we need to keep more slabs than we'd free… | |||||
Not Done Inline ActionsHere's a suggestion for rephrasing that into one loop where we calculate the number of slabs to keep up front. Take or leave according to your taste. partial = dom->ud_free_items - dom->ud_free_slabs * keg->uk_ipers; keep_slabs = 0; if (partial < keg->uk_reserve) { keep_slabs = min(dom->ud_free_slabs, howmany(keg->uk_reserve - partial, keg->uk_ipers)); } n = dom->ud_free_slabs - keep_slabs; for (i = min(keep_slabs, n); i > 0; i--) { slab = LIST_FIRST(&dom->ud_free_slab); LIST_REMOVE(slab, us_link); LIST_INSERT_HEAD(&freeslabs, slab, us_link); } if (keep_slabs < n) LIST_SWAP(&freeslabs, &dom->ud_free_slab, uma_slab, us_link); rlibby: Here's a suggestion for rephrasing that into one loop where we calculate the number of slabs to… | |||||
Done Inline ActionsThanks, that's much nicer than what I wrote. markj: Thanks, that's much nicer than what I wrote. | |||||
/* | |||||
* Maintain the keg's reserve of items, if any, while trying to avoid | |||||
* visiting individual slabs more than necessary. | |||||
Done Inline ActionsWhen I first read this "min" operation, my immediate was "what in the world is going on here." I think that this deserves a comment explaining that this is an optimization. alc: When I first read this "min" operation, my immediate was "what in the world is going on here."… | |||||
*/ | |||||
if (partial < reserve && reserve - partial > itofree / 2) { | |||||
itofree = 0; | |||||
Done Inline ActionsBut there are also free items in the partial slabs, and those work fine for reserve allocations. This loop needs to include some kind of comparison of ud_free_items vs uk_reserve. Maybe: free_items = dom->ud_free_items - n * keg->uk_ipers; while (free_items < keg->uk_reserve) { ... n--; free_items += keg->uk_ipers; } rlibby: But there are also free items in the partial slabs, and those work fine for reserve allocations. | |||||
while (dom->ud_free_items - (itofree + ipers) > reserve) { | |||||
Not Done Inline Actions>= ? If it's equal to reserve + ipers, we can subtract ipers and still meet reserve. rlibby: `>=` ? If it's equal to reserve + ipers, we can subtract ipers and still meet reserve. | |||||
slab = LIST_FIRST(&dom->ud_free_slab); | |||||
if (slab == NULL) | |||||
break; | |||||
LIST_REMOVE(slab, us_link); | |||||
Not Done Inline ActionsThis reverses the order. Do we care? If so, we can do tail inserts by keeping track of the tail and using LIST_INSERT_AFTER. rlibby: This reverses the order. Do we care? If so, we can do tail inserts by keeping track of the… | |||||
Done Inline ActionsI can't think of any reason this would hurt anything. markj: I can't think of any reason this would hurt anything. | |||||
LIST_INSERT_HEAD(&freeslabs, slab, us_link); | |||||
itofree += ipers; | |||||
} | |||||
Done Inline ActionsRather than remove from hash above and re-insert here, can't we just move the hash remove block to below the re-insert block (and walk &freeslabs instead)? rlibby: Rather than remove from hash above and re-insert here, can't we just move the hash remove block… | |||||
} else { | |||||
LIST_SWAP(&freeslabs, &dom->ud_free_slab, uma_slab, us_link); | |||||
while (dom->ud_free_items - itofree < reserve) { | |||||
slab = LIST_FIRST(&freeslabs); | |||||
if (slab == NULL) | |||||
break; | |||||
LIST_REMOVE(slab, us_link); | |||||
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); | |||||
itofree -= ipers; | |||||
} | |||||
} | |||||
if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) { | if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) { | ||||
LIST_FOREACH(slab, &dom->ud_free_slab, us_link) | LIST_FOREACH(slab, &freeslabs, us_link) | ||||
UMA_HASH_REMOVE(&keg->uk_hash, slab); | UMA_HASH_REMOVE(&keg->uk_hash, slab); | ||||
} | } | ||||
n = dom->ud_free_slabs; | dom->ud_free_items -= itofree; | ||||
LIST_SWAP(&freeslabs, &dom->ud_free_slab, uma_slab, us_link); | dom->ud_free_slabs -= itofree / ipers; | ||||
dom->ud_free_slabs = 0; | dom->ud_pages -= (itofree / ipers) * keg->uk_ppera; | ||||
dom->ud_free_items -= n * keg->uk_ipers; | KEG_UNLOCK(keg, domain); | ||||
dom->ud_pages -= n * keg->uk_ppera; | |||||
KEG_UNLOCK(keg, i); | |||||
LIST_FOREACH_SAFE(slab, &freeslabs, us_link, tmp) | LIST_FOREACH_SAFE(slab, &freeslabs, us_link, tmp) | ||||
keg_free_slab(keg, slab, keg->uk_ipers); | keg_free_slab(keg, slab, keg->uk_ipers); | ||||
} | } | ||||
/* | |||||
* Frees pages from a keg back to the system. This is done on demand from | |||||
* the pageout daemon. | |||||
* | |||||
* Returns nothing. | |||||
*/ | |||||
static void | |||||
keg_drain(uma_keg_t keg) | |||||
{ | |||||
int i; | |||||
if ((keg->uk_flags & UMA_ZONE_NOFREE) != 0) | |||||
return; | |||||
for (i = 0; i < vm_ndomains; i++) | |||||
keg_drain_domain(keg, i); | |||||
} | } | ||||
static void | static void | ||||
zone_reclaim(uma_zone_t zone, int waitok, bool drain) | zone_reclaim(uma_zone_t zone, int waitok, bool drain) | ||||
{ | { | ||||
/* | /* | ||||
* Set draining to interlock with zone_dtor() so we can release our | * Set draining to interlock with zone_dtor() so we can release our | ||||
▲ Show 20 Lines • Show All 942 Lines • ▼ Show 20 Lines | SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"ppera", CTLFLAG_RD, &keg->uk_ppera, 0, | "ppera", CTLFLAG_RD, &keg->uk_ppera, 0, | ||||
"pages per-slab allocation"); | "pages per-slab allocation"); | ||||
SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"ipers", CTLFLAG_RD, &keg->uk_ipers, 0, | "ipers", CTLFLAG_RD, &keg->uk_ipers, 0, | ||||
"items available per-slab"); | "items available per-slab"); | ||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"align", CTLFLAG_RD, &keg->uk_align, 0, | "align", CTLFLAG_RD, &keg->uk_align, 0, | ||||
"item alignment mask"); | "item alignment mask"); | ||||
SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | |||||
"reserve", CTLFLAG_RD, &keg->uk_reserve, 0, | |||||
"number of reserved items"); | |||||
SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, | ||||
"efficiency", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE, | "efficiency", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE, | ||||
keg, 0, sysctl_handle_uma_slab_efficiency, "I", | keg, 0, sysctl_handle_uma_slab_efficiency, "I", | ||||
"Slab utilization (100 - internal fragmentation %)"); | "Slab utilization (100 - internal fragmentation %)"); | ||||
domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(oid), | domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(oid), | ||||
OID_AUTO, "domain", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); | OID_AUTO, "domain", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); | ||||
for (i = 0; i < domains; i++) { | for (i = 0; i < domains; i++) { | ||||
dom = &keg->uk_domain[i]; | dom = &keg->uk_domain[i]; | ||||
▲ Show 20 Lines • Show All 3,083 Lines • Show Last 20 Lines |
Suggested comment to describe what follows: Are the free items in partially allocated slabs sufficient to meet the reserve? If not, compute the number of fully free slabs that must be kept.