Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/uma_core.c
Show First 20 Lines • Show All 230 Lines • ▼ Show 20 Lines | |||||
static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void page_free(void *, vm_size_t, uint8_t); | static void page_free(void *, vm_size_t, uint8_t); | ||||
static void pcpu_page_free(void *, vm_size_t, uint8_t); | static void pcpu_page_free(void *, vm_size_t, uint8_t); | ||||
static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int); | static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int); | ||||
static void cache_drain(uma_zone_t); | static void cache_drain(uma_zone_t); | ||||
static void bucket_drain(uma_zone_t, uma_bucket_t); | static void bucket_drain(uma_zone_t, uma_bucket_t); | ||||
static void bucket_cache_drain(uma_zone_t zone); | static void bucket_cache_reclaim(uma_zone_t zone, bool); | ||||
static int keg_ctor(void *, int, void *, int); | static int keg_ctor(void *, int, void *, int); | ||||
static void keg_dtor(void *, int, void *); | static void keg_dtor(void *, int, void *); | ||||
static int zone_ctor(void *, int, void *, int); | static int zone_ctor(void *, int, void *, int); | ||||
static void zone_dtor(void *, int, void *); | static void zone_dtor(void *, int, void *); | ||||
static int zero_init(void *, int, int); | static int zero_init(void *, int, int); | ||||
static void keg_small_init(uma_keg_t keg); | static void keg_small_init(uma_keg_t keg); | ||||
static void keg_large_init(uma_keg_t keg); | static void keg_large_init(uma_keg_t keg); | ||||
static void zone_foreach(void (*zfunc)(uma_zone_t)); | static void zone_foreach(void (*zfunc)(uma_zone_t)); | ||||
▲ Show 20 Lines • Show All 200 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static void | static void | ||||
bucket_zone_drain(void) | bucket_zone_drain(void) | ||||
{ | { | ||||
struct uma_bucket_zone *ubz; | struct uma_bucket_zone *ubz; | ||||
for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) | for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) | ||||
zone_drain(ubz->ubz_zone); | uma_zreclaim(ubz->ubz_zone, UMA_RECLAIM_DRAIN); | ||||
} | } | ||||
static uma_bucket_t | static uma_bucket_t | ||||
zone_try_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, const bool ws) | zone_try_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, const bool ws) | ||||
{ | { | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
if ((bucket = LIST_FIRST(&zdom->uzd_buckets)) != NULL) { | if ((bucket = LIST_FIRST(&zdom->uzd_buckets)) != NULL) { | ||||
▲ Show 20 Lines • Show All 292 Lines • ▼ Show 20 Lines | cache_drain(uma_zone_t zone) | ||||
/* | /* | ||||
* XXX: It is safe to not lock the per-CPU caches, because we're | * XXX: It is safe to not lock the per-CPU caches, because we're | ||||
* tearing down the zone anyway. I.e., there will be no further use | * tearing down the zone anyway. I.e., there will be no further use | ||||
* of the caches at this point. | * of the caches at this point. | ||||
* | * | ||||
* XXX: It would good to be able to assert that the zone is being | * XXX: It would good to be able to assert that the zone is being | ||||
* torn down to prevent improper use of cache_drain(). | * torn down to prevent improper use of cache_drain(). | ||||
* | * | ||||
* XXX: We lock the zone before passing into bucket_cache_drain() as | * XXX: We lock the zone before passing into bucket_cache_reclaim() as | ||||
* it is used elsewhere. Should the tear-down path be made special | * it is used elsewhere. Should the tear-down path be made special | ||||
* there in some form? | * there in some form? | ||||
*/ | */ | ||||
CPU_FOREACH(cpu) { | CPU_FOREACH(cpu) { | ||||
cache = &zone->uz_cpu[cpu]; | cache = &zone->uz_cpu[cpu]; | ||||
bucket_drain(zone, cache->uc_allocbucket); | bucket_drain(zone, cache->uc_allocbucket); | ||||
bucket_drain(zone, cache->uc_freebucket); | bucket_drain(zone, cache->uc_freebucket); | ||||
if (cache->uc_allocbucket != NULL) | if (cache->uc_allocbucket != NULL) | ||||
bucket_free(zone, cache->uc_allocbucket, NULL); | bucket_free(zone, cache->uc_allocbucket, NULL); | ||||
if (cache->uc_freebucket != NULL) | if (cache->uc_freebucket != NULL) | ||||
bucket_free(zone, cache->uc_freebucket, NULL); | bucket_free(zone, cache->uc_freebucket, NULL); | ||||
cache->uc_allocbucket = cache->uc_freebucket = NULL; | cache->uc_allocbucket = cache->uc_freebucket = NULL; | ||||
} | } | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
bucket_cache_drain(zone); | bucket_cache_reclaim(zone, true); | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
} | } | ||||
static void | static void | ||||
cache_shrink(uma_zone_t zone) | cache_shrink(uma_zone_t zone) | ||||
{ | { | ||||
if (zone->uz_flags & UMA_ZFLAG_INTERNAL) | if (zone->uz_flags & UMA_ZFLAG_INTERNAL) | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Safely drain per-CPU caches of a zone(s) to alloc bucket. | * Safely drain per-CPU caches of a zone(s) to alloc bucket. | ||||
* This is an expensive call because it needs to bind to all CPUs | * This is an expensive call because it needs to bind to all CPUs | ||||
* one by one and enter a critical section on each of them in order | * one by one and enter a critical section on each of them in order | ||||
* to safely access their cache buckets. | * to safely access their cache buckets. | ||||
* Zone lock must not be held on call this function. | * Zone lock must not be held on call this function. | ||||
*/ | */ | ||||
static void | static void | ||||
cache_drain_safe(uma_zone_t zone) | pcpu_cache_drain_safe(uma_zone_t zone) | ||||
{ | { | ||||
int cpu; | int cpu; | ||||
/* | /* | ||||
* Polite bucket sizes shrinking was not enouth, shrink aggressively. | * Polite bucket sizes shrinking was not enouth, shrink aggressively. | ||||
*/ | */ | ||||
if (zone) | if (zone) | ||||
cache_shrink(zone); | cache_shrink(zone); | ||||
Show All 11 Lines | else | ||||
zone_foreach(cache_drain_safe_cpu); | zone_foreach(cache_drain_safe_cpu); | ||||
} | } | ||||
thread_lock(curthread); | thread_lock(curthread); | ||||
sched_unbind(curthread); | sched_unbind(curthread); | ||||
thread_unlock(curthread); | thread_unlock(curthread); | ||||
} | } | ||||
/* | /* | ||||
* Drain the cached buckets from a zone. Expects a locked zone on entry. | * Reclaim cached buckets from a zone. All buckets are reclaimed if the caller | ||||
* requested a drain, otherwise the per-domain caches are trimmed to either | |||||
* estimated working set size. | |||||
*/ | */ | ||||
static void | static void | ||||
bucket_cache_drain(uma_zone_t zone) | bucket_cache_reclaim(uma_zone_t zone, bool drain) | ||||
{ | { | ||||
uma_zone_domain_t zdom; | uma_zone_domain_t zdom; | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
int i; | int i, target; | ||||
/* | |||||
* Drain the bucket queues and free the buckets. | |||||
*/ | |||||
for (i = 0; i < vm_ndomains; i++) { | for (i = 0; i < vm_ndomains; i++) { | ||||
zdom = &zone->uz_domain[i]; | zdom = &zone->uz_domain[i]; | ||||
while ((bucket = zone_try_fetch_bucket(zone, zdom, false)) != | target = drain ? 0 : zdom->uzd_wss; | ||||
jeff: I missed the review for the working set-size. Where is that? | |||||
markjAuthorUnsubmitted Not Done Inline ActionsD16666, just for the record, but you already commented on it. markj: D16666, just for the record, but you already commented on it. | |||||
NULL) { | while (zdom->uzd_nitems > target) { | ||||
bucket = zone_try_fetch_bucket(zone, zdom, false); | |||||
if (bucket == NULL) | |||||
break; | |||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
bucket_drain(zone, bucket); | bucket_drain(zone, bucket); | ||||
bucket_free(zone, bucket, NULL); | bucket_free(zone, bucket, NULL); | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Shrink further bucket sizes. Price of single zone lock collision | * Shrink the zone bucket size to ensure that the per-CPU caches | ||||
* is probably lower then price of global cache drain. | * don't grow too large. | ||||
*/ | */ | ||||
if (zone->uz_count > zone->uz_count_min) | if (zone->uz_count > zone->uz_count_min) | ||||
zone->uz_count--; | zone->uz_count--; | ||||
} | } | ||||
static void | static void | ||||
keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) | keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines | finished: | ||||
while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { | while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { | ||||
SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); | SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); | ||||
keg_free_slab(keg, slab, keg->uk_ipers); | keg_free_slab(keg, slab, keg->uk_ipers); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
zone_drain_wait(uma_zone_t zone, int waitok) | zone_reclaim(uma_zone_t zone, int waitok, bool drain) | ||||
{ | { | ||||
/* | /* | ||||
* Set draining to interlock with zone_dtor() so we can release our | * Set draining to interlock with zone_dtor() so we can release our | ||||
* locks as we go. Only dtor() should do a WAITOK call since it | * locks as we go. Only dtor() should do a WAITOK call since it | ||||
* is the only call that knows the structure will still be available | * is the only call that knows the structure will still be available | ||||
* when it wakes up. | * when it wakes up. | ||||
*/ | */ | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
while (zone->uz_flags & UMA_ZFLAG_DRAINING) { | while (zone->uz_flags & UMA_ZFLAG_RECLAIMING) { | ||||
if (waitok == M_NOWAIT) | if (waitok == M_NOWAIT) | ||||
goto out; | goto out; | ||||
msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1); | msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1); | ||||
} | } | ||||
zone->uz_flags |= UMA_ZFLAG_DRAINING; | zone->uz_flags |= UMA_ZFLAG_RECLAIMING; | ||||
bucket_cache_drain(zone); | bucket_cache_reclaim(zone, drain); | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
/* | /* | ||||
* The DRAINING flag protects us from being freed while | * The DRAINING flag protects us from being freed while | ||||
* we're running. Normally the uma_rwlock would protect us but we | * we're running. Normally the uma_rwlock would protect us but we | ||||
* must be able to release and acquire the right lock for each keg. | * must be able to release and acquire the right lock for each keg. | ||||
*/ | */ | ||||
zone_foreach_keg(zone, &keg_drain); | zone_foreach_keg(zone, &keg_drain); | ||||
ZONE_LOCK(zone); | ZONE_LOCK(zone); | ||||
zone->uz_flags &= ~UMA_ZFLAG_DRAINING; | zone->uz_flags &= ~UMA_ZFLAG_RECLAIMING; | ||||
wakeup(zone); | wakeup(zone); | ||||
out: | out: | ||||
ZONE_UNLOCK(zone); | ZONE_UNLOCK(zone); | ||||
} | } | ||||
void | static void | ||||
zone_drain(uma_zone_t zone) | zone_drain(uma_zone_t zone) | ||||
{ | { | ||||
zone_drain_wait(zone, M_NOWAIT); | zone_reclaim(zone, M_NOWAIT, true); | ||||
} | } | ||||
static void | |||||
zone_trim(uma_zone_t zone) | |||||
{ | |||||
zone_reclaim(zone, M_NOWAIT, false); | |||||
} | |||||
/* | /* | ||||
* Allocate a new slab for a keg. This does not insert the slab onto a list. | * Allocate a new slab for a keg. This does not insert the slab onto a list. | ||||
* | * | ||||
* Arguments: | * Arguments: | ||||
* wait Shall we wait? | * wait Shall we wait? | ||||
* | * | ||||
* Returns: | * Returns: | ||||
* The slab that was allocated or NULL if there is no memory and the | * The slab that was allocated or NULL if there is no memory and the | ||||
▲ Show 20 Lines • Show All 863 Lines • ▼ Show 20 Lines | zone_dtor(void *arg, int size, void *udata) | ||||
LIST_REMOVE(zone, uz_link); | LIST_REMOVE(zone, uz_link); | ||||
rw_wunlock(&uma_rwlock); | rw_wunlock(&uma_rwlock); | ||||
/* | /* | ||||
* XXX there are some races here where | * XXX there are some races here where | ||||
* the zone can be drained but zone lock | * the zone can be drained but zone lock | ||||
* released and then refilled before we | * released and then refilled before we | ||||
* remove it... we dont care for now | * remove it... we dont care for now | ||||
*/ | */ | ||||
zone_drain_wait(zone, M_WAITOK); | zone_reclaim(zone, M_WAITOK, true); | ||||
/* | /* | ||||
* Unlink all of our kegs. | * Unlink all of our kegs. | ||||
*/ | */ | ||||
while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) { | while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) { | ||||
klink->kl_keg = NULL; | klink->kl_keg = NULL; | ||||
LIST_REMOVE(klink, kl_link); | LIST_REMOVE(klink, kl_link); | ||||
if (klink == &zone->uz_klink) | if (klink == &zone->uz_klink) | ||||
continue; | continue; | ||||
▲ Show 20 Lines • Show All 1,714 Lines • ▼ Show 20 Lines | while (slabs > 0) { | ||||
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); | ||||
slabs--; | slabs--; | ||||
domain = (domain + 1) % vm_ndomains; | domain = (domain + 1) % vm_ndomains; | ||||
} | } | ||||
KEG_UNLOCK(keg); | KEG_UNLOCK(keg); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
static void | void | ||||
uma_reclaim_locked(bool kmem_danger) | uma_reclaim(int req) | ||||
{ | { | ||||
CTR0(KTR_UMA, "UMA: vm asked us to release pages!"); | CTR0(KTR_UMA, "UMA: vm asked us to release pages!"); | ||||
sx_assert(&uma_drain_lock, SA_XLOCKED); | sx_xlock(&uma_drain_lock); | ||||
bucket_enable(); | bucket_enable(); | ||||
switch (req) { | |||||
case UMA_RECLAIM_TRIM: | |||||
zone_foreach(zone_trim); | |||||
break; | |||||
case UMA_RECLAIM_DRAIN: | |||||
case UMA_RECLAIM_DRAIN_CPU: | |||||
zone_foreach(zone_drain); | zone_foreach(zone_drain); | ||||
if (vm_page_count_min() || kmem_danger) { | if (req == UMA_RECLAIM_DRAIN_CPU) { | ||||
cache_drain_safe(NULL); | pcpu_cache_drain_safe(NULL); | ||||
zone_foreach(zone_drain); | zone_foreach(zone_drain); | ||||
} | } | ||||
break; | |||||
default: | |||||
panic("unhandled reclamation request %d", req); | |||||
} | |||||
/* | /* | ||||
* Some slabs may have been freed but this zone will be visited early | * Some slabs may have been freed but this zone will be visited early | ||||
* we visit again so that we can free pages that are empty once other | * we visit again so that we can free pages that are empty once other | ||||
* zones are drained. We have to do the same for buckets. | * zones are drained. We have to do the same for buckets. | ||||
*/ | */ | ||||
zone_drain(slabzone); | zone_drain(slabzone); | ||||
bucket_zone_drain(); | bucket_zone_drain(); | ||||
} | |||||
void | |||||
uma_reclaim(void) | |||||
{ | |||||
sx_xlock(&uma_drain_lock); | |||||
uma_reclaim_locked(false); | |||||
sx_xunlock(&uma_drain_lock); | sx_xunlock(&uma_drain_lock); | ||||
} | } | ||||
static volatile int uma_reclaim_needed; | static volatile int uma_reclaim_needed; | ||||
void | void | ||||
uma_reclaim_wakeup(void) | uma_reclaim_wakeup(void) | ||||
{ | { | ||||
if (atomic_fetchadd_int(&uma_reclaim_needed, 1) == 0) | if (atomic_fetchadd_int(&uma_reclaim_needed, 1) == 0) | ||||
wakeup(uma_reclaim); | wakeup(uma_reclaim); | ||||
} | } | ||||
void | void | ||||
uma_reclaim_worker(void *arg __unused) | uma_reclaim_worker(void *arg __unused) | ||||
{ | { | ||||
for (;;) { | for (;;) { | ||||
sx_xlock(&uma_drain_lock); | sx_xlock(&uma_drain_lock); | ||||
while (atomic_load_int(&uma_reclaim_needed) == 0) | while (atomic_load_int(&uma_reclaim_needed) == 0) | ||||
sx_sleep(uma_reclaim, &uma_drain_lock, PVM, "umarcl", | sx_sleep(uma_reclaim, &uma_drain_lock, PVM, "umarcl", | ||||
hz); | hz); | ||||
sx_xunlock(&uma_drain_lock); | sx_xunlock(&uma_drain_lock); | ||||
EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_KMEM); | EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_KMEM); | ||||
sx_xlock(&uma_drain_lock); | uma_reclaim(UMA_RECLAIM_DRAIN_CPU); | ||||
uma_reclaim_locked(true); | |||||
atomic_store_int(&uma_reclaim_needed, 0); | atomic_store_int(&uma_reclaim_needed, 0); | ||||
sx_xunlock(&uma_drain_lock); | |||||
/* Don't fire more than once per-second. */ | /* Don't fire more than once per-second. */ | ||||
pause("umarclslp", hz); | pause("umarclslp", hz); | ||||
} | |||||
} | |||||
/* See uma.h */ | |||||
void | |||||
uma_zreclaim(uma_zone_t zone, int req) | |||||
{ | |||||
switch (req) { | |||||
case UMA_RECLAIM_TRIM: | |||||
zone_trim(zone); | |||||
break; | |||||
case UMA_RECLAIM_DRAIN: | |||||
zone_drain(zone); | |||||
break; | |||||
case UMA_RECLAIM_DRAIN_CPU: | |||||
pcpu_cache_drain_safe(zone); | |||||
zone_drain(zone); | |||||
break; | |||||
default: | |||||
panic("unhandled reclamation request %d", req); | |||||
} | } | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
int | int | ||||
uma_zone_exhausted(uma_zone_t zone) | uma_zone_exhausted(uma_zone_t zone) | ||||
{ | { | ||||
int full; | int full; | ||||
▲ Show 20 Lines • Show All 568 Lines • Show Last 20 Lines |
I missed the review for the working set-size. Where is that?