Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | |||||
#include "opt_ddb.h" | #include "opt_ddb.h" | ||||
#include "opt_param.h" | #include "opt_param.h" | ||||
#include "opt_vm.h" | #include "opt_vm.h" | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/bitset.h> | #include <sys/bitset.h> | ||||
#include <sys/domainset.h> | |||||
#include <sys/eventhandler.h> | #include <sys/eventhandler.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/types.h> | #include <sys/types.h> | ||||
#include <sys/limits.h> | #include <sys/limits.h> | ||||
#include <sys/queue.h> | #include <sys/queue.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/ktr.h> | #include <sys/ktr.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/random.h> | #include <sys/random.h> | ||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/sched.h> | #include <sys/sched.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/taskqueue.h> | #include <sys/taskqueue.h> | ||||
#include <sys/vmmeter.h> | #include <sys/vmmeter.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/vm_domainset.h> | |||||
#include <vm/vm_object.h> | #include <vm/vm_object.h> | ||||
#include <vm/vm_page.h> | #include <vm/vm_page.h> | ||||
#include <vm/vm_pageout.h> | #include <vm/vm_pageout.h> | ||||
#include <vm/vm_param.h> | #include <vm/vm_param.h> | ||||
#include <vm/vm_phys.h> | #include <vm/vm_phys.h> | ||||
#include <vm/vm_pagequeue.h> | #include <vm/vm_pagequeue.h> | ||||
#include <vm/vm_map.h> | #include <vm/vm_map.h> | ||||
#include <vm/vm_kern.h> | #include <vm/vm_kern.h> | ||||
▲ Show 20 Lines • Show All 896 Lines • ▼ Show 20 Lines | |||||
zone_drain(uma_zone_t zone) | zone_drain(uma_zone_t zone) | ||||
{ | { | ||||
zone_drain_wait(zone, M_NOWAIT); | zone_drain_wait(zone, M_NOWAIT); | ||||
} | } | ||||
/* | /* | ||||
* Allocate a new slab for a keg. This does not insert the slab onto a list. | * Allocate a new slab for a keg. This does not insert the slab onto a list. | ||||
* If the allocation was successful, the keg lock will be held upon return, | |||||
* otherwise the keg will be left unlocked. | |||||
* | * | ||||
* Arguments: | * Arguments: | ||||
* wait Shall we wait? | * wait Shall we wait? | ||||
* | * | ||||
* Returns: | * Returns: | ||||
* The slab that was allocated or NULL if there is no memory and the | * The slab that was allocated or NULL if there is no memory and the | ||||
* caller specified M_NOWAIT. | * caller specified M_NOWAIT. | ||||
*/ | */ | ||||
static uma_slab_t | static uma_slab_t | ||||
keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int wait) | keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int wait) | ||||
{ | { | ||||
uma_alloc allocf; | uma_alloc allocf; | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
unsigned long size; | unsigned long size; | ||||
uint8_t *mem; | uint8_t *mem; | ||||
uint8_t flags; | uint8_t flags; | ||||
int i; | int i; | ||||
KASSERT(domain >= 0 && domain < vm_ndomains, | KASSERT(domain >= 0 && domain < vm_ndomains, | ||||
("keg_alloc_slab: domain %d out of range", domain)); | ("keg_alloc_slab: domain %d out of range", domain)); | ||||
mtx_assert(&keg->uk_lock, MA_OWNED); | mtx_assert(&keg->uk_lock, MA_OWNED); | ||||
slab = NULL; | |||||
mem = NULL; | |||||
allocf = keg->uk_allocf; | allocf = keg->uk_allocf; | ||||
KEG_UNLOCK(keg); | KEG_UNLOCK(keg); | ||||
size = keg->uk_ppera * PAGE_SIZE; | |||||
slab = NULL; | |||||
mem = NULL; | |||||
if (keg->uk_flags & UMA_ZONE_OFFPAGE) { | if (keg->uk_flags & UMA_ZONE_OFFPAGE) { | ||||
slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, wait); | slab = zone_alloc_item(keg->uk_slabzone, NULL, domain, wait); | ||||
if (slab == NULL) | if (slab == NULL) | ||||
goto out; | goto out; | ||||
} | } | ||||
/* | /* | ||||
* This reproduces the old vm_zone behavior of zero filling pages the | * This reproduces the old vm_zone behavior of zero filling pages the | ||||
* first time they are added to a zone. | * first time they are added to a zone. | ||||
* | * | ||||
* Malloced items are zeroed in uma_zalloc. | * Malloced items are zeroed in uma_zalloc. | ||||
*/ | */ | ||||
if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) | if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) | ||||
wait |= M_ZERO; | wait |= M_ZERO; | ||||
else | else | ||||
wait &= ~M_ZERO; | wait &= ~M_ZERO; | ||||
if (keg->uk_flags & UMA_ZONE_NODUMP) | if (keg->uk_flags & UMA_ZONE_NODUMP) | ||||
wait |= M_NODUMP; | wait |= M_NODUMP; | ||||
/* zone is passed for legacy reasons. */ | /* zone is passed for legacy reasons. */ | ||||
size = keg->uk_ppera * PAGE_SIZE; | |||||
mem = allocf(zone, size, domain, &flags, wait); | mem = allocf(zone, size, domain, &flags, wait); | ||||
if (mem == NULL) { | if (mem == NULL) { | ||||
if (keg->uk_flags & UMA_ZONE_OFFPAGE) | if (keg->uk_flags & UMA_ZONE_OFFPAGE) | ||||
zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); | zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); | ||||
slab = NULL; | slab = NULL; | ||||
goto out; | goto out; | ||||
} | } | ||||
uma_total_inc(size); | uma_total_inc(size); | ||||
Show All 22 Lines | for (i = 0; i < keg->uk_ipers; i++) | ||||
keg->uk_size, wait) != 0) | keg->uk_size, wait) != 0) | ||||
break; | break; | ||||
if (i != keg->uk_ipers) { | if (i != keg->uk_ipers) { | ||||
keg_free_slab(keg, slab, i); | keg_free_slab(keg, slab, i); | ||||
slab = NULL; | slab = NULL; | ||||
goto out; | goto out; | ||||
} | } | ||||
} | } | ||||
out: | |||||
KEG_LOCK(keg); | KEG_LOCK(keg); | ||||
CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)", | CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)", | ||||
slab, keg->uk_name, keg); | slab, keg->uk_name, keg); | ||||
if (slab != NULL) { | |||||
if (keg->uk_flags & UMA_ZONE_HASH) | if (keg->uk_flags & UMA_ZONE_HASH) | ||||
UMA_HASH_INSERT(&keg->uk_hash, slab, mem); | UMA_HASH_INSERT(&keg->uk_hash, slab, mem); | ||||
keg->uk_pages += keg->uk_ppera; | keg->uk_pages += keg->uk_ppera; | ||||
keg->uk_free += keg->uk_ipers; | keg->uk_free += keg->uk_ipers; | ||||
} | |||||
out: | |||||
return (slab); | return (slab); | ||||
} | } | ||||
/* | /* | ||||
* This function is intended to be used early on in place of page_alloc() so | * This function is intended to be used early on in place of page_alloc() so | ||||
* that we may use the boot time page cache to satisfy allocations before | * that we may use the boot time page cache to satisfy allocations before | ||||
* the VM is ready. | * the VM is ready. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 450 Lines • ▼ Show 20 Lines | keg_ctor(void *mem, int size, void *udata, int flags) | ||||
uma_keg_t keg = mem; | uma_keg_t keg = mem; | ||||
uma_zone_t zone; | uma_zone_t zone; | ||||
bzero(keg, size); | bzero(keg, size); | ||||
keg->uk_size = arg->size; | keg->uk_size = arg->size; | ||||
keg->uk_init = arg->uminit; | keg->uk_init = arg->uminit; | ||||
keg->uk_fini = arg->fini; | keg->uk_fini = arg->fini; | ||||
keg->uk_align = arg->align; | keg->uk_align = arg->align; | ||||
keg->uk_cursor = 0; | |||||
keg->uk_free = 0; | keg->uk_free = 0; | ||||
keg->uk_reserve = 0; | keg->uk_reserve = 0; | ||||
keg->uk_pages = 0; | keg->uk_pages = 0; | ||||
keg->uk_flags = arg->flags; | keg->uk_flags = arg->flags; | ||||
keg->uk_slabzone = NULL; | keg->uk_slabzone = NULL; | ||||
/* | /* | ||||
* We use a global round-robin policy by default. Zones with | |||||
* UMA_ZONE_NUMA set will use first-touch instead, in which case the | |||||
* iterator is never run. | |||||
*/ | |||||
keg->uk_dr.dr_policy = DOMAINSET_RR(); | |||||
keg->uk_dr.dr_iter = 0; | |||||
/* | |||||
* The master zone is passed to us at keg-creation time. | * The master zone is passed to us at keg-creation time. | ||||
*/ | */ | ||||
zone = arg->zone; | zone = arg->zone; | ||||
keg->uk_name = zone->uz_name; | keg->uk_name = zone->uz_name; | ||||
if (arg->flags & UMA_ZONE_VM) | if (arg->flags & UMA_ZONE_VM) | ||||
keg->uk_flags |= UMA_ZFLAG_CACHEONLY; | keg->uk_flags |= UMA_ZFLAG_CACHEONLY; | ||||
▲ Show 20 Lines • Show All 1,024 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Find a slab with some space. Prefer slabs that are partially used over those | * Find a slab with some space. Prefer slabs that are partially used over those | ||||
* that are totally full. This helps to reduce fragmentation. | * that are totally full. This helps to reduce fragmentation. | ||||
* | * | ||||
* If 'rr' is 1, search all domains starting from 'domain'. Otherwise check | * If 'rr' is 1, search all domains starting from 'domain'. Otherwise check | ||||
* only 'domain'. | * only 'domain'. | ||||
*/ | */ | ||||
static uma_slab_t | static uma_slab_t | ||||
keg_first_slab(uma_keg_t keg, int domain, int rr) | keg_first_slab(uma_keg_t keg, int domain, bool rr) | ||||
{ | { | ||||
uma_domain_t dom; | uma_domain_t dom; | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
int start; | int start; | ||||
KASSERT(domain >= 0 && domain < vm_ndomains, | KASSERT(domain >= 0 && domain < vm_ndomains, | ||||
("keg_first_slab: domain %d out of range", domain)); | ("keg_first_slab: domain %d out of range", domain)); | ||||
Show All 12 Lines | do { | ||||
if (rr) | if (rr) | ||||
domain = (domain + 1) % vm_ndomains; | domain = (domain + 1) % vm_ndomains; | ||||
} while (domain != start); | } while (domain != start); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
static uma_slab_t | static uma_slab_t | ||||
keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, int flags) | keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags) | ||||
{ | { | ||||
uint32_t reserve; | |||||
mtx_assert(&keg->uk_lock, MA_OWNED); | |||||
reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; | |||||
if (keg->uk_free <= reserve) | |||||
return (NULL); | |||||
return (keg_first_slab(keg, domain, rr)); | |||||
} | |||||
static uma_slab_t | |||||
keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) | |||||
{ | |||||
struct vm_domainset_iter di; | |||||
uma_domain_t dom; | uma_domain_t dom; | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
int allocflags, domain, reserve, rr, start; | int aflags, domain; | ||||
bool rr; | |||||
restart: | |||||
mtx_assert(&keg->uk_lock, MA_OWNED); | mtx_assert(&keg->uk_lock, MA_OWNED); | ||||
slab = NULL; | |||||
reserve = 0; | |||||
allocflags = flags; | |||||
if ((flags & M_USE_RESERVE) == 0) | |||||
reserve = keg->uk_reserve; | |||||
/* | /* | ||||
* Round-robin for non first-touch zones when there is more than one | * Use the keg's policy if upper layers haven't already specified a | ||||
* domain. | * domain (as happens with first-touch zones). | ||||
* | |||||
* To avoid races we run the iterator with the keg lock held, but that | |||||
* means that we cannot allow the vm_domainset layer to sleep. Thus, | |||||
* clear M_WAITOK and handle low memory conditions locally. | |||||
*/ | */ | ||||
if (vm_ndomains == 1) | |||||
rdomain = 0; | |||||
rr = rdomain == UMA_ANYDOMAIN; | rr = rdomain == UMA_ANYDOMAIN; | ||||
if (rr) { | if (rr) { | ||||
start = keg->uk_cursor; | aflags = (flags & ~M_WAITOK) | M_NOWAIT; | ||||
do { | vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, | ||||
keg->uk_cursor = (keg->uk_cursor + 1) % vm_ndomains; | &aflags); | ||||
domain = keg->uk_cursor; | } else { | ||||
} while (VM_DOMAIN_EMPTY(domain) && domain != start); | aflags = flags; | ||||
domain = start = keg->uk_cursor; | domain = rdomain; | ||||
/* Only block on the second pass. */ | } | ||||
if ((flags & (M_WAITOK | M_NOVM)) == M_WAITOK) | |||||
allocflags = (allocflags & ~M_WAITOK) | M_NOWAIT; | |||||
} else | |||||
domain = start = rdomain; | |||||
again: | for (;;) { | ||||
do { | slab = keg_fetch_free_slab(keg, domain, rr, flags); | ||||
if (keg->uk_free > reserve && | if (slab != NULL) { | ||||
(slab = keg_first_slab(keg, domain, rr)) != NULL) { | |||||
MPASS(slab->us_keg == keg); | MPASS(slab->us_keg == keg); | ||||
return (slab); | return (slab); | ||||
} | } | ||||
/* | /* | ||||
* M_NOVM means don't ask at all! | * M_NOVM means don't ask at all! | ||||
*/ | */ | ||||
if (flags & M_NOVM) | if (flags & M_NOVM) | ||||
Show All 11 Lines | if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) { | ||||
zone_maxaction(zone); | zone_maxaction(zone); | ||||
} | } | ||||
if (flags & M_NOWAIT) | if (flags & M_NOWAIT) | ||||
return (NULL); | return (NULL); | ||||
zone->uz_sleeps++; | zone->uz_sleeps++; | ||||
msleep(keg, &keg->uk_lock, PVM, "keglimit", 0); | msleep(keg, &keg->uk_lock, PVM, "keglimit", 0); | ||||
continue; | continue; | ||||
} | } | ||||
slab = keg_alloc_slab(keg, zone, domain, allocflags); | slab = keg_alloc_slab(keg, zone, domain, aflags); | ||||
/* | /* | ||||
* If we got a slab here it's safe to mark it partially used | * If we got a slab here it's safe to mark it partially used | ||||
* and return. We assume that the caller is going to remove | * and return. We assume that the caller is going to remove | ||||
* at least one item. | * at least one item. | ||||
*/ | */ | ||||
if (slab) { | if (slab) { | ||||
MPASS(slab->us_keg == keg); | MPASS(slab->us_keg == keg); | ||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | ||||
return (slab); | return (slab); | ||||
} | } | ||||
if (rr) { | KEG_LOCK(keg); | ||||
do { | if (rr && vm_domainset_iter_policy(&di, &domain) != 0) { | ||||
domain = (domain + 1) % vm_ndomains; | if ((flags & M_WAITOK) != 0) { | ||||
} while (VM_DOMAIN_EMPTY(domain) && domain != start); | KEG_UNLOCK(keg); | ||||
vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); | |||||
KEG_LOCK(keg); | |||||
goto restart; | |||||
} | } | ||||
} while (domain != start); | break; | ||||
/* Retry domain scan with blocking. */ | |||||
if (allocflags != flags) { | |||||
allocflags = flags; | |||||
goto again; | |||||
} | } | ||||
} | |||||
/* | /* | ||||
* We might not have been able to get a slab but another cpu | * We might not have been able to get a slab but another cpu | ||||
* could have while we were unlocked. Check again before we | * could have while we were unlocked. Check again before we | ||||
* fail. | * fail. | ||||
*/ | */ | ||||
if (keg->uk_free > reserve && | if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) { | ||||
(slab = keg_first_slab(keg, domain, rr)) != NULL) { | |||||
MPASS(slab->us_keg == keg); | MPASS(slab->us_keg == keg); | ||||
return (slab); | return (slab); | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
static uma_slab_t | static uma_slab_t | ||||
zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int domain, int flags) | zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int domain, int flags) | ||||
▲ Show 20 Lines • Show All 858 Lines • ▼ Show 20 Lines | uma_prealloc(uma_zone_t zone, int items) | ||||
keg = zone_first_keg(zone); | keg = zone_first_keg(zone); | ||||
if (keg == NULL) | if (keg == NULL) | ||||
return; | return; | ||||
KEG_LOCK(keg); | KEG_LOCK(keg); | ||||
slabs = items / keg->uk_ipers; | slabs = items / keg->uk_ipers; | ||||
domain = 0; | domain = 0; | ||||
if (slabs * keg->uk_ipers < items) | if (slabs * keg->uk_ipers < items) | ||||
slabs++; | slabs++; | ||||
while (slabs > 0) { | while (slabs-- > 0) { | ||||
slab = keg_alloc_slab(keg, zone, domain, M_WAITOK); | slab = keg_alloc_slab(keg, zone, domain, M_WAITOK); | ||||
if (slab == NULL) | if (slab == NULL) | ||||
break; | return; | ||||
MPASS(slab->us_keg == keg); | MPASS(slab->us_keg == keg); | ||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); | ||||
slabs--; | |||||
do { | do { | ||||
domain = (domain + 1) % vm_ndomains; | domain = (domain + 1) % vm_ndomains; | ||||
} while (VM_DOMAIN_EMPTY(domain)); | } while (VM_DOMAIN_EMPTY(domain)); | ||||
} | } | ||||
KEG_UNLOCK(keg); | KEG_UNLOCK(keg); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
▲ Show 20 Lines • Show All 626 Lines • Show Last 20 Lines |