Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
Show First 20 Lines • Show All 95 Lines • ▼ Show 20 Lines | |||||
#include <vm/uma_dbg.h> | #include <vm/uma_dbg.h> | ||||
#include <ddb/ddb.h> | #include <ddb/ddb.h> | ||||
#ifdef DEBUG_MEMGUARD | #ifdef DEBUG_MEMGUARD | ||||
#include <vm/memguard.h> | #include <vm/memguard.h> | ||||
#endif | #endif | ||||
#include <machine/md_var.h> | |||||
/* | /* | ||||
* This is the zone and keg from which all zones are spawned. | * This is the zone and keg from which all zones are spawned. | ||||
*/ | */ | ||||
static uma_zone_t kegs; | static uma_zone_t kegs; | ||||
static uma_zone_t zones; | static uma_zone_t zones; | ||||
/* | /* | ||||
* These are the two zones from which all offpage uma_slab_ts are allocated. | * These are the two zones from which all offpage uma_slab_ts are allocated. | ||||
Show All 34 Lines | |||||
/* Linked list of all cache-only zones in the system */ | /* Linked list of all cache-only zones in the system */ | ||||
static LIST_HEAD(,uma_zone) uma_cachezones = | static LIST_HEAD(,uma_zone) uma_cachezones = | ||||
LIST_HEAD_INITIALIZER(uma_cachezones); | LIST_HEAD_INITIALIZER(uma_cachezones); | ||||
/* This RW lock protects the keg list */ | /* This RW lock protects the keg list */ | ||||
static struct rwlock_padalign __exclusive_cache_line uma_rwlock; | static struct rwlock_padalign __exclusive_cache_line uma_rwlock; | ||||
/* | /* | ||||
* Pointer and counter to pool of pages, that is preallocated at | * First available virual address for boot time allocations. | ||||
* startup to bootstrap UMA. | |||||
*/ | */ | ||||
static char *bootmem; | static vm_offset_t bootstart; | ||||
static int boot_pages; | static vm_offset_t bootmem; | ||||
static struct sx uma_reclaim_lock; | static struct sx uma_reclaim_lock; | ||||
/* | /* | ||||
* kmem soft limit, initialized by uma_set_limit(). Ensure that early | * kmem soft limit, initialized by uma_set_limit(). Ensure that early | ||||
* allocations don't trigger a wakeup of the reclaim thread. | * allocations don't trigger a wakeup of the reclaim thread. | ||||
*/ | */ | ||||
unsigned long uma_kmem_limit = LONG_MAX; | unsigned long uma_kmem_limit = LONG_MAX; | ||||
SYSCTL_ULONG(_vm, OID_AUTO, uma_kmem_limit, CTLFLAG_RD, &uma_kmem_limit, 0, | SYSCTL_ULONG(_vm, OID_AUTO, uma_kmem_limit, CTLFLAG_RD, &uma_kmem_limit, 0, | ||||
"UMA kernel memory soft limit"); | "UMA kernel memory soft limit"); | ||||
unsigned long uma_kmem_total; | unsigned long uma_kmem_total; | ||||
SYSCTL_ULONG(_vm, OID_AUTO, uma_kmem_total, CTLFLAG_RD, &uma_kmem_total, 0, | SYSCTL_ULONG(_vm, OID_AUTO, uma_kmem_total, CTLFLAG_RD, &uma_kmem_total, 0, | ||||
"UMA kernel memory usage"); | "UMA kernel memory usage"); | ||||
/* Is the VM done starting up? */ | /* Is the VM done starting up? */ | ||||
static enum { | static enum { | ||||
BOOT_COLD, | BOOT_COLD, | ||||
BOOT_STRAPPED, | BOOT_KVA, | ||||
BOOT_PAGEALLOC, | |||||
BOOT_BUCKETS, | |||||
BOOT_RUNNING, | BOOT_RUNNING, | ||||
BOOT_SHUTDOWN, | BOOT_SHUTDOWN, | ||||
} booted = BOOT_COLD; | } booted = BOOT_COLD; | ||||
/* | /* | ||||
* This is the handle used to schedule events that need to happen | * This is the handle used to schedule events that need to happen | ||||
* outside of the allocation fast path. | * outside of the allocation fast path. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | enum zfreeskip { | ||||
SKIP_NONE = 0, | SKIP_NONE = 0, | ||||
SKIP_CNT = 0x00000001, | SKIP_CNT = 0x00000001, | ||||
SKIP_DTOR = 0x00010000, | SKIP_DTOR = 0x00010000, | ||||
SKIP_FINI = 0x00020000, | SKIP_FINI = 0x00020000, | ||||
}; | }; | ||||
/* Prototypes.. */ | /* Prototypes.. */ | ||||
int uma_startup_count(int); | void uma_startup1(vm_offset_t); | ||||
void uma_startup(void *, int); | |||||
void uma_startup1(void); | |||||
void uma_startup2(void); | void uma_startup2(void); | ||||
static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void page_free(void *, vm_size_t, uint8_t); | static void page_free(void *, vm_size_t, uint8_t); | ||||
static void pcpu_page_free(void *, vm_size_t, uint8_t); | static void pcpu_page_free(void *, vm_size_t, uint8_t); | ||||
static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); | static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); | ||||
static void cache_drain(uma_zone_t); | static void cache_drain(uma_zone_t); | ||||
static void bucket_drain(uma_zone_t, uma_bucket_t); | static void bucket_drain(uma_zone_t, uma_bucket_t); | ||||
static void bucket_cache_reclaim(uma_zone_t zone, bool); | static void bucket_cache_reclaim(uma_zone_t zone, bool); | ||||
static int keg_ctor(void *, int, void *, int); | static int keg_ctor(void *, int, void *, int); | ||||
static void keg_dtor(void *, int, void *); | static void keg_dtor(void *, int, void *); | ||||
static int zone_ctor(void *, int, void *, int); | static int zone_ctor(void *, int, void *, int); | ||||
static void zone_dtor(void *, int, void *); | static void zone_dtor(void *, int, void *); | ||||
static int zero_init(void *, int, int); | static int zero_init(void *, int, int); | ||||
static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); | static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); | ||||
static void zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *), void *); | |||||
static void zone_timeout(uma_zone_t zone, void *); | static void zone_timeout(uma_zone_t zone, void *); | ||||
static int hash_alloc(struct uma_hash *, u_int); | static int hash_alloc(struct uma_hash *, u_int); | ||||
static int hash_expand(struct uma_hash *, struct uma_hash *); | static int hash_expand(struct uma_hash *, struct uma_hash *); | ||||
static void hash_free(struct uma_hash *hash); | static void hash_free(struct uma_hash *hash); | ||||
static void uma_timeout(void *); | static void uma_timeout(void *); | ||||
static void uma_startup3(void); | static void uma_startup3(void); | ||||
static void uma_shutdown(void); | static void uma_shutdown(void); | ||||
static void *zone_alloc_item(uma_zone_t, void *, int, int); | static void *zone_alloc_item(uma_zone_t, void *, int, int); | ||||
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* This routine checks to see whether or not it's safe to enable buckets. | * This routine checks to see whether or not it's safe to enable buckets. | ||||
*/ | */ | ||||
static void | static void | ||||
bucket_enable(void) | bucket_enable(void) | ||||
{ | { | ||||
KASSERT(booted >= BOOT_BUCKETS, ("Bucket enable before init")); | KASSERT(booted >= BOOT_KVA, ("Bucket enable before init")); | ||||
bucketdisable = vm_page_count_min(); | bucketdisable = vm_page_count_min(); | ||||
} | } | ||||
/* | /* | ||||
* Initialize bucket_zones, the array of zones of buckets of various sizes. | * Initialize bucket_zones, the array of zones of buckets of various sizes. | ||||
* | * | ||||
* For each zone, calculate the memory required for each bucket, consisting | * For each zone, calculate the memory required for each bucket, consisting | ||||
* of the header and an array of pointers. | * of the header and an array of pointers. | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | |||||
static uma_bucket_t | static uma_bucket_t | ||||
bucket_alloc(uma_zone_t zone, void *udata, int flags) | bucket_alloc(uma_zone_t zone, void *udata, int flags) | ||||
{ | { | ||||
struct uma_bucket_zone *ubz; | struct uma_bucket_zone *ubz; | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
/* | /* | ||||
* This is to stop us from allocating per cpu buckets while we're | * Don't allocate buckets in low memory situations. | ||||
* running out of vm.boot_pages. Otherwise, we would exhaust the | |||||
* boot pages. This also prevents us from allocating buckets in | |||||
* low memory situations. | |||||
*/ | */ | ||||
if (bucketdisable) | if (bucketdisable) | ||||
return (NULL); | return (NULL); | ||||
/* | /* | ||||
* To limit bucket recursion we store the original zone flags | * To limit bucket recursion we store the original zone flags | ||||
* in a cookie passed via zalloc_arg/zfree_arg. This allows the | * in a cookie passed via zalloc_arg/zfree_arg. This allows the | ||||
* NOVM flag to persist even through deep recursions. We also | * NOVM flag to persist even through deep recursions. We also | ||||
* store ZFLAG_BUCKET once we have recursed attempting to allocate | * store ZFLAG_BUCKET once we have recursed attempting to allocate | ||||
* a bucket for a bucket zone so we do not allow infinite bucket | * a bucket for a bucket zone so we do not allow infinite bucket | ||||
* recursion. This cookie will even persist to frees of unused | * recursion. This cookie will even persist to frees of unused | ||||
* buckets via the allocation path or bucket allocations in the | * buckets via the allocation path or bucket allocations in the | ||||
▲ Show 20 Lines • Show All 747 Lines • ▼ Show 20 Lines | keg_drain(uma_keg_t keg) | ||||
for (i = 0; i < vm_ndomains; i++) { | for (i = 0; i < vm_ndomains; i++) { | ||||
CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", | CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", | ||||
keg->uk_name, keg, i, dom->ud_free); | keg->uk_name, keg, i, dom->ud_free); | ||||
n = 0; | n = 0; | ||||
dom = &keg->uk_domain[i]; | dom = &keg->uk_domain[i]; | ||||
KEG_LOCK(keg, i); | KEG_LOCK(keg, i); | ||||
LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) { | LIST_FOREACH_SAFE(slab, &dom->ud_free_slab, us_link, tmp) { | ||||
/* We have nowhere to free these to. */ | |||||
if (slab->us_flags & UMA_SLAB_BOOT) | |||||
continue; | |||||
if (keg->uk_flags & UMA_ZFLAG_HASH) | if (keg->uk_flags & UMA_ZFLAG_HASH) | ||||
UMA_HASH_REMOVE(&keg->uk_hash, slab); | UMA_HASH_REMOVE(&keg->uk_hash, slab); | ||||
n++; | n++; | ||||
LIST_REMOVE(slab, us_link); | LIST_REMOVE(slab, us_link); | ||||
LIST_INSERT_HEAD(&freeslabs, slab, us_link); | LIST_INSERT_HEAD(&freeslabs, slab, us_link); | ||||
} | } | ||||
dom->ud_pages -= n * keg->uk_ppera; | dom->ud_pages -= n * keg->uk_ppera; | ||||
dom->ud_free -= n * keg->uk_ipers; | dom->ud_free -= n * keg->uk_ipers; | ||||
▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | |||||
* This function is intended to be used early on in place of page_alloc() so | * This function is intended to be used early on in place of page_alloc() so | ||||
* that we may use the boot time page cache to satisfy allocations before | * that we may use the boot time page cache to satisfy allocations before | ||||
* the VM is ready. | * the VM is ready. | ||||
*/ | */ | ||||
static void * | static void * | ||||
startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, | startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, | ||||
int wait) | int wait) | ||||
{ | { | ||||
uma_keg_t keg; | vm_paddr_t pa; | ||||
vm_page_t m; | |||||
void *mem; | void *mem; | ||||
int pages; | int pages; | ||||
int i; | |||||
keg = zone->uz_keg; | |||||
/* | |||||
* If we are in BOOT_BUCKETS or higher, than switch to real | |||||
* allocator. Zones with page sized slabs switch at BOOT_PAGEALLOC. | |||||
*/ | |||||
switch (booted) { | |||||
case BOOT_COLD: | |||||
case BOOT_STRAPPED: | |||||
break; | |||||
case BOOT_PAGEALLOC: | |||||
if (keg->uk_ppera > 1) | |||||
break; | |||||
default: | |||||
#ifdef UMA_MD_SMALL_ALLOC | |||||
keg->uk_allocf = (keg->uk_ppera > 1) ? | |||||
page_alloc : uma_small_alloc; | |||||
#else | |||||
keg->uk_allocf = page_alloc; | |||||
#endif | |||||
return keg->uk_allocf(zone, bytes, domain, pflag, wait); | |||||
} | |||||
/* | |||||
* Check our small startup cache to see if it has pages remaining. | |||||
*/ | |||||
pages = howmany(bytes, PAGE_SIZE); | pages = howmany(bytes, PAGE_SIZE); | ||||
KASSERT(pages > 0, ("%s can't reserve 0 pages", __func__)); | KASSERT(pages > 0, ("%s can't reserve 0 pages", __func__)); | ||||
if (pages > boot_pages) | |||||
panic("UMA zone \"%s\": Increase vm.boot_pages", zone->uz_name); | |||||
#ifdef DIAGNOSTIC | |||||
printf("%s from \"%s\", %d boot pages left\n", __func__, zone->uz_name, | |||||
boot_pages); | |||||
#endif | |||||
mem = bootmem; | |||||
boot_pages -= pages; | |||||
bootmem += pages * PAGE_SIZE; | |||||
*pflag = UMA_SLAB_BOOT; | *pflag = UMA_SLAB_BOOT; | ||||
m = vm_page_alloc_contig_domain(NULL, 0, domain, | |||||
malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED, pages, | |||||
(vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT); | |||||
if (m == NULL) | |||||
return (NULL); | |||||
pa = VM_PAGE_TO_PHYS(m); | |||||
for (i = 0; i < pages; i++, pa += PAGE_SIZE) { | |||||
#if defined(__aarch64__) || defined(__amd64__) || defined(__mips__) || \ | |||||
defined(__riscv) || defined(__powerpc64__) | |||||
if ((wait & M_NODUMP) == 0) | |||||
dump_add_page(pa); | |||||
#endif | |||||
} | |||||
/* Allocate KVA and indirectly advance bootmem. */ | |||||
mem = (void *)pmap_map(&bootmem, m->phys_addr, | |||||
m->phys_addr + (pages * PAGE_SIZE), VM_PROT_READ | VM_PROT_WRITE); | |||||
if ((wait & M_ZERO) != 0) | |||||
bzero(mem, pages * PAGE_SIZE); | |||||
return (mem); | return (mem); | ||||
} | } | ||||
static void | |||||
startup_free(void *mem, vm_size_t bytes) | |||||
{ | |||||
vm_offset_t va; | |||||
vm_page_t m; | |||||
va = (vm_offset_t)mem; | |||||
m = PHYS_TO_VM_PAGE(pmap_kextract(va)); | |||||
pmap_remove(kernel_pmap, va, va + bytes); | |||||
for (; bytes != 0; bytes -= PAGE_SIZE, m++) { | |||||
#if defined(__aarch64__) || defined(__amd64__) || defined(__mips__) || \ | |||||
defined(__riscv) || defined(__powerpc64__) | |||||
dump_drop_page(VM_PAGE_TO_PHYS(m)); | |||||
#endif | |||||
vm_page_unwire_noq(m); | |||||
vm_page_free(m); | |||||
} | |||||
} | |||||
/* | /* | ||||
* Allocates a number of pages from the system | * Allocates a number of pages from the system | ||||
* | * | ||||
* Arguments: | * Arguments: | ||||
* bytes The number of bytes requested | * bytes The number of bytes requested | ||||
* wait Shall we wait? | * wait Shall we wait? | ||||
* | * | ||||
* Returns: | * Returns: | ||||
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* Returns: | * Returns: | ||||
* Nothing | * Nothing | ||||
*/ | */ | ||||
static void | static void | ||||
page_free(void *mem, vm_size_t size, uint8_t flags) | page_free(void *mem, vm_size_t size, uint8_t flags) | ||||
{ | { | ||||
if ((flags & UMA_SLAB_BOOT) != 0) { | |||||
startup_free(mem, size); | |||||
return; | |||||
} | |||||
if ((flags & UMA_SLAB_KERNEL) == 0) | if ((flags & UMA_SLAB_KERNEL) == 0) | ||||
panic("UMA: page_free used with invalid flags %x", flags); | panic("UMA: page_free used with invalid flags %x", flags); | ||||
kmem_free((vm_offset_t)mem, size); | kmem_free((vm_offset_t)mem, size); | ||||
} | } | ||||
/* | /* | ||||
* Frees pcpu zone allocations | * Frees pcpu zone allocations | ||||
▲ Show 20 Lines • Show All 334 Lines • ▼ Show 20 Lines | #ifdef NUMA | ||||
else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) | else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) | ||||
keg->uk_flags |= UMA_ZONE_ROUNDROBIN; | keg->uk_flags |= UMA_ZONE_ROUNDROBIN; | ||||
#endif | #endif | ||||
/* | /* | ||||
* If we haven't booted yet we need allocations to go through the | * If we haven't booted yet we need allocations to go through the | ||||
* startup cache until the vm is ready. | * startup cache until the vm is ready. | ||||
*/ | */ | ||||
if (booted < BOOT_PAGEALLOC) | |||||
keg->uk_allocf = startup_alloc; | |||||
#ifdef UMA_MD_SMALL_ALLOC | #ifdef UMA_MD_SMALL_ALLOC | ||||
else if (keg->uk_ppera == 1) | if (keg->uk_ppera == 1) | ||||
keg->uk_allocf = uma_small_alloc; | keg->uk_allocf = uma_small_alloc; | ||||
else | |||||
#endif | #endif | ||||
if (booted < BOOT_KVA) | |||||
keg->uk_allocf = startup_alloc; | |||||
else if (keg->uk_flags & UMA_ZONE_PCPU) | else if (keg->uk_flags & UMA_ZONE_PCPU) | ||||
keg->uk_allocf = pcpu_page_alloc; | keg->uk_allocf = pcpu_page_alloc; | ||||
else | else | ||||
keg->uk_allocf = page_alloc; | keg->uk_allocf = page_alloc; | ||||
#ifdef UMA_MD_SMALL_ALLOC | #ifdef UMA_MD_SMALL_ALLOC | ||||
if (keg->uk_ppera == 1) | if (keg->uk_ppera == 1) | ||||
keg->uk_freef = uma_small_free; | keg->uk_freef = uma_small_free; | ||||
else | else | ||||
Show All 40 Lines | #endif | ||||
rw_wlock(&uma_rwlock); | rw_wlock(&uma_rwlock); | ||||
LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); | LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); | ||||
rw_wunlock(&uma_rwlock); | rw_wunlock(&uma_rwlock); | ||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
zone_kva_available(uma_zone_t zone, void *unused) | |||||
{ | |||||
uma_keg_t keg; | |||||
if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) | |||||
return; | |||||
KEG_GET(zone, keg); | |||||
if (keg->uk_allocf == startup_alloc) | |||||
keg->uk_allocf = page_alloc; | |||||
} | |||||
static void | |||||
zone_alloc_counters(uma_zone_t zone, void *unused) | zone_alloc_counters(uma_zone_t zone, void *unused) | ||||
{ | { | ||||
zone->uz_allocs = counter_u64_alloc(M_WAITOK); | zone->uz_allocs = counter_u64_alloc(M_WAITOK); | ||||
zone->uz_frees = counter_u64_alloc(M_WAITOK); | zone->uz_frees = counter_u64_alloc(M_WAITOK); | ||||
zone->uz_fails = counter_u64_alloc(M_WAITOK); | zone->uz_fails = counter_u64_alloc(M_WAITOK); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 424 Lines • ▼ Show 20 Lines | zone_dtor(void *arg, int size, void *udata) | ||||
counter_u64_free(zone->uz_allocs); | counter_u64_free(zone->uz_allocs); | ||||
counter_u64_free(zone->uz_frees); | counter_u64_free(zone->uz_frees); | ||||
counter_u64_free(zone->uz_fails); | counter_u64_free(zone->uz_fails); | ||||
free(zone->uz_ctlname, M_UMA); | free(zone->uz_ctlname, M_UMA); | ||||
ZONE_LOCK_FINI(zone); | ZONE_LOCK_FINI(zone); | ||||
ZONE_CROSS_LOCK_FINI(zone); | ZONE_CROSS_LOCK_FINI(zone); | ||||
} | } | ||||
/* | |||||
* Traverses every zone in the system and calls a callback | |||||
* | |||||
* Arguments: | |||||
* zfunc A pointer to a function which accepts a zone | |||||
* as an argument. | |||||
* | |||||
* Returns: | |||||
* Nothing | |||||
*/ | |||||
static void | static void | ||||
zone_foreach(void (*zfunc)(uma_zone_t, void *arg), void *arg) | zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *arg), void *arg) | ||||
{ | { | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
uma_zone_t zone; | uma_zone_t zone; | ||||
/* | |||||
* Before BOOT_RUNNING we are guaranteed to be single | |||||
* threaded, so locking isn't needed. Startup functions | |||||
* are allowed to use M_WAITOK. | |||||
*/ | |||||
if (__predict_true(booted >= BOOT_RUNNING)) | |||||
rw_rlock(&uma_rwlock); | |||||
LIST_FOREACH(keg, &uma_kegs, uk_link) { | LIST_FOREACH(keg, &uma_kegs, uk_link) { | ||||
LIST_FOREACH(zone, &keg->uk_zones, uz_link) | LIST_FOREACH(zone, &keg->uk_zones, uz_link) | ||||
zfunc(zone, arg); | zfunc(zone, arg); | ||||
} | } | ||||
LIST_FOREACH(zone, &uma_cachezones, uz_link) | LIST_FOREACH(zone, &uma_cachezones, uz_link) | ||||
zfunc(zone, arg); | zfunc(zone, arg); | ||||
if (__predict_true(booted >= BOOT_RUNNING)) | |||||
rw_runlock(&uma_rwlock); | |||||
} | } | ||||
/* | /* | ||||
* Count how many pages do we need to bootstrap. VM supplies | * Traverses every zone in the system and calls a callback | ||||
* its need in early zones in the argument, we add up our zones, | * | ||||
* which consist of the UMA Slabs, UMA Hash and 9 Bucket zones. The | * Arguments: | ||||
* zone of zones and zone of kegs are accounted separately. | * zfunc A pointer to a function which accepts a zone | ||||
* as an argument. | |||||
* | |||||
* Returns: | |||||
* Nothing | |||||
*/ | */ | ||||
#define UMA_BOOT_ZONES 12 | static void | ||||
static int zsize, ksize; | zone_foreach(void (*zfunc)(uma_zone_t, void *arg), void *arg) | ||||
int | |||||
uma_startup_count(int vm_zones) | |||||
{ | { | ||||
int zones, pages; | |||||
u_int zppera, zipers; | |||||
u_int kppera, kipers; | |||||
size_t space, size; | |||||
ksize = sizeof(struct uma_keg) + | rw_rlock(&uma_rwlock); | ||||
(sizeof(struct uma_domain) * vm_ndomains); | zone_foreach_unlocked(zfunc, arg); | ||||
ksize = roundup(ksize, UMA_SUPER_ALIGN); | rw_runlock(&uma_rwlock); | ||||
zsize = sizeof(struct uma_zone) + | |||||
(sizeof(struct uma_cache) * (mp_maxid + 1)) + | |||||
(sizeof(struct uma_zone_domain) * vm_ndomains); | |||||
zsize = roundup(zsize, UMA_SUPER_ALIGN); | |||||
/* | |||||
* Memory for the zone of kegs and its keg, and for zone | |||||
* of zones. Allocated directly in uma_startup(). | |||||
*/ | |||||
pages = howmany(zsize * 2 + ksize, PAGE_SIZE); | |||||
#ifdef UMA_MD_SMALL_ALLOC | |||||
zones = UMA_BOOT_ZONES; | |||||
#else | |||||
zones = UMA_BOOT_ZONES + vm_zones; | |||||
vm_zones = 0; | |||||
#endif | |||||
size = slab_sizeof(SLAB_MAX_SETSIZE); | |||||
space = slab_space(SLAB_MAX_SETSIZE); | |||||
/* Memory for the rest of startup zones, UMA and VM, ... */ | |||||
if (zsize > space) { | |||||
/* See keg_large_init(). */ | |||||
zppera = howmany(zsize + slab_sizeof(1), PAGE_SIZE); | |||||
zipers = 1; | |||||
zones += vm_zones; | |||||
} else { | |||||
zppera = 1; | |||||
zipers = space / zsize; | |||||
} | } | ||||
pages += howmany(zones, zipers) * zppera; | |||||
/* ... and their kegs. Note that zone of zones allocates a keg! */ | |||||
if (ksize > space) { | |||||
/* See keg_large_init(). */ | |||||
kppera = howmany(ksize + slab_sizeof(1), PAGE_SIZE); | |||||
kipers = 1; | |||||
} else { | |||||
kppera = 1; | |||||
kipers = space / ksize; | |||||
} | |||||
pages += howmany(zones + 1, kipers) * kppera; | |||||
/* | /* | ||||
* Allocate an additional slab for zones and kegs on NUMA | * Initialize the kernel memory allocator. This is done after pages can be | ||||
* systems. The round-robin allocation policy will populate at | * allocated but before general KVA is available. | ||||
* least one slab per-domain. | |||||
*/ | */ | ||||
pages += (vm_ndomains - 1) * (zppera + kppera); | |||||
return (pages); | |||||
} | |||||
void | void | ||||
uma_startup(void *mem, int npages) | uma_startup1(vm_offset_t virtual_avail) | ||||
{ | { | ||||
struct uma_zctor_args args; | struct uma_zctor_args args; | ||||
size_t ksize, zsize, size; | |||||
uma_keg_t masterkeg; | uma_keg_t masterkeg; | ||||
uintptr_t m; | uintptr_t m; | ||||
uint8_t pflag; | |||||
#ifdef DIAGNOSTIC | bootstart = bootmem = virtual_avail; | ||||
printf("Entering %s with %d boot pages configured\n", __func__, npages); | |||||
#endif | |||||
rw_init(&uma_rwlock, "UMA lock"); | rw_init(&uma_rwlock, "UMA lock"); | ||||
sx_init(&uma_reclaim_lock, "umareclaim"); | |||||
/* Use bootpages memory for the zone of zones and zone of kegs. */ | ksize = sizeof(struct uma_keg) + | ||||
m = (uintptr_t)mem; | (sizeof(struct uma_domain) * vm_ndomains); | ||||
ksize = roundup(ksize, UMA_SUPER_ALIGN); | |||||
zsize = sizeof(struct uma_zone) + | |||||
(sizeof(struct uma_cache) * (mp_maxid + 1)) + | |||||
(sizeof(struct uma_zone_domain) * vm_ndomains); | |||||
zsize = roundup(zsize, UMA_SUPER_ALIGN); | |||||
/* Allocate the zone of zones, zone of kegs, and zone of zones keg. */ | |||||
size = (zsize * 2) + ksize; | |||||
m = (uintptr_t)startup_alloc(NULL, size, 0, &pflag, M_NOWAIT | M_ZERO); | |||||
zones = (uma_zone_t)m; | zones = (uma_zone_t)m; | ||||
m += zsize; | m += zsize; | ||||
kegs = (uma_zone_t)m; | kegs = (uma_zone_t)m; | ||||
m += zsize; | m += zsize; | ||||
masterkeg = (uma_keg_t)m; | masterkeg = (uma_keg_t)m; | ||||
m += ksize; | |||||
m = roundup(m, PAGE_SIZE); | |||||
npages -= (m - (uintptr_t)mem) / PAGE_SIZE; | |||||
mem = (void *)m; | |||||
/* "manually" create the initial zone */ | /* "manually" create the initial zone */ | ||||
memset(&args, 0, sizeof(args)); | memset(&args, 0, sizeof(args)); | ||||
args.name = "UMA Kegs"; | args.name = "UMA Kegs"; | ||||
args.size = ksize; | args.size = ksize; | ||||
args.ctor = keg_ctor; | args.ctor = keg_ctor; | ||||
args.dtor = keg_dtor; | args.dtor = keg_dtor; | ||||
args.uminit = zero_init; | args.uminit = zero_init; | ||||
args.fini = NULL; | args.fini = NULL; | ||||
args.keg = masterkeg; | args.keg = masterkeg; | ||||
args.align = UMA_SUPER_ALIGN - 1; | args.align = UMA_SUPER_ALIGN - 1; | ||||
args.flags = UMA_ZFLAG_INTERNAL; | args.flags = UMA_ZFLAG_INTERNAL; | ||||
zone_ctor(kegs, zsize, &args, M_WAITOK); | zone_ctor(kegs, zsize, &args, M_WAITOK); | ||||
bootmem = mem; | |||||
boot_pages = npages; | |||||
args.name = "UMA Zones"; | args.name = "UMA Zones"; | ||||
args.size = zsize; | args.size = zsize; | ||||
args.ctor = zone_ctor; | args.ctor = zone_ctor; | ||||
args.dtor = zone_dtor; | args.dtor = zone_dtor; | ||||
args.uminit = zero_init; | args.uminit = zero_init; | ||||
args.fini = NULL; | args.fini = NULL; | ||||
args.keg = NULL; | args.keg = NULL; | ||||
args.align = UMA_SUPER_ALIGN - 1; | args.align = UMA_SUPER_ALIGN - 1; | ||||
args.flags = UMA_ZFLAG_INTERNAL; | args.flags = UMA_ZFLAG_INTERNAL; | ||||
zone_ctor(zones, zsize, &args, M_WAITOK); | zone_ctor(zones, zsize, &args, M_WAITOK); | ||||
/* Now make zones for slab headers */ | /* Now make zones for slab headers */ | ||||
slabzones[0] = uma_zcreate("UMA Slabs 0", SLABZONE0_SIZE, | slabzones[0] = uma_zcreate("UMA Slabs 0", SLABZONE0_SIZE, | ||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | ||||
slabzones[1] = uma_zcreate("UMA Slabs 1", SLABZONE1_SIZE, | slabzones[1] = uma_zcreate("UMA Slabs 1", SLABZONE1_SIZE, | ||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | ||||
hashzone = uma_zcreate("UMA Hash", | hashzone = uma_zcreate("UMA Hash", | ||||
sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, | sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, | ||||
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | ||||
booted = BOOT_STRAPPED; | bucket_init(); | ||||
} | } | ||||
void | #ifndef UMA_MD_SMALL_ALLOC | ||||
uma_startup1(void) | extern void vm_radix_reserve_kva(void); | ||||
{ | |||||
#ifdef DIAGNOSTIC | |||||
printf("Entering %s with %d boot pages left\n", __func__, boot_pages); | |||||
#endif | #endif | ||||
booted = BOOT_PAGEALLOC; | |||||
} | |||||
/* | |||||
* Advertise the availability of normal kva allocations and switch to | |||||
* the default back-end allocator. Marks the KVA we consumed on startup | |||||
* as used in the map. | |||||
*/ | |||||
void | void | ||||
uma_startup2(void) | uma_startup2(void) | ||||
{ | { | ||||
#ifdef DIAGNOSTIC | if (!PMAP_HAS_DMAP) { | ||||
printf("Entering %s with %d boot pages left\n", __func__, boot_pages); | vm_map_lock(kernel_map); | ||||
(void)vm_map_insert(kernel_map, NULL, 0, bootstart, bootmem, | |||||
VM_PROT_RW, VM_PROT_RW, MAP_NOFAULT); | |||||
vm_map_unlock(kernel_map); | |||||
} | |||||
#ifndef UMA_MD_SMALL_ALLOC | |||||
/* Set up radix zone to use noobj_alloc. */ | |||||
vm_radix_reserve_kva(); | |||||
#endif | #endif | ||||
sx_init(&uma_reclaim_lock, "umareclaim"); | |||||
bucket_init(); | booted = BOOT_KVA; | ||||
booted = BOOT_BUCKETS; | zone_foreach_unlocked(zone_kva_available, NULL); | ||||
bucket_enable(); | bucket_enable(); | ||||
} | } | ||||
/* | |||||
* Finish our initialization steps. | |||||
*/ | |||||
static void | static void | ||||
uma_startup3(void) | uma_startup3(void) | ||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
TUNABLE_INT_FETCH("vm.debug.divisor", &dbg_divisor); | TUNABLE_INT_FETCH("vm.debug.divisor", &dbg_divisor); | ||||
uma_dbg_cnt = counter_u64_alloc(M_WAITOK); | uma_dbg_cnt = counter_u64_alloc(M_WAITOK); | ||||
uma_skip_cnt = counter_u64_alloc(M_WAITOK); | uma_skip_cnt = counter_u64_alloc(M_WAITOK); | ||||
#endif | #endif | ||||
zone_foreach(zone_alloc_counters, NULL); | zone_foreach_unlocked(zone_alloc_counters, NULL); | ||||
zone_foreach(zone_alloc_sysctl, NULL); | zone_foreach_unlocked(zone_alloc_sysctl, NULL); | ||||
callout_init(&uma_callout, 1); | callout_init(&uma_callout, 1); | ||||
callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); | callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); | ||||
booted = BOOT_RUNNING; | booted = BOOT_RUNNING; | ||||
EVENTHANDLER_REGISTER(shutdown_post_sync, uma_shutdown, NULL, | EVENTHANDLER_REGISTER(shutdown_post_sync, uma_shutdown, NULL, | ||||
EVENTHANDLER_PRI_FIRST); | EVENTHANDLER_PRI_FIRST); | ||||
} | } | ||||
Show All 32 Lines | |||||
/* See uma.h */ | /* See uma.h */ | ||||
uma_zone_t | uma_zone_t | ||||
uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, | uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, | ||||
uma_init uminit, uma_fini fini, int align, uint32_t flags) | uma_init uminit, uma_fini fini, int align, uint32_t flags) | ||||
{ | { | ||||
struct uma_zctor_args args; | struct uma_zctor_args args; | ||||
uma_zone_t res; | uma_zone_t res; | ||||
bool locked; | |||||
KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"", | KASSERT(powerof2(align + 1), ("invalid zone alignment %d for \"%s\"", | ||||
align, name)); | align, name)); | ||||
/* This stuff is essential for the zone ctor */ | /* This stuff is essential for the zone ctor */ | ||||
memset(&args, 0, sizeof(args)); | memset(&args, 0, sizeof(args)); | ||||
args.name = name; | args.name = name; | ||||
args.size = size; | args.size = size; | ||||
Show All 15 Lines | if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOTOUCH | | ||||
args.uminit = trash_init; | args.uminit = trash_init; | ||||
args.fini = trash_fini; | args.fini = trash_fini; | ||||
} | } | ||||
#endif | #endif | ||||
args.align = align; | args.align = align; | ||||
args.flags = flags; | args.flags = flags; | ||||
args.keg = NULL; | args.keg = NULL; | ||||
if (booted < BOOT_BUCKETS) { | |||||
locked = false; | |||||
} else { | |||||
sx_slock(&uma_reclaim_lock); | sx_slock(&uma_reclaim_lock); | ||||
locked = true; | |||||
} | |||||
res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); | res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); | ||||
if (locked) | |||||
sx_sunlock(&uma_reclaim_lock); | sx_sunlock(&uma_reclaim_lock); | ||||
return (res); | return (res); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
uma_zone_t | uma_zone_t | ||||
uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, | uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, | ||||
uma_init zinit, uma_fini zfini, uma_zone_t master) | uma_init zinit, uma_fini zfini, uma_zone_t master) | ||||
{ | { | ||||
struct uma_zctor_args args; | struct uma_zctor_args args; | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
uma_zone_t res; | uma_zone_t res; | ||||
bool locked; | |||||
keg = master->uz_keg; | keg = master->uz_keg; | ||||
memset(&args, 0, sizeof(args)); | memset(&args, 0, sizeof(args)); | ||||
args.name = name; | args.name = name; | ||||
args.size = keg->uk_size; | args.size = keg->uk_size; | ||||
args.ctor = ctor; | args.ctor = ctor; | ||||
args.dtor = dtor; | args.dtor = dtor; | ||||
args.uminit = zinit; | args.uminit = zinit; | ||||
args.fini = zfini; | args.fini = zfini; | ||||
args.align = keg->uk_align; | args.align = keg->uk_align; | ||||
args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; | args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; | ||||
args.keg = keg; | args.keg = keg; | ||||
if (booted < BOOT_BUCKETS) { | |||||
locked = false; | |||||
} else { | |||||
sx_slock(&uma_reclaim_lock); | sx_slock(&uma_reclaim_lock); | ||||
locked = true; | |||||
} | |||||
/* XXX Attaches only one keg of potentially many. */ | |||||
res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); | res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); | ||||
if (locked) | |||||
sx_sunlock(&uma_reclaim_lock); | sx_sunlock(&uma_reclaim_lock); | ||||
return (res); | return (res); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
uma_zone_t | uma_zone_t | ||||
uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor, | uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor, | ||||
uma_init zinit, uma_fini zfini, uma_import zimport, | uma_init zinit, uma_fini zfini, uma_import zimport, | ||||
uma_release zrelease, void *arg, int flags) | uma_release zrelease, void *arg, int flags) | ||||
▲ Show 20 Lines • Show All 2,250 Lines • Show Last 20 Lines |