Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/uma_core.c
Show First 20 Lines • Show All 257 Lines • ▼ Show 20 Lines | |||||
void uma_startup1(vm_offset_t); | void uma_startup1(vm_offset_t); | ||||
void uma_startup2(void); | void uma_startup2(void); | ||||
static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | ||||
static void *contig_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); | |||||
static void page_free(void *, vm_size_t, uint8_t); | static void page_free(void *, vm_size_t, uint8_t); | ||||
static void pcpu_page_free(void *, vm_size_t, uint8_t); | static void pcpu_page_free(void *, vm_size_t, uint8_t); | ||||
static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); | static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); | ||||
static void cache_drain(uma_zone_t); | static void cache_drain(uma_zone_t); | ||||
static void bucket_drain(uma_zone_t, uma_bucket_t); | static void bucket_drain(uma_zone_t, uma_bucket_t); | ||||
static void bucket_cache_reclaim(uma_zone_t zone, bool); | static void bucket_cache_reclaim(uma_zone_t zone, bool); | ||||
static int keg_ctor(void *, int, void *, int); | static int keg_ctor(void *, int, void *, int); | ||||
static void keg_dtor(void *, int, void *); | static void keg_dtor(void *, int, void *); | ||||
▲ Show 20 Lines • Show All 1,329 Lines • ▼ Show 20 Lines | TAILQ_FOREACH(p, &alloctail, listq) { | ||||
pmap_qenter(zkva, &p, 1); | pmap_qenter(zkva, &p, 1); | ||||
zkva += PAGE_SIZE; | zkva += PAGE_SIZE; | ||||
} | } | ||||
return ((void *)retkva); | return ((void *)retkva); | ||||
} | } | ||||
/* | /* | ||||
* Allocate physically contiguous pages. | |||||
*/ | |||||
static void * | |||||
contig_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, | |||||
int wait) | |||||
{ | |||||
*pflag = UMA_SLAB_KERNEL; | |||||
return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain), | |||||
bytes, wait, 0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT)); | |||||
} | |||||
/* | |||||
* Frees a number of pages to the system | * Frees a number of pages to the system | ||||
* | * | ||||
* Arguments: | * Arguments: | ||||
* mem A pointer to the memory to be freed | * mem A pointer to the memory to be freed | ||||
* size The size of the memory being freed | * size The size of the memory being freed | ||||
* flags The original p->us_flags field | * flags The original p->us_flags field | ||||
* | * | ||||
* Returns: | * Returns: | ||||
* Nothing | * Nothing | ||||
*/ | */ | ||||
static void | static void | ||||
page_free(void *mem, vm_size_t size, uint8_t flags) | page_free(void *mem, vm_size_t size, uint8_t flags) | ||||
{ | { | ||||
if ((flags & UMA_SLAB_BOOT) != 0) { | if ((flags & UMA_SLAB_BOOT) != 0) { | ||||
startup_free(mem, size); | startup_free(mem, size); | ||||
return; | return; | ||||
} | } | ||||
if ((flags & UMA_SLAB_KERNEL) == 0) | KASSERT((flags & UMA_SLAB_KERNEL) != 0, | ||||
panic("UMA: page_free used with invalid flags %x", flags); | ("UMA: page_free used with invalid flags %x", flags)); | ||||
rlibby: Also gratuitous, but related. | |||||
kmem_free((vm_offset_t)mem, size); | kmem_free((vm_offset_t)mem, size); | ||||
} | } | ||||
/* | /* | ||||
* Frees pcpu zone allocations | * Frees pcpu zone allocations | ||||
* | * | ||||
* Arguments: | * Arguments: | ||||
* mem A pointer to the memory to be freed | * mem A pointer to the memory to be freed | ||||
▲ Show 20 Lines • Show All 306 Lines • ▼ Show 20 Lines | if (arg->flags & UMA_ZONE_VM) | ||||
keg->uk_flags |= UMA_ZFLAG_CACHEONLY; | keg->uk_flags |= UMA_ZFLAG_CACHEONLY; | ||||
if (arg->flags & UMA_ZONE_ZINIT) | if (arg->flags & UMA_ZONE_ZINIT) | ||||
keg->uk_init = zero_init; | keg->uk_init = zero_init; | ||||
if (arg->flags & UMA_ZONE_MALLOC) | if (arg->flags & UMA_ZONE_MALLOC) | ||||
keg->uk_flags |= UMA_ZFLAG_VTOSLAB; | keg->uk_flags |= UMA_ZFLAG_VTOSLAB; | ||||
KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || booted >= BOOT_KVA, | |||||
("Early boot PCPU zones not implemented, keg %s flags 0x%#b", | |||||
keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS)); | |||||
#ifndef SMP | #ifndef SMP | ||||
keg->uk_flags &= ~UMA_ZONE_PCPU; | keg->uk_flags &= ~UMA_ZONE_PCPU; | ||||
#endif | #endif | ||||
keg_layout(keg); | keg_layout(keg); | ||||
/* | /* | ||||
* Use a first-touch NUMA policy for all kegs that pmap_extract() | * Use a first-touch NUMA policy for all kegs that pmap_extract() | ||||
Show All 18 Lines | #ifdef UMA_MD_SMALL_ALLOC | ||||
if (keg->uk_ppera == 1) | if (keg->uk_ppera == 1) | ||||
keg->uk_allocf = uma_small_alloc; | keg->uk_allocf = uma_small_alloc; | ||||
else | else | ||||
#endif | #endif | ||||
if (booted < BOOT_KVA) | if (booted < BOOT_KVA) | ||||
keg->uk_allocf = startup_alloc; | keg->uk_allocf = startup_alloc; | ||||
else if (keg->uk_flags & UMA_ZONE_PCPU) | else if (keg->uk_flags & UMA_ZONE_PCPU) | ||||
keg->uk_allocf = pcpu_page_alloc; | keg->uk_allocf = pcpu_page_alloc; | ||||
else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1) | |||||
keg->uk_allocf = contig_alloc; | |||||
else | else | ||||
keg->uk_allocf = page_alloc; | keg->uk_allocf = page_alloc; | ||||
#ifdef UMA_MD_SMALL_ALLOC | #ifdef UMA_MD_SMALL_ALLOC | ||||
if (keg->uk_ppera == 1) | if (keg->uk_ppera == 1) | ||||
keg->uk_freef = uma_small_free; | keg->uk_freef = uma_small_free; | ||||
else | else | ||||
#endif | #endif | ||||
if (keg->uk_flags & UMA_ZONE_PCPU) | if (keg->uk_flags & UMA_ZONE_PCPU) | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
zone_kva_available(uma_zone_t zone, void *unused) | zone_kva_available(uma_zone_t zone, void *unused) | ||||
{ | { | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) | if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) | ||||
return; | return; | ||||
KEG_GET(zone, keg); | KEG_GET(zone, keg); | ||||
if (keg->uk_allocf == startup_alloc) | if (keg->uk_allocf == startup_alloc) { | ||||
if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && | |||||
jeffUnsubmitted Not Done Inline ActionsThere really shouldn't be any boot time contig allocations but I suppose it doesn't hurt to handle it. jeff: There really shouldn't be any boot time contig allocations but I suppose it doesn't hurt to… | |||||
rlibbyAuthorUnsubmitted Done Inline ActionsYes... I thought to handle it since it's easy and I think it works fine with startup_alloc. If you prefer, I can add it to the KASSERT about early boot PCPU alloc instead. rlibby: Yes... I thought to handle it since it's easy and I think it works fine with startup_alloc. If… | |||||
keg->uk_ppera > 1) | |||||
keg->uk_allocf = contig_alloc; | |||||
else | |||||
keg->uk_allocf = page_alloc; | keg->uk_allocf = page_alloc; | ||||
} | |||||
} | } | ||||
static void | static void | ||||
zone_alloc_counters(uma_zone_t zone, void *unused) | zone_alloc_counters(uma_zone_t zone, void *unused) | ||||
{ | { | ||||
zone->uz_allocs = counter_u64_alloc(M_WAITOK); | zone->uz_allocs = counter_u64_alloc(M_WAITOK); | ||||
zone->uz_frees = counter_u64_alloc(M_WAITOK); | zone->uz_frees = counter_u64_alloc(M_WAITOK); | ||||
▲ Show 20 Lines • Show All 2,953 Lines • Show Last 20 Lines |
Also gratuitous, but related.