Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/uma_core.c
Show First 20 Lines • Show All 106 Lines • ▼ Show 20 Lines | |||||
static struct uma_keg masterkeg; | static struct uma_keg masterkeg; | ||||
static struct uma_zone masterzone_k; | static struct uma_zone masterzone_k; | ||||
static struct uma_zone masterzone_z; | static struct uma_zone masterzone_z; | ||||
static uma_zone_t kegs = &masterzone_k; | static uma_zone_t kegs = &masterzone_k; | ||||
static uma_zone_t zones = &masterzone_z; | static uma_zone_t zones = &masterzone_z; | ||||
/* This is the zone from which all of uma_slab_t's are allocated. */ | /* This is the zone from which all of uma_slab_t's are allocated. */ | ||||
static uma_zone_t slabzone; | static uma_zone_t slabzone; | ||||
static uma_zone_t slabrefzone; /* With refcounters (for UMA_ZONE_REFCNT) */ | |||||
/* | /* | ||||
* The initial hash tables come out of this zone so they can be allocated | * The initial hash tables come out of this zone so they can be allocated | ||||
* prior to malloc coming up. | * prior to malloc coming up. | ||||
*/ | */ | ||||
static uma_zone_t hashzone; | static uma_zone_t hashzone; | ||||
/* The boot-time adjusted value for cache line alignment. */ | /* The boot-time adjusted value for cache line alignment. */ | ||||
Show All 26 Lines | |||||
static struct sx uma_drain_lock; | static struct sx uma_drain_lock; | ||||
/* Is the VM done starting up? */ | /* Is the VM done starting up? */ | ||||
static int booted = 0; | static int booted = 0; | ||||
#define UMA_STARTUP 1 | #define UMA_STARTUP 1 | ||||
#define UMA_STARTUP2 2 | #define UMA_STARTUP2 2 | ||||
/* | /* | ||||
* Only mbuf clusters use ref zones. Just provide enough references | |||||
* to support the one user. New code should not use the ref facility. | |||||
*/ | |||||
static const u_int uma_max_ipers_ref = PAGE_SIZE / MCLBYTES; | |||||
/* | |||||
* This is the handle used to schedule events that need to happen | * This is the handle used to schedule events that need to happen | ||||
* outside of the allocation fast path. | * outside of the allocation fast path. | ||||
*/ | */ | ||||
static struct callout uma_callout; | static struct callout uma_callout; | ||||
#define UMA_TIMEOUT 20 /* Seconds for callout interval. */ | #define UMA_TIMEOUT 20 /* Seconds for callout interval. */ | ||||
/* | /* | ||||
* This structure is passed as the zone ctor arg so that I don't have to create | * This structure is passed as the zone ctor arg so that I don't have to create | ||||
▲ Show 20 Lines • Show All 774 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* Returns: | * Returns: | ||||
* The slab that was allocated or NULL if there is no memory and the | * The slab that was allocated or NULL if there is no memory and the | ||||
* caller specified M_NOWAIT. | * caller specified M_NOWAIT. | ||||
*/ | */ | ||||
static uma_slab_t | static uma_slab_t | ||||
keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait) | keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait) | ||||
{ | { | ||||
uma_slabrefcnt_t slabref; | |||||
uma_alloc allocf; | uma_alloc allocf; | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
uint8_t *mem; | uint8_t *mem; | ||||
uint8_t flags; | uint8_t flags; | ||||
int i; | int i; | ||||
mtx_assert(&keg->uk_lock, MA_OWNED); | mtx_assert(&keg->uk_lock, MA_OWNED); | ||||
slab = NULL; | slab = NULL; | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | #endif | ||||
slab->us_keg = keg; | slab->us_keg = keg; | ||||
slab->us_data = mem; | slab->us_data = mem; | ||||
slab->us_freecount = keg->uk_ipers; | slab->us_freecount = keg->uk_ipers; | ||||
slab->us_flags = flags; | slab->us_flags = flags; | ||||
BIT_FILL(SLAB_SETSIZE, &slab->us_free); | BIT_FILL(SLAB_SETSIZE, &slab->us_free); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree); | BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree); | ||||
#endif | #endif | ||||
if (keg->uk_flags & UMA_ZONE_REFCNT) { | |||||
slabref = (uma_slabrefcnt_t)slab; | |||||
for (i = 0; i < keg->uk_ipers; i++) | |||||
slabref->us_refcnt[i] = 0; | |||||
} | |||||
if (keg->uk_init != NULL) { | if (keg->uk_init != NULL) { | ||||
for (i = 0; i < keg->uk_ipers; i++) | for (i = 0; i < keg->uk_ipers; i++) | ||||
if (keg->uk_init(slab->us_data + (keg->uk_rsize * i), | if (keg->uk_init(slab->us_data + (keg->uk_rsize * i), | ||||
keg->uk_size, wait) != 0) | keg->uk_size, wait) != 0) | ||||
break; | break; | ||||
if (i != keg->uk_ipers) { | if (i != keg->uk_ipers) { | ||||
keg_free_slab(keg, slab, i); | keg_free_slab(keg, slab, i); | ||||
▲ Show 20 Lines • Show All 231 Lines • ▼ Show 20 Lines | keg_small_init(uma_keg_t keg) | ||||
if (rsize & keg->uk_align) | if (rsize & keg->uk_align) | ||||
rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); | rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); | ||||
keg->uk_rsize = rsize; | keg->uk_rsize = rsize; | ||||
KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || | KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || | ||||
keg->uk_rsize < sizeof(struct pcpu), | keg->uk_rsize < sizeof(struct pcpu), | ||||
("%s: size %u too large", __func__, keg->uk_rsize)); | ("%s: size %u too large", __func__, keg->uk_rsize)); | ||||
if (keg->uk_flags & UMA_ZONE_REFCNT) | |||||
rsize += sizeof(uint32_t); | |||||
if (keg->uk_flags & UMA_ZONE_OFFPAGE) | if (keg->uk_flags & UMA_ZONE_OFFPAGE) | ||||
shsize = 0; | shsize = 0; | ||||
else | else | ||||
shsize = sizeof(struct uma_slab); | shsize = sizeof(struct uma_slab); | ||||
keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize; | keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize; | ||||
KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE, | KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE, | ||||
("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); | ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); | ||||
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | keg_large_init(uma_keg_t keg) | ||||
/* We can't do OFFPAGE if we're internal, bail out here. */ | /* We can't do OFFPAGE if we're internal, bail out here. */ | ||||
if (keg->uk_flags & UMA_ZFLAG_INTERNAL) | if (keg->uk_flags & UMA_ZFLAG_INTERNAL) | ||||
return; | return; | ||||
/* Check whether we have enough space to not do OFFPAGE. */ | /* Check whether we have enough space to not do OFFPAGE. */ | ||||
if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0) { | if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0) { | ||||
shsize = sizeof(struct uma_slab); | shsize = sizeof(struct uma_slab); | ||||
if (keg->uk_flags & UMA_ZONE_REFCNT) | |||||
shsize += keg->uk_ipers * sizeof(uint32_t); | |||||
if (shsize & UMA_ALIGN_PTR) | if (shsize & UMA_ALIGN_PTR) | ||||
shsize = (shsize & ~UMA_ALIGN_PTR) + | shsize = (shsize & ~UMA_ALIGN_PTR) + | ||||
(UMA_ALIGN_PTR + 1); | (UMA_ALIGN_PTR + 1); | ||||
if ((PAGE_SIZE * keg->uk_ppera) - keg->uk_rsize < shsize) | if ((PAGE_SIZE * keg->uk_ppera) - keg->uk_rsize < shsize) | ||||
keg->uk_flags |= UMA_ZONE_OFFPAGE; | keg->uk_flags |= UMA_ZONE_OFFPAGE; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | keg_ctor(void *mem, int size, void *udata, int flags) | ||||
keg->uk_name = zone->uz_name; | keg->uk_name = zone->uz_name; | ||||
if (arg->flags & UMA_ZONE_VM) | if (arg->flags & UMA_ZONE_VM) | ||||
keg->uk_flags |= UMA_ZFLAG_CACHEONLY; | keg->uk_flags |= UMA_ZFLAG_CACHEONLY; | ||||
if (arg->flags & UMA_ZONE_ZINIT) | if (arg->flags & UMA_ZONE_ZINIT) | ||||
keg->uk_init = zero_init; | keg->uk_init = zero_init; | ||||
if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC) | if (arg->flags & UMA_ZONE_MALLOC) | ||||
keg->uk_flags |= UMA_ZONE_VTOSLAB; | keg->uk_flags |= UMA_ZONE_VTOSLAB; | ||||
if (arg->flags & UMA_ZONE_PCPU) | if (arg->flags & UMA_ZONE_PCPU) | ||||
#ifdef SMP | #ifdef SMP | ||||
keg->uk_flags |= UMA_ZONE_OFFPAGE; | keg->uk_flags |= UMA_ZONE_OFFPAGE; | ||||
#else | #else | ||||
keg->uk_flags &= ~UMA_ZONE_PCPU; | keg->uk_flags &= ~UMA_ZONE_PCPU; | ||||
#endif | #endif | ||||
if (keg->uk_flags & UMA_ZONE_CACHESPREAD) { | if (keg->uk_flags & UMA_ZONE_CACHESPREAD) { | ||||
keg_cachespread_init(keg); | keg_cachespread_init(keg); | ||||
} else if (keg->uk_flags & UMA_ZONE_REFCNT) { | |||||
if (keg->uk_size > | |||||
(UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) - | |||||
sizeof(uint32_t))) | |||||
keg_large_init(keg); | |||||
else | |||||
keg_small_init(keg); | |||||
} else { | } else { | ||||
if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab))) | if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab))) | ||||
keg_large_init(keg); | keg_large_init(keg); | ||||
else | else | ||||
keg_small_init(keg); | keg_small_init(keg); | ||||
} | } | ||||
if (keg->uk_flags & UMA_ZONE_OFFPAGE) { | if (keg->uk_flags & UMA_ZONE_OFFPAGE) | ||||
if (keg->uk_flags & UMA_ZONE_REFCNT) { | |||||
if (keg->uk_ipers > uma_max_ipers_ref) | |||||
panic("Too many ref items per zone: %d > %d\n", | |||||
keg->uk_ipers, uma_max_ipers_ref); | |||||
keg->uk_slabzone = slabrefzone; | |||||
} else | |||||
keg->uk_slabzone = slabzone; | keg->uk_slabzone = slabzone; | ||||
} | |||||
/* | /* | ||||
* If we haven't booted yet we need allocations to go through the | * If we haven't booted yet we need allocations to go through the | ||||
* startup cache until the vm is ready. | * startup cache until the vm is ready. | ||||
*/ | */ | ||||
if (keg->uk_ppera == 1) { | if (keg->uk_ppera == 1) { | ||||
#ifdef UMA_MD_SMALL_ALLOC | #ifdef UMA_MD_SMALL_ALLOC | ||||
keg->uk_allocf = uma_small_alloc; | keg->uk_allocf = uma_small_alloc; | ||||
Show All 20 Lines | #endif | ||||
* justified offset into the memory on an ALIGN_PTR boundary. | * justified offset into the memory on an ALIGN_PTR boundary. | ||||
*/ | */ | ||||
if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) { | if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) { | ||||
u_int totsize; | u_int totsize; | ||||
/* Size of the slab struct and free list */ | /* Size of the slab struct and free list */ | ||||
totsize = sizeof(struct uma_slab); | totsize = sizeof(struct uma_slab); | ||||
/* Size of the reference counts. */ | |||||
if (keg->uk_flags & UMA_ZONE_REFCNT) | |||||
totsize += keg->uk_ipers * sizeof(uint32_t); | |||||
if (totsize & UMA_ALIGN_PTR) | if (totsize & UMA_ALIGN_PTR) | ||||
totsize = (totsize & ~UMA_ALIGN_PTR) + | totsize = (totsize & ~UMA_ALIGN_PTR) + | ||||
(UMA_ALIGN_PTR + 1); | (UMA_ALIGN_PTR + 1); | ||||
keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize; | keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize; | ||||
/* | /* | ||||
* The only way the following is possible is if with our | * The only way the following is possible is if with our | ||||
* UMA_ALIGN_PTR adjustments we are now bigger than | * UMA_ALIGN_PTR adjustments we are now bigger than | ||||
* UMA_SLAB_SIZE. I haven't checked whether this is | * UMA_SLAB_SIZE. I haven't checked whether this is | ||||
* mathematically possible for all cases, so we make | * mathematically possible for all cases, so we make | ||||
* sure here anyway. | * sure here anyway. | ||||
*/ | */ | ||||
totsize = keg->uk_pgoff + sizeof(struct uma_slab); | totsize = keg->uk_pgoff + sizeof(struct uma_slab); | ||||
if (keg->uk_flags & UMA_ZONE_REFCNT) | |||||
totsize += keg->uk_ipers * sizeof(uint32_t); | |||||
if (totsize > PAGE_SIZE * keg->uk_ppera) { | if (totsize > PAGE_SIZE * keg->uk_ppera) { | ||||
printf("zone %s ipers %d rsize %d size %d\n", | printf("zone %s ipers %d rsize %d size %d\n", | ||||
zone->uz_name, keg->uk_ipers, keg->uk_rsize, | zone->uz_name, keg->uk_ipers, keg->uk_rsize, | ||||
keg->uk_size); | keg->uk_size); | ||||
panic("UMA slab won't fit."); | panic("UMA slab won't fit."); | ||||
} | } | ||||
} | } | ||||
▲ Show 20 Lines • Show All 245 Lines • ▼ Show 20 Lines | |||||
/* Public functions */ | /* Public functions */ | ||||
/* See uma.h */ | /* See uma.h */ | ||||
void | void | ||||
uma_startup(void *bootmem, int boot_pages) | uma_startup(void *bootmem, int boot_pages) | ||||
{ | { | ||||
struct uma_zctor_args args; | struct uma_zctor_args args; | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
u_int slabsize; | |||||
int i; | int i; | ||||
#ifdef UMA_DEBUG | #ifdef UMA_DEBUG | ||||
printf("Creating uma keg headers zone and keg.\n"); | printf("Creating uma keg headers zone and keg.\n"); | ||||
#endif | #endif | ||||
rw_init(&uma_rwlock, "UMA lock"); | rw_init(&uma_rwlock, "UMA lock"); | ||||
/* "manually" create the initial zone */ | /* "manually" create the initial zone */ | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
/* Now make a zone for slab headers */ | /* Now make a zone for slab headers */ | ||||
slabzone = uma_zcreate("UMA Slabs", | slabzone = uma_zcreate("UMA Slabs", | ||||
sizeof(struct uma_slab), | sizeof(struct uma_slab), | ||||
NULL, NULL, NULL, NULL, | NULL, NULL, NULL, NULL, | ||||
UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | ||||
/* | |||||
* We also create a zone for the bigger slabs with reference | |||||
* counts in them, to accomodate UMA_ZONE_REFCNT zones. | |||||
*/ | |||||
slabsize = sizeof(struct uma_slab_refcnt); | |||||
slabsize += uma_max_ipers_ref * sizeof(uint32_t); | |||||
slabrefzone = uma_zcreate("UMA RCntSlabs", | |||||
slabsize, | |||||
NULL, NULL, NULL, NULL, | |||||
UMA_ALIGN_PTR, | |||||
UMA_ZFLAG_INTERNAL); | |||||
hashzone = uma_zcreate("UMA Hash", | hashzone = uma_zcreate("UMA Hash", | ||||
sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, | sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, | ||||
NULL, NULL, NULL, NULL, | NULL, NULL, NULL, NULL, | ||||
UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); | ||||
bucket_init(); | bucket_init(); | ||||
booted = UMA_STARTUP; | booted = UMA_STARTUP; | ||||
▲ Show 20 Lines • Show All 206 Lines • ▼ Show 20 Lines | uma_zsecond_add(uma_zone_t zone, uma_zone_t master) | ||||
} | } | ||||
/* | /* | ||||
* The new master must also use vtoslab(). | * The new master must also use vtoslab(). | ||||
*/ | */ | ||||
if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) { | if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) { | ||||
error = EINVAL; | error = EINVAL; | ||||
goto out; | goto out; | ||||
} | } | ||||
/* | /* | ||||
* Both must either be refcnt, or not be refcnt. | |||||
*/ | |||||
if ((zone->uz_flags & UMA_ZONE_REFCNT) != | |||||
(master->uz_flags & UMA_ZONE_REFCNT)) { | |||||
error = EINVAL; | |||||
goto out; | |||||
} | |||||
/* | |||||
* The underlying object must be the same size. rsize | * The underlying object must be the same size. rsize | ||||
* may be different. | * may be different. | ||||
*/ | */ | ||||
if (master->uz_size != zone->uz_size) { | if (master->uz_size != zone->uz_size) { | ||||
error = E2BIG; | error = E2BIG; | ||||
goto out; | goto out; | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 1,105 Lines • ▼ Show 20 Lines | while (slabs > 0) { | ||||
MPASS(slab->us_keg == keg); | MPASS(slab->us_keg == keg); | ||||
LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); | LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); | ||||
slabs--; | slabs--; | ||||
} | } | ||||
KEG_UNLOCK(keg); | KEG_UNLOCK(keg); | ||||
} | } | ||||
/* See uma.h */ | /* See uma.h */ | ||||
uint32_t * | |||||
uma_find_refcnt(uma_zone_t zone, void *item) | |||||
{ | |||||
uma_slabrefcnt_t slabref; | |||||
uma_slab_t slab; | |||||
uma_keg_t keg; | |||||
uint32_t *refcnt; | |||||
int idx; | |||||
slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK)); | |||||
slabref = (uma_slabrefcnt_t)slab; | |||||
keg = slab->us_keg; | |||||
KASSERT(keg->uk_flags & UMA_ZONE_REFCNT, | |||||
("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT")); | |||||
idx = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; | |||||
refcnt = &slabref->us_refcnt[idx]; | |||||
return refcnt; | |||||
} | |||||
/* See uma.h */ | |||||
static void | static void | ||||
uma_reclaim_locked(bool kmem_danger) | uma_reclaim_locked(bool kmem_danger) | ||||
{ | { | ||||
#ifdef UMA_DEBUG | #ifdef UMA_DEBUG | ||||
printf("UMA: vm asked us to release pages!\n"); | printf("UMA: vm asked us to release pages!\n"); | ||||
#endif | #endif | ||||
sx_assert(&uma_drain_lock, SA_XLOCKED); | sx_assert(&uma_drain_lock, SA_XLOCKED); | ||||
bucket_enable(); | bucket_enable(); | ||||
zone_foreach(zone_drain); | zone_foreach(zone_drain); | ||||
if (vm_page_count_min() || kmem_danger) { | if (vm_page_count_min() || kmem_danger) { | ||||
cache_drain_safe(NULL); | cache_drain_safe(NULL); | ||||
zone_foreach(zone_drain); | zone_foreach(zone_drain); | ||||
} | } | ||||
/* | /* | ||||
* Some slabs may have been freed but this zone will be visited early | * Some slabs may have been freed but this zone will be visited early | ||||
* we visit again so that we can free pages that are empty once other | * we visit again so that we can free pages that are empty once other | ||||
* zones are drained. We have to do the same for buckets. | * zones are drained. We have to do the same for buckets. | ||||
*/ | */ | ||||
zone_drain(slabzone); | zone_drain(slabzone); | ||||
zone_drain(slabrefzone); | |||||
bucket_zone_drain(); | bucket_zone_drain(); | ||||
} | } | ||||
void | void | ||||
uma_reclaim(void) | uma_reclaim(void) | ||||
{ | { | ||||
sx_xlock(&uma_drain_lock); | sx_xlock(&uma_drain_lock); | ||||
▲ Show 20 Lines • Show All 498 Lines • Show Last 20 Lines |