Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_core.c
/*- | /*- | ||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD | * SPDX-License-Identifier: BSD-2-Clause-FreeBSD | ||||
* | * | ||||
* Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org> | * Copyright (c) 2002-2019 Jeffrey Roberson <jeff@FreeBSD.org> | ||||
* Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> | * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> | ||||
* Copyright (c) 2004-2006 Robert N. M. Watson | * Copyright (c) 2004-2006 Robert N. M. Watson | ||||
* All rights reserved. | * All rights reserved. | ||||
* | * | ||||
* Redistribution and use in source and binary forms, with or without | * Redistribution and use in source and binary forms, with or without | ||||
* modification, are permitted provided that the following conditions | * modification, are permitted provided that the following conditions | ||||
* are met: | * are met: | ||||
* 1. Redistributions of source code must retain the above copyright | * 1. Redistributions of source code must retain the above copyright | ||||
▲ Show 20 Lines • Show All 257 Lines • ▼ Show 20 Lines | |||||
static void *zone_alloc_item_locked(uma_zone_t, void *, int, int); | static void *zone_alloc_item_locked(uma_zone_t, void *, int, int); | ||||
static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip); | static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip); | ||||
static void bucket_enable(void); | static void bucket_enable(void); | ||||
static void bucket_init(void); | static void bucket_init(void); | ||||
static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int); | static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int); | ||||
static void bucket_free(uma_zone_t zone, uma_bucket_t, void *); | static void bucket_free(uma_zone_t zone, uma_bucket_t, void *); | ||||
static void bucket_zone_drain(void); | static void bucket_zone_drain(void); | ||||
static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int); | static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int); | ||||
static uma_slab_t zone_fetch_slab(uma_zone_t, uma_keg_t, int, int); | |||||
static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); | static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); | ||||
static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); | static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); | ||||
static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, | static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, | ||||
uma_fini fini, int align, uint32_t flags); | uma_fini fini, int align, uint32_t flags); | ||||
static int zone_import(uma_zone_t, void **, int, int, int); | static int zone_import(uma_zone_t, void **, int, int, int); | ||||
static void zone_release(uma_zone_t, void **, int); | static void zone_release(uma_zone_t, void **, int); | ||||
static void uma_zero_item(void *, uma_zone_t); | static void uma_zero_item(void *, uma_zone_t); | ||||
static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int); | static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int); | ||||
▲ Show 20 Lines • Show All 918 Lines • ▼ Show 20 Lines | keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, | ||||
uma_total_inc(size); | uma_total_inc(size); | ||||
/* Point the slab into the allocated memory */ | /* Point the slab into the allocated memory */ | ||||
if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) | if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) | ||||
slab = (uma_slab_t )(mem + keg->uk_pgoff); | slab = (uma_slab_t )(mem + keg->uk_pgoff); | ||||
if (keg->uk_flags & UMA_ZONE_VTOSLAB) | if (keg->uk_flags & UMA_ZONE_VTOSLAB) | ||||
for (i = 0; i < keg->uk_ppera; i++) | for (i = 0; i < keg->uk_ppera; i++) | ||||
vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); | vsetzoneslab((vm_offset_t)mem + (i * PAGE_SIZE), | ||||
zone, slab); | |||||
slab->us_keg = keg; | |||||
slab->us_data = mem; | slab->us_data = mem; | ||||
slab->us_freecount = keg->uk_ipers; | slab->us_freecount = keg->uk_ipers; | ||||
slab->us_flags = sflags; | slab->us_flags = sflags; | ||||
slab->us_domain = domain; | slab->us_domain = domain; | ||||
BIT_FILL(SLAB_SETSIZE, &slab->us_free); | BIT_FILL(SLAB_SETSIZE, &slab->us_free); | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree); | BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree); | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 1,788 Lines • ▼ Show 20 Lines | vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, | ||||
&aflags); | &aflags); | ||||
} else { | } else { | ||||
aflags = flags; | aflags = flags; | ||||
domain = rdomain; | domain = rdomain; | ||||
} | } | ||||
for (;;) { | for (;;) { | ||||
slab = keg_fetch_free_slab(keg, domain, rr, flags); | slab = keg_fetch_free_slab(keg, domain, rr, flags); | ||||
if (slab != NULL) { | if (slab != NULL) | ||||
MPASS(slab->us_keg == keg); | |||||
return (slab); | return (slab); | ||||
} | |||||
/* | /* | ||||
* M_NOVM means don't ask at all! | * M_NOVM means don't ask at all! | ||||
*/ | */ | ||||
if (flags & M_NOVM) | if (flags & M_NOVM) | ||||
break; | break; | ||||
KASSERT(zone->uz_max_items == 0 || | KASSERT(zone->uz_max_items == 0 || | ||||
zone->uz_items <= zone->uz_max_items, | zone->uz_items <= zone->uz_max_items, | ||||
("%s: zone %p overflow", __func__, zone)); | ("%s: zone %p overflow", __func__, zone)); | ||||
slab = keg_alloc_slab(keg, zone, domain, flags, aflags); | slab = keg_alloc_slab(keg, zone, domain, flags, aflags); | ||||
/* | /* | ||||
* If we got a slab here it's safe to mark it partially used | * If we got a slab here it's safe to mark it partially used | ||||
* and return. We assume that the caller is going to remove | * and return. We assume that the caller is going to remove | ||||
* at least one item. | * at least one item. | ||||
*/ | */ | ||||
if (slab) { | if (slab) { | ||||
MPASS(slab->us_keg == keg); | |||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); | ||||
return (slab); | return (slab); | ||||
} | } | ||||
KEG_LOCK(keg); | KEG_LOCK(keg); | ||||
if (rr && vm_domainset_iter_policy(&di, &domain) != 0) { | if (rr && vm_domainset_iter_policy(&di, &domain) != 0) { | ||||
if ((flags & M_WAITOK) != 0) { | if ((flags & M_WAITOK) != 0) { | ||||
KEG_UNLOCK(keg); | KEG_UNLOCK(keg); | ||||
vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); | vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask); | ||||
KEG_LOCK(keg); | KEG_LOCK(keg); | ||||
goto restart; | goto restart; | ||||
} | } | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* We might not have been able to get a slab but another cpu | * We might not have been able to get a slab but another cpu | ||||
* could have while we were unlocked. Check again before we | * could have while we were unlocked. Check again before we | ||||
* fail. | * fail. | ||||
*/ | */ | ||||
if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) { | if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) { | ||||
MPASS(slab->us_keg == keg); | |||||
return (slab); | return (slab); | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
static uma_slab_t | |||||
zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int domain, int flags) | |||||
{ | |||||
uma_slab_t slab; | |||||
if (keg == NULL) { | |||||
keg = zone->uz_keg; | |||||
KEG_LOCK(keg); | |||||
} | |||||
for (;;) { | |||||
slab = keg_fetch_slab(keg, zone, domain, flags); | |||||
if (slab) | |||||
return (slab); | |||||
if (flags & (M_NOWAIT | M_NOVM)) | |||||
break; | |||||
} | |||||
KEG_UNLOCK(keg); | |||||
return (NULL); | |||||
} | |||||
static void * | static void * | ||||
slab_alloc_item(uma_keg_t keg, uma_slab_t slab) | slab_alloc_item(uma_keg_t keg, uma_slab_t slab) | ||||
{ | { | ||||
uma_domain_t dom; | uma_domain_t dom; | ||||
void *item; | void *item; | ||||
uint8_t freei; | uint8_t freei; | ||||
MPASS(keg == slab->us_keg); | |||||
KEG_LOCK_ASSERT(keg); | KEG_LOCK_ASSERT(keg); | ||||
freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1; | freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1; | ||||
BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free); | BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free); | ||||
item = slab->us_data + (keg->uk_rsize * freei); | item = slab->us_data + (keg->uk_rsize * freei); | ||||
slab->us_freecount--; | slab->us_freecount--; | ||||
keg->uk_free--; | keg->uk_free--; | ||||
Show All 13 Lines | zone_import(uma_zone_t zone, void **bucket, int max, int domain, int flags) | ||||
uma_slab_t slab; | uma_slab_t slab; | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
#ifdef NUMA | #ifdef NUMA | ||||
int stripe; | int stripe; | ||||
#endif | #endif | ||||
int i; | int i; | ||||
slab = NULL; | slab = NULL; | ||||
keg = NULL; | keg = zone->uz_keg; | ||||
KEG_LOCK(keg); | |||||
/* Try to keep the buckets totally full */ | /* Try to keep the buckets totally full */ | ||||
for (i = 0; i < max; ) { | for (i = 0; i < max; ) { | ||||
if ((slab = zone_fetch_slab(zone, keg, domain, flags)) == NULL) | if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) | ||||
break; | break; | ||||
keg = slab->us_keg; | |||||
#ifdef NUMA | #ifdef NUMA | ||||
stripe = howmany(max, vm_ndomains); | stripe = howmany(max, vm_ndomains); | ||||
#endif | #endif | ||||
while (slab->us_freecount && i < max) { | while (slab->us_freecount && i < max) { | ||||
bucket[i++] = slab_alloc_item(keg, slab); | bucket[i++] = slab_alloc_item(keg, slab); | ||||
if (keg->uk_free <= keg->uk_reserve) | if (keg->uk_free <= keg->uk_reserve) | ||||
break; | break; | ||||
#ifdef NUMA | #ifdef NUMA | ||||
Show All 9 Lines | #ifdef NUMA | ||||
vm_ndomains > 1 && --stripe == 0) | vm_ndomains > 1 && --stripe == 0) | ||||
break; | break; | ||||
#endif | #endif | ||||
} | } | ||||
/* Don't block if we allocated any successfully. */ | /* Don't block if we allocated any successfully. */ | ||||
flags &= ~M_WAITOK; | flags &= ~M_WAITOK; | ||||
flags |= M_NOWAIT; | flags |= M_NOWAIT; | ||||
} | } | ||||
if (slab != NULL) | |||||
KEG_UNLOCK(keg); | KEG_UNLOCK(keg); | ||||
return i; | return i; | ||||
} | } | ||||
static uma_bucket_t | static uma_bucket_t | ||||
zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) | zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) | ||||
{ | { | ||||
uma_bucket_t bucket; | uma_bucket_t bucket; | ||||
▲ Show 20 Lines • Show All 424 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
uma_keg_t keg; | uma_keg_t keg; | ||||
uma_domain_t dom; | uma_domain_t dom; | ||||
uint8_t freei; | uint8_t freei; | ||||
keg = zone->uz_keg; | keg = zone->uz_keg; | ||||
MPASS(zone->uz_lockptr == &keg->uk_lock); | MPASS(zone->uz_lockptr == &keg->uk_lock); | ||||
KEG_LOCK_ASSERT(keg); | KEG_LOCK_ASSERT(keg); | ||||
MPASS(keg == slab->us_keg); | |||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
/* Do we need to remove from any lists? */ | /* Do we need to remove from any lists? */ | ||||
if (slab->us_freecount+1 == keg->uk_ipers) { | if (slab->us_freecount+1 == keg->uk_ipers) { | ||||
LIST_REMOVE(slab, us_link); | LIST_REMOVE(slab, us_link); | ||||
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); | LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); | ||||
} else if (slab->us_freecount == 0) { | } else if (slab->us_freecount == 0) { | ||||
Show All 26 Lines | for (i = 0; i < cnt; i++) { | ||||
if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) { | if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) { | ||||
mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); | mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); | ||||
if (zone->uz_flags & UMA_ZONE_HASH) { | if (zone->uz_flags & UMA_ZONE_HASH) { | ||||
slab = hash_sfind(&keg->uk_hash, mem); | slab = hash_sfind(&keg->uk_hash, mem); | ||||
} else { | } else { | ||||
mem += keg->uk_pgoff; | mem += keg->uk_pgoff; | ||||
slab = (uma_slab_t)mem; | slab = (uma_slab_t)mem; | ||||
} | } | ||||
} else { | } else | ||||
slab = vtoslab((vm_offset_t)item); | slab = vtoslab((vm_offset_t)item); | ||||
MPASS(slab->us_keg == keg); | |||||
} | |||||
slab_free_item(zone, slab, item); | slab_free_item(zone, slab, item); | ||||
} | } | ||||
KEG_UNLOCK(keg); | KEG_UNLOCK(keg); | ||||
} | } | ||||
/* | /* | ||||
* Frees a single item to any zone. | * Frees a single item to any zone. | ||||
* | * | ||||
▲ Show 20 Lines • Show All 334 Lines • ▼ Show 20 Lines | uma_prealloc(uma_zone_t zone, int items) | ||||
while (slabs-- > 0) { | while (slabs-- > 0) { | ||||
aflags = M_NOWAIT; | aflags = M_NOWAIT; | ||||
vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, | vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, | ||||
&aflags); | &aflags); | ||||
for (;;) { | for (;;) { | ||||
slab = keg_alloc_slab(keg, zone, domain, M_WAITOK, | slab = keg_alloc_slab(keg, zone, domain, M_WAITOK, | ||||
aflags); | aflags); | ||||
if (slab != NULL) { | if (slab != NULL) { | ||||
MPASS(slab->us_keg == keg); | |||||
dom = &keg->uk_domain[slab->us_domain]; | dom = &keg->uk_domain[slab->us_domain]; | ||||
LIST_INSERT_HEAD(&dom->ud_free_slab, slab, | LIST_INSERT_HEAD(&dom->ud_free_slab, slab, | ||||
us_link); | us_link); | ||||
break; | break; | ||||
} | } | ||||
KEG_LOCK(keg); | KEG_LOCK(keg); | ||||
if (vm_domainset_iter_policy(&di, &domain) != 0) { | if (vm_domainset_iter_policy(&di, &domain) != 0) { | ||||
KEG_UNLOCK(keg); | KEG_UNLOCK(keg); | ||||
▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines | uma_large_malloc_domain(vm_size_t size, int domain, int wait) | ||||
} | } | ||||
slab = zone_alloc_item(slabzone, NULL, domain, wait); | slab = zone_alloc_item(slabzone, NULL, domain, wait); | ||||
if (slab == NULL) | if (slab == NULL) | ||||
return (NULL); | return (NULL); | ||||
policy = (domain == UMA_ANYDOMAIN) ? DOMAINSET_RR() : | policy = (domain == UMA_ANYDOMAIN) ? DOMAINSET_RR() : | ||||
DOMAINSET_FIXED(domain); | DOMAINSET_FIXED(domain); | ||||
addr = kmem_malloc_domainset(policy, size, wait); | addr = kmem_malloc_domainset(policy, size, wait); | ||||
if (addr != 0) { | if (addr != 0) { | ||||
vsetslab(addr, slab); | vsetzoneslab(addr, NULL, slab); | ||||
slab->us_data = (void *)addr; | slab->us_data = (void *)addr; | ||||
slab->us_flags = UMA_SLAB_KERNEL | UMA_SLAB_MALLOC; | slab->us_flags = UMA_SLAB_KERNEL | UMA_SLAB_MALLOC; | ||||
slab->us_size = size; | slab->us_size = size; | ||||
slab->us_domain = vm_phys_domain(PHYS_TO_VM_PAGE( | slab->us_domain = vm_phys_domain(PHYS_TO_VM_PAGE( | ||||
pmap_kextract(addr))); | pmap_kextract(addr))); | ||||
uma_total_inc(size); | uma_total_inc(size); | ||||
} else { | } else { | ||||
zone_free_item(slabzone, slab, NULL, SKIP_NONE); | zone_free_item(slabzone, slab, NULL, SKIP_NONE); | ||||
▲ Show 20 Lines • Show All 395 Lines • ▼ Show 20 Lines | uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item) | ||||
int freei; | int freei; | ||||
if (slab == NULL) { | if (slab == NULL) { | ||||
slab = uma_dbg_getslab(zone, item); | slab = uma_dbg_getslab(zone, item); | ||||
if (slab == NULL) | if (slab == NULL) | ||||
panic("uma: item %p did not belong to zone %s\n", | panic("uma: item %p did not belong to zone %s\n", | ||||
item, zone->uz_name); | item, zone->uz_name); | ||||
} | } | ||||
keg = slab->us_keg; | keg = zone->uz_keg; | ||||
freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; | freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; | ||||
if (BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree)) | if (BIT_ISSET(SLAB_SETSIZE, freei, &slab->us_debugfree)) | ||||
panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)\n", | panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)\n", | ||||
item, zone, zone->uz_name, slab, freei); | item, zone, zone->uz_name, slab, freei); | ||||
BIT_SET_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree); | BIT_SET_ATOMIC(SLAB_SETSIZE, freei, &slab->us_debugfree); | ||||
return; | return; | ||||
Show All 11 Lines | uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item) | ||||
int freei; | int freei; | ||||
if (slab == NULL) { | if (slab == NULL) { | ||||
slab = uma_dbg_getslab(zone, item); | slab = uma_dbg_getslab(zone, item); | ||||
if (slab == NULL) | if (slab == NULL) | ||||
panic("uma: Freed item %p did not belong to zone %s\n", | panic("uma: Freed item %p did not belong to zone %s\n", | ||||
item, zone->uz_name); | item, zone->uz_name); | ||||
} | } | ||||
keg = slab->us_keg; | keg = zone->uz_keg; | ||||
freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; | freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; | ||||
if (freei >= keg->uk_ipers) | if (freei >= keg->uk_ipers) | ||||
panic("Invalid free of %p from zone %p(%s) slab %p(%d)\n", | panic("Invalid free of %p from zone %p(%s) slab %p(%d)\n", | ||||
item, zone, zone->uz_name, slab, freei); | item, zone, zone->uz_name, slab, freei); | ||||
if (((freei * keg->uk_rsize) + slab->us_data) != item) | if (((freei * keg->uk_rsize) + slab->us_data) != item) | ||||
panic("Unaligned free of %p from zone %p(%s) slab %p(%d)\n", | panic("Unaligned free of %p from zone %p(%s) slab %p(%d)\n", | ||||
▲ Show 20 Lines • Show All 131 Lines • Show Last 20 Lines |