Changeset View
Changeset View
Standalone View
Standalone View
head/sys/vm/uma_int.h
Show First 20 Lines • Show All 362 Lines • ▼ Show 20 Lines | struct uma_keg { | ||||
const char *uk_name; /* Name of creating zone. */ | const char *uk_name; /* Name of creating zone. */ | ||||
LIST_ENTRY(uma_keg) uk_link; /* List of all kegs */ | LIST_ENTRY(uma_keg) uk_link; /* List of all kegs */ | ||||
/* Must be last, variable sized. */ | /* Must be last, variable sized. */ | ||||
struct uma_domain uk_domain[]; /* Keg's slab lists. */ | struct uma_domain uk_domain[]; /* Keg's slab lists. */ | ||||
}; | }; | ||||
typedef struct uma_keg * uma_keg_t; | typedef struct uma_keg * uma_keg_t; | ||||
#ifdef _KERNEL | |||||
#define KEG_ASSERT_COLD(k) \ | |||||
KASSERT(uma_keg_get_allocs((k)) == 0, \ | |||||
("keg %s initialization after use.", (k)->uk_name)) | |||||
/* | /* | ||||
* Free bits per-slab. | * Free bits per-slab. | ||||
*/ | */ | ||||
#define SLAB_MAX_SETSIZE (PAGE_SIZE / UMA_SMALLEST_UNIT) | #define SLAB_MAX_SETSIZE (PAGE_SIZE / UMA_SMALLEST_UNIT) | ||||
#define SLAB_MIN_SETSIZE _BITSET_BITS | #define SLAB_MIN_SETSIZE _BITSET_BITS | ||||
BITSET_DEFINE(noslabbits, 0); | BITSET_DEFINE(noslabbits, 0); | ||||
/* | /* | ||||
* The slab structure manages a single contiguous allocation from backing | * The slab structure manages a single contiguous allocation from backing | ||||
* store and subdivides it into individually allocatable items. | * store and subdivides it into individually allocatable items. | ||||
*/ | */ | ||||
struct uma_slab { | struct uma_slab { | ||||
LIST_ENTRY(uma_slab) us_link; /* slabs in zone */ | LIST_ENTRY(uma_slab) us_link; /* slabs in zone */ | ||||
uint16_t us_freecount; /* How many are free? */ | uint16_t us_freecount; /* How many are free? */ | ||||
uint8_t us_flags; /* Page flags see uma.h */ | uint8_t us_flags; /* Page flags see uma.h */ | ||||
uint8_t us_domain; /* Backing NUMA domain. */ | uint8_t us_domain; /* Backing NUMA domain. */ | ||||
struct noslabbits us_free; /* Free bitmask, flexible. */ | struct noslabbits us_free; /* Free bitmask, flexible. */ | ||||
}; | }; | ||||
_Static_assert(sizeof(struct uma_slab) == offsetof(struct uma_slab, us_free), | _Static_assert(sizeof(struct uma_slab) == __offsetof(struct uma_slab, us_free), | ||||
"us_free field must be last"); | "us_free field must be last"); | ||||
#if MAXMEMDOM >= 255 | _Static_assert(MAXMEMDOM < 255, | ||||
#error "Slab domain type insufficient" | "us_domain field is not wide enough"); | ||||
#endif | |||||
typedef struct uma_slab * uma_slab_t; | typedef struct uma_slab * uma_slab_t; | ||||
/* | /* | ||||
* On INVARIANTS builds, the slab contains a second bitset of the same size, | |||||
* "dbg_bits", which is laid out immediately after us_free. | |||||
*/ | |||||
#ifdef INVARIANTS | |||||
#define SLAB_BITSETS 2 | |||||
#else | |||||
#define SLAB_BITSETS 1 | |||||
#endif | |||||
/* These three functions are for embedded (!OFFPAGE) use only. */ | |||||
size_t slab_sizeof(int nitems); | |||||
size_t slab_space(int nitems); | |||||
int slab_ipers(size_t size, int align); | |||||
/* | |||||
* Slab structure with a full sized bitset and hash link for both | * Slab structure with a full sized bitset and hash link for both | ||||
* HASH and OFFPAGE zones. | * HASH and OFFPAGE zones. | ||||
*/ | */ | ||||
struct uma_hash_slab { | struct uma_hash_slab { | ||||
LIST_ENTRY(uma_hash_slab) uhs_hlink; /* Link for hash table */ | LIST_ENTRY(uma_hash_slab) uhs_hlink; /* Link for hash table */ | ||||
uint8_t *uhs_data; /* First item */ | uint8_t *uhs_data; /* First item */ | ||||
struct uma_slab uhs_slab; /* Must be last. */ | struct uma_slab uhs_slab; /* Must be last. */ | ||||
}; | }; | ||||
Show All 29 Lines | |||||
static inline int | static inline int | ||||
slab_item_index(uma_slab_t slab, uma_keg_t keg, void *item) | slab_item_index(uma_slab_t slab, uma_keg_t keg, void *item) | ||||
{ | { | ||||
uintptr_t data; | uintptr_t data; | ||||
data = (uintptr_t)slab_data(slab, keg); | data = (uintptr_t)slab_data(slab, keg); | ||||
return (((uintptr_t)item - data) / keg->uk_rsize); | return (((uintptr_t)item - data) / keg->uk_rsize); | ||||
} | } | ||||
#endif /* _KERNEL */ | |||||
STAILQ_HEAD(uma_bucketlist, uma_bucket); | STAILQ_HEAD(uma_bucketlist, uma_bucket); | ||||
struct uma_zone_domain { | struct uma_zone_domain { | ||||
struct uma_bucketlist uzd_buckets; /* full buckets */ | struct uma_bucketlist uzd_buckets; /* full buckets */ | ||||
uma_bucket_t uzd_cross; /* Fills from cross buckets. */ | uma_bucket_t uzd_cross; /* Fills from cross buckets. */ | ||||
long uzd_nitems; /* total item count */ | long uzd_nitems; /* total item count */ | ||||
long uzd_imax; /* maximum item count this period */ | long uzd_imax; /* maximum item count this period */ | ||||
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | |||||
#define KEG_UNLOCK(k, d) mtx_unlock(KEG_LOCKPTR(k, d)) | #define KEG_UNLOCK(k, d) mtx_unlock(KEG_LOCKPTR(k, d)) | ||||
#define KEG_LOCK_ASSERT(k, d) mtx_assert(KEG_LOCKPTR(k, d), MA_OWNED) | #define KEG_LOCK_ASSERT(k, d) mtx_assert(KEG_LOCKPTR(k, d), MA_OWNED) | ||||
#define KEG_GET(zone, keg) do { \ | #define KEG_GET(zone, keg) do { \ | ||||
(keg) = (zone)->uz_keg; \ | (keg) = (zone)->uz_keg; \ | ||||
KASSERT((void *)(keg) != NULL, \ | KASSERT((void *)(keg) != NULL, \ | ||||
("%s: Invalid zone %p type", __func__, (zone))); \ | ("%s: Invalid zone %p type", __func__, (zone))); \ | ||||
} while (0) | } while (0) | ||||
#define KEG_ASSERT_COLD(k) \ | |||||
KASSERT(uma_keg_get_allocs((k)) == 0, \ | |||||
("keg %s initialization after use.", (k)->uk_name)) | |||||
/* Domains are contiguous after the last CPU */ | /* Domains are contiguous after the last CPU */ | ||||
#define ZDOM_GET(z, n) \ | #define ZDOM_GET(z, n) \ | ||||
(&((uma_zone_domain_t)&(z)->uz_cpu[mp_maxid + 1])[n]) | (&((uma_zone_domain_t)&(z)->uz_cpu[mp_maxid + 1])[n]) | ||||
#define ZDOM_LOCK_INIT(z, zdom, lc) \ | #define ZDOM_LOCK_INIT(z, zdom, lc) \ | ||||
do { \ | do { \ | ||||
if ((lc)) \ | if ((lc)) \ | ||||
▲ Show 20 Lines • Show All 111 Lines • Show Last 20 Lines |