Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/uma_int.h
Show First 20 Lines • Show All 217 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Keg management structure | * Keg management structure | ||||
* | * | ||||
* TODO: Optimize for cache line size | * TODO: Optimize for cache line size | ||||
* | * | ||||
*/ | */ | ||||
struct uma_keg { | struct uma_keg { | ||||
struct mtx uk_lock; /* Lock for the keg */ | struct mtx uk_lock; /* Lock for the keg. MUST be first! */ | ||||
markj: Why? | |||||
Done Inline ActionsSo that we may use single pointer for uz_keg/uz_lockptr. Should I put that in comments? glebius: So that we may use single pointer for uz_keg/uz_lockptr. Should I put that in comments? | |||||
Done Inline ActionsRight, I see now. A comment would be helpful, I think. markj: Right, I see now. A comment would be helpful, I think. | |||||
Done Inline ActionsWill do. glebius: Will do. | |||||
struct uma_hash uk_hash; | struct uma_hash uk_hash; | ||||
LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */ | LIST_HEAD(,uma_zone) uk_zones; /* Keg's zones */ | ||||
struct domainset_ref uk_dr; /* Domain selection policy. */ | struct domainset_ref uk_dr; /* Domain selection policy. */ | ||||
uint32_t uk_align; /* Alignment mask */ | uint32_t uk_align; /* Alignment mask */ | ||||
uint32_t uk_pages; /* Total page count */ | uint32_t uk_pages; /* Total page count */ | ||||
uint32_t uk_free; /* Count of items free in slabs */ | uint32_t uk_free; /* Count of items free in slabs */ | ||||
uint32_t uk_reserve; /* Number of reserved items. */ | uint32_t uk_reserve; /* Number of reserved items. */ | ||||
uint32_t uk_size; /* Requested size of each item */ | uint32_t uk_size; /* Requested size of each item */ | ||||
uint32_t uk_rsize; /* Real size of each item */ | uint32_t uk_rsize; /* Real size of each item */ | ||||
uint32_t uk_maxpages; /* Maximum number of pages to alloc */ | /* 32 bit pad */ | ||||
Done Inline ActionsI don't believe there's any padding here. markj: I don't believe there's any padding here. | |||||
uma_init uk_init; /* Keg's init routine */ | uma_init uk_init; /* Keg's init routine */ | ||||
uma_fini uk_fini; /* Keg's fini routine */ | uma_fini uk_fini; /* Keg's fini routine */ | ||||
uma_alloc uk_allocf; /* Allocation function */ | uma_alloc uk_allocf; /* Allocation function */ | ||||
uma_free uk_freef; /* Free routine */ | uma_free uk_freef; /* Free routine */ | ||||
u_long uk_offset; /* Next free offset from base KVA */ | u_long uk_offset; /* Next free offset from base KVA */ | ||||
vm_offset_t uk_kva; /* Zone base KVA */ | vm_offset_t uk_kva; /* Zone base KVA */ | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | |||||
#if MAXMEMDOM >= 255 | #if MAXMEMDOM >= 255 | ||||
#error "Slab domain type insufficient" | #error "Slab domain type insufficient" | ||||
#endif | #endif | ||||
typedef struct uma_slab * uma_slab_t; | typedef struct uma_slab * uma_slab_t; | ||||
typedef uma_slab_t (*uma_slaballoc)(uma_zone_t, uma_keg_t, int, int); | typedef uma_slab_t (*uma_slaballoc)(uma_zone_t, uma_keg_t, int, int); | ||||
struct uma_klink { | |||||
LIST_ENTRY(uma_klink) kl_link; | |||||
uma_keg_t kl_keg; | |||||
}; | |||||
typedef struct uma_klink *uma_klink_t; | |||||
struct uma_zone_domain { | struct uma_zone_domain { | ||||
LIST_HEAD(,uma_bucket) uzd_buckets; /* full buckets */ | LIST_HEAD(,uma_bucket) uzd_buckets; /* full buckets */ | ||||
long uzd_nitems; /* total item count */ | long uzd_nitems; /* total item count */ | ||||
long uzd_imax; /* maximum item count this period */ | long uzd_imax; /* maximum item count this period */ | ||||
long uzd_imin; /* minimum item count this period */ | long uzd_imin; /* minimum item count this period */ | ||||
long uzd_wss; /* working set size estimate */ | long uzd_wss; /* working set size estimate */ | ||||
}; | }; | ||||
typedef struct uma_zone_domain * uma_zone_domain_t; | typedef struct uma_zone_domain * uma_zone_domain_t; | ||||
/* | /* | ||||
* Zone management structure | * Zone management structure | ||||
* | * | ||||
* TODO: Optimize for cache line size | * TODO: Optimize for cache line size | ||||
* | * | ||||
*/ | */ | ||||
struct uma_zone { | struct uma_zone { | ||||
/* Offset 0, used in alloc/free fast/medium fast path and const. */ | /* Offset 0, used in alloc/free fast/medium fast path and const. */ | ||||
struct mtx *uz_lockptr; | union { | ||||
const char *uz_name; /* Text name of the zone */ | uma_keg_t uz_keg; /* This zone's keg */ | ||||
Done Inline Actions"This zone's keg" markj: "This zone's keg" | |||||
struct mtx *uz_lockptr; /* To keg or to self */ | |||||
}; | |||||
struct uma_zone_domain *uz_domain; /* per-domain buckets */ | struct uma_zone_domain *uz_domain; /* per-domain buckets */ | ||||
uint32_t uz_flags; /* Flags inherited from kegs */ | uint32_t uz_flags; /* Flags inherited from kegs */ | ||||
uint32_t uz_size; /* Size inherited from kegs */ | uint32_t uz_size; /* Size inherited from kegs */ | ||||
uma_ctor uz_ctor; /* Constructor for each allocation */ | uma_ctor uz_ctor; /* Constructor for each allocation */ | ||||
uma_dtor uz_dtor; /* Destructor */ | uma_dtor uz_dtor; /* Destructor */ | ||||
uma_init uz_init; /* Initializer for each item */ | uint64_t uz_items; /* Total items count */ | ||||
uma_fini uz_fini; /* Finalizer for each item. */ | uint64_t uz_maxitems; /* Maximum number of items to alloc */ | ||||
Done Inline ActionsTo be consistent this should be uz_max_items. markj: To be consistent this should be uz_max_items. | |||||
uint32_t uz_sleepers; /* Number of sleepers on memory */ | |||||
uint16_t uz_count; /* Amount of items in full bucket */ | |||||
uint16_t uz_count_max; /* Maximum amount of items there */ | |||||
/* Offset 64, used in bucket replenish. */ | /* Offset 64, used in bucket replenish. */ | ||||
uma_import uz_import; /* Import new memory to cache. */ | uma_import uz_import; /* Import new memory to cache. */ | ||||
uma_release uz_release; /* Release memory from cache. */ | uma_release uz_release; /* Release memory from cache. */ | ||||
void *uz_arg; /* Import/release argument. */ | void *uz_arg; /* Import/release argument. */ | ||||
uma_init uz_init; /* Initializer for each item */ | |||||
uma_fini uz_fini; /* Finalizer for each item. */ | |||||
uma_slaballoc uz_slab; /* Allocate a slab from the backend. */ | uma_slaballoc uz_slab; /* Allocate a slab from the backend. */ | ||||
Not Done Inline ActionsThis is now used for a single pointer and can be removed. jeff: This is now used for a single pointer and can be removed. | |||||
Done Inline ActionsYes, I noticed that. At Netflix we already have that substituted for an other field. I will address that later. glebius: Yes, I noticed that. At Netflix we already have that substituted for an other field. I will… | |||||
uint16_t uz_count; /* Amount of items in full bucket */ | uint64_t uz_bktcount; /* Items in bucket cache */ | ||||
uint16_t uz_count_min; /* Minimal amount of items there */ | uint64_t uz_bktmax; /* Maximum bucket cache size */ | ||||
Done Inline ActionsAgain, I'd prefer consistency in the names: uz_bkt_count, uz_bkt_max. markj: Again, I'd prefer consistency in the names: uz_bkt_count, uz_bkt_max. | |||||
/* 32bit pad on 64bit. */ | |||||
LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ | |||||
LIST_HEAD(,uma_klink) uz_kegs; /* List of kegs. */ | |||||
/* Offset 128 Rare. */ | /* Offset 128 Rare. */ | ||||
Not Done Inline ActionsDid you verify this offset? jeff: Did you verify this offset? | |||||
Done Inline ActionsI did. Will verify once again pre-commit. glebius: I did. Will verify once again pre-commit. | |||||
/* | /* | ||||
* The lock is placed here to avoid adjacent line prefetcher | * The lock is placed here to avoid adjacent line prefetcher | ||||
* in fast paths and to take up space near infrequently accessed | * in fast paths and to take up space near infrequently accessed | ||||
* members to reduce alignment overhead. | * members to reduce alignment overhead. | ||||
*/ | */ | ||||
struct mtx uz_lock; /* Lock for the zone */ | struct mtx uz_lock; /* Lock for the zone */ | ||||
struct uma_klink uz_klink; /* klink for first keg. */ | LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ | ||||
const char *uz_name; /* Text name of the zone */ | |||||
/* The next two fields are used to print a rate-limited warnings. */ | /* The next two fields are used to print a rate-limited warnings. */ | ||||
const char *uz_warning; /* Warning to print on failure */ | const char *uz_warning; /* Warning to print on failure */ | ||||
struct timeval uz_ratecheck; /* Warnings rate-limiting */ | struct timeval uz_ratecheck; /* Warnings rate-limiting */ | ||||
struct task uz_maxaction; /* Task to run when at limit */ | struct task uz_maxaction; /* Task to run when at limit */ | ||||
uint16_t uz_count_min; /* Minimal amount of items in bucket */ | |||||
/* 16 bytes of pad. */ | /* Offset 256, stats. */ | ||||
uint64_t uz_allocs UMA_ALIGN; /* Total number of allocations */ | |||||
/* Offset 256, atomic stats. */ | |||||
volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */ | |||||
volatile u_long uz_fails; /* Total number of alloc failures */ | |||||
volatile u_long uz_frees; /* Total number of frees */ | |||||
uint64_t uz_sleeps; /* Total number of alloc sleeps */ | uint64_t uz_sleeps; /* Total number of alloc sleeps */ | ||||
uint64_t uz_frees; /* Total number of frees */ | |||||
volatile u_long uz_fails; /* Total number of alloc failures */ | |||||
/* | /* | ||||
* This HAS to be the last item because we adjust the zone size | * This HAS to be the last item because we adjust the zone size | ||||
* based on NCPU and then allocate the space for the zones. | * based on NCPU and then allocate the space for the zones. | ||||
*/ | */ | ||||
struct uma_cache uz_cpu[]; /* Per cpu caches */ | struct uma_cache uz_cpu[]; /* Per cpu caches */ | ||||
/* uz_domain follows here. */ | /* uz_domain follows here. */ | ||||
}; | }; | ||||
/* | /* | ||||
* These flags must not overlap with the UMA_ZONE flags specified in uma.h. | * These flags must not overlap with the UMA_ZONE flags specified in uma.h. | ||||
*/ | */ | ||||
#define UMA_ZFLAG_MULTI 0x04000000 /* Multiple kegs in the zone. */ | #define UMA_ZFLAG_MULTI 0x04000000 /* Multiple kegs in the zone. */ | ||||
#define UMA_ZFLAG_DRAINING 0x08000000 /* Running zone_drain. */ | #define UMA_ZFLAG_DRAINING 0x08000000 /* Running zone_drain. */ | ||||
#define UMA_ZFLAG_BUCKET 0x10000000 /* Bucket zone. */ | #define UMA_ZFLAG_BUCKET 0x10000000 /* Bucket zone. */ | ||||
#define UMA_ZFLAG_INTERNAL 0x20000000 /* No offpage no PCPU. */ | #define UMA_ZFLAG_INTERNAL 0x20000000 /* No offpage no PCPU. */ | ||||
#define UMA_ZFLAG_FULL 0x40000000 /* Reached uz_maxpages */ | |||||
#define UMA_ZFLAG_CACHEONLY 0x80000000 /* Don't ask VM for buckets. */ | #define UMA_ZFLAG_CACHEONLY 0x80000000 /* Don't ask VM for buckets. */ | ||||
#define UMA_ZFLAG_INHERIT \ | #define UMA_ZFLAG_INHERIT \ | ||||
(UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY | UMA_ZFLAG_BUCKET) | (UMA_ZFLAG_INTERNAL | UMA_ZFLAG_CACHEONLY | UMA_ZFLAG_BUCKET) | ||||
static inline uma_keg_t | |||||
zone_first_keg(uma_zone_t zone) | |||||
{ | |||||
uma_klink_t klink; | |||||
klink = LIST_FIRST(&zone->uz_kegs); | |||||
return (klink != NULL) ? klink->kl_keg : NULL; | |||||
} | |||||
#undef UMA_ALIGN | #undef UMA_ALIGN | ||||
#ifdef _KERNEL | #ifdef _KERNEL | ||||
/* Internal prototypes */ | /* Internal prototypes */ | ||||
static __inline uma_slab_t hash_sfind(struct uma_hash *hash, uint8_t *data); | static __inline uma_slab_t hash_sfind(struct uma_hash *hash, uint8_t *data); | ||||
void *uma_large_malloc(vm_size_t size, int wait); | void *uma_large_malloc(vm_size_t size, int wait); | ||||
void *uma_large_malloc_domain(vm_size_t size, int domain, int wait); | void *uma_large_malloc_domain(vm_size_t size, int domain, int wait); | ||||
void uma_large_free(uma_slab_t slab); | void uma_large_free(uma_slab_t slab); | ||||
Show All 9 Lines | else \ | ||||
mtx_init(&(k)->uk_lock, (k)->uk_name, \ | mtx_init(&(k)->uk_lock, (k)->uk_name, \ | ||||
"UMA zone", MTX_DEF | MTX_DUPOK); \ | "UMA zone", MTX_DEF | MTX_DUPOK); \ | ||||
} while (0) | } while (0) | ||||
#define KEG_LOCK_FINI(k) mtx_destroy(&(k)->uk_lock) | #define KEG_LOCK_FINI(k) mtx_destroy(&(k)->uk_lock) | ||||
#define KEG_LOCK(k) mtx_lock(&(k)->uk_lock) | #define KEG_LOCK(k) mtx_lock(&(k)->uk_lock) | ||||
#define KEG_UNLOCK(k) mtx_unlock(&(k)->uk_lock) | #define KEG_UNLOCK(k) mtx_unlock(&(k)->uk_lock) | ||||
#define KEG_GET(zone, keg) do { \ | |||||
(keg) = (zone)->uz_keg; \ | |||||
KASSERT((void *)(keg) != (void *)&(zone)->uz_lock, \ | |||||
("%s: Invalid zone %p type", __func__, (zone))); \ | |||||
} while (0) | |||||
#define ZONE_LOCK_INIT(z, lc) \ | #define ZONE_LOCK_INIT(z, lc) \ | ||||
do { \ | do { \ | ||||
if ((lc)) \ | if ((lc)) \ | ||||
mtx_init(&(z)->uz_lock, (z)->uz_name, \ | mtx_init(&(z)->uz_lock, (z)->uz_name, \ | ||||
(z)->uz_name, MTX_DEF | MTX_DUPOK); \ | (z)->uz_name, MTX_DEF | MTX_DUPOK); \ | ||||
else \ | else \ | ||||
mtx_init(&(z)->uz_lock, (z)->uz_name, \ | mtx_init(&(z)->uz_lock, (z)->uz_name, \ | ||||
"UMA zone", MTX_DEF | MTX_DUPOK); \ | "UMA zone", MTX_DEF | MTX_DUPOK); \ | ||||
} while (0) | } while (0) | ||||
#define ZONE_LOCK(z) mtx_lock((z)->uz_lockptr) | #define ZONE_LOCK(z) mtx_lock((z)->uz_lockptr) | ||||
#define ZONE_TRYLOCK(z) mtx_trylock((z)->uz_lockptr) | #define ZONE_TRYLOCK(z) mtx_trylock((z)->uz_lockptr) | ||||
#define ZONE_UNLOCK(z) mtx_unlock((z)->uz_lockptr) | #define ZONE_UNLOCK(z) mtx_unlock((z)->uz_lockptr) | ||||
#define ZONE_LOCK_ASSERT(z) mtx_assert((z)->uz_lockptr, MA_OWNED) | |||||
Done Inline ActionsSeems to be a dup? markj: Seems to be a dup? | |||||
#define ZONE_LOCK_FINI(z) mtx_destroy(&(z)->uz_lock) | #define ZONE_LOCK_FINI(z) mtx_destroy(&(z)->uz_lock) | ||||
#define ZONE_LOCK_ASSERT(z) mtx_assert((z)->uz_lockptr, MA_OWNED) | #define ZONE_LOCK_ASSERT(z) mtx_assert((z)->uz_lockptr, MA_OWNED) | ||||
/* | /* | ||||
* Find a slab within a hash table. This is used for OFFPAGE zones to lookup | * Find a slab within a hash table. This is used for OFFPAGE zones to lookup | ||||
* the slab structure. | * the slab structure. | ||||
* | * | ||||
* Arguments: | * Arguments: | ||||
▲ Show 20 Lines • Show All 53 Lines • Show Last 20 Lines |
Why?