Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/uma_int.h
Show First 20 Lines • Show All 465 Lines • ▼ Show 20 Lines | |||||
struct uma_zone_domain { | struct uma_zone_domain { | ||||
struct uma_bucketlist uzd_buckets; /* full buckets */ | struct uma_bucketlist uzd_buckets; /* full buckets */ | ||||
uma_bucket_t uzd_cross; /* Fills from cross buckets. */ | uma_bucket_t uzd_cross; /* Fills from cross buckets. */ | ||||
long uzd_nitems; /* total item count */ | long uzd_nitems; /* total item count */ | ||||
long uzd_imax; /* maximum item count this period */ | long uzd_imax; /* maximum item count this period */ | ||||
long uzd_imin; /* minimum item count this period */ | long uzd_imin; /* minimum item count this period */ | ||||
long uzd_wss; /* working set size estimate */ | long uzd_wss; /* working set size estimate */ | ||||
smr_seq_t uzd_seq; /* Lowest queued seq. */ | |||||
struct mtx uzd_lock; /* Lock for the domain */ | |||||
} __aligned(CACHE_LINE_SIZE); | } __aligned(CACHE_LINE_SIZE); | ||||
typedef struct uma_zone_domain * uma_zone_domain_t; | typedef struct uma_zone_domain * uma_zone_domain_t; | ||||
/* | /* | ||||
* Zone structure - per memory type. | * Zone structure - per memory type. | ||||
*/ | */ | ||||
struct uma_zone { | struct uma_zone { | ||||
/* Offset 0, used in alloc/free fast/medium fast path and const. */ | /* Offset 0, used in alloc/free fast/medium fast path and const. */ | ||||
uma_keg_t uz_keg; /* This zone's keg if !CACHE */ | |||||
struct uma_zone_domain *uz_domain; /* per-domain buckets */ | |||||
uint32_t uz_flags; /* Flags inherited from kegs */ | uint32_t uz_flags; /* Flags inherited from kegs */ | ||||
uint32_t uz_size; /* Size inherited from kegs */ | uint32_t uz_size; /* Size inherited from kegs */ | ||||
uma_ctor uz_ctor; /* Constructor for each allocation */ | uma_ctor uz_ctor; /* Constructor for each allocation */ | ||||
uma_dtor uz_dtor; /* Destructor */ | uma_dtor uz_dtor; /* Destructor */ | ||||
smr_t uz_smr; /* Safe memory reclaim context. */ | smr_t uz_smr; /* Safe memory reclaim context. */ | ||||
uint64_t uz_max_items; /* Maximum number of items to alloc */ | uint64_t uz_max_items; /* Maximum number of items to alloc */ | ||||
uint64_t uz_bkt_max; /* Maximum bucket cache size */ | |||||
markj: I dislike the mix of uz_bkt_ and uz_bucket_ in field names.
You might also amend the comment… | |||||
Done Inline ActionsI will go ahead and rename it. I thought it odd as well. jeff: I will go ahead and rename it. I thought it odd as well. | |||||
uint32_t uz_sleepers; /* Threads sleeping on limit */ | uint32_t uz_sleepers; /* Threads sleeping on limit */ | ||||
uint16_t uz_bucket_size; /* Number of items in full bucket */ | uint16_t uz_bucket_size; /* Number of items in full bucket */ | ||||
uint16_t uz_bucket_size_max; /* Maximum number of bucket items */ | uint16_t uz_bucket_size_max; /* Maximum number of bucket items */ | ||||
counter_u64_t uz_xdomain; /* Total number of cross-domain frees */ | |||||
/* Offset 64, used in bucket replenish. */ | /* Offset 64, used in bucket replenish. */ | ||||
uma_keg_t uz_keg; /* This zone's keg if !CACHE */ | |||||
uma_import uz_import; /* Import new memory to cache. */ | uma_import uz_import; /* Import new memory to cache. */ | ||||
uma_release uz_release; /* Release memory from cache. */ | uma_release uz_release; /* Release memory from cache. */ | ||||
void *uz_arg; /* Import/release argument. */ | void *uz_arg; /* Import/release argument. */ | ||||
uma_init uz_init; /* Initializer for each item */ | uma_init uz_init; /* Initializer for each item */ | ||||
uma_fini uz_fini; /* Finalizer for each item. */ | uma_fini uz_fini; /* Finalizer for each item. */ | ||||
void *uz_spare1; | volatile uint64_t uz_items; /* Total items count & sleepers */ | ||||
jeffAuthorUnsubmitted Done Inline ActionsThis one pains me slightly since it is written so often for mbufs. Maybe it should just be with garbage at the end still. jeff: This one pains me slightly since it is written so often for mbufs. Maybe it should just be… | |||||
uint64_t uz_bkt_count; /* Items in bucket cache */ | uint64_t uz_sleeps; /* Total number of alloc sleeps */ | ||||
uint64_t uz_bkt_max; /* Maximum bucket cache size */ | |||||
/* Offset 128 Rare. */ | /* Offset 128 Rare stats, misc read-only. */ | ||||
/* | |||||
* The lock is placed here to avoid adjacent line prefetcher | |||||
* in fast paths and to take up space near infrequently accessed | |||||
* members to reduce alignment overhead. | |||||
*/ | |||||
struct mtx uz_lock; /* Lock for the zone */ | |||||
LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ | LIST_ENTRY(uma_zone) uz_link; /* List of all zones in keg */ | ||||
const char *uz_name; /* Text name of the zone */ | |||||
/* The next two fields are used to print a rate-limited warnings. */ | |||||
const char *uz_warning; /* Warning to print on failure */ | |||||
struct timeval uz_ratecheck; /* Warnings rate-limiting */ | |||||
struct task uz_maxaction; /* Task to run when at limit */ | |||||
uint16_t uz_bucket_size_min; /* Min number of items in bucket */ | |||||
struct mtx_padalign uz_cross_lock; /* Cross domain free lock */ | |||||
/* Offset 256+, stats and misc. */ | |||||
counter_u64_t uz_allocs; /* Total number of allocations */ | counter_u64_t uz_allocs; /* Total number of allocations */ | ||||
counter_u64_t uz_frees; /* Total number of frees */ | counter_u64_t uz_frees; /* Total number of frees */ | ||||
counter_u64_t uz_fails; /* Total number of alloc failures */ | counter_u64_t uz_fails; /* Total number of alloc failures */ | ||||
uint64_t uz_sleeps; /* Total number of alloc sleeps */ | const char *uz_name; /* Text name of the zone */ | ||||
uint64_t uz_xdomain; /* Total number of cross-domain frees */ | |||||
volatile uint64_t uz_items; /* Total items count & sleepers */ | |||||
char *uz_ctlname; /* sysctl safe name string. */ | char *uz_ctlname; /* sysctl safe name string. */ | ||||
struct sysctl_oid *uz_oid; /* sysctl oid pointer. */ | |||||
int uz_namecnt; /* duplicate name count. */ | int uz_namecnt; /* duplicate name count. */ | ||||
uint16_t uz_bucket_size_min; /* Min number of items in bucket */ | |||||
uint16_t uz_pad0; | |||||
/* Offset 192, rare read-only. */ | |||||
struct sysctl_oid *uz_oid; /* sysctl oid pointer. */ | |||||
const char *uz_warning; /* Warning to print on failure */ | |||||
struct timeval uz_ratecheck; /* Warnings rate-limiting */ | |||||
struct task uz_maxaction; /* Task to run when at limit */ | |||||
/* Offset 256. */ | |||||
struct mtx uz_cross_lock; /* Cross domain free lock */ | |||||
jeffAuthorUnsubmitted Done Inline ActionsI was hoping to fit this in 256b total but it just doesn't quite work. This means there's a full 64 byte unused cacheline between here and the cpu array :( jeff: I was hoping to fit this in 256b total but it just doesn't quite work.
This means there's a… | |||||
/* | /* | ||||
* This HAS to be the last item because we adjust the zone size | * This HAS to be the last item because we adjust the zone size | ||||
* based on NCPU and then allocate the space for the zones. | * based on NCPU and then allocate the space for the zones. | ||||
*/ | */ | ||||
struct uma_cache uz_cpu[]; /* Per cpu caches */ | struct uma_cache uz_cpu[]; /* Per cpu caches */ | ||||
/* uz_domain follows here. */ | /* domains follow here. */ | ||||
}; | }; | ||||
/* | /* | ||||
* Macros for interpreting the uz_items field. 20 bits of sleeper count | * Macros for interpreting the uz_items field. 20 bits of sleeper count | ||||
* and 44 bit of item count. | * and 44 bit of item count. | ||||
*/ | */ | ||||
#define UZ_ITEMS_SLEEPER_SHIFT 44LL | #define UZ_ITEMS_SLEEPER_SHIFT 44LL | ||||
#define UZ_ITEMS_SLEEPERS_MAX ((1 << (64 - UZ_ITEMS_SLEEPER_SHIFT)) - 1) | #define UZ_ITEMS_SLEEPERS_MAX ((1 << (64 - UZ_ITEMS_SLEEPER_SHIFT)) - 1) | ||||
Show All 28 Lines | |||||
#define KEG_LOCK_FINI(k, d) mtx_destroy(KEG_LOCKPTR(k, d)) | #define KEG_LOCK_FINI(k, d) mtx_destroy(KEG_LOCKPTR(k, d)) | ||||
#define KEG_LOCK(k, d) \ | #define KEG_LOCK(k, d) \ | ||||
({ mtx_lock(KEG_LOCKPTR(k, d)); KEG_LOCKPTR(k, d); }) | ({ mtx_lock(KEG_LOCKPTR(k, d)); KEG_LOCKPTR(k, d); }) | ||||
#define KEG_UNLOCK(k, d) mtx_unlock(KEG_LOCKPTR(k, d)) | #define KEG_UNLOCK(k, d) mtx_unlock(KEG_LOCKPTR(k, d)) | ||||
#define KEG_LOCK_ASSERT(k, d) mtx_assert(KEG_LOCKPTR(k, d), MA_OWNED) | #define KEG_LOCK_ASSERT(k, d) mtx_assert(KEG_LOCKPTR(k, d), MA_OWNED) | ||||
#define KEG_GET(zone, keg) do { \ | #define KEG_GET(zone, keg) do { \ | ||||
(keg) = (zone)->uz_keg; \ | (keg) = (zone)->uz_keg; \ | ||||
KASSERT((void *)(keg) != (void *)&(zone)->uz_lock, \ | KASSERT((void *)(keg) != NULL, \ | ||||
("%s: Invalid zone %p type", __func__, (zone))); \ | ("%s: Invalid zone %p type", __func__, (zone))); \ | ||||
} while (0) | } while (0) | ||||
#define ZONE_LOCK_INIT(z, lc) \ | /* Domains are contiguous after the last CPU */ | ||||
#define ZDOM_GET(z, n) \ | |||||
(&((uma_zone_domain_t)&(z)->uz_cpu[mp_maxid + 1])[n]) | |||||
#define ZDOM_GET_LOCKED(z, n) \ | |||||
jeffAuthorUnsubmitted Done Inline ActionsUnused jeff: Unused | |||||
({ uma_zone_domain_t zd = ZDOM_GET((z), (n)); ZDOM_LOCK(zd); zd;}) | |||||
#define ZDOM_LOCK_INIT(z, zdom, lc) \ | |||||
do { \ | do { \ | ||||
if ((lc)) \ | if ((lc)) \ | ||||
mtx_init(&(z)->uz_lock, (z)->uz_name, \ | mtx_init(&(zdom)->uzd_lock, (z)->uz_name, \ | ||||
(z)->uz_name, MTX_DEF | MTX_DUPOK); \ | (z)->uz_name, MTX_DEF | MTX_DUPOK); \ | ||||
else \ | else \ | ||||
mtx_init(&(z)->uz_lock, (z)->uz_name, \ | mtx_init(&(zdom)->uzd_lock, (z)->uz_name, \ | ||||
"UMA zone", MTX_DEF | MTX_DUPOK); \ | "UMA zone", MTX_DEF | MTX_DUPOK); \ | ||||
} while (0) | } while (0) | ||||
#define ZDOM_LOCK_FINI(z) mtx_destroy(&(z)->uzd_lock) | |||||
#define ZDOM_LOCK_ASSERT(z) mtx_assert(&(z)->uzd_lock, MA_OWNED) | |||||
#define ZONE_LOCK(z) mtx_lock(&(z)->uz_lock) | #define ZDOM_LOCK(z) mtx_lock(&(z)->uzd_lock) | ||||
#define ZONE_TRYLOCK(z) mtx_trylock(&(z)->uz_lock) | #define ZDOM_OWNED(z) (mtx_owner(&(z)->uzd_lock) != NULL) | ||||
#define ZONE_UNLOCK(z) mtx_unlock(&(z)->uz_lock) | #define ZDOM_UNLOCK(z) mtx_unlock(&(z)->uzd_lock) | ||||
#define ZONE_LOCK_FINI(z) mtx_destroy(&(z)->uz_lock) | |||||
#define ZONE_LOCK_ASSERT(z) mtx_assert(&(z)->uz_lock, MA_OWNED) | #define ZONE_LOCK(z) ZDOM_LOCK(ZDOM_GET((z), 0)) | ||||
#define ZONE_UNLOCK(z) ZDOM_UNLOCK(ZDOM_GET((z), 0)) | |||||
#define ZONE_CROSS_LOCK_INIT(z) \ | #define ZONE_CROSS_LOCK_INIT(z) \ | ||||
mtx_init(&(z)->uz_cross_lock, "UMA Cross", NULL, MTX_DEF) | mtx_init(&(z)->uz_cross_lock, "UMA Cross", NULL, MTX_DEF) | ||||
#define ZONE_CROSS_LOCK(z) mtx_lock(&(z)->uz_cross_lock) | #define ZONE_CROSS_LOCK(z) mtx_lock(&(z)->uz_cross_lock) | ||||
#define ZONE_CROSS_UNLOCK(z) mtx_unlock(&(z)->uz_cross_lock) | #define ZONE_CROSS_UNLOCK(z) mtx_unlock(&(z)->uz_cross_lock) | ||||
#define ZONE_CROSS_LOCK_FINI(z) mtx_destroy(&(z)->uz_cross_lock) | #define ZONE_CROSS_LOCK_FINI(z) mtx_destroy(&(z)->uz_cross_lock) | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 90 Lines • Show Last 20 Lines |
I dislike the mix of uz_bkt_ and uz_bucket_ in field names.
You might also amend the comment, "Maxium per-domain bucket cache size".