Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/subr_smr.c
Show First 20 Lines • Show All 140 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* Any writer is free to advance wr seq at any time however it may busy | * Any writer is free to advance wr seq at any time however it may busy | ||||
* poll in pathological cases. | * poll in pathological cases. | ||||
*/ | */ | ||||
static uma_zone_t smr_shared_zone; | static uma_zone_t smr_shared_zone; | ||||
static uma_zone_t smr_zone; | static uma_zone_t smr_zone; | ||||
enum { | |||||
BOOT_COLD = 1, | |||||
BOOT_ZONE = 2, | |||||
BOOT_PCPU = 3, | |||||
} booted = BOOT_COLD; | |||||
static smr_shared_t smr_init_list; | |||||
#ifndef INVARIANTS | #ifndef INVARIANTS | ||||
#define SMR_SEQ_INIT 1 /* All valid sequence numbers are odd. */ | #define SMR_SEQ_INIT 1 /* All valid sequence numbers are odd. */ | ||||
#define SMR_SEQ_INCR 2 | #define SMR_SEQ_INCR 2 | ||||
/* | /* | ||||
* SMR_SEQ_MAX_DELTA is the maximum distance allowed between rd_seq and | * SMR_SEQ_MAX_DELTA is the maximum distance allowed between rd_seq and | ||||
* wr_seq. For the modular arithmetic to work a value of UNIT_MAX / 2 | * wr_seq. For the modular arithmetic to work a value of UNIT_MAX / 2 | ||||
* would be possible but it is checked after we increment the wr_seq so | * would be possible but it is checked after we increment the wr_seq so | ||||
▲ Show 20 Lines • Show All 420 Lines • ▼ Show 20 Lines | out: | ||||
* Serialize with smr_advance()/smr_exit(). The caller is now free | * Serialize with smr_advance()/smr_exit(). The caller is now free | ||||
* to modify memory as expected. | * to modify memory as expected. | ||||
*/ | */ | ||||
atomic_thread_fence_acq(); | atomic_thread_fence_acq(); | ||||
return (success); | return (success); | ||||
} | } | ||||
static void | |||||
smr_init_cpu(smr_shared_t s, int cpu, int limit, int flags) | |||||
{ | |||||
smr_t c; | |||||
c = zpcpu_get_cpu(s->s_cpu, cpu); | |||||
c->c_seq = SMR_SEQ_INVALID; | |||||
c->c_shared = s; | |||||
c->c_deferred = 0; | |||||
c->c_limit = limit; | |||||
c->c_flags = flags; | |||||
} | |||||
smr_t | smr_t | ||||
smr_create(const char *name, int limit, int flags) | smr_create(const char *name, int limit, int flags) | ||||
{ | { | ||||
smr_t smr, c; | smr_t smr; | ||||
smr_shared_t s; | smr_shared_t s; | ||||
int i; | int i; | ||||
KASSERT(booted >= BOOT_ZONE, | |||||
("%s: SMR is not yet initialized", __func__)); | |||||
s = uma_zalloc(smr_shared_zone, M_WAITOK); | s = uma_zalloc(smr_shared_zone, M_WAITOK); | ||||
smr = uma_zalloc_pcpu(smr_zone, M_WAITOK); | smr = uma_zalloc_pcpu(smr_zone, M_WAITOK); | ||||
s->s_name = name; | s->s_name = name; | ||||
s->s_rd_seq = s->s_wr.seq = SMR_SEQ_INIT; | s->s_rd_seq = s->s_wr.seq = SMR_SEQ_INIT; | ||||
s->s_wr.ticks = ticks; | s->s_wr.ticks = ticks; | ||||
s->s_cpu = smr; | |||||
/* Initialize all CPUS, not just those running. */ | smr_init_cpu(s, 0, limit, flags); | ||||
for (i = 0; i <= mp_maxid; i++) { | if (booted < BOOT_PCPU) { | ||||
c = zpcpu_get_cpu(smr, i); | /* | ||||
c->c_seq = SMR_SEQ_INVALID; | * Per-CPU memory for APs might not be accessible at this point. | ||||
c->c_shared = s; | * Defer initialization of the rest of the CPU sections. | ||||
c->c_deferred = 0; | */ | ||||
c->c_limit = limit; | s->s_next = smr_init_list; | ||||
c->c_flags = flags; | smr_init_list = s; | ||||
} | } else { | ||||
/* Initialize all CPUs, not just those running. */ | |||||
for (i = 1; i <= mp_maxid; i++) | |||||
smr_init_cpu(s, i, limit, flags); | |||||
atomic_thread_fence_seq_cst(); | atomic_thread_fence_seq_cst(); | ||||
} | |||||
return (smr); | return (smr); | ||||
} | } | ||||
void | void | ||||
smr_destroy(smr_t smr) | smr_destroy(smr_t smr) | ||||
{ | { | ||||
smr_synchronize(smr); | smr_synchronize(smr); | ||||
uma_zfree(smr_shared_zone, smr->c_shared); | uma_zfree(smr_shared_zone, smr->c_shared); | ||||
uma_zfree_pcpu(smr_zone, smr); | uma_zfree_pcpu(smr_zone, smr); | ||||
} | } | ||||
/* | /* | ||||
* Initialize the UMA slab zone. | * Initialize the UMA slab zone. | ||||
*/ | */ | ||||
void | static void | ||||
smr_init(void) | smr_init(void *arg __unused) | ||||
{ | { | ||||
rlibby: Style? | |||||
smr_shared_zone = uma_zcreate("SMR SHARED", sizeof(struct smr_shared), | smr_shared_zone = uma_zcreate("SMR SHARED", sizeof(struct smr_shared), | ||||
NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, 0); | NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, 0); | ||||
smr_zone = uma_zcreate("SMR CPU", sizeof(struct smr), | smr_zone = uma_zcreate("SMR CPU", sizeof(struct smr), | ||||
NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, UMA_ZONE_PCPU); | NULL, NULL, NULL, NULL, (CACHE_LINE_SIZE * 2) - 1, UMA_ZONE_PCPU); | ||||
booted = BOOT_ZONE; | |||||
} | } | ||||
SYSINIT(smr_init, SI_SUB_SMR, SI_ORDER_FIRST, smr_init, NULL); | |||||
static void | |||||
smr_init2(void *arg __unused) | |||||
{ | |||||
smr_shared_t s; | |||||
smr_t c0; | |||||
int i; | |||||
/* | |||||
* Complete initialization of per-CPU sections, started in smr_create(). | |||||
*/ | |||||
for (s = smr_init_list; s != NULL; s = s->s_next) { | |||||
c0 = zpcpu_get_cpu(s->s_cpu, 0); | |||||
for (i = 1; i <= mp_maxid; i++) | |||||
smr_init_cpu(s, i, c0->c_limit, c0->c_flags); | |||||
} | |||||
booted = BOOT_PCPU; | |||||
} | |||||
SYSINIT(smr_init2, SI_SUB_CPU, SI_ORDER_ANY, smr_init2, NULL); |
Style?