Changeset View
Changeset View
Standalone View
Standalone View
head/sys/sys/smr.h
Show First 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | #define SMR_ENTERED(smr) \ | ||||
(curthread->td_critnest != 0 && zpcpu_get((smr))->c_seq != SMR_SEQ_INVALID) | (curthread->td_critnest != 0 && zpcpu_get((smr))->c_seq != SMR_SEQ_INVALID) | ||||
#define SMR_ASSERT_ENTERED(smr) \ | #define SMR_ASSERT_ENTERED(smr) \ | ||||
KASSERT(SMR_ENTERED(smr), ("Not in smr section")) | KASSERT(SMR_ENTERED(smr), ("Not in smr section")) | ||||
#define SMR_ASSERT_NOT_ENTERED(smr) \ | #define SMR_ASSERT_NOT_ENTERED(smr) \ | ||||
KASSERT(!SMR_ENTERED(smr), ("In smr section.")); | KASSERT(!SMR_ENTERED(smr), ("In smr section.")); | ||||
#define SMR_ASSERT(ex, fn) \ | |||||
KASSERT((ex), (fn ": Assertion " #ex " failed at %s:%d", __FILE__, __LINE__)) | |||||
/* | |||||
* SMR Accessors are meant to provide safe access to SMR protected | |||||
* pointers and prevent misuse and accidental access. | |||||
* | |||||
* Accessors are grouped by type: | |||||
* entered - Use while in a read section (between smr_enter/smr_exit()) | |||||
* serialized - Use while holding a lock that serializes writers. Updates | |||||
* are synchronized with readers via included barriers. | |||||
* unserialized - Use after the memory is out of scope and not visible to | |||||
* readers. | |||||
* | |||||
* All acceses include a parameter for an assert to verify the required | |||||
* synchronization. For example, a writer might use: | |||||
* | |||||
* smr_serilized_store(pointer, value, mtx_owned(&writelock)); | |||||
* | |||||
* These are only enabled in INVARIANTS kernels. | |||||
*/ | |||||
/* Type restricting pointer access to force smr accessors. */ | |||||
#define SMR_TYPE_DECLARE(smrtype, type) \ | |||||
typedef struct { \ | |||||
type __ptr; /* Do not access directly */ \ | |||||
} smrtype | |||||
/* | |||||
* Read from an SMR protected pointer while in a read section. | |||||
*/ | |||||
#define smr_entered_load(p, smr) ({ \ | |||||
SMR_ASSERT(SMR_ENTERED((smr)), "smr_entered_load"); \ | |||||
(__typeof((p)->__ptr))atomic_load_acq_ptr((uintptr_t *)&(p)->__ptr); \ | |||||
}) | |||||
/* | |||||
* Read from an SMR protected pointer while serialized by an | |||||
* external mechanism. 'ex' should contain an assert that the | |||||
* external mechanism is held. i.e. mtx_owned() | |||||
*/ | |||||
#define smr_serialized_load(p, ex) ({ \ | |||||
SMR_ASSERT(ex, "smr_serialized_load"); \ | |||||
(__typeof((p)->__ptr))atomic_load_ptr((uintptr_t *)&(p)->__ptr);\ | |||||
}) | |||||
/* | |||||
* Store 'v' to an SMR protected pointer while serialized by an | |||||
* external mechanism. 'ex' should contain an assert that the | |||||
* external mechanism is held. i.e. mtx_owned() | |||||
*/ | |||||
#define smr_serialized_store(p, v, ex) do { \ | |||||
SMR_ASSERT(ex, "smr_serialized_store"); \ | |||||
__typeof((p)->__ptr) _v = (v); \ | |||||
atomic_store_rel_ptr((uintptr_t *)&(p)->__ptr, (uintptr_t)_v); \ | |||||
} while (0) | |||||
/* | |||||
* swap 'v' with an SMR protected pointer and return the old value | |||||
* while serialized by an external mechanism. 'ex' should contain | |||||
* an assert that the external mechanism is provided. i.e. mtx_owned() | |||||
*/ | |||||
#define smr_serialized_swap(p, v, ex) ({ \ | |||||
SMR_ASSERT(ex, "smr_serialized_swap"); \ | |||||
__typeof((p)->__ptr) _v = (v); \ | |||||
/* Release barrier guarantees contents are visible to reader */ \ | |||||
atomic_thread_fence_rel(); \ | |||||
(__typeof((p)->__ptr))atomic_swap_ptr( \ | |||||
(uintptr_t *)&(p)->__ptr, (uintptr_t)_v); \ | |||||
}) | |||||
/* | |||||
* Read from an SMR protected pointer when no serialization is required | |||||
* such as in the destructor callback or when the caller guarantees other | |||||
* synchronization. | |||||
*/ | |||||
#define smr_unserialized_load(p, ex) ({ \ | |||||
SMR_ASSERT(ex, "smr_unserialized_load"); \ | |||||
(__typeof((p)->__ptr))atomic_load_ptr((uintptr_t *)&(p)->__ptr);\ | |||||
}) | |||||
/* | |||||
* Store to an SMR protected pointer when no serialiation is required | |||||
* such as in the destructor callback or when the caller guarantees other | |||||
* synchronization. | |||||
*/ | |||||
#define smr_unserialized_store(p, v, ex) do { \ | |||||
SMR_ASSERT(ex, "smr_unserialized_store"); \ | |||||
__typeof((p)->__ptr) _v = (v); \ | |||||
atomic_store_ptr((uintptr_t *)&(p)->__ptr, (uintptr_t)_v); \ | |||||
} while (0) | |||||
/* | /* | ||||
* Return the current write sequence number. | * Return the current write sequence number. | ||||
*/ | */ | ||||
static inline smr_seq_t | static inline smr_seq_t | ||||
smr_shared_current(smr_shared_t s) | smr_shared_current(smr_shared_t s) | ||||
{ | { | ||||
return (atomic_load_int(&s->s_wr_seq)); | return (atomic_load_int(&s->s_wr_seq)); | ||||
▲ Show 20 Lines • Show All 117 Lines • Show Last 20 Lines |