Differential D23738 Diff 68657 sys/sys/smr.h

Changeset View

Standalone View

sys/sys/smr.h

Show All 39 Lines
* Readers synchronize with smr_enter()/exit() and writers may either		* Readers synchronize with smr_enter()/exit() and writers may either
* free directly to a SMR UMA zone or use smr_synchronize or wait.		* free directly to a SMR UMA zone or use smr_synchronize or wait.
*/		*/

/*		/*
* Modular arithmetic for comparing sequence numbers that have		* Modular arithmetic for comparing sequence numbers that have
* potentially wrapped. Copied from tcp_seq.h.		* potentially wrapped. Copied from tcp_seq.h.
*/		*/
#define SMR_SEQ_LT(a, b) ((int32_t)((a)-(b)) < 0)		#define SMR_SEQ_LT(a, b) ((smr_delta_t)((a)-(b)) < 0)
#define SMR_SEQ_LEQ(a, b) ((int32_t)((a)-(b)) <= 0)		#define SMR_SEQ_LEQ(a, b) ((smr_delta_t)((a)-(b)) <= 0)
#define SMR_SEQ_GT(a, b) ((int32_t)((a)-(b)) > 0)		#define SMR_SEQ_GT(a, b) ((smr_delta_t)((a)-(b)) > 0)
#define SMR_SEQ_GEQ(a, b) ((int32_t)((a)-(b)) >= 0)		#define SMR_SEQ_GEQ(a, b) ((smr_delta_t)((a)-(b)) >= 0)
#define SMR_SEQ_DELTA(a, b) ((int32_t)((a)-(b)))		#define SMR_SEQ_DELTA(a, b) ((smr_delta_t)((a)-(b)))
		#define SMR_SEQ_MIN(a, b) (SMR_SEQ_LT((a), (b)) ? (a) : (b))
		#define SMR_SEQ_MAX(a, b) (SMR_SEQ_GT((a), (b)) ? (a) : (b))
		rlibbyUnsubmitted Not Done Inline Actions Ought to add more parens around a and b. rlibby: Ought to add more parens around a and b.
		jeffAuthorUnsubmitted Done Inline Actions still need to do this. jeff: still need to do this.

#define SMR_SEQ_INVALID 0		#define SMR_SEQ_INVALID 0

/* Shared SMR state. */		/* Shared SMR state. */
struct smr_shared {		struct smr_shared {
const char s_name; / Name for debugging/reporting. */		const char s_name; / Name for debugging/reporting. */
smr_seq_t s_wr_seq; /* Current write sequence #. */		smr_seq_t s_wr_seq; /* Current write sequence #. */
smr_seq_t s_rd_seq; /* Minimum observed read sequence. */		smr_seq_t s_rd_seq; /* Minimum observed read sequence. */
};		};
typedef struct smr_shared *smr_shared_t;		typedef struct smr_shared *smr_shared_t;

/* Per-cpu SMR state. */		/* Per-cpu SMR state. */
struct smr {		struct smr {
smr_seq_t c_seq; /* Current observed sequence. */		smr_seq_t c_seq; /* Current observed sequence. */
smr_shared_t c_shared; /* Shared SMR state. */		smr_shared_t c_shared; /* Shared SMR state. */
int c_deferred; /* Deferred advance counter. */		int c_deferred; /* Deferred advance counter. */
		int c_limit; /* Deferred advance limit. */
		int c_flags; /* SMR Configuration */
};		};

		#define SMR_LAZY 0x0001 /* Higher latency write, fast read. */
		#define SMR_DEFERRED 0x0002 /* Aggregate updates to wr_seq. */

#define SMR_ENTERED(smr) \		#define SMR_ENTERED(smr) \
(curthread->td_critnest != 0 && zpcpu_get((smr))->c_seq != SMR_SEQ_INVALID)		(curthread->td_critnest != 0 && zpcpu_get((smr))->c_seq != SMR_SEQ_INVALID)

#define SMR_ASSERT_ENTERED(smr) \		#define SMR_ASSERT_ENTERED(smr) \
KASSERT(SMR_ENTERED(smr), ("Not in smr section"))		KASSERT(SMR_ENTERED(smr), ("Not in smr section"))

#define SMR_ASSERT_NOT_ENTERED(smr) \		#define SMR_ASSERT_NOT_ENTERED(smr) \
KASSERT(!SMR_ENTERED(smr), ("In smr section."));		KASSERT(!SMR_ENTERED(smr), ("In smr section."));
Show All 10 Lines
* serialized - Use while holding a lock that serializes writers. Updates		* serialized - Use while holding a lock that serializes writers. Updates
* are synchronized with readers via included barriers.		* are synchronized with readers via included barriers.
* unserialized - Use after the memory is out of scope and not visible to		* unserialized - Use after the memory is out of scope and not visible to
* readers.		* readers.
*		*
* All acceses include a parameter for an assert to verify the required		* All acceses include a parameter for an assert to verify the required
* synchronization. For example, a writer might use:		* synchronization. For example, a writer might use:
*		*
* smr_serilized_store(pointer, value, mtx_owned(&writelock));		* smr_serialized_store(pointer, value, mtx_owned(&writelock));
*		*
* These are only enabled in INVARIANTS kernels.		* These are only enabled in INVARIANTS kernels.
*/		*/

/* Type restricting pointer access to force smr accessors. */		/* Type restricting pointer access to force smr accessors. */
#define SMR_TYPE_DECLARE(smrtype, type) \		#define SMR_TYPE_DECLARE(smrtype, type) \
typedef struct { \		typedef struct { \
type __ptr; /* Do not access directly */ \		type __ptr; /* Do not access directly */ \
Show All 16 Lines	#define smr_serialized_load(p, ex) ({ \
SMR_ASSERT(ex, "smr_serialized_load"); \		SMR_ASSERT(ex, "smr_serialized_load"); \
(__typeof((p)->__ptr))atomic_load_ptr(&(p)->__ptr); \		(__typeof((p)->__ptr))atomic_load_ptr(&(p)->__ptr); \
})		})

/*		/*
* Store 'v' to an SMR protected pointer while serialized by an		* Store 'v' to an SMR protected pointer while serialized by an
* external mechanism. 'ex' should contain an assert that the		* external mechanism. 'ex' should contain an assert that the
* external mechanism is held. i.e. mtx_owned()		* external mechanism is held. i.e. mtx_owned()
		*
		* Writers that are serialized with mutual exclusion or on a single
		* thread should use smr_serialized_store() rather than swap.
*/		*/
#define smr_serialized_store(p, v, ex) do { \		#define smr_serialized_store(p, v, ex) do { \
SMR_ASSERT(ex, "smr_serialized_store"); \		SMR_ASSERT(ex, "smr_serialized_store"); \
__typeof((p)->__ptr) _v = (v); \		__typeof((p)->__ptr) _v = (v); \
atomic_store_rel_ptr((uintptr_t *)&(p)->__ptr, (uintptr_t)_v); \		atomic_store_rel_ptr((uintptr_t *)&(p)->__ptr, (uintptr_t)_v); \
} while (0)		} while (0)

/*		/*
* swap 'v' with an SMR protected pointer and return the old value		* swap 'v' with an SMR protected pointer and return the old value
* while serialized by an external mechanism. 'ex' should contain		* while serialized by an external mechanism. 'ex' should contain
* an assert that the external mechanism is provided. i.e. mtx_owned()		* an assert that the external mechanism is provided. i.e. mtx_owned()
		*
		* Swap permits multiple writers to update a pointer concurrently.
*/		*/
#define smr_serialized_swap(p, v, ex) ({ \		#define smr_serialized_swap(p, v, ex) ({ \
SMR_ASSERT(ex, "smr_serialized_swap"); \		SMR_ASSERT(ex, "smr_serialized_swap"); \
__typeof((p)->__ptr) _v = (v); \		__typeof((p)->__ptr) _v = (v); \
/* Release barrier guarantees contents are visible to reader */ \		/* Release barrier guarantees contents are visible to reader */ \
atomic_thread_fence_rel(); \		atomic_thread_fence_rel(); \
(__typeof((p)->__ptr))atomic_swap_ptr( \		(__typeof((p)->__ptr))atomic_swap_ptr( \
(uintptr_t *)&(p)->__ptr, (uintptr_t)_v); \		(uintptr_t *)&(p)->__ptr, (uintptr_t)_v); \
Show All 16 Lines
*/		*/
#define smr_unserialized_store(p, v, ex) do { \		#define smr_unserialized_store(p, v, ex) do { \
SMR_ASSERT(ex, "smr_unserialized_store"); \		SMR_ASSERT(ex, "smr_unserialized_store"); \
__typeof((p)->__ptr) _v = (v); \		__typeof((p)->__ptr) _v = (v); \
atomic_store_ptr((uintptr_t *)&(p)->__ptr, (uintptr_t)_v); \		atomic_store_ptr((uintptr_t *)&(p)->__ptr, (uintptr_t)_v); \
} while (0)		} while (0)

/*		/*
* Return the current write sequence number.		* Return the current write sequence number. This is not the same as the
		* current goal which may be in the future.
*/		*/
static inline smr_seq_t		static inline smr_seq_t
smr_shared_current(smr_shared_t s)		smr_shared_current(smr_shared_t s)
{		{

return (atomic_load_int(&s->s_wr_seq));		return (atomic_load_int(&s->s_wr_seq));
}		}

static inline smr_seq_t		static inline smr_seq_t
smr_current(smr_t smr)		smr_current(smr_t smr)
{		{

return (smr_shared_current(zpcpu_get(smr)->c_shared));		return (smr_shared_current(zpcpu_get(smr)->c_shared));
}		}

/*		/*
* Enter a read section.		* Enter a read section.
*/		*/
static inline void		static inline void
smr_enter(smr_t smr)		smr_enter(smr_t smr)
{		{

critical_enter();		critical_enter();
smr = zpcpu_get(smr);		smr = zpcpu_get(smr);
		KASSERT((smr->c_flags & SMR_LAZY) == 0,
		("smr_enter(%s) lazy smr.", smr->c_shared->s_name));
KASSERT(smr->c_seq == 0,		KASSERT(smr->c_seq == 0,
("smr_enter(%s) does not support recursion.",		("smr_enter(%s) does not support recursion.",
smr->c_shared->s_name));		smr->c_shared->s_name));

/*		/*
* Store the current observed write sequence number in our		* Store the current observed write sequence number in our
* per-cpu state so that it can be queried via smr_poll().		* per-cpu state so that it can be queried via smr_poll().
* Frees that are newer than this stored value will be		* Frees that are newer than this stored value will be
Show All 17 Lines
* Exit a read section.		* Exit a read section.
*/		*/
static inline void		static inline void
smr_exit(smr_t smr)		smr_exit(smr_t smr)
{		{

smr = zpcpu_get(smr);		smr = zpcpu_get(smr);
CRITICAL_ASSERT(curthread);		CRITICAL_ASSERT(curthread);
		KASSERT((smr->c_flags & SMR_LAZY) == 0,
		("smr_exit(%s) lazy smr.", smr->c_shared->s_name));
KASSERT(smr->c_seq != SMR_SEQ_INVALID,		KASSERT(smr->c_seq != SMR_SEQ_INVALID,
("smr_exit(%s) not in a smr section.", smr->c_shared->s_name));		("smr_exit(%s) not in a smr section.", smr->c_shared->s_name));

/*		/*
* Clear the recorded sequence number. This allows poll() to		* Clear the recorded sequence number. This allows poll() to
* detect CPUs not in read sections.		* detect CPUs not in read sections.
*		*
* Use release semantics to retire any stores before the sequence		* Use release semantics to retire any stores before the sequence
* number is cleared.		* number is cleared.
*/		*/
atomic_store_rel_int(&smr->c_seq, SMR_SEQ_INVALID);		atomic_store_rel_int(&smr->c_seq, SMR_SEQ_INVALID);
critical_exit();		critical_exit();
}		}

/*		/*
* Advances the write sequence number. Returns the sequence number		* Enter a lazy smr section. This is used for read-mostly state that
* required to ensure that all modifications are visible to readers.		* can tolerate a high free latency.
*/		*/
smr_seq_t smr_advance(smr_t smr);		static inline void
		smr_lazy_enter(smr_t smr)
		{

		critical_enter();
		smr = zpcpu_get(smr);
		KASSERT((smr->c_flags & SMR_LAZY) != 0,
		("smr_lazy_enter(%s) non-lazy smr.", smr->c_shared->s_name));
		KASSERT(smr->c_seq == 0,
		("smr_lazy_enter(%s) does not support recursion.",
		smr->c_shared->s_name));

/*		/*
* Advances the write sequence number only after N calls. Returns		* This needs no serialization. If an interrupt occurs before we
* the correct goal for a wr_seq that has not yet occurred. Used to		* assign sr_seq to c_seq any speculative loads will be discarded.
* minimize shared cacheline invalidations for frequent writers.		* If we assign a stale wr_seq value due to interrupt we use the
		* same algorithm that renders smr_enter() safe.
*/		*/
smr_seq_t smr_advance_deferred(smr_t smr, int limit);		smr->c_seq = smr_shared_current(smr->c_shared);
		}

/*		/*
		* Exit a lazy smr section. This is used for read-mostly state that
		* can tolerate a high free latency.
		*/
		static inline void
		smr_lazy_exit(smr_t smr)
		{

		smr = zpcpu_get(smr);
		CRITICAL_ASSERT(curthread);
		KASSERT((smr->c_flags & SMR_LAZY) != 0,
		("smr_lazy_enter(%s) non-lazy smr.", smr->c_shared->s_name));
		KASSERT(smr->c_seq != SMR_SEQ_INVALID,
		("smr_lazy_exit(%s) not in a smr section.", smr->c_shared->s_name));

		/*
		* All loads/stores must be retired before the sequence becomes
		* visible. The fence compiles away on amd64. Another
		* alternative would be to omit the fence but store the exit
		* time and wait 1 tick longer.
		*/
		atomic_thread_fence_rel();
		smr->c_seq = SMR_SEQ_INVALID;
		critical_exit();
		}

		/*
		* Advances the write sequence number. Returns the sequence number
		* required to ensure that all modifications are visible to readers.
		*/
		smr_seq_t smr_advance(smr_t smr);

		/*
* Returns true if a goal sequence has been reached. If		* Returns true if a goal sequence has been reached. If
* wait is true this will busy loop until success.		* wait is true this will busy loop until success.
*/		*/
bool smr_poll(smr_t smr, smr_seq_t goal, bool wait);		bool smr_poll(smr_t smr, smr_seq_t goal, bool wait);

/* Create a new SMR context. */		/* Create a new SMR context. */
smr_t smr_create(const char *name);		smr_t smr_create(const char *name, int limit, int flags);

		/* Destroy the context. */
void smr_destroy(smr_t smr);		void smr_destroy(smr_t smr);

/*		/*
* Blocking wait for all readers to observe 'goal'.		* Blocking wait for all readers to observe 'goal'.
*/		*/
static inline bool		static inline bool
smr_wait(smr_t smr, smr_seq_t goal)		smr_wait(smr_t smr, smr_seq_t goal)
{		{
Show All 23 Lines