Index: sys/kern/subr_smr.c =================================================================== --- sys/kern/subr_smr.c +++ sys/kern/subr_smr.c @@ -195,7 +195,7 @@ * odd and an observed value of 0 in a particular CPU means * it is not currently in a read section. */ - s = smr->c_shared; + s = zpcpu_get(smr)->c_shared; goal = atomic_fetchadd_int(&s->s_wr_seq, SMR_SEQ_INCR) + SMR_SEQ_INCR; /* @@ -209,6 +209,26 @@ return (goal); } +smr_seq_t +smr_advance_deferred(smr_t smr, int limit) +{ + smr_seq_t goal; + smr_t csmr; + + critical_enter(); + csmr = zpcpu_get(smr); + if (++csmr->c_deferred >= limit) { + goal = SMR_SEQ_INVALID; + csmr->c_deferred = 0; + } else + goal = smr_shared_current(csmr->c_shared) + SMR_SEQ_INCR; + critical_exit(); + if (goal != SMR_SEQ_INVALID) + return (goal); + + return (smr_advance(smr)); +} + /* * Poll to determine whether all readers have observed the 'goal' write * sequence number. @@ -242,14 +262,23 @@ */ success = true; critical_enter(); - s = smr->c_shared; + s = zpcpu_get(smr)->c_shared; /* * Acquire barrier loads s_wr_seq after s_rd_seq so that we can not * observe an updated read sequence that is larger than write. */ s_rd_seq = atomic_load_acq_int(&s->s_rd_seq); - s_wr_seq = smr_current(smr); + s_wr_seq = smr_shared_current(s); + + /* + * This may have come from a deferred advance. Consider one + * increment past the current wr_seq valid and make sure we + * have advanced far enough to succeed. + */ + if (goal == s_wr_seq + SMR_SEQ_INCR) + s_wr_seq = atomic_fetchadd_int(&s->s_wr_seq, + SMR_SEQ_INCR) + SMR_SEQ_INCR; /* * Detect whether the goal is valid and has already been observed. Index: sys/sys/smr.h =================================================================== --- sys/sys/smr.h +++ sys/sys/smr.h @@ -64,16 +64,24 @@ struct smr { smr_seq_t c_seq; /* Current observed sequence. */ smr_shared_t c_shared; /* Shared SMR state. */ + int c_deferred; /* Deferred advance counter. */ }; /* * Return the current write sequence number. */ +static inline smr_seq_t +smr_shared_current(smr_shared_t s) +{ + + return (atomic_load_int(&s->s_wr_seq)); +} + static inline smr_seq_t smr_current(smr_t smr) { - return (atomic_load_int(&smr->c_shared->s_wr_seq)); + return (smr_shared_current(zpcpu_get(smr)->c_shared)); } /* @@ -106,7 +114,7 @@ * is detected and handled there. */ /* This is an add because we do not have atomic_store_acq_int */ - atomic_add_acq_int(&smr->c_seq, smr_current(smr)); + atomic_add_acq_int(&smr->c_seq, smr_shared_current(smr->c_shared)); } /* @@ -138,6 +146,13 @@ */ smr_seq_t smr_advance(smr_t smr); +/* + * Advances the write sequence number only after N calls. Returns + * the correct goal for a wr_seq that has not yet occurred. Used to + * minimize shared cacheline invalidations for frequent writers. + */ +smr_seq_t smr_advance_deferred(smr_t smr, int limit); + /* * Returns true if a goal sequence has been reached. If * wait is true this will busy loop until success.