Index: sys/kern/kern_rmlock.c =================================================================== --- sys/kern/kern_rmlock.c +++ sys/kern/kern_rmlock.c @@ -991,17 +991,17 @@ struct rmslock_ipi { struct rmslock *rms; - cpuset_t signal; + struct smp_rendezvous_cpus_retry_arg srcra; }; static void -rms_wlock_IPI(void *arg) +rms_action_func(void *arg) { struct rmslock_ipi *rmsipi; struct rmslock *rms; int readers; - rmsipi = arg; + rmsipi = __containerof(arg, struct rmslock_ipi, srcra); rms = rmsipi->rms; if (*zpcpu_get(rms->readers_influx)) @@ -1009,65 +1009,40 @@ readers = zpcpu_replace(rms->readers_pcpu, 0); if (readers != 0) atomic_add_int(&rms->readers, readers); - CPU_CLR_ATOMIC(curcpu, &rmsipi->signal); + smp_rendezvous_cpus_done(arg); +} + +static void +rms_wait_func(void *arg, int cpu) +{ + struct rmslock_ipi *rmsipi; + struct rmslock *rms; + int *in_op; + + rmsipi = __containerof(arg, struct rmslock_ipi, srcra); + rms = rmsipi->rms; + + in_op = zpcpu_get_cpu(rms->readers_influx, cpu); + while (atomic_load_int(in_op)) + cpu_spinwait(); } static void rms_wlock_switch(struct rmslock *rms) { struct rmslock_ipi rmsipi; - int *in_op; - int cpu; MPASS(rms->readers == 0); MPASS(rms->writers == 1); rmsipi.rms = rms; - /* - * Publishes rms->writers. rlock and runlock will get this ordered - * via IPI in the worst case. - */ - atomic_thread_fence_rel(); - - /* - * Collect reader counts from all CPUs using an IPI. The handler can - * find itself running while the interrupted CPU was doing either - * rlock or runlock in which case it will fail. - * - * Successful attempts clear the cpu id in the bitmap. - * - * In case of failure we observe all failing CPUs not executing there to - * determine when to make the next attempt. Note that threads having - * the var set have preemption disabled. Setting of readers_influx - * only uses compiler barriers making these loads unreliable, which is - * fine -- the IPI handler will always see the correct result. - * - * We retry until all counts are collected. Forward progress is - * guaranteed by that fact that the total number of threads which can - * be caught like this is finite and they all are going to block on - * their own. - */ - CPU_COPY(&all_cpus, &rmsipi.signal); - for (;;) { - smp_rendezvous_cpus( - rmsipi.signal, - smp_no_rendezvous_barrier, - rms_wlock_IPI, - smp_no_rendezvous_barrier, - &rmsipi); - - if (CPU_EMPTY(&rmsipi.signal)) - break; - - CPU_FOREACH(cpu) { - if (!CPU_ISSET(cpu, &rmsipi.signal)) - continue; - in_op = zpcpu_get_cpu(rms->readers_influx, cpu); - while (atomic_load_int(in_op)) - cpu_spinwait(); - } - } + smp_rendezvous_cpus_retry(all_cpus, + smp_no_rendezvous_barrier, + rms_action_func, + smp_no_rendezvous_barrier, + rms_wait_func, + &rmsipi.srcra); } void Index: sys/kern/subr_smp.c =================================================================== --- sys/kern/subr_smp.c +++ sys/kern/subr_smp.c @@ -627,6 +627,48 @@ smp_rendezvous_cpus(all_cpus, setup_func, action_func, teardown_func, arg); } +void +smp_rendezvous_cpus_retry(cpuset_t map, + void (* setup_func)(void *), + void (* action_func)(void *), + void (* teardown_func)(void *), + void (* wait_func)(void *, int), + struct smp_rendezvous_cpus_retry_arg *arg) +{ + int cpu; + + /* + * Execute an action on all specified CPUs while retrying until they + * all acknowledge completion. + */ + CPU_COPY(&map, &arg->cpus); + for (;;) { + smp_rendezvous_cpus( + arg->cpus, + setup_func, + action_func, + teardown_func, + arg); + + if (CPU_EMPTY(&arg->cpus)) + break; + + CPU_FOREACH(cpu) { + if (!CPU_ISSET(cpu, &arg->cpus)) + continue; + wait_func(arg, cpu); + } + } + atomic_thread_fence_acq(); +} + +void +smp_rendezvous_cpus_done(struct smp_rendezvous_cpus_retry_arg *arg) +{ + + CPU_CLR_ATOMIC(curcpu, &arg->cpus); +} + static struct cpu_group group[MAXCPU * MAX_CACHE_LEVELS + 1]; struct cpu_group * Index: sys/kern/vfs_mount.c =================================================================== --- sys/kern/vfs_mount.c +++ sys/kern/vfs_mount.c @@ -1442,16 +1442,7 @@ MNT_IUNLOCK(mp); return; } - /* - * Paired with a fence in vfs_op_thread_enter(). See the comment - * above it for details. - */ - atomic_thread_fence_seq_cst(); vfs_op_barrier_wait(mp); - /* - * Paired with a fence in vfs_op_thread_exit(). - */ - atomic_thread_fence_acq(); CPU_FOREACH(cpu) { mp->mnt_ref += zpcpu_replace_cpu(mp->mnt_ref_pcpu, 0, cpu); @@ -1485,20 +1476,52 @@ MNT_IUNLOCK(mp); } -/* - * It is assumed the caller already posted at least an acquire barrier. - */ +struct vfs_op_barrier_ipi { + struct mount *mp; + struct smp_rendezvous_cpus_retry_arg srcra; +}; + +static void +vfs_op_action_func(void *arg) +{ + struct vfs_op_barrier_ipi *vfsopipi; + struct mount *mp; + + vfsopipi = __containerof(arg, struct vfs_op_barrier_ipi, srcra); + mp = vfsopipi->mp; + + if (!vfs_op_thread_entered(mp)) + smp_rendezvous_cpus_done(arg); +} + +static void +vfs_op_wait_func(void *arg, int cpu) +{ + struct vfs_op_barrier_ipi *vfsopipi; + struct mount *mp; + int *in_op; + + vfsopipi = __containerof(arg, struct vfs_op_barrier_ipi, srcra); + mp = vfsopipi->mp; + + in_op = zpcpu_get_cpu(mp->mnt_thread_in_ops_pcpu, cpu); + while (atomic_load_int(in_op)) + cpu_spinwait(); +} + void vfs_op_barrier_wait(struct mount *mp) { - int *in_op; - int cpu; + struct vfs_op_barrier_ipi vfsopipi; - CPU_FOREACH(cpu) { - in_op = zpcpu_get_cpu(mp->mnt_thread_in_ops_pcpu, cpu); - while (atomic_load_int(in_op)) - cpu_spinwait(); - } + vfsopipi.mp = mp; + + smp_rendezvous_cpus_retry(all_cpus, + smp_no_rendezvous_barrier, + vfs_op_action_func, + smp_no_rendezvous_barrier, + vfs_op_wait_func, + &vfsopipi.srcra); } #ifdef DIAGNOSTIC Index: sys/kern/vfs_subr.c =================================================================== --- sys/kern/vfs_subr.c +++ sys/kern/vfs_subr.c @@ -6029,10 +6029,6 @@ } MNT_IUNLOCK(mp); if (vp != NULL) { - /* - * Paired with a fence in vfs_op_thread_exit(). - */ - atomic_thread_fence_acq(); vfs_op_barrier_wait(mp); vrele(vp); } Index: sys/sys/mount.h =================================================================== --- sys/sys/mount.h +++ sys/sys/mount.h @@ -1009,7 +1009,7 @@ critical_enter(); \ MPASS(!vfs_op_thread_entered(mp)); \ zpcpu_set_protected(mp->mnt_thread_in_ops_pcpu, 1); \ - atomic_thread_fence_seq_cst(); \ + __compiler_membar(); \ if (__predict_false(mp->mnt_vfs_ops > 0)) { \ vfs_op_thread_exit(mp); \ _retval = false; \ @@ -1019,7 +1019,7 @@ #define vfs_op_thread_exit(mp) do { \ MPASS(vfs_op_thread_entered(mp)); \ - atomic_thread_fence_rel(); \ + __compiler_membar(); \ zpcpu_set_protected(mp->mnt_thread_in_ops_pcpu, 0); \ critical_exit(); \ } while (0) Index: sys/sys/smp.h =================================================================== --- sys/sys/smp.h +++ sys/sys/smp.h @@ -276,6 +276,19 @@ void (*)(void *), void (*)(void *), void *arg); + +struct smp_rendezvous_cpus_retry_arg { + cpuset_t cpus; +}; +void smp_rendezvous_cpus_retry(cpuset_t, + void (*)(void *), + void (*)(void *), + void (*)(void *), + void (*)(void *, int), + struct smp_rendezvous_cpus_retry_arg *); + +void smp_rendezvous_cpus_done(struct smp_rendezvous_cpus_retry_arg *); + #endif /* !LOCORE */ #endif /* _KERNEL */ #endif /* _SYS_SMP_H_ */