diff --git a/sys/kern/kern_rmlock.c b/sys/kern/kern_rmlock.c --- a/sys/kern/kern_rmlock.c +++ b/sys/kern/kern_rmlock.c @@ -366,7 +366,7 @@ * Check to see if the IPI granted us the lock after all. The load of * rmp_flags must happen after the tracker is removed from the list. */ - __compiler_membar(); + atomic_interrupt_fence(); if (tracker->rmp_flags) { /* Just add back tracker - we hold the lock. */ rm_tracker_add(pc, tracker); @@ -448,7 +448,7 @@ td->td_critnest++; /* critical_enter(); */ - __compiler_membar(); + atomic_interrupt_fence(); pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ @@ -456,7 +456,7 @@ sched_pin(); - __compiler_membar(); + atomic_interrupt_fence(); td->td_critnest--; @@ -873,17 +873,15 @@ * Concurrent writers take turns taking the lock while going off cpu. If this is * of concern for your usecase, this is not the right primitive. * - * Neither rms_rlock nor rms_runlock use fences. Instead compiler barriers are - * inserted to prevert reordering of generated code. Execution ordering is - * provided with the use of an IPI handler. + * Neither rms_rlock nor rms_runlock use thread fences. Instead interrupt + * fences are inserted to ensure ordering with the code executed in the IPI + * handler. * * No attempt is made to track which CPUs read locked at least once, * consequently write locking sends IPIs to all of them. This will become a * problem at some point. The easiest way to lessen it is to provide a bitmap. */ -#define rms_int_membar() __compiler_membar() - #define RMS_NOOWNER ((void *)0x1) #define RMS_TRANSIENT ((void *)0x2) #define RMS_FLAGMASK 0xf @@ -1030,14 +1028,14 @@ critical_enter(); pcpu = rms_int_pcpu(rms); rms_int_influx_enter(rms, pcpu); - rms_int_membar(); + atomic_interrupt_fence(); if (__predict_false(rms->writers > 0)) { rms_rlock_fallback(rms); return; } - rms_int_membar(); + atomic_interrupt_fence(); rms_int_readers_inc(rms, pcpu); - rms_int_membar(); + atomic_interrupt_fence(); rms_int_influx_exit(rms, pcpu); critical_exit(); } @@ -1052,15 +1050,15 @@ critical_enter(); pcpu = rms_int_pcpu(rms); rms_int_influx_enter(rms, pcpu); - rms_int_membar(); + atomic_interrupt_fence(); if (__predict_false(rms->writers > 0)) { rms_int_influx_exit(rms, pcpu); critical_exit(); return (0); } - rms_int_membar(); + atomic_interrupt_fence(); rms_int_readers_inc(rms, pcpu); - rms_int_membar(); + atomic_interrupt_fence(); rms_int_influx_exit(rms, pcpu); critical_exit(); return (1); @@ -1092,14 +1090,14 @@ critical_enter(); pcpu = rms_int_pcpu(rms); rms_int_influx_enter(rms, pcpu); - rms_int_membar(); + atomic_interrupt_fence(); if (__predict_false(rms->writers > 0)) { rms_runlock_fallback(rms); return; } - rms_int_membar(); + atomic_interrupt_fence(); rms_int_readers_dec(rms, pcpu); - rms_int_membar(); + atomic_interrupt_fence(); rms_int_influx_exit(rms, pcpu); critical_exit(); } diff --git a/sys/sys/kpilite.h b/sys/sys/kpilite.h --- a/sys/sys/kpilite.h +++ b/sys/sys/kpilite.h @@ -38,7 +38,7 @@ KASSERT((struct thread *)td == curthread, ("sched_pin called on non curthread")); td->td_pinned++; - __compiler_membar(); + atomic_interrupt_fence(); } static __inline void @@ -47,9 +47,8 @@ KASSERT((struct thread *)td == curthread, ("sched_unpin called on non curthread")); KASSERT(td->td_pinned > 0, ("sched_unpin called on non pinned thread")); - __compiler_membar(); + atomic_interrupt_fence(); td->td_pinned--; - __compiler_membar(); } #endif #endif diff --git a/sys/sys/mount.h b/sys/sys/mount.h --- a/sys/sys/mount.h +++ b/sys/sys/mount.h @@ -1091,7 +1091,7 @@ _mpcpu = vfs_mount_pcpu(mp); \ MPASS(mpcpu->mntp_thread_in_ops == 0); \ _mpcpu->mntp_thread_in_ops = 1; \ - __compiler_membar(); \ + atomic_interrupt_fence(); \ if (__predict_false(mp->mnt_vfs_ops > 0)) { \ vfs_op_thread_exit_crit(mp, _mpcpu); \ _retval_crit = false; \ @@ -1111,7 +1111,7 @@ #define vfs_op_thread_exit_crit(mp, _mpcpu) do { \ MPASS(_mpcpu == vfs_mount_pcpu(mp)); \ MPASS(_mpcpu->mntp_thread_in_ops == 1); \ - __compiler_membar(); \ + atomic_interrupt_fence(); \ _mpcpu->mntp_thread_in_ops = 0; \ } while (0) diff --git a/sys/sys/sched.h b/sys/sys/sched.h --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -173,13 +173,13 @@ sched_pin(void) { curthread->td_pinned++; - __compiler_membar(); + atomic_interrupt_fence(); } static __inline void sched_unpin(void) { - __compiler_membar(); + atomic_interrupt_fence(); curthread->td_pinned--; } diff --git a/sys/sys/systm.h b/sys/sys/systm.h --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -284,7 +284,7 @@ td = (struct thread_lite *)curthread; td->td_critnest++; - __compiler_membar(); + atomic_interrupt_fence(); } static __inline void @@ -295,9 +295,9 @@ td = (struct thread_lite *)curthread; KASSERT(td->td_critnest != 0, ("critical_exit: td_critnest == 0")); - __compiler_membar(); + atomic_interrupt_fence(); td->td_critnest--; - __compiler_membar(); + atomic_interrupt_fence(); if (__predict_false(td->td_owepreempt)) critical_exit_preempt();