Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_rmlock.c
Show First 20 Lines • Show All 360 Lines • ▼ Show 20 Lines | _rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) | ||||
/* Remove our tracker from the per-cpu list. */ | /* Remove our tracker from the per-cpu list. */ | ||||
rm_tracker_remove(pc, tracker); | rm_tracker_remove(pc, tracker); | ||||
/* | /* | ||||
* Check to see if the IPI granted us the lock after all. The load of | * Check to see if the IPI granted us the lock after all. The load of | ||||
* rmp_flags must happen after the tracker is removed from the list. | * rmp_flags must happen after the tracker is removed from the list. | ||||
*/ | */ | ||||
__compiler_membar(); | atomic_interrupt_fence(); | ||||
if (tracker->rmp_flags) { | if (tracker->rmp_flags) { | ||||
/* Just add back tracker - we hold the lock. */ | /* Just add back tracker - we hold the lock. */ | ||||
rm_tracker_add(pc, tracker); | rm_tracker_add(pc, tracker); | ||||
critical_exit(); | critical_exit(); | ||||
return (1); | return (1); | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | _rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) | ||||
tracker->rmp_thread = td; | tracker->rmp_thread = td; | ||||
tracker->rmp_rmlock = rm; | tracker->rmp_rmlock = rm; | ||||
if (rm->lock_object.lo_flags & LO_SLEEPABLE) | if (rm->lock_object.lo_flags & LO_SLEEPABLE) | ||||
THREAD_NO_SLEEPING(); | THREAD_NO_SLEEPING(); | ||||
td->td_critnest++; /* critical_enter(); */ | td->td_critnest++; /* critical_enter(); */ | ||||
__compiler_membar(); | atomic_interrupt_fence(); | ||||
pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ | pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ | ||||
rm_tracker_add(pc, tracker); | rm_tracker_add(pc, tracker); | ||||
sched_pin(); | sched_pin(); | ||||
__compiler_membar(); | atomic_interrupt_fence(); | ||||
td->td_critnest--; | td->td_critnest--; | ||||
/* | /* | ||||
* Fast path to combine two common conditions into a single | * Fast path to combine two common conditions into a single | ||||
* conditional jump. | * conditional jump. | ||||
*/ | */ | ||||
if (__predict_true(0 == (td->td_owepreempt | | if (__predict_true(0 == (td->td_owepreempt | | ||||
▲ Show 20 Lines • Show All 400 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* They are intended to be only used when write-locking is almost never needed | * They are intended to be only used when write-locking is almost never needed | ||||
* (e.g., they can guard against unloading a kernel module) while read-locking | * (e.g., they can guard against unloading a kernel module) while read-locking | ||||
* happens all the time. | * happens all the time. | ||||
* | * | ||||
* Concurrent writers take turns taking the lock while going off cpu. If this is | * Concurrent writers take turns taking the lock while going off cpu. If this is | ||||
* of concern for your usecase, this is not the right primitive. | * of concern for your usecase, this is not the right primitive. | ||||
* | * | ||||
* Neither rms_rlock nor rms_runlock use fences. Instead compiler barriers are | * Neither rms_rlock nor rms_runlock use thread fences. Instead interrupt | ||||
* inserted to prevert reordering of generated code. Execution ordering is | * fences are inserted to ensure ordering with the code executed in the IPI | ||||
* provided with the use of an IPI handler. | * handler. | ||||
* | * | ||||
* No attempt is made to track which CPUs read locked at least once, | * No attempt is made to track which CPUs read locked at least once, | ||||
* consequently write locking sends IPIs to all of them. This will become a | * consequently write locking sends IPIs to all of them. This will become a | ||||
* problem at some point. The easiest way to lessen it is to provide a bitmap. | * problem at some point. The easiest way to lessen it is to provide a bitmap. | ||||
*/ | */ | ||||
#define rms_int_membar() __compiler_membar() | |||||
#define RMS_NOOWNER ((void *)0x1) | #define RMS_NOOWNER ((void *)0x1) | ||||
#define RMS_TRANSIENT ((void *)0x2) | #define RMS_TRANSIENT ((void *)0x2) | ||||
#define RMS_FLAGMASK 0xf | #define RMS_FLAGMASK 0xf | ||||
struct rmslock_pcpu { | struct rmslock_pcpu { | ||||
int influx; | int influx; | ||||
int readers; | int readers; | ||||
}; | }; | ||||
▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines | rms_rlock(struct rmslock *rms) | ||||
struct rmslock_pcpu *pcpu; | struct rmslock_pcpu *pcpu; | ||||
WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__); | WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__); | ||||
MPASS(atomic_load_ptr(&rms->owner) != curthread); | MPASS(atomic_load_ptr(&rms->owner) != curthread); | ||||
critical_enter(); | critical_enter(); | ||||
pcpu = rms_int_pcpu(rms); | pcpu = rms_int_pcpu(rms); | ||||
rms_int_influx_enter(rms, pcpu); | rms_int_influx_enter(rms, pcpu); | ||||
rms_int_membar(); | atomic_interrupt_fence(); | ||||
if (__predict_false(rms->writers > 0)) { | if (__predict_false(rms->writers > 0)) { | ||||
rms_rlock_fallback(rms); | rms_rlock_fallback(rms); | ||||
return; | return; | ||||
} | } | ||||
rms_int_membar(); | atomic_interrupt_fence(); | ||||
rms_int_readers_inc(rms, pcpu); | rms_int_readers_inc(rms, pcpu); | ||||
rms_int_membar(); | atomic_interrupt_fence(); | ||||
rms_int_influx_exit(rms, pcpu); | rms_int_influx_exit(rms, pcpu); | ||||
critical_exit(); | critical_exit(); | ||||
} | } | ||||
int | int | ||||
rms_try_rlock(struct rmslock *rms) | rms_try_rlock(struct rmslock *rms) | ||||
{ | { | ||||
struct rmslock_pcpu *pcpu; | struct rmslock_pcpu *pcpu; | ||||
MPASS(atomic_load_ptr(&rms->owner) != curthread); | MPASS(atomic_load_ptr(&rms->owner) != curthread); | ||||
critical_enter(); | critical_enter(); | ||||
pcpu = rms_int_pcpu(rms); | pcpu = rms_int_pcpu(rms); | ||||
rms_int_influx_enter(rms, pcpu); | rms_int_influx_enter(rms, pcpu); | ||||
rms_int_membar(); | atomic_interrupt_fence(); | ||||
if (__predict_false(rms->writers > 0)) { | if (__predict_false(rms->writers > 0)) { | ||||
rms_int_influx_exit(rms, pcpu); | rms_int_influx_exit(rms, pcpu); | ||||
critical_exit(); | critical_exit(); | ||||
return (0); | return (0); | ||||
} | } | ||||
rms_int_membar(); | atomic_interrupt_fence(); | ||||
rms_int_readers_inc(rms, pcpu); | rms_int_readers_inc(rms, pcpu); | ||||
rms_int_membar(); | atomic_interrupt_fence(); | ||||
rms_int_influx_exit(rms, pcpu); | rms_int_influx_exit(rms, pcpu); | ||||
critical_exit(); | critical_exit(); | ||||
return (1); | return (1); | ||||
} | } | ||||
static void __noinline | static void __noinline | ||||
rms_runlock_fallback(struct rmslock *rms) | rms_runlock_fallback(struct rmslock *rms) | ||||
{ | { | ||||
Show All 15 Lines | |||||
void | void | ||||
rms_runlock(struct rmslock *rms) | rms_runlock(struct rmslock *rms) | ||||
{ | { | ||||
struct rmslock_pcpu *pcpu; | struct rmslock_pcpu *pcpu; | ||||
critical_enter(); | critical_enter(); | ||||
pcpu = rms_int_pcpu(rms); | pcpu = rms_int_pcpu(rms); | ||||
rms_int_influx_enter(rms, pcpu); | rms_int_influx_enter(rms, pcpu); | ||||
rms_int_membar(); | atomic_interrupt_fence(); | ||||
if (__predict_false(rms->writers > 0)) { | if (__predict_false(rms->writers > 0)) { | ||||
rms_runlock_fallback(rms); | rms_runlock_fallback(rms); | ||||
return; | return; | ||||
} | } | ||||
rms_int_membar(); | atomic_interrupt_fence(); | ||||
rms_int_readers_dec(rms, pcpu); | rms_int_readers_dec(rms, pcpu); | ||||
rms_int_membar(); | atomic_interrupt_fence(); | ||||
rms_int_influx_exit(rms, pcpu); | rms_int_influx_exit(rms, pcpu); | ||||
critical_exit(); | critical_exit(); | ||||
} | } | ||||
struct rmslock_ipi { | struct rmslock_ipi { | ||||
struct rmslock *rms; | struct rmslock *rms; | ||||
struct smp_rendezvous_cpus_retry_arg srcra; | struct smp_rendezvous_cpus_retry_arg srcra; | ||||
}; | }; | ||||
▲ Show 20 Lines • Show All 142 Lines • Show Last 20 Lines |