diff --git a/sys/kern/kern_rangelock.c b/sys/kern/kern_rangelock.c --- a/sys/kern/kern_rangelock.c +++ b/sys/kern/kern_rangelock.c @@ -66,7 +66,7 @@ * trylocks are same as normal locks but do not drain. */ -static int rangelock_cheat = 0; +static int rangelock_cheat = 1; SYSCTL_INT(_debug, OID_AUTO, rangelock_cheat, CTLFLAG_RWTUN, &rangelock_cheat, 0, ""); @@ -82,6 +82,24 @@ #define RL_RET_CHEAT_RLOCKED 0x1100 #define RL_RET_CHEAT_WLOCKED 0x2200 +static void +rangelock_cheat_drain(struct rangelock *lock) +{ + uintptr_t v; + + DROP_GIANT(); + for (;;) { + v = (uintptr_t)atomic_load_ptr(&lock->head); + if ((v & RL_CHEAT_DRAINING) == 0) + break; + sleepq_add(&lock->head, NULL, "ranged1", 0, 0); + sleepq_wait(&lock->head, PRI_USER); + sleepq_lock(&lock->head); + } + sleepq_release(&lock->head); + PICKUP_GIANT(); +} + static bool rangelock_cheat_lock(struct rangelock *lock, int locktype, bool trylock, void **cookie) @@ -99,17 +117,7 @@ } sleepq_lock(&lock->head); drain1: - DROP_GIANT(); - for (;;) { - v = (uintptr_t)atomic_load_ptr(&lock->head); - if ((v & RL_CHEAT_DRAINING) == 0) - break; - sleepq_add(&lock->head, NULL, "ranged1", 0, 0); - sleepq_wait(&lock->head, PRI_USER); - sleepq_lock(&lock->head); - } - sleepq_release(&lock->head); - PICKUP_GIANT(); + rangelock_cheat_drain(lock); return (false); } @@ -744,6 +752,46 @@ return (rangelock_lock_int(lock, true, start, end, RL_LOCK_WRITE)); } +/* + * If the caller asserts that it can obtain the range locks on the + * same lock simultaneously, switch to the non-cheat mode. Cheat mode + * cannot handle it, hanging in drain or trylock retries. + */ +void +rangelock_may_recurse(struct rangelock *lock) +{ + uintptr_t v, x; + + v = (uintptr_t)atomic_load_ptr(&lock->head); + if ((v & RL_CHEAT_CHEATING) == 0) + return; + + sleepq_lock(&lock->head); + for (;;) { + if ((v & RL_CHEAT_CHEATING) == 0) { + sleepq_release(&lock->head); + return; + } + + /* Cheating and locked, drain. */ + if ((v & RL_CHEAT_WLOCKED) != 0 || + (v & ~RL_CHEAT_MASK) >= RL_CHEAT_READER) { + x = v | RL_CHEAT_DRAINING; + if (atomic_fcmpset_rel_ptr(&lock->head, &v, x) != 0) { + rangelock_cheat_drain(lock); + return; + } + } + + /* Cheating and unlocked, clear RL_CHEAT_CHEATING. */ + x = 0; + if (atomic_fcmpset_rel_ptr(&lock->head, &v, x) != 0) { + sleepq_release(&lock->head); + return; + } + } +} + #ifdef INVARIANT_SUPPORT void _rangelock_cookie_assert(void *cookie, int what, const char *file, int line) diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c --- a/sys/kern/vfs_syscalls.c +++ b/sys/kern/vfs_syscalls.c @@ -4978,11 +4978,13 @@ * If infp and outfp refer to the same file, the byte ranges cannot * overlap. */ - if (invp == outvp && ((savinoff <= savoutoff && savinoff + len > - savoutoff) || (savinoff > savoutoff && savoutoff + len > - savinoff))) { - error = EINVAL; - goto out; + if (invp == outvp) { + if ((savinoff <= savoutoff && savinoff + len > savoutoff) || + (savinoff > savoutoff && savoutoff + len > savinoff)) { + error = EINVAL; + goto out; + } + rangelock_may_recurse(&invp->v_rl); } /* Range lock the byte ranges for both invp and outvp. */ diff --git a/sys/sys/rangelock.h b/sys/sys/rangelock.h --- a/sys/sys/rangelock.h +++ b/sys/sys/rangelock.h @@ -66,6 +66,7 @@ void *rangelock_trywlock(struct rangelock *lock, vm_ooffset_t start, vm_ooffset_t end); void rangelock_entry_free(struct rl_q_entry *e); +void rangelock_may_recurse(struct rangelock *lock); #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) void _rangelock_cookie_assert(void *cookie, int what, const char *file, int line);