Index: sys/compat/linux/linux_futex.c =================================================================== --- sys/compat/linux/linux_futex.c +++ sys/compat/linux/linux_futex.c @@ -41,6 +41,7 @@ #include #include #include +#include #ifdef COMPAT_LINUX32 #include @@ -465,8 +466,8 @@ continue; } + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); error = umtx_pi_claim(pi, td); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); @@ -504,9 +505,7 @@ if (error != 0) break; - umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_busy_unlocked(&uq->uq_key); /* * Set the contested bit so that a release in user space knows @@ -590,8 +589,8 @@ error = futex_key_get(args->uaddr, TYPE_PI_FUTEX, GET_SHARED(args), &key); if (error != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); error = umtx_pi_drop(td, &key, rb, &count); if (error != 0 || rb) { umtxq_unbusy(&key); @@ -645,8 +644,8 @@ umtx_key_release(&key); return (error); } + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); umtxq_unlock(&key); error = futex_atomic_op(td, args->val3, args->uaddr2, &op_ret); umtxq_lock(&key); @@ -704,9 +703,7 @@ umtx_key_release(&key); return (error); } - umtxq_lock(&key); - umtxq_busy(&key); - umtxq_unlock(&key); + umtxq_busy_unlocked(&key); error = fueword32(args->uaddr, &uval); if (error != 0) error = EFAULT; @@ -762,8 +759,8 @@ return (error); if (args->ts != NULL) linux_umtx_abs_timeout_init(&timo, args); + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); uq->uq_bitset = args->val3; umtxq_insert(uq); umtxq_unlock(&uq->uq_key); Index: sys/kern/kern_umtx.c =================================================================== --- sys/kern/kern_umtx.c +++ sys/kern/kern_umtx.c @@ -94,8 +94,7 @@ struct umtxq_chain *uc; \ \ uc = umtxq_getchain(key); \ - mtx_assert(&uc->uc_lock, MA_OWNED); \ - KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); \ + sx_assert(&uc->uc_busy, SX_XLOCKED); \ } while (0) #else #define UMTXQ_ASSERT_LOCKED_BUSY(key) do {} while (0) @@ -123,8 +122,6 @@ #define GET_SHARE(flags) \ (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) -#define BUSY_SPINS 200 - struct umtx_copyops { int (*copyin_timeout)(const void *uaddr, struct timespec *tsp); int (*copyin_umtx_time)(const void *uaddr, size_t size, @@ -337,8 +334,7 @@ LIST_INIT(&umtxq_chains[i][j].uc_queue[1]); LIST_INIT(&umtxq_chains[i][j].uc_spare_queue); TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list); - umtxq_chains[i][j].uc_busy = 0; - umtxq_chains[i][j].uc_waiters = 0; + sx_init(&umtxq_chains[i][j].uc_busy, "umtxqb"); #ifdef UMTX_PROFILING umtxq_chains[i][j].length = 0; umtxq_chains[i][j].max_length = 0; @@ -393,10 +389,6 @@ return (&umtxq_chains[0][key->hash]); } -/* - * Set chain to busy state when following operation - * may be blocked (kernel mutex can not be used). - */ void umtxq_busy(struct umtx_key *key) { @@ -404,25 +396,11 @@ uc = umtxq_getchain(key); mtx_assert(&uc->uc_lock, MA_OWNED); - if (uc->uc_busy) { -#ifdef SMP - if (smp_cpus > 1) { - int count = BUSY_SPINS; - if (count > 0) { - umtxq_unlock(key); - while (uc->uc_busy && --count > 0) - cpu_spinwait(); - umtxq_lock(key); - } - } -#endif - while (uc->uc_busy) { - uc->uc_waiters++; - msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); - uc->uc_waiters--; - } - } - uc->uc_busy = 1; + if (sx_try_xlock(&uc->uc_busy)) + return; + mtx_unlock(&uc->uc_lock); + umtxq_busy_unlocked(key); + mtx_lock(&uc->uc_lock); } /* @@ -431,23 +409,22 @@ void umtxq_unbusy(struct umtx_key *key) { +#ifdef INVARIANTS struct umtxq_chain *uc; uc = umtxq_getchain(key); mtx_assert(&uc->uc_lock, MA_OWNED); - KASSERT(uc->uc_busy != 0, ("not busy")); - uc->uc_busy = 0; - if (uc->uc_waiters) - wakeup_one(uc); +#endif + umtxq_unbusy_unlocked(key); } void umtxq_unbusy_unlocked(struct umtx_key *key) { + struct umtxq_chain *uc; - umtxq_lock(key); - umtxq_unbusy(key); - umtxq_unlock(key); + uc = umtxq_getchain(key); + sx_xunlock(&uc->uc_busy); } static struct umtxq_queue * @@ -946,8 +923,8 @@ AUTO_SHARE, &uq->uq_key)) != 0) return (error); + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); @@ -1035,8 +1012,8 @@ &key)) != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); @@ -1127,8 +1104,8 @@ AUTO_SHARE, &uq->uq_key)) != 0) return (error); + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); @@ -1216,8 +1193,8 @@ &key)) != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); @@ -1438,8 +1415,8 @@ GET_SHARE(flags), &uq->uq_key)) != 0) return (error); + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); @@ -1533,8 +1510,8 @@ &key)) != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); @@ -1595,8 +1572,8 @@ &key)) != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); @@ -1668,8 +1645,8 @@ return (error); owner = 0; + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); count = umtxq_count(&key); umtxq_unlock(&key); @@ -1995,16 +1972,12 @@ struct thread *td, *td1; struct umtx_q *uq1; int error, pri; -#ifdef INVARIANTS - struct umtxq_chain *uc; - uc = umtxq_getchain(&pi->pi_key); -#endif error = 0; td = uq->uq_thread; KASSERT(td == curthread, ("inconsistent uq_thread")); UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key)); - KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); + UMTXQ_ASSERT_LOCKED_BUSY(&uq->uq_key); umtxq_insert(uq); mtx_lock(&umtx_lock); if (pi->pi_owner == NULL) { @@ -2308,8 +2281,8 @@ MPASS(rv == 0); MPASS(owner == old_owner); + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); error = umtx_pi_claim(pi, td); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); @@ -2345,9 +2318,7 @@ if (error != 0) break; - umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_busy_unlocked(&uq->uq_key); /* * Set the contested bit so that a release in user space @@ -2449,8 +2420,8 @@ &key)) != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); error = umtx_pi_drop(td, &key, rb, &count); if (error != 0) { umtxq_unbusy(&key); @@ -2512,9 +2483,7 @@ su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); for (;;) { old_inherited_pri = uq->uq_inherited_pri; - umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_busy_unlocked(&uq->uq_key); rv = fueword32(&m->m_ceilings[0], &ceiling); if (rv == -1) { @@ -2694,9 +2663,7 @@ TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); - umtxq_lock(&key); - umtxq_busy(&key); - umtxq_unlock(&key); + umtxq_busy_unlocked(&key); /* * For priority protected mutex, always set unlocked state * to UMUTEX_CONTESTED, so that userland always enters kernel @@ -2759,9 +2726,7 @@ &uq->uq_key)) != 0) return (error); for (;;) { - umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_busy_unlocked(&uq->uq_key); rv = fueword32(&m->m_ceilings[0], &save_ceiling); if (rv == -1) { @@ -2926,8 +2891,8 @@ clockid = CLOCK_REALTIME; } + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); @@ -2996,8 +2961,8 @@ return (EFAULT); if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); cnt = umtxq_count(&key); nwake = umtxq_signal(&key, 1); if (cnt <= nwake) { @@ -3026,8 +2991,8 @@ if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); umtxq_signal(&key, INT_MAX); umtxq_unlock(&key); @@ -3102,9 +3067,7 @@ break; /* grab monitor lock */ - umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_busy_unlocked(&uq->uq_key); /* * re-read the state, in case it changed between the try-lock above @@ -3279,8 +3242,8 @@ if ((state & (URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS)) == 0 && blocked_readers != 0) { + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); umtxq_signal_queue(&uq->uq_key, INT_MAX, UMTX_SHARED_QUEUE); umtxq_unbusy(&uq->uq_key); @@ -3291,9 +3254,7 @@ } /* grab monitor lock */ - umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); - umtxq_unlock(&uq->uq_key); + umtxq_busy_unlocked(&uq->uq_key); /* * Re-read the state, in case it changed between the @@ -3510,8 +3471,8 @@ } if (count) { + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); umtxq_signal_queue(&uq->uq_key, count, q); umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); @@ -3542,8 +3503,8 @@ umtx_abs_timeout_init2(&timo, timeout); again: + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); rv = casueword32(&sem->_has_waiters, 0, &count1, 1); @@ -3602,8 +3563,8 @@ return (EFAULT); if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); cnt = umtxq_count(&key); if (cnt > 0) { /* @@ -3644,8 +3605,8 @@ error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key); if (error != 0) return (error); + umtxq_busy_unlocked(&uq->uq_key); umtxq_lock(&uq->uq_key); - umtxq_busy(&uq->uq_key); umtxq_insert(uq); umtxq_unlock(&uq->uq_key); rv = fueword32(&sem->_count, &count); @@ -3724,8 +3685,8 @@ return (EFAULT); if ((error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &key)) != 0) return (error); + umtxq_busy_unlocked(&key); umtxq_lock(&key); - umtxq_busy(&key); cnt = umtxq_count(&key); if (cnt > 0) { /* Index: sys/sys/umtxvar.h =================================================================== --- sys/sys/umtxvar.h +++ sys/sys/umtxvar.h @@ -174,11 +174,8 @@ LIST_HEAD(, umtxq_queue) uc_spare_queue; - /* Busy flag */ - char uc_busy; - - /* Chain lock waiters */ - int uc_waiters; + /* Busy lock */ + struct sx uc_busy; /* All PI in the list */ TAILQ_HEAD(,umtx_pi) uc_pi_list; @@ -250,6 +247,20 @@ mtx_lock(&_uc->uc_lock); \ } while (0) +/* + * Set chain to busy state when following operation + * may be blocked (kernel mutex can not be used). + * + * The code is a macro so that file/line information is taken from the caller. + */ +#define umtxq_busy_unlocked(key) do { \ + struct umtx_key *_key = (key); \ + struct umtxq_chain *_uc; \ + \ + _uc = umtxq_getchain(_key); \ + sx_xlock(&_uc->uc_busy); \ +} while (0) + /* * Unlock a chain. */