Index: sys/kern/kern_synch.c =================================================================== --- sys/kern/kern_synch.c +++ sys/kern/kern_synch.c @@ -336,81 +336,6 @@ (flags & C_CATCH) ? PCATCH : 0, wmesg, sbt, pr, flags)); } -/* - * Potentially release the last reference for refcount. Check for - * unlikely conditions and signal the caller as to whether it was - * the final ref. - */ -bool -refcount_release_last(volatile u_int *count, u_int n, u_int old) -{ - u_int waiter; - - waiter = old & REFCOUNT_WAITER; - old = REFCOUNT_COUNT(old); - if (__predict_false(n > old || REFCOUNT_SATURATED(old))) { - /* - * Avoid multiple destructor invocations if underflow occurred. - * This is not perfect since the memory backing the containing - * object may already have been reallocated. - */ - _refcount_update_saturated(count); - return (false); - } - - /* - * Attempt to atomically clear the waiter bit. Wakeup waiters - * if we are successful. - */ - if (waiter != 0 && atomic_cmpset_int(count, REFCOUNT_WAITER, 0)) - wakeup(__DEVOLATILE(u_int *, count)); - - /* - * Last reference. Signal the user to call the destructor. - * - * Ensure that the destructor sees all updates. This synchronizes - * with release fences from all routines which drop the count. - */ - atomic_thread_fence_acq(); - return (true); -} - -/* - * Wait for a refcount wakeup. This does not guarantee that the ref is still - * zero on return and may be subject to transient wakeups. Callers wanting - * a precise answer should use refcount_wait(). - */ -void -_refcount_sleep(volatile u_int *count, struct lock_object *lock, - const char *wmesg, int pri) -{ - void *wchan; - u_int old; - - if (REFCOUNT_COUNT(*count) == 0) { - if (lock != NULL) - LOCK_CLASS(lock)->lc_unlock(lock); - return; - } - wchan = __DEVOLATILE(void *, count); - sleepq_lock(wchan); - if (lock != NULL) - LOCK_CLASS(lock)->lc_unlock(lock); - old = *count; - for (;;) { - if (REFCOUNT_COUNT(old) == 0) { - sleepq_release(wchan); - return; - } - if (old & REFCOUNT_WAITER) - break; - if (atomic_fcmpset_int(count, &old, old | REFCOUNT_WAITER)) - break; - } - sleepq_add(wchan, NULL, wmesg, 0, 0); - sleepq_wait(wchan, pri); -} - /* * Make all threads sleeping on the specified identifier runnable. */ @@ -459,6 +384,56 @@ kick_proc0(); } +void +_blockcount_wakeup(u_int *count, u_int old) +{ + + KASSERT(BLOCKCOUNT_WAITERS(old), + ("blockcount_wakeup: no waiters on %p", count)); + + if (atomic_cmpset_int(count, BLOCKCOUNT_WAITERS_FLAG, 0)) + wakeup(__DEVOLATILE(u_int *, count)); +} + +/* + * Wait for a wakeup. This does not guarantee that the count is still zero on + * return and may be subject to transient wakeups. Callers wanting a precise + * answer should use refcount_wait(). + */ +void +_blockcount_sleep(u_int *count, struct lock_object *lock, const char *wmesg, + int prio) +{ + void *wchan; + uintptr_t lock_state; + u_int old; + + if (atomic_load_int(count) == 0) { + if (lock != NULL) + LOCK_CLASS(lock)->lc_unlock(lock); + return; + } + lock_state = 0; + wchan = __DEVOLATILE(void *, count); + sleepq_lock(wchan); + if (lock != NULL) + lock_state = LOCK_CLASS(lock)->lc_unlock(lock); + old = atomic_load_int(count); + do { + if (BLOCKCOUNT_COUNT(old) == 0) { + sleepq_release(wchan); + return; + } + if (BLOCKCOUNT_WAITERS(old)) + break; + } while (!atomic_fcmpset_int(count, &old, + old | BLOCKCOUNT_WAITERS_FLAG)); + sleepq_add(wchan, NULL, wmesg, 0, 0); + sleepq_wait(wchan, prio); + if (lock != NULL && (prio & PDROP) == 0) + LOCK_CLASS(lock)->lc_lock(lock, lock_state); +} + static void kdb_switch(void) { Index: sys/kern/vfs_bio.c =================================================================== --- sys/kern/vfs_bio.c +++ sys/kern/vfs_bio.c @@ -2854,9 +2854,9 @@ bool bogus; obj = bp->b_bufobj->bo_object; - KASSERT(REFCOUNT_COUNT(obj->paging_in_progress) >= bp->b_npages, + KASSERT(BLOCKCOUNT_COUNT(obj->paging_in_progress) >= bp->b_npages, ("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)", - REFCOUNT_COUNT(obj->paging_in_progress), bp->b_npages)); + BLOCKCOUNT_COUNT(obj->paging_in_progress), bp->b_npages)); vp = bp->b_vp; VNPASS(vp->v_holdcnt > 0, vp); Index: sys/sys/refcount.h =================================================================== --- sys/sys/refcount.h +++ sys/sys/refcount.h @@ -39,13 +39,8 @@ #define KASSERT(exp, msg) /* */ #endif -#define REFCOUNT_WAITER (1U << 31) /* Refcount has waiter. */ -#define REFCOUNT_SATURATION_VALUE (3U << 29) - -#define REFCOUNT_SATURATED(val) (((val) & (1U << 30)) != 0) -#define REFCOUNT_COUNT(x) ((x) & ~REFCOUNT_WAITER) - -bool refcount_release_last(volatile u_int *count, u_int n, u_int old); +#define REFCOUNT_SATURATED(val) (((val) & (1U << 31)) != 0) +#define REFCOUNT_SATURATION_VALUE (3U << 30) /* * Attempt to handle reference count overflow and underflow. Force the counter @@ -111,56 +106,6 @@ } } -static __inline bool -refcount_releasen(volatile u_int *count, u_int n) -{ - u_int old; - - KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, - ("refcount_releasen: n=%u too large", n)); - - /* - * Paired with acquire fence in refcount_release_last. - */ - atomic_thread_fence_rel(); - old = atomic_fetchadd_int(count, -n); - if (__predict_false(n >= REFCOUNT_COUNT(old) || - REFCOUNT_SATURATED(old))) - return (refcount_release_last(count, n, old)); - return (false); -} - -static __inline bool -refcount_release(volatile u_int *count) -{ - - return (refcount_releasen(count, 1)); -} - -#ifdef _KERNEL -struct lock_object; -void _refcount_sleep(volatile u_int *count, struct lock_object *, - const char *wmesg, int prio); - -static __inline void -refcount_sleep(volatile u_int *count, const char *wmesg, int prio) -{ - - _refcount_sleep(count, NULL, wmesg, prio); -} - -#define refcount_sleep_interlock(count, lock, wmesg, prio) \ - _refcount_sleep((count), (struct lock_object *)(lock), (wmesg), (prio)) - -static __inline void -refcount_wait(volatile u_int *count, const char *wmesg, int prio) -{ - - while (*count != 0) - refcount_sleep(count, wmesg, prio); -} -#endif - /* * This functions returns non-zero if the refcount was * incremented. Else zero is returned. @@ -172,7 +117,7 @@ old = *count; for (;;) { - if (REFCOUNT_COUNT(old) <= n) + if (old <= n) return (false); if (__predict_false(REFCOUNT_SATURATED(old))) return (true); @@ -185,7 +130,41 @@ refcount_acquire_if_not_zero(volatile u_int *count) { - return refcount_acquire_if_gt(count, 0); + return (refcount_acquire_if_gt(count, 0)); +} + +static __inline bool +refcount_releasen(volatile u_int *count, u_int n) +{ + u_int old; + + KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, + ("refcount_releasen: n=%u too large", n)); + + atomic_thread_fence_rel(); + old = atomic_fetchadd_int(count, -n); + if (__predict_false(old < n || REFCOUNT_SATURATED(old))) { + _refcount_update_saturated(count); + return (false); + } + if (old > n) + return (false); + + /* + * Last reference. Signal the user to call the destructor. + * + * Ensure that the destructor sees all updates. This synchronizes with + * release fences from all routines which drop the count. + */ + atomic_thread_fence_acq(); + return (true); +} + +static __inline bool +refcount_release(volatile u_int *count) +{ + + return (refcount_releasen(count, 1)); } static __inline __result_use_check bool @@ -197,12 +176,12 @@ ("refcount_release_if_gt: Use refcount_release for final ref")); old = *count; for (;;) { - if (REFCOUNT_COUNT(old) <= n) + if (old <= n) return (false); if (__predict_false(REFCOUNT_SATURATED(old))) return (true); /* - * Paired with acquire fence in refcount_release_last. + * Paired with acquire fence in refcount_releasen(). */ if (atomic_fcmpset_rel_int(count, &old, old - 1)) return (true); @@ -213,6 +192,64 @@ refcount_release_if_not_last(volatile u_int *count) { - return refcount_release_if_gt(count, 1); + return (refcount_release_if_gt(count, 1)); +} + +#ifdef _KERNEL + +struct lock_object; + +void _blockcount_sleep(u_int *count, struct lock_object *, const char *wmesg, + int prio); +void _blockcount_wakeup(u_int *count, u_int old); + +#define BLOCKCOUNT_WAITERS_FLAG (1U << 31) +#define BLOCKCOUNT_COUNT(count) ((count) & ~BLOCKCOUNT_WAITERS_FLAG) +#define BLOCKCOUNT_WAITERS(count) (((count) & BLOCKCOUNT_WAITERS_FLAG) != 0) + +static __inline void +blockcount_init(u_int *count) +{ + + atomic_store_int(count, 0); +} + +static __inline void +blockcount_acquire(u_int *count, u_int n) +{ +#ifdef INVARIANTS + u_int old; + + old = atomic_fetchadd_int(count, n); + KASSERT(old + n > old, ("%s: refcount overflow %p", __func__, count)); +#else + atomic_add_int(count, n); +#endif +} + +static __inline void +blockcount_release(u_int *count, u_int n) +{ + u_int old; + + old = atomic_fetchadd_int(count, -n); + KASSERT(old >= n, ("%s: counter underflow %p", __func__, count)); + if (BLOCKCOUNT_COUNT(old) == n && BLOCKCOUNT_WAITERS(old)) + _blockcount_wakeup(count, old); +} +#define blockcount_sleep(count, lo, wmesg, prio) \ + _blockcount_sleep((count), (struct lock_object *)(lo), (wmesg), (prio)) + +static __inline void +_blockcount_wait(u_int *count, struct lock_object *lo, const char *wmesg, + int prio) +{ + + while (atomic_load_int(count) != 0) + _blockcount_sleep(count, lo, wmesg, prio); } -#endif /* ! __SYS_REFCOUNT_H__ */ +#define blockcount_wait(count, lo, wmesg, prio) \ + _blockcount_wait((count), (struct lock_object *)(lo), (wmesg), (prio)) + +#endif /* _KERNEL */ +#endif /* !__SYS_REFCOUNT_H__ */ Index: sys/vm/vm_fault.c =================================================================== --- sys/vm/vm_fault.c +++ sys/vm/vm_fault.c @@ -377,7 +377,7 @@ { VM_OBJECT_ASSERT_WLOCKED(fs->first_object); - MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0); + MPASS(BLOCKCOUNT_COUNT(fs->first_object->paging_in_progress) > 0); if (!vm_map_trylock_read(fs->map)) { VM_OBJECT_WUNLOCK(fs->first_object); @@ -428,7 +428,7 @@ MPASS(fs->object == fs->first_object); VM_OBJECT_ASSERT_WLOCKED(fs->first_object); - MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0); + MPASS(BLOCKCOUNT_COUNT(fs->first_object->paging_in_progress) > 0); MPASS(fs->first_object->backing_object == NULL); MPASS(fs->lookup_still_valid); Index: sys/vm/vm_object.h =================================================================== --- sys/vm/vm_object.h +++ sys/vm/vm_object.h @@ -113,8 +113,8 @@ objtype_t type; /* type of pager */ u_short flags; /* see below */ u_short pg_color; /* (c) color of first page in obj */ - volatile u_int paging_in_progress; /* Paging (in or out) so don't collapse or destroy */ - volatile u_int busy; /* (a) object is busy, disallow page busy. */ + u_int paging_in_progress; /* (a) Paging (in or out) so don't collapse or destroy */ + u_int busy; /* (a) object is busy, disallow page busy. */ int resident_page_count; /* number of resident pages */ struct vm_object *backing_object; /* object that I'm a shadow of */ vm_ooffset_t backing_object_offset;/* Offset in backing object */ @@ -348,7 +348,11 @@ vm_object_busied(vm_object_t object) { - return (object->busy != 0); + /* + * This may harmlessly return a false positive if the waiter flag + * is still set. + */ + return (atomic_load_int(&object->busy) != 0); } #define VM_OBJECT_ASSERT_BUSY(object) MPASS(vm_object_busied((object))) Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -201,9 +201,9 @@ ("object %p has reservations", object)); #endif - KASSERT(REFCOUNT_COUNT(object->paging_in_progress) == 0, + KASSERT(BLOCKCOUNT_COUNT(object->paging_in_progress) == 0, ("object %p paging_in_progress = %d", - object, REFCOUNT_COUNT(object->paging_in_progress))); + object, BLOCKCOUNT_COUNT(object->paging_in_progress))); KASSERT(object->busy == 0, ("object %p busy = %d", object, object->busy)); @@ -231,7 +231,7 @@ object->type = OBJT_DEAD; vm_radix_init(&object->rtree); refcount_init(&object->ref_count, 0); - refcount_init(&object->paging_in_progress, 0); + blockcount_init(&object->paging_in_progress); refcount_init(&object->busy, 0); object->resident_page_count = 0; object->shadow_count = 0; @@ -363,34 +363,34 @@ vm_object_pip_add(vm_object_t object, short i) { - refcount_acquiren(&object->paging_in_progress, i); + blockcount_acquire(&object->paging_in_progress, i); } void vm_object_pip_wakeup(vm_object_t object) { - refcount_release(&object->paging_in_progress); + vm_object_pip_wakeupn(object, 1); } void vm_object_pip_wakeupn(vm_object_t object, short i) { - refcount_releasen(&object->paging_in_progress, i); + blockcount_release(&object->paging_in_progress, i); } /* - * Atomically drop the interlock and wait for pip to drain. This protects - * from sleep/wakeup races due to identity changes. The lock is not - * re-acquired on return. + * Atomically drop the object lock and wait for pip to drain. This protects + * from sleep/wakeup races due to identity changes. The lock is not re-acquired + * on return. */ static void vm_object_pip_sleep(vm_object_t object, char *waitid) { - refcount_sleep_interlock(&object->paging_in_progress, - &object->lock, waitid, PVM); + blockcount_sleep(&object->paging_in_progress, &object->lock, waitid, + PVM | PDROP); } void @@ -399,10 +399,8 @@ VM_OBJECT_ASSERT_WLOCKED(object); - while (REFCOUNT_COUNT(object->paging_in_progress) > 0) { - vm_object_pip_sleep(object, waitid); - VM_OBJECT_WLOCK(object); - } + blockcount_wait(&object->paging_in_progress, &object->lock, waitid, + PVM); } void @@ -411,8 +409,7 @@ VM_OBJECT_ASSERT_UNLOCKED(object); - while (REFCOUNT_COUNT(object->paging_in_progress) > 0) - refcount_wait(&object->paging_in_progress, waitid, PVM); + blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM); } /* @@ -955,7 +952,7 @@ */ vm_object_pip_wait(object, "objtrm"); - KASSERT(!REFCOUNT_COUNT(object->paging_in_progress), + KASSERT(!BLOCKCOUNT_COUNT(object->paging_in_progress), ("vm_object_terminate: pageout in progress")); KASSERT(object->ref_count == 0, @@ -2458,7 +2455,7 @@ VM_OBJECT_ASSERT_LOCKED(obj); - refcount_acquire(&obj->busy); + blockcount_acquire(&obj->busy, 1); /* The fence is required to order loads of page busy. */ atomic_thread_fence_acq_rel(); } @@ -2467,8 +2464,7 @@ vm_object_unbusy(vm_object_t obj) { - - refcount_release(&obj->busy); + blockcount_release(&obj->busy, 1); } void @@ -2477,8 +2473,8 @@ VM_OBJECT_ASSERT_UNLOCKED(obj); - if (obj->busy) - refcount_sleep(&obj->busy, wmesg, PVM); + if (atomic_load_int(&obj->busy)) + blockcount_sleep(&obj->busy, NULL, wmesg, PVM); } /* Index: sys/vm/vm_swapout.c =================================================================== --- sys/vm/vm_swapout.c +++ sys/vm/vm_swapout.c @@ -218,7 +218,7 @@ goto unlock_return; VM_OBJECT_ASSERT_LOCKED(object); if ((object->flags & OBJ_UNMANAGED) != 0 || - REFCOUNT_COUNT(object->paging_in_progress) > 0) + BLOCKCOUNT_COUNT(object->paging_in_progress) > 0) goto unlock_return; unmap = true;