Index: kern/kern_synch.c =================================================================== --- kern/kern_synch.c +++ kern/kern_synch.c @@ -335,72 +335,61 @@ } /* - * Potentially release the last reference for refcount. Check for - * unlikely conditions and signal the caller as to whether it was - * the final ref. + * Release the last reference of the given refcount and wakeup + * waiters, if any. */ -bool -refcount_release_last(volatile u_int *count, u_int n, u_int old) +void +refcount_release_last(volatile u_int *count) { - u_int waiter; + u_int old; - waiter = old & REFCOUNT_WAITER; - old = REFCOUNT_COUNT(old); - if (__predict_false(n > old || REFCOUNT_SATURATED(old))) { - /* - * Avoid multiple destructor invocations if underflow occurred. - * This is not perfect since the memory backing the containing - * object may already have been reallocated. - */ - _refcount_update_saturated(count); - return (false); + old = *count; + for (;;) { + if (atomic_fcmpset_int(count, &old, 0)) + break; } - /* - * Attempt to atomically clear the waiter bit. Wakeup waiters - * if we are successful. - */ - if (waiter != 0 && atomic_cmpset_int(count, REFCOUNT_WAITER, 0)) + /* check if we should wakeup other thread */ + if (old & REFCOUNT_WAIT_MASK) wakeup(__DEVOLATILE(u_int *, count)); - - /* - * Last reference. Signal the user to call the destructor. - * - * Ensure that the destructor sees all updates. The fence_rel - * at the start of refcount_releasen synchronizes with this fence. - */ - atomic_thread_fence_acq(); - return (true); } /* - * Wait for a refcount wakeup. This does not guarantee that the ref is still - * zero on return and may be subject to transient wakeups. Callers wanting - * a precise answer should use refcount_wait(). + * Wait for refcount to reach zero. This function should be paired with + * refcount_release_last() */ void -refcount_sleep(volatile u_int *count, const char *wmesg, int pri) +refcount_wait_last(volatile u_int *count, const char *wmesg, int pri) { void *wchan; u_int old; - if (REFCOUNT_COUNT(*count) == 0) - return; + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "refcount_wait_last() can sleep"); + wchan = __DEVOLATILE(void *, count); - sleepq_lock(wchan); - old = *count; for (;;) { - if (REFCOUNT_COUNT(old) == 0) { + old = *count; + if (old == 0) + break; + + DROP_GIANT(); + sleepq_lock(wchan); + /* try to get WAIT bit added */ + while ((old & REFCOUNT_WAIT_MASK) == 0) { + if (atomic_fcmpset_int(count, &old, old | REFCOUNT_WAIT_MASK)) + break; + else if (old == 0) + break; + } + if (old != 0) { + sleepq_add(wchan, NULL, wmesg, 0, 0); + sleepq_wait(wchan, pri); + } else { sleepq_release(wchan); - return; } - if (old & REFCOUNT_WAITER) - break; - if (atomic_fcmpset_int(count, &old, old | REFCOUNT_WAITER)) - break; + PICKUP_GIANT(); } - sleepq_add(wchan, NULL, wmesg, 0, 0); - sleepq_wait(wchan, pri); } /* Index: kern/vfs_bio.c =================================================================== --- kern/vfs_bio.c +++ kern/vfs_bio.c @@ -67,6 +67,7 @@ #include #include #include +#include #include #include #include @@ -2829,9 +2830,9 @@ bool bogus; obj = bp->b_bufobj->bo_object; - KASSERT(obj->paging_in_progress >= bp->b_npages, + KASSERT(REFCOUNT_VALUE(obj->paging_in_progress) >= bp->b_npages, ("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)", - obj->paging_in_progress, bp->b_npages)); + REFCOUNT_VALUE(obj->paging_in_progress), bp->b_npages)); vp = bp->b_vp; KASSERT(vp->v_holdcnt > 0, Index: sys/refcount.h =================================================================== --- sys/refcount.h +++ sys/refcount.h @@ -39,14 +39,14 @@ #define KASSERT(exp, msg) /* */ #endif -#define REFCOUNT_WAITER (1 << 31) /* Refcount has waiter. */ +#define REFCOUNT_WAIT_MASK (1U << 31) #define REFCOUNT_SATURATION_VALUE (3U << 29) #define REFCOUNT_SATURATED(val) (((val) & (1U << 30)) != 0) -#define REFCOUNT_COUNT(x) ((x) & ~REFCOUNT_WAITER) +#define REFCOUNT_VALUE(x) ((x) & ~REFCOUNT_WAIT_MASK) -bool refcount_release_last(volatile u_int *count, u_int n, u_int old); -void refcount_sleep(volatile u_int *count, const char *wmesg, int prio); +void refcount_release_last(volatile u_int *count); +void refcount_wait_last(volatile u_int *count, const char *wmesg, int prio); /* * Attempt to handle reference count overflow and underflow. Force the counter @@ -65,7 +65,7 @@ } static __inline void -refcount_init(volatile u_int *count, u_int value) +refcount_init(volatile u_int *count, const u_int value) { KASSERT(!REFCOUNT_SATURATED(value), ("invalid initial refcount value %u", value)); @@ -83,13 +83,12 @@ } static __inline void -refcount_acquiren(volatile u_int *count, u_int n) +refcount_acquiren(volatile u_int *count, const u_int n) { - u_int old; - KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, - ("refcount_acquiren: n %d too large", n)); + KASSERT(n < (REFCOUNT_SATURATION_VALUE / 2U), + ("refcount_acquiren: n=%u too large", n)); old = atomic_fetchadd_int(count, n); if (__predict_false(REFCOUNT_SATURATED(old))) _refcount_update_saturated(count); @@ -110,18 +109,29 @@ } static __inline bool -refcount_releasen(volatile u_int *count, u_int n) +refcount_releasen(volatile u_int *count, const u_int n) { u_int old; - KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, - ("refcount_releasen: n %d too large", n)); + KASSERT(n < (REFCOUNT_SATURATION_VALUE / 2U), + ("refcount_releasen: n=%u too large", n)); + atomic_thread_fence_rel(); old = atomic_fetchadd_int(count, -n); - if (__predict_false(n >= REFCOUNT_COUNT(old) || - REFCOUNT_SATURATED(old))) - return (refcount_release_last(count, n, old)); - return (false); + KASSERT(n <= REFCOUNT_VALUE(old), + ("refcount_releasen: n=%u > old=%u", n, REFCOUNT_VALUE(old))); + if (__predict_false(n != REFCOUNT_VALUE(old) || REFCOUNT_SATURATED(old))) + return (false); + + /* + * Last reference. Signal the user to call the destructor. + * + * Ensure that the destructor sees all updates. The fence_rel + * at the start of refcount_releasen synchronizes with this + * fence. + */ + atomic_thread_fence_acq(); + return (true); } static __inline bool @@ -131,14 +141,6 @@ return (refcount_releasen(count, 1)); } -static __inline void -refcount_wait(volatile u_int *count, const char *wmesg, int prio) -{ - - while (*count != 0) - refcount_sleep(count, wmesg, prio); -} - /* * This functions returns non-zero if the refcount was * incremented. Else zero is returned. @@ -150,7 +152,7 @@ old = *count; for (;;) { - if (REFCOUNT_COUNT(old) == 0) + if (__predict_false(REFCOUNT_VALUE(old) == 0)) return (false); if (__predict_false(REFCOUNT_SATURATED(old))) return (true); @@ -160,19 +162,44 @@ } static __inline __result_use_check bool -refcount_release_if_not_last(volatile u_int *count) +refcount_releasen_if_not_last(volatile u_int *count, const u_int n) { u_int old; + KASSERT(n < (REFCOUNT_SATURATION_VALUE / 2U), + ("refcount_releasen_if_not_last: n=%u too large", n)); + + atomic_thread_fence_rel(); old = *count; for (;;) { - if (REFCOUNT_COUNT(old) == 1) + KASSERT(n <= REFCOUNT_VALUE(old), + ("refcount_releasen_if_not_last: n=%u > old=%u", n, + REFCOUNT_VALUE(old))); + if (__predict_false(REFCOUNT_SATURATED(old))) { + return (true); + } else if (__predict_false(REFCOUNT_VALUE(old) == n)) { + /* + * Last reference. Signal the user to call the + * destructor. + * + * Ensure that the destructor sees all + * updates. The fence_rel at the start of + * refcount_release_if_not_last() synchronizes + * with this fence. + */ + atomic_thread_fence_acq(); return (false); - if (__predict_false(REFCOUNT_SATURATED(old))) + } else if (atomic_fcmpset_int(count, &old, old - n)) { return (true); - if (atomic_fcmpset_int(count, &old, old - 1)) - return (true); + } } } +static __inline __result_use_check bool +refcount_release_if_not_last(volatile u_int *count) +{ + + return (refcount_releasen_if_not_last(count, 1)); +} + #endif /* ! __SYS_REFCOUNT_H__ */ Index: vm/vm_fault.c =================================================================== --- vm/vm_fault.c +++ vm/vm_fault.c @@ -87,6 +87,7 @@ #include #include #include +#include #include #include #include @@ -339,7 +340,7 @@ { VM_OBJECT_ASSERT_WLOCKED(fs->first_object); - MPASS(fs->first_object->paging_in_progress > 0); + MPASS(REFCOUNT_VALUE(fs->first_object->paging_in_progress) > 0); if (!vm_map_trylock_read(fs->map)) { VM_OBJECT_WUNLOCK(fs->first_object); @@ -394,7 +395,7 @@ MPASS(fs->object == fs->first_object); VM_OBJECT_ASSERT_WLOCKED(fs->first_object); - MPASS(fs->first_object->paging_in_progress > 0); + MPASS(REFCOUNT_VALUE(fs->first_object->paging_in_progress) > 0); MPASS(fs->first_object->backing_object == NULL); MPASS(fs->lookup_still_valid); Index: vm/vm_object.c =================================================================== --- vm/vm_object.c +++ vm/vm_object.c @@ -195,9 +195,9 @@ ("object %p has reservations", object)); #endif - KASSERT(object->paging_in_progress == 0, + KASSERT(REFCOUNT_VALUE(object->paging_in_progress) == 0, ("object %p paging_in_progress = %d", - object, object->paging_in_progress)); + object, REFCOUNT_VALUE(object->paging_in_progress))); KASSERT(object->resident_page_count == 0, ("object %p resident_page_count = %d", object, object->resident_page_count)); @@ -379,14 +379,16 @@ vm_object_pip_wakeup(vm_object_t object) { - refcount_release(&object->paging_in_progress); + if (!refcount_release_if_not_last(&object->paging_in_progress)) + refcount_release_last(&object->paging_in_progress); } void vm_object_pip_wakeupn(vm_object_t object, short i) { - refcount_releasen(&object->paging_in_progress, i); + if (!refcount_releasen_if_not_last(&object->paging_in_progress, i)) + refcount_release_last(&object->paging_in_progress); } void @@ -395,9 +397,9 @@ VM_OBJECT_ASSERT_WLOCKED(object); - while (object->paging_in_progress) { + while (REFCOUNT_VALUE(object->paging_in_progress)) { VM_OBJECT_WUNLOCK(object); - refcount_wait(&object->paging_in_progress, waitid, PVM); + refcount_wait_last(&object->paging_in_progress, waitid, PVM); VM_OBJECT_WLOCK(object); } } @@ -408,8 +410,8 @@ VM_OBJECT_ASSERT_UNLOCKED(object); - while (object->paging_in_progress) - refcount_wait(&object->paging_in_progress, waitid, PVM); + while (REFCOUNT_VALUE(object->paging_in_progress)) + refcount_wait_last(&object->paging_in_progress, waitid, PVM); } /* @@ -577,7 +579,7 @@ robject->ref_count++; retry: - if (robject->paging_in_progress) { + if (REFCOUNT_VALUE(robject->paging_in_progress)) { VM_OBJECT_WUNLOCK(object); vm_object_pip_wait(robject, "objde1"); @@ -586,10 +588,10 @@ VM_OBJECT_WLOCK(object); goto retry; } - } else if (object->paging_in_progress) { + } else if (REFCOUNT_VALUE(object->paging_in_progress)) { VM_OBJECT_WUNLOCK(robject); VM_OBJECT_WUNLOCK(object); - refcount_wait( + refcount_wait_last( &object->paging_in_progress, "objde2", PVM); VM_OBJECT_WLOCK(robject); @@ -729,7 +731,7 @@ */ vm_object_pip_wait(object, "objtrm"); - KASSERT(!object->paging_in_progress, + KASSERT(!REFCOUNT_VALUE(object->paging_in_progress), ("vm_object_terminate: pageout in progress")); KASSERT(object->ref_count == 0, @@ -1660,8 +1662,8 @@ break; } - if (object->paging_in_progress != 0 || - backing_object->paging_in_progress != 0) { + if (REFCOUNT_VALUE(object->paging_in_progress) != 0 || + REFCOUNT_VALUE(backing_object->paging_in_progress) != 0) { vm_object_qcollapse(object); VM_OBJECT_WUNLOCK(backing_object); break; Index: vm/vm_swapout.c =================================================================== --- vm/vm_swapout.c +++ vm/vm_swapout.c @@ -90,6 +90,7 @@ #include #include #include +#include #include #include #include @@ -193,7 +194,7 @@ goto unlock_return; VM_OBJECT_ASSERT_LOCKED(object); if ((object->flags & OBJ_UNMANAGED) != 0 || - object->paging_in_progress != 0) + REFCOUNT_VALUE(object->paging_in_progress) != 0) goto unlock_return; remove_mode = 0;