Index: head/sys/kern/kern_synch.c =================================================================== --- head/sys/kern/kern_synch.c +++ head/sys/kern/kern_synch.c @@ -44,6 +44,7 @@ #include #include +#include #include #include #include @@ -52,7 +53,6 @@ #include #include #include -#include #include #include #include @@ -337,81 +337,6 @@ } /* - * Potentially release the last reference for refcount. Check for - * unlikely conditions and signal the caller as to whether it was - * the final ref. - */ -bool -refcount_release_last(volatile u_int *count, u_int n, u_int old) -{ - u_int waiter; - - waiter = old & REFCOUNT_WAITER; - old = REFCOUNT_COUNT(old); - if (__predict_false(n > old || REFCOUNT_SATURATED(old))) { - /* - * Avoid multiple destructor invocations if underflow occurred. - * This is not perfect since the memory backing the containing - * object may already have been reallocated. - */ - _refcount_update_saturated(count); - return (false); - } - - /* - * Attempt to atomically clear the waiter bit. Wakeup waiters - * if we are successful. - */ - if (waiter != 0 && atomic_cmpset_int(count, REFCOUNT_WAITER, 0)) - wakeup(__DEVOLATILE(u_int *, count)); - - /* - * Last reference. Signal the user to call the destructor. - * - * Ensure that the destructor sees all updates. This synchronizes - * with release fences from all routines which drop the count. - */ - atomic_thread_fence_acq(); - return (true); -} - -/* - * Wait for a refcount wakeup. This does not guarantee that the ref is still - * zero on return and may be subject to transient wakeups. Callers wanting - * a precise answer should use refcount_wait(). - */ -void -_refcount_sleep(volatile u_int *count, struct lock_object *lock, - const char *wmesg, int pri) -{ - void *wchan; - u_int old; - - if (REFCOUNT_COUNT(*count) == 0) { - if (lock != NULL) - LOCK_CLASS(lock)->lc_unlock(lock); - return; - } - wchan = __DEVOLATILE(void *, count); - sleepq_lock(wchan); - if (lock != NULL) - LOCK_CLASS(lock)->lc_unlock(lock); - old = *count; - for (;;) { - if (REFCOUNT_COUNT(old) == 0) { - sleepq_release(wchan); - return; - } - if (old & REFCOUNT_WAITER) - break; - if (atomic_fcmpset_int(count, &old, old | REFCOUNT_WAITER)) - break; - } - sleepq_add(wchan, NULL, wmesg, 0, 0); - sleepq_wait(wchan, pri); -} - -/* * Make all threads sleeping on the specified identifier runnable. */ void @@ -457,6 +382,82 @@ sleepq_release(ident); if (wakeup_swapper) kick_proc0(); +} + +/* + * Signal sleeping waiters after the counter has reached zero. + */ +void +_blockcount_wakeup(blockcount_t *bc, u_int old) +{ + + KASSERT(_BLOCKCOUNT_WAITERS(old), + ("%s: no waiters on %p", __func__, bc)); + + if (atomic_cmpset_int(&bc->__count, _BLOCKCOUNT_WAITERS_FLAG, 0)) + wakeup(bc); +} + +/* + * Wait for a wakeup. This does not guarantee that the count is still zero on + * return and may be subject to transient wakeups. Callers wanting a precise + * answer should use blockcount_wait() with an interlock. + * + * Return 0 if there is no work to wait for, and 1 if we slept waiting for work + * to complete. In the latter case the counter value must be re-read. + */ +int +_blockcount_sleep(blockcount_t *bc, struct lock_object *lock, const char *wmesg, + int prio) +{ + void *wchan; + uintptr_t lock_state; + u_int old; + int ret; + + KASSERT(lock != &Giant.lock_object, + ("%s: cannot use Giant as the interlock", __func__)); + + /* + * Synchronize with the fence in blockcount_release(). If we end up + * waiting, the sleepqueue lock acquisition will provide the required + * side effects. + * + * If there is no work to wait for, but waiters are present, try to put + * ourselves to sleep to avoid jumping ahead. + */ + if (atomic_load_acq_int(&bc->__count) == 0) { + if (lock != NULL && (prio & PDROP) != 0) + LOCK_CLASS(lock)->lc_unlock(lock); + return (0); + } + lock_state = 0; + wchan = bc; + sleepq_lock(wchan); + DROP_GIANT(); + if (lock != NULL) + lock_state = LOCK_CLASS(lock)->lc_unlock(lock); + old = blockcount_read(bc); + do { + if (_BLOCKCOUNT_COUNT(old) == 0) { + sleepq_release(wchan); + ret = 0; + goto out; + } + if (_BLOCKCOUNT_WAITERS(old)) + break; + } while (!atomic_fcmpset_int(&bc->__count, &old, + old | _BLOCKCOUNT_WAITERS_FLAG)); + sleepq_add(wchan, NULL, wmesg, 0, 0); + sleepq_wait(wchan, prio); + ret = 1; + +out: + PICKUP_GIANT(); + if (lock != NULL && (prio & PDROP) == 0) + LOCK_CLASS(lock)->lc_lock(lock, lock_state); + + return (ret); } static void Index: head/sys/kern/vfs_bio.c =================================================================== --- head/sys/kern/vfs_bio.c +++ head/sys/kern/vfs_bio.c @@ -2854,9 +2854,9 @@ bool bogus; obj = bp->b_bufobj->bo_object; - KASSERT(REFCOUNT_COUNT(obj->paging_in_progress) >= bp->b_npages, + KASSERT(blockcount_read(&obj->paging_in_progress) >= bp->b_npages, ("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)", - REFCOUNT_COUNT(obj->paging_in_progress), bp->b_npages)); + blockcount_read(&obj->paging_in_progress), bp->b_npages)); vp = bp->b_vp; VNPASS(vp->v_holdcnt > 0, vp); Index: head/sys/sys/_blockcount.h =================================================================== --- head/sys/sys/_blockcount.h +++ head/sys/sys/_blockcount.h @@ -0,0 +1,52 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Mark Johnston under sponsorship from + * the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __SYS__BLOCKCOUNT_H__ +#define __SYS__BLOCKCOUNT_H__ + +#include + +typedef struct _blockcount { + unsigned int __count; +} blockcount_t; + +#define _BLOCKCOUNT_WAITERS_FLAG (1U << 31) +#define _BLOCKCOUNT_COUNT(c) ((c) & ~_BLOCKCOUNT_WAITERS_FLAG) +#define _BLOCKCOUNT_WAITERS(c) (((c) & _BLOCKCOUNT_WAITERS_FLAG) != 0) + +static inline unsigned int +blockcount_read(blockcount_t *count) +{ + return (_BLOCKCOUNT_COUNT(atomic_load_int(&count->__count))); +} + +#endif /* !__SYS__BLOCKCOUNT_H__ */ Index: head/sys/sys/blockcount.h =================================================================== --- head/sys/sys/blockcount.h +++ head/sys/sys/blockcount.h @@ -0,0 +1,95 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2005 John Baldwin + * Copyright (c) 2020 The FreeBSD Foundation + * + * Portions of this software were developed by Mark Johnston under + * sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __SYS_BLOCKCOUNT_H__ +#define __SYS_BLOCKCOUNT_H__ + +#ifdef _KERNEL + +#include +#include + +struct lock_object; + +int _blockcount_sleep(blockcount_t *bc, struct lock_object *, const char *wmesg, + int prio); +void _blockcount_wakeup(blockcount_t *bc, u_int old); + +static __inline void +blockcount_init(blockcount_t *bc) +{ + atomic_store_int(&bc->__count, 0); +} + +static __inline void +blockcount_acquire(blockcount_t *bc, u_int n) +{ +#ifdef INVARIANTS + u_int old; + + old = atomic_fetchadd_int(&bc->__count, n); + KASSERT(old + n > old, ("%s: counter overflow %p", __func__, bc)); +#else + atomic_add_int(&bc->__count, n); +#endif +} + +static __inline void +blockcount_release(blockcount_t *bc, u_int n) +{ + u_int old; + + atomic_thread_fence_rel(); + old = atomic_fetchadd_int(&bc->__count, -n); + KASSERT(old >= n, ("%s: counter underflow %p", __func__, bc)); + if (_BLOCKCOUNT_COUNT(old) == n && _BLOCKCOUNT_WAITERS(old)) + _blockcount_wakeup(bc, old); +} + +static __inline void +_blockcount_wait(blockcount_t *bc, struct lock_object *lo, const char *wmesg, + int prio) +{ + KASSERT((prio & PDROP) == 0, ("%s: invalid prio %x", __func__, prio)); + + while (_blockcount_sleep(bc, lo, wmesg, prio) != 0) + ; +} + +#define blockcount_sleep(bc, lo, wmesg, prio) \ + _blockcount_sleep((bc), (struct lock_object *)(lo), (wmesg), (prio)) +#define blockcount_wait(bc, lo, wmesg, prio) \ + _blockcount_wait((bc), (struct lock_object *)(lo), (wmesg), (prio)) + +#endif /* _KERNEL */ +#endif /* !__SYS_BLOCKCOUNT_H__ */ Index: head/sys/sys/refcount.h =================================================================== --- head/sys/sys/refcount.h +++ head/sys/sys/refcount.h @@ -34,19 +34,15 @@ #ifdef _KERNEL #include +#include #else #include #define KASSERT(exp, msg) /* */ #endif -#define REFCOUNT_WAITER (1U << 31) /* Refcount has waiter. */ -#define REFCOUNT_SATURATION_VALUE (3U << 29) +#define REFCOUNT_SATURATED(val) (((val) & (1U << 31)) != 0) +#define REFCOUNT_SATURATION_VALUE (3U << 30) -#define REFCOUNT_SATURATED(val) (((val) & (1U << 30)) != 0) -#define REFCOUNT_COUNT(x) ((x) & ~REFCOUNT_WAITER) - -bool refcount_release_last(volatile u_int *count, u_int n, u_int old); - /* * Attempt to handle reference count overflow and underflow. Force the counter * to stay at the saturation value so that a counter overflow cannot trigger @@ -111,56 +107,6 @@ } } -static __inline bool -refcount_releasen(volatile u_int *count, u_int n) -{ - u_int old; - - KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, - ("refcount_releasen: n=%u too large", n)); - - /* - * Paired with acquire fence in refcount_release_last. - */ - atomic_thread_fence_rel(); - old = atomic_fetchadd_int(count, -n); - if (__predict_false(n >= REFCOUNT_COUNT(old) || - REFCOUNT_SATURATED(old))) - return (refcount_release_last(count, n, old)); - return (false); -} - -static __inline bool -refcount_release(volatile u_int *count) -{ - - return (refcount_releasen(count, 1)); -} - -#ifdef _KERNEL -struct lock_object; -void _refcount_sleep(volatile u_int *count, struct lock_object *, - const char *wmesg, int prio); - -static __inline void -refcount_sleep(volatile u_int *count, const char *wmesg, int prio) -{ - - _refcount_sleep(count, NULL, wmesg, prio); -} - -#define refcount_sleep_interlock(count, lock, wmesg, prio) \ - _refcount_sleep((count), (struct lock_object *)(lock), (wmesg), (prio)) - -static __inline void -refcount_wait(volatile u_int *count, const char *wmesg, int prio) -{ - - while (*count != 0) - refcount_sleep(count, wmesg, prio); -} -#endif - /* * This functions returns non-zero if the refcount was * incremented. Else zero is returned. @@ -172,7 +118,7 @@ old = *count; for (;;) { - if (REFCOUNT_COUNT(old) <= n) + if (old <= n) return (false); if (__predict_false(REFCOUNT_SATURATED(old))) return (true); @@ -185,9 +131,43 @@ refcount_acquire_if_not_zero(volatile u_int *count) { - return refcount_acquire_if_gt(count, 0); + return (refcount_acquire_if_gt(count, 0)); } +static __inline bool +refcount_releasen(volatile u_int *count, u_int n) +{ + u_int old; + + KASSERT(n < REFCOUNT_SATURATION_VALUE / 2, + ("refcount_releasen: n=%u too large", n)); + + atomic_thread_fence_rel(); + old = atomic_fetchadd_int(count, -n); + if (__predict_false(old < n || REFCOUNT_SATURATED(old))) { + _refcount_update_saturated(count); + return (false); + } + if (old > n) + return (false); + + /* + * Last reference. Signal the user to call the destructor. + * + * Ensure that the destructor sees all updates. This synchronizes with + * release fences from all routines which drop the count. + */ + atomic_thread_fence_acq(); + return (true); +} + +static __inline bool +refcount_release(volatile u_int *count) +{ + + return (refcount_releasen(count, 1)); +} + static __inline __result_use_check bool refcount_release_if_gt(volatile u_int *count, u_int n) { @@ -197,12 +177,12 @@ ("refcount_release_if_gt: Use refcount_release for final ref")); old = *count; for (;;) { - if (REFCOUNT_COUNT(old) <= n) + if (old <= n) return (false); if (__predict_false(REFCOUNT_SATURATED(old))) return (true); /* - * Paired with acquire fence in refcount_release_last. + * Paired with acquire fence in refcount_releasen(). */ if (atomic_fcmpset_rel_int(count, &old, old - 1)) return (true); @@ -213,6 +193,7 @@ refcount_release_if_not_last(volatile u_int *count) { - return refcount_release_if_gt(count, 1); + return (refcount_release_if_gt(count, 1)); } -#endif /* ! __SYS_REFCOUNT_H__ */ + +#endif /* !__SYS_REFCOUNT_H__ */ Index: head/sys/vm/vm_fault.c =================================================================== --- head/sys/vm/vm_fault.c +++ head/sys/vm/vm_fault.c @@ -377,7 +377,7 @@ { VM_OBJECT_ASSERT_WLOCKED(fs->first_object); - MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0); + MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); if (!vm_map_trylock_read(fs->map)) { VM_OBJECT_WUNLOCK(fs->first_object); @@ -428,7 +428,7 @@ MPASS(fs->object == fs->first_object); VM_OBJECT_ASSERT_WLOCKED(fs->first_object); - MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0); + MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); MPASS(fs->first_object->backing_object == NULL); MPASS(fs->lookup_still_valid); Index: head/sys/vm/vm_object.h =================================================================== --- head/sys/vm/vm_object.h +++ head/sys/vm/vm_object.h @@ -70,6 +70,7 @@ #define _VM_OBJECT_ #include +#include #include #include #include @@ -113,8 +114,8 @@ objtype_t type; /* type of pager */ u_short flags; /* see below */ u_short pg_color; /* (c) color of first page in obj */ - volatile u_int paging_in_progress; /* Paging (in or out) so don't collapse or destroy */ - volatile u_int busy; /* (a) object is busy, disallow page busy. */ + blockcount_t paging_in_progress; /* (a) Paging (in or out) so don't collapse or destroy */ + blockcount_t busy; /* (a) object is busy, disallow page busy. */ int resident_page_count; /* number of resident pages */ struct vm_object *backing_object; /* object that I'm a shadow of */ vm_ooffset_t backing_object_offset;/* Offset in backing object */ @@ -265,7 +266,7 @@ lock_class_rw.lc_lock(&(object)->lock.lock_object, (state)) #define VM_OBJECT_ASSERT_PAGING(object) \ - KASSERT((object)->paging_in_progress != 0, \ + KASSERT(blockcount_read(&(object)->paging_in_progress) != 0, \ ("vm_object %p is not paging", object)) #define VM_OBJECT_ASSERT_REFERENCE(object) \ KASSERT((object)->reference_count != 0, \ @@ -348,7 +349,7 @@ vm_object_busied(vm_object_t object) { - return (object->busy != 0); + return (blockcount_read(&object->busy) != 0); } #define VM_OBJECT_ASSERT_BUSY(object) MPASS(vm_object_busied((object))) Index: head/sys/vm/vm_object.c =================================================================== --- head/sys/vm/vm_object.c +++ head/sys/vm/vm_object.c @@ -71,6 +71,7 @@ #include #include +#include #include #include #include @@ -201,12 +202,11 @@ ("object %p has reservations", object)); #endif - KASSERT(REFCOUNT_COUNT(object->paging_in_progress) == 0, + KASSERT(blockcount_read(&object->paging_in_progress) == 0, ("object %p paging_in_progress = %d", - object, REFCOUNT_COUNT(object->paging_in_progress))); - KASSERT(object->busy == 0, - ("object %p busy = %d", - object, object->busy)); + object, blockcount_read(&object->paging_in_progress))); + KASSERT(!vm_object_busied(object), + ("object %p busy = %d", object, blockcount_read(&object->busy))); KASSERT(object->resident_page_count == 0, ("object %p resident_page_count = %d", object, object->resident_page_count)); @@ -231,8 +231,8 @@ object->type = OBJT_DEAD; vm_radix_init(&object->rtree); refcount_init(&object->ref_count, 0); - refcount_init(&object->paging_in_progress, 0); - refcount_init(&object->busy, 0); + blockcount_init(&object->paging_in_progress); + blockcount_init(&object->busy); object->resident_page_count = 0; object->shadow_count = 0; object->flags = OBJ_DEAD; @@ -363,34 +363,36 @@ vm_object_pip_add(vm_object_t object, short i) { - refcount_acquiren(&object->paging_in_progress, i); + if (i > 0) + blockcount_acquire(&object->paging_in_progress, i); } void vm_object_pip_wakeup(vm_object_t object) { - refcount_release(&object->paging_in_progress); + vm_object_pip_wakeupn(object, 1); } void vm_object_pip_wakeupn(vm_object_t object, short i) { - refcount_releasen(&object->paging_in_progress, i); + if (i > 0) + blockcount_release(&object->paging_in_progress, i); } /* - * Atomically drop the interlock and wait for pip to drain. This protects - * from sleep/wakeup races due to identity changes. The lock is not - * re-acquired on return. + * Atomically drop the object lock and wait for pip to drain. This protects + * from sleep/wakeup races due to identity changes. The lock is not re-acquired + * on return. */ static void vm_object_pip_sleep(vm_object_t object, const char *waitid) { - refcount_sleep_interlock(&object->paging_in_progress, - &object->lock, waitid, PVM); + (void)blockcount_sleep(&object->paging_in_progress, &object->lock, + waitid, PVM | PDROP); } void @@ -399,10 +401,8 @@ VM_OBJECT_ASSERT_WLOCKED(object); - while (REFCOUNT_COUNT(object->paging_in_progress) > 0) { - vm_object_pip_sleep(object, waitid); - VM_OBJECT_WLOCK(object); - } + blockcount_wait(&object->paging_in_progress, &object->lock, waitid, + PVM); } void @@ -411,8 +411,7 @@ VM_OBJECT_ASSERT_UNLOCKED(object); - while (REFCOUNT_COUNT(object->paging_in_progress) > 0) - refcount_wait(&object->paging_in_progress, waitid, PVM); + blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM); } /* @@ -955,7 +954,7 @@ */ vm_object_pip_wait(object, "objtrm"); - KASSERT(!REFCOUNT_COUNT(object->paging_in_progress), + KASSERT(!blockcount_read(&object->paging_in_progress), ("vm_object_terminate: pageout in progress")); KASSERT(object->ref_count == 0, @@ -2458,7 +2457,7 @@ VM_OBJECT_ASSERT_LOCKED(obj); - refcount_acquire(&obj->busy); + blockcount_acquire(&obj->busy, 1); /* The fence is required to order loads of page busy. */ atomic_thread_fence_acq_rel(); } @@ -2467,8 +2466,7 @@ vm_object_unbusy(vm_object_t obj) { - - refcount_release(&obj->busy); + blockcount_release(&obj->busy, 1); } void @@ -2477,8 +2475,7 @@ VM_OBJECT_ASSERT_UNLOCKED(obj); - if (obj->busy) - refcount_sleep(&obj->busy, wmesg, PVM); + (void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM); } /* Index: head/sys/vm/vm_pager.h =================================================================== --- head/sys/vm/vm_pager.h +++ head/sys/vm/vm_pager.h @@ -168,7 +168,7 @@ MPASS((object->flags & OBJ_POPULATE) != 0); MPASS(pidx < object->size); - MPASS(object->paging_in_progress > 0); + MPASS(blockcount_read(&object->paging_in_progress) > 0); return ((*pagertab[object->type]->pgo_populate)(object, pidx, fault_type, max_prot, first, last)); } Index: head/sys/vm/vm_swapout.c =================================================================== --- head/sys/vm/vm_swapout.c +++ head/sys/vm/vm_swapout.c @@ -218,7 +218,7 @@ goto unlock_return; VM_OBJECT_ASSERT_LOCKED(object); if ((object->flags & OBJ_UNMANAGED) != 0 || - REFCOUNT_COUNT(object->paging_in_progress) > 0) + blockcount_read(&object->paging_in_progress) > 0) goto unlock_return; unmap = true;