Page MenuHomeFreeBSD
Files F162733861

D23723.id68432.diff
No OneTemporary
Actions

D23723.id68432.diff
View Options

Index: sys/kern/kern_synch.c
===================================================================
--- sys/kern/kern_synch.c
+++ sys/kern/kern_synch.c
@@ -336,81 +336,6 @@
(flags & C_CATCH) ? PCATCH : 0, wmesg, sbt, pr, flags));
}
-/*
- * Potentially release the last reference for refcount. Check for
- * unlikely conditions and signal the caller as to whether it was
- * the final ref.
- */
-bool
-refcount_release_last(volatile u_int *count, u_int n, u_int old)
-{
- u_int waiter;
-
- waiter = old & REFCOUNT_WAITER;
- old = REFCOUNT_COUNT(old);
- if (__predict_false(n > old || REFCOUNT_SATURATED(old))) {
- /*
- * Avoid multiple destructor invocations if underflow occurred.
- * This is not perfect since the memory backing the containing
- * object may already have been reallocated.
- */
- _refcount_update_saturated(count);
- return (false);
- }
-
- /*
- * Attempt to atomically clear the waiter bit. Wakeup waiters
- * if we are successful.
- */
- if (waiter != 0 && atomic_cmpset_int(count, REFCOUNT_WAITER, 0))
- wakeup(__DEVOLATILE(u_int *, count));
-
- /*
- * Last reference. Signal the user to call the destructor.
- *
- * Ensure that the destructor sees all updates. This synchronizes
- * with release fences from all routines which drop the count.
- */
- atomic_thread_fence_acq();
- return (true);
-}
-
-/*
- * Wait for a refcount wakeup. This does not guarantee that the ref is still
- * zero on return and may be subject to transient wakeups. Callers wanting
- * a precise answer should use refcount_wait().
- */
-void
-_refcount_sleep(volatile u_int *count, struct lock_object *lock,
- const char *wmesg, int pri)
-{
- void *wchan;
- u_int old;
-
- if (REFCOUNT_COUNT(*count) == 0) {
- if (lock != NULL)
- LOCK_CLASS(lock)->lc_unlock(lock);
- return;
- }
- wchan = __DEVOLATILE(void *, count);
- sleepq_lock(wchan);
- if (lock != NULL)
- LOCK_CLASS(lock)->lc_unlock(lock);
- old = *count;
- for (;;) {
- if (REFCOUNT_COUNT(old) == 0) {
- sleepq_release(wchan);
- return;
- }
- if (old & REFCOUNT_WAITER)
- break;
- if (atomic_fcmpset_int(count, &old, old | REFCOUNT_WAITER))
- break;
- }
- sleepq_add(wchan, NULL, wmesg, 0, 0);
- sleepq_wait(wchan, pri);
-}
-
/*
* Make all threads sleeping on the specified identifier runnable.
*/
@@ -459,6 +384,56 @@
kick_proc0();
}
+void
+_blockcount_wakeup(u_int *count, u_int old)
+{
+
+ KASSERT(BLOCKCOUNT_WAITERS(old),
+ ("blockcount_wakeup: no waiters on %p", count));
+
+ if (atomic_cmpset_int(count, BLOCKCOUNT_WAITERS_FLAG, 0))
+ wakeup(__DEVOLATILE(u_int *, count));
+}
+
+/*
+ * Wait for a wakeup. This does not guarantee that the count is still zero on
+ * return and may be subject to transient wakeups. Callers wanting a precise
+ * answer should use refcount_wait().
+ */
+void
+_blockcount_sleep(u_int *count, struct lock_object *lock, const char *wmesg,
+ int prio)
+{
+ void *wchan;
+ uintptr_t lock_state;
+ u_int old;
+
+ if (atomic_load_int(count) == 0) {
+ if (lock != NULL)
+ LOCK_CLASS(lock)->lc_unlock(lock);
+ return;
+ }
+ lock_state = 0;
+ wchan = __DEVOLATILE(void *, count);
+ sleepq_lock(wchan);
+ if (lock != NULL)
+ lock_state = LOCK_CLASS(lock)->lc_unlock(lock);
+ old = atomic_load_int(count);
+ do {
+ if (BLOCKCOUNT_COUNT(old) == 0) {
+ sleepq_release(wchan);
+ return;
+ }
+ if (BLOCKCOUNT_WAITERS(old))
+ break;
+ } while (!atomic_fcmpset_int(count, &old,
+ old | BLOCKCOUNT_WAITERS_FLAG));
+ sleepq_add(wchan, NULL, wmesg, 0, 0);
+ sleepq_wait(wchan, prio);
+ if (lock != NULL && (prio & PDROP) == 0)
+ LOCK_CLASS(lock)->lc_lock(lock, lock_state);
+}
+
static void
kdb_switch(void)
{
Index: sys/kern/vfs_bio.c
===================================================================
--- sys/kern/vfs_bio.c
+++ sys/kern/vfs_bio.c
@@ -2854,9 +2854,9 @@
bool bogus;
obj = bp->b_bufobj->bo_object;
- KASSERT(REFCOUNT_COUNT(obj->paging_in_progress) >= bp->b_npages,
+ KASSERT(BLOCKCOUNT_COUNT(obj->paging_in_progress) >= bp->b_npages,
("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)",
- REFCOUNT_COUNT(obj->paging_in_progress), bp->b_npages));
+ BLOCKCOUNT_COUNT(obj->paging_in_progress), bp->b_npages));
vp = bp->b_vp;
VNPASS(vp->v_holdcnt > 0, vp);
Index: sys/net/if.c
===================================================================
--- sys/net/if.c
+++ sys/net/if.c
@@ -3380,9 +3380,6 @@
* counting, notifying the driver, handling routing messages, and releasing
* any dependent link layer state.
*/
-#ifdef MCAST_VERBOSE
-extern void kdb_backtrace(void);
-#endif
static void
if_freemulti_internal(struct ifmultiaddr *ifma)
{
@@ -3392,10 +3389,6 @@
if (ifma->ifma_lladdr != NULL)
free(ifma->ifma_lladdr, M_IFMADDR);
-#ifdef MCAST_VERBOSE
- kdb_backtrace();
- printf("%s freeing ifma: %p\n", __func__, ifma);
-#endif
free(ifma->ifma_addr, M_IFMADDR);
free(ifma, M_IFMADDR);
}
Index: sys/sys/refcount.h
===================================================================
--- sys/sys/refcount.h
+++ sys/sys/refcount.h
@@ -39,13 +39,8 @@
#define KASSERT(exp, msg) /* */
#endif
-#define REFCOUNT_WAITER (1U << 31) /* Refcount has waiter. */
-#define REFCOUNT_SATURATION_VALUE (3U << 29)
-
-#define REFCOUNT_SATURATED(val) (((val) & (1U << 30)) != 0)
-#define REFCOUNT_COUNT(x) ((x) & ~REFCOUNT_WAITER)
-
-bool refcount_release_last(volatile u_int *count, u_int n, u_int old);
+#define REFCOUNT_SATURATED(val) (((val) & (1U << 31)) != 0)
+#define REFCOUNT_SATURATION_VALUE (3U << 30)
/*
* Attempt to handle reference count overflow and underflow. Force the counter
@@ -111,56 +106,6 @@
}
}
-static __inline bool
-refcount_releasen(volatile u_int *count, u_int n)
-{
- u_int old;
-
- KASSERT(n < REFCOUNT_SATURATION_VALUE / 2,
- ("refcount_releasen: n=%u too large", n));
-
- /*
- * Paired with acquire fence in refcount_release_last.
- */
- atomic_thread_fence_rel();
- old = atomic_fetchadd_int(count, -n);
- if (__predict_false(n >= REFCOUNT_COUNT(old) ||
- REFCOUNT_SATURATED(old)))
- return (refcount_release_last(count, n, old));
- return (false);
-}
-
-static __inline bool
-refcount_release(volatile u_int *count)
-{
-
- return (refcount_releasen(count, 1));
-}
-
-#ifdef _KERNEL
-struct lock_object;
-void _refcount_sleep(volatile u_int *count, struct lock_object *,
- const char *wmesg, int prio);
-
-static __inline void
-refcount_sleep(volatile u_int *count, const char *wmesg, int prio)
-{
-
- _refcount_sleep(count, NULL, wmesg, prio);
-}
-
-#define refcount_sleep_interlock(count, lock, wmesg, prio) \
- _refcount_sleep((count), (struct lock_object *)(lock), (wmesg), (prio))
-
-static __inline void
-refcount_wait(volatile u_int *count, const char *wmesg, int prio)
-{
-
- while (*count != 0)
- refcount_sleep(count, wmesg, prio);
-}
-#endif
-
/*
* This functions returns non-zero if the refcount was
* incremented. Else zero is returned.
@@ -172,7 +117,7 @@
old = *count;
for (;;) {
- if (REFCOUNT_COUNT(old) <= n)
+ if (old <= n)
return (false);
if (__predict_false(REFCOUNT_SATURATED(old)))
return (true);
@@ -185,7 +130,41 @@
refcount_acquire_if_not_zero(volatile u_int *count)
{
- return refcount_acquire_if_gt(count, 0);
+ return (refcount_acquire_if_gt(count, 0));
+}
+
+static __inline bool
+refcount_releasen(volatile u_int *count, u_int n)
+{
+ u_int old;
+
+ KASSERT(n < REFCOUNT_SATURATION_VALUE / 2,
+ ("refcount_releasen: n=%u too large", n));
+
+ atomic_thread_fence_rel();
+ old = atomic_fetchadd_int(count, -n);
+ if (__predict_false(old < n || REFCOUNT_SATURATED(old))) {
+ _refcount_update_saturated(count);
+ return (false);
+ }
+ if (old > n)
+ return (false);
+
+ /*
+ * Last reference. Signal the user to call the destructor.
+ *
+ * Ensure that the destructor sees all updates. This synchronizes with
+ * release fences from all routines which drop the count.
+ */
+ atomic_thread_fence_acq();
+ return (true);
+}
+
+static __inline bool
+refcount_release(volatile u_int *count)
+{
+
+ return (refcount_releasen(count, 1));
}
static __inline __result_use_check bool
@@ -197,12 +176,12 @@
("refcount_release_if_gt: Use refcount_release for final ref"));
old = *count;
for (;;) {
- if (REFCOUNT_COUNT(old) <= n)
+ if (old <= n)
return (false);
if (__predict_false(REFCOUNT_SATURATED(old)))
return (true);
/*
- * Paired with acquire fence in refcount_release_last.
+ * Paired with acquire fence in refcount_releasen().
*/
if (atomic_fcmpset_rel_int(count, &old, old - 1))
return (true);
@@ -213,6 +192,64 @@
refcount_release_if_not_last(volatile u_int *count)
{
- return refcount_release_if_gt(count, 1);
+ return (refcount_release_if_gt(count, 1));
+}
+
+#ifdef _KERNEL
+
+struct lock_object;
+
+void _blockcount_sleep(u_int *count, struct lock_object *, const char *wmesg,
+ int prio);
+void _blockcount_wakeup(u_int *count, u_int old);
+
+#define BLOCKCOUNT_WAITERS_FLAG (1U << 31)
+#define BLOCKCOUNT_COUNT(count) ((count) & ~BLOCKCOUNT_WAITERS_FLAG)
+#define BLOCKCOUNT_WAITERS(count) (((count) & BLOCKCOUNT_WAITERS_FLAG) != 0)
+
+static __inline void
+blockcount_init(u_int *count)
+{
+
+ atomic_store_int(count, 0);
+}
+
+static __inline void
+blockcount_acquire(u_int *count, u_int n)
+{
+#ifdef INVARIANTS
+ u_int old;
+
+ old = atomic_fetchadd_int(count, n);
+ KASSERT(old + n > old, ("%s: refcount overflow %p", __func__, count));
+#else
+ atomic_add_int(count, n);
+#endif
+}
+
+static __inline void
+blockcount_release(u_int *count, u_int n)
+{
+ u_int old;
+
+ old = atomic_fetchadd_int(count, -n);
+ KASSERT(old >= n, ("%s: counter underflow %p", __func__, count));
+ if (BLOCKCOUNT_COUNT(old) == n && BLOCKCOUNT_WAITERS(old))
+ _blockcount_wakeup(count, old);
+}
+#define blockcount_sleep(count, lo, wmesg, prio) \
+ _blockcount_sleep((count), (struct lock_object *)(lo), (wmesg), (prio))
+
+static __inline void
+_blockcount_wait(u_int *count, struct lock_object *lo, const char *wmesg,
+ int prio)
+{
+
+ while (atomic_load_int(count) != 0)
+ _blockcount_sleep(count, lo, wmesg, prio);
}
-#endif /* ! __SYS_REFCOUNT_H__ */
+#define blockcount_wait(count, lo, wmesg, prio) \
+ _blockcount_wait((count), (struct lock_object *)(lo), (wmesg), (prio))
+
+#endif /* _KERNEL */
+#endif /* !__SYS_REFCOUNT_H__ */
Index: sys/vm/vm_fault.c
===================================================================
--- sys/vm/vm_fault.c
+++ sys/vm/vm_fault.c
@@ -377,7 +377,7 @@
{
VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
- MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0);
+ MPASS(BLOCKCOUNT_COUNT(fs->first_object->paging_in_progress) > 0);
if (!vm_map_trylock_read(fs->map)) {
VM_OBJECT_WUNLOCK(fs->first_object);
@@ -428,7 +428,7 @@
MPASS(fs->object == fs->first_object);
VM_OBJECT_ASSERT_WLOCKED(fs->first_object);
- MPASS(REFCOUNT_COUNT(fs->first_object->paging_in_progress) > 0);
+ MPASS(BLOCKCOUNT_COUNT(fs->first_object->paging_in_progress) > 0);
MPASS(fs->first_object->backing_object == NULL);
MPASS(fs->lookup_still_valid);
Index: sys/vm/vm_object.h
===================================================================
--- sys/vm/vm_object.h
+++ sys/vm/vm_object.h
@@ -113,8 +113,8 @@
objtype_t type; /* type of pager */
u_short flags; /* see below */
u_short pg_color; /* (c) color of first page in obj */
- volatile u_int paging_in_progress; /* Paging (in or out) so don't collapse or destroy */
- volatile u_int busy; /* (a) object is busy, disallow page busy. */
+ u_int paging_in_progress; /* (a) Paging (in or out) so don't collapse or destroy */
+ u_int busy; /* (a) object is busy, disallow page busy. */
int resident_page_count; /* number of resident pages */
struct vm_object *backing_object; /* object that I'm a shadow of */
vm_ooffset_t backing_object_offset;/* Offset in backing object */
@@ -348,7 +348,11 @@
vm_object_busied(vm_object_t object)
{
- return (object->busy != 0);
+ /*
+ * This may harmlessly return a false positive if the waiter flag
+ * is still set.
+ */
+ return (atomic_load_int(&object->busy) != 0);
}
#define VM_OBJECT_ASSERT_BUSY(object) MPASS(vm_object_busied((object)))
Index: sys/vm/vm_object.c
===================================================================
--- sys/vm/vm_object.c
+++ sys/vm/vm_object.c
@@ -201,9 +201,9 @@
("object %p has reservations",
object));
#endif
- KASSERT(REFCOUNT_COUNT(object->paging_in_progress) == 0,
+ KASSERT(BLOCKCOUNT_COUNT(object->paging_in_progress) == 0,
("object %p paging_in_progress = %d",
- object, REFCOUNT_COUNT(object->paging_in_progress)));
+ object, BLOCKCOUNT_COUNT(object->paging_in_progress)));
KASSERT(object->busy == 0,
("object %p busy = %d",
object, object->busy));
@@ -231,7 +231,7 @@
object->type = OBJT_DEAD;
vm_radix_init(&object->rtree);
refcount_init(&object->ref_count, 0);
- refcount_init(&object->paging_in_progress, 0);
+ blockcount_init(&object->paging_in_progress);
refcount_init(&object->busy, 0);
object->resident_page_count = 0;
object->shadow_count = 0;
@@ -363,34 +363,34 @@
vm_object_pip_add(vm_object_t object, short i)
{
- refcount_acquiren(&object->paging_in_progress, i);
+ blockcount_acquire(&object->paging_in_progress, i);
}
void
vm_object_pip_wakeup(vm_object_t object)
{
- refcount_release(&object->paging_in_progress);
+ vm_object_pip_wakeupn(object, 1);
}
void
vm_object_pip_wakeupn(vm_object_t object, short i)
{
- refcount_releasen(&object->paging_in_progress, i);
+ blockcount_release(&object->paging_in_progress, i);
}
/*
- * Atomically drop the interlock and wait for pip to drain. This protects
- * from sleep/wakeup races due to identity changes. The lock is not
- * re-acquired on return.
+ * Atomically drop the object lock and wait for pip to drain. This protects
+ * from sleep/wakeup races due to identity changes. The lock is not re-acquired
+ * on return.
*/
static void
vm_object_pip_sleep(vm_object_t object, char *waitid)
{
- refcount_sleep_interlock(&object->paging_in_progress,
- &object->lock, waitid, PVM);
+ blockcount_sleep(&object->paging_in_progress, &object->lock, waitid,
+ PVM | PDROP);
}
void
@@ -399,10 +399,8 @@
VM_OBJECT_ASSERT_WLOCKED(object);
- while (REFCOUNT_COUNT(object->paging_in_progress) > 0) {
- vm_object_pip_sleep(object, waitid);
- VM_OBJECT_WLOCK(object);
- }
+ blockcount_wait(&object->paging_in_progress, &object->lock, waitid,
+ PVM);
}
void
@@ -411,8 +409,7 @@
VM_OBJECT_ASSERT_UNLOCKED(object);
- while (REFCOUNT_COUNT(object->paging_in_progress) > 0)
- refcount_wait(&object->paging_in_progress, waitid, PVM);
+ blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);
}
/*
@@ -955,7 +952,7 @@
*/
vm_object_pip_wait(object, "objtrm");
- KASSERT(!REFCOUNT_COUNT(object->paging_in_progress),
+ KASSERT(!BLOCKCOUNT_COUNT(object->paging_in_progress),
("vm_object_terminate: pageout in progress"));
KASSERT(object->ref_count == 0,
@@ -2458,7 +2455,7 @@
VM_OBJECT_ASSERT_LOCKED(obj);
- refcount_acquire(&obj->busy);
+ blockcount_acquire(&obj->busy, 1);
/* The fence is required to order loads of page busy. */
atomic_thread_fence_acq_rel();
}
@@ -2467,8 +2464,7 @@
vm_object_unbusy(vm_object_t obj)
{
-
- refcount_release(&obj->busy);
+ blockcount_release(&obj->busy, 1);
}
void
@@ -2477,8 +2473,8 @@
VM_OBJECT_ASSERT_UNLOCKED(obj);
- if (obj->busy)
- refcount_sleep(&obj->busy, wmesg, PVM);
+ if (atomic_load_int(&obj->busy))
+ blockcount_sleep(&obj->busy, NULL, wmesg, PVM);
}
/*
Index: sys/vm/vm_swapout.c
===================================================================
--- sys/vm/vm_swapout.c
+++ sys/vm/vm_swapout.c
@@ -218,7 +218,7 @@
goto unlock_return;
VM_OBJECT_ASSERT_LOCKED(object);
if ((object->flags & OBJ_UNMANAGED) != 0 ||
- REFCOUNT_COUNT(object->paging_in_progress) > 0)
+ BLOCKCOUNT_COUNT(object->paging_in_progress) > 0)
goto unlock_return;
unmap = true;

File Metadata

Mime Type
text/plain
Expires
Fri, Jul 17, 7:21 AM (18 h, 16 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35161327
Default Alt Text
D23723.id68432.diff (15 KB)

Event Timeline