Page MenuHomeFreeBSD
Files F157112050

D22298.id64826.diff
No OneTemporary
Actions

D22298.id64826.diff
View Options

Index: head/sys/kern/kern_sendfile.c
===================================================================
--- head/sys/kern/kern_sendfile.c
+++ head/sys/kern/kern_sendfile.c
@@ -261,7 +261,7 @@
for (int i = 0; i < count; i++)
if (pg[i] != bogus_page)
- vm_page_xunbusy(pg[i]);
+ vm_page_xunbusy_unchecked(pg[i]);
if (error)
sfio->error = error;
Index: head/sys/vm/vm_page.h
===================================================================
--- head/sys/vm/vm_page.h
+++ head/sys/vm/vm_page.h
@@ -307,7 +307,13 @@
#define VPB_SHARERS_WORD(x) ((x) << VPB_SHARERS_SHIFT | VPB_BIT_SHARED)
#define VPB_ONE_SHARER (1 << VPB_SHARERS_SHIFT)
-#define VPB_SINGLE_EXCLUSIVER VPB_BIT_EXCLUSIVE
+#define VPB_SINGLE_EXCLUSIVE VPB_BIT_EXCLUSIVE
+#ifdef INVARIANTS
+#define VPB_CURTHREAD_EXCLUSIVE \
+ (VPB_BIT_EXCLUSIVE | ((u_int)(uintptr_t)curthread & ~VPB_BIT_FLAGMASK))
+#else
+#define VPB_CURTHREAD_EXCLUSIVE VPB_SINGLE_EXCLUSIVE
+#endif
#define VPB_UNBUSIED VPB_SHARERS_WORD(0)
@@ -649,6 +655,7 @@
void vm_page_wire(vm_page_t);
bool vm_page_wire_mapped(vm_page_t m);
void vm_page_xunbusy_hard(vm_page_t m);
+void vm_page_xunbusy_hard_unchecked(vm_page_t m);
void vm_page_set_validclean (vm_page_t, int, int);
void vm_page_clear_dirty(vm_page_t, int, int);
void vm_page_set_invalid(vm_page_t, int, int);
@@ -684,10 +691,19 @@
("vm_page_assert_unbusied: page %p busy @ %s:%d", \
(m), __FILE__, __LINE__))
-#define vm_page_assert_xbusied(m) \
+#define vm_page_assert_xbusied_unchecked(m) do { \
KASSERT(vm_page_xbusied(m), \
("vm_page_assert_xbusied: page %p not exclusive busy @ %s:%d", \
- (m), __FILE__, __LINE__))
+ (m), __FILE__, __LINE__)); \
+} while (0)
+#define vm_page_assert_xbusied(m) do { \
+ vm_page_assert_xbusied_unchecked(m); \
+ KASSERT((m->busy_lock & ~VPB_BIT_WAITERS) == \
+ VPB_CURTHREAD_EXCLUSIVE, \
+ ("vm_page_assert_xbusied: page %p busy_lock %#x not owned" \
+ " by me @ %s:%d", \
+ (m), (m)->busy_lock, __FILE__, __LINE__)); \
+} while (0)
#define vm_page_busied(m) \
((m)->busy_lock != VPB_UNBUSIED)
@@ -699,7 +715,7 @@
} while (0)
#define vm_page_xbusied(m) \
- (((m)->busy_lock & VPB_SINGLE_EXCLUSIVER) != 0)
+ (((m)->busy_lock & VPB_SINGLE_EXCLUSIVE) != 0)
#define vm_page_xbusy(m) do { \
if (!vm_page_tryxbusy(m)) \
@@ -710,8 +726,13 @@
/* Note: page m's lock must not be owned by the caller. */
#define vm_page_xunbusy(m) do { \
if (!atomic_cmpset_rel_int(&(m)->busy_lock, \
- VPB_SINGLE_EXCLUSIVER, VPB_UNBUSIED)) \
+ VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \
vm_page_xunbusy_hard(m); \
+} while (0)
+#define vm_page_xunbusy_unchecked(m) do { \
+ if (!atomic_cmpset_rel_int(&(m)->busy_lock, \
+ VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \
+ vm_page_xunbusy_hard_unchecked(m); \
} while (0)
#ifdef INVARIANTS
Index: head/sys/vm/vm_page.c
===================================================================
--- head/sys/vm/vm_page.c
+++ head/sys/vm/vm_page.c
@@ -437,7 +437,7 @@
bzero(marker, sizeof(*marker));
marker->flags = PG_MARKER;
marker->aflags = aflags;
- marker->busy_lock = VPB_SINGLE_EXCLUSIVER;
+ marker->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
marker->queue = queue;
}
@@ -939,18 +939,19 @@
int
vm_page_busy_tryupgrade(vm_page_t m)
{
- u_int x;
+ u_int ce, x;
vm_page_assert_sbusied(m);
x = m->busy_lock;
+ ce = VPB_CURTHREAD_EXCLUSIVE;
for (;;) {
if (VPB_SHARERS(x) > 1)
return (0);
KASSERT((x & ~VPB_BIT_WAITERS) == VPB_SHARERS_WORD(1),
("vm_page_busy_tryupgrade: invalid lock state"));
if (!atomic_fcmpset_acq_int(&m->busy_lock, &x,
- VPB_SINGLE_EXCLUSIVER | (x & VPB_BIT_WAITERS)))
+ ce | (x & VPB_BIT_WAITERS)))
continue;
return (1);
}
@@ -1108,7 +1109,7 @@
vm_object_t obj;
if (atomic_cmpset_acq_int(&(m)->busy_lock, VPB_UNBUSIED,
- VPB_SINGLE_EXCLUSIVER) == 0)
+ VPB_CURTHREAD_EXCLUSIVE) == 0)
return (0);
obj = m->object;
@@ -1119,6 +1120,14 @@
return (1);
}
+static void
+vm_page_xunbusy_hard_tail(vm_page_t m)
+{
+ atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED);
+ /* Wake the waiter. */
+ wakeup(m);
+}
+
/*
* vm_page_xunbusy_hard:
*
@@ -1127,14 +1136,15 @@
void
vm_page_xunbusy_hard(vm_page_t m)
{
-
vm_page_assert_xbusied(m);
+ vm_page_xunbusy_hard_tail(m);
+}
- /*
- * Wake the waiter.
- */
- atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED);
- wakeup(m);
+void
+vm_page_xunbusy_hard_unchecked(vm_page_t m)
+{
+ vm_page_assert_xbusied_unchecked(m);
+ vm_page_xunbusy_hard_tail(m);
}
/*
@@ -1228,7 +1238,7 @@
m->flags = PG_FICTITIOUS;
/* Fictitious pages don't use "order" or "pool". */
m->oflags = VPO_UNMANAGED;
- m->busy_lock = VPB_SINGLE_EXCLUSIVER;
+ m->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
/* Fictitious pages are unevictable. */
m->ref_count = 1;
pmap_page_init(m);
@@ -1318,7 +1328,7 @@
else
vm_page_deactivate(m);
vm_page_unlock(m);
- vm_page_xunbusy(m);
+ vm_page_xunbusy_unchecked(m);
}
/*
@@ -1967,7 +1977,7 @@
VPO_UNMANAGED : 0;
m->busy_lock = VPB_UNBUSIED;
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0)
- m->busy_lock = VPB_SINGLE_EXCLUSIVER;
+ m->busy_lock = VPB_CURTHREAD_EXCLUSIVE;
if ((req & VM_ALLOC_SBUSY) != 0)
m->busy_lock = VPB_SHARERS_WORD(1);
if (req & VM_ALLOC_WIRED) {
@@ -2161,7 +2171,7 @@
VPO_UNMANAGED : 0;
busy_lock = VPB_UNBUSIED;
if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0)
- busy_lock = VPB_SINGLE_EXCLUSIVER;
+ busy_lock = VPB_CURTHREAD_EXCLUSIVE;
if ((req & VM_ALLOC_SBUSY) != 0)
busy_lock = VPB_SHARERS_WORD(1);
if ((req & VM_ALLOC_WIRED) != 0)

File Metadata

Mime Type
text/plain
Expires
Tue, May 19, 1:05 PM (13 h, 44 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
33267861
Default Alt Text
D22298.id64826.diff (5 KB)

Event Timeline