Page MenuHomeFreeBSD
Files F145480406

D23446.id67592.diff
No OneTemporary
Actions

D23446.id67592.diff
View Options

Index: sys/vm/uma_core.c
===================================================================
--- sys/vm/uma_core.c
+++ sys/vm/uma_core.c
@@ -89,6 +89,7 @@
#include <vm/vm_param.h>
#include <vm/vm_phys.h>
#include <vm/vm_pagequeue.h>
+#include <vm/vm_radix.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
@@ -2655,6 +2656,8 @@
bucket_init();
smr_init();
+ uma_prealloc(zones, 10);
+ uma_prealloc(kegs, 10);
}
#ifndef UMA_MD_SMALL_ALLOC
@@ -2677,6 +2680,7 @@
vm_map_unlock(kernel_map);
}
+ vm_radix_zinit();
#ifndef UMA_MD_SMALL_ALLOC
/* Set up radix zone to use noobj_alloc. */
vm_radix_reserve_kva();
Index: sys/vm/vm_object.c
===================================================================
--- sys/vm/vm_object.c
+++ sys/vm/vm_object.c
@@ -313,7 +313,6 @@
#endif
vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
- vm_radix_zinit();
}
void
Index: sys/vm/vm_radix.h
===================================================================
--- sys/vm/vm_radix.h
+++ sys/vm/vm_radix.h
@@ -43,6 +43,7 @@
vm_page_t vm_radix_lookup(struct vm_radix *rtree, vm_pindex_t index);
vm_page_t vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index);
vm_page_t vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index);
+vm_page_t vm_radix_lookup_unlocked(struct vm_radix *rtree, vm_pindex_t index);
void vm_radix_reclaim_allnodes(struct vm_radix *rtree);
vm_page_t vm_radix_remove(struct vm_radix *rtree, vm_pindex_t index);
vm_page_t vm_radix_replace(struct vm_radix *rtree, vm_page_t newpage);
Index: sys/vm/vm_radix.c
===================================================================
--- sys/vm/vm_radix.c
+++ sys/vm/vm_radix.c
@@ -58,11 +58,14 @@
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
+#include <sys/proc.h>
#include <sys/vmmeter.h>
+#include <sys/smr.h>
#include <vm/uma.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
+#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_radix.h>
@@ -103,6 +106,7 @@
};
static uma_zone_t vm_radix_node_zone;
+static smr_t vm_radix_smr;
/*
* Allocate a radix node.
@@ -112,7 +116,7 @@
{
struct vm_radix_node *rnode;
- rnode = uma_zalloc(vm_radix_node_zone, M_NOWAIT);
+ rnode = uma_zalloc_smr(vm_radix_node_zone, M_NOWAIT);
if (rnode == NULL)
return (NULL);
rnode->rn_owner = owner;
@@ -128,7 +132,7 @@
vm_radix_node_put(struct vm_radix_node *rnode)
{
- uma_zfree(vm_radix_node_zone, rnode);
+ uma_zfree_smr(vm_radix_node_zone, rnode);
}
/*
@@ -172,7 +176,7 @@
vm_radix_setroot(struct vm_radix *rtree, struct vm_radix_node *rnode)
{
- rtree->rt_root = (uintptr_t)rnode;
+ atomic_store_rel_ptr(&rtree->rt_root, (uintptr_t)rnode);
}
/*
@@ -283,16 +287,6 @@
}
#endif
-static int
-vm_radix_node_zone_init(void *mem, int size __unused, int flags __unused)
-{
- struct vm_radix_node *rnode;
-
- rnode = mem;
- bzero(rnode, sizeof(*rnode));
- return (0);
-}
-
#ifndef UMA_MD_SMALL_ALLOC
void vm_radix_reserve_kva(void);
/*
@@ -331,7 +325,9 @@
#else
NULL,
#endif
- vm_radix_node_zone_init, NULL, VM_RADIX_PAD, UMA_ZONE_VM);
+ NULL, NULL, VM_RADIX_PAD,
+ UMA_ZONE_VM | UMA_ZONE_SMR | UMA_ZONE_ZINIT);
+ vm_radix_smr = uma_zone_get_smr(vm_radix_node_zone);
}
/*
@@ -342,7 +338,7 @@
vm_radix_insert(struct vm_radix *rtree, vm_page_t page)
{
vm_pindex_t index, newind;
- void **parentp;
+ volatile uintptr_t *parentp;
struct vm_radix_node *rnode, *tmp;
vm_page_t m;
int slot;
@@ -359,7 +355,7 @@
rtree->rt_root = (uintptr_t)page | VM_RADIX_ISLEAF;
return (0);
}
- parentp = (void **)&rtree->rt_root;
+ parentp = (uintptr_t *)&rtree->rt_root;
for (;;) {
if (vm_radix_isleaf(rnode)) {
m = vm_radix_topage(rnode);
@@ -371,9 +367,10 @@
clev + 1), 2, clev);
if (tmp == NULL)
return (ENOMEM);
- *parentp = tmp;
vm_radix_addpage(tmp, index, clev, page);
vm_radix_addpage(tmp, m->pindex, clev, m);
+ /* synchronize to keep leaf visible. */
+ atomic_store_rel_ptr(parentp, (uintptr_t)tmp);
return (0);
} else if (vm_radix_keybarr(rnode, index))
break;
@@ -383,7 +380,7 @@
vm_radix_addpage(rnode, index, rnode->rn_clev, page);
return (0);
}
- parentp = &rnode->rn_child[slot];
+ parentp = (uintptr_t *)&rnode->rn_child[slot];
rnode = rnode->rn_child[slot];
}
@@ -397,10 +394,12 @@
tmp = vm_radix_node_get(vm_radix_trimkey(index, clev + 1), 2, clev);
if (tmp == NULL)
return (ENOMEM);
- *parentp = tmp;
vm_radix_addpage(tmp, index, clev, page);
slot = vm_radix_slot(newind, clev);
tmp->rn_child[slot] = rnode;
+ /* Synchronize with lookup to keep the original entry visible. */
+ atomic_store_rel_ptr(parentp, (uintptr_t)tmp);
+
return (0);
}
@@ -419,6 +418,50 @@
return (vm_radix_isleaf(rnode));
}
+/*
+ * Returns the value stored at the index without requiring an external lock.
+ *
+ * If the index is not present, NULL is returned.
+ */
+vm_page_t
+vm_radix_lookup_unlocked(struct vm_radix *rtree, vm_pindex_t index)
+{
+ struct vm_radix_node *rnode;
+ struct vm_object *obj;
+ vm_page_t m;
+ int slot;
+
+ smr_enter(vm_radix_smr);
+retry:
+ m = NULL;
+ /* Synchronize with tree updates. */
+ rnode = (void *)atomic_load_acq_ptr(&rtree->rt_root);
+ while (rnode != NULL) {
+ if (vm_radix_isleaf(rnode)) {
+ m = vm_radix_topage(rnode);
+ obj = m->object;
+ if (obj == NULL || &obj->rtree != rtree)
+ goto retry;
+ if (m->pindex != index)
+ m = NULL;
+ break;
+ }
+
+ if (vm_radix_keybarr(rnode, index))
+ break;
+ slot = vm_radix_slot(index, rnode->rn_clev);
+ /* Synchronize with tree updates. */
+ rnode = (void *)atomic_load_acq_ptr(
+ (uintptr_t *)&rnode->rn_child[slot]);
+ }
+ smr_exit(vm_radix_smr);
+
+ return (m);
+}
+
+/*
+ * Look up the nearest entry at a position bigger than or equal to index.
+ */
/*
* Returns the value stored at the index. If the index is not present,
* NULL is returned.
@@ -438,7 +481,8 @@
return (m);
else
break;
- } else if (vm_radix_keybarr(rnode, index))
+ }
+ if (vm_radix_keybarr(rnode, index))
break;
slot = vm_radix_slot(index, rnode->rn_clev);
rnode = rnode->rn_child[slot];
@@ -713,7 +757,9 @@
slot = vm_radix_slot(index, parent->rn_clev);
KASSERT(parent->rn_child[slot] == rnode,
("%s: invalid child value", __func__));
- parent->rn_child[slot] = rnode->rn_child[i];
+ atomic_store_rel_ptr(
+ (uintptr_t *)&parent->rn_child[slot],
+ (uintptr_t)rnode->rn_child[i]);
}
rnode->rn_count--;
rnode->rn_child[i] = NULL;

File Metadata

Mime Type
text/plain
Expires
Sat, Feb 21, 9:30 AM (9 h, 45 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
28918147
Default Alt Text
D23446.id67592.diff (6 KB)

Event Timeline