Index: sys/vm/vm_init.c =================================================================== --- sys/vm/vm_init.c +++ sys/vm/vm_init.c @@ -91,6 +91,7 @@ #include #include #include +#include #include #include #include @@ -146,6 +147,7 @@ vm_object_init(); vm_map_startup(); kmem_init(virtual_avail, virtual_end); + vm_radix_zinit(); #ifndef UMA_MD_SMALL_ALLOC /* Set up radix zone to use noobj_alloc. */ Index: sys/vm/vm_object.c =================================================================== --- sys/vm/vm_object.c +++ sys/vm/vm_object.c @@ -309,7 +309,6 @@ #endif vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); - vm_radix_zinit(); } void Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -591,9 +591,11 @@ * before going fully functional. UMA will add its zones. * * VM startup zones: vmem, vmem_btag, VM OBJECT, RADIX NODE, MAP, - * KMAP ENTRY, MAP ENTRY, VMSPACE. + * KMAP ENTRY, MAP ENTRY, VMSPACE, SMR CPU, SMR GLOBAL. + * + * Add ncpu pages for a per-cpu zone. */ - boot_pages = uma_startup_count(8); + boot_pages = uma_startup_count(10) + mp_ncpus; #ifndef UMA_MD_SMALL_ALLOC /* vmem_startup() calls uma_prealloc(). */ Index: sys/vm/vm_radix.h =================================================================== --- sys/vm/vm_radix.h +++ sys/vm/vm_radix.h @@ -43,6 +43,7 @@ vm_page_t vm_radix_lookup(struct vm_radix *rtree, vm_pindex_t index); vm_page_t vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index); vm_page_t vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index); +vm_page_t vm_radix_lookup_unlocked(struct vm_radix *rtree, vm_pindex_t index); void vm_radix_reclaim_allnodes(struct vm_radix *rtree); vm_page_t vm_radix_remove(struct vm_radix *rtree, vm_pindex_t index); vm_page_t vm_radix_replace(struct vm_radix *rtree, vm_page_t newpage); Index: sys/vm/vm_radix.c =================================================================== --- sys/vm/vm_radix.c +++ sys/vm/vm_radix.c @@ -58,11 +58,14 @@ #include #include #include +#include #include #include +#include #include #include +#include #include #include @@ -103,6 +106,7 @@ }; static uma_zone_t vm_radix_node_zone; +static uma_smr_t vm_radix_smr; /* * Allocate a radix node. @@ -172,7 +176,7 @@ vm_radix_setroot(struct vm_radix *rtree, struct vm_radix_node *rnode) { - rtree->rt_root = (uintptr_t)rnode; + atomic_store_rel_ptr(&rtree->rt_root, (uintptr_t)rnode); } /* @@ -331,7 +335,9 @@ #else NULL, #endif - vm_radix_node_zone_init, NULL, VM_RADIX_PAD, UMA_ZONE_VM); + vm_radix_node_zone_init, NULL, VM_RADIX_PAD, + UMA_ZONE_VM | UMA_ZONE_SMR); + vm_radix_smr = uma_zone_get_smr(vm_radix_node_zone); } /* @@ -342,7 +348,7 @@ vm_radix_insert(struct vm_radix *rtree, vm_page_t page) { vm_pindex_t index, newind; - void **parentp; + volatile uintptr_t *parentp; struct vm_radix_node *rnode, *tmp; vm_page_t m; int slot; @@ -359,7 +365,7 @@ rtree->rt_root = (uintptr_t)page | VM_RADIX_ISLEAF; return (0); } - parentp = (void **)&rtree->rt_root; + parentp = (uintptr_t *)&rtree->rt_root; for (;;) { if (vm_radix_isleaf(rnode)) { m = vm_radix_topage(rnode); @@ -371,9 +377,10 @@ clev + 1), 2, clev); if (tmp == NULL) return (ENOMEM); - *parentp = tmp; vm_radix_addpage(tmp, index, clev, page); vm_radix_addpage(tmp, m->pindex, clev, m); + /* synchronize to keep leaf visible. */ + atomic_store_rel_ptr(parentp, (uintptr_t)tmp); return (0); } else if (vm_radix_keybarr(rnode, index)) break; @@ -383,7 +390,7 @@ vm_radix_addpage(rnode, index, rnode->rn_clev, page); return (0); } - parentp = &rnode->rn_child[slot]; + parentp = (uintptr_t *)&rnode->rn_child[slot]; rnode = rnode->rn_child[slot]; } @@ -397,10 +404,12 @@ tmp = vm_radix_node_get(vm_radix_trimkey(index, clev + 1), 2, clev); if (tmp == NULL) return (ENOMEM); - *parentp = tmp; vm_radix_addpage(tmp, index, clev, page); slot = vm_radix_slot(newind, clev); tmp->rn_child[slot] = rnode; + /* Synchronize with lookup to keep the original entry visible. */ + atomic_store_rel_ptr(parentp, (uintptr_t)tmp); + return (0); } @@ -419,6 +428,50 @@ return (vm_radix_isleaf(rnode)); } +/* + * Returns the value stored at the index without requiring an external lock. + * + * If the index is not present, NULL is returned. + */ +vm_page_t +vm_radix_lookup_unlocked(struct vm_radix *rtree, vm_pindex_t index) +{ + struct vm_radix_node *rnode; + struct vm_object *obj; + vm_page_t m; + int slot; + + uma_smr_enter(vm_radix_smr); +retry: + m = NULL; + /* Synchronize with tree updates. */ + rnode = (void *)atomic_load_acq_ptr(&rtree->rt_root); + while (rnode != NULL) { + if (vm_radix_isleaf(rnode)) { + m = vm_radix_topage(rnode); + obj = m->object; + if (obj == NULL || &obj->rtree != rtree) + goto retry; + if (m->pindex != index) + m = NULL; + break; + } + + if (vm_radix_keybarr(rnode, index)) + break; + slot = vm_radix_slot(index, rnode->rn_clev); + /* Synchronize with tree updates. */ + rnode = (void *)atomic_load_acq_ptr( + (uintptr_t *)&rnode->rn_child[slot]); + } + uma_smr_exit(vm_radix_smr); + + return (m); +} + +/* + * Look up the nearest entry at a position bigger than or equal to index. + */ /* * Returns the value stored at the index. If the index is not present, * NULL is returned. @@ -438,7 +491,8 @@ return (m); else break; - } else if (vm_radix_keybarr(rnode, index)) + } + if (vm_radix_keybarr(rnode, index)) break; slot = vm_radix_slot(index, rnode->rn_clev); rnode = rnode->rn_child[slot]; @@ -713,7 +767,9 @@ slot = vm_radix_slot(index, parent->rn_clev); KASSERT(parent->rn_child[slot] == rnode, ("%s: invalid child value", __func__)); - parent->rn_child[slot] = rnode->rn_child[i]; + atomic_store_rel_ptr( + (uintptr_t *)&parent->rn_child[slot], + (uintptr_t)rnode->rn_child[i]); } rnode->rn_count--; rnode->rn_child[i] = NULL;