Index: sys/vm/vm_radix.c
===================================================================
--- sys/vm/vm_radix.c
+++ sys/vm/vm_radix.c
@@ -527,205 +527,141 @@
 }
 
 /*
- * Look up the nearest entry at a position greater than or equal to index.
+ * Returns the least value stored at a position greater than or equal to index.
+ * Returns NULL if the all positions are less than index.
  */
-vm_page_t
-vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index)
+static __always_inline vm_page_t
+_vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index,
+    enum vm_radix_access access)
 {
-	struct vm_radix_node *stack[VM_RADIX_LIMIT];
+	struct vm_radix_node *nbr, *new_nbr, *rnode;
+	smrnode_t *p_nbr;
 	vm_page_t m;
-	struct vm_radix_node *child, *rnode;
-#ifdef INVARIANTS
-	int loops = 0;
-#endif
-	int slot, tos;
+	int next, slot;
 
-	rnode = vm_radix_root_load(rtree, LOCKED);
-	if (rnode == NULL)
-		return (NULL);
-	else if (vm_radix_isleaf(rnode)) {
-		m = vm_radix_topage(rnode);
-		if (m->pindex >= index)
-			return (m);
-		else
-			return (NULL);
-	}
-	tos = 0;
-	for (;;) {
-		/*
-		 * If the keys differ before the current bisection node,
-		 * then the search key might rollback to the earliest
-		 * available bisection node or to the smallest key
-		 * in the current node (if the owner is greater than the
-		 * search key).
-		 */
+	rnode = vm_radix_root_load(rtree, access);
+	p_nbr = NULL;
+	nbr = NULL;
+	while (rnode != NULL) {
+		if (vm_radix_isleaf(rnode)) {
+			m = vm_radix_topage(rnode);
+			if (m->pindex >= index)
+				return (m);
+			break;
+		}
 		if (vm_radix_keybarr(rnode, index)) {
-			if (index > rnode->rn_owner) {
-ascend:
-				KASSERT(++loops < 1000,
-				    ("vm_radix_lookup_ge: too many loops"));
-
-				/*
-				 * Pop nodes from the stack until either the
-				 * stack is empty or a node that could have a
-				 * matching descendant is found.
-				 */
-				do {
-					if (tos == 0)
-						return (NULL);
-					rnode = stack[--tos];
-				} while (vm_radix_slot(index,
-				    rnode->rn_clev) == (VM_RADIX_COUNT - 1));
-
-				/*
-				 * The following computation cannot overflow
-				 * because index's slot at the current level
-				 * is less than VM_RADIX_COUNT - 1.
-				 */
-				index = vm_radix_trimkey(index,
-				    rnode->rn_clev);
-				index += VM_RADIX_UNITLEVEL(rnode->rn_clev);
-			} else
-				index = rnode->rn_owner;
-			KASSERT(!vm_radix_keybarr(rnode, index),
-			    ("vm_radix_lookup_ge: keybarr failed"));
+			if (rnode->rn_owner >= index)
+				nbr = rnode;
+			break;
 		}
 		slot = vm_radix_slot(index, rnode->rn_clev);
-		child = vm_radix_node_load(&rnode->rn_child[slot], LOCKED);
-		if (vm_radix_isleaf(child)) {
-			m = vm_radix_topage(child);
-			if (m->pindex >= index)
-				return (m);
-		} else if (child != NULL)
-			goto descend;
-
 		/* Find the first set bit beyond the first slot+1 bits. */
-		slot = ffs(rnode->rn_popmap & (-2 << slot)) - 1;
-		if (slot < 0) {
-			/*
-			 * A page or edge greater than the search slot is not
-			 * found in the current node; ascend to the next
-			 * higher-level node.
-			 */
-			goto ascend;
+		while ((next = ffs(rnode->rn_popmap & (-2 << slot)) - 1) >= 0) {
+			p_nbr = &rnode->rn_child[next];
+			if (access == LOCKED)
+				break;
+			new_nbr = vm_radix_node_load(p_nbr, access);
+			if (new_nbr != NULL) {
+				nbr = new_nbr;
+				p_nbr = NULL;
+				break;
+			}
 		}
-		child = vm_radix_node_load(&rnode->rn_child[slot], LOCKED);
-		KASSERT(child != NULL, ("%s: bad popmap slot %d in rnode %p",
-		    __func__, slot, rnode));
-		if (vm_radix_isleaf(child))
-			return (vm_radix_topage(child));
-		index = vm_radix_trimkey(index, rnode->rn_clev + 1) +
-		    slot * VM_RADIX_UNITLEVEL(rnode->rn_clev);
-descend:
-		KASSERT(rnode->rn_clev > 0,
-		    ("vm_radix_lookup_ge: pushing leaf's parent"));
-		KASSERT(tos < VM_RADIX_LIMIT,
-		    ("vm_radix_lookup_ge: stack overflow"));
-		stack[tos++] = rnode;
-		rnode = child;
+		rnode = vm_radix_node_load(&rnode->rn_child[slot], access);
 	}
+	if (nbr == NULL) {
+		if (p_nbr == NULL)
+			return (NULL);
+		nbr = vm_radix_node_load(p_nbr, access);
+	}
+	while (!vm_radix_isleaf(nbr)) {
+		slot = ffs(nbr->rn_popmap) - 1;
+		new_nbr = vm_radix_node_load(&nbr->rn_child[slot], access);
+		if (access == LOCKED || new_nbr != NULL)
+			nbr = new_nbr;
+	}
+	return (vm_radix_topage(nbr));
 }
 
 /*
- * Look up the nearest entry at a position less than or equal to index.
+ * Returns the least value stored at a position greater than or equal to index
+ * assuming there is an external lock.
+ *
+ * Returns NULL if the all positions are less than index.
  */
 vm_page_t
-vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index)
+vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index)
 {
-	struct vm_radix_node *stack[VM_RADIX_LIMIT];
+	return (_vm_radix_lookup_ge(rtree, index, LOCKED));
+}
+
+/*
+ * Returns the greatest value stored at a position less than or equal to index.
+ * Returns NULL if the all positions are more than index.
+ */
+static __always_inline vm_page_t
+_vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index,
+    enum vm_radix_access access)
+{
+	struct vm_radix_node *nbr, *new_nbr, *rnode;
+	smrnode_t *p_nbr;
 	vm_page_t m;
-	struct vm_radix_node *child, *rnode;
-#ifdef INVARIANTS
-	int loops = 0;
-#endif
-	int slot, tos;
+	int next, slot;
 
-	rnode = vm_radix_root_load(rtree, LOCKED);
-	if (rnode == NULL)
-		return (NULL);
-	else if (vm_radix_isleaf(rnode)) {
-		m = vm_radix_topage(rnode);
-		if (m->pindex <= index)
-			return (m);
-		else
-			return (NULL);
-	}
-	tos = 0;
-	for (;;) {
-		/*
-		 * If the keys differ before the current bisection node,
-		 * then the search key might rollback to the earliest
-		 * available bisection node or to the largest key
-		 * in the current node (if the owner is smaller than the
-		 * search key).
-		 */
+	rnode = vm_radix_root_load(rtree, access);
+	p_nbr = NULL;
+	nbr = NULL;
+	while (rnode != NULL) {
+		if (vm_radix_isleaf(rnode)) {
+			m = vm_radix_topage(rnode);
+			if (m->pindex <= index)
+				return (m);
+			break;
+		}
 		if (vm_radix_keybarr(rnode, index)) {
-			if (index > rnode->rn_owner) {
-				index = rnode->rn_owner + VM_RADIX_COUNT *
-				    VM_RADIX_UNITLEVEL(rnode->rn_clev);
-			} else {
-ascend:
-				KASSERT(++loops < 1000,
-				    ("vm_radix_lookup_le: too many loops"));
-
-				/*
-				 * Pop nodes from the stack until either the
-				 * stack is empty or a node that could have a
-				 * matching descendant is found.
-				 */
-				do {
-					if (tos == 0)
-						return (NULL);
-					rnode = stack[--tos];
-				} while (vm_radix_slot(index,
-				    rnode->rn_clev) == 0);
-
-				/*
-				 * The following computation cannot overflow
-				 * because index's slot at the current level
-				 * is greater than 0.
-				 */
-				index = vm_radix_trimkey(index,
-				    rnode->rn_clev);
-			}
-			index--;
-			KASSERT(!vm_radix_keybarr(rnode, index),
-			    ("vm_radix_lookup_le: keybarr failed"));
+			if (rnode->rn_owner <= index)
+				nbr = rnode;
+			break;
 		}
 		slot = vm_radix_slot(index, rnode->rn_clev);
-		child = vm_radix_node_load(&rnode->rn_child[slot], LOCKED);
-		if (vm_radix_isleaf(child)) {
-			m = vm_radix_topage(child);
-			if (m->pindex <= index)
-				return (m);
-		} else if (child != NULL)
-			goto descend;
-
 		/* Find the last set bit among the first slot bits. */
-		slot = fls(rnode->rn_popmap & ((1 << slot) - 1)) - 1;
-		if (slot < 0) {
-			/*
-			 * A page or edge smaller than the search slot is not
-			 * found in the current node; ascend to the next
-			 * higher-level node.
-			 */
-			goto ascend;
+		while ((next = fls(rnode->rn_popmap & ~(-1 << slot))-1) >= 0) {
+			p_nbr = &rnode->rn_child[next];
+			if (access == LOCKED)
+				break;
+			new_nbr = vm_radix_node_load(p_nbr, access);
+			if (new_nbr != NULL) {
+				nbr = new_nbr;
+				p_nbr = NULL;
+				break;
+			}
 		}
-		child = vm_radix_node_load(&rnode->rn_child[slot], LOCKED);
-		KASSERT(child != NULL, ("%s: bad popmap slot %d in rnode %p",
-		    __func__, slot, rnode));
-		if (vm_radix_isleaf(child))
-			return (vm_radix_topage(child));
-		index = vm_radix_trimkey(index, rnode->rn_clev + 1) +
-		    (slot + 1) * VM_RADIX_UNITLEVEL(rnode->rn_clev) - 1;
-descend:
-		KASSERT(rnode->rn_clev > 0,
-		    ("vm_radix_lookup_le: pushing leaf's parent"));
-		KASSERT(tos < VM_RADIX_LIMIT,
-		    ("vm_radix_lookup_le: stack overflow"));
-		stack[tos++] = rnode;
-		rnode = child;
+		rnode = vm_radix_node_load(&rnode->rn_child[slot], access);
 	}
+	if (nbr == NULL) {
+		if (p_nbr == NULL)
+			return (NULL);
+		nbr = vm_radix_node_load(p_nbr, access);
+	}
+	while (!vm_radix_isleaf(nbr)) {
+		slot = fls(nbr->rn_popmap) - 1;
+		new_nbr = vm_radix_node_load(&nbr->rn_child[slot], access);
+		if (access == LOCKED || new_nbr != NULL)
+			nbr = new_nbr;
+	}
+	return (vm_radix_topage(nbr));
+}
+
+/*
+ * Returns the greatest value stored at a position less than or equal to index
+ * assuming there is an external lock.
+ *
+ * Returns NULL if the all positions are more than index.
+ */
+vm_page_t
+vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index)
+{
+	return (_vm_radix_lookup_le(rtree, index, LOCKED));
 }
 
 /*