Index: sys/kern/subr_pctrie.c =================================================================== --- sys/kern/subr_pctrie.c +++ sys/kern/subr_pctrie.c @@ -458,15 +458,15 @@ /* * Returns the value with the least index that is greater than or equal to the * specified index, or NULL if there are no such values. - * - * Requires that access be externally synchronized by a lock. */ -uint64_t * -pctrie_lookup_ge(struct pctrie *ptree, uint64_t index) +static __always_inline uint64_t * +_pctrie_lookup_ge(struct pctrie *ptree, uint64_t index, smr_t smr, + enum pctrie_access access) { - struct pctrie_node *node, *succ; + struct pctrie_node *node, *succ, *tmp; uint64_t *m; - int slot; + pn_popmap_t popmap; + int pos, slot; /* * Descend the trie as if performing an ordinary lookup for the @@ -481,7 +481,7 @@ * "succ". If "succ" is not NULL, then that lookup is guaranteed to * succeed. */ - node = pctrie_root_load(ptree, NULL, PCTRIE_LOCKED); + node = pctrie_root_load(ptree, NULL, access); succ = NULL; for (;;) { if (pctrie_isleaf(node)) { @@ -505,10 +505,22 @@ * a subtree of all pages with values > index, is available. If * so, remember to restart the search here. */ - if ((node->pn_popmap >> slot) > 1) - succ = node; + if (access == PCTRIE_LOCKED) { + if ((node->pn_popmap >> slot) > 1) + succ = node; + } + while (access != PCTRIE_LOCKED && + (popmap = node->pn_popmap >> slot) > 1) { + pos = slot + ffs(popmap >> 1); + tmp = pctrie_node_load(&node->pn_child[pos], NULL, + access); + if (tmp != PCTRIE_NULL) { + succ = tmp; + break; + } + } node = pctrie_node_load(&node->pn_child[slot], NULL, - PCTRIE_LOCKED); + access); } /* @@ -517,7 +529,7 @@ */ if (succ == NULL) return (NULL); - if (succ != node) { + if (access == PCTRIE_LOCKED && succ != node) { /* * Take a step to the next bigger sibling of the node chosen * last time. In that subtree, all values > index. @@ -528,7 +540,7 @@ __func__, slot, succ)); slot += ffs(succ->pn_popmap >> slot) - 1; succ = pctrie_node_load(&succ->pn_child[slot], NULL, - PCTRIE_LOCKED); + access); } /* @@ -538,29 +550,60 @@ KASSERT(succ->pn_popmap != 0, ("%s: no popmap children in node %p", __func__, succ)); slot = ffs(succ->pn_popmap) - 1; - succ = pctrie_node_load(&succ->pn_child[slot], NULL, - PCTRIE_LOCKED); + tmp = pctrie_node_load(&succ->pn_child[slot], NULL, + access); + if (access == PCTRIE_LOCKED || tmp != PCTRIE_NULL) + succ = tmp; } return (pctrie_toval(succ)); } /* - * Returns the value with the greatest index that is less than or equal to the + * Returns the value with the least index that is greater than or equal to the * specified index, or NULL if there are no such values. * * Requires that access be externally synchronized by a lock. */ uint64_t * -pctrie_lookup_le(struct pctrie *ptree, uint64_t index) +pctrie_lookup_ge(struct pctrie *ptree, uint64_t index) +{ + return (_pctrie_lookup_ge(ptree, index, NULL, PCTRIE_LOCKED)); +} + +/* + * Returns the value with the least index that is greater than or equal to the + * specified index, or NULL if there are no such values. + * + * Does not require that access be externally synchronized by a lock. + */ +uint64_t * +pctrie_lookup_ge_unlocked(struct pctrie *ptree, uint64_t index, smr_t smr) +{ + uint64_t *res; + + smr_enter(smr); + res = _pctrie_lookup_ge(ptree, index, smr, PCTRIE_SMR); + smr_exit(smr); + return (res); +} + +/* + * Returns the value with the greatest index that is less than or equal to the + * specified index, or NULL if there are no such values. + */ +static __always_inline uint64_t * +_pctrie_lookup_le(struct pctrie *ptree, uint64_t index, smr_t smr, + enum pctrie_access access) { - struct pctrie_node *node, *pred; + struct pctrie_node *node, *pred, *tmp; uint64_t *m; - int slot; + pn_popmap_t popmap; + int pos, slot; /* * Mirror the implementation of pctrie_lookup_ge, described above. */ - node = pctrie_root_load(ptree, NULL, PCTRIE_LOCKED); + node = pctrie_root_load(ptree, NULL, access); pred = NULL; for (;;) { if (pctrie_isleaf(node)) { @@ -573,32 +616,75 @@ pred = node; break; } - if ((node->pn_popmap & ((1 << slot) - 1)) != 0) - pred = node; + if (access == PCTRIE_LOCKED) { + if ((node->pn_popmap & ((1 << slot) - 1)) != 0) + pred = node; + } + while (access != PCTRIE_LOCKED && + (popmap = node->pn_popmap & ((1 << slot) - 1)) != 0) { + pos = fls(popmap) - 1; + tmp = pctrie_node_load(&node->pn_child[pos], NULL, + access); + if (tmp != PCTRIE_NULL) { + pred = tmp; + break; + } + } node = pctrie_node_load(&node->pn_child[slot], NULL, - PCTRIE_LOCKED); + access); } if (pred == NULL) return (NULL); - if (pred != node) { + if (access == PCTRIE_LOCKED && pred != node) { slot = pctrie_slot(pred, index); KASSERT((pred->pn_popmap & ((1 << slot) - 1)) != 0, ("%s: no popmap siblings before slot %d in node %p", __func__, slot, pred)); slot = fls(pred->pn_popmap & ((1 << slot) - 1)) - 1; pred = pctrie_node_load(&pred->pn_child[slot], NULL, - PCTRIE_LOCKED); + access); } while (!pctrie_isleaf(pred)) { KASSERT(pred->pn_popmap != 0, ("%s: no popmap children in node %p", __func__, pred)); slot = fls(pred->pn_popmap) - 1; - pred = pctrie_node_load(&pred->pn_child[slot], NULL, - PCTRIE_LOCKED); + tmp = pctrie_node_load(&pred->pn_child[slot], NULL, + access); + if (access == PCTRIE_LOCKED || tmp != PCTRIE_NULL) + pred = tmp; } return (pctrie_toval(pred)); } +/* + * Returns the value with the greatest index that is less than or equal to the + * specified index, or NULL if there are no such values. + * + * Requires that access be externally synchronized by a lock. + */ +uint64_t * +pctrie_lookup_le(struct pctrie *ptree, uint64_t index) +{ + return (_pctrie_lookup_le(ptree, index, NULL, PCTRIE_LOCKED)); +} + +/* + * Returns the value with the greatest index that is less than or equal to the + * specified index, or NULL if there are no such values. + * + * Does not require that access be externally synchronized by a lock. + */ +uint64_t * +pctrie_lookup_le_unlocked(struct pctrie *ptree, uint64_t index, smr_t smr) +{ + uint64_t *res; + + smr_enter(smr); + res = _pctrie_lookup_le(ptree, index, smr, PCTRIE_SMR); + smr_exit(smr); + return (res); +} + /* * Remove the specified index from the tree. * Panics if the key is not present. Index: sys/sys/pctrie.h =================================================================== --- sys/sys/pctrie.h +++ sys/sys/pctrie.h @@ -128,6 +128,10 @@ uint64_t *pctrie_lookup_le(struct pctrie *ptree, uint64_t key); uint64_t *pctrie_lookup_unlocked(struct pctrie *ptree, uint64_t key, smr_t smr); +uint64_t *pctrie_lookup_ge_unlocked(struct pctrie *ptree, uint64_t key, + smr_t smr); +uint64_t *pctrie_lookup_le_unlocked(struct pctrie *ptree, uint64_t key, + smr_t smr); void pctrie_reclaim_allnodes(struct pctrie *ptree, pctrie_free_t freefn); void pctrie_remove(struct pctrie *ptree, uint64_t key, Index: sys/vm/vm_radix.h =================================================================== --- sys/vm/vm_radix.h +++ sys/vm/vm_radix.h @@ -42,7 +42,12 @@ vm_page_t vm_radix_lookup(struct vm_radix *rtree, vm_pindex_t index); vm_page_t vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index); vm_page_t vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index); -vm_page_t vm_radix_lookup_unlocked(struct vm_radix *rtree, vm_pindex_t index); +vm_page_t vm_radix_lookup_unlocked(struct vm_radix *rtree, + vm_pindex_t index); +vm_page_t vm_radix_lookup_ge_unlocked(struct vm_radix *rtree, + vm_pindex_t index); +vm_page_t vm_radix_lookup_le_unlocked(struct vm_radix *rtree, + vm_pindex_t index); void vm_radix_reclaim_allnodes(struct vm_radix *rtree); vm_page_t vm_radix_remove(struct vm_radix *rtree, vm_pindex_t index); vm_page_t vm_radix_replace(struct vm_radix *rtree, vm_page_t newpage); Index: sys/vm/vm_radix.c =================================================================== --- sys/vm/vm_radix.c +++ sys/vm/vm_radix.c @@ -490,7 +490,7 @@ vm_radix_lookup(struct vm_radix *rtree, vm_pindex_t index) { - return _vm_radix_lookup(rtree, index, LOCKED); + return (_vm_radix_lookup(rtree, index, LOCKED)); } /* @@ -513,15 +513,15 @@ /* * Returns the page with the least pindex that is greater than or equal to the * specified pindex, or NULL if there are no such pages. - * - * Requires that access be externally synchronized by a lock. */ -vm_page_t -vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index) +static __always_inline vm_page_t +_vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index, + enum vm_radix_access access) { - struct vm_radix_node *rnode, *succ; + struct vm_radix_node *rnode, *succ, *tmp; vm_page_t m; - int slot; + rn_popmap_t popmap; + int pos, slot; /* * Descend the trie as if performing an ordinary lookup for the page @@ -536,7 +536,7 @@ * lookup starting from "succ". If "succ" is not NULL, then that lookup * is guaranteed to succeed. */ - rnode = vm_radix_root_load(rtree, LOCKED); + rnode = vm_radix_root_load(rtree, access); succ = NULL; for (;;) { if (vm_radix_isleaf(rnode)) { @@ -562,9 +562,20 @@ * bigger step, to a subtree of all pages with pindex > index, * is available. If so, remember to restart the search here. */ - if ((rnode->rn_popmap >> slot) > 1) - succ = rnode; - rnode = vm_radix_node_load(&rnode->rn_child[slot], LOCKED); + if (access == LOCKED) { + if ((rnode->rn_popmap >> slot) > 1) + succ = rnode; + } + while (access != LOCKED && + (popmap = rnode->rn_popmap >> slot) > 1) { + pos = slot + ffs(popmap >> 1); + tmp = vm_radix_node_load(&rnode->rn_child[pos], access); + if (tmp != VM_RADIX_NULL) { + succ = tmp; + break; + } + } + rnode = vm_radix_node_load(&rnode->rn_child[slot], access); } /* @@ -573,7 +584,7 @@ */ if (succ == NULL) return (NULL); - if (succ != rnode) { + if (access == LOCKED && succ != rnode) { /* * Take a step to the next bigger sibling of the node chosen * last time. In that subtree, all pages have pindex > index. @@ -583,7 +594,7 @@ ("%s: no popmap siblings past slot %d in node %p", __func__, slot, succ)); slot += ffs(succ->rn_popmap >> slot) - 1; - succ = vm_radix_node_load(&succ->rn_child[slot], LOCKED); + succ = vm_radix_node_load(&succ->rn_child[slot], access); } /* @@ -593,28 +604,60 @@ KASSERT(succ->rn_popmap != 0, ("%s: no popmap children in node %p", __func__, succ)); slot = ffs(succ->rn_popmap) - 1; - succ = vm_radix_node_load(&succ->rn_child[slot], LOCKED); + tmp = vm_radix_node_load(&succ->rn_child[slot], access); + if (access == LOCKED || tmp != VM_RADIX_NULL) + succ = tmp; } return (vm_radix_topage(succ)); } /* - * Returns the page with the greatest pindex that is less than or equal to the + * Returns the page with the least pindex that is greater than or equal to the * specified pindex, or NULL if there are no such pages. * * Requires that access be externally synchronized by a lock. */ vm_page_t -vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index) +vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index) +{ + return (_vm_radix_lookup_ge(rtree, index, LOCKED)); +} + +/* + * Returns the page with the least pindex that is greater than or equal to the + * specified pindex, or NULL if there are no such pages. + * + * Does not require an external lock. + */ +vm_page_t +vm_radix_lookup_ge_unlocked(struct vm_radix *rtree, vm_pindex_t index) { - struct vm_radix_node *pred, *rnode; vm_page_t m; - int slot; + + smr_enter(vm_radix_smr); + m = _vm_radix_lookup_ge(rtree, index, SMR); + smr_exit(vm_radix_smr); + + return (m); +} + +/* + * Returns the page with the greatest pindex that is less than or equal to the + * specified pindex, or NULL if there are no such pages. + */ +static __always_inline vm_page_t +_vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index, + enum vm_radix_access access) +{ + struct vm_radix_node *pred, *rnode, *tmp; + vm_page_t m; + rn_popmap_t popmap; + int pos, slot; /* * Mirror the implementation of vm_radix_lookup_ge, described above. */ - rnode = vm_radix_root_load(rtree, LOCKED); + rnode = vm_radix_root_load(rtree, access); pred = NULL; for (;;) { if (vm_radix_isleaf(rnode)) { @@ -628,29 +671,72 @@ pred = rnode; break; } - if ((rnode->rn_popmap & ((1 << slot) - 1)) != 0) - pred = rnode; - rnode = vm_radix_node_load(&rnode->rn_child[slot], LOCKED); + if (access == LOCKED) { + if ((rnode->rn_popmap & ((1 << slot) - 1)) != 0) + pred = rnode; + } + while (access != LOCKED && + (popmap = rnode->rn_popmap & ((1 << slot) - 1)) != 0) { + pos = fls(popmap) - 1; + tmp = vm_radix_node_load(&rnode->rn_child[pos], access); + if (tmp != VM_RADIX_NULL) { + pred = tmp; + break; + } + } + rnode = vm_radix_node_load(&rnode->rn_child[slot], access); } if (pred == NULL) return (NULL); - if (pred != rnode) { + if (access == LOCKED && pred != rnode) { slot = vm_radix_slot(pred, index); KASSERT((pred->rn_popmap & ((1 << slot) - 1)) != 0, ("%s: no popmap siblings before slot %d in node %p", __func__, slot, pred)); slot = fls(pred->rn_popmap & ((1 << slot) - 1)) - 1; - pred = vm_radix_node_load(&pred->rn_child[slot], LOCKED); + pred = vm_radix_node_load(&pred->rn_child[slot], access); } while (!vm_radix_isleaf(pred)) { KASSERT(pred->rn_popmap != 0, ("%s: no popmap children in node %p", __func__, pred)); slot = fls(pred->rn_popmap) - 1; - pred = vm_radix_node_load(&pred->rn_child[slot], LOCKED); + tmp = vm_radix_node_load(&pred->rn_child[slot], access); + if (access == LOCKED || tmp != VM_RADIX_NULL) + pred = tmp; } return (vm_radix_topage(pred)); } +/* + * Returns the page with the greatest pindex that is less than or equal to the + * specified pindex, or NULL if there are no such pages. + * + * Requires that access be externally synchronized by a lock. + */ +vm_page_t +vm_radix_lookup_le(struct vm_radix *rtree, vm_pindex_t index) +{ + return (_vm_radix_lookup_le(rtree, index, LOCKED)); +} + +/* + * Returns the page with the greatest pindex that is less than or equal to the + * specified pindex, or NULL if there are no such pages. + * + * Does not require an external lock. + */ +vm_page_t +vm_radix_lookup_le_unlocked(struct vm_radix *rtree, vm_pindex_t index) +{ + vm_page_t m; + + smr_enter(vm_radix_smr); + m = _vm_radix_lookup_le(rtree, index, SMR); + smr_exit(vm_radix_smr); + + return (m); +} + /* * Remove the specified index from the trie, and return the value stored at * that index. If the index is not present, return NULL.