diff --git a/sys/kern/subr_pctrie.c b/sys/kern/subr_pctrie.c --- a/sys/kern/subr_pctrie.c +++ b/sys/kern/subr_pctrie.c @@ -92,13 +92,29 @@ static __inline void pctrie_node_store(smr_pctnode_t *p, void *val, enum pctrie_access access); +/* + * Return the position in the array for a given level. + */ +static __inline int +pctrie_slot(uint64_t index, uint16_t level) +{ + return ((index >> (level * PCTRIE_WIDTH)) & PCTRIE_MASK); +} + +/* Computes the key (index) with the low-order 'level' radix-digits zeroed. */ +static __inline uint64_t +pctrie_trimkey(uint64_t index, uint16_t level) +{ + return (index & -PCTRIE_UNITLEVEL(level)); +} + /* * Allocate a node. Pre-allocation should ensure that the request * will always be satisfied. */ static struct pctrie_node * -pctrie_node_get(struct pctrie *ptree, pctrie_alloc_t allocfn, uint64_t owner, - uint16_t count, uint16_t clevel) +pctrie_node_get(struct pctrie *ptree, pctrie_alloc_t allocfn, uint64_t index, + uint16_t clevel) { struct pctrie_node *node; @@ -116,8 +132,8 @@ PCTRIE_UNSERIALIZED); node->pn_last = 0; } - node->pn_owner = owner; - node->pn_count = count; + node->pn_owner = pctrie_trimkey(index, clevel + 1); + node->pn_count = 2; node->pn_clev = clevel; return (node); } @@ -146,23 +162,6 @@ freefn(ptree, node); } -/* - * Return the position in the array for a given level. - */ -static __inline int -pctrie_slot(uint64_t index, uint16_t level) -{ - - return ((index >> (level * PCTRIE_WIDTH)) & PCTRIE_MASK); -} - -/* Computes the key (index) with the low-order 'level' radix-digits zeroed. */ -static __inline uint64_t -pctrie_trimkey(uint64_t index, uint16_t level) -{ - return (index & -PCTRIE_UNITLEVEL(level)); -} - /* * Fetch a node pointer from a slot. */ @@ -376,8 +375,7 @@ panic("%s: key %jx is already present", __func__, (uintmax_t)index); clev = pctrie_keydiff(*m, index); - tmp = pctrie_node_get(ptree, allocfn, - pctrie_trimkey(index, clev + 1), 2, clev); + tmp = pctrie_node_get(ptree, allocfn, index, clev); if (tmp == NULL) return (ENOMEM); /* These writes are not yet visible due to ordering. */ @@ -408,8 +406,7 @@ */ newind = node->pn_owner; clev = pctrie_keydiff(newind, index); - tmp = pctrie_node_get(ptree, allocfn, - pctrie_trimkey(index, clev + 1), 2, clev); + tmp = pctrie_node_get(ptree, allocfn, index, clev); if (tmp == NULL) return (ENOMEM); slot = pctrie_slot(newind, clev); diff --git a/sys/vm/vm_radix.c b/sys/vm/vm_radix.c --- a/sys/vm/vm_radix.c +++ b/sys/vm/vm_radix.c @@ -119,11 +119,27 @@ static void vm_radix_node_store(smrnode_t *p, struct vm_radix_node *v, enum vm_radix_access access); +/* + * Return the position in the array for a given level. + */ +static __inline int +vm_radix_slot(vm_pindex_t index, uint16_t level) +{ + return ((index >> (level * VM_RADIX_WIDTH)) & VM_RADIX_MASK); +} + +/* Computes the key (index) with the low-order 'level' radix-digits zeroed. */ +static __inline vm_pindex_t +vm_radix_trimkey(vm_pindex_t index, uint16_t level) +{ + return (index & -VM_RADIX_UNITLEVEL(level)); +} + /* * Allocate a radix node. */ static struct vm_radix_node * -vm_radix_node_get(vm_pindex_t owner, uint16_t count, uint16_t clevel) +vm_radix_node_get(vm_pindex_t index, uint16_t clevel) { struct vm_radix_node *rnode; @@ -141,8 +157,8 @@ NULL, UNSERIALIZED); rnode->rn_last = 0; } - rnode->rn_owner = owner; - rnode->rn_count = count; + rnode->rn_owner = vm_radix_trimkey(index, clevel + 1); + rnode->rn_count = 2; rnode->rn_clev = clevel; return (rnode); } @@ -171,23 +187,6 @@ uma_zfree_smr(vm_radix_node_zone, rnode); } -/* - * Return the position in the array for a given level. - */ -static __inline int -vm_radix_slot(vm_pindex_t index, uint16_t level) -{ - - return ((index >> (level * VM_RADIX_WIDTH)) & VM_RADIX_MASK); -} - -/* Computes the key (index) with the low-order 'level' radix-digits zeroed. */ -static __inline vm_pindex_t -vm_radix_trimkey(vm_pindex_t index, uint16_t level) -{ - return (index & -VM_RADIX_UNITLEVEL(level)); -} - /* * Fetch a node pointer from a slot in another node. */ @@ -416,8 +415,7 @@ panic("%s: key %jx is already present", __func__, (uintmax_t)index); clev = vm_radix_keydiff(m->pindex, index); - tmp = vm_radix_node_get(vm_radix_trimkey(index, - clev + 1), 2, clev); + tmp = vm_radix_node_get(index, clev); if (tmp == NULL) return (ENOMEM); /* These writes are not yet visible due to ordering. */ @@ -447,7 +445,7 @@ */ newind = rnode->rn_owner; clev = vm_radix_keydiff(newind, index); - tmp = vm_radix_node_get(vm_radix_trimkey(index, clev + 1), 2, clev); + tmp = vm_radix_node_get(index, clev); if (tmp == NULL) return (ENOMEM); slot = vm_radix_slot(newind, clev);