Index: sys/kern/subr_pctrie.c =================================================================== --- sys/kern/subr_pctrie.c +++ sys/kern/subr_pctrie.c @@ -797,9 +797,9 @@ * non-leaf child into 'node'. Repeat until a node has been stripped of all * children, and mark it for freeing, returning its parent. */ -static struct pctrie_node * -pctrie_reclaim_prune(struct pctrie_node **pnode, - struct pctrie_node *parent) +static __always_inline struct pctrie_node * +pctrie_reclaim_prune(struct pctrie_node **pnode, struct pctrie_node *parent, + pctrie_xfcb_t xform_cb, pctrie_cb_t callback, void *arg) { struct pctrie_node *child, *node; int slot; @@ -812,8 +812,11 @@ PCTRIE_UNSERIALIZED); pctrie_node_store(&node->pn_child[slot], PCTRIE_NULL, PCTRIE_UNSERIALIZED); - if (pctrie_isleaf(child)) + if (pctrie_isleaf(child)) { + if (callback != NULL) + xform_cb(callback, pctrie_toval(child), arg); continue; + } /* Climb one level down the trie. */ pctrie_node_store(&node->pn_child[0], parent, PCTRIE_UNSERIALIZED); @@ -827,8 +830,9 @@ /* * Recover the node parent from its first child and continue pruning. */ -struct pctrie_node * -pctrie_reclaim_resume(struct pctrie_node **pnode) +static __always_inline struct pctrie_node * +pctrie_reclaim_resume_compound(struct pctrie_node **pnode, + pctrie_xfcb_t xform_cb, pctrie_cb_t callback, void *arg) { struct pctrie_node *parent, *node; @@ -839,24 +843,58 @@ parent = pctrie_node_load(&node->pn_child[0], NULL, PCTRIE_UNSERIALIZED); pctrie_node_store(&node->pn_child[0], PCTRIE_NULL, PCTRIE_UNSERIALIZED); - return (pctrie_reclaim_prune(pnode, parent)); + return (pctrie_reclaim_prune(pnode, parent, xform_cb, callback, arg)); +} + +struct pctrie_node * +pctrie_reclaim_resume(struct pctrie_node **pnode) +{ + return (pctrie_reclaim_resume_compound(pnode, NULL, NULL, NULL)); +} + +struct pctrie_node * +pctrie_reclaim_resume_cb(struct pctrie_node **pnode, + pctrie_xfcb_t xform_cb, pctrie_cb_t callback, void *arg) +{ + return (pctrie_reclaim_resume_compound(pnode, xform_cb, callback, arg)); } /* * Find the trie root, and start pruning with a NULL parent. */ -struct pctrie_node * -pctrie_reclaim_begin(struct pctrie_node **pnode, - struct pctrie *ptree) +static __always_inline struct pctrie_node * +pctrie_reclaim_begin_compound(struct pctrie_node **pnode, + struct pctrie *ptree, + pctrie_xfcb_t xform_cb, pctrie_cb_t callback, void *arg) { struct pctrie_node *node; node = pctrie_root_load(ptree, NULL, PCTRIE_UNSERIALIZED); pctrie_root_store(ptree, PCTRIE_NULL, PCTRIE_UNSERIALIZED); - if (pctrie_isleaf(node)) + if (pctrie_isleaf(node)) { + if (node != PCTRIE_NULL && callback != NULL) + xform_cb(callback, pctrie_toval(node), arg); return (NULL); + } *pnode = node; - return (pctrie_reclaim_prune(pnode, NULL)); + return (pctrie_reclaim_prune(pnode, NULL, xform_cb, callback, arg)); +} + +/* + * Find the trie root, and start pruning with a NULL parent. + */ +struct pctrie_node * +pctrie_reclaim_begin(struct pctrie_node **pnode, struct pctrie *ptree) +{ + return (pctrie_reclaim_begin_compound(pnode, ptree, NULL, NULL, NULL)); +} + +struct pctrie_node * +pctrie_reclaim_begin_cb(struct pctrie_node **pnode, struct pctrie *ptree, + pctrie_xfcb_t xform_cb, pctrie_cb_t callback, void *arg) +{ + return (pctrie_reclaim_begin_compound(pnode, ptree, + xform_cb, callback, arg)); } /* Index: sys/sys/pctrie.h =================================================================== --- sys/sys/pctrie.h +++ sys/sys/pctrie.h @@ -34,6 +34,9 @@ #include #include +typedef void (*pctrie_cb_t)(void *ptr, void *arg); +typedef void (*pctrie_xfcb_t)(pctrie_cb_t callback, uint64_t *val, void *arg); + #ifdef _KERNEL #define PCTRIE_DEFINE_SMR(name, type, field, allocfn, freefn, smr) \ @@ -218,6 +221,26 @@ freefn(ptree, freenode); \ } \ \ +static __unused void \ +name##_PCTRIE_VAL2PTR_CB(pctrie_cb_t callback, uint64_t *val, void *arg)\ +{ \ + callback(name##_PCTRIE_VAL2PTR(val), arg); \ +} \ + \ +static __inline __unused void \ +name##_PCTRIE_RECLAIM_CALLBACK(struct pctrie *ptree, \ + pctrie_cb_t callback, void *arg) \ +{ \ + struct pctrie_node *freenode, *node; \ + \ + for (freenode = pctrie_reclaim_begin_cb(&node, ptree, \ + name##_PCTRIE_VAL2PTR_CB, callback, arg); \ + freenode != NULL; \ + freenode = pctrie_reclaim_resume_cb(&node, \ + name##_PCTRIE_VAL2PTR_CB, callback, arg)) \ + freefn(ptree, freenode); \ +} \ + \ static __inline __unused struct type * \ name##_PCTRIE_REPLACE(struct pctrie *ptree, struct type *ptr) \ { \ @@ -269,6 +292,11 @@ struct pctrie_node *pctrie_reclaim_begin(struct pctrie_node **pnode, struct pctrie *ptree); struct pctrie_node *pctrie_reclaim_resume(struct pctrie_node **pnode); +struct pctrie_node *pctrie_reclaim_begin_cb(struct pctrie_node **pnode, + struct pctrie *ptree, + pctrie_xfcb_t xform_cb, pctrie_cb_t callback, void *arg); +struct pctrie_node *pctrie_reclaim_resume_cb(struct pctrie_node **pnode, + pctrie_xfcb_t xform_cb, pctrie_cb_t callback, void *arg); uint64_t *pctrie_remove_lookup(struct pctrie *ptree, uint64_t index, struct pctrie_node **killnode); uint64_t *pctrie_replace(struct pctrie *ptree, uint64_t *newval);