Index: sys/kern/subr_eventhandler.c =================================================================== --- sys/kern/subr_eventhandler.c +++ sys/kern/subr_eventhandler.c @@ -68,7 +68,7 @@ NULL); static struct eventhandler_list * -eventhandler_find_or_create_list(const char *name) +eventhandler_find_or_create_list(const char *name, epoch_t *pepoch) { struct eventhandler_list *list, *new_list; @@ -86,12 +86,15 @@ mtx_lock(&eventhandler_mutex); list = _eventhandler_find_list(name); if (list != NULL) { + /* Verify that both list definitions are the same */ + MPASS(list->el_pepoch == pepoch); free(new_list, M_EVENTHANDLER); } else { CTR2(KTR_EVH, "%s: creating list \"%s\"", __func__, name); list = new_list; - TAILQ_INIT(&list->el_entries); + CK_LIST_INIT(&list->el_entries); list->el_name = (char *)(list + 1); + list->el_pepoch = pepoch; strcpy(list->el_name, name); mtx_init(&list->el_lock, list->el_name, "eventhandler list", MTX_DEF); @@ -107,9 +110,9 @@ */ static eventhandler_tag eventhandler_register_internal(struct eventhandler_list *list, - const char *name, eventhandler_tag epn) + const char *name, epoch_t *pepoch, eventhandler_tag epn) { - struct eventhandler_entry *ep; + struct eventhandler_entry *ep, *ep_last; KASSERT(eventhandler_lists_initted, ("eventhandler registered too early")); KASSERT(epn != NULL, ("%s: cannot register NULL event", __func__)); @@ -117,7 +120,7 @@ /* Do we need to find/create the list? */ if (list == NULL) { mtx_lock(&eventhandler_mutex); - list = eventhandler_find_or_create_list(name); + list = eventhandler_find_or_create_list(name, pepoch); mtx_unlock(&eventhandler_mutex); } @@ -128,15 +131,21 @@ CTR4(KTR_EVH, "%s: adding item %p (function %p) to \"%s\"", __func__, epn, ((struct eventhandler_entry_generic *)epn)->func, name); EHL_LOCK(list); - TAILQ_FOREACH(ep, &list->el_entries, ee_link) { + ep_last = NULL; + CK_LIST_FOREACH(ep, &list->el_entries, ee_link) { if (ep->ee_priority != EHE_DEAD_PRIORITY && epn->ee_priority < ep->ee_priority) { - TAILQ_INSERT_BEFORE(ep, epn, ee_link); + CK_LIST_INSERT_BEFORE(ep, epn, ee_link); break; } + ep_last = ep; + } + if (ep == NULL) { + if (ep_last != NULL) + CK_LIST_INSERT_AFTER(ep_last, epn, ee_link); + else + CK_LIST_INSERT_HEAD(&list->el_entries, epn, ee_link); } - if (ep == NULL) - TAILQ_INSERT_TAIL(&list->el_entries, epn, ee_link); EHL_UNLOCK(list); return(epn); } @@ -144,6 +153,13 @@ eventhandler_tag eventhandler_register(struct eventhandler_list *list, const char *name, void *func, void *arg, int priority) +{ + return (eventhandler_register_epoch(list, name, NULL, func, arg, priority)); +} + +eventhandler_tag +eventhandler_register_epoch(struct eventhandler_list *list, const char *name, + epoch_t *pepoch, void *func, void *arg, int priority) { struct eventhandler_entry_generic *eg; @@ -154,7 +170,7 @@ eg->ee.ee_arg = arg; eg->ee.ee_priority = priority; - return (eventhandler_register_internal(list, name, &eg->ee)); + return (eventhandler_register_internal(list, name, pepoch, &eg->ee)); } #ifdef VIMAGE @@ -180,10 +196,29 @@ eg->ee.ee_arg = &eg->v_ee; eg->ee.ee_priority = priority; - return (eventhandler_register_internal(list, name, &eg->ee)); + return (eventhandler_register_internal(list, name, NULL, &eg->ee)); } #endif +static void +destroy_ep_epoch(epoch_context_t ctx) +{ + struct eventhandler_entry *ep; + + ep = __containerof(ctx, struct eventhandler_entry, ee_epoch_ctx); + + free(ep, M_EVENTHANDLER); +} + +static void +free_list_item(struct eventhandler_list *list, struct eventhandler_entry *ep) +{ + if (EHL_TYPE_EPOCH(list)) + epoch_call(*(list->el_pepoch), destroy_ep_epoch, &ep->ee_epoch_ctx); + else + free(ep, M_EVENTHANDLER); +} + static void _eventhandler_deregister(struct eventhandler_list *list, eventhandler_tag tag, bool wait) @@ -193,11 +228,11 @@ EHL_LOCK_ASSERT(list, MA_OWNED); if (ep != NULL) { /* remove just this entry */ - if (list->el_runcount == 0) { + if ((list->el_runcount == 0) || EHL_TYPE_EPOCH(list)) { CTR3(KTR_EVH, "%s: removing item %p from \"%s\"", __func__, ep, list->el_name); - TAILQ_REMOVE(&list->el_entries, ep, ee_link); - free(ep, M_EVENTHANDLER); + CK_LIST_REMOVE(ep, ee_link); + free_list_item(list, ep); } else { CTR3(KTR_EVH, "%s: marking item %p from \"%s\" as dead", __func__, ep, list->el_name); @@ -205,18 +240,18 @@ } } else { /* remove entire list */ - if (list->el_runcount == 0) { + if ((list->el_runcount == 0) || EHL_TYPE_EPOCH(list)) { CTR2(KTR_EVH, "%s: removing all items from \"%s\"", __func__, list->el_name); - while (!TAILQ_EMPTY(&list->el_entries)) { - ep = TAILQ_FIRST(&list->el_entries); - TAILQ_REMOVE(&list->el_entries, ep, ee_link); - free(ep, M_EVENTHANDLER); + while (!CK_LIST_EMPTY(&list->el_entries)) { + ep = CK_LIST_FIRST(&list->el_entries); + CK_LIST_REMOVE(ep, ee_link); + free_list_item(list, ep); } } else { CTR2(KTR_EVH, "%s: marking all items from \"%s\" as dead", __func__, list->el_name); - TAILQ_FOREACH(ep, &list->el_entries, ee_link) + CK_LIST_FOREACH(ep, &list->el_entries, ee_link) ep->ee_priority = EHE_DEAD_PRIORITY; } } @@ -286,11 +321,14 @@ struct eventhandler_entry *ep, *en; int pruned = 0; + if (EHL_TYPE_EPOCH(list)) + return; + CTR2(KTR_EVH, "%s: pruning list \"%s\"", __func__, list->el_name); EHL_LOCK_ASSERT(list, MA_OWNED); - TAILQ_FOREACH_SAFE(ep, &list->el_entries, ee_link, en) { + CK_LIST_FOREACH_SAFE(ep, &list->el_entries, ee_link, en) { if (ep->ee_priority == EHE_DEAD_PRIORITY) { - TAILQ_REMOVE(&list->el_entries, ep, ee_link); + CK_LIST_REMOVE(ep, ee_link); free(ep, M_EVENTHANDLER); pruned++; } @@ -305,6 +343,12 @@ */ struct eventhandler_list * eventhandler_create_list(const char *name) +{ + return (eventhandler_create_list_epoch(name, NULL)); +} + +struct eventhandler_list * +eventhandler_create_list_epoch(const char *name, epoch_t *pepoch) { struct eventhandler_list *list; @@ -312,7 +356,7 @@ ("eventhandler list created too early")); mtx_lock(&eventhandler_mutex); - list = eventhandler_find_or_create_list(name); + list = eventhandler_find_or_create_list(name, pepoch); mtx_unlock(&eventhandler_mutex); return (list); Index: sys/sys/_eventhandler.h =================================================================== --- sys/sys/_eventhandler.h +++ sys/sys/_eventhandler.h @@ -32,12 +32,15 @@ #define _SYS__EVENTHANDLER_H_ #include +#include +#include struct eventhandler_entry { - TAILQ_ENTRY(eventhandler_entry) ee_link; + CK_LIST_ENTRY(eventhandler_entry) ee_link; int ee_priority; #define EHE_DEAD_PRIORITY (-1) void *ee_arg; + struct epoch_context ee_epoch_ctx; }; typedef struct eventhandler_entry *eventhandler_tag; Index: sys/sys/eventhandler.h =================================================================== --- sys/sys/eventhandler.h +++ sys/sys/eventhandler.h @@ -51,13 +51,16 @@ u_int el_runcount; struct mtx el_lock; TAILQ_ENTRY(eventhandler_list) el_link; - TAILQ_HEAD(,eventhandler_entry) el_entries; + CK_LIST_HEAD(,eventhandler_entry) el_entries; + epoch_t *el_pepoch; }; #define EHL_LOCK(p) mtx_lock(&(p)->el_lock) #define EHL_UNLOCK(p) mtx_unlock(&(p)->el_lock) #define EHL_LOCK_ASSERT(p, x) mtx_assert(&(p)->el_lock, x) +#define EHL_TYPE_EPOCH(list) ((list)->el_pepoch != NULL) + /* * Macro to invoke the handlers for a given event. */ @@ -70,7 +73,7 @@ KASSERT((list)->el_runcount > 0, \ ("eventhandler_invoke: runcount overflow")); \ CTR0(KTR_EVH, "eventhandler_invoke(\"" __STRING(name) "\")"); \ - TAILQ_FOREACH(_ep, &((list)->el_entries), ee_link) { \ + CK_LIST_FOREACH(_ep, &((list)->el_entries), ee_link) { \ if (_ep->ee_priority != EHE_DEAD_PRIORITY) { \ EHL_UNLOCK((list)); \ _t = (struct eventhandler_entry_ ## name *)_ep; \ @@ -88,6 +91,23 @@ EHL_UNLOCK((list)); \ } while (0) +/* + * Optimized version of the above that can be used for epoch-based lists + */ +#define _EVENTHANDLER_INVOKE_EPOCH(name, list, ...) do { \ + struct eventhandler_entry *_ep; \ + struct eventhandler_entry_ ## name *_t; \ + \ + MPASS(in_epoch(*((list)->el_pepoch))); \ + CTR0(KTR_EVH, "eventhandler_invoke_ep(\"" __STRING(name) "\")");\ + CK_LIST_FOREACH(_ep, &((list)->el_entries), ee_link) { \ + _t = (struct eventhandler_entry_ ## name *)_ep; \ + CTR1(KTR_EVH, "eventhandler_invoke_ep: executing %p", \ + (void *)_t->eh_func); \ + _t->eh_func(_ep->ee_arg , ## __VA_ARGS__); \ + } \ +} while (0) + /* * You can optionally use the EVENTHANDLER_LIST and EVENTHANDLER_DIRECT macros * to pre-define a symbol for the eventhandler list. This symbol can be used by @@ -103,7 +123,7 @@ struct eventhandler_list *_eventhandler_list_ ## name ; \ static void _ehl_init_ ## name (void * ctx __unused) \ { \ - _eventhandler_list_ ## name = eventhandler_create_list(#name); \ + _eventhandler_list_ ## name = eventhandler_create_list_epoch(#name, NULL); \ } \ SYSINIT(name ## _ehl_init, SI_SUB_EVENTHANDLER, SI_ORDER_ANY, \ _ehl_init_ ## name, NULL); \ @@ -113,12 +133,31 @@ struct eventhandler_list *_el; \ \ _el = _eventhandler_list_ ## name ; \ - if (!TAILQ_EMPTY(&_el->el_entries)) { \ + if (!CK_LIST_EMPTY(&_el->el_entries)) { \ EHL_LOCK(_el); \ _EVENTHANDLER_INVOKE(name, _el , ## __VA_ARGS__); \ } \ } while (0) +#define EVENTHANDLER_DEFINE_EPOCH(name, _epoch) \ +struct eventhandler_list *_eventhandler_list_ ## name ; \ +static void _ehl_init_ ## name (void * ctx __unused) \ +{ \ + _eventhandler_list_ ## name = eventhandler_create_list_epoch(#name, &_epoch); \ +} \ +SYSINIT(name ## _ehl_init, SI_SUB_EVENTHANDLER, SI_ORDER_ANY, \ + _ehl_init_ ## name, NULL); \ + struct __hack + +#define EVENTHANDLER_INVOKE_EPOCH(name, ...) do { \ + struct eventhandler_list *_el; \ + \ + _el = _eventhandler_list_ ## name ; \ + if (!CK_LIST_EMPTY(&_el->el_entries)) { \ + _EVENTHANDLER_INVOKE_EPOCH(name, _el , ## __VA_ARGS__); \ + } \ +} while (0) + #define EVENTHANDLER_DEFINE(name, func, arg, priority) \ static eventhandler_tag name ## _tag; \ static void name ## _evh_init(void *ctx) \ @@ -139,7 +178,10 @@ } while (0) #define EVENTHANDLER_REGISTER(name, func, arg, priority) \ - eventhandler_register(NULL, #name, func, arg, priority) + eventhandler_register_epoch(NULL, #name, NULL, func, arg, priority) + +#define EVENTHANDLER_REGISTER_EPOCH(name, epoch, func, arg, priority) \ + eventhandler_register_epoch(NULL, #name, &epoch, func, arg, priority) #define EVENTHANDLER_DEREGISTER(name, tag) \ do { \ @@ -159,6 +201,8 @@ eventhandler_tag eventhandler_register(struct eventhandler_list *list, const char *name, void *func, void *arg, int priority); +eventhandler_tag eventhandler_register_epoch(struct eventhandler_list *list, + const char *name, epoch_t *pepoch, void *func, void *arg, int priority); void eventhandler_deregister(struct eventhandler_list *list, eventhandler_tag tag); void eventhandler_deregister_nowait(struct eventhandler_list *list, @@ -166,6 +210,8 @@ struct eventhandler_list *eventhandler_find_list(const char *name); void eventhandler_prune_list(struct eventhandler_list *list); struct eventhandler_list *eventhandler_create_list(const char *name); +struct eventhandler_list *eventhandler_create_list_epoch(const char *name, + epoch_t *pepoch); #ifdef VIMAGE typedef void (*vimage_iterator_func_t)(void *, ...);