Index: sys/kern/subr_eventhandler.c =================================================================== --- sys/kern/subr_eventhandler.c +++ sys/kern/subr_eventhandler.c @@ -37,6 +37,7 @@ #include #include #include +#include #include static MALLOC_DEFINE(M_EVENTHANDLER, "eventhandler", "Event handler records"); @@ -317,3 +318,361 @@ return (list); } + +/* + * Preemptible and sleepable event handler points. + */ +static void +eventhandler_preemptible_lock(struct eventhandler_common *ehc) +{ + struct eventhandler_preemptible *ehp; + + ehp = __containerof(ehc, struct eventhandler_preemptible, common); + rm_wlock(&ehp->lock); +} + +static void +eventhandler_preemptible_unlock(struct eventhandler_common *ehc) +{ + struct eventhandler_preemptible *ehp; + + ehp = __containerof(ehc, struct eventhandler_preemptible, common); + rm_wunlock(&ehp->lock); +} + +static void +eventhandler_sleepable_lock(struct eventhandler_common *ehc) +{ + struct eventhandler_sleepable *ehs; + + ehs = __containerof(ehc, struct eventhandler_sleepable, common); + rms_wlock(&ehs->lock); +} + +static void +eventhandler_sleepable_unlock(struct eventhandler_common *ehc) +{ + struct eventhandler_sleepable *ehs; + + ehs = __containerof(ehc, struct eventhandler_sleepable, common); + rms_wunlock(&ehs->lock); +} + +static void +eventhandler_common_validate(struct eventhandler_common *ehc) +{ + int i; + + for (i = 0; i < ehc->count; i++) { + if (ehc->cb[i] != NULL) { + switch (ehc->prio[i]) { + case EVENTHANDLER_PRIORITY_FIRST: + case EVENTHANDLER_PRIORITY_ANY: + case EVENTHANDLER_PRIORITY_LAST: + break; + case EVENTHANDLER_PRIORITY_INVALID: + default: + panic("%s bad priority %d at %d\n", ehc->ident, + ehc->prio[i], i); + break; + } + } + } + + for (i = 1; i < ehc->count; i++) { + if (ehc->prio[i] < ehc->prio[i - 1]) { + panic("%s unsorted priorities (%d < %d) at %d\n", ehc->ident, + ehc->prio[i], ehc->prio[i - 1], i); + } + } + + for (i = ehc->count; i < ehc->size; i++) { + if (ehc->cb[i] != NULL) { + panic("%s callback %p found past the end at %d\n", + ehc->ident, ehc->cb[i], i); + } + if (ehc->prio[i] != EVENTHANDLER_PRIORITY_INVALID) { + panic("%s prio %d found past the end at %d\n", + ehc->ident, ehc->prio[i], i); + } + } +} + +static void +eventhandler_common_assert_prio(struct eventhandler_common *ehc, + enum eventhandler_priority prio) +{ + + switch (prio) { + case EVENTHANDLER_PRIORITY_FIRST: + case EVENTHANDLER_PRIORITY_ANY: + case EVENTHANDLER_PRIORITY_LAST: + break; + default: + case EVENTHANDLER_PRIORITY_INVALID: + panic("invalid priority %d passed", prio); + break; + } +} + +void +eventhandler_preemptible_init(struct eventhandler_preemptible *ehp) +{ + + rm_init(&ehp->lock, ehp->common.ident); +} + +void +eventhandler_sleepable_init(struct eventhandler_sleepable *ehs) +{ + + rms_init(&ehs->lock, ehs->common.ident); +} + +static void +eventhandler_common_realloc(struct eventhandler_common *ehc, + void (*lock)(struct eventhandler_common *), + void (*unlock)(struct eventhandler_common *)) +{ + eventhandler_cb **newcb, **tofree_cb; + size_t newsize; + enum eventhandler_priority *newprio, *tofree_prio; + + MPASS(ehc->count == ehc->size); + + for (;;) { + if (ehc->size == 0) + newsize = 8; + else + newsize = ehc->size * 2; + + unlock(ehc); + + newcb = malloc(sizeof(ehc->cb[0]) * newsize, + M_EVENTHANDLER, M_WAITOK | M_ZERO); + newprio = malloc(sizeof(ehc->prio[0]) * newsize, + M_EVENTHANDLER, M_WAITOK | M_ZERO); + + lock(ehc); + + if (ehc->count == newsize) { + free(newcb, M_EVENTHANDLER); + free(newprio, M_EVENTHANDLER); + continue; + } + + if (ehc->size >= newsize) { + free(newcb, M_EVENTHANDLER); + free(newprio, M_EVENTHANDLER); + MPASS(ehc->count < ehc->size); + return; + } + + break; + } + + tofree_cb = ehc->cb; + tofree_prio = ehc->prio; + + memcpy(newcb, ehc->cb, sizeof(ehc->cb[0]) * ehc->size); + memcpy(newprio, ehc->prio, sizeof(ehc->prio[0]) * ehc->size); + + ehc->cb = newcb; + ehc->prio = newprio; + ehc->size = newsize; + + free(tofree_cb, M_EVENTHANDLER); + free(tofree_prio, M_EVENTHANDLER); +} + +static void +eventhandler_common_register(struct eventhandler_common *ehc, void *pcb, + enum eventhandler_priority prio, void (*lock)(struct eventhandler_common *), + void (*unlock)(struct eventhandler_common *)) +{ + eventhandler_cb *cb; + int i; + + eventhandler_common_assert_prio(ehc, prio); + + cb = pcb; + lock(ehc); + MPASS(ehc->count <= ehc->size); + + if (ehc->count == ehc->size) { + eventhandler_common_realloc(ehc, lock, unlock); + MPASS(ehc->count < ehc->size); + } + + eventhandler_common_validate(ehc); + + for (i = 0; i < ehc->size; i++) { + if (ehc->cb[i] == cb) + panic("callback %p already registered", cb); + } + + switch (prio) { + case EVENTHANDLER_PRIORITY_INVALID: + /* appease clang */ + __builtin_unreachable(); + break; + case EVENTHANDLER_PRIORITY_FIRST: + memmove(&ehc->cb[1], &ehc->cb[0], + sizeof(ehc->cb[0]) * ehc->count); + memmove(&ehc->prio[1], &ehc->prio[0], + sizeof(ehc->prio[0]) * ehc->count); + ehc->cb[0] = cb; + ehc->prio[0] = prio; + ehc->count++; + break; + case EVENTHANDLER_PRIORITY_ANY: + for (i = 0; i < ehc->count; i++) { + if (ehc->prio[i] != EVENTHANDLER_PRIORITY_FIRST) + break; + } + /* + * Move all entries to cover for EVENTHANDLER_PRIORITY_LAST. + */ + memmove(&ehc->cb[i + 1], &ehc->cb[i], + sizeof(ehc->cb[0]) * (ehc->count - i)); + memmove(&ehc->prio[i + 1], &ehc->prio[i], + sizeof(ehc->prio[0]) * (ehc->count - i)); + ehc->cb[i] = cb; + ehc->prio[i] = prio; + ehc->count++; + break; + case EVENTHANDLER_PRIORITY_LAST: + ehc->cb[ehc->count] = cb; + ehc->prio[ehc->count] = prio; + ehc->count++; + break; + } + eventhandler_common_validate(ehc); + unlock(ehc); +} + +void +eventhandler_preemptible_register(struct eventhandler_preemptible *ehp, void *pcb, + enum eventhandler_priority prio) +{ + struct eventhandler_common *ehc; + + ehc = &ehp->common; + eventhandler_common_register(ehc, pcb, prio, eventhandler_preemptible_lock, + eventhandler_preemptible_unlock); +} + +void +eventhandler_sleepable_register(struct eventhandler_sleepable *ehs, void *pcb, + enum eventhandler_priority prio) +{ + struct eventhandler_common *ehc; + + ehc = &ehs->common; + eventhandler_common_register(ehc, pcb, prio, eventhandler_sleepable_lock, + eventhandler_sleepable_unlock); +} + +static void +eventhandler_common_deregister(struct eventhandler_common *ehc, void *pcb, + void (*lock)(struct eventhandler_common *), + void (*unlock)(struct eventhandler_common *)) +{ + eventhandler_cb *cb; + int i; + + cb = pcb; + lock(ehc); + MPASS(ehc->count <= ehc->size); + eventhandler_common_validate(ehc); + + for (i = 0; i < ehc->count; i++) { + if (ehc->cb[i] == cb) + break; + } + + if (i == ehc->count) + panic("callback %p not found", cb); + + switch (ehc->prio[i]) { + case EVENTHANDLER_PRIORITY_INVALID: + /* appease clang */ + __builtin_unreachable(); + break; + case EVENTHANDLER_PRIORITY_FIRST: + case EVENTHANDLER_PRIORITY_ANY: + case EVENTHANDLER_PRIORITY_LAST: + memmove(&ehc->cb[i], &ehc->cb[i + 1], + sizeof(ehc->cb[0]) * (ehc->count - i - 1)); + memmove(&ehc->prio[i], &ehc->prio[i + 1], + sizeof(ehc->prio[0]) * (ehc->count - i - 1)); + break; + } + ehc->cb[ehc->count - 1] = NULL; + ehc->prio[ehc->count - 1] = EVENTHANDLER_PRIORITY_INVALID; + ehc->count--; + + eventhandler_common_validate(ehc); + unlock(ehc); +} + +void +eventhandler_preemptible_deregister(struct eventhandler_preemptible *ehp, void *pcb) +{ + struct eventhandler_common *ehc; + + ehc = &ehp->common; + eventhandler_common_deregister(ehc, pcb, eventhandler_preemptible_lock, + eventhandler_preemptible_unlock); +} + +void +eventhandler_sleepable_deregister(struct eventhandler_sleepable *ehs, void *pcb) +{ + struct eventhandler_common *ehc; + + ehc = &ehs->common; + eventhandler_common_deregister(ehc, pcb, eventhandler_sleepable_lock, + eventhandler_sleepable_unlock); +} + +static void +eventhandler_common_invoke(struct eventhandler_common *ehc, void *arg) +{ + int i; + + for (i = 0; i < ehc->count; i++) { + (ehc->cb[i])(arg); + } +} + +void +eventhandler_preemptible_invoke(struct eventhandler_preemptible *ehs, void *arg) +{ + struct rm_priotracker tracker; + struct eventhandler_common *ehc; + + ehc = &ehs->common; + if (atomic_load_int(&ehc->count) == 0) + return; + + rm_rlock(&ehs->lock, &tracker); + eventhandler_common_invoke(ehc, arg); + rm_runlock(&ehs->lock, &tracker); +} + +void +eventhandler_sleepable_invoke(struct eventhandler_sleepable *ehs, void *arg) +{ + struct eventhandler_common *ehc; + + rms_assert_rlock_ok(ehs->lock); + + ehc = &ehs->common; + if (atomic_load_int(&ehc->count) == 0) + return; + + rms_rlock(&ehs->lock); + eventhandler_common_invoke(ehc, arg); + rms_runlock(&ehs->lock); +} Index: sys/sys/_eventhandler.h =================================================================== --- sys/sys/_eventhandler.h +++ sys/sys/_eventhandler.h @@ -69,4 +69,82 @@ }; \ struct __hack +/* + * Headers to accomodate _rmlock.h + */ +#include +#include +#include +#include +#include +#include + +typedef void eventhandler_cb(void *); + +enum eventhandler_priority { + EVENTHANDLER_PRIORITY_INVALID, + EVENTHANDLER_PRIORITY_FIRST, + EVENTHANDLER_PRIORITY_ANY, + EVENTHANDLER_PRIORITY_LAST +}; + +struct eventhandler_common { + eventhandler_cb **cb; + u_int count; + u_int size; + const char *ident; + enum eventhandler_priority *prio; +}; + +struct eventhandler_preemptible { + struct rmlock lock; + struct eventhandler_common common; +}; + +struct eventhandler_sleepable { + struct rmslock lock; + struct eventhandler_common common; +}; + +#define EVENTHANDLER_PREEMPTIBLE_DECLARE(name) \ +extern struct eventhandler_preemptible eventhandler_preemptible_ ## name + +#define EVENTHANDLER_PREEMPTIBLE_REGISTER(name, func, prio) \ + eventhandler_preemptible_register(&eventhandler_preemptible_ ## name, func, \ + EVENTHANDLER_PRIORITY_ ## prio) + +#define EVENTHANDLER_PREEMPTIBLE_DEREGISTER(name, func) \ + eventhandler_preemptible_deregister(&eventhandler_preemptible_ ## name, func) + +#define EVENTHANDLER_PREEMPTIBLE_INVOKE(name, arg) \ + eventhandler_preemptible_invoke(&eventhandler_preemptible_ ## name, arg) + +#define EVENTHANDLER_PREEMPTIBLE_DEFINE(name) \ +struct eventhandler_preemptible eventhandler_preemptible_ ## name = { \ + .common.ident = __STRING(name), \ +}; \ +SYSINIT(_ehs_init_ ## name, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, \ + eventhandler_preemptible_init, &eventhandler_preemptible_ ## name); + +#define EVENTHANDLER_SLEEPABLE_DECLARE(name) \ +extern struct eventhandler_sleepable eventhandler_sleepable_ ## name + +#define EVENTHANDLER_SLEEPABLE_REGISTER(name, func, prio) \ + { eventhandler_cb *__typecheck __unused = func; } \ + eventhandler_sleepable_register(&eventhandler_sleepable_ ## name, func, \ + EVENTHANDLER_PRIORITY_ ## prio) + +#define EVENTHANDLER_SLEEPABLE_DEREGISTER(name, func) \ + eventhandler_sleepable_deregister(&eventhandler_sleepable_ ## name, func) + +#define EVENTHANDLER_SLEEPABLE_INVOKE(name, arg) \ + eventhandler_sleepable_invoke(&eventhandler_sleepable_ ## name, arg) + +#define EVENTHANDLER_SLEEPABLE_DEFINE(name) \ +struct eventhandler_sleepable eventhandler_sleepable_ ## name = { \ + .common.ident = __STRING(name), \ +}; \ +SYSINIT(_ehs_init_ ## name, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, \ + eventhandler_sleepable_init, &eventhandler_sleepable_ ## name); + #endif Index: sys/sys/eventhandler.h =================================================================== --- sys/sys/eventhandler.h +++ sys/sys/eventhandler.h @@ -138,8 +138,15 @@ _EVENTHANDLER_INVOKE(name, _el , ## __VA_ARGS__); \ } while (0) -#define EVENTHANDLER_REGISTER(name, func, arg, priority) \ - eventhandler_register(NULL, #name, func, arg, priority) +/* + * preemptible and sleepable vars get declared in order to trigger compilation + * failure if the passed event already exists by that type. + */ +#define EVENTHANDLER_REGISTER(name, func, arg, priority) ({ \ + extern int eventhandler_preemptible_ ## name; \ + extern int eventhandler_sleepable_ ## name; \ + eventhandler_register(NULL, #name, func, arg, priority); \ +}) #define EVENTHANDLER_DEREGISTER(name, tag) \ do { \ @@ -321,4 +328,19 @@ typedef void (*rt_addrmsg_fn)(void *, struct ifaddr *, int); EVENTHANDLER_DECLARE(rt_addrmsg, rt_addrmsg_fn); +/* + * Preemptible and sleepable eventhandler support. + */ +void eventhandler_preemptible_init(struct eventhandler_preemptible *); +void eventhandler_preemptible_register(struct eventhandler_preemptible *, void *, + enum eventhandler_priority); +void eventhandler_preemptible_deregister(struct eventhandler_preemptible *, void *); +void eventhandler_preemptible_invoke(struct eventhandler_preemptible *, void *); + +void eventhandler_sleepable_init(struct eventhandler_sleepable *); +void eventhandler_sleepable_register(struct eventhandler_sleepable *, void *, + enum eventhandler_priority); +void eventhandler_sleepable_deregister(struct eventhandler_sleepable *, void *); +void eventhandler_sleepable_invoke(struct eventhandler_sleepable *, void *); + #endif /* _SYS_EVENTHANDLER_H_ */