diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index f71ba6efaf5..3bb0becb9c9 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -569,10 +569,10 @@ proc0_init(void *dummy __unused) * Call the init and ctor for the new thread and proc. We wait * to do this until all other structures are fairly sane. */ - EVENTHANDLER_INVOKE(process_init, p); - EVENTHANDLER_INVOKE(thread_init, td); - EVENTHANDLER_INVOKE(process_ctor, p); - EVENTHANDLER_INVOKE(thread_ctor, td); + EVENTHANDLER_STATIC_INVOKE(process_init, p); + EVENTHANDLER_STATIC_INVOKE(thread_init, td); + EVENTHANDLER_STATIC_INVOKE(process_ctor, p); + EVENTHANDLER_STATIC_INVOKE(thread_ctor, td); /* * Charge root for one process. diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c index 953da2d908a..70fa5f1577c 100644 --- a/sys/kern/kern_exec.c +++ b/sys/kern/kern_exec.c @@ -1074,7 +1074,7 @@ exec_new_vmspace(struct image_params *imgp, struct sysentvec *sv) imgp->sysent = sv; /* May be called with Giant held */ - EVENTHANDLER_INVOKE(process_exec, p, imgp); + EVENTHANDLER_STATIC_INVOKE(process_exec, p, imgp); /* * Blow away entire process VM, if address space not shared, diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c index d05951e150c..ad86035b75c 100644 --- a/sys/kern/kern_exit.c +++ b/sys/kern/kern_exit.c @@ -329,7 +329,7 @@ exit1(struct thread *td, int rval, int signo) * Event handler could change exit status. * XXX what if one of these generates an error? */ - EVENTHANDLER_INVOKE(process_exit, p); + EVENTHANDLER_STATIC_INVOKE(process_exit, p); /* * If parent is waiting for us to exit or exec, diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 997a783522e..bac4787815c 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -699,7 +699,7 @@ do_fork(struct thread *td, struct fork_req *fr, struct proc *p2, struct thread * * Both processes are set up, now check if any loadable modules want * to adjust anything. */ - EVENTHANDLER_INVOKE(process_fork, p1, p2, fr->fr_flags); + EVENTHANDLER_STATIC_INVOKE(process_fork, p1, p2, fr->fr_flags); /* * Set the child start time and mark the process as being complete. diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c index de829228b1d..ac28436a774 100644 --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -151,6 +151,14 @@ const int thread_off_td_oncpu = offsetof(struct thread, td_oncpu); const int thread_off_td_pcb = offsetof(struct thread, td_pcb); const int thread_off_td_plist = offsetof(struct thread, td_plist); +EVENTHANDLER_STATIC_DEFINE(process_ctor); +EVENTHANDLER_STATIC_DEFINE(process_dtor); +EVENTHANDLER_STATIC_DEFINE(process_init); +EVENTHANDLER_STATIC_DEFINE(process_fini); +EVENTHANDLER_STATIC_DEFINE(process_exit); +EVENTHANDLER_STATIC_DEFINE(process_fork); +EVENTHANDLER_STATIC_DEFINE(process_exec); + int kstack_pages = KSTACK_PAGES; SYSCTL_INT(_kern, OID_AUTO, kstack_pages, CTLFLAG_RD, &kstack_pages, 0, "Kernel stack size in pages"); @@ -195,12 +203,12 @@ proc_ctor(void *mem, int size, void *arg, int flags) p = (struct proc *)mem; SDT_PROBE4(proc, , ctor , entry, p, size, arg, flags); - EVENTHANDLER_INVOKE(process_ctor, p); + EVENTHANDLER_STATIC_INVOKE(process_ctor, p); SDT_PROBE4(proc, , ctor , return, p, size, arg, flags); td = FIRST_THREAD_IN_PROC(p); if (td != NULL) { /* Make sure all thread constructors are executed */ - EVENTHANDLER_INVOKE(thread_ctor, td); + EVENTHANDLER_STATIC_INVOKE(thread_ctor, td); } return (0); } @@ -230,9 +238,9 @@ proc_dtor(void *mem, int size, void *arg) MPASS(td->td_su == NULL); /* Make sure all thread destructors are executed */ - EVENTHANDLER_INVOKE(thread_dtor, td); + EVENTHANDLER_STATIC_INVOKE(thread_dtor, td); } - EVENTHANDLER_INVOKE(process_dtor, p); + EVENTHANDLER_STATIC_INVOKE(process_dtor, p); if (p->p_ksi != NULL) KASSERT(! KSI_ONQ(p->p_ksi), ("SIGCHLD queue")); SDT_PROBE3(proc, , dtor, return, p, size, arg); @@ -256,7 +264,7 @@ proc_init(void *mem, int size, int flags) cv_init(&p->p_pwait, "ppwait"); cv_init(&p->p_dbgwait, "dbgwait"); TAILQ_INIT(&p->p_threads); /* all threads in proc */ - EVENTHANDLER_INVOKE(process_init, p); + EVENTHANDLER_STATIC_INVOKE(process_init, p); p->p_stats = pstats_alloc(); p->p_pgrp = NULL; SDT_PROBE3(proc, , init, return, p, size, flags); @@ -274,7 +282,7 @@ proc_fini(void *mem, int size) struct proc *p; p = (struct proc *)mem; - EVENTHANDLER_INVOKE(process_fini, p); + EVENTHANDLER_STATIC_INVOKE(process_fini, p); pstats_free(p->p_stats); thread_free(FIRST_THREAD_IN_PROC(p)); mtx_destroy(&p->p_mtx); diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index d3e7e306a68..a93e1bd4bae 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -144,6 +144,11 @@ struct tidhashhead *tidhashtbl; u_long tidhash; struct rwlock tidhash_lock; +EVENTHANDLER_STATIC_DEFINE(thread_ctor); +EVENTHANDLER_STATIC_DEFINE(thread_dtor); +EVENTHANDLER_STATIC_DEFINE(thread_init); +EVENTHANDLER_STATIC_DEFINE(thread_fini); + static lwpid_t tid_alloc(void) { @@ -201,7 +206,7 @@ thread_ctor(void *mem, int size, void *arg, int flags) */ td->td_critnest = 1; td->td_lend_user_pri = PRI_MAX; - EVENTHANDLER_INVOKE(thread_ctor, td); + EVENTHANDLER_STATIC_INVOKE(thread_ctor, td); #ifdef AUDIT audit_thread_alloc(td); #endif @@ -247,7 +252,7 @@ thread_dtor(void *mem, int size, void *arg) td_softdep_cleanup(td); MPASS(td->td_su == NULL); - EVENTHANDLER_INVOKE(thread_dtor, td); + EVENTHANDLER_STATIC_INVOKE(thread_dtor, td); tid_free(td->td_tid); } @@ -264,7 +269,7 @@ thread_init(void *mem, int size, int flags) td->td_sleepqueue = sleepq_alloc(); td->td_turnstile = turnstile_alloc(); td->td_rlqe = NULL; - EVENTHANDLER_INVOKE(thread_init, td); + EVENTHANDLER_STATIC_INVOKE(thread_init, td); umtx_thread_init(td); td->td_kstack = 0; td->td_sel = NULL; @@ -280,7 +285,7 @@ thread_fini(void *mem, int size) struct thread *td; td = (struct thread *)mem; - EVENTHANDLER_INVOKE(thread_fini, td); + EVENTHANDLER_STATIC_INVOKE(thread_fini, td); rlqentry_free(td->td_rlqe); turnstile_free(td->td_turnstile); sleepq_free(td->td_sleepqueue); diff --git a/sys/kern/subr_eventhandler.c b/sys/kern/subr_eventhandler.c index 793210cfe94..080d3fd654c 100644 --- a/sys/kern/subr_eventhandler.c +++ b/sys/kern/subr_eventhandler.c @@ -133,10 +133,22 @@ eventhandler_register_internal(struct eventhandler_list *list, } if (ep == NULL) TAILQ_INSERT_TAIL(&list->el_entries, epn, ee_link); + list->el_flags |= EHL_NONEMPTY; EHL_UNLOCK(list); return(epn); } +void +_eventhandler_insert_static(void *ctx) +{ + struct eventhandler_list *list = ctx; + KASSERT(eventhandler_lists_initted, + ("eventhandler registered too early")); + mtx_lock(&eventhandler_mutex); + TAILQ_INSERT_HEAD(&eventhandler_lists, list, el_link); + mtx_unlock(&eventhandler_mutex); +} + eventhandler_tag eventhandler_register(struct eventhandler_list *list, const char *name, void *func, void *arg, int priority) @@ -216,6 +228,8 @@ _eventhandler_deregister(struct eventhandler_list *list, eventhandler_tag tag, ep->ee_priority = EHE_DEAD_PRIORITY; } } + if (TAILQ_EMPTY(&list->el_entries)) + list->el_flags &= ~EHL_NONEMPTY; while (wait && list->el_runcount > 0) mtx_sleep(list, &list->el_lock, 0, "evhrm", 0); EHL_UNLOCK(list); diff --git a/sys/sys/eventhandler.h b/sys/sys/eventhandler.h index 7fccffffa8a..30fa0c7f324 100644 --- a/sys/sys/eventhandler.h +++ b/sys/sys/eventhandler.h @@ -53,6 +53,7 @@ struct eventhandler_list { char *el_name; int el_flags; #define EHL_INITTED (1<<0) +#define EHL_NONEMPTY (1<<1) u_int el_runcount; struct mtx el_lock; TAILQ_ENTRY(eventhandler_list) el_link; @@ -97,11 +98,46 @@ typedef struct eventhandler_entry *eventhandler_tag; EHL_UNLOCK((list)); \ } while (0) +void _eventhandler_insert_static(void *ctx); + +/* + * Static event handler lists require the eventhandler list be present at link + * time. They don't allow addition of entries to unknown eventhandler lists. + * + * Static handler lists must be defined once by the "owner" of the eventhandler + * list, and the declaration must be in scope at any point the list is + * manipulated. + */ +#define EVENTHANDLER_STATIC_DECLARE(name, type) \ +extern struct eventhandler_list Xeventhandler_list_ ## name ; \ +struct eventhandler_entry_ ## name { \ + struct eventhandler_entry ee; \ + type eh_func; \ +}; \ +struct __hack + +#define EVENTHANDLER_STATIC_DEFINE(name) \ +struct eventhandler_list Xeventhandler_list_ ## name = { #name }; \ +SYSINIT(name ## _evh_insert, SI_SUB_CONFIGURE, SI_ORDER_ANY, \ + _eventhandler_insert_static, &Xeventhandler_list_ ## name); \ + struct __hack + +#define EVENTHANDLER_STATIC_INVOKE(name, ...) \ +do { \ + struct eventhandler_list *_el = &Xeventhandler_list_ ## name ; \ + \ + if (_el->el_flags & EHL_INITTED && \ + _el->el_flags & EHL_NONEMPTY) { \ + EHL_LOCK(_el); \ + _EVENTHANDLER_INVOKE(name, _el , ## __VA_ARGS__); \ + } \ +} while (0) + /* - * Slow handlers are entirely dynamic; lists are created + * Non-static handlers are entirely dynamic; lists are created * when entries are added to them, and thus have no concept of "owner", * - * Slow handlers need to be declared, but do not need to be defined. The + * These handlers need to be declared, but do not need to be defined. The * declaration must be in scope wherever the handler is to be invoked. */ #define EVENTHANDLER_DECLARE(name, type) \ @@ -228,13 +264,13 @@ typedef void (*proc_ctor_fn)(void *, struct proc *); typedef void (*proc_dtor_fn)(void *, struct proc *); typedef void (*proc_init_fn)(void *, struct proc *); typedef void (*proc_fini_fn)(void *, struct proc *); -EVENTHANDLER_DECLARE(process_ctor, proc_ctor_fn); -EVENTHANDLER_DECLARE(process_dtor, proc_dtor_fn); -EVENTHANDLER_DECLARE(process_init, proc_init_fn); -EVENTHANDLER_DECLARE(process_fini, proc_fini_fn); -EVENTHANDLER_DECLARE(process_exit, exitlist_fn); -EVENTHANDLER_DECLARE(process_fork, forklist_fn); -EVENTHANDLER_DECLARE(process_exec, execlist_fn); +EVENTHANDLER_STATIC_DECLARE(process_ctor, proc_ctor_fn); +EVENTHANDLER_STATIC_DECLARE(process_dtor, proc_dtor_fn); +EVENTHANDLER_STATIC_DECLARE(process_init, proc_init_fn); +EVENTHANDLER_STATIC_DECLARE(process_fini, proc_fini_fn); +EVENTHANDLER_STATIC_DECLARE(process_exit, exitlist_fn); +EVENTHANDLER_STATIC_DECLARE(process_fork, forklist_fn); +EVENTHANDLER_STATIC_DECLARE(process_exec, execlist_fn); /* * application dump event @@ -253,10 +289,10 @@ typedef void (*thread_ctor_fn)(void *, struct thread *); typedef void (*thread_dtor_fn)(void *, struct thread *); typedef void (*thread_fini_fn)(void *, struct thread *); typedef void (*thread_init_fn)(void *, struct thread *); -EVENTHANDLER_DECLARE(thread_ctor, thread_ctor_fn); -EVENTHANDLER_DECLARE(thread_dtor, thread_dtor_fn); -EVENTHANDLER_DECLARE(thread_init, thread_init_fn); -EVENTHANDLER_DECLARE(thread_fini, thread_fini_fn); +EVENTHANDLER_STATIC_DECLARE(thread_ctor, thread_ctor_fn); +EVENTHANDLER_STATIC_DECLARE(thread_dtor, thread_dtor_fn); +EVENTHANDLER_STATIC_DECLARE(thread_init, thread_init_fn); +EVENTHANDLER_STATIC_DECLARE(thread_fini, thread_fini_fn); typedef void (*uma_zone_chfn)(void *); EVENTHANDLER_DECLARE(nmbclusters_change, uma_zone_chfn);