diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -140,66 +140,132 @@ * - 0, 'compiled in but silent by default' * - 1, 'compiled in but verbose by default' (default) */ -int verbose_sysinit = VERBOSE_SYSINIT; +static int verbose_sysinit = VERBOSE_SYSINIT; TUNABLE_INT("debug.verbose_sysinit", &verbose_sysinit); +SYSCTL_INT(_debug, OID_AUTO, verbose_sysinit, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, + &verbose_sysinit, 0, "Verbose sysinit enabled, if non-zero"); #endif #ifdef INVARIANTS FEATURE(invariants, "Kernel compiled with INVARIANTS, may affect performance"); #endif +extern sysinit_func_t * const __init_array_start[] __hidden; +extern sysinit_func_t * const __init_array_end[] __hidden; +extern sysinit_func_t * const __fini_array_start[] __hidden; +extern sysinit_func_t * const __fini_array_end[] __hidden; + /* - * This ensures that there is at least one entry so that the sysinit_set - * symbol is not undefined. A sybsystem ID of SI_SUB_DUMMY is never - * executed. + * Compare two sysinit structures. */ +static int +sysinit_compare(const struct sysinit *pa, const struct sysinit *pb, int neg) +{ + if (pa->subsystem > pb->subsystem) + return (neg); + else if (pa->subsystem < pb->subsystem) + return (-neg); + else if (pa->order > pb->order) + return (neg); + else if (pa->order < pb->order) + return (-neg); + else + return (0); +} + +SYSINIT_DECLARE_HEAD(sysinit_head); +SYSINIT_DECLARE_HEAD(sysuninit_head); + +#ifdef VERBOSE_SYSINIT +static size_t sysinit_insert_ops; static void -placeholder(void) +sysinit_insert_ops_print(void) { + if (bootverbose != 0 || verbose_sysinit != 0) + printf("sysinit_insert_sorted() looped %zu times\n", sysinit_insert_ops); } -SYSINIT(placeholder, SI_SUB_DUMMY, SI_ORDER_ANY, placeholder); +SYSINIT(sysinit_insert_ops_print, SI_SUB_LAST, SI_ORDER_ANY, sysinit_insert_ops_print); +#endif /* - * The sysinit table itself. Items are checked off as the are run. - * If we want to register new sysinit types, add them to newsysinit. + * Insert a sysinit structure into a list sorted by subsystem and order. + * + * @param ptr Pointer to the sysinit structure to be inserted. + * @param phead Pointer to the head structure for the sysinits. + * @param neg 1: Ascending order -1: Descending order */ -SET_DECLARE(sysinit_set, struct sysinit); -struct sysinit **sysinit, **sysinit_end; -struct sysinit **newsysinit, **newsysinit_end; +void +sysinit_insert_sorted(struct sysinit *ptr, sysinit_head_t *phead, int neg) +{ + /* don't insert reserved entries */ + if (ptr->subsystem == SI_SUB_DUMMY) + return; + + if (TAILQ_FIRST(&phead->h) == NULL) { + TAILQ_INSERT_TAIL(&phead->h, ptr, entry); + } else { + for (;;) { + struct sysinit *pnext; + struct sysinit *pprev; + int cmp; + + pnext = TAILQ_NEXT(phead->l, entry); + pprev = TAILQ_PREV(phead->l, __sysinit_head, entry); + + cmp = sysinit_compare(ptr, phead->l, neg); + if (cmp < 0) { + /* go backwards in list */ + if (pprev == NULL) { + TAILQ_INSERT_HEAD(&phead->h, ptr, entry); + break; + } else { + phead->l = pprev; + } + } else if (pnext == NULL || sysinit_compare(ptr, pnext, neg) < 0) { + /* insert after last element */ + TAILQ_INSERT_AFTER(&phead->h, phead->l, ptr, entry); + break; + } else { + /* advance to next element */ + phead->l = pnext; + } +#ifdef VERBOSE_SYSINIT + sysinit_insert_ops++; +#endif + } + } + phead->l = ptr; +} /* - * Merge a new sysinit set into the current set, reallocating it if - * necessary. This can only be called after malloc is running. + * Remove a sysinit structure from a list. + + * @param ptr Pointer to the sysinit structure to be removed + * @param phead Pointer to the head structure for the sysinits */ void -sysinit_add(struct sysinit **set, struct sysinit **set_end) +sysinit_remove(struct sysinit *ptr, sysinit_head_t *phead) { - struct sysinit **newset; - struct sysinit **sipp; - struct sysinit **xipp; - int count; - - count = set_end - set; - if (newsysinit) - count += newsysinit_end - newsysinit; - else - count += sysinit_end - sysinit; - newset = malloc(count * sizeof(*sipp), M_TEMP, M_NOWAIT); - if (newset == NULL) - panic("cannot malloc for sysinit"); - xipp = newset; - if (newsysinit) - for (sipp = newsysinit; sipp < newsysinit_end; sipp++) - *xipp++ = *sipp; - else - for (sipp = sysinit; sipp < sysinit_end; sipp++) - *xipp++ = *sipp; - for (sipp = set; sipp < set_end; sipp++) - *xipp++ = *sipp; - if (newsysinit) - free(newsysinit, M_TEMP); - newsysinit = newset; - newsysinit_end = newset + count; + /* update last pointer to next element, if it was removed */ + if (phead->l == ptr) { + phead->l = TAILQ_NEXT(ptr, entry); + /* if cannot go forward, then go backwards */ + if (phead->l == NULL) + phead->l = TAILQ_PREV(ptr, __sysinit_head, entry); + } + TAILQ_REMOVE(&phead->h, ptr, entry); +} + +void +sysinit_constructor_register(struct sysinit *ptr) +{ + sysinit_insert_sorted(ptr, &sysinit_head, 1); +} + +void +sysinit_destructor_register(struct sysinit *ptr) +{ + sysinit_insert_sorted(ptr, &sysuninit_head, -1); } #if defined (DDB) && defined(VERBOSE_SYSINIT) @@ -220,6 +286,53 @@ } #endif +void +sysinit_execute(sysinit_head_t *phead, const char *fname) +{ + struct sysinit *ptr; + enum sysinit_sub_id last = SI_SUB_DUMMY; +#if defined(VERBOSE_SYSINIT) + bool verbose = false; +#endif + + while ((ptr = TAILQ_FIRST(&phead->h))) { + sysinit_remove(ptr, phead); + + if (last != ptr->subsystem) { + last = ptr->subsystem; + BOOTTRACE("%s: sysinit execute 0x%07X", fname, last); + +#if defined(VERBOSE_SYSINIT) + verbose = last > SI_SUB_COPYRIGHT && verbose_sysinit != 0; + if (verbose) + printf("subsystem 0x%07X\n", last); +#endif + } + +#if defined(VERBOSE_SYSINIT) + if (verbose) { +#if defined(DDB) + const char *func; + + func = symbol_name((vm_offset_t)ptr->func, + DB_STGY_PROC); + if (func != NULL) + printf(" %s()... ", func); + else +#endif + printf(" %p()... ", ptr->func); + } +#endif + /* execute sysinit */ + ptr->func(); + +#if defined(VERBOSE_SYSINIT) + if (verbose) + printf("done.\n"); +#endif + } +} + /* * System startup; initialize the world, create process 0, mount root * filesystem, and fork to create init and pagedaemon. Most of the @@ -234,115 +347,36 @@ void mi_startup(void) { - - struct sysinit **sipp; /* system initialization*/ - struct sysinit **xipp; /* interior loop of sort*/ - struct sysinit *save; /* bubble*/ - - int last; -#if defined(VERBOSE_SYSINIT) - int verbose; -#endif + sysinit_func_t * const * si_start; + sysinit_func_t * const * si_stop; TSENTER(); if (boothowto & RB_VERBOSE) bootverbose++; - if (sysinit == NULL) { - sysinit = SET_BEGIN(sysinit_set); - sysinit_end = SET_LIMIT(sysinit_set); - } - -restart: - /* - * Perform a bubble sort of the system initialization objects by - * their subsystem (primary key) and order (secondary key). - */ - TSENTER2("bubblesort"); - for (sipp = sysinit; sipp < sysinit_end; sipp++) { - for (xipp = sipp + 1; xipp < sysinit_end; xipp++) { - if ((*sipp)->subsystem < (*xipp)->subsystem || - ((*sipp)->subsystem == (*xipp)->subsystem && - (*sipp)->order <= (*xipp)->order)) - continue; /* skip*/ - save = *sipp; - *sipp = *xipp; - *xipp = save; - } - } - TSEXIT2("bubblesort"); + si_start = __init_array_start; + si_stop = __init_array_end; - last = SI_SUB_COPYRIGHT; -#if defined(VERBOSE_SYSINIT) - verbose = 0; -#if !defined(DDB) +#if defined(VERBOSE_SYSINIT) && !defined(DDB) printf("VERBOSE_SYSINIT: DDB not enabled, symbol lookups disabled.\n"); #endif -#endif - - /* - * Traverse the (now) ordered list of system initialization tasks. - * Perform each task, and continue on to the next task. - */ - for (sipp = sysinit; sipp < sysinit_end; sipp++) { - if ((*sipp)->subsystem == SI_SUB_DUMMY) - continue; /* skip dummy task(s)*/ - - if ((*sipp)->subsystem == SI_SUB_DONE) - continue; - - if ((*sipp)->subsystem > last) - BOOTTRACE_INIT("sysinit 0x%7x", (*sipp)->subsystem); + /* run all the kernel constructors */ + for (; si_start != si_stop; si_start++) { #if defined(VERBOSE_SYSINIT) - if ((*sipp)->subsystem > last && verbose_sysinit != 0) { - verbose = 1; - printf("subsystem %x\n", last); + if (bootverbose != 0) { + printf("Constructor[%zu/%zu]=%p\n", + (size_t)(si_start - __init_array_start), + (size_t)(si_stop - __init_array_start - 1), *si_start); } - if (verbose) { -#if defined(DDB) - const char *func, *data; - - func = symbol_name((vm_offset_t)(*sipp)->func, - DB_STGY_PROC); - data = symbol_name((vm_offset_t)(*sipp)->udata, - DB_STGY_ANY); - if (func != NULL && data != NULL) - printf(" %s(&%s)... ", func, data); - else if (func != NULL) - printf(" %s(%p)... ", func, (*sipp)->udata); - else -#endif - printf(" %p(%p)... ", (*sipp)->func, - (*sipp)->udata); - } -#endif - - /* Call function */ - (*((*sipp)->func))((*sipp)->udata); - -#if defined(VERBOSE_SYSINIT) - if (verbose) - printf("done.\n"); #endif - - /* Check off the one we're just done */ - last = (*sipp)->subsystem; - (*sipp)->subsystem = SI_SUB_DONE; - - /* Check if we've installed more sysinit items via KLD */ - if (newsysinit != NULL) { - if (sysinit != SET_BEGIN(sysinit_set)) - free(sysinit, M_TEMP); - sysinit = newsysinit; - sysinit_end = newsysinit_end; - newsysinit = NULL; - newsysinit_end = NULL; - goto restart; - } + (*si_start)(); } + /* execute all the sysinits */ + sysinit_execute(&sysinit_head, "kernel"); + TSEXIT(); /* Here so we don't overlap with start_init. */ BOOTTRACE("mi_startup done"); @@ -871,9 +905,9 @@ */ #ifdef DDB static void -db_show_print_syinit(struct sysinit *sip, bool ddb) +db_show_print_function(sysinit_func_t *func, bool ddb) { - const char *sname, *funcname; + const char *funcname; c_db_sym_t sym; db_expr_t offset; @@ -883,31 +917,44 @@ else \ printf(__VA_ARGS__) - if (sip == NULL) { - xprint("%s: no sysinit * given\n", __func__); + if (func == NULL) { + xprint("%s: no function given\n", __func__); return; } - sym = db_search_symbol((vm_offset_t)sip, DB_STGY_ANY, &offset); - db_symbol_values(sym, &sname, NULL); - sym = db_search_symbol((vm_offset_t)sip->func, DB_STGY_PROC, &offset); + sym = db_search_symbol((vm_offset_t)func, DB_STGY_PROC, &offset); db_symbol_values(sym, &funcname, NULL); - xprint("%s(%p)\n", (sname != NULL) ? sname : "", sip); - xprint(" %#08x %#08x\n", sip->subsystem, sip->order); - xprint(" %p(%s)(%p)\n", - sip->func, (funcname != NULL) ? funcname : "", sip->udata); + xprint("%s(%p)\n", funcname, func); #undef xprint } -DB_SHOW_COMMAND_FLAGS(sysinit, db_show_sysinit, DB_CMD_MEMSAFE) +DB_SHOW_COMMAND_FLAGS(constructors, db_show_constructors, DB_CMD_MEMSAFE) +{ + sysinit_func_t * const * si_start; + sysinit_func_t * const * si_stop; + + si_start = __init_array_start; + si_stop = __init_array_end; + + db_printf("CONSTRUCTORS vs Name(Ptr)\n"); + for (; si_start != si_stop; si_start++) { + db_show_print_function(*si_start, true); + if (db_pager_quit) + break; + } +} + +DB_SHOW_COMMAND_FLAGS(destructors, db_show_destructors, DB_CMD_MEMSAFE) { - struct sysinit **sipp; + sysinit_func_t * const * si_start; + sysinit_func_t * const * si_stop; + + si_start = __fini_array_start; + si_stop = __fini_array_end; - db_printf("SYSINIT vs Name(Ptr)\n"); - db_printf(" Subsystem Order\n"); - db_printf(" Function(Name)(Arg)\n"); - for (sipp = sysinit; sipp < sysinit_end; sipp++) { - db_show_print_syinit(*sipp, true); + db_printf("DESTRUCTORS vs Name(Ptr)\n"); + for (; si_start != si_stop; si_start++) { + db_show_print_function(*si_start, true); if (db_pager_quit) break; } diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c --- a/sys/kern/kern_linker.c +++ b/sys/kern/kern_linker.c @@ -135,6 +135,14 @@ (a) = next_file_id; \ } while (0) +#define LINKER_LOOKUP_ARRAY(file,name,start,stop) ({ \ + start = (sysinit_func_t * const *) \ + linker_file_lookup_symbol_internal(file, #name "_start", 0);\ + stop = (sysinit_func_t * const *) \ + linker_file_lookup_symbol_internal(file, #name "_end", 0); \ + (start) != NULL && (stop) != NULL; \ +}) + /* XXX wrong name; we're looking at version provision tags here, not modules */ typedef TAILQ_HEAD(, modlist) modlisthead_t; struct modlist { @@ -195,54 +203,27 @@ static void linker_file_sysinit(linker_file_t lf) { - struct sysinit **start, **stop, **sipp, **xipp, *save; - int last; + sysinit_func_t * const * si_start; + sysinit_func_t * const * si_stop; - KLD_DPF(FILE, ("linker_file_sysinit: calling SYSINITs for %s\n", + KLD_DPF(FILE, ("linker_file_sysinit: calling constructors for %s\n", lf->filename)); sx_assert(&kld_sx, SA_XLOCKED); - if (linker_file_lookup_set(lf, "sysinit_set", &start, &stop, NULL) != 0) + if (LINKER_LOOKUP_ARRAY(lf, __init_array, si_start, si_stop) == false) return; - /* - * Perform a bubble sort of the system initialization objects by - * their subsystem (primary key) and order (secondary key). - * - * Since some things care about execution order, this is the operation - * which ensures continued function. - */ - for (sipp = start; sipp < stop; sipp++) { - for (xipp = sipp + 1; xipp < stop; xipp++) { - if ((*sipp)->subsystem < (*xipp)->subsystem || - ((*sipp)->subsystem == (*xipp)->subsystem && - (*sipp)->order <= (*xipp)->order)) - continue; /* skip */ - save = *sipp; - *sipp = *xipp; - *xipp = save; - } - } - /* - * Traverse the (now) ordered list of system initialization tasks. - * Perform each task, and continue on to the next task. - */ - last = SI_SUB_DUMMY; sx_xunlock(&kld_sx); mtx_lock(&Giant); - for (sipp = start; sipp < stop; sipp++) { - if ((*sipp)->subsystem == SI_SUB_DUMMY) - continue; /* skip dummy task(s) */ - if ((*sipp)->subsystem > last) - BOOTTRACE("%s: sysinit 0x%7x", lf->filename, - (*sipp)->subsystem); + /* run all the constructors provided by the kernel module */ + for (; si_start != si_stop; si_start++) + (*si_start)(); + + /* run all the registered SYSINITs */ + sysinit_execute(&sysinit_head, lf->filename); - /* Call function */ - (*((*sipp)->func)) ((*sipp)->udata); - last = (*sipp)->subsystem; - } mtx_unlock(&Giant); sx_xlock(&kld_sx); } @@ -250,56 +231,27 @@ static void linker_file_sysuninit(linker_file_t lf) { - struct sysinit **start, **stop, **sipp, **xipp, *save; - int last; + sysinit_func_t * const * si_start; + sysinit_func_t * const * si_stop; - KLD_DPF(FILE, ("linker_file_sysuninit: calling SYSUNINITs for %s\n", + KLD_DPF(FILE, ("linker_file_sysuninit: calling destructors for %s\n", lf->filename)); sx_assert(&kld_sx, SA_XLOCKED); - if (linker_file_lookup_set(lf, "sysuninit_set", &start, &stop, - NULL) != 0) + if (LINKER_LOOKUP_ARRAY(lf, __fini_array, si_start, si_stop) == false) return; - /* - * Perform a reverse bubble sort of the system initialization objects - * by their subsystem (primary key) and order (secondary key). - * - * Since some things care about execution order, this is the operation - * which ensures continued function. - */ - for (sipp = start; sipp < stop; sipp++) { - for (xipp = sipp + 1; xipp < stop; xipp++) { - if ((*sipp)->subsystem > (*xipp)->subsystem || - ((*sipp)->subsystem == (*xipp)->subsystem && - (*sipp)->order >= (*xipp)->order)) - continue; /* skip */ - save = *sipp; - *sipp = *xipp; - *xipp = save; - } - } - - /* - * Traverse the (now) ordered list of system initialization tasks. - * Perform each task, and continue on to the next task. - */ sx_xunlock(&kld_sx); mtx_lock(&Giant); - last = SI_SUB_DUMMY; - for (sipp = start; sipp < stop; sipp++) { - if ((*sipp)->subsystem == SI_SUB_DUMMY) - continue; /* skip dummy task(s) */ - - if ((*sipp)->subsystem > last) - BOOTTRACE("%s: sysuninit 0x%7x", lf->filename, - (*sipp)->subsystem); - - /* Call function */ - (*((*sipp)->func)) ((*sipp)->udata); - last = (*sipp)->subsystem; - } + + /* run all the destructors provided by the kernel module */ + for (; si_start != si_stop; si_start++) + (*si_start)(); + + /* run all the registered SYSUNINITs */ + sysinit_execute(&sysuninit_head, lf->filename); + mtx_unlock(&Giant); sx_xlock(&kld_sx); } @@ -1581,7 +1533,8 @@ int nver; int resolves; modlist_t mod; - struct sysinit **si_start, **si_stop; + sysinit_func_t * const * si_start; + sysinit_func_t * const * si_stop; TAILQ_INIT(&loaded_files); TAILQ_INIT(&depended_files); @@ -1762,9 +1715,10 @@ linker_file_register_modules(lf); if (!TAILQ_EMPTY(&lf->modules)) lf->flags |= LINKER_FILE_MODULES; - if (linker_file_lookup_set(lf, "sysinit_set", &si_start, - &si_stop, NULL) == 0) - sysinit_add(si_start, si_stop); + if (LINKER_LOOKUP_ARRAY(lf, __init_array, si_start, si_stop)) { + for (; si_start != si_stop; si_start++) + (*si_start)(); + } linker_file_register_sysctls(lf, true); lf->flags |= LINKER_FILE_LINKED; continue; diff --git a/sys/net/vnet.h b/sys/net/vnet.h --- a/sys/net/vnet.h +++ b/sys/net/vnet.h @@ -320,6 +320,8 @@ */ #include +typedef void (*sysinit_cfunc_t)(const void *); + /* * SYSINIT/SYSUNINIT variants that provide per-vnet constructors and * destructors. diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h --- a/sys/sys/kernel.h +++ b/sys/sys/kernel.h @@ -49,11 +49,10 @@ #include -#ifdef _KERNEL - -/* for intrhook below */ #include +#ifdef _KERNEL + /* for timestamping SYSINITs; other files may assume this is included here */ #include @@ -88,7 +87,6 @@ */ enum sysinit_sub_id { SI_SUB_DUMMY = 0x0000000, /* not executed; for linker*/ - SI_SUB_DONE = 0x0000001, /* processed*/ SI_SUB_TUNABLES = 0x0700000, /* establish tunable values */ SI_SUB_COPYRIGHT = 0x0800001, /* first use of console*/ SI_SUB_VM = 0x1000000, /* virtual memory system init */ @@ -192,37 +190,30 @@ SI_ORDER_ANY = 0xfffffff /* last*/ }; -/* - * A system initialization call instance - * - * At the moment there is one instance of sysinit. We probably do not - * want two which is why this code is if'd out, but we definitely want - * to discern SYSINIT's which take non-constant data pointers and - * SYSINIT's which take constant data pointers, - * - * The C_* macros take functions expecting const void * arguments - * while the non-C_* macros take functions expecting just void * arguments. - * - * With -Wcast-qual on, the compiler issues warnings: - * - if we pass non-const data or functions taking non-const data - * to a C_* macro. - * - * - if we pass const data to the normal macros - * - * However, no warning is issued if we pass a function taking const data - * through a normal non-const macro. This is ok because the function is - * saying it won't modify the data so we don't care whether the data is - * modifiable or not. - */ +struct sysinit; + +TAILQ_HEAD(__sysinit_head, sysinit); + +typedef struct { + struct __sysinit_head h; + struct sysinit *l; /* last entry added - to optimise sorted insertion */ +} sysinit_head_t; + +#define SYSINIT_DECLARE_HEAD(name) \ + sysinit_head_t name = { .h = TAILQ_HEAD_INITIALIZER(name.h), .l = NULL } + +#define SYSINIT_FOREACH(ptr, head) \ + TAILQ_FOREACH(ptr, &(head)->h, entry) -typedef void (*sysinit_nfunc_t)(void *); -typedef void (*sysinit_cfunc_t)(const void *); +typedef TAILQ_ENTRY(sysinit) sysinit_entry_t; + +typedef void (sysinit_func_t)(void); struct sysinit { - enum sysinit_sub_id subsystem; /* subsystem identifier*/ - enum sysinit_elem_order order; /* init order within subsystem*/ - sysinit_cfunc_t func; /* function */ - const void *udata; /* multiplexer/argument */ + sysinit_entry_t entry; /* entry for list */ + sysinit_func_t *func; /* function pointer */ + enum sysinit_sub_id subsystem; /* subsystem identifier */ + enum sysinit_elem_order order; /* init order within subsystem */ }; /* @@ -243,11 +234,18 @@ do { } while (0) #endif +#define __SI_PRIORITY_START 100 +#define __SI_PRIORITY_NUM 257 /* should be a prime number */ + +#define __SI_PRIORITY_HASH(sub, order) \ + (((sub) % __SI_PRIORITY_NUM) + \ + ((order) % __SI_PRIORITY_NUM) + __SI_PRIORITY_START) + /* * Explanation of arguments for the __SI_REGISTER() macro: * * @param uniq An identifier for the needed functions and structures. - * @param type sysinit_set or sysuninit_set, depending on use case. + * @param type constructor or destructor, depending on use case. * @param _sub SI_SUB_XXX enum. * @param _order SI_ORDER_XXX enum. * @param _func Pointer to callback function. @@ -259,37 +257,49 @@ * pointer casting between types is avoided entirely. */ #define __SI_REGISTER(uniq, type, _sub, _order, _func, ...) \ +_Static_assert((_sub) > SI_SUB_DUMMY && (_sub) <= SI_SUB_LAST, \ + "Subsystem is out of range"); \ +_Static_assert((_order) >= 0 && (_order) <= SI_ORDER_ANY, \ + "Order is out of range"); \ static void \ -type##_##uniq##_shim(const void *arg __unused) \ +type##_##uniq##_shim(void) \ { \ __SI_FUNCTION_PRE(type, _func); \ (_func)(__VA_ARGS__); \ __SI_FUNCTION_POST(type, _func); \ } \ -static struct sysinit type##_##uniq = { \ - .subsystem = _sub, \ - .order = _order, \ - .func = &type##_##uniq##_shim, \ - .udata = NULL, \ -}; \ -DATA_WSET(type, type##_##uniq); \ +static void __attribute__((type(__SI_PRIORITY_HASH(_sub, _order)))) \ + type##_##uniq##_register(void) { \ + static struct sysinit __data = { \ + .subsystem = _sub, \ + .order = _order, \ + .func = &type##_##uniq##_shim, \ + }; \ + sysinit_##type##_register(&__data); \ +} \ struct __hack /* * Sysinit API macro, called on kernel or module load: */ #define SYSINIT(uniq, sub, order, ...) \ - __SI_REGISTER(uniq, sysinit_set, sub, order, __VA_ARGS__) + __SI_REGISTER(uniq, constructor, sub, order, __VA_ARGS__) /* * Sysuninit API macros, called on module unload: */ #define SYSUNINIT(uniq, sub, order, ...) \ - __SI_REGISTER(uniq, sysuinit_set, sub, order, __VA_ARGS__) - -void sysinit_add(struct sysinit **set, struct sysinit **set_end); + __SI_REGISTER(uniq, destructor, sub, order, __VA_ARGS__) #ifdef _KERNEL +extern sysinit_head_t sysinit_head; +extern sysinit_head_t sysuninit_head; + +void sysinit_insert_sorted(struct sysinit *, sysinit_head_t *, int neg); +void sysinit_remove(struct sysinit *, sysinit_head_t *); +void sysinit_constructor_register(struct sysinit *); +void sysinit_destructor_register(struct sysinit *); +void sysinit_execute(sysinit_head_t *, const char *); /* * Infrastructure for tunable 'constants'. Value may be specified at compile