Index: sys/arm/arm/elf_machdep.c =================================================================== --- sys/arm/arm/elf_machdep.c +++ sys/arm/arm/elf_machdep.c @@ -47,6 +47,7 @@ #include #include +#include #ifdef VFP #include #endif @@ -309,12 +310,21 @@ cpu_l2cache_wb_range((vm_offset_t)lf->address, (vm_size_t)lf->size); cpu_icache_sync_range((vm_offset_t)lf->address, (vm_size_t)lf->size); #endif + + /* + * Inform the stack(9) code of the new module, so it can acquire its + * per-module unwind data. + */ + unwind_module_loaded(lf); + return (0); } int -elf_cpu_unload_file(linker_file_t lf __unused) +elf_cpu_unload_file(linker_file_t lf) { + /* Inform the stack(9) code that this module is gone. */ + unwind_module_unloaded(lf); return (0); } Index: sys/arm/arm/unwind.c =================================================================== --- sys/arm/arm/unwind.c +++ sys/arm/arm/unwind.c @@ -32,8 +32,11 @@ __FBSDID("$FreeBSD$"); #include -#include +#include #include +#include +#include +#include #include #include @@ -94,79 +97,238 @@ uint32_t insn; }; -/* Expand a 31-bit signed value to a 32-bit signed value */ -static __inline int32_t -expand_prel31(uint32_t prel31) +/* + * Local cache of unwind info for loaded modules. + * + * To unwind the stack through the code in a loaded module, we need to access + * the module's exidx unwind data. To locate that data, one must search the + * elf section headers for the SHT_ARM_EXIDX section. Those headers are + * available at the time the module is being loaded, but are discarded by time + * the load process has completed. Code in kern/link_elf.c locates the data we + * need and stores it into the linker_file structure before calling the arm + * machdep routine for handling loaded modules (in arm/elf_machdep.c). That + * function calls into this code to pass along the unwind info, which we save + * into one of these module_info structures. + * + * Because we have to help stack(9) gather stack info at any time, including in + * contexts where sleeping is not allowed, we cannot use linker_file_foreach() + * to walk the kernel's list of linker_file structs, because doing so requires + * acquiring an exclusive sx_lock. So instead, we keep a local list of these + * structures, one for each loaded module (and one for the kernel itself that we + * synthesize at init time). New entries are added to the end of this list as + * needed, but entries are never deleted from the list. Instead, they are + * cleared out in-place to mark them as unused. That means the code doing stack + * unwinding can always safely walk the list without locking, because the + * structure of the list never changes in a way that would cause the walker to + * follow a bad link. + * + * A cleared-out entry on the list has module start=UINTPTR_MAX and end=0, so + * start <= addr < end cannot be true for any value of addr being searched for. + * We also don't have to worry about races where we look up the unwind info just + * before a module is unloaded and try to access it concurrently with or just + * after the unloading happens in another thread, because that means the path of + * execution leads through a now-unloaded module, and that's already well into + * undefined-behavior territory. + * + * List entries marked as unused get reused when new modules are loaded. We + * don't worry about holding a few unused bytes of memory in the list after + * unloading a module. Even if you loaded 1000 modules, did something that used + * them, then unloaded them all, we'd be wasting about 20K. In real-world + * scenarios the waste will be maybe a couple hundred bytes at most. + */ +struct module_info { + uintptr_t module_start; /* Start of loaded module */ + uintptr_t module_end; /* End of loaded module */ + uintptr_t exidx_start; /* Start of unwind data */ + uintptr_t exidx_end; /* End of unwind data */ + STAILQ_ENTRY(module_info) + link; /* Link to next entry */ +}; +static STAILQ_HEAD(, module_info) module_list; + +/* + * Hide ugly casting in somewhat-less-ugly macros. + * CADDR - cast a pointer or number to caddr_t. + * UADDR - cast a pointer or number to uintptr_t. + */ +#define CADDR(addr) ((caddr_t)(void*)(uintptr_t)(addr)) +#define UADDR(addr) ((uintptr_t)(addr)) + +/* + * Clear the info in an existing module_info entry on the list. The + * module_start/end addresses are set to values that cannot match any real + * memory address. The entry remains on the list, but will be ignored until it + * is populated with new data. + */ +static void +clear_module_info(struct module_info *info) { + info->module_start = UINTPTR_MAX; + info->module_end = 0; +} - return ((int32_t)(prel31 & 0x7fffffffu) << 1) / 2; +/* + * Populate an existing module_info entry (which is already on the list) with + * the info for a new module. + */ +static void +populate_module_info(struct module_info *info, linker_file_t lf) +{ + + /* + * Careful! The module_start and module_end fields must not be set + * until all other data in the structure is valid. + */ + info->exidx_start = UADDR(lf->exidx_addr); + info->exidx_end = UADDR(lf->exidx_addr) + lf->exidx_size; + info->module_start = UADDR(lf->address); + info->module_end = UADDR(lf->address) + lf->size; } -struct search_context { - uint32_t addr; - caddr_t exidx_start; - caddr_t exidx_end; -}; +/* + * Create a new empty module_info entry and add it to the tail of the list. + */ +static struct module_info * +create_module_info(void) +{ + struct module_info *info; -static int -module_search(linker_file_t lf, void *context) + info = malloc(sizeof(*info), M_CACHE, M_WAITOK | M_ZERO); + clear_module_info(info); + STAILQ_INSERT_TAIL(&module_list, info, link); + return (info); +} + +/* + * Search for a module_info entry on the list whose address range contains the + * given address. If the search address is zero (no module will be loaded at + * zero), then we're looking for an empty item to reuse, which is indicated by + * module_start being set to UINTPTR_MAX in the entry. + */ +static struct module_info * +find_module_info(uintptr_t addr) { - struct search_context *sc = context; - linker_symval_t symval; - c_linker_sym_t sym; + struct module_info *info; - if (lf->address <= (caddr_t)sc->addr && - (lf->address + lf->size) >= (caddr_t)sc->addr) { - if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_start", &sym) == 0 || - LINKER_LOOKUP_SYMBOL(lf, "exidx_start", &sym) == 0) && - LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) - sc->exidx_start = symval.value; + STAILQ_FOREACH(info, &module_list, link) { + if ((addr >= info->module_start && addr < info->module_end) || + (addr == 0 && info->module_start == UINTPTR_MAX)) + return (info); + } + return (NULL); +} - if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_end", &sym) == 0 || - LINKER_LOOKUP_SYMBOL(lf, "exidx_end", &sym) == 0) && - LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) - sc->exidx_end = symval.value; +/* + * Handle the loading of a new module by populating a module_info for it. This + * is called for both preloaded and dynamically loaded modules. + */ +void +unwind_module_loaded(struct linker_file *lf) +{ + struct module_info *info; - if (sc->exidx_start != NULL && sc->exidx_end != NULL) - return (1); - panic("Invalid module %s, no unwind tables\n", lf->filename); + /* + * A module that contains only data may have no unwind info; don't + * create any module info for it. + */ + if (lf->exidx_size == 0) + return; + + /* + * Find an unused entry in the existing list to reuse. If we don't find + * one, create a new one and link it into the list. This is the only + * place the module_list is modified. Adding a new entry to the list + * will not perturb any other threads currently walking the list. This + * function is invoked while kern_linker is still holding its lock + * to prevent its module list from being modified, so we don't have to + * worry about racing other threads doing an insert concurrently. + */ + if ((info = find_module_info(0)) == NULL) { + info = create_module_info(); } + populate_module_info(info, lf); +} + +/* Handle the unloading of a module. */ +void +unwind_module_unloaded(struct linker_file *lf) +{ + struct module_info *info; + + /* + * A module that contains only data may have no unwind info and there + * won't be a list entry for it. + */ + if (lf->exidx_size == 0) + return; + + /* + * When a module is unloaded, we clear the info out of its entry in the + * module list, making that entry available for later reuse. + */ + if ((info = find_module_info(UADDR(lf->address))) == NULL) { + printf("arm unwind: module '%s' not on list at unload time\n", + lf->filename); + return; + } + clear_module_info(info); +} + +/* + * Initialization must run fairly early, as soon as malloc(9) is available, and + * definitely before witness, which uses stack(9). We synthesize a module_info + * entry for the kernel, because unwind_module_loaded() doesn't get called for + * it. Also, it is unlike other modules in that the elf metadata for locating + * the unwind tables might be stripped, so instead we have to use the + * _exidx_start/end symbols created by ldscript.arm. + */ +static int +module_info_init(void *arg __unused) +{ + struct linker_file thekernel; + + STAILQ_INIT(&module_list); + + thekernel.filename = "kernel"; + thekernel.address = CADDR(&_start); + thekernel.size = UADDR(&_end) - UADDR(&_start); + thekernel.exidx_addr = CADDR(&_exidx_start); + thekernel.exidx_size = UADDR(&_exidx_end) - UADDR(&_exidx_start); + populate_module_info(create_module_info(), &thekernel); + return (0); } +SYSINIT(unwind_init, SI_SUB_KMEM, SI_ORDER_ANY, module_info_init, NULL); +/* Expand a 31-bit signed value to a 32-bit signed value */ +static __inline int32_t +expand_prel31(uint32_t prel31) +{ + + return ((int32_t)(prel31 & 0x7fffffffu) << 1) / 2; +} + /* * Perform a binary search of the index table to find the function * with the largest address that doesn't exceed addr. */ static struct unwind_idx * -find_index(uint32_t addr, int search_modules) +find_index(uint32_t addr) { - struct search_context sc; - caddr_t idx_start, idx_end; + struct module_info *info; unsigned int min, mid, max; struct unwind_idx *start; struct unwind_idx *item; int32_t prel31_addr; uint32_t func_addr; - start = (struct unwind_idx *)&_exidx_start; - idx_start = (caddr_t)&_exidx_start; - idx_end = (caddr_t)&_exidx_end; + info = find_module_info(addr); + if (info == NULL) + return NULL; - /* This may acquire a lock */ - if (search_modules) { - bzero(&sc, sizeof(sc)); - sc.addr = addr; - if (linker_file_foreach(module_search, &sc) != 0 && - sc.exidx_start != NULL && sc.exidx_end != NULL) { - start = (struct unwind_idx *)sc.exidx_start; - idx_start = sc.exidx_start; - idx_end = sc.exidx_end; - } - } - min = 0; - max = (idx_end - idx_start) / sizeof(struct unwind_idx); + max = (info->exidx_end - info->exidx_start) / sizeof(struct unwind_idx); + start = (struct unwind_idx *)CADDR(info->exidx_start); while (min != max) { mid = min + (max - min + 1) / 2; @@ -377,11 +539,17 @@ return 0; } +/* + * Unwind a single stack frame. + * Return 0 on success or 1 if the stack cannot be unwound any further. + * + * XXX The can_lock argument is no longer germane; a sweep of callers should be + * made to remove it after this new code has proven itself for a while. + */ int -unwind_stack_one(struct unwind_state *state, int can_lock) +unwind_stack_one(struct unwind_state *state, int can_lock __unused) { struct unwind_idx *index; - int finished; /* Reset the mask of updated registers */ state->update_mask = 0; @@ -390,26 +558,20 @@ state->start_pc = state->registers[PC]; /* Find the item to run */ - index = find_index(state->start_pc, can_lock); + index = find_index(state->start_pc); + if (index == NULL || index->insn == EXIDX_CANTUNWIND) + return 1; - finished = 0; - if (index->insn != EXIDX_CANTUNWIND) { - if (index->insn & (1U << 31)) { - /* The data is within the instruction */ - state->insn = &index->insn; - } else { - /* A prel31 offset to the unwind table */ - state->insn = (uint32_t *) - ((uintptr_t)&index->insn + - expand_prel31(index->insn)); - } - /* Run the unwind function */ - finished = unwind_tab(state); + if (index->insn & (1U << 31)) { + /* The data is within the instruction */ + state->insn = &index->insn; + } else { + /* A prel31 offset to the unwind table */ + state->insn = (uint32_t *) + ((uintptr_t)&index->insn + + expand_prel31(index->insn)); } - /* This is the top of the stack, finish */ - if (index->insn == EXIDX_CANTUNWIND) - finished = 1; - - return (finished); + /* Run the unwind function, return its finished/not-finished status. */ + return (unwind_tab(state)); } Index: sys/arm/include/stack.h =================================================================== --- sys/arm/include/stack.h +++ sys/arm/include/stack.h @@ -55,6 +55,14 @@ #define LR 14 #define PC 15 +#ifdef _KERNEL + int unwind_stack_one(struct unwind_state *, int); + +struct linker_file; +void unwind_module_loaded(struct linker_file *); +void unwind_module_unloaded(struct linker_file *); + +#endif #endif /* !_MACHINE_STACK_H_ */ Index: sys/kern/kern_linker.c =================================================================== --- sys/kern/kern_linker.c +++ sys/kern/kern_linker.c @@ -624,6 +624,10 @@ lf->ndeps = 0; lf->deps = NULL; lf->loadcnt = ++loadcnt; +#ifdef __arm__ + lf->exidx_addr = 0; + lf->exidx_size = 0; +#endif STAILQ_INIT(&lf->common); TAILQ_INIT(&lf->modules); TAILQ_INSERT_TAIL(&linker_files, lf, link); Index: sys/kern/link_elf.c =================================================================== --- sys/kern/link_elf.c +++ sys/kern/link_elf.c @@ -773,6 +773,50 @@ #endif } +#ifdef __arm__ +/* + * Locate the ARM exception/unwind table info for DDB and stack(9) use by + * searching for the section header that describes it. There may be no unwind + * info, for example in a module containing only data. + */ +static void +link_elf_locate_exidx(linker_file_t lf, Elf_Shdr *shdr, int nhdr) +{ + int i; + + for (i = 0; i < nhdr; i++) { + if (shdr[i].sh_type == SHT_ARM_EXIDX) { + lf->exidx_addr = shdr[i].sh_addr + lf->address; + lf->exidx_size = shdr[i].sh_size; + break; + } + } +} + +/* + * Locate the section headers metadata in a preloaded module, then use it to + * locate the exception/unwind table in the module. The size of the metadata + * block is stored in a uint32 word immediately before the data itself, and a + * comment in preload_search_info() says it is safe to rely on that. + */ +static void +link_elf_locate_exidx_preload(struct linker_file *lf, caddr_t modptr) +{ + uint32_t *modinfo; + Elf_Shdr *shdr; + uint32_t nhdr; + + modinfo = (uint32_t *)preload_search_info(modptr, + MODINFO_METADATA | MODINFOMD_SHDR); + if (modinfo != NULL) { + shdr = (Elf_Shdr *)modinfo; + nhdr = modinfo[-1] / sizeof(Elf_Shdr); + link_elf_locate_exidx(lf, shdr, nhdr); + } +} + +#endif /* __arm__ */ + static int link_elf_link_preload(linker_class_t cls, const char *filename, linker_file_t *result) @@ -828,6 +872,10 @@ lf->ctors_size = *ctors_sizep; } +#ifdef __arm__ + link_elf_locate_exidx_preload(lf, modptr); +#endif + error = parse_dynamic(ef); if (error == 0) error = parse_dpcpu(ef); @@ -1225,6 +1273,11 @@ ef->ddbstrtab = ef->strbase; nosyms: + +#ifdef __arm__ + link_elf_locate_exidx(lf, shdr, hdr->e_shnum); +#endif + error = link_elf_link_common_finish(lf); if (error != 0) goto out;