Index: sys/arm/arm/db_trace.c =================================================================== --- sys/arm/arm/db_trace.c +++ sys/arm/arm/db_trace.c @@ -68,7 +68,7 @@ finished = false; while (!finished) { - finished = unwind_stack_one(state, 1); + finished = unwind_stack_one(state); /* Print the frame details */ sym = db_search_symbol(state->start_pc, DB_STGY_ANY, &offset); Index: sys/arm/arm/unwind.c =================================================================== --- sys/arm/arm/unwind.c +++ sys/arm/arm/unwind.c @@ -32,13 +32,24 @@ __FBSDID("$FreeBSD$"); #include +#include #include +#include +#include #include +#include +#include +#include +#include #include +#include + #include "linker_if.h" +static MALLOC_DEFINE(M_ARMUNWIND, "unwind info", "unwind info for arm"); + /* * Definitions for the instruction interpreter. * @@ -107,69 +118,162 @@ return ((int32_t)(prel31 & 0x7fffffffu) << 1) / 2; } -struct search_context { - uint32_t addr; - caddr_t exidx_start; - caddr_t exidx_end; +struct module_idx { + SLIST_ENTRY(module_idx) module_next; + caddr_t address; + size_t size; + caddr_t start; + caddr_t end; }; -static int -module_search(linker_file_t lf, void *context) +static SLIST_HEAD(slisthead, module_idx) modules = + SLIST_HEAD_INITIALIZER(head); + +static struct mtx mtx_modules; + +static void +unwind_kld_load(void *arg __unused, linker_file_t lf) { - struct search_context *sc = context; + caddr_t exidx_start, exidx_end; + struct module_idx *m, *newentry; linker_symval_t symval; c_linker_sym_t sym; - if (lf->address <= (caddr_t)sc->addr && - (lf->address + lf->size) >= (caddr_t)sc->addr) { - if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_start", &sym) == 0 || - LINKER_LOOKUP_SYMBOL(lf, "exidx_start", &sym) == 0) && - LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) - sc->exidx_start = symval.value; - - if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_end", &sym) == 0 || - LINKER_LOOKUP_SYMBOL(lf, "exidx_end", &sym) == 0) && - LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) - sc->exidx_end = symval.value; - - if (sc->exidx_start != NULL && sc->exidx_end != NULL) - return (1); - panic("Invalid module %s, no unwind tables\n", lf->filename); + if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_start", &sym) == 0 || + LINKER_LOOKUP_SYMBOL(lf, "_exidx_start", &sym) == 0) && + LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) + exidx_start = symval.value; + else + return; + + if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_end", &sym) == 0 || + LINKER_LOOKUP_SYMBOL(lf, "_exidx_end", &sym) == 0) && + LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) + exidx_end = symval.value; + else + return; + + newentry = malloc(sizeof(*newentry), M_ARMUNWIND, M_WAITOK); + mtx_lock(&mtx_modules); + SLIST_FOREACH(m, &modules, module_next) { + if (m->address == 0 || m->address == lf->address) { + m->start = exidx_start; + m->end = exidx_end; + m->size = lf->size; + /* set address last */ + atomic_store_rel_ptr((uint32_t*)&m->address, (uint32_t)lf->address); + + mtx_unlock(&mtx_modules); + free(newentry, M_ARMUNWIND); + return; + } } + + newentry->start = exidx_start; + newentry->end = exidx_end; + newentry->size = lf->size; + newentry->address = lf->address; + /* + * in order to avoid race, we cannot use macro directly + * SLIST_INSERT_HEAD(&modules, newentry, module_next); + */ + SLIST_NEXT(newentry, module_next) = SLIST_FIRST(&modules); + atomic_thread_fence_acq_rel(); + SLIST_FIRST(&modules) = newentry; + + mtx_unlock(&mtx_modules); +} + +static void +unwind_kld_unload(void *arg, const char *filename, caddr_t address, size_t length) +{ + struct module_idx *m; + mtx_lock(&mtx_modules); + SLIST_FOREACH(m, &modules, module_next) { + if (m->address == address) { + m->address = 0; + mtx_unlock(&mtx_modules); + + return; + } + } + + mtx_unlock(&mtx_modules); +} + +static int +module_search(linker_file_t lf, void* arg) +{ + unwind_kld_load(arg, lf); + return (1); +} + +eventhandler_tag unwind_kld_load_tag; +eventhandler_tag unwind_kld_unload_tag; + +static int +unwind_initialize(void) +{ + mtx_init(&mtx_modules, "unwind module lock", NULL, MTX_DEF); + + linker_file_foreach(module_search, NULL); + + /* Register callbacks for linker file load and unload events. */ + unwind_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, + unwind_kld_load, NULL, EVENTHANDLER_PRI_ANY); + unwind_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, + unwind_kld_unload, NULL, EVENTHANDLER_PRI_ANY); + return (0); } +SYSINIT(unwind_init, SI_SUB_KLD, SI_ORDER_ANY, unwind_initialize, NULL); /* * Perform a binary search of the index table to find the function * with the largest address that doesn't exceed addr. */ static struct unwind_idx * -find_index(uint32_t addr, int search_modules) +find_index(uint32_t addr) { - struct search_context sc; caddr_t idx_start, idx_end; + caddr_t daddr, dstart, dend; unsigned int min, mid, max; - struct unwind_idx *start; + struct unwind_idx *start = NULL; struct unwind_idx *item; int32_t prel31_addr; uint32_t func_addr; - - start = (struct unwind_idx *)&exidx_start; - idx_start = (caddr_t)&exidx_start; - idx_end = (caddr_t)&exidx_end; - - /* This may acquire a lock */ - if (search_modules) { - bzero(&sc, sizeof(sc)); - sc.addr = addr; - if (linker_file_foreach(module_search, &sc) != 0 && - sc.exidx_start != NULL && sc.exidx_end != NULL) { - start = (struct unwind_idx *)sc.exidx_start; - idx_start = sc.exidx_start; - idx_end = sc.exidx_end; + struct module_idx *m; + size_t size; + + SLIST_FOREACH(m, &modules, module_next) { + do { + daddr = (caddr_t)atomic_load_acq_ptr((uint32_t *)&m->address); + if (daddr == 0) + break; + + size = m->size; + dstart = m->start; + dend = m->end; + + /* Ensure start, end, size have been loaded before address */ + atomic_thread_fence_acq(); + } while (daddr != m->address); + + if (daddr != 0 && daddr <= (caddr_t)addr && + (daddr + size) >= (caddr_t)addr) { + start = (struct unwind_idx *)dstart; + idx_start = dstart; + idx_end = dend; + break; } } + if (start == NULL) { + start = (struct unwind_idx *)&exidx_start; + idx_start = (caddr_t)&exidx_start; + idx_end = (caddr_t)&exidx_end; + } + min = 0; max = (idx_end - idx_start) / sizeof(struct unwind_idx); @@ -383,7 +487,7 @@ } int -unwind_stack_one(struct unwind_state *state, int can_lock) +unwind_stack_one(struct unwind_state *state) { struct unwind_idx *index; int finished; @@ -395,7 +499,7 @@ state->start_pc = state->registers[PC]; /* Find the item to run */ - index = find_index(state->start_pc, can_lock); + index = find_index(state->start_pc); finished = 0; if (index->insn != EXIDX_CANTUNWIND) { @@ -412,6 +516,11 @@ finished = unwind_tab(state); } + /* if pc or sp is invalid, finish unwind */ + if (!INKERNEL(state->registers[SP]) || + !INKERNEL(state->registers[PC])) + finished = 1; + /* This is the top of the stack, finish */ if (index->insn == EXIDX_CANTUNWIND) finished = 1; Index: sys/arm/include/atomic.h =================================================================== --- sys/arm/include/atomic.h +++ sys/arm/include/atomic.h @@ -89,6 +89,8 @@ #define atomic_cmpset_acq_ptr atomic_cmpset_acq_32 #define atomic_store_ptr atomic_store_32 #define atomic_store_rel_ptr atomic_store_rel_32 +#define atomic_load_acq_ptr atomic_load_acq_32 +#define atomic_store_rel_ptr atomic_store_rel_32 #define atomic_add_int atomic_add_32 #define atomic_add_acq_int atomic_add_acq_32 Index: sys/arm/include/stack.h =================================================================== --- sys/arm/include/stack.h +++ sys/arm/include/stack.h @@ -55,6 +55,6 @@ #define LR 14 #define PC 15 -int unwind_stack_one(struct unwind_state *, int); +int unwind_stack_one(struct unwind_state *); #endif /* !_MACHINE_STACK_H_ */ Index: sys/cddl/dev/dtrace/arm/dtrace_isa.c =================================================================== --- sys/cddl/dev/dtrace/arm/dtrace_isa.c +++ sys/cddl/dev/dtrace/arm/dtrace_isa.c @@ -89,7 +89,9 @@ while (depth < pcstack_limit) { int done; - done = unwind_stack_one(&state, 1); + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + done = unwind_stack_one(&state); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); /* * NB: Unlike some other architectures, we don't need to @@ -156,7 +158,9 @@ state.registers[PC] = (uint32_t)dtrace_getstackdepth; do { - done = unwind_stack_one(&state, 1); + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + done = unwind_stack_one(&state); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); depth++; } while (!done);