Changeset View
Changeset View
Standalone View
Standalone View
sys/arm/arm/unwind.c
Show All 26 Lines | |||||
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | ||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||||
*/ | */ | ||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/systm.h> | #include <sys/kernel.h> | ||||
#include <sys/linker.h> | #include <sys/linker.h> | ||||
#include <sys/malloc.h> | |||||
#include <sys/queue.h> | |||||
#include <sys/systm.h> | |||||
#include <machine/machdep.h> | #include <machine/machdep.h> | ||||
#include <machine/stack.h> | #include <machine/stack.h> | ||||
#include "linker_if.h" | #include "linker_if.h" | ||||
/* | /* | ||||
* Definitions for the instruction interpreter. | * Definitions for the instruction interpreter. | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
#define INSN_VSP_LARGE_INC 0xb2 | #define INSN_VSP_LARGE_INC 0xb2 | ||||
/* An item in the exception index table */ | /* An item in the exception index table */ | ||||
struct unwind_idx { | struct unwind_idx { | ||||
uint32_t offset; | uint32_t offset; | ||||
uint32_t insn; | uint32_t insn; | ||||
}; | }; | ||||
/* Expand a 31-bit signed value to a 32-bit signed value */ | /* | ||||
static __inline int32_t | * Local cache of unwind info for loaded modules. | ||||
expand_prel31(uint32_t prel31) | * | ||||
* To unwind the stack through the code in a loaded module, we need to access | |||||
* the module's exidx unwind data. To locate that data, one must search the | |||||
* elf section headers for the SHT_ARM_EXIDX section. Those headers are | |||||
* available at the time the module is being loaded, but are discarded by time | |||||
* the load process has completed. Code in kern/link_elf.c locates the data we | |||||
* need and stores it into the linker_file structure before calling the arm | |||||
* machdep routine for handling loaded modules (in arm/elf_machdep.c). That | |||||
* function calls into this code to pass along the unwind info, which we save | |||||
* into one of these module_info structures. | |||||
* | |||||
* Because we have to help stack(9) gather stack info at any time, including in | |||||
* contexts where sleeping is not allowed, we cannot use linker_file_foreach() | |||||
* to walk the kernel's list of linker_file structs, because doing so requires | |||||
* acquiring an exclusive sx_lock. So instead, we keep a local list of these | |||||
* structures, one for each loaded module (and one for the kernel itself that we | |||||
* synthesize at init time). New entries are added to the end of this list as | |||||
* needed, but entries are never deleted from the list. Instead, they are | |||||
* cleared out in-place to mark them as unused. That means the code doing stack | |||||
* unwinding can always safely walk the list without locking, because the | |||||
* structure of the list never changes in a way that would cause the walker to | |||||
* follow a bad link. | |||||
* | |||||
* A cleared-out entry on the list has module start=UINTPTR_MAX and end=0, so | |||||
* start <= addr < end cannot be true for any value of addr being searched for. | |||||
* We also don't have to worry about races where we look up the unwind info just | |||||
* before a module is unloaded and try to access it concurrently with or just | |||||
* after the unloading happens in another thread, because that means the path of | |||||
* execution leads through a now-unloaded module, and that's already well into | |||||
* undefined-behavior territory. | |||||
* | |||||
* List entries marked as unused get reused when new modules are loaded. We | |||||
* don't worry about holding a few unused bytes of memory in the list after | |||||
* unloading a module. Even if you loaded 1000 modules, did something that used | |||||
* them, then unloaded them all, we'd be wasting about 20K. In real-world | |||||
* scenarios the waste will be maybe a couple hundred bytes at most. | |||||
*/ | |||||
struct module_info { | |||||
uintptr_t module_start; /* Start of loaded module */ | |||||
uintptr_t module_end; /* End of loaded module */ | |||||
uintptr_t exidx_start; /* Start of unwind data */ | |||||
uintptr_t exidx_end; /* End of unwind data */ | |||||
STAILQ_ENTRY(module_info) | |||||
link; /* Link to next entry */ | |||||
}; | |||||
static STAILQ_HEAD(, module_info) module_list; | |||||
/* | |||||
* Hide ugly casting in somewhat-less-ugly macros. | |||||
* CADDR - cast a pointer or number to caddr_t. | |||||
* UADDR - cast a pointer or number to uintptr_t. | |||||
*/ | |||||
#define CADDR(addr) ((caddr_t)(void*)(uintptr_t)(addr)) | |||||
#define UADDR(addr) ((uintptr_t)(addr)) | |||||
/* | |||||
* Clear the info in an existing module_info entry on the list. The | |||||
* module_start/end addresses are set to values that cannot match any real | |||||
* memory address. The entry remains on the list, but will be ignored until it | |||||
* is populated with new data. | |||||
*/ | |||||
static void | |||||
clear_module_info(struct module_info *info) | |||||
{ | { | ||||
info->module_start = UINTPTR_MAX; | |||||
info->module_end = 0; | |||||
} | |||||
return ((int32_t)(prel31 & 0x7fffffffu) << 1) / 2; | /* | ||||
* Populate an existing module_info entry (which is already on the list) with | |||||
* the info for a new module. | |||||
*/ | |||||
static void | |||||
populate_module_info(struct module_info *info, linker_file_t lf) | |||||
{ | |||||
/* | |||||
* Careful! The module_start and module_end fields must not be set | |||||
* until all other data in the structure is valid. | |||||
*/ | |||||
info->exidx_start = UADDR(lf->exidx_addr); | |||||
info->exidx_end = UADDR(lf->exidx_addr) + lf->exidx_size; | |||||
info->module_start = UADDR(lf->address); | |||||
info->module_end = UADDR(lf->address) + lf->size; | |||||
} | } | ||||
struct search_context { | /* | ||||
uint32_t addr; | * Create a new empty module_info entry and add it to the tail of the list. | ||||
caddr_t exidx_start; | */ | ||||
caddr_t exidx_end; | static struct module_info * | ||||
}; | create_module_info(void) | ||||
{ | |||||
struct module_info *info; | |||||
static int | info = malloc(sizeof(*info), M_CACHE, M_WAITOK | M_ZERO); | ||||
module_search(linker_file_t lf, void *context) | clear_module_info(info); | ||||
STAILQ_INSERT_TAIL(&module_list, info, link); | |||||
return (info); | |||||
} | |||||
/* | |||||
* Search for a module_info entry on the list whose address range contains the | |||||
* given address. If the search address is zero (no module will be loaded at | |||||
* zero), then we're looking for an empty item to reuse, which is indicated by | |||||
* module_start being set to UINTPTR_MAX in the entry. | |||||
*/ | |||||
static struct module_info * | |||||
find_module_info(uintptr_t addr) | |||||
{ | { | ||||
struct search_context *sc = context; | struct module_info *info; | ||||
linker_symval_t symval; | |||||
c_linker_sym_t sym; | |||||
if (lf->address <= (caddr_t)sc->addr && | STAILQ_FOREACH(info, &module_list, link) { | ||||
(lf->address + lf->size) >= (caddr_t)sc->addr) { | if ((addr >= info->module_start && addr < info->module_end) || | ||||
if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_start", &sym) == 0 || | (addr == 0 && info->module_start == UINTPTR_MAX)) | ||||
LINKER_LOOKUP_SYMBOL(lf, "exidx_start", &sym) == 0) && | return (info); | ||||
LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) | } | ||||
sc->exidx_start = symval.value; | return (NULL); | ||||
} | |||||
if ((LINKER_LOOKUP_SYMBOL(lf, "__exidx_end", &sym) == 0 || | /* | ||||
LINKER_LOOKUP_SYMBOL(lf, "exidx_end", &sym) == 0) && | * Handle the loading of a new module by populating a module_info for it. This | ||||
LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) | * is called for both preloaded and dynamically loaded modules. | ||||
sc->exidx_end = symval.value; | */ | ||||
void | |||||
unwind_module_loaded(struct linker_file *lf) | |||||
{ | |||||
struct module_info *info; | |||||
if (sc->exidx_start != NULL && sc->exidx_end != NULL) | /* | ||||
return (1); | * A module that contains only data may have no unwind info; don't | ||||
panic("Invalid module %s, no unwind tables\n", lf->filename); | * create any module info for it. | ||||
*/ | |||||
if (lf->exidx_size == 0) | |||||
return; | |||||
/* | |||||
* Find an unused entry in the existing list to reuse. If we don't find | |||||
* one, create a new one and link it into the list. This is the only | |||||
* place the module_list is modified. Adding a new entry to the list | |||||
* will not perturb any other threads currently walking the list. This | |||||
* function is invoked while kern_linker is still holding its lock | |||||
* to prevent its module list from being modified, so we don't have to | |||||
* worry about racing other threads doing an insert concurrently. | |||||
*/ | |||||
if ((info = find_module_info(0)) == NULL) { | |||||
info = create_module_info(); | |||||
} | } | ||||
populate_module_info(info, lf); | |||||
} | |||||
/* Handle the unloading of a module. */ | |||||
void | |||||
unwind_module_unloaded(struct linker_file *lf) | |||||
{ | |||||
struct module_info *info; | |||||
/* | |||||
* A module that contains only data may have no unwind info and there | |||||
* won't be a list entry for it. | |||||
*/ | |||||
if (lf->exidx_size == 0) | |||||
return; | |||||
/* | |||||
* When a module is unloaded, we clear the info out of its entry in the | |||||
* module list, making that entry available for later reuse. | |||||
*/ | |||||
if ((info = find_module_info(UADDR(lf->address))) == NULL) { | |||||
printf("arm unwind: module '%s' not on list at unload time\n", | |||||
lf->filename); | |||||
return; | |||||
} | |||||
clear_module_info(info); | |||||
} | |||||
/* | |||||
* Initialization must run fairly early, as soon as malloc(9) is available, and | |||||
* definitely before witness, which uses stack(9). We synthesize a module_info | |||||
* entry for the kernel, because unwind_module_loaded() doesn't get called for | |||||
* it. Also, it is unlike other modules in that the elf metadata for locating | |||||
* the unwind tables might be stripped, so instead we have to use the | |||||
* _exidx_start/end symbols created by ldscript.arm. | |||||
*/ | |||||
static int | |||||
module_info_init(void *arg __unused) | |||||
{ | |||||
struct linker_file thekernel; | |||||
STAILQ_INIT(&module_list); | |||||
thekernel.filename = "kernel"; | |||||
thekernel.address = CADDR(&_start); | |||||
thekernel.size = UADDR(&_end) - UADDR(&_start); | |||||
thekernel.exidx_addr = CADDR(&_exidx_start); | |||||
thekernel.exidx_size = UADDR(&_exidx_end) - UADDR(&_exidx_start); | |||||
populate_module_info(create_module_info(), &thekernel); | |||||
return (0); | return (0); | ||||
} | } | ||||
SYSINIT(unwind_init, SI_SUB_KMEM, SI_ORDER_ANY, module_info_init, NULL); | |||||
/* Expand a 31-bit signed value to a 32-bit signed value */ | |||||
static __inline int32_t | |||||
expand_prel31(uint32_t prel31) | |||||
{ | |||||
return ((int32_t)(prel31 & 0x7fffffffu) << 1) / 2; | |||||
} | |||||
/* | /* | ||||
* Perform a binary search of the index table to find the function | * Perform a binary search of the index table to find the function | ||||
* with the largest address that doesn't exceed addr. | * with the largest address that doesn't exceed addr. | ||||
*/ | */ | ||||
static struct unwind_idx * | static struct unwind_idx * | ||||
find_index(uint32_t addr, int search_modules) | find_index(uint32_t addr) | ||||
{ | { | ||||
struct search_context sc; | struct module_info *info; | ||||
caddr_t idx_start, idx_end; | |||||
unsigned int min, mid, max; | unsigned int min, mid, max; | ||||
struct unwind_idx *start; | struct unwind_idx *start; | ||||
struct unwind_idx *item; | struct unwind_idx *item; | ||||
int32_t prel31_addr; | int32_t prel31_addr; | ||||
uint32_t func_addr; | uint32_t func_addr; | ||||
start = (struct unwind_idx *)&_exidx_start; | info = find_module_info(addr); | ||||
idx_start = (caddr_t)&_exidx_start; | if (info == NULL) | ||||
idx_end = (caddr_t)&_exidx_end; | return NULL; | ||||
/* This may acquire a lock */ | |||||
if (search_modules) { | |||||
bzero(&sc, sizeof(sc)); | |||||
sc.addr = addr; | |||||
if (linker_file_foreach(module_search, &sc) != 0 && | |||||
sc.exidx_start != NULL && sc.exidx_end != NULL) { | |||||
start = (struct unwind_idx *)sc.exidx_start; | |||||
idx_start = sc.exidx_start; | |||||
idx_end = sc.exidx_end; | |||||
} | |||||
} | |||||
min = 0; | min = 0; | ||||
max = (idx_end - idx_start) / sizeof(struct unwind_idx); | max = (info->exidx_end - info->exidx_start) / sizeof(struct unwind_idx); | ||||
start = (struct unwind_idx *)CADDR(info->exidx_start); | |||||
while (min != max) { | while (min != max) { | ||||
mid = min + (max - min + 1) / 2; | mid = min + (max - min + 1) / 2; | ||||
item = &start[mid]; | item = &start[mid]; | ||||
prel31_addr = expand_prel31(item->offset); | prel31_addr = expand_prel31(item->offset); | ||||
func_addr = (uint32_t)&item->offset + prel31_addr; | func_addr = (uint32_t)&item->offset + prel31_addr; | ||||
▲ Show 20 Lines • Show All 194 Lines • ▼ Show 20 Lines | if (state->registers[PC] == 0) { | ||||
*/ | */ | ||||
if (state->start_pc != state->registers[PC]) | if (state->start_pc != state->registers[PC]) | ||||
state->update_mask |= 1 << PC; | state->update_mask |= 1 << PC; | ||||
} | } | ||||
return 0; | return 0; | ||||
} | } | ||||
/* | |||||
* Unwind a single stack frame. | |||||
* Return 0 on success or 1 if the stack cannot be unwound any further. | |||||
* | |||||
* XXX The can_lock argument is no longer germane; a sweep of callers should be | |||||
* made to remove it after this new code has proven itself for a while. | |||||
*/ | |||||
int | int | ||||
unwind_stack_one(struct unwind_state *state, int can_lock) | unwind_stack_one(struct unwind_state *state, int can_lock __unused) | ||||
{ | { | ||||
struct unwind_idx *index; | struct unwind_idx *index; | ||||
int finished; | |||||
/* Reset the mask of updated registers */ | /* Reset the mask of updated registers */ | ||||
state->update_mask = 0; | state->update_mask = 0; | ||||
/* The pc value is correct and will be overwritten, save it */ | /* The pc value is correct and will be overwritten, save it */ | ||||
state->start_pc = state->registers[PC]; | state->start_pc = state->registers[PC]; | ||||
/* Find the item to run */ | /* Find the item to run */ | ||||
index = find_index(state->start_pc, can_lock); | index = find_index(state->start_pc); | ||||
if (index == NULL || index->insn == EXIDX_CANTUNWIND) | |||||
return 1; | |||||
finished = 0; | |||||
if (index->insn != EXIDX_CANTUNWIND) { | |||||
if (index->insn & (1U << 31)) { | if (index->insn & (1U << 31)) { | ||||
/* The data is within the instruction */ | /* The data is within the instruction */ | ||||
state->insn = &index->insn; | state->insn = &index->insn; | ||||
} else { | } else { | ||||
/* A prel31 offset to the unwind table */ | /* A prel31 offset to the unwind table */ | ||||
state->insn = (uint32_t *) | state->insn = (uint32_t *) | ||||
((uintptr_t)&index->insn + | ((uintptr_t)&index->insn + | ||||
expand_prel31(index->insn)); | expand_prel31(index->insn)); | ||||
} | } | ||||
/* Run the unwind function */ | |||||
finished = unwind_tab(state); | |||||
} | |||||
/* This is the top of the stack, finish */ | /* Run the unwind function, return its finished/not-finished status. */ | ||||
if (index->insn == EXIDX_CANTUNWIND) | return (unwind_tab(state)); | ||||
finished = 1; | |||||
return (finished); | |||||
} | } |