diff --git a/sys/arm64/arm64/db_interface.c b/sys/arm64/arm64/db_interface.c index 5138bf3f1cab..289886658905 100644 --- a/sys/arm64/arm64/db_interface.c +++ b/sys/arm64/arm64/db_interface.c @@ -1,194 +1,179 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #ifdef KDB #include #endif #include #include #include #include #include #include static int db_frame(struct db_variable *vp, db_expr_t *valuep, int op) { long *reg; if (kdb_frame == NULL) return (0); reg = (long *)((uintptr_t)kdb_frame + (db_expr_t)vp->valuep); if (op == DB_VAR_GET) *valuep = *reg; else *reg = *valuep; return (1); } #define DB_OFFSET(x) (db_expr_t *)offsetof(struct trapframe, x) struct db_variable db_regs[] = { { "spsr", DB_OFFSET(tf_spsr), db_frame }, { "x0", DB_OFFSET(tf_x[0]), db_frame }, { "x1", DB_OFFSET(tf_x[1]), db_frame }, { "x2", DB_OFFSET(tf_x[2]), db_frame }, { "x3", DB_OFFSET(tf_x[3]), db_frame }, { "x4", DB_OFFSET(tf_x[4]), db_frame }, { "x5", DB_OFFSET(tf_x[5]), db_frame }, { "x6", DB_OFFSET(tf_x[6]), db_frame }, { "x7", DB_OFFSET(tf_x[7]), db_frame }, { "x8", DB_OFFSET(tf_x[8]), db_frame }, { "x9", DB_OFFSET(tf_x[9]), db_frame }, { "x10", DB_OFFSET(tf_x[10]), db_frame }, { "x11", DB_OFFSET(tf_x[11]), db_frame }, { "x12", DB_OFFSET(tf_x[12]), db_frame }, { "x13", DB_OFFSET(tf_x[13]), db_frame }, { "x14", DB_OFFSET(tf_x[14]), db_frame }, { "x15", DB_OFFSET(tf_x[15]), db_frame }, { "x16", DB_OFFSET(tf_x[16]), db_frame }, { "x17", DB_OFFSET(tf_x[17]), db_frame }, { "x18", DB_OFFSET(tf_x[18]), db_frame }, { "x19", DB_OFFSET(tf_x[19]), db_frame }, { "x20", DB_OFFSET(tf_x[20]), db_frame }, { "x21", DB_OFFSET(tf_x[21]), db_frame }, { "x22", DB_OFFSET(tf_x[22]), db_frame }, { "x23", DB_OFFSET(tf_x[23]), db_frame }, { "x24", DB_OFFSET(tf_x[24]), db_frame }, { "x25", DB_OFFSET(tf_x[25]), db_frame }, { "x26", DB_OFFSET(tf_x[26]), db_frame }, { "x27", DB_OFFSET(tf_x[27]), db_frame }, { "x28", DB_OFFSET(tf_x[28]), db_frame }, { "x29", DB_OFFSET(tf_x[29]), db_frame }, { "lr", DB_OFFSET(tf_lr), db_frame }, { "elr", DB_OFFSET(tf_elr), db_frame }, { "sp", DB_OFFSET(tf_sp), db_frame }, }; struct db_variable *db_eregs = db_regs + nitems(db_regs); void db_show_mdpcpu(struct pcpu *pc) { } /* * Read bytes from kernel address space for debugger. */ int db_read_bytes(vm_offset_t addr, size_t size, char *data) { jmp_buf jb; void *prev_jb; const char *src; int ret; uint64_t tmp64; uint32_t tmp32; uint16_t tmp16; prev_jb = kdb_jmpbuf(jb); ret = setjmp(jb); if (ret == 0) { src = (const char *)addr; if (size == 8 && (addr & 7) == 0) { tmp64 = *((const int *)src); src = (const char *)&tmp64; } else if (size == 4 && (addr & 3) == 0) { tmp32 = *((const int *)src); src = (const char *)&tmp32; } else if (size == 2 && (addr & 1) == 0) { tmp16 = *((const short *)src); src = (const char *)&tmp16; } while (size-- > 0) *data++ = *src++; } (void)kdb_jmpbuf(prev_jb); return (ret); } /* * Write bytes to kernel address space for debugger. */ int db_write_bytes(vm_offset_t addr, size_t size, char *data) { jmp_buf jb; void *prev_jb; char *dst; + size_t i; int ret; - uint64_t tmp64; - uint32_t tmp32; - uint16_t tmp16; prev_jb = kdb_jmpbuf(jb); ret = setjmp(jb); if (ret == 0) { - if (size == 8 && (addr & 7) == 0) { - dst = (char *)&tmp64; - while (size-- > 0) - *dst++ = *data++; - *((uint64_t *)addr) = tmp64; - } else if (size == 4 && (addr & 3) == 0) { - dst = (char *)&tmp32; - while (size-- > 0) - *dst++ = *data++; - *((uint32_t *)addr) = tmp32; - } else if (size == 2 && (addr & 1) == 0) { - dst = (char *)&tmp16; - while (size-- > 0) - *dst++ = *data++; - *((uint32_t *)addr) = tmp16; + if (!arm64_get_writable_addr(addr, &addr)) { + ret = 1; } else { dst = (char *)addr; - while (size-- > 0) + for (i = 0; i < size; i++) *dst++ = *data++; - } - dsb(ish); + dsb(ish); - /* Clean D-cache and invalidate I-cache */ - cpu_dcache_wb_range(addr, (vm_size_t)size); - cpu_icache_sync_range(addr, (vm_size_t)size); + /* Clean D-cache and invalidate I-cache */ + cpu_dcache_wb_range(addr, (vm_size_t)size); + cpu_icache_sync_range(addr, (vm_size_t)size); + } } (void)kdb_jmpbuf(prev_jb); return (ret); } diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c index 67f6f66241f8..9ade7dde99fa 100644 --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -1,956 +1,992 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include "opt_acpi.h" #include "opt_platform.h" #include "opt_ddb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef VFP #include #endif #ifdef DEV_ACPI #include #include #endif #ifdef FDT #include #include #endif enum arm64_bus arm64_bus_method = ARM64_BUS_NONE; /* * XXX: The .bss is assumed to be in the boot CPU NUMA domain. If not we * could relocate this, but will need to keep the same virtual address as * it's reverenced by the EARLY_COUNTER macro. */ struct pcpu pcpu0; #if defined(PERTHREAD_SSP) /* * The boot SSP canary. Will be replaced with a per-thread canary when * scheduling has started. */ uintptr_t boot_canary = 0x49a2d892bc05a0b1ul; #endif static struct trapframe proc0_tf; int early_boot = 1; int cold = 1; static int boot_el; struct kva_md_info kmi; int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ int has_pan; /* * Physical address of the EFI System Table. Stashed from the metadata hints * passed into the kernel and used by the EFI code to call runtime services. */ vm_paddr_t efi_systbl_phys; static struct efi_map_header *efihdr; /* pagezero_* implementations are provided in support.S */ void pagezero_simple(void *); void pagezero_cache(void *); /* pagezero_simple is default pagezero */ void (*pagezero)(void *p) = pagezero_simple; int (*apei_nmi)(void); #if defined(PERTHREAD_SSP_WARNING) static void print_ssp_warning(void *data __unused) { printf("WARNING: Per-thread SSP is enabled but the compiler is too old to support it\n"); } SYSINIT(ssp_warn, SI_SUB_COPYRIGHT, SI_ORDER_ANY, print_ssp_warning, NULL); SYSINIT(ssp_warn2, SI_SUB_LAST, SI_ORDER_ANY, print_ssp_warning, NULL); #endif static void pan_setup(void) { uint64_t id_aa64mfr1; id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); if (ID_AA64MMFR1_PAN_VAL(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE) has_pan = 1; } void pan_enable(void) { /* * The LLVM integrated assembler doesn't understand the PAN * PSTATE field. Because of this we need to manually create * the instruction in an asm block. This is equivalent to: * msr pan, #1 * * This sets the PAN bit, stopping the kernel from accessing * memory when userspace can also access it unless the kernel * uses the userspace load/store instructions. */ if (has_pan) { WRITE_SPECIALREG(sctlr_el1, READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN); __asm __volatile(".inst 0xd500409f | (0x1 << 8)"); } } bool has_hyp(void) { return (boot_el == 2); } static void cpu_startup(void *dummy) { vm_paddr_t size; int i; printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)realmem), ptoa((uintmax_t)realmem) / 1024 / 1024); if (bootverbose) { printf("Physical memory chunk(s):\n"); for (i = 0; phys_avail[i + 1] != 0; i += 2) { size = phys_avail[i + 1] - phys_avail[i]; printf("%#016jx - %#016jx, %ju bytes (%ju pages)\n", (uintmax_t)phys_avail[i], (uintmax_t)phys_avail[i + 1] - 1, (uintmax_t)size, (uintmax_t)size / PAGE_SIZE); } } printf("avail memory = %ju (%ju MB)\n", ptoa((uintmax_t)vm_free_count()), ptoa((uintmax_t)vm_free_count()) / 1024 / 1024); undef_init(); install_cpu_errata(); vm_ksubmap_init(&kmi); bufinit(); vm_pager_bufferinit(); } SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); static void late_ifunc_resolve(void *dummy __unused) { link_elf_late_ireloc(); } SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL); int cpu_idle_wakeup(int cpu) { return (0); } void cpu_idle(int busy) { spinlock_enter(); if (!busy) cpu_idleclock(); if (!sched_runnable()) __asm __volatile( "dsb sy \n" "wfi \n"); if (!busy) cpu_activeclock(); spinlock_exit(); } void cpu_halt(void) { /* We should have shutdown by now, if not enter a low power sleep */ intr_disable(); while (1) { __asm __volatile("wfi"); } } /* * Flush the D-cache for non-DMA I/O so that the I-cache can * be made coherent later. */ void cpu_flush_dcache(void *ptr, size_t len) { /* ARM64TODO TBD */ } /* Get current clock frequency for the given CPU ID. */ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { struct pcpu *pc; pc = pcpu_find(cpu_id); if (pc == NULL || rate == NULL) return (EINVAL); if (pc->pc_clock == 0) return (EOPNOTSUPP); *rate = pc->pc_clock; return (0); } void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) { pcpu->pc_acpi_id = 0xffffffff; pcpu->pc_mpidr = 0xffffffff; } void spinlock_enter(void) { struct thread *td; register_t daif; td = curthread; if (td->td_md.md_spinlock_count == 0) { daif = intr_disable(); td->td_md.md_spinlock_count = 1; td->td_md.md_saved_daif = daif; critical_enter(); } else td->td_md.md_spinlock_count++; } void spinlock_exit(void) { struct thread *td; register_t daif; td = curthread; daif = td->td_md.md_saved_daif; td->td_md.md_spinlock_count--; if (td->td_md.md_spinlock_count == 0) { critical_exit(); intr_restore(daif); } } /* * Construct a PCB from a trapframe. This is called from kdb_trap() where * we want to start a backtrace from the function that caused us to enter * the debugger. We have the context in the trapframe, but base the trace * on the PCB. The PCB doesn't have to be perfect, as long as it contains * enough for a backtrace. */ void makectx(struct trapframe *tf, struct pcb *pcb) { int i; for (i = 0; i < nitems(pcb->pcb_x); i++) pcb->pcb_x[i] = tf->tf_x[i]; /* NB: pcb_lr is the PC, see PC_REGS() in db_machdep.h */ pcb->pcb_lr = tf->tf_elr; pcb->pcb_sp = tf->tf_sp; } static void init_proc0(vm_offset_t kstack) { struct pcpu *pcpup; pcpup = cpuid_to_pcpu[0]; MPASS(pcpup != NULL); proc_linkup0(&proc0, &thread0); thread0.td_kstack = kstack; thread0.td_kstack_pages = KSTACK_PAGES; #if defined(PERTHREAD_SSP) thread0.td_md.md_canary = boot_canary; #endif thread0.td_pcb = (struct pcb *)(thread0.td_kstack + thread0.td_kstack_pages * PAGE_SIZE) - 1; thread0.td_pcb->pcb_fpflags = 0; thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate; thread0.td_pcb->pcb_vfpcpu = UINT_MAX; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; /* * Unmask SError exceptions. They are used to signal a RAS failure, * or other hardware error. */ serror_enable(); } +/* + * Get an address to be used to write to kernel data that may be mapped + * read-only, e.g. to patch kernel code. + */ +bool +arm64_get_writable_addr(vm_offset_t addr, vm_offset_t *out) +{ + vm_paddr_t pa; + + /* Check if the page is writable */ + if (PAR_SUCCESS(arm64_address_translate_s1e1w(addr))) { + *out = addr; + return (true); + } + + /* + * Find the physical address of the given page. + */ + if (!pmap_klookup(addr, &pa)) { + return (false); + } + + /* + * If it is within the DMAP region and is writable use that. + */ + if (PHYS_IN_DMAP(pa)) { + addr = PHYS_TO_DMAP(pa); + if (PAR_SUCCESS(arm64_address_translate_s1e1w(addr))) { + *out = addr; + return (true); + } + } + + return (false); +} + typedef struct { uint32_t type; uint64_t phys_start; uint64_t virt_start; uint64_t num_pages; uint64_t attr; } EFI_MEMORY_DESCRIPTOR; typedef void (*efi_map_entry_cb)(struct efi_md *); static void foreach_efi_map_entry(struct efi_map_header *efihdr, efi_map_entry_cb cb) { struct efi_md *map, *p; size_t efisz; int ndesc, i; /* * Memory map data provided by UEFI via the GetMemoryMap * Boot Services API. */ efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; map = (struct efi_md *)((uint8_t *)efihdr + efisz); if (efihdr->descriptor_size == 0) return; ndesc = efihdr->memory_size / efihdr->descriptor_size; for (i = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p, efihdr->descriptor_size)) { cb(p); } } static void exclude_efi_map_entry(struct efi_md *p) { switch (p->md_type) { case EFI_MD_TYPE_CODE: case EFI_MD_TYPE_DATA: case EFI_MD_TYPE_BS_CODE: case EFI_MD_TYPE_BS_DATA: case EFI_MD_TYPE_FREE: /* * We're allowed to use any entry with these types. */ break; default: physmem_exclude_region(p->md_phys, p->md_pages * PAGE_SIZE, EXFLAG_NOALLOC); } } static void exclude_efi_map_entries(struct efi_map_header *efihdr) { foreach_efi_map_entry(efihdr, exclude_efi_map_entry); } static void add_efi_map_entry(struct efi_md *p) { switch (p->md_type) { case EFI_MD_TYPE_RT_DATA: /* * Runtime data will be excluded after the DMAP * region is created to stop it from being added * to phys_avail. */ case EFI_MD_TYPE_CODE: case EFI_MD_TYPE_DATA: case EFI_MD_TYPE_BS_CODE: case EFI_MD_TYPE_BS_DATA: case EFI_MD_TYPE_FREE: /* * We're allowed to use any entry with these types. */ physmem_hardware_region(p->md_phys, p->md_pages * PAGE_SIZE); break; } } static void add_efi_map_entries(struct efi_map_header *efihdr) { foreach_efi_map_entry(efihdr, add_efi_map_entry); } static void print_efi_map_entry(struct efi_md *p) { const char *type; static const char *types[] = { "Reserved", "LoaderCode", "LoaderData", "BootServicesCode", "BootServicesData", "RuntimeServicesCode", "RuntimeServicesData", "ConventionalMemory", "UnusableMemory", "ACPIReclaimMemory", "ACPIMemoryNVS", "MemoryMappedIO", "MemoryMappedIOPortSpace", "PalCode", "PersistentMemory" }; if (p->md_type < nitems(types)) type = types[p->md_type]; else type = ""; printf("%23s %012lx %12p %08lx ", type, p->md_phys, p->md_virt, p->md_pages); if (p->md_attr & EFI_MD_ATTR_UC) printf("UC "); if (p->md_attr & EFI_MD_ATTR_WC) printf("WC "); if (p->md_attr & EFI_MD_ATTR_WT) printf("WT "); if (p->md_attr & EFI_MD_ATTR_WB) printf("WB "); if (p->md_attr & EFI_MD_ATTR_UCE) printf("UCE "); if (p->md_attr & EFI_MD_ATTR_WP) printf("WP "); if (p->md_attr & EFI_MD_ATTR_RP) printf("RP "); if (p->md_attr & EFI_MD_ATTR_XP) printf("XP "); if (p->md_attr & EFI_MD_ATTR_NV) printf("NV "); if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE) printf("MORE_RELIABLE "); if (p->md_attr & EFI_MD_ATTR_RO) printf("RO "); if (p->md_attr & EFI_MD_ATTR_RT) printf("RUNTIME"); printf("\n"); } static void print_efi_map_entries(struct efi_map_header *efihdr) { printf("%23s %12s %12s %8s %4s\n", "Type", "Physical", "Virtual", "#Pages", "Attr"); foreach_efi_map_entry(efihdr, print_efi_map_entry); } #ifdef FDT static void try_load_dtb(caddr_t kmdp) { vm_offset_t dtbp; dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); #if defined(FDT_DTB_STATIC) /* * In case the device tree blob was not retrieved (from metadata) try * to use the statically embedded one. */ if (dtbp == 0) dtbp = (vm_offset_t)&fdt_static_dtb; #endif if (dtbp == (vm_offset_t)NULL) { #ifndef TSLOG printf("ERROR loading DTB\n"); #endif return; } if (OF_install(OFW_FDT, 0) == FALSE) panic("Cannot install FDT"); if (OF_init((void *)dtbp) != 0) panic("OF_init failed with the found device tree"); parse_fdt_bootargs(); } #endif static bool bus_probe(void) { bool has_acpi, has_fdt; char *order, *env; has_acpi = has_fdt = false; #ifdef FDT has_fdt = (OF_peer(0) != 0); #endif #ifdef DEV_ACPI has_acpi = (AcpiOsGetRootPointer() != 0); #endif env = kern_getenv("kern.cfg.order"); if (env != NULL) { order = env; while (order != NULL) { if (has_acpi && strncmp(order, "acpi", 4) == 0 && (order[4] == ',' || order[4] == '\0')) { arm64_bus_method = ARM64_BUS_ACPI; break; } if (has_fdt && strncmp(order, "fdt", 3) == 0 && (order[3] == ',' || order[3] == '\0')) { arm64_bus_method = ARM64_BUS_FDT; break; } order = strchr(order, ','); } freeenv(env); /* If we set the bus method it is valid */ if (arm64_bus_method != ARM64_BUS_NONE) return (true); } /* If no order or an invalid order was set use the default */ if (arm64_bus_method == ARM64_BUS_NONE) { if (has_fdt) arm64_bus_method = ARM64_BUS_FDT; else if (has_acpi) arm64_bus_method = ARM64_BUS_ACPI; } /* * If no option was set the default is valid, otherwise we are * setting one to get cninit() working, then calling panic to tell * the user about the invalid bus setup. */ return (env == NULL); } static void cache_setup(void) { int dczva_line_shift; uint32_t dczid_el0; identify_cache(READ_SPECIALREG(ctr_el0)); dczid_el0 = READ_SPECIALREG(dczid_el0); /* Check if dc zva is not prohibited */ if (dczid_el0 & DCZID_DZP) dczva_line_size = 0; else { /* Same as with above calculations */ dczva_line_shift = DCZID_BS_SIZE(dczid_el0); dczva_line_size = sizeof(int) << dczva_line_shift; /* Change pagezero function */ pagezero = pagezero_cache; } } int memory_mapping_mode(vm_paddr_t pa) { struct efi_md *map, *p; size_t efisz; int ndesc, i; if (efihdr == NULL) return (VM_MEMATTR_WRITE_BACK); /* * Memory map data provided by UEFI via the GetMemoryMap * Boot Services API. */ efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; map = (struct efi_md *)((uint8_t *)efihdr + efisz); if (efihdr->descriptor_size == 0) return (VM_MEMATTR_WRITE_BACK); ndesc = efihdr->memory_size / efihdr->descriptor_size; for (i = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p, efihdr->descriptor_size)) { if (pa < p->md_phys || pa >= p->md_phys + p->md_pages * EFI_PAGE_SIZE) continue; if (p->md_type == EFI_MD_TYPE_IOMEM || p->md_type == EFI_MD_TYPE_IOPORT) return (VM_MEMATTR_DEVICE); else if ((p->md_attr & EFI_MD_ATTR_WB) != 0 || p->md_type == EFI_MD_TYPE_RECLAIM) return (VM_MEMATTR_WRITE_BACK); else if ((p->md_attr & EFI_MD_ATTR_WT) != 0) return (VM_MEMATTR_WRITE_THROUGH); else if ((p->md_attr & EFI_MD_ATTR_WC) != 0) return (VM_MEMATTR_WRITE_COMBINING); break; } return (VM_MEMATTR_DEVICE); } void initarm(struct arm64_bootparams *abp) { struct efi_fb *efifb; struct pcpu *pcpup; char *env; #ifdef FDT struct mem_region mem_regions[FDT_MEM_REGIONS]; int mem_regions_sz; phandle_t root; char dts_version[255]; #endif vm_offset_t lastaddr; caddr_t kmdp; bool valid; TSRAW(&thread0, TS_ENTER, __func__, NULL); boot_el = abp->boot_el; /* Parse loader or FDT boot parametes. Determine last used address. */ lastaddr = parse_boot_param(abp); /* Find the kernel address */ kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); identify_cpu(0); update_special_regs(0); link_elf_ireloc(kmdp); try_load_dtb(kmdp); efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t); /* Load the physical memory ranges */ efihdr = (struct efi_map_header *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_MAP); if (efihdr != NULL) add_efi_map_entries(efihdr); #ifdef FDT else { /* Grab physical memory regions information from device tree. */ if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, NULL) != 0) panic("Cannot get physical memory regions"); physmem_hardware_regions(mem_regions, mem_regions_sz); } if (fdt_get_reserved_mem(mem_regions, &mem_regions_sz) == 0) physmem_exclude_regions(mem_regions, mem_regions_sz, EXFLAG_NODUMP | EXFLAG_NOALLOC); #endif /* Exclude the EFI framebuffer from our view of physical memory. */ efifb = (struct efi_fb *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_FB); if (efifb != NULL) physmem_exclude_region(efifb->fb_addr, efifb->fb_size, EXFLAG_NOALLOC); /* Set the pcpu data, this is needed by pmap_bootstrap */ pcpup = &pcpu0; pcpu_init(pcpup, 0, sizeof(struct pcpu)); /* * Set the pcpu pointer with a backup in tpidr_el1 to be * loaded when entering the kernel from userland. */ __asm __volatile( "mov x18, %0 \n" "msr tpidr_el1, %0" :: "r"(pcpup)); /* locore.S sets sp_el0 to &thread0 so no need to set it here. */ PCPU_SET(curthread, &thread0); PCPU_SET(midr, get_midr()); /* Do basic tuning, hz etc */ init_param1(); cache_setup(); pan_setup(); /* Bootstrap enough of pmap to enter the kernel proper */ pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt, KERNBASE - abp->kern_delta, lastaddr - KERNBASE); /* Exclude entries neexed in teh DMAP region, but not phys_avail */ if (efihdr != NULL) exclude_efi_map_entries(efihdr); physmem_init_kernel_globals(); devmap_bootstrap(0, NULL); valid = bus_probe(); cninit(); set_ttbr0(abp->kern_ttbr0); cpu_tlb_flushID(); if (!valid) panic("Invalid bus configuration: %s", kern_getenv("kern.cfg.order")); /* * Dump the boot metadata. We have to wait for cninit() since console * output is required. If it's grossly incorrect the kernel will never * make it this far. */ if (getenv_is_true("debug.dump_modinfo_at_boot")) preload_dump(); init_proc0(abp->kern_stack); msgbufinit(msgbufp, msgbufsize); mutex_init(); init_param2(physmem); dbg_init(); kdb_init(); pan_enable(); kcsan_cpu_init(0); env = kern_getenv("kernelname"); if (env != NULL) strlcpy(kernelname, env, sizeof(kernelname)); #ifdef FDT if (arm64_bus_method == ARM64_BUS_FDT) { root = OF_finddevice("/"); if (OF_getprop(root, "freebsd,dts-version", dts_version, sizeof(dts_version)) > 0) { if (strcmp(LINUX_DTS_VERSION, dts_version) != 0) printf("WARNING: DTB version is %s while kernel expects %s, " "please update the DTB in the ESP\n", dts_version, LINUX_DTS_VERSION); } else { printf("WARNING: Cannot find freebsd,dts-version property, " "cannot check DTB compliance\n"); } } #endif if (boothowto & RB_VERBOSE) { if (efihdr != NULL) print_efi_map_entries(efihdr); physmem_print_tables(); } early_boot = 0; TSEXIT(); } void dbg_init(void) { /* Clear OS lock */ WRITE_SPECIALREG(oslar_el1, 0); /* This permits DDB to use debug registers for watchpoints. */ dbg_monitor_init(); /* TODO: Eventually will need to initialize debug registers here. */ } #ifdef DDB #include DB_SHOW_COMMAND(specialregs, db_show_spregs) { #define PRINT_REG(reg) \ db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg)) PRINT_REG(actlr_el1); PRINT_REG(afsr0_el1); PRINT_REG(afsr1_el1); PRINT_REG(aidr_el1); PRINT_REG(amair_el1); PRINT_REG(ccsidr_el1); PRINT_REG(clidr_el1); PRINT_REG(contextidr_el1); PRINT_REG(cpacr_el1); PRINT_REG(csselr_el1); PRINT_REG(ctr_el0); PRINT_REG(currentel); PRINT_REG(daif); PRINT_REG(dczid_el0); PRINT_REG(elr_el1); PRINT_REG(esr_el1); PRINT_REG(far_el1); #if 0 /* ARM64TODO: Enable VFP before reading floating-point registers */ PRINT_REG(fpcr); PRINT_REG(fpsr); #endif PRINT_REG(id_aa64afr0_el1); PRINT_REG(id_aa64afr1_el1); PRINT_REG(id_aa64dfr0_el1); PRINT_REG(id_aa64dfr1_el1); PRINT_REG(id_aa64isar0_el1); PRINT_REG(id_aa64isar1_el1); PRINT_REG(id_aa64pfr0_el1); PRINT_REG(id_aa64pfr1_el1); PRINT_REG(id_afr0_el1); PRINT_REG(id_dfr0_el1); PRINT_REG(id_isar0_el1); PRINT_REG(id_isar1_el1); PRINT_REG(id_isar2_el1); PRINT_REG(id_isar3_el1); PRINT_REG(id_isar4_el1); PRINT_REG(id_isar5_el1); PRINT_REG(id_mmfr0_el1); PRINT_REG(id_mmfr1_el1); PRINT_REG(id_mmfr2_el1); PRINT_REG(id_mmfr3_el1); #if 0 /* Missing from llvm */ PRINT_REG(id_mmfr4_el1); #endif PRINT_REG(id_pfr0_el1); PRINT_REG(id_pfr1_el1); PRINT_REG(isr_el1); PRINT_REG(mair_el1); PRINT_REG(midr_el1); PRINT_REG(mpidr_el1); PRINT_REG(mvfr0_el1); PRINT_REG(mvfr1_el1); PRINT_REG(mvfr2_el1); PRINT_REG(revidr_el1); PRINT_REG(sctlr_el1); PRINT_REG(sp_el0); PRINT_REG(spsel); PRINT_REG(spsr_el1); PRINT_REG(tcr_el1); PRINT_REG(tpidr_el0); PRINT_REG(tpidr_el1); PRINT_REG(tpidrro_el0); PRINT_REG(ttbr0_el1); PRINT_REG(ttbr1_el1); PRINT_REG(vbar_el1); #undef PRINT_REG } DB_SHOW_COMMAND(vtop, db_show_vtop) { uint64_t phys; if (have_addr) { phys = arm64_address_translate_s1e1r(addr); db_printf("EL1 physical address reg (read): 0x%016lx\n", phys); phys = arm64_address_translate_s1e1w(addr); db_printf("EL1 physical address reg (write): 0x%016lx\n", phys); phys = arm64_address_translate_s1e0r(addr); db_printf("EL0 physical address reg (read): 0x%016lx\n", phys); phys = arm64_address_translate_s1e0w(addr); db_printf("EL0 physical address reg (write): 0x%016lx\n", phys); } else db_printf("show vtop \n"); } #endif diff --git a/sys/arm64/include/cpufunc.h b/sys/arm64/include/cpufunc.h index 94af62380de3..16133903841f 100644 --- a/sys/arm64/include/cpufunc.h +++ b/sys/arm64/include/cpufunc.h @@ -1,251 +1,252 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_CPUFUNC_H_ #define _MACHINE_CPUFUNC_H_ static __inline void breakpoint(void) { __asm("brk #0"); } #ifdef _KERNEL #define HAVE_INLINE_FFS static __inline __pure2 int ffs(int mask) { return (__builtin_ffs(mask)); } #define HAVE_INLINE_FFSL static __inline __pure2 int ffsl(long mask) { return (__builtin_ffsl(mask)); } #define HAVE_INLINE_FFSLL static __inline __pure2 int ffsll(long long mask) { return (__builtin_ffsll(mask)); } #define HAVE_INLINE_FLS static __inline __pure2 int fls(int mask) { return (mask == 0 ? 0 : 8 * sizeof(mask) - __builtin_clz((u_int)mask)); } #define HAVE_INLINE_FLSL static __inline __pure2 int flsl(long mask) { return (mask == 0 ? 0 : 8 * sizeof(mask) - __builtin_clzl((u_long)mask)); } #define HAVE_INLINE_FLSLL static __inline __pure2 int flsll(long long mask) { return (mask == 0 ? 0 : 8 * sizeof(mask) - __builtin_clzll((unsigned long long)mask)); } #include void pan_enable(void); static __inline register_t dbg_disable(void) { uint32_t ret; __asm __volatile( "mrs %x0, daif \n" "msr daifset, #(" __XSTRING(DAIF_D) ") \n" : "=&r" (ret)); return (ret); } static __inline void dbg_enable(void) { __asm __volatile("msr daifclr, #(" __XSTRING(DAIF_D) ")"); } static __inline register_t intr_disable(void) { /* DAIF is a 32-bit register */ uint32_t ret; __asm __volatile( "mrs %x0, daif \n" "msr daifset, #(" __XSTRING(DAIF_INTR) ") \n" : "=&r" (ret)); return (ret); } static __inline void intr_restore(register_t s) { WRITE_SPECIALREG(daif, s); } static __inline void intr_enable(void) { __asm __volatile("msr daifclr, #(" __XSTRING(DAIF_INTR) ")"); } static __inline void serror_enable(void) { __asm __volatile("msr daifclr, #(" __XSTRING(DAIF_A) ")"); } static __inline register_t get_midr(void) { uint64_t midr; midr = READ_SPECIALREG(midr_el1); return (midr); } static __inline register_t get_mpidr(void) { uint64_t mpidr; mpidr = READ_SPECIALREG(mpidr_el1); return (mpidr); } static __inline void clrex(void) { /* * Ensure compiler barrier, otherwise the monitor clear might * occur too late for us ? */ __asm __volatile("clrex" : : : "memory"); } static __inline void set_ttbr0(uint64_t ttbr0) { __asm __volatile( "msr ttbr0_el1, %0 \n" "isb \n" : : "r" (ttbr0)); } static __inline void invalidate_icache(void) { __asm __volatile( "ic ialluis \n" "dsb ish \n" "isb \n"); } static __inline void invalidate_local_icache(void) { __asm __volatile( "ic iallu \n" "dsb nsh \n" "isb \n"); } extern bool icache_aliasing; extern bool icache_vmid; extern int64_t dcache_line_size; extern int64_t icache_line_size; extern int64_t idcache_line_size; extern int64_t dczva_line_size; #define cpu_nullop() arm64_nullop() #define cpufunc_nullop() arm64_nullop() #define cpu_tlb_flushID() arm64_tlb_flushID() #define cpu_dcache_wbinv_range(a, s) arm64_dcache_wbinv_range((a), (s)) #define cpu_dcache_inv_range(a, s) arm64_dcache_inv_range((a), (s)) #define cpu_dcache_wb_range(a, s) arm64_dcache_wb_range((a), (s)) extern void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t); #define cpu_icache_sync_range(a, s) arm64_icache_sync_range((a), (s)) #define cpu_icache_sync_range_checked(a, s) arm64_icache_sync_range_checked((a), (s)) void arm64_nullop(void); void arm64_tlb_flushID(void); void arm64_dic_idc_icache_sync_range(vm_offset_t, vm_size_t); void arm64_aliasing_icache_sync_range(vm_offset_t, vm_size_t); int arm64_icache_sync_range_checked(vm_offset_t, vm_size_t); void arm64_dcache_wbinv_range(vm_offset_t, vm_size_t); void arm64_dcache_inv_range(vm_offset_t, vm_size_t); void arm64_dcache_wb_range(vm_offset_t, vm_size_t); +bool arm64_get_writable_addr(vm_offset_t, vm_offset_t *); #endif /* _KERNEL */ #endif /* _MACHINE_CPUFUNC_H_ */ diff --git a/sys/cddl/dev/fbt/aarch64/fbt_isa.c b/sys/cddl/dev/fbt/aarch64/fbt_isa.c index 6a21c7f403de..adc355e3296b 100644 --- a/sys/cddl/dev/fbt/aarch64/fbt_isa.c +++ b/sys/cddl/dev/fbt/aarch64/fbt_isa.c @@ -1,217 +1,221 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2006-2008 John Birrell jb@freebsd.org * Portions Copyright 2013 Justin Hibbits jhibbits@freebsd.org * Portions Copyright 2013 Howard Su howardsu@freebsd.org * Portions Copyright 2015 Ruslan Bukin * * $FreeBSD$ */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include "fbt.h" #define AARCH64_BRK 0xd4200000 #define AARCH64_BRK_IMM16_SHIFT 5 #define AARCH64_BRK_IMM16_VAL (0x40d << AARCH64_BRK_IMM16_SHIFT) #define FBT_PATCHVAL (AARCH64_BRK | AARCH64_BRK_IMM16_VAL) #define FBT_ENTRY "entry" #define FBT_RETURN "return" #define FBT_AFRAMES 4 int fbt_invop(uintptr_t addr, struct trapframe *frame, uintptr_t rval) { solaris_cpu_t *cpu; fbt_probe_t *fbt; cpu = &solaris_cpu[curcpu]; fbt = fbt_probetab[FBT_ADDR2NDX(addr)]; for (; fbt != NULL; fbt = fbt->fbtp_hashnext) { if ((uintptr_t)fbt->fbtp_patchpoint != addr) continue; cpu->cpu_dtrace_caller = addr; if (fbt->fbtp_roffset == 0) { dtrace_probe(fbt->fbtp_id, frame->tf_x[0], frame->tf_x[1], frame->tf_x[2], frame->tf_x[3], frame->tf_x[4]); } else { dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0); } cpu->cpu_dtrace_caller = 0; return (fbt->fbtp_savedval); } return (0); } void fbt_patch_tracepoint(fbt_probe_t *fbt, fbt_patchval_t val) { + vm_offset_t addr; - *fbt->fbtp_patchpoint = val; + if (!arm64_get_writable_addr((vm_offset_t)fbt->fbtp_patchpoint, &addr)) + panic("%s: Unable to write new instruction", __func__); + + *(fbt_patchval_t *)addr = val; cpu_icache_sync_range((vm_offset_t)fbt->fbtp_patchpoint, 4); } int fbt_provide_module_function(linker_file_t lf, int symindx, linker_symval_t *symval, void *opaque) { fbt_probe_t *fbt, *retfbt; uint32_t *target, *start; uint32_t *instr, *limit; const char *name; char *modname; bool found; int offs; modname = opaque; name = symval->name; /* Check if function is excluded from instrumentation */ if (fbt_excluded(name)) return (0); /* * Instrumenting certain exception handling functions can lead to FBT * recursion, so exclude from instrumentation. */ if (strcmp(name, "handle_el1h_sync") == 0 || strcmp(name, "do_el1h_sync") == 0) return (1); instr = (uint32_t *)(symval->value); limit = (uint32_t *)(symval->value + symval->size); /* Look for stp (pre-indexed) operation */ found = false; for (; instr < limit; instr++) { /* Some functions start with "stp xt1, xt2, [xn, ]!" */ if ((*instr & LDP_STP_MASK) == STP_64) { /* * Assume any other store of this type means we * are past the function prolog. */ if (((*instr >> ADDR_SHIFT) & ADDR_MASK) == 31) found = true; break; } /* * Some functions start with a "sub sp, sp, " * Sometimes the compiler will have a sub instruction that * is not of the above type so don't stop if we see one. */ if ((*instr & SUB_MASK) == SUB_INSTR && ((*instr >> SUB_RD_SHIFT) & SUB_R_MASK) == 31 && ((*instr >> SUB_RN_SHIFT) & SUB_R_MASK) == 31) { found = true; break; } } if (!found) return (0); fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, FBT_AFRAMES, fbt); fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; if ((*instr & SUB_MASK) == SUB_INSTR) fbt->fbtp_rval = DTRACE_INVOP_SUB; else fbt->fbtp_rval = DTRACE_INVOP_STP; fbt->fbtp_symindx = symindx; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; retfbt = NULL; again: for (; instr < limit; instr++) { if (*instr == RET_INSTR) break; else if ((*instr & B_MASK) == B_INSTR) { offs = (*instr & B_DATA_MASK); offs *= 4; target = (instr + offs); start = (uint32_t *)symval->value; if (target >= limit || target < start) break; } } if (instr >= limit) return (0); /* * We have a winner! */ fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO); fbt->fbtp_name = name; if (retfbt == NULL) { fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_RETURN, FBT_AFRAMES, fbt); } else { retfbt->fbtp_probenext = fbt; fbt->fbtp_id = retfbt->fbtp_id; } retfbt = fbt; fbt->fbtp_patchpoint = instr; fbt->fbtp_ctl = lf; fbt->fbtp_loadcnt = lf->loadcnt; fbt->fbtp_symindx = symindx; if ((*instr & B_MASK) == B_INSTR) fbt->fbtp_rval = DTRACE_INVOP_B; else fbt->fbtp_rval = DTRACE_INVOP_RET; fbt->fbtp_roffset = (uintptr_t)instr - (uintptr_t)symval->value; fbt->fbtp_savedval = *instr; fbt->fbtp_patchval = FBT_PATCHVAL; fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)]; fbt_probetab[FBT_ADDR2NDX(instr)] = fbt; lf->fbt_nentries++; instr++; goto again; }