diff --git a/sys/riscv/riscv/machdep.c b/sys/riscv/riscv/machdep.c index 5872ef9f94ac..61f86f9e6958 100644 --- a/sys/riscv/riscv/machdep.c +++ b/sys/riscv/riscv/machdep.c @@ -1,609 +1,610 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_ddb.h" #include "opt_kstack_pages.h" #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #endif #ifdef FDT #include #include #include #endif struct pcpu __pcpu[MAXCPU]; static struct trapframe proc0_tf; int early_boot = 1; int cold = 1; #define DTB_SIZE_MAX (1024 * 1024) struct kva_md_info kmi; int64_t dcache_line_size; /* The minimum D cache line size */ int64_t icache_line_size; /* The minimum I cache line size */ int64_t idcache_line_size; /* The minimum cache line size */ #define BOOT_HART_INVALID 0xffffffff uint32_t boot_hart = BOOT_HART_INVALID; /* The hart we booted on. */ cpuset_t all_harts; extern int *end; static char static_kenv[PAGE_SIZE]; static void cpu_startup(void *dummy) { sbi_print_version(); printcpuinfo(0); printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)realmem), ptoa((uintmax_t)realmem) / (1024 * 1024)); /* * Display any holes after the first chunk of extended memory. */ if (bootverbose) { int indx; printf("Physical memory chunk(s):\n"); for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) { vm_paddr_t size; size = phys_avail[indx + 1] - phys_avail[indx]; printf( "0x%016jx - 0x%016jx, %ju bytes (%ju pages)\n", (uintmax_t)phys_avail[indx], (uintmax_t)phys_avail[indx + 1] - 1, (uintmax_t)size, (uintmax_t)size / PAGE_SIZE); } } vm_ksubmap_init(&kmi); printf("avail memory = %ju (%ju MB)\n", ptoa((uintmax_t)vm_free_count()), ptoa((uintmax_t)vm_free_count()) / (1024 * 1024)); if (bootverbose) devmap_print_table(); bufinit(); vm_pager_bufferinit(); } SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); int cpu_idle_wakeup(int cpu) { return (0); } void cpu_idle(int busy) { spinlock_enter(); if (!busy) cpu_idleclock(); if (!sched_runnable()) __asm __volatile( "fence \n" "wfi \n"); if (!busy) cpu_activeclock(); spinlock_exit(); } void cpu_halt(void) { /* * Try to power down using the HSM SBI extension and fall back to a * simple wfi loop. */ intr_disable(); if (sbi_probe_extension(SBI_EXT_ID_HSM) != 0) sbi_hsm_hart_stop(); for (;;) __asm __volatile("wfi"); /* NOTREACHED */ } /* * Flush the D-cache for non-DMA I/O so that the I-cache can * be made coherent later. */ void cpu_flush_dcache(void *ptr, size_t len) { /* TBD */ } /* Get current clock frequency for the given CPU ID. */ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { panic("cpu_est_clockrate"); } void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) { } void spinlock_enter(void) { struct thread *td; register_t reg; td = curthread; if (td->td_md.md_spinlock_count == 0) { reg = intr_disable(); td->td_md.md_spinlock_count = 1; td->td_md.md_saved_sstatus_ie = reg; critical_enter(); } else td->td_md.md_spinlock_count++; } void spinlock_exit(void) { struct thread *td; register_t sstatus_ie; td = curthread; sstatus_ie = td->td_md.md_saved_sstatus_ie; td->td_md.md_spinlock_count--; if (td->td_md.md_spinlock_count == 0) { critical_exit(); intr_restore(sstatus_ie); } } /* * Construct a PCB from a trapframe. This is called from kdb_trap() where * we want to start a backtrace from the function that caused us to enter * the debugger. We have the context in the trapframe, but base the trace * on the PCB. The PCB doesn't have to be perfect, as long as it contains * enough for a backtrace. */ void makectx(struct trapframe *tf, struct pcb *pcb) { memcpy(pcb->pcb_s, tf->tf_s, sizeof(tf->tf_s)); pcb->pcb_ra = tf->tf_sepc; pcb->pcb_sp = tf->tf_sp; pcb->pcb_gp = tf->tf_gp; pcb->pcb_tp = tf->tf_tp; } static void init_proc0(vm_offset_t kstack) { struct pcpu *pcpup; pcpup = &__pcpu[0]; proc_linkup0(&proc0, &thread0); thread0.td_kstack = kstack; thread0.td_kstack_pages = KSTACK_PAGES; thread0.td_pcb = (struct pcb *)(thread0.td_kstack + thread0.td_kstack_pages * PAGE_SIZE) - 1; thread0.td_pcb->pcb_fpflags = 0; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; } #ifdef FDT static void try_load_dtb(caddr_t kmdp) { vm_offset_t dtbp; dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); #if defined(FDT_DTB_STATIC) /* * In case the device tree blob was not retrieved (from metadata) try * to use the statically embedded one. */ if (dtbp == (vm_offset_t)NULL) dtbp = (vm_offset_t)&fdt_static_dtb; #endif if (dtbp == (vm_offset_t)NULL) { printf("ERROR loading DTB\n"); return; } if (OF_install(OFW_FDT, 0) == FALSE) panic("Cannot install FDT"); if (OF_init((void *)dtbp) != 0) panic("OF_init failed with the found device tree"); } #endif static void cache_setup(void) { /* TODO */ dcache_line_size = 0; icache_line_size = 0; idcache_line_size = 0; } /* * Fake up a boot descriptor table. */ static void fake_preload_metadata(struct riscv_bootparams *rvbp) { static uint32_t fake_preload[48]; vm_offset_t lastaddr; size_t fake_size, dtb_size; #define PRELOAD_PUSH_VALUE(type, value) do { \ *(type *)((char *)fake_preload + fake_size) = (value); \ fake_size += sizeof(type); \ } while (0) #define PRELOAD_PUSH_STRING(str) do { \ uint32_t ssize; \ ssize = strlen(str) + 1; \ PRELOAD_PUSH_VALUE(uint32_t, ssize); \ strcpy(((char *)fake_preload + fake_size), str); \ fake_size += ssize; \ fake_size = roundup(fake_size, sizeof(u_long)); \ } while (0) fake_size = 0; lastaddr = (vm_offset_t)&end; PRELOAD_PUSH_VALUE(uint32_t, MODINFO_NAME); PRELOAD_PUSH_STRING("kernel"); PRELOAD_PUSH_VALUE(uint32_t, MODINFO_TYPE); PRELOAD_PUSH_STRING("elf kernel"); PRELOAD_PUSH_VALUE(uint32_t, MODINFO_ADDR); PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t)); PRELOAD_PUSH_VALUE(uint64_t, KERNBASE); PRELOAD_PUSH_VALUE(uint32_t, MODINFO_SIZE); PRELOAD_PUSH_VALUE(uint32_t, sizeof(size_t)); PRELOAD_PUSH_VALUE(uint64_t, (size_t)((vm_offset_t)&end - KERNBASE)); /* Copy the DTB to KVA space. */ lastaddr = roundup(lastaddr, sizeof(int)); PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_DTBP); PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t)); PRELOAD_PUSH_VALUE(vm_offset_t, lastaddr); dtb_size = fdt_totalsize(rvbp->dtbp_virt); memmove((void *)lastaddr, (const void *)rvbp->dtbp_virt, dtb_size); lastaddr = roundup(lastaddr + dtb_size, sizeof(int)); PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_KERNEND); PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t)); PRELOAD_PUSH_VALUE(vm_offset_t, lastaddr); PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_HOWTO); PRELOAD_PUSH_VALUE(uint32_t, sizeof(int)); PRELOAD_PUSH_VALUE(int, RB_VERBOSE); /* End marker */ PRELOAD_PUSH_VALUE(uint32_t, 0); PRELOAD_PUSH_VALUE(uint32_t, 0); preload_metadata = (caddr_t)fake_preload; /* Check if bootloader clobbered part of the kernel with the DTB. */ KASSERT(rvbp->dtbp_phys + dtb_size <= rvbp->kern_phys || rvbp->dtbp_phys >= rvbp->kern_phys + (lastaddr - KERNBASE), ("FDT (%lx-%lx) and kernel (%lx-%lx) overlap", rvbp->dtbp_phys, rvbp->dtbp_phys + dtb_size, rvbp->kern_phys, rvbp->kern_phys + (lastaddr - KERNBASE))); KASSERT(fake_size < sizeof(fake_preload), ("Too many fake_preload items")); if (boothowto & RB_VERBOSE) printf("FDT phys (%lx-%lx), kernel phys (%lx-%lx)\n", rvbp->dtbp_phys, rvbp->dtbp_phys + dtb_size, rvbp->kern_phys, rvbp->kern_phys + (lastaddr - KERNBASE)); } /* Support for FDT configurations only. */ CTASSERT(FDT); #ifdef FDT static void parse_fdt_bootargs(void) { char bootargs[512]; bootargs[sizeof(bootargs) - 1] = '\0'; if (fdt_get_chosen_bootargs(bootargs, sizeof(bootargs) - 1) == 0) { boothowto |= boot_parse_cmdline(bootargs); } } #endif static vm_offset_t parse_metadata(void) { caddr_t kmdp; vm_offset_t lastaddr; #ifdef DDB vm_offset_t ksym_start, ksym_end; #endif char *kern_envp; /* Find the kernel address */ kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); KASSERT(kmdp != NULL, ("No preload metadata found!")); /* Read the boot metadata */ boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *); if (kern_envp != NULL) init_static_kenv(kern_envp, 0); else init_static_kenv(static_kenv, sizeof(static_kenv)); #ifdef DDB ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t); ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t); db_fetch_ksymtab(ksym_start, ksym_end, 0); #endif #ifdef FDT try_load_dtb(kmdp); if (kern_envp == NULL) parse_fdt_bootargs(); #endif return (lastaddr); } void initriscv(struct riscv_bootparams *rvbp) { struct mem_region mem_regions[FDT_MEM_REGIONS]; struct pcpu *pcpup; int mem_regions_sz; vm_offset_t lastaddr; vm_size_t kernlen; #ifdef FDT phandle_t chosen; uint32_t hart; #endif char *env; TSRAW(&thread0, TS_ENTER, __func__, NULL); /* Set the pcpu data, this is needed by pmap_bootstrap */ pcpup = &__pcpu[0]; pcpu_init(pcpup, 0, sizeof(struct pcpu)); /* Set the pcpu pointer */ __asm __volatile("mv tp, %0" :: "r"(pcpup)); PCPU_SET(curthread, &thread0); /* Initialize SBI interface. */ sbi_init(); /* Parse the boot metadata. */ if (rvbp->modulep != 0) { preload_metadata = (caddr_t)rvbp->modulep; } else { fake_preload_metadata(rvbp); } lastaddr = parse_metadata(); #ifdef FDT /* * Look for the boot hart ID. This was either passed in directly from * the SBI firmware and handled by locore, or was stored in the device * tree by an earlier boot stage. */ chosen = OF_finddevice("/chosen"); if (OF_getencprop(chosen, "boot-hartid", &hart, sizeof(hart)) != -1) { boot_hart = hart; } #endif if (boot_hart == BOOT_HART_INVALID) { panic("Boot hart ID was not properly set"); } pcpup->pc_hart = boot_hart; #ifdef FDT /* * Exclude reserved memory specified by the device tree. Typically, * this contains an entry for memory used by the runtime SBI firmware. */ if (fdt_get_reserved_mem(mem_regions, &mem_regions_sz) == 0) { physmem_exclude_regions(mem_regions, mem_regions_sz, EXFLAG_NODUMP | EXFLAG_NOALLOC); } /* Grab physical memory regions information from device tree. */ if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, NULL) != 0) { panic("Cannot get physical memory regions"); } physmem_hardware_regions(mem_regions, mem_regions_sz); #endif /* * Identify CPU/ISA features. */ identify_cpu(0); /* Do basic tuning, hz etc */ init_param1(); cache_setup(); - /* Bootstrap enough of pmap to enter the kernel proper */ - kernlen = (lastaddr - KERNBASE); - pmap_bootstrap(rvbp->kern_l1pt, rvbp->kern_phys, kernlen); - #ifdef FDT /* * XXX: Unconditionally exclude the lowest 2MB of physical memory, as * this area is assumed to contain the SBI firmware. This is a little * fragile, but it is consistent with the platforms we support so far. * * TODO: remove this when the all regular booting methods properly * report their reserved memory in the device tree. */ physmem_exclude_region(mem_regions[0].mr_start, L2_SIZE, EXFLAG_NODUMP | EXFLAG_NOALLOC); #endif + + /* Bootstrap enough of pmap to enter the kernel proper */ + kernlen = (lastaddr - KERNBASE); + pmap_bootstrap(rvbp->kern_l1pt, rvbp->kern_phys, kernlen); + physmem_init_kernel_globals(); /* Establish static device mappings */ devmap_bootstrap(0, NULL); cninit(); /* * Dump the boot metadata. We have to wait for cninit() since console * output is required. If it's grossly incorrect the kernel will never * make it this far. */ if (getenv_is_true("debug.dump_modinfo_at_boot")) preload_dump(); init_proc0(rvbp->kern_stack); msgbufinit(msgbufp, msgbufsize); mutex_init(); init_param2(physmem); kdb_init(); #ifdef KDB if ((boothowto & RB_KDB) != 0) kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); #endif env = kern_getenv("kernelname"); if (env != NULL) strlcpy(kernelname, env, sizeof(kernelname)); if (boothowto & RB_VERBOSE) physmem_print_tables(); early_boot = 0; if (bootverbose && kstack_pages != KSTACK_PAGES) printf("kern.kstack_pages = %d ignored for thread0\n", kstack_pages); TSEXIT(); }