Index: sys/arm64/arm64/locore.S =================================================================== --- sys/arm64/arm64/locore.S +++ sys/arm64/arm64/locore.S @@ -28,6 +28,7 @@ #include "assym.inc" #include "opt_kstack_pages.h" +#include "opt_platform.h" #include #include #include @@ -54,11 +55,12 @@ .globl _start _start: #ifdef LINUX_BOOT_ABI - /* - * See Documentation/arm64/booting.txt in the Linux kernel. - * This is needed to boot using U-Boot's booti command. - */ +#define FDT_MAGIC 0xEDFE0DD0 /* FDT blob Magic */ +/* + * See Documentation/arm64/booting.txt in the Linux kernel. + * This is needed to boot using U-Boot's booti command. + */ #define BOOTI_MAGIC 0x644d5241 #define UBOOT_IMAGE_OFFSET 0 #define UBOOT_IMAGE_SIZE _end - _start @@ -371,24 +373,26 @@ */ mov x6, #(KERNBASE) + mov x19, xzr +#ifdef LINUX_BOOT_ABI and x7, x0, x6 cmp x7, x6 b.eq 1f - /* booted from U-Boot */ + /* Booted from U-Boot */ + mov x19, #1 ldr x7, .Lend sub x8, x7, x6 /* kernel size = end - begin */ b 2f 1: +#endif /* booted from FreeBSD loader */ sub x8, x0, x6 /* size = modulep - begin */ +2: /* Add two 2MiB pages for the module data and round up */ ldr x7, =(3 * L2_SIZE - 1) add x8, x8, x7 -2: /* Get the number of l2 pages to allocate, rounded down */ lsr x10, x8, #(L2_SHIFT) - /* Add 1 to get actual count */ - add x10, x10, #1 /* Create the kernel space L2 table */ mov x6, x26 @@ -442,6 +446,21 @@ mov x10, #1 bl build_l1_block_pagetable +#ifdef LINUX_BOOT_ABI + cbz x19, 1f + ldr x8, [x0] + ldr w9, =FDT_MAGIC + cmp w8, w9 + b.ne 1f + /* Create VA = PA map for DTB (x0) */ + mov x7, #NORMAL_UNCACHED + mov x8, x0 + mov x9, x0 /* VA start (== PA start) */ + mov x10, #1 + bl build_l1_block_pagetable +1: +#endif + /* Move to the l0 table */ add x27, x27, #PAGE_SIZE Index: sys/arm64/arm64/machdep.c =================================================================== --- sys/arm64/arm64/machdep.c +++ sys/arm64/arm64/machdep.c @@ -1091,7 +1091,7 @@ #ifdef FDT try_load_dtb(kmdp); #ifdef LINUX_BOOT_ABI - parse_bootargs(&lastaddr, abp); + parse_fdt_bootargs(); #endif #endif @@ -1148,6 +1148,10 @@ /* Exclude entries neexed in teh DMAP region, but not phys_avail */ if (efihdr != NULL) exclude_efi_map_entries(efihdr); + +#ifdef LINUX_BOOT_ABI + fdt_initrd(); +#endif arm_physmem_init_kernel_globals(); devmap_bootstrap(0, NULL); Index: sys/arm64/arm64/machdep_boot.c =================================================================== --- sys/arm64/arm64/machdep_boot.c +++ sys/arm64/arm64/machdep_boot.c @@ -1,5 +1,4 @@ /*- - * Copyright (c) 2019 Juniper Networks, Inc * Copyright (c) 2004 Olivier Houchard * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. @@ -46,12 +45,11 @@ #include #ifdef FDT +#include #include #include #include #include -#include -#include #include #endif @@ -70,27 +68,24 @@ preload_size = roundup(preload_size, sizeof(u_long)); \ } while (0) -static int build_l2_block_pagetable(vm_offset_t, uint64_t, - struct arm64_bootparams *); - +#define CMDLINE_GUARD "FreeBSD:" #define INITRD_START "linux,initrd-start" #define INITRD_END "linux,initrd-end" #define KENV_SIZE 2048 -static char static_kenv[KENV_SIZE]; -static caddr_t metadata_endptr; +static char static_kenv[KENV_SIZE]; +static size_t initrd_size; +static vm_paddr_t initrd_start; #endif -#define PMAP_BOOTSTRAP_PAGES 2 - -extern vm_offset_t end; +extern vm_offset_t end; +static caddr_t fake_metadata_endptr; /* * Fake up a boot descriptor table */ static vm_offset_t -fake_preload_metadata(void *dtb_ptr, size_t dtb_size, - struct arm64_bootparams *abp) +fake_preload_metadata(void *dtb_ptr, size_t dtb_size) { vm_offset_t lastaddr; static char fake_preload[256]; @@ -118,8 +113,7 @@ lastaddr = roundup(lastaddr, sizeof(vm_offset_t)); #ifdef FDT - if (dtb_ptr != NULL && - (build_l2_block_pagetable(lastaddr, dtb_size, abp) == 0)) { + if (dtb_ptr != NULL) { /* Copy DTB to KVA space and insert it into module chain. */ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_DTBP); PRELOAD_PUSH_VALUE(uint32_t, sizeof(uint64_t)); @@ -129,9 +123,9 @@ } lastaddr = roundup(lastaddr, sizeof(vm_offset_t)); - /* End marker */ - metadata_endptr = preload_ptr; #endif + /* End marker */ + fake_metadata_endptr = preload_ptr; PRELOAD_PUSH_VALUE(uint32_t, 0); PRELOAD_PUSH_VALUE(uint32_t, 0); @@ -157,99 +151,33 @@ if (dtb_ptr && fdt_check_header(dtb_ptr) == 0) dtb_size = fdt_totalsize(dtb_ptr); #endif - return (fake_preload_metadata(dtb_ptr, dtb_size, abp)); + return (fake_preload_metadata(dtb_ptr, dtb_size)); } #ifdef FDT -/* - * Builds count 2 MiB page table entries. - * During startup, locore.S maps kernel memory in L2 page table. - * Create space to copy size bytes following the kernel memory. - * See build_l2_block_pagetable in locore.S - */ -static int -build_l2_block_pagetable(vm_offset_t lastaddr, uint64_t size, - struct arm64_bootparams *abp) +void fdt_initrd_params(size_t *size, vm_paddr_t *start_pa) { - vm_offset_t l2_block_entry, *l2pt_entry; - int32_t count_2mib; - volatile uint64_t output_bits; - - /* Number of 2MiB pages */ - count_2mib = ((lastaddr - KERNBASE) + size) >> L2_SHIFT; - - /* All the memory must not cross a 1GiB boundary */ - if (count_2mib >= Ln_ENTRIES) { - printf("%s: Adding %#lx bytes makes kernel cross 1GiB boundary\n", - __FUNCTION__, size); - return EINVAL; - } - - /* size fits within the last 2MiB page table entry */ - if (((lastaddr - KERNBASE) >> L2_SHIFT) == count_2mib) - return 0; - - /* Build the L2 block entry */ - l2_block_entry = ATTR_IDX(VM_MEMATTR_WRITE_BACK) | L2_BLOCK | ATTR_AF; -#ifdef SMP - l2_block_entry |= ATTR_SH(ATTR_SH_IS); -#endif - /* Number of 2MiB pages mapped to kernel */ - count_2mib = (lastaddr - KERNBASE) >> L2_SHIFT; - - /* Go to last L2 page table entry. Each pagetable entry is 8 bytes */ - l2pt_entry = (vm_offset_t*)((abp->kern_l1pt - PAGE_SIZE) + - (count_2mib << 3)); - output_bits = (*l2pt_entry++ >> L2_SHIFT) + 1; - - /* Build count 2MiB page table entries */ - for (count_2mib = size >> L2_SHIFT; count_2mib >= 0; - l2pt_entry++, output_bits++, count_2mib--) - *l2pt_entry = (output_bits << L2_SHIFT) | l2_block_entry; - return 0; + *size = initrd_size; + *start_pa = initrd_start; } /* - * Align start addr to 1GiB boundary and build L1 page table entry for it. - * See build_l1_block_pagetable in locore.S + * The physical memory region where the initrd image is loaded + * is marked as an exclude region. */ -static void -build_l1_block_pagetable(vm_offset_t start, struct arm64_bootparams *abp) -{ - vm_offset_t l1_table_idx, l1_block_entry, phy_addr, *l1_table_entry; - - /* Find the table index */ - l1_table_idx = (start >> L1_SHIFT) & Ln_ADDR_MASK; - - /* Build the L1 block entry */ - l1_block_entry = ATTR_nG | ATTR_IDX(VM_MEMATTR_UNCACHEABLE) | - L1_BLOCK | ATTR_AF; -#ifdef SMP - l1_block_entry |= ATTR_SH(ATTR_SH_IS); -#endif - - /* Set the physical address */ - phy_addr = l1_block_entry | (l1_table_idx << L1_SHIFT); - - /* Index of L1 pagetable. Each pagetable entry is 8 bytes */ - l1_table_entry = (vm_offset_t*)((abp->kern_l0pt + PAGE_SIZE) + - (l1_table_idx << 3)); - *l1_table_entry = phy_addr; -} - -/* - * Copy the initrd image passed using U-Boot's booti command into - * KVA space. - */ -static void -linux_load_initrd(vm_offset_t *lastaddr, struct arm64_bootparams *abp) +void +fdt_initrd(void) { phandle_t chosen; - uint64_t initrd_start = 0, initrd_end = 0; - uint64_t initrd_size; - caddr_t preload_ptr; - size_t preload_size = 0; + vm_paddr_t initrd_end = 0; + + /* + * Fake metadata is used to support boot from U-Boot. Process + * initrd args from FDT blob. + */ + if (fake_metadata_endptr == NULL) + return; if ((chosen = OF_finddevice("/chosen")) == -1) return; @@ -267,15 +195,16 @@ if ((initrd_size = (initrd_end - initrd_start)) <= 0) return; - if (build_l2_block_pagetable(*lastaddr, initrd_size, abp) != 0) - return; - - build_l1_block_pagetable(initrd_start, abp); + arm_physmem_exclude_region(initrd_start, initrd_size, EXFLAG_NOALLOC); +} - /* Copy the initrd image to virtual address space */ - memmove((void*)(*lastaddr), (void*)initrd_start, initrd_size); +void +load_initrd_metadata(vm_size_t initrd_size, vm_offset_t va) +{ + size_t preload_size = 0; + caddr_t preload_ptr; - preload_ptr = metadata_endptr; + preload_ptr = fake_metadata_endptr; PRELOAD_PUSH_VALUE(uint32_t, MODINFO_NAME); PRELOAD_PUSH_STRING("initrd"); @@ -289,42 +218,42 @@ PRELOAD_PUSH_VALUE(uint32_t, MODINFO_ADDR); PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t)); - PRELOAD_PUSH_VALUE(uint64_t, *lastaddr); - - *lastaddr += initrd_size; - *lastaddr = roundup(*lastaddr, sizeof(vm_offset_t)); + PRELOAD_PUSH_VALUE(uint64_t, va); /* End marker */ - metadata_endptr = preload_ptr; + fake_metadata_endptr = preload_ptr; PRELOAD_PUSH_VALUE(uint32_t, 0); PRELOAD_PUSH_VALUE(uint32_t, 0); } /* - * Parse initrd image arguments, bootargs passed in FDT from U-Boot. + * Parse bootargs passed in FDT from U-Boot into kenv and boot options. */ void -parse_bootargs(vm_offset_t *lastaddr, struct arm64_bootparams *abp) +parse_fdt_bootargs(void) { + char buf[KENV_SIZE]; + char *cmdline; - /* - * Fake metadata is used to support boot from U-Boot. Process bootargs, - * initrd args from FDT blob set in fake medadata. - */ - if (metadata_endptr == NULL) + /* Fake metadata is used to support boot from U-Boot. */ + if (fake_metadata_endptr == NULL) return; - /* Booted from U-Boot */ - linux_load_initrd(lastaddr, abp); + cmdline = buf; + if (fdt_get_chosen_bootargs(cmdline, sizeof(cmdline)) != 0) + return; - /* - * L2 PTEs map addresses in order of kernel, dtb, initrd image. - * Add L2 pages at the end for pmap to bootstrap L2, L3 PTEs, etc. - */ - build_l2_block_pagetable(*lastaddr, - (PMAP_BOOTSTRAP_PAGES * L2_SIZE) - 1, abp); + /* Skip leading spaces. */ + while (isspace(*cmdline)) + cmdline++; + + /* Test and remove guard. */ + if (strncasecmp(cmdline, CMDLINE_GUARD, sizeof(CMDLINE_GUARD))) + return; + cmdline += sizeof(CMDLINE_GUARD); init_static_kenv(static_kenv, sizeof(static_kenv)); - ofw_parse_bootargs(); + + boothowto |= boot_parse_cmdline(cmdline); } #endif Index: sys/arm64/arm64/pmap.c =================================================================== --- sys/arm64/arm64/pmap.c +++ sys/arm64/arm64/pmap.c @@ -106,6 +106,7 @@ */ #include "opt_vm.h" +#include "opt_platform.h" #include #include @@ -896,6 +897,39 @@ m->md.pv_memattr = VM_MEMATTR_WRITE_BACK; } +#if defined(FDT) && defined(LINUX_BOOT_ABI) +static void +pmap_map_initrd(void) +{ + int lvl; + pd_entry_t *pde; + vm_size_t size; + vm_paddr_t pa; + vm_offset_t va, offset; + + fdt_initrd_params(&size, &pa); + if ( size == 0 || pa == 0) + return; + + offset = pa & PAGE_MASK; + size = round_page(offset + size); + + va = kva_alloc(size); + if (va == 0) + panic("%s: Couldn't allocate KVA", __func__); + + pde = pmap_pde(kernel_pmap, va, &lvl); + KASSERT(lvl == 2, ("%s: Invalid level %d", __func__, lvl)); + + /* L3 table is linked */ + va = trunc_page(va); + pa = trunc_page(pa); + pmap_kenter(va, size, pa, CACHED_MEMORY); + + load_initrd_metadata(size, va); +} +#endif + /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap @@ -944,6 +978,10 @@ TAILQ_INIT(&pv_dummy.pv_list); vm_initialized = 1; + +#if defined(FDT) && defined(LINUX_BOOT_ABI) + pmap_map_initrd(); +#endif } static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2, CTLFLAG_RD, 0, Index: sys/arm64/conf/GENERIC =================================================================== --- sys/arm64/conf/GENERIC +++ sys/arm64/conf/GENERIC @@ -77,7 +77,6 @@ options RCTL # Resource limits options SMP options INTRNG -options LINUX_BOOT_ABI # Boot using booti command from U-Boot # Debugging support. Always need this: options KDB # Enable kernel debugger support. Index: sys/arm64/include/machdep.h =================================================================== --- sys/arm64/include/machdep.h +++ sys/arm64/include/machdep.h @@ -49,8 +49,11 @@ void initarm(struct arm64_bootparams *); extern void (*pagezero)(void *); vm_offset_t linux_parse_boot_param(struct arm64_bootparams *); -#ifdef LINUX_BOOT_ABI -void parse_bootargs(vm_offset_t *, struct arm64_bootparams *); +#if defined(FDT) && defined(LINUX_BOOT_ABI) +void fdt_initrd(void); +void fdt_initrd_params(size_t *, vm_paddr_t *); +void load_initrd_metadata(vm_size_t, vm_offset_t); +void parse_fdt_bootargs(void); #endif #endif /* _MACHINE_MACHDEP_H_ */ Index: sys/conf/options.arm64 =================================================================== --- sys/conf/options.arm64 +++ sys/conf/options.arm64 @@ -2,7 +2,7 @@ ARM64 opt_global.h INTRNG opt_global.h -LINUX_BOOT_ABI opt_global.h +LINUX_BOOT_ABI opt_platform.h SOCDEV_PA opt_global.h SOCDEV_VA opt_global.h THUNDERX_PASS_1_1_ERRATA opt_global.h