diff --git a/stand/kboot/kboot/arch/aarch64/tramp.S b/stand/kboot/kboot/arch/aarch64/tramp.S index 32616f239c50..f58a95cb8463 100644 --- a/stand/kboot/kboot/arch/aarch64/tramp.S +++ b/stand/kboot/kboot/arch/aarch64/tramp.S @@ -1,101 +1,103 @@ /* * Copyright (c) 2022, Netflix, Inc. * * SPDX-License-Identifier: BSD-2-Clause */ /* * This is the trampoline that starts the FreeBSD kernel. Since the Linux kernel * calls this routine with no args, and has a different environment than the boot * loader provides and that the kernel expects, this code is responsible for setting * all that up and calling the normal kernel entry point. It's analogous ot the * "purgatory" code in the linux kernel. Details about these operations are * contained in comments below. On aarch64, the kernel will start all the APs so * we don't have to worry about them here. */ /* * Keep in sync with exec.c. Kexec starts aarch64_tramp w/o any * parameters, so store them here. * * struct trampoline_data { * uint64_t entry; // 0 (PA where kernel loaded) * uint64_t modulep; // 8 module metadata * uint64_t memmap_src; // 16 Linux-provided memory map PA * uint64_t memmap_dst; // 24 Module data copy PA * uint64_t memmap_len; // 32 Length to copy * }; * * FreeBSD's arm64 entry point is _start which assumes: * MMU on with an identity map, or off * D-Cache: off * I-Cache: on or off * We are loaded at a 2MiB aligned address * Module data (modulep) pointer in x0 * * The rest of the boot loader tells Linux to land the kernel in its final * location with the needed alignment, etc. It does this, and then we take over. * * The linux kernel will helpfully turn off the MMU, flush the caches, disables * them, etc. It calls the tramp with two args: FDT blob address in x0 and the * EL2 vectors in x1. Currently, we make use of neither of these parameters: we * pass whatever dtb we think we need as part of the module data and we're a bit * weak on hypervisor support at the moment. _start's requirements are all * satisifed. * * This trampoline sets up the arguments the kernel expects and jumps to the * kernel _start address. We pass the modulep pointer in x0, as _start * expects. We assume that the various cache flushing, invalidation, etc that * linux did during or after copying the data down is sufficient, though we may * need to be mindful of cache flushing if we run in EL2 (TBD). * * Note, if TRAMP_MEMMAP_SRC is non-zero, then we have to copy the Linux * provided UEFI memory map. It's easier to do that here. In kboot we couldn't * access the physical memory, and it's a chicken and egg problem later in the * kernel. */ #define TRAMP_ENTRY 0 #define TRAMP_MODULEP 8 #define TRAMP_MEMMAP_SRC 16 #define TRAMP_MEMMAP_DST 24 #define TRAMP_MEMMAP_LEN 32 #define TRAMP_TOTAL 40 .text .globl tramp tramp: adr x8, trampoline_data ldr x10, [x8, #TRAMP_MEMMAP_SRC] cmp x10, xzr b.eq 9f /* * Copy over the memory map into area we have reserved for it. Assume * the copy is a multiple of 8, since we know table entries are made up * of several 64-bit quantities. */ ldp x11, x12, [x8, #TRAMP_MEMMAP_DST] /* x12 = len */ 1: ldr x13, [x10], #8 str x13, [x11], #8 subs x12, x12, #8 b.hi 1b 9: ldp x9, x0, [x8, #TRAMP_ENTRY] /* x0 = modulep */ br x9 .p2align 4 trampoline_data: .space TRAMP_TOTAL #define TMPSTACKSIZE 48 /* 16 bytes for args +8 for pushq/popfq + 24 spare */ .space TMPSTACKSIZE tramp_end: /* padding doubles as stack */ .data .globl tramp_size tramp_size: .long tramp_end-tramp .globl tramp_data_offset tramp_data_offset: .long trampoline_data-tramp + + .section .note.GNU-stack,"",%progbits diff --git a/stand/kboot/kboot/arch/amd64/amd64_tramp.S b/stand/kboot/kboot/arch/amd64/amd64_tramp.S index b95e99cbaf0f..2f2f765d3547 100644 --- a/stand/kboot/kboot/arch/amd64/amd64_tramp.S +++ b/stand/kboot/kboot/arch/amd64/amd64_tramp.S @@ -1,107 +1,109 @@ /*- * Copyright (c) 2022 Netflix, Inc * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This is the trampoline that starts the FreeBSD kernel. Since the Linux kernel * calls this routine with no args, and has a different environment than the * boot loader provides and that the kernel expects, this code is responsible * for setting all that up and calling the normal kernel entry point. It's * analogous to the "purgatory" code in the linux kernel. Details about these * operations are contained in comments below. On amd64, the kernel starts all * the APs so we don't have to worry about them here. */ /* * Keep in sync with elf64_freebsd.c. Kexec starts tramp w/o any parameters, so * store them here. This is constructed to be a useful stack: * * struct trampoline_data { * uint64_t pt4; // Page table address to pop * uint64_t entry; // return address to jump to kernel * uint32_t fill1; // 0 * uint32_t modulep; // 4 module metadata * uint32_t kernend; // 8 kernel end * uint32_t fill2; // 12 * }; * * loader.kboot will construct a stack that btext expects, which is arguments on * the stack, not in registers, and these args are 32-bit not 64 * * Processor is already in long mode when we're called, paging is enabled and * boot loader loads things such that: * - kernel mapped at KERNBASE, aligned to 2MB, below 4GB, contiguous memory * - there is a 2M hole at KERNBASE (KERNSTART = KERNBASE + 2M) * - kernel is mapped with 2M superpages * - The kernel, modules and metadata is in first 4GB which is unity mapped * - There's additional memory after loader provided data for early allocations * * Unlike EFI, we don't support copying the staging area. We tell Linux to land * the kernel in its final location with the needed alignment, etc. We copy the * trampoline code to 1MB offset above KERNBASE since that memory is otherwise * free and safely above the lower 1MB swamp we inherited from IBM PC, though * this code makes no assumptions about where that might. * * Thus, the trampoline just needs to set %rsp to that stack pop the %cr3 value, * set it and then retq to jump to the kernel with its stack args filled in. * Since the handoff to this code used to be from 32-bit code, it uses the i386 * calling conventions which put the arguments on the stack. The kernel's btext * routine expects this setup. */ .text .globl tramp tramp: cli /* Make sure we don't get interrupted. */ leaq tramp_pt4(%rip), %rsp /* Setup our pre-filled-in stack */ popq %rax /* Pop off the PT4 ptr for %cr3 */ movq %rax, %cr3 /* set the page table */ retq /* Return addr and args already on stack */ /* * The following is the stack for the above code. The stack will increase in * address as things are popped off of it, so we start with the stack pointing * to tramp_pt4. */ .p2align 3 /* Stack has to be 8 byte aligned */ trampoline_data: tramp_pt4: .quad 0 /* New %cr3 value */ tramp_entry: .quad 0 /* Entry to kernel (btext) */ /* %rsp points here on entry to amd64 kernel's btext */ .long 0 /* 0 filler, ignored (current loaders set to 0) */ tramp_modulep: .long 0 /* 4 moudlep */ tramp_kernend: .long 0 /* 8 kernend */ .long 0 /* 12 alignment filler (also 0) */ tramp_end: .data .type tramp_size,@object .globl tramp_size tramp_size: .long tramp_end-tramp .size tramp_size, 4 .type tramp_data_offset,@object .globl tramp_data_offset tramp_data_offset: .long trampoline_data-tramp .size tramp_data_offset, 4 + + .section .note.GNU-stack,"",%progbits diff --git a/stand/kboot/kboot/arch/powerpc64/kerneltramp.S b/stand/kboot/kboot/arch/powerpc64/kerneltramp.S index 211a6e474d2a..e9678dc02e72 100644 --- a/stand/kboot/kboot/arch/powerpc64/kerneltramp.S +++ b/stand/kboot/kboot/arch/powerpc64/kerneltramp.S @@ -1,101 +1,103 @@ /* * This is the analog to the kexec "purgatory" code * * The goal here is to call the actual kernel entry point with the arguments it * expects when kexec calls into it with no arguments. The value of the kernel * entry point and arguments r3-r7 are copied into the trampoline text (which * can be executed from any address) at bytes 8-32. kexec begins execution * of APs at 0x60 bytes past the entry point, executing in a copy relocated * to the absolute address 0x60. Here we implement a loop waiting on the release * of a lock by the kernel at 0x40. * */ #include .globl CNAME(kerneltramp),CNAME(szkerneltramp) CNAME(kerneltramp): mflr %r9 bl 2f .space 24 /* branch address, r3-r7 */ /* * MUST BE IN SYNC WITH: * struct trampoline_data { * uint32_t kernel_entry; * uint32_t dtb; * uint32_t phys_mem_offset; * uint32_t of_entry; * uint32_t mdp; * uint32_t mdp_size; * }; */ . = kerneltramp + 0x40 /* AP spinlock */ .long 0 . = kerneltramp + 0x60 /* AP entry point */ li %r3,0x40 1: lwz %r1,0(%r3) cmpwi %r1,0 beq 1b /* Jump into CPU reset */ li %r0,0x100 icbi 0,%r0 isync sync ba 0x100 2: /* Continuation of kerneltramp */ mflr %r8 mtlr %r9 mfmsr %r10 andi. %r10, %r10, 1 /* test MSR_LE */ bne little_endian /* We're starting in BE */ big_endian: lwz %r3,4(%r8) lwz %r4,8(%r8) lwz %r5,12(%r8) lwz %r6,16(%r8) lwz %r7,20(%r8) lwz %r10, 0(%r8) mtctr %r10 bctr /* We're starting in LE */ little_endian: /* Entries are BE, swap them during load. */ li %r10, 4 lwbrx %r3, %r8, %r10 li %r10, 8 lwbrx %r4, %r8, %r10 li %r10, 12 lwbrx %r5, %r8, %r10 li %r10, 16 lwbrx %r6, %r8, %r10 li %r10, 20 lwbrx %r7, %r8, %r10 /* Clear MSR_LE flag to enter the BE world */ mfmsr %r10 clrrdi %r10, %r10, 1 mtsrr1 %r10 /* Entry is at 0(%r8) */ li %r10, 0 lwbrx %r10, %r8, %r10 mtsrr0 %r10 rfid endkerneltramp: .data CNAME(szkerneltramp): .long endkerneltramp - CNAME(kerneltramp) + + .section .note.GNU-stack,"",%progbits diff --git a/stand/kboot/libkboot/arch/aarch64/host_syscall.S b/stand/kboot/libkboot/arch/aarch64/host_syscall.S index db3ecf0f885d..3b1c345f2cf6 100644 --- a/stand/kboot/libkboot/arch/aarch64/host_syscall.S +++ b/stand/kboot/libkboot/arch/aarch64/host_syscall.S @@ -1,18 +1,20 @@ #include /* * Emulate the Linux system call interface. System call number in x8. * Args in x0, x1, x2, x3, x4 and x5. Return in x0. */ ENTRY(host_syscall) mov x8, x0 mov x0, x1 mov x1, x2 mov x2, x3 mov x3, x4 mov x4, x5 mov x5, x6 svc 0 ret /* Note: We're exposing the raw return value to the caller */ END(host_syscall) + + .section .note.GNU-stack,"",%progbits diff --git a/stand/kboot/libkboot/arch/amd64/host_syscall.S b/stand/kboot/libkboot/arch/amd64/host_syscall.S index 5bf0fca0cec1..0869bfe245f6 100644 --- a/stand/kboot/libkboot/arch/amd64/host_syscall.S +++ b/stand/kboot/libkboot/arch/amd64/host_syscall.S @@ -1,29 +1,31 @@ #include /* * Emulate the Linux system call interface. The system call number is set in * %rax, and %rdi, %rsi, %rdx, %r10, %r8, %r9 have the 6 system call * arguments. errno is returned as a negative value, but we use it more as a * flag something went wrong rather than using its value. * * Note: For system calls, we use %r10 instead of %rcx for the 4th argument. * See section A.2.1 for the Linux calling conventions of the ABI spec * https://web.archive.org/web/20160801075146/http://www.x86-64.org/documentation/abi.pdf * In addition to the below, %r11 and %rcx are destroyed, negative * values are ERRNO for %rax between -1 and -4095 otherwise the system * call is successful. Unlike other Unix systems, carry isn't used to * signal an error in the system call. We expose the raw system call * result, rather than do the POSIX converion to -1 and setting errno. */ ENTRY(host_syscall) movq %rdi, %rax /* SYS_ number in %rax */ movq %rsi, %rdi /* arg2 -> 1 */ movq %rdx, %rsi /* arg3 -> 2 */ movq %rcx, %rdx /* arg4 -> 3 */ movq %r8, %r10 /* arg5 -> 4 */ movq %r9, %r8 /* arg6 -> 5 */ movq 8(%rsp),%r9 /* arg7 -> 6 from stack. */ syscall ret /* Note: We're exposing the raw return value to the caller */ END(host_syscall) + + .section .note.GNU-stack,"",%progbits diff --git a/stand/kboot/libkboot/arch/powerpc64/host_syscall.S b/stand/kboot/libkboot/arch/powerpc64/host_syscall.S index f9108065ebfa..84fde2041704 100644 --- a/stand/kboot/libkboot/arch/powerpc64/host_syscall.S +++ b/stand/kboot/libkboot/arch/powerpc64/host_syscall.S @@ -1,32 +1,34 @@ #include /* * Emulate the Linux system call interface. The system call number is set in * %r0, and %r3 -> %r8 have the 6 system call arguments. errno is returned * as a negative value, but we use it more as a flag something went wrong * rather than using its value. * * Return value in %r3. If it is positive or < -4096, it's a successful * system call. If it is between -1 and -4095 then it's an failed system * call with -x as the errno. Errors from the kernel are signaled via the * the 'so' bit, but we don't test that here at all. There are at most 6 * arguments to system calls in Linux. * * We expose the raw system call result, rather than do the POSIX * conversion to -1 and setting errno. * * Note: The code this replaced used bso to set %r3 to 0 for the read and * open system calls for reasons that are still under investigation. */ ENTRY(host_syscall) mr %r0, %r3 /* SYS_ number in $r0 */ mr %r3, %r4 /* arg2 -> 1 */ mr %r4, %r5 /* arg3 -> 2 */ mr %r5, %r6 /* arg4 -> 3 */ mr %r6, %r7 /* arg5 -> 4 */ mr %r7, %r8 /* arg6 -> 5 */ mr %r8, %r9 /* arg7 -> 6 */ sc blr /* Note: We're exposing the raw return value to the caller */ END(host_syscall) + + .section .note.GNU-stack,"",%progbits