Index: sys/amd64/amd64/efirt_machdep.c =================================================================== --- sys/amd64/amd64/efirt_machdep.c +++ sys/amd64/amd64/efirt_machdep.c @@ -55,6 +55,7 @@ #include #include #include +#include #include #include #include @@ -266,6 +267,7 @@ curpmap = PCPU_GET(curpmap); PMAP_LOCK_ASSERT(curpmap, MA_OWNED); + curthread->td_md.md_efirt_dis_pf = vm_fault_disable_pagefaults(); /* * IPI TLB shootdown handler invltlb_pcid_handler() reloads @@ -300,6 +302,7 @@ curpmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid : 0)); if (!pmap_pcid_enabled) invltlb(); + vm_fault_enable_pagefaults(curthread->td_md.md_efirt_dis_pf); } /* XXX debug stuff */ Index: sys/amd64/amd64/efirt_support.S =================================================================== --- /dev/null +++ sys/amd64/amd64/efirt_support.S @@ -0,0 +1,116 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2018 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include + +#include "assym.inc" + + .text +ENTRY(efi_rt_arch_call) + pushq %rbp + movq %rsp, %rbp + + movq %rbx, EC_RBX(%rdi) + movq %rsp, EC_RSP(%rdi) + movq %rbp, EC_RBP(%rdi) + movq %r12, EC_R12(%rdi) + movq %r13, EC_R13(%rdi) + movq %r14, EC_R14(%rdi) + movq %r15, EC_R15(%rdi) + movq PCPU(CURTHREAD), %rax + movq %rdi, TD_MD+MD_EFIRT_TMP(%rax) + movq PCPU(CURPCB), %rsi + + movl EC_ARGCNT(%rdi), %ecx + movl %ecx, %ebx + movl $4, %eax + cmpl %eax, %ecx + cmovbl %eax, %ecx + shll $3, %ecx + subq %rcx, %rsp + + cmpl $0, %ebx + jz 1f + movq EC_ARG1(%rdi), %rcx + decl %ebx + jz 1f + movq EC_ARG2(%rdi), %rdx + decl %ebx + jz 1f + movq EC_ARG3(%rdi), %r8 + decl %ebx + jz 1f + movq EC_ARG4(%rdi), %r9 + decl %ebx + jz 1f + movq EC_ARG5(%rdi), %rax + movq %rax, 4*8(%rsp) + decl %ebx + jz 1f + movq $efi_rt_panic_str, %rdi + call panic +1: movq EC_FPTR(%rdi), %rax + movq $efi_rt_fault, PCB_ONFAULT(%rsi) + callq *%rax + + movq PCPU(CURTHREAD), %rbx + movq TD_MD+MD_EFIRT_TMP(%rbx), %rdi + movq %rax, EC_EFI_STATUS(%rdi) + movq PCPU(CURPCB), %rsi + xorl %eax, %eax + movq %rax, PCB_ONFAULT(%rsi) + +efi_rt_arch_call_tail: + movq EC_R15(%rdi), %r15 + movq EC_R14(%rdi), %r14 + movq EC_R13(%rdi), %r13 + movq EC_R12(%rdi), %r12 + movq EC_RBP(%rdi), %rbp + movq EC_RSP(%rdi), %rsp + movq EC_RBX(%rdi), %rbx + + popq %rbp + ret +END(efi_rt_arch_call) + +ENTRY(efi_rt_fault) + xorl %eax, %eax + movq PCPU(CURPCB), %rsi + movq %rax, PCB_ONFAULT(%rsi) + movl $EFAULT, %eax + movq PCPU(CURTHREAD), %rbx + movq TD_MD+MD_EFIRT_TMP(%rbx), %rdi + jmp efi_rt_arch_call_tail +END(efi_rt_fault) + +efi_rt_panic_str: .asciz "efi_rt_arch_call: too many args" Index: sys/amd64/amd64/genassym.c =================================================================== --- sys/amd64/amd64/genassym.c +++ sys/amd64/amd64/genassym.c @@ -68,6 +68,7 @@ #include #include #include +#include ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace)); ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap)); @@ -77,12 +78,15 @@ ASSYM(MD_LDT, offsetof(struct mdproc, md_ldt)); ASSYM(MD_LDT_SD, offsetof(struct mdproc, md_ldt_sd)); +ASSYM(MD_EFIRT_TMP, offsetof(struct mdthread, md_efirt_tmp)); + ASSYM(TD_LOCK, offsetof(struct thread, td_lock)); ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); ASSYM(TD_PFLAGS, offsetof(struct thread, td_pflags)); ASSYM(TD_PROC, offsetof(struct thread, td_proc)); ASSYM(TD_FRAME, offsetof(struct thread, td_frame)); +ASSYM(TD_MD, offsetof(struct thread, td_md)); ASSYM(TDF_ASTPENDING, TDF_ASTPENDING); ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED); @@ -249,3 +253,19 @@ #ifdef HWPMC_HOOKS ASSYM(PMC_FN_USER_CALLCHAIN, PMC_FN_USER_CALLCHAIN); #endif + +ASSYM(EC_EFI_STATUS, offsetof(struct efirt_callinfo, ec_efi_status)); +ASSYM(EC_FPTR, offsetof(struct efirt_callinfo, ec_fptr)); +ASSYM(EC_ARGCNT, offsetof(struct efirt_callinfo, ec_argcnt)); +ASSYM(EC_ARG1, offsetof(struct efirt_callinfo, ec_arg1)); +ASSYM(EC_ARG2, offsetof(struct efirt_callinfo, ec_arg2)); +ASSYM(EC_ARG3, offsetof(struct efirt_callinfo, ec_arg3)); +ASSYM(EC_ARG4, offsetof(struct efirt_callinfo, ec_arg4)); +ASSYM(EC_ARG5, offsetof(struct efirt_callinfo, ec_arg5)); +ASSYM(EC_RBX, offsetof(struct efirt_callinfo, ec_rbx)); +ASSYM(EC_RSP, offsetof(struct efirt_callinfo, ec_rsp)); +ASSYM(EC_RBP, offsetof(struct efirt_callinfo, ec_rbp)); +ASSYM(EC_R12, offsetof(struct efirt_callinfo, ec_r12)); +ASSYM(EC_R13, offsetof(struct efirt_callinfo, ec_r13)); +ASSYM(EC_R14, offsetof(struct efirt_callinfo, ec_r14)); +ASSYM(EC_R15, offsetof(struct efirt_callinfo, ec_r15)); Index: sys/amd64/amd64/trap.c =================================================================== --- sys/amd64/amd64/trap.c +++ sys/amd64/amd64/trap.c @@ -806,7 +806,7 @@ * If nx protection of the usermode portion of kernel page * tables caused trap, panic. */ - if (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 && usermode && + if (usermode && PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 && pg_nx != 0 && (frame->tf_err & (PGEX_P | PGEX_W | PGEX_U | PGEX_I)) == (PGEX_P | PGEX_U | PGEX_I) && (curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK)== Index: sys/amd64/include/efi.h =================================================================== --- sys/amd64/include/efi.h +++ sys/amd64/include/efi.h @@ -48,9 +48,30 @@ #ifdef _KERNEL #include -#define EFI_TIME_LOCK() mtx_lock(&atrtc_time_lock); -#define EFI_TIME_UNLOCK() mtx_unlock(&atrtc_time_lock); -#define EFI_TIME_OWNED() mtx_assert(&atrtc_time_lock, MA_OWNED); +#define EFI_TIME_LOCK() mtx_lock(&atrtc_time_lock) +#define EFI_TIME_UNLOCK() mtx_unlock(&atrtc_time_lock) +#define EFI_TIME_OWNED() mtx_assert(&atrtc_time_lock, MA_OWNED) + +#define EFI_RT_HANDLE_FAULTS_DEFAULT 1 #endif +struct efirt_callinfo { + const char *ec_name; + register_t ec_efi_status; + register_t ec_fptr; + register_t ec_argcnt; + register_t ec_arg1; + register_t ec_arg2; + register_t ec_arg3; + register_t ec_arg4; + register_t ec_arg5; + register_t ec_rbx; + register_t ec_rsp; + register_t ec_rbp; + register_t ec_r12; + register_t ec_r13; + register_t ec_r14; + register_t ec_r15; +}; + #endif /* __AMD64_INCLUDE_EFI_H_ */ Index: sys/amd64/include/proc.h =================================================================== --- sys/amd64/include/proc.h +++ sys/amd64/include/proc.h @@ -62,6 +62,8 @@ register_t md_saved_flags; /* (k) */ register_t md_spurflt_addr; /* (k) Spurious page fault address. */ struct pmap_invl_gen md_invl_gen; + register_t md_efirt_tmp; /* (k) */ + int md_efirt_dis_pf; /* (k) */ }; struct mdproc { Index: sys/arm64/arm64/efirt_machdep.c =================================================================== --- sys/arm64/arm64/efirt_machdep.c +++ sys/arm64/arm64/efirt_machdep.c @@ -268,3 +268,10 @@ "isb \n" : : "r"(td->td_proc->p_md.md_l0addr)); } + +int +efi_rt_arch_call(struct efirt_callinfo *ec) +{ + + panic("not implemented"); +} Index: sys/arm64/include/efi.h =================================================================== --- sys/arm64/include/efi.h +++ sys/arm64/include/efi.h @@ -39,6 +39,20 @@ #define EFI_TIME_LOCK() #define EFI_TIME_UNLOCK() #define EFI_TIME_OWNED() + +#define EFI_RT_HANDLE_FAULTS_DEFAULT 0 #endif +struct efirt_callinfo { + const char *ec_name; + register_t ec_efi_status; + register_t ec_fptr; + register_t ec_argcnt; + register_t ec_arg1; + register_t ec_arg2; + register_t ec_arg3; + register_t ec_arg4; + register_t ec_arg5; +}; + #endif /* __ARM64_INCLUDE_EFI_H_ */ Index: sys/conf/files.amd64 =================================================================== --- sys/conf/files.amd64 +++ sys/conf/files.amd64 @@ -134,6 +134,7 @@ amd64/amd64/db_interface.c optional ddb amd64/amd64/db_trace.c optional ddb amd64/amd64/efirt_machdep.c optional efirt +amd64/amd64/efirt_support.S optional efirt amd64/amd64/elf_machdep.c standard amd64/amd64/exception.S standard amd64/amd64/fpu.c standard Index: sys/dev/efidev/efirt.c =================================================================== --- sys/dev/efidev/efirt.c +++ sys/dev/efidev/efirt.c @@ -293,22 +293,89 @@ return (ENOENT); } +static int efi_rt_handle_faults = EFI_RT_HANDLE_FAULTS_DEFAULT; +SYSCTL_INT(_machdep, OID_AUTO, efi_rt_handle_faults, CTLFLAG_RWTUN, + &efi_rt_handle_faults, 0, + "Call EFI RT methods with fault handler wrapper around"); + static int -efi_get_time_locked(struct efi_tm *tm, struct efi_tmcap *tmcap) +efi_rt_arch_call_nofault(struct efirt_callinfo *ec) +{ + + switch (ec->ec_argcnt) { + case 0: + ec->ec_efi_status = ((register_t (*)(void))ec->ec_fptr)(); + break; + case 1: + ec->ec_efi_status = ((register_t (*)(register_t))ec->ec_fptr) + (ec->ec_arg1); + break; + case 2: + ec->ec_efi_status = ((register_t (*)(register_t, register_t)) + ec->ec_fptr)(ec->ec_arg1, ec->ec_arg2); + break; + case 3: + ec->ec_efi_status = ((register_t (*)(register_t, register_t, + register_t))ec->ec_fptr)(ec->ec_arg1, ec->ec_arg2, + ec->ec_arg3); + break; + case 4: + ec->ec_efi_status = ((register_t (*)(register_t, register_t, + register_t, register_t))ec->ec_fptr)(ec->ec_arg1, + ec->ec_arg2, ec->ec_arg3, ec->ec_arg4); + break; + case 5: + ec->ec_efi_status = ((register_t (*)(register_t, register_t, + register_t, register_t, register_t))ec->ec_fptr)( + ec->ec_arg1, ec->ec_arg2, ec->ec_arg3, ec->ec_arg4, + ec->ec_arg5); + break; + default: + panic("efi_rt_arch_call: %d args", (int)ec->ec_argcnt); + } + + return (0); +} + +static int +efi_call(struct efirt_callinfo *ecp) { - efi_status status; int error; - EFI_TIME_OWNED() error = efi_enter(); if (error != 0) return (error); - status = efi_runtime->rt_gettime(tm, tmcap); + error = efi_rt_handle_faults ? efi_rt_arch_call(ecp) : + efi_rt_arch_call_nofault(ecp); efi_leave(); - error = efi_status_to_errno(status); + if (error == 0) + error = efi_status_to_errno(ecp->ec_efi_status); + else if (bootverbose) + printf("EFI %s call faulted, error %d\n", ecp->ec_name, error); return (error); } +#define EFI_RT_METHOD_PA(method) \ + ((uintptr_t)((struct efi_rt *)efi_phys_to_kva((uintptr_t) \ + efi_runtime))->method) + +static int +efi_get_time_locked(struct efi_tm *tm, struct efi_tmcap *tmcap) +{ + struct efirt_callinfo ec; + + EFI_TIME_OWNED(); + if (efi_runtime == NULL) + return (ENXIO); + bzero(&ec, sizeof(ec)); + ec.ec_name = "rt_gettime"; + ec.ec_argcnt = 2; + ec.ec_arg1 = (uintptr_t)tm; + ec.ec_arg2 = (uintptr_t)tmcap; + ec.ec_fptr = EFI_RT_METHOD_PA(rt_gettime); + return (efi_call(&ec)); +} + int efi_get_time(struct efi_tm *tm) { @@ -317,7 +384,7 @@ if (efi_runtime == NULL) return (ENXIO); - EFI_TIME_LOCK() + EFI_TIME_LOCK(); /* * UEFI spec states that the Capabilities argument to GetTime is * optional, but some UEFI implementations choke when passed a NULL @@ -325,7 +392,7 @@ * to workaround such implementations. */ error = efi_get_time_locked(tm, &dummy); - EFI_TIME_UNLOCK() + EFI_TIME_UNLOCK(); return (error); } @@ -337,39 +404,44 @@ if (efi_runtime == NULL) return (ENXIO); - EFI_TIME_LOCK() + EFI_TIME_LOCK(); error = efi_get_time_locked(&dummy, tmcap); - EFI_TIME_UNLOCK() + EFI_TIME_UNLOCK(); return (error); } int efi_reset_system(void) { - int error; + struct efirt_callinfo ec; - error = efi_enter(); - if (error != 0) - return (error); - efi_runtime->rt_reset(EFI_RESET_WARM, 0, 0, NULL); - efi_leave(); - return (EIO); + if (efi_runtime == NULL) + return (ENXIO); + bzero(&ec, sizeof(ec)); + ec.ec_name = "rt_reset"; + ec.ec_argcnt = 4; + ec.ec_arg1 = (uintptr_t)EFI_RESET_WARM; + ec.ec_arg1 = (uintptr_t)0; + ec.ec_arg1 = (uintptr_t)0; + ec.ec_arg1 = (uintptr_t)NULL; + ec.ec_fptr = EFI_RT_METHOD_PA(rt_reset); + return (efi_call(&ec)); } static int efi_set_time_locked(struct efi_tm *tm) { - efi_status status; - int error; + struct efirt_callinfo ec; EFI_TIME_OWNED(); - error = efi_enter(); - if (error != 0) - return (error); - status = efi_runtime->rt_settime(tm); - efi_leave(); - error = efi_status_to_errno(status); - return (error); + if (efi_runtime == NULL) + return (ENXIO); + bzero(&ec, sizeof(ec)); + ec.ec_name = "rt_settime"; + ec.ec_argcnt = 1; + ec.ec_arg1 = (uintptr_t)tm; + ec.ec_fptr = EFI_RT_METHOD_PA(rt_settime); + return (efi_call(&ec)); } int @@ -379,9 +451,9 @@ if (efi_runtime == NULL) return (ENXIO); - EFI_TIME_LOCK() + EFI_TIME_LOCK(); error = efi_set_time_locked(tm); - EFI_TIME_UNLOCK() + EFI_TIME_UNLOCK(); return (error); } @@ -389,47 +461,57 @@ efi_var_get(efi_char *name, struct uuid *vendor, uint32_t *attrib, size_t *datasize, void *data) { - efi_status status; - int error; + struct efirt_callinfo ec; - error = efi_enter(); - if (error != 0) - return (error); - status = efi_runtime->rt_getvar(name, vendor, attrib, datasize, data); - efi_leave(); - error = efi_status_to_errno(status); - return (error); + if (efi_runtime == NULL) + return (ENXIO); + bzero(&ec, sizeof(ec)); + ec.ec_argcnt = 5; + ec.ec_name = "rt_getvar"; + ec.ec_arg1 = (uintptr_t)name; + ec.ec_arg2 = (uintptr_t)vendor; + ec.ec_arg3 = (uintptr_t)attrib; + ec.ec_arg4 = (uintptr_t)datasize; + ec.ec_arg5 = (uintptr_t)data; + ec.ec_fptr = EFI_RT_METHOD_PA(rt_getvar); + return (efi_call(&ec)); } int efi_var_nextname(size_t *namesize, efi_char *name, struct uuid *vendor) { - efi_status status; - int error; + struct efirt_callinfo ec; - error = efi_enter(); - if (error != 0) - return (error); - status = efi_runtime->rt_scanvar(namesize, name, vendor); - efi_leave(); - error = efi_status_to_errno(status); - return (error); + if (efi_runtime == NULL) + return (ENXIO); + bzero(&ec, sizeof(ec)); + ec.ec_argcnt = 3; + ec.ec_name = "rt_scanvar"; + ec.ec_arg1 = (uintptr_t)namesize; + ec.ec_arg2 = (uintptr_t)name; + ec.ec_arg3 = (uintptr_t)vendor; + ec.ec_fptr = EFI_RT_METHOD_PA(rt_scanvar); + return (efi_call(&ec)); } int efi_var_set(efi_char *name, struct uuid *vendor, uint32_t attrib, size_t datasize, void *data) { - efi_status status; - int error; + struct efirt_callinfo ec; - error = efi_enter(); - if (error != 0) - return (error); - status = efi_runtime->rt_setvar(name, vendor, attrib, datasize, data); - efi_leave(); - error = efi_status_to_errno(status); - return (error); + if (efi_runtime == NULL) + return (ENXIO); + bzero(&ec, sizeof(ec)); + ec.ec_argcnt = 5; + ec.ec_name = "rt_setvar"; + ec.ec_arg1 = (uintptr_t)name; + ec.ec_arg2 = (uintptr_t)vendor; + ec.ec_arg3 = (uintptr_t)attrib; + ec.ec_arg4 = (uintptr_t)datasize; + ec.ec_arg5 = (uintptr_t)data; + ec.ec_fptr = EFI_RT_METHOD_PA(rt_setvar); + return (efi_call(&ec)); } static int Index: sys/dev/efidev/efirtc.c =================================================================== --- sys/dev/efidev/efirtc.c +++ sys/dev/efidev/efirtc.c @@ -74,7 +74,8 @@ */ if ((error = efi_get_time(&tm)) != 0) { if (bootverbose) - device_printf(dev, "cannot read EFI realtime clock\n"); + device_printf(dev, "cannot read EFI realtime clock, " + "error %d\n", error); return (error); } device_set_desc(dev, "EFI Realtime Clock"); Index: sys/kern/kern_thread.c =================================================================== --- sys/kern/kern_thread.c +++ sys/kern/kern_thread.c @@ -83,7 +83,7 @@ "struct thread KBI td_pflags"); _Static_assert(offsetof(struct thread, td_frame) == 0x470, "struct thread KBI td_frame"); -_Static_assert(offsetof(struct thread, td_emuldata) == 0x518, +_Static_assert(offsetof(struct thread, td_emuldata) == 0x528, "struct thread KBI td_emuldata"); _Static_assert(offsetof(struct proc, p_flag) == 0xb0, "struct proc KBI p_flag"); Index: sys/kern/subr_rtc.c =================================================================== --- sys/kern/subr_rtc.c +++ sys/kern/subr_rtc.c @@ -138,6 +138,7 @@ { struct timespec ts; struct rtc_instance *rtc; + int error; rtc = arg; if (!(rtc->flags & CLOCKF_SETTIME_NO_TS)) { @@ -150,7 +151,9 @@ ts.tv_sec = 0; ts.tv_nsec = 0; } - CLOCK_SETTIME(rtc->clockdev, &ts); + error = CLOCK_SETTIME(rtc->clockdev, &ts); + if (error != 0 && bootverbose) + device_printf(rtc->clockdev, "CLOCK_SETTIME error %d\n", error); } static void Index: sys/modules/efirt/Makefile =================================================================== --- sys/modules/efirt/Makefile +++ sys/modules/efirt/Makefile @@ -8,4 +8,11 @@ SRCS+= efirtc.c SRCS+= device_if.h bus_if.h clock_if.h +.if ${MACHINE_CPUARCH} == "amd64" +SRCS+= efirt_support.S +efirt_support.o: efirt_support.S assym.inc + ${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \ + ${.IMPSRC} -o ${.TARGET} +.endif + .include Index: sys/sys/efi.h =================================================================== --- sys/sys/efi.h +++ sys/sys/efi.h @@ -173,6 +173,7 @@ int efi_arch_enter(void); void efi_arch_leave(void); vm_offset_t efi_phys_to_kva(vm_paddr_t); +int efi_rt_arch_call(struct efirt_callinfo *); bool efi_create_1t1_map(struct efi_md *, int, int); void efi_destroy_1t1_map(void);