diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -159,7 +159,7 @@ INTR_HANDLER xen_intr_upcall KMSAN_ENTER movq %rsp, %rdi - call xen_intr_handle_upcall + call xen_arch_intr_handle_upcall KMSAN_LEAVE jmp doreti #endif diff --git a/sys/conf/files.x86 b/sys/conf/files.x86 --- a/sys/conf/files.x86 +++ b/sys/conf/files.x86 @@ -347,3 +347,4 @@ x86/xen/hvm.c optional xenhvm x86/xen/xen_intr.c optional xenhvm x86/xen/xen_apic.c optional xenhvm smp +x86/xen/xen_arch_intr.c optional xenhvm diff --git a/sys/dev/xen/xenpci/xenpci.c b/sys/dev/xen/xenpci/xenpci.c --- a/sys/dev/xen/xenpci/xenpci.c +++ b/sys/dev/xen/xenpci/xenpci.c @@ -52,13 +52,6 @@ #include -static int -xenpci_intr_filter(void *trap_frame) -{ - xen_intr_handle_upcall(trap_frame); - return (FILTER_HANDLED); -} - static int xenpci_irq_init(device_t device, struct xenpci_softc *scp) { @@ -66,7 +59,7 @@ error = BUS_SETUP_INTR(device_get_parent(device), device, scp->res_irq, INTR_MPSAFE|INTR_TYPE_MISC, - xenpci_intr_filter, NULL, /*trap_frame*/NULL, + xen_intr_handle_upcall, NULL, NULL, &scp->intr_cookie); if (error) return error; diff --git a/sys/i386/i386/apic_vector.S b/sys/i386/i386/apic_vector.S --- a/sys/i386/i386/apic_vector.S +++ b/sys/i386/i386/apic_vector.S @@ -183,7 +183,7 @@ cld KENTER pushl %esp - movl $xen_intr_handle_upcall, %eax + movl $xen_arch_intr_handle_upcall, %eax call *%eax add $4, %esp jmp doreti diff --git a/sys/x86/xen/xen_arch_intr.c b/sys/x86/xen/xen_arch_intr.c new file mode 100644 --- /dev/null +++ b/sys/x86/xen/xen_arch_intr.c @@ -0,0 +1,103 @@ +/*- + * SPDX-License-Identifier: MIT OR GPL-2.0-only + * + * Copyright © 2013 Spectra Logic Corporation + * Copyright © 2018 John Baldwin/The FreeBSD Foundation + * Copyright © 2019 Roger Pau Monné/Citrix Systems R&D + * Copyright © 2021 Elliott Mitchell + * + * This file may be distributed separately from the Linux kernel, or + * incorporated into other software packages, subject to the following license: + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this source file (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +/************************ Xen x86 interrupt interface ************************/ + +/* + * Pointers to the interrupt counters + */ +DPCPU_DEFINE_STATIC(u_long *, pintrcnt); + +static void +xen_intrcnt_init(void *dummy __unused) +{ + unsigned int i; + + if (!xen_domain()) + return; + + CPU_FOREACH(i) { + char buf[MAXCOMLEN + 1]; + + snprintf(buf, sizeof(buf), "cpu%d:xen", i); + intrcnt_add(buf, DPCPU_ID_PTR(i, pintrcnt)); + } +} +SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL); + +/* + * Transition from assembly language, called from + * sys/{amd64/amd64|i386/i386}/apic_vector.S + */ +extern void xen_arch_intr_handle_upcall(struct trapframe *); +void +xen_arch_intr_handle_upcall(struct trapframe *trap_frame) +{ + struct trapframe *old; + + /* + * Disable preemption in order to always check and fire events + * on the right vCPU + */ + critical_enter(); + + ++*DPCPU_GET(pintrcnt); + + ++curthread->td_intr_nesting_level; + old = curthread->td_intr_frame; + curthread->td_intr_frame = trap_frame; + + xen_intr_handle_upcall(NULL); + + curthread->td_intr_frame = old; + --curthread->td_intr_nesting_level; + + if (xen_evtchn_needs_ack) + lapic_eoi(); + + critical_exit(); +} diff --git a/sys/x86/xen/xen_intr.c b/sys/x86/xen/xen_intr.c --- a/sys/x86/xen/xen_intr.c +++ b/sys/x86/xen/xen_intr.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include @@ -56,7 +57,6 @@ #include #include -#include #include #include #include @@ -102,9 +102,6 @@ */ u_int last_processed_l2i; - /** Pointer to this CPU's interrupt statistic counter. */ - u_long *evtchn_intrcnt; - /** * A bitmap of ports that can be serviced from this CPU. * A set bit means interrupt handling is enabled. @@ -261,25 +258,6 @@ xen_set_bit(port, pcpu->evtchn_enabled); } -/** - * Allocate and register a per-cpu Xen upcall interrupt counter. - * - * \param cpu The cpu for which to register this interrupt count. - */ -static void -xen_intr_intrcnt_add(u_int cpu) -{ - char buf[MAXCOMLEN + 1]; - struct xen_intr_pcpu_data *pcpu; - - pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); - if (pcpu->evtchn_intrcnt != NULL) - return; - - snprintf(buf, sizeof(buf), "cpu%d:xen", cpu); - intrcnt_add(buf, &pcpu->evtchn_intrcnt); -} - /** * Search for an already allocated but currently unused Xen interrupt * source object. @@ -526,9 +504,10 @@ * * \param trap_frame The trap frame context for the current interrupt. */ -void -xen_intr_handle_upcall(struct trapframe *trap_frame) +int +xen_intr_handle_upcall(void *unused __unused) { + struct trapframe *trap_frame = curthread->td_intr_frame; u_int l1i, l2i, port, cpu __diagused; u_long masked_l1, masked_l2; struct xenisrc *isrc; @@ -536,11 +515,8 @@ struct xen_intr_pcpu_data *pc; u_long l1, l2; - /* - * Disable preemption in order to always check and fire events - * on the right vCPU - */ - critical_enter(); + /* We must remain on the same vCPU during this function */ + CRITICAL_ASSERT(curthread); cpu = PCPU_GET(cpuid); pc = DPCPU_PTR(xen_intr_pcpu); @@ -551,19 +527,15 @@ } v->evtchn_upcall_pending = 0; - -#if 0 -#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */ +/* No need for a barrier on x86 -- XCHG is a barrier on x86. */ +#if !defined(__amd64__) && !defined(__i386__) /* Clear master flag /before/ clearing selector flag. */ wmb(); #endif -#endif - l1 = atomic_readandclear_long(&v->evtchn_pending_sel); l1i = pc->last_processed_l1i; l2i = pc->last_processed_l2i; - (*pc->evtchn_intrcnt)++; while (l1 != 0) { l1i = (l1i + 1) % LONG_BIT; @@ -627,10 +599,7 @@ } } - if (xen_evtchn_needs_ack) - lapic_eoi(); - - critical_exit(); + return (FILTER_HANDLED); } static int @@ -682,23 +651,6 @@ } SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL); -static void -xen_intrcnt_init(void *dummy __unused) -{ - unsigned int i; - - if (!xen_domain()) - return; - - /* - * Register interrupt count manually as we aren't guaranteed to see a - * call to xen_intr_assign_cpu() before our first interrupt. - */ - CPU_FOREACH(i) - xen_intr_intrcnt_add(i); -} -SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL); - void xen_intr_alloc_irqs(void) { diff --git a/sys/xen/xen_intr.h b/sys/xen/xen_intr.h --- a/sys/xen/xen_intr.h +++ b/sys/xen/xen_intr.h @@ -38,7 +38,10 @@ /** Registered Xen interrupt callback handle. */ typedef void * xen_intr_handle_t; -void xen_intr_handle_upcall(struct trapframe *trap_frame); +/* + * Main handler for Xen event channel interrupts + */ +extern driver_filter_t xen_intr_handle_upcall; /** * Associate an already allocated local event channel port an interrupt