diff --git a/sys/x86/xen/hvm.c b/sys/x86/xen/hvm.c index 85f274175ac6..6336602c8bc4 100644 --- a/sys/x86/xen/hvm.c +++ b/sys/x86/xen/hvm.c @@ -1,633 +1,633 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2008, 2013 Citrix Systems, Inc. * Copyright (c) 2012 Spectra Logic Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*--------------------------- Forward Declarations ---------------------------*/ static void xen_hvm_cpu_init(void); /*-------------------------------- Global Data -------------------------------*/ #ifdef SMP struct cpu_ops xen_hvm_cpu_ops = { .cpu_init = xen_hvm_cpu_init, .cpu_resume = xen_hvm_cpu_init }; #endif static MALLOC_DEFINE(M_XENHVM, "xen_hvm", "Xen HVM PV Support"); /** * If non-zero, the hypervisor has been configured to use a direct * IDT event callback for interrupt injection. */ int xen_vector_callback_enabled; /** * Signal whether the vector injected for the event channel upcall requires to * be EOI'ed on the local APIC. */ bool xen_evtchn_needs_ack; /*------------------------------- Per-CPU Data -------------------------------*/ DPCPU_DECLARE(struct vcpu_info *, vcpu_info); /*------------------------------ Sysctl tunables -----------------------------*/ int xen_disable_pv_disks = 0; int xen_disable_pv_nics = 0; TUNABLE_INT("hw.xen.disable_pv_disks", &xen_disable_pv_disks); TUNABLE_INT("hw.xen.disable_pv_nics", &xen_disable_pv_nics); /*---------------------- XEN Hypervisor Probe and Setup ----------------------*/ void xen_emergency_print(const char *str, size_t size) { outsb(XEN_HVM_DEBUGCONS_IOPORT, str, size); } uint32_t xen_cpuid_base; static uint32_t xen_hvm_cpuid_base(void) { uint32_t base, regs[4]; for (base = 0x40000000; base < 0x40010000; base += 0x100) { do_cpuid(base, regs); if (!memcmp("XenVMMXenVMM", ®s[1], 12) && (regs[0] - base) >= 2) return (base); } return (0); } static void hypervisor_quirks(unsigned int major, unsigned int minor) { #ifdef SMP if (((major < 4) || (major == 4 && minor <= 5)) && msix_disable_migration == -1) { /* * Xen hypervisors prior to 4.6.0 do not properly * handle updates to enabled MSI-X table entries, * so disable MSI-X interrupt migration in that * case. */ if (bootverbose) printf( "Disabling MSI-X interrupt migration due to Xen hypervisor bug.\n" "Set machdep.msix_disable_migration=0 to forcefully enable it.\n"); msix_disable_migration = 1; } #endif } static void hypervisor_version(void) { uint32_t regs[4]; int major, minor; - do_cpuid(xen_cpuid_base + 1, regs); + do_cpuid(hv_base + 1, regs); major = regs[0] >> 16; minor = regs[0] & 0xffff; printf("XEN: Hypervisor version %d.%d detected.\n", major, minor); hypervisor_quirks(major, minor); } /* * Allocate and fill in the hypcall page. */ int xen_hvm_init_hypercall_stubs(enum xen_hvm_init_type init_type) { uint32_t regs[4]; if (xen_cpuid_base != 0) /* Already setup. */ goto out; xen_cpuid_base = xen_hvm_cpuid_base(); if (xen_cpuid_base == 0) return (ENXIO); /* * Find the hypercall pages. */ do_cpuid(xen_cpuid_base + 2, regs); if (regs[0] != 1) return (EINVAL); wrmsr(regs[1], (init_type == XEN_HVM_INIT_EARLY) ? (vm_paddr_t)((uintptr_t)&hypercall_page - KERNBASE) : vtophys(&hypercall_page)); out: hypervisor_version(); return (0); } /* * Translate linear to physical address when still running on the bootloader * created page-tables. */ static vm_paddr_t early_init_vtop(void *addr) { /* * Using a KASSERT won't print anything, as this is before console * initialization. */ if (__predict_false((uintptr_t)addr < KERNBASE)) { xc_printf("invalid linear address: %#lx\n", (uintptr_t)addr); halt(); } return ((uintptr_t)addr - KERNBASE #ifdef __amd64__ + kernphys - KERNLOAD #endif ); } static int map_shared_info(void) { /* * TODO shared info page should be mapped in an unpopulated (IOW: * non-RAM) address. But finding one at this point in boot is * complicated, hence re-use a RAM address for the time being. This * sadly causes super-page shattering in the second stage translation * page tables. */ static union { shared_info_t shared_info; uint8_t raw[PAGE_SIZE]; } shared_page __attribute__((aligned(PAGE_SIZE))); static struct xen_add_to_physmap xatp = { .domid = DOMID_SELF, .space = XENMAPSPACE_shared_info, }; int rc; _Static_assert(sizeof(shared_page) == PAGE_SIZE, "invalid Xen shared_info struct size"); if (xatp.gpfn == 0) xatp.gpfn = atop(early_init_vtop(&shared_page.shared_info)); rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp); if (rc != 0) { xc_printf("cannot map shared info page: %d\n", rc); HYPERVISOR_shared_info = NULL; } else if (HYPERVISOR_shared_info == NULL) HYPERVISOR_shared_info = &shared_page.shared_info; return (rc); } static void fixup_console(void) { struct xen_platform_op op = { .cmd = XENPF_get_dom0_console, }; xenpf_dom0_console_t *console = &op.u.dom0_console; union { struct efi_fb efi; struct vbe_fb vbe; } *fb = NULL; int size; caddr_t kmdp; kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); if (kmdp == NULL) { xc_printf("Unable to find kernel metadata\n"); return; } size = HYPERVISOR_platform_op(&op); if (size < 0) { xc_printf("Failed to get video console info: %d\n", size); return; } switch (console->video_type) { case XEN_VGATYPE_VESA_LFB: fb = (__typeof__ (fb))preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_VBE_FB); if (fb == NULL) { xc_printf("No VBE FB in kernel metadata\n"); return; } _Static_assert(offsetof(struct vbe_fb, fb_bpp) == offsetof(struct efi_fb, fb_mask_reserved) + sizeof(fb->efi.fb_mask_reserved), "Bad structure overlay\n"); fb->vbe.fb_bpp = console->u.vesa_lfb.bits_per_pixel; /* FALLTHROUGH */ case XEN_VGATYPE_EFI_LFB: if (fb == NULL) { fb = (__typeof__ (fb))preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_FB); if (fb == NULL) { xc_printf("No EFI FB in kernel metadata\n"); return; } } fb->efi.fb_addr = console->u.vesa_lfb.lfb_base; if (size > offsetof(xenpf_dom0_console_t, u.vesa_lfb.ext_lfb_base)) fb->efi.fb_addr |= (uint64_t)console->u.vesa_lfb.ext_lfb_base << 32; fb->efi.fb_size = console->u.vesa_lfb.lfb_size << 16; fb->efi.fb_height = console->u.vesa_lfb.height; fb->efi.fb_width = console->u.vesa_lfb.width; fb->efi.fb_stride = (console->u.vesa_lfb.bytes_per_line << 3) / console->u.vesa_lfb.bits_per_pixel; #define FBMASK(c) \ ((~0u << console->u.vesa_lfb.c ## _pos) & \ (~0u >> (32 - console->u.vesa_lfb.c ## _pos - \ console->u.vesa_lfb.c ## _size))) fb->efi.fb_mask_red = FBMASK(red); fb->efi.fb_mask_green = FBMASK(green); fb->efi.fb_mask_blue = FBMASK(blue); fb->efi.fb_mask_reserved = FBMASK(rsvd); #undef FBMASK break; default: xc_printf("Video console type unsupported\n"); return; } } /* Early initialization when running as a Xen guest. */ void xen_early_init(void) { uint32_t regs[4]; int rc; xen_cpuid_base = xen_hvm_cpuid_base(); if (xen_cpuid_base == 0) return; /* Find the hypercall pages. */ do_cpuid(xen_cpuid_base + 2, regs); if (regs[0] != 1) { xc_printf("Invalid number of hypercall pages %u\n", regs[0]); vm_guest = VM_GUEST_VM; return; } wrmsr(regs[1], early_init_vtop(&hypercall_page)); rc = map_shared_info(); if (rc != 0) { vm_guest = VM_GUEST_VM; return; } if (xen_initial_domain()) /* Fixup video console information in case Xen changed the mode. */ fixup_console(); } static void xen_hvm_init_shared_info_page(void) { struct xen_add_to_physmap xatp; if (xen_pv_domain()) { /* * Already setup in the PV case, shared_info is passed inside * of the start_info struct at start of day. */ return; } if (HYPERVISOR_shared_info == NULL) { HYPERVISOR_shared_info = malloc(PAGE_SIZE, M_XENHVM, M_NOWAIT); if (HYPERVISOR_shared_info == NULL) panic("Unable to allocate Xen shared info page"); } xatp.domid = DOMID_SELF; xatp.idx = 0; xatp.space = XENMAPSPACE_shared_info; xatp.gpfn = vtophys(HYPERVISOR_shared_info) >> PAGE_SHIFT; if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp)) panic("HYPERVISOR_memory_op failed"); } static int set_percpu_callback(unsigned int vcpu) { struct xen_hvm_evtchn_upcall_vector vec; int error; vec.vcpu = vcpu; vec.vector = IDT_EVTCHN; error = HYPERVISOR_hvm_op(HVMOP_set_evtchn_upcall_vector, &vec); return (error != 0 ? xen_translate_error(error) : 0); } /* * Tell the hypervisor how to contact us for event channel callbacks. */ void xen_hvm_set_callback(device_t dev) { struct xen_hvm_param xhp; int irq; if (xen_vector_callback_enabled) return; xhp.domid = DOMID_SELF; xhp.index = HVM_PARAM_CALLBACK_IRQ; if (xen_feature(XENFEAT_hvm_callback_vector) != 0) { int error; error = set_percpu_callback(0); if (error == 0) { xen_evtchn_needs_ack = true; /* Trick toolstack to think we are enlightened */ xhp.value = 1; } else xhp.value = HVM_CALLBACK_VECTOR(IDT_EVTCHN); error = HYPERVISOR_hvm_op(HVMOP_set_param, &xhp); if (error == 0) { xen_vector_callback_enabled = 1; return; } else if (xen_evtchn_needs_ack) panic("Unable to setup fake HVM param: %d", error); printf("Xen HVM callback vector registration failed (%d). " "Falling back to emulated device interrupt\n", error); } xen_vector_callback_enabled = 0; if (dev == NULL) { /* * Called from early boot or resume. * xenpci will invoke us again later. */ return; } irq = pci_get_irq(dev); if (irq < 16) { xhp.value = HVM_CALLBACK_GSI(irq); } else { u_int slot; u_int pin; slot = pci_get_slot(dev); pin = pci_get_intpin(dev) - 1; xhp.value = HVM_CALLBACK_PCI_INTX(slot, pin); } if (HYPERVISOR_hvm_op(HVMOP_set_param, &xhp) != 0) panic("Can't set evtchn callback"); } #define XEN_MAGIC_IOPORT 0x10 enum { XMI_MAGIC = 0x49d2, XMI_UNPLUG_IDE_DISKS = 0x01, XMI_UNPLUG_NICS = 0x02, XMI_UNPLUG_IDE_EXCEPT_PRI_MASTER = 0x04 }; static void xen_hvm_disable_emulated_devices(void) { u_short disable_devs = 0; if (xen_pv_domain()) { /* * No emulated devices in the PV case, so no need to unplug * anything. */ if (xen_disable_pv_disks != 0 || xen_disable_pv_nics != 0) printf("PV devices cannot be disabled in PV guests\n"); return; } if (inw(XEN_MAGIC_IOPORT) != XMI_MAGIC) return; if (xen_disable_pv_disks == 0) { if (bootverbose) printf("XEN: disabling emulated disks\n"); disable_devs |= XMI_UNPLUG_IDE_DISKS; } if (xen_disable_pv_nics == 0) { if (bootverbose) printf("XEN: disabling emulated nics\n"); disable_devs |= XMI_UNPLUG_NICS; } if (disable_devs != 0) outw(XEN_MAGIC_IOPORT, disable_devs); } static void xen_hvm_init(enum xen_hvm_init_type init_type) { int error; int i; if (!xen_domain() || init_type == XEN_HVM_INIT_CANCELLED_SUSPEND) return; error = xen_hvm_init_hypercall_stubs(init_type); switch (init_type) { case XEN_HVM_INIT_LATE: if (error != 0) return; setup_xen_features(); #ifdef SMP cpu_ops = xen_hvm_cpu_ops; #endif break; case XEN_HVM_INIT_RESUME: if (error != 0) panic("Unable to init Xen hypercall stubs on resume"); /* Clear stale vcpu_info. */ CPU_FOREACH(i) DPCPU_ID_SET(i, vcpu_info, NULL); break; default: panic("Unsupported HVM initialization type"); } xen_vector_callback_enabled = 0; xen_evtchn_needs_ack = false; xen_hvm_set_callback(NULL); /* * On (PV)HVM domains we need to request the hypervisor to * fill the shared info page, for PVH guest the shared_info page * is passed inside the start_info struct and is already set, so this * functions are no-ops. */ xen_hvm_init_shared_info_page(); xen_hvm_disable_emulated_devices(); } void xen_hvm_suspend(void) { } void xen_hvm_resume(bool suspend_cancelled) { xen_hvm_init(suspend_cancelled ? XEN_HVM_INIT_CANCELLED_SUSPEND : XEN_HVM_INIT_RESUME); /* Register vcpu_info area for CPU#0. */ xen_hvm_cpu_init(); } static void xen_hvm_sysinit(void *arg __unused) { xen_hvm_init(XEN_HVM_INIT_LATE); } SYSINIT(xen_hvm_init, SI_SUB_HYPERVISOR, SI_ORDER_FIRST, xen_hvm_sysinit, NULL); static void xen_hvm_cpu_init(void) { uint32_t regs[4]; int rc; if (!xen_domain()) return; if (DPCPU_GET(vcpu_info) != NULL) { /* * vcpu_info is already set. We're resuming * from a failed migration and our pre-suspend * configuration is still valid. */ return; } /* * Set vCPU ID. If available fetch the ID from CPUID, if not just use * the ACPI ID. */ - KASSERT(xen_cpuid_base != 0, ("Invalid base Xen CPUID leaf")); - cpuid_count(xen_cpuid_base + 4, 0, regs); + KASSERT(hv_base != 0, ("Invalid base Xen CPUID leaf")); + cpuid_count(hv_base + 4, 0, regs); KASSERT((regs[0] & XEN_HVM_CPUID_VCPU_ID_PRESENT) || !xen_pv_domain(), ("Xen PV domain without vcpu_id in cpuid")); PCPU_SET(vcpu_id, (regs[0] & XEN_HVM_CPUID_VCPU_ID_PRESENT) ? regs[1] : PCPU_GET(acpi_id)); if (xen_evtchn_needs_ack && !IS_BSP()) { /* * Setup the per-vpcu event channel upcall vector. This is only * required when using the new HVMOP_set_evtchn_upcall_vector * hypercall, which allows using a different vector for each * vCPU. Note that FreeBSD uses the same vector for all vCPUs * because it's not dynamically allocated. */ rc = set_percpu_callback(PCPU_GET(vcpu_id)); if (rc != 0) panic("Event channel upcall vector setup failed: %d", rc); } xen_setup_vcpu_info(); } SYSINIT(xen_hvm_cpu_init, SI_SUB_INTR, SI_ORDER_FIRST, xen_hvm_cpu_init, NULL); bool xen_has_iommu_maps(void) { uint32_t regs[4]; - KASSERT(xen_cpuid_base != 0, ("Invalid base Xen CPUID leaf")); - cpuid_count(xen_cpuid_base + 4, 0, regs); + KASSERT(hv_base != 0, ("Invalid base Xen CPUID leaf")); + cpuid_count(hv_base + 4, 0, regs); return (regs[0] & XEN_HVM_CPUID_IOMMU_MAPPINGS); } diff --git a/sys/x86/xen/xen_apic.c b/sys/x86/xen/xen_apic.c index 4f2b03879227..994dc3e0804c 100644 --- a/sys/x86/xen/xen_apic.c +++ b/sys/x86/xen/xen_apic.c @@ -1,361 +1,361 @@ /* * Copyright (c) 2014 Roger Pau MonnĂ© * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*--------------------------- Forward Declarations ---------------------------*/ static driver_filter_t xen_smp_rendezvous_action; #ifdef __amd64__ static driver_filter_t xen_invlop; #else static driver_filter_t xen_invltlb; static driver_filter_t xen_invlpg; static driver_filter_t xen_invlrng; static driver_filter_t xen_invlcache; #endif static driver_filter_t xen_ipi_bitmap_handler; static driver_filter_t xen_cpustop_handler; static driver_filter_t xen_cpususpend_handler; static driver_filter_t xen_ipi_swi_handler; /*---------------------------------- Macros ----------------------------------*/ #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) /*--------------------------------- Xen IPIs ---------------------------------*/ struct xen_ipi_handler { driver_filter_t *filter; const char *description; }; static struct xen_ipi_handler xen_ipis[] = { [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, #ifdef __amd64__ [IPI_TO_IDX(IPI_INVLOP)] = { xen_invlop, "itlb"}, #else [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, #endif [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, [IPI_TO_IDX(IPI_SWI)] = { xen_ipi_swi_handler, "sw" }, }; /* * Save previous (native) handler as a fallback. Xen < 4.7 doesn't support * VCPUOP_send_nmi for HVM guests, and thus we need a fallback in that case: * * https://lists.freebsd.org/archives/freebsd-xen/2022-January/000032.html */ void (*native_ipi_vectored)(u_int, int); /*------------------------------- Per-CPU Data -------------------------------*/ DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); /*------------------------------- Xen PV APIC --------------------------------*/ #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field) static int send_nmi(int dest) { unsigned int cpu; int rc = 0; /* * NMIs are not routed over event channels, and instead delivered as on * native using the exception vector (#2). Triggering them can be done * using the local APIC, or an hypercall as a shortcut like it's done * below. */ switch(dest) { case APIC_IPI_DEST_SELF: rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL); break; case APIC_IPI_DEST_ALL: CPU_FOREACH(cpu) { rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_ID_GET(cpu, vcpu_id), NULL); if (rc != 0) break; } break; case APIC_IPI_DEST_OTHERS: CPU_FOREACH(cpu) { if (cpu != PCPU_GET(cpuid)) { rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_ID_GET(cpu, vcpu_id), NULL); if (rc != 0) break; } } break; default: rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL); break; } return rc; } #undef PCPU_ID_GET static void xen_pv_lapic_ipi_vectored(u_int vector, int dest) { xen_intr_handle_t *ipi_handle; int ipi_idx, to_cpu, self; static bool pvnmi = true; if (vector >= IPI_NMI_FIRST) { if (pvnmi) { int rc = send_nmi(dest); if (rc != 0) { printf( "Sending NMI using hypercall failed (%d) switching to APIC\n", rc); pvnmi = false; native_ipi_vectored(vector, dest); } } else native_ipi_vectored(vector, dest); return; } ipi_idx = IPI_TO_IDX(vector); if (ipi_idx >= nitems(xen_ipis)) panic("IPI out of range"); switch(dest) { case APIC_IPI_DEST_SELF: ipi_handle = DPCPU_GET(ipi_handle); xen_intr_signal(ipi_handle[ipi_idx]); break; case APIC_IPI_DEST_ALL: CPU_FOREACH(to_cpu) { ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); xen_intr_signal(ipi_handle[ipi_idx]); } break; case APIC_IPI_DEST_OTHERS: self = PCPU_GET(cpuid); CPU_FOREACH(to_cpu) { if (to_cpu != self) { ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); xen_intr_signal(ipi_handle[ipi_idx]); } } break; default: to_cpu = apic_cpuid(dest); ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); xen_intr_signal(ipi_handle[ipi_idx]); break; } } /*---------------------------- XEN PV IPI Handlers ---------------------------*/ /* * These are C clones of the ASM functions found in apic_vector. */ static int xen_ipi_bitmap_handler(void *arg) { ipi_bitmap_handler(*curthread->td_intr_frame); return (FILTER_HANDLED); } static int xen_smp_rendezvous_action(void *arg) { #ifdef COUNT_IPIS (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ smp_rendezvous_action(); return (FILTER_HANDLED); } #ifdef __amd64__ static int xen_invlop(void *arg) { invlop_handler(); return (FILTER_HANDLED); } #else /* __i386__ */ static int xen_invltlb(void *arg) { invltlb_handler(); return (FILTER_HANDLED); } static int xen_invlpg(void *arg) { invlpg_handler(); return (FILTER_HANDLED); } static int xen_invlrng(void *arg) { invlrng_handler(); return (FILTER_HANDLED); } static int xen_invlcache(void *arg) { invlcache_handler(); return (FILTER_HANDLED); } #endif /* __amd64__ */ static int xen_cpustop_handler(void *arg) { cpustop_handler(); return (FILTER_HANDLED); } static int xen_cpususpend_handler(void *arg) { cpususpend_handler(); return (FILTER_HANDLED); } static int xen_ipi_swi_handler(void *arg) { ipi_swi_handler(*curthread->td_intr_frame); return (FILTER_HANDLED); } /*----------------------------- XEN PV IPI setup -----------------------------*/ /* * Those functions are provided outside of the Xen PV APIC implementation * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, * because on PVHVM there's an emulated LAPIC provided by Xen. */ static void xen_cpu_ipi_init(int cpu) { xen_intr_handle_t *ipi_handle; const struct xen_ipi_handler *ipi; int idx, rc; ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { if (ipi->filter == NULL) { ipi_handle[idx] = NULL; continue; } rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, INTR_TYPE_TTY, &ipi_handle[idx]); if (rc != 0) panic("Unable to allocate a XEN IPI port"); xen_intr_describe(ipi_handle[idx], "%s", ipi->description); } } static void xen_setup_cpus(void) { uint32_t regs[4]; int i; if (!xen_vector_callback_enabled) return; /* * Check whether the APIC virtualization is hardware assisted, as * that's faster than using event channels because it avoids the VM * exit. */ - KASSERT(xen_cpuid_base != 0, ("Invalid base Xen CPUID leaf")); - cpuid_count(xen_cpuid_base + 4, 0, regs); + KASSERT(hv_base != 0, ("Invalid base Xen CPUID leaf")); + cpuid_count(hv_base + 4, 0, regs); if ((x2apic_mode && (regs[0] & XEN_HVM_CPUID_X2APIC_VIRT)) || (!x2apic_mode && (regs[0] & XEN_HVM_CPUID_APIC_ACCESS_VIRT))) return; CPU_FOREACH(i) xen_cpu_ipi_init(i); /* Set the xen pv ipi ops to replace the native ones */ ipi_vectored = xen_pv_lapic_ipi_vectored; native_ipi_vectored = ipi_vectored; } /* Switch to using PV IPIs as soon as the vcpu_id is set. */ SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL);