diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index cea9b79a0ec8..5515213cd154 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -1,275 +1,275 @@ /*- * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * Copyright (c) 2014-2018 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by * Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: vector.s, 386BSD 0.1 unknown origin * $FreeBSD$ */ /* * Interrupt entry points for external interrupts triggered by I/O APICs * as well as IPI handlers. */ #include "opt_smp.h" #include "assym.inc" #include #include #include #ifdef SMP #define LK lock ; #else #define LK #endif .text SUPERALIGN_TEXT /* End Of Interrupt to APIC */ as_lapic_eoi: cmpl $0,x2apic_mode jne 1f movq lapic_map,%rax movl $0,LA_EOI(%rax) ret 1: movl $MSR_APIC_EOI,%ecx xorl %eax,%eax xorl %edx,%edx wrmsr ret /* * I/O Interrupt Entry Point. Rather than having one entry point for * each interrupt source, we use one entry point for each 32-bit word * in the ISR. The handler determines the highest bit set in the ISR, * translates that into a vector, and passes the vector to the * lapic_handle_intr() function. */ .macro ISR_VEC index, vec_name INTR_HANDLER \vec_name KMSAN_ENTER cmpl $0,x2apic_mode je 1f movl $(MSR_APIC_ISR0 + \index),%ecx rdmsr jmp 2f 1: movq lapic_map, %rdx /* pointer to local APIC */ movl LA_ISR + 16 * (\index)(%rdx), %eax /* load ISR */ 2: bsrl %eax, %eax /* index of highest set bit in ISR */ jz 3f addl $(32 * \index),%eax movq %rsp, %rsi movl %eax, %edi /* pass the IRQ */ call lapic_handle_intr 3: KMSAN_LEAVE jmp doreti .endm /* * Handle "spurious INTerrupts". * Notes: * This is different than the "spurious INTerrupt" generated by an * 8259 PIC for missing INTs. See the APIC documentation for details. * This routine should NOT do an 'EOI' cycle. */ .text SUPERALIGN_TEXT IDTVEC(spuriousint) /* No EOI cycle used here */ jmp doreti_iret ISR_VEC 1, apic_isr1 ISR_VEC 2, apic_isr2 ISR_VEC 3, apic_isr3 ISR_VEC 4, apic_isr4 ISR_VEC 5, apic_isr5 ISR_VEC 6, apic_isr6 ISR_VEC 7, apic_isr7 /* * Local APIC periodic timer handler. */ INTR_HANDLER timerint KMSAN_ENTER movq %rsp, %rdi call lapic_handle_timer KMSAN_LEAVE jmp doreti /* * Local APIC CMCI handler. */ INTR_HANDLER cmcint KMSAN_ENTER call lapic_handle_cmc KMSAN_LEAVE jmp doreti /* * Local APIC error interrupt handler. */ INTR_HANDLER errorint KMSAN_ENTER call lapic_handle_error KMSAN_LEAVE jmp doreti #ifdef XENHVM /* * Xen event channel upcall interrupt handler. * Only used when the hypervisor supports direct vector callbacks. */ INTR_HANDLER xen_intr_upcall KMSAN_ENTER movq %rsp, %rdi - call xen_intr_handle_upcall + call xen_arch_intr_handle_upcall KMSAN_LEAVE jmp doreti #endif #ifdef SMP /* * Global address space TLB shootdown. */ .text SUPERALIGN_TEXT /* * IPI handler for cache and TLB shootdown */ INTR_HANDLER invlop KMSAN_ENTER call invlop_handler call as_lapic_eoi KMSAN_LEAVE jmp ld_regs /* * Handler for IPIs sent via the per-cpu IPI bitmap. */ INTR_HANDLER ipi_intr_bitmap_handler call as_lapic_eoi KMSAN_ENTER call ipi_bitmap_handler KMSAN_LEAVE jmp doreti /* * Executed by a CPU when it receives an IPI_STOP from another CPU. */ INTR_HANDLER cpustop call as_lapic_eoi KMSAN_ENTER call cpustop_handler KMSAN_LEAVE jmp doreti /* * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. */ INTR_HANDLER cpususpend KMSAN_ENTER call cpususpend_handler call as_lapic_eoi KMSAN_LEAVE jmp doreti /* * Executed by a CPU when it receives an IPI_SWI. */ INTR_HANDLER ipi_swi call as_lapic_eoi KMSAN_ENTER call ipi_swi_handler KMSAN_LEAVE jmp doreti /* * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. * * - Calls the generic rendezvous action function. */ INTR_HANDLER rendezvous #ifdef COUNT_IPIS movl PCPU(CPUID), %eax movq ipi_rendezvous_counts(,%rax,8), %rax incq (%rax) #endif KMSAN_ENTER call smp_rendezvous_action call as_lapic_eoi KMSAN_LEAVE jmp doreti /* * IPI handler whose purpose is to interrupt the CPU with minimum overhead. * This is used by bhyve to force a host cpu executing in guest context to * trap into the hypervisor. * * This handler is different from other IPI handlers in the following aspects: * * 1. It doesn't push a trapframe on the stack. * * This implies that a DDB backtrace involving 'justreturn' will skip the * function that was interrupted by this handler. * * 2. It doesn't 'swapgs' when userspace is interrupted. * * The 'justreturn' handler does not access any pcpu data so it is not an * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI * whose handler already doesn't trust GS.base when kernel code is interrupted. */ .text SUPERALIGN_TEXT IDTVEC(justreturn) pushq %rax pushq %rcx pushq %rdx call as_lapic_eoi popq %rdx popq %rcx popq %rax jmp doreti_iret INTR_HANDLER justreturn1 call as_lapic_eoi jmp doreti #endif /* SMP */ diff --git a/sys/conf/files.x86 b/sys/conf/files.x86 index 8774139eee3a..54f6e8c8260a 100644 --- a/sys/conf/files.x86 +++ b/sys/conf/files.x86 @@ -1,349 +1,350 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # $FreeBSD$ # # This file contains all the x86 devices and such that are # common between i386 and amd64, but aren't applicable to # any other architecture we support. # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # atkbdmap.h optional atkbd_dflt_keymap \ compile-with "${KEYMAP} -L ${ATKBD_DFLT_KEYMAP} | ${KEYMAP_FIX} > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "atkbdmap.h" cddl/dev/fbt/x86/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" cddl/dev/dtrace/x86/dis_tables.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" cddl/dev/dtrace/x86/instr_size.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" crypto/aesni/aesni.c optional aesni aesni_ghash.o optional aesni \ dependency "$S/crypto/aesni/aesni_ghash.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ghash.o" aesni_ccm.o optional aesni \ dependency "$S/crypto/aesni/aesni_ccm.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ccm.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} -mmmx -msse -msse4 -maes ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_wrap.o" intel_sha1.o optional aesni \ dependency "$S/crypto/aesni/intel_sha1.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha1.o" intel_sha256.o optional aesni \ dependency "$S/crypto/aesni/intel_sha256.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha256.o" crypto/openssl/ossl_x86.c optional ossl crypto/via/padlock.c optional padlock crypto/via/padlock_cipher.c optional padlock crypto/via/padlock_hash.c optional padlock dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_if.m standard dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/acpica/acpi_pxm.c optional acpi dev/acpica/acpi_timer.c optional acpi dev/amdsbwd/amdsbwd.c optional amdsbwd dev/amdsmn/amdsmn.c optional amdsmn | amdtemp dev/amdtemp/amdtemp.c optional amdtemp dev/arcmsr/arcmsr.c optional arcmsr pci dev/asmc/asmc.c optional asmc isa dev/atkbdc/atkbd.c optional atkbd atkbdc dev/atkbdc/atkbd_atkbdc.c optional atkbd atkbdc dev/atkbdc/atkbdc.c optional atkbdc dev/atkbdc/atkbdc_isa.c optional atkbdc isa dev/atkbdc/atkbdc_subr.c optional atkbdc dev/atkbdc/psm.c optional psm atkbdc dev/bxe/bxe.c optional bxe pci dev/bxe/bxe_stats.c optional bxe pci dev/bxe/bxe_debug.c optional bxe pci dev/bxe/ecore_sp.c optional bxe pci dev/bxe/bxe_elink.c optional bxe pci dev/bxe/57710_init_values.c optional bxe pci dev/bxe/57711_init_values.c optional bxe pci dev/bxe/57712_init_values.c optional bxe pci dev/coretemp/coretemp.c optional coretemp dev/cpuctl/cpuctl.c optional cpuctl dev/dpms/dpms.c optional dpms dev/fb/fb.c optional fb | vga dev/fb/s3_pci.c optional s3pci dev/fb/vesa.c optional vga vesa dev/fb/vga.c optional vga dev/fdc/fdc.c optional fdc dev/fdc/fdc_acpi.c optional fdc dev/fdc/fdc_isa.c optional fdc isa dev/gpio/bytgpio.c optional bytgpio dev/gpio/chvgpio.c optional chvgpio dev/hpt27xx/hpt27xx_os_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_osm_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_config.c optional hpt27xx dev/hpt27xx/$M-elf.hpt27xx_lib.o optional hpt27xx dev/hptmv/entry.c optional hptmv dev/hptmv/mv.c optional hptmv dev/hptmv/gui_lib.c optional hptmv dev/hptmv/hptproc.c optional hptmv dev/hptmv/ioctl.c optional hptmv dev/hptmv/$M-elf.hptmvraid.o optional hptmv dev/hptnr/hptnr_os_bsd.c optional hptnr dev/hptnr/hptnr_osm_bsd.c optional hptnr dev/hptnr/hptnr_config.c optional hptnr dev/hptnr/$M-elf.hptnr_lib.o optional hptnr dev/hptrr/hptrr_os_bsd.c optional hptrr dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr dev/hptrr/$M-elf.hptrr_lib.o optional hptrr dev/hwpmc/hwpmc_amd.c optional hwpmc dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_core.c optional hwpmc dev/hwpmc/hwpmc_uncore.c optional hwpmc dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/hyperv/hvsock/hv_sock.c optional hyperv dev/hyperv/input/hv_hid.c optional hyperv hvhid dev/hyperv/input/hv_kbd.c optional hyperv dev/hyperv/input/hv_kbdc.c optional hyperv dev/hyperv/pcib/vmbus_pcib.c optional hyperv pci dev/hyperv/netvsc/hn_nvs.c optional hyperv dev/hyperv/netvsc/hn_rndis.c optional hyperv dev/hyperv/netvsc/if_hn.c optional hyperv dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c optional hyperv dev/hyperv/utilities/hv_kvp.c optional hyperv dev/hyperv/utilities/hv_snapshot.c optional hyperv dev/hyperv/utilities/vmbus_heartbeat.c optional hyperv dev/hyperv/utilities/vmbus_ic.c optional hyperv dev/hyperv/utilities/vmbus_shutdown.c optional hyperv dev/hyperv/utilities/vmbus_timesync.c optional hyperv dev/hyperv/vmbus/hyperv.c optional hyperv dev/hyperv/vmbus/x86/hyperv_x86.c optional hyperv dev/hyperv/vmbus/x86/vmbus_x86.c optional hyperv dev/hyperv/vmbus/hyperv_busdma.c optional hyperv dev/hyperv/vmbus/vmbus.c optional hyperv pci dev/hyperv/vmbus/vmbus_br.c optional hyperv dev/hyperv/vmbus/vmbus_chan.c optional hyperv dev/hyperv/vmbus/vmbus_et.c optional hyperv dev/hyperv/vmbus/vmbus_if.m optional hyperv dev/hyperv/vmbus/vmbus_res.c optional hyperv dev/hyperv/vmbus/vmbus_xact.c optional hyperv dev/ichwd/ichwd.c optional ichwd dev/imcsmb/imcsmb.c optional imcsmb dev/imcsmb/imcsmb_pci.c optional imcsmb pci dev/intel/pchtherm.c optional pchtherm dev/intel/spi.c optional intelspi dev/io/iodev.c optional io dev/iommu/busdma_iommu.c optional acpi iommu pci dev/iommu/iommu_gas.c optional acpi iommu pci dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_isa.c optional ipmi isa dev/ipmi/ipmi_kcs.c optional ipmi dev/ipmi/ipmi_smic.c optional ipmi dev/ipmi/ipmi_smbus.c optional ipmi smbus dev/ipmi/ipmi_smbios.c optional ipmi dev/ipmi/ipmi_ssif.c optional ipmi smbus dev/ipmi/ipmi_pci.c optional ipmi pci dev/isci/isci.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/isci_controller.c optional isci dev/isci/isci_domain.c optional isci dev/isci/isci_interrupt.c optional isci dev/isci/isci_io_request.c optional isci dev/isci/isci_logger.c optional isci dev/isci/isci_oem_parameters.c optional isci dev/isci/isci_remote_device.c optional isci dev/isci/isci_sysctl.c optional isci dev/isci/isci_task_request.c optional isci dev/isci/isci_timer.c optional isci dev/isci/scil/sati.c optional isci dev/isci/scil/sati_abort_task_set.c optional isci dev/isci/scil/sati_atapi.c optional isci dev/isci/scil/sati_device.c optional isci dev/isci/scil/sati_inquiry.c optional isci dev/isci/scil/sati_log_sense.c optional isci dev/isci/scil/sati_lun_reset.c optional isci dev/isci/scil/sati_mode_pages.c optional isci dev/isci/scil/sati_mode_select.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/sati_mode_sense.c optional isci dev/isci/scil/sati_mode_sense_10.c optional isci dev/isci/scil/sati_mode_sense_6.c optional isci dev/isci/scil/sati_move.c optional isci dev/isci/scil/sati_passthrough.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/sati_read.c optional isci dev/isci/scil/sati_read_buffer.c optional isci dev/isci/scil/sati_read_capacity.c optional isci dev/isci/scil/sati_reassign_blocks.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/sati_report_luns.c optional isci dev/isci/scil/sati_request_sense.c optional isci dev/isci/scil/sati_start_stop_unit.c optional isci dev/isci/scil/sati_synchronize_cache.c optional isci dev/isci/scil/sati_test_unit_ready.c optional isci dev/isci/scil/sati_unmap.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/sati_util.c optional isci dev/isci/scil/sati_verify.c optional isci dev/isci/scil/sati_write.c optional isci dev/isci/scil/sati_write_and_verify.c optional isci dev/isci/scil/sati_write_buffer.c optional isci dev/isci/scil/sati_write_long.c optional isci dev/isci/scil/sci_abstract_list.c optional isci dev/isci/scil/sci_base_controller.c optional isci dev/isci/scil/sci_base_domain.c optional isci dev/isci/scil/sci_base_iterator.c optional isci dev/isci/scil/sci_base_library.c optional isci dev/isci/scil/sci_base_logger.c optional isci dev/isci/scil/sci_base_memory_descriptor_list.c optional isci dev/isci/scil/sci_base_memory_descriptor_list_decorator.c optional isci dev/isci/scil/sci_base_object.c optional isci dev/isci/scil/sci_base_observer.c optional isci dev/isci/scil/sci_base_phy.c optional isci dev/isci/scil/sci_base_port.c optional isci dev/isci/scil/sci_base_remote_device.c optional isci dev/isci/scil/sci_base_request.c optional isci dev/isci/scil/sci_base_state_machine.c optional isci dev/isci/scil/sci_base_state_machine_logger.c optional isci dev/isci/scil/sci_base_state_machine_observer.c optional isci dev/isci/scil/sci_base_subject.c optional isci dev/isci/scil/sci_util.c optional isci dev/isci/scil/scic_sds_controller.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/scic_sds_library.c optional isci dev/isci/scil/scic_sds_pci.c optional isci dev/isci/scil/scic_sds_phy.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/scic_sds_port.c optional isci dev/isci/scil/scic_sds_port_configuration_agent.c optional isci dev/isci/scil/scic_sds_remote_device.c optional isci dev/isci/scil/scic_sds_remote_node_context.c optional isci dev/isci/scil/scic_sds_remote_node_table.c optional isci dev/isci/scil/scic_sds_request.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/scic_sds_sgpio.c optional isci dev/isci/scil/scic_sds_smp_remote_device.c optional isci dev/isci/scil/scic_sds_smp_request.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/scic_sds_ssp_request.c optional isci dev/isci/scil/scic_sds_stp_packet_request.c optional isci dev/isci/scil/scic_sds_stp_remote_device.c optional isci dev/isci/scil/scic_sds_stp_request.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/scic_sds_unsolicited_frame_control.c optional isci dev/isci/scil/scif_sas_controller.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/scif_sas_controller_state_handlers.c optional isci dev/isci/scil/scif_sas_controller_states.c optional isci dev/isci/scil/scif_sas_domain.c optional isci dev/isci/scil/scif_sas_domain_state_handlers.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/scif_sas_domain_states.c optional isci dev/isci/scil/scif_sas_high_priority_request_queue.c optional isci dev/isci/scil/scif_sas_internal_io_request.c optional isci dev/isci/scil/scif_sas_io_request.c optional isci dev/isci/scil/scif_sas_io_request_state_handlers.c optional isci dev/isci/scil/scif_sas_io_request_states.c optional isci dev/isci/scil/scif_sas_library.c optional isci dev/isci/scil/scif_sas_remote_device.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substates.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/scif_sas_remote_device_starting_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substates.c optional isci dev/isci/scil/scif_sas_remote_device_state_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_states.c optional isci dev/isci/scil/scif_sas_request.c optional isci dev/isci/scil/scif_sas_smp_activity_clear_affiliation.c optional isci dev/isci/scil/scif_sas_smp_io_request.c optional isci dev/isci/scil/scif_sas_smp_phy.c optional isci dev/isci/scil/scif_sas_smp_remote_device.c optional isci \ compile-with "${NORMAL_C} ${NO_WUNUSED_BUT_SET_VARIABLE}" dev/isci/scil/scif_sas_stp_io_request.c optional isci dev/isci/scil/scif_sas_stp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_task_request.c optional isci dev/isci/scil/scif_sas_task_request.c optional isci dev/isci/scil/scif_sas_task_request_state_handlers.c optional isci dev/isci/scil/scif_sas_task_request_states.c optional isci dev/isci/scil/scif_sas_timer.c optional isci dev/itwd/itwd.c optional itwd dev/kvm_clock/kvm_clock.c optional kvm_clock dev/mana/gdma_main.c optional mana dev/mana/mana_en.c optional mana dev/mana/mana_sysctl.c optional mana dev/mana/shm_channel.c optional mana dev/mana/hw_channel.c optional mana dev/mana/gdma_util.c optional mana dev/qat_c2xxx/qat.c optional qat_c2xxx dev/qat_c2xxx/qat_ae.c optional qat_c2xxx dev/qat_c2xxx/qat_c2xxx.c optional qat_c2xxx dev/qat_c2xxx/qat_hw15.c optional qat_c2xxx dev/smbios/smbios_subr.c standard libkern/strcmp.c standard libkern/strncmp.c standard libkern/x86/crc32_sse42.c standard # # x86 shared code between IA32 and AMD64 architectures # x86/acpica/OsdEnvironment.c optional acpi x86/acpica/acpi_apm.c optional acpi x86/acpica/srat.c optional acpi x86/bios/vpd.c optional vpd x86/cpufreq/est.c optional cpufreq x86/cpufreq/hwpstate_amd.c optional cpufreq x86/cpufreq/hwpstate_intel.c optional cpufreq x86/cpufreq/p4tcc.c optional cpufreq x86/cpufreq/powernow.c optional cpufreq x86/iommu/intel_ctx.c optional acpi iommu pci x86/iommu/intel_drv.c optional acpi iommu pci x86/iommu/intel_fault.c optional acpi iommu pci x86/iommu/intel_idpgtbl.c optional acpi iommu pci x86/iommu/intel_intrmap.c optional acpi iommu pci x86/iommu/intel_qi.c optional acpi iommu pci x86/iommu/intel_quirks.c optional acpi iommu pci x86/iommu/intel_utils.c optional acpi iommu pci x86/isa/atrtc.c standard x86/isa/clock.c standard x86/isa/isa.c optional isa x86/isa/isa_dma.c optional isa x86/isa/nmi.c standard x86/isa/orm.c optional isa x86/pci/pci_bus.c optional pci x86/pci/qpi.c optional pci x86/x86/autoconf.c standard x86/x86/bus_machdep.c standard x86/x86/busdma_bounce.c standard x86/x86/busdma_machdep.c standard x86/x86/cpu_machdep.c standard x86/x86/dbreg.c optional ddb | gdb x86/x86/dump_machdep.c standard x86/x86/fdt_machdep.c optional fdt x86/x86/identcpu.c standard x86/x86/intr_machdep.c standard x86/x86/legacy.c standard x86/x86/mca.c standard x86/x86/x86_mem.c optional mem x86/x86/mp_x86.c optional smp x86/x86/nexus.c standard x86/x86/pvclock.c optional kvm_clock | xenhvm x86/x86/stack_machdep.c optional ddb | stack x86/x86/tsc.c standard x86/x86/ucode.c standard x86/x86/delay.c standard x86/xen/hvm.c optional xenhvm x86/xen/xen_intr.c optional xenhvm x86/xen/xen_apic.c optional xenhvm smp +x86/xen/xen_arch_intr.c optional xenhvm diff --git a/sys/dev/xen/xenpci/xenpci.c b/sys/dev/xen/xenpci/xenpci.c index ca54b691b8ad..7dd69688a00e 100644 --- a/sys/dev/xen/xenpci/xenpci.c +++ b/sys/dev/xen/xenpci/xenpci.c @@ -1,234 +1,227 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2008 Citrix Systems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -static int -xenpci_intr_filter(void *trap_frame) -{ - xen_intr_handle_upcall(trap_frame); - return (FILTER_HANDLED); -} - static int xenpci_irq_init(device_t device, struct xenpci_softc *scp) { int error; error = BUS_SETUP_INTR(device_get_parent(device), device, scp->res_irq, INTR_MPSAFE|INTR_TYPE_MISC, - xenpci_intr_filter, NULL, /*trap_frame*/NULL, + xen_intr_handle_upcall, NULL, NULL, &scp->intr_cookie); if (error) return error; #ifdef SMP /* * When using the PCI event delivery callback we cannot assign * events to specific vCPUs, so all events are delivered to vCPU#0 by * Xen. Since the PCI interrupt can fire on any CPU by default, we * need to bind it to vCPU#0 in order to ensure that * xen_intr_handle_upcall always gets called on vCPU#0. */ error = BUS_BIND_INTR(device_get_parent(device), device, scp->res_irq, 0); if (error) return error; #endif xen_hvm_set_callback(device); return (0); } /* * Deallocate anything allocated by xenpci_allocate_resources. */ static int xenpci_deallocate_resources(device_t dev) { struct xenpci_softc *scp = device_get_softc(dev); if (scp->res_irq != 0) { bus_deactivate_resource(dev, SYS_RES_IRQ, scp->rid_irq, scp->res_irq); bus_release_resource(dev, SYS_RES_IRQ, scp->rid_irq, scp->res_irq); scp->res_irq = 0; } return (0); } /* * Allocate irq and memory resources. */ static int xenpci_allocate_resources(device_t dev) { struct xenpci_softc *scp = device_get_softc(dev); scp->res_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &scp->rid_irq, RF_SHAREABLE|RF_ACTIVE); if (scp->res_irq == NULL) { printf("xenpci Could not allocate irq.\n"); goto errexit; } return (0); errexit: /* Cleanup anything we may have assigned. */ xenpci_deallocate_resources(dev); return (ENXIO); /* For want of a better idea. */ } /* * Probe - just check device ID. */ static int xenpci_probe(device_t dev) { if (pci_get_devid(dev) != 0x00015853) return (ENXIO); device_set_desc(dev, "Xen Platform Device"); return (BUS_PROBE_DEFAULT); } /* * Attach - find resources and talk to Xen. */ static int xenpci_attach(device_t dev) { struct xenpci_softc *scp = device_get_softc(dev); int error; error = xenpci_allocate_resources(dev); if (error) { device_printf(dev, "xenpci_allocate_resources failed(%d).\n", error); goto errexit; } /* * Hook the irq up to evtchn */ error = xenpci_irq_init(dev, scp); if (error) { device_printf(dev, "xenpci_irq_init failed(%d).\n", error); goto errexit; } return (0); errexit: /* * Undo anything we may have done. */ xenpci_deallocate_resources(dev); return (error); } /* * Detach - reverse anything done by attach. */ static int xenpci_detach(device_t dev) { struct xenpci_softc *scp = device_get_softc(dev); device_t parent = device_get_parent(dev); /* * Take our interrupt handler out of the list of handlers * that can handle this irq. */ if (scp->intr_cookie != NULL) { if (BUS_TEARDOWN_INTR(parent, dev, scp->res_irq, scp->intr_cookie) != 0) device_printf(dev, "intr teardown failed.. continuing\n"); scp->intr_cookie = NULL; } /* * Deallocate any system resources we may have * allocated on behalf of this driver. */ return (xenpci_deallocate_resources(dev)); } static int xenpci_resume(device_t dev) { xen_hvm_set_callback(dev); return (0); } static device_method_t xenpci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, xenpci_probe), DEVMETHOD(device_attach, xenpci_attach), DEVMETHOD(device_detach, xenpci_detach), DEVMETHOD(device_resume, xenpci_resume), { 0, 0 } }; static driver_t xenpci_driver = { "xenpci", xenpci_methods, sizeof(struct xenpci_softc), }; DRIVER_MODULE(xenpci, pci, xenpci_driver, 0, 0); diff --git a/sys/i386/i386/apic_vector.S b/sys/i386/i386/apic_vector.S index 6c4f8b78fe55..270e0d0393ab 100644 --- a/sys/i386/i386/apic_vector.S +++ b/sys/i386/i386/apic_vector.S @@ -1,336 +1,336 @@ /*- * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: vector.s, 386BSD 0.1 unknown origin * $FreeBSD$ */ /* * Interrupt entry points for external interrupts triggered by I/O APICs * as well as IPI handlers. */ #include "opt_smp.h" #include #include #include #include #include "assym.inc" .text SUPERALIGN_TEXT /* End Of Interrupt to APIC */ as_lapic_eoi: cmpl $0,x2apic_mode jne 1f movl lapic_map,%eax movl $0,LA_EOI(%eax) ret 1: movl $MSR_APIC_EOI,%ecx xorl %eax,%eax xorl %edx,%edx wrmsr ret /* * I/O Interrupt Entry Point. Rather than having one entry point for * each interrupt source, we use one entry point for each 32-bit word * in the ISR. The handler determines the highest bit set in the ISR, * translates that into a vector, and passes the vector to the * lapic_handle_intr() function. */ .macro ISR_VEC index, vec_name .text SUPERALIGN_TEXT .globl X\()\vec_name\()_pti, X\()\vec_name X\()\vec_name\()_pti: X\()\vec_name: PUSH_FRAME SET_KERNEL_SREGS cld KENTER cmpl $0,x2apic_mode je 2f movl $(MSR_APIC_ISR0 + \index),%ecx rdmsr jmp 3f 2: movl lapic_map, %edx /* pointer to local APIC */ movl LA_ISR + 16 * \index(%edx), %eax /* load ISR */ 3: bsrl %eax, %eax /* index of highest set bit in ISR */ jz 4f addl $(32 * \index),%eax pushl %esp pushl %eax /* pass the IRQ */ movl $lapic_handle_intr, %eax call *%eax addl $8, %esp /* discard parameter */ 4: jmp doreti .endm /* * Handle "spurious INTerrupts". * Notes: * This is different than the "spurious INTerrupt" generated by an * 8259 PIC for missing INTs. See the APIC documentation for details. * This routine should NOT do an 'EOI' cycle. */ .text SUPERALIGN_TEXT IDTVEC(spuriousint) /* No EOI cycle used here */ iret ISR_VEC 1, apic_isr1 ISR_VEC 2, apic_isr2 ISR_VEC 3, apic_isr3 ISR_VEC 4, apic_isr4 ISR_VEC 5, apic_isr5 ISR_VEC 6, apic_isr6 ISR_VEC 7, apic_isr7 /* * Local APIC periodic timer handler. */ .text SUPERALIGN_TEXT IDTVEC(timerint_pti) IDTVEC(timerint) PUSH_FRAME SET_KERNEL_SREGS cld KENTER pushl %esp movl $lapic_handle_timer, %eax call *%eax add $4, %esp jmp doreti /* * Local APIC CMCI handler. */ .text SUPERALIGN_TEXT IDTVEC(cmcint_pti) IDTVEC(cmcint) PUSH_FRAME SET_KERNEL_SREGS cld KENTER movl $lapic_handle_cmc, %eax call *%eax jmp doreti /* * Local APIC error interrupt handler. */ .text SUPERALIGN_TEXT IDTVEC(errorint_pti) IDTVEC(errorint) PUSH_FRAME SET_KERNEL_SREGS cld KENTER movl $lapic_handle_error, %eax call *%eax jmp doreti #ifdef XENHVM /* * Xen event channel upcall interrupt handler. * Only used when the hypervisor supports direct vector callbacks. */ .text SUPERALIGN_TEXT IDTVEC(xen_intr_upcall) PUSH_FRAME SET_KERNEL_SREGS cld KENTER pushl %esp - movl $xen_intr_handle_upcall, %eax + movl $xen_arch_intr_handle_upcall, %eax call *%eax add $4, %esp jmp doreti #endif #ifdef SMP /* * Global address space TLB shootdown. */ .text SUPERALIGN_TEXT invltlb_ret: call as_lapic_eoi jmp doreti SUPERALIGN_TEXT IDTVEC(invltlb) PUSH_FRAME SET_KERNEL_SREGS cld KENTER movl $invltlb_handler, %eax call *%eax jmp invltlb_ret /* * Single page TLB shootdown */ .text SUPERALIGN_TEXT IDTVEC(invlpg) PUSH_FRAME SET_KERNEL_SREGS cld KENTER movl $invlpg_handler, %eax call *%eax jmp invltlb_ret /* * Page range TLB shootdown. */ .text SUPERALIGN_TEXT IDTVEC(invlrng) PUSH_FRAME SET_KERNEL_SREGS cld KENTER movl $invlrng_handler, %eax call *%eax jmp invltlb_ret /* * Invalidate cache. */ .text SUPERALIGN_TEXT IDTVEC(invlcache) PUSH_FRAME SET_KERNEL_SREGS cld KENTER movl $invlcache_handler, %eax call *%eax jmp invltlb_ret /* * Handler for IPIs sent via the per-cpu IPI bitmap. */ .text SUPERALIGN_TEXT IDTVEC(ipi_intr_bitmap_handler) PUSH_FRAME SET_KERNEL_SREGS cld KENTER call as_lapic_eoi movl $ipi_bitmap_handler, %eax call *%eax jmp doreti /* * Executed by a CPU when it receives an IPI_STOP from another CPU. */ .text SUPERALIGN_TEXT IDTVEC(cpustop) PUSH_FRAME SET_KERNEL_SREGS cld KENTER call as_lapic_eoi movl $cpustop_handler, %eax call *%eax jmp doreti /* * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. */ .text SUPERALIGN_TEXT IDTVEC(cpususpend) PUSH_FRAME SET_KERNEL_SREGS cld KENTER call as_lapic_eoi movl $cpususpend_handler, %eax call *%eax jmp doreti /* * Executed by a CPU when it receives an IPI_SWI. */ .text SUPERALIGN_TEXT IDTVEC(ipi_swi) PUSH_FRAME SET_KERNEL_SREGS cld KENTER call as_lapic_eoi movl $ipi_swi_handler, %eax call *%eax jmp doreti /* * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. * * - Calls the generic rendezvous action function. */ .text SUPERALIGN_TEXT IDTVEC(rendezvous) PUSH_FRAME SET_KERNEL_SREGS cld KENTER #ifdef COUNT_IPIS movl PCPU(CPUID), %eax movl ipi_rendezvous_counts(,%eax,4), %eax incl (%eax) #endif movl $smp_rendezvous_action, %eax call *%eax call as_lapic_eoi jmp doreti #endif /* SMP */ diff --git a/sys/x86/xen/xen_arch_intr.c b/sys/x86/xen/xen_arch_intr.c new file mode 100644 index 000000000000..d946ca781355 --- /dev/null +++ b/sys/x86/xen/xen_arch_intr.c @@ -0,0 +1,103 @@ +/*- + * SPDX-License-Identifier: MIT OR GPL-2.0-only + * + * Copyright © 2013 Spectra Logic Corporation + * Copyright © 2018 John Baldwin/The FreeBSD Foundation + * Copyright © 2019 Roger Pau Monné/Citrix Systems R&D + * Copyright © 2021 Elliott Mitchell + * + * This file may be distributed separately from the Linux kernel, or + * incorporated into other software packages, subject to the following license: + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this source file (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +/************************ Xen x86 interrupt interface ************************/ + +/* + * Pointers to the interrupt counters + */ +DPCPU_DEFINE_STATIC(u_long *, pintrcnt); + +static void +xen_intrcnt_init(void *dummy __unused) +{ + unsigned int i; + + if (!xen_domain()) + return; + + CPU_FOREACH(i) { + char buf[MAXCOMLEN + 1]; + + snprintf(buf, sizeof(buf), "cpu%d:xen", i); + intrcnt_add(buf, DPCPU_ID_PTR(i, pintrcnt)); + } +} +SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL); + +/* + * Transition from assembly language, called from + * sys/{amd64/amd64|i386/i386}/apic_vector.S + */ +extern void xen_arch_intr_handle_upcall(struct trapframe *); +void +xen_arch_intr_handle_upcall(struct trapframe *trap_frame) +{ + struct trapframe *old; + + /* + * Disable preemption in order to always check and fire events + * on the right vCPU + */ + critical_enter(); + + ++*DPCPU_GET(pintrcnt); + + ++curthread->td_intr_nesting_level; + old = curthread->td_intr_frame; + curthread->td_intr_frame = trap_frame; + + xen_intr_handle_upcall(NULL); + + curthread->td_intr_frame = old; + --curthread->td_intr_nesting_level; + + if (xen_evtchn_needs_ack) + lapic_eoi(); + + critical_exit(); +} diff --git a/sys/x86/xen/xen_intr.c b/sys/x86/xen/xen_intr.c index 1bfe941bc67b..0c4d2cd9222b 100644 --- a/sys/x86/xen/xen_intr.c +++ b/sys/x86/xen/xen_intr.c @@ -1,1441 +1,1393 @@ /****************************************************************************** * xen_intr.c * * Xen event and interrupt services for x86 HVM guests. * * Copyright (c) 2002-2005, K A Fraser * Copyright (c) 2005, Intel Corporation * Copyright (c) 2012, Spectra Logic Corporation * Copyright © 2021-2023, Elliott Mitchell * * This file may be distributed separately from the Linux kernel, or * incorporated into other software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include #include #include #include #include -#include #include #include #include #include #include #ifdef DDB #include #endif /* The code below is the implementation of 2L event channels. */ #define NR_EVENT_CHANNELS EVTCHN_2L_NR_CHANNELS static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services"); /* * Lock for x86-related structures. Notably modifying * xen_intr_auto_vector_count, and allocating interrupts require this lock be * held. * * ->xi_type == EVTCHN_TYPE_UNBOUND indicates a xenisrc is under control of * this lock and operations require it be held. */ static struct mtx xen_intr_x86_lock; static u_int first_evtchn_irq; /** * Per-cpu event channel processing state. */ struct xen_intr_pcpu_data { /** * The last event channel bitmap section (level one bit) processed. * This is used to ensure we scan all ports before * servicing an already servied port again. */ u_int last_processed_l1i; /** * The last event channel processed within the event channel * bitmap being scanned. */ u_int last_processed_l2i; - /** Pointer to this CPU's interrupt statistic counter. */ - u_long *evtchn_intrcnt; - /** * A bitmap of ports that can be serviced from this CPU. * A set bit means interrupt handling is enabled. */ u_long evtchn_enabled[sizeof(u_long) * 8]; }; /* * Start the scan at port 0 by initializing the last scanned * location as the highest numbered event channel port. */ DPCPU_DEFINE_STATIC(struct xen_intr_pcpu_data, xen_intr_pcpu) = { .last_processed_l1i = LONG_BIT - 1, .last_processed_l2i = LONG_BIT - 1 }; DPCPU_DECLARE(struct vcpu_info *, vcpu_info); #define INVALID_EVTCHN (~(evtchn_port_t)0) /* Invalid event channel */ #define is_valid_evtchn(x) ((uintmax_t)(x) < NR_EVENT_CHANNELS) struct xenisrc { struct intsrc xi_intsrc; enum evtchn_type xi_type; u_int xi_cpu; /* VCPU for delivery. */ u_int xi_vector; /* Global isrc vector number. */ evtchn_port_t xi_port; u_int xi_virq; void *xi_cookie; bool xi_close:1; /* close on unbind? */ bool xi_masked:1; volatile u_int xi_refcount; }; static void xen_intr_suspend(struct pic *); static void xen_intr_resume(struct pic *, bool suspend_cancelled); static void xen_intr_enable_source(struct intsrc *isrc); static void xen_intr_disable_source(struct intsrc *isrc, int eoi); static void xen_intr_eoi_source(struct intsrc *isrc); static void xen_intr_enable_intr(struct intsrc *isrc); static void xen_intr_disable_intr(struct intsrc *isrc); static int xen_intr_vector(struct intsrc *isrc); static int xen_intr_source_pending(struct intsrc *isrc); static int xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig, enum intr_polarity pol); static int xen_intr_assign_cpu(struct intsrc *isrc, u_int to_cpu); static int xen_intr_pic_assign_cpu(struct intsrc *isrc, u_int apic_id); /** * PIC interface for all event channel port types except physical IRQs. */ struct pic xen_intr_pic = { .pic_enable_source = xen_intr_enable_source, .pic_disable_source = xen_intr_disable_source, .pic_eoi_source = xen_intr_eoi_source, .pic_enable_intr = xen_intr_enable_intr, .pic_disable_intr = xen_intr_disable_intr, .pic_vector = xen_intr_vector, .pic_source_pending = xen_intr_source_pending, .pic_suspend = xen_intr_suspend, .pic_resume = xen_intr_resume, .pic_config_intr = xen_intr_config_intr, .pic_assign_cpu = xen_intr_pic_assign_cpu, }; /* * Lock for interrupt core data. * * Modifying xen_intr_port_to_isrc[], or isrc->xi_port (implies the former) * requires this lock be held. Any time this lock is not held, the condition * `!xen_intr_port_to_isrc[i] || (xen_intr_port_to_isrc[i]->ix_port == i)` * MUST be true for all values of i which are valid indicies of the array. * * Acquire/release operations for isrc->xi_refcount require this lock be held. */ static struct mtx xen_intr_isrc_lock; static u_int xen_intr_auto_vector_count; static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS]; /*------------------------- Private Functions --------------------------------*/ /** * Retrieve a handle for a Xen interrupt source. * * \param isrc A valid Xen interrupt source structure. * * \returns A handle suitable for use with xen_intr_isrc_from_handle() * to retrieve the original Xen interrupt source structure. */ static inline xen_intr_handle_t xen_intr_handle_from_isrc(struct xenisrc *isrc) { return (isrc); } /** * Lookup a Xen interrupt source object given an interrupt binding handle. * * \param handle A handle initialized by a previous call to * xen_intr_bind_isrc(). * * \returns A pointer to the Xen interrupt source object associated * with the given interrupt handle. NULL if no association * currently exists. */ static inline struct xenisrc * xen_intr_isrc_from_handle(xen_intr_handle_t handle) { return ((struct xenisrc *)handle); } /** * Disable signal delivery for an event channel port on the * specified CPU. * * \param port The event channel port to mask. * * This API is used to manage the port<=>CPU binding of event * channel handlers. * * \note This operation does not preclude reception of an event * for this event channel on another CPU. To mask the * event channel globally, use evtchn_mask(). */ static inline void evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port) { struct xen_intr_pcpu_data *pcpu; pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); xen_clear_bit(port, pcpu->evtchn_enabled); } /** * Enable signal delivery for an event channel port on the * specified CPU. * * \param port The event channel port to unmask. * * This API is used to manage the port<=>CPU binding of event * channel handlers. * * \note This operation does not guarantee that event delivery * is enabled for this event channel port. The port must * also be globally enabled. See evtchn_unmask(). */ static inline void evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port) { struct xen_intr_pcpu_data *pcpu; pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); xen_set_bit(port, pcpu->evtchn_enabled); } -/** - * Allocate and register a per-cpu Xen upcall interrupt counter. - * - * \param cpu The cpu for which to register this interrupt count. - */ -static void -xen_intr_intrcnt_add(u_int cpu) -{ - char buf[MAXCOMLEN + 1]; - struct xen_intr_pcpu_data *pcpu; - - pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); - if (pcpu->evtchn_intrcnt != NULL) - return; - - snprintf(buf, sizeof(buf), "cpu%d:xen", cpu); - intrcnt_add(buf, &pcpu->evtchn_intrcnt); -} - /** * Search for an already allocated but currently unused Xen interrupt * source object. * * \param type Restrict the search to interrupt sources of the given * type. * * \return A pointer to a free Xen interrupt source object or NULL. */ static struct xenisrc * xen_intr_find_unused_isrc(enum evtchn_type type) { int isrc_idx; mtx_assert(&xen_intr_x86_lock, MA_OWNED); for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) { struct xenisrc *isrc; u_int vector; vector = first_evtchn_irq + isrc_idx; isrc = (struct xenisrc *)intr_lookup_source(vector); /* * Since intr_register_source() must be called while unlocked, * isrc == NULL *will* occur, though very infrequently. * * This also allows a very small gap where a foreign intrusion * into Xen's interrupt range could be examined by this test. */ if (__predict_true(isrc != NULL) && __predict_true(isrc->xi_intsrc.is_pic == &xen_intr_pic) && isrc->xi_type == EVTCHN_TYPE_UNBOUND) { KASSERT(isrc->xi_intsrc.is_handlers == 0, ("Free evtchn still has handlers")); isrc->xi_type = type; return (isrc); } } return (NULL); } /** * Allocate a Xen interrupt source object. * * \param type The type of interrupt source to create. * * \return A pointer to a newly allocated Xen interrupt source * object or NULL. */ static struct xenisrc * xen_intr_alloc_isrc(enum evtchn_type type) { static int warned; struct xenisrc *isrc; unsigned int vector; int error; mtx_lock(&xen_intr_x86_lock); isrc = xen_intr_find_unused_isrc(type); if (isrc != NULL) { mtx_unlock(&xen_intr_x86_lock); return (isrc); } if (xen_intr_auto_vector_count >= NR_EVENT_CHANNELS) { if (!warned) { warned = 1; printf("%s: Xen interrupts exhausted.\n", __func__); } mtx_unlock(&xen_intr_x86_lock); return (NULL); } vector = first_evtchn_irq + xen_intr_auto_vector_count; xen_intr_auto_vector_count++; KASSERT((intr_lookup_source(vector) == NULL), ("Trying to use an already allocated vector")); mtx_unlock(&xen_intr_x86_lock); isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO); isrc->xi_intsrc.is_pic = &xen_intr_pic; isrc->xi_vector = vector; isrc->xi_type = type; error = intr_register_source(&isrc->xi_intsrc); if (error != 0) panic("%s(): failed registering interrupt %u, error=%d\n", __func__, vector, error); return (isrc); } /** * Attempt to free an active Xen interrupt source object. * * \param isrc The interrupt source object to release. * * \returns EBUSY if the source is still in use, otherwise 0. */ static int xen_intr_release_isrc(struct xenisrc *isrc) { KASSERT(isrc->xi_intsrc.is_handlers == 0, ("Release called, but xenisrc still in use")); mtx_lock(&xen_intr_isrc_lock); if (is_valid_evtchn(isrc->xi_port)) { evtchn_mask_port(isrc->xi_port); evtchn_clear_port(isrc->xi_port); /* Rebind port to CPU 0. */ evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); evtchn_cpu_unmask_port(0, isrc->xi_port); if (isrc->xi_close != 0) { struct evtchn_close close = { .port = isrc->xi_port }; if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) panic("EVTCHNOP_close failed"); } xen_intr_port_to_isrc[isrc->xi_port] = NULL; } /* not reachable from xen_intr_port_to_isrc[], unlock */ mtx_unlock(&xen_intr_isrc_lock); mtx_lock(&xen_intr_x86_lock); isrc->xi_type = EVTCHN_TYPE_UNBOUND; mtx_unlock(&xen_intr_x86_lock); return (0); } /** * Associate an interrupt handler with an already allocated local Xen * event channel port. * * \param isrcp The returned Xen interrupt object associated with * the specified local port. * \param local_port The event channel to bind. * \param type The event channel type of local_port. * \param intr_owner The device making this bind request. * \param filter An interrupt filter handler. Specify NULL * to always dispatch to the ithread handler. * \param handler An interrupt ithread handler. Optional (can * specify NULL) if all necessary event actions * are performed by filter. * \param arg Argument to present to both filter and handler. * \param irqflags Interrupt handler flags. See sys/bus.h. * \param handlep Pointer to an opaque handle used to manage this * registration. * * \returns 0 on success, otherwise an errno. */ static int xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port, enum evtchn_type type, const char *intr_owner, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, xen_intr_handle_t *const port_handlep) { struct xenisrc *isrc; int error; *isrcp = NULL; if (port_handlep == NULL) { printf("%s: %s: Bad event handle\n", intr_owner, __func__); return (EINVAL); } *port_handlep = NULL; isrc = xen_intr_alloc_isrc(type); if (isrc == NULL) return (ENOSPC); isrc->xi_cookie = NULL; isrc->xi_port = local_port; isrc->xi_close = false; isrc->xi_cpu = 0; refcount_init(&isrc->xi_refcount, 1); mtx_lock(&xen_intr_isrc_lock); xen_intr_port_to_isrc[isrc->xi_port] = isrc; mtx_unlock(&xen_intr_isrc_lock); #ifdef SMP if (type == EVTCHN_TYPE_PORT) { /* * By default all interrupts are assigned to vCPU#0 * unless specified otherwise, so shuffle them to balance * the interrupt load. */ xen_intr_assign_cpu(&isrc->xi_intsrc, apic_cpuid(intr_next_cpu(0))); } #endif /* * If a filter or handler function is provided, add it to the event. * Otherwise the event channel is left masked and without a handler, * the caller is in charge of setting that up. */ if (filter != NULL || handler != NULL) { error = xen_intr_add_handler(intr_owner, filter, handler, arg, flags, xen_intr_handle_from_isrc(isrc)); if (error != 0) { xen_intr_release_isrc(isrc); return (error); } } *isrcp = isrc; /* Assign the opaque handler */ *port_handlep = xen_intr_handle_from_isrc(isrc); return (0); } /** * Determine the event channel ports at the given section of the * event port bitmap which have pending events for the given cpu. * * \param pcpu The Xen interrupt pcpu data for the cpu being queried. * \param sh The Xen shared info area. * \param idx The index of the section of the event channel bitmap to * inspect. * * \returns A u_long with bits set for every event channel with pending * events. */ static inline u_long xen_intr_active_ports(const struct xen_intr_pcpu_data *const pcpu, const u_int idx) { volatile const shared_info_t *const sh = HYPERVISOR_shared_info; CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(sh->evtchn_pending[0])); CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(pcpu->evtchn_enabled[0])); CTASSERT(sizeof(sh->evtchn_mask) == sizeof(sh->evtchn_pending)); CTASSERT(sizeof(sh->evtchn_mask) == sizeof(pcpu->evtchn_enabled)); return (sh->evtchn_pending[idx] & ~sh->evtchn_mask[idx] & pcpu->evtchn_enabled[idx]); } /** * Interrupt handler for processing all Xen event channel events. * * \param trap_frame The trap frame context for the current interrupt. */ -void -xen_intr_handle_upcall(struct trapframe *trap_frame) +int +xen_intr_handle_upcall(void *unused __unused) { + struct trapframe *trap_frame = curthread->td_intr_frame; u_int l1i, l2i, port, cpu __diagused; u_long masked_l1, masked_l2; struct xenisrc *isrc; vcpu_info_t *v; struct xen_intr_pcpu_data *pc; u_long l1, l2; - /* - * Disable preemption in order to always check and fire events - * on the right vCPU - */ - critical_enter(); + /* We must remain on the same vCPU during this function */ + CRITICAL_ASSERT(curthread); cpu = PCPU_GET(cpuid); pc = DPCPU_PTR(xen_intr_pcpu); v = DPCPU_GET(vcpu_info); if (!xen_has_percpu_evtchn()) { KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU")); } v->evtchn_upcall_pending = 0; - -#if 0 -#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */ +/* No need for a barrier on x86 -- XCHG is a barrier on x86. */ +#if !defined(__amd64__) && !defined(__i386__) /* Clear master flag /before/ clearing selector flag. */ wmb(); #endif -#endif - l1 = atomic_readandclear_long(&v->evtchn_pending_sel); l1i = pc->last_processed_l1i; l2i = pc->last_processed_l2i; - (*pc->evtchn_intrcnt)++; while (l1 != 0) { l1i = (l1i + 1) % LONG_BIT; masked_l1 = l1 & ((~0UL) << l1i); if (masked_l1 == 0) { /* * if we masked out all events, wrap around * to the beginning. */ l1i = LONG_BIT - 1; l2i = LONG_BIT - 1; continue; } l1i = ffsl(masked_l1) - 1; do { l2 = xen_intr_active_ports(pc, l1i); l2i = (l2i + 1) % LONG_BIT; masked_l2 = l2 & ((~0UL) << l2i); if (masked_l2 == 0) { /* if we masked out all events, move on */ l2i = LONG_BIT - 1; break; } l2i = ffsl(masked_l2) - 1; /* process port */ port = (l1i * LONG_BIT) + l2i; evtchn_clear_port(port); isrc = xen_intr_port_to_isrc[port]; if (__predict_false(isrc == NULL)) continue; /* Make sure we are firing on the right vCPU */ KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)), ("Received unexpected event on vCPU#%u, event bound to vCPU#%u", PCPU_GET(cpuid), isrc->xi_cpu)); intr_execute_handlers(&isrc->xi_intsrc, trap_frame); /* * If this is the final port processed, * we'll pick up here+1 next time. */ pc->last_processed_l1i = l1i; pc->last_processed_l2i = l2i; } while (l2i != LONG_BIT - 1); l2 = xen_intr_active_ports(pc, l1i); if (l2 == 0) { /* * We handled all ports, so we can clear the * selector bit. */ l1 &= ~(1UL << l1i); } } - if (xen_evtchn_needs_ack) - lapic_eoi(); - - critical_exit(); + return (FILTER_HANDLED); } static int xen_intr_init(void *dummy __unused) { shared_info_t *s = HYPERVISOR_shared_info; struct xen_intr_pcpu_data *pcpu; int i; if (!xen_domain()) return (0); _Static_assert(is_valid_evtchn(0), "is_valid_evtchn(0) fails (unused by Xen, but valid by interface"); _Static_assert(is_valid_evtchn(NR_EVENT_CHANNELS - 1), "is_valid_evtchn(max) fails (is a valid channel)"); _Static_assert(!is_valid_evtchn(NR_EVENT_CHANNELS), "is_valid_evtchn(>max) fails (NOT a valid channel)"); _Static_assert(!is_valid_evtchn(~(evtchn_port_t)0), "is_valid_evtchn(maxint) fails (overflow?)"); _Static_assert(!is_valid_evtchn(INVALID_EVTCHN), "is_valid_evtchn(INVALID_EVTCHN) fails (must be invalid!)"); _Static_assert(!is_valid_evtchn(-1), "is_valid_evtchn(-1) fails (negative are invalid)"); mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF); mtx_init(&xen_intr_x86_lock, "xen-x86-table-lock", NULL, MTX_DEF); /* * Set the per-cpu mask of CPU#0 to enable all, since by default all * event channels are bound to CPU#0. */ CPU_FOREACH(i) { pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, sizeof(pcpu->evtchn_enabled)); } for (i = 0; i < nitems(s->evtchn_mask); i++) atomic_store_rel_long(&s->evtchn_mask[i], ~0); intr_register_pic(&xen_intr_pic); if (bootverbose) printf("Xen interrupt system initialized\n"); return (0); } SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL); -static void -xen_intrcnt_init(void *dummy __unused) -{ - unsigned int i; - - if (!xen_domain()) - return; - - /* - * Register interrupt count manually as we aren't guaranteed to see a - * call to xen_intr_assign_cpu() before our first interrupt. - */ - CPU_FOREACH(i) - xen_intr_intrcnt_add(i); -} -SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL); - void xen_intr_alloc_irqs(void) { if (num_io_irqs > UINT_MAX - NR_EVENT_CHANNELS) panic("IRQ allocation overflow (num_msi_irqs too high?)"); first_evtchn_irq = num_io_irqs; num_io_irqs += NR_EVENT_CHANNELS; } /*--------------------------- Common PIC Functions ---------------------------*/ /** * Prepare this PIC for system suspension. */ static void xen_intr_suspend(struct pic *unused) { } static void xen_rebind_ipi(struct xenisrc *isrc) { #ifdef SMP u_int cpu = isrc->xi_cpu; u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu); int error; struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi); if (error != 0) panic("unable to rebind xen IPI: %d", error); isrc->xi_port = bind_ipi.port; #else panic("Resume IPI event channel on UP"); #endif } static void xen_rebind_virq(struct xenisrc *isrc) { u_int cpu = isrc->xi_cpu; u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu); int error; struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq, .vcpu = vcpu_id }; error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq); if (error != 0) panic("unable to rebind xen VIRQ#%u: %d", isrc->xi_virq, error); isrc->xi_port = bind_virq.port; } static struct xenisrc * xen_intr_rebind_isrc(struct xenisrc *isrc) { #ifdef SMP u_int cpu = isrc->xi_cpu; int error; #endif struct xenisrc *prev; switch (isrc->xi_type) { case EVTCHN_TYPE_IPI: xen_rebind_ipi(isrc); break; case EVTCHN_TYPE_VIRQ: xen_rebind_virq(isrc); break; default: return (NULL); } prev = xen_intr_port_to_isrc[isrc->xi_port]; xen_intr_port_to_isrc[isrc->xi_port] = isrc; #ifdef SMP isrc->xi_cpu = 0; error = xen_intr_assign_cpu(&isrc->xi_intsrc, cpu); if (error) panic("%s(): unable to rebind Xen channel %u to vCPU%u: %d", __func__, isrc->xi_port, cpu, error); #endif evtchn_unmask_port(isrc->xi_port); return (prev); } /** * Return this PIC to service after being suspended. */ static void xen_intr_resume(struct pic *unused, bool suspend_cancelled) { shared_info_t *s = HYPERVISOR_shared_info; u_int isrc_idx; int i; if (suspend_cancelled) return; /* Reset the per-CPU masks */ CPU_FOREACH(i) { struct xen_intr_pcpu_data *pcpu; pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, sizeof(pcpu->evtchn_enabled)); } /* Mask all event channels. */ for (i = 0; i < nitems(s->evtchn_mask); i++) atomic_store_rel_long(&s->evtchn_mask[i], ~0); /* Clear existing port mappings */ for (isrc_idx = 0; isrc_idx < NR_EVENT_CHANNELS; ++isrc_idx) if (xen_intr_port_to_isrc[isrc_idx] != NULL) xen_intr_port_to_isrc[isrc_idx]->xi_port = INVALID_EVTCHN; /* Remap in-use isrcs, using xen_intr_port_to_isrc as listing */ for (isrc_idx = 0; isrc_idx < NR_EVENT_CHANNELS; ++isrc_idx) { struct xenisrc *cur = xen_intr_port_to_isrc[isrc_idx]; /* empty or entry already taken care of */ if (cur == NULL || cur->xi_port == isrc_idx) continue; xen_intr_port_to_isrc[isrc_idx] = NULL; do { KASSERT(!is_valid_evtchn(cur->xi_port), ("%s(): Multiple channels on single intr?", __func__)); cur = xen_intr_rebind_isrc(cur); } while (cur != NULL); } } /** * Disable a Xen interrupt source. * * \param isrc The interrupt source to disable. */ static void xen_intr_disable_intr(struct intsrc *base_isrc) { struct xenisrc *isrc = (struct xenisrc *)base_isrc; evtchn_mask_port(isrc->xi_port); } /** * Determine the global interrupt vector number for * a Xen interrupt source. * * \param isrc The interrupt source to query. * * \return The vector number corresponding to the given interrupt source. */ static int xen_intr_vector(struct intsrc *base_isrc) { struct xenisrc *isrc = (struct xenisrc *)base_isrc; return (isrc->xi_vector); } /** * Determine whether or not interrupt events are pending on the * the given interrupt source. * * \param isrc The interrupt source to query. * * \returns 0 if no events are pending, otherwise non-zero. */ static int xen_intr_source_pending(struct intsrc *isrc) { /* * EventChannels are edge triggered and never masked. * There can be no pending events. */ return (0); } /** * Perform configuration of an interrupt source. * * \param isrc The interrupt source to configure. * \param trig Edge or level. * \param pol Active high or low. * * \returns 0 if no events are pending, otherwise non-zero. */ static int xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig, enum intr_polarity pol) { /* Configuration is only possible via the evtchn apis. */ return (ENODEV); } /** * Configure CPU affinity for interrupt source event delivery. * * \param isrc The interrupt source to configure. * \param to_cpu The id of the CPU for handling future events. * * \returns 0 if successful, otherwise an errno. */ static int xen_intr_assign_cpu(struct intsrc *base_isrc, u_int to_cpu) { #ifdef SMP struct evtchn_bind_vcpu bind_vcpu; struct xenisrc *isrc; u_int vcpu_id = XEN_CPUID_TO_VCPUID(to_cpu); int error, masked; if (!xen_has_percpu_evtchn()) return (EOPNOTSUPP); mtx_lock(&xen_intr_isrc_lock); isrc = (struct xenisrc *)base_isrc; if (!is_valid_evtchn(isrc->xi_port)) { mtx_unlock(&xen_intr_isrc_lock); return (EINVAL); } /* * Mask the event channel while binding it to prevent interrupt * delivery with an inconsistent state in isrc->xi_cpu. */ masked = evtchn_test_and_set_mask(isrc->xi_port); if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) || (isrc->xi_type == EVTCHN_TYPE_IPI)) { /* * Virtual IRQs are associated with a cpu by * the Hypervisor at evtchn_bind_virq time, so * all we need to do is update the per-CPU masks. */ evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); isrc->xi_cpu = to_cpu; evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); goto out; } bind_vcpu.port = isrc->xi_port; bind_vcpu.vcpu = vcpu_id; error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu); if (isrc->xi_cpu != to_cpu) { if (error == 0) { /* Commit to new binding by removing the old one. */ evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); isrc->xi_cpu = to_cpu; evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); } } out: if (masked == 0) evtchn_unmask_port(isrc->xi_port); mtx_unlock(&xen_intr_isrc_lock); return (0); #else return (EOPNOTSUPP); #endif } /* Wrapper of xen_intr_assign_cpu to use as pic callbacks */ static int xen_intr_pic_assign_cpu(struct intsrc *isrc, u_int apic_id) { return (xen_intr_assign_cpu(isrc, apic_cpuid(apic_id))); } /*------------------- Virtual Interrupt Source PIC Functions -----------------*/ /* * Mask a level triggered interrupt source. * * \param isrc The interrupt source to mask (if necessary). * \param eoi If non-zero, perform any necessary end-of-interrupt * acknowledgements. */ static void xen_intr_disable_source(struct intsrc *base_isrc, int eoi) { struct xenisrc *isrc; isrc = (struct xenisrc *)base_isrc; /* * NB: checking if the event channel is already masked is * needed because the event channel user-space device * masks event channels on its filter as part of its * normal operation, and those shouldn't be automatically * unmasked by the generic interrupt code. The event channel * device will unmask them when needed. */ isrc->xi_masked = !!evtchn_test_and_set_mask(isrc->xi_port); } /* * Unmask a level triggered interrupt source. * * \param isrc The interrupt source to unmask (if necessary). */ static void xen_intr_enable_source(struct intsrc *base_isrc) { struct xenisrc *isrc; isrc = (struct xenisrc *)base_isrc; if (isrc->xi_masked == 0) evtchn_unmask_port(isrc->xi_port); } /* * Perform any necessary end-of-interrupt acknowledgements. * * \param isrc The interrupt source to EOI. */ static void xen_intr_eoi_source(struct intsrc *base_isrc) { } /* * Enable and unmask the interrupt source. * * \param isrc The interrupt source to enable. */ static void xen_intr_enable_intr(struct intsrc *base_isrc) { struct xenisrc *isrc = (struct xenisrc *)base_isrc; evtchn_unmask_port(isrc->xi_port); } /*--------------------------- Public Functions -------------------------------*/ /*------- API comments for these methods can be found in xen/xenintr.h -------*/ int xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep) { struct xenisrc *isrc; int error; error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT, device_get_nameunit(dev), filter, handler, arg, flags, port_handlep); if (error != 0) return (error); /* * The Event Channel API didn't open this port, so it is not * responsible for closing it automatically on unbind. */ isrc->xi_close = 0; return (0); } int xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep) { struct xenisrc *isrc; struct evtchn_alloc_unbound alloc_unbound; int error; alloc_unbound.dom = DOMID_SELF; alloc_unbound.remote_dom = remote_domain; error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, &alloc_unbound); if (error != 0) { /* * XXX Trap Hypercall error code Linuxisms in * the HYPERCALL layer. */ return (-error); } error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT, device_get_nameunit(dev), filter, handler, arg, flags, port_handlep); if (error != 0) { evtchn_close_t close = { .port = alloc_unbound.port }; if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) panic("EVTCHNOP_close failed"); return (error); } isrc->xi_close = 1; return (0); } int xen_intr_bind_remote_port(device_t dev, u_int remote_domain, u_int remote_port, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep) { struct xenisrc *isrc; struct evtchn_bind_interdomain bind_interdomain; int error; bind_interdomain.remote_dom = remote_domain; bind_interdomain.remote_port = remote_port; error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, &bind_interdomain); if (error != 0) { /* * XXX Trap Hypercall error code Linuxisms in * the HYPERCALL layer. */ return (-error); } error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port, EVTCHN_TYPE_PORT, device_get_nameunit(dev), filter, handler, arg, flags, port_handlep); if (error) { evtchn_close_t close = { .port = bind_interdomain.local_port }; if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) panic("EVTCHNOP_close failed"); return (error); } /* * The Event Channel API opened this port, so it is * responsible for closing it automatically on unbind. */ isrc->xi_close = 1; return (0); } int xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep) { u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu); struct xenisrc *isrc; struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id }; int error; isrc = NULL; error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq); if (error != 0) { /* * XXX Trap Hypercall error code Linuxisms in * the HYPERCALL layer. */ return (-error); } error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, device_get_nameunit(dev), filter, handler, arg, flags, port_handlep); #ifdef SMP if (error == 0) error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); #endif if (error != 0) { evtchn_close_t close = { .port = bind_virq.port }; xen_intr_unbind(*port_handlep); if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) panic("EVTCHNOP_close failed"); return (error); } #ifdef SMP if (isrc->xi_cpu != cpu) { /* * Too early in the boot process for the generic interrupt * code to perform the binding. Update our event channel * masks manually so events can't fire on the wrong cpu * during AP startup. */ xen_intr_assign_cpu(&isrc->xi_intsrc, cpu); } #endif /* * The Event Channel API opened this port, so it is * responsible for closing it automatically on unbind. */ isrc->xi_close = 1; isrc->xi_virq = virq; return (0); } int xen_intr_alloc_and_bind_ipi(u_int cpu, driver_filter_t filter, enum intr_type flags, xen_intr_handle_t *port_handlep) { #ifdef SMP u_int vcpu_id = XEN_CPUID_TO_VCPUID(cpu); struct xenisrc *isrc; struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; /* Same size as the one used by intr_handler->ih_name. */ char name[MAXCOMLEN + 1]; int error; isrc = NULL; error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi); if (error != 0) { /* * XXX Trap Hypercall error code Linuxisms in * the HYPERCALL layer. */ return (-error); } snprintf(name, sizeof(name), "cpu%u", cpu); error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI, name, filter, NULL, NULL, flags, port_handlep); if (error != 0) { evtchn_close_t close = { .port = bind_ipi.port }; xen_intr_unbind(*port_handlep); if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) panic("EVTCHNOP_close failed"); return (error); } if (isrc->xi_cpu != cpu) { /* * Too early in the boot process for the generic interrupt * code to perform the binding. Update our event channel * masks manually so events can't fire on the wrong cpu * during AP startup. */ xen_intr_assign_cpu(&isrc->xi_intsrc, cpu); } /* * The Event Channel API opened this port, so it is * responsible for closing it automatically on unbind. */ isrc->xi_close = 1; return (0); #else return (EOPNOTSUPP); #endif } int xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...) { char descr[MAXCOMLEN + 1]; struct xenisrc *isrc; va_list ap; isrc = xen_intr_isrc_from_handle(port_handle); if (isrc == NULL) return (EINVAL); va_start(ap, fmt); vsnprintf(descr, sizeof(descr), fmt, ap); va_end(ap); return (intr_describe(isrc->xi_vector, isrc->xi_cookie, descr)); } void xen_intr_unbind(xen_intr_handle_t *port_handlep) { struct xenisrc *isrc; KASSERT(port_handlep != NULL, ("NULL xen_intr_handle_t passed to %s", __func__)); isrc = xen_intr_isrc_from_handle(*port_handlep); *port_handlep = NULL; if (isrc == NULL) return; mtx_lock(&xen_intr_isrc_lock); if (refcount_release(&isrc->xi_refcount) == 0) { mtx_unlock(&xen_intr_isrc_lock); return; } mtx_unlock(&xen_intr_isrc_lock); if (isrc->xi_cookie != NULL) intr_remove_handler(isrc->xi_cookie); xen_intr_release_isrc(isrc); } void xen_intr_signal(xen_intr_handle_t handle) { struct xenisrc *isrc; isrc = xen_intr_isrc_from_handle(handle); if (isrc != NULL) { KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT || isrc->xi_type == EVTCHN_TYPE_IPI, ("evtchn_signal on something other than a local port")); struct evtchn_send send = { .port = isrc->xi_port }; (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send); } } evtchn_port_t xen_intr_port(xen_intr_handle_t handle) { struct xenisrc *isrc; isrc = xen_intr_isrc_from_handle(handle); if (isrc == NULL) return (0); return (isrc->xi_port); } int xen_intr_add_handler(const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, xen_intr_handle_t handle) { struct xenisrc *isrc; int error; isrc = xen_intr_isrc_from_handle(handle); if (isrc == NULL || isrc->xi_cookie != NULL) return (EINVAL); error = intr_add_handler(name, isrc->xi_vector,filter, handler, arg, flags|INTR_EXCL, &isrc->xi_cookie, 0); if (error != 0) printf("%s: %s: add handler failed: %d\n", name, __func__, error); return (error); } int xen_intr_get_evtchn_from_port(evtchn_port_t port, xen_intr_handle_t *handlep) { if (!is_valid_evtchn(port)) return (EINVAL); if (handlep == NULL) { return (EINVAL); } mtx_lock(&xen_intr_isrc_lock); if (xen_intr_port_to_isrc[port] == NULL) { mtx_unlock(&xen_intr_isrc_lock); return (EINVAL); } refcount_acquire(&xen_intr_port_to_isrc[port]->xi_refcount); mtx_unlock(&xen_intr_isrc_lock); /* Assign the opaque handler */ *handlep = xen_intr_handle_from_isrc(xen_intr_port_to_isrc[port]); return (0); } #ifdef DDB static const char * xen_intr_print_type(enum evtchn_type type) { static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = { [EVTCHN_TYPE_UNBOUND] = "UNBOUND", [EVTCHN_TYPE_VIRQ] = "VIRQ", [EVTCHN_TYPE_IPI] = "IPI", [EVTCHN_TYPE_PORT] = "PORT", }; if (type >= EVTCHN_TYPE_COUNT) return ("UNKNOWN"); return (evtchn_type_to_string[type]); } static void xen_intr_dump_port(struct xenisrc *isrc) { struct xen_intr_pcpu_data *pcpu; shared_info_t *s = HYPERVISOR_shared_info; u_int i; db_printf("Port %d Type: %s\n", isrc->xi_port, xen_intr_print_type(isrc->xi_type)); if (isrc->xi_type == EVTCHN_TYPE_VIRQ) db_printf("\tVirq: %u\n", isrc->xi_virq); db_printf("\tMasked: %d Pending: %d\n", !!xen_test_bit(isrc->xi_port, &s->evtchn_mask[0]), !!xen_test_bit(isrc->xi_port, &s->evtchn_pending[0])); db_printf("\tPer-CPU Masks: "); CPU_FOREACH(i) { pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); db_printf("cpu#%u: %d ", i, !!xen_test_bit(isrc->xi_port, pcpu->evtchn_enabled)); } db_printf("\n"); } DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn) { u_int i; if (!xen_domain()) { db_printf("Only available on Xen guests\n"); return; } for (i = 0; i < NR_EVENT_CHANNELS; i++) { struct xenisrc *isrc; isrc = xen_intr_port_to_isrc[i]; if (isrc == NULL) continue; xen_intr_dump_port(isrc); } } #endif /* DDB */ diff --git a/sys/xen/xen_intr.h b/sys/xen/xen_intr.h index 5ebce0678222..d8e85b73d95e 100644 --- a/sys/xen/xen_intr.h +++ b/sys/xen/xen_intr.h @@ -1,240 +1,243 @@ /****************************************************************************** * xen_intr.h * * APIs for managing Xen event channel, virtual IRQ, and physical IRQ * notifications. * * Copyright (c) 2004, K A Fraser * Copyright (c) 2012, Spectra Logic Corporation * * This file may be distributed separately from the Linux kernel, or * incorporated into other software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * $FreeBSD$ */ #ifndef _XEN_INTR_H_ #define _XEN_INTR_H_ #include /** Registered Xen interrupt callback handle. */ typedef void * xen_intr_handle_t; -void xen_intr_handle_upcall(struct trapframe *trap_frame); +/* + * Main handler for Xen event channel interrupts + */ +extern driver_filter_t xen_intr_handle_upcall; /** * Associate an already allocated local event channel port an interrupt * handler. * * \param dev The device making this bind request. * \param local_port The event channel to bind. * \param filter An interrupt filter handler. Specify NULL * to always dispatch to the ithread handler. * \param handler An interrupt ithread handler. Optional (can * specify NULL) if all necessary event actions * are performed by filter. * \param arg Argument to present to both filter and handler. * \param irqflags Interrupt handler flags. See sys/bus.h. * \param handlep Pointer to an opaque handle used to manage this * registration. * * \returns 0 on success, otherwise an errno. */ int xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type irqflags, xen_intr_handle_t *handlep); /** * Allocate a local event channel port, accessible by the specified * remote/foreign domain and, if successful, associate the port with * the specified interrupt handler. * * \param dev The device making this bind request. * \param remote_domain Remote domain grant permission to signal the * newly allocated local port. * \param filter An interrupt filter handler. Specify NULL * to always dispatch to the ithread handler. * \param handler An interrupt ithread handler. Optional (can * specify NULL) if all necessary event actions * are performed by filter. * \param arg Argument to present to both filter and handler. * \param irqflags Interrupt handler flags. See sys/bus.h. * \param handlep Pointer to an opaque handle used to manage this * registration. * * \returns 0 on success, otherwise an errno. */ int xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type irqflags, xen_intr_handle_t *handlep); /** * Associate the specified interrupt handler with the remote event * channel port specified by remote_domain and remote_port. * * \param dev The device making this bind request. * \param remote_domain The domain peer for this event channel connection. * \param remote_port Remote domain's local port number for this event * channel port. * \param filter An interrupt filter handler. Specify NULL * to always dispatch to the ithread handler. * \param handler An interrupt ithread handler. Optional (can * specify NULL) if all necessary event actions * are performed by filter. * \param arg Argument to present to both filter and handler. * \param irqflags Interrupt handler flags. See sys/bus.h. * \param handlep Pointer to an opaque handle used to manage this * registration. * * \returns 0 on success, otherwise an errno. */ int xen_intr_bind_remote_port(device_t dev, u_int remote_domain, evtchn_port_t remote_port, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type irqflags, xen_intr_handle_t *handlep); /** * Associate the specified interrupt handler with the specified Xen * virtual interrupt source. * * \param dev The device making this bind request. * \param virq The Xen virtual IRQ number for the Xen interrupt * source being hooked. * \param cpu The cpu on which interrupt events should be delivered. * \param filter An interrupt filter handler. Specify NULL * to always dispatch to the ithread handler. * \param handler An interrupt ithread handler. Optional (can * specify NULL) if all necessary event actions * are performed by filter. * \param arg Argument to present to both filter and handler. * \param irqflags Interrupt handler flags. See sys/bus.h. * \param handlep Pointer to an opaque handle used to manage this * registration. * * \returns 0 on success, otherwise an errno. */ int xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type irqflags, xen_intr_handle_t *handlep); /** * Allocate a local event channel port for servicing interprocessor * interupts and, if successful, associate the port with the specified * interrupt handler. * * \param cpu The cpu receiving the IPI. * \param filter The interrupt filter servicing this IPI. * \param irqflags Interrupt handler flags. See sys/bus.h. * \param handlep Pointer to an opaque handle used to manage this * registration. * * \returns 0 on success, otherwise an errno. */ int xen_intr_alloc_and_bind_ipi(u_int cpu, driver_filter_t filter, enum intr_type irqflags, xen_intr_handle_t *handlep); /** * Unbind an interrupt handler from its interrupt source. * * \param handlep A pointer to the opaque handle that was initialized * at the time the interrupt source was bound. * * \returns 0 on success, otherwise an errno. * * \note The event channel, if any, that was allocated at bind time is * closed upon successful return of this method. * * \note It is always safe to call xen_intr_unbind() on a handle that * has been initilized to NULL. */ void xen_intr_unbind(xen_intr_handle_t *handle); /** * Add a description to an interrupt handler. * * \param handle The opaque handle that was initialized at the time * the interrupt source was bound. * * \param fmt The sprintf compatible format string for the description, * followed by optional sprintf arguments. * * \returns 0 on success, otherwise an errno. */ int xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...) __attribute__((format(printf, 2, 3))); /** * Signal the remote peer of an interrupt source associated with an * event channel port. * * \param handle The opaque handle that was initialized at the time * the interrupt source was bound. * * \note For xen interrupt sources other than event channel ports, * this method takes no action. */ void xen_intr_signal(xen_intr_handle_t handle); /** * Get the local event channel port number associated with this interrupt * source. * * \param handle The opaque handle that was initialized at the time * the interrupt source was bound. * * \returns 0 if the handle is invalid, otherwise positive port number. */ evtchn_port_t xen_intr_port(xen_intr_handle_t handle); /** * Bind an event channel port with a handler * * \param dev The device making this bind request. * \param filter An interrupt filter handler. Specify NULL * to always dispatch to the ithread handler. * \param handler An interrupt ithread handler. Optional (can * specify NULL) if all necessary event actions * are performed by filter. * \param arg Argument to present to both filter and handler. * \param irqflags Interrupt handler flags. See sys/bus.h. * \param handle Opaque handle used to manage this registration. * * \returns 0 on success, otherwise an errno. */ int xen_intr_add_handler(const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, xen_intr_handle_t handle); /** * Get a reference to an event channel port * * \param port Event channel port to which we get a reference. * \param handlep Pointer to an opaque handle used to manage this * registration. * * \returns 0 on success, otherwise an errno. */ int xen_intr_get_evtchn_from_port(evtchn_port_t port, xen_intr_handle_t *handlep); #endif /* _XEN_INTR_H_ */