Page MenuHomeFreeBSD
Files F151093637

D9880.diff
No OneTemporary
Actions

D9880.diff
View Options

Index: head/sys/amd64/amd64/apic_vector.S
===================================================================
--- head/sys/amd64/amd64/apic_vector.S
+++ head/sys/amd64/amd64/apic_vector.S
@@ -50,22 +50,6 @@
#define LK
#endif
- .text
- SUPERALIGN_TEXT
- /* End Of Interrupt to APIC */
-as_lapic_eoi:
- cmpl $0,x2apic_mode
- jne 1f
- movq lapic_map,%rax
- movl $0,LA_EOI(%rax)
- ret
-1:
- movl $MSR_APIC_EOI,%ecx
- xorl %eax,%eax
- xorl %edx,%edx
- wrmsr
- ret
-
/*
* I/O Interrupt Entry Point. Rather than having one entry point for
* each interrupt source, we use one entry point for each 32-bit word
@@ -182,7 +166,7 @@
SUPERALIGN_TEXT
invltlb_ret:
- call as_lapic_eoi
+ call native_lapic_eoi
POP_FRAME
jmp doreti_iret
@@ -191,18 +175,21 @@
PUSH_FRAME
call invltlb_handler
+ movl $IPI_INVLTLB, %edi
jmp invltlb_ret
IDTVEC(invltlb_pcid)
PUSH_FRAME
call invltlb_pcid_handler
+ movl $IPI_INVLTLB, %edi
jmp invltlb_ret
IDTVEC(invltlb_invpcid)
PUSH_FRAME
call invltlb_invpcid_handler
+ movl $IPI_INVLTLB, %edi
jmp invltlb_ret
/*
@@ -215,6 +202,7 @@
PUSH_FRAME
call invlpg_handler
+ movl $IPI_INVLPG, %edi
jmp invltlb_ret
/*
@@ -226,6 +214,7 @@
PUSH_FRAME
call invlrng_handler
+ movl $IPI_INVLRNG, %edi
jmp invltlb_ret
/*
@@ -237,6 +226,7 @@
PUSH_FRAME
call invlcache_handler
+ movl $IPI_INVLCACHE, %edi
jmp invltlb_ret
/*
@@ -247,7 +237,8 @@
IDTVEC(ipi_intr_bitmap_handler)
PUSH_FRAME
- call as_lapic_eoi
+ movl $IPI_BITMAP_VECTOR, %edi
+ call native_lapic_eoi
FAKE_MCOUNT(TF_RIP(%rsp))
@@ -263,7 +254,8 @@
IDTVEC(cpustop)
PUSH_FRAME
- call as_lapic_eoi
+ movl $IPI_STOP, %edi
+ call native_lapic_eoi
call cpustop_handler
jmp doreti
@@ -277,7 +269,8 @@
PUSH_FRAME
call cpususpend_handler
- call as_lapic_eoi
+ movl $IPI_SUSPEND, %edi
+ call native_lapic_eoi
jmp doreti
/*
@@ -295,37 +288,22 @@
incq (%rax)
#endif
call smp_rendezvous_action
- call as_lapic_eoi
+ movl $IPI_RENDEZVOUS, %edi
+ call native_lapic_eoi
jmp doreti
/*
* IPI handler whose purpose is to interrupt the CPU with minimum overhead.
* This is used by bhyve to force a host cpu executing in guest context to
* trap into the hypervisor.
- *
- * This handler is different from other IPI handlers in the following aspects:
- *
- * 1. It doesn't push a trapframe on the stack.
- *
- * This implies that a DDB backtrace involving 'justreturn' will skip the
- * function that was interrupted by this handler.
- *
- * 2. It doesn't 'swapgs' when userspace is interrupted.
- *
- * The 'justreturn' handler does not access any pcpu data so it is not an
- * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI
- * whose handler already doesn't trust GS.base when kernel code is interrupted.
*/
.text
SUPERALIGN_TEXT
IDTVEC(justreturn)
- pushq %rax
- pushq %rcx
- pushq %rdx
- call as_lapic_eoi
- popq %rdx
- popq %rcx
- popq %rax
+ PUSH_FRAME
+ movl vmm_ipinum, %edi
+ call native_lapic_eoi
+ POP_FRAME
jmp doreti_iret
#endif /* SMP */
Index: head/sys/amd64/amd64/genassym.c
===================================================================
--- head/sys/amd64/amd64/genassym.c
+++ head/sys/amd64/amd64/genassym.c
@@ -44,6 +44,7 @@
#include <sys/assym.h>
#include <sys/bio.h>
#include <sys/buf.h>
+#include <sys/bus.h>
#include <sys/proc.h>
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
@@ -61,7 +62,9 @@
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <sys/proc.h>
+#include <machine/intr_machdep.h>
#include <x86/apicreg.h>
+#include <x86/apicvar.h>
#include <machine/cpu.h>
#include <machine/pcb.h>
#include <machine/sigframe.h>
@@ -215,10 +218,19 @@
ASSYM(PC_COMMONTSSP, offsetof(struct pcpu, pc_commontssp));
ASSYM(PC_TSS, offsetof(struct pcpu, pc_tss));
ASSYM(PC_PM_SAVE_CNT, offsetof(struct pcpu, pc_pm_save_cnt));
-
+
ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL);
ASSYM(LA_ISR, LAPIC_ISR0 * LAPIC_MEM_MUL);
+ASSYM(IPI_INVLTLB, IPI_INVLTLB);
+ASSYM(IPI_INVLPG, IPI_INVLPG);
+ASSYM(IPI_INVLRNG, IPI_INVLRNG);
+ASSYM(IPI_INVLCACHE, IPI_INVLCACHE);
+ASSYM(IPI_BITMAP_VECTOR, IPI_BITMAP_VECTOR);
+ASSYM(IPI_STOP, IPI_STOP);
+ASSYM(IPI_SUSPEND, IPI_SUSPEND);
+ASSYM(IPI_RENDEZVOUS, IPI_RENDEZVOUS);
+
ASSYM(KCSEL, GSEL(GCODE_SEL, SEL_KPL));
ASSYM(KDSEL, GSEL(GDATA_SEL, SEL_KPL));
ASSYM(KUCSEL, GSEL(GUCODE_SEL, SEL_UPL));
Index: head/sys/amd64/vmm/amd/svm.c
===================================================================
--- head/sys/amd64/vmm/amd/svm.c
+++ head/sys/amd64/vmm/amd/svm.c
@@ -48,6 +48,7 @@
#include <machine/vmm_dev.h>
#include <machine/vmm_instruction_emul.h>
+#include "vmm_host.h"
#include "vmm_lapic.h"
#include "vmm_stat.h"
#include "vmm_ktr.h"
@@ -1620,7 +1621,7 @@
* Although not explicitly specified in APMv2 the
* relative priorities were verified empirically.
*/
- ipi_cpu(curcpu, IPI_AST); /* XXX vmm_ipinum? */
+ ipi_cpu(curcpu, vmm_ipinum);
} else {
vm_nmi_clear(sc->vm, vcpu);
Index: head/sys/amd64/vmm/vmm.c
===================================================================
--- head/sys/amd64/vmm/vmm.c
+++ head/sys/amd64/vmm/vmm.c
@@ -215,7 +215,6 @@
&halt_detection_enabled, 0,
"Halt VM if all vcpus execute HLT with interrupts disabled");
-static int vmm_ipinum;
SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
"IPI vector used for vcpu notifications");
Index: head/sys/amd64/vmm/vmm_host.h
===================================================================
--- head/sys/amd64/vmm/vmm_host.h
+++ head/sys/amd64/vmm/vmm_host.h
@@ -33,6 +33,8 @@
#error "no user-serviceable parts inside"
#endif
+extern int vmm_ipinum;
+
struct xsave_limits {
int xsave_enabled;
uint64_t xcr0_allowed;
Index: head/sys/i386/i386/apic_vector.s
===================================================================
--- head/sys/i386/i386/apic_vector.s
+++ head/sys/i386/i386/apic_vector.s
@@ -44,22 +44,6 @@
#include "assym.s"
- .text
- SUPERALIGN_TEXT
- /* End Of Interrupt to APIC */
-as_lapic_eoi:
- cmpl $0,x2apic_mode
- jne 1f
- movl lapic_map,%eax
- movl $0,LA_EOI(%eax)
- ret
-1:
- movl $MSR_APIC_EOI,%ecx
- xorl %eax,%eax
- xorl %edx,%edx
- wrmsr
- ret
-
/*
* I/O Interrupt Entry Point. Rather than having one entry point for
* each interrupt source, we use one entry point for each 32-bit word
@@ -188,7 +172,8 @@
.text
SUPERALIGN_TEXT
invltlb_ret:
- call as_lapic_eoi
+ call native_lapic_eoi
+ add $4, %esp
POP_FRAME
iret
@@ -200,6 +185,7 @@
call invltlb_handler
+ pushl $IPI_INVLTLB
jmp invltlb_ret
/*
@@ -214,6 +200,7 @@
call invlpg_handler
+ pushl $IPI_INVLPG
jmp invltlb_ret
/*
@@ -228,6 +215,7 @@
call invlrng_handler
+ pushl $IPI_INVLRNG
jmp invltlb_ret
/*
@@ -242,6 +230,7 @@
call invlcache_handler
+ pushl $IPI_INVLCACHE
jmp invltlb_ret
/*
@@ -254,8 +243,10 @@
SET_KERNEL_SREGS
cld
- call as_lapic_eoi
-
+ pushl $IPI_BITMAP_VECTOR
+ call native_lapic_eoi
+ add $4, %esp
+
FAKE_MCOUNT(TF_EIP(%esp))
call ipi_bitmap_handler
@@ -272,7 +263,9 @@
SET_KERNEL_SREGS
cld
- call as_lapic_eoi
+ pushl $IPI_STOP
+ call native_lapic_eoi
+ add $4, %esp
call cpustop_handler
POP_FRAME
@@ -288,7 +281,9 @@
SET_KERNEL_SREGS
cld
- call as_lapic_eoi
+ pushl $IPI_SUSPEND
+ call native_lapic_eoi
+ add $4, %esp
call cpususpend_handler
POP_FRAME
@@ -313,7 +308,9 @@
#endif
call smp_rendezvous_action
- call as_lapic_eoi
+ pushl $IPI_RENDEZVOUS
+ call native_lapic_eoi
+ add $4, %esp
POP_FRAME
iret
Index: head/sys/i386/i386/genassym.c
===================================================================
--- head/sys/i386/i386/genassym.c
+++ head/sys/i386/i386/genassym.c
@@ -70,7 +70,10 @@
#include <nfsclient/nfs.h>
#include <nfs/nfsdiskless.h>
#ifdef DEV_APIC
+#include <sys/bus.h>
+#include <machine/intr_machdep.h>
#include <x86/apicreg.h>
+#include <x86/apicvar.h>
#endif
#include <machine/cpu.h>
#include <machine/pcb.h>
@@ -219,6 +222,15 @@
#ifdef DEV_APIC
ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL);
ASSYM(LA_ISR, LAPIC_ISR0 * LAPIC_MEM_MUL);
+
+ASSYM(IPI_INVLTLB, IPI_INVLTLB);
+ASSYM(IPI_INVLPG, IPI_INVLPG);
+ASSYM(IPI_INVLRNG, IPI_INVLRNG);
+ASSYM(IPI_INVLCACHE, IPI_INVLCACHE);
+ASSYM(IPI_BITMAP_VECTOR, IPI_BITMAP_VECTOR);
+ASSYM(IPI_STOP, IPI_STOP);
+ASSYM(IPI_SUSPEND, IPI_SUSPEND);
+ASSYM(IPI_RENDEZVOUS, IPI_RENDEZVOUS);
#endif
ASSYM(KCSEL, GSEL(GCODE_SEL, SEL_KPL));
Index: head/sys/x86/include/apicvar.h
===================================================================
--- head/sys/x86/include/apicvar.h
+++ head/sys/x86/include/apicvar.h
@@ -210,7 +210,7 @@
void (*setup)(int);
void (*dump)(const char *);
void (*disable)(void);
- void (*eoi)(void);
+ void (*eoi)(u_int vector);
int (*id)(void);
int (*intr_pending)(u_int);
void (*set_logical_id)(u_int, u_int, u_int);
@@ -301,10 +301,10 @@
}
static inline void
-lapic_eoi(void)
+lapic_eoi(u_int vector)
{
- apic_ops.eoi();
+ apic_ops.eoi(vector);
}
static inline int
@@ -469,6 +469,7 @@
return (apic_ops.set_lvt_triggermode(apic_id, lvt, trigger));
}
+void native_lapic_eoi(u_int vector);
void lapic_handle_cmc(void);
void lapic_handle_error(void);
void lapic_handle_intr(int vector, struct trapframe *frame);
Index: head/sys/x86/x86/io_apic.c
===================================================================
--- head/sys/x86/x86/io_apic.c
+++ head/sys/x86/x86/io_apic.c
@@ -151,10 +151,10 @@
volatile uint32_t *apic_eoi;
uint32_t low1;
- lapic_eoi();
+ src = (struct ioapic_intsrc *)isrc;
+ lapic_eoi(src->io_vector);
if (!lapic_eoi_suppression)
return;
- src = (struct ioapic_intsrc *)isrc;
if (src->io_edgetrigger)
return;
io = (struct ioapic *)isrc->is_pic;
Index: head/sys/x86/x86/local_apic.c
===================================================================
--- head/sys/x86/x86/local_apic.c
+++ head/sys/x86/x86/local_apic.c
@@ -83,6 +83,9 @@
#define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL)
#endif
+#define INTEL_SEOI 1
+#define AMD_SEOI 2
+
/* Sanity checks on IDT vectors. */
CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
@@ -184,6 +187,10 @@
#ifdef SMP
static uint64_t lapic_ipi_wait_mult;
#endif
+#ifdef __amd64__
+/* IPI vector used for VMM VCPU notifications. */
+int vmm_ipinum;
+#endif
SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options");
SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
@@ -312,7 +319,6 @@
static void native_lapic_setup(int boot);
static void native_lapic_dump(const char *str);
static void native_lapic_disable(void);
-static void native_lapic_eoi(void);
static int native_lapic_id(void);
static int native_lapic_intr_pending(u_int vector);
static u_int native_apic_cpuid(u_int apic_id);
@@ -447,6 +453,31 @@
return (lvt_mode_impl(la, elvt, idx, value));
}
+static inline uint32_t
+amd_read_ext_features(void)
+{
+ uint32_t version;
+
+ if (cpu_vendor_id != CPU_VENDOR_AMD)
+ return (0);
+ version = lapic_read32(LAPIC_VERSION);
+ if ((version & APIC_VER_AMD_EXT_SPACE) != 0)
+ return (lapic_read32(LAPIC_EXT_FEATURES));
+ return (0);
+}
+
+static inline uint32_t
+amd_read_elvt_count(void)
+{
+ uint32_t extf;
+ uint32_t count;
+
+ extf = amd_read_ext_features();
+ count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT;
+ count = min(count, APIC_ELVT_MAX + 1);
+ return (count);
+}
+
/*
* Map the local APIC and setup necessary interrupt vectors.
*/
@@ -456,9 +487,9 @@
#ifdef SMP
uint64_t r, r1, r2, rx;
#endif
- uint32_t ver;
+ uint32_t extf, ver;
u_int regs[4];
- int i, arat;
+ int i, arat, seoi_enable;
/*
* Enable x2APIC mode if possible. Map the local APIC
@@ -546,16 +577,27 @@
*/
ver = lapic_read32(LAPIC_VERSION);
if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) {
- lapic_eoi_suppression = 1;
+ lapic_eoi_suppression = INTEL_SEOI;
+ } else {
+ extf = amd_read_ext_features();
+ if ((extf & APIC_EXTF_SEIO_CAP) != 0)
+ lapic_eoi_suppression = AMD_SEOI;
+ }
+ if (lapic_eoi_suppression != 0) {
+ seoi_enable = 1;
if (vm_guest == VM_GUEST_KVM) {
if (bootverbose)
printf(
"KVM -- disabling lapic eoi suppression\n");
- lapic_eoi_suppression = 0;
+ seoi_enable = 0;
}
TUNABLE_INT_FETCH("hw.lapic_eoi_suppression",
- &lapic_eoi_suppression);
+ &seoi_enable);
+ if (seoi_enable == 0)
+ lapic_eoi_suppression = 0;
}
+ if (lapic_eoi_suppression != 0)
+ printf("LAPIC specific EOI enabled\n");
#ifdef SMP
#define LOOPS 100000
@@ -642,32 +684,6 @@
#endif
}
-static inline uint32_t
-amd_read_ext_features(void)
-{
- uint32_t version;
-
- if (cpu_vendor_id != CPU_VENDOR_AMD)
- return (0);
- version = lapic_read32(LAPIC_VERSION);
- if ((version & APIC_VER_AMD_EXT_SPACE) != 0)
- return (lapic_read32(LAPIC_EXT_FEATURES));
- else
- return (0);
-}
-
-static inline uint32_t
-amd_read_elvt_count(void)
-{
- uint32_t extf;
- uint32_t count;
-
- extf = amd_read_ext_features();
- count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT;
- count = min(count, APIC_ELVT_MAX + 1);
- return (count);
-}
-
/*
* Dump contents of local APIC registers
*/
@@ -702,9 +718,11 @@
extf = amd_read_ext_features();
if (extf != 0) {
printf(" AMD ext features: 0x%08x\n", extf);
+ extf = lapic_read32(LAPIC_EXT_CTRL);
+ printf(" AMD ext control: 0x%08x\n", extf);
elvt_count = amd_read_elvt_count();
for (i = 0; i < elvt_count; i++)
- printf(" AMD elvt%d: 0x%08x\n", i,
+ printf(" AMD elvt%d: 0x%08x\n", i,
lapic_read32(LAPIC_EXT_LVT0 + i));
}
}
@@ -1022,9 +1040,15 @@
value = lapic_read32(LAPIC_SVR);
value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS);
value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT;
- if (lapic_eoi_suppression)
+ if (lapic_eoi_suppression == INTEL_SEOI)
value |= APIC_SVR_EOI_SUPPRESSION;
lapic_write32(LAPIC_SVR, value);
+
+ if (lapic_eoi_suppression == AMD_SEOI) {
+ value = lapic_read32(LAPIC_EXT_CTRL);
+ value |= APIC_EXTF_SEIO_CAP;
+ lapic_write32(LAPIC_EXT_CTRL, value);
+ }
}
/* Reset the local APIC on the BSP during resume. */
@@ -1227,11 +1251,14 @@
#endif
}
-static void
-native_lapic_eoi(void)
+void
+native_lapic_eoi(u_int vector)
{
- lapic_write32_nofence(LAPIC_EOI, 0);
+ if (lapic_eoi_suppression == AMD_SEOI)
+ lapic_write32(LAPIC_EXT_SEOI, vector);
+ else
+ lapic_write32_nofence(LAPIC_EOI, 0);
}
void
@@ -1252,7 +1279,7 @@
struct thread *td;
/* Send EOI first thing. */
- lapic_eoi();
+ lapic_eoi(APIC_TIMER_INT);
#if defined(SMP) && !defined(SCHED_ULE)
/*
@@ -1373,7 +1400,7 @@
lapic_handle_cmc(void)
{
- lapic_eoi();
+ lapic_eoi(APIC_CMC_INT);
cmc_intr();
}
@@ -1447,7 +1474,7 @@
esr = lapic_read32(LAPIC_ESR);
printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr);
- lapic_eoi();
+ lapic_eoi(APIC_ERROR_INT);
}
static u_int
Index: head/sys/x86/x86/msi.c
===================================================================
--- head/sys/x86/x86/msi.c
+++ head/sys/x86/x86/msi.c
@@ -177,16 +177,18 @@
static void
msi_disable_source(struct intsrc *isrc, int eoi)
{
+ struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
if (eoi == PIC_EOI)
- lapic_eoi();
+ lapic_eoi(msi->msi_vector);
}
static void
msi_eoi_source(struct intsrc *isrc)
{
+ struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
- lapic_eoi();
+ lapic_eoi(msi->msi_vector);
}
static void
Index: head/sys/x86/xen/xen_apic.c
===================================================================
--- head/sys/x86/xen/xen_apic.c
+++ head/sys/x86/xen/xen_apic.c
@@ -147,7 +147,7 @@
}
static void
-xen_pv_lapic_eoi(void)
+xen_pv_lapic_eoi(u_int vector)
{
XEN_APIC_UNSUPPORTED;

File Metadata

Mime Type
text/plain
Expires
Mon, Apr 6, 11:54 PM (14 h, 50 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
31002822
Default Alt Text
D9880.diff (15 KB)

Event Timeline