Changeset View
Standalone View
sys/x86/x86/mp_x86.c
Show First 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | |||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/bus.h> | #include <sys/bus.h> | ||||
#include <sys/cons.h> /* cngetc() */ | #include <sys/cons.h> /* cngetc() */ | ||||
#include <sys/cpuset.h> | #include <sys/cpuset.h> | ||||
#include <sys/csan.h> | #include <sys/csan.h> | ||||
#ifdef GPROF | #ifdef GPROF | ||||
#include <sys/gmon.h> | #include <sys/gmon.h> | ||||
#endif | #endif | ||||
#include <sys/interrupt.h> | |||||
#include <sys/kdb.h> | #include <sys/kdb.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/ktr.h> | #include <sys/ktr.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/memrange.h> | #include <sys/memrange.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/pcpu.h> | #include <sys/pcpu.h> | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | |||||
static u_long *ipi_preempt_counts[MAXCPU]; | static u_long *ipi_preempt_counts[MAXCPU]; | ||||
static u_long *ipi_ast_counts[MAXCPU]; | static u_long *ipi_ast_counts[MAXCPU]; | ||||
u_long *ipi_invltlb_counts[MAXCPU]; | u_long *ipi_invltlb_counts[MAXCPU]; | ||||
u_long *ipi_invlrng_counts[MAXCPU]; | u_long *ipi_invlrng_counts[MAXCPU]; | ||||
u_long *ipi_invlpg_counts[MAXCPU]; | u_long *ipi_invlpg_counts[MAXCPU]; | ||||
u_long *ipi_invlcache_counts[MAXCPU]; | u_long *ipi_invlcache_counts[MAXCPU]; | ||||
u_long *ipi_rendezvous_counts[MAXCPU]; | u_long *ipi_rendezvous_counts[MAXCPU]; | ||||
static u_long *ipi_hardclock_counts[MAXCPU]; | static u_long *ipi_hardclock_counts[MAXCPU]; | ||||
static u_long *ipi_swi_counts[MAXCPU]; | |||||
#endif | #endif | ||||
/* Default cpu_ops implementation. */ | /* Default cpu_ops implementation. */ | ||||
struct cpu_ops cpu_ops; | struct cpu_ops cpu_ops; | ||||
/* | /* | ||||
* Local data and functions. | * Local data and functions. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 1,108 Lines • ▼ Show 20 Lines | lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | | ||||
vector, apic_id); | vector, apic_id); | ||||
if (!lapic_ipi_wait(100)) | if (!lapic_ipi_wait(100)) | ||||
panic("Failed to deliver second STARTUP IPI to APIC %d", | panic("Failed to deliver second STARTUP IPI to APIC %d", | ||||
apic_id); | apic_id); | ||||
DELAY(200); /* wait ~200uS */ | DELAY(200); /* wait ~200uS */ | ||||
} | } | ||||
/* | static bool | ||||
* Send an IPI to specified CPU handling the bitmap logic. | ipi_bitmap_set(int cpu, u_int ipi) | ||||
*/ | |||||
void | |||||
ipi_send_cpu(int cpu, u_int ipi) | |||||
{ | { | ||||
u_int bitmap, old, new; | u_int bitmap, old, new; | ||||
u_int *cpu_bitmap; | u_int *cpu_bitmap; | ||||
KASSERT((u_int)cpu < MAXCPU && cpu_apic_ids[cpu] != -1, | |||||
("IPI to non-existent CPU %d", cpu)); | |||||
if (IPI_IS_BITMAPED(ipi)) { | |||||
bitmap = 1 << ipi; | bitmap = 1 << ipi; | ||||
ipi = IPI_BITMAP_VECTOR; | |||||
cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap; | cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap; | ||||
old = *cpu_bitmap; | old = *cpu_bitmap; | ||||
for (;;) { | for (;;) { | ||||
if ((old & bitmap) == bitmap) | if ((old & bitmap) != 0) | ||||
break; | break; | ||||
new = old | bitmap; | new = old | bitmap; | ||||
if (atomic_fcmpset_int(cpu_bitmap, &old, new)) | if (atomic_fcmpset_int(cpu_bitmap, &old, new)) | ||||
break; | break; | ||||
} | } | ||||
if (old) | return (old != 0); | ||||
} | |||||
/* | |||||
* Send an IPI to specified CPU handling the bitmap logic. | |||||
*/ | |||||
static void | |||||
ipi_send_cpu(int cpu, u_int ipi) | |||||
{ | |||||
KASSERT((u_int)cpu < MAXCPU && cpu_apic_ids[cpu] != -1, | |||||
("IPI to non-existent CPU %d", cpu)); | |||||
if (IPI_IS_BITMAPED(ipi)) { | |||||
if (ipi_bitmap_set(cpu, ipi)) | |||||
return; | return; | ||||
ipi = IPI_BITMAP_VECTOR; | |||||
} | } | ||||
lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); | lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); | ||||
} | } | ||||
void | void | ||||
ipi_bitmap_handler(struct trapframe frame) | ipi_bitmap_handler(struct trapframe frame) | ||||
{ | { | ||||
struct trapframe *oldframe; | struct trapframe *oldframe; | ||||
Show All 39 Lines | #ifdef COUNT_IPIS | ||||
(*ipi_hardclock_counts[cpu])++; | (*ipi_hardclock_counts[cpu])++; | ||||
#endif | #endif | ||||
hardclockintr(); | hardclockintr(); | ||||
} | } | ||||
td->td_intr_frame = oldframe; | td->td_intr_frame = oldframe; | ||||
td->td_intr_nesting_level--; | td->td_intr_nesting_level--; | ||||
if (ipi_bitmap & (1 << IPI_HARDCLOCK)) | if (ipi_bitmap & (1 << IPI_HARDCLOCK)) | ||||
critical_exit(); | critical_exit(); | ||||
if (ipi_bitmap & (1 << IPI_SWI)) { | |||||
#ifdef COUNT_IPIS | |||||
(*ipi_swi_counts[cpu])++; | |||||
#endif | |||||
intr_event_handle(delay_intr_event, &frame); | |||||
} | } | ||||
} | |||||
/* | /* | ||||
* send an IPI to a set of cpus. | * send an IPI to a set of cpus. | ||||
*/ | */ | ||||
void | void | ||||
ipi_selected(cpuset_t cpus, u_int ipi) | ipi_selected(cpuset_t cpus, u_int ipi) | ||||
{ | { | ||||
int cpu; | int cpu; | ||||
Show All 35 Lines | |||||
/* | /* | ||||
* send an IPI to all CPUs EXCEPT myself | * send an IPI to all CPUs EXCEPT myself | ||||
*/ | */ | ||||
void | void | ||||
ipi_all_but_self(u_int ipi) | ipi_all_but_self(u_int ipi) | ||||
{ | { | ||||
cpuset_t other_cpus; | cpuset_t other_cpus; | ||||
int cpu, c; | |||||
/* | |||||
* IPI_STOP_HARD maps to a NMI and the trap handler needs a bit | |||||
* of help in order to understand what is the source. | |||||
* Set the mask of receiving CPUs for this purpose. | |||||
*/ | |||||
if (ipi == IPI_STOP_HARD) { | |||||
other_cpus = all_cpus; | other_cpus = all_cpus; | ||||
CPU_CLR(PCPU_GET(cpuid), &other_cpus); | CPU_CLR(PCPU_GET(cpuid), &other_cpus); | ||||
CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &other_cpus); | |||||
} | |||||
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); | |||||
if (IPI_IS_BITMAPED(ipi)) { | if (IPI_IS_BITMAPED(ipi)) { | ||||
ipi_selected(other_cpus, ipi); | cpu = PCPU_GET(cpuid); | ||||
return; | CPU_FOREACH(c) { | ||||
if (c != cpu) | |||||
ipi_bitmap_set(c, ipi); | |||||
} | } | ||||
ipi = IPI_BITMAP_VECTOR; | |||||
} | |||||
lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); | |||||
} | |||||
/* | /* | ||||
* send an IPI to myself | |||||
*/ | |||||
void | |||||
ipi_self(u_int ipi) | |||||
{ | |||||
/* | |||||
* IPI_STOP_HARD maps to a NMI and the trap handler needs a bit | * IPI_STOP_HARD maps to a NMI and the trap handler needs a bit | ||||
* of help in order to understand what is the source. | * of help in order to understand what is the source. | ||||
* Set the mask of receiving CPUs for this purpose. | * Set the mask of receiving CPUs for this purpose. | ||||
*/ | */ | ||||
if (ipi == IPI_STOP_HARD) | if (ipi == IPI_STOP_HARD) | ||||
CPU_OR_ATOMIC(&ipi_stop_nmi_pending, &other_cpus); | CPU_SET_ATOMIC(PCPU_GET(cpuid), &ipi_stop_nmi_pending); | ||||
CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); | CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); | ||||
lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); | if (IPI_IS_BITMAPED(ipi)) { | ||||
if (ipi_bitmap_set(PCPU_GET(cpuid), ipi)) | |||||
return; | |||||
ipi = IPI_BITMAP_VECTOR; | |||||
} | } | ||||
lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); | |||||
} | |||||
void | |||||
ipi_self_from_nmi(u_int ipi) | |||||
{ | |||||
ipi_self(ipi); | |||||
/* Wait for IPI to finish. */ | |||||
if (!lapic_ipi_wait(50000)) { | |||||
kib: This is not needed in x2APIC mode. There is no pending state. | |||||
Done Inline Actionslapic_ipi_wait() does nothing on x2APIC. Would you prefer to see explicit check here too? mav: lapic_ipi_wait() does nothing on x2APIC. Would you prefer to see explicit check here too? | |||||
Not Done Inline ActionsIn fact, I am not sure that we want ipi_self() to take 'ipi' as a parameter, instead of a vector. Look at what IPIs that are not vector provide: HARDCLOCK, HARD_STOP etc. It does not make sense to send any of that to self. I thought for a second that IPI_AST does make sense, but then realized that it requires setting TDF_ASTPENDING to have any effect, which means that it cannot be done from nmi context. Then ipi_self_from_nmi() (do we need ipi_self() at all ?) would start as if (x2apic_mode) { lapic_write_self_ipi(vector); return; } kib: In fact, I am not sure that we want ipi_self() to take 'ipi' as a parameter, instead of a… | |||||
Not Done Inline ActionsI was just trying to extend existing KPI without custom limitations. You may be right that it is overkill, but I don't care much about performance of something happening once in a while. mav: I was just trying to extend existing KPI without custom limitations. You may be right that it… | |||||
Not Done Inline ActionsI mean ipi_self() cannot be correctly used with IPI parameter. It can be only utilized when you pass it non-vectored non-special vectors. Then it makes sense to not copy existing interface to not make wrong assumptions in reader' head. kib: I mean ipi_self() cannot be correctly used with IPI parameter. It can be only utilized when… | |||||
Not Done Inline ActionsI am not getting why ipi_self() cannot be correctly used with IPI parameter. All the code setting bitmaps uses atomics and should work from any context. If you are worrying about thread being migrated, then we already have ipi_all_but_self(), that seems already quietly assumes the same. mav: I am not getting why ipi_self() cannot be correctly used with IPI parameter. All the code… | |||||
Not Done Inline ActionsIt is not ipi_self() as such, but the use of any of the IPI (vs. vectored interrupts). You cannot HARD_STOP itself, you cannot schedule AST to yourself from NMI context because it requires td_lock, and so on. kib: It is not ipi_self() as such, but the use of any of the IPI (vs. vectored interrupts). You… | |||||
Not Done Inline ActionsI've added my IPI_SWI as bitmapped to not waste a vector. So it does require ipi there. Do you think it makes sense to waste a vector for such a rare thing? I still think unification is good. If you don't like bitmapped IPIs, then may be we should remove them all together turning into vectors? It would simplify the code much more than making this case different. mav: I've added my IPI_SWI as bitmapped to not waste a vector. So it does require ipi there. Do… | |||||
Not Done Inline ActionsI think it is much easier to make it operate correctly with the distinguished vector. I would not use the term 'waste' for it. That said, I still want the patch split into (at least) two parts, and lets finish the APIC bit first. kib: I think it is much easier to make it operate correctly with the distinguished vector. I would… | |||||
Not Done Inline ActionsCalling lapic_write_self_ipi() directly from ipi_self_from_nmi() would be a layering violation for apic_ops KPI used by Xen. I haven't looked what Xen can actually do from NMI, but I bet this code would be completely wrong there. So if we are any way going through lapic_ipi_vectored(), then my code is just right, and if you want me to remove the bitmap IPIs handling from there just for sake of removal, then I've already made a version using separate vector for IPI_SELF. It touches much more files to manage the vector, but if you wish so -- I am fine with it. mav: Calling lapic_write_self_ipi() directly from ipi_self_from_nmi() would be a layering violation… | |||||
Not Done Inline ActionsI do not believe that RAS features are relevant to the Xen guest environment, even for dom0. I would the dedicated vector variant. But still, please split, kib: I do not believe that RAS features are relevant to the Xen guest environment, even for dom0.
I… | |||||
if (KERNEL_PANICKED()) | |||||
return; | |||||
else | |||||
panic("APIC: IPI is stuck"); | |||||
} | |||||
} | |||||
int | int | ||||
ipi_nmi_handler(void) | ipi_nmi_handler(void) | ||||
{ | { | ||||
u_int cpuid; | u_int cpuid; | ||||
/* | /* | ||||
* As long as there is not a simple way to know about a NMI's | * As long as there is not a simple way to know about a NMI's | ||||
* source, if the bitmask for the current CPU is present in | * source, if the bitmask for the current CPU is present in | ||||
▲ Show 20 Lines • Show All 234 Lines • ▼ Show 20 Lines | CPU_FOREACH(i) { | ||||
snprintf(buf, sizeof(buf), "cpu%d:preempt", i); | snprintf(buf, sizeof(buf), "cpu%d:preempt", i); | ||||
intrcnt_add(buf, &ipi_preempt_counts[i]); | intrcnt_add(buf, &ipi_preempt_counts[i]); | ||||
snprintf(buf, sizeof(buf), "cpu%d:ast", i); | snprintf(buf, sizeof(buf), "cpu%d:ast", i); | ||||
intrcnt_add(buf, &ipi_ast_counts[i]); | intrcnt_add(buf, &ipi_ast_counts[i]); | ||||
snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i); | snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i); | ||||
intrcnt_add(buf, &ipi_rendezvous_counts[i]); | intrcnt_add(buf, &ipi_rendezvous_counts[i]); | ||||
snprintf(buf, sizeof(buf), "cpu%d:hardclock", i); | snprintf(buf, sizeof(buf), "cpu%d:hardclock", i); | ||||
intrcnt_add(buf, &ipi_hardclock_counts[i]); | intrcnt_add(buf, &ipi_hardclock_counts[i]); | ||||
snprintf(buf, sizeof(buf), "cpu%d:swi", i); | |||||
intrcnt_add(buf, &ipi_swi_counts[i]); | |||||
} | } | ||||
} | } | ||||
SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL); | SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL); | ||||
#endif | #endif |
This is not needed in x2APIC mode. There is no pending state.