Page MenuHomeFreeBSD
Differential D25754 Diff 74810 sys/x86/x86/local_apic.c

Changeset View

Standalone View

View Options

sys/x86/x86/local_apic.c

Show First 20 Lines • Show All 248 Lines • ▼ Show 20 Lines if (x2apic_mode) {
wrmsr(MSR_APIC_000 + reg, val); wrmsr(MSR_APIC_000 + reg, val);
} else { } else {
*(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val; *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val;
} }
} }
#ifdef SMP #ifdef SMP
static uint64_t static uint64_t
lapic_read_icr(void)
{
uint64_t v;
uint32_t vhi, vlo;
if (x2apic_mode) {
v = rdmsr(MSR_APIC_000 + LAPIC_ICR_LO);
} else {
vhi = lapic_read32(LAPIC_ICR_HI);
vlo = lapic_read32(LAPIC_ICR_LO);
v = ((uint64_t)vhi << 32) | vlo;
}
return (v);
}
static uint64_t
lapic_read_icr_lo(void) lapic_read_icr_lo(void)
{ {
return (lapic_read32(LAPIC_ICR_LO)); return (lapic_read32(LAPIC_ICR_LO));
} }
static void static void
lapic_write_icr(uint32_t vhi, uint32_t vlo) lapic_write_icr(uint32_t vhi, uint32_t vlo)
{ {
register_t saveintr;
uint64_t v; uint64_t v;
if (x2apic_mode) { if (x2apic_mode) {
v = ((uint64_t)vhi << 32) | vlo; v = ((uint64_t)vhi << 32) | vlo;
mfence(); mfence();
wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v); wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v);
} else { } else {
saveintr = intr_disable();
lapic_write32(LAPIC_ICR_HI, vhi); lapic_write32(LAPIC_ICR_HI, vhi);
lapic_write32(LAPIC_ICR_LO, vlo); lapic_write32(LAPIC_ICR_LO, vlo);
intr_restore(saveintr);
} }
} }
static void
lapic_write_icr_lo(uint32_t vlo)
{
if (x2apic_mode) {
mfence();
wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, vlo);
} else {
lapic_write32(LAPIC_ICR_LO, vlo);
}
}
static void
lapic_write_self_ipi(uint32_t vector)
{
KASSERT(x2apic_mode, ("SELF IPI write in xAPIC mode"));
wrmsr(MSR_APIC_000 + LAPIC_SELF_IPI, vector);
kibUnsubmitted
Done
Inline Actions

You do not need this mfence when interrupting yourself. It will not make anything ordered that was not ordered without it, on the same core.

kib: You do not need this mfence when interrupting yourself. It will not make anything ordered that…
mavAuthorUnsubmitted
Done
Inline Actions

OK. I haven't thought it is "self" when copy-pasted. ;)

mav: OK. I haven't thought it is "self" when copy-pasted. ;)
}
#endif /* SMP */ #endif /* SMP */
static void static void
native_lapic_enable_x2apic(void) native_lapic_enable_x2apic(void)
{ {
uint64_t apic_base; uint64_t apic_base;
apic_base = rdmsr(MSR_APICBASE); apic_base = rdmsr(MSR_APICBASE);
▲ Show 20 Lines • Show All 1,685 Lines • ▼ Show 20 Lines for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) {
ia32_pause(); ia32_pause();
} }
return (0); return (0);
} }
static void static void
native_lapic_ipi_raw(register_t icrlo, u_int dest) native_lapic_ipi_raw(register_t icrlo, u_int dest)
{ {
uint64_t icr; uint32_t icrhi;
uint32_t vhi, vlo;
register_t saveintr;
/* XXX: Need more sanity checking of icrlo? */ /* XXX: Need more sanity checking of icrlo? */
KASSERT(x2apic_mode || lapic_map != NULL, KASSERT(x2apic_mode || lapic_map != NULL,
("%s called too early", __func__)); ("%s called too early", __func__));
KASSERT(x2apic_mode || KASSERT(x2apic_mode ||
(dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
("%s: invalid dest field", __func__)); ("%s: invalid dest field", __func__));
KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0, KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
("%s: reserved bits set in ICR LO register", __func__)); ("%s: reserved bits set in ICR LO register", __func__));
/* Set destination in ICR HI register if it is being used. */
if (!x2apic_mode) {
saveintr = intr_disable();
icr = lapic_read_icr();
}
if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) { if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
if (x2apic_mode) { if (x2apic_mode)
vhi = dest; icrhi = dest;
else
icrhi = dest << APIC_ID_SHIFT;
lapic_write_icr(icrhi, icrlo);
} else { } else {
vhi = icr >> 32; lapic_write_icr_lo(icrlo);
vhi &= ~APIC_ID_MASK;
vhi |= dest << APIC_ID_SHIFT;
} }
} else {
vhi = 0;
} }
/* Program the contents of the IPI and dispatch it. */
if (x2apic_mode) {
vlo = icrlo;
} else {
vlo = icr;
vlo &= APIC_ICRLO_RESV_MASK;
vlo |= icrlo;
}
lapic_write_icr(vhi, vlo);
if (!x2apic_mode)
intr_restore(saveintr);
}
#define BEFORE_SPIN 50000 #define BEFORE_SPIN 50000
#ifdef DETECT_DEADLOCK #ifdef DETECT_DEADLOCK
#define AFTER_SPIN 50 #define AFTER_SPIN 50
#endif #endif
static void static void
native_lapic_ipi_vectored(u_int vector, int dest) native_lapic_ipi_vectored(u_int vector, int dest)
{ {
register_t icrlo, destfield; register_t icrlo, destfield;
KASSERT((vector & ~APIC_VECTOR_MASK) == 0, KASSERT((vector & ~APIC_VECTOR_MASK) == 0,
("%s: invalid vector %d", __func__, vector)); ("%s: invalid vector %d", __func__, vector));
icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
/*
* NMI IPIs are just fake vectors used to send a NMI. Use special rules
* regarding NMIs if passed, otherwise specify the vector.
*/
if (vector >= IPI_NMI_FIRST)
icrlo |= APIC_DELMODE_NMI;
else
icrlo |= vector | APIC_DELMODE_FIXED;
destfield = 0; destfield = 0;
switch (dest) { switch (dest) {
case APIC_IPI_DEST_SELF: case APIC_IPI_DEST_SELF:
icrlo |= APIC_DEST_SELF; if (x2apic_mode && vector < IPI_NMI_FIRST) {
lapic_write_self_ipi(vector);
return;
}
icrlo = APIC_DEST_SELF;
break; break;
case APIC_IPI_DEST_ALL: case APIC_IPI_DEST_ALL:
icrlo |= APIC_DEST_ALLISELF; icrlo = APIC_DEST_ALLISELF;
break; break;
case APIC_IPI_DEST_OTHERS: case APIC_IPI_DEST_OTHERS:
icrlo |= APIC_DEST_ALLESELF; icrlo = APIC_DEST_ALLESELF;
break; break;
default: default:
icrlo = 0;
KASSERT(x2apic_mode || KASSERT(x2apic_mode ||
(dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
("%s: invalid destination 0x%x", __func__, dest)); ("%s: invalid destination 0x%x", __func__, dest));
destfield = dest; destfield = dest;
} }
/*
* NMI IPIs are just fake vectors used to send a NMI. Use special rules
* regarding NMIs if passed, otherwise specify the vector.
*/
if (vector >= IPI_NMI_FIRST)
icrlo |= APIC_DELMODE_NMI;
else
icrlo |= vector | APIC_DELMODE_FIXED;
icrlo |= APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT;
/* Wait for an earlier IPI to finish. */ /* Wait for an earlier IPI to finish. */
if (!lapic_ipi_wait(BEFORE_SPIN)) { if (!lapic_ipi_wait(BEFORE_SPIN)) {
if (KERNEL_PANICKED()) if (KERNEL_PANICKED())
return; return;
else else
panic("APIC: Previous IPI is stuck"); panic("APIC: Previous IPI is stuck");
} }
▲ Show 20 Lines • Show All 92 Lines • Show Last 20 Lines