Changeset View
Changeset View
Standalone View
Standalone View
sys/x86/x86/msi.c
Show All 31 Lines | |||||
* Support for PCI Message Signalled Interrupts (MSI). MSI interrupts on | * Support for PCI Message Signalled Interrupts (MSI). MSI interrupts on | ||||
* x86 are basically APIC messages that the northbridge delivers directly | * x86 are basically APIC messages that the northbridge delivers directly | ||||
* to the local APICs as if they had come from an I/O APIC. | * to the local APICs as if they had come from an I/O APIC. | ||||
*/ | */ | ||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include "opt_acpi.h" | |||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/bus.h> | #include <sys/bus.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/sx.h> | #include <sys/sx.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <x86/apicreg.h> | #include <x86/apicreg.h> | ||||
#include <machine/cputypes.h> | #include <machine/cputypes.h> | ||||
#include <machine/md_var.h> | #include <machine/md_var.h> | ||||
#include <machine/frame.h> | #include <machine/frame.h> | ||||
#include <machine/intr_machdep.h> | #include <machine/intr_machdep.h> | ||||
#include <x86/apicvar.h> | #include <x86/apicvar.h> | ||||
#include <x86/iommu/iommu_intrmap.h> | |||||
#include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||
#include <dev/pci/pcivar.h> | #include <dev/pci/pcivar.h> | ||||
/* Fields in address for Intel MSI messages. */ | /* Fields in address for Intel MSI messages. */ | ||||
#define MSI_INTEL_ADDR_DEST 0x000ff000 | #define MSI_INTEL_ADDR_DEST 0x000ff000 | ||||
#define MSI_INTEL_ADDR_RH 0x00000008 | #define MSI_INTEL_ADDR_RH 0x00000008 | ||||
# define MSI_INTEL_ADDR_RH_ON 0x00000008 | # define MSI_INTEL_ADDR_RH_ON 0x00000008 | ||||
# define MSI_INTEL_ADDR_RH_OFF 0x00000000 | # define MSI_INTEL_ADDR_RH_OFF 0x00000000 | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
struct msi_intsrc { | struct msi_intsrc { | ||||
struct intsrc msi_intsrc; | struct intsrc msi_intsrc; | ||||
device_t msi_dev; /* Owning device. (g) */ | device_t msi_dev; /* Owning device. (g) */ | ||||
struct msi_intsrc *msi_first; /* First source in group. */ | struct msi_intsrc *msi_first; /* First source in group. */ | ||||
u_int msi_irq; /* IRQ cookie. */ | u_int msi_irq; /* IRQ cookie. */ | ||||
u_int msi_msix; /* MSI-X message. */ | u_int msi_msix; /* MSI-X message. */ | ||||
u_int msi_vector:8; /* IDT vector. */ | u_int msi_vector:8; /* IDT vector. */ | ||||
u_int msi_cpu:8; /* Local APIC ID. (g) */ | u_int msi_cpu; /* Local APIC ID. (g) */ | ||||
u_int msi_count:8; /* Messages in this group. (g) */ | u_int msi_count:8; /* Messages in this group. (g) */ | ||||
u_int msi_maxcount:8; /* Alignment for this group. (g) */ | u_int msi_maxcount:8; /* Alignment for this group. (g) */ | ||||
int *msi_irqs; /* Group's IRQ list. (g) */ | int *msi_irqs; /* Group's IRQ list. (g) */ | ||||
u_int msi_remap_cookie; | |||||
}; | }; | ||||
static void msi_create_source(void); | static void msi_create_source(void); | ||||
static void msi_enable_source(struct intsrc *isrc); | static void msi_enable_source(struct intsrc *isrc); | ||||
static void msi_disable_source(struct intsrc *isrc, int eoi); | static void msi_disable_source(struct intsrc *isrc, int eoi); | ||||
static void msi_eoi_source(struct intsrc *isrc); | static void msi_eoi_source(struct intsrc *isrc); | ||||
static void msi_enable_intr(struct intsrc *isrc); | static void msi_enable_intr(struct intsrc *isrc); | ||||
static void msi_disable_intr(struct intsrc *isrc); | static void msi_disable_intr(struct intsrc *isrc); | ||||
static int msi_vector(struct intsrc *isrc); | static int msi_vector(struct intsrc *isrc); | ||||
static int msi_source_pending(struct intsrc *isrc); | static int msi_source_pending(struct intsrc *isrc); | ||||
static int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig, | static int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig, | ||||
enum intr_polarity pol); | enum intr_polarity pol); | ||||
static int msi_assign_cpu(struct intsrc *isrc, u_int apic_id); | static int msi_assign_cpu(struct intsrc *isrc, u_int apic_id); | ||||
struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source, | struct pic msi_pic = { | ||||
msi_enable_intr, msi_disable_intr, msi_vector, | .pic_enable_source = msi_enable_source, | ||||
msi_source_pending, NULL, NULL, msi_config_intr, | .pic_disable_source = msi_disable_source, | ||||
msi_assign_cpu }; | .pic_eoi_source = msi_eoi_source, | ||||
.pic_enable_intr = msi_enable_intr, | |||||
.pic_disable_intr = msi_disable_intr, | |||||
.pic_vector = msi_vector, | |||||
.pic_source_pending = msi_source_pending, | |||||
.pic_suspend = NULL, | |||||
.pic_resume = NULL, | |||||
.pic_config_intr = msi_config_intr, | |||||
.pic_assign_cpu = msi_assign_cpu, | |||||
.pic_reprogram_pin = NULL, | |||||
}; | |||||
static int msi_enabled; | static int msi_enabled; | ||||
static int msi_last_irq; | static int msi_last_irq; | ||||
static struct mtx msi_lock; | static struct mtx msi_lock; | ||||
static void | static void | ||||
msi_enable_source(struct intsrc *isrc) | msi_enable_source(struct intsrc *isrc) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 171 Lines • ▼ Show 20 Lines | |||||
* Try to allocate 'count' interrupt sources with contiguous IDT values. | * Try to allocate 'count' interrupt sources with contiguous IDT values. | ||||
*/ | */ | ||||
int | int | ||||
msi_alloc(device_t dev, int count, int maxcount, int *irqs) | msi_alloc(device_t dev, int count, int maxcount, int *irqs) | ||||
{ | { | ||||
struct msi_intsrc *msi, *fsrc; | struct msi_intsrc *msi, *fsrc; | ||||
u_int cpu; | u_int cpu; | ||||
int cnt, i, *mirqs, vector; | int cnt, i, *mirqs, vector; | ||||
#ifdef ACPI_DMAR | |||||
u_int cookies[count]; | |||||
int error; | |||||
#endif | |||||
if (!msi_enabled) | if (!msi_enabled) | ||||
return (ENXIO); | return (ENXIO); | ||||
if (count > 1) | if (count > 1) | ||||
mirqs = malloc(count * sizeof(*mirqs), M_MSI, M_WAITOK); | mirqs = malloc(count * sizeof(*mirqs), M_MSI, M_WAITOK); | ||||
else | else | ||||
mirqs = NULL; | mirqs = NULL; | ||||
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | again: | ||||
cpu = intr_next_cpu(); | cpu = intr_next_cpu(); | ||||
vector = apic_alloc_vectors(cpu, irqs, count, maxcount); | vector = apic_alloc_vectors(cpu, irqs, count, maxcount); | ||||
if (vector == 0) { | if (vector == 0) { | ||||
mtx_unlock(&msi_lock); | mtx_unlock(&msi_lock); | ||||
free(mirqs, M_MSI); | free(mirqs, M_MSI); | ||||
return (ENOSPC); | return (ENOSPC); | ||||
} | } | ||||
#ifdef ACPI_DMAR | |||||
mtx_unlock(&msi_lock); | |||||
error = iommu_alloc_msi_intr(dev, cookies, count); | |||||
mtx_lock(&msi_lock); | |||||
if (error == EOPNOTSUPP) | |||||
error = 0; | |||||
if (error != 0) { | |||||
for (i = 0; i < count; i++) | |||||
apic_free_vector(cpu, vector + i, irqs[i]); | |||||
free(mirqs, M_MSI); | |||||
return (error); | |||||
} | |||||
for (i = 0; i < count; i++) { | |||||
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); | |||||
msi->msi_remap_cookie = cookies[i]; | |||||
} | |||||
#endif | |||||
/* Assign IDT vectors and make these messages owned by 'dev'. */ | /* Assign IDT vectors and make these messages owned by 'dev'. */ | ||||
fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]); | fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]); | ||||
for (i = 0; i < count; i++) { | for (i = 0; i < count; i++) { | ||||
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); | msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); | ||||
msi->msi_cpu = cpu; | msi->msi_cpu = cpu; | ||||
msi->msi_dev = dev; | msi->msi_dev = dev; | ||||
msi->msi_vector = vector + i; | msi->msi_vector = vector + i; | ||||
if (bootverbose) | if (bootverbose) | ||||
printf( | printf( | ||||
"msi: routing MSI IRQ %d to local APIC %u vector %u\n", | "msi: routing MSI IRQ %d to local APIC %u vector %u\n", | ||||
msi->msi_irq, msi->msi_cpu, msi->msi_vector); | msi->msi_irq, msi->msi_cpu, msi->msi_vector); | ||||
msi->msi_first = fsrc; | msi->msi_first = fsrc; | ||||
KASSERT(msi->msi_intsrc.is_handlers == 0, | KASSERT(msi->msi_intsrc.is_handlers == 0, | ||||
("dead MSI has handlers")); | ("dead MSI has handlers")); | ||||
} | } | ||||
fsrc->msi_count = count; | fsrc->msi_count = count; | ||||
fsrc->msi_maxcount = maxcount; | fsrc->msi_maxcount = maxcount; | ||||
if (count > 1) | if (count > 1) | ||||
bcopy(irqs, mirqs, count * sizeof(*mirqs)); | bcopy(irqs, mirqs, count * sizeof(*mirqs)); | ||||
fsrc->msi_irqs = mirqs; | fsrc->msi_irqs = mirqs; | ||||
mtx_unlock(&msi_lock); | mtx_unlock(&msi_lock); | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
msi_release(int *irqs, int count) | msi_release(int *irqs, int count) | ||||
{ | { | ||||
struct msi_intsrc *msi, *first; | struct msi_intsrc *msi, *first; | ||||
int i; | int i; | ||||
Show All 27 Lines | msi_release(int *irqs, int count) | ||||
} | } | ||||
KASSERT(first->msi_dev != NULL, ("unowned group")); | KASSERT(first->msi_dev != NULL, ("unowned group")); | ||||
/* Clear all the extra messages in the group. */ | /* Clear all the extra messages in the group. */ | ||||
for (i = 1; i < count; i++) { | for (i = 1; i < count; i++) { | ||||
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); | msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]); | ||||
KASSERT(msi->msi_first == first, ("message not in group")); | KASSERT(msi->msi_first == first, ("message not in group")); | ||||
KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch")); | KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch")); | ||||
#ifdef ACPI_DMAR | |||||
iommu_unmap_msi_intr(first->msi_dev, msi->msi_remap_cookie); | |||||
#endif | |||||
msi->msi_first = NULL; | msi->msi_first = NULL; | ||||
msi->msi_dev = NULL; | msi->msi_dev = NULL; | ||||
apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq); | apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq); | ||||
msi->msi_vector = 0; | msi->msi_vector = 0; | ||||
} | } | ||||
/* Clear out the first message. */ | /* Clear out the first message. */ | ||||
#ifdef ACPI_DMAR | |||||
mtx_unlock(&msi_lock); | |||||
iommu_unmap_msi_intr(first->msi_dev, first->msi_remap_cookie); | |||||
mtx_lock(&msi_lock); | |||||
#endif | |||||
first->msi_first = NULL; | first->msi_first = NULL; | ||||
first->msi_dev = NULL; | first->msi_dev = NULL; | ||||
apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq); | apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq); | ||||
first->msi_vector = 0; | first->msi_vector = 0; | ||||
first->msi_count = 0; | first->msi_count = 0; | ||||
first->msi_maxcount = 0; | first->msi_maxcount = 0; | ||||
free(first->msi_irqs, M_MSI); | free(first->msi_irqs, M_MSI); | ||||
first->msi_irqs = NULL; | first->msi_irqs = NULL; | ||||
mtx_unlock(&msi_lock); | mtx_unlock(&msi_lock); | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
msi_map(int irq, uint64_t *addr, uint32_t *data) | msi_map(int irq, uint64_t *addr, uint32_t *data) | ||||
{ | { | ||||
struct msi_intsrc *msi; | struct msi_intsrc *msi; | ||||
int error; | |||||
#ifdef ACPI_DMAR | |||||
struct msi_intsrc *msi1; | |||||
int i, k; | |||||
#endif | |||||
mtx_lock(&msi_lock); | mtx_lock(&msi_lock); | ||||
msi = (struct msi_intsrc *)intr_lookup_source(irq); | msi = (struct msi_intsrc *)intr_lookup_source(irq); | ||||
if (msi == NULL) { | if (msi == NULL) { | ||||
mtx_unlock(&msi_lock); | mtx_unlock(&msi_lock); | ||||
return (ENOENT); | return (ENOENT); | ||||
} | } | ||||
Show All 11 Lines | #endif | ||||
if (!msi->msi_msix) { | if (!msi->msi_msix) { | ||||
if (msi->msi_first == NULL) { | if (msi->msi_first == NULL) { | ||||
mtx_unlock(&msi_lock); | mtx_unlock(&msi_lock); | ||||
return (ENXIO); | return (ENXIO); | ||||
} | } | ||||
msi = msi->msi_first; | msi = msi->msi_first; | ||||
} | } | ||||
#ifdef ACPI_DMAR | |||||
if (!msi->msi_msix) { | |||||
for (k = msi->msi_count - 1, i = FIRST_MSI_INT; k > 0 && | |||||
jhb: Would it help if we kept a linked list of an MSI "group"? The "first" MSI would have a… | |||||
kibAuthorUnsubmitted Not Done Inline ActionsSure, this would eliminate the loop over the whole set of msi interrupt sources. On the other hand, this would trade CPU time for memory. The interrup setup is performed once usually, while +8 bytes per msi_intsrc are consumed for the whole duration of the device life. This is small amount, but is it worth that ? kib: Sure, this would eliminate the loop over the whole set of msi interrupt sources.
On the other… | |||||
i < FIRST_MSI_INT + NUM_MSI_INTS; i++) { | |||||
if (i == msi->msi_irq) | |||||
continue; | |||||
msi1 = (struct msi_intsrc *)intr_lookup_source(i); | |||||
if (!msi1->msi_msix && msi1->msi_first == msi) { | |||||
mtx_unlock(&msi_lock); | |||||
iommu_map_msi_intr(msi1->msi_dev, | |||||
msi1->msi_cpu, msi1->msi_vector, | |||||
msi1->msi_remap_cookie, NULL, NULL); | |||||
k--; | |||||
mtx_lock(&msi_lock); | |||||
} | |||||
} | |||||
} | |||||
mtx_unlock(&msi_lock); | |||||
error = iommu_map_msi_intr(msi->msi_dev, msi->msi_cpu, | |||||
msi->msi_vector, msi->msi_remap_cookie, addr, data); | |||||
#else | |||||
mtx_unlock(&msi_lock); | |||||
error = EOPNOTSUPP; | |||||
#endif | |||||
if (error == EOPNOTSUPP) { | |||||
*addr = INTEL_ADDR(msi); | *addr = INTEL_ADDR(msi); | ||||
*data = INTEL_DATA(msi); | *data = INTEL_DATA(msi); | ||||
mtx_unlock(&msi_lock); | error = 0; | ||||
return (0); | |||||
} | } | ||||
return (error); | |||||
} | |||||
int | int | ||||
msix_alloc(device_t dev, int *irq) | msix_alloc(device_t dev, int *irq) | ||||
{ | { | ||||
struct msi_intsrc *msi; | struct msi_intsrc *msi; | ||||
u_int cpu; | u_int cpu; | ||||
int i, vector; | int i, vector; | ||||
#ifdef ACPI_DMAR | |||||
u_int cookie; | |||||
int error; | |||||
#endif | |||||
if (!msi_enabled) | if (!msi_enabled) | ||||
return (ENXIO); | return (ENXIO); | ||||
again: | again: | ||||
mtx_lock(&msi_lock); | mtx_lock(&msi_lock); | ||||
/* Find a free IRQ. */ | /* Find a free IRQ. */ | ||||
Show All 25 Lines | again: | ||||
/* Allocate an IDT vector. */ | /* Allocate an IDT vector. */ | ||||
cpu = intr_next_cpu(); | cpu = intr_next_cpu(); | ||||
vector = apic_alloc_vector(cpu, i); | vector = apic_alloc_vector(cpu, i); | ||||
if (vector == 0) { | if (vector == 0) { | ||||
mtx_unlock(&msi_lock); | mtx_unlock(&msi_lock); | ||||
return (ENOSPC); | return (ENOSPC); | ||||
} | } | ||||
msi->msi_dev = dev; | |||||
#ifdef ACPI_DMAR | |||||
mtx_unlock(&msi_lock); | |||||
error = iommu_alloc_msi_intr(dev, &cookie, 1); | |||||
mtx_lock(&msi_lock); | |||||
if (error == EOPNOTSUPP) | |||||
error = 0; | |||||
if (error != 0) { | |||||
msi->msi_dev = NULL; | |||||
apic_free_vector(cpu, vector, i); | |||||
return (error); | |||||
} | |||||
msi->msi_remap_cookie = cookie; | |||||
#endif | |||||
if (bootverbose) | if (bootverbose) | ||||
printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n", | printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n", | ||||
msi->msi_irq, cpu, vector); | msi->msi_irq, cpu, vector); | ||||
/* Setup source. */ | /* Setup source. */ | ||||
msi->msi_cpu = cpu; | msi->msi_cpu = cpu; | ||||
msi->msi_dev = dev; | |||||
msi->msi_first = msi; | msi->msi_first = msi; | ||||
msi->msi_vector = vector; | msi->msi_vector = vector; | ||||
msi->msi_msix = 1; | msi->msi_msix = 1; | ||||
msi->msi_count = 1; | msi->msi_count = 1; | ||||
msi->msi_maxcount = 1; | msi->msi_maxcount = 1; | ||||
msi->msi_irqs = NULL; | msi->msi_irqs = NULL; | ||||
KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers")); | KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers")); | ||||
Show All 19 Lines | msix_release(int irq) | ||||
if (!msi->msi_msix) { | if (!msi->msi_msix) { | ||||
mtx_unlock(&msi_lock); | mtx_unlock(&msi_lock); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
KASSERT(msi->msi_dev != NULL, ("unowned message")); | KASSERT(msi->msi_dev != NULL, ("unowned message")); | ||||
/* Clear out the message. */ | /* Clear out the message. */ | ||||
#ifdef ACPI_DMAR | |||||
mtx_unlock(&msi_lock); | |||||
iommu_unmap_msi_intr(msi->msi_dev, msi->msi_remap_cookie); | |||||
mtx_lock(&msi_lock); | |||||
#endif | |||||
msi->msi_first = NULL; | msi->msi_first = NULL; | ||||
msi->msi_dev = NULL; | msi->msi_dev = NULL; | ||||
apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq); | apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq); | ||||
msi->msi_vector = 0; | msi->msi_vector = 0; | ||||
msi->msi_msix = 0; | msi->msi_msix = 0; | ||||
msi->msi_count = 0; | msi->msi_count = 0; | ||||
msi->msi_maxcount = 0; | msi->msi_maxcount = 0; | ||||
mtx_unlock(&msi_lock); | mtx_unlock(&msi_lock); | ||||
return (0); | return (0); | ||||
} | } |
Would it help if we kept a linked list of an MSI "group"? The "first" MSI would have a SLIST_HEAD and the others would be linked in order perhaps? That would make this loop simpler and perhaps easier to read?