Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/subr_intr.c
Show First 20 Lines • Show All 95 Lines • ▼ Show 20 Lines | |||||
/* Interrupt controller definition. */ | /* Interrupt controller definition. */ | ||||
struct intr_pic { | struct intr_pic { | ||||
SLIST_ENTRY(intr_pic) pic_next; | SLIST_ENTRY(intr_pic) pic_next; | ||||
intptr_t pic_xref; /* hardware identification */ | intptr_t pic_xref; /* hardware identification */ | ||||
device_t pic_dev; | device_t pic_dev; | ||||
#define FLAG_PIC (1 << 0) | #define FLAG_PIC (1 << 0) | ||||
#define FLAG_MSI (1 << 1) | #define FLAG_MSI (1 << 1) | ||||
u_int pic_flags; | u_int pic_flags; | ||||
u_int pic_irq_base; | |||||
u_int pic_irq_next; | |||||
u_int pic_irq_count; | |||||
struct intr_irqsrc ** pic_sources; | |||||
}; | }; | ||||
static struct mtx pic_list_lock; | static struct mtx pic_list_lock; | ||||
static SLIST_HEAD(, intr_pic) pic_list; | static SLIST_HEAD(, intr_pic) pic_list; | ||||
static u_int pic_next_irq_base; | |||||
static struct intr_pic *pic_lookup(device_t dev, intptr_t xref); | static struct intr_pic *pic_lookup(device_t dev, intptr_t xref); | ||||
/* Interrupt source definition. */ | /* Interrupt source definition. */ | ||||
static struct mtx isrc_table_lock; | static struct mtx isrc_table_lock; | ||||
static struct intr_irqsrc *irq_sources[NIRQ]; | |||||
u_int irq_next_free; | |||||
/* | /* | ||||
* XXX - All stuff around struct intr_dev_data is considered as temporary | * XXX - All stuff around struct intr_dev_data is considered as temporary | ||||
* until better place for storing struct intr_map_data will be find. | * until better place for storing struct intr_map_data will be find. | ||||
* | * | ||||
* For now, there are two global interrupt numbers spaces: | * For now, there are two global interrupt numbers spaces: | ||||
* <0, NIRQ) ... interrupts without config data | * <0, NIRQ) ... interrupts without config data | ||||
* managed in irq_sources[] | * managed in irq_sources[] | ||||
Show All 12 Lines | struct intr_dev_data { | ||||
struct intr_map_data * idd_data; | struct intr_map_data * idd_data; | ||||
struct intr_irqsrc * idd_isrc; | struct intr_irqsrc * idd_isrc; | ||||
}; | }; | ||||
static struct intr_dev_data *intr_ddata_tab[2 * NIRQ]; | static struct intr_dev_data *intr_ddata_tab[2 * NIRQ]; | ||||
static u_int intr_ddata_first_unused; | static u_int intr_ddata_first_unused; | ||||
#define IRQ_DDATA_BASE 10000 | #define IRQ_DDATA_BASE 10000 | ||||
CTASSERT(IRQ_DDATA_BASE > nitems(irq_sources)); | |||||
#ifdef SMP | #ifdef SMP | ||||
static boolean_t irq_assign_cpu = FALSE; | static boolean_t irq_assign_cpu = FALSE; | ||||
#endif | #endif | ||||
/* | /* | ||||
* - 2 counters for each I/O interrupt. | * - 2 counters for each I/O interrupt. | ||||
* - MAXCPU counters for each IPI counters for SMP. | * - MAXCPU counters for each IPI counters for SMP. | ||||
▲ Show 20 Lines • Show All 198 Lines • ▼ Show 20 Lines | |||||
* There could be various strategies how to allocate free interrupt number | * There could be various strategies how to allocate free interrupt number | ||||
* (resource handle) for new interrupt source. | * (resource handle) for new interrupt source. | ||||
* | * | ||||
* 1. Handles are always allocated forward, so handles are not recycled | * 1. Handles are always allocated forward, so handles are not recycled | ||||
* immediately. However, if only one free handle left which is reused | * immediately. However, if only one free handle left which is reused | ||||
* constantly... | * constantly... | ||||
*/ | */ | ||||
static inline int | static inline int | ||||
isrc_alloc_irq(struct intr_irqsrc *isrc) | isrc_alloc_irq(struct intr_pic *pic, struct intr_irqsrc *isrc) | ||||
{ | { | ||||
u_int maxirqs, irq; | u_int maxirqs, irq; | ||||
mtx_assert(&isrc_table_lock, MA_OWNED); | mtx_assert(&isrc_table_lock, MA_OWNED); | ||||
maxirqs = nitems(irq_sources); | maxirqs = pic->pic_irq_base + pic->pic_irq_count; | ||||
if (irq_next_free >= maxirqs) | if (pic->pic_irq_next >= maxirqs) | ||||
return (ENOSPC); | return (ENOSPC); | ||||
for (irq = irq_next_free; irq < maxirqs; irq++) { | for (irq = pic->pic_irq_next; irq < maxirqs; irq++) { | ||||
if (irq_sources[irq] == NULL) | if (pic->pic_sources[irq] == NULL) | ||||
goto found; | goto found; | ||||
} | } | ||||
for (irq = 0; irq < irq_next_free; irq++) { | for (irq = pic->pic_irq_base; irq < pic->pic_irq_next; irq++) { | ||||
if (irq_sources[irq] == NULL) | if (pic->pic_sources[irq] == NULL) | ||||
goto found; | goto found; | ||||
} | } | ||||
irq_next_free = maxirqs; | pic->pic_irq_next = maxirqs; | ||||
return (ENOSPC); | return (ENOSPC); | ||||
found: | found: | ||||
isrc->isrc_irq = irq; | isrc->isrc_irq = irq; | ||||
irq_sources[irq] = isrc; | pic->pic_sources[irq - pic->pic_irq_base] = isrc; | ||||
irq_next_free = irq + 1; | pic->pic_irq_next = irq + 1; | ||||
if (irq_next_free >= maxirqs) | if (pic->pic_irq_next >= maxirqs) | ||||
irq_next_free = 0; | pic->pic_irq_next = pic->pic_irq_base; | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Free unique interrupt number (resource handle) from interrupt source. | * Free unique interrupt number (resource handle) from interrupt source. | ||||
*/ | */ | ||||
static inline int | static inline int | ||||
isrc_free_irq(struct intr_irqsrc *isrc) | isrc_free_irq(struct intr_pic *pic, struct intr_irqsrc *isrc) | ||||
{ | { | ||||
u_int irq; | |||||
mtx_assert(&isrc_table_lock, MA_OWNED); | mtx_assert(&isrc_table_lock, MA_OWNED); | ||||
if (isrc->isrc_irq >= nitems(irq_sources)) | irq = isrc->isrc_irq - pic->pic_irq_base; | ||||
if (irq > pic->pic_irq_count) | |||||
return (EINVAL); | return (EINVAL); | ||||
if (irq_sources[isrc->isrc_irq] != isrc) | if (pic->pic_sources[irq] != isrc) | ||||
return (EINVAL); | return (EINVAL); | ||||
irq_sources[isrc->isrc_irq] = NULL; | pic->pic_sources[irq] = NULL; | ||||
isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ | isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Lookup interrupt source by interrupt number (resource handle). | * Lookup interrupt source by interrupt number (resource handle). | ||||
*/ | */ | ||||
static inline struct intr_irqsrc * | static inline struct intr_irqsrc * | ||||
isrc_lookup(u_int irq) | isrc_lookup(u_int irq) | ||||
{ | { | ||||
struct intr_pic *pic; | |||||
if (irq < nitems(irq_sources)) | SLIST_FOREACH(pic, &pic_list, pic_next) { | ||||
return (irq_sources[irq]); | if (irq >= pic->pic_irq_base && irq < pic->pic_irq_count) | ||||
return (pic->pic_sources[irq - pic->pic_irq_base]); | |||||
} | |||||
return (NULL); | return (NULL); | ||||
} | } | ||||
/* | /* | ||||
* Initialize interrupt source and register it into global interrupt table. | * Initialize interrupt source and register it into global interrupt table. | ||||
*/ | */ | ||||
int | int | ||||
intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, | intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, | ||||
skra: `struct pic*` argument instead of `device_t`? | |||||
andrewAuthorUnsubmitted Not Done Inline ActionsCan I hold off changing this ABI for now & update D6304 on top of this? I have other needs for a driver to know its struct intr_pic *. andrew: Can I hold off changing this ABI for now & update D6304 on top of this? I have other needs for… | |||||
const char *fmt, ...) | const char *fmt, ...) | ||||
{ | { | ||||
struct intr_pic *pic; | |||||
int error; | int error; | ||||
va_list ap; | va_list ap; | ||||
pic = pic_lookup(isrc->isrc_dev, 0); | |||||
skraUnsubmitted Done Inline Actionsisrc->isrc_dev is initialized in this function on line 445. skra: isrc->isrc_dev is initialized in this function on line 445. | |||||
/* This can only happen if the pic disappears somehow */ | |||||
KASSERT(pic != NULL, ("%s: Unable to find isrc pic", __func__)); | |||||
bzero(isrc, sizeof(struct intr_irqsrc)); | bzero(isrc, sizeof(struct intr_irqsrc)); | ||||
isrc->isrc_dev = dev; | isrc->isrc_dev = dev; | ||||
skraUnsubmitted Not Done Inline Actionspic->pic_dev? skra: pic->pic_dev? | |||||
andrewAuthorUnsubmitted Not Done Inline ActionsShould be the same? andrew: Should be the same? | |||||
isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ | isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ | ||||
isrc->isrc_flags = flags; | isrc->isrc_flags = flags; | ||||
va_start(ap, fmt); | va_start(ap, fmt); | ||||
vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); | vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); | ||||
va_end(ap); | va_end(ap); | ||||
mtx_lock(&isrc_table_lock); | mtx_lock(&isrc_table_lock); | ||||
error = isrc_alloc_irq(isrc); | error = isrc_alloc_irq(pic, isrc); | ||||
if (error != 0) { | if (error != 0) { | ||||
mtx_unlock(&isrc_table_lock); | mtx_unlock(&isrc_table_lock); | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Setup interrupt counters, but not for IPI sources. Those are setup | * Setup interrupt counters, but not for IPI sources. Those are setup | ||||
* later and only for used ones (up to INTR_IPI_COUNT) to not exhaust | * later and only for used ones (up to INTR_IPI_COUNT) to not exhaust | ||||
* our counter pool. | * our counter pool. | ||||
*/ | */ | ||||
if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) | if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) | ||||
isrc_setup_counters(isrc); | isrc_setup_counters(isrc); | ||||
mtx_unlock(&isrc_table_lock); | mtx_unlock(&isrc_table_lock); | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Deregister interrupt source from global interrupt table. | * Deregister interrupt source from global interrupt table. | ||||
*/ | */ | ||||
int | int | ||||
intr_isrc_deregister(struct intr_irqsrc *isrc) | intr_isrc_deregister(struct intr_irqsrc *isrc) | ||||
skraUnsubmitted Not Done Inline Actionsadd struct pic* argument? skra: add struct pic* argument? | |||||
{ | { | ||||
struct intr_pic *pic; | |||||
int error; | int error; | ||||
pic = pic_lookup(isrc->isrc_dev, 0); | |||||
/* This can only happen if the pic disappears somehow */ | |||||
KASSERT(pic != NULL, ("%s: Unable to find isrc pic", __func__)); | |||||
mtx_lock(&isrc_table_lock); | mtx_lock(&isrc_table_lock); | ||||
if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) | if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) | ||||
isrc_release_counters(isrc); | isrc_release_counters(isrc); | ||||
error = isrc_free_irq(isrc); | error = isrc_free_irq(pic, isrc); | ||||
mtx_unlock(&isrc_table_lock); | mtx_unlock(&isrc_table_lock); | ||||
return (error); | return (error); | ||||
} | } | ||||
#ifdef SMP | #ifdef SMP | ||||
/* | /* | ||||
* A support function for a PIC to decide if provided ISRC should be inited | * A support function for a PIC to decide if provided ISRC should be inited | ||||
* on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of | * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of | ||||
▲ Show 20 Lines • Show All 430 Lines • ▼ Show 20 Lines | pic_destroy(device_t dev, intptr_t xref) | ||||
free(pic, M_INTRNG); | free(pic, M_INTRNG); | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* Register interrupt controller. | * Register interrupt controller. | ||||
*/ | */ | ||||
int | int | ||||
intr_pic_register(device_t dev, intptr_t xref) | intr_pic_register(device_t dev, intptr_t xref, u_int count) | ||||
{ | { | ||||
struct intr_pic *pic; | struct intr_pic *pic; | ||||
if (dev == NULL) | if (dev == NULL) | ||||
return (EINVAL); | return (EINVAL); | ||||
pic = pic_create(dev, xref); | pic = pic_create(dev, xref); | ||||
if (pic == NULL) | if (pic == NULL) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
pic->pic_sources = malloc(sizeof(*pic->pic_sources) * count, M_INTRNG, | |||||
M_WAITOK | M_ZERO); | |||||
pic->pic_irq_base = atomic_fetchadd_int(&pic_next_irq_base, count); | |||||
pic->pic_irq_next = pic->pic_irq_base; | |||||
pic->pic_irq_count = count; | |||||
KASSERT((pic->pic_irq_base + count) < IRQ_DDATA_BASE, | |||||
("%s: PIC %s is attempting to handle device data irqs, " | |||||
"base: %u count: %u", __func__, device_get_nameunit(dev), | |||||
pic->pic_irq_base, count)); | |||||
pic->pic_flags |= FLAG_PIC; | pic->pic_flags |= FLAG_PIC; | ||||
debugf("PIC %p registered for %s <dev %p, xref %x>\n", pic, | debugf("PIC %p registered for %s <dev %p, xref %x>\n", pic, | ||||
device_get_nameunit(dev), dev, xref); | device_get_nameunit(dev), dev, xref); | ||||
return (0); | return (0); | ||||
} | } | ||||
/* | /* | ||||
* Unregister interrupt controller. | * Unregister interrupt controller. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, | ||||
irq_root_arg = arg; | irq_root_arg = arg; | ||||
irq_root_ipicount = ipicount; | irq_root_ipicount = ipicount; | ||||
debugf("irq root set to %s\n", device_get_nameunit(dev)); | debugf("irq root set to %s\n", device_get_nameunit(dev)); | ||||
return (0); | return (0); | ||||
} | } | ||||
int | int | ||||
intr_pic_intr_count(device_t dev, u_int *count) | |||||
skraUnsubmitted Not Done Inline ActionsMaybe not needed at all? skra: Maybe not needed at all? | |||||
{ | |||||
struct intr_pic *pic; | |||||
pic = pic_lookup(dev, 0); | |||||
if (pic == NULL) | |||||
return (ESRCH); | |||||
KASSERT((pic->pic_flags & FLAG_PIC) != 0, | |||||
("%s: Found a non-PIC controller: %s", __func__, | |||||
device_get_name(pic->pic_dev))); | |||||
*count = pic->pic_irq_count; | |||||
return (0); | |||||
} | |||||
int | |||||
intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data, | intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data, | ||||
u_int *irqp) | u_int *irqp) | ||||
{ | { | ||||
int error; | int error; | ||||
struct intr_irqsrc *isrc; | struct intr_irqsrc *isrc; | ||||
struct intr_pic *pic; | struct intr_pic *pic; | ||||
if (data == NULL) | if (data == NULL) | ||||
▲ Show 20 Lines • Show All 219 Lines • ▼ Show 20 Lines | intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) | ||||
do { | do { | ||||
last_cpu++; | last_cpu++; | ||||
if (last_cpu > mp_maxid) | if (last_cpu > mp_maxid) | ||||
last_cpu = 0; | last_cpu = 0; | ||||
} while (!CPU_ISSET(last_cpu, cpumask)); | } while (!CPU_ISSET(last_cpu, cpumask)); | ||||
return (last_cpu); | return (last_cpu); | ||||
} | } | ||||
/* | |||||
* Distribute all the interrupt sources among the available | |||||
* CPUs once the AP's have been launched. | |||||
*/ | |||||
static void | static void | ||||
intr_irq_shuffle(void *arg __unused) | intr_irq_shuffle_pic(struct intr_pic *pic) | ||||
{ | { | ||||
struct intr_irqsrc *isrc; | struct intr_irqsrc *isrc; | ||||
u_int i; | u_int i; | ||||
if (mp_ncpus == 1) | mtx_assert(&isrc_table_lock, MA_OWNED); | ||||
return; | |||||
mtx_lock(&isrc_table_lock); | for (i = 0; i < pic->pic_irq_count; i++) { | ||||
irq_assign_cpu = TRUE; | isrc = pic->pic_sources[i]; | ||||
for (i = 0; i < NIRQ; i++) { | |||||
isrc = irq_sources[i]; | |||||
if (isrc == NULL || isrc->isrc_handlers == 0 || | if (isrc == NULL || isrc->isrc_handlers == 0 || | ||||
isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) | isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) | ||||
continue; | continue; | ||||
if (isrc->isrc_event != NULL && | if (isrc->isrc_event != NULL && | ||||
isrc->isrc_flags & INTR_ISRCF_BOUND && | isrc->isrc_flags & INTR_ISRCF_BOUND && | ||||
isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) | isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) | ||||
panic("%s: CPU inconsistency", __func__); | panic("%s: CPU inconsistency", __func__); | ||||
if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) | if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) | ||||
CPU_ZERO(&isrc->isrc_cpu); /* start again */ | CPU_ZERO(&isrc->isrc_cpu); /* start again */ | ||||
/* | /* | ||||
* We are in wicked position here if the following call fails | * We are in wicked position here if the following call fails | ||||
* for bound ISRC. The best thing we can do is to clear | * for bound ISRC. The best thing we can do is to clear | ||||
* isrc_cpu so inconsistency with ie_cpu will be detectable. | * isrc_cpu so inconsistency with ie_cpu will be detectable. | ||||
*/ | */ | ||||
if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) | if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) | ||||
CPU_ZERO(&isrc->isrc_cpu); | CPU_ZERO(&isrc->isrc_cpu); | ||||
} | } | ||||
} | |||||
/* | |||||
* Distribute all the interrupt sources among the available | |||||
* CPUs once the AP's have been launched. | |||||
*/ | |||||
static void | |||||
intr_irq_shuffle(void *arg __unused) | |||||
{ | |||||
struct intr_pic *pic; | |||||
if (mp_ncpus == 1) | |||||
return; | |||||
mtx_lock(&isrc_table_lock); | |||||
irq_assign_cpu = TRUE; | |||||
SLIST_FOREACH(pic, &pic_list, pic_next) { | |||||
intr_irq_shuffle_pic(pic); | |||||
} | |||||
mtx_unlock(&isrc_table_lock); | mtx_unlock(&isrc_table_lock); | ||||
} | } | ||||
SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); | SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); | ||||
#else | #else | ||||
u_int | u_int | ||||
intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) | intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | intr_pic_init_secondary(void) | ||||
PIC_INIT_SECONDARY(intr_irq_root_dev); | PIC_INIT_SECONDARY(intr_irq_root_dev); | ||||
//mtx_unlock(&isrc_table_lock); | //mtx_unlock(&isrc_table_lock); | ||||
} | } | ||||
#endif | #endif | ||||
#ifdef DDB | #ifdef DDB | ||||
DB_SHOW_COMMAND(irqs, db_show_irqs) | DB_SHOW_COMMAND(irqs, db_show_irqs) | ||||
{ | { | ||||
struct intr_pic *pic; | |||||
u_int i, irqsum; | u_int i, irqsum; | ||||
u_long num; | u_long num; | ||||
struct intr_irqsrc *isrc; | struct intr_irqsrc *isrc; | ||||
for (irqsum = 0, i = 0; i < NIRQ; i++) { | irqsum = 0; | ||||
isrc = irq_sources[i]; | SLIST_FOREACH(pic, &pic_list, pic_next) { | ||||
for (i = 0; i < pic->pic_irq_count; i++) { | |||||
isrc = pic->pic_sources[i]; | |||||
if (isrc == NULL) | if (isrc == NULL) | ||||
continue; | continue; | ||||
num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; | num = | ||||
isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; | |||||
db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, | db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, | ||||
skraUnsubmitted Done Inline ActionsDon't use i but isrc->isrc_irq or pic->pic_irq_base + i. skra: Don't use `i` but `isrc->isrc_irq` or `pic->pic_irq_base + i`. | |||||
isrc->isrc_name, isrc->isrc_cpu.__bits[0], | isrc->isrc_name, isrc->isrc_cpu.__bits[0], | ||||
isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num); | isrc->isrc_flags & INTR_ISRCF_BOUND ? | ||||
" (bound)" : "", num); | |||||
irqsum += num; | irqsum += num; | ||||
} | |||||
} | } | ||||
db_printf("irq total %u\n", irqsum); | db_printf("irq total %u\n", irqsum); | ||||
} | } | ||||
#endif | #endif |
struct pic* argument instead of device_t?