Changeset View
Changeset View
Standalone View
Standalone View
kern/kern_intr.c
Show First 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | |||||
struct proc *intrproc; | struct proc *intrproc; | ||||
static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); | static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); | ||||
static int intr_storm_threshold = 0; | static int intr_storm_threshold = 0; | ||||
SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RWTUN, | SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RWTUN, | ||||
&intr_storm_threshold, 0, | &intr_storm_threshold, 0, | ||||
"Number of consecutive interrupts before storm protection is enabled"); | "Number of consecutive interrupts before storm protection is enabled"); | ||||
static int intr_epoch_batch = 1000; | |||||
SYSCTL_INT(_hw, OID_AUTO, intr_epoch_batch, CTLFLAG_RWTUN, &intr_epoch_batch, | |||||
0, "Maximum interrupt handler executions without re-entering epoch(9)"); | |||||
static TAILQ_HEAD(, intr_event) event_list = | static TAILQ_HEAD(, intr_event) event_list = | ||||
TAILQ_HEAD_INITIALIZER(event_list); | TAILQ_HEAD_INITIALIZER(event_list); | ||||
static struct mtx event_lock; | static struct mtx event_lock; | ||||
MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); | MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); | ||||
static void intr_event_update(struct intr_event *ie); | static void intr_event_update(struct intr_event *ie); | ||||
static int intr_event_schedule_thread(struct intr_event *ie); | static int intr_event_schedule_thread(struct intr_event *ie); | ||||
static struct intr_thread *ithread_create(const char *name); | static struct intr_thread *ithread_create(const char *name); | ||||
static void ithread_destroy(struct intr_thread *ithread); | static void ithread_destroy(struct intr_thread *ithread); | ||||
static void ithread_execute_handlers(struct proc *p, | static void ithread_execute_handlers(struct proc *p, | ||||
struct intr_event *ie); | struct intr_event *ie); | ||||
static void ithread_loop(void *); | static void ithread_loop(void *); | ||||
static void ithread_update(struct intr_thread *ithd); | static void ithread_update(struct intr_thread *ithd); | ||||
static void start_softintr(void *); | static void start_softintr(void *); | ||||
/* Map an interrupt type to an ithread priority. */ | /* Map an interrupt type to an ithread priority. */ | ||||
u_char | u_char | ||||
intr_priority(enum intr_type flags) | intr_priority(enum intr_type flags) | ||||
{ | { | ||||
u_char pri; | u_char pri; | ||||
flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | | switch (flags & INTR_TYPE_MASK) { | ||||
INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); | |||||
switch (flags) { | |||||
case INTR_TYPE_TTY: | case INTR_TYPE_TTY: | ||||
pri = PI_TTY; | pri = PI_TTY; | ||||
break; | break; | ||||
case INTR_TYPE_BIO: | case INTR_TYPE_BIO: | ||||
pri = PI_DISK; | pri = PI_DISK; | ||||
break; | break; | ||||
case INTR_TYPE_NET: | case INTR_TYPE_NET: | ||||
pri = PI_NET; | pri = PI_NET; | ||||
break; | break; | ||||
case INTR_TYPE_CAM: | case INTR_TYPE_CAM: | ||||
pri = PI_DISK; | pri = PI_DISK; | ||||
break; | break; | ||||
case INTR_TYPE_AV: | case INTR_TYPE_AV: | ||||
pri = PI_AV; | pri = PI_AV; | ||||
break; | break; | ||||
case INTR_TYPE_CLK: | case INTR_TYPE_CLK: | ||||
pri = PI_REALTIME; | pri = PI_REALTIME; | ||||
break; | break; | ||||
case INTR_TYPE_MISC: | case INTR_TYPE_MISC: | ||||
pri = PI_DULL; /* don't care */ | pri = PI_DULL; /* don't care */ | ||||
break; | break; | ||||
default: | default: | ||||
/* We didn't specify an interrupt level. */ | /* We didn't specify an interrupt level. */ | ||||
panic("intr_priority: no interrupt type in flags"); | panic("intr_priority: Invalid interrupt flags(0x%08x)", flags); | ||||
} | } | ||||
return (pri); | |||||
return pri; | |||||
} | } | ||||
/* | /* | ||||
* Update an ithread based on the associated intr_event. | * Update an ithread based on the associated intr_event. | ||||
*/ | */ | ||||
static void | static void | ||||
ithread_update(struct intr_thread *ithd) | ithread_update(struct intr_thread *ithd) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 426 Lines • ▼ Show 20 Lines | intr_event_add_handler(struct intr_event *ie, const char *name, | ||||
ih->ih_event = ie; | ih->ih_event = ie; | ||||
ih->ih_pri = pri; | ih->ih_pri = pri; | ||||
if (flags & INTR_EXCL) | if (flags & INTR_EXCL) | ||||
ih->ih_flags = IH_EXCLUSIVE; | ih->ih_flags = IH_EXCLUSIVE; | ||||
if (flags & INTR_MPSAFE) | if (flags & INTR_MPSAFE) | ||||
ih->ih_flags |= IH_MPSAFE; | ih->ih_flags |= IH_MPSAFE; | ||||
if (flags & INTR_ENTROPY) | if (flags & INTR_ENTROPY) | ||||
ih->ih_flags |= IH_ENTROPY; | ih->ih_flags |= IH_ENTROPY; | ||||
if (flags & INTR_TYPE_NET) | if (flags & INTR_EPOCH) { | ||||
ih->ih_flags |= IH_NET; | switch (flags & INTR_TYPE_MASK) { | ||||
case INTR_TYPE_NET: | |||||
ih->ih_epoch = &net_epoch_preempt; | |||||
break; | |||||
default: | |||||
free(ih, M_ITHREAD); | |||||
return (EINVAL); | |||||
} | |||||
} | |||||
/* We can only have one exclusive handler in a event. */ | /* We can only have one exclusive handler in a event. */ | ||||
mtx_lock(&ie->ie_lock); | mtx_lock(&ie->ie_lock); | ||||
if (!CK_SLIST_EMPTY(&ie->ie_handlers)) { | if (!CK_SLIST_EMPTY(&ie->ie_handlers)) { | ||||
if ((flags & INTR_EXCL) || | if ((flags & INTR_EXCL) || | ||||
(CK_SLIST_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { | (CK_SLIST_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { | ||||
mtx_unlock(&ie->ie_lock); | mtx_unlock(&ie->ie_lock); | ||||
free(ih, M_ITHREAD); | free(ih, M_ITHREAD); | ||||
return (EINVAL); | return (EINVAL); | ||||
▲ Show 20 Lines • Show All 479 Lines • ▼ Show 20 Lines | swi_remove(void *cookie) | ||||
return (intr_event_remove_handler(cookie)); | return (intr_event_remove_handler(cookie)); | ||||
} | } | ||||
static void | static void | ||||
intr_event_execute_handlers(struct proc *p, struct intr_event *ie) | intr_event_execute_handlers(struct proc *p, struct intr_event *ie) | ||||
{ | { | ||||
struct intr_handler *ih, *ihn, *ihp; | struct intr_handler *ih, *ihn, *ihp; | ||||
struct epoch_tracker et; | |||||
ihp = NULL; | ihp = NULL; | ||||
CK_SLIST_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { | CK_SLIST_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { | ||||
/* | /* | ||||
* If this handler is marked for death, remove it from | * If this handler is marked for death, remove it from | ||||
* the list of handlers and wake up the sleeper. | * the list of handlers and wake up the sleeper. | ||||
*/ | */ | ||||
if (ih->ih_flags & IH_DEAD) { | if (ih->ih_flags & IH_DEAD) { | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | CK_SLIST_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { | ||||
/* Execute this handler. */ | /* Execute this handler. */ | ||||
CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", | CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", | ||||
__func__, p->p_pid, (void *)ih->ih_handler, | __func__, p->p_pid, (void *)ih->ih_handler, | ||||
ih->ih_argument, ih->ih_name, ih->ih_flags); | ih->ih_argument, ih->ih_name, ih->ih_flags); | ||||
if (!(ih->ih_flags & IH_MPSAFE)) | if (!(ih->ih_flags & IH_MPSAFE)) | ||||
mtx_lock(&Giant); | mtx_lock(&Giant); | ||||
if (ih->ih_epoch != NULL) | |||||
epoch_enter_preempt(ih->ih_epoch, &et); | |||||
ih->ih_handler(ih->ih_argument); | ih->ih_handler(ih->ih_argument); | ||||
if (ih->ih_epoch != NULL) | |||||
epoch_exit_preempt(ih->ih_epoch, &et); | |||||
if (!(ih->ih_flags & IH_MPSAFE)) | if (!(ih->ih_flags & IH_MPSAFE)) | ||||
mtx_unlock(&Giant); | mtx_unlock(&Giant); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
ithread_execute_handlers(struct proc *p, struct intr_event *ie) | ithread_execute_handlers(struct proc *p, struct intr_event *ie) | ||||
{ | { | ||||
Show All 36 Lines | |||||
} | } | ||||
/* | /* | ||||
* This is the main code for interrupt threads. | * This is the main code for interrupt threads. | ||||
*/ | */ | ||||
static void | static void | ||||
ithread_loop(void *arg) | ithread_loop(void *arg) | ||||
{ | { | ||||
struct epoch_tracker et; | |||||
struct intr_thread *ithd; | struct intr_thread *ithd; | ||||
struct intr_event *ie; | struct intr_event *ie; | ||||
struct thread *td; | struct thread *td; | ||||
struct proc *p; | struct proc *p; | ||||
int wake, epoch_count; | int wake; | ||||
td = curthread; | td = curthread; | ||||
p = td->td_proc; | p = td->td_proc; | ||||
ithd = (struct intr_thread *)arg; | ithd = (struct intr_thread *)arg; | ||||
KASSERT(ithd->it_thread == td, | KASSERT(ithd->it_thread == td, | ||||
("%s: ithread and proc linkage out of sync", __func__)); | ("%s: ithread and proc linkage out of sync", __func__)); | ||||
ie = ithd->it_event; | ie = ithd->it_event; | ||||
ie->ie_count = 0; | ie->ie_count = 0; | ||||
Show All 18 Lines | for (;;) { | ||||
* Service interrupts. If another interrupt arrives while | * Service interrupts. If another interrupt arrives while | ||||
* we are running, it will set it_need to note that we | * we are running, it will set it_need to note that we | ||||
* should make another pass. | * should make another pass. | ||||
* | * | ||||
* The load_acq part of the following cmpset ensures | * The load_acq part of the following cmpset ensures | ||||
* that the load of ih_need in ithread_execute_handlers() | * that the load of ih_need in ithread_execute_handlers() | ||||
* is ordered after the load of it_need here. | * is ordered after the load of it_need here. | ||||
*/ | */ | ||||
if (ie->ie_hflags & IH_NET) { | while (atomic_cmpset_acq_int(&ithd->it_need, 1, 0) != 0) | ||||
epoch_count = 0; | |||||
NET_EPOCH_ENTER(et); | |||||
} | |||||
while (atomic_cmpset_acq_int(&ithd->it_need, 1, 0) != 0) { | |||||
ithread_execute_handlers(p, ie); | ithread_execute_handlers(p, ie); | ||||
if ((ie->ie_hflags & IH_NET) && | |||||
++epoch_count >= intr_epoch_batch) { | |||||
NET_EPOCH_EXIT(et); | |||||
epoch_count = 0; | |||||
NET_EPOCH_ENTER(et); | |||||
} | |||||
} | |||||
if (ie->ie_hflags & IH_NET) | |||||
NET_EPOCH_EXIT(et); | |||||
WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); | WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); | ||||
mtx_assert(&Giant, MA_NOTOWNED); | mtx_assert(&Giant, MA_NOTOWNED); | ||||
/* | /* | ||||
* Processed all our interrupts. Now get the sched | * Processed all our interrupts. Now get the sched | ||||
* lock. This may take a while and it_need may get | * lock. This may take a while and it_need may get | ||||
* set again, so we have to check it again. | * set again, so we have to check it again. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 374 Lines • Show Last 20 Lines |