Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_intr.c
Show All 24 Lines | |||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | ||||
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||||
*/ | */ | ||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include "opt_ddb.h" | #include "opt_ddb.h" | ||||
#include "opt_hwpmc_hooks.h" | |||||
#include "opt_kstack_usage_prof.h" | #include "opt_kstack_usage_prof.h" | ||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/bus.h> | #include <sys/bus.h> | ||||
#include <sys/conf.h> | #include <sys/conf.h> | ||||
#include <sys/cpuset.h> | #include <sys/cpuset.h> | ||||
#include <sys/rtprio.h> | #include <sys/rtprio.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
Show All 29 Lines | |||||
/* | /* | ||||
* Describe an interrupt thread. There is one of these per interrupt event. | * Describe an interrupt thread. There is one of these per interrupt event. | ||||
*/ | */ | ||||
struct intr_thread { | struct intr_thread { | ||||
struct intr_event *it_event; | struct intr_event *it_event; | ||||
struct thread *it_thread; /* Kernel thread. */ | struct thread *it_thread; /* Kernel thread. */ | ||||
int it_flags; /* (j) IT_* flags. */ | int it_flags; /* (j) IT_* flags. */ | ||||
int it_need; /* Needs service. */ | int it_need; /* Needs service. */ | ||||
#ifdef HWPMC_HOOKS | |||||
kib: I suggest to have it_waiting defined always, the effect on memory usage is minuscule. | |||||
int it_waiting; /* Waiting in the runq. */ | |||||
kibUnsubmitted Not Done Inline ActionsWhat is the locking protocol for it_waiting? Is it locked by it_thread thread lock? kib: What is the locking protocol for it_waiting? Is it locked by it_thread thread lock? | |||||
#endif | |||||
}; | }; | ||||
/* Interrupt thread flags kept in it_flags */ | /* Interrupt thread flags kept in it_flags */ | ||||
#define IT_DEAD 0x000001 /* Thread is waiting to exit. */ | #define IT_DEAD 0x000001 /* Thread is waiting to exit. */ | ||||
#define IT_WAIT 0x000002 /* Thread is waiting for completion. */ | #define IT_WAIT 0x000002 /* Thread is waiting for completion. */ | ||||
struct intr_entropy { | struct intr_entropy { | ||||
struct thread *td; | struct thread *td; | ||||
Show All 15 Lines | |||||
SYSCTL_INT(_hw, OID_AUTO, intr_epoch_batch, CTLFLAG_RWTUN, &intr_epoch_batch, | SYSCTL_INT(_hw, OID_AUTO, intr_epoch_batch, CTLFLAG_RWTUN, &intr_epoch_batch, | ||||
0, "Maximum interrupt handler executions without re-entering epoch(9)"); | 0, "Maximum interrupt handler executions without re-entering epoch(9)"); | ||||
static TAILQ_HEAD(, intr_event) event_list = | static TAILQ_HEAD(, intr_event) event_list = | ||||
TAILQ_HEAD_INITIALIZER(event_list); | TAILQ_HEAD_INITIALIZER(event_list); | ||||
static struct mtx event_lock; | static struct mtx event_lock; | ||||
MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); | MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); | ||||
static void intr_event_update(struct intr_event *ie); | static void intr_event_update(struct intr_event *ie); | ||||
static int intr_event_schedule_thread(struct intr_event *ie); | static int intr_event_schedule_thread(struct intr_event *ie, struct trapframe *frame); | ||||
static struct intr_thread *ithread_create(const char *name); | static struct intr_thread *ithread_create(const char *name); | ||||
static void ithread_destroy(struct intr_thread *ithread); | static void ithread_destroy(struct intr_thread *ithread); | ||||
static void ithread_execute_handlers(struct proc *p, | static void ithread_execute_handlers(struct proc *p, | ||||
struct intr_event *ie); | struct intr_event *ie); | ||||
static void ithread_loop(void *); | static void ithread_loop(void *); | ||||
static void ithread_update(struct intr_thread *ithd); | static void ithread_update(struct intr_thread *ithd); | ||||
static void start_softintr(void *); | static void start_softintr(void *); | ||||
#ifdef HWPMC_HOOKS | |||||
#include <sys/pmckern.h> | |||||
PMC_SOFT_DEFINE( , , intr, all); | |||||
PMC_SOFT_DEFINE( , , intr, ithread); | |||||
PMC_SOFT_DEFINE( , , intr, filter); | |||||
PMC_SOFT_DEFINE( , , intr, stray); | |||||
PMC_SOFT_DEFINE( , , intr, schedule); | |||||
PMC_SOFT_DEFINE( , , intr, waiting); | |||||
PMC_SOFT_DEFINE( , , intr, waiting_5); | |||||
#endif | |||||
/* Map an interrupt type to an ithread priority. */ | /* Map an interrupt type to an ithread priority. */ | ||||
u_char | u_char | ||||
intr_priority(enum intr_type flags) | intr_priority(enum intr_type flags) | ||||
{ | { | ||||
u_char pri; | u_char pri; | ||||
flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | | flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | | ||||
INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); | INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); | ||||
▲ Show 20 Lines • Show All 642 Lines • ▼ Show 20 Lines | KASSERT((handler->ih_flags & IH_DEAD) == 0, | ||||
("update for a removed handler")); | ("update for a removed handler")); | ||||
if (ie->ie_thread == NULL) { | if (ie->ie_thread == NULL) { | ||||
intr_event_barrier(ie); | intr_event_barrier(ie); | ||||
return; | return; | ||||
} | } | ||||
if ((handler->ih_flags & IH_CHANGED) == 0) { | if ((handler->ih_flags & IH_CHANGED) == 0) { | ||||
handler->ih_flags |= IH_CHANGED; | handler->ih_flags |= IH_CHANGED; | ||||
intr_event_schedule_thread(ie); | intr_event_schedule_thread(ie, NULL); | ||||
} | } | ||||
while ((handler->ih_flags & IH_CHANGED) != 0) | while ((handler->ih_flags & IH_CHANGED) != 0) | ||||
msleep(handler, &ie->ie_lock, 0, "ih_barr", 0); | msleep(handler, &ie->ie_lock, 0, "ih_barr", 0); | ||||
} | } | ||||
/* | /* | ||||
* Sleep until an ithread finishes executing an interrupt handler. | * Sleep until an ithread finishes executing an interrupt handler. | ||||
* | * | ||||
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines | #endif | ||||
* Let the interrupt thread do the job. | * Let the interrupt thread do the job. | ||||
* The interrupt source is disabled when the interrupt thread is | * The interrupt source is disabled when the interrupt thread is | ||||
* running, so it does not have to worry about interaction with | * running, so it does not have to worry about interaction with | ||||
* intr_event_handle(). | * intr_event_handle(). | ||||
*/ | */ | ||||
KASSERT((handler->ih_flags & IH_DEAD) == 0, | KASSERT((handler->ih_flags & IH_DEAD) == 0, | ||||
("duplicate handle remove")); | ("duplicate handle remove")); | ||||
handler->ih_flags |= IH_DEAD; | handler->ih_flags |= IH_DEAD; | ||||
intr_event_schedule_thread(ie); | intr_event_schedule_thread(ie, NULL); | ||||
while (handler->ih_flags & IH_DEAD) | while (handler->ih_flags & IH_DEAD) | ||||
msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); | msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); | ||||
intr_event_update(ie); | intr_event_update(ie); | ||||
#ifdef notyet | #ifdef notyet | ||||
/* | /* | ||||
* XXX: This could be bad in the case of ppbus(8). Also, I think | * XXX: This could be bad in the case of ppbus(8). Also, I think | ||||
* this could lead to races of stale data when servicing an | * this could lead to races of stale data when servicing an | ||||
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | intr_event_resume_handler(void *cookie) | ||||
mtx_lock(&ie->ie_lock); | mtx_lock(&ie->ie_lock); | ||||
handler->ih_flags &= ~IH_SUSP; | handler->ih_flags &= ~IH_SUSP; | ||||
intr_handler_barrier(handler); | intr_handler_barrier(handler); | ||||
mtx_unlock(&ie->ie_lock); | mtx_unlock(&ie->ie_lock); | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
intr_event_schedule_thread(struct intr_event *ie) | intr_event_schedule_thread(struct intr_event *ie, struct trapframe *frame) | ||||
{ | { | ||||
struct intr_entropy entropy; | struct intr_entropy entropy; | ||||
struct intr_thread *it; | struct intr_thread *it; | ||||
struct thread *td; | struct thread *td; | ||||
struct thread *ctd; | struct thread *ctd; | ||||
/* | /* | ||||
* If no ithread or no handlers, then we have a stray interrupt. | * If no ithread or no handlers, then we have a stray interrupt. | ||||
Show All 25 Lines | intr_event_schedule_thread(struct intr_event *ie, struct trapframe *frame) | ||||
* | * | ||||
* Use store_rel to arrange that the store to ih_need in | * Use store_rel to arrange that the store to ih_need in | ||||
* swi_sched() is before the store to it_need and prepare for | * swi_sched() is before the store to it_need and prepare for | ||||
* transfer of this order to loads in the ithread. | * transfer of this order to loads in the ithread. | ||||
*/ | */ | ||||
atomic_store_rel_int(&it->it_need, 1); | atomic_store_rel_int(&it->it_need, 1); | ||||
thread_lock(td); | thread_lock(td); | ||||
if (TD_AWAITING_INTR(td)) { | if (TD_AWAITING_INTR(td)) { | ||||
#ifdef HWPMC_HOOKS | |||||
it->it_waiting = 0; | |||||
if (frame != NULL) | |||||
PMC_SOFT_CALL_TF( , , intr, schedule, frame); | |||||
else | |||||
PMC_SOFT_CALL( , , intr, schedule); | |||||
#endif | |||||
CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, td->td_proc->p_pid, | CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, td->td_proc->p_pid, | ||||
td->td_name); | td->td_name); | ||||
TD_CLR_IWAIT(td); | TD_CLR_IWAIT(td); | ||||
sched_add(td, SRQ_INTR); | sched_add(td, SRQ_INTR); | ||||
} else { | } else { | ||||
#ifdef HWPMC_HOOKS | |||||
it->it_waiting++; | |||||
if (frame != NULL) | |||||
PMC_SOFT_CALL_TF( , , intr, waiting, frame); | |||||
else | |||||
PMC_SOFT_CALL( , , intr, waiting); | |||||
if (it->it_waiting > 5) { | |||||
kibUnsubmitted Not Done Inline ActionsCan you explain the desired semantic for intr waiting_5 soft event? kib: Can you explain the desired semantic for intr waiting_5 soft event? | |||||
if (frame != NULL) | |||||
PMC_SOFT_CALL_TF( , , intr, waiting_5, frame); | |||||
else | |||||
PMC_SOFT_CALL( , , intr, waiting_5); | |||||
} | |||||
#endif | |||||
CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", | CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", | ||||
__func__, td->td_proc->p_pid, td->td_name, it->it_need, TD_GET_STATE(td)); | __func__, td->td_proc->p_pid, td->td_name, it->it_need, TD_GET_STATE(td)); | ||||
thread_unlock(td); | thread_unlock(td); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | swi_sched(void *cookie, int flags) | ||||
if (flags & SWI_FROMNMI) { | if (flags & SWI_FROMNMI) { | ||||
#if defined(SMP) && (defined(__i386__) || defined(__amd64__)) | #if defined(SMP) && (defined(__i386__) || defined(__amd64__)) | ||||
KASSERT(ie == clk_intr_event, | KASSERT(ie == clk_intr_event, | ||||
("SWI_FROMNMI used not with clk_intr_event")); | ("SWI_FROMNMI used not with clk_intr_event")); | ||||
ipi_self_from_nmi(IPI_SWI); | ipi_self_from_nmi(IPI_SWI); | ||||
#endif | #endif | ||||
} else { | } else { | ||||
VM_CNT_INC(v_soft); | VM_CNT_INC(v_soft); | ||||
error = intr_event_schedule_thread(ie); | error = intr_event_schedule_thread(ie, NULL); | ||||
KASSERT(error == 0, ("stray software interrupt")); | KASSERT(error == 0, ("stray software interrupt")); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Remove a software interrupt handler. Currently this code does not | * Remove a software interrupt handler. Currently this code does not | ||||
* remove the associated interrupt event if it becomes empty. Calling code | * remove the associated interrupt event if it becomes empty. Calling code | ||||
* may do so manually via intr_event_destroy(), but that's not really | * may do so manually via intr_event_destroy(), but that's not really | ||||
▲ Show 20 Lines • Show All 274 Lines • ▼ Show 20 Lines | CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) { | ||||
} | } | ||||
CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__, | CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__, | ||||
ih->ih_filter, ih->ih_argument == NULL ? frame : | ih->ih_filter, ih->ih_argument == NULL ? frame : | ||||
ih->ih_argument, ih->ih_name); | ih->ih_argument, ih->ih_name); | ||||
if (ih->ih_argument == NULL) | if (ih->ih_argument == NULL) | ||||
ret = ih->ih_filter(frame); | ret = ih->ih_filter(frame); | ||||
else | else | ||||
ret = ih->ih_filter(ih->ih_argument); | ret = ih->ih_filter(ih->ih_argument); | ||||
#ifdef HWPMC_HOOKS | |||||
PMC_SOFT_CALL_TF( , , intr, all, frame); | |||||
#endif | |||||
KASSERT(ret == FILTER_STRAY || | KASSERT(ret == FILTER_STRAY || | ||||
((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && | ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && | ||||
(ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), | (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), | ||||
("%s: incorrect return value %#x from %s", __func__, ret, | ("%s: incorrect return value %#x from %s", __func__, ret, | ||||
ih->ih_name)); | ih->ih_name)); | ||||
filter = filter || ret == FILTER_HANDLED; | filter = filter || ret == FILTER_HANDLED; | ||||
#ifdef HWPMC_HOOKS | |||||
if (ret & FILTER_SCHEDULE_THREAD) | |||||
PMC_SOFT_CALL_TF( , , intr, ithread, frame); | |||||
else if (ret & FILTER_HANDLED) | |||||
PMC_SOFT_CALL_TF( , , intr, filter, frame); | |||||
else if (ret == FILTER_STRAY) | |||||
PMC_SOFT_CALL_TF( , , intr, stray, frame); | |||||
#endif | |||||
/* | /* | ||||
* Wrapper handler special handling: | * Wrapper handler special handling: | ||||
* | * | ||||
* in some particular cases (like pccard and pccbb), | * in some particular cases (like pccard and pccbb), | ||||
* the _real_ device handler is wrapped in a couple of | * the _real_ device handler is wrapped in a couple of | ||||
* functions - a filter wrapper and an ithread wrapper. | * functions - a filter wrapper and an ithread wrapper. | ||||
* In this case (and just in this case), the filter wrapper | * In this case (and just in this case), the filter wrapper | ||||
Show All 20 Lines | if (thread) { | ||||
if (ie->ie_post_filter != NULL) | if (ie->ie_post_filter != NULL) | ||||
ie->ie_post_filter(ie->ie_source); | ie->ie_post_filter(ie->ie_source); | ||||
} | } | ||||
/* Schedule the ithread if needed. */ | /* Schedule the ithread if needed. */ | ||||
if (thread) { | if (thread) { | ||||
int error __unused; | int error __unused; | ||||
error = intr_event_schedule_thread(ie); | error = intr_event_schedule_thread(ie, NULL); | ||||
Not Done Inline ActionsWhy not pass frame here? mhorne: Why not pass `frame` here? | |||||
KASSERT(error == 0, ("bad stray interrupt")); | KASSERT(error == 0, ("bad stray interrupt")); | ||||
} | } | ||||
critical_exit(); | critical_exit(); | ||||
td->td_intr_nesting_level--; | td->td_intr_nesting_level--; | ||||
#ifdef notyet | #ifdef notyet | ||||
/* The interrupt is not aknowledged by any filter and has no ithread. */ | /* The interrupt is not aknowledged by any filter and has no ithread. */ | ||||
if (!thread && !filter) | if (!thread && !filter) | ||||
return (EINVAL); | return (EINVAL); | ||||
▲ Show 20 Lines • Show All 242 Lines • Show Last 20 Lines |
I suggest to have it_waiting defined always, the effect on memory usage is minuscule.