Index: head/sys/kern/kern_intr.c =================================================================== --- head/sys/kern/kern_intr.c (revision 271711) +++ head/sys/kern/kern_intr.c (revision 271712) @@ -1,1932 +1,1932 @@ /*- * Copyright (c) 1997, Stefan Esser * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #include #endif /* * Describe an interrupt thread. There is one of these per interrupt event. */ struct intr_thread { struct intr_event *it_event; struct thread *it_thread; /* Kernel thread. */ int it_flags; /* (j) IT_* flags. */ int it_need; /* Needs service. */ }; /* Interrupt thread flags kept in it_flags */ #define IT_DEAD 0x000001 /* Thread is waiting to exit. */ #define IT_WAIT 0x000002 /* Thread is waiting for completion. */ struct intr_entropy { struct thread *td; uintptr_t event; }; struct intr_event *clk_intr_event; struct intr_event *tty_intr_event; void *vm_ih; struct proc *intrproc; static MALLOC_DEFINE(M_ITHREAD, "ithread", "Interrupt Threads"); static int intr_storm_threshold = 1000; SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RWTUN, &intr_storm_threshold, 0, "Number of consecutive interrupts before storm protection is enabled"); static TAILQ_HEAD(, intr_event) event_list = TAILQ_HEAD_INITIALIZER(event_list); static struct mtx event_lock; MTX_SYSINIT(intr_event_list, &event_lock, "intr event list", MTX_DEF); static void intr_event_update(struct intr_event *ie); #ifdef INTR_FILTER static int intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *ithd); static int intr_filter_loop(struct intr_event *ie, struct trapframe *frame, struct intr_thread **ithd); static struct intr_thread *ithread_create(const char *name, struct intr_handler *ih); #else static int intr_event_schedule_thread(struct intr_event *ie); static struct intr_thread *ithread_create(const char *name); #endif static void ithread_destroy(struct intr_thread *ithread); static void ithread_execute_handlers(struct proc *p, struct intr_event *ie); #ifdef INTR_FILTER static void priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih); #endif static void ithread_loop(void *); static void ithread_update(struct intr_thread *ithd); static void start_softintr(void *); /* Map an interrupt type to an ithread priority. */ u_char intr_priority(enum intr_type flags) { u_char pri; flags &= (INTR_TYPE_TTY | INTR_TYPE_BIO | INTR_TYPE_NET | INTR_TYPE_CAM | INTR_TYPE_MISC | INTR_TYPE_CLK | INTR_TYPE_AV); switch (flags) { case INTR_TYPE_TTY: pri = PI_TTY; break; case INTR_TYPE_BIO: pri = PI_DISK; break; case INTR_TYPE_NET: pri = PI_NET; break; case INTR_TYPE_CAM: pri = PI_DISK; break; case INTR_TYPE_AV: pri = PI_AV; break; case INTR_TYPE_CLK: pri = PI_REALTIME; break; case INTR_TYPE_MISC: pri = PI_DULL; /* don't care */ break; default: /* We didn't specify an interrupt level. */ panic("intr_priority: no interrupt type in flags"); } return pri; } /* * Update an ithread based on the associated intr_event. */ static void ithread_update(struct intr_thread *ithd) { struct intr_event *ie; struct thread *td; u_char pri; ie = ithd->it_event; td = ithd->it_thread; /* Determine the overall priority of this event. */ if (TAILQ_EMPTY(&ie->ie_handlers)) pri = PRI_MAX_ITHD; else pri = TAILQ_FIRST(&ie->ie_handlers)->ih_pri; /* Update name and priority. */ strlcpy(td->td_name, ie->ie_fullname, sizeof(td->td_name)); #ifdef KTR sched_clear_tdname(td); #endif thread_lock(td); sched_prio(td, pri); thread_unlock(td); } /* * Regenerate the full name of an interrupt event and update its priority. */ static void intr_event_update(struct intr_event *ie) { struct intr_handler *ih; char *last; int missed, space; /* Start off with no entropy and just the name of the event. */ mtx_assert(&ie->ie_lock, MA_OWNED); strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); ie->ie_flags &= ~IE_ENTROPY; missed = 0; space = 1; /* Run through all the handlers updating values. */ TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { if (strlen(ie->ie_fullname) + strlen(ih->ih_name) + 1 < sizeof(ie->ie_fullname)) { strcat(ie->ie_fullname, " "); strcat(ie->ie_fullname, ih->ih_name); space = 0; } else missed++; if (ih->ih_flags & IH_ENTROPY) ie->ie_flags |= IE_ENTROPY; } /* * If the handler names were too long, add +'s to indicate missing * names. If we run out of room and still have +'s to add, change * the last character from a + to a *. */ last = &ie->ie_fullname[sizeof(ie->ie_fullname) - 2]; while (missed-- > 0) { if (strlen(ie->ie_fullname) + 1 == sizeof(ie->ie_fullname)) { if (*last == '+') { *last = '*'; break; } else *last = '+'; } else if (space) { strcat(ie->ie_fullname, " +"); space = 0; } else strcat(ie->ie_fullname, "+"); } /* * If this event has an ithread, update it's priority and * name. */ if (ie->ie_thread != NULL) ithread_update(ie->ie_thread); CTR2(KTR_INTR, "%s: updated %s", __func__, ie->ie_fullname); } int intr_event_create(struct intr_event **event, void *source, int flags, int irq, void (*pre_ithread)(void *), void (*post_ithread)(void *), - void (*post_filter)(void *), int (*assign_cpu)(void *, u_char), + void (*post_filter)(void *), int (*assign_cpu)(void *, int), const char *fmt, ...) { struct intr_event *ie; va_list ap; /* The only valid flag during creation is IE_SOFT. */ if ((flags & ~IE_SOFT) != 0) return (EINVAL); ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); ie->ie_source = source; ie->ie_pre_ithread = pre_ithread; ie->ie_post_ithread = post_ithread; ie->ie_post_filter = post_filter; ie->ie_assign_cpu = assign_cpu; ie->ie_flags = flags; ie->ie_irq = irq; ie->ie_cpu = NOCPU; TAILQ_INIT(&ie->ie_handlers); mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); va_start(ap, fmt); vsnprintf(ie->ie_name, sizeof(ie->ie_name), fmt, ap); va_end(ap); strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname)); mtx_lock(&event_lock); TAILQ_INSERT_TAIL(&event_list, ie, ie_list); mtx_unlock(&event_lock); if (event != NULL) *event = ie; CTR2(KTR_INTR, "%s: created %s", __func__, ie->ie_name); return (0); } /* * Bind an interrupt event to the specified CPU. Note that not all * platforms support binding an interrupt to a CPU. For those * platforms this request will fail. For supported platforms, any * associated ithreads as well as the primary interrupt context will * be bound to the specificed CPU. Using a cpu id of NOCPU unbinds * the interrupt event. */ int -intr_event_bind(struct intr_event *ie, u_char cpu) +intr_event_bind(struct intr_event *ie, int cpu) { lwpid_t id; int error; /* Need a CPU to bind to. */ if (cpu != NOCPU && CPU_ABSENT(cpu)) return (EINVAL); if (ie->ie_assign_cpu == NULL) return (EOPNOTSUPP); error = priv_check(curthread, PRIV_SCHED_CPUSET_INTR); if (error) return (error); /* * If we have any ithreads try to set their mask first to verify * permissions, etc. */ mtx_lock(&ie->ie_lock); if (ie->ie_thread != NULL) { id = ie->ie_thread->it_thread->td_tid; mtx_unlock(&ie->ie_lock); error = cpuset_setithread(id, cpu); if (error) return (error); } else mtx_unlock(&ie->ie_lock); error = ie->ie_assign_cpu(ie->ie_source, cpu); if (error) { mtx_lock(&ie->ie_lock); if (ie->ie_thread != NULL) { cpu = ie->ie_cpu; id = ie->ie_thread->it_thread->td_tid; mtx_unlock(&ie->ie_lock); (void)cpuset_setithread(id, cpu); } else mtx_unlock(&ie->ie_lock); return (error); } mtx_lock(&ie->ie_lock); ie->ie_cpu = cpu; mtx_unlock(&ie->ie_lock); return (error); } static struct intr_event * intr_lookup(int irq) { struct intr_event *ie; mtx_lock(&event_lock); TAILQ_FOREACH(ie, &event_list, ie_list) if (ie->ie_irq == irq && (ie->ie_flags & IE_SOFT) == 0 && TAILQ_FIRST(&ie->ie_handlers) != NULL) break; mtx_unlock(&event_lock); return (ie); } int intr_setaffinity(int irq, void *m) { struct intr_event *ie; cpuset_t *mask; u_char cpu; int n; mask = m; cpu = NOCPU; /* * If we're setting all cpus we can unbind. Otherwise make sure * only one cpu is in the set. */ if (CPU_CMP(cpuset_root, mask)) { for (n = 0; n < CPU_SETSIZE; n++) { if (!CPU_ISSET(n, mask)) continue; if (cpu != NOCPU) return (EINVAL); cpu = (u_char)n; } } ie = intr_lookup(irq); if (ie == NULL) return (ESRCH); return (intr_event_bind(ie, cpu)); } int intr_getaffinity(int irq, void *m) { struct intr_event *ie; cpuset_t *mask; mask = m; ie = intr_lookup(irq); if (ie == NULL) return (ESRCH); CPU_ZERO(mask); mtx_lock(&ie->ie_lock); if (ie->ie_cpu == NOCPU) CPU_COPY(cpuset_root, mask); else CPU_SET(ie->ie_cpu, mask); mtx_unlock(&ie->ie_lock); return (0); } int intr_event_destroy(struct intr_event *ie) { mtx_lock(&event_lock); mtx_lock(&ie->ie_lock); if (!TAILQ_EMPTY(&ie->ie_handlers)) { mtx_unlock(&ie->ie_lock); mtx_unlock(&event_lock); return (EBUSY); } TAILQ_REMOVE(&event_list, ie, ie_list); #ifndef notyet if (ie->ie_thread != NULL) { ithread_destroy(ie->ie_thread); ie->ie_thread = NULL; } #endif mtx_unlock(&ie->ie_lock); mtx_unlock(&event_lock); mtx_destroy(&ie->ie_lock); free(ie, M_ITHREAD); return (0); } #ifndef INTR_FILTER static struct intr_thread * ithread_create(const char *name) { struct intr_thread *ithd; struct thread *td; int error; ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); error = kproc_kthread_add(ithread_loop, ithd, &intrproc, &td, RFSTOPPED | RFHIGHPID, 0, "intr", "%s", name); if (error) panic("kproc_create() failed with %d", error); thread_lock(td); sched_class(td, PRI_ITHD); TD_SET_IWAIT(td); thread_unlock(td); td->td_pflags |= TDP_ITHREAD; ithd->it_thread = td; CTR2(KTR_INTR, "%s: created %s", __func__, name); return (ithd); } #else static struct intr_thread * ithread_create(const char *name, struct intr_handler *ih) { struct intr_thread *ithd; struct thread *td; int error; ithd = malloc(sizeof(struct intr_thread), M_ITHREAD, M_WAITOK | M_ZERO); error = kproc_kthread_add(ithread_loop, ih, &intrproc, &td, RFSTOPPED | RFHIGHPID, 0, "intr", "%s", name); if (error) panic("kproc_create() failed with %d", error); thread_lock(td); sched_class(td, PRI_ITHD); TD_SET_IWAIT(td); thread_unlock(td); td->td_pflags |= TDP_ITHREAD; ithd->it_thread = td; CTR2(KTR_INTR, "%s: created %s", __func__, name); return (ithd); } #endif static void ithread_destroy(struct intr_thread *ithread) { struct thread *td; CTR2(KTR_INTR, "%s: killing %s", __func__, ithread->it_event->ie_name); td = ithread->it_thread; thread_lock(td); ithread->it_flags |= IT_DEAD; if (TD_AWAITING_INTR(td)) { TD_CLR_IWAIT(td); sched_add(td, SRQ_INTR); } thread_unlock(td); } #ifndef INTR_FILTER int intr_event_add_handler(struct intr_event *ie, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, void **cookiep) { struct intr_handler *ih, *temp_ih; struct intr_thread *it; if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) return (EINVAL); /* Allocate and populate an interrupt handler structure. */ ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); ih->ih_filter = filter; ih->ih_handler = handler; ih->ih_argument = arg; strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); ih->ih_event = ie; ih->ih_pri = pri; if (flags & INTR_EXCL) ih->ih_flags = IH_EXCLUSIVE; if (flags & INTR_MPSAFE) ih->ih_flags |= IH_MPSAFE; if (flags & INTR_ENTROPY) ih->ih_flags |= IH_ENTROPY; /* We can only have one exclusive handler in a event. */ mtx_lock(&ie->ie_lock); if (!TAILQ_EMPTY(&ie->ie_handlers)) { if ((flags & INTR_EXCL) || (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { mtx_unlock(&ie->ie_lock); free(ih, M_ITHREAD); return (EINVAL); } } /* Create a thread if we need one. */ while (ie->ie_thread == NULL && handler != NULL) { if (ie->ie_flags & IE_ADDING_THREAD) msleep(ie, &ie->ie_lock, 0, "ithread", 0); else { ie->ie_flags |= IE_ADDING_THREAD; mtx_unlock(&ie->ie_lock); it = ithread_create("intr: newborn"); mtx_lock(&ie->ie_lock); ie->ie_flags &= ~IE_ADDING_THREAD; ie->ie_thread = it; it->it_event = ie; ithread_update(it); wakeup(ie); } } /* Add the new handler to the event in priority order. */ TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { if (temp_ih->ih_pri > ih->ih_pri) break; } if (temp_ih == NULL) TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); else TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); intr_event_update(ie); CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, ie->ie_name); mtx_unlock(&ie->ie_lock); if (cookiep != NULL) *cookiep = ih; return (0); } #else int intr_event_add_handler(struct intr_event *ie, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, void **cookiep) { struct intr_handler *ih, *temp_ih; struct intr_thread *it; if (ie == NULL || name == NULL || (handler == NULL && filter == NULL)) return (EINVAL); /* Allocate and populate an interrupt handler structure. */ ih = malloc(sizeof(struct intr_handler), M_ITHREAD, M_WAITOK | M_ZERO); ih->ih_filter = filter; ih->ih_handler = handler; ih->ih_argument = arg; strlcpy(ih->ih_name, name, sizeof(ih->ih_name)); ih->ih_event = ie; ih->ih_pri = pri; if (flags & INTR_EXCL) ih->ih_flags = IH_EXCLUSIVE; if (flags & INTR_MPSAFE) ih->ih_flags |= IH_MPSAFE; if (flags & INTR_ENTROPY) ih->ih_flags |= IH_ENTROPY; /* We can only have one exclusive handler in a event. */ mtx_lock(&ie->ie_lock); if (!TAILQ_EMPTY(&ie->ie_handlers)) { if ((flags & INTR_EXCL) || (TAILQ_FIRST(&ie->ie_handlers)->ih_flags & IH_EXCLUSIVE)) { mtx_unlock(&ie->ie_lock); free(ih, M_ITHREAD); return (EINVAL); } } /* For filtered handlers, create a private ithread to run on. */ if (filter != NULL && handler != NULL) { mtx_unlock(&ie->ie_lock); it = ithread_create("intr: newborn", ih); mtx_lock(&ie->ie_lock); it->it_event = ie; ih->ih_thread = it; ithread_update(it); /* XXX - do we really need this?!?!? */ } else { /* Create the global per-event thread if we need one. */ while (ie->ie_thread == NULL && handler != NULL) { if (ie->ie_flags & IE_ADDING_THREAD) msleep(ie, &ie->ie_lock, 0, "ithread", 0); else { ie->ie_flags |= IE_ADDING_THREAD; mtx_unlock(&ie->ie_lock); it = ithread_create("intr: newborn", ih); mtx_lock(&ie->ie_lock); ie->ie_flags &= ~IE_ADDING_THREAD; ie->ie_thread = it; it->it_event = ie; ithread_update(it); wakeup(ie); } } } /* Add the new handler to the event in priority order. */ TAILQ_FOREACH(temp_ih, &ie->ie_handlers, ih_next) { if (temp_ih->ih_pri > ih->ih_pri) break; } if (temp_ih == NULL) TAILQ_INSERT_TAIL(&ie->ie_handlers, ih, ih_next); else TAILQ_INSERT_BEFORE(temp_ih, ih, ih_next); intr_event_update(ie); CTR3(KTR_INTR, "%s: added %s to %s", __func__, ih->ih_name, ie->ie_name); mtx_unlock(&ie->ie_lock); if (cookiep != NULL) *cookiep = ih; return (0); } #endif /* * Append a description preceded by a ':' to the name of the specified * interrupt handler. */ int intr_event_describe_handler(struct intr_event *ie, void *cookie, const char *descr) { struct intr_handler *ih; size_t space; char *start; mtx_lock(&ie->ie_lock); #ifdef INVARIANTS TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { if (ih == cookie) break; } if (ih == NULL) { mtx_unlock(&ie->ie_lock); panic("handler %p not found in interrupt event %p", cookie, ie); } #endif ih = cookie; /* * Look for an existing description by checking for an * existing ":". This assumes device names do not include * colons. If one is found, prepare to insert the new * description at that point. If one is not found, find the * end of the name to use as the insertion point. */ start = strchr(ih->ih_name, ':'); if (start == NULL) start = strchr(ih->ih_name, 0); /* * See if there is enough remaining room in the string for the * description + ":". The "- 1" leaves room for the trailing * '\0'. The "+ 1" accounts for the colon. */ space = sizeof(ih->ih_name) - (start - ih->ih_name) - 1; if (strlen(descr) + 1 > space) { mtx_unlock(&ie->ie_lock); return (ENOSPC); } /* Append a colon followed by the description. */ *start = ':'; strcpy(start + 1, descr); intr_event_update(ie); mtx_unlock(&ie->ie_lock); return (0); } /* * Return the ie_source field from the intr_event an intr_handler is * associated with. */ void * intr_handler_source(void *cookie) { struct intr_handler *ih; struct intr_event *ie; ih = (struct intr_handler *)cookie; if (ih == NULL) return (NULL); ie = ih->ih_event; KASSERT(ie != NULL, ("interrupt handler \"%s\" has a NULL interrupt event", ih->ih_name)); return (ie->ie_source); } /* * Sleep until an ithread finishes executing an interrupt handler. * * XXX Doesn't currently handle interrupt filters or fast interrupt * handlers. This is intended for compatibility with linux drivers * only. Do not use in BSD code. */ void _intr_drain(int irq) { struct intr_event *ie; struct intr_thread *ithd; struct thread *td; ie = intr_lookup(irq); if (ie == NULL) return; if (ie->ie_thread == NULL) return; ithd = ie->ie_thread; td = ithd->it_thread; /* * We set the flag and wait for it to be cleared to avoid * long delays with potentially busy interrupt handlers * were we to only sample TD_AWAITING_INTR() every tick. */ thread_lock(td); if (!TD_AWAITING_INTR(td)) { ithd->it_flags |= IT_WAIT; while (ithd->it_flags & IT_WAIT) { thread_unlock(td); pause("idrain", 1); thread_lock(td); } } thread_unlock(td); return; } #ifndef INTR_FILTER int intr_event_remove_handler(void *cookie) { struct intr_handler *handler = (struct intr_handler *)cookie; struct intr_event *ie; #ifdef INVARIANTS struct intr_handler *ih; #endif #ifdef notyet int dead; #endif if (handler == NULL) return (EINVAL); ie = handler->ih_event; KASSERT(ie != NULL, ("interrupt handler \"%s\" has a NULL interrupt event", handler->ih_name)); mtx_lock(&ie->ie_lock); CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, ie->ie_name); #ifdef INVARIANTS TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) if (ih == handler) goto ok; mtx_unlock(&ie->ie_lock); panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", ih->ih_name, ie->ie_name); ok: #endif /* * If there is no ithread, then just remove the handler and return. * XXX: Note that an INTR_FAST handler might be running on another * CPU! */ if (ie->ie_thread == NULL) { TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); mtx_unlock(&ie->ie_lock); free(handler, M_ITHREAD); return (0); } /* * If the interrupt thread is already running, then just mark this * handler as being dead and let the ithread do the actual removal. * * During a cold boot while cold is set, msleep() does not sleep, * so we have to remove the handler here rather than letting the * thread do it. */ thread_lock(ie->ie_thread->it_thread); if (!TD_AWAITING_INTR(ie->ie_thread->it_thread) && !cold) { handler->ih_flags |= IH_DEAD; /* * Ensure that the thread will process the handler list * again and remove this handler if it has already passed * it on the list. */ atomic_store_rel_int(&ie->ie_thread->it_need, 1); } else TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); thread_unlock(ie->ie_thread->it_thread); while (handler->ih_flags & IH_DEAD) msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); intr_event_update(ie); #ifdef notyet /* * XXX: This could be bad in the case of ppbus(8). Also, I think * this could lead to races of stale data when servicing an * interrupt. */ dead = 1; TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { if (!(ih->ih_flags & IH_FAST)) { dead = 0; break; } } if (dead) { ithread_destroy(ie->ie_thread); ie->ie_thread = NULL; } #endif mtx_unlock(&ie->ie_lock); free(handler, M_ITHREAD); return (0); } static int intr_event_schedule_thread(struct intr_event *ie) { struct intr_entropy entropy; struct intr_thread *it; struct thread *td; struct thread *ctd; struct proc *p; /* * If no ithread or no handlers, then we have a stray interrupt. */ if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || ie->ie_thread == NULL) return (EINVAL); ctd = curthread; it = ie->ie_thread; td = it->it_thread; p = td->td_proc; /* * If any of the handlers for this ithread claim to be good * sources of entropy, then gather some. */ if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, p->p_pid, td->td_name); entropy.event = (uintptr_t)ie; entropy.td = ctd; random_harvest(&entropy, sizeof(entropy), 2, RANDOM_INTERRUPT); } KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); /* * Set it_need to tell the thread to keep running if it is already * running. Then, lock the thread and see if we actually need to * put it on the runqueue. */ atomic_store_rel_int(&it->it_need, 1); thread_lock(td); if (TD_AWAITING_INTR(td)) { CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, td->td_name); TD_CLR_IWAIT(td); sched_add(td, SRQ_INTR); } else { CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", __func__, p->p_pid, td->td_name, it->it_need, td->td_state); } thread_unlock(td); return (0); } #else int intr_event_remove_handler(void *cookie) { struct intr_handler *handler = (struct intr_handler *)cookie; struct intr_event *ie; struct intr_thread *it; #ifdef INVARIANTS struct intr_handler *ih; #endif #ifdef notyet int dead; #endif if (handler == NULL) return (EINVAL); ie = handler->ih_event; KASSERT(ie != NULL, ("interrupt handler \"%s\" has a NULL interrupt event", handler->ih_name)); mtx_lock(&ie->ie_lock); CTR3(KTR_INTR, "%s: removing %s from %s", __func__, handler->ih_name, ie->ie_name); #ifdef INVARIANTS TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) if (ih == handler) goto ok; mtx_unlock(&ie->ie_lock); panic("interrupt handler \"%s\" not found in interrupt event \"%s\"", ih->ih_name, ie->ie_name); ok: #endif /* * If there are no ithreads (per event and per handler), then * just remove the handler and return. * XXX: Note that an INTR_FAST handler might be running on another CPU! */ if (ie->ie_thread == NULL && handler->ih_thread == NULL) { TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); mtx_unlock(&ie->ie_lock); free(handler, M_ITHREAD); return (0); } /* Private or global ithread? */ it = (handler->ih_thread) ? handler->ih_thread : ie->ie_thread; /* * If the interrupt thread is already running, then just mark this * handler as being dead and let the ithread do the actual removal. * * During a cold boot while cold is set, msleep() does not sleep, * so we have to remove the handler here rather than letting the * thread do it. */ thread_lock(it->it_thread); if (!TD_AWAITING_INTR(it->it_thread) && !cold) { handler->ih_flags |= IH_DEAD; /* * Ensure that the thread will process the handler list * again and remove this handler if it has already passed * it on the list. */ atomic_store_rel_int(&it->it_need, 1); } else TAILQ_REMOVE(&ie->ie_handlers, handler, ih_next); thread_unlock(it->it_thread); while (handler->ih_flags & IH_DEAD) msleep(handler, &ie->ie_lock, 0, "iev_rmh", 0); /* * At this point, the handler has been disconnected from the event, * so we can kill the private ithread if any. */ if (handler->ih_thread) { ithread_destroy(handler->ih_thread); handler->ih_thread = NULL; } intr_event_update(ie); #ifdef notyet /* * XXX: This could be bad in the case of ppbus(8). Also, I think * this could lead to races of stale data when servicing an * interrupt. */ dead = 1; TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { if (handler != NULL) { dead = 0; break; } } if (dead) { ithread_destroy(ie->ie_thread); ie->ie_thread = NULL; } #endif mtx_unlock(&ie->ie_lock); free(handler, M_ITHREAD); return (0); } static int intr_event_schedule_thread(struct intr_event *ie, struct intr_thread *it) { struct intr_entropy entropy; struct thread *td; struct thread *ctd; struct proc *p; /* * If no ithread or no handlers, then we have a stray interrupt. */ if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers) || it == NULL) return (EINVAL); ctd = curthread; td = it->it_thread; p = td->td_proc; /* * If any of the handlers for this ithread claim to be good * sources of entropy, then gather some. */ if (harvest.interrupt && ie->ie_flags & IE_ENTROPY) { CTR3(KTR_INTR, "%s: pid %d (%s) gathering entropy", __func__, p->p_pid, td->td_name); entropy.event = (uintptr_t)ie; entropy.td = ctd; random_harvest(&entropy, sizeof(entropy), 2, RANDOM_INTERRUPT); } KASSERT(p != NULL, ("ithread %s has no process", ie->ie_name)); /* * Set it_need to tell the thread to keep running if it is already * running. Then, lock the thread and see if we actually need to * put it on the runqueue. */ atomic_store_rel_int(&it->it_need, 1); thread_lock(td); if (TD_AWAITING_INTR(td)) { CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, td->td_name); TD_CLR_IWAIT(td); sched_add(td, SRQ_INTR); } else { CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", __func__, p->p_pid, td->td_name, it->it_need, td->td_state); } thread_unlock(td); return (0); } #endif /* * Allow interrupt event binding for software interrupt handlers -- a no-op, * since interrupts are generated in software rather than being directed by * a PIC. */ static int -swi_assign_cpu(void *arg, u_char cpu) +swi_assign_cpu(void *arg, int cpu) { return (0); } /* * Add a software interrupt handler to a specified event. If a given event * is not specified, then a new event is created. */ int swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, void *arg, int pri, enum intr_type flags, void **cookiep) { struct intr_event *ie; int error; if (flags & INTR_ENTROPY) return (EINVAL); ie = (eventp != NULL) ? *eventp : NULL; if (ie != NULL) { if (!(ie->ie_flags & IE_SOFT)) return (EINVAL); } else { error = intr_event_create(&ie, NULL, IE_SOFT, 0, NULL, NULL, NULL, swi_assign_cpu, "swi%d:", pri); if (error) return (error); if (eventp != NULL) *eventp = ie; } error = intr_event_add_handler(ie, name, NULL, handler, arg, PI_SWI(pri), flags, cookiep); return (error); } /* * Schedule a software interrupt thread. */ void swi_sched(void *cookie, int flags) { struct intr_handler *ih = (struct intr_handler *)cookie; struct intr_event *ie = ih->ih_event; struct intr_entropy entropy; int error; CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name, ih->ih_need); if (harvest.swi) { CTR2(KTR_INTR, "swi_sched: pid %d (%s) gathering entropy", curproc->p_pid, curthread->td_name); entropy.event = (uintptr_t)ih; entropy.td = curthread; random_harvest(&entropy, sizeof(entropy), 1, RANDOM_SWI); } /* * Set ih_need for this handler so that if the ithread is already * running it will execute this handler on the next pass. Otherwise, * it will execute it the next time it runs. */ atomic_store_rel_int(&ih->ih_need, 1); if (!(flags & SWI_DELAY)) { PCPU_INC(cnt.v_soft); #ifdef INTR_FILTER error = intr_event_schedule_thread(ie, ie->ie_thread); #else error = intr_event_schedule_thread(ie); #endif KASSERT(error == 0, ("stray software interrupt")); } } /* * Remove a software interrupt handler. Currently this code does not * remove the associated interrupt event if it becomes empty. Calling code * may do so manually via intr_event_destroy(), but that's not really * an optimal interface. */ int swi_remove(void *cookie) { return (intr_event_remove_handler(cookie)); } #ifdef INTR_FILTER static void priv_ithread_execute_handler(struct proc *p, struct intr_handler *ih) { struct intr_event *ie; ie = ih->ih_event; /* * If this handler is marked for death, remove it from * the list of handlers and wake up the sleeper. */ if (ih->ih_flags & IH_DEAD) { mtx_lock(&ie->ie_lock); TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); ih->ih_flags &= ~IH_DEAD; wakeup(ih); mtx_unlock(&ie->ie_lock); return; } /* Execute this handler. */ CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument, ih->ih_name, ih->ih_flags); if (!(ih->ih_flags & IH_MPSAFE)) mtx_lock(&Giant); ih->ih_handler(ih->ih_argument); if (!(ih->ih_flags & IH_MPSAFE)) mtx_unlock(&Giant); } #endif /* * This is a public function for use by drivers that mux interrupt * handlers for child devices from their interrupt handler. */ void intr_event_execute_handlers(struct proc *p, struct intr_event *ie) { struct intr_handler *ih, *ihn; TAILQ_FOREACH_SAFE(ih, &ie->ie_handlers, ih_next, ihn) { /* * If this handler is marked for death, remove it from * the list of handlers and wake up the sleeper. */ if (ih->ih_flags & IH_DEAD) { mtx_lock(&ie->ie_lock); TAILQ_REMOVE(&ie->ie_handlers, ih, ih_next); ih->ih_flags &= ~IH_DEAD; wakeup(ih); mtx_unlock(&ie->ie_lock); continue; } /* Skip filter only handlers */ if (ih->ih_handler == NULL) continue; /* * For software interrupt threads, we only execute * handlers that have their need flag set. Hardware * interrupt threads always invoke all of their handlers. */ if (ie->ie_flags & IE_SOFT) { if (atomic_load_acq_int(&ih->ih_need) == 0) continue; else atomic_store_rel_int(&ih->ih_need, 0); } /* Execute this handler. */ CTR6(KTR_INTR, "%s: pid %d exec %p(%p) for %s flg=%x", __func__, p->p_pid, (void *)ih->ih_handler, ih->ih_argument, ih->ih_name, ih->ih_flags); if (!(ih->ih_flags & IH_MPSAFE)) mtx_lock(&Giant); ih->ih_handler(ih->ih_argument); if (!(ih->ih_flags & IH_MPSAFE)) mtx_unlock(&Giant); } } static void ithread_execute_handlers(struct proc *p, struct intr_event *ie) { /* Interrupt handlers should not sleep. */ if (!(ie->ie_flags & IE_SOFT)) THREAD_NO_SLEEPING(); intr_event_execute_handlers(p, ie); if (!(ie->ie_flags & IE_SOFT)) THREAD_SLEEPING_OK(); /* * Interrupt storm handling: * * If this interrupt source is currently storming, then throttle * it to only fire the handler once per clock tick. * * If this interrupt source is not currently storming, but the * number of back to back interrupts exceeds the storm threshold, * then enter storming mode. */ if (intr_storm_threshold != 0 && ie->ie_count >= intr_storm_threshold && !(ie->ie_flags & IE_SOFT)) { /* Report the message only once every second. */ if (ppsratecheck(&ie->ie_warntm, &ie->ie_warncnt, 1)) { printf( "interrupt storm detected on \"%s\"; throttling interrupt source\n", ie->ie_name); } pause("istorm", 1); } else ie->ie_count++; /* * Now that all the handlers have had a chance to run, reenable * the interrupt source. */ if (ie->ie_post_ithread != NULL) ie->ie_post_ithread(ie->ie_source); } #ifndef INTR_FILTER /* * This is the main code for interrupt threads. */ static void ithread_loop(void *arg) { struct intr_thread *ithd; struct intr_event *ie; struct thread *td; struct proc *p; int wake; td = curthread; p = td->td_proc; ithd = (struct intr_thread *)arg; KASSERT(ithd->it_thread == td, ("%s: ithread and proc linkage out of sync", __func__)); ie = ithd->it_event; ie->ie_count = 0; wake = 0; /* * As long as we have interrupts outstanding, go through the * list of handlers, giving each one a go at it. */ for (;;) { /* * If we are an orphaned thread, then just die. */ if (ithd->it_flags & IT_DEAD) { CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, p->p_pid, td->td_name); free(ithd, M_ITHREAD); kthread_exit(); } /* * Service interrupts. If another interrupt arrives while * we are running, it will set it_need to note that we * should make another pass. */ while (atomic_load_acq_int(&ithd->it_need) != 0) { /* * This might need a full read and write barrier * to make sure that this write posts before any * of the memory or device accesses in the * handlers. */ atomic_store_rel_int(&ithd->it_need, 0); ithread_execute_handlers(p, ie); } WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); mtx_assert(&Giant, MA_NOTOWNED); /* * Processed all our interrupts. Now get the sched * lock. This may take a while and it_need may get * set again, so we have to check it again. */ thread_lock(td); if ((atomic_load_acq_int(&ithd->it_need) == 0) && !(ithd->it_flags & (IT_DEAD | IT_WAIT))) { TD_SET_IWAIT(td); ie->ie_count = 0; mi_switch(SW_VOL | SWT_IWAIT, NULL); } if (ithd->it_flags & IT_WAIT) { wake = 1; ithd->it_flags &= ~IT_WAIT; } thread_unlock(td); if (wake) { wakeup(ithd); wake = 0; } } } /* * Main interrupt handling body. * * Input: * o ie: the event connected to this interrupt. * o frame: some archs (i.e. i386) pass a frame to some. * handlers as their main argument. * Return value: * o 0: everything ok. * o EINVAL: stray interrupt. */ int intr_event_handle(struct intr_event *ie, struct trapframe *frame) { struct intr_handler *ih; struct trapframe *oldframe; struct thread *td; int error, ret, thread; td = curthread; /* An interrupt with no event or handlers is a stray interrupt. */ if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) return (EINVAL); /* * Execute fast interrupt handlers directly. * To support clock handlers, if a handler registers * with a NULL argument, then we pass it a pointer to * a trapframe as its argument. */ td->td_intr_nesting_level++; thread = 0; ret = 0; critical_enter(); oldframe = td->td_intr_frame; td->td_intr_frame = frame; TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { if (ih->ih_filter == NULL) { thread = 1; continue; } CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__, ih->ih_filter, ih->ih_argument == NULL ? frame : ih->ih_argument, ih->ih_name); if (ih->ih_argument == NULL) ret = ih->ih_filter(frame); else ret = ih->ih_filter(ih->ih_argument); KASSERT(ret == FILTER_STRAY || ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), ("%s: incorrect return value %#x from %s", __func__, ret, ih->ih_name)); /* * Wrapper handler special handling: * * in some particular cases (like pccard and pccbb), * the _real_ device handler is wrapped in a couple of * functions - a filter wrapper and an ithread wrapper. * In this case (and just in this case), the filter wrapper * could ask the system to schedule the ithread and mask * the interrupt source if the wrapped handler is composed * of just an ithread handler. * * TODO: write a generic wrapper to avoid people rolling * their own */ if (!thread) { if (ret == FILTER_SCHEDULE_THREAD) thread = 1; } } td->td_intr_frame = oldframe; if (thread) { if (ie->ie_pre_ithread != NULL) ie->ie_pre_ithread(ie->ie_source); } else { if (ie->ie_post_filter != NULL) ie->ie_post_filter(ie->ie_source); } /* Schedule the ithread if needed. */ if (thread) { error = intr_event_schedule_thread(ie); #ifndef XEN KASSERT(error == 0, ("bad stray interrupt")); #else if (error != 0) log(LOG_WARNING, "bad stray interrupt"); #endif } critical_exit(); td->td_intr_nesting_level--; return (0); } #else /* * This is the main code for interrupt threads. */ static void ithread_loop(void *arg) { struct intr_thread *ithd; struct intr_handler *ih; struct intr_event *ie; struct thread *td; struct proc *p; int priv; int wake; td = curthread; p = td->td_proc; ih = (struct intr_handler *)arg; priv = (ih->ih_thread != NULL) ? 1 : 0; ithd = (priv) ? ih->ih_thread : ih->ih_event->ie_thread; KASSERT(ithd->it_thread == td, ("%s: ithread and proc linkage out of sync", __func__)); ie = ithd->it_event; ie->ie_count = 0; wake = 0; /* * As long as we have interrupts outstanding, go through the * list of handlers, giving each one a go at it. */ for (;;) { /* * If we are an orphaned thread, then just die. */ if (ithd->it_flags & IT_DEAD) { CTR3(KTR_INTR, "%s: pid %d (%s) exiting", __func__, p->p_pid, td->td_name); free(ithd, M_ITHREAD); kthread_exit(); } /* * Service interrupts. If another interrupt arrives while * we are running, it will set it_need to note that we * should make another pass. */ while (atomic_load_acq_int(&ithd->it_need) != 0) { /* * This might need a full read and write barrier * to make sure that this write posts before any * of the memory or device accesses in the * handlers. */ atomic_store_rel_int(&ithd->it_need, 0); if (priv) priv_ithread_execute_handler(p, ih); else ithread_execute_handlers(p, ie); } WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread"); mtx_assert(&Giant, MA_NOTOWNED); /* * Processed all our interrupts. Now get the sched * lock. This may take a while and it_need may get * set again, so we have to check it again. */ thread_lock(td); if ((atomic_load_acq_int(&ithd->it_need) == 0) && !(ithd->it_flags & (IT_DEAD | IT_WAIT))) { TD_SET_IWAIT(td); ie->ie_count = 0; mi_switch(SW_VOL | SWT_IWAIT, NULL); } if (ithd->it_flags & IT_WAIT) { wake = 1; ithd->it_flags &= ~IT_WAIT; } thread_unlock(td); if (wake) { wakeup(ithd); wake = 0; } } } /* * Main loop for interrupt filter. * * Some architectures (i386, amd64 and arm) require the optional frame * parameter, and use it as the main argument for fast handler execution * when ih_argument == NULL. * * Return value: * o FILTER_STRAY: No filter recognized the event, and no * filter-less handler is registered on this * line. * o FILTER_HANDLED: A filter claimed the event and served it. * o FILTER_SCHEDULE_THREAD: No filter claimed the event, but there's at * least one filter-less handler on this line. * o FILTER_HANDLED | * FILTER_SCHEDULE_THREAD: A filter claimed the event, and asked for * scheduling the per-handler ithread. * * In case an ithread has to be scheduled, in *ithd there will be a * pointer to a struct intr_thread containing the thread to be * scheduled. */ static int intr_filter_loop(struct intr_event *ie, struct trapframe *frame, struct intr_thread **ithd) { struct intr_handler *ih; void *arg; int ret, thread_only; ret = 0; thread_only = 0; TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { /* * Execute fast interrupt handlers directly. * To support clock handlers, if a handler registers * with a NULL argument, then we pass it a pointer to * a trapframe as its argument. */ arg = ((ih->ih_argument == NULL) ? frame : ih->ih_argument); CTR5(KTR_INTR, "%s: exec %p/%p(%p) for %s", __func__, ih->ih_filter, ih->ih_handler, arg, ih->ih_name); if (ih->ih_filter != NULL) ret = ih->ih_filter(arg); else { thread_only = 1; continue; } KASSERT(ret == FILTER_STRAY || ((ret & (FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) != 0 && (ret & ~(FILTER_SCHEDULE_THREAD | FILTER_HANDLED)) == 0), ("%s: incorrect return value %#x from %s", __func__, ret, ih->ih_name)); if (ret & FILTER_STRAY) continue; else { *ithd = ih->ih_thread; return (ret); } } /* * No filters handled the interrupt and we have at least * one handler without a filter. In this case, we schedule * all of the filter-less handlers to run in the ithread. */ if (thread_only) { *ithd = ie->ie_thread; return (FILTER_SCHEDULE_THREAD); } return (FILTER_STRAY); } /* * Main interrupt handling body. * * Input: * o ie: the event connected to this interrupt. * o frame: some archs (i.e. i386) pass a frame to some. * handlers as their main argument. * Return value: * o 0: everything ok. * o EINVAL: stray interrupt. */ int intr_event_handle(struct intr_event *ie, struct trapframe *frame) { struct intr_thread *ithd; struct trapframe *oldframe; struct thread *td; int thread; ithd = NULL; td = curthread; if (ie == NULL || TAILQ_EMPTY(&ie->ie_handlers)) return (EINVAL); td->td_intr_nesting_level++; thread = 0; critical_enter(); oldframe = td->td_intr_frame; td->td_intr_frame = frame; thread = intr_filter_loop(ie, frame, &ithd); if (thread & FILTER_HANDLED) { if (ie->ie_post_filter != NULL) ie->ie_post_filter(ie->ie_source); } else { if (ie->ie_pre_ithread != NULL) ie->ie_pre_ithread(ie->ie_source); } td->td_intr_frame = oldframe; critical_exit(); /* Interrupt storm logic */ if (thread & FILTER_STRAY) { ie->ie_count++; if (ie->ie_count < intr_storm_threshold) printf("Interrupt stray detection not present\n"); } /* Schedule an ithread if needed. */ if (thread & FILTER_SCHEDULE_THREAD) { if (intr_event_schedule_thread(ie, ithd) != 0) panic("%s: impossible stray interrupt", __func__); } td->td_intr_nesting_level--; return (0); } #endif #ifdef DDB /* * Dump details about an interrupt handler */ static void db_dump_intrhand(struct intr_handler *ih) { int comma; db_printf("\t%-10s ", ih->ih_name); switch (ih->ih_pri) { case PI_REALTIME: db_printf("CLK "); break; case PI_AV: db_printf("AV "); break; case PI_TTY: db_printf("TTY "); break; case PI_NET: db_printf("NET "); break; case PI_DISK: db_printf("DISK"); break; case PI_DULL: db_printf("DULL"); break; default: if (ih->ih_pri >= PI_SOFT) db_printf("SWI "); else db_printf("%4u", ih->ih_pri); break; } db_printf(" "); if (ih->ih_filter != NULL) { db_printf("[F]"); db_printsym((uintptr_t)ih->ih_filter, DB_STGY_PROC); } if (ih->ih_handler != NULL) { if (ih->ih_filter != NULL) db_printf(","); db_printf("[H]"); db_printsym((uintptr_t)ih->ih_handler, DB_STGY_PROC); } db_printf("(%p)", ih->ih_argument); if (ih->ih_need || (ih->ih_flags & (IH_EXCLUSIVE | IH_ENTROPY | IH_DEAD | IH_MPSAFE)) != 0) { db_printf(" {"); comma = 0; if (ih->ih_flags & IH_EXCLUSIVE) { if (comma) db_printf(", "); db_printf("EXCL"); comma = 1; } if (ih->ih_flags & IH_ENTROPY) { if (comma) db_printf(", "); db_printf("ENTROPY"); comma = 1; } if (ih->ih_flags & IH_DEAD) { if (comma) db_printf(", "); db_printf("DEAD"); comma = 1; } if (ih->ih_flags & IH_MPSAFE) { if (comma) db_printf(", "); db_printf("MPSAFE"); comma = 1; } if (ih->ih_need) { if (comma) db_printf(", "); db_printf("NEED"); } db_printf("}"); } db_printf("\n"); } /* * Dump details about a event. */ void db_dump_intr_event(struct intr_event *ie, int handlers) { struct intr_handler *ih; struct intr_thread *it; int comma; db_printf("%s ", ie->ie_fullname); it = ie->ie_thread; if (it != NULL) db_printf("(pid %d)", it->it_thread->td_proc->p_pid); else db_printf("(no thread)"); if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 || (it != NULL && it->it_need)) { db_printf(" {"); comma = 0; if (ie->ie_flags & IE_SOFT) { db_printf("SOFT"); comma = 1; } if (ie->ie_flags & IE_ENTROPY) { if (comma) db_printf(", "); db_printf("ENTROPY"); comma = 1; } if (ie->ie_flags & IE_ADDING_THREAD) { if (comma) db_printf(", "); db_printf("ADDING_THREAD"); comma = 1; } if (it != NULL && it->it_need) { if (comma) db_printf(", "); db_printf("NEED"); } db_printf("}"); } db_printf("\n"); if (handlers) TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) db_dump_intrhand(ih); } /* * Dump data about interrupt handlers */ DB_SHOW_COMMAND(intr, db_show_intr) { struct intr_event *ie; int all, verbose; verbose = strchr(modif, 'v') != NULL; all = strchr(modif, 'a') != NULL; TAILQ_FOREACH(ie, &event_list, ie_list) { if (!all && TAILQ_EMPTY(&ie->ie_handlers)) continue; db_dump_intr_event(ie, verbose); if (db_pager_quit) break; } } #endif /* DDB */ /* * Start standard software interrupt threads */ static void start_softintr(void *dummy) { if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih)) panic("died while creating vm swi ithread"); } SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, NULL); /* * Sysctls used by systat and others: hw.intrnames and hw.intrcnt. * The data for this machine dependent, and the declarations are in machine * dependent code. The layout of intrnames and intrcnt however is machine * independent. * * We do not know the length of intrcnt and intrnames at compile time, so * calculate things at run time. */ static int sysctl_intrnames(SYSCTL_HANDLER_ARGS) { return (sysctl_handle_opaque(oidp, intrnames, sintrnames, req)); } SYSCTL_PROC(_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE | CTLFLAG_RD, NULL, 0, sysctl_intrnames, "", "Interrupt Names"); static int sysctl_intrcnt(SYSCTL_HANDLER_ARGS) { #ifdef SCTL_MASK32 uint32_t *intrcnt32; unsigned i; int error; if (req->flags & SCTL_MASK32) { if (!req->oldptr) return (sysctl_handle_opaque(oidp, NULL, sintrcnt / 2, req)); intrcnt32 = malloc(sintrcnt / 2, M_TEMP, M_NOWAIT); if (intrcnt32 == NULL) return (ENOMEM); for (i = 0; i < sintrcnt / sizeof (u_long); i++) intrcnt32[i] = intrcnt[i]; error = sysctl_handle_opaque(oidp, intrcnt32, sintrcnt / 2, req); free(intrcnt32, M_TEMP); return (error); } #endif return (sysctl_handle_opaque(oidp, intrcnt, sintrcnt, req)); } SYSCTL_PROC(_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE | CTLFLAG_RD, NULL, 0, sysctl_intrcnt, "", "Interrupt Counts"); #ifdef DDB /* * DDB command to dump the interrupt statistics. */ DB_SHOW_COMMAND(intrcnt, db_show_intrcnt) { u_long *i; char *cp; u_int j; cp = intrnames; j = 0; for (i = intrcnt; j < (sintrcnt / sizeof(u_long)) && !db_pager_quit; i++, j++) { if (*cp == '\0') break; if (*i != 0) db_printf("%s\t%lu\n", cp, *i); cp += strlen(cp) + 1; } } #endif Index: head/sys/mips/cavium/ciu.c =================================================================== --- head/sys/mips/cavium/ciu.c (revision 271711) +++ head/sys/mips/cavium/ciu.c (revision 271712) @@ -1,486 +1,486 @@ /*- * Copyright (c) 2010 Juli Mallett * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include /* * This bus sits between devices/buses and nexus and handles CIU interrupts * and passes everything else through. It should really be a nexus subclass * or something, but for now this will be sufficient. */ #define CIU_IRQ_HARD (0) #define CIU_IRQ_EN0_BEGIN OCTEON_IRQ_WORKQ0 #define CIU_IRQ_EN0_END OCTEON_IRQ_BOOTDMA #define CIU_IRQ_EN0_COUNT ((CIU_IRQ_EN0_END - CIU_IRQ_EN0_BEGIN) + 1) #define CIU_IRQ_EN1_BEGIN OCTEON_IRQ_WDOG0 #define CIU_IRQ_EN1_END OCTEON_IRQ_DFM #define CIU_IRQ_EN1_COUNT ((CIU_IRQ_EN1_END - CIU_IRQ_EN1_BEGIN) + 1) struct ciu_softc { struct rman irq_rman; struct resource *ciu_irq; }; static mips_intrcnt_t ciu_en0_intrcnt[CIU_IRQ_EN0_COUNT]; static mips_intrcnt_t ciu_en1_intrcnt[CIU_IRQ_EN1_COUNT]; static struct intr_event *ciu_en0_intr_events[CIU_IRQ_EN0_COUNT]; static struct intr_event *ciu_en1_intr_events[CIU_IRQ_EN1_COUNT]; static int ciu_probe(device_t); static int ciu_attach(device_t); static struct resource *ciu_alloc_resource(device_t, device_t, int, int *, u_long, u_long, u_long, u_int); static int ciu_setup_intr(device_t, device_t, struct resource *, int, driver_filter_t *, driver_intr_t *, void *, void **); static int ciu_teardown_intr(device_t, device_t, struct resource *, void *); static int ciu_bind_intr(device_t, device_t, struct resource *, int); static int ciu_describe_intr(device_t, device_t, struct resource *, void *, const char *); static void ciu_hinted_child(device_t, const char *, int); static void ciu_en0_intr_mask(void *); static void ciu_en0_intr_unmask(void *); #ifdef SMP -static int ciu_en0_intr_bind(void *, u_char); +static int ciu_en0_intr_bind(void *, int); #endif static void ciu_en1_intr_mask(void *); static void ciu_en1_intr_unmask(void *); #ifdef SMP -static int ciu_en1_intr_bind(void *, u_char); +static int ciu_en1_intr_bind(void *, int); #endif static int ciu_intr(void *); static int ciu_probe(device_t dev) { if (device_get_unit(dev) != 0) return (ENXIO); device_set_desc(dev, "Cavium Octeon Central Interrupt Unit"); return (BUS_PROBE_NOWILDCARD); } static int ciu_attach(device_t dev) { char name[MAXCOMLEN + 1]; struct ciu_softc *sc; unsigned i; int error; int rid; sc = device_get_softc(dev); rid = 0; sc->ciu_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, CIU_IRQ_HARD, CIU_IRQ_HARD, 1, RF_ACTIVE); if (sc->ciu_irq == NULL) { device_printf(dev, "could not allocate irq%d\n", CIU_IRQ_HARD); return (ENXIO); } error = bus_setup_intr(dev, sc->ciu_irq, INTR_TYPE_MISC, ciu_intr, NULL, sc, NULL); if (error != 0) { device_printf(dev, "bus_setup_intr failed: %d\n", error); return (error); } sc->irq_rman.rm_type = RMAN_ARRAY; sc->irq_rman.rm_descr = "CIU IRQ"; error = rman_init(&sc->irq_rman); if (error != 0) return (error); /* * We have two contiguous IRQ regions, use a single rman. */ error = rman_manage_region(&sc->irq_rman, CIU_IRQ_EN0_BEGIN, CIU_IRQ_EN1_END); if (error != 0) return (error); for (i = 0; i < CIU_IRQ_EN0_COUNT; i++) { snprintf(name, sizeof name, "int%d:", i + CIU_IRQ_EN0_BEGIN); ciu_en0_intrcnt[i] = mips_intrcnt_create(name); } for (i = 0; i < CIU_IRQ_EN1_COUNT; i++) { snprintf(name, sizeof name, "int%d:", i + CIU_IRQ_EN1_BEGIN); ciu_en1_intrcnt[i] = mips_intrcnt_create(name); } bus_generic_probe(dev); bus_generic_attach(dev); return (0); } static struct resource * ciu_alloc_resource(device_t bus, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags) { struct resource *res; struct ciu_softc *sc; sc = device_get_softc(bus); switch (type) { case SYS_RES_IRQ: break; default: return (bus_alloc_resource(device_get_parent(bus), type, rid, start, end, count, flags)); } /* * One interrupt at a time for now. */ if (start != end) return (NULL); res = rman_reserve_resource(&sc->irq_rman, start, end, count, flags, child); if (res != NULL) return (res); return (NULL); } static int ciu_setup_intr(device_t bus, device_t child, struct resource *res, int flags, driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep) { struct intr_event *event, **eventp; void (*mask_func)(void *); void (*unmask_func)(void *); - int (*bind_func)(void *, u_char); + int (*bind_func)(void *, int); mips_intrcnt_t intrcnt; int error; int irq; irq = rman_get_start(res); if (irq <= CIU_IRQ_EN0_END) { eventp = &ciu_en0_intr_events[irq - CIU_IRQ_EN0_BEGIN]; intrcnt = ciu_en0_intrcnt[irq - CIU_IRQ_EN0_BEGIN]; mask_func = ciu_en0_intr_mask; unmask_func = ciu_en0_intr_unmask; #ifdef SMP bind_func = ciu_en0_intr_bind; #endif } else { eventp = &ciu_en1_intr_events[irq - CIU_IRQ_EN1_BEGIN]; intrcnt = ciu_en1_intrcnt[irq - CIU_IRQ_EN1_BEGIN]; mask_func = ciu_en1_intr_mask; unmask_func = ciu_en1_intr_unmask; #ifdef SMP bind_func = ciu_en1_intr_bind; #endif } #if !defined(SMP) bind_func = NULL; #endif if ((event = *eventp) == NULL) { error = intr_event_create(eventp, (void *)(uintptr_t)irq, 0, irq, mask_func, unmask_func, NULL, bind_func, "int%d", irq); if (error != 0) return (error); event = *eventp; unmask_func((void *)(uintptr_t)irq); } intr_event_add_handler(event, device_get_nameunit(child), filter, intr, arg, intr_priority(flags), flags, cookiep); mips_intrcnt_setname(intrcnt, event->ie_fullname); return (0); } static int ciu_teardown_intr(device_t bus, device_t child, struct resource *res, void *cookie) { int error; error = intr_event_remove_handler(cookie); if (error != 0) return (error); return (0); } #ifdef SMP static int ciu_bind_intr(device_t bus, device_t child, struct resource *res, int cpu) { struct intr_event *event; int irq; irq = rman_get_start(res); if (irq <= CIU_IRQ_EN0_END) event = ciu_en0_intr_events[irq - CIU_IRQ_EN0_BEGIN]; else event = ciu_en1_intr_events[irq - CIU_IRQ_EN1_BEGIN]; return (intr_event_bind(event, cpu)); } #endif static int ciu_describe_intr(device_t bus, device_t child, struct resource *res, void *cookie, const char *descr) { struct intr_event *event; mips_intrcnt_t intrcnt; int error; int irq; irq = rman_get_start(res); if (irq <= CIU_IRQ_EN0_END) { event = ciu_en0_intr_events[irq - CIU_IRQ_EN0_BEGIN]; intrcnt = ciu_en0_intrcnt[irq - CIU_IRQ_EN0_BEGIN]; } else { event = ciu_en1_intr_events[irq - CIU_IRQ_EN1_BEGIN]; intrcnt = ciu_en1_intrcnt[irq - CIU_IRQ_EN1_BEGIN]; } error = intr_event_describe_handler(event, cookie, descr); if (error != 0) return (error); mips_intrcnt_setname(intrcnt, event->ie_fullname); return (0); } static void ciu_hinted_child(device_t bus, const char *dname, int dunit) { BUS_ADD_CHILD(bus, 0, dname, dunit); } static void ciu_en0_intr_mask(void *arg) { uint64_t mask; int irq; irq = (uintptr_t)arg; mask = cvmx_read_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num()*2)); mask &= ~(1ull << (irq - CIU_IRQ_EN0_BEGIN)); cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num()*2), mask); } static void ciu_en0_intr_unmask(void *arg) { uint64_t mask; int irq; irq = (uintptr_t)arg; mask = cvmx_read_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num()*2)); mask |= 1ull << (irq - CIU_IRQ_EN0_BEGIN); cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num()*2), mask); } #ifdef SMP static int -ciu_en0_intr_bind(void *arg, u_char target) +ciu_en0_intr_bind(void *arg, int target) { uint64_t mask; int core; int irq; irq = (uintptr_t)arg; CPU_FOREACH(core) { mask = cvmx_read_csr(CVMX_CIU_INTX_EN0(core*2)); if (core == target) mask |= 1ull << (irq - CIU_IRQ_EN0_BEGIN); else mask &= ~(1ull << (irq - CIU_IRQ_EN0_BEGIN)); cvmx_write_csr(CVMX_CIU_INTX_EN0(core*2), mask); } return (0); } #endif static void ciu_en1_intr_mask(void *arg) { uint64_t mask; int irq; irq = (uintptr_t)arg; mask = cvmx_read_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num()*2)); mask &= ~(1ull << (irq - CIU_IRQ_EN1_BEGIN)); cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num()*2), mask); } static void ciu_en1_intr_unmask(void *arg) { uint64_t mask; int irq; irq = (uintptr_t)arg; mask = cvmx_read_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num()*2)); mask |= 1ull << (irq - CIU_IRQ_EN1_BEGIN); cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num()*2), mask); } #ifdef SMP static int -ciu_en1_intr_bind(void *arg, u_char target) +ciu_en1_intr_bind(void *arg, int target) { uint64_t mask; int core; int irq; irq = (uintptr_t)arg; CPU_FOREACH(core) { mask = cvmx_read_csr(CVMX_CIU_INTX_EN1(core*2)); if (core == target) mask |= 1ull << (irq - CIU_IRQ_EN1_BEGIN); else mask &= ~(1ull << (irq - CIU_IRQ_EN1_BEGIN)); cvmx_write_csr(CVMX_CIU_INTX_EN1(core*2), mask); } return (0); } #endif static int ciu_intr(void *arg) { struct ciu_softc *sc; uint64_t en0_sum, en1_sum; uint64_t en0_mask, en1_mask; int irq_index; int error; sc = arg; (void)sc; en0_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(cvmx_get_core_num()*2)); en1_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1); en0_mask = cvmx_read_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num()*2)); en1_mask = cvmx_read_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num()*2)); en0_sum &= en0_mask; en1_sum &= en1_mask; if (en0_sum == 0 && en1_sum == 0) return (FILTER_STRAY); for (irq_index = 0; en0_sum != 0; irq_index++, en0_sum >>= 1) { if ((en0_sum & 1) == 0) continue; mips_intrcnt_inc(ciu_en0_intrcnt[irq_index]); error = intr_event_handle(ciu_en0_intr_events[irq_index], NULL); if (error != 0) printf("%s: stray en0 irq%d\n", __func__, irq_index); } for (irq_index = 0; en1_sum != 0; irq_index++, en1_sum >>= 1) { if ((en1_sum & 1) == 0) continue; mips_intrcnt_inc(ciu_en1_intrcnt[irq_index]); error = intr_event_handle(ciu_en1_intr_events[irq_index], NULL); if (error != 0) printf("%s: stray en1 irq%d\n", __func__, irq_index); } return (FILTER_HANDLED); } static device_method_t ciu_methods[] = { DEVMETHOD(device_probe, ciu_probe), DEVMETHOD(device_attach, ciu_attach), DEVMETHOD(bus_alloc_resource, ciu_alloc_resource), DEVMETHOD(bus_activate_resource,bus_generic_activate_resource), DEVMETHOD(bus_setup_intr, ciu_setup_intr), DEVMETHOD(bus_teardown_intr, ciu_teardown_intr), #ifdef SMP DEVMETHOD(bus_bind_intr, ciu_bind_intr), #endif DEVMETHOD(bus_describe_intr, ciu_describe_intr), DEVMETHOD(bus_add_child, bus_generic_add_child), DEVMETHOD(bus_hinted_child, ciu_hinted_child), { 0, 0 } }; static driver_t ciu_driver = { "ciu", ciu_methods, sizeof(struct ciu_softc), }; static devclass_t ciu_devclass; DRIVER_MODULE(ciu, nexus, ciu_driver, ciu_devclass, 0, 0); Index: head/sys/powerpc/powerpc/intr_machdep.c =================================================================== --- head/sys/powerpc/powerpc/intr_machdep.c (revision 271711) +++ head/sys/powerpc/powerpc/intr_machdep.c (revision 271712) @@ -1,606 +1,606 @@ /*- * Copyright (c) 1991 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /*- * Copyright (c) 2002 Benno Rice. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 * form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20 * * $FreeBSD$ */ #include "opt_isa.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" #define MAX_STRAY_LOG 5 static MALLOC_DEFINE(M_INTR, "intr", "interrupt handler data"); struct powerpc_intr { struct intr_event *event; long *cntp; u_int irq; device_t pic; u_int intline; u_int vector; u_int cntindex; cpuset_t cpu; enum intr_trigger trig; enum intr_polarity pol; int fwcode; int ipi; }; struct pic { device_t dev; uint32_t node; u_int irqs; u_int ipis; int base; }; static u_int intrcnt_index = 0; static struct mtx intr_table_lock; static struct powerpc_intr *powerpc_intrs[INTR_VECTORS]; static struct pic piclist[MAX_PICS]; static u_int nvectors; /* Allocated vectors */ static u_int npics; /* PICs registered */ #ifdef DEV_ISA static u_int nirqs = 16; /* Allocated IRQS (ISA pre-allocated). */ #else static u_int nirqs = 0; /* Allocated IRQs. */ #endif static u_int stray_count; device_t root_pic; #ifdef SMP static void *ipi_cookie; #endif static void intr_init(void *dummy __unused) { mtx_init(&intr_table_lock, "intr sources lock", NULL, MTX_DEF); } SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); #ifdef SMP static void smp_intr_init(void *dummy __unused) { struct powerpc_intr *i; int vector; for (vector = 0; vector < nvectors; vector++) { i = powerpc_intrs[vector]; if (i != NULL && i->pic == root_pic) PIC_BIND(i->pic, i->intline, i->cpu); } } SYSINIT(smp_intr_init, SI_SUB_SMP, SI_ORDER_ANY, smp_intr_init, NULL); #endif static void intrcnt_setname(const char *name, int index) { snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", MAXCOMLEN, name); } void intrcnt_add(const char *name, u_long **countp) { int idx; idx = atomic_fetchadd_int(&intrcnt_index, 1); *countp = &intrcnt[idx]; intrcnt_setname(name, idx); } static struct powerpc_intr * intr_lookup(u_int irq) { char intrname[16]; struct powerpc_intr *i, *iscan; int vector; mtx_lock(&intr_table_lock); for (vector = 0; vector < nvectors; vector++) { i = powerpc_intrs[vector]; if (i != NULL && i->irq == irq) { mtx_unlock(&intr_table_lock); return (i); } } i = malloc(sizeof(*i), M_INTR, M_NOWAIT); if (i == NULL) { mtx_unlock(&intr_table_lock); return (NULL); } i->event = NULL; i->cntp = NULL; i->trig = INTR_TRIGGER_CONFORM; i->pol = INTR_POLARITY_CONFORM; i->irq = irq; i->pic = NULL; i->vector = -1; i->fwcode = 0; i->ipi = 0; #ifdef SMP i->cpu = all_cpus; #else CPU_SETOF(0, &i->cpu); #endif for (vector = 0; vector < INTR_VECTORS && vector <= nvectors; vector++) { iscan = powerpc_intrs[vector]; if (iscan != NULL && iscan->irq == irq) break; if (iscan == NULL && i->vector == -1) i->vector = vector; iscan = NULL; } if (iscan == NULL && i->vector != -1) { powerpc_intrs[i->vector] = i; i->cntindex = atomic_fetchadd_int(&intrcnt_index, 1); i->cntp = &intrcnt[i->cntindex]; sprintf(intrname, "irq%u:", i->irq); intrcnt_setname(intrname, i->cntindex); nvectors++; } mtx_unlock(&intr_table_lock); if (iscan != NULL || i->vector == -1) { free(i, M_INTR); i = iscan; } return (i); } static int powerpc_map_irq(struct powerpc_intr *i) { struct pic *p; u_int cnt; int idx; for (idx = 0; idx < npics; idx++) { p = &piclist[idx]; cnt = p->irqs + p->ipis; if (i->irq >= p->base && i->irq < p->base + cnt) break; } if (idx == npics) return (EINVAL); i->intline = i->irq - p->base; i->pic = p->dev; /* Try a best guess if that failed */ if (i->pic == NULL) i->pic = root_pic; return (0); } static void powerpc_intr_eoi(void *arg) { struct powerpc_intr *i = arg; PIC_EOI(i->pic, i->intline); } static void powerpc_intr_pre_ithread(void *arg) { struct powerpc_intr *i = arg; PIC_MASK(i->pic, i->intline); PIC_EOI(i->pic, i->intline); } static void powerpc_intr_post_ithread(void *arg) { struct powerpc_intr *i = arg; PIC_UNMASK(i->pic, i->intline); } static int -powerpc_assign_intr_cpu(void *arg, u_char cpu) +powerpc_assign_intr_cpu(void *arg, int cpu) { #ifdef SMP struct powerpc_intr *i = arg; if (cpu == NOCPU) i->cpu = all_cpus; else CPU_SETOF(cpu, &i->cpu); if (!cold && i->pic != NULL && i->pic == root_pic) PIC_BIND(i->pic, i->intline, i->cpu); return (0); #else return (EOPNOTSUPP); #endif } void powerpc_register_pic(device_t dev, uint32_t node, u_int irqs, u_int ipis, u_int atpic) { struct pic *p; u_int irq; int idx; mtx_lock(&intr_table_lock); /* XXX see powerpc_get_irq(). */ for (idx = 0; idx < npics; idx++) { p = &piclist[idx]; if (p->node != node) continue; if (node != 0 || p->dev == dev) break; } p = &piclist[idx]; p->dev = dev; p->node = node; p->irqs = irqs; p->ipis = ipis; if (idx == npics) { #ifdef DEV_ISA p->base = (atpic) ? 0 : nirqs; #else p->base = nirqs; #endif irq = p->base + irqs + ipis; nirqs = MAX(nirqs, irq); npics++; } mtx_unlock(&intr_table_lock); } u_int powerpc_get_irq(uint32_t node, u_int pin) { int idx; if (node == 0) return (pin); mtx_lock(&intr_table_lock); for (idx = 0; idx < npics; idx++) { if (piclist[idx].node == node) { mtx_unlock(&intr_table_lock); return (piclist[idx].base + pin); } } /* * XXX we should never encounter an unregistered PIC, but that * can only be done when we properly support bus enumeration * using multiple passes. Until then, fake an entry and give it * some adhoc maximum number of IRQs and IPIs. */ piclist[idx].dev = NULL; piclist[idx].node = node; piclist[idx].irqs = 124; piclist[idx].ipis = 4; piclist[idx].base = nirqs; nirqs += 128; npics++; mtx_unlock(&intr_table_lock); return (piclist[idx].base + pin); } int powerpc_enable_intr(void) { struct powerpc_intr *i; int error, vector; #ifdef SMP int n; #endif if (npics == 0) panic("no PIC detected\n"); if (root_pic == NULL) root_pic = piclist[0].dev; #ifdef SMP /* Install an IPI handler. */ if (mp_ncpus > 1) { for (n = 0; n < npics; n++) { if (piclist[n].dev != root_pic) continue; KASSERT(piclist[n].ipis != 0, ("%s: SMP root PIC does not supply any IPIs", __func__)); error = powerpc_setup_intr("IPI", MAP_IRQ(piclist[n].node, piclist[n].irqs), powerpc_ipi_handler, NULL, NULL, INTR_TYPE_MISC | INTR_EXCL, &ipi_cookie); if (error) { printf("unable to setup IPI handler\n"); return (error); } /* * Some subterfuge: disable late EOI and mark this * as an IPI to the dispatch layer. */ i = intr_lookup(MAP_IRQ(piclist[n].node, piclist[n].irqs)); i->event->ie_post_filter = NULL; i->ipi = 1; } } #endif for (vector = 0; vector < nvectors; vector++) { i = powerpc_intrs[vector]; if (i == NULL) continue; error = powerpc_map_irq(i); if (error) continue; if (i->trig == -1) PIC_TRANSLATE_CODE(i->pic, i->intline, i->fwcode, &i->trig, &i->pol); if (i->trig != INTR_TRIGGER_CONFORM || i->pol != INTR_POLARITY_CONFORM) PIC_CONFIG(i->pic, i->intline, i->trig, i->pol); if (i->event != NULL) PIC_ENABLE(i->pic, i->intline, vector); } return (0); } int powerpc_setup_intr(const char *name, u_int irq, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep) { struct powerpc_intr *i; int error, enable = 0; i = intr_lookup(irq); if (i == NULL) return (ENOMEM); if (i->event == NULL) { error = intr_event_create(&i->event, (void *)i, 0, irq, powerpc_intr_pre_ithread, powerpc_intr_post_ithread, powerpc_intr_eoi, powerpc_assign_intr_cpu, "irq%u:", irq); if (error) return (error); enable = 1; } error = intr_event_add_handler(i->event, name, filter, handler, arg, intr_priority(flags), flags, cookiep); mtx_lock(&intr_table_lock); intrcnt_setname(i->event->ie_fullname, i->cntindex); mtx_unlock(&intr_table_lock); if (!cold) { error = powerpc_map_irq(i); if (!error) { if (i->trig == -1) PIC_TRANSLATE_CODE(i->pic, i->intline, i->fwcode, &i->trig, &i->pol); if (i->trig != INTR_TRIGGER_CONFORM || i->pol != INTR_POLARITY_CONFORM) PIC_CONFIG(i->pic, i->intline, i->trig, i->pol); if (i->pic == root_pic) PIC_BIND(i->pic, i->intline, i->cpu); if (enable) PIC_ENABLE(i->pic, i->intline, i->vector); } } return (error); } int powerpc_teardown_intr(void *cookie) { return (intr_event_remove_handler(cookie)); } #ifdef SMP int powerpc_bind_intr(u_int irq, u_char cpu) { struct powerpc_intr *i; i = intr_lookup(irq); if (i == NULL) return (ENOMEM); return (intr_event_bind(i->event, cpu)); } #endif int powerpc_fw_config_intr(int irq, int sense_code) { struct powerpc_intr *i; i = intr_lookup(irq); if (i == NULL) return (ENOMEM); i->trig = -1; i->pol = INTR_POLARITY_CONFORM; i->fwcode = sense_code; if (!cold && i->pic != NULL) { PIC_TRANSLATE_CODE(i->pic, i->intline, i->fwcode, &i->trig, &i->pol); PIC_CONFIG(i->pic, i->intline, i->trig, i->pol); } return (0); } int powerpc_config_intr(int irq, enum intr_trigger trig, enum intr_polarity pol) { struct powerpc_intr *i; i = intr_lookup(irq); if (i == NULL) return (ENOMEM); i->trig = trig; i->pol = pol; if (!cold && i->pic != NULL) PIC_CONFIG(i->pic, i->intline, trig, pol); return (0); } void powerpc_dispatch_intr(u_int vector, struct trapframe *tf) { struct powerpc_intr *i; struct intr_event *ie; i = powerpc_intrs[vector]; if (i == NULL) goto stray; (*i->cntp)++; ie = i->event; KASSERT(ie != NULL, ("%s: interrupt without an event", __func__)); /* * IPIs are magical and need to be EOI'ed before filtering. * This prevents races in IPI handling. */ if (i->ipi) PIC_EOI(i->pic, i->intline); if (intr_event_handle(ie, tf) != 0) { goto stray; } return; stray: stray_count++; if (stray_count <= MAX_STRAY_LOG) { printf("stray irq %d\n", i ? i->irq : -1); if (stray_count >= MAX_STRAY_LOG) { printf("got %d stray interrupts, not logging anymore\n", MAX_STRAY_LOG); } } if (i != NULL) PIC_MASK(i->pic, i->intline); } Index: head/sys/sparc64/sparc64/intr_machdep.c =================================================================== --- head/sys/sparc64/sparc64/intr_machdep.c (revision 271711) +++ head/sys/sparc64/sparc64/intr_machdep.c (revision 271712) @@ -1,557 +1,557 @@ /*- * Copyright (c) 1991 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 * form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MAX_STRAY_LOG 5 CTASSERT((1 << IV_SHIFT) == sizeof(struct intr_vector)); ih_func_t *intr_handlers[PIL_MAX]; uint16_t pil_countp[PIL_MAX]; static uint16_t pil_stray_count[PIL_MAX]; struct intr_vector intr_vectors[IV_MAX]; uint16_t intr_countp[IV_MAX]; static uint16_t intr_stray_count[IV_MAX]; static const char *const pil_names[] = { "stray", "low", /* PIL_LOW */ "preempt", /* PIL_PREEMPT */ "ithrd", /* PIL_ITHREAD */ "rndzvs", /* PIL_RENDEZVOUS */ "ast", /* PIL_AST */ "hardclock", /* PIL_HARDCLOCK */ "stray", "stray", "stray", "stray", "filter", /* PIL_FILTER */ "bridge", /* PIL_BRIDGE */ "stop", /* PIL_STOP */ "tick", /* PIL_TICK */ }; /* protect the intr_vectors table */ static struct sx intr_table_lock; /* protect intrcnt_index */ static struct mtx intrcnt_lock; #ifdef SMP static int assign_cpu; static void intr_assign_next_cpu(struct intr_vector *iv); static void intr_shuffle_irqs(void *arg __unused); #endif -static int intr_assign_cpu(void *arg, u_char cpu); +static int intr_assign_cpu(void *arg, int cpu); static void intr_execute_handlers(void *); static void intr_stray_level(struct trapframe *); static void intr_stray_vector(void *); static int intrcnt_setname(const char *, int); static void intrcnt_updatename(int, const char *, int); static void intrcnt_updatename(int vec, const char *name, int ispil) { static int intrcnt_index, stray_pil_index, stray_vec_index; int name_index; mtx_lock_spin(&intrcnt_lock); if (intrnames[0] == '\0') { /* for bitbucket */ if (bootverbose) printf("initalizing intr_countp\n"); intrcnt_setname("???", intrcnt_index++); stray_vec_index = intrcnt_index++; intrcnt_setname("stray", stray_vec_index); for (name_index = 0; name_index < IV_MAX; name_index++) intr_countp[name_index] = stray_vec_index; stray_pil_index = intrcnt_index++; intrcnt_setname("pil", stray_pil_index); for (name_index = 0; name_index < PIL_MAX; name_index++) pil_countp[name_index] = stray_pil_index; } if (name == NULL) name = "???"; if (!ispil && intr_countp[vec] != stray_vec_index) name_index = intr_countp[vec]; else if (ispil && pil_countp[vec] != stray_pil_index) name_index = pil_countp[vec]; else name_index = intrcnt_index++; if (intrcnt_setname(name, name_index)) name_index = 0; if (!ispil) intr_countp[vec] = name_index; else pil_countp[vec] = name_index; mtx_unlock_spin(&intrcnt_lock); } static int intrcnt_setname(const char *name, int index) { if ((MAXCOMLEN + 1) * index >= sintrnames) return (E2BIG); snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", MAXCOMLEN, name); return (0); } void intr_setup(int pri, ih_func_t *ihf, int vec, iv_func_t *ivf, void *iva) { char pilname[MAXCOMLEN + 1]; register_t s; s = intr_disable(); if (vec != -1) { intr_vectors[vec].iv_func = ivf; intr_vectors[vec].iv_arg = iva; intr_vectors[vec].iv_pri = pri; intr_vectors[vec].iv_vec = vec; } intr_handlers[pri] = ihf; intr_restore(s); snprintf(pilname, MAXCOMLEN + 1, "pil%d: %s", pri, pil_names[pri]); intrcnt_updatename(pri, pilname, 1); } static void intr_stray_level(struct trapframe *tf) { uint64_t level; level = tf->tf_level; if (pil_stray_count[level] < MAX_STRAY_LOG) { printf("stray level interrupt %ld\n", level); pil_stray_count[level]++; if (pil_stray_count[level] >= MAX_STRAY_LOG) printf("got %d stray level interrupt %ld's: not " "logging anymore\n", MAX_STRAY_LOG, level); } } static void intr_stray_vector(void *cookie) { struct intr_vector *iv; u_int vec; iv = cookie; vec = iv->iv_vec; if (intr_stray_count[vec] < MAX_STRAY_LOG) { printf("stray vector interrupt %d\n", vec); intr_stray_count[vec]++; if (intr_stray_count[vec] >= MAX_STRAY_LOG) printf("got %d stray vector interrupt %d's: not " "logging anymore\n", MAX_STRAY_LOG, vec); } } void intr_init1() { int i; /* Mark all interrupts as being stray. */ for (i = 0; i < PIL_MAX; i++) intr_handlers[i] = intr_stray_level; for (i = 0; i < IV_MAX; i++) { intr_vectors[i].iv_func = intr_stray_vector; intr_vectors[i].iv_arg = &intr_vectors[i]; intr_vectors[i].iv_pri = PIL_LOW; intr_vectors[i].iv_vec = i; intr_vectors[i].iv_refcnt = 0; } intr_handlers[PIL_LOW] = intr_fast; } void intr_init2() { sx_init(&intr_table_lock, "intr sources"); mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); } static int -intr_assign_cpu(void *arg, u_char cpu) +intr_assign_cpu(void *arg, int cpu) { #ifdef SMP struct pcpu *pc; struct intr_vector *iv; /* * Don't do anything during early boot. We will pick up the * assignment once the APs are started. */ if (assign_cpu && cpu != NOCPU) { pc = pcpu_find(cpu); if (pc == NULL) return (EINVAL); iv = arg; sx_xlock(&intr_table_lock); iv->iv_mid = pc->pc_mid; iv->iv_ic->ic_assign(iv); sx_xunlock(&intr_table_lock); } return (0); #else return (EOPNOTSUPP); #endif } static void intr_execute_handlers(void *cookie) { struct intr_vector *iv; iv = cookie; if (__predict_false(intr_event_handle(iv->iv_event, NULL) != 0)) intr_stray_vector(iv); } int intr_controller_register(int vec, const struct intr_controller *ic, void *icarg) { struct intr_event *ie; struct intr_vector *iv; int error; if (vec < 0 || vec >= IV_MAX) return (EINVAL); sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; ie = iv->iv_event; sx_xunlock(&intr_table_lock); if (ie != NULL) return (EEXIST); error = intr_event_create(&ie, iv, 0, vec, NULL, ic->ic_clear, ic->ic_clear, intr_assign_cpu, "vec%d:", vec); if (error != 0) return (error); sx_xlock(&intr_table_lock); if (iv->iv_event != NULL) { sx_xunlock(&intr_table_lock); intr_event_destroy(ie); return (EEXIST); } iv->iv_ic = ic; iv->iv_icarg = icarg; iv->iv_event = ie; iv->iv_mid = PCPU_GET(mid); sx_xunlock(&intr_table_lock); return (0); } int inthand_add(const char *name, int vec, driver_filter_t *filt, driver_intr_t *handler, void *arg, int flags, void **cookiep) { const struct intr_controller *ic; struct intr_event *ie; struct intr_handler *ih; struct intr_vector *iv; int error, filter; if (vec < 0 || vec >= IV_MAX) return (EINVAL); /* * INTR_BRIDGE filters/handlers are special purpose only, allowing * them to be shared just would complicate things unnecessarily. */ if ((flags & INTR_BRIDGE) != 0 && (flags & INTR_EXCL) == 0) return (EINVAL); sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; ic = iv->iv_ic; ie = iv->iv_event; sx_xunlock(&intr_table_lock); if (ic == NULL || ie == NULL) return (EINVAL); error = intr_event_add_handler(ie, name, filt, handler, arg, intr_priority(flags), flags, cookiep); if (error != 0) return (error); sx_xlock(&intr_table_lock); /* Disable the interrupt while we fiddle with it. */ ic->ic_disable(iv); iv->iv_refcnt++; if (iv->iv_refcnt == 1) intr_setup((flags & INTR_BRIDGE) != 0 ? PIL_BRIDGE : filt != NULL ? PIL_FILTER : PIL_ITHREAD, intr_fast, vec, intr_execute_handlers, iv); else if (filt != NULL) { /* * Check if we need to upgrade from PIL_ITHREAD to PIL_FILTER. * Given that apart from the on-board SCCs and UARTs shared * interrupts are rather uncommon on sparc64 this sould be * pretty rare in practice. */ filter = 0; TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { if (ih->ih_filter != NULL && ih->ih_filter != filt) { filter = 1; break; } } if (filter == 0) intr_setup(PIL_FILTER, intr_fast, vec, intr_execute_handlers, iv); } intr_stray_count[vec] = 0; intrcnt_updatename(vec, ie->ie_fullname, 0); #ifdef SMP if (assign_cpu) intr_assign_next_cpu(iv); #endif ic->ic_enable(iv); /* Ensure the interrupt is cleared, it might have triggered before. */ if (ic->ic_clear != NULL) ic->ic_clear(iv); sx_xunlock(&intr_table_lock); return (0); } int inthand_remove(int vec, void *cookie) { struct intr_vector *iv; int error; if (vec < 0 || vec >= IV_MAX) return (EINVAL); error = intr_event_remove_handler(cookie); if (error == 0) { /* * XXX: maybe this should be done regardless of whether * intr_event_remove_handler() succeeded? */ sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; iv->iv_refcnt--; if (iv->iv_refcnt == 0) { /* * Don't disable the interrupt for now, so that * stray interrupts get detected... */ intr_setup(PIL_LOW, intr_fast, vec, intr_stray_vector, iv); } sx_xunlock(&intr_table_lock); } return (error); } /* Add a description to an active interrupt handler. */ int intr_describe(int vec, void *ih, const char *descr) { struct intr_vector *iv; int error; if (vec < 0 || vec >= IV_MAX) return (EINVAL); sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; if (iv == NULL) { sx_xunlock(&intr_table_lock); return (EINVAL); } error = intr_event_describe_handler(iv->iv_event, ih, descr); if (error) { sx_xunlock(&intr_table_lock); return (error); } intrcnt_updatename(vec, iv->iv_event->ie_fullname, 0); sx_xunlock(&intr_table_lock); return (error); } #ifdef SMP /* * Support for balancing interrupt sources across CPUs. For now we just * allocate CPUs round-robin. */ static cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1); static int current_cpu; static void intr_assign_next_cpu(struct intr_vector *iv) { struct pcpu *pc; sx_assert(&intr_table_lock, SA_XLOCKED); /* * Assign this source to a CPU in a round-robin fashion. */ pc = pcpu_find(current_cpu); if (pc == NULL) return; iv->iv_mid = pc->pc_mid; iv->iv_ic->ic_assign(iv); do { current_cpu++; if (current_cpu > mp_maxid) current_cpu = 0; } while (!CPU_ISSET(current_cpu, &intr_cpus)); } /* Attempt to bind the specified IRQ to the specified CPU. */ int intr_bind(int vec, u_char cpu) { struct intr_vector *iv; int error; if (vec < 0 || vec >= IV_MAX) return (EINVAL); sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; if (iv == NULL) { sx_xunlock(&intr_table_lock); return (EINVAL); } error = intr_event_bind(iv->iv_event, cpu); sx_xunlock(&intr_table_lock); return (error); } /* * Add a CPU to our mask of valid CPUs that can be destinations of * interrupts. */ void intr_add_cpu(u_int cpu) { if (cpu >= MAXCPU) panic("%s: Invalid CPU ID", __func__); if (bootverbose) printf("INTR: Adding CPU %d as a target\n", cpu); CPU_SET(cpu, &intr_cpus); } /* * Distribute all the interrupt sources among the available CPUs once the * APs have been launched. */ static void intr_shuffle_irqs(void *arg __unused) { struct pcpu *pc; struct intr_vector *iv; int i; /* Don't bother on UP. */ if (mp_ncpus == 1) return; sx_xlock(&intr_table_lock); assign_cpu = 1; for (i = 0; i < IV_MAX; i++) { iv = &intr_vectors[i]; if (iv != NULL && iv->iv_refcnt > 0) { /* * If this event is already bound to a CPU, * then assign the source to that CPU instead * of picking one via round-robin. */ if (iv->iv_event->ie_cpu != NOCPU && (pc = pcpu_find(iv->iv_event->ie_cpu)) != NULL) { iv->iv_mid = pc->pc_mid; iv->iv_ic->ic_assign(iv); } else intr_assign_next_cpu(iv); } } sx_xunlock(&intr_table_lock); } SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, NULL); #endif Index: head/sys/sys/interrupt.h =================================================================== --- head/sys/sys/interrupt.h (revision 271711) +++ head/sys/sys/interrupt.h (revision 271712) @@ -1,186 +1,186 @@ /*- * Copyright (c) 1997, Stefan Esser * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_INTERRUPT_H_ #define _SYS_INTERRUPT_H_ #include #include struct intr_event; struct intr_thread; struct trapframe; /* * Describe a hardware interrupt handler. * * Multiple interrupt handlers for a specific event can be chained * together. */ struct intr_handler { driver_filter_t *ih_filter; /* Filter handler function. */ driver_intr_t *ih_handler; /* Threaded handler function. */ void *ih_argument; /* Argument to pass to handlers. */ int ih_flags; char ih_name[MAXCOMLEN + 1]; /* Name of handler. */ struct intr_event *ih_event; /* Event we are connected to. */ int ih_need; /* Needs service. */ TAILQ_ENTRY(intr_handler) ih_next; /* Next handler for this event. */ u_char ih_pri; /* Priority of this handler. */ struct intr_thread *ih_thread; /* Ithread for filtered handler. */ }; /* Interrupt handle flags kept in ih_flags */ #define IH_EXCLUSIVE 0x00000002 /* Exclusive interrupt. */ #define IH_ENTROPY 0x00000004 /* Device is a good entropy source. */ #define IH_DEAD 0x00000008 /* Handler should be removed. */ #define IH_MPSAFE 0x80000000 /* Handler does not need Giant. */ /* * Describe an interrupt event. An event holds a list of handlers. * The 'pre_ithread', 'post_ithread', 'post_filter', and 'assign_cpu' * hooks are used to invoke MD code for certain operations. * * The 'pre_ithread' hook is called when an interrupt thread for * handlers without filters is scheduled. It is responsible for * ensuring that 1) the system won't be swamped with an interrupt * storm from the associated source while the ithread runs and 2) the * current CPU is able to receive interrupts from other interrupt * sources. The first is usually accomplished by disabling * level-triggered interrupts until the ithread completes. The second * is accomplished on some platforms by acknowledging the interrupt * via an EOI. * * The 'post_ithread' hook is invoked when an ithread finishes. It is * responsible for ensuring that the associated interrupt source will * trigger an interrupt when it is asserted in the future. Usually * this is implemented by enabling a level-triggered interrupt that * was previously disabled via the 'pre_ithread' hook. * * The 'post_filter' hook is invoked when a filter handles an * interrupt. It is responsible for ensuring that the current CPU is * able to receive interrupts again. On some platforms this is done * by acknowledging the interrupts via an EOI. * * The 'assign_cpu' hook is used to bind an interrupt source to a * specific CPU. If the interrupt cannot be bound, this function may * return an error. * * Note that device drivers may also use interrupt events to manage * multiplexing interrupt interrupt handler into handlers for child * devices. In that case, the above hooks are not used. The device * can create an event for its interrupt resource and register child * event handlers with that event. It can then use * intr_event_execute_handlers() to execute non-filter handlers. * Currently filter handlers are not supported by this, but that can * be added by splitting out the filter loop from intr_event_handle() * if desired. */ struct intr_event { TAILQ_ENTRY(intr_event) ie_list; TAILQ_HEAD(, intr_handler) ie_handlers; /* Interrupt handlers. */ char ie_name[MAXCOMLEN + 1]; /* Individual event name. */ char ie_fullname[MAXCOMLEN + 1]; struct mtx ie_lock; void *ie_source; /* Cookie used by MD code. */ struct intr_thread *ie_thread; /* Thread we are connected to. */ void (*ie_pre_ithread)(void *); void (*ie_post_ithread)(void *); void (*ie_post_filter)(void *); - int (*ie_assign_cpu)(void *, u_char); + int (*ie_assign_cpu)(void *, int); int ie_flags; int ie_count; /* Loop counter. */ int ie_warncnt; /* Rate-check interrupt storm warns. */ struct timeval ie_warntm; int ie_irq; /* Physical irq number if !SOFT. */ - u_char ie_cpu; /* CPU this event is bound to. */ + int ie_cpu; /* CPU this event is bound to. */ }; /* Interrupt event flags kept in ie_flags. */ #define IE_SOFT 0x000001 /* Software interrupt. */ #define IE_ENTROPY 0x000002 /* Interrupt is an entropy source. */ #define IE_ADDING_THREAD 0x000004 /* Currently building an ithread. */ /* Flags to pass to sched_swi. */ #define SWI_DELAY 0x2 /* * Software interrupt numbers in priority order. The priority determines * the priority of the corresponding interrupt thread. */ #define SWI_TTY 0 #define SWI_NET 1 #define SWI_CAMBIO 2 #define SWI_VM 3 #define SWI_CLOCK 4 #define SWI_TQ_FAST 5 #define SWI_TQ 6 #define SWI_TQ_GIANT 6 struct proc; extern struct intr_event *tty_intr_event; extern struct intr_event *clk_intr_event; extern void *vm_ih; /* Counts and names for statistics (defined in MD code). */ extern u_long intrcnt[]; /* counts for for each device and stray */ extern char intrnames[]; /* string table containing device names */ extern size_t sintrcnt; /* size of intrcnt table */ extern size_t sintrnames; /* size of intrnames table */ #ifdef DDB void db_dump_intr_event(struct intr_event *ie, int handlers); #endif u_char intr_priority(enum intr_type flags); int intr_event_add_handler(struct intr_event *ie, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, void **cookiep); -int intr_event_bind(struct intr_event *ie, u_char cpu); +int intr_event_bind(struct intr_event *ie, int cpu); int intr_event_create(struct intr_event **event, void *source, int flags, int irq, void (*pre_ithread)(void *), void (*post_ithread)(void *), void (*post_filter)(void *), - int (*assign_cpu)(void *, u_char), const char *fmt, ...) + int (*assign_cpu)(void *, int), const char *fmt, ...) __printflike(9, 10); int intr_event_describe_handler(struct intr_event *ie, void *cookie, const char *descr); int intr_event_destroy(struct intr_event *ie); void intr_event_execute_handlers(struct proc *p, struct intr_event *ie); int intr_event_handle(struct intr_event *ie, struct trapframe *frame); int intr_event_remove_handler(void *cookie); int intr_getaffinity(int irq, void *mask); void *intr_handler_source(void *cookie); int intr_setaffinity(int irq, void *mask); void _intr_drain(int irq); /* Linux compat only. */ int swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, void *arg, int pri, enum intr_type flags, void **cookiep); void swi_sched(void *cookie, int flags); int swi_remove(void *cookie); #endif Index: head/sys/x86/x86/intr_machdep.c =================================================================== --- head/sys/x86/x86/intr_machdep.c (revision 271711) +++ head/sys/x86/x86/intr_machdep.c (revision 271712) @@ -1,565 +1,565 @@ /*- * Copyright (c) 2003 John Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * Machine dependent interrupt code for x86. For x86, we have to * deal with different PICs. Thus, we use the passed in vector to lookup * an interrupt source associated with that vector. The interrupt source * describes which PIC the source belongs to and includes methods to handle * that source. */ #include "opt_atpic.h" #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #endif #ifndef DEV_ATPIC #include #include #include #include #ifdef PC98 #include #else #include #endif #endif #define MAX_STRAY_LOG 5 typedef void (*mask_fn)(void *); static int intrcnt_index; static struct intsrc *interrupt_sources[NUM_IO_INTS]; static struct mtx intr_table_lock; static struct mtx intrcnt_lock; static TAILQ_HEAD(pics_head, pic) pics; #ifdef SMP static int assign_cpu; #endif u_long intrcnt[INTRCNT_COUNT]; char intrnames[INTRCNT_COUNT * (MAXCOMLEN + 1)]; size_t sintrcnt = sizeof(intrcnt); size_t sintrnames = sizeof(intrnames); -static int intr_assign_cpu(void *arg, u_char cpu); +static int intr_assign_cpu(void *arg, int cpu); static void intr_disable_src(void *arg); static void intr_init(void *__dummy); static int intr_pic_registered(struct pic *pic); static void intrcnt_setname(const char *name, int index); static void intrcnt_updatename(struct intsrc *is); static void intrcnt_register(struct intsrc *is); static int intr_pic_registered(struct pic *pic) { struct pic *p; TAILQ_FOREACH(p, &pics, pics) { if (p == pic) return (1); } return (0); } /* * Register a new interrupt controller (PIC). This is to support suspend * and resume where we suspend/resume controllers rather than individual * sources. This also allows controllers with no active sources (such as * 8259As in a system using the APICs) to participate in suspend and resume. */ int intr_register_pic(struct pic *pic) { int error; mtx_lock(&intr_table_lock); if (intr_pic_registered(pic)) error = EBUSY; else { TAILQ_INSERT_TAIL(&pics, pic, pics); error = 0; } mtx_unlock(&intr_table_lock); return (error); } /* * Register a new interrupt source with the global interrupt system. * The global interrupts need to be disabled when this function is * called. */ int intr_register_source(struct intsrc *isrc) { int error, vector; KASSERT(intr_pic_registered(isrc->is_pic), ("unregistered PIC")); vector = isrc->is_pic->pic_vector(isrc); if (interrupt_sources[vector] != NULL) return (EEXIST); error = intr_event_create(&isrc->is_event, isrc, 0, vector, intr_disable_src, (mask_fn)isrc->is_pic->pic_enable_source, (mask_fn)isrc->is_pic->pic_eoi_source, intr_assign_cpu, "irq%d:", vector); if (error) return (error); mtx_lock(&intr_table_lock); if (interrupt_sources[vector] != NULL) { mtx_unlock(&intr_table_lock); intr_event_destroy(isrc->is_event); return (EEXIST); } intrcnt_register(isrc); interrupt_sources[vector] = isrc; isrc->is_handlers = 0; mtx_unlock(&intr_table_lock); return (0); } struct intsrc * intr_lookup_source(int vector) { return (interrupt_sources[vector]); } int intr_add_handler(const char *name, int vector, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep) { struct intsrc *isrc; int error; isrc = intr_lookup_source(vector); if (isrc == NULL) return (EINVAL); error = intr_event_add_handler(isrc->is_event, name, filter, handler, arg, intr_priority(flags), flags, cookiep); if (error == 0) { mtx_lock(&intr_table_lock); intrcnt_updatename(isrc); isrc->is_handlers++; if (isrc->is_handlers == 1) { isrc->is_pic->pic_enable_intr(isrc); isrc->is_pic->pic_enable_source(isrc); } mtx_unlock(&intr_table_lock); } return (error); } int intr_remove_handler(void *cookie) { struct intsrc *isrc; int error; isrc = intr_handler_source(cookie); error = intr_event_remove_handler(cookie); if (error == 0) { mtx_lock(&intr_table_lock); isrc->is_handlers--; if (isrc->is_handlers == 0) { isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI); isrc->is_pic->pic_disable_intr(isrc); } intrcnt_updatename(isrc); mtx_unlock(&intr_table_lock); } return (error); } int intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol) { struct intsrc *isrc; isrc = intr_lookup_source(vector); if (isrc == NULL) return (EINVAL); return (isrc->is_pic->pic_config_intr(isrc, trig, pol)); } static void intr_disable_src(void *arg) { struct intsrc *isrc; isrc = arg; isrc->is_pic->pic_disable_source(isrc, PIC_EOI); } void intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame) { struct intr_event *ie; int vector; /* * We count software interrupts when we process them. The * code here follows previous practice, but there's an * argument for counting hardware interrupts when they're * processed too. */ (*isrc->is_count)++; PCPU_INC(cnt.v_intr); ie = isrc->is_event; /* * XXX: We assume that IRQ 0 is only used for the ISA timer * device (clk). */ vector = isrc->is_pic->pic_vector(isrc); if (vector == 0) clkintr_pending = 1; /* * For stray interrupts, mask and EOI the source, bump the * stray count, and log the condition. */ if (intr_event_handle(ie, frame) != 0) { isrc->is_pic->pic_disable_source(isrc, PIC_EOI); (*isrc->is_straycount)++; if (*isrc->is_straycount < MAX_STRAY_LOG) log(LOG_ERR, "stray irq%d\n", vector); else if (*isrc->is_straycount == MAX_STRAY_LOG) log(LOG_CRIT, "too many stray irq %d's: not logging anymore\n", vector); } } void intr_resume(bool suspend_cancelled) { struct pic *pic; #ifndef DEV_ATPIC atpic_reset(); #endif mtx_lock(&intr_table_lock); TAILQ_FOREACH(pic, &pics, pics) { if (pic->pic_resume != NULL) pic->pic_resume(pic, suspend_cancelled); } mtx_unlock(&intr_table_lock); } void intr_suspend(void) { struct pic *pic; mtx_lock(&intr_table_lock); TAILQ_FOREACH_REVERSE(pic, &pics, pics_head, pics) { if (pic->pic_suspend != NULL) pic->pic_suspend(pic); } mtx_unlock(&intr_table_lock); } static int -intr_assign_cpu(void *arg, u_char cpu) +intr_assign_cpu(void *arg, int cpu) { #ifdef SMP struct intsrc *isrc; int error; /* * Don't do anything during early boot. We will pick up the * assignment once the APs are started. */ if (assign_cpu && cpu != NOCPU) { isrc = arg; mtx_lock(&intr_table_lock); error = isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); mtx_unlock(&intr_table_lock); } else error = 0; return (error); #else return (EOPNOTSUPP); #endif } static void intrcnt_setname(const char *name, int index) { snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", MAXCOMLEN, name); } static void intrcnt_updatename(struct intsrc *is) { intrcnt_setname(is->is_event->ie_fullname, is->is_index); } static void intrcnt_register(struct intsrc *is) { char straystr[MAXCOMLEN + 1]; KASSERT(is->is_event != NULL, ("%s: isrc with no event", __func__)); mtx_lock_spin(&intrcnt_lock); is->is_index = intrcnt_index; intrcnt_index += 2; snprintf(straystr, MAXCOMLEN + 1, "stray irq%d", is->is_pic->pic_vector(is)); intrcnt_updatename(is); is->is_count = &intrcnt[is->is_index]; intrcnt_setname(straystr, is->is_index + 1); is->is_straycount = &intrcnt[is->is_index + 1]; mtx_unlock_spin(&intrcnt_lock); } void intrcnt_add(const char *name, u_long **countp) { mtx_lock_spin(&intrcnt_lock); *countp = &intrcnt[intrcnt_index]; intrcnt_setname(name, intrcnt_index); intrcnt_index++; mtx_unlock_spin(&intrcnt_lock); } static void intr_init(void *dummy __unused) { intrcnt_setname("???", 0); intrcnt_index = 1; TAILQ_INIT(&pics); mtx_init(&intr_table_lock, "intr sources", NULL, MTX_DEF); mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); } SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); #ifndef DEV_ATPIC /* Initialize the two 8259A's to a known-good shutdown state. */ void atpic_reset(void) { outb(IO_ICU1, ICW1_RESET | ICW1_IC4); outb(IO_ICU1 + ICU_IMR_OFFSET, IDT_IO_INTS); outb(IO_ICU1 + ICU_IMR_OFFSET, IRQ_MASK(ICU_SLAVEID)); outb(IO_ICU1 + ICU_IMR_OFFSET, MASTER_MODE); outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff); outb(IO_ICU1, OCW3_SEL | OCW3_RR); outb(IO_ICU2, ICW1_RESET | ICW1_IC4); outb(IO_ICU2 + ICU_IMR_OFFSET, IDT_IO_INTS + 8); outb(IO_ICU2 + ICU_IMR_OFFSET, ICU_SLAVEID); outb(IO_ICU2 + ICU_IMR_OFFSET, SLAVE_MODE); outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff); outb(IO_ICU2, OCW3_SEL | OCW3_RR); } #endif /* Add a description to an active interrupt handler. */ int intr_describe(u_int vector, void *ih, const char *descr) { struct intsrc *isrc; int error; isrc = intr_lookup_source(vector); if (isrc == NULL) return (EINVAL); error = intr_event_describe_handler(isrc->is_event, ih, descr); if (error) return (error); intrcnt_updatename(isrc); return (0); } #ifdef DDB /* * Dump data about interrupt handlers */ DB_SHOW_COMMAND(irqs, db_show_irqs) { struct intsrc **isrc; int i, verbose; if (strcmp(modif, "v") == 0) verbose = 1; else verbose = 0; isrc = interrupt_sources; for (i = 0; i < NUM_IO_INTS && !db_pager_quit; i++, isrc++) if (*isrc != NULL) db_dump_intr_event((*isrc)->is_event, verbose); } #endif #ifdef SMP /* * Support for balancing interrupt sources across CPUs. For now we just * allocate CPUs round-robin. */ static cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1); static int current_cpu; /* * Return the CPU that the next interrupt source should use. For now * this just returns the next local APIC according to round-robin. */ u_int intr_next_cpu(void) { u_int apic_id; /* Leave all interrupts on the BSP during boot. */ if (!assign_cpu) return (PCPU_GET(apic_id)); mtx_lock_spin(&icu_lock); apic_id = cpu_apic_ids[current_cpu]; do { current_cpu++; if (current_cpu > mp_maxid) current_cpu = 0; } while (!CPU_ISSET(current_cpu, &intr_cpus)); mtx_unlock_spin(&icu_lock); return (apic_id); } /* Attempt to bind the specified IRQ to the specified CPU. */ int intr_bind(u_int vector, u_char cpu) { struct intsrc *isrc; isrc = intr_lookup_source(vector); if (isrc == NULL) return (EINVAL); return (intr_event_bind(isrc->is_event, cpu)); } /* * Add a CPU to our mask of valid CPUs that can be destinations of * interrupts. */ void intr_add_cpu(u_int cpu) { if (cpu >= MAXCPU) panic("%s: Invalid CPU ID", __func__); if (bootverbose) printf("INTR: Adding local APIC %d as a target\n", cpu_apic_ids[cpu]); CPU_SET(cpu, &intr_cpus); } /* * Distribute all the interrupt sources among the available CPUs once the * AP's have been launched. */ static void intr_shuffle_irqs(void *arg __unused) { struct intsrc *isrc; int i; #ifdef XEN /* * Doesn't work yet */ return; #endif /* Don't bother on UP. */ if (mp_ncpus == 1) return; /* Round-robin assign a CPU to each enabled source. */ mtx_lock(&intr_table_lock); assign_cpu = 1; for (i = 0; i < NUM_IO_INTS; i++) { isrc = interrupt_sources[i]; if (isrc != NULL && isrc->is_handlers > 0) { /* * If this event is already bound to a CPU, * then assign the source to that CPU instead * of picking one via round-robin. Note that * this is careful to only advance the * round-robin if the CPU assignment succeeds. */ if (isrc->is_event->ie_cpu != NOCPU) (void)isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[isrc->is_event->ie_cpu]); else if (isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]) == 0) (void)intr_next_cpu(); } } mtx_unlock(&intr_table_lock); } SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, NULL); #else /* * Always route interrupts to the current processor in the UP case. */ u_int intr_next_cpu(void) { return (PCPU_GET(apic_id)); } #endif