Changeset View
Changeset View
Standalone View
Standalone View
sys/xen/xen_intr.c
Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
#include <sys/interrupt.h> | #include <sys/interrupt.h> | ||||
#include <sys/pcpu.h> | #include <sys/pcpu.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/refcount.h> | #include <sys/refcount.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <machine/intr_machdep.h> | |||||
#include <machine/smp.h> | #include <machine/smp.h> | ||||
#include <machine/stdarg.h> | #include <machine/stdarg.h> | ||||
#include <machine/xen/synch_bitops.h> | #include <machine/xen/synch_bitops.h> | ||||
#include <xen/xen-os.h> | #include <xen/xen-os.h> | ||||
#include <xen/arch-intr.h> | #include <xen/arch-intr.h> | ||||
#include <xen/hvm.h> | #include <xen/hvm.h> | ||||
#include <xen/hypervisor.h> | #include <xen/hypervisor.h> | ||||
#include <xen/xen_intr.h> | #include <xen/xen_intr.h> | ||||
#include <dev/xen/xenpci/xenpcivar.h> | #include <dev/xen/xenpci/xenpcivar.h> | ||||
#include <dev/pci/pcivar.h> | #include <dev/pci/pcivar.h> | ||||
#ifdef DDB | #ifdef DDB | ||||
#include <ddb/ddb.h> | #include <ddb/ddb.h> | ||||
#endif | #endif | ||||
static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services"); | static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services"); | ||||
/* | |||||
* Lock for x86-related structures. Notably modifying | |||||
* xen_intr_auto_vector_count, and allocating interrupts require this lock be | |||||
* held. | |||||
*/ | |||||
static struct mtx xen_intr_x86_lock; | |||||
static u_int first_evtchn_irq; | |||||
/** | /** | ||||
* Per-cpu event channel processing state. | * Per-cpu event channel processing state. | ||||
*/ | */ | ||||
struct xen_intr_pcpu_data { | struct xen_intr_pcpu_data { | ||||
/** | /** | ||||
* The last event channel bitmap section (level one bit) processed. | * The last event channel bitmap section (level one bit) processed. | ||||
* This is used to ensure we scan all ports before | * This is used to ensure we scan all ports before | ||||
* servicing an already servied port again. | * servicing an already servied port again. | ||||
Show All 31 Lines | |||||
* Lock for interrupt core data. Notably modifying xen_intr_port_to_isrc[], | * Lock for interrupt core data. Notably modifying xen_intr_port_to_isrc[], | ||||
* or isrc->xi_port (which implies the former) requires this lock be held. | * or isrc->xi_port (which implies the former) requires this lock be held. | ||||
* | * | ||||
* Any time this lock is not held, the condition | * Any time this lock is not held, the condition | ||||
* `!xen_intr_port_to_isrc[i] || (xen_intr_port_to_isrc[i]->ix_port == i)` | * `!xen_intr_port_to_isrc[i] || (xen_intr_port_to_isrc[i]->ix_port == i)` | ||||
* MUST be true for all values of i which are valid indicies of the array. | * MUST be true for all values of i which are valid indicies of the array. | ||||
*/ | */ | ||||
static struct mtx xen_intr_isrc_lock; | static struct mtx xen_intr_isrc_lock; | ||||
static u_int xen_intr_auto_vector_count; | |||||
static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS]; | static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS]; | ||||
/*------------------------- Private Functions --------------------------------*/ | /*------------------------- Private Functions --------------------------------*/ | ||||
/** | /** | ||||
* Retrieve a handle for a Xen interrupt source. | * Retrieve a handle for a Xen interrupt source. | ||||
* | * | ||||
* \param isrc A valid Xen interrupt source structure. | * \param isrc A valid Xen interrupt source structure. | ||||
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines | xen_intr_intrcnt_add(u_int cpu) | ||||
if (pcpu->evtchn_intrcnt != NULL) | if (pcpu->evtchn_intrcnt != NULL) | ||||
return; | return; | ||||
snprintf(buf, sizeof(buf), "cpu%d:xen", cpu); | snprintf(buf, sizeof(buf), "cpu%d:xen", cpu); | ||||
intrcnt_add(buf, &pcpu->evtchn_intrcnt); | intrcnt_add(buf, &pcpu->evtchn_intrcnt); | ||||
} | } | ||||
/** | /** | ||||
* Search for an already allocated but currently unused Xen interrupt | |||||
* source object. | |||||
* | |||||
* \param type Restrict the search to interrupt sources of the given | |||||
* type. | |||||
* | |||||
* \return A pointer to a free Xen interrupt source object or NULL. | |||||
*/ | |||||
static struct xenisrc * | |||||
xen_intr_find_unused_isrc(enum evtchn_type type) | |||||
{ | |||||
int isrc_idx; | |||||
mtx_assert(&xen_intr_x86_lock, MA_OWNED); | |||||
for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) { | |||||
struct xenisrc *isrc; | |||||
u_int vector; | |||||
vector = first_evtchn_irq + isrc_idx; | |||||
isrc = (struct xenisrc *)intr_lookup_source(vector); | |||||
if (isrc != NULL | |||||
&& isrc->xi_type == EVTCHN_TYPE_UNBOUND) { | |||||
KASSERT(xen_arch_intr_has_handlers(isrc), | |||||
("Free evtchn still has handlers")); | |||||
isrc->xi_type = type; | |||||
return (isrc); | |||||
} | |||||
} | |||||
return (NULL); | |||||
} | |||||
/** | |||||
* Allocate a Xen interrupt source object. | |||||
* | |||||
* \param type The type of interrupt source to create. | |||||
* | |||||
* \return A pointer to a newly allocated Xen interrupt source | |||||
* object or NULL. | |||||
*/ | |||||
static struct xenisrc * | |||||
xen_intr_alloc_isrc(enum evtchn_type type, evtchn_port_t port) | |||||
{ | |||||
static int warned; | |||||
struct xenisrc *isrc; | |||||
unsigned int vector; | |||||
mtx_lock(&xen_intr_x86_lock); | |||||
isrc = xen_intr_find_unused_isrc(type); | |||||
if (isrc != NULL) { | |||||
mtx_unlock(&xen_intr_x86_lock); | |||||
goto out; | |||||
} | |||||
if (xen_intr_auto_vector_count >= NR_EVENT_CHANNELS) { | |||||
if (!warned) { | |||||
warned = 1; | |||||
printf("%s: Event channels exhausted.\n", __func__); | |||||
} | |||||
mtx_unlock(&xen_intr_x86_lock); | |||||
return (NULL); | |||||
} | |||||
vector = first_evtchn_irq + xen_intr_auto_vector_count; | |||||
xen_intr_auto_vector_count++; | |||||
KASSERT((intr_lookup_source(vector) == NULL), | |||||
("Trying to use an already allocated vector")); | |||||
mtx_unlock(&xen_intr_x86_lock); | |||||
isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO); | |||||
isrc->xi_arch.xai_intsrc.is_pic = &xen_intr_pic; | |||||
isrc->xi_arch.xai_vector = vector; | |||||
isrc->xi_type = type; | |||||
intr_register_source(&isrc->xi_arch.xai_intsrc); | |||||
out: | |||||
/* xen_intr_assign_cpu() requires this to be set */ | |||||
isrc->xi_port = port; | |||||
#ifdef SMP | |||||
if (type == EVTCHN_TYPE_PORT) { | |||||
/* | |||||
* By default all interrupts are assigned to vCPU#0 | |||||
* unless specified otherwise, so shuffle them to balance | |||||
* the interrupt load. | |||||
*/ | |||||
xen_intr_assign_cpu(isrc, intr_next_cpu(0)); | |||||
} | |||||
#endif | |||||
return (isrc); | |||||
} | |||||
/** | |||||
* Attempt to free an active Xen interrupt source object. | * Attempt to free an active Xen interrupt source object. | ||||
* | * | ||||
* \param isrc The interrupt source object to release. | * \param isrc The interrupt source object to release. | ||||
* | * | ||||
* \returns EBUSY if the source is still in use, otherwise 0. | * \returns EBUSY if the source is still in use, otherwise 0. | ||||
*/ | */ | ||||
static int | static int | ||||
xen_intr_release_isrc(struct xenisrc *isrc) | xen_intr_release_isrc(struct xenisrc *isrc) | ||||
Show All 14 Lines | if (isrc->xi_close != 0 && isrc->xi_port < NR_EVENT_CHANNELS) { | ||||
if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) | if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) | ||||
panic("EVTCHNOP_close failed"); | panic("EVTCHNOP_close failed"); | ||||
} | } | ||||
xen_intr_port_to_isrc[isrc->xi_port] = NULL; | xen_intr_port_to_isrc[isrc->xi_port] = NULL; | ||||
/* not reachable from xen_intr_port_to_isrc[], therefore unlock */ | /* not reachable from xen_intr_port_to_isrc[], therefore unlock */ | ||||
mtx_unlock(&xen_intr_isrc_lock); | mtx_unlock(&xen_intr_isrc_lock); | ||||
isrc->xi_cpu = 0; | xen_arch_intr_release(M_XENINTR, isrc); | ||||
isrc->xi_port = ~0; | |||||
isrc->xi_cookie = NULL; | |||||
/* | |||||
* Fun with locking here. xen_intr_x86_lock is actually controlling | |||||
* *allocation*. This means the isrc isn't under control of the lock | |||||
* until ->xi_type == EVTCHN_TYPE_UNBOUND. The consequence is | |||||
* atomic_store_rel() is appropriate since we merely need the other | |||||
* stores to complete before this one. This one simply needs to | |||||
* complete atomically. | |||||
*/ | |||||
atomic_store_rel(&isrc->xi_type, EVTCHN_TYPE_UNBOUND); | |||||
return (0); | return (0); | ||||
} | } | ||||
/** | /** | ||||
* Associate an interrupt handler with an already allocated local Xen | * Associate an interrupt handler with an already allocated local Xen | ||||
* event channel port. | * event channel port. | ||||
* | * | ||||
* \param isrcp The returned Xen interrupt object associated with | * \param isrcp The returned Xen interrupt object associated with | ||||
Show All 23 Lines | xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port, | ||||
int error; | int error; | ||||
*isrcp = NULL; | *isrcp = NULL; | ||||
if (port_handlep == NULL) { | if (port_handlep == NULL) { | ||||
printf("%s: %s: Bad event handle\n", intr_owner, __func__); | printf("%s: %s: Bad event handle\n", intr_owner, __func__); | ||||
return (EINVAL); | return (EINVAL); | ||||
} | } | ||||
isrc = xen_intr_alloc_isrc(type, local_port); | isrc = xen_arch_intr_alloc(M_XENINTR, intr_owner, type, local_port); | ||||
if (isrc == NULL) | if (isrc == NULL) | ||||
return (ENOSPC); | return (ENOSPC); | ||||
mtx_lock(&xen_intr_isrc_lock); | mtx_lock(&xen_intr_isrc_lock); | ||||
xen_intr_port_to_isrc[isrc->xi_port] = isrc; | xen_intr_port_to_isrc[isrc->xi_port] = isrc; | ||||
refcount_init(&isrc->xi_refcount, 1); | refcount_init(&isrc->xi_refcount, 1); | ||||
mtx_unlock(&xen_intr_isrc_lock); | mtx_unlock(&xen_intr_isrc_lock); | ||||
/* Assign the opaque handler */ | /* Assign the opaque handler */ | ||||
▲ Show 20 Lines • Show All 170 Lines • ▼ Show 20 Lines | xen_intr_init(void *dummy __unused) | ||||
struct xen_intr_pcpu_data *pcpu; | struct xen_intr_pcpu_data *pcpu; | ||||
int i; | int i; | ||||
if (!xen_domain()) | if (!xen_domain()) | ||||
return (0); | return (0); | ||||
mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF); | mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF); | ||||
mtx_init(&xen_intr_x86_lock, "xen-x86-table-lock", NULL, MTX_DEF); | |||||
/* | /* | ||||
* Set the per-cpu mask of CPU#0 to enable all, since by default all | * Set the per-cpu mask of CPU#0 to enable all, since by default all | ||||
* event channels are bound to CPU#0. | * event channels are bound to CPU#0. | ||||
*/ | */ | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); | pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); | ||||
memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, | memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, | ||||
sizeof(pcpu->evtchn_enabled)); | sizeof(pcpu->evtchn_enabled)); | ||||
Show All 22 Lines | xen_intrcnt_init(void *dummy __unused) | ||||
/* | /* | ||||
* Register interrupt count manually as we aren't guaranteed to see a | * Register interrupt count manually as we aren't guaranteed to see a | ||||
* call to xen_intr_assign_cpu() before our first interrupt. | * call to xen_intr_assign_cpu() before our first interrupt. | ||||
*/ | */ | ||||
CPU_FOREACH(i) | CPU_FOREACH(i) | ||||
xen_intr_intrcnt_add(i); | xen_intr_intrcnt_add(i); | ||||
} | } | ||||
SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL); | SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL); | ||||
void | |||||
xen_intr_alloc_irqs(void) | |||||
{ | |||||
if (num_io_irqs > UINT_MAX - NR_EVENT_CHANNELS) | |||||
panic("IRQ allocation overflow (num_msi_irqs too high?)"); | |||||
first_evtchn_irq = num_io_irqs; | |||||
num_io_irqs += NR_EVENT_CHANNELS; | |||||
} | |||||
/*--------------------------- Common PIC Functions ---------------------------*/ | /*--------------------------- Common PIC Functions ---------------------------*/ | ||||
/** | /** | ||||
* Prepare this PIC for system suspension. | * Prepare this PIC for system suspension. | ||||
*/ | */ | ||||
void | void | ||||
xen_intr_suspend(void) | xen_intr_suspend(void) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 661 Lines • Show Last 20 Lines |