Changeset View
Changeset View
Standalone View
Standalone View
sys/x86/xen/xen_intr.c
Show First 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | |||||
#include <sys/pcpu.h> | #include <sys/pcpu.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/smp.h> | #include <sys/smp.h> | ||||
#include <sys/refcount.h> | #include <sys/refcount.h> | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/pmap.h> | #include <vm/pmap.h> | ||||
#include <machine/intr_machdep.h> | |||||
#include <machine/smp.h> | #include <machine/smp.h> | ||||
#include <machine/stdarg.h> | #include <machine/stdarg.h> | ||||
#include <xen/xen-os.h> | #include <xen/xen-os.h> | ||||
#include <xen/hypervisor.h> | #include <xen/hypervisor.h> | ||||
#include <xen/xen_intr.h> | #include <xen/xen_intr.h> | ||||
#include <xen/evtchn/evtchnvar.h> | #include <xen/evtchn/evtchnvar.h> | ||||
#include <dev/xen/xenpci/xenpcivar.h> | #include <dev/xen/xenpci/xenpcivar.h> | ||||
#include <dev/pci/pcivar.h> | #include <dev/pci/pcivar.h> | ||||
#include <machine/xen/arch-intr.h> | |||||
#ifdef DDB | #ifdef DDB | ||||
#include <ddb/ddb.h> | #include <ddb/ddb.h> | ||||
#endif | #endif | ||||
/* The code below is the implementation of 2L event channels. */ | |||||
#define NR_EVENT_CHANNELS EVTCHN_2L_NR_CHANNELS | |||||
static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services"); | static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services"); | ||||
/* | /* | ||||
* Lock for x86-related structures. Notably modifying | * Lock for x86-related structures. Notably modifying | ||||
* xen_intr_auto_vector_count, and allocating interrupts require this lock be | * xen_intr_auto_vector_count, and allocating interrupts require this lock be | ||||
* held. | * held. | ||||
* | * | ||||
* ->xi_type == EVTCHN_TYPE_UNBOUND indicates a xenisrc is under control of | * ->xi_type == EVTCHN_TYPE_UNBOUND indicates a xenisrc is under control of | ||||
Show All 36 Lines | DPCPU_DEFINE_STATIC(struct xen_intr_pcpu_data, xen_intr_pcpu) = { | ||||
.last_processed_l2i = LONG_BIT - 1 | .last_processed_l2i = LONG_BIT - 1 | ||||
}; | }; | ||||
DPCPU_DECLARE(struct vcpu_info *, vcpu_info); | DPCPU_DECLARE(struct vcpu_info *, vcpu_info); | ||||
#define INVALID_EVTCHN (~(evtchn_port_t)0) /* Invalid event channel */ | #define INVALID_EVTCHN (~(evtchn_port_t)0) /* Invalid event channel */ | ||||
#define is_valid_evtchn(x) ((uintmax_t)(x) < NR_EVENT_CHANNELS) | #define is_valid_evtchn(x) ((uintmax_t)(x) < NR_EVENT_CHANNELS) | ||||
struct xenisrc { | |||||
struct intsrc xi_intsrc; | |||||
enum evtchn_type xi_type; | |||||
u_int xi_cpu; /* VCPU for delivery. */ | |||||
u_int xi_vector; /* Global isrc vector number. */ | |||||
evtchn_port_t xi_port; | |||||
u_int xi_virq; | |||||
void *xi_cookie; | |||||
bool xi_close:1; /* close on unbind? */ | |||||
bool xi_masked:1; | |||||
volatile u_int xi_refcount; | |||||
}; | |||||
static void xen_intr_suspend(struct pic *); | static void xen_intr_suspend(struct pic *); | ||||
static void xen_intr_resume(struct pic *, bool suspend_cancelled); | static void xen_intr_resume(struct pic *, bool suspend_cancelled); | ||||
static void xen_intr_enable_source(struct intsrc *isrc); | static void xen_intr_enable_source(struct intsrc *isrc); | ||||
static void xen_intr_disable_source(struct intsrc *isrc, int eoi); | static void xen_intr_disable_source(struct intsrc *isrc, int eoi); | ||||
static void xen_intr_eoi_source(struct intsrc *isrc); | static void xen_intr_eoi_source(struct intsrc *isrc); | ||||
static void xen_intr_enable_intr(struct intsrc *isrc); | static void xen_intr_enable_intr(struct intsrc *isrc); | ||||
static void xen_intr_disable_intr(struct intsrc *isrc); | static void xen_intr_disable_intr(struct intsrc *isrc); | ||||
static int xen_intr_vector(struct intsrc *isrc); | static int xen_intr_vector(struct intsrc *isrc); | ||||
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines | for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) { | ||||
/* | /* | ||||
* Since intr_register_source() must be called while unlocked, | * Since intr_register_source() must be called while unlocked, | ||||
* isrc == NULL *will* occur, though very infrequently. | * isrc == NULL *will* occur, though very infrequently. | ||||
* | * | ||||
* This also allows a very small gap where a foreign intrusion | * This also allows a very small gap where a foreign intrusion | ||||
* into Xen's interrupt range could be examined by this test. | * into Xen's interrupt range could be examined by this test. | ||||
*/ | */ | ||||
if (__predict_true(isrc != NULL) && | if (__predict_true(isrc != NULL) && | ||||
__predict_true(isrc->xi_intsrc.is_pic == &xen_intr_pic) && | __predict_true(isrc->xi_arch.intsrc.is_pic == | ||||
&xen_intr_pic) && | |||||
isrc->xi_type == EVTCHN_TYPE_UNBOUND) { | isrc->xi_type == EVTCHN_TYPE_UNBOUND) { | ||||
KASSERT(isrc->xi_intsrc.is_handlers == 0, | KASSERT(isrc->xi_arch.intsrc.is_handlers == 0, | ||||
("Free evtchn still has handlers")); | ("Free evtchn still has handlers")); | ||||
isrc->xi_type = type; | isrc->xi_type = type; | ||||
return (isrc); | return (isrc); | ||||
} | } | ||||
} | } | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
Show All 32 Lines | xen_intr_alloc_isrc(enum evtchn_type type) | ||||
vector = first_evtchn_irq + xen_intr_auto_vector_count; | vector = first_evtchn_irq + xen_intr_auto_vector_count; | ||||
xen_intr_auto_vector_count++; | xen_intr_auto_vector_count++; | ||||
KASSERT((intr_lookup_source(vector) == NULL), | KASSERT((intr_lookup_source(vector) == NULL), | ||||
("Trying to use an already allocated vector")); | ("Trying to use an already allocated vector")); | ||||
mtx_unlock(&xen_intr_x86_lock); | mtx_unlock(&xen_intr_x86_lock); | ||||
isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO); | isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO); | ||||
isrc->xi_intsrc.is_pic = &xen_intr_pic; | isrc->xi_arch.intsrc.is_pic = &xen_intr_pic; | ||||
isrc->xi_vector = vector; | isrc->xi_arch.vector = vector; | ||||
isrc->xi_type = type; | isrc->xi_type = type; | ||||
error = intr_register_source(&isrc->xi_intsrc); | error = intr_register_source(&isrc->xi_arch.intsrc); | ||||
if (error != 0) | if (error != 0) | ||||
panic("%s(): failed registering interrupt %u, error=%d\n", | panic("%s(): failed registering interrupt %u, error=%d\n", | ||||
__func__, vector, error); | __func__, vector, error); | ||||
return (isrc); | return (isrc); | ||||
} | } | ||||
/** | /** | ||||
* Attempt to free an active Xen interrupt source object. | * Attempt to free an active Xen interrupt source object. | ||||
* | * | ||||
* \param isrc The interrupt source object to release. | * \param isrc The interrupt source object to release. | ||||
* | * | ||||
* \returns EBUSY if the source is still in use, otherwise 0. | * \returns EBUSY if the source is still in use, otherwise 0. | ||||
*/ | */ | ||||
static int | static int | ||||
xen_intr_release_isrc(struct xenisrc *isrc) | xen_intr_release_isrc(struct xenisrc *isrc) | ||||
{ | { | ||||
KASSERT(isrc->xi_intsrc.is_handlers == 0, | KASSERT(isrc->xi_arch.intsrc.is_handlers == 0, | ||||
("Release called, but xenisrc still in use")); | ("Release called, but xenisrc still in use")); | ||||
mtx_lock(&xen_intr_isrc_lock); | mtx_lock(&xen_intr_isrc_lock); | ||||
if (is_valid_evtchn(isrc->xi_port)) { | if (is_valid_evtchn(isrc->xi_port)) { | ||||
evtchn_mask_port(isrc->xi_port); | evtchn_mask_port(isrc->xi_port); | ||||
evtchn_clear_port(isrc->xi_port); | evtchn_clear_port(isrc->xi_port); | ||||
/* Rebind port to CPU 0. */ | /* Rebind port to CPU 0. */ | ||||
evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); | evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | |||||
#ifdef SMP | #ifdef SMP | ||||
if (type == EVTCHN_TYPE_PORT) { | if (type == EVTCHN_TYPE_PORT) { | ||||
/* | /* | ||||
* By default all interrupts are assigned to vCPU#0 | * By default all interrupts are assigned to vCPU#0 | ||||
* unless specified otherwise, so shuffle them to balance | * unless specified otherwise, so shuffle them to balance | ||||
* the interrupt load. | * the interrupt load. | ||||
*/ | */ | ||||
xen_intr_assign_cpu(&isrc->xi_intsrc, | xen_intr_assign_cpu(&isrc->xi_arch.intsrc, | ||||
apic_cpuid(intr_next_cpu(0))); | apic_cpuid(intr_next_cpu(0))); | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* If a filter or handler function is provided, add it to the event. | * If a filter or handler function is provided, add it to the event. | ||||
* Otherwise the event channel is left masked and without a handler, | * Otherwise the event channel is left masked and without a handler, | ||||
* the caller is in charge of setting that up. | * the caller is in charge of setting that up. | ||||
▲ Show 20 Lines • Show All 114 Lines • ▼ Show 20 Lines | do { | ||||
if (__predict_false(isrc == NULL)) | if (__predict_false(isrc == NULL)) | ||||
continue; | continue; | ||||
/* Make sure we are firing on the right vCPU */ | /* Make sure we are firing on the right vCPU */ | ||||
KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)), | KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)), | ||||
("Received unexpected event on vCPU#%u, event bound to vCPU#%u", | ("Received unexpected event on vCPU#%u, event bound to vCPU#%u", | ||||
PCPU_GET(cpuid), isrc->xi_cpu)); | PCPU_GET(cpuid), isrc->xi_cpu)); | ||||
intr_execute_handlers(&isrc->xi_intsrc, trap_frame); | intr_execute_handlers(&isrc->xi_arch.intsrc, | ||||
trap_frame); | |||||
/* | /* | ||||
* If this is the final port processed, | * If this is the final port processed, | ||||
* we'll pick up here+1 next time. | * we'll pick up here+1 next time. | ||||
*/ | */ | ||||
pc->last_processed_l1i = l1i; | pc->last_processed_l1i = l1i; | ||||
pc->last_processed_l2i = l2i; | pc->last_processed_l2i = l2i; | ||||
▲ Show 20 Lines • Show All 137 Lines • ▼ Show 20 Lines | default: | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
prev = xen_intr_port_to_isrc[isrc->xi_port]; | prev = xen_intr_port_to_isrc[isrc->xi_port]; | ||||
xen_intr_port_to_isrc[isrc->xi_port] = isrc; | xen_intr_port_to_isrc[isrc->xi_port] = isrc; | ||||
#ifdef SMP | #ifdef SMP | ||||
isrc->xi_cpu = 0; | isrc->xi_cpu = 0; | ||||
error = xen_intr_assign_cpu(&isrc->xi_intsrc, cpu); | error = xen_intr_assign_cpu(&isrc->xi_arch.intsrc, cpu); | ||||
if (error) | if (error) | ||||
panic("%s(): unable to rebind Xen channel %u to vCPU%u: %d", | panic("%s(): unable to rebind Xen channel %u to vCPU%u: %d", | ||||
__func__, isrc->xi_port, cpu, error); | __func__, isrc->xi_port, cpu, error); | ||||
#endif | #endif | ||||
evtchn_unmask_port(isrc->xi_port); | evtchn_unmask_port(isrc->xi_port); | ||||
return (prev); | return (prev); | ||||
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | |||||
* | * | ||||
* \return The vector number corresponding to the given interrupt source. | * \return The vector number corresponding to the given interrupt source. | ||||
*/ | */ | ||||
static int | static int | ||||
xen_intr_vector(struct intsrc *base_isrc) | xen_intr_vector(struct intsrc *base_isrc) | ||||
{ | { | ||||
struct xenisrc *isrc = (struct xenisrc *)base_isrc; | struct xenisrc *isrc = (struct xenisrc *)base_isrc; | ||||
return (isrc->xi_vector); | return (isrc->xi_arch.vector); | ||||
} | } | ||||
/** | /** | ||||
* Determine whether or not interrupt events are pending on the | * Determine whether or not interrupt events are pending on the | ||||
* the given interrupt source. | * the given interrupt source. | ||||
* | * | ||||
* \param isrc The interrupt source to query. | * \param isrc The interrupt source to query. | ||||
* | * | ||||
▲ Show 20 Lines • Show All 286 Lines • ▼ Show 20 Lines | xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu, | ||||
} | } | ||||
error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, | error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, | ||||
device_get_nameunit(dev), filter, handler, arg, flags, | device_get_nameunit(dev), filter, handler, arg, flags, | ||||
port_handlep); | port_handlep); | ||||
#ifdef SMP | #ifdef SMP | ||||
if (error == 0) | if (error == 0) | ||||
error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); | error = intr_event_bind(isrc->xi_arch.intsrc.is_event, cpu); | ||||
#endif | #endif | ||||
if (error != 0) { | if (error != 0) { | ||||
evtchn_close_t close = { .port = bind_virq.port }; | evtchn_close_t close = { .port = bind_virq.port }; | ||||
xen_intr_unbind(*port_handlep); | xen_intr_unbind(*port_handlep); | ||||
if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) | if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) | ||||
panic("EVTCHNOP_close failed"); | panic("EVTCHNOP_close failed"); | ||||
return (error); | return (error); | ||||
} | } | ||||
#ifdef SMP | #ifdef SMP | ||||
if (isrc->xi_cpu != cpu) { | if (isrc->xi_cpu != cpu) { | ||||
/* | /* | ||||
* Too early in the boot process for the generic interrupt | * Too early in the boot process for the generic interrupt | ||||
* code to perform the binding. Update our event channel | * code to perform the binding. Update our event channel | ||||
* masks manually so events can't fire on the wrong cpu | * masks manually so events can't fire on the wrong cpu | ||||
* during AP startup. | * during AP startup. | ||||
*/ | */ | ||||
xen_intr_assign_cpu(&isrc->xi_intsrc, cpu); | xen_intr_assign_cpu(&isrc->xi_arch.intsrc, cpu); | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* The Event Channel API opened this port, so it is | * The Event Channel API opened this port, so it is | ||||
* responsible for closing it automatically on unbind. | * responsible for closing it automatically on unbind. | ||||
*/ | */ | ||||
isrc->xi_close = 1; | isrc->xi_close = 1; | ||||
Show All 39 Lines | #ifdef SMP | ||||
if (isrc->xi_cpu != cpu) { | if (isrc->xi_cpu != cpu) { | ||||
/* | /* | ||||
* Too early in the boot process for the generic interrupt | * Too early in the boot process for the generic interrupt | ||||
* code to perform the binding. Update our event channel | * code to perform the binding. Update our event channel | ||||
* masks manually so events can't fire on the wrong cpu | * masks manually so events can't fire on the wrong cpu | ||||
* during AP startup. | * during AP startup. | ||||
*/ | */ | ||||
xen_intr_assign_cpu(&isrc->xi_intsrc, cpu); | xen_intr_assign_cpu(&isrc->xi_arch.intsrc, cpu); | ||||
} | } | ||||
/* | /* | ||||
* The Event Channel API opened this port, so it is | * The Event Channel API opened this port, so it is | ||||
* responsible for closing it automatically on unbind. | * responsible for closing it automatically on unbind. | ||||
*/ | */ | ||||
isrc->xi_close = 1; | isrc->xi_close = 1; | ||||
return (0); | return (0); | ||||
Show All 11 Lines | xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...) | ||||
isrc = xen_intr_isrc_from_handle(port_handle); | isrc = xen_intr_isrc_from_handle(port_handle); | ||||
if (isrc == NULL) | if (isrc == NULL) | ||||
return (EINVAL); | return (EINVAL); | ||||
va_start(ap, fmt); | va_start(ap, fmt); | ||||
vsnprintf(descr, sizeof(descr), fmt, ap); | vsnprintf(descr, sizeof(descr), fmt, ap); | ||||
va_end(ap); | va_end(ap); | ||||
return (intr_describe(isrc->xi_vector, isrc->xi_cookie, descr)); | return (intr_describe(isrc->xi_arch.vector, isrc->xi_cookie, descr)); | ||||
} | } | ||||
void | void | ||||
xen_intr_unbind(xen_intr_handle_t *port_handlep) | xen_intr_unbind(xen_intr_handle_t *port_handlep) | ||||
{ | { | ||||
struct xenisrc *isrc; | struct xenisrc *isrc; | ||||
KASSERT(port_handlep != NULL, | KASSERT(port_handlep != NULL, | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
struct xenisrc *isrc; | struct xenisrc *isrc; | ||||
int error; | int error; | ||||
isrc = xen_intr_isrc_from_handle(handle); | isrc = xen_intr_isrc_from_handle(handle); | ||||
if (isrc == NULL || isrc->xi_cookie != NULL) | if (isrc == NULL || isrc->xi_cookie != NULL) | ||||
return (EINVAL); | return (EINVAL); | ||||
error = intr_add_handler(name, isrc->xi_vector,filter, handler, arg, | error = intr_add_handler(name, isrc->xi_arch.vector, filter, handler, | ||||
flags|INTR_EXCL, &isrc->xi_cookie, 0); | arg, flags|INTR_EXCL, &isrc->xi_cookie, 0); | ||||
if (error != 0) | if (error != 0) | ||||
printf("%s: %s: add handler failed: %d\n", name, __func__, | printf("%s: %s: add handler failed: %d\n", name, __func__, | ||||
error); | error); | ||||
return (error); | return (error); | ||||
} | } | ||||
int | int | ||||
▲ Show 20 Lines • Show All 86 Lines • Show Last 20 Lines |