diff --git a/sys/x86/xen/xen_intr.c b/sys/x86/xen/xen_intr.c
index ebbae25c2d9e..646c9b4728fc 100644
--- a/sys/x86/xen/xen_intr.c
+++ b/sys/x86/xen/xen_intr.c
@@ -1,1375 +1,1372 @@
/******************************************************************************
* xen_intr.c
*
* Xen event and interrupt services for x86 HVM guests.
*
* Copyright (c) 2002-2005, K A Fraser
* Copyright (c) 2005, Intel Corporation <xiaofeng.ling@intel.com>
* Copyright (c) 2012, Spectra Logic Corporation
*
* This file may be distributed separately from the Linux kernel, or
* incorporated into other software packages, subject to the following license:
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this source file (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use, copy, modify,
* merge, publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/interrupt.h>
#include <sys/pcpu.h>
#include <sys/smp.h>
#include <sys/refcount.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/intr_machdep.h>
#include <x86/apicvar.h>
#include <x86/apicreg.h>
#include <machine/smp.h>
#include <machine/stdarg.h>
#include <machine/xen/synch_bitops.h>
#include <machine/xen/xen-os.h>
#include <xen/xen-os.h>
#include <xen/hvm.h>
#include <xen/hypervisor.h>
#include <xen/xen_intr.h>
#include <xen/evtchn/evtchnvar.h>
#include <dev/xen/xenpci/xenpcivar.h>
#include <dev/pci/pcivar.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services");
static u_int first_evtchn_irq;
/**
* Per-cpu event channel processing state.
*/
struct xen_intr_pcpu_data {
/**
* The last event channel bitmap section (level one bit) processed.
* This is used to ensure we scan all ports before
* servicing an already servied port again.
*/
u_int last_processed_l1i;
/**
* The last event channel processed within the event channel
* bitmap being scanned.
*/
u_int last_processed_l2i;
/** Pointer to this CPU's interrupt statistic counter. */
u_long *evtchn_intrcnt;
/**
* A bitmap of ports that can be serviced from this CPU.
* A set bit means interrupt handling is enabled.
*/
u_long evtchn_enabled[sizeof(u_long) * 8];
};
/*
* Start the scan at port 0 by initializing the last scanned
* location as the highest numbered event channel port.
*/
DPCPU_DEFINE_STATIC(struct xen_intr_pcpu_data, xen_intr_pcpu) = {
.last_processed_l1i = LONG_BIT - 1,
.last_processed_l2i = LONG_BIT - 1
};
DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
#define XEN_INVALID_EVTCHN 0 /* Invalid event channel */
#define is_valid_evtchn(x) ((x) != XEN_INVALID_EVTCHN)
struct xenisrc {
struct intsrc xi_intsrc;
enum evtchn_type xi_type;
int xi_cpu; /* VCPU for delivery. */
int xi_vector; /* Global isrc vector number. */
evtchn_port_t xi_port;
int xi_virq;
void *xi_cookie;
u_int xi_close:1; /* close on unbind? */
u_int xi_activehi:1;
u_int xi_edgetrigger:1;
u_int xi_masked:1;
volatile u_int xi_refcount;
};
static void xen_intr_suspend(struct pic *);
static void xen_intr_resume(struct pic *, bool suspend_cancelled);
static void xen_intr_enable_source(struct intsrc *isrc);
static void xen_intr_disable_source(struct intsrc *isrc, int eoi);
static void xen_intr_eoi_source(struct intsrc *isrc);
static void xen_intr_enable_intr(struct intsrc *isrc);
static void xen_intr_disable_intr(struct intsrc *isrc);
static int xen_intr_vector(struct intsrc *isrc);
static int xen_intr_source_pending(struct intsrc *isrc);
static int xen_intr_config_intr(struct intsrc *isrc,
enum intr_trigger trig, enum intr_polarity pol);
static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id);
/**
* PIC interface for all event channel port types except physical IRQs.
*/
struct pic xen_intr_pic = {
.pic_enable_source = xen_intr_enable_source,
.pic_disable_source = xen_intr_disable_source,
.pic_eoi_source = xen_intr_eoi_source,
.pic_enable_intr = xen_intr_enable_intr,
.pic_disable_intr = xen_intr_disable_intr,
.pic_vector = xen_intr_vector,
.pic_source_pending = xen_intr_source_pending,
.pic_suspend = xen_intr_suspend,
.pic_resume = xen_intr_resume,
.pic_config_intr = xen_intr_config_intr,
.pic_assign_cpu = xen_intr_assign_cpu
};
static struct mtx xen_intr_isrc_lock;
static u_int xen_intr_auto_vector_count;
static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS];
/*------------------------- Private Functions --------------------------------*/
/**
* Retrieve a handle for a Xen interrupt source.
*
* \param isrc A valid Xen interrupt source structure.
*
* \returns A handle suitable for use with xen_intr_isrc_from_handle()
* to retrieve the original Xen interrupt source structure.
*/
static inline xen_intr_handle_t
xen_intr_handle_from_isrc(struct xenisrc *isrc)
{
return (isrc);
}
/**
* Lookup a Xen interrupt source object given an interrupt binding handle.
*
* \param handle A handle initialized by a previous call to
* xen_intr_bind_isrc().
*
* \returns A pointer to the Xen interrupt source object associated
* with the given interrupt handle. NULL if no association
* currently exists.
*/
static inline struct xenisrc *
xen_intr_isrc_from_handle(xen_intr_handle_t handle)
{
return ((struct xenisrc *)handle);
}
/**
* Disable signal delivery for an event channel port on the
* specified CPU.
*
* \param port The event channel port to mask.
*
* This API is used to manage the port<=>CPU binding of event
* channel handlers.
*
* \note This operation does not preclude reception of an event
* for this event channel on another CPU. To mask the
* event channel globally, use evtchn_mask().
*/
static inline void
evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port)
{
struct xen_intr_pcpu_data *pcpu;
pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
xen_clear_bit(port, pcpu->evtchn_enabled);
}
/**
* Enable signal delivery for an event channel port on the
* specified CPU.
*
* \param port The event channel port to unmask.
*
* This API is used to manage the port<=>CPU binding of event
* channel handlers.
*
* \note This operation does not guarantee that event delivery
* is enabled for this event channel port. The port must
* also be globally enabled. See evtchn_unmask().
*/
static inline void
evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port)
{
struct xen_intr_pcpu_data *pcpu;
pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
xen_set_bit(port, pcpu->evtchn_enabled);
}
/**
* Allocate and register a per-cpu Xen upcall interrupt counter.
*
* \param cpu The cpu for which to register this interrupt count.
*/
static void
xen_intr_intrcnt_add(u_int cpu)
{
char buf[MAXCOMLEN + 1];
struct xen_intr_pcpu_data *pcpu;
pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
if (pcpu->evtchn_intrcnt != NULL)
return;
snprintf(buf, sizeof(buf), "cpu%d:xen", cpu);
intrcnt_add(buf, &pcpu->evtchn_intrcnt);
}
/**
* Search for an already allocated but currently unused Xen interrupt
* source object.
*
* \param type Restrict the search to interrupt sources of the given
* type.
*
* \return A pointer to a free Xen interrupt source object or NULL.
*/
static struct xenisrc *
xen_intr_find_unused_isrc(enum evtchn_type type)
{
int isrc_idx;
KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held"));
for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) {
struct xenisrc *isrc;
u_int vector;
vector = first_evtchn_irq + isrc_idx;
isrc = (struct xenisrc *)intr_lookup_source(vector);
if (isrc != NULL
&& isrc->xi_type == EVTCHN_TYPE_UNBOUND) {
KASSERT(isrc->xi_intsrc.is_handlers == 0,
("Free evtchn still has handlers"));
isrc->xi_type = type;
return (isrc);
}
}
return (NULL);
}
/**
* Allocate a Xen interrupt source object.
*
* \param type The type of interrupt source to create.
*
* \return A pointer to a newly allocated Xen interrupt source
* object or NULL.
*/
static struct xenisrc *
xen_intr_alloc_isrc(enum evtchn_type type)
{
static int warned;
struct xenisrc *isrc;
unsigned int vector;
KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held"));
if (xen_intr_auto_vector_count > NR_EVENT_CHANNELS) {
if (!warned) {
warned = 1;
- printf("xen_intr_alloc: Event channels exhausted.\n");
+ printf("%s: Event channels exhausted.\n", __func__);
}
return (NULL);
}
vector = first_evtchn_irq + xen_intr_auto_vector_count;
xen_intr_auto_vector_count++;
KASSERT((intr_lookup_source(vector) == NULL),
("Trying to use an already allocated vector"));
mtx_unlock(&xen_intr_isrc_lock);
isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO);
isrc->xi_intsrc.is_pic = &xen_intr_pic;
isrc->xi_vector = vector;
isrc->xi_type = type;
intr_register_source(&isrc->xi_intsrc);
mtx_lock(&xen_intr_isrc_lock);
return (isrc);
}
/**
* Attempt to free an active Xen interrupt source object.
*
* \param isrc The interrupt source object to release.
*
* \returns EBUSY if the source is still in use, otherwise 0.
*/
static int
xen_intr_release_isrc(struct xenisrc *isrc)
{
mtx_lock(&xen_intr_isrc_lock);
KASSERT(isrc->xi_intsrc.is_handlers == 0,
("Release called, but xenisrc still in use"));
evtchn_mask_port(isrc->xi_port);
evtchn_clear_port(isrc->xi_port);
/* Rebind port to CPU 0. */
evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
evtchn_cpu_unmask_port(0, isrc->xi_port);
if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) {
struct evtchn_close close = { .port = isrc->xi_port };
if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
panic("EVTCHNOP_close failed");
}
xen_intr_port_to_isrc[isrc->xi_port] = NULL;
isrc->xi_cpu = 0;
isrc->xi_type = EVTCHN_TYPE_UNBOUND;
isrc->xi_port = 0;
isrc->xi_cookie = NULL;
mtx_unlock(&xen_intr_isrc_lock);
return (0);
}
/**
* Associate an interrupt handler with an already allocated local Xen
* event channel port.
*
* \param isrcp The returned Xen interrupt object associated with
* the specified local port.
* \param local_port The event channel to bind.
* \param type The event channel type of local_port.
* \param intr_owner The device making this bind request.
* \param filter An interrupt filter handler. Specify NULL
* to always dispatch to the ithread handler.
* \param handler An interrupt ithread handler. Optional (can
* specify NULL) if all necessary event actions
* are performed by filter.
* \param arg Argument to present to both filter and handler.
* \param irqflags Interrupt handler flags. See sys/bus.h.
* \param handlep Pointer to an opaque handle used to manage this
* registration.
*
* \returns 0 on success, otherwise an errno.
*/
static int
xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port,
enum evtchn_type type, const char *intr_owner, driver_filter_t filter,
driver_intr_t handler, void *arg, enum intr_type flags,
xen_intr_handle_t *port_handlep)
{
struct xenisrc *isrc;
int error;
*isrcp = NULL;
if (port_handlep == NULL) {
- printf("%s: xen_intr_bind_isrc: Bad event handle\n",
- intr_owner);
+ printf("%s: %s: Bad event handle\n", intr_owner, __func__);
return (EINVAL);
}
mtx_lock(&xen_intr_isrc_lock);
isrc = xen_intr_find_unused_isrc(type);
if (isrc == NULL) {
isrc = xen_intr_alloc_isrc(type);
if (isrc == NULL) {
mtx_unlock(&xen_intr_isrc_lock);
return (ENOSPC);
}
}
isrc->xi_port = local_port;
xen_intr_port_to_isrc[local_port] = isrc;
refcount_init(&isrc->xi_refcount, 1);
mtx_unlock(&xen_intr_isrc_lock);
/* Assign the opaque handler */
*port_handlep = xen_intr_handle_from_isrc(isrc);
#ifdef SMP
if (type == EVTCHN_TYPE_PORT) {
/*
* By default all interrupts are assigned to vCPU#0
* unless specified otherwise, so shuffle them to balance
* the interrupt load.
*/
xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu(0));
}
#endif
if (filter == NULL && handler == NULL) {
/*
* No filter/handler provided, leave the event channel
* masked and without a valid handler, the caller is
* in charge of setting that up.
*/
*isrcp = isrc;
return (0);
}
error = xen_intr_add_handler(intr_owner, filter, handler, arg, flags,
*port_handlep);
if (error != 0) {
xen_intr_release_isrc(isrc);
return (error);
}
*isrcp = isrc;
return (0);
}
/**
* Determine the event channel ports at the given section of the
* event port bitmap which have pending events for the given cpu.
*
* \param pcpu The Xen interrupt pcpu data for the cpu being querried.
* \param sh The Xen shared info area.
* \param idx The index of the section of the event channel bitmap to
* inspect.
*
* \returns A u_long with bits set for every event channel with pending
* events.
*/
static inline u_long
xen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh,
u_int idx)
{
CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(sh->evtchn_pending[0]));
CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(pcpu->evtchn_enabled[0]));
CTASSERT(sizeof(sh->evtchn_mask) == sizeof(sh->evtchn_pending));
CTASSERT(sizeof(sh->evtchn_mask) == sizeof(pcpu->evtchn_enabled));
return (sh->evtchn_pending[idx]
& ~sh->evtchn_mask[idx]
& pcpu->evtchn_enabled[idx]);
}
/**
* Interrupt handler for processing all Xen event channel events.
*
* \param trap_frame The trap frame context for the current interrupt.
*/
void
xen_intr_handle_upcall(struct trapframe *trap_frame)
{
u_int l1i, l2i, port, cpu;
u_long masked_l1, masked_l2;
struct xenisrc *isrc;
shared_info_t *s;
vcpu_info_t *v;
struct xen_intr_pcpu_data *pc;
u_long l1, l2;
/*
* Disable preemption in order to always check and fire events
* on the right vCPU
*/
critical_enter();
cpu = PCPU_GET(cpuid);
pc = DPCPU_PTR(xen_intr_pcpu);
s = HYPERVISOR_shared_info;
v = DPCPU_GET(vcpu_info);
if (xen_hvm_domain() && !xen_vector_callback_enabled) {
KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU"));
}
v->evtchn_upcall_pending = 0;
#if 0
#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
/* Clear master flag /before/ clearing selector flag. */
wmb();
#endif
#endif
l1 = atomic_readandclear_long(&v->evtchn_pending_sel);
l1i = pc->last_processed_l1i;
l2i = pc->last_processed_l2i;
(*pc->evtchn_intrcnt)++;
while (l1 != 0) {
l1i = (l1i + 1) % LONG_BIT;
masked_l1 = l1 & ((~0UL) << l1i);
if (masked_l1 == 0) {
/*
* if we masked out all events, wrap around
* to the beginning.
*/
l1i = LONG_BIT - 1;
l2i = LONG_BIT - 1;
continue;
}
l1i = ffsl(masked_l1) - 1;
do {
l2 = xen_intr_active_ports(pc, s, l1i);
l2i = (l2i + 1) % LONG_BIT;
masked_l2 = l2 & ((~0UL) << l2i);
if (masked_l2 == 0) {
/* if we masked out all events, move on */
l2i = LONG_BIT - 1;
break;
}
l2i = ffsl(masked_l2) - 1;
/* process port */
port = (l1i * LONG_BIT) + l2i;
synch_clear_bit(port, &s->evtchn_pending[0]);
isrc = xen_intr_port_to_isrc[port];
if (__predict_false(isrc == NULL))
continue;
/* Make sure we are firing on the right vCPU */
KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)),
("Received unexpected event on vCPU#%d, event bound to vCPU#%d",
PCPU_GET(cpuid), isrc->xi_cpu));
intr_execute_handlers(&isrc->xi_intsrc, trap_frame);
/*
* If this is the final port processed,
* we'll pick up here+1 next time.
*/
pc->last_processed_l1i = l1i;
pc->last_processed_l2i = l2i;
} while (l2i != LONG_BIT - 1);
l2 = xen_intr_active_ports(pc, s, l1i);
if (l2 == 0) {
/*
* We handled all ports, so we can clear the
* selector bit.
*/
l1 &= ~(1UL << l1i);
}
}
if (xen_evtchn_needs_ack)
lapic_eoi();
critical_exit();
}
static int
xen_intr_init(void *dummy __unused)
{
shared_info_t *s = HYPERVISOR_shared_info;
struct xen_intr_pcpu_data *pcpu;
int i;
if (!xen_domain())
return (0);
mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF);
/*
* Set the per-cpu mask of CPU#0 to enable all, since by default all
* event channels are bound to CPU#0.
*/
CPU_FOREACH(i) {
pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
sizeof(pcpu->evtchn_enabled));
}
for (i = 0; i < nitems(s->evtchn_mask); i++)
atomic_store_rel_long(&s->evtchn_mask[i], ~0);
intr_register_pic(&xen_intr_pic);
if (bootverbose)
printf("Xen interrupt system initialized\n");
return (0);
}
SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL);
static void
xen_intrcnt_init(void *dummy __unused)
{
unsigned int i;
if (!xen_domain())
return;
/*
* Register interrupt count manually as we aren't guaranteed to see a
* call to xen_intr_assign_cpu() before our first interrupt.
*/
CPU_FOREACH(i)
xen_intr_intrcnt_add(i);
}
SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL);
void
xen_intr_alloc_irqs(void)
{
if (num_io_irqs > UINT_MAX - NR_EVENT_CHANNELS)
panic("IRQ allocation overflow (num_msi_irqs too high?)");
first_evtchn_irq = num_io_irqs;
num_io_irqs += NR_EVENT_CHANNELS;
}
/*--------------------------- Common PIC Functions ---------------------------*/
/**
* Prepare this PIC for system suspension.
*/
static void
xen_intr_suspend(struct pic *unused)
{
}
static void
xen_rebind_ipi(struct xenisrc *isrc)
{
#ifdef SMP
int cpu = isrc->xi_cpu;
int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
int error;
struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
&bind_ipi);
if (error != 0)
panic("unable to rebind xen IPI: %d", error);
isrc->xi_port = bind_ipi.port;
isrc->xi_cpu = 0;
xen_intr_port_to_isrc[bind_ipi.port] = isrc;
error = xen_intr_assign_cpu(&isrc->xi_intsrc,
cpu_apic_ids[cpu]);
if (error)
panic("unable to bind xen IPI to CPU#%d: %d",
cpu, error);
evtchn_unmask_port(bind_ipi.port);
#else
panic("Resume IPI event channel on UP");
#endif
}
static void
xen_rebind_virq(struct xenisrc *isrc)
{
int cpu = isrc->xi_cpu;
int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
int error;
struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq,
.vcpu = vcpu_id };
error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
&bind_virq);
if (error != 0)
panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error);
isrc->xi_port = bind_virq.port;
isrc->xi_cpu = 0;
xen_intr_port_to_isrc[bind_virq.port] = isrc;
#ifdef SMP
error = xen_intr_assign_cpu(&isrc->xi_intsrc,
cpu_apic_ids[cpu]);
if (error)
panic("unable to bind xen VIRQ#%d to CPU#%d: %d",
isrc->xi_virq, cpu, error);
#endif
evtchn_unmask_port(bind_virq.port);
}
/**
* Return this PIC to service after being suspended.
*/
static void
xen_intr_resume(struct pic *unused, bool suspend_cancelled)
{
shared_info_t *s = HYPERVISOR_shared_info;
struct xenisrc *isrc;
u_int isrc_idx;
int i;
if (suspend_cancelled)
return;
/* Reset the per-CPU masks */
CPU_FOREACH(i) {
struct xen_intr_pcpu_data *pcpu;
pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
sizeof(pcpu->evtchn_enabled));
}
/* Mask all event channels. */
for (i = 0; i < nitems(s->evtchn_mask); i++)
atomic_store_rel_long(&s->evtchn_mask[i], ~0);
/* Remove port -> isrc mappings */
memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc));
/* Free unused isrcs and rebind VIRQs and IPIs */
for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx++) {
u_int vector;
vector = first_evtchn_irq + isrc_idx;
isrc = (struct xenisrc *)intr_lookup_source(vector);
if (isrc != NULL) {
isrc->xi_port = 0;
switch (isrc->xi_type) {
case EVTCHN_TYPE_IPI:
xen_rebind_ipi(isrc);
break;
case EVTCHN_TYPE_VIRQ:
xen_rebind_virq(isrc);
break;
default:
break;
}
}
}
}
/**
* Disable a Xen interrupt source.
*
* \param isrc The interrupt source to disable.
*/
static void
xen_intr_disable_intr(struct intsrc *base_isrc)
{
struct xenisrc *isrc = (struct xenisrc *)base_isrc;
evtchn_mask_port(isrc->xi_port);
}
/**
* Determine the global interrupt vector number for
* a Xen interrupt source.
*
* \param isrc The interrupt source to query.
*
* \return The vector number corresponding to the given interrupt source.
*/
static int
xen_intr_vector(struct intsrc *base_isrc)
{
struct xenisrc *isrc = (struct xenisrc *)base_isrc;
return (isrc->xi_vector);
}
/**
* Determine whether or not interrupt events are pending on the
* the given interrupt source.
*
* \param isrc The interrupt source to query.
*
* \returns 0 if no events are pending, otherwise non-zero.
*/
static int
xen_intr_source_pending(struct intsrc *isrc)
{
/*
* EventChannels are edge triggered and never masked.
* There can be no pending events.
*/
return (0);
}
/**
* Perform configuration of an interrupt source.
*
* \param isrc The interrupt source to configure.
* \param trig Edge or level.
* \param pol Active high or low.
*
* \returns 0 if no events are pending, otherwise non-zero.
*/
static int
xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig,
enum intr_polarity pol)
{
/* Configuration is only possible via the evtchn apis. */
return (ENODEV);
}
/**
* Configure CPU affinity for interrupt source event delivery.
*
* \param isrc The interrupt source to configure.
* \param apic_id The apic id of the CPU for handling future events.
*
* \returns 0 if successful, otherwise an errno.
*/
static int
xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id)
{
#ifdef SMP
struct evtchn_bind_vcpu bind_vcpu;
struct xenisrc *isrc;
u_int to_cpu, vcpu_id;
int error, masked;
if (xen_vector_callback_enabled == 0)
return (EOPNOTSUPP);
to_cpu = apic_cpuid(apic_id);
vcpu_id = pcpu_find(to_cpu)->pc_vcpu_id;
mtx_lock(&xen_intr_isrc_lock);
isrc = (struct xenisrc *)base_isrc;
if (!is_valid_evtchn(isrc->xi_port)) {
mtx_unlock(&xen_intr_isrc_lock);
return (EINVAL);
}
/*
* Mask the event channel while binding it to prevent interrupt
* delivery with an inconsistent state in isrc->xi_cpu.
*/
masked = evtchn_test_and_set_mask(isrc->xi_port);
if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) ||
(isrc->xi_type == EVTCHN_TYPE_IPI)) {
/*
* Virtual IRQs are associated with a cpu by
* the Hypervisor at evtchn_bind_virq time, so
* all we need to do is update the per-CPU masks.
*/
evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
isrc->xi_cpu = to_cpu;
evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
goto out;
}
bind_vcpu.port = isrc->xi_port;
bind_vcpu.vcpu = vcpu_id;
error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu);
if (isrc->xi_cpu != to_cpu) {
if (error == 0) {
/* Commit to new binding by removing the old one. */
evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
isrc->xi_cpu = to_cpu;
evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
}
}
out:
if (masked == 0)
evtchn_unmask_port(isrc->xi_port);
mtx_unlock(&xen_intr_isrc_lock);
return (0);
#else
return (EOPNOTSUPP);
#endif
}
/*------------------- Virtual Interrupt Source PIC Functions -----------------*/
/*
* Mask a level triggered interrupt source.
*
* \param isrc The interrupt source to mask (if necessary).
* \param eoi If non-zero, perform any necessary end-of-interrupt
* acknowledgements.
*/
static void
xen_intr_disable_source(struct intsrc *base_isrc, int eoi)
{
struct xenisrc *isrc;
isrc = (struct xenisrc *)base_isrc;
/*
* NB: checking if the event channel is already masked is
* needed because the event channel user-space device
* masks event channels on its filter as part of its
* normal operation, and those shouldn't be automatically
* unmasked by the generic interrupt code. The event channel
* device will unmask them when needed.
*/
isrc->xi_masked = !!evtchn_test_and_set_mask(isrc->xi_port);
}
/*
* Unmask a level triggered interrupt source.
*
* \param isrc The interrupt source to unmask (if necessary).
*/
static void
xen_intr_enable_source(struct intsrc *base_isrc)
{
struct xenisrc *isrc;
isrc = (struct xenisrc *)base_isrc;
if (isrc->xi_masked == 0)
evtchn_unmask_port(isrc->xi_port);
}
/*
* Perform any necessary end-of-interrupt acknowledgements.
*
* \param isrc The interrupt source to EOI.
*/
static void
xen_intr_eoi_source(struct intsrc *base_isrc)
{
}
/*
* Enable and unmask the interrupt source.
*
* \param isrc The interrupt source to enable.
*/
static void
xen_intr_enable_intr(struct intsrc *base_isrc)
{
struct xenisrc *isrc = (struct xenisrc *)base_isrc;
evtchn_unmask_port(isrc->xi_port);
}
/*--------------------------- Public Functions -------------------------------*/
/*------- API comments for these methods can be found in xen/xenintr.h -------*/
int
xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port,
driver_filter_t filter, driver_intr_t handler, void *arg,
enum intr_type flags, xen_intr_handle_t *port_handlep)
{
struct xenisrc *isrc;
int error;
error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT,
device_get_nameunit(dev), filter, handler, arg, flags,
port_handlep);
if (error != 0)
return (error);
/*
* The Event Channel API didn't open this port, so it is not
* responsible for closing it automatically on unbind.
*/
isrc->xi_close = 0;
return (0);
}
int
xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain,
driver_filter_t filter, driver_intr_t handler, void *arg,
enum intr_type flags, xen_intr_handle_t *port_handlep)
{
struct xenisrc *isrc;
struct evtchn_alloc_unbound alloc_unbound;
int error;
alloc_unbound.dom = DOMID_SELF;
alloc_unbound.remote_dom = remote_domain;
error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
&alloc_unbound);
if (error != 0) {
/*
* XXX Trap Hypercall error code Linuxisms in
* the HYPERCALL layer.
*/
return (-error);
}
error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT,
device_get_nameunit(dev), filter, handler, arg, flags,
port_handlep);
if (error != 0) {
evtchn_close_t close = { .port = alloc_unbound.port };
if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
panic("EVTCHNOP_close failed");
return (error);
}
isrc->xi_close = 1;
return (0);
}
int
xen_intr_bind_remote_port(device_t dev, u_int remote_domain,
u_int remote_port, driver_filter_t filter, driver_intr_t handler,
void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep)
{
struct xenisrc *isrc;
struct evtchn_bind_interdomain bind_interdomain;
int error;
bind_interdomain.remote_dom = remote_domain;
bind_interdomain.remote_port = remote_port;
error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
&bind_interdomain);
if (error != 0) {
/*
* XXX Trap Hypercall error code Linuxisms in
* the HYPERCALL layer.
*/
return (-error);
}
error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port,
EVTCHN_TYPE_PORT, device_get_nameunit(dev), filter, handler, arg,
flags, port_handlep);
if (error) {
evtchn_close_t close = { .port = bind_interdomain.local_port };
if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
panic("EVTCHNOP_close failed");
return (error);
}
/*
* The Event Channel API opened this port, so it is
* responsible for closing it automatically on unbind.
*/
isrc->xi_close = 1;
return (0);
}
int
xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu,
driver_filter_t filter, driver_intr_t handler, void *arg,
enum intr_type flags, xen_intr_handle_t *port_handlep)
{
int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
struct xenisrc *isrc;
struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id };
int error;
isrc = NULL;
error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
if (error != 0) {
/*
* XXX Trap Hypercall error code Linuxisms in
* the HYPERCALL layer.
*/
return (-error);
}
error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ,
device_get_nameunit(dev), filter, handler, arg, flags,
port_handlep);
#ifdef SMP
if (error == 0)
error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
#endif
if (error != 0) {
evtchn_close_t close = { .port = bind_virq.port };
xen_intr_unbind(*port_handlep);
if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
panic("EVTCHNOP_close failed");
return (error);
}
#ifdef SMP
if (isrc->xi_cpu != cpu) {
/*
* Too early in the boot process for the generic interrupt
* code to perform the binding. Update our event channel
* masks manually so events can't fire on the wrong cpu
* during AP startup.
*/
xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
}
#endif
/*
* The Event Channel API opened this port, so it is
* responsible for closing it automatically on unbind.
*/
isrc->xi_close = 1;
isrc->xi_virq = virq;
return (0);
}
int
xen_intr_alloc_and_bind_ipi(u_int cpu, driver_filter_t filter,
enum intr_type flags, xen_intr_handle_t *port_handlep)
{
#ifdef SMP
int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
struct xenisrc *isrc;
struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
/* Same size as the one used by intr_handler->ih_name. */
char name[MAXCOMLEN + 1];
int error;
isrc = NULL;
error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
if (error != 0) {
/*
* XXX Trap Hypercall error code Linuxisms in
* the HYPERCALL layer.
*/
return (-error);
}
snprintf(name, sizeof(name), "cpu%u", cpu);
error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI,
name, filter, NULL, NULL, flags, port_handlep);
if (error != 0) {
evtchn_close_t close = { .port = bind_ipi.port };
xen_intr_unbind(*port_handlep);
if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
panic("EVTCHNOP_close failed");
return (error);
}
if (isrc->xi_cpu != cpu) {
/*
* Too early in the boot process for the generic interrupt
* code to perform the binding. Update our event channel
* masks manually so events can't fire on the wrong cpu
* during AP startup.
*/
xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
}
/*
* The Event Channel API opened this port, so it is
* responsible for closing it automatically on unbind.
*/
isrc->xi_close = 1;
return (0);
#else
return (EOPNOTSUPP);
#endif
}
int
xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...)
{
char descr[MAXCOMLEN + 1];
struct xenisrc *isrc;
va_list ap;
isrc = xen_intr_isrc_from_handle(port_handle);
if (isrc == NULL)
return (EINVAL);
va_start(ap, fmt);
vsnprintf(descr, sizeof(descr), fmt, ap);
va_end(ap);
return (intr_describe(isrc->xi_vector, isrc->xi_cookie, descr));
}
void
xen_intr_unbind(xen_intr_handle_t *port_handlep)
{
struct xenisrc *isrc;
KASSERT(port_handlep != NULL,
- ("NULL xen_intr_handle_t passed to xen_intr_unbind"));
+ ("NULL xen_intr_handle_t passed to %s", __func__));
isrc = xen_intr_isrc_from_handle(*port_handlep);
*port_handlep = NULL;
if (isrc == NULL)
return;
mtx_lock(&xen_intr_isrc_lock);
if (refcount_release(&isrc->xi_refcount) == 0) {
mtx_unlock(&xen_intr_isrc_lock);
return;
}
mtx_unlock(&xen_intr_isrc_lock);
if (isrc->xi_cookie != NULL)
intr_remove_handler(isrc->xi_cookie);
xen_intr_release_isrc(isrc);
}
void
xen_intr_signal(xen_intr_handle_t handle)
{
struct xenisrc *isrc;
isrc = xen_intr_isrc_from_handle(handle);
if (isrc != NULL) {
KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT ||
isrc->xi_type == EVTCHN_TYPE_IPI,
("evtchn_signal on something other than a local port"));
struct evtchn_send send = { .port = isrc->xi_port };
(void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
}
}
evtchn_port_t
xen_intr_port(xen_intr_handle_t handle)
{
struct xenisrc *isrc;
isrc = xen_intr_isrc_from_handle(handle);
if (isrc == NULL)
return (0);
return (isrc->xi_port);
}
int
xen_intr_add_handler(const char *name, driver_filter_t filter,
driver_intr_t handler, void *arg, enum intr_type flags,
xen_intr_handle_t handle)
{
struct xenisrc *isrc;
int error;
isrc = xen_intr_isrc_from_handle(handle);
if (isrc == NULL || isrc->xi_cookie != NULL)
return (EINVAL);
error = intr_add_handler(name, isrc->xi_vector,filter, handler, arg,
flags|INTR_EXCL, &isrc->xi_cookie, 0);
- if (error != 0) {
- printf(
- "%s: xen_intr_add_handler: intr_add_handler failed: %d\n",
- name, error);
- }
+ if (error != 0)
+ printf("%s: %s: add handler failed: %d\n", name, __func__,
+ error);
return (error);
}
int
xen_intr_get_evtchn_from_port(evtchn_port_t port, xen_intr_handle_t *handlep)
{
if (!is_valid_evtchn(port) || port >= NR_EVENT_CHANNELS)
return (EINVAL);
if (handlep == NULL) {
return (EINVAL);
}
mtx_lock(&xen_intr_isrc_lock);
if (xen_intr_port_to_isrc[port] == NULL) {
mtx_unlock(&xen_intr_isrc_lock);
return (EINVAL);
}
refcount_acquire(&xen_intr_port_to_isrc[port]->xi_refcount);
mtx_unlock(&xen_intr_isrc_lock);
/* Assign the opaque handler */
*handlep = xen_intr_handle_from_isrc(xen_intr_port_to_isrc[port]);
return (0);
}
#ifdef DDB
static const char *
xen_intr_print_type(enum evtchn_type type)
{
static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = {
[EVTCHN_TYPE_UNBOUND] = "UNBOUND",
[EVTCHN_TYPE_VIRQ] = "VIRQ",
[EVTCHN_TYPE_IPI] = "IPI",
[EVTCHN_TYPE_PORT] = "PORT",
};
if (type >= EVTCHN_TYPE_COUNT)
return ("UNKNOWN");
return (evtchn_type_to_string[type]);
}
static void
xen_intr_dump_port(struct xenisrc *isrc)
{
struct xen_intr_pcpu_data *pcpu;
shared_info_t *s = HYPERVISOR_shared_info;
int i;
db_printf("Port %d Type: %s\n",
isrc->xi_port, xen_intr_print_type(isrc->xi_type));
if (isrc->xi_type == EVTCHN_TYPE_VIRQ)
db_printf("\tVirq: %d\n", isrc->xi_virq);
db_printf("\tMasked: %d Pending: %d\n",
!!xen_test_bit(isrc->xi_port, &s->evtchn_mask[0]),
!!xen_test_bit(isrc->xi_port, &s->evtchn_pending[0]));
db_printf("\tPer-CPU Masks: ");
CPU_FOREACH(i) {
pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
db_printf("cpu#%d: %d ", i,
!!xen_test_bit(isrc->xi_port, pcpu->evtchn_enabled));
}
db_printf("\n");
}
DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn)
{
int i;
if (!xen_domain()) {
db_printf("Only available on Xen guests\n");
return;
}
for (i = 0; i < NR_EVENT_CHANNELS; i++) {
struct xenisrc *isrc;
isrc = xen_intr_port_to_isrc[i];
if (isrc == NULL)
continue;
xen_intr_dump_port(isrc);
}
}
#endif /* DDB */