Page MenuHomeFreeBSD
Differential D20389 Diff 60908 sys/amd64/vmm/io/vioapic.c

Changeset View

Standalone View

View Options

sys/amd64/vmm/io/vioapic.c

/*- /*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
* *
* Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com> * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
* Copyright (c) 2013 Neel Natu <neel@freebsd.org> * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
* All rights reserved. * All rights reserved.
* Copyright (c) 2019 Joyent, Inc.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions * modification, are permitted provided that the following conditions
* are met: * are met:
* 1. Redistributions of source code must retain the above copyright * 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer. * notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright * 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the * notice, this list of conditions and the following disclaimer in the
Show All 16 Lines
#include <sys/cdefs.h> #include <sys/cdefs.h>
__FBSDID("$FreeBSD$"); __FBSDID("$FreeBSD$");
#include <sys/param.h> #include <sys/param.h>
#include <sys/queue.h> #include <sys/queue.h>
#include <sys/lock.h> #include <sys/lock.h>
#include <sys/mutex.h> #include <sys/mutex.h>
#include <sys/sx.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <sys/kernel.h> #include <sys/kernel.h>
#include <sys/malloc.h> #include <sys/malloc.h>
#include <sys/cpuset.h>
markjUnsubmitted
Not Done
Inline Actions

This should be sorted after param.h. (param.h is a special header that always gets included first.)

markj: This should be sorted after param.h. (param.h is a special header that always gets included…
#include <x86/apicreg.h> #include <x86/apicreg.h>
#include <machine/vmm.h> #include <machine/vmm.h>
#include "vmm_ktr.h" #include "vmm_ktr.h"
#include "vmm_lapic.h" #include "vmm_lapic.h"
#include "vlapic.h" #include "vlapic.h"
#include "vioapic.h" #include "vioapic.h"
#define IOREGSEL 0x00 #define IOREGSEL 0x00
#define IOWIN 0x10 #define IOWIN 0x10
#define REDIR_ENTRIES 32 #define REDIR_ENTRIES 32
#define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS)) #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
struct vioapic { struct vioapic {
struct vm *vm; struct vm *vm;
struct mtx mtx; struct mtx mtx;
struct sx wlock;
uint32_t id; uint32_t id;
uint32_t ioregsel; uint32_t ioregsel;
struct { struct {
uint64_t reg; uint64_t reg;
int acnt; /* sum of pin asserts (+1) and deasserts (-1) */ int acnt; /* sum of pin asserts (+1) and deasserts (-1) */
} rtbl[REDIR_ENTRIES]; } rtbl[REDIR_ENTRIES];
}; };
#define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx)) #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx))
#define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx)) #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx))
#define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx)) #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx))
#define VIOAPIC_WRITE_LOCK(vioapic) sx_xlock(&(vioapic)->wlock)
#define VIOAPIC_WRITE_UNLOCK(vioapic) sx_xunlock(&(vioapic)->wlock)
#define VIOAPIC_WRITE_LOCKED(vioapic) sx_xlocked(&(vioapic)->wlock)
static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic"); static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
#define VIOAPIC_CTR1(vioapic, fmt, a1) \ #define VIOAPIC_CTR1(vioapic, fmt, a1) \
VM_CTR1((vioapic)->vm, fmt, a1) VM_CTR1((vioapic)->vm, fmt, a1)
#define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \ #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \
VM_CTR2((vioapic)->vm, fmt, a1, a2) VM_CTR2((vioapic)->vm, fmt, a1, a2)
▲ Show 20 Lines • Show All 138 Lines • ▼ Show 20 Lines
int int
vioapic_pulse_irq(struct vm *vm, int irq) vioapic_pulse_irq(struct vm *vm, int irq)
{ {
return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE)); return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
} }
#define REDIR_IS_PHYS(reg) (((reg) & IOART_DESTMOD) == IOART_DESTPHY)
#define REDIR_IS_LOWPRIO(reg) (((reg) & IOART_DELMOD) == IOART_DELLOPRI)
/* Level-triggered interrupts only valid in fixed and low-priority modes */
#define REDIR_IS_LVLTRIG(reg) \
(((reg) & IOART_TRGRLVL) != 0 && \
(((reg) & IOART_DELMOD) == IOART_DELFIXED || REDIR_IS_LOWPRIO(reg)))
#define REDIR_DEST(reg) ((reg) >> (32 + APIC_ID_SHIFT))
#define REDIR_VECTOR(reg) ((reg) & IOART_INTVEC)
/* /*
* Reset the vlapic's trigger-mode register to reflect the ioapic pin * Given a redirection entry, determine which vCPUs would be targeted.
* configuration.
*/ */
static void static void
vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg) vioapic_calcdest(struct vioapic *vioapic, uint64_t redir_ent, cpuset_t *dmask)
{ {
struct vioapic *vioapic;
struct vlapic *vlapic;
uint32_t low, high, dest;
int delmode, pin, vector;
bool level, phys;
vlapic = vm_lapic(vm, vcpuid); /*
vioapic = vm_ioapic(vm); * When calculating interrupt destinations with vlapic_calcdest(), the
* legacy xAPIC format is assumed, since the system lacks interrupt
* redirection hardware.
* See vlapic_deliver_intr() for more details.
*/
vlapic_calcdest(vioapic->vm, dmask, REDIR_DEST(redir_ent),
REDIR_IS_PHYS(redir_ent), REDIR_IS_LOWPRIO(redir_ent), false);
}
VIOAPIC_LOCK(vioapic);
/* /*
* Reset all vectors to be edge-triggered. * Across all redirection entries utilizing a specified vector, determine the
* set of vCPUs which would be targeted by a level-triggered interrupt.
*/ */
vlapic_reset_tmr(vlapic); static void
for (pin = 0; pin < REDIR_ENTRIES; pin++) { vioapic_tmr_active(struct vioapic *vioapic, uint8_t vec, cpuset_t *result)
low = vioapic->rtbl[pin].reg; {
high = vioapic->rtbl[pin].reg >> 32; u_int i;
level = low & IOART_TRGRLVL ? true : false; CPU_ZERO(result);
if (!level) if (vec == 0) {
return;
}
for (i = 0; i < REDIR_ENTRIES; i++) {
cpuset_t dest;
const uint64_t val = vioapic->rtbl[i].reg;
if (!REDIR_IS_LVLTRIG(val) || REDIR_VECTOR(val) != vec) {
continue; continue;
}
CPU_ZERO(&dest);
vioapic_calcdest(vioapic, val, &dest);
CPU_OR(result, &dest);
}
}
/* /*
* For a level-triggered 'pin' let the vlapic figure out if * Update TMR state in vLAPICs after changes to vIOAPIC pin configuration
* an assertion on this 'pin' would result in an interrupt
* being delivered to it. If yes, then it will modify the
* TMR bit associated with this vector to level-triggered.
*/ */
phys = ((low & IOART_DESTMOD) == IOART_DESTPHY); static void
delmode = low & IOART_DELMOD; vioapic_update_tmrs(struct vioapic *vioapic, int vcpuid, uint64_t oldval,
vector = low & IOART_INTVEC; uint64_t newval)
dest = high >> APIC_ID_SHIFT; {
vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector); cpuset_t active, allset, newset, oldset;
struct vm *vm;
uint8_t newvec, oldvec;
KASSERT(VIOAPIC_LOCKED(vioapic),
("vioapic_update_tmrs: vioapic is not locked"));
markjUnsubmitted
Not Done
Inline Actions

This assertion is redundant.

markj: This assertion is redundant.
KASSERT(VIOAPIC_WRITE_LOCKED(vioapic),
("vioapic_update_tmrs: vioapic write lock not held"));
vm = vioapic->vm;
CPU_ZERO(&allset);
CPU_ZERO(&newset);
CPU_ZERO(&oldset);
newvec = oldvec = 0;
if (REDIR_IS_LVLTRIG(oldval)) {
vioapic_calcdest(vioapic, oldval, &oldset);
CPU_OR(&allset, &oldset);
oldvec = REDIR_VECTOR(oldval);
} }
if (REDIR_IS_LVLTRIG(newval)) {
vioapic_calcdest(vioapic, newval, &newset);
CPU_OR(&allset, &newset);
newvec = REDIR_VECTOR(newval);
}
if (CPU_EMPTY(&allset) ||
(CPU_CMP(&oldset, &newset) == 0 && oldvec == newvec)) {
return;
}
/*
* Since the write to the redirection table has already occurred, a
* scan of level-triggered entries referencing the old vector will find
* only entries which are now currently valid.
*/
vioapic_tmr_active(vioapic, oldvec, &active);
/*
* Drop VIOAPIC_LOCK while updateing TMRs in case any of the affected
* vCPUs require sleeping until they are in an appropriate state.
*/
VIOAPIC_UNLOCK(vioapic); VIOAPIC_UNLOCK(vioapic);
while (!CPU_EMPTY(&allset)) {
struct vlapic *vlapic;
u_int i;
i = CPU_FFS(&allset) - 1;
CPU_CLR(i, &allset);
if (oldvec == newvec &&
CPU_ISSET(i, &oldset) && CPU_ISSET(i, &newset)) {
continue;
} }
if (i != vcpuid) {
vcpu_block_run(vm, i);
}
vlapic = vm_lapic(vm, i);
if (CPU_ISSET(i, &oldset)) {
/*
* Perform the deassertion if no other level-triggered
* IOAPIC entries target this vCPU with the old vector
*
* Note: Sharing of vectors like that should be
* extremely rare in modern operating systems and was
* previously unsupported by the bhyve vIOAPIC.
*/
if (!CPU_ISSET(i, &active)) {
vlapic_tmr_set(vlapic, oldvec, false);
}
}
if (CPU_ISSET(i, &newset)) {
vlapic_tmr_set(vlapic, newvec, true);
}
if (i != vcpuid) {
vcpu_unblock_run(vm, i);
}
}
VIOAPIC_LOCK(vioapic);
}
static uint32_t static uint32_t
vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr) vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr)
{ {
int regnum, pin, rshift; int regnum, pin, rshift;
regnum = addr & 0xff; regnum = addr & 0xff;
switch (regnum) { switch (regnum) {
case IOAPIC_ID: case IOAPIC_ID:
Show All 25 Lines
} }
static void static void
vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data) vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data)
{ {
uint64_t data64, mask64; uint64_t data64, mask64;
uint64_t last, changed; uint64_t last, changed;
int regnum, pin, lshift; int regnum, pin, lshift;
cpuset_t allvcpus;
regnum = addr & 0xff; regnum = addr & 0xff;
switch (regnum) { switch (regnum) {
case IOAPIC_ID: case IOAPIC_ID:
vioapic->id = data & APIC_ID_MASK; vioapic->id = data & APIC_ID_MASK;
break; break;
case IOAPIC_VER: case IOAPIC_VER:
case IOAPIC_ARB: case IOAPIC_ARB:
Show All 19 Lines if (regnum >= IOAPIC_REDTBL &&
vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS; vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS; vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx", VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
pin, vioapic->rtbl[pin].reg); pin, vioapic->rtbl[pin].reg);
/* /*
* If any fields in the redirection table entry (except mask * If any fields in the redirection table entry (except mask
* or polarity) have changed then rendezvous all the vcpus * or polarity) have changed then update the trigger-mode
* to update their vlapic trigger-mode registers. * registers on all the vlapics.
*/ */
changed = last ^ vioapic->rtbl[pin].reg; changed = last ^ vioapic->rtbl[pin].reg;
if (changed & ~(IOART_INTMASK | IOART_INTPOL)) { if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate " VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
"vlapic trigger-mode register", pin); "vlapic trigger-mode register", pin);
VIOAPIC_UNLOCK(vioapic); vioapic_update_tmrs(vioapic, vcpuid, last,
allvcpus = vm_active_cpus(vioapic->vm); vioapic->rtbl[pin].reg);
vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus,
vioapic_update_tmr, NULL);
VIOAPIC_LOCK(vioapic);
} }
/* /*
* Generate an interrupt if the following conditions are met: * Generate an interrupt if the following conditions are met:
* - pin is not masked * - pin is not masked
* - previous interrupt has been EOIed * - previous interrupt has been EOIed
* - pin level is asserted * - pin level is asserted
*/ */
Show All 20 Linesvioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa,
* IOREGSEL (offset 0) and IOWIN (offset 16) registers. * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
*/ */
if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) { if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
if (doread) if (doread)
*data = 0; *data = 0;
return (0); return (0);
} }
if (!doread) {
/*
* When writing the vioapic registers which result in TMR
* updates, an unbounded sleep is possible while waiting for
* certain vCPUs to reach acceptable states. Since the
* VIOAPIC_LOCK mutex cannot be held during such a sleep, an
* additional synchronization mechanism is needed to prevent
* conflicting writes.
*/
VIOAPIC_WRITE_LOCK(vioapic);
}
VIOAPIC_LOCK(vioapic); VIOAPIC_LOCK(vioapic);
if (offset == IOREGSEL) { if (offset == IOREGSEL) {
if (doread) if (doread)
*data = vioapic->ioregsel; *data = vioapic->ioregsel;
else else
vioapic->ioregsel = *data; vioapic->ioregsel = *data;
} else { } else {
if (doread) { if (doread) {
*data = vioapic_read(vioapic, vcpuid, *data = vioapic_read(vioapic, vcpuid,
vioapic->ioregsel); vioapic->ioregsel);
} else { } else {
vioapic_write(vioapic, vcpuid, vioapic->ioregsel, vioapic_write(vioapic, vcpuid, vioapic->ioregsel,
*data); *data);
} }
} }
VIOAPIC_UNLOCK(vioapic); VIOAPIC_UNLOCK(vioapic);
if (!doread) {
VIOAPIC_WRITE_UNLOCK(vioapic);
}
return (0); return (0);
} }
int int
vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval, vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval,
int size, void *arg) int size, void *arg)
{ {
int error; int error;
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines
{ {
int i; int i;
struct vioapic *vioapic; struct vioapic *vioapic;
vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO); vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
vioapic->vm = vm; vioapic->vm = vm;
mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN); mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
sx_init(&vioapic->wlock, "vioapic write lock");
/* Initialize all redirection entries to mask all interrupts */ /* Initialize all redirection entries to mask all interrupts */
for (i = 0; i < REDIR_ENTRIES; i++) for (i = 0; i < REDIR_ENTRIES; i++)
vioapic->rtbl[i].reg = 0x0001000000010000UL; vioapic->rtbl[i].reg = 0x0001000000010000UL;
return (vioapic); return (vioapic);
} }
void void
vioapic_cleanup(struct vioapic *vioapic) vioapic_cleanup(struct vioapic *vioapic)
{ {
sx_destroy(&vioapic->wlock);
free(vioapic, M_VIOAPIC); free(vioapic, M_VIOAPIC);
} }
int int
vioapic_pincount(struct vm *vm) vioapic_pincount(struct vm *vm)
{ {
return (REDIR_ENTRIES); return (REDIR_ENTRIES);
} }