Changeset View
Changeset View
Standalone View
Standalone View
usr.sbin/bhyve/pci_passthru.c
Show First 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | |||||
#include <string.h> | #include <string.h> | ||||
#include <err.h> | #include <err.h> | ||||
#include <errno.h> | #include <errno.h> | ||||
#include <fcntl.h> | #include <fcntl.h> | ||||
#include <sysexits.h> | #include <sysexits.h> | ||||
#include <unistd.h> | #include <unistd.h> | ||||
#include <machine/vmm.h> | #include <machine/vmm.h> | ||||
#include <vmmapi.h> | |||||
#include "pci_emul.h" | |||||
#include "mem.h" | #include "mem.h" | ||||
#include "pci_passthru.h" | |||||
#ifndef _PATH_DEVPCI | #ifndef _PATH_DEVPCI | ||||
#define _PATH_DEVPCI "/dev/pci" | #define _PATH_DEVPCI "/dev/pci" | ||||
#endif | #endif | ||||
#ifndef _PATH_DEVIO | #ifndef _PATH_DEVIO | ||||
#define _PATH_DEVIO "/dev/io" | #define _PATH_DEVIO "/dev/io" | ||||
#endif | #endif | ||||
#ifndef _PATH_MEM | #ifndef _PATH_MEM | ||||
#define _PATH_MEM "/dev/mem" | #define _PATH_MEM "/dev/mem" | ||||
#endif | #endif | ||||
#define LEGACY_SUPPORT 1 | #define LEGACY_SUPPORT 1 | ||||
#define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1) | #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1) | ||||
#define MSIX_CAPLEN 12 | #define MSIX_CAPLEN 12 | ||||
static int pcifd = -1; | static int pcifd = -1; | ||||
static int iofd = -1; | static int iofd = -1; | ||||
static int memfd = -1; | static int memfd = -1; | ||||
struct passthru_softc { | |||||
struct pci_devinst *psc_pi; | |||||
struct pcibar psc_bar[PCI_BARMAX + 1]; | |||||
struct { | |||||
int capoff; | |||||
int msgctrl; | |||||
int emulated; | |||||
} psc_msi; | |||||
struct { | |||||
int capoff; | |||||
} psc_msix; | |||||
struct pcisel psc_sel; | |||||
}; | |||||
static int | static int | ||||
msi_caplen(int msgctrl) | msi_caplen(int msgctrl) | ||||
{ | { | ||||
int len; | int len; | ||||
len = 10; /* minimum length of msi capability */ | len = 10; /* minimum length of msi capability */ | ||||
if (msgctrl & PCIM_MSICTRL_64BIT) | if (msgctrl & PCIM_MSICTRL_64BIT) | ||||
len += 4; | len += 4; | ||||
#if 0 | #if 0 | ||||
/* | /* | ||||
* Ignore the 'mask' and 'pending' bits in the MSI capability. | * Ignore the 'mask' and 'pending' bits in the MSI capability. | ||||
* We'll let the guest manipulate them directly. | * We'll let the guest manipulate them directly. | ||||
*/ | */ | ||||
if (msgctrl & PCIM_MSICTRL_VECTOR) | if (msgctrl & PCIM_MSICTRL_VECTOR) | ||||
len += 10; | len += 10; | ||||
#endif | #endif | ||||
return (len); | return (len); | ||||
} | } | ||||
static uint32_t | uint32_t | ||||
read_config(const struct pcisel *sel, long reg, int width) | read_config(const struct pcisel *sel, long reg, int width) | ||||
{ | { | ||||
struct pci_io pi; | struct pci_io pi; | ||||
bzero(&pi, sizeof(pi)); | bzero(&pi, sizeof(pi)); | ||||
pi.pi_sel = *sel; | pi.pi_sel = *sel; | ||||
pi.pi_reg = reg; | pi.pi_reg = reg; | ||||
pi.pi_width = width; | pi.pi_width = width; | ||||
if (ioctl(pcifd, PCIOCREAD, &pi) < 0) | if (ioctl(pcifd, PCIOCREAD, &pi) < 0) | ||||
return (0); /* XXX */ | return (0); /* XXX */ | ||||
else | else | ||||
return (pi.pi_data); | return (pi.pi_data); | ||||
} | } | ||||
static void | void | ||||
write_config(const struct pcisel *sel, long reg, int width, uint32_t data) | write_config(const struct pcisel *sel, long reg, int width, uint32_t data) | ||||
{ | { | ||||
struct pci_io pi; | struct pci_io pi; | ||||
bzero(&pi, sizeof(pi)); | bzero(&pi, sizeof(pi)); | ||||
pi.pi_sel = *sel; | pi.pi_sel = *sel; | ||||
pi.pi_reg = reg; | pi.pi_reg = reg; | ||||
pi.pi_width = width; | pi.pi_width = width; | ||||
pi.pi_data = data; | pi.pi_data = data; | ||||
(void)ioctl(pcifd, PCIOCWRITE, &pi); /* XXX */ | (void)ioctl(pcifd, PCIOCWRITE, &pi); /* XXX */ | ||||
} | } | ||||
int | |||||
passthru_modify_pptdev_mmio(struct vmctx *ctx, struct passthru_softc *sc, struct passthru_mmio_mapping *map, int registration) | |||||
{ | |||||
if (registration == PT_MAP_PPTDEV_MMIO) | |||||
return vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, map->gpa, map->len, map->hpa); | |||||
else | |||||
return vm_unmap_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, map->gpa, map->len); | |||||
} | |||||
static int | |||||
passthru_modify_bar_registration(struct pci_devinst *pi, int idx, int registration) | |||||
{ | |||||
int error; | |||||
struct passthru_softc *sc; | |||||
struct passthru_mmio_mapping map; | |||||
sc = pi->pi_arg; | |||||
/* | |||||
* If the guest writes a new value to a 64-bit BAR, two writes are neccessary. | |||||
* vm_map_pptdev_mmio can fail in that case due to an invalid address after the first write. | |||||
* To avoid it, skip registration in that case. | |||||
*/ | |||||
if ((registration == PT_MAP_PPTDEV_MMIO) && (pi->pi_bar[idx].type == PCIBAR_MEM64)) | |||||
if ((pci_get_cfgdata32(pi, PCIR_BAR(idx + 0)) == ~0U) || | |||||
(pci_get_cfgdata32(pi, PCIR_BAR(idx + 1)) == ~0U)) | |||||
return 0; | |||||
if (idx != pci_msix_table_bar(pi)) { | |||||
map.gpa = pi->pi_bar[idx].addr; | |||||
map.len = pi->pi_bar[idx].size; | |||||
map.hpa = sc->psc_bar[idx].addr; | |||||
return passthru_modify_pptdev_mmio(pi->pi_vmctx, sc, &map, registration); | |||||
} | |||||
/* special handling for MSI-X table */ | |||||
uint32_t table_offset, table_size; | |||||
table_offset = rounddown2(pi->pi_msix.table_offset, 4096); | |||||
table_size = pi->pi_msix.table_offset - table_offset; | |||||
table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; | |||||
table_size = roundup2(table_size, 4096); | |||||
map.gpa = pi->pi_bar[idx].addr; | |||||
map.len = table_offset; | |||||
map.hpa = sc->psc_bar[idx].addr; | |||||
/* map/unmap everything before MSI-X table */ | |||||
if (map.len > 0) | |||||
if ((error = passthru_modify_pptdev_mmio(pi->pi_vmctx, sc, &map, registration)) != 0) | |||||
return error; | |||||
map.gpa += table_offset + table_size; | |||||
map.len = pi->pi_bar[idx].size - (table_offset + table_size); | |||||
map.hpa += table_offset + table_size; | |||||
/* map/unmap everything behind MSI-X table */ | |||||
if (map.len > 0) | |||||
if ((error = passthru_modify_pptdev_mmio(pi->pi_vmctx, sc, &map, registration)) != 0) | |||||
return error; | |||||
return (0); | |||||
} | |||||
#ifdef LEGACY_SUPPORT | #ifdef LEGACY_SUPPORT | ||||
static int | static int | ||||
passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr) | passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr) | ||||
{ | { | ||||
int capoff, i; | int capoff, i; | ||||
struct msicap msicap; | struct msicap msicap; | ||||
u_char *capdata; | u_char *capdata; | ||||
▲ Show 20 Lines • Show All 273 Lines • ▼ Show 20 Lines | if (pi->pi_msix.enabled) { | ||||
} | } | ||||
} | } | ||||
} | } | ||||
static int | static int | ||||
init_msix_table(struct vmctx *ctx, struct passthru_softc *sc, uint64_t base) | init_msix_table(struct vmctx *ctx, struct passthru_softc *sc, uint64_t base) | ||||
{ | { | ||||
int b, s, f; | int b, s, f; | ||||
int error, idx; | int idx; | ||||
size_t len, remaining; | size_t remaining; | ||||
uint32_t table_size, table_offset; | uint32_t table_size, table_offset; | ||||
uint32_t pba_size, pba_offset; | uint32_t pba_size, pba_offset; | ||||
vm_paddr_t start; | vm_paddr_t start; | ||||
struct pci_devinst *pi = sc->psc_pi; | struct pci_devinst *pi = sc->psc_pi; | ||||
assert(pci_msix_table_bar(pi) >= 0 && pci_msix_pba_bar(pi) >= 0); | assert(pci_msix_table_bar(pi) >= 0 && pci_msix_pba_bar(pi) >= 0); | ||||
b = sc->psc_sel.pc_bus; | b = sc->psc_sel.pc_bus; | ||||
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | if (pba_offset >= table_offset + table_size || | ||||
warn( | warn( | ||||
"Failed to map PBA page for MSI-X on %d/%d/%d", | "Failed to map PBA page for MSI-X on %d/%d/%d", | ||||
b, s, f); | b, s, f); | ||||
return (-1); | return (-1); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
/* Map everything before the MSI-X table */ | |||||
if (table_offset > 0) { | |||||
len = table_offset; | |||||
error = vm_map_pptdev_mmio(ctx, b, s, f, start, len, base); | |||||
if (error) | |||||
return (error); | |||||
base += len; | |||||
start += len; | |||||
remaining -= len; | |||||
} | |||||
/* Skip the MSI-X table */ | |||||
base += table_size; | |||||
start += table_size; | |||||
remaining -= table_size; | |||||
/* Map everything beyond the end of the MSI-X table */ | |||||
if (remaining > 0) { | |||||
len = remaining; | |||||
error = vm_map_pptdev_mmio(ctx, b, s, f, start, len, base); | |||||
if (error) | |||||
return (error); | |||||
} | |||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
cfginitbar(struct vmctx *ctx, struct passthru_softc *sc) | cfginitbar(struct vmctx *ctx, struct passthru_softc *sc) | ||||
{ | { | ||||
int i, error; | int i, error; | ||||
struct pci_devinst *pi; | struct pci_devinst *pi; | ||||
Show All 39 Lines | if (bartype != PCIBAR_IO) { | ||||
return (-1); | return (-1); | ||||
} | } | ||||
} | } | ||||
/* Cache information about the "real" BAR */ | /* Cache information about the "real" BAR */ | ||||
sc->psc_bar[i].type = bartype; | sc->psc_bar[i].type = bartype; | ||||
sc->psc_bar[i].size = size; | sc->psc_bar[i].size = size; | ||||
sc->psc_bar[i].addr = base; | sc->psc_bar[i].addr = base; | ||||
sc->psc_bar[i].lobits = 0; | |||||
/* Allocate the BAR in the guest I/O or MMIO space */ | /* Allocate the BAR in the guest I/O or MMIO space */ | ||||
error = pci_emul_alloc_bar(pi, i, bartype, size); | error = pci_emul_alloc_bar(pi, i, bartype, size); | ||||
if (error) | if (error) | ||||
return (-1); | return (-1); | ||||
/* Use same lobits as physical bar */ | |||||
uint8_t lobits = read_config(&sc->psc_sel, PCIR_BAR(i), 0x01); | |||||
if (bartype == PCIBAR_MEM32 || bartype == PCIBAR_MEM64) { | |||||
lobits &= ~PCIM_BAR_MEM_BASE; | |||||
} else { | |||||
lobits &= ~PCIM_BAR_IO_BASE; | |||||
} | |||||
sc->psc_bar[i].lobits = lobits; | |||||
pi->pi_bar[i].lobits = lobits; | |||||
/* The MSI-X table needs special handling */ | /* The MSI-X table needs special handling */ | ||||
if (i == pci_msix_table_bar(pi)) { | if (i == pci_msix_table_bar(pi)) { | ||||
error = init_msix_table(ctx, sc, base); | error = init_msix_table(ctx, sc, base); | ||||
if (error) | if (error) | ||||
return (-1); | return (-1); | ||||
} else if (bartype != PCIBAR_IO) { | |||||
/* Map the physical BAR in the guest MMIO space */ | |||||
error = vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus, | |||||
sc->psc_sel.pc_dev, sc->psc_sel.pc_func, | |||||
pi->pi_bar[i].addr, pi->pi_bar[i].size, base); | |||||
if (error) | |||||
return (-1); | |||||
} | } | ||||
/* | /* | ||||
* 64-bit BAR takes up two slots so skip the next one. | * 64-bit BAR takes up two slots so skip the next one. | ||||
*/ | */ | ||||
if (bartype == PCIBAR_MEM64) { | if (bartype == PCIBAR_MEM64) { | ||||
i++; | i++; | ||||
assert(i <= PCI_BARMAX); | assert(i <= PCI_BARMAX); | ||||
Show All 24 Lines | cfginit(struct vmctx *ctx, struct pci_devinst *pi, int bus, int slot, int func) | ||||
} | } | ||||
if (cfginitbar(ctx, sc) != 0) { | if (cfginitbar(ctx, sc) != 0) { | ||||
warnx("failed to initialize BARs for PCI %d/%d/%d", | warnx("failed to initialize BARs for PCI %d/%d/%d", | ||||
bus, slot, func); | bus, slot, func); | ||||
goto done; | goto done; | ||||
} | } | ||||
pci_set_cfgdata16(pi, PCIR_COMMAND, read_config(&sc->psc_sel, | /* sync command register */ | ||||
PCIR_COMMAND, 2)); | write_config(&sc->psc_sel, PCIR_COMMAND, 0x02, | ||||
pci_get_cfgdata16(pi, PCIR_COMMAND)); | |||||
error = 0; /* success */ | error = 0; /* success */ | ||||
done: | done: | ||||
return (error); | return (error); | ||||
} | } | ||||
#define PPT_PCIR_PROT(reg) ((sc->psc_pcir_prot_map[reg / 4] >> (reg & 0x03)) & PPT_PCIR_PROT_MASK) | |||||
int | |||||
set_pcir_prot(struct passthru_softc *sc, uint32_t reg, uint32_t len, uint8_t prot) | |||||
{ | |||||
if (reg > PCI_REGMAX || reg + len > PCI_REGMAX + 1) | |||||
return (-1); | |||||
prot &= PPT_PCIR_PROT_MASK; | |||||
for (int i = reg; i < reg + len; ++i) { | |||||
/* delete old prot value */ | |||||
sc->psc_pcir_prot_map[i / 4] &= ~(PPT_PCIR_PROT_MASK << (i & 0x03)); | |||||
/* set new prot value */ | |||||
sc->psc_pcir_prot_map[i / 4] |= prot << (i & 0x03); | |||||
} | |||||
return (0); | |||||
} | |||||
static int | static int | ||||
is_pcir_writable(struct passthru_softc *sc, uint32_t reg) | |||||
{ | |||||
if (reg > PCI_REGMAX) | |||||
return (0); | |||||
return ((PPT_PCIR_PROT(reg) & PPT_PCIR_PROT_WO) != 0); | |||||
} | |||||
static int | |||||
is_pcir_readable(struct passthru_softc *sc, uint32_t reg) | |||||
{ | |||||
if (reg > PCI_REGMAX) | |||||
return (0); | |||||
return ((PPT_PCIR_PROT(reg) & PPT_PCIR_PROT_RO) != 0); | |||||
} | |||||
static int | |||||
passthru_init_quirks(struct vmctx *ctx, struct pci_devinst *pi, char *opts) | |||||
{ | |||||
struct passthru_softc *sc = pi->pi_arg; | |||||
uint16_t vendor = read_config(&sc->psc_sel, PCIR_VENDOR, 0x02); | |||||
uint8_t class = read_config(&sc->psc_sel, PCIR_CLASS, 0x01); | |||||
/* currently only display devices have quirks */ | |||||
if (class != PCIC_DISPLAY) | |||||
return (0); | |||||
if (vendor == PCI_VENDOR_INTEL) | |||||
return gvt_d_init(ctx, pi, opts); | |||||
return (0); | |||||
} | |||||
static void | |||||
passthru_deinit_quirks(struct vmctx *ctx, struct pci_devinst *pi) | |||||
{ | |||||
struct passthru_softc *sc = pi->pi_arg; | |||||
uint16_t vendor = read_config(&sc->psc_sel, PCIR_VENDOR, 0x02); | |||||
uint8_t class = read_config(&sc->psc_sel, PCIR_CLASS, 0x01); | |||||
/* currently only display devices have quirks */ | |||||
if (class != PCIC_DISPLAY) | |||||
return; | |||||
if (vendor == PCI_VENDOR_INTEL) | |||||
return gvt_d_deinit(ctx, pi); | |||||
return; | |||||
} | |||||
static int | |||||
passthru_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) | passthru_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) | ||||
{ | { | ||||
int bus, slot, func, error, memflags; | int bus, slot, func, error, memflags; | ||||
struct passthru_softc *sc; | struct passthru_softc *sc; | ||||
#ifndef WITHOUT_CAPSICUM | #ifndef WITHOUT_CAPSICUM | ||||
cap_rights_t rights; | cap_rights_t rights; | ||||
cap_ioctl_t pci_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR }; | cap_ioctl_t pci_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR }; | ||||
cap_ioctl_t io_ioctls[] = { IODEV_PIO }; | cap_ioctl_t io_ioctls[] = { IODEV_PIO }; | ||||
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | #endif | ||||
} | } | ||||
sc = calloc(1, sizeof(struct passthru_softc)); | sc = calloc(1, sizeof(struct passthru_softc)); | ||||
pi->pi_arg = sc; | pi->pi_arg = sc; | ||||
sc->psc_pi = pi; | sc->psc_pi = pi; | ||||
/* initialize config space */ | /* initialize config space */ | ||||
error = cfginit(ctx, pi, bus, slot, func); | if ((error = cfginit(ctx, pi, bus, slot, func)) != 0) | ||||
goto done; | |||||
/* allow access to all PCI registers */ | |||||
if ((error = set_pcir_prot(sc, 0, PCI_REGMAX + 1, PPT_PCIR_PROT_RW)) != 0) | |||||
goto done; | |||||
if ((error = passthru_init_quirks(ctx, pi, opts)) != 0) | |||||
goto done; | |||||
error = 0; /* success */ | |||||
done: | done: | ||||
if (error) { | if (error) { | ||||
passthru_deinit_quirks(ctx, pi); | |||||
free(sc); | free(sc); | ||||
vm_unassign_pptdev(ctx, bus, slot, func); | vm_unassign_pptdev(ctx, bus, slot, func); | ||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
bar_access(int coff) | bar_access(int coff) | ||||
Show All 33 Lines | |||||
static int | static int | ||||
passthru_cfgread(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, | passthru_cfgread(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, | ||||
int coff, int bytes, uint32_t *rv) | int coff, int bytes, uint32_t *rv) | ||||
{ | { | ||||
struct passthru_softc *sc; | struct passthru_softc *sc; | ||||
sc = pi->pi_arg; | sc = pi->pi_arg; | ||||
/* skip for protected PCI registers */ | |||||
if (!is_pcir_readable(sc, coff)) | |||||
return (-1); | |||||
/* | /* | ||||
* PCI BARs and MSI capability is emulated. | * PCI BARs and MSI capability is emulated. | ||||
*/ | */ | ||||
if (bar_access(coff) || msicap_access(sc, coff)) | if (bar_access(coff) || msicap_access(sc, coff)) | ||||
return (-1); | return (-1); | ||||
#ifdef LEGACY_SUPPORT | #ifdef LEGACY_SUPPORT | ||||
/* | /* | ||||
Show All 30 Lines | passthru_cfgwrite(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, | ||||
int coff, int bytes, uint32_t val) | int coff, int bytes, uint32_t val) | ||||
{ | { | ||||
int error, msix_table_entries, i; | int error, msix_table_entries, i; | ||||
struct passthru_softc *sc; | struct passthru_softc *sc; | ||||
uint16_t cmd_old; | uint16_t cmd_old; | ||||
sc = pi->pi_arg; | sc = pi->pi_arg; | ||||
/* skip for protected PCI registers */ | |||||
if (!is_pcir_writable(sc, coff)) | |||||
return (-1); | |||||
/* | /* | ||||
* PCI BARs are emulated | * PCI BARs are emulated | ||||
*/ | */ | ||||
if (bar_access(coff)) | if (bar_access(coff)) | ||||
return (-1); | return (-1); | ||||
/* | /* | ||||
* MSI capability is emulated | * MSI capability is emulated | ||||
▲ Show 20 Lines • Show All 106 Lines • ▼ Show 20 Lines | if (baridx == pci_msix_table_bar(pi)) { | ||||
(void)ioctl(iofd, IODEV_PIO, &pio); | (void)ioctl(iofd, IODEV_PIO, &pio); | ||||
val = pio.val; | val = pio.val; | ||||
} | } | ||||
return (val); | return (val); | ||||
} | } | ||||
static int | |||||
passthru_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx, | |||||
int enabled, uint64_t address) | |||||
{ | |||||
int error; | |||||
if (pi->pi_bar[baridx].type == PCIBAR_IO) | |||||
return (-1); | |||||
error = passthru_modify_bar_registration(pi, baridx, enabled); | |||||
assert(error == 0); | |||||
return error; | |||||
} | |||||
struct pci_devemu passthru = { | struct pci_devemu passthru = { | ||||
.pe_emu = "passthru", | .pe_emu = "passthru", | ||||
.pe_init = passthru_init, | .pe_init = passthru_init, | ||||
.pe_cfgwrite = passthru_cfgwrite, | .pe_cfgwrite = passthru_cfgwrite, | ||||
.pe_cfgread = passthru_cfgread, | .pe_cfgread = passthru_cfgread, | ||||
.pe_barwrite = passthru_write, | .pe_barwrite = passthru_write, | ||||
.pe_barread = passthru_read, | .pe_barread = passthru_read, | ||||
.pe_baraddr = passthru_addr, | |||||
}; | }; | ||||
PCI_EMUL_SET(passthru); | PCI_EMUL_SET(passthru); |