Differential D26209 Diff 79590 usr.sbin/bhyve/pci_passthru.c

Changeset View

Standalone View

usr.sbin/bhyve/pci_passthru.c

Show First 20 Lines • Show All 52 Lines • ▼ Show 20 Lines
#include <string.h>		#include <string.h>
#include <err.h>		#include <err.h>
#include <errno.h>		#include <errno.h>
#include <fcntl.h>		#include <fcntl.h>
#include <sysexits.h>		#include <sysexits.h>
#include <unistd.h>		#include <unistd.h>

#include <machine/vmm.h>		#include <machine/vmm.h>
#include <vmmapi.h>
#include "pci_emul.h"
#include "mem.h"		#include "mem.h"
		#include "pci_passthru.h"

#ifndef _PATH_DEVPCI		#ifndef _PATH_DEVPCI
#define _PATH_DEVPCI "/dev/pci"		#define _PATH_DEVPCI "/dev/pci"
#endif		#endif

#ifndef _PATH_DEVIO		#ifndef _PATH_DEVIO
#define _PATH_DEVIO "/dev/io"		#define _PATH_DEVIO "/dev/io"
#endif		#endif

#ifndef _PATH_MEM		#ifndef _PATH_MEM
#define _PATH_MEM "/dev/mem"		#define _PATH_MEM "/dev/mem"
#endif		#endif

#define LEGACY_SUPPORT 1		#define LEGACY_SUPPORT 1

#define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1)		#define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1)
#define MSIX_CAPLEN 12		#define MSIX_CAPLEN 12

static int pcifd = -1;		static int pcifd = -1;
static int iofd = -1;		static int iofd = -1;
static int memfd = -1;		static int memfd = -1;

struct passthru_softc {
struct pci_devinst *psc_pi;
struct pcibar psc_bar[PCI_BARMAX + 1];
struct {
int capoff;
int msgctrl;
int emulated;
} psc_msi;
struct {
int capoff;
} psc_msix;
struct pcisel psc_sel;
};

static int		static int
msi_caplen(int msgctrl)		msi_caplen(int msgctrl)
{		{
int len;		int len;

len = 10; /* minimum length of msi capability */		len = 10; /* minimum length of msi capability */

if (msgctrl & PCIM_MSICTRL_64BIT)		if (msgctrl & PCIM_MSICTRL_64BIT)
len += 4;		len += 4;

#if 0		#if 0
/*		/*
* Ignore the 'mask' and 'pending' bits in the MSI capability.		* Ignore the 'mask' and 'pending' bits in the MSI capability.
* We'll let the guest manipulate them directly.		* We'll let the guest manipulate them directly.
*/		*/
if (msgctrl & PCIM_MSICTRL_VECTOR)		if (msgctrl & PCIM_MSICTRL_VECTOR)
len += 10;		len += 10;
#endif		#endif

return (len);		return (len);
}		}

static uint32_t		uint32_t
read_config(const struct pcisel *sel, long reg, int width)		read_config(const struct pcisel *sel, long reg, int width)
{		{
struct pci_io pi;		struct pci_io pi;

bzero(&pi, sizeof(pi));		bzero(&pi, sizeof(pi));
pi.pi_sel = *sel;		pi.pi_sel = *sel;
pi.pi_reg = reg;		pi.pi_reg = reg;
pi.pi_width = width;		pi.pi_width = width;

if (ioctl(pcifd, PCIOCREAD, &pi) < 0)		if (ioctl(pcifd, PCIOCREAD, &pi) < 0)
return (0); /* XXX */		return (0); /* XXX */
else		else
return (pi.pi_data);		return (pi.pi_data);
}		}

static void		void
write_config(const struct pcisel *sel, long reg, int width, uint32_t data)		write_config(const struct pcisel *sel, long reg, int width, uint32_t data)
{		{
struct pci_io pi;		struct pci_io pi;

bzero(&pi, sizeof(pi));		bzero(&pi, sizeof(pi));
pi.pi_sel = *sel;		pi.pi_sel = *sel;
pi.pi_reg = reg;		pi.pi_reg = reg;
pi.pi_width = width;		pi.pi_width = width;
pi.pi_data = data;		pi.pi_data = data;

(void)ioctl(pcifd, PCIOCWRITE, &pi); /* XXX */		(void)ioctl(pcifd, PCIOCWRITE, &pi); /* XXX */
}		}

		int
		passthru_modify_pptdev_mmio(struct vmctx ctx, struct passthru_softc sc, struct passthru_mmio_mapping *map, int registration)
		{
		if (registration == PT_MAP_PPTDEV_MMIO)
		return vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, map->gpa, map->len, map->hpa);
		else
		return vm_unmap_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, map->gpa, map->len);
		}

		int
		passthru_modify_bar_registration(struct pci_devinst *pi, int idx, int registration)
		{
		int error;
		struct passthru_softc *sc;
		struct passthru_mmio_mapping map;

		sc = pi->pi_arg;

		/*
		* If the guest writes a new value to a 64-bit BAR, two writes are neccessary.
		* vm_map_pptdev_mmio can fail in that case due to an invalid address after the first write.
		* To avoid it, skip registration in that case.
		*/
		if ((registration == PT_MAP_PPTDEV_MMIO) && (pi->pi_bar[idx].type == PCIBAR_MEM64))
		if ((pci_get_cfgdata32(pi, PCIR_BAR(idx + 0)) == ~0U) \|\|
		(pci_get_cfgdata32(pi, PCIR_BAR(idx + 1)) == ~0U))
		return 0;

		if (idx != pci_msix_table_bar(pi)) {
		map.gpa = pi->pi_bar[idx].addr;
		map.len = pi->pi_bar[idx].size;
		map.hpa = sc->psc_bar[idx].addr;
		return passthru_modify_pptdev_mmio(pi->pi_vmctx, sc, &map, registration);
		}

		// special handling for MSI-X table
		uint32_t table_offset, table_size;

		table_offset = rounddown2(pi->pi_msix.table_offset, 4096);
		table_size = pi->pi_msix.table_offset - table_offset;
		table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
		table_size = roundup2(table_size, 4096);

		map.gpa = pi->pi_bar[idx].addr;
		map.len = table_offset;
		map.hpa = sc->psc_bar[idx].addr;

		// map/unmap everything before MSI-X table
		if (map.len > 0)
		if ((error = passthru_modify_pptdev_mmio(pi->pi_vmctx, sc, &map, registration)) != 0)
		return error;

		map.gpa += table_offset + table_size;
		map.len = pi->pi_bar[idx].size - (table_offset + table_size);
		map.hpa += table_offset + table_size;

		// map/unmap everything behind MSI-X table
		if (map.len > 0)
		if ((error = passthru_modify_pptdev_mmio(pi->pi_vmctx, sc, &map, registration)) != 0)
		return error;

		return (0);
		}

#ifdef LEGACY_SUPPORT		#ifdef LEGACY_SUPPORT
static int		static int
passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr)		passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr)
{		{
int capoff, i;		int capoff, i;
struct msicap msicap;		struct msicap msicap;
u_char *capdata;		u_char *capdata;

▲ Show 20 Lines • Show All 391 Lines • ▼ Show 20 Lines	if (bartype != PCIBAR_IO) {
return (-1);		return (-1);
}		}
}		}

/* Cache information about the "real" BAR */		/* Cache information about the "real" BAR */
sc->psc_bar[i].type = bartype;		sc->psc_bar[i].type = bartype;
sc->psc_bar[i].size = size;		sc->psc_bar[i].size = size;
sc->psc_bar[i].addr = base;		sc->psc_bar[i].addr = base;
		sc->psc_bar[i].lobits = 0;

/* Allocate the BAR in the guest I/O or MMIO space */		/* Allocate the BAR in the guest I/O or MMIO space */
error = pci_emul_alloc_pbar(pi, i, base, bartype, size);		error = pci_emul_alloc_pbar(pi, i, base, bartype, size);
if (error)		if (error)
return (-1);		return (-1);

		/* Use same prefetchable property as physical bar */
		if (is_gvt_d(pi)) {
		uint8_t lobits = pci_get_cfgdata8(pi, PCIR_BAR(i));
		if (bartype == PCIBAR_MEM32 \|\| bartype == PCIBAR_MEM64) {
		if (bar.pbi_base & PCIM_BAR_MEM_PREFETCH)
		lobits \|= PCIM_BAR_MEM_PREFETCH;
		else
		lobits &= ~PCIM_BAR_MEM_PREFETCH;
		pci_set_cfgdata8(pi, PCIR_BAR(i), lobits);
		lobits &= ~PCIM_BAR_MEM_BASE;
		}
		else {
		lobits \|= PCIM_BAR_IO_SPACE;
		pci_set_cfgdata8(pi, PCIR_BAR(i), lobits);
		lobits &= ~PCIM_BAR_IO_BASE;
		}
		sc->psc_bar[i].lobits = lobits;
		pi->pi_bar[i].lobits = lobits;
		}

/* The MSI-X table needs special handling */		/* The MSI-X table needs special handling */
if (i == pci_msix_table_bar(pi)) {		if (i == pci_msix_table_bar(pi)) {
error = init_msix_table(ctx, sc, base);		error = init_msix_table(ctx, sc, base);
if (error)		if (error)
return (-1);		return (-1);
}		}

/*		/*
Show All 37 Lines	cfginit(struct vmctx ctx, struct pci_devinst pi, int bus, int slot, int func)
pci_set_cfgdata16(pi, PCIR_COMMAND, read_config(&sc->psc_sel,		pci_set_cfgdata16(pi, PCIR_COMMAND, read_config(&sc->psc_sel,
PCIR_COMMAND, 2));		PCIR_COMMAND, 2));

error = 0; /* success */		error = 0; /* success */
done:		done:
return (error);		return (error);
}		}

static int		int
passthru_init(struct vmctx ctx, struct pci_devinst pi, char *opts)		passthru_init(struct vmctx ctx, struct pci_devinst pi, char *opts)
{		{
int bus, slot, func, error, memflags;		int bus, slot, func, error, memflags;
struct passthru_softc *sc;		struct passthru_softc *sc;
#ifndef WITHOUT_CAPSICUM		#ifndef WITHOUT_CAPSICUM
cap_rights_t rights;		cap_rights_t rights;
cap_ioctl_t pci_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR };		cap_ioctl_t pci_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR };
cap_ioctl_t io_ioctls[] = { IODEV_PIO };		cap_ioctl_t io_ioctls[] = { IODEV_PIO };
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines	#endif
}		}

sc = calloc(1, sizeof(struct passthru_softc));		sc = calloc(1, sizeof(struct passthru_softc));

pi->pi_arg = sc;		pi->pi_arg = sc;
sc->psc_pi = pi;		sc->psc_pi = pi;

/* initialize config space */		/* initialize config space */
error = cfginit(ctx, pi, bus, slot, func);		if ((error = cfginit(ctx, pi, bus, slot, func)) != 0)
		goto done;

		if (pi->pi_d->pe_quirks_init)
		if ((error = pi->pi_d->pe_quirks_init(ctx, pi, opts)) != 0)
		goto done;
done:		done:
if (error) {		if (error) {
		if (pi->pi_d->pe_quirks_deinit)
		pi->pi_d->pe_quirks_deinit(ctx, pi);
free(sc);		free(sc);
vm_unassign_pptdev(ctx, bus, slot, func);		vm_unassign_pptdev(ctx, bus, slot, func);
}		}
return (error);		return (error);
}		}

static int		int
bar_access(int coff)		bar_access(int coff)
{		{
if (coff >= PCIR_BAR(0) && coff < PCIR_BAR(PCI_BARMAX + 1))		if (coff >= PCIR_BAR(0) && coff < PCIR_BAR(PCI_BARMAX + 1))
return (1);		return (1);
else		else
return (0);		return (0);
}		}

Show All 19 Lines	msixcap_access(struct passthru_softc *sc, int coff)
if (sc->psc_msix.capoff == 0)		if (sc->psc_msix.capoff == 0)
return (0);		return (0);

return (coff >= sc->psc_msix.capoff &&		return (coff >= sc->psc_msix.capoff &&
coff < sc->psc_msix.capoff + MSIX_CAPLEN);		coff < sc->psc_msix.capoff + MSIX_CAPLEN);
}		}

static int		static int
		passthru_cfgread_bar(struct vmctx ctx, int vcpu, struct pci_devinst pi,
		int coff, int bytes, uint32_t *rv)
		{
		const int idx = (coff - PCIR_BAR(0)) / 4;
		int update_idx = idx;
		if (pi->pi_bar[idx].type == PCIBAR_MEMHI64)
		--update_idx;

		if (pci_get_cfgdata32(pi, PCIR_BAR(idx)) != ~0U) {
		// return address of BAR
		if (bytes == 1)
		*rv = pci_get_cfgdata8(pi, coff);
		else if (bytes == 2)
		*rv = pci_get_cfgdata16(pi, coff);
		else
		*rv = pci_get_cfgdata32(pi, coff);

		return (0);
		}

		// return size of BAR
		uint64_t size = ~(uint64_t)(pi->pi_bar[update_idx].size - 1);
		size \|= pi->pi_bar[update_idx].lobits;
		if (pi->pi_bar[idx].type == PCIBAR_MEMHI64)
		size >>= 32;
		assert(bytes == 4);
		*rv = size;

		return (0);
		}

		int
passthru_cfgread(struct vmctx ctx, int vcpu, struct pci_devinst pi,		passthru_cfgread(struct vmctx ctx, int vcpu, struct pci_devinst pi,
int coff, int bytes, uint32_t *rv)		int coff, int bytes, uint32_t *rv)
{		{
struct passthru_softc *sc;		struct passthru_softc *sc;

sc = pi->pi_arg;		sc = pi->pi_arg;

/*		/*
* PCI BARs and MSI capability is emulated.		* PCI BARs and MSI capability is emulated.
*/		*/
if (bar_access(coff) \|\| msicap_access(sc, coff))		if (msicap_access(sc, coff))
return (-1);		return (-1);

		if (bar_access(coff)) {
		if (is_gvt_d(pi))
		return passthru_cfgread_bar(ctx, vcpu, pi, coff, bytes, rv);
		else
		return (-1);
		}

#ifdef LEGACY_SUPPORT		#ifdef LEGACY_SUPPORT
/*		/*
* Emulate PCIR_CAP_PTR if this device does not support MSI capability		* Emulate PCIR_CAP_PTR if this device does not support MSI capability
* natively.		* natively.
*/		*/
if (sc->psc_msi.emulated) {		if (sc->psc_msi.emulated) {
if (coff >= PCIR_CAP_PTR && coff < PCIR_CAP_PTR + 4)		if (coff >= PCIR_CAP_PTR && coff < PCIR_CAP_PTR + 4)
return (-1);		return (-1);
Show All 15 Lines	#endif

/* Everything else just read from the device's config space */		/* Everything else just read from the device's config space */
*rv = read_config(&sc->psc_sel, coff, bytes);		*rv = read_config(&sc->psc_sel, coff, bytes);

return (0);		return (0);
}		}

static int		static int
		passthru_cfgwrite_bar(struct vmctx ctx, int vcpu, struct pci_devinst pi,
		int coff, int bytes, uint32_t val)
		{
		const int idx = (coff - PCIR_BAR(0)) / 4;
		int update_idx = idx;

		switch (pi->pi_bar[idx].type) {
		case PCIBAR_MEMHI64:
		--update_idx;
		case PCIBAR_IO:
		case PCIBAR_MEM32:
		case PCIBAR_MEM64:
		{
		const uint16_t cmd = pci_get_cfgdata16(pi, PCIR_COMMAND);
		if ((cmd & PCIM_CMD_MEMEN && pi->pi_bar[idx].type != PCIBAR_IO) \|\|
		(cmd & PCIM_CMD_PORTEN && pi->pi_bar[idx].type == PCIBAR_IO)) {
		passthru_modify_bar_registration(pi, update_idx, 0);
		}

		if (val == ~0U) {
		// guest wants to read size of BAR
		pci_set_cfgdata32(pi, coff, ~0U);
		pi->pi_bar[update_idx].addr = 0;
		break;
		}

		// guest sets address of BAR
		uint64_t mask, bar;
		mask = ~(pi->pi_bar[update_idx].size - 1);
		if (pi->pi_bar[idx].type == PCIBAR_MEMHI64)
		mask >>= 32;
		bar = val & mask;
		if (pi->pi_bar[idx].type != PCIBAR_MEMHI64)
		bar \|= pi->pi_bar[update_idx].lobits;
		pci_set_cfgdata32(pi, coff, bar);

		/* Only register BAR if it contains a valid address */
		uint32_t lo, hi;

		lo = pci_get_cfgdata32(pi, PCIR_BAR(update_idx));
		if (pi->pi_bar[update_idx].type == PCIBAR_IO) {
		if ((lo & PCIM_BAR_IO_BASE) == PCIM_BAR_IO_BASE)
		lo = ~0U;
		else
		lo &= PCIM_BAR_IO_BASE;
		} else {
		if ((lo & PCIM_BAR_MEM_BASE) == PCIM_BAR_MEM_BASE)
		lo = ~0U;
		else
		lo &= PCIM_BAR_MEM_BASE;
		}

		if (pi->pi_bar[update_idx].type == PCIBAR_MEM64)
		hi = pci_get_cfgdata32(pi, PCIR_BAR(update_idx + 1));
		else
		hi = 0;

		if (lo != ~0U && hi != ~0U) {
		pi->pi_bar[update_idx].addr = (uint64_t)lo \| ((uint64_t)hi << 32U);
		if ((cmd & PCIM_CMD_MEMEN && pi->pi_bar[idx].type != PCIBAR_IO) \|\|
		(cmd & PCIM_CMD_PORTEN && pi->pi_bar[idx].type == PCIBAR_IO)) {
		passthru_modify_bar_registration(pi, update_idx, 1);
		}
		}
		else
		pi->pi_bar[update_idx].addr = 0;
		break;
		}
		default:
		pi->pi_bar[idx].addr = 0;
		break;
		}
		return (0);
		}

		int
passthru_cfgwrite(struct vmctx ctx, int vcpu, struct pci_devinst pi,		passthru_cfgwrite(struct vmctx ctx, int vcpu, struct pci_devinst pi,
int coff, int bytes, uint32_t val)		int coff, int bytes, uint32_t val)
{		{
int error, msix_table_entries, i;		int error, msix_table_entries, i;
struct passthru_softc *sc;		struct passthru_softc *sc;
uint16_t cmd_old;		uint16_t cmd_old;

sc = pi->pi_arg;		sc = pi->pi_arg;

/*		/*
* PCI BARs are emulated		* PCI BARs are emulated
*/		*/
if (bar_access(coff))		if (bar_access(coff)) {
		if (is_gvt_d(pi))
		return passthru_cfgwrite_bar(ctx, vcpu, pi, coff, bytes, val);
		else
return (-1);		return (-1);
		}

/*		/*
* MSI capability is emulated		* MSI capability is emulated
*/		*/
if (msicap_access(sc, coff)) {		if (msicap_access(sc, coff)) {
pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff,		pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff,
PCIY_MSI);		PCIY_MSI);
error = vm_setup_pptdev_msi(ctx, vcpu, sc->psc_sel.pc_bus,		error = vm_setup_pptdev_msi(ctx, vcpu, sc->psc_sel.pc_bus,
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines	if (coff == PCIR_COMMAND) {
else if (bytes == 2)		else if (bytes == 2)
pci_set_cfgdata16(pi, PCIR_COMMAND, val);		pci_set_cfgdata16(pi, PCIR_COMMAND, val);
pci_emul_cmd_changed(pi, cmd_old);		pci_emul_cmd_changed(pi, cmd_old);
}		}

return (0);		return (0);
}		}

static void		void
passthru_write(struct vmctx ctx, int vcpu, struct pci_devinst pi, int baridx,		passthru_write(struct vmctx ctx, int vcpu, struct pci_devinst pi, int baridx,
uint64_t offset, int size, uint64_t value)		uint64_t offset, int size, uint64_t value)
{		{
struct passthru_softc *sc;		struct passthru_softc *sc;
struct iodev_pio_req pio;		struct iodev_pio_req pio;

sc = pi->pi_arg;		sc = pi->pi_arg;

if (baridx == pci_msix_table_bar(pi)) {		if (baridx == pci_msix_table_bar(pi)) {
msix_table_write(ctx, vcpu, sc, offset, size, value);		msix_table_write(ctx, vcpu, sc, offset, size, value);
} else {		} else {
assert(pi->pi_bar[baridx].type == PCIBAR_IO);		assert(pi->pi_bar[baridx].type == PCIBAR_IO);
bzero(&pio, sizeof(struct iodev_pio_req));		bzero(&pio, sizeof(struct iodev_pio_req));
pio.access = IODEV_PIO_WRITE;		pio.access = IODEV_PIO_WRITE;
pio.port = sc->psc_bar[baridx].addr + offset;		pio.port = sc->psc_bar[baridx].addr + offset;
pio.width = size;		pio.width = size;
pio.val = value;		pio.val = value;

(void)ioctl(iofd, IODEV_PIO, &pio);		(void)ioctl(iofd, IODEV_PIO, &pio);
}		}
}		}

static uint64_t		uint64_t
passthru_read(struct vmctx ctx, int vcpu, struct pci_devinst pi, int baridx,		passthru_read(struct vmctx ctx, int vcpu, struct pci_devinst pi, int baridx,
uint64_t offset, int size)		uint64_t offset, int size)
{		{
struct passthru_softc *sc;		struct passthru_softc *sc;
struct iodev_pio_req pio;		struct iodev_pio_req pio;
uint64_t val;		uint64_t val;

sc = pi->pi_arg;		sc = pi->pi_arg;
▲ Show 20 Lines • Show All 115 Lines • Show Last 20 Lines