Differential D27456 Diff 82255 usr.sbin/bhyve/pci_passthru.c

Changeset View

Standalone View

usr.sbin/bhyve/pci_passthru.c

Show First 20 Lines • Show All 52 Lines • ▼ Show 20 Lines
#include <string.h>		#include <string.h>
#include <err.h>		#include <err.h>
#include <errno.h>		#include <errno.h>
#include <fcntl.h>		#include <fcntl.h>
#include <sysexits.h>		#include <sysexits.h>
#include <unistd.h>		#include <unistd.h>

#include <machine/vmm.h>		#include <machine/vmm.h>
#include <vmmapi.h>
#include "pci_emul.h"
#include "mem.h"		#include "mem.h"
		#include "pci_passthru.h"

#ifndef _PATH_DEVPCI		#ifndef _PATH_DEVPCI
#define _PATH_DEVPCI "/dev/pci"		#define _PATH_DEVPCI "/dev/pci"
#endif		#endif

#ifndef _PATH_DEVIO		#ifndef _PATH_DEVIO
#define _PATH_DEVIO "/dev/io"		#define _PATH_DEVIO "/dev/io"
#endif		#endif

#ifndef _PATH_MEM		#ifndef _PATH_MEM
#define _PATH_MEM "/dev/mem"		#define _PATH_MEM "/dev/mem"
#endif		#endif

#define LEGACY_SUPPORT 1		#define LEGACY_SUPPORT 1

#define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1)		#define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1)
#define MSIX_CAPLEN 12		#define MSIX_CAPLEN 12

static int pcifd = -1;		static int pcifd = -1;
static int iofd = -1;		static int iofd = -1;
static int memfd = -1;		static int memfd = -1;

struct passthru_softc {
struct pci_devinst *psc_pi;
struct pcibar psc_bar[PCI_BARMAX + 1];
struct {
int capoff;
int msgctrl;
int emulated;
} psc_msi;
struct {
int capoff;
} psc_msix;
struct pcisel psc_sel;
};

static int		static int
msi_caplen(int msgctrl)		msi_caplen(int msgctrl)
{		{
int len;		int len;

len = 10; /* minimum length of msi capability */		len = 10; /* minimum length of msi capability */

if (msgctrl & PCIM_MSICTRL_64BIT)		if (msgctrl & PCIM_MSICTRL_64BIT)
len += 4;		len += 4;

#if 0		#if 0
/*		/*
* Ignore the 'mask' and 'pending' bits in the MSI capability.		* Ignore the 'mask' and 'pending' bits in the MSI capability.
* We'll let the guest manipulate them directly.		* We'll let the guest manipulate them directly.
*/		*/
if (msgctrl & PCIM_MSICTRL_VECTOR)		if (msgctrl & PCIM_MSICTRL_VECTOR)
len += 10;		len += 10;
#endif		#endif

return (len);		return (len);
}		}

static uint32_t		uint32_t
read_config(const struct pcisel *sel, long reg, int width)		read_config(const struct pcisel *sel, long reg, int width)
{		{
struct pci_io pi;		struct pci_io pi;

bzero(&pi, sizeof(pi));		bzero(&pi, sizeof(pi));
pi.pi_sel = *sel;		pi.pi_sel = *sel;
pi.pi_reg = reg;		pi.pi_reg = reg;
pi.pi_width = width;		pi.pi_width = width;

if (ioctl(pcifd, PCIOCREAD, &pi) < 0)		if (ioctl(pcifd, PCIOCREAD, &pi) < 0)
return (0); /* XXX */		return (0); /* XXX */
else		else
return (pi.pi_data);		return (pi.pi_data);
}		}

static void		void
write_config(const struct pcisel *sel, long reg, int width, uint32_t data)		write_config(const struct pcisel *sel, long reg, int width, uint32_t data)
{		{
struct pci_io pi;		struct pci_io pi;

bzero(&pi, sizeof(pi));		bzero(&pi, sizeof(pi));
pi.pi_sel = *sel;		pi.pi_sel = *sel;
pi.pi_reg = reg;		pi.pi_reg = reg;
pi.pi_width = width;		pi.pi_width = width;
pi.pi_data = data;		pi.pi_data = data;

(void)ioctl(pcifd, PCIOCWRITE, &pi); /* XXX */		(void)ioctl(pcifd, PCIOCWRITE, &pi); /* XXX */
}		}

		int
		passthru_modify_pptdev_mmio(struct vmctx ctx, struct passthru_softc sc, struct passthru_mmio_mapping *map, int registration)
		{
		if (registration == PT_MAP_PPTDEV_MMIO)
		return vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, map->gpa, map->len, map->hpa);
		else
		return vm_unmap_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, map->gpa, map->len);
		}

		static int
		passthru_modify_bar_registration(struct pci_devinst *pi, int idx, int registration)
		{
		int error;
		struct passthru_softc *sc;
		struct passthru_mmio_mapping map;

		sc = pi->pi_arg;

		/*
		* If the guest writes a new value to a 64-bit BAR, two writes are neccessary.
		* vm_map_pptdev_mmio can fail in that case due to an invalid address after the first write.
		* To avoid it, skip registration in that case.
		*/
		if ((registration == PT_MAP_PPTDEV_MMIO) && (pi->pi_bar[idx].type == PCIBAR_MEM64))
		if ((pci_get_cfgdata32(pi, PCIR_BAR(idx + 0)) == ~0U) \|\|
		(pci_get_cfgdata32(pi, PCIR_BAR(idx + 1)) == ~0U))
		return 0;

		if (idx != pci_msix_table_bar(pi)) {
		map.gpa = pi->pi_bar[idx].addr;
		map.len = pi->pi_bar[idx].size;
		map.hpa = sc->psc_bar[idx].addr;
		return passthru_modify_pptdev_mmio(pi->pi_vmctx, sc, &map, registration);
		}

		/* special handling for MSI-X table */
		uint32_t table_offset, table_size;

		table_offset = rounddown2(pi->pi_msix.table_offset, 4096);
		table_size = pi->pi_msix.table_offset - table_offset;
		table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE;
		table_size = roundup2(table_size, 4096);

		map.gpa = pi->pi_bar[idx].addr;
		map.len = table_offset;
		map.hpa = sc->psc_bar[idx].addr;

		/* map/unmap everything before MSI-X table */
		if (map.len > 0)
		if ((error = passthru_modify_pptdev_mmio(pi->pi_vmctx, sc, &map, registration)) != 0)
		return error;

		map.gpa += table_offset + table_size;
		map.len = pi->pi_bar[idx].size - (table_offset + table_size);
		map.hpa += table_offset + table_size;

		/* map/unmap everything behind MSI-X table */
		if (map.len > 0)
		if ((error = passthru_modify_pptdev_mmio(pi->pi_vmctx, sc, &map, registration)) != 0)
		return error;

		return (0);
		}

#ifdef LEGACY_SUPPORT		#ifdef LEGACY_SUPPORT
static int		static int
passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr)		passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr)
{		{
int capoff, i;		int capoff, i;
struct msicap msicap;		struct msicap msicap;
u_char *capdata;		u_char *capdata;

▲ Show 20 Lines • Show All 273 Lines • ▼ Show 20 Lines	if (pi->pi_msix.enabled) {
}		}
}		}
}		}

static int		static int
init_msix_table(struct vmctx ctx, struct passthru_softc sc, uint64_t base)		init_msix_table(struct vmctx ctx, struct passthru_softc sc, uint64_t base)
{		{
int b, s, f;		int b, s, f;
int error, idx;		int idx;
size_t len, remaining;		size_t remaining;
uint32_t table_size, table_offset;		uint32_t table_size, table_offset;
uint32_t pba_size, pba_offset;		uint32_t pba_size, pba_offset;
vm_paddr_t start;		vm_paddr_t start;
struct pci_devinst *pi = sc->psc_pi;		struct pci_devinst *pi = sc->psc_pi;

assert(pci_msix_table_bar(pi) >= 0 && pci_msix_pba_bar(pi) >= 0);		assert(pci_msix_table_bar(pi) >= 0 && pci_msix_pba_bar(pi) >= 0);

b = sc->psc_sel.pc_bus;		b = sc->psc_sel.pc_bus;
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines	if (pba_offset >= table_offset + table_size \|\|
warn(		warn(
"Failed to map PBA page for MSI-X on %d/%d/%d",		"Failed to map PBA page for MSI-X on %d/%d/%d",
b, s, f);		b, s, f);
return (-1);		return (-1);
}		}
}		}
}		}

/* Map everything before the MSI-X table */
if (table_offset > 0) {
len = table_offset;
error = vm_map_pptdev_mmio(ctx, b, s, f, start, len, base);
if (error)
return (error);

base += len;
start += len;
remaining -= len;
}

/* Skip the MSI-X table */
base += table_size;
start += table_size;
remaining -= table_size;

/* Map everything beyond the end of the MSI-X table */
if (remaining > 0) {
len = remaining;
error = vm_map_pptdev_mmio(ctx, b, s, f, start, len, base);
if (error)
return (error);
}

return (0);		return (0);
}		}

static int		static int
cfginitbar(struct vmctx ctx, struct passthru_softc sc)		cfginitbar(struct vmctx ctx, struct passthru_softc sc)
{		{
int i, error;		int i, error;
struct pci_devinst *pi;		struct pci_devinst *pi;
Show All 39 Lines	if (bartype != PCIBAR_IO) {
return (-1);		return (-1);
}		}
}		}

/* Cache information about the "real" BAR */		/* Cache information about the "real" BAR */
sc->psc_bar[i].type = bartype;		sc->psc_bar[i].type = bartype;
sc->psc_bar[i].size = size;		sc->psc_bar[i].size = size;
sc->psc_bar[i].addr = base;		sc->psc_bar[i].addr = base;
		sc->psc_bar[i].lobits = 0;

/* Allocate the BAR in the guest I/O or MMIO space */		/* Allocate the BAR in the guest I/O or MMIO space */
error = pci_emul_alloc_bar(pi, i, bartype, size);		error = pci_emul_alloc_bar(pi, i, bartype, size);
if (error)		if (error)
return (-1);		return (-1);

		/* Use same lobits as physical bar */
		uint8_t lobits = read_config(&sc->psc_sel, PCIR_BAR(i), 0x01);
		if (bartype == PCIBAR_MEM32 \|\| bartype == PCIBAR_MEM64) {
		lobits &= ~PCIM_BAR_MEM_BASE;
		} else {
		lobits &= ~PCIM_BAR_IO_BASE;
		}
		sc->psc_bar[i].lobits = lobits;
		pi->pi_bar[i].lobits = lobits;

/* The MSI-X table needs special handling */		/* The MSI-X table needs special handling */
if (i == pci_msix_table_bar(pi)) {		if (i == pci_msix_table_bar(pi)) {
error = init_msix_table(ctx, sc, base);		error = init_msix_table(ctx, sc, base);
if (error)		if (error)
return (-1);		return (-1);
} else if (bartype != PCIBAR_IO) {
/* Map the physical BAR in the guest MMIO space */
error = vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus,
sc->psc_sel.pc_dev, sc->psc_sel.pc_func,
pi->pi_bar[i].addr, pi->pi_bar[i].size, base);
if (error)
return (-1);
}		}

/*		/*
* 64-bit BAR takes up two slots so skip the next one.		* 64-bit BAR takes up two slots so skip the next one.
*/		*/
if (bartype == PCIBAR_MEM64) {		if (bartype == PCIBAR_MEM64) {
i++;		i++;
assert(i <= PCI_BARMAX);		assert(i <= PCI_BARMAX);
Show All 24 Lines	cfginit(struct vmctx ctx, struct pci_devinst pi, int bus, int slot, int func)
}		}

if (cfginitbar(ctx, sc) != 0) {		if (cfginitbar(ctx, sc) != 0) {
warnx("failed to initialize BARs for PCI %d/%d/%d",		warnx("failed to initialize BARs for PCI %d/%d/%d",
bus, slot, func);		bus, slot, func);
goto done;		goto done;
}		}

pci_set_cfgdata16(pi, PCIR_COMMAND, read_config(&sc->psc_sel,		/* sync command register */
PCIR_COMMAND, 2));		write_config(&sc->psc_sel, PCIR_COMMAND, 0x02,
		pci_get_cfgdata16(pi, PCIR_COMMAND));

error = 0; /* success */		error = 0; /* success */
done:		done:
return (error);		return (error);
}		}

		#define PPT_PCIR_PROT(reg) ((sc->psc_pcir_prot_map[reg / 4] >> (reg & 0x03)) & PPT_PCIR_PROT_MASK)

		int
		set_pcir_prot(struct passthru_softc *sc, uint32_t reg, uint32_t len, uint8_t prot)
		{
		if (reg > PCI_REGMAX \|\| reg + len > PCI_REGMAX + 1)
		return (-1);

		prot &= PPT_PCIR_PROT_MASK;

		for (int i = reg; i < reg + len; ++i) {
		/* delete old prot value */
		sc->psc_pcir_prot_map[i / 4] &= ~(PPT_PCIR_PROT_MASK << (i & 0x03));
		/* set new prot value */
		sc->psc_pcir_prot_map[i / 4] \|= prot << (i & 0x03);
		}

		return (0);
		}

static int		static int
		is_pcir_writable(struct passthru_softc *sc, uint32_t reg)
		{
		if (reg > PCI_REGMAX)
		return (0);

		return ((PPT_PCIR_PROT(reg) & PPT_PCIR_PROT_WO) != 0);
		}

		static int
		is_pcir_readable(struct passthru_softc *sc, uint32_t reg)
		{
		if (reg > PCI_REGMAX)
		return (0);

		return ((PPT_PCIR_PROT(reg) & PPT_PCIR_PROT_RO) != 0);
		}

		static int
		passthru_init_quirks(struct vmctx ctx, struct pci_devinst pi, char *opts)
		{
		struct passthru_softc *sc = pi->pi_arg;

		uint16_t vendor = read_config(&sc->psc_sel, PCIR_VENDOR, 0x02);
		uint8_t class = read_config(&sc->psc_sel, PCIR_CLASS, 0x01);

		/* currently only display devices have quirks */
		if (class != PCIC_DISPLAY)
		return (0);

		if (vendor == PCI_VENDOR_AMD)
		return apu_d_init(ctx, pi, opts);

		return (0);
		}

		static void
		passthru_deinit_quirks(struct vmctx ctx, struct pci_devinst pi)
		{
		struct passthru_softc *sc = pi->pi_arg;

		//uint16_t vendor = read_config(&sc->psc_sel, PCIR_VENDOR, 0x02);
		uint8_t class = read_config(&sc->psc_sel, PCIR_CLASS, 0x01);

		/* currently only display devices have quirks */
		if (class != PCIC_DISPLAY)
		return;

		return;
		}

		static int
passthru_init(struct vmctx ctx, struct pci_devinst pi, char *opts)		passthru_init(struct vmctx ctx, struct pci_devinst pi, char *opts)
{		{
int bus, slot, func, error, memflags;		int bus, slot, func, error, memflags;
struct passthru_softc *sc;		struct passthru_softc *sc;
#ifndef WITHOUT_CAPSICUM		#ifndef WITHOUT_CAPSICUM
cap_rights_t rights;		cap_rights_t rights;
cap_ioctl_t pci_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR };		cap_ioctl_t pci_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR };
cap_ioctl_t io_ioctls[] = { IODEV_PIO };		cap_ioctl_t io_ioctls[] = { IODEV_PIO };
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines	#endif
}		}

sc = calloc(1, sizeof(struct passthru_softc));		sc = calloc(1, sizeof(struct passthru_softc));

pi->pi_arg = sc;		pi->pi_arg = sc;
sc->psc_pi = pi;		sc->psc_pi = pi;

/* initialize config space */		/* initialize config space */
error = cfginit(ctx, pi, bus, slot, func);		if ((error = cfginit(ctx, pi, bus, slot, func)) != 0)
		goto done;

		/* allow access to all PCI registers */
		if ((error = set_pcir_prot(sc, 0, PCI_REGMAX + 1, PPT_PCIR_PROT_RW)) != 0)
		goto done;

		if ((error = passthru_init_quirks(ctx, pi, opts)) != 0)
		goto done;

		error = 0; /* success */
done:		done:
if (error) {		if (error) {
		passthru_deinit_quirks(ctx, pi);
free(sc);		free(sc);
vm_unassign_pptdev(ctx, bus, slot, func);		vm_unassign_pptdev(ctx, bus, slot, func);
}		}
return (error);		return (error);
}		}

static int		static int
bar_access(int coff)		bar_access(int coff)
Show All 33 Lines
static int		static int
passthru_cfgread(struct vmctx ctx, int vcpu, struct pci_devinst pi,		passthru_cfgread(struct vmctx ctx, int vcpu, struct pci_devinst pi,
int coff, int bytes, uint32_t *rv)		int coff, int bytes, uint32_t *rv)
{		{
struct passthru_softc *sc;		struct passthru_softc *sc;

sc = pi->pi_arg;		sc = pi->pi_arg;

		/* skip for protected PCI registers */
		if (!is_pcir_readable(sc, coff))
		return (-1);

/*		/*
* PCI BARs and MSI capability is emulated.		* PCI BARs and MSI capability is emulated.
*/		*/
if (bar_access(coff) \|\| msicap_access(sc, coff))		if (bar_access(coff) \|\| msicap_access(sc, coff))
return (-1);		return (-1);

		/*
		* PCI ROM is emulated
		*/
		if (coff >= PCIR_BIOS && coff < PCIR_BIOS + 4)
		return (-1);

#ifdef LEGACY_SUPPORT		#ifdef LEGACY_SUPPORT
/*		/*
* Emulate PCIR_CAP_PTR if this device does not support MSI capability		* Emulate PCIR_CAP_PTR if this device does not support MSI capability
* natively.		* natively.
*/		*/
if (sc->psc_msi.emulated) {		if (sc->psc_msi.emulated) {
if (coff >= PCIR_CAP_PTR && coff < PCIR_CAP_PTR + 4)		if (coff >= PCIR_CAP_PTR && coff < PCIR_CAP_PTR + 4)
return (-1);		return (-1);
Show All 24 Lines	passthru_cfgwrite(struct vmctx ctx, int vcpu, struct pci_devinst pi,
int coff, int bytes, uint32_t val)		int coff, int bytes, uint32_t val)
{		{
int error, msix_table_entries, i;		int error, msix_table_entries, i;
struct passthru_softc *sc;		struct passthru_softc *sc;
uint16_t cmd_old;		uint16_t cmd_old;

sc = pi->pi_arg;		sc = pi->pi_arg;

		/* skip for protected PCI registers */
		if (!is_pcir_writable(sc, coff))
		return (-1);

/*		/*
* PCI BARs are emulated		* PCI BARs are emulated
*/		*/
if (bar_access(coff))		if (bar_access(coff))
return (-1);		return (-1);

/*		/*
		* PCI ROM is emulated
		*/
		if (coff >= PCIR_BIOS && coff < PCIR_BIOS + 4)
		return (-1);

		/*
* MSI capability is emulated		* MSI capability is emulated
*/		*/
if (msicap_access(sc, coff)) {		if (msicap_access(sc, coff)) {
pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff,		pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff,
PCIY_MSI);		PCIY_MSI);
error = vm_setup_pptdev_msi(ctx, vcpu, sc->psc_sel.pc_bus,		error = vm_setup_pptdev_msi(ctx, vcpu, sc->psc_sel.pc_bus,
sc->psc_sel.pc_dev, sc->psc_sel.pc_func,		sc->psc_sel.pc_dev, sc->psc_sel.pc_func,
pi->pi_msi.addr, pi->pi_msi.msg_data,		pi->pi_msi.addr, pi->pi_msi.msg_data,
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines	if (baridx == pci_msix_table_bar(pi)) {
(void)ioctl(iofd, IODEV_PIO, &pio);		(void)ioctl(iofd, IODEV_PIO, &pio);

val = pio.val;		val = pio.val;
}		}

return (val);		return (val);
}		}

		static int
		passthru_addr_rom(struct pci_devinst *pi, int idx, int enabled)
		{
		struct passthru_softc *sc = pi->pi_arg;

		uint16_t vendor = read_config(&sc->psc_sel, PCIR_VENDOR, 0x02);
		uint8_t class = read_config(&sc->psc_sel, PCIR_CLASS, 0x01);

		/* currently only display devices have ROMs */
		if (class != PCIC_DISPLAY)
		return (0);

		if (vendor == PCI_VENDOR_AMD)
		return apu_d_addr_rom(pi, idx, enabled);

		return (0);
		}

		static int
		passthru_addr(struct vmctx ctx, struct pci_devinst pi, int idx,
		int enabled, uint64_t address)
		{
		int error;

		switch (pi->pi_bar[idx].type) {
		case PCIBAR_IO:
		/* IO BARs are emulated */
		return (-1);
		case PCIBAR_ROM:
		/* Only quirk devices have a ROM */
		assert(idx == PCI_BARMAX + 1);
		error = passthru_addr_rom(pi, idx , enabled);
		break;
		case PCIBAR_MEM32:
		case PCIBAR_MEM64:
		error = passthru_modify_bar_registration(pi, idx, enabled);
		break;
		default:
		error = EINVAL;
		break;
		}

		assert(error == 0);
		return error;
		}

struct pci_devemu passthru = {		struct pci_devemu passthru = {
.pe_emu = "passthru",		.pe_emu = "passthru",
.pe_init = passthru_init,		.pe_init = passthru_init,
.pe_cfgwrite = passthru_cfgwrite,		.pe_cfgwrite = passthru_cfgwrite,
.pe_cfgread = passthru_cfgread,		.pe_cfgread = passthru_cfgread,
.pe_barwrite = passthru_write,		.pe_barwrite = passthru_write,
.pe_barread = passthru_read,		.pe_barread = passthru_read,
		.pe_baraddr = passthru_addr,
};		};
PCI_EMUL_SET(passthru);		PCI_EMUL_SET(passthru);