diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8 index a352a7ba2a93..1de8c516b785 100644 --- a/usr.sbin/bhyve/bhyve.8 +++ b/usr.sbin/bhyve/bhyve.8 @@ -1,923 +1,942 @@ .\" Copyright (c) 2013 Peter Grehan .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd October 13, 2021 +.Dd August 19, 2022 .Dt BHYVE 8 .Os .Sh NAME .Nm bhyve .Nd "run a guest operating system inside a virtual machine" .Sh SYNOPSIS .Nm .Op Fl AaCDeHhPSuWwxY .Oo .Sm off .Fl c\~ .Oo .Op Cm cpus= .Ar numcpus .Oc .Op Cm ,sockets= Ar n .Op Cm ,cores= Ar n .Op Cm ,threads= Ar n .Oc .Sm on .Op Fl G Ar port .Op Fl k Ar config_file .Oo Fl l .Sm off .Ar lpcdev Op Cm \&, Ar conf .Sm on .Oc .Oo Fl m .Sm off .Ar memsize .Oo .Cm K | Cm k | Cm M | Cm m | Cm G | Cm g | Cm T | Cm t .Oc .Sm on .Oc .Op Fl o Ar var Ns Cm = Ns Ar value .Op Fl p Ar vcpu Ns Cm \&: Ns Ar hostcpu .Op Fl r Ar file .Sm off .Oo Fl s\~ .Ar slot Cm \&, Ar emulation Op Cm \&, Ar conf .Sm on .Oc .Op Fl U Ar uuid .Ar vmname .Nm .Fl l Cm help .Nm .Fl s Cm help .Sh DESCRIPTION .Nm is a hypervisor that runs guest operating systems inside a virtual machine. .Pp Parameters such as the number of virtual CPUs, amount of guest memory, and I/O connectivity can be specified with command-line parameters. .Pp If not using a boot ROM, the guest operating system must be loaded with .Xr bhyveload 8 or a similar boot loader before running .Nm , otherwise, it is enough to run .Nm with a boot ROM of choice. .Pp .Nm runs until the guest operating system reboots or an unhandled hypervisor exit is detected. .Sh OPTIONS .Bl -tag -width 10n .It Fl A Generate ACPI tables. Required for .Fx Ns /amd64 guests. .It Fl a The guest's local APIC is configured in xAPIC mode. The xAPIC mode is the default setting so this option is redundant. It will be deprecated in a future version. .It Fl C Include guest memory in core file. .It Fl c Op Ar setting ... Number of guest virtual CPUs and/or the CPU topology. The default value for each of .Ar numcpus , .Ar sockets , .Ar cores , and .Ar threads is 1. The current maximum number of guest virtual CPUs is 16. If .Ar numcpus is not specified then it will be calculated from the other arguments. The topology must be consistent in that the .Ar numcpus must equal the product of .Ar sockets , .Ar cores , and .Ar threads . If a .Ar setting is specified more than once the last one has precedence. .It Fl D Destroy the VM on guest initiated power-off. .It Fl e Force .Nm to exit when a guest issues an access to an I/O port that is not emulated. This is intended for debug purposes. .It Fl G Ar port Start a debug server that uses the GDB protocol to export guest state to a debugger. An IPv4 TCP socket will be bound to the supplied .Ar port to listen for debugger connections. Only a single debugger may be attached to the debug server at a time. If .Ar port begins with .Sq w , .Nm will pause execution at the first instruction waiting for a debugger to attach. .It Fl H Yield the virtual CPU thread when a HLT instruction is detected. If this option is not specified, virtual CPUs will use 100% of a host CPU. .It Fl h Print help message and exit. .It Fl k Ar config_file Set configuration variables from a simple, key-value config file. Each line of the config file is expected to consist of a config variable name, an equals sign .Pq Sq = , and a value. No spaces are permitted between the variable name, equals sign, or value. Blank lines and lines starting with .Sq # are ignored. See .Xr bhyve_config 5 for more details. .It Fl l Cm help Print a list of supported LPC devices. .It Fl l Ar lpcdev Ns Op Cm \&, Ns Ar conf Allow devices behind the LPC PCI-ISA bridge to be configured. The only supported devices are the TTY-class devices .Cm com1 , com2 , com3 , and .Cm com4 , the boot ROM device .Cm bootrom , and the debug/test device .Cm pc-testdev . .Pp The possible values for the .Ar conf argument are listed in the .Fl s flag description. .It Xo .Fl m Ar memsize Ns Oo .Sm off .Cm K | k | M | m | G | g | T | t .Sm on .Oc .Xc Set the guest physical memory size This must be the same size that was given to .Xr bhyveload 8 . .Pp The size argument may be suffixed with one of .Cm K , M , G or .Cm T (either upper or lower case) to indicate a multiple of kilobytes, megabytes, gigabytes, or terabytes. If no suffix is given, the value is assumed to be in megabytes. .Pp The default is 256M. .It Fl o Ar var Ns Cm = Ns Ar value Set the configuration variable .Ar var to .Ar value . .It Fl P Force the guest virtual CPU to exit when a PAUSE instruction is detected. .It Fl p Ar vcpu Ns Cm \& : Ns Ar hostcpu Pin guest's virtual CPU .Em vcpu to .Em hostcpu . .It Fl r Ar file Resume a guest from a snapshot. The guest memory contents are restored from .Ar file , and the guest device and vCPU state are restored from the file .Dq Ar file Ns .kern . .Pp Note that the current snapshot file format requires that the configuration of devices in the new VM match the VM from which the snapshot was taken by specifying the same .Fl s and .Fl l options. The count of vCPUs and memory configuration are read from the snapshot. .It Fl S Wire guest memory. .It Fl s Cm help Print a list of supported PCI devices. .It Fl s Ar slot Ns Cm \&, Ns Ar emulation Ns Op Cm \&, Ns Ar conf Configure a virtual PCI slot and function. .Pp .Nm provides PCI bus emulation and virtual devices that can be attached to slots on the bus. There are 32 available slots, with the option of providing up to 8 functions per slot. .Pp The .Ar slot can be specified in one of the following formats: .Pp .Bl -bullet -compact .It .Ar pcislot .It .Sm off .Ar pcislot Cm \&: Ar function .Sm on .It .Sm off .Ar bus Cm \&: Ar pcislot Cm \&: Ar function .Sm on .El .Pp The .Ar pcislot value is 0 to 31. The optional .Ar function value is 0 to 7. The optional .Ar bus value is 0 to 255. If not specified, the .Ar function value defaults to 0. If not specified, the .Ar bus value defaults to 0. .Pp The .Ar emulation argument can be one of the following: .Bl -tag -width "amd_hostbridge" .It Cm hostbridge A simple host bridge. This is usually configured at slot 0, and is required by most guest operating systems. .It Cm amd_hostbridge Emulation identical to .Cm hostbridge using a PCI vendor ID of AMD. .It Cm passthru PCI pass-through device. .It Cm virtio-net Virtio network interface. .It Cm virtio-blk Virtio block storage interface. .It Cm virtio-scsi Virtio SCSI interface. .It Cm virtio-9p Virtio 9p (VirtFS) interface. .It Cm virtio-rnd Virtio RNG interface. .It Cm virtio-console Virtio console interface, which exposes multiple ports to the guest in the form of simple char devices for simple IO between the guest and host userspaces. .It Cm ahci AHCI controller attached to arbitrary devices. .It Cm ahci-cd AHCI controller attached to an ATAPI CD/DVD. .It Cm ahci-hd AHCI controller attached to a SATA hard drive. .It Cm e1000 Intel e82545 network interface. .It Cm uart PCI 16550 serial device. .It Cm lpc LPC PCI-ISA bridge with COM1, COM2, COM3, and COM4 16550 serial ports, a boot ROM, and, optionally, the debug/test device. The LPC bridge emulation can only be configured on bus 0. .It Cm fbuf Raw framebuffer device attached to VNC server. .It Cm xhci eXtensible Host Controller Interface (xHCI) USB controller. .It Cm nvme NVM Express (NVMe) controller. .It Cm hda High Definition Audio Controller. .El .Pp The optional parameter .Ar conf describes the backend for device emulations. If .Ar conf is not specified, the device emulation has no backend and can be considered unconnected. .Pp Network device backends: .Sm off .Bl -bullet .It .Xo .Cm tap Ar N .Op Cm \&,mac= Ar xx:xx:xx:xx:xx:xx .Op Cm \&,mtu= Ar N .Xc .It .Xo .Cm vmnet Ar N .Op Cm \&,mac= Ar xx:xx:xx:xx:xx:xx .Op Cm \&,mtu= Ar N .Xc .It .Xo .Cm netgraph,path= Ar ADDRESS Cm \&,peerhook= Ar HOOK .Op Cm \&,socket= Ar NAME .Op Cm \&,hook= Ar HOOK .Op Cm \&,mac= Ar xx:xx:xx:xx:xx:xx .Op Cm \&,mtu= Ar N .Xc .El .Sm on +.Pp If .Cm mac is not specified, the MAC address is derived from a fixed OUI and the remaining bytes from an MD5 hash of the slot and function numbers and the device name. .Pp The MAC address is an ASCII string in .Xr ethers 5 format. .Pp With .Cm virtio-net devices, the .Cm mtu parameter can be specified to inform the guest about the largest MTU that should be allowed, expressed in bytes. .Pp With .Cm netgraph backend, the .Cm path and .Cm peerhook parameters must be specified to set the destination node and corresponding hook. The optional parameters .Cm socket and .Cm hook may be used to set the .Xr ng_socket 4 node name and source hook. The .Ar ADDRESS , .Ar HOOK , and .Ar NAME must comply with .Xr netgraph 4 addressing rules. .Pp Block storage device backends: .Sm off .Bl -bullet .It .Ar /filename Op Cm \&, Ar block-device-options .It .Ar /dev/xxx Op Cm \&, Ar block-device-options .El .Sm on .Pp The .Ar block-device-options are: .Bl -tag -width 10n .It Cm nocache Open the file with .Dv O_DIRECT . .It Cm direct Open the file using .Dv O_SYNC . .It Cm ro Force the file to be opened read-only. .It Cm sectorsize= Ns Ar logical Ns Oo Cm \&/ Ns Ar physical Oc Specify the logical and physical sector sizes of the emulated disk. The physical sector size is optional and is equal to the logical sector size if not explicitly specified. .It Cm nodelete Disable emulation of guest trim requests via .Dv DIOCGDELETE requests. .El .Pp SCSI device backends: .Sm off .Bl -bullet .It .Pa /dev/cam/ctl Oo Ar pp Cm \&. Ar vp Oc Oo Cm \&, Ar scsi-device-options Oc .El .Sm on .Pp The .Ar scsi-device-options are: .Bl -tag -width 10n .It Cm iid= Ns Ar IID Initiator ID to use when sending requests to specified CTL port. The default value is 0. .El .Pp 9P device backends: .Sm off .Bl -bullet .It .Ar sharename Cm = Ar /path/to/share Op Cm \&, Ar 9p-device-options .El .Sm on .Pp The .Ar 9p-device-options are: .Bl -tag -width 10n .It Cm ro Expose the share in read-only mode. .El .Pp TTY device backends: .Bl -tag -width 10n .It Cm stdio Connect the serial port to the standard input and output of the .Nm process. .It Ar /dev/xxx Use the host TTY device for serial port I/O. .El .Pp Boot ROM device backends: .Bl -tag -width 10n .It Ar romfile Map .Ar romfile in the guest address space reserved for boot firmware. .El .Pp Pass-through device backends: -.Bl -tag -width 10n -.It Ns Ar slot Ns Cm \&/ Ns Ar bus Ns Cm \&/ Ns Ar function -Connect to a PCI device on the host at the selector described by +.Sm off +.Bl -bullet +.It +.Cm ppt Ar N Oo , Ar passthru-device-options Oc +.It +.Ns Ar bus Cm \&/ Ar slot Cm \&/ Ar function +.Op , Ar passthru-device-options +.It +.Cm pci Ar bus Cm : Ar slot Cm : Ns Ar function +.Op , Ar passthru-device-options +.El +.Sm on +.Pp +Connect to a PCI device on the host either named ppt +.Ns Ar N +or at the selector described by .Ar slot , .Ar bus , and .Ar function numbers. +.Pp +The +.Ar passthru-device-options +are: +.Bl -tag -width 10n .It Cm rom= Ns Ar romfile Add .Ar romfile as option ROM to the PCI device. The ROM will be loaded by firmware and should be capable of initializing the device. .El .Pp Guest memory must be wired using the .Fl S option when a pass-through device is configured. .Pp The host device must have been reserved at boot-time using the .Va pptdevs loader variable as described in .Xr vmm 4 . .Pp Virtio console device backends: .Bl -bullet .Sm off .It .Cm port1= Ns Ar /path/to/port1.sock Ns Op Cm ,port Ns Ar N Cm \&= Ns Ar /path/to/port2.sock No \~ Ar ... .Sm on .El .Pp A maximum of 16 ports per device can be created. Every port is named and corresponds to a Unix domain socket created by .Nm . .Nm accepts at most one connection per port at a time. .Pp Limitations: .Bl -bullet .It Due to lack of destructors in .Nm , sockets on the filesystem must be cleaned up manually after .Nm exits. .It There is no way to use the .Dq console port feature, nor the console port resize at present. .It Emergency write is advertised, but no-op at present. .El .Pp Framebuffer devices backends: .Bl -bullet .Sm off .It .Op Cm rfb= Ar ip-and-port .Op Cm ,w= Ar width .Op Cm ,h= Ar height .Op Cm ,vga= Ar vgaconf .Op Cm ,wait .Op Cm ,password= Ar password .Sm on .El .Pp Configuration options are defined as follows: .Bl -tag -width 10n .It Cm rfb= Ns Ar ip-and-port Pq or Cm tcp= Ns Ar ip-and-port An IP address and a port VNC should listen on. There are two formats: .Pp .Bl -bullet -compact .It .Sm off .Op Ar IPv4 Cm \&: .Ar port .Sm on .It .Sm off .Cm \&[ Ar IPv6%zone Cm \&] Cm \&: Ar port .Sm on .El .Pp The default is to listen on localhost IPv4 address and default VNC port 5900. An IPv6 address must be enclosed in square brackets and may contain an optional zone identifier. .It Cm w= Ns Ar width No and Cm h= Ns Ar height A display resolution, width and height, respectively. If not specified, a default resolution of 1024x768 pixels will be used. Minimal supported resolution is 640x480 pixels, and maximum is 1920x1200 pixels. .It Cm vga= Ns Ar vgaconf Possible values for this option are .Cm io (default), .Cm on , and .Cm off . PCI graphics cards have a dual personality in that they are standard PCI devices with BAR addressing, but may also implicitly decode legacy VGA I/O space .Pq Ad 0x3c0-3df and memory space .Pq 64KB at Ad 0xA0000 . The default .Cm io option should be used for guests that attempt to issue BIOS calls which result in I/O port queries, and fail to boot if I/O decode is disabled. .Pp The .Cm on option should be used along with the CSM BIOS capability in UEFI to boot traditional BIOS guests that require the legacy VGA I/O and memory regions to be available. .Pp The .Cm off option should be used for the UEFI guests that assume that VGA adapter is present if they detect the I/O ports. An example of such a guest is .Ox in UEFI mode. .Pp Please refer to the .Nm .Fx wiki page .Pq Lk https://wiki.freebsd.org/bhyve for configuration notes of particular guests. .It Cm wait Instruct .Nm to only boot upon the initiation of a VNC connection, simplifying the installation of operating systems that require immediate keyboard input. This can be removed for post-installation use. .It Cm password= Ns Ar password This type of authentication is known to be cryptographically weak and is not intended for use on untrusted networks. Many implementations will want to use stronger security, such as running the session over an encrypted channel provided by IPsec or SSH. .El .Pp xHCI USB device backends: .Bl -tag -width 10n .It Cm tablet A USB tablet device which provides precise cursor synchronization when using VNC. .El .Pp NVMe device backends: .Bl -bullet .Sm off .It .Ar devpath .Op Cm ,maxq= Ar # .Op Cm ,qsz= Ar # .Op Cm ,ioslots= Ar # .Op Cm ,sectsz= Ar # .Op Cm ,ser= Ar # .Op Cm ,eui64= Ar # .Op Cm ,dsm= Ar opt .Sm on .El .Pp Configuration options are defined as follows: .Bl -tag -width 10n .It Ar devpath Accepted device paths are: .Ar /dev/blockdev or .Ar /path/to/image or .Cm ram= Ns Ar size_in_MiB . .It Cm maxq Max number of queues. .It Cm qsz Max elements in each queue. .It Cm ioslots Max number of concurrent I/O requests. .It Cm sectsz Sector size (defaults to blockif sector size). .It Cm ser Serial number with maximum 20 characters. .It Cm eui64 IEEE Extended Unique Identifier (8 byte value). .It Cm dsm DataSet Management support. Supported values are: .Cm auto , enable , and .Cm disable . .El .Pp AHCI device backends: .Bl -bullet .It .Sm off .Op Oo Cm hd\&: | cd\&: Oc Ar path .Op Cm ,nmrr= Ar nmrr .Op Cm ,ser= Ar # .Op Cm ,rev= Ar # .Op Cm ,model= Ar # .Sm on .El .Pp Configuration options are defined as follows: .Bl -tag -width 10n .It Cm nmrr Nominal Media Rotation Rate, known as RPM. Value 1 will indicate device as Solid State Disk. Default value is 0, not report. .It Cm ser Serial Number with maximum 20 characters. .It Cm rev Revision Number with maximum 8 characters. .It Cm model Model Number with maximum 40 characters. .El .Pp HD Audio device backends: .Bl -bullet .It .Sm off .Op Cm play= Ar playback .Op Cm ,rec= Ar recording .Sm on .El .Pp Configuration options are defined as follows: .Bl -tag -width 10n .It Cm play Playback device, typically .Ar /dev/dsp0 . .It Cm rec Recording device, typically .Ar /dev/dsp0 . .El .It Fl U Ar uuid Set the universally unique identifier .Pq UUID in the guest's System Management BIOS System Information structure. By default a UUID is generated from the host's hostname and .Ar vmname . .It Fl u RTC keeps UTC time. .It Fl W Force virtio PCI device emulations to use MSI interrupts instead of MSI-X interrupts. .It Fl w Ignore accesses to unimplemented Model Specific Registers (MSRs). This is intended for debug purposes. .It Fl x The guest's local APIC is configured in x2APIC mode. .It Fl Y Disable MPtable generation. .It Ar vmname Alphanumeric name of the guest. This should be the same as that created by .Xr bhyveload 8 . .El .Sh CONFIGURATION VARIABLES .Nm uses an internal tree of configuration variables to describe global and per-device settings. When .Nm starts, it parses command line options (including config files) in the order given on the command line. Each command line option sets one or more configuration variables. For example, the .Fl s option creates a new tree node for a PCI device and sets one or more variables under that node including the device model and device model-specific variables. Variables may be set multiple times during this parsing stage with the final value overriding previous values. .Pp Once all of the command line options have been processed, the configuration values are frozen. .Nm then uses the value of configuration values to initialize device models and global settings. .Pp More details on configuration variables can be found in .Xr bhyve_config 5 . .Sh DEBUG SERVER The current debug server provides limited support for debuggers. .Ss Registers Each virtual CPU is exposed to the debugger as a thread. .Pp General purpose registers can be queried for each virtual CPU, but other registers such as floating-point and system registers cannot be queried. .Ss Memory Memory (including memory mapped I/O regions) can be read and written by the debugger. Memory operations use virtual addresses that are resolved to physical addresses via the current virtual CPU's active address translation. .Ss Control The running guest can be interrupted by the debugger at any time .Pq for example, by pressing Ctrl-C in the debugger . .Pp Single stepping is only supported on Intel CPUs supporting the MTRAP VM exit. .Pp Breakpoints are supported on Intel CPUs that support single stepping. Note that continuing from a breakpoint while interrupts are enabled in the guest may not work as expected due to timer interrupts firing while single stepping over the breakpoint. .Sh SIGNAL HANDLING .Nm deals with the following signals: .Pp .Bl -tag -width SIGTERM -compact .It SIGTERM Trigger ACPI poweroff for a VM .El .Sh EXIT STATUS Exit status indicates how the VM was terminated: .Pp .Bl -tag -width indent -compact .It 0 rebooted .It 1 powered off .It 2 halted .It 3 triple fault .It 4 exited due to an error .El .Sh EXAMPLES If not using a boot ROM, the guest operating system must have been loaded with .Xr bhyveload 8 or a similar boot loader before .Xr bhyve 4 can be run. Otherwise, the boot loader is not needed. .Pp To run a virtual machine with 1GB of memory, two virtual CPUs, a virtio block device backed by the .Pa /my/image filesystem image, and a serial port for the console: .Bd -literal -offset indent bhyve -c 2 -s 0,hostbridge -s 1,lpc -s 2,virtio-blk,/my/image \\ -l com1,stdio -A -H -P -m 1G vm1 .Ed .Pp Run a 24GB single-CPU virtual machine with three network ports, one of which has a MAC address specified: .Bd -literal -offset indent bhyve -s 0,hostbridge -s 1,lpc -s 2:0,virtio-net,tap0 \\ -s 2:1,virtio-net,tap1 \\ -s 2:2,virtio-net,tap2,mac=00:be:fa:76:45:00 \\ -s 3,virtio-blk,/my/image -l com1,stdio \\ -A -H -P -m 24G bigvm .Ed .Pp Run an 8GB quad-CPU virtual machine with 8 AHCI SATA disks, an AHCI ATAPI CD-ROM, a single virtio network port, an AMD hostbridge, and the console port connected to an .Xr nmdm 4 null-modem device. .Bd -literal -offset indent bhyve -c 4 \\ -s 0,amd_hostbridge -s 1,lpc \\ -s 1:0,ahci,hd:/images/disk.1,hd:/images/disk.2,\\ hd:/images/disk.3,hd:/images/disk.4,\\ hd:/images/disk.5,hd:/images/disk.6,\\ hd:/images/disk.7,hd:/images/disk.8,\\ cd:/images/install.iso \\ -s 3,virtio-net,tap0 \\ -l com1,/dev/nmdm0A \\ -A -H -P -m 8G .Ed .Pp Run a UEFI virtual machine with a display resolution of 800 by 600 pixels that can be accessed via VNC at: 0.0.0.0:5900. .Bd -literal -offset indent bhyve -c 2 -m 4G -w -H \\ -s 0,hostbridge \\ -s 3,ahci-cd,/path/to/uefi-OS-install.iso \\ -s 4,ahci-hd,disk.img \\ -s 5,virtio-net,tap0 \\ -s 29,fbuf,tcp=0.0.0.0:5900,w=800,h=600,wait \\ -s 30,xhci,tablet \\ -s 31,lpc -l com1,stdio \\ -l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \\ uefivm .Ed .Pp Run a UEFI virtual machine with a VNC display that is bound to all IPv6 addresses on port 5900. .Bd -literal -offset indent bhyve -c 2 -m 4G -w -H \\ -s 0,hostbridge \\ -s 4,ahci-hd,disk.img \\ -s 5,virtio-net,tap0 \\ -s 29,fbuf,tcp=[::]:5900,w=800,h=600 \\ -s 30,xhci,tablet \\ -s 31,lpc -l com1,stdio \\ -l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \\ uefivm .Ed .Sh SEE ALSO .Xr bhyve 4 , .Xr netgraph 4 , .Xr ng_socket 4 , .Xr nmdm 4 , .Xr vmm 4 , .Xr bhyve_config 5 , .Xr ethers 5 , .Xr bhyvectl 8 , .Xr bhyveload 8 .Pp .Rs .%A Intel .%B 64 and IA-32 Architectures Software Developer’s Manual .%V Volume 3 .Re .Sh HISTORY .Nm first appeared in .Fx 10.0 . .Sh AUTHORS .An Neel Natu Aq Mt neel@freebsd.org .An Peter Grehan Aq Mt grehan@freebsd.org diff --git a/usr.sbin/bhyve/bhyve_config.5 b/usr.sbin/bhyve/bhyve_config.5 index fbbea50bc7be..b4160a37620d 100644 --- a/usr.sbin/bhyve/bhyve_config.5 +++ b/usr.sbin/bhyve/bhyve_config.5 @@ -1,620 +1,630 @@ .\" SPDX-License-Identifier: BSD-2-Clause .\" .\" Copyright (c) 2021 John H. Baldwin .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd September 17, 2021 +.Dd August 19, 2022 .Dt BHYVE_CONFIG 5 .Os .Sh NAME .Nm bhyve_config .Nd "bhyve configuration variables" .Sh DESCRIPTION .Xr bhyve 8 uses a hierarchical tree of configuration variables to describe global and per-device settings. Internal nodes in this tree do not have a value, only leaf nodes have values. This manual describes the configuration variables understood by .Xr bhyve 8 . If additional variables are defined, .Xr bhyve 8 will ignore them and will not emit errors for unknown variables. However, these additional variables can be referenced by other variables as described below. .Sh VARIABLE VALUES Configuration variable values are stored as strings. A configuration variable value may refer to one or more other configuration values by name. Instances of the pattern .Sq % Ns Pq Ar var are replaced by the value of the configuration variable .Va var . To avoid unwanted expansion, .Sq % characters can be escaped by a leading .Sq % . For example, if a configuration variable .Va disk uses the value .Pa /dev/zvol/bhyve/%(name) , then the final value of the .Va disk variable will be set to the path of a ZFS volume whose name matches the name of the virtual machine on the pool .Pa bhyve . .Pp Some configuration variables may be interpreted as a boolean value. For those variables the following case-insensitive values may be used to indicate true: .Pp .Bl -bullet -offset indent -compact .It true .It on .It yes .It 1 .El .Pp The following values may be used to indicate false: .Pp .Bl -bullet -offset indent -compact .It false .It off .It no .It 0 .El .Pp Some configuration variables may be interperted as an integer. For those variables, any syntax supported by .Xr strtol 3 may be used. .Sh GLOBAL SETTINGS .Ss Architecture Neutral Settings .Bl -column "memory.guest_in_core" "integer" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va name Ta string Ta Ta The name of the VM. .It Va cpus Ta integer Ta 1 Ta The total number of virtual CPUs. .It Va cores Ta integer Ta 1 Ta The number of virtual cores in each virtual socket. .It Va threads Ta integer Ta 1 Ta The number of virtual CPUs in each virtual core. .It Va sockets Ta integer Ta 1 Ta The number of virtual sockets. .It Va memory.guest_in_core Ta bool Ta false Ta Include guest memory in core file. .It Va memory.size Ta string Ta 256M Ta Guest physical memory size in bytes. The value must be formatted as described in .Xr expand_number 3 . .It Va memory.wired Ta bool Ta false Ta Wire guest memory. .It Va acpi_tables Ta bool Ta false Ta Generate ACPI tables. .It Va destroy_on_poweroff Ta bool Ta false Ta Destroy the VM on guest-initiated power-off. .It Va gdb.port Ta integer Ta 0 Ta TCP port number for the debug server. If this is set to a non-zero value, a debug server will listen for connections on this port. .It Va gdb.wait Ta bool Ta false Ta If the debug server is enabled, wait for a debugger to connect before starting the guest. .It Va rtc.use_localtime Ta bool Ta true Ta The real time clock uses the local time of the host. If this is set to false, the real time clock uses UTC. .It Va uuid Ta string Ta Ta The universally unique identifier (UUID) to use in the guest's System Management BIOS System Information structure. If an explicit value is not set, a valid UUID is generated from the host's hostname and the VM name. .It Va virtio_msix Ta bool Ta true Ta Use MSI-X interrupts for PCI VirtIO devices. If set to false, MSI interrupts are used instead. .It Va config.dump Ta bool Ta false Ta If this value is set to true after .Xr bhyve 8 has finished parsing command line options, then .Xr bhyve 8 will write all of its configuration variables to stdout and exit. No VM will be started. .It Va bios.vendor Ta string Ta BHYVE Ta This value is used for the guest's System Management BIOS System Information structure. .It Va bios.version Ta string Ta 14.0 Ta This value is used for the guest's System Management BIOS System Information structure. .It Va bios.release_date Ta string Ta 10/17/2021 Ta This value is used for the guest's System Management BIOS System Information structure. .It Va system.family_name Ta string Ta Virtual Machine Ta Family the computer belongs to. This value is used for the guest's System Management BIOS System Information structure. .It Va system.manufacturer Ta string Ta FreeBSD Ta This value is used for the guest's System Management BIOS System Information structure. .It Va system.product_name Ta string Ta BHYVE Ta This value is used for the guest's System Management BIOS System Information structure. .It Va system.serial_number Ta string Ta None Ta This value is used for the guest's System Management BIOS System Information structure. .It Va system.sku Ta string Ta None Ta Stock keeping unit of the computer. It's also called product ID or purchase order number. This value is used for the guest's System Management BIOS System Information structure. .It Va system.version Ta string Ta 1.0 Ta This value is used for the guest's System Management BIOS System Information structure. .It Va board.manufacturer Ta string Ta FreeBSD Ta This value is used for the guest's System Management BIOS System Information structure. .It Va board.product_name Ta string Ta BHYVE Ta This value is used for the guest's System Management BIOS System Information structure. .It Va board.version Ta string Ta 1.0 Ta This value is used for the guest's System Management BIOS System Information structure. .It Va board.serial_number Ta string Ta None Ta This value is used for the guest's System Management BIOS System Information structure. .It Va board.asset_tag Ta string Ta None Ta This value is used for the guest's System Management BIOS System Information structure. .It Va board.location Ta string Ta None Ta Describes the board's location within the chassis. This value is used for the guest's System Management BIOS System Information structure. .It Va chassis.manufacturer Ta string Ta FreeBSD Ta This value is used for the guest's System Management BIOS System Information structure. .It Va chassis.version Ta string Ta 1.0 Ta This value is used for the guest's System Management BIOS System Information structure. .It Va chassis.serial_number Ta string Ta None Ta This value is used for the guest's System Management BIOS System Information structure. .It Va chassis.asset_tag Ta string Ta None Ta This value is used for the guest's System Management BIOS System Information structure. .It Va chassis.sku Ta string Ta None Ta Stock keeping unit of the chassis. It's also called product ID or purchase order number. This value is used for the guest's System Management BIOS System Information structure. .El .Ss x86-Specific Settings .Bl -column "x86.vmexit_on_pause" "integer" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va x86.mptable Ta bool Ta true Ta Generate an MPTable. .It Va x86.x2apic Ta bool Ta false Ta Configure guest's local APICs in x2APIC mode. .It Va x86.strictio Ta bool Ta false Ta Exit if a guest accesses an I/O port that is not emulated. By default, writes are ignored and reads return all bits set. .It Va x86.strictmsr Ta bool Ta true Ta Inject a general protection fault if a guest accesses a Model Specific Register (MSR) that is not emulated. If this is false, writes are ignored and reads return zero. .It Va x86.vmexit_on_hlt Ta bool Ta false Ta Force a VM exit when a guest CPU executes the .Dv HLT instruction. This allows idle guest CPUs to yield the host CPU. .It Va x86.vmexit_on_pause Ta bool Ta false Ta Force a VM exit when a guest CPU executes the .Dv PAUSE instruction. .El .Sh DEVICE SETTINGS Device settings are stored under a device node. The device node's name is set by the parent bus of the device. .Ss PCI Device Settings PCI devices are described by a device node named .Dq pci Ns Ar bus . Ns Ar slot . Ns Ar function where each of .Ar bus , .Ar slot , and .Ar function are formatted as decimal values with no padding. All PCI device nodes must contain a configuration variable named .Dq device which specifies the device model to use. The following PCI device models are supported: .Bl -tag -indent .It Li hostbridge Provide a simple PCI-Host bridge device. This is usually configured at pci0:0:0 and is required by most guest operating systems. .It Li ahci AHCI storage controller. .It Li e1000 Intel e82545 network interface. .It Li fbuf VGA framebuffer device attached to VNC server. .It Li lpc LPC PCI-ISA bridge with COM1-COM4 16550 serial ports, a boot ROM, and an optional debug/test device. This device must be configured on bus 0. .It Li hda High Definition audio controller. .It Li nvme NVM Express (NVMe) controller. .It Li passthru PCI pass-through device. .It Li uart PCI 16550 serial device. .It Li virtio-9p VirtIO 9p (VirtFS) interface. .It Li virtio-blk VirtIO block storage interface. .It Li virtio-console VirtIO console interface. .It Li virtio-net VirtIO network interface. .It Li virtio-rnd VirtIO RNG interface. .It Li virtio-scsi VirtIO SCSI interface. .It Li xhci Extensible Host Controller Interface (XHCI) USB controller. .El .Ss USB Device Settings USB controller devices contain zero or more child USB devices attached to slots. Each USB device stores its settings in a node named .Dq slot. Ns Va N under the controller's device node. .Va N is the number of the slot to which the USB device is attached. Note that USB slot numbers begin at 1. All USB device nodes must contain a configuration variable named .Dq device which specifies the device model to use. The following USB device models are supported: .Bl -tag -indent .It Li tablet A USB tablet device which provides precise cursor synchronization when using VNC. .El .Ss Block Device Settings Block devices use the following settings to configure their backing store. These settings are stored in the configuration node of the respective device. .Bl -column "sectorsize" "logical[/physical]" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It path Ta string Ta Ta The path of the file or disk device to use as the backing store. .It nocache Ta bool Ta false Ta Disable caching on the backing file by opening the backing file with .Dv O_DIRECT . .It nodelete Ta bool Ta false Ta Disable emulation of guest trim requests via .Dv DIOCGDELETE requests. .It sync Ta bool Ta false Ta Write changes to the backing file with synchronous writes. .It direct Ta bool Ta false Ta An alias for .Va sync . .It ro Ta bool Ta false Ta Disable writes to the backing file. .It sectorsize Ta Va logical Ns Op / Ns Va physical Ta Ta Specify the logical and physical sector size of the emulated disk. If the physical size is not specified, it is equal to the logical size. .El .Ss Network Backend Settings Network devices use the following settings to configure their backend. The backend is responsible for passing packets between the device model and a desired destination. Configuring a backend requires setting the .Va backend variable to one of the following values: .Bl -tag .It tap Ns Va N Use the named .Xr tap 4 interface as the backend. .It vmnet Ns Va N Use the named .Xr vmnet 4 interface as the backend. .It netgraph Use a .Xr netgraph 4 socket hook as the backend. This backend uses the following additional variables: .Bl -column "peerhook" "Format" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va path Ta string Ta Ta The name of the .Xr netgraph 4 destination node. .It Va peerhook Ta string Ta Ta The name of the destination hook. .It Va socket Ta string Ta Ta The name of the created .Xr ng_socket 4 node. .It Va hook Ta string Ta vmlink Ta The name of the source hook on the created .Xr ng_socket 4 node. .El .It netmap: Ns Va interface Use .Xr netmap 4 on a network interface as the backend. .It vale Ns Va bridge : Ns Va port Use a port on a .Xr vale 4 bridge as the backend. .El .Ss UART Device Settings .Bl -column "Name" "Format" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va path Ta path Ta Ta Backend device for the serial port. Either the pathname of a character device or .Dq stdio to use standard input and output of the .Xr bhyve 8 process. .El .Ss Host Bridge Settings .Bl -column "vendor" "integer" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va vendor Ta integer Ta 0x1275 Ta PCI vendor ID. .It Va devid Ta integer Ta 0x1275 Ta PCI device ID. .El .Ss AHCI Controller Settings AHCI controller devices contain zero or more ports each of which provides a storage device. Each port stores its settings in a node named .Dq port. Ns Va N under the controller's device node. The .Va N values are formatted as successive decimal values starting with 0. In addition to the block device settings described above, each port supports the following settings: .Bl -column "model" "integer" "generated" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va type Ta string Ta Ta The type of storage device to emulate. Must be set to either .Dq cd or .Dq hd . .It Va nmrr Ta integer Ta 0 Ta Nominal Media Rotation Rate, also known as RPM. A value 1 of indicates a device with no rate such as a Solid State Disk. .It Va ser Ta string Ta generated Ta Serial number of up to twenty characters. A default serial number is generated using a hash of the backing store's pathname. .It Va rev Ta string Ta 001 Ta Revision number of up to eight characters. .It Va model Ta string Ta Ta Model number of up to forty characters. Separate default model strings are used for .Dq cd and .Dq hd device types. .El .Ss e1000 Settings In addition to the network backend settings, Intel e82545 network interfaces support the following variables: .Bl -column "Name" "MAC address" "generated" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va mac Ta MAC address Ta generated Ta MAC address. If an explicit address is not provided, a MAC address is generated from a hash of the device's PCI address. .El .Ss Frame Buffer Settings .Bl -column "password" "[IP:]port" "127.0.0.1:5900" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va wait Ta bool Ta false Ta Wait for a remote connection before starting the VM. .It Va rfb Ta Oo Ar IP Ns : Oc Ns Ar port Ta 127.0.0.1:5900 Ta TCP address to listen on for remote connections. The IP address must be given as a numeric address. IPv6 addresses must be enclosed in square brackets and support scoped identifiers as described in .Xr getaddrinfo 3 . A bare port number may be given in which case the IPv4 localhost address is used. .It Va vga Ta string Ta io Ta VGA configuration. More details are provided in .Xr bhyve 8 . .It Va w Ta integer Ta 1024 Ta Frame buffer width in pixels. .It Va h Ta integer Ta 768 Ta Frame buffer height in pixels. .It Va password Ta string Ta Ta Password to use for VNC authentication. This type of authentication is known to be cryptographically weak and is not intended for use on untrusted networks. .El .Ss High Definition Audio Settings .Bl -column "Name" "Format" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va play Ta path Ta Ta Host playback device, typically .Pa /dev/dsp0 . .It Va rec Ta path Ta Ta Host recording device, typically .Pa /dev/dsp0 . .El .Ss LPC Device Settings The LPC bridge stores its configuration under a top-level .Va lpc node rather than under the PCI LPC device's node. The following nodes are available under .Va lpc : .Bl -column "pc-testdev" "Format" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va bootrom Ta path Ta Ta Path to a boot ROM. The contents of this file are copied into the guest's memory ending just before the 4GB physical address. If a boot ROM is present, a firmware interface device is also enabled for use by the boot ROM. .It Va com1 Ta node Ta Ta Settings for the COM1 serial port device. .It Va com2 Ta node Ta Ta Settings for the COM2 serial port device. .It Va com3 Ta node Ta Ta Settings for the COM3 serial port device. .It Va com4 Ta node Ta Ta Settings for the COM4 serial port device. .It Va pc-testdev Ta bool Ta false Ta Enable the PC debug/test device. .El .Ss NVMe Controller Settings Each NVMe controller supports a single storage device. The device can be backed either by a memory disk described by the .Va ram variable, or a block device using the the block device settings described above. In addition, each controller supports the following settings: .Bl -column "ioslots" "Format" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va maxq Ta integer Ta 16 Ta Maximum number of I/O submission and completion queue pairs. .It Va qsz Ta integer Ta 2058 Ta Number of elements in each I/O queue. .It Va ioslots Ta integer Ta 8 Ta Maximum number of concurrent I/O requests. .It Va sectsz Ta integer Ta Ta Sector size. Can be one of 512, 4096, or 8192. Devices backed by a memory disk use 4096 as the default. Devices backed by a block device use the block device's sector size as the default. .It Va ser Ta string Ta Ta Serial number of up to twenty characters. A default serial number is generated using a hash of the device's PCI address. .It Va eui64 Ta integer Ta Ta IEEE Extended Unique Identifier. If an EUI is not provided, a default is generated using a checksum of the device's PCI address. .It Va dsm Ta string Ta auto Ta Whether or not to advertise DataSet Management support. One of .Dq auto , .Dq enable , or .Dq disable . The .Dq auto setting only advertises support if the backing store supports resource freeing, for example via TRIM. .It Va ram Ta integer Ta Ta If set, allocate a memory disk as the backing store. The value of this variable is the size of the memory disk in megabytes. .El .Ss PCI Passthrough Settings +The +.Xr ppt 4 +device driver must be attached to the +PCI device being passed through. +The device to pass through can be identified either by name or its +host PCI bus location. .Bl -column "Name" "integer" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va bus Ta integer Ta Ta Host PCI bus address of device to pass through. .It Va slot Ta integer Ta Ta Host PCI slot address of device to pass through. .It Va func Ta integer Ta Ta Host PCI function address of device to pass through. +.It Va pptdev Ta string Ta Ta +Name of a +.Xr ppt 4 +device to pass through. .It Va rom Ta path Ta Ta ROM file of the device which will be executed by OVMF to init the device. .El .Ss VirtIO 9p Settings Each VirtIO 9p device exposes a single filesystem from a host path. .Bl -column "sharename" "Format" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va sharename Ta string Ta Ta The share name exposed to the guest. .It Va path Ta path Ta Ta The path of a directory on the host to export to the guest. .It Va ro Ta bool Ta false Ta If true, the guest filesystem is read-only. .El .Ss VirtIO Block Device Settings In addition to the block device settings described above, each VirtIO block device supports the following settings: .Bl -column "model" "integer" "generated" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va ser Ta string Ta generated Ta Serial number of up to twenty characters. A default serial number is generated using a hash of the backing store's pathname. .El .Ss VirtIO Console Device Settings Each VirtIO Console device contains one or more console ports. Each port stores its settings in a node named .Dq port. Ns Va N under the controller's device node. The .Va N values are formatted as successive decimal values starting with 0. Each port supports the following settings: .Bl -column "Name" "Format" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va name Ta string Ta Ta The name of the port exposed to the guest. .It Va path Ta path Ta Ta The path of a UNIX domain socket providing the host connection for the port. .El .Ss VirtIO Network Interface Settings In addition to the network backend settings, VirtIO network interfaces support the following variables: .Bl -column "Name" "MAC address" "generated" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va mac Ta MAC address Ta generated Ta MAC address. If an explicit address is not provided, a MAC address is generated from a hash of the device's PCI address. .It Va mtu Ta integer Ta 1500 Ta The largest supported MTU advertised to the guest. .El .Ss VirtIO SCSI Settings .Bl -column "Name" "integer" "Default" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description .It Va dev Ta path Ta Ta The path of a CAM target layer (CTL) device to export: .Pa /dev/cam/ctl Ns Oo Ar pp . Ns Ar vp Oc . .It Va iid Ta integer Ta 0 Ta Initiator ID to use when sending requests to the CTL port. .El .Sh SEE ALSO .Xr expand_number 3 , .Xr getaddrinfo 3 , .Xr strtol 3 , .Xr netgraph 4 , .Xr netmap 4 , .Xr ng_socket 4 , .Xr tap 4 , .Xr vale 4 , .Xr vmnet 4 , .Xr bhyve 8 diff --git a/usr.sbin/bhyve/pci_passthru.c b/usr.sbin/bhyve/pci_passthru.c index 0dfaf5908670..9940a13250ac 100644 --- a/usr.sbin/bhyve/pci_passthru.c +++ b/usr.sbin/bhyve/pci_passthru.c @@ -1,1145 +1,1227 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include __FBSDID("$FreeBSD$"); #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif +#include #include #include #include #include #include #include #include #include #include #include "config.h" #include "debug.h" #include "mem.h" #include "pci_passthru.h" #ifndef _PATH_DEVPCI #define _PATH_DEVPCI "/dev/pci" #endif #define LEGACY_SUPPORT 1 #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1) #define MSIX_CAPLEN 12 static int pcifd = -1; struct passthru_softc { struct pci_devinst *psc_pi; /* ROM is handled like a BAR */ struct pcibar psc_bar[PCI_BARMAX_WITH_ROM + 1]; struct { int capoff; int msgctrl; int emulated; } psc_msi; struct { int capoff; } psc_msix; struct pcisel psc_sel; }; static int msi_caplen(int msgctrl) { int len; len = 10; /* minimum length of msi capability */ if (msgctrl & PCIM_MSICTRL_64BIT) len += 4; #if 0 /* * Ignore the 'mask' and 'pending' bits in the MSI capability. * We'll let the guest manipulate them directly. */ if (msgctrl & PCIM_MSICTRL_VECTOR) len += 10; #endif return (len); } static int pcifd_init(void) { pcifd = open(_PATH_DEVPCI, O_RDWR, 0); if (pcifd < 0) { warn("failed to open %s", _PATH_DEVPCI); return (1); } #ifndef WITHOUT_CAPSICUM cap_rights_t pcifd_rights; cap_rights_init(&pcifd_rights, CAP_IOCTL, CAP_READ, CAP_WRITE); if (caph_rights_limit(pcifd, &pcifd_rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); const cap_ioctl_t pcifd_ioctls[] = { PCIOCREAD, PCIOCWRITE, PCIOCGETBAR, - PCIOCBARIO, PCIOCBARMMAP }; + PCIOCBARIO, PCIOCBARMMAP, PCIOCGETCONF }; if (caph_ioctls_limit(pcifd, pcifd_ioctls, nitems(pcifd_ioctls)) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif return (0); } uint32_t read_config(const struct pcisel *sel, long reg, int width) { struct pci_io pi; if (pcifd < 0 && pcifd_init()) { return (0); } bzero(&pi, sizeof(pi)); pi.pi_sel = *sel; pi.pi_reg = reg; pi.pi_width = width; if (ioctl(pcifd, PCIOCREAD, &pi) < 0) return (0); /* XXX */ else return (pi.pi_data); } void write_config(const struct pcisel *sel, long reg, int width, uint32_t data) { struct pci_io pi; if (pcifd < 0 && pcifd_init()) { return; } bzero(&pi, sizeof(pi)); pi.pi_sel = *sel; pi.pi_reg = reg; pi.pi_width = width; pi.pi_data = data; (void)ioctl(pcifd, PCIOCWRITE, &pi); /* XXX */ } #ifdef LEGACY_SUPPORT static int passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr) { int capoff, i; struct msicap msicap; u_char *capdata; pci_populate_msicap(&msicap, msgnum, nextptr); /* * XXX * Copy the msi capability structure in the last 16 bytes of the * config space. This is wrong because it could shadow something * useful to the device. */ capoff = 256 - roundup(sizeof(msicap), 4); capdata = (u_char *)&msicap; for (i = 0; i < sizeof(msicap); i++) pci_set_cfgdata8(pi, capoff + i, capdata[i]); return (capoff); } #endif /* LEGACY_SUPPORT */ static int cfginitmsi(struct passthru_softc *sc) { int i, ptr, capptr, cap, sts, caplen, table_size; uint32_t u32; struct pcisel sel; struct pci_devinst *pi; struct msixcap msixcap; uint32_t *msixcap_ptr; pi = sc->psc_pi; sel = sc->psc_sel; /* * Parse the capabilities and cache the location of the MSI * and MSI-X capabilities. */ sts = read_config(&sel, PCIR_STATUS, 2); if (sts & PCIM_STATUS_CAPPRESENT) { ptr = read_config(&sel, PCIR_CAP_PTR, 1); while (ptr != 0 && ptr != 0xff) { cap = read_config(&sel, ptr + PCICAP_ID, 1); if (cap == PCIY_MSI) { /* * Copy the MSI capability into the config * space of the emulated pci device */ sc->psc_msi.capoff = ptr; sc->psc_msi.msgctrl = read_config(&sel, ptr + 2, 2); sc->psc_msi.emulated = 0; caplen = msi_caplen(sc->psc_msi.msgctrl); capptr = ptr; while (caplen > 0) { u32 = read_config(&sel, capptr, 4); pci_set_cfgdata32(pi, capptr, u32); caplen -= 4; capptr += 4; } } else if (cap == PCIY_MSIX) { /* * Copy the MSI-X capability */ sc->psc_msix.capoff = ptr; caplen = 12; msixcap_ptr = (uint32_t*) &msixcap; capptr = ptr; while (caplen > 0) { u32 = read_config(&sel, capptr, 4); *msixcap_ptr = u32; pci_set_cfgdata32(pi, capptr, u32); caplen -= 4; capptr += 4; msixcap_ptr++; } } ptr = read_config(&sel, ptr + PCICAP_NEXTPTR, 1); } } if (sc->psc_msix.capoff != 0) { pi->pi_msix.pba_bar = msixcap.pba_info & PCIM_MSIX_BIR_MASK; pi->pi_msix.pba_offset = msixcap.pba_info & ~PCIM_MSIX_BIR_MASK; pi->pi_msix.table_bar = msixcap.table_info & PCIM_MSIX_BIR_MASK; pi->pi_msix.table_offset = msixcap.table_info & ~PCIM_MSIX_BIR_MASK; pi->pi_msix.table_count = MSIX_TABLE_COUNT(msixcap.msgctrl); pi->pi_msix.pba_size = PBA_SIZE(pi->pi_msix.table_count); /* Allocate the emulated MSI-X table array */ table_size = pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; pi->pi_msix.table = calloc(1, table_size); /* Mask all table entries */ for (i = 0; i < pi->pi_msix.table_count; i++) { pi->pi_msix.table[i].vector_control |= PCIM_MSIX_VCTRL_MASK; } } #ifdef LEGACY_SUPPORT /* * If the passthrough device does not support MSI then craft a * MSI capability for it. We link the new MSI capability at the * head of the list of capabilities. */ if ((sts & PCIM_STATUS_CAPPRESENT) != 0 && sc->psc_msi.capoff == 0) { int origptr, msiptr; origptr = read_config(&sel, PCIR_CAP_PTR, 1); msiptr = passthru_add_msicap(pi, 1, origptr); sc->psc_msi.capoff = msiptr; sc->psc_msi.msgctrl = pci_get_cfgdata16(pi, msiptr + 2); sc->psc_msi.emulated = 1; pci_set_cfgdata8(pi, PCIR_CAP_PTR, msiptr); } #endif /* Make sure one of the capabilities is present */ if (sc->psc_msi.capoff == 0 && sc->psc_msix.capoff == 0) return (-1); else return (0); } static uint64_t msix_table_read(struct passthru_softc *sc, uint64_t offset, int size) { struct pci_devinst *pi; struct msix_table_entry *entry; uint8_t *src8; uint16_t *src16; uint32_t *src32; uint64_t *src64; uint64_t data; size_t entry_offset; uint32_t table_offset; int index, table_count; pi = sc->psc_pi; table_offset = pi->pi_msix.table_offset; table_count = pi->pi_msix.table_count; if (offset < table_offset || offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) { switch (size) { case 1: src8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset); data = *src8; break; case 2: src16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset); data = *src16; break; case 4: src32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset); data = *src32; break; case 8: src64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset); data = *src64; break; default: return (-1); } return (data); } offset -= table_offset; index = offset / MSIX_TABLE_ENTRY_SIZE; assert(index < table_count); entry = &pi->pi_msix.table[index]; entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; switch (size) { case 1: src8 = (uint8_t *)((uint8_t *)entry + entry_offset); data = *src8; break; case 2: src16 = (uint16_t *)((uint8_t *)entry + entry_offset); data = *src16; break; case 4: src32 = (uint32_t *)((uint8_t *)entry + entry_offset); data = *src32; break; case 8: src64 = (uint64_t *)((uint8_t *)entry + entry_offset); data = *src64; break; default: return (-1); } return (data); } static void msix_table_write(struct vmctx *ctx, int vcpu, struct passthru_softc *sc, uint64_t offset, int size, uint64_t data) { struct pci_devinst *pi; struct msix_table_entry *entry; uint8_t *dest8; uint16_t *dest16; uint32_t *dest32; uint64_t *dest64; size_t entry_offset; uint32_t table_offset, vector_control; int index, table_count; pi = sc->psc_pi; table_offset = pi->pi_msix.table_offset; table_count = pi->pi_msix.table_count; if (offset < table_offset || offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) { switch (size) { case 1: dest8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset); *dest8 = data; break; case 2: dest16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset); *dest16 = data; break; case 4: dest32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset); *dest32 = data; break; case 8: dest64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset); *dest64 = data; break; } return; } offset -= table_offset; index = offset / MSIX_TABLE_ENTRY_SIZE; assert(index < table_count); entry = &pi->pi_msix.table[index]; entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; /* Only 4 byte naturally-aligned writes are supported */ assert(size == 4); assert(entry_offset % 4 == 0); vector_control = entry->vector_control; dest32 = (uint32_t *)((void *)entry + entry_offset); *dest32 = data; /* If MSI-X hasn't been enabled, do nothing */ if (pi->pi_msix.enabled) { /* If the entry is masked, don't set it up */ if ((entry->vector_control & PCIM_MSIX_VCTRL_MASK) == 0 || (vector_control & PCIM_MSIX_VCTRL_MASK) == 0) { (void)vm_setup_pptdev_msix(ctx, vcpu, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, index, entry->addr, entry->msg_data, entry->vector_control); } } } static int init_msix_table(struct vmctx *ctx, struct passthru_softc *sc) { struct pci_devinst *pi = sc->psc_pi; struct pci_bar_mmap pbm; int b, s, f; uint32_t table_size, table_offset; assert(pci_msix_table_bar(pi) >= 0 && pci_msix_pba_bar(pi) >= 0); b = sc->psc_sel.pc_bus; s = sc->psc_sel.pc_dev; f = sc->psc_sel.pc_func; /* * Map the region of the BAR containing the MSI-X table. This is * necessary for two reasons: * 1. The PBA may reside in the first or last page containing the MSI-X * table. * 2. While PCI devices are not supposed to use the page(s) containing * the MSI-X table for other purposes, some do in practice. */ memset(&pbm, 0, sizeof(pbm)); pbm.pbm_sel = sc->psc_sel; pbm.pbm_flags = PCIIO_BAR_MMAP_RW; pbm.pbm_reg = PCIR_BAR(pi->pi_msix.table_bar); pbm.pbm_memattr = VM_MEMATTR_DEVICE; if (ioctl(pcifd, PCIOCBARMMAP, &pbm) != 0) { warn("Failed to map MSI-X table BAR on %d/%d/%d", b, s, f); return (-1); } assert(pbm.pbm_bar_off == 0); pi->pi_msix.mapped_addr = (uint8_t *)(uintptr_t)pbm.pbm_map_base; pi->pi_msix.mapped_size = pbm.pbm_map_length; table_offset = rounddown2(pi->pi_msix.table_offset, 4096); table_size = pi->pi_msix.table_offset - table_offset; table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; table_size = roundup2(table_size, 4096); /* * Unmap any pages not containing the table, we do not need to emulate * accesses to them. Avoid releasing address space to help ensure that * a buggy out-of-bounds access causes a crash. */ if (table_offset != 0) if (mprotect(pi->pi_msix.mapped_addr, table_offset, PROT_NONE) != 0) warn("Failed to unmap MSI-X table BAR region"); if (table_offset + table_size != pi->pi_msix.mapped_size) if (mprotect( pi->pi_msix.mapped_addr + table_offset + table_size, pi->pi_msix.mapped_size - (table_offset + table_size), PROT_NONE) != 0) warn("Failed to unmap MSI-X table BAR region"); return (0); } static int cfginitbar(struct vmctx *ctx, struct passthru_softc *sc) { int i, error; struct pci_devinst *pi; struct pci_bar_io bar; enum pcibar_type bartype; uint64_t base, size; pi = sc->psc_pi; /* * Initialize BAR registers */ for (i = 0; i <= PCI_BARMAX; i++) { bzero(&bar, sizeof(bar)); bar.pbi_sel = sc->psc_sel; bar.pbi_reg = PCIR_BAR(i); if (ioctl(pcifd, PCIOCGETBAR, &bar) < 0) continue; if (PCI_BAR_IO(bar.pbi_base)) { bartype = PCIBAR_IO; base = bar.pbi_base & PCIM_BAR_IO_BASE; } else { switch (bar.pbi_base & PCIM_BAR_MEM_TYPE) { case PCIM_BAR_MEM_64: bartype = PCIBAR_MEM64; break; default: bartype = PCIBAR_MEM32; break; } base = bar.pbi_base & PCIM_BAR_MEM_BASE; } size = bar.pbi_length; if (bartype != PCIBAR_IO) { if (((base | size) & PAGE_MASK) != 0) { warnx("passthru device %d/%d/%d BAR %d: " "base %#lx or size %#lx not page aligned\n", sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, i, base, size); return (-1); } } /* Cache information about the "real" BAR */ sc->psc_bar[i].type = bartype; sc->psc_bar[i].size = size; sc->psc_bar[i].addr = base; sc->psc_bar[i].lobits = 0; /* Allocate the BAR in the guest I/O or MMIO space */ error = pci_emul_alloc_bar(pi, i, bartype, size); if (error) return (-1); /* Use same lobits as physical bar */ uint8_t lobits = read_config(&sc->psc_sel, PCIR_BAR(i), 0x01); if (bartype == PCIBAR_MEM32 || bartype == PCIBAR_MEM64) { lobits &= ~PCIM_BAR_MEM_BASE; } else { lobits &= ~PCIM_BAR_IO_BASE; } sc->psc_bar[i].lobits = lobits; pi->pi_bar[i].lobits = lobits; /* The MSI-X table needs special handling */ if (i == pci_msix_table_bar(pi)) { error = init_msix_table(ctx, sc); if (error) return (-1); } /* * 64-bit BAR takes up two slots so skip the next one. */ if (bartype == PCIBAR_MEM64) { i++; assert(i <= PCI_BARMAX); sc->psc_bar[i].type = PCIBAR_MEMHI64; } } return (0); } static int cfginit(struct vmctx *ctx, struct pci_devinst *pi, int bus, int slot, int func) { int error; struct passthru_softc *sc; error = 1; sc = pi->pi_arg; bzero(&sc->psc_sel, sizeof(struct pcisel)); sc->psc_sel.pc_bus = bus; sc->psc_sel.pc_dev = slot; sc->psc_sel.pc_func = func; if (cfginitmsi(sc) != 0) { warnx("failed to initialize MSI for PCI %d/%d/%d", bus, slot, func); goto done; } if (cfginitbar(ctx, sc) != 0) { warnx("failed to initialize BARs for PCI %d/%d/%d", bus, slot, func); goto done; } write_config(&sc->psc_sel, PCIR_COMMAND, 2, pci_get_cfgdata16(pi, PCIR_COMMAND)); /* * We need to do this after PCIR_COMMAND got possibly updated, e.g., * a BAR was enabled, as otherwise the PCIOCBARMMAP might fail on us. */ if (pci_msix_table_bar(pi) >= 0) { error = init_msix_table(ctx, sc); if (error != 0) { warnx( "failed to initialize MSI-X table for PCI %d/%d/%d: %d", bus, slot, func, error); goto done; } } error = 0; /* success */ done: return (error); } static int passthru_legacy_config(nvlist_t *nvl, const char *opts) { + const char *cp; + char *tofree; char value[16]; int bus, slot, func; if (opts == NULL) return (0); - if (sscanf(opts, "%d/%d/%d", &bus, &slot, &func) != 3) { + cp = strchr(opts, ','); + + if (strncmp(opts, "ppt", strlen("ppt")) == 0) { + tofree = strndup(opts, cp - opts); + set_config_value_node(nvl, "pptdev", tofree); + free(tofree); + } else if (sscanf(opts, "pci0:%d:%d:%d", &bus, &slot, &func) == 3 || + sscanf(opts, "pci%d:%d:%d", &bus, &slot, &func) == 3 || + sscanf(opts, "%d/%d/%d", &bus, &slot, &func) == 3) { + snprintf(value, sizeof(value), "%d", bus); + set_config_value_node(nvl, "bus", value); + snprintf(value, sizeof(value), "%d", slot); + set_config_value_node(nvl, "slot", value); + snprintf(value, sizeof(value), "%d", func); + set_config_value_node(nvl, "func", value); + } else { EPRINTLN("passthru: invalid options \"%s\"", opts); return (-1); } - snprintf(value, sizeof(value), "%d", bus); - set_config_value_node(nvl, "bus", value); - snprintf(value, sizeof(value), "%d", slot); - set_config_value_node(nvl, "slot", value); - snprintf(value, sizeof(value), "%d", func); - set_config_value_node(nvl, "func", value); - - opts = strchr(opts, ','); - if (opts == NULL) { + if (cp == NULL) { return (0); } - return pci_parse_legacy_config(nvl, opts + 1); + return (pci_parse_legacy_config(nvl, cp + 1)); } static int passthru_init_rom(struct vmctx *const ctx, struct passthru_softc *const sc, const char *const romfile) { if (romfile == NULL) { return (0); } const int fd = open(romfile, O_RDONLY); if (fd < 0) { warnx("%s: can't open romfile \"%s\"", __func__, romfile); return (-1); } struct stat sbuf; if (fstat(fd, &sbuf) < 0) { warnx("%s: can't fstat romfile \"%s\"", __func__, romfile); close(fd); return (-1); } const uint64_t rom_size = sbuf.st_size; void *const rom_data = mmap(NULL, rom_size, PROT_READ, MAP_SHARED, fd, 0); if (rom_data == MAP_FAILED) { warnx("%s: unable to mmap romfile \"%s\" (%d)", __func__, romfile, errno); close(fd); return (-1); } void *rom_addr; int error = pci_emul_alloc_rom(sc->psc_pi, rom_size, &rom_addr); if (error) { warnx("%s: failed to alloc rom segment", __func__); munmap(rom_data, rom_size); close(fd); return (error); } memcpy(rom_addr, rom_data, rom_size); sc->psc_bar[PCI_ROM_IDX].type = PCIBAR_ROM; sc->psc_bar[PCI_ROM_IDX].addr = (uint64_t)rom_addr; sc->psc_bar[PCI_ROM_IDX].size = rom_size; munmap(rom_data, rom_size); close(fd); return (0); } +static bool +passthru_lookup_pptdev(const char *name, int *bus, int *slot, int *func) +{ + struct pci_conf_io pc; + struct pci_conf conf[1]; + struct pci_match_conf patterns[1]; + char *cp; + + bzero(&pc, sizeof(struct pci_conf_io)); + pc.match_buf_len = sizeof(conf); + pc.matches = conf; + + bzero(&patterns, sizeof(patterns)); + + /* + * The pattern structure requires the unit to be split out from + * the driver name. Walk backwards from the end of the name to + * find the start of the unit. + */ + cp = strchr(name, '\0'); + assert(cp != NULL); + while (cp != name && isdigit(cp[-1])) + cp--; + if (cp == name || !isdigit(*cp)) { + EPRINTLN("Invalid passthru device name %s", name); + return (false); + } + if ((size_t)(cp - name) + 1 > sizeof(patterns[0].pd_name)) { + EPRINTLN("Passthru device name %s is too long", name); + return (false); + } + memcpy(patterns[0].pd_name, name, cp - name); + patterns[0].pd_unit = strtol(cp, &cp, 10); + if (*cp != '\0') { + EPRINTLN("Invalid passthru device name %s", name); + return (false); + } + patterns[0].flags = PCI_GETCONF_MATCH_NAME | PCI_GETCONF_MATCH_UNIT; + pc.num_patterns = 1; + pc.pat_buf_len = sizeof(patterns); + pc.patterns = patterns; + + if (ioctl(pcifd, PCIOCGETCONF, &pc) == -1) { + EPRINTLN("ioctl(PCIOCGETCONF): %s", strerror(errno)); + return (false); + } + if (pc.status != PCI_GETCONF_LAST_DEVICE && + pc.status != PCI_GETCONF_MORE_DEVS) { + EPRINTLN("error returned from PCIOCGETCONF ioctl"); + return (false); + } + if (pc.num_matches == 0) { + EPRINTLN("Passthru device %s not found", name); + return (false); + } + + if (conf[0].pc_sel.pc_domain != 0) { + EPRINTLN("Passthru device %s on unsupported domain", name); + return (false); + } + *bus = conf[0].pc_sel.pc_bus; + *slot = conf[0].pc_sel.pc_dev; + *func = conf[0].pc_sel.pc_func; + return (true); +} + static int passthru_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl) { int bus, slot, func, error, memflags; struct passthru_softc *sc; const char *value; sc = NULL; error = 1; memflags = vm_get_memflags(ctx); if (!(memflags & VM_MEM_F_WIRED)) { warnx("passthru requires guest memory to be wired"); return (error); } if (pcifd < 0 && pcifd_init()) { return (error); } #define GET_INT_CONFIG(var, name) do { \ value = get_config_value_node(nvl, name); \ if (value == NULL) { \ EPRINTLN("passthru: missing required %s setting", name); \ return (error); \ } \ var = atoi(value); \ } while (0) - GET_INT_CONFIG(bus, "bus"); - GET_INT_CONFIG(slot, "slot"); - GET_INT_CONFIG(func, "func"); + value = get_config_value_node(nvl, "pptdev"); + if (value != NULL) { + if (!passthru_lookup_pptdev(value, &bus, &slot, &func)) + return (error); + } else { + GET_INT_CONFIG(bus, "bus"); + GET_INT_CONFIG(slot, "slot"); + GET_INT_CONFIG(func, "func"); + } if (vm_assign_pptdev(ctx, bus, slot, func) != 0) { warnx("PCI device at %d/%d/%d is not using the ppt(4) driver", bus, slot, func); goto done; } sc = calloc(1, sizeof(struct passthru_softc)); pi->pi_arg = sc; sc->psc_pi = pi; /* initialize config space */ if ((error = cfginit(ctx, pi, bus, slot, func)) != 0) goto done; /* initialize ROM */ if ((error = passthru_init_rom(ctx, sc, get_config_value_node(nvl, "rom"))) != 0) goto done; error = 0; /* success */ done: if (error) { free(sc); vm_unassign_pptdev(ctx, bus, slot, func); } return (error); } static int bar_access(int coff) { if ((coff >= PCIR_BAR(0) && coff < PCIR_BAR(PCI_BARMAX + 1)) || coff == PCIR_BIOS) return (1); else return (0); } static int msicap_access(struct passthru_softc *sc, int coff) { int caplen; if (sc->psc_msi.capoff == 0) return (0); caplen = msi_caplen(sc->psc_msi.msgctrl); if (coff >= sc->psc_msi.capoff && coff < sc->psc_msi.capoff + caplen) return (1); else return (0); } static int msixcap_access(struct passthru_softc *sc, int coff) { if (sc->psc_msix.capoff == 0) return (0); return (coff >= sc->psc_msix.capoff && coff < sc->psc_msix.capoff + MSIX_CAPLEN); } static int passthru_cfgread(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int coff, int bytes, uint32_t *rv) { struct passthru_softc *sc; sc = pi->pi_arg; /* * PCI BARs and MSI capability is emulated. */ if (bar_access(coff) || msicap_access(sc, coff) || msixcap_access(sc, coff)) return (-1); #ifdef LEGACY_SUPPORT /* * Emulate PCIR_CAP_PTR if this device does not support MSI capability * natively. */ if (sc->psc_msi.emulated) { if (coff >= PCIR_CAP_PTR && coff < PCIR_CAP_PTR + 4) return (-1); } #endif /* * Emulate the command register. If a single read reads both the * command and status registers, read the status register from the * device's config space. */ if (coff == PCIR_COMMAND) { if (bytes <= 2) return (-1); *rv = read_config(&sc->psc_sel, PCIR_STATUS, 2) << 16 | pci_get_cfgdata16(pi, PCIR_COMMAND); return (0); } /* Everything else just read from the device's config space */ *rv = read_config(&sc->psc_sel, coff, bytes); return (0); } static int passthru_cfgwrite(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int coff, int bytes, uint32_t val) { int error, msix_table_entries, i; struct passthru_softc *sc; uint16_t cmd_old; sc = pi->pi_arg; /* * PCI BARs are emulated */ if (bar_access(coff)) return (-1); /* * MSI capability is emulated */ if (msicap_access(sc, coff)) { pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff, PCIY_MSI); error = vm_setup_pptdev_msi(ctx, vcpu, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, pi->pi_msi.addr, pi->pi_msi.msg_data, pi->pi_msi.maxmsgnum); if (error != 0) err(1, "vm_setup_pptdev_msi"); return (0); } if (msixcap_access(sc, coff)) { pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msix.capoff, PCIY_MSIX); if (pi->pi_msix.enabled) { msix_table_entries = pi->pi_msix.table_count; for (i = 0; i < msix_table_entries; i++) { error = vm_setup_pptdev_msix(ctx, vcpu, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, i, pi->pi_msix.table[i].addr, pi->pi_msix.table[i].msg_data, pi->pi_msix.table[i].vector_control); if (error) err(1, "vm_setup_pptdev_msix"); } } else { error = vm_disable_pptdev_msix(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func); if (error) err(1, "vm_disable_pptdev_msix"); } return (0); } #ifdef LEGACY_SUPPORT /* * If this device does not support MSI natively then we cannot let * the guest disable legacy interrupts from the device. It is the * legacy interrupt that is triggering the virtual MSI to the guest. */ if (sc->psc_msi.emulated && pci_msi_enabled(pi)) { if (coff == PCIR_COMMAND && bytes == 2) val &= ~PCIM_CMD_INTxDIS; } #endif write_config(&sc->psc_sel, coff, bytes, val); if (coff == PCIR_COMMAND) { cmd_old = pci_get_cfgdata16(pi, PCIR_COMMAND); if (bytes == 1) pci_set_cfgdata8(pi, PCIR_COMMAND, val); else if (bytes == 2) pci_set_cfgdata16(pi, PCIR_COMMAND, val); pci_emul_cmd_changed(pi, cmd_old); } return (0); } static void passthru_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct passthru_softc *sc; struct pci_bar_ioreq pio; sc = pi->pi_arg; if (baridx == pci_msix_table_bar(pi)) { msix_table_write(ctx, vcpu, sc, offset, size, value); } else { assert(pi->pi_bar[baridx].type == PCIBAR_IO); assert(size == 1 || size == 2 || size == 4); assert(offset <= UINT32_MAX && offset + size <= UINT32_MAX); bzero(&pio, sizeof(pio)); pio.pbi_sel = sc->psc_sel; pio.pbi_op = PCIBARIO_WRITE; pio.pbi_bar = baridx; pio.pbi_offset = (uint32_t)offset; pio.pbi_width = size; pio.pbi_value = (uint32_t)value; (void)ioctl(pcifd, PCIOCBARIO, &pio); } } static uint64_t passthru_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct passthru_softc *sc; struct pci_bar_ioreq pio; uint64_t val; sc = pi->pi_arg; if (baridx == pci_msix_table_bar(pi)) { val = msix_table_read(sc, offset, size); } else { assert(pi->pi_bar[baridx].type == PCIBAR_IO); assert(size == 1 || size == 2 || size == 4); assert(offset <= UINT32_MAX && offset + size <= UINT32_MAX); bzero(&pio, sizeof(pio)); pio.pbi_sel = sc->psc_sel; pio.pbi_op = PCIBARIO_READ; pio.pbi_bar = baridx; pio.pbi_offset = (uint32_t)offset; pio.pbi_width = size; (void)ioctl(pcifd, PCIOCBARIO, &pio); val = pio.pbi_value; } return (val); } static void passthru_msix_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct passthru_softc *sc; size_t remaining; uint32_t table_size, table_offset; sc = pi->pi_arg; table_offset = rounddown2(pi->pi_msix.table_offset, 4096); if (table_offset > 0) { if (!enabled) { if (vm_unmap_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, address, table_offset) != 0) warnx("pci_passthru: unmap_pptdev_mmio failed"); } else { if (vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, address, table_offset, sc->psc_bar[baridx].addr) != 0) warnx("pci_passthru: map_pptdev_mmio failed"); } } table_size = pi->pi_msix.table_offset - table_offset; table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; table_size = roundup2(table_size, 4096); remaining = pi->pi_bar[baridx].size - table_offset - table_size; if (remaining > 0) { address += table_offset + table_size; if (!enabled) { if (vm_unmap_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, address, remaining) != 0) warnx("pci_passthru: unmap_pptdev_mmio failed"); } else { if (vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, address, remaining, sc->psc_bar[baridx].addr + table_offset + table_size) != 0) warnx("pci_passthru: map_pptdev_mmio failed"); } } } static void passthru_mmio_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct passthru_softc *sc; sc = pi->pi_arg; if (!enabled) { if (vm_unmap_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, address, sc->psc_bar[baridx].size) != 0) warnx("pci_passthru: unmap_pptdev_mmio failed"); } else { if (vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus, sc->psc_sel.pc_dev, sc->psc_sel.pc_func, address, sc->psc_bar[baridx].size, sc->psc_bar[baridx].addr) != 0) warnx("pci_passthru: map_pptdev_mmio failed"); } } static void passthru_addr_rom(struct pci_devinst *const pi, const int idx, const int enabled) { const uint64_t addr = pi->pi_bar[idx].addr; const uint64_t size = pi->pi_bar[idx].size; if (!enabled) { if (vm_munmap_memseg(pi->pi_vmctx, addr, size) != 0) { errx(4, "%s: munmap_memseg @ [%016lx - %016lx] failed", __func__, addr, addr + size); } } else { if (vm_mmap_memseg(pi->pi_vmctx, addr, VM_PCIROM, pi->pi_romoffset, size, PROT_READ | PROT_EXEC) != 0) { errx(4, "%s: mnmap_memseg @ [%016lx - %016lx] failed", __func__, addr, addr + size); } } } static void passthru_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { switch (pi->pi_bar[baridx].type) { case PCIBAR_IO: /* IO BARs are emulated */ break; case PCIBAR_ROM: passthru_addr_rom(pi, baridx, enabled); break; case PCIBAR_MEM32: case PCIBAR_MEM64: if (baridx == pci_msix_table_bar(pi)) passthru_msix_addr(ctx, pi, baridx, enabled, address); else passthru_mmio_addr(ctx, pi, baridx, enabled, address); break; default: errx(4, "%s: invalid BAR type %d", __func__, pi->pi_bar[baridx].type); } } static const struct pci_devemu passthru = { .pe_emu = "passthru", .pe_init = passthru_init, .pe_legacy_config = passthru_legacy_config, .pe_cfgwrite = passthru_cfgwrite, .pe_cfgread = passthru_cfgread, .pe_barwrite = passthru_write, .pe_barread = passthru_read, .pe_baraddr = passthru_addr, }; PCI_EMUL_SET(passthru);