diff --git a/share/man/man4/virtio.4 b/share/man/man4/virtio.4 index 1e5ea0e4a7da..0598d5d8621d 100644 --- a/share/man/man4/virtio.4 +++ b/share/man/man4/virtio.4 @@ -1,110 +1,134 @@ .\" Copyright (c) 2011 Bryan Venteicher .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd May 26, 2019 +.Dd March 2, 2026 .Dt VIRTIO 4 .Os .Sh NAME .Nm virtio .Nd VirtIO Device Support .Sh SYNOPSIS To compile VirtIO device support into the kernel, place the following lines in your kernel configuration file: .Bd -ragged -offset indent .Cd "device virtio" .Cd "device virtio_pci" .Ed .Pp Alternatively, to load VirtIO support as modules at boot time, place the following lines in .Xr loader.conf 5 : .Bd -literal -offset indent virtio_load="YES" virtio_pci_load="YES" .Ed .Sh DESCRIPTION VirtIO is a specification for para-virtualized I/O in a virtual machine (VM). Traditionally, the hypervisor emulated real devices such as an Ethernet interface or disk controller to provide the VM with I/O. This emulation is often inefficient. .Pp VirtIO defines an interface for efficient I/O between the hypervisor and VM. The .Nm module provides a shared memory transport called a virtqueue. The -.Xr virtio_pci +.Sy virtio_pci device driver represents an emulated PCI device that the hypervisor makes available to the VM. This device provides the probing, configuration, and interrupt notifications needed to interact with the hypervisor. .Fx supports the following VirtIO devices: .Bl -hang -offset indent -width xxxxxxxx .It Sy Ethernet An emulated Ethernet device is provided by the .Xr vtnet 4 device driver. .It Sy Block An emulated disk controller is provided by the .Xr virtio_blk 4 device driver. .It Sy Console Provided by the .Xr virtio_console 4 driver. .It Sy Entropy Provided by the .Xr virtio_random 4 driver. .It Sy Balloon A pseudo-device to allow the VM to release memory back to the hypervisor is provided by the .Xr virtio_balloon 4 device driver. .It Sy GPU Graphics support is provided by the .Xr virtio_gpu 4 device driver. .It Sy SCSI An emulated SCSI HBA is provided by the .Xr virtio_scsi 4 device driver. .El +.Sh LOADER TUNABLES +Tunables can be set at the +.Xr loader 8 +prompt before booting the kernel or stored in +.Xr loader.conf 5 . +.Bl -tag -width "hw.virtio.pci.transitional" +.It Va hw.virtio.pci.disable_msix +If set to 1, disables MSI-X. +The default value is 0. +.It Va hw.virtio.pci.transitional +For a transitional +.Nm +device, this tunable specifies whether to negotiate +modern mode and use the modern +.Nm +driver +.Pq 1 +or to negotiate legacy mode and +use the legacy +.Nm +driver +.Pq 1 . +The default value is 0. +.El .Sh SEE ALSO .Xr virtio_balloon 4 , .Xr virtio_blk 4 , .Xr virtio_console 4 , .Xr virtio_gpu 4 , .Xr virtio_random 4 , .Xr virtio_scsi 4 , .Xr vtnet 4 .Sh HISTORY Support for VirtIO first appeared in .Fx 9.0 . .Sh AUTHORS .An -nosplit .Fx support for VirtIO was first added by .An Bryan Venteicher Aq Mt bryanv@FreeBSD.org . diff --git a/sys/dev/virtio/pci/virtio_pci.c b/sys/dev/virtio/pci/virtio_pci.c index b7b34b448f6e..2d33d99014ae 100644 --- a/sys/dev/virtio/pci/virtio_pci.c +++ b/sys/dev/virtio/pci/virtio_pci.c @@ -1,994 +1,999 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_pci_if.h" #include "virtio_if.h" static void vtpci_describe_features(struct vtpci_common *, const char *, uint64_t); static int vtpci_alloc_msix(struct vtpci_common *, int); static int vtpci_alloc_msi(struct vtpci_common *); static int vtpci_alloc_intr_msix_pervq(struct vtpci_common *); static int vtpci_alloc_intr_msix_shared(struct vtpci_common *); static int vtpci_alloc_intr_msi(struct vtpci_common *); static int vtpci_alloc_intr_intx(struct vtpci_common *); static int vtpci_alloc_interrupt(struct vtpci_common *, int, int, struct vtpci_interrupt *); static void vtpci_free_interrupt(struct vtpci_common *, struct vtpci_interrupt *); static void vtpci_free_interrupts(struct vtpci_common *); static void vtpci_free_virtqueues(struct vtpci_common *); static void vtpci_cleanup_setup_intr_attempt(struct vtpci_common *); static int vtpci_alloc_intr_resources(struct vtpci_common *); static int vtpci_setup_intx_interrupt(struct vtpci_common *, enum intr_type); static int vtpci_setup_pervq_msix_interrupts(struct vtpci_common *, enum intr_type); static int vtpci_set_host_msix_vectors(struct vtpci_common *); static int vtpci_setup_msix_interrupts(struct vtpci_common *, enum intr_type); static int vtpci_setup_intrs(struct vtpci_common *, enum intr_type); static int vtpci_reinit_virtqueue(struct vtpci_common *, int); static void vtpci_intx_intr(void *); static int vtpci_vq_shared_intr_filter(void *); static void vtpci_vq_shared_intr(void *); static int vtpci_vq_intr_filter(void *); static void vtpci_vq_intr(void *); static void vtpci_config_intr(void *); static void vtpci_setup_sysctl(struct vtpci_common *); #define vtpci_setup_msi_interrupt vtpci_setup_intx_interrupt /* * This module contains two drivers: * - virtio_pci_legacy for pre-V1 support * - virtio_pci_modern for V1 support */ MODULE_VERSION(virtio_pci, 1); MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); +SYSCTL_DECL(_hw_virtio); +SYSCTL_NODE(_hw_virtio, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "VirtIO PCI driver parameters"); + int vtpci_disable_msix = 0; -TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); +SYSCTL_INT(_hw_virtio_pci, OID_AUTO, disable_msix, CTLFLAG_RDTUN, + &vtpci_disable_msix, 0, "If set to 1, disables MSI-X."); static uint8_t vtpci_read_isr(struct vtpci_common *cn) { return (VIRTIO_PCI_READ_ISR(cn->vtpci_dev)); } static uint16_t vtpci_get_vq_size(struct vtpci_common *cn, int idx) { return (VIRTIO_PCI_GET_VQ_SIZE(cn->vtpci_dev, idx)); } static bus_size_t vtpci_get_vq_notify_off(struct vtpci_common *cn, int idx) { return (VIRTIO_PCI_GET_VQ_NOTIFY_OFF(cn->vtpci_dev, idx)); } static void vtpci_set_vq(struct vtpci_common *cn, struct virtqueue *vq) { VIRTIO_PCI_SET_VQ(cn->vtpci_dev, vq); } static void vtpci_disable_vq(struct vtpci_common *cn, int idx) { VIRTIO_PCI_DISABLE_VQ(cn->vtpci_dev, idx); } static int vtpci_register_cfg_msix(struct vtpci_common *cn, struct vtpci_interrupt *intr) { return (VIRTIO_PCI_REGISTER_CFG_MSIX(cn->vtpci_dev, intr)); } static int vtpci_register_vq_msix(struct vtpci_common *cn, int idx, struct vtpci_interrupt *intr) { return (VIRTIO_PCI_REGISTER_VQ_MSIX(cn->vtpci_dev, idx, intr)); } void vtpci_init(struct vtpci_common *cn, device_t dev, bool modern) { cn->vtpci_dev = dev; pci_enable_busmaster(dev); if (modern) cn->vtpci_flags |= VTPCI_FLAG_MODERN; if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) cn->vtpci_flags |= VTPCI_FLAG_NO_MSI; if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) cn->vtpci_flags |= VTPCI_FLAG_NO_MSIX; vtpci_setup_sysctl(cn); } int vtpci_add_child(struct vtpci_common *cn) { device_t dev, child; dev = cn->vtpci_dev; child = device_add_child(dev, NULL, DEVICE_UNIT_ANY); if (child == NULL) { device_printf(dev, "cannot create child device\n"); return (ENOMEM); } cn->vtpci_child_dev = child; return (0); } int vtpci_delete_child(struct vtpci_common *cn) { device_t dev; int error; dev = cn->vtpci_dev; error = bus_generic_detach(dev); if (error) return (error); return (0); } void vtpci_child_detached(struct vtpci_common *cn) { vtpci_release_child_resources(cn); cn->vtpci_child_feat_desc = NULL; cn->vtpci_host_features = 0; cn->vtpci_features = 0; } int vtpci_reinit(struct vtpci_common *cn) { int idx, error; for (idx = 0; idx < cn->vtpci_nvqs; idx++) { error = vtpci_reinit_virtqueue(cn, idx); if (error) return (error); } if (vtpci_is_msix_enabled(cn)) { error = vtpci_set_host_msix_vectors(cn); if (error) return (error); } return (0); } static void vtpci_describe_features(struct vtpci_common *cn, const char *msg, uint64_t features) { device_t dev, child; dev = cn->vtpci_dev; child = cn->vtpci_child_dev; if (device_is_attached(child) || bootverbose == 0) return; virtio_describe(dev, msg, features, cn->vtpci_child_feat_desc); } uint64_t vtpci_negotiate_features(struct vtpci_common *cn, uint64_t child_features, uint64_t host_features) { uint64_t features; cn->vtpci_host_features = host_features; vtpci_describe_features(cn, "host", host_features); /* * Limit negotiated features to what the driver, virtqueue, and * host all support. */ features = host_features & child_features; features = virtio_filter_transport_features(features); cn->vtpci_features = features; vtpci_describe_features(cn, "negotiated", features); return (features); } bool vtpci_with_feature(struct vtpci_common *cn, uint64_t feature) { return ((cn->vtpci_features & feature) != 0); } int vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result) { device_t dev; int error; dev = cn->vtpci_dev; error = 0; switch (index) { case VIRTIO_IVAR_SUBDEVICE: *result = pci_get_subdevice(dev); break; case VIRTIO_IVAR_VENDOR: *result = pci_get_vendor(dev); break; case VIRTIO_IVAR_DEVICE: *result = pci_get_device(dev); break; case VIRTIO_IVAR_SUBVENDOR: *result = pci_get_subvendor(dev); break; case VIRTIO_IVAR_MODERN: *result = vtpci_is_modern(cn); break; default: error = ENOENT; } return (error); } int vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value) { int error; error = 0; switch (index) { case VIRTIO_IVAR_FEATURE_DESC: cn->vtpci_child_feat_desc = (void *) value; break; default: error = ENOENT; } return (error); } int vtpci_alloc_virtqueues(struct vtpci_common *cn, int nvqs, struct vq_alloc_info *vq_info) { device_t dev; int idx, align, error; dev = cn->vtpci_dev; /* * This is VIRTIO_PCI_VRING_ALIGN from legacy VirtIO. In modern VirtIO, * the tables do not have to be allocated contiguously, but we do so * anyways. */ align = 4096; if (cn->vtpci_nvqs != 0) return (EALREADY); if (nvqs <= 0) return (EINVAL); cn->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), M_DEVBUF, M_NOWAIT | M_ZERO); if (cn->vtpci_vqs == NULL) return (ENOMEM); for (idx = 0; idx < nvqs; idx++) { struct vtpci_virtqueue *vqx; struct vq_alloc_info *info; struct virtqueue *vq; bus_size_t notify_offset; uint16_t size; vqx = &cn->vtpci_vqs[idx]; info = &vq_info[idx]; size = vtpci_get_vq_size(cn, idx); notify_offset = vtpci_get_vq_notify_off(cn, idx); error = virtqueue_alloc(dev, idx, size, notify_offset, align, ~(vm_paddr_t)0, info, &vq); if (error) { device_printf(dev, "cannot allocate virtqueue %d: %d\n", idx, error); break; } vtpci_set_vq(cn, vq); vqx->vtv_vq = *info->vqai_vq = vq; vqx->vtv_no_intr = info->vqai_intr == NULL; cn->vtpci_nvqs++; } if (error) vtpci_free_virtqueues(cn); return (error); } static int vtpci_alloc_msix(struct vtpci_common *cn, int nvectors) { device_t dev; int nmsix, cnt, required; dev = cn->vtpci_dev; /* Allocate an additional vector for the config changes. */ required = nvectors + 1; nmsix = pci_msix_count(dev); if (nmsix < required) return (1); cnt = required; if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { cn->vtpci_nmsix_resources = required; return (0); } pci_release_msi(dev); return (1); } static int vtpci_alloc_msi(struct vtpci_common *cn) { device_t dev; int nmsi, cnt, required; dev = cn->vtpci_dev; required = 1; nmsi = pci_msi_count(dev); if (nmsi < required) return (1); cnt = required; if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) return (0); pci_release_msi(dev); return (1); } static int vtpci_alloc_intr_msix_pervq(struct vtpci_common *cn) { int i, nvectors, error; if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) return (ENOTSUP); for (nvectors = 0, i = 0; i < cn->vtpci_nvqs; i++) { if (cn->vtpci_vqs[i].vtv_no_intr == 0) nvectors++; } error = vtpci_alloc_msix(cn, nvectors); if (error) return (error); cn->vtpci_flags |= VTPCI_FLAG_MSIX; return (0); } static int vtpci_alloc_intr_msix_shared(struct vtpci_common *cn) { int error; if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) return (ENOTSUP); error = vtpci_alloc_msix(cn, 1); if (error) return (error); cn->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; return (0); } static int vtpci_alloc_intr_msi(struct vtpci_common *cn) { int error; /* Only BHyVe supports MSI. */ if (cn->vtpci_flags & VTPCI_FLAG_NO_MSI) return (ENOTSUP); error = vtpci_alloc_msi(cn); if (error) return (error); cn->vtpci_flags |= VTPCI_FLAG_MSI; return (0); } static int vtpci_alloc_intr_intx(struct vtpci_common *cn) { cn->vtpci_flags |= VTPCI_FLAG_INTX; return (0); } static int vtpci_alloc_interrupt(struct vtpci_common *cn, int rid, int flags, struct vtpci_interrupt *intr) { struct resource *irq; irq = bus_alloc_resource_any(cn->vtpci_dev, SYS_RES_IRQ, &rid, flags); if (irq == NULL) return (ENXIO); intr->vti_irq = irq; intr->vti_rid = rid; return (0); } static void vtpci_free_interrupt(struct vtpci_common *cn, struct vtpci_interrupt *intr) { device_t dev; dev = cn->vtpci_dev; if (intr->vti_handler != NULL) { bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); intr->vti_handler = NULL; } if (intr->vti_irq != NULL) { bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, intr->vti_irq); intr->vti_irq = NULL; intr->vti_rid = -1; } } static void vtpci_free_interrupts(struct vtpci_common *cn) { struct vtpci_interrupt *intr; int i, nvq_intrs; vtpci_free_interrupt(cn, &cn->vtpci_device_interrupt); if (cn->vtpci_nmsix_resources != 0) { nvq_intrs = cn->vtpci_nmsix_resources - 1; cn->vtpci_nmsix_resources = 0; if ((intr = cn->vtpci_msix_vq_interrupts) != NULL) { for (i = 0; i < nvq_intrs; i++, intr++) vtpci_free_interrupt(cn, intr); free(cn->vtpci_msix_vq_interrupts, M_DEVBUF); cn->vtpci_msix_vq_interrupts = NULL; } } if (cn->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) pci_release_msi(cn->vtpci_dev); cn->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; } static void vtpci_free_virtqueues(struct vtpci_common *cn) { struct vtpci_virtqueue *vqx; int idx; for (idx = 0; idx < cn->vtpci_nvqs; idx++) { vtpci_disable_vq(cn, idx); vqx = &cn->vtpci_vqs[idx]; virtqueue_free(vqx->vtv_vq); vqx->vtv_vq = NULL; } free(cn->vtpci_vqs, M_DEVBUF); cn->vtpci_vqs = NULL; cn->vtpci_nvqs = 0; } void vtpci_release_child_resources(struct vtpci_common *cn) { vtpci_free_interrupts(cn); vtpci_free_virtqueues(cn); } static void vtpci_cleanup_setup_intr_attempt(struct vtpci_common *cn) { int idx; if (cn->vtpci_flags & VTPCI_FLAG_MSIX) { vtpci_register_cfg_msix(cn, NULL); for (idx = 0; idx < cn->vtpci_nvqs; idx++) vtpci_register_vq_msix(cn, idx, NULL); } vtpci_free_interrupts(cn); } static int vtpci_alloc_intr_resources(struct vtpci_common *cn) { struct vtpci_interrupt *intr; int i, rid, flags, nvq_intrs, error; flags = RF_ACTIVE; if (cn->vtpci_flags & VTPCI_FLAG_INTX) { rid = 0; flags |= RF_SHAREABLE; } else rid = 1; /* * When using INTX or MSI interrupts, this resource handles all * interrupts. When using MSIX, this resource handles just the * configuration changed interrupt. */ intr = &cn->vtpci_device_interrupt; error = vtpci_alloc_interrupt(cn, rid, flags, intr); if (error || cn->vtpci_flags & (VTPCI_FLAG_INTX | VTPCI_FLAG_MSI)) return (error); /* * Now allocate the interrupts for the virtqueues. This may be one * for all the virtqueues, or one for each virtqueue. Subtract one * below for because of the configuration changed interrupt. */ nvq_intrs = cn->vtpci_nmsix_resources - 1; cn->vtpci_msix_vq_interrupts = malloc(nvq_intrs * sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); if (cn->vtpci_msix_vq_interrupts == NULL) return (ENOMEM); intr = cn->vtpci_msix_vq_interrupts; for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { error = vtpci_alloc_interrupt(cn, rid, flags, intr); if (error) return (error); } return (0); } static int vtpci_setup_intx_interrupt(struct vtpci_common *cn, enum intr_type type) { struct vtpci_interrupt *intr; int error; intr = &cn->vtpci_device_interrupt; error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, vtpci_intx_intr, cn, &intr->vti_handler); return (error); } static int vtpci_setup_pervq_msix_interrupts(struct vtpci_common *cn, enum intr_type type) { struct vtpci_virtqueue *vqx; struct vtpci_interrupt *intr; int i, error; intr = cn->vtpci_msix_vq_interrupts; for (i = 0; i < cn->vtpci_nvqs; i++) { vqx = &cn->vtpci_vqs[i]; if (vqx->vtv_no_intr) continue; error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, &intr->vti_handler); if (error) return (error); intr++; } return (0); } static int vtpci_set_host_msix_vectors(struct vtpci_common *cn) { struct vtpci_interrupt *intr, *tintr; int idx, error; intr = &cn->vtpci_device_interrupt; error = vtpci_register_cfg_msix(cn, intr); if (error) return (error); intr = cn->vtpci_msix_vq_interrupts; for (idx = 0; idx < cn->vtpci_nvqs; idx++) { if (cn->vtpci_vqs[idx].vtv_no_intr) tintr = NULL; else tintr = intr; error = vtpci_register_vq_msix(cn, idx, tintr); if (error) break; /* * For shared MSIX, all the virtqueues share the first * interrupt. */ if (!cn->vtpci_vqs[idx].vtv_no_intr && (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) intr++; } return (error); } static int vtpci_setup_msix_interrupts(struct vtpci_common *cn, enum intr_type type) { struct vtpci_interrupt *intr; int error; intr = &cn->vtpci_device_interrupt; error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, vtpci_config_intr, cn, &intr->vti_handler); if (error) return (error); if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { intr = &cn->vtpci_msix_vq_interrupts[0]; error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, cn, &intr->vti_handler); } else error = vtpci_setup_pervq_msix_interrupts(cn, type); return (error ? error : vtpci_set_host_msix_vectors(cn)); } static int vtpci_setup_intrs(struct vtpci_common *cn, enum intr_type type) { int error; type |= INTR_MPSAFE; KASSERT(cn->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, ("%s: no interrupt type selected %#x", __func__, cn->vtpci_flags)); error = vtpci_alloc_intr_resources(cn); if (error) return (error); if (cn->vtpci_flags & VTPCI_FLAG_INTX) error = vtpci_setup_intx_interrupt(cn, type); else if (cn->vtpci_flags & VTPCI_FLAG_MSI) error = vtpci_setup_msi_interrupt(cn, type); else error = vtpci_setup_msix_interrupts(cn, type); return (error); } int vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type) { device_t dev; int attempt, error; dev = cn->vtpci_dev; for (attempt = 0; attempt < 5; attempt++) { /* * Start with the most desirable interrupt configuration and * fallback towards less desirable ones. */ switch (attempt) { case 0: error = vtpci_alloc_intr_msix_pervq(cn); break; case 1: error = vtpci_alloc_intr_msix_shared(cn); break; case 2: error = vtpci_alloc_intr_msi(cn); break; case 3: error = vtpci_alloc_intr_intx(cn); break; default: device_printf(dev, "exhausted all interrupt allocation attempts\n"); return (ENXIO); } if (error == 0 && vtpci_setup_intrs(cn, type) == 0) break; vtpci_cleanup_setup_intr_attempt(cn); } if (bootverbose) { if (cn->vtpci_flags & VTPCI_FLAG_INTX) device_printf(dev, "using legacy interrupt\n"); else if (cn->vtpci_flags & VTPCI_FLAG_MSI) device_printf(dev, "using MSI interrupt\n"); else if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) device_printf(dev, "using shared MSIX interrupts\n"); else device_printf(dev, "using per VQ MSIX interrupts\n"); } return (0); } static int vtpci_reinit_virtqueue(struct vtpci_common *cn, int idx) { struct vtpci_virtqueue *vqx; struct virtqueue *vq; int error; vqx = &cn->vtpci_vqs[idx]; vq = vqx->vtv_vq; KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); error = virtqueue_reinit(vq, vtpci_get_vq_size(cn, idx)); if (error == 0) vtpci_set_vq(cn, vq); return (error); } static void vtpci_intx_intr(void *xcn) { struct vtpci_common *cn; struct vtpci_virtqueue *vqx; int i; uint8_t isr; cn = xcn; isr = vtpci_read_isr(cn); if (isr & VIRTIO_PCI_ISR_CONFIG) vtpci_config_intr(cn); if (isr & VIRTIO_PCI_ISR_INTR) { vqx = &cn->vtpci_vqs[0]; for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) virtqueue_intr(vqx->vtv_vq); } } } static int vtpci_vq_shared_intr_filter(void *xcn) { struct vtpci_common *cn; struct vtpci_virtqueue *vqx; int i, rc; cn = xcn; vqx = &cn->vtpci_vqs[0]; rc = 0; for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) rc |= virtqueue_intr_filter(vqx->vtv_vq); } return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); } static void vtpci_vq_shared_intr(void *xcn) { struct vtpci_common *cn; struct vtpci_virtqueue *vqx; int i; cn = xcn; vqx = &cn->vtpci_vqs[0]; for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) virtqueue_intr(vqx->vtv_vq); } } static int vtpci_vq_intr_filter(void *xvq) { struct virtqueue *vq; int rc; vq = xvq; rc = virtqueue_intr_filter(vq); return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); } static void vtpci_vq_intr(void *xvq) { struct virtqueue *vq; vq = xvq; virtqueue_intr(vq); } static void vtpci_config_intr(void *xcn) { struct vtpci_common *cn; device_t child; cn = xcn; child = cn->vtpci_child_dev; if (child != NULL) VIRTIO_CONFIG_CHANGE(child); } static int vtpci_feature_sysctl(struct sysctl_req *req, struct vtpci_common *cn, uint64_t features) { struct sbuf *sb; int error; sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); error = virtio_describe_sbuf(sb, features, cn->vtpci_child_feat_desc); sbuf_delete(sb); return (error); } static int vtpci_host_features_sysctl(SYSCTL_HANDLER_ARGS) { struct vtpci_common *cn; cn = arg1; return (vtpci_feature_sysctl(req, cn, cn->vtpci_host_features)); } static int vtpci_negotiated_features_sysctl(SYSCTL_HANDLER_ARGS) { struct vtpci_common *cn; cn = arg1; return (vtpci_feature_sysctl(req, cn, cn->vtpci_features)); } static void vtpci_setup_sysctl(struct vtpci_common *cn) { device_t dev; struct sysctl_ctx_list *ctx; struct sysctl_oid *tree; struct sysctl_oid_list *child; dev = cn->vtpci_dev; ctx = device_get_sysctl_ctx(dev); tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nvqs", CTLFLAG_RD, &cn->vtpci_nvqs, 0, "Number of virtqueues"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "host_features", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0, vtpci_host_features_sysctl, "A", "Features supported by the host"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "negotiated_features", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0, vtpci_negotiated_features_sysctl, "A", "Features negotiated"); } diff --git a/sys/dev/virtio/pci/virtio_pci_modern.c b/sys/dev/virtio/pci/virtio_pci_modern.c index eb1d5a1e6989..108fd2b5f8e9 100644 --- a/sys/dev/virtio/pci/virtio_pci_modern.c +++ b/sys/dev/virtio/pci/virtio_pci_modern.c @@ -1,1442 +1,1447 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for the modern VirtIO PCI interface. */ #include #include #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_bus_if.h" #include "virtio_pci_if.h" #include "virtio_if.h" struct vtpci_modern_resource_map { struct resource_map vtrm_map; int vtrm_cap_offset; int vtrm_bar; int vtrm_offset; int vtrm_length; int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */ }; struct vtpci_modern_bar_resource { struct resource *vtbr_res; int vtbr_type; }; struct vtpci_modern_softc { device_t vtpci_dev; struct vtpci_common vtpci_common; uint32_t vtpci_notify_offset_multiplier; uint16_t vtpci_devid; int vtpci_msix_bar; struct resource *vtpci_msix_res; struct vtpci_modern_resource_map vtpci_common_res_map; struct vtpci_modern_resource_map vtpci_notify_res_map; struct vtpci_modern_resource_map vtpci_isr_res_map; struct vtpci_modern_resource_map vtpci_device_res_map; #define VTPCI_MODERN_MAX_BARS 6 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS]; }; static int vtpci_modern_probe(device_t); static int vtpci_modern_attach(device_t); static int vtpci_modern_detach(device_t); static int vtpci_modern_suspend(device_t); static int vtpci_modern_resume(device_t); static int vtpci_modern_shutdown(device_t); static void vtpci_modern_driver_added(device_t, driver_t *); static void vtpci_modern_child_detached(device_t, device_t); static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *); static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t); static uint8_t vtpci_modern_read_isr(device_t); static uint16_t vtpci_modern_get_vq_size(device_t, int); static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int); static void vtpci_modern_set_vq(device_t, struct virtqueue *); static void vtpci_modern_disable_vq(device_t, int); static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int, struct vtpci_interrupt *); static int vtpci_modern_register_cfg_msix(device_t, struct vtpci_interrupt *); static int vtpci_modern_register_vq_msix(device_t, int idx, struct vtpci_interrupt *); static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t); static int vtpci_modern_finalize_features(device_t); static bool vtpci_modern_with_feature(device_t, uint64_t); static int vtpci_modern_alloc_virtqueues(device_t, int, struct vq_alloc_info *); static int vtpci_modern_setup_interrupts(device_t, enum intr_type); static void vtpci_modern_stop(device_t); static int vtpci_modern_reinit(device_t, uint64_t); static void vtpci_modern_reinit_complete(device_t); static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t); static int vtpci_modern_config_generation(device_t); static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int); static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int); static int vtpci_modern_probe_configs(device_t); static int vtpci_modern_find_cap(device_t, uint8_t, int *); static int vtpci_modern_map_configs(struct vtpci_modern_softc *); static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *); static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *, uint8_t, int, int, struct vtpci_modern_resource_map *); static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int); static struct resource *vtpci_modern_get_bar_resource( struct vtpci_modern_softc *, int, int); static struct resource *vtpci_modern_alloc_bar_resource( struct vtpci_modern_softc *, int, int); static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *); static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *, struct vtpci_modern_resource_map *); static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *, struct vtpci_modern_resource_map *); static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *); static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *); static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *); static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *); static void vtpci_modern_write_features(struct vtpci_modern_softc *, uint64_t); static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int); static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *); static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t); static void vtpci_modern_reset(struct vtpci_modern_softc *); static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *); static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *, bus_size_t); static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *, bus_size_t); static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *, bus_size_t); static void vtpci_modern_write_common_1(struct vtpci_modern_softc *, bus_size_t, uint8_t); static void vtpci_modern_write_common_2(struct vtpci_modern_softc *, bus_size_t, uint16_t); static void vtpci_modern_write_common_4(struct vtpci_modern_softc *, bus_size_t, uint32_t); static void vtpci_modern_write_common_8(struct vtpci_modern_softc *, bus_size_t, uint64_t); static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *, bus_size_t, uint16_t); static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *, bus_size_t); static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *, bus_size_t); static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *, bus_size_t); static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *, bus_size_t); static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *, bus_size_t); static void vtpci_modern_write_device_1(struct vtpci_modern_softc *, bus_size_t, uint8_t); static void vtpci_modern_write_device_2(struct vtpci_modern_softc *, bus_size_t, uint16_t); static void vtpci_modern_write_device_4(struct vtpci_modern_softc *, bus_size_t, uint32_t); static void vtpci_modern_write_device_8(struct vtpci_modern_softc *, bus_size_t, uint64_t); /* Tunables. */ +SYSCTL_DECL(_hw_virtio_pci); + static int vtpci_modern_transitional = 0; -TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional); +SYSCTL_INT(_hw_virtio_pci, OID_AUTO, transitional, CTLFLAG_RDTUN, + &vtpci_modern_transitional, 0, + "If 0, a transitional VirtIO device is used in legacy mode; otherwise, in modern mode."); static device_method_t vtpci_modern_methods[] = { /* Device interface. */ DEVMETHOD(device_probe, vtpci_modern_probe), DEVMETHOD(device_attach, vtpci_modern_attach), DEVMETHOD(device_detach, vtpci_modern_detach), DEVMETHOD(device_suspend, vtpci_modern_suspend), DEVMETHOD(device_resume, vtpci_modern_resume), DEVMETHOD(device_shutdown, vtpci_modern_shutdown), /* Bus interface. */ DEVMETHOD(bus_driver_added, vtpci_modern_driver_added), DEVMETHOD(bus_child_detached, vtpci_modern_child_detached), DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar), DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar), /* VirtIO PCI interface. */ DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr), DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size), DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off), DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq), DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq), DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix), DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix), /* VirtIO bus interface. */ DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features), DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features), DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature), DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues), DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts), DEVMETHOD(virtio_bus_stop, vtpci_modern_stop), DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit), DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete), DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq), DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation), DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config), DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config), DEVMETHOD_END }; static driver_t vtpci_modern_driver = { .name = "virtio_pci", .methods = vtpci_modern_methods, .size = sizeof(struct vtpci_modern_softc) }; DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0); static int vtpci_modern_probe(device_t dev) { const char *name; uint16_t devid; if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) return (ENXIO); if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX) return (ENXIO); if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) { if (!vtpci_modern_transitional) return (ENXIO); devid = pci_get_subdevice(dev); } else devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; if (vtpci_modern_probe_configs(dev) != 0) return (ENXIO); name = virtio_device_name(devid); if (name == NULL) name = "Unknown"; device_set_descf(dev, "VirtIO PCI (modern) %s adapter", name); return (BUS_PROBE_DEFAULT); } static int vtpci_modern_attach(device_t dev) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); sc->vtpci_dev = dev; vtpci_init(&sc->vtpci_common, dev, true); if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) sc->vtpci_devid = pci_get_subdevice(dev); else sc->vtpci_devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; error = vtpci_modern_map_configs(sc); if (error) { device_printf(dev, "cannot map configs\n"); vtpci_modern_unmap_configs(sc); return (error); } vtpci_modern_reset(sc); /* Tell the host we've noticed this device. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); error = vtpci_add_child(&sc->vtpci_common); if (error) goto fail; vtpci_modern_probe_and_attach_child(sc); return (0); fail: vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); vtpci_modern_detach(dev); return (error); } static int vtpci_modern_detach(device_t dev) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); error = vtpci_delete_child(&sc->vtpci_common); if (error) return (error); vtpci_modern_reset(sc); vtpci_modern_unmap_configs(sc); return (0); } static int vtpci_modern_suspend(device_t dev) { return (bus_generic_suspend(dev)); } static int vtpci_modern_resume(device_t dev) { return (bus_generic_resume(dev)); } static int vtpci_modern_shutdown(device_t dev) { (void) bus_generic_shutdown(dev); /* Forcibly stop the host device. */ vtpci_modern_stop(dev); return (0); } static void vtpci_modern_driver_added(device_t dev, driver_t *driver) { vtpci_modern_probe_and_attach_child(device_get_softc(dev)); } static void vtpci_modern_child_detached(device_t dev, device_t child) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_reset(sc); vtpci_child_detached(&sc->vtpci_common); /* After the reset, retell the host we've noticed this device. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); } static int vtpci_modern_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { struct vtpci_modern_softc *sc; struct vtpci_common *cn; sc = device_get_softc(dev); cn = &sc->vtpci_common; if (vtpci_child_device(cn) != child) return (ENOENT); switch (index) { case VIRTIO_IVAR_DEVTYPE: *result = sc->vtpci_devid; break; default: return (vtpci_read_ivar(cn, index, result)); } return (0); } static int vtpci_modern_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { struct vtpci_modern_softc *sc; struct vtpci_common *cn; sc = device_get_softc(dev); cn = &sc->vtpci_common; if (vtpci_child_device(cn) != child) return (ENOENT); switch (index) { default: return (vtpci_write_ivar(cn, index, value)); } return (0); } static uint64_t vtpci_modern_negotiate_features(device_t dev, uint64_t child_features) { struct vtpci_modern_softc *sc; uint64_t host_features, features; sc = device_get_softc(dev); host_features = vtpci_modern_read_features(sc); /* * Since the driver was added as a child of the modern PCI bus, * always add the V1 flag. */ child_features |= VIRTIO_F_VERSION_1; features = vtpci_negotiate_features(&sc->vtpci_common, child_features, host_features); vtpci_modern_write_features(sc, features); return (features); } static int vtpci_modern_finalize_features(device_t dev) { struct vtpci_modern_softc *sc; uint8_t status; sc = device_get_softc(dev); /* * Must re-read the status after setting it to verify the negotiated * features were accepted by the device. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK); status = vtpci_modern_get_status(sc); if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { device_printf(dev, "desired features were not accepted\n"); return (ENOTSUP); } return (0); } static bool vtpci_modern_with_feature(device_t dev, uint64_t feature) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); return (vtpci_with_feature(&sc->vtpci_common, feature)); } static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *sc) { uint32_t features0, features1; vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0); features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1); features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); return (((uint64_t) features1 << 32) | features0); } static void vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features) { uint32_t features0, features1; features0 = features; features1 = features >> 32; vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0); vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0); vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1); vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1); } static int vtpci_modern_alloc_virtqueues(device_t dev, int nvqs, struct vq_alloc_info *vq_info) { struct vtpci_modern_softc *sc; struct vtpci_common *cn; uint16_t max_nvqs; sc = device_get_softc(dev); cn = &sc->vtpci_common; max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ); if (nvqs > max_nvqs) { device_printf(sc->vtpci_dev, "requested virtqueue count %d " "exceeds max %d\n", nvqs, max_nvqs); return (E2BIG); } return (vtpci_alloc_virtqueues(cn, nvqs, vq_info)); } static int vtpci_modern_setup_interrupts(device_t dev, enum intr_type type) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); error = vtpci_setup_interrupts(&sc->vtpci_common, type); if (error == 0) vtpci_modern_enable_virtqueues(sc); return (error); } static void vtpci_modern_stop(device_t dev) { vtpci_modern_reset(device_get_softc(dev)); } static int vtpci_modern_reinit(device_t dev, uint64_t features) { struct vtpci_modern_softc *sc; struct vtpci_common *cn; int error; sc = device_get_softc(dev); cn = &sc->vtpci_common; /* * Redrive the device initialization. This is a bit of an abuse of * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to * play nice. * * We do not allow the host device to change from what was originally * negotiated beyond what the guest driver changed. MSIX state should * not change, number of virtqueues and their size remain the same, etc. * This will need to be rethought when we want to support migration. */ if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) vtpci_modern_stop(dev); /* * Quickly drive the status through ACK and DRIVER. The device does * not become usable again until DRIVER_OK in reinit complete. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); /* * TODO: Check that features are not added as to what was * originally negotiated. */ vtpci_modern_negotiate_features(dev, features); error = vtpci_modern_finalize_features(dev); if (error) { device_printf(dev, "cannot finalize features during reinit\n"); return (error); } error = vtpci_reinit(cn); if (error) return (error); return (0); } static void vtpci_modern_reinit_complete(device_t dev) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_enable_virtqueues(sc); vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); } static void vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_write_notify_2(sc, offset, queue); } static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *sc) { return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS)); } static void vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status) { if (status != VIRTIO_CONFIG_STATUS_RESET) status |= vtpci_modern_get_status(sc); vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status); } static int vtpci_modern_config_generation(device_t dev) { struct vtpci_modern_softc *sc; uint8_t gen; sc = device_get_softc(dev); gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION); return (gen); } static void vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst, int length) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { panic("%s: attempt to read dev config but not present", __func__); } switch (length) { case 1: *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset); break; case 2: *(uint16_t *) dst = virtio_htog16(true, vtpci_modern_read_device_2(sc, offset)); break; case 4: *(uint32_t *) dst = virtio_htog32(true, vtpci_modern_read_device_4(sc, offset)); break; case 8: *(uint64_t *) dst = virtio_htog64(true, vtpci_modern_read_device_8(sc, offset)); break; default: panic("%s: device %s invalid device read length %d offset %d", __func__, device_get_nameunit(dev), length, (int) offset); } } static void vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src, int length) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { panic("%s: attempt to write dev config but not present", __func__); } switch (length) { case 1: vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src); break; case 2: { uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src); vtpci_modern_write_device_2(sc, offset, val); break; } case 4: { uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src); vtpci_modern_write_device_4(sc, offset, val); break; } case 8: { uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src); vtpci_modern_write_device_8(sc, offset, val); break; } default: panic("%s: device %s invalid device write length %d offset %d", __func__, device_get_nameunit(dev), length, (int) offset); } } static int vtpci_modern_probe_configs(device_t dev) { int error; /* * These config capabilities must be present. The DEVICE_CFG * capability is only present if the device requires it. */ error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL); if (error) { device_printf(dev, "cannot find COMMON_CFG capability\n"); return (error); } error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL); if (error) { device_printf(dev, "cannot find NOTIFY_CFG capability\n"); return (error); } error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL); if (error) { device_printf(dev, "cannot find ISR_CFG capability\n"); return (error); } return (0); } static int vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset) { uint32_t type, bar; int capreg, error; for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg); error == 0; error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) { type = pci_read_config(dev, capreg + offsetof(struct virtio_pci_cap, cfg_type), 1); bar = pci_read_config(dev, capreg + offsetof(struct virtio_pci_cap, bar), 1); /* Must ignore reserved BARs. */ if (bar >= VTPCI_MODERN_MAX_BARS) continue; if (type == cfg_type) { if (cap_offset != NULL) *cap_offset = capreg; break; } } return (error); } static int vtpci_modern_map_common_config(struct vtpci_modern_softc *sc) { device_t dev; int error; dev = sc->vtpci_dev; error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG, sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map); if (error) { device_printf(dev, "cannot find cap COMMON_CFG resource\n"); return (error); } error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map); if (error) { device_printf(dev, "cannot alloc resource for COMMON_CFG\n"); return (error); } return (0); } static int vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc) { device_t dev; int cap_offset, error; dev = sc->vtpci_dev; error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG, -1, 2, &sc->vtpci_notify_res_map); if (error) { device_printf(dev, "cannot find cap NOTIFY_CFG resource\n"); return (error); } cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset; sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4); error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map); if (error) { device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n"); return (error); } return (0); } static int vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc) { device_t dev; int error; dev = sc->vtpci_dev; error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG, sizeof(uint8_t), 1, &sc->vtpci_isr_res_map); if (error) { device_printf(dev, "cannot find cap ISR_CFG resource\n"); return (error); } error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map); if (error) { device_printf(dev, "cannot alloc resource for ISR_CFG\n"); return (error); } return (0); } static int vtpci_modern_map_device_config(struct vtpci_modern_softc *sc) { device_t dev; int error; dev = sc->vtpci_dev; error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG, -1, 4, &sc->vtpci_device_res_map); if (error == ENOENT) { /* Device configuration is optional depending on device. */ return (0); } else if (error) { device_printf(dev, "cannot find cap DEVICE_CFG resource\n"); return (error); } error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map); if (error) { device_printf(dev, "cannot alloc resource for DEVICE_CFG\n"); return (error); } return (0); } static int vtpci_modern_map_configs(struct vtpci_modern_softc *sc) { int error; error = vtpci_modern_map_common_config(sc); if (error) return (error); error = vtpci_modern_map_notify_config(sc); if (error) return (error); error = vtpci_modern_map_isr_config(sc); if (error) return (error); error = vtpci_modern_map_device_config(sc); if (error) return (error); vtpci_modern_alloc_msix_resource(sc); return (0); } static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc) { vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map); vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map); vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map); vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map); vtpci_modern_free_bar_resources(sc); vtpci_modern_free_msix_resource(sc); sc->vtpci_notify_offset_multiplier = 0; } static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type, int min_size, int alignment, struct vtpci_modern_resource_map *res) { device_t dev; int cap_offset, offset, length, error; uint8_t bar, cap_length; dev = sc->vtpci_dev; error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset); if (error) return (error); cap_length = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1); if (cap_length < sizeof(struct virtio_pci_cap)) { device_printf(dev, "cap %u length %d less than expected\n", cfg_type, cap_length); return (ENXIO); } bar = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_cap, bar), 1); offset = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_cap, offset), 4); length = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_cap, length), 4); if (min_size != -1 && length < min_size) { device_printf(dev, "cap %u struct length %d less than min %d\n", cfg_type, length, min_size); return (ENXIO); } if (offset % alignment) { device_printf(dev, "cap %u struct offset %d not aligned to %d\n", cfg_type, offset, alignment); return (ENXIO); } /* BMV: TODO Can we determine the size of the BAR here? */ res->vtrm_cap_offset = cap_offset; res->vtrm_bar = bar; res->vtrm_offset = offset; res->vtrm_length = length; res->vtrm_type = vtpci_modern_bar_type(sc, bar); return (0); } static int vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar) { uint32_t val; /* * The BAR described by a config capability may be either an IOPORT or * MEM, but we must know the type when calling bus_alloc_resource(). */ val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4); if (PCI_BAR_IO(val)) return (SYS_RES_IOPORT); else return (SYS_RES_MEMORY); } static struct resource * vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) { struct resource *res; MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); res = sc->vtpci_bar_res[bar].vtbr_res; MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type); return (res); } static struct resource * vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) { struct resource *res; int rid; MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT); res = sc->vtpci_bar_res[bar].vtbr_res; if (res != NULL) { MPASS(sc->vtpci_bar_res[bar].vtbr_type == type); return (res); } rid = PCIR_BAR(bar); res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid, RF_ACTIVE | RF_UNMAPPED); if (res != NULL) { sc->vtpci_bar_res[bar].vtbr_res = res; sc->vtpci_bar_res[bar].vtbr_type = type; } return (res); } static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc) { device_t dev; struct resource *res; int bar, rid, type; dev = sc->vtpci_dev; for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) { res = sc->vtpci_bar_res[bar].vtbr_res; type = sc->vtpci_bar_res[bar].vtbr_type; if (res != NULL) { rid = PCIR_BAR(bar); bus_release_resource(dev, type, rid, res); sc->vtpci_bar_res[bar].vtbr_res = NULL; sc->vtpci_bar_res[bar].vtbr_type = 0; } } } static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc, struct vtpci_modern_resource_map *map) { struct resource_map_request req; struct resource *res; int type; type = map->vtrm_type; res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type); if (res == NULL) return (ENXIO); resource_init_map_request(&req); req.offset = map->vtrm_offset; req.length = map->vtrm_length; return (bus_map_resource(sc->vtpci_dev, type, res, &req, &map->vtrm_map)); } static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc, struct vtpci_modern_resource_map *map) { struct resource *res; int type; type = map->vtrm_type; res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type); if (res != NULL && map->vtrm_map.r_size != 0) { bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map); bzero(map, sizeof(struct vtpci_modern_resource_map)); } } static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc) { device_t dev; int bar; dev = sc->vtpci_dev; if (!vtpci_is_msix_available(&sc->vtpci_common) || (bar = pci_msix_table_bar(dev)) == -1) return; /* TODO: Can this BAR be in the 0-5 range? */ sc->vtpci_msix_bar = bar; if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE)) == NULL) device_printf(dev, "Unable to map MSIX table\n"); } static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc) { device_t dev; dev = sc->vtpci_dev; if (sc->vtpci_msix_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar, sc->vtpci_msix_res); sc->vtpci_msix_bar = 0; sc->vtpci_msix_res = NULL; } } static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc) { device_t dev, child; dev = sc->vtpci_dev; child = vtpci_child_device(&sc->vtpci_common); if (child == NULL || device_get_state(child) != DS_NOTPRESENT) return; if (device_probe(child) != 0) return; vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); if (device_attach(child) != 0) { vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); /* Reset state for later attempt. */ vtpci_modern_child_detached(dev, child); } else { vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); VIRTIO_ATTACH_COMPLETED(child); } } static int vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset, struct vtpci_interrupt *intr) { uint16_t vector; if (intr != NULL) { /* Map from guest rid to host vector. */ vector = intr->vti_rid - 1; } else vector = VIRTIO_MSI_NO_VECTOR; vtpci_modern_write_common_2(sc, offset, vector); return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV); } static int vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr); if (error) { device_printf(dev, "unable to register config MSIX interrupt\n"); return (error); } return (0); } static int vtpci_modern_register_vq_msix(device_t dev, int idx, struct vtpci_interrupt *intr) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, idx); error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr); if (error) { device_printf(dev, "unable to register virtqueue MSIX interrupt\n"); return (error); } return (0); } static void vtpci_modern_reset(struct vtpci_modern_softc *sc) { /* * Setting the status to RESET sets the host device to the * original, uninitialized state. Must poll the status until * the reset is complete. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) cpu_spinwait(); } static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx) { vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx); } static uint8_t vtpci_modern_read_isr(device_t dev) { return (vtpci_modern_read_isr_1(device_get_softc(dev), 0)); } static uint16_t vtpci_modern_get_vq_size(device_t dev, int idx) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, idx); return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE)); } static bus_size_t vtpci_modern_get_vq_notify_off(device_t dev, int idx) { struct vtpci_modern_softc *sc; uint16_t q_notify_off; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, idx); q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF); return (q_notify_off * sc->vtpci_notify_offset_multiplier); } static void vtpci_modern_set_vq(device_t dev, struct virtqueue *vq) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, virtqueue_index(vq)); /* BMV: Currently we never adjust the device's proposed VQ size. */ vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq)); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq)); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq)); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq)); } static void vtpci_modern_disable_vq(device_t dev, int idx) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, idx); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL); } static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc) { int idx; for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) { vtpci_modern_select_virtqueue(sc, idx); vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1); } } static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off)); } static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off) { return virtio_htog16(true, bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off)); } static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off) { return virtio_htog32(true, bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off)); } static void vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off, uint8_t val) { bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val); } static void vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off, uint16_t val) { bus_write_2(&sc->vtpci_common_res_map.vtrm_map, off, virtio_gtoh16(true, val)); } static void vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(&sc->vtpci_common_res_map.vtrm_map, off, virtio_gtoh32(true, val)); } static void vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off, uint64_t val) { uint32_t val0, val1; val0 = (uint32_t) val; val1 = val >> 32; vtpci_modern_write_common_4(sc, off, val0); vtpci_modern_write_common_4(sc, off + 4, val1); } static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off, uint16_t val) { bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val); } static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off)); } static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off)); } static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off)); } static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off)); } static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off) { device_t dev; int gen; uint32_t val0, val1; dev = sc->vtpci_dev; /* * Treat the 64-bit field as two 32-bit fields. Use the generation * to ensure a consistent read. */ do { gen = vtpci_modern_config_generation(dev); val0 = vtpci_modern_read_device_4(sc, off); val1 = vtpci_modern_read_device_4(sc, off + 4); } while (gen != vtpci_modern_config_generation(dev)); return (((uint64_t) val1 << 32) | val0); } static void vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off, uint8_t val) { bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val); } static void vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off, uint16_t val) { bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val); } static void vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val); } static void vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off, uint64_t val) { uint32_t val0, val1; val0 = (uint32_t) val; val1 = val >> 32; vtpci_modern_write_device_4(sc, off, val0); vtpci_modern_write_device_4(sc, off + 4, val1); } diff --git a/sys/dev/virtio/virtio.c b/sys/dev/virtio/virtio.c index 75d65ba4a8c8..be5edcafa7ff 100644 --- a/sys/dev/virtio/virtio.c +++ b/sys/dev/virtio/virtio.c @@ -1,384 +1,388 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include +#include #include #include #include #include #include #include #include #include #include #include "virtio_bus_if.h" static int virtio_modevent(module_t, int, void *); static const char *virtio_feature_name(uint64_t, struct virtio_feature_desc *); static struct virtio_ident { uint16_t devid; const char *name; } virtio_ident_table[] = { { VIRTIO_ID_NETWORK, "Network" }, { VIRTIO_ID_BLOCK, "Block" }, { VIRTIO_ID_CONSOLE, "Console" }, { VIRTIO_ID_ENTROPY, "Entropy" }, { VIRTIO_ID_BALLOON, "Balloon" }, { VIRTIO_ID_IOMEMORY, "IOMemory" }, { VIRTIO_ID_RPMSG, "Remote Processor Messaging" }, { VIRTIO_ID_SCSI, "SCSI" }, { VIRTIO_ID_9P, "9P Transport" }, { VIRTIO_ID_RPROC_SERIAL, "Remote Processor Serial" }, { VIRTIO_ID_CAIF, "CAIF" }, { VIRTIO_ID_GPU, "GPU" }, { VIRTIO_ID_INPUT, "Input" }, { VIRTIO_ID_VSOCK, "VSOCK Transport" }, { VIRTIO_ID_CRYPTO, "Crypto" }, { VIRTIO_ID_IOMMU, "IOMMU" }, { VIRTIO_ID_SOUND, "Sound" }, { VIRTIO_ID_FS, "Filesystem" }, { VIRTIO_ID_PMEM, "Persistent Memory" }, { VIRTIO_ID_RPMB, "RPMB" }, { VIRTIO_ID_SCMI, "SCMI" }, { VIRTIO_ID_GPIO, "GPIO" }, { 0, NULL } }; /* Device independent features. */ static struct virtio_feature_desc virtio_common_feature_desc[] = { { VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" }, /* Legacy */ { VIRTIO_F_ANY_LAYOUT, "AnyLayout" }, /* Legacy */ { VIRTIO_RING_F_INDIRECT_DESC, "RingIndirectDesc" }, { VIRTIO_RING_F_EVENT_IDX, "RingEventIdx" }, { VIRTIO_F_BAD_FEATURE, "BadFeature" }, /* Legacy */ { VIRTIO_F_VERSION_1, "Version1" }, { VIRTIO_F_IOMMU_PLATFORM, "IOMMUPlatform" }, { 0, NULL } }; +SYSCTL_NODE(_hw, OID_AUTO, virtio, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "VirtIO driver parameters"); + const char * virtio_device_name(uint16_t devid) { struct virtio_ident *ident; for (ident = virtio_ident_table; ident->name != NULL; ident++) { if (ident->devid == devid) return (ident->name); } return (NULL); } static const char * virtio_feature_name(uint64_t val, struct virtio_feature_desc *desc) { int i, j; struct virtio_feature_desc *descs[2] = { desc, virtio_common_feature_desc }; for (i = 0; i < 2; i++) { if (descs[i] == NULL) continue; for (j = 0; descs[i][j].vfd_val != 0; j++) { if (val == descs[i][j].vfd_val) return (descs[i][j].vfd_str); } } return (NULL); } int virtio_describe_sbuf(struct sbuf *sb, uint64_t features, struct virtio_feature_desc *desc) { const char *name; uint64_t val; int n; sbuf_printf(sb, "%#jx", (uintmax_t) features); for (n = 0, val = 1ULL << 63; val != 0; val >>= 1) { /* * BAD_FEATURE is used to detect broken Linux clients * and therefore is not applicable to FreeBSD. */ if (((features & val) == 0) || val == VIRTIO_F_BAD_FEATURE) continue; if (n++ == 0) sbuf_cat(sb, " <"); else sbuf_cat(sb, ","); name = virtio_feature_name(val, desc); if (name == NULL) sbuf_printf(sb, "%#jx", (uintmax_t) val); else sbuf_cat(sb, name); } if (n > 0) sbuf_cat(sb, ">"); return (sbuf_finish(sb)); } void virtio_describe(device_t dev, const char *msg, uint64_t features, struct virtio_feature_desc *desc) { struct sbuf sb; char *buf; int error; if ((buf = malloc(1024, M_TEMP, M_NOWAIT)) == NULL) { error = ENOMEM; goto out; } sbuf_new(&sb, buf, 1024, SBUF_FIXEDLEN); sbuf_printf(&sb, "%s features: ", msg); error = virtio_describe_sbuf(&sb, features, desc); if (error == 0) device_printf(dev, "%s\n", sbuf_data(&sb)); sbuf_delete(&sb); free(buf, M_TEMP); out: if (error != 0) { device_printf(dev, "%s features: %#jx\n", msg, (uintmax_t) features); } } uint64_t virtio_filter_transport_features(uint64_t features) { uint64_t transport, mask; transport = (1ULL << (VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START)) - 1; transport <<= VIRTIO_TRANSPORT_F_START; mask = -1ULL & ~transport; mask |= VIRTIO_RING_F_INDIRECT_DESC; mask |= VIRTIO_RING_F_EVENT_IDX; mask |= VIRTIO_F_VERSION_1; return (features & mask); } bool virtio_bus_is_modern(device_t dev) { uintptr_t modern; virtio_read_ivar(dev, VIRTIO_IVAR_MODERN, &modern); return (modern != 0); } void virtio_read_device_config_array(device_t dev, bus_size_t offset, void *dst, int size, int count) { int i, gen; do { gen = virtio_config_generation(dev); for (i = 0; i < count; i++) { virtio_read_device_config(dev, offset + i * size, (uint8_t *) dst + i * size, size); } } while (gen != virtio_config_generation(dev)); } /* * VirtIO bus method wrappers. */ void virtio_read_ivar(device_t dev, int ivar, uintptr_t *val) { *val = -1; BUS_READ_IVAR(device_get_parent(dev), dev, ivar, val); } void virtio_write_ivar(device_t dev, int ivar, uintptr_t val) { BUS_WRITE_IVAR(device_get_parent(dev), dev, ivar, val); } uint64_t virtio_negotiate_features(device_t dev, uint64_t child_features) { return (VIRTIO_BUS_NEGOTIATE_FEATURES(device_get_parent(dev), child_features)); } int virtio_finalize_features(device_t dev) { return (VIRTIO_BUS_FINALIZE_FEATURES(device_get_parent(dev))); } int virtio_alloc_virtqueues(device_t dev, int nvqs, struct vq_alloc_info *info) { return (VIRTIO_BUS_ALLOC_VIRTQUEUES(device_get_parent(dev), nvqs, info)); } int virtio_setup_intr(device_t dev, enum intr_type type) { return (VIRTIO_BUS_SETUP_INTR(device_get_parent(dev), type)); } bool virtio_with_feature(device_t dev, uint64_t feature) { return (VIRTIO_BUS_WITH_FEATURE(device_get_parent(dev), feature)); } void virtio_stop(device_t dev) { VIRTIO_BUS_STOP(device_get_parent(dev)); } int virtio_reinit(device_t dev, uint64_t features) { return (VIRTIO_BUS_REINIT(device_get_parent(dev), features)); } void virtio_reinit_complete(device_t dev) { VIRTIO_BUS_REINIT_COMPLETE(device_get_parent(dev)); } int virtio_config_generation(device_t dev) { return (VIRTIO_BUS_CONFIG_GENERATION(device_get_parent(dev))); } void virtio_read_device_config(device_t dev, bus_size_t offset, void *dst, int len) { VIRTIO_BUS_READ_DEVICE_CONFIG(device_get_parent(dev), offset, dst, len); } void virtio_write_device_config(device_t dev, bus_size_t offset, const void *dst, int len) { VIRTIO_BUS_WRITE_DEVICE_CONFIG(device_get_parent(dev), offset, dst, len); } int virtio_child_pnpinfo(device_t busdev __unused, device_t child, struct sbuf *sb) { /* * All of these PCI fields will be only 16 bits, but on the vtmmio bus * the corresponding fields (only "vendor" and "device_type") are 32 * bits. Many virtio drivers can attach below either bus. * Gratuitously expand these two fields to 32-bits to allow sharing PNP * match table data between the mostly-similar buses. * * Subdevice and device_type are redundant in both buses, so I don't * see a lot of PNP utility in exposing the same value under a * different name. */ sbuf_printf(sb, "vendor=0x%08x device=0x%04x subvendor=0x%04x " "device_type=0x%08x", (unsigned)virtio_get_vendor(child), (unsigned)virtio_get_device(child), (unsigned)virtio_get_subvendor(child), (unsigned)virtio_get_device_type(child)); return (0); } static int virtio_modevent(module_t mod, int type, void *unused) { int error; switch (type) { case MOD_LOAD: case MOD_QUIESCE: case MOD_UNLOAD: case MOD_SHUTDOWN: error = 0; break; default: error = EOPNOTSUPP; break; } return (error); } static moduledata_t virtio_mod = { "virtio", virtio_modevent, 0 }; DECLARE_MODULE(virtio, virtio_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); MODULE_VERSION(virtio, 1);