Index: head/sys/dev/virtio/pci/virtio_pci.c =================================================================== --- head/sys/dev/virtio/pci/virtio_pci.c (revision 267521) +++ head/sys/dev/virtio/pci/virtio_pci.c (revision 267522) @@ -1,1330 +1,1329 @@ /*- * Copyright (c) 2011, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for the VirtIO PCI interface. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include #include "virtio_bus_if.h" #include "virtio_if.h" struct vtpci_interrupt { struct resource *vti_irq; int vti_rid; void *vti_handler; }; struct vtpci_virtqueue { struct virtqueue *vtv_vq; int vtv_no_intr; }; struct vtpci_softc { device_t vtpci_dev; struct resource *vtpci_res; struct resource *vtpci_msix_res; uint64_t vtpci_features; uint32_t vtpci_flags; #define VTPCI_FLAG_NO_MSI 0x0001 #define VTPCI_FLAG_NO_MSIX 0x0002 #define VTPCI_FLAG_LEGACY 0x1000 #define VTPCI_FLAG_MSI 0x2000 #define VTPCI_FLAG_MSIX 0x4000 #define VTPCI_FLAG_SHARED_MSIX 0x8000 #define VTPCI_FLAG_ITYPE_MASK 0xF000 /* This "bus" will only ever have one child. */ device_t vtpci_child_dev; struct virtio_feature_desc *vtpci_child_feat_desc; int vtpci_nvqs; struct vtpci_virtqueue *vtpci_vqs; /* * Ideally, each virtqueue that the driver provides a callback for will * receive its own MSIX vector. If there are not sufficient vectors * available, then attempt to have all the VQs share one vector. For * MSIX, the configuration changed notifications must be on their own * vector. * * If MSIX is not available, we will attempt to have the whole device * share one MSI vector, and then, finally, one legacy interrupt. */ struct vtpci_interrupt vtpci_device_interrupt; struct vtpci_interrupt *vtpci_msix_vq_interrupts; int vtpci_nmsix_resources; }; static int vtpci_probe(device_t); static int vtpci_attach(device_t); static int vtpci_detach(device_t); static int vtpci_suspend(device_t); static int vtpci_resume(device_t); static int vtpci_shutdown(device_t); static void vtpci_driver_added(device_t, driver_t *); static void vtpci_child_detached(device_t, device_t); static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); static int vtpci_write_ivar(device_t, device_t, int, uintptr_t); static uint64_t vtpci_negotiate_features(device_t, uint64_t); static int vtpci_with_feature(device_t, uint64_t); static int vtpci_alloc_virtqueues(device_t, int, int, struct vq_alloc_info *); static int vtpci_setup_intr(device_t, enum intr_type); static void vtpci_stop(device_t); static int vtpci_reinit(device_t, uint64_t); static void vtpci_reinit_complete(device_t); static void vtpci_notify_virtqueue(device_t, uint16_t); static uint8_t vtpci_get_status(device_t); static void vtpci_set_status(device_t, uint8_t); static void vtpci_read_dev_config(device_t, bus_size_t, void *, int); static void vtpci_write_dev_config(device_t, bus_size_t, void *, int); static void vtpci_describe_features(struct vtpci_softc *, const char *, uint64_t); static void vtpci_probe_and_attach_child(struct vtpci_softc *); static int vtpci_alloc_msix(struct vtpci_softc *, int); static int vtpci_alloc_msi(struct vtpci_softc *); static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *); static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *); static int vtpci_alloc_intr_msi(struct vtpci_softc *); static int vtpci_alloc_intr_legacy(struct vtpci_softc *); static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int, struct vtpci_interrupt *); static int vtpci_alloc_intr_resources(struct vtpci_softc *); static int vtpci_setup_legacy_interrupt(struct vtpci_softc *, enum intr_type); static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *, enum intr_type); static int vtpci_setup_msix_interrupts(struct vtpci_softc *, enum intr_type); static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type); static int vtpci_register_msix_vector(struct vtpci_softc *, int, struct vtpci_interrupt *); static int vtpci_set_host_msix_vectors(struct vtpci_softc *); static int vtpci_reinit_virtqueue(struct vtpci_softc *, int); static void vtpci_free_interrupt(struct vtpci_softc *, struct vtpci_interrupt *); static void vtpci_free_interrupts(struct vtpci_softc *); static void vtpci_free_virtqueues(struct vtpci_softc *); static void vtpci_release_child_resources(struct vtpci_softc *); static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *); static void vtpci_reset(struct vtpci_softc *); static void vtpci_select_virtqueue(struct vtpci_softc *, int); static void vtpci_legacy_intr(void *); static int vtpci_vq_shared_intr_filter(void *); static void vtpci_vq_shared_intr(void *); static int vtpci_vq_intr_filter(void *); static void vtpci_vq_intr(void *); static void vtpci_config_intr(void *); #define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt #define VIRTIO_PCI_CONFIG(_sc) \ VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0) /* * I/O port read/write wrappers. */ #define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) #define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o)) #define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o)) #define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) #define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) #define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) /* Tunables. */ static int vtpci_disable_msix = 0; TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); static device_method_t vtpci_methods[] = { /* Device interface. */ DEVMETHOD(device_probe, vtpci_probe), DEVMETHOD(device_attach, vtpci_attach), DEVMETHOD(device_detach, vtpci_detach), DEVMETHOD(device_suspend, vtpci_suspend), DEVMETHOD(device_resume, vtpci_resume), DEVMETHOD(device_shutdown, vtpci_shutdown), /* Bus interface. */ DEVMETHOD(bus_driver_added, vtpci_driver_added), DEVMETHOD(bus_child_detached, vtpci_child_detached), DEVMETHOD(bus_read_ivar, vtpci_read_ivar), DEVMETHOD(bus_write_ivar, vtpci_write_ivar), /* VirtIO bus interface. */ DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), DEVMETHOD(virtio_bus_stop, vtpci_stop), DEVMETHOD(virtio_bus_reinit, vtpci_reinit), DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), DEVMETHOD_END }; static driver_t vtpci_driver = { "virtio_pci", vtpci_methods, sizeof(struct vtpci_softc) }; devclass_t vtpci_devclass; DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); MODULE_VERSION(virtio_pci, 1); MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); static int vtpci_probe(device_t dev) { char desc[36]; const char *name; if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) return (ENXIO); if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) return (ENXIO); if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) return (ENXIO); name = virtio_device_name(pci_get_subdevice(dev)); if (name == NULL) name = "Unknown"; snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); device_set_desc_copy(dev, desc); return (BUS_PROBE_DEFAULT); } static int vtpci_attach(device_t dev) { struct vtpci_softc *sc; device_t child; int rid; sc = device_get_softc(dev); sc->vtpci_dev = dev; pci_enable_busmaster(dev); rid = PCIR_BAR(0); sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, RF_ACTIVE); if (sc->vtpci_res == NULL) { device_printf(dev, "cannot map I/O space\n"); return (ENXIO); } if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) sc->vtpci_flags |= VTPCI_FLAG_NO_MSI; if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { rid = PCIR_BAR(1); sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); } if (sc->vtpci_msix_res == NULL) sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX; vtpci_reset(sc); /* Tell the host we've noticed this device. */ vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); if ((child = device_add_child(dev, NULL, -1)) == NULL) { device_printf(dev, "cannot create child device\n"); vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); vtpci_detach(dev); return (ENOMEM); } sc->vtpci_child_dev = child; vtpci_probe_and_attach_child(sc); return (0); } static int vtpci_detach(device_t dev) { struct vtpci_softc *sc; device_t child; int error; sc = device_get_softc(dev); if ((child = sc->vtpci_child_dev) != NULL) { error = device_delete_child(dev, child); if (error) return (error); sc->vtpci_child_dev = NULL; } vtpci_reset(sc); if (sc->vtpci_msix_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), sc->vtpci_msix_res); sc->vtpci_msix_res = NULL; } if (sc->vtpci_res != NULL) { bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), sc->vtpci_res); sc->vtpci_res = NULL; } return (0); } static int vtpci_suspend(device_t dev) { return (bus_generic_suspend(dev)); } static int vtpci_resume(device_t dev) { return (bus_generic_resume(dev)); } static int vtpci_shutdown(device_t dev) { (void) bus_generic_shutdown(dev); /* Forcibly stop the host device. */ vtpci_stop(dev); return (0); } static void vtpci_driver_added(device_t dev, driver_t *driver) { struct vtpci_softc *sc; sc = device_get_softc(dev); vtpci_probe_and_attach_child(sc); } static void vtpci_child_detached(device_t dev, device_t child) { struct vtpci_softc *sc; sc = device_get_softc(dev); vtpci_reset(sc); vtpci_release_child_resources(sc); } static int vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { struct vtpci_softc *sc; sc = device_get_softc(dev); if (sc->vtpci_child_dev != child) return (ENOENT); switch (index) { case VIRTIO_IVAR_DEVTYPE: case VIRTIO_IVAR_SUBDEVICE: *result = pci_get_subdevice(dev); break; case VIRTIO_IVAR_VENDOR: *result = pci_get_vendor(dev); break; case VIRTIO_IVAR_DEVICE: *result = pci_get_device(dev); break; case VIRTIO_IVAR_SUBVENDOR: *result = pci_get_subdevice(dev); break; default: return (ENOENT); } return (0); } static int vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { struct vtpci_softc *sc; sc = device_get_softc(dev); if (sc->vtpci_child_dev != child) return (ENOENT); switch (index) { case VIRTIO_IVAR_FEATURE_DESC: sc->vtpci_child_feat_desc = (void *) value; break; default: return (ENOENT); } return (0); } static uint64_t vtpci_negotiate_features(device_t dev, uint64_t child_features) { struct vtpci_softc *sc; uint64_t host_features, features; sc = device_get_softc(dev); host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES); vtpci_describe_features(sc, "host", host_features); /* * Limit negotiated features to what the driver, virtqueue, and * host all support. */ features = host_features & child_features; features = virtqueue_filter_features(features); sc->vtpci_features = features; vtpci_describe_features(sc, "negotiated", features); vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); return (features); } static int vtpci_with_feature(device_t dev, uint64_t feature) { struct vtpci_softc *sc; sc = device_get_softc(dev); return ((sc->vtpci_features & feature) != 0); } static int vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, struct vq_alloc_info *vq_info) { struct vtpci_softc *sc; struct virtqueue *vq; struct vtpci_virtqueue *vqx; struct vq_alloc_info *info; int idx, error; uint16_t size; sc = device_get_softc(dev); if (sc->vtpci_nvqs != 0) return (EALREADY); if (nvqs <= 0) return (EINVAL); sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->vtpci_vqs == NULL) return (ENOMEM); for (idx = 0; idx < nvqs; idx++) { vqx = &sc->vtpci_vqs[idx]; info = &vq_info[idx]; vtpci_select_virtqueue(sc, idx); size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN, 0xFFFFFFFFUL, info, &vq); if (error) { device_printf(dev, "cannot allocate virtqueue %d: %d\n", idx, error); break; } vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); vqx->vtv_vq = *info->vqai_vq = vq; vqx->vtv_no_intr = info->vqai_intr == NULL; sc->vtpci_nvqs++; } if (error) vtpci_free_virtqueues(sc); return (error); } static int vtpci_setup_intr(device_t dev, enum intr_type type) { struct vtpci_softc *sc; int attempt, error; sc = device_get_softc(dev); for (attempt = 0; attempt < 5; attempt++) { /* * Start with the most desirable interrupt configuration and * fallback towards less desirable ones. */ switch (attempt) { case 0: error = vtpci_alloc_intr_msix_pervq(sc); break; case 1: error = vtpci_alloc_intr_msix_shared(sc); break; case 2: error = vtpci_alloc_intr_msi(sc); break; case 3: error = vtpci_alloc_intr_legacy(sc); break; default: device_printf(dev, "exhausted all interrupt allocation attempts\n"); return (ENXIO); } if (error == 0 && vtpci_setup_interrupts(sc, type) == 0) break; vtpci_cleanup_setup_intr_attempt(sc); } if (bootverbose) { if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) device_printf(dev, "using legacy interrupt\n"); else if (sc->vtpci_flags & VTPCI_FLAG_MSI) device_printf(dev, "using MSI interrupt\n"); else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) device_printf(dev, "using shared MSIX interrupts\n"); else device_printf(dev, "using per VQ MSIX interrupts\n"); } return (0); } static void vtpci_stop(device_t dev) { vtpci_reset(device_get_softc(dev)); } static int vtpci_reinit(device_t dev, uint64_t features) { struct vtpci_softc *sc; int idx, error; sc = device_get_softc(dev); /* * Redrive the device initialization. This is a bit of an abuse of * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to * play nice. * * We do not allow the host device to change from what was originally * negotiated beyond what the guest driver changed. MSIX state should * not change, number of virtqueues and their size remain the same, etc. * This will need to be rethought when we want to support migration. */ if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) vtpci_stop(dev); /* * Quickly drive the status through ACK and DRIVER. The device * does not become usable again until vtpci_reinit_complete(). */ vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); vtpci_negotiate_features(dev, features); for (idx = 0; idx < sc->vtpci_nvqs; idx++) { error = vtpci_reinit_virtqueue(sc, idx); if (error) return (error); } if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { error = vtpci_set_host_msix_vectors(sc); if (error) return (error); } return (0); } static void vtpci_reinit_complete(device_t dev) { vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); } static void vtpci_notify_virtqueue(device_t dev, uint16_t queue) { struct vtpci_softc *sc; sc = device_get_softc(dev); vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); } static uint8_t vtpci_get_status(device_t dev) { struct vtpci_softc *sc; sc = device_get_softc(dev); return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); } static void vtpci_set_status(device_t dev, uint8_t status) { struct vtpci_softc *sc; sc = device_get_softc(dev); if (status != VIRTIO_CONFIG_STATUS_RESET) status |= vtpci_get_status(dev); vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); } static void vtpci_read_dev_config(device_t dev, bus_size_t offset, void *dst, int length) { struct vtpci_softc *sc; bus_size_t off; uint8_t *d; int size; sc = device_get_softc(dev); off = VIRTIO_PCI_CONFIG(sc) + offset; for (d = dst; length > 0; d += size, off += size, length -= size) { if (length >= 4) { size = 4; *(uint32_t *)d = vtpci_read_config_4(sc, off); } else if (length >= 2) { size = 2; *(uint16_t *)d = vtpci_read_config_2(sc, off); } else { size = 1; *d = vtpci_read_config_1(sc, off); } } } static void vtpci_write_dev_config(device_t dev, bus_size_t offset, void *src, int length) { struct vtpci_softc *sc; bus_size_t off; uint8_t *s; int size; sc = device_get_softc(dev); off = VIRTIO_PCI_CONFIG(sc) + offset; for (s = src; length > 0; s += size, off += size, length -= size) { if (length >= 4) { size = 4; vtpci_write_config_4(sc, off, *(uint32_t *)s); } else if (length >= 2) { size = 2; vtpci_write_config_2(sc, off, *(uint16_t *)s); } else { size = 1; vtpci_write_config_1(sc, off, *s); } } } static void vtpci_describe_features(struct vtpci_softc *sc, const char *msg, uint64_t features) { device_t dev, child; dev = sc->vtpci_dev; child = sc->vtpci_child_dev; if (device_is_attached(child) && bootverbose == 0) return; virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); } static void vtpci_probe_and_attach_child(struct vtpci_softc *sc) { device_t dev, child; dev = sc->vtpci_dev; child = sc->vtpci_child_dev; if (child == NULL) return; if (device_get_state(child) != DS_NOTPRESENT) return; if (device_probe(child) != 0) return; vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); if (device_attach(child) != 0) { vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); vtpci_reset(sc); vtpci_release_child_resources(sc); /* Reset status for future attempt. */ vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); } else { vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); VIRTIO_ATTACH_COMPLETED(child); } } static int vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) { device_t dev; int nmsix, cnt, required; dev = sc->vtpci_dev; /* Allocate an additional vector for the config changes. */ required = nvectors + 1; nmsix = pci_msix_count(dev); if (nmsix < required) return (1); cnt = required; if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { sc->vtpci_nmsix_resources = required; return (0); } pci_release_msi(dev); return (1); } static int vtpci_alloc_msi(struct vtpci_softc *sc) { device_t dev; int nmsi, cnt, required; dev = sc->vtpci_dev; required = 1; nmsi = pci_msi_count(dev); if (nmsi < required) return (1); cnt = required; if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) return (0); pci_release_msi(dev); return (1); } static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc) { int i, nvectors, error; if (vtpci_disable_msix != 0 || sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) return (ENOTSUP); for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) { if (sc->vtpci_vqs[i].vtv_no_intr == 0) nvectors++; } error = vtpci_alloc_msix(sc, nvectors); if (error) return (error); sc->vtpci_flags |= VTPCI_FLAG_MSIX; return (0); } static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc) { int error; if (vtpci_disable_msix != 0 || sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) return (ENOTSUP); error = vtpci_alloc_msix(sc, 1); if (error) return (error); sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; return (0); } static int vtpci_alloc_intr_msi(struct vtpci_softc *sc) { int error; /* Only BHyVe supports MSI. */ if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI) return (ENOTSUP); error = vtpci_alloc_msi(sc); if (error) return (error); sc->vtpci_flags |= VTPCI_FLAG_MSI; return (0); } static int vtpci_alloc_intr_legacy(struct vtpci_softc *sc) { sc->vtpci_flags |= VTPCI_FLAG_LEGACY; return (0); } static int vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, struct vtpci_interrupt *intr) { struct resource *irq; irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags); if (irq == NULL) return (ENXIO); intr->vti_irq = irq; intr->vti_rid = rid; return (0); } static int vtpci_alloc_intr_resources(struct vtpci_softc *sc) { struct vtpci_interrupt *intr; int i, rid, flags, nvq_intrs, error; rid = 0; flags = RF_ACTIVE; if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) flags |= RF_SHAREABLE; else rid = 1; /* * For legacy and MSI interrupts, this single resource handles all * interrupts. For MSIX, this resource is used for the configuration * changed interrupt. */ intr = &sc->vtpci_device_interrupt; error = vtpci_alloc_interrupt(sc, rid, flags, intr); if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI)) return (error); /* Subtract one for the configuration changed interrupt. */ nvq_intrs = sc->vtpci_nmsix_resources - 1; intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs * sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->vtpci_msix_vq_interrupts == NULL) return (ENOMEM); for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { error = vtpci_alloc_interrupt(sc, rid, flags, intr); if (error) return (error); } return (0); } static int vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type) { struct vtpci_interrupt *intr; int error; intr = &sc->vtpci_device_interrupt; error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL, vtpci_legacy_intr, sc, &intr->vti_handler); return (error); } static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) { struct vtpci_virtqueue *vqx; struct vtpci_interrupt *intr; int i, error; intr = sc->vtpci_msix_vq_interrupts; for (i = 0; i < sc->vtpci_nvqs; i++) { vqx = &sc->vtpci_vqs[i]; if (vqx->vtv_no_intr) continue; error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, &intr->vti_handler); if (error) return (error); intr++; } return (0); } static int vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) { device_t dev; struct vtpci_interrupt *intr; int error; dev = sc->vtpci_dev; intr = &sc->vtpci_device_interrupt; error = bus_setup_intr(dev, intr->vti_irq, type, NULL, vtpci_config_intr, sc, &intr->vti_handler); if (error) return (error); if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { intr = sc->vtpci_msix_vq_interrupts; error = bus_setup_intr(dev, intr->vti_irq, type, vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc, &intr->vti_handler); } else error = vtpci_setup_pervq_msix_interrupts(sc, type); return (error ? error : vtpci_set_host_msix_vectors(sc)); } static int vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type) { int error; type |= INTR_MPSAFE; KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags)); error = vtpci_alloc_intr_resources(sc); if (error) return (error); if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) error = vtpci_setup_legacy_interrupt(sc, type); else if (sc->vtpci_flags & VTPCI_FLAG_MSI) error = vtpci_setup_msi_interrupt(sc, type); else error = vtpci_setup_msix_interrupts(sc, type); return (error); } static int vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, struct vtpci_interrupt *intr) { device_t dev; uint16_t vector; dev = sc->vtpci_dev; if (intr != NULL) { /* Map from guest rid to host vector. */ vector = intr->vti_rid - 1; } else vector = VIRTIO_MSI_NO_VECTOR; vtpci_write_config_2(sc, offset, vector); /* Read vector to determine if the host had sufficient resources. */ if (vtpci_read_config_2(sc, offset) != vector) { device_printf(dev, "insufficient host resources for MSIX interrupts\n"); return (ENODEV); } return (0); } static int vtpci_set_host_msix_vectors(struct vtpci_softc *sc) { struct vtpci_interrupt *intr, *tintr; int idx, offset, error; intr = &sc->vtpci_device_interrupt; offset = VIRTIO_MSI_CONFIG_VECTOR; error = vtpci_register_msix_vector(sc, offset, intr); if (error) return (error); intr = sc->vtpci_msix_vq_interrupts; offset = VIRTIO_MSI_QUEUE_VECTOR; for (idx = 0; idx < sc->vtpci_nvqs; idx++) { vtpci_select_virtqueue(sc, idx); if (sc->vtpci_vqs[idx].vtv_no_intr) tintr = NULL; else tintr = intr; error = vtpci_register_msix_vector(sc, offset, tintr); if (error) break; /* * For shared MSIX, all the virtqueues share the first * interrupt. */ if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) intr++; } return (error); } static int vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx) { struct vtpci_virtqueue *vqx; struct virtqueue *vq; int error; uint16_t size; vqx = &sc->vtpci_vqs[idx]; vq = vqx->vtv_vq; KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); vtpci_select_virtqueue(sc, idx); size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); error = virtqueue_reinit(vq, size); if (error) return (error); vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); return (0); } static void vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr) { device_t dev; dev = sc->vtpci_dev; if (intr->vti_handler != NULL) { bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); intr->vti_handler = NULL; } if (intr->vti_irq != NULL) { bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, intr->vti_irq); intr->vti_irq = NULL; intr->vti_rid = -1; } } static void vtpci_free_interrupts(struct vtpci_softc *sc) { struct vtpci_interrupt *intr; int i, nvq_intrs; vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt); if (sc->vtpci_nmsix_resources != 0) { nvq_intrs = sc->vtpci_nmsix_resources - 1; sc->vtpci_nmsix_resources = 0; intr = sc->vtpci_msix_vq_interrupts; if (intr != NULL) { for (i = 0; i < nvq_intrs; i++, intr++) vtpci_free_interrupt(sc, intr); free(sc->vtpci_msix_vq_interrupts, M_DEVBUF); sc->vtpci_msix_vq_interrupts = NULL; } } if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) pci_release_msi(sc->vtpci_dev); sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; } static void vtpci_free_virtqueues(struct vtpci_softc *sc) { struct vtpci_virtqueue *vqx; int idx; for (idx = 0; idx < sc->vtpci_nvqs; idx++) { vqx = &sc->vtpci_vqs[idx]; vtpci_select_virtqueue(sc, idx); vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); virtqueue_free(vqx->vtv_vq); vqx->vtv_vq = NULL; } free(sc->vtpci_vqs, M_DEVBUF); sc->vtpci_vqs = NULL; sc->vtpci_nvqs = 0; } static void vtpci_release_child_resources(struct vtpci_softc *sc) { vtpci_free_interrupts(sc); vtpci_free_virtqueues(sc); } static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc) { int idx; if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR, VIRTIO_MSI_NO_VECTOR); for (idx = 0; idx < sc->vtpci_nvqs; idx++) { vtpci_select_virtqueue(sc, idx); vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR, VIRTIO_MSI_NO_VECTOR); } } vtpci_free_interrupts(sc); } static void vtpci_reset(struct vtpci_softc *sc) { /* * Setting the status to RESET sets the host device to * the original, uninitialized state. */ vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); } static void vtpci_select_virtqueue(struct vtpci_softc *sc, int idx) { vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); } static void vtpci_legacy_intr(void *xsc) { struct vtpci_softc *sc; struct vtpci_virtqueue *vqx; int i; uint8_t isr; sc = xsc; vqx = &sc->vtpci_vqs[0]; /* Reading the ISR also clears it. */ isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); if (isr & VIRTIO_PCI_ISR_CONFIG) vtpci_config_intr(sc); if (isr & VIRTIO_PCI_ISR_INTR) { for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) virtqueue_intr(vqx->vtv_vq); } } } static int vtpci_vq_shared_intr_filter(void *xsc) { struct vtpci_softc *sc; struct vtpci_virtqueue *vqx; int i, rc; rc = 0; sc = xsc; vqx = &sc->vtpci_vqs[0]; for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) rc |= virtqueue_intr_filter(vqx->vtv_vq); } return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); } static void vtpci_vq_shared_intr(void *xsc) { struct vtpci_softc *sc; struct vtpci_virtqueue *vqx; int i; sc = xsc; vqx = &sc->vtpci_vqs[0]; for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) virtqueue_intr(vqx->vtv_vq); } } static int vtpci_vq_intr_filter(void *xvq) { struct virtqueue *vq; int rc; vq = xvq; rc = virtqueue_intr_filter(vq); return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); } static void vtpci_vq_intr(void *xvq) { struct virtqueue *vq; vq = xvq; virtqueue_intr(vq); } static void vtpci_config_intr(void *xsc) { struct vtpci_softc *sc; device_t child; sc = xsc; child = sc->vtpci_child_dev; if (child != NULL) VIRTIO_CONFIG_CHANGE(child); } Index: head/sys/dev/virtio/virtio.h =================================================================== --- head/sys/dev/virtio/virtio.h (revision 267521) +++ head/sys/dev/virtio/virtio.h (revision 267522) @@ -1,143 +1,144 @@ /*- * Copyright (c) 2014, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _VIRTIO_H_ #define _VIRTIO_H_ #include +#include struct vq_alloc_info; /* * Each virtqueue indirect descriptor list must be physically contiguous. * To allow us to malloc(9) each list individually, limit the number * supported to what will fit in one page. With 4KB pages, this is a limit * of 256 descriptors. If there is ever a need for more, we can switch to * contigmalloc(9) for the larger allocations, similar to what * bus_dmamem_alloc(9) does. * * Note the sizeof(struct vring_desc) is 16 bytes. */ #define VIRTIO_MAX_INDIRECT ((int) (PAGE_SIZE / 16)) /* * VirtIO instance variables indices. */ #define VIRTIO_IVAR_DEVTYPE 1 #define VIRTIO_IVAR_FEATURE_DESC 2 #define VIRTIO_IVAR_VENDOR 3 #define VIRTIO_IVAR_DEVICE 4 #define VIRTIO_IVAR_SUBVENDOR 5 #define VIRTIO_IVAR_SUBDEVICE 6 struct virtio_feature_desc { uint64_t vfd_val; const char *vfd_str; }; const char *virtio_device_name(uint16_t devid); void virtio_describe(device_t dev, const char *msg, uint64_t features, struct virtio_feature_desc *feature_desc); /* * VirtIO Bus Methods. */ void virtio_read_ivar(device_t dev, int ivar, uintptr_t *val); void virtio_write_ivar(device_t dev, int ivar, uintptr_t val); uint64_t virtio_negotiate_features(device_t dev, uint64_t child_features); int virtio_alloc_virtqueues(device_t dev, int flags, int nvqs, struct vq_alloc_info *info); int virtio_setup_intr(device_t dev, enum intr_type type); int virtio_with_feature(device_t dev, uint64_t feature); void virtio_stop(device_t dev); int virtio_reinit(device_t dev, uint64_t features); void virtio_reinit_complete(device_t dev); /* * Read/write a variable amount from the device specific (ie, network) * configuration region. This region is encoded in the same endian as * the guest. */ void virtio_read_device_config(device_t dev, bus_size_t offset, void *dst, int length); void virtio_write_device_config(device_t dev, bus_size_t offset, void *src, int length); /* Inlined device specific read/write functions for common lengths. */ #define VIRTIO_RDWR_DEVICE_CONFIG(size, type) \ static inline type \ __CONCAT(virtio_read_dev_config_,size)(device_t dev, \ bus_size_t offset) \ { \ type val; \ virtio_read_device_config(dev, offset, &val, sizeof(type)); \ return (val); \ } \ \ static inline void \ __CONCAT(virtio_write_dev_config_,size)(device_t dev, \ bus_size_t offset, type val) \ { \ virtio_write_device_config(dev, offset, &val, sizeof(type)); \ } VIRTIO_RDWR_DEVICE_CONFIG(1, uint8_t); VIRTIO_RDWR_DEVICE_CONFIG(2, uint16_t); VIRTIO_RDWR_DEVICE_CONFIG(4, uint32_t); #undef VIRTIO_RDWR_DEVICE_CONFIG #define VIRTIO_READ_IVAR(name, ivar) \ static inline int \ __CONCAT(virtio_get_,name)(device_t dev) \ { \ uintptr_t val; \ virtio_read_ivar(dev, ivar, &val); \ return ((int) val); \ } VIRTIO_READ_IVAR(device_type, VIRTIO_IVAR_DEVTYPE); VIRTIO_READ_IVAR(vendor, VIRTIO_IVAR_VENDOR); VIRTIO_READ_IVAR(device, VIRTIO_IVAR_DEVICE); VIRTIO_READ_IVAR(subvendor, VIRTIO_IVAR_SUBVENDOR); VIRTIO_READ_IVAR(subdevice, VIRTIO_IVAR_SUBDEVICE); #undef VIRTIO_READ_IVAR #define VIRTIO_WRITE_IVAR(name, ivar) \ static inline void \ __CONCAT(virtio_set_,name)(device_t dev, void *val) \ { \ virtio_write_ivar(dev, ivar, (uintptr_t) val); \ } VIRTIO_WRITE_IVAR(feature_desc, VIRTIO_IVAR_FEATURE_DESC); #undef VIRTIO_WRITE_IVAR #endif /* _VIRTIO_H_ */ Index: head/sys/dev/virtio/virtio_config.h =================================================================== --- head/sys/dev/virtio/virtio_config.h (revision 267521) +++ head/sys/dev/virtio/virtio_config.h (revision 267522) @@ -1,61 +1,67 @@ /*- * This header is BSD licensed so anyone can use the definitions to implement * compatible drivers/servers. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of IBM nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _VIRTIO_CONFIG_H_ #define _VIRTIO_CONFIG_H_ /* Status byte for guest to report progress. */ #define VIRTIO_CONFIG_STATUS_RESET 0x00 #define VIRTIO_CONFIG_STATUS_ACK 0x01 #define VIRTIO_CONFIG_STATUS_DRIVER 0x03 #define VIRTIO_CONFIG_STATUS_DRIVER_OK 0x04 #define VIRTIO_CONFIG_STATUS_FAILED 0x80 /* * Generate interrupt when the virtqueue ring is * completely used, even if we've suppressed them. */ #define VIRTIO_F_NOTIFY_ON_EMPTY (1 << 24) +/* Support for indirect buffer descriptors. */ +#define VIRTIO_RING_F_INDIRECT_DESC (1 << 28) + +/* Support to suppress interrupt until specific index is reached. */ +#define VIRTIO_RING_F_EVENT_IDX (1 << 29) + /* * The guest should never negotiate this feature; it * is used to detect faulty drivers. */ #define VIRTIO_F_BAD_FEATURE (1 << 30) /* * Some VirtIO feature bits (currently bits 28 through 31) are * reserved for the transport being used (eg. virtio_ring), the * rest are per-device feature bits. */ #define VIRTIO_TRANSPORT_F_START 28 #define VIRTIO_TRANSPORT_F_END 32 #endif /* _VIRTIO_CONFIG_H_ */ Index: head/sys/dev/virtio/virtqueue.c =================================================================== --- head/sys/dev/virtio/virtqueue.c (revision 267521) +++ head/sys/dev/virtio/virtqueue.c (revision 267522) @@ -1,820 +1,819 @@ /*- * Copyright (c) 2011, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * Implements the virtqueue interface as basically described * in the original VirtIO paper. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include #include "virtio_bus_if.h" struct virtqueue { device_t vq_dev; char vq_name[VIRTQUEUE_MAX_NAME_SZ]; uint16_t vq_queue_index; uint16_t vq_nentries; uint32_t vq_flags; #define VIRTQUEUE_FLAG_INDIRECT 0x0001 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002 int vq_alignment; int vq_ring_size; void *vq_ring_mem; int vq_max_indirect_size; int vq_indirect_mem_size; virtqueue_intr_t *vq_intrhand; void *vq_intrhand_arg; struct vring vq_ring; uint16_t vq_free_cnt; uint16_t vq_queued_cnt; /* * Head of the free chain in the descriptor table. If * there are no free descriptors, this will be set to * VQ_RING_DESC_CHAIN_END. */ uint16_t vq_desc_head_idx; /* * Last consumed descriptor in the used table, * trails vq_ring.used->idx. */ uint16_t vq_used_cons_idx; struct vq_desc_extra { void *cookie; struct vring_desc *indirect; vm_paddr_t indirect_paddr; uint16_t ndescs; } vq_descx[0]; }; /* * The maximum virtqueue size is 2^15. Use that value as the end of * descriptor chain terminator since it will never be a valid index * in the descriptor table. This is used to verify we are correctly * handling vq_free_cnt. */ #define VQ_RING_DESC_CHAIN_END 32768 #define VQASSERT(_vq, _exp, _msg, ...) \ KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \ ##__VA_ARGS__)) #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \ VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \ "invalid ring index: %d, max: %d", (_idx), \ (_vq)->vq_nentries) #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \ VQASSERT((_vq), (_vq)->vq_desc_head_idx == \ VQ_RING_DESC_CHAIN_END, "full ring terminated " \ "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx) static int virtqueue_init_indirect(struct virtqueue *vq, int); static void virtqueue_free_indirect(struct virtqueue *vq); static void virtqueue_init_indirect_list(struct virtqueue *, struct vring_desc *); static void vq_ring_init(struct virtqueue *); static void vq_ring_update_avail(struct virtqueue *, uint16_t); static uint16_t vq_ring_enqueue_segments(struct virtqueue *, struct vring_desc *, uint16_t, struct sglist *, int, int); static int vq_ring_use_indirect(struct virtqueue *, int); static void vq_ring_enqueue_indirect(struct virtqueue *, void *, struct sglist *, int, int); static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t); static int vq_ring_must_notify_host(struct virtqueue *); static void vq_ring_notify_host(struct virtqueue *); static void vq_ring_free_chain(struct virtqueue *, uint16_t); uint64_t virtqueue_filter_features(uint64_t features) { uint64_t mask; mask = (1 << VIRTIO_TRANSPORT_F_START) - 1; mask |= VIRTIO_RING_F_INDIRECT_DESC; mask |= VIRTIO_RING_F_EVENT_IDX; return (features & mask); } int virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align, vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp) { struct virtqueue *vq; int error; *vqp = NULL; error = 0; if (size == 0) { device_printf(dev, "virtqueue %d (%s) does not exist (size is zero)\n", queue, info->vqai_name); return (ENODEV); } else if (!powerof2(size)) { device_printf(dev, "virtqueue %d (%s) size is not a power of 2: %d\n", queue, info->vqai_name, size); return (ENXIO); } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) { device_printf(dev, "virtqueue %d (%s) requested too many " "indirect descriptors: %d, max %d\n", queue, info->vqai_name, info->vqai_maxindirsz, VIRTIO_MAX_INDIRECT); return (EINVAL); } vq = malloc(sizeof(struct virtqueue) + size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO); if (vq == NULL) { device_printf(dev, "cannot allocate virtqueue\n"); return (ENOMEM); } vq->vq_dev = dev; strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name)); vq->vq_queue_index = queue; vq->vq_alignment = align; vq->vq_nentries = size; vq->vq_free_cnt = size; vq->vq_intrhand = info->vqai_intr; vq->vq_intrhand_arg = info->vqai_intr_arg; if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0) vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX; if (info->vqai_maxindirsz > 1) { error = virtqueue_init_indirect(vq, info->vqai_maxindirsz); if (error) goto fail; } vq->vq_ring_size = round_page(vring_size(size, align)); vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF, M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0); if (vq->vq_ring_mem == NULL) { device_printf(dev, "cannot allocate memory for virtqueue ring\n"); error = ENOMEM; goto fail; } vq_ring_init(vq); virtqueue_disable_intr(vq); *vqp = vq; fail: if (error) virtqueue_free(vq); return (error); } static int virtqueue_init_indirect(struct virtqueue *vq, int indirect_size) { device_t dev; struct vq_desc_extra *dxp; int i, size; dev = vq->vq_dev; if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) { /* * Indirect descriptors requested by the driver but not * negotiated. Return zero to keep the initialization * going: we'll run fine without. */ if (bootverbose) device_printf(dev, "virtqueue %d (%s) requested " "indirect descriptors but not negotiated\n", vq->vq_queue_index, vq->vq_name); return (0); } size = indirect_size * sizeof(struct vring_desc); vq->vq_max_indirect_size = indirect_size; vq->vq_indirect_mem_size = size; vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT; for (i = 0; i < vq->vq_nentries; i++) { dxp = &vq->vq_descx[i]; dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT); if (dxp->indirect == NULL) { device_printf(dev, "cannot allocate indirect list\n"); return (ENOMEM); } dxp->indirect_paddr = vtophys(dxp->indirect); virtqueue_init_indirect_list(vq, dxp->indirect); } return (0); } static void virtqueue_free_indirect(struct virtqueue *vq) { struct vq_desc_extra *dxp; int i; for (i = 0; i < vq->vq_nentries; i++) { dxp = &vq->vq_descx[i]; if (dxp->indirect == NULL) break; free(dxp->indirect, M_DEVBUF); dxp->indirect = NULL; dxp->indirect_paddr = 0; } vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT; vq->vq_indirect_mem_size = 0; } static void virtqueue_init_indirect_list(struct virtqueue *vq, struct vring_desc *indirect) { int i; bzero(indirect, vq->vq_indirect_mem_size); for (i = 0; i < vq->vq_max_indirect_size - 1; i++) indirect[i].next = i + 1; indirect[i].next = VQ_RING_DESC_CHAIN_END; } int virtqueue_reinit(struct virtqueue *vq, uint16_t size) { struct vq_desc_extra *dxp; int i; if (vq->vq_nentries != size) { device_printf(vq->vq_dev, "%s: '%s' changed size; old=%hu, new=%hu\n", __func__, vq->vq_name, vq->vq_nentries, size); return (EINVAL); } /* Warn if the virtqueue was not properly cleaned up. */ if (vq->vq_free_cnt != vq->vq_nentries) { device_printf(vq->vq_dev, "%s: warning '%s' virtqueue not empty, " "leaking %d entries\n", __func__, vq->vq_name, vq->vq_nentries - vq->vq_free_cnt); } vq->vq_desc_head_idx = 0; vq->vq_used_cons_idx = 0; vq->vq_queued_cnt = 0; vq->vq_free_cnt = vq->vq_nentries; /* To be safe, reset all our allocated memory. */ bzero(vq->vq_ring_mem, vq->vq_ring_size); for (i = 0; i < vq->vq_nentries; i++) { dxp = &vq->vq_descx[i]; dxp->cookie = NULL; dxp->ndescs = 0; if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) virtqueue_init_indirect_list(vq, dxp->indirect); } vq_ring_init(vq); virtqueue_disable_intr(vq); return (0); } void virtqueue_free(struct virtqueue *vq) { if (vq->vq_free_cnt != vq->vq_nentries) { device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, " "leaking %d entries\n", vq->vq_name, vq->vq_nentries - vq->vq_free_cnt); } if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) virtqueue_free_indirect(vq); if (vq->vq_ring_mem != NULL) { contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF); vq->vq_ring_size = 0; vq->vq_ring_mem = NULL; } free(vq, M_DEVBUF); } vm_paddr_t virtqueue_paddr(struct virtqueue *vq) { return (vtophys(vq->vq_ring_mem)); } int virtqueue_size(struct virtqueue *vq) { return (vq->vq_nentries); } int virtqueue_empty(struct virtqueue *vq) { return (vq->vq_nentries == vq->vq_free_cnt); } int virtqueue_full(struct virtqueue *vq) { return (vq->vq_free_cnt == 0); } void virtqueue_notify(struct virtqueue *vq) { /* Ensure updated avail->idx is visible to host. */ mb(); if (vq_ring_must_notify_host(vq)) vq_ring_notify_host(vq); vq->vq_queued_cnt = 0; } int virtqueue_nused(struct virtqueue *vq) { uint16_t used_idx, nused; used_idx = vq->vq_ring.used->idx; nused = (uint16_t)(used_idx - vq->vq_used_cons_idx); VQASSERT(vq, nused <= vq->vq_nentries, "used more than available"); return (nused); } int virtqueue_intr_filter(struct virtqueue *vq) { if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) return (0); virtqueue_disable_intr(vq); return (1); } void virtqueue_intr(struct virtqueue *vq) { vq->vq_intrhand(vq->vq_intrhand_arg); } int virtqueue_enable_intr(struct virtqueue *vq) { return (vq_ring_enable_interrupt(vq, 0)); } int virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint) { uint16_t ndesc, avail_idx; avail_idx = vq->vq_ring.avail->idx; ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx); switch (hint) { case VQ_POSTPONE_SHORT: ndesc = ndesc / 4; break; case VQ_POSTPONE_LONG: ndesc = (ndesc * 3) / 4; break; case VQ_POSTPONE_EMPTIED: break; } return (vq_ring_enable_interrupt(vq, ndesc)); } /* * Note this is only considered a hint to the host. */ void virtqueue_disable_intr(struct virtqueue *vq) { if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx - vq->vq_nentries - 1; } else vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; } int virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg, int readable, int writable) { struct vq_desc_extra *dxp; int needed; uint16_t head_idx, idx; needed = readable + writable; VQASSERT(vq, cookie != NULL, "enqueuing with no cookie"); VQASSERT(vq, needed == sg->sg_nseg, "segment count mismatch, %d, %d", needed, sg->sg_nseg); VQASSERT(vq, needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size, "too many segments to enqueue: %d, %d/%d", needed, vq->vq_nentries, vq->vq_max_indirect_size); if (needed < 1) return (EINVAL); if (vq->vq_free_cnt == 0) return (ENOSPC); if (vq_ring_use_indirect(vq, needed)) { vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable); return (0); } else if (vq->vq_free_cnt < needed) return (EMSGSIZE); head_idx = vq->vq_desc_head_idx; VQ_RING_ASSERT_VALID_IDX(vq, head_idx); dxp = &vq->vq_descx[head_idx]; VQASSERT(vq, dxp->cookie == NULL, "cookie already exists for index %d", head_idx); dxp->cookie = cookie; dxp->ndescs = needed; idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx, sg, readable, writable); vq->vq_desc_head_idx = idx; vq->vq_free_cnt -= needed; if (vq->vq_free_cnt == 0) VQ_RING_ASSERT_CHAIN_TERM(vq); else VQ_RING_ASSERT_VALID_IDX(vq, idx); vq_ring_update_avail(vq, head_idx); return (0); } void * virtqueue_dequeue(struct virtqueue *vq, uint32_t *len) { struct vring_used_elem *uep; void *cookie; uint16_t used_idx, desc_idx; if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) return (NULL); used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1); uep = &vq->vq_ring.used->ring[used_idx]; rmb(); desc_idx = (uint16_t) uep->id; if (len != NULL) *len = uep->len; vq_ring_free_chain(vq, desc_idx); cookie = vq->vq_descx[desc_idx].cookie; VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx); vq->vq_descx[desc_idx].cookie = NULL; return (cookie); } void * virtqueue_poll(struct virtqueue *vq, uint32_t *len) { void *cookie; while ((cookie = virtqueue_dequeue(vq, len)) == NULL) cpu_spinwait(); return (cookie); } void * virtqueue_drain(struct virtqueue *vq, int *last) { void *cookie; int idx; cookie = NULL; idx = *last; while (idx < vq->vq_nentries && cookie == NULL) { if ((cookie = vq->vq_descx[idx].cookie) != NULL) { vq->vq_descx[idx].cookie = NULL; /* Free chain to keep free count consistent. */ vq_ring_free_chain(vq, idx); } idx++; } *last = idx; return (cookie); } void virtqueue_dump(struct virtqueue *vq) { if (vq == NULL) return; printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; " "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; " "used.idx=%d; avail.flags=0x%x; used.flags=0x%x\n", vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx, vq->vq_ring.avail->idx, vq->vq_used_cons_idx, vq->vq_ring.used->idx, vq->vq_ring.avail->flags, vq->vq_ring.used->flags); } static void vq_ring_init(struct virtqueue *vq) { struct vring *vr; char *ring_mem; int i, size; ring_mem = vq->vq_ring_mem; size = vq->vq_nentries; vr = &vq->vq_ring; vring_init(vr, size, ring_mem, vq->vq_alignment); for (i = 0; i < size - 1; i++) vr->desc[i].next = i + 1; vr->desc[i].next = VQ_RING_DESC_CHAIN_END; } static void vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx) { uint16_t avail_idx; /* * Place the head of the descriptor chain into the next slot and make * it usable to the host. The chain is made available now rather than * deferring to virtqueue_notify() in the hopes that if the host is * currently running on another CPU, we can keep it processing the new * descriptor. */ avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1); vq->vq_ring.avail->ring[avail_idx] = desc_idx; wmb(); vq->vq_ring.avail->idx++; /* Keep pending count until virtqueue_notify(). */ vq->vq_queued_cnt++; } static uint16_t vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc, uint16_t head_idx, struct sglist *sg, int readable, int writable) { struct sglist_seg *seg; struct vring_desc *dp; int i, needed; uint16_t idx; needed = readable + writable; for (i = 0, idx = head_idx, seg = sg->sg_segs; i < needed; i++, idx = dp->next, seg++) { VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END, "premature end of free desc chain"); dp = &desc[idx]; dp->addr = seg->ss_paddr; dp->len = seg->ss_len; dp->flags = 0; if (i < needed - 1) dp->flags |= VRING_DESC_F_NEXT; if (i >= readable) dp->flags |= VRING_DESC_F_WRITE; } return (idx); } static int vq_ring_use_indirect(struct virtqueue *vq, int needed) { if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0) return (0); if (vq->vq_max_indirect_size < needed) return (0); if (needed < 2) return (0); return (1); } static void vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie, struct sglist *sg, int readable, int writable) { struct vring_desc *dp; struct vq_desc_extra *dxp; int needed; uint16_t head_idx; needed = readable + writable; VQASSERT(vq, needed <= vq->vq_max_indirect_size, "enqueuing too many indirect descriptors"); head_idx = vq->vq_desc_head_idx; VQ_RING_ASSERT_VALID_IDX(vq, head_idx); dp = &vq->vq_ring.desc[head_idx]; dxp = &vq->vq_descx[head_idx]; VQASSERT(vq, dxp->cookie == NULL, "cookie already exists for index %d", head_idx); dxp->cookie = cookie; dxp->ndescs = 1; dp->addr = dxp->indirect_paddr; dp->len = needed * sizeof(struct vring_desc); dp->flags = VRING_DESC_F_INDIRECT; vq_ring_enqueue_segments(vq, dxp->indirect, 0, sg, readable, writable); vq->vq_desc_head_idx = dp->next; vq->vq_free_cnt--; if (vq->vq_free_cnt == 0) VQ_RING_ASSERT_CHAIN_TERM(vq); else VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx); vq_ring_update_avail(vq, head_idx); } static int vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc) { /* * Enable interrupts, making sure we get the latest index of * what's already been consumed. */ if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc; else vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; mb(); /* * Enough items may have already been consumed to meet our threshold * since we last checked. Let our caller know so it processes the new * entries. */ if (virtqueue_nused(vq) > ndesc) return (1); return (0); } static int vq_ring_must_notify_host(struct virtqueue *vq) { uint16_t new_idx, prev_idx, event_idx; if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { new_idx = vq->vq_ring.avail->idx; prev_idx = new_idx - vq->vq_queued_cnt; event_idx = vring_avail_event(&vq->vq_ring); return (vring_need_event(event_idx, new_idx, prev_idx) != 0); } return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0); } static void vq_ring_notify_host(struct virtqueue *vq) { VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index); } static void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) { struct vring_desc *dp; struct vq_desc_extra *dxp; VQ_RING_ASSERT_VALID_IDX(vq, desc_idx); dp = &vq->vq_ring.desc[desc_idx]; dxp = &vq->vq_descx[desc_idx]; if (vq->vq_free_cnt == 0) VQ_RING_ASSERT_CHAIN_TERM(vq); vq->vq_free_cnt += dxp->ndescs; dxp->ndescs--; if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { while (dp->flags & VRING_DESC_F_NEXT) { VQ_RING_ASSERT_VALID_IDX(vq, dp->next); dp = &vq->vq_ring.desc[dp->next]; dxp->ndescs--; } } VQASSERT(vq, dxp->ndescs == 0, "failed to free entire desc chain, remaining: %d", dxp->ndescs); /* * We must append the existing free chain, if any, to the end of * newly freed chain. If the virtqueue was completely used, then * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). */ dp->next = vq->vq_desc_head_idx; vq->vq_desc_head_idx = desc_idx; } Index: head/sys/dev/virtio/virtqueue.h =================================================================== --- head/sys/dev/virtio/virtqueue.h (revision 267521) +++ head/sys/dev/virtio/virtqueue.h (revision 267522) @@ -1,104 +1,98 @@ /*- * Copyright (c) 2011, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _VIRTIO_VIRTQUEUE_H #define _VIRTIO_VIRTQUEUE_H struct virtqueue; struct sglist; -/* Support for indirect buffer descriptors. */ -#define VIRTIO_RING_F_INDIRECT_DESC (1 << 28) - -/* Support to suppress interrupt until specific index is reached. */ -#define VIRTIO_RING_F_EVENT_IDX (1 << 29) - /* Device callback for a virtqueue interrupt. */ typedef void virtqueue_intr_t(void *); /* * Hint on how long the next interrupt should be postponed. This is * only used when the EVENT_IDX feature is negotiated. */ typedef enum { VQ_POSTPONE_SHORT, VQ_POSTPONE_LONG, VQ_POSTPONE_EMPTIED /* Until all available desc are used. */ } vq_postpone_t; #define VIRTQUEUE_MAX_NAME_SZ 32 /* One for each virtqueue the device wishes to allocate. */ struct vq_alloc_info { char vqai_name[VIRTQUEUE_MAX_NAME_SZ]; int vqai_maxindirsz; virtqueue_intr_t *vqai_intr; void *vqai_intr_arg; struct virtqueue **vqai_vq; }; #define VQ_ALLOC_INFO_INIT(_i,_nsegs,_intr,_arg,_vqp,_str,...) do { \ snprintf((_i)->vqai_name, VIRTQUEUE_MAX_NAME_SZ, _str, \ ##__VA_ARGS__); \ (_i)->vqai_maxindirsz = (_nsegs); \ (_i)->vqai_intr = (_intr); \ (_i)->vqai_intr_arg = (_arg); \ (_i)->vqai_vq = (_vqp); \ } while (0) uint64_t virtqueue_filter_features(uint64_t features); int virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align, vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp); void *virtqueue_drain(struct virtqueue *vq, int *last); void virtqueue_free(struct virtqueue *vq); int virtqueue_reinit(struct virtqueue *vq, uint16_t size); int virtqueue_intr_filter(struct virtqueue *vq); void virtqueue_intr(struct virtqueue *vq); int virtqueue_enable_intr(struct virtqueue *vq); int virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint); void virtqueue_disable_intr(struct virtqueue *vq); /* Get physical address of the virtqueue ring. */ vm_paddr_t virtqueue_paddr(struct virtqueue *vq); int virtqueue_full(struct virtqueue *vq); int virtqueue_empty(struct virtqueue *vq); int virtqueue_size(struct virtqueue *vq); int virtqueue_nused(struct virtqueue *vq); void virtqueue_notify(struct virtqueue *vq); void virtqueue_dump(struct virtqueue *vq); int virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg, int readable, int writable); void *virtqueue_dequeue(struct virtqueue *vq, uint32_t *len); void *virtqueue_poll(struct virtqueue *vq, uint32_t *len); #endif /* _VIRTIO_VIRTQUEUE_H */