diff --git a/sys/dev/virtio/balloon/virtio_balloon.c b/sys/dev/virtio/balloon/virtio_balloon.c index c5d0e54dbee4..e1c373291d0b 100644 --- a/sys/dev/virtio/balloon/virtio_balloon.c +++ b/sys/dev/virtio/balloon/virtio_balloon.c @@ -1,589 +1,589 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for VirtIO memory balloon devices. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_if.h" struct vtballoon_softc { device_t vtballoon_dev; struct mtx vtballoon_mtx; uint64_t vtballoon_features; uint32_t vtballoon_flags; #define VTBALLOON_FLAG_DETACH 0x01 struct virtqueue *vtballoon_inflate_vq; struct virtqueue *vtballoon_deflate_vq; uint32_t vtballoon_desired_npages; uint32_t vtballoon_current_npages; TAILQ_HEAD(,vm_page) vtballoon_pages; struct thread *vtballoon_td; uint32_t *vtballoon_page_frames; int vtballoon_timeout; }; static struct virtio_feature_desc vtballoon_feature_desc[] = { { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" }, { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" }, { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" }, { 0, NULL } }; static int vtballoon_probe(device_t); static int vtballoon_attach(device_t); static int vtballoon_detach(device_t); static int vtballoon_config_change(device_t); static int vtballoon_negotiate_features(struct vtballoon_softc *); static int vtballoon_setup_features(struct vtballoon_softc *); static int vtballoon_alloc_virtqueues(struct vtballoon_softc *); static void vtballoon_vq_intr(void *); static void vtballoon_inflate(struct vtballoon_softc *, int); static void vtballoon_deflate(struct vtballoon_softc *, int); static void vtballoon_send_page_frames(struct vtballoon_softc *, struct virtqueue *, int); static void vtballoon_pop(struct vtballoon_softc *); static void vtballoon_stop(struct vtballoon_softc *); static vm_page_t vtballoon_alloc_page(struct vtballoon_softc *); static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t); static int vtballoon_sleep(struct vtballoon_softc *); static void vtballoon_thread(void *); static void vtballoon_setup_sysctl(struct vtballoon_softc *); #define vtballoon_modern(_sc) \ (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0) /* Features desired/implemented by this driver. */ #define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST /* Timeout between retries when the balloon needs inflating. */ #define VTBALLOON_LOWMEM_TIMEOUT hz /* * Maximum number of pages we'll request to inflate or deflate * the balloon in one virtqueue request. Both Linux and NetBSD * have settled on 256, doing up to 1MB at a time. */ #define VTBALLOON_PAGES_PER_REQUEST 256 /* Must be able to fix all pages frames in one page (segment). */ CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE); #define VTBALLOON_MTX(_sc) &(_sc)->vtballoon_mtx #define VTBALLOON_LOCK_INIT(_sc, _name) mtx_init(VTBALLOON_MTX((_sc)), _name, \ "VirtIO Balloon Lock", MTX_DEF) #define VTBALLOON_LOCK(_sc) mtx_lock(VTBALLOON_MTX((_sc))) #define VTBALLOON_UNLOCK(_sc) mtx_unlock(VTBALLOON_MTX((_sc))) #define VTBALLOON_LOCK_DESTROY(_sc) mtx_destroy(VTBALLOON_MTX((_sc))) static device_method_t vtballoon_methods[] = { /* Device methods. */ DEVMETHOD(device_probe, vtballoon_probe), DEVMETHOD(device_attach, vtballoon_attach), DEVMETHOD(device_detach, vtballoon_detach), /* VirtIO methods. */ DEVMETHOD(virtio_config_change, vtballoon_config_change), DEVMETHOD_END }; static driver_t vtballoon_driver = { "vtballoon", vtballoon_methods, sizeof(struct vtballoon_softc) }; VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver, 0, 0); MODULE_VERSION(virtio_balloon, 1); MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1); VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON, "VirtIO Balloon Adapter"); static int vtballoon_probe(device_t dev) { return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon)); } static int vtballoon_attach(device_t dev) { struct vtballoon_softc *sc; int error; sc = device_get_softc(dev); sc->vtballoon_dev = dev; virtio_set_feature_desc(dev, vtballoon_feature_desc); VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev)); TAILQ_INIT(&sc->vtballoon_pages); vtballoon_setup_sysctl(sc); error = vtballoon_setup_features(sc); if (error) { device_printf(dev, "cannot setup features\n"); goto fail; } sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->vtballoon_page_frames == NULL) { error = ENOMEM; device_printf(dev, "cannot allocate page frame request array\n"); goto fail; } error = vtballoon_alloc_virtqueues(sc); if (error) { device_printf(dev, "cannot allocate virtqueues\n"); goto fail; } error = virtio_setup_intr(dev, INTR_TYPE_MISC); if (error) { device_printf(dev, "cannot setup virtqueue interrupts\n"); goto fail; } error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td, 0, 0, "virtio_balloon"); if (error) { device_printf(dev, "cannot create balloon kthread\n"); goto fail; } virtqueue_enable_intr(sc->vtballoon_inflate_vq); virtqueue_enable_intr(sc->vtballoon_deflate_vq); fail: if (error) vtballoon_detach(dev); return (error); } static int vtballoon_detach(device_t dev) { struct vtballoon_softc *sc; sc = device_get_softc(dev); if (sc->vtballoon_td != NULL) { VTBALLOON_LOCK(sc); sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH; wakeup_one(sc); msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0); VTBALLOON_UNLOCK(sc); sc->vtballoon_td = NULL; } if (device_is_attached(dev)) { vtballoon_pop(sc); vtballoon_stop(sc); } if (sc->vtballoon_page_frames != NULL) { free(sc->vtballoon_page_frames, M_DEVBUF); sc->vtballoon_page_frames = NULL; } VTBALLOON_LOCK_DESTROY(sc); return (0); } static int vtballoon_config_change(device_t dev) { struct vtballoon_softc *sc; sc = device_get_softc(dev); VTBALLOON_LOCK(sc); wakeup_one(sc); VTBALLOON_UNLOCK(sc); return (1); } static int vtballoon_negotiate_features(struct vtballoon_softc *sc) { device_t dev; uint64_t features; dev = sc->vtballoon_dev; features = VTBALLOON_FEATURES; sc->vtballoon_features = virtio_negotiate_features(dev, features); return (virtio_finalize_features(dev)); } static int vtballoon_setup_features(struct vtballoon_softc *sc) { int error; error = vtballoon_negotiate_features(sc); if (error) return (error); return (0); } static int vtballoon_alloc_virtqueues(struct vtballoon_softc *sc) { device_t dev; struct vq_alloc_info vq_info[2]; int nvqs; dev = sc->vtballoon_dev; nvqs = 2; VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc, &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev)); VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc, &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev)); - return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); + return (virtio_alloc_virtqueues(dev, nvqs, vq_info)); } static void vtballoon_vq_intr(void *xsc) { struct vtballoon_softc *sc; sc = xsc; VTBALLOON_LOCK(sc); wakeup_one(sc); VTBALLOON_UNLOCK(sc); } static void vtballoon_inflate(struct vtballoon_softc *sc, int npages) { struct virtqueue *vq; vm_page_t m; int i; vq = sc->vtballoon_inflate_vq; if (npages > VTBALLOON_PAGES_PER_REQUEST) npages = VTBALLOON_PAGES_PER_REQUEST; for (i = 0; i < npages; i++) { if ((m = vtballoon_alloc_page(sc)) == NULL) { sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT; break; } sc->vtballoon_page_frames[i] = VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; KASSERT(m->a.queue == PQ_NONE, ("%s: allocated page %p on queue", __func__, m)); TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q); } if (i > 0) vtballoon_send_page_frames(sc, vq, i); } static void vtballoon_deflate(struct vtballoon_softc *sc, int npages) { TAILQ_HEAD(, vm_page) free_pages; struct virtqueue *vq; vm_page_t m; int i; vq = sc->vtballoon_deflate_vq; TAILQ_INIT(&free_pages); if (npages > VTBALLOON_PAGES_PER_REQUEST) npages = VTBALLOON_PAGES_PER_REQUEST; for (i = 0; i < npages; i++) { m = TAILQ_FIRST(&sc->vtballoon_pages); KASSERT(m != NULL, ("%s: no more pages to deflate", __func__)); sc->vtballoon_page_frames[i] = VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q); TAILQ_INSERT_TAIL(&free_pages, m, plinks.q); } if (i > 0) { /* Always tell host first before freeing the pages. */ vtballoon_send_page_frames(sc, vq, i); while ((m = TAILQ_FIRST(&free_pages)) != NULL) { TAILQ_REMOVE(&free_pages, m, plinks.q); vtballoon_free_page(sc, m); } } KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) && sc->vtballoon_current_npages == 0) || (!TAILQ_EMPTY(&sc->vtballoon_pages) && sc->vtballoon_current_npages != 0), ("%s: bogus page count %d", __func__, sc->vtballoon_current_npages)); } static void vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq, int npages) { struct sglist sg; struct sglist_seg segs[1]; void *c; int error __diagused; sglist_init(&sg, 1, segs); error = sglist_append(&sg, sc->vtballoon_page_frames, npages * sizeof(uint32_t)); KASSERT(error == 0, ("error adding page frames to sglist")); error = virtqueue_enqueue(vq, vq, &sg, 1, 0); KASSERT(error == 0, ("error enqueuing page frames to virtqueue")); virtqueue_notify(vq); /* * Inflate and deflate operations are done synchronously. The * interrupt handler will wake us up. */ VTBALLOON_LOCK(sc); while ((c = virtqueue_dequeue(vq, NULL)) == NULL) msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0); VTBALLOON_UNLOCK(sc); KASSERT(c == vq, ("unexpected balloon operation response")); } static void vtballoon_pop(struct vtballoon_softc *sc) { while (!TAILQ_EMPTY(&sc->vtballoon_pages)) vtballoon_deflate(sc, sc->vtballoon_current_npages); } static void vtballoon_stop(struct vtballoon_softc *sc) { virtqueue_disable_intr(sc->vtballoon_inflate_vq); virtqueue_disable_intr(sc->vtballoon_deflate_vq); virtio_stop(sc->vtballoon_dev); } static vm_page_t vtballoon_alloc_page(struct vtballoon_softc *sc) { vm_page_t m; m = vm_page_alloc_noobj(VM_ALLOC_NODUMP); if (m != NULL) sc->vtballoon_current_npages++; return (m); } static void vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m) { vm_page_free(m); sc->vtballoon_current_npages--; } static uint32_t vtballoon_desired_size(struct vtballoon_softc *sc) { uint32_t desired; desired = virtio_read_dev_config_4(sc->vtballoon_dev, offsetof(struct virtio_balloon_config, num_pages)); if (vtballoon_modern(sc)) return (desired); else return (le32toh(desired)); } static void vtballoon_update_size(struct vtballoon_softc *sc) { uint32_t npages; npages = sc->vtballoon_current_npages; if (!vtballoon_modern(sc)) npages = htole32(npages); virtio_write_dev_config_4(sc->vtballoon_dev, offsetof(struct virtio_balloon_config, actual), npages); } static int vtballoon_sleep(struct vtballoon_softc *sc) { int rc, timeout; uint32_t current, desired; rc = 0; current = sc->vtballoon_current_npages; VTBALLOON_LOCK(sc); for (;;) { if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) { rc = 1; break; } desired = vtballoon_desired_size(sc); sc->vtballoon_desired_npages = desired; /* * If given, use non-zero timeout on the first time through * the loop. On subsequent times, timeout will be zero so * we will reevaluate the desired size of the balloon and * break out to retry if needed. */ timeout = sc->vtballoon_timeout; sc->vtballoon_timeout = 0; if (current > desired) break; if (current < desired && timeout == 0) break; msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout); } VTBALLOON_UNLOCK(sc); return (rc); } static void vtballoon_thread(void *xsc) { struct vtballoon_softc *sc; uint32_t current, desired; sc = xsc; for (;;) { if (vtballoon_sleep(sc) != 0) break; current = sc->vtballoon_current_npages; desired = sc->vtballoon_desired_npages; if (desired != current) { if (desired > current) vtballoon_inflate(sc, desired - current); else vtballoon_deflate(sc, current - desired); vtballoon_update_size(sc); } } kthread_exit(); } static void vtballoon_setup_sysctl(struct vtballoon_softc *sc) { device_t dev; struct sysctl_ctx_list *ctx; struct sysctl_oid *tree; struct sysctl_oid_list *child; dev = sc->vtballoon_dev; ctx = device_get_sysctl_ctx(dev); tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired", CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t), "Desired balloon size in pages"); SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current", CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t), "Current balloon size in pages"); } diff --git a/sys/dev/virtio/block/virtio_blk.c b/sys/dev/virtio/block/virtio_blk.c index eb693c7f2f51..43952a93dc75 100644 --- a/sys/dev/virtio/block/virtio_blk.c +++ b/sys/dev/virtio/block/virtio_blk.c @@ -1,1662 +1,1662 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for VirtIO block devices. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_if.h" struct vtblk_request { struct vtblk_softc *vbr_sc; bus_dmamap_t vbr_mapp; /* Fields after this point are zeroed for each request. */ struct virtio_blk_outhdr vbr_hdr; struct bio *vbr_bp; uint8_t vbr_ack; uint8_t vbr_requeue_on_error; uint8_t vbr_busdma_wait; int vbr_error; TAILQ_ENTRY(vtblk_request) vbr_link; }; enum vtblk_cache_mode { VTBLK_CACHE_WRITETHROUGH, VTBLK_CACHE_WRITEBACK, VTBLK_CACHE_MAX }; struct vtblk_softc { device_t vtblk_dev; struct mtx vtblk_mtx; uint64_t vtblk_features; uint32_t vtblk_flags; #define VTBLK_FLAG_INDIRECT 0x0001 #define VTBLK_FLAG_DETACH 0x0002 #define VTBLK_FLAG_SUSPEND 0x0004 #define VTBLK_FLAG_BARRIER 0x0008 #define VTBLK_FLAG_WCE_CONFIG 0x0010 #define VTBLK_FLAG_BUSDMA_WAIT 0x0020 #define VTBLK_FLAG_BUSDMA_ALIGN 0x0040 struct virtqueue *vtblk_vq; struct sglist *vtblk_sglist; bus_dma_tag_t vtblk_dmat; struct disk *vtblk_disk; struct bio_queue_head vtblk_bioq; TAILQ_HEAD(, vtblk_request) vtblk_req_free; TAILQ_HEAD(, vtblk_request) vtblk_req_ready; struct vtblk_request *vtblk_req_ordered; int vtblk_max_nsegs; int vtblk_request_count; enum vtblk_cache_mode vtblk_write_cache; struct bio_queue vtblk_dump_queue; struct vtblk_request vtblk_dump_request; }; static struct virtio_feature_desc vtblk_feature_desc[] = { { VIRTIO_BLK_F_BARRIER, "HostBarrier" }, { VIRTIO_BLK_F_SIZE_MAX, "MaxSegSize" }, { VIRTIO_BLK_F_SEG_MAX, "MaxNumSegs" }, { VIRTIO_BLK_F_GEOMETRY, "DiskGeometry" }, { VIRTIO_BLK_F_RO, "ReadOnly" }, { VIRTIO_BLK_F_BLK_SIZE, "BlockSize" }, { VIRTIO_BLK_F_SCSI, "SCSICmds" }, { VIRTIO_BLK_F_FLUSH, "FlushCmd" }, { VIRTIO_BLK_F_TOPOLOGY, "Topology" }, { VIRTIO_BLK_F_CONFIG_WCE, "ConfigWCE" }, { VIRTIO_BLK_F_MQ, "Multiqueue" }, { VIRTIO_BLK_F_DISCARD, "Discard" }, { VIRTIO_BLK_F_WRITE_ZEROES, "WriteZeros" }, { 0, NULL } }; static int vtblk_modevent(module_t, int, void *); static int vtblk_probe(device_t); static int vtblk_attach(device_t); static int vtblk_detach(device_t); static int vtblk_suspend(device_t); static int vtblk_resume(device_t); static int vtblk_shutdown(device_t); static int vtblk_attach_completed(device_t); static int vtblk_config_change(device_t); static int vtblk_open(struct disk *); static int vtblk_close(struct disk *); static int vtblk_ioctl(struct disk *, u_long, void *, int, struct thread *); static int vtblk_dump(void *, void *, off_t, size_t); static void vtblk_strategy(struct bio *); static int vtblk_negotiate_features(struct vtblk_softc *); static int vtblk_setup_features(struct vtblk_softc *); static int vtblk_maximum_segments(struct vtblk_softc *, struct virtio_blk_config *); static int vtblk_alloc_virtqueue(struct vtblk_softc *); static void vtblk_resize_disk(struct vtblk_softc *, uint64_t); static void vtblk_alloc_disk(struct vtblk_softc *, struct virtio_blk_config *); static void vtblk_create_disk(struct vtblk_softc *); static int vtblk_request_prealloc(struct vtblk_softc *); static void vtblk_request_free(struct vtblk_softc *); static struct vtblk_request * vtblk_request_dequeue(struct vtblk_softc *); static void vtblk_request_enqueue(struct vtblk_softc *, struct vtblk_request *); static struct vtblk_request * vtblk_request_next_ready(struct vtblk_softc *); static void vtblk_request_requeue_ready(struct vtblk_softc *, struct vtblk_request *); static struct vtblk_request * vtblk_request_next(struct vtblk_softc *); static struct vtblk_request * vtblk_request_bio(struct vtblk_softc *); static int vtblk_request_execute(struct vtblk_request *, int); static void vtblk_request_execute_cb(void *, bus_dma_segment_t *, int, int); static int vtblk_request_error(struct vtblk_request *); static void vtblk_queue_completed(struct vtblk_softc *, struct bio_queue *); static void vtblk_done_completed(struct vtblk_softc *, struct bio_queue *); static void vtblk_drain_vq(struct vtblk_softc *); static void vtblk_drain(struct vtblk_softc *); static void vtblk_startio(struct vtblk_softc *); static void vtblk_bio_done(struct vtblk_softc *, struct bio *, int); static void vtblk_read_config(struct vtblk_softc *, struct virtio_blk_config *); static void vtblk_ident(struct vtblk_softc *); static int vtblk_poll_request(struct vtblk_softc *, struct vtblk_request *); static int vtblk_quiesce(struct vtblk_softc *); static void vtblk_vq_intr(void *); static void vtblk_stop(struct vtblk_softc *); static void vtblk_dump_quiesce(struct vtblk_softc *); static int vtblk_dump_write(struct vtblk_softc *, void *, off_t, size_t); static int vtblk_dump_flush(struct vtblk_softc *); static void vtblk_dump_complete(struct vtblk_softc *); static void vtblk_set_write_cache(struct vtblk_softc *, int); static int vtblk_write_cache_enabled(struct vtblk_softc *sc, struct virtio_blk_config *); static int vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS); static void vtblk_setup_sysctl(struct vtblk_softc *); static int vtblk_tunable_int(struct vtblk_softc *, const char *, int); #define vtblk_modern(_sc) (((_sc)->vtblk_features & VIRTIO_F_VERSION_1) != 0) #define vtblk_htog16(_sc, _val) virtio_htog16(vtblk_modern(_sc), _val) #define vtblk_htog32(_sc, _val) virtio_htog32(vtblk_modern(_sc), _val) #define vtblk_htog64(_sc, _val) virtio_htog64(vtblk_modern(_sc), _val) #define vtblk_gtoh16(_sc, _val) virtio_gtoh16(vtblk_modern(_sc), _val) #define vtblk_gtoh32(_sc, _val) virtio_gtoh32(vtblk_modern(_sc), _val) #define vtblk_gtoh64(_sc, _val) virtio_gtoh64(vtblk_modern(_sc), _val) /* Tunables. */ static int vtblk_no_ident = 0; TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident); static int vtblk_writecache_mode = -1; TUNABLE_INT("hw.vtblk.writecache_mode", &vtblk_writecache_mode); #define VTBLK_COMMON_FEATURES \ (VIRTIO_BLK_F_SIZE_MAX | \ VIRTIO_BLK_F_SEG_MAX | \ VIRTIO_BLK_F_GEOMETRY | \ VIRTIO_BLK_F_RO | \ VIRTIO_BLK_F_BLK_SIZE | \ VIRTIO_BLK_F_FLUSH | \ VIRTIO_BLK_F_TOPOLOGY | \ VIRTIO_BLK_F_CONFIG_WCE | \ VIRTIO_BLK_F_DISCARD | \ VIRTIO_RING_F_INDIRECT_DESC) #define VTBLK_MODERN_FEATURES (VTBLK_COMMON_FEATURES) #define VTBLK_LEGACY_FEATURES (VIRTIO_BLK_F_BARRIER | VTBLK_COMMON_FEATURES) #define VTBLK_MTX(_sc) &(_sc)->vtblk_mtx #define VTBLK_LOCK_INIT(_sc, _name) \ mtx_init(VTBLK_MTX((_sc)), (_name), \ "VirtIO Block Lock", MTX_DEF) #define VTBLK_LOCK(_sc) mtx_lock(VTBLK_MTX((_sc))) #define VTBLK_UNLOCK(_sc) mtx_unlock(VTBLK_MTX((_sc))) #define VTBLK_LOCK_DESTROY(_sc) mtx_destroy(VTBLK_MTX((_sc))) #define VTBLK_LOCK_ASSERT(_sc) mtx_assert(VTBLK_MTX((_sc)), MA_OWNED) #define VTBLK_LOCK_ASSERT_NOTOWNED(_sc) \ mtx_assert(VTBLK_MTX((_sc)), MA_NOTOWNED) #define VTBLK_DISK_NAME "vtbd" #define VTBLK_QUIESCE_TIMEOUT (30 * hz) #define VTBLK_BSIZE 512 /* * Each block request uses at least two segments - one for the header * and one for the status. */ #define VTBLK_MIN_SEGMENTS 2 static device_method_t vtblk_methods[] = { /* Device methods. */ DEVMETHOD(device_probe, vtblk_probe), DEVMETHOD(device_attach, vtblk_attach), DEVMETHOD(device_detach, vtblk_detach), DEVMETHOD(device_suspend, vtblk_suspend), DEVMETHOD(device_resume, vtblk_resume), DEVMETHOD(device_shutdown, vtblk_shutdown), /* VirtIO methods. */ DEVMETHOD(virtio_attach_completed, vtblk_attach_completed), DEVMETHOD(virtio_config_change, vtblk_config_change), DEVMETHOD_END }; static driver_t vtblk_driver = { "vtblk", vtblk_methods, sizeof(struct vtblk_softc) }; VIRTIO_DRIVER_MODULE(virtio_blk, vtblk_driver, vtblk_modevent, NULL); MODULE_VERSION(virtio_blk, 1); MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1); VIRTIO_SIMPLE_PNPINFO(virtio_blk, VIRTIO_ID_BLOCK, "VirtIO Block Adapter"); static int vtblk_modevent(module_t mod, int type, void *unused) { int error; error = 0; switch (type) { case MOD_LOAD: case MOD_QUIESCE: case MOD_UNLOAD: case MOD_SHUTDOWN: break; default: error = EOPNOTSUPP; break; } return (error); } static int vtblk_probe(device_t dev) { return (VIRTIO_SIMPLE_PROBE(dev, virtio_blk)); } static int vtblk_attach(device_t dev) { struct vtblk_softc *sc; struct virtio_blk_config blkcfg; int error; sc = device_get_softc(dev); sc->vtblk_dev = dev; virtio_set_feature_desc(dev, vtblk_feature_desc); VTBLK_LOCK_INIT(sc, device_get_nameunit(dev)); bioq_init(&sc->vtblk_bioq); TAILQ_INIT(&sc->vtblk_dump_queue); TAILQ_INIT(&sc->vtblk_req_free); TAILQ_INIT(&sc->vtblk_req_ready); vtblk_setup_sysctl(sc); error = vtblk_setup_features(sc); if (error) { device_printf(dev, "cannot setup features\n"); goto fail; } vtblk_read_config(sc, &blkcfg); /* * With the current sglist(9) implementation, it is not easy * for us to support a maximum segment size as adjacent * segments are coalesced. For now, just make sure it's larger * than the maximum supported transfer size. */ if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) { if (blkcfg.size_max < maxphys) { error = ENOTSUP; device_printf(dev, "host requires unsupported " "maximum segment size feature\n"); goto fail; } } sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg); if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) { error = EINVAL; device_printf(dev, "fewer than minimum number of segments " "allowed: %d\n", sc->vtblk_max_nsegs); goto fail; } sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT); if (sc->vtblk_sglist == NULL) { error = ENOMEM; device_printf(dev, "cannot allocate sglist\n"); goto fail; } /* * If vtblk_max_nsegs == VTBLK_MIN_SEGMENTS + 1, the device only * supports a single data segment; in that case we need busdma to * align to a page boundary so we can send a *contiguous* page size * request to the host. */ if (sc->vtblk_max_nsegs == VTBLK_MIN_SEGMENTS + 1) sc->vtblk_flags |= VTBLK_FLAG_BUSDMA_ALIGN; error = bus_dma_tag_create( bus_get_dma_tag(dev), /* parent */ (sc->vtblk_flags & VTBLK_FLAG_BUSDMA_ALIGN) ? PAGE_SIZE : 1, 0, /* boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ maxphys, /* max request size */ sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS, /* max # segments */ maxphys, /* maxsegsize */ 0, /* flags */ busdma_lock_mutex, /* lockfunc */ &sc->vtblk_mtx, /* lockarg */ &sc->vtblk_dmat); if (error) { device_printf(dev, "cannot create bus dma tag\n"); goto fail; } #ifdef __powerpc__ /* * Virtio uses physical addresses rather than bus addresses, so we * need to ask busdma to skip the iommu physical->bus mapping. At * present, this is only a thing on the powerpc architectures. */ bus_dma_tag_set_iommu(sc->vtblk_dmat, NULL, NULL); #endif error = vtblk_alloc_virtqueue(sc); if (error) { device_printf(dev, "cannot allocate virtqueue\n"); goto fail; } error = vtblk_request_prealloc(sc); if (error) { device_printf(dev, "cannot preallocate requests\n"); goto fail; } vtblk_alloc_disk(sc, &blkcfg); error = virtio_setup_intr(dev, INTR_TYPE_BIO | INTR_ENTROPY); if (error) { device_printf(dev, "cannot setup virtqueue interrupt\n"); goto fail; } virtqueue_enable_intr(sc->vtblk_vq); fail: if (error) vtblk_detach(dev); return (error); } static int vtblk_detach(device_t dev) { struct vtblk_softc *sc; sc = device_get_softc(dev); VTBLK_LOCK(sc); sc->vtblk_flags |= VTBLK_FLAG_DETACH; if (device_is_attached(dev)) vtblk_stop(sc); VTBLK_UNLOCK(sc); vtblk_drain(sc); if (sc->vtblk_disk != NULL) { disk_destroy(sc->vtblk_disk); sc->vtblk_disk = NULL; } if (sc->vtblk_dmat != NULL) { bus_dma_tag_destroy(sc->vtblk_dmat); sc->vtblk_dmat = NULL; } if (sc->vtblk_sglist != NULL) { sglist_free(sc->vtblk_sglist); sc->vtblk_sglist = NULL; } VTBLK_LOCK_DESTROY(sc); return (0); } static int vtblk_suspend(device_t dev) { struct vtblk_softc *sc; int error; sc = device_get_softc(dev); VTBLK_LOCK(sc); sc->vtblk_flags |= VTBLK_FLAG_SUSPEND; /* XXX BMV: virtio_stop(), etc needed here? */ error = vtblk_quiesce(sc); if (error) sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND; VTBLK_UNLOCK(sc); return (error); } static int vtblk_resume(device_t dev) { struct vtblk_softc *sc; sc = device_get_softc(dev); VTBLK_LOCK(sc); /* XXX BMV: virtio_reinit(), etc needed here? */ sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND; vtblk_startio(sc); VTBLK_UNLOCK(sc); return (0); } static int vtblk_shutdown(device_t dev) { return (0); } static int vtblk_attach_completed(device_t dev) { struct vtblk_softc *sc; sc = device_get_softc(dev); /* * Create disk after attach as VIRTIO_BLK_T_GET_ID can only be * processed after the device acknowledged * VIRTIO_CONFIG_STATUS_DRIVER_OK. */ vtblk_create_disk(sc); return (0); } static int vtblk_config_change(device_t dev) { struct vtblk_softc *sc; struct virtio_blk_config blkcfg; uint64_t capacity; sc = device_get_softc(dev); vtblk_read_config(sc, &blkcfg); /* Capacity is always in 512-byte units. */ capacity = blkcfg.capacity * VTBLK_BSIZE; if (sc->vtblk_disk->d_mediasize != capacity) vtblk_resize_disk(sc, capacity); return (0); } static int vtblk_open(struct disk *dp) { struct vtblk_softc *sc; if ((sc = dp->d_drv1) == NULL) return (ENXIO); return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0); } static int vtblk_close(struct disk *dp) { struct vtblk_softc *sc; if ((sc = dp->d_drv1) == NULL) return (ENXIO); return (0); } static int vtblk_ioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td) { struct vtblk_softc *sc; if ((sc = dp->d_drv1) == NULL) return (ENXIO); return (ENOTTY); } static int vtblk_dump(void *arg, void *virtual, off_t offset, size_t length) { struct disk *dp; struct vtblk_softc *sc; int error; dp = arg; error = 0; if ((sc = dp->d_drv1) == NULL) return (ENXIO); VTBLK_LOCK(sc); vtblk_dump_quiesce(sc); if (length > 0) error = vtblk_dump_write(sc, virtual, offset, length); if (error || (virtual == NULL && offset == 0)) vtblk_dump_complete(sc); VTBLK_UNLOCK(sc); return (error); } static void vtblk_strategy(struct bio *bp) { struct vtblk_softc *sc; if ((sc = bp->bio_disk->d_drv1) == NULL) { vtblk_bio_done(NULL, bp, EINVAL); return; } if ((bp->bio_cmd != BIO_READ) && (bp->bio_cmd != BIO_WRITE) && (bp->bio_cmd != BIO_FLUSH) && (bp->bio_cmd != BIO_DELETE)) { vtblk_bio_done(sc, bp, EOPNOTSUPP); return; } VTBLK_LOCK(sc); if (sc->vtblk_flags & VTBLK_FLAG_DETACH) { VTBLK_UNLOCK(sc); vtblk_bio_done(sc, bp, ENXIO); return; } bioq_insert_tail(&sc->vtblk_bioq, bp); vtblk_startio(sc); VTBLK_UNLOCK(sc); } static int vtblk_negotiate_features(struct vtblk_softc *sc) { device_t dev; uint64_t features; dev = sc->vtblk_dev; features = virtio_bus_is_modern(dev) ? VTBLK_MODERN_FEATURES : VTBLK_LEGACY_FEATURES; sc->vtblk_features = virtio_negotiate_features(dev, features); return (virtio_finalize_features(dev)); } static int vtblk_setup_features(struct vtblk_softc *sc) { device_t dev; int error; dev = sc->vtblk_dev; error = vtblk_negotiate_features(sc); if (error) return (error); if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) sc->vtblk_flags |= VTBLK_FLAG_INDIRECT; if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE)) sc->vtblk_flags |= VTBLK_FLAG_WCE_CONFIG; /* Legacy. */ if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER)) sc->vtblk_flags |= VTBLK_FLAG_BARRIER; return (0); } static int vtblk_maximum_segments(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) { device_t dev; int nsegs; dev = sc->vtblk_dev; nsegs = VTBLK_MIN_SEGMENTS; if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) { nsegs += MIN(blkcfg->seg_max, maxphys / PAGE_SIZE + 1); if (sc->vtblk_flags & VTBLK_FLAG_INDIRECT) nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); } else nsegs += 1; return (nsegs); } static int vtblk_alloc_virtqueue(struct vtblk_softc *sc) { device_t dev; struct vq_alloc_info vq_info; dev = sc->vtblk_dev; VQ_ALLOC_INFO_INIT(&vq_info, sc->vtblk_max_nsegs, vtblk_vq_intr, sc, &sc->vtblk_vq, "%s request", device_get_nameunit(dev)); - return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info)); + return (virtio_alloc_virtqueues(dev, 1, &vq_info)); } static void vtblk_resize_disk(struct vtblk_softc *sc, uint64_t new_capacity) { device_t dev; struct disk *dp; int error; dev = sc->vtblk_dev; dp = sc->vtblk_disk; dp->d_mediasize = new_capacity; if (bootverbose) { device_printf(dev, "resized to %juMB (%ju %u byte sectors)\n", (uintmax_t) dp->d_mediasize >> 20, (uintmax_t) dp->d_mediasize / dp->d_sectorsize, dp->d_sectorsize); } error = disk_resize(dp, M_NOWAIT); if (error) { device_printf(dev, "disk_resize(9) failed, error: %d\n", error); } } static void vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) { device_t dev; struct disk *dp; dev = sc->vtblk_dev; sc->vtblk_disk = dp = disk_alloc(); dp->d_open = vtblk_open; dp->d_close = vtblk_close; dp->d_ioctl = vtblk_ioctl; dp->d_strategy = vtblk_strategy; dp->d_name = VTBLK_DISK_NAME; dp->d_unit = device_get_unit(dev); dp->d_drv1 = sc; dp->d_flags = DISKFLAG_UNMAPPED_BIO | DISKFLAG_DIRECT_COMPLETION; dp->d_hba_vendor = virtio_get_vendor(dev); dp->d_hba_device = virtio_get_device(dev); dp->d_hba_subvendor = virtio_get_subvendor(dev); dp->d_hba_subdevice = virtio_get_subdevice(dev); if (virtio_with_feature(dev, VIRTIO_BLK_F_RO)) dp->d_flags |= DISKFLAG_WRITE_PROTECT; else { if (virtio_with_feature(dev, VIRTIO_BLK_F_FLUSH)) dp->d_flags |= DISKFLAG_CANFLUSHCACHE; dp->d_dump = vtblk_dump; } /* Capacity is always in 512-byte units. */ dp->d_mediasize = blkcfg->capacity * VTBLK_BSIZE; if (virtio_with_feature(dev, VIRTIO_BLK_F_BLK_SIZE)) dp->d_sectorsize = blkcfg->blk_size; else dp->d_sectorsize = VTBLK_BSIZE; /* * The VirtIO maximum I/O size is given in terms of segments. * However, FreeBSD limits I/O size by logical buffer size, not * by physically contiguous pages. Therefore, we have to assume * no pages are contiguous. This may impose an artificially low * maximum I/O size. But in practice, since QEMU advertises 128 * segments, this gives us a maximum IO size of 125 * PAGE_SIZE, * which is typically greater than maxphys. Eventually we should * just advertise maxphys and split buffers that are too big. * * If we're not asking busdma to align data to page boundaries, the * maximum I/O size is reduced by PAGE_SIZE in order to accommodate * unaligned I/Os. */ dp->d_maxsize = (sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS) * PAGE_SIZE; if ((sc->vtblk_flags & VTBLK_FLAG_BUSDMA_ALIGN) == 0) dp->d_maxsize -= PAGE_SIZE; if (virtio_with_feature(dev, VIRTIO_BLK_F_GEOMETRY)) { dp->d_fwsectors = blkcfg->geometry.sectors; dp->d_fwheads = blkcfg->geometry.heads; } if (virtio_with_feature(dev, VIRTIO_BLK_F_TOPOLOGY) && blkcfg->topology.physical_block_exp > 0) { dp->d_stripesize = dp->d_sectorsize * (1 << blkcfg->topology.physical_block_exp); dp->d_stripeoffset = (dp->d_stripesize - blkcfg->topology.alignment_offset * dp->d_sectorsize) % dp->d_stripesize; } if (virtio_with_feature(dev, VIRTIO_BLK_F_DISCARD)) { dp->d_flags |= DISKFLAG_CANDELETE; dp->d_delmaxsize = blkcfg->max_discard_sectors * VTBLK_BSIZE; } if (vtblk_write_cache_enabled(sc, blkcfg) != 0) sc->vtblk_write_cache = VTBLK_CACHE_WRITEBACK; else sc->vtblk_write_cache = VTBLK_CACHE_WRITETHROUGH; } static void vtblk_create_disk(struct vtblk_softc *sc) { struct disk *dp; dp = sc->vtblk_disk; vtblk_ident(sc); device_printf(sc->vtblk_dev, "%juMB (%ju %u byte sectors)\n", (uintmax_t) dp->d_mediasize >> 20, (uintmax_t) dp->d_mediasize / dp->d_sectorsize, dp->d_sectorsize); disk_create(dp, DISK_VERSION); } static int vtblk_request_prealloc(struct vtblk_softc *sc) { struct vtblk_request *req; int i, nreqs; nreqs = virtqueue_size(sc->vtblk_vq); /* * Preallocate sufficient requests to keep the virtqueue full. Each * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce * the number allocated when indirect descriptors are not available. */ if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0) nreqs /= VTBLK_MIN_SEGMENTS; for (i = 0; i < nreqs; i++) { req = malloc(sizeof(struct vtblk_request), M_DEVBUF, M_NOWAIT); if (req == NULL) return (ENOMEM); req->vbr_sc = sc; if (bus_dmamap_create(sc->vtblk_dmat, 0, &req->vbr_mapp)) { free(req, M_DEVBUF); return (ENOMEM); } MPASS(sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr)) == 1); MPASS(sglist_count(&req->vbr_ack, sizeof(req->vbr_ack)) == 1); sc->vtblk_request_count++; vtblk_request_enqueue(sc, req); } return (0); } static void vtblk_request_free(struct vtblk_softc *sc) { struct vtblk_request *req; MPASS(TAILQ_EMPTY(&sc->vtblk_req_ready)); while ((req = vtblk_request_dequeue(sc)) != NULL) { sc->vtblk_request_count--; bus_dmamap_destroy(sc->vtblk_dmat, req->vbr_mapp); free(req, M_DEVBUF); } KASSERT(sc->vtblk_request_count == 0, ("%s: leaked %d requests", __func__, sc->vtblk_request_count)); } static struct vtblk_request * vtblk_request_dequeue(struct vtblk_softc *sc) { struct vtblk_request *req; req = TAILQ_FIRST(&sc->vtblk_req_free); if (req != NULL) { TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link); bzero(&req->vbr_hdr, sizeof(struct vtblk_request) - offsetof(struct vtblk_request, vbr_hdr)); } return (req); } static void vtblk_request_enqueue(struct vtblk_softc *sc, struct vtblk_request *req) { TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link); } static struct vtblk_request * vtblk_request_next_ready(struct vtblk_softc *sc) { struct vtblk_request *req; req = TAILQ_FIRST(&sc->vtblk_req_ready); if (req != NULL) TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link); return (req); } static void vtblk_request_requeue_ready(struct vtblk_softc *sc, struct vtblk_request *req) { /* NOTE: Currently, there will be at most one request in the queue. */ TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link); } static struct vtblk_request * vtblk_request_next(struct vtblk_softc *sc) { struct vtblk_request *req; req = vtblk_request_next_ready(sc); if (req != NULL) return (req); return (vtblk_request_bio(sc)); } static struct vtblk_request * vtblk_request_bio(struct vtblk_softc *sc) { struct bio_queue_head *bioq; struct vtblk_request *req; struct bio *bp; bioq = &sc->vtblk_bioq; if (bioq_first(bioq) == NULL) return (NULL); req = vtblk_request_dequeue(sc); if (req == NULL) return (NULL); bp = bioq_takefirst(bioq); req->vbr_bp = bp; req->vbr_ack = -1; req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1); switch (bp->bio_cmd) { case BIO_FLUSH: req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH); req->vbr_hdr.sector = 0; break; case BIO_READ: req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_IN); req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE); break; case BIO_WRITE: req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT); req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE); break; case BIO_DELETE: req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_DISCARD); req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE); break; default: panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd); } if (bp->bio_flags & BIO_ORDERED) req->vbr_hdr.type |= vtblk_gtoh32(sc, VIRTIO_BLK_T_BARRIER); return (req); } static int vtblk_request_execute(struct vtblk_request *req, int flags) { struct vtblk_softc *sc = req->vbr_sc; struct bio *bp = req->vbr_bp; int error = 0; /* * Call via bus_dmamap_load_bio or directly depending on whether we * have a buffer we need to map. If we don't have a busdma map, * try to perform the I/O directly and hope that it works (this will * happen when dumping). */ if ((req->vbr_mapp != NULL) && (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE)) { error = bus_dmamap_load_bio(sc->vtblk_dmat, req->vbr_mapp, req->vbr_bp, vtblk_request_execute_cb, req, flags); if (error == EINPROGRESS) { req->vbr_busdma_wait = 1; sc->vtblk_flags |= VTBLK_FLAG_BUSDMA_WAIT; } } else { vtblk_request_execute_cb(req, NULL, 0, 0); } return (error ? error : req->vbr_error); } static void vtblk_request_execute_cb(void * callback_arg, bus_dma_segment_t * segs, int nseg, int error) { struct vtblk_request *req; struct vtblk_softc *sc; struct virtqueue *vq; struct sglist *sg; struct bio *bp; int ordered, readable, writable, i; req = (struct vtblk_request *)callback_arg; sc = req->vbr_sc; vq = sc->vtblk_vq; sg = sc->vtblk_sglist; bp = req->vbr_bp; ordered = 0; writable = 0; /* * If we paused request queueing while we waited for busdma to call us * asynchronously, unpause it now; this request made it through so we * don't need to worry about others getting ahead of us. (Note that we * hold the device mutex so nothing will happen until after we return * anyway.) */ if (req->vbr_busdma_wait) sc->vtblk_flags &= ~VTBLK_FLAG_BUSDMA_WAIT; /* Fail on errors from busdma. */ if (error) goto out1; /* * Some hosts (such as bhyve) do not implement the barrier feature, * so we emulate it in the driver by allowing the barrier request * to be the only one in flight. */ if ((sc->vtblk_flags & VTBLK_FLAG_BARRIER) == 0) { if (sc->vtblk_req_ordered != NULL) { error = EBUSY; goto out; } if (bp->bio_flags & BIO_ORDERED) { if (!virtqueue_empty(vq)) { error = EBUSY; goto out; } ordered = 1; req->vbr_hdr.type &= vtblk_gtoh32(sc, ~VIRTIO_BLK_T_BARRIER); } } sglist_reset(sg); sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr)); if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { /* * We cast bus_addr_t to vm_paddr_t here; since we skip the * iommu mapping (see vtblk_attach) this should be safe. */ for (i = 0; i < nseg; i++) { error = sglist_append_phys(sg, (vm_paddr_t)segs[i].ds_addr, segs[i].ds_len); if (error || sg->sg_nseg == sg->sg_maxseg) { panic("%s: bio %p data buffer too big %d", __func__, bp, error); } } /* Special handling for dump, which bypasses busdma. */ if (req->vbr_mapp == NULL) { error = sglist_append_bio(sg, bp); if (error || sg->sg_nseg == sg->sg_maxseg) { panic("%s: bio %p data buffer too big %d", __func__, bp, error); } } /* BIO_READ means the host writes into our buffer. */ if (bp->bio_cmd == BIO_READ) writable = sg->sg_nseg - 1; } else if (bp->bio_cmd == BIO_DELETE) { struct virtio_blk_discard_write_zeroes *discard; discard = malloc(sizeof(*discard), M_DEVBUF, M_NOWAIT | M_ZERO); if (discard == NULL) { error = ENOMEM; goto out; } bp->bio_driver1 = discard; discard->sector = vtblk_gtoh64(sc, bp->bio_offset / VTBLK_BSIZE); discard->num_sectors = vtblk_gtoh32(sc, bp->bio_bcount / VTBLK_BSIZE); error = sglist_append(sg, discard, sizeof(*discard)); if (error || sg->sg_nseg == sg->sg_maxseg) { panic("%s: bio %p data buffer too big %d", __func__, bp, error); } } writable++; sglist_append(sg, &req->vbr_ack, sizeof(uint8_t)); readable = sg->sg_nseg - writable; if (req->vbr_mapp != NULL) { switch (bp->bio_cmd) { case BIO_READ: bus_dmamap_sync(sc->vtblk_dmat, req->vbr_mapp, BUS_DMASYNC_PREREAD); break; case BIO_WRITE: bus_dmamap_sync(sc->vtblk_dmat, req->vbr_mapp, BUS_DMASYNC_PREWRITE); break; } } error = virtqueue_enqueue(vq, req, sg, readable, writable); if (error == 0 && ordered) sc->vtblk_req_ordered = req; /* * If we were called asynchronously, we need to notify the queue that * we've added a new request, since the notification from startio was * performed already. */ if (error == 0 && req->vbr_busdma_wait) virtqueue_notify(vq); out: if (error && (req->vbr_mapp != NULL)) bus_dmamap_unload(sc->vtblk_dmat, req->vbr_mapp); out1: if (error && req->vbr_requeue_on_error) vtblk_request_requeue_ready(sc, req); req->vbr_error = error; } static int vtblk_request_error(struct vtblk_request *req) { int error; switch (req->vbr_ack) { case VIRTIO_BLK_S_OK: error = 0; break; case VIRTIO_BLK_S_UNSUPP: error = ENOTSUP; break; default: error = EIO; break; } return (error); } static void vtblk_queue_completed(struct vtblk_softc *sc, struct bio_queue *queue) { struct vtblk_request *req; struct bio *bp; while ((req = virtqueue_dequeue(sc->vtblk_vq, NULL)) != NULL) { if (sc->vtblk_req_ordered != NULL) { MPASS(sc->vtblk_req_ordered == req); sc->vtblk_req_ordered = NULL; } bp = req->vbr_bp; if (req->vbr_mapp != NULL) { switch (bp->bio_cmd) { case BIO_READ: bus_dmamap_sync(sc->vtblk_dmat, req->vbr_mapp, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->vtblk_dmat, req->vbr_mapp); break; case BIO_WRITE: bus_dmamap_sync(sc->vtblk_dmat, req->vbr_mapp, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->vtblk_dmat, req->vbr_mapp); break; } } bp->bio_error = vtblk_request_error(req); TAILQ_INSERT_TAIL(queue, bp, bio_queue); vtblk_request_enqueue(sc, req); } } static void vtblk_done_completed(struct vtblk_softc *sc, struct bio_queue *queue) { struct bio *bp, *tmp; TAILQ_FOREACH_SAFE(bp, queue, bio_queue, tmp) { if (bp->bio_error != 0) disk_err(bp, "hard error", -1, 1); vtblk_bio_done(sc, bp, bp->bio_error); } } static void vtblk_drain_vq(struct vtblk_softc *sc) { struct virtqueue *vq; struct vtblk_request *req; int last; vq = sc->vtblk_vq; last = 0; while ((req = virtqueue_drain(vq, &last)) != NULL) { vtblk_bio_done(sc, req->vbr_bp, ENXIO); vtblk_request_enqueue(sc, req); } sc->vtblk_req_ordered = NULL; KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); } static void vtblk_drain(struct vtblk_softc *sc) { struct bio_queue_head *bioq; struct vtblk_request *req; struct bio *bp; bioq = &sc->vtblk_bioq; if (sc->vtblk_vq != NULL) { struct bio_queue queue; TAILQ_INIT(&queue); vtblk_queue_completed(sc, &queue); vtblk_done_completed(sc, &queue); vtblk_drain_vq(sc); } while ((req = vtblk_request_next_ready(sc)) != NULL) { vtblk_bio_done(sc, req->vbr_bp, ENXIO); vtblk_request_enqueue(sc, req); } while (bioq_first(bioq) != NULL) { bp = bioq_takefirst(bioq); vtblk_bio_done(sc, bp, ENXIO); } vtblk_request_free(sc); } static void vtblk_startio(struct vtblk_softc *sc) { struct virtqueue *vq; struct vtblk_request *req; int enq; VTBLK_LOCK_ASSERT(sc); vq = sc->vtblk_vq; enq = 0; if (sc->vtblk_flags & (VTBLK_FLAG_SUSPEND | VTBLK_FLAG_BUSDMA_WAIT)) return; while (!virtqueue_full(vq)) { req = vtblk_request_next(sc); if (req == NULL) break; req->vbr_requeue_on_error = 1; if (vtblk_request_execute(req, BUS_DMA_WAITOK)) break; enq++; } if (enq > 0) virtqueue_notify(vq); } static void vtblk_bio_done(struct vtblk_softc *sc, struct bio *bp, int error) { /* Because of GEOM direct dispatch, we cannot hold any locks. */ if (sc != NULL) VTBLK_LOCK_ASSERT_NOTOWNED(sc); if (error) { bp->bio_resid = bp->bio_bcount; bp->bio_error = error; bp->bio_flags |= BIO_ERROR; } else { kmsan_mark_bio(bp, KMSAN_STATE_INITED); } if (bp->bio_driver1 != NULL) { free(bp->bio_driver1, M_DEVBUF); bp->bio_driver1 = NULL; } biodone(bp); } #define VTBLK_GET_CONFIG(_dev, _feature, _field, _cfg) \ if (virtio_with_feature(_dev, _feature)) { \ virtio_read_device_config(_dev, \ offsetof(struct virtio_blk_config, _field), \ &(_cfg)->_field, sizeof((_cfg)->_field)); \ } static void vtblk_read_config(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) { device_t dev; dev = sc->vtblk_dev; bzero(blkcfg, sizeof(struct virtio_blk_config)); /* The capacity is always available. */ virtio_read_device_config(dev, offsetof(struct virtio_blk_config, capacity), &blkcfg->capacity, sizeof(blkcfg->capacity)); /* Read the configuration if the feature was negotiated. */ VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SIZE_MAX, size_max, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SEG_MAX, seg_max, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry.cylinders, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry.heads, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry.sectors, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_BLK_SIZE, blk_size, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology.physical_block_exp, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology.alignment_offset, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology.min_io_size, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology.opt_io_size, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, wce, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_DISCARD, max_discard_sectors, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_DISCARD, max_discard_seg, blkcfg); VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_DISCARD, discard_sector_alignment, blkcfg); } #undef VTBLK_GET_CONFIG static void vtblk_ident(struct vtblk_softc *sc) { struct bio buf; struct disk *dp; struct vtblk_request *req; int len, error; dp = sc->vtblk_disk; len = MIN(VIRTIO_BLK_ID_BYTES, DISK_IDENT_SIZE); if (vtblk_tunable_int(sc, "no_ident", vtblk_no_ident) != 0) return; req = vtblk_request_dequeue(sc); if (req == NULL) return; req->vbr_ack = -1; req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_GET_ID); req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1); req->vbr_hdr.sector = 0; req->vbr_bp = &buf; g_reset_bio(&buf); buf.bio_cmd = BIO_READ; buf.bio_data = dp->d_ident; buf.bio_bcount = len; VTBLK_LOCK(sc); error = vtblk_poll_request(sc, req); VTBLK_UNLOCK(sc); vtblk_request_enqueue(sc, req); if (error) { device_printf(sc->vtblk_dev, "error getting device identifier: %d\n", error); } } static int vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req) { struct virtqueue *vq; int error; vq = sc->vtblk_vq; if (!virtqueue_empty(vq)) return (EBUSY); error = vtblk_request_execute(req, BUS_DMA_NOWAIT); if (error) return (error); virtqueue_notify(vq); virtqueue_poll(vq, NULL); error = vtblk_request_error(req); if (error && bootverbose) { device_printf(sc->vtblk_dev, "%s: IO error: %d\n", __func__, error); } return (error); } static int vtblk_quiesce(struct vtblk_softc *sc) { int error; VTBLK_LOCK_ASSERT(sc); error = 0; while (!virtqueue_empty(sc->vtblk_vq)) { if (mtx_sleep(&sc->vtblk_vq, VTBLK_MTX(sc), PRIBIO, "vtblkq", VTBLK_QUIESCE_TIMEOUT) == EWOULDBLOCK) { error = EBUSY; break; } } return (error); } static void vtblk_vq_intr(void *xsc) { struct vtblk_softc *sc; struct virtqueue *vq; struct bio_queue queue; sc = xsc; vq = sc->vtblk_vq; TAILQ_INIT(&queue); VTBLK_LOCK(sc); again: if (sc->vtblk_flags & VTBLK_FLAG_DETACH) goto out; vtblk_queue_completed(sc, &queue); vtblk_startio(sc); if (virtqueue_enable_intr(vq) != 0) { virtqueue_disable_intr(vq); goto again; } if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND) wakeup(&sc->vtblk_vq); out: VTBLK_UNLOCK(sc); vtblk_done_completed(sc, &queue); } static void vtblk_stop(struct vtblk_softc *sc) { virtqueue_disable_intr(sc->vtblk_vq); virtio_stop(sc->vtblk_dev); } static void vtblk_dump_quiesce(struct vtblk_softc *sc) { /* * Spin here until all the requests in-flight at the time of the * dump are completed and queued. The queued requests will be * biodone'd once the dump is finished. */ while (!virtqueue_empty(sc->vtblk_vq)) vtblk_queue_completed(sc, &sc->vtblk_dump_queue); } static int vtblk_dump_write(struct vtblk_softc *sc, void *virtual, off_t offset, size_t length) { struct bio buf; struct vtblk_request *req; req = &sc->vtblk_dump_request; req->vbr_sc = sc; req->vbr_ack = -1; req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT); req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1); req->vbr_hdr.sector = vtblk_gtoh64(sc, offset / VTBLK_BSIZE); req->vbr_bp = &buf; g_reset_bio(&buf); buf.bio_cmd = BIO_WRITE; buf.bio_data = virtual; buf.bio_bcount = length; return (vtblk_poll_request(sc, req)); } static int vtblk_dump_flush(struct vtblk_softc *sc) { struct bio buf; struct vtblk_request *req; req = &sc->vtblk_dump_request; req->vbr_sc = sc; req->vbr_ack = -1; req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH); req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1); req->vbr_hdr.sector = 0; req->vbr_bp = &buf; g_reset_bio(&buf); buf.bio_cmd = BIO_FLUSH; return (vtblk_poll_request(sc, req)); } static void vtblk_dump_complete(struct vtblk_softc *sc) { vtblk_dump_flush(sc); VTBLK_UNLOCK(sc); vtblk_done_completed(sc, &sc->vtblk_dump_queue); VTBLK_LOCK(sc); } static void vtblk_set_write_cache(struct vtblk_softc *sc, int wc) { /* Set either writeback (1) or writethrough (0) mode. */ virtio_write_dev_config_1(sc->vtblk_dev, offsetof(struct virtio_blk_config, wce), wc); } static int vtblk_write_cache_enabled(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg) { int wc; if (sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) { wc = vtblk_tunable_int(sc, "writecache_mode", vtblk_writecache_mode); if (wc >= 0 && wc < VTBLK_CACHE_MAX) vtblk_set_write_cache(sc, wc); else wc = blkcfg->wce; } else wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_FLUSH); return (wc); } static int vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS) { struct vtblk_softc *sc; int wc, error; sc = oidp->oid_arg1; wc = sc->vtblk_write_cache; error = sysctl_handle_int(oidp, &wc, 0, req); if (error || req->newptr == NULL) return (error); if ((sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) == 0) return (EPERM); if (wc < 0 || wc >= VTBLK_CACHE_MAX) return (EINVAL); VTBLK_LOCK(sc); sc->vtblk_write_cache = wc; vtblk_set_write_cache(sc, sc->vtblk_write_cache); VTBLK_UNLOCK(sc); return (0); } static void vtblk_setup_sysctl(struct vtblk_softc *sc) { device_t dev; struct sysctl_ctx_list *ctx; struct sysctl_oid *tree; struct sysctl_oid_list *child; dev = sc->vtblk_dev; ctx = device_get_sysctl_ctx(dev); tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "writecache_mode", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, vtblk_write_cache_sysctl, "I", "Write cache mode (writethrough (0) or writeback (1))"); } static int vtblk_tunable_int(struct vtblk_softc *sc, const char *knob, int def) { char path[64]; snprintf(path, sizeof(path), "hw.vtblk.%d.%s", device_get_unit(sc->vtblk_dev), knob); TUNABLE_INT_FETCH(path, &def); return (def); } diff --git a/sys/dev/virtio/console/virtio_console.c b/sys/dev/virtio/console/virtio_console.c index d8048d0f1f6e..b69a473cf8a4 100644 --- a/sys/dev/virtio/console/virtio_console.c +++ b/sys/dev/virtio/console/virtio_console.c @@ -1,1514 +1,1514 @@ /*- * Copyright (c) 2014, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for VirtIO console devices. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_if.h" #define VTCON_MAX_PORTS 32 #define VTCON_TTY_PREFIX "V" #define VTCON_TTY_ALIAS_PREFIX "vtcon" #define VTCON_BULK_BUFSZ 128 #define VTCON_CTRL_BUFSZ 128 /* * The buffers cannot cross more than one page boundary due to the * size of the sglist segment array used. */ CTASSERT(VTCON_BULK_BUFSZ <= PAGE_SIZE); CTASSERT(VTCON_CTRL_BUFSZ <= PAGE_SIZE); CTASSERT(sizeof(struct virtio_console_config) <= VTCON_CTRL_BUFSZ); struct vtcon_softc; struct vtcon_softc_port; struct vtcon_port { struct mtx vtcport_mtx; struct vtcon_softc *vtcport_sc; struct vtcon_softc_port *vtcport_scport; struct tty *vtcport_tty; struct virtqueue *vtcport_invq; struct virtqueue *vtcport_outvq; int vtcport_id; int vtcport_flags; #define VTCON_PORT_FLAG_GONE 0x01 #define VTCON_PORT_FLAG_CONSOLE 0x02 #define VTCON_PORT_FLAG_ALIAS 0x04 #if defined(KDB) int vtcport_alt_break_state; #endif }; #define VTCON_PORT_LOCK(_port) mtx_lock(&(_port)->vtcport_mtx) #define VTCON_PORT_UNLOCK(_port) mtx_unlock(&(_port)->vtcport_mtx) struct vtcon_softc_port { struct vtcon_softc *vcsp_sc; struct vtcon_port *vcsp_port; struct virtqueue *vcsp_invq; struct virtqueue *vcsp_outvq; }; struct vtcon_softc { device_t vtcon_dev; struct mtx vtcon_mtx; uint64_t vtcon_features; uint32_t vtcon_max_ports; uint32_t vtcon_flags; #define VTCON_FLAG_DETACHED 0x01 #define VTCON_FLAG_SIZE 0x02 #define VTCON_FLAG_MULTIPORT 0x04 /* * Ports can be added and removed during runtime, but we have * to allocate all the virtqueues during attach. This array is * indexed by the port ID. */ struct vtcon_softc_port *vtcon_ports; struct task vtcon_ctrl_task; struct virtqueue *vtcon_ctrl_rxvq; struct virtqueue *vtcon_ctrl_txvq; struct mtx vtcon_ctrl_tx_mtx; }; #define VTCON_LOCK(_sc) mtx_lock(&(_sc)->vtcon_mtx) #define VTCON_UNLOCK(_sc) mtx_unlock(&(_sc)->vtcon_mtx) #define VTCON_LOCK_ASSERT(_sc) \ mtx_assert(&(_sc)->vtcon_mtx, MA_OWNED) #define VTCON_LOCK_ASSERT_NOTOWNED(_sc) \ mtx_assert(&(_sc)->vtcon_mtx, MA_NOTOWNED) #define VTCON_CTRL_TX_LOCK(_sc) mtx_lock(&(_sc)->vtcon_ctrl_tx_mtx) #define VTCON_CTRL_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->vtcon_ctrl_tx_mtx) #define VTCON_ASSERT_VALID_PORTID(_sc, _id) \ KASSERT((_id) >= 0 && (_id) < (_sc)->vtcon_max_ports, \ ("%s: port ID %d out of range", __func__, _id)) #define VTCON_FEATURES VIRTIO_CONSOLE_F_MULTIPORT static struct virtio_feature_desc vtcon_feature_desc[] = { { VIRTIO_CONSOLE_F_SIZE, "ConsoleSize" }, { VIRTIO_CONSOLE_F_MULTIPORT, "MultiplePorts" }, { VIRTIO_CONSOLE_F_EMERG_WRITE, "EmergencyWrite" }, { 0, NULL } }; static int vtcon_modevent(module_t, int, void *); static void vtcon_drain_all(void); static int vtcon_probe(device_t); static int vtcon_attach(device_t); static int vtcon_detach(device_t); static int vtcon_config_change(device_t); static int vtcon_setup_features(struct vtcon_softc *); static int vtcon_negotiate_features(struct vtcon_softc *); static int vtcon_alloc_scports(struct vtcon_softc *); static int vtcon_alloc_virtqueues(struct vtcon_softc *); static void vtcon_read_config(struct vtcon_softc *, struct virtio_console_config *); static void vtcon_determine_max_ports(struct vtcon_softc *, struct virtio_console_config *); static void vtcon_destroy_ports(struct vtcon_softc *); static void vtcon_stop(struct vtcon_softc *); static int vtcon_ctrl_event_enqueue(struct vtcon_softc *, struct virtio_console_control *); static int vtcon_ctrl_event_create(struct vtcon_softc *); static void vtcon_ctrl_event_requeue(struct vtcon_softc *, struct virtio_console_control *); static int vtcon_ctrl_event_populate(struct vtcon_softc *); static void vtcon_ctrl_event_drain(struct vtcon_softc *); static int vtcon_ctrl_init(struct vtcon_softc *); static void vtcon_ctrl_deinit(struct vtcon_softc *); static void vtcon_ctrl_port_add_event(struct vtcon_softc *, int); static void vtcon_ctrl_port_remove_event(struct vtcon_softc *, int); static void vtcon_ctrl_port_console_event(struct vtcon_softc *, int); static void vtcon_ctrl_port_open_event(struct vtcon_softc *, int); static void vtcon_ctrl_port_name_event(struct vtcon_softc *, int, const char *, size_t); static void vtcon_ctrl_process_event(struct vtcon_softc *, struct virtio_console_control *, void *, size_t); static void vtcon_ctrl_task_cb(void *, int); static void vtcon_ctrl_event_intr(void *); static void vtcon_ctrl_poll(struct vtcon_softc *, struct virtio_console_control *control); static void vtcon_ctrl_send_control(struct vtcon_softc *, uint32_t, uint16_t, uint16_t); static int vtcon_port_enqueue_buf(struct vtcon_port *, void *, size_t); static int vtcon_port_create_buf(struct vtcon_port *); static void vtcon_port_requeue_buf(struct vtcon_port *, void *); static int vtcon_port_populate(struct vtcon_port *); static void vtcon_port_destroy(struct vtcon_port *); static int vtcon_port_create(struct vtcon_softc *, int); static void vtcon_port_dev_alias(struct vtcon_port *, const char *, size_t); static void vtcon_port_drain_bufs(struct virtqueue *); static void vtcon_port_drain(struct vtcon_port *); static void vtcon_port_teardown(struct vtcon_port *); static void vtcon_port_change_size(struct vtcon_port *, uint16_t, uint16_t); static void vtcon_port_update_console_size(struct vtcon_softc *); static void vtcon_port_enable_intr(struct vtcon_port *); static void vtcon_port_disable_intr(struct vtcon_port *); static void vtcon_port_in(struct vtcon_port *); static void vtcon_port_intr(void *); static void vtcon_port_out(struct vtcon_port *, void *, int); static void vtcon_port_submit_event(struct vtcon_port *, uint16_t, uint16_t); static int vtcon_tty_open(struct tty *); static void vtcon_tty_close(struct tty *); static void vtcon_tty_outwakeup(struct tty *); static void vtcon_tty_free(void *); static void vtcon_get_console_size(struct vtcon_softc *, uint16_t *, uint16_t *); static void vtcon_enable_interrupts(struct vtcon_softc *); static void vtcon_disable_interrupts(struct vtcon_softc *); #define vtcon_modern(_sc) (((_sc)->vtcon_features & VIRTIO_F_VERSION_1) != 0) #define vtcon_htog16(_sc, _val) virtio_htog16(vtcon_modern(_sc), _val) #define vtcon_htog32(_sc, _val) virtio_htog32(vtcon_modern(_sc), _val) #define vtcon_htog64(_sc, _val) virtio_htog64(vtcon_modern(_sc), _val) #define vtcon_gtoh16(_sc, _val) virtio_gtoh16(vtcon_modern(_sc), _val) #define vtcon_gtoh32(_sc, _val) virtio_gtoh32(vtcon_modern(_sc), _val) #define vtcon_gtoh64(_sc, _val) virtio_gtoh64(vtcon_modern(_sc), _val) static int vtcon_pending_free; static struct ttydevsw vtcon_tty_class = { .tsw_flags = 0, .tsw_open = vtcon_tty_open, .tsw_close = vtcon_tty_close, .tsw_outwakeup = vtcon_tty_outwakeup, .tsw_free = vtcon_tty_free, }; static device_method_t vtcon_methods[] = { /* Device methods. */ DEVMETHOD(device_probe, vtcon_probe), DEVMETHOD(device_attach, vtcon_attach), DEVMETHOD(device_detach, vtcon_detach), /* VirtIO methods. */ DEVMETHOD(virtio_config_change, vtcon_config_change), DEVMETHOD_END }; static driver_t vtcon_driver = { "vtcon", vtcon_methods, sizeof(struct vtcon_softc) }; VIRTIO_DRIVER_MODULE(virtio_console, vtcon_driver, vtcon_modevent, NULL); MODULE_VERSION(virtio_console, 1); MODULE_DEPEND(virtio_console, virtio, 1, 1, 1); VIRTIO_SIMPLE_PNPINFO(virtio_console, VIRTIO_ID_CONSOLE, "VirtIO Console Adapter"); static int vtcon_modevent(module_t mod, int type, void *unused) { int error; switch (type) { case MOD_LOAD: error = 0; break; case MOD_QUIESCE: error = 0; break; case MOD_UNLOAD: vtcon_drain_all(); error = 0; break; case MOD_SHUTDOWN: error = 0; break; default: error = EOPNOTSUPP; break; } return (error); } static void vtcon_drain_all(void) { int first; for (first = 1; vtcon_pending_free != 0; first = 0) { if (first != 0) { printf("virtio_console: Waiting for all detached TTY " "devices to have open fds closed.\n"); } pause("vtcondra", hz); } } static int vtcon_probe(device_t dev) { return (VIRTIO_SIMPLE_PROBE(dev, virtio_console)); } static int vtcon_attach(device_t dev) { struct vtcon_softc *sc; struct virtio_console_config concfg; int error; sc = device_get_softc(dev); sc->vtcon_dev = dev; virtio_set_feature_desc(dev, vtcon_feature_desc); mtx_init(&sc->vtcon_mtx, "vtconmtx", NULL, MTX_DEF); mtx_init(&sc->vtcon_ctrl_tx_mtx, "vtconctrlmtx", NULL, MTX_DEF); error = vtcon_setup_features(sc); if (error) { device_printf(dev, "cannot setup features\n"); goto fail; } vtcon_read_config(sc, &concfg); vtcon_determine_max_ports(sc, &concfg); error = vtcon_alloc_scports(sc); if (error) { device_printf(dev, "cannot allocate softc port structures\n"); goto fail; } error = vtcon_alloc_virtqueues(sc); if (error) { device_printf(dev, "cannot allocate virtqueues\n"); goto fail; } if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) { TASK_INIT(&sc->vtcon_ctrl_task, 0, vtcon_ctrl_task_cb, sc); error = vtcon_ctrl_init(sc); if (error) goto fail; } else { error = vtcon_port_create(sc, 0); if (error) goto fail; if (sc->vtcon_flags & VTCON_FLAG_SIZE) vtcon_port_update_console_size(sc); } error = virtio_setup_intr(dev, INTR_TYPE_TTY); if (error) { device_printf(dev, "cannot setup virtqueue interrupts\n"); goto fail; } vtcon_enable_interrupts(sc); vtcon_ctrl_send_control(sc, VIRTIO_CONSOLE_BAD_ID, VIRTIO_CONSOLE_DEVICE_READY, 1); fail: if (error) vtcon_detach(dev); return (error); } static int vtcon_detach(device_t dev) { struct vtcon_softc *sc; sc = device_get_softc(dev); VTCON_LOCK(sc); sc->vtcon_flags |= VTCON_FLAG_DETACHED; if (device_is_attached(dev)) vtcon_stop(sc); VTCON_UNLOCK(sc); if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) { taskqueue_drain(taskqueue_thread, &sc->vtcon_ctrl_task); vtcon_ctrl_deinit(sc); } vtcon_destroy_ports(sc); mtx_destroy(&sc->vtcon_mtx); mtx_destroy(&sc->vtcon_ctrl_tx_mtx); return (0); } static int vtcon_config_change(device_t dev) { struct vtcon_softc *sc; sc = device_get_softc(dev); /* * When the multiport feature is negotiated, all configuration * changes are done through control virtqueue events. */ if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0) { if (sc->vtcon_flags & VTCON_FLAG_SIZE) vtcon_port_update_console_size(sc); } return (0); } static int vtcon_negotiate_features(struct vtcon_softc *sc) { device_t dev; uint64_t features; dev = sc->vtcon_dev; features = VTCON_FEATURES; sc->vtcon_features = virtio_negotiate_features(dev, features); return (virtio_finalize_features(dev)); } static int vtcon_setup_features(struct vtcon_softc *sc) { device_t dev; int error; dev = sc->vtcon_dev; error = vtcon_negotiate_features(sc); if (error) return (error); if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_SIZE)) sc->vtcon_flags |= VTCON_FLAG_SIZE; if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_MULTIPORT)) sc->vtcon_flags |= VTCON_FLAG_MULTIPORT; return (0); } #define VTCON_GET_CONFIG(_dev, _feature, _field, _cfg) \ if (virtio_with_feature(_dev, _feature)) { \ virtio_read_device_config(_dev, \ offsetof(struct virtio_console_config, _field), \ &(_cfg)->_field, sizeof((_cfg)->_field)); \ } static void vtcon_read_config(struct vtcon_softc *sc, struct virtio_console_config *concfg) { device_t dev; dev = sc->vtcon_dev; bzero(concfg, sizeof(struct virtio_console_config)); VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, cols, concfg); VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, rows, concfg); VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_MULTIPORT, max_nr_ports, concfg); } #undef VTCON_GET_CONFIG static int vtcon_alloc_scports(struct vtcon_softc *sc) { struct vtcon_softc_port *scport; int max, i; max = sc->vtcon_max_ports; sc->vtcon_ports = malloc(sizeof(struct vtcon_softc_port) * max, M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->vtcon_ports == NULL) return (ENOMEM); for (i = 0; i < max; i++) { scport = &sc->vtcon_ports[i]; scport->vcsp_sc = sc; } return (0); } static int vtcon_alloc_virtqueues(struct vtcon_softc *sc) { device_t dev; struct vq_alloc_info *info; struct vtcon_softc_port *scport; int i, idx, portidx, nvqs, error; dev = sc->vtcon_dev; nvqs = sc->vtcon_max_ports * 2; if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) nvqs += 2; info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT); if (info == NULL) return (ENOMEM); for (i = 0, idx = 0, portidx = 0; i < nvqs / 2; i++, idx += 2) { if (i == 1) { /* The control virtqueues are after the first port. */ VQ_ALLOC_INFO_INIT(&info[idx], 0, vtcon_ctrl_event_intr, sc, &sc->vtcon_ctrl_rxvq, "%s-control rx", device_get_nameunit(dev)); VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL, sc, &sc->vtcon_ctrl_txvq, "%s-control tx", device_get_nameunit(dev)); continue; } scport = &sc->vtcon_ports[portidx]; VQ_ALLOC_INFO_INIT(&info[idx], 0, vtcon_port_intr, scport, &scport->vcsp_invq, "%s-port%d in", device_get_nameunit(dev), i); VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL, NULL, &scport->vcsp_outvq, "%s-port%d out", device_get_nameunit(dev), i); portidx++; } - error = virtio_alloc_virtqueues(dev, 0, nvqs, info); + error = virtio_alloc_virtqueues(dev, nvqs, info); free(info, M_TEMP); return (error); } static void vtcon_determine_max_ports(struct vtcon_softc *sc, struct virtio_console_config *concfg) { if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) { sc->vtcon_max_ports = min(concfg->max_nr_ports, VTCON_MAX_PORTS); if (sc->vtcon_max_ports == 0) sc->vtcon_max_ports = 1; } else sc->vtcon_max_ports = 1; } static void vtcon_destroy_ports(struct vtcon_softc *sc) { struct vtcon_softc_port *scport; struct vtcon_port *port; struct virtqueue *vq; int i; if (sc->vtcon_ports == NULL) return; VTCON_LOCK(sc); for (i = 0; i < sc->vtcon_max_ports; i++) { scport = &sc->vtcon_ports[i]; port = scport->vcsp_port; if (port != NULL) { scport->vcsp_port = NULL; VTCON_PORT_LOCK(port); VTCON_UNLOCK(sc); vtcon_port_teardown(port); VTCON_LOCK(sc); } vq = scport->vcsp_invq; if (vq != NULL) vtcon_port_drain_bufs(vq); } VTCON_UNLOCK(sc); free(sc->vtcon_ports, M_DEVBUF); sc->vtcon_ports = NULL; } static void vtcon_stop(struct vtcon_softc *sc) { vtcon_disable_interrupts(sc); virtio_stop(sc->vtcon_dev); } static int vtcon_ctrl_event_enqueue(struct vtcon_softc *sc, struct virtio_console_control *control) { struct sglist_seg segs[2]; struct sglist sg; struct virtqueue *vq; int error __diagused; vq = sc->vtcon_ctrl_rxvq; sglist_init(&sg, 2, segs); error = sglist_append(&sg, control, VTCON_CTRL_BUFSZ); KASSERT(error == 0, ("%s: error %d adding control to sglist", __func__, error)); return (virtqueue_enqueue(vq, control, &sg, 0, sg.sg_nseg)); } static int vtcon_ctrl_event_create(struct vtcon_softc *sc) { struct virtio_console_control *control; int error; control = malloc(VTCON_CTRL_BUFSZ, M_DEVBUF, M_ZERO | M_NOWAIT); if (control == NULL) return (ENOMEM); error = vtcon_ctrl_event_enqueue(sc, control); if (error) free(control, M_DEVBUF); return (error); } static void vtcon_ctrl_event_requeue(struct vtcon_softc *sc, struct virtio_console_control *control) { int error __diagused; bzero(control, VTCON_CTRL_BUFSZ); error = vtcon_ctrl_event_enqueue(sc, control); KASSERT(error == 0, ("%s: cannot requeue control buffer %d", __func__, error)); } static int vtcon_ctrl_event_populate(struct vtcon_softc *sc) { struct virtqueue *vq; int nbufs, error; vq = sc->vtcon_ctrl_rxvq; error = ENOSPC; for (nbufs = 0; !virtqueue_full(vq); nbufs++) { error = vtcon_ctrl_event_create(sc); if (error) break; } if (nbufs > 0) { virtqueue_notify(vq); error = 0; } return (error); } static void vtcon_ctrl_event_drain(struct vtcon_softc *sc) { struct virtio_console_control *control; struct virtqueue *vq; int last; vq = sc->vtcon_ctrl_rxvq; last = 0; if (vq == NULL) return; VTCON_LOCK(sc); while ((control = virtqueue_drain(vq, &last)) != NULL) free(control, M_DEVBUF); VTCON_UNLOCK(sc); } static int vtcon_ctrl_init(struct vtcon_softc *sc) { int error; error = vtcon_ctrl_event_populate(sc); return (error); } static void vtcon_ctrl_deinit(struct vtcon_softc *sc) { vtcon_ctrl_event_drain(sc); } static void vtcon_ctrl_port_add_event(struct vtcon_softc *sc, int id) { device_t dev; int error; dev = sc->vtcon_dev; /* This single thread only way for ports to be created. */ if (sc->vtcon_ports[id].vcsp_port != NULL) { device_printf(dev, "%s: adding port %d, but already exists\n", __func__, id); return; } error = vtcon_port_create(sc, id); if (error) { device_printf(dev, "%s: cannot create port %d: %d\n", __func__, id, error); vtcon_ctrl_send_control(sc, id, VIRTIO_CONSOLE_PORT_READY, 0); return; } } static void vtcon_ctrl_port_remove_event(struct vtcon_softc *sc, int id) { device_t dev; struct vtcon_softc_port *scport; struct vtcon_port *port; dev = sc->vtcon_dev; scport = &sc->vtcon_ports[id]; VTCON_LOCK(sc); port = scport->vcsp_port; if (port == NULL) { VTCON_UNLOCK(sc); device_printf(dev, "%s: remove port %d, but does not exist\n", __func__, id); return; } scport->vcsp_port = NULL; VTCON_PORT_LOCK(port); VTCON_UNLOCK(sc); vtcon_port_teardown(port); } static void vtcon_ctrl_port_console_event(struct vtcon_softc *sc, int id) { device_t dev; struct vtcon_softc_port *scport; struct vtcon_port *port; dev = sc->vtcon_dev; scport = &sc->vtcon_ports[id]; VTCON_LOCK(sc); port = scport->vcsp_port; if (port == NULL) { VTCON_UNLOCK(sc); device_printf(dev, "%s: console port %d, but does not exist\n", __func__, id); return; } VTCON_PORT_LOCK(port); VTCON_UNLOCK(sc); port->vtcport_flags |= VTCON_PORT_FLAG_CONSOLE; vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1); VTCON_PORT_UNLOCK(port); } static void vtcon_ctrl_port_open_event(struct vtcon_softc *sc, int id) { device_t dev; struct vtcon_softc_port *scport; struct vtcon_port *port; dev = sc->vtcon_dev; scport = &sc->vtcon_ports[id]; VTCON_LOCK(sc); port = scport->vcsp_port; if (port == NULL) { VTCON_UNLOCK(sc); device_printf(dev, "%s: open port %d, but does not exist\n", __func__, id); return; } VTCON_PORT_LOCK(port); VTCON_UNLOCK(sc); vtcon_port_enable_intr(port); VTCON_PORT_UNLOCK(port); } static void vtcon_ctrl_port_name_event(struct vtcon_softc *sc, int id, const char *name, size_t len) { device_t dev; struct vtcon_softc_port *scport; struct vtcon_port *port; dev = sc->vtcon_dev; scport = &sc->vtcon_ports[id]; /* * The VirtIO specification says the NUL terminator is not included in * the length, but QEMU includes it. Adjust the length if needed. */ if (name == NULL || len == 0) return; if (name[len - 1] == '\0') { len--; if (len == 0) return; } VTCON_LOCK(sc); port = scport->vcsp_port; if (port == NULL) { VTCON_UNLOCK(sc); device_printf(dev, "%s: name port %d, but does not exist\n", __func__, id); return; } VTCON_PORT_LOCK(port); VTCON_UNLOCK(sc); vtcon_port_dev_alias(port, name, len); VTCON_PORT_UNLOCK(port); } static void vtcon_ctrl_process_event(struct vtcon_softc *sc, struct virtio_console_control *control, void *data, size_t data_len) { device_t dev; uint32_t id; uint16_t event; dev = sc->vtcon_dev; id = vtcon_htog32(sc, control->id); event = vtcon_htog16(sc, control->event); if (id >= sc->vtcon_max_ports) { device_printf(dev, "%s: event %d invalid port ID %d\n", __func__, event, id); return; } switch (event) { case VIRTIO_CONSOLE_PORT_ADD: vtcon_ctrl_port_add_event(sc, id); break; case VIRTIO_CONSOLE_PORT_REMOVE: vtcon_ctrl_port_remove_event(sc, id); break; case VIRTIO_CONSOLE_CONSOLE_PORT: vtcon_ctrl_port_console_event(sc, id); break; case VIRTIO_CONSOLE_RESIZE: break; case VIRTIO_CONSOLE_PORT_OPEN: vtcon_ctrl_port_open_event(sc, id); break; case VIRTIO_CONSOLE_PORT_NAME: vtcon_ctrl_port_name_event(sc, id, (const char *)data, data_len); break; } } static void vtcon_ctrl_task_cb(void *xsc, int pending) { struct vtcon_softc *sc; struct virtqueue *vq; struct virtio_console_control *control; void *data; size_t data_len; int detached; uint32_t len; sc = xsc; vq = sc->vtcon_ctrl_rxvq; VTCON_LOCK(sc); while ((detached = (sc->vtcon_flags & VTCON_FLAG_DETACHED)) == 0) { control = virtqueue_dequeue(vq, &len); if (control == NULL) break; if (len > sizeof(struct virtio_console_control)) { data = (void *) &control[1]; data_len = len - sizeof(struct virtio_console_control); } else { data = NULL; data_len = 0; } VTCON_UNLOCK(sc); vtcon_ctrl_process_event(sc, control, data, data_len); VTCON_LOCK(sc); vtcon_ctrl_event_requeue(sc, control); } if (!detached) { virtqueue_notify(vq); if (virtqueue_enable_intr(vq) != 0) taskqueue_enqueue(taskqueue_thread, &sc->vtcon_ctrl_task); } VTCON_UNLOCK(sc); } static void vtcon_ctrl_event_intr(void *xsc) { struct vtcon_softc *sc; sc = xsc; /* * Only some events require us to potentially block, but it * easier to just defer all event handling to the taskqueue. */ taskqueue_enqueue(taskqueue_thread, &sc->vtcon_ctrl_task); } static void vtcon_ctrl_poll(struct vtcon_softc *sc, struct virtio_console_control *control) { struct sglist_seg segs[2]; struct sglist sg; struct virtqueue *vq; int error; vq = sc->vtcon_ctrl_txvq; sglist_init(&sg, 2, segs); error = sglist_append(&sg, control, sizeof(struct virtio_console_control)); KASSERT(error == 0, ("%s: error %d adding control to sglist", __func__, error)); /* * We cannot use the softc lock to serialize access to this * virtqueue since this is called from the tty layer with the * port lock held. Acquiring the softc would violate our lock * ordering. */ VTCON_CTRL_TX_LOCK(sc); KASSERT(virtqueue_empty(vq), ("%s: virtqueue is not emtpy", __func__)); error = virtqueue_enqueue(vq, control, &sg, sg.sg_nseg, 0); if (error == 0) { virtqueue_notify(vq); virtqueue_poll(vq, NULL); } VTCON_CTRL_TX_UNLOCK(sc); } static void vtcon_ctrl_send_control(struct vtcon_softc *sc, uint32_t portid, uint16_t event, uint16_t value) { struct virtio_console_control control; if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0) return; control.id = vtcon_gtoh32(sc, portid); control.event = vtcon_gtoh16(sc, event); control.value = vtcon_gtoh16(sc, value); vtcon_ctrl_poll(sc, &control); } static int vtcon_port_enqueue_buf(struct vtcon_port *port, void *buf, size_t len) { struct sglist_seg segs[2]; struct sglist sg; struct virtqueue *vq; int error; vq = port->vtcport_invq; sglist_init(&sg, 2, segs); error = sglist_append(&sg, buf, len); KASSERT(error == 0, ("%s: error %d adding buffer to sglist", __func__, error)); error = virtqueue_enqueue(vq, buf, &sg, 0, sg.sg_nseg); return (error); } static int vtcon_port_create_buf(struct vtcon_port *port) { void *buf; int error; buf = malloc(VTCON_BULK_BUFSZ, M_DEVBUF, M_ZERO | M_NOWAIT); if (buf == NULL) return (ENOMEM); error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ); if (error) free(buf, M_DEVBUF); return (error); } static void vtcon_port_requeue_buf(struct vtcon_port *port, void *buf) { int error __diagused; error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ); KASSERT(error == 0, ("%s: cannot requeue input buffer %d", __func__, error)); } static int vtcon_port_populate(struct vtcon_port *port) { struct virtqueue *vq; int nbufs, error; vq = port->vtcport_invq; error = ENOSPC; for (nbufs = 0; !virtqueue_full(vq); nbufs++) { error = vtcon_port_create_buf(port); if (error) break; } if (nbufs > 0) { virtqueue_notify(vq); error = 0; } return (error); } static void vtcon_port_destroy(struct vtcon_port *port) { port->vtcport_sc = NULL; port->vtcport_scport = NULL; port->vtcport_invq = NULL; port->vtcport_outvq = NULL; port->vtcport_id = -1; mtx_destroy(&port->vtcport_mtx); free(port, M_DEVBUF); } static int vtcon_port_init_vqs(struct vtcon_port *port) { struct vtcon_softc_port *scport; int error; scport = port->vtcport_scport; port->vtcport_invq = scport->vcsp_invq; port->vtcport_outvq = scport->vcsp_outvq; /* * Free any data left over from when this virtqueue was in use by a * prior port. We have not yet notified the host that the port is * ready, so assume nothing in the virtqueue can be for us. */ vtcon_port_drain(port); KASSERT(virtqueue_empty(port->vtcport_invq), ("%s: in virtqueue is not empty", __func__)); KASSERT(virtqueue_empty(port->vtcport_outvq), ("%s: out virtqueue is not empty", __func__)); error = vtcon_port_populate(port); if (error) return (error); return (0); } static int vtcon_port_create(struct vtcon_softc *sc, int id) { device_t dev; struct vtcon_softc_port *scport; struct vtcon_port *port; int error; dev = sc->vtcon_dev; scport = &sc->vtcon_ports[id]; VTCON_ASSERT_VALID_PORTID(sc, id); MPASS(scport->vcsp_port == NULL); port = malloc(sizeof(struct vtcon_port), M_DEVBUF, M_NOWAIT | M_ZERO); if (port == NULL) return (ENOMEM); port->vtcport_sc = sc; port->vtcport_scport = scport; port->vtcport_id = id; mtx_init(&port->vtcport_mtx, "vtcpmtx", NULL, MTX_DEF); port->vtcport_tty = tty_alloc_mutex(&vtcon_tty_class, port, &port->vtcport_mtx); error = vtcon_port_init_vqs(port); if (error) { VTCON_PORT_LOCK(port); vtcon_port_teardown(port); return (error); } VTCON_LOCK(sc); VTCON_PORT_LOCK(port); scport->vcsp_port = port; vtcon_port_enable_intr(port); vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_READY, 1); VTCON_PORT_UNLOCK(port); VTCON_UNLOCK(sc); tty_makedev(port->vtcport_tty, NULL, "%s%r.%r", VTCON_TTY_PREFIX, device_get_unit(dev), id); return (0); } static void vtcon_port_dev_alias(struct vtcon_port *port, const char *name, size_t len) { struct vtcon_softc *sc; struct cdev *pdev; struct tty *tp; int i, error; sc = port->vtcport_sc; tp = port->vtcport_tty; if (port->vtcport_flags & VTCON_PORT_FLAG_ALIAS) return; /* Port name is UTF-8, but we can only handle ASCII. */ for (i = 0; i < len; i++) { if (!isascii(name[i])) return; } /* * Port name may not conform to the devfs requirements so we cannot use * tty_makealias() because the MAKEDEV_CHECKNAME flag must be specified. */ error = make_dev_alias_p(MAKEDEV_NOWAIT | MAKEDEV_CHECKNAME, &pdev, tp->t_dev, "%s/%*s", VTCON_TTY_ALIAS_PREFIX, (int)len, name); if (error) { device_printf(sc->vtcon_dev, "%s: cannot make dev alias (%s/%*s) error %d\n", __func__, VTCON_TTY_ALIAS_PREFIX, (int)len, name, error); } else port->vtcport_flags |= VTCON_PORT_FLAG_ALIAS; } static void vtcon_port_drain_bufs(struct virtqueue *vq) { void *buf; int last; last = 0; while ((buf = virtqueue_drain(vq, &last)) != NULL) free(buf, M_DEVBUF); } static void vtcon_port_drain(struct vtcon_port *port) { vtcon_port_drain_bufs(port->vtcport_invq); } static void vtcon_port_teardown(struct vtcon_port *port) { struct tty *tp; tp = port->vtcport_tty; port->vtcport_flags |= VTCON_PORT_FLAG_GONE; if (tp != NULL) { atomic_add_int(&vtcon_pending_free, 1); tty_rel_gone(tp); } else vtcon_port_destroy(port); } static void vtcon_port_change_size(struct vtcon_port *port, uint16_t cols, uint16_t rows) { struct tty *tp; struct winsize sz; tp = port->vtcport_tty; if (tp == NULL) return; bzero(&sz, sizeof(struct winsize)); sz.ws_col = cols; sz.ws_row = rows; tty_set_winsize(tp, &sz); } static void vtcon_port_update_console_size(struct vtcon_softc *sc) { struct vtcon_port *port; struct vtcon_softc_port *scport; uint16_t cols, rows; vtcon_get_console_size(sc, &cols, &rows); /* * For now, assume the first (only) port is the console. Note * QEMU does not implement this feature yet. */ scport = &sc->vtcon_ports[0]; VTCON_LOCK(sc); port = scport->vcsp_port; if (port != NULL) { VTCON_PORT_LOCK(port); VTCON_UNLOCK(sc); vtcon_port_change_size(port, cols, rows); VTCON_PORT_UNLOCK(port); } else VTCON_UNLOCK(sc); } static void vtcon_port_enable_intr(struct vtcon_port *port) { /* * NOTE: The out virtqueue is always polled, so its interrupt * kept disabled. */ virtqueue_enable_intr(port->vtcport_invq); } static void vtcon_port_disable_intr(struct vtcon_port *port) { if (port->vtcport_invq != NULL) virtqueue_disable_intr(port->vtcport_invq); if (port->vtcport_outvq != NULL) virtqueue_disable_intr(port->vtcport_outvq); } static void vtcon_port_in(struct vtcon_port *port) { struct virtqueue *vq; struct tty *tp; char *buf; uint32_t len; int i, deq; tp = port->vtcport_tty; vq = port->vtcport_invq; again: deq = 0; while ((buf = virtqueue_dequeue(vq, &len)) != NULL) { for (i = 0; i < len; i++) { #if defined(KDB) if (port->vtcport_flags & VTCON_PORT_FLAG_CONSOLE) kdb_alt_break(buf[i], &port->vtcport_alt_break_state); #endif ttydisc_rint(tp, buf[i], 0); } vtcon_port_requeue_buf(port, buf); deq++; } ttydisc_rint_done(tp); if (deq > 0) virtqueue_notify(vq); if (virtqueue_enable_intr(vq) != 0) goto again; } static void vtcon_port_intr(void *scportx) { struct vtcon_softc_port *scport; struct vtcon_softc *sc; struct vtcon_port *port; scport = scportx; sc = scport->vcsp_sc; VTCON_LOCK(sc); port = scport->vcsp_port; if (port == NULL) { VTCON_UNLOCK(sc); return; } VTCON_PORT_LOCK(port); VTCON_UNLOCK(sc); if ((port->vtcport_flags & VTCON_PORT_FLAG_GONE) == 0) vtcon_port_in(port); VTCON_PORT_UNLOCK(port); } static void vtcon_port_out(struct vtcon_port *port, void *buf, int bufsize) { struct sglist_seg segs[2]; struct sglist sg; struct virtqueue *vq; int error; vq = port->vtcport_outvq; KASSERT(virtqueue_empty(vq), ("%s: port %p out virtqueue not emtpy", __func__, port)); sglist_init(&sg, 2, segs); error = sglist_append(&sg, buf, bufsize); KASSERT(error == 0, ("%s: error %d adding buffer to sglist", __func__, error)); error = virtqueue_enqueue(vq, buf, &sg, sg.sg_nseg, 0); if (error == 0) { virtqueue_notify(vq); virtqueue_poll(vq, NULL); } } static void vtcon_port_submit_event(struct vtcon_port *port, uint16_t event, uint16_t value) { struct vtcon_softc *sc; sc = port->vtcport_sc; vtcon_ctrl_send_control(sc, port->vtcport_id, event, value); } static int vtcon_tty_open(struct tty *tp) { struct vtcon_port *port; port = tty_softc(tp); if (port->vtcport_flags & VTCON_PORT_FLAG_GONE) return (ENXIO); vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1); return (0); } static void vtcon_tty_close(struct tty *tp) { struct vtcon_port *port; port = tty_softc(tp); if (port->vtcport_flags & VTCON_PORT_FLAG_GONE) return; vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 0); } static void vtcon_tty_outwakeup(struct tty *tp) { struct vtcon_port *port; char buf[VTCON_BULK_BUFSZ]; int len; port = tty_softc(tp); if (port->vtcport_flags & VTCON_PORT_FLAG_GONE) return; while ((len = ttydisc_getc(tp, buf, sizeof(buf))) != 0) vtcon_port_out(port, buf, len); } static void vtcon_tty_free(void *xport) { struct vtcon_port *port; port = xport; vtcon_port_destroy(port); atomic_subtract_int(&vtcon_pending_free, 1); } static void vtcon_get_console_size(struct vtcon_softc *sc, uint16_t *cols, uint16_t *rows) { struct virtio_console_config concfg; KASSERT(sc->vtcon_flags & VTCON_FLAG_SIZE, ("%s: size feature not negotiated", __func__)); vtcon_read_config(sc, &concfg); *cols = concfg.cols; *rows = concfg.rows; } static void vtcon_enable_interrupts(struct vtcon_softc *sc) { struct vtcon_softc_port *scport; struct vtcon_port *port; int i; VTCON_LOCK(sc); if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) virtqueue_enable_intr(sc->vtcon_ctrl_rxvq); for (i = 0; i < sc->vtcon_max_ports; i++) { scport = &sc->vtcon_ports[i]; port = scport->vcsp_port; if (port == NULL) continue; VTCON_PORT_LOCK(port); vtcon_port_enable_intr(port); VTCON_PORT_UNLOCK(port); } VTCON_UNLOCK(sc); } static void vtcon_disable_interrupts(struct vtcon_softc *sc) { struct vtcon_softc_port *scport; struct vtcon_port *port; int i; VTCON_LOCK_ASSERT(sc); if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) virtqueue_disable_intr(sc->vtcon_ctrl_rxvq); for (i = 0; i < sc->vtcon_max_ports; i++) { scport = &sc->vtcon_ports[i]; port = scport->vcsp_port; if (port == NULL) continue; VTCON_PORT_LOCK(port); vtcon_port_disable_intr(port); VTCON_PORT_UNLOCK(port); } } diff --git a/sys/dev/virtio/gpu/virtio_gpu.c b/sys/dev/virtio/gpu/virtio_gpu.c index 0472bc98b3ba..0009d7bb3ec1 100644 --- a/sys/dev/virtio/gpu/virtio_gpu.c +++ b/sys/dev/virtio/gpu/virtio_gpu.c @@ -1,697 +1,697 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013, Bryan Venteicher * All rights reserved. * Copyright (c) 2023, Arm Ltd * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for VirtIO GPU device. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fb_if.h" #define VTGPU_FEATURES 0 /* The guest can allocate resource IDs, we only need one */ #define VTGPU_RESOURCE_ID 1 struct vtgpu_softc { /* Must be first so we can cast from info -> softc */ struct fb_info vtgpu_fb_info; struct virtio_gpu_config vtgpu_gpucfg; device_t vtgpu_dev; uint64_t vtgpu_features; struct virtqueue *vtgpu_ctrl_vq; uint64_t vtgpu_next_fence; bool vtgpu_have_fb_info; }; static int vtgpu_modevent(module_t, int, void *); static int vtgpu_probe(device_t); static int vtgpu_attach(device_t); static int vtgpu_detach(device_t); static int vtgpu_negotiate_features(struct vtgpu_softc *); static int vtgpu_setup_features(struct vtgpu_softc *); static void vtgpu_read_config(struct vtgpu_softc *, struct virtio_gpu_config *); static int vtgpu_alloc_virtqueue(struct vtgpu_softc *); static int vtgpu_get_display_info(struct vtgpu_softc *); static int vtgpu_create_2d(struct vtgpu_softc *); static int vtgpu_attach_backing(struct vtgpu_softc *); static int vtgpu_set_scanout(struct vtgpu_softc *, uint32_t, uint32_t, uint32_t, uint32_t); static int vtgpu_transfer_to_host_2d(struct vtgpu_softc *, uint32_t, uint32_t, uint32_t, uint32_t); static int vtgpu_resource_flush(struct vtgpu_softc *, uint32_t, uint32_t, uint32_t, uint32_t); static vd_blank_t vtgpu_fb_blank; static vd_bitblt_text_t vtgpu_fb_bitblt_text; static vd_bitblt_bmp_t vtgpu_fb_bitblt_bitmap; static vd_drawrect_t vtgpu_fb_drawrect; static vd_setpixel_t vtgpu_fb_setpixel; static struct vt_driver vtgpu_fb_driver = { .vd_name = "virtio_gpu", .vd_init = vt_fb_init, .vd_fini = vt_fb_fini, .vd_blank = vtgpu_fb_blank, .vd_bitblt_text = vtgpu_fb_bitblt_text, .vd_invalidate_text = vt_fb_invalidate_text, .vd_bitblt_bmp = vtgpu_fb_bitblt_bitmap, .vd_drawrect = vtgpu_fb_drawrect, .vd_setpixel = vtgpu_fb_setpixel, .vd_postswitch = vt_fb_postswitch, .vd_priority = VD_PRIORITY_GENERIC+10, .vd_fb_ioctl = vt_fb_ioctl, .vd_fb_mmap = NULL, /* No mmap as we need to signal the host */ .vd_suspend = vt_fb_suspend, .vd_resume = vt_fb_resume, }; VT_DRIVER_DECLARE(vt_vtgpu, vtgpu_fb_driver); static void vtgpu_fb_blank(struct vt_device *vd, term_color_t color) { struct vtgpu_softc *sc; struct fb_info *info; info = vd->vd_softc; sc = (struct vtgpu_softc *)info; vt_fb_blank(vd, color); vtgpu_transfer_to_host_2d(sc, 0, 0, sc->vtgpu_fb_info.fb_width, sc->vtgpu_fb_info.fb_height); vtgpu_resource_flush(sc, 0, 0, sc->vtgpu_fb_info.fb_width, sc->vtgpu_fb_info.fb_height); } static void vtgpu_fb_bitblt_text(struct vt_device *vd, const struct vt_window *vw, const term_rect_t *area) { struct vtgpu_softc *sc; struct fb_info *info; int x, y, width, height; info = vd->vd_softc; sc = (struct vtgpu_softc *)info; vt_fb_bitblt_text(vd, vw, area); x = area->tr_begin.tp_col * vw->vw_font->vf_width + vw->vw_draw_area.tr_begin.tp_col; y = area->tr_begin.tp_row * vw->vw_font->vf_height + vw->vw_draw_area.tr_begin.tp_row; width = area->tr_end.tp_col * vw->vw_font->vf_width + vw->vw_draw_area.tr_begin.tp_col - x; height = area->tr_end.tp_row * vw->vw_font->vf_height + vw->vw_draw_area.tr_begin.tp_row - y; vtgpu_transfer_to_host_2d(sc, x, y, width, height); vtgpu_resource_flush(sc, x, y, width, height); } static void vtgpu_fb_bitblt_bitmap(struct vt_device *vd, const struct vt_window *vw, const uint8_t *pattern, const uint8_t *mask, unsigned int width, unsigned int height, unsigned int x, unsigned int y, term_color_t fg, term_color_t bg) { struct vtgpu_softc *sc; struct fb_info *info; info = vd->vd_softc; sc = (struct vtgpu_softc *)info; vt_fb_bitblt_bitmap(vd, vw, pattern, mask, width, height, x, y, fg, bg); vtgpu_transfer_to_host_2d(sc, x, y, width, height); vtgpu_resource_flush(sc, x, y, width, height); } static void vtgpu_fb_drawrect(struct vt_device *vd, int x1, int y1, int x2, int y2, int fill, term_color_t color) { struct vtgpu_softc *sc; struct fb_info *info; int width, height; info = vd->vd_softc; sc = (struct vtgpu_softc *)info; vt_fb_drawrect(vd, x1, y1, x2, y2, fill, color); width = x2 - x1 + 1; height = y2 - y1 + 1; vtgpu_transfer_to_host_2d(sc, x1, y1, width, height); vtgpu_resource_flush(sc, x1, y1, width, height); } static void vtgpu_fb_setpixel(struct vt_device *vd, int x, int y, term_color_t color) { struct vtgpu_softc *sc; struct fb_info *info; info = vd->vd_softc; sc = (struct vtgpu_softc *)info; vt_fb_setpixel(vd, x, y, color); vtgpu_transfer_to_host_2d(sc, x, y, 1, 1); vtgpu_resource_flush(sc, x, y, 1, 1); } static struct virtio_feature_desc vtgpu_feature_desc[] = { { VIRTIO_GPU_F_VIRGL, "VirGL" }, { VIRTIO_GPU_F_EDID, "EDID" }, { VIRTIO_GPU_F_RESOURCE_UUID, "ResUUID" }, { VIRTIO_GPU_F_RESOURCE_BLOB, "ResBlob" }, { VIRTIO_GPU_F_CONTEXT_INIT, "ContextInit" }, { 0, NULL } }; static device_method_t vtgpu_methods[] = { /* Device methods. */ DEVMETHOD(device_probe, vtgpu_probe), DEVMETHOD(device_attach, vtgpu_attach), DEVMETHOD(device_detach, vtgpu_detach), DEVMETHOD_END }; static driver_t vtgpu_driver = { "vtgpu", vtgpu_methods, sizeof(struct vtgpu_softc) }; VIRTIO_DRIVER_MODULE(virtio_gpu, vtgpu_driver, vtgpu_modevent, NULL); MODULE_VERSION(virtio_gpu, 1); MODULE_DEPEND(virtio_gpu, virtio, 1, 1, 1); VIRTIO_SIMPLE_PNPINFO(virtio_gpu, VIRTIO_ID_GPU, "VirtIO GPU"); static int vtgpu_modevent(module_t mod, int type, void *unused) { int error; switch (type) { case MOD_LOAD: case MOD_QUIESCE: case MOD_UNLOAD: case MOD_SHUTDOWN: error = 0; break; default: error = EOPNOTSUPP; break; } return (error); } static int vtgpu_probe(device_t dev) { return (VIRTIO_SIMPLE_PROBE(dev, virtio_gpu)); } static int vtgpu_attach(device_t dev) { struct vtgpu_softc *sc; int error; sc = device_get_softc(dev); sc->vtgpu_have_fb_info = false; sc->vtgpu_dev = dev; sc->vtgpu_next_fence = 1; virtio_set_feature_desc(dev, vtgpu_feature_desc); error = vtgpu_setup_features(sc); if (error != 0) { device_printf(dev, "cannot setup features\n"); goto fail; } vtgpu_read_config(sc, &sc->vtgpu_gpucfg); error = vtgpu_alloc_virtqueue(sc); if (error != 0) { device_printf(dev, "cannot allocate virtqueue\n"); goto fail; } virtio_setup_intr(dev, INTR_TYPE_TTY); /* Read the device info to get the display size */ error = vtgpu_get_display_info(sc); if (error != 0) { goto fail; } /* * TODO: This doesn't need to be contigmalloc as we * can use scatter-gather lists. */ sc->vtgpu_fb_info.fb_vbase = (vm_offset_t)contigmalloc( sc->vtgpu_fb_info.fb_size, M_DEVBUF, M_WAITOK|M_ZERO, 0, ~0, 4, 0); sc->vtgpu_fb_info.fb_pbase = pmap_kextract(sc->vtgpu_fb_info.fb_vbase); /* Create the 2d resource */ error = vtgpu_create_2d(sc); if (error != 0) { goto fail; } /* Attach the backing memory */ error = vtgpu_attach_backing(sc); if (error != 0) { goto fail; } /* Set the scanout to link the framebuffer to the display scanout */ error = vtgpu_set_scanout(sc, 0, 0, sc->vtgpu_fb_info.fb_width, sc->vtgpu_fb_info.fb_height); if (error != 0) { goto fail; } vt_allocate(&vtgpu_fb_driver, &sc->vtgpu_fb_info); sc->vtgpu_have_fb_info = true; error = vtgpu_transfer_to_host_2d(sc, 0, 0, sc->vtgpu_fb_info.fb_width, sc->vtgpu_fb_info.fb_height); if (error != 0) goto fail; error = vtgpu_resource_flush(sc, 0, 0, sc->vtgpu_fb_info.fb_width, sc->vtgpu_fb_info.fb_height); fail: if (error != 0) vtgpu_detach(dev); return (error); } static int vtgpu_detach(device_t dev) { struct vtgpu_softc *sc; sc = device_get_softc(dev); if (sc->vtgpu_have_fb_info) vt_deallocate(&vtgpu_fb_driver, &sc->vtgpu_fb_info); if (sc->vtgpu_fb_info.fb_vbase != 0) { MPASS(sc->vtgpu_fb_info.fb_size != 0); contigfree((void *)sc->vtgpu_fb_info.fb_vbase, sc->vtgpu_fb_info.fb_size, M_DEVBUF); } /* TODO: Tell the host we are detaching */ return (0); } static int vtgpu_negotiate_features(struct vtgpu_softc *sc) { device_t dev; uint64_t features; dev = sc->vtgpu_dev; features = VTGPU_FEATURES; sc->vtgpu_features = virtio_negotiate_features(dev, features); return (virtio_finalize_features(dev)); } static int vtgpu_setup_features(struct vtgpu_softc *sc) { int error; error = vtgpu_negotiate_features(sc); if (error != 0) return (error); return (0); } static void vtgpu_read_config(struct vtgpu_softc *sc, struct virtio_gpu_config *gpucfg) { device_t dev; dev = sc->vtgpu_dev; bzero(gpucfg, sizeof(struct virtio_gpu_config)); #define VTGPU_GET_CONFIG(_dev, _field, _cfg) \ virtio_read_device_config(_dev, \ offsetof(struct virtio_gpu_config, _field), \ &(_cfg)->_field, sizeof((_cfg)->_field)) \ VTGPU_GET_CONFIG(dev, events_read, gpucfg); VTGPU_GET_CONFIG(dev, events_clear, gpucfg); VTGPU_GET_CONFIG(dev, num_scanouts, gpucfg); VTGPU_GET_CONFIG(dev, num_capsets, gpucfg); #undef VTGPU_GET_CONFIG } static int vtgpu_alloc_virtqueue(struct vtgpu_softc *sc) { device_t dev; struct vq_alloc_info vq_info[2]; int nvqs; dev = sc->vtgpu_dev; nvqs = 1; VQ_ALLOC_INFO_INIT(&vq_info[0], 0, NULL, sc, &sc->vtgpu_ctrl_vq, "%s control", device_get_nameunit(dev)); - return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); + return (virtio_alloc_virtqueues(dev, nvqs, vq_info)); } static int vtgpu_req_resp(struct vtgpu_softc *sc, void *req, size_t reqlen, void *resp, size_t resplen) { struct sglist sg; struct sglist_seg segs[2]; int error; sglist_init(&sg, 2, segs); error = sglist_append(&sg, req, reqlen); if (error != 0) { device_printf(sc->vtgpu_dev, "Unable to append the request to the sglist: %d\n", error); return (error); } error = sglist_append(&sg, resp, resplen); if (error != 0) { device_printf(sc->vtgpu_dev, "Unable to append the response buffer to the sglist: %d\n", error); return (error); } error = virtqueue_enqueue(sc->vtgpu_ctrl_vq, resp, &sg, 1, 1); if (error != 0) { device_printf(sc->vtgpu_dev, "Enqueue failed: %d\n", error); return (error); } virtqueue_notify(sc->vtgpu_ctrl_vq); virtqueue_poll(sc->vtgpu_ctrl_vq, NULL); return (0); } static int vtgpu_get_display_info(struct vtgpu_softc *sc) { struct { struct virtio_gpu_ctrl_hdr req; char pad; struct virtio_gpu_resp_display_info resp; } s = { 0 }; int error; s.req.type = htole32(VIRTIO_GPU_CMD_GET_DISPLAY_INFO); s.req.flags = htole32(VIRTIO_GPU_FLAG_FENCE); s.req.fence_id = htole64(atomic_fetchadd_64(&sc->vtgpu_next_fence, 1)); error = vtgpu_req_resp(sc, &s.req, sizeof(s.req), &s.resp, sizeof(s.resp)); if (error != 0) return (error); for (int i = 0; i < sc->vtgpu_gpucfg.num_scanouts; i++) { if (s.resp.pmodes[i].enabled != 0) MPASS(i == 0); sc->vtgpu_fb_info.fb_name = device_get_nameunit(sc->vtgpu_dev); sc->vtgpu_fb_info.fb_width = le32toh(s.resp.pmodes[i].r.width); sc->vtgpu_fb_info.fb_height = le32toh(s.resp.pmodes[i].r.height); /* 32 bits per pixel */ sc->vtgpu_fb_info.fb_bpp = 32; sc->vtgpu_fb_info.fb_depth = 32; sc->vtgpu_fb_info.fb_size = sc->vtgpu_fb_info.fb_width * sc->vtgpu_fb_info.fb_height * 4; sc->vtgpu_fb_info.fb_stride = sc->vtgpu_fb_info.fb_width * 4; return (0); } return (ENXIO); } static int vtgpu_create_2d(struct vtgpu_softc *sc) { struct { struct virtio_gpu_resource_create_2d req; char pad; struct virtio_gpu_ctrl_hdr resp; } s = { 0 }; int error; s.req.hdr.type = htole32(VIRTIO_GPU_CMD_RESOURCE_CREATE_2D); s.req.hdr.flags = htole32(VIRTIO_GPU_FLAG_FENCE); s.req.hdr.fence_id = htole64( atomic_fetchadd_64(&sc->vtgpu_next_fence, 1)); s.req.resource_id = htole32(VTGPU_RESOURCE_ID); s.req.format = htole32(VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM); s.req.width = htole32(sc->vtgpu_fb_info.fb_width); s.req.height = htole32(sc->vtgpu_fb_info.fb_height); error = vtgpu_req_resp(sc, &s.req, sizeof(s.req), &s.resp, sizeof(s.resp)); if (error != 0) return (error); if (s.resp.type != htole32(VIRTIO_GPU_RESP_OK_NODATA)) { device_printf(sc->vtgpu_dev, "Invalid reponse type %x\n", le32toh(s.resp.type)); return (EINVAL); } return (0); } static int vtgpu_attach_backing(struct vtgpu_softc *sc) { struct { struct { struct virtio_gpu_resource_attach_backing backing; struct virtio_gpu_mem_entry mem[1]; } req; char pad; struct virtio_gpu_ctrl_hdr resp; } s = { 0 }; int error; s.req.backing.hdr.type = htole32(VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING); s.req.backing.hdr.flags = htole32(VIRTIO_GPU_FLAG_FENCE); s.req.backing.hdr.fence_id = htole64( atomic_fetchadd_64(&sc->vtgpu_next_fence, 1)); s.req.backing.resource_id = htole32(VTGPU_RESOURCE_ID); s.req.backing.nr_entries = htole32(1); s.req.mem[0].addr = htole32(sc->vtgpu_fb_info.fb_pbase); s.req.mem[0].length = htole32(sc->vtgpu_fb_info.fb_size); error = vtgpu_req_resp(sc, &s.req, sizeof(s.req), &s.resp, sizeof(s.resp)); if (error != 0) return (error); if (s.resp.type != htole32(VIRTIO_GPU_RESP_OK_NODATA)) { device_printf(sc->vtgpu_dev, "Invalid reponse type %x\n", le32toh(s.resp.type)); return (EINVAL); } return (0); } static int vtgpu_set_scanout(struct vtgpu_softc *sc, uint32_t x, uint32_t y, uint32_t width, uint32_t height) { struct { struct virtio_gpu_set_scanout req; char pad; struct virtio_gpu_ctrl_hdr resp; } s = { 0 }; int error; s.req.hdr.type = htole32(VIRTIO_GPU_CMD_SET_SCANOUT); s.req.hdr.flags = htole32(VIRTIO_GPU_FLAG_FENCE); s.req.hdr.fence_id = htole64( atomic_fetchadd_64(&sc->vtgpu_next_fence, 1)); s.req.r.x = htole32(x); s.req.r.y = htole32(y); s.req.r.width = htole32(width); s.req.r.height = htole32(height); s.req.scanout_id = 0; s.req.resource_id = htole32(VTGPU_RESOURCE_ID); error = vtgpu_req_resp(sc, &s.req, sizeof(s.req), &s.resp, sizeof(s.resp)); if (error != 0) return (error); if (s.resp.type != htole32(VIRTIO_GPU_RESP_OK_NODATA)) { device_printf(sc->vtgpu_dev, "Invalid reponse type %x\n", le32toh(s.resp.type)); return (EINVAL); } return (0); } static int vtgpu_transfer_to_host_2d(struct vtgpu_softc *sc, uint32_t x, uint32_t y, uint32_t width, uint32_t height) { struct { struct virtio_gpu_transfer_to_host_2d req; char pad; struct virtio_gpu_ctrl_hdr resp; } s = { 0 }; int error; s.req.hdr.type = htole32(VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D); s.req.hdr.flags = htole32(VIRTIO_GPU_FLAG_FENCE); s.req.hdr.fence_id = htole64( atomic_fetchadd_64(&sc->vtgpu_next_fence, 1)); s.req.r.x = htole32(x); s.req.r.y = htole32(y); s.req.r.width = htole32(width); s.req.r.height = htole32(height); s.req.offset = htole64((y * sc->vtgpu_fb_info.fb_width + x) * (sc->vtgpu_fb_info.fb_bpp / 8)); s.req.resource_id = htole32(VTGPU_RESOURCE_ID); error = vtgpu_req_resp(sc, &s.req, sizeof(s.req), &s.resp, sizeof(s.resp)); if (error != 0) return (error); if (s.resp.type != htole32(VIRTIO_GPU_RESP_OK_NODATA)) { device_printf(sc->vtgpu_dev, "Invalid reponse type %x\n", le32toh(s.resp.type)); return (EINVAL); } return (0); } static int vtgpu_resource_flush(struct vtgpu_softc *sc, uint32_t x, uint32_t y, uint32_t width, uint32_t height) { struct { struct virtio_gpu_resource_flush req; char pad; struct virtio_gpu_ctrl_hdr resp; } s = { 0 }; int error; s.req.hdr.type = htole32(VIRTIO_GPU_CMD_RESOURCE_FLUSH); s.req.hdr.flags = htole32(VIRTIO_GPU_FLAG_FENCE); s.req.hdr.fence_id = htole64( atomic_fetchadd_64(&sc->vtgpu_next_fence, 1)); s.req.r.x = htole32(x); s.req.r.y = htole32(y); s.req.r.width = htole32(width); s.req.r.height = htole32(height); s.req.resource_id = htole32(VTGPU_RESOURCE_ID); error = vtgpu_req_resp(sc, &s.req, sizeof(s.req), &s.resp, sizeof(s.resp)); if (error != 0) return (error); if (s.resp.type != htole32(VIRTIO_GPU_RESP_OK_NODATA)) { device_printf(sc->vtgpu_dev, "Invalid reponse type %x\n", le32toh(s.resp.type)); return (EINVAL); } return (0); } diff --git a/sys/dev/virtio/mmio/virtio_mmio.c b/sys/dev/virtio/mmio/virtio_mmio.c index 6059ef76eceb..c706f91ecd15 100644 --- a/sys/dev/virtio/mmio/virtio_mmio.c +++ b/sys/dev/virtio/mmio/virtio_mmio.c @@ -1,1027 +1,1027 @@ /*- * Copyright (c) 2014 Ruslan Bukin * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) * ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by Andrew Turner * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * VirtIO MMIO interface. * This driver is heavily based on VirtIO PCI interface driver. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_mmio_if.h" #include "virtio_bus_if.h" #include "virtio_if.h" struct vtmmio_virtqueue { struct virtqueue *vtv_vq; int vtv_no_intr; }; static int vtmmio_detach(device_t); static int vtmmio_suspend(device_t); static int vtmmio_resume(device_t); static int vtmmio_shutdown(device_t); static void vtmmio_driver_added(device_t, driver_t *); static void vtmmio_child_detached(device_t, device_t); static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *); static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t); static uint64_t vtmmio_negotiate_features(device_t, uint64_t); static int vtmmio_finalize_features(device_t); static bool vtmmio_with_feature(device_t, uint64_t); static void vtmmio_set_virtqueue(struct vtmmio_softc *sc, struct virtqueue *vq, uint32_t size); -static int vtmmio_alloc_virtqueues(device_t, int, int, +static int vtmmio_alloc_virtqueues(device_t, int, struct vq_alloc_info *); static int vtmmio_setup_intr(device_t, enum intr_type); static void vtmmio_stop(device_t); static void vtmmio_poll(device_t); static int vtmmio_reinit(device_t, uint64_t); static void vtmmio_reinit_complete(device_t); static void vtmmio_notify_virtqueue(device_t, uint16_t, bus_size_t); static int vtmmio_config_generation(device_t); static uint8_t vtmmio_get_status(device_t); static void vtmmio_set_status(device_t, uint8_t); static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); static uint64_t vtmmio_read_dev_config_8(struct vtmmio_softc *, bus_size_t); static void vtmmio_write_dev_config(device_t, bus_size_t, const void *, int); static void vtmmio_describe_features(struct vtmmio_softc *, const char *, uint64_t); static void vtmmio_probe_and_attach_child(struct vtmmio_softc *); static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int); static void vtmmio_free_interrupts(struct vtmmio_softc *); static void vtmmio_free_virtqueues(struct vtmmio_softc *); static void vtmmio_release_child_resources(struct vtmmio_softc *); static void vtmmio_reset(struct vtmmio_softc *); static void vtmmio_select_virtqueue(struct vtmmio_softc *, int); static void vtmmio_vq_intr(void *); /* * I/O port read/write wrappers. */ #define vtmmio_write_config_1(sc, o, v) \ do { \ if (sc->platform != NULL) \ VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ bus_write_1((sc)->res[0], (o), (v)); \ if (sc->platform != NULL) \ VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ } while (0) #define vtmmio_write_config_2(sc, o, v) \ do { \ if (sc->platform != NULL) \ VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ bus_write_2((sc)->res[0], (o), (v)); \ if (sc->platform != NULL) \ VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ } while (0) #define vtmmio_write_config_4(sc, o, v) \ do { \ if (sc->platform != NULL) \ VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ bus_write_4((sc)->res[0], (o), (v)); \ if (sc->platform != NULL) \ VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ } while (0) #define vtmmio_read_config_1(sc, o) \ bus_read_1((sc)->res[0], (o)) #define vtmmio_read_config_2(sc, o) \ bus_read_2((sc)->res[0], (o)) #define vtmmio_read_config_4(sc, o) \ bus_read_4((sc)->res[0], (o)) static device_method_t vtmmio_methods[] = { /* Device interface. */ DEVMETHOD(device_attach, vtmmio_attach), DEVMETHOD(device_detach, vtmmio_detach), DEVMETHOD(device_suspend, vtmmio_suspend), DEVMETHOD(device_resume, vtmmio_resume), DEVMETHOD(device_shutdown, vtmmio_shutdown), /* Bus interface. */ DEVMETHOD(bus_driver_added, vtmmio_driver_added), DEVMETHOD(bus_child_detached, vtmmio_child_detached), DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), DEVMETHOD(bus_read_ivar, vtmmio_read_ivar), DEVMETHOD(bus_write_ivar, vtmmio_write_ivar), /* VirtIO bus interface. */ DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features), DEVMETHOD(virtio_bus_finalize_features, vtmmio_finalize_features), DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature), DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), DEVMETHOD(virtio_bus_stop, vtmmio_stop), DEVMETHOD(virtio_bus_poll, vtmmio_poll), DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), DEVMETHOD(virtio_bus_config_generation, vtmmio_config_generation), DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config), DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config), DEVMETHOD_END }; DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods, sizeof(struct vtmmio_softc)); MODULE_VERSION(virtio_mmio, 1); int vtmmio_probe(device_t dev) { struct vtmmio_softc *sc; int rid; uint32_t magic, version; sc = device_get_softc(dev); rid = 0; sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->res[0] == NULL) { device_printf(dev, "Cannot allocate memory window.\n"); return (ENXIO); } magic = vtmmio_read_config_4(sc, VIRTIO_MMIO_MAGIC_VALUE); if (magic != VIRTIO_MMIO_MAGIC_VIRT) { device_printf(dev, "Bad magic value %#x\n", magic); bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); return (ENXIO); } version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION); if (version < 1 || version > 2) { device_printf(dev, "Unsupported version: %#x\n", version); bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); return (ENXIO); } if (vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID) == 0) { bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); return (ENXIO); } bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); device_set_desc(dev, "VirtIO MMIO adapter"); return (BUS_PROBE_DEFAULT); } static int vtmmio_setup_intr(device_t dev, enum intr_type type) { struct vtmmio_softc *sc; int rid; int err; sc = device_get_softc(dev); if (sc->platform != NULL) { err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, vtmmio_vq_intr, sc); if (err == 0) { /* Okay we have backend-specific interrupts */ return (0); } } rid = 0; sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (!sc->res[1]) { device_printf(dev, "Can't allocate interrupt\n"); return (ENXIO); } if (bus_setup_intr(dev, sc->res[1], type | INTR_MPSAFE, NULL, vtmmio_vq_intr, sc, &sc->ih)) { device_printf(dev, "Can't setup the interrupt\n"); return (ENXIO); } return (0); } int vtmmio_attach(device_t dev) { struct vtmmio_softc *sc; device_t child; int rid; sc = device_get_softc(dev); sc->dev = dev; rid = 0; sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->res[0] == NULL) { device_printf(dev, "Cannot allocate memory window.\n"); return (ENXIO); } sc->vtmmio_version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION); vtmmio_reset(sc); /* Tell the host we've noticed this device. */ vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); if ((child = device_add_child(dev, NULL, -1)) == NULL) { device_printf(dev, "Cannot create child device.\n"); vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); vtmmio_detach(dev); return (ENOMEM); } sc->vtmmio_child_dev = child; vtmmio_probe_and_attach_child(sc); return (0); } static int vtmmio_detach(device_t dev) { struct vtmmio_softc *sc; device_t child; int error; sc = device_get_softc(dev); if ((child = sc->vtmmio_child_dev) != NULL) { error = device_delete_child(dev, child); if (error) return (error); sc->vtmmio_child_dev = NULL; } vtmmio_reset(sc); if (sc->res[0] != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->res[0]); sc->res[0] = NULL; } return (0); } static int vtmmio_suspend(device_t dev) { return (bus_generic_suspend(dev)); } static int vtmmio_resume(device_t dev) { return (bus_generic_resume(dev)); } static int vtmmio_shutdown(device_t dev) { (void) bus_generic_shutdown(dev); /* Forcibly stop the host device. */ vtmmio_stop(dev); return (0); } static void vtmmio_driver_added(device_t dev, driver_t *driver) { struct vtmmio_softc *sc; sc = device_get_softc(dev); vtmmio_probe_and_attach_child(sc); } static void vtmmio_child_detached(device_t dev, device_t child) { struct vtmmio_softc *sc; sc = device_get_softc(dev); vtmmio_reset(sc); vtmmio_release_child_resources(sc); } static int vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { struct vtmmio_softc *sc; sc = device_get_softc(dev); if (sc->vtmmio_child_dev != child) return (ENOENT); switch (index) { case VIRTIO_IVAR_DEVTYPE: case VIRTIO_IVAR_SUBDEVICE: *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID); break; case VIRTIO_IVAR_VENDOR: *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID); break; case VIRTIO_IVAR_SUBVENDOR: case VIRTIO_IVAR_DEVICE: /* * Dummy value for fields not present in this bus. Used by * bus-agnostic virtio_child_pnpinfo. */ *result = 0; break; case VIRTIO_IVAR_MODERN: /* * There are several modern (aka MMIO v2) spec compliance * issues with this driver, but keep the status quo. */ *result = sc->vtmmio_version > 1; break; default: return (ENOENT); } return (0); } static int vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { struct vtmmio_softc *sc; sc = device_get_softc(dev); if (sc->vtmmio_child_dev != child) return (ENOENT); switch (index) { case VIRTIO_IVAR_FEATURE_DESC: sc->vtmmio_child_feat_desc = (void *) value; break; default: return (ENOENT); } return (0); } static uint64_t vtmmio_negotiate_features(device_t dev, uint64_t child_features) { struct vtmmio_softc *sc; uint64_t host_features, features; sc = device_get_softc(dev); if (sc->vtmmio_version > 1) { child_features |= VIRTIO_F_VERSION_1; } vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 1); host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); host_features <<= 32; vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 0); host_features |= vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); vtmmio_describe_features(sc, "host", host_features); /* * Limit negotiated features to what the driver, virtqueue, and * host all support. */ features = host_features & child_features; features = virtio_filter_transport_features(features); sc->vtmmio_features = features; vtmmio_describe_features(sc, "negotiated", features); vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES_SEL, 1); vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features >> 32); vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES_SEL, 0); vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features); return (features); } static int vtmmio_finalize_features(device_t dev) { struct vtmmio_softc *sc; uint8_t status; sc = device_get_softc(dev); if (sc->vtmmio_version > 1) { /* * Must re-read the status after setting it to verify the * negotiated features were accepted by the device. */ vtmmio_set_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); status = vtmmio_get_status(dev); if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { device_printf(dev, "desired features were not accepted\n"); return (ENOTSUP); } } return (0); } static bool vtmmio_with_feature(device_t dev, uint64_t feature) { struct vtmmio_softc *sc; sc = device_get_softc(dev); return ((sc->vtmmio_features & feature) != 0); } static void vtmmio_set_virtqueue(struct vtmmio_softc *sc, struct virtqueue *vq, uint32_t size) { vm_paddr_t paddr; vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); if (sc->vtmmio_version == 1) { vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, VIRTIO_MMIO_VRING_ALIGN); paddr = virtqueue_paddr(vq); vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, paddr >> PAGE_SHIFT); } else { paddr = virtqueue_desc_paddr(vq); vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_LOW, paddr); vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_HIGH, ((uint64_t)paddr) >> 32); paddr = virtqueue_avail_paddr(vq); vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_LOW, paddr); vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_HIGH, ((uint64_t)paddr) >> 32); paddr = virtqueue_used_paddr(vq); vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_LOW, paddr); vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_HIGH, ((uint64_t)paddr) >> 32); vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 1); } } static int -vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, +vtmmio_alloc_virtqueues(device_t dev, int nvqs, struct vq_alloc_info *vq_info) { struct vtmmio_virtqueue *vqx; struct vq_alloc_info *info; struct vtmmio_softc *sc; struct virtqueue *vq; uint32_t size; int idx, error; sc = device_get_softc(dev); if (sc->vtmmio_nvqs != 0) return (EALREADY); if (nvqs <= 0) return (EINVAL); sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue), M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->vtmmio_vqs == NULL) return (ENOMEM); if (sc->vtmmio_version == 1) { vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, (1 << PAGE_SHIFT)); } for (idx = 0; idx < nvqs; idx++) { vqx = &sc->vtmmio_vqs[idx]; info = &vq_info[idx]; vtmmio_select_virtqueue(sc, idx); size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); error = virtqueue_alloc(dev, idx, size, VIRTIO_MMIO_QUEUE_NOTIFY, VIRTIO_MMIO_VRING_ALIGN, ~(vm_paddr_t)0, info, &vq); if (error) { device_printf(dev, "cannot allocate virtqueue %d: %d\n", idx, error); break; } vtmmio_set_virtqueue(sc, vq, size); vqx->vtv_vq = *info->vqai_vq = vq; vqx->vtv_no_intr = info->vqai_intr == NULL; sc->vtmmio_nvqs++; } if (error) vtmmio_free_virtqueues(sc); return (error); } static void vtmmio_stop(device_t dev) { vtmmio_reset(device_get_softc(dev)); } static void vtmmio_poll(device_t dev) { struct vtmmio_softc *sc; sc = device_get_softc(dev); if (sc->platform != NULL) VIRTIO_MMIO_POLL(sc->platform); } static int vtmmio_reinit(device_t dev, uint64_t features) { struct vtmmio_softc *sc; int idx, error; sc = device_get_softc(dev); if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) vtmmio_stop(dev); /* * Quickly drive the status through ACK and DRIVER. The device * does not become usable again until vtmmio_reinit_complete(). */ vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); /* * TODO: Check that features are not added as to what was * originally negotiated. */ vtmmio_negotiate_features(dev, features); error = vtmmio_finalize_features(dev); if (error) { device_printf(dev, "cannot finalize features during reinit\n"); return (error); } if (sc->vtmmio_version == 1) { vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, (1 << PAGE_SHIFT)); } for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { error = vtmmio_reinit_virtqueue(sc, idx); if (error) return (error); } return (0); } static void vtmmio_reinit_complete(device_t dev) { vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); } static void vtmmio_notify_virtqueue(device_t dev, uint16_t queue, bus_size_t offset) { struct vtmmio_softc *sc; sc = device_get_softc(dev); MPASS(offset == VIRTIO_MMIO_QUEUE_NOTIFY); vtmmio_write_config_4(sc, offset, queue); } static int vtmmio_config_generation(device_t dev) { struct vtmmio_softc *sc; uint32_t gen; sc = device_get_softc(dev); if (sc->vtmmio_version > 1) gen = vtmmio_read_config_4(sc, VIRTIO_MMIO_CONFIG_GENERATION); else gen = 0; return (gen); } static uint8_t vtmmio_get_status(device_t dev) { struct vtmmio_softc *sc; sc = device_get_softc(dev); return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS)); } static void vtmmio_set_status(device_t dev, uint8_t status) { struct vtmmio_softc *sc; sc = device_get_softc(dev); if (status != VIRTIO_CONFIG_STATUS_RESET) status |= vtmmio_get_status(dev); vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status); } static void vtmmio_read_dev_config(device_t dev, bus_size_t offset, void *dst, int length) { struct vtmmio_softc *sc; bus_size_t off; uint8_t *d; int size; sc = device_get_softc(dev); off = VIRTIO_MMIO_CONFIG + offset; /* * The non-legacy MMIO specification adds the following restriction: * * 4.2.2.2: For the device-specific configuration space, the driver * MUST use 8 bit wide accesses for 8 bit wide fields, 16 bit wide * and aligned accesses for 16 bit wide fields and 32 bit wide and * aligned accesses for 32 and 64 bit wide fields. * * The endianness also varies between non-legacy and legacy: * * 2.4: Note: The device configuration space uses the little-endian * format for multi-byte fields. * * 2.4.3: Note that for legacy interfaces, device configuration space * is generally the guest’s native endian, rather than PCI’s * little-endian. The correct endian-ness is documented for each * device. */ if (sc->vtmmio_version > 1) { switch (length) { case 1: *(uint8_t *)dst = vtmmio_read_config_1(sc, off); break; case 2: *(uint16_t *)dst = le16toh(vtmmio_read_config_2(sc, off)); break; case 4: *(uint32_t *)dst = le32toh(vtmmio_read_config_4(sc, off)); break; case 8: *(uint64_t *)dst = vtmmio_read_dev_config_8(sc, off); break; default: panic("%s: invalid length %d\n", __func__, length); } return; } for (d = dst; length > 0; d += size, off += size, length -= size) { #ifdef ALLOW_WORD_ALIGNED_ACCESS if (length >= 4) { size = 4; *(uint32_t *)d = vtmmio_read_config_4(sc, off); } else if (length >= 2) { size = 2; *(uint16_t *)d = vtmmio_read_config_2(sc, off); } else #endif { size = 1; *d = vtmmio_read_config_1(sc, off); } } } static uint64_t vtmmio_read_dev_config_8(struct vtmmio_softc *sc, bus_size_t off) { device_t dev; int gen; uint32_t val0, val1; dev = sc->dev; do { gen = vtmmio_config_generation(dev); val0 = le32toh(vtmmio_read_config_4(sc, off)); val1 = le32toh(vtmmio_read_config_4(sc, off + 4)); } while (gen != vtmmio_config_generation(dev)); return (((uint64_t) val1 << 32) | val0); } static void vtmmio_write_dev_config(device_t dev, bus_size_t offset, const void *src, int length) { struct vtmmio_softc *sc; bus_size_t off; const uint8_t *s; int size; sc = device_get_softc(dev); off = VIRTIO_MMIO_CONFIG + offset; /* * The non-legacy MMIO specification adds size and alignment * restrctions. It also changes the endianness from native-endian to * little-endian. See vtmmio_read_dev_config. */ if (sc->vtmmio_version > 1) { switch (length) { case 1: vtmmio_write_config_1(sc, off, *(const uint8_t *)src); break; case 2: vtmmio_write_config_2(sc, off, htole16(*(const uint16_t *)src)); break; case 4: vtmmio_write_config_4(sc, off, htole32(*(const uint32_t *)src)); break; case 8: vtmmio_write_config_4(sc, off, htole32(*(const uint64_t *)src)); vtmmio_write_config_4(sc, off + 4, htole32((*(const uint64_t *)src) >> 32)); break; default: panic("%s: invalid length %d\n", __func__, length); } return; } for (s = src; length > 0; s += size, off += size, length -= size) { #ifdef ALLOW_WORD_ALIGNED_ACCESS if (length >= 4) { size = 4; vtmmio_write_config_4(sc, off, *(uint32_t *)s); } else if (length >= 2) { size = 2; vtmmio_write_config_2(sc, off, *(uint16_t *)s); } else #endif { size = 1; vtmmio_write_config_1(sc, off, *s); } } } static void vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg, uint64_t features) { device_t dev, child; dev = sc->dev; child = sc->vtmmio_child_dev; if (device_is_attached(child) || bootverbose == 0) return; virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc); } static void vtmmio_probe_and_attach_child(struct vtmmio_softc *sc) { device_t dev, child; dev = sc->dev; child = sc->vtmmio_child_dev; if (child == NULL) return; if (device_get_state(child) != DS_NOTPRESENT) { return; } if (device_probe(child) != 0) { return; } vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); if (device_attach(child) != 0) { vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); vtmmio_reset(sc); vtmmio_release_child_resources(sc); /* Reset status for future attempt. */ vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); } else { vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); VIRTIO_ATTACH_COMPLETED(child); } } static int vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx) { struct vtmmio_virtqueue *vqx; struct virtqueue *vq; int error; uint16_t size; vqx = &sc->vtmmio_vqs[idx]; vq = vqx->vtv_vq; KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); vtmmio_select_virtqueue(sc, idx); size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); error = virtqueue_reinit(vq, size); if (error) return (error); vtmmio_set_virtqueue(sc, vq, size); return (0); } static void vtmmio_free_interrupts(struct vtmmio_softc *sc) { if (sc->ih != NULL) bus_teardown_intr(sc->dev, sc->res[1], sc->ih); if (sc->res[1] != NULL) bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]); } static void vtmmio_free_virtqueues(struct vtmmio_softc *sc) { struct vtmmio_virtqueue *vqx; int idx; for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { vqx = &sc->vtmmio_vqs[idx]; vtmmio_select_virtqueue(sc, idx); if (sc->vtmmio_version > 1) { vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 0); vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_READY); } else vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0); virtqueue_free(vqx->vtv_vq); vqx->vtv_vq = NULL; } free(sc->vtmmio_vqs, M_DEVBUF); sc->vtmmio_vqs = NULL; sc->vtmmio_nvqs = 0; } static void vtmmio_release_child_resources(struct vtmmio_softc *sc) { vtmmio_free_interrupts(sc); vtmmio_free_virtqueues(sc); } static void vtmmio_reset(struct vtmmio_softc *sc) { /* * Setting the status to RESET sets the host device to * the original, uninitialized state. */ vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET); } static void vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx) { vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); } static void vtmmio_vq_intr(void *arg) { struct vtmmio_virtqueue *vqx; struct vtmmio_softc *sc; struct virtqueue *vq; uint32_t status; int idx; sc = arg; status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS); vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status); /* The config changed */ if (status & VIRTIO_MMIO_INT_CONFIG) if (sc->vtmmio_child_dev != NULL) VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev); /* Notify all virtqueues. */ if (status & VIRTIO_MMIO_INT_VRING) { for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { vqx = &sc->vtmmio_vqs[idx]; if (vqx->vtv_no_intr == 0) { vq = vqx->vtv_vq; virtqueue_intr(vq); } } } } diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c index f7ad385b3955..6445a9ebb998 100644 --- a/sys/dev/virtio/network/if_vtnet.c +++ b/sys/dev/virtio/network/if_vtnet.c @@ -1,4456 +1,4448 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for VirtIO network devices. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_if.h" #include "opt_inet.h" #include "opt_inet6.h" #if defined(INET) || defined(INET6) #include #endif static int vtnet_modevent(module_t, int, void *); static int vtnet_probe(device_t); static int vtnet_attach(device_t); static int vtnet_detach(device_t); static int vtnet_suspend(device_t); static int vtnet_resume(device_t); static int vtnet_shutdown(device_t); static int vtnet_attach_completed(device_t); static int vtnet_config_change(device_t); static int vtnet_negotiate_features(struct vtnet_softc *); static int vtnet_setup_features(struct vtnet_softc *); static int vtnet_init_rxq(struct vtnet_softc *, int); static int vtnet_init_txq(struct vtnet_softc *, int); static int vtnet_alloc_rxtx_queues(struct vtnet_softc *); static void vtnet_free_rxtx_queues(struct vtnet_softc *); static int vtnet_alloc_rx_filters(struct vtnet_softc *); static void vtnet_free_rx_filters(struct vtnet_softc *); static int vtnet_alloc_virtqueues(struct vtnet_softc *); static int vtnet_alloc_interface(struct vtnet_softc *); static int vtnet_setup_interface(struct vtnet_softc *); static int vtnet_ioctl_mtu(struct vtnet_softc *, u_int); static int vtnet_ioctl_ifflags(struct vtnet_softc *); static int vtnet_ioctl_multi(struct vtnet_softc *); static int vtnet_ioctl_ifcap(struct vtnet_softc *, struct ifreq *); static int vtnet_ioctl(if_t, u_long, caddr_t); static uint64_t vtnet_get_counter(if_t, ift_counter); static int vtnet_rxq_populate(struct vtnet_rxq *); static void vtnet_rxq_free_mbufs(struct vtnet_rxq *); static struct mbuf * vtnet_rx_alloc_buf(struct vtnet_softc *, int , struct mbuf **); static int vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *, struct mbuf *, int); static int vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int); static int vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *); static int vtnet_rxq_new_buf(struct vtnet_rxq *); static int vtnet_rxq_csum_needs_csum(struct vtnet_rxq *, struct mbuf *, uint16_t, int, struct virtio_net_hdr *); static int vtnet_rxq_csum_data_valid(struct vtnet_rxq *, struct mbuf *, uint16_t, int, struct virtio_net_hdr *); static int vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *, struct virtio_net_hdr *); static void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int); static void vtnet_rxq_discard_buf(struct vtnet_rxq *, struct mbuf *); static int vtnet_rxq_merged_eof(struct vtnet_rxq *, struct mbuf *, int); static void vtnet_rxq_input(struct vtnet_rxq *, struct mbuf *, struct virtio_net_hdr *); static int vtnet_rxq_eof(struct vtnet_rxq *); static void vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries); static void vtnet_rx_vq_intr(void *); static void vtnet_rxq_tq_intr(void *, int); static int vtnet_txq_intr_threshold(struct vtnet_txq *); static int vtnet_txq_below_threshold(struct vtnet_txq *); static int vtnet_txq_notify(struct vtnet_txq *); static void vtnet_txq_free_mbufs(struct vtnet_txq *); static int vtnet_txq_offload_ctx(struct vtnet_txq *, struct mbuf *, int *, int *, int *); static int vtnet_txq_offload_tso(struct vtnet_txq *, struct mbuf *, int, int, struct virtio_net_hdr *); static struct mbuf * vtnet_txq_offload(struct vtnet_txq *, struct mbuf *, struct virtio_net_hdr *); static int vtnet_txq_enqueue_buf(struct vtnet_txq *, struct mbuf **, struct vtnet_tx_header *); static int vtnet_txq_encap(struct vtnet_txq *, struct mbuf **, int); #ifdef VTNET_LEGACY_TX static void vtnet_start_locked(struct vtnet_txq *, if_t); static void vtnet_start(if_t); #else static int vtnet_txq_mq_start_locked(struct vtnet_txq *, struct mbuf *); static int vtnet_txq_mq_start(if_t, struct mbuf *); static void vtnet_txq_tq_deferred(void *, int); #endif static void vtnet_txq_start(struct vtnet_txq *); static void vtnet_txq_tq_intr(void *, int); static int vtnet_txq_eof(struct vtnet_txq *); static void vtnet_tx_vq_intr(void *); static void vtnet_tx_start_all(struct vtnet_softc *); #ifndef VTNET_LEGACY_TX static void vtnet_qflush(if_t); #endif static int vtnet_watchdog(struct vtnet_txq *); static void vtnet_accum_stats(struct vtnet_softc *, struct vtnet_rxq_stats *, struct vtnet_txq_stats *); static void vtnet_tick(void *); static void vtnet_start_taskqueues(struct vtnet_softc *); static void vtnet_free_taskqueues(struct vtnet_softc *); static void vtnet_drain_taskqueues(struct vtnet_softc *); static void vtnet_drain_rxtx_queues(struct vtnet_softc *); static void vtnet_stop_rendezvous(struct vtnet_softc *); static void vtnet_stop(struct vtnet_softc *); static int vtnet_virtio_reinit(struct vtnet_softc *); static void vtnet_init_rx_filters(struct vtnet_softc *); static int vtnet_init_rx_queues(struct vtnet_softc *); static int vtnet_init_tx_queues(struct vtnet_softc *); static int vtnet_init_rxtx_queues(struct vtnet_softc *); static void vtnet_set_active_vq_pairs(struct vtnet_softc *); static void vtnet_update_rx_offloads(struct vtnet_softc *); static int vtnet_reinit(struct vtnet_softc *); static void vtnet_init_locked(struct vtnet_softc *, int); static void vtnet_init(void *); static void vtnet_free_ctrl_vq(struct vtnet_softc *); static void vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *, struct sglist *, int, int); static int vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *); static int vtnet_ctrl_guest_offloads(struct vtnet_softc *, uint64_t); static int vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t); static int vtnet_ctrl_rx_cmd(struct vtnet_softc *, uint8_t, bool); static int vtnet_set_promisc(struct vtnet_softc *, bool); static int vtnet_set_allmulti(struct vtnet_softc *, bool); static void vtnet_rx_filter(struct vtnet_softc *); static void vtnet_rx_filter_mac(struct vtnet_softc *); static int vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t); static void vtnet_rx_filter_vlan(struct vtnet_softc *); static void vtnet_update_vlan_filter(struct vtnet_softc *, int, uint16_t); static void vtnet_register_vlan(void *, if_t, uint16_t); static void vtnet_unregister_vlan(void *, if_t, uint16_t); static void vtnet_update_speed_duplex(struct vtnet_softc *); static int vtnet_is_link_up(struct vtnet_softc *); static void vtnet_update_link_status(struct vtnet_softc *); static int vtnet_ifmedia_upd(if_t); static void vtnet_ifmedia_sts(if_t, struct ifmediareq *); static void vtnet_get_macaddr(struct vtnet_softc *); static void vtnet_set_macaddr(struct vtnet_softc *); static void vtnet_attached_set_macaddr(struct vtnet_softc *); static void vtnet_vlan_tag_remove(struct mbuf *); static void vtnet_set_rx_process_limit(struct vtnet_softc *); static void vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *, struct sysctl_oid_list *, struct vtnet_rxq *); static void vtnet_setup_txq_sysctl(struct sysctl_ctx_list *, struct sysctl_oid_list *, struct vtnet_txq *); static void vtnet_setup_queue_sysctl(struct vtnet_softc *); static void vtnet_load_tunables(struct vtnet_softc *); static void vtnet_setup_sysctl(struct vtnet_softc *); static int vtnet_rxq_enable_intr(struct vtnet_rxq *); static void vtnet_rxq_disable_intr(struct vtnet_rxq *); static int vtnet_txq_enable_intr(struct vtnet_txq *); static void vtnet_txq_disable_intr(struct vtnet_txq *); static void vtnet_enable_rx_interrupts(struct vtnet_softc *); static void vtnet_enable_tx_interrupts(struct vtnet_softc *); static void vtnet_enable_interrupts(struct vtnet_softc *); static void vtnet_disable_rx_interrupts(struct vtnet_softc *); static void vtnet_disable_tx_interrupts(struct vtnet_softc *); static void vtnet_disable_interrupts(struct vtnet_softc *); static int vtnet_tunable_int(struct vtnet_softc *, const char *, int); DEBUGNET_DEFINE(vtnet); #define vtnet_htog16(_sc, _val) virtio_htog16(vtnet_modern(_sc), _val) #define vtnet_htog32(_sc, _val) virtio_htog32(vtnet_modern(_sc), _val) #define vtnet_htog64(_sc, _val) virtio_htog64(vtnet_modern(_sc), _val) #define vtnet_gtoh16(_sc, _val) virtio_gtoh16(vtnet_modern(_sc), _val) #define vtnet_gtoh32(_sc, _val) virtio_gtoh32(vtnet_modern(_sc), _val) #define vtnet_gtoh64(_sc, _val) virtio_gtoh64(vtnet_modern(_sc), _val) /* Tunables. */ static SYSCTL_NODE(_hw, OID_AUTO, vtnet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "VirtIO Net driver parameters"); static int vtnet_csum_disable = 0; SYSCTL_INT(_hw_vtnet, OID_AUTO, csum_disable, CTLFLAG_RDTUN, &vtnet_csum_disable, 0, "Disables receive and send checksum offload"); static int vtnet_fixup_needs_csum = 0; SYSCTL_INT(_hw_vtnet, OID_AUTO, fixup_needs_csum, CTLFLAG_RDTUN, &vtnet_fixup_needs_csum, 0, "Calculate valid checksum for NEEDS_CSUM packets"); static int vtnet_tso_disable = 0; SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_disable, CTLFLAG_RDTUN, &vtnet_tso_disable, 0, "Disables TSO"); static int vtnet_lro_disable = 0; SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_disable, CTLFLAG_RDTUN, &vtnet_lro_disable, 0, "Disables hardware LRO"); static int vtnet_mq_disable = 0; SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_disable, CTLFLAG_RDTUN, &vtnet_mq_disable, 0, "Disables multiqueue support"); static int vtnet_mq_max_pairs = VTNET_MAX_QUEUE_PAIRS; SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_max_pairs, CTLFLAG_RDTUN, &vtnet_mq_max_pairs, 0, "Maximum number of multiqueue pairs"); static int vtnet_tso_maxlen = IP_MAXPACKET; SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN, &vtnet_tso_maxlen, 0, "TSO burst limit"); static int vtnet_rx_process_limit = 1024; SYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, &vtnet_rx_process_limit, 0, "Number of RX segments processed in one pass"); static int vtnet_lro_entry_count = 128; SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN, &vtnet_lro_entry_count, 0, "Software LRO entry count"); /* Enable sorted LRO, and the depth of the mbuf queue. */ static int vtnet_lro_mbufq_depth = 0; SYSCTL_UINT(_hw_vtnet, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN, &vtnet_lro_mbufq_depth, 0, "Depth of software LRO mbuf queue"); static uma_zone_t vtnet_tx_header_zone; static struct virtio_feature_desc vtnet_feature_desc[] = { { VIRTIO_NET_F_CSUM, "TxChecksum" }, { VIRTIO_NET_F_GUEST_CSUM, "RxChecksum" }, { VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, "CtrlRxOffloads" }, { VIRTIO_NET_F_MAC, "MAC" }, { VIRTIO_NET_F_GSO, "TxGSO" }, { VIRTIO_NET_F_GUEST_TSO4, "RxLROv4" }, { VIRTIO_NET_F_GUEST_TSO6, "RxLROv6" }, { VIRTIO_NET_F_GUEST_ECN, "RxLROECN" }, { VIRTIO_NET_F_GUEST_UFO, "RxUFO" }, { VIRTIO_NET_F_HOST_TSO4, "TxTSOv4" }, { VIRTIO_NET_F_HOST_TSO6, "TxTSOv6" }, { VIRTIO_NET_F_HOST_ECN, "TxTSOECN" }, { VIRTIO_NET_F_HOST_UFO, "TxUFO" }, { VIRTIO_NET_F_MRG_RXBUF, "MrgRxBuf" }, { VIRTIO_NET_F_STATUS, "Status" }, { VIRTIO_NET_F_CTRL_VQ, "CtrlVq" }, { VIRTIO_NET_F_CTRL_RX, "CtrlRxMode" }, { VIRTIO_NET_F_CTRL_VLAN, "CtrlVLANFilter" }, { VIRTIO_NET_F_CTRL_RX_EXTRA, "CtrlRxModeExtra" }, { VIRTIO_NET_F_GUEST_ANNOUNCE, "GuestAnnounce" }, { VIRTIO_NET_F_MQ, "Multiqueue" }, { VIRTIO_NET_F_CTRL_MAC_ADDR, "CtrlMacAddr" }, { VIRTIO_NET_F_SPEED_DUPLEX, "SpeedDuplex" }, { 0, NULL } }; static device_method_t vtnet_methods[] = { /* Device methods. */ DEVMETHOD(device_probe, vtnet_probe), DEVMETHOD(device_attach, vtnet_attach), DEVMETHOD(device_detach, vtnet_detach), DEVMETHOD(device_suspend, vtnet_suspend), DEVMETHOD(device_resume, vtnet_resume), DEVMETHOD(device_shutdown, vtnet_shutdown), /* VirtIO methods. */ DEVMETHOD(virtio_attach_completed, vtnet_attach_completed), DEVMETHOD(virtio_config_change, vtnet_config_change), DEVMETHOD_END }; #ifdef DEV_NETMAP #include #endif static driver_t vtnet_driver = { .name = "vtnet", .methods = vtnet_methods, .size = sizeof(struct vtnet_softc) }; VIRTIO_DRIVER_MODULE(vtnet, vtnet_driver, vtnet_modevent, NULL); MODULE_VERSION(vtnet, 1); MODULE_DEPEND(vtnet, virtio, 1, 1, 1); #ifdef DEV_NETMAP MODULE_DEPEND(vtnet, netmap, 1, 1, 1); #endif VIRTIO_SIMPLE_PNPINFO(vtnet, VIRTIO_ID_NETWORK, "VirtIO Networking Adapter"); static int vtnet_modevent(module_t mod __unused, int type, void *unused __unused) { int error = 0; static int loaded = 0; switch (type) { case MOD_LOAD: if (loaded++ == 0) { vtnet_tx_header_zone = uma_zcreate("vtnet_tx_hdr", sizeof(struct vtnet_tx_header), NULL, NULL, NULL, NULL, 0, 0); #ifdef DEBUGNET /* * We need to allocate from this zone in the transmit path, so ensure * that we have at least one item per header available. * XXX add a separate zone like we do for mbufs? otherwise we may alloc * buckets */ uma_zone_reserve(vtnet_tx_header_zone, DEBUGNET_MAX_IN_FLIGHT * 2); uma_prealloc(vtnet_tx_header_zone, DEBUGNET_MAX_IN_FLIGHT * 2); #endif } break; case MOD_QUIESCE: if (uma_zone_get_cur(vtnet_tx_header_zone) > 0) error = EBUSY; break; case MOD_UNLOAD: if (--loaded == 0) { uma_zdestroy(vtnet_tx_header_zone); vtnet_tx_header_zone = NULL; } break; case MOD_SHUTDOWN: break; default: error = EOPNOTSUPP; break; } return (error); } static int vtnet_probe(device_t dev) { return (VIRTIO_SIMPLE_PROBE(dev, vtnet)); } static int vtnet_attach(device_t dev) { struct vtnet_softc *sc; int error; sc = device_get_softc(dev); sc->vtnet_dev = dev; virtio_set_feature_desc(dev, vtnet_feature_desc); VTNET_CORE_LOCK_INIT(sc); callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0); vtnet_load_tunables(sc); error = vtnet_alloc_interface(sc); if (error) { device_printf(dev, "cannot allocate interface\n"); goto fail; } vtnet_setup_sysctl(sc); error = vtnet_setup_features(sc); if (error) { device_printf(dev, "cannot setup features\n"); goto fail; } error = vtnet_alloc_rx_filters(sc); if (error) { device_printf(dev, "cannot allocate Rx filters\n"); goto fail; } error = vtnet_alloc_rxtx_queues(sc); if (error) { device_printf(dev, "cannot allocate queues\n"); goto fail; } error = vtnet_alloc_virtqueues(sc); if (error) { device_printf(dev, "cannot allocate virtqueues\n"); goto fail; } error = vtnet_setup_interface(sc); if (error) { device_printf(dev, "cannot setup interface\n"); goto fail; } error = virtio_setup_intr(dev, INTR_TYPE_NET); if (error) { device_printf(dev, "cannot setup interrupts\n"); ether_ifdetach(sc->vtnet_ifp); goto fail; } #ifdef DEV_NETMAP vtnet_netmap_attach(sc); #endif vtnet_start_taskqueues(sc); fail: if (error) vtnet_detach(dev); return (error); } static int vtnet_detach(device_t dev) { struct vtnet_softc *sc; if_t ifp; sc = device_get_softc(dev); ifp = sc->vtnet_ifp; if (device_is_attached(dev)) { VTNET_CORE_LOCK(sc); vtnet_stop(sc); VTNET_CORE_UNLOCK(sc); callout_drain(&sc->vtnet_tick_ch); vtnet_drain_taskqueues(sc); ether_ifdetach(ifp); } #ifdef DEV_NETMAP netmap_detach(ifp); #endif if (sc->vtnet_pfil != NULL) { pfil_head_unregister(sc->vtnet_pfil); sc->vtnet_pfil = NULL; } vtnet_free_taskqueues(sc); if (sc->vtnet_vlan_attach != NULL) { EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach); sc->vtnet_vlan_attach = NULL; } if (sc->vtnet_vlan_detach != NULL) { EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vtnet_vlan_detach); sc->vtnet_vlan_detach = NULL; } ifmedia_removeall(&sc->vtnet_media); if (ifp != NULL) { if_free(ifp); sc->vtnet_ifp = NULL; } vtnet_free_rxtx_queues(sc); vtnet_free_rx_filters(sc); if (sc->vtnet_ctrl_vq != NULL) vtnet_free_ctrl_vq(sc); VTNET_CORE_LOCK_DESTROY(sc); return (0); } static int vtnet_suspend(device_t dev) { struct vtnet_softc *sc; sc = device_get_softc(dev); VTNET_CORE_LOCK(sc); vtnet_stop(sc); sc->vtnet_flags |= VTNET_FLAG_SUSPENDED; VTNET_CORE_UNLOCK(sc); return (0); } static int vtnet_resume(device_t dev) { struct vtnet_softc *sc; if_t ifp; sc = device_get_softc(dev); ifp = sc->vtnet_ifp; VTNET_CORE_LOCK(sc); if (if_getflags(ifp) & IFF_UP) vtnet_init_locked(sc, 0); sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED; VTNET_CORE_UNLOCK(sc); return (0); } static int vtnet_shutdown(device_t dev) { /* * Suspend already does all of what we need to * do here; we just never expect to be resumed. */ return (vtnet_suspend(dev)); } static int vtnet_attach_completed(device_t dev) { struct vtnet_softc *sc; sc = device_get_softc(dev); VTNET_CORE_LOCK(sc); vtnet_attached_set_macaddr(sc); VTNET_CORE_UNLOCK(sc); return (0); } static int vtnet_config_change(device_t dev) { struct vtnet_softc *sc; sc = device_get_softc(dev); VTNET_CORE_LOCK(sc); vtnet_update_link_status(sc); if (sc->vtnet_link_active != 0) vtnet_tx_start_all(sc); VTNET_CORE_UNLOCK(sc); return (0); } static int vtnet_negotiate_features(struct vtnet_softc *sc) { device_t dev; uint64_t features, negotiated_features; int no_csum; dev = sc->vtnet_dev; features = virtio_bus_is_modern(dev) ? VTNET_MODERN_FEATURES : VTNET_LEGACY_FEATURES; /* * TSO and LRO are only available when their corresponding checksum * offload feature is also negotiated. */ no_csum = vtnet_tunable_int(sc, "csum_disable", vtnet_csum_disable); if (no_csum) features &= ~(VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM); if (no_csum || vtnet_tunable_int(sc, "tso_disable", vtnet_tso_disable)) features &= ~VTNET_TSO_FEATURES; if (no_csum || vtnet_tunable_int(sc, "lro_disable", vtnet_lro_disable)) features &= ~VTNET_LRO_FEATURES; #ifndef VTNET_LEGACY_TX if (vtnet_tunable_int(sc, "mq_disable", vtnet_mq_disable)) features &= ~VIRTIO_NET_F_MQ; #else features &= ~VIRTIO_NET_F_MQ; #endif negotiated_features = virtio_negotiate_features(dev, features); if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) { uint16_t mtu; mtu = virtio_read_dev_config_2(dev, offsetof(struct virtio_net_config, mtu)); if (mtu < VTNET_MIN_MTU /* || mtu > VTNET_MAX_MTU */) { device_printf(dev, "Invalid MTU value: %d. " "MTU feature disabled.\n", mtu); features &= ~VIRTIO_NET_F_MTU; negotiated_features = virtio_negotiate_features(dev, features); } } if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) { uint16_t npairs; npairs = virtio_read_dev_config_2(dev, offsetof(struct virtio_net_config, max_virtqueue_pairs)); if (npairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || npairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) { device_printf(dev, "Invalid max_virtqueue_pairs value: " "%d. Multiqueue feature disabled.\n", npairs); features &= ~VIRTIO_NET_F_MQ; negotiated_features = virtio_negotiate_features(dev, features); } } if (virtio_with_feature(dev, VTNET_LRO_FEATURES) && virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) { /* * LRO without mergeable buffers requires special care. This * is not ideal because every receive buffer must be large * enough to hold the maximum TCP packet, the Ethernet header, * and the header. This requires up to 34 descriptors with * MCLBYTES clusters. If we do not have indirect descriptors, * LRO is disabled since the virtqueue will not contain very * many receive buffers. */ if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) { device_printf(dev, "Host LRO disabled since both mergeable buffers " "and indirect descriptors were not negotiated\n"); features &= ~VTNET_LRO_FEATURES; negotiated_features = virtio_negotiate_features(dev, features); } else sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG; } sc->vtnet_features = negotiated_features; sc->vtnet_negotiated_features = negotiated_features; return (virtio_finalize_features(dev)); } static int vtnet_setup_features(struct vtnet_softc *sc) { device_t dev; int error; dev = sc->vtnet_dev; error = vtnet_negotiate_features(sc); if (error) return (error); if (virtio_with_feature(dev, VIRTIO_F_VERSION_1)) sc->vtnet_flags |= VTNET_FLAG_MODERN; if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) sc->vtnet_flags |= VTNET_FLAG_INDIRECT; if (virtio_with_feature(dev, VIRTIO_RING_F_EVENT_IDX)) sc->vtnet_flags |= VTNET_FLAG_EVENT_IDX; if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) { /* This feature should always be negotiated. */ sc->vtnet_flags |= VTNET_FLAG_MAC; } if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) { sc->vtnet_max_mtu = virtio_read_dev_config_2(dev, offsetof(struct virtio_net_config, mtu)); } else sc->vtnet_max_mtu = VTNET_MAX_MTU; if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) { sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS; sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); } else if (vtnet_modern(sc)) { /* This is identical to the mergeable header. */ sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_v1); } else sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr); if (vtnet_modern(sc) || sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_INLINE; else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) sc->vtnet_rx_nsegs = VTNET_RX_SEGS_LRO_NOMRG; else sc->vtnet_rx_nsegs = VTNET_RX_SEGS_HDR_SEPARATE; /* * Favor "hardware" LRO if negotiated, but support software LRO as * a fallback; there is usually little benefit (or worse) with both. */ if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) == 0 && virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6) == 0) sc->vtnet_flags |= VTNET_FLAG_SW_LRO; if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MAX; else sc->vtnet_tx_nsegs = VTNET_TX_SEGS_MIN; sc->vtnet_req_vq_pairs = 1; sc->vtnet_max_vq_pairs = 1; if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) { sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ; if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX)) sc->vtnet_flags |= VTNET_FLAG_CTRL_RX; if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN)) sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER; if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR)) sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC; if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) { sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev, offsetof(struct virtio_net_config, max_virtqueue_pairs)); } } if (sc->vtnet_max_vq_pairs > 1) { int req; /* * Limit the maximum number of requested queue pairs to the * number of CPUs and the configured maximum. */ req = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs); if (req < 0) req = 1; if (req == 0) req = mp_ncpus; if (req > sc->vtnet_max_vq_pairs) req = sc->vtnet_max_vq_pairs; if (req > mp_ncpus) req = mp_ncpus; if (req > 1) { sc->vtnet_req_vq_pairs = req; sc->vtnet_flags |= VTNET_FLAG_MQ; } } return (0); } static int vtnet_init_rxq(struct vtnet_softc *sc, int id) { struct vtnet_rxq *rxq; rxq = &sc->vtnet_rxqs[id]; snprintf(rxq->vtnrx_name, sizeof(rxq->vtnrx_name), "%s-rx%d", device_get_nameunit(sc->vtnet_dev), id); mtx_init(&rxq->vtnrx_mtx, rxq->vtnrx_name, NULL, MTX_DEF); rxq->vtnrx_sc = sc; rxq->vtnrx_id = id; rxq->vtnrx_sg = sglist_alloc(sc->vtnet_rx_nsegs, M_NOWAIT); if (rxq->vtnrx_sg == NULL) return (ENOMEM); #if defined(INET) || defined(INET6) if (vtnet_software_lro(sc)) { if (tcp_lro_init_args(&rxq->vtnrx_lro, sc->vtnet_ifp, sc->vtnet_lro_entry_count, sc->vtnet_lro_mbufq_depth) != 0) return (ENOMEM); } #endif NET_TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq); rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT, taskqueue_thread_enqueue, &rxq->vtnrx_tq); return (rxq->vtnrx_tq == NULL ? ENOMEM : 0); } static int vtnet_init_txq(struct vtnet_softc *sc, int id) { struct vtnet_txq *txq; txq = &sc->vtnet_txqs[id]; snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d", device_get_nameunit(sc->vtnet_dev), id); mtx_init(&txq->vtntx_mtx, txq->vtntx_name, NULL, MTX_DEF); txq->vtntx_sc = sc; txq->vtntx_id = id; txq->vtntx_sg = sglist_alloc(sc->vtnet_tx_nsegs, M_NOWAIT); if (txq->vtntx_sg == NULL) return (ENOMEM); #ifndef VTNET_LEGACY_TX txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF, M_NOWAIT, &txq->vtntx_mtx); if (txq->vtntx_br == NULL) return (ENOMEM); TASK_INIT(&txq->vtntx_defrtask, 0, vtnet_txq_tq_deferred, txq); #endif TASK_INIT(&txq->vtntx_intrtask, 0, vtnet_txq_tq_intr, txq); txq->vtntx_tq = taskqueue_create(txq->vtntx_name, M_NOWAIT, taskqueue_thread_enqueue, &txq->vtntx_tq); if (txq->vtntx_tq == NULL) return (ENOMEM); return (0); } static int vtnet_alloc_rxtx_queues(struct vtnet_softc *sc) { int i, npairs, error; npairs = sc->vtnet_max_vq_pairs; sc->vtnet_rxqs = malloc(sizeof(struct vtnet_rxq) * npairs, M_DEVBUF, M_NOWAIT | M_ZERO); sc->vtnet_txqs = malloc(sizeof(struct vtnet_txq) * npairs, M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->vtnet_rxqs == NULL || sc->vtnet_txqs == NULL) return (ENOMEM); for (i = 0; i < npairs; i++) { error = vtnet_init_rxq(sc, i); if (error) return (error); error = vtnet_init_txq(sc, i); if (error) return (error); } vtnet_set_rx_process_limit(sc); vtnet_setup_queue_sysctl(sc); return (0); } static void vtnet_destroy_rxq(struct vtnet_rxq *rxq) { rxq->vtnrx_sc = NULL; rxq->vtnrx_id = -1; #if defined(INET) || defined(INET6) tcp_lro_free(&rxq->vtnrx_lro); #endif if (rxq->vtnrx_sg != NULL) { sglist_free(rxq->vtnrx_sg); rxq->vtnrx_sg = NULL; } if (mtx_initialized(&rxq->vtnrx_mtx) != 0) mtx_destroy(&rxq->vtnrx_mtx); } static void vtnet_destroy_txq(struct vtnet_txq *txq) { txq->vtntx_sc = NULL; txq->vtntx_id = -1; if (txq->vtntx_sg != NULL) { sglist_free(txq->vtntx_sg); txq->vtntx_sg = NULL; } #ifndef VTNET_LEGACY_TX if (txq->vtntx_br != NULL) { buf_ring_free(txq->vtntx_br, M_DEVBUF); txq->vtntx_br = NULL; } #endif if (mtx_initialized(&txq->vtntx_mtx) != 0) mtx_destroy(&txq->vtntx_mtx); } static void vtnet_free_rxtx_queues(struct vtnet_softc *sc) { int i; if (sc->vtnet_rxqs != NULL) { for (i = 0; i < sc->vtnet_max_vq_pairs; i++) vtnet_destroy_rxq(&sc->vtnet_rxqs[i]); free(sc->vtnet_rxqs, M_DEVBUF); sc->vtnet_rxqs = NULL; } if (sc->vtnet_txqs != NULL) { for (i = 0; i < sc->vtnet_max_vq_pairs; i++) vtnet_destroy_txq(&sc->vtnet_txqs[i]); free(sc->vtnet_txqs, M_DEVBUF); sc->vtnet_txqs = NULL; } } static int vtnet_alloc_rx_filters(struct vtnet_softc *sc) { if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { sc->vtnet_mac_filter = malloc(sizeof(struct vtnet_mac_filter), M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->vtnet_mac_filter == NULL) return (ENOMEM); } if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { sc->vtnet_vlan_filter = malloc(sizeof(uint32_t) * VTNET_VLAN_FILTER_NWORDS, M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->vtnet_vlan_filter == NULL) return (ENOMEM); } return (0); } static void vtnet_free_rx_filters(struct vtnet_softc *sc) { if (sc->vtnet_mac_filter != NULL) { free(sc->vtnet_mac_filter, M_DEVBUF); sc->vtnet_mac_filter = NULL; } if (sc->vtnet_vlan_filter != NULL) { free(sc->vtnet_vlan_filter, M_DEVBUF); sc->vtnet_vlan_filter = NULL; } } static int vtnet_alloc_virtqueues(struct vtnet_softc *sc) { device_t dev; struct vq_alloc_info *info; struct vtnet_rxq *rxq; struct vtnet_txq *txq; - int i, idx, flags, nvqs, error; + int i, idx, nvqs, error; dev = sc->vtnet_dev; - flags = 0; nvqs = sc->vtnet_max_vq_pairs * 2; if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) nvqs++; info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT); if (info == NULL) return (ENOMEM); for (i = 0, idx = 0; i < sc->vtnet_req_vq_pairs; i++, idx += 2) { rxq = &sc->vtnet_rxqs[i]; VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs, vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq, "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id); txq = &sc->vtnet_txqs[i]; VQ_ALLOC_INFO_INIT(&info[idx+1], sc->vtnet_tx_nsegs, vtnet_tx_vq_intr, txq, &txq->vtntx_vq, "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id); } /* These queues will not be used so allocate the minimum resources. */ for (/**/; i < sc->vtnet_max_vq_pairs; i++, idx += 2) { rxq = &sc->vtnet_rxqs[i]; VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, rxq, &rxq->vtnrx_vq, "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id); txq = &sc->vtnet_txqs[i]; VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL, txq, &txq->vtntx_vq, "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id); } if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, NULL, &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev)); } - /* - * TODO: Enable interrupt binding if this is multiqueue. This will - * only matter when per-virtqueue MSIX is available. - */ - if (sc->vtnet_flags & VTNET_FLAG_MQ) - flags |= 0; - - error = virtio_alloc_virtqueues(dev, flags, nvqs, info); + error = virtio_alloc_virtqueues(dev, nvqs, info); free(info, M_TEMP); return (error); } static int vtnet_alloc_interface(struct vtnet_softc *sc) { device_t dev; if_t ifp; dev = sc->vtnet_dev; ifp = if_alloc(IFT_ETHER); if (ifp == NULL) return (ENOMEM); sc->vtnet_ifp = ifp; if_setsoftc(ifp, sc); if_initname(ifp, device_get_name(dev), device_get_unit(dev)); return (0); } static int vtnet_setup_interface(struct vtnet_softc *sc) { device_t dev; struct pfil_head_args pa; if_t ifp; dev = sc->vtnet_dev; ifp = sc->vtnet_ifp; if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); if_setbaudrate(ifp, IF_Gbps(10)); if_setinitfn(ifp, vtnet_init); if_setioctlfn(ifp, vtnet_ioctl); if_setgetcounterfn(ifp, vtnet_get_counter); #ifndef VTNET_LEGACY_TX if_settransmitfn(ifp, vtnet_txq_mq_start); if_setqflushfn(ifp, vtnet_qflush); #else struct virtqueue *vq = sc->vtnet_txqs[0].vtntx_vq; if_setstartfn(ifp, vtnet_start); if_setsendqlen(ifp, virtqueue_size(vq) - 1); if_setsendqready(ifp); #endif vtnet_get_macaddr(sc); if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)) if_setcapabilitiesbit(ifp, IFCAP_LINKSTATE, 0); ifmedia_init(&sc->vtnet_media, 0, vtnet_ifmedia_upd, vtnet_ifmedia_sts); ifmedia_add(&sc->vtnet_media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&sc->vtnet_media, IFM_ETHER | IFM_AUTO); if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) { int gso; if_setcapabilitiesbit(ifp, IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6, 0); gso = virtio_with_feature(dev, VIRTIO_NET_F_GSO); if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4)) if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) if_setcapabilitiesbit(ifp, IFCAP_TSO6, 0); if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN)) sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; if (if_getcapabilities(ifp) & (IFCAP_TSO4 | IFCAP_TSO6)) { int tso_maxlen; if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTSO, 0); tso_maxlen = vtnet_tunable_int(sc, "tso_maxlen", vtnet_tso_maxlen); if_sethwtsomax(ifp, tso_maxlen - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)); if_sethwtsomaxsegcount(ifp, sc->vtnet_tx_nsegs - 1); if_sethwtsomaxsegsize(ifp, PAGE_SIZE); } } if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) { if_setcapabilitiesbit(ifp, IFCAP_RXCSUM, 0); #ifdef notyet /* BMV: Rx checksums not distinguished between IPv4 and IPv6. */ if_setcapabilitiesbit(ifp, IFCAP_RXCSUM_IPV6, 0); #endif if (vtnet_tunable_int(sc, "fixup_needs_csum", vtnet_fixup_needs_csum) != 0) sc->vtnet_flags |= VTNET_FLAG_FIXUP_NEEDS_CSUM; /* Support either "hardware" or software LRO. */ if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); } if (if_getcapabilities(ifp) & (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6)) { /* * VirtIO does not support VLAN tagging, but we can fake * it by inserting and removing the 802.1Q header during * transmit and receive. We are then able to do checksum * offloading of VLAN frames. */ if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM, 0); } if (sc->vtnet_max_mtu >= ETHERMTU_JUMBO) if_setcapabilitiesbit(ifp, IFCAP_JUMBO_MTU, 0); if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU, 0); /* * Capabilities after here are not enabled by default. */ if_setcapenable(ifp, if_getcapabilities(ifp)); if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0); sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST); sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); } ether_ifattach(ifp, sc->vtnet_hwaddr); /* Tell the upper layer(s) we support long frames. */ if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); DEBUGNET_SET(ifp, vtnet); pa.pa_version = PFIL_VERSION; pa.pa_flags = PFIL_IN; pa.pa_type = PFIL_TYPE_ETHERNET; pa.pa_headname = if_name(ifp); sc->vtnet_pfil = pfil_head_register(&pa); return (0); } static int vtnet_rx_cluster_size(struct vtnet_softc *sc, int mtu) { int framesz; if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) return (MJUMPAGESIZE); else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) return (MCLBYTES); /* * Try to scale the receive mbuf cluster size from the MTU. We * could also use the VQ size to influence the selected size, * but that would only matter for very small queues. */ if (vtnet_modern(sc)) { MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr_v1)); framesz = sizeof(struct virtio_net_hdr_v1); } else framesz = sizeof(struct vtnet_rx_header); framesz += sizeof(struct ether_vlan_header) + mtu; if (framesz <= MCLBYTES) return (MCLBYTES); else if (framesz <= MJUMPAGESIZE) return (MJUMPAGESIZE); else if (framesz <= MJUM9BYTES) return (MJUM9BYTES); /* Sane default; avoid 16KB clusters. */ return (MCLBYTES); } static int vtnet_ioctl_mtu(struct vtnet_softc *sc, u_int mtu) { if_t ifp; int clustersz; ifp = sc->vtnet_ifp; VTNET_CORE_LOCK_ASSERT(sc); if (if_getmtu(ifp) == mtu) return (0); else if (mtu < ETHERMIN || mtu > sc->vtnet_max_mtu) return (EINVAL); if_setmtu(ifp, mtu); clustersz = vtnet_rx_cluster_size(sc, mtu); if (clustersz != sc->vtnet_rx_clustersz && if_getdrvflags(ifp) & IFF_DRV_RUNNING) { if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); vtnet_init_locked(sc, 0); } return (0); } static int vtnet_ioctl_ifflags(struct vtnet_softc *sc) { if_t ifp; int drv_running; ifp = sc->vtnet_ifp; drv_running = (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0; VTNET_CORE_LOCK_ASSERT(sc); if ((if_getflags(ifp) & IFF_UP) == 0) { if (drv_running) vtnet_stop(sc); goto out; } if (!drv_running) { vtnet_init_locked(sc, 0); goto out; } if ((if_getflags(ifp) ^ sc->vtnet_if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) { if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) vtnet_rx_filter(sc); else { /* * We don't support filtering out multicast, so * ALLMULTI is always set. */ if_setflagbits(ifp, IFF_ALLMULTI, 0); if_setflagbits(ifp, IFF_PROMISC, 0); } } out: sc->vtnet_if_flags = if_getflags(ifp); return (0); } static int vtnet_ioctl_multi(struct vtnet_softc *sc) { if_t ifp; ifp = sc->vtnet_ifp; VTNET_CORE_LOCK_ASSERT(sc); if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX && if_getdrvflags(ifp) & IFF_DRV_RUNNING) vtnet_rx_filter_mac(sc); return (0); } static int vtnet_ioctl_ifcap(struct vtnet_softc *sc, struct ifreq *ifr) { if_t ifp; int mask, reinit, update; ifp = sc->vtnet_ifp; mask = (ifr->ifr_reqcap & if_getcapabilities(ifp)) ^ if_getcapenable(ifp); reinit = update = 0; VTNET_CORE_LOCK_ASSERT(sc); if (mask & IFCAP_TXCSUM) if_togglecapenable(ifp, IFCAP_TXCSUM); if (mask & IFCAP_TXCSUM_IPV6) if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); if (mask & IFCAP_TSO4) if_togglecapenable(ifp, IFCAP_TSO4); if (mask & IFCAP_TSO6) if_togglecapenable(ifp, IFCAP_TSO6); if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) { /* * These Rx features require the negotiated features to * be updated. Avoid a full reinit if possible. */ if (sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) update = 1; else reinit = 1; /* BMV: Avoid needless renegotiation for just software LRO. */ if ((mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) == IFCAP_LRO && vtnet_software_lro(sc)) reinit = update = 0; if (mask & IFCAP_RXCSUM) if_togglecapenable(ifp, IFCAP_RXCSUM); if (mask & IFCAP_RXCSUM_IPV6) if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); if (mask & IFCAP_LRO) if_togglecapenable(ifp, IFCAP_LRO); /* * VirtIO does not distinguish between IPv4 and IPv6 checksums * so treat them as a pair. Guest TSO (LRO) requires receive * checksums. */ if (if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { if_setcapenablebit(ifp, IFCAP_RXCSUM, 0); #ifdef notyet if_setcapenablebit(ifp, IFCAP_RXCSUM_IPV6, 0); #endif } else if_setcapenablebit(ifp, 0, (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)); } if (mask & IFCAP_VLAN_HWFILTER) { /* These Rx features require renegotiation. */ reinit = 1; if (mask & IFCAP_VLAN_HWFILTER) if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER); } if (mask & IFCAP_VLAN_HWTSO) if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); if (mask & IFCAP_VLAN_HWTAGGING) if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { if (reinit) { if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); vtnet_init_locked(sc, 0); } else if (update) vtnet_update_rx_offloads(sc); } return (0); } static int vtnet_ioctl(if_t ifp, u_long cmd, caddr_t data) { struct vtnet_softc *sc; struct ifreq *ifr; int error; sc = if_getsoftc(ifp); ifr = (struct ifreq *) data; error = 0; switch (cmd) { case SIOCSIFMTU: VTNET_CORE_LOCK(sc); error = vtnet_ioctl_mtu(sc, ifr->ifr_mtu); VTNET_CORE_UNLOCK(sc); break; case SIOCSIFFLAGS: VTNET_CORE_LOCK(sc); error = vtnet_ioctl_ifflags(sc); VTNET_CORE_UNLOCK(sc); break; case SIOCADDMULTI: case SIOCDELMULTI: VTNET_CORE_LOCK(sc); error = vtnet_ioctl_multi(sc); VTNET_CORE_UNLOCK(sc); break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd); break; case SIOCSIFCAP: VTNET_CORE_LOCK(sc); error = vtnet_ioctl_ifcap(sc, ifr); VTNET_CORE_UNLOCK(sc); VLAN_CAPABILITIES(ifp); break; default: error = ether_ioctl(ifp, cmd, data); break; } VTNET_CORE_LOCK_ASSERT_NOTOWNED(sc); return (error); } static int vtnet_rxq_populate(struct vtnet_rxq *rxq) { struct virtqueue *vq; int nbufs, error; #ifdef DEV_NETMAP error = vtnet_netmap_rxq_populate(rxq); if (error >= 0) return (error); #endif /* DEV_NETMAP */ vq = rxq->vtnrx_vq; error = ENOSPC; for (nbufs = 0; !virtqueue_full(vq); nbufs++) { error = vtnet_rxq_new_buf(rxq); if (error) break; } if (nbufs > 0) { virtqueue_notify(vq); /* * EMSGSIZE signifies the virtqueue did not have enough * entries available to hold the last mbuf. This is not * an error. */ if (error == EMSGSIZE) error = 0; } return (error); } static void vtnet_rxq_free_mbufs(struct vtnet_rxq *rxq) { struct virtqueue *vq; struct mbuf *m; int last; #ifdef DEV_NETMAP struct netmap_kring *kring = netmap_kring_on(NA(rxq->vtnrx_sc->vtnet_ifp), rxq->vtnrx_id, NR_RX); #else /* !DEV_NETMAP */ void *kring = NULL; #endif /* !DEV_NETMAP */ vq = rxq->vtnrx_vq; last = 0; while ((m = virtqueue_drain(vq, &last)) != NULL) { if (kring == NULL) m_freem(m); } KASSERT(virtqueue_empty(vq), ("%s: mbufs remaining in rx queue %p", __func__, rxq)); } static struct mbuf * vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp) { struct mbuf *m_head, *m_tail, *m; int i, size; m_head = NULL; size = sc->vtnet_rx_clustersz; KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, ("%s: mbuf %d chain requested without LRO_NOMRG", __func__, nbufs)); for (i = 0; i < nbufs; i++) { m = m_getjcl(M_NOWAIT, MT_DATA, i == 0 ? M_PKTHDR : 0, size); if (m == NULL) { sc->vtnet_stats.mbuf_alloc_failed++; m_freem(m_head); return (NULL); } m->m_len = size; if (m_head != NULL) { m_tail->m_next = m; m_tail = m; } else m_head = m_tail = m; } if (m_tailp != NULL) *m_tailp = m_tail; return (m_head); } /* * Slow path for when LRO without mergeable buffers is negotiated. */ static int vtnet_rxq_replace_lro_nomrg_buf(struct vtnet_rxq *rxq, struct mbuf *m0, int len0) { struct vtnet_softc *sc; struct mbuf *m, *m_prev, *m_new, *m_tail; int len, clustersz, nreplace, error; sc = rxq->vtnrx_sc; clustersz = sc->vtnet_rx_clustersz; m_prev = NULL; m_tail = NULL; nreplace = 0; m = m0; len = len0; /* * Since these mbuf chains are so large, avoid allocating a complete * replacement when the received frame did not consume the entire * chain. Unused mbufs are moved to the tail of the replacement mbuf. */ while (len > 0) { if (m == NULL) { sc->vtnet_stats.rx_frame_too_large++; return (EMSGSIZE); } /* * Every mbuf should have the expected cluster size since that * is also used to allocate the replacements. */ KASSERT(m->m_len == clustersz, ("%s: mbuf size %d not expected cluster size %d", __func__, m->m_len, clustersz)); m->m_len = MIN(m->m_len, len); len -= m->m_len; m_prev = m; m = m->m_next; nreplace++; } KASSERT(nreplace > 0 && nreplace <= sc->vtnet_rx_nmbufs, ("%s: invalid replacement mbuf count %d max %d", __func__, nreplace, sc->vtnet_rx_nmbufs)); m_new = vtnet_rx_alloc_buf(sc, nreplace, &m_tail); if (m_new == NULL) { m_prev->m_len = clustersz; return (ENOBUFS); } /* * Move any unused mbufs from the received mbuf chain onto the * end of the replacement chain. */ if (m_prev->m_next != NULL) { m_tail->m_next = m_prev->m_next; m_prev->m_next = NULL; } error = vtnet_rxq_enqueue_buf(rxq, m_new); if (error) { /* * The replacement is suppose to be an copy of the one * dequeued so this is a very unexpected error. * * Restore the m0 chain to the original state if it was * modified so we can then discard it. */ if (m_tail->m_next != NULL) { m_prev->m_next = m_tail->m_next; m_tail->m_next = NULL; } m_prev->m_len = clustersz; sc->vtnet_stats.rx_enq_replacement_failed++; m_freem(m_new); } return (error); } static int vtnet_rxq_replace_buf(struct vtnet_rxq *rxq, struct mbuf *m, int len) { struct vtnet_softc *sc; struct mbuf *m_new; int error; sc = rxq->vtnrx_sc; if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) return (vtnet_rxq_replace_lro_nomrg_buf(rxq, m, len)); MPASS(m->m_next == NULL); if (m->m_len < len) return (EMSGSIZE); m_new = vtnet_rx_alloc_buf(sc, 1, NULL); if (m_new == NULL) return (ENOBUFS); error = vtnet_rxq_enqueue_buf(rxq, m_new); if (error) { sc->vtnet_stats.rx_enq_replacement_failed++; m_freem(m_new); } else m->m_len = len; return (error); } static int vtnet_rxq_enqueue_buf(struct vtnet_rxq *rxq, struct mbuf *m) { struct vtnet_softc *sc; struct sglist *sg; int header_inlined, error; sc = rxq->vtnrx_sc; sg = rxq->vtnrx_sg; KASSERT(m->m_next == NULL || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, ("%s: mbuf chain without LRO_NOMRG", __func__)); VTNET_RXQ_LOCK_ASSERT(rxq); sglist_reset(sg); header_inlined = vtnet_modern(sc) || (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) != 0; /* TODO: ANY_LAYOUT */ if (header_inlined) error = sglist_append_mbuf(sg, m); else { struct vtnet_rx_header *rxhdr = mtod(m, struct vtnet_rx_header *); MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr)); /* Append the header and remaining mbuf data. */ error = sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size); if (error) return (error); error = sglist_append(sg, &rxhdr[1], m->m_len - sizeof(struct vtnet_rx_header)); if (error) return (error); if (m->m_next != NULL) error = sglist_append_mbuf(sg, m->m_next); } if (error) return (error); return (virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg)); } static int vtnet_rxq_new_buf(struct vtnet_rxq *rxq) { struct vtnet_softc *sc; struct mbuf *m; int error; sc = rxq->vtnrx_sc; m = vtnet_rx_alloc_buf(sc, sc->vtnet_rx_nmbufs, NULL); if (m == NULL) return (ENOBUFS); error = vtnet_rxq_enqueue_buf(rxq, m); if (error) m_freem(m); return (error); } static int vtnet_rxq_csum_needs_csum(struct vtnet_rxq *rxq, struct mbuf *m, uint16_t etype, int hoff, struct virtio_net_hdr *hdr) { struct vtnet_softc *sc; int error; sc = rxq->vtnrx_sc; /* * NEEDS_CSUM corresponds to Linux's CHECKSUM_PARTIAL, but FreeBSD does * not have an analogous CSUM flag. The checksum has been validated, * but is incomplete (TCP/UDP pseudo header). * * The packet is likely from another VM on the same host that itself * performed checksum offloading so Tx/Rx is basically a memcpy and * the checksum has little value. * * Default to receiving the packet as-is for performance reasons, but * this can cause issues if the packet is to be forwarded because it * does not contain a valid checksum. This patch may be helpful: * https://reviews.freebsd.org/D6611. In the meantime, have the driver * compute the checksum if requested. * * BMV: Need to add an CSUM_PARTIAL flag? */ if ((sc->vtnet_flags & VTNET_FLAG_FIXUP_NEEDS_CSUM) == 0) { error = vtnet_rxq_csum_data_valid(rxq, m, etype, hoff, hdr); return (error); } /* * Compute the checksum in the driver so the packet will contain a * valid checksum. The checksum is at csum_offset from csum_start. */ switch (etype) { #if defined(INET) || defined(INET6) case ETHERTYPE_IP: case ETHERTYPE_IPV6: { int csum_off, csum_end; uint16_t csum; csum_off = hdr->csum_start + hdr->csum_offset; csum_end = csum_off + sizeof(uint16_t); /* Assume checksum will be in the first mbuf. */ if (m->m_len < csum_end || m->m_pkthdr.len < csum_end) return (1); /* * Like in_delayed_cksum()/in6_delayed_cksum(), compute the * checksum and write it at the specified offset. We could * try to verify the packet: csum_start should probably * correspond to the start of the TCP/UDP header. * * BMV: Need to properly handle UDP with zero checksum. Is * the IPv4 header checksum implicitly validated? */ csum = in_cksum_skip(m, m->m_pkthdr.len, hdr->csum_start); *(uint16_t *)(mtodo(m, csum_off)) = csum; m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; m->m_pkthdr.csum_data = 0xFFFF; break; } #endif default: sc->vtnet_stats.rx_csum_bad_ethtype++; return (1); } return (0); } static int vtnet_rxq_csum_data_valid(struct vtnet_rxq *rxq, struct mbuf *m, uint16_t etype, int hoff, struct virtio_net_hdr *hdr __unused) { #if 0 struct vtnet_softc *sc; #endif int protocol; #if 0 sc = rxq->vtnrx_sc; #endif switch (etype) { #if defined(INET) case ETHERTYPE_IP: if (__predict_false(m->m_len < hoff + sizeof(struct ip))) protocol = IPPROTO_DONE; else { struct ip *ip = (struct ip *)(m->m_data + hoff); protocol = ip->ip_p; } break; #endif #if defined(INET6) case ETHERTYPE_IPV6: if (__predict_false(m->m_len < hoff + sizeof(struct ip6_hdr)) || ip6_lasthdr(m, hoff, IPPROTO_IPV6, &protocol) < 0) protocol = IPPROTO_DONE; break; #endif default: protocol = IPPROTO_DONE; break; } switch (protocol) { case IPPROTO_TCP: case IPPROTO_UDP: m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; m->m_pkthdr.csum_data = 0xFFFF; break; default: /* * FreeBSD does not support checksum offloading of this * protocol. Let the stack re-verify the checksum later * if the protocol is supported. */ #if 0 if_printf(sc->vtnet_ifp, "%s: checksum offload of unsupported protocol " "etype=%#x protocol=%d csum_start=%d csum_offset=%d\n", __func__, etype, protocol, hdr->csum_start, hdr->csum_offset); #endif break; } return (0); } static int vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m, struct virtio_net_hdr *hdr) { const struct ether_header *eh; int hoff; uint16_t etype; eh = mtod(m, const struct ether_header *); etype = ntohs(eh->ether_type); if (etype == ETHERTYPE_VLAN) { /* TODO BMV: Handle QinQ. */ const struct ether_vlan_header *evh = mtod(m, const struct ether_vlan_header *); etype = ntohs(evh->evl_proto); hoff = sizeof(struct ether_vlan_header); } else hoff = sizeof(struct ether_header); if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) return (vtnet_rxq_csum_needs_csum(rxq, m, etype, hoff, hdr)); else /* VIRTIO_NET_HDR_F_DATA_VALID */ return (vtnet_rxq_csum_data_valid(rxq, m, etype, hoff, hdr)); } static void vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *rxq, int nbufs) { struct mbuf *m; while (--nbufs > 0) { m = virtqueue_dequeue(rxq->vtnrx_vq, NULL); if (m == NULL) break; vtnet_rxq_discard_buf(rxq, m); } } static void vtnet_rxq_discard_buf(struct vtnet_rxq *rxq, struct mbuf *m) { int error __diagused; /* * Requeue the discarded mbuf. This should always be successful * since it was just dequeued. */ error = vtnet_rxq_enqueue_buf(rxq, m); KASSERT(error == 0, ("%s: cannot requeue discarded mbuf %d", __func__, error)); } static int vtnet_rxq_merged_eof(struct vtnet_rxq *rxq, struct mbuf *m_head, int nbufs) { struct vtnet_softc *sc; struct virtqueue *vq; struct mbuf *m_tail; sc = rxq->vtnrx_sc; vq = rxq->vtnrx_vq; m_tail = m_head; while (--nbufs > 0) { struct mbuf *m; uint32_t len; m = virtqueue_dequeue(vq, &len); if (m == NULL) { rxq->vtnrx_stats.vrxs_ierrors++; goto fail; } if (vtnet_rxq_new_buf(rxq) != 0) { rxq->vtnrx_stats.vrxs_iqdrops++; vtnet_rxq_discard_buf(rxq, m); if (nbufs > 1) vtnet_rxq_discard_merged_bufs(rxq, nbufs); goto fail; } if (m->m_len < len) len = m->m_len; m->m_len = len; m->m_flags &= ~M_PKTHDR; m_head->m_pkthdr.len += len; m_tail->m_next = m; m_tail = m; } return (0); fail: sc->vtnet_stats.rx_mergeable_failed++; m_freem(m_head); return (1); } #if defined(INET) || defined(INET6) static int vtnet_lro_rx(struct vtnet_rxq *rxq, struct mbuf *m) { struct lro_ctrl *lro; lro = &rxq->vtnrx_lro; if (lro->lro_mbuf_max != 0) { tcp_lro_queue_mbuf(lro, m); return (0); } return (tcp_lro_rx(lro, m, 0)); } #endif static void vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m, struct virtio_net_hdr *hdr) { struct vtnet_softc *sc; if_t ifp; sc = rxq->vtnrx_sc; ifp = sc->vtnet_ifp; if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { struct ether_header *eh = mtod(m, struct ether_header *); if (eh->ether_type == htons(ETHERTYPE_VLAN)) { vtnet_vlan_tag_remove(m); /* * With the 802.1Q header removed, update the * checksum starting location accordingly. */ if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) hdr->csum_start -= ETHER_VLAN_ENCAP_LEN; } } m->m_pkthdr.flowid = rxq->vtnrx_id; M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); if (hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM | VIRTIO_NET_HDR_F_DATA_VALID)) { if (vtnet_rxq_csum(rxq, m, hdr) == 0) rxq->vtnrx_stats.vrxs_csum++; else rxq->vtnrx_stats.vrxs_csum_failed++; } if (hdr->gso_size != 0) { switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { case VIRTIO_NET_HDR_GSO_TCPV4: case VIRTIO_NET_HDR_GSO_TCPV6: m->m_pkthdr.lro_nsegs = howmany(m->m_pkthdr.len, hdr->gso_size); rxq->vtnrx_stats.vrxs_host_lro++; break; } } rxq->vtnrx_stats.vrxs_ipackets++; rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len; #if defined(INET) || defined(INET6) if (vtnet_software_lro(sc) && if_getcapenable(ifp) & IFCAP_LRO) { if (vtnet_lro_rx(rxq, m) == 0) return; } #endif if_input(ifp, m); } static int vtnet_rxq_eof(struct vtnet_rxq *rxq) { struct virtio_net_hdr lhdr, *hdr; struct vtnet_softc *sc; if_t ifp; struct virtqueue *vq; int deq, count; sc = rxq->vtnrx_sc; vq = rxq->vtnrx_vq; ifp = sc->vtnet_ifp; deq = 0; count = sc->vtnet_rx_process_limit; VTNET_RXQ_LOCK_ASSERT(rxq); while (count-- > 0) { struct mbuf *m; uint32_t len, nbufs, adjsz; m = virtqueue_dequeue(vq, &len); if (m == NULL) break; deq++; if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) { rxq->vtnrx_stats.vrxs_ierrors++; vtnet_rxq_discard_buf(rxq, m); continue; } if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) { struct virtio_net_hdr_mrg_rxbuf *mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *); kmsan_mark(mhdr, sizeof(*mhdr), KMSAN_STATE_INITED); nbufs = vtnet_htog16(sc, mhdr->num_buffers); adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf); } else if (vtnet_modern(sc)) { nbufs = 1; /* num_buffers is always 1 */ adjsz = sizeof(struct virtio_net_hdr_v1); } else { nbufs = 1; adjsz = sizeof(struct vtnet_rx_header); /* * Account for our gap between the header and start of * data to keep the segments separated. */ len += VTNET_RX_HEADER_PAD; } if (vtnet_rxq_replace_buf(rxq, m, len) != 0) { rxq->vtnrx_stats.vrxs_iqdrops++; vtnet_rxq_discard_buf(rxq, m); if (nbufs > 1) vtnet_rxq_discard_merged_bufs(rxq, nbufs); continue; } m->m_pkthdr.len = len; m->m_pkthdr.rcvif = ifp; m->m_pkthdr.csum_flags = 0; if (nbufs > 1) { /* Dequeue the rest of chain. */ if (vtnet_rxq_merged_eof(rxq, m, nbufs) != 0) continue; } kmsan_mark_mbuf(m, KMSAN_STATE_INITED); /* * Save an endian swapped version of the header prior to it * being stripped. The header is always at the start of the * mbuf data. num_buffers was already saved (and not needed) * so use the standard header. */ hdr = mtod(m, struct virtio_net_hdr *); lhdr.flags = hdr->flags; lhdr.gso_type = hdr->gso_type; lhdr.hdr_len = vtnet_htog16(sc, hdr->hdr_len); lhdr.gso_size = vtnet_htog16(sc, hdr->gso_size); lhdr.csum_start = vtnet_htog16(sc, hdr->csum_start); lhdr.csum_offset = vtnet_htog16(sc, hdr->csum_offset); m_adj(m, adjsz); if (PFIL_HOOKED_IN(sc->vtnet_pfil)) { pfil_return_t pfil; pfil = pfil_mbuf_in(sc->vtnet_pfil, &m, ifp, NULL); switch (pfil) { case PFIL_DROPPED: case PFIL_CONSUMED: continue; default: KASSERT(pfil == PFIL_PASS, ("Filter returned %d!", pfil)); } } vtnet_rxq_input(rxq, m, &lhdr); } if (deq > 0) { #if defined(INET) || defined(INET6) if (vtnet_software_lro(sc)) tcp_lro_flush_all(&rxq->vtnrx_lro); #endif virtqueue_notify(vq); } return (count > 0 ? 0 : EAGAIN); } static void vtnet_rx_vq_process(struct vtnet_rxq *rxq, int tries) { struct vtnet_softc *sc; if_t ifp; u_int more; #ifdef DEV_NETMAP int nmirq; #endif /* DEV_NETMAP */ sc = rxq->vtnrx_sc; ifp = sc->vtnet_ifp; if (__predict_false(rxq->vtnrx_id >= sc->vtnet_act_vq_pairs)) { /* * Ignore this interrupt. Either this is a spurious interrupt * or multiqueue without per-VQ MSIX so every queue needs to * be polled (a brain dead configuration we could try harder * to avoid). */ vtnet_rxq_disable_intr(rxq); return; } VTNET_RXQ_LOCK(rxq); #ifdef DEV_NETMAP /* * We call netmap_rx_irq() under lock to prevent concurrent calls. * This is not necessary to serialize the access to the RX vq, but * rather to avoid races that may happen if this interface is * attached to a VALE switch, which would cause received packets * to stall in the RX queue (nm_kr_tryget() could find the kring * busy when called from netmap_bwrap_intr_notify()). */ nmirq = netmap_rx_irq(ifp, rxq->vtnrx_id, &more); if (nmirq != NM_IRQ_PASS) { VTNET_RXQ_UNLOCK(rxq); if (nmirq == NM_IRQ_RESCHED) { taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); } return; } #endif /* DEV_NETMAP */ again: if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { VTNET_RXQ_UNLOCK(rxq); return; } more = vtnet_rxq_eof(rxq); if (more || vtnet_rxq_enable_intr(rxq) != 0) { if (!more) vtnet_rxq_disable_intr(rxq); /* * This is an occasional condition or race (when !more), * so retry a few times before scheduling the taskqueue. */ if (tries-- > 0) goto again; rxq->vtnrx_stats.vrxs_rescheduled++; VTNET_RXQ_UNLOCK(rxq); taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); } else VTNET_RXQ_UNLOCK(rxq); } static void vtnet_rx_vq_intr(void *xrxq) { struct vtnet_rxq *rxq; rxq = xrxq; vtnet_rx_vq_process(rxq, VTNET_INTR_DISABLE_RETRIES); } static void vtnet_rxq_tq_intr(void *xrxq, int pending __unused) { struct vtnet_rxq *rxq; rxq = xrxq; vtnet_rx_vq_process(rxq, 0); } static int vtnet_txq_intr_threshold(struct vtnet_txq *txq) { struct vtnet_softc *sc; int threshold; sc = txq->vtntx_sc; /* * The Tx interrupt is disabled until the queue free count falls * below our threshold. Completed frames are drained from the Tx * virtqueue before transmitting new frames and in the watchdog * callout, so the frequency of Tx interrupts is greatly reduced, * at the cost of not freeing mbufs as quickly as they otherwise * would be. */ threshold = virtqueue_size(txq->vtntx_vq) / 4; /* * Without indirect descriptors, leave enough room for the most * segments we handle. */ if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) == 0 && threshold < sc->vtnet_tx_nsegs) threshold = sc->vtnet_tx_nsegs; return (threshold); } static int vtnet_txq_below_threshold(struct vtnet_txq *txq) { struct virtqueue *vq; vq = txq->vtntx_vq; return (virtqueue_nfree(vq) <= txq->vtntx_intr_threshold); } static int vtnet_txq_notify(struct vtnet_txq *txq) { struct virtqueue *vq; vq = txq->vtntx_vq; txq->vtntx_watchdog = VTNET_TX_TIMEOUT; virtqueue_notify(vq); if (vtnet_txq_enable_intr(txq) == 0) return (0); /* * Drain frames that were completed since last checked. If this * causes the queue to go above the threshold, the caller should * continue transmitting. */ if (vtnet_txq_eof(txq) != 0 && vtnet_txq_below_threshold(txq) == 0) { virtqueue_disable_intr(vq); return (1); } return (0); } static void vtnet_txq_free_mbufs(struct vtnet_txq *txq) { struct virtqueue *vq; struct vtnet_tx_header *txhdr; int last; #ifdef DEV_NETMAP struct netmap_kring *kring = netmap_kring_on(NA(txq->vtntx_sc->vtnet_ifp), txq->vtntx_id, NR_TX); #else /* !DEV_NETMAP */ void *kring = NULL; #endif /* !DEV_NETMAP */ vq = txq->vtntx_vq; last = 0; while ((txhdr = virtqueue_drain(vq, &last)) != NULL) { if (kring == NULL) { m_freem(txhdr->vth_mbuf); uma_zfree(vtnet_tx_header_zone, txhdr); } } KASSERT(virtqueue_empty(vq), ("%s: mbufs remaining in tx queue %p", __func__, txq)); } /* * BMV: This can go away once we finally have offsets in the mbuf header. */ static int vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m, int *etype, int *proto, int *start) { struct vtnet_softc *sc; struct ether_vlan_header *evh; #if defined(INET) || defined(INET6) int offset; #endif sc = txq->vtntx_sc; evh = mtod(m, struct ether_vlan_header *); if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { /* BMV: We should handle nested VLAN tags too. */ *etype = ntohs(evh->evl_proto); #if defined(INET) || defined(INET6) offset = sizeof(struct ether_vlan_header); #endif } else { *etype = ntohs(evh->evl_encap_proto); #if defined(INET) || defined(INET6) offset = sizeof(struct ether_header); #endif } switch (*etype) { #if defined(INET) case ETHERTYPE_IP: { struct ip *ip, iphdr; if (__predict_false(m->m_len < offset + sizeof(struct ip))) { m_copydata(m, offset, sizeof(struct ip), (caddr_t) &iphdr); ip = &iphdr; } else ip = (struct ip *)(m->m_data + offset); *proto = ip->ip_p; *start = offset + (ip->ip_hl << 2); break; } #endif #if defined(INET6) case ETHERTYPE_IPV6: *proto = -1; *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto); /* Assert the network stack sent us a valid packet. */ KASSERT(*start > offset, ("%s: mbuf %p start %d offset %d proto %d", __func__, m, *start, offset, *proto)); break; #endif default: sc->vtnet_stats.tx_csum_unknown_ethtype++; return (EINVAL); } return (0); } static int vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type, int offset, struct virtio_net_hdr *hdr) { static struct timeval lastecn; static int curecn; struct vtnet_softc *sc; struct tcphdr *tcp, tcphdr; sc = txq->vtntx_sc; if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) { m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr); tcp = &tcphdr; } else tcp = (struct tcphdr *)(m->m_data + offset); hdr->hdr_len = vtnet_gtoh16(sc, offset + (tcp->th_off << 2)); hdr->gso_size = vtnet_gtoh16(sc, m->m_pkthdr.tso_segsz); hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 : VIRTIO_NET_HDR_GSO_TCPV6; if (__predict_false(tcp->th_flags & TH_CWR)) { /* * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In * FreeBSD, ECN support is not on a per-interface basis, * but globally via the net.inet.tcp.ecn.enable sysctl * knob. The default is off. */ if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) { if (ppsratecheck(&lastecn, &curecn, 1)) if_printf(sc->vtnet_ifp, "TSO with ECN not negotiated with host\n"); return (ENOTSUP); } hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; } txq->vtntx_stats.vtxs_tso++; return (0); } static struct mbuf * vtnet_txq_offload(struct vtnet_txq *txq, struct mbuf *m, struct virtio_net_hdr *hdr) { struct vtnet_softc *sc; int flags, etype, csum_start, proto, error; sc = txq->vtntx_sc; flags = m->m_pkthdr.csum_flags; error = vtnet_txq_offload_ctx(txq, m, &etype, &proto, &csum_start); if (error) goto drop; if (flags & (VTNET_CSUM_OFFLOAD | VTNET_CSUM_OFFLOAD_IPV6)) { /* Sanity check the parsed mbuf matches the offload flags. */ if (__predict_false((flags & VTNET_CSUM_OFFLOAD && etype != ETHERTYPE_IP) || (flags & VTNET_CSUM_OFFLOAD_IPV6 && etype != ETHERTYPE_IPV6))) { sc->vtnet_stats.tx_csum_proto_mismatch++; goto drop; } hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = vtnet_gtoh16(sc, csum_start); hdr->csum_offset = vtnet_gtoh16(sc, m->m_pkthdr.csum_data); txq->vtntx_stats.vtxs_csum++; } if (flags & (CSUM_IP_TSO | CSUM_IP6_TSO)) { /* * Sanity check the parsed mbuf IP protocol is TCP, and * VirtIO TSO reqires the checksum offloading above. */ if (__predict_false(proto != IPPROTO_TCP)) { sc->vtnet_stats.tx_tso_not_tcp++; goto drop; } else if (__predict_false((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) { sc->vtnet_stats.tx_tso_without_csum++; goto drop; } error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr); if (error) goto drop; } return (m); drop: m_freem(m); return (NULL); } static int vtnet_txq_enqueue_buf(struct vtnet_txq *txq, struct mbuf **m_head, struct vtnet_tx_header *txhdr) { struct vtnet_softc *sc; struct virtqueue *vq; struct sglist *sg; struct mbuf *m; int error; sc = txq->vtntx_sc; vq = txq->vtntx_vq; sg = txq->vtntx_sg; m = *m_head; sglist_reset(sg); error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size); if (error != 0 || sg->sg_nseg != 1) { KASSERT(0, ("%s: cannot add header to sglist error %d nseg %d", __func__, error, sg->sg_nseg)); goto fail; } error = sglist_append_mbuf(sg, m); if (error) { m = m_defrag(m, M_NOWAIT); if (m == NULL) goto fail; *m_head = m; sc->vtnet_stats.tx_defragged++; error = sglist_append_mbuf(sg, m); if (error) goto fail; } txhdr->vth_mbuf = m; error = virtqueue_enqueue(vq, txhdr, sg, sg->sg_nseg, 0); return (error); fail: sc->vtnet_stats.tx_defrag_failed++; m_freem(*m_head); *m_head = NULL; return (ENOBUFS); } static int vtnet_txq_encap(struct vtnet_txq *txq, struct mbuf **m_head, int flags) { struct vtnet_tx_header *txhdr; struct virtio_net_hdr *hdr; struct mbuf *m; int error; m = *m_head; M_ASSERTPKTHDR(m); txhdr = uma_zalloc(vtnet_tx_header_zone, flags | M_ZERO); if (txhdr == NULL) { m_freem(m); *m_head = NULL; return (ENOMEM); } /* * Always use the non-mergeable header, regardless if mergable headers * were negotiated, because for transmit num_buffers is always zero. * The vtnet_hdr_size is used to enqueue the right header size segment. */ hdr = &txhdr->vth_uhdr.hdr; if (m->m_flags & M_VLANTAG) { m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); if ((*m_head = m) == NULL) { error = ENOBUFS; goto fail; } m->m_flags &= ~M_VLANTAG; } if (m->m_pkthdr.csum_flags & VTNET_CSUM_ALL_OFFLOAD) { m = vtnet_txq_offload(txq, m, hdr); if ((*m_head = m) == NULL) { error = ENOBUFS; goto fail; } } error = vtnet_txq_enqueue_buf(txq, m_head, txhdr); fail: if (error) uma_zfree(vtnet_tx_header_zone, txhdr); return (error); } #ifdef VTNET_LEGACY_TX static void vtnet_start_locked(struct vtnet_txq *txq, if_t ifp) { struct vtnet_softc *sc; struct virtqueue *vq; struct mbuf *m0; int tries, enq; sc = txq->vtntx_sc; vq = txq->vtntx_vq; tries = 0; VTNET_TXQ_LOCK_ASSERT(txq); if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || sc->vtnet_link_active == 0) return; vtnet_txq_eof(txq); again: enq = 0; while (!if_sendq_empty(ifp)) { if (virtqueue_full(vq)) break; m0 = if_dequeue(ifp); if (m0 == NULL) break; if (vtnet_txq_encap(txq, &m0, M_NOWAIT) != 0) { if (m0 != NULL) if_sendq_prepend(ifp, m0); break; } enq++; ETHER_BPF_MTAP(ifp, m0); } if (enq > 0 && vtnet_txq_notify(txq) != 0) { if (tries++ < VTNET_NOTIFY_RETRIES) goto again; txq->vtntx_stats.vtxs_rescheduled++; taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); } } static void vtnet_start(if_t ifp) { struct vtnet_softc *sc; struct vtnet_txq *txq; sc = if_getsoftc(ifp); txq = &sc->vtnet_txqs[0]; VTNET_TXQ_LOCK(txq); vtnet_start_locked(txq, ifp); VTNET_TXQ_UNLOCK(txq); } #else /* !VTNET_LEGACY_TX */ static int vtnet_txq_mq_start_locked(struct vtnet_txq *txq, struct mbuf *m) { struct vtnet_softc *sc; struct virtqueue *vq; struct buf_ring *br; if_t ifp; int enq, tries, error; sc = txq->vtntx_sc; vq = txq->vtntx_vq; br = txq->vtntx_br; ifp = sc->vtnet_ifp; tries = 0; error = 0; VTNET_TXQ_LOCK_ASSERT(txq); if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || sc->vtnet_link_active == 0) { if (m != NULL) error = drbr_enqueue(ifp, br, m); return (error); } if (m != NULL) { error = drbr_enqueue(ifp, br, m); if (error) return (error); } vtnet_txq_eof(txq); again: enq = 0; while ((m = drbr_peek(ifp, br)) != NULL) { if (virtqueue_full(vq)) { drbr_putback(ifp, br, m); break; } if (vtnet_txq_encap(txq, &m, M_NOWAIT) != 0) { if (m != NULL) drbr_putback(ifp, br, m); else drbr_advance(ifp, br); break; } drbr_advance(ifp, br); enq++; ETHER_BPF_MTAP(ifp, m); } if (enq > 0 && vtnet_txq_notify(txq) != 0) { if (tries++ < VTNET_NOTIFY_RETRIES) goto again; txq->vtntx_stats.vtxs_rescheduled++; taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_intrtask); } return (0); } static int vtnet_txq_mq_start(if_t ifp, struct mbuf *m) { struct vtnet_softc *sc; struct vtnet_txq *txq; int i, npairs, error; sc = if_getsoftc(ifp); npairs = sc->vtnet_act_vq_pairs; if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) i = m->m_pkthdr.flowid % npairs; else i = curcpu % npairs; txq = &sc->vtnet_txqs[i]; if (VTNET_TXQ_TRYLOCK(txq) != 0) { error = vtnet_txq_mq_start_locked(txq, m); VTNET_TXQ_UNLOCK(txq); } else { error = drbr_enqueue(ifp, txq->vtntx_br, m); taskqueue_enqueue(txq->vtntx_tq, &txq->vtntx_defrtask); } return (error); } static void vtnet_txq_tq_deferred(void *xtxq, int pending __unused) { struct vtnet_softc *sc; struct vtnet_txq *txq; txq = xtxq; sc = txq->vtntx_sc; VTNET_TXQ_LOCK(txq); if (!drbr_empty(sc->vtnet_ifp, txq->vtntx_br)) vtnet_txq_mq_start_locked(txq, NULL); VTNET_TXQ_UNLOCK(txq); } #endif /* VTNET_LEGACY_TX */ static void vtnet_txq_start(struct vtnet_txq *txq) { struct vtnet_softc *sc; if_t ifp; sc = txq->vtntx_sc; ifp = sc->vtnet_ifp; #ifdef VTNET_LEGACY_TX if (!if_sendq_empty(ifp)) vtnet_start_locked(txq, ifp); #else if (!drbr_empty(ifp, txq->vtntx_br)) vtnet_txq_mq_start_locked(txq, NULL); #endif } static void vtnet_txq_tq_intr(void *xtxq, int pending __unused) { struct vtnet_softc *sc; struct vtnet_txq *txq; if_t ifp; txq = xtxq; sc = txq->vtntx_sc; ifp = sc->vtnet_ifp; VTNET_TXQ_LOCK(txq); if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { VTNET_TXQ_UNLOCK(txq); return; } vtnet_txq_eof(txq); vtnet_txq_start(txq); VTNET_TXQ_UNLOCK(txq); } static int vtnet_txq_eof(struct vtnet_txq *txq) { struct virtqueue *vq; struct vtnet_tx_header *txhdr; struct mbuf *m; int deq; vq = txq->vtntx_vq; deq = 0; VTNET_TXQ_LOCK_ASSERT(txq); while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) { m = txhdr->vth_mbuf; deq++; txq->vtntx_stats.vtxs_opackets++; txq->vtntx_stats.vtxs_obytes += m->m_pkthdr.len; if (m->m_flags & M_MCAST) txq->vtntx_stats.vtxs_omcasts++; m_freem(m); uma_zfree(vtnet_tx_header_zone, txhdr); } if (virtqueue_empty(vq)) txq->vtntx_watchdog = 0; return (deq); } static void vtnet_tx_vq_intr(void *xtxq) { struct vtnet_softc *sc; struct vtnet_txq *txq; if_t ifp; txq = xtxq; sc = txq->vtntx_sc; ifp = sc->vtnet_ifp; if (__predict_false(txq->vtntx_id >= sc->vtnet_act_vq_pairs)) { /* * Ignore this interrupt. Either this is a spurious interrupt * or multiqueue without per-VQ MSIX so every queue needs to * be polled (a brain dead configuration we could try harder * to avoid). */ vtnet_txq_disable_intr(txq); return; } #ifdef DEV_NETMAP if (netmap_tx_irq(ifp, txq->vtntx_id) != NM_IRQ_PASS) return; #endif /* DEV_NETMAP */ VTNET_TXQ_LOCK(txq); if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { VTNET_TXQ_UNLOCK(txq); return; } vtnet_txq_eof(txq); vtnet_txq_start(txq); VTNET_TXQ_UNLOCK(txq); } static void vtnet_tx_start_all(struct vtnet_softc *sc) { struct vtnet_txq *txq; int i; VTNET_CORE_LOCK_ASSERT(sc); for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { txq = &sc->vtnet_txqs[i]; VTNET_TXQ_LOCK(txq); vtnet_txq_start(txq); VTNET_TXQ_UNLOCK(txq); } } #ifndef VTNET_LEGACY_TX static void vtnet_qflush(if_t ifp) { struct vtnet_softc *sc; struct vtnet_txq *txq; struct mbuf *m; int i; sc = if_getsoftc(ifp); for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { txq = &sc->vtnet_txqs[i]; VTNET_TXQ_LOCK(txq); while ((m = buf_ring_dequeue_sc(txq->vtntx_br)) != NULL) m_freem(m); VTNET_TXQ_UNLOCK(txq); } if_qflush(ifp); } #endif static int vtnet_watchdog(struct vtnet_txq *txq) { if_t ifp; ifp = txq->vtntx_sc->vtnet_ifp; VTNET_TXQ_LOCK(txq); if (txq->vtntx_watchdog == 1) { /* * Only drain completed frames if the watchdog is about to * expire. If any frames were drained, there may be enough * free descriptors now available to transmit queued frames. * In that case, the timer will immediately be decremented * below, but the timeout is generous enough that should not * be a problem. */ if (vtnet_txq_eof(txq) != 0) vtnet_txq_start(txq); } if (txq->vtntx_watchdog == 0 || --txq->vtntx_watchdog) { VTNET_TXQ_UNLOCK(txq); return (0); } VTNET_TXQ_UNLOCK(txq); if_printf(ifp, "watchdog timeout on queue %d\n", txq->vtntx_id); return (1); } static void vtnet_accum_stats(struct vtnet_softc *sc, struct vtnet_rxq_stats *rxacc, struct vtnet_txq_stats *txacc) { bzero(rxacc, sizeof(struct vtnet_rxq_stats)); bzero(txacc, sizeof(struct vtnet_txq_stats)); for (int i = 0; i < sc->vtnet_max_vq_pairs; i++) { struct vtnet_rxq_stats *rxst; struct vtnet_txq_stats *txst; rxst = &sc->vtnet_rxqs[i].vtnrx_stats; rxacc->vrxs_ipackets += rxst->vrxs_ipackets; rxacc->vrxs_ibytes += rxst->vrxs_ibytes; rxacc->vrxs_iqdrops += rxst->vrxs_iqdrops; rxacc->vrxs_csum += rxst->vrxs_csum; rxacc->vrxs_csum_failed += rxst->vrxs_csum_failed; rxacc->vrxs_rescheduled += rxst->vrxs_rescheduled; txst = &sc->vtnet_txqs[i].vtntx_stats; txacc->vtxs_opackets += txst->vtxs_opackets; txacc->vtxs_obytes += txst->vtxs_obytes; txacc->vtxs_csum += txst->vtxs_csum; txacc->vtxs_tso += txst->vtxs_tso; txacc->vtxs_rescheduled += txst->vtxs_rescheduled; } } static uint64_t vtnet_get_counter(if_t ifp, ift_counter cnt) { struct vtnet_softc *sc; struct vtnet_rxq_stats rxaccum; struct vtnet_txq_stats txaccum; sc = if_getsoftc(ifp); vtnet_accum_stats(sc, &rxaccum, &txaccum); switch (cnt) { case IFCOUNTER_IPACKETS: return (rxaccum.vrxs_ipackets); case IFCOUNTER_IQDROPS: return (rxaccum.vrxs_iqdrops); case IFCOUNTER_IERRORS: return (rxaccum.vrxs_ierrors); case IFCOUNTER_OPACKETS: return (txaccum.vtxs_opackets); #ifndef VTNET_LEGACY_TX case IFCOUNTER_OBYTES: return (txaccum.vtxs_obytes); case IFCOUNTER_OMCASTS: return (txaccum.vtxs_omcasts); #endif default: return (if_get_counter_default(ifp, cnt)); } } static void vtnet_tick(void *xsc) { struct vtnet_softc *sc; if_t ifp; int i, timedout; sc = xsc; ifp = sc->vtnet_ifp; timedout = 0; VTNET_CORE_LOCK_ASSERT(sc); for (i = 0; i < sc->vtnet_act_vq_pairs; i++) timedout |= vtnet_watchdog(&sc->vtnet_txqs[i]); if (timedout != 0) { if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); vtnet_init_locked(sc, 0); } else callout_schedule(&sc->vtnet_tick_ch, hz); } static void vtnet_start_taskqueues(struct vtnet_softc *sc) { device_t dev; struct vtnet_rxq *rxq; struct vtnet_txq *txq; int i, error; dev = sc->vtnet_dev; /* * Errors here are very difficult to recover from - we cannot * easily fail because, if this is during boot, we will hang * when freeing any successfully started taskqueues because * the scheduler isn't up yet. * * Most drivers just ignore the return value - it only fails * with ENOMEM so an error is not likely. */ for (i = 0; i < sc->vtnet_req_vq_pairs; i++) { rxq = &sc->vtnet_rxqs[i]; error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET, "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id); if (error) { device_printf(dev, "failed to start rx taskq %d\n", rxq->vtnrx_id); } txq = &sc->vtnet_txqs[i]; error = taskqueue_start_threads(&txq->vtntx_tq, 1, PI_NET, "%s txq %d", device_get_nameunit(dev), txq->vtntx_id); if (error) { device_printf(dev, "failed to start tx taskq %d\n", txq->vtntx_id); } } } static void vtnet_free_taskqueues(struct vtnet_softc *sc) { struct vtnet_rxq *rxq; struct vtnet_txq *txq; int i; for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { rxq = &sc->vtnet_rxqs[i]; if (rxq->vtnrx_tq != NULL) { taskqueue_free(rxq->vtnrx_tq); rxq->vtnrx_tq = NULL; } txq = &sc->vtnet_txqs[i]; if (txq->vtntx_tq != NULL) { taskqueue_free(txq->vtntx_tq); txq->vtntx_tq = NULL; } } } static void vtnet_drain_taskqueues(struct vtnet_softc *sc) { struct vtnet_rxq *rxq; struct vtnet_txq *txq; int i; for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { rxq = &sc->vtnet_rxqs[i]; if (rxq->vtnrx_tq != NULL) taskqueue_drain(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); txq = &sc->vtnet_txqs[i]; if (txq->vtntx_tq != NULL) { taskqueue_drain(txq->vtntx_tq, &txq->vtntx_intrtask); #ifndef VTNET_LEGACY_TX taskqueue_drain(txq->vtntx_tq, &txq->vtntx_defrtask); #endif } } } static void vtnet_drain_rxtx_queues(struct vtnet_softc *sc) { struct vtnet_rxq *rxq; struct vtnet_txq *txq; int i; for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { rxq = &sc->vtnet_rxqs[i]; vtnet_rxq_free_mbufs(rxq); txq = &sc->vtnet_txqs[i]; vtnet_txq_free_mbufs(txq); } } static void vtnet_stop_rendezvous(struct vtnet_softc *sc) { struct vtnet_rxq *rxq; struct vtnet_txq *txq; int i; VTNET_CORE_LOCK_ASSERT(sc); /* * Lock and unlock the per-queue mutex so we known the stop * state is visible. Doing only the active queues should be * sufficient, but it does not cost much extra to do all the * queues. */ for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { rxq = &sc->vtnet_rxqs[i]; VTNET_RXQ_LOCK(rxq); VTNET_RXQ_UNLOCK(rxq); txq = &sc->vtnet_txqs[i]; VTNET_TXQ_LOCK(txq); VTNET_TXQ_UNLOCK(txq); } } static void vtnet_stop(struct vtnet_softc *sc) { device_t dev; if_t ifp; dev = sc->vtnet_dev; ifp = sc->vtnet_ifp; VTNET_CORE_LOCK_ASSERT(sc); if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); sc->vtnet_link_active = 0; callout_stop(&sc->vtnet_tick_ch); /* Only advisory. */ vtnet_disable_interrupts(sc); #ifdef DEV_NETMAP /* Stop any pending txsync/rxsync and disable them. */ netmap_disable_all_rings(ifp); #endif /* DEV_NETMAP */ /* * Stop the host adapter. This resets it to the pre-initialized * state. It will not generate any interrupts until after it is * reinitialized. */ virtio_stop(dev); vtnet_stop_rendezvous(sc); vtnet_drain_rxtx_queues(sc); sc->vtnet_act_vq_pairs = 1; } static int vtnet_virtio_reinit(struct vtnet_softc *sc) { device_t dev; if_t ifp; uint64_t features; int error; dev = sc->vtnet_dev; ifp = sc->vtnet_ifp; features = sc->vtnet_negotiated_features; /* * Re-negotiate with the host, removing any disabled receive * features. Transmit features are disabled only on our side * via if_capenable and if_hwassist. */ if ((if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) == 0) features &= ~(VIRTIO_NET_F_GUEST_CSUM | VTNET_LRO_FEATURES); if ((if_getcapenable(ifp) & IFCAP_LRO) == 0) features &= ~VTNET_LRO_FEATURES; if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) == 0) features &= ~VIRTIO_NET_F_CTRL_VLAN; error = virtio_reinit(dev, features); if (error) { device_printf(dev, "virtio reinit error %d\n", error); return (error); } sc->vtnet_features = features; virtio_reinit_complete(dev); return (0); } static void vtnet_init_rx_filters(struct vtnet_softc *sc) { if_t ifp; ifp = sc->vtnet_ifp; if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { vtnet_rx_filter(sc); vtnet_rx_filter_mac(sc); } if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) vtnet_rx_filter_vlan(sc); } static int vtnet_init_rx_queues(struct vtnet_softc *sc) { device_t dev; if_t ifp; struct vtnet_rxq *rxq; int i, clustersz, error; dev = sc->vtnet_dev; ifp = sc->vtnet_ifp; clustersz = vtnet_rx_cluster_size(sc, if_getmtu(ifp)); sc->vtnet_rx_clustersz = clustersz; if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) { sc->vtnet_rx_nmbufs = howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE, clustersz); KASSERT(sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs, ("%s: too many rx mbufs %d for %d segments", __func__, sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs)); } else sc->vtnet_rx_nmbufs = 1; for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { rxq = &sc->vtnet_rxqs[i]; /* Hold the lock to satisfy asserts. */ VTNET_RXQ_LOCK(rxq); error = vtnet_rxq_populate(rxq); VTNET_RXQ_UNLOCK(rxq); if (error) { device_printf(dev, "cannot populate Rx queue %d\n", i); return (error); } } return (0); } static int vtnet_init_tx_queues(struct vtnet_softc *sc) { struct vtnet_txq *txq; int i; for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { txq = &sc->vtnet_txqs[i]; txq->vtntx_watchdog = 0; txq->vtntx_intr_threshold = vtnet_txq_intr_threshold(txq); #ifdef DEV_NETMAP netmap_reset(NA(sc->vtnet_ifp), NR_TX, i, 0); #endif /* DEV_NETMAP */ } return (0); } static int vtnet_init_rxtx_queues(struct vtnet_softc *sc) { int error; error = vtnet_init_rx_queues(sc); if (error) return (error); error = vtnet_init_tx_queues(sc); if (error) return (error); return (0); } static void vtnet_set_active_vq_pairs(struct vtnet_softc *sc) { device_t dev; int npairs; dev = sc->vtnet_dev; if ((sc->vtnet_flags & VTNET_FLAG_MQ) == 0) { sc->vtnet_act_vq_pairs = 1; return; } npairs = sc->vtnet_req_vq_pairs; if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) { device_printf(dev, "cannot set active queue pairs to %d, " "falling back to 1 queue pair\n", npairs); npairs = 1; } sc->vtnet_act_vq_pairs = npairs; } static void vtnet_update_rx_offloads(struct vtnet_softc *sc) { if_t ifp; uint64_t features; int error; ifp = sc->vtnet_ifp; features = sc->vtnet_features; VTNET_CORE_LOCK_ASSERT(sc); if (if_getcapabilities(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { if (if_getcapenable(ifp) & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) features |= VIRTIO_NET_F_GUEST_CSUM; else features &= ~VIRTIO_NET_F_GUEST_CSUM; } if (if_getcapabilities(ifp) & IFCAP_LRO && !vtnet_software_lro(sc)) { if (if_getcapenable(ifp) & IFCAP_LRO) features |= VTNET_LRO_FEATURES; else features &= ~VTNET_LRO_FEATURES; } error = vtnet_ctrl_guest_offloads(sc, features & (VIRTIO_NET_F_GUEST_CSUM | VIRTIO_NET_F_GUEST_TSO4 | VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN | VIRTIO_NET_F_GUEST_UFO)); if (error) { device_printf(sc->vtnet_dev, "%s: cannot update Rx features\n", __func__); if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); vtnet_init_locked(sc, 0); } } else sc->vtnet_features = features; } static int vtnet_reinit(struct vtnet_softc *sc) { if_t ifp; int error; ifp = sc->vtnet_ifp; bcopy(if_getlladdr(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN); error = vtnet_virtio_reinit(sc); if (error) return (error); vtnet_set_macaddr(sc); vtnet_set_active_vq_pairs(sc); if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) vtnet_init_rx_filters(sc); if_sethwassist(ifp, 0); if (if_getcapenable(ifp) & IFCAP_TXCSUM) if_sethwassistbits(ifp, VTNET_CSUM_OFFLOAD, 0); if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) if_sethwassistbits(ifp, VTNET_CSUM_OFFLOAD_IPV6, 0); if (if_getcapenable(ifp) & IFCAP_TSO4) if_sethwassistbits(ifp, CSUM_IP_TSO, 0); if (if_getcapenable(ifp) & IFCAP_TSO6) if_sethwassistbits(ifp, CSUM_IP6_TSO, 0); error = vtnet_init_rxtx_queues(sc); if (error) return (error); return (0); } static void vtnet_init_locked(struct vtnet_softc *sc, int init_mode) { if_t ifp; ifp = sc->vtnet_ifp; VTNET_CORE_LOCK_ASSERT(sc); if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) return; vtnet_stop(sc); #ifdef DEV_NETMAP /* Once stopped we can update the netmap flags, if necessary. */ switch (init_mode) { case VTNET_INIT_NETMAP_ENTER: nm_set_native_flags(NA(ifp)); break; case VTNET_INIT_NETMAP_EXIT: nm_clear_native_flags(NA(ifp)); break; } #endif /* DEV_NETMAP */ if (vtnet_reinit(sc) != 0) { vtnet_stop(sc); return; } if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); vtnet_update_link_status(sc); vtnet_enable_interrupts(sc); callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc); #ifdef DEV_NETMAP /* Re-enable txsync/rxsync. */ netmap_enable_all_rings(ifp); #endif /* DEV_NETMAP */ } static void vtnet_init(void *xsc) { struct vtnet_softc *sc; sc = xsc; VTNET_CORE_LOCK(sc); vtnet_init_locked(sc, 0); VTNET_CORE_UNLOCK(sc); } static void vtnet_free_ctrl_vq(struct vtnet_softc *sc) { /* * The control virtqueue is only polled and therefore it should * already be empty. */ KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq), ("%s: ctrl vq %p not empty", __func__, sc->vtnet_ctrl_vq)); } static void vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie, struct sglist *sg, int readable, int writable) { struct virtqueue *vq; vq = sc->vtnet_ctrl_vq; MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ); VTNET_CORE_LOCK_ASSERT(sc); if (!virtqueue_empty(vq)) return; /* * Poll for the response, but the command is likely completed before * returning from the notify. */ if (virtqueue_enqueue(vq, cookie, sg, readable, writable) == 0) { virtqueue_notify(vq); virtqueue_poll(vq, NULL); } } static int vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr) { struct sglist_seg segs[3]; struct sglist sg; struct { struct virtio_net_ctrl_hdr hdr __aligned(2); uint8_t pad1; uint8_t addr[ETHER_ADDR_LEN] __aligned(8); uint8_t pad2; uint8_t ack; } s; int error; error = 0; MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_MAC); s.hdr.class = VIRTIO_NET_CTRL_MAC; s.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET; bcopy(hwaddr, &s.addr[0], ETHER_ADDR_LEN); s.ack = VIRTIO_NET_ERR; sglist_init(&sg, nitems(segs), segs); error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); error |= sglist_append(&sg, &s.addr[0], ETHER_ADDR_LEN); error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); MPASS(error == 0 && sg.sg_nseg == nitems(segs)); if (error == 0) vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); return (s.ack == VIRTIO_NET_OK ? 0 : EIO); } static int vtnet_ctrl_guest_offloads(struct vtnet_softc *sc, uint64_t offloads) { struct sglist_seg segs[3]; struct sglist sg; struct { struct virtio_net_ctrl_hdr hdr __aligned(2); uint8_t pad1; uint64_t offloads __aligned(8); uint8_t pad2; uint8_t ack; } s; int error; error = 0; MPASS(sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); s.hdr.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; s.hdr.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; s.offloads = vtnet_gtoh64(sc, offloads); s.ack = VIRTIO_NET_ERR; sglist_init(&sg, nitems(segs), segs); error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); error |= sglist_append(&sg, &s.offloads, sizeof(uint64_t)); error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); MPASS(error == 0 && sg.sg_nseg == nitems(segs)); if (error == 0) vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); return (s.ack == VIRTIO_NET_OK ? 0 : EIO); } static int vtnet_ctrl_mq_cmd(struct vtnet_softc *sc, uint16_t npairs) { struct sglist_seg segs[3]; struct sglist sg; struct { struct virtio_net_ctrl_hdr hdr __aligned(2); uint8_t pad1; struct virtio_net_ctrl_mq mq __aligned(2); uint8_t pad2; uint8_t ack; } s; int error; error = 0; MPASS(sc->vtnet_flags & VTNET_FLAG_MQ); s.hdr.class = VIRTIO_NET_CTRL_MQ; s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; s.mq.virtqueue_pairs = vtnet_gtoh16(sc, npairs); s.ack = VIRTIO_NET_ERR; sglist_init(&sg, nitems(segs), segs); error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); error |= sglist_append(&sg, &s.mq, sizeof(struct virtio_net_ctrl_mq)); error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); MPASS(error == 0 && sg.sg_nseg == nitems(segs)); if (error == 0) vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); return (s.ack == VIRTIO_NET_OK ? 0 : EIO); } static int vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, uint8_t cmd, bool on) { struct sglist_seg segs[3]; struct sglist sg; struct { struct virtio_net_ctrl_hdr hdr __aligned(2); uint8_t pad1; uint8_t onoff; uint8_t pad2; uint8_t ack; } s; int error; error = 0; MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX); s.hdr.class = VIRTIO_NET_CTRL_RX; s.hdr.cmd = cmd; s.onoff = on; s.ack = VIRTIO_NET_ERR; sglist_init(&sg, nitems(segs), segs); error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t)); error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); MPASS(error == 0 && sg.sg_nseg == nitems(segs)); if (error == 0) vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); return (s.ack == VIRTIO_NET_OK ? 0 : EIO); } static int vtnet_set_promisc(struct vtnet_softc *sc, bool on) { return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on)); } static int vtnet_set_allmulti(struct vtnet_softc *sc, bool on) { return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on)); } static void vtnet_rx_filter(struct vtnet_softc *sc) { device_t dev; if_t ifp; dev = sc->vtnet_dev; ifp = sc->vtnet_ifp; VTNET_CORE_LOCK_ASSERT(sc); if (vtnet_set_promisc(sc, if_getflags(ifp) & IFF_PROMISC) != 0) { device_printf(dev, "cannot %s promiscuous mode\n", if_getflags(ifp) & IFF_PROMISC ? "enable" : "disable"); } if (vtnet_set_allmulti(sc, if_getflags(ifp) & IFF_ALLMULTI) != 0) { device_printf(dev, "cannot %s all-multicast mode\n", if_getflags(ifp) & IFF_ALLMULTI ? "enable" : "disable"); } } static u_int vtnet_copy_ifaddr(void *arg, struct sockaddr_dl *sdl, u_int ucnt) { struct vtnet_softc *sc = arg; if (memcmp(LLADDR(sdl), sc->vtnet_hwaddr, ETHER_ADDR_LEN) == 0) return (0); if (ucnt < VTNET_MAX_MAC_ENTRIES) bcopy(LLADDR(sdl), &sc->vtnet_mac_filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN); return (1); } static u_int vtnet_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int mcnt) { struct vtnet_mac_filter *filter = arg; if (mcnt < VTNET_MAX_MAC_ENTRIES) bcopy(LLADDR(sdl), &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN); return (1); } static void vtnet_rx_filter_mac(struct vtnet_softc *sc) { struct virtio_net_ctrl_hdr hdr __aligned(2); struct vtnet_mac_filter *filter; struct sglist_seg segs[4]; struct sglist sg; if_t ifp; bool promisc, allmulti; u_int ucnt, mcnt; int error; uint8_t ack; ifp = sc->vtnet_ifp; filter = sc->vtnet_mac_filter; error = 0; MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX); VTNET_CORE_LOCK_ASSERT(sc); /* Unicast MAC addresses: */ ucnt = if_foreach_lladdr(ifp, vtnet_copy_ifaddr, sc); promisc = (ucnt > VTNET_MAX_MAC_ENTRIES); if (promisc) { ucnt = 0; if_printf(ifp, "more than %d MAC addresses assigned, " "falling back to promiscuous mode\n", VTNET_MAX_MAC_ENTRIES); } /* Multicast MAC addresses: */ mcnt = if_foreach_llmaddr(ifp, vtnet_copy_maddr, filter); allmulti = (mcnt > VTNET_MAX_MAC_ENTRIES); if (allmulti) { mcnt = 0; if_printf(ifp, "more than %d multicast MAC addresses " "assigned, falling back to all-multicast mode\n", VTNET_MAX_MAC_ENTRIES); } if (promisc && allmulti) goto out; filter->vmf_unicast.nentries = vtnet_gtoh32(sc, ucnt); filter->vmf_multicast.nentries = vtnet_gtoh32(sc, mcnt); hdr.class = VIRTIO_NET_CTRL_MAC; hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; ack = VIRTIO_NET_ERR; sglist_init(&sg, nitems(segs), segs); error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); error |= sglist_append(&sg, &filter->vmf_unicast, sizeof(uint32_t) + ucnt * ETHER_ADDR_LEN); error |= sglist_append(&sg, &filter->vmf_multicast, sizeof(uint32_t) + mcnt * ETHER_ADDR_LEN); error |= sglist_append(&sg, &ack, sizeof(uint8_t)); MPASS(error == 0 && sg.sg_nseg == nitems(segs)); if (error == 0) vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); if (ack != VIRTIO_NET_OK) if_printf(ifp, "error setting host MAC filter table\n"); out: if (promisc != 0 && vtnet_set_promisc(sc, true) != 0) if_printf(ifp, "cannot enable promiscuous mode\n"); if (allmulti != 0 && vtnet_set_allmulti(sc, true) != 0) if_printf(ifp, "cannot enable all-multicast mode\n"); } static int vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) { struct sglist_seg segs[3]; struct sglist sg; struct { struct virtio_net_ctrl_hdr hdr __aligned(2); uint8_t pad1; uint16_t tag __aligned(2); uint8_t pad2; uint8_t ack; } s; int error; error = 0; MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER); s.hdr.class = VIRTIO_NET_CTRL_VLAN; s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; s.tag = vtnet_gtoh16(sc, tag); s.ack = VIRTIO_NET_ERR; sglist_init(&sg, nitems(segs), segs); error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr)); error |= sglist_append(&sg, &s.tag, sizeof(uint16_t)); error |= sglist_append(&sg, &s.ack, sizeof(uint8_t)); MPASS(error == 0 && sg.sg_nseg == nitems(segs)); if (error == 0) vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1); return (s.ack == VIRTIO_NET_OK ? 0 : EIO); } static void vtnet_rx_filter_vlan(struct vtnet_softc *sc) { int i, bit; uint32_t w; uint16_t tag; MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER); VTNET_CORE_LOCK_ASSERT(sc); /* Enable the filter for each configured VLAN. */ for (i = 0; i < VTNET_VLAN_FILTER_NWORDS; i++) { w = sc->vtnet_vlan_filter[i]; while ((bit = ffs(w) - 1) != -1) { w &= ~(1 << bit); tag = sizeof(w) * CHAR_BIT * i + bit; if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) { device_printf(sc->vtnet_dev, "cannot enable VLAN %d filter\n", tag); } } } } static void vtnet_update_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) { if_t ifp; int idx, bit; ifp = sc->vtnet_ifp; idx = (tag >> 5) & 0x7F; bit = tag & 0x1F; if (tag == 0 || tag > 4095) return; VTNET_CORE_LOCK(sc); if (add) sc->vtnet_vlan_filter[idx] |= (1 << bit); else sc->vtnet_vlan_filter[idx] &= ~(1 << bit); if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER && if_getdrvflags(ifp) & IFF_DRV_RUNNING && vtnet_exec_vlan_filter(sc, add, tag) != 0) { device_printf(sc->vtnet_dev, "cannot %s VLAN %d %s the host filter table\n", add ? "add" : "remove", tag, add ? "to" : "from"); } VTNET_CORE_UNLOCK(sc); } static void vtnet_register_vlan(void *arg, if_t ifp, uint16_t tag) { if (if_getsoftc(ifp) != arg) return; vtnet_update_vlan_filter(arg, 1, tag); } static void vtnet_unregister_vlan(void *arg, if_t ifp, uint16_t tag) { if (if_getsoftc(ifp) != arg) return; vtnet_update_vlan_filter(arg, 0, tag); } static void vtnet_update_speed_duplex(struct vtnet_softc *sc) { if_t ifp; uint32_t speed; ifp = sc->vtnet_ifp; if ((sc->vtnet_features & VIRTIO_NET_F_SPEED_DUPLEX) == 0) return; /* BMV: Ignore duplex. */ speed = virtio_read_dev_config_4(sc->vtnet_dev, offsetof(struct virtio_net_config, speed)); if (speed != UINT32_MAX) if_setbaudrate(ifp, IF_Mbps(speed)); } static int vtnet_is_link_up(struct vtnet_softc *sc) { uint16_t status; if ((sc->vtnet_features & VIRTIO_NET_F_STATUS) == 0) return (1); status = virtio_read_dev_config_2(sc->vtnet_dev, offsetof(struct virtio_net_config, status)); return ((status & VIRTIO_NET_S_LINK_UP) != 0); } static void vtnet_update_link_status(struct vtnet_softc *sc) { if_t ifp; int link; ifp = sc->vtnet_ifp; VTNET_CORE_LOCK_ASSERT(sc); link = vtnet_is_link_up(sc); /* Notify if the link status has changed. */ if (link != 0 && sc->vtnet_link_active == 0) { vtnet_update_speed_duplex(sc); sc->vtnet_link_active = 1; if_link_state_change(ifp, LINK_STATE_UP); } else if (link == 0 && sc->vtnet_link_active != 0) { sc->vtnet_link_active = 0; if_link_state_change(ifp, LINK_STATE_DOWN); } } static int vtnet_ifmedia_upd(if_t ifp __unused) { return (EOPNOTSUPP); } static void vtnet_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr) { struct vtnet_softc *sc; sc = if_getsoftc(ifp); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; VTNET_CORE_LOCK(sc); if (vtnet_is_link_up(sc) != 0) { ifmr->ifm_status |= IFM_ACTIVE; ifmr->ifm_active |= IFM_10G_T | IFM_FDX; } else ifmr->ifm_active |= IFM_NONE; VTNET_CORE_UNLOCK(sc); } static void vtnet_get_macaddr(struct vtnet_softc *sc) { if (sc->vtnet_flags & VTNET_FLAG_MAC) { virtio_read_device_config_array(sc->vtnet_dev, offsetof(struct virtio_net_config, mac), &sc->vtnet_hwaddr[0], sizeof(uint8_t), ETHER_ADDR_LEN); } else { /* Generate a random locally administered unicast address. */ sc->vtnet_hwaddr[0] = 0xB2; arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0); } } static void vtnet_set_macaddr(struct vtnet_softc *sc) { device_t dev; int error; dev = sc->vtnet_dev; if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) { error = vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr); if (error) device_printf(dev, "unable to set MAC address\n"); return; } /* MAC in config is read-only in modern VirtIO. */ if (!vtnet_modern(sc) && sc->vtnet_flags & VTNET_FLAG_MAC) { for (int i = 0; i < ETHER_ADDR_LEN; i++) { virtio_write_dev_config_1(dev, offsetof(struct virtio_net_config, mac) + i, sc->vtnet_hwaddr[i]); } } } static void vtnet_attached_set_macaddr(struct vtnet_softc *sc) { /* Assign MAC address if it was generated. */ if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) vtnet_set_macaddr(sc); } static void vtnet_vlan_tag_remove(struct mbuf *m) { struct ether_vlan_header *evh; evh = mtod(m, struct ether_vlan_header *); m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag); m->m_flags |= M_VLANTAG; /* Strip the 802.1Q header. */ bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN, ETHER_HDR_LEN - ETHER_TYPE_LEN); m_adj(m, ETHER_VLAN_ENCAP_LEN); } static void vtnet_set_rx_process_limit(struct vtnet_softc *sc) { int limit; limit = vtnet_tunable_int(sc, "rx_process_limit", vtnet_rx_process_limit); if (limit < 0) limit = INT_MAX; sc->vtnet_rx_process_limit = limit; } static void vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child, struct vtnet_rxq *rxq) { struct sysctl_oid *node; struct sysctl_oid_list *list; struct vtnet_rxq_stats *stats; char namebuf[16]; snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vtnrx_id); node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Receive Queue"); list = SYSCTL_CHILDREN(node); stats = &rxq->vtnrx_stats; SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ipackets", CTLFLAG_RD, &stats->vrxs_ipackets, "Receive packets"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ibytes", CTLFLAG_RD, &stats->vrxs_ibytes, "Receive bytes"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "iqdrops", CTLFLAG_RD, &stats->vrxs_iqdrops, "Receive drops"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "ierrors", CTLFLAG_RD, &stats->vrxs_ierrors, "Receive errors"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, &stats->vrxs_csum, "Receive checksum offloaded"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", CTLFLAG_RD, &stats->vrxs_csum_failed, "Receive checksum offload failed"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "host_lro", CTLFLAG_RD, &stats->vrxs_host_lro, "Receive host segmentation offloaded"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, &stats->vrxs_rescheduled, "Receive interrupt handler rescheduled"); } static void vtnet_setup_txq_sysctl(struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child, struct vtnet_txq *txq) { struct sysctl_oid *node; struct sysctl_oid_list *list; struct vtnet_txq_stats *stats; char namebuf[16]; snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vtntx_id); node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Transmit Queue"); list = SYSCTL_CHILDREN(node); stats = &txq->vtntx_stats; SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "opackets", CTLFLAG_RD, &stats->vtxs_opackets, "Transmit packets"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "obytes", CTLFLAG_RD, &stats->vtxs_obytes, "Transmit bytes"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "omcasts", CTLFLAG_RD, &stats->vtxs_omcasts, "Transmit multicasts"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD, &stats->vtxs_csum, "Transmit checksum offloaded"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD, &stats->vtxs_tso, "Transmit TCP segmentation offloaded"); SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD, &stats->vtxs_rescheduled, "Transmit interrupt handler rescheduled"); } static void vtnet_setup_queue_sysctl(struct vtnet_softc *sc) { device_t dev; struct sysctl_ctx_list *ctx; struct sysctl_oid *tree; struct sysctl_oid_list *child; int i; dev = sc->vtnet_dev; ctx = device_get_sysctl_ctx(dev); tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); for (i = 0; i < sc->vtnet_req_vq_pairs; i++) { vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]); vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]); } } static void vtnet_setup_stat_sysctl(struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child, struct vtnet_softc *sc) { struct vtnet_statistics *stats; struct vtnet_rxq_stats rxaccum; struct vtnet_txq_stats txaccum; vtnet_accum_stats(sc, &rxaccum, &txaccum); stats = &sc->vtnet_stats; stats->rx_csum_offloaded = rxaccum.vrxs_csum; stats->rx_csum_failed = rxaccum.vrxs_csum_failed; stats->rx_task_rescheduled = rxaccum.vrxs_rescheduled; stats->tx_csum_offloaded = txaccum.vtxs_csum; stats->tx_tso_offloaded = txaccum.vtxs_tso; stats->tx_task_rescheduled = txaccum.vtxs_rescheduled; SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "mbuf_alloc_failed", CTLFLAG_RD, &stats->mbuf_alloc_failed, "Mbuf cluster allocation failures"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_frame_too_large", CTLFLAG_RD, &stats->rx_frame_too_large, "Received frame larger than the mbuf chain"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_enq_replacement_failed", CTLFLAG_RD, &stats->rx_enq_replacement_failed, "Enqueuing the replacement receive mbuf failed"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_mergeable_failed", CTLFLAG_RD, &stats->rx_mergeable_failed, "Mergeable buffers receive failures"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ethtype", CTLFLAG_RD, &stats->rx_csum_bad_ethtype, "Received checksum offloaded buffer with unsupported " "Ethernet type"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_ipproto", CTLFLAG_RD, &stats->rx_csum_bad_ipproto, "Received checksum offloaded buffer with incorrect IP protocol"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_offset", CTLFLAG_RD, &stats->rx_csum_bad_offset, "Received checksum offloaded buffer with incorrect offset"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_bad_proto", CTLFLAG_RD, &stats->rx_csum_bad_proto, "Received checksum offloaded buffer with incorrect protocol"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_failed", CTLFLAG_RD, &stats->rx_csum_failed, "Received buffer checksum offload failed"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_csum_offloaded", CTLFLAG_RD, &stats->rx_csum_offloaded, "Received buffer checksum offload succeeded"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "rx_task_rescheduled", CTLFLAG_RD, &stats->rx_task_rescheduled, "Times the receive interrupt task rescheduled itself"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_unknown_ethtype", CTLFLAG_RD, &stats->tx_csum_unknown_ethtype, "Aborted transmit of checksum offloaded buffer with unknown " "Ethernet type"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_proto_mismatch", CTLFLAG_RD, &stats->tx_csum_proto_mismatch, "Aborted transmit of checksum offloaded buffer because mismatched " "protocols"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp", CTLFLAG_RD, &stats->tx_tso_not_tcp, "Aborted transmit of TSO buffer with non TCP protocol"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_without_csum", CTLFLAG_RD, &stats->tx_tso_without_csum, "Aborted transmit of TSO buffer without TCP checksum offload"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged", CTLFLAG_RD, &stats->tx_defragged, "Transmit mbufs defragged"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defrag_failed", CTLFLAG_RD, &stats->tx_defrag_failed, "Aborted transmit of buffer because defrag failed"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_offloaded", CTLFLAG_RD, &stats->tx_csum_offloaded, "Offloaded checksum of transmitted buffer"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_offloaded", CTLFLAG_RD, &stats->tx_tso_offloaded, "Segmentation offload of transmitted buffer"); SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_task_rescheduled", CTLFLAG_RD, &stats->tx_task_rescheduled, "Times the transmit interrupt task rescheduled itself"); } static void vtnet_setup_sysctl(struct vtnet_softc *sc) { device_t dev; struct sysctl_ctx_list *ctx; struct sysctl_oid *tree; struct sysctl_oid_list *child; dev = sc->vtnet_dev; ctx = device_get_sysctl_ctx(dev); tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs", CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0, "Number of maximum supported virtqueue pairs"); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "req_vq_pairs", CTLFLAG_RD, &sc->vtnet_req_vq_pairs, 0, "Number of requested virtqueue pairs"); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs", CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0, "Number of active virtqueue pairs"); vtnet_setup_stat_sysctl(ctx, child, sc); } static void vtnet_load_tunables(struct vtnet_softc *sc) { sc->vtnet_lro_entry_count = vtnet_tunable_int(sc, "lro_entry_count", vtnet_lro_entry_count); if (sc->vtnet_lro_entry_count < TCP_LRO_ENTRIES) sc->vtnet_lro_entry_count = TCP_LRO_ENTRIES; sc->vtnet_lro_mbufq_depth = vtnet_tunable_int(sc, "lro_mbufq_depth", vtnet_lro_mbufq_depth); } static int vtnet_rxq_enable_intr(struct vtnet_rxq *rxq) { return (virtqueue_enable_intr(rxq->vtnrx_vq)); } static void vtnet_rxq_disable_intr(struct vtnet_rxq *rxq) { virtqueue_disable_intr(rxq->vtnrx_vq); } static int vtnet_txq_enable_intr(struct vtnet_txq *txq) { struct virtqueue *vq; vq = txq->vtntx_vq; if (vtnet_txq_below_threshold(txq) != 0) return (virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG)); /* * The free count is above our threshold. Keep the Tx interrupt * disabled until the queue is fuller. */ return (0); } static void vtnet_txq_disable_intr(struct vtnet_txq *txq) { virtqueue_disable_intr(txq->vtntx_vq); } static void vtnet_enable_rx_interrupts(struct vtnet_softc *sc) { struct vtnet_rxq *rxq; int i; for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { rxq = &sc->vtnet_rxqs[i]; if (vtnet_rxq_enable_intr(rxq) != 0) taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask); } } static void vtnet_enable_tx_interrupts(struct vtnet_softc *sc) { int i; for (i = 0; i < sc->vtnet_act_vq_pairs; i++) vtnet_txq_enable_intr(&sc->vtnet_txqs[i]); } static void vtnet_enable_interrupts(struct vtnet_softc *sc) { vtnet_enable_rx_interrupts(sc); vtnet_enable_tx_interrupts(sc); } static void vtnet_disable_rx_interrupts(struct vtnet_softc *sc) { int i; for (i = 0; i < sc->vtnet_max_vq_pairs; i++) vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]); } static void vtnet_disable_tx_interrupts(struct vtnet_softc *sc) { int i; for (i = 0; i < sc->vtnet_max_vq_pairs; i++) vtnet_txq_disable_intr(&sc->vtnet_txqs[i]); } static void vtnet_disable_interrupts(struct vtnet_softc *sc) { vtnet_disable_rx_interrupts(sc); vtnet_disable_tx_interrupts(sc); } static int vtnet_tunable_int(struct vtnet_softc *sc, const char *knob, int def) { char path[64]; snprintf(path, sizeof(path), "hw.vtnet.%d.%s", device_get_unit(sc->vtnet_dev), knob); TUNABLE_INT_FETCH(path, &def); return (def); } #ifdef DEBUGNET static void vtnet_debugnet_init(if_t ifp, int *nrxr, int *ncl, int *clsize) { struct vtnet_softc *sc; sc = if_getsoftc(ifp); VTNET_CORE_LOCK(sc); *nrxr = sc->vtnet_req_vq_pairs; *ncl = DEBUGNET_MAX_IN_FLIGHT; *clsize = sc->vtnet_rx_clustersz; VTNET_CORE_UNLOCK(sc); } static void vtnet_debugnet_event(if_t ifp __unused, enum debugnet_ev event) { struct vtnet_softc *sc; static bool sw_lro_enabled = false; /* * Disable software LRO, since it would require entering the network * epoch when calling vtnet_txq_eof() in vtnet_debugnet_poll(). */ sc = if_getsoftc(ifp); switch (event) { case DEBUGNET_START: sw_lro_enabled = (sc->vtnet_flags & VTNET_FLAG_SW_LRO) != 0; if (sw_lro_enabled) sc->vtnet_flags &= ~VTNET_FLAG_SW_LRO; break; case DEBUGNET_END: if (sw_lro_enabled) sc->vtnet_flags |= VTNET_FLAG_SW_LRO; break; } } static int vtnet_debugnet_transmit(if_t ifp, struct mbuf *m) { struct vtnet_softc *sc; struct vtnet_txq *txq; int error; sc = if_getsoftc(ifp); if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return (EBUSY); txq = &sc->vtnet_txqs[0]; error = vtnet_txq_encap(txq, &m, M_NOWAIT | M_USE_RESERVE); if (error == 0) (void)vtnet_txq_notify(txq); return (error); } static int vtnet_debugnet_poll(if_t ifp, int count) { struct vtnet_softc *sc; int i; sc = if_getsoftc(ifp); if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return (EBUSY); (void)vtnet_txq_eof(&sc->vtnet_txqs[0]); for (i = 0; i < sc->vtnet_act_vq_pairs; i++) (void)vtnet_rxq_eof(&sc->vtnet_rxqs[i]); return (0); } #endif /* DEBUGNET */ diff --git a/sys/dev/virtio/pci/virtio_pci.c b/sys/dev/virtio/pci/virtio_pci.c index 1470dc83949c..b9aa36ef6d0f 100644 --- a/sys/dev/virtio/pci/virtio_pci.c +++ b/sys/dev/virtio/pci/virtio_pci.c @@ -1,999 +1,999 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_pci_if.h" #include "virtio_if.h" static void vtpci_describe_features(struct vtpci_common *, const char *, uint64_t); static int vtpci_alloc_msix(struct vtpci_common *, int); static int vtpci_alloc_msi(struct vtpci_common *); static int vtpci_alloc_intr_msix_pervq(struct vtpci_common *); static int vtpci_alloc_intr_msix_shared(struct vtpci_common *); static int vtpci_alloc_intr_msi(struct vtpci_common *); static int vtpci_alloc_intr_intx(struct vtpci_common *); static int vtpci_alloc_interrupt(struct vtpci_common *, int, int, struct vtpci_interrupt *); static void vtpci_free_interrupt(struct vtpci_common *, struct vtpci_interrupt *); static void vtpci_free_interrupts(struct vtpci_common *); static void vtpci_free_virtqueues(struct vtpci_common *); static void vtpci_cleanup_setup_intr_attempt(struct vtpci_common *); static int vtpci_alloc_intr_resources(struct vtpci_common *); static int vtpci_setup_intx_interrupt(struct vtpci_common *, enum intr_type); static int vtpci_setup_pervq_msix_interrupts(struct vtpci_common *, enum intr_type); static int vtpci_set_host_msix_vectors(struct vtpci_common *); static int vtpci_setup_msix_interrupts(struct vtpci_common *, enum intr_type); static int vtpci_setup_intrs(struct vtpci_common *, enum intr_type); static int vtpci_reinit_virtqueue(struct vtpci_common *, int); static void vtpci_intx_intr(void *); static int vtpci_vq_shared_intr_filter(void *); static void vtpci_vq_shared_intr(void *); static int vtpci_vq_intr_filter(void *); static void vtpci_vq_intr(void *); static void vtpci_config_intr(void *); static void vtpci_setup_sysctl(struct vtpci_common *); #define vtpci_setup_msi_interrupt vtpci_setup_intx_interrupt /* * This module contains two drivers: * - virtio_pci_legacy for pre-V1 support * - virtio_pci_modern for V1 support */ MODULE_VERSION(virtio_pci, 1); MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); int vtpci_disable_msix = 0; TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); static uint8_t vtpci_read_isr(struct vtpci_common *cn) { return (VIRTIO_PCI_READ_ISR(cn->vtpci_dev)); } static uint16_t vtpci_get_vq_size(struct vtpci_common *cn, int idx) { return (VIRTIO_PCI_GET_VQ_SIZE(cn->vtpci_dev, idx)); } static bus_size_t vtpci_get_vq_notify_off(struct vtpci_common *cn, int idx) { return (VIRTIO_PCI_GET_VQ_NOTIFY_OFF(cn->vtpci_dev, idx)); } static void vtpci_set_vq(struct vtpci_common *cn, struct virtqueue *vq) { VIRTIO_PCI_SET_VQ(cn->vtpci_dev, vq); } static void vtpci_disable_vq(struct vtpci_common *cn, int idx) { VIRTIO_PCI_DISABLE_VQ(cn->vtpci_dev, idx); } static int vtpci_register_cfg_msix(struct vtpci_common *cn, struct vtpci_interrupt *intr) { return (VIRTIO_PCI_REGISTER_CFG_MSIX(cn->vtpci_dev, intr)); } static int vtpci_register_vq_msix(struct vtpci_common *cn, int idx, struct vtpci_interrupt *intr) { return (VIRTIO_PCI_REGISTER_VQ_MSIX(cn->vtpci_dev, idx, intr)); } void vtpci_init(struct vtpci_common *cn, device_t dev, bool modern) { cn->vtpci_dev = dev; pci_enable_busmaster(dev); if (modern) cn->vtpci_flags |= VTPCI_FLAG_MODERN; if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) cn->vtpci_flags |= VTPCI_FLAG_NO_MSI; if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) cn->vtpci_flags |= VTPCI_FLAG_NO_MSIX; vtpci_setup_sysctl(cn); } int vtpci_add_child(struct vtpci_common *cn) { device_t dev, child; dev = cn->vtpci_dev; child = device_add_child(dev, NULL, -1); if (child == NULL) { device_printf(dev, "cannot create child device\n"); return (ENOMEM); } cn->vtpci_child_dev = child; return (0); } int vtpci_delete_child(struct vtpci_common *cn) { device_t dev, child; int error; dev = cn->vtpci_dev; child = cn->vtpci_child_dev; if (child != NULL) { error = device_delete_child(dev, child); if (error) return (error); cn->vtpci_child_dev = NULL; } return (0); } void vtpci_child_detached(struct vtpci_common *cn) { vtpci_release_child_resources(cn); cn->vtpci_child_feat_desc = NULL; cn->vtpci_host_features = 0; cn->vtpci_features = 0; } int vtpci_reinit(struct vtpci_common *cn) { int idx, error; for (idx = 0; idx < cn->vtpci_nvqs; idx++) { error = vtpci_reinit_virtqueue(cn, idx); if (error) return (error); } if (vtpci_is_msix_enabled(cn)) { error = vtpci_set_host_msix_vectors(cn); if (error) return (error); } return (0); } static void vtpci_describe_features(struct vtpci_common *cn, const char *msg, uint64_t features) { device_t dev, child; dev = cn->vtpci_dev; child = cn->vtpci_child_dev; if (device_is_attached(child) || bootverbose == 0) return; virtio_describe(dev, msg, features, cn->vtpci_child_feat_desc); } uint64_t vtpci_negotiate_features(struct vtpci_common *cn, uint64_t child_features, uint64_t host_features) { uint64_t features; cn->vtpci_host_features = host_features; vtpci_describe_features(cn, "host", host_features); /* * Limit negotiated features to what the driver, virtqueue, and * host all support. */ features = host_features & child_features; features = virtio_filter_transport_features(features); cn->vtpci_features = features; vtpci_describe_features(cn, "negotiated", features); return (features); } bool vtpci_with_feature(struct vtpci_common *cn, uint64_t feature) { return ((cn->vtpci_features & feature) != 0); } int vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result) { device_t dev; int error; dev = cn->vtpci_dev; error = 0; switch (index) { case VIRTIO_IVAR_SUBDEVICE: *result = pci_get_subdevice(dev); break; case VIRTIO_IVAR_VENDOR: *result = pci_get_vendor(dev); break; case VIRTIO_IVAR_DEVICE: *result = pci_get_device(dev); break; case VIRTIO_IVAR_SUBVENDOR: *result = pci_get_subvendor(dev); break; case VIRTIO_IVAR_MODERN: *result = vtpci_is_modern(cn); break; default: error = ENOENT; } return (error); } int vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value) { int error; error = 0; switch (index) { case VIRTIO_IVAR_FEATURE_DESC: cn->vtpci_child_feat_desc = (void *) value; break; default: error = ENOENT; } return (error); } int -vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs, +vtpci_alloc_virtqueues(struct vtpci_common *cn, int nvqs, struct vq_alloc_info *vq_info) { device_t dev; int idx, align, error; dev = cn->vtpci_dev; /* * This is VIRTIO_PCI_VRING_ALIGN from legacy VirtIO. In modern VirtIO, * the tables do not have to be allocated contiguously, but we do so * anyways. */ align = 4096; if (cn->vtpci_nvqs != 0) return (EALREADY); if (nvqs <= 0) return (EINVAL); cn->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), M_DEVBUF, M_NOWAIT | M_ZERO); if (cn->vtpci_vqs == NULL) return (ENOMEM); for (idx = 0; idx < nvqs; idx++) { struct vtpci_virtqueue *vqx; struct vq_alloc_info *info; struct virtqueue *vq; bus_size_t notify_offset; uint16_t size; vqx = &cn->vtpci_vqs[idx]; info = &vq_info[idx]; size = vtpci_get_vq_size(cn, idx); notify_offset = vtpci_get_vq_notify_off(cn, idx); error = virtqueue_alloc(dev, idx, size, notify_offset, align, ~(vm_paddr_t)0, info, &vq); if (error) { device_printf(dev, "cannot allocate virtqueue %d: %d\n", idx, error); break; } vtpci_set_vq(cn, vq); vqx->vtv_vq = *info->vqai_vq = vq; vqx->vtv_no_intr = info->vqai_intr == NULL; cn->vtpci_nvqs++; } if (error) vtpci_free_virtqueues(cn); return (error); } static int vtpci_alloc_msix(struct vtpci_common *cn, int nvectors) { device_t dev; int nmsix, cnt, required; dev = cn->vtpci_dev; /* Allocate an additional vector for the config changes. */ required = nvectors + 1; nmsix = pci_msix_count(dev); if (nmsix < required) return (1); cnt = required; if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { cn->vtpci_nmsix_resources = required; return (0); } pci_release_msi(dev); return (1); } static int vtpci_alloc_msi(struct vtpci_common *cn) { device_t dev; int nmsi, cnt, required; dev = cn->vtpci_dev; required = 1; nmsi = pci_msi_count(dev); if (nmsi < required) return (1); cnt = required; if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) return (0); pci_release_msi(dev); return (1); } static int vtpci_alloc_intr_msix_pervq(struct vtpci_common *cn) { int i, nvectors, error; if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) return (ENOTSUP); for (nvectors = 0, i = 0; i < cn->vtpci_nvqs; i++) { if (cn->vtpci_vqs[i].vtv_no_intr == 0) nvectors++; } error = vtpci_alloc_msix(cn, nvectors); if (error) return (error); cn->vtpci_flags |= VTPCI_FLAG_MSIX; return (0); } static int vtpci_alloc_intr_msix_shared(struct vtpci_common *cn) { int error; if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) return (ENOTSUP); error = vtpci_alloc_msix(cn, 1); if (error) return (error); cn->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; return (0); } static int vtpci_alloc_intr_msi(struct vtpci_common *cn) { int error; /* Only BHyVe supports MSI. */ if (cn->vtpci_flags & VTPCI_FLAG_NO_MSI) return (ENOTSUP); error = vtpci_alloc_msi(cn); if (error) return (error); cn->vtpci_flags |= VTPCI_FLAG_MSI; return (0); } static int vtpci_alloc_intr_intx(struct vtpci_common *cn) { cn->vtpci_flags |= VTPCI_FLAG_INTX; return (0); } static int vtpci_alloc_interrupt(struct vtpci_common *cn, int rid, int flags, struct vtpci_interrupt *intr) { struct resource *irq; irq = bus_alloc_resource_any(cn->vtpci_dev, SYS_RES_IRQ, &rid, flags); if (irq == NULL) return (ENXIO); intr->vti_irq = irq; intr->vti_rid = rid; return (0); } static void vtpci_free_interrupt(struct vtpci_common *cn, struct vtpci_interrupt *intr) { device_t dev; dev = cn->vtpci_dev; if (intr->vti_handler != NULL) { bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); intr->vti_handler = NULL; } if (intr->vti_irq != NULL) { bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, intr->vti_irq); intr->vti_irq = NULL; intr->vti_rid = -1; } } static void vtpci_free_interrupts(struct vtpci_common *cn) { struct vtpci_interrupt *intr; int i, nvq_intrs; vtpci_free_interrupt(cn, &cn->vtpci_device_interrupt); if (cn->vtpci_nmsix_resources != 0) { nvq_intrs = cn->vtpci_nmsix_resources - 1; cn->vtpci_nmsix_resources = 0; if ((intr = cn->vtpci_msix_vq_interrupts) != NULL) { for (i = 0; i < nvq_intrs; i++, intr++) vtpci_free_interrupt(cn, intr); free(cn->vtpci_msix_vq_interrupts, M_DEVBUF); cn->vtpci_msix_vq_interrupts = NULL; } } if (cn->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) pci_release_msi(cn->vtpci_dev); cn->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; } static void vtpci_free_virtqueues(struct vtpci_common *cn) { struct vtpci_virtqueue *vqx; int idx; for (idx = 0; idx < cn->vtpci_nvqs; idx++) { vtpci_disable_vq(cn, idx); vqx = &cn->vtpci_vqs[idx]; virtqueue_free(vqx->vtv_vq); vqx->vtv_vq = NULL; } free(cn->vtpci_vqs, M_DEVBUF); cn->vtpci_vqs = NULL; cn->vtpci_nvqs = 0; } void vtpci_release_child_resources(struct vtpci_common *cn) { vtpci_free_interrupts(cn); vtpci_free_virtqueues(cn); } static void vtpci_cleanup_setup_intr_attempt(struct vtpci_common *cn) { int idx; if (cn->vtpci_flags & VTPCI_FLAG_MSIX) { vtpci_register_cfg_msix(cn, NULL); for (idx = 0; idx < cn->vtpci_nvqs; idx++) vtpci_register_vq_msix(cn, idx, NULL); } vtpci_free_interrupts(cn); } static int vtpci_alloc_intr_resources(struct vtpci_common *cn) { struct vtpci_interrupt *intr; int i, rid, flags, nvq_intrs, error; flags = RF_ACTIVE; if (cn->vtpci_flags & VTPCI_FLAG_INTX) { rid = 0; flags |= RF_SHAREABLE; } else rid = 1; /* * When using INTX or MSI interrupts, this resource handles all * interrupts. When using MSIX, this resource handles just the * configuration changed interrupt. */ intr = &cn->vtpci_device_interrupt; error = vtpci_alloc_interrupt(cn, rid, flags, intr); if (error || cn->vtpci_flags & (VTPCI_FLAG_INTX | VTPCI_FLAG_MSI)) return (error); /* * Now allocate the interrupts for the virtqueues. This may be one * for all the virtqueues, or one for each virtqueue. Subtract one * below for because of the configuration changed interrupt. */ nvq_intrs = cn->vtpci_nmsix_resources - 1; cn->vtpci_msix_vq_interrupts = malloc(nvq_intrs * sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); if (cn->vtpci_msix_vq_interrupts == NULL) return (ENOMEM); intr = cn->vtpci_msix_vq_interrupts; for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { error = vtpci_alloc_interrupt(cn, rid, flags, intr); if (error) return (error); } return (0); } static int vtpci_setup_intx_interrupt(struct vtpci_common *cn, enum intr_type type) { struct vtpci_interrupt *intr; int error; intr = &cn->vtpci_device_interrupt; error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, vtpci_intx_intr, cn, &intr->vti_handler); return (error); } static int vtpci_setup_pervq_msix_interrupts(struct vtpci_common *cn, enum intr_type type) { struct vtpci_virtqueue *vqx; struct vtpci_interrupt *intr; int i, error; intr = cn->vtpci_msix_vq_interrupts; for (i = 0; i < cn->vtpci_nvqs; i++) { vqx = &cn->vtpci_vqs[i]; if (vqx->vtv_no_intr) continue; error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, &intr->vti_handler); if (error) return (error); intr++; } return (0); } static int vtpci_set_host_msix_vectors(struct vtpci_common *cn) { struct vtpci_interrupt *intr, *tintr; int idx, error; intr = &cn->vtpci_device_interrupt; error = vtpci_register_cfg_msix(cn, intr); if (error) return (error); intr = cn->vtpci_msix_vq_interrupts; for (idx = 0; idx < cn->vtpci_nvqs; idx++) { if (cn->vtpci_vqs[idx].vtv_no_intr) tintr = NULL; else tintr = intr; error = vtpci_register_vq_msix(cn, idx, tintr); if (error) break; /* * For shared MSIX, all the virtqueues share the first * interrupt. */ if (!cn->vtpci_vqs[idx].vtv_no_intr && (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) intr++; } return (error); } static int vtpci_setup_msix_interrupts(struct vtpci_common *cn, enum intr_type type) { struct vtpci_interrupt *intr; int error; intr = &cn->vtpci_device_interrupt; error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, vtpci_config_intr, cn, &intr->vti_handler); if (error) return (error); if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { intr = &cn->vtpci_msix_vq_interrupts[0]; error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, cn, &intr->vti_handler); } else error = vtpci_setup_pervq_msix_interrupts(cn, type); return (error ? error : vtpci_set_host_msix_vectors(cn)); } static int vtpci_setup_intrs(struct vtpci_common *cn, enum intr_type type) { int error; type |= INTR_MPSAFE; KASSERT(cn->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, ("%s: no interrupt type selected %#x", __func__, cn->vtpci_flags)); error = vtpci_alloc_intr_resources(cn); if (error) return (error); if (cn->vtpci_flags & VTPCI_FLAG_INTX) error = vtpci_setup_intx_interrupt(cn, type); else if (cn->vtpci_flags & VTPCI_FLAG_MSI) error = vtpci_setup_msi_interrupt(cn, type); else error = vtpci_setup_msix_interrupts(cn, type); return (error); } int vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type) { device_t dev; int attempt, error; dev = cn->vtpci_dev; for (attempt = 0; attempt < 5; attempt++) { /* * Start with the most desirable interrupt configuration and * fallback towards less desirable ones. */ switch (attempt) { case 0: error = vtpci_alloc_intr_msix_pervq(cn); break; case 1: error = vtpci_alloc_intr_msix_shared(cn); break; case 2: error = vtpci_alloc_intr_msi(cn); break; case 3: error = vtpci_alloc_intr_intx(cn); break; default: device_printf(dev, "exhausted all interrupt allocation attempts\n"); return (ENXIO); } if (error == 0 && vtpci_setup_intrs(cn, type) == 0) break; vtpci_cleanup_setup_intr_attempt(cn); } if (bootverbose) { if (cn->vtpci_flags & VTPCI_FLAG_INTX) device_printf(dev, "using legacy interrupt\n"); else if (cn->vtpci_flags & VTPCI_FLAG_MSI) device_printf(dev, "using MSI interrupt\n"); else if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) device_printf(dev, "using shared MSIX interrupts\n"); else device_printf(dev, "using per VQ MSIX interrupts\n"); } return (0); } static int vtpci_reinit_virtqueue(struct vtpci_common *cn, int idx) { struct vtpci_virtqueue *vqx; struct virtqueue *vq; int error; vqx = &cn->vtpci_vqs[idx]; vq = vqx->vtv_vq; KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); error = virtqueue_reinit(vq, vtpci_get_vq_size(cn, idx)); if (error == 0) vtpci_set_vq(cn, vq); return (error); } static void vtpci_intx_intr(void *xcn) { struct vtpci_common *cn; struct vtpci_virtqueue *vqx; int i; uint8_t isr; cn = xcn; isr = vtpci_read_isr(cn); if (isr & VIRTIO_PCI_ISR_CONFIG) vtpci_config_intr(cn); if (isr & VIRTIO_PCI_ISR_INTR) { vqx = &cn->vtpci_vqs[0]; for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) virtqueue_intr(vqx->vtv_vq); } } } static int vtpci_vq_shared_intr_filter(void *xcn) { struct vtpci_common *cn; struct vtpci_virtqueue *vqx; int i, rc; cn = xcn; vqx = &cn->vtpci_vqs[0]; rc = 0; for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) rc |= virtqueue_intr_filter(vqx->vtv_vq); } return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); } static void vtpci_vq_shared_intr(void *xcn) { struct vtpci_common *cn; struct vtpci_virtqueue *vqx; int i; cn = xcn; vqx = &cn->vtpci_vqs[0]; for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { if (vqx->vtv_no_intr == 0) virtqueue_intr(vqx->vtv_vq); } } static int vtpci_vq_intr_filter(void *xvq) { struct virtqueue *vq; int rc; vq = xvq; rc = virtqueue_intr_filter(vq); return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); } static void vtpci_vq_intr(void *xvq) { struct virtqueue *vq; vq = xvq; virtqueue_intr(vq); } static void vtpci_config_intr(void *xcn) { struct vtpci_common *cn; device_t child; cn = xcn; child = cn->vtpci_child_dev; if (child != NULL) VIRTIO_CONFIG_CHANGE(child); } static int vtpci_feature_sysctl(struct sysctl_req *req, struct vtpci_common *cn, uint64_t features) { struct sbuf *sb; int error; sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); if (sb == NULL) return (ENOMEM); error = virtio_describe_sbuf(sb, features, cn->vtpci_child_feat_desc); sbuf_delete(sb); return (error); } static int vtpci_host_features_sysctl(SYSCTL_HANDLER_ARGS) { struct vtpci_common *cn; cn = arg1; return (vtpci_feature_sysctl(req, cn, cn->vtpci_host_features)); } static int vtpci_negotiated_features_sysctl(SYSCTL_HANDLER_ARGS) { struct vtpci_common *cn; cn = arg1; return (vtpci_feature_sysctl(req, cn, cn->vtpci_features)); } static void vtpci_setup_sysctl(struct vtpci_common *cn) { device_t dev; struct sysctl_ctx_list *ctx; struct sysctl_oid *tree; struct sysctl_oid_list *child; dev = cn->vtpci_dev; ctx = device_get_sysctl_ctx(dev); tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nvqs", CTLFLAG_RD, &cn->vtpci_nvqs, 0, "Number of virtqueues"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "host_features", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0, vtpci_host_features_sysctl, "A", "Features supported by the host"); SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "negotiated_features", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0, vtpci_negotiated_features_sysctl, "A", "Features negotiated"); } diff --git a/sys/dev/virtio/pci/virtio_pci.h b/sys/dev/virtio/pci/virtio_pci.h index 401cba25bbf4..2449cfc2826c 100644 --- a/sys/dev/virtio/pci/virtio_pci.h +++ b/sys/dev/virtio/pci/virtio_pci.h @@ -1,130 +1,130 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _VIRTIO_PCI_H #define _VIRTIO_PCI_H struct vtpci_interrupt { struct resource *vti_irq; int vti_rid; void *vti_handler; }; struct vtpci_virtqueue { struct virtqueue *vtv_vq; int vtv_no_intr; int vtv_notify_offset; }; struct vtpci_common { device_t vtpci_dev; uint64_t vtpci_host_features; uint64_t vtpci_features; struct vtpci_virtqueue *vtpci_vqs; int vtpci_nvqs; uint32_t vtpci_flags; #define VTPCI_FLAG_NO_MSI 0x0001 #define VTPCI_FLAG_NO_MSIX 0x0002 #define VTPCI_FLAG_MODERN 0x0004 #define VTPCI_FLAG_INTX 0x1000 #define VTPCI_FLAG_MSI 0x2000 #define VTPCI_FLAG_MSIX 0x4000 #define VTPCI_FLAG_SHARED_MSIX 0x8000 #define VTPCI_FLAG_ITYPE_MASK 0xF000 /* The VirtIO PCI "bus" will only ever have one child. */ device_t vtpci_child_dev; struct virtio_feature_desc *vtpci_child_feat_desc; /* * Ideally, each virtqueue that the driver provides a callback for will * receive its own MSIX vector. If there are not sufficient vectors * available, then attempt to have all the VQs share one vector. For * MSIX, the configuration changed notifications must be on their own * vector. * * If MSIX is not available, attempt to have the whole device share * one MSI vector, and then, finally, one intx interrupt. */ struct vtpci_interrupt vtpci_device_interrupt; struct vtpci_interrupt *vtpci_msix_vq_interrupts; int vtpci_nmsix_resources; }; extern int vtpci_disable_msix; static inline device_t vtpci_child_device(struct vtpci_common *cn) { return (cn->vtpci_child_dev); } static inline bool vtpci_is_msix_available(struct vtpci_common *cn) { return ((cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) == 0); } static inline bool vtpci_is_msix_enabled(struct vtpci_common *cn) { return ((cn->vtpci_flags & VTPCI_FLAG_MSIX) != 0); } static inline bool vtpci_is_modern(struct vtpci_common *cn) { return ((cn->vtpci_flags & VTPCI_FLAG_MODERN) != 0); } static inline int vtpci_virtqueue_count(struct vtpci_common *cn) { return (cn->vtpci_nvqs); } void vtpci_init(struct vtpci_common *cn, device_t dev, bool modern); int vtpci_add_child(struct vtpci_common *cn); int vtpci_delete_child(struct vtpci_common *cn); void vtpci_child_detached(struct vtpci_common *cn); int vtpci_reinit(struct vtpci_common *cn); uint64_t vtpci_negotiate_features(struct vtpci_common *cn, uint64_t child_features, uint64_t host_features); bool vtpci_with_feature(struct vtpci_common *cn, uint64_t feature); int vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result); int vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value); -int vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs, +int vtpci_alloc_virtqueues(struct vtpci_common *cn, int nvqs, struct vq_alloc_info *vq_info); int vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type); void vtpci_release_child_resources(struct vtpci_common *cn); #endif /* _VIRTIO_PCI_H */ diff --git a/sys/dev/virtio/pci/virtio_pci_legacy.c b/sys/dev/virtio/pci/virtio_pci_legacy.c index 72d637fb0f55..9853546e1af2 100644 --- a/sys/dev/virtio/pci/virtio_pci_legacy.c +++ b/sys/dev/virtio/pci/virtio_pci_legacy.c @@ -1,767 +1,767 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for the legacy VirtIO PCI interface. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_bus_if.h" #include "virtio_pci_if.h" #include "virtio_if.h" struct vtpci_legacy_softc { device_t vtpci_dev; struct vtpci_common vtpci_common; int vtpci_res_type; struct resource *vtpci_res; struct resource *vtpci_msix_table_res; struct resource *vtpci_msix_pba_res; }; static int vtpci_legacy_probe(device_t); static int vtpci_legacy_attach(device_t); static int vtpci_legacy_detach(device_t); static int vtpci_legacy_suspend(device_t); static int vtpci_legacy_resume(device_t); static int vtpci_legacy_shutdown(device_t); static void vtpci_legacy_driver_added(device_t, driver_t *); static void vtpci_legacy_child_detached(device_t, device_t); static int vtpci_legacy_read_ivar(device_t, device_t, int, uintptr_t *); static int vtpci_legacy_write_ivar(device_t, device_t, int, uintptr_t); static uint8_t vtpci_legacy_read_isr(device_t); static uint16_t vtpci_legacy_get_vq_size(device_t, int); static bus_size_t vtpci_legacy_get_vq_notify_off(device_t, int); static void vtpci_legacy_set_vq(device_t, struct virtqueue *); static void vtpci_legacy_disable_vq(device_t, int); static int vtpci_legacy_register_cfg_msix(device_t, struct vtpci_interrupt *); static int vtpci_legacy_register_vq_msix(device_t, int idx, struct vtpci_interrupt *); static uint64_t vtpci_legacy_negotiate_features(device_t, uint64_t); static bool vtpci_legacy_with_feature(device_t, uint64_t); -static int vtpci_legacy_alloc_virtqueues(device_t, int, int, +static int vtpci_legacy_alloc_virtqueues(device_t, int, struct vq_alloc_info *); static int vtpci_legacy_setup_interrupts(device_t, enum intr_type); static void vtpci_legacy_stop(device_t); static int vtpci_legacy_reinit(device_t, uint64_t); static void vtpci_legacy_reinit_complete(device_t); static void vtpci_legacy_notify_vq(device_t, uint16_t, bus_size_t); static void vtpci_legacy_read_dev_config(device_t, bus_size_t, void *, int); static void vtpci_legacy_write_dev_config(device_t, bus_size_t, const void *, int); static bool vtpci_legacy_setup_msix(struct vtpci_legacy_softc *sc); static void vtpci_legacy_teardown_msix(struct vtpci_legacy_softc *sc); static int vtpci_legacy_alloc_resources(struct vtpci_legacy_softc *); static void vtpci_legacy_free_resources(struct vtpci_legacy_softc *); static void vtpci_legacy_probe_and_attach_child(struct vtpci_legacy_softc *); static uint8_t vtpci_legacy_get_status(struct vtpci_legacy_softc *); static void vtpci_legacy_set_status(struct vtpci_legacy_softc *, uint8_t); static void vtpci_legacy_select_virtqueue(struct vtpci_legacy_softc *, int); static void vtpci_legacy_reset(struct vtpci_legacy_softc *); #define VIRTIO_PCI_LEGACY_CONFIG(_sc) \ VIRTIO_PCI_CONFIG_OFF(vtpci_is_msix_enabled(&(_sc)->vtpci_common)) #define vtpci_legacy_read_config_1(sc, o) \ bus_read_1((sc)->vtpci_res, (o)) #define vtpci_legacy_write_config_1(sc, o, v) \ bus_write_1((sc)->vtpci_res, (o), (v)) /* * VirtIO specifies that PCI Configuration area is guest endian. However, * since PCI devices are inherently little-endian, on big-endian systems * the bus layer transparently converts it to BE. For virtio-legacy, this * conversion is undesired, so an extra byte swap is required to fix it. */ #define vtpci_legacy_read_config_2(sc, o) \ le16toh(bus_read_2((sc)->vtpci_res, (o))) #define vtpci_legacy_read_config_4(sc, o) \ le32toh(bus_read_4((sc)->vtpci_res, (o))) #define vtpci_legacy_write_config_2(sc, o, v) \ bus_write_2((sc)->vtpci_res, (o), (htole16(v))) #define vtpci_legacy_write_config_4(sc, o, v) \ bus_write_4((sc)->vtpci_res, (o), (htole32(v))) /* PCI Header LE. On BE systems the bus layer takes care of byte swapping. */ #define vtpci_legacy_read_header_2(sc, o) \ bus_read_2((sc)->vtpci_res, (o)) #define vtpci_legacy_read_header_4(sc, o) \ bus_read_4((sc)->vtpci_res, (o)) #define vtpci_legacy_write_header_2(sc, o, v) \ bus_write_2((sc)->vtpci_res, (o), (v)) #define vtpci_legacy_write_header_4(sc, o, v) \ bus_write_4((sc)->vtpci_res, (o), (v)) static device_method_t vtpci_legacy_methods[] = { /* Device interface. */ DEVMETHOD(device_probe, vtpci_legacy_probe), DEVMETHOD(device_attach, vtpci_legacy_attach), DEVMETHOD(device_detach, vtpci_legacy_detach), DEVMETHOD(device_suspend, vtpci_legacy_suspend), DEVMETHOD(device_resume, vtpci_legacy_resume), DEVMETHOD(device_shutdown, vtpci_legacy_shutdown), /* Bus interface. */ DEVMETHOD(bus_driver_added, vtpci_legacy_driver_added), DEVMETHOD(bus_child_detached, vtpci_legacy_child_detached), DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), DEVMETHOD(bus_read_ivar, vtpci_legacy_read_ivar), DEVMETHOD(bus_write_ivar, vtpci_legacy_write_ivar), /* VirtIO PCI interface. */ DEVMETHOD(virtio_pci_read_isr, vtpci_legacy_read_isr), DEVMETHOD(virtio_pci_get_vq_size, vtpci_legacy_get_vq_size), DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_legacy_get_vq_notify_off), DEVMETHOD(virtio_pci_set_vq, vtpci_legacy_set_vq), DEVMETHOD(virtio_pci_disable_vq, vtpci_legacy_disable_vq), DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_legacy_register_cfg_msix), DEVMETHOD(virtio_pci_register_vq_msix, vtpci_legacy_register_vq_msix), /* VirtIO bus interface. */ DEVMETHOD(virtio_bus_negotiate_features, vtpci_legacy_negotiate_features), DEVMETHOD(virtio_bus_with_feature, vtpci_legacy_with_feature), DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_legacy_alloc_virtqueues), DEVMETHOD(virtio_bus_setup_intr, vtpci_legacy_setup_interrupts), DEVMETHOD(virtio_bus_stop, vtpci_legacy_stop), DEVMETHOD(virtio_bus_reinit, vtpci_legacy_reinit), DEVMETHOD(virtio_bus_reinit_complete, vtpci_legacy_reinit_complete), DEVMETHOD(virtio_bus_notify_vq, vtpci_legacy_notify_vq), DEVMETHOD(virtio_bus_read_device_config, vtpci_legacy_read_dev_config), DEVMETHOD(virtio_bus_write_device_config, vtpci_legacy_write_dev_config), DEVMETHOD_END }; static driver_t vtpci_legacy_driver = { .name = "virtio_pci", .methods = vtpci_legacy_methods, .size = sizeof(struct vtpci_legacy_softc) }; DRIVER_MODULE(virtio_pci_legacy, pci, vtpci_legacy_driver, 0, 0); static int vtpci_legacy_probe(device_t dev) { char desc[64]; const char *name; if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) return (ENXIO); if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || pci_get_device(dev) > VIRTIO_PCI_DEVICEID_LEGACY_MAX) return (ENXIO); if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) return (ENXIO); name = virtio_device_name(pci_get_subdevice(dev)); if (name == NULL) name = "Unknown"; snprintf(desc, sizeof(desc), "VirtIO PCI (legacy) %s adapter", name); device_set_desc_copy(dev, desc); /* Prefer transitional modern VirtIO PCI. */ return (BUS_PROBE_LOW_PRIORITY); } static int vtpci_legacy_attach(device_t dev) { struct vtpci_legacy_softc *sc; int error; sc = device_get_softc(dev); sc->vtpci_dev = dev; vtpci_init(&sc->vtpci_common, dev, false); error = vtpci_legacy_alloc_resources(sc); if (error) { device_printf(dev, "cannot map I/O space nor memory space\n"); return (error); } if (vtpci_is_msix_available(&sc->vtpci_common) && !vtpci_legacy_setup_msix(sc)) { device_printf(dev, "cannot setup MSI-x resources\n"); error = ENXIO; goto fail; } vtpci_legacy_reset(sc); /* Tell the host we've noticed this device. */ vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); error = vtpci_add_child(&sc->vtpci_common); if (error) goto fail; vtpci_legacy_probe_and_attach_child(sc); return (0); fail: vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); vtpci_legacy_detach(dev); return (error); } static int vtpci_legacy_detach(device_t dev) { struct vtpci_legacy_softc *sc; int error; sc = device_get_softc(dev); error = vtpci_delete_child(&sc->vtpci_common); if (error) return (error); vtpci_legacy_reset(sc); vtpci_legacy_teardown_msix(sc); vtpci_legacy_free_resources(sc); return (0); } static int vtpci_legacy_suspend(device_t dev) { return (bus_generic_suspend(dev)); } static int vtpci_legacy_resume(device_t dev) { return (bus_generic_resume(dev)); } static int vtpci_legacy_shutdown(device_t dev) { (void) bus_generic_shutdown(dev); /* Forcibly stop the host device. */ vtpci_legacy_stop(dev); return (0); } static void vtpci_legacy_driver_added(device_t dev, driver_t *driver) { vtpci_legacy_probe_and_attach_child(device_get_softc(dev)); } static void vtpci_legacy_child_detached(device_t dev, device_t child) { struct vtpci_legacy_softc *sc; sc = device_get_softc(dev); vtpci_legacy_reset(sc); vtpci_child_detached(&sc->vtpci_common); /* After the reset, retell the host we've noticed this device. */ vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); } static int vtpci_legacy_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { struct vtpci_legacy_softc *sc; struct vtpci_common *cn; sc = device_get_softc(dev); cn = &sc->vtpci_common; if (vtpci_child_device(cn) != child) return (ENOENT); switch (index) { case VIRTIO_IVAR_DEVTYPE: *result = pci_get_subdevice(dev); break; default: return (vtpci_read_ivar(cn, index, result)); } return (0); } static int vtpci_legacy_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { struct vtpci_legacy_softc *sc; struct vtpci_common *cn; sc = device_get_softc(dev); cn = &sc->vtpci_common; if (vtpci_child_device(cn) != child) return (ENOENT); switch (index) { default: return (vtpci_write_ivar(cn, index, value)); } return (0); } static uint64_t vtpci_legacy_negotiate_features(device_t dev, uint64_t child_features) { struct vtpci_legacy_softc *sc; uint64_t host_features, features; sc = device_get_softc(dev); host_features = vtpci_legacy_read_header_4(sc, VIRTIO_PCI_HOST_FEATURES); features = vtpci_negotiate_features(&sc->vtpci_common, child_features, host_features); vtpci_legacy_write_header_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); return (features); } static bool vtpci_legacy_with_feature(device_t dev, uint64_t feature) { struct vtpci_legacy_softc *sc; sc = device_get_softc(dev); return (vtpci_with_feature(&sc->vtpci_common, feature)); } static int -vtpci_legacy_alloc_virtqueues(device_t dev, int flags, int nvqs, +vtpci_legacy_alloc_virtqueues(device_t dev, int nvqs, struct vq_alloc_info *vq_info) { struct vtpci_legacy_softc *sc; struct vtpci_common *cn; sc = device_get_softc(dev); cn = &sc->vtpci_common; - return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info)); + return (vtpci_alloc_virtqueues(cn, nvqs, vq_info)); } static int vtpci_legacy_setup_interrupts(device_t dev, enum intr_type type) { struct vtpci_legacy_softc *sc; sc = device_get_softc(dev); return (vtpci_setup_interrupts(&sc->vtpci_common, type)); } static void vtpci_legacy_stop(device_t dev) { vtpci_legacy_reset(device_get_softc(dev)); } static int vtpci_legacy_reinit(device_t dev, uint64_t features) { struct vtpci_legacy_softc *sc; struct vtpci_common *cn; int error; sc = device_get_softc(dev); cn = &sc->vtpci_common; /* * Redrive the device initialization. This is a bit of an abuse of * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to * play nice. * * We do not allow the host device to change from what was originally * negotiated beyond what the guest driver changed. MSIX state should * not change, number of virtqueues and their size remain the same, etc. * This will need to be rethought when we want to support migration. */ if (vtpci_legacy_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) vtpci_legacy_stop(dev); /* * Quickly drive the status through ACK and DRIVER. The device does * not become usable again until DRIVER_OK in reinit complete. */ vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); vtpci_legacy_negotiate_features(dev, features); error = vtpci_reinit(cn); if (error) return (error); return (0); } static void vtpci_legacy_reinit_complete(device_t dev) { struct vtpci_legacy_softc *sc; sc = device_get_softc(dev); vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); } static void vtpci_legacy_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) { struct vtpci_legacy_softc *sc; sc = device_get_softc(dev); MPASS(offset == VIRTIO_PCI_QUEUE_NOTIFY); vtpci_legacy_write_header_2(sc, offset, queue); } static uint8_t vtpci_legacy_get_status(struct vtpci_legacy_softc *sc) { return (vtpci_legacy_read_config_1(sc, VIRTIO_PCI_STATUS)); } static void vtpci_legacy_set_status(struct vtpci_legacy_softc *sc, uint8_t status) { if (status != VIRTIO_CONFIG_STATUS_RESET) status |= vtpci_legacy_get_status(sc); vtpci_legacy_write_config_1(sc, VIRTIO_PCI_STATUS, status); } static void vtpci_legacy_read_dev_config(device_t dev, bus_size_t offset, void *dst, int length) { struct vtpci_legacy_softc *sc; bus_size_t off; uint8_t *d; int i; sc = device_get_softc(dev); off = VIRTIO_PCI_LEGACY_CONFIG(sc) + offset; d = dst; for (i = 0; i < length; i++) { d[i] = vtpci_legacy_read_config_1(sc, off + i); } } static void vtpci_legacy_write_dev_config(device_t dev, bus_size_t offset, const void *src, int length) { struct vtpci_legacy_softc *sc; bus_size_t off; const uint8_t *s; int i; sc = device_get_softc(dev); off = VIRTIO_PCI_LEGACY_CONFIG(sc) + offset; s = src; for (i = 0; i < length; i++) { vtpci_legacy_write_config_1(sc, off + i, s[i]); } } static bool vtpci_legacy_setup_msix(struct vtpci_legacy_softc *sc) { device_t dev; int rid, table_rid; dev = sc->vtpci_dev; rid = table_rid = pci_msix_table_bar(dev); if (rid != PCIR_BAR(0)) { sc->vtpci_msix_table_res = bus_alloc_resource_any( dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->vtpci_msix_table_res == NULL) return (false); } rid = pci_msix_pba_bar(dev); if (rid != table_rid && rid != PCIR_BAR(0)) { sc->vtpci_msix_pba_res = bus_alloc_resource_any( dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->vtpci_msix_pba_res == NULL) return (false); } return (true); } static void vtpci_legacy_teardown_msix(struct vtpci_legacy_softc *sc) { device_t dev; dev = sc->vtpci_dev; if (sc->vtpci_msix_pba_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->vtpci_msix_pba_res), sc->vtpci_msix_pba_res); sc->vtpci_msix_pba_res = NULL; } if (sc->vtpci_msix_table_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->vtpci_msix_table_res), sc->vtpci_msix_table_res); sc->vtpci_msix_table_res = NULL; } } static int vtpci_legacy_alloc_resources(struct vtpci_legacy_softc *sc) { const int res_types[] = { SYS_RES_IOPORT, SYS_RES_MEMORY }; device_t dev; int rid, i; dev = sc->vtpci_dev; /* * Most hypervisors export the common configuration structure in IO * space, but some use memory space; try both. */ for (i = 0; nitems(res_types); i++) { rid = PCIR_BAR(0); sc->vtpci_res_type = res_types[i]; sc->vtpci_res = bus_alloc_resource_any(dev, res_types[i], &rid, RF_ACTIVE); if (sc->vtpci_res != NULL) break; } if (sc->vtpci_res == NULL) return (ENXIO); return (0); } static void vtpci_legacy_free_resources(struct vtpci_legacy_softc *sc) { device_t dev; dev = sc->vtpci_dev; if (sc->vtpci_res != NULL) { bus_release_resource(dev, sc->vtpci_res_type, PCIR_BAR(0), sc->vtpci_res); sc->vtpci_res = NULL; } } static void vtpci_legacy_probe_and_attach_child(struct vtpci_legacy_softc *sc) { device_t dev, child; dev = sc->vtpci_dev; child = vtpci_child_device(&sc->vtpci_common); if (child == NULL || device_get_state(child) != DS_NOTPRESENT) return; if (device_probe(child) != 0) return; vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); if (device_attach(child) != 0) { vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); /* Reset status for future attempt. */ vtpci_legacy_child_detached(dev, child); } else { vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); VIRTIO_ATTACH_COMPLETED(child); } } static int vtpci_legacy_register_msix(struct vtpci_legacy_softc *sc, int offset, struct vtpci_interrupt *intr) { uint16_t vector; if (intr != NULL) { /* Map from guest rid to host vector. */ vector = intr->vti_rid - 1; } else vector = VIRTIO_MSI_NO_VECTOR; vtpci_legacy_write_header_2(sc, offset, vector); return (vtpci_legacy_read_header_2(sc, offset) == vector ? 0 : ENODEV); } static int vtpci_legacy_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) { struct vtpci_legacy_softc *sc; int error; sc = device_get_softc(dev); error = vtpci_legacy_register_msix(sc, VIRTIO_MSI_CONFIG_VECTOR, intr); if (error) { device_printf(dev, "unable to register config MSIX interrupt\n"); return (error); } return (0); } static int vtpci_legacy_register_vq_msix(device_t dev, int idx, struct vtpci_interrupt *intr) { struct vtpci_legacy_softc *sc; int error; sc = device_get_softc(dev); vtpci_legacy_select_virtqueue(sc, idx); error = vtpci_legacy_register_msix(sc, VIRTIO_MSI_QUEUE_VECTOR, intr); if (error) { device_printf(dev, "unable to register virtqueue MSIX interrupt\n"); return (error); } return (0); } static void vtpci_legacy_reset(struct vtpci_legacy_softc *sc) { /* * Setting the status to RESET sets the host device to the * original, uninitialized state. */ vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); (void) vtpci_legacy_get_status(sc); } static void vtpci_legacy_select_virtqueue(struct vtpci_legacy_softc *sc, int idx) { vtpci_legacy_write_header_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); } static uint8_t vtpci_legacy_read_isr(device_t dev) { struct vtpci_legacy_softc *sc; sc = device_get_softc(dev); return (vtpci_legacy_read_config_1(sc, VIRTIO_PCI_ISR)); } static uint16_t vtpci_legacy_get_vq_size(device_t dev, int idx) { struct vtpci_legacy_softc *sc; sc = device_get_softc(dev); vtpci_legacy_select_virtqueue(sc, idx); return (vtpci_legacy_read_header_2(sc, VIRTIO_PCI_QUEUE_NUM)); } static bus_size_t vtpci_legacy_get_vq_notify_off(device_t dev, int idx) { return (VIRTIO_PCI_QUEUE_NOTIFY); } static void vtpci_legacy_set_vq(device_t dev, struct virtqueue *vq) { struct vtpci_legacy_softc *sc; sc = device_get_softc(dev); vtpci_legacy_select_virtqueue(sc, virtqueue_index(vq)); vtpci_legacy_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); } static void vtpci_legacy_disable_vq(device_t dev, int idx) { struct vtpci_legacy_softc *sc; sc = device_get_softc(dev); vtpci_legacy_select_virtqueue(sc, idx); vtpci_legacy_write_header_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); } diff --git a/sys/dev/virtio/pci/virtio_pci_modern.c b/sys/dev/virtio/pci/virtio_pci_modern.c index 84d9511798e0..6e3b08e244c0 100644 --- a/sys/dev/virtio/pci/virtio_pci_modern.c +++ b/sys/dev/virtio/pci/virtio_pci_modern.c @@ -1,1445 +1,1445 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for the modern VirtIO PCI interface. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_bus_if.h" #include "virtio_pci_if.h" #include "virtio_if.h" struct vtpci_modern_resource_map { struct resource_map vtrm_map; int vtrm_cap_offset; int vtrm_bar; int vtrm_offset; int vtrm_length; int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */ }; struct vtpci_modern_bar_resource { struct resource *vtbr_res; int vtbr_type; }; struct vtpci_modern_softc { device_t vtpci_dev; struct vtpci_common vtpci_common; uint32_t vtpci_notify_offset_multiplier; uint16_t vtpci_devid; int vtpci_msix_bar; struct resource *vtpci_msix_res; struct vtpci_modern_resource_map vtpci_common_res_map; struct vtpci_modern_resource_map vtpci_notify_res_map; struct vtpci_modern_resource_map vtpci_isr_res_map; struct vtpci_modern_resource_map vtpci_device_res_map; #define VTPCI_MODERN_MAX_BARS 6 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS]; }; static int vtpci_modern_probe(device_t); static int vtpci_modern_attach(device_t); static int vtpci_modern_detach(device_t); static int vtpci_modern_suspend(device_t); static int vtpci_modern_resume(device_t); static int vtpci_modern_shutdown(device_t); static void vtpci_modern_driver_added(device_t, driver_t *); static void vtpci_modern_child_detached(device_t, device_t); static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *); static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t); static uint8_t vtpci_modern_read_isr(device_t); static uint16_t vtpci_modern_get_vq_size(device_t, int); static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int); static void vtpci_modern_set_vq(device_t, struct virtqueue *); static void vtpci_modern_disable_vq(device_t, int); static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int, struct vtpci_interrupt *); static int vtpci_modern_register_cfg_msix(device_t, struct vtpci_interrupt *); static int vtpci_modern_register_vq_msix(device_t, int idx, struct vtpci_interrupt *); static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t); static int vtpci_modern_finalize_features(device_t); static bool vtpci_modern_with_feature(device_t, uint64_t); -static int vtpci_modern_alloc_virtqueues(device_t, int, int, +static int vtpci_modern_alloc_virtqueues(device_t, int, struct vq_alloc_info *); static int vtpci_modern_setup_interrupts(device_t, enum intr_type); static void vtpci_modern_stop(device_t); static int vtpci_modern_reinit(device_t, uint64_t); static void vtpci_modern_reinit_complete(device_t); static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t); static int vtpci_modern_config_generation(device_t); static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int); static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int); static int vtpci_modern_probe_configs(device_t); static int vtpci_modern_find_cap(device_t, uint8_t, int *); static int vtpci_modern_map_configs(struct vtpci_modern_softc *); static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *); static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *, uint8_t, int, int, struct vtpci_modern_resource_map *); static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int); static struct resource *vtpci_modern_get_bar_resource( struct vtpci_modern_softc *, int, int); static struct resource *vtpci_modern_alloc_bar_resource( struct vtpci_modern_softc *, int, int); static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *); static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *, struct vtpci_modern_resource_map *); static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *, struct vtpci_modern_resource_map *); static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *); static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *); static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *); static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *); static void vtpci_modern_write_features(struct vtpci_modern_softc *, uint64_t); static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int); static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *); static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t); static void vtpci_modern_reset(struct vtpci_modern_softc *); static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *); static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *, bus_size_t); static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *, bus_size_t); static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *, bus_size_t); static void vtpci_modern_write_common_1(struct vtpci_modern_softc *, bus_size_t, uint8_t); static void vtpci_modern_write_common_2(struct vtpci_modern_softc *, bus_size_t, uint16_t); static void vtpci_modern_write_common_4(struct vtpci_modern_softc *, bus_size_t, uint32_t); static void vtpci_modern_write_common_8(struct vtpci_modern_softc *, bus_size_t, uint64_t); static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *, bus_size_t, uint16_t); static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *, bus_size_t); static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *, bus_size_t); static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *, bus_size_t); static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *, bus_size_t); static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *, bus_size_t); static void vtpci_modern_write_device_1(struct vtpci_modern_softc *, bus_size_t, uint8_t); static void vtpci_modern_write_device_2(struct vtpci_modern_softc *, bus_size_t, uint16_t); static void vtpci_modern_write_device_4(struct vtpci_modern_softc *, bus_size_t, uint32_t); static void vtpci_modern_write_device_8(struct vtpci_modern_softc *, bus_size_t, uint64_t); /* Tunables. */ static int vtpci_modern_transitional = 0; TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional); static device_method_t vtpci_modern_methods[] = { /* Device interface. */ DEVMETHOD(device_probe, vtpci_modern_probe), DEVMETHOD(device_attach, vtpci_modern_attach), DEVMETHOD(device_detach, vtpci_modern_detach), DEVMETHOD(device_suspend, vtpci_modern_suspend), DEVMETHOD(device_resume, vtpci_modern_resume), DEVMETHOD(device_shutdown, vtpci_modern_shutdown), /* Bus interface. */ DEVMETHOD(bus_driver_added, vtpci_modern_driver_added), DEVMETHOD(bus_child_detached, vtpci_modern_child_detached), DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar), DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar), /* VirtIO PCI interface. */ DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr), DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size), DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off), DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq), DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq), DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix), DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix), /* VirtIO bus interface. */ DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features), DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features), DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature), DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues), DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts), DEVMETHOD(virtio_bus_stop, vtpci_modern_stop), DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit), DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete), DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq), DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation), DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config), DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config), DEVMETHOD_END }; static driver_t vtpci_modern_driver = { .name = "virtio_pci", .methods = vtpci_modern_methods, .size = sizeof(struct vtpci_modern_softc) }; DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0); static int vtpci_modern_probe(device_t dev) { char desc[64]; const char *name; uint16_t devid; if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) return (ENXIO); if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX) return (ENXIO); if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) { if (!vtpci_modern_transitional) return (ENXIO); devid = pci_get_subdevice(dev); } else devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; if (vtpci_modern_probe_configs(dev) != 0) return (ENXIO); name = virtio_device_name(devid); if (name == NULL) name = "Unknown"; snprintf(desc, sizeof(desc), "VirtIO PCI (modern) %s adapter", name); device_set_desc_copy(dev, desc); return (BUS_PROBE_DEFAULT); } static int vtpci_modern_attach(device_t dev) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); sc->vtpci_dev = dev; vtpci_init(&sc->vtpci_common, dev, true); if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) sc->vtpci_devid = pci_get_subdevice(dev); else sc->vtpci_devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN; error = vtpci_modern_map_configs(sc); if (error) { device_printf(dev, "cannot map configs\n"); vtpci_modern_unmap_configs(sc); return (error); } vtpci_modern_reset(sc); /* Tell the host we've noticed this device. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); error = vtpci_add_child(&sc->vtpci_common); if (error) goto fail; vtpci_modern_probe_and_attach_child(sc); return (0); fail: vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); vtpci_modern_detach(dev); return (error); } static int vtpci_modern_detach(device_t dev) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); error = vtpci_delete_child(&sc->vtpci_common); if (error) return (error); vtpci_modern_reset(sc); vtpci_modern_unmap_configs(sc); return (0); } static int vtpci_modern_suspend(device_t dev) { return (bus_generic_suspend(dev)); } static int vtpci_modern_resume(device_t dev) { return (bus_generic_resume(dev)); } static int vtpci_modern_shutdown(device_t dev) { (void) bus_generic_shutdown(dev); /* Forcibly stop the host device. */ vtpci_modern_stop(dev); return (0); } static void vtpci_modern_driver_added(device_t dev, driver_t *driver) { vtpci_modern_probe_and_attach_child(device_get_softc(dev)); } static void vtpci_modern_child_detached(device_t dev, device_t child) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_reset(sc); vtpci_child_detached(&sc->vtpci_common); /* After the reset, retell the host we've noticed this device. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); } static int vtpci_modern_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { struct vtpci_modern_softc *sc; struct vtpci_common *cn; sc = device_get_softc(dev); cn = &sc->vtpci_common; if (vtpci_child_device(cn) != child) return (ENOENT); switch (index) { case VIRTIO_IVAR_DEVTYPE: *result = sc->vtpci_devid; break; default: return (vtpci_read_ivar(cn, index, result)); } return (0); } static int vtpci_modern_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { struct vtpci_modern_softc *sc; struct vtpci_common *cn; sc = device_get_softc(dev); cn = &sc->vtpci_common; if (vtpci_child_device(cn) != child) return (ENOENT); switch (index) { default: return (vtpci_write_ivar(cn, index, value)); } return (0); } static uint64_t vtpci_modern_negotiate_features(device_t dev, uint64_t child_features) { struct vtpci_modern_softc *sc; uint64_t host_features, features; sc = device_get_softc(dev); host_features = vtpci_modern_read_features(sc); /* * Since the driver was added as a child of the modern PCI bus, * always add the V1 flag. */ child_features |= VIRTIO_F_VERSION_1; features = vtpci_negotiate_features(&sc->vtpci_common, child_features, host_features); vtpci_modern_write_features(sc, features); return (features); } static int vtpci_modern_finalize_features(device_t dev) { struct vtpci_modern_softc *sc; uint8_t status; sc = device_get_softc(dev); /* * Must re-read the status after setting it to verify the negotiated * features were accepted by the device. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK); status = vtpci_modern_get_status(sc); if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { device_printf(dev, "desired features were not accepted\n"); return (ENOTSUP); } return (0); } static bool vtpci_modern_with_feature(device_t dev, uint64_t feature) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); return (vtpci_with_feature(&sc->vtpci_common, feature)); } static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *sc) { uint32_t features0, features1; vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0); features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1); features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF); return (((uint64_t) features1 << 32) | features0); } static void vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features) { uint32_t features0, features1; features0 = features; features1 = features >> 32; vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0); vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0); vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1); vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1); } static int -vtpci_modern_alloc_virtqueues(device_t dev, int flags, int nvqs, +vtpci_modern_alloc_virtqueues(device_t dev, int nvqs, struct vq_alloc_info *vq_info) { struct vtpci_modern_softc *sc; struct vtpci_common *cn; uint16_t max_nvqs; sc = device_get_softc(dev); cn = &sc->vtpci_common; max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ); if (nvqs > max_nvqs) { device_printf(sc->vtpci_dev, "requested virtqueue count %d " "exceeds max %d\n", nvqs, max_nvqs); return (E2BIG); } - return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info)); + return (vtpci_alloc_virtqueues(cn, nvqs, vq_info)); } static int vtpci_modern_setup_interrupts(device_t dev, enum intr_type type) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); error = vtpci_setup_interrupts(&sc->vtpci_common, type); if (error == 0) vtpci_modern_enable_virtqueues(sc); return (error); } static void vtpci_modern_stop(device_t dev) { vtpci_modern_reset(device_get_softc(dev)); } static int vtpci_modern_reinit(device_t dev, uint64_t features) { struct vtpci_modern_softc *sc; struct vtpci_common *cn; int error; sc = device_get_softc(dev); cn = &sc->vtpci_common; /* * Redrive the device initialization. This is a bit of an abuse of * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to * play nice. * * We do not allow the host device to change from what was originally * negotiated beyond what the guest driver changed. MSIX state should * not change, number of virtqueues and their size remain the same, etc. * This will need to be rethought when we want to support migration. */ if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) vtpci_modern_stop(dev); /* * Quickly drive the status through ACK and DRIVER. The device does * not become usable again until DRIVER_OK in reinit complete. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK); vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); /* * TODO: Check that features are not added as to what was * originally negotiated. */ vtpci_modern_negotiate_features(dev, features); error = vtpci_modern_finalize_features(dev); if (error) { device_printf(dev, "cannot finalize features during reinit\n"); return (error); } error = vtpci_reinit(cn); if (error) return (error); return (0); } static void vtpci_modern_reinit_complete(device_t dev) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_enable_virtqueues(sc); vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); } static void vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_write_notify_2(sc, offset, queue); } static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *sc) { return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS)); } static void vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status) { if (status != VIRTIO_CONFIG_STATUS_RESET) status |= vtpci_modern_get_status(sc); vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status); } static int vtpci_modern_config_generation(device_t dev) { struct vtpci_modern_softc *sc; uint8_t gen; sc = device_get_softc(dev); gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION); return (gen); } static void vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst, int length) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { panic("%s: attempt to read dev config but not present", __func__); } switch (length) { case 1: *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset); break; case 2: *(uint16_t *) dst = virtio_htog16(true, vtpci_modern_read_device_2(sc, offset)); break; case 4: *(uint32_t *) dst = virtio_htog32(true, vtpci_modern_read_device_4(sc, offset)); break; case 8: *(uint64_t *) dst = virtio_htog64(true, vtpci_modern_read_device_8(sc, offset)); break; default: panic("%s: device %s invalid device read length %d offset %d", __func__, device_get_nameunit(dev), length, (int) offset); } } static void vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src, int length) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) { panic("%s: attempt to write dev config but not present", __func__); } switch (length) { case 1: vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src); break; case 2: { uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src); vtpci_modern_write_device_2(sc, offset, val); break; } case 4: { uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src); vtpci_modern_write_device_4(sc, offset, val); break; } case 8: { uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src); vtpci_modern_write_device_8(sc, offset, val); break; } default: panic("%s: device %s invalid device write length %d offset %d", __func__, device_get_nameunit(dev), length, (int) offset); } } static int vtpci_modern_probe_configs(device_t dev) { int error; /* * These config capabilities must be present. The DEVICE_CFG * capability is only present if the device requires it. */ error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL); if (error) { device_printf(dev, "cannot find COMMON_CFG capability\n"); return (error); } error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL); if (error) { device_printf(dev, "cannot find NOTIFY_CFG capability\n"); return (error); } error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL); if (error) { device_printf(dev, "cannot find ISR_CFG capability\n"); return (error); } return (0); } static int vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset) { uint32_t type, bar; int capreg, error; for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg); error == 0; error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) { type = pci_read_config(dev, capreg + offsetof(struct virtio_pci_cap, cfg_type), 1); bar = pci_read_config(dev, capreg + offsetof(struct virtio_pci_cap, bar), 1); /* Must ignore reserved BARs. */ if (bar >= VTPCI_MODERN_MAX_BARS) continue; if (type == cfg_type) { if (cap_offset != NULL) *cap_offset = capreg; break; } } return (error); } static int vtpci_modern_map_common_config(struct vtpci_modern_softc *sc) { device_t dev; int error; dev = sc->vtpci_dev; error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG, sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map); if (error) { device_printf(dev, "cannot find cap COMMON_CFG resource\n"); return (error); } error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map); if (error) { device_printf(dev, "cannot alloc resource for COMMON_CFG\n"); return (error); } return (0); } static int vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc) { device_t dev; int cap_offset, error; dev = sc->vtpci_dev; error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG, -1, 2, &sc->vtpci_notify_res_map); if (error) { device_printf(dev, "cannot find cap NOTIFY_CFG resource\n"); return (error); } cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset; sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4); error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map); if (error) { device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n"); return (error); } return (0); } static int vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc) { device_t dev; int error; dev = sc->vtpci_dev; error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG, sizeof(uint8_t), 1, &sc->vtpci_isr_res_map); if (error) { device_printf(dev, "cannot find cap ISR_CFG resource\n"); return (error); } error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map); if (error) { device_printf(dev, "cannot alloc resource for ISR_CFG\n"); return (error); } return (0); } static int vtpci_modern_map_device_config(struct vtpci_modern_softc *sc) { device_t dev; int error; dev = sc->vtpci_dev; error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG, -1, 4, &sc->vtpci_device_res_map); if (error == ENOENT) { /* Device configuration is optional depending on device. */ return (0); } else if (error) { device_printf(dev, "cannot find cap DEVICE_CFG resource\n"); return (error); } error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map); if (error) { device_printf(dev, "cannot alloc resource for DEVICE_CFG\n"); return (error); } return (0); } static int vtpci_modern_map_configs(struct vtpci_modern_softc *sc) { int error; error = vtpci_modern_map_common_config(sc); if (error) return (error); error = vtpci_modern_map_notify_config(sc); if (error) return (error); error = vtpci_modern_map_isr_config(sc); if (error) return (error); error = vtpci_modern_map_device_config(sc); if (error) return (error); vtpci_modern_alloc_msix_resource(sc); return (0); } static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc) { vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map); vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map); vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map); vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map); vtpci_modern_free_bar_resources(sc); vtpci_modern_free_msix_resource(sc); sc->vtpci_notify_offset_multiplier = 0; } static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type, int min_size, int alignment, struct vtpci_modern_resource_map *res) { device_t dev; int cap_offset, offset, length, error; uint8_t bar, cap_length; dev = sc->vtpci_dev; error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset); if (error) return (error); cap_length = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1); if (cap_length < sizeof(struct virtio_pci_cap)) { device_printf(dev, "cap %u length %d less than expected\n", cfg_type, cap_length); return (ENXIO); } bar = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_cap, bar), 1); offset = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_cap, offset), 4); length = pci_read_config(dev, cap_offset + offsetof(struct virtio_pci_cap, length), 4); if (min_size != -1 && length < min_size) { device_printf(dev, "cap %u struct length %d less than min %d\n", cfg_type, length, min_size); return (ENXIO); } if (offset % alignment) { device_printf(dev, "cap %u struct offset %d not aligned to %d\n", cfg_type, offset, alignment); return (ENXIO); } /* BMV: TODO Can we determine the size of the BAR here? */ res->vtrm_cap_offset = cap_offset; res->vtrm_bar = bar; res->vtrm_offset = offset; res->vtrm_length = length; res->vtrm_type = vtpci_modern_bar_type(sc, bar); return (0); } static int vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar) { uint32_t val; /* * The BAR described by a config capability may be either an IOPORT or * MEM, but we must know the type when calling bus_alloc_resource(). */ val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4); if (PCI_BAR_IO(val)) return (SYS_RES_IOPORT); else return (SYS_RES_MEMORY); } static struct resource * vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) { struct resource *res; MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); res = sc->vtpci_bar_res[bar].vtbr_res; MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type); return (res); } static struct resource * vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar, int type) { struct resource *res; int rid; MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS); MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT); res = sc->vtpci_bar_res[bar].vtbr_res; if (res != NULL) { MPASS(sc->vtpci_bar_res[bar].vtbr_type == type); return (res); } rid = PCIR_BAR(bar); res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid, RF_ACTIVE | RF_UNMAPPED); if (res != NULL) { sc->vtpci_bar_res[bar].vtbr_res = res; sc->vtpci_bar_res[bar].vtbr_type = type; } return (res); } static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc) { device_t dev; struct resource *res; int bar, rid, type; dev = sc->vtpci_dev; for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) { res = sc->vtpci_bar_res[bar].vtbr_res; type = sc->vtpci_bar_res[bar].vtbr_type; if (res != NULL) { rid = PCIR_BAR(bar); bus_release_resource(dev, type, rid, res); sc->vtpci_bar_res[bar].vtbr_res = NULL; sc->vtpci_bar_res[bar].vtbr_type = 0; } } } static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc, struct vtpci_modern_resource_map *map) { struct resource_map_request req; struct resource *res; int type; type = map->vtrm_type; res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type); if (res == NULL) return (ENXIO); resource_init_map_request(&req); req.offset = map->vtrm_offset; req.length = map->vtrm_length; return (bus_map_resource(sc->vtpci_dev, type, res, &req, &map->vtrm_map)); } static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc, struct vtpci_modern_resource_map *map) { struct resource *res; int type; type = map->vtrm_type; res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type); if (res != NULL && map->vtrm_map.r_size != 0) { bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map); bzero(map, sizeof(struct vtpci_modern_resource_map)); } } static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc) { device_t dev; int bar; dev = sc->vtpci_dev; if (!vtpci_is_msix_available(&sc->vtpci_common) || (bar = pci_msix_table_bar(dev)) == -1) return; /* TODO: Can this BAR be in the 0-5 range? */ sc->vtpci_msix_bar = bar; if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE)) == NULL) device_printf(dev, "Unable to map MSIX table\n"); } static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc) { device_t dev; dev = sc->vtpci_dev; if (sc->vtpci_msix_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar, sc->vtpci_msix_res); sc->vtpci_msix_bar = 0; sc->vtpci_msix_res = NULL; } } static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc) { device_t dev, child; dev = sc->vtpci_dev; child = vtpci_child_device(&sc->vtpci_common); if (child == NULL || device_get_state(child) != DS_NOTPRESENT) return; if (device_probe(child) != 0) return; vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER); if (device_attach(child) != 0) { vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED); /* Reset state for later attempt. */ vtpci_modern_child_detached(dev, child); } else { vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK); VIRTIO_ATTACH_COMPLETED(child); } } static int vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset, struct vtpci_interrupt *intr) { uint16_t vector; if (intr != NULL) { /* Map from guest rid to host vector. */ vector = intr->vti_rid - 1; } else vector = VIRTIO_MSI_NO_VECTOR; vtpci_modern_write_common_2(sc, offset, vector); return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV); } static int vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr); if (error) { device_printf(dev, "unable to register config MSIX interrupt\n"); return (error); } return (0); } static int vtpci_modern_register_vq_msix(device_t dev, int idx, struct vtpci_interrupt *intr) { struct vtpci_modern_softc *sc; int error; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, idx); error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr); if (error) { device_printf(dev, "unable to register virtqueue MSIX interrupt\n"); return (error); } return (0); } static void vtpci_modern_reset(struct vtpci_modern_softc *sc) { /* * Setting the status to RESET sets the host device to the * original, uninitialized state. Must poll the status until * the reset is complete. */ vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET); while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET) cpu_spinwait(); } static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx) { vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx); } static uint8_t vtpci_modern_read_isr(device_t dev) { return (vtpci_modern_read_isr_1(device_get_softc(dev), 0)); } static uint16_t vtpci_modern_get_vq_size(device_t dev, int idx) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, idx); return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE)); } static bus_size_t vtpci_modern_get_vq_notify_off(device_t dev, int idx) { struct vtpci_modern_softc *sc; uint16_t q_notify_off; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, idx); q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF); return (q_notify_off * sc->vtpci_notify_offset_multiplier); } static void vtpci_modern_set_vq(device_t dev, struct virtqueue *vq) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, virtqueue_index(vq)); /* BMV: Currently we never adjust the device's proposed VQ size. */ vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq)); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq)); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq)); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq)); } static void vtpci_modern_disable_vq(device_t dev, int idx) { struct vtpci_modern_softc *sc; sc = device_get_softc(dev); vtpci_modern_select_virtqueue(sc, idx); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL); vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL); } static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc) { int idx; for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) { vtpci_modern_select_virtqueue(sc, idx); vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1); } } static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off)); } static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off) { return virtio_htog16(true, bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off)); } static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off) { return virtio_htog32(true, bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off)); } static void vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off, uint8_t val) { bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val); } static void vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off, uint16_t val) { bus_write_2(&sc->vtpci_common_res_map.vtrm_map, off, virtio_gtoh16(true, val)); } static void vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(&sc->vtpci_common_res_map.vtrm_map, off, virtio_gtoh32(true, val)); } static void vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off, uint64_t val) { uint32_t val0, val1; val0 = (uint32_t) val; val1 = val >> 32; vtpci_modern_write_common_4(sc, off, val0); vtpci_modern_write_common_4(sc, off + 4, val1); } static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off, uint16_t val) { bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val); } static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off)); } static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off)); } static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off)); } static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off) { return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off)); } static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off) { device_t dev; int gen; uint32_t val0, val1; dev = sc->vtpci_dev; /* * Treat the 64-bit field as two 32-bit fields. Use the generation * to ensure a consistent read. */ do { gen = vtpci_modern_config_generation(dev); val0 = vtpci_modern_read_device_4(sc, off); val1 = vtpci_modern_read_device_4(sc, off + 4); } while (gen != vtpci_modern_config_generation(dev)); return (((uint64_t) val1 << 32) | val0); } static void vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off, uint8_t val) { bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val); } static void vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off, uint16_t val) { bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val); } static void vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val); } static void vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off, uint64_t val) { uint32_t val0, val1; val0 = (uint32_t) val; val1 = val >> 32; vtpci_modern_write_device_4(sc, off, val0); vtpci_modern_write_device_4(sc, off + 4, val1); } diff --git a/sys/dev/virtio/random/virtio_random.c b/sys/dev/virtio/random/virtio_random.c index d54e2e6b70d4..0b6ee1dcbf6b 100644 --- a/sys/dev/virtio/random/virtio_random.c +++ b/sys/dev/virtio/random/virtio_random.c @@ -1,330 +1,330 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for VirtIO entropy device. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct vtrnd_softc { device_t vtrnd_dev; uint64_t vtrnd_features; struct virtqueue *vtrnd_vq; eventhandler_tag eh; bool inactive; struct sglist *vtrnd_sg; uint32_t *vtrnd_value; }; static int vtrnd_modevent(module_t, int, void *); static int vtrnd_probe(device_t); static int vtrnd_attach(device_t); static int vtrnd_detach(device_t); static int vtrnd_shutdown(device_t); static int vtrnd_negotiate_features(struct vtrnd_softc *); static int vtrnd_setup_features(struct vtrnd_softc *); static int vtrnd_alloc_virtqueue(struct vtrnd_softc *); static int vtrnd_harvest(struct vtrnd_softc *, void *, size_t *); static void vtrnd_enqueue(struct vtrnd_softc *sc); static unsigned vtrnd_read(void *, unsigned); #define VTRND_FEATURES 0 static struct virtio_feature_desc vtrnd_feature_desc[] = { { 0, NULL } }; static struct random_source random_vtrnd = { .rs_ident = "VirtIO Entropy Adapter", .rs_source = RANDOM_PURE_VIRTIO, .rs_read = vtrnd_read, }; /* Kludge for API limitations of random(4). */ static _Atomic(struct vtrnd_softc *) g_vtrnd_softc; static device_method_t vtrnd_methods[] = { /* Device methods. */ DEVMETHOD(device_probe, vtrnd_probe), DEVMETHOD(device_attach, vtrnd_attach), DEVMETHOD(device_detach, vtrnd_detach), DEVMETHOD(device_shutdown, vtrnd_shutdown), DEVMETHOD_END }; static driver_t vtrnd_driver = { "vtrnd", vtrnd_methods, sizeof(struct vtrnd_softc) }; VIRTIO_DRIVER_MODULE(virtio_random, vtrnd_driver, vtrnd_modevent, NULL); MODULE_VERSION(virtio_random, 1); MODULE_DEPEND(virtio_random, virtio, 1, 1, 1); MODULE_DEPEND(virtio_random, random_device, 1, 1, 1); VIRTIO_SIMPLE_PNPINFO(virtio_random, VIRTIO_ID_ENTROPY, "VirtIO Entropy Adapter"); static int vtrnd_modevent(module_t mod, int type, void *unused) { int error; switch (type) { case MOD_LOAD: case MOD_QUIESCE: case MOD_UNLOAD: case MOD_SHUTDOWN: error = 0; break; default: error = EOPNOTSUPP; break; } return (error); } static int vtrnd_probe(device_t dev) { return (VIRTIO_SIMPLE_PROBE(dev, virtio_random)); } static int vtrnd_attach(device_t dev) { struct vtrnd_softc *sc, *exp; size_t len; int error; sc = device_get_softc(dev); sc->vtrnd_dev = dev; virtio_set_feature_desc(dev, vtrnd_feature_desc); len = sizeof(*sc->vtrnd_value) * HARVESTSIZE; sc->vtrnd_value = malloc_aligned(len, len, M_DEVBUF, M_WAITOK); sc->vtrnd_sg = sglist_build(sc->vtrnd_value, len, M_WAITOK); error = vtrnd_setup_features(sc); if (error) { device_printf(dev, "cannot setup features\n"); goto fail; } error = vtrnd_alloc_virtqueue(sc); if (error) { device_printf(dev, "cannot allocate virtqueue\n"); goto fail; } exp = NULL; if (!atomic_compare_exchange_strong_explicit(&g_vtrnd_softc, &exp, sc, memory_order_release, memory_order_acquire)) { error = EEXIST; goto fail; } sc->eh = EVENTHANDLER_REGISTER(shutdown_post_sync, vtrnd_shutdown, dev, SHUTDOWN_PRI_LAST + 1); /* ??? */ if (sc->eh == NULL) { device_printf(dev, "Shutdown event registration failed\n"); error = ENXIO; goto fail; } sc->inactive = false; random_source_register(&random_vtrnd); vtrnd_enqueue(sc); fail: if (error) vtrnd_detach(dev); return (error); } static int vtrnd_detach(device_t dev) { struct vtrnd_softc *sc; uint32_t rdlen; sc = device_get_softc(dev); KASSERT( atomic_load_explicit(&g_vtrnd_softc, memory_order_acquire) == sc, ("only one global instance at a time")); sc->inactive = true; if (sc->eh != NULL) { EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->eh); sc->eh = NULL; } random_source_deregister(&random_vtrnd); /* clear the queue */ virtqueue_poll(sc->vtrnd_vq, &rdlen); atomic_store_explicit(&g_vtrnd_softc, NULL, memory_order_release); sglist_free(sc->vtrnd_sg); zfree(sc->vtrnd_value, M_DEVBUF); return (0); } static int vtrnd_shutdown(device_t dev) { struct vtrnd_softc *sc; sc = device_get_softc(dev); sc->inactive = true; return(0); } static int vtrnd_negotiate_features(struct vtrnd_softc *sc) { device_t dev; uint64_t features; dev = sc->vtrnd_dev; features = VTRND_FEATURES; sc->vtrnd_features = virtio_negotiate_features(dev, features); return (virtio_finalize_features(dev)); } static int vtrnd_setup_features(struct vtrnd_softc *sc) { int error; error = vtrnd_negotiate_features(sc); if (error) return (error); return (0); } static int vtrnd_alloc_virtqueue(struct vtrnd_softc *sc) { device_t dev; struct vq_alloc_info vq_info; dev = sc->vtrnd_dev; VQ_ALLOC_INFO_INIT(&vq_info, 0, NULL, sc, &sc->vtrnd_vq, "%s request", device_get_nameunit(dev)); - return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info)); + return (virtio_alloc_virtqueues(dev, 1, &vq_info)); } static void vtrnd_enqueue(struct vtrnd_softc *sc) { struct virtqueue *vq; int error __diagused; vq = sc->vtrnd_vq; KASSERT(virtqueue_empty(vq), ("%s: non-empty queue", __func__)); error = virtqueue_enqueue(vq, sc, sc->vtrnd_sg, 0, 1); KASSERT(error == 0, ("%s: virtqueue_enqueue returned error: %d", __func__, error)); virtqueue_notify(vq); } static int vtrnd_harvest(struct vtrnd_softc *sc, void *buf, size_t *sz) { struct virtqueue *vq; void *cookie; uint32_t rdlen; if (sc->inactive) return (EDEADLK); vq = sc->vtrnd_vq; cookie = virtqueue_dequeue(vq, &rdlen); if (cookie == NULL) return (EAGAIN); KASSERT(cookie == sc, ("%s: cookie mismatch", __func__)); *sz = MIN(rdlen, *sz); memcpy(buf, sc->vtrnd_value, *sz); vtrnd_enqueue(sc); return (0); } static unsigned vtrnd_read(void *buf, unsigned usz) { struct vtrnd_softc *sc; size_t sz; int error; sc = g_vtrnd_softc; if (sc == NULL) return (0); sz = usz; error = vtrnd_harvest(sc, buf, &sz); if (error != 0) return (0); return (sz); } diff --git a/sys/dev/virtio/scsi/virtio_scsi.c b/sys/dev/virtio/scsi/virtio_scsi.c index 0355962c7af9..f6278434e5f9 100644 --- a/sys/dev/virtio/scsi/virtio_scsi.c +++ b/sys/dev/virtio/scsi/virtio_scsi.c @@ -1,2369 +1,2369 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* Driver for VirtIO SCSI devices. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_if.h" static int vtscsi_modevent(module_t, int, void *); static int vtscsi_probe(device_t); static int vtscsi_attach(device_t); static int vtscsi_detach(device_t); static int vtscsi_suspend(device_t); static int vtscsi_resume(device_t); static int vtscsi_negotiate_features(struct vtscsi_softc *); static int vtscsi_setup_features(struct vtscsi_softc *); static void vtscsi_read_config(struct vtscsi_softc *, struct virtio_scsi_config *); static int vtscsi_maximum_segments(struct vtscsi_softc *, int); static int vtscsi_alloc_virtqueues(struct vtscsi_softc *); static void vtscsi_check_sizes(struct vtscsi_softc *); static void vtscsi_write_device_config(struct vtscsi_softc *); static int vtscsi_reinit(struct vtscsi_softc *); static int vtscsi_alloc_cam(struct vtscsi_softc *); static int vtscsi_register_cam(struct vtscsi_softc *); static void vtscsi_free_cam(struct vtscsi_softc *); static void vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *); static int vtscsi_register_async(struct vtscsi_softc *); static void vtscsi_deregister_async(struct vtscsi_softc *); static void vtscsi_cam_action(struct cam_sim *, union ccb *); static void vtscsi_cam_poll(struct cam_sim *); static void vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *, union ccb *); static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *, union ccb *); static void vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *); static void vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *); static void vtscsi_cam_abort(struct vtscsi_softc *, union ccb *); static void vtscsi_cam_path_inquiry(struct vtscsi_softc *, struct cam_sim *, union ccb *); static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *, struct sglist *, struct ccb_scsiio *); static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *, struct vtscsi_request *, int *, int *); static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *); static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *, struct vtscsi_request *); static void vtscsi_timedout_scsi_cmd(void *); static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *); static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *, struct ccb_scsiio *, struct virtio_scsi_cmd_resp *); static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *, struct vtscsi_request *); static void vtscsi_poll_ctrl_req(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_execute_ctrl_req(struct vtscsi_softc *, struct vtscsi_request *, struct sglist *, int, int, int); static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c, struct vtscsi_request *); static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *, struct vtscsi_request *); static void vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *); static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []); static void vtscsi_init_scsi_cmd_req(struct vtscsi_softc *, struct ccb_scsiio *, struct virtio_scsi_cmd_req *); static void vtscsi_init_ctrl_tmf_req(struct vtscsi_softc *, struct ccb_hdr *, uint32_t, uintptr_t, struct virtio_scsi_ctrl_tmf_req *); static void vtscsi_freeze_simq(struct vtscsi_softc *, int); static int vtscsi_thaw_simq(struct vtscsi_softc *, int); static void vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t, lun_id_t); static void vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t, lun_id_t); static void vtscsi_execute_rescan_bus(struct vtscsi_softc *); static void vtscsi_handle_event(struct vtscsi_softc *, struct virtio_scsi_event *); static int vtscsi_enqueue_event_buf(struct vtscsi_softc *, struct virtio_scsi_event *); static int vtscsi_init_event_vq(struct vtscsi_softc *); static void vtscsi_reinit_event_vq(struct vtscsi_softc *); static void vtscsi_drain_event_vq(struct vtscsi_softc *); static void vtscsi_complete_vqs_locked(struct vtscsi_softc *); static void vtscsi_complete_vqs(struct vtscsi_softc *); static void vtscsi_drain_vqs(struct vtscsi_softc *); static void vtscsi_cancel_request(struct vtscsi_softc *, struct vtscsi_request *); static void vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *); static void vtscsi_stop(struct vtscsi_softc *); static int vtscsi_reset_bus(struct vtscsi_softc *); static void vtscsi_init_request(struct vtscsi_softc *, struct vtscsi_request *); static int vtscsi_alloc_requests(struct vtscsi_softc *); static void vtscsi_free_requests(struct vtscsi_softc *); static void vtscsi_enqueue_request(struct vtscsi_softc *, struct vtscsi_request *); static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *); static void vtscsi_complete_request(struct vtscsi_request *); static void vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *); static void vtscsi_control_vq_intr(void *); static void vtscsi_event_vq_intr(void *); static void vtscsi_request_vq_intr(void *); static void vtscsi_disable_vqs_intr(struct vtscsi_softc *); static void vtscsi_enable_vqs_intr(struct vtscsi_softc *); static void vtscsi_get_tunables(struct vtscsi_softc *); static void vtscsi_setup_sysctl(struct vtscsi_softc *); static void vtscsi_printf_req(struct vtscsi_request *, const char *, const char *, ...); #define vtscsi_modern(_sc) (((_sc)->vtscsi_features & VIRTIO_F_VERSION_1) != 0) #define vtscsi_htog16(_sc, _val) virtio_htog16(vtscsi_modern(_sc), _val) #define vtscsi_htog32(_sc, _val) virtio_htog32(vtscsi_modern(_sc), _val) #define vtscsi_htog64(_sc, _val) virtio_htog64(vtscsi_modern(_sc), _val) #define vtscsi_gtoh16(_sc, _val) virtio_gtoh16(vtscsi_modern(_sc), _val) #define vtscsi_gtoh32(_sc, _val) virtio_gtoh32(vtscsi_modern(_sc), _val) #define vtscsi_gtoh64(_sc, _val) virtio_gtoh64(vtscsi_modern(_sc), _val) /* Global tunables. */ /* * The current QEMU VirtIO SCSI implementation does not cancel in-flight * IO during virtio_stop(). So in-flight requests still complete after the * device reset. We would have to wait for all the in-flight IO to complete, * which defeats the typical purpose of a bus reset. We could simulate the * bus reset with either I_T_NEXUS_RESET of all the targets, or with * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the * control virtqueue). But this isn't very useful if things really go off * the rails, so default to disabled for now. */ static int vtscsi_bus_reset_disable = 1; TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable); static struct virtio_feature_desc vtscsi_feature_desc[] = { { VIRTIO_SCSI_F_INOUT, "InOut" }, { VIRTIO_SCSI_F_HOTPLUG, "Hotplug" }, { VIRTIO_SCSI_F_CHANGE, "ChangeEvent" }, { VIRTIO_SCSI_F_T10_PI, "T10PI" }, { 0, NULL } }; static device_method_t vtscsi_methods[] = { /* Device methods. */ DEVMETHOD(device_probe, vtscsi_probe), DEVMETHOD(device_attach, vtscsi_attach), DEVMETHOD(device_detach, vtscsi_detach), DEVMETHOD(device_suspend, vtscsi_suspend), DEVMETHOD(device_resume, vtscsi_resume), DEVMETHOD_END }; static driver_t vtscsi_driver = { "vtscsi", vtscsi_methods, sizeof(struct vtscsi_softc) }; VIRTIO_DRIVER_MODULE(virtio_scsi, vtscsi_driver, vtscsi_modevent, NULL); MODULE_VERSION(virtio_scsi, 1); MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1); MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1); VIRTIO_SIMPLE_PNPINFO(virtio_scsi, VIRTIO_ID_SCSI, "VirtIO SCSI Adapter"); static int vtscsi_modevent(module_t mod, int type, void *unused) { int error; switch (type) { case MOD_LOAD: case MOD_QUIESCE: case MOD_UNLOAD: case MOD_SHUTDOWN: error = 0; break; default: error = EOPNOTSUPP; break; } return (error); } static int vtscsi_probe(device_t dev) { return (VIRTIO_SIMPLE_PROBE(dev, virtio_scsi)); } static int vtscsi_attach(device_t dev) { struct vtscsi_softc *sc; struct virtio_scsi_config scsicfg; int error; sc = device_get_softc(dev); sc->vtscsi_dev = dev; virtio_set_feature_desc(dev, vtscsi_feature_desc); VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev)); TAILQ_INIT(&sc->vtscsi_req_free); vtscsi_get_tunables(sc); vtscsi_setup_sysctl(sc); error = vtscsi_setup_features(sc); if (error) { device_printf(dev, "cannot setup features\n"); goto fail; } vtscsi_read_config(sc, &scsicfg); sc->vtscsi_max_channel = scsicfg.max_channel; sc->vtscsi_max_target = scsicfg.max_target; sc->vtscsi_max_lun = scsicfg.max_lun; sc->vtscsi_event_buf_size = scsicfg.event_info_size; vtscsi_write_device_config(sc); sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max); sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT); if (sc->vtscsi_sglist == NULL) { error = ENOMEM; device_printf(dev, "cannot allocate sglist\n"); goto fail; } error = vtscsi_alloc_virtqueues(sc); if (error) { device_printf(dev, "cannot allocate virtqueues\n"); goto fail; } vtscsi_check_sizes(sc); error = vtscsi_init_event_vq(sc); if (error) { device_printf(dev, "cannot populate the eventvq\n"); goto fail; } error = vtscsi_alloc_requests(sc); if (error) { device_printf(dev, "cannot allocate requests\n"); goto fail; } error = vtscsi_alloc_cam(sc); if (error) { device_printf(dev, "cannot allocate CAM structures\n"); goto fail; } error = virtio_setup_intr(dev, INTR_TYPE_CAM); if (error) { device_printf(dev, "cannot setup virtqueue interrupts\n"); goto fail; } vtscsi_enable_vqs_intr(sc); /* * Register with CAM after interrupts are enabled so we will get * notified of the probe responses. */ error = vtscsi_register_cam(sc); if (error) { device_printf(dev, "cannot register with CAM\n"); goto fail; } fail: if (error) vtscsi_detach(dev); return (error); } static int vtscsi_detach(device_t dev) { struct vtscsi_softc *sc; sc = device_get_softc(dev); VTSCSI_LOCK(sc); sc->vtscsi_flags |= VTSCSI_FLAG_DETACH; if (device_is_attached(dev)) vtscsi_stop(sc); VTSCSI_UNLOCK(sc); vtscsi_complete_vqs(sc); vtscsi_drain_vqs(sc); vtscsi_free_cam(sc); vtscsi_free_requests(sc); if (sc->vtscsi_sglist != NULL) { sglist_free(sc->vtscsi_sglist); sc->vtscsi_sglist = NULL; } VTSCSI_LOCK_DESTROY(sc); return (0); } static int vtscsi_suspend(device_t dev) { return (0); } static int vtscsi_resume(device_t dev) { return (0); } static int vtscsi_negotiate_features(struct vtscsi_softc *sc) { device_t dev; uint64_t features; dev = sc->vtscsi_dev; features = VTSCSI_FEATURES; sc->vtscsi_features = virtio_negotiate_features(dev, features); return (virtio_finalize_features(dev)); } static int vtscsi_setup_features(struct vtscsi_softc *sc) { device_t dev; int error; dev = sc->vtscsi_dev; error = vtscsi_negotiate_features(sc); if (error) return (error); if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT; if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT)) sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL; if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG)) sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG; return (0); } #define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \ virtio_read_device_config(_dev, \ offsetof(struct virtio_scsi_config, _field), \ &(_cfg)->_field, sizeof((_cfg)->_field)) \ static void vtscsi_read_config(struct vtscsi_softc *sc, struct virtio_scsi_config *scsicfg) { device_t dev; dev = sc->vtscsi_dev; bzero(scsicfg, sizeof(struct virtio_scsi_config)); VTSCSI_GET_CONFIG(dev, num_queues, scsicfg); VTSCSI_GET_CONFIG(dev, seg_max, scsicfg); VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg); VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg); VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg); VTSCSI_GET_CONFIG(dev, sense_size, scsicfg); VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg); VTSCSI_GET_CONFIG(dev, max_channel, scsicfg); VTSCSI_GET_CONFIG(dev, max_target, scsicfg); VTSCSI_GET_CONFIG(dev, max_lun, scsicfg); } #undef VTSCSI_GET_CONFIG static int vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max) { int nsegs; nsegs = VTSCSI_MIN_SEGMENTS; if (seg_max > 0) { nsegs += MIN(seg_max, maxphys / PAGE_SIZE + 1); if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); } else nsegs += 1; return (nsegs); } static int vtscsi_alloc_virtqueues(struct vtscsi_softc *sc) { device_t dev; struct vq_alloc_info vq_info[3]; int nvqs; dev = sc->vtscsi_dev; nvqs = 3; VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtscsi_control_vq_intr, sc, &sc->vtscsi_control_vq, "%s control", device_get_nameunit(dev)); VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtscsi_event_vq_intr, sc, &sc->vtscsi_event_vq, "%s event", device_get_nameunit(dev)); VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs, vtscsi_request_vq_intr, sc, &sc->vtscsi_request_vq, "%s request", device_get_nameunit(dev)); - return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); + return (virtio_alloc_virtqueues(dev, nvqs, vq_info)); } static void vtscsi_check_sizes(struct vtscsi_softc *sc) { int rqsize; if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) { /* * Ensure the assertions in virtqueue_enqueue(), * even if the hypervisor reports a bad seg_max. */ rqsize = virtqueue_size(sc->vtscsi_request_vq); if (sc->vtscsi_max_nsegs > rqsize) { device_printf(sc->vtscsi_dev, "clamping seg_max (%d %d)\n", sc->vtscsi_max_nsegs, rqsize); sc->vtscsi_max_nsegs = rqsize; } } } static void vtscsi_write_device_config(struct vtscsi_softc *sc) { virtio_write_dev_config_4(sc->vtscsi_dev, offsetof(struct virtio_scsi_config, sense_size), VIRTIO_SCSI_SENSE_SIZE); /* * This is the size in the virtio_scsi_cmd_req structure. Note * this value (32) is larger than the maximum CAM CDB size (16). */ virtio_write_dev_config_4(sc->vtscsi_dev, offsetof(struct virtio_scsi_config, cdb_size), VIRTIO_SCSI_CDB_SIZE); } static int vtscsi_reinit(struct vtscsi_softc *sc) { device_t dev; int error; dev = sc->vtscsi_dev; error = virtio_reinit(dev, sc->vtscsi_features); if (error == 0) { vtscsi_write_device_config(sc); virtio_reinit_complete(dev); vtscsi_reinit_event_vq(sc); vtscsi_enable_vqs_intr(sc); } vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error); return (error); } static int vtscsi_alloc_cam(struct vtscsi_softc *sc) { device_t dev; struct cam_devq *devq; int openings; dev = sc->vtscsi_dev; openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS; devq = cam_simq_alloc(openings); if (devq == NULL) { device_printf(dev, "cannot allocate SIM queue\n"); return (ENOMEM); } sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll, "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1, openings, devq); if (sc->vtscsi_sim == NULL) { cam_simq_free(devq); device_printf(dev, "cannot allocate SIM\n"); return (ENOMEM); } return (0); } static int vtscsi_register_cam(struct vtscsi_softc *sc) { device_t dev; int registered, error; dev = sc->vtscsi_dev; registered = 0; VTSCSI_LOCK(sc); if (xpt_bus_register(sc->vtscsi_sim, dev, 0) != CAM_SUCCESS) { error = ENOMEM; device_printf(dev, "cannot register XPT bus\n"); goto fail; } registered = 1; if (xpt_create_path(&sc->vtscsi_path, NULL, cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { error = ENOMEM; device_printf(dev, "cannot create bus path\n"); goto fail; } if (vtscsi_register_async(sc) != CAM_REQ_CMP) { error = EIO; device_printf(dev, "cannot register async callback\n"); goto fail; } VTSCSI_UNLOCK(sc); return (0); fail: if (sc->vtscsi_path != NULL) { xpt_free_path(sc->vtscsi_path); sc->vtscsi_path = NULL; } if (registered != 0) xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); VTSCSI_UNLOCK(sc); return (error); } static void vtscsi_free_cam(struct vtscsi_softc *sc) { VTSCSI_LOCK(sc); if (sc->vtscsi_path != NULL) { vtscsi_deregister_async(sc); xpt_free_path(sc->vtscsi_path); sc->vtscsi_path = NULL; xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); } if (sc->vtscsi_sim != NULL) { cam_sim_free(sc->vtscsi_sim, 1); sc->vtscsi_sim = NULL; } VTSCSI_UNLOCK(sc); } static void vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg) { struct cam_sim *sim; struct vtscsi_softc *sc; sim = cb_arg; sc = cam_sim_softc(sim); vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code); /* * TODO Once QEMU supports event reporting, we should * (un)subscribe to events here. */ switch (code) { case AC_FOUND_DEVICE: break; case AC_LOST_DEVICE: break; } } static int vtscsi_register_async(struct vtscsi_softc *sc) { struct ccb_setasync csa; memset(&csa, 0, sizeof(csa)); xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); csa.ccb_h.func_code = XPT_SASYNC_CB; csa.event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE; csa.callback = vtscsi_cam_async; csa.callback_arg = sc->vtscsi_sim; xpt_action((union ccb *) &csa); return (csa.ccb_h.status); } static void vtscsi_deregister_async(struct vtscsi_softc *sc) { struct ccb_setasync csa; memset(&csa, 0, sizeof(csa)); xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); csa.ccb_h.func_code = XPT_SASYNC_CB; csa.event_enable = 0; csa.callback = vtscsi_cam_async; csa.callback_arg = sc->vtscsi_sim; xpt_action((union ccb *) &csa); } static void vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb) { struct vtscsi_softc *sc; struct ccb_hdr *ccbh; sc = cam_sim_softc(sim); ccbh = &ccb->ccb_h; VTSCSI_LOCK_OWNED(sc); if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) { /* * The VTSCSI_MTX is briefly dropped between setting * VTSCSI_FLAG_DETACH and deregistering with CAM, so * drop any CCBs that come in during that window. */ ccbh->status = CAM_NO_HBA; xpt_done(ccb); return; } switch (ccbh->func_code) { case XPT_SCSI_IO: vtscsi_cam_scsi_io(sc, sim, ccb); break; case XPT_SET_TRAN_SETTINGS: ccbh->status = CAM_FUNC_NOTAVAIL; xpt_done(ccb); break; case XPT_GET_TRAN_SETTINGS: vtscsi_cam_get_tran_settings(sc, ccb); break; case XPT_RESET_BUS: vtscsi_cam_reset_bus(sc, ccb); break; case XPT_RESET_DEV: vtscsi_cam_reset_dev(sc, ccb); break; case XPT_ABORT: vtscsi_cam_abort(sc, ccb); break; case XPT_CALC_GEOMETRY: cam_calc_geometry(&ccb->ccg, 1); xpt_done(ccb); break; case XPT_PATH_INQ: vtscsi_cam_path_inquiry(sc, sim, ccb); break; default: vtscsi_dprintf(sc, VTSCSI_ERROR, "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code); ccbh->status = CAM_REQ_INVALID; xpt_done(ccb); break; } } static void vtscsi_cam_poll(struct cam_sim *sim) { struct vtscsi_softc *sc; sc = cam_sim_softc(sim); vtscsi_complete_vqs_locked(sc); } static void vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim, union ccb *ccb) { struct ccb_hdr *ccbh; struct ccb_scsiio *csio; int error; ccbh = &ccb->ccb_h; csio = &ccb->csio; if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) { error = EINVAL; ccbh->status = CAM_REQ_INVALID; goto done; } if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH && (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) { error = EINVAL; ccbh->status = CAM_REQ_INVALID; goto done; } error = vtscsi_start_scsi_cmd(sc, ccb); done: if (error) { vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status); xpt_done(ccb); } } static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb) { struct ccb_trans_settings *cts; struct ccb_trans_settings_scsi *scsi; cts = &ccb->cts; scsi = &cts->proto_specific.scsi; cts->protocol = PROTO_SCSI; cts->protocol_version = SCSI_REV_SPC3; cts->transport = XPORT_SAS; cts->transport_version = 0; scsi->valid = CTS_SCSI_VALID_TQ; scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); } static void vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb) { int error; error = vtscsi_reset_bus(sc); if (error == 0) ccb->ccb_h.status = CAM_REQ_CMP; else ccb->ccb_h.status = CAM_REQ_CMP_ERR; vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n", error, ccb, ccb->ccb_h.status); xpt_done(ccb); } static void vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb) { struct ccb_hdr *ccbh; struct vtscsi_request *req; int error; ccbh = &ccb->ccb_h; req = vtscsi_dequeue_request(sc); if (req == NULL) { error = EAGAIN; vtscsi_freeze_simq(sc, VTSCSI_REQUEST); goto fail; } req->vsr_ccb = ccb; error = vtscsi_execute_reset_dev_cmd(sc, req); if (error == 0) return; vtscsi_enqueue_request(sc, req); fail: vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", error, req, ccb); if (error == EAGAIN) ccbh->status = CAM_RESRC_UNAVAIL; else ccbh->status = CAM_REQ_CMP_ERR; xpt_done(ccb); } static void vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb) { struct vtscsi_request *req; struct ccb_hdr *ccbh; int error; ccbh = &ccb->ccb_h; req = vtscsi_dequeue_request(sc); if (req == NULL) { error = EAGAIN; vtscsi_freeze_simq(sc, VTSCSI_REQUEST); goto fail; } req->vsr_ccb = ccb; error = vtscsi_execute_abort_task_cmd(sc, req); if (error == 0) return; vtscsi_enqueue_request(sc, req); fail: vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", error, req, ccb); if (error == EAGAIN) ccbh->status = CAM_RESRC_UNAVAIL; else ccbh->status = CAM_REQ_CMP_ERR; xpt_done(ccb); } static void vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim, union ccb *ccb) { device_t dev; struct ccb_pathinq *cpi; dev = sc->vtscsi_dev; cpi = &ccb->cpi; vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb); cpi->version_num = 1; cpi->hba_inquiry = PI_TAG_ABLE; cpi->target_sprt = 0; cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED; if (vtscsi_bus_reset_disable != 0) cpi->hba_misc |= PIM_NOBUSRESET; cpi->hba_eng_cnt = 0; cpi->max_target = sc->vtscsi_max_target; cpi->max_lun = sc->vtscsi_max_lun; cpi->initiator_id = cpi->max_target + 1; strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->bus_id = cam_sim_bus(sim); cpi->base_transfer_speed = 300000; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_SPC3; cpi->transport = XPORT_SAS; cpi->transport_version = 0; cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) * PAGE_SIZE; cpi->hba_vendor = virtio_get_vendor(dev); cpi->hba_device = virtio_get_device(dev); cpi->hba_subvendor = virtio_get_subvendor(dev); cpi->hba_subdevice = virtio_get_subdevice(dev); ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); } static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg, struct ccb_scsiio *csio) { struct ccb_hdr *ccbh; struct bus_dma_segment *dseg; int i, error; ccbh = &csio->ccb_h; error = 0; switch ((ccbh->flags & CAM_DATA_MASK)) { case CAM_DATA_VADDR: error = sglist_append(sg, csio->data_ptr, csio->dxfer_len); break; case CAM_DATA_PADDR: error = sglist_append_phys(sg, (vm_paddr_t)(vm_offset_t) csio->data_ptr, csio->dxfer_len); break; case CAM_DATA_SG: for (i = 0; i < csio->sglist_cnt && error == 0; i++) { dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; error = sglist_append(sg, (void *)(vm_offset_t) dseg->ds_addr, dseg->ds_len); } break; case CAM_DATA_SG_PADDR: for (i = 0; i < csio->sglist_cnt && error == 0; i++) { dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; error = sglist_append_phys(sg, (vm_paddr_t) dseg->ds_addr, dseg->ds_len); } break; case CAM_DATA_BIO: error = sglist_append_bio(sg, (struct bio *) csio->data_ptr); break; default: error = EINVAL; break; } return (error); } static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req, int *readable, int *writable) { struct sglist *sg; struct ccb_hdr *ccbh; struct ccb_scsiio *csio; struct virtio_scsi_cmd_req *cmd_req; struct virtio_scsi_cmd_resp *cmd_resp; int error; sg = sc->vtscsi_sglist; csio = &req->vsr_ccb->csio; ccbh = &csio->ccb_h; cmd_req = &req->vsr_cmd_req; cmd_resp = &req->vsr_cmd_resp; sglist_reset(sg); sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req)); if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) { error = vtscsi_sg_append_scsi_buf(sc, sg, csio); /* At least one segment must be left for the response. */ if (error || sg->sg_nseg == sg->sg_maxseg) goto fail; } *readable = sg->sg_nseg; sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp)); if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) { error = vtscsi_sg_append_scsi_buf(sc, sg, csio); if (error) goto fail; } *writable = sg->sg_nseg - *readable; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d " "writable=%d\n", req, ccbh, *readable, *writable); return (0); fail: /* * This should never happen unless maxio was incorrectly set. */ vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0); vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p " "nseg=%d maxseg=%d\n", error, req, ccbh, sg->sg_nseg, sg->sg_maxseg); return (EFBIG); } static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct sglist *sg; struct virtqueue *vq; struct ccb_scsiio *csio; struct ccb_hdr *ccbh; struct virtio_scsi_cmd_req *cmd_req; struct virtio_scsi_cmd_resp *cmd_resp; int readable, writable, error; sg = sc->vtscsi_sglist; vq = sc->vtscsi_request_vq; csio = &req->vsr_ccb->csio; ccbh = &csio->ccb_h; cmd_req = &req->vsr_cmd_req; cmd_resp = &req->vsr_cmd_resp; vtscsi_init_scsi_cmd_req(sc, csio, cmd_req); error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable); if (error) return (error); req->vsr_complete = vtscsi_complete_scsi_cmd; cmd_resp->response = -1; error = virtqueue_enqueue(vq, req, sg, readable, writable); if (error) { vtscsi_dprintf(sc, VTSCSI_ERROR, "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh); ccbh->status = CAM_REQUEUE_REQ; vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ); return (error); } ccbh->status |= CAM_SIM_QUEUED; ccbh->ccbh_vtscsi_req = req; virtqueue_notify(vq); if (ccbh->timeout != CAM_TIME_INFINITY) { req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET; callout_reset_sbt(&req->vsr_callout, SBT_1MS * ccbh->timeout, 0, vtscsi_timedout_scsi_cmd, req, 0); } vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n", req, ccbh); return (0); } static int vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb) { struct vtscsi_request *req; int error; req = vtscsi_dequeue_request(sc); if (req == NULL) { ccb->ccb_h.status = CAM_REQUEUE_REQ; vtscsi_freeze_simq(sc, VTSCSI_REQUEST); return (ENOBUFS); } req->vsr_ccb = ccb; error = vtscsi_execute_scsi_cmd(sc, req); if (error) vtscsi_enqueue_request(sc, req); return (error); } static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct virtio_scsi_ctrl_tmf_resp *tmf_resp; struct vtscsi_request *to_req; uint8_t response; tmf_resp = &req->vsr_tmf_resp; response = tmf_resp->response; to_req = req->vsr_timedout_req; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n", req, to_req, response); vtscsi_enqueue_request(sc, req); /* * The timedout request could have completed between when the * abort task was sent and when the host processed it. */ if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT) return; /* The timedout request was successfully aborted. */ if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) return; /* Don't bother if the device is going away. */ if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) return; /* The timedout request will be aborted by the reset. */ if (sc->vtscsi_flags & VTSCSI_FLAG_RESET) return; vtscsi_reset_bus(sc); } static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *to_req) { struct sglist *sg; struct ccb_hdr *to_ccbh; struct vtscsi_request *req; struct virtio_scsi_ctrl_tmf_req *tmf_req; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; int error; sg = sc->vtscsi_sglist; to_ccbh = &to_req->vsr_ccb->ccb_h; req = vtscsi_dequeue_request(sc); if (req == NULL) { error = ENOBUFS; goto fail; } tmf_req = &req->vsr_tmf_req; tmf_resp = &req->vsr_tmf_resp; vtscsi_init_ctrl_tmf_req(sc, to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, (uintptr_t) to_ccbh, tmf_req); sglist_reset(sg); sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); req->vsr_timedout_req = to_req; req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd; tmf_resp->response = -1; error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, VTSCSI_EXECUTE_ASYNC); if (error == 0) return (0); vtscsi_enqueue_request(sc, req); fail: vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p " "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh); return (error); } static void vtscsi_timedout_scsi_cmd(void *xreq) { struct vtscsi_softc *sc; struct vtscsi_request *to_req; to_req = xreq; sc = to_req->vsr_softc; vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n", to_req, to_req->vsr_ccb, to_req->vsr_state); /* Don't bother if the device is going away. */ if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) return; /* * Bail if the request is not in use. We likely raced when * stopping the callout handler or it has already been aborted. */ if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE || (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0) return; /* * Complete the request queue in case the timedout request is * actually just pending. */ vtscsi_complete_vq(sc, sc->vtscsi_request_vq); if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE) return; sc->vtscsi_stats.scsi_cmd_timeouts++; to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT; if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0) return; vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n"); vtscsi_reset_bus(sc); } static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp) { cam_status status; switch (cmd_resp->response) { case VIRTIO_SCSI_S_OK: status = CAM_REQ_CMP; break; case VIRTIO_SCSI_S_OVERRUN: status = CAM_DATA_RUN_ERR; break; case VIRTIO_SCSI_S_ABORTED: status = CAM_REQ_ABORTED; break; case VIRTIO_SCSI_S_BAD_TARGET: status = CAM_SEL_TIMEOUT; break; case VIRTIO_SCSI_S_RESET: status = CAM_SCSI_BUS_RESET; break; case VIRTIO_SCSI_S_BUSY: status = CAM_SCSI_BUSY; break; case VIRTIO_SCSI_S_TRANSPORT_FAILURE: case VIRTIO_SCSI_S_TARGET_FAILURE: case VIRTIO_SCSI_S_NEXUS_FAILURE: status = CAM_SCSI_IT_NEXUS_LOST; break; default: /* VIRTIO_SCSI_S_FAILURE */ status = CAM_REQ_CMP_ERR; break; } return (status); } static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc, struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp) { uint32_t resp_sense_length; cam_status status; csio->scsi_status = cmd_resp->status; csio->resid = vtscsi_htog32(sc, cmd_resp->resid); if (csio->scsi_status == SCSI_STATUS_OK) status = CAM_REQ_CMP; else status = CAM_SCSI_STATUS_ERROR; resp_sense_length = vtscsi_htog32(sc, cmd_resp->sense_len); if (resp_sense_length > 0) { status |= CAM_AUTOSNS_VALID; if (resp_sense_length < csio->sense_len) csio->sense_resid = csio->sense_len - resp_sense_length; else csio->sense_resid = 0; memcpy(&csio->sense_data, cmd_resp->sense, csio->sense_len - csio->sense_resid); } vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR, "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n", csio, csio->scsi_status, csio->resid, csio->sense_resid); return (status); } static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct ccb_hdr *ccbh; struct ccb_scsiio *csio; struct virtio_scsi_cmd_resp *cmd_resp; cam_status status; csio = &req->vsr_ccb->csio; ccbh = &csio->ccb_h; cmd_resp = &req->vsr_cmd_resp; KASSERT(ccbh->ccbh_vtscsi_req == req, ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req)); if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) callout_stop(&req->vsr_callout); status = vtscsi_scsi_cmd_cam_status(cmd_resp); if (status == CAM_REQ_ABORTED) { if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT) status = CAM_CMD_TIMEOUT; } else if (status == CAM_REQ_CMP) status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp); if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) { status |= CAM_DEV_QFRZN; xpt_freeze_devq(ccbh->path, 1); } if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) status |= CAM_RELEASE_SIMQ; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n", req, ccbh, status); ccbh->status = status; xpt_done(req->vsr_ccb); vtscsi_enqueue_request(sc, req); } static void vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req) { /* XXX We probably shouldn't poll forever. */ req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED; do vtscsi_complete_vq(sc, sc->vtscsi_control_vq); while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0); req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED; } static int vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req, struct sglist *sg, int readable, int writable, int flag) { struct virtqueue *vq; int error; vq = sc->vtscsi_control_vq; MPASS(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL); error = virtqueue_enqueue(vq, req, sg, readable, writable); if (error) { /* * Return EAGAIN when the virtqueue does not have enough * descriptors available. */ if (error == ENOSPC || error == EMSGSIZE) error = EAGAIN; return (error); } virtqueue_notify(vq); if (flag == VTSCSI_EXECUTE_POLL) vtscsi_poll_ctrl_req(sc, req); return (0); } static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { union ccb *ccb; struct ccb_hdr *ccbh; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; ccb = req->vsr_ccb; ccbh = &ccb->ccb_h; tmf_resp = &req->vsr_tmf_resp; switch (tmf_resp->response) { case VIRTIO_SCSI_S_FUNCTION_COMPLETE: ccbh->status = CAM_REQ_CMP; break; case VIRTIO_SCSI_S_FUNCTION_REJECTED: ccbh->status = CAM_UA_ABORT; break; default: ccbh->status = CAM_REQ_CMP_ERR; break; } xpt_done(ccb); vtscsi_enqueue_request(sc, req); } static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct sglist *sg; struct ccb_abort *cab; struct ccb_hdr *ccbh; struct ccb_hdr *abort_ccbh; struct vtscsi_request *abort_req; struct virtio_scsi_ctrl_tmf_req *tmf_req; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; int error; sg = sc->vtscsi_sglist; cab = &req->vsr_ccb->cab; ccbh = &cab->ccb_h; tmf_req = &req->vsr_tmf_req; tmf_resp = &req->vsr_tmf_resp; /* CCB header and request that's to be aborted. */ abort_ccbh = &cab->abort_ccb->ccb_h; abort_req = abort_ccbh->ccbh_vtscsi_req; if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) { error = EINVAL; goto fail; } /* Only attempt to abort requests that could be in-flight. */ if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) { error = EALREADY; goto fail; } abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED; if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) callout_stop(&abort_req->vsr_callout); vtscsi_init_ctrl_tmf_req(sc, ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, (uintptr_t) abort_ccbh, tmf_req); sglist_reset(sg); sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); req->vsr_complete = vtscsi_complete_abort_task_cmd; tmf_resp->response = -1; error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, VTSCSI_EXECUTE_ASYNC); fail: vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p " "abort_req=%p\n", error, req, abort_ccbh, abort_req); return (error); } static void vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { union ccb *ccb; struct ccb_hdr *ccbh; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; ccb = req->vsr_ccb; ccbh = &ccb->ccb_h; tmf_resp = &req->vsr_tmf_resp; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n", req, ccb, tmf_resp->response); if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) { ccbh->status = CAM_REQ_CMP; vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id, ccbh->target_lun); } else ccbh->status = CAM_REQ_CMP_ERR; xpt_done(ccb); vtscsi_enqueue_request(sc, req); } static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) { struct sglist *sg; struct ccb_resetdev *crd; struct ccb_hdr *ccbh; struct virtio_scsi_ctrl_tmf_req *tmf_req; struct virtio_scsi_ctrl_tmf_resp *tmf_resp; uint32_t subtype; int error; sg = sc->vtscsi_sglist; crd = &req->vsr_ccb->crd; ccbh = &crd->ccb_h; tmf_req = &req->vsr_tmf_req; tmf_resp = &req->vsr_tmf_resp; if (ccbh->target_lun == CAM_LUN_WILDCARD) subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET; else subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET; vtscsi_init_ctrl_tmf_req(sc, ccbh, subtype, 0, tmf_req); sglist_reset(sg); sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); req->vsr_complete = vtscsi_complete_reset_dev_cmd; tmf_resp->response = -1; error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, VTSCSI_EXECUTE_ASYNC); vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n", error, req, ccbh); return (error); } static void vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id) { *target_id = lun[1]; *lun_id = (lun[2] << 8) | lun[3]; } static void vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[]) { lun[0] = 1; lun[1] = ccbh->target_id; lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F); lun[3] = ccbh->target_lun & 0xFF; } static void vtscsi_init_scsi_cmd_req(struct vtscsi_softc *sc, struct ccb_scsiio *csio, struct virtio_scsi_cmd_req *cmd_req) { uint8_t attr; switch (csio->tag_action) { case MSG_HEAD_OF_Q_TAG: attr = VIRTIO_SCSI_S_HEAD; break; case MSG_ORDERED_Q_TAG: attr = VIRTIO_SCSI_S_ORDERED; break; case MSG_ACA_TASK: attr = VIRTIO_SCSI_S_ACA; break; default: /* MSG_SIMPLE_Q_TAG */ attr = VIRTIO_SCSI_S_SIMPLE; break; } vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun); cmd_req->tag = vtscsi_gtoh64(sc, (uintptr_t) csio); cmd_req->task_attr = attr; memcpy(cmd_req->cdb, csio->ccb_h.flags & CAM_CDB_POINTER ? csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes, csio->cdb_len); } static void vtscsi_init_ctrl_tmf_req(struct vtscsi_softc *sc, struct ccb_hdr *ccbh, uint32_t subtype, uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req) { vtscsi_set_request_lun(ccbh, tmf_req->lun); tmf_req->type = vtscsi_gtoh32(sc, VIRTIO_SCSI_T_TMF); tmf_req->subtype = vtscsi_gtoh32(sc, subtype); tmf_req->tag = vtscsi_gtoh64(sc, tag); } static void vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason) { int frozen; frozen = sc->vtscsi_frozen; if (reason & VTSCSI_REQUEST && (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0) sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS; if (reason & VTSCSI_REQUEST_VQ && (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0) sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL; /* Freeze the SIMQ if transitioned to frozen. */ if (frozen == 0 && sc->vtscsi_frozen != 0) { vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n"); xpt_freeze_simq(sc->vtscsi_sim, 1); } } static int vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason) { int thawed; if (sc->vtscsi_frozen == 0 || reason == 0) return (0); if (reason & VTSCSI_REQUEST && sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS; if (reason & VTSCSI_REQUEST_VQ && sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL; thawed = sc->vtscsi_frozen == 0; if (thawed != 0) vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n"); return (thawed); } static void vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code, target_id_t target_id, lun_id_t lun_id) { struct cam_path *path; /* Use the wildcard path from our softc for bus announcements. */ if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) { xpt_async(ac_code, sc->vtscsi_path, NULL); return; } if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim), target_id, lun_id) != CAM_REQ_CMP) { vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n"); return; } xpt_async(ac_code, path, NULL); xpt_free_path(path); } static void vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id, lun_id_t lun_id) { union ccb *ccb; cam_status status; ccb = xpt_alloc_ccb_nowait(); if (ccb == NULL) { vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n"); return; } status = xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(sc->vtscsi_sim), target_id, lun_id); if (status != CAM_REQ_CMP) { xpt_free_ccb(ccb); return; } xpt_rescan(ccb); } static void vtscsi_execute_rescan_bus(struct vtscsi_softc *sc) { vtscsi_execute_rescan(sc, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); } static void vtscsi_transport_reset_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event) { target_id_t target_id; lun_id_t lun_id; vtscsi_get_request_lun(event->lun, &target_id, &lun_id); switch (event->reason) { case VIRTIO_SCSI_EVT_RESET_RESCAN: case VIRTIO_SCSI_EVT_RESET_REMOVED: vtscsi_execute_rescan(sc, target_id, lun_id); break; default: device_printf(sc->vtscsi_dev, "unhandled transport event reason: %d\n", event->reason); break; } } static void vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event) { int error __diagused; if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) { switch (event->event) { case VIRTIO_SCSI_T_TRANSPORT_RESET: vtscsi_transport_reset_event(sc, event); break; default: device_printf(sc->vtscsi_dev, "unhandled event: %d\n", event->event); break; } } else vtscsi_execute_rescan_bus(sc); /* * This should always be successful since the buffer * was just dequeued. */ error = vtscsi_enqueue_event_buf(sc, event); KASSERT(error == 0, ("cannot requeue event buffer: %d", error)); } static int vtscsi_enqueue_event_buf(struct vtscsi_softc *sc, struct virtio_scsi_event *event) { struct sglist *sg; struct virtqueue *vq; int size, error; sg = sc->vtscsi_sglist; vq = sc->vtscsi_event_vq; size = sc->vtscsi_event_buf_size; bzero(event, size); sglist_reset(sg); error = sglist_append(sg, event, size); if (error) return (error); error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg); if (error) return (error); virtqueue_notify(vq); return (0); } static int vtscsi_init_event_vq(struct vtscsi_softc *sc) { struct virtio_scsi_event *event; int i, size, error; /* * The first release of QEMU with VirtIO SCSI support would crash * when attempting to notify the event virtqueue. This was fixed * when hotplug support was added. */ if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) size = sc->vtscsi_event_buf_size; else size = 0; if (size < sizeof(struct virtio_scsi_event)) return (0); for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { event = &sc->vtscsi_event_bufs[i]; error = vtscsi_enqueue_event_buf(sc, event); if (error) break; } /* * Even just one buffer is enough. Missed events are * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag. */ if (i > 0) error = 0; return (error); } static void vtscsi_reinit_event_vq(struct vtscsi_softc *sc) { struct virtio_scsi_event *event; int i, error; if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 || sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event)) return; for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { event = &sc->vtscsi_event_bufs[i]; error = vtscsi_enqueue_event_buf(sc, event); if (error) break; } KASSERT(i > 0, ("cannot reinit event vq: %d", error)); } static void vtscsi_drain_event_vq(struct vtscsi_softc *sc) { struct virtqueue *vq; int last; vq = sc->vtscsi_event_vq; last = 0; while (virtqueue_drain(vq, &last) != NULL) ; KASSERT(virtqueue_empty(vq), ("eventvq not empty")); } static void vtscsi_complete_vqs_locked(struct vtscsi_softc *sc) { VTSCSI_LOCK_OWNED(sc); if (sc->vtscsi_request_vq != NULL) vtscsi_complete_vq(sc, sc->vtscsi_request_vq); if (sc->vtscsi_control_vq != NULL) vtscsi_complete_vq(sc, sc->vtscsi_control_vq); } static void vtscsi_complete_vqs(struct vtscsi_softc *sc) { VTSCSI_LOCK(sc); vtscsi_complete_vqs_locked(sc); VTSCSI_UNLOCK(sc); } static void vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req) { union ccb *ccb; int detach; ccb = req->vsr_ccb; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb); /* * The callout must be drained when detaching since the request is * about to be freed. The VTSCSI_MTX must not be held for this in * case the callout is pending because there is a deadlock potential. * Otherwise, the virtqueue is being drained because of a bus reset * so we only need to attempt to stop the callouts. */ detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0; if (detach != 0) VTSCSI_LOCK_NOTOWNED(sc); else VTSCSI_LOCK_OWNED(sc); if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) { if (detach != 0) callout_drain(&req->vsr_callout); else callout_stop(&req->vsr_callout); } if (ccb != NULL) { if (detach != 0) { VTSCSI_LOCK(sc); ccb->ccb_h.status = CAM_NO_HBA; } else ccb->ccb_h.status = CAM_REQUEUE_REQ; xpt_done(ccb); if (detach != 0) VTSCSI_UNLOCK(sc); } vtscsi_enqueue_request(sc, req); } static void vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq) { struct vtscsi_request *req; int last; last = 0; vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq); while ((req = virtqueue_drain(vq, &last)) != NULL) vtscsi_cancel_request(sc, req); KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); } static void vtscsi_drain_vqs(struct vtscsi_softc *sc) { if (sc->vtscsi_control_vq != NULL) vtscsi_drain_vq(sc, sc->vtscsi_control_vq); if (sc->vtscsi_request_vq != NULL) vtscsi_drain_vq(sc, sc->vtscsi_request_vq); if (sc->vtscsi_event_vq != NULL) vtscsi_drain_event_vq(sc); } static void vtscsi_stop(struct vtscsi_softc *sc) { vtscsi_disable_vqs_intr(sc); virtio_stop(sc->vtscsi_dev); } static int vtscsi_reset_bus(struct vtscsi_softc *sc) { int error; VTSCSI_LOCK_OWNED(sc); if (vtscsi_bus_reset_disable != 0) { device_printf(sc->vtscsi_dev, "bus reset disabled\n"); return (0); } sc->vtscsi_flags |= VTSCSI_FLAG_RESET; /* * vtscsi_stop() will cause the in-flight requests to be canceled. * Those requests are then completed here so CAM will retry them * after the reset is complete. */ vtscsi_stop(sc); vtscsi_complete_vqs_locked(sc); /* Rid the virtqueues of any remaining requests. */ vtscsi_drain_vqs(sc); /* * Any resource shortage that froze the SIMQ cannot persist across * a bus reset so ensure it gets thawed here. */ if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) xpt_release_simq(sc->vtscsi_sim, 0); error = vtscsi_reinit(sc); if (error) { device_printf(sc->vtscsi_dev, "reinitialization failed, stopping device...\n"); vtscsi_stop(sc); } else vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET; return (error); } static void vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req) { #ifdef INVARIANTS int req_nsegs, resp_nsegs; req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq)); resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp)); KASSERT(req_nsegs == 1, ("request crossed page boundary")); KASSERT(resp_nsegs == 1, ("response crossed page boundary")); #endif req->vsr_softc = sc; callout_init_mtx(&req->vsr_callout, VTSCSI_MTX(sc), 0); } static int vtscsi_alloc_requests(struct vtscsi_softc *sc) { struct vtscsi_request *req; int i, nreqs; /* * Commands destined for either the request or control queues come * from the same SIM queue. Use the size of the request virtqueue * as it (should) be much more frequently used. Some additional * requests are allocated for internal (TMF) use. */ nreqs = virtqueue_size(sc->vtscsi_request_vq); if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) nreqs /= VTSCSI_MIN_SEGMENTS; nreqs += VTSCSI_RESERVED_REQUESTS; for (i = 0; i < nreqs; i++) { req = malloc(sizeof(struct vtscsi_request), M_DEVBUF, M_NOWAIT); if (req == NULL) return (ENOMEM); vtscsi_init_request(sc, req); sc->vtscsi_nrequests++; vtscsi_enqueue_request(sc, req); } return (0); } static void vtscsi_free_requests(struct vtscsi_softc *sc) { struct vtscsi_request *req; while ((req = vtscsi_dequeue_request(sc)) != NULL) { KASSERT(callout_active(&req->vsr_callout) == 0, ("request callout still active")); sc->vtscsi_nrequests--; free(req, M_DEVBUF); } KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d", sc->vtscsi_nrequests)); } static void vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req) { KASSERT(req->vsr_softc == sc, ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc)); vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); /* A request is available so the SIMQ could be released. */ if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0) xpt_release_simq(sc->vtscsi_sim, 1); req->vsr_ccb = NULL; req->vsr_complete = NULL; req->vsr_ptr0 = NULL; req->vsr_state = VTSCSI_REQ_STATE_FREE; req->vsr_flags = 0; bzero(&req->vsr_ureq, sizeof(req->vsr_ureq)); bzero(&req->vsr_uresp, sizeof(req->vsr_uresp)); /* * We insert at the tail of the queue in order to make it * very unlikely a request will be reused if we race with * stopping its callout handler. */ TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link); } static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *sc) { struct vtscsi_request *req; req = TAILQ_FIRST(&sc->vtscsi_req_free); if (req != NULL) { req->vsr_state = VTSCSI_REQ_STATE_INUSE; TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link); } else sc->vtscsi_stats.dequeue_no_requests++; vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); return (req); } static void vtscsi_complete_request(struct vtscsi_request *req) { if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED) req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE; if (req->vsr_complete != NULL) req->vsr_complete(req->vsr_softc, req); } static void vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq) { struct vtscsi_request *req; VTSCSI_LOCK_OWNED(sc); while ((req = virtqueue_dequeue(vq, NULL)) != NULL) vtscsi_complete_request(req); } static void vtscsi_control_vq_intr(void *xsc) { struct vtscsi_softc *sc; struct virtqueue *vq; sc = xsc; vq = sc->vtscsi_control_vq; again: VTSCSI_LOCK(sc); vtscsi_complete_vq(sc, sc->vtscsi_control_vq); if (virtqueue_enable_intr(vq) != 0) { virtqueue_disable_intr(vq); VTSCSI_UNLOCK(sc); goto again; } VTSCSI_UNLOCK(sc); } static void vtscsi_event_vq_intr(void *xsc) { struct vtscsi_softc *sc; struct virtqueue *vq; struct virtio_scsi_event *event; sc = xsc; vq = sc->vtscsi_event_vq; again: VTSCSI_LOCK(sc); while ((event = virtqueue_dequeue(vq, NULL)) != NULL) vtscsi_handle_event(sc, event); if (virtqueue_enable_intr(vq) != 0) { virtqueue_disable_intr(vq); VTSCSI_UNLOCK(sc); goto again; } VTSCSI_UNLOCK(sc); } static void vtscsi_request_vq_intr(void *xsc) { struct vtscsi_softc *sc; struct virtqueue *vq; sc = xsc; vq = sc->vtscsi_request_vq; again: VTSCSI_LOCK(sc); vtscsi_complete_vq(sc, sc->vtscsi_request_vq); if (virtqueue_enable_intr(vq) != 0) { virtqueue_disable_intr(vq); VTSCSI_UNLOCK(sc); goto again; } VTSCSI_UNLOCK(sc); } static void vtscsi_disable_vqs_intr(struct vtscsi_softc *sc) { virtqueue_disable_intr(sc->vtscsi_control_vq); virtqueue_disable_intr(sc->vtscsi_event_vq); virtqueue_disable_intr(sc->vtscsi_request_vq); } static void vtscsi_enable_vqs_intr(struct vtscsi_softc *sc) { virtqueue_enable_intr(sc->vtscsi_control_vq); virtqueue_enable_intr(sc->vtscsi_event_vq); virtqueue_enable_intr(sc->vtscsi_request_vq); } static void vtscsi_get_tunables(struct vtscsi_softc *sc) { char tmpstr[64]; TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug); snprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level", device_get_unit(sc->vtscsi_dev)); TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug); } static void vtscsi_setup_sysctl(struct vtscsi_softc *sc) { device_t dev; struct vtscsi_statistics *stats; struct sysctl_ctx_list *ctx; struct sysctl_oid *tree; struct sysctl_oid_list *child; dev = sc->vtscsi_dev; stats = &sc->vtscsi_stats; ctx = device_get_sysctl_ctx(dev); tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level", CTLFLAG_RW, &sc->vtscsi_debug, 0, "Debug level"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts", CTLFLAG_RD, &stats->scsi_cmd_timeouts, "SCSI command timeouts"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests", CTLFLAG_RD, &stats->dequeue_no_requests, "No available requests to dequeue"); } static void vtscsi_printf_req(struct vtscsi_request *req, const char *func, const char *fmt, ...) { struct vtscsi_softc *sc; union ccb *ccb; struct sbuf sb; va_list ap; char str[192]; char path_str[64]; if (req == NULL) return; sc = req->vsr_softc; ccb = req->vsr_ccb; va_start(ap, fmt); sbuf_new(&sb, str, sizeof(str), 0); if (ccb == NULL) { sbuf_printf(&sb, "(noperiph:%s%d:%u): ", cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim), cam_sim_bus(sc->vtscsi_sim)); } else { xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str)); sbuf_cat(&sb, path_str); if (ccb->ccb_h.func_code == XPT_SCSI_IO) { scsi_command_string(&ccb->csio, &sb); sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len); } } sbuf_vprintf(&sb, fmt, ap); va_end(ap); sbuf_finish(&sb); printf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func, sbuf_data(&sb)); } diff --git a/sys/dev/virtio/virtio.c b/sys/dev/virtio/virtio.c index 45a657542e28..8cfb045183b8 100644 --- a/sys/dev/virtio/virtio.c +++ b/sys/dev/virtio/virtio.c @@ -1,379 +1,378 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "virtio_bus_if.h" static int virtio_modevent(module_t, int, void *); static const char *virtio_feature_name(uint64_t, struct virtio_feature_desc *); static struct virtio_ident { uint16_t devid; const char *name; } virtio_ident_table[] = { { VIRTIO_ID_NETWORK, "Network" }, { VIRTIO_ID_BLOCK, "Block" }, { VIRTIO_ID_CONSOLE, "Console" }, { VIRTIO_ID_ENTROPY, "Entropy" }, { VIRTIO_ID_BALLOON, "Balloon" }, { VIRTIO_ID_IOMEMORY, "IOMemory" }, { VIRTIO_ID_RPMSG, "Remote Processor Messaging" }, { VIRTIO_ID_SCSI, "SCSI" }, { VIRTIO_ID_9P, "9P Transport" }, { VIRTIO_ID_RPROC_SERIAL, "Remote Processor Serial" }, { VIRTIO_ID_CAIF, "CAIF" }, { VIRTIO_ID_GPU, "GPU" }, { VIRTIO_ID_INPUT, "Input" }, { VIRTIO_ID_VSOCK, "VSOCK Transport" }, { VIRTIO_ID_CRYPTO, "Crypto" }, { 0, NULL } }; /* Device independent features. */ static struct virtio_feature_desc virtio_common_feature_desc[] = { { VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" }, /* Legacy */ { VIRTIO_F_ANY_LAYOUT, "AnyLayout" }, /* Legacy */ { VIRTIO_RING_F_INDIRECT_DESC, "RingIndirectDesc" }, { VIRTIO_RING_F_EVENT_IDX, "RingEventIdx" }, { VIRTIO_F_BAD_FEATURE, "BadFeature" }, /* Legacy */ { VIRTIO_F_VERSION_1, "Version1" }, { VIRTIO_F_IOMMU_PLATFORM, "IOMMUPlatform" }, { 0, NULL } }; const char * virtio_device_name(uint16_t devid) { struct virtio_ident *ident; for (ident = virtio_ident_table; ident->name != NULL; ident++) { if (ident->devid == devid) return (ident->name); } return (NULL); } static const char * virtio_feature_name(uint64_t val, struct virtio_feature_desc *desc) { int i, j; struct virtio_feature_desc *descs[2] = { desc, virtio_common_feature_desc }; for (i = 0; i < 2; i++) { if (descs[i] == NULL) continue; for (j = 0; descs[i][j].vfd_val != 0; j++) { if (val == descs[i][j].vfd_val) return (descs[i][j].vfd_str); } } return (NULL); } int virtio_describe_sbuf(struct sbuf *sb, uint64_t features, struct virtio_feature_desc *desc) { const char *name; uint64_t val; int n; sbuf_printf(sb, "%#jx", (uintmax_t) features); for (n = 0, val = 1ULL << 63; val != 0; val >>= 1) { /* * BAD_FEATURE is used to detect broken Linux clients * and therefore is not applicable to FreeBSD. */ if (((features & val) == 0) || val == VIRTIO_F_BAD_FEATURE) continue; if (n++ == 0) sbuf_cat(sb, " <"); else sbuf_cat(sb, ","); name = virtio_feature_name(val, desc); if (name == NULL) sbuf_printf(sb, "%#jx", (uintmax_t) val); else sbuf_cat(sb, name); } if (n > 0) sbuf_cat(sb, ">"); return (sbuf_finish(sb)); } void virtio_describe(device_t dev, const char *msg, uint64_t features, struct virtio_feature_desc *desc) { struct sbuf sb; char *buf; int error; if ((buf = malloc(1024, M_TEMP, M_NOWAIT)) == NULL) { error = ENOMEM; goto out; } sbuf_new(&sb, buf, 1024, SBUF_FIXEDLEN); sbuf_printf(&sb, "%s features: ", msg); error = virtio_describe_sbuf(&sb, features, desc); if (error == 0) device_printf(dev, "%s\n", sbuf_data(&sb)); sbuf_delete(&sb); free(buf, M_TEMP); out: if (error != 0) { device_printf(dev, "%s features: %#jx\n", msg, (uintmax_t) features); } } uint64_t virtio_filter_transport_features(uint64_t features) { uint64_t transport, mask; transport = (1ULL << (VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START)) - 1; transport <<= VIRTIO_TRANSPORT_F_START; mask = -1ULL & ~transport; mask |= VIRTIO_RING_F_INDIRECT_DESC; mask |= VIRTIO_RING_F_EVENT_IDX; mask |= VIRTIO_F_VERSION_1; return (features & mask); } bool virtio_bus_is_modern(device_t dev) { uintptr_t modern; virtio_read_ivar(dev, VIRTIO_IVAR_MODERN, &modern); return (modern != 0); } void virtio_read_device_config_array(device_t dev, bus_size_t offset, void *dst, int size, int count) { int i, gen; do { gen = virtio_config_generation(dev); for (i = 0; i < count; i++) { virtio_read_device_config(dev, offset + i * size, (uint8_t *) dst + i * size, size); } } while (gen != virtio_config_generation(dev)); } /* * VirtIO bus method wrappers. */ void virtio_read_ivar(device_t dev, int ivar, uintptr_t *val) { *val = -1; BUS_READ_IVAR(device_get_parent(dev), dev, ivar, val); } void virtio_write_ivar(device_t dev, int ivar, uintptr_t val) { BUS_WRITE_IVAR(device_get_parent(dev), dev, ivar, val); } uint64_t virtio_negotiate_features(device_t dev, uint64_t child_features) { return (VIRTIO_BUS_NEGOTIATE_FEATURES(device_get_parent(dev), child_features)); } int virtio_finalize_features(device_t dev) { return (VIRTIO_BUS_FINALIZE_FEATURES(device_get_parent(dev))); } int -virtio_alloc_virtqueues(device_t dev, int flags, int nvqs, +virtio_alloc_virtqueues(device_t dev, int nvqs, struct vq_alloc_info *info) { - return (VIRTIO_BUS_ALLOC_VIRTQUEUES(device_get_parent(dev), flags, - nvqs, info)); + return (VIRTIO_BUS_ALLOC_VIRTQUEUES(device_get_parent(dev), nvqs, info)); } int virtio_setup_intr(device_t dev, enum intr_type type) { return (VIRTIO_BUS_SETUP_INTR(device_get_parent(dev), type)); } bool virtio_with_feature(device_t dev, uint64_t feature) { return (VIRTIO_BUS_WITH_FEATURE(device_get_parent(dev), feature)); } void virtio_stop(device_t dev) { VIRTIO_BUS_STOP(device_get_parent(dev)); } int virtio_reinit(device_t dev, uint64_t features) { return (VIRTIO_BUS_REINIT(device_get_parent(dev), features)); } void virtio_reinit_complete(device_t dev) { VIRTIO_BUS_REINIT_COMPLETE(device_get_parent(dev)); } int virtio_config_generation(device_t dev) { return (VIRTIO_BUS_CONFIG_GENERATION(device_get_parent(dev))); } void virtio_read_device_config(device_t dev, bus_size_t offset, void *dst, int len) { VIRTIO_BUS_READ_DEVICE_CONFIG(device_get_parent(dev), offset, dst, len); } void virtio_write_device_config(device_t dev, bus_size_t offset, const void *dst, int len) { VIRTIO_BUS_WRITE_DEVICE_CONFIG(device_get_parent(dev), offset, dst, len); } int virtio_child_pnpinfo(device_t busdev __unused, device_t child, struct sbuf *sb) { /* * All of these PCI fields will be only 16 bits, but on the vtmmio bus * the corresponding fields (only "vendor" and "device_type") are 32 * bits. Many virtio drivers can attach below either bus. * Gratuitously expand these two fields to 32-bits to allow sharing PNP * match table data between the mostly-similar buses. * * Subdevice and device_type are redundant in both buses, so I don't * see a lot of PNP utility in exposing the same value under a * different name. */ sbuf_printf(sb, "vendor=0x%08x device=0x%04x subvendor=0x%04x " "device_type=0x%08x", (unsigned)virtio_get_vendor(child), (unsigned)virtio_get_device(child), (unsigned)virtio_get_subvendor(child), (unsigned)virtio_get_device_type(child)); return (0); } static int virtio_modevent(module_t mod, int type, void *unused) { int error; switch (type) { case MOD_LOAD: case MOD_QUIESCE: case MOD_UNLOAD: case MOD_SHUTDOWN: error = 0; break; default: error = EOPNOTSUPP; break; } return (error); } static moduledata_t virtio_mod = { "virtio", virtio_modevent, 0 }; DECLARE_MODULE(virtio, virtio_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); MODULE_VERSION(virtio, 1); diff --git a/sys/dev/virtio/virtio.h b/sys/dev/virtio/virtio.h index 96ebaf653428..855dafd63033 100644 --- a/sys/dev/virtio/virtio.h +++ b/sys/dev/virtio/virtio.h @@ -1,191 +1,191 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014, Bryan Venteicher * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _VIRTIO_H_ #define _VIRTIO_H_ #include #include #include #ifdef _KERNEL struct sbuf; struct vq_alloc_info; /* * Each virtqueue indirect descriptor list must be physically contiguous. * To allow us to malloc(9) each list individually, limit the number * supported to what will fit in one page. With 4KB pages, this is a limit * of 256 descriptors. If there is ever a need for more, we can switch to * contigmalloc(9) for the larger allocations, similar to what * bus_dmamem_alloc(9) does. * * Note the sizeof(struct vring_desc) is 16 bytes. */ #define VIRTIO_MAX_INDIRECT ((int) (PAGE_SIZE / 16)) /* * VirtIO instance variables indices. */ #define VIRTIO_IVAR_DEVTYPE 1 #define VIRTIO_IVAR_FEATURE_DESC 2 #define VIRTIO_IVAR_VENDOR 3 #define VIRTIO_IVAR_DEVICE 4 #define VIRTIO_IVAR_SUBVENDOR 5 #define VIRTIO_IVAR_SUBDEVICE 6 #define VIRTIO_IVAR_MODERN 7 struct virtio_feature_desc { uint64_t vfd_val; const char *vfd_str; }; #define VIRTIO_DRIVER_MODULE(name, driver, evh, arg) \ DRIVER_MODULE(name, virtio_mmio, driver, evh, arg); \ DRIVER_MODULE(name, virtio_pci, driver, evh, arg) struct virtio_pnp_match { uint32_t device_type; const char *description; }; #define VIRTIO_SIMPLE_PNPINFO(driver, devtype, desc) \ static const struct virtio_pnp_match driver ## _match = { \ .device_type = devtype, \ .description = desc, \ }; \ MODULE_PNP_INFO("U32:device_type;D:#", virtio_mmio, driver, \ &driver ## _match, 1); \ MODULE_PNP_INFO("U32:device_type;D:#", virtio_pci, driver, \ &driver ## _match, 1) #define VIRTIO_SIMPLE_PROBE(dev, driver) \ (virtio_simple_probe(dev, &driver ## _match)) const char *virtio_device_name(uint16_t devid); void virtio_describe(device_t dev, const char *msg, uint64_t features, struct virtio_feature_desc *desc); int virtio_describe_sbuf(struct sbuf *sb, uint64_t features, struct virtio_feature_desc *desc); uint64_t virtio_filter_transport_features(uint64_t features); bool virtio_bus_is_modern(device_t dev); void virtio_read_device_config_array(device_t dev, bus_size_t offset, void *dst, int size, int count); /* * VirtIO Bus Methods. */ void virtio_read_ivar(device_t dev, int ivar, uintptr_t *val); void virtio_write_ivar(device_t dev, int ivar, uintptr_t val); uint64_t virtio_negotiate_features(device_t dev, uint64_t child_features); int virtio_finalize_features(device_t dev); -int virtio_alloc_virtqueues(device_t dev, int flags, int nvqs, +int virtio_alloc_virtqueues(device_t dev, int nvqs, struct vq_alloc_info *info); int virtio_setup_intr(device_t dev, enum intr_type type); bool virtio_with_feature(device_t dev, uint64_t feature); void virtio_stop(device_t dev); int virtio_config_generation(device_t dev); int virtio_reinit(device_t dev, uint64_t features); void virtio_reinit_complete(device_t dev); int virtio_child_pnpinfo(device_t busdev, device_t child, struct sbuf *sb); /* * Read/write a variable amount from the device specific (ie, network) * configuration region. This region is encoded in the same endian as * the guest. */ void virtio_read_device_config(device_t dev, bus_size_t offset, void *dst, int length); void virtio_write_device_config(device_t dev, bus_size_t offset, const void *src, int length); /* Inlined device specific read/write functions for common lengths. */ #define VIRTIO_RDWR_DEVICE_CONFIG(size, type) \ static inline type \ __CONCAT(virtio_read_dev_config_,size)(device_t dev, \ bus_size_t offset) \ { \ type val; \ virtio_read_device_config(dev, offset, &val, sizeof(type)); \ return (val); \ } \ \ static inline void \ __CONCAT(virtio_write_dev_config_,size)(device_t dev, \ bus_size_t offset, type val) \ { \ virtio_write_device_config(dev, offset, &val, sizeof(type)); \ } VIRTIO_RDWR_DEVICE_CONFIG(1, uint8_t); VIRTIO_RDWR_DEVICE_CONFIG(2, uint16_t); VIRTIO_RDWR_DEVICE_CONFIG(4, uint32_t); #undef VIRTIO_RDWR_DEVICE_CONFIG #define VIRTIO_READ_IVAR(name, ivar) \ static inline int \ __CONCAT(virtio_get_,name)(device_t dev) \ { \ uintptr_t val; \ virtio_read_ivar(dev, ivar, &val); \ return ((int) val); \ } VIRTIO_READ_IVAR(device_type, VIRTIO_IVAR_DEVTYPE); VIRTIO_READ_IVAR(vendor, VIRTIO_IVAR_VENDOR); VIRTIO_READ_IVAR(device, VIRTIO_IVAR_DEVICE); VIRTIO_READ_IVAR(subvendor, VIRTIO_IVAR_SUBVENDOR); VIRTIO_READ_IVAR(subdevice, VIRTIO_IVAR_SUBDEVICE); VIRTIO_READ_IVAR(modern, VIRTIO_IVAR_MODERN); #undef VIRTIO_READ_IVAR #define VIRTIO_WRITE_IVAR(name, ivar) \ static inline void \ __CONCAT(virtio_set_,name)(device_t dev, void *val) \ { \ virtio_write_ivar(dev, ivar, (uintptr_t) val); \ } VIRTIO_WRITE_IVAR(feature_desc, VIRTIO_IVAR_FEATURE_DESC); #undef VIRTIO_WRITE_IVAR static inline int virtio_simple_probe(device_t dev, const struct virtio_pnp_match *match) { if (virtio_get_device_type(dev) != match->device_type) return (ENXIO); device_set_desc(dev, match->description); return (BUS_PROBE_DEFAULT); } #endif /* _KERNEL */ #endif /* _VIRTIO_H_ */ diff --git a/sys/dev/virtio/virtio_bus_if.m b/sys/dev/virtio/virtio_bus_if.m index 848c6ac05b62..57ae90bdc917 100644 --- a/sys/dev/virtio/virtio_bus_if.m +++ b/sys/dev/virtio/virtio_bus_if.m @@ -1,116 +1,115 @@ #- # Copyright (c) 2011, Bryan Venteicher # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # #include #include INTERFACE virtio_bus; HEADER { struct vq_alloc_info; }; CODE { static int virtio_bus_default_finalize_features(device_t dev) { return (0); } static int virtio_bus_default_config_generation(device_t dev) { return (0); } }; METHOD uint64_t negotiate_features { device_t dev; uint64_t child_features; }; METHOD int finalize_features { device_t dev; } DEFAULT virtio_bus_default_finalize_features; METHOD bool with_feature { device_t dev; uint64_t feature; }; METHOD int alloc_virtqueues { device_t dev; - int flags; int nvqs; struct vq_alloc_info *info; }; METHOD int setup_intr { device_t dev; enum intr_type type; }; METHOD void stop { device_t dev; }; METHOD int reinit { device_t dev; uint64_t features; }; METHOD void reinit_complete { device_t dev; }; METHOD void notify_vq { device_t dev; uint16_t queue; bus_size_t offset; }; METHOD int config_generation { device_t dev; } DEFAULT virtio_bus_default_config_generation; METHOD void read_device_config { device_t dev; bus_size_t offset; void *dst; int len; }; METHOD void write_device_config { device_t dev; bus_size_t offset; const void *src; int len; }; METHOD void poll { device_t dev; };