diff --git a/sys/dev/beri/virtio/network/if_vtbe.c b/sys/dev/beri/virtio/network/if_vtbe.c index a08a8a2918cf..01e4be973a00 100644 --- a/sys/dev/beri/virtio/network/if_vtbe.c +++ b/sys/dev/beri/virtio/network/if_vtbe.c @@ -1,655 +1,656 @@ /*- * Copyright (c) 2014 Ruslan Bukin * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * BERI Virtio Networking Frontend */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include +#include +#include #include #include "pio_if.h" #define DPRINTF(fmt, args...) printf(fmt, ##args) #define READ4(_sc, _reg) \ bus_read_4((_sc)->res[0], _reg) #define WRITE4(_sc, _reg, _val) \ bus_write_4((_sc)->res[0], _reg, _val) #define VTBE_LOCK(sc) mtx_lock(&(sc)->mtx) #define VTBE_UNLOCK(sc) mtx_unlock(&(sc)->mtx) #define VTBE_ASSERT_LOCKED(sc) mtx_assert(&(sc)->mtx, MA_OWNED); #define VTBE_ASSERT_UNLOCKED(sc) mtx_assert(&(sc)->mtx, MA_NOTOWNED); /* * Driver data and defines. */ #define DESC_COUNT 256 struct vtbe_softc { struct resource *res[2]; bus_space_tag_t bst; bus_space_handle_t bsh; device_t dev; struct ifnet *ifp; int if_flags; struct mtx mtx; boolean_t is_attached; int beri_mem_offset; device_t pio_send; device_t pio_recv; int opened; struct vqueue_info vs_queues[2]; int vs_curq; int hdrsize; }; static struct resource_spec vtbe_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { -1, 0 } }; static void vtbe_txfinish_locked(struct vtbe_softc *sc); static void vtbe_rxfinish_locked(struct vtbe_softc *sc); static void vtbe_stop_locked(struct vtbe_softc *sc); static int pio_enable_irq(struct vtbe_softc *sc, int enable); static void vtbe_txstart_locked(struct vtbe_softc *sc) { struct virtio_net_hdr_mrg_rxbuf *vnh; struct iovec iov[DESC_COUNT]; struct vqueue_info *vq; struct iovec *riov; struct ifnet *ifp; struct mbuf *m; struct uio uio; int enqueued; int iolen; int error; int *addr; int reg; int len; int n; VTBE_ASSERT_LOCKED(sc); /* RX queue */ vq = &sc->vs_queues[0]; if (!vq_has_descs(vq)) { return; } ifp = sc->ifp; if (ifp->if_drv_flags & IFF_DRV_OACTIVE) { return; } enqueued = 0; if (!vq_ring_ready(vq)) return; vq->vq_save_used = be16toh(vq->vq_used->idx); for (;;) { if (!vq_has_descs(vq)) { ifp->if_drv_flags |= IFF_DRV_OACTIVE; break; } IFQ_DRV_DEQUEUE(&ifp->if_snd, m); if (m == NULL) { break; } n = vq_getchain(sc->beri_mem_offset, vq, iov, DESC_COUNT, NULL); KASSERT(n >= 1 && n <= DESC_COUNT, ("wrong descriptors num %d", n)); addr = iov[0].iov_base; len = iov[0].iov_len; vnh = iov[0].iov_base; memset(vnh, 0, sc->hdrsize); vnh->num_buffers = htobe16(1); iov[0].iov_len -= sc->hdrsize; iov[0].iov_base = (void *)((uintptr_t)iov[0].iov_base + sc->hdrsize); riov = &iov[0]; uio.uio_resid = iov[0].iov_len; uio.uio_iov = riov; uio.uio_segflg = UIO_SYSSPACE; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_rw = UIO_READ; error = m_mbuftouio(&uio, m, 0); if (error) panic("m_mbuftouio failed\n"); iolen = (len - iov[0].iov_len - sc->hdrsize); vq_relchain(vq, iov, 0, iolen + sc->hdrsize); paddr_unmap((void *)addr, len); if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); BPF_MTAP(ifp, m); m_freem(m); ++enqueued; } if (enqueued != 0) { reg = htobe32(VIRTIO_MMIO_INT_VRING); WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); PIO_SET(sc->pio_send, Q_INTR, 1); } } static void vtbe_txstart(struct ifnet *ifp) { struct vtbe_softc *sc = ifp->if_softc; VTBE_LOCK(sc); vtbe_txstart_locked(sc); VTBE_UNLOCK(sc); } static void vtbe_stop_locked(struct vtbe_softc *sc) { struct ifnet *ifp; VTBE_ASSERT_LOCKED(sc); ifp = sc->ifp; ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); } static void vtbe_init_locked(struct vtbe_softc *sc) { struct ifnet *ifp = sc->ifp; VTBE_ASSERT_LOCKED(sc); if (ifp->if_drv_flags & IFF_DRV_RUNNING) return; ifp->if_drv_flags |= IFF_DRV_RUNNING; } static void vtbe_init(void *if_softc) { struct vtbe_softc *sc = if_softc; VTBE_LOCK(sc); vtbe_init_locked(sc); VTBE_UNLOCK(sc); } static int vtbe_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { struct ifmediareq *ifmr; struct vtbe_softc *sc; struct ifreq *ifr; int mask, error; sc = ifp->if_softc; ifr = (struct ifreq *)data; error = 0; switch (cmd) { case SIOCSIFFLAGS: VTBE_LOCK(sc); if (ifp->if_flags & IFF_UP) { pio_enable_irq(sc, 1); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { vtbe_init_locked(sc); } } else { pio_enable_irq(sc, 0); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { vtbe_stop_locked(sc); } } sc->if_flags = ifp->if_flags; VTBE_UNLOCK(sc); break; case SIOCADDMULTI: case SIOCDELMULTI: break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: ifmr = (struct ifmediareq *)data; ifmr->ifm_count = 1; ifmr->ifm_status = (IFM_AVALID | IFM_ACTIVE); ifmr->ifm_active = (IFM_ETHER | IFM_10G_T | IFM_FDX); ifmr->ifm_current = ifmr->ifm_active; break; case SIOCSIFCAP: mask = ifp->if_capenable ^ ifr->ifr_reqcap; if (mask & IFCAP_VLAN_MTU) { ifp->if_capenable ^= IFCAP_VLAN_MTU; } break; case SIOCSIFADDR: pio_enable_irq(sc, 1); default: error = ether_ioctl(ifp, cmd, data); break; } return (error); } static void vtbe_txfinish_locked(struct vtbe_softc *sc) { struct ifnet *ifp; VTBE_ASSERT_LOCKED(sc); ifp = sc->ifp; } static int vq_init(struct vtbe_softc *sc) { struct vqueue_info *vq; uint8_t *base; int size; int reg; int pfn; vq = &sc->vs_queues[sc->vs_curq]; vq->vq_qsize = DESC_COUNT; reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); pfn = be32toh(reg); vq->vq_pfn = pfn; size = vring_size(vq->vq_qsize, VRING_ALIGN); base = paddr_map(sc->beri_mem_offset, (pfn << PAGE_SHIFT), size); /* First pages are descriptors */ vq->vq_desc = (struct vring_desc *)base; base += vq->vq_qsize * sizeof(struct vring_desc); /* Then avail ring */ vq->vq_avail = (struct vring_avail *)base; base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); /* Then it's rounded up to the next page */ base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); /* And the last pages are the used ring */ vq->vq_used = (struct vring_used *)base; /* Mark queue as allocated, and start at 0 when we use it. */ vq->vq_flags = VQ_ALLOC; vq->vq_last_avail = 0; return (0); } static void vtbe_proc_rx(struct vtbe_softc *sc, struct vqueue_info *vq) { struct iovec iov[DESC_COUNT]; struct ifnet *ifp; struct uio uio; struct mbuf *m; int iolen; int i; int n; ifp = sc->ifp; n = vq_getchain(sc->beri_mem_offset, vq, iov, DESC_COUNT, NULL); KASSERT(n >= 1 && n <= DESC_COUNT, ("wrong n %d", n)); iolen = 0; for (i = 1; i < n; i++) { iolen += iov[i].iov_len; } uio.uio_resid = iolen; uio.uio_iov = &iov[1]; uio.uio_segflg = UIO_SYSSPACE; uio.uio_iovcnt = (n - 1); uio.uio_rw = UIO_WRITE; if ((m = m_uiotombuf(&uio, M_NOWAIT, 0, ETHER_ALIGN, M_PKTHDR)) == NULL) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); goto done; } m->m_pkthdr.rcvif = ifp; if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); CURVNET_SET(ifp->if_vnet); VTBE_UNLOCK(sc); (*ifp->if_input)(ifp, m); VTBE_LOCK(sc); CURVNET_RESTORE(); done: vq_relchain(vq, iov, n, iolen + sc->hdrsize); } static void vtbe_rxfinish_locked(struct vtbe_softc *sc) { struct vqueue_info *vq; int reg; /* TX queue */ vq = &sc->vs_queues[1]; if (!vq_ring_ready(vq)) return; /* Process new descriptors */ vq->vq_save_used = be16toh(vq->vq_used->idx); while (vq_has_descs(vq)) { vtbe_proc_rx(sc, vq); } /* Interrupt the other side */ reg = htobe32(VIRTIO_MMIO_INT_VRING); WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); PIO_SET(sc->pio_send, Q_INTR, 1); } static void vtbe_intr(void *arg) { struct vtbe_softc *sc; int pending; uint32_t reg; sc = arg; VTBE_LOCK(sc); reg = PIO_READ(sc->pio_recv); /* Ack */ PIO_SET(sc->pio_recv, reg, 0); pending = htobe32(reg); if (pending & Q_SEL) { reg = READ4(sc, VIRTIO_MMIO_QUEUE_SEL); sc->vs_curq = be32toh(reg); } if (pending & Q_PFN) { vq_init(sc); } if (pending & Q_NOTIFY) { /* beri rx / arm tx notify */ vtbe_txfinish_locked(sc); } if (pending & Q_NOTIFY1) { vtbe_rxfinish_locked(sc); } VTBE_UNLOCK(sc); } static int vtbe_get_hwaddr(struct vtbe_softc *sc, uint8_t *hwaddr) { int rnd; /* * Generate MAC address, use 'bsd' + random 24 low-order bits. */ rnd = arc4random() & 0x00ffffff; hwaddr[0] = 'b'; hwaddr[1] = 's'; hwaddr[2] = 'd'; hwaddr[3] = rnd >> 16; hwaddr[4] = rnd >> 8; hwaddr[5] = rnd >> 0; return (0); } static int pio_enable_irq(struct vtbe_softc *sc, int enable) { /* * IRQ lines should be disabled while reprogram FPGA core. */ if (enable) { if (sc->opened == 0) { sc->opened = 1; PIO_SETUP_IRQ(sc->pio_recv, vtbe_intr, sc); } } else { if (sc->opened == 1) { PIO_TEARDOWN_IRQ(sc->pio_recv); sc->opened = 0; } } return (0); } static int vtbe_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtnet")) return (ENXIO); device_set_desc(dev, "Virtio BERI Ethernet Controller"); return (BUS_PROBE_DEFAULT); } static int vtbe_attach(device_t dev) { uint8_t macaddr[ETHER_ADDR_LEN]; struct vtbe_softc *sc; struct ifnet *ifp; int reg; sc = device_get_softc(dev); sc->dev = dev; sc->hdrsize = sizeof(struct virtio_net_hdr_mrg_rxbuf); if (bus_alloc_resources(dev, vtbe_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } /* Memory interface */ sc->bst = rman_get_bustag(sc->res[0]); sc->bsh = rman_get_bushandle(sc->res[0]); mtx_init(&sc->mtx, device_get_nameunit(sc->dev), MTX_NETWORK_LOCK, MTX_DEF); if (setup_offset(dev, &sc->beri_mem_offset) != 0) return (ENXIO); if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) return (ENXIO); if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) return (ENXIO); /* Setup MMIO */ /* Specify that we provide network device */ reg = htobe32(VIRTIO_ID_NETWORK); WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); /* The number of desc we support */ reg = htobe32(DESC_COUNT); WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); /* Our features */ reg = htobe32(VIRTIO_NET_F_MAC | VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_NOTIFY_ON_EMPTY); WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); /* Get MAC */ if (vtbe_get_hwaddr(sc, macaddr)) { device_printf(sc->dev, "can't get mac\n"); return (ENXIO); } /* Set up the ethernet interface. */ sc->ifp = ifp = if_alloc(IFT_ETHER); ifp->if_baudrate = IF_Gbps(10); ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = (IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | IFF_PROMISC); ifp->if_capabilities = IFCAP_VLAN_MTU; ifp->if_capenable = ifp->if_capabilities; ifp->if_start = vtbe_txstart; ifp->if_ioctl = vtbe_ioctl; ifp->if_init = vtbe_init; IFQ_SET_MAXLEN(&ifp->if_snd, DESC_COUNT - 1); ifp->if_snd.ifq_drv_maxlen = DESC_COUNT - 1; IFQ_SET_READY(&ifp->if_snd); ifp->if_hdrlen = sizeof(struct ether_vlan_header); /* All ready to run, attach the ethernet interface. */ ether_ifattach(ifp, macaddr); sc->is_attached = true; return (0); } static device_method_t vtbe_methods[] = { DEVMETHOD(device_probe, vtbe_probe), DEVMETHOD(device_attach, vtbe_attach), { 0, 0 } }; static driver_t vtbe_driver = { "vtbe", vtbe_methods, sizeof(struct vtbe_softc), }; static devclass_t vtbe_devclass; DRIVER_MODULE(vtbe, simplebus, vtbe_driver, vtbe_devclass, 0, 0); MODULE_DEPEND(vtbe, ether, 1, 1, 1); diff --git a/sys/dev/beri/virtio/virtio.c b/sys/dev/beri/virtio/virtio.c index cb7c2934d223..6921c8a874e5 100644 --- a/sys/dev/beri/virtio/virtio.c +++ b/sys/dev/beri/virtio/virtio.c @@ -1,249 +1,248 @@ /*- * Copyright (c) 2014 Ruslan Bukin * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * BERI virtio mmio backend common methods */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include #include #include #include #include "pio_if.h" int vq_ring_ready(struct vqueue_info *vq) { return (vq->vq_flags & VQ_ALLOC); } int vq_has_descs(struct vqueue_info *vq) { return (vq_ring_ready(vq) && vq->vq_last_avail != be16toh(vq->vq_avail->idx)); } void * paddr_map(uint32_t offset, uint32_t phys, uint32_t size) { bus_space_handle_t bsh; if (bus_space_map(fdtbus_bs_tag, (phys + offset), size, 0, &bsh) != 0) { panic("Couldn't map 0x%08x\n", (phys + offset)); } return (void *)(bsh); } void paddr_unmap(void *phys, uint32_t size) { bus_space_unmap(fdtbus_bs_tag, (bus_space_handle_t)phys, size); } static inline void _vq_record(uint32_t offs, int i, volatile struct vring_desc *vd, struct iovec *iov, int n_iov, uint16_t *flags) { if (i >= n_iov) return; iov[i].iov_base = paddr_map(offs, be64toh(vd->addr), be32toh(vd->len)); iov[i].iov_len = be32toh(vd->len); if (flags != NULL) flags[i] = be16toh(vd->flags); } int vq_getchain(uint32_t offs, struct vqueue_info *vq, struct iovec *iov, int n_iov, uint16_t *flags) { volatile struct vring_desc *vdir, *vindir, *vp; int idx, ndesc, n_indir; int head, next; int i; idx = vq->vq_last_avail; ndesc = (be16toh(vq->vq_avail->idx) - idx); if (ndesc == 0) return (0); head = be16toh(vq->vq_avail->ring[idx & (vq->vq_qsize - 1)]); next = head; for (i = 0; i < VQ_MAX_DESCRIPTORS; next = be16toh(vdir->next)) { vdir = &vq->vq_desc[next]; if ((be16toh(vdir->flags) & VRING_DESC_F_INDIRECT) == 0) { _vq_record(offs, i, vdir, iov, n_iov, flags); i++; } else { n_indir = be32toh(vdir->len) / 16; vindir = paddr_map(offs, be64toh(vdir->addr), be32toh(vdir->len)); next = 0; for (;;) { vp = &vindir[next]; _vq_record(offs, i, vp, iov, n_iov, flags); i+=1; if ((be16toh(vp->flags) & \ VRING_DESC_F_NEXT) == 0) break; next = be16toh(vp->next); } paddr_unmap(__DEVOLATILE(void *, vindir), be32toh(vdir->len)); } if ((be16toh(vdir->flags) & VRING_DESC_F_NEXT) == 0) return (i); } return (i); } void vq_relchain(struct vqueue_info *vq, struct iovec *iov, int n, uint32_t iolen) { volatile struct vring_used_elem *vue; volatile struct vring_used *vu; uint16_t head, uidx, mask; int i; mask = vq->vq_qsize - 1; vu = vq->vq_used; head = be16toh(vq->vq_avail->ring[vq->vq_last_avail++ & mask]); uidx = be16toh(vu->idx); vue = &vu->ring[uidx++ & mask]; vue->id = htobe32(head); vue->len = htobe32(iolen); vu->idx = htobe16(uidx); /* Clean up */ for (i = 1; i < (n-1); i++) { paddr_unmap((void *)iov[i].iov_base, iov[i].iov_len); } } int setup_pio(device_t dev, char *name, device_t *pio_dev) { phandle_t pio_node; struct fdt_ic *ic; phandle_t xref; phandle_t node; if ((node = ofw_bus_get_node(dev)) == -1) return (ENXIO); if (OF_searchencprop(node, name, &xref, sizeof(xref)) == -1) { return (ENXIO); } pio_node = OF_node_from_xref(xref); SLIST_FOREACH(ic, &fdt_ic_list_head, fdt_ics) { if (ic->iph == pio_node) { *pio_dev = ic->dev; PIO_CONFIGURE(*pio_dev, PIO_OUT_ALL, PIO_UNMASK_ALL); return (0); } } return (ENXIO); } int setup_offset(device_t dev, uint32_t *offset) { pcell_t dts_value[2]; phandle_t mem_node; phandle_t xref; phandle_t node; int len; if ((node = ofw_bus_get_node(dev)) == -1) return (ENXIO); if (OF_searchencprop(node, "beri-mem", &xref, sizeof(xref)) == -1) { return (ENXIO); } mem_node = OF_node_from_xref(xref); if ((len = OF_getproplen(mem_node, "reg")) <= 0) return (ENXIO); OF_getencprop(mem_node, "reg", dts_value, len); *offset = dts_value[0]; return (0); } diff --git a/sys/dev/beri/virtio/virtio_block.c b/sys/dev/beri/virtio/virtio_block.c index 12b6e0edc9e0..b763cb927c1c 100644 --- a/sys/dev/beri/virtio/virtio_block.c +++ b/sys/dev/beri/virtio/virtio_block.c @@ -1,558 +1,559 @@ /*- * Copyright (c) 2014 Ruslan Bukin * All rights reserved. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * BERI virtio block backend driver */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include +#include +#include #include #include "pio_if.h" #define DPRINTF(fmt, ...) /* We use indirect descriptors */ #define NUM_DESCS 1 #define NUM_QUEUES 1 #define VTBLK_BLK_ID_BYTES 20 #define VTBLK_MAXSEGS 256 struct beri_vtblk_softc { struct resource *res[1]; bus_space_tag_t bst; bus_space_handle_t bsh; struct cdev *cdev; device_t dev; int opened; device_t pio_recv; device_t pio_send; struct vqueue_info vs_queues[NUM_QUEUES]; char ident[VTBLK_BLK_ID_BYTES]; struct ucred *cred; struct vnode *vnode; struct thread *vtblk_ktd; struct sx sc_mtx; int beri_mem_offset; struct md_ioctl *mdio; struct virtio_blk_config *cfg; }; static struct resource_spec beri_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { -1, 0 } }; static int vtblk_rdwr(struct beri_vtblk_softc *sc, struct iovec *iov, int cnt, int offset, int operation, int iolen) { struct vnode *vp; struct mount *mp; struct uio auio; int error; bzero(&auio, sizeof(auio)); vp = sc->vnode; KASSERT(vp != NULL, ("file not opened")); auio.uio_iov = iov; auio.uio_iovcnt = cnt; auio.uio_offset = offset; auio.uio_segflg = UIO_SYSSPACE; auio.uio_rw = operation; auio.uio_resid = iolen; auio.uio_td = curthread; if (operation == 0) { vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred); VOP_UNLOCK(vp, 0); } else { (void) vn_start_write(vp, &mp, V_WAIT); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); error = VOP_WRITE(vp, &auio, IO_SYNC, sc->cred); VOP_UNLOCK(vp, 0); vn_finished_write(mp); } return (error); } static void vtblk_proc(struct beri_vtblk_softc *sc, struct vqueue_info *vq) { struct iovec iov[VTBLK_MAXSEGS + 2]; uint16_t flags[VTBLK_MAXSEGS + 2]; struct virtio_blk_outhdr *vbh; uint8_t *status; off_t offset; int iolen; int type; int i, n; int err; n = vq_getchain(sc->beri_mem_offset, vq, iov, VTBLK_MAXSEGS + 2, flags); KASSERT(n >= 2 && n <= VTBLK_MAXSEGS + 2, ("wrong n value %d", n)); vbh = iov[0].iov_base; status = iov[n-1].iov_base; KASSERT(iov[n-1].iov_len == 1, ("iov_len == %d", iov[n-1].iov_len)); type = be32toh(vbh->type) & ~VIRTIO_BLK_T_BARRIER; offset = be64toh(vbh->sector) * DEV_BSIZE; iolen = 0; for (i = 1; i < (n-1); i++) { iolen += iov[i].iov_len; } switch (type) { case VIRTIO_BLK_T_OUT: case VIRTIO_BLK_T_IN: err = vtblk_rdwr(sc, iov + 1, i - 1, offset, type, iolen); break; case VIRTIO_BLK_T_GET_ID: /* Assume a single buffer */ strlcpy(iov[1].iov_base, sc->ident, MIN(iov[1].iov_len, sizeof(sc->ident))); err = 0; break; case VIRTIO_BLK_T_FLUSH: /* Possible? */ default: err = -ENOSYS; break; } if (err < 0) { if (err == -ENOSYS) { *status = VIRTIO_BLK_S_UNSUPP; } else *status = VIRTIO_BLK_S_IOERR; } else *status = VIRTIO_BLK_S_OK; vq_relchain(vq, iov, n, 1); } static int close_file(struct beri_vtblk_softc *sc, struct thread *td) { int error; if (sc->vnode != NULL) { vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); sc->vnode->v_vflag &= ~VV_MD; VOP_UNLOCK(sc->vnode, 0); error = vn_close(sc->vnode, (FREAD|FWRITE), sc->cred, td); if (error != 0) return (error); sc->vnode = NULL; } if (sc->cred != NULL) crfree(sc->cred); return (0); } static int open_file(struct beri_vtblk_softc *sc, struct thread *td) { struct nameidata nd; struct vattr vattr; int error; int flags; flags = (FREAD | FWRITE); NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->mdio->md_file, td); error = vn_open(&nd, &flags, 0, NULL); if (error != 0) return (error); NDFREE(&nd, NDF_ONLY_PNBUF); if (nd.ni_vp->v_type != VREG) { return (EINVAL); } error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); if (error != 0) return (error); if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); if (nd.ni_vp->v_iflag & VI_DOOMED) { return (1); } } nd.ni_vp->v_vflag |= VV_MD; VOP_UNLOCK(nd.ni_vp, 0); sc->vnode = nd.ni_vp; sc->cred = crhold(td->td_ucred); return (0); } static int vtblk_notify(struct beri_vtblk_softc *sc) { struct vqueue_info *vq; int queue; int reg; vq = &sc->vs_queues[0]; if (!vq_ring_ready(vq)) return (0); if (!sc->opened) return (0); reg = READ2(sc, VIRTIO_MMIO_QUEUE_NOTIFY); queue = be16toh(reg); KASSERT(queue == 0, ("we support single queue only")); /* Process new descriptors */ vq = &sc->vs_queues[queue]; vq->vq_save_used = be16toh(vq->vq_used->idx); while (vq_has_descs(vq)) vtblk_proc(sc, vq); /* Interrupt the other side */ if ((be16toh(vq->vq_avail->flags) & VRING_AVAIL_F_NO_INTERRUPT) == 0) { reg = htobe32(VIRTIO_MMIO_INT_VRING); WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); PIO_SET(sc->pio_send, Q_INTR, 1); } return (0); } static int vq_init(struct beri_vtblk_softc *sc) { struct vqueue_info *vq; uint8_t *base; int size; int reg; int pfn; vq = &sc->vs_queues[0]; vq->vq_qsize = NUM_DESCS; reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); pfn = be32toh(reg); vq->vq_pfn = pfn; size = vring_size(vq->vq_qsize, VRING_ALIGN); base = paddr_map(sc->beri_mem_offset, (pfn << PAGE_SHIFT), size); /* First pages are descriptors */ vq->vq_desc = (struct vring_desc *)base; base += vq->vq_qsize * sizeof(struct vring_desc); /* Then avail ring */ vq->vq_avail = (struct vring_avail *)base; base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); /* Then it's rounded up to the next page */ base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); /* And the last pages are the used ring */ vq->vq_used = (struct vring_used *)base; /* Mark queue as allocated, and start at 0 when we use it. */ vq->vq_flags = VQ_ALLOC; vq->vq_last_avail = 0; return (0); } static void vtblk_thread(void *arg) { struct beri_vtblk_softc *sc; int err; sc = arg; sx_xlock(&sc->sc_mtx); for (;;) { err = msleep(sc, &sc->sc_mtx, PCATCH | PZERO, "prd", hz); vtblk_notify(sc); } sx_xunlock(&sc->sc_mtx); kthread_exit(); } static int backend_info(struct beri_vtblk_softc *sc) { struct virtio_blk_config *cfg; uint32_t *s; int reg; int i; /* Specify that we provide block device */ reg = htobe32(VIRTIO_ID_BLOCK); WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); /* Queue size */ reg = htobe32(NUM_DESCS); WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); /* Our features */ reg = htobe32(VIRTIO_RING_F_INDIRECT_DESC | VIRTIO_BLK_F_BLK_SIZE | VIRTIO_BLK_F_SEG_MAX); WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); cfg = sc->cfg; cfg->capacity = htobe64(sc->mdio->md_mediasize / DEV_BSIZE); cfg->size_max = 0; /* not negotiated */ cfg->seg_max = htobe32(VTBLK_MAXSEGS); cfg->blk_size = htobe32(DEV_BSIZE); s = (uint32_t *)cfg; for (i = 0; i < sizeof(struct virtio_blk_config); i+=4) { WRITE4(sc, VIRTIO_MMIO_CONFIG + i, *s); s+=1; } sprintf(sc->ident, "Virtio block backend"); return (0); } static void vtblk_intr(void *arg) { struct beri_vtblk_softc *sc; int pending; int reg; sc = arg; reg = PIO_READ(sc->pio_recv); /* Ack */ PIO_SET(sc->pio_recv, reg, 0); pending = htobe32(reg); if (pending & Q_PFN) { vq_init(sc); } if (pending & Q_NOTIFY) { wakeup(sc); } } static int beri_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td) { struct beri_vtblk_softc *sc; int err; sc = dev->si_drv1; switch (cmd) { case MDIOCATTACH: /* take file as argument */ if (sc->vnode != NULL) { /* Already opened */ return (1); } sc->mdio = (struct md_ioctl *)addr; backend_info(sc); DPRINTF("opening file, td 0x%08x\n", (int)td); err = open_file(sc, td); if (err) return (err); PIO_SETUP_IRQ(sc->pio_recv, vtblk_intr, sc); sc->opened = 1; break; case MDIOCDETACH: if (sc->vnode == 0) { /* File not opened */ return (1); } sc->opened = 0; DPRINTF("closing file, td 0x%08x\n", (int)td); err = close_file(sc, td); if (err) return (err); PIO_TEARDOWN_IRQ(sc->pio_recv); break; default: break; } return (0); } static struct cdevsw beri_cdevsw = { .d_version = D_VERSION, .d_ioctl = beri_ioctl, .d_name = "virtio block backend", }; static int beri_vtblk_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtblk")) return (ENXIO); device_set_desc(dev, "SRI-Cambridge BERI block"); return (BUS_PROBE_DEFAULT); } static int beri_vtblk_attach(device_t dev) { struct beri_vtblk_softc *sc; int error; sc = device_get_softc(dev); sc->dev = dev; if (bus_alloc_resources(dev, beri_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } /* Memory interface */ sc->bst = rman_get_bustag(sc->res[0]); sc->bsh = rman_get_bushandle(sc->res[0]); sc->cfg = malloc(sizeof(struct virtio_blk_config), M_DEVBUF, M_NOWAIT|M_ZERO); sx_init(&sc->sc_mtx, device_get_nameunit(sc->dev)); error = kthread_add(vtblk_thread, sc, NULL, &sc->vtblk_ktd, 0, 0, "beri_virtio_block"); if (error) { device_printf(dev, "cannot create kthread\n"); return (ENXIO); } if (setup_offset(dev, &sc->beri_mem_offset) != 0) return (ENXIO); if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) return (ENXIO); if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) return (ENXIO); sc->cdev = make_dev(&beri_cdevsw, 0, UID_ROOT, GID_WHEEL, S_IRWXU, "beri_vtblk"); if (sc->cdev == NULL) { device_printf(dev, "Failed to create character device.\n"); return (ENXIO); } sc->cdev->si_drv1 = sc; return (0); } static device_method_t beri_vtblk_methods[] = { DEVMETHOD(device_probe, beri_vtblk_probe), DEVMETHOD(device_attach, beri_vtblk_attach), { 0, 0 } }; static driver_t beri_vtblk_driver = { "beri_vtblk", beri_vtblk_methods, sizeof(struct beri_vtblk_softc), }; static devclass_t beri_vtblk_devclass; DRIVER_MODULE(beri_vtblk, simplebus, beri_vtblk_driver, beri_vtblk_devclass, 0, 0);