Index: stable/10/sys/dev/netmap/if_lem_netmap.h =================================================================== --- stable/10/sys/dev/netmap/if_lem_netmap.h (revision 278778) +++ stable/10/sys/dev/netmap/if_lem_netmap.h (revision 278779) @@ -1,497 +1,497 @@ /* * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ * * netmap support for: lem * * For details on netmap support please see ixgbe_netmap.h */ #include #include #include #include /* vtophys ? */ #include extern int netmap_adaptive_io; /* * Register/unregister. We are already under netmap lock. */ static int lem_netmap_reg(struct netmap_adapter *na, int onoff) { struct ifnet *ifp = na->ifp; struct adapter *adapter = ifp->if_softc; EM_CORE_LOCK(adapter); lem_disable_intr(adapter); /* Tell the stack that the interface is no longer active */ ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); #ifndef EM_LEGACY_IRQ // XXX do we need this ? taskqueue_block(adapter->tq); taskqueue_drain(adapter->tq, &adapter->rxtx_task); taskqueue_drain(adapter->tq, &adapter->link_task); #endif /* !EM_LEGCY_IRQ */ /* enable or disable flags and callbacks in na and ifp */ if (onoff) { nm_set_native_flags(na); } else { nm_clear_native_flags(na); } lem_init_locked(adapter); /* also enable intr */ #ifndef EM_LEGACY_IRQ taskqueue_unblock(adapter->tq); // XXX do we need this ? #endif /* !EM_LEGCY_IRQ */ EM_CORE_UNLOCK(adapter); return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1); } /* * Reconcile kernel and user view of the transmit ring. */ static int lem_netmap_txsync(struct netmap_kring *kring, int flags) { struct netmap_adapter *na = kring->na; struct ifnet *ifp = na->ifp; struct netmap_ring *ring = kring->ring; u_int nm_i; /* index into the netmap ring */ u_int nic_i; /* index into the NIC ring */ u_int const lim = kring->nkr_num_slots - 1; u_int const head = kring->rhead; /* generate an interrupt approximately every half ring */ u_int report_frequency = kring->nkr_num_slots >> 1; /* device-specific */ struct adapter *adapter = ifp->if_softc; #ifdef NIC_PARAVIRT struct paravirt_csb *csb = adapter->csb; uint64_t *csbd = (uint64_t *)(csb + 1); #endif /* NIC_PARAVIRT */ bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, BUS_DMASYNC_POSTREAD); /* * First part: process new packets to send. */ nm_i = kring->nr_hwcur; if (nm_i != head) { /* we have new packets to send */ #ifdef NIC_PARAVIRT int do_kick = 0; uint64_t t = 0; // timestamp int n = head - nm_i; if (n < 0) n += lim + 1; if (csb) { t = rdtsc(); /* last timestamp */ csbd[16] += t - csbd[0]; /* total Wg */ csbd[17] += n; /* Wg count */ csbd[0] = t; } #endif /* NIC_PARAVIRT */ nic_i = netmap_idx_k2n(kring, nm_i); while (nm_i != head) { struct netmap_slot *slot = &ring->slot[nm_i]; u_int len = slot->len; uint64_t paddr; void *addr = PNMB(na, slot, &paddr); /* device-specific */ struct e1000_tx_desc *curr = &adapter->tx_desc_base[nic_i]; struct em_buffer *txbuf = &adapter->tx_buffer_area[nic_i]; int flags = (slot->flags & NS_REPORT || nic_i == 0 || nic_i == report_frequency) ? E1000_TXD_CMD_RS : 0; NM_CHECK_ADDR_LEN(na, addr, len); if (slot->flags & NS_BUF_CHANGED) { /* buffer has changed, reload map */ curr->buffer_addr = htole64(paddr); netmap_reload_map(na, adapter->txtag, txbuf->map, addr); } slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); /* Fill the slot in the NIC ring. */ curr->upper.data = 0; curr->lower.data = htole32(adapter->txd_cmd | len | (E1000_TXD_CMD_EOP | flags) ); bus_dmamap_sync(adapter->txtag, txbuf->map, BUS_DMASYNC_PREWRITE); nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); // XXX might try an early kick } kring->nr_hwcur = head; /* synchronize the NIC ring */ bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); #ifdef NIC_PARAVIRT /* set unconditionally, then also kick if needed */ if (csb) { t = rdtsc(); if (csb->host_need_txkick == 2) { /* can compute an update of delta */ int64_t delta = t - csbd[3]; if (delta < 0) delta = -delta; if (csbd[8] == 0 || delta < csbd[8]) { csbd[8] = delta; csbd[9]++; } csbd[10]++; } csb->guest_tdt = nic_i; csbd[18] += t - csbd[0]; // total wp csbd[19] += n; } if (!csb || !csb->guest_csb_on || (csb->host_need_txkick & 1)) do_kick = 1; if (do_kick) #endif /* NIC_PARAVIRT */ /* (re)start the tx unit up to slot nic_i (excluded) */ E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), nic_i); #ifdef NIC_PARAVIRT if (do_kick) { uint64_t t1 = rdtsc(); csbd[20] += t1 - t; // total Np csbd[21]++; } #endif /* NIC_PARAVIRT */ } /* * Second part: reclaim buffers for completed transmissions. */ if (ticks != kring->last_reclaim || flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) { kring->last_reclaim = ticks; /* record completed transmissions using TDH */ #ifdef NIC_PARAVIRT /* host updates tdh unconditionally, and we have * no side effects on reads, so we can read from there * instead of exiting. */ if (csb) { static int drain = 0, nodrain=0, good = 0, bad = 0, fail = 0; u_int x = adapter->next_tx_to_clean; csbd[19]++; // XXX count reclaims nic_i = csb->host_tdh; if (csb->guest_csb_on) { if (nic_i == x) { bad++; csbd[24]++; // failed reclaims /* no progress, request kick and retry */ csb->guest_need_txkick = 1; mb(); // XXX barrier nic_i = csb->host_tdh; } else { good++; } if (nic_i != x) { csb->guest_need_txkick = 2; if (nic_i == csb->guest_tdt) drain++; else nodrain++; #if 1 if (netmap_adaptive_io) { /* new mechanism: last half ring (or so) * released one slot at a time. * This effectively makes the system spin. * * Take next_to_clean + 1 as a reference. * tdh must be ahead or equal * On entry, the logical order is * x < tdh = nic_i * We first push tdh up to avoid wraps. * The limit is tdh-ll (half ring). * if tdh-256 < x we report x; * else we report tdh-256 */ u_int tdh = nic_i; u_int ll = csbd[15]; u_int delta = lim/8; if (netmap_adaptive_io == 2 || ll > delta) csbd[15] = ll = delta; else if (netmap_adaptive_io == 1 && ll > 1) { csbd[15]--; } if (nic_i >= kring->nkr_num_slots) { RD(5, "bad nic_i %d on input", nic_i); } x = nm_next(x, lim); if (tdh < x) tdh += lim + 1; if (tdh <= x + ll) { nic_i = x; csbd[25]++; //report n + 1; } else { tdh = nic_i; if (tdh < ll) tdh += lim + 1; nic_i = tdh - ll; csbd[26]++; // report tdh - ll } } #endif } else { /* we stop, count whether we are idle or not */ int bh_active = csb->host_need_txkick & 2 ? 4 : 0; csbd[27+ csb->host_need_txkick]++; if (netmap_adaptive_io == 1) { if (bh_active && csbd[15] > 1) csbd[15]--; else if (!bh_active && csbd[15] < lim/2) csbd[15]++; } bad--; fail++; } } RD(1, "drain %d nodrain %d good %d retry %d fail %d", drain, nodrain, good, bad, fail); } else #endif /* !NIC_PARAVIRT */ nic_i = E1000_READ_REG(&adapter->hw, E1000_TDH(0)); if (nic_i >= kring->nkr_num_slots) { /* XXX can it happen ? */ D("TDH wrap %d", nic_i); nic_i -= kring->nkr_num_slots; } adapter->next_tx_to_clean = nic_i; kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); } nm_txsync_finalize(kring); return 0; } /* * Reconcile kernel and user view of the receive ring. */ static int lem_netmap_rxsync(struct netmap_kring *kring, int flags) { struct netmap_adapter *na = kring->na; struct ifnet *ifp = na->ifp; struct netmap_ring *ring = kring->ring; u_int nm_i; /* index into the netmap ring */ u_int nic_i; /* index into the NIC ring */ u_int n; u_int const lim = kring->nkr_num_slots - 1; u_int const head = nm_rxsync_prologue(kring); int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; /* device-specific */ struct adapter *adapter = ifp->if_softc; #ifdef NIC_PARAVIRT struct paravirt_csb *csb = adapter->csb; uint32_t csb_mode = csb && csb->guest_csb_on; uint32_t do_host_rxkick = 0; #endif /* NIC_PARAVIRT */ if (head > lim) return netmap_ring_reinit(kring); #ifdef NIC_PARAVIRT if (csb_mode) { force_update = 1; csb->guest_need_rxkick = 0; } #endif /* NIC_PARAVIRT */ /* XXX check sync modes */ bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); /* * First part: import newly received packets. */ if (netmap_no_pendintr || force_update) { uint16_t slot_flags = kring->nkr_slot_flags; nic_i = adapter->next_rx_desc_to_check; nm_i = netmap_idx_n2k(kring, nic_i); for (n = 0; ; n++) { struct e1000_rx_desc *curr = &adapter->rx_desc_base[nic_i]; uint32_t staterr = le32toh(curr->status); int len; #ifdef NIC_PARAVIRT if (csb_mode) { if ((staterr & E1000_RXD_STAT_DD) == 0) { /* don't bother to retry if more than 1 pkt */ if (n > 1) break; csb->guest_need_rxkick = 1; wmb(); staterr = le32toh(curr->status); if ((staterr & E1000_RXD_STAT_DD) == 0) { break; } else { /* we are good */ csb->guest_need_rxkick = 0; } } } else #endif /* NIC_PARAVIRT */ if ((staterr & E1000_RXD_STAT_DD) == 0) break; len = le16toh(curr->length) - 4; // CRC if (len < 0) { RD(5, "bogus pkt (%d) size %d nic idx %d", n, len, nic_i); len = 0; } ring->slot[nm_i].len = len; ring->slot[nm_i].flags = slot_flags; bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[nic_i].map, BUS_DMASYNC_POSTREAD); nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); } if (n) { /* update the state variables */ #ifdef NIC_PARAVIRT if (csb_mode) { if (n > 1) { /* leave one spare buffer so we avoid rxkicks */ nm_i = nm_prev(nm_i, lim); nic_i = nm_prev(nic_i, lim); n--; } else { csb->guest_need_rxkick = 1; } } #endif /* NIC_PARAVIRT */ ND("%d new packets at nic %d nm %d tail %d", n, adapter->next_rx_desc_to_check, netmap_idx_n2k(kring, adapter->next_rx_desc_to_check), kring->nr_hwtail); adapter->next_rx_desc_to_check = nic_i; - // ifp->if_ipackets += n; + // if_inc_counter(ifp, IFCOUNTER_IPACKETS, n); kring->nr_hwtail = nm_i; } kring->nr_kflags &= ~NKR_PENDINTR; } /* * Second part: skip past packets that userspace has released. */ nm_i = kring->nr_hwcur; if (nm_i != head) { nic_i = netmap_idx_k2n(kring, nm_i); for (n = 0; nm_i != head; n++) { struct netmap_slot *slot = &ring->slot[nm_i]; uint64_t paddr; void *addr = PNMB(na, slot, &paddr); struct e1000_rx_desc *curr = &adapter->rx_desc_base[nic_i]; struct em_buffer *rxbuf = &adapter->rx_buffer_area[nic_i]; if (addr == NETMAP_BUF_BASE(na)) /* bad buf */ goto ring_reset; if (slot->flags & NS_BUF_CHANGED) { /* buffer has changed, reload map */ curr->buffer_addr = htole64(paddr); netmap_reload_map(na, adapter->rxtag, rxbuf->map, addr); slot->flags &= ~NS_BUF_CHANGED; } curr->status = 0; bus_dmamap_sync(adapter->rxtag, rxbuf->map, BUS_DMASYNC_PREREAD); #ifdef NIC_PARAVIRT if (csb_mode && csb->host_rxkick_at == nic_i) do_host_rxkick = 1; #endif /* NIC_PARAVIRT */ nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); } kring->nr_hwcur = head; bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* * IMPORTANT: we must leave one free slot in the ring, * so move nic_i back by one unit */ nic_i = nm_prev(nic_i, lim); #ifdef NIC_PARAVIRT /* set unconditionally, then also kick if needed */ if (csb) csb->guest_rdt = nic_i; if (!csb_mode || do_host_rxkick) #endif /* NIC_PARAVIRT */ E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), nic_i); } /* tell userspace that there might be new packets */ nm_rxsync_finalize(kring); return 0; ring_reset: return netmap_ring_reinit(kring); } static void lem_netmap_attach(struct adapter *adapter) { struct netmap_adapter na; bzero(&na, sizeof(na)); na.ifp = adapter->ifp; na.na_flags = NAF_BDG_MAYSLEEP; na.num_tx_desc = adapter->num_tx_desc; na.num_rx_desc = adapter->num_rx_desc; na.nm_txsync = lem_netmap_txsync; na.nm_rxsync = lem_netmap_rxsync; na.nm_register = lem_netmap_reg; na.num_tx_rings = na.num_rx_rings = 1; netmap_attach(&na); } /* end of file */ Index: stable/10/sys/dev/netmap/if_re_netmap.h =================================================================== --- stable/10/sys/dev/netmap/if_re_netmap.h (revision 278778) +++ stable/10/sys/dev/netmap/if_re_netmap.h (revision 278779) @@ -1,379 +1,379 @@ /* * Copyright (C) 2011-2014 Luigi Rizzo. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ * * netmap support for: re * * For more details on netmap support please see ixgbe_netmap.h */ #include #include #include #include /* vtophys ? */ #include /* * Register/unregister. We are already under netmap lock. */ static int re_netmap_reg(struct netmap_adapter *na, int onoff) { struct ifnet *ifp = na->ifp; struct rl_softc *adapter = ifp->if_softc; RL_LOCK(adapter); re_stop(adapter); /* also clears IFF_DRV_RUNNING */ if (onoff) { nm_set_native_flags(na); } else { nm_clear_native_flags(na); } re_init_locked(adapter); /* also enables intr */ RL_UNLOCK(adapter); return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1); } /* * Reconcile kernel and user view of the transmit ring. */ static int re_netmap_txsync(struct netmap_kring *kring, int flags) { struct netmap_adapter *na = kring->na; struct ifnet *ifp = na->ifp; struct netmap_ring *ring = kring->ring; u_int nm_i; /* index into the netmap ring */ u_int nic_i; /* index into the NIC ring */ u_int n; u_int const lim = kring->nkr_num_slots - 1; u_int const head = kring->rhead; /* device-specific */ struct rl_softc *sc = ifp->if_softc; struct rl_txdesc *txd = sc->rl_ldata.rl_tx_desc; bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, sc->rl_ldata.rl_tx_list_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); // XXX extra postwrite ? /* * First part: process new packets to send. */ nm_i = kring->nr_hwcur; if (nm_i != head) { /* we have new packets to send */ nic_i = sc->rl_ldata.rl_tx_prodidx; // XXX or netmap_idx_k2n(kring, nm_i); for (n = 0; nm_i != head; n++) { struct netmap_slot *slot = &ring->slot[nm_i]; u_int len = slot->len; uint64_t paddr; void *addr = PNMB(na, slot, &paddr); /* device-specific */ struct rl_desc *desc = &sc->rl_ldata.rl_tx_list[nic_i]; int cmd = slot->len | RL_TDESC_CMD_EOF | RL_TDESC_CMD_OWN | RL_TDESC_CMD_SOF ; NM_CHECK_ADDR_LEN(na, addr, len); if (nic_i == lim) /* mark end of ring */ cmd |= RL_TDESC_CMD_EOR; if (slot->flags & NS_BUF_CHANGED) { /* buffer has changed, reload map */ desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); netmap_reload_map(na, sc->rl_ldata.rl_tx_mtag, txd[nic_i].tx_dmamap, addr); } slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); /* Fill the slot in the NIC ring. */ desc->rl_cmdstat = htole32(cmd); /* make sure changes to the buffer are synced */ bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag, txd[nic_i].tx_dmamap, BUS_DMASYNC_PREWRITE); nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); } sc->rl_ldata.rl_tx_prodidx = nic_i; kring->nr_hwcur = head; /* synchronize the NIC ring */ bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, sc->rl_ldata.rl_tx_list_map, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); /* start ? */ CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START); } /* * Second part: reclaim buffers for completed transmissions. */ if (flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) { nic_i = sc->rl_ldata.rl_tx_considx; for (n = 0; nic_i != sc->rl_ldata.rl_tx_prodidx; n++, nic_i = RL_TX_DESC_NXT(sc, nic_i)) { uint32_t cmdstat = le32toh(sc->rl_ldata.rl_tx_list[nic_i].rl_cmdstat); if (cmdstat & RL_TDESC_STAT_OWN) break; } if (n > 0) { sc->rl_ldata.rl_tx_considx = nic_i; sc->rl_ldata.rl_tx_free += n; kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); } } nm_txsync_finalize(kring); return 0; } /* * Reconcile kernel and user view of the receive ring. */ static int re_netmap_rxsync(struct netmap_kring *kring, int flags) { struct netmap_adapter *na = kring->na; struct ifnet *ifp = na->ifp; struct netmap_ring *ring = kring->ring; u_int nm_i; /* index into the netmap ring */ u_int nic_i; /* index into the NIC ring */ u_int n; u_int const lim = kring->nkr_num_slots - 1; u_int const head = nm_rxsync_prologue(kring); int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; /* device-specific */ struct rl_softc *sc = ifp->if_softc; struct rl_rxdesc *rxd = sc->rl_ldata.rl_rx_desc; if (head > lim) return netmap_ring_reinit(kring); bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag, sc->rl_ldata.rl_rx_list_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); /* * First part: import newly received packets. * * This device uses all the buffers in the ring, so we need * another termination condition in addition to RL_RDESC_STAT_OWN * cleared (all buffers could have it cleared). The easiest one * is to stop right before nm_hwcur. */ if (netmap_no_pendintr || force_update) { uint16_t slot_flags = kring->nkr_slot_flags; uint32_t stop_i = nm_prev(kring->nr_hwcur, lim); nic_i = sc->rl_ldata.rl_rx_prodidx; /* next pkt to check */ nm_i = netmap_idx_n2k(kring, nic_i); while (nm_i != stop_i) { struct rl_desc *cur_rx = &sc->rl_ldata.rl_rx_list[nic_i]; uint32_t rxstat = le32toh(cur_rx->rl_cmdstat); uint32_t total_len; if ((rxstat & RL_RDESC_STAT_OWN) != 0) break; total_len = rxstat & sc->rl_rxlenmask; /* XXX subtract crc */ total_len = (total_len < 4) ? 0 : total_len - 4; ring->slot[nm_i].len = total_len; ring->slot[nm_i].flags = slot_flags; /* sync was in re_newbuf() */ bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, rxd[nic_i].rx_dmamap, BUS_DMASYNC_POSTREAD); - // sc->rl_ifp->if_ipackets++; + // if_inc_counter(sc->rl_ifp, IFCOUNTER_IPACKETS, 1); nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); } sc->rl_ldata.rl_rx_prodidx = nic_i; kring->nr_hwtail = nm_i; kring->nr_kflags &= ~NKR_PENDINTR; } /* * Second part: skip past packets that userspace has released. */ nm_i = kring->nr_hwcur; if (nm_i != head) { nic_i = netmap_idx_k2n(kring, nm_i); for (n = 0; nm_i != head; n++) { struct netmap_slot *slot = &ring->slot[nm_i]; uint64_t paddr; void *addr = PNMB(na, slot, &paddr); struct rl_desc *desc = &sc->rl_ldata.rl_rx_list[nic_i]; int cmd = NETMAP_BUF_SIZE(na) | RL_RDESC_CMD_OWN; if (addr == NETMAP_BUF_BASE(na)) /* bad buf */ goto ring_reset; if (nic_i == lim) /* mark end of ring */ cmd |= RL_RDESC_CMD_EOR; if (slot->flags & NS_BUF_CHANGED) { /* buffer has changed, reload map */ desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); netmap_reload_map(na, sc->rl_ldata.rl_rx_mtag, rxd[nic_i].rx_dmamap, addr); slot->flags &= ~NS_BUF_CHANGED; } desc->rl_cmdstat = htole32(cmd); bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, rxd[nic_i].rx_dmamap, BUS_DMASYNC_PREREAD); nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); } kring->nr_hwcur = head; bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag, sc->rl_ldata.rl_rx_list_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); } /* tell userspace that there might be new packets */ nm_rxsync_finalize(kring); return 0; ring_reset: return netmap_ring_reinit(kring); } /* * Additional routines to init the tx and rx rings. * In other drivers we do that inline in the main code. */ static void re_netmap_tx_init(struct rl_softc *sc) { struct rl_txdesc *txd; struct rl_desc *desc; int i, n; struct netmap_adapter *na = NA(sc->rl_ifp); struct netmap_slot *slot; slot = netmap_reset(na, NR_TX, 0, 0); /* slot is NULL if we are not in native netmap mode */ if (!slot) return; /* in netmap mode, overwrite addresses and maps */ txd = sc->rl_ldata.rl_tx_desc; desc = sc->rl_ldata.rl_tx_list; n = sc->rl_ldata.rl_tx_desc_cnt; /* l points in the netmap ring, i points in the NIC ring */ for (i = 0; i < n; i++) { uint64_t paddr; int l = netmap_idx_n2k(&na->tx_rings[0], i); void *addr = PNMB(na, slot + l, &paddr); desc[i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); desc[i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); netmap_load_map(na, sc->rl_ldata.rl_tx_mtag, txd[i].tx_dmamap, addr); } } static void re_netmap_rx_init(struct rl_softc *sc) { struct netmap_adapter *na = NA(sc->rl_ifp); struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0); struct rl_desc *desc = sc->rl_ldata.rl_rx_list; uint32_t cmdstat; uint32_t nic_i, max_avail; uint32_t const n = sc->rl_ldata.rl_rx_desc_cnt; if (!slot) return; /* * Do not release the slots owned by userspace, * and also keep one empty. */ max_avail = n - 1 - nm_kr_rxspace(&na->rx_rings[0]); for (nic_i = 0; nic_i < n; nic_i++) { void *addr; uint64_t paddr; uint32_t nm_i = netmap_idx_n2k(&na->rx_rings[0], nic_i); addr = PNMB(na, slot + nm_i, &paddr); netmap_reload_map(na, sc->rl_ldata.rl_rx_mtag, sc->rl_ldata.rl_rx_desc[nic_i].rx_dmamap, addr); bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, sc->rl_ldata.rl_rx_desc[nic_i].rx_dmamap, BUS_DMASYNC_PREREAD); desc[nic_i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); desc[nic_i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); cmdstat = NETMAP_BUF_SIZE(na); if (nic_i == n - 1) /* mark the end of ring */ cmdstat |= RL_RDESC_CMD_EOR; if (nic_i < max_avail) cmdstat |= RL_RDESC_CMD_OWN; desc[nic_i].rl_cmdstat = htole32(cmdstat); } } static void re_netmap_attach(struct rl_softc *sc) { struct netmap_adapter na; bzero(&na, sizeof(na)); na.ifp = sc->rl_ifp; na.na_flags = NAF_BDG_MAYSLEEP; na.num_tx_desc = sc->rl_ldata.rl_tx_desc_cnt; na.num_rx_desc = sc->rl_ldata.rl_rx_desc_cnt; na.nm_txsync = re_netmap_txsync; na.nm_rxsync = re_netmap_rxsync; na.nm_register = re_netmap_reg; na.num_tx_rings = na.num_rx_rings = 1; netmap_attach(&na); } /* end of file */ Index: stable/10/sys/dev/netmap/netmap.c =================================================================== --- stable/10/sys/dev/netmap/netmap.c (revision 278778) +++ stable/10/sys/dev/netmap/netmap.c (revision 278779) @@ -1,3071 +1,3093 @@ /* * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ * * This module supports memory mapped access to network devices, * see netmap(4). * * The module uses a large, memory pool allocated by the kernel * and accessible as mmapped memory by multiple userspace threads/processes. * The memory pool contains packet buffers and "netmap rings", * i.e. user-accessible copies of the interface's queues. * * Access to the network card works like this: * 1. a process/thread issues one or more open() on /dev/netmap, to create * select()able file descriptor on which events are reported. * 2. on each descriptor, the process issues an ioctl() to identify * the interface that should report events to the file descriptor. * 3. on each descriptor, the process issues an mmap() request to * map the shared memory region within the process' address space. * The list of interesting queues is indicated by a location in * the shared memory region. * 4. using the functions in the netmap(4) userspace API, a process * can look up the occupation state of a queue, access memory buffers, * and retrieve received packets or enqueue packets to transmit. * 5. using some ioctl()s the process can synchronize the userspace view * of the queue with the actual status in the kernel. This includes both * receiving the notification of new packets, and transmitting new * packets on the output interface. * 6. select() or poll() can be used to wait for events on individual * transmit or receive queues (or all queues for a given interface). * SYNCHRONIZATION (USER) The netmap rings and data structures may be shared among multiple user threads or even independent processes. Any synchronization among those threads/processes is delegated to the threads themselves. Only one thread at a time can be in a system call on the same netmap ring. The OS does not enforce this and only guarantees against system crashes in case of invalid usage. LOCKING (INTERNAL) Within the kernel, access to the netmap rings is protected as follows: - a spinlock on each ring, to handle producer/consumer races on RX rings attached to the host stack (against multiple host threads writing from the host stack to the same ring), and on 'destination' rings attached to a VALE switch (i.e. RX rings in VALE ports, and TX rings in NIC/host ports) protecting multiple active senders for the same destination) - an atomic variable to guarantee that there is at most one instance of *_*xsync() on the ring at any time. For rings connected to user file descriptors, an atomic_test_and_set() protects this, and the lock on the ring is not actually used. For NIC RX rings connected to a VALE switch, an atomic_test_and_set() is also used to prevent multiple executions (the driver might indeed already guarantee this). For NIC TX rings connected to a VALE switch, the lock arbitrates access to the queue (both when allocating buffers and when pushing them out). - *xsync() should be protected against initializations of the card. On FreeBSD most devices have the reset routine protected by a RING lock (ixgbe, igb, em) or core lock (re). lem is missing the RING protection on rx_reset(), this should be added. On linux there is an external lock on the tx path, which probably also arbitrates access to the reset routine. XXX to be revised - a per-interface core_lock protecting access from the host stack while interfaces may be detached from netmap mode. XXX there should be no need for this lock if we detach the interfaces only while they are down. --- VALE SWITCH --- NMG_LOCK() serializes all modifications to switches and ports. A switch cannot be deleted until all ports are gone. For each switch, an SX lock (RWlock on linux) protects deletion of ports. When configuring or deleting a new port, the lock is acquired in exclusive mode (after holding NMG_LOCK). When forwarding, the lock is acquired in shared mode (without NMG_LOCK). The lock is held throughout the entire forwarding cycle, during which the thread may incur in a page fault. Hence it is important that sleepable shared locks are used. On the rx ring, the per-port lock is grabbed initially to reserve a number of slot in the ring, then the lock is released, packets are copied from source to destination, and then the lock is acquired again and the receive ring is updated. (A similar thing is done on the tx ring for NIC and host stack ports attached to the switch) */ /* --- internals ---- * * Roadmap to the code that implements the above. * * > 1. a process/thread issues one or more open() on /dev/netmap, to create * > select()able file descriptor on which events are reported. * * Internally, we allocate a netmap_priv_d structure, that will be * initialized on ioctl(NIOCREGIF). * * os-specific: * FreeBSD: netmap_open (netmap_freebsd.c). The priv is * per-thread. * linux: linux_netmap_open (netmap_linux.c). The priv is * per-open. * * > 2. on each descriptor, the process issues an ioctl() to identify * > the interface that should report events to the file descriptor. * * Implemented by netmap_ioctl(), NIOCREGIF case, with nmr->nr_cmd==0. * Most important things happen in netmap_get_na() and * netmap_do_regif(), called from there. Additional details can be * found in the comments above those functions. * * In all cases, this action creates/takes-a-reference-to a * netmap_*_adapter describing the port, and allocates a netmap_if * and all necessary netmap rings, filling them with netmap buffers. * * In this phase, the sync callbacks for each ring are set (these are used * in steps 5 and 6 below). The callbacks depend on the type of adapter. * The adapter creation/initialization code puts them in the * netmap_adapter (fields na->nm_txsync and na->nm_rxsync). Then, they * are copied from there to the netmap_kring's during netmap_do_regif(), by * the nm_krings_create() callback. All the nm_krings_create callbacks * actually call netmap_krings_create() to perform this and the other * common stuff. netmap_krings_create() also takes care of the host rings, * if needed, by setting their sync callbacks appropriately. * * Additional actions depend on the kind of netmap_adapter that has been * registered: * * - netmap_hw_adapter: [netmap.c] * This is a system netdev/ifp with native netmap support. * The ifp is detached from the host stack by redirecting: * - transmissions (from the network stack) to netmap_transmit() * - receive notifications to the nm_notify() callback for * this adapter. The callback is normally netmap_notify(), unless * the ifp is attached to a bridge using bwrap, in which case it * is netmap_bwrap_intr_notify(). * * - netmap_generic_adapter: [netmap_generic.c] * A system netdev/ifp without native netmap support. * * (the decision about native/non native support is taken in * netmap_get_hw_na(), called by netmap_get_na()) * * - netmap_vp_adapter [netmap_vale.c] * Returned by netmap_get_bdg_na(). * This is a persistent or ephemeral VALE port. Ephemeral ports * are created on the fly if they don't already exist, and are * always attached to a bridge. * Persistent VALE ports must must be created seperately, and i * then attached like normal NICs. The NIOCREGIF we are examining * will find them only if they had previosly been created and * attached (see VALE_CTL below). * * - netmap_pipe_adapter [netmap_pipe.c] * Returned by netmap_get_pipe_na(). * Both pipe ends are created, if they didn't already exist. * * - netmap_monitor_adapter [netmap_monitor.c] * Returned by netmap_get_monitor_na(). * If successful, the nm_sync callbacks of the monitored adapter * will be intercepted by the returned monitor. * * - netmap_bwrap_adapter [netmap_vale.c] * Cannot be obtained in this way, see VALE_CTL below * * * os-specific: * linux: we first go through linux_netmap_ioctl() to * adapt the FreeBSD interface to the linux one. * * * > 3. on each descriptor, the process issues an mmap() request to * > map the shared memory region within the process' address space. * > The list of interesting queues is indicated by a location in * > the shared memory region. * * os-specific: * FreeBSD: netmap_mmap_single (netmap_freebsd.c). * linux: linux_netmap_mmap (netmap_linux.c). * * > 4. using the functions in the netmap(4) userspace API, a process * > can look up the occupation state of a queue, access memory buffers, * > and retrieve received packets or enqueue packets to transmit. * * these actions do not involve the kernel. * * > 5. using some ioctl()s the process can synchronize the userspace view * > of the queue with the actual status in the kernel. This includes both * > receiving the notification of new packets, and transmitting new * > packets on the output interface. * * These are implemented in netmap_ioctl(), NIOCTXSYNC and NIOCRXSYNC * cases. They invoke the nm_sync callbacks on the netmap_kring * structures, as initialized in step 2 and maybe later modified * by a monitor. Monitors, however, will always call the original * callback before doing anything else. * * * > 6. select() or poll() can be used to wait for events on individual * > transmit or receive queues (or all queues for a given interface). * * Implemented in netmap_poll(). This will call the same nm_sync() * callbacks as in step 5 above. * * os-specific: * linux: we first go through linux_netmap_poll() to adapt * the FreeBSD interface to the linux one. * * * ---- VALE_CTL ----- * * VALE switches are controlled by issuing a NIOCREGIF with a non-null * nr_cmd in the nmreq structure. These subcommands are handled by * netmap_bdg_ctl() in netmap_vale.c. Persistent VALE ports are created * and destroyed by issuing the NETMAP_BDG_NEWIF and NETMAP_BDG_DELIF * subcommands, respectively. * * Any network interface known to the system (including a persistent VALE * port) can be attached to a VALE switch by issuing the * NETMAP_BDG_ATTACH subcommand. After the attachment, persistent VALE ports * look exactly like ephemeral VALE ports (as created in step 2 above). The * attachment of other interfaces, instead, requires the creation of a * netmap_bwrap_adapter. Moreover, the attached interface must be put in * netmap mode. This may require the creation of a netmap_generic_adapter if * we have no native support for the interface, or if generic adapters have * been forced by sysctl. * * Both persistent VALE ports and bwraps are handled by netmap_get_bdg_na(), * called by nm_bdg_ctl_attach(), and discriminated by the nm_bdg_attach() * callback. In the case of the bwrap, the callback creates the * netmap_bwrap_adapter. The initialization of the bwrap is then * completed by calling netmap_do_regif() on it, in the nm_bdg_ctl() * callback (netmap_bwrap_bdg_ctl in netmap_vale.c). * A generic adapter for the wrapped ifp will be created if needed, when * netmap_get_bdg_na() calls netmap_get_hw_na(). * * * ---- DATAPATHS ----- * * -= SYSTEM DEVICE WITH NATIVE SUPPORT =- * * na == NA(ifp) == netmap_hw_adapter created in DEVICE_netmap_attach() * * - tx from netmap userspace: * concurrently: * 1) ioctl(NIOCTXSYNC)/netmap_poll() in process context * kring->nm_sync() == DEVICE_netmap_txsync() * 2) device interrupt handler * na->nm_notify() == netmap_notify() * - rx from netmap userspace: * concurrently: * 1) ioctl(NIOCRXSYNC)/netmap_poll() in process context * kring->nm_sync() == DEVICE_netmap_rxsync() * 2) device interrupt handler * na->nm_notify() == netmap_notify() * - tx from host stack * concurrently: * 1) host stack * netmap_transmit() * na->nm_notify == netmap_notify() * 2) ioctl(NIOCRXSYNC)/netmap_poll() in process context * kring->nm_sync() == netmap_rxsync_from_host_compat * netmap_rxsync_from_host(na, NULL, NULL) * - tx to host stack * ioctl(NIOCTXSYNC)/netmap_poll() in process context * kring->nm_sync() == netmap_txsync_to_host_compat * netmap_txsync_to_host(na) * NM_SEND_UP() * FreeBSD: na->if_input() == ?? XXX * linux: netif_rx() with NM_MAGIC_PRIORITY_RX * * * * -= SYSTEM DEVICE WITH GENERIC SUPPORT =- * * * * -= VALE PORT =- * * * * -= NETMAP PIPE =- * * * * -= SYSTEM DEVICE WITH NATIVE SUPPORT, CONNECTED TO VALE, NO HOST RINGS =- * * * * -= SYSTEM DEVICE WITH NATIVE SUPPORT, CONNECTED TO VALE, WITH HOST RINGS =- * * * * -= SYSTEM DEVICE WITH GENERIC SUPPORT, CONNECTED TO VALE, NO HOST RINGS =- * * * * -= SYSTEM DEVICE WITH GENERIC SUPPORT, CONNECTED TO VALE, WITH HOST RINGS =- * * * */ /* * OS-specific code that is used only within this file. * Other OS-specific code that must be accessed by drivers * is present in netmap_kern.h */ #if defined(__FreeBSD__) #include /* prerequisite */ #include #include #include /* defines used in kernel.h */ #include /* types used in module initialization */ #include /* cdevsw struct, UID, GID */ #include /* FIONBIO */ #include #include /* struct socket */ #include #include #include #include /* sockaddrs */ #include #include #include #include #include #include #include /* BIOCIMMEDIATE */ #include /* bus_dmamap_* */ #include #include /* reduce conditional code */ // linux API, use for the knlist in FreeBSD -#define init_waitqueue_head(x) knlist_init_mtx(&(x)->si_note, NULL) +/* use a private mutex for the knlist */ +#define init_waitqueue_head(x) do { \ + struct mtx *m = &(x)->m; \ + mtx_init(m, "nm_kn_lock", NULL, MTX_DEF); \ + knlist_init_mtx(&(x)->si.si_note, m); \ + } while (0) -void freebsd_selwakeup(struct selinfo *si, int pri); +#define OS_selrecord(a, b) selrecord(a, &((b)->si)) #define OS_selwakeup(a, b) freebsd_selwakeup(a, b) #elif defined(linux) #include "bsd_glue.h" #elif defined(__APPLE__) #warning OSX support is only partial #include "osx_glue.h" #else #error Unsupported platform #endif /* unsupported */ /* * common headers */ #include #include #include MALLOC_DEFINE(M_NETMAP, "netmap", "Network memory map"); /* * The following variables are used by the drivers and replicate * fields in the global memory pool. They only refer to buffers * used by physical interfaces. */ u_int netmap_total_buffers; u_int netmap_buf_size; char *netmap_buffer_base; /* also address of an invalid buffer */ /* user-controlled variables */ int netmap_verbose; static int netmap_no_timestamp; /* don't timestamp on rxsync */ SYSCTL_NODE(_dev, OID_AUTO, netmap, CTLFLAG_RW, 0, "Netmap args"); SYSCTL_INT(_dev_netmap, OID_AUTO, verbose, CTLFLAG_RW, &netmap_verbose, 0, "Verbose mode"); SYSCTL_INT(_dev_netmap, OID_AUTO, no_timestamp, CTLFLAG_RW, &netmap_no_timestamp, 0, "no_timestamp"); int netmap_mitigate = 1; SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, ""); int netmap_no_pendintr = 1; SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr, CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets."); int netmap_txsync_retry = 2; SYSCTL_INT(_dev_netmap, OID_AUTO, txsync_retry, CTLFLAG_RW, &netmap_txsync_retry, 0 , "Number of txsync loops in bridge's flush."); int netmap_adaptive_io = 0; SYSCTL_INT(_dev_netmap, OID_AUTO, adaptive_io, CTLFLAG_RW, &netmap_adaptive_io, 0 , "Adaptive I/O on paravirt"); int netmap_flags = 0; /* debug flags */ int netmap_fwd = 0; /* force transparent mode */ int netmap_mmap_unreg = 0; /* allow mmap of unregistered fds */ /* * netmap_admode selects the netmap mode to use. * Invalid values are reset to NETMAP_ADMODE_BEST */ enum { NETMAP_ADMODE_BEST = 0, /* use native, fallback to generic */ NETMAP_ADMODE_NATIVE, /* either native or none */ NETMAP_ADMODE_GENERIC, /* force generic */ NETMAP_ADMODE_LAST }; static int netmap_admode = NETMAP_ADMODE_BEST; int netmap_generic_mit = 100*1000; /* Generic mitigation interval in nanoseconds. */ int netmap_generic_ringsize = 1024; /* Generic ringsize. */ int netmap_generic_rings = 1; /* number of queues in generic. */ SYSCTL_INT(_dev_netmap, OID_AUTO, flags, CTLFLAG_RW, &netmap_flags, 0 , ""); SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0 , ""); SYSCTL_INT(_dev_netmap, OID_AUTO, mmap_unreg, CTLFLAG_RW, &netmap_mmap_unreg, 0, ""); SYSCTL_INT(_dev_netmap, OID_AUTO, admode, CTLFLAG_RW, &netmap_admode, 0 , ""); SYSCTL_INT(_dev_netmap, OID_AUTO, generic_mit, CTLFLAG_RW, &netmap_generic_mit, 0 , ""); SYSCTL_INT(_dev_netmap, OID_AUTO, generic_ringsize, CTLFLAG_RW, &netmap_generic_ringsize, 0 , ""); SYSCTL_INT(_dev_netmap, OID_AUTO, generic_rings, CTLFLAG_RW, &netmap_generic_rings, 0 , ""); NMG_LOCK_T netmap_global_lock; static void nm_kr_get(struct netmap_kring *kr) { while (NM_ATOMIC_TEST_AND_SET(&kr->nr_busy)) tsleep(kr, 0, "NM_KR_GET", 4); } /* * mark the ring as stopped, and run through the locks * to make sure other users get to see it. */ static void netmap_disable_ring(struct netmap_kring *kr) { kr->nkr_stopped = 1; nm_kr_get(kr); mtx_lock(&kr->q_lock); mtx_unlock(&kr->q_lock); nm_kr_put(kr); } /* stop or enable a single tx ring */ void netmap_set_txring(struct netmap_adapter *na, u_int ring_id, int stopped) { if (stopped) netmap_disable_ring(na->tx_rings + ring_id); else na->tx_rings[ring_id].nkr_stopped = 0; /* nofify that the stopped state has changed. This is currently *only used by bwrap to propagate the state to its own krings. * (see netmap_bwrap_intr_notify). */ na->nm_notify(na, ring_id, NR_TX, NAF_DISABLE_NOTIFY); } /* stop or enable a single rx ring */ void netmap_set_rxring(struct netmap_adapter *na, u_int ring_id, int stopped) { if (stopped) netmap_disable_ring(na->rx_rings + ring_id); else na->rx_rings[ring_id].nkr_stopped = 0; /* nofify that the stopped state has changed. This is currently *only used by bwrap to propagate the state to its own krings. * (see netmap_bwrap_intr_notify). */ na->nm_notify(na, ring_id, NR_RX, NAF_DISABLE_NOTIFY); } /* stop or enable all the rings of na */ void netmap_set_all_rings(struct netmap_adapter *na, int stopped) { int i; u_int ntx, nrx; if (!nm_netmap_on(na)) return; ntx = netmap_real_tx_rings(na); nrx = netmap_real_rx_rings(na); for (i = 0; i < ntx; i++) { netmap_set_txring(na, i, stopped); } for (i = 0; i < nrx; i++) { netmap_set_rxring(na, i, stopped); } } /* * Convenience function used in drivers. Waits for current txsync()s/rxsync()s * to finish and prevents any new one from starting. Call this before turning * netmap mode off, or before removing the harware rings (e.g., on module * onload). As a rule of thumb for linux drivers, this should be placed near * each napi_disable(). */ void netmap_disable_all_rings(struct ifnet *ifp) { netmap_set_all_rings(NA(ifp), 1 /* stopped */); } /* * Convenience function used in drivers. Re-enables rxsync and txsync on the * adapter's rings In linux drivers, this should be placed near each * napi_enable(). */ void netmap_enable_all_rings(struct ifnet *ifp) { netmap_set_all_rings(NA(ifp), 0 /* enabled */); } /* * generic bound_checking function */ u_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg) { u_int oldv = *v; const char *op = NULL; if (dflt < lo) dflt = lo; if (dflt > hi) dflt = hi; if (oldv < lo) { *v = dflt; op = "Bump"; } else if (oldv > hi) { *v = hi; op = "Clamp"; } if (op && msg) printf("%s %s to %d (was %d)\n", op, msg, *v, oldv); return *v; } /* * packet-dump function, user-supplied or static buffer. * The destination buffer must be at least 30+4*len */ const char * nm_dump_buf(char *p, int len, int lim, char *dst) { static char _dst[8192]; int i, j, i0; static char hex[] ="0123456789abcdef"; char *o; /* output position */ #define P_HI(x) hex[((x) & 0xf0)>>4] #define P_LO(x) hex[((x) & 0xf)] #define P_C(x) ((x) >= 0x20 && (x) <= 0x7e ? (x) : '.') if (!dst) dst = _dst; if (lim <= 0 || lim > len) lim = len; o = dst; sprintf(o, "buf 0x%p len %d lim %d\n", p, len, lim); o += strlen(o); /* hexdump routine */ for (i = 0; i < lim; ) { sprintf(o, "%5d: ", i); o += strlen(o); memset(o, ' ', 48); i0 = i; for (j=0; j < 16 && i < lim; i++, j++) { o[j*3] = P_HI(p[i]); o[j*3+1] = P_LO(p[i]); } i = i0; for (j=0; j < 16 && i < lim; i++, j++) o[j + 48] = P_C(p[i]); o[j+48] = '\n'; o += j+49; } *o = '\0'; #undef P_HI #undef P_LO #undef P_C return dst; } /* * Fetch configuration from the device, to cope with dynamic * reconfigurations after loading the module. */ /* call with NMG_LOCK held */ int netmap_update_config(struct netmap_adapter *na) { u_int txr, txd, rxr, rxd; txr = txd = rxr = rxd = 0; - if (na->nm_config) { - na->nm_config(na, &txr, &txd, &rxr, &rxd); - } else { + if (na->nm_config == NULL || + na->nm_config(na, &txr, &txd, &rxr, &rxd)) { /* take whatever we had at init time */ txr = na->num_tx_rings; txd = na->num_tx_desc; rxr = na->num_rx_rings; rxd = na->num_rx_desc; } if (na->num_tx_rings == txr && na->num_tx_desc == txd && na->num_rx_rings == rxr && na->num_rx_desc == rxd) return 0; /* nothing changed */ if (netmap_verbose || na->active_fds > 0) { D("stored config %s: txring %d x %d, rxring %d x %d", na->name, na->num_tx_rings, na->num_tx_desc, na->num_rx_rings, na->num_rx_desc); D("new config %s: txring %d x %d, rxring %d x %d", na->name, txr, txd, rxr, rxd); } if (na->active_fds == 0) { D("configuration changed (but fine)"); na->num_tx_rings = txr; na->num_tx_desc = txd; na->num_rx_rings = rxr; na->num_rx_desc = rxd; return 0; } D("configuration changed while active, this is bad..."); return 1; } /* kring->nm_sync callback for the host tx ring */ static int netmap_txsync_to_host_compat(struct netmap_kring *kring, int flags) { (void)flags; /* unused */ netmap_txsync_to_host(kring->na); return 0; } /* kring->nm_sync callback for the host rx ring */ static int netmap_rxsync_from_host_compat(struct netmap_kring *kring, int flags) { (void)flags; /* unused */ netmap_rxsync_from_host(kring->na, NULL, NULL); return 0; } /* create the krings array and initialize the fields common to all adapters. * The array layout is this: * * +----------+ * na->tx_rings ----->| | \ * | | } na->num_tx_ring * | | / * +----------+ * | | host tx kring * na->rx_rings ----> +----------+ * | | \ * | | } na->num_rx_rings * | | / * +----------+ * | | host rx kring * +----------+ * na->tailroom ----->| | \ * | | } tailroom bytes * | | / * +----------+ * * Note: for compatibility, host krings are created even when not needed. * The tailroom space is currently used by vale ports for allocating leases. */ /* call with NMG_LOCK held */ int netmap_krings_create(struct netmap_adapter *na, u_int tailroom) { u_int i, len, ndesc; struct netmap_kring *kring; u_int ntx, nrx; /* account for the (possibly fake) host rings */ ntx = na->num_tx_rings + 1; nrx = na->num_rx_rings + 1; len = (ntx + nrx) * sizeof(struct netmap_kring) + tailroom; na->tx_rings = malloc((size_t)len, M_DEVBUF, M_NOWAIT | M_ZERO); if (na->tx_rings == NULL) { D("Cannot allocate krings"); return ENOMEM; } na->rx_rings = na->tx_rings + ntx; /* * All fields in krings are 0 except the one initialized below. * but better be explicit on important kring fields. */ ndesc = na->num_tx_desc; for (i = 0; i < ntx; i++) { /* Transmit rings */ kring = &na->tx_rings[i]; bzero(kring, sizeof(*kring)); kring->na = na; kring->ring_id = i; kring->nkr_num_slots = ndesc; if (i < na->num_tx_rings) { kring->nm_sync = na->nm_txsync; } else if (i == na->num_tx_rings) { kring->nm_sync = netmap_txsync_to_host_compat; } /* * IMPORTANT: Always keep one slot empty. */ kring->rhead = kring->rcur = kring->nr_hwcur = 0; kring->rtail = kring->nr_hwtail = ndesc - 1; snprintf(kring->name, sizeof(kring->name) - 1, "%s TX%d", na->name, i); ND("ktx %s h %d c %d t %d", kring->name, kring->rhead, kring->rcur, kring->rtail); mtx_init(&kring->q_lock, "nm_txq_lock", NULL, MTX_DEF); init_waitqueue_head(&kring->si); } ndesc = na->num_rx_desc; for (i = 0; i < nrx; i++) { /* Receive rings */ kring = &na->rx_rings[i]; bzero(kring, sizeof(*kring)); kring->na = na; kring->ring_id = i; kring->nkr_num_slots = ndesc; if (i < na->num_rx_rings) { kring->nm_sync = na->nm_rxsync; } else if (i == na->num_rx_rings) { kring->nm_sync = netmap_rxsync_from_host_compat; } kring->rhead = kring->rcur = kring->nr_hwcur = 0; kring->rtail = kring->nr_hwtail = 0; snprintf(kring->name, sizeof(kring->name) - 1, "%s RX%d", na->name, i); ND("krx %s h %d c %d t %d", kring->name, kring->rhead, kring->rcur, kring->rtail); mtx_init(&kring->q_lock, "nm_rxq_lock", NULL, MTX_DEF); init_waitqueue_head(&kring->si); } init_waitqueue_head(&na->tx_si); init_waitqueue_head(&na->rx_si); na->tailroom = na->rx_rings + nrx; return 0; } +#ifdef __FreeBSD__ +static void +netmap_knlist_destroy(NM_SELINFO_T *si) +{ + /* XXX kqueue(9) needed; these will mirror knlist_init. */ + knlist_delete(&si->si.si_note, curthread, 0 /* not locked */ ); + knlist_destroy(&si->si.si_note); + /* now we don't need the mutex anymore */ + mtx_destroy(&si->m); +} +#endif /* __FreeBSD__ */ + + /* undo the actions performed by netmap_krings_create */ /* call with NMG_LOCK held */ void netmap_krings_delete(struct netmap_adapter *na) { struct netmap_kring *kring = na->tx_rings; /* we rely on the krings layout described above */ for ( ; kring != na->tailroom; kring++) { mtx_destroy(&kring->q_lock); + netmap_knlist_destroy(&kring->si); } free(na->tx_rings, M_DEVBUF); na->tx_rings = na->rx_rings = na->tailroom = NULL; } /* * Destructor for NIC ports. They also have an mbuf queue * on the rings connected to the host so we need to purge * them first. */ /* call with NMG_LOCK held */ static void netmap_hw_krings_delete(struct netmap_adapter *na) { struct mbq *q = &na->rx_rings[na->num_rx_rings].rx_queue; ND("destroy sw mbq with len %d", mbq_len(q)); mbq_purge(q); mbq_safe_destroy(q); netmap_krings_delete(na); } /* create a new netmap_if for a newly registered fd. * If this is the first registration of the adapter, * also create the netmap rings and their in-kernel view, * the netmap krings. */ /* call with NMG_LOCK held */ static struct netmap_if* netmap_if_new(struct netmap_adapter *na) { struct netmap_if *nifp; if (netmap_update_config(na)) { /* configuration mismatch, report and fail */ return NULL; } if (na->active_fds) /* already registered */ goto final; /* create and init the krings arrays. * Depending on the adapter, this may also create * the netmap rings themselves */ if (na->nm_krings_create(na)) return NULL; /* create all missing netmap rings */ if (netmap_mem_rings_create(na)) goto cleanup; final: /* in all cases, create a new netmap if */ nifp = netmap_mem_if_new(na); if (nifp == NULL) goto cleanup; return (nifp); cleanup: if (na->active_fds == 0) { netmap_mem_rings_delete(na); na->nm_krings_delete(na); } return NULL; } /* grab a reference to the memory allocator, if we don't have one already. The * reference is taken from the netmap_adapter registered with the priv. */ /* call with NMG_LOCK held */ static int netmap_get_memory_locked(struct netmap_priv_d* p) { struct netmap_mem_d *nmd; int error = 0; if (p->np_na == NULL) { if (!netmap_mmap_unreg) return ENODEV; /* for compatibility with older versions of the API * we use the global allocator when no interface has been * registered */ nmd = &nm_mem; } else { nmd = p->np_na->nm_mem; } if (p->np_mref == NULL) { error = netmap_mem_finalize(nmd, p->np_na); if (!error) p->np_mref = nmd; } else if (p->np_mref != nmd) { /* a virtual port has been registered, but previous * syscalls already used the global allocator. * We cannot continue */ error = ENODEV; } return error; } /* call with NMG_LOCK *not* held */ int netmap_get_memory(struct netmap_priv_d* p) { int error; NMG_LOCK(); error = netmap_get_memory_locked(p); NMG_UNLOCK(); return error; } /* call with NMG_LOCK held */ static int netmap_have_memory_locked(struct netmap_priv_d* p) { return p->np_mref != NULL; } /* call with NMG_LOCK held */ static void netmap_drop_memory_locked(struct netmap_priv_d* p) { if (p->np_mref) { netmap_mem_deref(p->np_mref, p->np_na); p->np_mref = NULL; } } /* * Call nm_register(ifp,0) to stop netmap mode on the interface and * revert to normal operation. * The second argument is the nifp to work on. In some cases it is * not attached yet to the netmap_priv_d so we need to pass it as * a separate argument. */ /* call with NMG_LOCK held */ static void netmap_do_unregif(struct netmap_priv_d *priv, struct netmap_if *nifp) { struct netmap_adapter *na = priv->np_na; NMG_LOCK_ASSERT(); na->active_fds--; if (na->active_fds <= 0) { /* last instance */ if (netmap_verbose) D("deleting last instance for %s", na->name); /* * (TO CHECK) This function is only called * when the last reference to this file descriptor goes * away. This means we cannot have any pending poll() * or interrupt routine operating on the structure. * XXX The file may be closed in a thread while * another thread is using it. * Linux keeps the file opened until the last reference * by any outstanding ioctl/poll or mmap is gone. * FreeBSD does not track mmap()s (but we do) and * wakes up any sleeping poll(). Need to check what * happens if the close() occurs while a concurrent * syscall is running. */ na->nm_register(na, 0); /* off, clear flags */ /* Wake up any sleeping threads. netmap_poll will * then return POLLERR * XXX The wake up now must happen during *_down(), when * we order all activities to stop. -gl */ - /* XXX kqueue(9) needed; these will mirror knlist_init. */ - /* knlist_destroy(&na->tx_si.si_note); */ - /* knlist_destroy(&na->rx_si.si_note); */ + netmap_knlist_destroy(&na->tx_si); + netmap_knlist_destroy(&na->rx_si); /* delete rings and buffers */ netmap_mem_rings_delete(na); na->nm_krings_delete(na); } /* delete the nifp */ netmap_mem_if_delete(na, nifp); } /* call with NMG_LOCK held */ static __inline int nm_tx_si_user(struct netmap_priv_d *priv) { return (priv->np_na != NULL && (priv->np_txqlast - priv->np_txqfirst > 1)); } /* call with NMG_LOCK held */ static __inline int nm_rx_si_user(struct netmap_priv_d *priv) { return (priv->np_na != NULL && (priv->np_rxqlast - priv->np_rxqfirst > 1)); } /* * Destructor of the netmap_priv_d, called when the fd has * no active open() and mmap(). Also called in error paths. * * returns 1 if this is the last instance and we can free priv */ /* call with NMG_LOCK held */ int netmap_dtor_locked(struct netmap_priv_d *priv) { struct netmap_adapter *na = priv->np_na; #ifdef __FreeBSD__ /* * np_refcount is the number of active mmaps on * this file descriptor */ if (--priv->np_refcount > 0) { return 0; } #endif /* __FreeBSD__ */ if (!na) { return 1; //XXX is it correct? } netmap_do_unregif(priv, priv->np_nifp); priv->np_nifp = NULL; netmap_drop_memory_locked(priv); if (priv->np_na) { if (nm_tx_si_user(priv)) na->tx_si_users--; if (nm_rx_si_user(priv)) na->rx_si_users--; netmap_adapter_put(na); priv->np_na = NULL; } return 1; } /* call with NMG_LOCK *not* held */ void netmap_dtor(void *data) { struct netmap_priv_d *priv = data; int last_instance; NMG_LOCK(); last_instance = netmap_dtor_locked(priv); NMG_UNLOCK(); if (last_instance) { bzero(priv, sizeof(*priv)); /* for safety */ free(priv, M_DEVBUF); } } /* * Handlers for synchronization of the queues from/to the host. * Netmap has two operating modes: * - in the default mode, the rings connected to the host stack are * just another ring pair managed by userspace; * - in transparent mode (XXX to be defined) incoming packets * (from the host or the NIC) are marked as NS_FORWARD upon * arrival, and the user application has a chance to reset the * flag for packets that should be dropped. * On the RXSYNC or poll(), packets in RX rings between * kring->nr_kcur and ring->cur with NS_FORWARD still set are moved * to the other side. * The transfer NIC --> host is relatively easy, just encapsulate * into mbufs and we are done. The host --> NIC side is slightly * harder because there might not be room in the tx ring so it * might take a while before releasing the buffer. */ /* * pass a chain of buffers to the host stack as coming from 'dst' * We do not need to lock because the queue is private. */ static void netmap_send_up(struct ifnet *dst, struct mbq *q) { struct mbuf *m; /* send packets up, outside the lock */ while ((m = mbq_dequeue(q)) != NULL) { if (netmap_verbose & NM_VERB_HOST) D("sending up pkt %p size %d", m, MBUF_LEN(m)); NM_SEND_UP(dst, m); } mbq_destroy(q); } /* * put a copy of the buffers marked NS_FORWARD into an mbuf chain. * Take packets from hwcur to ring->head marked NS_FORWARD (or forced) * and pass them up. Drop remaining packets in the unlikely event * of an mbuf shortage. */ static void netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force) { u_int const lim = kring->nkr_num_slots - 1; u_int const head = kring->ring->head; u_int n; struct netmap_adapter *na = kring->na; for (n = kring->nr_hwcur; n != head; n = nm_next(n, lim)) { struct mbuf *m; struct netmap_slot *slot = &kring->ring->slot[n]; if ((slot->flags & NS_FORWARD) == 0 && !force) continue; if (slot->len < 14 || slot->len > NETMAP_BUF_SIZE(na)) { RD(5, "bad pkt at %d len %d", n, slot->len); continue; } slot->flags &= ~NS_FORWARD; // XXX needed ? /* XXX TODO: adapt to the case of a multisegment packet */ m = m_devget(NMB(na, slot), slot->len, 0, na->ifp, NULL); if (m == NULL) break; mbq_enqueue(q, m); } } /* * Send to the NIC rings packets marked NS_FORWARD between * kring->nr_hwcur and kring->rhead * Called under kring->rx_queue.lock on the sw rx ring, */ static u_int netmap_sw_to_nic(struct netmap_adapter *na) { struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings]; struct netmap_slot *rxslot = kring->ring->slot; u_int i, rxcur = kring->nr_hwcur; u_int const head = kring->rhead; u_int const src_lim = kring->nkr_num_slots - 1; u_int sent = 0; /* scan rings to find space, then fill as much as possible */ for (i = 0; i < na->num_tx_rings; i++) { struct netmap_kring *kdst = &na->tx_rings[i]; struct netmap_ring *rdst = kdst->ring; u_int const dst_lim = kdst->nkr_num_slots - 1; /* XXX do we trust ring or kring->rcur,rtail ? */ for (; rxcur != head && !nm_ring_empty(rdst); rxcur = nm_next(rxcur, src_lim) ) { struct netmap_slot *src, *dst, tmp; u_int dst_cur = rdst->cur; src = &rxslot[rxcur]; if ((src->flags & NS_FORWARD) == 0 && !netmap_fwd) continue; sent++; dst = &rdst->slot[dst_cur]; tmp = *src; src->buf_idx = dst->buf_idx; src->flags = NS_BUF_CHANGED; dst->buf_idx = tmp.buf_idx; dst->len = tmp.len; dst->flags = NS_BUF_CHANGED; rdst->cur = nm_next(dst_cur, dst_lim); } /* if (sent) XXX txsync ? */ } return sent; } /* * netmap_txsync_to_host() passes packets up. We are called from a * system call in user process context, and the only contention * can be among multiple user threads erroneously calling * this routine concurrently. */ void netmap_txsync_to_host(struct netmap_adapter *na) { struct netmap_kring *kring = &na->tx_rings[na->num_tx_rings]; struct netmap_ring *ring = kring->ring; u_int const lim = kring->nkr_num_slots - 1; u_int const head = kring->rhead; struct mbq q; /* Take packets from hwcur to head and pass them up. * force head = cur since netmap_grab_packets() stops at head * In case of no buffers we give up. At the end of the loop, * the queue is drained in all cases. */ mbq_init(&q); ring->cur = head; netmap_grab_packets(kring, &q, 1 /* force */); ND("have %d pkts in queue", mbq_len(&q)); kring->nr_hwcur = head; kring->nr_hwtail = head + lim; if (kring->nr_hwtail > lim) kring->nr_hwtail -= lim + 1; nm_txsync_finalize(kring); netmap_send_up(na->ifp, &q); } /* * rxsync backend for packets coming from the host stack. * They have been put in kring->rx_queue by netmap_transmit(). * We protect access to the kring using kring->rx_queue.lock * * This routine also does the selrecord if called from the poll handler * (we know because td != NULL). * * NOTE: on linux, selrecord() is defined as a macro and uses pwait * as an additional hidden argument. * returns the number of packets delivered to tx queues in * transparent mode, or a negative value if error */ int netmap_rxsync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait) { struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings]; struct netmap_ring *ring = kring->ring; u_int nm_i, n; u_int const lim = kring->nkr_num_slots - 1; u_int const head = kring->rhead; int ret = 0; struct mbq *q = &kring->rx_queue; (void)pwait; /* disable unused warnings */ (void)td; mbq_lock(q); /* First part: import newly received packets */ n = mbq_len(q); if (n) { /* grab packets from the queue */ struct mbuf *m; uint32_t stop_i; nm_i = kring->nr_hwtail; stop_i = nm_prev(nm_i, lim); while ( nm_i != stop_i && (m = mbq_dequeue(q)) != NULL ) { int len = MBUF_LEN(m); struct netmap_slot *slot = &ring->slot[nm_i]; m_copydata(m, 0, len, NMB(na, slot)); ND("nm %d len %d", nm_i, len); if (netmap_verbose) D("%s", nm_dump_buf(NMB(na, slot),len, 128, NULL)); slot->len = len; slot->flags = kring->nkr_slot_flags; nm_i = nm_next(nm_i, lim); m_freem(m); } kring->nr_hwtail = nm_i; } /* * Second part: skip past packets that userspace has released. */ nm_i = kring->nr_hwcur; if (nm_i != head) { /* something was released */ if (netmap_fwd || kring->ring->flags & NR_FORWARD) ret = netmap_sw_to_nic(na); kring->nr_hwcur = head; } nm_rxsync_finalize(kring); /* access copies of cur,tail in the kring */ if (kring->rcur == kring->rtail && td) /* no bufs available */ - selrecord(td, &kring->si); + OS_selrecord(td, &kring->si); mbq_unlock(q); return ret; } /* Get a netmap adapter for the port. * * If it is possible to satisfy the request, return 0 * with *na containing the netmap adapter found. * Otherwise return an error code, with *na containing NULL. * * When the port is attached to a bridge, we always return * EBUSY. * Otherwise, if the port is already bound to a file descriptor, * then we unconditionally return the existing adapter into *na. * In all the other cases, we return (into *na) either native, * generic or NULL, according to the following table: * * native_support * active_fds dev.netmap.admode YES NO * ------------------------------------------------------- * >0 * NA(ifp) NA(ifp) * * 0 NETMAP_ADMODE_BEST NATIVE GENERIC * 0 NETMAP_ADMODE_NATIVE NATIVE NULL * 0 NETMAP_ADMODE_GENERIC GENERIC GENERIC * */ int netmap_get_hw_na(struct ifnet *ifp, struct netmap_adapter **na) { /* generic support */ int i = netmap_admode; /* Take a snapshot. */ int error = 0; struct netmap_adapter *prev_na; struct netmap_generic_adapter *gna; *na = NULL; /* default */ /* reset in case of invalid value */ if (i < NETMAP_ADMODE_BEST || i >= NETMAP_ADMODE_LAST) i = netmap_admode = NETMAP_ADMODE_BEST; if (NETMAP_CAPABLE(ifp)) { prev_na = NA(ifp); /* If an adapter already exists, return it if * there are active file descriptors or if * netmap is not forced to use generic * adapters. */ if (NETMAP_OWNED_BY_ANY(prev_na) || i != NETMAP_ADMODE_GENERIC || prev_na->na_flags & NAF_FORCE_NATIVE #ifdef WITH_PIPES /* ugly, but we cannot allow an adapter switch * if some pipe is referring to this one */ || prev_na->na_next_pipe > 0 #endif ) { *na = prev_na; return 0; } } /* If there isn't native support and netmap is not allowed * to use generic adapters, we cannot satisfy the request. */ if (!NETMAP_CAPABLE(ifp) && i == NETMAP_ADMODE_NATIVE) return EOPNOTSUPP; /* Otherwise, create a generic adapter and return it, * saving the previously used netmap adapter, if any. * * Note that here 'prev_na', if not NULL, MUST be a * native adapter, and CANNOT be a generic one. This is * true because generic adapters are created on demand, and * destroyed when not used anymore. Therefore, if the adapter * currently attached to an interface 'ifp' is generic, it * must be that * (NA(ifp)->active_fds > 0 || NETMAP_OWNED_BY_KERN(NA(ifp))). * Consequently, if NA(ifp) is generic, we will enter one of * the branches above. This ensures that we never override * a generic adapter with another generic adapter. */ prev_na = NA(ifp); error = generic_netmap_attach(ifp); if (error) return error; *na = NA(ifp); gna = (struct netmap_generic_adapter*)NA(ifp); gna->prev = prev_na; /* save old na */ if (prev_na != NULL) { ifunit_ref(ifp->if_xname); // XXX add a refcount ? netmap_adapter_get(prev_na); } ND("Created generic NA %p (prev %p)", gna, gna->prev); return 0; } /* * MUST BE CALLED UNDER NMG_LOCK() * * Get a refcounted reference to a netmap adapter attached * to the interface specified by nmr. * This is always called in the execution of an ioctl(). * * Return ENXIO if the interface specified by the request does * not exist, ENOTSUP if netmap is not supported by the interface, * EBUSY if the interface is already attached to a bridge, * EINVAL if parameters are invalid, ENOMEM if needed resources * could not be allocated. * If successful, hold a reference to the netmap adapter. * * No reference is kept on the real interface, which may then * disappear at any time. */ int netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na, int create) { struct ifnet *ifp = NULL; int error = 0; struct netmap_adapter *ret = NULL; *na = NULL; /* default return value */ NMG_LOCK_ASSERT(); /* we cascade through all possibile types of netmap adapter. * All netmap_get_*_na() functions return an error and an na, * with the following combinations: * * error na * 0 NULL type doesn't match * !0 NULL type matches, but na creation/lookup failed * 0 !NULL type matches and na created/found * !0 !NULL impossible */ /* try to see if this is a monitor port */ error = netmap_get_monitor_na(nmr, na, create); if (error || *na != NULL) return error; /* try to see if this is a pipe port */ error = netmap_get_pipe_na(nmr, na, create); if (error || *na != NULL) return error; /* try to see if this is a bridge port */ error = netmap_get_bdg_na(nmr, na, create); if (error) return error; if (*na != NULL) /* valid match in netmap_get_bdg_na() */ goto pipes; /* * This must be a hardware na, lookup the name in the system. * Note that by hardware we actually mean "it shows up in ifconfig". * This may still be a tap, a veth/epair, or even a * persistent VALE port. */ ifp = ifunit_ref(nmr->nr_name); if (ifp == NULL) { return ENXIO; } error = netmap_get_hw_na(ifp, &ret); if (error) goto out; *na = ret; netmap_adapter_get(ret); pipes: /* * If we are opening a pipe whose parent was not in netmap mode, * we have to allocate the pipe array now. * XXX get rid of this clumsiness (2014-03-15) */ error = netmap_pipe_alloc(*na, nmr); out: if (error && ret != NULL) netmap_adapter_put(ret); if (ifp) if_rele(ifp); /* allow live unloading of drivers modules */ return error; } /* * validate parameters on entry for *_txsync() * Returns ring->cur if ok, or something >= kring->nkr_num_slots * in case of error. * * rhead, rcur and rtail=hwtail are stored from previous round. * hwcur is the next packet to send to the ring. * * We want * hwcur <= *rhead <= head <= cur <= tail = *rtail <= hwtail * * hwcur, rhead, rtail and hwtail are reliable */ u_int nm_txsync_prologue(struct netmap_kring *kring) { struct netmap_ring *ring = kring->ring; u_int head = ring->head; /* read only once */ u_int cur = ring->cur; /* read only once */ u_int n = kring->nkr_num_slots; ND(5, "%s kcur %d ktail %d head %d cur %d tail %d", kring->name, kring->nr_hwcur, kring->nr_hwtail, ring->head, ring->cur, ring->tail); #if 1 /* kernel sanity checks; but we can trust the kring. */ if (kring->nr_hwcur >= n || kring->rhead >= n || kring->rtail >= n || kring->nr_hwtail >= n) goto error; #endif /* kernel sanity checks */ /* * user sanity checks. We only use 'cur', * A, B, ... are possible positions for cur: * * 0 A cur B tail C n-1 * 0 D tail E cur F n-1 * * B, F, D are valid. A, C, E are wrong */ if (kring->rtail >= kring->rhead) { /* want rhead <= head <= rtail */ if (head < kring->rhead || head > kring->rtail) goto error; /* and also head <= cur <= rtail */ if (cur < head || cur > kring->rtail) goto error; } else { /* here rtail < rhead */ /* we need head outside rtail .. rhead */ if (head > kring->rtail && head < kring->rhead) goto error; /* two cases now: head <= rtail or head >= rhead */ if (head <= kring->rtail) { /* want head <= cur <= rtail */ if (cur < head || cur > kring->rtail) goto error; } else { /* head >= rhead */ /* cur must be outside rtail..head */ if (cur > kring->rtail && cur < head) goto error; } } if (ring->tail != kring->rtail) { RD(5, "tail overwritten was %d need %d", ring->tail, kring->rtail); ring->tail = kring->rtail; } kring->rhead = head; kring->rcur = cur; return head; error: RD(5, "%s kring error: hwcur %d rcur %d hwtail %d cur %d tail %d", kring->name, kring->nr_hwcur, kring->rcur, kring->nr_hwtail, cur, ring->tail); return n; } /* * validate parameters on entry for *_rxsync() * Returns ring->head if ok, kring->nkr_num_slots on error. * * For a valid configuration, * hwcur <= head <= cur <= tail <= hwtail * * We only consider head and cur. * hwcur and hwtail are reliable. * */ u_int nm_rxsync_prologue(struct netmap_kring *kring) { struct netmap_ring *ring = kring->ring; uint32_t const n = kring->nkr_num_slots; uint32_t head, cur; ND("%s kc %d kt %d h %d c %d t %d", kring->name, kring->nr_hwcur, kring->nr_hwtail, ring->head, ring->cur, ring->tail); /* * Before storing the new values, we should check they do not * move backwards. However: * - head is not an issue because the previous value is hwcur; * - cur could in principle go back, however it does not matter * because we are processing a brand new rxsync() */ cur = kring->rcur = ring->cur; /* read only once */ head = kring->rhead = ring->head; /* read only once */ #if 1 /* kernel sanity checks */ if (kring->nr_hwcur >= n || kring->nr_hwtail >= n) goto error; #endif /* kernel sanity checks */ /* user sanity checks */ if (kring->nr_hwtail >= kring->nr_hwcur) { /* want hwcur <= rhead <= hwtail */ if (head < kring->nr_hwcur || head > kring->nr_hwtail) goto error; /* and also rhead <= rcur <= hwtail */ if (cur < head || cur > kring->nr_hwtail) goto error; } else { /* we need rhead outside hwtail..hwcur */ if (head < kring->nr_hwcur && head > kring->nr_hwtail) goto error; /* two cases now: head <= hwtail or head >= hwcur */ if (head <= kring->nr_hwtail) { /* want head <= cur <= hwtail */ if (cur < head || cur > kring->nr_hwtail) goto error; } else { /* cur must be outside hwtail..head */ if (cur < head && cur > kring->nr_hwtail) goto error; } } if (ring->tail != kring->rtail) { RD(5, "%s tail overwritten was %d need %d", kring->name, ring->tail, kring->rtail); ring->tail = kring->rtail; } return head; error: RD(5, "kring error: hwcur %d rcur %d hwtail %d head %d cur %d tail %d", kring->nr_hwcur, kring->rcur, kring->nr_hwtail, kring->rhead, kring->rcur, ring->tail); return n; } /* * Error routine called when txsync/rxsync detects an error. * Can't do much more than resetting head =cur = hwcur, tail = hwtail * Return 1 on reinit. * * This routine is only called by the upper half of the kernel. * It only reads hwcur (which is changed only by the upper half, too) * and hwtail (which may be changed by the lower half, but only on * a tx ring and only to increase it, so any error will be recovered * on the next call). For the above, we don't strictly need to call * it under lock. */ int netmap_ring_reinit(struct netmap_kring *kring) { struct netmap_ring *ring = kring->ring; u_int i, lim = kring->nkr_num_slots - 1; int errors = 0; // XXX KASSERT nm_kr_tryget RD(10, "called for %s", kring->name); // XXX probably wrong to trust userspace kring->rhead = ring->head; kring->rcur = ring->cur; kring->rtail = ring->tail; if (ring->cur > lim) errors++; if (ring->head > lim) errors++; if (ring->tail > lim) errors++; for (i = 0; i <= lim; i++) { u_int idx = ring->slot[i].buf_idx; u_int len = ring->slot[i].len; if (idx < 2 || idx >= netmap_total_buffers) { RD(5, "bad index at slot %d idx %d len %d ", i, idx, len); ring->slot[i].buf_idx = 0; ring->slot[i].len = 0; } else if (len > NETMAP_BUF_SIZE(kring->na)) { ring->slot[i].len = 0; RD(5, "bad len at slot %d idx %d len %d", i, idx, len); } } if (errors) { RD(10, "total %d errors", errors); RD(10, "%s reinit, cur %d -> %d tail %d -> %d", kring->name, ring->cur, kring->nr_hwcur, ring->tail, kring->nr_hwtail); ring->head = kring->rhead = kring->nr_hwcur; ring->cur = kring->rcur = kring->nr_hwcur; ring->tail = kring->rtail = kring->nr_hwtail; } return (errors ? 1 : 0); } /* interpret the ringid and flags fields of an nmreq, by translating them * into a pair of intervals of ring indices: * * [priv->np_txqfirst, priv->np_txqlast) and * [priv->np_rxqfirst, priv->np_rxqlast) * */ int netmap_interp_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags) { struct netmap_adapter *na = priv->np_na; u_int j, i = ringid & NETMAP_RING_MASK; u_int reg = flags & NR_REG_MASK; if (reg == NR_REG_DEFAULT) { /* convert from old ringid to flags */ if (ringid & NETMAP_SW_RING) { reg = NR_REG_SW; } else if (ringid & NETMAP_HW_RING) { reg = NR_REG_ONE_NIC; } else { reg = NR_REG_ALL_NIC; } D("deprecated API, old ringid 0x%x -> ringid %x reg %d", ringid, i, reg); } switch (reg) { case NR_REG_ALL_NIC: case NR_REG_PIPE_MASTER: case NR_REG_PIPE_SLAVE: priv->np_txqfirst = 0; priv->np_txqlast = na->num_tx_rings; priv->np_rxqfirst = 0; priv->np_rxqlast = na->num_rx_rings; ND("%s %d %d", "ALL/PIPE", priv->np_rxqfirst, priv->np_rxqlast); break; case NR_REG_SW: case NR_REG_NIC_SW: if (!(na->na_flags & NAF_HOST_RINGS)) { D("host rings not supported"); return EINVAL; } priv->np_txqfirst = (reg == NR_REG_SW ? na->num_tx_rings : 0); priv->np_txqlast = na->num_tx_rings + 1; priv->np_rxqfirst = (reg == NR_REG_SW ? na->num_rx_rings : 0); priv->np_rxqlast = na->num_rx_rings + 1; ND("%s %d %d", reg == NR_REG_SW ? "SW" : "NIC+SW", priv->np_rxqfirst, priv->np_rxqlast); break; case NR_REG_ONE_NIC: if (i >= na->num_tx_rings && i >= na->num_rx_rings) { D("invalid ring id %d", i); return EINVAL; } /* if not enough rings, use the first one */ j = i; if (j >= na->num_tx_rings) j = 0; priv->np_txqfirst = j; priv->np_txqlast = j + 1; j = i; if (j >= na->num_rx_rings) j = 0; priv->np_rxqfirst = j; priv->np_rxqlast = j + 1; break; default: D("invalid regif type %d", reg); return EINVAL; } priv->np_flags = (flags & ~NR_REG_MASK) | reg; if (netmap_verbose) { D("%s: tx [%d,%d) rx [%d,%d) id %d", na->name, priv->np_txqfirst, priv->np_txqlast, priv->np_rxqfirst, priv->np_rxqlast, i); } return 0; } /* * Set the ring ID. For devices with a single queue, a request * for all rings is the same as a single ring. */ static int netmap_set_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags) { struct netmap_adapter *na = priv->np_na; int error; error = netmap_interp_ringid(priv, ringid, flags); if (error) { return error; } priv->np_txpoll = (ringid & NETMAP_NO_TX_POLL) ? 0 : 1; /* optimization: count the users registered for more than * one ring, which are the ones sleeping on the global queue. * The default netmap_notify() callback will then * avoid signaling the global queue if nobody is using it */ if (nm_tx_si_user(priv)) na->tx_si_users++; if (nm_rx_si_user(priv)) na->rx_si_users++; return 0; } /* * possibly move the interface to netmap-mode. * If success it returns a pointer to netmap_if, otherwise NULL. * This must be called with NMG_LOCK held. * * The following na callbacks are called in the process: * * na->nm_config() [by netmap_update_config] * (get current number and size of rings) * * We have a generic one for linux (netmap_linux_config). * The bwrap has to override this, since it has to forward * the request to the wrapped adapter (netmap_bwrap_config). * * XXX netmap_if_new calls this again (2014-03-15) * * na->nm_krings_create() [by netmap_if_new] * (create and init the krings array) * * One of the following: * * * netmap_hw_krings_create, (hw ports) * creates the standard layout for the krings * and adds the mbq (used for the host rings). * * * netmap_vp_krings_create (VALE ports) * add leases and scratchpads * * * netmap_pipe_krings_create (pipes) * create the krings and rings of both ends and * cross-link them * * * netmap_monitor_krings_create (monitors) * avoid allocating the mbq * * * netmap_bwrap_krings_create (bwraps) * create both the brap krings array, * the krings array of the wrapped adapter, and * (if needed) the fake array for the host adapter * * na->nm_register(, 1) * (put the adapter in netmap mode) * * This may be one of the following: * (XXX these should be either all *_register or all *_reg 2014-03-15) * * * netmap_hw_register (hw ports) * checks that the ifp is still there, then calls * the hardware specific callback; * * * netmap_vp_reg (VALE ports) * If the port is connected to a bridge, * set the NAF_NETMAP_ON flag under the * bridge write lock. * * * netmap_pipe_reg (pipes) * inform the other pipe end that it is no * longer responsibile for the lifetime of this * pipe end * * * netmap_monitor_reg (monitors) * intercept the sync callbacks of the monitored * rings * * * netmap_bwrap_register (bwraps) * cross-link the bwrap and hwna rings, * forward the request to the hwna, override * the hwna notify callback (to get the frames * coming from outside go through the bridge). * * XXX maybe netmap_if_new() should be merged with this (2014-03-15). * */ struct netmap_if * netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na, uint16_t ringid, uint32_t flags, int *err) { struct netmap_if *nifp = NULL; int error, need_mem = 0; NMG_LOCK_ASSERT(); /* ring configuration may have changed, fetch from the card */ netmap_update_config(na); priv->np_na = na; /* store the reference */ error = netmap_set_ringid(priv, ringid, flags); if (error) goto out; /* ensure allocators are ready */ need_mem = !netmap_have_memory_locked(priv); if (need_mem) { error = netmap_get_memory_locked(priv); ND("get_memory returned %d", error); if (error) goto out; } /* Allocate a netmap_if and, if necessary, all the netmap_ring's */ nifp = netmap_if_new(na); if (nifp == NULL) { /* allocation failed */ error = ENOMEM; goto out; } na->active_fds++; if (!nm_netmap_on(na)) { /* Netmap not active, set the card in netmap mode * and make it use the shared buffers. */ /* cache the allocator info in the na */ na->na_lut = netmap_mem_get_lut(na->nm_mem); ND("%p->na_lut == %p", na, na->na_lut); na->na_lut_objtotal = netmap_mem_get_buftotal(na->nm_mem); na->na_lut_objsize = netmap_mem_get_bufsize(na->nm_mem); error = na->nm_register(na, 1); /* mode on */ if (error) { netmap_do_unregif(priv, nifp); nifp = NULL; } } out: *err = error; if (error) { /* we should drop the allocator, but only * if we were the ones who grabbed it */ if (need_mem) netmap_drop_memory_locked(priv); priv->np_na = NULL; } if (nifp != NULL) { /* * advertise that the interface is ready bt setting ni_nifp. * The barrier is needed because readers (poll and *SYNC) * check for priv->np_nifp != NULL without locking */ wmb(); /* make sure previous writes are visible to all CPUs */ priv->np_nifp = nifp; } return nifp; } /* * ioctl(2) support for the "netmap" device. * * Following a list of accepted commands: * - NIOCGINFO * - SIOCGIFADDR just for convenience * - NIOCREGIF * - NIOCTXSYNC * - NIOCRXSYNC * * Return 0 on success, errno otherwise. */ int netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td) { struct netmap_priv_d *priv = NULL; struct nmreq *nmr = (struct nmreq *) data; struct netmap_adapter *na = NULL; int error; u_int i, qfirst, qlast; struct netmap_if *nifp; struct netmap_kring *krings; (void)dev; /* UNUSED */ (void)fflag; /* UNUSED */ if (cmd == NIOCGINFO || cmd == NIOCREGIF) { /* truncate name */ nmr->nr_name[sizeof(nmr->nr_name) - 1] = '\0'; if (nmr->nr_version != NETMAP_API) { D("API mismatch for %s got %d need %d", nmr->nr_name, nmr->nr_version, NETMAP_API); nmr->nr_version = NETMAP_API; } if (nmr->nr_version < NETMAP_MIN_API || nmr->nr_version > NETMAP_MAX_API) { return EINVAL; } } CURVNET_SET(TD_TO_VNET(td)); error = devfs_get_cdevpriv((void **)&priv); if (error) { CURVNET_RESTORE(); /* XXX ENOENT should be impossible, since the priv * is now created in the open */ return (error == ENOENT ? ENXIO : error); } switch (cmd) { case NIOCGINFO: /* return capabilities etc */ if (nmr->nr_cmd == NETMAP_BDG_LIST) { error = netmap_bdg_ctl(nmr, NULL); break; } NMG_LOCK(); do { /* memsize is always valid */ struct netmap_mem_d *nmd = &nm_mem; u_int memflags; if (nmr->nr_name[0] != '\0') { /* get a refcount */ error = netmap_get_na(nmr, &na, 1 /* create */); if (error) break; nmd = na->nm_mem; /* get memory allocator */ } error = netmap_mem_get_info(nmd, &nmr->nr_memsize, &memflags, &nmr->nr_arg2); if (error) break; if (na == NULL) /* only memory info */ break; nmr->nr_offset = 0; nmr->nr_rx_slots = nmr->nr_tx_slots = 0; netmap_update_config(na); nmr->nr_rx_rings = na->num_rx_rings; nmr->nr_tx_rings = na->num_tx_rings; nmr->nr_rx_slots = na->num_rx_desc; nmr->nr_tx_slots = na->num_tx_desc; netmap_adapter_put(na); } while (0); NMG_UNLOCK(); break; case NIOCREGIF: /* possibly attach/detach NIC and VALE switch */ i = nmr->nr_cmd; if (i == NETMAP_BDG_ATTACH || i == NETMAP_BDG_DETACH || i == NETMAP_BDG_VNET_HDR || i == NETMAP_BDG_NEWIF || i == NETMAP_BDG_DELIF) { error = netmap_bdg_ctl(nmr, NULL); break; } else if (i != 0) { D("nr_cmd must be 0 not %d", i); error = EINVAL; break; } /* protect access to priv from concurrent NIOCREGIF */ NMG_LOCK(); do { u_int memflags; if (priv->np_na != NULL) { /* thread already registered */ error = EBUSY; break; } /* find the interface and a reference */ error = netmap_get_na(nmr, &na, 1 /* create */); /* keep reference */ if (error) break; if (NETMAP_OWNED_BY_KERN(na)) { netmap_adapter_put(na); error = EBUSY; break; } nifp = netmap_do_regif(priv, na, nmr->nr_ringid, nmr->nr_flags, &error); if (!nifp) { /* reg. failed, release priv and ref */ netmap_adapter_put(na); priv->np_nifp = NULL; break; } priv->np_td = td; // XXX kqueue, debugging only /* return the offset of the netmap_if object */ nmr->nr_rx_rings = na->num_rx_rings; nmr->nr_tx_rings = na->num_tx_rings; nmr->nr_rx_slots = na->num_rx_desc; nmr->nr_tx_slots = na->num_tx_desc; error = netmap_mem_get_info(na->nm_mem, &nmr->nr_memsize, &memflags, &nmr->nr_arg2); if (error) { netmap_adapter_put(na); break; } if (memflags & NETMAP_MEM_PRIVATE) { *(uint32_t *)(uintptr_t)&nifp->ni_flags |= NI_PRIV_MEM; } priv->np_txsi = (priv->np_txqlast - priv->np_txqfirst > 1) ? &na->tx_si : &na->tx_rings[priv->np_txqfirst].si; priv->np_rxsi = (priv->np_rxqlast - priv->np_rxqfirst > 1) ? &na->rx_si : &na->rx_rings[priv->np_rxqfirst].si; if (nmr->nr_arg3) { D("requested %d extra buffers", nmr->nr_arg3); nmr->nr_arg3 = netmap_extra_alloc(na, &nifp->ni_bufs_head, nmr->nr_arg3); D("got %d extra buffers", nmr->nr_arg3); } nmr->nr_offset = netmap_mem_if_offset(na->nm_mem, nifp); } while (0); NMG_UNLOCK(); break; case NIOCTXSYNC: case NIOCRXSYNC: nifp = priv->np_nifp; if (nifp == NULL) { error = ENXIO; break; } - rmb(); /* make sure following reads are not from cache */ + mb(); /* make sure following reads are not from cache */ na = priv->np_na; /* we have a reference */ if (na == NULL) { D("Internal error: nifp != NULL && na == NULL"); error = ENXIO; break; } if (!nm_netmap_on(na)) { error = ENXIO; break; } if (cmd == NIOCTXSYNC) { krings = na->tx_rings; qfirst = priv->np_txqfirst; qlast = priv->np_txqlast; } else { krings = na->rx_rings; qfirst = priv->np_rxqfirst; qlast = priv->np_rxqlast; } for (i = qfirst; i < qlast; i++) { struct netmap_kring *kring = krings + i; if (nm_kr_tryget(kring)) { error = EBUSY; goto out; } if (cmd == NIOCTXSYNC) { if (netmap_verbose & NM_VERB_TXSYNC) D("pre txsync ring %d cur %d hwcur %d", i, kring->ring->cur, kring->nr_hwcur); if (nm_txsync_prologue(kring) >= kring->nkr_num_slots) { netmap_ring_reinit(kring); } else { kring->nm_sync(kring, NAF_FORCE_RECLAIM); } if (netmap_verbose & NM_VERB_TXSYNC) D("post txsync ring %d cur %d hwcur %d", i, kring->ring->cur, kring->nr_hwcur); } else { kring->nm_sync(kring, NAF_FORCE_READ); microtime(&na->rx_rings[i].ring->ts); } nm_kr_put(kring); } break; case NIOCCONFIG: error = netmap_bdg_config(nmr); break; #ifdef __FreeBSD__ case FIONBIO: case FIOASYNC: ND("FIONBIO/FIOASYNC are no-ops"); break; case BIOCIMMEDIATE: case BIOCGHDRCMPLT: case BIOCSHDRCMPLT: case BIOCSSEESENT: D("ignore BIOCIMMEDIATE/BIOCSHDRCMPLT/BIOCSHDRCMPLT/BIOCSSEESENT"); break; default: /* allow device-specific ioctls */ { struct ifnet *ifp = ifunit_ref(nmr->nr_name); if (ifp == NULL) { error = ENXIO; } else { struct socket so; bzero(&so, sizeof(so)); so.so_vnet = ifp->if_vnet; // so->so_proto not null. error = ifioctl(&so, cmd, data, td); if_rele(ifp); } break; } #else /* linux */ default: error = EOPNOTSUPP; #endif /* linux */ } out: CURVNET_RESTORE(); return (error); } /* * select(2) and poll(2) handlers for the "netmap" device. * * Can be called for one or more queues. * Return true the event mask corresponding to ready events. * If there are no ready events, do a selrecord on either individual * selinfo or on the global one. * Device-dependent parts (locking and sync of tx/rx rings) * are done through callbacks. * * On linux, arguments are really pwait, the poll table, and 'td' is struct file * * The first one is remapped to pwait as selrecord() uses the name as an * hidden argument. */ int netmap_poll(struct cdev *dev, int events, struct thread *td) { struct netmap_priv_d *priv = NULL; struct netmap_adapter *na; struct netmap_kring *kring; u_int i, check_all_tx, check_all_rx, want_tx, want_rx, revents = 0; struct mbq q; /* packets from hw queues to host stack */ void *pwait = dev; /* linux compatibility */ int is_kevent = 0; /* * In order to avoid nested locks, we need to "double check" * txsync and rxsync if we decide to do a selrecord(). * retry_tx (and retry_rx, later) prevent looping forever. */ int retry_tx = 1, retry_rx = 1; (void)pwait; mbq_init(&q); /* * XXX kevent has curthread->tp_fop == NULL, * so devfs_get_cdevpriv() fails. We circumvent this by passing * priv as the first argument, which is also useful to avoid * the selrecord() which are not necessary in that case. */ if (devfs_get_cdevpriv((void **)&priv) != 0) { is_kevent = 1; if (netmap_verbose) D("called from kevent"); priv = (struct netmap_priv_d *)dev; } if (priv == NULL) return POLLERR; if (priv->np_nifp == NULL) { D("No if registered"); return POLLERR; } rmb(); /* make sure following reads are not from cache */ na = priv->np_na; if (!nm_netmap_on(na)) return POLLERR; if (netmap_verbose & 0x8000) D("device %s events 0x%x", na->name, events); want_tx = events & (POLLOUT | POLLWRNORM); want_rx = events & (POLLIN | POLLRDNORM); /* * check_all_{tx|rx} are set if the card has more than one queue AND * the file descriptor is bound to all of them. If so, we sleep on * the "global" selinfo, otherwise we sleep on individual selinfo * (FreeBSD only allows two selinfo's per file descriptor). * The interrupt routine in the driver wake one or the other * (or both) depending on which clients are active. * * rxsync() is only called if we run out of buffers on a POLLIN. * txsync() is called if we run out of buffers on POLLOUT, or * there are pending packets to send. The latter can be disabled * passing NETMAP_NO_TX_POLL in the NIOCREG call. */ check_all_tx = nm_tx_si_user(priv); check_all_rx = nm_rx_si_user(priv); /* * We start with a lock free round which is cheap if we have * slots available. If this fails, then lock and call the sync * routines. */ for (i = priv->np_rxqfirst; want_rx && i < priv->np_rxqlast; i++) { kring = &na->rx_rings[i]; /* XXX compare ring->cur and kring->tail */ if (!nm_ring_empty(kring->ring)) { revents |= want_rx; want_rx = 0; /* also breaks the loop */ } } for (i = priv->np_txqfirst; want_tx && i < priv->np_txqlast; i++) { kring = &na->tx_rings[i]; /* XXX compare ring->cur and kring->tail */ if (!nm_ring_empty(kring->ring)) { revents |= want_tx; want_tx = 0; /* also breaks the loop */ } } /* * If we want to push packets out (priv->np_txpoll) or * want_tx is still set, we must issue txsync calls * (on all rings, to avoid that the tx rings stall). * XXX should also check cur != hwcur on the tx rings. * Fortunately, normal tx mode has np_txpoll set. */ if (priv->np_txpoll || want_tx) { /* * The first round checks if anyone is ready, if not * do a selrecord and another round to handle races. * want_tx goes to 0 if any space is found, and is * used to skip rings with no pending transmissions. */ flush_tx: for (i = priv->np_txqfirst; i < priv->np_txqlast; i++) { int found = 0; kring = &na->tx_rings[i]; if (!want_tx && kring->ring->cur == kring->nr_hwcur) continue; /* only one thread does txsync */ if (nm_kr_tryget(kring)) { /* either busy or stopped * XXX if the ring is stopped, sleeping would * be better. In current code, however, we only * stop the rings for brief intervals (2014-03-14) */ if (netmap_verbose) RD(2, "%p lost race on txring %d, ok", priv, i); continue; } if (nm_txsync_prologue(kring) >= kring->nkr_num_slots) { netmap_ring_reinit(kring); revents |= POLLERR; } else { if (kring->nm_sync(kring, 0)) revents |= POLLERR; } /* * If we found new slots, notify potential * listeners on the same ring. * Since we just did a txsync, look at the copies * of cur,tail in the kring. */ found = kring->rcur != kring->rtail; nm_kr_put(kring); if (found) { /* notify other listeners */ revents |= want_tx; want_tx = 0; na->nm_notify(na, i, NR_TX, 0); } } if (want_tx && retry_tx && !is_kevent) { - selrecord(td, check_all_tx ? + OS_selrecord(td, check_all_tx ? &na->tx_si : &na->tx_rings[priv->np_txqfirst].si); retry_tx = 0; goto flush_tx; } } /* * If want_rx is still set scan receive rings. * Do it on all rings because otherwise we starve. */ if (want_rx) { int send_down = 0; /* transparent mode */ /* two rounds here for race avoidance */ do_retry_rx: for (i = priv->np_rxqfirst; i < priv->np_rxqlast; i++) { int found = 0; kring = &na->rx_rings[i]; if (nm_kr_tryget(kring)) { if (netmap_verbose) RD(2, "%p lost race on rxring %d, ok", priv, i); continue; } /* * transparent mode support: collect packets * from the rxring(s). * XXX NR_FORWARD should only be read on * physical or NIC ports */ if (netmap_fwd ||kring->ring->flags & NR_FORWARD) { ND(10, "forwarding some buffers up %d to %d", kring->nr_hwcur, kring->ring->cur); netmap_grab_packets(kring, &q, netmap_fwd); } if (kring->nm_sync(kring, 0)) revents |= POLLERR; if (netmap_no_timestamp == 0 || kring->ring->flags & NR_TIMESTAMP) { microtime(&kring->ring->ts); } /* after an rxsync we can use kring->rcur, rtail */ found = kring->rcur != kring->rtail; nm_kr_put(kring); if (found) { revents |= want_rx; retry_rx = 0; na->nm_notify(na, i, NR_RX, 0); } } /* transparent mode XXX only during first pass ? */ if (na->na_flags & NAF_HOST_RINGS) { kring = &na->rx_rings[na->num_rx_rings]; if (check_all_rx && (netmap_fwd || kring->ring->flags & NR_FORWARD)) { /* XXX fix to use kring fields */ if (nm_ring_empty(kring->ring)) send_down = netmap_rxsync_from_host(na, td, dev); if (!nm_ring_empty(kring->ring)) revents |= want_rx; } } if (retry_rx && !is_kevent) - selrecord(td, check_all_rx ? + OS_selrecord(td, check_all_rx ? &na->rx_si : &na->rx_rings[priv->np_rxqfirst].si); if (send_down > 0 || retry_rx) { retry_rx = 0; if (send_down) goto flush_tx; /* and retry_rx */ else goto do_retry_rx; } } /* * Transparent mode: marked bufs on rx rings between * kring->nr_hwcur and ring->head * are passed to the other endpoint. * * In this mode we also scan the sw rxring, which in * turn passes packets up. * * XXX Transparent mode at the moment requires to bind all * rings to a single file descriptor. */ if (q.head && na->ifp != NULL) netmap_send_up(na->ifp, &q); return (revents); } /*-------------------- driver support routines -------------------*/ static int netmap_hw_krings_create(struct netmap_adapter *); /* default notify callback */ static int netmap_notify(struct netmap_adapter *na, u_int n_ring, enum txrx tx, int flags) { struct netmap_kring *kring; if (tx == NR_TX) { kring = na->tx_rings + n_ring; OS_selwakeup(&kring->si, PI_NET); /* optimization: avoid a wake up on the global * queue if nobody has registered for more * than one ring */ if (na->tx_si_users > 0) OS_selwakeup(&na->tx_si, PI_NET); } else { kring = na->rx_rings + n_ring; OS_selwakeup(&kring->si, PI_NET); /* optimization: same as above */ if (na->rx_si_users > 0) OS_selwakeup(&na->rx_si, PI_NET); } return 0; } /* called by all routines that create netmap_adapters. * Attach na to the ifp (if any) and provide defaults * for optional callbacks. Defaults assume that we * are creating an hardware netmap_adapter. */ int netmap_attach_common(struct netmap_adapter *na) { struct ifnet *ifp = na->ifp; if (na->num_tx_rings == 0 || na->num_rx_rings == 0) { D("%s: invalid rings tx %d rx %d", na->name, na->num_tx_rings, na->num_rx_rings); return EINVAL; } /* ifp is NULL for virtual adapters (bwrap, non-persistent VALE ports, * pipes, monitors). For bwrap we actually have a non-null ifp for * use by the external modules, but that is set after this * function has been called. * XXX this is ugly, maybe split this function in two (2014-03-14) */ if (ifp != NULL) { WNA(ifp) = na; /* the following is only needed for na that use the host port. * XXX do we have something similar for linux ? */ #ifdef __FreeBSD__ na->if_input = ifp->if_input; /* for netmap_send_up */ #endif /* __FreeBSD__ */ NETMAP_SET_CAPABLE(ifp); } if (na->nm_krings_create == NULL) { /* we assume that we have been called by a driver, * since other port types all provide their own * nm_krings_create */ na->nm_krings_create = netmap_hw_krings_create; na->nm_krings_delete = netmap_hw_krings_delete; } if (na->nm_notify == NULL) na->nm_notify = netmap_notify; na->active_fds = 0; if (na->nm_mem == NULL) /* use the global allocator */ na->nm_mem = &nm_mem; if (na->nm_bdg_attach == NULL) /* no special nm_bdg_attach callback. On VALE * attach, we need to interpose a bwrap */ na->nm_bdg_attach = netmap_bwrap_attach; return 0; } /* standard cleanup, called by all destructors */ void netmap_detach_common(struct netmap_adapter *na) { if (na->ifp != NULL) WNA(na->ifp) = NULL; /* XXX do we need this? */ if (na->tx_rings) { /* XXX should not happen */ D("freeing leftover tx_rings"); na->nm_krings_delete(na); } netmap_pipe_dealloc(na); if (na->na_flags & NAF_MEM_OWNER) netmap_mem_private_delete(na->nm_mem); bzero(na, sizeof(*na)); free(na, M_DEVBUF); } /* Wrapper for the register callback provided hardware drivers. * na->ifp == NULL means the the driver module has been * unloaded, so we cannot call into it. * Note that module unloading, in our patched linux drivers, * happens under NMG_LOCK and after having stopped all the * nic rings (see netmap_detach). This provides sufficient * protection for the other driver-provied callbacks * (i.e., nm_config and nm_*xsync), that therefore don't need * to wrapped. */ static int netmap_hw_register(struct netmap_adapter *na, int onoff) { struct netmap_hw_adapter *hwna = (struct netmap_hw_adapter*)na; if (na->ifp == NULL) return onoff ? ENXIO : 0; return hwna->nm_hw_register(na, onoff); } /* * Initialize a ``netmap_adapter`` object created by driver on attach. * We allocate a block of memory with room for a struct netmap_adapter * plus two sets of N+2 struct netmap_kring (where N is the number * of hardware rings): * krings 0..N-1 are for the hardware queues. * kring N is for the host stack queue * kring N+1 is only used for the selinfo for all queues. // XXX still true ? * Return 0 on success, ENOMEM otherwise. */ int netmap_attach(struct netmap_adapter *arg) { struct netmap_hw_adapter *hwna = NULL; // XXX when is arg == NULL ? struct ifnet *ifp = arg ? arg->ifp : NULL; if (arg == NULL || ifp == NULL) goto fail; hwna = malloc(sizeof(*hwna), M_DEVBUF, M_NOWAIT | M_ZERO); if (hwna == NULL) goto fail; hwna->up = *arg; hwna->up.na_flags |= NAF_HOST_RINGS; strncpy(hwna->up.name, ifp->if_xname, sizeof(hwna->up.name)); hwna->nm_hw_register = hwna->up.nm_register; hwna->up.nm_register = netmap_hw_register; if (netmap_attach_common(&hwna->up)) { free(hwna, M_DEVBUF); goto fail; } netmap_adapter_get(&hwna->up); #ifdef linux if (ifp->netdev_ops) { /* prepare a clone of the netdev ops */ #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 28) hwna->nm_ndo.ndo_start_xmit = ifp->netdev_ops; #else hwna->nm_ndo = *ifp->netdev_ops; #endif } hwna->nm_ndo.ndo_start_xmit = linux_netmap_start_xmit; if (ifp->ethtool_ops) { hwna->nm_eto = *ifp->ethtool_ops; } hwna->nm_eto.set_ringparam = linux_netmap_set_ringparam; #ifdef ETHTOOL_SCHANNELS hwna->nm_eto.set_channels = linux_netmap_set_channels; #endif if (arg->nm_config == NULL) { hwna->up.nm_config = netmap_linux_config; } #endif /* linux */ D("success for %s tx %d/%d rx %d/%d queues/slots", hwna->up.name, hwna->up.num_tx_rings, hwna->up.num_tx_desc, hwna->up.num_rx_rings, hwna->up.num_rx_desc ); return 0; fail: D("fail, arg %p ifp %p na %p", arg, ifp, hwna); if (ifp) netmap_detach(ifp); return (hwna ? EINVAL : ENOMEM); } void NM_DBG(netmap_adapter_get)(struct netmap_adapter *na) { if (!na) { return; } refcount_acquire(&na->na_refcount); } /* returns 1 iff the netmap_adapter is destroyed */ int NM_DBG(netmap_adapter_put)(struct netmap_adapter *na) { if (!na) return 1; if (!refcount_release(&na->na_refcount)) return 0; if (na->nm_dtor) na->nm_dtor(na); netmap_detach_common(na); return 1; } /* nm_krings_create callback for all hardware native adapters */ int netmap_hw_krings_create(struct netmap_adapter *na) { int ret = netmap_krings_create(na, 0); if (ret == 0) { /* initialize the mbq for the sw rx ring */ mbq_safe_init(&na->rx_rings[na->num_rx_rings].rx_queue); ND("initialized sw rx queue %d", na->num_rx_rings); } return ret; } /* * Called on module unload by the netmap-enabled drivers */ void netmap_detach(struct ifnet *ifp) { struct netmap_adapter *na = NA(ifp); if (!na) return; NMG_LOCK(); netmap_disable_all_rings(ifp); if (!netmap_adapter_put(na)) { /* someone is still using the adapter, * tell them that the interface is gone */ na->ifp = NULL; // XXX also clear NAF_NATIVE_ON ? na->na_flags &= ~NAF_NETMAP_ON; /* give them a chance to notice */ netmap_enable_all_rings(ifp); } NMG_UNLOCK(); } /* * Intercept packets from the network stack and pass them * to netmap as incoming packets on the 'software' ring. * * We only store packets in a bounded mbq and then copy them * in the relevant rxsync routine. * * We rely on the OS to make sure that the ifp and na do not go * away (typically the caller checks for IFF_DRV_RUNNING or the like). * In nm_register() or whenever there is a reinitialization, * we make sure to make the mode change visible here. */ int netmap_transmit(struct ifnet *ifp, struct mbuf *m) { struct netmap_adapter *na = NA(ifp); struct netmap_kring *kring; u_int len = MBUF_LEN(m); u_int error = ENOBUFS; struct mbq *q; int space; // XXX [Linux] we do not need this lock // if we follow the down/configure/up protocol -gl // mtx_lock(&na->core_lock); if (!nm_netmap_on(na)) { D("%s not in netmap mode anymore", na->name); error = ENXIO; goto done; } kring = &na->rx_rings[na->num_rx_rings]; q = &kring->rx_queue; // XXX reconsider long packets if we handle fragments if (len > NETMAP_BUF_SIZE(na)) { /* too long for us */ D("%s from_host, drop packet size %d > %d", na->name, len, NETMAP_BUF_SIZE(na)); goto done; } /* protect against rxsync_from_host(), netmap_sw_to_nic() * and maybe other instances of netmap_transmit (the latter * not possible on Linux). * Also avoid overflowing the queue. */ mbq_lock(q); space = kring->nr_hwtail - kring->nr_hwcur; if (space < 0) space += kring->nkr_num_slots; if (space + mbq_len(q) >= kring->nkr_num_slots - 1) { // XXX RD(10, "%s full hwcur %d hwtail %d qlen %d len %d m %p", na->name, kring->nr_hwcur, kring->nr_hwtail, mbq_len(q), len, m); } else { mbq_enqueue(q, m); ND(10, "%s %d bufs in queue len %d m %p", na->name, mbq_len(q), len, m); /* notify outside the lock */ m = NULL; error = 0; } mbq_unlock(q); done: if (m) m_freem(m); /* unconditionally wake up listeners */ na->nm_notify(na, na->num_rx_rings, NR_RX, 0); /* this is normally netmap_notify(), but for nics * connected to a bridge it is netmap_bwrap_intr_notify(), * that possibly forwards the frames through the switch */ return (error); } /* * netmap_reset() is called by the driver routines when reinitializing * a ring. The driver is in charge of locking to protect the kring. * If native netmap mode is not set just return NULL. */ struct netmap_slot * netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n, u_int new_cur) { struct netmap_kring *kring; int new_hwofs, lim; if (!nm_native_on(na)) { ND("interface not in native netmap mode"); return NULL; /* nothing to reinitialize */ } /* XXX note- in the new scheme, we are not guaranteed to be * under lock (e.g. when called on a device reset). * In this case, we should set a flag and do not trust too * much the values. In practice: TODO * - set a RESET flag somewhere in the kring * - do the processing in a conservative way * - let the *sync() fixup at the end. */ if (tx == NR_TX) { if (n >= na->num_tx_rings) return NULL; kring = na->tx_rings + n; // XXX check whether we should use hwcur or rcur new_hwofs = kring->nr_hwcur - new_cur; } else { if (n >= na->num_rx_rings) return NULL; kring = na->rx_rings + n; new_hwofs = kring->nr_hwtail - new_cur; } lim = kring->nkr_num_slots - 1; if (new_hwofs > lim) new_hwofs -= lim + 1; /* Always set the new offset value and realign the ring. */ if (netmap_verbose) D("%s %s%d hwofs %d -> %d, hwtail %d -> %d", na->name, tx == NR_TX ? "TX" : "RX", n, kring->nkr_hwofs, new_hwofs, kring->nr_hwtail, tx == NR_TX ? lim : kring->nr_hwtail); kring->nkr_hwofs = new_hwofs; if (tx == NR_TX) { kring->nr_hwtail = kring->nr_hwcur + lim; if (kring->nr_hwtail > lim) kring->nr_hwtail -= lim + 1; } #if 0 // def linux /* XXX check that the mappings are correct */ /* need ring_nr, adapter->pdev, direction */ buffer_info->dma = dma_map_single(&pdev->dev, addr, adapter->rx_buffer_len, DMA_FROM_DEVICE); if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { D("error mapping rx netmap buffer %d", i); // XXX fix error handling } #endif /* linux */ /* * Wakeup on the individual and global selwait * We do the wakeup here, but the ring is not yet reconfigured. * However, we are under lock so there are no races. */ na->nm_notify(na, n, tx, 0); return kring->ring->slot; } /* * Dispatch rx/tx interrupts to the netmap rings. * * "work_done" is non-null on the RX path, NULL for the TX path. * We rely on the OS to make sure that there is only one active * instance per queue, and that there is appropriate locking. * * The 'notify' routine depends on what the ring is attached to. * - for a netmap file descriptor, do a selwakeup on the individual * waitqueue, plus one on the global one if needed * (see netmap_notify) * - for a nic connected to a switch, call the proper forwarding routine * (see netmap_bwrap_intr_notify) */ void netmap_common_irq(struct ifnet *ifp, u_int q, u_int *work_done) { struct netmap_adapter *na = NA(ifp); struct netmap_kring *kring; q &= NETMAP_RING_MASK; if (netmap_verbose) { RD(5, "received %s queue %d", work_done ? "RX" : "TX" , q); } if (work_done) { /* RX path */ if (q >= na->num_rx_rings) return; // not a physical queue kring = na->rx_rings + q; kring->nr_kflags |= NKR_PENDINTR; // XXX atomic ? na->nm_notify(na, q, NR_RX, 0); *work_done = 1; /* do not fire napi again */ } else { /* TX path */ if (q >= na->num_tx_rings) return; // not a physical queue kring = na->tx_rings + q; na->nm_notify(na, q, NR_TX, 0); } } /* * Default functions to handle rx/tx interrupts from a physical device. * "work_done" is non-null on the RX path, NULL for the TX path. * * If the card is not in netmap mode, simply return 0, * so that the caller proceeds with regular processing. * Otherwise call netmap_common_irq() and return 1. * * If the card is connected to a netmap file descriptor, * do a selwakeup on the individual queue, plus one on the global one * if needed (multiqueue card _and_ there are multiqueue listeners), * and return 1. * * Finally, if called on rx from an interface connected to a switch, * calls the proper forwarding routine, and return 1. */ int netmap_rx_irq(struct ifnet *ifp, u_int q, u_int *work_done) { struct netmap_adapter *na = NA(ifp); /* * XXX emulated netmap mode sets NAF_SKIP_INTR so * we still use the regular driver even though the previous * check fails. It is unclear whether we should use * nm_native_on() here. */ if (!nm_netmap_on(na)) return 0; if (na->na_flags & NAF_SKIP_INTR) { ND("use regular interrupt"); return 0; } netmap_common_irq(ifp, q, work_done); return 1; } /* * Module loader and unloader * * netmap_init() creates the /dev/netmap device and initializes * all global variables. Returns 0 on success, errno on failure * (but there is no chance) * * netmap_fini() destroys everything. */ static struct cdev *netmap_dev; /* /dev/netmap character device. */ extern struct cdevsw netmap_cdevsw; void netmap_fini(void) { // XXX destroy_bridges() ? if (netmap_dev) destroy_dev(netmap_dev); netmap_mem_fini(); NMG_LOCK_DESTROY(); printf("netmap: unloaded module.\n"); } int netmap_init(void) { int error; NMG_LOCK_INIT(); error = netmap_mem_init(); if (error != 0) goto fail; - /* XXX could use make_dev_credv() to get error number */ - netmap_dev = make_dev(&netmap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0660, + /* + * MAKEDEV_ETERNAL_KLD avoids an expensive check on syscalls + * when the module is compiled in. + * XXX could use make_dev_credv() to get error number + */ + netmap_dev = make_dev_credf(MAKEDEV_ETERNAL_KLD, + &netmap_cdevsw, 0, NULL, UID_ROOT, GID_WHEEL, 0600, "netmap"); if (!netmap_dev) goto fail; netmap_init_bridges(); #ifdef __FreeBSD__ nm_vi_init_index(); #endif printf("netmap: loaded module\n"); return (0); fail: netmap_fini(); return (EINVAL); /* may be incorrect */ } Index: stable/10/sys/dev/netmap/netmap_freebsd.c =================================================================== --- stable/10/sys/dev/netmap/netmap_freebsd.c (revision 278778) +++ stable/10/sys/dev/netmap/netmap_freebsd.c (revision 278779) @@ -1,831 +1,832 @@ /* * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* $FreeBSD$ */ #include #include #include #include /* defines used in kernel.h */ #include /* POLLIN, POLLOUT */ #include /* types used in module initialization */ #include /* DEV_MODULE */ #include #include #include /* vtophys */ #include /* vtophys */ #include #include #include #include #include #include #include /* sockaddrs */ #include #include #include #include /* IFT_ETHER */ #include /* ether_ifdetach */ #include /* LLADDR */ #include /* bus_dmamap_* */ #include /* in6_cksum_pseudo() */ #include /* in_pseudo(), in_cksum_hdr() */ #include #include #include /* ======================== FREEBSD-SPECIFIC ROUTINES ================== */ rawsum_t nm_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum) { /* TODO XXX please use the FreeBSD implementation for this. */ uint16_t *words = (uint16_t *)data; int nw = len / 2; int i; for (i = 0; i < nw; i++) cur_sum += be16toh(words[i]); if (len & 1) cur_sum += (data[len-1] << 8); return cur_sum; } /* Fold a raw checksum: 'cur_sum' is in host byte order, while the * return value is in network byte order. */ uint16_t nm_csum_fold(rawsum_t cur_sum) { /* TODO XXX please use the FreeBSD implementation for this. */ while (cur_sum >> 16) cur_sum = (cur_sum & 0xFFFF) + (cur_sum >> 16); return htobe16((~cur_sum) & 0xFFFF); } uint16_t nm_csum_ipv4(struct nm_iphdr *iph) { #if 0 return in_cksum_hdr((void *)iph); #else return nm_csum_fold(nm_csum_raw((uint8_t*)iph, sizeof(struct nm_iphdr), 0)); #endif } void nm_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, size_t datalen, uint16_t *check) { #ifdef INET uint16_t pseudolen = datalen + iph->protocol; /* Compute and insert the pseudo-header cheksum. */ *check = in_pseudo(iph->saddr, iph->daddr, htobe16(pseudolen)); /* Compute the checksum on TCP/UDP header + payload * (includes the pseudo-header). */ *check = nm_csum_fold(nm_csum_raw(data, datalen, 0)); #else static int notsupported = 0; if (!notsupported) { notsupported = 1; D("inet4 segmentation not supported"); } #endif } void nm_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, size_t datalen, uint16_t *check) { #ifdef INET6 *check = in6_cksum_pseudo((void*)ip6h, datalen, ip6h->nexthdr, 0); *check = nm_csum_fold(nm_csum_raw(data, datalen, 0)); #else static int notsupported = 0; if (!notsupported) { notsupported = 1; D("inet6 segmentation not supported"); } #endif } /* * Intercept the rx routine in the standard device driver. * Second argument is non-zero to intercept, 0 to restore */ int netmap_catch_rx(struct netmap_adapter *na, int intercept) { struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; struct ifnet *ifp = na->ifp; if (intercept) { if (gna->save_if_input) { D("cannot intercept again"); return EINVAL; /* already set */ } gna->save_if_input = ifp->if_input; ifp->if_input = generic_rx_handler; } else { if (!gna->save_if_input){ D("cannot restore"); return EINVAL; /* not saved */ } ifp->if_input = gna->save_if_input; gna->save_if_input = NULL; } return 0; } /* * Intercept the packet steering routine in the tx path, * so that we can decide which queue is used for an mbuf. * Second argument is non-zero to intercept, 0 to restore. * On freebsd we just intercept if_transmit. */ void netmap_catch_tx(struct netmap_generic_adapter *gna, int enable) { struct netmap_adapter *na = &gna->up.up; struct ifnet *ifp = na->ifp; if (enable) { na->if_transmit = ifp->if_transmit; ifp->if_transmit = netmap_transmit; } else { ifp->if_transmit = na->if_transmit; } } /* * Transmit routine used by generic_netmap_txsync(). Returns 0 on success * and non-zero on error (which may be packet drops or other errors). * addr and len identify the netmap buffer, m is the (preallocated) * mbuf to use for transmissions. * * We should add a reference to the mbuf so the m_freem() at the end * of the transmission does not consume resources. * * On FreeBSD, and on multiqueue cards, we can force the queue using * if ((m->m_flags & M_FLOWID) != 0) * i = m->m_pkthdr.flowid % adapter->num_queues; * else * i = curcpu % adapter->num_queues; * */ int generic_xmit_frame(struct ifnet *ifp, struct mbuf *m, void *addr, u_int len, u_int ring_nr) { int ret; /* * The mbuf should be a cluster from our special pool, * so we do not need to do an m_copyback but just copy * (and eventually, just reference the netmap buffer) */ if (GET_MBUF_REFCNT(m) != 1) { D("invalid refcnt %d for %p", GET_MBUF_REFCNT(m), m); panic("in generic_xmit_frame"); } // XXX the ext_size check is unnecessary if we link the netmap buf if (m->m_ext.ext_size < len) { RD(5, "size %d < len %d", m->m_ext.ext_size, len); len = m->m_ext.ext_size; } if (0) { /* XXX seems to have negligible benefits */ m->m_ext.ext_buf = m->m_data = addr; } else { bcopy(addr, m->m_data, len); } m->m_len = m->m_pkthdr.len = len; // inc refcount. All ours, we could skip the atomic atomic_fetchadd_int(PNT_MBUF_REFCNT(m), 1); m->m_flags |= M_FLOWID; m->m_pkthdr.flowid = ring_nr; m->m_pkthdr.rcvif = ifp; /* used for tx notification */ ret = NA(ifp)->if_transmit(ifp, m); return ret; } #if __FreeBSD_version >= 1100005 struct netmap_adapter * netmap_getna(if_t ifp) { return (NA((struct ifnet *)ifp)); } #endif /* __FreeBSD_version >= 1100005 */ /* * The following two functions are empty until we have a generic * way to extract the info from the ifp */ int generic_find_num_desc(struct ifnet *ifp, unsigned int *tx, unsigned int *rx) { D("called, in tx %d rx %d", *tx, *rx); return 0; } void generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq) { D("called, in txq %d rxq %d", *txq, *rxq); *txq = netmap_generic_rings; *rxq = netmap_generic_rings; } void netmap_mitigation_init(struct nm_generic_mit *mit, int idx, struct netmap_adapter *na) { ND("called"); mit->mit_pending = 0; mit->mit_ring_idx = idx; mit->mit_na = na; } void netmap_mitigation_start(struct nm_generic_mit *mit) { ND("called"); } void netmap_mitigation_restart(struct nm_generic_mit *mit) { ND("called"); } int netmap_mitigation_active(struct nm_generic_mit *mit) { ND("called"); return 0; } void netmap_mitigation_cleanup(struct nm_generic_mit *mit) { ND("called"); } static int nm_vi_dummy(struct ifnet *ifp, u_long cmd, caddr_t addr) { return EINVAL; } static void nm_vi_start(struct ifnet *ifp) { panic("nm_vi_start() must not be called"); } /* * Index manager of persistent virtual interfaces. * It is used to decide the lowest byte of the MAC address. * We use the same algorithm with management of bridge port index. */ #define NM_VI_MAX 255 static struct { uint8_t index[NM_VI_MAX]; /* XXX just for a reasonable number */ uint8_t active; struct mtx lock; } nm_vi_indices; void nm_vi_init_index(void) { int i; for (i = 0; i < NM_VI_MAX; i++) nm_vi_indices.index[i] = i; nm_vi_indices.active = 0; mtx_init(&nm_vi_indices.lock, "nm_vi_indices_lock", NULL, MTX_DEF); } /* return -1 if no index available */ static int nm_vi_get_index(void) { int ret; mtx_lock(&nm_vi_indices.lock); ret = nm_vi_indices.active == NM_VI_MAX ? -1 : nm_vi_indices.index[nm_vi_indices.active++]; mtx_unlock(&nm_vi_indices.lock); return ret; } static void nm_vi_free_index(uint8_t val) { int i, lim; mtx_lock(&nm_vi_indices.lock); lim = nm_vi_indices.active; for (i = 0; i < lim; i++) { if (nm_vi_indices.index[i] == val) { /* swap index[lim-1] and j */ int tmp = nm_vi_indices.index[lim-1]; nm_vi_indices.index[lim-1] = val; nm_vi_indices.index[i] = tmp; nm_vi_indices.active--; break; } } if (lim == nm_vi_indices.active) D("funny, index %u didn't found", val); mtx_unlock(&nm_vi_indices.lock); } #undef NM_VI_MAX /* * Implementation of a netmap-capable virtual interface that * registered to the system. * It is based on if_tap.c and ip_fw_log.c in FreeBSD 9. * * Note: Linux sets refcount to 0 on allocation of net_device, * then increments it on registration to the system. * FreeBSD sets refcount to 1 on if_alloc(), and does not * increment this refcount on if_attach(). */ int nm_vi_persist(const char *name, struct ifnet **ret) { struct ifnet *ifp; u_short macaddr_hi; uint32_t macaddr_mid; u_char eaddr[6]; int unit = nm_vi_get_index(); /* just to decide MAC address */ if (unit < 0) return EBUSY; /* * We use the same MAC address generation method with tap * except for the highest octet is 00:be instead of 00:bd */ macaddr_hi = htons(0x00be); /* XXX tap + 1 */ macaddr_mid = (uint32_t) ticks; bcopy(&macaddr_hi, eaddr, sizeof(short)); bcopy(&macaddr_mid, &eaddr[2], sizeof(uint32_t)); eaddr[5] = (uint8_t)unit; ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { D("if_alloc failed"); return ENOMEM; } if_initname(ifp, name, IF_DUNIT_NONE); ifp->if_mtu = 65536; ifp->if_flags = IFF_UP | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_init = (void *)nm_vi_dummy; ifp->if_ioctl = nm_vi_dummy; ifp->if_start = nm_vi_start; ifp->if_mtu = ETHERMTU; IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen); ifp->if_capabilities |= IFCAP_LINKSTATE; ifp->if_capenable |= IFCAP_LINKSTATE; ether_ifattach(ifp, eaddr); *ret = ifp; return 0; } /* unregister from the system and drop the final refcount */ void nm_vi_detach(struct ifnet *ifp) { nm_vi_free_index(((char *)IF_LLADDR(ifp))[5]); ether_ifdetach(ifp); if_free(ifp); } /* * In order to track whether pages are still mapped, we hook into * the standard cdev_pager and intercept the constructor and * destructor. */ struct netmap_vm_handle_t { struct cdev *dev; struct netmap_priv_d *priv; }; static int netmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color) { struct netmap_vm_handle_t *vmh = handle; if (netmap_verbose) D("handle %p size %jd prot %d foff %jd", handle, (intmax_t)size, prot, (intmax_t)foff); + if (color) + *color = 0; dev_ref(vmh->dev); return 0; } static void netmap_dev_pager_dtor(void *handle) { struct netmap_vm_handle_t *vmh = handle; struct cdev *dev = vmh->dev; struct netmap_priv_d *priv = vmh->priv; if (netmap_verbose) D("handle %p", handle); netmap_dtor(priv); free(vmh, M_DEVBUF); dev_rel(dev); } static int netmap_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot, vm_page_t *mres) { struct netmap_vm_handle_t *vmh = object->handle; struct netmap_priv_d *priv = vmh->priv; vm_paddr_t paddr; vm_page_t page; vm_memattr_t memattr; vm_pindex_t pidx; ND("object %p offset %jd prot %d mres %p", object, (intmax_t)offset, prot, mres); memattr = object->memattr; pidx = OFF_TO_IDX(offset); paddr = netmap_mem_ofstophys(priv->np_mref, offset); if (paddr == 0) return VM_PAGER_FAIL; if (((*mres)->flags & PG_FICTITIOUS) != 0) { /* * If the passed in result page is a fake page, update it with * the new physical address. */ page = *mres; vm_page_updatefake(page, paddr, memattr); } else { /* * Replace the passed in reqpage page with our own fake page and * free up the all of the original pages. */ #ifndef VM_OBJECT_WUNLOCK /* FreeBSD < 10.x */ #define VM_OBJECT_WUNLOCK VM_OBJECT_UNLOCK #define VM_OBJECT_WLOCK VM_OBJECT_LOCK #endif /* VM_OBJECT_WUNLOCK */ VM_OBJECT_WUNLOCK(object); page = vm_page_getfake(paddr, memattr); VM_OBJECT_WLOCK(object); vm_page_lock(*mres); vm_page_free(*mres); vm_page_unlock(*mres); *mres = page; vm_page_insert(page, object, pidx); } page->valid = VM_PAGE_BITS_ALL; return (VM_PAGER_OK); } static struct cdev_pager_ops netmap_cdev_pager_ops = { .cdev_pg_ctor = netmap_dev_pager_ctor, .cdev_pg_dtor = netmap_dev_pager_dtor, .cdev_pg_fault = netmap_dev_pager_fault, }; static int netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff, vm_size_t objsize, vm_object_t *objp, int prot) { int error; struct netmap_vm_handle_t *vmh; struct netmap_priv_d *priv; vm_object_t obj; if (netmap_verbose) D("cdev %p foff %jd size %jd objp %p prot %d", cdev, (intmax_t )*foff, (intmax_t )objsize, objp, prot); vmh = malloc(sizeof(struct netmap_vm_handle_t), M_DEVBUF, M_NOWAIT | M_ZERO); if (vmh == NULL) return ENOMEM; vmh->dev = cdev; NMG_LOCK(); error = devfs_get_cdevpriv((void**)&priv); if (error) goto err_unlock; vmh->priv = priv; priv->np_refcount++; NMG_UNLOCK(); error = netmap_get_memory(priv); if (error) goto err_deref; obj = cdev_pager_allocate(vmh, OBJT_DEVICE, &netmap_cdev_pager_ops, objsize, prot, *foff, NULL); if (obj == NULL) { D("cdev_pager_allocate failed"); error = EINVAL; goto err_deref; } *objp = obj; return 0; err_deref: NMG_LOCK(); priv->np_refcount--; err_unlock: NMG_UNLOCK(); // err: free(vmh, M_DEVBUF); return error; } // XXX can we remove this ? static int netmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td) { if (netmap_verbose) D("dev %p fflag 0x%x devtype %d td %p", dev, fflag, devtype, td); return 0; } static int netmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td) { struct netmap_priv_d *priv; int error; (void)dev; (void)oflags; (void)devtype; (void)td; // XXX wait or nowait ? priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF, M_NOWAIT | M_ZERO); if (priv == NULL) return ENOMEM; error = devfs_set_cdevpriv(priv, netmap_dtor); if (error) return error; priv->np_refcount = 1; return 0; } /******************** kqueue support ****************/ /* * The OS_selwakeup also needs to issue a KNOTE_UNLOCKED. * We use a non-zero argument to distinguish the call from the one * in kevent_scan() which instead also needs to run netmap_poll(). * The knote uses a global mutex for the time being. We might * try to reuse the one in the si, but it is not allocated * permanently so it might be a bit tricky. * * The *kqfilter function registers one or another f_event * depending on read or write mode. * In the call to f_event() td_fpop is NULL so any child function * calling devfs_get_cdevpriv() would fail - and we need it in * netmap_poll(). As a workaround we store priv into kn->kn_hook * and pass it as first argument to netmap_poll(), which then * uses the failure to tell that we are called from f_event() * and do not need the selrecord(). */ -void freebsd_selwakeup(struct selinfo *si, int pri); void -freebsd_selwakeup(struct selinfo *si, int pri) +freebsd_selwakeup(struct nm_selinfo *si, int pri) { if (netmap_verbose) - D("on knote %p", &si->si_note); - selwakeuppri(si, pri); + D("on knote %p", &si->si.si_note); + selwakeuppri(&si->si, pri); /* use a non-zero hint to tell the notification from the * call done in kqueue_scan() which uses 0 */ - KNOTE_UNLOCKED(&si->si_note, 0x100 /* notification */); + KNOTE_UNLOCKED(&si->si.si_note, 0x100 /* notification */); } static void netmap_knrdetach(struct knote *kn) { struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; - struct selinfo *si = priv->np_rxsi; + struct selinfo *si = &priv->np_rxsi->si; D("remove selinfo %p", si); knlist_remove(&si->si_note, kn, 0); } static void netmap_knwdetach(struct knote *kn) { struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; - struct selinfo *si = priv->np_txsi; + struct selinfo *si = &priv->np_txsi->si; D("remove selinfo %p", si); knlist_remove(&si->si_note, kn, 0); } /* * callback from notifies (generated externally) and our * calls to kevent(). The former we just return 1 (ready) * since we do not know better. * In the latter we call netmap_poll and return 0/1 accordingly. */ static int netmap_knrw(struct knote *kn, long hint, int events) { struct netmap_priv_d *priv; int revents; if (hint != 0) { ND(5, "call from notify"); return 1; /* assume we are ready */ } priv = kn->kn_hook; /* the notification may come from an external thread, * in which case we do not want to run the netmap_poll * This should be filtered above, but check just in case. */ if (curthread != priv->np_td) { /* should not happen */ RD(5, "curthread changed %p %p", curthread, priv->np_td); return 1; } else { revents = netmap_poll((void *)priv, events, curthread); return (events & revents) ? 1 : 0; } } static int netmap_knread(struct knote *kn, long hint) { return netmap_knrw(kn, hint, POLLIN); } static int netmap_knwrite(struct knote *kn, long hint) { return netmap_knrw(kn, hint, POLLOUT); } static struct filterops netmap_rfiltops = { .f_isfd = 1, .f_detach = netmap_knrdetach, .f_event = netmap_knread, }; static struct filterops netmap_wfiltops = { .f_isfd = 1, .f_detach = netmap_knwdetach, .f_event = netmap_knwrite, }; /* * This is called when a thread invokes kevent() to record * a change in the configuration of the kqueue(). * The 'priv' should be the same as in the netmap device. */ static int netmap_kqfilter(struct cdev *dev, struct knote *kn) { struct netmap_priv_d *priv; int error; struct netmap_adapter *na; - struct selinfo *si; + struct nm_selinfo *si; int ev = kn->kn_filter; if (ev != EVFILT_READ && ev != EVFILT_WRITE) { D("bad filter request %d", ev); return 1; } error = devfs_get_cdevpriv((void**)&priv); if (error) { D("device not yet setup"); return 1; } na = priv->np_na; if (na == NULL) { D("no netmap adapter for this file descriptor"); return 1; } /* the si is indicated in the priv */ si = (ev == EVFILT_WRITE) ? priv->np_txsi : priv->np_rxsi; // XXX lock(priv) ? kn->kn_fop = (ev == EVFILT_WRITE) ? &netmap_wfiltops : &netmap_rfiltops; kn->kn_hook = priv; - knlist_add(&si->si_note, kn, 1); + knlist_add(&si->si.si_note, kn, 1); // XXX unlock(priv) ND("register %p %s td %p priv %p kn %p np_nifp %p kn_fp/fpop %s", na, na->ifp->if_xname, curthread, priv, kn, priv->np_nifp, kn->kn_fp == curthread->td_fpop ? "match" : "MISMATCH"); return 0; } struct cdevsw netmap_cdevsw = { .d_version = D_VERSION, .d_name = "netmap", .d_open = netmap_open, .d_mmap_single = netmap_mmap_single, .d_ioctl = netmap_ioctl, .d_poll = netmap_poll, .d_kqfilter = netmap_kqfilter, .d_close = netmap_close, }; /*--- end of kqueue support ----*/ /* * Kernel entry point. * * Initialize/finalize the module and return. * * Return 0 on success, errno on failure. */ static int netmap_loader(__unused struct module *module, int event, __unused void *arg) { int error = 0; switch (event) { case MOD_LOAD: error = netmap_init(); break; case MOD_UNLOAD: netmap_fini(); break; default: error = EOPNOTSUPP; break; } return (error); } DEV_MODULE(netmap, netmap_loader, NULL); Index: stable/10/sys/dev/netmap/netmap_generic.c =================================================================== --- stable/10/sys/dev/netmap/netmap_generic.c (revision 278778) +++ stable/10/sys/dev/netmap/netmap_generic.c (revision 278779) @@ -1,863 +1,863 @@ /* * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This module implements netmap support on top of standard, * unmodified device drivers. * * A NIOCREGIF request is handled here if the device does not * have native support. TX and RX rings are emulated as follows: * * NIOCREGIF * We preallocate a block of TX mbufs (roughly as many as * tx descriptors; the number is not critical) to speed up * operation during transmissions. The refcount on most of * these buffers is artificially bumped up so we can recycle * them more easily. Also, the destructor is intercepted * so we use it as an interrupt notification to wake up * processes blocked on a poll(). * * For each receive ring we allocate one "struct mbq" * (an mbuf tailq plus a spinlock). We intercept packets * (through if_input) * on the receive path and put them in the mbq from which * netmap receive routines can grab them. * * TX: * in the generic_txsync() routine, netmap buffers are copied * (or linked, in a future) to the preallocated mbufs * and pushed to the transmit queue. Some of these mbufs * (those with NS_REPORT, or otherwise every half ring) * have the refcount=1, others have refcount=2. * When the destructor is invoked, we take that as * a notification that all mbufs up to that one in * the specific ring have been completed, and generate * the equivalent of a transmit interrupt. * * RX: * */ #ifdef __FreeBSD__ #include /* prerequisite */ __FBSDID("$FreeBSD$"); #include #include #include #include /* PROT_EXEC */ #include #include /* sockaddrs */ #include #include #include #include /* bus_dmamap_* in netmap_kern.h */ // XXX temporary - D() defined here #include #include #include #define rtnl_lock() ND("rtnl_lock called") #define rtnl_unlock() ND("rtnl_unlock called") #define MBUF_TXQ(m) ((m)->m_pkthdr.flowid) #define MBUF_RXQ(m) ((m)->m_pkthdr.flowid) #define smp_mb() /* * FreeBSD mbuf allocator/deallocator in emulation mode: * * We allocate EXT_PACKET mbuf+clusters, but need to set M_NOFREE * so that the destructor, if invoked, will not free the packet. * In principle we should set the destructor only on demand, * but since there might be a race we better do it on allocation. * As a consequence, we also need to set the destructor or we * would leak buffers. */ /* * mbuf wrappers */ /* mbuf destructor, also need to change the type to EXT_EXTREF, * add an M_NOFREE flag, and then clear the flag and * chain into uma_zfree(zone_pack, mf) * (or reinstall the buffer ?) */ #define SET_MBUF_DESTRUCTOR(m, fn) do { \ (m)->m_ext.ext_free = (void *)fn; \ (m)->m_ext.ext_type = EXT_EXTREF; \ } while (0) static void netmap_default_mbuf_destructor(struct mbuf *m) { /* restore original mbuf */ m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg1; m->m_ext.ext_arg1 = NULL; m->m_ext.ext_type = EXT_PACKET; m->m_ext.ext_free = NULL; if (GET_MBUF_REFCNT(m) == 0) SET_MBUF_REFCNT(m, 1); uma_zfree(zone_pack, m); } static inline struct mbuf * netmap_get_mbuf(int len) { struct mbuf *m; m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR | M_NOFREE); if (m) { m->m_ext.ext_arg1 = m->m_ext.ext_buf; // XXX save m->m_ext.ext_free = (void *)netmap_default_mbuf_destructor; m->m_ext.ext_type = EXT_EXTREF; ND(5, "create m %p refcnt %d", m, GET_MBUF_REFCNT(m)); } return m; } #else /* linux */ #include "bsd_glue.h" #include /* rtnl_[un]lock() */ #include /* struct ethtool_ops, get_ringparam */ #include //#define REG_RESET #endif /* linux */ /* Common headers. */ #include #include #include /* ======================== usage stats =========================== */ #ifdef RATE_GENERIC #define IFRATE(x) x struct rate_stats { unsigned long txpkt; unsigned long txsync; unsigned long txirq; unsigned long rxpkt; unsigned long rxirq; unsigned long rxsync; }; struct rate_context { unsigned refcount; struct timer_list timer; struct rate_stats new; struct rate_stats old; }; #define RATE_PRINTK(_NAME_) \ printk( #_NAME_ " = %lu Hz\n", (cur._NAME_ - ctx->old._NAME_)/RATE_PERIOD); #define RATE_PERIOD 2 static void rate_callback(unsigned long arg) { struct rate_context * ctx = (struct rate_context *)arg; struct rate_stats cur = ctx->new; int r; RATE_PRINTK(txpkt); RATE_PRINTK(txsync); RATE_PRINTK(txirq); RATE_PRINTK(rxpkt); RATE_PRINTK(rxsync); RATE_PRINTK(rxirq); printk("\n"); ctx->old = cur; r = mod_timer(&ctx->timer, jiffies + msecs_to_jiffies(RATE_PERIOD * 1000)); if (unlikely(r)) D("[v1000] Error: mod_timer()"); } static struct rate_context rate_ctx; void generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi) { if (txp) rate_ctx.new.txpkt++; if (txs) rate_ctx.new.txsync++; if (txi) rate_ctx.new.txirq++; if (rxp) rate_ctx.new.rxpkt++; if (rxs) rate_ctx.new.rxsync++; if (rxi) rate_ctx.new.rxirq++; } #else /* !RATE */ #define IFRATE(x) #endif /* !RATE */ /* =============== GENERIC NETMAP ADAPTER SUPPORT ================= */ /* * Wrapper used by the generic adapter layer to notify * the poller threads. Differently from netmap_rx_irq(), we check * only NAF_NETMAP_ON instead of NAF_NATIVE_ON to enable the irq. */ static void netmap_generic_irq(struct ifnet *ifp, u_int q, u_int *work_done) { struct netmap_adapter *na = NA(ifp); if (unlikely(!nm_netmap_on(na))) return; netmap_common_irq(ifp, q, work_done); } /* Enable/disable netmap mode for a generic network interface. */ static int generic_netmap_register(struct netmap_adapter *na, int enable) { struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; struct mbuf *m; int error; int i, r; if (!na) return EINVAL; #ifdef REG_RESET error = ifp->netdev_ops->ndo_stop(ifp); if (error) { return error; } #endif /* REG_RESET */ if (enable) { /* Enable netmap mode. */ /* Init the mitigation support on all the rx queues. */ gna->mit = malloc(na->num_rx_rings * sizeof(struct nm_generic_mit), M_DEVBUF, M_NOWAIT | M_ZERO); if (!gna->mit) { D("mitigation allocation failed"); error = ENOMEM; goto out; } for (r=0; rnum_rx_rings; r++) netmap_mitigation_init(&gna->mit[r], r, na); /* Initialize the rx queue, as generic_rx_handler() can * be called as soon as netmap_catch_rx() returns. */ for (r=0; rnum_rx_rings; r++) { mbq_safe_init(&na->rx_rings[r].rx_queue); } /* * Preallocate packet buffers for the tx rings. */ for (r=0; rnum_tx_rings; r++) na->tx_rings[r].tx_pool = NULL; for (r=0; rnum_tx_rings; r++) { na->tx_rings[r].tx_pool = malloc(na->num_tx_desc * sizeof(struct mbuf *), M_DEVBUF, M_NOWAIT | M_ZERO); if (!na->tx_rings[r].tx_pool) { D("tx_pool allocation failed"); error = ENOMEM; goto free_tx_pools; } for (i=0; inum_tx_desc; i++) na->tx_rings[r].tx_pool[i] = NULL; for (i=0; inum_tx_desc; i++) { m = netmap_get_mbuf(NETMAP_BUF_SIZE(na)); if (!m) { D("tx_pool[%d] allocation failed", i); error = ENOMEM; goto free_tx_pools; } na->tx_rings[r].tx_pool[i] = m; } } rtnl_lock(); /* Prepare to intercept incoming traffic. */ error = netmap_catch_rx(na, 1); if (error) { D("netdev_rx_handler_register() failed (%d)", error); goto register_handler; } na->na_flags |= NAF_NETMAP_ON; /* Make netmap control the packet steering. */ netmap_catch_tx(gna, 1); rtnl_unlock(); #ifdef RATE_GENERIC if (rate_ctx.refcount == 0) { D("setup_timer()"); memset(&rate_ctx, 0, sizeof(rate_ctx)); setup_timer(&rate_ctx.timer, &rate_callback, (unsigned long)&rate_ctx); if (mod_timer(&rate_ctx.timer, jiffies + msecs_to_jiffies(1500))) { D("Error: mod_timer()"); } } rate_ctx.refcount++; #endif /* RATE */ } else if (na->tx_rings[0].tx_pool) { /* Disable netmap mode. We enter here only if the previous generic_netmap_register(na, 1) was successfull. If it was not, na->tx_rings[0].tx_pool was set to NULL by the error handling code below. */ rtnl_lock(); na->na_flags &= ~NAF_NETMAP_ON; /* Release packet steering control. */ netmap_catch_tx(gna, 0); /* Do not intercept packets on the rx path. */ netmap_catch_rx(na, 0); rtnl_unlock(); /* Free the mbufs going to the netmap rings */ for (r=0; rnum_rx_rings; r++) { mbq_safe_purge(&na->rx_rings[r].rx_queue); mbq_safe_destroy(&na->rx_rings[r].rx_queue); } for (r=0; rnum_rx_rings; r++) netmap_mitigation_cleanup(&gna->mit[r]); free(gna->mit, M_DEVBUF); for (r=0; rnum_tx_rings; r++) { for (i=0; inum_tx_desc; i++) { m_freem(na->tx_rings[r].tx_pool[i]); } free(na->tx_rings[r].tx_pool, M_DEVBUF); } #ifdef RATE_GENERIC if (--rate_ctx.refcount == 0) { D("del_timer()"); del_timer(&rate_ctx.timer); } #endif } #ifdef REG_RESET error = ifp->netdev_ops->ndo_open(ifp); if (error) { goto free_tx_pools; } #endif return 0; register_handler: rtnl_unlock(); free_tx_pools: for (r=0; rnum_tx_rings; r++) { if (na->tx_rings[r].tx_pool == NULL) continue; for (i=0; inum_tx_desc; i++) if (na->tx_rings[r].tx_pool[i]) m_freem(na->tx_rings[r].tx_pool[i]); free(na->tx_rings[r].tx_pool, M_DEVBUF); na->tx_rings[r].tx_pool = NULL; } for (r=0; rnum_rx_rings; r++) { netmap_mitigation_cleanup(&gna->mit[r]); mbq_safe_destroy(&na->rx_rings[r].rx_queue); } free(gna->mit, M_DEVBUF); out: return error; } /* * Callback invoked when the device driver frees an mbuf used * by netmap to transmit a packet. This usually happens when * the NIC notifies the driver that transmission is completed. */ static void generic_mbuf_destructor(struct mbuf *m) { netmap_generic_irq(MBUF_IFP(m), MBUF_TXQ(m), NULL); #ifdef __FreeBSD__ if (netmap_verbose) RD(5, "Tx irq (%p) queue %d index %d" , m, MBUF_TXQ(m), (int)(uintptr_t)m->m_ext.ext_arg1); netmap_default_mbuf_destructor(m); #endif /* __FreeBSD__ */ IFRATE(rate_ctx.new.txirq++); } extern int netmap_adaptive_io; /* Record completed transmissions and update hwtail. * * The oldest tx buffer not yet completed is at nr_hwtail + 1, * nr_hwcur is the first unsent buffer. */ static u_int generic_netmap_tx_clean(struct netmap_kring *kring) { u_int const lim = kring->nkr_num_slots - 1; u_int nm_i = nm_next(kring->nr_hwtail, lim); u_int hwcur = kring->nr_hwcur; u_int n = 0; struct mbuf **tx_pool = kring->tx_pool; while (nm_i != hwcur) { /* buffers not completed */ struct mbuf *m = tx_pool[nm_i]; if (unlikely(m == NULL)) { /* this is done, try to replenish the entry */ tx_pool[nm_i] = m = netmap_get_mbuf(NETMAP_BUF_SIZE(kring->na)); if (unlikely(m == NULL)) { D("mbuf allocation failed, XXX error"); // XXX how do we proceed ? break ? return -ENOMEM; } } else if (GET_MBUF_REFCNT(m) != 1) { break; /* This mbuf is still busy: its refcnt is 2. */ } n++; nm_i = nm_next(nm_i, lim); #if 0 /* rate adaptation */ if (netmap_adaptive_io > 1) { if (n >= netmap_adaptive_io) break; } else if (netmap_adaptive_io) { /* if hwcur - nm_i < lim/8 do an early break * so we prevent the sender from stalling. See CVT. */ if (hwcur >= nm_i) { if (hwcur - nm_i < lim/2) break; } else { if (hwcur + lim + 1 - nm_i < lim/2) break; } } #endif } kring->nr_hwtail = nm_prev(nm_i, lim); ND("tx completed [%d] -> hwtail %d", n, kring->nr_hwtail); return n; } /* * We have pending packets in the driver between nr_hwtail +1 and hwcur. * Compute a position in the middle, to be used to generate * a notification. */ static inline u_int generic_tx_event_middle(struct netmap_kring *kring, u_int hwcur) { u_int n = kring->nkr_num_slots; u_int ntc = nm_next(kring->nr_hwtail, n-1); u_int e; if (hwcur >= ntc) { e = (hwcur + ntc) / 2; } else { /* wrap around */ e = (hwcur + n + ntc) / 2; if (e >= n) { e -= n; } } if (unlikely(e >= n)) { D("This cannot happen"); e = 0; } return e; } /* * We have pending packets in the driver between nr_hwtail+1 and hwcur. * Schedule a notification approximately in the middle of the two. * There is a race but this is only called within txsync which does * a double check. */ static void generic_set_tx_event(struct netmap_kring *kring, u_int hwcur) { struct mbuf *m; u_int e; if (nm_next(kring->nr_hwtail, kring->nkr_num_slots -1) == hwcur) { return; /* all buffers are free */ } e = generic_tx_event_middle(kring, hwcur); m = kring->tx_pool[e]; ND(5, "Request Event at %d mbuf %p refcnt %d", e, m, m ? GET_MBUF_REFCNT(m) : -2 ); if (m == NULL) { /* This can happen if there is already an event on the netmap slot 'e': There is nothing to do. */ return; } kring->tx_pool[e] = NULL; SET_MBUF_DESTRUCTOR(m, generic_mbuf_destructor); // XXX wmb() ? /* Decrement the refcount an free it if we have the last one. */ m_freem(m); smp_mb(); } /* * generic_netmap_txsync() transforms netmap buffers into mbufs * and passes them to the standard device driver * (ndo_start_xmit() or ifp->if_transmit() ). * On linux this is not done directly, but using dev_queue_xmit(), * since it implements the TX flow control (and takes some locks). */ static int generic_netmap_txsync(struct netmap_kring *kring, int flags) { struct netmap_adapter *na = kring->na; struct ifnet *ifp = na->ifp; struct netmap_ring *ring = kring->ring; u_int nm_i; /* index into the netmap ring */ // j u_int const lim = kring->nkr_num_slots - 1; u_int const head = kring->rhead; u_int ring_nr = kring->ring_id; IFRATE(rate_ctx.new.txsync++); // TODO: handle the case of mbuf allocation failure rmb(); /* * First part: process new packets to send. */ nm_i = kring->nr_hwcur; if (nm_i != head) { /* we have new packets to send */ while (nm_i != head) { struct netmap_slot *slot = &ring->slot[nm_i]; u_int len = slot->len; void *addr = NMB(na, slot); /* device-specific */ struct mbuf *m; int tx_ret; NM_CHECK_ADDR_LEN(na, addr, len); /* Tale a mbuf from the tx pool and copy in the user packet. */ m = kring->tx_pool[nm_i]; if (unlikely(!m)) { RD(5, "This should never happen"); kring->tx_pool[nm_i] = m = netmap_get_mbuf(NETMAP_BUF_SIZE(na)); if (unlikely(m == NULL)) { D("mbuf allocation failed"); break; } } /* XXX we should ask notifications when NS_REPORT is set, * or roughly every half frame. We can optimize this * by lazily requesting notifications only when a * transmission fails. Probably the best way is to * break on failures and set notifications when * ring->cur == ring->tail || nm_i != cur */ tx_ret = generic_xmit_frame(ifp, m, addr, len, ring_nr); if (unlikely(tx_ret)) { ND(5, "start_xmit failed: err %d [nm_i %u, head %u, hwtail %u]", tx_ret, nm_i, head, kring->nr_hwtail); /* * No room for this mbuf in the device driver. * Request a notification FOR A PREVIOUS MBUF, * then call generic_netmap_tx_clean(kring) to do the * double check and see if we can free more buffers. * If there is space continue, else break; * NOTE: the double check is necessary if the problem * occurs in the txsync call after selrecord(). * Also, we need some way to tell the caller that not * all buffers were queued onto the device (this was * not a problem with native netmap driver where space * is preallocated). The bridge has a similar problem * and we solve it there by dropping the excess packets. */ generic_set_tx_event(kring, nm_i); if (generic_netmap_tx_clean(kring)) { /* space now available */ continue; } else { break; } } slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); nm_i = nm_next(nm_i, lim); IFRATE(rate_ctx.new.txpkt ++); } /* Update hwcur to the next slot to transmit. */ kring->nr_hwcur = nm_i; /* not head, we could break early */ } /* * Second, reclaim completed buffers */ if (flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) { /* No more available slots? Set a notification event * on a netmap slot that will be cleaned in the future. * No doublecheck is performed, since txsync() will be * called twice by netmap_poll(). */ generic_set_tx_event(kring, nm_i); } ND("tx #%d, hwtail = %d", n, kring->nr_hwtail); generic_netmap_tx_clean(kring); nm_txsync_finalize(kring); return 0; } /* * This handler is registered (through netmap_catch_rx()) * within the attached network interface * in the RX subsystem, so that every mbuf passed up by * the driver can be stolen to the network stack. * Stolen packets are put in a queue where the * generic_netmap_rxsync() callback can extract them. */ void generic_rx_handler(struct ifnet *ifp, struct mbuf *m) { struct netmap_adapter *na = NA(ifp); struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; u_int work_done; u_int rr = MBUF_RXQ(m); // receive ring number if (rr >= na->num_rx_rings) { rr = rr % na->num_rx_rings; // XXX expensive... } /* limit the size of the queue */ if (unlikely(mbq_len(&na->rx_rings[rr].rx_queue) > 1024)) { m_freem(m); } else { mbq_safe_enqueue(&na->rx_rings[rr].rx_queue, m); } if (netmap_generic_mit < 32768) { /* no rx mitigation, pass notification up */ netmap_generic_irq(na->ifp, rr, &work_done); IFRATE(rate_ctx.new.rxirq++); } else { /* same as send combining, filter notification if there is a * pending timer, otherwise pass it up and start a timer. */ if (likely(netmap_mitigation_active(&gna->mit[rr]))) { /* Record that there is some pending work. */ gna->mit[rr].mit_pending = 1; } else { netmap_generic_irq(na->ifp, rr, &work_done); IFRATE(rate_ctx.new.rxirq++); netmap_mitigation_start(&gna->mit[rr]); } } } /* * generic_netmap_rxsync() extracts mbufs from the queue filled by * generic_netmap_rx_handler() and puts their content in the netmap * receive ring. * Access must be protected because the rx handler is asynchronous, */ static int generic_netmap_rxsync(struct netmap_kring *kring, int flags) { struct netmap_ring *ring = kring->ring; struct netmap_adapter *na = kring->na; u_int nm_i; /* index into the netmap ring */ //j, u_int n; u_int const lim = kring->nkr_num_slots - 1; u_int const head = nm_rxsync_prologue(kring); int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; if (head > lim) return netmap_ring_reinit(kring); /* * First part: import newly received packets. */ if (netmap_no_pendintr || force_update) { /* extract buffers from the rx queue, stop at most one * slot before nr_hwcur (stop_i) */ uint16_t slot_flags = kring->nkr_slot_flags; u_int stop_i = nm_prev(kring->nr_hwcur, lim); nm_i = kring->nr_hwtail; /* first empty slot in the receive ring */ for (n = 0; nm_i != stop_i; n++) { int len; void *addr = NMB(na, &ring->slot[nm_i]); struct mbuf *m; /* we only check the address here on generic rx rings */ if (addr == NETMAP_BUF_BASE(na)) { /* Bad buffer */ return netmap_ring_reinit(kring); } /* * Call the locked version of the function. * XXX Ideally we could grab a batch of mbufs at once * and save some locking overhead. */ m = mbq_safe_dequeue(&kring->rx_queue); if (!m) /* no more data */ break; len = MBUF_LEN(m); m_copydata(m, 0, len, addr); ring->slot[nm_i].len = len; ring->slot[nm_i].flags = slot_flags; m_freem(m); nm_i = nm_next(nm_i, lim); } if (n) { kring->nr_hwtail = nm_i; IFRATE(rate_ctx.new.rxpkt += n); } kring->nr_kflags &= ~NKR_PENDINTR; } // XXX should we invert the order ? /* * Second part: skip past packets that userspace has released. */ nm_i = kring->nr_hwcur; if (nm_i != head) { /* Userspace has released some packets. */ for (n = 0; nm_i != head; n++) { struct netmap_slot *slot = &ring->slot[nm_i]; slot->flags &= ~NS_BUF_CHANGED; nm_i = nm_next(nm_i, lim); } kring->nr_hwcur = head; } /* tell userspace that there might be new packets. */ nm_rxsync_finalize(kring); IFRATE(rate_ctx.new.rxsync++); return 0; } static void generic_netmap_dtor(struct netmap_adapter *na) { struct ifnet *ifp = na->ifp; struct netmap_generic_adapter *gna = (struct netmap_generic_adapter*)na; struct netmap_adapter *prev_na = gna->prev; if (prev_na != NULL) { D("Released generic NA %p", gna); if_rele(na->ifp); netmap_adapter_put(prev_na); } if (ifp != NULL) { WNA(ifp) = prev_na; D("Restored native NA %p", prev_na); na->ifp = NULL; } } /* * generic_netmap_attach() makes it possible to use netmap on * a device without native netmap support. * This is less performant than native support but potentially * faster than raw sockets or similar schemes. * * In this "emulated" mode, netmap rings do not necessarily * have the same size as those in the NIC. We use a default * value and possibly override it if the OS has ways to fetch the * actual configuration. */ int generic_netmap_attach(struct ifnet *ifp) { struct netmap_adapter *na; struct netmap_generic_adapter *gna; int retval; u_int num_tx_desc, num_rx_desc; num_tx_desc = num_rx_desc = netmap_generic_ringsize; /* starting point */ - generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc); + generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc); /* ignore errors */ ND("Netmap ring size: TX = %d, RX = %d", num_tx_desc, num_rx_desc); if (num_tx_desc == 0 || num_rx_desc == 0) { D("Device has no hw slots (tx %u, rx %u)", num_tx_desc, num_rx_desc); return EINVAL; } gna = malloc(sizeof(*gna), M_DEVBUF, M_NOWAIT | M_ZERO); if (gna == NULL) { D("no memory on attach, give up"); return ENOMEM; } na = (struct netmap_adapter *)gna; na->ifp = ifp; na->num_tx_desc = num_tx_desc; na->num_rx_desc = num_rx_desc; na->nm_register = &generic_netmap_register; na->nm_txsync = &generic_netmap_txsync; na->nm_rxsync = &generic_netmap_rxsync; na->nm_dtor = &generic_netmap_dtor; /* when using generic, NAF_NETMAP_ON is set so we force * NAF_SKIP_INTR to use the regular interrupt handler */ na->na_flags = NAF_SKIP_INTR | NAF_HOST_RINGS; ND("[GNA] num_tx_queues(%d), real_num_tx_queues(%d), len(%lu)", ifp->num_tx_queues, ifp->real_num_tx_queues, ifp->tx_queue_len); ND("[GNA] num_rx_queues(%d), real_num_rx_queues(%d)", ifp->num_rx_queues, ifp->real_num_rx_queues); generic_find_num_queues(ifp, &na->num_tx_rings, &na->num_rx_rings); retval = netmap_attach_common(na); if (retval) { free(gna, M_DEVBUF); } return retval; } Index: stable/10/sys/dev/netmap/netmap_kern.h =================================================================== --- stable/10/sys/dev/netmap/netmap_kern.h (revision 278778) +++ stable/10/sys/dev/netmap/netmap_kern.h (revision 278779) @@ -1,1621 +1,1645 @@ /* * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved. * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ * * The header contains the definitions of constants and function * prototypes used only in kernelspace. */ #ifndef _NET_NETMAP_KERN_H_ #define _NET_NETMAP_KERN_H_ #define WITH_VALE // comment out to disable VALE support #define WITH_PIPES #define WITH_MONITOR +#define WITH_GENERIC #if defined(__FreeBSD__) #define likely(x) __builtin_expect((long)!!(x), 1L) #define unlikely(x) __builtin_expect((long)!!(x), 0L) #define NM_LOCK_T struct mtx + +/* netmap global lock */ #define NMG_LOCK_T struct sx #define NMG_LOCK_INIT() sx_init(&netmap_global_lock, \ "netmap global lock") #define NMG_LOCK_DESTROY() sx_destroy(&netmap_global_lock) #define NMG_LOCK() sx_xlock(&netmap_global_lock) #define NMG_UNLOCK() sx_xunlock(&netmap_global_lock) #define NMG_LOCK_ASSERT() sx_assert(&netmap_global_lock, SA_XLOCKED) -#define NM_SELINFO_T struct selinfo +#define NM_SELINFO_T struct nm_selinfo #define MBUF_LEN(m) ((m)->m_pkthdr.len) #define MBUF_IFP(m) ((m)->m_pkthdr.rcvif) #define NM_SEND_UP(ifp, m) ((NA(ifp))->if_input)(ifp, m) #define NM_ATOMIC_T volatile int // XXX ? /* atomic operations */ #include #define NM_ATOMIC_TEST_AND_SET(p) (!atomic_cmpset_acq_int((p), 0, 1)) #define NM_ATOMIC_CLEAR(p) atomic_store_rel_int((p), 0) +#if __FreeBSD_version >= 1100030 +#define WNA(_ifp) (_ifp)->if_netmap +#else /* older FreeBSD */ +#define WNA(_ifp) (_ifp)->if_pspare[0] +#endif /* older FreeBSD */ + #if __FreeBSD_version >= 1100005 struct netmap_adapter *netmap_getna(if_t ifp); #endif #if __FreeBSD_version >= 1100027 #define GET_MBUF_REFCNT(m) ((m)->m_ext.ext_cnt ? *((m)->m_ext.ext_cnt) : -1) #define SET_MBUF_REFCNT(m, x) *((m)->m_ext.ext_cnt) = x #define PNT_MBUF_REFCNT(m) ((m)->m_ext.ext_cnt) #else #define GET_MBUF_REFCNT(m) ((m)->m_ext.ref_cnt ? *((m)->m_ext.ref_cnt) : -1) #define SET_MBUF_REFCNT(m, x) *((m)->m_ext.ref_cnt) = x #define PNT_MBUF_REFCNT(m) ((m)->m_ext.ref_cnt) #endif MALLOC_DECLARE(M_NETMAP); +struct nm_selinfo { + struct selinfo si; + struct mtx m; +}; + +void freebsd_selwakeup(struct nm_selinfo *si, int pri); + // XXX linux struct, not used in FreeBSD struct net_device_ops { }; struct ethtool_ops { }; struct hrtimer { }; #elif defined (linux) #define NM_LOCK_T safe_spinlock_t // see bsd_glue.h #define NM_SELINFO_T wait_queue_head_t #define MBUF_LEN(m) ((m)->len) #define MBUF_IFP(m) ((m)->dev) #define NM_SEND_UP(ifp, m) \ do { \ m->priority = NM_MAGIC_PRIORITY_RX; \ netif_rx(m); \ } while (0) #define NM_ATOMIC_T volatile long unsigned int -// XXX a mtx would suffice here too 20130404 gl -#define NMG_LOCK_T struct semaphore -#define NMG_LOCK_INIT() sema_init(&netmap_global_lock, 1) -#define NMG_LOCK_DESTROY() -#define NMG_LOCK() down(&netmap_global_lock) -#define NMG_UNLOCK() up(&netmap_global_lock) -#define NMG_LOCK_ASSERT() // XXX to be completed +#define NM_MTX_T struct mutex +#define NM_MTX_INIT(m, s) do { (void)s; mutex_init(&(m)); } while (0) +#define NM_MTX_DESTROY(m) do { (void)m; } while (0) +#define NM_MTX_LOCK(m) mutex_lock(&(m)) +#define NM_MTX_UNLOCK(m) mutex_unlock(&(m)) +#define NM_MTX_LOCK_ASSERT(m) mutex_is_locked(&(m)) +#define NMG_LOCK_T NM_MTX_T +#define NMG_LOCK_INIT() NM_MTX_INIT(netmap_global_lock, \ + "netmap_global_lock") +#define NMG_LOCK_DESTROY() NM_MTX_DESTROY(netmap_global_lock) +#define NMG_LOCK() NM_MTX_LOCK(netmap_global_lock) +#define NMG_UNLOCK() NM_MTX_UNLOCK(netmap_global_lock) +#define NMG_LOCK_ASSERT() NM_MTX_LOCK_ASSERT(netmap_global_lock) + #ifndef DEV_NETMAP #define DEV_NETMAP #endif /* DEV_NETMAP */ #elif defined (__APPLE__) #warning apple support is incomplete. #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #define NM_LOCK_T IOLock * #define NM_SELINFO_T struct selinfo #define MBUF_LEN(m) ((m)->m_pkthdr.len) #define NM_SEND_UP(ifp, m) ((ifp)->if_input)(ifp, m) #else #error unsupported platform #endif /* end - platform-specific code */ #define ND(format, ...) #define D(format, ...) \ do { \ struct timeval __xxts; \ microtime(&__xxts); \ printf("%03d.%06d [%4d] %-25s " format "\n", \ (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ __LINE__, __FUNCTION__, ##__VA_ARGS__); \ } while (0) /* rate limited, lps indicates how many per second */ #define RD(lps, format, ...) \ do { \ static int t0, __cnt; \ if (t0 != time_second) { \ t0 = time_second; \ __cnt = 0; \ } \ if (__cnt++ < lps) \ D(format, ##__VA_ARGS__); \ } while (0) struct netmap_adapter; struct nm_bdg_fwd; struct nm_bridge; struct netmap_priv_d; const char *nm_dump_buf(char *p, int len, int lim, char *dst); #include "netmap_mbq.h" extern NMG_LOCK_T netmap_global_lock; /* * private, kernel view of a ring. Keeps track of the status of * a ring across system calls. * * nr_hwcur index of the next buffer to refill. * It corresponds to ring->head * at the time the system call returns. * * nr_hwtail index of the first buffer owned by the kernel. * On RX, hwcur->hwtail are receive buffers * not yet released. hwcur is advanced following * ring->head, hwtail is advanced on incoming packets, * and a wakeup is generated when hwtail passes ring->cur * On TX, hwcur->rcur have been filled by the sender * but not sent yet to the NIC; rcur->hwtail are available * for new transmissions, and hwtail->hwcur-1 are pending * transmissions not yet acknowledged. * * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots. * This is so that, on a reset, buffers owned by userspace are not * modified by the kernel. In particular: * RX rings: the next empty buffer (hwtail + hwofs) coincides with * the next empty buffer as known by the hardware (next_to_check or so). * TX rings: hwcur + hwofs coincides with next_to_send * * For received packets, slot->flags is set to nkr_slot_flags * so we can provide a proper initial value (e.g. set NS_FORWARD * when operating in 'transparent' mode). * * The following fields are used to implement lock-free copy of packets * from input to output ports in VALE switch: * nkr_hwlease buffer after the last one being copied. * A writer in nm_bdg_flush reserves N buffers * from nr_hwlease, advances it, then does the * copy outside the lock. * In RX rings (used for VALE ports), * nkr_hwtail <= nkr_hwlease < nkr_hwcur+N-1 * In TX rings (used for NIC or host stack ports) * nkr_hwcur <= nkr_hwlease < nkr_hwtail * nkr_leases array of nkr_num_slots where writers can report * completion of their block. NR_NOSLOT (~0) indicates * that the writer has not finished yet * nkr_lease_idx index of next free slot in nr_leases, to be assigned * * The kring is manipulated by txsync/rxsync and generic netmap function. * * Concurrent rxsync or txsync on the same ring are prevented through * by nm_kr_(try)lock() which in turn uses nr_busy. This is all we need * for NIC rings, and for TX rings attached to the host stack. * * RX rings attached to the host stack use an mbq (rx_queue) on both * rxsync_from_host() and netmap_transmit(). The mbq is protected * by its internal lock. * * RX rings attached to the VALE switch are accessed by both senders * and receiver. They are protected through the q_lock on the RX ring. */ struct netmap_kring { struct netmap_ring *ring; uint32_t nr_hwcur; uint32_t nr_hwtail; /* * Copies of values in user rings, so we do not need to look * at the ring (which could be modified). These are set in the * *sync_prologue()/finalize() routines. */ uint32_t rhead; uint32_t rcur; uint32_t rtail; uint32_t nr_kflags; /* private driver flags */ #define NKR_PENDINTR 0x1 // Pending interrupt. uint32_t nkr_num_slots; /* * On a NIC reset, the NIC ring indexes may be reset but the * indexes in the netmap rings remain the same. nkr_hwofs * keeps track of the offset between the two. */ int32_t nkr_hwofs; uint16_t nkr_slot_flags; /* initial value for flags */ /* last_reclaim is opaque marker to help reduce the frequency * of operations such as reclaiming tx buffers. A possible use * is set it to ticks and do the reclaim only once per tick. */ uint64_t last_reclaim; NM_SELINFO_T si; /* poll/select wait queue */ NM_LOCK_T q_lock; /* protects kring and ring. */ NM_ATOMIC_T nr_busy; /* prevent concurrent syscalls */ struct netmap_adapter *na; - /* The folloiwing fields are for VALE switch support */ + /* The following fields are for VALE switch support */ struct nm_bdg_fwd *nkr_ft; uint32_t *nkr_leases; #define NR_NOSLOT ((uint32_t)~0) /* used in nkr_*lease* */ uint32_t nkr_hwlease; uint32_t nkr_lease_idx; /* while nkr_stopped is set, no new [tr]xsync operations can * be started on this kring. * This is used by netmap_disable_all_rings() * to find a synchronization point where critical data * structures pointed to by the kring can be added or removed */ volatile int nkr_stopped; /* Support for adapters without native netmap support. * On tx rings we preallocate an array of tx buffers * (same size as the netmap ring), on rx rings we * store incoming mbufs in a queue that is drained by * a rxsync. */ struct mbuf **tx_pool; // u_int nr_ntc; /* Emulation of a next-to-clean RX ring pointer. */ struct mbq rx_queue; /* intercepted rx mbufs. */ uint32_t ring_id; /* debugging */ char name[64]; /* diagnostic */ /* [tx]sync callback for this kring. * The default nm_kring_create callback (netmap_krings_create) * sets the nm_sync callback of each hardware tx(rx) kring to * the corresponding nm_txsync(nm_rxsync) taken from the * netmap_adapter; moreover, it sets the sync callback * of the host tx(rx) ring to netmap_txsync_to_host * (netmap_rxsync_from_host). * * Overrides: the above configuration is not changed by * any of the nm_krings_create callbacks. */ int (*nm_sync)(struct netmap_kring *kring, int flags); #ifdef WITH_PIPES struct netmap_kring *pipe; /* if this is a pipe ring, * pointer to the other end */ struct netmap_ring *save_ring; /* pointer to hidden rings * (see netmap_pipe.c for details) */ #endif /* WITH_PIPES */ #ifdef WITH_MONITOR /* pointer to the adapter that is monitoring this kring (if any) */ struct netmap_monitor_adapter *monitor; /* * Monitors work by intercepting the txsync and/or rxsync of the * monitored krings. This is implemented by replacing * the nm_sync pointer above and saving the previous * one in save_sync below. */ int (*save_sync)(struct netmap_kring *kring, int flags); #endif } __attribute__((__aligned__(64))); /* return the next index, with wraparound */ static inline uint32_t nm_next(uint32_t i, uint32_t lim) { return unlikely (i == lim) ? 0 : i + 1; } /* return the previous index, with wraparound */ static inline uint32_t nm_prev(uint32_t i, uint32_t lim) { return unlikely (i == 0) ? lim : i - 1; } /* * * Here is the layout for the Rx and Tx rings. RxRING TxRING +-----------------+ +-----------------+ | | | | |XXX free slot XXX| |XXX free slot XXX| +-----------------+ +-----------------+ head->| owned by user |<-hwcur | not sent to nic |<-hwcur | | | yet | +-----------------+ | | cur->| available to | | | | user, not read | +-----------------+ | yet | cur->| (being | | | | prepared) | | | | | +-----------------+ + ------ + tail->| |<-hwtail | |<-hwlease | (being | ... | | ... | prepared) | ... | | ... +-----------------+ ... | | ... | |<-hwlease +-----------------+ | | tail->| |<-hwtail | | | | | | | | | | | | +-----------------+ +-----------------+ * The cur/tail (user view) and hwcur/hwtail (kernel view) * are used in the normal operation of the card. * * When a ring is the output of a switch port (Rx ring for * a VALE port, Tx ring for the host stack or NIC), slots * are reserved in blocks through 'hwlease' which points * to the next unused slot. * On an Rx ring, hwlease is always after hwtail, * and completions cause hwtail to advance. * On a Tx ring, hwlease is always between cur and hwtail, * and completions cause cur to advance. * * nm_kr_space() returns the maximum number of slots that * can be assigned. * nm_kr_lease() reserves the required number of buffers, * advances nkr_hwlease and also returns an entry in * a circular array where completions should be reported. */ enum txrx { NR_RX = 0, NR_TX = 1 }; struct netmap_vp_adapter; // forward /* * The "struct netmap_adapter" extends the "struct adapter" * (or equivalent) device descriptor. * It contains all base fields needed to support netmap operation. * There are in fact different types of netmap adapters * (native, generic, VALE switch...) so a netmap_adapter is * just the first field in the derived type. */ struct netmap_adapter { /* * On linux we do not have a good way to tell if an interface * is netmap-capable. So we always use the following trick: * NA(ifp) points here, and the first entry (which hopefully * always exists and is at least 32 bits) contains a magic * value which we can use to detect that the interface is good. */ uint32_t magic; uint32_t na_flags; /* enabled, and other flags */ #define NAF_SKIP_INTR 1 /* use the regular interrupt handler. * useful during initialization */ #define NAF_SW_ONLY 2 /* forward packets only to sw adapter */ #define NAF_BDG_MAYSLEEP 4 /* the bridge is allowed to sleep when * forwarding packets coming from this * interface */ #define NAF_MEM_OWNER 8 /* the adapter is responsible for the * deallocation of the memory allocator */ #define NAF_NATIVE_ON 16 /* the adapter is native and the attached * interface is in netmap mode. * Virtual ports (vale, pipe, monitor...) * should never use this flag. */ #define NAF_NETMAP_ON 32 /* netmap is active (either native or * emulated). Where possible (e.g. FreeBSD) * IFCAP_NETMAP also mirrors this flag. */ #define NAF_HOST_RINGS 64 /* the adapter supports the host rings */ #define NAF_FORCE_NATIVE 128 /* the adapter is always NATIVE */ #define NAF_BUSY (1U<<31) /* the adapter is used internally and * cannot be registered from userspace */ int active_fds; /* number of user-space descriptors using this interface, which is equal to the number of struct netmap_if objs in the mapped region. */ u_int num_rx_rings; /* number of adapter receive rings */ u_int num_tx_rings; /* number of adapter transmit rings */ u_int num_tx_desc; /* number of descriptor in each queue */ u_int num_rx_desc; /* tx_rings and rx_rings are private but allocated * as a contiguous chunk of memory. Each array has * N+1 entries, for the adapter queues and for the host queue. */ struct netmap_kring *tx_rings; /* array of TX rings. */ struct netmap_kring *rx_rings; /* array of RX rings. */ void *tailroom; /* space below the rings array */ /* (used for leases) */ NM_SELINFO_T tx_si, rx_si; /* global wait queues */ /* count users of the global wait queues */ int tx_si_users, rx_si_users; void *pdev; /* used to store pci device */ /* copy of if_qflush and if_transmit pointers, to intercept * packets from the network stack when netmap is active. */ int (*if_transmit)(struct ifnet *, struct mbuf *); /* copy of if_input for netmap_send_up() */ void (*if_input)(struct ifnet *, struct mbuf *); /* references to the ifnet and device routines, used by * the generic netmap functions. */ struct ifnet *ifp; /* adapter is ifp->if_softc */ /*---- callbacks for this netmap adapter -----*/ /* * nm_dtor() is the cleanup routine called when destroying * the adapter. * Called with NMG_LOCK held. * * nm_register() is called on NIOCREGIF and close() to enter * or exit netmap mode on the NIC * Called with NNG_LOCK held. * * nm_txsync() pushes packets to the underlying hw/switch * * nm_rxsync() collects packets from the underlying hw/switch * * nm_config() returns configuration information from the OS * Called with NMG_LOCK held. * * nm_krings_create() create and init the tx_rings and * rx_rings arrays of kring structures. In particular, * set the nm_sync callbacks for each ring. * There is no need to also allocate the corresponding * netmap_rings, since netmap_mem_rings_create() will always * be called to provide the missing ones. * Called with NNG_LOCK held. * * nm_krings_delete() cleanup and delete the tx_rings and rx_rings * arrays * Called with NMG_LOCK held. * * nm_notify() is used to act after data have become available * (or the stopped state of the ring has changed) * For hw devices this is typically a selwakeup(), * but for NIC/host ports attached to a switch (or vice-versa) * we also need to invoke the 'txsync' code downstream. */ void (*nm_dtor)(struct netmap_adapter *); int (*nm_register)(struct netmap_adapter *, int onoff); int (*nm_txsync)(struct netmap_kring *kring, int flags); int (*nm_rxsync)(struct netmap_kring *kring, int flags); #define NAF_FORCE_READ 1 #define NAF_FORCE_RECLAIM 2 /* return configuration information */ int (*nm_config)(struct netmap_adapter *, u_int *txr, u_int *txd, u_int *rxr, u_int *rxd); int (*nm_krings_create)(struct netmap_adapter *); void (*nm_krings_delete)(struct netmap_adapter *); int (*nm_notify)(struct netmap_adapter *, u_int ring, enum txrx, int flags); #define NAF_DISABLE_NOTIFY 8 /* notify that the stopped state of the * ring has changed (kring->nkr_stopped) */ #ifdef WITH_VALE /* * nm_bdg_attach() initializes the na_vp field to point * to an adapter that can be attached to a VALE switch. If the * current adapter is already a VALE port, na_vp is simply a cast; * otherwise, na_vp points to a netmap_bwrap_adapter. * If applicable, this callback also initializes na_hostvp, * that can be used to connect the adapter host rings to the * switch. * Called with NMG_LOCK held. * * nm_bdg_ctl() is called on the actual attach/detach to/from * to/from the switch, to perform adapter-specific * initializations * Called with NMG_LOCK held. */ int (*nm_bdg_attach)(const char *bdg_name, struct netmap_adapter *); int (*nm_bdg_ctl)(struct netmap_adapter *, struct nmreq *, int); /* adapter used to attach this adapter to a VALE switch (if any) */ struct netmap_vp_adapter *na_vp; /* adapter used to attach the host rings of this adapter * to a VALE switch (if any) */ struct netmap_vp_adapter *na_hostvp; #endif /* standard refcount to control the lifetime of the adapter * (it should be equal to the lifetime of the corresponding ifp) */ int na_refcount; /* memory allocator (opaque) * We also cache a pointer to the lut_entry for translating * buffer addresses, and the total number of buffers. */ struct netmap_mem_d *nm_mem; struct lut_entry *na_lut; uint32_t na_lut_objtotal; /* max buffer index */ uint32_t na_lut_objsize; /* buffer size */ /* additional information attached to this adapter * by other netmap subsystems. Currently used by * bwrap and LINUX/v1000. */ void *na_private; #ifdef WITH_PIPES /* array of pipes that have this adapter as a parent */ struct netmap_pipe_adapter **na_pipes; int na_next_pipe; /* next free slot in the array */ int na_max_pipes; /* size of the array */ #endif /* WITH_PIPES */ char name[64]; }; /* * If the NIC is owned by the kernel * (i.e., bridge), neither another bridge nor user can use it; * if the NIC is owned by a user, only users can share it. * Evaluation must be done under NMG_LOCK(). */ #define NETMAP_OWNED_BY_KERN(na) ((na)->na_flags & NAF_BUSY) #define NETMAP_OWNED_BY_ANY(na) \ (NETMAP_OWNED_BY_KERN(na) || ((na)->active_fds > 0)) /* * derived netmap adapters for various types of ports */ struct netmap_vp_adapter { /* VALE software port */ struct netmap_adapter up; /* * Bridge support: * * bdg_port is the port number used in the bridge; * na_bdg points to the bridge this NA is attached to. */ int bdg_port; struct nm_bridge *na_bdg; int retry; /* Offset of ethernet header for each packet. */ u_int virt_hdr_len; /* Maximum Frame Size, used in bdg_mismatch_datapath() */ u_int mfs; }; struct netmap_hw_adapter { /* physical device */ struct netmap_adapter up; struct net_device_ops nm_ndo; // XXX linux only struct ethtool_ops nm_eto; // XXX linux only const struct ethtool_ops* save_ethtool; int (*nm_hw_register)(struct netmap_adapter *, int onoff); }; +#ifdef WITH_GENERIC /* Mitigation support. */ struct nm_generic_mit { struct hrtimer mit_timer; int mit_pending; int mit_ring_idx; /* index of the ring being mitigated */ struct netmap_adapter *mit_na; /* backpointer */ }; struct netmap_generic_adapter { /* emulated device */ struct netmap_hw_adapter up; /* Pointer to a previously used netmap adapter. */ struct netmap_adapter *prev; /* generic netmap adapters support: * a net_device_ops struct overrides ndo_select_queue(), * save_if_input saves the if_input hook (FreeBSD), * mit implements rx interrupt mitigation, */ struct net_device_ops generic_ndo; void (*save_if_input)(struct ifnet *, struct mbuf *); struct nm_generic_mit *mit; #ifdef linux netdev_tx_t (*save_start_xmit)(struct mbuf *, struct ifnet *); #endif }; +#endif /* WITH_GENERIC */ static __inline int netmap_real_tx_rings(struct netmap_adapter *na) { return na->num_tx_rings + !!(na->na_flags & NAF_HOST_RINGS); } static __inline int netmap_real_rx_rings(struct netmap_adapter *na) { return na->num_rx_rings + !!(na->na_flags & NAF_HOST_RINGS); } #ifdef WITH_VALE /* * Bridge wrapper for non VALE ports attached to a VALE switch. * * The real device must already have its own netmap adapter (hwna). * The bridge wrapper and the hwna adapter share the same set of * netmap rings and buffers, but they have two separate sets of * krings descriptors, with tx/rx meanings swapped: * * netmap * bwrap krings rings krings hwna * +------+ +------+ +-----+ +------+ +------+ * |tx_rings->| |\ /| |----| |<-tx_rings| * | | +------+ \ / +-----+ +------+ | | * | | X | | * | | / \ | | * | | +------+/ \+-----+ +------+ | | * |rx_rings->| | | |----| |<-rx_rings| * | | +------+ +-----+ +------+ | | * +------+ +------+ * * - packets coming from the bridge go to the brwap rx rings, * which are also the hwna tx rings. The bwrap notify callback * will then complete the hwna tx (see netmap_bwrap_notify). * * - packets coming from the outside go to the hwna rx rings, * which are also the bwrap tx rings. The (overwritten) hwna * notify method will then complete the bridge tx * (see netmap_bwrap_intr_notify). * * The bridge wrapper may optionally connect the hwna 'host' rings * to the bridge. This is done by using a second port in the * bridge and connecting it to the 'host' netmap_vp_adapter * contained in the netmap_bwrap_adapter. The brwap host adapter * cross-links the hwna host rings in the same way as shown above. * * - packets coming from the bridge and directed to the host stack * are handled by the bwrap host notify callback * (see netmap_bwrap_host_notify) * * - packets coming from the host stack are still handled by the * overwritten hwna notify callback (netmap_bwrap_intr_notify), * but are diverted to the host adapter depending on the ring number. * */ struct netmap_bwrap_adapter { struct netmap_vp_adapter up; struct netmap_vp_adapter host; /* for host rings */ struct netmap_adapter *hwna; /* the underlying device */ /* backup of the hwna notify callback */ int (*save_notify)(struct netmap_adapter *, u_int ring, enum txrx, int flags); /* backup of the hwna memory allocator */ struct netmap_mem_d *save_nmd; /* * When we attach a physical interface to the bridge, we * allow the controlling process to terminate, so we need * a place to store the n_detmap_priv_d data structure. * This is only done when physical interfaces * are attached to a bridge. */ struct netmap_priv_d *na_kpriv; }; int netmap_bwrap_attach(const char *name, struct netmap_adapter *); #endif /* WITH_VALE */ #ifdef WITH_PIPES #define NM_MAXPIPES 64 /* max number of pipes per adapter */ struct netmap_pipe_adapter { struct netmap_adapter up; u_int id; /* pipe identifier */ int role; /* either NR_REG_PIPE_MASTER or NR_REG_PIPE_SLAVE */ struct netmap_adapter *parent; /* adapter that owns the memory */ struct netmap_pipe_adapter *peer; /* the other end of the pipe */ int peer_ref; /* 1 iff we are holding a ref to the peer */ u_int parent_slot; /* index in the parent pipe array */ }; #endif /* WITH_PIPES */ /* return slots reserved to rx clients; used in drivers */ static inline uint32_t nm_kr_rxspace(struct netmap_kring *k) { int space = k->nr_hwtail - k->nr_hwcur; if (space < 0) space += k->nkr_num_slots; ND("preserving %d rx slots %d -> %d", space, k->nr_hwcur, k->nr_hwtail); return space; } /* True if no space in the tx ring. only valid after txsync_prologue */ static inline int nm_kr_txempty(struct netmap_kring *kring) { return kring->rcur == kring->nr_hwtail; } /* * protect against multiple threads using the same ring. * also check that the ring has not been stopped. * We only care for 0 or !=0 as a return code. */ #define NM_KR_BUSY 1 #define NM_KR_STOPPED 2 static __inline void nm_kr_put(struct netmap_kring *kr) { NM_ATOMIC_CLEAR(&kr->nr_busy); } static __inline int nm_kr_tryget(struct netmap_kring *kr) { /* check a first time without taking the lock * to avoid starvation for nm_kr_get() */ if (unlikely(kr->nkr_stopped)) { ND("ring %p stopped (%d)", kr, kr->nkr_stopped); return NM_KR_STOPPED; } if (unlikely(NM_ATOMIC_TEST_AND_SET(&kr->nr_busy))) return NM_KR_BUSY; /* check a second time with lock held */ if (unlikely(kr->nkr_stopped)) { ND("ring %p stopped (%d)", kr, kr->nkr_stopped); nm_kr_put(kr); return NM_KR_STOPPED; } return 0; } /* * The following functions are used by individual drivers to * support netmap operation. * * netmap_attach() initializes a struct netmap_adapter, allocating the * struct netmap_ring's and the struct selinfo. * * netmap_detach() frees the memory allocated by netmap_attach(). * * netmap_transmit() replaces the if_transmit routine of the interface, * and is used to intercept packets coming from the stack. * * netmap_load_map/netmap_reload_map are helper routines to set/reset * the dmamap for a packet buffer * * netmap_reset() is a helper routine to be called in the hw driver * when reinitializing a ring. It should not be called by * virtual ports (vale, pipes, monitor) */ int netmap_attach(struct netmap_adapter *); void netmap_detach(struct ifnet *); int netmap_transmit(struct ifnet *, struct mbuf *); struct netmap_slot *netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n, u_int new_cur); int netmap_ring_reinit(struct netmap_kring *); /* default functions to handle rx/tx interrupts */ int netmap_rx_irq(struct ifnet *, u_int, u_int *); #define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL) void netmap_common_irq(struct ifnet *, u_int, u_int *work_done); #ifdef WITH_VALE /* functions used by external modules to interface with VALE */ #define netmap_vp_to_ifp(_vp) ((_vp)->up.ifp) #define netmap_ifp_to_vp(_ifp) (NA(_ifp)->na_vp) #define netmap_ifp_to_host_vp(_ifp) (NA(_ifp)->na_hostvp) #define netmap_bdg_idx(_vp) ((_vp)->bdg_port) const char *netmap_bdg_name(struct netmap_vp_adapter *); #else /* !WITH_VALE */ #define netmap_vp_to_ifp(_vp) NULL #define netmap_ifp_to_vp(_ifp) NULL #define netmap_ifp_to_host_vp(_ifp) NULL #define netmap_bdg_idx(_vp) -1 #define netmap_bdg_name(_vp) NULL #endif /* WITH_VALE */ static inline int nm_native_on(struct netmap_adapter *na) { return na && na->na_flags & NAF_NATIVE_ON; } static inline int nm_netmap_on(struct netmap_adapter *na) { return na && na->na_flags & NAF_NETMAP_ON; } /* set/clear native flags and if_transmit/netdev_ops */ static inline void nm_set_native_flags(struct netmap_adapter *na) { struct ifnet *ifp = na->ifp; na->na_flags |= (NAF_NATIVE_ON | NAF_NETMAP_ON); #ifdef IFCAP_NETMAP /* or FreeBSD ? */ ifp->if_capenable |= IFCAP_NETMAP; #endif #ifdef __FreeBSD__ na->if_transmit = ifp->if_transmit; ifp->if_transmit = netmap_transmit; #else na->if_transmit = (void *)ifp->netdev_ops; ifp->netdev_ops = &((struct netmap_hw_adapter *)na)->nm_ndo; ((struct netmap_hw_adapter *)na)->save_ethtool = ifp->ethtool_ops; ifp->ethtool_ops = &((struct netmap_hw_adapter*)na)->nm_eto; #endif } static inline void nm_clear_native_flags(struct netmap_adapter *na) { struct ifnet *ifp = na->ifp; #ifdef __FreeBSD__ ifp->if_transmit = na->if_transmit; #else ifp->netdev_ops = (void *)na->if_transmit; ifp->ethtool_ops = ((struct netmap_hw_adapter*)na)->save_ethtool; #endif na->na_flags &= ~(NAF_NATIVE_ON | NAF_NETMAP_ON); #ifdef IFCAP_NETMAP /* or FreeBSD ? */ ifp->if_capenable &= ~IFCAP_NETMAP; #endif } /* * validates parameters in the ring/kring, returns a value for head * If any error, returns ring_size to force a reinit. */ uint32_t nm_txsync_prologue(struct netmap_kring *); /* * validates parameters in the ring/kring, returns a value for head, * and the 'reserved' value in the argument. * If any error, returns ring_size lim to force a reinit. */ uint32_t nm_rxsync_prologue(struct netmap_kring *); /* * update kring and ring at the end of txsync. */ static inline void nm_txsync_finalize(struct netmap_kring *kring) { /* update ring tail to what the kernel knows */ kring->ring->tail = kring->rtail = kring->nr_hwtail; /* note, head/rhead/hwcur might be behind cur/rcur * if no carrier */ ND(5, "%s now hwcur %d hwtail %d head %d cur %d tail %d", kring->name, kring->nr_hwcur, kring->nr_hwtail, kring->rhead, kring->rcur, kring->rtail); } /* * update kring and ring at the end of rxsync */ static inline void nm_rxsync_finalize(struct netmap_kring *kring) { /* tell userspace that there might be new packets */ //struct netmap_ring *ring = kring->ring; ND("head %d cur %d tail %d -> %d", ring->head, ring->cur, ring->tail, kring->nr_hwtail); kring->ring->tail = kring->rtail = kring->nr_hwtail; /* make a copy of the state for next round */ kring->rhead = kring->ring->head; kring->rcur = kring->ring->cur; } /* check/fix address and len in tx rings */ #if 1 /* debug version */ #define NM_CHECK_ADDR_LEN(_na, _a, _l) do { \ if (_a == NETMAP_BUF_BASE(_na) || _l > NETMAP_BUF_SIZE(_na)) { \ RD(5, "bad addr/len ring %d slot %d idx %d len %d", \ kring->ring_id, nm_i, slot->buf_idx, len); \ if (_l > NETMAP_BUF_SIZE(_na)) \ _l = NETMAP_BUF_SIZE(_na); \ } } while (0) #else /* no debug version */ #define NM_CHECK_ADDR_LEN(_na, _a, _l) do { \ if (_l > NETMAP_BUF_SIZE(_na)) \ _l = NETMAP_BUF_SIZE(_na); \ } while (0) #endif /*---------------------------------------------------------------*/ /* * Support routines used by netmap subsystems * (native drivers, VALE, generic, pipes, monitors, ...) */ /* common routine for all functions that create a netmap adapter. It performs * two main tasks: * - if the na points to an ifp, mark the ifp as netmap capable * using na as its native adapter; * - provide defaults for the setup callbacks and the memory allocator */ int netmap_attach_common(struct netmap_adapter *); /* common actions to be performed on netmap adapter destruction */ void netmap_detach_common(struct netmap_adapter *); /* fill priv->np_[tr]xq{first,last} using the ringid and flags information * coming from a struct nmreq */ int netmap_interp_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags); /* update the ring parameters (number and size of tx and rx rings). * It calls the nm_config callback, if available. */ int netmap_update_config(struct netmap_adapter *na); /* create and initialize the common fields of the krings array. * using the information that must be already available in the na. * tailroom can be used to request the allocation of additional * tailroom bytes after the krings array. This is used by * netmap_vp_adapter's (i.e., VALE ports) to make room for * leasing-related data structures */ int netmap_krings_create(struct netmap_adapter *na, u_int tailroom); /* deletes the kring array of the adapter. The array must have * been created using netmap_krings_create */ void netmap_krings_delete(struct netmap_adapter *na); /* set the stopped/enabled status of ring * When stopping, they also wait for all current activity on the ring to * terminate. The status change is then notified using the na nm_notify * callback. */ void netmap_set_txring(struct netmap_adapter *, u_int ring_id, int stopped); void netmap_set_rxring(struct netmap_adapter *, u_int ring_id, int stopped); /* set the stopped/enabled status of all rings of the adapter. */ void netmap_set_all_rings(struct netmap_adapter *, int stopped); /* convenience wrappers for netmap_set_all_rings, used in drivers */ void netmap_disable_all_rings(struct ifnet *); void netmap_enable_all_rings(struct ifnet *); int netmap_rxsync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait); struct netmap_if * netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na, uint16_t ringid, uint32_t flags, int *err); u_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg); int netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na, int create); int netmap_get_hw_na(struct ifnet *ifp, struct netmap_adapter **na); #ifdef WITH_VALE /* * The following bridge-related functions are used by other * kernel modules. * * VALE only supports unicast or broadcast. The lookup * function can return 0 .. NM_BDG_MAXPORTS-1 for regular ports, * NM_BDG_MAXPORTS for broadcast, NM_BDG_MAXPORTS+1 for unknown. * XXX in practice "unknown" might be handled same as broadcast. */ typedef u_int (*bdg_lookup_fn_t)(struct nm_bdg_fwd *ft, uint8_t *ring_nr, const struct netmap_vp_adapter *); typedef int (*bdg_config_fn_t)(struct nm_ifreq *); typedef void (*bdg_dtor_fn_t)(const struct netmap_vp_adapter *); struct netmap_bdg_ops { bdg_lookup_fn_t lookup; bdg_config_fn_t config; bdg_dtor_fn_t dtor; }; u_int netmap_bdg_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring, const struct netmap_vp_adapter *); #define NM_BDG_MAXPORTS 254 /* up to 254 */ #define NM_BDG_BROADCAST NM_BDG_MAXPORTS #define NM_BDG_NOPORT (NM_BDG_MAXPORTS+1) #define NM_NAME "vale" /* prefix for bridge port name */ /* these are redefined in case of no VALE support */ int netmap_get_bdg_na(struct nmreq *nmr, struct netmap_adapter **na, int create); void netmap_init_bridges(void); int netmap_bdg_ctl(struct nmreq *nmr, struct netmap_bdg_ops *bdg_ops); int netmap_bdg_config(struct nmreq *nmr); #else /* !WITH_VALE */ #define netmap_get_bdg_na(_1, _2, _3) 0 #define netmap_init_bridges(_1) #define netmap_bdg_ctl(_1, _2) EINVAL #endif /* !WITH_VALE */ #ifdef WITH_PIPES /* max number of pipes per device */ #define NM_MAXPIPES 64 /* XXX how many? */ /* in case of no error, returns the actual number of pipes in nmr->nr_arg1 */ int netmap_pipe_alloc(struct netmap_adapter *, struct nmreq *nmr); void netmap_pipe_dealloc(struct netmap_adapter *); int netmap_get_pipe_na(struct nmreq *nmr, struct netmap_adapter **na, int create); #else /* !WITH_PIPES */ #define NM_MAXPIPES 0 #define netmap_pipe_alloc(_1, _2) EOPNOTSUPP #define netmap_pipe_dealloc(_1) #define netmap_get_pipe_na(_1, _2, _3) 0 #endif #ifdef WITH_MONITOR int netmap_get_monitor_na(struct nmreq *nmr, struct netmap_adapter **na, int create); #else #define netmap_get_monitor_na(_1, _2, _3) 0 #endif /* Various prototypes */ int netmap_poll(struct cdev *dev, int events, struct thread *td); int netmap_init(void); void netmap_fini(void); int netmap_get_memory(struct netmap_priv_d* p); void netmap_dtor(void *data); int netmap_dtor_locked(struct netmap_priv_d *priv); int netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td); /* netmap_adapter creation/destruction */ // #define NM_DEBUG_PUTGET 1 #ifdef NM_DEBUG_PUTGET #define NM_DBG(f) __##f void __netmap_adapter_get(struct netmap_adapter *na); #define netmap_adapter_get(na) \ do { \ struct netmap_adapter *__na = na; \ D("getting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount); \ __netmap_adapter_get(__na); \ } while (0) int __netmap_adapter_put(struct netmap_adapter *na); #define netmap_adapter_put(na) \ ({ \ struct netmap_adapter *__na = na; \ D("putting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount); \ __netmap_adapter_put(__na); \ }) #else /* !NM_DEBUG_PUTGET */ #define NM_DBG(f) f void netmap_adapter_get(struct netmap_adapter *na); int netmap_adapter_put(struct netmap_adapter *na); #endif /* !NM_DEBUG_PUTGET */ /* * module variables */ #define NETMAP_BUF_BASE(na) ((na)->na_lut[0].vaddr) #define NETMAP_BUF_SIZE(na) ((na)->na_lut_objsize) extern int netmap_mitigate; // XXX not really used extern int netmap_no_pendintr; extern int netmap_verbose; // XXX debugging enum { /* verbose flags */ NM_VERB_ON = 1, /* generic verbose */ NM_VERB_HOST = 0x2, /* verbose host stack */ NM_VERB_RXSYNC = 0x10, /* verbose on rxsync/txsync */ NM_VERB_TXSYNC = 0x20, NM_VERB_RXINTR = 0x100, /* verbose on rx/tx intr (driver) */ NM_VERB_TXINTR = 0x200, NM_VERB_NIC_RXSYNC = 0x1000, /* verbose on rx/tx intr (driver) */ NM_VERB_NIC_TXSYNC = 0x2000, }; extern int netmap_txsync_retry; extern int netmap_generic_mit; extern int netmap_generic_ringsize; extern int netmap_generic_rings; /* * NA returns a pointer to the struct netmap adapter from the ifp, * WNA is used to write it. */ -#ifndef WNA -#define WNA(_ifp) (_ifp)->if_pspare[0] -#endif #define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp)) /* * Macros to determine if an interface is netmap capable or netmap enabled. * See the magic field in struct netmap_adapter. */ #ifdef __FreeBSD__ /* * on FreeBSD just use if_capabilities and if_capenable. */ #define NETMAP_CAPABLE(ifp) (NA(ifp) && \ (ifp)->if_capabilities & IFCAP_NETMAP ) #define NETMAP_SET_CAPABLE(ifp) \ (ifp)->if_capabilities |= IFCAP_NETMAP #else /* linux */ /* * on linux: * we check if NA(ifp) is set and its first element has a related * magic value. The capenable is within the struct netmap_adapter. */ #define NETMAP_MAGIC 0x52697a7a #define NETMAP_CAPABLE(ifp) (NA(ifp) && \ ((uint32_t)(uintptr_t)NA(ifp) ^ NA(ifp)->magic) == NETMAP_MAGIC ) #define NETMAP_SET_CAPABLE(ifp) \ NA(ifp)->magic = ((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC #endif /* linux */ #ifdef __FreeBSD__ /* Assigns the device IOMMU domain to an allocator. * Returns -ENOMEM in case the domain is different */ #define nm_iommu_group_id(dev) (0) /* Callback invoked by the dma machinery after a successful dmamap_load */ static void netmap_dmamap_cb(__unused void *arg, __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error) { } /* bus_dmamap_load wrapper: call aforementioned function if map != NULL. * XXX can we do it without a callback ? */ static inline void netmap_load_map(struct netmap_adapter *na, bus_dma_tag_t tag, bus_dmamap_t map, void *buf) { if (map) bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na), netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); } static inline void netmap_unload_map(struct netmap_adapter *na, bus_dma_tag_t tag, bus_dmamap_t map) { if (map) bus_dmamap_unload(tag, map); } /* update the map when a buffer changes. */ static inline void netmap_reload_map(struct netmap_adapter *na, bus_dma_tag_t tag, bus_dmamap_t map, void *buf) { if (map) { bus_dmamap_unload(tag, map); bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na), netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); } } #else /* linux */ int nm_iommu_group_id(bus_dma_tag_t dev); extern size_t netmap_mem_get_bufsize(struct netmap_mem_d *); #include static inline void netmap_load_map(struct netmap_adapter *na, bus_dma_tag_t tag, bus_dmamap_t map, void *buf) { if (map) { *map = dma_map_single(na->pdev, buf, netmap_mem_get_bufsize(na->nm_mem), DMA_BIDIRECTIONAL); } } static inline void netmap_unload_map(struct netmap_adapter *na, bus_dma_tag_t tag, bus_dmamap_t map) { u_int sz = netmap_mem_get_bufsize(na->nm_mem); if (*map) { dma_unmap_single(na->pdev, *map, sz, DMA_BIDIRECTIONAL); } } static inline void netmap_reload_map(struct netmap_adapter *na, bus_dma_tag_t tag, bus_dmamap_t map, void *buf) { u_int sz = netmap_mem_get_bufsize(na->nm_mem); if (*map) { dma_unmap_single(na->pdev, *map, sz, DMA_BIDIRECTIONAL); } *map = dma_map_single(na->pdev, buf, sz, DMA_BIDIRECTIONAL); } /* * XXX How do we redefine these functions: * * on linux we need * dma_map_single(&pdev->dev, virt_addr, len, direction) * dma_unmap_single(&adapter->pdev->dev, phys_addr, len, direction * The len can be implicit (on netmap it is NETMAP_BUF_SIZE) * unfortunately the direction is not, so we need to change * something to have a cross API */ #if 0 struct e1000_buffer *buffer_info = &tx_ring->buffer_info[l]; /* set time_stamp *before* dma to help avoid a possible race */ buffer_info->time_stamp = jiffies; buffer_info->mapped_as_page = false; buffer_info->length = len; //buffer_info->next_to_watch = l; /* reload dma map */ dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, NETMAP_BUF_SIZE, DMA_TO_DEVICE); buffer_info->dma = dma_map_single(&adapter->pdev->dev, addr, NETMAP_BUF_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { D("dma mapping error"); /* goto dma_error; See e1000_put_txbuf() */ /* XXX reset */ } tx_desc->buffer_addr = htole64(buffer_info->dma); //XXX #endif /* * The bus_dmamap_sync() can be one of wmb() or rmb() depending on direction. */ #define bus_dmamap_sync(_a, _b, _c) #endif /* linux */ /* * functions to map NIC to KRING indexes (n2k) and vice versa (k2n) */ static inline int netmap_idx_n2k(struct netmap_kring *kr, int idx) { int n = kr->nkr_num_slots; idx += kr->nkr_hwofs; if (idx < 0) return idx + n; else if (idx < n) return idx; else return idx - n; } static inline int netmap_idx_k2n(struct netmap_kring *kr, int idx) { int n = kr->nkr_num_slots; idx -= kr->nkr_hwofs; if (idx < 0) return idx + n; else if (idx < n) return idx; else return idx - n; } /* Entries of the look-up table. */ struct lut_entry { void *vaddr; /* virtual address. */ vm_paddr_t paddr; /* physical address. */ }; struct netmap_obj_pool; /* * NMB return the virtual address of a buffer (buffer 0 on bad index) * PNMB also fills the physical address */ static inline void * NMB(struct netmap_adapter *na, struct netmap_slot *slot) { struct lut_entry *lut = na->na_lut; uint32_t i = slot->buf_idx; return (unlikely(i >= na->na_lut_objtotal)) ? lut[0].vaddr : lut[i].vaddr; } static inline void * PNMB(struct netmap_adapter *na, struct netmap_slot *slot, uint64_t *pp) { uint32_t i = slot->buf_idx; struct lut_entry *lut = na->na_lut; void *ret = (i >= na->na_lut_objtotal) ? lut[0].vaddr : lut[i].vaddr; *pp = (i >= na->na_lut_objtotal) ? lut[0].paddr : lut[i].paddr; return ret; } /* Generic version of NMB, which uses device-specific memory. */ void netmap_txsync_to_host(struct netmap_adapter *na); /* * Structure associated to each thread which registered an interface. * * The first 4 fields of this structure are written by NIOCREGIF and * read by poll() and NIOC?XSYNC. * * There is low contention among writers (a correct user program * should have none) and among writers and readers, so we use a * single global lock to protect the structure initialization; * since initialization involves the allocation of memory, * we reuse the memory allocator lock. * * Read access to the structure is lock free. Readers must check that * np_nifp is not NULL before using the other fields. * If np_nifp is NULL initialization has not been performed, * so they should return an error to userspace. * * The ref_done field is used to regulate access to the refcount in the * memory allocator. The refcount must be incremented at most once for * each open("/dev/netmap"). The increment is performed by the first * function that calls netmap_get_memory() (currently called by * mmap(), NIOCGINFO and NIOCREGIF). * If the refcount is incremented, it is then decremented when the * private structure is destroyed. */ struct netmap_priv_d { struct netmap_if * volatile np_nifp; /* netmap if descriptor. */ struct netmap_adapter *np_na; uint32_t np_flags; /* from the ioctl */ u_int np_txqfirst, np_txqlast; /* range of tx rings to scan */ u_int np_rxqfirst, np_rxqlast; /* range of rx rings to scan */ uint16_t np_txpoll; /* XXX and also np_rxpoll ? */ struct netmap_mem_d *np_mref; /* use with NMG_LOCK held */ /* np_refcount is only used on FreeBSD */ int np_refcount; /* use with NMG_LOCK held */ /* pointers to the selinfo to be used for selrecord. * Either the local or the global one depending on the * number of rings. */ NM_SELINFO_T *np_rxsi, *np_txsi; struct thread *np_td; /* kqueue, just debugging */ }; #ifdef WITH_MONITOR struct netmap_monitor_adapter { struct netmap_adapter up; struct netmap_priv_d priv; uint32_t flags; }; #endif /* WITH_MONITOR */ +#ifdef WITH_GENERIC /* * generic netmap emulation for devices that do not have * native netmap support. */ int generic_netmap_attach(struct ifnet *ifp); int netmap_catch_rx(struct netmap_adapter *na, int intercept); void generic_rx_handler(struct ifnet *ifp, struct mbuf *m);; void netmap_catch_tx(struct netmap_generic_adapter *na, int enable); int generic_xmit_frame(struct ifnet *ifp, struct mbuf *m, void *addr, u_int len, u_int ring_nr); int generic_find_num_desc(struct ifnet *ifp, u_int *tx, u_int *rx); void generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq); //#define RATE_GENERIC /* Enables communication statistics for generic. */ #ifdef RATE_GENERIC void generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi); #else #define generic_rate(txp, txs, txi, rxp, rxs, rxi) #endif /* * netmap_mitigation API. This is used by the generic adapter * to reduce the number of interrupt requests/selwakeup * to clients on incoming packets. */ void netmap_mitigation_init(struct nm_generic_mit *mit, int idx, struct netmap_adapter *na); void netmap_mitigation_start(struct nm_generic_mit *mit); void netmap_mitigation_restart(struct nm_generic_mit *mit); int netmap_mitigation_active(struct nm_generic_mit *mit); void netmap_mitigation_cleanup(struct nm_generic_mit *mit); +#endif /* WITH_GENERIC */ /* Shared declarations for the VALE switch. */ /* * Each transmit queue accumulates a batch of packets into * a structure before forwarding. Packets to the same * destination are put in a list using ft_next as a link field. * ft_frags and ft_next are valid only on the first fragment. */ struct nm_bdg_fwd { /* forwarding entry for a bridge */ void *ft_buf; /* netmap or indirect buffer */ uint8_t ft_frags; /* how many fragments (only on 1st frag) */ uint8_t _ft_port; /* dst port (unused) */ uint16_t ft_flags; /* flags, e.g. indirect */ uint16_t ft_len; /* src fragment len */ uint16_t ft_next; /* next packet to same destination */ }; /* struct 'virtio_net_hdr' from linux. */ struct nm_vnet_hdr { #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */ #define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */ uint8_t flags; #define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */ #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */ #define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */ #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* GSO frame, IPv6 TCP */ #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* TCP has ECN set */ uint8_t gso_type; uint16_t hdr_len; uint16_t gso_size; uint16_t csum_start; uint16_t csum_offset; }; #define WORST_CASE_GSO_HEADER (14+40+60) /* IPv6 + TCP */ /* Private definitions for IPv4, IPv6, UDP and TCP headers. */ struct nm_iphdr { uint8_t version_ihl; uint8_t tos; uint16_t tot_len; uint16_t id; uint16_t frag_off; uint8_t ttl; uint8_t protocol; uint16_t check; uint32_t saddr; uint32_t daddr; /*The options start here. */ }; struct nm_tcphdr { uint16_t source; uint16_t dest; uint32_t seq; uint32_t ack_seq; uint8_t doff; /* Data offset + Reserved */ uint8_t flags; uint16_t window; uint16_t check; uint16_t urg_ptr; }; struct nm_udphdr { uint16_t source; uint16_t dest; uint16_t len; uint16_t check; }; struct nm_ipv6hdr { uint8_t priority_version; uint8_t flow_lbl[3]; uint16_t payload_len; uint8_t nexthdr; uint8_t hop_limit; uint8_t saddr[16]; uint8_t daddr[16]; }; /* Type used to store a checksum (in host byte order) that hasn't been * folded yet. */ #define rawsum_t uint32_t rawsum_t nm_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum); uint16_t nm_csum_ipv4(struct nm_iphdr *iph); void nm_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, size_t datalen, uint16_t *check); void nm_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, size_t datalen, uint16_t *check); uint16_t nm_csum_fold(rawsum_t cur_sum); void bdg_mismatch_datapath(struct netmap_vp_adapter *na, struct netmap_vp_adapter *dst_na, struct nm_bdg_fwd *ft_p, struct netmap_ring *ring, u_int *j, u_int lim, u_int *howmany); /* persistent virtual port routines */ int nm_vi_persist(const char *, struct ifnet **); void nm_vi_detach(struct ifnet *); void nm_vi_init_index(void); #endif /* _NET_NETMAP_KERN_H_ */ Index: stable/10/sys/dev/netmap/netmap_monitor.c =================================================================== --- stable/10/sys/dev/netmap/netmap_monitor.c (revision 278778) +++ stable/10/sys/dev/netmap/netmap_monitor.c (revision 278779) @@ -1,498 +1,498 @@ /* * Copyright (C) 2014 Giuseppe Lettieri. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ * * Monitors * * netmap monitors can be used to do zero-copy monitoring of network traffic * on another adapter, when the latter adapter is working in netmap mode. * * Monitors offer to userspace the same interface as any other netmap port, * with as many pairs of netmap rings as the monitored adapter. * However, only the rx rings are actually used. Each monitor rx ring receives * the traffic transiting on both the tx and rx corresponding rings in the * monitored adapter. During registration, the user can choose if she wants * to intercept tx only, rx only, or both tx and rx traffic. * * The monitor only sees the frames after they have been consumed in the * monitored adapter: * * - For tx traffic, this is after the slots containing the frames have been * marked as free. Note that this may happen at a considerably delay after * frame transmission, since freeing of slots is often done lazily. * * - For rx traffic, this is after the consumer on the monitored adapter * has released them. In most cases, the consumer is a userspace * application which may have modified the frame contents. * * If the monitor is not able to cope with the stream of frames, excess traffic * will be dropped. * * Each ring can be monitored by at most one monitor. This may change in the * future, if we implement monitor chaining. * */ #if defined(__FreeBSD__) #include /* prerequisite */ #include #include #include /* defines used in kernel.h */ #include /* types used in module initialization */ #include #include #include #include #include #include #include /* sockaddrs */ #include #include #include /* bus_dmamap_* */ #include #elif defined(linux) #include "bsd_glue.h" #elif defined(__APPLE__) #warning OSX support is only partial #include "osx_glue.h" #else #error Unsupported platform #endif /* unsupported */ /* * common headers */ #include #include #include #ifdef WITH_MONITOR #define NM_MONITOR_MAXSLOTS 4096 /* monitor works by replacing the nm_sync callbacks in the monitored rings. * The actions to be performed are the same on both tx and rx rings, so we * have collected them here */ static int netmap_monitor_parent_sync(struct netmap_kring *kring, int flags, u_int* ringptr) { struct netmap_monitor_adapter *mna = kring->monitor; struct netmap_kring *mkring = &mna->up.rx_rings[kring->ring_id]; struct netmap_ring *ring = kring->ring, *mring = mkring->ring; int error; int rel_slots, free_slots, busy; u_int beg, end, i; u_int lim = kring->nkr_num_slots - 1, mlim = mkring->nkr_num_slots - 1; /* get the relased slots (rel_slots) */ beg = *ringptr; error = kring->save_sync(kring, flags); if (error) return error; end = *ringptr; rel_slots = end - beg; if (rel_slots < 0) rel_slots += kring->nkr_num_slots; if (!rel_slots) { return 0; } /* we need to lock the monitor receive ring, since it * is the target of bot tx and rx traffic from the monitored * adapter */ mtx_lock(&mkring->q_lock); /* get the free slots available on the monitor ring */ i = mkring->nr_hwtail; busy = i - mkring->nr_hwcur; if (busy < 0) busy += mkring->nkr_num_slots; free_slots = mlim - busy; if (!free_slots) { mtx_unlock(&mkring->q_lock); return 0; } /* swap min(free_slots, rel_slots) slots */ if (free_slots < rel_slots) { beg += (rel_slots - free_slots); if (beg > lim) beg = 0; rel_slots = free_slots; } for ( ; rel_slots; rel_slots--) { struct netmap_slot *s = &ring->slot[beg]; struct netmap_slot *ms = &mring->slot[i]; uint32_t tmp; tmp = ms->buf_idx; ms->buf_idx = s->buf_idx; s->buf_idx = tmp; tmp = ms->len; ms->len = s->len; s->len = tmp; s->flags |= NS_BUF_CHANGED; beg = nm_next(beg, lim); i = nm_next(i, mlim); } - wmb(); + mb(); mkring->nr_hwtail = i; mtx_unlock(&mkring->q_lock); /* notify the new frames to the monitor */ mna->up.nm_notify(&mna->up, mkring->ring_id, NR_RX, 0); return 0; } /* callback used to replace the nm_sync callback in the monitored tx rings */ static int netmap_monitor_parent_txsync(struct netmap_kring *kring, int flags) { ND("%s %x", kring->name, flags); return netmap_monitor_parent_sync(kring, flags, &kring->nr_hwtail); } /* callback used to replace the nm_sync callback in the monitored rx rings */ static int netmap_monitor_parent_rxsync(struct netmap_kring *kring, int flags) { ND("%s %x", kring->name, flags); return netmap_monitor_parent_sync(kring, flags, &kring->rcur); } /* nm_sync callback for the monitor's own tx rings. * This makes no sense and always returns error */ static int netmap_monitor_txsync(struct netmap_kring *kring, int flags) { D("%s %x", kring->name, flags); return EIO; } /* nm_sync callback for the monitor's own rx rings. * Note that the lock in netmap_monitor_parent_sync only protects * writers among themselves. Synchronization between writers * (i.e., netmap_monitor_parent_txsync and netmap_monitor_parent_rxsync) * and readers (i.e., netmap_monitor_rxsync) relies on memory barriers. */ static int netmap_monitor_rxsync(struct netmap_kring *kring, int flags) { ND("%s %x", kring->name, flags); kring->nr_hwcur = kring->rcur; - rmb(); + mb(); nm_rxsync_finalize(kring); return 0; } /* nm_krings_create callbacks for monitors. * We could use the default netmap_hw_krings_monitor, but * we don't need the mbq. */ static int netmap_monitor_krings_create(struct netmap_adapter *na) { return netmap_krings_create(na, 0); } /* nm_register callback for monitors. * * On registration, replace the nm_sync callbacks in the monitored * rings with our own, saving the previous ones in the monitored * rings themselves, where they are used by netmap_monitor_parent_sync. * * On de-registration, restore the original callbacks. We need to * stop traffic while we are doing this, since the monitored adapter may * have already started executing a netmap_monitor_parent_sync * and may not like the kring->save_sync pointer to become NULL. */ static int netmap_monitor_reg(struct netmap_adapter *na, int onoff) { struct netmap_monitor_adapter *mna = (struct netmap_monitor_adapter *)na; struct netmap_priv_d *priv = &mna->priv; struct netmap_adapter *pna = priv->np_na; struct netmap_kring *kring; int i; ND("%p: onoff %d", na, onoff); if (onoff) { if (!nm_netmap_on(pna)) { /* parent left netmap mode, fatal */ return ENXIO; } if (mna->flags & NR_MONITOR_TX) { for (i = priv->np_txqfirst; i < priv->np_txqlast; i++) { kring = &pna->tx_rings[i]; kring->save_sync = kring->nm_sync; kring->nm_sync = netmap_monitor_parent_txsync; } } if (mna->flags & NR_MONITOR_RX) { for (i = priv->np_rxqfirst; i < priv->np_rxqlast; i++) { kring = &pna->rx_rings[i]; kring->save_sync = kring->nm_sync; kring->nm_sync = netmap_monitor_parent_rxsync; } } na->na_flags |= NAF_NETMAP_ON; } else { if (!nm_netmap_on(pna)) { /* parent left netmap mode, nothing to restore */ return 0; } na->na_flags &= ~NAF_NETMAP_ON; if (mna->flags & NR_MONITOR_TX) { for (i = priv->np_txqfirst; i < priv->np_txqlast; i++) { netmap_set_txring(pna, i, 1 /* stopped */); kring = &pna->tx_rings[i]; kring->nm_sync = kring->save_sync; kring->save_sync = NULL; netmap_set_txring(pna, i, 0 /* enabled */); } } if (mna->flags & NR_MONITOR_RX) { for (i = priv->np_rxqfirst; i < priv->np_rxqlast; i++) { netmap_set_rxring(pna, i, 1 /* stopped */); kring = &pna->rx_rings[i]; kring->nm_sync = kring->save_sync; kring->save_sync = NULL; netmap_set_rxring(pna, i, 0 /* enabled */); } } } return 0; } /* nm_krings_delete callback for monitors */ static void netmap_monitor_krings_delete(struct netmap_adapter *na) { netmap_krings_delete(na); } /* nm_dtor callback for monitors */ static void netmap_monitor_dtor(struct netmap_adapter *na) { struct netmap_monitor_adapter *mna = (struct netmap_monitor_adapter *)na; struct netmap_priv_d *priv = &mna->priv; struct netmap_adapter *pna = priv->np_na; int i; ND("%p", na); if (nm_netmap_on(pna)) { /* parent still in netmap mode, mark its krings as free */ if (mna->flags & NR_MONITOR_TX) { for (i = priv->np_txqfirst; i < priv->np_txqlast; i++) { pna->tx_rings[i].monitor = NULL; } } if (mna->flags & NR_MONITOR_RX) { for (i = priv->np_rxqfirst; i < priv->np_rxqlast; i++) { pna->rx_rings[i].monitor = NULL; } } } netmap_adapter_put(pna); } /* check if nmr is a request for a monitor adapter that we can satisfy */ int netmap_get_monitor_na(struct nmreq *nmr, struct netmap_adapter **na, int create) { struct nmreq pnmr; struct netmap_adapter *pna; /* parent adapter */ struct netmap_monitor_adapter *mna; int i, error; if ((nmr->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX)) == 0) { ND("not a monitor"); return 0; } /* this is a request for a monitor adapter */ D("flags %x", nmr->nr_flags); mna = malloc(sizeof(*mna), M_DEVBUF, M_NOWAIT | M_ZERO); if (mna == NULL) { D("memory error"); return ENOMEM; } /* first, try to find the adapter that we want to monitor * We use the same nmr, after we have turned off the monitor flags. * In this way we can potentially monitor everything netmap understands, * except other monitors. */ memcpy(&pnmr, nmr, sizeof(pnmr)); pnmr.nr_flags &= ~(NR_MONITOR_TX | NR_MONITOR_RX); error = netmap_get_na(&pnmr, &pna, create); if (error) { D("parent lookup failed: %d", error); return error; } D("found parent: %s", pna->name); if (!nm_netmap_on(pna)) { /* parent not in netmap mode */ /* XXX we can wait for the parent to enter netmap mode, * by intercepting its nm_register callback (2014-03-16) */ D("%s not in netmap mode", pna->name); error = EINVAL; goto put_out; } /* grab all the rings we need in the parent */ mna->priv.np_na = pna; error = netmap_interp_ringid(&mna->priv, nmr->nr_ringid, nmr->nr_flags); if (error) { D("ringid error"); goto put_out; } if (nmr->nr_flags & NR_MONITOR_TX) { for (i = mna->priv.np_txqfirst; i < mna->priv.np_txqlast; i++) { struct netmap_kring *kring = &pna->tx_rings[i]; if (kring->monitor) { error = EBUSY; D("ring busy"); goto release_out; } kring->monitor = mna; } } if (nmr->nr_flags & NR_MONITOR_RX) { for (i = mna->priv.np_rxqfirst; i < mna->priv.np_rxqlast; i++) { struct netmap_kring *kring = &pna->rx_rings[i]; if (kring->monitor) { error = EBUSY; D("ring busy"); goto release_out; } kring->monitor = mna; } } snprintf(mna->up.name, sizeof(mna->up.name), "mon:%s", pna->name); /* the monitor supports the host rings iff the parent does */ mna->up.na_flags = (pna->na_flags & NAF_HOST_RINGS); mna->up.nm_txsync = netmap_monitor_txsync; mna->up.nm_rxsync = netmap_monitor_rxsync; mna->up.nm_register = netmap_monitor_reg; mna->up.nm_dtor = netmap_monitor_dtor; mna->up.nm_krings_create = netmap_monitor_krings_create; mna->up.nm_krings_delete = netmap_monitor_krings_delete; mna->up.nm_mem = pna->nm_mem; mna->up.na_lut = pna->na_lut; mna->up.na_lut_objtotal = pna->na_lut_objtotal; mna->up.na_lut_objsize = pna->na_lut_objsize; mna->up.num_tx_rings = 1; // XXX we don't need it, but field can't be zero /* we set the number of our rx_rings to be max(num_rx_rings, num_rx_rings) * in the parent */ mna->up.num_rx_rings = pna->num_rx_rings; if (pna->num_tx_rings > pna->num_rx_rings) mna->up.num_rx_rings = pna->num_tx_rings; /* by default, the number of slots is the same as in * the parent rings, but the user may ask for a different * number */ mna->up.num_tx_desc = nmr->nr_tx_slots; nm_bound_var(&mna->up.num_tx_desc, pna->num_tx_desc, 1, NM_MONITOR_MAXSLOTS, NULL); mna->up.num_rx_desc = nmr->nr_rx_slots; nm_bound_var(&mna->up.num_rx_desc, pna->num_rx_desc, 1, NM_MONITOR_MAXSLOTS, NULL); error = netmap_attach_common(&mna->up); if (error) { D("attach_common error"); goto release_out; } /* remember the traffic directions we have to monitor */ mna->flags = (nmr->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX)); *na = &mna->up; netmap_adapter_get(*na); /* write the configuration back */ nmr->nr_tx_rings = mna->up.num_tx_rings; nmr->nr_rx_rings = mna->up.num_rx_rings; nmr->nr_tx_slots = mna->up.num_tx_desc; nmr->nr_rx_slots = mna->up.num_rx_desc; /* keep the reference to the parent */ D("monitor ok"); return 0; release_out: D("monitor error"); for (i = mna->priv.np_txqfirst; i < mna->priv.np_txqlast; i++) { if (pna->tx_rings[i].monitor == mna) pna->tx_rings[i].monitor = NULL; } for (i = mna->priv.np_rxqfirst; i < mna->priv.np_rxqlast; i++) { if (pna->rx_rings[i].monitor == mna) pna->rx_rings[i].monitor = NULL; } put_out: netmap_adapter_put(pna); free(mna, M_DEVBUF); return error; } #endif /* WITH_MONITOR */ Index: stable/10/sys/dev/netmap/netmap_pipe.c =================================================================== --- stable/10/sys/dev/netmap/netmap_pipe.c (revision 278778) +++ stable/10/sys/dev/netmap/netmap_pipe.c (revision 278779) @@ -1,688 +1,688 @@ /* * Copyright (C) 2014 Giuseppe Lettieri. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* $FreeBSD$ */ #if defined(__FreeBSD__) #include /* prerequisite */ #include #include #include /* defines used in kernel.h */ #include /* types used in module initialization */ #include #include #include #include #include #include #include /* sockaddrs */ #include #include #include /* bus_dmamap_* */ #include #elif defined(linux) #include "bsd_glue.h" #elif defined(__APPLE__) #warning OSX support is only partial #include "osx_glue.h" #else #error Unsupported platform #endif /* unsupported */ /* * common headers */ #include #include #include #ifdef WITH_PIPES #define NM_PIPE_MAXSLOTS 4096 int netmap_default_pipes = 0; /* default number of pipes for each nic */ SYSCTL_DECL(_dev_netmap); SYSCTL_INT(_dev_netmap, OID_AUTO, default_pipes, CTLFLAG_RW, &netmap_default_pipes, 0 , ""); /* allocate the pipe array in the parent adapter */ int netmap_pipe_alloc(struct netmap_adapter *na, struct nmreq *nmr) { size_t len; int mode = nmr->nr_flags & NR_REG_MASK; u_int npipes; if (mode == NR_REG_PIPE_MASTER || mode == NR_REG_PIPE_SLAVE) { /* this is for our parent, not for us */ return 0; } /* TODO: we can resize the array if the new * request can accomodate the already existing pipes */ if (na->na_pipes) { nmr->nr_arg1 = na->na_max_pipes; return 0; } npipes = nmr->nr_arg1; if (npipes == 0) npipes = netmap_default_pipes; nm_bound_var(&npipes, 0, 0, NM_MAXPIPES, NULL); if (npipes == 0) { /* really zero, nothing to alloc */ goto out; } len = sizeof(struct netmap_pipe_adapter *) * npipes; na->na_pipes = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO); if (na->na_pipes == NULL) return ENOMEM; na->na_max_pipes = npipes; na->na_next_pipe = 0; out: nmr->nr_arg1 = npipes; return 0; } /* deallocate the parent array in the parent adapter */ void netmap_pipe_dealloc(struct netmap_adapter *na) { if (na->na_pipes) { ND("freeing pipes for %s", na->name); free(na->na_pipes, M_DEVBUF); na->na_pipes = NULL; na->na_max_pipes = 0; na->na_next_pipe = 0; } } /* find a pipe endpoint with the given id among the parent's pipes */ static struct netmap_pipe_adapter * netmap_pipe_find(struct netmap_adapter *parent, u_int pipe_id) { int i; struct netmap_pipe_adapter *na; for (i = 0; i < parent->na_next_pipe; i++) { na = parent->na_pipes[i]; if (na->id == pipe_id) { return na; } } return NULL; } /* add a new pipe endpoint to the parent array */ static int netmap_pipe_add(struct netmap_adapter *parent, struct netmap_pipe_adapter *na) { if (parent->na_next_pipe >= parent->na_max_pipes) { D("%s: no space left for pipes", parent->name); return ENOMEM; } parent->na_pipes[parent->na_next_pipe] = na; na->parent_slot = parent->na_next_pipe; parent->na_next_pipe++; return 0; } /* remove the given pipe endpoint from the parent array */ static void netmap_pipe_remove(struct netmap_adapter *parent, struct netmap_pipe_adapter *na) { u_int n; n = --parent->na_next_pipe; if (n != na->parent_slot) { parent->na_pipes[na->parent_slot] = parent->na_pipes[n]; } parent->na_pipes[n] = NULL; } static int netmap_pipe_txsync(struct netmap_kring *txkring, int flags) { struct netmap_kring *rxkring = txkring->pipe; u_int limit; /* slots to transfer */ u_int j, k, lim_tx = txkring->nkr_num_slots - 1, lim_rx = rxkring->nkr_num_slots - 1; int m, busy; ND("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name); ND(2, "before: hwcur %d hwtail %d cur %d head %d tail %d", txkring->nr_hwcur, txkring->nr_hwtail, txkring->rcur, txkring->rhead, txkring->rtail); j = rxkring->nr_hwtail; /* RX */ k = txkring->nr_hwcur; /* TX */ m = txkring->rhead - txkring->nr_hwcur; /* new slots */ if (m < 0) m += txkring->nkr_num_slots; limit = m; - m = rxkring->nkr_num_slots - 1; /* max avail space on destination */ + m = lim_rx; /* max avail space on destination */ busy = j - rxkring->nr_hwcur; /* busy slots */ if (busy < 0) - busy += txkring->nkr_num_slots; + busy += rxkring->nkr_num_slots; m -= busy; /* subtract busy slots */ ND(2, "m %d limit %d", m, limit); if (m < limit) limit = m; if (limit == 0) { /* either the rxring is full, or nothing to send */ nm_txsync_finalize(txkring); /* actually useless */ return 0; } while (limit-- > 0) { struct netmap_slot *rs = &rxkring->save_ring->slot[j]; struct netmap_slot *ts = &txkring->ring->slot[k]; struct netmap_slot tmp; /* swap the slots */ tmp = *rs; *rs = *ts; *ts = tmp; /* no need to report the buffer change */ j = nm_next(j, lim_rx); k = nm_next(k, lim_tx); } - wmb(); /* make sure the slots are updated before publishing them */ + mb(); /* make sure the slots are updated before publishing them */ rxkring->nr_hwtail = j; txkring->nr_hwcur = k; txkring->nr_hwtail = nm_prev(k, lim_tx); nm_txsync_finalize(txkring); ND(2, "after: hwcur %d hwtail %d cur %d head %d tail %d j %d", txkring->nr_hwcur, txkring->nr_hwtail, txkring->rcur, txkring->rhead, txkring->rtail, j); - wmb(); /* make sure rxkring->nr_hwtail is updated before notifying */ + mb(); /* make sure rxkring->nr_hwtail is updated before notifying */ rxkring->na->nm_notify(rxkring->na, rxkring->ring_id, NR_RX, 0); return 0; } static int netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags) { struct netmap_kring *txkring = rxkring->pipe; uint32_t oldhwcur = rxkring->nr_hwcur; ND("%s %x <- %s", rxkring->name, flags, txkring->name); rxkring->nr_hwcur = rxkring->rhead; /* recover user-relased slots */ ND(5, "hwcur %d hwtail %d cur %d head %d tail %d", rxkring->nr_hwcur, rxkring->nr_hwtail, rxkring->rcur, rxkring->rhead, rxkring->rtail); - rmb(); /* paired with the first wmb() in txsync */ + mb(); /* paired with the first mb() in txsync */ nm_rxsync_finalize(rxkring); if (oldhwcur != rxkring->nr_hwcur) { /* we have released some slots, notify the other end */ - wmb(); /* make sure nr_hwcur is updated before notifying */ + mb(); /* make sure nr_hwcur is updated before notifying */ txkring->na->nm_notify(txkring->na, txkring->ring_id, NR_TX, 0); } return 0; } /* Pipe endpoints are created and destroyed together, so that endopoints do not * have to check for the existence of their peer at each ?xsync. * * To play well with the existing netmap infrastructure (refcounts etc.), we * adopt the following strategy: * * 1) The first endpoint that is created also creates the other endpoint and * grabs a reference to it. * * state A) user1 --> endpoint1 --> endpoint2 * * 2) If, starting from state A, endpoint2 is then registered, endpoint1 gives * its reference to the user: * * state B) user1 --> endpoint1 endpoint2 <--- user2 * * 3) Assume that, starting from state B endpoint2 is closed. In the unregister * callback endpoint2 notes that endpoint1 is still active and adds a reference * from endpoint1 to itself. When user2 then releases her own reference, * endpoint2 is not destroyed and we are back to state A. A symmetrical state * would be reached if endpoint1 were released instead. * * 4) If, starting from state A, endpoint1 is closed, the destructor notes that * it owns a reference to endpoint2 and releases it. * * Something similar goes on for the creation and destruction of the krings. */ /* netmap_pipe_krings_delete. * * There are two cases: * * 1) state is * * usr1 --> e1 --> e2 * * and we are e1. We have to create both sets * of krings. * * 2) state is * * usr1 --> e1 --> e2 * * and we are e2. e1 is certainly registered and our * krings already exist, but they may be hidden. */ static int netmap_pipe_krings_create(struct netmap_adapter *na) { struct netmap_pipe_adapter *pna = (struct netmap_pipe_adapter *)na; struct netmap_adapter *ona = &pna->peer->up; int error = 0; if (pna->peer_ref) { int i; /* case 1) above */ D("%p: case 1, create everything", na); error = netmap_krings_create(na, 0); if (error) goto err; /* we also create all the rings, since we need to * update the save_ring pointers. * netmap_mem_rings_create (called by our caller) * will not create the rings again */ error = netmap_mem_rings_create(na); if (error) goto del_krings1; /* update our hidden ring pointers */ for (i = 0; i < na->num_tx_rings + 1; i++) na->tx_rings[i].save_ring = na->tx_rings[i].ring; for (i = 0; i < na->num_rx_rings + 1; i++) na->rx_rings[i].save_ring = na->rx_rings[i].ring; /* now, create krings and rings of the other end */ error = netmap_krings_create(ona, 0); if (error) goto del_rings1; error = netmap_mem_rings_create(ona); if (error) goto del_krings2; for (i = 0; i < ona->num_tx_rings + 1; i++) ona->tx_rings[i].save_ring = ona->tx_rings[i].ring; for (i = 0; i < ona->num_rx_rings + 1; i++) ona->rx_rings[i].save_ring = ona->rx_rings[i].ring; /* cross link the krings */ for (i = 0; i < na->num_tx_rings; i++) { na->tx_rings[i].pipe = pna->peer->up.rx_rings + i; na->rx_rings[i].pipe = pna->peer->up.tx_rings + i; pna->peer->up.tx_rings[i].pipe = na->rx_rings + i; pna->peer->up.rx_rings[i].pipe = na->tx_rings + i; } } else { int i; /* case 2) above */ /* recover the hidden rings */ ND("%p: case 2, hidden rings", na); for (i = 0; i < na->num_tx_rings + 1; i++) na->tx_rings[i].ring = na->tx_rings[i].save_ring; for (i = 0; i < na->num_rx_rings + 1; i++) na->rx_rings[i].ring = na->rx_rings[i].save_ring; } return 0; del_krings2: netmap_krings_delete(ona); del_rings1: netmap_mem_rings_delete(na); del_krings1: netmap_krings_delete(na); err: return error; } /* netmap_pipe_reg. * * There are two cases on registration (onoff==1) * * 1.a) state is * * usr1 --> e1 --> e2 * * and we are e1. Nothing special to do. * * 1.b) state is * * usr1 --> e1 --> e2 <-- usr2 * * and we are e2. Drop the ref e1 is holding. * * There are two additional cases on unregister (onoff==0) * * 2.a) state is * * usr1 --> e1 --> e2 * * and we are e1. Nothing special to do, e2 will * be cleaned up by the destructor of e1. * * 2.b) state is * * usr1 --> e1 e2 <-- usr2 * * and we are either e1 or e2. Add a ref from the * other end and hide our rings. */ static int netmap_pipe_reg(struct netmap_adapter *na, int onoff) { struct netmap_pipe_adapter *pna = (struct netmap_pipe_adapter *)na; ND("%p: onoff %d", na, onoff); if (onoff) { na->na_flags |= NAF_NETMAP_ON; } else { na->na_flags &= ~NAF_NETMAP_ON; } if (pna->peer_ref) { ND("%p: case 1.a or 2.a, nothing to do", na); return 0; } if (onoff) { ND("%p: case 1.b, drop peer", na); pna->peer->peer_ref = 0; netmap_adapter_put(na); } else { int i; ND("%p: case 2.b, grab peer", na); netmap_adapter_get(na); pna->peer->peer_ref = 1; /* hide our rings from netmap_mem_rings_delete */ for (i = 0; i < na->num_tx_rings + 1; i++) { na->tx_rings[i].ring = NULL; } for (i = 0; i < na->num_rx_rings + 1; i++) { na->rx_rings[i].ring = NULL; } } return 0; } /* netmap_pipe_krings_delete. * * There are two cases: * * 1) state is * * usr1 --> e1 --> e2 * * and we are e1 (e2 is not registered, so krings_delete cannot be * called on it); * * 2) state is * * usr1 --> e1 e2 <-- usr2 * * and we are either e1 or e2. * * In the former case we have to also delete the krings of e2; * in the latter case we do nothing (note that our krings * have already been hidden in the unregister callback). */ static void netmap_pipe_krings_delete(struct netmap_adapter *na) { struct netmap_pipe_adapter *pna = (struct netmap_pipe_adapter *)na; struct netmap_adapter *ona; /* na of the other end */ int i; if (!pna->peer_ref) { ND("%p: case 2, kept alive by peer", na); return; } /* case 1) above */ ND("%p: case 1, deleting everyhing", na); netmap_krings_delete(na); /* also zeroes tx_rings etc. */ /* restore the ring to be deleted on the peer */ ona = &pna->peer->up; if (ona->tx_rings == NULL) { /* already deleted, we must be on an * cleanup-after-error path */ return; } for (i = 0; i < ona->num_tx_rings + 1; i++) ona->tx_rings[i].ring = ona->tx_rings[i].save_ring; for (i = 0; i < ona->num_rx_rings + 1; i++) ona->rx_rings[i].ring = ona->rx_rings[i].save_ring; netmap_mem_rings_delete(ona); netmap_krings_delete(ona); } static void netmap_pipe_dtor(struct netmap_adapter *na) { struct netmap_pipe_adapter *pna = (struct netmap_pipe_adapter *)na; ND("%p", na); if (pna->peer_ref) { ND("%p: clean up peer", na); pna->peer_ref = 0; netmap_adapter_put(&pna->peer->up); } if (pna->role == NR_REG_PIPE_MASTER) netmap_pipe_remove(pna->parent, pna); netmap_adapter_put(pna->parent); pna->parent = NULL; } int netmap_get_pipe_na(struct nmreq *nmr, struct netmap_adapter **na, int create) { struct nmreq pnmr; struct netmap_adapter *pna; /* parent adapter */ struct netmap_pipe_adapter *mna, *sna, *req; u_int pipe_id; int role = nmr->nr_flags & NR_REG_MASK; int error; ND("flags %x", nmr->nr_flags); if (role != NR_REG_PIPE_MASTER && role != NR_REG_PIPE_SLAVE) { ND("not a pipe"); return 0; } role = nmr->nr_flags & NR_REG_MASK; /* first, try to find the parent adapter */ bzero(&pnmr, sizeof(pnmr)); memcpy(&pnmr.nr_name, nmr->nr_name, IFNAMSIZ); /* pass to parent the requested number of pipes */ pnmr.nr_arg1 = nmr->nr_arg1; error = netmap_get_na(&pnmr, &pna, create); if (error) { ND("parent lookup failed: %d", error); return error; } ND("found parent: %s", na->name); if (NETMAP_OWNED_BY_KERN(pna)) { ND("parent busy"); error = EBUSY; goto put_out; } /* next, lookup the pipe id in the parent list */ req = NULL; pipe_id = nmr->nr_ringid & NETMAP_RING_MASK; mna = netmap_pipe_find(pna, pipe_id); if (mna) { if (mna->role == role) { ND("found %d directly at %d", pipe_id, mna->parent_slot); req = mna; } else { ND("found %d indirectly at %d", pipe_id, mna->parent_slot); req = mna->peer; } /* the pipe we have found already holds a ref to the parent, * so we need to drop the one we got from netmap_get_na() */ netmap_adapter_put(pna); goto found; } ND("pipe %d not found, create %d", pipe_id, create); if (!create) { error = ENODEV; goto put_out; } /* we create both master and slave. * The endpoint we were asked for holds a reference to * the other one. */ mna = malloc(sizeof(*mna), M_DEVBUF, M_NOWAIT | M_ZERO); if (mna == NULL) { error = ENOMEM; goto put_out; } snprintf(mna->up.name, sizeof(mna->up.name), "%s{%d", pna->name, pipe_id); mna->id = pipe_id; mna->role = NR_REG_PIPE_MASTER; mna->parent = pna; mna->up.nm_txsync = netmap_pipe_txsync; mna->up.nm_rxsync = netmap_pipe_rxsync; mna->up.nm_register = netmap_pipe_reg; mna->up.nm_dtor = netmap_pipe_dtor; mna->up.nm_krings_create = netmap_pipe_krings_create; mna->up.nm_krings_delete = netmap_pipe_krings_delete; mna->up.nm_mem = pna->nm_mem; mna->up.na_lut = pna->na_lut; mna->up.na_lut_objtotal = pna->na_lut_objtotal; mna->up.na_lut_objsize = pna->na_lut_objsize; mna->up.num_tx_rings = 1; mna->up.num_rx_rings = 1; mna->up.num_tx_desc = nmr->nr_tx_slots; nm_bound_var(&mna->up.num_tx_desc, pna->num_tx_desc, 1, NM_PIPE_MAXSLOTS, NULL); mna->up.num_rx_desc = nmr->nr_rx_slots; nm_bound_var(&mna->up.num_rx_desc, pna->num_rx_desc, 1, NM_PIPE_MAXSLOTS, NULL); error = netmap_attach_common(&mna->up); if (error) goto free_mna; /* register the master with the parent */ error = netmap_pipe_add(pna, mna); if (error) goto free_mna; /* create the slave */ sna = malloc(sizeof(*mna), M_DEVBUF, M_NOWAIT | M_ZERO); if (sna == NULL) { error = ENOMEM; goto free_mna; } /* most fields are the same, copy from master and then fix */ *sna = *mna; snprintf(sna->up.name, sizeof(sna->up.name), "%s}%d", pna->name, pipe_id); sna->role = NR_REG_PIPE_SLAVE; error = netmap_attach_common(&sna->up); if (error) goto free_sna; /* join the two endpoints */ mna->peer = sna; sna->peer = mna; /* we already have a reference to the parent, but we * need another one for the other endpoint we created */ netmap_adapter_get(pna); if (role == NR_REG_PIPE_MASTER) { req = mna; mna->peer_ref = 1; netmap_adapter_get(&sna->up); } else { req = sna; sna->peer_ref = 1; netmap_adapter_get(&mna->up); } ND("created master %p and slave %p", mna, sna); found: ND("pipe %d %s at %p", pipe_id, (req->role == NR_REG_PIPE_MASTER ? "master" : "slave"), req); *na = &req->up; netmap_adapter_get(*na); /* write the configuration back */ nmr->nr_tx_rings = req->up.num_tx_rings; nmr->nr_rx_rings = req->up.num_rx_rings; nmr->nr_tx_slots = req->up.num_tx_desc; nmr->nr_rx_slots = req->up.num_rx_desc; /* keep the reference to the parent. * It will be released by the req destructor */ return 0; free_sna: free(sna, M_DEVBUF); free_mna: free(mna, M_DEVBUF); put_out: netmap_adapter_put(pna); return error; } #endif /* WITH_PIPES */