Index: sys/net/iflib.c =================================================================== --- sys/net/iflib.c +++ sys/net/iflib.c @@ -988,6 +988,8 @@ nm_i = kring->nr_hwcur; if (nm_i != head) { /* we have new packets to send */ + uint32_t pkt_len = 0, pkt_slot = 0; + int pkt_start = -1, flags = 0; pkt_info_zero(&pi); pi.ipi_segs = txq->ift_segs; pi.ipi_qsidx = kring->ring_id; @@ -1002,22 +1004,36 @@ u_int len = slot->len; uint64_t paddr; void *addr = PNMB(na, slot, &paddr); - int flags = (slot->flags & NS_REPORT || + flags |= (slot->flags & NS_REPORT || nic_i == 0 || nic_i == report_frequency) ? IPI_TX_INTR : 0; /* device-specific */ - pi.ipi_len = len; - pi.ipi_segs[0].ds_addr = paddr; - pi.ipi_segs[0].ds_len = len; - pi.ipi_nsegs = 1; - pi.ipi_ndescs = 0; - pi.ipi_pidx = nic_i; - pi.ipi_flags = flags; - - /* Fill the slot in the NIC ring. */ - ctx->isc_txd_encap(ctx->ifc_softc, &pi); - DBG_COUNTER_INC(tx_encap); + if (pkt_start < 0) + pkt_start = nic_i; + + pi.ipi_segs[pkt_slot].ds_addr = paddr; + pi.ipi_segs[pkt_slot].ds_len = len; + if (len) { + pkt_len += len; + pkt_slot++; + } + + if (!(slot->flags & NS_MOREFRAG)) { + pi.ipi_len = pkt_len; + pi.ipi_nsegs = pkt_slot; + pi.ipi_pidx = pkt_start; + pi.ipi_ndescs = 0; + pi.ipi_flags = flags; + + /* Fill the slot in the NIC ring. */ + ctx->isc_txd_encap(ctx->ifc_softc, &pi); + DBG_COUNTER_INC(tx_encap); + + /* Reinit params */ + flags = pkt_slot = pkt_len = 0; + pkt_start = -1; + } /* prefetch for next round */ __builtin_prefetch(&ring->slot[nm_i + 1]); @@ -1036,7 +1052,7 @@ txq->ift_sds.ifsd_map[nic_i], BUS_DMASYNC_PREWRITE); - slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); + slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED | NS_MOREFRAG); nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); } @@ -1099,6 +1115,7 @@ u_int n; u_int const lim = kring->nkr_num_slots - 1; int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; + int i = 0; if_ctx_t ctx = ifp->if_softc; if_shared_ctx_t sctx = ctx->ifc_sctx; @@ -1158,19 +1175,34 @@ ri.iri_qsidx = kring->ring_id; ri.iri_ifp = ctx->ifc_ifp; ri.iri_cidx = *cidxp; + i = 0; error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); - ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen; - ring->slot[nm_i].flags = 0; - if (have_rxcq) { - *cidxp = ri.iri_cidx; - while (*cidxp >= scctx->isc_nrxd[0]) - *cidxp -= scctx->isc_nrxd[0]; - } - bus_dmamap_sync(fl->ifl_buf_tag, - fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); - nm_i = nm_next(nm_i, lim); - fl->ifl_cidx = nic_i = nm_next(nic_i, lim); + do { + if (error) { + ring->slot[nm_i].len = 0; + ring->slot[nm_i].flags = 0; + } else { + ring->slot[nm_i].len = ri.iri_frags[i].irf_len; //ri.iri_len - crclen; + if (i == (ri.iri_nfrags - 1)) { + ring->slot[nm_i].len -= crclen; + ring->slot[nm_i].flags = 0; + } else + ring->slot[nm_i].flags |= NS_MOREFRAG; + } + + if (have_rxcq) { + *cidxp = ri.iri_cidx; + while (*cidxp >= scctx->isc_nrxd[0]) + *cidxp -= scctx->isc_nrxd[0]; + } + + bus_dmamap_sync(fl->ifl_buf_tag, + fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); + nm_i = nm_next(nm_i, lim); + fl->ifl_cidx = nic_i = nm_next(nic_i, lim); + } while (++i < ri.iri_nfrags); + } if (n) { /* update the state variables */ if (netmap_no_pendintr && !force_update) { @@ -1217,7 +1249,7 @@ bzero(&na, sizeof(na)); na.ifp = ctx->ifc_ifp; - na.na_flags = NAF_BDG_MAYSLEEP; + na.na_flags = NAF_BDG_MAYSLEEP | NAF_MOREFRAG; MPASS(ctx->ifc_softc_ctx.isc_ntxqsets); MPASS(ctx->ifc_softc_ctx.isc_nrxqsets); @@ -2617,7 +2649,8 @@ } } else { fl->ifl_sds.ifsd_m[cidx] = NULL; - *pf_rv = PFIL_PASS; + if (pf_rv != NULL) + *pf_rv = PFIL_PASS; } if (unload && irf->irf_len != 0) Index: tools/tools/netmap/bridge.c =================================================================== --- tools/tools/netmap/bridge.c +++ tools/tools/netmap/bridge.c @@ -36,24 +36,18 @@ * set of queues ? */ static int -rx_slots_avail(struct nmport_d *d) +slots_avail(struct nmport_d *d, int tx) { u_int i, tot = 0; - for (i = d->first_rx_ring; i <= d->last_rx_ring; i++) { - tot += nm_ring_space(NETMAP_RXRING(d->nifp, i)); - } - - return tot; -} - -static int -tx_slots_avail(struct nmport_d *d) -{ - u_int i, tot = 0; - - for (i = d->first_tx_ring; i <= d->last_tx_ring; i++) { - tot += nm_ring_space(NETMAP_TXRING(d->nifp, i)); + if (tx) { + for (i = d->first_tx_ring; i <= d->last_tx_ring; i++) { + tot += nm_ring_space(NETMAP_TXRING(d->nifp, i)); + } + } else { + for (i = d->first_rx_ring; i <= d->last_rx_ring; i++) { + tot += nm_ring_space(NETMAP_RXRING(d->nifp, i)); + } } return tot; @@ -68,6 +62,7 @@ u_int limit, const char *msg) { u_int j, k, m = 0; + u_int pkt_len = 0; /* print a warning if any of the ring flags is set (e.g. NM_REINIT) */ if (rxring->flags || txring->flags) @@ -103,21 +98,32 @@ } ts->len = rs->len; if (zerocopy) { - uint32_t pkt = ts->buf_idx; + uint32_t flags, pkt; + pkt = ts->buf_idx; ts->buf_idx = rs->buf_idx; rs->buf_idx = pkt; /* report the buffer change. */ ts->flags |= NS_BUF_CHANGED; rs->flags |= NS_BUF_CHANGED; /* copy the NS_MOREFRAG */ + flags = rs->flags; rs->flags = (rs->flags & ~NS_MOREFRAG) | (ts->flags & NS_MOREFRAG); + ts->flags = (ts->flags & ~NS_MOREFRAG) | (flags & NS_MOREFRAG); + j = nm_ring_next(rxring, j); + k = nm_ring_next(txring, k); } else { char *rxbuf = NETMAP_BUF(rxring, rs->buf_idx); char *txbuf = NETMAP_BUF(txring, ts->buf_idx); - nm_pkt_copy(rxbuf, txbuf, ts->len); + nm_pkt_copy(rxbuf, txbuf + pkt_len, ts->len); + pkt_len += ts->len; + + j = nm_ring_next(rxring, j); + if (!(rs->flags & NS_MOREFRAG)) { + ts->len = pkt_len; + k = nm_ring_next(txring, k); + pkt_len = 0; + } } - j = nm_ring_next(rxring, j); - k = nm_ring_next(txring, k); } rxring->head = rxring->cur = j; txring->head = txring->cur = k; @@ -317,8 +323,8 @@ int n0, n1, ret; pollfd[0].events = pollfd[1].events = 0; pollfd[0].revents = pollfd[1].revents = 0; - n0 = rx_slots_avail(pa); - n1 = rx_slots_avail(pb); + n0 = slots_avail(pa, 0); + n1 = slots_avail(pb, 0); #if defined(_WIN32) || defined(BUSYWAIT) if (n0) { ioctl(pollfd[1].fd, NIOCTXSYNC, NULL); @@ -352,14 +358,14 @@ ret <= 0 ? "timeout" : "ok", pollfd[0].events, pollfd[0].revents, - rx_slots_avail(pa), + slots_avail(pa, 0), NETMAP_RXRING(pa->nifp, pa->cur_rx_ring)->head, - tx_slots_avail(pa), + slots_avail(pa, 1), pollfd[1].events, pollfd[1].revents, - rx_slots_avail(pb), + slots_avail(pb, 0), NETMAP_RXRING(pb->nifp, pb->cur_rx_ring)->head, - tx_slots_avail(pb) + slots_avail(pb, 1) ); if (ret < 0) continue;