Page Menu
Home
FreeBSD
Search
Configure Global Search
Log In
Files
F177637519
D4295.1794913098.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Flag For Later
Award Token
Authored By
Unknown
Size
26 KB
Referenced Files
None
Subscribers
None
D4295.1794913098.diff
View Options
Index: share/doc/notes/tcp_backpressure_design.txt
===================================================================
--- /dev/null
+++ share/doc/notes/tcp_backpressure_design.txt
@@ -0,0 +1,88 @@
+inpcb portion of implementation:
+
+private:
+ - initialize an inpcb tailq per-cpu
+ - on non-iflib systems initialize a taskqueue thread per-cpu
+
+inp_rexmt_fn() task function:
+ - call inp_rexmt function for each inpcb in tailq
+ - inp_rexmt or the function it is a wrapper for should just
+ call inp_rexmt_enqueue when ENOBUFS is returned so that inpcbs
+ will end up being serviced round robin for all participating
+ interfaces
+ - Would be faster to skip further inp_rexmt (tcp_output) calls for a queue
+ that had already returned ENOBUFS. However, that would require callers
+ that check the return code of tcp_output to handle an ENOBUFS return.
+ Consider this a future optimization pending indication that much
+ time is spent in the txq overrun state.
+
+public:
+inp_rexmt_enqueue():
+ - initialize inp_rexmt to passed function
+ - enqueue inpcb to the tailq for the cpu corresponding to the flowid
+
+inp_rexmt_start(uint32_t qid, uint32_t nqs):
+ - wakeup taskq threads corresponding to qid
+
+
+Driver portion of implementation:
+
+Backpressure adds a new state machine to driver packet processing:
+
+
+
+ !full ifq ifq >= half full
+ _________ _______
+ | | | |
+ __\ /___ | full ifq ____\ /__ |
+ | OPEN | --|------------->| CLOSED|--|
+ -------- --------- |
+ / \ |
+ | ifq < half full |
+ |----------------------------------
+
+
+
+The term ifq refers to any software queue including struct ifq, buf_ring and
+np_ring. When a queue becomes full the driver will return ENOBUFS on any
+further requests until the software ring transtions to less than half full.
+When the driver enqueues the packet that transitions the software queue back
+to the open state it calls inp_rxmt_start with a cpuset indicating the cpus that
+map to the software ring whose state has just transitioned. For example, a
+hypothetical driver has 4 hw tx queues paired with 4 sw queues feeding them on
+a system with 8 logical cores. If the software queue that is used by cores 4
+and 5 transitions from CLOSED to open, the driver will call inp_rxmt_start
+with a cpuset corresponding to 4 and 5.
+
+
+TCP portion of implementation:
+
+Legacy TCP behavior of tcp_output:
+
+ switch (error) {
+ <..>
+ case ENOBUFS:
+ if (!tcp_timer_active(tp, TT_REXMT) &&
+ !tcp_timer_active(tp, TT_PERSIST))
+ tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur);
+ tp->snd_cwnd = tp->t_maxseg;
+ return (0);
+ <...>
+
+With backpressure supported the inpcb is simply put at the end of a tailq
+corresponding to the current cpu:
+ switch (error) {
+ <..>
+ case ENOBUFS:
+ inp_rexmt_enqueue(tp->t_inp, tcp_output_rexmt);
+ return (0);
+ <...>
+
+where tcp_output_rexmt is a wrapper for tcp_output():
+
+void
+tcp_output_rexmt(struct inpcb *inp)
+{
+
+ (void)tcp_output(tp->inp_ppcb);
+}
Index: sys/dev/bxe/bxe.c
===================================================================
--- sys/dev/bxe/bxe.c
+++ sys/dev/bxe/bxe.c
@@ -6810,7 +6810,7 @@
#if __FreeBSD_version >= 800000
fp->tx_br = buf_ring_alloc(BXE_BR_SIZE, M_DEVBUF,
- M_NOWAIT, &fp->tx_mtx);
+ M_NOWAIT, &fp->tx_mtx, i, sc->num_queues);
if (fp->tx_br == NULL) {
BLOGE(sc, "buf_ring alloc fail for fp[%02d]\n", i);
goto bxe_alloc_fp_buffers_error;
Index: sys/dev/cxgb/cxgb_sge.c
===================================================================
--- sys/dev/cxgb/cxgb_sge.c
+++ sys/dev/cxgb/cxgb_sge.c
@@ -2421,13 +2421,18 @@
{
struct sge_qset *q = &sc->sge.qs[id];
int i, ret = 0;
+ int nqsets;
MTX_INIT(&q->lock, q->namebuf, NULL, MTX_DEF);
q->port = pi;
q->adap = sc;
+ for (nqsets = i = 0; i < sc->params.nports; i++) {
+ struct port_info *ptmp = &sc->port[i];
+ nqsets += ptmp->nqsets;
+ }
if ((q->txq[TXQ_ETH].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size,
- M_DEVBUF, M_WAITOK, &q->lock)) == NULL) {
+ M_DEVBUF, M_WAITOK, &q->lock, id, nqsets)) == NULL) {
device_printf(sc->dev, "failed to allocate mbuf ring\n");
goto err;
}
Index: sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.c
===================================================================
--- sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.c
+++ sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.c
@@ -94,7 +94,7 @@
u32 rarray[16];
mtx_init(fifo_lock, "cxio fifo", NULL, MTX_DEF|MTX_DUPOK);
- *fifo = buf_ring_alloc(nr, M_DEVBUF, M_NOWAIT, fifo_lock);
+ *fifo = buf_ring_alloc(nr, M_DEVBUF, M_NOWAIT, fifo_lock, 0, 0);
if (*fifo == NULL)
return (-ENOMEM);
#if 0
@@ -154,7 +154,7 @@
mtx_init(&rdev_p->rscp->qpid_fifo_lock, "qpid fifo", NULL, MTX_DEF);
rdev_p->rscp->qpid_fifo = buf_ring_alloc(T3_MAX_NUM_QP, M_DEVBUF,
- M_NOWAIT, &rdev_p->rscp->qpid_fifo_lock);
+ M_NOWAIT, &rdev_p->rscp->qpid_fifo_lock, 0, 0);
if (rdev_p->rscp->qpid_fifo == NULL)
return (-ENOMEM);
Index: sys/dev/e1000/if_em.c
===================================================================
--- sys/dev/e1000/if_em.c
+++ sys/dev/e1000/if_em.c
@@ -3288,7 +3288,7 @@
#if __FreeBSD_version >= 800000
/* Allocate a buf ring */
txr->br = buf_ring_alloc(4096, M_DEVBUF,
- M_WAITOK, &txr->tx_mtx);
+ M_WAITOK, &txr->tx_mtx, i, adapter->num_queues);
#endif
}
Index: sys/dev/e1000/if_igb.c
===================================================================
--- sys/dev/e1000/if_igb.c
+++ sys/dev/e1000/if_igb.c
@@ -3375,7 +3375,8 @@
#ifndef IGB_LEGACY_TX
/* Allocate a buf ring */
txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF,
- M_WAITOK, &txr->tx_mtx);
+ M_WAITOK, &txr->tx_mtx, i,
+ adapter->num_queues);
#endif
}
Index: sys/dev/ifmlx4/en_tx.c
===================================================================
--- sys/dev/ifmlx4/en_tx.c
+++ sys/dev/ifmlx4/en_tx.c
@@ -89,7 +89,8 @@
/* Allocate the buf ring */
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
- M_WAITOK, &ring->tx_lock.m);
+ M_WAITOK, &ring->tx_lock.m, queue_idx,
+ priv->tx_ring_num);
if (ring->br == NULL) {
en_err(priv, "Failed allocating tx_info ring\n");
return -ENOMEM;
Index: sys/dev/ixgbe/ix_txrx.c
===================================================================
--- sys/dev/ixgbe/ix_txrx.c
+++ sys/dev/ixgbe/ix_txrx.c
@@ -2222,8 +2222,9 @@
}
#ifndef IXGBE_LEGACY_TX
/* Allocate a buf ring */
- txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
- M_WAITOK, &txr->tx_mtx);
+ txr->br = buf_ring_allocIXGBE_BR_SIZE, M_DEVBUF,
+ M_WAITOK, &txr->tx_mtx, i,
+ adapter->num_queues);
if (txr->br == NULL) {
device_printf(dev,
"Critical Failure setting up buf ring\n");
Index: sys/dev/mxge/if_mxge.c
===================================================================
--- sys/dev/mxge/if_mxge.c
+++ sys/dev/mxge/if_mxge.c
@@ -4435,7 +4435,7 @@
mtx_init(&ss->tx.mtx, ss->tx.mtx_name, NULL, MTX_DEF);
#ifdef IFNET_BUF_RING
ss->tx.br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK,
- &ss->tx.mtx);
+ &ss->tx.mtx, i, sc->num_slices);
#endif
}
Index: sys/dev/oce/oce_queue.c
===================================================================
--- sys/dev/oce/oce_queue.c
+++ sys/dev/oce/oce_queue.c
@@ -45,7 +45,7 @@
*****************************************************/
static struct oce_wq *oce_wq_init(POCE_SOFTC sc,
- uint32_t q_len, uint32_t wq_type);
+ uint32_t q_len, uint32_t wq_type, int idx);
static int oce_wq_create(struct oce_wq *wq, struct oce_eq *eq);
static void oce_wq_free(struct oce_wq *wq);
static void oce_wq_del(struct oce_wq *wq);
@@ -94,7 +94,7 @@
/* alloc TX/RX queues */
for_all_wq_queues(sc, wq, i) {
sc->wq[i] = oce_wq_init(sc, sc->tx_ring_size,
- NIC_WQ_TYPE_STANDARD);
+ NIC_WQ_TYPE_STANDARD, i);
if (!sc->wq[i])
goto error;
@@ -202,7 +202,7 @@
* @returns the pointer to the WQ created or NULL on failure
*/
static struct
-oce_wq *oce_wq_init(POCE_SOFTC sc, uint32_t q_len, uint32_t wq_type)
+oce_wq *oce_wq_init(POCE_SOFTC sc, uint32_t q_len, uint32_t wq_type, int idx)
{
struct oce_wq *wq;
int rc = 0, i;
@@ -258,7 +258,7 @@
#if __FreeBSD_version >= 800000
/* Allocate buf ring for multiqueue*/
wq->br = buf_ring_alloc(4096, M_DEVBUF,
- M_WAITOK, &wq->tx_lock.mutex);
+ M_WAITOK, &wq->tx_lock.mutex, idx, sc->nwqs);
if (!wq->br)
goto free_wq;
#endif
Index: sys/dev/virtio/network/if_vtnet.c
===================================================================
--- sys/dev/virtio/network/if_vtnet.c
+++ sys/dev/virtio/network/if_vtnet.c
@@ -702,8 +702,10 @@
vtnet_init_txq(struct vtnet_softc *sc, int id)
{
struct vtnet_txq *txq;
+ int npairs;
txq = &sc->vtnet_txqs[id];
+ npairs = sc->vtnet_max_vq_pairs;
snprintf(txq->vtntx_name, sizeof(txq->vtntx_name), "%s-tx%d",
device_get_nameunit(sc->vtnet_dev), id);
@@ -718,7 +720,7 @@
#ifndef VTNET_LEGACY_TX
txq->vtntx_br = buf_ring_alloc(VTNET_DEFAULT_BUFRING_SIZE, M_DEVBUF,
- M_NOWAIT, &txq->vtntx_mtx);
+ M_NOWAIT, &txq->vtntx_mtx, id, npairs);
if (txq->vtntx_br == NULL)
return (ENOMEM);
Index: sys/dev/vmware/vmxnet3/if_vmx.c
===================================================================
--- sys/dev/vmware/vmxnet3/if_vmx.c
+++ sys/dev/vmware/vmxnet3/if_vmx.c
@@ -990,7 +990,7 @@
TASK_INIT(&txq->vxtxq_defrtask, 0, vmxnet3_txq_tq_deferred, txq);
txq->vxtxq_br = buf_ring_alloc(VMXNET3_DEF_BUFRING_SIZE, M_DEVBUF,
- M_NOWAIT, &txq->vxtxq_mtx);
+ M_NOWAIT, &txq->vxtxq_mtx, q, sc->vmx_max_ntxqueues);
if (txq->vxtxq_br == NULL)
return (ENOMEM);
#endif
Index: sys/dev/vxge/vxge.c
===================================================================
--- sys/dev/vxge/vxge.c
+++ sys/dev/vxge/vxge.c
@@ -2295,7 +2295,8 @@
}
#if __FreeBSD_version >= 800000
vpath->br = buf_ring_alloc(VXGE_DEFAULT_BR_SIZE, M_DEVBUF,
- M_WAITOK, &vpath->mtx_tx);
+ M_WAITOK, &vpath->mtx_tx, i,
+ vdev->no_of_vpath);
if (vpath->br == NULL) {
err = ENOMEM;
break;
Index: sys/kern/subr_bufring.c
===================================================================
--- sys/kern/subr_bufring.c
+++ sys/kern/subr_bufring.c
@@ -50,7 +50,7 @@
static struct buf_ring *
-buf_ring_alloc_(int count, struct malloc_type *type, int flags, struct mtx *lock, int brflags)
+buf_ring_alloc_(int count, struct malloc_type *type, int flags, struct mtx *lock, int brflags, int id, int nqs)
{
struct buf_ring *br;
int alloc_count;
@@ -70,22 +70,24 @@
br->br_prod_mask = br->br_cons_mask = count-1;
br->br_prod_head = br->br_cons_head = 0;
br->br_prod_tail = br->br_cons_tail = 0;
+ br->br_id = id;
+ br->br_nqs = nqs;
return (br);
}
struct buf_ring *
-buf_ring_alloc(int count, struct malloc_type *type, int flags, struct mtx *lock)
+buf_ring_alloc(int count, struct malloc_type *type, int flags, struct mtx *lock, int id, int nqs)
{
- return (buf_ring_alloc_(count, type, flags, lock, 0));
+ return (buf_ring_alloc_(count, type, flags, lock, 0, id, nqs));
}
struct buf_ring *
buf_ring_aligned_alloc(int count, struct malloc_type *type, int flags, struct mtx *lock)
{
- return (buf_ring_alloc_(count, type, flags, lock, BR_FLAGS_ALIGNED));
+ return (buf_ring_alloc_(count, type, flags, lock, BR_FLAGS_ALIGNED, 0, 0));
}
void
Index: sys/net/altq/if_altq.h
===================================================================
--- sys/net/altq/if_altq.h
+++ sys/net/altq/if_altq.h
@@ -44,6 +44,7 @@
struct mbuf *ifq_tail;
int ifq_len;
int ifq_maxlen;
+ int ifq_closed;
struct mtx ifq_mtx;
/* driver owned queue (used for bulk dequeue and prepend) UNLOCKED */
Index: sys/net/if.c
===================================================================
--- sys/net/if.c
+++ sys/net/if.c
@@ -81,6 +81,7 @@
#include <net/ethernet.h>
#include <netinet/in.h>
#include <netinet/in_var.h>
+#include <netinet/in_pcb.h>
#include <netinet/ip.h>
#include <netinet/ip_carp.h>
#ifdef INET
@@ -3424,6 +3425,16 @@
(*(ifp)->if_start)(ifp);
}
+void
+if_rexmt_start(int qid, int nqs)
+{
+
+#if defined(INET6) || defined(INET)
+ inp_rexmt_start(qid, nqs);
+#endif
+}
+
+
/*
* Backwards compatibility interface for drivers
* that have not implemented it
Index: sys/net/iflib.h
===================================================================
--- sys/net/iflib.h
+++ sys/net/iflib.h
@@ -208,7 +208,7 @@
if_softc_ctx_t iflib_get_softc_ctx(if_ctx_t ctx);
if_shared_ctx_t iflib_get_sctx(if_ctx_t ctx);
-void iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN]);
+void iflib_set_mac(if_ctx_t ctx, uint8_t mac[6/*ETHER_ADDR_LEN*/]);
@@ -245,6 +245,7 @@
void iflib_irq_free(if_ctx_t ctx, if_irq_t irq);
+void iflib_io_tqg_attach(struct grouptask *gtask, void *uniq, int cpu, char *name);
void iflib_tx_intr_deferred(if_ctx_t ctx, int txqid);
void iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid);
Index: sys/net/iflib.c
===================================================================
--- sys/net/iflib.c
+++ sys/net/iflib.c
@@ -52,6 +52,7 @@
#include <net/mp_ring.h>
#include <netinet/in.h>
+#include <netinet/in_pcb.h>
#include <netinet/tcp_lro.h>
#include <machine/bus.h>
@@ -288,12 +289,13 @@
iflib_sd_t ift_sds;
int ift_nbr;
struct mp_ring **ift_br;
- struct grouptask ift_task;
- int ift_qstatus;
+ struct grouptask ift_task;
+ int ift_qstatus;
int ift_active;
- int ift_watchdog_time;
+ int ift_watchdog_time;
struct iflib_filter_info ift_filter_info;
- iflib_dma_info_t ift_ifdi;
+ iflib_dma_info_t ift_ifdi;
+ int ift_closed;
};
struct iflib_fl {
@@ -2290,6 +2292,10 @@
int i, count, pkt_sent, bytes_sent, mcast_sent, avail;
avail = IDXDIFF(pidx, cidx, r->size);
+ if (avail < (r->size >> 1)) {
+ txq->ift_closed = FALSE;
+ inp_rexmt_start(txq->ift_id, ctx->ifc_softc_ctx.isc_nqsets);
+ }
if (ctx->ifc_flags & IFC_QFLUSH) {
DBG_COUNTER_INC(txq_drain_flushing);
for (i = 0; i < avail; i++) {
@@ -2505,6 +2511,26 @@
m_freem(m);
return (0);
}
+
+ qidx = 0;
+ if ((NQSETS(ctx) > 1) && M_HASHTYPE_GET(m))
+ qidx = QIDX(ctx, m);
+ /*
+ * XXX calculate buf_ring based on flowid (divvy up bits?)
+ */
+ txq = &ctx->ifc_txqs[qidx];
+
+
+ if (txq->ift_closed) {
+ while (m != NULL) {
+ next = m->m_nextpkt;
+ m->m_nextpkt = NULL;
+ m_freem(m);
+ m = next;
+ }
+ return (ENOBUFS);
+ }
+
qidx = count = 0;
mp = marr;
next = m;
@@ -2526,17 +2552,11 @@
next = next->m_nextpkt;
mp[i]->m_nextpkt = NULL;
}
- if ((NQSETS(ctx) > 1) && M_HASHTYPE_GET(m))
- qidx = QIDX(ctx, m);
- /*
- * XXX calculate buf_ring based on flowid (divvy up bits?)
- */
- txq = &ctx->ifc_txqs[qidx];
-
DBG_COUNTER_INC(tx_seen);
err = mp_ring_enqueue(txq->ift_br[0], (void **)mp, count, IFLIB_BUDGET);
/* drain => err = iflib_txq_transmit(ifp, txq, m); */
if (err) {
+ txq->ift_closed = TRUE;
for (i = 0; i < count; i++)
m_freem(mp[i]);
mp_ring_check_drainage(txq->ift_br[0], BATCH_SIZE);
@@ -3717,6 +3737,14 @@
}
void
+iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name)
+{
+
+ taskqgroup_attach_cpu(gctx->igc_io_tqg, gt, uniq, cpu, -1, name);
+}
+
+
+void
iflib_link_state_change(if_ctx_t ctx, int link_state)
{
if_t ifp = ctx->ifc_ifp;
Index: sys/net/ifq.h
===================================================================
--- sys/net/ifq.h
+++ sys/net/ifq.h
@@ -57,6 +57,7 @@
struct mbuf *ifq_tail;
int ifq_len;
int ifq_maxlen;
+ int ifq_closed;
struct mtx ifq_mtx;
};
@@ -71,6 +72,7 @@
#define IF_UNLOCK(ifq) mtx_unlock(&(ifq)->ifq_mtx)
#define IF_LOCK_ASSERT(ifq) mtx_assert(&(ifq)->ifq_mtx, MA_OWNED)
#define _IF_QFULL(ifq) ((ifq)->ifq_len >= (ifq)->ifq_maxlen)
+#define _IF_QCLOSED(ifq) ((ifq)->ifq_closed == TRUE)
#define _IF_QLEN(ifq) ((ifq)->ifq_len)
#define _IF_ENQUEUE(ifq, m) do { \
@@ -162,12 +164,14 @@
#define IFQ_ENQUEUE(ifq, m, err) \
do { \
IF_LOCK(ifq); \
- if (ALTQ_IS_ENABLED(ifq)) \
+ if (ALTQ_IS_ENABLED(ifq)) { \
ALTQ_ENQUEUE(ifq, m, NULL, err); \
- else { \
- if (_IF_QFULL(ifq)) { \
+ (ifq)->ifq_closed = TRUE; \
+ } else { \
+ if (_IF_QCLOSED(ifq) || _IF_QFULL(ifq)) { \
m_freem(m); \
(err) = ENOBUFS; \
+ (ifq)->ifq_closed = TRUE; \
} else { \
_IF_ENQUEUE(ifq, m); \
(err) = 0; \
@@ -182,8 +186,14 @@
(m) = tbr_dequeue_ptr(ifq, ALTDQ_REMOVE); \
else if (ALTQ_IS_ENABLED(ifq)) \
ALTQ_DEQUEUE(ifq, m); \
- else \
+ else { \
_IF_DEQUEUE(ifq, m); \
+ if (_IF_QCLOSED(ifq) && \
+ (ifq)->ifq_len < ((ifq)->ifq_maxlen >> 1)) { \
+ (ifq)->ifq_closed = FALSE; \
+ if_rexmt_start(0, 1); \
+ } \
+ } \
} while (0)
#define IFQ_DEQUEUE(ifq, m) \
@@ -216,6 +226,7 @@
ALTQ_PURGE(ifq); \
} else \
_IF_DRAIN(ifq); \
+ (ifq)->ifq_closed = FALSE; \
} while (0)
#define IFQ_PURGE(ifq) \
@@ -329,7 +340,6 @@
error = buf_ring_enqueue(br, m);
if (error)
m_freem(m);
-
return (error);
}
Index: sys/netinet/in_pcb.h
===================================================================
--- sys/netinet/in_pcb.h
+++ sys/netinet/in_pcb.h
@@ -161,6 +161,7 @@
* (p) - Protected by the pcbinfo lock for the inpcb
* (l) - Protected by the pcblist lock for the inpcb
* (h) - Protected by the pcbhash lock for the inpcb
+ * (r) - Protected by the rexmt lock
* (s) - Protected by another subsystem's locks
* (x) - Undefined locking
*
@@ -240,6 +241,8 @@
struct llentry *inp_lle; /* cached L2 information */
struct rtentry *inp_rt; /* cached L3 information */
struct rwlock inp_lock;
+ STAILQ_ENTRY(inpcb) inp_rexmt_entry; /* (i/) */
+ void (*inp_rexmt) (struct inpcb *);
};
#define inp_fport inp_inc.inc_fport
#define inp_lport inp_inc.inc_lport
@@ -609,6 +612,7 @@
#define INP_RSS_BUCKET_SET 0x00000080 /* IP_RSS_LISTEN_BUCKET is set */
#define INP_RECVFLOWID 0x00000100 /* populate recv datagram with flow info */
#define INP_RECVRSSBUCKETID 0x00000200 /* populate recv datagram with bucket id */
+#define INP_IN_REXMTQ 0x00000400 /* inpcb is referenced by the pcpu rexmit q */
/*
* Flags passed to in_pcblookup*() functions.
@@ -728,6 +732,9 @@
struct sockaddr *
in_sockaddr(in_port_t port, struct in_addr *addr);
void in_pcbsosetlabel(struct socket *so);
+void inp_rexmt_enqueue(struct inpcb *inp, void (*inp_rexmt) (struct inpcb *));
+void inp_rexmt_start(uint32_t qid, uint32_t nqs);
+void inp_rexmt_stop(struct inpcb *inp);
#endif /* _KERNEL */
#endif /* !_NETINET_IN_PCB_H_ */
Index: sys/netinet/in_pcb.c
===================================================================
--- sys/netinet/in_pcb.c
+++ sys/netinet/in_pcb.c
@@ -2387,6 +2387,159 @@
*fp = inp->inp_fport;
}
+
+/* backpressure start */
+#include <sys/smp.h>
+#include <sys/taskqueue.h>
+
+#include <net/iflib.h>
+
+STAILQ_HEAD(inp_rexmt_head, inpcb) *inp_rexmt_list, *inp_rexmt_worklist;
+static struct mtx *inp_rexmt_lock;
+static struct grouptask *inp_rexmt_gt;
+static counter_u64_t inp_rexmt_count;
+SYSCTL_COUNTER_U64(_net_inet_ip, OID_AUTO, rexmt, CTLFLAG_RD, &inp_rexmt_count,
+ "Number of times inpcb was enqueued for rexmit");
+#define INP_REXMT_LOCK(i) mtx_lock(&inp_rexmt_lock[i])
+#define INP_REXMT_UNLOCK(i) mtx_unlock(&inp_rexmt_lock[i])
+
+void inp_rexmt_fn(void *context __unused, int pending __unused);
+
+static void
+inp_rexmt_init(const void *arg __unused)
+{
+ int i;
+ struct grouptask *gt;
+
+ inp_rexmt_list = malloc(sizeof(struct inp_rexmt_head)*mp_ncpus, M_PCB, M_WAITOK|M_ZERO);
+ inp_rexmt_worklist = malloc(sizeof(struct inp_rexmt_head)*mp_ncpus, M_PCB, M_WAITOK|M_ZERO);
+ inp_rexmt_lock = malloc(sizeof(struct mtx)*mp_ncpus, M_PCB, M_WAITOK|M_ZERO);
+ inp_rexmt_gt = malloc(sizeof(struct grouptask)*mp_ncpus, M_PCB, M_WAITOK|M_ZERO);
+ inp_rexmt_count = counter_u64_alloc(M_WAITOK|M_ZERO);
+ for (i = 0; i < mp_ncpus; i++) {
+ STAILQ_INIT(&inp_rexmt_list[i]);
+ STAILQ_INIT(&inp_rexmt_worklist[i]);
+ mtx_init(&inp_rexmt_lock[i], "rexmt", NULL, MTX_DEF);
+ gt = &inp_rexmt_gt[i];
+ GROUPTASK_INIT(gt, 0, inp_rexmt_fn, NULL);
+ iflib_io_tqg_attach(gt, gt, i, "rexmt");
+ }
+}
+/* must be post-SI_SUB_SMP */
+SYSINIT(inp_rexmt_init, SI_SUB_LAST, SI_ORDER_ANY,
+ inp_rexmt_init, NULL);
+
+void
+inp_rexmt_enqueue(struct inpcb *inp, void (*inp_rexmt) (struct inpcb *))
+{
+ int cpuid;
+
+ INP_WLOCK_ASSERT(inp);
+ if (inp->inp_flags2 & INP_IN_REXMTQ)
+ return;
+
+ counter_u64_add(inp_rexmt_count, 1);
+ inp->inp_flags2 |= INP_IN_REXMTQ;
+ inp->inp_rexmt = inp_rexmt;
+ in_pcbref(inp);
+
+ /*
+ * having a consistent reversible mapping is more important than
+ * optimal cpu locality - so wo don't go the route of:
+ * cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype);
+ */
+ cpuid = inp->inp_flowid % mp_ncpus;
+
+ INP_REXMT_LOCK(cpuid);
+ /*
+ * If performance is an issue on systems with multiple interfaces
+ * it would make sense to hash the inp based on it's ifp so that
+ * rexmt_start only wakes up inpcbs on the callers interface
+ */
+ STAILQ_INSERT_TAIL(&inp_rexmt_list[cpuid], inp, inp_rexmt_entry);
+ INP_REXMT_UNLOCK(cpuid);
+}
+
+void
+inp_rexmt_fn(void *context __unused, int pending __unused)
+{
+ struct inpcb *inp;
+ int cpuid;
+ struct inp_rexmt_head *worklist, *list;
+ void (*inp_rexmt) (struct inpcb *);
+
+ cpuid = curcpu;
+ worklist = &inp_rexmt_worklist[cpuid];
+ list = &inp_rexmt_list[cpuid];
+ if (STAILQ_EMPTY(worklist)) {
+ INP_REXMT_LOCK(cpuid);
+ STAILQ_SWAP(worklist, list, inpcb);
+ INP_REXMT_UNLOCK(cpuid);
+ }
+
+ inp = STAILQ_FIRST(worklist);
+ STAILQ_REMOVE_HEAD(worklist, inp_rexmt_entry);
+ INP_WLOCK(inp);
+ inp_rexmt = inp->inp_rexmt;
+ inp->inp_flags2 &= ~INP_IN_REXMTQ;
+ inp->inp_rexmt = NULL;
+ if (inp_rexmt != NULL)
+ inp_rexmt(inp);
+ if (!in_pcbrele_wlocked(inp))
+ INP_WUNLOCK(inp);
+ /* reschedule task to continue work */
+ if (!STAILQ_EMPTY(worklist))
+ GROUPTASK_ENQUEUE(&inp_rexmt_gt[cpuid]);
+}
+
+void
+inp_rexmt_stop(struct inpcb *inp)
+{
+
+ INP_WLOCK_ASSERT(inp);
+
+ /* we rely on tq thread to call rele on the inpcb */
+ if ((inp->inp_flags2 & INP_IN_REXMTQ) == 0)
+ return;
+
+ inp->inp_rexmt = NULL;
+}
+
+void
+inp_rexmt_start(uint32_t qid, uint32_t nqs)
+{
+ int i, start, count;
+
+ MPASS(nqs > 0);
+
+ if (nqs == 1) {
+ count = mp_ncpus;
+ start = 0;
+ } else if (nqs == mp_ncpus) {
+ count = 1;
+ start = qid;
+ } else if (nqs < mp_ncpus && (mp_ncpus % nqs) == 0) {
+ count = mp_ncpus/nqs;
+ start = count*qid;
+ } else if (nqs > mp_ncpus && (nqs % mp_ncpus) == 0) {
+ count = 1;
+ start = qid/mp_ncpus;
+ } else {
+ /* XXX chance for future optimization
+ * there isn't a trivial way to map queues to
+ * cpus in this case
+ */
+ count = mp_ncpus;
+ start = 0;
+ }
+
+ for (i = start; i < start + count; i++)
+ GROUPTASK_ENQUEUE(&inp_rexmt_gt[i]);
+}
+
+/* backpressure end */
+
+
struct inpcb *
so_sotoinpcb(struct socket *so)
{
Index: sys/netinet/tcp_output.c
===================================================================
--- sys/netinet/tcp_output.c
+++ sys/netinet/tcp_output.c
@@ -126,6 +126,13 @@
&VNET_NAME(tcp_autosndbuf_max), 0,
"Max size of automatic send buffer");
+VNET_DEFINE(int, tcp_output_enobufs) = 0;
+#define V_tcp_output_enobufs VNET(tcp_output_enobufs)
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_output_enobufs, CTLFLAG_VNET | CTLFLAG_RW,
+ &VNET_NAME(tcp_output_enobufs), 0,
+ "number of times ENOBUFS returned");
+
+
static void inline hhook_run_tcp_est_out(struct tcpcb *tp,
struct tcphdr *th, struct tcpopt *to,
long len, int tso);
@@ -164,6 +171,14 @@
CC_ALGO(tp)->after_idle(tp->ccv);
}
+
+static void
+tcp_rexmt_output(struct inpcb *inp)
+{
+
+ (void) tcp_output(inp->inp_ppcb);
+}
+
/*
* Tcp output routine: figure out what should be sent and send it.
*/
@@ -1498,10 +1513,8 @@
tp->t_softerror = error;
return (error);
case ENOBUFS:
- if (!tcp_timer_active(tp, TT_REXMT) &&
- !tcp_timer_active(tp, TT_PERSIST))
- tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur);
- tp->snd_cwnd = tp->t_maxseg;
+ atomic_add_int(&V_tcp_output_enobufs, 1);
+ inp_rexmt_enqueue(tp->t_inpcb, tcp_rexmt_output);
return (0);
case EMSGSIZE:
/*
Index: sys/ofed/drivers/net/mlx4/en_tx.c
===================================================================
--- sys/ofed/drivers/net/mlx4/en_tx.c
+++ sys/ofed/drivers/net/mlx4/en_tx.c
@@ -89,7 +89,8 @@
/* Allocate the buf ring */
ring->br = buf_ring_alloc(MLX4_EN_DEF_TX_QUEUE_SIZE, M_DEVBUF,
- M_WAITOK, &ring->tx_lock.m);
+ M_WAITOK, &ring->tx_lock.m, queue_idx,
+ priv->tx_ring_num);
if (ring->br == NULL) {
en_err(priv, "Failed allocating tx_info ring\n");
return -ENOMEM;
Index: sys/sys/buf_ring.h
===================================================================
--- sys/sys/buf_ring.h
+++ sys/sys/buf_ring.h
@@ -41,6 +41,8 @@
#include <sys/mutex.h>
#endif
+void if_rexmt_start(int qid, int nqs);
+
/* cache line align buf ring entries */
#define BR_FLAGS_ALIGNED 0x1
@@ -61,6 +63,9 @@
volatile uint32_t br_cons_tail;
int br_cons_size;
int br_cons_mask;
+ int br_id;
+ int br_nqs;
+ int br_closed;
#ifdef DEBUG_BUFRING
struct mtx *br_lock;
#endif
@@ -70,6 +75,9 @@
struct br_entry_ br_ring[0] __aligned(CACHE_LINE_SIZE);
};
+
+static __inline int buf_ring_count(struct buf_ring *br);
+
/*
* ring entry accessors to allow us to make ring entry
* alignment determined at runtime
@@ -191,6 +199,10 @@
buf, i, br->br_prod_tail, br->br_cons_tail);
#endif
critical_enter();
+ if (br->br_closed == TRUE) {
+ critical_exit();
+ return (ENOBUFS);
+ }
do {
prod_head = br->br_prod_head;
@@ -206,6 +218,7 @@
continue;
br->br_drops++;
+ br->br_closed = TRUE;
critical_exit();
return (ENOBUFS);
}
@@ -333,7 +346,10 @@
br->br_cons_tail, cons_head);
#endif
atomic_store_rel_32(&br->br_cons_tail, cons_next);
-
+ if (br->br_closed == TRUE && buf_ring_count(br) < (br->br_prod_size >> 1)) {
+ br->br_closed = FALSE;
+ if_rexmt_start(br->br_id, br->br_nqs);
+ }
return ((void *)(uintptr_t)buf);
}
@@ -366,6 +382,10 @@
*/
br->br_ring[cons_head].bre_ptr = NULL;
atomic_store_rel_32(&br->br_cons_tail, cons_next);
+ if (br->br_closed == TRUE && buf_ring_count(br) < (br->br_prod_size >> 1)) {
+ br->br_closed = FALSE;
+ if_rexmt_start(br->br_id, br->br_nqs);
+ }
}
/*
@@ -451,7 +471,7 @@
}
struct buf_ring *buf_ring_alloc(int count, struct malloc_type *type, int flags,
- struct mtx *);
+ struct mtx *, int id, int nqs);
struct buf_ring *buf_ring_aligned_alloc(int count, struct malloc_type *type, int flags,
struct mtx *);
void buf_ring_free(struct buf_ring *br, struct malloc_type *type);
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Tue, Nov 17, 10:58 AM (18 h, 46 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41798486
Default Alt Text
D4295.1794913098.diff (26 KB)
Attached To
Mode
D4295: Add driver backpressure
Attached
Detach File
Event Timeline