diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c --- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c @@ -498,15 +498,38 @@ } } +static void +mlx5e_input_mbs(struct mlx5e_rq *rq, struct mbuf **mbs, int c) +{ + int i; + + mtx_unlock(&rq->mtx); + + for (i = 0; i < c; i++) { +#if !defined(HAVE_TCP_LRO_RX) + tcp_lro_queue_mbuf(&rq->lro, mbs[i]); +#else + if (mb->m_pkthdr.csum_flags == 0 || + (if_getcapenable(rq->ifp) & IFCAP_LRO) == 0 || + rq->lro.lro_cnt == 0 || + tcp_lro_rx(&rq->lro, mbs[i], 0) != 0) { + if_input(rq->ifp, mbs[i]); + } +#endif + } + + mtx_lock(&rq->mtx); +} + static int -mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) +mlx5e_poll_rx_cq(struct mlx5e_rq *rq, struct mbuf **mbs, int budget, int *cp) { struct pfil_head *pfil; - int i, rv; + int i, c, rv; CURVNET_SET_QUIET(if_getvnet(rq->ifp)); pfil = rq->channel->priv->pfil; - for (i = 0; i < budget; i++) { + for (i = 0, c = 0; i < budget; i++) { struct mlx5e_rx_wqe *wqe; struct mlx5_cqe64 *cqe; struct mbuf *mb; @@ -595,16 +618,7 @@ mb->m_pkthdr.numa_domain = if_getnumadomain(rq->ifp); #endif -#if !defined(HAVE_TCP_LRO_RX) - tcp_lro_queue_mbuf(&rq->lro, mb); -#else - if (mb->m_pkthdr.csum_flags == 0 || - (if_getcapenable(rq->ifp) & IFCAP_LRO) == 0 || - rq->lro.lro_cnt == 0 || - tcp_lro_rx(&rq->lro, mb, 0) != 0) { - if_input(rq->ifp, mb); - } -#endif + mbs[c++] = mb; wq_ll_pop: mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, &wqe->next.next_wqe_index); @@ -615,6 +629,7 @@ /* ensure cq space is freed before enabling more cqes */ atomic_thread_fence_rel(); + *cp = c; return (i); } @@ -623,7 +638,8 @@ { struct mlx5e_channel *c = container_of(mcq, struct mlx5e_channel, rq.cq.mcq); struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); - int i = 0; + struct mbuf *mbs[MLX5E_RX_BUDGET_MAX]; + int i = 0, mbs_count, wqe_processed; #ifdef HAVE_PER_CQ_EVENT_PACKET #if (MHLEN < 15) @@ -657,9 +673,11 @@ * Polling the entire CQ without posting new WQEs results in * lack of receive WQEs during heavy traffic scenarios. */ - while (1) { - if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) != - MLX5E_RX_BUDGET_MAX) + for (;;) { + wqe_processed = mlx5e_poll_rx_cq(rq, mbs, MLX5E_RX_BUDGET_MAX, + &mbs_count); + mlx5e_input_mbs(rq, mbs, mbs_count); + if (wqe_processed != MLX5E_RX_BUDGET_MAX) break; i += MLX5E_RX_BUDGET_MAX; if (i >= MLX5E_BUDGET_MAX) @@ -670,9 +688,10 @@ /* check for dynamic interrupt moderation callback */ if (rq->dim.mode != NET_DIM_CQ_PERIOD_MODE_DISABLED) net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes); - mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock)); - tcp_lro_flush_all(&rq->lro); + mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK( + &rq->channel->priv->doorbell_lock)); mtx_unlock(&rq->mtx); + tcp_lro_flush_all(&rq->lro); for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) { mtx_lock(&c->sq[j].lock);