diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c --- a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c @@ -498,15 +498,38 @@ } } +static void +mlx5e_input_mbs(struct mlx5e_rq *rq, struct mbufq *mbsp) +{ + struct mbuf *mb; + + mtx_unlock(&rq->mtx); + + while ((mb = mbufq_dequeue(mbsp)) != NULL) { +#if !defined(HAVE_TCP_LRO_RX) + tcp_lro_queue_mbuf(&rq->lro, mb); +#else + if (mb->m_pkthdr.csum_flags == 0 || + (if_getcapenable(rq->ifp) & IFCAP_LRO) == 0 || + rq->lro.lro_cnt == 0 || + tcp_lro_rx(&rq->lro, mb, 0) != 0) { + if_input(rq->ifp, mb); + } +#endif + } + + mtx_lock(&rq->mtx); +} + static int -mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) +mlx5e_poll_rx_cq(struct mlx5e_rq *rq, struct mbufq *mbsp, int budget) { struct pfil_head *pfil; - int i, rv; + int i, c, rv; CURVNET_SET_QUIET(if_getvnet(rq->ifp)); pfil = rq->channel->priv->pfil; - for (i = 0; i < budget; i++) { + for (i = 0, c = 0; i < budget; i++) { struct mlx5e_rx_wqe *wqe; struct mlx5_cqe64 *cqe; struct mbuf *mb; @@ -595,16 +618,7 @@ mb->m_pkthdr.numa_domain = if_getnumadomain(rq->ifp); #endif -#if !defined(HAVE_TCP_LRO_RX) - tcp_lro_queue_mbuf(&rq->lro, mb); -#else - if (mb->m_pkthdr.csum_flags == 0 || - (if_getcapenable(rq->ifp) & IFCAP_LRO) == 0 || - rq->lro.lro_cnt == 0 || - tcp_lro_rx(&rq->lro, mb, 0) != 0) { - if_input(rq->ifp, mb); - } -#endif + mbufq_enqueue(mbsp, mb); wq_ll_pop: mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, &wqe->next.next_wqe_index); @@ -623,7 +637,10 @@ { struct mlx5e_channel *c = container_of(mcq, struct mlx5e_channel, rq.cq.mcq); struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); - int i = 0; + struct mbufq mbs; + int i, wqe_processed; + + mbufq_init(&mbs, 0); #ifdef HAVE_PER_CQ_EVENT_PACKET #if (MHLEN < 15) @@ -657,22 +674,23 @@ * Polling the entire CQ without posting new WQEs results in * lack of receive WQEs during heavy traffic scenarios. */ - while (1) { - if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) != - MLX5E_RX_BUDGET_MAX) - break; - i += MLX5E_RX_BUDGET_MAX; - if (i >= MLX5E_BUDGET_MAX) - break; + for (i = 0; i < MLX5E_BUDGET_MAX; i+= wqe_processed) { + wqe_processed = mlx5e_poll_rx_cq(rq, &mbs, + MLX5E_RX_BUDGET_MAX); + mlx5e_input_mbs(rq, &mbs); + MPASS(mbufq_empty(&mbs)); mlx5e_post_rx_wqes(rq); + if (wqe_processed != MLX5E_RX_BUDGET_MAX) + break; } - mlx5e_post_rx_wqes(rq); + /* check for dynamic interrupt moderation callback */ if (rq->dim.mode != NET_DIM_CQ_PERIOD_MODE_DISABLED) net_dim(&rq->dim, rq->stats.packets, rq->stats.bytes); - mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock)); - tcp_lro_flush_all(&rq->lro); + mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK( + &rq->channel->priv->doorbell_lock)); mtx_unlock(&rq->mtx); + tcp_lro_flush_all(&rq->lro); for (int j = 0; j != MLX5E_MAX_TX_NUM_TC; j++) { mtx_lock(&c->sq[j].lock);