Differential D11560 Diff 30657 sys/dev/mlx4/mlx4_en/mlx4_en_rx.c

Changeset View

Standalone View

sys/dev/mlx4/mlx4_en/mlx4_en_rx.c

Show First 20 Lines • Show All 47 Lines • ▼ Show 20 Lines
static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,		static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring,		struct mlx4_en_rx_ring *ring,
int index)		int index)
{		{
struct mlx4_en_rx_desc rx_desc = (struct mlx4_en_rx_desc )		struct mlx4_en_rx_desc rx_desc = (struct mlx4_en_rx_desc )
(ring->buf + (ring->stride * index));		(ring->buf + (ring->stride * index));
int possible_frags;		int possible_frags;
int i;		int i;
		int ip_align;

		ip_align = MLX4_NET_IP_ALIGN;
/* Set size and memtype fields */		/* Set size and memtype fields */
rx_desc->data[0].byte_count = cpu_to_be32(priv->rx_mb_size - MLX4_NET_IP_ALIGN);		for (i = 0; i < priv->num_frags; i++) {
rx_desc->data[0].lkey = cpu_to_be32(priv->mdev->mr.key);		rx_desc->data[i].byte_count =
		cpu_to_be32(priv->frag_info[i].frag_size - ip_align);
		rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key);

		/* Adjust only the first fragment for IP header alignment. */
		ip_align = 0;
		}

/*		/*
* If the number of used fragments does not fill up the ring		* If the number of used fragments does not fill up the ring
* stride, remaining (unused) fragments must be padded with		* stride, remaining (unused) fragments must be padded with
* null address/size and a special memory key:		* null address/size and a special memory key:
*/		*/
possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;		possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
for (i = 1; i < possible_frags; i++) {		for (i = priv->num_frags; i < possible_frags; i++) {
rx_desc->data[i].byte_count = 0;		rx_desc->data[i].byte_count = 0;
rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);		rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
rx_desc->data[i].addr = 0;		rx_desc->data[i].addr = 0;
}		}

}		}

static int		static int
mlx4_en_alloc_buf(struct mlx4_en_rx_ring *ring,		mlx4_en_alloc_buf(struct mlx4_en_priv priv, struct mlx4_en_rx_ring ring,
__be64 pdma, struct mlx4_en_rx_mbuf mb_list)		__be64 pdma, struct mlx4_en_rx_mbuf mb_list, int flags, int frag_size)
{		{
bus_dma_segment_t segs[1];		bus_dma_segment_t segs[1];
bus_dmamap_t map;
struct mbuf *mb;		struct mbuf *mb;
int nsegs;		int nsegs;
int err;		int err;

/* try to allocate a new spare mbuf */
if (unlikely(ring->spare.mbuf == NULL)) {
mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size);
if (unlikely(mb == NULL))
return (-ENOMEM);
/* setup correct length */
mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size;

/* make sure IP header gets aligned */
m_adj(mb, MLX4_NET_IP_ALIGN);

/* load spare mbuf into BUSDMA */
err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, ring->spare.dma_map,
mb, segs, &nsegs, BUS_DMA_NOWAIT);
if (unlikely(err != 0)) {
m_freem(mb);
return (err);
}

/* store spare info */
ring->spare.mbuf = mb;
ring->spare.paddr_be = cpu_to_be64(segs[0].ds_addr);

bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map,
BUS_DMASYNC_PREREAD);
}

/* synchronize and unload the current mbuf, if any */		/* synchronize and unload the current mbuf, if any */
if (likely(mb_list->mbuf != NULL)) {		if (likely(mb_list->mbuf != NULL)) {
bus_dmamap_sync(ring->dma_tag, mb_list->dma_map,		bus_dmamap_sync(ring->dma_tag, mb_list->dma_map,
BUS_DMASYNC_POSTREAD);		BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(ring->dma_tag, mb_list->dma_map);		bus_dmamap_unload(ring->dma_tag, mb_list->dma_map);
}		}

mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, ring->rx_mb_size);		mb = m_getjcl(M_NOWAIT, MT_DATA, flags, frag_size);
if (unlikely(mb == NULL))		if (unlikely(mb == NULL)) {
goto use_spare;		priv->port_stats.rx_alloc_failed++;
		return -ENOMEM;
		}

/* setup correct length */		/* setup correct length */
mb->m_pkthdr.len = mb->m_len = ring->rx_mb_size;		mb->m_len = frag_size;

/* make sure IP header gets aligned */		/* make sure IP header gets aligned */
		if (flags & M_PKTHDR) {
		mb->m_pkthdr.len = frag_size;
m_adj(mb, MLX4_NET_IP_ALIGN);		m_adj(mb, MLX4_NET_IP_ALIGN);
		}

err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map,		err = -bus_dmamap_load_mbuf_sg(ring->dma_tag, mb_list->dma_map,
mb, segs, &nsegs, BUS_DMA_NOWAIT);		mb, segs, &nsegs, BUS_DMA_NOWAIT);
if (unlikely(err != 0)) {		if (unlikely(err != 0)) {
m_freem(mb);		m_freem(mb);
goto use_spare;		return (-err);
}		}

*pdma = cpu_to_be64(segs[0].ds_addr);		*pdma = cpu_to_be64(segs[0].ds_addr);
mb_list->mbuf = mb;		mb_list->mbuf = mb;

bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD);		bus_dmamap_sync(ring->dma_tag, mb_list->dma_map, BUS_DMASYNC_PREREAD);
return (0);		return (0);
		}

use_spare:		static void
/* swap DMA maps */		mlx4_en_free_buf(struct mlx4_en_priv priv, struct mlx4_en_rx_ring ring,
map = mb_list->dma_map;		struct mlx4_en_rx_mbuf *mb_list)
mb_list->dma_map = ring->spare.dma_map;		{
ring->spare.dma_map = map;		bus_dmamap_t map;
		bus_dma_tag_t tag;
		int nr;

/* swap MBUFs */		for (nr = 0; nr < priv->num_frags; nr++) {
mb_list->mbuf = ring->spare.mbuf;		en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
ring->spare.mbuf = NULL;

/* store physical address */		if (mb_list->mbuf != NULL) {
*pdma = ring->spare.paddr_be;		map = mb_list->dma_map;
return (0);		tag = ring->dma_tag;

		bus_dmamap_sync(tag, map, BUS_DMASYNC_POSTREAD);
		bus_dmamap_unload(tag, map);
		m_freem(mb_list->mbuf);
		mb_list->mbuf = NULL; /* safety clearing */
}		}
		mb_list++;
		}
		}

static void		static void
mlx4_en_free_buf(struct mlx4_en_rx_ring ring, struct mlx4_en_rx_mbuf mb_list)		mlx4_en_free_rx_desc(struct mlx4_en_priv priv, struct mlx4_en_rx_ring ring,
		int index)
{		{
bus_dmamap_t map = mb_list->dma_map;		struct mlx4_en_rx_mbuf *mb_list;
bus_dmamap_sync(ring->dma_tag, map, BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(ring->dma_tag, map);		mb_list = ring->mbuf + (index << priv->log_mbuf);
m_freem(mb_list->mbuf);		mlx4_en_free_buf(priv, ring, mb_list);
mb_list->mbuf = NULL; /* safety clearing */
}		}

static int		static int
mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,		mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring, int index)		struct mlx4_en_rx_ring *ring, int index)
{		{
struct mlx4_en_rx_desc rx_desc = (struct mlx4_en_rx_desc )		struct mlx4_en_rx_desc rx_desc = (struct mlx4_en_rx_desc )
(ring->buf + (index * ring->stride));		(ring->buf + (index * ring->stride));
struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + index;		struct mlx4_en_rx_mbuf *mb_list = ring->mbuf + (index << priv->log_mbuf);
		int i;
		int flags;

mb_list->mbuf = NULL;		mlx4_en_free_buf(priv, ring, mb_list);

if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list)) {		flags = M_PKTHDR;
priv->port_stats.rx_alloc_failed++;		for (i = 0; i < priv->num_frags; i++) {
return (-ENOMEM);		if (mlx4_en_alloc_buf(priv, ring, &rx_desc->data[i].addr, &mb_list[i],
		flags, priv->frag_info[i].frag_size))
		goto err;
		flags = 0;
}		}
return (0);		return (0);

		err:
		mlx4_en_free_buf(priv, ring, mb_list);
		return -ENOMEM;
}		}

static inline void		static inline void
mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)		mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)
{		{
*ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);		*ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
}		}

Show All 32 Lines	static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
return 0;		return 0;

reduce_rings:		reduce_rings:
for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {		for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
ring = priv->rx_ring[ring_ind];		ring = priv->rx_ring[ring_ind];
while (ring->actual_size > new_size) {		while (ring->actual_size > new_size) {
ring->actual_size--;		ring->actual_size--;
ring->prod--;		ring->prod--;
mlx4_en_free_buf(ring,		mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
ring->mbuf + ring->actual_size);
}		}
}		}

return 0;		return 0;
}		}

static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,		static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring *ring)		struct mlx4_en_rx_ring *ring)
{		{
int index;		int index;

en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",		en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",
ring->cons, ring->prod);		ring->cons, ring->prod);

/* Unmap and free Rx buffers */		/* Unmap and free Rx buffers */
BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size);		BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size);
while (ring->cons != ring->prod) {		while (ring->cons != ring->prod) {
index = ring->cons & ring->size_mask;		index = ring->cons & ring->size_mask;
en_dbg(DRV, priv, "Processing descriptor:%d\n", index);		en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
mlx4_en_free_buf(ring, ring->mbuf + index);		mlx4_en_free_rx_desc(priv, ring, index);
++ring->cons;		++ring->cons;
}		}
}		}

		#if MLX4_EN_MAX_RX_FRAGS == 3
		static int frag_sizes[] = {
		FRAG_SZ0,
		FRAG_SZ1,
		FRAG_SZ2,
		};
		#elif MLX4_EN_MAX_RX_FRAGS == 2
		static int frag_sizes[] = {
		FRAG_SZ0,
		FRAG_SZ1,
		};
		#else
		#error "Unknown MAX_RX_FRAGS"
		#endif

void mlx4_en_calc_rx_buf(struct net_device *dev)		void mlx4_en_calc_rx_buf(struct net_device *dev)
{		{
struct mlx4_en_priv *priv = netdev_priv(dev);		struct mlx4_en_priv *priv = netdev_priv(dev);
int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN +		int eff_mtu = dev->if_mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN +
MLX4_NET_IP_ALIGN;		MLX4_NET_IP_ALIGN;
		int buf_size = 0;
		int i, frag;

if (eff_mtu > MJUM16BYTES) {		/*
en_err(priv, "MTU(%d) is too big\n", dev->if_mtu);		* Try to fit packets into a single mbuf+cluster, but we have to split
eff_mtu = MJUM16BYTES;		* frames across multiple mbufs if the MTU is greater than the page
} else if (eff_mtu > MJUM9BYTES) {		* size so that we don't don't trigger the (very expensive) contiguous
eff_mtu = MJUM16BYTES;		* memory allocator during normal rx operation.
} else if (eff_mtu > MJUMPAGESIZE) {		*/
eff_mtu = MJUM9BYTES;		if (eff_mtu <= MCLBYTES) {
} else if (eff_mtu > MCLBYTES) {		priv->frag_info[0].frag_size = MCLBYTES;
eff_mtu = MJUMPAGESIZE;		priv->num_frags = 1;
		} else if (eff_mtu <= MJUMPAGESIZE) {
		priv->frag_info[0].frag_size = MJUMPAGESIZE;
		priv->num_frags = 1;
} else {		} else {
eff_mtu = MCLBYTES;		for (i = 0, frag = 0; buf_size < eff_mtu; frag++, i++) {
		/*
		* Allocate small to large but only as much as is needed for
		* the tail.
		*/
		while (i > 0 && eff_mtu - buf_size <= frag_sizes[i - 1])
		i--;
		priv->frag_info[frag].frag_size = frag_sizes[i];
		buf_size += priv->frag_info[frag].frag_size;
}		}

		priv->num_frags = frag;
		}
priv->rx_mb_size = eff_mtu;		priv->rx_mb_size = eff_mtu;
		priv->log_mbuf = ROUNDUP_LOG2(priv->num_frags);

en_dbg(DRV, priv, "Effective RX MTU: %d bytes\n", eff_mtu);		en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d "
		"num_frags:%d):\n", eff_mtu, priv->num_frags);
		for (i = 0; i < priv->num_frags; i++) {
		en_dbg(DRV, priv, " frag:%d - size:%d\n", i,
		priv->frag_info[i].frag_size);
}		}
		}

int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,		int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
struct mlx4_en_rx_ring **pring,		struct mlx4_en_rx_ring **pring,
u32 size, int node)		u32 size, int node)
{		{
struct mlx4_en_dev *mdev = priv->mdev;		struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_rx_ring *ring;		struct mlx4_en_rx_ring *ring;
int err;		int err;
int tmp;		size_t ring_size_bytes;
uint32_t x;		uint32_t x;

ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL);		ring = kzalloc(sizeof(struct mlx4_en_rx_ring), GFP_KERNEL);
if (!ring) {		if (!ring) {
en_err(priv, "Failed to allocate RX ring structure\n");		en_err(priv, "Failed to allocate RX ring structure\n");
return -ENOMEM;		return -ENOMEM;
}		}

/* Create DMA descriptor TAG */		/* Create DMA descriptor TAG */
if ((err = -bus_dma_tag_create(		if ((err = -bus_dma_tag_create(
bus_get_dma_tag(mdev->pdev->dev.bsddev),		bus_get_dma_tag(mdev->pdev->dev.bsddev),
1, /* any alignment */		1, /* any alignment */
0, /* no boundary */		0, /* no boundary */
BUS_SPACE_MAXADDR, /* lowaddr */		BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */		BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */		NULL, NULL, /* filter, filterarg */
MJUM16BYTES, /* maxsize */		MJUMPAGESIZE, /* maxsize */
1, /* nsegments */		1, /* nsegments */
MJUM16BYTES, /* maxsegsize */		MJUMPAGESIZE, /* maxsegsize */
0, /* flags */		0, /* flags */
NULL, NULL, /* lockfunc, lockfuncarg */		NULL, NULL, /* lockfunc, lockfuncarg */
&ring->dma_tag))) {		&ring->dma_tag))) {
en_err(priv, "Failed to create DMA tag\n");		en_err(priv, "Failed to create DMA tag\n");
goto err_ring;		goto err_ring;
}		}

ring->prod = 0;		ring->prod = 0;
ring->cons = 0;		ring->cons = 0;
ring->size = size;		ring->size = size;
ring->size_mask = size - 1;		ring->size_mask = size - 1;
ring->stride = roundup_pow_of_two(		ring->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
sizeof(struct mlx4_en_rx_desc) + DS_SIZE);		DS_SIZE * MLX4_EN_MAX_RX_FRAGS);
ring->log_stride = ffs(ring->stride) - 1;		ring->log_stride = ffs(ring->stride) - 1;
ring->buf_size = ring->size * ring->stride + TXBB_SIZE;		ring->buf_size = ring->size * ring->stride + TXBB_SIZE;

tmp = size * sizeof(struct mlx4_en_rx_mbuf);		ring->num_mbufs = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS);
		ring_size_bytes = ring->num_mbufs * sizeof(struct mlx4_en_rx_mbuf);

ring->mbuf = kzalloc(tmp, GFP_KERNEL);		ring->mbuf = kzalloc(ring_size_bytes, GFP_KERNEL);
if (ring->mbuf == NULL) {		if (ring->mbuf == NULL) {
err = -ENOMEM;		err = -ENOMEM;
goto err_dma_tag;		goto err_dma_tag;
}		}

err = -bus_dmamap_create(ring->dma_tag, 0, &ring->spare.dma_map);		for (x = 0; x != ring->num_mbufs; x++) {
if (err != 0)
goto err_info;

for (x = 0; x != size; x++) {
err = -bus_dmamap_create(ring->dma_tag, 0,		err = -bus_dmamap_create(ring->dma_tag, 0,
&ring->mbuf[x].dma_map);		&ring->mbuf[x].dma_map);
if (err != 0) {		if (err != 0) {
while (x--)		while (x--)
bus_dmamap_destroy(ring->dma_tag,		bus_dmamap_destroy(ring->dma_tag,
ring->mbuf[x].dma_map);		ring->mbuf[x].dma_map);
goto err_info;		goto err_info;
}		}
}		}
en_dbg(DRV, priv, "Allocated MBUF ring at addr:%p size:%d\n",		en_dbg(DRV, priv, "Allocated MBUF ring at addr:%p size:%zd\n",
ring->mbuf, tmp);		ring->mbuf, ring_size_bytes);

err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,		err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
ring->buf_size, 2 * PAGE_SIZE);		ring->buf_size, 2 * PAGE_SIZE);
if (err)		if (err)
goto err_dma_map;		goto err_dma_map;

err = mlx4_en_map_buffer(&ring->wqres.buf);		err = mlx4_en_map_buffer(&ring->wqres.buf);
if (err) {		if (err) {
Show All 22 Lines
}		}

int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)		int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
{		{
struct mlx4_en_rx_ring *ring;		struct mlx4_en_rx_ring *ring;
int i;		int i;
int ring_ind;		int ring_ind;
int err;		int err;
int stride = roundup_pow_of_two(		int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
sizeof(struct mlx4_en_rx_desc) + DS_SIZE);		DS_SIZE * priv->num_frags);

for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {		for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
ring = priv->rx_ring[ring_ind];		ring = priv->rx_ring[ring_ind];

ring->prod = 0;		ring->prod = 0;
ring->cons = 0;		ring->cons = 0;
ring->actual_size = 0;		ring->actual_size = 0;
ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn;		ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn;
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines	void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
u32 size, u16 stride)		u32 size, u16 stride)
{		{
struct mlx4_en_dev *mdev = priv->mdev;		struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_rx_ring ring = pring;		struct mlx4_en_rx_ring ring = pring;
uint32_t x;		uint32_t x;

mlx4_en_unmap_buffer(&ring->wqres.buf);		mlx4_en_unmap_buffer(&ring->wqres.buf);
mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);		mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
for (x = 0; x != size; x++)		for (x = 0; x != ring->num_mbufs; x++)
bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map);		bus_dmamap_destroy(ring->dma_tag, ring->mbuf[x].dma_map);
/* free spare mbuf, if any */
if (ring->spare.mbuf != NULL) {
bus_dmamap_sync(ring->dma_tag, ring->spare.dma_map,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(ring->dma_tag, ring->spare.dma_map);
m_freem(ring->spare.mbuf);
}
bus_dmamap_destroy(ring->dma_tag, ring->spare.dma_map);
vfree(ring->mbuf);		vfree(ring->mbuf);
bus_dma_tag_destroy(ring->dma_tag);		bus_dma_tag_destroy(ring->dma_tag);
kfree(ring);		kfree(ring);
*pring = NULL;		*pring = NULL;
#ifdef CONFIG_RFS_ACCEL		#ifdef CONFIG_RFS_ACCEL
mlx4_en_cleanup_filters(priv, ring);		mlx4_en_cleanup_filters(priv, ring);
#endif		#endif
}		}
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines	static inline int invalid_cqe(struct mlx4_en_priv *priv,
if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {		if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");		en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
return 1;		return 1;
}		}

return 0;		return 0;
}		}

		/*
		* Collect up the packet fragments, represented by individual mbufs, into a
		* single mbuf chain ready to be passed up the stack. As mbufs are removed from
		* the ring replace them with newly allocated ones; if we fail to allocate an
		* mbuf then drop the current packet and return an error. This ensures that the
		* ring is always in a state where it is ready to receive packets.
		*/
		static int
		mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
		struct mlx4_en_rx_ring ring, struct mlx4_en_rx_desc rx_desc,
		struct mlx4_en_rx_mbuf *mb_list, int length)
		{
		struct mlx4_en_frag_info *frag_info;
		struct mbuf mb, first_mb, *prev_mb;
		int flags, nr, align_len, mb_len;

		first_mb = mb_list[0].mbuf;
		prev_mb = NULL;
		first_mb->m_pkthdr.len = length;
		flags = M_PKTHDR;
		align_len = MLX4_NET_IP_ALIGN;

		/* Collect used fragments while replacing them in the HW descriptors */
		for (nr = 0; nr < priv->num_frags; nr++) {
		frag_info = &priv->frag_info[nr];

		mb = mb_list[nr].mbuf;

		/* Allocate a replacement page */
		if (mlx4_en_alloc_buf(priv, ring, &rx_desc->data[nr].addr,
		&mb_list[nr], flags, priv->frag_info[nr].frag_size))
		goto fail;
		kibUnsubmitted Not Done Inline Actions Does this mean that we claim that the reception of the packet failed if we were unable to allocate the replacement mbuf or page for the used fragment ? If yes. should we split that ? kib: Does this mean that we claim that the reception of the packet failed if we were unable to…

		if (prev_mb != NULL)
		prev_mb->m_next = mb;
		mb_len = frag_info->frag_size - align_len;
		prev_mb = mb;

		if (length <= mb_len)
		break;

		mb->m_len = mb_len;
		length -= mb_len;
		flags = 0;
		align_len = 0;
		}
		/* Adjust size of last fragment to match actual length */
		prev_mb->m_len = min(length, prev_mb->m_len);;
		prev_mb->m_next = NULL;
		return (0);

		fail:
		/*
		* At this point the fragments have been partially extracted and
		* replaced. Free the mbufs that are no longer referenced by the ring.
		*/
		if (first_mb != mb_list[0].mbuf)
		m_freem(first_mb);
		return (-ENOMEM);

		}

static struct mbuf *		static struct mbuf *
mlx4_en_rx_mb(struct mlx4_en_priv priv, struct mlx4_en_rx_ring ring,		mlx4_en_rx_mb(struct mlx4_en_priv priv, struct mlx4_en_rx_ring ring,
		kibUnsubmitted Not Done Inline Actions I would say that after such refactoring, the function is no longer useful. You can inline its call into process_rx_cq(). kib: I would say that after such refactoring, the function is no longer useful. You can inline its…
struct mlx4_en_rx_desc rx_desc, struct mlx4_en_rx_mbuf mb_list,		struct mlx4_en_rx_desc rx_desc, struct mlx4_en_rx_mbuf mb_list,
int length)		int length)
{		{
struct mbuf *mb;		struct mbuf *mb;

/* get mbuf */
mb = mb_list->mbuf;		mb = mb_list->mbuf;

/* collect used fragment while atomically replacing it */		/* Move relevant fragments to mb */
if (mlx4_en_alloc_buf(ring, &rx_desc->data[0].addr, mb_list))		if (unlikely(mlx4_en_complete_rx_desc(priv, ring, rx_desc, mb_list, length)))
return (NULL);		return NULL;

/* range check hardware computed value */
if (unlikely(length > mb->m_len))
length = mb->m_len;

/* update total packet length in packet header */
mb->m_len = mb->m_pkthdr.len = length;
return (mb);		return (mb);
}		}

/* For cpu arch with cache line of 64B the performance is better when cqe size==64B		/* For cpu arch with cache line of 64B the performance is better when cqe size==64B
* To enlarge cqe size from 32B to 64B --> 32B of garbage (i.e. 0xccccccc)		* To enlarge cqe size from 32B to 64B --> 32B of garbage (i.e. 0xccccccc)
* was added in the beginning of each cqe (the real data is in the corresponding 32B).		* was added in the beginning of each cqe (the real data is in the corresponding 32B).
* The following calc ensures that when factor==1, it means we are aligned to 64B		* The following calc ensures that when factor==1, it means we are aligned to 64B
* and we get the real cqe data*/		* and we get the real cqe data*/
Show All 23 Lines	int mlx4_en_process_rx_cq(struct net_device dev, struct mlx4_en_cq cq, int budget)
* descriptor offset can be deducted from the CQE index instead of		* descriptor offset can be deducted from the CQE index instead of
* reading 'cqe->index' */		* reading 'cqe->index' */
index = cons_index & size_mask;		index = cons_index & size_mask;
cqe = &buf[CQE_FACTOR_INDEX(index, factor)];		cqe = &buf[CQE_FACTOR_INDEX(index, factor)];

/* Process all completed CQEs */		/* Process all completed CQEs */
while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,		while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
cons_index & size)) {		cons_index & size)) {
mb_list = ring->mbuf + index;		mb_list = ring->mbuf + (index << priv->log_mbuf);
rx_desc = (struct mlx4_en_rx_desc *)		rx_desc = (struct mlx4_en_rx_desc *)
(ring->buf + (index << ring->log_stride));		(ring->buf + (index << ring->log_stride));

/*		/*
* make sure we read the CQE after we read the ownership bit		* make sure we read the CQE after we read the ownership bit
*/		*/
rmb();		rmb();

▲ Show 20 Lines • Show All 326 Lines • Show Last 20 Lines