Page MenuHomeFreeBSD
Differential D50786 Diff 156989 sys/dev/gve/gve_rx_dqo.c

Changeset View

Standalone View

View Options

sys/dev/gve/gve_rx_dqo.c

Show First 20 Lines • Show All 134 Lines • ▼ Show 20 Lines if (rx->com.qpl == NULL) {
device_printf(priv->dev, device_printf(priv->dev,
"Failed to alloc QPL for rx ring %d", i); "Failed to alloc QPL for rx ring %d", i);
err = ENOMEM; err = ENOMEM;
goto abort; goto abort;
} }
return (0); return (0);
} }
bus_size_t max_seg_size = gve_rx_dqo_mbuf_segment_size(priv);
err = bus_dma_tag_create( err = bus_dma_tag_create(
bus_get_dma_tag(priv->dev), /* parent */ bus_get_dma_tag(priv->dev), /* parent */
1, 0, /* alignment, bounds */ 1, 0, /* alignment, bounds */
BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* lowaddr */
BUS_SPACE_MAXADDR, /* highaddr */ BUS_SPACE_MAXADDR, /* highaddr */
NULL, NULL, /* filter, filterarg */ NULL, NULL, /* filter, filterarg */
MCLBYTES, /* maxsize */ max_seg_size, /* maxsize */
1, /* nsegments */ 1, /* nsegments */
MCLBYTES, /* maxsegsize */ max_seg_size, /* maxsegsize */
0, /* flags */ 0, /* flags */
NULL, /* lockfunc */ NULL, /* lockfunc */
NULL, /* lockarg */ NULL, /* lockarg */
&rx->dqo.buf_dmatag); &rx->dqo.buf_dmatag);
if (err != 0) { if (err != 0) {
device_printf(priv->dev, device_printf(priv->dev,
"%s: bus_dma_tag_create failed: %d\n", "%s: bus_dma_tag_create failed: %d\n",
__func__, err); __func__, err);
▲ Show 20 Lines • Show All 152 Lines • ▼ Show 20 Linesgve_rx_post_new_mbuf_dqo(struct gve_rx_ring *rx, int how)
buf = SLIST_FIRST(&rx->dqo.free_bufs); buf = SLIST_FIRST(&rx->dqo.free_bufs);
if (__predict_false(!buf)) { if (__predict_false(!buf)) {
device_printf(rx->com.priv->dev, device_printf(rx->com.priv->dev,
"Unexpected empty free bufs list\n"); "Unexpected empty free bufs list\n");
return (ENOBUFS); return (ENOBUFS);
} }
SLIST_REMOVE_HEAD(&rx->dqo.free_bufs, slist_entry); SLIST_REMOVE_HEAD(&rx->dqo.free_bufs, slist_entry);
buf->mbuf = m_getcl(how, MT_DATA, M_PKTHDR); bus_size_t segment_size = gve_rx_dqo_mbuf_segment_size(rx->com.priv);
buf->mbuf = m_getjcl(how, MT_DATA, M_PKTHDR, segment_size);
veethebee_google.comAuthorUnsubmitted
Done
Inline Actions

@markj, @delphij or any other folks familiar with memory management in freebsd, do you have any insight on whether 4k mbuf jumbo clusters are as readily available as 2k mbuf clusters? From my understanding, fetching mbuf clusters only performs one fetch as there is a memory region for mbufs with pre-attached 2k clusters but when getting a 4k jumbo cluster, an mbuf is fetched separately from fetching a jumbo cluster. Would this pose any issues in replacing the use of 2k clusters with 4k jumbo clusters?

veethebee_google.com: @markj, @delphij or any other folks familiar with memory management in freebsd, do you have any…
markjUnsubmitted
Done
Inline Actions

By "fetch" you mean allocate from UMA? It shouldn't make a functional difference. There's a "packet" zone which caches mbuf headers with a 2KB cluster attached, whereas a 4KB packet buffer requires two back-to-back allocations. The packet zone is an optimization to elide a call to uma_zalloc(), but aside from extra performance overhead, the driver shouldn't notice.

Are you able to do some benchmarking to measure CPU usage and throughput with small packets to compare the two allocation strategies?

markj: By "fetch" you mean allocate from UMA? It shouldn't make a functional difference. There's a…
veethebee_google.comAuthorUnsubmitted
Done
Inline Actions

Yup, I meant the allocate from UMA. We did do extensive performance testing and didn't see any significant differences between the two.

So it sounds like replacing 2k buffers with 4k buffers shouldn't really have any issues overall. Thanks so much!

veethebee_google.com: Yup, I meant the allocate from UMA. We did do extensive performance testing and didn't see any…
if (__predict_false(!buf->mbuf)) { if (__predict_false(!buf->mbuf)) {
err = ENOMEM; err = ENOMEM;
counter_enter(); counter_enter();
counter_u64_add_protected(rx->stats.rx_mbuf_mclget_null, 1); counter_u64_add_protected(rx->stats.rx_mbuf_mclget_null, 1);
counter_exit(); counter_exit();
goto abort_with_buf; goto abort_with_buf;
} }
buf->mbuf->m_len = MCLBYTES; buf->mbuf->m_len = segment_size;
err = bus_dmamap_load_mbuf_sg(rx->dqo.buf_dmatag, buf->dmamap, err = bus_dmamap_load_mbuf_sg(rx->dqo.buf_dmatag, buf->dmamap,
buf->mbuf, segs, &nsegs, BUS_DMA_NOWAIT); buf->mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
KASSERT(nsegs == 1, ("dma segs for a cluster mbuf is not 1")); KASSERT(nsegs == 1, ("dma segs for a cluster mbuf is not 1"));
if (__predict_false(err != 0)) { if (__predict_false(err != 0)) {
counter_enter(); counter_enter();
counter_u64_add_protected(rx->stats.rx_mbuf_dmamap_err, 1); counter_u64_add_protected(rx->stats.rx_mbuf_dmamap_err, 1);
counter_exit(); counter_exit();
Show All 29 Linesgve_rx_post_qpl_buf_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_dqo *buf,
composed_id.buf_id = buf - rx->dqo.bufs; composed_id.buf_id = buf - rx->dqo.bufs;
composed_id.frag_num = frag_num; composed_id.frag_num = frag_num;
desc->buf_id = htole16(composed_id.all); desc->buf_id = htole16(composed_id.all);
page_dma_handle = gve_get_page_dma_handle(rx, buf); page_dma_handle = gve_get_page_dma_handle(rx, buf);
bus_dmamap_sync(page_dma_handle->tag, page_dma_handle->map, bus_dmamap_sync(page_dma_handle->tag, page_dma_handle->map,
BUS_DMASYNC_PREREAD); BUS_DMASYNC_PREREAD);
desc->buf_addr = htole64(page_dma_handle->bus_addr + desc->buf_addr = htole64(page_dma_handle->bus_addr +
frag_num * GVE_DEFAULT_RX_BUFFER_SIZE); frag_num * rx->com.priv->rx_buf_size_dqo);
buf->num_nic_frags++; buf->num_nic_frags++;
gve_rx_advance_head_dqo(rx); gve_rx_advance_head_dqo(rx);
} }
static void static void
gve_rx_maybe_extract_from_used_bufs(struct gve_rx_ring *rx, bool just_one) gve_rx_maybe_extract_from_used_bufs(struct gve_rx_ring *rx, bool just_one)
{ {
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Linesgve_rx_post_new_dqo_qpl_buf(struct gve_rx_ring *rx)
if (__predict_false(buf == NULL)) { if (__predict_false(buf == NULL)) {
gve_rx_maybe_extract_from_used_bufs(rx, /*just_one=*/true); gve_rx_maybe_extract_from_used_bufs(rx, /*just_one=*/true);
buf = SLIST_FIRST(&rx->dqo.free_bufs); buf = SLIST_FIRST(&rx->dqo.free_bufs);
if (__predict_false(buf == NULL)) if (__predict_false(buf == NULL))
return (ENOBUFS); return (ENOBUFS);
} }
gve_rx_post_qpl_buf_dqo(rx, buf, buf->next_idx); gve_rx_post_qpl_buf_dqo(rx, buf, buf->next_idx);
if (buf->next_idx == GVE_DQ_NUM_FRAGS_IN_PAGE - 1) if (buf->next_idx == gve_get_dq_num_frags_in_page(rx->com.priv) - 1)
buf->next_idx = 0; buf->next_idx = 0;
else else
buf->next_idx++; buf->next_idx++;
/* /*
* We have posted all the frags in this buf to the NIC. * We have posted all the frags in this buf to the NIC.
* - buf will enter used_bufs once the last completion arrives. * - buf will enter used_bufs once the last completion arrives.
* - It will renter free_bufs in gve_rx_maybe_extract_from_used_bufs * - It will renter free_bufs in gve_rx_maybe_extract_from_used_bufs
▲ Show 20 Lines • Show All 295 Lines • ▼ Show 20 Lines
static void * static void *
gve_get_cpu_addr_for_qpl_buf(struct gve_rx_ring *rx, gve_get_cpu_addr_for_qpl_buf(struct gve_rx_ring *rx,
struct gve_rx_buf_dqo *buf, uint8_t buf_frag_num) struct gve_rx_buf_dqo *buf, uint8_t buf_frag_num)
{ {
int page_idx = buf - rx->dqo.bufs; int page_idx = buf - rx->dqo.bufs;
void *va = rx->com.qpl->dmas[page_idx].cpu_addr; void *va = rx->com.qpl->dmas[page_idx].cpu_addr;
va = (char *)va + (buf_frag_num * GVE_DEFAULT_RX_BUFFER_SIZE); va = (char *)va + (buf_frag_num * rx->com.priv->rx_buf_size_dqo);
return (va); return (va);
} }
static int static int
gve_rx_add_clmbuf_to_ctx(struct gve_rx_ring *rx, gve_rx_add_clmbuf_to_ctx(struct gve_rx_ring *rx,
struct gve_rx_ctx *ctx, struct gve_rx_buf_dqo *buf, struct gve_rx_ctx *ctx, struct gve_rx_buf_dqo *buf,
uint8_t buf_frag_num, uint16_t frag_len) uint8_t buf_frag_num, uint16_t frag_len)
{ {
void *va = gve_get_cpu_addr_for_qpl_buf(rx, buf, buf_frag_num); void *va = gve_get_cpu_addr_for_qpl_buf(rx, buf, buf_frag_num);
struct mbuf *mbuf; struct mbuf *mbuf;
bus_size_t segment_size = gve_rx_dqo_mbuf_segment_size(rx->com.priv);
if (ctx->mbuf_tail == NULL) { if (ctx->mbuf_tail == NULL) {
mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, segment_size);
if (mbuf == NULL) if (mbuf == NULL)
return (ENOMEM); return (ENOMEM);
ctx->mbuf_head = mbuf; ctx->mbuf_head = mbuf;
ctx->mbuf_tail = mbuf; ctx->mbuf_tail = mbuf;
} else { } else {
mbuf = m_getcl(M_NOWAIT, MT_DATA, 0); mbuf = m_getjcl(M_NOWAIT, MT_DATA, 0, segment_size);
if (mbuf == NULL) if (mbuf == NULL)
return (ENOMEM); return (ENOMEM);
ctx->mbuf_tail->m_next = mbuf; ctx->mbuf_tail->m_next = mbuf;
ctx->mbuf_tail = mbuf; ctx->mbuf_tail = mbuf;
} }
mbuf->m_len = frag_len; mbuf->m_len = frag_len;
ctx->total_size += frag_len; ctx->total_size += frag_len;
Show All 31 Linesgve_rx_add_extmbuf_to_ctx(struct gve_rx_ring *rx,
} }
mbuf->m_len = frag_len; mbuf->m_len = frag_len;
ctx->total_size += frag_len; ctx->total_size += frag_len;
page_idx = buf - rx->dqo.bufs; page_idx = buf - rx->dqo.bufs;
page = rx->com.qpl->pages[page_idx]; page = rx->com.qpl->pages[page_idx];
page_addr = rx->com.qpl->dmas[page_idx].cpu_addr; page_addr = rx->com.qpl->dmas[page_idx].cpu_addr;
va = (char *)page_addr + (buf_frag_num * GVE_DEFAULT_RX_BUFFER_SIZE); va = (char *)page_addr + (buf_frag_num * rx->com.priv->rx_buf_size_dqo);
/* /*
* Grab an extra ref to the page so that gve_mextadd_free * Grab an extra ref to the page so that gve_mextadd_free
* does not end up freeing the page while the interface exists. * does not end up freeing the page while the interface exists.
*/ */
vm_page_wire(page); vm_page_wire(page);
counter_enter(); counter_enter();
Show All 29 Linesgve_rx_dqo_qpl(struct gve_priv *priv, struct gve_rx_ring *rx,
if (__predict_false(buf_id >= rx->dqo.buf_cnt)) { if (__predict_false(buf_id >= rx->dqo.buf_cnt)) {
device_printf(priv->dev, "Invalid rx buf id %d on rxq %d, issuing reset\n", device_printf(priv->dev, "Invalid rx buf id %d on rxq %d, issuing reset\n",
buf_id, rx->com.id); buf_id, rx->com.id);
gve_schedule_reset(priv); gve_schedule_reset(priv);
goto drop_frag_clear_ctx; goto drop_frag_clear_ctx;
} }
buf = &rx->dqo.bufs[buf_id]; buf = &rx->dqo.bufs[buf_id];
if (__predict_false(buf->num_nic_frags == 0 || if (__predict_false(buf->num_nic_frags == 0 ||
buf_frag_num > GVE_DQ_NUM_FRAGS_IN_PAGE - 1)) { buf_frag_num > gve_get_dq_num_frags_in_page(priv) - 1)) {
device_printf(priv->dev, "Spurious compl for buf id %d on rxq %d " device_printf(priv->dev, "Spurious compl for buf id %d on rxq %d "
"with buf_frag_num %d and num_nic_frags %d, issuing reset\n", "with buf_frag_num %d and num_nic_frags %d, issuing reset\n",
buf_id, rx->com.id, buf_frag_num, buf->num_nic_frags); buf_id, rx->com.id, buf_frag_num, buf->num_nic_frags);
gve_schedule_reset(priv); gve_schedule_reset(priv);
goto drop_frag_clear_ctx; goto drop_frag_clear_ctx;
} }
buf->num_nic_frags--; buf->num_nic_frags--;
▲ Show 20 Lines • Show All 165 Lines • Show Last 20 Lines