Differential D23949 Diff 69519 head/sys/dev/vmware/vmxnet3/if_vmx.c

Changeset View

Standalone View

head/sys/dev/vmware/vmxnet3/if_vmx.c

Show First 20 Lines • Show All 1,488 Lines • ▼ Show 20 Lines	vmxnet3_isc_rxd_pkt_get(void *vsc, if_rxd_info_t ri)
struct vmxnet3_comp_ring *rxc;		struct vmxnet3_comp_ring *rxc;
struct vmxnet3_rxcompdesc *rxcd;		struct vmxnet3_rxcompdesc *rxcd;
struct vmxnet3_rxring *rxr;		struct vmxnet3_rxring *rxr;
struct vmxnet3_rxdesc *rxd;		struct vmxnet3_rxdesc *rxd;
if_rxd_frag_t frag;		if_rxd_frag_t frag;
int cqidx;		int cqidx;
uint16_t total_len;		uint16_t total_len;
uint8_t nfrags;		uint8_t nfrags;
		uint8_t i;
uint8_t flid;		uint8_t flid;

sc = vsc;		sc = vsc;
scctx = sc->vmx_scctx;		scctx = sc->vmx_scctx;
rxq = &sc->vmx_rxq[ri->iri_qsidx];		rxq = &sc->vmx_rxq[ri->iri_qsidx];
rxc = &rxq->vxrxq_comp_ring;		rxc = &rxq->vxrxq_comp_ring;

/*		/*
* Get a single packet starting at the given index in the completion		* Get a single packet starting at the given index in the completion
* queue. That we have been called indicates that		* queue. That we have been called indicates that
* vmxnet3_isc_rxd_available() has already verified that either		* vmxnet3_isc_rxd_available() has already verified that either
* there is a complete packet available starting at the given index,		* there is a complete packet available starting at the given index,
* or there are one or more zero length packets starting at the		* or there are one or more zero length packets starting at the
* given index followed by a complete packet, so no verification of		* given index followed by a complete packet, so no verification of
* ownership of the descriptors (and no associated read barrier) is		* ownership of the descriptors (and no associated read barrier) is
* required here.		* required here.
*/		*/
cqidx = ri->iri_cidx;		cqidx = ri->iri_cidx;
rxcd = &rxc->vxcr_u.rxcd[cqidx];		rxcd = &rxc->vxcr_u.rxcd[cqidx];
while (rxcd->len == 0) {		while (rxcd->len == 0) {
KASSERT(rxcd->sop && rxcd->eop,		KASSERT(rxcd->sop && rxcd->eop,
("%s: zero-length packet without both sop and eop set",		("%s: zero-length packet without both sop and eop set",
__func__));		__func__));
		rxc->vxcr_zero_length++;
if (++cqidx == rxc->vxcr_ndesc) {		if (++cqidx == rxc->vxcr_ndesc) {
cqidx = 0;		cqidx = 0;
rxc->vxcr_gen ^= 1;		rxc->vxcr_gen ^= 1;
}		}
rxcd = &rxc->vxcr_u.rxcd[cqidx];		rxcd = &rxc->vxcr_u.rxcd[cqidx];
}		}
KASSERT(rxcd->sop, ("%s: expected sop", __func__));		KASSERT(rxcd->sop, ("%s: expected sop", __func__));

Show All 39 Lines	case VMXNET3_RCD_RSS_TYPE_NONE:
ri->iri_rsstype = M_HASHTYPE_NONE;		ri->iri_rsstype = M_HASHTYPE_NONE;
break;		break;
default:		default:
ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH;		ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH;
break;		break;
}		}
}		}

/* VLAN */
if (rxcd->vlan) {
ri->iri_flags \|= M_VLANTAG;
ri->iri_vtag = rxcd->vtag;
}

/* Checksum offload */
if (!rxcd->no_csum) {
uint32_t csum_flags = 0;

if (rxcd->ipv4) {
csum_flags \|= CSUM_IP_CHECKED;
if (rxcd->ipcsum_ok)
csum_flags \|= CSUM_IP_VALID;
}
if (!rxcd->fragment && (rxcd->tcp \|\| rxcd->udp)) {
csum_flags \|= CSUM_L4_CALC;
if (rxcd->csum_ok) {
csum_flags \|= CSUM_L4_VALID;
ri->iri_csum_data = 0xffff;
}
}
ri->iri_csum_flags = csum_flags;
}

/*		/*
* The queue numbering scheme used for rxcd->qid is as follows:		* The queue numbering scheme used for rxcd->qid is as follows:
* - All of the command ring 0s are numbered [0, nrxqsets - 1]		* - All of the command ring 0s are numbered [0, nrxqsets - 1]
* - All of the command ring 1s are numbered [nrxqsets, 2*nrxqsets - 1]		* - All of the command ring 1s are numbered [nrxqsets, 2*nrxqsets - 1]
*		*
* Thus, rxcd->qid less than nrxqsets indicates command ring (and		* Thus, rxcd->qid less than nrxqsets indicates command ring (and
* flid) 0, and rxcd->qid greater than or equal to nrxqsets		* flid) 0, and rxcd->qid greater than or equal to nrxqsets
* indicates command ring (and flid) 1.		* indicates command ring (and flid) 1.
Show All 19 Lines	if (++cqidx == rxc->vxcr_ndesc) {
rxc->vxcr_gen ^= 1;		rxc->vxcr_gen ^= 1;
}		}
} while (!rxcd->eop);		} while (!rxcd->eop);

ri->iri_cidx = cqidx;		ri->iri_cidx = cqidx;
ri->iri_nfrags = nfrags;		ri->iri_nfrags = nfrags;
ri->iri_len = total_len;		ri->iri_len = total_len;

		/*
		* If there's an error, the last descriptor in the packet will
		* have the error indicator set. In this case, set all
		* fragment lengths to zero. This will cause iflib to discard
		* the packet, but process all associated descriptors through
		* the refill mechanism.
		*/
		if (__predict_false(rxcd->error)) {
		rxc->vxcr_pkt_errors++;
		for (i = 0; i < nfrags; i++) {
		frag = &ri->iri_frags[i];
		frag->irf_len = 0;
		}
		} else {
		/* Checksum offload information is in the last descriptor. */
		if (!rxcd->no_csum) {
		uint32_t csum_flags = 0;

		if (rxcd->ipv4) {
		csum_flags \|= CSUM_IP_CHECKED;
		if (rxcd->ipcsum_ok)
		csum_flags \|= CSUM_IP_VALID;
		}
		if (!rxcd->fragment && (rxcd->tcp \|\| rxcd->udp)) {
		csum_flags \|= CSUM_L4_CALC;
		if (rxcd->csum_ok) {
		csum_flags \|= CSUM_L4_VALID;
		ri->iri_csum_data = 0xffff;
		}
		}
		ri->iri_csum_flags = csum_flags;
		}

		/* VLAN information is in the last descriptor. */
		if (rxcd->vlan) {
		ri->iri_flags \|= M_VLANTAG;
		ri->iri_vtag = rxcd->vtag;
		}
		}

return (0);		return (0);
}		}

static void		static void
vmxnet3_isc_rxd_refill(void *vsc, if_rxd_update_t iru)		vmxnet3_isc_rxd_refill(void *vsc, if_rxd_update_t iru)
{		{
struct vmxnet3_softc *sc;		struct vmxnet3_softc *sc;
struct vmxnet3_rxqueue *rxq;		struct vmxnet3_rxqueue *rxq;
struct vmxnet3_rxring *rxr;		struct vmxnet3_rxring *rxr;
struct vmxnet3_rxdesc *rxd;		struct vmxnet3_rxdesc *rxd;
uint64_t *paddrs;		uint64_t *paddrs;
int count;		int count;
int len;		int len;
int pidx;		int idx;
int i;		int i;
uint8_t flid;		uint8_t flid;
uint8_t btype;		uint8_t btype;

count = iru->iru_count;		count = iru->iru_count;
len = iru->iru_buf_size;		len = iru->iru_buf_size;
pidx = iru->iru_pidx;
flid = iru->iru_flidx;		flid = iru->iru_flidx;
paddrs = iru->iru_paddrs;		paddrs = iru->iru_paddrs;

sc = vsc;		sc = vsc;
rxq = &sc->vmx_rxq[iru->iru_qsidx];		rxq = &sc->vmx_rxq[iru->iru_qsidx];
rxr = &rxq->vxrxq_cmd_ring[flid];		rxr = &rxq->vxrxq_cmd_ring[flid];
rxd = rxr->vxrxr_rxd;		rxd = rxr->vxrxr_rxd;

/*		/*
* Command ring 0 is filled with BTYPE_HEAD descriptors, and		* Command ring 0 is filled with BTYPE_HEAD descriptors, and
* command ring 1 is filled with BTYPE_BODY descriptors.		* command ring 1 is filled with BTYPE_BODY descriptors.
*/		*/
btype = (flid == 0) ? VMXNET3_BTYPE_HEAD : VMXNET3_BTYPE_BODY;		btype = (flid == 0) ? VMXNET3_BTYPE_HEAD : VMXNET3_BTYPE_BODY;
for (i = 0; i < count; i++) {		/*
rxd[pidx].addr = paddrs[i];		* The refill entries from iflib will advance monotonically,
rxd[pidx].len = len;		* but the refilled descriptors may not be contiguous due to
rxd[pidx].btype = btype;		* earlier skipping of descriptors by the device. The refill
rxd[pidx].gen = rxr->vxrxr_gen;		* entries from iflib need an entire state update, while the
		* descriptors previously skipped by the device only need to
		* have their generation numbers updated.
		*/
		idx = rxr->vxrxr_refill_start;
		i = 0;
		do {
		if (idx == iru->iru_idxs[i]) {
		rxd[idx].addr = paddrs[i];
		rxd[idx].len = len;
		rxd[idx].btype = btype;
		i++;
		} else
		rxr->vxrxr_desc_skips++;
		rxd[idx].gen = rxr->vxrxr_gen;

if (++pidx == rxr->vxrxr_ndesc) {		if (++idx == rxr->vxrxr_ndesc) {
pidx = 0;		idx = 0;
rxr->vxrxr_gen ^= 1;		rxr->vxrxr_gen ^= 1;
}		}
		} while (i != count);
		rxr->vxrxr_refill_start = idx;
}		}
}

static void		static void
vmxnet3_isc_rxd_flush(void *vsc, uint16_t rxqid, uint8_t flid, qidx_t pidx)		vmxnet3_isc_rxd_flush(void *vsc, uint16_t rxqid, uint8_t flid, qidx_t pidx)
{		{
struct vmxnet3_softc *sc;		struct vmxnet3_softc *sc;
struct vmxnet3_rxqueue *rxq;		struct vmxnet3_rxqueue *rxq;
struct vmxnet3_rxring *rxr;		struct vmxnet3_rxring *rxr;
bus_size_t r;		bus_size_t r;
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines	vmxnet3_rxinit(struct vmxnet3_softc sc, struct vmxnet3_rxqueue rxq)

/*		/*
* The descriptors will be populated with buffers during a		* The descriptors will be populated with buffers during a
* subsequent invocation of vmxnet3_isc_rxd_refill()		* subsequent invocation of vmxnet3_isc_rxd_refill()
*/		*/
for (i = 0; i < sc->vmx_sctx->isc_nrxqs - 1; i++) {		for (i = 0; i < sc->vmx_sctx->isc_nrxqs - 1; i++) {
rxr = &rxq->vxrxq_cmd_ring[i];		rxr = &rxq->vxrxq_cmd_ring[i];
rxr->vxrxr_gen = VMXNET3_INIT_GEN;		rxr->vxrxr_gen = VMXNET3_INIT_GEN;
		rxr->vxrxr_desc_skips = 0;
		rxr->vxrxr_refill_start = 0;
/*		/*
* iflib has zeroed out the descriptor array during the		* iflib has zeroed out the descriptor array during the
* prior attach or stop		* prior attach or stop
*/		*/
}		}

for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {		for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
rxr = &rxq->vxrxq_cmd_ring[i];		rxr = &rxq->vxrxq_cmd_ring[i];
rxr->vxrxr_gen = 0;		rxr->vxrxr_gen = 0;
		rxr->vxrxr_desc_skips = 0;
		rxr->vxrxr_refill_start = 0;
bzero(rxr->vxrxr_rxd,		bzero(rxr->vxrxr_rxd,
rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));		rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
}		}

rxc = &rxq->vxrxq_comp_ring;		rxc = &rxq->vxrxq_comp_ring;
rxc->vxcr_next = 0;		rxc->vxcr_next = 0;
rxc->vxcr_gen = VMXNET3_INIT_GEN;		rxc->vxcr_gen = VMXNET3_INIT_GEN;
		rxc->vxcr_zero_length = 0;
		rxc->vxcr_pkt_errors = 0;
/*		/*
* iflib has zeroed out the descriptor array during the prior attach		* iflib has zeroed out the descriptor array during the prior attach
* or stop		* or stop
*/		*/
}		}

static void		static void
vmxnet3_reinit_queues(struct vmxnet3_softc *sc)		vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
▲ Show 20 Lines • Show All 428 Lines • ▼ Show 20 Lines	for (i = 0; i < scctx->isc_nrxqsets; i++) {
node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,		node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,
"debug", CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "");		"debug", CTLFLAG_RD \| CTLFLAG_MPSAFE, NULL, "");
list = SYSCTL_CHILDREN(node);		list = SYSCTL_CHILDREN(node);

SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,
&rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");		&rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");
SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,
&rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");		&rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");
		SYSCTL_ADD_U64(ctx, list, OID_AUTO, "cmd0_desc_skips", CTLFLAG_RD,
		&rxq->vxrxq_cmd_ring[0].vxrxr_desc_skips, 0, "");
SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,
&rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");		&rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");
SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,
&rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");		&rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");
		SYSCTL_ADD_U64(ctx, list, OID_AUTO, "cmd1_desc_skips", CTLFLAG_RD,
		&rxq->vxrxq_cmd_ring[1].vxrxr_desc_skips, 0, "");
SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,		SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
&rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");		&rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");
SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,		SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
&rxq->vxrxq_comp_ring.vxcr_gen, 0, "");		&rxq->vxrxq_comp_ring.vxcr_gen, 0, "");
		SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_zero_length", CTLFLAG_RD,
		&rxq->vxrxq_comp_ring.vxcr_zero_length, 0, "");
		SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_pkt_errors", CTLFLAG_RD,
		&rxq->vxrxq_comp_ring.vxcr_pkt_errors, 0, "");
}		}
}		}

static void		static void
vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,		vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,
struct sysctl_ctx_list ctx, struct sysctl_oid_list child)		struct sysctl_ctx_list ctx, struct sysctl_oid_list child)
{		{
if_softc_ctx_t scctx;		if_softc_ctx_t scctx;
▲ Show 20 Lines • Show All 161 Lines • Show Last 20 Lines