D24598.id71298.diff
No OneTemporary
Actions

Size

26 KB

Referenced Files

None

Subscribers

None

D24598.id71298.diff
View Options

	Index: head/sys/dev/cxgbe/crypto/t4_kern_tls.c
	===================================================================
	--- head/sys/dev/cxgbe/crypto/t4_kern_tls.c
	+++ head/sys/dev/cxgbe/crypto/t4_kern_tls.c
	@@ -906,7 +906,7 @@

	MBUF_EXT_PGS_ASSERT(m_tls);
	ext_pgs = &m_tls->m_ext_pgs;
	- hdr = (void *)ext_pgs->m_epg_hdr;
	+ hdr = (void *)m_tls->m_epg_hdr;
	plen = ntohs(hdr->tls_length);

	/*
	@@ -962,7 +962,7 @@

	MBUF_EXT_PGS_ASSERT(m_tls);
	ext_pgs = &m_tls->m_ext_pgs;
	- hdr = (void *)ext_pgs->m_epg_hdr;
	+ hdr = (void *)m_tls->m_epg_hdr;
	plen = ntohs(hdr->tls_length);
	#ifdef INVARIANTS
	mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
	@@ -1040,7 +1040,7 @@
	return (wr_len);
	}

	- hdr = (void *)ext_pgs->m_epg_hdr;
	+ hdr = (void *)m_tls->m_epg_hdr;
	plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
	if (tlen < plen) {
	plen = tlen;
	@@ -1064,7 +1064,7 @@
	wr_len += roundup2(imm_len, 16);

	/* TLS record payload via DSGL. */
	- *nsegsp = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len + offset,
	+ *nsegsp = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len + offset,
	plen - (ext_pgs->hdr_len + offset));
	wr_len += ktls_sgl_size(*nsegsp);

	@@ -1543,7 +1543,7 @@
	(m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));

	/* Copy the subset of the TLS header requested. */
	- copy_to_txd(&txq->eq, (char *)ext_pgs->m_epg_hdr +
	+ copy_to_txd(&txq->eq, (char *)m_tls->m_epg_hdr +
	mtod(m_tls, vm_offset_t), &out, m_tls->m_len);
	txq->imm_wrs++;

	@@ -1604,7 +1604,7 @@
	/* Locate the TLS header. */
	MBUF_EXT_PGS_ASSERT(m_tls);
	ext_pgs = &m_tls->m_ext_pgs;
	- hdr = (void *)ext_pgs->m_epg_hdr;
	+ hdr = (void *)m_tls->m_epg_hdr;
	plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;

	/* Determine how much of the TLS record to send. */
	@@ -1799,7 +1799,7 @@

	/* Recalculate 'nsegs' if cached value is not available. */
	if (nsegs == 0)
	- nsegs = sglist_count_ext_pgs(ext_pgs, ext_pgs->hdr_len +
	+ nsegs = sglist_count_ext_pgs(m_tls, ext_pgs->hdr_len +
	offset, plen - (ext_pgs->hdr_len + offset));

	/* Calculate the size of the TLS work request. */
	@@ -2031,7 +2031,7 @@
	/* Populate the TLS header */
	out = (void *)(tx_data + 1);
	if (offset == 0) {
	- memcpy(out, ext_pgs->m_epg_hdr, ext_pgs->hdr_len);
	+ memcpy(out, m_tls->m_epg_hdr, ext_pgs->hdr_len);
	out += ext_pgs->hdr_len;
	}

	@@ -2067,7 +2067,7 @@

	/* SGL for record payload */
	sglist_reset(txq->gl);
	- if (sglist_append_ext_pgs(txq->gl, ext_pgs, ext_pgs->hdr_len + offset,
	+ if (sglist_append_ext_pgs(txq->gl, m_tls, ext_pgs->hdr_len + offset,
	plen - (ext_pgs->hdr_len + offset)) != 0) {
	#ifdef INVARIANTS
	panic("%s: failed to append sglist", __func__);
	Index: head/sys/dev/cxgbe/t4_sge.c
	===================================================================
	--- head/sys/dev/cxgbe/t4_sge.c
	+++ head/sys/dev/cxgbe/t4_sge.c
	@@ -2435,7 +2435,7 @@
	off = 0;
	len -= seglen;
	paddr = pmap_kextract(
	- (vm_offset_t)&ext_pgs->m_epg_hdr[segoff]);
	+ (vm_offset_t)&m->m_epg_hdr[segoff]);
	if (*nextaddr != paddr)
	nsegs++;
	*nextaddr = paddr + seglen;
	@@ -2454,7 +2454,7 @@
	off = 0;
	seglen = min(seglen, len);
	len -= seglen;
	- paddr = ext_pgs->m_epg_pa[i] + segoff;
	+ paddr = m->m_epg_pa[i] + segoff;
	if (*nextaddr != paddr)
	nsegs++;
	*nextaddr = paddr + seglen;
	@@ -2463,7 +2463,7 @@
	if (len != 0) {
	seglen = min(len, ext_pgs->trail_len - off);
	len -= seglen;
	- paddr = pmap_kextract((vm_offset_t)&ext_pgs->m_epg_trail[off]);
	+ paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]);
	if (*nextaddr != paddr)
	nsegs++;
	*nextaddr = paddr + seglen;
	Index: head/sys/dev/cxgbe/tom/t4_cpl_io.c
	===================================================================
	--- head/sys/dev/cxgbe/tom/t4_cpl_io.c
	+++ head/sys/dev/cxgbe/tom/t4_cpl_io.c
	@@ -1935,7 +1935,7 @@
	#endif

	for (int i = 0; i < ext_pgs->npgs; i++) {
	- pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
	+ pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
	vm_page_unwire(pg, PQ_ACTIVE);
	}

	@@ -2003,7 +2003,7 @@
	(npages - 2) * PAGE_SIZE;
	}
	for (i = 0; i < npages; i++)
	- ext_pgs->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
	+ m->m_epg_pa[i] = VM_PAGE_TO_PHYS(pgs[i]);

	m->m_len = mlen;
	m->m_ext.ext_size = npages * PAGE_SIZE;
	Index: head/sys/dev/cxgbe/tom/t4_tls.c
	===================================================================
	--- head/sys/dev/cxgbe/tom/t4_tls.c
	+++ head/sys/dev/cxgbe/tom/t4_tls.c
	@@ -1623,26 +1623,24 @@

	#ifdef KERN_TLS
	static int
	-count_ext_pgs_segs(struct mbuf_ext_pgs *ext_pgs,
	- struct mbuf_ext_pgs_data *ext_pgs_data)
	+count_ext_pgs_segs(struct mbuf *m)
	{
	vm_paddr_t nextpa;
	u_int i, nsegs;

	- MPASS(ext_pgs->npgs > 0);
	+ MPASS(m->m_ext_pgs.npgs > 0);
	nsegs = 1;
	- nextpa = ext_pgs_data->pa[0] + PAGE_SIZE;
	- for (i = 1; i < ext_pgs->npgs; i++) {
	- if (nextpa != ext_pgs_data->pa[i])
	+ nextpa = m->m_epg_pa[0] + PAGE_SIZE;
	+ for (i = 1; i < m->m_ext_pgs.npgs; i++) {
	+ if (nextpa != m->m_epg_pa[i])
	nsegs++;
	- nextpa = ext_pgs_data->pa[i] + PAGE_SIZE;
	+ nextpa = m->m_epg_pa[i] + PAGE_SIZE;
	}
	return (nsegs);
	}

	static void
	-write_ktlstx_sgl(void dst, struct mbuf_ext_pgs ext_pgs,
	- struct mbuf_ext_pgs_data *ext_pgs_data, int nsegs)
	+write_ktlstx_sgl(void dst, struct mbuf m, int nsegs)
	{
	struct ulptx_sgl *usgl = dst;
	vm_paddr_t pa;
	@@ -1655,15 +1653,15 @@
	V_ULPTX_NSGE(nsegs));

	/* Figure out the first S/G length. */
	- pa = ext_pgs_data->pa[0] + ext_pgs->first_pg_off;
	+ pa = m->m_epg_pa[0] + m->m_ext_pgs.first_pg_off;
	usgl->addr0 = htobe64(pa);
	- len = mbuf_ext_pg_len(ext_pgs, 0, ext_pgs->first_pg_off);
	+ len = mbuf_ext_pg_len(&m->m_ext_pgs, 0, m->m_ext_pgs.first_pg_off);
	pa += len;
	- for (i = 1; i < ext_pgs->npgs; i++) {
	- if (ext_pgs_data->pa[i] != pa)
	+ for (i = 1; i < m->m_ext_pgs.npgs; i++) {
	+ if (m->m_epg_pa[i] != pa)
	break;
	- len += mbuf_ext_pg_len(ext_pgs, i, 0);
	- pa += mbuf_ext_pg_len(ext_pgs, i, 0);
	+ len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
	+ pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
	}
	usgl->len0 = htobe32(len);
	#ifdef INVARIANTS
	@@ -1671,21 +1669,21 @@
	#endif

	j = -1;
	- for (; i < ext_pgs->npgs; i++) {
	- if (j == -1 \|\| ext_pgs_data->pa[i] != pa) {
	+ for (; i < m->m_ext_pgs.npgs; i++) {
	+ if (j == -1 \|\| m->m_epg_pa[i] != pa) {
	if (j >= 0)
	usgl->sge[j / 2].len[j & 1] = htobe32(len);
	j++;
	#ifdef INVARIANTS
	nsegs--;
	#endif
	- pa = ext_pgs_data->pa[i];
	+ pa = m->m_epg_pa[i];
	usgl->sge[j / 2].addr[j & 1] = htobe64(pa);
	- len = mbuf_ext_pg_len(ext_pgs, i, 0);
	+ len = mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
	pa += len;
	} else {
	- len += mbuf_ext_pg_len(ext_pgs, i, 0);
	- pa += mbuf_ext_pg_len(ext_pgs, i, 0);
	+ len += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
	+ pa += mbuf_ext_pg_len(&m->m_ext_pgs, i, 0);
	}
	}
	if (j >= 0) {
	@@ -1694,8 +1692,7 @@
	if ((j & 1) == 0)
	usgl->sge[j / 2].len[1] = htobe32(0);
	}
	- KASSERT(nsegs == 0, ("%s: nsegs %d, ext_pgs %p", __func__, nsegs,
	- ext_pgs));
	+ KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m));
	}

	/*
	@@ -1813,8 +1810,7 @@
	wr_len += AES_BLOCK_LEN;

	/* Account for SGL in work request length. */
	- nsegs = count_ext_pgs_segs(&m->m_ext_pgs,
	- &m->m_ext.ext_pgs);
	+ nsegs = count_ext_pgs_segs(m);
	wr_len += sizeof(struct ulptx_sgl) +
	((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;

	@@ -1892,8 +1888,7 @@
	memcpy(buf, thdr + 1, toep->tls.iv_len);
	buf += AES_BLOCK_LEN;

	- write_ktlstx_sgl(buf, &m->m_ext_pgs, &m->m_ext.ext_pgs,
	- nsegs);
	+ write_ktlstx_sgl(buf, m, nsegs);

	KASSERT(toep->tx_credits >= credits,
	("%s: not enough credits", __func__));
	Index: head/sys/kern/kern_mbuf.c
	===================================================================
	--- head/sys/kern/kern_mbuf.c
	+++ head/sys/kern/kern_mbuf.c
	@@ -311,9 +311,6 @@
	/* Ensure that MSIZE is a power of 2. */
	CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);

	-_Static_assert(offsetof(struct mbuf, m_ext) ==
	- offsetof(struct mbuf, m_ext_pgs.m_ext),
	- "m_ext offset mismatch between mbuf and ext_pgs");
	_Static_assert(sizeof(struct mbuf) <= MSIZE,
	"size of mbuf exceeds MSIZE");
	/*
	@@ -984,7 +981,7 @@
	goto fail;
	m_new->m_len = seglen;
	prev = top = m_new;
	- memcpy(mtod(m_new, void *), &ext_pgs->m_epg_hdr[segoff],
	+ memcpy(mtod(m_new, void *), &m->m_epg_hdr[segoff],
	seglen);
	}
	}
	@@ -1002,7 +999,7 @@
	seglen = min(seglen, len);
	len -= seglen;

	- pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
	+ pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
	m_new = m_get(M_NOWAIT, MT_DATA);
	if (m_new == NULL)
	goto fail;
	@@ -1036,7 +1033,7 @@
	else
	prev->m_next = m_new;
	m_new->m_len = len;
	- memcpy(mtod(m_new, void *), &ext_pgs->m_epg_trail[off], len);
	+ memcpy(mtod(m_new, void *), &m->m_epg_trail[off], len);
	}

	if (ref_inc != 0) {
	@@ -1154,8 +1151,9 @@

	#ifdef INVARIANT_SUPPORT
	void
	-mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
	+mb_ext_pgs_check(struct mbuf *m)
	{
	+ struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;

	/*
	* NB: This expects a non-empty buffer (npgs > 0 and
	@@ -1163,7 +1161,7 @@
	*/
	KASSERT(ext_pgs->npgs > 0,
	("ext_pgs with no valid pages: %p", ext_pgs));
	- KASSERT(ext_pgs->npgs <= nitems(ext_pgs->m_epg_pa),
	+ KASSERT(ext_pgs->npgs <= nitems(m->m_epg_pa),
	("ext_pgs with too many pages: %p", ext_pgs));
	KASSERT(ext_pgs->nrdy <= ext_pgs->npgs,
	("ext_pgs with too many ready pages: %p", ext_pgs));
	@@ -1178,9 +1176,9 @@
	PAGE_SIZE, ("ext_pgs with single page too large: %p",
	ext_pgs));
	}
	- KASSERT(ext_pgs->hdr_len <= sizeof(ext_pgs->m_epg_hdr),
	+ KASSERT(ext_pgs->hdr_len <= sizeof(m->m_epg_hdr),
	("ext_pgs with too large header length: %p", ext_pgs));
	- KASSERT(ext_pgs->trail_len <= sizeof(ext_pgs->m_epg_trail),
	+ KASSERT(ext_pgs->trail_len <= sizeof(m->m_epg_trail),
	("ext_pgs with too large header length: %p", ext_pgs));
	}
	#endif
	Index: head/sys/kern/kern_sendfile.c
	===================================================================
	--- head/sys/kern/kern_sendfile.c
	+++ head/sys/kern/kern_sendfile.c
	@@ -203,7 +203,7 @@
	for (i = 0; i < ext_pgs->npgs; i++) {
	if (cache_last && i == ext_pgs->npgs - 1)
	flags = 0;
	- pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
	+ pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
	vm_page_release(pg, flags);
	}

	@@ -1046,11 +1046,11 @@
	ext_pgs->nrdy++;
	}

	- ext_pgs->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
	+ m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga);
	ext_pgs->npgs++;
	xfs = xfsize(i, npages, off, space);
	ext_pgs->last_pg_len = xfs;
	- MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs);
	+ MBUF_EXT_PGS_ASSERT_SANITY(m0);
	mtail->m_len += xfs;
	mtail->m_ext.ext_size += PAGE_SIZE;
	continue;
	Index: head/sys/kern/subr_bus_dma.c
	===================================================================
	--- head/sys/kern/subr_bus_dma.c
	+++ head/sys/kern/subr_bus_dma.c
	@@ -141,7 +141,7 @@
	off = 0;
	len -= seglen;
	error = _bus_dmamap_load_buffer(dmat, map,
	- &ext_pgs->m_epg_hdr[segoff], seglen, kernel_pmap,
	+ &m->m_epg_hdr[segoff], seglen, kernel_pmap,
	flags, segs, nsegs);
	}
	}
	@@ -159,7 +159,7 @@
	seglen = min(seglen, len);
	len -= seglen;
	error = _bus_dmamap_load_phys(dmat, map,
	- ext_pgs->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
	+ m->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs);
	pgoff = 0;
	};
	if (len != 0 && error == 0) {
	@@ -167,7 +167,7 @@
	("off + len > trail (%d + %d > %d)", off, len,
	ext_pgs->trail_len));
	error = _bus_dmamap_load_buffer(dmat, map,
	- &ext_pgs->m_epg_trail[off], len, kernel_pmap, flags, segs,
	+ &m->m_epg_trail[off], len, kernel_pmap, flags, segs,
	nsegs);
	}
	return (error);
	Index: head/sys/kern/subr_sglist.c
	===================================================================
	--- head/sys/kern/subr_sglist.c
	+++ head/sys/kern/subr_sglist.c
	@@ -223,8 +223,9 @@
	* describe an EXT_PGS buffer.
	*/
	int
	-sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off, size_t len)
	+sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len)
	{
	+ struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
	vm_paddr_t nextaddr, paddr;
	size_t seglen, segoff;
	int i, nsegs, pglen, pgoff;
	@@ -242,7 +243,7 @@
	seglen = MIN(seglen, len);
	off = 0;
	len -= seglen;
	- nsegs += sglist_count(&ext_pgs->m_epg_hdr[segoff],
	+ nsegs += sglist_count(&m->m_epg_hdr[segoff],
	seglen);
	}
	}
	@@ -260,7 +261,7 @@
	off = 0;
	seglen = MIN(seglen, len);
	len -= seglen;
	- paddr = ext_pgs->m_epg_pa[i] + segoff;
	+ paddr = m->m_epg_pa[i] + segoff;
	if (paddr != nextaddr)
	nsegs++;
	nextaddr = paddr + seglen;
	@@ -269,7 +270,7 @@
	if (len != 0) {
	seglen = MIN(len, ext_pgs->trail_len - off);
	len -= seglen;
	- nsegs += sglist_count(&ext_pgs->m_epg_trail[off], seglen);
	+ nsegs += sglist_count(&m->m_epg_trail[off], seglen);
	}
	KASSERT(len == 0, ("len != 0"));
	return (nsegs);
	@@ -284,8 +285,7 @@
	{

	MBUF_EXT_PGS_ASSERT(m);
	- return (sglist_count_ext_pgs(&m->m_ext_pgs, mtod(m, vm_offset_t),
	- m->m_len));
	+ return (sglist_count_ext_pgs(m, mtod(m, vm_offset_t), m->m_len));
	}

	/*
	@@ -395,9 +395,9 @@
	* fails with EFBIG.
	*/
	int
	-sglist_append_ext_pgs(struct sglist sg, struct mbuf_ext_pgs ext_pgs,
	- size_t off, size_t len)
	+sglist_append_ext_pgs(struct sglist sg, struct mbuf m, size_t off, size_t len)
	{
	+ struct mbuf_ext_pgs *ext_pgs = &m->m_ext_pgs;
	size_t seglen, segoff;
	vm_paddr_t paddr;
	int error, i, pglen, pgoff;
	@@ -413,7 +413,7 @@
	off = 0;
	len -= seglen;
	error = sglist_append(sg,
	- &ext_pgs->m_epg_hdr[segoff], seglen);
	+ &m->m_epg_hdr[segoff], seglen);
	}
	}
	pgoff = ext_pgs->first_pg_off;
	@@ -429,7 +429,7 @@
	off = 0;
	seglen = MIN(seglen, len);
	len -= seglen;
	- paddr = ext_pgs->m_epg_pa[i] + segoff;
	+ paddr = m->m_epg_pa[i] + segoff;
	error = sglist_append_phys(sg, paddr, seglen);
	pgoff = 0;
	};
	@@ -437,7 +437,7 @@
	seglen = MIN(len, ext_pgs->trail_len - off);
	len -= seglen;
	error = sglist_append(sg,
	- &ext_pgs->m_epg_trail[off], seglen);
	+ &m->m_epg_trail[off], seglen);
	}
	if (error == 0)
	KASSERT(len == 0, ("len != 0"));
	@@ -455,8 +455,7 @@

	/* for now, all unmapped mbufs are assumed to be EXT_PGS */
	MBUF_EXT_PGS_ASSERT(m);
	- return (sglist_append_ext_pgs(sg, &m->m_ext_pgs,
	- mtod(m, vm_offset_t), m->m_len));
	+ return (sglist_append_ext_pgs(sg, m, mtod(m, vm_offset_t), m->m_len));
	}

	/*
	Index: head/sys/kern/uipc_ktls.c
	===================================================================
	--- head/sys/kern/uipc_ktls.c
	+++ head/sys/kern/uipc_ktls.c
	@@ -1374,7 +1374,7 @@
	m->m_len += pgs->hdr_len + pgs->trail_len;

	/* Populate the TLS header. */
	- tlshdr = (void *)pgs->m_epg_hdr;
	+ tlshdr = (void *)m->m_epg_hdr;
	tlshdr->tls_vmajor = tls->params.tls_vmajor;

	/*
	@@ -1387,7 +1387,7 @@
	tlshdr->tls_type = TLS_RLTYPE_APP;
	/* save the real record type for later */
	pgs->record_type = record_type;
	- pgs->m_epg_trail[0] = record_type;
	+ m->m_epg_trail[0] = record_type;
	} else {
	tlshdr->tls_vminor = tls->params.tls_vminor;
	tlshdr->tls_type = record_type;
	@@ -1552,7 +1552,7 @@
	len = mbuf_ext_pg_len(pgs, i, off);
	src_iov[i].iov_len = len;
	src_iov[i].iov_base =
	- (char )(void )PHYS_TO_DMAP(pgs->m_epg_pa[i]) +
	+ (char )(void )PHYS_TO_DMAP(m->m_epg_pa[i]) +
	off;

	if (is_anon) {
	@@ -1576,8 +1576,8 @@
	npages += i;

	error = (*tls->sw_encrypt)(tls,
	- (const struct tls_record_layer *)pgs->m_epg_hdr,
	- pgs->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
	+ (const struct tls_record_layer *)m->m_epg_hdr,
	+ m->m_epg_trail, src_iov, dst_iov, i, pgs->seqno,
	pgs->record_type);
	if (error) {
	counter_u64_add(ktls_offload_failed_crypto, 1);
	@@ -1595,7 +1595,7 @@

	/* Replace them with the new pages. */
	for (i = 0; i < pgs->npgs; i++)
	- pgs->m_epg_pa[i] = parray[i];
	+ m->m_epg_pa[i] = parray[i];

	/* Use the basic free routine. */
	m->m_ext.ext_free = mb_free_mext_pgs;
	Index: head/sys/kern/uipc_mbuf.c
	===================================================================
	--- head/sys/kern/uipc_mbuf.c
	+++ head/sys/kern/uipc_mbuf.c
	@@ -163,11 +163,11 @@
	#if defined(__LP64__)
	CTASSERT(offsetof(struct mbuf, m_dat) == 32);
	CTASSERT(sizeof(struct pkthdr) == 56);
	-CTASSERT(sizeof(struct m_ext) == 168);
	+CTASSERT(sizeof(struct m_ext) == 160);
	#else
	CTASSERT(offsetof(struct mbuf, m_dat) == 24);
	CTASSERT(sizeof(struct pkthdr) == 48);
	-CTASSERT(sizeof(struct m_ext) == 184);
	+CTASSERT(sizeof(struct m_ext) == 180);
	#endif

	/*
	@@ -195,19 +195,30 @@
	KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n));

	/*
	- * Cache access optimization. For most kinds of external
	- * storage we don't need full copy of m_ext, since the
	- * holder of the 'ext_count' is responsible to carry the
	- * free routine and its arguments. Exclusion is EXT_EXTREF,
	- * where 'ext_cnt' doesn't point into mbuf at all.
	+ * Cache access optimization.
	+ *
	+ * o Regular M_EXT storage doesn't need full copy of m_ext, since
	+ * the holder of the 'ext_count' is responsible to carry the free
	+ * routine and its arguments.
	+ * o EXT_PGS data is split between main part of mbuf and m_ext, the
	+ * main part is copied in full, the m_ext part is similar to M_EXT.
	+ * o EXT_EXTREF, where 'ext_cnt' doesn't point into mbuf at all, is
	+ * special - it needs full copy of m_ext into each mbuf, since any
	+ * copy could end up as the last to free.
	*/
	- if (m->m_ext.ext_type == EXT_EXTREF)
	- bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext));
	- else if (m->m_ext.ext_type == EXT_PGS)
	+ switch (m->m_ext.ext_type) {
	+ case EXT_PGS:
	+ bcopy(&m->m_ext, &n->m_ext, m_epg_copylen);
	bcopy(&m->m_ext_pgs, &n->m_ext_pgs,
	sizeof(struct mbuf_ext_pgs));
	- else
	+ break;
	+ case EXT_EXTREF:
	+ bcopy(&m->m_ext, &n->m_ext, sizeof(struct m_ext));
	+ break;
	+ default:
	bcopy(&m->m_ext, &n->m_ext, m_ext_copylen);
	+ }
	+
	n->m_flags \|= M_EXT;
	n->m_flags \|= m->m_flags & (M_RDONLY \| M_NOMAP);

	@@ -1623,7 +1634,7 @@
	MBUF_EXT_PGS_ASSERT(m);
	ext_pgs = &m->m_ext_pgs;
	for (int i = 0; i < ext_pgs->npgs; i++) {
	- pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
	+ pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
	vm_page_unwire_noq(pg);
	vm_page_free(pg);
	}
	@@ -1681,11 +1692,11 @@
	}
	}
	pg_array[i]->flags &= ~PG_ZERO;
	- pgs->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
	+ mb->m_epg_pa[i] = VM_PAGE_TO_PHYS(pg_array[i]);
	pgs->npgs++;
	}
	pgs->last_pg_len = length - PAGE_SIZE * (pgs->npgs - 1);
	- MBUF_EXT_PGS_ASSERT_SANITY(pgs);
	+ MBUF_EXT_PGS_ASSERT_SANITY(mb);
	total -= length;
	error = uiomove_fromphys(pg_array, 0, length, uio);
	if (error != 0)
	@@ -1788,7 +1799,8 @@
	seglen = min(seglen, len);
	off = 0;
	len -= seglen;
	- error = uiomove(&ext_pgs->m_epg_hdr[segoff], seglen, uio);
	+ error = uiomove(__DECONST(void *,
	+ &m->m_epg_hdr[segoff]), seglen, uio);
	}
	}
	pgoff = ext_pgs->first_pg_off;
	@@ -1804,7 +1816,7 @@
	off = 0;
	seglen = min(seglen, len);
	len -= seglen;
	- pg = PHYS_TO_VM_PAGE(ext_pgs->m_epg_pa[i]);
	+ pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
	error = uiomove_fromphys(&pg, segoff, seglen, uio);
	pgoff = 0;
	};
	@@ -1812,7 +1824,8 @@
	KASSERT((off + len) <= ext_pgs->trail_len,
	("off + len > trail (%d + %d > %d, m_off = %d)", off, len,
	ext_pgs->trail_len, m_off));
	- error = uiomove(&ext_pgs->m_epg_trail[off], len, uio);
	+ error = uiomove(__DECONST(void *, &m->m_epg_trail[off]),
	+ len, uio);
	}
	return (error);
	}
	Index: head/sys/sys/mbuf.h
	===================================================================
	--- head/sys/sys/mbuf.h
	+++ head/sys/sys/mbuf.h
	@@ -231,13 +231,6 @@

	#define MBUF_PEXT_FLAG_ANON 1 /* Data can be encrypted in place. */

	-
	-struct mbuf_ext_pgs_data {
	- vm_paddr_t pa[MBUF_PEXT_MAX_PGS]; /* phys addrs of pgs */
	- char trail[MBUF_PEXT_TRAIL_LEN]; /* TLS trailer */
	- char hdr[MBUF_PEXT_HDR_LEN]; /* TLS header */
	-};
	-
	struct ktls_session;
	struct socket;

	@@ -266,49 +259,49 @@
	uint32_t ext_size; /* size of buffer, for ext_free */
	uint32_t ext_type:8, /* type of external storage */
	ext_flags:24; /* external storage mbuf flags */
	- char ext_buf; / start of buffer */
	- /*
	- * Fields below store the free context for the external storage.
	- * They are valid only in the refcount carrying mbuf, the one with
	- * EXT_FLAG_EMBREF flag, with exclusion for EXT_EXTREF type, where
	- * the free context is copied into all mbufs that use same external
	- * storage.
	- */
	-#define m_ext_copylen offsetof(struct m_ext, ext_free)
	- m_ext_free_t ext_free; / free routine if not the usual */
	- void ext_arg1; / optional argument pointer */
	union {
	- void ext_arg2; / optional argument pointer */
	- struct mbuf_ext_pgs_data ext_pgs;
	+ struct {
	+ /*
	+ * Regular M_EXT mbuf:
	+ * o ext_buf always points to the external buffer.
	+ * o ext_free (below) and two optional arguments
	+ * ext_arg1 and ext_arg2 store the free context for
	+ * the external storage. They are set only in the
	+ * refcount carrying mbuf, the one with
	+ * EXT_FLAG_EMBREF flag, with exclusion for
	+ * EXT_EXTREF type, where the free context is copied
	+ * into all mbufs that use same external storage.
	+ */
	+ char ext_buf; / start of buffer */
	+#define m_ext_copylen offsetof(struct m_ext, ext_arg2)
	+ void *ext_arg2;
	+ };
	+ struct {
	+ /*
	+ * Multi-page M_EXTPG mbuf:
	+ * o extpg_pa - page vector.
	+ * o extpg_trail and extpg_hdr - TLS trailer and
	+ * header.
	+ * Uses ext_free and may also use ext_arg1.
	+ */
	+ vm_paddr_t extpg_pa[MBUF_PEXT_MAX_PGS];
	+ char extpg_trail[MBUF_PEXT_TRAIL_LEN];
	+ char extpg_hdr[MBUF_PEXT_HDR_LEN];
	+ /* Pretend these 3 fields are part of mbuf itself. */
	+#define m_epg_pa m_ext.extpg_pa
	+#define m_epg_trail m_ext.extpg_trail
	+#define m_epg_hdr m_ext.extpg_hdr
	+#define m_epg_copylen offsetof(struct m_ext, ext_free)
	+ };
	};
	+ /*
	+ * Free method and optional argument pointer, both
	+ * used by M_EXT and M_EXTPG.
	+ */
	+ m_ext_free_t *ext_free;
	+ void *ext_arg1;
	};

	-struct mbuf_ext_pgs {
	- uint8_t npgs; /* Number of attached pages */
	- uint8_t nrdy; /* Pages with I/O pending */
	- uint8_t hdr_len; /* TLS header length */
	- uint8_t trail_len; /* TLS trailer length */
	- uint16_t first_pg_off; /* Offset into 1st page */
	- uint16_t last_pg_len; /* Length of last page */
	- uint8_t flags; /* Flags */
	- uint8_t record_type;
	- uint8_t spare[2];
	- int enc_cnt;
	- struct ktls_session tls; / TLS session */
	- struct socket *so;
	- uint64_t seqno;
	- struct mbuf *mbuf;
	- STAILQ_ENTRY(mbuf_ext_pgs) stailq;
	-#if !defined(__LP64__)
	- uint8_t pad[8]; /* pad to size of pkthdr */
	-#endif
	- struct m_ext m_ext;
	-};
	-
	-#define m_epg_hdr m_ext.ext_pgs.hdr
	-#define m_epg_trail m_ext.ext_pgs.trail
	-#define m_epg_pa m_ext.ext_pgs.pa
	-
	/*
	* The core of the mbuf object along with some shortcut defines for practical
	* purposes.
	@@ -347,15 +340,48 @@
	* order to support future work on variable-size mbufs.
	*/
	union {
	- union {
	- struct {
	- struct pkthdr m_pkthdr; /* M_PKTHDR set */
	- union {
	- struct m_ext m_ext; /* M_EXT set */
	- char m_pktdat[0];
	- };
	+ struct {
	+ union {
	+ /* M_PKTHDR set. */
	+ struct pkthdr m_pkthdr;
	+
	+ /* M_EXTPG set.
	+ * Multi-page M_EXTPG mbuf has its meta data
	+ * split between the mbuf_ext_pgs structure
	+ * and m_ext. It carries vector of pages,
	+ * optional header and trailer char vectors
	+ * and pointers to socket/TLS data.
	+ */
	+ struct mbuf_ext_pgs {
	+ /* Overall count of pages and count of
	+ * pages with I/O pending. */
	+ uint8_t npgs;
	+ uint8_t nrdy;
	+ /* TLS header and trailer lengths.
	+ * The data itself resides in m_ext. */
	+ uint8_t hdr_len;
	+ uint8_t trail_len;
	+ /* Offset into 1st page and lenght of
	+ * data in the last page. */
	+ uint16_t first_pg_off;
	+ uint16_t last_pg_len;
	+ uint8_t flags;
	+ uint8_t record_type;
	+ uint8_t spare[2];
	+ int enc_cnt;
	+ struct ktls_session *tls;
	+ struct socket *so;
	+ uint64_t seqno;
	+ struct mbuf *mbuf;
	+ STAILQ_ENTRY(mbuf_ext_pgs) stailq;
	+ } m_ext_pgs;
	};
	- struct mbuf_ext_pgs m_ext_pgs;
	+ union {
	+ /* M_EXT or M_EXTPG set. */
	+ struct m_ext m_ext;
	+ /* M_PKTHDR set, neither M_EXT nor M_EXTPG. */
	+ char m_pktdat[0];
	+ };
	};
	char m_dat[0]; /* !M_PKTHDR, !M_EXT */
	};
	@@ -375,12 +401,12 @@
	}

	#ifdef INVARIANT_SUPPORT
	-void mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs);
	+void mb_ext_pgs_check(struct mbuf *m);
	#endif
	#ifdef INVARIANTS
	-#define MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs) mb_ext_pgs_check((ext_pgs))
	+#define MBUF_EXT_PGS_ASSERT_SANITY(m) mb_ext_pgs_check((m))
	#else
	-#define MBUF_EXT_PGS_ASSERT_SANITY(ext_pgs)
	+#define MBUF_EXT_PGS_ASSERT_SANITY(m)
	#endif
	#endif

	Index: head/sys/sys/sglist.h
	===================================================================
	--- head/sys/sys/sglist.h
	+++ head/sys/sys/sglist.h
	@@ -57,7 +57,6 @@

	struct bio;
	struct mbuf;
	-struct mbuf_ext_pgs;
	struct uio;

	static __inline void
	@@ -88,8 +87,8 @@
	struct sglist *sglist_alloc(int nsegs, int mflags);
	int sglist_append(struct sglist sg, void buf, size_t len);
	int sglist_append_bio(struct sglist sg, struct bio bp);
	-int sglist_append_ext_pgs(struct sglist sg, struct mbuf_ext_pgs ext_pgs,
	- size_t off, size_t len);
	+int sglist_append_ext_pgs(struct sglist sg, struct mbuf m, size_t off,
	+ size_t len);
	int sglist_append_mb_ext_pgs(struct sglist sg, struct mbuf m);
	int sglist_append_mbuf(struct sglist sg, struct mbuf m0);
	int sglist_append_phys(struct sglist *sg, vm_paddr_t paddr,
	@@ -105,8 +104,7 @@
	struct sglist sglist_clone(struct sglist sg, int mflags);
	int sglist_consume_uio(struct sglist sg, struct uio uio, size_t resid);
	int sglist_count(void *buf, size_t len);
	-int sglist_count_ext_pgs(struct mbuf_ext_pgs *ext_pgs, size_t off,
	- size_t len);
	+int sglist_count_ext_pgs(struct mbuf *m, size_t off, size_t len);
	int sglist_count_mb_ext_pgs(struct mbuf *m);
	int sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len);
	void sglist_free(struct sglist *sg);

File Metadata

Mime Type: text/plain
Expires: Mon, Feb 9, 4:07 AM (13 h, 40 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 28534549
Default Alt Text: D24598.id71298.diff (26 KB)

D24598.id71298.diffNo OneTemporaryActions

D24598.id71298.diffView Options

File Metadata

Event Timeline

D24598.id71298.diff
No OneTemporary
Actions

D24598.id71298.diff
View Options