diff --git a/share/man/man9/crypto_buffer.9 b/share/man/man9/crypto_buffer.9 --- a/share/man/man9/crypto_buffer.9 +++ b/share/man/man9/crypto_buffer.9 @@ -30,7 +30,7 @@ .\" .\" $FreeBSD$ .\" -.Dd September 24, 2020 +.Dd May 25, 2021 .Dt CRYPTO_BUFFER 9 .Os .Sh NAME @@ -195,7 +195,10 @@ A scatter/gather list of kernel buffers as described in .Xr uio 9 . .It Dv CRYPTO_BUF_MBUF -A network memory buffer as described in +A chain of network memory buffers as described in +.Xr mbuf 9 . +.It Dv CRYPTO_BUF_SINGLE_MBUF +A single network memory buffer as described in .Xr mbuf 9 . .It Dv CRYPTO_BUF_VMPAGE A scatter/gather list of @@ -220,7 +223,9 @@ A pointer to a .Vt struct mbuf for -.Dv CRYPTO_BUF_MBUF . +.Dv CRYPTO_BUF_MBUF +and +.Dv CRYPTO_BUF_SINGLE_MBUF . .It Fa cb_uio A pointer to a .Vt struct uio diff --git a/sys/crypto/ccp/ccp.c b/sys/crypto/ccp/ccp.c --- a/sys/crypto/ccp/ccp.c +++ b/sys/crypto/ccp/ccp.c @@ -101,6 +101,9 @@ case CRYPTO_BUF_MBUF: error = sglist_append_mbuf(sg, cb->cb_mbuf); break; + case CRYPTO_BUF_SINGLE_MBUF: + error = sglist_append_single_mbuf(sg, cb->cb_mbuf); + break; case CRYPTO_BUF_UIO: error = sglist_append_uio(sg, cb->cb_uio); break; diff --git a/sys/dev/cxgbe/crypto/t4_crypto.c b/sys/dev/cxgbe/crypto/t4_crypto.c --- a/sys/dev/cxgbe/crypto/t4_crypto.c +++ b/sys/dev/cxgbe/crypto/t4_crypto.c @@ -271,6 +271,9 @@ case CRYPTO_BUF_MBUF: error = sglist_append_mbuf(sg, cb->cb_mbuf); break; + case CRYPTO_BUF_SINGLE_MBUF: + error = sglist_append_single_mbuf(sg, cb->cb_mbuf); + break; case CRYPTO_BUF_UIO: error = sglist_append_uio(sg, cb->cb_uio); break; diff --git a/sys/dev/sec/sec.c b/sys/dev/sec/sec.c --- a/sys/dev/sec/sec.c +++ b/sys/dev/sec/sec.c @@ -850,6 +850,9 @@ case CRYPTO_BUF_MBUF: size = m_length(crp->crp_buf.cb_mbuf, NULL); break; + case CRYPTO_BUF_SINGLE_MBUF: + size = crp->crp_buf.cb_mbuf->m_len; + break; case CRYPTO_BUF_VMPAGE: size = PAGE_SIZE - crp->crp_buf.cb_vm_page_offset; break; diff --git a/sys/kern/subr_bus_dma.c b/sys/kern/subr_bus_dma.c --- a/sys/kern/subr_bus_dma.c +++ b/sys/kern/subr_bus_dma.c @@ -172,6 +172,27 @@ return (error); } +/* + * Load a single mbuf. + */ +static int +_bus_dmamap_load_single_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, + struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags) +{ + int error; + + error = 0; + if ((m->m_flags & M_EXTPG) != 0) + error = _bus_dmamap_load_mbuf_epg(dmat, map, m, segs, nsegs, + flags); + else + error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len, + kernel_pmap, flags | BUS_DMA_LOAD_MBUF, segs, nsegs); + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, flags, error, *nsegs); + return (error); +} + /* * Load an mbuf chain. */ @@ -658,6 +679,10 @@ error = _bus_dmamap_load_mbuf_sg(dmat, map, cb->cb_mbuf, NULL, &nsegs, flags); break; + case CRYPTO_BUF_SINGLE_MBUF: + error = _bus_dmamap_load_single_mbuf(dmat, map, cb->cb_mbuf, + NULL, &nsegs, flags); + break; case CRYPTO_BUF_UIO: error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs, flags); diff --git a/sys/mips/nlm/dev/sec/nlmsec.c b/sys/mips/nlm/dev/sec/nlmsec.c --- a/sys/mips/nlm/dev/sec/nlmsec.c +++ b/sys/mips/nlm/dev/sec/nlmsec.c @@ -468,12 +468,15 @@ switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: { struct mbuf *m = NULL; m = crp->crp_buf.cb_mbuf; while (m != NULL) { *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len); + if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF) + break; m = m->m_next; } break; diff --git a/sys/mips/nlm/dev/sec/nlmseclib.c b/sys/mips/nlm/dev/sec/nlmseclib.c --- a/sys/mips/nlm/dev/sec/nlmseclib.c +++ b/sys/mips/nlm/dev/sec/nlmseclib.c @@ -112,6 +112,7 @@ switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: { struct mbuf *m = NULL; @@ -123,6 +124,8 @@ dstseg = nlm_crypto_fill_dst_seg(cmd->paramp, dstseg, mtod(m,caddr_t), m->m_len); } + if (crp->crp_buf.cb_type == CRYPTO_BUF_SINGLE_MBUF) + break; m = m->m_next; } break; diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c --- a/sys/opencrypto/criov.c +++ b/sys/opencrypto/criov.c @@ -344,6 +344,7 @@ cc->cc_buf_len = cb->cb_buf_len; break; case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: cc->cc_mbuf = cb->cb_mbuf; break; case CRYPTO_BUF_VMPAGE: @@ -389,6 +390,10 @@ break; } break; + case CRYPTO_BUF_SINGLE_MBUF: + MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + amount); + cc->cc_offset += amount; + break; case CRYPTO_BUF_VMPAGE: for (;;) { SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage, @@ -436,6 +441,7 @@ case CRYPTO_BUF_CONTIG: return (cc->cc_buf); case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: if (cc->cc_mbuf == NULL) return (NULL); if (cc->cc_mbuf->m_flags & M_EXTPG) @@ -463,6 +469,7 @@ case CRYPTO_BUF_VMPAGE: return (PAGE_SIZE - cc->cc_offset); case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: if (cc->cc_mbuf == NULL) return (0); if (cc->cc_mbuf->m_flags & M_EXTPG) @@ -516,6 +523,11 @@ break; } break; + case CRYPTO_BUF_SINGLE_MBUF: + MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size); + m_copyback(cc->cc_mbuf, cc->cc_offset, size, src); + cc->cc_offset += size; + break; case CRYPTO_BUF_VMPAGE: for (;;) { dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( @@ -599,6 +611,11 @@ break; } break; + case CRYPTO_BUF_SINGLE_MBUF: + MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size); + m_copydata(cc->cc_mbuf, cc->cc_offset, size, dst); + cc->cc_offset += size; + break; case CRYPTO_BUF_VMPAGE: for (;;) { src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( @@ -699,6 +716,7 @@ cb = &crp->crp_buf; switch (cb->cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: m_copyback(cb->cb_mbuf, off, size, src); break; #if CRYPTO_MAY_HAVE_VMPAGE @@ -731,6 +749,7 @@ switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: m_copydata(crp->crp_buf.cb_mbuf, off, size, dst); break; #if CRYPTO_MAY_HAVE_VMPAGE @@ -765,6 +784,7 @@ switch (cb->cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: error = m_apply(cb->cb_mbuf, off, len, (int (*)(void *, void *, u_int))f, arg); break; @@ -843,6 +863,7 @@ switch (cb->cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: return (m_contiguous_subsegment(cb->cb_mbuf, skip, len)); case CRYPTO_BUF_UIO: return (cuio_contiguous_segment(cb->cb_uio, skip, len)); diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -1155,6 +1155,8 @@ if (cb->cb_mbuf->m_flags & M_PKTHDR) return (cb->cb_mbuf->m_pkthdr.len); return (m_length(cb->cb_mbuf, NULL)); + case CRYPTO_BUF_SINGLE_MBUF: + return (cb->cb_mbuf->m_len); case CRYPTO_BUF_VMPAGE: return (cb->cb_vm_page_len); case CRYPTO_BUF_UIO: diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h --- a/sys/opencrypto/cryptodev.h +++ b/sys/opencrypto/cryptodev.h @@ -354,7 +354,8 @@ CRYPTO_BUF_UIO, CRYPTO_BUF_MBUF, CRYPTO_BUF_VMPAGE, - CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE + CRYPTO_BUF_SINGLE_MBUF, + CRYPTO_BUF_LAST = CRYPTO_BUF_SINGLE_MBUF }; /* @@ -480,6 +481,13 @@ cb->cb_type = CRYPTO_BUF_MBUF; } +static __inline void +_crypto_use_single_mbuf(struct crypto_buffer *cb, struct mbuf *m) +{ + cb->cb_mbuf = m; + cb->cb_type = CRYPTO_BUF_SINGLE_MBUF; +} + static __inline void _crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len, int offset) @@ -509,6 +517,12 @@ _crypto_use_mbuf(&crp->crp_buf, m); } +static __inline void +crypto_use_single_mbuf(struct cryptop *crp, struct mbuf *m) +{ + _crypto_use_single_mbuf(&crp->crp_buf, m); +} + static __inline void crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset) { @@ -533,6 +547,12 @@ _crypto_use_mbuf(&crp->crp_obuf, m); } +static __inline void +crypto_use_output_single_mbuf(struct cryptop *crp, struct mbuf *m) +{ + _crypto_use_single_mbuf(&crp->crp_obuf, m); +} + static __inline void crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset) diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c --- a/sys/opencrypto/cryptosoft.c +++ b/sys/opencrypto/cryptosoft.c @@ -1105,6 +1105,7 @@ if (result < crp->crp_payload_length) { switch (crp->crp_buf.cb_type) { case CRYPTO_BUF_MBUF: + case CRYPTO_BUF_SINGLE_MBUF: adj = result - crp->crp_payload_length; m_adj(crp->crp_buf.cb_mbuf, adj); break;