Index: head/share/man/man9/crypto_buffer.9 =================================================================== --- head/share/man/man9/crypto_buffer.9 +++ head/share/man/man9/crypto_buffer.9 @@ -30,7 +30,7 @@ .\" .\" $FreeBSD$ .\" -.Dd May 25, 2020 +.Dd August 12, 2020 .Dt CRYPTO_BUFFER 9 .Os .Sh NAME @@ -197,10 +197,17 @@ .It Dv CRYPTO_BUF_MBUF A network memory buffer as described in .Xr mbuf 9 . +.It Dv CRYPTO_BUF_VMPAGE +A scatter/gather list of +.Vt vm_page_t +structures describing pages in the kernel's address space. +This buffer type is only available if +.Dv CRYPTO_HAS_VMPAGE +is true. .El .Pp The structure also contains the following type-specific fields: -.Bl -tag -width " cb_buf_len" +.Bl -tag -width " cb_vm_page_offset" .It Fa cb_buf A pointer to the start of a .Dv CRYPTO_BUF_CONTIG @@ -219,6 +226,19 @@ .Vt struct uio for .Dv CRYPTO_BUF_UIO . +.It Fa cb_vm_page +A pointer to an array of +.Vt struct vm_page +for +.Dv CRYPTO_BUF_VMPAGE . +.It Fa cb_vm_page_len +The total amount of data included in the +.Fa cb_vm_page +array, in bytes. +.It Fa cb_vm_page_offset +Offset in bytes in the first page of +.Fa cb_vm_page +where valid data begins. .El .Ss Cursors Cursors provide a mechanism for iterating over a data buffer. Index: head/share/man/man9/crypto_request.9 =================================================================== --- head/share/man/man9/crypto_request.9 +++ head/share/man/man9/crypto_request.9 @@ -30,7 +30,7 @@ .\" .\" $FreeBSD$ .\" -.Dd July 16, 2020 +.Dd August 12, 2020 .Dt CRYPTO_REQUEST 9 .Os .Sh NAME @@ -55,11 +55,15 @@ .Ft void .Fn crypto_use_uio "struct cryptop *crp" "struct uio *uio" .Ft void +.Fn crypto_use_vmpage "struct cryptop *crp" "vm_page_t *pages" "int len" "int offset" +.Ft void .Fn crypto_use_output_buf "struct cryptop *crp" "void *buf" "int len" .Ft void .Fn crypto_use_output_mbuf "struct cryptop *crp" "struct mbuf *m" .Ft void .Fn crypto_use_output_uio "struct cryptop *crp" "struct uio *uio" +.Ft void +.Fn crypto_use_output_vmpage "struct cryptop *crp" "vm_page_t *pages" "int len" "int offset" .Sh DESCRIPTION Each symmetric cryptographic operation in the kernel is described by an instance of @@ -141,7 +145,7 @@ All requests must have a valid .Fa crp_buf initialized by one of the following functions: -.Bl -tag -width "Fn crypto_use_mbuf" +.Bl -tag -width "Fn crypto_use_vmpage" .It Fn crypto_use_buf Uses an array of .Fa len @@ -156,12 +160,16 @@ Uses the scatter/gather list .Fa uio as the data buffer. +.It Fn crypto_use_vmpage +Uses the array of +.Vt vm_page_t +structures as the data buffer. .El .Pp One of the following functions should be used to initialize .Fa crp_obuf for requests that use separate input and output buffers: -.Bl -tag -width "Fn crypto_use_output_mbuf" +.Bl -tag -width "Fn crypto_use_output_vmpage" .It Fn crypto_use_output_buf Uses an array of .Fa len @@ -176,6 +184,10 @@ Uses the scatter/gather list .Fa uio as the output buffer. +.It Fn crypto_use_output_vmpage +Uses the array of +.Vt vm_page_t +structures as the output buffer. .El .Ss Request Regions Each request describes one or more regions in the data buffers. Index: head/sys/crypto/ccp/ccp.c =================================================================== --- head/sys/crypto/ccp/ccp.c +++ head/sys/crypto/ccp/ccp.c @@ -107,6 +107,10 @@ case CRYPTO_BUF_CONTIG: error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len); break; + case CRYPTO_BUF_VMPAGE: + error = sglist_append_vmpages(sg, cb->cb_vm_page, + cb->cb_vm_page_len, cb->cb_vm_page_offset); + break; default: error = EINVAL; } Index: head/sys/dev/cxgbe/crypto/t4_crypto.c =================================================================== --- head/sys/dev/cxgbe/crypto/t4_crypto.c +++ head/sys/dev/cxgbe/crypto/t4_crypto.c @@ -272,6 +272,10 @@ case CRYPTO_BUF_CONTIG: error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len); break; + case CRYPTO_BUF_VMPAGE: + error = sglist_append_vmpages(sg, cb->cb_vm_page, + cb->cb_vm_page_len, cb->cb_vm_page_offset); + break; default: error = EINVAL; } Index: head/sys/dev/sec/sec.c =================================================================== --- head/sys/dev/sec/sec.c +++ head/sys/dev/sec/sec.c @@ -851,6 +851,9 @@ case CRYPTO_BUF_MBUF: size = m_length(crp->crp_buf.cb_mbuf, NULL); break; + case CRYPTO_BUF_VMPAGE: + size = PAGE_SIZE - cb->cb_vm_page_offset; + break; default: return (EINVAL); } Index: head/sys/kern/subr_bus_dma.c =================================================================== --- head/sys/kern/subr_bus_dma.c +++ head/sys/kern/subr_bus_dma.c @@ -661,6 +661,11 @@ error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs, flags); break; + case CRYPTO_BUF_VMPAGE: + error = _bus_dmamap_load_ma(dmat, map, cb->cb_vm_page, + cb->cb_vm_page_len, cb->cb_vm_page_offset, flags, NULL, + &nsegs); + break; default: error = EINVAL; } Index: head/sys/opencrypto/criov.c =================================================================== --- head/sys/opencrypto/criov.c +++ head/sys/opencrypto/criov.c @@ -40,12 +40,21 @@ #include #include #include +#include +#include + +#include +#include +#include + #include +SDT_PROVIDER_DECLARE(opencrypto); + /* - * This macro is only for avoiding code duplication, as we need to skip - * given number of bytes in the same way in three functions below. + * These macros are only for avoiding code duplication, as we need to skip + * given number of bytes in the same way in several functions below. */ #define CUIO_SKIP() do { \ KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \ @@ -60,6 +69,18 @@ } \ } while (0) +#define CVM_PAGE_SKIP() do { \ + KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \ + KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \ + while (off > 0) { \ + if (off < PAGE_SIZE) \ + break; \ + processed += PAGE_SIZE - off; \ + off -= PAGE_SIZE - off; \ + pages++; \ + } \ +} while (0) + static void cuio_copydata(struct uio* uio, int off, int len, caddr_t cp) { @@ -128,6 +149,96 @@ return (-1); } +#if CRYPTO_MAY_HAVE_VMPAGE +/* + * Apply function f to the data in a vm_page_t list starting "off" bytes from + * the beginning, continuing for "len" bytes. + */ +static int +cvm_page_apply(vm_page_t *pages, int off, int len, + int (*f)(void *, const void *, u_int), void *arg) +{ + int processed = 0; + unsigned count; + int rval; + + CVM_PAGE_SKIP(); + while (len > 0) { + char *kaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)); + count = min(PAGE_SIZE - off, len); + rval = (*f)(arg, kaddr + off, count); + if (rval) + return (rval); + len -= count; + processed += count; + off = 0; + pages++; + } + return (0); +} + +static inline void * +cvm_page_contiguous_segment(vm_page_t *pages, size_t skip, int len) +{ + if ((skip + len - 1) / PAGE_SIZE > skip / PAGE_SIZE) + return (NULL); + + pages += (skip / PAGE_SIZE); + skip -= rounddown(skip, PAGE_SIZE); + return (((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages))) + skip); +} + +/* + * Copy len bytes of data from the vm_page_t array, skipping the first off + * bytes, into the pointer cp. Return the number of bytes skipped and copied. + * Does not verify the length of the array. + */ +static int +cvm_page_copyback(vm_page_t *pages, int off, int len, c_caddr_t cp) +{ + int processed = 0; + unsigned count; + + CVM_PAGE_SKIP(); + while (len > 0) { + count = min(PAGE_SIZE - off, len); + bcopy(cp, (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off, + count); + len -= count; + cp += count; + processed += count; + off = 0; + pages++; + } + return (processed); +} + +/* + * Copy len bytes of data from the pointer cp into the vm_page_t array, + * skipping the first off bytes, Return the number of bytes skipped and copied. + * Does not verify the length of the array. + */ +static int +cvm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp) +{ + int processed = 0; + unsigned count; + + CVM_PAGE_SKIP(); + while (len > 0) { + count = min(PAGE_SIZE - off, len); + bcopy(((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off), cp, + count); + len -= count; + cp += count; + processed += count; + off = 0; + pages++; + } + return processed; +} +#endif /* CRYPTO_MAY_HAVE_VMPAGE */ + void crypto_cursor_init(struct crypto_buffer_cursor *cc, const struct crypto_buffer *cb) @@ -142,6 +253,11 @@ case CRYPTO_BUF_MBUF: cc->cc_mbuf = cb->cb_mbuf; break; + case CRYPTO_BUF_VMPAGE: + cc->cc_vmpage = cb->cb_vm_page; + cc->cc_buf_len = cb->cb_vm_page_len; + cc->cc_offset = cb->cb_vm_page_offset; + break; case CRYPTO_BUF_UIO: cc->cc_iov = cb->cb_uio->uio_iov; break; @@ -153,6 +269,8 @@ } } +SDT_PROBE_DEFINE2(opencrypto, criov, cursor_advance, vmpage, "struct crypto_buffer_cursor*", "size_t"); + void crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount) { @@ -178,6 +296,24 @@ break; } break; + case CRYPTO_BUF_VMPAGE: + for (;;) { + SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage, + cc, amount); + remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len); + if (amount < remain) { + cc->cc_buf_len -= amount; + cc->cc_offset += amount; + break; + } + cc->cc_buf_len -= remain; + amount -= remain; + cc->cc_vmpage++; + cc->cc_offset = 0; + if (amount == 0 || cc->cc_buf_len == 0) + break; + } + break; case CRYPTO_BUF_UIO: for (;;) { remain = cc->cc_iov->iov_len - cc->cc_offset; @@ -212,6 +348,9 @@ KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0, ("%s: not supported for unmapped mbufs", __func__)); return (mtod(cc->cc_mbuf, char *) + cc->cc_offset); + case CRYPTO_BUF_VMPAGE: + return ((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( + *cc->cc_vmpage)) + cc->cc_offset); case CRYPTO_BUF_UIO: return ((char *)cc->cc_iov->iov_base + cc->cc_offset); default: @@ -228,6 +367,8 @@ switch (cc->cc_type) { case CRYPTO_BUF_CONTIG: return (cc->cc_buf_len); + case CRYPTO_BUF_VMPAGE: + return (PAGE_SIZE - cc->cc_offset); case CRYPTO_BUF_MBUF: if (cc->cc_mbuf == NULL) return (0); @@ -278,6 +419,26 @@ break; } break; + case CRYPTO_BUF_VMPAGE: + for (;;) { + dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( + *cc->cc_vmpage)) + cc->cc_offset; + remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len); + todo = MIN(remain, size); + memcpy(dst, src, todo); + src += todo; + cc->cc_buf_len -= todo; + if (todo < remain) { + cc->cc_offset += todo; + break; + } + size -= todo; + cc->cc_vmpage++; + cc->cc_offset = 0; + if (size == 0) + break; + } + break; case CRYPTO_BUF_UIO: for (;;) { dst = (char *)cc->cc_iov->iov_base + cc->cc_offset; @@ -339,6 +500,26 @@ break; } break; + case CRYPTO_BUF_VMPAGE: + for (;;) { + src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( + *cc->cc_vmpage)) + cc->cc_offset; + remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len); + todo = MIN(remain, size); + memcpy(dst, src, todo); + src += todo; + cc->cc_buf_len -= todo; + if (todo < remain) { + cc->cc_offset += todo; + break; + } + size -= todo; + cc->cc_vmpage++; + cc->cc_offset = 0; + if (size == 0) + break; + } + break; case CRYPTO_BUF_UIO: for (;;) { src = (const char *)cc->cc_iov->iov_base + @@ -421,6 +602,15 @@ case CRYPTO_BUF_MBUF: m_copyback(cb->cb_mbuf, off, size, src); break; +#if CRYPTO_MAY_HAVE_VMPAGE + case CRYPTO_BUF_VMPAGE: + MPASS(size <= cb->cb_vm_page_len); + MPASS(size + off <= + cb->cb_vm_page_len + cb->cb_vm_page_offset); + cvm_page_copyback(cb->cb_vm_page, + off + cb->cb_vm_page_offset, size, src); + break; +#endif /* CRYPTO_MAY_HAVE_VMPAGE */ case CRYPTO_BUF_UIO: cuio_copyback(cb->cb_uio, off, size, src); break; @@ -444,6 +634,15 @@ case CRYPTO_BUF_MBUF: m_copydata(crp->crp_buf.cb_mbuf, off, size, dst); break; +#if CRYPTO_MAY_HAVE_VMPAGE + case CRYPTO_BUF_VMPAGE: + MPASS(size <= crp->crp_buf.cb_vm_page_len); + MPASS(size + off <= crp->crp_buf.cb_vm_page_len + + crp->crp_buf.cb_vm_page_offset); + cvm_page_copydata(crp->crp_buf.cb_vm_page, + off + crp->crp_buf.cb_vm_page_offset, size, dst); + break; +#endif /* CRYPTO_MAY_HAVE_VMPAGE */ case CRYPTO_BUF_UIO: cuio_copydata(crp->crp_buf.cb_uio, off, size, dst); break; @@ -473,6 +672,12 @@ case CRYPTO_BUF_UIO: error = cuio_apply(cb->cb_uio, off, len, f, arg); break; +#if CRYPTO_MAY_HAVE_VMPAGE + case CRYPTO_BUF_VMPAGE: + error = cvm_page_apply(cb->cb_vm_page, + off + cb->cb_vm_page_offset, len, f, arg); + break; +#endif /* CRYPTO_MAY_HAVE_VMPAGE */ case CRYPTO_BUF_CONTIG: MPASS(off + len <= cb->cb_buf_len); error = (*f)(arg, cb->cb_buf + off, len); @@ -540,6 +745,12 @@ return (m_contiguous_subsegment(cb->cb_mbuf, skip, len)); case CRYPTO_BUF_UIO: return (cuio_contiguous_segment(cb->cb_uio, skip, len)); +#if CRYPTO_MAY_HAVE_VMPAGE + case CRYPTO_BUF_VMPAGE: + MPASS(skip + len <= cb->cb_vm_page_len); + return (cvm_page_contiguous_segment(cb->cb_vm_page, + skip + cb->cb_vm_page_offset, len)); +#endif /* CRYPTO_MAY_HAVE_VMPAGE */ case CRYPTO_BUF_CONTIG: MPASS(skip + len <= cb->cb_buf_len); return (cb->cb_buf + skip); Index: head/sys/opencrypto/crypto.c =================================================================== --- head/sys/opencrypto/crypto.c +++ head/sys/opencrypto/crypto.c @@ -78,7 +78,9 @@ #include +#include #include + #include #include #include @@ -1218,6 +1220,8 @@ if (cb->cb_mbuf->m_flags & M_PKTHDR) return (cb->cb_mbuf->m_pkthdr.len); return (m_length(cb->cb_mbuf, NULL)); + case CRYPTO_BUF_VMPAGE: + return (cb->cb_vm_page_len); case CRYPTO_BUF_UIO: return (cb->cb_uio->uio_resid); default: @@ -1232,9 +1236,25 @@ { KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST, ("incoming crp with invalid %s buffer type", name)); - if (cb->cb_type == CRYPTO_BUF_CONTIG) + switch (cb->cb_type) { + case CRYPTO_BUF_CONTIG: KASSERT(cb->cb_buf_len >= 0, ("incoming crp with -ve %s buffer length", name)); + break; + case CRYPTO_BUF_VMPAGE: + KASSERT(CRYPTO_HAS_VMPAGE, + ("incoming crp uses dmap on supported arch")); + KASSERT(cb->cb_vm_page_len >= 0, + ("incoming crp with -ve %s buffer length", name)); + KASSERT(cb->cb_vm_page_offset >= 0, + ("incoming crp with -ve %s buffer offset", name)); + KASSERT(cb->cb_vm_page_offset < PAGE_SIZE, + ("incoming crp with %s buffer offset greater than page size" + , name)); + break; + default: + break; + } } static void Index: head/sys/opencrypto/cryptodev.h =================================================================== --- head/sys/opencrypto/cryptodev.h +++ head/sys/opencrypto/cryptodev.h @@ -205,6 +205,15 @@ #define CRYPTO_FLAG_HARDWARE 0x01000000 /* hardware accelerated */ #define CRYPTO_FLAG_SOFTWARE 0x02000000 /* software implementation */ +/* Does the kernel support vmpage buffers on this platform? */ +#ifdef __powerpc__ +#define CRYPTO_MAY_HAVE_VMPAGE 1 +#else +#define CRYPTO_MAY_HAVE_VMPAGE ( PMAP_HAS_DMAP ) +#endif +/* Does the currently running system support vmpage buffers on this platform? */ +#define CRYPTO_HAS_VMPAGE ( PMAP_HAS_DMAP ) + /* NB: deprecated */ struct session_op { u_int32_t cipher; /* ie. CRYPTO_AES_CBC */ @@ -387,7 +396,8 @@ CRYPTO_BUF_CONTIG, CRYPTO_BUF_UIO, CRYPTO_BUF_MBUF, - CRYPTO_BUF_LAST = CRYPTO_BUF_MBUF + CRYPTO_BUF_VMPAGE, + CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE }; /* @@ -402,6 +412,11 @@ int cb_buf_len; }; struct mbuf *cb_mbuf; + struct { + vm_page_t *cb_vm_page; + int cb_vm_page_len; + int cb_vm_page_offset; + }; struct uio *cb_uio; }; enum crypto_buffer_type cb_type; @@ -415,11 +430,15 @@ char *cc_buf; struct mbuf *cc_mbuf; struct iovec *cc_iov; + vm_page_t *cc_vmpage; }; - union { - int cc_buf_len; - size_t cc_offset; - }; + /* Optional bytes of valid data remaining */ + int cc_buf_len; + /* + * Optional offset within the current buffer segment where + * valid data begins + */ + size_t cc_offset; enum crypto_buffer_type cc_type; }; @@ -509,6 +528,16 @@ } static __inline void +_crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len, + int offset) +{ + cb->cb_vm_page = pages; + cb->cb_vm_page_len = len; + cb->cb_vm_page_offset = offset; + cb->cb_type = CRYPTO_BUF_VMPAGE; +} + +static __inline void _crypto_use_uio(struct crypto_buffer *cb, struct uio *uio) { cb->cb_uio = uio; @@ -528,6 +557,12 @@ } static __inline void +crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset) +{ + _crypto_use_vmpage(&crp->crp_buf, pages, len, offset); +} + +static __inline void crypto_use_uio(struct cryptop *crp, struct uio *uio) { _crypto_use_uio(&crp->crp_buf, uio); @@ -543,6 +578,13 @@ crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m) { _crypto_use_mbuf(&crp->crp_obuf, m); +} + +static __inline void +crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len, + int offset) +{ + _crypto_use_vmpage(&crp->crp_obuf, pages, len, offset); } static __inline void Index: head/sys/opencrypto/cryptosoft.c =================================================================== --- head/sys/opencrypto/cryptosoft.c +++ head/sys/opencrypto/cryptosoft.c @@ -980,6 +980,10 @@ } } break; + case CRYPTO_BUF_VMPAGE: + adj = crp->crp_payload_length - result; + crp->crp_buf.cb_vm_page_len -= adj; + break; default: break; }