Page MenuHomeFreeBSD
Files F106056619

D25671.diff
No OneTemporary
Actions

D25671.diff
View Options

Index: head/share/man/man9/crypto_buffer.9
===================================================================
--- head/share/man/man9/crypto_buffer.9
+++ head/share/man/man9/crypto_buffer.9
@@ -30,7 +30,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd May 25, 2020
+.Dd August 12, 2020
.Dt CRYPTO_BUFFER 9
.Os
.Sh NAME
@@ -197,10 +197,17 @@
.It Dv CRYPTO_BUF_MBUF
A network memory buffer as described in
.Xr mbuf 9 .
+.It Dv CRYPTO_BUF_VMPAGE
+A scatter/gather list of
+.Vt vm_page_t
+structures describing pages in the kernel's address space.
+This buffer type is only available if
+.Dv CRYPTO_HAS_VMPAGE
+is true.
.El
.Pp
The structure also contains the following type-specific fields:
-.Bl -tag -width " cb_buf_len"
+.Bl -tag -width " cb_vm_page_offset"
.It Fa cb_buf
A pointer to the start of a
.Dv CRYPTO_BUF_CONTIG
@@ -219,6 +226,19 @@
.Vt struct uio
for
.Dv CRYPTO_BUF_UIO .
+.It Fa cb_vm_page
+A pointer to an array of
+.Vt struct vm_page
+for
+.Dv CRYPTO_BUF_VMPAGE .
+.It Fa cb_vm_page_len
+The total amount of data included in the
+.Fa cb_vm_page
+array, in bytes.
+.It Fa cb_vm_page_offset
+Offset in bytes in the first page of
+.Fa cb_vm_page
+where valid data begins.
.El
.Ss Cursors
Cursors provide a mechanism for iterating over a data buffer.
Index: head/share/man/man9/crypto_request.9
===================================================================
--- head/share/man/man9/crypto_request.9
+++ head/share/man/man9/crypto_request.9
@@ -30,7 +30,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd July 16, 2020
+.Dd August 12, 2020
.Dt CRYPTO_REQUEST 9
.Os
.Sh NAME
@@ -55,11 +55,15 @@
.Ft void
.Fn crypto_use_uio "struct cryptop *crp" "struct uio *uio"
.Ft void
+.Fn crypto_use_vmpage "struct cryptop *crp" "vm_page_t *pages" "int len" "int offset"
+.Ft void
.Fn crypto_use_output_buf "struct cryptop *crp" "void *buf" "int len"
.Ft void
.Fn crypto_use_output_mbuf "struct cryptop *crp" "struct mbuf *m"
.Ft void
.Fn crypto_use_output_uio "struct cryptop *crp" "struct uio *uio"
+.Ft void
+.Fn crypto_use_output_vmpage "struct cryptop *crp" "vm_page_t *pages" "int len" "int offset"
.Sh DESCRIPTION
Each symmetric cryptographic operation in the kernel is described by
an instance of
@@ -141,7 +145,7 @@
All requests must have a valid
.Fa crp_buf
initialized by one of the following functions:
-.Bl -tag -width "Fn crypto_use_mbuf"
+.Bl -tag -width "Fn crypto_use_vmpage"
.It Fn crypto_use_buf
Uses an array of
.Fa len
@@ -156,12 +160,16 @@
Uses the scatter/gather list
.Fa uio
as the data buffer.
+.It Fn crypto_use_vmpage
+Uses the array of
+.Vt vm_page_t
+structures as the data buffer.
.El
.Pp
One of the following functions should be used to initialize
.Fa crp_obuf
for requests that use separate input and output buffers:
-.Bl -tag -width "Fn crypto_use_output_mbuf"
+.Bl -tag -width "Fn crypto_use_output_vmpage"
.It Fn crypto_use_output_buf
Uses an array of
.Fa len
@@ -176,6 +184,10 @@
Uses the scatter/gather list
.Fa uio
as the output buffer.
+.It Fn crypto_use_output_vmpage
+Uses the array of
+.Vt vm_page_t
+structures as the output buffer.
.El
.Ss Request Regions
Each request describes one or more regions in the data buffers.
Index: head/sys/crypto/ccp/ccp.c
===================================================================
--- head/sys/crypto/ccp/ccp.c
+++ head/sys/crypto/ccp/ccp.c
@@ -107,6 +107,10 @@
case CRYPTO_BUF_CONTIG:
error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
break;
+ case CRYPTO_BUF_VMPAGE:
+ error = sglist_append_vmpages(sg, cb->cb_vm_page,
+ cb->cb_vm_page_len, cb->cb_vm_page_offset);
+ break;
default:
error = EINVAL;
}
Index: head/sys/dev/cxgbe/crypto/t4_crypto.c
===================================================================
--- head/sys/dev/cxgbe/crypto/t4_crypto.c
+++ head/sys/dev/cxgbe/crypto/t4_crypto.c
@@ -272,6 +272,10 @@
case CRYPTO_BUF_CONTIG:
error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
break;
+ case CRYPTO_BUF_VMPAGE:
+ error = sglist_append_vmpages(sg, cb->cb_vm_page,
+ cb->cb_vm_page_len, cb->cb_vm_page_offset);
+ break;
default:
error = EINVAL;
}
Index: head/sys/dev/sec/sec.c
===================================================================
--- head/sys/dev/sec/sec.c
+++ head/sys/dev/sec/sec.c
@@ -851,6 +851,9 @@
case CRYPTO_BUF_MBUF:
size = m_length(crp->crp_buf.cb_mbuf, NULL);
break;
+ case CRYPTO_BUF_VMPAGE:
+ size = PAGE_SIZE - cb->cb_vm_page_offset;
+ break;
default:
return (EINVAL);
}
Index: head/sys/kern/subr_bus_dma.c
===================================================================
--- head/sys/kern/subr_bus_dma.c
+++ head/sys/kern/subr_bus_dma.c
@@ -661,6 +661,11 @@
error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs,
flags);
break;
+ case CRYPTO_BUF_VMPAGE:
+ error = _bus_dmamap_load_ma(dmat, map, cb->cb_vm_page,
+ cb->cb_vm_page_len, cb->cb_vm_page_offset, flags, NULL,
+ &nsegs);
+ break;
default:
error = EINVAL;
}
Index: head/sys/opencrypto/criov.c
===================================================================
--- head/sys/opencrypto/criov.c
+++ head/sys/opencrypto/criov.c
@@ -40,12 +40,21 @@
#include <sys/uio.h>
#include <sys/limits.h>
#include <sys/lock.h>
+#include <sys/sdt.h>
+#include <machine/vmparam.h>
+
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+#include <vm/pmap.h>
+
#include <opencrypto/cryptodev.h>
+SDT_PROVIDER_DECLARE(opencrypto);
+
/*
- * This macro is only for avoiding code duplication, as we need to skip
- * given number of bytes in the same way in three functions below.
+ * These macros are only for avoiding code duplication, as we need to skip
+ * given number of bytes in the same way in several functions below.
*/
#define CUIO_SKIP() do { \
KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
@@ -60,6 +69,18 @@
} \
} while (0)
+#define CVM_PAGE_SKIP() do { \
+ KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
+ KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
+ while (off > 0) { \
+ if (off < PAGE_SIZE) \
+ break; \
+ processed += PAGE_SIZE - off; \
+ off -= PAGE_SIZE - off; \
+ pages++; \
+ } \
+} while (0)
+
static void
cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
{
@@ -128,6 +149,96 @@
return (-1);
}
+#if CRYPTO_MAY_HAVE_VMPAGE
+/*
+ * Apply function f to the data in a vm_page_t list starting "off" bytes from
+ * the beginning, continuing for "len" bytes.
+ */
+static int
+cvm_page_apply(vm_page_t *pages, int off, int len,
+ int (*f)(void *, const void *, u_int), void *arg)
+{
+ int processed = 0;
+ unsigned count;
+ int rval;
+
+ CVM_PAGE_SKIP();
+ while (len > 0) {
+ char *kaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages));
+ count = min(PAGE_SIZE - off, len);
+ rval = (*f)(arg, kaddr + off, count);
+ if (rval)
+ return (rval);
+ len -= count;
+ processed += count;
+ off = 0;
+ pages++;
+ }
+ return (0);
+}
+
+static inline void *
+cvm_page_contiguous_segment(vm_page_t *pages, size_t skip, int len)
+{
+ if ((skip + len - 1) / PAGE_SIZE > skip / PAGE_SIZE)
+ return (NULL);
+
+ pages += (skip / PAGE_SIZE);
+ skip -= rounddown(skip, PAGE_SIZE);
+ return (((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages))) + skip);
+}
+
+/*
+ * Copy len bytes of data from the vm_page_t array, skipping the first off
+ * bytes, into the pointer cp. Return the number of bytes skipped and copied.
+ * Does not verify the length of the array.
+ */
+static int
+cvm_page_copyback(vm_page_t *pages, int off, int len, c_caddr_t cp)
+{
+ int processed = 0;
+ unsigned count;
+
+ CVM_PAGE_SKIP();
+ while (len > 0) {
+ count = min(PAGE_SIZE - off, len);
+ bcopy(cp, (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off,
+ count);
+ len -= count;
+ cp += count;
+ processed += count;
+ off = 0;
+ pages++;
+ }
+ return (processed);
+}
+
+/*
+ * Copy len bytes of data from the pointer cp into the vm_page_t array,
+ * skipping the first off bytes, Return the number of bytes skipped and copied.
+ * Does not verify the length of the array.
+ */
+static int
+cvm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp)
+{
+ int processed = 0;
+ unsigned count;
+
+ CVM_PAGE_SKIP();
+ while (len > 0) {
+ count = min(PAGE_SIZE - off, len);
+ bcopy(((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off), cp,
+ count);
+ len -= count;
+ cp += count;
+ processed += count;
+ off = 0;
+ pages++;
+ }
+ return processed;
+}
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
+
void
crypto_cursor_init(struct crypto_buffer_cursor *cc,
const struct crypto_buffer *cb)
@@ -142,6 +253,11 @@
case CRYPTO_BUF_MBUF:
cc->cc_mbuf = cb->cb_mbuf;
break;
+ case CRYPTO_BUF_VMPAGE:
+ cc->cc_vmpage = cb->cb_vm_page;
+ cc->cc_buf_len = cb->cb_vm_page_len;
+ cc->cc_offset = cb->cb_vm_page_offset;
+ break;
case CRYPTO_BUF_UIO:
cc->cc_iov = cb->cb_uio->uio_iov;
break;
@@ -153,6 +269,8 @@
}
}
+SDT_PROBE_DEFINE2(opencrypto, criov, cursor_advance, vmpage, "struct crypto_buffer_cursor*", "size_t");
+
void
crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
{
@@ -178,6 +296,24 @@
break;
}
break;
+ case CRYPTO_BUF_VMPAGE:
+ for (;;) {
+ SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
+ cc, amount);
+ remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
+ if (amount < remain) {
+ cc->cc_buf_len -= amount;
+ cc->cc_offset += amount;
+ break;
+ }
+ cc->cc_buf_len -= remain;
+ amount -= remain;
+ cc->cc_vmpage++;
+ cc->cc_offset = 0;
+ if (amount == 0 || cc->cc_buf_len == 0)
+ break;
+ }
+ break;
case CRYPTO_BUF_UIO:
for (;;) {
remain = cc->cc_iov->iov_len - cc->cc_offset;
@@ -212,6 +348,9 @@
KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
("%s: not supported for unmapped mbufs", __func__));
return (mtod(cc->cc_mbuf, char *) + cc->cc_offset);
+ case CRYPTO_BUF_VMPAGE:
+ return ((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
+ *cc->cc_vmpage)) + cc->cc_offset);
case CRYPTO_BUF_UIO:
return ((char *)cc->cc_iov->iov_base + cc->cc_offset);
default:
@@ -228,6 +367,8 @@
switch (cc->cc_type) {
case CRYPTO_BUF_CONTIG:
return (cc->cc_buf_len);
+ case CRYPTO_BUF_VMPAGE:
+ return (PAGE_SIZE - cc->cc_offset);
case CRYPTO_BUF_MBUF:
if (cc->cc_mbuf == NULL)
return (0);
@@ -278,6 +419,26 @@
break;
}
break;
+ case CRYPTO_BUF_VMPAGE:
+ for (;;) {
+ dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
+ *cc->cc_vmpage)) + cc->cc_offset;
+ remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
+ todo = MIN(remain, size);
+ memcpy(dst, src, todo);
+ src += todo;
+ cc->cc_buf_len -= todo;
+ if (todo < remain) {
+ cc->cc_offset += todo;
+ break;
+ }
+ size -= todo;
+ cc->cc_vmpage++;
+ cc->cc_offset = 0;
+ if (size == 0)
+ break;
+ }
+ break;
case CRYPTO_BUF_UIO:
for (;;) {
dst = (char *)cc->cc_iov->iov_base + cc->cc_offset;
@@ -339,6 +500,26 @@
break;
}
break;
+ case CRYPTO_BUF_VMPAGE:
+ for (;;) {
+ src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
+ *cc->cc_vmpage)) + cc->cc_offset;
+ remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
+ todo = MIN(remain, size);
+ memcpy(dst, src, todo);
+ src += todo;
+ cc->cc_buf_len -= todo;
+ if (todo < remain) {
+ cc->cc_offset += todo;
+ break;
+ }
+ size -= todo;
+ cc->cc_vmpage++;
+ cc->cc_offset = 0;
+ if (size == 0)
+ break;
+ }
+ break;
case CRYPTO_BUF_UIO:
for (;;) {
src = (const char *)cc->cc_iov->iov_base +
@@ -421,6 +602,15 @@
case CRYPTO_BUF_MBUF:
m_copyback(cb->cb_mbuf, off, size, src);
break;
+#if CRYPTO_MAY_HAVE_VMPAGE
+ case CRYPTO_BUF_VMPAGE:
+ MPASS(size <= cb->cb_vm_page_len);
+ MPASS(size + off <=
+ cb->cb_vm_page_len + cb->cb_vm_page_offset);
+ cvm_page_copyback(cb->cb_vm_page,
+ off + cb->cb_vm_page_offset, size, src);
+ break;
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_UIO:
cuio_copyback(cb->cb_uio, off, size, src);
break;
@@ -444,6 +634,15 @@
case CRYPTO_BUF_MBUF:
m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
break;
+#if CRYPTO_MAY_HAVE_VMPAGE
+ case CRYPTO_BUF_VMPAGE:
+ MPASS(size <= crp->crp_buf.cb_vm_page_len);
+ MPASS(size + off <= crp->crp_buf.cb_vm_page_len +
+ crp->crp_buf.cb_vm_page_offset);
+ cvm_page_copydata(crp->crp_buf.cb_vm_page,
+ off + crp->crp_buf.cb_vm_page_offset, size, dst);
+ break;
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_UIO:
cuio_copydata(crp->crp_buf.cb_uio, off, size, dst);
break;
@@ -473,6 +672,12 @@
case CRYPTO_BUF_UIO:
error = cuio_apply(cb->cb_uio, off, len, f, arg);
break;
+#if CRYPTO_MAY_HAVE_VMPAGE
+ case CRYPTO_BUF_VMPAGE:
+ error = cvm_page_apply(cb->cb_vm_page,
+ off + cb->cb_vm_page_offset, len, f, arg);
+ break;
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_CONTIG:
MPASS(off + len <= cb->cb_buf_len);
error = (*f)(arg, cb->cb_buf + off, len);
@@ -540,6 +745,12 @@
return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
case CRYPTO_BUF_UIO:
return (cuio_contiguous_segment(cb->cb_uio, skip, len));
+#if CRYPTO_MAY_HAVE_VMPAGE
+ case CRYPTO_BUF_VMPAGE:
+ MPASS(skip + len <= cb->cb_vm_page_len);
+ return (cvm_page_contiguous_segment(cb->cb_vm_page,
+ skip + cb->cb_vm_page_offset, len));
+#endif /* CRYPTO_MAY_HAVE_VMPAGE */
case CRYPTO_BUF_CONTIG:
MPASS(skip + len <= cb->cb_buf_len);
return (cb->cb_buf + skip);
Index: head/sys/opencrypto/crypto.c
===================================================================
--- head/sys/opencrypto/crypto.c
+++ head/sys/opencrypto/crypto.c
@@ -78,7 +78,9 @@
#include <ddb/ddb.h>
+#include <machine/vmparam.h>
#include <vm/uma.h>
+
#include <crypto/intake.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/xform_auth.h>
@@ -1218,6 +1220,8 @@
if (cb->cb_mbuf->m_flags & M_PKTHDR)
return (cb->cb_mbuf->m_pkthdr.len);
return (m_length(cb->cb_mbuf, NULL));
+ case CRYPTO_BUF_VMPAGE:
+ return (cb->cb_vm_page_len);
case CRYPTO_BUF_UIO:
return (cb->cb_uio->uio_resid);
default:
@@ -1232,9 +1236,25 @@
{
KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
("incoming crp with invalid %s buffer type", name));
- if (cb->cb_type == CRYPTO_BUF_CONTIG)
+ switch (cb->cb_type) {
+ case CRYPTO_BUF_CONTIG:
KASSERT(cb->cb_buf_len >= 0,
("incoming crp with -ve %s buffer length", name));
+ break;
+ case CRYPTO_BUF_VMPAGE:
+ KASSERT(CRYPTO_HAS_VMPAGE,
+ ("incoming crp uses dmap on supported arch"));
+ KASSERT(cb->cb_vm_page_len >= 0,
+ ("incoming crp with -ve %s buffer length", name));
+ KASSERT(cb->cb_vm_page_offset >= 0,
+ ("incoming crp with -ve %s buffer offset", name));
+ KASSERT(cb->cb_vm_page_offset < PAGE_SIZE,
+ ("incoming crp with %s buffer offset greater than page size"
+ , name));
+ break;
+ default:
+ break;
+ }
}
static void
Index: head/sys/opencrypto/cryptodev.h
===================================================================
--- head/sys/opencrypto/cryptodev.h
+++ head/sys/opencrypto/cryptodev.h
@@ -205,6 +205,15 @@
#define CRYPTO_FLAG_HARDWARE 0x01000000 /* hardware accelerated */
#define CRYPTO_FLAG_SOFTWARE 0x02000000 /* software implementation */
+/* Does the kernel support vmpage buffers on this platform? */
+#ifdef __powerpc__
+#define CRYPTO_MAY_HAVE_VMPAGE 1
+#else
+#define CRYPTO_MAY_HAVE_VMPAGE ( PMAP_HAS_DMAP )
+#endif
+/* Does the currently running system support vmpage buffers on this platform? */
+#define CRYPTO_HAS_VMPAGE ( PMAP_HAS_DMAP )
+
/* NB: deprecated */
struct session_op {
u_int32_t cipher; /* ie. CRYPTO_AES_CBC */
@@ -387,7 +396,8 @@
CRYPTO_BUF_CONTIG,
CRYPTO_BUF_UIO,
CRYPTO_BUF_MBUF,
- CRYPTO_BUF_LAST = CRYPTO_BUF_MBUF
+ CRYPTO_BUF_VMPAGE,
+ CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE
};
/*
@@ -402,6 +412,11 @@
int cb_buf_len;
};
struct mbuf *cb_mbuf;
+ struct {
+ vm_page_t *cb_vm_page;
+ int cb_vm_page_len;
+ int cb_vm_page_offset;
+ };
struct uio *cb_uio;
};
enum crypto_buffer_type cb_type;
@@ -415,11 +430,15 @@
char *cc_buf;
struct mbuf *cc_mbuf;
struct iovec *cc_iov;
+ vm_page_t *cc_vmpage;
};
- union {
- int cc_buf_len;
- size_t cc_offset;
- };
+ /* Optional bytes of valid data remaining */
+ int cc_buf_len;
+ /*
+ * Optional offset within the current buffer segment where
+ * valid data begins
+ */
+ size_t cc_offset;
enum crypto_buffer_type cc_type;
};
@@ -509,6 +528,16 @@
}
static __inline void
+_crypto_use_vmpage(struct crypto_buffer *cb, vm_page_t *pages, int len,
+ int offset)
+{
+ cb->cb_vm_page = pages;
+ cb->cb_vm_page_len = len;
+ cb->cb_vm_page_offset = offset;
+ cb->cb_type = CRYPTO_BUF_VMPAGE;
+}
+
+static __inline void
_crypto_use_uio(struct crypto_buffer *cb, struct uio *uio)
{
cb->cb_uio = uio;
@@ -528,6 +557,12 @@
}
static __inline void
+crypto_use_vmpage(struct cryptop *crp, vm_page_t *pages, int len, int offset)
+{
+ _crypto_use_vmpage(&crp->crp_buf, pages, len, offset);
+}
+
+static __inline void
crypto_use_uio(struct cryptop *crp, struct uio *uio)
{
_crypto_use_uio(&crp->crp_buf, uio);
@@ -543,6 +578,13 @@
crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m)
{
_crypto_use_mbuf(&crp->crp_obuf, m);
+}
+
+static __inline void
+crypto_use_output_vmpage(struct cryptop *crp, vm_page_t *pages, int len,
+ int offset)
+{
+ _crypto_use_vmpage(&crp->crp_obuf, pages, len, offset);
}
static __inline void
Index: head/sys/opencrypto/cryptosoft.c
===================================================================
--- head/sys/opencrypto/cryptosoft.c
+++ head/sys/opencrypto/cryptosoft.c
@@ -980,6 +980,10 @@
}
}
break;
+ case CRYPTO_BUF_VMPAGE:
+ adj = crp->crp_payload_length - result;
+ crp->crp_buf.cb_vm_page_len -= adj;
+ break;
default:
break;
}

File Metadata

Mime Type
text/plain
Expires
Wed, Dec 25, 3:43 PM (11 h, 50 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
15597078
Default Alt Text
D25671.diff (17 KB)

Event Timeline