Index: sys/crypto/ccp/ccp.c =================================================================== --- sys/crypto/ccp/ccp.c +++ sys/crypto/ccp/ccp.c @@ -44,6 +44,8 @@ #include #include +#include + #ifdef DDB #include #endif @@ -106,6 +108,10 @@ break; case CRYPTO_BUF_CONTIG: error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len); + break; + case CRYPTO_BUF_VMPAGE: + error = sglist_append_vmpages(sg, cb->cb_vm_page, + cb->cb_vm_page_len, cb->cb_vm_page_offset); break; default: error = EINVAL; Index: sys/dev/cxgbe/crypto/t4_crypto.c =================================================================== --- sys/dev/cxgbe/crypto/t4_crypto.c +++ sys/dev/cxgbe/crypto/t4_crypto.c @@ -36,6 +36,8 @@ #include #include +#include + #include #include @@ -271,6 +273,10 @@ break; case CRYPTO_BUF_CONTIG: error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len); + break; + case CRYPTO_BUF_VMPAGE: + error = sglist_append_vmpages(sg, cb->cb_vm_page, + cb->cb_vm_page_len, cb->cb_vm_page_offset); break; default: error = EINVAL; Index: sys/dev/sec/sec.c =================================================================== --- sys/dev/sec/sec.c +++ sys/dev/sec/sec.c @@ -49,6 +49,7 @@ #include #include #include +#include #include #include @@ -850,6 +851,9 @@ break; case CRYPTO_BUF_MBUF: size = m_length(crp->crp_buf.cb_mbuf, NULL); + break; + case CRYPTO_BUF_VMPAGE: + size = PAGE_SIZE - cb->cb_vm_page_offset; break; default: return (EINVAL); Index: sys/geom/eli/g_eli.c =================================================================== --- sys/geom/eli/g_eli.c +++ sys/geom/eli/g_eli.c @@ -49,6 +49,8 @@ #include #include +#include + #include #include @@ -972,6 +974,16 @@ */ pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX); pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE; + if (PMAP_HAS_DMAP) { + /* + * On DMAP architectures we can use unmapped I/O. But don't + * use it with data integrity verification. That module needs + * much more intimate access to the bio's data in order to + * add/remove HMAC fields, and split bios that exceed MAXPHYS. + */ + if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0) + pp->flags |= G_PF_ACCEPT_UNMAPPED; + } pp->mediasize = sc->sc_mediasize; pp->sectorsize = sc->sc_sectorsize; LIST_FOREACH(gap, &bpp->aliases, ga_next) Index: sys/geom/eli/g_eli_privacy.c =================================================================== --- sys/geom/eli/g_eli_privacy.c +++ sys/geom/eli/g_eli_privacy.c @@ -45,6 +45,8 @@ #include #include +#include + #include #include @@ -63,6 +65,28 @@ MALLOC_DECLARE(M_ELI); /* + * Copy data from a (potentially unmapped) bio to a kernelspace buffer. + * + * The buffer must have at least as much room as bp->bio_length. + */ +static void +g_eli_bio_copyin(struct bio *bp, void *kaddr) +{ + struct uio uio; + struct iovec iov[1]; + + iov[0].iov_base = kaddr; + iov[0].iov_len = bp->bio_length; + uio.uio_iov = iov; + uio.uio_iovcnt = 1; + uio.uio_offset = 0; + uio.uio_resid = bp->bio_length; + uio.uio_segflg = UIO_SYSSPACE; + uio.uio_rw = UIO_READ; + uiomove_fromphys(bp->bio_ma, bp->bio_ma_offset, bp->bio_length, &uio); +} + +/* * The function is called after we read and decrypt data. * * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver @@ -98,8 +122,7 @@ */ if (bp->bio_inbed < bp->bio_children) return (0); - free(bp->bio_driver2, M_ELI); - bp->bio_driver2 = NULL; + if (bp->bio_error != 0) { G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).", bp->bio_error); @@ -167,6 +190,7 @@ return (0); } cbp->bio_data = bp->bio_driver2; + cbp->bio_flags &= ~BIO_UNMAPPED; cbp->bio_done = g_eli_write_done; cp = LIST_FIRST(&gp->consumer); cbp->bio_to = cp->provider; @@ -236,10 +260,12 @@ { struct g_eli_softc *sc; struct cryptop *crp; + vm_page_t *pages; u_int i, nsec, secsize; off_t dstoff; - u_char *data; + u_char *data = NULL; int error; + int pages_offset; G_ELI_LOGREQ(3, bp, "%s", __func__); @@ -258,16 +284,39 @@ if (bp->bio_cmd == BIO_WRITE) { data = malloc(bp->bio_length, M_ELI, M_WAITOK); bp->bio_driver2 = data; - bcopy(bp->bio_data, data, bp->bio_length); - } else - data = bp->bio_data; + /* + * This copy could be eliminated by using crypto's output + * buffer, instead of using a single overwriting buffer. + */ + if ((bp->bio_flags & BIO_UNMAPPED) != 0) + g_eli_bio_copyin(bp, data); + else + bcopy(bp->bio_data, data, bp->bio_length); + } else { + if ((bp->bio_flags & BIO_UNMAPPED) != 0) { + pages = bp->bio_ma; + pages_offset = bp->bio_ma_offset; + } else { + data = bp->bio_data; + } + } for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) { crp = crypto_getreq(wr->w_sid, M_WAITOK); - crypto_use_buf(crp, data, secsize); + if (data) { + crypto_use_buf(crp, data, secsize); + data += secsize; + } else { + KASSERT(PMAP_HAS_DMAP, + ("geli can't use unmapped I/O on non-dmap arches")); + MPASS(pages != NULL); + crypto_use_vm_page(crp, pages, secsize, pages_offset); + pages_offset += secsize; + pages += pages_offset >> PAGE_SHIFT; + pages_offset &= PAGE_MASK; + } crp->crp_opaque = (void *)bp; - data += secsize; if (bp->bio_cmd == BIO_WRITE) { crp->crp_op = CRYPTO_OP_ENCRYPT; crp->crp_callback = g_eli_crypto_write_done; Index: sys/kern/subr_bus_dma.c =================================================================== --- sys/kern/subr_bus_dma.c +++ sys/kern/subr_bus_dma.c @@ -46,6 +46,8 @@ #include #include +#include + #include #include #include @@ -660,6 +662,11 @@ case CRYPTO_BUF_UIO: error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs, flags); + break; + case CRYPTO_BUF_VMPAGE: + error = _bus_dmamap_load_ma(dmat, map, cb->cb_vm_page, + cb->cb_vm_page_len, cb->cb_vm_page_offset, flags, NULL, + &nsegs); break; default: error = EINVAL; Index: sys/opencrypto/criov.c =================================================================== --- sys/opencrypto/criov.c +++ sys/opencrypto/criov.c @@ -41,11 +41,17 @@ #include #include +#include + +#include +#include +#include + #include /* - * This macro is only for avoiding code duplication, as we need to skip - * given number of bytes in the same way in three functions below. + * These macros are only for avoiding code duplication, as we need to skip + * given number of bytes in the same way in several functions below. */ #define CUIO_SKIP() do { \ KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \ @@ -59,6 +65,17 @@ iov++; \ } \ } while (0) +#define VM_PAGE_SKIP() do { \ + KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \ + KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \ + while (off > 0) { \ + if (off < PAGE_SIZE) \ + break; \ + processed += PAGE_SIZE - off; \ + off -= PAGE_SIZE - off; \ + pages++; \ + } \ +} while (0) static void cuio_copydata(struct uio* uio, int off, int len, caddr_t cp) @@ -128,6 +145,94 @@ return (-1); } +/* + * Apply function f to the data in a vm_page_t list starting "off" bytes from + * the beginning, continuing for "len" bytes. + */ +static int +vm_page_apply(vm_page_t *pages, int off, int len, + int (*f)(void *, const void *, u_int), void *arg) +{ + int processed = 0; + unsigned count; + int rval; + + VM_PAGE_SKIP(); + while (len > 0) { + char *kaddr = (char*)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)); + count = min(PAGE_SIZE - off, len); + rval = (*f)(arg, kaddr + off, count); + if (rval) + return (rval); + len -= count; + processed += count; + off = 0; + pages++; + } + return (0); +} + +static inline void * +vm_page_contiguous_segment(vm_page_t *pages, size_t skip, int len) +{ + if ((skip + len - 1) / PAGE_SIZE > skip / PAGE_SIZE) + return (NULL); + + pages += (skip / PAGE_SIZE); + skip -= rounddown(skip, PAGE_SIZE); + return (((char*)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages))) + skip); +} + +/* + * Copy len bytes of data from the vm_page_t array, skipping the first off + * bytes, into the pointer cp. Return the number of bytes skipped and copied. + * Does not verify the length of the array. + */ +static int +vm_page_copyback(vm_page_t *pages, int off, int len, c_caddr_t cp) +{ + int processed = 0; + unsigned count; + + VM_PAGE_SKIP(); + while (len > 0) { + count = min(PAGE_SIZE - off, len); + bcopy(cp, (char*)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off, + count); + len -= count; + cp += count; + processed += count; + off = 0; + pages++; + } + return processed; +} + +/* + * Copy len bytes of data from the pointer cp into the vm_page_t array, + * skipping the first off bytes, Return the number of bytes skipped and copied. + * Does not verify the length of the array. + */ +static int +vm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp) +{ + int processed = 0; + unsigned count; + + VM_PAGE_SKIP(); + while (len > 0) { + count = min(PAGE_SIZE - off, len); + bcopy(((char*)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off), cp, + count); + len -= count; + cp += count; + processed += count; + off = 0; + pages++; + } + return processed; +} + void crypto_cursor_init(struct crypto_buffer_cursor *cc, const struct crypto_buffer *cb) @@ -142,6 +247,11 @@ case CRYPTO_BUF_MBUF: cc->cc_mbuf = cb->cb_mbuf; break; + case CRYPTO_BUF_VMPAGE: + cc->cc_vm_page = cb->cb_vm_page; + cc->cc_buf_len = cb->cb_vm_page_len; + cc->cc_offset = cb->cb_vm_page_offset; + break; case CRYPTO_BUF_UIO: cc->cc_iov = cb->cb_uio->uio_iov; break; @@ -178,6 +288,21 @@ break; } break; + case CRYPTO_BUF_VMPAGE: + for (;;) { + remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len); + cc->cc_buf_len -= amount; + if (amount < remain) { + cc->cc_offset += amount; + break; + } + amount -= remain; + cc->cc_vm_page++; + cc->cc_offset = 0; + if (amount == 0) + break; + } + break; case CRYPTO_BUF_UIO: for (;;) { remain = cc->cc_iov->iov_len - cc->cc_offset; @@ -212,6 +337,9 @@ KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0, ("%s: not supported for unmapped mbufs", __func__)); return (mtod(cc->cc_mbuf, char *) + cc->cc_offset); + case CRYPTO_BUF_VMPAGE: + return ((char*)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( + *cc->cc_vm_page)) + cc->cc_offset); case CRYPTO_BUF_UIO: return ((char *)cc->cc_iov->iov_base + cc->cc_offset); default: @@ -228,6 +356,8 @@ switch (cc->cc_type) { case CRYPTO_BUF_CONTIG: return (cc->cc_buf_len); + case CRYPTO_BUF_VMPAGE: + return (PAGE_SIZE - cc->cc_offset); case CRYPTO_BUF_MBUF: if (cc->cc_mbuf == NULL) return (0); @@ -278,6 +408,26 @@ break; } break; + case CRYPTO_BUF_VMPAGE: + for (;;) { + dst = (char*)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( + *cc->cc_vm_page)) + cc->cc_offset; + remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len); + todo = MIN(remain, size); + memcpy(dst, src, todo); + src += todo; + cc->cc_buf_len -= todo; + if (todo < remain) { + cc->cc_offset += todo; + break; + } + size -= todo; + cc->cc_vm_page++; + cc->cc_offset = 0; + if (size == 0) + break; + } + break; case CRYPTO_BUF_UIO: for (;;) { dst = (char *)cc->cc_iov->iov_base + cc->cc_offset; @@ -339,6 +489,26 @@ break; } break; + case CRYPTO_BUF_VMPAGE: + for (;;) { + src = (char*)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( + *cc->cc_vm_page)) + cc->cc_offset; + remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len); + todo = MIN(remain, size); + memcpy(dst, src, todo); + src += todo; + cc->cc_buf_len -= todo; + if (todo < remain) { + cc->cc_offset += todo; + break; + } + size -= todo; + cc->cc_vm_page++; + cc->cc_offset = 0; + if (size == 0) + break; + } + break; case CRYPTO_BUF_UIO: for (;;) { src = (const char *)cc->cc_iov->iov_base + @@ -421,6 +591,12 @@ case CRYPTO_BUF_MBUF: m_copyback(cb->cb_mbuf, off, size, src); break; + case CRYPTO_BUF_VMPAGE: + MPASS(size <= cb->cb_vm_page_len); + MPASS(size + off <= cb->cb_vm_page_len + cb->cb_vm_page_offset); + vm_page_copyback(cb->cb_vm_page, off + cb->cb_vm_page_offset, + size, src); + break; case CRYPTO_BUF_UIO: cuio_copyback(cb->cb_uio, off, size, src); break; @@ -444,6 +620,13 @@ case CRYPTO_BUF_MBUF: m_copydata(crp->crp_buf.cb_mbuf, off, size, dst); break; + case CRYPTO_BUF_VMPAGE: + MPASS(size <= crp->crp_buf.cb_vm_page_len); + MPASS(size + off <= crp->crp_buf.cb_vm_page_len + + crp->crp_buf.cb_vm_page_offset); + vm_page_copydata(crp->crp_buf.cb_vm_page, + off + crp->crp_buf.cb_vm_page_offset, size, dst); + break; case CRYPTO_BUF_UIO: cuio_copydata(crp->crp_buf.cb_uio, off, size, dst); break; @@ -473,6 +656,10 @@ case CRYPTO_BUF_UIO: error = cuio_apply(cb->cb_uio, off, len, f, arg); break; + case CRYPTO_BUF_VMPAGE: + error = vm_page_apply(cb->cb_vm_page, + off + cb->cb_vm_page_offset, len, f, arg); + break; case CRYPTO_BUF_CONTIG: MPASS(off + len <= cb->cb_buf_len); error = (*f)(arg, cb->cb_buf + off, len); @@ -540,6 +727,10 @@ return (m_contiguous_subsegment(cb->cb_mbuf, skip, len)); case CRYPTO_BUF_UIO: return (cuio_contiguous_segment(cb->cb_uio, skip, len)); + case CRYPTO_BUF_VMPAGE: + MPASS(skip + len <= cb->cb_vm_page_len); + return (vm_page_contiguous_segment(cb->cb_vm_page, + skip + cb->cb_vm_page_offset, len)); case CRYPTO_BUF_CONTIG: MPASS(skip + len <= cb->cb_buf_len); return (cb->cb_buf + skip); Index: sys/opencrypto/crypto.c =================================================================== --- sys/opencrypto/crypto.c +++ sys/opencrypto/crypto.c @@ -78,7 +78,9 @@ #include +#include #include + #include #include #include @@ -1240,6 +1242,8 @@ if (cb->cb_mbuf->m_flags & M_PKTHDR) return (cb->cb_mbuf->m_pkthdr.len); return (m_length(cb->cb_mbuf, NULL)); + case CRYPTO_BUF_VMPAGE: + return (cb->cb_vm_page_len); case CRYPTO_BUF_UIO: return (cb->cb_uio->uio_resid); default: @@ -1254,9 +1258,25 @@ { KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST, ("incoming crp with invalid %s buffer type", name)); - if (cb->cb_type == CRYPTO_BUF_CONTIG) + switch (cb->cb_type) { + case CRYPTO_BUF_CONTIG: KASSERT(cb->cb_buf_len >= 0, ("incoming crp with -ve %s buffer length", name)); + break; + case CRYPTO_BUF_VMPAGE: + KASSERT(PMAP_HAS_DMAP, + ("incoming crp uses dmap on supported arch")); + KASSERT(cb->cb_vm_page_len >= 0, + ("incoming crp with -ve %s buffer length", name)); + KASSERT(cb->cb_vm_page_offset >= 0, + ("incoming crp with -ve %s buffer offset", name)); + KASSERT(cb->cb_vm_page_offset < PAGE_SIZE, + ("incoming crp with %s buffer offset greater than page size" + , name)); + break; + default: + break; + } } static void Index: sys/opencrypto/cryptodev.h =================================================================== --- sys/opencrypto/cryptodev.h +++ sys/opencrypto/cryptodev.h @@ -387,7 +387,8 @@ CRYPTO_BUF_CONTIG, CRYPTO_BUF_UIO, CRYPTO_BUF_MBUF, - CRYPTO_BUF_LAST = CRYPTO_BUF_MBUF + CRYPTO_BUF_VMPAGE, + CRYPTO_BUF_LAST = CRYPTO_BUF_VMPAGE }; /* @@ -402,6 +403,11 @@ int cb_buf_len; }; struct mbuf *cb_mbuf; + struct { + vm_page_t *cb_vm_page; + int cb_vm_page_len; + int cb_vm_page_offset; + }; struct uio *cb_uio; }; enum crypto_buffer_type cb_type; @@ -415,11 +421,15 @@ char *cc_buf; struct mbuf *cc_mbuf; struct iovec *cc_iov; + vm_page_t *cc_vm_page; }; - union { - int cc_buf_len; - size_t cc_offset; - }; + /* Optional bytes of valid data remaining */ + int cc_buf_len; + /* + * Optional offset within the current buffer segment where + * valid data begins + */ + size_t cc_offset; enum crypto_buffer_type cc_type; }; @@ -509,6 +519,16 @@ } static __inline void +_crypto_use_vm_page(struct crypto_buffer *cb, vm_page_t *pages, int len, + int offset) +{ + cb->cb_vm_page = pages; + cb->cb_vm_page_len = len; + cb->cb_vm_page_offset = offset; + cb->cb_type = CRYPTO_BUF_VMPAGE; +} + +static __inline void _crypto_use_uio(struct crypto_buffer *cb, struct uio *uio) { cb->cb_uio = uio; @@ -528,6 +548,13 @@ } static __inline void +crypto_use_vm_page(struct cryptop *crp, vm_page_t *pages, int len, + int offset) +{ + _crypto_use_vm_page(&crp->crp_buf, pages, len, offset); +} + +static __inline void crypto_use_uio(struct cryptop *crp, struct uio *uio) { _crypto_use_uio(&crp->crp_buf, uio); @@ -543,6 +570,13 @@ crypto_use_output_mbuf(struct cryptop *crp, struct mbuf *m) { _crypto_use_mbuf(&crp->crp_obuf, m); +} + +static __inline void +crypto_use_output_vm_page(struct cryptop *crp, vm_page_t *pages, int len, + int offset) +{ + _crypto_use_vm_page(&crp->crp_obuf, pages, len, offset); } static __inline void Index: sys/opencrypto/cryptosoft.c =================================================================== --- sys/opencrypto/cryptosoft.c +++ sys/opencrypto/cryptosoft.c @@ -47,6 +47,8 @@ #include #include +#include + #include #include @@ -979,6 +981,10 @@ uio->uio_iovcnt--; } } + break; + case CRYPTO_BUF_VMPAGE: + adj = crp->crp_payload_length - result; + crp->crp_buf.cb_vm_page_len -= adj; break; default: break;