Page MenuHomeFreeBSD
Differential D12452 Diff 33337 sys/crypto/sha_sse/sha_sse.c

Changeset View

Standalone View

View Options

sys/crypto/sha_sse/sha_sse.c

  • This file was added.
/*-
* Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
* Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
* Copyright (c) 2014 The FreeBSD Foundation
* Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
* All rights reserved.
*
* Portions of this software were developed by John-Mark Gurney
* under sponsorship of the FreeBSD Foundation and
* Rubicon Communications, LLC (Netgate).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/kobj.h>
#include <sys/libkern.h>
#include <sys/lock.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/rwlock.h>
#include <sys/bus.h>
#include <sys/uio.h>
#include <sys/mbuf.h>
#include <sys/smp.h>
#include <crypto/sha1.h>
#include <crypto/sha2/sha256.h>
#include <crypto/sha_sse/sha_sse.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/gmac.h>
#include <cryptodev_if.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
#if defined(__i386__)
#include <machine/npx.h>
#elif defined(__amd64__)
#include <machine/fpu.h>
#endif
static struct mtx_padalign *ctx_mtx;
static struct fpu_kern_ctx **ctx_fpu;
struct sha_sse_softc {
int dieing;
int32_t cid;
uint32_t sid;
struct rwlock lock;
TAILQ_HEAD(sha_sse_sessions_head, sha_sse_session) sessions;
};
struct sha_sse_session {
/* Same as the SHA256 Blocksize. */
uint8_t key[SHA1_HMAC_BLOCK_LEN] __aligned(16);
int algo;
int used;
int mlen;
uint32_t id;
TAILQ_ENTRY(sha_sse_session) next;
};
#define ACQUIRE_CTX(i, ctx) \
do { \
kibUnsubmitted
Not Done
Inline Actions

This locking should be unified between aes and sha drivers. Best would be to move the stuff into the common header and probably some support subroutines.

kib: This locking should be unified between aes and sha drivers. Best would be to move the stuff…
cemAuthorUnsubmitted
Not Done
Inline Actions

Seems like an orthogonal cleanup. Any suggestion where it should go?

cem: Seems like an orthogonal cleanup. Any suggestion where it should go?
(i) = PCPU_GET(cpuid); \
mtx_lock(&ctx_mtx[(i)]); \
(ctx) = ctx_fpu[(i)]; \
} while (0)
#define RELEASE_CTX(i, ctx) \
do { \
mtx_unlock(&ctx_mtx[(i)]); \
(i) = -1; \
(ctx) = NULL; \
} while (0)
static int sha_sse_newsession(device_t, uint32_t *sidp, struct cryptoini *cri);
static int sha_sse_freesession(device_t, uint64_t tid);
static void sha_sse_freesession_locked(struct sha_sse_softc *sc,
struct sha_sse_session *ses);
static int sha_sse_cipher_setup(struct sha_sse_session *ses,
struct cryptoini *encini);
static int sha_sse_cipher_process(struct sha_sse_session *ses,
struct cryptop *crp);
MALLOC_DEFINE(M_SHA_SSE, "sha_sse_data", "SHA_SSE Data");
static void
sha_sse_identify(driver_t *drv, device_t parent)
{
/* NB: order 10 is so we get attached after h/w devices */
if (device_find_child(parent, "sha_sse", -1) == NULL &&
BUS_ADD_CHILD(parent, 10, "sha_sse", -1) == 0)
panic("sha_sse: could not attach");
}
static int
sha_sse_probe(device_t dev)
{
if ((cpu_stdext_feature & CPUID_STDEXT_SHA) == 0) {
device_printf(dev, "No SHA support.\n");
return (EINVAL);
}
if ((cpu_feature2 & CPUID2_SSSE3) == 0) {
device_printf(dev, "No SSSE3 support.\n");
return (EINVAL);
}
device_set_desc_copy(dev, "SHA1,SHA2");
return (0);
}
static void
sha_sse_cleanctx(void)
{
int i;
/* XXX - no way to return driverid */
CPU_FOREACH(i) {
if (ctx_fpu[i] != NULL) {
mtx_destroy(&ctx_mtx[i]);
fpu_kern_free_ctx(ctx_fpu[i]);
}
ctx_fpu[i] = NULL;
}
free(ctx_mtx, M_SHA_SSE);
ctx_mtx = NULL;
free(ctx_fpu, M_SHA_SSE);
ctx_fpu = NULL;
}
static int
sha_sse_attach(device_t dev)
{
struct sha_sse_softc *sc;
int i;
sc = device_get_softc(dev);
sc->dieing = 0;
TAILQ_INIT(&sc->sessions);
sc->sid = 1;
sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE |
CRYPTOCAP_F_SYNC);
if (sc->cid < 0) {
device_printf(dev, "Could not get crypto driver id.\n");
return (ENOMEM);
}
ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_SHA_SSE,
M_WAITOK | M_ZERO);
ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_SHA_SSE,
M_WAITOK | M_ZERO);
CPU_FOREACH(i) {
ctx_fpu[i] = fpu_kern_alloc_ctx(0);
mtx_init(&ctx_mtx[i], "shafpumtx", NULL, MTX_DEF | MTX_NEW);
}
rw_init(&sc->lock, "sha_sse_lock");
crypto_register(sc->cid, CRYPTO_SHA1, 0, 0);
crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
return (0);
}
static int
sha_sse_detach(device_t dev)
{
struct sha_sse_softc *sc;
struct sha_sse_session *ses;
sc = device_get_softc(dev);
rw_wlock(&sc->lock);
TAILQ_FOREACH(ses, &sc->sessions, next) {
if (ses->used) {
rw_wunlock(&sc->lock);
device_printf(dev,
"Cannot detach, sessions still active.\n");
return (EBUSY);
}
}
sc->dieing = 1;
while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) {
TAILQ_REMOVE(&sc->sessions, ses, next);
free(ses, M_SHA_SSE);
}
rw_wunlock(&sc->lock);
crypto_unregister_all(sc->cid);
rw_destroy(&sc->lock);
sha_sse_cleanctx();
return (0);
}
static int
sha_sse_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
{
struct sha_sse_softc *sc;
struct sha_sse_session *ses;
struct cryptoini *encini;
int error;
if (sidp == NULL || cri == NULL) {
CRYPTDEB("no sidp or cri");
return (EINVAL);
}
sc = device_get_softc(dev);
if (sc->dieing)
return (EINVAL);
ses = NULL;
encini = NULL;
for (; cri != NULL; cri = cri->cri_next) {
switch (cri->cri_alg) {
case CRYPTO_SHA1:
case CRYPTO_SHA1_HMAC:
case CRYPTO_SHA2_256_HMAC:
if (encini != NULL) {
CRYPTDEB("encini already set");
return (EINVAL);
}
encini = cri;
break;
default:
CRYPTDEB("unhandled algorithm");
return (EINVAL);
}
}
if (encini == NULL) {
CRYPTDEB("no cipher");
return (EINVAL);
}
rw_wlock(&sc->lock);
if (sc->dieing) {
rw_wunlock(&sc->lock);
return (EINVAL);
}
/*
* Free sessions goes first, so if first session is used, we need to
* allocate one.
*/
ses = TAILQ_FIRST(&sc->sessions);
if (ses == NULL || ses->used) {
ses = malloc(sizeof(*ses), M_SHA_SSE, M_NOWAIT | M_ZERO);
if (ses == NULL) {
rw_wunlock(&sc->lock);
return (ENOMEM);
}
ses->id = sc->sid++;
} else {
TAILQ_REMOVE(&sc->sessions, ses, next);
}
ses->used = 1;
TAILQ_INSERT_TAIL(&sc->sessions, ses, next);
rw_wunlock(&sc->lock);
ses->algo = encini->cri_alg;
error = sha_sse_cipher_setup(ses, encini);
if (error != 0) {
CRYPTDEB("setup failed");
rw_wlock(&sc->lock);
sha_sse_freesession_locked(sc, ses);
rw_wunlock(&sc->lock);
return (error);
}
*sidp = ses->id;
return (0);
}
static void
sha_sse_freesession_locked(struct sha_sse_softc *sc, struct sha_sse_session *ses)
{
uint32_t sid;
rw_assert(&sc->lock, RA_WLOCKED);
sid = ses->id;
TAILQ_REMOVE(&sc->sessions, ses, next);
explicit_bzero(ses, sizeof(*ses));
ses->id = sid;
TAILQ_INSERT_HEAD(&sc->sessions, ses, next);
}
static int
sha_sse_freesession(device_t dev, uint64_t tid)
{
struct sha_sse_softc *sc;
struct sha_sse_session *ses;
uint32_t sid;
sc = device_get_softc(dev);
sid = ((uint32_t)tid) & 0xffffffff;
rw_wlock(&sc->lock);
TAILQ_FOREACH_REVERSE(ses, &sc->sessions, sha_sse_sessions_head, next) {
if (ses->id == sid)
break;
}
if (ses == NULL) {
rw_wunlock(&sc->lock);
return (EINVAL);
}
sha_sse_freesession_locked(sc, ses);
rw_wunlock(&sc->lock);
return (0);
}
static int
sha_sse_process(device_t dev, struct cryptop *crp, int hint __unused)
{
struct sha_sse_softc *sc = device_get_softc(dev);
struct sha_sse_session *ses = NULL;
int error;
error = 0;
/* Sanity check. */
if (crp == NULL)
return (EINVAL);
if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
error = EINVAL;
goto out;
}
rw_rlock(&sc->lock);
TAILQ_FOREACH_REVERSE(ses, &sc->sessions, sha_sse_sessions_head, next) {
if (ses->id == (crp->crp_sid & 0xffffffff))
break;
}
rw_runlock(&sc->lock);
if (ses == NULL) {
error = EINVAL;
goto out;
}
error = sha_sse_cipher_process(ses, crp);
if (error != 0)
goto out;
out:
crp->crp_etype = error;
crypto_done(crp);
return (error);
}
static device_method_t sha_sse_methods[] = {
DEVMETHOD(device_identify, sha_sse_identify),
DEVMETHOD(device_probe, sha_sse_probe),
DEVMETHOD(device_attach, sha_sse_attach),
DEVMETHOD(device_detach, sha_sse_detach),
DEVMETHOD(cryptodev_newsession, sha_sse_newsession),
DEVMETHOD(cryptodev_freesession, sha_sse_freesession),
DEVMETHOD(cryptodev_process, sha_sse_process),
{0, 0},
};
static driver_t sha_sse_driver = {
"sha_sse",
sha_sse_methods,
sizeof(struct sha_sse_softc),
};
static devclass_t sha_sse_devclass;
DRIVER_MODULE(sha_sse, nexus, sha_sse_driver, sha_sse_devclass, 0, 0);
MODULE_VERSION(sha_sse, 1);
MODULE_DEPEND(sha_sse, crypto, 1, 1, 1);
static int
sha_sse_cipher_setup(struct sha_sse_session *ses, struct cryptoini *encini)
{
int error, keylen;
keylen = encini->cri_klen / 8;
if (keylen > sizeof(ses->key)) {
printf("%s: keylen:%d ses->key:%zu\n", __func__, keylen, sizeof(ses->key));
error = EINVAL;
goto out;
}
if (ses->algo == CRYPTO_SHA1 && keylen > 0) {
printf("%s: algo:%d keylen:%d\n", __func__, ses->algo, keylen);
error = EINVAL;
goto out;
}
rlibbyUnsubmitted
Done
Inline Actions

Do we actually need a fpu_kern_enter in this procedure? It doesn't seem to be used. And there's no requirement from the API that the same thread that does setup does process, right?

rlibby: Do we actually need a `fpu_kern_enter` in this procedure? It doesn't seem to be used. And…
cemAuthorUnsubmitted
Not Done
Inline Actions

Guess not — presumably it was used in aesni for key scheduling. I will remove it.

I don't believe there's any requirement from the API that the same thread does process and setup.

cem: Guess not — presumably it was used in aesni for key scheduling. I will remove it. I don't…
memcpy(ses->key, encini->cri_key, keylen);
ses->mlen = encini->cri_mlen;
out:
return (error);
}
static int
intel_sha1_update(void *vctx, const void *vdata, u_int datalen)
{
struct sha1_ctxt *ctx = vctx;
const char *data = vdata;
size_t gaplen;
size_t gapstart;
size_t off;
size_t copysiz;
u_int blocks;
off = 0;
/* Do any aligned blocks without redundant copying. */
if (datalen >= 64 && ctx->count % 64 == 0) {
blocks = datalen / 64;
ctx->c.b64[0] += blocks * 64 * 8;
intel_sha1_step(ctx->h.b32, data + off, blocks);
off += blocks * 64;
}
while (off < datalen) {
gapstart = ctx->count % 64;
gaplen = 64 - gapstart;
copysiz = (gaplen < datalen - off) ? gaplen : datalen - off;
bcopy(&data[off], &ctx->m.b8[gapstart], copysiz);
ctx->count += copysiz;
ctx->count %= 64;
ctx->c.b64[0] += copysiz * 8;
if (ctx->count % 64 == 0)
intel_sha1_step(ctx->h.b32, (void *)ctx->m.b8, 1);
off += copysiz;
}
return (0);
}
static void
SHA1_Finalize_fn(void *digest, void *ctx)
{
sha1_result(ctx, digest);
}
static int
intel_sha256_update(void *vctx, const void *vdata, u_int len)
{
SHA256_CTX *ctx = vctx;
uint64_t bitlen;
uint32_t r;
u_int blocks;
const unsigned char *src = vdata;
/* Number of bytes left in the buffer from previous updates */
r = (ctx->count >> 3) & 0x3f;
/* Convert the length into a number of bits */
bitlen = len << 3;
/* Update number of bits */
ctx->count += bitlen;
/* Handle the case where we don't need to perform any transforms */
if (len < 64 - r) {
memcpy(&ctx->buf[r], src, len);
return (0);
}
/* Finish the current block */
memcpy(&ctx->buf[r], src, 64 - r);
intel_sha256_step(ctx->state, ctx->buf, 1);
src += 64 - r;
len -= 64 - r;
/* Perform complete blocks */
if (len >= 64) {
blocks = len / 64;
intel_sha256_step(ctx->state, src, blocks);
src += blocks * 64;
len -= blocks * 64;
}
/* Copy left over data into buffer */
memcpy(ctx->buf, src, len);
return (0);
}
static void
SHA256_Finalize_fn(void *digest, void *ctx)
{
SHA256_Final(digest, ctx);
}
/*
* Compute the HASH( (key ^ xorbyte) || buf )
*/
static void
hmac_internal(void *ctx, uint32_t *res,
int (*update)(void *, const void *, u_int),
void (*finalize)(void *, void *), uint8_t *key, uint8_t xorbyte,
const void *buf, size_t off, size_t buflen, int crpflags)
{
size_t i;
for (i = 0; i < 64; i++)
key[i] ^= xorbyte;
update(ctx, key, 64);
for (i = 0; i < 64; i++)
key[i] ^= xorbyte;
crypto_apply(crpflags, __DECONST(void *, buf), off, buflen,
__DECONST(int (*)(void *, void *, u_int), update), ctx);
finalize(res, ctx);
}
static int
sha_sse_cipher_process(struct sha_sse_session *ses, struct cryptop *crp)
{
struct SHA256Context s2ctx;
struct sha1_ctxt sctx __aligned(16);
uint32_t res[8];
struct fpu_kern_ctx *ctx;
struct cryptodesc *crd;
int error, kt, ctxidx, hashlen;
error = 0;
kt = is_fpu_kern_thread(0);
if (!kt) {
ACQUIRE_CTX(ctxidx, ctx);
error = fpu_kern_enter(curthread, ctx,
FPU_KERN_NORMAL|FPU_KERN_KTHR);
if (error != 0)
goto out2;
}
crd = crp->crp_desc;
if (crd->crd_next != NULL || crd->crd_flags != 0) {
error = EINVAL;
goto out;
}
switch (ses->algo) {
case CRYPTO_SHA1_HMAC:
hashlen = SHA1_HASH_LEN;
/* Inner hash: (K ^ IPAD) || data */
sha1_init(&sctx);
hmac_internal(&sctx, res, intel_sha1_update, SHA1_Finalize_fn,
ses->key, 0x36, crp->crp_buf, crd->crd_skip, crd->crd_len,
crp->crp_flags);
/* Outer hash: (K ^ OPAD) || inner hash */
sha1_init(&sctx);
hmac_internal(&sctx, res, intel_sha1_update, SHA1_Finalize_fn,
ses->key, 0x5C, res, 0, hashlen, 0);
break;
case CRYPTO_SHA1:
hashlen = SHA1_HASH_LEN;
sha1_init(&sctx);
crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
crd->crd_len, __DECONST(int (*)(void *, void *, u_int),
intel_sha1_update), &sctx);
sha1_result(&sctx, (void *)res);
break;
case CRYPTO_SHA2_256_HMAC:
hashlen = SHA2_256_HASH_LEN;
/* Inner hash: (K ^ IPAD) || data */
SHA256_Init(&s2ctx);
hmac_internal(&s2ctx, res, intel_sha256_update,
SHA256_Finalize_fn, ses->key, 0x36, crp->crp_buf,
crd->crd_skip, crd->crd_len, crp->crp_flags);
/* Outer hash: (K ^ OPAD) || inner hash */
SHA256_Init(&s2ctx);
hmac_internal(&s2ctx, res, intel_sha256_update,
SHA256_Finalize_fn, ses->key, 0x5C, res, 0, hashlen, 0);
break;
}
if (ses->mlen != 0 && ses->mlen < hashlen)
hashlen = ses->mlen;
crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, hashlen,
(void *)res);
out:
if (!kt) {
fpu_kern_leave(curthread, ctx);
out2:
RELEASE_CTX(ctxidx, ctx);
}
return (error);
}