Page MenuHomeFreeBSD
Differential D27454 Diff 82193 sys/crypto/armv8/armv8_crypto.c

Changeset View

Standalone View

View Options

sys/crypto/armv8/armv8_crypto.c

/*- /*-
* Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org> * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
* Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org> * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
* Copyright (c) 2014,2016 The FreeBSD Foundation * Copyright (c) 2014,2016 The FreeBSD Foundation
* Copyright (c) 2020 Ampere Computing
* All rights reserved. * All rights reserved.
* *
* Portions of this software were developed by John-Mark Gurney * Portions of this software were developed by John-Mark Gurney
* under sponsorship of the FreeBSD Foundation and * under sponsorship of the FreeBSD Foundation and
* Rubicon Communications, LLC (Netgate). * Rubicon Communications, LLC (Netgate).
* *
* This software was developed by Andrew Turner under * This software was developed by Andrew Turner under
* sponsorship from the FreeBSD Foundation. * sponsorship from the FreeBSD Foundation.
Show All 40 Lines
#include <sys/queue.h> #include <sys/queue.h>
#include <sys/rwlock.h> #include <sys/rwlock.h>
#include <sys/smp.h> #include <sys/smp.h>
#include <sys/uio.h> #include <sys/uio.h>
#include <machine/vfp.h> #include <machine/vfp.h>
#include <opencrypto/cryptodev.h> #include <opencrypto/cryptodev.h>
#include <opencrypto/gmac.h>
#include <cryptodev_if.h> #include <cryptodev_if.h>
#include <crypto/armv8/armv8_crypto.h> #include <crypto/armv8/armv8_crypto.h>
#include <crypto/rijndael/rijndael.h> #include <crypto/rijndael/rijndael.h>
struct armv8_crypto_softc { struct armv8_crypto_softc {
int dieing; int dieing;
int32_t cid; int32_t cid;
struct rwlock lock; struct rwlock lock;
bool has_pmul;
}; };
static struct mtx *ctx_mtx; static struct mtx *ctx_mtx;
static struct fpu_kern_ctx **ctx_vfp; static struct fpu_kern_ctx **ctx_vfp;
#define AQUIRE_CTX(i, ctx) \ #define AQUIRE_CTX(i, ctx) \
do { \ do { \
(i) = PCPU_GET(cpuid); \ (i) = PCPU_GET(cpuid); \
Show All 27 Lines
{ {
uint64_t reg; uint64_t reg;
int ret = ENXIO; int ret = ENXIO;
reg = READ_SPECIALREG(id_aa64isar0_el1); reg = READ_SPECIALREG(id_aa64isar0_el1);
switch (ID_AA64ISAR0_AES_VAL(reg)) { switch (ID_AA64ISAR0_AES_VAL(reg)) {
case ID_AA64ISAR0_AES_BASE: case ID_AA64ISAR0_AES_BASE:
ret = 0;
device_set_desc_copy(dev, "AES-CBC,AES-XTS");
jhbUnsubmitted
Not Done
Inline Actions

This can just use 'device_set_desc'. It's not a dynamic string, so a copy isn't required. That is an old bug, but not worth perpetuating further.

jhb: This can just use 'device_set_desc'. It's not a dynamic string, so a copy isn't required.
break;
case ID_AA64ISAR0_AES_PMULL: case ID_AA64ISAR0_AES_PMULL:
ret = 0; ret = 0;
device_set_desc_copy(dev, "AES-CBC,AES-XTS,AES-GCM");
break; break;
default:
break;
} }
device_set_desc_copy(dev, "AES-CBC,AES-XTS");
/* TODO: Check more fields as we support more features */ /* TODO: Check more fields as we support more features */
return (ret); return (ret);
} }
static int static int
armv8_crypto_attach(device_t dev) armv8_crypto_attach(device_t dev)
{ {
struct armv8_crypto_softc *sc; struct armv8_crypto_softc *sc;
uint64_t reg;
int i; int i;
sc = device_get_softc(dev); sc = device_get_softc(dev);
sc->dieing = 0; sc->dieing = 0;
reg = READ_SPECIALREG(id_aa64isar0_el1);
if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL)
sc->has_pmul = true;
sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session), sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE); CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
if (sc->cid < 0) { if (sc->cid < 0) {
device_printf(dev, "Could not get crypto driver id.\n"); device_printf(dev, "Could not get crypto driver id.\n");
return (ENOMEM); return (ENOMEM);
} }
rw_init(&sc->lock, "armv8crypto"); rw_init(&sc->lock, "armv8crypto");
Show All 36 Linesarmv8_crypto_detach(device_t dev)
free(ctx_mtx, M_ARMV8_CRYPTO); free(ctx_mtx, M_ARMV8_CRYPTO);
ctx_mtx = NULL; ctx_mtx = NULL;
free(ctx_vfp, M_ARMV8_CRYPTO); free(ctx_vfp, M_ARMV8_CRYPTO);
ctx_vfp = NULL; ctx_vfp = NULL;
return (0); return (0);
} }
#define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)
static int static int
armv8_crypto_probesession(device_t dev, armv8_crypto_probesession(device_t dev,
const struct crypto_session_params *csp) const struct crypto_session_params *csp)
{ {
struct armv8_crypto_softc *sc;
if (csp->csp_flags != 0) sc = device_get_softc(dev);
if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
return (EINVAL); return (EINVAL);
switch (csp->csp_mode) { switch (csp->csp_mode) {
case CSP_MODE_AEAD:
switch (csp->csp_cipher_alg) {
jhbUnsubmitted
Done
Inline Actions

This should also check csp_flags and only accept the flags it supports (for KTLS you will want CSP_F_SEPARATE_AAD and CSP_F_SEPARATE_OUTPUT IIRC). This makes it future-proof against new flags being added in the future.

jhb: This should also check `csp_flags` and only accept the flags it supports (for KTLS you will…
case CRYPTO_AES_NIST_GCM_16:
if (!sc->has_pmul)
return (EINVAL);
if (csp->csp_ivlen != AES_GCM_IV_LEN)
return (EINVAL);
if (csp->csp_auth_mlen != 0 &&
csp->csp_auth_mlen != GMAC_DIGEST_LEN)
return (EINVAL);
switch (csp->csp_cipher_klen * 8) {
case 128:
case 192:
case 256:
break;
default:
return (EINVAL);
}
break;
default:
return (EINVAL);
}
break;
case CSP_MODE_CIPHER: case CSP_MODE_CIPHER:
switch (csp->csp_cipher_alg) { switch (csp->csp_cipher_alg) {
jhbUnsubmitted
Done
Inline Actions

It seems like you support separate output buffers with AES-CBC in your changes, so you should perhaps just keep the csp_flags check in one place and whitelist the flags supported there the way aesni(4) does?

jhb: It seems like you support separate output buffers with AES-CBC in your changes, so you should…
case CRYPTO_AES_CBC: case CRYPTO_AES_CBC:
if (csp->csp_ivlen != AES_BLOCK_LEN) if (csp->csp_ivlen != AES_BLOCK_LEN)
return (EINVAL); return (EINVAL);
switch (csp->csp_cipher_klen * 8) { switch (csp->csp_cipher_klen * 8) {
case 128: case 128:
case 192: case 192:
case 256: case 256:
break; break;
Show All 17 Lines case CSP_MODE_CIPHER:
} }
break; break;
default: default:
return (EINVAL); return (EINVAL);
} }
return (CRYPTODEV_PROBE_ACCEL_SOFTWARE); return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
} }
static void static int
armv8_crypto_cipher_setup(struct armv8_crypto_session *ses, armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
const struct crypto_session_params *csp, const uint8_t *key, int keylen) const struct crypto_session_params *csp, const uint8_t *key, int keylen)
{ {
int i; __uint128_val_t H;
struct fpu_kern_ctx *ctx;
int kt, i;
if (csp->csp_cipher_alg == CRYPTO_AES_XTS) if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
keylen /= 2; keylen /= 2;
switch (keylen * 8) { switch (keylen * 8) {
case 128: case 128:
ses->rounds = AES128_ROUNDS;
break;
case 192: case 192:
ses->rounds = AES192_ROUNDS;
break;
case 256: case 256:
ses->rounds = AES256_ROUNDS;
break; break;
default: default:
panic("invalid AES key length"); return (EINVAL);
} }
rijndaelKeySetupEnc(ses->enc_schedule, key, keylen * 8); kt = is_fpu_kern_thread(0);
rijndaelKeySetupDec(ses->dec_schedule, key, keylen * 8); if (!kt) {
if (csp->csp_cipher_alg == CRYPTO_AES_XTS) AQUIRE_CTX(i, ctx);
rijndaelKeySetupEnc(ses->xts_schedule, key + keylen, keylen * 8); fpu_kern_enter(curthread, ctx,
FPU_KERN_NORMAL | FPU_KERN_KTHR);
}
for (i = 0; i < nitems(ses->enc_schedule); i++) { aes_v8_set_encrypt_key(key,
ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]); keylen * 8, &ses->enc_schedule);
ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]);
if (csp->csp_cipher_alg == CRYPTO_AES_XTS) if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) ||
ses->xts_schedule[i] = bswap32(ses->xts_schedule[i]); (csp->csp_cipher_alg == CRYPTO_AES_CBC))
aes_v8_set_decrypt_key(key,
keylen * 8, &ses->dec_schedule);
if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
memset(H.c, 0, sizeof(H.c));
aes_v8_encrypt(H.c, H.c, &ses->enc_schedule);
H.u[0] = bswap64(H.u[0]);
H.u[1] = bswap64(H.u[1]);
gcm_init_v8(ses->Htable, H.u);
} }
if (!kt) {
fpu_kern_leave(curthread, ctx);
RELEASE_CTX(i, ctx);
} }
return (0);
jhbUnsubmitted
Done
Inline Actions

Should some of this logic be conditional on the session? E.g. only initializing H for GCM sessions, and only initializing a decrypt key for AES-CBC (and XTS). AES-CTR (which GCM and CCM both use for their cipher side) does not use a separate decryption key schedule but instead uses the encryption key to generate a keystream xored with the cipher/plain text in both directions.

jhb: Should some of this logic be conditional on the session? E.g. only initializing H for GCM…
}
static int static int
armv8_crypto_newsession(device_t dev, crypto_session_t cses, armv8_crypto_newsession(device_t dev, crypto_session_t cses,
const struct crypto_session_params *csp) const struct crypto_session_params *csp)
{ {
struct armv8_crypto_softc *sc; struct armv8_crypto_softc *sc;
struct armv8_crypto_session *ses; struct armv8_crypto_session *ses;
int error;
jhbUnsubmitted
Done
Inline Actions

Perhaps call this error as that is more common in the kernel?

jhb: Perhaps call this `error` as that is more common in the kernel?
sc = device_get_softc(dev); sc = device_get_softc(dev);
rw_wlock(&sc->lock); rw_wlock(&sc->lock);
if (sc->dieing) { if (sc->dieing) {
rw_wunlock(&sc->lock); rw_wunlock(&sc->lock);
return (EINVAL); return (EINVAL);
} }
ses = crypto_get_driver_session(cses); ses = crypto_get_driver_session(cses);
armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key, error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
csp->csp_cipher_klen); csp->csp_cipher_klen);
rw_wunlock(&sc->lock); rw_wunlock(&sc->lock);
return (0); return (error);
} }
static int static int
armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused) armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
{ {
struct armv8_crypto_session *ses; struct armv8_crypto_session *ses;
int error;
/* We can only handle full blocks for now */
if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
error = EINVAL;
goto out;
}
ses = crypto_get_driver_session(crp->crp_session); ses = crypto_get_driver_session(crp->crp_session);
error = armv8_crypto_cipher_process(ses, crp); crp->crp_etype = armv8_crypto_cipher_process(ses, crp);
out:
crp->crp_etype = error;
crypto_done(crp); crypto_done(crp);
return (0); return (0);
} }
static uint8_t * static uint8_t *
armv8_crypto_cipher_alloc(struct cryptop *crp, int *allocated) armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated)
{ {
uint8_t *addr; uint8_t *addr;
addr = crypto_contiguous_subsegment(crp, crp->crp_payload_start, addr = crypto_contiguous_subsegment(crp, start, length);
crp->crp_payload_length);
if (addr != NULL) { if (addr != NULL) {
*allocated = 0; *allocated = 0;
return (addr); return (addr);
} }
addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT); addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
if (addr != NULL) { if (addr != NULL) {
*allocated = 1; *allocated = 1;
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, start, length, addr);
crp->crp_payload_length, addr);
} else } else
*allocated = 0; *allocated = 0;
return (addr); return (addr);
} }
static int static int
armv8_crypto_cipher_process(struct armv8_crypto_session *ses, armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
struct cryptop *crp) struct cryptop *crp)
{ {
const struct crypto_session_params *csp; const struct crypto_session_params *csp;
struct fpu_kern_ctx *ctx; struct fpu_kern_ctx *ctx;
uint8_t *buf; uint8_t *buf, *authbuf, *outbuf;
uint8_t iv[AES_BLOCK_LEN]; uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
int allocated, i; int allocated, authallocated, outallocated, i;
int encflag; int encflag;
int kt; int kt;
int error;
bool outcopy;
csp = crypto_get_params(crp->crp_session); csp = crypto_get_params(crp->crp_session);
encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
buf = armv8_crypto_cipher_alloc(crp, &allocated); allocated = 0;
outallocated = 0;
authallocated = 0;
authbuf = NULL;
kt = 1;
buf = armv8_crypto_cipher_alloc(crp, crp->crp_payload_start,
crp->crp_payload_length, &allocated);
if (buf == NULL) if (buf == NULL)
return (ENOMEM); return (ENOMEM);
if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
if (crp->crp_aad != NULL)
authbuf = crp->crp_aad;
else
authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start,
crp->crp_aad_length, &authallocated);
if (authbuf == NULL) {
error = ENOMEM;
goto out;
}
}
if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf,
crp->crp_payload_output_start, crp->crp_payload_length);
if (outbuf == NULL) {
outcopy = true;
if (allocated)
outbuf = buf;
else {
outbuf = malloc(crp->crp_payload_length,
M_ARMV8_CRYPTO, M_NOWAIT);
if (outbuf == NULL) {
error = ENOMEM;
goto out;
}
outallocated = true;
}
} else
outcopy = false;
} else {
outbuf = buf;
outcopy = allocated;
}
kt = is_fpu_kern_thread(0); kt = is_fpu_kern_thread(0);
if (!kt) { if (!kt) {
AQUIRE_CTX(i, ctx); AQUIRE_CTX(i, ctx);
fpu_kern_enter(curthread, ctx, fpu_kern_enter(curthread, ctx,
FPU_KERN_NORMAL | FPU_KERN_KTHR); FPU_KERN_NORMAL | FPU_KERN_KTHR);
} }
if (crp->crp_cipher_key != NULL) { if (crp->crp_cipher_key != NULL) {
armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key, armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
csp->csp_cipher_klen); csp->csp_cipher_klen);
} }
crypto_read_iv(crp, iv); crypto_read_iv(crp, iv);
/* Do work */ /* Do work */
switch (csp->csp_cipher_alg) { switch (csp->csp_cipher_alg) {
jhbUnsubmitted
Done
Inline Actions

I think you can instead remove the err == 0 check before the copy for the output buffer and set 'err = 0' just before the out label since all the places that set err jump to out. I would probably also spell it error.

jhb: I think you can instead remove the `err == 0` check before the copy for the output buffer and…
case CRYPTO_AES_CBC: case CRYPTO_AES_CBC:
if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
error = EINVAL;
goto out;
}
if (encflag) if (encflag)
armv8_aes_encrypt_cbc(ses->rounds, ses->enc_schedule, armv8_aes_encrypt_cbc(&ses->enc_schedule,
crp->crp_payload_length, buf, buf, iv); crp->crp_payload_length, buf, buf, iv);
else else
armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule, armv8_aes_decrypt_cbc(&ses->dec_schedule,
crp->crp_payload_length, buf, iv); crp->crp_payload_length, buf, iv);
break; break;
case CRYPTO_AES_XTS: case CRYPTO_AES_XTS:
if (encflag) if (encflag)
armv8_aes_encrypt_xts(ses->rounds, ses->enc_schedule, armv8_aes_encrypt_xts(&ses->enc_schedule,
ses->xts_schedule, crp->crp_payload_length, buf, &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
buf, iv); buf, iv);
else else
armv8_aes_decrypt_xts(ses->rounds, ses->dec_schedule, armv8_aes_decrypt_xts(&ses->dec_schedule,
ses->xts_schedule, crp->crp_payload_length, buf, &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
buf, iv); buf, iv);
break; break;
case CRYPTO_AES_NIST_GCM_16:
if (encflag) {
jhbUnsubmitted
Done
Inline Actions

It's really nicer to reject this in the probesession callback instead. Then you are only checking once per session (and can fall back to another driver) rather than checking once per-op (and failing every request sent if the check fails).

jhb: It's really nicer to reject this in the probesession callback instead. Then you are only…
memset(tag, 0, sizeof(tag));
armv8_aes_encrypt_gcm(&ses->enc_schedule,
crp->crp_payload_length,
buf, outbuf,
crp->crp_aad_length, authbuf,
jhbUnsubmitted
Done
Inline Actions

Same, though I think you already check this in probesession, so there's no need to re-check it here.

jhb: Same, though I think you already check this in probesession, so there's no need to re-check it…
tag, iv, ses->Htable);
crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
tag);
} else {
crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
tag);
if (armv8_aes_decrypt_gcm(&ses->enc_schedule,
crp->crp_payload_length,
buf, outbuf,
crp->crp_aad_length, authbuf,
tag, iv, ses->Htable) != 0) {
error = EBADMSG;
goto out;
} }
}
break;
}
if (allocated) if (outcopy)
crypto_copyback(crp, crp->crp_payload_start, crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ?
crp->crp_payload_length, buf); crp->crp_payload_output_start : crp->crp_payload_start,
crp->crp_payload_length, outbuf);
error = 0;
out:
if (!kt) { if (!kt) {
fpu_kern_leave(curthread, ctx); fpu_kern_leave(curthread, ctx);
RELEASE_CTX(i, ctx); RELEASE_CTX(i, ctx);
} }
if (allocated) if (allocated)
zfree(buf, M_ARMV8_CRYPTO); zfree(buf, M_ARMV8_CRYPTO);
return (0); if (authallocated)
zfree(authbuf, M_ARMV8_CRYPTO);
if (outallocated)
zfree(outbuf, M_ARMV8_CRYPTO);
explicit_bzero(iv, sizeof(iv));
explicit_bzero(tag, sizeof(tag));
return (error);
} }
static device_method_t armv8_crypto_methods[] = { static device_method_t armv8_crypto_methods[] = {
DEVMETHOD(device_identify, armv8_crypto_identify), DEVMETHOD(device_identify, armv8_crypto_identify),
DEVMETHOD(device_probe, armv8_crypto_probe), DEVMETHOD(device_probe, armv8_crypto_probe),
DEVMETHOD(device_attach, armv8_crypto_attach), DEVMETHOD(device_attach, armv8_crypto_attach),
DEVMETHOD(device_detach, armv8_crypto_detach), DEVMETHOD(device_detach, armv8_crypto_detach),
Show All 13 Lines