diff --git a/sys/crypto/chacha20/chacha-sw.c b/sys/crypto/chacha20/chacha-sw.c --- a/sys/crypto/chacha20/chacha-sw.c +++ b/sys/crypto/chacha20/chacha-sw.c @@ -32,6 +32,14 @@ chacha_encrypt_bytes(ctx, in, out, CHACHA_BLOCKLEN); } +static void +chacha20_xform_crypt_multi(void *ctx, const uint8_t *in, uint8_t *out, + size_t len) +{ + KASSERT(len % CHACHA_BLOCKLEN == 0, ("%s: invalid length", __func__)); + chacha_encrypt_bytes(ctx, in, out, len); +} + static void chacha20_xform_crypt_last(void *ctx, const uint8_t *in, uint8_t *out, size_t len) @@ -49,10 +57,12 @@ .ivsize = CHACHA_NONCELEN + CHACHA_CTRLEN, .minkey = CHACHA_MINKEYLEN, .maxkey = 32, - .encrypt = chacha20_xform_crypt, - .decrypt = chacha20_xform_crypt, .setkey = chacha20_xform_setkey, .reinit = chacha20_xform_reinit, + .encrypt = chacha20_xform_crypt, + .decrypt = chacha20_xform_crypt, + .encrypt_multi = chacha20_xform_crypt_multi, + .decrypt_multi = chacha20_xform_crypt_multi, .encrypt_last = chacha20_xform_crypt_last, .decrypt_last = chacha20_xform_crypt_last, }; diff --git a/sys/opencrypto/xform_aes_cbc.c b/sys/opencrypto/xform_aes_cbc.c --- a/sys/opencrypto/xform_aes_cbc.c +++ b/sys/opencrypto/xform_aes_cbc.c @@ -61,6 +61,8 @@ static int aes_cbc_setkey(void *, const uint8_t *, int); static void aes_cbc_encrypt(void *, const uint8_t *, uint8_t *); static void aes_cbc_decrypt(void *, const uint8_t *, uint8_t *); +static void aes_cbc_encrypt_multi(void *, const uint8_t *, uint8_t *, size_t); +static void aes_cbc_decrypt_multi(void *, const uint8_t *, uint8_t *, size_t); static void aes_cbc_reinit(void *, const uint8_t *, size_t); /* Encryption instances */ @@ -72,10 +74,12 @@ .ivsize = AES_BLOCK_LEN, .minkey = AES_MIN_KEY, .maxkey = AES_MAX_KEY, - .encrypt = aes_cbc_encrypt, - .decrypt = aes_cbc_decrypt, .setkey = aes_cbc_setkey, .reinit = aes_cbc_reinit, + .encrypt = aes_cbc_encrypt, + .decrypt = aes_cbc_decrypt, + .encrypt_multi = aes_cbc_encrypt_multi, + .decrypt_multi = aes_cbc_decrypt_multi, }; /* @@ -106,6 +110,43 @@ explicit_bzero(block, sizeof(block)); } +static void +aes_cbc_encrypt_multi(void *vctx, const uint8_t *in, uint8_t *out, size_t len) +{ + struct aes_cbc_ctx *ctx = vctx; + + KASSERT(len % AES_BLOCK_LEN == 0, ("%s: invalid length", __func__)); + while (len > 0) { + for (u_int i = 0; i < AES_BLOCK_LEN; i++) + out[i] = in[i] ^ ctx->iv[i]; + rijndael_encrypt(&ctx->key, out, out); + memcpy(ctx->iv, out, AES_BLOCK_LEN); + out += AES_BLOCK_LEN; + in += AES_BLOCK_LEN; + len -= AES_BLOCK_LEN; + } +} + +static void +aes_cbc_decrypt_multi(void *vctx, const uint8_t *in, uint8_t *out, size_t len) +{ + struct aes_cbc_ctx *ctx = vctx; + char block[AES_BLOCK_LEN]; + + KASSERT(len % AES_BLOCK_LEN == 0, ("%s: invalid length", __func__)); + while (len > 0) { + memcpy(block, in, AES_BLOCK_LEN); + rijndael_decrypt(&ctx->key, in, out); + for (u_int i = 0; i < AES_BLOCK_LEN; i++) + out[i] ^= ctx->iv[i]; + memcpy(ctx->iv, block, AES_BLOCK_LEN); + out += AES_BLOCK_LEN; + in += AES_BLOCK_LEN; + len -= AES_BLOCK_LEN; + } + explicit_bzero(block, sizeof(block)); +} + static int aes_cbc_setkey(void *vctx, const uint8_t *key, int len) { diff --git a/sys/opencrypto/xform_aes_icm.c b/sys/opencrypto/xform_aes_icm.c --- a/sys/opencrypto/xform_aes_icm.c +++ b/sys/opencrypto/xform_aes_icm.c @@ -66,6 +66,7 @@ static int aes_icm_setkey(void *, const uint8_t *, int); static void aes_icm_crypt(void *, const uint8_t *, uint8_t *); +static void aes_icm_crypt_multi(void *, const uint8_t *, uint8_t *, size_t); static void aes_icm_crypt_last(void *, const uint8_t *, uint8_t *, size_t); static void aes_icm_reinit(void *, const uint8_t *, size_t); static int aes_gcm_setkey(void *, const uint8_t *, int); @@ -87,10 +88,12 @@ .ivsize = AES_BLOCK_LEN, .minkey = AES_MIN_KEY, .maxkey = AES_MAX_KEY, - .encrypt = aes_icm_crypt, - .decrypt = aes_icm_crypt, .setkey = aes_icm_setkey, .reinit = aes_icm_reinit, + .encrypt = aes_icm_crypt, + .decrypt = aes_icm_crypt, + .encrypt_multi = aes_icm_crypt_multi, + .decrypt_multi = aes_icm_crypt_multi, .encrypt_last = aes_icm_crypt_last, .decrypt_last = aes_icm_crypt_last, }; @@ -105,10 +108,12 @@ .minkey = AES_MIN_KEY, .maxkey = AES_MAX_KEY, .macsize = AES_GMAC_HASH_LEN, - .encrypt = aes_icm_crypt, - .decrypt = aes_icm_crypt, .setkey = aes_gcm_setkey, .reinit = aes_gcm_reinit, + .encrypt = aes_icm_crypt, + .decrypt = aes_icm_crypt, + .encrypt_multi = aes_icm_crypt_multi, + .decrypt_multi = aes_icm_crypt_multi, .encrypt_last = aes_icm_crypt_last, .decrypt_last = aes_icm_crypt_last, .update = aes_gcm_update, @@ -124,10 +129,12 @@ .ivsize = AES_CCM_IV_LEN, .minkey = AES_MIN_KEY, .maxkey = AES_MAX_KEY, .macsize = AES_CBC_MAC_HASH_LEN, - .encrypt = aes_icm_crypt, - .decrypt = aes_icm_crypt, .setkey = aes_ccm_setkey, .reinit = aes_ccm_reinit, + .encrypt = aes_icm_crypt, + .decrypt = aes_icm_crypt, + .encrypt_multi = aes_icm_crypt_multi, + .decrypt_multi = aes_icm_crypt_multi, .encrypt_last = aes_icm_crypt_last, .decrypt_last = aes_icm_crypt_last, .update = aes_ccm_update, @@ -197,6 +204,31 @@ break; } +static void +aes_icm_crypt_multi(void *key, const uint8_t *in, uint8_t *out, size_t len) +{ + struct aes_icm_ctx *ctx = key; + uint8_t keystream[AESICM_BLOCKSIZE]; + int i; + + KASSERT(len % AESICM_BLOCKSIZE == 0, ("%s: invalid length", __func__)); + while (len > 0) { + rijndaelEncrypt(ctx->ac_ek, ctx->ac_nr, ctx->ac_block, keystream); + for (i = 0; i < AESICM_BLOCKSIZE; i++) + out[i] = in[i] ^ keystream[i]; + + /* increment counter */ + for (i = AESICM_BLOCKSIZE - 1; i >= 0; i--) + if (++ctx->ac_block[i]) /* continue on overflow */ + break; + + out += AESICM_BLOCKSIZE; + in += AESICM_BLOCKSIZE; + len -= AESICM_BLOCKSIZE; + } + explicit_bzero(keystream, sizeof(keystream)); +} + static void aes_icm_crypt_last(void *key, const uint8_t *in, uint8_t *out, size_t len) { diff --git a/sys/opencrypto/xform_aes_xts.c b/sys/opencrypto/xform_aes_xts.c --- a/sys/opencrypto/xform_aes_xts.c +++ b/sys/opencrypto/xform_aes_xts.c @@ -56,6 +56,8 @@ static int aes_xts_setkey(void *, const uint8_t *, int); static void aes_xts_encrypt(void *, const uint8_t *, uint8_t *); static void aes_xts_decrypt(void *, const uint8_t *, uint8_t *); +static void aes_xts_encrypt_multi(void *, const uint8_t *, uint8_t *, size_t); +static void aes_xts_decrypt_multi(void *, const uint8_t *, uint8_t *, size_t); static void aes_xts_reinit(void *, const uint8_t *, size_t); /* Encryption instances */ @@ -67,10 +69,12 @@ .ivsize = AES_XTS_IV_LEN, .minkey = AES_XTS_MIN_KEY, .maxkey = AES_XTS_MAX_KEY, + .setkey = aes_xts_setkey, + .reinit = aes_xts_reinit, .encrypt = aes_xts_encrypt, .decrypt = aes_xts_decrypt, - .setkey = aes_xts_setkey, - .reinit = aes_xts_reinit + .encrypt_multi = aes_xts_encrypt_multi, + .decrypt_multi = aes_xts_decrypt_multi, }; /* @@ -105,44 +109,63 @@ static void aes_xts_crypt(struct aes_xts_ctx *ctx, const uint8_t *in, uint8_t *out, - u_int do_encrypt) + size_t len, bool do_encrypt) { uint8_t block[AES_XTS_BLOCKSIZE]; u_int i, carry_in, carry_out; - for (i = 0; i < AES_XTS_BLOCKSIZE; i++) - block[i] = in[i] ^ ctx->tweak[i]; - - if (do_encrypt) - rijndael_encrypt(&ctx->key1, block, out); - else - rijndael_decrypt(&ctx->key1, block, out); - - for (i = 0; i < AES_XTS_BLOCKSIZE; i++) - out[i] ^= ctx->tweak[i]; - - /* Exponentiate tweak */ - carry_in = 0; - for (i = 0; i < AES_XTS_BLOCKSIZE; i++) { - carry_out = ctx->tweak[i] & 0x80; - ctx->tweak[i] = (ctx->tweak[i] << 1) | (carry_in ? 1 : 0); - carry_in = carry_out; + KASSERT(len % AES_XTS_BLOCKSIZE == 0, ("%s: invalid length", __func__)); + while (len > 0) { + for (i = 0; i < AES_XTS_BLOCKSIZE; i++) + block[i] = in[i] ^ ctx->tweak[i]; + + if (do_encrypt) + rijndael_encrypt(&ctx->key1, block, out); + else + rijndael_decrypt(&ctx->key1, block, out); + + for (i = 0; i < AES_XTS_BLOCKSIZE; i++) + out[i] ^= ctx->tweak[i]; + + /* Exponentiate tweak */ + carry_in = 0; + for (i = 0; i < AES_XTS_BLOCKSIZE; i++) { + carry_out = ctx->tweak[i] & 0x80; + ctx->tweak[i] = (ctx->tweak[i] << 1) | (carry_in ? 1 : 0); + carry_in = carry_out; + } + if (carry_in) + ctx->tweak[0] ^= AES_XTS_ALPHA; + + in += AES_XTS_BLOCKSIZE; + out += AES_XTS_BLOCKSIZE; + len -= AES_XTS_BLOCKSIZE; } - if (carry_in) - ctx->tweak[0] ^= AES_XTS_ALPHA; explicit_bzero(block, sizeof(block)); } static void aes_xts_encrypt(void *key, const uint8_t *in, uint8_t *out) { - aes_xts_crypt(key, in, out, 1); + aes_xts_crypt(key, in, out, AES_XTS_BLOCKSIZE, true); } static void aes_xts_decrypt(void *key, const uint8_t *in, uint8_t *out) { - aes_xts_crypt(key, in, out, 0); + aes_xts_crypt(key, in, out, AES_XTS_BLOCKSIZE, false); +} + +static void +aes_xts_encrypt_multi(void *vctx, const uint8_t *in, uint8_t *out, size_t len) +{ + aes_xts_crypt(vctx, in, out, len, true); +} + +static void +aes_xts_decrypt_multi(void *vctx, const uint8_t *in, uint8_t *out, size_t len) +{ + aes_xts_crypt(vctx, in, out, len, false); } static int diff --git a/sys/opencrypto/xform_chacha20_poly1305.c b/sys/opencrypto/xform_chacha20_poly1305.c --- a/sys/opencrypto/xform_chacha20_poly1305.c +++ b/sys/opencrypto/xform_chacha20_poly1305.c @@ -98,6 +98,24 @@ ctx->ic++; } +static void +chacha20_poly1305_crypt_multi(void *vctx, const uint8_t *in, uint8_t *out, size_t len) +{ + struct chacha20_poly1305_ctx *ctx = vctx; + int error __diagused; + + KASSERT(len % CHACHA20_NATIVE_BLOCK_LEN == 0, ("%s: invalid length", + __func__)); + if (ctx->ietf) + error = crypto_stream_chacha20_ietf_xor_ic(out, in, len, + ctx->nonce, ctx->ic, ctx->key); + else + error = crypto_stream_chacha20_xor_ic(out, in, len, ctx->nonce, + ctx->ic, ctx->key); + KASSERT(error == 0, ("%s failed: %d", __func__, error)); + ctx->ic += len / CHACHA20_NATIVE_BLOCK_LEN; +} + static void chacha20_poly1305_crypt_last(void *vctx, const uint8_t *in, uint8_t *out, size_t len) @@ -142,10 +160,12 @@ .minkey = CHACHA20_POLY1305_KEY, .maxkey = CHACHA20_POLY1305_KEY, .macsize = POLY1305_HASH_LEN, - .encrypt = chacha20_poly1305_crypt, - .decrypt = chacha20_poly1305_crypt, .setkey = chacha20_poly1305_setkey, .reinit = chacha20_poly1305_reinit, + .encrypt = chacha20_poly1305_crypt, + .decrypt = chacha20_poly1305_crypt, + .encrypt_multi = chacha20_poly1305_crypt_multi, + .decrypt_multi = chacha20_poly1305_crypt_multi, .encrypt_last = chacha20_poly1305_crypt_last, .decrypt_last = chacha20_poly1305_crypt_last, .update = chacha20_poly1305_update, @@ -197,10 +217,12 @@ .minkey = XCHACHA20_POLY1305_KEY, .maxkey = XCHACHA20_POLY1305_KEY, .macsize = POLY1305_HASH_LEN, - .encrypt = chacha20_poly1305_crypt, - .decrypt = chacha20_poly1305_crypt, .setkey = xchacha20_poly1305_setkey, .reinit = xchacha20_poly1305_reinit, + .encrypt = chacha20_poly1305_crypt, + .decrypt = chacha20_poly1305_crypt, + .encrypt_multi = chacha20_poly1305_crypt_multi, + .decrypt_multi = chacha20_poly1305_crypt_multi, .encrypt_last = chacha20_poly1305_crypt_last, .decrypt_last = chacha20_poly1305_crypt_last, .update = chacha20_poly1305_update, diff --git a/sys/opencrypto/xform_cml.c b/sys/opencrypto/xform_cml.c --- a/sys/opencrypto/xform_cml.c +++ b/sys/opencrypto/xform_cml.c @@ -61,6 +61,8 @@ static int cml_setkey(void *, const uint8_t *, int); static void cml_encrypt(void *, const uint8_t *, uint8_t *); static void cml_decrypt(void *, const uint8_t *, uint8_t *); +static void cml_encrypt_multi(void *, const uint8_t *, uint8_t *, size_t); +static void cml_decrypt_multi(void *, const uint8_t *, uint8_t *, size_t); static void cml_reinit(void *, const uint8_t *, size_t); /* Encryption instances */ @@ -72,10 +74,12 @@ .ivsize = CAMELLIA_BLOCK_LEN, .minkey = CAMELLIA_MIN_KEY, .maxkey = CAMELLIA_MAX_KEY, - .encrypt = cml_encrypt, - .decrypt = cml_decrypt, .setkey = cml_setkey, .reinit = cml_reinit, + .encrypt = cml_encrypt, + .decrypt = cml_decrypt, + .encrypt_multi = cml_encrypt_multi, + .decrypt_multi = cml_decrypt_multi, }; /* @@ -106,6 +110,45 @@ explicit_bzero(block, sizeof(block)); } +static void +cml_encrypt_multi(void *vctx, const uint8_t *in, uint8_t *out, size_t len) +{ + struct camellia_cbc_ctx *ctx = vctx; + + KASSERT(len % CAMELLIA_BLOCK_LEN == 0, ("%s: invalid length", + __func__)); + while (len > 0) { + for (u_int i = 0; i < CAMELLIA_BLOCK_LEN; i++) + out[i] = in[i] ^ ctx->iv[i]; + camellia_encrypt(&ctx->state, out, out); + memcpy(ctx->iv, out, CAMELLIA_BLOCK_LEN); + out += CAMELLIA_BLOCK_LEN; + in += CAMELLIA_BLOCK_LEN; + len -= CAMELLIA_BLOCK_LEN; + } +} + +static void +cml_decrypt_multi(void *vctx, const uint8_t *in, uint8_t *out, size_t len) +{ + struct camellia_cbc_ctx *ctx = vctx; + char block[CAMELLIA_BLOCK_LEN]; + + KASSERT(len % CAMELLIA_BLOCK_LEN == 0, ("%s: invalid length", + __func__)); + while (len > 0) { + memcpy(block, in, CAMELLIA_BLOCK_LEN); + camellia_decrypt(&ctx->state, in, out); + for (u_int i = 0; i < CAMELLIA_BLOCK_LEN; i++) + out[i] ^= ctx->iv[i]; + memcpy(ctx->iv, block, CAMELLIA_BLOCK_LEN); + out += CAMELLIA_BLOCK_LEN; + in += CAMELLIA_BLOCK_LEN; + len -= CAMELLIA_BLOCK_LEN; + } + explicit_bzero(block, sizeof(block)); +} + static int cml_setkey(void *vctx, const uint8_t *key, int len) { diff --git a/sys/opencrypto/xform_enc.h b/sys/opencrypto/xform_enc.h --- a/sys/opencrypto/xform_enc.h +++ b/sys/opencrypto/xform_enc.h @@ -56,14 +56,26 @@ uint16_t minkey, maxkey; uint16_t macsize; /* For AEAD ciphers. */ + /* Initialize context and set key. */ + int (*setkey) (void *, const uint8_t *, int len); + + /* Supply context with nonce/IV. */ + void (*reinit) (void *, const uint8_t *, size_t); + /* * Encrypt/decrypt a single block. For stream ciphers this * encrypts/decrypts a single "native" block. */ void (*encrypt) (void *, const uint8_t *, uint8_t *); void (*decrypt) (void *, const uint8_t *, uint8_t *); - int (*setkey) (void *, const uint8_t *, int len); - void (*reinit) (void *, const uint8_t *, size_t); + + /* + * Encrypt/decrypt multiple blocks. For stream ciphers this + * encrypts/decrypts multiple "native" blocks. The fourth + * argument is a count of bytes. + */ + void (*encrypt_multi) (void *, const uint8_t *, uint8_t *, size_t); + void (*decrypt_multi) (void *, const uint8_t *, uint8_t *, size_t); /* * For stream ciphers, encrypt/decrypt the final partial block diff --git a/sys/opencrypto/xform_null.c b/sys/opencrypto/xform_null.c --- a/sys/opencrypto/xform_null.c +++ b/sys/opencrypto/xform_null.c @@ -55,6 +55,7 @@ static int null_setkey(void *, const uint8_t *, int); static void null_crypt(void *, const uint8_t *, uint8_t *); +static void null_crypt_multi(void *, const uint8_t *, uint8_t *, size_t); static void null_init(void *); static void null_reinit(void *ctx, const uint8_t *buf, u_int len); @@ -70,9 +71,11 @@ .ivsize = 0, .minkey = NULL_MIN_KEY, .maxkey = NULL_MAX_KEY, + .setkey = null_setkey, .encrypt = null_crypt, .decrypt = null_crypt, - .setkey = null_setkey, + .encrypt_multi = null_crypt_multi, + .decrypt_multi = null_crypt_multi, }; /* Authentication instances */ @@ -98,6 +101,11 @@ { } +static void +null_crypt_multi(void *key, const uint8_t *in, uint8_t *out, size_t len) +{ +} + static int null_setkey(void *sched, const uint8_t *key, int len) {