Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -2992,6 +2992,7 @@ geom/eli/g_eli.c optional geom_eli geom/eli/g_eli_crypto.c optional geom_eli geom/eli/g_eli_ctl.c optional geom_eli +geom/eli/g_eli_hmac.c optional geom_eli geom/eli/g_eli_integrity.c optional geom_eli geom/eli/g_eli_key.c optional geom_eli geom/eli/g_eli_key_cache.c optional geom_eli Index: sys/geom/eli/g_eli.h =================================================================== --- sys/geom/eli/g_eli.h +++ sys/geom/eli/g_eli.h @@ -39,8 +39,6 @@ #include #include #include -#include -#include #include #else #include @@ -48,6 +46,8 @@ #include #include #endif +#include +#include #ifndef _OpenSSL_ #include #endif @@ -131,15 +131,15 @@ /* Switch data encryption key every 2^20 blocks. */ #define G_ELI_KEY_SHIFT 20 +#define G_ELI_CRYPTO_UNKNOWN 0 +#define G_ELI_CRYPTO_HW 1 +#define G_ELI_CRYPTO_SW 2 + #ifdef _KERNEL extern int g_eli_debug; extern u_int g_eli_overwrites; extern u_int g_eli_batch; -#define G_ELI_CRYPTO_UNKNOWN 0 -#define G_ELI_CRYPTO_HW 1 -#define G_ELI_CRYPTO_SW 2 - #define G_ELI_DEBUG(lvl, ...) do { \ if (g_eli_debug >= (lvl)) { \ printf("GEOM_ELI"); \ @@ -172,6 +172,10 @@ LIST_ENTRY(g_eli_worker) w_next; }; +#else /* !_KERNEL */ +typedef int boolean_t; +#endif /* _KERNEL */ + struct g_eli_softc { struct g_geom *sc_geom; u_int sc_version; @@ -201,13 +205,31 @@ u_int sc_data_per_sector; boolean_t sc_cpubind; +#ifdef _KERNEL /* Only for software cryptography. */ struct bio_queue_head sc_queue; struct mtx sc_queue_mtx; LIST_HEAD(, g_eli_worker) sc_workers; +#endif /* _KERNEL */ }; #define sc_name sc_geom->name -#endif /* _KERNEL */ + +#define G_ELI_KEY_MAGIC 0xe11341c + +struct g_eli_key { + /* Key value, must be first in the structure. */ + uint8_t gek_key[G_ELI_DATAKEYLEN]; + /* Magic. */ + int gek_magic; + /* Key number. */ + uint64_t gek_keyno; + /* Reference counter. */ + int gek_count; + /* Keeps keys sorted by most recent use. */ + TAILQ_ENTRY(g_eli_key) gek_next; + /* Keeps keys sorted by number. */ + RB_ENTRY(g_eli_key) gek_link; +}; struct g_eli_metadata { char md_magic[16]; /* Magic value. */ @@ -568,6 +590,60 @@ return (0); } +static __inline void +eli_metadata_softc(struct g_eli_softc *sc, const struct g_eli_metadata *md, + u_int sectorsize, off_t mediasize) +{ + + sc->sc_version = md->md_version; + sc->sc_inflight = 0; + sc->sc_crypto = G_ELI_CRYPTO_UNKNOWN; + sc->sc_flags = md->md_flags; + /* Backward compatibility. */ + if (md->md_version < G_ELI_VERSION_04) + sc->sc_flags |= G_ELI_FLAG_NATIVE_BYTE_ORDER; + if (md->md_version < G_ELI_VERSION_05) + sc->sc_flags |= G_ELI_FLAG_SINGLE_KEY; + if (md->md_version < G_ELI_VERSION_06 && + (sc->sc_flags & G_ELI_FLAG_AUTH) != 0) { + sc->sc_flags |= G_ELI_FLAG_FIRST_KEY; + } + if (md->md_version < G_ELI_VERSION_07) + sc->sc_flags |= G_ELI_FLAG_ENC_IVKEY; + sc->sc_ealgo = md->md_ealgo; + + if (sc->sc_flags & G_ELI_FLAG_AUTH) { + sc->sc_akeylen = sizeof(sc->sc_akey) * 8; + sc->sc_aalgo = md->md_aalgo; + sc->sc_alen = g_eli_hashlen(sc->sc_aalgo); + + sc->sc_data_per_sector = sectorsize - sc->sc_alen; + /* + * Some hash functions (like SHA1 and RIPEMD160) generates hash + * which length is not multiple of 128 bits, but we want data + * length to be multiple of 128, so we can encrypt without + * padding. The line below rounds down data length to multiple + * of 128 bits. + */ + sc->sc_data_per_sector -= sc->sc_data_per_sector % 16; + + sc->sc_bytes_per_sector = + (md->md_sectorsize - 1) / sc->sc_data_per_sector + 1; + sc->sc_bytes_per_sector *= sectorsize; + } + sc->sc_sectorsize = md->md_sectorsize; + sc->sc_mediasize = mediasize; + if (!(sc->sc_flags & G_ELI_FLAG_ONETIME)) + sc->sc_mediasize -= sectorsize; + if (!(sc->sc_flags & G_ELI_FLAG_AUTH)) + sc->sc_mediasize -= (sc->sc_mediasize % sc->sc_sectorsize); + else { + sc->sc_mediasize /= sc->sc_bytes_per_sector; + sc->sc_mediasize *= sc->sc_sectorsize; + } + sc->sc_ekeylen = md->md_keylen; +} + #ifdef _KERNEL int g_eli_read_metadata(struct g_class *mp, struct g_provider *pp, struct g_eli_metadata *md); @@ -582,8 +658,6 @@ void g_eli_read_done(struct bio *bp); void g_eli_write_done(struct bio *bp); int g_eli_crypto_rerun(struct cryptop *crp); -void g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv, - size_t size); void g_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker); void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp); @@ -591,6 +665,8 @@ void g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp); void g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp); #endif +void g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv, + size_t size); void g_eli_mkey_hmac(unsigned char *mkey, const unsigned char *key); int g_eli_mkey_decrypt(const struct g_eli_metadata *md, @@ -619,6 +695,8 @@ void g_eli_crypto_hmac(const uint8_t *hkey, size_t hkeysize, const uint8_t *data, size_t datasize, uint8_t *md, size_t mdsize); +void g_eli_key_fill(struct g_eli_softc *sc, struct g_eli_key *key, + uint64_t keyno); #ifdef _KERNEL void g_eli_key_init(struct g_eli_softc *sc); void g_eli_key_destroy(struct g_eli_softc *sc); Index: sys/geom/eli/g_eli.c =================================================================== --- sys/geom/eli/g_eli.c +++ sys/geom/eli/g_eli.c @@ -571,40 +571,6 @@ } } -/* - * Here we generate IV. It is unique for every sector. - */ -void -g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv, - size_t size) -{ - uint8_t off[8]; - - if ((sc->sc_flags & G_ELI_FLAG_NATIVE_BYTE_ORDER) != 0) - bcopy(&offset, off, sizeof(off)); - else - le64enc(off, (uint64_t)offset); - - switch (sc->sc_ealgo) { - case CRYPTO_AES_XTS: - bcopy(off, iv, sizeof(off)); - bzero(iv + sizeof(off), size - sizeof(off)); - break; - default: - { - u_char hash[SHA256_DIGEST_LENGTH]; - SHA256_CTX ctx; - - /* Copy precalculated SHA256 context for IV-Key. */ - bcopy(&sc->sc_ivctx, &ctx, sizeof(ctx)); - SHA256_Update(&ctx, off, sizeof(off)); - SHA256_Final(hash, &ctx); - bcopy(hash, iv, MIN(sizeof(hash), size)); - break; - } - } -} - int g_eli_read_metadata(struct g_class *mp, struct g_provider *pp, struct g_eli_metadata *md) @@ -751,44 +717,9 @@ else gp->access = g_std_access; - sc->sc_version = md->md_version; - sc->sc_inflight = 0; - sc->sc_crypto = G_ELI_CRYPTO_UNKNOWN; - sc->sc_flags = md->md_flags; - /* Backward compatibility. */ - if (md->md_version < G_ELI_VERSION_04) - sc->sc_flags |= G_ELI_FLAG_NATIVE_BYTE_ORDER; - if (md->md_version < G_ELI_VERSION_05) - sc->sc_flags |= G_ELI_FLAG_SINGLE_KEY; - if (md->md_version < G_ELI_VERSION_06 && - (sc->sc_flags & G_ELI_FLAG_AUTH) != 0) { - sc->sc_flags |= G_ELI_FLAG_FIRST_KEY; - } - if (md->md_version < G_ELI_VERSION_07) - sc->sc_flags |= G_ELI_FLAG_ENC_IVKEY; - sc->sc_ealgo = md->md_ealgo; + eli_metadata_softc(sc, md, bpp->sectorsize, bpp->mediasize); sc->sc_nkey = nkey; - if (sc->sc_flags & G_ELI_FLAG_AUTH) { - sc->sc_akeylen = sizeof(sc->sc_akey) * 8; - sc->sc_aalgo = md->md_aalgo; - sc->sc_alen = g_eli_hashlen(sc->sc_aalgo); - - sc->sc_data_per_sector = bpp->sectorsize - sc->sc_alen; - /* - * Some hash functions (like SHA1 and RIPEMD160) generates hash - * which length is not multiple of 128 bits, but we want data - * length to be multiple of 128, so we can encrypt without - * padding. The line below rounds down data length to multiple - * of 128 bits. - */ - sc->sc_data_per_sector -= sc->sc_data_per_sector % 16; - - sc->sc_bytes_per_sector = - (md->md_sectorsize - 1) / sc->sc_data_per_sector + 1; - sc->sc_bytes_per_sector *= bpp->sectorsize; - } - gp->softc = sc; sc->sc_geom = gp; @@ -831,22 +762,10 @@ goto failed; } - sc->sc_sectorsize = md->md_sectorsize; - sc->sc_mediasize = bpp->mediasize; - if (!(sc->sc_flags & G_ELI_FLAG_ONETIME)) - sc->sc_mediasize -= bpp->sectorsize; - if (!(sc->sc_flags & G_ELI_FLAG_AUTH)) - sc->sc_mediasize -= (sc->sc_mediasize % sc->sc_sectorsize); - else { - sc->sc_mediasize /= sc->sc_bytes_per_sector; - sc->sc_mediasize *= sc->sc_sectorsize; - } - /* * Remember the keys in our softc structure. */ g_eli_mkey_propagate(sc, mkey); - sc->sc_ekeylen = md->md_keylen; LIST_INIT(&sc->sc_workers); Index: sys/geom/eli/g_eli_crypto.c =================================================================== --- sys/geom/eli/g_eli_crypto.c +++ sys/geom/eli/g_eli_crypto.c @@ -221,75 +221,3 @@ return (g_eli_crypto_cipher(algo, 0, data, datasize, key, keysize)); } - -void -g_eli_crypto_hmac_init(struct hmac_ctx *ctx, const uint8_t *hkey, - size_t hkeylen) -{ - u_char k_ipad[128], key[128]; - SHA512_CTX lctx; - u_int i; - - bzero(key, sizeof(key)); - if (hkeylen == 0) - ; /* do nothing */ - else if (hkeylen <= 128) - bcopy(hkey, key, hkeylen); - else { - /* If key is longer than 128 bytes reset it to key = SHA512(key). */ - SHA512_Init(&lctx); - SHA512_Update(&lctx, hkey, hkeylen); - SHA512_Final(key, &lctx); - } - - /* XOR key with ipad and opad values. */ - for (i = 0; i < sizeof(key); i++) { - k_ipad[i] = key[i] ^ 0x36; - ctx->k_opad[i] = key[i] ^ 0x5c; - } - bzero(key, sizeof(key)); - /* Perform inner SHA512. */ - SHA512_Init(&ctx->shactx); - SHA512_Update(&ctx->shactx, k_ipad, sizeof(k_ipad)); - bzero(k_ipad, sizeof(k_ipad)); -} - -void -g_eli_crypto_hmac_update(struct hmac_ctx *ctx, const uint8_t *data, - size_t datasize) -{ - - SHA512_Update(&ctx->shactx, data, datasize); -} - -void -g_eli_crypto_hmac_final(struct hmac_ctx *ctx, uint8_t *md, size_t mdsize) -{ - u_char digest[SHA512_MDLEN]; - SHA512_CTX lctx; - - SHA512_Final(digest, &ctx->shactx); - /* Perform outer SHA512. */ - SHA512_Init(&lctx); - SHA512_Update(&lctx, ctx->k_opad, sizeof(ctx->k_opad)); - bzero(ctx, sizeof(*ctx)); - SHA512_Update(&lctx, digest, sizeof(digest)); - SHA512_Final(digest, &lctx); - bzero(&lctx, sizeof(lctx)); - /* mdsize == 0 means "Give me the whole hash!" */ - if (mdsize == 0) - mdsize = SHA512_MDLEN; - bcopy(digest, md, mdsize); - bzero(digest, sizeof(digest)); -} - -void -g_eli_crypto_hmac(const uint8_t *hkey, size_t hkeysize, const uint8_t *data, - size_t datasize, uint8_t *md, size_t mdsize) -{ - struct hmac_ctx ctx; - - g_eli_crypto_hmac_init(&ctx, hkey, hkeysize); - g_eli_crypto_hmac_update(&ctx, data, datasize); - g_eli_crypto_hmac_final(&ctx, md, mdsize); -} Index: sys/geom/eli/g_eli_hmac.c =================================================================== --- sys/geom/eli/g_eli_hmac.c +++ sys/geom/eli/g_eli_hmac.c @@ -43,185 +43,6 @@ #endif #include -#ifdef _KERNEL -MALLOC_DECLARE(M_ELI); - -static int -g_eli_crypto_done(struct cryptop *crp) -{ - - crp->crp_opaque = (void *)crp; - wakeup(crp); - return (0); -} - -static int -g_eli_crypto_cipher(u_int algo, int enc, u_char *data, size_t datasize, - const u_char *key, size_t keysize) -{ - struct cryptoini cri; - struct cryptop *crp; - struct cryptodesc *crd; - uint64_t sid; - u_char *p; - int error; - - KASSERT(algo != CRYPTO_AES_XTS, - ("%s: CRYPTO_AES_XTS unexpected here", __func__)); - - bzero(&cri, sizeof(cri)); - cri.cri_alg = algo; - cri.cri_key = __DECONST(void *, key); - cri.cri_klen = keysize; - error = crypto_newsession(&sid, &cri, CRYPTOCAP_F_SOFTWARE); - if (error != 0) - return (error); - p = malloc(sizeof(*crp) + sizeof(*crd), M_ELI, M_NOWAIT | M_ZERO); - if (p == NULL) { - crypto_freesession(sid); - return (ENOMEM); - } - crp = (struct cryptop *)p; p += sizeof(*crp); - crd = (struct cryptodesc *)p; p += sizeof(*crd); - - crd->crd_skip = 0; - crd->crd_len = datasize; - crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; - if (enc) - crd->crd_flags |= CRD_F_ENCRYPT; - crd->crd_alg = algo; - crd->crd_key = __DECONST(void *, key); - crd->crd_klen = keysize; - bzero(crd->crd_iv, sizeof(crd->crd_iv)); - crd->crd_next = NULL; - - crp->crp_sid = sid; - crp->crp_ilen = datasize; - crp->crp_olen = datasize; - crp->crp_opaque = NULL; - crp->crp_callback = g_eli_crypto_done; - crp->crp_buf = (void *)data; - crp->crp_flags = CRYPTO_F_CBIFSYNC; - crp->crp_desc = crd; - - error = crypto_dispatch(crp); - if (error == 0) { - while (crp->crp_opaque == NULL) - tsleep(crp, PRIBIO, "geli", hz / 5); - error = crp->crp_etype; - } - - free(crp, M_ELI); - crypto_freesession(sid); - return (error); -} -#else /* !_KERNEL */ -static int -g_eli_crypto_cipher(u_int algo, int enc, u_char *data, size_t datasize, - const u_char *key, size_t keysize) -{ - EVP_CIPHER_CTX ctx; - const EVP_CIPHER *type; - u_char iv[keysize]; - int outsize; - - assert(algo != CRYPTO_AES_XTS); - - switch (algo) { - case CRYPTO_NULL_CBC: - type = EVP_enc_null(); - break; - case CRYPTO_AES_CBC: - switch (keysize) { - case 128: - type = EVP_aes_128_cbc(); - break; - case 192: - type = EVP_aes_192_cbc(); - break; - case 256: - type = EVP_aes_256_cbc(); - break; - default: - return (EINVAL); - } - break; - case CRYPTO_BLF_CBC: - type = EVP_bf_cbc(); - break; -#ifndef OPENSSL_NO_CAMELLIA - case CRYPTO_CAMELLIA_CBC: - switch (keysize) { - case 128: - type = EVP_camellia_128_cbc(); - break; - case 192: - type = EVP_camellia_192_cbc(); - break; - case 256: - type = EVP_camellia_256_cbc(); - break; - default: - return (EINVAL); - } - break; -#endif - case CRYPTO_3DES_CBC: - type = EVP_des_ede3_cbc(); - break; - default: - return (EINVAL); - } - - EVP_CIPHER_CTX_init(&ctx); - - EVP_CipherInit_ex(&ctx, type, NULL, NULL, NULL, enc); - EVP_CIPHER_CTX_set_key_length(&ctx, keysize / 8); - EVP_CIPHER_CTX_set_padding(&ctx, 0); - bzero(iv, sizeof(iv)); - EVP_CipherInit_ex(&ctx, NULL, NULL, key, iv, enc); - - if (EVP_CipherUpdate(&ctx, data, &outsize, data, datasize) == 0) { - EVP_CIPHER_CTX_cleanup(&ctx); - return (EINVAL); - } - assert(outsize == (int)datasize); - - if (EVP_CipherFinal_ex(&ctx, data + outsize, &outsize) == 0) { - EVP_CIPHER_CTX_cleanup(&ctx); - return (EINVAL); - } - assert(outsize == 0); - - EVP_CIPHER_CTX_cleanup(&ctx); - return (0); -} -#endif /* !_KERNEL */ - -int -g_eli_crypto_encrypt(u_int algo, u_char *data, size_t datasize, - const u_char *key, size_t keysize) -{ - - /* We prefer AES-CBC for metadata protection. */ - if (algo == CRYPTO_AES_XTS) - algo = CRYPTO_AES_CBC; - - return (g_eli_crypto_cipher(algo, 1, data, datasize, key, keysize)); -} - -int -g_eli_crypto_decrypt(u_int algo, u_char *data, size_t datasize, - const u_char *key, size_t keysize) -{ - - /* We prefer AES-CBC for metadata protection. */ - if (algo == CRYPTO_AES_XTS) - algo = CRYPTO_AES_CBC; - - return (g_eli_crypto_cipher(algo, 0, data, datasize, key, keysize)); -} - void g_eli_crypto_hmac_init(struct hmac_ctx *ctx, const uint8_t *hkey, size_t hkeylen) @@ -293,3 +114,37 @@ g_eli_crypto_hmac_update(&ctx, data, datasize); g_eli_crypto_hmac_final(&ctx, md, mdsize); } + +/* + * Here we generate IV. It is unique for every sector. + */ +void +g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv, + size_t size) +{ + uint8_t off[8]; + + if ((sc->sc_flags & G_ELI_FLAG_NATIVE_BYTE_ORDER) != 0) + bcopy(&offset, off, sizeof(off)); + else + le64enc(off, (uint64_t)offset); + + switch (sc->sc_ealgo) { + case CRYPTO_AES_XTS: + bcopy(off, iv, sizeof(off)); + bzero(iv + sizeof(off), size - sizeof(off)); + break; + default: + { + u_char hash[SHA256_DIGEST_LENGTH]; + SHA256_CTX ctx; + + /* Copy precalculated SHA256 context for IV-Key. */ + bcopy(&sc->sc_ivctx, &ctx, sizeof(ctx)); + SHA256_Update(&ctx, off, sizeof(off)); + SHA256_Final(hash, &ctx); + bcopy(hash, iv, MIN(sizeof(hash), size)); + break; + } + } +} Index: sys/geom/eli/g_eli_key_cache.c =================================================================== --- sys/geom/eli/g_eli_key_cache.c +++ sys/geom/eli/g_eli_key_cache.c @@ -28,17 +28,20 @@ __FBSDID("$FreeBSD$"); #include +#ifdef _KERNEL #include #include #include #include #include #include +#endif /* _KERNEL */ #include #include +#ifdef _KERNEL MALLOC_DECLARE(M_ELI); SYSCTL_DECL(_kern_geom_eli); @@ -56,22 +59,7 @@ SYSCTL_UQUAD(_kern_geom_eli, OID_AUTO, key_cache_misses, CTLFLAG_RW, &g_eli_key_cache_misses, 0, "Key cache misses"); -#define G_ELI_KEY_MAGIC 0xe11341c - -struct g_eli_key { - /* Key value, must be first in the structure. */ - uint8_t gek_key[G_ELI_DATAKEYLEN]; - /* Magic. */ - int gek_magic; - /* Key number. */ - uint64_t gek_keyno; - /* Reference counter. */ - int gek_count; - /* Keeps keys sorted by most recent use. */ - TAILQ_ENTRY(g_eli_key) gek_next; - /* Keeps keys sorted by number. */ - RB_ENTRY(g_eli_key) gek_link; -}; +#endif /* _KERNEL */ static int g_eli_key_cmp(const struct g_eli_key *a, const struct g_eli_key *b) @@ -84,10 +72,7 @@ return (0); } -RB_PROTOTYPE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp); -RB_GENERATE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp); - -static void +void g_eli_key_fill(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno) { const uint8_t *ekey; @@ -110,6 +95,10 @@ key->gek_magic = G_ELI_KEY_MAGIC; } +#ifdef _KERNEL +RB_PROTOTYPE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp); +RB_GENERATE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp); + static struct g_eli_key * g_eli_key_allocate(struct g_eli_softc *sc, uint64_t keyno) { @@ -350,3 +339,4 @@ } mtx_unlock(&sc->sc_ekeys_lock); } +#endif /* _KERNEL */ Index: sys/geom/eli/pkcs5v2.c =================================================================== --- sys/geom/eli/pkcs5v2.c +++ sys/geom/eli/pkcs5v2.c @@ -83,6 +83,7 @@ } #ifndef _KERNEL +#ifndef _STAND /* * Return the number of microseconds needed for 'interations' iterations. */ @@ -120,4 +121,5 @@ } return (((intmax_t)iterations * (intmax_t)usecs) / v); } +#endif /* !_STAND */ #endif /* !_KERNEL */ Index: sys/modules/geom/geom_eli/Makefile =================================================================== --- sys/modules/geom/geom_eli/Makefile +++ sys/modules/geom/geom_eli/Makefile @@ -6,6 +6,7 @@ SRCS= g_eli.c SRCS+= g_eli_crypto.c SRCS+= g_eli_ctl.c +SRCS+= g_eli_hmac.c SRCS+= g_eli_integrity.c SRCS+= g_eli_key.c SRCS+= g_eli_key_cache.c