diff --git a/contrib/pam_zfs_key/pam_zfs_key.c b/contrib/pam_zfs_key/pam_zfs_key.c index 4cafc37b9b47..0856c7534f0d 100644 --- a/contrib/pam_zfs_key/pam_zfs_key.c +++ b/contrib/pam_zfs_key/pam_zfs_key.c @@ -1,795 +1,803 @@ /* * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Copyright (c) 2020, Felix Dörre * All rights reserved. */ #include #include #include #include #include #include #define PAM_SM_AUTH #define PAM_SM_PASSWORD #define PAM_SM_SESSION #include #if defined(__linux__) #include #elif defined(__FreeBSD__) #include static void pam_syslog(pam_handle_t *pamh, int loglevel, const char *fmt, ...) { va_list args; va_start(args, fmt); vsyslog(loglevel, fmt, args); va_end(args); } #endif #include #include #include #include #include #include #include static const char PASSWORD_VAR_NAME[] = "pam_zfs_key_authtok"; static libzfs_handle_t *g_zfs; static void destroy_pw(pam_handle_t *pamh, void *data, int errcode); typedef struct { size_t len; char *value; } pw_password_t; static pw_password_t * alloc_pw_size(size_t len) { pw_password_t *pw = malloc(sizeof (pw_password_t)); if (!pw) { return (NULL); } pw->len = len; - pw->value = malloc(len); + /* + * The use of malloc() triggers a spurious gcc 11 -Wmaybe-uninitialized + * warning in the mlock() function call below, so use calloc(). + */ + pw->value = calloc(len, 1); if (!pw->value) { free(pw); return (NULL); } mlock(pw->value, pw->len); return (pw); } static pw_password_t * alloc_pw_string(const char *source) { pw_password_t *pw = malloc(sizeof (pw_password_t)); if (!pw) { return (NULL); } pw->len = strlen(source) + 1; - pw->value = malloc(pw->len); + /* + * The use of malloc() triggers a spurious gcc 11 -Wmaybe-uninitialized + * warning in the mlock() function call below, so use calloc(). + */ + pw->value = calloc(pw->len, 1); if (!pw->value) { free(pw); return (NULL); } mlock(pw->value, pw->len); memcpy(pw->value, source, pw->len); return (pw); } static void pw_free(pw_password_t *pw) { bzero(pw->value, pw->len); munlock(pw->value, pw->len); free(pw->value); free(pw); } static pw_password_t * pw_fetch(pam_handle_t *pamh) { const char *token; if (pam_get_authtok(pamh, PAM_AUTHTOK, &token, NULL) != PAM_SUCCESS) { pam_syslog(pamh, LOG_ERR, "couldn't get password from PAM stack"); return (NULL); } if (!token) { pam_syslog(pamh, LOG_ERR, "token from PAM stack is null"); return (NULL); } return (alloc_pw_string(token)); } static const pw_password_t * pw_fetch_lazy(pam_handle_t *pamh) { pw_password_t *pw = pw_fetch(pamh); if (pw == NULL) { return (NULL); } int ret = pam_set_data(pamh, PASSWORD_VAR_NAME, pw, destroy_pw); if (ret != PAM_SUCCESS) { pw_free(pw); pam_syslog(pamh, LOG_ERR, "pam_set_data failed"); return (NULL); } return (pw); } static const pw_password_t * pw_get(pam_handle_t *pamh) { const pw_password_t *authtok = NULL; int ret = pam_get_data(pamh, PASSWORD_VAR_NAME, (const void**)(&authtok)); if (ret == PAM_SUCCESS) return (authtok); if (ret == PAM_NO_MODULE_DATA) return (pw_fetch_lazy(pamh)); pam_syslog(pamh, LOG_ERR, "password not available"); return (NULL); } static int pw_clear(pam_handle_t *pamh) { int ret = pam_set_data(pamh, PASSWORD_VAR_NAME, NULL, NULL); if (ret != PAM_SUCCESS) { pam_syslog(pamh, LOG_ERR, "clearing password failed"); return (-1); } return (0); } static void destroy_pw(pam_handle_t *pamh, void *data, int errcode) { if (data != NULL) { pw_free((pw_password_t *)data); } } static int pam_zfs_init(pam_handle_t *pamh) { int error = 0; if ((g_zfs = libzfs_init()) == NULL) { error = errno; pam_syslog(pamh, LOG_ERR, "Zfs initialization error: %s", libzfs_error_init(error)); } return (error); } static void pam_zfs_free(void) { libzfs_fini(g_zfs); } static pw_password_t * prepare_passphrase(pam_handle_t *pamh, zfs_handle_t *ds, const char *passphrase, nvlist_t *nvlist) { pw_password_t *key = alloc_pw_size(WRAPPING_KEY_LEN); if (!key) { return (NULL); } uint64_t salt; uint64_t iters; if (nvlist != NULL) { int fd = open("/dev/urandom", O_RDONLY); if (fd < 0) { pw_free(key); return (NULL); } int bytes_read = 0; char *buf = (char *)&salt; size_t bytes = sizeof (uint64_t); while (bytes_read < bytes) { ssize_t len = read(fd, buf + bytes_read, bytes - bytes_read); if (len < 0) { close(fd); pw_free(key); return (NULL); } bytes_read += len; } close(fd); if (nvlist_add_uint64(nvlist, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), salt)) { pam_syslog(pamh, LOG_ERR, "failed to add salt to nvlist"); pw_free(key); return (NULL); } iters = DEFAULT_PBKDF2_ITERATIONS; if (nvlist_add_uint64(nvlist, zfs_prop_to_name( ZFS_PROP_PBKDF2_ITERS), iters)) { pam_syslog(pamh, LOG_ERR, "failed to add iters to nvlist"); pw_free(key); return (NULL); } } else { salt = zfs_prop_get_int(ds, ZFS_PROP_PBKDF2_SALT); iters = zfs_prop_get_int(ds, ZFS_PROP_PBKDF2_ITERS); } salt = LE_64(salt); if (!PKCS5_PBKDF2_HMAC_SHA1((char *)passphrase, strlen(passphrase), (uint8_t *)&salt, sizeof (uint64_t), iters, WRAPPING_KEY_LEN, (uint8_t *)key->value)) { pam_syslog(pamh, LOG_ERR, "pbkdf failed"); pw_free(key); return (NULL); } return (key); } static int is_key_loaded(pam_handle_t *pamh, const char *ds_name) { zfs_handle_t *ds = zfs_open(g_zfs, ds_name, ZFS_TYPE_FILESYSTEM); if (ds == NULL) { pam_syslog(pamh, LOG_ERR, "dataset %s not found", ds_name); return (-1); } int keystatus = zfs_prop_get_int(ds, ZFS_PROP_KEYSTATUS); zfs_close(ds); return (keystatus != ZFS_KEYSTATUS_UNAVAILABLE); } static int change_key(pam_handle_t *pamh, const char *ds_name, const char *passphrase) { zfs_handle_t *ds = zfs_open(g_zfs, ds_name, ZFS_TYPE_FILESYSTEM); if (ds == NULL) { pam_syslog(pamh, LOG_ERR, "dataset %s not found", ds_name); return (-1); } nvlist_t *nvlist = fnvlist_alloc(); pw_password_t *key = prepare_passphrase(pamh, ds, passphrase, nvlist); if (key == NULL) { nvlist_free(nvlist); zfs_close(ds); return (-1); } if (nvlist_add_string(nvlist, zfs_prop_to_name(ZFS_PROP_KEYLOCATION), "prompt")) { pam_syslog(pamh, LOG_ERR, "nvlist_add failed for keylocation"); pw_free(key); nvlist_free(nvlist); zfs_close(ds); return (-1); } if (nvlist_add_uint64(nvlist, zfs_prop_to_name(ZFS_PROP_KEYFORMAT), ZFS_KEYFORMAT_PASSPHRASE)) { pam_syslog(pamh, LOG_ERR, "nvlist_add failed for keyformat"); pw_free(key); nvlist_free(nvlist); zfs_close(ds); return (-1); } int ret = lzc_change_key(ds_name, DCP_CMD_NEW_KEY, nvlist, (uint8_t *)key->value, WRAPPING_KEY_LEN); pw_free(key); if (ret) { pam_syslog(pamh, LOG_ERR, "change_key failed: %d", ret); nvlist_free(nvlist); zfs_close(ds); return (-1); } nvlist_free(nvlist); zfs_close(ds); return (0); } static int decrypt_mount(pam_handle_t *pamh, const char *ds_name, const char *passphrase) { zfs_handle_t *ds = zfs_open(g_zfs, ds_name, ZFS_TYPE_FILESYSTEM); if (ds == NULL) { pam_syslog(pamh, LOG_ERR, "dataset %s not found", ds_name); return (-1); } pw_password_t *key = prepare_passphrase(pamh, ds, passphrase, NULL); if (key == NULL) { zfs_close(ds); return (-1); } int ret = lzc_load_key(ds_name, B_FALSE, (uint8_t *)key->value, WRAPPING_KEY_LEN); pw_free(key); if (ret) { pam_syslog(pamh, LOG_ERR, "load_key failed: %d", ret); zfs_close(ds); return (-1); } ret = zfs_mount(ds, NULL, 0); if (ret) { pam_syslog(pamh, LOG_ERR, "mount failed: %d", ret); zfs_close(ds); return (-1); } zfs_close(ds); return (0); } static int unmount_unload(pam_handle_t *pamh, const char *ds_name) { zfs_handle_t *ds = zfs_open(g_zfs, ds_name, ZFS_TYPE_FILESYSTEM); if (ds == NULL) { pam_syslog(pamh, LOG_ERR, "dataset %s not found", ds_name); return (-1); } int ret = zfs_unmount(ds, NULL, 0); if (ret) { pam_syslog(pamh, LOG_ERR, "zfs_unmount failed with: %d", ret); zfs_close(ds); return (-1); } ret = lzc_unload_key(ds_name); if (ret) { pam_syslog(pamh, LOG_ERR, "unload_key failed with: %d", ret); zfs_close(ds); return (-1); } zfs_close(ds); return (0); } typedef struct { char *homes_prefix; char *runstatedir; char *homedir; char *dsname; uid_t uid; const char *username; int unmount_and_unload; } zfs_key_config_t; static int zfs_key_config_load(pam_handle_t *pamh, zfs_key_config_t *config, int argc, const char **argv) { config->homes_prefix = strdup("rpool/home"); if (config->homes_prefix == NULL) { pam_syslog(pamh, LOG_ERR, "strdup failure"); return (-1); } config->runstatedir = strdup(RUNSTATEDIR "/pam_zfs_key"); if (config->runstatedir == NULL) { pam_syslog(pamh, LOG_ERR, "strdup failure"); free(config->homes_prefix); return (-1); } const char *name; if (pam_get_user(pamh, &name, NULL) != PAM_SUCCESS) { pam_syslog(pamh, LOG_ERR, "couldn't get username from PAM stack"); free(config->runstatedir); free(config->homes_prefix); return (-1); } struct passwd *entry = getpwnam(name); if (!entry) { free(config->runstatedir); free(config->homes_prefix); return (-1); } config->uid = entry->pw_uid; config->username = name; config->unmount_and_unload = 1; config->dsname = NULL; config->homedir = NULL; for (int c = 0; c < argc; c++) { if (strncmp(argv[c], "homes=", 6) == 0) { free(config->homes_prefix); config->homes_prefix = strdup(argv[c] + 6); } else if (strncmp(argv[c], "runstatedir=", 12) == 0) { free(config->runstatedir); config->runstatedir = strdup(argv[c] + 12); } else if (strcmp(argv[c], "nounmount") == 0) { config->unmount_and_unload = 0; } else if (strcmp(argv[c], "prop_mountpoint") == 0) { config->homedir = strdup(entry->pw_dir); } } return (0); } static void zfs_key_config_free(zfs_key_config_t *config) { free(config->homes_prefix); free(config->runstatedir); free(config->homedir); free(config->dsname); } static int find_dsname_by_prop_value(zfs_handle_t *zhp, void *data) { zfs_type_t type = zfs_get_type(zhp); zfs_key_config_t *target = data; char mountpoint[ZFS_MAXPROPLEN]; /* Skip any datasets whose type does not match */ if ((type & ZFS_TYPE_FILESYSTEM) == 0) { zfs_close(zhp); return (0); } /* Skip any datasets whose mountpoint does not match */ (void) zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint, sizeof (mountpoint), NULL, NULL, 0, B_FALSE); if (strcmp(target->homedir, mountpoint) != 0) { zfs_close(zhp); return (0); } target->dsname = strdup(zfs_get_name(zhp)); zfs_close(zhp); return (1); } static char * zfs_key_config_get_dataset(zfs_key_config_t *config) { if (config->homedir != NULL && config->homes_prefix != NULL) { zfs_handle_t *zhp = zfs_open(g_zfs, config->homes_prefix, ZFS_TYPE_FILESYSTEM); if (zhp == NULL) { pam_syslog(NULL, LOG_ERR, "dataset %s not found", config->homes_prefix); zfs_close(zhp); return (NULL); } (void) zfs_iter_filesystems(zhp, find_dsname_by_prop_value, config); zfs_close(zhp); char *dsname = config->dsname; config->dsname = NULL; return (dsname); } size_t len = ZFS_MAX_DATASET_NAME_LEN; size_t total_len = strlen(config->homes_prefix) + 1 + strlen(config->username); if (total_len > len) { return (NULL); } char *ret = malloc(len + 1); if (!ret) { return (NULL); } ret[0] = 0; strcat(ret, config->homes_prefix); strcat(ret, "/"); strcat(ret, config->username); return (ret); } static int zfs_key_config_modify_session_counter(pam_handle_t *pamh, zfs_key_config_t *config, int delta) { const char *runtime_path = config->runstatedir; if (mkdir(runtime_path, S_IRWXU) != 0 && errno != EEXIST) { pam_syslog(pamh, LOG_ERR, "Can't create runtime path: %d", errno); return (-1); } if (chown(runtime_path, 0, 0) != 0) { pam_syslog(pamh, LOG_ERR, "Can't chown runtime path: %d", errno); return (-1); } if (chmod(runtime_path, S_IRWXU) != 0) { pam_syslog(pamh, LOG_ERR, "Can't chmod runtime path: %d", errno); return (-1); } size_t runtime_path_len = strlen(runtime_path); size_t counter_path_len = runtime_path_len + 1 + 10; char *counter_path = malloc(counter_path_len + 1); if (!counter_path) { return (-1); } counter_path[0] = 0; strcat(counter_path, runtime_path); snprintf(counter_path + runtime_path_len, counter_path_len, "/%d", config->uid); const int fd = open(counter_path, O_RDWR | O_CLOEXEC | O_CREAT | O_NOFOLLOW, S_IRUSR | S_IWUSR); free(counter_path); if (fd < 0) { pam_syslog(pamh, LOG_ERR, "Can't open counter file: %d", errno); return (-1); } if (flock(fd, LOCK_EX) != 0) { pam_syslog(pamh, LOG_ERR, "Can't lock counter file: %d", errno); close(fd); return (-1); } char counter[20]; char *pos = counter; int remaining = sizeof (counter) - 1; int ret; counter[sizeof (counter) - 1] = 0; while (remaining > 0 && (ret = read(fd, pos, remaining)) > 0) { remaining -= ret; pos += ret; } *pos = 0; long int counter_value = strtol(counter, NULL, 10); counter_value += delta; if (counter_value < 0) { counter_value = 0; } lseek(fd, 0, SEEK_SET); if (ftruncate(fd, 0) != 0) { pam_syslog(pamh, LOG_ERR, "Can't truncate counter file: %d", errno); close(fd); return (-1); } snprintf(counter, sizeof (counter), "%ld", counter_value); remaining = strlen(counter); pos = counter; while (remaining > 0 && (ret = write(fd, pos, remaining)) > 0) { remaining -= ret; pos += ret; } close(fd); return (counter_value); } __attribute__((visibility("default"))) PAM_EXTERN int pam_sm_authenticate(pam_handle_t *pamh, int flags, int argc, const char **argv) { if (pw_fetch_lazy(pamh) == NULL) { return (PAM_AUTH_ERR); } return (PAM_SUCCESS); } __attribute__((visibility("default"))) PAM_EXTERN int pam_sm_setcred(pam_handle_t *pamh, int flags, int argc, const char **argv) { return (PAM_SUCCESS); } __attribute__((visibility("default"))) PAM_EXTERN int pam_sm_chauthtok(pam_handle_t *pamh, int flags, int argc, const char **argv) { if (geteuid() != 0) { pam_syslog(pamh, LOG_ERR, "Cannot zfs_mount when not being root."); return (PAM_PERM_DENIED); } zfs_key_config_t config; if (zfs_key_config_load(pamh, &config, argc, argv) == -1) { return (PAM_SERVICE_ERR); } if (config.uid < 1000) { zfs_key_config_free(&config); return (PAM_SUCCESS); } { if (pam_zfs_init(pamh) != 0) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } char *dataset = zfs_key_config_get_dataset(&config); if (!dataset) { pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } int key_loaded = is_key_loaded(pamh, dataset); if (key_loaded == -1) { free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } free(dataset); pam_zfs_free(); if (! key_loaded) { pam_syslog(pamh, LOG_ERR, "key not loaded, returning try_again"); zfs_key_config_free(&config); return (PAM_PERM_DENIED); } } if ((flags & PAM_UPDATE_AUTHTOK) != 0) { const pw_password_t *token = pw_get(pamh); if (token == NULL) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } if (pam_zfs_init(pamh) != 0) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } char *dataset = zfs_key_config_get_dataset(&config); if (!dataset) { pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } if (change_key(pamh, dataset, token->value) == -1) { free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } free(dataset); pam_zfs_free(); zfs_key_config_free(&config); if (pw_clear(pamh) == -1) { return (PAM_SERVICE_ERR); } } else { zfs_key_config_free(&config); } return (PAM_SUCCESS); } PAM_EXTERN int pam_sm_open_session(pam_handle_t *pamh, int flags, int argc, const char **argv) { if (geteuid() != 0) { pam_syslog(pamh, LOG_ERR, "Cannot zfs_mount when not being root."); return (PAM_SUCCESS); } zfs_key_config_t config; zfs_key_config_load(pamh, &config, argc, argv); if (config.uid < 1000) { zfs_key_config_free(&config); return (PAM_SUCCESS); } int counter = zfs_key_config_modify_session_counter(pamh, &config, 1); if (counter != 1) { zfs_key_config_free(&config); return (PAM_SUCCESS); } const pw_password_t *token = pw_get(pamh); if (token == NULL) { zfs_key_config_free(&config); return (PAM_SESSION_ERR); } if (pam_zfs_init(pamh) != 0) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } char *dataset = zfs_key_config_get_dataset(&config); if (!dataset) { pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } if (decrypt_mount(pamh, dataset, token->value) == -1) { free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } free(dataset); pam_zfs_free(); zfs_key_config_free(&config); if (pw_clear(pamh) == -1) { return (PAM_SERVICE_ERR); } return (PAM_SUCCESS); } __attribute__((visibility("default"))) PAM_EXTERN int pam_sm_close_session(pam_handle_t *pamh, int flags, int argc, const char **argv) { if (geteuid() != 0) { pam_syslog(pamh, LOG_ERR, "Cannot zfs_mount when not being root."); return (PAM_SUCCESS); } zfs_key_config_t config; zfs_key_config_load(pamh, &config, argc, argv); if (config.uid < 1000) { zfs_key_config_free(&config); return (PAM_SUCCESS); } int counter = zfs_key_config_modify_session_counter(pamh, &config, -1); if (counter != 0) { zfs_key_config_free(&config); return (PAM_SUCCESS); } if (config.unmount_and_unload) { if (pam_zfs_init(pamh) != 0) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } char *dataset = zfs_key_config_get_dataset(&config); if (!dataset) { pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SESSION_ERR); } if (unmount_unload(pamh, dataset) == -1) { free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SESSION_ERR); } free(dataset); pam_zfs_free(); } zfs_key_config_free(&config); return (PAM_SUCCESS); } diff --git a/include/sys/crypto/api.h b/include/sys/crypto/api.h index 7c3c465513de..8aecfeaff0f4 100644 --- a/include/sys/crypto/api.h +++ b/include/sys/crypto/api.h @@ -1,425 +1,425 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _SYS_CRYPTO_API_H #define _SYS_CRYPTO_API_H #ifdef __cplusplus extern "C" { #endif #include #include typedef long crypto_req_id_t; typedef void *crypto_bc_t; typedef void *crypto_context_t; typedef void *crypto_ctx_template_t; typedef uint32_t crypto_call_flag_t; /* crypto_call_flag's values */ #define CRYPTO_ALWAYS_QUEUE 0x00000001 /* ALWAYS queue the req. */ #define CRYPTO_NOTIFY_OPDONE 0x00000002 /* Notify intermediate steps */ #define CRYPTO_SKIP_REQID 0x00000004 /* Skip request ID generation */ #define CRYPTO_RESTRICTED 0x00000008 /* cannot use restricted prov */ typedef struct { crypto_call_flag_t cr_flag; void (*cr_callback_func)(void *, int); void *cr_callback_arg; crypto_req_id_t cr_reqid; } crypto_call_req_t; /* * Returns the mechanism type corresponding to a mechanism name. */ #define CRYPTO_MECH_INVALID ((uint64_t)-1) -extern crypto_mech_type_t crypto_mech2id(crypto_mech_name_t name); +extern crypto_mech_type_t crypto_mech2id(char *name); /* * Create and destroy context templates. */ extern int crypto_create_ctx_template(crypto_mechanism_t *mech, crypto_key_t *key, crypto_ctx_template_t *tmpl, int kmflag); extern void crypto_destroy_ctx_template(crypto_ctx_template_t tmpl); /* * Single and multi-part digest operations. */ extern int crypto_digest(crypto_mechanism_t *mech, crypto_data_t *data, crypto_data_t *digest, crypto_call_req_t *cr); extern int crypto_digest_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_data_t *, crypto_data_t *, crypto_call_req_t *); extern int crypto_digest_init(crypto_mechanism_t *mech, crypto_context_t *ctxp, crypto_call_req_t *cr); extern int crypto_digest_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_context_t *, crypto_call_req_t *); extern int crypto_digest_update(crypto_context_t ctx, crypto_data_t *data, crypto_call_req_t *cr); extern int crypto_digest_final(crypto_context_t ctx, crypto_data_t *digest, crypto_call_req_t *cr); /* * Single and multi-part MAC operations. */ extern int crypto_mac(crypto_mechanism_t *mech, crypto_data_t *data, crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac, crypto_call_req_t *cr); extern int crypto_mac_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_data_t *, crypto_key_t *, crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); extern int crypto_mac_verify(crypto_mechanism_t *mech, crypto_data_t *data, crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac, crypto_call_req_t *cr); extern int crypto_mac_verify_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_data_t *, crypto_key_t *, crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); extern int crypto_mac_init(crypto_mechanism_t *mech, crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); extern int crypto_mac_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, crypto_context_t *, crypto_call_req_t *); extern int crypto_mac_update(crypto_context_t ctx, crypto_data_t *data, crypto_call_req_t *cr); extern int crypto_mac_final(crypto_context_t ctx, crypto_data_t *data, crypto_call_req_t *cr); /* * Single and multi-part sign with private key operations. */ extern int crypto_sign(crypto_mechanism_t *mech, crypto_key_t *key, crypto_data_t *data, crypto_ctx_template_t tmpl, crypto_data_t *signature, crypto_call_req_t *cr); extern int crypto_sign_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); extern int crypto_sign_init(crypto_mechanism_t *mech, crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); extern int crypto_sign_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, crypto_context_t *, crypto_call_req_t *); extern int crypto_sign_update(crypto_context_t ctx, crypto_data_t *data, crypto_call_req_t *cr); extern int crypto_sign_final(crypto_context_t ctx, crypto_data_t *signature, crypto_call_req_t *cr); extern int crypto_sign_recover_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t tmpl, crypto_context_t *, crypto_call_req_t *); extern int crypto_sign_recover(crypto_mechanism_t *mech, crypto_key_t *key, crypto_data_t *data, crypto_ctx_template_t tmpl, crypto_data_t *signature, crypto_call_req_t *cr); extern int crypto_sign_recover_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); /* * Single and multi-part verify with public key operations. */ extern int crypto_verify(crypto_mechanism_t *mech, crypto_key_t *key, crypto_data_t *data, crypto_ctx_template_t tmpl, crypto_data_t *signature, crypto_call_req_t *cr); extern int crypto_verify_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); extern int crypto_verify_init(crypto_mechanism_t *mech, crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); extern int crypto_verify_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, crypto_context_t *, crypto_call_req_t *); extern int crypto_verify_update(crypto_context_t ctx, crypto_data_t *data, crypto_call_req_t *cr); extern int crypto_verify_final(crypto_context_t ctx, crypto_data_t *signature, crypto_call_req_t *cr); extern int crypto_verify_recover_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t tmpl, crypto_context_t *, crypto_call_req_t *); extern int crypto_verify_recover(crypto_mechanism_t *mech, crypto_key_t *key, crypto_data_t *signature, crypto_ctx_template_t tmpl, crypto_data_t *data, crypto_call_req_t *cr); extern int crypto_verify_recover_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); /* * Single and multi-part encryption operations. */ extern int crypto_encrypt(crypto_mechanism_t *mech, crypto_data_t *plaintext, crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *ciphertext, crypto_call_req_t *cr); extern int crypto_encrypt_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_data_t *, crypto_key_t *, crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); extern int crypto_encrypt_init(crypto_mechanism_t *mech, crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); extern int crypto_encrypt_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, crypto_context_t *, crypto_call_req_t *); extern int crypto_encrypt_update(crypto_context_t ctx, crypto_data_t *plaintext, crypto_data_t *ciphertext, crypto_call_req_t *cr); extern int crypto_encrypt_final(crypto_context_t ctx, crypto_data_t *ciphertext, crypto_call_req_t *cr); /* * Single and multi-part decryption operations. */ extern int crypto_decrypt(crypto_mechanism_t *mech, crypto_data_t *ciphertext, crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *plaintext, crypto_call_req_t *cr); extern int crypto_decrypt_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_data_t *, crypto_key_t *, crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); extern int crypto_decrypt_init(crypto_mechanism_t *mech, crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); extern int crypto_decrypt_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, crypto_context_t *, crypto_call_req_t *); extern int crypto_decrypt_update(crypto_context_t ctx, crypto_data_t *ciphertext, crypto_data_t *plaintext, crypto_call_req_t *cr); extern int crypto_decrypt_final(crypto_context_t ctx, crypto_data_t *plaintext, crypto_call_req_t *cr); /* * Single and multi-part encrypt/MAC dual operations. */ extern int crypto_encrypt_mac(crypto_mechanism_t *encr_mech, crypto_mechanism_t *mac_mech, crypto_data_t *pt, crypto_key_t *encr_key, crypto_key_t *mac_key, crypto_ctx_template_t encr_tmpl, crypto_ctx_template_t mac_tmpl, crypto_dual_data_t *ct, crypto_data_t *mac, crypto_call_req_t *cr); extern int crypto_encrypt_mac_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_mechanism_t *, crypto_data_t *, crypto_key_t *, crypto_key_t *, crypto_ctx_template_t, crypto_ctx_template_t, crypto_dual_data_t *, crypto_data_t *, crypto_call_req_t *); extern int crypto_encrypt_mac_init(crypto_mechanism_t *encr_mech, crypto_mechanism_t *mac_mech, crypto_key_t *encr_key, crypto_key_t *mac_key, crypto_ctx_template_t encr_tmpl, crypto_ctx_template_t mac_tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); extern int crypto_encrypt_mac_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_mechanism_t *, crypto_key_t *, crypto_key_t *, crypto_ctx_template_t, crypto_ctx_template_t, crypto_context_t *, crypto_call_req_t *); extern int crypto_encrypt_mac_update(crypto_context_t ctx, crypto_data_t *pt, crypto_dual_data_t *ct, crypto_call_req_t *cr); extern int crypto_encrypt_mac_final(crypto_context_t ctx, crypto_dual_data_t *ct, crypto_data_t *mac, crypto_call_req_t *cr); /* * Single and multi-part MAC/decrypt dual operations. */ extern int crypto_mac_decrypt(crypto_mechanism_t *mac_mech, crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct, crypto_key_t *mac_key, crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr); extern int crypto_mac_decrypt_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *mac_mech, crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct, crypto_key_t *mac_key, crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr); extern int crypto_mac_verify_decrypt(crypto_mechanism_t *mac_mech, crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct, crypto_key_t *mac_key, crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr); extern int crypto_mac_verify_decrypt_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *mac_mech, crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct, crypto_key_t *mac_key, crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr); extern int crypto_mac_decrypt_init(crypto_mechanism_t *mac_mech, crypto_mechanism_t *decr_mech, crypto_key_t *mac_key, crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); extern int crypto_mac_decrypt_init_prov(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *mac_mech, crypto_mechanism_t *decr_mech, crypto_key_t *mac_key, crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); extern int crypto_mac_decrypt_update(crypto_context_t ctx, crypto_dual_data_t *ct, crypto_data_t *pt, crypto_call_req_t *cr); extern int crypto_mac_decrypt_final(crypto_context_t ctx, crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr); /* Session Management */ extern int crypto_session_open(crypto_provider_t, crypto_session_id_t *, crypto_call_req_t *); extern int crypto_session_close(crypto_provider_t, crypto_session_id_t, crypto_call_req_t *); extern int crypto_session_login(crypto_provider_t, crypto_session_id_t, crypto_user_type_t, char *, size_t, crypto_call_req_t *); extern int crypto_session_logout(crypto_provider_t, crypto_session_id_t, crypto_call_req_t *); /* Object Management */ extern int crypto_object_copy(crypto_provider_t, crypto_session_id_t, crypto_object_id_t, crypto_object_attribute_t *, uint_t, crypto_object_id_t *, crypto_call_req_t *); extern int crypto_object_create(crypto_provider_t, crypto_session_id_t, crypto_object_attribute_t *, uint_t, crypto_object_id_t *, crypto_call_req_t *); extern int crypto_object_destroy(crypto_provider_t, crypto_session_id_t, crypto_object_id_t, crypto_call_req_t *); extern int crypto_object_get_attribute_value(crypto_provider_t, crypto_session_id_t, crypto_object_id_t, crypto_object_attribute_t *, uint_t, crypto_call_req_t *); extern int crypto_object_get_size(crypto_provider_t, crypto_session_id_t, crypto_object_id_t, size_t *, crypto_call_req_t *); extern int crypto_object_find_final(crypto_provider_t, void *, crypto_call_req_t *); extern int crypto_object_find_init(crypto_provider_t, crypto_session_id_t, crypto_object_attribute_t *, uint_t, void **, crypto_call_req_t *); extern int crypto_object_find(crypto_provider_t, void *, crypto_object_id_t *, uint_t *, uint_t, crypto_call_req_t *); extern int crypto_object_set_attribute_value(crypto_provider_t, crypto_session_id_t, crypto_object_id_t, crypto_object_attribute_t *, uint_t, crypto_call_req_t *); /* Key Management */ extern int crypto_key_derive(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_object_attribute_t *, uint_t, crypto_object_id_t *, crypto_call_req_t *); extern int crypto_key_generate(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_object_attribute_t *, uint_t, crypto_object_id_t *, crypto_call_req_t *); extern int crypto_key_generate_pair(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_object_attribute_t *, uint_t, crypto_object_attribute_t *, uint_t, crypto_object_id_t *, crypto_object_id_t *, crypto_call_req_t *); extern int crypto_key_unwrap(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, uchar_t *, size_t *, crypto_object_attribute_t *, uint_t, crypto_object_id_t *, crypto_call_req_t *); extern int crypto_key_wrap(crypto_provider_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_object_id_t *, uchar_t *, size_t *, crypto_call_req_t *); extern int crypto_key_check_prov(crypto_provider_t, crypto_mechanism_t *mech, crypto_key_t *key); extern int crypto_key_check(crypto_mechanism_t *mech, crypto_key_t *key); /* * Routines to cancel a single asynchronous request or all asynchronous * requests associated with a particular context. */ extern void crypto_cancel_req(crypto_req_id_t req); extern void crypto_cancel_ctx(crypto_context_t ctx); /* * crypto_get_mech_list(9F) allocates and returns the list of currently * supported cryptographic mechanisms. */ extern crypto_mech_name_t *crypto_get_mech_list(uint_t *count, int kmflag); extern void crypto_free_mech_list(crypto_mech_name_t *mech_names, uint_t count); extern crypto_provider_t crypto_get_provider(char *, char *, char *); extern int crypto_get_provinfo(crypto_provider_t, crypto_provider_ext_info_t *); extern void crypto_release_provider(crypto_provider_t); /* * A kernel consumer can request to be notified when some particular event * occurs. The valid events, callback function type, and functions to * be called to register or unregister for notification are defined below. */ #define CRYPTO_EVENT_MECHS_CHANGED 0x00000001 #define CRYPTO_EVENT_PROVIDER_REGISTERED 0x00000002 #define CRYPTO_EVENT_PROVIDER_UNREGISTERED 0x00000004 typedef enum { CRYPTO_MECH_ADDED = 1, CRYPTO_MECH_REMOVED } crypto_event_change_t; /* The event_arg argument structure for CRYPTO_EVENT_PROVIDERS_CHANGE event */ typedef struct crypto_notify_event_change { crypto_mech_name_t ec_mech_name; crypto_provider_type_t ec_provider_type; crypto_event_change_t ec_change; } crypto_notify_event_change_t; typedef void *crypto_notify_handle_t; typedef void (*crypto_notify_callback_t)(uint32_t event_mask, void *event_arg); extern crypto_notify_handle_t crypto_notify_events( crypto_notify_callback_t nf, uint32_t event_mask); extern void crypto_unnotify_events(crypto_notify_handle_t); /* * crypto_bufcall(9F) group of routines. */ extern crypto_bc_t crypto_bufcall_alloc(void); extern int crypto_bufcall_free(crypto_bc_t bc); extern int crypto_bufcall(crypto_bc_t bc, void (*func)(void *arg), void *arg); extern int crypto_unbufcall(crypto_bc_t bc); /* * To obtain the list of key size ranges supported by a mechanism. */ #define CRYPTO_MECH_USAGE_ENCRYPT 0x00000001 #define CRYPTO_MECH_USAGE_DECRYPT 0x00000002 #define CRYPTO_MECH_USAGE_MAC 0x00000004 typedef uint32_t crypto_mech_usage_t; typedef struct crypto_mechanism_info { size_t mi_min_key_size; size_t mi_max_key_size; crypto_keysize_unit_t mi_keysize_unit; /* for mi_xxx_key_size */ crypto_mech_usage_t mi_usage; } crypto_mechanism_info_t; #ifdef _SYSCALL32 typedef struct crypto_mechanism_info32 { size32_t mi_min_key_size; size32_t mi_max_key_size; crypto_keysize_unit_t mi_keysize_unit; /* for mi_xxx_key_size */ crypto_mech_usage_t mi_usage; } crypto_mechanism_info32_t; #endif /* _SYSCALL32 */ extern int crypto_get_all_mech_info(crypto_mech_type_t, crypto_mechanism_info_t **, uint_t *, int); extern void crypto_free_all_mech_info(crypto_mechanism_info_t *, uint_t); #ifdef __cplusplus } #endif #endif /* _SYS_CRYPTO_API_H */ diff --git a/include/sys/dnode.h b/include/sys/dnode.h index 3208b60f0e7b..de6492bb7618 100644 --- a/include/sys/dnode.h +++ b/include/sys/dnode.h @@ -1,627 +1,627 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2018 by Delphix. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. */ #ifndef _SYS_DNODE_H #define _SYS_DNODE_H #include #include #include #include #include #include #include #include #include #ifdef __cplusplus extern "C" { #endif /* * dnode_hold() flags. */ #define DNODE_MUST_BE_ALLOCATED 1 #define DNODE_MUST_BE_FREE 2 #define DNODE_DRY_RUN 4 /* * dnode_next_offset() flags. */ #define DNODE_FIND_HOLE 1 #define DNODE_FIND_BACKWARDS 2 #define DNODE_FIND_HAVELOCK 4 /* * Fixed constants. */ #define DNODE_SHIFT 9 /* 512 bytes */ #define DN_MIN_INDBLKSHIFT 12 /* 4k */ /* * If we ever increase this value beyond 20, we need to revisit all logic that * does x << level * ebps to handle overflow. With a 1M indirect block size, * 4 levels of indirect blocks would not be able to guarantee addressing an * entire object, so 5 levels will be used, but 5 * (20 - 7) = 65. */ #define DN_MAX_INDBLKSHIFT 17 /* 128k */ #define DNODE_BLOCK_SHIFT 14 /* 16k */ #define DNODE_CORE_SIZE 64 /* 64 bytes for dnode sans blkptrs */ #define DN_MAX_OBJECT_SHIFT 48 /* 256 trillion (zfs_fid_t limit) */ #define DN_MAX_OFFSET_SHIFT 64 /* 2^64 bytes in a dnode */ /* * dnode id flags * * Note: a file will never ever have its ids moved from bonus->spill */ #define DN_ID_CHKED_BONUS 0x1 #define DN_ID_CHKED_SPILL 0x2 #define DN_ID_OLD_EXIST 0x4 #define DN_ID_NEW_EXIST 0x8 /* * Derived constants. */ #define DNODE_MIN_SIZE (1 << DNODE_SHIFT) #define DNODE_MAX_SIZE (1 << DNODE_BLOCK_SHIFT) #define DNODE_BLOCK_SIZE (1 << DNODE_BLOCK_SHIFT) #define DNODE_MIN_SLOTS (DNODE_MIN_SIZE >> DNODE_SHIFT) #define DNODE_MAX_SLOTS (DNODE_MAX_SIZE >> DNODE_SHIFT) #define DN_BONUS_SIZE(dnsize) ((dnsize) - DNODE_CORE_SIZE - \ (1 << SPA_BLKPTRSHIFT)) #define DN_SLOTS_TO_BONUSLEN(slots) DN_BONUS_SIZE((slots) << DNODE_SHIFT) #define DN_OLD_MAX_BONUSLEN (DN_BONUS_SIZE(DNODE_MIN_SIZE)) #define DN_MAX_NBLKPTR ((DNODE_MIN_SIZE - DNODE_CORE_SIZE) >> SPA_BLKPTRSHIFT) #define DN_MAX_OBJECT (1ULL << DN_MAX_OBJECT_SHIFT) #define DN_ZERO_BONUSLEN (DN_BONUS_SIZE(DNODE_MAX_SIZE) + 1) #define DN_KILL_SPILLBLK (1) #define DN_SLOT_UNINIT ((void *)NULL) /* Uninitialized */ #define DN_SLOT_FREE ((void *)1UL) /* Free slot */ #define DN_SLOT_ALLOCATED ((void *)2UL) /* Allocated slot */ #define DN_SLOT_INTERIOR ((void *)3UL) /* Interior allocated slot */ #define DN_SLOT_IS_PTR(dn) ((void *)dn > DN_SLOT_INTERIOR) #define DN_SLOT_IS_VALID(dn) ((void *)dn != NULL) #define DNODES_PER_BLOCK_SHIFT (DNODE_BLOCK_SHIFT - DNODE_SHIFT) #define DNODES_PER_BLOCK (1ULL << DNODES_PER_BLOCK_SHIFT) /* * This is inaccurate if the indblkshift of the particular object is not the * max. But it's only used by userland to calculate the zvol reservation. */ #define DNODES_PER_LEVEL_SHIFT (DN_MAX_INDBLKSHIFT - SPA_BLKPTRSHIFT) #define DNODES_PER_LEVEL (1ULL << DNODES_PER_LEVEL_SHIFT) #define DN_MAX_LEVELS (DIV_ROUND_UP(DN_MAX_OFFSET_SHIFT - SPA_MINBLOCKSHIFT, \ DN_MIN_INDBLKSHIFT - SPA_BLKPTRSHIFT) + 1) #define DN_BONUS(dnp) ((void*)((dnp)->dn_bonus + \ (((dnp)->dn_nblkptr - 1) * sizeof (blkptr_t)))) #define DN_MAX_BONUS_LEN(dnp) \ ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) ? \ (uint8_t *)DN_SPILL_BLKPTR(dnp) - (uint8_t *)DN_BONUS(dnp) : \ (uint8_t *)(dnp + (dnp->dn_extra_slots + 1)) - (uint8_t *)DN_BONUS(dnp)) #define DN_USED_BYTES(dnp) (((dnp)->dn_flags & DNODE_FLAG_USED_BYTES) ? \ (dnp)->dn_used : (dnp)->dn_used << SPA_MINBLOCKSHIFT) #define EPB(blkshift, typeshift) (1 << (blkshift - typeshift)) struct dmu_buf_impl; struct objset; struct zio; enum dnode_dirtycontext { DN_UNDIRTIED, DN_DIRTY_OPEN, DN_DIRTY_SYNC }; /* Is dn_used in bytes? if not, it's in multiples of SPA_MINBLOCKSIZE */ #define DNODE_FLAG_USED_BYTES (1 << 0) #define DNODE_FLAG_USERUSED_ACCOUNTED (1 << 1) /* Does dnode have a SA spill blkptr in bonus? */ #define DNODE_FLAG_SPILL_BLKPTR (1 << 2) /* User/Group/Project dnode accounting */ #define DNODE_FLAG_USEROBJUSED_ACCOUNTED (1 << 3) /* * This mask defines the set of flags which are "portable", meaning * that they can be preserved when doing a raw encrypted zfs send. * Flags included in this mask will be protected by AAD when the block * of dnodes is encrypted. */ #define DNODE_CRYPT_PORTABLE_FLAGS_MASK (DNODE_FLAG_SPILL_BLKPTR) /* * VARIABLE-LENGTH (LARGE) DNODES * * The motivation for variable-length dnodes is to eliminate the overhead * associated with using spill blocks. Spill blocks are used to store * system attribute data (i.e. file metadata) that does not fit in the * dnode's bonus buffer. By allowing a larger bonus buffer area the use of * a spill block can be avoided. Spill blocks potentially incur an * additional read I/O for every dnode in a dnode block. As a worst case * example, reading 32 dnodes from a 16k dnode block and all of the spill * blocks could issue 33 separate reads. Now suppose those dnodes have size * 1024 and therefore don't need spill blocks. Then the worst case number - * of blocks read is reduced to from 33 to two--one per dnode block. + * of blocks read is reduced from 33 to two--one per dnode block. * * ZFS-on-Linux systems that make heavy use of extended attributes benefit * from this feature. In particular, ZFS-on-Linux supports the xattr=sa * dataset property which allows file extended attribute data to be stored * in the dnode bonus buffer as an alternative to the traditional * directory-based format. Workloads such as SELinux and the Lustre * distributed filesystem often store enough xattr data to force spill * blocks when xattr=sa is in effect. Large dnodes may therefore provide a * performance benefit to such systems. Other use cases that benefit from * this feature include files with large ACLs and symbolic links with long * target names. * * The size of a dnode may be a multiple of 512 bytes up to the size of a * dnode block (currently 16384 bytes). The dn_extra_slots field of the * on-disk dnode_phys_t structure describes the size of the physical dnode * on disk. The field represents how many "extra" dnode_phys_t slots a * dnode consumes in its dnode block. This convention results in a value of * 0 for 512 byte dnodes which preserves on-disk format compatibility with * older software which doesn't support large dnodes. * * Similarly, the in-memory dnode_t structure has a dn_num_slots field * to represent the total number of dnode_phys_t slots consumed on disk. * Thus dn->dn_num_slots is 1 greater than the corresponding * dnp->dn_extra_slots. This difference in convention was adopted * because, unlike on-disk structures, backward compatibility is not a * concern for in-memory objects, so we used a more natural way to * represent size for a dnode_t. * * The default size for newly created dnodes is determined by the value of * the "dnodesize" dataset property. By default the property is set to * "legacy" which is compatible with older software. Setting the property * to "auto" will allow the filesystem to choose the most suitable dnode * size. Currently this just sets the default dnode size to 1k, but future * code improvements could dynamically choose a size based on observed * workload patterns. Dnodes of varying sizes can coexist within the same * dataset and even within the same dnode block. */ typedef struct dnode_phys { uint8_t dn_type; /* dmu_object_type_t */ uint8_t dn_indblkshift; /* ln2(indirect block size) */ uint8_t dn_nlevels; /* 1=dn_blkptr->data blocks */ uint8_t dn_nblkptr; /* length of dn_blkptr */ uint8_t dn_bonustype; /* type of data in bonus buffer */ uint8_t dn_checksum; /* ZIO_CHECKSUM type */ uint8_t dn_compress; /* ZIO_COMPRESS type */ uint8_t dn_flags; /* DNODE_FLAG_* */ uint16_t dn_datablkszsec; /* data block size in 512b sectors */ uint16_t dn_bonuslen; /* length of dn_bonus */ uint8_t dn_extra_slots; /* # of subsequent slots consumed */ uint8_t dn_pad2[3]; /* accounting is protected by dn_dirty_mtx */ uint64_t dn_maxblkid; /* largest allocated block ID */ uint64_t dn_used; /* bytes (or sectors) of disk space */ /* * Both dn_pad2 and dn_pad3 are protected by the block's MAC. This * allows us to protect any fields that might be added here in the * future. In either case, developers will want to check - * zio_crypt_init_uios_dnode() to ensure the new field is being - * protected properly. + * zio_crypt_init_uios_dnode() and zio_crypt_do_dnode_hmac_updates() + * to ensure the new field is being protected and updated properly. */ uint64_t dn_pad3[4]; /* * The tail region is 448 bytes for a 512 byte dnode, and * correspondingly larger for larger dnode sizes. The spill * block pointer, when present, is always at the end of the tail * region. There are three ways this space may be used, using * a 512 byte dnode for this diagram: * * 0 64 128 192 256 320 384 448 (offset) * +---------------+---------------+---------------+-------+ * | dn_blkptr[0] | dn_blkptr[1] | dn_blkptr[2] | / | * +---------------+---------------+---------------+-------+ * | dn_blkptr[0] | dn_bonus[0..319] | * +---------------+-----------------------+---------------+ * | dn_blkptr[0] | dn_bonus[0..191] | dn_spill | * +---------------+-----------------------+---------------+ */ union { blkptr_t dn_blkptr[1+DN_OLD_MAX_BONUSLEN/sizeof (blkptr_t)]; struct { blkptr_t __dn_ignore1; uint8_t dn_bonus[DN_OLD_MAX_BONUSLEN]; }; struct { blkptr_t __dn_ignore2; uint8_t __dn_ignore3[DN_OLD_MAX_BONUSLEN - sizeof (blkptr_t)]; blkptr_t dn_spill; }; }; } dnode_phys_t; #define DN_SPILL_BLKPTR(dnp) ((blkptr_t *)((char *)(dnp) + \ (((dnp)->dn_extra_slots + 1) << DNODE_SHIFT) - (1 << SPA_BLKPTRSHIFT))) struct dnode { /* * Protects the structure of the dnode, including the number of levels * of indirection (dn_nlevels), dn_maxblkid, and dn_next_* */ krwlock_t dn_struct_rwlock; /* Our link on dn_objset->os_dnodes list; protected by os_lock. */ list_node_t dn_link; /* immutable: */ struct objset *dn_objset; uint64_t dn_object; struct dmu_buf_impl *dn_dbuf; struct dnode_handle *dn_handle; dnode_phys_t *dn_phys; /* pointer into dn->dn_dbuf->db.db_data */ /* * Copies of stuff in dn_phys. They're valid in the open * context (eg. even before the dnode is first synced). * Where necessary, these are protected by dn_struct_rwlock. */ dmu_object_type_t dn_type; /* object type */ uint16_t dn_bonuslen; /* bonus length */ uint8_t dn_bonustype; /* bonus type */ uint8_t dn_nblkptr; /* number of blkptrs (immutable) */ uint8_t dn_checksum; /* ZIO_CHECKSUM type */ uint8_t dn_compress; /* ZIO_COMPRESS type */ uint8_t dn_nlevels; uint8_t dn_indblkshift; uint8_t dn_datablkshift; /* zero if blksz not power of 2! */ uint8_t dn_moved; /* Has this dnode been moved? */ uint16_t dn_datablkszsec; /* in 512b sectors */ uint32_t dn_datablksz; /* in bytes */ uint64_t dn_maxblkid; uint8_t dn_next_type[TXG_SIZE]; uint8_t dn_num_slots; /* metadnode slots consumed on disk */ uint8_t dn_next_nblkptr[TXG_SIZE]; uint8_t dn_next_nlevels[TXG_SIZE]; uint8_t dn_next_indblkshift[TXG_SIZE]; uint8_t dn_next_bonustype[TXG_SIZE]; uint8_t dn_rm_spillblk[TXG_SIZE]; /* for removing spill blk */ uint16_t dn_next_bonuslen[TXG_SIZE]; uint32_t dn_next_blksz[TXG_SIZE]; /* next block size in bytes */ uint64_t dn_next_maxblkid[TXG_SIZE]; /* next maxblkid in bytes */ /* protected by dn_dbufs_mtx; declared here to fill 32-bit hole */ uint32_t dn_dbufs_count; /* count of dn_dbufs */ /* protected by os_lock: */ multilist_node_t dn_dirty_link[TXG_SIZE]; /* next on dataset's dirty */ /* protected by dn_mtx: */ kmutex_t dn_mtx; list_t dn_dirty_records[TXG_SIZE]; struct range_tree *dn_free_ranges[TXG_SIZE]; uint64_t dn_allocated_txg; uint64_t dn_free_txg; uint64_t dn_assigned_txg; uint64_t dn_dirty_txg; /* txg dnode was last dirtied */ kcondvar_t dn_notxholds; kcondvar_t dn_nodnholds; enum dnode_dirtycontext dn_dirtyctx; void *dn_dirtyctx_firstset; /* dbg: contents meaningless */ /* protected by own devices */ zfs_refcount_t dn_tx_holds; zfs_refcount_t dn_holds; kmutex_t dn_dbufs_mtx; /* * Descendent dbufs, ordered by dbuf_compare. Note that dn_dbufs * can contain multiple dbufs of the same (level, blkid) when a * dbuf is marked DB_EVICTING without being removed from * dn_dbufs. To maintain the avl invariant that there cannot be * duplicate entries, we order the dbufs by an arbitrary value - * their address in memory. This means that dn_dbufs cannot be used to * directly look up a dbuf. Instead, callers must use avl_walk, have * a reference to the dbuf, or look up a non-existent node with * db_state = DB_SEARCH (see dbuf_free_range for an example). */ avl_tree_t dn_dbufs; /* protected by dn_struct_rwlock */ struct dmu_buf_impl *dn_bonus; /* bonus buffer dbuf */ boolean_t dn_have_spill; /* have spill or are spilling */ /* parent IO for current sync write */ zio_t *dn_zio; /* used in syncing context */ uint64_t dn_oldused; /* old phys used bytes */ uint64_t dn_oldflags; /* old phys dn_flags */ uint64_t dn_olduid, dn_oldgid, dn_oldprojid; uint64_t dn_newuid, dn_newgid, dn_newprojid; int dn_id_flags; /* holds prefetch structure */ struct zfetch dn_zfetch; }; /* * Since AVL already has embedded element counter, use dn_dbufs_count * only for dbufs not counted there (bonus buffers) and just add them. */ #define DN_DBUFS_COUNT(dn) ((dn)->dn_dbufs_count + \ avl_numnodes(&(dn)->dn_dbufs)) /* * We use this (otherwise unused) bit to indicate if the value of * dn_next_maxblkid[txgoff] is valid to use in dnode_sync(). */ #define DMU_NEXT_MAXBLKID_SET (1ULL << 63) /* * Adds a level of indirection between the dbuf and the dnode to avoid * iterating descendent dbufs in dnode_move(). Handles are not allocated * individually, but as an array of child dnodes in dnode_hold_impl(). */ typedef struct dnode_handle { /* Protects dnh_dnode from modification by dnode_move(). */ zrlock_t dnh_zrlock; dnode_t *dnh_dnode; } dnode_handle_t; typedef struct dnode_children { dmu_buf_user_t dnc_dbu; /* User evict data */ size_t dnc_count; /* number of children */ dnode_handle_t dnc_children[]; /* sized dynamically */ } dnode_children_t; typedef struct free_range { avl_node_t fr_node; uint64_t fr_blkid; uint64_t fr_nblks; } free_range_t; void dnode_special_open(struct objset *dd, dnode_phys_t *dnp, uint64_t object, dnode_handle_t *dnh); void dnode_special_close(dnode_handle_t *dnh); void dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx); void dnode_setbonus_type(dnode_t *dn, dmu_object_type_t, dmu_tx_t *tx); void dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx); int dnode_hold(struct objset *dd, uint64_t object, void *ref, dnode_t **dnp); int dnode_hold_impl(struct objset *dd, uint64_t object, int flag, int dn_slots, void *ref, dnode_t **dnp); boolean_t dnode_add_ref(dnode_t *dn, void *ref); void dnode_rele(dnode_t *dn, void *ref); void dnode_rele_and_unlock(dnode_t *dn, void *tag, boolean_t evicting); int dnode_try_claim(objset_t *os, uint64_t object, int slots); void dnode_setdirty(dnode_t *dn, dmu_tx_t *tx); void dnode_set_dirtyctx(dnode_t *dn, dmu_tx_t *tx, void *tag); void dnode_sync(dnode_t *dn, dmu_tx_t *tx); void dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx); void dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, dmu_object_type_t bonustype, int bonuslen, int dn_slots, boolean_t keep_spill, dmu_tx_t *tx); void dnode_free(dnode_t *dn, dmu_tx_t *tx); void dnode_byteswap(dnode_phys_t *dnp); void dnode_buf_byteswap(void *buf, size_t size); void dnode_verify(dnode_t *dn); int dnode_set_nlevels(dnode_t *dn, int nlevels, dmu_tx_t *tx); int dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx); void dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx); void dnode_diduse_space(dnode_t *dn, int64_t space); void dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx, boolean_t have_read, boolean_t force); uint64_t dnode_block_freed(dnode_t *dn, uint64_t blkid); void dnode_init(void); void dnode_fini(void); int dnode_next_offset(dnode_t *dn, int flags, uint64_t *off, int minlvl, uint64_t blkfill, uint64_t txg); void dnode_evict_dbufs(dnode_t *dn); void dnode_evict_bonus(dnode_t *dn); void dnode_free_interior_slots(dnode_t *dn); #define DNODE_IS_DIRTY(_dn) \ ((_dn)->dn_dirty_txg >= spa_syncing_txg((_dn)->dn_objset->os_spa)) #define DNODE_IS_CACHEABLE(_dn) \ ((_dn)->dn_objset->os_primary_cache == ZFS_CACHE_ALL || \ (DMU_OT_IS_METADATA((_dn)->dn_type) && \ (_dn)->dn_objset->os_primary_cache == ZFS_CACHE_METADATA)) #define DNODE_META_IS_CACHEABLE(_dn) \ ((_dn)->dn_objset->os_primary_cache == ZFS_CACHE_ALL || \ (_dn)->dn_objset->os_primary_cache == ZFS_CACHE_METADATA) /* * Used for dnodestats kstat. */ typedef struct dnode_stats { /* * Number of failed attempts to hold a meta dnode dbuf. */ kstat_named_t dnode_hold_dbuf_hold; /* * Number of failed attempts to read a meta dnode dbuf. */ kstat_named_t dnode_hold_dbuf_read; /* * Number of times dnode_hold(..., DNODE_MUST_BE_ALLOCATED) was able * to hold the requested object number which was allocated. This is * the common case when looking up any allocated object number. */ kstat_named_t dnode_hold_alloc_hits; /* * Number of times dnode_hold(..., DNODE_MUST_BE_ALLOCATED) was not * able to hold the request object number because it was not allocated. */ kstat_named_t dnode_hold_alloc_misses; /* * Number of times dnode_hold(..., DNODE_MUST_BE_ALLOCATED) was not * able to hold the request object number because the object number * refers to an interior large dnode slot. */ kstat_named_t dnode_hold_alloc_interior; /* * Number of times dnode_hold(..., DNODE_MUST_BE_ALLOCATED) needed * to retry acquiring slot zrl locks due to contention. */ kstat_named_t dnode_hold_alloc_lock_retry; /* * Number of times dnode_hold(..., DNODE_MUST_BE_ALLOCATED) did not * need to create the dnode because another thread did so after * dropping the read lock but before acquiring the write lock. */ kstat_named_t dnode_hold_alloc_lock_misses; /* * Number of times dnode_hold(..., DNODE_MUST_BE_ALLOCATED) found * a free dnode instantiated by dnode_create() but not yet allocated * by dnode_allocate(). */ kstat_named_t dnode_hold_alloc_type_none; /* * Number of times dnode_hold(..., DNODE_MUST_BE_FREE) was able * to hold the requested range of free dnode slots. */ kstat_named_t dnode_hold_free_hits; /* * Number of times dnode_hold(..., DNODE_MUST_BE_FREE) was not * able to hold the requested range of free dnode slots because * at least one slot was allocated. */ kstat_named_t dnode_hold_free_misses; /* * Number of times dnode_hold(..., DNODE_MUST_BE_FREE) was not * able to hold the requested range of free dnode slots because * after acquiring the zrl lock at least one slot was allocated. */ kstat_named_t dnode_hold_free_lock_misses; /* * Number of times dnode_hold(..., DNODE_MUST_BE_FREE) needed * to retry acquiring slot zrl locks due to contention. */ kstat_named_t dnode_hold_free_lock_retry; /* * Number of times dnode_hold(..., DNODE_MUST_BE_FREE) requested * a range of dnode slots which were held by another thread. */ kstat_named_t dnode_hold_free_refcount; /* * Number of times dnode_hold(..., DNODE_MUST_BE_FREE) requested * a range of dnode slots which would overflow the dnode_phys_t. */ kstat_named_t dnode_hold_free_overflow; /* * Number of times dnode_free_interior_slots() needed to retry * acquiring a slot zrl lock due to contention. */ kstat_named_t dnode_free_interior_lock_retry; /* * Number of new dnodes allocated by dnode_allocate(). */ kstat_named_t dnode_allocate; /* * Number of dnodes re-allocated by dnode_reallocate(). */ kstat_named_t dnode_reallocate; /* * Number of meta dnode dbufs evicted. */ kstat_named_t dnode_buf_evict; /* * Number of times dmu_object_alloc*() reached the end of the existing * object ID chunk and advanced to a new one. */ kstat_named_t dnode_alloc_next_chunk; /* * Number of times multiple threads attempted to allocate a dnode * from the same block of free dnodes. */ kstat_named_t dnode_alloc_race; /* * Number of times dmu_object_alloc*() was forced to advance to the * next meta dnode dbuf due to an error from dmu_object_next(). */ kstat_named_t dnode_alloc_next_block; /* * Statistics for tracking dnodes which have been moved. */ kstat_named_t dnode_move_invalid; kstat_named_t dnode_move_recheck1; kstat_named_t dnode_move_recheck2; kstat_named_t dnode_move_special; kstat_named_t dnode_move_handle; kstat_named_t dnode_move_rwlock; kstat_named_t dnode_move_active; } dnode_stats_t; extern dnode_stats_t dnode_stats; #define DNODE_STAT_INCR(stat, val) \ atomic_add_64(&dnode_stats.stat.value.ui64, (val)); #define DNODE_STAT_BUMP(stat) \ DNODE_STAT_INCR(stat, 1); #ifdef ZFS_DEBUG #define dprintf_dnode(dn, fmt, ...) do { \ if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ char __db_buf[32]; \ uint64_t __db_obj = (dn)->dn_object; \ if (__db_obj == DMU_META_DNODE_OBJECT) \ (void) strcpy(__db_buf, "mdn"); \ else \ (void) snprintf(__db_buf, sizeof (__db_buf), "%lld", \ (u_longlong_t)__db_obj);\ dprintf_ds((dn)->dn_objset->os_dsl_dataset, "obj=%s " fmt, \ __db_buf, __VA_ARGS__); \ } \ _NOTE(CONSTCOND) } while (0) #define DNODE_VERIFY(dn) dnode_verify(dn) #define FREE_VERIFY(db, start, end, tx) free_verify(db, start, end, tx) #else #define dprintf_dnode(db, fmt, ...) #define DNODE_VERIFY(dn) #define FREE_VERIFY(db, start, end, tx) #endif #ifdef __cplusplus } #endif #endif /* _SYS_DNODE_H */ diff --git a/module/os/linux/zfs/zio_crypt.c b/module/os/linux/zfs/zio_crypt.c index 94406999cb89..52e62f4d1da4 100644 --- a/module/os/linux/zfs/zio_crypt.c +++ b/module/os/linux/zfs/zio_crypt.c @@ -1,2037 +1,2043 @@ /* * CDDL HEADER START * * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * CDDL HEADER END */ /* * Copyright (c) 2017, Datto, Inc. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include /* * This file is responsible for handling all of the details of generating * encryption parameters and performing encryption and authentication. * * BLOCK ENCRYPTION PARAMETERS: * Encryption /Authentication Algorithm Suite (crypt): * The encryption algorithm, mode, and key length we are going to use. We * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit * keys. All authentication is currently done with SHA512-HMAC. * * Plaintext: * The unencrypted data that we want to encrypt. * * Initialization Vector (IV): * An initialization vector for the encryption algorithms. This is used to * "tweak" the encryption algorithms so that two blocks of the same data are * encrypted into different ciphertext outputs, thus obfuscating block patterns. * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is * never reused with the same encryption key. This value is stored unencrypted * and must simply be provided to the decryption function. We use a 96 bit IV * (as recommended by NIST) for all block encryption. For non-dedup blocks we * derive the IV randomly. The first 64 bits of the IV are stored in the second * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of * level 0 blocks is the number of allocated dnodes in that block. The on-disk * format supports at most 2^15 slots per L0 dnode block, because the maximum * block size is 16MB (2^24). In either case, for level 0 blocks this number * will still be smaller than UINT32_MAX so it is safe to store the IV in the * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count * for the dnode code. * * Master key: * This is the most important secret data of an encrypted dataset. It is used * along with the salt to generate that actual encryption keys via HKDF. We * do not use the master key to directly encrypt any data because there are * theoretical limits on how much data can actually be safely encrypted with * any encryption mode. The master key is stored encrypted on disk with the * user's wrapping key. Its length is determined by the encryption algorithm. * For details on how this is stored see the block comment in dsl_crypt.c * * Salt: * Used as an input to the HKDF function, along with the master key. We use a * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt * can be used for encrypting many blocks, so we cache the current salt and the * associated derived key in zio_crypt_t so we do not need to derive it again * needlessly. * * Encryption Key: * A secret binary key, generated from an HKDF function used to encrypt and * decrypt data. * * Message Authentication Code (MAC) * The MAC is an output of authenticated encryption modes such as AES-GCM and * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted * data on disk and return garbage to the application. Effectively, it is a * checksum that can not be reproduced by an attacker. We store the MAC in the * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated * regular checksum of the ciphertext which can be used for scrubbing. * * OBJECT AUTHENTICATION: * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because * they contain some info that always needs to be readable. To prevent this * data from being altered, we authenticate this data using SHA512-HMAC. This * will produce a MAC (similar to the one produced via encryption) which can * be used to verify the object was not modified. HMACs do not require key * rotation or IVs, so we can keep up to the full 3 copies of authenticated * data. * * ZIL ENCRYPTION: * ZIL blocks have their bp written to disk ahead of the associated data, so we * cannot store the MAC there as we normally do. For these blocks the MAC is * stored in the embedded checksum within the zil_chain_t header. The salt and * IV are generated for the block on bp allocation instead of at encryption * time. In addition, ZIL blocks have some pieces that must be left in plaintext * for claiming even though all of the sensitive user data still needs to be * encrypted. The function zio_crypt_init_uios_zil() handles parsing which * pieces of the block need to be encrypted. All data that is not encrypted is * authenticated using the AAD mechanisms that the supported encryption modes * provide for. In order to preserve the semantics of the ZIL for encrypted * datasets, the ZIL is not protected at the objset level as described below. * * DNODE ENCRYPTION: * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left * in plaintext for scrubbing and claiming, but the bonus buffers might contain * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing * which pieces of the block need to be encrypted. For more details about * dnode authentication and encryption, see zio_crypt_init_uios_dnode(). * * OBJECT SET AUTHENTICATION: * Up to this point, everything we have encrypted and authenticated has been * at level 0 (or -2 for the ZIL). If we did not do any further work the * on-disk format would be susceptible to attacks that deleted or rearranged * the order of level 0 blocks. Ideally, the cleanest solution would be to * maintain a tree of authentication MACs going up the bp tree. However, this * presents a problem for raw sends. Send files do not send information about * indirect blocks so there would be no convenient way to transfer the MACs and * they cannot be recalculated on the receive side without the master key which * would defeat one of the purposes of raw sends in the first place. Instead, * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs * from the level below. We also include some portable fields from blk_prop such * as the lsize and compression algorithm to prevent the data from being * misinterpreted. * * At the objset level, we maintain 2 separate 256 bit MACs in the * objset_phys_t. The first one is "portable" and is the logical root of the * MAC tree maintained in the metadnode's bps. The second, is "local" and is * used as the root MAC for the user accounting objects, which are also not * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload * of the send file. The useraccounting code ensures that the useraccounting * info is not present upon a receive, so the local MAC can simply be cleared * out at that time. For more info about objset_phys_t authentication, see * zio_crypt_do_objset_hmacs(). * * CONSIDERATIONS FOR DEDUP: * In order for dedup to work, blocks that we want to dedup with one another * need to use the same IV and encryption key, so that they will have the same * ciphertext. Normally, one should never reuse an IV with the same encryption * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both * blocks. In this case, however, since we are using the same plaintext as * well all that we end up with is a duplicate of the original ciphertext we * already had. As a result, an attacker with read access to the raw disk will * be able to tell which blocks are the same but this information is given away * by dedup anyway. In order to get the same IVs and encryption keys for * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC * here so that a reproducible checksum of the plaintext is never available to * the attacker. The HMAC key is kept alongside the master key, encrypted on * disk. The first 64 bits of the HMAC are used in place of the random salt, and * the next 96 bits are used as the IV. As a result of this mechanism, dedup * will only work within a clone family since encrypted dedup requires use of * the same master and HMAC keys. */ /* * After encrypting many blocks with the same key we may start to run up * against the theoretical limits of how much data can securely be encrypted * with a single key using the supported encryption modes. The most obvious * limitation is that our risk of generating 2 equivalent 96 bit IVs increases * the more IVs we generate (which both GCM and CCM modes strictly forbid). * This risk actually grows surprisingly quickly over time according to the * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have * generated n IVs with a cryptographically secure RNG, the approximate * probability p(n) of a collision is given as: * * p(n) ~= e^(-n*(n-1)/(2*(2^96))) * * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html] * * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion * we must not write more than 398,065,730 blocks with the same encryption key. * Therefore, we rotate our keys after 400,000,000 blocks have been written by * generating a new random 64 bit salt for our HKDF encryption key generation * function. */ #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000 #define ZFS_CURRENT_MAX_SALT_USES \ (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT)) unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; typedef struct blkptr_auth_buf { uint64_t bab_prop; /* blk_prop - portable mask */ - uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */ + uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */ uint64_t bab_pad; /* reserved for future use */ } blkptr_auth_buf_t; zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = { {"", ZC_TYPE_NONE, 0, "inherit"}, {"", ZC_TYPE_NONE, 0, "on"}, {"", ZC_TYPE_NONE, 0, "off"}, {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"}, {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"}, {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"}, {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"}, {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"}, {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"} }; void zio_crypt_key_destroy(zio_crypt_key_t *key) { rw_destroy(&key->zk_salt_lock); /* free crypto templates */ crypto_destroy_ctx_template(key->zk_current_tmpl); crypto_destroy_ctx_template(key->zk_hmac_tmpl); /* zero out sensitive data */ bzero(key, sizeof (zio_crypt_key_t)); } int zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key) { int ret; crypto_mechanism_t mech; uint_t keydata_len; ASSERT(key != NULL); ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); keydata_len = zio_crypt_table[crypt].ci_keylen; bzero(key, sizeof (zio_crypt_key_t)); /* fill keydata buffers and salt with random data */ ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t)); if (ret != 0) goto error; ret = random_get_bytes(key->zk_master_keydata, keydata_len); if (ret != 0) goto error; ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); if (ret != 0) goto error; ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); if (ret != 0) goto error; /* derive the current key from the master key */ ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); if (ret != 0) goto error; /* initialize keys for the ICP */ key->zk_current_key.ck_format = CRYPTO_KEY_RAW; key->zk_current_key.ck_data = key->zk_current_keydata; key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; key->zk_hmac_key.ck_data = &key->zk_hmac_key; key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); /* * Initialize the crypto templates. It's ok if this fails because * this is just an optimization. */ mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); ret = crypto_create_ctx_template(&mech, &key->zk_current_key, &key->zk_current_tmpl, KM_SLEEP); if (ret != CRYPTO_SUCCESS) key->zk_current_tmpl = NULL; mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, &key->zk_hmac_tmpl, KM_SLEEP); if (ret != CRYPTO_SUCCESS) key->zk_hmac_tmpl = NULL; key->zk_crypt = crypt; key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION; key->zk_salt_count = 0; rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); return (0); error: zio_crypt_key_destroy(key); return (ret); } static int zio_crypt_key_change_salt(zio_crypt_key_t *key) { int ret = 0; uint8_t salt[ZIO_DATA_SALT_LEN]; crypto_mechanism_t mech; uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen; /* generate a new salt */ ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN); if (ret != 0) goto error; rw_enter(&key->zk_salt_lock, RW_WRITER); /* someone beat us to the salt rotation, just unlock and return */ if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES) goto out_unlock; /* derive the current key from the master key and the new salt */ ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); if (ret != 0) goto out_unlock; /* assign the salt and reset the usage count */ bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN); key->zk_salt_count = 0; /* destroy the old context template and create the new one */ crypto_destroy_ctx_template(key->zk_current_tmpl); ret = crypto_create_ctx_template(&mech, &key->zk_current_key, &key->zk_current_tmpl, KM_SLEEP); if (ret != CRYPTO_SUCCESS) key->zk_current_tmpl = NULL; rw_exit(&key->zk_salt_lock); return (0); out_unlock: rw_exit(&key->zk_salt_lock); error: return (ret); } /* See comment above zfs_key_max_salt_uses definition for details */ int zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt) { int ret; boolean_t salt_change; rw_enter(&key->zk_salt_lock, RW_READER); bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN); salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >= ZFS_CURRENT_MAX_SALT_USES); rw_exit(&key->zk_salt_lock); if (salt_change) { ret = zio_crypt_key_change_salt(key); if (ret != 0) goto error; } return (0); error: return (ret); } /* * This function handles all encryption and decryption in zfs. When * encrypting it expects puio to reference the plaintext and cuio to * reference the ciphertext. cuio must have enough space for the * ciphertext + room for a MAC. datalen should be the length of the * plaintext / ciphertext alone. */ static int zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key, crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen, zfs_uio_t *puio, zfs_uio_t *cuio, uint8_t *authbuf, uint_t auth_len) { int ret; crypto_data_t plaindata, cipherdata; CK_AES_CCM_PARAMS ccmp; CK_AES_GCM_PARAMS gcmp; crypto_mechanism_t mech; zio_crypt_info_t crypt_info; uint_t plain_full_len, maclen; ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW); /* lookup the encryption info */ crypt_info = zio_crypt_table[crypt]; /* the mac will always be the last iovec_t in the cipher uio */ maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len; ASSERT(maclen <= ZIO_DATA_MAC_LEN); /* setup encryption mechanism (same as crypt) */ mech.cm_type = crypto_mech2id(crypt_info.ci_mechname); /* * Strangely, the ICP requires that plain_full_len must include * the MAC length when decrypting, even though the UIO does not * need to have the extra space allocated. */ if (encrypt) { plain_full_len = datalen; } else { plain_full_len = datalen + maclen; } /* * setup encryption params (currently only AES CCM and AES GCM * are supported) */ if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) { ccmp.ulNonceSize = ZIO_DATA_IV_LEN; ccmp.ulAuthDataSize = auth_len; ccmp.authData = authbuf; ccmp.ulMACSize = maclen; ccmp.nonce = ivbuf; ccmp.ulDataSize = plain_full_len; mech.cm_param = (char *)(&ccmp); mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS); } else { gcmp.ulIvLen = ZIO_DATA_IV_LEN; gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN); gcmp.ulAADLen = auth_len; gcmp.pAAD = authbuf; gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen); gcmp.pIv = ivbuf; mech.cm_param = (char *)(&gcmp); mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); } /* populate the cipher and plain data structs. */ plaindata.cd_format = CRYPTO_DATA_UIO; plaindata.cd_offset = 0; plaindata.cd_uio = puio; plaindata.cd_miscdata = NULL; plaindata.cd_length = plain_full_len; cipherdata.cd_format = CRYPTO_DATA_UIO; cipherdata.cd_offset = 0; cipherdata.cd_uio = cuio; cipherdata.cd_miscdata = NULL; cipherdata.cd_length = datalen + maclen; /* perform the actual encryption */ if (encrypt) { ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } } else { ret = crypto_decrypt(&mech, &cipherdata, key, tmpl, &plaindata, NULL); if (ret != CRYPTO_SUCCESS) { ASSERT3U(ret, ==, CRYPTO_INVALID_MAC); ret = SET_ERROR(ECKSUM); goto error; } } return (0); error: return (ret); } int zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) { int ret; zfs_uio_t puio, cuio; uint64_t aad[3]; iovec_t plain_iovecs[2], cipher_iovecs[3]; uint64_t crypt = key->zk_crypt; uint_t enc_len, keydata_len, aad_len; ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); keydata_len = zio_crypt_table[crypt].ci_keylen; /* generate iv for wrapping the master and hmac key */ ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN); if (ret != 0) goto error; /* initialize zfs_uio_ts */ plain_iovecs[0].iov_base = key->zk_master_keydata; plain_iovecs[0].iov_len = keydata_len; plain_iovecs[1].iov_base = key->zk_hmac_keydata; plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; cipher_iovecs[0].iov_base = keydata_out; cipher_iovecs[0].iov_len = keydata_len; cipher_iovecs[1].iov_base = hmac_keydata_out; cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; cipher_iovecs[2].iov_base = mac; cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; /* * Although we don't support writing to the old format, we do * support rewrapping the key so that the user can move and * quarantine datasets on the old format. */ if (key->zk_version == 0) { aad_len = sizeof (uint64_t); aad[0] = LE_64(key->zk_guid); } else { ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); aad_len = sizeof (uint64_t) * 3; aad[0] = LE_64(key->zk_guid); aad[1] = LE_64(crypt); aad[2] = LE_64(key->zk_version); } enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN; puio.uio_iov = plain_iovecs; puio.uio_iovcnt = 2; puio.uio_segflg = UIO_SYSSPACE; cuio.uio_iov = cipher_iovecs; cuio.uio_iovcnt = 3; cuio.uio_segflg = UIO_SYSSPACE; /* encrypt the keys and store the resulting ciphertext and mac */ ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len, &puio, &cuio, (uint8_t *)aad, aad_len); if (ret != 0) goto error; return (0); error: return (ret); } int zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv, uint8_t *mac, zio_crypt_key_t *key) { crypto_mechanism_t mech; zfs_uio_t puio, cuio; uint64_t aad[3]; iovec_t plain_iovecs[2], cipher_iovecs[3]; uint_t enc_len, keydata_len, aad_len; int ret; ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); keydata_len = zio_crypt_table[crypt].ci_keylen; /* initialize zfs_uio_ts */ plain_iovecs[0].iov_base = key->zk_master_keydata; plain_iovecs[0].iov_len = keydata_len; plain_iovecs[1].iov_base = key->zk_hmac_keydata; plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; cipher_iovecs[0].iov_base = keydata; cipher_iovecs[0].iov_len = keydata_len; cipher_iovecs[1].iov_base = hmac_keydata; cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; cipher_iovecs[2].iov_base = mac; cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; if (version == 0) { aad_len = sizeof (uint64_t); aad[0] = LE_64(guid); } else { ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); aad_len = sizeof (uint64_t) * 3; aad[0] = LE_64(guid); aad[1] = LE_64(crypt); aad[2] = LE_64(version); } enc_len = keydata_len + SHA512_HMAC_KEYLEN; puio.uio_iov = plain_iovecs; puio.uio_segflg = UIO_SYSSPACE; puio.uio_iovcnt = 2; cuio.uio_iov = cipher_iovecs; cuio.uio_iovcnt = 3; cuio.uio_segflg = UIO_SYSSPACE; /* decrypt the keys and store the result in the output buffers */ ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len, &puio, &cuio, (uint8_t *)aad, aad_len); if (ret != 0) goto error; /* generate a fresh salt */ ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); if (ret != 0) goto error; /* derive the current key from the master key */ ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); if (ret != 0) goto error; /* initialize keys for ICP */ key->zk_current_key.ck_format = CRYPTO_KEY_RAW; key->zk_current_key.ck_data = key->zk_current_keydata; key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; key->zk_hmac_key.ck_data = key->zk_hmac_keydata; key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); /* * Initialize the crypto templates. It's ok if this fails because * this is just an optimization. */ mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); ret = crypto_create_ctx_template(&mech, &key->zk_current_key, &key->zk_current_tmpl, KM_SLEEP); if (ret != CRYPTO_SUCCESS) key->zk_current_tmpl = NULL; mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, &key->zk_hmac_tmpl, KM_SLEEP); if (ret != CRYPTO_SUCCESS) key->zk_hmac_tmpl = NULL; key->zk_crypt = crypt; key->zk_version = version; key->zk_guid = guid; key->zk_salt_count = 0; return (0); error: zio_crypt_key_destroy(key); return (ret); } int zio_crypt_generate_iv(uint8_t *ivbuf) { int ret; /* randomly generate the IV */ ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN); if (ret != 0) goto error; return (0); error: bzero(ivbuf, ZIO_DATA_IV_LEN); return (ret); } int zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, uint8_t *digestbuf, uint_t digestlen) { int ret; crypto_mechanism_t mech; crypto_data_t in_data, digest_data; uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH]; ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH); /* initialize sha512-hmac mechanism and crypto data */ mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); mech.cm_param = NULL; mech.cm_param_len = 0; /* initialize the crypto data */ in_data.cd_format = CRYPTO_DATA_RAW; in_data.cd_offset = 0; in_data.cd_length = datalen; in_data.cd_raw.iov_base = (char *)data; in_data.cd_raw.iov_len = in_data.cd_length; digest_data.cd_format = CRYPTO_DATA_RAW; digest_data.cd_offset = 0; digest_data.cd_length = SHA512_DIGEST_LENGTH; digest_data.cd_raw.iov_base = (char *)raw_digestbuf; digest_data.cd_raw.iov_len = digest_data.cd_length; /* generate the hmac */ ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl, &digest_data, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } bcopy(raw_digestbuf, digestbuf, digestlen); return (0); error: bzero(digestbuf, digestlen); return (ret); } int zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, uint8_t *ivbuf, uint8_t *salt) { int ret; uint8_t digestbuf[SHA512_DIGEST_LENGTH]; ret = zio_crypt_do_hmac(key, data, datalen, digestbuf, SHA512_DIGEST_LENGTH); if (ret != 0) return (ret); bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN); bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN); return (0); } /* * The following functions are used to encode and decode encryption parameters * into blkptr_t and zil_header_t. The ICP wants to use these parameters as * byte strings, which normally means that these strings would not need to deal * with byteswapping at all. However, both blkptr_t and zil_header_t may be * byteswapped by lower layers and so we must "undo" that byteswap here upon * decoding and encoding in a non-native byteorder. These functions require * that the byteorder bit is correct before being called. */ void zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv) { uint64_t val64; uint32_t val32; ASSERT(BP_IS_ENCRYPTED(bp)); if (!BP_SHOULD_BYTESWAP(bp)) { bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t)); bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t)); bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); BP_SET_IV2(bp, val32); } else { bcopy(salt, &val64, sizeof (uint64_t)); bp->blk_dva[2].dva_word[0] = BSWAP_64(val64); bcopy(iv, &val64, sizeof (uint64_t)); bp->blk_dva[2].dva_word[1] = BSWAP_64(val64); bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); BP_SET_IV2(bp, BSWAP_32(val32)); } } void zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv) { uint64_t val64; uint32_t val32; ASSERT(BP_IS_PROTECTED(bp)); /* for convenience, so callers don't need to check */ if (BP_IS_AUTHENTICATED(bp)) { bzero(salt, ZIO_DATA_SALT_LEN); bzero(iv, ZIO_DATA_IV_LEN); return; } if (!BP_SHOULD_BYTESWAP(bp)) { bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t)); bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t)); val32 = (uint32_t)BP_GET_IV2(bp); bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); } else { val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]); bcopy(&val64, salt, sizeof (uint64_t)); val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]); bcopy(&val64, iv, sizeof (uint64_t)); val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp)); bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); } } void zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac) { uint64_t val64; ASSERT(BP_USES_CRYPT(bp)); ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET); if (!BP_SHOULD_BYTESWAP(bp)) { bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t)); bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3], sizeof (uint64_t)); } else { bcopy(mac, &val64, sizeof (uint64_t)); bp->blk_cksum.zc_word[2] = BSWAP_64(val64); bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t)); bp->blk_cksum.zc_word[3] = BSWAP_64(val64); } } void zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac) { uint64_t val64; ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp)); /* for convenience, so callers don't need to check */ if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { bzero(mac, ZIO_DATA_MAC_LEN); return; } if (!BP_SHOULD_BYTESWAP(bp)) { bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t)); bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t), sizeof (uint64_t)); } else { val64 = BSWAP_64(bp->blk_cksum.zc_word[2]); bcopy(&val64, mac, sizeof (uint64_t)); val64 = BSWAP_64(bp->blk_cksum.zc_word[3]); bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t)); } } void zio_crypt_encode_mac_zil(void *data, uint8_t *mac) { zil_chain_t *zilc = data; bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t)); bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3], sizeof (uint64_t)); } void zio_crypt_decode_mac_zil(const void *data, uint8_t *mac) { /* * The ZIL MAC is embedded in the block it protects, which will * not have been byteswapped by the time this function has been called. * As a result, we don't need to worry about byteswapping the MAC. */ const zil_chain_t *zilc = data; bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t)); bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t), sizeof (uint64_t)); } /* * This routine takes a block of dnodes (src_abd) and copies only the bonus * buffers to the same offsets in the dst buffer. datalen should be the size * of both the src_abd and the dst buffer (not just the length of the bonus * buffers). */ void zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen) { uint_t i, max_dnp = datalen >> DNODE_SHIFT; uint8_t *src; dnode_phys_t *dnp, *sdnp, *ddnp; src = abd_borrow_buf_copy(src_abd, datalen); sdnp = (dnode_phys_t *)src; ddnp = (dnode_phys_t *)dst; for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { dnp = &sdnp[i]; if (dnp->dn_type != DMU_OT_NONE && DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && dnp->dn_bonuslen != 0) { bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), DN_MAX_BONUS_LEN(dnp)); } } abd_return_buf(src_abd, src, datalen); } /* * This function decides what fields from blk_prop are included in * the on-disk various MAC algorithms. */ static void zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) { /* * Version 0 did not properly zero out all non-portable fields * as it should have done. We maintain this code so that we can * do read-only imports of pools on this version. */ if (version == 0) { BP_SET_DEDUP(bp, 0); BP_SET_CHECKSUM(bp, 0); BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); return; } ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); /* * The hole_birth feature might set these fields even if this bp * is a hole. We zero them out here to guarantee that raw sends * will function with or without the feature. */ if (BP_IS_HOLE(bp)) { bp->blk_prop = 0ULL; return; } /* * At L0 we want to verify these fields to ensure that data blocks * can not be reinterpreted. For instance, we do not want an attacker * to trick us into returning raw lz4 compressed data to the user * by modifying the compression bits. At higher levels, we cannot * enforce this policy since raw sends do not convey any information * about indirect blocks, so these values might be different on the * receive side. Fortunately, this does not open any new attack * vectors, since any alterations that can be made to a higher level * bp must still verify the correct order of the layer below it. */ if (BP_GET_LEVEL(bp) != 0) { BP_SET_BYTEORDER(bp, 0); BP_SET_COMPRESS(bp, 0); /* * psize cannot be set to zero or it will trigger * asserts, but the value doesn't really matter as * long as it is constant. */ BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); } BP_SET_DEDUP(bp, 0); BP_SET_CHECKSUM(bp, 0); } static void zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp, blkptr_auth_buf_t *bab, uint_t *bab_len) { blkptr_t tmpbp = *bp; if (should_bswap) byteswap_uint64_array(&tmpbp, sizeof (blkptr_t)); ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp)); ASSERT0(BP_IS_EMBEDDED(&tmpbp)); zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac); /* * We always MAC blk_prop in LE to ensure portability. This * must be done after decoding the mac, since the endianness * will get zero'd out here. */ zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version); bab->bab_prop = LE_64(tmpbp.blk_prop); bab->bab_pad = 0ULL; /* version 0 did not include the padding */ *bab_len = sizeof (blkptr_auth_buf_t); if (version == 0) *bab_len -= sizeof (uint64_t); } static int zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version, boolean_t should_bswap, blkptr_t *bp) { int ret; uint_t bab_len; blkptr_auth_buf_t bab; crypto_data_t cd; zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); cd.cd_format = CRYPTO_DATA_RAW; cd.cd_offset = 0; cd.cd_length = bab_len; cd.cd_raw.iov_base = (char *)&bab; cd.cd_raw.iov_len = cd.cd_length; ret = crypto_mac_update(ctx, &cd, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } return (0); error: return (ret); } static void zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version, boolean_t should_bswap, blkptr_t *bp) { uint_t bab_len; blkptr_auth_buf_t bab; zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); SHA2Update(ctx, &bab, bab_len); } static void zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version, boolean_t should_bswap, blkptr_t *bp) { uint_t bab_len; blkptr_auth_buf_t bab; zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); bcopy(&bab, *aadp, bab_len); *aadp += bab_len; *aad_len += bab_len; } static int zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, boolean_t should_bswap, dnode_phys_t *dnp) { int ret, i; - dnode_phys_t *adnp; + dnode_phys_t *adnp, tmp_dncore; + size_t dn_core_size = offsetof(dnode_phys_t, dn_blkptr); boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); crypto_data_t cd; - uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)]; cd.cd_format = CRYPTO_DATA_RAW; cd.cd_offset = 0; - /* authenticate the core dnode (masking out non-portable bits) */ - bcopy(dnp, tmp_dncore, sizeof (tmp_dncore)); - adnp = (dnode_phys_t *)tmp_dncore; + /* + * Authenticate the core dnode (masking out non-portable bits). + * We only copy the first 64 bytes we operate on to avoid the overhead + * of copying 512-64 unneeded bytes. The compiler seems to be fine + * with that. + */ + bcopy(dnp, &tmp_dncore, dn_core_size); + adnp = &tmp_dncore; + if (le_bswap) { adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec); adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen); adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid); adnp->dn_used = BSWAP_64(adnp->dn_used); } adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; adnp->dn_used = 0; - cd.cd_length = sizeof (tmp_dncore); + cd.cd_length = dn_core_size; cd.cd_raw.iov_base = (char *)adnp; cd.cd_raw.iov_len = cd.cd_length; ret = crypto_mac_update(ctx, &cd, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } for (i = 0; i < dnp->dn_nblkptr; i++) { ret = zio_crypt_bp_do_hmac_updates(ctx, version, should_bswap, &dnp->dn_blkptr[i]); if (ret != 0) goto error; } if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { ret = zio_crypt_bp_do_hmac_updates(ctx, version, should_bswap, DN_SPILL_BLKPTR(dnp)); if (ret != 0) goto error; } return (0); error: return (ret); } /* * objset_phys_t blocks introduce a number of exceptions to the normal * authentication process. objset_phys_t's contain 2 separate HMACS for * protecting the integrity of their data. The portable_mac protects the * metadnode. This MAC can be sent with a raw send and protects against * reordering of data within the metadnode. The local_mac protects the user * accounting objects which are not sent from one system to another. * * In addition, objset blocks are the only blocks that can be modified and * written to disk without the key loaded under certain circumstances. During * zil_claim() we need to be able to update the zil_header_t to complete * claiming log blocks and during raw receives we need to write out the * portable_mac from the send file. Both of these actions are possible * because these fields are not protected by either MAC so neither one will * need to modify the MACs without the key. However, when the modified blocks * are written out they will be byteswapped into the host machine's native * endianness which will modify fields protected by the MAC. As a result, MAC * calculation for objset blocks works slightly differently from other block * types. Where other block types MAC the data in whatever endianness is * written to disk, objset blocks always MAC little endian version of their * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP() * and le_bswap indicates whether a byteswap is needed to get this block * into little endian format. */ int zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) { int ret; crypto_mechanism_t mech; crypto_context_t ctx; crypto_data_t cd; objset_phys_t *osp = data; uint64_t intval; boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH]; uint8_t raw_local_mac[SHA512_DIGEST_LENGTH]; /* initialize HMAC mechanism */ mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); mech.cm_param = NULL; mech.cm_param_len = 0; cd.cd_format = CRYPTO_DATA_RAW; cd.cd_offset = 0; /* calculate the portable MAC from the portable fields and metadnode */ ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } /* add in the os_type */ intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type); cd.cd_length = sizeof (uint64_t); cd.cd_raw.iov_base = (char *)&intval; cd.cd_raw.iov_len = cd.cd_length; ret = crypto_mac_update(ctx, &cd, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } /* add in the portable os_flags */ intval = osp->os_flags; if (should_bswap) intval = BSWAP_64(intval); intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK; if (!ZFS_HOST_BYTEORDER) intval = BSWAP_64(intval); cd.cd_length = sizeof (uint64_t); cd.cd_raw.iov_base = (char *)&intval; cd.cd_raw.iov_len = cd.cd_length; ret = crypto_mac_update(ctx, &cd, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } /* add in fields from the metadnode */ ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, should_bswap, &osp->os_meta_dnode); if (ret) goto error; /* store the final digest in a temporary buffer and copy what we need */ cd.cd_length = SHA512_DIGEST_LENGTH; cd.cd_raw.iov_base = (char *)raw_portable_mac; cd.cd_raw.iov_len = cd.cd_length; ret = crypto_mac_final(ctx, &cd, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); /* * The local MAC protects the user, group and project accounting. * If these objects are not present, the local MAC is zeroed out. */ if ((datalen >= OBJSET_PHYS_SIZE_V3 && osp->os_userused_dnode.dn_type == DMU_OT_NONE && osp->os_groupused_dnode.dn_type == DMU_OT_NONE && osp->os_projectused_dnode.dn_type == DMU_OT_NONE) || (datalen >= OBJSET_PHYS_SIZE_V2 && osp->os_userused_dnode.dn_type == DMU_OT_NONE && osp->os_groupused_dnode.dn_type == DMU_OT_NONE) || (datalen <= OBJSET_PHYS_SIZE_V1)) { bzero(local_mac, ZIO_OBJSET_MAC_LEN); return (0); } /* calculate the local MAC from the userused and groupused dnodes */ ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } /* add in the non-portable os_flags */ intval = osp->os_flags; if (should_bswap) intval = BSWAP_64(intval); intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK; if (!ZFS_HOST_BYTEORDER) intval = BSWAP_64(intval); cd.cd_length = sizeof (uint64_t); cd.cd_raw.iov_base = (char *)&intval; cd.cd_raw.iov_len = cd.cd_length; ret = crypto_mac_update(ctx, &cd, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } /* add in fields from the user accounting dnodes */ if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) { ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, should_bswap, &osp->os_userused_dnode); if (ret) goto error; } if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) { ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, should_bswap, &osp->os_groupused_dnode); if (ret) goto error; } if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE && datalen >= OBJSET_PHYS_SIZE_V3) { ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, should_bswap, &osp->os_projectused_dnode); if (ret) goto error; } /* store the final digest in a temporary buffer and copy what we need */ cd.cd_length = SHA512_DIGEST_LENGTH; cd.cd_raw.iov_base = (char *)raw_local_mac; cd.cd_raw.iov_len = cd.cd_length; ret = crypto_mac_final(ctx, &cd, NULL); if (ret != CRYPTO_SUCCESS) { ret = SET_ERROR(EIO); goto error; } bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN); return (0); error: bzero(portable_mac, ZIO_OBJSET_MAC_LEN); bzero(local_mac, ZIO_OBJSET_MAC_LEN); return (ret); } static void zio_crypt_destroy_uio(zfs_uio_t *uio) { if (uio->uio_iov) kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t)); } /* * This function parses an uncompressed indirect block and returns a checksum * of all the portable fields from all of the contained bps. The portable * fields are the MAC and all of the fields from blk_prop except for the dedup, * checksum, and psize bits. For an explanation of the purpose of this, see * the comment block on object set authentication. */ static int zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum) { blkptr_t *bp; int i, epb = datalen >> SPA_BLKPTRSHIFT; SHA2_CTX ctx; uint8_t digestbuf[SHA512_DIGEST_LENGTH]; /* checksum all of the MACs from the layer below */ SHA2Init(SHA512, &ctx); for (i = 0, bp = buf; i < epb; i++, bp++) { zio_crypt_bp_do_indrect_checksum_updates(&ctx, version, byteswap, bp); } SHA2Final(digestbuf, &ctx); if (generate) { bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN); return (0); } if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) return (SET_ERROR(ECKSUM)); return (0); } int zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf, uint_t datalen, boolean_t byteswap, uint8_t *cksum) { int ret; /* * Unfortunately, callers of this function will not always have * easy access to the on-disk format version. This info is * normally found in the DSL Crypto Key, but the checksum-of-MACs * is expected to be verifiable even when the key isn't loaded. * Here, instead of doing a ZAP lookup for the version for each * zio, we simply try both existing formats. */ ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf, datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum); if (ret == ECKSUM) { ASSERT(!generate); ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf, datalen, 0, byteswap, cksum); } return (ret); } int zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, uint_t datalen, boolean_t byteswap, uint8_t *cksum) { int ret; void *buf; buf = abd_borrow_buf_copy(abd, datalen); ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen, byteswap, cksum); abd_return_buf(abd, buf, datalen); return (ret); } /* * Special case handling routine for encrypting / decrypting ZIL blocks. * We do not check for the older ZIL chain because the encryption feature * was not available before the newer ZIL chain was introduced. The goal * here is to encrypt everything except the blkptr_t of a lr_write_t and * the zil_chain_t header. Everything that is not encrypted is authenticated. */ static int zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt) { int ret; uint64_t txtype, lr_len; uint_t nr_src, nr_dst, crypt_len; uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; zil_chain_t *zilc; lr_t *lr; uint8_t *aadbuf = zio_buf_alloc(datalen); /* cipherbuf always needs an extra iovec for the MAC */ if (encrypt) { src = plainbuf; dst = cipherbuf; nr_src = 0; nr_dst = 1; } else { src = cipherbuf; dst = plainbuf; nr_src = 1; nr_dst = 0; } bzero(dst, datalen); /* find the start and end record of the log block */ zilc = (zil_chain_t *)src; slrp = src + sizeof (zil_chain_t); aadp = aadbuf; blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused); /* calculate the number of encrypted iovecs we will need */ for (; slrp < blkend; slrp += lr_len) { lr = (lr_t *)slrp; if (!byteswap) { txtype = lr->lrc_txtype; lr_len = lr->lrc_reclen; } else { txtype = BSWAP_64(lr->lrc_txtype); lr_len = BSWAP_64(lr->lrc_reclen); } nr_iovecs++; if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t)) nr_iovecs++; } nr_src += nr_iovecs; nr_dst += nr_iovecs; /* allocate the iovec arrays */ if (nr_src != 0) { src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); if (src_iovecs == NULL) { ret = SET_ERROR(ENOMEM); goto error; } } if (nr_dst != 0) { dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); if (dst_iovecs == NULL) { ret = SET_ERROR(ENOMEM); goto error; } } /* * Copy the plain zil header over and authenticate everything except * the checksum that will store our MAC. If we are writing the data * the embedded checksum will not have been calculated yet, so we don't * authenticate that. */ bcopy(src, dst, sizeof (zil_chain_t)); bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t)); aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t); aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t); /* loop over records again, filling in iovecs */ nr_iovecs = 0; slrp = src + sizeof (zil_chain_t); dlrp = dst + sizeof (zil_chain_t); for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) { lr = (lr_t *)slrp; if (!byteswap) { txtype = lr->lrc_txtype; lr_len = lr->lrc_reclen; } else { txtype = BSWAP_64(lr->lrc_txtype); lr_len = BSWAP_64(lr->lrc_reclen); } /* copy the common lr_t */ bcopy(slrp, dlrp, sizeof (lr_t)); bcopy(slrp, aadp, sizeof (lr_t)); aadp += sizeof (lr_t); aad_len += sizeof (lr_t); ASSERT3P(src_iovecs, !=, NULL); ASSERT3P(dst_iovecs, !=, NULL); /* * If this is a TX_WRITE record we want to encrypt everything * except the bp if exists. If the bp does exist we want to * authenticate it. */ if (txtype == TX_WRITE) { crypt_len = sizeof (lr_write_t) - sizeof (lr_t) - sizeof (blkptr_t); src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); src_iovecs[nr_iovecs].iov_len = crypt_len; dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); dst_iovecs[nr_iovecs].iov_len = crypt_len; /* copy the bp now since it will not be encrypted */ bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), dlrp + sizeof (lr_write_t) - sizeof (blkptr_t), sizeof (blkptr_t)); bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), aadp, sizeof (blkptr_t)); aadp += sizeof (blkptr_t); aad_len += sizeof (blkptr_t); nr_iovecs++; total_len += crypt_len; if (lr_len != sizeof (lr_write_t)) { crypt_len = lr_len - sizeof (lr_write_t); src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_write_t); src_iovecs[nr_iovecs].iov_len = crypt_len; dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_write_t); dst_iovecs[nr_iovecs].iov_len = crypt_len; nr_iovecs++; total_len += crypt_len; } } else { crypt_len = lr_len - sizeof (lr_t); src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); src_iovecs[nr_iovecs].iov_len = crypt_len; dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); dst_iovecs[nr_iovecs].iov_len = crypt_len; nr_iovecs++; total_len += crypt_len; } } *no_crypt = (nr_iovecs == 0); *enc_len = total_len; *authbuf = aadbuf; *auth_len = aad_len; if (encrypt) { puio->uio_iov = src_iovecs; puio->uio_iovcnt = nr_src; cuio->uio_iov = dst_iovecs; cuio->uio_iovcnt = nr_dst; } else { puio->uio_iov = dst_iovecs; puio->uio_iovcnt = nr_dst; cuio->uio_iov = src_iovecs; cuio->uio_iovcnt = nr_src; } return (0); error: zio_buf_free(aadbuf, datalen); if (src_iovecs != NULL) kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); if (dst_iovecs != NULL) kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); *enc_len = 0; *authbuf = NULL; *auth_len = 0; *no_crypt = B_FALSE; puio->uio_iov = NULL; puio->uio_iovcnt = 0; cuio->uio_iov = NULL; cuio->uio_iovcnt = 0; return (ret); } /* * Special case handling routine for encrypting / decrypting dnode blocks. */ static int zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt) { int ret; uint_t nr_src, nr_dst, crypt_len; uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; uint8_t *src, *dst, *aadp; dnode_phys_t *dnp, *adnp, *sdnp, *ddnp; uint8_t *aadbuf = zio_buf_alloc(datalen); if (encrypt) { src = plainbuf; dst = cipherbuf; nr_src = 0; nr_dst = 1; } else { src = cipherbuf; dst = plainbuf; nr_src = 1; nr_dst = 0; } sdnp = (dnode_phys_t *)src; ddnp = (dnode_phys_t *)dst; aadp = aadbuf; /* * Count the number of iovecs we will need to do the encryption by * counting the number of bonus buffers that need to be encrypted. */ for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { /* * This block may still be byteswapped. However, all of the * values we use are either uint8_t's (for which byteswapping * is a noop) or a * != 0 check, which will work regardless * of whether or not we byteswap. */ if (sdnp[i].dn_type != DMU_OT_NONE && DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) && sdnp[i].dn_bonuslen != 0) { nr_iovecs++; } } nr_src += nr_iovecs; nr_dst += nr_iovecs; if (nr_src != 0) { src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); if (src_iovecs == NULL) { ret = SET_ERROR(ENOMEM); goto error; } } if (nr_dst != 0) { dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); if (dst_iovecs == NULL) { ret = SET_ERROR(ENOMEM); goto error; } } nr_iovecs = 0; /* * Iterate through the dnodes again, this time filling in the uios * we allocated earlier. We also concatenate any data we want to * authenticate onto aadbuf. */ for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { dnp = &sdnp[i]; /* copy over the core fields and blkptrs (kept as plaintext) */ bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp); if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]), sizeof (blkptr_t)); } /* * Handle authenticated data. We authenticate everything in * the dnode that can be brought over when we do a raw send. * This includes all of the core fields as well as the MACs * stored in the bp checksums and all of the portable bits * from blk_prop. We include the dnode padding here in case it * ever gets used in the future. Some dn_flags and dn_used are * not portable so we mask those out values out of the * authenticated data. */ crypt_len = offsetof(dnode_phys_t, dn_blkptr); bcopy(dnp, aadp, crypt_len); adnp = (dnode_phys_t *)aadp; adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; adnp->dn_used = 0; aadp += crypt_len; aad_len += crypt_len; for (j = 0; j < dnp->dn_nblkptr; j++) { zio_crypt_bp_do_aad_updates(&aadp, &aad_len, version, byteswap, &dnp->dn_blkptr[j]); } if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { zio_crypt_bp_do_aad_updates(&aadp, &aad_len, version, byteswap, DN_SPILL_BLKPTR(dnp)); } /* * If this bonus buffer needs to be encrypted, we prepare an * iovec_t. The encryption / decryption functions will fill * this in for us with the encrypted or decrypted data. * Otherwise we add the bonus buffer to the authenticated * data buffer and copy it over to the destination. The * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that * we can guarantee alignment with the AES block size * (128 bits). */ crypt_len = DN_MAX_BONUS_LEN(dnp); if (dnp->dn_type != DMU_OT_NONE && DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && dnp->dn_bonuslen != 0) { ASSERT3U(nr_iovecs, <, nr_src); ASSERT3U(nr_iovecs, <, nr_dst); ASSERT3P(src_iovecs, !=, NULL); ASSERT3P(dst_iovecs, !=, NULL); src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp); src_iovecs[nr_iovecs].iov_len = crypt_len; dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]); dst_iovecs[nr_iovecs].iov_len = crypt_len; nr_iovecs++; total_len += crypt_len; } else { bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len); bcopy(DN_BONUS(dnp), aadp, crypt_len); aadp += crypt_len; aad_len += crypt_len; } } *no_crypt = (nr_iovecs == 0); *enc_len = total_len; *authbuf = aadbuf; *auth_len = aad_len; if (encrypt) { puio->uio_iov = src_iovecs; puio->uio_iovcnt = nr_src; cuio->uio_iov = dst_iovecs; cuio->uio_iovcnt = nr_dst; } else { puio->uio_iov = dst_iovecs; puio->uio_iovcnt = nr_dst; cuio->uio_iov = src_iovecs; cuio->uio_iovcnt = nr_src; } return (0); error: zio_buf_free(aadbuf, datalen); if (src_iovecs != NULL) kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); if (dst_iovecs != NULL) kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); *enc_len = 0; *authbuf = NULL; *auth_len = 0; *no_crypt = B_FALSE; puio->uio_iov = NULL; puio->uio_iovcnt = 0; cuio->uio_iov = NULL; cuio->uio_iovcnt = 0; return (ret); } static int zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len) { int ret; uint_t nr_plain = 1, nr_cipher = 2; iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; /* allocate the iovecs for the plain and cipher data */ plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t), KM_SLEEP); if (!plain_iovecs) { ret = SET_ERROR(ENOMEM); goto error; } cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t), KM_SLEEP); if (!cipher_iovecs) { ret = SET_ERROR(ENOMEM); goto error; } plain_iovecs[0].iov_base = plainbuf; plain_iovecs[0].iov_len = datalen; cipher_iovecs[0].iov_base = cipherbuf; cipher_iovecs[0].iov_len = datalen; *enc_len = datalen; puio->uio_iov = plain_iovecs; puio->uio_iovcnt = nr_plain; cuio->uio_iov = cipher_iovecs; cuio->uio_iovcnt = nr_cipher; return (0); error: if (plain_iovecs != NULL) kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t)); if (cipher_iovecs != NULL) kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t)); *enc_len = 0; puio->uio_iov = NULL; puio->uio_iovcnt = 0; cuio->uio_iov = NULL; cuio->uio_iovcnt = 0; return (ret); } /* * This function builds up the plaintext (puio) and ciphertext (cuio) uios so * that they can be used for encryption and decryption by zio_do_crypt_uio(). * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks * requiring special handling to parse out pieces that are to be encrypted. The * authbuf is used by these special cases to store additional authenticated * data (AAD) for the encryption modes. */ static int zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uint8_t *mac, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt) { int ret; iovec_t *mac_iov; ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE); /* route to handler */ switch (ot) { case DMU_OT_INTENT_LOG: ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, no_crypt); break; case DMU_OT_DNODE: ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, no_crypt); break; default: ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, datalen, puio, cuio, enc_len); *authbuf = NULL; *auth_len = 0; *no_crypt = B_FALSE; break; } if (ret != 0) goto error; /* populate the uios */ puio->uio_segflg = UIO_SYSSPACE; cuio->uio_segflg = UIO_SYSSPACE; mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]); mac_iov->iov_base = mac; mac_iov->iov_len = ZIO_DATA_MAC_LEN; return (0); error: return (ret); } /* * Primary encryption / decryption entrypoint for zio data. */ int zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf, boolean_t *no_crypt) { int ret; boolean_t locked = B_FALSE; uint64_t crypt = key->zk_crypt; uint_t keydata_len = zio_crypt_table[crypt].ci_keylen; uint_t enc_len, auth_len; zfs_uio_t puio, cuio; uint8_t enc_keydata[MASTER_KEY_MAX_LEN]; crypto_key_t tmp_ckey, *ckey = NULL; crypto_ctx_template_t tmpl; uint8_t *authbuf = NULL; /* * If the needed key is the current one, just use it. Otherwise we * need to generate a temporary one from the given salt + master key. * If we are encrypting, we must return a copy of the current salt * so that it can be stored in the blkptr_t. */ rw_enter(&key->zk_salt_lock, RW_READER); locked = B_TRUE; if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) { ckey = &key->zk_current_key; tmpl = key->zk_current_tmpl; } else { rw_exit(&key->zk_salt_lock); locked = B_FALSE; ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len); if (ret != 0) goto error; tmp_ckey.ck_format = CRYPTO_KEY_RAW; tmp_ckey.ck_data = enc_keydata; tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len); ckey = &tmp_ckey; tmpl = NULL; } /* * Attempt to use QAT acceleration if we can. We currently don't * do this for metadnode and ZIL blocks, since they have a much * more involved buffer layout and the qat_crypt() function only * works in-place. */ if (qat_crypt_use_accel(datalen) && ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) { uint8_t *srcbuf, *dstbuf; if (encrypt) { srcbuf = plainbuf; dstbuf = cipherbuf; } else { srcbuf = cipherbuf; dstbuf = plainbuf; } ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf, dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen); if (ret == CPA_STATUS_SUCCESS) { if (locked) { rw_exit(&key->zk_salt_lock); locked = B_FALSE; } return (0); } /* If the hardware implementation fails fall back to software */ } bzero(&puio, sizeof (zfs_uio_t)); bzero(&cuio, sizeof (zfs_uio_t)); /* create uios for encryption */ ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, &authbuf, &auth_len, no_crypt); if (ret != 0) goto error; /* perform the encryption / decryption in software */ ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len, &puio, &cuio, authbuf, auth_len); if (ret != 0) goto error; if (locked) { rw_exit(&key->zk_salt_lock); locked = B_FALSE; } if (authbuf != NULL) zio_buf_free(authbuf, datalen); if (ckey == &tmp_ckey) bzero(enc_keydata, keydata_len); zio_crypt_destroy_uio(&puio); zio_crypt_destroy_uio(&cuio); return (0); error: if (locked) rw_exit(&key->zk_salt_lock); if (authbuf != NULL) zio_buf_free(authbuf, datalen); if (ckey == &tmp_ckey) bzero(enc_keydata, keydata_len); zio_crypt_destroy_uio(&puio); zio_crypt_destroy_uio(&cuio); return (ret); } /* * Simple wrapper around zio_do_crypt_data() to work with abd's instead of * linear buffers. */ int zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt) { int ret; void *ptmp, *ctmp; if (encrypt) { ptmp = abd_borrow_buf_copy(pabd, datalen); ctmp = abd_borrow_buf(cabd, datalen); } else { ptmp = abd_borrow_buf(pabd, datalen); ctmp = abd_borrow_buf_copy(cabd, datalen); } ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac, datalen, ptmp, ctmp, no_crypt); if (ret != 0) goto error; if (encrypt) { abd_return_buf(pabd, ptmp, datalen); abd_return_buf_copy(cabd, ctmp, datalen); } else { abd_return_buf_copy(pabd, ptmp, datalen); abd_return_buf(cabd, ctmp, datalen); } return (0); error: if (encrypt) { abd_return_buf(pabd, ptmp, datalen); abd_return_buf_copy(cabd, ctmp, datalen); } else { abd_return_buf_copy(pabd, ptmp, datalen); abd_return_buf(cabd, ctmp, datalen); } return (ret); } #if defined(_KERNEL) /* BEGIN CSTYLED */ module_param(zfs_key_max_salt_uses, ulong, 0644); MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " "can be used for generating encryption keys before it is rotated"); /* END CSTYLED */ #endif