diff --git a/contrib/pam_zfs_key/pam_zfs_key.c b/contrib/pam_zfs_key/pam_zfs_key.c index c617a6e6b370..a0bc172c6f44 100644 --- a/contrib/pam_zfs_key/pam_zfs_key.c +++ b/contrib/pam_zfs_key/pam_zfs_key.c @@ -1,1087 +1,1088 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Copyright (c) 2020, Felix Dörre * All rights reserved. */ #include #include #include #include #include #include #define PAM_SM_AUTH #define PAM_SM_PASSWORD #define PAM_SM_SESSION #include #if defined(__linux__) #include #define MAP_FLAGS MAP_PRIVATE | MAP_ANONYMOUS #elif defined(__FreeBSD__) #include static void pam_syslog(pam_handle_t *pamh, int loglevel, const char *fmt, ...) { (void) pamh; va_list args; va_start(args, fmt); vsyslog(loglevel, fmt, args); va_end(args); } #define MAP_FLAGS MAP_PRIVATE | MAP_ANON | MAP_NOCORE #endif #include #include #include #include #include #include #include #include #include static const char PASSWORD_VAR_NAME[] = "pam_zfs_key_authtok"; static const char OLD_PASSWORD_VAR_NAME[] = "pam_zfs_key_oldauthtok"; static libzfs_handle_t *g_zfs; static void destroy_pw(pam_handle_t *pamh, void *data, int errcode); typedef int (*mlock_func_t) (const void *, size_t); typedef struct { size_t len; char *value; } pw_password_t; /* * Try to mlock(2) or munlock(2) addr while handling EAGAIN by retrying ten * times and sleeping 10 milliseconds in between for a total of 0.1 * seconds. lock_func must point to either mlock(2) or munlock(2). */ static int try_lock(mlock_func_t lock_func, const void *addr, size_t len) { int err; int retries = 10; useconds_t sleep_dur = 10 * 1000; if ((err = (*lock_func)(addr, len)) != EAGAIN) { return (err); } for (int i = retries; i > 0; --i) { (void) usleep(sleep_dur); if ((err = (*lock_func)(addr, len)) != EAGAIN) { break; } } return (err); } static pw_password_t * alloc_pw_size(size_t len) { pw_password_t *pw = malloc(sizeof (pw_password_t)); if (!pw) { return (NULL); } pw->len = len; /* * We use mmap(2) rather than malloc(3) since later on we mlock(2) the * memory region. Since mlock(2) and munlock(2) operate on whole memory * pages we should allocate a whole page here as mmap(2) does. Further * this ensures that the addresses passed to mlock(2) an munlock(2) are * on a page boundary as suggested by FreeBSD and required by some * other implementations. Finally we avoid inadvertently munlocking * memory mlocked by an concurrently running instance of us. */ pw->value = mmap(NULL, pw->len, PROT_READ | PROT_WRITE, MAP_FLAGS, -1, 0); if (pw->value == MAP_FAILED) { free(pw); return (NULL); } if (try_lock(mlock, pw->value, pw->len) != 0) { (void) munmap(pw->value, pw->len); free(pw); return (NULL); } return (pw); } static pw_password_t * alloc_pw_string(const char *source) { size_t len = strlen(source) + 1; pw_password_t *pw = alloc_pw_size(len); if (!pw) { return (NULL); } memcpy(pw->value, source, pw->len); return (pw); } static void pw_free(pw_password_t *pw) { memset(pw->value, 0, pw->len); if (try_lock(munlock, pw->value, pw->len) == 0) { (void) munmap(pw->value, pw->len); } free(pw); } static pw_password_t * pw_fetch(pam_handle_t *pamh, int tok) { const char *token; if (pam_get_authtok(pamh, tok, &token, NULL) != PAM_SUCCESS) { pam_syslog(pamh, LOG_ERR, "couldn't get password from PAM stack"); return (NULL); } if (!token) { pam_syslog(pamh, LOG_ERR, "token from PAM stack is null"); return (NULL); } return (alloc_pw_string(token)); } static const pw_password_t * pw_fetch_lazy(pam_handle_t *pamh, int tok, const char *var_name) { pw_password_t *pw = pw_fetch(pamh, tok); if (pw == NULL) { return (NULL); } int ret = pam_set_data(pamh, var_name, pw, destroy_pw); if (ret != PAM_SUCCESS) { pw_free(pw); pam_syslog(pamh, LOG_ERR, "pam_set_data failed"); return (NULL); } return (pw); } static const pw_password_t * pw_get(pam_handle_t *pamh, int tok, const char *var_name) { const pw_password_t *authtok = NULL; int ret = pam_get_data(pamh, var_name, (const void**)(&authtok)); if (ret == PAM_SUCCESS) return (authtok); if (ret == PAM_NO_MODULE_DATA) return (pw_fetch_lazy(pamh, tok, var_name)); pam_syslog(pamh, LOG_ERR, "password not available"); return (NULL); } static int pw_clear(pam_handle_t *pamh, const char *var_name) { int ret = pam_set_data(pamh, var_name, NULL, NULL); if (ret != PAM_SUCCESS) { pam_syslog(pamh, LOG_ERR, "clearing password failed"); return (-1); } return (0); } static void destroy_pw(pam_handle_t *pamh, void *data, int errcode) { (void) pamh, (void) errcode; if (data != NULL) { pw_free((pw_password_t *)data); } } static int pam_zfs_init(pam_handle_t *pamh) { int error = 0; if ((g_zfs = libzfs_init()) == NULL) { error = errno; pam_syslog(pamh, LOG_ERR, "Zfs initialization error: %s", libzfs_error_init(error)); } return (error); } static void pam_zfs_free(void) { libzfs_fini(g_zfs); } static pw_password_t * prepare_passphrase(pam_handle_t *pamh, zfs_handle_t *ds, const char *passphrase, nvlist_t *nvlist) { pw_password_t *key = alloc_pw_size(WRAPPING_KEY_LEN); if (!key) { return (NULL); } uint64_t salt; uint64_t iters; if (nvlist != NULL) { int fd = open("/dev/urandom", O_RDONLY); if (fd < 0) { pw_free(key); return (NULL); } int bytes_read = 0; char *buf = (char *)&salt; size_t bytes = sizeof (uint64_t); while (bytes_read < bytes) { ssize_t len = read(fd, buf + bytes_read, bytes - bytes_read); if (len < 0) { close(fd); pw_free(key); return (NULL); } bytes_read += len; } close(fd); if (nvlist_add_uint64(nvlist, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), salt)) { pam_syslog(pamh, LOG_ERR, "failed to add salt to nvlist"); pw_free(key); return (NULL); } iters = DEFAULT_PBKDF2_ITERATIONS; if (nvlist_add_uint64(nvlist, zfs_prop_to_name( ZFS_PROP_PBKDF2_ITERS), iters)) { pam_syslog(pamh, LOG_ERR, "failed to add iters to nvlist"); pw_free(key); return (NULL); } } else { salt = zfs_prop_get_int(ds, ZFS_PROP_PBKDF2_SALT); iters = zfs_prop_get_int(ds, ZFS_PROP_PBKDF2_ITERS); } salt = LE_64(salt); if (!PKCS5_PBKDF2_HMAC_SHA1((char *)passphrase, strlen(passphrase), (uint8_t *)&salt, sizeof (uint64_t), iters, WRAPPING_KEY_LEN, (uint8_t *)key->value)) { pam_syslog(pamh, LOG_ERR, "pbkdf failed"); pw_free(key); return (NULL); } return (key); } static int is_key_loaded(pam_handle_t *pamh, const char *ds_name) { zfs_handle_t *ds = zfs_open(g_zfs, ds_name, ZFS_TYPE_FILESYSTEM); if (ds == NULL) { pam_syslog(pamh, LOG_ERR, "dataset %s not found", ds_name); return (-1); } int keystatus = zfs_prop_get_int(ds, ZFS_PROP_KEYSTATUS); zfs_close(ds); return (keystatus != ZFS_KEYSTATUS_UNAVAILABLE); } static int change_key(pam_handle_t *pamh, const char *ds_name, const char *passphrase) { zfs_handle_t *ds = zfs_open(g_zfs, ds_name, ZFS_TYPE_FILESYSTEM); if (ds == NULL) { pam_syslog(pamh, LOG_ERR, "dataset %s not found", ds_name); return (-1); } nvlist_t *nvlist = fnvlist_alloc(); pw_password_t *key = prepare_passphrase(pamh, ds, passphrase, nvlist); if (key == NULL) { nvlist_free(nvlist); zfs_close(ds); return (-1); } if (nvlist_add_string(nvlist, zfs_prop_to_name(ZFS_PROP_KEYLOCATION), "prompt")) { pam_syslog(pamh, LOG_ERR, "nvlist_add failed for keylocation"); pw_free(key); nvlist_free(nvlist); zfs_close(ds); return (-1); } if (nvlist_add_uint64(nvlist, zfs_prop_to_name(ZFS_PROP_KEYFORMAT), ZFS_KEYFORMAT_PASSPHRASE)) { pam_syslog(pamh, LOG_ERR, "nvlist_add failed for keyformat"); pw_free(key); nvlist_free(nvlist); zfs_close(ds); return (-1); } int ret = lzc_change_key(ds_name, DCP_CMD_NEW_KEY, nvlist, (uint8_t *)key->value, WRAPPING_KEY_LEN); pw_free(key); if (ret) { pam_syslog(pamh, LOG_ERR, "change_key failed: %d", ret); nvlist_free(nvlist); zfs_close(ds); return (-1); } nvlist_free(nvlist); zfs_close(ds); return (0); } typedef struct { char *homes_prefix; char *runstatedir; char *homedir; char *dsname; uid_t uid_min; uid_t uid_max; uid_t uid; const char *username; boolean_t unmount_and_unload; boolean_t force_unmount; boolean_t recursive_homes; boolean_t mount_recursively; } zfs_key_config_t; static int zfs_key_config_load(pam_handle_t *pamh, zfs_key_config_t *config, int argc, const char **argv) { config->homes_prefix = strdup("rpool/home"); if (config->homes_prefix == NULL) { pam_syslog(pamh, LOG_ERR, "strdup failure"); return (PAM_SERVICE_ERR); } config->runstatedir = strdup(RUNSTATEDIR "/pam_zfs_key"); if (config->runstatedir == NULL) { pam_syslog(pamh, LOG_ERR, "strdup failure"); free(config->homes_prefix); return (PAM_SERVICE_ERR); } const char *name; if (pam_get_user(pamh, &name, NULL) != PAM_SUCCESS) { pam_syslog(pamh, LOG_ERR, "couldn't get username from PAM stack"); free(config->runstatedir); free(config->homes_prefix); return (PAM_SERVICE_ERR); } struct passwd *entry = getpwnam(name); if (!entry) { free(config->runstatedir); free(config->homes_prefix); return (PAM_USER_UNKNOWN); } config->uid_min = 1000; config->uid_max = MAXUID; config->uid = entry->pw_uid; config->username = name; config->unmount_and_unload = B_TRUE; config->force_unmount = B_FALSE; config->recursive_homes = B_FALSE; config->mount_recursively = B_FALSE; config->dsname = NULL; config->homedir = NULL; for (int c = 0; c < argc; c++) { if (strncmp(argv[c], "homes=", 6) == 0) { free(config->homes_prefix); config->homes_prefix = strdup(argv[c] + 6); } else if (strncmp(argv[c], "runstatedir=", 12) == 0) { free(config->runstatedir); config->runstatedir = strdup(argv[c] + 12); } else if (strncmp(argv[c], "uid_min=", 8) == 0) { sscanf(argv[c] + 8, "%u", &config->uid_min); } else if (strncmp(argv[c], "uid_max=", 8) == 0) { sscanf(argv[c] + 8, "%u", &config->uid_max); } else if (strcmp(argv[c], "nounmount") == 0) { config->unmount_and_unload = B_FALSE; } else if (strcmp(argv[c], "forceunmount") == 0) { config->force_unmount = B_TRUE; } else if (strcmp(argv[c], "recursive_homes") == 0) { config->recursive_homes = B_TRUE; } else if (strcmp(argv[c], "mount_recursively") == 0) { config->mount_recursively = B_TRUE; } else if (strcmp(argv[c], "prop_mountpoint") == 0) { if (config->homedir == NULL) config->homedir = strdup(entry->pw_dir); } } return (PAM_SUCCESS); } typedef struct { pam_handle_t *pamh; zfs_key_config_t *target; } mount_umount_dataset_data_t; static int mount_dataset(zfs_handle_t *zhp, void *data) { mount_umount_dataset_data_t *mount_umount_dataset_data = data; zfs_key_config_t *target = mount_umount_dataset_data->target; pam_handle_t *pamh = mount_umount_dataset_data->pamh; /* Refresh properties to get the latest key status */ zfs_refresh_properties(zhp); int ret = 0; /* Check if dataset type is filesystem */ if (zhp->zfs_type != ZFS_TYPE_FILESYSTEM) { pam_syslog(pamh, LOG_DEBUG, "dataset is not filesystem: %s, skipping.", zfs_get_name(zhp)); return (0); } /* Check if encryption key is available */ if (zfs_prop_get_int(zhp, ZFS_PROP_KEYSTATUS) == ZFS_KEYSTATUS_UNAVAILABLE) { pam_syslog(pamh, LOG_WARNING, "key unavailable for: %s, skipping", zfs_get_name(zhp)); return (0); } /* Check if prop canmount is on */ if (zfs_prop_get_int(zhp, ZFS_PROP_CANMOUNT) != ZFS_CANMOUNT_ON) { pam_syslog(pamh, LOG_INFO, "canmount is not on for: %s, skipping", zfs_get_name(zhp)); return (0); } /* Get mountpoint prop for check */ char mountpoint[ZFS_MAXPROPLEN]; if ((ret = zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint, sizeof (mountpoint), NULL, NULL, 0, 1)) != 0) { pam_syslog(pamh, LOG_ERR, "failed to get mountpoint prop: %d", ret); return (-1); } /* Check if mountpoint isn't none or legacy */ if (strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) == 0 || strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) == 0) { pam_syslog(pamh, LOG_INFO, "mountpoint is none or legacy for: %s, skipping", zfs_get_name(zhp)); return (0); } /* Don't mount the dataset if already mounted */ if (zfs_is_mounted(zhp, NULL)) { pam_syslog(pamh, LOG_INFO, "already mounted: %s", zfs_get_name(zhp)); return (0); } /* Mount the dataset */ ret = zfs_mount(zhp, NULL, 0); if (ret) { pam_syslog(pamh, LOG_ERR, "zfs_mount failed for %s with: %d", zfs_get_name(zhp), ret); return (ret); } /* Recursively mount children if the recursive flag is set */ if (target->mount_recursively) { ret = zfs_iter_filesystems_v2(zhp, 0, mount_dataset, data); if (ret != 0) { pam_syslog(pamh, LOG_ERR, "child iteration failed: %d", ret); return (-1); } } return (ret); } static int umount_dataset(zfs_handle_t *zhp, void *data) { mount_umount_dataset_data_t *mount_umount_dataset_data = data; zfs_key_config_t *target = mount_umount_dataset_data->target; pam_handle_t *pamh = mount_umount_dataset_data->pamh; int ret = 0; /* Recursively umount children if the recursive flag is set */ if (target->mount_recursively) { ret = zfs_iter_filesystems_v2(zhp, 0, umount_dataset, data); if (ret != 0) { pam_syslog(pamh, LOG_ERR, "child iteration failed: %d", ret); return (-1); } } /* Check if dataset type is filesystem */ if (zhp->zfs_type != ZFS_TYPE_FILESYSTEM) { pam_syslog(pamh, LOG_DEBUG, "dataset is not filesystem: %s, skipping", zfs_get_name(zhp)); return (0); } /* Don't umount the dataset if already unmounted */ if (zfs_is_mounted(zhp, NULL) == 0) { pam_syslog(pamh, LOG_INFO, "already unmounted: %s", zfs_get_name(zhp)); return (0); } /* Unmount the dataset */ ret = zfs_unmount(zhp, NULL, target->force_unmount ? MS_FORCE : 0); if (ret) { pam_syslog(pamh, LOG_ERR, "zfs_unmount failed for %s with: %d", zfs_get_name(zhp), ret); return (ret); } return (ret); } static int decrypt_mount(pam_handle_t *pamh, zfs_key_config_t *config, const char *ds_name, const char *passphrase, boolean_t noop) { zfs_handle_t *ds = zfs_open(g_zfs, ds_name, ZFS_TYPE_FILESYSTEM); if (ds == NULL) { pam_syslog(pamh, LOG_ERR, "dataset %s not found", ds_name); return (-1); } pw_password_t *key = prepare_passphrase(pamh, ds, passphrase, NULL); if (key == NULL) { zfs_close(ds); return (-1); } int ret = lzc_load_key(ds_name, noop, (uint8_t *)key->value, WRAPPING_KEY_LEN); pw_free(key); if (ret && ret != EEXIST) { pam_syslog(pamh, LOG_ERR, "load_key failed: %d", ret); zfs_close(ds); return (-1); } if (noop) { zfs_close(ds); return (0); } mount_umount_dataset_data_t data; data.pamh = pamh; data.target = config; ret = mount_dataset(ds, &data); if (ret != 0) { pam_syslog(pamh, LOG_ERR, "mount failed: %d", ret); zfs_close(ds); return (-1); } zfs_close(ds); return (0); } static int unmount_unload(pam_handle_t *pamh, const char *ds_name, zfs_key_config_t *target) { zfs_handle_t *ds = zfs_open(g_zfs, ds_name, ZFS_TYPE_FILESYSTEM); if (ds == NULL) { pam_syslog(pamh, LOG_ERR, "dataset %s not found", ds_name); return (-1); } mount_umount_dataset_data_t data; data.pamh = pamh; data.target = target; int ret = umount_dataset(ds, &data); if (ret) { pam_syslog(pamh, LOG_ERR, "unmount_dataset failed with: %d", ret); zfs_close(ds); return (-1); } ret = lzc_unload_key(ds_name); if (ret) { pam_syslog(pamh, LOG_ERR, "unload_key failed with: %d", ret); zfs_close(ds); return (-1); } zfs_close(ds); return (0); } static void zfs_key_config_free(zfs_key_config_t *config) { free(config->homes_prefix); free(config->runstatedir); free(config->homedir); free(config->dsname); } static int find_dsname_by_prop_value(zfs_handle_t *zhp, void *data) { zfs_type_t type = zfs_get_type(zhp); zfs_key_config_t *target = data; char mountpoint[ZFS_MAXPROPLEN]; /* Skip any datasets whose type does not match */ if ((type & ZFS_TYPE_FILESYSTEM) == 0) { zfs_close(zhp); return (0); } /* Skip any datasets whose mountpoint does not match */ (void) zfs_prop_get(zhp, ZFS_PROP_MOUNTPOINT, mountpoint, sizeof (mountpoint), NULL, NULL, 0, B_FALSE); if (strcmp(target->homedir, mountpoint) != 0) { if (target->recursive_homes) { (void) zfs_iter_filesystems_v2(zhp, 0, find_dsname_by_prop_value, target); } zfs_close(zhp); return (target->dsname != NULL); } target->dsname = strdup(zfs_get_name(zhp)); zfs_close(zhp); return (1); } static char * zfs_key_config_get_dataset(pam_handle_t *pamh, zfs_key_config_t *config) { if (config->homedir != NULL && config->homes_prefix != NULL) { if (strcmp(config->homes_prefix, "*") == 0) { (void) zfs_iter_root(g_zfs, find_dsname_by_prop_value, config); } else { zfs_handle_t *zhp = zfs_open(g_zfs, config->homes_prefix, ZFS_TYPE_FILESYSTEM); if (zhp == NULL) { pam_syslog(pamh, LOG_ERR, "dataset %s not found", config->homes_prefix); return (NULL); } (void) zfs_iter_filesystems_v2(zhp, 0, find_dsname_by_prop_value, config); zfs_close(zhp); } char *dsname = config->dsname; config->dsname = NULL; return (dsname); } if (config->homes_prefix == NULL) { return (NULL); } size_t len = ZFS_MAX_DATASET_NAME_LEN; size_t total_len = strlen(config->homes_prefix) + 1 + strlen(config->username); if (total_len > len) { return (NULL); } char *ret = malloc(len + 1); if (!ret) { return (NULL); } ret[0] = 0; (void) snprintf(ret, len + 1, "%s/%s", config->homes_prefix, config->username); return (ret); } static int zfs_key_config_modify_session_counter(pam_handle_t *pamh, zfs_key_config_t *config, int delta) { const char *runtime_path = config->runstatedir; if (mkdir(runtime_path, S_IRWXU) != 0 && errno != EEXIST) { pam_syslog(pamh, LOG_ERR, "Can't create runtime path: %d", errno); return (-1); } if (chown(runtime_path, 0, 0) != 0) { pam_syslog(pamh, LOG_ERR, "Can't chown runtime path: %d", errno); return (-1); } if (chmod(runtime_path, S_IRWXU) != 0) { pam_syslog(pamh, LOG_ERR, "Can't chmod runtime path: %d", errno); return (-1); } char *counter_path; if (asprintf(&counter_path, "%s/%u", runtime_path, config->uid) == -1) return (-1); const int fd = open(counter_path, O_RDWR | O_CLOEXEC | O_CREAT | O_NOFOLLOW, S_IRUSR | S_IWUSR); free(counter_path); if (fd < 0) { pam_syslog(pamh, LOG_ERR, "Can't open counter file: %d", errno); return (-1); } if (flock(fd, LOCK_EX) != 0) { pam_syslog(pamh, LOG_ERR, "Can't lock counter file: %d", errno); close(fd); return (-1); } char counter[20]; char *pos = counter; int remaining = sizeof (counter) - 1; int ret; counter[sizeof (counter) - 1] = 0; while (remaining > 0 && (ret = read(fd, pos, remaining)) > 0) { remaining -= ret; pos += ret; } *pos = 0; long int counter_value = strtol(counter, NULL, 10); counter_value += delta; if (counter_value < 0) { counter_value = 0; } lseek(fd, 0, SEEK_SET); if (ftruncate(fd, 0) != 0) { pam_syslog(pamh, LOG_ERR, "Can't truncate counter file: %d", errno); close(fd); return (-1); } snprintf(counter, sizeof (counter), "%ld", counter_value); remaining = strlen(counter); pos = counter; while (remaining > 0 && (ret = write(fd, pos, remaining)) > 0) { remaining -= ret; pos += ret; } close(fd); return (counter_value); } __attribute__((visibility("default"))) PAM_EXTERN int pam_sm_authenticate(pam_handle_t *pamh, int flags, int argc, const char **argv) { (void) flags; if (geteuid() != 0) { pam_syslog(pamh, LOG_ERR, "Cannot zfs_mount when not being root."); return (PAM_SERVICE_ERR); } zfs_key_config_t config; int config_err = zfs_key_config_load(pamh, &config, argc, argv); if (config_err != PAM_SUCCESS) { return (config_err); } if (config.uid < config.uid_min || config.uid > config.uid_max) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } const pw_password_t *token = pw_fetch_lazy(pamh, PAM_AUTHTOK, PASSWORD_VAR_NAME); if (token == NULL) { zfs_key_config_free(&config); return (PAM_AUTH_ERR); } if (pam_zfs_init(pamh) != 0) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } char *dataset = zfs_key_config_get_dataset(pamh, &config); if (!dataset) { pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } if (decrypt_mount(pamh, &config, dataset, token->value, B_TRUE) == -1) { free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_AUTH_ERR); } free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SUCCESS); } __attribute__((visibility("default"))) PAM_EXTERN int pam_sm_setcred(pam_handle_t *pamh, int flags, int argc, const char **argv) { (void) pamh, (void) flags, (void) argc, (void) argv; return (PAM_SUCCESS); } __attribute__((visibility("default"))) PAM_EXTERN int pam_sm_chauthtok(pam_handle_t *pamh, int flags, int argc, const char **argv) { if (geteuid() != 0) { pam_syslog(pamh, LOG_ERR, "Cannot zfs_mount when not being root."); return (PAM_PERM_DENIED); } zfs_key_config_t config; if (zfs_key_config_load(pamh, &config, argc, argv) != PAM_SUCCESS) { return (PAM_SERVICE_ERR); } if (config.uid < config.uid_min || config.uid > config.uid_max) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } const pw_password_t *old_token = pw_get(pamh, PAM_OLDAUTHTOK, OLD_PASSWORD_VAR_NAME); { if (pam_zfs_init(pamh) != 0) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } char *dataset = zfs_key_config_get_dataset(pamh, &config); if (!dataset) { pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } if (!old_token) { pam_syslog(pamh, LOG_ERR, "old password from PAM stack is null"); free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } if (decrypt_mount(pamh, &config, dataset, old_token->value, B_TRUE) == -1) { pam_syslog(pamh, LOG_ERR, "old token mismatch"); free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_PERM_DENIED); } } if ((flags & PAM_UPDATE_AUTHTOK) != 0) { const pw_password_t *token = pw_get(pamh, PAM_AUTHTOK, PASSWORD_VAR_NAME); if (token == NULL) { pam_syslog(pamh, LOG_ERR, "new password unavailable"); pam_zfs_free(); zfs_key_config_free(&config); pw_clear(pamh, OLD_PASSWORD_VAR_NAME); return (PAM_SERVICE_ERR); } char *dataset = zfs_key_config_get_dataset(pamh, &config); if (!dataset) { pam_zfs_free(); zfs_key_config_free(&config); pw_clear(pamh, OLD_PASSWORD_VAR_NAME); pw_clear(pamh, PASSWORD_VAR_NAME); return (PAM_SERVICE_ERR); } int was_loaded = is_key_loaded(pamh, dataset); if (!was_loaded && decrypt_mount(pamh, &config, dataset, old_token->value, B_FALSE) == -1) { free(dataset); pam_zfs_free(); zfs_key_config_free(&config); pw_clear(pamh, OLD_PASSWORD_VAR_NAME); pw_clear(pamh, PASSWORD_VAR_NAME); return (PAM_SERVICE_ERR); } int changed = change_key(pamh, dataset, token->value); if (!was_loaded) { unmount_unload(pamh, dataset, &config); } free(dataset); pam_zfs_free(); zfs_key_config_free(&config); if (pw_clear(pamh, OLD_PASSWORD_VAR_NAME) == -1 || pw_clear(pamh, PASSWORD_VAR_NAME) == -1 || changed == -1) { return (PAM_SERVICE_ERR); } } else { zfs_key_config_free(&config); } return (PAM_SUCCESS); } PAM_EXTERN int pam_sm_open_session(pam_handle_t *pamh, int flags, int argc, const char **argv) { (void) flags; if (geteuid() != 0) { pam_syslog(pamh, LOG_ERR, "Cannot zfs_mount when not being root."); return (PAM_SUCCESS); } zfs_key_config_t config; if (zfs_key_config_load(pamh, &config, argc, argv) != PAM_SUCCESS) { return (PAM_SESSION_ERR); } if (config.uid < config.uid_min || config.uid > config.uid_max) { zfs_key_config_free(&config); return (PAM_SUCCESS); } int counter = zfs_key_config_modify_session_counter(pamh, &config, 1); if (counter != 1) { zfs_key_config_free(&config); return (PAM_SUCCESS); } const pw_password_t *token = pw_get(pamh, PAM_AUTHTOK, PASSWORD_VAR_NAME); if (token == NULL) { zfs_key_config_free(&config); return (PAM_SESSION_ERR); } if (pam_zfs_init(pamh) != 0) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } char *dataset = zfs_key_config_get_dataset(pamh, &config); if (!dataset) { pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } if (decrypt_mount(pamh, &config, dataset, token->value, B_FALSE) == -1) { free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } free(dataset); pam_zfs_free(); zfs_key_config_free(&config); if (pw_clear(pamh, PASSWORD_VAR_NAME) == -1) { return (PAM_SERVICE_ERR); } return (PAM_SUCCESS); } __attribute__((visibility("default"))) PAM_EXTERN int pam_sm_close_session(pam_handle_t *pamh, int flags, int argc, const char **argv) { (void) flags; if (geteuid() != 0) { pam_syslog(pamh, LOG_ERR, "Cannot zfs_mount when not being root."); return (PAM_SUCCESS); } zfs_key_config_t config; if (zfs_key_config_load(pamh, &config, argc, argv) != PAM_SUCCESS) { return (PAM_SESSION_ERR); } if (config.uid < config.uid_min || config.uid > config.uid_max) { zfs_key_config_free(&config); return (PAM_SUCCESS); } int counter = zfs_key_config_modify_session_counter(pamh, &config, -1); if (counter != 0) { zfs_key_config_free(&config); return (PAM_SUCCESS); } if (config.unmount_and_unload) { if (pam_zfs_init(pamh) != 0) { zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } char *dataset = zfs_key_config_get_dataset(pamh, &config); if (!dataset) { pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SESSION_ERR); } if (unmount_unload(pamh, dataset, &config) == -1) { free(dataset); pam_zfs_free(); zfs_key_config_free(&config); return (PAM_SESSION_ERR); } free(dataset); pam_zfs_free(); } zfs_key_config_free(&config); return (PAM_SUCCESS); } diff --git a/include/sys/zstd/zstd.h b/include/sys/zstd/zstd.h index 6d212b082f9a..8c3c6e67e9c1 100644 --- a/include/sys/zstd/zstd.h +++ b/include/sys/zstd/zstd.h @@ -1,231 +1,232 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2016-2018, Klara Inc. * Copyright (c) 2016-2018, Allan Jude * Copyright (c) 2018-2020, Sebastian Gottschall * Copyright (c) 2019-2020, Michael Niewöhner * Copyright (c) 2020, The FreeBSD Foundation [1] * * [1] Portions of this software were developed by Allan Jude * under sponsorship from the FreeBSD Foundation. */ #ifndef _ZFS_ZSTD_H #define _ZFS_ZSTD_H #ifdef __cplusplus extern "C" { #endif /* * ZSTD block header * NOTE: all fields in this header are in big endian order. */ typedef struct zfs_zstd_header { /* Compressed size of data */ uint32_t c_len; /* * Version and compression level * We used to use a union to reference compression level * and version easily, but as it turns out, relying on the * ordering of bitfields is not remotely portable. * So now we have get/set functions in zfs_zstd.c for * manipulating this in just the right way forever. */ uint32_t raw_version_level; char data[]; } zfs_zstdhdr_t; /* * Simple struct to pass the data from raw_version_level around. */ typedef struct zfs_zstd_meta { uint8_t level; uint32_t version; } zfs_zstdmeta_t; /* * kstat helper macros */ #define ZSTDSTAT(stat) (zstd_stats.stat.value.ui64) #define ZSTDSTAT_ZERO(stat) \ atomic_store_64(&zstd_stats.stat.value.ui64, 0) #define ZSTDSTAT_ADD(stat, val) \ atomic_add_64(&zstd_stats.stat.value.ui64, (val)) #define ZSTDSTAT_SUB(stat, val) \ atomic_sub_64(&zstd_stats.stat.value.ui64, (val)) #define ZSTDSTAT_BUMP(stat) ZSTDSTAT_ADD(stat, 1) /* (de)init for user space / kernel emulation */ int zstd_init(void); void zstd_fini(void); size_t zfs_zstd_compress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int level); int zfs_zstd_get_level(void *s_start, size_t s_len, uint8_t *level); int zfs_zstd_decompress_level(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, uint8_t *level); int zfs_zstd_decompress(abd_t *src, abd_t *dst, size_t s_len, size_t d_len, int n); void zfs_zstd_cache_reap_now(void); /* * So, the reason we have all these complicated set/get functions is that * originally, in the zstd "header" we wrote out to disk, we used a 32-bit * bitfield to store the "level" (8 bits) and "version" (24 bits). * * Unfortunately, bitfields make few promises about how they're arranged in * memory... * * By way of example, if we were using version 1.4.5 and level 3, it'd be * level = 0x03, version = 10405/0x0028A5, which gets broken into Vhigh = 0x00, * Vmid = 0x28, Vlow = 0xA5. We include these positions below to help follow * which data winds up where. * * As a consequence, we wound up with little endian platforms with a layout * like this in memory: * * 0 8 16 24 32 * +-------+-------+-------+-------+ * | Vlow | Vmid | Vhigh | level | * +-------+-------+-------+-------+ * =A5 =28 =00 =03 * * ...and then, after being run through BE_32(), serializing this out to * disk: * * 0 8 16 24 32 * +-------+-------+-------+-------+ * | level | Vhigh | Vmid | Vlow | * +-------+-------+-------+-------+ * =03 =00 =28 =A5 * * while on big-endian systems, since BE_32() is a noop there, both in * memory and on disk, we wind up with: * * 0 8 16 24 32 * +-------+-------+-------+-------+ * | Vhigh | Vmid | Vlow | level | * +-------+-------+-------+-------+ * =00 =28 =A5 =03 * * (Vhigh is always 0 until version exceeds 6.55.35. Vmid and Vlow are the * other two bytes of the "version" data.) * * So now we use the BF32_SET macros to get consistent behavior (the * ondisk LE encoding, since x86 currently rules the world) across * platforms, but the "get" behavior requires that we check each of the * bytes in the aforementioned former-bitfield for 0x00, and from there, * we can know which possible layout we're dealing with. (Only the two * that have been observed in the wild are illustrated above, but handlers * for all 4 positions of 0x00 are implemented. */ static inline void zfs_get_hdrmeta(const zfs_zstdhdr_t *blob, zfs_zstdmeta_t *res) { uint32_t raw = blob->raw_version_level; uint8_t findme = 0xff; int shift; for (shift = 0; shift < 4; shift++) { findme = BF32_GET(raw, 8*shift, 8); if (findme == 0) break; } switch (shift) { case 0: res->level = BF32_GET(raw, 24, 8); res->version = BSWAP_32(raw); res->version = BF32_GET(res->version, 8, 24); break; case 1: res->level = BF32_GET(raw, 0, 8); res->version = BSWAP_32(raw); res->version = BF32_GET(res->version, 0, 24); break; case 2: res->level = BF32_GET(raw, 24, 8); res->version = BF32_GET(raw, 0, 24); break; case 3: res->level = BF32_GET(raw, 0, 8); res->version = BF32_GET(raw, 8, 24); break; default: res->level = 0; res->version = 0; break; } } static inline uint8_t zfs_get_hdrlevel(const zfs_zstdhdr_t *blob) { uint8_t level = 0; zfs_zstdmeta_t res; zfs_get_hdrmeta(blob, &res); level = res.level; return (level); } static inline uint32_t zfs_get_hdrversion(const zfs_zstdhdr_t *blob) { uint32_t version = 0; zfs_zstdmeta_t res; zfs_get_hdrmeta(blob, &res); version = res.version; return (version); } static inline void zfs_set_hdrversion(zfs_zstdhdr_t *blob, uint32_t version) { /* cppcheck-suppress syntaxError */ BF32_SET(blob->raw_version_level, 0, 24, version); } static inline void zfs_set_hdrlevel(zfs_zstdhdr_t *blob, uint8_t level) { /* cppcheck-suppress syntaxError */ BF32_SET(blob->raw_version_level, 24, 8, level); } #ifdef __cplusplus } #endif #endif /* _ZFS_ZSTD_H */ diff --git a/lib/libzutil/os/linux/zutil_setproctitle.c b/lib/libzutil/os/linux/zutil_setproctitle.c index e63acceb4f8e..fc727e29ce2a 100644 --- a/lib/libzutil/os/linux/zutil_setproctitle.c +++ b/lib/libzutil/os/linux/zutil_setproctitle.c @@ -1,276 +1,277 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * Copyright © 2013 Guillem Jover * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include static struct { /* Original value. */ const char *arg0; /* Title space available. */ char *base, *end; /* Pointer to original nul character within base. */ char *nul; boolean_t warned; boolean_t reset; int error; } SPT; #define LIBBSD_IS_PATHNAME_SEPARATOR(c) ((c) == '/') #define SPT_MAXTITLE 255 extern const char *__progname; static const char * getprogname(void) { return (__progname); } static void setprogname(const char *progname) { size_t i; for (i = strlen(progname); i > 0; i--) { if (LIBBSD_IS_PATHNAME_SEPARATOR(progname[i - 1])) { __progname = progname + i; return; } } __progname = progname; } static inline size_t spt_min(size_t a, size_t b) { return ((a < b) ? a : b); } static int spt_copyenv(int envc, char *envp[]) { char **envcopy; char *eq; int envsize; int i, error = 0; if (environ != envp) return (0); /* * Make a copy of the old environ array of pointers, in case * clearenv() or setenv() is implemented to free the internal * environ array, because we will need to access the old environ * contents to make the new copy. */ envsize = (envc + 1) * sizeof (char *); envcopy = malloc(envsize); if (envcopy == NULL) return (errno); memcpy(envcopy, envp, envsize); environ = NULL; for (i = 0; envcopy[i]; i++) { eq = strchr(envcopy[i], '='); if (eq == NULL) continue; *eq = '\0'; if (setenv(envcopy[i], eq + 1, 1) < 0) error = errno; *eq = '='; if (error) { clearenv(); environ = envp; free(envcopy); return (error); } } /* * Dispose of the shallow copy, now that we've finished transfering * the old environment. */ free(envcopy); return (0); } static int spt_copyargs(int argc, char *argv[]) { char *tmp; int i; for (i = 1; i < argc || (i >= argc && argv[i]); i++) { if (argv[i] == NULL) continue; tmp = strdup(argv[i]); if (tmp == NULL) return (errno); argv[i] = tmp; } return (0); } void zfs_setproctitle_init(int argc, char *argv[], char *envp[]) { char *base, *end, *nul, *tmp; int i, envc, error; /* Try to make sure we got called with main() arguments. */ if (argc < 0) return; base = argv[0]; if (base == NULL) return; nul = base + strlen(base); end = nul + 1; for (i = 0; i < argc || (i >= argc && argv[i]); i++) { if (argv[i] == NULL || argv[i] != end) continue; end = argv[i] + strlen(argv[i]) + 1; } for (i = 0; envp[i]; i++) { if (envp[i] != end) continue; end = envp[i] + strlen(envp[i]) + 1; } envc = i; SPT.arg0 = strdup(argv[0]); if (SPT.arg0 == NULL) { SPT.error = errno; return; } tmp = strdup(getprogname()); if (tmp == NULL) { SPT.error = errno; return; } setprogname(tmp); error = spt_copyenv(envc, envp); if (error) { SPT.error = error; return; } error = spt_copyargs(argc, argv); if (error) { SPT.error = error; return; } SPT.nul = nul; SPT.base = base; SPT.end = end; } void zfs_setproctitle(const char *fmt, ...) { /* Use buffer in case argv[0] is passed. */ char buf[SPT_MAXTITLE + 1]; va_list ap; char *nul; int len; if (SPT.base == NULL) { if (!SPT.warned) { warnx("setproctitle not initialized, please" "call zfs_setproctitle_init()"); SPT.warned = B_TRUE; } return; } if (fmt) { if (fmt[0] == '-') { /* Skip program name prefix. */ fmt++; len = 0; } else { /* Print program name heading for grep. */ snprintf(buf, sizeof (buf), "%s: ", getprogname()); len = strlen(buf); } va_start(ap, fmt); len += vsnprintf(buf + len, sizeof (buf) - len, fmt, ap); va_end(ap); } else { len = snprintf(buf, sizeof (buf), "%s", SPT.arg0); } if (len <= 0) { SPT.error = errno; return; } if (!SPT.reset) { memset(SPT.base, 0, SPT.end - SPT.base); SPT.reset = B_TRUE; } else { memset(SPT.base, 0, spt_min(sizeof (buf), SPT.end - SPT.base)); } len = spt_min(len, spt_min(sizeof (buf), SPT.end - SPT.base) - 1); memcpy(SPT.base, buf, len); nul = SPT.base + len; if (nul < SPT.nul) { *SPT.nul = '.'; } else if (nul == SPT.nul && nul + 1 < SPT.end) { *SPT.nul = ' '; *++nul = '\0'; } } diff --git a/module/lua/setjmp/setjmp_mips.S b/module/lua/setjmp/setjmp_mips.S index 0084fbfa4bec..9f52b83b85c3 100644 --- a/module/lua/setjmp/setjmp_mips.S +++ b/module/lua/setjmp/setjmp_mips.S @@ -1,105 +1,106 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * Copyright (c) 2000, 2001, 2002, 2003, 2004, 2005, 2008, 2009 * The President and Fellows of Harvard College. * Copyright (c) 2017 MIPS Technologies, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include /* * setjmp and longjmp for MIPS. */ .text .set noreorder /* * int setjmp(jmp_buf jb); * * Save the current state so we can return again from the call later * if/when longjmp is called. (If the function that called setjmp * returns before longjmp is called, the results are undefined. We * only need to save registers, not the whole contents of the stack.) */ LEAF(setjmp) /* * jmp_buf is in a0. We need to save s0-s8, sp, gp, and ra in it. * Don't store more registers without adjusting machine/setjmp.h. */ REG_S sp, 0(a0) /* save registers */ REG_S ra, 1*SZREG(a0) REG_S gp, 2*SZREG(a0) REG_S s0, 3*SZREG(a0) REG_S s1, 4*SZREG(a0) REG_S s2, 5*SZREG(a0) REG_S s3, 6*SZREG(a0) REG_S s4, 7*SZREG(a0) REG_S s5, 8*SZREG(a0) REG_S s6, 9*SZREG(a0) REG_S s7, 10*SZREG(a0) REG_S s8, 11*SZREG(a0) jr ra /* done */ move v0, zero /* return 0 (in delay slot) */ END(setjmp) /* * void longjmp(jmp_buf jb, int code); */ LEAF(longjmp) /* * jmp_buf is in a0. Return code is in a1. * We need to restore s0-s8, sp, gp, and ra from the jmp_buf. * The return code is forced to 1 if 0 is passed in. */ sltiu t0, a1, 1 /* set t0 to 1 if return code is 0... otherwise 0 */ addu a1, a1, t0 /* update the return code */ REG_L sp, 0(a0) /* restore registers */ REG_L ra, 1*SZREG(a0) REG_L gp, 2*SZREG(a0) REG_L s0, 3*SZREG(a0) REG_L s1, 4*SZREG(a0) REG_L s2, 5*SZREG(a0) REG_L s3, 6*SZREG(a0) REG_L s4, 7*SZREG(a0) REG_L s5, 8*SZREG(a0) REG_L s6, 9*SZREG(a0) REG_L s7, 10*SZREG(a0) REG_L s8, 11*SZREG(a0) jr ra /* return, to where setjmp was called from */ move v0, a1 /* set return value */ END(longjmp) #ifdef __ELF__ .section .note.GNU-stack,"",%progbits #endif diff --git a/module/lua/setjmp/setjmp_sparc64.S b/module/lua/setjmp/setjmp_sparc64.S index e1099643de92..08abf4b28311 100644 --- a/module/lua/setjmp/setjmp_sparc64.S +++ b/module/lua/setjmp/setjmp_sparc64.S @@ -1,105 +1,106 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This software was developed by the Computer Systems Engineering group * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and * contributed to Berkeley. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $Header: _setjmp.s,v 1.1 91/07/06 16:45:53 torek Exp */ #if defined(LIBC_SCCS) && !defined(lint) #if 0 .asciz "@(#)_setjmp.s 8.1 (Berkeley) 6/4/93" #else RCSID("$NetBSD: _setjmp.S,v 1.4 1998/10/08 02:27:59 eeh Exp $") #endif #endif /* LIBC_SCCS and not lint */ #define _JB_FP 0x0 #define _JB_PC 0x8 #define _JB_SP 0x10 .register %g2,#ignore .register %g3,#ignore #define ENTRY(x) \ .text ; \ .balign 32 ; \ .globl x ; \ .type x,@function ; \ x: #define END(x) \ .size x, . - x /* * C library -- setjmp, longjmp * * longjmp(a,v) * will generate a "return(v?v:1)" from * the last call to * setjmp(a) * by restoring the previous context. */ ENTRY(setjmp) stx %sp, [%o0 + _JB_SP] stx %o7, [%o0 + _JB_PC] stx %fp, [%o0 + _JB_FP] retl clr %o0 END(setjmp) ENTRY(longjmp) mov 1, %g1 movrnz %o1, %o1, %g1 mov %o0, %g2 ldx [%g2 + _JB_FP], %g3 1: cmp %fp, %g3 bl,a 1b restore be,a 2f ldx [%g2 + _JB_SP], %o0 .Lbotch: illtrap 2: cmp %o0, %sp bge,a 3f mov %o0, %sp b,a .Lbotch nop 3: ldx [%g2 + _JB_PC], %o7 retl mov %g1, %o0 END(longjmp) #ifdef __ELF__ .section .note.GNU-stack,"",%progbits #endif diff --git a/module/zstd/include/aarch64_compat.h b/module/zstd/include/aarch64_compat.h index 088517d3d23b..9500a832b81c 100644 --- a/module/zstd/include/aarch64_compat.h +++ b/module/zstd/include/aarch64_compat.h @@ -1,37 +1,38 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2018-2020, Sebastian Gottschall */ #ifdef _KERNEL #undef __aarch64__ #endif diff --git a/module/zstd/include/limits.h b/module/zstd/include/limits.h index 3bf5b67765ae..45b57f6e9f24 100644 --- a/module/zstd/include/limits.h +++ b/module/zstd/include/limits.h @@ -1,63 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2014-2019, Allan Jude * Copyright (c) 2020, Brian Behlendorf * Copyright (c) 2020, Michael Niewöhner */ #ifndef _ZSTD_LIMITS_H #define _ZSTD_LIMITS_H #ifdef __cplusplus extern "C" { #endif #ifdef _KERNEL #if defined(__FreeBSD__) #include #elif defined(__linux__) #include #include #else #error "Unsupported platform" #endif #else /* !_KERNEL */ #include_next #endif /* _KERNEL */ #ifdef __cplusplus } #endif #endif /* _ZSTD_LIMITS_H */ diff --git a/module/zstd/include/stddef.h b/module/zstd/include/stddef.h index 3f46fb8b033e..24c250fa5d27 100644 --- a/module/zstd/include/stddef.h +++ b/module/zstd/include/stddef.h @@ -1,62 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2014-2019, Allan Jude * Copyright (c) 2020, Brian Behlendorf * Copyright (c) 2020, Michael Niewöhner */ #ifndef _ZSTD_STDDEF_H #define _ZSTD_STDDEF_H #ifdef __cplusplus extern "C" { #endif #ifdef _KERNEL #if defined(__FreeBSD__) #include #elif defined(__linux__) #include #else #error "Unsupported platform" #endif #else /* !_KERNEL */ #include_next #endif /* _KERNEL */ #ifdef __cplusplus } #endif #endif /* _ZSTD_STDDEF_H */ diff --git a/module/zstd/include/stdint.h b/module/zstd/include/stdint.h index 2d98a556c23e..31e11f578687 100644 --- a/module/zstd/include/stdint.h +++ b/module/zstd/include/stdint.h @@ -1,62 +1,63 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2014-2019, Allan Jude * Copyright (c) 2020, Brian Behlendorf * Copyright (c) 2020, Michael Niewöhner */ #ifndef _ZSTD_STDINT_H #define _ZSTD_STDINT_H #ifdef __cplusplus extern "C" { #endif #ifdef _KERNEL #if defined(__FreeBSD__) #include #elif defined(__linux__) #include #else #error "Unsupported platform" #endif #else /* !_KERNEL */ #include_next #endif /* _KERNEL */ #ifdef __cplusplus } #endif #endif /* _ZSTD_STDINT_H */ diff --git a/module/zstd/include/stdio.h b/module/zstd/include/stdio.h index 5a7c6ec69916..50b698fa2072 100644 --- a/module/zstd/include/stdio.h +++ b/module/zstd/include/stdio.h @@ -1,54 +1,55 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2014-2019, Allan Jude * Copyright (c) 2020, Brian Behlendorf * Copyright (c) 2020, Michael Niewöhner */ #ifndef _ZSTD_STDIO_H #define _ZSTD_STDIO_H #ifdef __cplusplus extern "C" { #endif #ifndef _KERNEL #include_next #endif /* _KERNEL */ #ifdef __cplusplus } #endif #endif /* _ZSTD_STDIO_H */ diff --git a/module/zstd/include/stdlib.h b/module/zstd/include/stdlib.h index c341a0c84884..cc66184fa2a4 100644 --- a/module/zstd/include/stdlib.h +++ b/module/zstd/include/stdlib.h @@ -1,58 +1,59 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2014-2019, Allan Jude * Copyright (c) 2020, Brian Behlendorf * Copyright (c) 2020, Michael Niewöhner */ #ifndef _ZSTD_STDLIB_H #define _ZSTD_STDLIB_H #ifdef __cplusplus extern "C" { #endif #undef GCC_VERSION /* * Define calloc, malloc, free to make building work. They are never really used * in zstdlib.c since allocation is done in zstd.c. */ #define calloc(n, sz) NULL #define malloc(sz) NULL #define free(ptr) #ifdef __cplusplus } #endif #endif /* _ZSTD_STDLIB_H */ diff --git a/module/zstd/include/string.h b/module/zstd/include/string.h index 7474e7f1af0f..b058bafc1ad5 100644 --- a/module/zstd/include/string.h +++ b/module/zstd/include/string.h @@ -1,63 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2014-2019, Allan Jude * Copyright (c) 2020, Brian Behlendorf * Copyright (c) 2020, Michael Niewöhner */ #ifndef _ZSTD_STRING_H #define _ZSTD_STRING_H #ifdef __cplusplus extern "C" { #endif #ifdef _KERNEL #if defined(__FreeBSD__) #include /* u_int, u_char */ #include /* memcpy, memset */ #elif defined(__linux__) #include /* memcpy, memset */ #else #error "Unsupported platform" #endif #else /* !_KERNEL */ #include_next #endif /* _KERNEL */ #ifdef __cplusplus } #endif #endif /* _ZSTD_STRING_H */ diff --git a/module/zstd/include/zstd_compat_wrapper.h b/module/zstd/include/zstd_compat_wrapper.h index 4e6561f31a68..51e67ab6b8ad 100644 --- a/module/zstd/include/zstd_compat_wrapper.h +++ b/module/zstd/include/zstd_compat_wrapper.h @@ -1,422 +1,423 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2020, Sebastian Gottschall */ /* * This wrapper fixes a problem, in case the ZFS filesystem driver, is compiled * statically into the kernel. * This will cause a symbol collision with the older in-kernel zstd library. * * On update, truncate this file at the scissor line, rebuild the module, * and make gen-zstd-symbols. */ #define MEM_MODULE #define XXH_NAMESPACE ZSTD_ #define XXH_PRIVATE_API #define XXH_INLINE_ALL #define ZSTD_LEGACY_SUPPORT 0 #define ZSTD_LIB_DICTBUILDER 0 #define ZSTD_LIB_DEPRECATED 0 #define ZSTD_NOBENCH #define DEBUGLEVEL 0 #ifdef _KERNEL #define ZSTD_DEPS_ASSERT #endif /* -- >8 -- */ /* lib/common/entropy_common.o: */ #define FSE_getErrorName zfs_FSE_getErrorName #define FSE_isError zfs_FSE_isError #define FSE_readNCount zfs_FSE_readNCount #define FSE_versionNumber zfs_FSE_versionNumber #define HUF_getErrorName zfs_HUF_getErrorName #define HUF_isError zfs_HUF_isError #define HUF_readStats zfs_HUF_readStats /* lib/common/error_private.o: */ #define ERR_getErrorString zfs_ERR_getErrorString /* lib/common/fse_decompress.o: */ #define FSE_buildDTable_raw zfs_FSE_buildDTable_raw #define FSE_buildDTable_rle zfs_FSE_buildDTable_rle #define FSE_buildDTable zfs_FSE_buildDTable #define FSE_decompress_usingDTable zfs_FSE_decompress_usingDTable #define FSE_decompress_wksp zfs_FSE_decompress_wksp #define FSE_decompress zfs_FSE_decompress /* lib/common/pool.o: */ #define POOL_add zfs_POOL_add #define POOL_create_advanced zfs_POOL_create_advanced #define POOL_create zfs_POOL_create #define POOL_free zfs_POOL_free #define POOL_resize zfs_POOL_resize #define POOL_sizeof zfs_POOL_sizeof #define POOL_tryAdd zfs_POOL_tryAdd /* lib/common/zstd_common.o: */ #define ZSTD_calloc zfs_ZSTD_calloc #define ZSTD_free zfs_ZSTD_free #define ZSTD_getErrorCode zfs_ZSTD_getErrorCode #define ZSTD_getErrorName zfs_ZSTD_getErrorName #define ZSTD_getErrorString zfs_ZSTD_getErrorString #define ZSTD_isError zfs_ZSTD_isError #define ZSTD_malloc zfs_ZSTD_malloc #define ZSTD_versionNumber zfs_ZSTD_versionNumber #define ZSTD_versionString zfs_ZSTD_versionString /* lib/compress/fse_compress.o: */ #define FSE_buildCTable_raw zfs_FSE_buildCTable_raw #define FSE_buildCTable_rle zfs_FSE_buildCTable_rle #define FSE_buildCTable_wksp zfs_FSE_buildCTable_wksp #define FSE_buildCTable zfs_FSE_buildCTable #define FSE_compress2 zfs_FSE_compress2 #define FSE_compressBound zfs_FSE_compressBound #define FSE_compress_usingCTable zfs_FSE_compress_usingCTable #define FSE_compress_wksp zfs_FSE_compress_wksp #define FSE_compress zfs_FSE_compress #define FSE_createCTable zfs_FSE_createCTable #define FSE_freeCTable zfs_FSE_freeCTable #define FSE_NCountWriteBound zfs_FSE_NCountWriteBound #define FSE_normalizeCount zfs_FSE_normalizeCount #define FSE_optimalTableLog_internal zfs_FSE_optimalTableLog_internal #define FSE_optimalTableLog zfs_FSE_optimalTableLog #define FSE_writeNCount zfs_FSE_writeNCount /* lib/compress/hist.o: */ #define HIST_countFast_wksp zfs_HIST_countFast_wksp #define HIST_countFast zfs_HIST_countFast #define HIST_count_simple zfs_HIST_count_simple #define HIST_count_wksp zfs_HIST_count_wksp #define HIST_count zfs_HIST_count #define HIST_isError zfs_HIST_isError /* lib/compress/huf_compress.o: */ #define HUF_buildCTable_wksp zfs_HUF_buildCTable_wksp #define HUF_buildCTable zfs_HUF_buildCTable #define HUF_compress1X_repeat zfs_HUF_compress1X_repeat #define HUF_compress1X_usingCTable zfs_HUF_compress1X_usingCTable #define HUF_compress1X_wksp zfs_HUF_compress1X_wksp #define HUF_compress1X zfs_HUF_compress1X #define HUF_compress2 zfs_HUF_compress2 #define HUF_compress4X_repeat zfs_HUF_compress4X_repeat #define HUF_compress4X_usingCTable zfs_HUF_compress4X_usingCTable #define HUF_compress4X_wksp zfs_HUF_compress4X_wksp #define HUF_compressBound zfs_HUF_compressBound #define HUF_compress zfs_HUF_compress #define HUF_estimateCompressedSize zfs_HUF_estimateCompressedSize #define HUF_getNbBits zfs_HUF_getNbBits #define HUF_optimalTableLog zfs_HUF_optimalTableLog #define HUF_readCTable zfs_HUF_readCTable #define HUF_validateCTable zfs_HUF_validateCTable #define HUF_writeCTable zfs_HUF_writeCTable /* lib/compress/zstd_compress_literals.o: */ #define ZSTD_compressLiterals zfs_ZSTD_compressLiterals #define ZSTD_compressRleLiteralsBlock zfs_ZSTD_compressRleLiteralsBlock #define ZSTD_noCompressLiterals zfs_ZSTD_noCompressLiterals /* lib/compress/zstd_compress_sequences.o: */ #define ZSTD_buildCTable zfs_ZSTD_buildCTable #define ZSTD_crossEntropyCost zfs_ZSTD_crossEntropyCost #define ZSTD_encodeSequences zfs_ZSTD_encodeSequences #define ZSTD_fseBitCost zfs_ZSTD_fseBitCost #define ZSTD_selectEncodingType zfs_ZSTD_selectEncodingType /* lib/compress/zstd_compress_superblock.o: */ #define ZSTD_compressSuperBlock zfs_ZSTD_compressSuperBlock /* lib/compress/zstd_compress.o: */ #define ZSTD_adjustCParams zfs_ZSTD_adjustCParams #define ZSTD_CCtx_getParameter zfs_ZSTD_CCtx_getParameter #define ZSTD_CCtx_loadDictionary_advanced zfs_ZSTD_CCtx_loadDictionary_advanced #define ZSTD_CCtx_loadDictionary_byReference zfs_ZSTD_CCtx_loadDictionary_byReference #define ZSTD_CCtx_loadDictionary zfs_ZSTD_CCtx_loadDictionary #define ZSTD_CCtxParams_getParameter zfs_ZSTD_CCtxParams_getParameter #define ZSTD_CCtxParams_init_advanced zfs_ZSTD_CCtxParams_init_advanced #define ZSTD_CCtxParams_init zfs_ZSTD_CCtxParams_init #define ZSTD_CCtxParams_reset zfs_ZSTD_CCtxParams_reset #define ZSTD_CCtxParams_setParameter zfs_ZSTD_CCtxParams_setParameter #define ZSTD_CCtx_refCDict zfs_ZSTD_CCtx_refCDict #define ZSTD_CCtx_refPrefix_advanced zfs_ZSTD_CCtx_refPrefix_advanced #define ZSTD_CCtx_refPrefix zfs_ZSTD_CCtx_refPrefix #define ZSTD_CCtx_reset zfs_ZSTD_CCtx_reset #define ZSTD_CCtx_setParametersUsingCCtxParams zfs_ZSTD_CCtx_setParametersUsingCCtxParams #define ZSTD_CCtx_setParameter zfs_ZSTD_CCtx_setParameter #define ZSTD_CCtx_setPledgedSrcSize zfs_ZSTD_CCtx_setPledgedSrcSize #define ZSTD_checkCParams zfs_ZSTD_checkCParams #define ZSTD_compress2 zfs_ZSTD_compress2 #define ZSTD_compress_advanced_internal zfs_ZSTD_compress_advanced_internal #define ZSTD_compress_advanced zfs_ZSTD_compress_advanced #define ZSTD_compressBegin_advanced_internal zfs_ZSTD_compressBegin_advanced_internal #define ZSTD_compressBegin_advanced zfs_ZSTD_compressBegin_advanced #define ZSTD_compressBegin_usingCDict_advanced zfs_ZSTD_compressBegin_usingCDict_advanced #define ZSTD_compressBegin_usingCDict zfs_ZSTD_compressBegin_usingCDict #define ZSTD_compressBegin_usingDict zfs_ZSTD_compressBegin_usingDict #define ZSTD_compressBegin zfs_ZSTD_compressBegin #define ZSTD_compressBlock zfs_ZSTD_compressBlock #define ZSTD_compressBound zfs_ZSTD_compressBound #define ZSTD_compressCCtx zfs_ZSTD_compressCCtx #define ZSTD_compressContinue zfs_ZSTD_compressContinue #define ZSTD_compressEnd zfs_ZSTD_compressEnd #define ZSTD_compressStream2_simpleArgs zfs_ZSTD_compressStream2_simpleArgs #define ZSTD_compressStream2 zfs_ZSTD_compressStream2 #define ZSTD_compressStream zfs_ZSTD_compressStream #define ZSTD_compress_usingCDict_advanced zfs_ZSTD_compress_usingCDict_advanced #define ZSTD_compress_usingCDict zfs_ZSTD_compress_usingCDict #define ZSTD_compress_usingDict zfs_ZSTD_compress_usingDict #define ZSTD_compress zfs_ZSTD_compress #define ZSTD_copyCCtx zfs_ZSTD_copyCCtx #define ZSTD_cParam_getBounds zfs_ZSTD_cParam_getBounds #define ZSTD_createCCtx_advanced zfs_ZSTD_createCCtx_advanced #define ZSTD_createCCtxParams zfs_ZSTD_createCCtxParams #define ZSTD_createCCtx zfs_ZSTD_createCCtx #define ZSTD_createCDict_advanced zfs_ZSTD_createCDict_advanced #define ZSTD_createCDict_byReference zfs_ZSTD_createCDict_byReference #define ZSTD_createCDict zfs_ZSTD_createCDict #define ZSTD_createCStream_advanced zfs_ZSTD_createCStream_advanced #define ZSTD_createCStream zfs_ZSTD_createCStream #define ZSTD_CStreamInSize zfs_ZSTD_CStreamInSize #define ZSTD_CStreamOutSize zfs_ZSTD_CStreamOutSize #define ZSTD_cycleLog zfs_ZSTD_cycleLog #define ZSTD_endStream zfs_ZSTD_endStream #define ZSTD_estimateCCtxSize_usingCCtxParams zfs_ZSTD_estimateCCtxSize_usingCCtxParams #define ZSTD_estimateCCtxSize_usingCParams zfs_ZSTD_estimateCCtxSize_usingCParams #define ZSTD_estimateCCtxSize zfs_ZSTD_estimateCCtxSize #define ZSTD_estimateCDictSize_advanced zfs_ZSTD_estimateCDictSize_advanced #define ZSTD_estimateCDictSize zfs_ZSTD_estimateCDictSize #define ZSTD_estimateCStreamSize_usingCCtxParams zfs_ZSTD_estimateCStreamSize_usingCCtxParams #define ZSTD_estimateCStreamSize_usingCParams zfs_ZSTD_estimateCStreamSize_usingCParams #define ZSTD_estimateCStreamSize zfs_ZSTD_estimateCStreamSize #define ZSTD_flushStream zfs_ZSTD_flushStream #define ZSTD_freeCCtxParams zfs_ZSTD_freeCCtxParams #define ZSTD_freeCCtx zfs_ZSTD_freeCCtx #define ZSTD_freeCDict zfs_ZSTD_freeCDict #define ZSTD_freeCStream zfs_ZSTD_freeCStream #define ZSTD_getBlockSize zfs_ZSTD_getBlockSize #define ZSTD_getCParamsFromCCtxParams zfs_ZSTD_getCParamsFromCCtxParams #define ZSTD_getCParamsFromCDict zfs_ZSTD_getCParamsFromCDict #define ZSTD_getCParams zfs_ZSTD_getCParams #define ZSTD_getFrameProgression zfs_ZSTD_getFrameProgression #define ZSTD_getParams zfs_ZSTD_getParams #define ZSTD_getSeqStore zfs_ZSTD_getSeqStore #define ZSTD_getSequences zfs_ZSTD_getSequences #define ZSTD_initCStream_advanced zfs_ZSTD_initCStream_advanced #define ZSTD_initCStream_internal zfs_ZSTD_initCStream_internal #define ZSTD_initCStream_srcSize zfs_ZSTD_initCStream_srcSize #define ZSTD_initCStream_usingCDict_advanced zfs_ZSTD_initCStream_usingCDict_advanced #define ZSTD_initCStream_usingCDict zfs_ZSTD_initCStream_usingCDict #define ZSTD_initCStream_usingDict zfs_ZSTD_initCStream_usingDict #define ZSTD_initCStream zfs_ZSTD_initCStream #define ZSTD_initStaticCCtx zfs_ZSTD_initStaticCCtx #define ZSTD_initStaticCDict zfs_ZSTD_initStaticCDict #define ZSTD_initStaticCStream zfs_ZSTD_initStaticCStream #define ZSTD_invalidateRepCodes zfs_ZSTD_invalidateRepCodes #define ZSTD_loadCEntropy zfs_ZSTD_loadCEntropy #define ZSTD_maxCLevel zfs_ZSTD_maxCLevel #define ZSTD_minCLevel zfs_ZSTD_minCLevel #define ZSTD_referenceExternalSequences zfs_ZSTD_referenceExternalSequences #define ZSTD_reset_compressedBlockState zfs_ZSTD_reset_compressedBlockState #define ZSTD_resetCStream zfs_ZSTD_resetCStream #define ZSTD_resetSeqStore zfs_ZSTD_resetSeqStore #define ZSTD_selectBlockCompressor zfs_ZSTD_selectBlockCompressor #define ZSTD_seqToCodes zfs_ZSTD_seqToCodes #define ZSTD_sizeof_CCtx zfs_ZSTD_sizeof_CCtx #define ZSTD_sizeof_CDict zfs_ZSTD_sizeof_CDict #define ZSTD_sizeof_CStream zfs_ZSTD_sizeof_CStream #define ZSTD_toFlushNow zfs_ZSTD_toFlushNow #define ZSTD_writeLastEmptyBlock zfs_ZSTD_writeLastEmptyBlock /* lib/compress/zstd_double_fast.o: */ #define ZSTD_compressBlock_doubleFast_dictMatchState zfs_ZSTD_compressBlock_doubleFast_dictMatchState #define ZSTD_compressBlock_doubleFast_extDict zfs_ZSTD_compressBlock_doubleFast_extDict #define ZSTD_compressBlock_doubleFast zfs_ZSTD_compressBlock_doubleFast #define ZSTD_fillDoubleHashTable zfs_ZSTD_fillDoubleHashTable /* lib/compress/zstd_fast.o: */ #define ZSTD_compressBlock_fast_dictMatchState zfs_ZSTD_compressBlock_fast_dictMatchState #define ZSTD_compressBlock_fast_extDict zfs_ZSTD_compressBlock_fast_extDict #define ZSTD_compressBlock_fast zfs_ZSTD_compressBlock_fast #define ZSTD_fillHashTable zfs_ZSTD_fillHashTable /* lib/compress/zstd_lazy.o: */ #define ZSTD_compressBlock_btlazy2_dictMatchState zfs_ZSTD_compressBlock_btlazy2_dictMatchState #define ZSTD_compressBlock_btlazy2_extDict zfs_ZSTD_compressBlock_btlazy2_extDict #define ZSTD_compressBlock_btlazy2 zfs_ZSTD_compressBlock_btlazy2 #define ZSTD_compressBlock_greedy_dictMatchState zfs_ZSTD_compressBlock_greedy_dictMatchState #define ZSTD_compressBlock_greedy_extDict zfs_ZSTD_compressBlock_greedy_extDict #define ZSTD_compressBlock_greedy zfs_ZSTD_compressBlock_greedy #define ZSTD_compressBlock_lazy2_dictMatchState zfs_ZSTD_compressBlock_lazy2_dictMatchState #define ZSTD_compressBlock_lazy2_extDict zfs_ZSTD_compressBlock_lazy2_extDict #define ZSTD_compressBlock_lazy2 zfs_ZSTD_compressBlock_lazy2 #define ZSTD_compressBlock_lazy_dictMatchState zfs_ZSTD_compressBlock_lazy_dictMatchState #define ZSTD_compressBlock_lazy_extDict zfs_ZSTD_compressBlock_lazy_extDict #define ZSTD_compressBlock_lazy zfs_ZSTD_compressBlock_lazy #define ZSTD_insertAndFindFirstIndex zfs_ZSTD_insertAndFindFirstIndex /* lib/compress/zstd_ldm.o: */ #define ZSTD_ldm_adjustParameters zfs_ZSTD_ldm_adjustParameters #define ZSTD_ldm_blockCompress zfs_ZSTD_ldm_blockCompress #define ZSTD_ldm_fillHashTable zfs_ZSTD_ldm_fillHashTable #define ZSTD_ldm_generateSequences zfs_ZSTD_ldm_generateSequences #define ZSTD_ldm_getMaxNbSeq zfs_ZSTD_ldm_getMaxNbSeq #define ZSTD_ldm_getTableSize zfs_ZSTD_ldm_getTableSize #define ZSTD_ldm_skipSequences zfs_ZSTD_ldm_skipSequences /* lib/compress/zstd_opt.o: */ #define ZSTD_compressBlock_btopt_dictMatchState zfs_ZSTD_compressBlock_btopt_dictMatchState #define ZSTD_compressBlock_btopt_extDict zfs_ZSTD_compressBlock_btopt_extDict #define ZSTD_compressBlock_btopt zfs_ZSTD_compressBlock_btopt #define ZSTD_compressBlock_btultra2 zfs_ZSTD_compressBlock_btultra2 #define ZSTD_compressBlock_btultra_dictMatchState zfs_ZSTD_compressBlock_btultra_dictMatchState #define ZSTD_compressBlock_btultra_extDict zfs_ZSTD_compressBlock_btultra_extDict #define ZSTD_compressBlock_btultra zfs_ZSTD_compressBlock_btultra #define ZSTD_updateTree zfs_ZSTD_updateTree /* lib/decompress/huf_decompress.o: */ #define HUF_decompress1X1_DCtx_wksp_bmi2 zfs_HUF_decompress1X1_DCtx_wksp_bmi2 #define HUF_decompress1X1_DCtx_wksp zfs_HUF_decompress1X1_DCtx_wksp #define HUF_decompress1X1_DCtx zfs_HUF_decompress1X1_DCtx #define HUF_decompress1X1_usingDTable zfs_HUF_decompress1X1_usingDTable #define HUF_decompress1X1 zfs_HUF_decompress1X1 #define HUF_decompress1X2_DCtx_wksp zfs_HUF_decompress1X2_DCtx_wksp #define HUF_decompress1X2_DCtx zfs_HUF_decompress1X2_DCtx #define HUF_decompress1X2_usingDTable zfs_HUF_decompress1X2_usingDTable #define HUF_decompress1X2 zfs_HUF_decompress1X2 #define HUF_decompress1X_DCtx_wksp zfs_HUF_decompress1X_DCtx_wksp #define HUF_decompress1X_DCtx zfs_HUF_decompress1X_DCtx #define HUF_decompress1X_usingDTable_bmi2 zfs_HUF_decompress1X_usingDTable_bmi2 #define HUF_decompress1X_usingDTable zfs_HUF_decompress1X_usingDTable #define HUF_decompress4X1_DCtx_wksp zfs_HUF_decompress4X1_DCtx_wksp #define HUF_decompress4X1_DCtx zfs_HUF_decompress4X1_DCtx #define HUF_decompress4X1_usingDTable zfs_HUF_decompress4X1_usingDTable #define HUF_decompress4X1 zfs_HUF_decompress4X1 #define HUF_decompress4X2_DCtx_wksp zfs_HUF_decompress4X2_DCtx_wksp #define HUF_decompress4X2_DCtx zfs_HUF_decompress4X2_DCtx #define HUF_decompress4X2_usingDTable zfs_HUF_decompress4X2_usingDTable #define HUF_decompress4X2 zfs_HUF_decompress4X2 #define HUF_decompress4X_DCtx zfs_HUF_decompress4X_DCtx #define HUF_decompress4X_hufOnly_wksp_bmi2 zfs_HUF_decompress4X_hufOnly_wksp_bmi2 #define HUF_decompress4X_hufOnly_wksp zfs_HUF_decompress4X_hufOnly_wksp #define HUF_decompress4X_hufOnly zfs_HUF_decompress4X_hufOnly #define HUF_decompress4X_usingDTable_bmi2 zfs_HUF_decompress4X_usingDTable_bmi2 #define HUF_decompress4X_usingDTable zfs_HUF_decompress4X_usingDTable #define HUF_decompress zfs_HUF_decompress #define HUF_readDTableX1_wksp zfs_HUF_readDTableX1_wksp #define HUF_readDTableX1 zfs_HUF_readDTableX1 #define HUF_readDTableX2_wksp zfs_HUF_readDTableX2_wksp #define HUF_readDTableX2 zfs_HUF_readDTableX2 #define HUF_selectDecoder zfs_HUF_selectDecoder /* lib/decompress/zstd_ddict.o: */ #define ZSTD_copyDDictParameters zfs_ZSTD_copyDDictParameters #define ZSTD_createDDict_advanced zfs_ZSTD_createDDict_advanced #define ZSTD_createDDict_byReference zfs_ZSTD_createDDict_byReference #define ZSTD_createDDict zfs_ZSTD_createDDict #define ZSTD_DDict_dictContent zfs_ZSTD_DDict_dictContent #define ZSTD_DDict_dictSize zfs_ZSTD_DDict_dictSize #define ZSTD_estimateDDictSize zfs_ZSTD_estimateDDictSize #define ZSTD_freeDDict zfs_ZSTD_freeDDict #define ZSTD_getDictID_fromDDict zfs_ZSTD_getDictID_fromDDict #define ZSTD_initStaticDDict zfs_ZSTD_initStaticDDict #define ZSTD_sizeof_DDict zfs_ZSTD_sizeof_DDict /* lib/decompress/zstd_decompress.o: */ #define ZSTD_copyDCtx zfs_ZSTD_copyDCtx #define ZSTD_createDCtx_advanced zfs_ZSTD_createDCtx_advanced #define ZSTD_createDCtx zfs_ZSTD_createDCtx #define ZSTD_createDStream_advanced zfs_ZSTD_createDStream_advanced #define ZSTD_createDStream zfs_ZSTD_createDStream #define ZSTD_DCtx_loadDictionary_advanced zfs_ZSTD_DCtx_loadDictionary_advanced #define ZSTD_DCtx_loadDictionary_byReference zfs_ZSTD_DCtx_loadDictionary_byReference #define ZSTD_DCtx_loadDictionary zfs_ZSTD_DCtx_loadDictionary #define ZSTD_DCtx_refDDict zfs_ZSTD_DCtx_refDDict #define ZSTD_DCtx_refPrefix_advanced zfs_ZSTD_DCtx_refPrefix_advanced #define ZSTD_DCtx_refPrefix zfs_ZSTD_DCtx_refPrefix #define ZSTD_DCtx_reset zfs_ZSTD_DCtx_reset #define ZSTD_DCtx_setFormat zfs_ZSTD_DCtx_setFormat #define ZSTD_DCtx_setMaxWindowSize zfs_ZSTD_DCtx_setMaxWindowSize #define ZSTD_DCtx_setParameter zfs_ZSTD_DCtx_setParameter #define ZSTD_decodingBufferSize_min zfs_ZSTD_decodingBufferSize_min #define ZSTD_decompressBegin_usingDDict zfs_ZSTD_decompressBegin_usingDDict #define ZSTD_decompressBegin_usingDict zfs_ZSTD_decompressBegin_usingDict #define ZSTD_decompressBegin zfs_ZSTD_decompressBegin #define ZSTD_decompressBound zfs_ZSTD_decompressBound #define ZSTD_decompressContinue zfs_ZSTD_decompressContinue #define ZSTD_decompressDCtx zfs_ZSTD_decompressDCtx #define ZSTD_decompressStream_simpleArgs zfs_ZSTD_decompressStream_simpleArgs #define ZSTD_decompressStream zfs_ZSTD_decompressStream #define ZSTD_decompress_usingDDict zfs_ZSTD_decompress_usingDDict #define ZSTD_decompress_usingDict zfs_ZSTD_decompress_usingDict #define ZSTD_decompress zfs_ZSTD_decompress #define ZSTD_dParam_getBounds zfs_ZSTD_dParam_getBounds #define ZSTD_DStreamInSize zfs_ZSTD_DStreamInSize #define ZSTD_DStreamOutSize zfs_ZSTD_DStreamOutSize #define ZSTD_estimateDCtxSize zfs_ZSTD_estimateDCtxSize #define ZSTD_estimateDStreamSize_fromFrame zfs_ZSTD_estimateDStreamSize_fromFrame #define ZSTD_estimateDStreamSize zfs_ZSTD_estimateDStreamSize #define ZSTD_findDecompressedSize zfs_ZSTD_findDecompressedSize #define ZSTD_findFrameCompressedSize zfs_ZSTD_findFrameCompressedSize #define ZSTD_frameHeaderSize zfs_ZSTD_frameHeaderSize #define ZSTD_freeDCtx zfs_ZSTD_freeDCtx #define ZSTD_freeDStream zfs_ZSTD_freeDStream #define ZSTD_getDecompressedSize zfs_ZSTD_getDecompressedSize #define ZSTD_getDictID_fromDict zfs_ZSTD_getDictID_fromDict #define ZSTD_getDictID_fromFrame zfs_ZSTD_getDictID_fromFrame #define ZSTD_getFrameContentSize zfs_ZSTD_getFrameContentSize #define ZSTD_getFrameHeader_advanced zfs_ZSTD_getFrameHeader_advanced #define ZSTD_getFrameHeader zfs_ZSTD_getFrameHeader #define ZSTD_initDStream_usingDDict zfs_ZSTD_initDStream_usingDDict #define ZSTD_initDStream_usingDict zfs_ZSTD_initDStream_usingDict #define ZSTD_initDStream zfs_ZSTD_initDStream #define ZSTD_initStaticDCtx zfs_ZSTD_initStaticDCtx #define ZSTD_initStaticDStream zfs_ZSTD_initStaticDStream #define ZSTD_insertBlock zfs_ZSTD_insertBlock #define ZSTD_isFrame zfs_ZSTD_isFrame #define ZSTD_loadDEntropy zfs_ZSTD_loadDEntropy #define ZSTD_nextInputType zfs_ZSTD_nextInputType #define ZSTD_nextSrcSizeToDecompress zfs_ZSTD_nextSrcSizeToDecompress #define ZSTD_resetDStream zfs_ZSTD_resetDStream #define ZSTD_sizeof_DCtx zfs_ZSTD_sizeof_DCtx #define ZSTD_sizeof_DStream zfs_ZSTD_sizeof_DStream /* lib/decompress/zstd_decompress_block.o: */ #define ZSTD_buildFSETable zfs_ZSTD_buildFSETable #define ZSTD_checkContinuity zfs_ZSTD_checkContinuity #define ZSTD_decodeLiteralsBlock zfs_ZSTD_decodeLiteralsBlock #define ZSTD_decodeSeqHeaders zfs_ZSTD_decodeSeqHeaders #define ZSTD_decompressBlock_internal zfs_ZSTD_decompressBlock_internal #define ZSTD_decompressBlock zfs_ZSTD_decompressBlock #define ZSTD_getcBlockSize zfs_ZSTD_getcBlockSize diff --git a/module/zstd/zfs_zstd.c b/module/zstd/zfs_zstd.c index e113962f65b6..b42066fdb7c3 100644 --- a/module/zstd/zfs_zstd.c +++ b/module/zstd/zfs_zstd.c @@ -1,921 +1,922 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2016-2018, Klara Inc. * Copyright (c) 2016-2018, Allan Jude * Copyright (c) 2018-2020, Sebastian Gottschall * Copyright (c) 2019-2020, Michael Niewöhner * Copyright (c) 2020, The FreeBSD Foundation [1] * * [1] Portions of this software were developed by Allan Jude * under sponsorship from the FreeBSD Foundation. */ #include #include #include #include #include #include #define ZSTD_STATIC_LINKING_ONLY #include "lib/zstd.h" #include "lib/common/zstd_errors.h" static uint_t zstd_earlyabort_pass = 1; static int zstd_cutoff_level = ZIO_ZSTD_LEVEL_3; static unsigned int zstd_abort_size = (128 * 1024); static kstat_t *zstd_ksp = NULL; typedef struct zstd_stats { kstat_named_t zstd_stat_alloc_fail; kstat_named_t zstd_stat_alloc_fallback; kstat_named_t zstd_stat_com_alloc_fail; kstat_named_t zstd_stat_dec_alloc_fail; kstat_named_t zstd_stat_com_inval; kstat_named_t zstd_stat_dec_inval; kstat_named_t zstd_stat_dec_header_inval; kstat_named_t zstd_stat_com_fail; kstat_named_t zstd_stat_dec_fail; /* * LZ4 first-pass early abort verdict */ kstat_named_t zstd_stat_lz4pass_allowed; kstat_named_t zstd_stat_lz4pass_rejected; /* * zstd-1 second-pass early abort verdict */ kstat_named_t zstd_stat_zstdpass_allowed; kstat_named_t zstd_stat_zstdpass_rejected; /* * We excluded this from early abort for some reason */ kstat_named_t zstd_stat_passignored; kstat_named_t zstd_stat_passignored_size; kstat_named_t zstd_stat_buffers; kstat_named_t zstd_stat_size; } zstd_stats_t; static zstd_stats_t zstd_stats = { { "alloc_fail", KSTAT_DATA_UINT64 }, { "alloc_fallback", KSTAT_DATA_UINT64 }, { "compress_alloc_fail", KSTAT_DATA_UINT64 }, { "decompress_alloc_fail", KSTAT_DATA_UINT64 }, { "compress_level_invalid", KSTAT_DATA_UINT64 }, { "decompress_level_invalid", KSTAT_DATA_UINT64 }, { "decompress_header_invalid", KSTAT_DATA_UINT64 }, { "compress_failed", KSTAT_DATA_UINT64 }, { "decompress_failed", KSTAT_DATA_UINT64 }, { "lz4pass_allowed", KSTAT_DATA_UINT64 }, { "lz4pass_rejected", KSTAT_DATA_UINT64 }, { "zstdpass_allowed", KSTAT_DATA_UINT64 }, { "zstdpass_rejected", KSTAT_DATA_UINT64 }, { "passignored", KSTAT_DATA_UINT64 }, { "passignored_size", KSTAT_DATA_UINT64 }, { "buffers", KSTAT_DATA_UINT64 }, { "size", KSTAT_DATA_UINT64 }, }; #ifdef _KERNEL static int kstat_zstd_update(kstat_t *ksp, int rw) { ASSERT(ksp != NULL); if (rw == KSTAT_WRITE && ksp == zstd_ksp) { ZSTDSTAT_ZERO(zstd_stat_alloc_fail); ZSTDSTAT_ZERO(zstd_stat_alloc_fallback); ZSTDSTAT_ZERO(zstd_stat_com_alloc_fail); ZSTDSTAT_ZERO(zstd_stat_dec_alloc_fail); ZSTDSTAT_ZERO(zstd_stat_com_inval); ZSTDSTAT_ZERO(zstd_stat_dec_inval); ZSTDSTAT_ZERO(zstd_stat_dec_header_inval); ZSTDSTAT_ZERO(zstd_stat_com_fail); ZSTDSTAT_ZERO(zstd_stat_dec_fail); ZSTDSTAT_ZERO(zstd_stat_lz4pass_allowed); ZSTDSTAT_ZERO(zstd_stat_lz4pass_rejected); ZSTDSTAT_ZERO(zstd_stat_zstdpass_allowed); ZSTDSTAT_ZERO(zstd_stat_zstdpass_rejected); ZSTDSTAT_ZERO(zstd_stat_passignored); ZSTDSTAT_ZERO(zstd_stat_passignored_size); } return (0); } #endif /* Enums describing the allocator type specified by kmem_type in zstd_kmem */ enum zstd_kmem_type { ZSTD_KMEM_UNKNOWN = 0, /* Allocation type using kmem_vmalloc */ ZSTD_KMEM_DEFAULT, /* Pool based allocation using mempool_alloc */ ZSTD_KMEM_POOL, /* Reserved fallback memory for decompression only */ ZSTD_KMEM_DCTX, ZSTD_KMEM_COUNT, }; /* Structure for pooled memory objects */ struct zstd_pool { void *mem; size_t size; kmutex_t barrier; hrtime_t timeout; }; /* Global structure for handling memory allocations */ struct zstd_kmem { enum zstd_kmem_type kmem_type; size_t kmem_size; struct zstd_pool *pool; }; /* Fallback memory structure used for decompression only if memory runs out */ struct zstd_fallback_mem { size_t mem_size; void *mem; kmutex_t barrier; }; struct zstd_levelmap { int16_t zstd_level; enum zio_zstd_levels level; }; /* * ZSTD memory handlers * * For decompression we use a different handler which also provides fallback * memory allocation in case memory runs out. * * The ZSTD handlers were split up for the most simplified implementation. */ static void *zstd_alloc(void *opaque, size_t size); static void *zstd_dctx_alloc(void *opaque, size_t size); static void zstd_free(void *opaque, void *ptr); /* Compression memory handler */ static const ZSTD_customMem zstd_malloc = { zstd_alloc, zstd_free, NULL, }; /* Decompression memory handler */ static const ZSTD_customMem zstd_dctx_malloc = { zstd_dctx_alloc, zstd_free, NULL, }; /* Level map for converting ZFS internal levels to ZSTD levels and vice versa */ static struct zstd_levelmap zstd_levels[] = { {ZIO_ZSTD_LEVEL_1, ZIO_ZSTD_LEVEL_1}, {ZIO_ZSTD_LEVEL_2, ZIO_ZSTD_LEVEL_2}, {ZIO_ZSTD_LEVEL_3, ZIO_ZSTD_LEVEL_3}, {ZIO_ZSTD_LEVEL_4, ZIO_ZSTD_LEVEL_4}, {ZIO_ZSTD_LEVEL_5, ZIO_ZSTD_LEVEL_5}, {ZIO_ZSTD_LEVEL_6, ZIO_ZSTD_LEVEL_6}, {ZIO_ZSTD_LEVEL_7, ZIO_ZSTD_LEVEL_7}, {ZIO_ZSTD_LEVEL_8, ZIO_ZSTD_LEVEL_8}, {ZIO_ZSTD_LEVEL_9, ZIO_ZSTD_LEVEL_9}, {ZIO_ZSTD_LEVEL_10, ZIO_ZSTD_LEVEL_10}, {ZIO_ZSTD_LEVEL_11, ZIO_ZSTD_LEVEL_11}, {ZIO_ZSTD_LEVEL_12, ZIO_ZSTD_LEVEL_12}, {ZIO_ZSTD_LEVEL_13, ZIO_ZSTD_LEVEL_13}, {ZIO_ZSTD_LEVEL_14, ZIO_ZSTD_LEVEL_14}, {ZIO_ZSTD_LEVEL_15, ZIO_ZSTD_LEVEL_15}, {ZIO_ZSTD_LEVEL_16, ZIO_ZSTD_LEVEL_16}, {ZIO_ZSTD_LEVEL_17, ZIO_ZSTD_LEVEL_17}, {ZIO_ZSTD_LEVEL_18, ZIO_ZSTD_LEVEL_18}, {ZIO_ZSTD_LEVEL_19, ZIO_ZSTD_LEVEL_19}, {-1, ZIO_ZSTD_LEVEL_FAST_1}, {-2, ZIO_ZSTD_LEVEL_FAST_2}, {-3, ZIO_ZSTD_LEVEL_FAST_3}, {-4, ZIO_ZSTD_LEVEL_FAST_4}, {-5, ZIO_ZSTD_LEVEL_FAST_5}, {-6, ZIO_ZSTD_LEVEL_FAST_6}, {-7, ZIO_ZSTD_LEVEL_FAST_7}, {-8, ZIO_ZSTD_LEVEL_FAST_8}, {-9, ZIO_ZSTD_LEVEL_FAST_9}, {-10, ZIO_ZSTD_LEVEL_FAST_10}, {-20, ZIO_ZSTD_LEVEL_FAST_20}, {-30, ZIO_ZSTD_LEVEL_FAST_30}, {-40, ZIO_ZSTD_LEVEL_FAST_40}, {-50, ZIO_ZSTD_LEVEL_FAST_50}, {-60, ZIO_ZSTD_LEVEL_FAST_60}, {-70, ZIO_ZSTD_LEVEL_FAST_70}, {-80, ZIO_ZSTD_LEVEL_FAST_80}, {-90, ZIO_ZSTD_LEVEL_FAST_90}, {-100, ZIO_ZSTD_LEVEL_FAST_100}, {-500, ZIO_ZSTD_LEVEL_FAST_500}, {-1000, ZIO_ZSTD_LEVEL_FAST_1000}, }; /* * This variable represents the maximum count of the pool based on the number * of CPUs plus some buffer. We default to cpu count * 4, see init_zstd. */ static int pool_count = 16; #define ZSTD_POOL_MAX pool_count #define ZSTD_POOL_TIMEOUT 60 * 2 static struct zstd_fallback_mem zstd_dctx_fallback; static struct zstd_pool *zstd_mempool_cctx; static struct zstd_pool *zstd_mempool_dctx; /* * The library zstd code expects these if ADDRESS_SANITIZER gets defined, * and while ASAN does this, KASAN defines that and does not. So to avoid * changing the external code, we do this. */ #if defined(ZFS_ASAN_ENABLED) #define ADDRESS_SANITIZER 1 #endif #if defined(_KERNEL) && defined(ADDRESS_SANITIZER) void __asan_unpoison_memory_region(void const volatile *addr, size_t size); void __asan_poison_memory_region(void const volatile *addr, size_t size); void __asan_unpoison_memory_region(void const volatile *addr, size_t size) {}; void __asan_poison_memory_region(void const volatile *addr, size_t size) {}; #endif static void zstd_mempool_reap(struct zstd_pool *zstd_mempool) { struct zstd_pool *pool; if (!zstd_mempool || !ZSTDSTAT(zstd_stat_buffers)) { return; } /* free obsolete slots */ for (int i = 0; i < ZSTD_POOL_MAX; i++) { pool = &zstd_mempool[i]; if (pool->mem && mutex_tryenter(&pool->barrier)) { /* Free memory if unused object older than 2 minutes */ if (pool->mem && gethrestime_sec() > pool->timeout) { vmem_free(pool->mem, pool->size); ZSTDSTAT_SUB(zstd_stat_buffers, 1); ZSTDSTAT_SUB(zstd_stat_size, pool->size); pool->mem = NULL; pool->size = 0; pool->timeout = 0; } mutex_exit(&pool->barrier); } } } /* * Try to get a cached allocated buffer from memory pool or allocate a new one * if necessary. If a object is older than 2 minutes and does not fit the * requested size, it will be released and a new cached entry will be allocated. * If other pooled objects are detected without being used for 2 minutes, they * will be released, too. * * The concept is that high frequency memory allocations of bigger objects are * expensive. So if a lot of work is going on, allocations will be kept for a * while and can be reused in that time frame. * * The scheduled release will be updated every time a object is reused. */ static void * zstd_mempool_alloc(struct zstd_pool *zstd_mempool, size_t size) { struct zstd_pool *pool; struct zstd_kmem *mem = NULL; if (!zstd_mempool) { return (NULL); } /* Seek for preallocated memory slot and free obsolete slots */ for (int i = 0; i < ZSTD_POOL_MAX; i++) { pool = &zstd_mempool[i]; /* * This lock is simply a marker for a pool object being in use. * If it's already hold, it will be skipped. * * We need to create it before checking it to avoid race * conditions caused by running in a threaded context. * * The lock is later released by zstd_mempool_free. */ if (mutex_tryenter(&pool->barrier)) { /* * Check if objects fits the size, if so we take it and * update the timestamp. */ if (pool->mem && size <= pool->size) { pool->timeout = gethrestime_sec() + ZSTD_POOL_TIMEOUT; mem = pool->mem; return (mem); } mutex_exit(&pool->barrier); } } /* * If no preallocated slot was found, try to fill in a new one. * * We run a similar algorithm twice here to avoid pool fragmentation. * The first one may generate holes in the list if objects get released. * We always make sure that these holes get filled instead of adding new * allocations constantly at the end. */ for (int i = 0; i < ZSTD_POOL_MAX; i++) { pool = &zstd_mempool[i]; if (mutex_tryenter(&pool->barrier)) { /* Object is free, try to allocate new one */ if (!pool->mem) { mem = vmem_alloc(size, KM_SLEEP); if (mem) { ZSTDSTAT_ADD(zstd_stat_buffers, 1); ZSTDSTAT_ADD(zstd_stat_size, size); pool->mem = mem; pool->size = size; /* Keep track for later release */ mem->pool = pool; mem->kmem_type = ZSTD_KMEM_POOL; mem->kmem_size = size; } } if (size <= pool->size) { /* Update timestamp */ pool->timeout = gethrestime_sec() + ZSTD_POOL_TIMEOUT; return (pool->mem); } mutex_exit(&pool->barrier); } } /* * If the pool is full or the allocation failed, try lazy allocation * instead. */ if (!mem) { mem = vmem_alloc(size, KM_NOSLEEP); if (mem) { mem->pool = NULL; mem->kmem_type = ZSTD_KMEM_DEFAULT; mem->kmem_size = size; } } return (mem); } /* Mark object as released by releasing the barrier mutex */ static void zstd_mempool_free(struct zstd_kmem *z) { mutex_exit(&z->pool->barrier); } /* Convert ZFS internal enum to ZSTD level */ static int zstd_enum_to_level(enum zio_zstd_levels level, int16_t *zstd_level) { if (level > 0 && level <= ZIO_ZSTD_LEVEL_19) { *zstd_level = zstd_levels[level - 1].zstd_level; return (0); } if (level >= ZIO_ZSTD_LEVEL_FAST_1 && level <= ZIO_ZSTD_LEVEL_FAST_1000) { *zstd_level = zstd_levels[level - ZIO_ZSTD_LEVEL_FAST_1 + ZIO_ZSTD_LEVEL_19].zstd_level; return (0); } /* Invalid/unknown zfs compression enum - this should never happen. */ return (1); } /* Compress block using zstd */ static size_t zfs_zstd_compress_impl(void *s_start, void *d_start, size_t s_len, size_t d_len, int level) { size_t c_len; int16_t zstd_level; zfs_zstdhdr_t *hdr; ZSTD_CCtx *cctx; hdr = (zfs_zstdhdr_t *)d_start; /* Skip compression if the specified level is invalid */ if (zstd_enum_to_level(level, &zstd_level)) { ZSTDSTAT_BUMP(zstd_stat_com_inval); return (s_len); } ASSERT3U(d_len, >=, sizeof (*hdr)); ASSERT3U(d_len, <=, s_len); ASSERT3U(zstd_level, !=, 0); cctx = ZSTD_createCCtx_advanced(zstd_malloc); /* * Out of kernel memory, gently fall through - this will disable * compression in zio_compress_data */ if (!cctx) { ZSTDSTAT_BUMP(zstd_stat_com_alloc_fail); return (s_len); } /* Set the compression level */ ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, zstd_level); /* Use the "magicless" zstd header which saves us 4 header bytes */ ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless); /* * Disable redundant checksum calculation and content size storage since * this is already done by ZFS itself. */ ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0); ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0); c_len = ZSTD_compress2(cctx, hdr->data, d_len - sizeof (*hdr), s_start, s_len); ZSTD_freeCCtx(cctx); /* Error in the compression routine, disable compression. */ if (ZSTD_isError(c_len)) { /* * If we are aborting the compression because the saves are * too small, that is not a failure. Everything else is a * failure, so increment the compression failure counter. */ int err = ZSTD_getErrorCode(c_len); if (err != ZSTD_error_dstSize_tooSmall) { ZSTDSTAT_BUMP(zstd_stat_com_fail); dprintf("Error: %s", ZSTD_getErrorString(err)); } return (s_len); } /* * Encode the compressed buffer size at the start. We'll need this in * decompression to counter the effects of padding which might be added * to the compressed buffer and which, if unhandled, would confuse the * hell out of our decompression function. */ hdr->c_len = BE_32(c_len); /* * Check version for overflow. * The limit of 24 bits must not be exceeded. This allows a maximum * version 1677.72.15 which we don't expect to be ever reached. */ ASSERT3U(ZSTD_VERSION_NUMBER, <=, 0xFFFFFF); /* * Encode the compression level as well. We may need to know the * original compression level if compressed_arc is disabled, to match * the compression settings to write this block to the L2ARC. * * Encode the actual level, so if the enum changes in the future, we * will be compatible. * * The upper 24 bits store the ZSTD version to be able to provide * future compatibility, since new versions might enhance the * compression algorithm in a way, where the compressed data will * change. * * As soon as such incompatibility occurs, handling code needs to be * added, differentiating between the versions. */ zfs_set_hdrversion(hdr, ZSTD_VERSION_NUMBER); zfs_set_hdrlevel(hdr, level); hdr->raw_version_level = BE_32(hdr->raw_version_level); return (c_len + sizeof (*hdr)); } static size_t zfs_zstd_compress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int level) { int16_t zstd_level; if (zstd_enum_to_level(level, &zstd_level)) { ZSTDSTAT_BUMP(zstd_stat_com_inval); return (s_len); } /* * A zstd early abort heuristic. * * - Zeroth, if this is <= zstd-3, or < zstd_abort_size (currently * 128k), don't try any of this, just go. * (because experimentally that was a reasonable cutoff for a perf win * with tiny ratio change) * - First, we try LZ4 compression, and if it doesn't early abort, we * jump directly to whatever compression level we intended to try. * - Second, we try zstd-1 - if that errors out (usually, but not * exclusively, if it would overflow), we give up early. * * If it works, instead we go on and compress anyway. * * Why two passes? LZ4 alone gets you a lot of the way, but on highly * compressible data, it was losing up to 8.5% of the compressed * savings versus no early abort, and all the zstd-fast levels are * worse indications on their own than LZ4, and don't improve the LZ4 * pass noticably if stacked like this. */ size_t actual_abort_size = zstd_abort_size; if (zstd_earlyabort_pass > 0 && zstd_level >= zstd_cutoff_level && s_len >= actual_abort_size) { int pass_len = 1; abd_t sabd, dabd; abd_get_from_buf_struct(&sabd, s_start, s_len); abd_get_from_buf_struct(&dabd, d_start, d_len); pass_len = zfs_lz4_compress(&sabd, &dabd, s_len, d_len, 0); abd_free(&dabd); abd_free(&sabd); if (pass_len < d_len) { ZSTDSTAT_BUMP(zstd_stat_lz4pass_allowed); goto keep_trying; } ZSTDSTAT_BUMP(zstd_stat_lz4pass_rejected); pass_len = zfs_zstd_compress_impl(s_start, d_start, s_len, d_len, ZIO_ZSTD_LEVEL_1); if (pass_len == s_len || pass_len <= 0 || pass_len > d_len) { ZSTDSTAT_BUMP(zstd_stat_zstdpass_rejected); return (s_len); } ZSTDSTAT_BUMP(zstd_stat_zstdpass_allowed); } else { ZSTDSTAT_BUMP(zstd_stat_passignored); if (s_len < actual_abort_size) { ZSTDSTAT_BUMP(zstd_stat_passignored_size); } } keep_trying: return (zfs_zstd_compress_impl(s_start, d_start, s_len, d_len, level)); } /* Decompress block using zstd and return its stored level */ static int zfs_zstd_decompress_level_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, uint8_t *level) { ZSTD_DCtx *dctx; size_t result; int16_t zstd_level; uint32_t c_len; const zfs_zstdhdr_t *hdr; zfs_zstdhdr_t hdr_copy; hdr = (const zfs_zstdhdr_t *)s_start; c_len = BE_32(hdr->c_len); /* * Make a copy instead of directly converting the header, since we must * not modify the original data that may be used again later. */ hdr_copy.raw_version_level = BE_32(hdr->raw_version_level); uint8_t curlevel = zfs_get_hdrlevel(&hdr_copy); /* * NOTE: We ignore the ZSTD version for now. As soon as any * incompatibility occurs, it has to be handled accordingly. * The version can be accessed via `hdr_copy.version`. */ /* * Convert and check the level * An invalid level is a strong indicator for data corruption! In such * case return an error so the upper layers can try to fix it. */ if (zstd_enum_to_level(curlevel, &zstd_level)) { ZSTDSTAT_BUMP(zstd_stat_dec_inval); return (1); } ASSERT3U(d_len, >=, s_len); ASSERT3U(curlevel, !=, ZIO_COMPLEVEL_INHERIT); /* Invalid compressed buffer size encoded at start */ if (c_len + sizeof (*hdr) > s_len) { ZSTDSTAT_BUMP(zstd_stat_dec_header_inval); return (1); } dctx = ZSTD_createDCtx_advanced(zstd_dctx_malloc); if (!dctx) { ZSTDSTAT_BUMP(zstd_stat_dec_alloc_fail); return (1); } /* Set header type to "magicless" */ ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless); /* Decompress the data and release the context */ result = ZSTD_decompressDCtx(dctx, d_start, d_len, hdr->data, c_len); ZSTD_freeDCtx(dctx); /* * Returns 0 on success (decompression function returned non-negative) * and non-zero on failure (decompression function returned negative. */ if (ZSTD_isError(result)) { ZSTDSTAT_BUMP(zstd_stat_dec_fail); return (1); } if (level) { *level = curlevel; } return (0); } /* Decompress datablock using zstd */ static int zfs_zstd_decompress_buf(void *s_start, void *d_start, size_t s_len, size_t d_len, int level __maybe_unused) { return (zfs_zstd_decompress_level_buf(s_start, d_start, s_len, d_len, NULL)); } ZFS_COMPRESS_WRAP_DECL(zfs_zstd_compress) ZFS_DECOMPRESS_WRAP_DECL(zfs_zstd_decompress) ZFS_DECOMPRESS_LEVEL_WRAP_DECL(zfs_zstd_decompress_level) /* Allocator for zstd compression context using mempool_allocator */ static void * zstd_alloc(void *opaque __maybe_unused, size_t size) { size_t nbytes = sizeof (struct zstd_kmem) + size; struct zstd_kmem *z = NULL; z = (struct zstd_kmem *)zstd_mempool_alloc(zstd_mempool_cctx, nbytes); if (!z) { ZSTDSTAT_BUMP(zstd_stat_alloc_fail); return (NULL); } return ((void*)z + (sizeof (struct zstd_kmem))); } /* * Allocator for zstd decompression context using mempool_allocator with * fallback to reserved memory if allocation fails */ static void * zstd_dctx_alloc(void *opaque __maybe_unused, size_t size) { size_t nbytes = sizeof (struct zstd_kmem) + size; struct zstd_kmem *z = NULL; enum zstd_kmem_type type = ZSTD_KMEM_DEFAULT; z = (struct zstd_kmem *)zstd_mempool_alloc(zstd_mempool_dctx, nbytes); if (!z) { /* Try harder, decompression shall not fail */ z = vmem_alloc(nbytes, KM_SLEEP); if (z) { z->pool = NULL; } ZSTDSTAT_BUMP(zstd_stat_alloc_fail); } else { return ((void*)z + (sizeof (struct zstd_kmem))); } /* Fallback if everything fails */ if (!z) { /* * Barrier since we only can handle it in a single thread. All * other following threads need to wait here until decompression * is completed. zstd_free will release this barrier later. */ mutex_enter(&zstd_dctx_fallback.barrier); z = zstd_dctx_fallback.mem; type = ZSTD_KMEM_DCTX; ZSTDSTAT_BUMP(zstd_stat_alloc_fallback); } /* Allocation should always be successful */ if (!z) { return (NULL); } z->kmem_type = type; z->kmem_size = nbytes; return ((void*)z + (sizeof (struct zstd_kmem))); } /* Free allocated memory by its specific type */ static void zstd_free(void *opaque __maybe_unused, void *ptr) { struct zstd_kmem *z = (ptr - sizeof (struct zstd_kmem)); enum zstd_kmem_type type; ASSERT3U(z->kmem_type, <, ZSTD_KMEM_COUNT); ASSERT3U(z->kmem_type, >, ZSTD_KMEM_UNKNOWN); type = z->kmem_type; switch (type) { case ZSTD_KMEM_DEFAULT: vmem_free(z, z->kmem_size); break; case ZSTD_KMEM_POOL: zstd_mempool_free(z); break; case ZSTD_KMEM_DCTX: mutex_exit(&zstd_dctx_fallback.barrier); break; default: break; } } /* Allocate fallback memory to ensure safe decompression */ static void __init create_fallback_mem(struct zstd_fallback_mem *mem, size_t size) { mem->mem_size = size; mem->mem = vmem_zalloc(mem->mem_size, KM_SLEEP); mutex_init(&mem->barrier, NULL, MUTEX_DEFAULT, NULL); } /* Initialize memory pool barrier mutexes */ static void __init zstd_mempool_init(void) { zstd_mempool_cctx = kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP); zstd_mempool_dctx = kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP); for (int i = 0; i < ZSTD_POOL_MAX; i++) { mutex_init(&zstd_mempool_cctx[i].barrier, NULL, MUTEX_DEFAULT, NULL); mutex_init(&zstd_mempool_dctx[i].barrier, NULL, MUTEX_DEFAULT, NULL); } } /* Initialize zstd-related memory handling */ static int __init zstd_meminit(void) { zstd_mempool_init(); /* * Estimate the size of the fallback decompression context. * The expected size on x64 with current ZSTD should be about 160 KB. */ create_fallback_mem(&zstd_dctx_fallback, P2ROUNDUP(ZSTD_estimateDCtxSize() + sizeof (struct zstd_kmem), PAGESIZE)); return (0); } /* Release object from pool and free memory */ static void release_pool(struct zstd_pool *pool) { mutex_destroy(&pool->barrier); vmem_free(pool->mem, pool->size); pool->mem = NULL; pool->size = 0; } /* Release memory pool objects */ static void zstd_mempool_deinit(void) { for (int i = 0; i < ZSTD_POOL_MAX; i++) { release_pool(&zstd_mempool_cctx[i]); release_pool(&zstd_mempool_dctx[i]); } kmem_free(zstd_mempool_dctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool)); kmem_free(zstd_mempool_cctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool)); zstd_mempool_dctx = NULL; zstd_mempool_cctx = NULL; } /* release unused memory from pool */ void zfs_zstd_cache_reap_now(void) { /* * Short-circuit if there are no buffers to begin with. */ if (ZSTDSTAT(zstd_stat_buffers) == 0) return; /* * calling alloc with zero size seeks * and releases old unused objects */ zstd_mempool_reap(zstd_mempool_cctx); zstd_mempool_reap(zstd_mempool_dctx); } extern int __init zstd_init(void) { /* Set pool size by using maximum sane thread count * 4 */ pool_count = (boot_ncpus * 4); zstd_meminit(); /* Initialize kstat */ zstd_ksp = kstat_create("zfs", 0, "zstd", "misc", KSTAT_TYPE_NAMED, sizeof (zstd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); if (zstd_ksp != NULL) { zstd_ksp->ks_data = &zstd_stats; kstat_install(zstd_ksp); #ifdef _KERNEL zstd_ksp->ks_update = kstat_zstd_update; #endif } return (0); } extern void zstd_fini(void) { /* Deinitialize kstat */ if (zstd_ksp != NULL) { kstat_delete(zstd_ksp); zstd_ksp = NULL; } /* Release fallback memory */ vmem_free(zstd_dctx_fallback.mem, zstd_dctx_fallback.mem_size); mutex_destroy(&zstd_dctx_fallback.barrier); /* Deinit memory pool */ zstd_mempool_deinit(); } #if defined(_KERNEL) #ifdef __FreeBSD__ module_init(zstd_init); module_exit(zstd_fini); #endif ZFS_MODULE_PARAM(zfs, zstd_, earlyabort_pass, UINT, ZMOD_RW, "Enable early abort attempts when using zstd"); ZFS_MODULE_PARAM(zfs, zstd_, abort_size, UINT, ZMOD_RW, "Minimal size of block to attempt early abort"); #endif diff --git a/module/zstd/zstd-in.c b/module/zstd/zstd-in.c index 121f375e5515..fb56b19fefd9 100644 --- a/module/zstd/zstd-in.c +++ b/module/zstd/zstd-in.c @@ -1,68 +1,69 @@ +// SPDX-License-Identifier: BSD-3-Clause /* * BSD 3-Clause Clear License * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. * Copyright (c) 2019-2020, Michael Niewöhner */ #define MEM_MODULE #define XXH_NAMESPACE ZSTD_ #define XXH_PRIVATE_API #define XXH_INLINE_ALL #define ZSTD_LEGACY_SUPPORT 0 #define ZSTD_LIB_DICTBUILDER 0 #define ZSTD_LIB_DEPRECATED 0 #define ZSTD_NOBENCH #include "common/debug.c" #include "common/entropy_common.c" #include "common/error_private.c" #include "common/fse_decompress.c" #include "common/pool.c" #include "common/zstd_common.c" #include "compress/fse_compress.c" #include "compress/hist.c" #include "compress/huf_compress.c" #include "compress/zstd_compress_literals.c" #include "compress/zstd_compress_sequences.c" #include "compress/zstd_compress_superblock.c" #include "compress/zstd_compress.c" #include "compress/zstd_double_fast.c" #include "compress/zstd_fast.c" #include "compress/zstd_lazy.c" #include "compress/zstd_ldm.c" #include "compress/zstd_opt.c" #include "decompress/huf_decompress.c" #include "decompress/zstd_ddict.c" #include "decompress/zstd_decompress.c" #include "decompress/zstd_decompress_block.c"