Index: Makefile.inc1 =================================================================== --- Makefile.inc1 +++ Makefile.inc1 @@ -2305,7 +2305,7 @@ ${_cddl_lib_libctf} \ lib/libutil lib/libpjdlog ${_lib_libypclnt} lib/libz lib/msun \ ${_secure_lib_libcrypto} ${_lib_libldns} \ - ${_secure_lib_libssh} ${_secure_lib_libssl} + ${_secure_lib_libssh} ${_secure_lib_libssl} lib/libzstd .if ${MK_GNUCXX} != "no" _prebuild_libs+= gnu/lib/libstdc++ gnu/lib/libsupc++ Index: cddl/lib/libzfs/Makefile =================================================================== --- cddl/lib/libzfs/Makefile +++ cddl/lib/libzfs/Makefile @@ -6,7 +6,7 @@ .PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libzfs/common LIB= zfs -LIBADD= md pthread umem util uutil m avl bsdxml geom nvpair z zfs_core +LIBADD= md pthread umem util uutil m avl bsdxml geom nvpair z zfs_core zstd SRCS= deviceid.c \ fsshare.c \ mkdirp.c \ Index: cddl/lib/libzpool/Makefile =================================================================== --- cddl/lib/libzpool/Makefile +++ cddl/lib/libzpool/Makefile @@ -54,8 +54,9 @@ CFLAGS+= -I${SRCTOP}/lib/libpthread/thread CFLAGS+= -I${SRCTOP}/lib/libpthread/sys CFLAGS+= -I${SRCTOP}/lib/libthr/arch/${MACHINE_CPUARCH}/include +CFLAGS+= -I${.CURDIR}/../../../contrib/zstd/lib -LIBADD= md pthread z nvpair avl umem +LIBADD= md pthread z nvpair avl umem zstd # atomic.S doesn't like profiling. MK_PROFILE= no Index: lib/libstand/Makefile =================================================================== --- lib/libstand/Makefile +++ lib/libstand/Makefile @@ -109,6 +109,15 @@ sed -e 's||"stand.h"|' \ ${.ALLSRC} > ${.TARGET} +# decompression functionality from libzstd +# NOTE: to actually test this functionality after libbz2 upgrade compile +# loader(8) with LOADER_ZSTD_SUPPORT defined +.PATH: ${LIBSTAND_SRC}/../../contrib/zstd/lib/common +.PATH: ${LIBSTAND_SRC}/../../contrib/zstd/lib/decompress +CFLAGS+=-D_BOOTSTRAP +CFLAGS+=-I${LIBSTAND_SRC}/../../contrib/zstd/lib -I${LIBSTAND_SRC}/../../contrib/zstd/lib/common +SRCS+= entropy_common.c error_private.c fse_decompress.c xxhash.c zstd_common.c huf_decompress.c zstd_decompress.c + # decompression functionality from zlib .PATH: ${LIBSTAND_SRC}/../../contrib/zlib CFLAGS+=-DHAVE_MEMCPY -I${LIBSTAND_SRC}/../../contrib/zlib Index: rescue/rescue/Makefile =================================================================== --- rescue/rescue/Makefile +++ rescue/rescue/Makefile @@ -129,7 +129,7 @@ CRUNCH_LIBS+= -l80211 -lalias -lcam -lncursesw -ldevstat -lipsec -llzma .if ${MK_ZFS} != "no" -CRUNCH_LIBS+= -lavl -lzpool -lzfs_core -lzfs -lnvpair -lpthread -luutil -lumem +CRUNCH_LIBS+= -lavl -lzpool -lzfs_core -lzfs -lnvpair -lpthread -luutil -lumem -lprivatezstd .else # liblzma needs pthread CRUNCH_LIBS+= -lpthread Index: share/mk/src.libnames.mk =================================================================== --- share/mk/src.libnames.mk +++ share/mk/src.libnames.mk @@ -326,9 +326,9 @@ _DP_fifolog= z _DP_ipf= kvm _DP_zfs= md pthread umem util uutil m nvpair avl bsdxml geom nvpair z \ - zfs_core + zfs_core zstd _DP_zfs_core= nvpair -_DP_zpool= md pthread z nvpair avl umem +_DP_zpool= md pthread z nvpair avl umem zstd .if ${MK_OFED} != "no" _DP_cxgb4= ibverbs pthread _DP_ibcm= ibverbs Index: sys/boot/efi/boot1/Makefile =================================================================== --- sys/boot/efi/boot1/Makefile +++ sys/boot/efi/boot1/Makefile @@ -40,6 +40,7 @@ # Do not unroll skein loops, reduce code size CFLAGS+= -DSKEIN_LOOP=111 .PATH: ${.CURDIR}/../../../crypto/skein +CFLAGS+= -I${SRCTOP}/contrib/zstd/lib .endif .if ${COMPILER_TYPE} == "gcc" && ${COMPILER_VERSION} > 40201 @@ -90,8 +91,9 @@ # __aeabi_* (arm) or __divdi3 (i386). # as well as required string and memory functions for all platforms. # +LIBSTAND= ${OBJTOP}/lib/libstand/libstand.a DPADD+= ${LIBEFI} ${LIBSTAND} -LDADD+= ${LIBEFI} -lstand +LDADD+= ${LIBEFI} ${LIBSTAND} DPADD+= ${LDSCRIPT} Index: sys/boot/efi/boot1/boot1.c =================================================================== --- sys/boot/efi/boot1/boot1.c +++ sys/boot/efi/boot1/boot1.c @@ -73,6 +73,19 @@ return (NULL); } +void * +Calloc(size_t number, size_t len, const char *file __unused, int line __unused) +{ + uintptr_t bytes = (uintptr_t)number * (uintptr_t)len; + void *out; + + if ((out = Malloc(bytes, file, line)) != NULL) { + bzero(out, bytes); + } + + return(out); +} + void Free(void *buf, const char *file __unused, int line __unused) { Index: sys/boot/efi/loader/Makefile =================================================================== --- sys/boot/efi/loader/Makefile +++ sys/boot/efi/loader/Makefile @@ -29,7 +29,7 @@ # Do not unroll skein loops, reduce code size CFLAGS+= -DSKEIN_LOOP=111 .PATH: ${.CURDIR}/../../../crypto/skein - +CFLAGS+= -I${SRCTOP}/contrib/zstd/lib # Disable warnings that are currently incompatible with the zfs boot code CWARNFLAGS.zfs.c+= -Wno-sign-compare CWARNFLAGS.zfs.c+= -Wno-array-bounds @@ -149,7 +149,7 @@ --output-target=${EFI_TARGET} ${.ALLSRC} ${.TARGET} LIBEFI= ${.OBJDIR}/../libefi/libefi.a - +LIBSTAND= ${OBJTOP}/lib/libstand/libstand.a DPADD= ${LIBFICL} ${LIBEFI} ${LIBFDT} ${LIBEFI_FDT} ${LIBSTAND} \ ${LDSCRIPT} LDADD= ${LIBFICL} ${LIBEFI} ${LIBFDT} ${LIBEFI_FDT} ${LIBSTAND} Index: sys/boot/i386/gptzfsboot/Makefile =================================================================== --- sys/boot/i386/gptzfsboot/Makefile +++ sys/boot/i386/gptzfsboot/Makefile @@ -30,6 +30,7 @@ -I${.CURDIR}/../../zfs \ -I${.CURDIR}/../../../cddl/boot/zfs \ -I${.CURDIR}/../../../crypto/skein \ + -I${SRCTOP}/contrib/zstd/lib \ -I${.CURDIR}/../btx/lib -I. \ -I${.CURDIR}/../boot2 \ -I${.CURDIR}/../../.. \ @@ -87,7 +88,8 @@ gptzfsboot.out: ${BTXCRT} zfsboot.o sio.o gpt.o drv.o cons.o util.o \ skein.o skein_block.o ${OPENCRYPTO_XTS} - ${LD} ${LD_FLAGS} -Ttext ${ORG2} -o ${.TARGET} ${.ALLSRC} ${LIBGELIBOOT} ${LIBSTAND} + ${LD} ${LD_FLAGS} -Ttext ${ORG2} -o ${.TARGET} ${.ALLSRC} ${LIBGELIBOOT} \ + ${LIBSTAND} zfsboot.o: ${.CURDIR}/../../zfs/zfsimpl.c Index: sys/boot/i386/zfsboot/Makefile =================================================================== --- sys/boot/i386/zfsboot/Makefile +++ sys/boot/i386/zfsboot/Makefile @@ -27,6 +27,7 @@ -I${.CURDIR}/../../zfs \ -I${.CURDIR}/../../../cddl/boot/zfs \ -I${.CURDIR}/../../../crypto/skein \ + -I${SRCTOP}/contrib/zstd/lib \ -I${.CURDIR}/../btx/lib -I. \ -I${.CURDIR}/../boot2 \ -Wall -Waggregate-return -Wbad-function-cast -Wno-cast-align \ Index: sys/boot/i386/zfsboot/zfsboot.c =================================================================== --- sys/boot/i386/zfsboot/zfsboot.c +++ sys/boot/i386/zfsboot/zfsboot.c @@ -126,7 +126,20 @@ static void bios_getmem(void); void *malloc(size_t n); void free(void *ptr); +void panic(const char *fmt, ...); int main(void); + +void +panic(const char *fmt, ...) +{ + + /* We don't have vprintf, just punt */ + printf("PANIC: %s\n", fmt); + + for (;;) + ; + /* NOTREACHED */ +} void * malloc(size_t n) Index: sys/boot/userboot/userboot/Makefile =================================================================== --- sys/boot/userboot/userboot/Makefile +++ sys/boot/userboot/userboot/Makefile @@ -60,6 +60,7 @@ .PATH: ${.CURDIR}/../../common .include "${.CURDIR}/../../common/Makefile.inc" CFLAGS+= -I${.CURDIR}/../../common +CFLAGS+= -I${SRCTOP}/contrib/zstd/lib CFLAGS+= -I. DPADD+= ${LIBFICL} ${LIBZFSBOOT} ${LIBSTAND} LDADD+= ${LIBFICL} ${LIBZFSBOOT} ${LIBSTAND} Index: sys/boot/userboot/zfs/Makefile =================================================================== --- sys/boot/userboot/zfs/Makefile +++ sys/boot/userboot/zfs/Makefile @@ -15,6 +15,7 @@ CFLAGS+= -I${.CURDIR}/../../../../lib/libstand CFLAGS+= -I${.CURDIR}/../../../cddl/boot/zfs CFLAGS+= -I${.CURDIR}/../../../crypto/skein +CFLAGS+= -I${SRCTOP}/contrib/zstd/lib CFLAGS+= -ffreestanding -fPIC CFLAGS+= -Wformat -Wall Index: sys/boot/zfs/Makefile =================================================================== --- sys/boot/zfs/Makefile +++ sys/boot/zfs/Makefile @@ -15,6 +15,7 @@ CFLAGS+= -I${.CURDIR}/../../../lib/libstand CFLAGS+= -I${.CURDIR}/../../cddl/boot/zfs CFLAGS+= -I${.CURDIR}/../../crypto/skein +CFLAGS+= -I${SRCTOP}/contrib/zstd/lib .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" CFLAGS+= -march=i386 Index: sys/boot/zfs/zfsimpl.c =================================================================== --- sys/boot/zfs/zfsimpl.c +++ sys/boot/zfs/zfsimpl.c @@ -61,6 +61,7 @@ "org.illumos:sha512", "org.illumos:skein", "org.zfsonlinux:large_dnode", + "org.freebsd:zstd_compress", NULL }; Index: sys/cddl/boot/zfs/zfsimpl.h =================================================================== --- sys/cddl/boot/zfs/zfsimpl.h +++ sys/cddl/boot/zfs/zfsimpl.h @@ -561,6 +561,9 @@ ZIO_COMPRESS_GZIP_9, ZIO_COMPRESS_ZLE, ZIO_COMPRESS_LZ4, + ZIO_COMPRESS_ZSTD_MIN, + ZIO_COMPRESS_ZSTD_DEF, + ZIO_COMPRESS_ZSTD_MAX, ZIO_COMPRESS_FUNCTIONS }; Index: sys/cddl/boot/zfs/zfssubr.c =================================================================== --- sys/cddl/boot/zfs/zfssubr.c +++ sys/cddl/boot/zfs/zfssubr.c @@ -162,6 +162,7 @@ #include "lzjb.c" #include "zle.c" #include "lz4.c" +#include "zstd.c" /* * Compression vectors. @@ -183,6 +184,9 @@ {NULL, NULL, 9, "gzip-9"}, {NULL, zle_decompress, 64, "zle"}, {NULL, lz4_decompress, 0, "lz4"}, + {NULL, zstd_decompress, ZIO_ZSTD_LEVEL_MIN, "zstd-min"}, + {NULL, zstd_decompress, ZIO_ZSTD_LEVEL_DEF, "zstd-def"}, + {NULL, zstd_decompress, ZIO_ZSTD_LEVEL_MAX, "zstd-max"}, }; static void Index: sys/cddl/boot/zfs/zstd.c =================================================================== --- /dev/null +++ sys/cddl/boot/zfs/zstd.c @@ -0,0 +1,91 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2016-2017 by ScaleEngine Inc. All rights reserved. + * Copyright (c) 2017 Allan Jude . All rights reserved. + */ + +#include + +#define ZSTD_STATIC_LINKING_ONLY +#include + +/* Allocate just one decompression context and reuse it */ +static char *dctxbuf = NULL; +static size_t dctxsize; + +static int +zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) +{ + const char *src = s_start; + uint32_t bufsiz = htonl(*(uint32_t *)src); + size_t result; + + ASSERT(d_len >= s_len); + + /* invalid compressed buffer size encoded at start */ + if (bufsiz + sizeof (bufsiz) > s_len) { + printf("ZFS: Failed to decode decompression header\n"); + return (1); + } + + /* + * Returns 0 on success (decompression function returned non-negative) + * and non-zero on failure (decompression function returned negative. + */ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + result = ZSTD_decompress(d_start, d_len, &src[sizeof (bufsiz)], bufsiz); + if (ZSTD_isError(result)) { + printf("ZFS: Failed to decompress block: %s\n", + ZSTD_getErrorName(result)); + return (-1); + } +#else + ZSTD_DCtx *dctx; + + if (dctxbuf == NULL) { + dctxsize = roundup2(ZSTD_estimateDCtxSize() + 8, PAGE_SIZE); + /* Pointer must be 8 byte aligned */ + dctxbuf = (char *)roundup2((size_t)malloc(dctxsize), 8); + if (dctxbuf == NULL) { + printf("ZFS: memory allocation failure\n"); + return (-1); + } + } + + dctx = ZSTD_initStaticDCtx(dctxbuf, dctxsize); + if (dctx == NULL) { + printf("ZFS: failed to initialize decompression context\n"); + return (-1); + } + + result = ZSTD_decompressDCtx(dctx, d_start, d_len, + &src[sizeof (bufsiz)], bufsiz); + if (ZSTD_isError(result)) { + printf("ZFS: Failed to decompress block: %s\n", + ZSTD_getErrorName(result)); + return (-1); + } +#endif + + return (0); +} Index: sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.h =================================================================== --- sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.h +++ sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.h @@ -57,6 +57,7 @@ #ifdef illumos SPA_FEATURE_EDONR, #endif + SPA_FEATURE_ZSTD_COMPRESS, SPA_FEATURES } spa_feature_t; Index: sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c =================================================================== --- sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c +++ sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c @@ -247,4 +247,10 @@ "Edon-R hash algorithm.", ZFEATURE_FLAG_PER_DATASET, NULL); #endif + + zfeature_register(SPA_FEATURE_ZSTD_COMPRESS, + "org.freebsd:zstd_compress", "zstd_compress", + "zstd compression algorithm support.", + ZFEATURE_FLAG_PER_DATASET, NULL); + } Index: sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c =================================================================== --- sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c +++ sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c @@ -116,6 +116,10 @@ { "gzip-9", ZIO_COMPRESS_GZIP_9 }, { "zle", ZIO_COMPRESS_ZLE }, { "lz4", ZIO_COMPRESS_LZ4 }, + { "zstd", ZIO_COMPRESS_ZSTD_DEF }, /* zstd default */ + { "zstd-min", ZIO_COMPRESS_ZSTD_MIN }, + { "zstd-def", ZIO_COMPRESS_ZSTD_DEF }, + { "zstd-max", ZIO_COMPRESS_ZSTD_MAX }, { NULL } }; @@ -250,7 +254,8 @@ zprop_register_index(ZFS_PROP_COMPRESSION, "compression", ZIO_COMPRESS_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, - "on | off | lzjb | gzip | gzip-[1-9] | zle | lz4", + "on | off | lzjb | gzip | gzip-[1-9] | zle | lz4" + " | zstd | zstd-min | zstd-def | zstd-max", "COMPRESS", compress_table); zprop_register_index(ZFS_PROP_SNAPDIR, "snapdir", ZFS_SNAPDIR_HIDDEN, PROP_INHERIT, ZFS_TYPE_FILESYSTEM, Index: sys/cddl/contrib/opensolaris/uts/common/Makefile.files =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/Makefile.files +++ sys/cddl/contrib/opensolaris/uts/common/Makefile.files @@ -110,7 +110,8 @@ zio_compress.o \ zio_inject.o \ zle.o \ - zrlock.o + zrlock.o \ + zstd.o ZFS_SHARED_OBJS += \ zfeature_common.o \ Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c @@ -2019,6 +2019,7 @@ metaslab_alloc_trace_init(); zio_init(); lz4_init(); + zstd_init(); dmu_init(); zil_init(); vdev_cache_stat_init(); @@ -2047,6 +2048,7 @@ zil_fini(); dmu_fini(); lz4_fini(); + zstd_fini(); zio_fini(); metaslab_alloc_trace_fini(); range_tree_fini(); Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h @@ -124,6 +124,9 @@ #define BOOTFS_COMPRESS_VALID(compress) \ ((compress) == ZIO_COMPRESS_LZJB || \ (compress) == ZIO_COMPRESS_LZ4 || \ + (compress) == ZIO_COMPRESS_ZSTD_MIN || \ + (compress) == ZIO_COMPRESS_ZSTD_DEF || \ + (compress) == ZIO_COMPRESS_ZSTD_MAX || \ (compress) == ZIO_COMPRESS_ON || \ (compress) == ZIO_COMPRESS_OFF) Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_compress.h =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_compress.h +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_compress.h @@ -54,9 +54,16 @@ ZIO_COMPRESS_GZIP_9, ZIO_COMPRESS_ZLE, ZIO_COMPRESS_LZ4, + ZIO_COMPRESS_ZSTD_MIN, + ZIO_COMPRESS_ZSTD_DEF, + ZIO_COMPRESS_ZSTD_MAX, ZIO_COMPRESS_FUNCTIONS }; +#define ZIO_ZSTD_LEVEL_MIN 1 +#define ZIO_ZSTD_LEVEL_DEF 3 +#define ZIO_ZSTD_LEVEL_MAX 19 + /* Common signature for all zio compress functions. */ typedef size_t zio_compress_func_t(void *src, void *dst, size_t s_len, size_t d_len, int); @@ -103,6 +110,12 @@ extern size_t lz4_compress(void *src, void *dst, size_t s_len, size_t d_len, int level); extern int lz4_decompress(void *src, void *dst, size_t s_len, size_t d_len, + int level); +extern void zstd_init(void); +extern void zstd_fini(void); +extern size_t zstd_compress(void *src, void *dst, size_t s_len, size_t d_len, + int level); +extern int zstd_decompress(void *src, void *dst, size_t s_len, size_t d_len, int level); /* Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_impl.h =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_impl.h +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_impl.h @@ -76,9 +76,9 @@ * the supported transformations: * * Compression: - * ZFS supports three different flavors of compression -- gzip, lzjb, and - * zle. Compression occurs as part of the write pipeline and is performed - * in the ZIO_STAGE_WRITE_BP_INIT stage. + * ZFS supports five different flavors of compression -- gzip, lzjb, lz4, zle, + * and ZStandard. Compression occurs as part of the write pipeline and is + * performed in the ZIO_STAGE_WRITE_BP_INIT stage. * * Dedup: * Dedup reads are handled by the ZIO_STAGE_DDT_READ_START and Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c @@ -3997,6 +3997,21 @@ spa_close(spa, FTAG); } + if (intval >= ZIO_COMPRESS_ZSTD_MIN && + intval <= ZIO_COMPRESS_ZSTD_MAX) { + spa_t *spa; + + if ((err = spa_open(dsname, &spa, FTAG)) != 0) + return (err); + + if (!spa_feature_is_enabled(spa, + SPA_FEATURE_ZSTD_COMPRESS)) { + spa_close(spa, FTAG); + return (SET_ERROR(ENOTSUP)); + } + spa_close(spa, FTAG); + } + /* * If this is a bootable dataset then * verify that the compression algorithm Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c @@ -74,7 +74,10 @@ {"gzip-8", 8, gzip_compress, gzip_decompress}, {"gzip-9", 9, gzip_compress, gzip_decompress}, {"zle", 64, zle_compress, zle_decompress}, - {"lz4", 0, lz4_compress, lz4_decompress} + {"lz4", 0, lz4_compress, lz4_decompress}, + {"zstd-min", ZIO_ZSTD_LEVEL_MIN, zstd_compress, zstd_decompress}, + {"zstd-def", ZIO_ZSTD_LEVEL_DEF, zstd_compress, zstd_decompress}, + {"zstd-max", ZIO_ZSTD_LEVEL_MAX, zstd_compress, zstd_decompress}, }; enum zio_compress Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zstd.c =================================================================== --- /dev/null +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zstd.c @@ -0,0 +1,333 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2016-2017 by ScaleEngine Inc. All rights reserved. + * Copyright (c) 2017 Allan Jude . All rights reserved. + */ + +#include +#include +#include +#include +#include + +#define ZSTD_STATIC_LINKING_ONLY +#include + +#define ZSTD_KMEM_MAGIC 0x20160831 + +/* + * XXX: TODO: Investigate using ZSTD_compressBlock for small blocks + */ + +static size_t real_zstd_compress(const char *source, char *dest, int isize, + int osize, int level); +static size_t real_zstd_decompress(const char *source, char *dest, int isize, + int maxosize); + +void *zstd_alloc(void *opaque, size_t size); +void zstd_free(void *opaque, void *ptr); + +static const ZSTD_customMem zstd_malloc = { + zstd_alloc, + zstd_free, + NULL, +}; + +enum zstd_kmem_type { + ZSTD_KMEM_UNKNOWN = 0, + ZSTD_KMEM_CCTX, + ZSTD_KMEM_WRKSPC_4K_MIN, + ZSTD_KMEM_WRKSPC_4K_DEF, + ZSTD_KMEM_WRKSPC_4K_MAX, + ZSTD_KMEM_WRKSPC_16K_MIN, + ZSTD_KMEM_WRKSPC_16K_DEF, + ZSTD_KMEM_WRKSPC_16K_MAX, + ZSTD_KMEM_WRKSPC_128K_MIN, + ZSTD_KMEM_WRKSPC_128K_DEF, + ZSTD_KMEM_WRKSPC_128K_MAX, + /* zfs_max_recordsize */ + ZSTD_KMEM_WRKSPC_MRS_MIN, + ZSTD_KMEM_WRKSPC_MRS_DEF, + ZSTD_KMEM_WRKSPC_MRS_MAX, + /* SPA_MAXBLOCKSIZE */ + ZSTD_KMEM_WRKSPC_MBS_MIN, + ZSTD_KMEM_WRKSPC_MBS_DEF, + ZSTD_KMEM_WRKSPC_MBS_MAX, + ZSTD_KMEM_DCTX, + ZSTD_KMEM_COUNT, +}; + +struct zstd_kmem { + uint_t kmem_magic; + enum zstd_kmem_type kmem_type; + size_t kmem_size; +}; + +struct zstd_kmem_config { + size_t block_size; + int compress_level; + char* cache_name; +}; + +static kmem_cache_t *zstd_kmem_cache[ZSTD_KMEM_COUNT] = { NULL }; +static struct zstd_kmem zstd_cache_size[ZSTD_KMEM_COUNT] = { { ZSTD_KMEM_MAGIC, 0, 0 } }; +static struct zstd_kmem_config zstd_cache_config[ZSTD_KMEM_COUNT] = { + { 0, 0, "zstd_unknown" }, + { 0, 0, "zstd_cctx" }, + { 4096, ZIO_ZSTD_LEVEL_MIN, "zstd_wrkspc_4k_min" }, + { 4096, ZIO_ZSTD_LEVEL_DEF, "zstd_wrkspc_4k_def" }, + { 4096, ZIO_ZSTD_LEVEL_MAX, "zstd_wrkspc_4k_max" }, + { 16384, ZIO_ZSTD_LEVEL_MIN, "zstd_wrkspc_16k_min" }, + { 16384, ZIO_ZSTD_LEVEL_DEF, "zstd_wrkspc_16k_def" }, + { 16384, ZIO_ZSTD_LEVEL_MAX, "zstd_wrkspc_16k_max" }, + { SPA_OLD_MAXBLOCKSIZE, ZIO_ZSTD_LEVEL_MIN, "zstd_wrkspc_128k_min" }, + { SPA_OLD_MAXBLOCKSIZE, ZIO_ZSTD_LEVEL_DEF, "zstd_wrkspc_128k_def" }, + { SPA_OLD_MAXBLOCKSIZE, ZIO_ZSTD_LEVEL_MAX, "zstd_wrkspc_128k_max" }, + /* zfs_max_recordsize, but we cannot use that at compile time */ + { 1 * 1024 * 1024, ZIO_ZSTD_LEVEL_MIN, "zstd_wrkspc_mrs_min" }, + { 1 * 1024 * 1024, ZIO_ZSTD_LEVEL_DEF, "zstd_wrkspc_mrs_def" }, + { 1 * 1024 * 1024, ZIO_ZSTD_LEVEL_MAX, "zstd_wrkspc_mrs_max" }, + { SPA_MAXBLOCKSIZE, ZIO_ZSTD_LEVEL_MIN, "zstd_wrkspc_mbs_min" }, + { SPA_MAXBLOCKSIZE, ZIO_ZSTD_LEVEL_DEF, "zstd_wrkspc_mbs_def" }, + { SPA_MAXBLOCKSIZE, ZIO_ZSTD_LEVEL_MAX, "zstd_wrkspc_mbs_max" }, + { 0, 0, "zstd_dctx" }, + }; + +static int +zstd_compare(const void *a, const void *b) +{ + struct zstd_kmem *x, *y; + + x = (struct zstd_kmem*)a; + y = (struct zstd_kmem*)b; + + ASSERT(x->kmem_magic == ZSTD_KMEM_MAGIC); + ASSERT(y->kmem_magic == ZSTD_KMEM_MAGIC); + + if (x->kmem_size > y->kmem_size) { + return (1); + } else if (x->kmem_size == y->kmem_size) { + return (0); + } else { + return (-1); + } +} + +size_t +zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) +{ + uint32_t bufsiz; + char *dest = d_start; + + ASSERT(d_len >= sizeof (bufsiz)); + ASSERT(d_len <= s_len); + + /* XXX: this could overflow, but we never have blocks that big */ + bufsiz = real_zstd_compress(s_start, &dest[sizeof (bufsiz)], s_len, + d_len - sizeof (bufsiz), n); + + /* Signal an error if the compression routine returned an error. */ + if (ZSTD_isError(bufsiz)) + return (s_len); + + /* + * Encode the compresed buffer size at the start. We'll need this in + * decompression to counter the effects of padding which might be + * added to the compressed buffer and which, if unhandled, would + * confuse the hell out of our decompression function. + */ + *(uint32_t *)dest = BE_32(bufsiz); + + return (bufsiz + sizeof (bufsiz)); +} + +int +zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) +{ + const char *src = s_start; + uint32_t bufsiz = BE_IN32(src); + + ASSERT(d_len >= s_len); + + /* invalid compressed buffer size encoded at start */ + if (bufsiz + sizeof (bufsiz) > s_len) + return (1); + + /* + * Returns 0 on success (decompression function returned non-negative) + * and non-zero on failure (decompression function returned negative. + */ + if (ZSTD_isError(real_zstd_decompress(&src[sizeof (bufsiz)], d_start, + bufsiz, d_len))) + return (-1); + + return (0); +} + +static size_t +real_zstd_compress(const char *source, char *dest, int isize, int osize, + int level) +{ + size_t result; + ZSTD_CCtx *cctx; + + /* XXX: In ZSTD 1.3+ consider using ZSTD_initStaticCCtx() instead */ + cctx = ZSTD_createCCtx_advanced(zstd_malloc); + /* + * out of kernel memory, gently fall through - this will disable + * compression in zio_compress_data + */ + if (cctx == NULL) + return (-1); + + result = ZSTD_compressCCtx(cctx, dest, osize, source, isize, level); + + ZSTD_freeCCtx(cctx); + return (result); +} + + +static size_t +real_zstd_decompress(const char *source, char *dest, int isize, int maxosize) +{ + size_t result; + ZSTD_DCtx *dctx; + + dctx = ZSTD_createDCtx_advanced(zstd_malloc); + if (dctx == NULL) + return (-1); + + result = ZSTD_decompressDCtx(dctx, dest, maxosize, source, isize); + + ZSTD_freeDCtx(dctx); + return (result); +} + +extern void * +zstd_alloc(void *opaque __unused, size_t size) +{ + size_t nbytes = sizeof(struct zstd_kmem) + size; + struct zstd_kmem *z; + enum zstd_kmem_type type; + int i; + + type = ZSTD_KMEM_UNKNOWN; + for (i = 0; i < ZSTD_KMEM_COUNT; i++) { + if (nbytes <= zstd_cache_size[i].kmem_size) { + type = zstd_cache_size[i].kmem_type; + z = kmem_cache_alloc(zstd_kmem_cache[type], + KM_NOSLEEP | M_ZERO); + break; + } + } + /* No matching cache */ + if (type == ZSTD_KMEM_UNKNOWN) { + z = kmem_alloc(nbytes, KM_NOSLEEP | M_ZERO); + } + if (z == NULL) { + return (NULL); + } + + z->kmem_magic = ZSTD_KMEM_MAGIC; + z->kmem_type = type; + z->kmem_size = nbytes; + + return ((void*)z + (sizeof(struct zstd_kmem))); +} + +extern void +zstd_free(void *opaque __unused, void *ptr) +{ + struct zstd_kmem *z = ptr - sizeof(struct zstd_kmem); + + ASSERT(z->kmem_magic == ZSTD_KMEM_MAGIC); + ASSERT(z->kmem_type < ZSTD_KMEM_COUNT); + ASSERT(z->kmem_type >= ZSTD_KMEM_UNKNOWN); + + if (z->kmem_type == ZSTD_KMEM_UNKNOWN) { + kmem_free(z, z->kmem_size); + } else { + kmem_cache_free(zstd_kmem_cache[z->kmem_type], z); + } +} + +extern void +zstd_init(void) +{ + int i; + + /* Deal with values that are not compile time constants */ + for (i = ZSTD_KMEM_WRKSPC_MRS_MIN; i <= ZSTD_KMEM_WRKSPC_MRS_MAX; i++) { + zstd_cache_config[i].block_size = zfs_max_recordsize; + } + + /* There is no estimate for the CCtx */ + zstd_cache_size[1].kmem_magic = ZSTD_KMEM_MAGIC; + zstd_cache_size[1].kmem_type = 1; + zstd_cache_size[1].kmem_size = roundup2(zstd_cache_config[1].block_size + + sizeof(struct zstd_kmem), PAGE_SIZE); + zstd_kmem_cache[1] = kmem_cache_create( + zstd_cache_config[1].cache_name, zstd_cache_size[1].kmem_size, + 0, NULL, NULL, NULL, NULL, NULL, 0); + + /* + * Estimate the size of the ZSTD CCtx workspace required for each record + * size at each compression level. + */ + for (i = 2; i < ZSTD_KMEM_DCTX; i++) { + zstd_cache_size[i].kmem_magic = ZSTD_KMEM_MAGIC; + zstd_cache_size[i].kmem_type = i; + zstd_cache_size[i].kmem_size = roundup2(ZSTD_estimateCCtxSize_advanced( + ZSTD_getCParams(zstd_cache_config[i].compress_level, + zstd_cache_config[i].block_size, 0)) + + sizeof(struct zstd_kmem), PAGE_SIZE); + zstd_kmem_cache[i] = kmem_cache_create( + zstd_cache_config[i].cache_name, zstd_cache_size[i].kmem_size, + 0, NULL, NULL, NULL, NULL, NULL, 0); + } + /* Estimate the size of the decompression context */ + zstd_cache_size[i].kmem_magic = ZSTD_KMEM_MAGIC; + zstd_cache_size[i].kmem_type = i; + zstd_cache_size[i].kmem_size = roundup2(ZSTD_estimateDCtxSize() + + sizeof(struct zstd_kmem), PAGE_SIZE); + zstd_kmem_cache[i] = kmem_cache_create(zstd_cache_config[i].cache_name, + zstd_cache_size[i].kmem_size, 0, NULL, NULL, NULL, NULL, NULL, 0); + + /* Sort the kmem caches for later searching */ + qsort(zstd_cache_size, ZSTD_KMEM_COUNT, sizeof(struct zstd_kmem), + zstd_compare); + +} + +extern void +zstd_fini(void) +{ + int i, type; + + for (i = 0; i < ZSTD_KMEM_COUNT; i++) { + type = zstd_cache_size[i].kmem_type; + if (zstd_kmem_cache[type] != NULL) { + kmem_cache_destroy(zstd_kmem_cache[type]); + } + } +} Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -252,6 +252,7 @@ cddl/contrib/opensolaris/uts/common/fs/zfs/zio_inject.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zle.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c optional zfs compile-with "${ZFS_C}" +cddl/contrib/opensolaris/uts/common/fs/zfs/zstd.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/callb.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/fm.c optional zfs compile-with "${ZFS_C}" Index: sys/modules/zfs/Makefile =================================================================== --- sys/modules/zfs/Makefile +++ sys/modules/zfs/Makefile @@ -72,6 +72,17 @@ .PATH: ${SYSDIR}/crypto/skein SRCS+= skein.c skein_block.c +CFLAGS+=-DZSTD_HEAPMODE=1 +.PATH: ${SYSDIR}/../contrib/zstd/lib +CFLAGS+=-I${SYSDIR}/../contrib/zstd/lib +.PATH: ${SYSDIR}/../contrib/zstd/lib/common +CFLAGS+=-I${SYSDIR}/../contrib/zstd/lib/common +SRCS+= zstd_common.c xxhash.c fse_decompress.c entropy_common.c error_private.c pool.c zstd_kmalloc.c +.PATH: ${SYSDIR}/../contrib/zstd/lib/compress +SRCS+= zstd_compress.c zstdmt_compress.c fse_compress.c huf_compress.c +.PATH: ${SYSDIR}/../contrib/zstd/lib/decompress +SRCS+= zstd_decompress.c huf_decompress.c + .PATH: ${SUNW}/common/zfs .include "${SUNW}/uts/common/Makefile.files" .PATH: ${SUNW}/uts/common/fs/zfs