Index: Makefile.inc1 =================================================================== --- Makefile.inc1 +++ Makefile.inc1 @@ -2255,7 +2255,7 @@ ${_cddl_lib_libctf} \ lib/libutil lib/libpjdlog ${_lib_libypclnt} lib/libz lib/msun \ ${_secure_lib_libcrypto} ${_lib_libldns} \ - ${_secure_lib_libssh} ${_secure_lib_libssl} + ${_secure_lib_libssh} ${_secure_lib_libssl} lib/libzstd .if ${MK_GNUCXX} != "no" _prebuild_libs+= gnu/lib/libstdc++ gnu/lib/libsupc++ Index: cddl/lib/libzfs/Makefile =================================================================== --- cddl/lib/libzfs/Makefile +++ cddl/lib/libzfs/Makefile @@ -6,7 +6,7 @@ .PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libzfs/common LIB= zfs -LIBADD= md pthread umem util uutil m avl bsdxml geom nvpair z zfs_core +LIBADD= md pthread umem util uutil m avl bsdxml geom nvpair z zfs_core zstd SRCS= deviceid.c \ fsshare.c \ mkdirp.c \ Index: cddl/lib/libzpool/Makefile =================================================================== --- cddl/lib/libzpool/Makefile +++ cddl/lib/libzpool/Makefile @@ -54,8 +54,9 @@ CFLAGS+= -I${SRCTOP}/lib/libpthread/thread CFLAGS+= -I${SRCTOP}/lib/libpthread/sys CFLAGS+= -I${SRCTOP}/lib/libthr/arch/${MACHINE_CPUARCH}/include +CFLAGS+= -I${.CURDIR}/../../../contrib/zstd/lib -LIBADD= md pthread z nvpair avl umem +LIBADD= md pthread z nvpair avl umem zstd # atomic.S doesn't like profiling. MK_PROFILE= no Index: contrib/zstd/lib/common/error_private.h =================================================================== --- contrib/zstd/lib/common/error_private.h +++ contrib/zstd/lib/common/error_private.h @@ -20,7 +20,11 @@ /* **************************************** * Dependencies ******************************************/ +#ifdef _KERNEL +#include /* size_t */ +#else #include /* size_t */ +#endif #include "zstd_errors.h" /* enum list */ Index: contrib/zstd/lib/common/fse.h =================================================================== --- contrib/zstd/lib/common/fse.h +++ contrib/zstd/lib/common/fse.h @@ -42,7 +42,11 @@ /*-***************************************** * Dependencies ******************************************/ +#ifdef _KERNEL +#include /* size_t, ptrdiff_t */ +#else #include /* size_t, ptrdiff_t */ +#endif /*-***************************************** Index: contrib/zstd/lib/common/fse_decompress.c =================================================================== --- contrib/zstd/lib/common/fse_decompress.c +++ contrib/zstd/lib/common/fse_decompress.c @@ -57,8 +57,13 @@ /* ************************************************************** * Includes ****************************************************************/ +#ifdef _KERNEL +#include /* malloc, free */ +#include /* memset */ +#else #include /* malloc, free, qsort */ #include /* memcpy, memset */ +#endif #include "bitstream.h" #define FSE_STATIC_LINKING_ONLY #include "fse.h" @@ -101,12 +106,20 @@ FSE_DTable* FSE_createDTable (unsigned tableLog) { if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; +#ifdef _KERNEL + return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32), M_TEMP, M_WAITOK); +#else return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +#endif } void FSE_freeDTable (FSE_DTable* dt) { +#ifdef _KERNEL + free(dt, M_TEMP); +#else free(dt); +#endif } size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) Index: contrib/zstd/lib/common/huf.h =================================================================== --- contrib/zstd/lib/common/huf.h +++ contrib/zstd/lib/common/huf.h @@ -40,7 +40,11 @@ /* *** Dependencies *** */ +#ifdef _KERNEL +#include /* size_t */ +#else #include /* size_t */ +#endif /* *** library symbols visibility *** */ Index: contrib/zstd/lib/common/mem.h =================================================================== --- contrib/zstd/lib/common/mem.h +++ contrib/zstd/lib/common/mem.h @@ -17,8 +17,13 @@ /*-**************************************** * Dependencies ******************************************/ +#ifdef _KERNEL +#include /* size_t */ +#include /* memcpy */ +#else #include /* size_t, ptrdiff_t */ #include /* memcpy */ +#endif /*-**************************************** @@ -47,7 +52,9 @@ * Basic Types *****************************************************************/ #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +#ifndef _KERNEL # include +#endif typedef uint8_t BYTE; typedef uint16_t U16; typedef int16_t S16; Index: contrib/zstd/lib/common/pool.h =================================================================== --- contrib/zstd/lib/common/pool.h +++ contrib/zstd/lib/common/pool.h @@ -14,7 +14,11 @@ #endif +#ifdef _KERNEL +#include +#else #include /* size_t */ +#endif typedef struct POOL_ctx_s POOL_ctx; Index: contrib/zstd/lib/common/pool.c =================================================================== --- contrib/zstd/lib/common/pool.c +++ contrib/zstd/lib/common/pool.c @@ -9,8 +9,13 @@ /* ====== Dependencies ======= */ +#ifdef _KERNEL +#include +#include +#else #include /* size_t */ #include /* malloc, calloc, free */ +#endif #include "pool.h" /* ====== Compiler specifics ====== */ Index: contrib/zstd/lib/common/xxhash.h =================================================================== --- contrib/zstd/lib/common/xxhash.h +++ contrib/zstd/lib/common/xxhash.h @@ -75,7 +75,11 @@ /* **************************** * Definitions ******************************/ +#ifdef _KERNEL +#include /* size_t */ +#else #include /* size_t */ +#endif typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; Index: contrib/zstd/lib/common/xxhash.c =================================================================== --- contrib/zstd/lib/common/xxhash.c +++ contrib/zstd/lib/common/xxhash.c @@ -97,11 +97,20 @@ ***************************************/ /* Modify the local functions below should you wish to use some other memory routines */ /* for malloc(), free() */ +#ifdef _KERNEL +#include +#include +static void* XXH_malloc(size_t s) { return malloc(s, M_TEMP, M_WAITOK); } +static void XXH_free (void* p) { free(p, M_TEMP); } +/* for memcpy() */ +#include +#else #include static void* XXH_malloc(size_t s) { return malloc(s); } static void XXH_free (void* p) { free(p); } /* for memcpy() */ #include +#endif static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } #ifndef XXH_STATIC_LINKING_ONLY @@ -135,7 +144,9 @@ #ifndef MEM_MODULE # define MEM_MODULE # if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +#ifndef _KERNEL # include +#endif typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; Index: contrib/zstd/lib/common/zstd_common.c =================================================================== --- contrib/zstd/lib/common/zstd_common.c +++ contrib/zstd/lib/common/zstd_common.c @@ -12,7 +12,9 @@ /*-************************************* * Dependencies ***************************************/ +#ifndef _KERNEL #include /* malloc */ +#endif #include "error_private.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ @@ -50,15 +52,23 @@ /* default uses stdlib */ void* ZSTD_defaultAllocFunction(void* opaque, size_t size) { +#ifdef _KERNEL + void* address = zstd_alloc(opaque, size); +#else void* address = malloc(size); (void)opaque; +#endif return address; } void ZSTD_defaultFreeFunction(void* opaque, void* address) { +#ifdef _KERNEL + zstd_free(opaque, address); +#else (void)opaque; free(address); +#endif } void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) Index: contrib/zstd/lib/common/zstd_errors.h =================================================================== --- contrib/zstd/lib/common/zstd_errors.h +++ contrib/zstd/lib/common/zstd_errors.h @@ -15,7 +15,11 @@ #endif /*===== dependency =====*/ +#ifdef _KERNEL +#include /* size_t */ +#else #include /* size_t */ +#endif /* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ Index: contrib/zstd/lib/compress/fse_compress.c =================================================================== --- contrib/zstd/lib/compress/fse_compress.c +++ contrib/zstd/lib/compress/fse_compress.c @@ -56,9 +56,14 @@ /* ************************************************************** * Includes ****************************************************************/ +#ifdef _KERNEL +#include /* malloc, free */ +#include /* memcpy, memset */ +#else #include /* malloc, free, qsort */ #include /* memcpy, memset */ #include /* printf (debug) */ +#endif #include "bitstream.h" #define FSE_STATIC_LINKING_ONLY #include "fse.h" @@ -468,10 +473,20 @@ size_t size; if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); +#ifdef _KERNEL + return (FSE_CTable*)malloc(size, M_TEMP, M_WAITOK); +#else return (FSE_CTable*)malloc(size); +#endif } -void FSE_freeCTable (FSE_CTable* ct) { free(ct); } +void FSE_freeCTable (FSE_CTable* ct) { +#ifdef _KERNEL + free(ct, M_TEMP); +#else + free(ct); +#endif +} /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) Index: contrib/zstd/lib/compress/huf_compress.c =================================================================== --- contrib/zstd/lib/compress/huf_compress.c +++ contrib/zstd/lib/compress/huf_compress.c @@ -43,8 +43,12 @@ /* ************************************************************** * Includes ****************************************************************/ +#ifdef _KERNEL +#include /* memcpy, memset */ +#else #include /* memcpy, memset */ #include /* printf (debug) */ +#endif #include "bitstream.h" #define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ #include "fse.h" /* header compression */ Index: contrib/zstd/lib/compress/zstd_compress.c =================================================================== --- contrib/zstd/lib/compress/zstd_compress.c +++ contrib/zstd/lib/compress/zstd_compress.c @@ -11,7 +11,11 @@ /*-************************************* * Dependencies ***************************************/ +#ifdef _KERNEL +#include /* memcpy, memset */ +#else #include /* memset */ +#endif #include "mem.h" #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ #include "fse.h" Index: contrib/zstd/lib/decompress/huf_decompress.c =================================================================== --- contrib/zstd/lib/decompress/huf_decompress.c +++ contrib/zstd/lib/decompress/huf_decompress.c @@ -54,7 +54,11 @@ /* ************************************************************** * Dependencies ****************************************************************/ +#ifdef _KERNEL +#include /* memcpy, memset */ +#else #include /* memcpy, memset */ +#endif #include "bitstream.h" /* BIT_* */ #include "fse.h" /* header compression */ #define HUF_STATIC_LINKING_ONLY Index: contrib/zstd/lib/decompress/zstd_decompress.c =================================================================== --- contrib/zstd/lib/decompress/zstd_decompress.c +++ contrib/zstd/lib/decompress/zstd_decompress.c @@ -41,7 +41,11 @@ /*-******************************************************* * Dependencies *********************************************************/ +#ifdef _KERNEL +#include /* memcpy, memset */ +#else #include /* memcpy, memmove, memset */ +#endif #include "mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY #include "fse.h" Index: contrib/zstd/lib/zstd.h =================================================================== --- contrib/zstd/lib/zstd.h +++ contrib/zstd/lib/zstd.h @@ -15,7 +15,13 @@ #define ZSTD_H_235446 /* ====== Dependency ======*/ +#ifdef _KERNEL +#include /* size_t */ +extern void *zstd_alloc(void *opaque, size_t size); +extern void zstd_free(void *opaque, void *ptr); +#else #include /* size_t */ +#endif /* ===== ZSTDLIB_API : control library symbols visibility ===== */ Index: rescue/rescue/Makefile =================================================================== --- rescue/rescue/Makefile +++ rescue/rescue/Makefile @@ -129,7 +129,7 @@ CRUNCH_LIBS+= -l80211 -lalias -lcam -lncursesw -ldevstat -lipsec -llzma .if ${MK_ZFS} != "no" -CRUNCH_LIBS+= -lavl -lzpool -lzfs_core -lzfs -lnvpair -lpthread -luutil -lumem +CRUNCH_LIBS+= -lavl -lzpool -lzfs_core -lzfs -lnvpair -lpthread -luutil -lumem -lprivatezstd .else # liblzma needs pthread CRUNCH_LIBS+= -lpthread Index: share/mk/src.libnames.mk =================================================================== --- share/mk/src.libnames.mk +++ share/mk/src.libnames.mk @@ -325,9 +325,9 @@ _DP_fifolog= z _DP_ipf= kvm _DP_zfs= md pthread umem util uutil m nvpair avl bsdxml geom nvpair z \ - zfs_core + zfs_core zstd _DP_zfs_core= nvpair -_DP_zpool= md pthread z nvpair avl umem +_DP_zpool= md pthread z nvpair avl umem zstd .if ${MK_OFED} != "no" _DP_cxgb4= ibverbs pthread _DP_ibcm= ibverbs Index: sys/boot/zfs/zfsimpl.c =================================================================== --- sys/boot/zfs/zfsimpl.c +++ sys/boot/zfs/zfsimpl.c @@ -60,6 +60,7 @@ "org.open-zfs:large_blocks", "org.illumos:sha512", "org.illumos:skein", + "org.freebsd:zstd_compress", NULL }; Index: sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.h =================================================================== --- sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.h +++ sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.h @@ -57,6 +57,7 @@ #ifdef illumos SPA_FEATURE_EDONR, #endif + SPA_FEATURE_ZSTD_COMPRESS, SPA_FEATURES } spa_feature_t; Index: sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c =================================================================== --- sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c +++ sys/cddl/contrib/opensolaris/common/zfs/zfeature_common.c @@ -247,4 +247,10 @@ "Edon-R hash algorithm.", ZFEATURE_FLAG_PER_DATASET, NULL); #endif + + zfeature_register(SPA_FEATURE_ZSTD_COMPRESS, + "org.freebsd:zstd_compress", "zstd_compress", + "zstd compression algorithm support.", + ZFEATURE_FLAG_PER_DATASET, NULL); + } Index: sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c =================================================================== --- sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c +++ sys/cddl/contrib/opensolaris/common/zfs/zfs_prop.c @@ -116,6 +116,9 @@ { "gzip-9", ZIO_COMPRESS_GZIP_9 }, { "zle", ZIO_COMPRESS_ZLE }, { "lz4", ZIO_COMPRESS_LZ4 }, + { "zstd-min", ZIO_COMPRESS_ZSTD_MIN }, + { "zstd", ZIO_COMPRESS_ZSTD_DEFAULT }, /* zstd default */ + { "zstd-max", ZIO_COMPRESS_ZSTD_MAX }, { NULL } }; @@ -250,7 +253,8 @@ zprop_register_index(ZFS_PROP_COMPRESSION, "compression", ZIO_COMPRESS_DEFAULT, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, - "on | off | lzjb | gzip | gzip-[1-9] | zle | lz4", + "on | off | lzjb | gzip | gzip-[1-9] | zle | lz4" + " | zstd | zstd-min | zstd-max", "COMPRESS", compress_table); zprop_register_index(ZFS_PROP_SNAPDIR, "snapdir", ZFS_SNAPDIR_HIDDEN, PROP_INHERIT, ZFS_TYPE_FILESYSTEM, Index: sys/cddl/contrib/opensolaris/uts/common/Makefile.files =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/Makefile.files +++ sys/cddl/contrib/opensolaris/uts/common/Makefile.files @@ -110,7 +110,8 @@ zio_compress.o \ zio_inject.o \ zle.o \ - zrlock.o + zrlock.o \ + zstd.o ZFS_SHARED_OBJS += \ zfeature_common.o \ Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c @@ -2017,6 +2017,7 @@ metaslab_alloc_trace_init(); zio_init(); lz4_init(); + zstd_init(); dmu_init(); zil_init(); vdev_cache_stat_init(); @@ -2043,6 +2044,7 @@ zil_fini(); dmu_fini(); lz4_fini(); + zstd_fini(); zio_fini(); metaslab_alloc_trace_fini(); range_tree_fini(); Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_compress.h =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_compress.h +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_compress.h @@ -54,9 +54,16 @@ ZIO_COMPRESS_GZIP_9, ZIO_COMPRESS_ZLE, ZIO_COMPRESS_LZ4, + ZIO_COMPRESS_ZSTD_MIN, + ZIO_COMPRESS_ZSTD_DEFAULT, + ZIO_COMPRESS_ZSTD_MAX, ZIO_COMPRESS_FUNCTIONS }; +#define ZIO_ZSTD_LEVEL_MIN 1 +#define ZIO_ZSTD_LEVEL_DEFAULT 3 +#define ZIO_ZSTD_LEVEL_MAX 19 + /* Common signature for all zio compress functions. */ typedef size_t zio_compress_func_t(void *src, void *dst, size_t s_len, size_t d_len, int); @@ -104,6 +111,14 @@ int level); extern int lz4_decompress(void *src, void *dst, size_t s_len, size_t d_len, int level); +extern void zstd_init(void); +extern void zstd_fini(void); +extern size_t zstd_compress(void *src, void *dst, size_t s_len, size_t d_len, + int level); +extern int zstd_decompress(void *src, void *dst, size_t s_len, size_t d_len, + int level); +extern void *zstd_alloc(void *opaque, size_t size); +extern void zstd_free(void *opaque, void *ptr); /* * Compress and decompress data if necessary. Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_impl.h =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_impl.h +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio_impl.h @@ -76,9 +76,9 @@ * the supported transformations: * * Compression: - * ZFS supports three different flavors of compression -- gzip, lzjb, and - * zle. Compression occurs as part of the write pipeline and is performed - * in the ZIO_STAGE_WRITE_BP_INIT stage. + * ZFS supports five different flavors of compression -- gzip, lzjb, lz4, zle, + * and ZStandard. Compression occurs as part of the write pipeline and is + * performed in the ZIO_STAGE_WRITE_BP_INIT stage. * * Dedup: * Dedup reads are handled by the ZIO_STAGE_DDT_READ_START and Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c @@ -3974,6 +3974,21 @@ spa_close(spa, FTAG); } + if (intval >= ZIO_COMPRESS_ZSTD_MIN && + intval <= ZIO_COMPRESS_ZSTD_MAX) { + spa_t *spa; + + if ((err = spa_open(dsname, &spa, FTAG)) != 0) + return (err); + + if (!spa_feature_is_enabled(spa, + SPA_FEATURE_ZSTD_COMPRESS)) { + spa_close(spa, FTAG); + return (SET_ERROR(ENOTSUP)); + } + spa_close(spa, FTAG); + } + /* * If this is a bootable dataset then * verify that the compression algorithm Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c =================================================================== --- sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio_compress.c @@ -74,7 +74,10 @@ {"gzip-8", 8, gzip_compress, gzip_decompress}, {"gzip-9", 9, gzip_compress, gzip_decompress}, {"zle", 64, zle_compress, zle_decompress}, - {"lz4", 0, lz4_compress, lz4_decompress} + {"lz4", 0, lz4_compress, lz4_decompress}, + {"zstd-min", ZIO_ZSTD_LEVEL_MIN, zstd_compress, zstd_decompress}, + {"zstd", ZIO_ZSTD_LEVEL_DEFAULT, zstd_compress, zstd_decompress}, + {"zstd-max", ZIO_ZSTD_LEVEL_MAX, zstd_compress, zstd_decompress}, }; enum zio_compress Index: sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zstd.c =================================================================== --- /dev/null +++ sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zstd.c @@ -0,0 +1,330 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2016 ScaleEngine, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include + +#define ZSTD_STATIC_LINKING_ONLY +#include + +struct uma_zone { + struct mtx_padalign uz_lock; /* Lock for the zone */ + struct mtx_padalign *uz_lockptr; + const char *uz_name; /* Text name of the zone */ +}; + +/* FreeBSD: Use heap for all platforms for now */ +#define HEAPMODE 1 + +#define ZSTD_KMEM_MAGIC 0x20160831 + +/* + * XXX: TODO: Investigate using ZSTD_compressBlock for small blocks + * XXX: TODO: use kmem_cache like lz4.c does + */ + +static size_t real_zstd_compress(const char *source, char *dest, int isize, + int osize, int level); +static size_t real_zstd_decompress(const char *source, char *dest, int isize, + int maxosize); + +void *zstd_alloc(void *opaque, size_t size); +void *zstd_calloc(void *opaque, size_t number, size_t size); +void zstd_free(void *opaque, void *ptr); + +static const ZSTD_customMem zstd_malloc = { + zstd_alloc, + /* requires 1.3.0: zstd_calloc, */ + zstd_free, + NULL, +}; + +enum zstd_kmem_type { + ZSTD_KMEM_OPAQUE = 0, + ZSTD_KMEM_CCTX_MIN, + ZSTD_KMEM_CCTX_DEFAULT, + ZSTD_KMEM_CCTX_MAX, + ZSTD_KMEM_DCTX, + ZSTD_KMEM_COUNT, +}; + +struct zstd_kmem { + uint_t kmem_magic; + enum zstd_kmem_type kmem_type; + uint_t kmem_size; +}; + +static kmem_cache_t *zstd_kmem_cache[ZSTD_KMEM_COUNT] = { NULL }; +static size_t zcctx_min, zcctx_default, zcctx_max, zdctx; + +size_t +zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) +{ + uint32_t bufsiz; + char *dest = d_start; + + ASSERT(d_len >= sizeof (bufsiz)); + ASSERT(d_len <= s_len); + + /* XXX: this could overflow, but we never have blocks that big */ + bufsiz = real_zstd_compress(s_start, &dest[sizeof (bufsiz)], s_len, + d_len - sizeof (bufsiz), n); + + /* Signal an error if the compression routine returned an error. */ + if (ZSTD_isError(bufsiz)) + return (s_len); + + /* + * Encode the compresed buffer size at the start. We'll need this in + * decompression to counter the effects of padding which might be + * added to the compressed buffer and which, if unhandled, would + * confuse the hell out of our decompression function. + */ + *(uint32_t *)dest = BE_32(bufsiz); + + return (bufsiz + sizeof (bufsiz)); +} + +int +zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) +{ + const char *src = s_start; + uint32_t bufsiz = BE_IN32(src); + + ASSERT(d_len >= s_len); + + /* invalid compressed buffer size encoded at start */ + if (bufsiz + sizeof (bufsiz) > s_len) + return (1); + + /* + * Returns 0 on success (decompression function returned non-negative) + * and non-zero on failure (decompression function returned negative. + */ + if (ZSTD_isError(real_zstd_decompress(&src[sizeof (bufsiz)], d_start, + bufsiz, d_len))) + return (-1); + + return (0); +} + +static size_t +real_zstd_compress(const char *source, char *dest, int isize, int osize, + int level) +{ +#if HEAPMODE + size_t result; + ZSTD_CCtx *cctx; + + /* XXX: In ZSTD 1.3+ use ZSTD_initStaticCCtx() instead */ + /* XXX: Could we reuse one cctx per thread? instead of alloc/free */ + cctx = ZSTD_createCCtx_advanced(zstd_malloc); + /* + * out of kernel memory, gently fall through - this will disable + * compression in zio_compress_data + */ + if (cctx == NULL) + return (-1); + + result = ZSTD_compressCCtx(cctx, dest, osize, source, isize, level); + + ZSTD_freeCCtx(cctx); + return (result); +#else + ZSTD_CCtx ctx; + void *ctx; + + return (ZSTD_compressCCtx(&ctx, dest, osize, source, isize, level)); +#endif +} + + +static size_t +real_zstd_decompress(const char *source, char *dest, int isize, int maxosize) +{ +#if HEAPMODE + size_t result; + ZSTD_DCtx *dctx; + + dctx = ZSTD_createDCtx_advanced(zstd_malloc); + if (dctx == NULL) + return (-1); + + result = ZSTD_decompressDCtx(dctx, dest, maxosize, source, isize); + + ZSTD_freeDCtx(dctx); + return (result); +#else + ZSTD_DCtx dctx; + + return (ZSTD_decompressDCtx(&dctx, dest, maxosize, source, isize)); +#endif +} + +extern void * +zstd_alloc(void *opaque, size_t size) +{ + size_t nbytes = sizeof(struct zstd_kmem) + size; + struct zstd_kmem *z; + enum zstd_kmem_type type; + + if (opaque != NULL) { + z = kmem_cache_alloc(opaque, KM_NOSLEEP | M_ZERO); + type = ZSTD_KMEM_OPAQUE; + } else if (size == zcctx_min) { + z = kmem_cache_alloc(zstd_kmem_cache[ZSTD_KMEM_CCTX_MIN], + KM_NOSLEEP | M_ZERO); + type = ZSTD_KMEM_CCTX_MIN; + } else if (size == zcctx_default) { + z = kmem_cache_alloc(zstd_kmem_cache[ZSTD_KMEM_CCTX_DEFAULT], + KM_NOSLEEP | M_ZERO); + type = ZSTD_KMEM_CCTX_DEFAULT; + } else if (size == zcctx_max) { + z = kmem_cache_alloc(zstd_kmem_cache[ZSTD_KMEM_CCTX_MAX], + KM_NOSLEEP | M_ZERO); + type = ZSTD_KMEM_CCTX_MAX; + } else if (size == zdctx) { + z = kmem_cache_alloc(zstd_kmem_cache[ZSTD_KMEM_DCTX], + KM_NOSLEEP | M_ZERO); + type = ZSTD_KMEM_DCTX; + } else { + z = kmem_alloc(nbytes, KM_NOSLEEP | M_ZERO); + type = ZSTD_KMEM_OPAQUE; + } + if (z == NULL) + return (NULL); + + z->kmem_magic = ZSTD_KMEM_MAGIC; + z->kmem_type = type; + z->kmem_size = nbytes; + + return ((void*)z + (sizeof(struct zstd_kmem))); +} + +#if 0 +extern void * +zstd_calloc(void *opaque, size_t number, size_t size) +{ + size_t nbytes = sizeof(struct zstd_kmem) + (number * size); + struct zstd_kmem *z; + enum zstd_kmem_type type; + + if (opaque != NULL) { + z = kmem_cache_alloc(opaque, KM_NOSLEEP | M_ZERO); + type = ZSTD_KMEM_OPAQUE; + } else { + z = kmem_alloc(nbytes, KM_NOSLEEP | M_ZERO); + type = ZSTD_KMEM_OPAQUE; + } + + if (z == NULL) + return (NULL); + + z->kmem_magic = ZSTD_KMEM_MAGIC; + z->kmem_type = type; + z->kmem_size = nbytes; + + return (z + sizeof(struct zstd_kmem)); +} +#endif + +extern void +zstd_free(void *opaque, void *ptr) +{ + struct zstd_kmem *z = ptr - sizeof(struct zstd_kmem); + +if (z->kmem_magic != ZSTD_KMEM_MAGIC) { + printf("%s:%u: ERROR!: magic=%x opaque=%p, ptr=%p z=%p\n", __func__, __LINE__, z->kmem_magic, opaque, ptr, z); +} + ASSERT(z->kmem_magic == ZSTD_KMEM_MAGIC); + + switch (z->kmem_type) { + case ZSTD_KMEM_CCTX_MIN: + case ZSTD_KMEM_CCTX_DEFAULT: + case ZSTD_KMEM_CCTX_MAX: + case ZSTD_KMEM_DCTX: + kmem_cache_free(zstd_kmem_cache[z->kmem_type], z); + break; + case ZSTD_KMEM_OPAQUE: + if (opaque != NULL) { + kmem_cache_free(opaque, z); + } else { + kmem_free(z, z->kmem_size); + } + break; + default: + panic("%s:%u: unrecognized memory allocation type: %d", + __func__, __LINE__, z->kmem_type); + break; + } +} + +extern void +zstd_init(void) +{ + + zcctx_min = ZSTD_estimateCCtxSize( + ZSTD_getCParams(ZIO_ZSTD_LEVEL_MIN, 0, 0)); + zcctx_default = ZSTD_estimateCCtxSize( + ZSTD_getCParams(ZIO_ZSTD_LEVEL_DEFAULT, 0, 0)); + zcctx_max = ZSTD_estimateCCtxSize( + ZSTD_getCParams(ZIO_ZSTD_LEVEL_MAX, 0, 0)); + zdctx = ZSTD_estimateDCtxSize(); +printf("%s:%u: min=%zu, default=%zu, max=%zu, dctx=%zu\n", + __func__, __LINE__, zcctx_min, zcctx_default, zcctx_max, zdctx); +#if HEAPMODE + zstd_kmem_cache[ZSTD_KMEM_CCTX_MIN] = kmem_cache_create( + "zstd_cctx_min", + roundup2(zcctx_min + sizeof(struct zstd_kmem), PAGE_SIZE), + 0, NULL, NULL, NULL, NULL, NULL, 0); + zstd_kmem_cache[ZSTD_KMEM_CCTX_DEFAULT] = kmem_cache_create( + "zstd_cctx_default", + roundup2(zcctx_default + sizeof(struct zstd_kmem), PAGE_SIZE), + 0, NULL, NULL, NULL, NULL, NULL, 0); + zstd_kmem_cache[ZSTD_KMEM_CCTX_MAX] = kmem_cache_create( + "zstd_cctx_max", + roundup2(zcctx_max + sizeof(struct zstd_kmem), PAGE_SIZE), + 0, NULL, NULL, NULL, NULL, NULL, 0); + zstd_kmem_cache[ZSTD_KMEM_DCTX] = kmem_cache_create( + "zstd_dctx", + roundup2(zdctx + sizeof(struct zstd_kmem), PAGE_SIZE), + 0, NULL, NULL, NULL, NULL, NULL, 0); + +#endif +} + +extern void +zstd_fini(void) +{ + +#if HEAPMODE + kmem_cache_destroy(zstd_kmem_cache[ZSTD_KMEM_CCTX_MIN]); + kmem_cache_destroy(zstd_kmem_cache[ZSTD_KMEM_CCTX_DEFAULT]); + kmem_cache_destroy(zstd_kmem_cache[ZSTD_KMEM_CCTX_MAX]); + kmem_cache_destroy(zstd_kmem_cache[ZSTD_KMEM_DCTX]); +#endif +} Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -250,6 +250,7 @@ cddl/contrib/opensolaris/uts/common/fs/zfs/zio_inject.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zle.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zrlock.c optional zfs compile-with "${ZFS_C}" +cddl/contrib/opensolaris/uts/common/fs/zfs/zstd.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/callb.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/fm.c optional zfs compile-with "${ZFS_C}" Index: sys/modules/zfs/Makefile =================================================================== --- sys/modules/zfs/Makefile +++ sys/modules/zfs/Makefile @@ -72,6 +72,17 @@ .PATH: ${SYSDIR}/crypto/skein SRCS+= skein.c skein_block.c +CFLAGS+=-DZSTD_HEAPMODE=1 +.PATH: ${SYSDIR}/../contrib/zstd/lib +CFLAGS+=-I${SYSDIR}/../contrib/zstd/lib +.PATH: ${SYSDIR}/../contrib/zstd/lib/common +CFLAGS+=-I${SYSDIR}/../contrib/zstd/lib/common +SRCS+= zstd_common.c xxhash.c fse_decompress.c entropy_common.c error_private.c +.PATH: ${SYSDIR}/../contrib/zstd/lib/compress +SRCS+= zstd_compress.c fse_compress.c huf_compress.c +.PATH: ${SYSDIR}/../contrib/zstd/lib/decompress +SRCS+= zstd_decompress.c huf_decompress.c + .PATH: ${SUNW}/common/zfs .include "${SUNW}/uts/common/Makefile.files" .PATH: ${SUNW}/uts/common/fs/zfs