diff --git a/include/sys/ddt.h b/include/sys/ddt.h index e6bbaa10e9ac..a15354057683 100644 --- a/include/sys/ddt.h +++ b/include/sys/ddt.h @@ -1,252 +1,253 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or https://opensource.org/licenses/CDDL-1.0. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016 by Delphix. All rights reserved. */ #ifndef _SYS_DDT_H #define _SYS_DDT_H #include #include #include #include #include #ifdef __cplusplus extern "C" { #endif struct abd; /* * On-disk DDT formats, in the desired search order (newest version first). */ enum ddt_type { DDT_TYPE_ZAP = 0, DDT_TYPES }; /* * DDT classes, in the desired search order (highest replication level first). */ enum ddt_class { DDT_CLASS_DITTO = 0, DDT_CLASS_DUPLICATE, DDT_CLASS_UNIQUE, DDT_CLASSES }; #define DDT_TYPE_CURRENT 0 /* * On-disk ddt entry: key (name) and physical storage (value). */ typedef struct ddt_key { zio_cksum_t ddk_cksum; /* 256-bit block checksum */ /* * Encoded with logical & physical size, encryption, and compression, * as follows: * +-------+-------+-------+-------+-------+-------+-------+-------+ * | 0 | 0 | 0 |X| comp| PSIZE | LSIZE | * +-------+-------+-------+-------+-------+-------+-------+-------+ */ uint64_t ddk_prop; } ddt_key_t; #define DDK_GET_LSIZE(ddk) \ BF64_GET_SB((ddk)->ddk_prop, 0, 16, SPA_MINBLOCKSHIFT, 1) #define DDK_SET_LSIZE(ddk, x) \ BF64_SET_SB((ddk)->ddk_prop, 0, 16, SPA_MINBLOCKSHIFT, 1, x) #define DDK_GET_PSIZE(ddk) \ BF64_GET_SB((ddk)->ddk_prop, 16, 16, SPA_MINBLOCKSHIFT, 1) #define DDK_SET_PSIZE(ddk, x) \ BF64_SET_SB((ddk)->ddk_prop, 16, 16, SPA_MINBLOCKSHIFT, 1, x) #define DDK_GET_COMPRESS(ddk) BF64_GET((ddk)->ddk_prop, 32, 7) #define DDK_SET_COMPRESS(ddk, x) BF64_SET((ddk)->ddk_prop, 32, 7, x) #define DDK_GET_CRYPT(ddk) BF64_GET((ddk)->ddk_prop, 39, 1) #define DDK_SET_CRYPT(ddk, x) BF64_SET((ddk)->ddk_prop, 39, 1, x) #define DDE_GET_NDVAS(dde) (DDK_GET_CRYPT(&dde->dde_key) \ ? SPA_DVAS_PER_BP - 1 : SPA_DVAS_PER_BP) typedef struct ddt_phys { dva_t ddp_dva[SPA_DVAS_PER_BP]; uint64_t ddp_refcnt; uint64_t ddp_phys_birth; } ddt_phys_t; /* * Note, we no longer generate new DDT_PHYS_DITTO-type blocks. However, * we maintain the ability to free existing dedup-ditto blocks. */ enum ddt_phys_type { DDT_PHYS_DITTO = 0, DDT_PHYS_SINGLE = 1, DDT_PHYS_DOUBLE = 2, DDT_PHYS_TRIPLE = 3, DDT_PHYS_TYPES }; /* * In-core ddt entry */ struct ddt_entry { /* key must be first for ddt_key_compare */ ddt_key_t dde_key; ddt_phys_t dde_phys[DDT_PHYS_TYPES]; zio_t *dde_lead_zio[DDT_PHYS_TYPES]; struct abd *dde_repair_abd; enum ddt_type dde_type; enum ddt_class dde_class; uint8_t dde_loading; uint8_t dde_loaded; kcondvar_t dde_cv; avl_node_t dde_node; }; /* * In-core ddt */ struct ddt { kmutex_t ddt_lock; avl_tree_t ddt_tree; avl_tree_t ddt_repair_tree; enum zio_checksum ddt_checksum; spa_t *ddt_spa; objset_t *ddt_os; uint64_t ddt_stat_object; uint64_t ddt_object[DDT_TYPES][DDT_CLASSES]; ddt_histogram_t ddt_histogram[DDT_TYPES][DDT_CLASSES]; ddt_histogram_t ddt_histogram_cache[DDT_TYPES][DDT_CLASSES]; ddt_object_t ddt_object_stats[DDT_TYPES][DDT_CLASSES]; avl_node_t ddt_node; }; /* * In-core and on-disk bookmark for DDT walks */ typedef struct ddt_bookmark { uint64_t ddb_class; uint64_t ddb_type; uint64_t ddb_checksum; uint64_t ddb_cursor; } ddt_bookmark_t; /* * Ops vector to access a specific DDT object type. */ typedef struct ddt_ops { char ddt_op_name[32]; int (*ddt_op_create)(objset_t *os, uint64_t *object, dmu_tx_t *tx, boolean_t prehash); int (*ddt_op_destroy)(objset_t *os, uint64_t object, dmu_tx_t *tx); int (*ddt_op_lookup)(objset_t *os, uint64_t object, ddt_entry_t *dde); void (*ddt_op_prefetch)(objset_t *os, uint64_t object, ddt_entry_t *dde); int (*ddt_op_update)(objset_t *os, uint64_t object, ddt_entry_t *dde, dmu_tx_t *tx); int (*ddt_op_remove)(objset_t *os, uint64_t object, ddt_entry_t *dde, dmu_tx_t *tx); int (*ddt_op_walk)(objset_t *os, uint64_t object, ddt_entry_t *dde, uint64_t *walk); int (*ddt_op_count)(objset_t *os, uint64_t object, uint64_t *count); } ddt_ops_t; #define DDT_NAMELEN 107 extern void ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class clazz, char *name); extern int ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class clazz, uint64_t *walk, ddt_entry_t *dde); extern int ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class clazz, uint64_t *count); extern int ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class clazz, dmu_object_info_t *); extern boolean_t ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class clazz); extern void ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg); extern void ddt_bp_create(enum zio_checksum checksum, const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp); extern void ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp); extern void ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp); extern void ddt_phys_clear(ddt_phys_t *ddp); extern void ddt_phys_addref(ddt_phys_t *ddp); extern void ddt_phys_decref(ddt_phys_t *ddp); extern void ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg); extern ddt_phys_t *ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp); extern uint64_t ddt_phys_total_refcnt(const ddt_entry_t *dde); +extern void ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg); extern void ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg); extern void ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src); extern void ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh); extern boolean_t ddt_histogram_empty(const ddt_histogram_t *ddh); extern void ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo); extern void ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh); extern void ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total); extern uint64_t ddt_get_dedup_dspace(spa_t *spa); extern uint64_t ddt_get_pool_dedup_ratio(spa_t *spa); extern ddt_t *ddt_select(spa_t *spa, const blkptr_t *bp); extern void ddt_enter(ddt_t *ddt); extern void ddt_exit(ddt_t *ddt); extern void ddt_init(void); extern void ddt_fini(void); extern ddt_entry_t *ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add); extern void ddt_prefetch(spa_t *spa, const blkptr_t *bp); extern void ddt_remove(ddt_t *ddt, ddt_entry_t *dde); extern boolean_t ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp); extern ddt_entry_t *ddt_repair_start(ddt_t *ddt, const blkptr_t *bp); extern void ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde); extern int ddt_key_compare(const void *x1, const void *x2); extern void ddt_create(spa_t *spa); extern int ddt_load(spa_t *spa); extern void ddt_unload(spa_t *spa); extern void ddt_sync(spa_t *spa, uint64_t txg); extern int ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde); extern int ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class clazz, ddt_entry_t *dde, dmu_tx_t *tx); extern boolean_t ddt_addref(spa_t *spa, const blkptr_t *bp); extern const ddt_ops_t ddt_zap_ops; #ifdef __cplusplus } #endif #endif /* _SYS_DDT_H */ diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am index 3c986a707d2d..42f3404db5a9 100644 --- a/lib/libzpool/Makefile.am +++ b/lib/libzpool/Makefile.am @@ -1,214 +1,215 @@ libzpool_la_CFLAGS = $(AM_CFLAGS) $(KERNEL_CFLAGS) $(LIBRARY_CFLAGS) libzpool_la_CFLAGS += $(ZLIB_CFLAGS) libzpool_la_CPPFLAGS = $(AM_CPPFLAGS) $(FORCEDEBUG_CPPFLAGS) libzpool_la_CPPFLAGS += -I$(srcdir)/include/os/@ac_system_l@/zfs libzpool_la_CPPFLAGS += -DLIB_ZPOOL_BUILD lib_LTLIBRARIES += libzpool.la CPPCHECKTARGETS += libzpool.la dist_libzpool_la_SOURCES = \ %D%/kernel.c \ %D%/taskq.c \ %D%/util.c nodist_libzpool_la_SOURCES = \ module/lua/lapi.c \ module/lua/lauxlib.c \ module/lua/lbaselib.c \ module/lua/lcode.c \ module/lua/lcompat.c \ module/lua/lcorolib.c \ module/lua/lctype.c \ module/lua/ldebug.c \ module/lua/ldo.c \ module/lua/lfunc.c \ module/lua/lgc.c \ module/lua/llex.c \ module/lua/lmem.c \ module/lua/lobject.c \ module/lua/lopcodes.c \ module/lua/lparser.c \ module/lua/lstate.c \ module/lua/lstring.c \ module/lua/lstrlib.c \ module/lua/ltable.c \ module/lua/ltablib.c \ module/lua/ltm.c \ module/lua/lvm.c \ module/lua/lzio.c \ \ module/os/linux/zfs/abd_os.c \ module/os/linux/zfs/arc_os.c \ module/os/linux/zfs/trace.c \ module/os/linux/zfs/vdev_file.c \ module/os/linux/zfs/vdev_label_os.c \ module/os/linux/zfs/zfs_debug.c \ module/os/linux/zfs/zfs_racct.c \ module/os/linux/zfs/zfs_znode.c \ module/os/linux/zfs/zio_crypt.c \ \ module/zcommon/cityhash.c \ module/zcommon/zfeature_common.c \ module/zcommon/zfs_comutil.c \ module/zcommon/zfs_deleg.c \ module/zcommon/zfs_fletcher.c \ module/zcommon/zfs_fletcher_aarch64_neon.c \ module/zcommon/zfs_fletcher_avx512.c \ module/zcommon/zfs_fletcher_intel.c \ module/zcommon/zfs_fletcher_sse.c \ module/zcommon/zfs_fletcher_superscalar.c \ module/zcommon/zfs_fletcher_superscalar4.c \ module/zcommon/zfs_namecheck.c \ module/zcommon/zfs_prop.c \ module/zcommon/zpool_prop.c \ module/zcommon/zprop_common.c \ \ module/zfs/abd.c \ module/zfs/aggsum.c \ module/zfs/arc.c \ module/zfs/blake3_zfs.c \ module/zfs/blkptr.c \ module/zfs/bplist.c \ module/zfs/bpobj.c \ module/zfs/bptree.c \ module/zfs/bqueue.c \ module/zfs/btree.c \ module/zfs/brt.c \ module/zfs/dbuf.c \ module/zfs/dbuf_stats.c \ module/zfs/ddt.c \ + module/zfs/ddt_stats.c \ module/zfs/ddt_zap.c \ module/zfs/dmu.c \ module/zfs/dmu_diff.c \ module/zfs/dmu_object.c \ module/zfs/dmu_objset.c \ module/zfs/dmu_recv.c \ module/zfs/dmu_redact.c \ module/zfs/dmu_send.c \ module/zfs/dmu_traverse.c \ module/zfs/dmu_tx.c \ module/zfs/dmu_zfetch.c \ module/zfs/dnode.c \ module/zfs/dnode_sync.c \ module/zfs/dsl_bookmark.c \ module/zfs/dsl_crypt.c \ module/zfs/dsl_dataset.c \ module/zfs/dsl_deadlist.c \ module/zfs/dsl_deleg.c \ module/zfs/dsl_destroy.c \ module/zfs/dsl_dir.c \ module/zfs/dsl_pool.c \ module/zfs/dsl_prop.c \ module/zfs/dsl_scan.c \ module/zfs/dsl_synctask.c \ module/zfs/dsl_userhold.c \ module/zfs/edonr_zfs.c \ module/zfs/fm.c \ module/zfs/gzip.c \ module/zfs/hkdf.c \ module/zfs/lz4.c \ module/zfs/lz4_zfs.c \ module/zfs/lzjb.c \ module/zfs/metaslab.c \ module/zfs/mmp.c \ module/zfs/multilist.c \ module/zfs/objlist.c \ module/zfs/pathname.c \ module/zfs/range_tree.c \ module/zfs/refcount.c \ module/zfs/rrwlock.c \ module/zfs/sa.c \ module/zfs/sha2_zfs.c \ module/zfs/skein_zfs.c \ module/zfs/spa.c \ module/zfs/spa_checkpoint.c \ module/zfs/spa_config.c \ module/zfs/spa_errlog.c \ module/zfs/spa_history.c \ module/zfs/spa_log_spacemap.c \ module/zfs/spa_misc.c \ module/zfs/spa_stats.c \ module/zfs/space_map.c \ module/zfs/space_reftree.c \ module/zfs/txg.c \ module/zfs/uberblock.c \ module/zfs/unique.c \ module/zfs/vdev.c \ module/zfs/vdev_draid.c \ module/zfs/vdev_draid_rand.c \ module/zfs/vdev_indirect.c \ module/zfs/vdev_indirect_births.c \ module/zfs/vdev_indirect_mapping.c \ module/zfs/vdev_initialize.c \ module/zfs/vdev_label.c \ module/zfs/vdev_mirror.c \ module/zfs/vdev_missing.c \ module/zfs/vdev_queue.c \ module/zfs/vdev_raidz.c \ module/zfs/vdev_raidz_math.c \ module/zfs/vdev_raidz_math_aarch64_neon.c \ module/zfs/vdev_raidz_math_aarch64_neonx2.c \ module/zfs/vdev_raidz_math_avx2.c \ module/zfs/vdev_raidz_math_avx512bw.c \ module/zfs/vdev_raidz_math_avx512f.c \ module/zfs/vdev_raidz_math_powerpc_altivec.c \ module/zfs/vdev_raidz_math_scalar.c \ module/zfs/vdev_raidz_math_sse2.c \ module/zfs/vdev_raidz_math_ssse3.c \ module/zfs/vdev_rebuild.c \ module/zfs/vdev_removal.c \ module/zfs/vdev_root.c \ module/zfs/vdev_trim.c \ module/zfs/zap.c \ module/zfs/zap_leaf.c \ module/zfs/zap_micro.c \ module/zfs/zcp.c \ module/zfs/zcp_get.c \ module/zfs/zcp_global.c \ module/zfs/zcp_iter.c \ module/zfs/zcp_set.c \ module/zfs/zcp_synctask.c \ module/zfs/zfeature.c \ module/zfs/zfs_byteswap.c \ module/zfs/zfs_chksum.c \ module/zfs/zfs_fm.c \ module/zfs/zfs_fuid.c \ module/zfs/zfs_ratelimit.c \ module/zfs/zfs_rlock.c \ module/zfs/zfs_sa.c \ module/zfs/zil.c \ module/zfs/zio.c \ module/zfs/zio_checksum.c \ module/zfs/zio_compress.c \ module/zfs/zio_inject.c \ module/zfs/zle.c \ module/zfs/zrlock.c \ module/zfs/zthr.c libzpool_la_LIBADD = \ libicp.la \ libunicode.la \ libnvpair.la \ libzstd.la \ libzutil.la libzpool_la_LIBADD += $(LIBCLOCK_GETTIME) $(ZLIB_LIBS) -ldl -lm libzpool_la_LDFLAGS = -pthread if !ASAN_ENABLED libzpool_la_LDFLAGS += -Wl,-z,defs endif if BUILD_FREEBSD libzpool_la_LIBADD += -lgeom endif libzpool_la_LDFLAGS += -version-info 5:0:0 if TARGET_CPU_POWERPC module/zfs/libzpool_la-vdev_raidz_math_powerpc_altivec.$(OBJEXT) : CFLAGS += -maltivec module/zfs/libzpool_la-vdev_raidz_math_powerpc_altivec.l$(OBJEXT): CFLAGS += -maltivec endif diff --git a/module/Kbuild.in b/module/Kbuild.in index fb22bfe733c0..7e08374fa2b9 100644 --- a/module/Kbuild.in +++ b/module/Kbuild.in @@ -1,504 +1,505 @@ # When integrated in to a monolithic kernel the spl module must appear # first. This ensures its module initialization function is run before # any of the other module initialization functions which depend on it. ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement ZFS_MODULE_CFLAGS += -Wmissing-prototypes ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@ @NO_FORMAT_ZERO_LENGTH@ ifneq ($(KBUILD_EXTMOD),) zfs_include = @abs_top_srcdir@/include icp_include = @abs_srcdir@/icp/include zstd_include = @abs_srcdir@/zstd/include ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h ZFS_MODULE_CFLAGS += -I@abs_top_builddir@/include src = @abs_srcdir@ obj = @abs_builddir@ else zfs_include = $(srctree)/include/zfs icp_include = $(srctree)/$(src)/icp/include zstd_include = $(srctree)/$(src)/zstd/include ZFS_MODULE_CFLAGS += -include $(zfs_include)/zfs_config.h endif ZFS_MODULE_CFLAGS += -I$(zfs_include)/os/linux/kernel ZFS_MODULE_CFLAGS += -I$(zfs_include)/os/linux/spl ZFS_MODULE_CFLAGS += -I$(zfs_include)/os/linux/zfs ZFS_MODULE_CFLAGS += -I$(zfs_include) ZFS_MODULE_CPPFLAGS += -D_KERNEL ZFS_MODULE_CPPFLAGS += @KERNEL_DEBUG_CPPFLAGS@ # KASAN enables -Werror=frame-larger-than=1024, which # breaks oh so many parts of our build. ifeq ($(CONFIG_KASAN),y) ZFS_MODULE_CFLAGS += -Wno-error=frame-larger-than= endif # Generated binary search code is particularly bad with this optimization. # Oddly, range_tree.c is not affected when unrolling is not done and dsl_scan.c # is not affected when unrolling is done. # Disable it until the following upstream issue is resolved: # https://github.com/llvm/llvm-project/issues/62790 ifeq ($(CONFIG_X86),y) ifeq ($(CONFIG_CC_IS_CLANG),y) CFLAGS_zfs/dsl_scan.o += -mllvm -x86-cmov-converter=false CFLAGS_zfs/metaslab.o += -mllvm -x86-cmov-converter=false CFLAGS_zfs/range_tree.o += -mllvm -x86-cmov-converter=false CFLAGS_zfs/zap_micro.o += -mllvm -x86-cmov-converter=false endif endif ifneq ($(KBUILD_EXTMOD),) @CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include @CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@ endif asflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS) ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS) ifeq ($(CONFIG_ARM64),y) CFLAGS_REMOVE_zcommon/zfs_fletcher_aarch64_neon.o += -mgeneral-regs-only CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neon.o += -mgeneral-regs-only CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neonx2.o += -mgeneral-regs-only endif # Suppress unused-value warnings in sparc64 architecture headers ccflags-$(CONFIG_SPARC64) += -Wno-unused-value obj-$(CONFIG_ZFS) := spl.o zfs.o SPL_OBJS := \ spl-atomic.o \ spl-condvar.o \ spl-cred.o \ spl-err.o \ spl-generic.o \ spl-kmem-cache.o \ spl-kmem.o \ spl-kstat.o \ spl-proc.o \ spl-procfs-list.o \ spl-shrinker.o \ spl-taskq.o \ spl-thread.o \ spl-trace.o \ spl-tsd.o \ spl-vmem.o \ spl-xdr.o \ spl-zlib.o \ spl-zone.o spl-objs += $(addprefix os/linux/spl/,$(SPL_OBJS)) zfs-objs += avl/avl.o ICP_OBJS := \ algs/aes/aes_impl.o \ algs/aes/aes_impl_generic.o \ algs/aes/aes_modes.o \ algs/blake3/blake3.o \ algs/blake3/blake3_generic.o \ algs/blake3/blake3_impl.o \ algs/edonr/edonr.o \ algs/modes/cbc.o \ algs/modes/ccm.o \ algs/modes/ctr.o \ algs/modes/ecb.o \ algs/modes/gcm.o \ algs/modes/gcm_generic.o \ algs/modes/modes.o \ algs/sha2/sha2_generic.o \ algs/sha2/sha256_impl.o \ algs/sha2/sha512_impl.o \ algs/skein/skein.o \ algs/skein/skein_block.o \ algs/skein/skein_iv.o \ api/kcf_cipher.o \ api/kcf_ctxops.o \ api/kcf_mac.o \ core/kcf_callprov.o \ core/kcf_mech_tabs.o \ core/kcf_prov_lib.o \ core/kcf_prov_tabs.o \ core/kcf_sched.o \ illumos-crypto.o \ io/aes.o \ io/sha2_mod.o \ io/skein_mod.o \ spi/kcf_spi.o ICP_OBJS_X86_64 := \ asm-x86_64/aes/aes_aesni.o \ asm-x86_64/aes/aes_amd64.o \ asm-x86_64/aes/aeskey.o \ asm-x86_64/blake3/blake3_avx2.o \ asm-x86_64/blake3/blake3_avx512.o \ asm-x86_64/blake3/blake3_sse2.o \ asm-x86_64/blake3/blake3_sse41.o \ asm-x86_64/sha2/sha256-x86_64.o \ asm-x86_64/sha2/sha512-x86_64.o \ asm-x86_64/modes/aesni-gcm-x86_64.o \ asm-x86_64/modes/gcm_pclmulqdq.o \ asm-x86_64/modes/ghash-x86_64.o ICP_OBJS_X86 := \ algs/aes/aes_impl_aesni.o \ algs/aes/aes_impl_x86-64.o \ algs/modes/gcm_pclmulqdq.o ICP_OBJS_ARM := \ asm-arm/sha2/sha256-armv7.o \ asm-arm/sha2/sha512-armv7.o ICP_OBJS_ARM64 := \ asm-aarch64/blake3/b3_aarch64_sse2.o \ asm-aarch64/blake3/b3_aarch64_sse41.o \ asm-aarch64/sha2/sha256-armv8.o \ asm-aarch64/sha2/sha512-armv8.o ICP_OBJS_PPC_PPC64 := \ asm-ppc64/blake3/b3_ppc64le_sse2.o \ asm-ppc64/blake3/b3_ppc64le_sse41.o \ asm-ppc64/sha2/sha256-p8.o \ asm-ppc64/sha2/sha512-p8.o \ asm-ppc64/sha2/sha256-ppc.o \ asm-ppc64/sha2/sha512-ppc.o zfs-objs += $(addprefix icp/,$(ICP_OBJS)) zfs-$(CONFIG_X86) += $(addprefix icp/,$(ICP_OBJS_X86)) zfs-$(CONFIG_UML_X86)+= $(addprefix icp/,$(ICP_OBJS_X86)) zfs-$(CONFIG_X86_64) += $(addprefix icp/,$(ICP_OBJS_X86_64)) zfs-$(CONFIG_ARM) += $(addprefix icp/,$(ICP_OBJS_ARM)) zfs-$(CONFIG_ARM64) += $(addprefix icp/,$(ICP_OBJS_ARM64)) zfs-$(CONFIG_PPC) += $(addprefix icp/,$(ICP_OBJS_PPC_PPC64)) zfs-$(CONFIG_PPC64) += $(addprefix icp/,$(ICP_OBJS_PPC_PPC64)) $(addprefix $(obj)/icp/,$(ICP_OBJS) $(ICP_OBJS_X86) $(ICP_OBJS_X86_64) \ $(ICP_OBJS_ARM64) $(ICP_OBJS_PPC_PPC64)) : asflags-y += -I$(icp_include) -I$(zfs_include)/os/linux/spl -I$(zfs_include) $(addprefix $(obj)/icp/,$(ICP_OBJS) $(ICP_OBJS_X86) $(ICP_OBJS_X86_64) \ $(ICP_OBJS_ARM64) $(ICP_OBJS_PPC_PPC64)) : ccflags-y += -I$(icp_include) -I$(zfs_include)/os/linux/spl -I$(zfs_include) # Suppress objtool "return with modified stack frame" warnings. OBJECT_FILES_NON_STANDARD_aesni-gcm-x86_64.o := y # Suppress objtool "unsupported stack pointer realignment" warnings. # See #6950 for the reasoning. OBJECT_FILES_NON_STANDARD_sha256-x86_64.o := y OBJECT_FILES_NON_STANDARD_sha512-x86_64.o := y LUA_OBJS := \ lapi.o \ lauxlib.o \ lbaselib.o \ lcode.o \ lcompat.o \ lcorolib.o \ lctype.o \ ldebug.o \ ldo.o \ lfunc.o \ lgc.o \ llex.o \ lmem.o \ lobject.o \ lopcodes.o \ lparser.o \ lstate.o \ lstring.o \ lstrlib.o \ ltable.o \ ltablib.o \ ltm.o \ lvm.o \ lzio.o \ setjmp/setjmp.o zfs-objs += $(addprefix lua/,$(LUA_OBJS)) NVPAIR_OBJS := \ fnvpair.o \ nvpair.o \ nvpair_alloc_fixed.o \ nvpair_alloc_spl.o zfs-objs += $(addprefix nvpair/,$(NVPAIR_OBJS)) UNICODE_OBJS := \ u8_textprep.o \ uconv.o zfs-objs += $(addprefix unicode/,$(UNICODE_OBJS)) ZCOMMON_OBJS := \ cityhash.o \ zfeature_common.o \ zfs_comutil.o \ zfs_deleg.o \ zfs_fletcher.o \ zfs_fletcher_superscalar.o \ zfs_fletcher_superscalar4.o \ zfs_namecheck.o \ zfs_prop.o \ zpool_prop.o \ zprop_common.o ZCOMMON_OBJS_X86 := \ zfs_fletcher_avx512.o \ zfs_fletcher_intel.o \ zfs_fletcher_sse.o ZCOMMON_OBJS_ARM64 := \ zfs_fletcher_aarch64_neon.o zfs-objs += $(addprefix zcommon/,$(ZCOMMON_OBJS)) zfs-$(CONFIG_X86) += $(addprefix zcommon/,$(ZCOMMON_OBJS_X86)) zfs-$(CONFIG_UML_X86)+= $(addprefix zcommon/,$(ZCOMMON_OBJS_X86)) zfs-$(CONFIG_ARM64) += $(addprefix zcommon/,$(ZCOMMON_OBJS_ARM64)) # Zstd uses -O3 by default, so we should follow ZFS_ZSTD_FLAGS := -O3 # -fno-tree-vectorize gets set for gcc in zstd/common/compiler.h # Set it for other compilers, too. ZFS_ZSTD_FLAGS += -fno-tree-vectorize # SSE register return with SSE disabled if -march=znverX is passed ZFS_ZSTD_FLAGS += -U__BMI__ # Quiet warnings about frame size due to unused code in unmodified zstd lib ZFS_ZSTD_FLAGS += -Wframe-larger-than=20480 ZSTD_OBJS := \ zfs_zstd.o \ zstd_sparc.o ZSTD_UPSTREAM_OBJS := \ lib/common/entropy_common.o \ lib/common/error_private.o \ lib/common/fse_decompress.o \ lib/common/pool.o \ lib/common/zstd_common.o \ lib/compress/fse_compress.o \ lib/compress/hist.o \ lib/compress/huf_compress.o \ lib/compress/zstd_compress.o \ lib/compress/zstd_compress_literals.o \ lib/compress/zstd_compress_sequences.o \ lib/compress/zstd_compress_superblock.o \ lib/compress/zstd_double_fast.o \ lib/compress/zstd_fast.o \ lib/compress/zstd_lazy.o \ lib/compress/zstd_ldm.o \ lib/compress/zstd_opt.o \ lib/decompress/huf_decompress.o \ lib/decompress/zstd_ddict.o \ lib/decompress/zstd_decompress.o \ lib/decompress/zstd_decompress_block.o zfs-objs += $(addprefix zstd/,$(ZSTD_OBJS) $(ZSTD_UPSTREAM_OBJS)) # Disable aarch64 neon SIMD instructions for kernel mode $(addprefix $(obj)/zstd/,$(ZSTD_OBJS) $(ZSTD_UPSTREAM_OBJS)) : ccflags-y += -I$(zstd_include) $(ZFS_ZSTD_FLAGS) $(addprefix $(obj)/zstd/,$(ZSTD_OBJS) $(ZSTD_UPSTREAM_OBJS)) : asflags-y += -I$(zstd_include) $(addprefix $(obj)/zstd/,$(ZSTD_UPSTREAM_OBJS)) : ccflags-y += -include $(zstd_include)/aarch64_compat.h -include $(zstd_include)/zstd_compat_wrapper.h -Wp,-w $(obj)/zstd/zfs_zstd.o : ccflags-y += -include $(zstd_include)/zstd_compat_wrapper.h ZFS_OBJS := \ abd.o \ aggsum.o \ arc.o \ blake3_zfs.o \ blkptr.o \ bplist.o \ bpobj.o \ bptree.o \ bqueue.o \ brt.o \ btree.o \ dataset_kstats.o \ dbuf.o \ dbuf_stats.o \ ddt.o \ + ddt_stats.o \ ddt_zap.o \ dmu.o \ dmu_diff.o \ dmu_object.o \ dmu_objset.o \ dmu_recv.o \ dmu_redact.o \ dmu_send.o \ dmu_traverse.o \ dmu_tx.o \ dmu_zfetch.o \ dnode.o \ dnode_sync.o \ dsl_bookmark.o \ dsl_crypt.o \ dsl_dataset.o \ dsl_deadlist.o \ dsl_deleg.o \ dsl_destroy.o \ dsl_dir.o \ dsl_pool.o \ dsl_prop.o \ dsl_scan.o \ dsl_synctask.o \ dsl_userhold.o \ edonr_zfs.o \ fm.o \ gzip.o \ hkdf.o \ lz4.o \ lz4_zfs.o \ lzjb.o \ metaslab.o \ mmp.o \ multilist.o \ objlist.o \ pathname.o \ range_tree.o \ refcount.o \ rrwlock.o \ sa.o \ sha2_zfs.o \ skein_zfs.o \ spa.o \ spa_checkpoint.o \ spa_config.o \ spa_errlog.o \ spa_history.o \ spa_log_spacemap.o \ spa_misc.o \ spa_stats.o \ space_map.o \ space_reftree.o \ txg.o \ uberblock.o \ unique.o \ vdev.o \ vdev_draid.o \ vdev_draid_rand.o \ vdev_indirect.o \ vdev_indirect_births.o \ vdev_indirect_mapping.o \ vdev_initialize.o \ vdev_label.o \ vdev_mirror.o \ vdev_missing.o \ vdev_queue.o \ vdev_raidz.o \ vdev_raidz_math.o \ vdev_raidz_math_scalar.o \ vdev_rebuild.o \ vdev_removal.o \ vdev_root.o \ vdev_trim.o \ zap.o \ zap_leaf.o \ zap_micro.o \ zcp.o \ zcp_get.o \ zcp_global.o \ zcp_iter.o \ zcp_set.o \ zcp_synctask.o \ zfeature.o \ zfs_byteswap.o \ zfs_chksum.o \ zfs_fm.o \ zfs_fuid.o \ zfs_impl.o \ zfs_ioctl.o \ zfs_log.o \ zfs_onexit.o \ zfs_quota.o \ zfs_ratelimit.o \ zfs_replay.o \ zfs_rlock.o \ zfs_sa.o \ zfs_vnops.o \ zil.o \ zio.o \ zio_checksum.o \ zio_compress.o \ zio_inject.o \ zle.o \ zrlock.o \ zthr.o \ zvol.o ZFS_OBJS_OS := \ abd_os.o \ arc_os.o \ mmp_os.o \ policy.o \ qat.o \ qat_compress.o \ qat_crypt.o \ spa_misc_os.o \ trace.o \ vdev_disk.o \ vdev_file.o \ vdev_label_os.o \ zfs_acl.o \ zfs_ctldir.o \ zfs_debug.o \ zfs_dir.o \ zfs_file_os.o \ zfs_ioctl_os.o \ zfs_racct.o \ zfs_sysfs.o \ zfs_uio.o \ zfs_vfsops.o \ zfs_vnops_os.o \ zfs_znode.o \ zio_crypt.o \ zpl_ctldir.o \ zpl_export.o \ zpl_file.o \ zpl_file_range.o \ zpl_inode.o \ zpl_super.o \ zpl_xattr.o \ zvol_os.o ZFS_OBJS_X86 := \ vdev_raidz_math_avx2.o \ vdev_raidz_math_avx512bw.o \ vdev_raidz_math_avx512f.o \ vdev_raidz_math_sse2.o \ vdev_raidz_math_ssse3.o ZFS_OBJS_ARM64 := \ vdev_raidz_math_aarch64_neon.o \ vdev_raidz_math_aarch64_neonx2.o ZFS_OBJS_PPC_PPC64 := \ vdev_raidz_math_powerpc_altivec.o zfs-objs += $(addprefix zfs/,$(ZFS_OBJS)) $(addprefix os/linux/zfs/,$(ZFS_OBJS_OS)) zfs-$(CONFIG_X86) += $(addprefix zfs/,$(ZFS_OBJS_X86)) zfs-$(CONFIG_UML_X86)+= $(addprefix zfs/,$(ZFS_OBJS_X86)) zfs-$(CONFIG_ARM64) += $(addprefix zfs/,$(ZFS_OBJS_ARM64)) zfs-$(CONFIG_PPC) += $(addprefix zfs/,$(ZFS_OBJS_PPC_PPC64)) zfs-$(CONFIG_PPC64) += $(addprefix zfs/,$(ZFS_OBJS_PPC_PPC64)) UBSAN_SANITIZE_zap_leaf.o := n UBSAN_SANITIZE_zap_micro.o := n UBSAN_SANITIZE_sa.o := n # Suppress incorrect warnings from versions of objtool which are not # aware of x86 EVEX prefix instructions used for AVX512. OBJECT_FILES_NON_STANDARD_vdev_raidz_math_avx512bw.o := y OBJECT_FILES_NON_STANDARD_vdev_raidz_math_avx512f.o := y ifeq ($(CONFIG_ALTIVEC),y) $(obj)/zfs/vdev_raidz_math_powerpc_altivec.o : c_flags += -maltivec endif diff --git a/module/Makefile.bsd b/module/Makefile.bsd index 0c4d8bfe1159..1b0110d3ae9a 100644 --- a/module/Makefile.bsd +++ b/module/Makefile.bsd @@ -1,543 +1,544 @@ .if !defined(WITH_CTF) WITH_CTF=1 .endif .include SRCDIR=${.CURDIR} INCDIR=${.CURDIR:H}/include KMOD= openzfs .PATH: ${SRCDIR}/avl \ ${SRCDIR}/lua \ ${SRCDIR}/nvpair \ ${SRCDIR}/icp/algs/blake3 \ ${SRCDIR}/icp/algs/edonr \ ${SRCDIR}/icp/algs/sha2 \ ${SRCDIR}/icp/asm-aarch64/blake3 \ ${SRCDIR}/icp/asm-aarch64/sha2 \ ${SRCDIR}/icp/asm-arm/sha2 \ ${SRCDIR}/icp/asm-ppc64/sha2 \ ${SRCDIR}/icp/asm-ppc64/blake3 \ ${SRCDIR}/icp/asm-x86_64/blake3 \ ${SRCDIR}/icp/asm-x86_64/sha2 \ ${SRCDIR}/os/freebsd/spl \ ${SRCDIR}/os/freebsd/zfs \ ${SRCDIR}/unicode \ ${SRCDIR}/zcommon \ ${SRCDIR}/zfs \ ${SRCDIR}/zstd \ ${SRCDIR}/zstd/lib/common \ ${SRCDIR}/zstd/lib/compress \ ${SRCDIR}/zstd/lib/decompress CFLAGS+= -I${INCDIR} CFLAGS+= -I${SRCDIR}/icp/include CFLAGS+= -I${INCDIR}/os/freebsd CFLAGS+= -I${INCDIR}/os/freebsd/spl CFLAGS+= -I${INCDIR}/os/freebsd/zfs CFLAGS+= -I${SRCDIR}/zstd/include CFLAGS+= -include ${INCDIR}/os/freebsd/spl/sys/ccompile.h CFLAGS+= -I${.CURDIR} CFLAGS+= -D__KERNEL__ -DFREEBSD_NAMECACHE -DBUILDING_ZFS -D__BSD_VISIBLE=1 \ -DHAVE_UIO_ZEROCOPY -DWITHOUT_NETDUMP -D__KERNEL -D_SYS_CONDVAR_H_ \ -D_SYS_VMEM_H_ -DKDTRACE_HOOKS -DCOMPAT_FREEBSD11 .if ${MACHINE_ARCH} == "amd64" CFLAGS+= -D__x86_64 -DHAVE_SSE2 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2 \ -DHAVE_AVX -DHAVE_AVX2 -DHAVE_AVX512F -DHAVE_AVX512VL -DHAVE_AVX512BW .endif .if defined(WITH_DEBUG) && ${WITH_DEBUG} == "true" CFLAGS+= -DZFS_DEBUG -g .if defined(WITH_INVARIANTS) && ${WITH_INVARIANTS} == "true" CFLAGS+= -DINVARIANTS -DWITNESS -DOPENSOLARIS_WITNESS .endif .if defined(WITH_O0) && ${WITH_O0} == "true" CFLAGS+= -O0 .endif .else CFLAGS += -DNDEBUG .endif .if defined(WITH_VFS_DEBUG) && ${WITH_VFS_DEBUG} == "true" # kernel must also be built with this option for this to work CFLAGS+= -DDEBUG_VFS_LOCKS .endif .if defined(WITH_GCOV) && ${WITH_GCOV} == "true" CFLAGS+= -fprofile-arcs -ftest-coverage .endif DEBUG_FLAGS=-g .if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" || \ ${MACHINE_ARCH} == "powerpcspe" || ${MACHINE_ARCH} == "arm" CFLAGS+= -DBITS_PER_LONG=32 .else CFLAGS+= -DBITS_PER_LONG=64 .endif SRCS= vnode_if.h device_if.h bus_if.h # avl SRCS+= avl.c # icp SRCS+= edonr.c #icp/algs/blake3 SRCS+= blake3.c \ blake3_generic.c \ blake3_impl.c #icp/asm-aarch64/blake3 SRCS+= b3_aarch64_sse2.S \ b3_aarch64_sse41.S #icp/asm-ppc64/blake3 SRCS+= b3_ppc64le_sse2.S \ b3_ppc64le_sse41.S #icp/asm-x86_64/blake3 SRCS+= blake3_avx2.S \ blake3_avx512.S \ blake3_sse2.S \ blake3_sse41.S #icp/algs/sha2 SRCS+= sha2_generic.c \ sha256_impl.c \ sha512_impl.c #icp/asm-arm/sha2 SRCS+= sha256-armv7.S \ sha512-armv7.S #icp/asm-aarch64/sha2 SRCS+= sha256-armv8.S \ sha512-armv8.S #icp/asm-ppc64/sha2 SRCS+= sha256-p8.S \ sha512-p8.S \ sha256-ppc.S \ sha512-ppc.S #icp/asm-x86_64/sha2 SRCS+= sha256-x86_64.S \ sha512-x86_64.S #lua SRCS+= lapi.c \ lauxlib.c \ lbaselib.c \ lcode.c \ lcompat.c \ lcorolib.c \ lctype.c \ ldebug.c \ ldo.c \ lfunc.c \ lgc.c \ llex.c \ lmem.c \ lobject.c \ lopcodes.c \ lparser.c \ lstate.c \ lstring.c \ lstrlib.c \ ltable.c \ ltablib.c \ ltm.c \ lvm.c \ lzio.c #nvpair SRCS+= nvpair.c \ fnvpair.c \ nvpair_alloc_spl.c \ nvpair_alloc_fixed.c #os/freebsd/spl SRCS+= acl_common.c \ callb.c \ list.c \ spl_acl.c \ spl_cmn_err.c \ spl_dtrace.c \ spl_kmem.c \ spl_kstat.c \ spl_misc.c \ spl_policy.c \ spl_procfs_list.c \ spl_string.c \ spl_sunddi.c \ spl_sysevent.c \ spl_taskq.c \ spl_uio.c \ spl_vfs.c \ spl_vm.c \ spl_zlib.c \ spl_zone.c .if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" || \ ${MACHINE_ARCH} == "powerpcspe" || ${MACHINE_ARCH} == "arm" SRCS+= spl_atomic.c .endif #os/freebsd/zfs SRCS+= abd_os.c \ arc_os.c \ crypto_os.c \ dmu_os.c \ event_os.c \ hkdf.c \ kmod_core.c \ spa_os.c \ sysctl_os.c \ vdev_file.c \ vdev_geom.c \ vdev_label_os.c \ zfs_acl.c \ zfs_ctldir.c \ zfs_debug.c \ zfs_dir.c \ zfs_ioctl_compat.c \ zfs_ioctl_os.c \ zfs_racct.c \ zfs_vfsops.c \ zfs_vnops_os.c \ zfs_znode.c \ zio_crypt.c \ zvol_os.c #unicode SRCS+= uconv.c \ u8_textprep.c #zcommon SRCS+= zfeature_common.c \ zfs_comutil.c \ zfs_deleg.c \ zfs_fletcher.c \ zfs_fletcher_avx512.c \ zfs_fletcher_intel.c \ zfs_fletcher_sse.c \ zfs_fletcher_superscalar.c \ zfs_fletcher_superscalar4.c \ zfs_namecheck.c \ zfs_prop.c \ zpool_prop.c \ zprop_common.c #zfs SRCS+= abd.c \ aggsum.c \ arc.c \ blake3_zfs.c \ blkptr.c \ bplist.c \ bpobj.c \ brt.c \ btree.c \ cityhash.c \ dbuf.c \ dbuf_stats.c \ bptree.c \ bqueue.c \ dataset_kstats.c \ ddt.c \ + ddt_stats.c \ ddt_zap.c \ dmu.c \ dmu_diff.c \ dmu_object.c \ dmu_objset.c \ dmu_recv.c \ dmu_redact.c \ dmu_send.c \ dmu_traverse.c \ dmu_tx.c \ dmu_zfetch.c \ dnode.c \ dnode_sync.c \ dsl_dataset.c \ dsl_deadlist.c \ dsl_deleg.c \ dsl_bookmark.c \ dsl_dir.c \ dsl_crypt.c \ dsl_destroy.c \ dsl_pool.c \ dsl_prop.c \ dsl_scan.c \ dsl_synctask.c \ dsl_userhold.c \ edonr_zfs.c \ fm.c \ gzip.c \ lzjb.c \ lz4.c \ lz4_zfs.c \ metaslab.c \ mmp.c \ multilist.c \ objlist.c \ pathname.c \ range_tree.c \ refcount.c \ rrwlock.c \ sa.c \ sha2_zfs.c \ skein_zfs.c \ spa.c \ spa_checkpoint.c \ spa_config.c \ spa_errlog.c \ spa_history.c \ spa_log_spacemap.c \ spa_misc.c \ spa_stats.c \ space_map.c \ space_reftree.c \ txg.c \ uberblock.c \ unique.c \ vdev.c \ vdev_draid.c \ vdev_draid_rand.c \ vdev_indirect.c \ vdev_indirect_births.c \ vdev_indirect_mapping.c \ vdev_initialize.c \ vdev_label.c \ vdev_mirror.c \ vdev_missing.c \ vdev_queue.c \ vdev_raidz.c \ vdev_raidz_math.c \ vdev_raidz_math_scalar.c \ vdev_raidz_math_avx2.c \ vdev_raidz_math_avx512bw.c \ vdev_raidz_math_avx512f.c \ vdev_raidz_math_sse2.c \ vdev_raidz_math_ssse3.c \ vdev_rebuild.c \ vdev_removal.c \ vdev_root.c \ vdev_trim.c \ zap.c \ zap_leaf.c \ zap_micro.c \ zcp.c \ zcp_get.c \ zcp_global.c \ zcp_iter.c \ zcp_set.c \ zcp_synctask.c \ zfeature.c \ zfs_byteswap.c \ zfs_chksum.c \ zfs_file_os.c \ zfs_fm.c \ zfs_fuid.c \ zfs_impl.c \ zfs_ioctl.c \ zfs_log.c \ zfs_onexit.c \ zfs_quota.c \ zfs_ratelimit.c \ zfs_replay.c \ zfs_rlock.c \ zfs_sa.c \ zfs_vnops.c \ zil.c \ zio.c \ zio_checksum.c \ zio_compress.c \ zio_inject.c \ zle.c \ zrlock.c \ zthr.c \ zvol.c #zstd SRCS+= zfs_zstd.c \ entropy_common.c \ error_private.c \ fse_compress.c \ fse_decompress.c \ hist.c \ huf_compress.c \ huf_decompress.c \ pool.c \ xxhash.c \ zstd_common.c \ zstd_compress.c \ zstd_compress_literals.c \ zstd_compress_sequences.c \ zstd_compress_superblock.c \ zstd_ddict.c \ zstd_decompress.c \ zstd_decompress_block.c \ zstd_double_fast.c \ zstd_fast.c \ zstd_lazy.c \ zstd_ldm.c \ zstd_opt.c beforeinstall: .if ${MK_DEBUG_FILES} != "no" mtree -eu \ -f /etc/mtree/BSD.debug.dist \ -p ${DESTDIR}/usr/lib .endif .include # Generated binary search code is particularly bad with this optimization. # Oddly, range_tree.c is not affected when unrolling is not done and dsl_scan.c # is not affected when unrolling is done. # Disable it until the following upstream issue is resolved: # https://github.com/llvm/llvm-project/issues/62790 .if ${CC} == "clang" .if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "amd64" CFLAGS.dsl_scan.c= -mllvm -x86-cmov-converter=false CFLAGS.metaslab.c= -mllvm -x86-cmov-converter=false CFLAGS.range_tree.c= -mllvm -x86-cmov-converter=false CFLAGS.zap_micro.c= -mllvm -x86-cmov-converter=false .endif .endif CFLAGS.sysctl_os.c= -include ../zfs_config.h CFLAGS.xxhash.c+= -include ${SYSDIR}/sys/_null.h CFLAGS.gcc+= -Wno-pointer-to-int-cast CFLAGS.abd.c= -Wno-cast-qual CFLAGS.ddt.c= -Wno-cast-qual CFLAGS.dmu.c= -Wno-cast-qual CFLAGS.dmu_traverse.c= -Wno-cast-qual CFLAGS.dnode.c= ${NO_WUNUSED_BUT_SET_VARIABLE} CFLAGS.dsl_deadlist.c= -Wno-cast-qual CFLAGS.dsl_dir.c= -Wno-cast-qual CFLAGS.dsl_prop.c= -Wno-cast-qual CFLAGS.edonr.c= -Wno-cast-qual CFLAGS.fm.c= -Wno-cast-qual CFLAGS.hist.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.lapi.c= -Wno-cast-qual CFLAGS.lcompat.c= -Wno-cast-qual CFLAGS.ldo.c= ${NO_WINFINITE_RECURSION} CFLAGS.lobject.c= -Wno-cast-qual CFLAGS.ltable.c= -Wno-cast-qual CFLAGS.lvm.c= -Wno-cast-qual CFLAGS.lz4.c= -Wno-cast-qual CFLAGS.lz4_zfs.c= -Wno-cast-qual CFLAGS.nvpair.c= -Wno-cast-qual -DHAVE_RPC_TYPES ${NO_WSTRINGOP_OVERREAD} CFLAGS.pool.c+= ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.pool.c= -U__BMI__ -fno-tree-vectorize CFLAGS.spa.c= -Wno-cast-qual CFLAGS.spa_misc.c= -Wno-cast-qual CFLAGS.spl_string.c= -Wno-cast-qual CFLAGS.spl_vm.c= -Wno-cast-qual CFLAGS.spl_zlib.c= -Wno-cast-qual CFLAGS.u8_textprep.c= -Wno-cast-qual CFLAGS.vdev_draid.c= -Wno-cast-qual CFLAGS.vdev_raidz.c= -Wno-cast-qual CFLAGS.vdev_raidz_math.c= -Wno-cast-qual CFLAGS.vdev_raidz_math_avx2.c= -Wno-cast-qual -Wno-duplicate-decl-specifier CFLAGS.vdev_raidz_math_avx512f.c= -Wno-cast-qual -Wno-duplicate-decl-specifier CFLAGS.vdev_raidz_math_scalar.c= -Wno-cast-qual CFLAGS.vdev_raidz_math_sse2.c= -Wno-cast-qual -Wno-duplicate-decl-specifier CFLAGS.zap_leaf.c= -Wno-cast-qual CFLAGS.zap_micro.c= -Wno-cast-qual CFLAGS.zcp.c= -Wno-cast-qual CFLAGS.zfs_fletcher.c= -Wno-cast-qual -Wno-pointer-arith CFLAGS.zfs_fletcher_avx512.c= -Wno-cast-qual -Wno-pointer-arith CFLAGS.zfs_fletcher_intel.c= -Wno-cast-qual -Wno-pointer-arith CFLAGS.zfs_fletcher_sse.c= -Wno-cast-qual -Wno-pointer-arith CFLAGS.zfs_fm.c= -Wno-cast-qual ${NO_WUNUSED_BUT_SET_VARIABLE} CFLAGS.zfs_ioctl.c= -Wno-cast-qual CFLAGS.zfs_log.c= -Wno-cast-qual CFLAGS.zfs_vnops_os.c= -Wno-pointer-arith CFLAGS.zfs_zstd.c= -Wno-cast-qual -Wno-pointer-arith CFLAGS.zil.c= -Wno-cast-qual CFLAGS.zio.c= -Wno-cast-qual CFLAGS.zprop_common.c= -Wno-cast-qual CFLAGS.zrlock.c= -Wno-cast-qual #zstd CFLAGS.entropy_common.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.error_private.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.fse_compress.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} ${NO_WUNUSED_BUT_SET_VARIABLE} CFLAGS.fse_decompress.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.huf_compress.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.huf_decompress.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.xxhash.c+= -U__BMI__ -fno-tree-vectorize CFLAGS.xxhash.c+= ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_common.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_compress.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_compress_literals.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_compress_sequences.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_compress_superblock.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} ${NO_WUNUSED_BUT_SET_VARIABLE} CFLAGS.zstd_ddict.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_decompress.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_decompress_block.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_double_fast.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_fast.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_lazy.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_ldm.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} CFLAGS.zstd_opt.c= -U__BMI__ -fno-tree-vectorize ${NO_WBITWISE_INSTEAD_OF_LOGICAL} .if ${MACHINE_CPUARCH} == "aarch64" __ZFS_ZSTD_AARCH64_FLAGS= -include ${SRCDIR}/zstd/include/aarch64_compat.h CFLAGS.zstd.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.entropy_common.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.error_private.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.fse_compress.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.fse_decompress.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.hist.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.huf_compress.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.huf_decompress.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.pool.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.xxhash.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_common.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_compress.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_compress_literals.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_compress_sequences.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_compress_superblock.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_ddict.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_decompress.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_decompress_block.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_double_fast.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_fast.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_lazy.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_ldm.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_opt.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} sha256-armv8.o: sha256-armv8.S ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC} \ -o ${.TARGET} ${CTFCONVERT_CMD} sha512-armv8.o: sha512-armv8.S ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC} \ -o ${.TARGET} ${CTFCONVERT_CMD} b3_aarch64_sse2.o: b3_aarch64_sse2.S ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC} \ -o ${.TARGET} ${CTFCONVERT_CMD} b3_aarch64_sse41.o: b3_aarch64_sse41.S ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC} \ -o ${.TARGET} ${CTFCONVERT_CMD} .endif diff --git a/module/zfs/ddt.c b/module/zfs/ddt.c index f54a51842ebb..43fc2aef5200 100644 --- a/module/zfs/ddt.c +++ b/module/zfs/ddt.c @@ -1,1208 +1,1037 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or https://opensource.org/licenses/CDDL-1.0. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright (c) 2022 by Pawel Jakub Dawidek */ #include #include #include #include #include #include #include #include #include #include #include #include static kmem_cache_t *ddt_cache; static kmem_cache_t *ddt_entry_cache; /* * Enable/disable prefetching of dedup-ed blocks which are going to be freed. */ int zfs_dedup_prefetch = 0; static const ddt_ops_t *const ddt_ops[DDT_TYPES] = { &ddt_zap_ops, }; static const char *const ddt_class_name[DDT_CLASSES] = { "ditto", "duplicate", "unique", }; static void ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class, dmu_tx_t *tx) { spa_t *spa = ddt->ddt_spa; objset_t *os = ddt->ddt_os; uint64_t *objectp = &ddt->ddt_object[type][class]; boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_flags & ZCHECKSUM_FLAG_DEDUP; char name[DDT_NAMELEN]; ddt_object_name(ddt, type, class, name); ASSERT3U(*objectp, ==, 0); VERIFY0(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash)); ASSERT3U(*objectp, !=, 0); VERIFY0(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name, sizeof (uint64_t), 1, objectp, tx)); VERIFY0(zap_add(os, spa->spa_ddt_stat_object, name, sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), &ddt->ddt_histogram[type][class], tx)); } static void ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class, dmu_tx_t *tx) { spa_t *spa = ddt->ddt_spa; objset_t *os = ddt->ddt_os; uint64_t *objectp = &ddt->ddt_object[type][class]; uint64_t count; char name[DDT_NAMELEN]; ddt_object_name(ddt, type, class, name); ASSERT3U(*objectp, !=, 0); ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class])); VERIFY0(ddt_object_count(ddt, type, class, &count)); VERIFY0(count); VERIFY0(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx)); VERIFY0(zap_remove(os, spa->spa_ddt_stat_object, name, tx)); VERIFY0(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx)); memset(&ddt->ddt_object_stats[type][class], 0, sizeof (ddt_object_t)); *objectp = 0; } static int ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class) { ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; dmu_object_info_t doi; uint64_t count; char name[DDT_NAMELEN]; int error; ddt_object_name(ddt, type, class, name); error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name, sizeof (uint64_t), 1, &ddt->ddt_object[type][class]); if (error != 0) return (error); error = zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), &ddt->ddt_histogram[type][class]); if (error != 0) return (error); /* * Seed the cached statistics. */ error = ddt_object_info(ddt, type, class, &doi); if (error) return (error); error = ddt_object_count(ddt, type, class, &count); if (error) return (error); ddo->ddo_count = count; ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; return (0); } static void ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class, dmu_tx_t *tx) { ddt_object_t *ddo = &ddt->ddt_object_stats[type][class]; dmu_object_info_t doi; uint64_t count; char name[DDT_NAMELEN]; ddt_object_name(ddt, type, class, name); VERIFY0(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name, sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t), &ddt->ddt_histogram[type][class], tx)); /* * Cache DDT statistics; this is the only time they'll change. */ VERIFY0(ddt_object_info(ddt, type, class, &doi)); VERIFY0(ddt_object_count(ddt, type, class, &count)); ddo->ddo_count = count; ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9; ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size; } static int ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class, ddt_entry_t *dde) { if (!ddt_object_exists(ddt, type, class)) return (SET_ERROR(ENOENT)); return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os, ddt->ddt_object[type][class], dde)); } static void ddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class, ddt_entry_t *dde) { if (!ddt_object_exists(ddt, type, class)) return; ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os, ddt->ddt_object[type][class], dde); } int ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class, ddt_entry_t *dde, dmu_tx_t *tx) { ASSERT(ddt_object_exists(ddt, type, class)); return (ddt_ops[type]->ddt_op_update(ddt->ddt_os, ddt->ddt_object[type][class], dde, tx)); } static int ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class, ddt_entry_t *dde, dmu_tx_t *tx) { ASSERT(ddt_object_exists(ddt, type, class)); return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os, ddt->ddt_object[type][class], dde, tx)); } int ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class, uint64_t *walk, ddt_entry_t *dde) { ASSERT(ddt_object_exists(ddt, type, class)); return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os, ddt->ddt_object[type][class], dde, walk)); } int ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class, uint64_t *count) { ASSERT(ddt_object_exists(ddt, type, class)); return (ddt_ops[type]->ddt_op_count(ddt->ddt_os, ddt->ddt_object[type][class], count)); } int ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class, dmu_object_info_t *doi) { if (!ddt_object_exists(ddt, type, class)) return (SET_ERROR(ENOENT)); return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class], doi)); } boolean_t ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class) { return (!!ddt->ddt_object[type][class]); } void ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class, char *name) { (void) snprintf(name, DDT_NAMELEN, DMU_POOL_DDT, zio_checksum_table[ddt->ddt_checksum].ci_name, ddt_ops[type]->ddt_op_name, ddt_class_name[class]); } void ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg) { ASSERT3U(txg, !=, 0); for (int d = 0; d < SPA_DVAS_PER_BP; d++) bp->blk_dva[d] = ddp->ddp_dva[d]; BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth); } /* * The bp created via this function may be used for repairs and scrub, but it * will be missing the salt / IV required to do a full decrypting read. */ void ddt_bp_create(enum zio_checksum checksum, const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp) { BP_ZERO(bp); if (ddp != NULL) ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth); bp->blk_cksum = ddk->ddk_cksum; BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk)); BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk)); BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk)); BP_SET_CRYPT(bp, DDK_GET_CRYPT(ddk)); BP_SET_FILL(bp, 1); BP_SET_CHECKSUM(bp, checksum); BP_SET_TYPE(bp, DMU_OT_DEDUP); BP_SET_LEVEL(bp, 0); BP_SET_DEDUP(bp, 1); BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); } void ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp) { ddk->ddk_cksum = bp->blk_cksum; ddk->ddk_prop = 0; ASSERT(BP_IS_ENCRYPTED(bp) || !BP_USES_CRYPT(bp)); DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp)); DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp)); DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp)); DDK_SET_CRYPT(ddk, BP_USES_CRYPT(bp)); } void ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp) { ASSERT0(ddp->ddp_phys_birth); for (int d = 0; d < SPA_DVAS_PER_BP; d++) ddp->ddp_dva[d] = bp->blk_dva[d]; ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp); } void ddt_phys_clear(ddt_phys_t *ddp) { memset(ddp, 0, sizeof (*ddp)); } void ddt_phys_addref(ddt_phys_t *ddp) { ddp->ddp_refcnt++; } void ddt_phys_decref(ddt_phys_t *ddp) { if (ddp) { ASSERT3U(ddp->ddp_refcnt, >, 0); ddp->ddp_refcnt--; } } void ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg) { blkptr_t blk; ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); /* * We clear the dedup bit so that zio_free() will actually free the * space, rather than just decrementing the refcount in the DDT. */ BP_SET_DEDUP(&blk, 0); ddt_phys_clear(ddp); zio_free(ddt->ddt_spa, txg, &blk); } ddt_phys_t * ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp) { ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys; for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) && BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth) return (ddp); } return (NULL); } uint64_t ddt_phys_total_refcnt(const ddt_entry_t *dde) { uint64_t refcnt = 0; for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) refcnt += dde->dde_phys[p].ddp_refcnt; return (refcnt); } -static void -ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds) -{ - spa_t *spa = ddt->ddt_spa; - ddt_phys_t *ddp = dde->dde_phys; - ddt_key_t *ddk = &dde->dde_key; - uint64_t lsize = DDK_GET_LSIZE(ddk); - uint64_t psize = DDK_GET_PSIZE(ddk); - - memset(dds, 0, sizeof (*dds)); - - for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { - uint64_t dsize = 0; - uint64_t refcnt = ddp->ddp_refcnt; - - if (ddp->ddp_phys_birth == 0) - continue; - - for (int d = 0; d < DDE_GET_NDVAS(dde); d++) - dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]); - - dds->dds_blocks += 1; - dds->dds_lsize += lsize; - dds->dds_psize += psize; - dds->dds_dsize += dsize; - - dds->dds_ref_blocks += refcnt; - dds->dds_ref_lsize += lsize * refcnt; - dds->dds_ref_psize += psize * refcnt; - dds->dds_ref_dsize += dsize * refcnt; - } -} - -void -ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg) -{ - const uint64_t *s = (const uint64_t *)src; - uint64_t *d = (uint64_t *)dst; - uint64_t *d_end = (uint64_t *)(dst + 1); - - ASSERT(neg == 0 || neg == -1ULL); /* add or subtract */ - - for (int i = 0; i < d_end - d; i++) - d[i] += (s[i] ^ neg) - neg; -} - -static void -ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg) -{ - ddt_stat_t dds; - ddt_histogram_t *ddh; - int bucket; - - ddt_stat_generate(ddt, dde, &dds); - - bucket = highbit64(dds.dds_ref_blocks) - 1; - ASSERT3U(bucket, >=, 0); - - ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class]; - - ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg); -} - -void -ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src) -{ - for (int h = 0; h < 64; h++) - ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0); -} - -void -ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh) -{ - memset(dds, 0, sizeof (*dds)); - - for (int h = 0; h < 64; h++) - ddt_stat_add(dds, &ddh->ddh_stat[h], 0); -} - -boolean_t -ddt_histogram_empty(const ddt_histogram_t *ddh) -{ - const uint64_t *s = (const uint64_t *)ddh; - const uint64_t *s_end = (const uint64_t *)(ddh + 1); - - while (s < s_end) - if (*s++ != 0) - return (B_FALSE); - - return (B_TRUE); -} - -void -ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total) -{ - /* Sum the statistics we cached in ddt_object_sync(). */ - for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { - ddt_t *ddt = spa->spa_ddt[c]; - for (enum ddt_type type = 0; type < DDT_TYPES; type++) { - for (enum ddt_class class = 0; class < DDT_CLASSES; - class++) { - ddt_object_t *ddo = - &ddt->ddt_object_stats[type][class]; - ddo_total->ddo_count += ddo->ddo_count; - ddo_total->ddo_dspace += ddo->ddo_dspace; - ddo_total->ddo_mspace += ddo->ddo_mspace; - } - } - } - - /* ... and compute the averages. */ - if (ddo_total->ddo_count != 0) { - ddo_total->ddo_dspace /= ddo_total->ddo_count; - ddo_total->ddo_mspace /= ddo_total->ddo_count; - } -} - -void -ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh) -{ - for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { - ddt_t *ddt = spa->spa_ddt[c]; - for (enum ddt_type type = 0; type < DDT_TYPES && ddt; type++) { - for (enum ddt_class class = 0; class < DDT_CLASSES; - class++) { - ddt_histogram_add(ddh, - &ddt->ddt_histogram_cache[type][class]); - } - } - } -} - -void -ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) -{ - ddt_histogram_t *ddh_total; - - ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); - ddt_get_dedup_histogram(spa, ddh_total); - ddt_histogram_stat(dds_total, ddh_total); - kmem_free(ddh_total, sizeof (ddt_histogram_t)); -} - -uint64_t -ddt_get_dedup_dspace(spa_t *spa) -{ - ddt_stat_t dds_total; - - if (spa->spa_dedup_dspace != ~0ULL) - return (spa->spa_dedup_dspace); - - memset(&dds_total, 0, sizeof (ddt_stat_t)); - - /* Calculate and cache the stats */ - ddt_get_dedup_stats(spa, &dds_total); - spa->spa_dedup_dspace = dds_total.dds_ref_dsize - dds_total.dds_dsize; - return (spa->spa_dedup_dspace); -} - -uint64_t -ddt_get_pool_dedup_ratio(spa_t *spa) -{ - ddt_stat_t dds_total = { 0 }; - - ddt_get_dedup_stats(spa, &dds_total); - if (dds_total.dds_dsize == 0) - return (100); - - return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize); -} - ddt_t * ddt_select(spa_t *spa, const blkptr_t *bp) { return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]); } void ddt_enter(ddt_t *ddt) { mutex_enter(&ddt->ddt_lock); } void ddt_exit(ddt_t *ddt) { mutex_exit(&ddt->ddt_lock); } void ddt_init(void) { ddt_cache = kmem_cache_create("ddt_cache", sizeof (ddt_t), 0, NULL, NULL, NULL, NULL, NULL, 0); ddt_entry_cache = kmem_cache_create("ddt_entry_cache", sizeof (ddt_entry_t), 0, NULL, NULL, NULL, NULL, NULL, 0); } void ddt_fini(void) { kmem_cache_destroy(ddt_entry_cache); kmem_cache_destroy(ddt_cache); } static ddt_entry_t * ddt_alloc(const ddt_key_t *ddk) { ddt_entry_t *dde; dde = kmem_cache_alloc(ddt_entry_cache, KM_SLEEP); memset(dde, 0, sizeof (ddt_entry_t)); cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL); dde->dde_key = *ddk; return (dde); } static void ddt_free(ddt_entry_t *dde) { ASSERT(!dde->dde_loading); for (int p = 0; p < DDT_PHYS_TYPES; p++) ASSERT3P(dde->dde_lead_zio[p], ==, NULL); if (dde->dde_repair_abd != NULL) abd_free(dde->dde_repair_abd); cv_destroy(&dde->dde_cv); kmem_cache_free(ddt_entry_cache, dde); } void ddt_remove(ddt_t *ddt, ddt_entry_t *dde) { ASSERT(MUTEX_HELD(&ddt->ddt_lock)); avl_remove(&ddt->ddt_tree, dde); ddt_free(dde); } ddt_entry_t * ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add) { ddt_key_t search; ddt_entry_t *dde; enum ddt_type type; enum ddt_class class; avl_index_t where; int error; ASSERT(MUTEX_HELD(&ddt->ddt_lock)); ddt_key_fill(&search, bp); dde = avl_find(&ddt->ddt_tree, &search, &where); if (dde == NULL) { if (!add) return (NULL); dde = ddt_alloc(&search); avl_insert(&ddt->ddt_tree, dde, where); } while (dde->dde_loading) cv_wait(&dde->dde_cv, &ddt->ddt_lock); if (dde->dde_loaded) return (dde); dde->dde_loading = B_TRUE; ddt_exit(ddt); error = ENOENT; for (type = 0; type < DDT_TYPES; type++) { for (class = 0; class < DDT_CLASSES; class++) { error = ddt_object_lookup(ddt, type, class, dde); if (error != ENOENT) { ASSERT0(error); break; } } if (error != ENOENT) break; } ddt_enter(ddt); ASSERT(!dde->dde_loaded); ASSERT(dde->dde_loading); dde->dde_type = type; /* will be DDT_TYPES if no entry found */ dde->dde_class = class; /* will be DDT_CLASSES if no entry found */ dde->dde_loaded = B_TRUE; dde->dde_loading = B_FALSE; if (error == 0) ddt_stat_update(ddt, dde, -1ULL); cv_broadcast(&dde->dde_cv); return (dde); } void ddt_prefetch(spa_t *spa, const blkptr_t *bp) { ddt_t *ddt; ddt_entry_t dde; if (!zfs_dedup_prefetch || bp == NULL || !BP_GET_DEDUP(bp)) return; /* * We only remove the DDT once all tables are empty and only * prefetch dedup blocks when there are entries in the DDT. * Thus no locking is required as the DDT can't disappear on us. */ ddt = ddt_select(spa, bp); ddt_key_fill(&dde.dde_key, bp); for (enum ddt_type type = 0; type < DDT_TYPES; type++) { for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { ddt_object_prefetch(ddt, type, class, &dde); } } } /* * Key comparison. Any struct wanting to make use of this function must have * the key as the first element. */ #define DDT_KEY_CMP_LEN (sizeof (ddt_key_t) / sizeof (uint16_t)) typedef struct ddt_key_cmp { uint16_t u16[DDT_KEY_CMP_LEN]; } ddt_key_cmp_t; int ddt_key_compare(const void *x1, const void *x2) { const ddt_key_cmp_t *k1 = (const ddt_key_cmp_t *)x1; const ddt_key_cmp_t *k2 = (const ddt_key_cmp_t *)x2; int32_t cmp = 0; for (int i = 0; i < DDT_KEY_CMP_LEN; i++) { cmp = (int32_t)k1->u16[i] - (int32_t)k2->u16[i]; if (likely(cmp)) break; } return (TREE_ISIGN(cmp)); } static ddt_t * ddt_table_alloc(spa_t *spa, enum zio_checksum c) { ddt_t *ddt; ddt = kmem_cache_alloc(ddt_cache, KM_SLEEP); memset(ddt, 0, sizeof (ddt_t)); mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL); avl_create(&ddt->ddt_tree, ddt_key_compare, sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); avl_create(&ddt->ddt_repair_tree, ddt_key_compare, sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node)); ddt->ddt_checksum = c; ddt->ddt_spa = spa; ddt->ddt_os = spa->spa_meta_objset; return (ddt); } static void ddt_table_free(ddt_t *ddt) { ASSERT0(avl_numnodes(&ddt->ddt_tree)); ASSERT0(avl_numnodes(&ddt->ddt_repair_tree)); avl_destroy(&ddt->ddt_tree); avl_destroy(&ddt->ddt_repair_tree); mutex_destroy(&ddt->ddt_lock); kmem_cache_free(ddt_cache, ddt); } void ddt_create(spa_t *spa) { spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM; for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) spa->spa_ddt[c] = ddt_table_alloc(spa, c); } int ddt_load(spa_t *spa) { int error; ddt_create(spa); error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DDT_STATS, sizeof (uint64_t), 1, &spa->spa_ddt_stat_object); if (error) return (error == ENOENT ? 0 : error); for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { ddt_t *ddt = spa->spa_ddt[c]; for (enum ddt_type type = 0; type < DDT_TYPES; type++) { for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { error = ddt_object_load(ddt, type, class); if (error != 0 && error != ENOENT) return (error); } } /* * Seed the cached histograms. */ memcpy(&ddt->ddt_histogram_cache, ddt->ddt_histogram, sizeof (ddt->ddt_histogram)); spa->spa_dedup_dspace = ~0ULL; } return (0); } void ddt_unload(spa_t *spa) { for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { if (spa->spa_ddt[c]) { ddt_table_free(spa->spa_ddt[c]); spa->spa_ddt[c] = NULL; } } } boolean_t ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp) { ddt_t *ddt; ddt_entry_t *dde; if (!BP_GET_DEDUP(bp)) return (B_FALSE); if (max_class == DDT_CLASS_UNIQUE) return (B_TRUE); ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)]; dde = kmem_cache_alloc(ddt_entry_cache, KM_SLEEP); ddt_key_fill(&(dde->dde_key), bp); for (enum ddt_type type = 0; type < DDT_TYPES; type++) { for (enum ddt_class class = 0; class <= max_class; class++) { if (ddt_object_lookup(ddt, type, class, dde) == 0) { kmem_cache_free(ddt_entry_cache, dde); return (B_TRUE); } } } kmem_cache_free(ddt_entry_cache, dde); return (B_FALSE); } ddt_entry_t * ddt_repair_start(ddt_t *ddt, const blkptr_t *bp) { ddt_key_t ddk; ddt_entry_t *dde; ddt_key_fill(&ddk, bp); dde = ddt_alloc(&ddk); for (enum ddt_type type = 0; type < DDT_TYPES; type++) { for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { /* * We can only do repair if there are multiple copies * of the block. For anything in the UNIQUE class, * there's definitely only one copy, so don't even try. */ if (class != DDT_CLASS_UNIQUE && ddt_object_lookup(ddt, type, class, dde) == 0) return (dde); } } memset(dde->dde_phys, 0, sizeof (dde->dde_phys)); return (dde); } void ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde) { avl_index_t where; ddt_enter(ddt); if (dde->dde_repair_abd != NULL && spa_writeable(ddt->ddt_spa) && avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL) avl_insert(&ddt->ddt_repair_tree, dde, where); else ddt_free(dde); ddt_exit(ddt); } static void ddt_repair_entry_done(zio_t *zio) { ddt_entry_t *rdde = zio->io_private; ddt_free(rdde); } static void ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio) { ddt_phys_t *ddp = dde->dde_phys; ddt_phys_t *rddp = rdde->dde_phys; ddt_key_t *ddk = &dde->dde_key; ddt_key_t *rddk = &rdde->dde_key; zio_t *zio; blkptr_t blk; zio = zio_null(rio, rio->io_spa, NULL, ddt_repair_entry_done, rdde, rio->io_flags); for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) { if (ddp->ddp_phys_birth == 0 || ddp->ddp_phys_birth != rddp->ddp_phys_birth || memcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva))) continue; ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk); zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk, rdde->dde_repair_abd, DDK_GET_PSIZE(rddk), NULL, NULL, ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL)); } zio_nowait(zio); } static void ddt_repair_table(ddt_t *ddt, zio_t *rio) { spa_t *spa = ddt->ddt_spa; ddt_entry_t *dde, *rdde_next, *rdde; avl_tree_t *t = &ddt->ddt_repair_tree; blkptr_t blk; if (spa_sync_pass(spa) > 1) return; ddt_enter(ddt); for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) { rdde_next = AVL_NEXT(t, rdde); avl_remove(&ddt->ddt_repair_tree, rdde); ddt_exit(ddt); ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk); dde = ddt_repair_start(ddt, &blk); ddt_repair_entry(ddt, dde, rdde, rio); ddt_repair_done(ddt, dde); ddt_enter(ddt); } ddt_exit(ddt); } static void ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg) { dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool; ddt_phys_t *ddp = dde->dde_phys; ddt_key_t *ddk = &dde->dde_key; enum ddt_type otype = dde->dde_type; enum ddt_type ntype = DDT_TYPE_CURRENT; enum ddt_class oclass = dde->dde_class; enum ddt_class nclass; uint64_t total_refcnt = 0; ASSERT(dde->dde_loaded); ASSERT(!dde->dde_loading); for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { ASSERT3P(dde->dde_lead_zio[p], ==, NULL); if (ddp->ddp_phys_birth == 0) { ASSERT0(ddp->ddp_refcnt); continue; } if (p == DDT_PHYS_DITTO) { /* * Note, we no longer create DDT-DITTO blocks, but we * don't want to leak any written by older software. */ ddt_phys_free(ddt, ddk, ddp, txg); continue; } if (ddp->ddp_refcnt == 0) ddt_phys_free(ddt, ddk, ddp, txg); total_refcnt += ddp->ddp_refcnt; } /* We do not create new DDT-DITTO blocks. */ ASSERT0(dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth); if (total_refcnt > 1) nclass = DDT_CLASS_DUPLICATE; else nclass = DDT_CLASS_UNIQUE; if (otype != DDT_TYPES && (otype != ntype || oclass != nclass || total_refcnt == 0)) { VERIFY0(ddt_object_remove(ddt, otype, oclass, dde, tx)); ASSERT3U( ddt_object_lookup(ddt, otype, oclass, dde), ==, ENOENT); } if (total_refcnt != 0) { dde->dde_type = ntype; dde->dde_class = nclass; ddt_stat_update(ddt, dde, 0); if (!ddt_object_exists(ddt, ntype, nclass)) ddt_object_create(ddt, ntype, nclass, tx); VERIFY0(ddt_object_update(ddt, ntype, nclass, dde, tx)); /* * If the class changes, the order that we scan this bp * changes. If it decreases, we could miss it, so * scan it right now. (This covers both class changing * while we are doing ddt_walk(), and when we are * traversing.) */ if (nclass < oclass) { dsl_scan_ddt_entry(dp->dp_scan, ddt->ddt_checksum, dde, tx); } } } static void ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg) { spa_t *spa = ddt->ddt_spa; ddt_entry_t *dde; void *cookie = NULL; if (avl_numnodes(&ddt->ddt_tree) == 0) return; ASSERT3U(spa->spa_uberblock.ub_version, >=, SPA_VERSION_DEDUP); if (spa->spa_ddt_stat_object == 0) { spa->spa_ddt_stat_object = zap_create_link(ddt->ddt_os, DMU_OT_DDT_STATS, DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DDT_STATS, tx); } while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) { ddt_sync_entry(ddt, dde, tx, txg); ddt_free(dde); } for (enum ddt_type type = 0; type < DDT_TYPES; type++) { uint64_t add, count = 0; for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { if (ddt_object_exists(ddt, type, class)) { ddt_object_sync(ddt, type, class, tx); VERIFY0(ddt_object_count(ddt, type, class, &add)); count += add; } } for (enum ddt_class class = 0; class < DDT_CLASSES; class++) { if (count == 0 && ddt_object_exists(ddt, type, class)) ddt_object_destroy(ddt, type, class, tx); } } memcpy(&ddt->ddt_histogram_cache, ddt->ddt_histogram, sizeof (ddt->ddt_histogram)); spa->spa_dedup_dspace = ~0ULL; } void ddt_sync(spa_t *spa, uint64_t txg) { dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; dmu_tx_t *tx; zio_t *rio; ASSERT3U(spa_syncing_txg(spa), ==, txg); tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); rio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SELF_HEAL); /* * This function may cause an immediate scan of ddt blocks (see * the comment above dsl_scan_ddt() for details). We set the * scan's root zio here so that we can wait for any scan IOs in * addition to the regular ddt IOs. */ ASSERT3P(scn->scn_zio_root, ==, NULL); scn->scn_zio_root = rio; for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { ddt_t *ddt = spa->spa_ddt[c]; if (ddt == NULL) continue; ddt_sync_table(ddt, tx, txg); ddt_repair_table(ddt, rio); } (void) zio_wait(rio); scn->scn_zio_root = NULL; dmu_tx_commit(tx); } int ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde) { do { do { do { ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum]; int error = ENOENT; if (ddt_object_exists(ddt, ddb->ddb_type, ddb->ddb_class)) { error = ddt_object_walk(ddt, ddb->ddb_type, ddb->ddb_class, &ddb->ddb_cursor, dde); } dde->dde_type = ddb->ddb_type; dde->dde_class = ddb->ddb_class; if (error == 0) return (0); if (error != ENOENT) return (error); ddb->ddb_cursor = 0; } while (++ddb->ddb_checksum < ZIO_CHECKSUM_FUNCTIONS); ddb->ddb_checksum = 0; } while (++ddb->ddb_type < DDT_TYPES); ddb->ddb_type = 0; } while (++ddb->ddb_class < DDT_CLASSES); return (SET_ERROR(ENOENT)); } /* * This function is used by Block Cloning (brt.c) to increase reference * counter for the DDT entry if the block is already in DDT. * * Return false if the block, despite having the D bit set, is not present * in the DDT. Currently this is not possible but might be in the future. * See the comment below. */ boolean_t ddt_addref(spa_t *spa, const blkptr_t *bp) { ddt_t *ddt; ddt_entry_t *dde; boolean_t result; spa_config_enter(spa, SCL_ZIO, FTAG, RW_READER); ddt = ddt_select(spa, bp); ddt_enter(ddt); dde = ddt_lookup(ddt, bp, B_TRUE); ASSERT3P(dde, !=, NULL); if (dde->dde_type < DDT_TYPES) { ddt_phys_t *ddp; ASSERT3S(dde->dde_class, <, DDT_CLASSES); ddp = &dde->dde_phys[BP_GET_NDVAS(bp)]; /* * This entry already existed (dde_type is real), so it must * have refcnt >0 at the start of this txg. We are called from * brt_pending_apply(), before frees are issued, so the refcnt * can't be lowered yet. Therefore, it must be >0. We assert * this because if the order of BRT and DDT interactions were * ever to change and the refcnt was ever zero here, then * likely further action is required to fill out the DDT entry, * and this is a place that is likely to be missed in testing. */ ASSERT3U(ddp->ddp_refcnt, >, 0); ddt_phys_addref(ddp); result = B_TRUE; } else { /* * At the time of implementating this if the block has the * DEDUP flag set it must exist in the DEDUP table, but * there are many advocates that want ability to remove * entries from DDT with refcnt=1. If this will happen, * we may have a block with the DEDUP set, but which doesn't * have a corresponding entry in the DDT. Be ready. */ ASSERT3S(dde->dde_class, ==, DDT_CLASSES); ddt_remove(ddt, dde); result = B_FALSE; } ddt_exit(ddt); spa_config_exit(spa, SCL_ZIO, FTAG); return (result); } ZFS_MODULE_PARAM(zfs_dedup, zfs_dedup_, prefetch, INT, ZMOD_RW, "Enable prefetching dedup-ed blks"); diff --git a/module/zfs/ddt_stats.c b/module/zfs/ddt_stats.c new file mode 100644 index 000000000000..05d0c22f92a5 --- /dev/null +++ b/module/zfs/ddt_stats.c @@ -0,0 +1,203 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or https://opensource.org/licenses/CDDL-1.0. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2016 by Delphix. All rights reserved. + * Copyright (c) 2022 by Pawel Jakub Dawidek + * Copyright (c) 2023, Klara Inc. + */ + +#include +#include +#include +#include + +static void +ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds) +{ + spa_t *spa = ddt->ddt_spa; + ddt_phys_t *ddp = dde->dde_phys; + ddt_key_t *ddk = &dde->dde_key; + uint64_t lsize = DDK_GET_LSIZE(ddk); + uint64_t psize = DDK_GET_PSIZE(ddk); + + memset(dds, 0, sizeof (*dds)); + + for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { + uint64_t dsize = 0; + uint64_t refcnt = ddp->ddp_refcnt; + + if (ddp->ddp_phys_birth == 0) + continue; + + for (int d = 0; d < DDE_GET_NDVAS(dde); d++) + dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]); + + dds->dds_blocks += 1; + dds->dds_lsize += lsize; + dds->dds_psize += psize; + dds->dds_dsize += dsize; + + dds->dds_ref_blocks += refcnt; + dds->dds_ref_lsize += lsize * refcnt; + dds->dds_ref_psize += psize * refcnt; + dds->dds_ref_dsize += dsize * refcnt; + } +} + +void +ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg) +{ + const uint64_t *s = (const uint64_t *)src; + uint64_t *d = (uint64_t *)dst; + uint64_t *d_end = (uint64_t *)(dst + 1); + + ASSERT(neg == 0 || neg == -1ULL); /* add or subtract */ + + for (int i = 0; i < d_end - d; i++) + d[i] += (s[i] ^ neg) - neg; +} + +void +ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg) +{ + ddt_stat_t dds; + ddt_histogram_t *ddh; + int bucket; + + ddt_stat_generate(ddt, dde, &dds); + + bucket = highbit64(dds.dds_ref_blocks) - 1; + ASSERT3U(bucket, >=, 0); + + ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class]; + + ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg); +} + +void +ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src) +{ + for (int h = 0; h < 64; h++) + ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0); +} + +void +ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh) +{ + memset(dds, 0, sizeof (*dds)); + + for (int h = 0; h < 64; h++) + ddt_stat_add(dds, &ddh->ddh_stat[h], 0); +} + +boolean_t +ddt_histogram_empty(const ddt_histogram_t *ddh) +{ + const uint64_t *s = (const uint64_t *)ddh; + const uint64_t *s_end = (const uint64_t *)(ddh + 1); + + while (s < s_end) + if (*s++ != 0) + return (B_FALSE); + + return (B_TRUE); +} + +void +ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total) +{ + /* Sum the statistics we cached in ddt_object_sync(). */ + for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { + ddt_t *ddt = spa->spa_ddt[c]; + for (enum ddt_type type = 0; type < DDT_TYPES; type++) { + for (enum ddt_class class = 0; class < DDT_CLASSES; + class++) { + ddt_object_t *ddo = + &ddt->ddt_object_stats[type][class]; + ddo_total->ddo_count += ddo->ddo_count; + ddo_total->ddo_dspace += ddo->ddo_dspace; + ddo_total->ddo_mspace += ddo->ddo_mspace; + } + } + } + + /* ... and compute the averages. */ + if (ddo_total->ddo_count != 0) { + ddo_total->ddo_dspace /= ddo_total->ddo_count; + ddo_total->ddo_mspace /= ddo_total->ddo_count; + } +} + +void +ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh) +{ + for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) { + ddt_t *ddt = spa->spa_ddt[c]; + for (enum ddt_type type = 0; type < DDT_TYPES && ddt; type++) { + for (enum ddt_class class = 0; class < DDT_CLASSES; + class++) { + ddt_histogram_add(ddh, + &ddt->ddt_histogram_cache[type][class]); + } + } + } +} + +void +ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) +{ + ddt_histogram_t *ddh_total; + + ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); + ddt_get_dedup_histogram(spa, ddh_total); + ddt_histogram_stat(dds_total, ddh_total); + kmem_free(ddh_total, sizeof (ddt_histogram_t)); +} + +uint64_t +ddt_get_dedup_dspace(spa_t *spa) +{ + ddt_stat_t dds_total; + + if (spa->spa_dedup_dspace != ~0ULL) + return (spa->spa_dedup_dspace); + + memset(&dds_total, 0, sizeof (ddt_stat_t)); + + /* Calculate and cache the stats */ + ddt_get_dedup_stats(spa, &dds_total); + spa->spa_dedup_dspace = dds_total.dds_ref_dsize - dds_total.dds_dsize; + return (spa->spa_dedup_dspace); +} + +uint64_t +ddt_get_pool_dedup_ratio(spa_t *spa) +{ + ddt_stat_t dds_total = { 0 }; + + ddt_get_dedup_stats(spa, &dds_total); + if (dds_total.dds_dsize == 0) + return (100); + + return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize); +}