diff --git a/Makefile.am b/Makefile.am
index d4b75264da73..37a9bfdf8697 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,241 +1,242 @@
include $(top_srcdir)/config/Shellcheck.am
ACLOCAL_AMFLAGS = -I config
SUBDIRS = include
if BUILD_LINUX
SUBDIRS += rpm
endif
if CONFIG_USER
SUBDIRS += man scripts lib tests cmd etc contrib
if BUILD_LINUX
SUBDIRS += udev
endif
endif
if CONFIG_KERNEL
SUBDIRS += module
extradir = $(prefix)/src/zfs-$(VERSION)
extra_HEADERS = zfs.release.in zfs_config.h.in
if BUILD_LINUX
kerneldir = $(prefix)/src/zfs-$(VERSION)/$(LINUX_VERSION)
nodist_kernel_HEADERS = zfs.release zfs_config.h module/$(LINUX_SYMBOLS)
endif
endif
AUTOMAKE_OPTIONS = foreign
EXTRA_DIST = autogen.sh copy-builtin
EXTRA_DIST += config/config.awk config/rpm.am config/deb.am config/tgz.am
EXTRA_DIST += AUTHORS CODE_OF_CONDUCT.md COPYRIGHT LICENSE META NEWS NOTICE
EXTRA_DIST += README.md RELEASES.md
EXTRA_DIST += module/lua/README.zfs module/os/linux/spl/README.md
# Include all the extra licensing information for modules
EXTRA_DIST += module/icp/algs/skein/THIRDPARTYLICENSE
EXTRA_DIST += module/icp/algs/skein/THIRDPARTYLICENSE.descrip
EXTRA_DIST += module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman
EXTRA_DIST += module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman.descrip
EXTRA_DIST += module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl
EXTRA_DIST += module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl.descrip
EXTRA_DIST += module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.cryptogams
EXTRA_DIST += module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.cryptogams.descrip
EXTRA_DIST += module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.openssl
EXTRA_DIST += module/icp/asm-x86_64/modes/THIRDPARTYLICENSE.openssl.descrip
EXTRA_DIST += module/os/linux/spl/THIRDPARTYLICENSE.gplv2
EXTRA_DIST += module/os/linux/spl/THIRDPARTYLICENSE.gplv2.descrip
EXTRA_DIST += module/zfs/THIRDPARTYLICENSE.cityhash
EXTRA_DIST += module/zfs/THIRDPARTYLICENSE.cityhash.descrip
@CODE_COVERAGE_RULES@
GITREV = include/zfs_gitrev.h
PHONY = gitrev
gitrev:
$(AM_V_GEN)$(top_srcdir)/scripts/make_gitrev.sh $(GITREV)
all: gitrev
# Double-colon rules are allowed; there are multiple independent definitions.
maintainer-clean-local::
-$(RM) $(GITREV)
distclean-local::
-$(RM) -R autom4te*.cache build
-find . \( -name SCCS -o -name BitKeeper -o -name .svn -o -name CVS \
-o -name .pc -o -name .hg -o -name .git \) -prune -o \
\( -name '*.orig' -o -name '*.rej' -o -name '*~' \
-o -name '*.bak' -o -name '#*#' -o -name '.*.orig' \
-o -name '.*.rej' -o -size 0 -o -name '*%' -o -name '.*.cmd' \
-o -name 'core' -o -name 'Makefile' -o -name 'Module.symvers' \
-o -name '*.order' -o -name '*.markers' -o -name '*.gcda' \
-o -name '*.gcno' \) \
-type f -print | xargs $(RM)
all-local:
-[ -x ${top_builddir}/scripts/zfs-tests.sh ] && \
${top_builddir}/scripts/zfs-tests.sh -c
dist-hook:
$(AM_V_GEN)$(top_srcdir)/scripts/make_gitrev.sh -D $(distdir) $(GITREV)
$(SED) ${ac_inplace} -e 's/Release:[[:print:]]*/Release: $(RELEASE)/' \
$(distdir)/META
if BUILD_LINUX
# For compatibility, create a matching spl-x.y.z directly which contains
# symlinks to the updated header and object file locations. These
# compatibility links will be removed in the next major release.
if CONFIG_KERNEL
install-data-hook:
rm -rf $(DESTDIR)$(prefix)/src/spl-$(VERSION) && \
mkdir $(DESTDIR)$(prefix)/src/spl-$(VERSION) && \
cd $(DESTDIR)$(prefix)/src/spl-$(VERSION) && \
ln -s ../zfs-$(VERSION)/include/spl include && \
ln -s ../zfs-$(VERSION)/$(LINUX_VERSION) $(LINUX_VERSION) && \
ln -s ../zfs-$(VERSION)/zfs_config.h.in spl_config.h.in && \
ln -s ../zfs-$(VERSION)/zfs.release.in spl.release.in && \
cd $(DESTDIR)$(prefix)/src/zfs-$(VERSION)/$(LINUX_VERSION) && \
ln -fs zfs_config.h spl_config.h && \
ln -fs zfs.release spl.release
endif
endif
PHONY += codecheck
codecheck: cstyle shellcheck checkbashisms flake8 mancheck testscheck vcscheck
PHONY += checkstyle
checkstyle: codecheck commitcheck
PHONY += commitcheck
commitcheck:
@if git rev-parse --git-dir > /dev/null 2>&1; then \
${top_srcdir}/scripts/commitcheck.sh; \
fi
if HAVE_PARALLEL
cstyle_line = -print0 | parallel -X0 ${top_srcdir}/scripts/cstyle.pl -cpP {}
else
cstyle_line = -exec ${top_srcdir}/scripts/cstyle.pl -cpP {} +
endif
PHONY += cstyle
cstyle:
@find ${top_srcdir} -name build -prune \
-o -type f -name '*.[hc]' \
! -name 'zfs_config.*' ! -name '*.mod.c' \
! -name 'opt_global.h' ! -name '*_if*.h' \
! -path './module/zstd/lib/*' \
+ ! -path './module/zfs/lz4.c' \
$(cstyle_line)
filter_executable = -exec test -x '{}' \; -print
SHELLCHECKDIRS = cmd contrib etc scripts tests
SHELLCHECKSCRIPTS = autogen.sh
PHONY += checkabi storeabi
checklibabiversion:
libabiversion=`abidw -v | $(SED) 's/[^0-9]//g'`; \
if test $$libabiversion -lt "200"; then \
/bin/echo -e "\n" \
"*** Please use libabigail 2.0.0 version or newer;\n" \
"*** otherwise results are not consistent!\n" \
"(or see https://github.com/openzfs/libabigail-docker )\n"; \
exit 1; \
fi;
checkabi: checklibabiversion lib
$(MAKE) -C lib checkabi
storeabi: checklibabiversion lib
$(MAKE) -C lib storeabi
PHONY += mancheck
mancheck:
${top_srcdir}/scripts/mancheck.sh ${top_srcdir}/man ${top_srcdir}/tests/test-runner/man
if BUILD_LINUX
stat_fmt = -c '%A %n'
else
stat_fmt = -f '%Sp %N'
endif
PHONY += testscheck
testscheck:
@find ${top_srcdir}/tests/zfs-tests -type f \
\( -name '*.ksh' -not ${filter_executable} \) -o \
\( -name '*.kshlib' ${filter_executable} \) -o \
\( -name '*.shlib' ${filter_executable} \) -o \
\( -name '*.cfg' ${filter_executable} \) | \
xargs -r stat ${stat_fmt} | \
awk '{c++; print} END {if(c>0) exit 1}'
PHONY += vcscheck
vcscheck:
@if git rev-parse --git-dir > /dev/null 2>&1; then \
git ls-files . --exclude-standard --others | \
awk '{c++; print} END {if(c>0) exit 1}' ; \
fi
PHONY += lint
lint: cppcheck paxcheck
CPPCHECKDIRS = cmd lib module
PHONY += cppcheck
cppcheck: $(CPPCHECKDIRS)
@if test -n "$(CPPCHECK)"; then \
set -e ; for dir in $(CPPCHECKDIRS) ; do \
$(MAKE) -C $$dir cppcheck ; \
done \
else \
echo "skipping cppcheck because cppcheck is not installed"; \
fi
PHONY += paxcheck
paxcheck:
@if type scanelf > /dev/null 2>&1; then \
${top_srcdir}/scripts/paxcheck.sh ${top_builddir}; \
else \
echo "skipping paxcheck because scanelf is not installed"; \
fi
PHONY += flake8
flake8:
@if type flake8 > /dev/null 2>&1; then \
flake8 ${top_srcdir}; \
else \
echo "skipping flake8 because flake8 is not installed"; \
fi
PHONY += ctags
ctags:
$(RM) tags
find $(top_srcdir) -name '.?*' -prune \
-o -type f -name '*.[hcS]' -print | xargs ctags -a
PHONY += etags
etags:
$(RM) TAGS
find $(top_srcdir) -name '.?*' -prune \
-o -type f -name '*.[hcS]' -print | xargs etags -a
PHONY += cscopelist
cscopelist:
find $(top_srcdir) -name '.?*' -prune \
-o -type f -name '*.[hc]' -print >cscope.files
PHONY += tags
tags: ctags etags
PHONY += pkg pkg-dkms pkg-kmod pkg-utils
pkg: @DEFAULT_PACKAGE@
pkg-dkms: @DEFAULT_PACKAGE@-dkms
pkg-kmod: @DEFAULT_PACKAGE@-kmod
pkg-utils: @DEFAULT_PACKAGE@-utils
include config/rpm.am
include config/deb.am
include config/tgz.am
.PHONY: $(PHONY)
diff --git a/config/ax_code_coverage.m4 b/config/ax_code_coverage.m4
index 3e3c666f3c54..cd2498358df2 100644
--- a/config/ax_code_coverage.m4
+++ b/config/ax_code_coverage.m4
@@ -1,268 +1,269 @@
# ===========================================================================
# https://www.gnu.org/software/autoconf-archive/ax_code_coverage.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_CODE_COVERAGE()
#
# DESCRIPTION
#
# Defines CODE_COVERAGE_CPPFLAGS, CODE_COVERAGE_CFLAGS,
# CODE_COVERAGE_CXXFLAGS and CODE_COVERAGE_LIBS which should be included
# in the CPPFLAGS, CFLAGS CXXFLAGS and LIBS/LIBADD variables of every
# build target (program or library) which should be built with code
# coverage support. Also defines CODE_COVERAGE_RULES which should be
# substituted in your Makefile; and $enable_code_coverage which can be
# used in subsequent configure output. CODE_COVERAGE_ENABLED is defined
# and substituted, and corresponds to the value of the
# --enable-code-coverage option, which defaults to being disabled.
#
# Test also for gcov program and create GCOV variable that could be
# substituted.
#
# Note that all optimization flags in CFLAGS must be disabled when code
# coverage is enabled.
#
# Usage example:
#
# configure.ac:
#
# AX_CODE_COVERAGE
#
# Makefile.am:
#
# @CODE_COVERAGE_RULES@
# my_program_LIBS = ... $(CODE_COVERAGE_LIBS) ...
# my_program_CPPFLAGS = ... $(CODE_COVERAGE_CPPFLAGS) ...
# my_program_CFLAGS = ... $(CODE_COVERAGE_CFLAGS) ...
# my_program_CXXFLAGS = ... $(CODE_COVERAGE_CXXFLAGS) ...
#
# This results in a "check-code-coverage" rule being added to any
# Makefile.am which includes "@CODE_COVERAGE_RULES@" (assuming the module
# has been configured with --enable-code-coverage). Running `make
# check-code-coverage` in that directory will run the module's test suite
# (`make check`) and build a code coverage report detailing the code which
# was touched, then print the URI for the report.
#
# In earlier versions of this macro, CODE_COVERAGE_LDFLAGS was defined
# instead of CODE_COVERAGE_LIBS. They are both still defined, but use of
# CODE_COVERAGE_LIBS is preferred for clarity; CODE_COVERAGE_LDFLAGS is
# deprecated. They have the same value.
#
# This code was derived from Makefile.decl in GLib, originally licensed
# under LGPLv2.1+.
#
# LICENSE
#
# Copyright (c) 2012, 2016 Philip Withnall
# Copyright (c) 2012 Xan Lopez
# Copyright (c) 2012 Christian Persch
# Copyright (c) 2012 Paolo Borelli
# Copyright (c) 2012 Dan Winship
# Copyright (c) 2015 Bastien ROUCARIES
#
# This library is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or (at
# your option) any later version.
#
# This library is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
# General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see .
#serial 25
AC_DEFUN([AX_CODE_COVERAGE],[
dnl Check for --enable-code-coverage
AC_REQUIRE([AC_PROG_SED])
# allow to override gcov location
AC_ARG_WITH([gcov],
[AS_HELP_STRING([--with-gcov[=GCOV]], [use given GCOV for coverage (GCOV=gcov).])],
[_AX_CODE_COVERAGE_GCOV_PROG_WITH=$with_gcov],
[_AX_CODE_COVERAGE_GCOV_PROG_WITH=gcov])
AC_MSG_CHECKING([whether to build with code coverage support])
AC_ARG_ENABLE([code-coverage],
AS_HELP_STRING([--enable-code-coverage],
[Whether to enable code coverage support]),,
enable_code_coverage=no)
AM_CONDITIONAL([CODE_COVERAGE_ENABLED], [test x$enable_code_coverage = xyes])
AC_SUBST([CODE_COVERAGE_ENABLED], [$enable_code_coverage])
AC_MSG_RESULT($enable_code_coverage)
AS_IF([ test "$enable_code_coverage" = "yes" ], [
# check for gcov
AC_CHECK_TOOL([GCOV],
[$_AX_CODE_COVERAGE_GCOV_PROG_WITH],
[:])
AS_IF([test "X$GCOV" = "X:"],
[AC_MSG_ERROR([gcov is needed to do coverage])])
AC_SUBST([GCOV])
dnl Check if gcc is being used
AS_IF([ test "$GCC" = "no" ], [
AC_MSG_ERROR([not compiling with gcc, which is required for gcov code coverage])
])
AC_CHECK_PROG([LCOV], [lcov], [lcov])
AC_CHECK_PROG([GENHTML], [genhtml], [genhtml])
AS_IF([ test -z "$LCOV" ], [
AC_MSG_ERROR([To enable code coverage reporting you must have lcov installed])
])
AS_IF([ test -z "$GENHTML" ], [
AC_MSG_ERROR([Could not find genhtml from the lcov package])
])
dnl Build the code coverage flags
dnl Define CODE_COVERAGE_LDFLAGS for backwards compatibility
CODE_COVERAGE_CPPFLAGS=""
CODE_COVERAGE_CFLAGS="-O0 -g -fprofile-arcs -ftest-coverage"
CODE_COVERAGE_CXXFLAGS="-O0 -g -fprofile-arcs -ftest-coverage"
CODE_COVERAGE_LIBS="-lgcov"
CODE_COVERAGE_LDFLAGS="$CODE_COVERAGE_LIBS"
AC_SUBST([CODE_COVERAGE_CPPFLAGS])
AC_SUBST([CODE_COVERAGE_CFLAGS])
AC_SUBST([CODE_COVERAGE_CXXFLAGS])
AC_SUBST([CODE_COVERAGE_LIBS])
AC_SUBST([CODE_COVERAGE_LDFLAGS])
[CODE_COVERAGE_RULES_CHECK='
-$(A''M_V_at)$(MAKE) $(AM_MAKEFLAGS) -k check
$(A''M_V_at)$(MAKE) $(AM_MAKEFLAGS) code-coverage-capture
']
[CODE_COVERAGE_RULES_CAPTURE='
$(code_coverage_v_lcov_cap)$(LCOV) $(code_coverage_quiet) $(addprefix --directory ,$(CODE_COVERAGE_DIRECTORY)) --capture --output-file "$(CODE_COVERAGE_OUTPUT_FILE).tmp" --test-name "$(call code_coverage_sanitize,$(PACKAGE_NAME)-$(PACKAGE_VERSION))" --no-checksum --compat-libtool $(CODE_COVERAGE_LCOV_SHOPTS) $(CODE_COVERAGE_LCOV_OPTIONS)
$(code_coverage_v_lcov_ign)$(LCOV) $(code_coverage_quiet) $(addprefix --directory ,$(CODE_COVERAGE_DIRECTORY)) --remove "$(CODE_COVERAGE_OUTPUT_FILE).tmp" $(CODE_COVERAGE_IGNORE_PATTERN) --output-file "$(CODE_COVERAGE_OUTPUT_FILE)" $(CODE_COVERAGE_LCOV_SHOPTS) $(CODE_COVERAGE_LCOV_RMOPTS)
-@rm -f $(CODE_COVERAGE_OUTPUT_FILE).tmp
$(code_coverage_v_genhtml)LANG=C $(GENHTML) $(code_coverage_quiet) $(addprefix --prefix ,$(CODE_COVERAGE_DIRECTORY)) --output-directory "$(CODE_COVERAGE_OUTPUT_DIRECTORY)" --title "$(PACKAGE_NAME)-$(PACKAGE_VERSION) Code Coverage" --legend --show-details "$(CODE_COVERAGE_OUTPUT_FILE)" $(CODE_COVERAGE_GENHTML_OPTIONS)
@echo "file://$(abs_builddir)/$(CODE_COVERAGE_OUTPUT_DIRECTORY)/index.html"
']
[CODE_COVERAGE_RULES_CLEAN='
clean: code-coverage-clean
distclean: code-coverage-clean
code-coverage-clean:
-$(LCOV) --directory $(top_builddir) -z
-rm -rf $(CODE_COVERAGE_OUTPUT_FILE) $(CODE_COVERAGE_OUTPUT_FILE).tmp $(CODE_COVERAGE_OUTPUT_DIRECTORY)
-find . \( -name "*.gcda" -o -name "*.gcno" -o -name "*.gcov" \) -delete
']
], [
[CODE_COVERAGE_RULES_CHECK='
@echo "Need to reconfigure with --enable-code-coverage"
']
CODE_COVERAGE_RULES_CAPTURE="$CODE_COVERAGE_RULES_CHECK"
CODE_COVERAGE_RULES_CLEAN=''
])
[CODE_COVERAGE_RULES='
# Code coverage
#
# Optional:
# - CODE_COVERAGE_DIRECTORY: Top-level directory for code coverage reporting.
# Multiple directories may be specified, separated by whitespace.
# (Default: $(top_builddir))
# - CODE_COVERAGE_OUTPUT_FILE: Filename and path for the .info file generated
# by lcov for code coverage. (Default:
# $(PACKAGE_NAME)-$(PACKAGE_VERSION)-coverage.info)
# - CODE_COVERAGE_OUTPUT_DIRECTORY: Directory for generated code coverage
# reports to be created. (Default:
# $(PACKAGE_NAME)-$(PACKAGE_VERSION)-coverage)
# - CODE_COVERAGE_BRANCH_COVERAGE: Set to 1 to enforce branch coverage,
# set to 0 to disable it and leave empty to stay with the default.
# (Default: empty)
# - CODE_COVERAGE_LCOV_SHOPTS_DEFAULT: Extra options shared between both lcov
# instances. (Default: based on $CODE_COVERAGE_BRANCH_COVERAGE)
# - CODE_COVERAGE_LCOV_SHOPTS: Extra options to shared between both lcov
# instances. (Default: $CODE_COVERAGE_LCOV_SHOPTS_DEFAULT)
# - CODE_COVERAGE_LCOV_OPTIONS_GCOVPATH: --gcov-tool pathtogcov
# - CODE_COVERAGE_LCOV_OPTIONS_DEFAULT: Extra options to pass to the
# collecting lcov instance. (Default: $CODE_COVERAGE_LCOV_OPTIONS_GCOVPATH)
# - CODE_COVERAGE_LCOV_OPTIONS: Extra options to pass to the collecting lcov
# instance. (Default: $CODE_COVERAGE_LCOV_OPTIONS_DEFAULT)
# - CODE_COVERAGE_LCOV_RMOPTS_DEFAULT: Extra options to pass to the filtering
# lcov instance. (Default: empty)
# - CODE_COVERAGE_LCOV_RMOPTS: Extra options to pass to the filtering lcov
# instance. (Default: $CODE_COVERAGE_LCOV_RMOPTS_DEFAULT)
# - CODE_COVERAGE_GENHTML_OPTIONS_DEFAULT: Extra options to pass to the
# genhtml instance. (Default: based on $CODE_COVERAGE_BRANCH_COVERAGE)
# - CODE_COVERAGE_GENHTML_OPTIONS: Extra options to pass to the genhtml
# instance. (Default: $CODE_COVERAGE_GENHTML_OPTIONS_DEFAULT)
# - CODE_COVERAGE_IGNORE_PATTERN: Extra glob pattern of files to ignore
#
# The generated report will be titled using the $(PACKAGE_NAME) and
# $(PACKAGE_VERSION). In order to add the current git hash to the title,
# use the git-version-gen script, available online.
# Optional variables
CODE_COVERAGE_DIRECTORY ?= $(top_builddir)
CODE_COVERAGE_OUTPUT_FILE ?= $(PACKAGE_NAME)-$(PACKAGE_VERSION)-coverage.info
CODE_COVERAGE_OUTPUT_DIRECTORY ?= $(PACKAGE_NAME)-$(PACKAGE_VERSION)-coverage
CODE_COVERAGE_BRANCH_COVERAGE ?=
CODE_COVERAGE_LCOV_SHOPTS_DEFAULT ?= $(if $(CODE_COVERAGE_BRANCH_COVERAGE),\
--rc lcov_branch_coverage=$(CODE_COVERAGE_BRANCH_COVERAGE))
CODE_COVERAGE_LCOV_SHOPTS ?= $(CODE_COVERAGE_LCOV_SHOPTS_DEFAULT)
CODE_COVERAGE_LCOV_OPTIONS_GCOVPATH ?= --gcov-tool "$(GCOV)"
CODE_COVERAGE_LCOV_OPTIONS_DEFAULT ?= $(CODE_COVERAGE_LCOV_OPTIONS_GCOVPATH)
CODE_COVERAGE_LCOV_OPTIONS ?= $(CODE_COVERAGE_LCOV_OPTIONS_DEFAULT)
CODE_COVERAGE_LCOV_RMOPTS_DEFAULT ?=
CODE_COVERAGE_LCOV_RMOPTS ?= $(CODE_COVERAGE_LCOV_RMOPTS_DEFAULT)
CODE_COVERAGE_GENHTML_OPTIONS_DEFAULT ?=\
$(if $(CODE_COVERAGE_BRANCH_COVERAGE),\
--rc genhtml_branch_coverage=$(CODE_COVERAGE_BRANCH_COVERAGE))
CODE_COVERAGE_GENHTML_OPTIONS ?= $(CODE_COVERAGE_GENHTML_OPTIONS_DEFAULT)
# Add any folders you want to ignore here
# Ignore tmp and tests themselves
CODE_COVERAGE_IGNORE_PATTERN ?= "/tmp/*" "*/tests/*"
CODE_COVERAGE_IGNORE_PATTERN += "*/module/zstd/lib/*"
+CODE_COVERAGE_IGNORE_PATTERN += "*/module/zfs/lz4.c"
GITIGNOREFILES ?=
GITIGNOREFILES += $(CODE_COVERAGE_OUTPUT_FILE) $(CODE_COVERAGE_OUTPUT_DIRECTORY)
code_coverage_v_lcov_cap = $(code_coverage_v_lcov_cap_$(V))
code_coverage_v_lcov_cap_ = $(code_coverage_v_lcov_cap_$(AM_DEFAULT_VERBOSITY))
code_coverage_v_lcov_cap_0 = @echo " LCOV --capture"\
$(CODE_COVERAGE_OUTPUT_FILE);
code_coverage_v_lcov_ign = $(code_coverage_v_lcov_ign_$(V))
code_coverage_v_lcov_ign_ = $(code_coverage_v_lcov_ign_$(AM_DEFAULT_VERBOSITY))
code_coverage_v_lcov_ign_0 = @echo " LCOV --remove /tmp/*"\
$(CODE_COVERAGE_IGNORE_PATTERN);
code_coverage_v_genhtml = $(code_coverage_v_genhtml_$(V))
code_coverage_v_genhtml_ = $(code_coverage_v_genhtml_$(AM_DEFAULT_VERBOSITY))
code_coverage_v_genhtml_0 = @echo " GEN " $(CODE_COVERAGE_OUTPUT_DIRECTORY);
code_coverage_quiet = $(code_coverage_quiet_$(V))
code_coverage_quiet_ = $(code_coverage_quiet_$(AM_DEFAULT_VERBOSITY))
code_coverage_quiet_0 = --quiet
# sanitizes the test-name: replaces with underscores: dashes and dots
code_coverage_sanitize = $(subst -,_,$(subst .,_,$(1)))
# Use recursive makes in order to ignore errors during check
check-code-coverage:'"$CODE_COVERAGE_RULES_CHECK"'
# Capture code coverage data
code-coverage-capture: code-coverage-capture-hook'"$CODE_COVERAGE_RULES_CAPTURE"'
# Hook rule executed before code-coverage-capture, overridable by the user
code-coverage-capture-hook:
'"$CODE_COVERAGE_RULES_CLEAN"'
A''M_DISTCHECK_CONFIGURE_FLAGS ?=
A''M_DISTCHECK_CONFIGURE_FLAGS += --disable-code-coverage
.PHONY: check-code-coverage code-coverage-capture code-coverage-capture-hook code-coverage-clean
']
AC_SUBST([CODE_COVERAGE_RULES])
m4_ifdef([_AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE([CODE_COVERAGE_RULES])])
])
diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am
index dce3f81b829d..e49577ec4a63 100644
--- a/lib/libzpool/Makefile.am
+++ b/lib/libzpool/Makefile.am
@@ -1,240 +1,241 @@
include $(top_srcdir)/config/Rules.am
VPATH = \
$(top_srcdir)/module/zfs \
$(top_srcdir)/module/zcommon \
$(top_srcdir)/module/lua \
$(top_srcdir)/module/os/linux/zfs \
$(top_srcdir)/lib/libzpool
if BUILD_FREEBSD
DEFAULT_INCLUDES += -I$(top_srcdir)/include/os/freebsd/zfs
endif
if BUILD_LINUX
DEFAULT_INCLUDES += -I$(top_srcdir)/include/os/linux/zfs
endif
# Unconditionally enable debugging for libzpool
AM_CPPFLAGS += -DDEBUG -UNDEBUG -DZFS_DEBUG
# Suppress unused but set variable warnings often due to ASSERTs
AM_CFLAGS += $(NO_UNUSED_BUT_SET_VARIABLE)
# Includes kernel code generate warnings for large stack frames
AM_CFLAGS += $(FRAME_LARGER_THAN)
AM_CFLAGS += $(ZLIB_CFLAGS)
AM_CFLAGS += -DLIB_ZPOOL_BUILD
lib_LTLIBRARIES = libzpool.la
USER_C = \
kernel.c \
taskq.c \
util.c
KERNEL_C = \
zfeature_common.c \
zfs_comutil.c \
zfs_deleg.c \
zfs_fletcher.c \
zfs_fletcher_aarch64_neon.c \
zfs_fletcher_avx512.c \
zfs_fletcher_intel.c \
zfs_fletcher_sse.c \
zfs_fletcher_superscalar.c \
zfs_fletcher_superscalar4.c \
zfs_namecheck.c \
zfs_prop.c \
zpool_prop.c \
zprop_common.c \
abd.c \
abd_os.c \
aggsum.c \
arc.c \
arc_os.c \
blkptr.c \
bplist.c \
bpobj.c \
bptree.c \
bqueue.c \
btree.c \
cityhash.c \
dbuf.c \
dbuf_stats.c \
ddt.c \
ddt_zap.c \
dmu.c \
dmu_diff.c \
dmu_object.c \
dmu_objset.c \
dmu_recv.c \
dmu_redact.c \
dmu_send.c \
dmu_traverse.c \
dmu_tx.c \
dmu_zfetch.c \
dnode.c \
dnode_sync.c \
dsl_bookmark.c \
dsl_crypt.c \
dsl_dataset.c \
dsl_deadlist.c \
dsl_deleg.c \
dsl_destroy.c \
dsl_dir.c \
dsl_pool.c \
dsl_prop.c \
dsl_scan.c \
dsl_synctask.c \
dsl_userhold.c \
edonr_zfs.c \
fm.c \
gzip.c \
hkdf.c \
lz4.c \
+ lz4_zfs.c \
lzjb.c \
metaslab.c \
mmp.c \
multilist.c \
objlist.c \
pathname.c \
range_tree.c \
refcount.c \
rrwlock.c \
sa.c \
sha256.c \
skein_zfs.c \
spa.c \
spa_boot.c \
spa_checkpoint.c \
spa_config.c \
spa_errlog.c \
spa_history.c \
spa_log_spacemap.c \
spa_misc.c \
spa_stats.c \
space_map.c \
space_reftree.c \
trace.c \
txg.c \
uberblock.c \
unique.c \
vdev.c \
vdev_cache.c \
vdev_draid.c \
vdev_draid_rand.c \
vdev_file.c \
vdev_indirect.c \
vdev_indirect_births.c \
vdev_indirect_mapping.c \
vdev_initialize.c \
vdev_label.c \
vdev_mirror.c \
vdev_missing.c \
vdev_queue.c \
vdev_raidz.c \
vdev_raidz_math.c \
vdev_raidz_math_aarch64_neon.c \
vdev_raidz_math_aarch64_neonx2.c \
vdev_raidz_math_avx2.c \
vdev_raidz_math_avx512bw.c \
vdev_raidz_math_avx512f.c \
vdev_raidz_math_powerpc_altivec.c \
vdev_raidz_math_scalar.c \
vdev_raidz_math_sse2.c \
vdev_raidz_math_ssse3.c \
vdev_rebuild.c \
vdev_removal.c \
vdev_root.c \
vdev_trim.c \
zap.c \
zap_leaf.c \
zap_micro.c \
zcp.c \
zcp_get.c \
zcp_global.c \
zcp_iter.c \
zcp_set.c \
zcp_synctask.c \
zfeature.c \
zfs_byteswap.c \
zfs_debug.c \
zfs_fm.c \
zfs_fuid.c \
zfs_racct.c \
zfs_sa.c \
zfs_ratelimit.c \
zfs_rlock.c \
zfs_znode.c \
zil.c \
zio.c \
zio_checksum.c \
zio_compress.c \
zio_crypt.c \
zio_inject.c \
zle.c \
zrlock.c \
zthr.c
LUA_C = \
lapi.c \
lauxlib.c \
lbaselib.c \
lcode.c \
lcompat.c \
lcorolib.c \
lctype.c \
ldebug.c \
ldo.c \
lfunc.c \
lgc.c \
llex.c \
lmem.c \
lobject.c \
lopcodes.c \
lparser.c \
lstate.c \
lstring.c \
lstrlib.c \
ltable.c \
ltablib.c \
ltm.c \
lvm.c \
lzio.c
dist_libzpool_la_SOURCES = \
$(USER_C)
nodist_libzpool_la_SOURCES = \
$(KERNEL_C) \
$(LUA_C)
libzpool_la_LIBADD = \
$(abs_top_builddir)/lib/libicp/libicp.la \
$(abs_top_builddir)/lib/libunicode/libunicode.la \
$(abs_top_builddir)/lib/libnvpair/libnvpair.la \
$(abs_top_builddir)/lib/libzstd/libzstd.la \
$(abs_top_builddir)/lib/libzutil/libzutil.la
libzpool_la_LIBADD += $(LIBCLOCK_GETTIME) $(ZLIB_LIBS) -ldl -lm
libzpool_la_LDFLAGS = -pthread
if !ASAN_ENABLED
libzpool_la_LDFLAGS += -Wl,-z,defs
endif
if BUILD_FREEBSD
libzpool_la_LIBADD += -lgeom
endif
libzpool_la_LDFLAGS += -version-info 5:0:0
if TARGET_CPU_POWERPC
vdev_raidz_math_powerpc_altivec.$(OBJEXT): CFLAGS += -maltivec
vdev_raidz_math_powerpc_altivec.l$(OBJEXT): CFLAGS += -maltivec
endif
include $(top_srcdir)/config/CppCheck.am
diff --git a/module/Makefile.bsd b/module/Makefile.bsd
index 315be2808605..5172394e6ac8 100644
--- a/module/Makefile.bsd
+++ b/module/Makefile.bsd
@@ -1,375 +1,376 @@
.if !defined(WITH_CTF)
WITH_CTF=1
.endif
.include
SRCDIR=${.CURDIR}
INCDIR=${.CURDIR:H}/include
KMOD= openzfs
.PATH: ${SRCDIR}/avl \
${SRCDIR}/lua \
${SRCDIR}/nvpair \
${SRCDIR}/icp/algs/edonr \
${SRCDIR}/os/freebsd/spl \
${SRCDIR}/os/freebsd/zfs \
${SRCDIR}/unicode \
${SRCDIR}/zcommon \
${SRCDIR}/zfs \
${SRCDIR}/zstd \
${SRCDIR}/zstd/lib
CFLAGS+= -I${.OBJDIR:H}/include
CFLAGS+= -I${INCDIR}
CFLAGS+= -I${INCDIR}/os/freebsd
CFLAGS+= -I${INCDIR}/os/freebsd/spl
CFLAGS+= -I${INCDIR}/os/freebsd/zfs
CFLAGS+= -I${SRCDIR}/zstd/include
CFLAGS+= -include ${INCDIR}/os/freebsd/spl/sys/ccompile.h
CFLAGS+= -D__KERNEL__ -DFREEBSD_NAMECACHE -DBUILDING_ZFS -D__BSD_VISIBLE=1 \
-DHAVE_UIO_ZEROCOPY -DWITHOUT_NETDUMP -D__KERNEL -D_SYS_CONDVAR_H_ \
-D_SYS_VMEM_H_ -DKDTRACE_HOOKS -DSMP -DHAVE_KSID -DCOMPAT_FREEBSD11
.if ${MACHINE_ARCH} == "amd64"
CFLAGS+= -DHAVE_AVX2 -DHAVE_AVX -D__x86_64 -DHAVE_SSE2 -DHAVE_AVX512F -DHAVE_SSSE3
.endif
.if defined(WITH_DEBUG) && ${WITH_DEBUG} == "true"
CFLAGS+= -DZFS_DEBUG -g
.if defined(WITH_INVARIANTS) && ${WITH_INVARIANTS} == "true"
CFLAGS+= -DINVARIANTS -DWITNESS -DOPENSOLARIS_WITNESS
.endif
.if defined(WITH_O0) && ${WITH_O0} == "true"
CFLAGS+= -O0
.endif
.else
CFLAGS += -DNDEBUG
.endif
.if defined(WITH_VFS_DEBUG) && ${WITH_VFS_DEBUG} == "true"
# kernel must also be built with this option for this to work
CFLAGS+= -DDEBUG_VFS_LOCKS
.endif
.if defined(WITH_GCOV) && ${WITH_GCOV} == "true"
CFLAGS+= -fprofile-arcs -ftest-coverage
.endif
DEBUG_FLAGS=-g
.if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" || \
${MACHINE_ARCH} == "arm"
CFLAGS+= -DBITS_PER_LONG=32
.else
CFLAGS+= -DBITS_PER_LONG=64
.endif
SRCS= vnode_if.h device_if.h bus_if.h
# avl
SRCS+= avl.c
# icp
SRCS+= edonr.c
#lua
SRCS+= lapi.c \
lauxlib.c \
lbaselib.c \
lcode.c \
lcompat.c \
lcorolib.c \
lctype.c \
ldebug.c \
ldo.c \
lfunc.c \
lgc.c \
llex.c \
lmem.c \
lobject.c \
lopcodes.c \
lparser.c \
lstate.c \
lstring.c \
lstrlib.c \
ltable.c \
ltablib.c \
ltm.c \
lvm.c \
lzio.c
#nvpair
SRCS+= nvpair.c \
fnvpair.c \
nvpair_alloc_spl.c \
nvpair_alloc_fixed.c
#os/freebsd/spl
SRCS+= acl_common.c \
callb.c \
list.c \
sha256c.c \
sha512c.c \
spl_acl.c \
spl_cmn_err.c \
spl_dtrace.c \
spl_kmem.c \
spl_kstat.c \
spl_misc.c \
spl_policy.c \
spl_procfs_list.c \
spl_string.c \
spl_sunddi.c \
spl_sysevent.c \
spl_taskq.c \
spl_uio.c \
spl_vfs.c \
spl_vm.c \
spl_zlib.c \
spl_zone.c
.if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "powerpc" || \
${MACHINE_ARCH} == "arm"
SRCS+= spl_atomic.c
.endif
#os/freebsd/zfs
SRCS+= abd_os.c \
arc_os.c \
crypto_os.c \
dmu_os.c \
hkdf.c \
kmod_core.c \
spa_os.c \
sysctl_os.c \
vdev_file.c \
vdev_geom.c \
vdev_label_os.c \
zfs_acl.c \
zfs_ctldir.c \
zfs_debug.c \
zfs_dir.c \
zfs_ioctl_compat.c \
zfs_ioctl_os.c \
zfs_racct.c \
zfs_vfsops.c \
zfs_vnops_os.c \
zfs_znode.c \
zio_crypt.c \
zvol_os.c
#unicode
SRCS+= uconv.c \
u8_textprep.c
#zcommon
SRCS+= zfeature_common.c \
zfs_comutil.c \
zfs_deleg.c \
zfs_fletcher.c \
zfs_fletcher_avx512.c \
zfs_fletcher_intel.c \
zfs_fletcher_sse.c \
zfs_fletcher_superscalar.c \
zfs_fletcher_superscalar4.c \
zfs_namecheck.c \
zfs_prop.c \
zpool_prop.c \
zprop_common.c
#zfs
SRCS+= abd.c \
aggsum.c \
arc.c \
blkptr.c \
bplist.c \
bpobj.c \
btree.c \
cityhash.c \
dbuf.c \
dbuf_stats.c \
bptree.c \
bqueue.c \
dataset_kstats.c \
ddt.c \
ddt_zap.c \
dmu.c \
dmu_diff.c \
dmu_object.c \
dmu_objset.c \
dmu_recv.c \
dmu_redact.c \
dmu_send.c \
dmu_traverse.c \
dmu_tx.c \
dmu_zfetch.c \
dnode.c \
dnode_sync.c \
dsl_dataset.c \
dsl_deadlist.c \
dsl_deleg.c \
dsl_bookmark.c \
dsl_dir.c \
dsl_crypt.c \
dsl_destroy.c \
dsl_pool.c \
dsl_prop.c \
dsl_scan.c \
dsl_synctask.c \
dsl_userhold.c \
edonr_zfs.c \
fm.c \
gzip.c \
lzjb.c \
lz4.c \
+ lz4_zfs.c \
metaslab.c \
mmp.c \
multilist.c \
objlist.c \
pathname.c \
range_tree.c \
refcount.c \
rrwlock.c \
sa.c \
sha256.c \
skein_zfs.c \
spa.c \
spa_boot.c \
spa_checkpoint.c \
spa_config.c \
spa_errlog.c \
spa_history.c \
spa_log_spacemap.c \
spa_misc.c \
spa_stats.c \
space_map.c \
space_reftree.c \
txg.c \
uberblock.c \
unique.c \
vdev.c \
vdev_cache.c \
vdev_draid.c \
vdev_draid_rand.c \
vdev_indirect.c \
vdev_indirect_births.c \
vdev_indirect_mapping.c \
vdev_initialize.c \
vdev_label.c \
vdev_mirror.c \
vdev_missing.c \
vdev_queue.c \
vdev_raidz.c \
vdev_raidz_math.c \
vdev_raidz_math_scalar.c \
vdev_raidz_math_avx2.c \
vdev_raidz_math_avx512bw.c \
vdev_raidz_math_avx512f.c \
vdev_raidz_math_sse2.c \
vdev_raidz_math_ssse3.c \
vdev_rebuild.c \
vdev_removal.c \
vdev_root.c \
vdev_trim.c \
zap.c \
zap_leaf.c \
zap_micro.c \
zcp.c \
zcp_get.c \
zcp_global.c \
zcp_iter.c \
zcp_set.c \
zcp_synctask.c \
zfeature.c \
zfs_byteswap.c \
zfs_file_os.c \
zfs_fm.c \
zfs_fuid.c \
zfs_ioctl.c \
zfs_log.c \
zfs_onexit.c \
zfs_quota.c \
zfs_ratelimit.c \
zfs_replay.c \
zfs_rlock.c \
zfs_sa.c \
zfs_vnops.c \
zil.c \
zio.c \
zio_checksum.c \
zio_compress.c \
zio_inject.c \
zle.c \
zrlock.c \
zthr.c \
zvol.c
#zstd
SRCS+= zfs_zstd.c \
zstd.c
beforeinstall:
.if ${MK_DEBUG_FILES} != "no"
mtree -eu \
-f /etc/mtree/BSD.debug.dist \
-p ${DESTDIR}/usr/lib
.endif
.include
CFLAGS.gcc+= -Wno-pointer-to-int-cast
CFLAGS.lapi.c= -Wno-cast-qual
CFLAGS.lcompat.c= -Wno-cast-qual
CFLAGS.lobject.c= -Wno-cast-qual
CFLAGS.ltable.c= -Wno-cast-qual
CFLAGS.lvm.c= -Wno-cast-qual
CFLAGS.nvpair.c= -DHAVE_RPC_TYPES -Wno-cast-qual
CFLAGS.spl_string.c= -Wno-cast-qual
CFLAGS.spl_vm.c= -Wno-cast-qual
CFLAGS.spl_zlib.c= -Wno-cast-qual
CFLAGS.abd.c= -Wno-cast-qual
CFLAGS.zfs_log.c= -Wno-cast-qual
CFLAGS.zfs_vnops_os.c= -Wno-pointer-arith
CFLAGS.u8_textprep.c= -Wno-cast-qual
CFLAGS.zfs_fletcher.c= -Wno-cast-qual -Wno-pointer-arith
CFLAGS.zfs_fletcher_intel.c= -Wno-cast-qual -Wno-pointer-arith
CFLAGS.zfs_fletcher_sse.c= -Wno-cast-qual -Wno-pointer-arith
CFLAGS.zfs_fletcher_avx512.c= -Wno-cast-qual -Wno-pointer-arith
CFLAGS.zprop_common.c= -Wno-cast-qual
CFLAGS.ddt.c= -Wno-cast-qual
CFLAGS.dmu.c= -Wno-cast-qual
CFLAGS.dmu_traverse.c= -Wno-cast-qual
CFLAGS.dsl_dir.c= -Wno-cast-qual
CFLAGS.dsl_deadlist.c= -Wno-cast-qual
CFLAGS.dsl_prop.c= -Wno-cast-qual
CFLAGS.edonr.c=-Wno-cast-qual
CFLAGS.fm.c= -Wno-cast-qual
-CFLAGS.lz4.c= -Wno-cast-qual
+CFLAGS.lz4_zfs.c= -Wno-cast-qual
CFLAGS.spa.c= -Wno-cast-qual
CFLAGS.spa_misc.c= -Wno-cast-qual
CFLAGS.sysctl_os.c= -include ../zfs_config.h
CFLAGS.vdev_draid.c= -Wno-cast-qual
CFLAGS.vdev_raidz.c= -Wno-cast-qual
CFLAGS.vdev_raidz_math.c= -Wno-cast-qual
CFLAGS.vdev_raidz_math_scalar.c= -Wno-cast-qual
CFLAGS.vdev_raidz_math_avx2.c= -Wno-cast-qual -Wno-duplicate-decl-specifier
CFLAGS.vdev_raidz_math_avx512f.c= -Wno-cast-qual -Wno-duplicate-decl-specifier
CFLAGS.vdev_raidz_math_sse2.c= -Wno-cast-qual -Wno-duplicate-decl-specifier
CFLAGS.zap_leaf.c= -Wno-cast-qual
CFLAGS.zap_micro.c= -Wno-cast-qual
CFLAGS.zcp.c= -Wno-cast-qual
CFLAGS.zfs_fm.c= -Wno-cast-qual
CFLAGS.zfs_ioctl.c= -Wno-cast-qual
CFLAGS.zil.c= -Wno-cast-qual
CFLAGS.zio.c= -Wno-cast-qual
CFLAGS.zrlock.c= -Wno-cast-qual
CFLAGS.zfs_zstd.c= -Wno-cast-qual -Wno-pointer-arith
CFLAGS.zstd.c= -fno-tree-vectorize -U__BMI__
diff --git a/module/zfs/Makefile.in b/module/zfs/Makefile.in
index 653ea0da9bcc..30dc91a7eb59 100644
--- a/module/zfs/Makefile.in
+++ b/module/zfs/Makefile.in
@@ -1,157 +1,158 @@
ifneq ($(KBUILD_EXTMOD),)
src = @abs_srcdir@
obj = @abs_builddir@
mfdir = $(obj)
else
mfdir = $(srctree)/$(src)
endif
MODULE := zfs
obj-$(CONFIG_ZFS) := $(MODULE).o
# Suppress unused-value warnings in sparc64 architecture headers
ccflags-$(CONFIG_SPARC64) += -Wno-unused-value
$(MODULE)-objs += abd.o
$(MODULE)-objs += aggsum.o
$(MODULE)-objs += arc.o
$(MODULE)-objs += blkptr.o
$(MODULE)-objs += bplist.o
$(MODULE)-objs += bpobj.o
$(MODULE)-objs += bptree.o
$(MODULE)-objs += btree.o
$(MODULE)-objs += bqueue.o
$(MODULE)-objs += dataset_kstats.o
$(MODULE)-objs += dbuf.o
$(MODULE)-objs += dbuf_stats.o
$(MODULE)-objs += ddt.o
$(MODULE)-objs += ddt_zap.o
$(MODULE)-objs += dmu.o
$(MODULE)-objs += dmu_diff.o
$(MODULE)-objs += dmu_object.o
$(MODULE)-objs += dmu_objset.o
$(MODULE)-objs += dmu_recv.o
$(MODULE)-objs += dmu_redact.o
$(MODULE)-objs += dmu_send.o
$(MODULE)-objs += dmu_traverse.o
$(MODULE)-objs += dmu_tx.o
$(MODULE)-objs += dmu_zfetch.o
$(MODULE)-objs += dnode.o
$(MODULE)-objs += dnode_sync.o
$(MODULE)-objs += dsl_bookmark.o
$(MODULE)-objs += dsl_crypt.o
$(MODULE)-objs += dsl_dataset.o
$(MODULE)-objs += dsl_deadlist.o
$(MODULE)-objs += dsl_deleg.o
$(MODULE)-objs += dsl_destroy.o
$(MODULE)-objs += dsl_dir.o
$(MODULE)-objs += dsl_pool.o
$(MODULE)-objs += dsl_prop.o
$(MODULE)-objs += dsl_scan.o
$(MODULE)-objs += dsl_synctask.o
$(MODULE)-objs += dsl_userhold.o
$(MODULE)-objs += edonr_zfs.o
$(MODULE)-objs += fm.o
$(MODULE)-objs += gzip.o
$(MODULE)-objs += hkdf.o
$(MODULE)-objs += lz4.o
+$(MODULE)-objs += lz4_zfs.o
$(MODULE)-objs += lzjb.o
$(MODULE)-objs += metaslab.o
$(MODULE)-objs += mmp.o
$(MODULE)-objs += multilist.o
$(MODULE)-objs += objlist.o
$(MODULE)-objs += pathname.o
$(MODULE)-objs += range_tree.o
$(MODULE)-objs += refcount.o
$(MODULE)-objs += rrwlock.o
$(MODULE)-objs += sa.o
$(MODULE)-objs += sha256.o
$(MODULE)-objs += skein_zfs.o
$(MODULE)-objs += spa.o
$(MODULE)-objs += spa_boot.o
$(MODULE)-objs += spa_checkpoint.o
$(MODULE)-objs += spa_config.o
$(MODULE)-objs += spa_errlog.o
$(MODULE)-objs += spa_history.o
$(MODULE)-objs += spa_log_spacemap.o
$(MODULE)-objs += spa_misc.o
$(MODULE)-objs += spa_stats.o
$(MODULE)-objs += space_map.o
$(MODULE)-objs += space_reftree.o
$(MODULE)-objs += txg.o
$(MODULE)-objs += uberblock.o
$(MODULE)-objs += unique.o
$(MODULE)-objs += vdev.o
$(MODULE)-objs += vdev_cache.o
$(MODULE)-objs += vdev_draid.o
$(MODULE)-objs += vdev_draid_rand.o
$(MODULE)-objs += vdev_indirect.o
$(MODULE)-objs += vdev_indirect_births.o
$(MODULE)-objs += vdev_indirect_mapping.o
$(MODULE)-objs += vdev_initialize.o
$(MODULE)-objs += vdev_label.o
$(MODULE)-objs += vdev_mirror.o
$(MODULE)-objs += vdev_missing.o
$(MODULE)-objs += vdev_queue.o
$(MODULE)-objs += vdev_raidz.o
$(MODULE)-objs += vdev_raidz_math.o
$(MODULE)-objs += vdev_raidz_math_scalar.o
$(MODULE)-objs += vdev_rebuild.o
$(MODULE)-objs += vdev_removal.o
$(MODULE)-objs += vdev_root.o
$(MODULE)-objs += vdev_trim.o
$(MODULE)-objs += zap.o
$(MODULE)-objs += zap_leaf.o
$(MODULE)-objs += zap_micro.o
$(MODULE)-objs += zcp.o
$(MODULE)-objs += zcp_get.o
$(MODULE)-objs += zcp_global.o
$(MODULE)-objs += zcp_iter.o
$(MODULE)-objs += zcp_set.o
$(MODULE)-objs += zcp_synctask.o
$(MODULE)-objs += zfeature.o
$(MODULE)-objs += zfs_byteswap.o
$(MODULE)-objs += zfs_fm.o
$(MODULE)-objs += zfs_fuid.o
$(MODULE)-objs += zfs_ioctl.o
$(MODULE)-objs += zfs_log.o
$(MODULE)-objs += zfs_onexit.o
$(MODULE)-objs += zfs_quota.o
$(MODULE)-objs += zfs_ratelimit.o
$(MODULE)-objs += zfs_replay.o
$(MODULE)-objs += zfs_rlock.o
$(MODULE)-objs += zfs_sa.o
$(MODULE)-objs += zfs_vnops.o
$(MODULE)-objs += zil.o
$(MODULE)-objs += zio.o
$(MODULE)-objs += zio_checksum.o
$(MODULE)-objs += zio_compress.o
$(MODULE)-objs += zio_inject.o
$(MODULE)-objs += zle.o
$(MODULE)-objs += zrlock.o
$(MODULE)-objs += zthr.o
$(MODULE)-objs += zvol.o
# Suppress incorrect warnings from versions of objtool which are not
# aware of x86 EVEX prefix instructions used for AVX512.
OBJECT_FILES_NON_STANDARD_vdev_raidz_math_avx512bw.o := y
OBJECT_FILES_NON_STANDARD_vdev_raidz_math_avx512f.o := y
$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_sse2.o
$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_ssse3.o
$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_avx2.o
$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_avx512f.o
$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_avx512bw.o
$(MODULE)-$(CONFIG_ARM64) += vdev_raidz_math_aarch64_neon.o
$(MODULE)-$(CONFIG_ARM64) += vdev_raidz_math_aarch64_neonx2.o
$(MODULE)-$(CONFIG_PPC) += vdev_raidz_math_powerpc_altivec.o
$(MODULE)-$(CONFIG_PPC64) += vdev_raidz_math_powerpc_altivec.o
ifeq ($(CONFIG_ALTIVEC),y)
$(obj)/vdev_raidz_math_powerpc_altivec.o: c_flags += -maltivec
endif
include $(mfdir)/../os/linux/zfs/Makefile
diff --git a/module/zfs/lz4.c b/module/zfs/lz4.c
index eba8f01b5065..800635b3449e 100644
--- a/module/zfs/lz4.c
+++ b/module/zfs/lz4.c
@@ -1,1029 +1,987 @@
/*
- * LZ4 - Fast LZ compression algorithm
- * Header File
- * Copyright (C) 2011-2013, Yann Collet.
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
+ LZ4 - Fast LZ compression algorithm
+ Copyright (C) 2011-present, Yann Collet.
+
+ BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ You can contact the author at :
+ - LZ4 homepage : http://www.lz4.org
+ - LZ4 source repository : https://github.com/lz4/lz4
+*/
+
+/*
+ * This file contains unmodified code from lz4 1.9.3's decompressor, plus
+ * associated macros and constants.
*
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * It also contains a couple of defines from the old lz4.c to make things
+ * fit together smoothly.
*
- * You can contact the author at :
- * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- * - LZ4 source repository : http://code.google.com/p/lz4/
*/
#include
-#include
-static int real_LZ4_compress(const char *source, char *dest, int isize,
- int osize);
-static int LZ4_uncompress_unknownOutputSize(const char *source, char *dest,
+int LZ4_uncompress_unknownOutputSize(const char *source, char *dest,
int isize, int maxOutputSize);
-static int LZ4_compressCtx(void *ctx, const char *source, char *dest,
- int isize, int osize);
-static int LZ4_compress64kCtx(void *ctx, const char *source, char *dest,
- int isize, int osize);
-
-static kmem_cache_t *lz4_cache;
-
-size_t
-lz4_compress_zfs(void *s_start, void *d_start, size_t s_len,
- size_t d_len, int n)
-{
- (void) n;
- uint32_t bufsiz;
- char *dest = d_start;
-
- ASSERT(d_len >= sizeof (bufsiz));
-
- bufsiz = real_LZ4_compress(s_start, &dest[sizeof (bufsiz)], s_len,
- d_len - sizeof (bufsiz));
-
- /* Signal an error if the compression routine returned zero. */
- if (bufsiz == 0)
- return (s_len);
-
- /*
- * The exact compressed size is needed by the decompression routine,
- * so it is stored at the start of the buffer. Note that this may be
- * less than the compressed block size, which is rounded up to a
- * multiple of 1< s_len)
- return (1);
-
- /*
- * Returns 0 on success (decompression function returned non-negative)
- * and non-zero on failure (decompression function returned negative).
- */
- return (LZ4_uncompress_unknownOutputSize(&src[sizeof (bufsiz)],
- d_start, bufsiz, d_len) < 0);
-}
-
-/*
- * LZ4 API Description:
- *
- * Simple Functions:
- * real_LZ4_compress() :
- * isize : is the input size. Max supported value is ~1.9GB
- * return : the number of bytes written in buffer dest
- * or 0 if the compression fails (if LZ4_COMPRESSMIN is set).
- * note : destination buffer must be already allocated.
- * destination buffer must be sized to handle worst cases
- * situations (input data not compressible) worst case size
- * evaluation is provided by function LZ4_compressBound().
- *
- * real_LZ4_uncompress() :
- * osize : is the output size, therefore the original size
- * return : the number of bytes read in the source buffer.
- * If the source stream is malformed, the function will stop
- * decoding and return a negative result, indicating the byte
- * position of the faulty instruction. This function never
- * writes beyond dest + osize, and is therefore protected
- * against malicious data packets.
- * note : destination buffer must be already allocated
- * note : real_LZ4_uncompress() is not used in ZFS so its code
- * is not present here.
- *
- * Advanced Functions
- *
- * LZ4_compressBound() :
- * Provides the maximum size that LZ4 may output in a "worst case"
- * scenario (input data not compressible) primarily useful for memory
- * allocation of output buffer.
- *
- * isize : is the input size. Max supported value is ~1.9GB
- * return : maximum output size in a "worst case" scenario
- * note : this function is limited by "int" range (2^31-1)
- *
- * LZ4_uncompress_unknownOutputSize() :
- * isize : is the input size, therefore the compressed size
- * maxOutputSize : is the size of the destination buffer (which must be
- * already allocated)
- * return : the number of bytes decoded in the destination buffer
- * (necessarily <= maxOutputSize). If the source stream is
- * malformed, the function will stop decoding and return a
- * negative result, indicating the byte position of the faulty
- * instruction. This function never writes beyond dest +
- * maxOutputSize, and is therefore protected against malicious
- * data packets.
- * note : Destination buffer must be already allocated.
- * This version is slightly slower than real_LZ4_uncompress()
- *
- * LZ4_compressCtx() :
- * This function explicitly handles the CTX memory structure.
- *
- * ILLUMOS CHANGES: the CTX memory structure must be explicitly allocated
- * by the caller (either on the stack or using kmem_cache_alloc). Passing
- * NULL isn't valid.
- *
- * LZ4_compress64kCtx() :
- * Same as LZ4_compressCtx(), but specific to small inputs (<64KB).
- * isize *Must* be <64KB, otherwise the output will be corrupted.
- *
- * ILLUMOS CHANGES: the CTX memory structure must be explicitly allocated
- * by the caller (either on the stack or using kmem_cache_alloc). Passing
- * NULL isn't valid.
- */
/*
* Tuning parameters
*/
/*
* COMPRESSIONLEVEL: Increasing this value improves compression ratio
* Lowering this value reduces memory usage. Reduced memory usage
* typically improves speed, due to cache effect (ex: L1 32KB for Intel,
* L1 64KB for AMD). Memory usage formula : N->2^(N+2) Bytes
* (examples : 12 -> 16KB ; 17 -> 512KB)
*/
#define COMPRESSIONLEVEL 12
/*
* NOTCOMPRESSIBLE_CONFIRMATION: Decreasing this value will make the
* algorithm skip faster data segments considered "incompressible".
* This may decrease compression ratio dramatically, but will be
* faster on incompressible data. Increasing this value will make
* the algorithm search more before declaring a segment "incompressible".
* This could improve compression a bit, but will be slower on
* incompressible data. The default value (6) is recommended.
*/
#define NOTCOMPRESSIBLE_CONFIRMATION 6
-/*
- * BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE: This will provide a boost to
- * performance for big endian cpu, but the resulting compressed stream
- * will be incompatible with little-endian CPU. You can set this option
- * to 1 in situations where data will stay within closed environment.
- * This option is useless on Little_Endian CPU (such as x86).
- */
-/* #define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 */
-
-/*
- * CPU Feature Detection
- */
-
-/* 32 or 64 bits ? */
-#if defined(_LP64)
-#define LZ4_ARCH64 1
-#else
-#define LZ4_ARCH64 0
-#endif
-
/*
* Little Endian or Big Endian?
* Note: overwrite the below #define if you know your architecture endianness.
*/
#if defined(_ZFS_BIG_ENDIAN)
#define LZ4_BIG_ENDIAN 1
#else
/*
* Little Endian assumed. PDP Endian and other very rare endian format
* are unsupported.
*/
#undef LZ4_BIG_ENDIAN
#endif
-/*
- * Unaligned memory access is automatically enabled for "common" CPU,
- * such as x86. For others CPU, the compiler will be more cautious, and
- * insert extra code to ensure aligned access is respected. If you know
- * your target CPU supports unaligned memory access, you may want to
- * force this option manually to improve performance
+/*-************************************
+* CPU Feature Detection
+**************************************/
+/* LZ4_FORCE_MEMORY_ACCESS
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
+ * The below switch allow to select different access method for improved performance.
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+ * Method 2 : direct access. This method is portable but violate C standard.
+ * It can generate buggy code on targets which assembly generation depends on alignment.
+ * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * Prefer these methods in priority order (0 > 1 > 2)
*/
-#if defined(__ARM_FEATURE_UNALIGNED)
-#define LZ4_FORCE_UNALIGNED_ACCESS 1
+#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */
+# if defined(__GNUC__) && \
+ ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \
+ || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+# define LZ4_FORCE_MEMORY_ACCESS 2
+# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__)
+# define LZ4_FORCE_MEMORY_ACCESS 1
+# endif
#endif
+/*
+ * LZ4_FORCE_SW_BITCOUNT
+ * Define this parameter if your target system or compiler does not support hardware bit count
+ */
/*
* Illumos : we can't use GCC's __builtin_ctz family of builtins in the
* kernel
* Linux : we can use GCC's __builtin_ctz family of builtins in the
* kernel
*/
#undef LZ4_FORCE_SW_BITCOUNT
-#if defined(__sparc)
+#if defined(__sunos__)
#define LZ4_FORCE_SW_BITCOUNT
#endif
/*
* Compiler Options
*/
/* Disable restrict */
#define restrict
/*
* Linux : GCC_VERSION is defined as of 3.9-rc1, so undefine it.
* torvalds/linux@3f3f8d2f48acfd8ed3b8e6b7377935da57b27b16
*/
#ifdef GCC_VERSION
#undef GCC_VERSION
#endif
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
-#define expect(expr, value) (__builtin_expect((expr), (value)))
+#ifndef LZ4_FORCE_INLINE
+# ifdef _MSC_VER /* Visual Studio */
+# define LZ4_FORCE_INLINE static __forceinline
+# else
+# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define LZ4_FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define LZ4_FORCE_INLINE static inline
+# endif
+# else
+# define LZ4_FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
+# endif /* _MSC_VER */
+#endif /* LZ4_FORCE_INLINE */
+
+/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE
+ * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8,
+ * together with a simple 8-byte copy loop as a fall-back path.
+ * However, this optimization hurts the decompression speed by >30%,
+ * because the execution does not go to the optimized loop
+ * for typical compressible data, and all of the preamble checks
+ * before going to the fall-back path become useless overhead.
+ * This optimization happens only with the -O3 flag, and -O2 generates
+ * a simple 8-byte copy loop.
+ * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8
+ * functions are annotated with __attribute__((optimize("O2"))),
+ * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute
+ * of LZ4_wildCopy8 does not affect the compression speed.
+ */
+#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__)
+# define LZ4_FORCE_O2 __attribute__((optimize("O2")))
+# undef LZ4_FORCE_INLINE
+# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline))
#else
-#define expect(expr, value) (expr)
+# define LZ4_FORCE_O2
+#endif
+
+#ifndef expect
+#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__)
+# define expect(expr,value) (__builtin_expect ((expr),(value)) )
+#else
+# define expect(expr,value) (expr)
+#endif
#endif
#ifndef likely
#define likely(expr) expect((expr) != 0, 1)
#endif
#ifndef unlikely
#define unlikely(expr) expect((expr) != 0, 0)
#endif
-#define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | \
- (((x) & 0xffu) << 8)))
+#ifndef _KERNEL
+#include /* malloc, calloc, free */
+#include /* memset, memcpy */
+#endif
+#define ALLOC(s) malloc(s)
+#define ALLOC_AND_ZERO(s) calloc(1,s)
+#define FREEMEM(p) free(p)
+
+#define MEM_INIT(p,v,s) memset((p),(v),(s))
+
-/* Basic types */
-#define BYTE uint8_t
-#define U16 uint16_t
-#define U32 uint32_t
-#define S32 int32_t
-#define U64 uint64_t
+/*-************************************
+* Common Constants
+**************************************/
+#define MINMATCH 4
-#ifndef LZ4_FORCE_UNALIGNED_ACCESS
-#pragma pack(1)
+#define WILDCOPYLENGTH 8
+#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
+#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */
+#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */
+#define FASTLOOP_SAFE_DISTANCE 64
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */
+# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */
#endif
-typedef struct _U16_S {
- U16 v;
-} U16_S;
-typedef struct _U32_S {
- U32 v;
-} U32_S;
-typedef struct _U64_S {
- U64 v;
-} U64_S;
-
-#ifndef LZ4_FORCE_UNALIGNED_ACCESS
-#pragma pack()
+#define LZ4_DISTANCE_ABSOLUTE_MAX 65535
+#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */
+# error "LZ4_DISTANCE_MAX is too big : must be <= 65535"
#endif
-#define A64(x) (((U64_S *)(x))->v)
-#define A32(x) (((U32_S *)(x))->v)
-#define A16(x) (((U16_S *)(x))->v)
+#define ML_BITS 4
+#define ML_MASK ((1U< 2 ? \
- NOTCOMPRESSIBLE_CONFIRMATION : 2)
+/*-************************************
+* Types
+**************************************/
+#ifndef _KERNEL
+#include
+#endif
+#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#ifndef _KERNEL
+#include
+#endif
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+ typedef uintptr_t uptrval;
+#else
+# if UINT_MAX != 4294967295UL
+# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4"
+# endif
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+ typedef unsigned long long U64;
+ typedef size_t uptrval; /* generally true, except OpenVMS-64 */
+#endif
-#define COPYLENGTH 8
-#define LASTLITERALS 5
-#define MFLIMIT (COPYLENGTH + MINMATCH)
-#define MINLENGTH (MFLIMIT + 1)
+#if defined(__x86_64__)
+ typedef U64 reg_t; /* 64-bits in x32 mode */
+#else
+ typedef size_t reg_t; /* 32-bits in x32 mode */
+#endif
-#define MAXD_LOG 16
-#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
+typedef enum {
+ notLimited = 0,
+ limitedOutput = 1,
+ fillOutput = 2
+} limitedOutput_directive;
-#define ML_BITS 4
-#define ML_MASK ((1U<= 4)
+#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size)
#else
-#define LZ4_READ_LITTLEENDIAN_16(d, s, p) { d = (s) - A16(p); }
-#define LZ4_WRITE_LITTLEENDIAN_16(p, v) { A16(p) = v; p += 2; }
+#define LZ4_memcpy(dst, src, size) memcpy(dst, src, size)
#endif
+static unsigned LZ4_isLittleEndian(void)
+{
+ const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
+ return one.c[0];
+}
+
+
+#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2)
+/* lie to the compiler about data alignment; use with caution */
+
+static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; }
-/* Local structures */
-struct refTables {
- HTYPE hashTable[HASHTABLESIZE];
-};
+static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
+static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
+#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1)
-/* Macros */
-#define LZ4_HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH * 8) - \
- HASH_LOG))
-#define LZ4_HASH_VALUE(p) LZ4_HASH_FUNCTION(A32(p))
-#define LZ4_WILDCOPY(s, d, e) do { LZ4_COPYPACKET(s, d) } while (d < e);
-#define LZ4_BLINDCOPY(s, d, l) { BYTE* e = (d) + l; LZ4_WILDCOPY(s, d, e); \
- d = e; }
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign;
+static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
-/* Private functions */
-#if LZ4_ARCH64
+static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
-static inline int
-LZ4_NbCommonBytes(register U64 val)
+#else /* safe and portable access using memcpy() */
+
+static U16 LZ4_read16(const void* memPtr)
{
-#if defined(LZ4_BIG_ENDIAN)
-#if ((defined(__GNUC__) && (GCC_VERSION >= 304)) || defined(__clang__)) && \
- !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clzll(val) >> 3);
-#else
- int r;
- if (!(val >> 32)) {
- r = 4;
- } else {
- r = 0;
- val >>= 32;
- }
- if (!(val >> 16)) {
- r += 2;
- val >>= 8;
- } else {
- val >>= 24;
- }
- r += (!val);
- return (r);
-#endif
-#else
-#if ((defined(__GNUC__) && (GCC_VERSION >= 304)) || defined(__clang__)) && \
- !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctzll(val) >> 3);
-#else
- static const int DeBruijnBytePos[64] =
- { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5,
- 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5,
- 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4,
- 4, 5, 7, 2, 6, 5, 7, 6, 7, 7
- };
- return DeBruijnBytePos[((U64) ((val & -val) * 0x0218A392CDABBD3F)) >>
- 58];
-#endif
-#endif
+ U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val;
}
-#else
-
-static inline int
-LZ4_NbCommonBytes(register U32 val)
+static void LZ4_write32(void* memPtr, U32 value)
{
-#if defined(LZ4_BIG_ENDIAN)
-#if ((defined(__GNUC__) && (GCC_VERSION >= 304)) || defined(__clang__)) && \
- !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clz(val) >> 3);
-#else
- int r;
- if (!(val >> 16)) {
- r = 2;
- val >>= 8;
- } else {
- r = 0;
- val >>= 24;
- }
- r += (!val);
- return (r);
-#endif
-#else
-#if defined(__GNUC__) && (GCC_VERSION >= 304) && \
- !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctz(val) >> 3);
-#else
- static const int DeBruijnBytePos[32] = {
- 0, 0, 3, 0, 3, 1, 3, 0,
- 3, 2, 2, 1, 3, 2, 0, 1,
- 3, 3, 1, 2, 2, 2, 2, 0,
- 3, 1, 2, 0, 1, 0, 1, 1
- };
- return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >>
- 27];
-#endif
-#endif
+ LZ4_memcpy(memPtr, &value, sizeof(value));
}
-#endif
+#endif /* LZ4_FORCE_MEMORY_ACCESS */
-/* Compression functions */
+static U16 LZ4_readLE16(const void* memPtr)
+{
+ if (LZ4_isLittleEndian()) {
+ return LZ4_read16(memPtr);
+ } else {
+ const BYTE* p = (const BYTE*)memPtr;
+ return (U16)((U16)p[0] + (p[1]<<8));
+ }
+}
-static int
-LZ4_compressCtx(void *ctx, const char *source, char *dest, int isize,
- int osize)
+/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */
+LZ4_FORCE_INLINE
+void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd)
{
- struct refTables *srt = (struct refTables *)ctx;
- HTYPE *HashTable = (HTYPE *) (srt->hashTable);
-
- const BYTE *ip = (BYTE *) source;
- INITBASE(base);
- const BYTE *anchor = ip;
- const BYTE *const iend = ip + isize;
- const BYTE *const oend = (BYTE *) dest + osize;
- const BYTE *const mflimit = iend - MFLIMIT;
-#define matchlimit (iend - LASTLITERALS)
-
- BYTE *op = (BYTE *) dest;
-
- int len, length;
- const int skipStrength = SKIPSTRENGTH;
- U32 forwardH;
-
-
- /* Init */
- if (isize < MINLENGTH)
- goto _last_literals;
-
- /* First Byte */
- HashTable[LZ4_HASH_VALUE(ip)] = ip - base;
- ip++;
- forwardH = LZ4_HASH_VALUE(ip);
-
- /* Main Loop */
- for (;;) {
- int findMatchAttempts = (1U << skipStrength) + 3;
- const BYTE *forwardIp = ip;
- const BYTE *ref;
- BYTE *token;
-
- /* Find a match */
- do {
- U32 h = forwardH;
- int step = findMatchAttempts++ >> skipStrength;
- ip = forwardIp;
- forwardIp = ip + step;
-
- if (unlikely(forwardIp > mflimit)) {
- goto _last_literals;
- }
-
- forwardH = LZ4_HASH_VALUE(forwardIp);
- ref = base + HashTable[h];
- HashTable[h] = ip - base;
-
- } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip)));
-
- /* Catch up */
- while ((ip > anchor) && (ref > (BYTE *) source) &&
- unlikely(ip[-1] == ref[-1])) {
- ip--;
- ref--;
- }
-
- /* Encode Literal length */
- length = ip - anchor;
- token = op++;
-
- /* Check output limit */
- if (unlikely(op + length + (2 + 1 + LASTLITERALS) +
- (length >> 8) > oend))
- return (0);
-
- if (length >= (int)RUN_MASK) {
- *token = (RUN_MASK << ML_BITS);
- len = length - RUN_MASK;
- for (; len > 254; len -= 255)
- *op++ = 255;
- *op++ = (BYTE)len;
- } else
- *token = (length << ML_BITS);
-
- /* Copy Literals */
- LZ4_BLINDCOPY(anchor, op, length);
-
- _next_match:
- /* Encode Offset */
- LZ4_WRITE_LITTLEENDIAN_16(op, ip - ref);
-
- /* Start Counting */
- ip += MINMATCH;
- ref += MINMATCH; /* MinMatch verified */
- anchor = ip;
- while (likely(ip < matchlimit - (STEPSIZE - 1))) {
- UARCH diff = AARCH(ref) ^ AARCH(ip);
- if (!diff) {
- ip += STEPSIZE;
- ref += STEPSIZE;
- continue;
- }
- ip += LZ4_NbCommonBytes(diff);
- goto _endCount;
- }
-#if LZ4_ARCH64
- if ((ip < (matchlimit - 3)) && (A32(ref) == A32(ip))) {
- ip += 4;
- ref += 4;
- }
-#endif
- if ((ip < (matchlimit - 1)) && (A16(ref) == A16(ip))) {
- ip += 2;
- ref += 2;
- }
- if ((ip < matchlimit) && (*ref == *ip))
- ip++;
- _endCount:
-
- /* Encode MatchLength */
- len = (ip - anchor);
- /* Check output limit */
- if (unlikely(op + (1 + LASTLITERALS) + (len >> 8) > oend))
- return (0);
- if (len >= (int)ML_MASK) {
- *token += ML_MASK;
- len -= ML_MASK;
- for (; len > 509; len -= 510) {
- *op++ = 255;
- *op++ = 255;
- }
- if (len > 254) {
- len -= 255;
- *op++ = 255;
- }
- *op++ = (BYTE)len;
- } else
- *token += len;
-
- /* Test end of chunk */
- if (ip > mflimit) {
- anchor = ip;
- break;
- }
- /* Fill table */
- HashTable[LZ4_HASH_VALUE(ip - 2)] = ip - 2 - base;
-
- /* Test next position */
- ref = base + HashTable[LZ4_HASH_VALUE(ip)];
- HashTable[LZ4_HASH_VALUE(ip)] = ip - base;
- if ((ref > ip - (MAX_DISTANCE + 1)) && (A32(ref) == A32(ip))) {
- token = op++;
- *token = 0;
- goto _next_match;
- }
- /* Prepare next loop */
- anchor = ip++;
- forwardH = LZ4_HASH_VALUE(ip);
- }
-
- _last_literals:
- /* Encode Last Literals */
- {
- int lastRun = iend - anchor;
- if (op + lastRun + 1 + ((lastRun + 255 - RUN_MASK) / 255) >
- oend)
- return (0);
- if (lastRun >= (int)RUN_MASK) {
- *op++ = (RUN_MASK << ML_BITS);
- lastRun -= RUN_MASK;
- for (; lastRun > 254; lastRun -= 255) {
- *op++ = 255;
- }
- *op++ = (BYTE)lastRun;
- } else
- *op++ = (lastRun << ML_BITS);
- (void) memcpy(op, anchor, iend - anchor);
- op += iend - anchor;
- }
-
- /* End */
- return (int)(((char *)op) - dest);
+ BYTE* d = (BYTE*)dstPtr;
+ const BYTE* s = (const BYTE*)srcPtr;
+ BYTE* const e = (BYTE*)dstEnd;
+
+ do { LZ4_memcpy(d,s,8); d+=8; s+=8; } while (d> ((MINMATCH*8) - \
- HASHLOG64K))
-#define LZ4_HASH64K_VALUE(p) LZ4_HASH64K_FUNCTION(A32(p))
+LZ4_FORCE_INLINE void
+LZ4_memcpy_using_offset_base(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)
+{
+ assert(srcPtr + offset == dstPtr);
+ if (offset < 8) {
+ LZ4_write32(dstPtr, 0); /* silence an msan warning when offset==0 */
+ dstPtr[0] = srcPtr[0];
+ dstPtr[1] = srcPtr[1];
+ dstPtr[2] = srcPtr[2];
+ dstPtr[3] = srcPtr[3];
+ srcPtr += inc32table[offset];
+ LZ4_memcpy(dstPtr+4, srcPtr, 4);
+ srcPtr -= dec64table[offset];
+ dstPtr += 8;
+ } else {
+ LZ4_memcpy(dstPtr, srcPtr, 8);
+ dstPtr += 8;
+ srcPtr += 8;
+ }
+
+ LZ4_wildCopy8(dstPtr, srcPtr, dstEnd);
+}
-static int
-LZ4_compress64kCtx(void *ctx, const char *source, char *dest, int isize,
- int osize)
+/* customized variant of memcpy, which can overwrite up to 32 bytes beyond dstEnd
+ * this version copies two times 16 bytes (instead of one time 32 bytes)
+ * because it must be compatible with offsets >= 16. */
+LZ4_FORCE_INLINE void
+LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd)
{
- struct refTables *srt = (struct refTables *)ctx;
- U16 *HashTable = (U16 *) (srt->hashTable);
-
- const BYTE *ip = (BYTE *) source;
- const BYTE *anchor = ip;
- const BYTE *const base = ip;
- const BYTE *const iend = ip + isize;
- const BYTE *const oend = (BYTE *) dest + osize;
- const BYTE *const mflimit = iend - MFLIMIT;
-#define matchlimit (iend - LASTLITERALS)
-
- BYTE *op = (BYTE *) dest;
-
- int len, length;
- const int skipStrength = SKIPSTRENGTH;
- U32 forwardH;
-
- /* Init */
- if (isize < MINLENGTH)
- goto _last_literals;
-
- /* First Byte */
- ip++;
- forwardH = LZ4_HASH64K_VALUE(ip);
-
- /* Main Loop */
- for (;;) {
- int findMatchAttempts = (1U << skipStrength) + 3;
- const BYTE *forwardIp = ip;
- const BYTE *ref;
- BYTE *token;
-
- /* Find a match */
- do {
- U32 h = forwardH;
- int step = findMatchAttempts++ >> skipStrength;
- ip = forwardIp;
- forwardIp = ip + step;
-
- if (forwardIp > mflimit) {
- goto _last_literals;
- }
-
- forwardH = LZ4_HASH64K_VALUE(forwardIp);
- ref = base + HashTable[h];
- HashTable[h] = ip - base;
-
- } while (A32(ref) != A32(ip));
-
- /* Catch up */
- while ((ip > anchor) && (ref > (BYTE *) source) &&
- (ip[-1] == ref[-1])) {
- ip--;
- ref--;
- }
-
- /* Encode Literal length */
- length = ip - anchor;
- token = op++;
-
- /* Check output limit */
- if (unlikely(op + length + (2 + 1 + LASTLITERALS) +
- (length >> 8) > oend))
- return (0);
-
- if (length >= (int)RUN_MASK) {
- *token = (RUN_MASK << ML_BITS);
- len = length - RUN_MASK;
- for (; len > 254; len -= 255)
- *op++ = 255;
- *op++ = (BYTE)len;
- } else
- *token = (length << ML_BITS);
-
- /* Copy Literals */
- LZ4_BLINDCOPY(anchor, op, length);
-
- _next_match:
- /* Encode Offset */
- LZ4_WRITE_LITTLEENDIAN_16(op, ip - ref);
-
- /* Start Counting */
- ip += MINMATCH;
- ref += MINMATCH; /* MinMatch verified */
- anchor = ip;
- while (ip < matchlimit - (STEPSIZE - 1)) {
- UARCH diff = AARCH(ref) ^ AARCH(ip);
- if (!diff) {
- ip += STEPSIZE;
- ref += STEPSIZE;
- continue;
- }
- ip += LZ4_NbCommonBytes(diff);
- goto _endCount;
- }
-#if LZ4_ARCH64
- if ((ip < (matchlimit - 3)) && (A32(ref) == A32(ip))) {
- ip += 4;
- ref += 4;
- }
-#endif
- if ((ip < (matchlimit - 1)) && (A16(ref) == A16(ip))) {
- ip += 2;
- ref += 2;
- }
- if ((ip < matchlimit) && (*ref == *ip))
- ip++;
- _endCount:
-
- /* Encode MatchLength */
- len = (ip - anchor);
- /* Check output limit */
- if (unlikely(op + (1 + LASTLITERALS) + (len >> 8) > oend))
- return (0);
- if (len >= (int)ML_MASK) {
- *token += ML_MASK;
- len -= ML_MASK;
- for (; len > 509; len -= 510) {
- *op++ = 255;
- *op++ = 255;
- }
- if (len > 254) {
- len -= 255;
- *op++ = 255;
- }
- *op++ = (BYTE)len;
- } else
- *token += len;
-
- /* Test end of chunk */
- if (ip > mflimit) {
- anchor = ip;
- break;
- }
- /* Fill table */
- HashTable[LZ4_HASH64K_VALUE(ip - 2)] = ip - 2 - base;
-
- /* Test next position */
- ref = base + HashTable[LZ4_HASH64K_VALUE(ip)];
- HashTable[LZ4_HASH64K_VALUE(ip)] = ip - base;
- if (A32(ref) == A32(ip)) {
- token = op++;
- *token = 0;
- goto _next_match;
- }
- /* Prepare next loop */
- anchor = ip++;
- forwardH = LZ4_HASH64K_VALUE(ip);
- }
-
- _last_literals:
- /* Encode Last Literals */
- {
- int lastRun = iend - anchor;
- if (op + lastRun + 1 + ((lastRun + 255 - RUN_MASK) / 255) >
- oend)
- return (0);
- if (lastRun >= (int)RUN_MASK) {
- *op++ = (RUN_MASK << ML_BITS);
- lastRun -= RUN_MASK;
- for (; lastRun > 254; lastRun -= 255)
- *op++ = 255;
- *op++ = (BYTE)lastRun;
- } else
- *op++ = (lastRun << ML_BITS);
- (void) memcpy(op, anchor, iend - anchor);
- op += iend - anchor;
- }
-
- /* End */
- return (int)(((char *)op) - dest);
+ BYTE* d = (BYTE*)dstPtr;
+ const BYTE* s = (const BYTE*)srcPtr;
+ BYTE* const e = (BYTE*)dstEnd;
+
+ do { LZ4_memcpy(d,s,16); LZ4_memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH
+ * - there is at least 8 bytes available to write after dstEnd */
+LZ4_FORCE_INLINE void
+LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)
{
- void *ctx;
- int result;
+ BYTE v[8];
+
+ assert(dstEnd >= dstPtr + MINMATCH);
+
+ switch(offset) {
+ case 1:
+ MEM_INIT(v, *srcPtr, 8);
+ break;
+ case 2:
+ LZ4_memcpy(v, srcPtr, 2);
+ LZ4_memcpy(&v[2], srcPtr, 2);
+ LZ4_memcpy(&v[4], v, 4);
+ break;
+ case 4:
+ LZ4_memcpy(v, srcPtr, 4);
+ LZ4_memcpy(&v[4], srcPtr, 4);
+ break;
+ default:
+ LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset);
+ return;
+ }
+
+ LZ4_memcpy(dstPtr, v, 8);
+ dstPtr += 8;
+ while (dstPtr < dstEnd) {
+ LZ4_memcpy(dstPtr, v, 8);
+ dstPtr += 8;
+ }
+}
+#endif
+
+
+/*-************************************
+* Local Structures and types
+**************************************/
+typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t;
- ASSERT(lz4_cache != NULL);
- ctx = kmem_cache_alloc(lz4_cache, KM_SLEEP);
+/**
+ * This enum distinguishes several different modes of accessing previous
+ * content in the stream.
+ *
+ * - noDict : There is no preceding content.
+ * - withPrefix64k : Table entries up to ctx->dictSize before the current blob
+ * blob being compressed are valid and refer to the preceding
+ * content (of length ctx->dictSize), which is available
+ * contiguously preceding in memory the content currently
+ * being compressed.
+ * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere
+ * else in memory, starting at ctx->dictionary with length
+ * ctx->dictSize.
+ * - usingDictCtx : Like usingExtDict, but everything concerning the preceding
+ * content is in a separate context, pointed to by
+ * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table
+ * entries in the current context that refer to positions
+ * preceding the beginning of the current compression are
+ * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx
+ * ->dictSize describe the location and size of the preceding
+ * content, and matches are found by looking in the ctx
+ * ->dictCtx->hashTable.
+ */
+typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive;
+typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
- /*
- * out of kernel memory, gently fall through - this will disable
- * compression in zio_compress_data
- */
- if (ctx == NULL)
- return (0);
+/*-*******************************
+ * Decompression functions
+ ********************************/
- memset(ctx, 0, sizeof (struct refTables));
+typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
+typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive;
- if (isize < LZ4_64KLIMIT)
- result = LZ4_compress64kCtx(ctx, source, dest, isize, osize);
- else
- result = LZ4_compressCtx(ctx, source, dest, isize, osize);
+typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error;
- kmem_cache_free(lz4_cache, ctx);
- return (result);
+LZ4_FORCE_INLINE unsigned
+read_variable_length(const BYTE**ip, const BYTE* lencheck,
+ int loop_check, int initial_check,
+ variable_length_error* error)
+{
+ U32 length = 0;
+ U32 s;
+ if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */
+ *error = initial_error;
+ return length;
+ }
+ do {
+ s = **ip;
+ (*ip)++;
+ length += s;
+ if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */
+ *error = loop_error;
+ return length;
+ }
+ } while (s==255);
+
+ return length;
}
-/* Decompression functions */
+#define LZ4_STATIC_ASSERT(c) ASSERT(c)
-/*
- * Note: The decoding functions real_LZ4_uncompress() and
- * LZ4_uncompress_unknownOutputSize() are safe against "buffer overflow"
- * attack type. They will never write nor read outside of the provided
- * output buffers. LZ4_uncompress_unknownOutputSize() also insures that
- * it will never read outside of the input buffer. A corrupted input
- * will produce an error result, a negative int, indicating the position
- * of the error within input stream.
- *
- * Note[2]: real_LZ4_uncompress(), referred to above, is not used in ZFS so
- * its code is not present here.
- */
-static const int dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0};
-#if LZ4_ARCH64
-static const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3};
+/*! LZ4_decompress_generic() :
+ * This generic decompression function covers all use cases.
+ * It shall be instantiated several times, using different sets of directives.
+ * Note that it is important for performance that this function really get inlined,
+ * in order to remove useless branches during compilation optimization.
+ */
+LZ4_FORCE_INLINE int
+LZ4_decompress_generic(
+ const char* const src,
+ char* const dst,
+ int srcSize,
+ int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */
+
+ endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */
+ earlyEnd_directive partialDecoding, /* full, partial */
+ dict_directive dict, /* noDict, withPrefix64k, usingExtDict */
+ const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */
+ const BYTE* const dictStart, /* only if dict==usingExtDict */
+ const size_t dictSize /* note : = 0 if noDict */
+ )
+{
+ if (src == NULL) { return -1; }
+
+ { const BYTE* ip = (const BYTE*) src;
+ const BYTE* const iend = ip + srcSize;
+
+ BYTE* op = (BYTE*) dst;
+ BYTE* const oend = op + outputSize;
+ BYTE* cpy;
+
+ const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize;
+
+ const int safeDecode = (endOnInput==endOnInputSize);
+ const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
+
+
+ /* Set up the "end" pointers for the shortcut. */
+ const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/;
+ const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/;
+
+ const BYTE* match;
+ size_t offset;
+ unsigned token;
+ size_t length;
+
+
+ DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize);
+
+ /* Special cases */
+ assert(lowPrefix <= op);
+ if ((endOnInput) && (unlikely(outputSize==0))) {
+ /* Empty output buffer */
+ if (partialDecoding) return 0;
+ return ((srcSize==1) && (*ip==0)) ? 0 : -1;
+ }
+ if ((!endOnInput) && (unlikely(outputSize==0))) { return (*ip==0 ? 1 : -1); }
+ if ((endOnInput) && unlikely(srcSize==0)) { return -1; }
+
+ /* Currently the fast loop shows a regression on qualcomm arm chips. */
+#if LZ4_FAST_DEC_LOOP
+ if ((oend - op) < FASTLOOP_SAFE_DISTANCE) {
+ DEBUGLOG(6, "skip fast decode loop");
+ goto safe_decode;
+ }
+
+ /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */
+ while (1) {
+ /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */
+ assert(oend - op >= FASTLOOP_SAFE_DISTANCE);
+ if (endOnInput) { assert(ip < iend); }
+ token = *ip++;
+ length = token >> ML_BITS; /* literal length */
+
+ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
+
+ /* decode literal length */
+ if (length == RUN_MASK) {
+ variable_length_error error = ok;
+ length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error);
+ if (error == initial_error) { goto _output_error; }
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
+ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
+
+ /* copy literals */
+ cpy = op+length;
+ LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
+ if (endOnInput) { /* LZ4_decompress_safe() */
+ if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }
+ LZ4_wildCopy32(op, ip, cpy);
+ } else { /* LZ4_decompress_fast() */
+ if (cpy>oend-8) { goto safe_literal_copy; }
+ LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
+ * it doesn't know input length, and only relies on end-of-block properties */
+ }
+ ip += length; op = cpy;
+ } else {
+ cpy = op+length;
+ if (endOnInput) { /* LZ4_decompress_safe() */
+ DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
+ /* We don't need to check oend, since we check it once for each loop below */
+ if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; }
+ /* Literals can only be 14, but hope compilers optimize if we copy by a register size */
+ LZ4_memcpy(op, ip, 16);
+ } else { /* LZ4_decompress_fast() */
+ /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time :
+ * it doesn't know input length, and relies on end-of-block properties */
+ LZ4_memcpy(op, ip, 8);
+ if (length > 8) { LZ4_memcpy(op+8, ip+8, 8); }
+ }
+ ip += length; op = cpy;
+ }
+
+ /* get offset */
+ offset = LZ4_readLE16(ip); ip+=2;
+ match = op - offset;
+ assert(match <= op);
+
+ /* get matchlength */
+ length = token & ML_MASK;
+
+ if (length == ML_MASK) {
+ variable_length_error error = ok;
+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
+ length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error);
+ if (error != ok) { goto _output_error; }
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */
+ length += MINMATCH;
+ if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
+ goto safe_match_copy;
+ }
+ } else {
+ length += MINMATCH;
+ if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
+ goto safe_match_copy;
+ }
+
+ /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */
+ if ((dict == withPrefix64k) || (match >= lowPrefix)) {
+ if (offset >= 8) {
+ assert(match >= lowPrefix);
+ assert(match <= op);
+ assert(op + 18 <= oend);
+
+ LZ4_memcpy(op, match, 8);
+ LZ4_memcpy(op+8, match+8, 8);
+ LZ4_memcpy(op+16, match+16, 2);
+ op += length;
+ continue;
+ } } }
+
+ if (checkOffset && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */
+ /* match starting within external dictionary */
+ if ((dict==usingExtDict) && (match < lowPrefix)) {
+ if (unlikely(op+length > oend-LASTLITERALS)) {
+ if (partialDecoding) {
+ DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd");
+ length = MIN(length, (size_t)(oend-op));
+ } else {
+ goto _output_error; /* end-of-block condition violated */
+ } }
+
+ if (length <= (size_t)(lowPrefix-match)) {
+ /* match fits entirely within external dictionary : just copy */
+ memmove(op, dictEnd - (lowPrefix-match), length);
+ op += length;
+ } else {
+ /* match stretches into both external dictionary and current block */
+ size_t const copySize = (size_t)(lowPrefix - match);
+ size_t const restSize = length - copySize;
+ LZ4_memcpy(op, dictEnd - copySize, copySize);
+ op += copySize;
+ if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */
+ BYTE* const endOfMatch = op + restSize;
+ const BYTE* copyFrom = lowPrefix;
+ while (op < endOfMatch) { *op++ = *copyFrom++; }
+ } else {
+ LZ4_memcpy(op, lowPrefix, restSize);
+ op += restSize;
+ } }
+ continue;
+ }
+
+ /* copy match within block */
+ cpy = op + length;
+
+ assert((op <= oend) && (oend-op >= 32));
+ if (unlikely(offset<16)) {
+ LZ4_memcpy_using_offset(op, match, cpy, offset);
+ } else {
+ LZ4_wildCopy32(op, match, cpy);
+ }
+
+ op = cpy; /* wildcopy correction */
+ }
+ safe_decode:
#endif
-static int
-LZ4_uncompress_unknownOutputSize(const char *source, char *dest, int isize,
- int maxOutputSize)
-{
- /* Local Variables */
- const BYTE *restrict ip = (const BYTE *) source;
- const BYTE *const iend = ip + isize;
- const BYTE *ref;
-
- BYTE *op = (BYTE *) dest;
- BYTE *const oend = op + maxOutputSize;
- BYTE *cpy;
-
- /* Main Loop */
- while (ip < iend) {
- unsigned token;
- size_t length;
-
- /* get runlength */
- token = *ip++;
- if ((length = (token >> ML_BITS)) == RUN_MASK) {
- int s = 255;
- while ((ip < iend) && (s == 255)) {
- s = *ip++;
- if (unlikely(length > (size_t)(length + s)))
- goto _output_error;
- length += s;
- }
- }
- /* copy literals */
- cpy = op + length;
- /* CORNER-CASE: cpy might overflow. */
- if (cpy < op)
- goto _output_error; /* cpy was overflowed, bail! */
- if ((cpy > oend - COPYLENGTH) ||
- (ip + length > iend - COPYLENGTH)) {
- if (cpy > oend)
- /* Error: writes beyond output buffer */
- goto _output_error;
- if (ip + length != iend)
- /*
- * Error: LZ4 format requires to consume all
- * input at this stage
- */
- goto _output_error;
- (void) memcpy(op, ip, length);
- op += length;
- /* Necessarily EOF, due to parsing restrictions */
- break;
- }
- LZ4_WILDCOPY(ip, op, cpy);
- ip -= (op - cpy);
- op = cpy;
-
- /* get offset */
- LZ4_READ_LITTLEENDIAN_16(ref, cpy, ip);
- ip += 2;
- if (ref < (BYTE * const) dest)
- /*
- * Error: offset creates reference outside of
- * destination buffer
- */
- goto _output_error;
-
- /* get matchlength */
- if ((length = (token & ML_MASK)) == ML_MASK) {
- while (ip < iend) {
- int s = *ip++;
- if (unlikely(length > (size_t)(length + s)))
- goto _output_error;
- length += s;
- if (s == 255)
- continue;
- break;
- }
- }
- /* copy repeated sequence */
- if (unlikely(op - ref < STEPSIZE)) {
-#if LZ4_ARCH64
- int dec64 = dec64table[op - ref];
-#else
- const int dec64 = 0;
+ /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */
+ while (1) {
+ token = *ip++;
+ length = token >> ML_BITS; /* literal length */
+
+ assert(!endOnInput || ip <= iend); /* ip < iend before the increment */
+
+ /* A two-stage shortcut for the most common case:
+ * 1) If the literal length is 0..14, and there is enough space,
+ * enter the shortcut and copy 16 bytes on behalf of the literals
+ * (in the fast mode, only 8 bytes can be safely copied this way).
+ * 2) Further if the match length is 4..18, copy 18 bytes in a similar
+ * manner; but we ensure that there's enough space in the output for
+ * those 18 bytes earlier, upon entering the shortcut (in other words,
+ * there is a combined check for both stages).
+ */
+ if ( (endOnInput ? length != RUN_MASK : length <= 8)
+ /* strictly "less than" on input, to re-enter the loop with at least one byte */
+ && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) {
+ /* Copy the literals */
+ LZ4_memcpy(op, ip, endOnInput ? 16 : 8);
+ op += length; ip += length;
+
+ /* The second stage: prepare for match copying, decode full info.
+ * If it doesn't work out, the info won't be wasted. */
+ length = token & ML_MASK; /* match length */
+ offset = LZ4_readLE16(ip); ip += 2;
+ match = op - offset;
+ assert(match <= op); /* check overflow */
+
+ /* Do not deal with overlapping matches. */
+ if ( (length != ML_MASK)
+ && (offset >= 8)
+ && (dict==withPrefix64k || match >= lowPrefix) ) {
+ /* Copy the match. */
+ LZ4_memcpy(op + 0, match + 0, 8);
+ LZ4_memcpy(op + 8, match + 8, 8);
+ LZ4_memcpy(op +16, match +16, 2);
+ op += length + MINMATCH;
+ /* Both stages worked, load the next token. */
+ continue;
+ }
+
+ /* The second stage didn't work out, but the info is ready.
+ * Propel it right to the point of match copying. */
+ goto _copy_match;
+ }
+
+ /* decode literal length */
+ if (length == RUN_MASK) {
+ variable_length_error error = ok;
+ length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error);
+ if (error == initial_error) { goto _output_error; }
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */
+ if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
+ }
+
+ /* copy literals */
+ cpy = op+length;
+#if LZ4_FAST_DEC_LOOP
+ safe_literal_copy:
#endif
- op[0] = ref[0];
- op[1] = ref[1];
- op[2] = ref[2];
- op[3] = ref[3];
- op += 4;
- ref += 4;
- ref -= dec32table[op - ref];
- A32(op) = A32(ref);
- op += STEPSIZE - 4;
- ref -= dec64;
- } else {
- LZ4_COPYSTEP(ref, op);
- }
- cpy = op + length - (STEPSIZE - 4);
- if (cpy > oend - COPYLENGTH) {
- if (cpy > oend)
- /*
- * Error: request to write outside of
- * destination buffer
- */
- goto _output_error;
-#if LZ4_ARCH64
- if ((ref + COPYLENGTH) > oend)
-#else
- if ((ref + COPYLENGTH) > oend ||
- (op + COPYLENGTH) > oend)
+ LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
+ if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) )
+ || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) )
+ {
+ /* We've either hit the input parsing restriction or the output parsing restriction.
+ * In the normal scenario, decoding a full block, it must be the last sequence,
+ * otherwise it's an error (invalid input or dimensions).
+ * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow.
+ */
+ if (partialDecoding) {
+ /* Since we are partial decoding we may be in this block because of the output parsing
+ * restriction, which is not valid since the output buffer is allowed to be undersized.
+ */
+ assert(endOnInput);
+ DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end")
+ DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length);
+ DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op));
+ DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip));
+ /* Finishing in the middle of a literals segment,
+ * due to lack of input.
+ */
+ if (ip+length > iend) {
+ length = (size_t)(iend-ip);
+ cpy = op + length;
+ }
+ /* Finishing in the middle of a literals segment,
+ * due to lack of output space.
+ */
+ if (cpy > oend) {
+ cpy = oend;
+ assert(op<=oend);
+ length = (size_t)(oend-op);
+ }
+ } else {
+ /* We must be on the last sequence because of the parsing limitations so check
+ * that we exactly regenerate the original size (must be exact when !endOnInput).
+ */
+ if ((!endOnInput) && (cpy != oend)) { goto _output_error; }
+ /* We must be on the last sequence (or invalid) because of the parsing limitations
+ * so check that we exactly consume the input and don't overrun the output buffer.
+ */
+ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) {
+ DEBUGLOG(6, "should have been last run of literals")
+ DEBUGLOG(6, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend);
+ DEBUGLOG(6, "or cpy(%p) > oend(%p)", cpy, oend);
+ goto _output_error;
+ }
+ }
+ memmove(op, ip, length); /* supports overlapping memory regions; only matters for in-place decompression scenarios */
+ ip += length;
+ op += length;
+ /* Necessarily EOF when !partialDecoding.
+ * When partialDecoding, it is EOF if we've either
+ * filled the output buffer or
+ * can't proceed with reading an offset for following match.
+ */
+ if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) {
+ break;
+ }
+ } else {
+ LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */
+ ip += length; op = cpy;
+ }
+
+ /* get offset */
+ offset = LZ4_readLE16(ip); ip+=2;
+ match = op - offset;
+
+ /* get matchlength */
+ length = token & ML_MASK;
+
+ _copy_match:
+ if (length == ML_MASK) {
+ variable_length_error error = ok;
+ length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error);
+ if (error != ok) goto _output_error;
+ if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */
+ }
+ length += MINMATCH;
+
+#if LZ4_FAST_DEC_LOOP
+ safe_match_copy:
#endif
- goto _output_error;
- LZ4_SECURECOPY(ref, op, (oend - COPYLENGTH));
- while (op < cpy)
- *op++ = *ref++;
- op = cpy;
- if (op == oend)
- /*
- * Check EOF (should never happen, since
- * last 5 bytes are supposed to be literals)
- */
- goto _output_error;
- continue;
- }
- LZ4_SECURECOPY(ref, op, cpy);
- op = cpy; /* correction */
- }
-
- /* end of decoding */
- return (int)(((char *)op) - dest);
-
- /* write overflow error detected */
- _output_error:
- return (-1);
+ if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */
+ /* match starting within external dictionary */
+ if ((dict==usingExtDict) && (match < lowPrefix)) {
+ if (unlikely(op+length > oend-LASTLITERALS)) {
+ if (partialDecoding) length = MIN(length, (size_t)(oend-op));
+ else goto _output_error; /* doesn't respect parsing restriction */
+ }
+
+ if (length <= (size_t)(lowPrefix-match)) {
+ /* match fits entirely within external dictionary : just copy */
+ memmove(op, dictEnd - (lowPrefix-match), length);
+ op += length;
+ } else {
+ /* match stretches into both external dictionary and current block */
+ size_t const copySize = (size_t)(lowPrefix - match);
+ size_t const restSize = length - copySize;
+ LZ4_memcpy(op, dictEnd - copySize, copySize);
+ op += copySize;
+ if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */
+ BYTE* const endOfMatch = op + restSize;
+ const BYTE* copyFrom = lowPrefix;
+ while (op < endOfMatch) *op++ = *copyFrom++;
+ } else {
+ LZ4_memcpy(op, lowPrefix, restSize);
+ op += restSize;
+ } }
+ continue;
+ }
+ assert(match >= lowPrefix);
+
+ /* copy match within block */
+ cpy = op + length;
+
+ /* partialDecoding : may end anywhere within the block */
+ assert(op<=oend);
+ if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
+ size_t const mlen = MIN(length, (size_t)(oend-op));
+ const BYTE* const matchEnd = match + mlen;
+ BYTE* const copyEnd = op + mlen;
+ if (matchEnd > op) { /* overlap copy */
+ while (op < copyEnd) { *op++ = *match++; }
+ } else {
+ LZ4_memcpy(op, match, mlen);
+ }
+ op = copyEnd;
+ if (op == oend) { break; }
+ continue;
+ }
+
+ if (unlikely(offset<8)) {
+ LZ4_write32(op, 0); /* silence msan warning when offset==0 */
+ op[0] = match[0];
+ op[1] = match[1];
+ op[2] = match[2];
+ op[3] = match[3];
+ match += inc32table[offset];
+ LZ4_memcpy(op+4, match, 4);
+ match -= dec64table[offset];
+ } else {
+ LZ4_memcpy(op, match, 8);
+ match += 8;
+ }
+ op += 8;
+
+ if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) {
+ BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1);
+ if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */
+ if (op < oCopyLimit) {
+ LZ4_wildCopy8(op, match, oCopyLimit);
+ match += oCopyLimit - op;
+ op = oCopyLimit;
+ }
+ while (op < cpy) { *op++ = *match++; }
+ } else {
+ LZ4_memcpy(op, match, 8);
+ if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); }
+ }
+ op = cpy; /* wildcopy correction */
+ }
+
+ /* end of decoding */
+ if (endOnInput) {
+ DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst));
+ return (int) (((char*)op)-dst); /* Nb of output bytes decoded */
+ } else {
+ return (int) (((const char*)ip)-src); /* Nb of input bytes read */
+ }
+
+ /* Overflow error detected */
+ _output_error:
+ return (int) (-(((const char*)ip)-src))-1;
+ }
}
-void
-lz4_init(void)
-{
- lz4_cache = kmem_cache_create("lz4_cache",
- sizeof (struct refTables), 0, NULL, NULL, NULL, NULL, NULL, 0);
-}
+/*
+ * LZ4_uncompress_unknownOutputSize() :
+ * isize : is the input size, therefore the compressed size
+ * maxOutputSize : is the size of the destination buffer (which must be
+ * already allocated)
+ * return : the number of bytes decoded in the destination buffer
+ * (necessarily <= maxOutputSize). If the source stream is
+ * malformed, the function will stop decoding and return a
+ * negative result, indicating the byte position of the faulty
+ * instruction. This function never writes beyond dest +
+ * maxOutputSize, and is therefore protected against malicious
+ * data packets.
+ * note : Destination buffer must be already allocated.
+ * This version is slightly slower than real_LZ4_uncompress()
+ *
+ */
-void
-lz4_fini(void)
+/*
+ * Note: In upstream code, LZ4_uncompress_unknownOutputSize is now a legacy
+ * wrapper for LZ4_decompress_safe which is a wrapper for
+ * LZ4_decompress_generic; this wrapper flattens that, rather than
+ * rewriting the callers.
+ */
+int LZ4_uncompress_unknownOutputSize(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
- if (lz4_cache) {
- kmem_cache_destroy(lz4_cache);
- lz4_cache = NULL;
- }
+ return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize,
+ endOnInputSize, decode_full_block, noDict,
+ (BYTE*)dest, NULL, 0);
}
diff --git a/module/zfs/lz4.c b/module/zfs/lz4_zfs.c
similarity index 85%
copy from module/zfs/lz4.c
copy to module/zfs/lz4_zfs.c
index eba8f01b5065..e28215cf3501 100644
--- a/module/zfs/lz4.c
+++ b/module/zfs/lz4_zfs.c
@@ -1,1029 +1,880 @@
/*
* LZ4 - Fast LZ compression algorithm
* Header File
* Copyright (C) 2011-2013, Yann Collet.
* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* You can contact the author at :
* - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
* - LZ4 source repository : http://code.google.com/p/lz4/
*/
+/*
+ * N.B. - This file seems to be based on LZ4 r85, dated Dec 10, 2012
+ */
+
#include
#include
static int real_LZ4_compress(const char *source, char *dest, int isize,
int osize);
-static int LZ4_uncompress_unknownOutputSize(const char *source, char *dest,
- int isize, int maxOutputSize);
static int LZ4_compressCtx(void *ctx, const char *source, char *dest,
int isize, int osize);
static int LZ4_compress64kCtx(void *ctx, const char *source, char *dest,
int isize, int osize);
+/* See lz4.c */
+int LZ4_uncompress_unknownOutputSize(const char *source, char *dest,
+ int isize, int maxOutputSize);
+
static kmem_cache_t *lz4_cache;
size_t
lz4_compress_zfs(void *s_start, void *d_start, size_t s_len,
size_t d_len, int n)
{
(void) n;
uint32_t bufsiz;
char *dest = d_start;
ASSERT(d_len >= sizeof (bufsiz));
bufsiz = real_LZ4_compress(s_start, &dest[sizeof (bufsiz)], s_len,
d_len - sizeof (bufsiz));
/* Signal an error if the compression routine returned zero. */
if (bufsiz == 0)
return (s_len);
/*
* The exact compressed size is needed by the decompression routine,
* so it is stored at the start of the buffer. Note that this may be
* less than the compressed block size, which is rounded up to a
* multiple of 1< s_len)
return (1);
/*
* Returns 0 on success (decompression function returned non-negative)
* and non-zero on failure (decompression function returned negative).
*/
return (LZ4_uncompress_unknownOutputSize(&src[sizeof (bufsiz)],
d_start, bufsiz, d_len) < 0);
}
/*
* LZ4 API Description:
*
* Simple Functions:
* real_LZ4_compress() :
* isize : is the input size. Max supported value is ~1.9GB
* return : the number of bytes written in buffer dest
* or 0 if the compression fails (if LZ4_COMPRESSMIN is set).
* note : destination buffer must be already allocated.
* destination buffer must be sized to handle worst cases
* situations (input data not compressible) worst case size
* evaluation is provided by function LZ4_compressBound().
*
* real_LZ4_uncompress() :
* osize : is the output size, therefore the original size
* return : the number of bytes read in the source buffer.
* If the source stream is malformed, the function will stop
* decoding and return a negative result, indicating the byte
* position of the faulty instruction. This function never
* writes beyond dest + osize, and is therefore protected
* against malicious data packets.
* note : destination buffer must be already allocated
* note : real_LZ4_uncompress() is not used in ZFS so its code
* is not present here.
*
* Advanced Functions
*
* LZ4_compressBound() :
* Provides the maximum size that LZ4 may output in a "worst case"
* scenario (input data not compressible) primarily useful for memory
* allocation of output buffer.
*
* isize : is the input size. Max supported value is ~1.9GB
* return : maximum output size in a "worst case" scenario
* note : this function is limited by "int" range (2^31-1)
*
* LZ4_uncompress_unknownOutputSize() :
* isize : is the input size, therefore the compressed size
* maxOutputSize : is the size of the destination buffer (which must be
* already allocated)
* return : the number of bytes decoded in the destination buffer
* (necessarily <= maxOutputSize). If the source stream is
* malformed, the function will stop decoding and return a
* negative result, indicating the byte position of the faulty
* instruction. This function never writes beyond dest +
* maxOutputSize, and is therefore protected against malicious
* data packets.
* note : Destination buffer must be already allocated.
* This version is slightly slower than real_LZ4_uncompress()
*
* LZ4_compressCtx() :
* This function explicitly handles the CTX memory structure.
*
* ILLUMOS CHANGES: the CTX memory structure must be explicitly allocated
* by the caller (either on the stack or using kmem_cache_alloc). Passing
* NULL isn't valid.
*
* LZ4_compress64kCtx() :
* Same as LZ4_compressCtx(), but specific to small inputs (<64KB).
* isize *Must* be <64KB, otherwise the output will be corrupted.
*
* ILLUMOS CHANGES: the CTX memory structure must be explicitly allocated
* by the caller (either on the stack or using kmem_cache_alloc). Passing
* NULL isn't valid.
*/
/*
* Tuning parameters
*/
/*
* COMPRESSIONLEVEL: Increasing this value improves compression ratio
* Lowering this value reduces memory usage. Reduced memory usage
* typically improves speed, due to cache effect (ex: L1 32KB for Intel,
* L1 64KB for AMD). Memory usage formula : N->2^(N+2) Bytes
* (examples : 12 -> 16KB ; 17 -> 512KB)
*/
#define COMPRESSIONLEVEL 12
/*
* NOTCOMPRESSIBLE_CONFIRMATION: Decreasing this value will make the
* algorithm skip faster data segments considered "incompressible".
* This may decrease compression ratio dramatically, but will be
* faster on incompressible data. Increasing this value will make
* the algorithm search more before declaring a segment "incompressible".
* This could improve compression a bit, but will be slower on
* incompressible data. The default value (6) is recommended.
*/
#define NOTCOMPRESSIBLE_CONFIRMATION 6
/*
* BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE: This will provide a boost to
* performance for big endian cpu, but the resulting compressed stream
* will be incompatible with little-endian CPU. You can set this option
* to 1 in situations where data will stay within closed environment.
* This option is useless on Little_Endian CPU (such as x86).
*/
/* #define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 */
/*
* CPU Feature Detection
*/
/* 32 or 64 bits ? */
#if defined(_LP64)
#define LZ4_ARCH64 1
#else
#define LZ4_ARCH64 0
#endif
/*
* Little Endian or Big Endian?
* Note: overwrite the below #define if you know your architecture endianness.
*/
#if defined(_ZFS_BIG_ENDIAN)
#define LZ4_BIG_ENDIAN 1
#else
/*
* Little Endian assumed. PDP Endian and other very rare endian format
* are unsupported.
*/
#undef LZ4_BIG_ENDIAN
#endif
/*
* Unaligned memory access is automatically enabled for "common" CPU,
* such as x86. For others CPU, the compiler will be more cautious, and
* insert extra code to ensure aligned access is respected. If you know
* your target CPU supports unaligned memory access, you may want to
* force this option manually to improve performance
*/
#if defined(__ARM_FEATURE_UNALIGNED)
#define LZ4_FORCE_UNALIGNED_ACCESS 1
#endif
/*
* Illumos : we can't use GCC's __builtin_ctz family of builtins in the
* kernel
* Linux : we can use GCC's __builtin_ctz family of builtins in the
* kernel
*/
#undef LZ4_FORCE_SW_BITCOUNT
#if defined(__sparc)
#define LZ4_FORCE_SW_BITCOUNT
#endif
/*
* Compiler Options
*/
/* Disable restrict */
#define restrict
/*
* Linux : GCC_VERSION is defined as of 3.9-rc1, so undefine it.
* torvalds/linux@3f3f8d2f48acfd8ed3b8e6b7377935da57b27b16
*/
#ifdef GCC_VERSION
#undef GCC_VERSION
#endif
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
#define expect(expr, value) (__builtin_expect((expr), (value)))
#else
#define expect(expr, value) (expr)
#endif
#ifndef likely
#define likely(expr) expect((expr) != 0, 1)
#endif
#ifndef unlikely
#define unlikely(expr) expect((expr) != 0, 0)
#endif
#define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | \
(((x) & 0xffu) << 8)))
/* Basic types */
#define BYTE uint8_t
#define U16 uint16_t
#define U32 uint32_t
#define S32 int32_t
#define U64 uint64_t
#ifndef LZ4_FORCE_UNALIGNED_ACCESS
#pragma pack(1)
#endif
typedef struct _U16_S {
U16 v;
} U16_S;
typedef struct _U32_S {
U32 v;
} U32_S;
typedef struct _U64_S {
U64 v;
} U64_S;
#ifndef LZ4_FORCE_UNALIGNED_ACCESS
#pragma pack()
#endif
#define A64(x) (((U64_S *)(x))->v)
#define A32(x) (((U32_S *)(x))->v)
#define A16(x) (((U16_S *)(x))->v)
/*
* Constants
*/
#define MINMATCH 4
#define HASH_LOG COMPRESSIONLEVEL
#define HASHTABLESIZE (1 << HASH_LOG)
#define HASH_MASK (HASHTABLESIZE - 1)
#define SKIPSTRENGTH (NOTCOMPRESSIBLE_CONFIRMATION > 2 ? \
NOTCOMPRESSIBLE_CONFIRMATION : 2)
#define COPYLENGTH 8
#define LASTLITERALS 5
#define MFLIMIT (COPYLENGTH + MINMATCH)
#define MINLENGTH (MFLIMIT + 1)
#define MAXD_LOG 16
#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
#define ML_BITS 4
#define ML_MASK ((1U<> ((MINMATCH * 8) - \
HASH_LOG))
#define LZ4_HASH_VALUE(p) LZ4_HASH_FUNCTION(A32(p))
#define LZ4_WILDCOPY(s, d, e) do { LZ4_COPYPACKET(s, d) } while (d < e);
#define LZ4_BLINDCOPY(s, d, l) { BYTE* e = (d) + l; LZ4_WILDCOPY(s, d, e); \
d = e; }
/* Private functions */
#if LZ4_ARCH64
static inline int
LZ4_NbCommonBytes(register U64 val)
{
#if defined(LZ4_BIG_ENDIAN)
#if ((defined(__GNUC__) && (GCC_VERSION >= 304)) || defined(__clang__)) && \
!defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clzll(val) >> 3);
#else
int r;
if (!(val >> 32)) {
r = 4;
} else {
r = 0;
val >>= 32;
}
if (!(val >> 16)) {
r += 2;
val >>= 8;
} else {
val >>= 24;
}
r += (!val);
return (r);
#endif
#else
#if ((defined(__GNUC__) && (GCC_VERSION >= 304)) || defined(__clang__)) && \
!defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctzll(val) >> 3);
#else
static const int DeBruijnBytePos[64] =
{ 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5,
3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5,
5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4,
4, 5, 7, 2, 6, 5, 7, 6, 7, 7
};
return DeBruijnBytePos[((U64) ((val & -val) * 0x0218A392CDABBD3F)) >>
58];
#endif
#endif
}
#else
static inline int
LZ4_NbCommonBytes(register U32 val)
{
#if defined(LZ4_BIG_ENDIAN)
#if ((defined(__GNUC__) && (GCC_VERSION >= 304)) || defined(__clang__)) && \
!defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_clz(val) >> 3);
#else
int r;
if (!(val >> 16)) {
r = 2;
val >>= 8;
} else {
r = 0;
val >>= 24;
}
r += (!val);
return (r);
#endif
#else
#if defined(__GNUC__) && (GCC_VERSION >= 304) && \
!defined(LZ4_FORCE_SW_BITCOUNT)
return (__builtin_ctz(val) >> 3);
#else
static const int DeBruijnBytePos[32] = {
0, 0, 3, 0, 3, 1, 3, 0,
3, 2, 2, 1, 3, 2, 0, 1,
3, 3, 1, 2, 2, 2, 2, 0,
3, 1, 2, 0, 1, 0, 1, 1
};
return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >>
27];
#endif
#endif
}
#endif
/* Compression functions */
static int
LZ4_compressCtx(void *ctx, const char *source, char *dest, int isize,
int osize)
{
struct refTables *srt = (struct refTables *)ctx;
HTYPE *HashTable = (HTYPE *) (srt->hashTable);
const BYTE *ip = (BYTE *) source;
INITBASE(base);
const BYTE *anchor = ip;
const BYTE *const iend = ip + isize;
const BYTE *const oend = (BYTE *) dest + osize;
const BYTE *const mflimit = iend - MFLIMIT;
#define matchlimit (iend - LASTLITERALS)
BYTE *op = (BYTE *) dest;
int len, length;
const int skipStrength = SKIPSTRENGTH;
U32 forwardH;
/* Init */
if (isize < MINLENGTH)
goto _last_literals;
/* First Byte */
HashTable[LZ4_HASH_VALUE(ip)] = ip - base;
ip++;
forwardH = LZ4_HASH_VALUE(ip);
/* Main Loop */
for (;;) {
int findMatchAttempts = (1U << skipStrength) + 3;
const BYTE *forwardIp = ip;
const BYTE *ref;
BYTE *token;
/* Find a match */
do {
U32 h = forwardH;
int step = findMatchAttempts++ >> skipStrength;
ip = forwardIp;
forwardIp = ip + step;
if (unlikely(forwardIp > mflimit)) {
goto _last_literals;
}
forwardH = LZ4_HASH_VALUE(forwardIp);
ref = base + HashTable[h];
HashTable[h] = ip - base;
} while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip)));
/* Catch up */
while ((ip > anchor) && (ref > (BYTE *) source) &&
unlikely(ip[-1] == ref[-1])) {
ip--;
ref--;
}
/* Encode Literal length */
length = ip - anchor;
token = op++;
/* Check output limit */
if (unlikely(op + length + (2 + 1 + LASTLITERALS) +
(length >> 8) > oend))
return (0);
if (length >= (int)RUN_MASK) {
*token = (RUN_MASK << ML_BITS);
len = length - RUN_MASK;
for (; len > 254; len -= 255)
*op++ = 255;
*op++ = (BYTE)len;
} else
*token = (length << ML_BITS);
/* Copy Literals */
LZ4_BLINDCOPY(anchor, op, length);
_next_match:
/* Encode Offset */
LZ4_WRITE_LITTLEENDIAN_16(op, ip - ref);
/* Start Counting */
ip += MINMATCH;
ref += MINMATCH; /* MinMatch verified */
anchor = ip;
while (likely(ip < matchlimit - (STEPSIZE - 1))) {
UARCH diff = AARCH(ref) ^ AARCH(ip);
if (!diff) {
ip += STEPSIZE;
ref += STEPSIZE;
continue;
}
ip += LZ4_NbCommonBytes(diff);
goto _endCount;
}
#if LZ4_ARCH64
if ((ip < (matchlimit - 3)) && (A32(ref) == A32(ip))) {
ip += 4;
ref += 4;
}
#endif
if ((ip < (matchlimit - 1)) && (A16(ref) == A16(ip))) {
ip += 2;
ref += 2;
}
if ((ip < matchlimit) && (*ref == *ip))
ip++;
_endCount:
/* Encode MatchLength */
len = (ip - anchor);
/* Check output limit */
if (unlikely(op + (1 + LASTLITERALS) + (len >> 8) > oend))
return (0);
if (len >= (int)ML_MASK) {
*token += ML_MASK;
len -= ML_MASK;
for (; len > 509; len -= 510) {
*op++ = 255;
*op++ = 255;
}
if (len > 254) {
len -= 255;
*op++ = 255;
}
*op++ = (BYTE)len;
} else
*token += len;
/* Test end of chunk */
if (ip > mflimit) {
anchor = ip;
break;
}
/* Fill table */
HashTable[LZ4_HASH_VALUE(ip - 2)] = ip - 2 - base;
/* Test next position */
ref = base + HashTable[LZ4_HASH_VALUE(ip)];
HashTable[LZ4_HASH_VALUE(ip)] = ip - base;
if ((ref > ip - (MAX_DISTANCE + 1)) && (A32(ref) == A32(ip))) {
token = op++;
*token = 0;
goto _next_match;
}
/* Prepare next loop */
anchor = ip++;
forwardH = LZ4_HASH_VALUE(ip);
}
_last_literals:
/* Encode Last Literals */
{
int lastRun = iend - anchor;
if (op + lastRun + 1 + ((lastRun + 255 - RUN_MASK) / 255) >
oend)
return (0);
if (lastRun >= (int)RUN_MASK) {
*op++ = (RUN_MASK << ML_BITS);
lastRun -= RUN_MASK;
for (; lastRun > 254; lastRun -= 255) {
*op++ = 255;
}
*op++ = (BYTE)lastRun;
} else
*op++ = (lastRun << ML_BITS);
(void) memcpy(op, anchor, iend - anchor);
op += iend - anchor;
}
/* End */
return (int)(((char *)op) - dest);
}
/* Note : this function is valid only if isize < LZ4_64KLIMIT */
#define LZ4_64KLIMIT ((1 << 16) + (MFLIMIT - 1))
#define HASHLOG64K (HASH_LOG + 1)
#define HASH64KTABLESIZE (1U << HASHLOG64K)
#define LZ4_HASH64K_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8) - \
HASHLOG64K))
#define LZ4_HASH64K_VALUE(p) LZ4_HASH64K_FUNCTION(A32(p))
static int
LZ4_compress64kCtx(void *ctx, const char *source, char *dest, int isize,
int osize)
{
struct refTables *srt = (struct refTables *)ctx;
U16 *HashTable = (U16 *) (srt->hashTable);
const BYTE *ip = (BYTE *) source;
const BYTE *anchor = ip;
const BYTE *const base = ip;
const BYTE *const iend = ip + isize;
const BYTE *const oend = (BYTE *) dest + osize;
const BYTE *const mflimit = iend - MFLIMIT;
#define matchlimit (iend - LASTLITERALS)
BYTE *op = (BYTE *) dest;
int len, length;
const int skipStrength = SKIPSTRENGTH;
U32 forwardH;
/* Init */
if (isize < MINLENGTH)
goto _last_literals;
/* First Byte */
ip++;
forwardH = LZ4_HASH64K_VALUE(ip);
/* Main Loop */
for (;;) {
int findMatchAttempts = (1U << skipStrength) + 3;
const BYTE *forwardIp = ip;
const BYTE *ref;
BYTE *token;
/* Find a match */
do {
U32 h = forwardH;
int step = findMatchAttempts++ >> skipStrength;
ip = forwardIp;
forwardIp = ip + step;
if (forwardIp > mflimit) {
goto _last_literals;
}
forwardH = LZ4_HASH64K_VALUE(forwardIp);
ref = base + HashTable[h];
HashTable[h] = ip - base;
} while (A32(ref) != A32(ip));
/* Catch up */
while ((ip > anchor) && (ref > (BYTE *) source) &&
(ip[-1] == ref[-1])) {
ip--;
ref--;
}
/* Encode Literal length */
length = ip - anchor;
token = op++;
/* Check output limit */
if (unlikely(op + length + (2 + 1 + LASTLITERALS) +
(length >> 8) > oend))
return (0);
if (length >= (int)RUN_MASK) {
*token = (RUN_MASK << ML_BITS);
len = length - RUN_MASK;
for (; len > 254; len -= 255)
*op++ = 255;
*op++ = (BYTE)len;
} else
*token = (length << ML_BITS);
/* Copy Literals */
LZ4_BLINDCOPY(anchor, op, length);
_next_match:
/* Encode Offset */
LZ4_WRITE_LITTLEENDIAN_16(op, ip - ref);
/* Start Counting */
ip += MINMATCH;
ref += MINMATCH; /* MinMatch verified */
anchor = ip;
while (ip < matchlimit - (STEPSIZE - 1)) {
UARCH diff = AARCH(ref) ^ AARCH(ip);
if (!diff) {
ip += STEPSIZE;
ref += STEPSIZE;
continue;
}
ip += LZ4_NbCommonBytes(diff);
goto _endCount;
}
#if LZ4_ARCH64
if ((ip < (matchlimit - 3)) && (A32(ref) == A32(ip))) {
ip += 4;
ref += 4;
}
#endif
if ((ip < (matchlimit - 1)) && (A16(ref) == A16(ip))) {
ip += 2;
ref += 2;
}
if ((ip < matchlimit) && (*ref == *ip))
ip++;
_endCount:
/* Encode MatchLength */
len = (ip - anchor);
/* Check output limit */
if (unlikely(op + (1 + LASTLITERALS) + (len >> 8) > oend))
return (0);
if (len >= (int)ML_MASK) {
*token += ML_MASK;
len -= ML_MASK;
for (; len > 509; len -= 510) {
*op++ = 255;
*op++ = 255;
}
if (len > 254) {
len -= 255;
*op++ = 255;
}
*op++ = (BYTE)len;
} else
*token += len;
/* Test end of chunk */
if (ip > mflimit) {
anchor = ip;
break;
}
/* Fill table */
HashTable[LZ4_HASH64K_VALUE(ip - 2)] = ip - 2 - base;
/* Test next position */
ref = base + HashTable[LZ4_HASH64K_VALUE(ip)];
HashTable[LZ4_HASH64K_VALUE(ip)] = ip - base;
if (A32(ref) == A32(ip)) {
token = op++;
*token = 0;
goto _next_match;
}
/* Prepare next loop */
anchor = ip++;
forwardH = LZ4_HASH64K_VALUE(ip);
}
_last_literals:
/* Encode Last Literals */
{
int lastRun = iend - anchor;
if (op + lastRun + 1 + ((lastRun + 255 - RUN_MASK) / 255) >
oend)
return (0);
if (lastRun >= (int)RUN_MASK) {
*op++ = (RUN_MASK << ML_BITS);
lastRun -= RUN_MASK;
for (; lastRun > 254; lastRun -= 255)
*op++ = 255;
*op++ = (BYTE)lastRun;
} else
*op++ = (lastRun << ML_BITS);
(void) memcpy(op, anchor, iend - anchor);
op += iend - anchor;
}
/* End */
return (int)(((char *)op) - dest);
}
static int
real_LZ4_compress(const char *source, char *dest, int isize, int osize)
{
void *ctx;
int result;
ASSERT(lz4_cache != NULL);
ctx = kmem_cache_alloc(lz4_cache, KM_SLEEP);
/*
* out of kernel memory, gently fall through - this will disable
* compression in zio_compress_data
*/
if (ctx == NULL)
return (0);
memset(ctx, 0, sizeof (struct refTables));
if (isize < LZ4_64KLIMIT)
result = LZ4_compress64kCtx(ctx, source, dest, isize, osize);
else
result = LZ4_compressCtx(ctx, source, dest, isize, osize);
kmem_cache_free(lz4_cache, ctx);
return (result);
}
-/* Decompression functions */
-
-/*
- * Note: The decoding functions real_LZ4_uncompress() and
- * LZ4_uncompress_unknownOutputSize() are safe against "buffer overflow"
- * attack type. They will never write nor read outside of the provided
- * output buffers. LZ4_uncompress_unknownOutputSize() also insures that
- * it will never read outside of the input buffer. A corrupted input
- * will produce an error result, a negative int, indicating the position
- * of the error within input stream.
- *
- * Note[2]: real_LZ4_uncompress(), referred to above, is not used in ZFS so
- * its code is not present here.
- */
-
-static const int dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0};
-#if LZ4_ARCH64
-static const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3};
-#endif
-
-static int
-LZ4_uncompress_unknownOutputSize(const char *source, char *dest, int isize,
- int maxOutputSize)
-{
- /* Local Variables */
- const BYTE *restrict ip = (const BYTE *) source;
- const BYTE *const iend = ip + isize;
- const BYTE *ref;
-
- BYTE *op = (BYTE *) dest;
- BYTE *const oend = op + maxOutputSize;
- BYTE *cpy;
-
- /* Main Loop */
- while (ip < iend) {
- unsigned token;
- size_t length;
-
- /* get runlength */
- token = *ip++;
- if ((length = (token >> ML_BITS)) == RUN_MASK) {
- int s = 255;
- while ((ip < iend) && (s == 255)) {
- s = *ip++;
- if (unlikely(length > (size_t)(length + s)))
- goto _output_error;
- length += s;
- }
- }
- /* copy literals */
- cpy = op + length;
- /* CORNER-CASE: cpy might overflow. */
- if (cpy < op)
- goto _output_error; /* cpy was overflowed, bail! */
- if ((cpy > oend - COPYLENGTH) ||
- (ip + length > iend - COPYLENGTH)) {
- if (cpy > oend)
- /* Error: writes beyond output buffer */
- goto _output_error;
- if (ip + length != iend)
- /*
- * Error: LZ4 format requires to consume all
- * input at this stage
- */
- goto _output_error;
- (void) memcpy(op, ip, length);
- op += length;
- /* Necessarily EOF, due to parsing restrictions */
- break;
- }
- LZ4_WILDCOPY(ip, op, cpy);
- ip -= (op - cpy);
- op = cpy;
-
- /* get offset */
- LZ4_READ_LITTLEENDIAN_16(ref, cpy, ip);
- ip += 2;
- if (ref < (BYTE * const) dest)
- /*
- * Error: offset creates reference outside of
- * destination buffer
- */
- goto _output_error;
-
- /* get matchlength */
- if ((length = (token & ML_MASK)) == ML_MASK) {
- while (ip < iend) {
- int s = *ip++;
- if (unlikely(length > (size_t)(length + s)))
- goto _output_error;
- length += s;
- if (s == 255)
- continue;
- break;
- }
- }
- /* copy repeated sequence */
- if (unlikely(op - ref < STEPSIZE)) {
-#if LZ4_ARCH64
- int dec64 = dec64table[op - ref];
-#else
- const int dec64 = 0;
-#endif
- op[0] = ref[0];
- op[1] = ref[1];
- op[2] = ref[2];
- op[3] = ref[3];
- op += 4;
- ref += 4;
- ref -= dec32table[op - ref];
- A32(op) = A32(ref);
- op += STEPSIZE - 4;
- ref -= dec64;
- } else {
- LZ4_COPYSTEP(ref, op);
- }
- cpy = op + length - (STEPSIZE - 4);
- if (cpy > oend - COPYLENGTH) {
- if (cpy > oend)
- /*
- * Error: request to write outside of
- * destination buffer
- */
- goto _output_error;
-#if LZ4_ARCH64
- if ((ref + COPYLENGTH) > oend)
-#else
- if ((ref + COPYLENGTH) > oend ||
- (op + COPYLENGTH) > oend)
-#endif
- goto _output_error;
- LZ4_SECURECOPY(ref, op, (oend - COPYLENGTH));
- while (op < cpy)
- *op++ = *ref++;
- op = cpy;
- if (op == oend)
- /*
- * Check EOF (should never happen, since
- * last 5 bytes are supposed to be literals)
- */
- goto _output_error;
- continue;
- }
- LZ4_SECURECOPY(ref, op, cpy);
- op = cpy; /* correction */
- }
-
- /* end of decoding */
- return (int)(((char *)op) - dest);
-
- /* write overflow error detected */
- _output_error:
- return (-1);
-}
-
void
lz4_init(void)
{
lz4_cache = kmem_cache_create("lz4_cache",
sizeof (struct refTables), 0, NULL, NULL, NULL, NULL, NULL, 0);
}
void
lz4_fini(void)
{
if (lz4_cache) {
kmem_cache_destroy(lz4_cache);
lz4_cache = NULL;
}
}