diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 77b1269a9de4..a5107e6fc872 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -1,89 +1,90 @@ include $(top_srcdir)/config/Shellcheck.am pkgdatadir = $(datadir)/@PACKAGE@ dist_pkgdata_SCRIPTS = \ zimport.sh \ zfs.sh \ zfs-tests.sh \ zloop.sh \ zfs-helpers.sh EXTRA_SCRIPTS = \ commitcheck.sh \ common.sh.in \ dkms.mkconf \ dkms.postbuild \ kmodtool \ make_gitrev.sh \ man-dates.sh \ paxcheck.sh \ mancheck.sh EXTRA_DIST = \ cstyle.pl \ enum-extract.pl \ zfs2zol-patch.sed \ zol2zfs-patch.sed \ $(EXTRA_SCRIPTS) SHELLCHECKSCRIPTS = $(EXTRA_SCRIPTS) +SHELLCHECK_OPTS = --enable=all define EXTRA_ENVIRONMENT # Only required for in-tree use export INTREE="yes" export GDB="libtool --mode=execute gdb" export LDMOD=/sbin/insmod export CMD_DIR=@abs_top_builddir@/cmd export UDEV_RULE_DIR=@abs_top_builddir@/udev/rules.d export ZEDLET_ETC_DIR=$$CMD_DIR/zed/zed.d export ZEDLET_LIBEXEC_DIR=$$CMD_DIR/zed/zed.d export ZPOOL_SCRIPT_DIR=$$CMD_DIR/zpool/zpool.d export ZPOOL_SCRIPTS_PATH=$$CMD_DIR/zpool/zpool.d export ZPOOL_COMPAT_DIR=$$CMD_DIR/zpool/compatibility.d export CONTRIB_DIR=@abs_top_builddir@/contrib export LIB_DIR=@abs_top_builddir@/lib export SYSCONF_DIR=@abs_top_builddir@/etc export INSTALL_UDEV_DIR=@udevdir@ export INSTALL_UDEV_RULE_DIR=@udevruledir@ export INSTALL_MOUNT_HELPER_DIR=@mounthelperdir@ export INSTALL_SYSCONF_DIR=@sysconfdir@ export INSTALL_PYTHON_DIR=@pythonsitedir@ export KMOD_SPL=@abs_top_builddir@/module/spl/spl.ko export KMOD_ZAVL=@abs_top_builddir@/module/avl/zavl.ko export KMOD_ZNVPAIR=@abs_top_builddir@/module/nvpair/znvpair.ko export KMOD_ZUNICODE=@abs_top_builddir@/module/unicode/zunicode.ko export KMOD_ZCOMMON=@abs_top_builddir@/module/zcommon/zcommon.ko export KMOD_ZLUA=@abs_top_builddir@/module/lua/zlua.ko export KMOD_ICP=@abs_top_builddir@/module/icp/icp.ko export KMOD_ZFS=@abs_top_builddir@/module/zfs/zfs.ko export KMOD_FREEBSD=@abs_top_builddir@/module/openzfs.ko export KMOD_ZZSTD=@abs_top_builddir@/module/zstd/zzstd.ko endef export EXTRA_ENVIRONMENT all-local: -$(SED) -e '\|^export BIN_DIR=|s|$$|@abs_top_builddir@/bin|' \ -e '\|^export SBIN_DIR=|s|$$|@abs_top_builddir@/bin|' \ -e '\|^export LIBEXEC_DIR=|s|$$|@abs_top_builddir@/bin|' \ -e '\|^export ZTS_DIR=|s|$$|@abs_top_srcdir@/tests|' \ -e '\|^export SCRIPT_DIR=|s|$$|@abs_top_srcdir@/scripts|' \ $(abs_top_srcdir)/scripts/common.sh.in >common.sh -echo "$$EXTRA_ENVIRONMENT" >>common.sh clean-local: -$(RM) common.sh install-data-hook: -$(SED) -e '\|^export BIN_DIR=|s|$$|@bindir@|' \ -e '\|^export SBIN_DIR=|s|$$|@sbindir@|' \ -e '\|^export LIBEXEC_DIR=|s|$$|@zfsexecdir@|' \ -e '\|^export ZTS_DIR=|s|$$|@datadir@/@PACKAGE@|' \ -e '\|^export SCRIPT_DIR=|s|$$|@datadir@/@PACKAGE@|' \ $(abs_top_srcdir)/scripts/common.sh.in \ >$(DESTDIR)$(datadir)/@PACKAGE@/common.sh diff --git a/scripts/commitcheck.sh b/scripts/commitcheck.sh index cb9fd66c6f46..1b1d097501db 100755 --- a/scripts/commitcheck.sh +++ b/scripts/commitcheck.sh @@ -1,123 +1,123 @@ #!/bin/sh REF="HEAD" # test commit body for length # lines containing urls are exempt for the length limit. test_commit_bodylength() { length="72" body=$(git log --no-show-signature -n 1 --pretty=%b "$REF" | grep -Ev "http(s)*://" | grep -E -m 1 ".{$((length + 1))}") if [ -n "$body" ]; then echo "error: commit message body contains line over ${length} characters" return 1 fi return 0 } # check for a tagged line check_tagged_line() { regex='^[[:space:]]*'"$1"':[[:space:]][[:print:]]+[[:space:]]<[[:graph:]]+>$' foundline=$(git log --no-show-signature -n 1 "$REF" | grep -E -m 1 "$regex") if [ -z "$foundline" ]; then echo "error: missing \"$1\"" return 1 fi return 0 } # check commit message for a normal commit new_change_commit() { error=0 # subject is not longer than 72 characters long_subject=$(git log --no-show-signature -n 1 --pretty=%s "$REF" | grep -E -m 1 '.{73}') if [ -n "$long_subject" ]; then echo "error: commit subject over 72 characters" error=1 fi # need a signed off by if ! check_tagged_line "Signed-off-by" ; then error=1 fi # ensure that no lines in the body of the commit are over 72 characters if ! test_commit_bodylength ; then error=1 fi - return $error + return "$error" } is_coverity_fix() { # subject starts with Fix coverity defects means it's a coverity fix subject=$(git log --no-show-signature -n 1 --pretty=%s "$REF" | grep -E -m 1 '^Fix coverity defects') if [ -n "$subject" ]; then return 0 fi return 1 } coverity_fix_commit() { error=0 # subject starts with Fix coverity defects: CID dddd, dddd... subject=$(git log --no-show-signature -n 1 --pretty=%s "$REF" | grep -E -m 1 'Fix coverity defects: CID [[:digit:]]+(, [[:digit:]]+)*') if [ -z "$subject" ]; then echo "error: Coverity defect fixes must have a subject line that starts with \"Fix coverity defects: CID dddd\"" error=1 fi # need a signed off by if ! check_tagged_line "Signed-off-by" ; then error=1 fi # test each summary line for the proper format OLDIFS=$IFS IFS=' ' for line in $(git log --no-show-signature -n 1 --pretty=%b "$REF" | grep -E '^CID'); do if ! echo "$line" | grep -qE '^CID [[:digit:]]+: ([[:graph:]]+|[[:space:]])+ \(([[:upper:]]|\_)+\)'; then echo "error: commit message has an improperly formatted CID defect line" error=1 fi done IFS=$OLDIFS # ensure that no lines in the body of the commit are over 72 characters if ! test_commit_bodylength; then error=1 fi - return $error + return "$error" } if [ -n "$1" ]; then REF="$1" fi # if coverity fix, test against that if is_coverity_fix; then if ! coverity_fix_commit; then exit 1 else exit 0 fi fi # have a normal commit if ! new_change_commit ; then exit 1 fi exit 0 diff --git a/scripts/kmodtool b/scripts/kmodtool index b1021596997e..afbb6ab3b03c 100755 --- a/scripts/kmodtool +++ b/scripts/kmodtool @@ -1,625 +1,628 @@ #!/usr/bin/env bash # shellcheck disable=SC2086 # kmodtool - Helper script for building kernel module RPMs # Copyright (c) 2003-2012 Ville Skyttä , # Thorsten Leemhuis # Nicolas Chauvet # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. shopt -s extglob myprog="kmodtool-${repo}" myver="0.12.1" kmodname= build_kernels="current" kernels_known_variants= kernel_versions= kernel_versions_to_build_for= prefix= filterfile= target= buildroot= dashvariant= error_out() { local errorlevel=${1} shift echo "Error: $*" >&2 # the next line is not multi-line safe -- not needed *yet* echo "%global kmodtool_check echo \"kmodtool error: $*\"; exit ${errorlevel};" exit "${errorlevel}" } print_rpmtemplate_header() { echo - echo '%global kmodinstdir_prefix '${prefix}/lib/modules/ - echo '%global kmodinstdir_postfix '/extra/${kmodname}/ - echo '%global kernel_versions '${kernel_versions} + echo "%global kmodinstdir_prefix ${prefix}/lib/modules/" + echo "%global kmodinstdir_postfix /extra/${kmodname}/" + echo "%global kernel_versions ${kernel_versions}" echo } print_akmodtemplate () { echo cat <= %{?epoch:%{epoch}:}%{version} Provides: ${kmodname}-kmod = %{?epoch:%{epoch}:}%{version}-%{release} EOF - if [[ ${obsolete_name} ]]; then + if [ -n "${obsolete_name}" ]; then echo "Provides: akmod-${obsolete_name} = ${obsolete_version}" echo "Obsoletes: akmod-${obsolete_name} < ${obsolete_version}" fi cat < /dev/null & %files -n akmod-${kmodname} %defattr(-,root,root,-) %{_usrsrc}/akmods/* EOF } print_akmodmeta () { cat <= %{?epoch:%{epoch}:}%{version} %if 0%{?rhel} == 6 || 0%{?centos} == 6 Requires(post): module-init-tools Requires(postun): module-init-tools %else Requires(post): kmod Requires(postun): kmod %endif EOF - if [[ ${obsolete_name} ]]; then + if [ -n "${obsolete_name}" ]; then echo "Provides: kmod-${obsolete_name}-${kernel_uname_r} = ${obsolete_version}" echo "Obsoletes: kmod-${obsolete_name}-${kernel_uname_r} < ${obsolete_version}" fi # second part - if [[ ! "${customkernel}" ]]; then + if [ -z "${customkernel}" ]; then cat < /dev/null || : -elif [[ -f "/lib/modules/${kernel_uname_r}/System.map" ]]; then +elif [ -f "/lib/modules/${kernel_uname_r}/System.map" ]; then ${prefix}${depmod_path} -aeF /lib/modules/${kernel_uname_r}/System.map ${kernel_uname_r} > /dev/null || : else ${prefix}${depmod_path} -ae ${kernel_uname_r} &> /dev/null || : fi %postun -n kmod-${kmodname}-${kernel_uname_r} -if [[ -f "/boot/System.map-${kernel_uname_r}" ]]; then +if [ -f "/boot/System.map-${kernel_uname_r}" ]; then ${prefix}${depmod_path} -aF /boot/System.map-${kernel_uname_r} ${kernel_uname_r} &> /dev/null || : -elif [[ -f "/lib/modules/${kernel_uname_r}/System.map" ]]; then +elif [ -f "/lib/modules/${kernel_uname_r}/System.map" ]; then ${prefix}${depmod_path} -aF /lib/modules/${kernel_uname_r}/System.map ${kernel_uname_r} &> /dev/null || : else ${prefix}${depmod_path} -a ${kernel_uname_r} &> /dev/null || : fi EOF else cat < /dev/null || : +[ "\$(uname -r)" = "${kernel_uname_r}" ] && ${prefix}${depmod_path} -a > /dev/null || : %postun -n kmod-${kmodname}-${kernel_uname_r} -[[ "\$(uname -r)" == "${kernel_uname_r}" ]] && ${prefix}${depmod_path} -a > /dev/null || : +[ "\$(uname -r)" = "${kernel_uname_r}" ] && ${prefix}${depmod_path} -a > /dev/null || : EOF fi # third part cat <= %{?epoch:%{epoch}:}%{version}-%{release}" fi - if [[ ${obsolete_name} ]]; then + if [ -n "${obsolete_name}" ]; then echo "Provides: kmod-${obsolete_name}-devel = ${obsolete_version}" echo "Obsoletes: kmod-${obsolete_name}-devel < ${obsolete_version}" fi cat < objects for the newest kernel. %files -n kmod-${kmodname}-devel %defattr(644,root,root,755) %{_usrsrc}/${kmodname}-%{version} EOF - if [[ ${obsolete_name} ]]; then + if [ -n "${obsolete_name}" ]; then echo "%{_usrsrc}/${obsolete_name}-%{version}" fi for kernel in ${1}; do local kernel_uname_r=${kernel} echo "%exclude %{_usrsrc}/${kmodname}-%{version}/${kernel_uname_r}" - if [[ ${obsolete_name} ]]; then + if [ -n "${obsolete_name}" ]; then echo "%exclude %{_usrsrc}/${obsolete_name}-%{version}/${kernel_uname_r}" fi done echo echo } print_rpmtemplate_per_kmoddevelpkg () { - if [[ "${1}" == "--custom" ]]; then + if [ "${1}" = "--custom" ]; then shift local customkernel=true - elif [[ "${1}" == "--redhat" ]]; then + elif [ "${1}" = "--redhat" ]; then # this is needed for akmods shift local redhatkernel=true fi local kernel_uname_r=${1} local kernel_variant="${2:+-${2}}" # first part cat <= %{?epoch:%{epoch}:}%{version}-%{release} %{?KmodsMetaRequires:Requires: %{?KmodsMetaRequires}} EOF - if [[ ${obsolete_name} ]]; then + if [ -n "${obsolete_name}" ]; then echo "Provides: kmod-${obsolete_name}${kernel_variant} = ${obsolete_version}" echo "Obsoletes: kmod-${obsolete_name}${kernel_variant} < ${obsolete_version}" fi cat < -- filter the results with grep --file " echo " --for-kernels -- created templates only for these kernels" echo " --kmodname -- name of the kmod (required)" echo " --devel -- make kmod-devel package" echo " --noakmod -- no akmod package" echo " --repo -- use buildsys-build--kerneldevpkgs" echo " --target -- target-arch (required)" echo " --buildroot -- Build root (place to look for build files)" } -while [ "${1}" ] ; do +while [ -n "${1}" ] ; do case "${1}" in --filterfile) shift - if [[ ! "${1}" ]] ; then + if [ -z "${1}" ] ; then error_out 2 "Please provide path to a filter-file together with --filterfile" >&2 - elif [[ ! -e "${1}" ]]; then + elif [ ! -e "${1}" ]; then error_out 2 "Filterfile ${1} not found" >&2 fi filterfile="${1}" shift ;; --kmodname) shift - if [[ ! "${1}" ]] ; then + if [ -z "${1}" ] ; then error_out 2 "Please provide the name of the kmod together with --kmodname" >&2 fi # strip pending -kmod kmodname="${1%%-kmod}" shift ;; --devel) shift devel="true" ;; --prefix) shift - if [[ ! "${1}" ]] ; then + if [ -z "${1}" ] ; then error_out 2 "Please provide a prefix with --prefix" >&2 fi prefix="${1}" shift ;; --repo) shift - if [[ ! "${1}" ]] ; then + if [ -z "${1}" ] ; then error_out 2 "Please provide the name of the repo together with --repo" >&2 fi repo=${1} shift ;; --for-kernels) shift - if [[ ! "${1}" ]] ; then + if [ -z "${1}" ] ; then error_out 2 "Please provide the name of the kmod together with --kmodname" >&2 fi for_kernels="${1}" shift ;; --noakmod) shift noakmod="true" ;; --obsolete-name) shift - if [[ ! "${1}" ]] ; then + if [ -z "${1}" ] ; then error_out 2 "Please provide the name of the kmod to obsolete together with --obsolete-name" >&2 fi obsolete_name="${1}" shift ;; --obsolete-version) shift - if [[ ! "${1}" ]] ; then + if [ -z "${1}" ] ; then error_out 2 "Please provide the version of the kmod to obsolete together with --obsolete-version" >&2 fi obsolete_version="${1}" shift ;; --target) shift target="${1}" shift ;; --akmod) shift build_kernels="akmod" ;; --newest) shift build_kernels="newest" ;; --current) shift build_kernels="current" ;; --buildroot) shift buildroot="${1}" shift ;; --help) myprog_help exit 0 ;; --version) echo "${myprog} ${myver}" exit 0 ;; *) echo "Error: Unknown option '${1}'." >&2 usage >&2 exit 2 ;; esac done -if [[ -e ./kmodtool-kernel-variants ]]; then +if [ -e ./kmodtool-kernel-variants ]; then kernels_known_variants="$(cat ./kmodtool-kernel-variants)" -elif [[ -e /usr/share/kmodtool/kernel-variants ]] ; then +elif [ -e /usr/share/kmodtool/kernel-variants ] ; then kernels_known_variants="$(cat /usr/share/kmodtool/kernel-variants)" else kernels_known_variants="@(smp?(-debug)|PAE?(-debug)|debug|kdump|xen|kirkwood|highbank|imx|omap|tegra)" fi # general sanity checks -if [[ ! "${target}" ]]; then +if [ -z "${target}" ]; then error_out 2 "please pass target arch with --target" -elif [[ ! "${kmodname}" ]]; then +elif [ -z "${kmodname}" ]; then error_out 2 "please pass kmodname with --kmodname" -elif [[ ! "${kernels_known_variants}" ]] ; then +elif [ -z "${kernels_known_variants}" ] ; then error_out 2 "could not determine known variants" -elif { [[ "${obsolete_name}" ]] && [[ ! "${obsolete_version}" ]]; } || { [[ ! "${obsolete_name}" ]] && [[ "${obsolete_version}" ]]; } ; then +elif { [ -n "${obsolete_name}" ] && [ -z "${obsolete_version}" ]; } || { [ -z "${obsolete_name}" ] && [ -n "${obsolete_version}" ]; } ; then error_out 2 "you need to provide both --obsolete-name and --obsolete-version" fi # go -if [[ "${for_kernels}" ]]; then +if [ -n "${for_kernels}" ]; then # this is easy: print_customrpmtemplate "${for_kernels}" -elif [[ "${build_kernels}" == "akmod" ]]; then +elif [ "${build_kernels}" = "akmod" ]; then # do only a akmod package print_akmodtemplate print_akmodmeta else # seems we are on out own to decide for which kernels to build # we need more sanity checks in this case - if [[ ! "${repo}" ]]; then + if [ -z "${repo}" ]; then error_out 2 "please provide repo name with --repo" - elif ! command -v "buildsys-build-${repo}-kerneldevpkgs" &> /dev/null ; then + elif ! command -v "buildsys-build-${repo}-kerneldevpkgs" > /dev/null 2>&1; then error_out 2 "buildsys-build-${repo}-kerneldevpkgs not found" fi # call buildsys-build-${repo}-kerneldevpkgs to get the list of kernels cmdoptions="--target ${target}" # filterfile to filter list of kernels? - if [[ "${filterfile}" ]] ; then + if [ -n "${filterfile}" ] ; then cmdoptions="${cmdoptions} --filterfile ${filterfile}" fi - kernel_versions_to_build_for="$(buildsys-build-${repo}-kerneldevpkgs --${build_kernels} ${cmdoptions})" + kernel_versions_to_build_for=$(buildsys-build-${repo}-kerneldevpkgs "--${build_kernels}" ${cmdoptions}) returncode=$? - if (( returncode != 0 )); then - error_out 2 "buildsys-build-${repo}-kerneldevpkgs failed: $(buildsys-build-${repo}-kerneldevpkgs --${build_kernels} ${cmdoptions})" + if [ "$returncode" -ne 0 ]; then + + error_out 2 "buildsys-build-${repo}-kerneldevpkgs failed: ${kernel_versions_to_build_for}" fi - if [[ "${build_kernels}" == "current" ]] && [[ ! "${noakmod}" ]]; then + if [ "${build_kernels}" = "current" ] && [ -z "${noakmod}" ]; then print_akmodtemplate fi print_rpmtemplate fi diff --git a/scripts/make_gitrev.sh b/scripts/make_gitrev.sh index e7f4ce8844d5..4a6a98f52929 100755 --- a/scripts/make_gitrev.sh +++ b/scripts/make_gitrev.sh @@ -1,78 +1,80 @@ #!/bin/sh # # CDDL HEADER START # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # CDDL HEADER END # # Copyright (c) 2018 by Delphix. All rights reserved. # Copyright (c) 2018 by Matthew Thode. All rights reserved. # # Generate zfs_gitrev.h. Note that we need to do this for every # invocation of `make`, including for incremental builds. Therefore we # can't use a zfs_gitrev.h.in file which would be processed only when # `configure` is run. # -set -e -u +set -eu dist=no distdir=. while getopts D: flag do case $flag in \?) echo "Usage: $0 [-D distdir] [file]" >&2; exit 1;; D) dist=yes; distdir=${OPTARG};; + *) ;; esac done shift $((OPTIND - 1)) top_srcdir="$(dirname "$0")/.." GITREV="${1:-include/zfs_gitrev.h}" # GITREV should be a relative path (relative to top_builddir or distdir) case "${GITREV}" in /*) echo "Error: ${GITREV} should be a relative path" >&2 exit 1;; + *) ;; esac ZFS_GITREV=$({ cd "${top_srcdir}" && git describe --always --long --dirty 2>/dev/null; } || :) if [ -z "${ZFS_GITREV}" ] then # If the source directory is not a git repository, check if the file # already exists (in the source) if [ -f "${top_srcdir}/${GITREV}" ] then ZFS_GITREV=$(sed -n \ '1s/^#define[[:blank:]]ZFS_META_GITREV "\([^"]*\)"$/\1/p' \ "${top_srcdir}/${GITREV}") fi -elif [ ${dist} = yes ] +elif [ "${dist}" = yes ] then # Append -dist when creating distributed sources from a git repository ZFS_GITREV="${ZFS_GITREV}-dist" fi ZFS_GITREV=${ZFS_GITREV:-unknown} GITREVTMP="${GITREV}~" printf '#define\tZFS_META_GITREV "%s"\n' "${ZFS_GITREV}" >"${GITREVTMP}" GITREV="${distdir}/${GITREV}" if cmp -s "${GITREV}" "${GITREVTMP}" then rm -f "${GITREVTMP}" else mv -f "${GITREVTMP}" "${GITREV}" fi diff --git a/scripts/paxcheck.sh b/scripts/paxcheck.sh index 27acc95364aa..aba770e9e6f3 100755 --- a/scripts/paxcheck.sh +++ b/scripts/paxcheck.sh @@ -1,43 +1,43 @@ #!/bin/sh if ! command -v scanelf > /dev/null; then echo "scanelf (from pax-utils) is required for these checks." >&2 exit 3 fi RET=0 # check for exec stacks OUT=$(scanelf -qyRAF '%e %p' "$1") if [ x"${OUT}" != x ]; then RET=2 echo "The following files contain writable and executable sections" echo " Files with such sections will not work properly (or at all!) on some" echo " architectures/operating systems." echo " For more information, see:" echo " https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart" echo echo "${OUT}" echo fi # check for TEXTRELS OUT=$(scanelf -qyRAF '%T %p' "$1") if [ x"${OUT}" != x ]; then RET=2 echo "The following files contain runtime text relocations" echo " Text relocations force the dynamic linker to perform extra" echo " work at startup, waste system resources, and may pose a security" echo " risk. On some architectures, the code may not even function" echo " properly, if at all." echo " For more information, see:" echo " https://wiki.gentoo.org/wiki/Hardened/HOWTO_locate_and_fix_textrels" echo echo "${OUT}" echo fi -exit $RET +exit "$RET" diff --git a/scripts/zfs-helpers.sh b/scripts/zfs-helpers.sh index 02b4922006f2..a86a6eb61a60 100755 --- a/scripts/zfs-helpers.sh +++ b/scripts/zfs-helpers.sh @@ -1,194 +1,197 @@ #!/bin/sh +# shellcheck disable=SC2154 # # This script is designed to facilitate in-tree development and testing # by installing symlinks on your system which refer to in-tree helper # utilities. These helper utilities must be installed to in order to # exercise all ZFS functionality. By using symbolic links and keeping # the scripts in-tree during development they can be easily modified # and those changes tracked. # # Use the following configuration option to override the installation # paths for these scripts. The correct path is automatically set for # most distributions but you can optionally set it for your environment. # # --with-mounthelperdir=DIR install mount.zfs in dir [/sbin] # --with-udevdir=DIR install udev helpers [default=check] # --with-udevruledir=DIR install udev rules [default=UDEVDIR/rules.d] # --sysconfdir=DIR install zfs configuration files [PREFIX/etc] # BASE_DIR=$(dirname "$0") SCRIPT_COMMON=common.sh if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then . "${BASE_DIR}/${SCRIPT_COMMON}" else echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 fi PROG=zfs-helpers.sh DRYRUN="no" INSTALL="no" REMOVE="no" VERBOSE="no" fail() { echo "${PROG}: $1" >&2 exit 1 } msg() { if [ "$VERBOSE" = "yes" ]; then echo "$@" fi } usage() { cat << EOF USAGE: $0 [dhirv] DESCRIPTION: Install/remove the ZFS helper utilities. OPTIONS: -d Dry run -h Show this message -i Install the helper utilities -r Remove the helper utilities -v Verbose $0 -iv $0 -r EOF } while getopts 'hdirv' OPTION; do case $OPTION in h) usage exit 1 ;; d) DRYRUN="yes" ;; i) INSTALL="yes" ;; r) REMOVE="yes" ;; v) VERBOSE="yes" ;; ?) usage exit ;; + *) + ;; esac done if [ "$INSTALL" = "yes" ] && [ "$REMOVE" = "yes" ]; then fail "Specify -i or -r but not both" fi if [ "$INSTALL" = "no" ] && [ "$REMOVE" = "no" ]; then fail "Either -i or -r must be specified" fi if [ "$(id -u)" != "0" ]; then fail "Must run as root" fi if [ "$INTREE" != "yes" ]; then fail "Must be run in-tree" fi if [ "$VERBOSE" = "yes" ]; then echo "--- Configuration ---" echo "udevdir: $INSTALL_UDEV_DIR" echo "udevruledir: $INSTALL_UDEV_RULE_DIR" echo "mounthelperdir: $INSTALL_MOUNT_HELPER_DIR" echo "sysconfdir: $INSTALL_SYSCONF_DIR" echo "pythonsitedir: $INSTALL_PYTHON_DIR" echo "dryrun: $DRYRUN" echo fi install() { src=$1 dst=$2 if [ -h "$dst" ]; then echo "Symlink exists: $dst" elif [ -e "$dst" ]; then echo "File exists: $dst" elif [ ! -e "$src" ]; then echo "Source missing: $src" else msg "ln -s $src $dst" if [ "$DRYRUN" = "no" ]; then DIR=$(dirname "$dst") mkdir -p "$DIR" >/dev/null 2>&1 ln -s "$src" "$dst" fi fi } remove() { dst=$1 if [ -h "$dst" ]; then msg "rm $dst" rm "$dst" DIR=$(dirname "$dst") rmdir "$DIR" >/dev/null 2>&1 elif [ -e "$dst" ]; then echo "Expected symlink: $dst" fi } if [ "${INSTALL}" = "yes" ]; then install "$CMD_DIR/mount_zfs/mount.zfs" \ "$INSTALL_MOUNT_HELPER_DIR/mount.zfs" install "$CMD_DIR/fsck_zfs/fsck.zfs" \ "$INSTALL_MOUNT_HELPER_DIR/fsck.zfs" install "$CMD_DIR/zvol_id/zvol_id" \ "$INSTALL_UDEV_DIR/zvol_id" install "$CMD_DIR/vdev_id/vdev_id" \ "$INSTALL_UDEV_DIR/vdev_id" install "$UDEV_RULE_DIR/60-zvol.rules" \ "$INSTALL_UDEV_RULE_DIR/60-zvol.rules" install "$UDEV_RULE_DIR/69-vdev.rules" \ "$INSTALL_UDEV_RULE_DIR/69-vdev.rules" install "$UDEV_RULE_DIR/90-zfs.rules" \ "$INSTALL_UDEV_RULE_DIR/90-zfs.rules" install "$CMD_DIR/zpool/zpool.d" \ "$INSTALL_SYSCONF_DIR/zfs/zpool.d" install "$CONTRIB_DIR/pyzfs/libzfs_core" \ "$INSTALL_PYTHON_DIR/libzfs_core" # Ideally we would install these in the configured ${libdir}, which is # by default "/usr/local/lib and unfortunately not included in the # dynamic linker search path. install "$(find "$LIB_DIR/libzfs_core" -type f -name 'libzfs_core.so*')" \ "/lib/libzfs_core.so" install "$(find "$LIB_DIR/libnvpair" -type f -name 'libnvpair.so*')" \ "/lib/libnvpair.so" ldconfig else remove "$INSTALL_MOUNT_HELPER_DIR/mount.zfs" remove "$INSTALL_MOUNT_HELPER_DIR/fsck.zfs" remove "$INSTALL_UDEV_DIR/zvol_id" remove "$INSTALL_UDEV_DIR/vdev_id" remove "$INSTALL_UDEV_RULE_DIR/60-zvol.rules" remove "$INSTALL_UDEV_RULE_DIR/69-vdev.rules" remove "$INSTALL_UDEV_RULE_DIR/90-zfs.rules" remove "$INSTALL_SYSCONF_DIR/zfs/zpool.d" remove "$INSTALL_PYTHON_DIR/libzfs_core" remove "/lib/libzfs_core.so" remove "/lib/libnvpair.so" ldconfig fi exit 0 diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh index f871a51d34c2..aaaf9ddfc25d 100755 --- a/scripts/zfs-tests.sh +++ b/scripts/zfs-tests.sh @@ -1,748 +1,751 @@ #!/bin/sh +# shellcheck disable=SC2154 # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License, Version 1.0 only # (the "License"). You may not use this file except in compliance # with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2020 OmniOS Community Edition (OmniOSce) Association. # BASE_DIR=$(dirname "$0") SCRIPT_COMMON=common.sh if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then . "${BASE_DIR}/${SCRIPT_COMMON}" else echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 fi PROG=zfs-tests.sh VERBOSE="no" QUIET="" CLEANUP="yes" CLEANUPALL="no" LOOPBACK="yes" STACK_TRACER="no" FILESIZE="4G" DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run" RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES} FILEDIR=${FILEDIR:-/var/tmp} DISKS=${DISKS:-""} SINGLETEST="" SINGLETESTUSER="root" TAGS="" ITERATIONS=1 ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh" ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh" UNAME=$(uname -s) RERUN="" # Override some defaults if on FreeBSD if [ "$UNAME" = "FreeBSD" ] ; then TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"} LOSETUP=/sbin/mdconfig DMSETUP=/sbin/gpart else ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh" TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"} LOSETUP=${LOSETUP:-/sbin/losetup} DMSETUP=${DMSETUP:-/sbin/dmsetup} fi # # Log an informational message when additional verbosity is enabled. # msg() { if [ "$VERBOSE" = "yes" ]; then echo "$@" fi } # # Log a failure message, cleanup, and return an error. # fail() { echo "$PROG: $1" >&2 cleanup exit 1 } cleanup_freebsd_loopback() { for TEST_LOOPBACK in ${LOOPBACKS}; do if [ -c "/dev/${TEST_LOOPBACK}" ]; then sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" || echo "Failed to destroy: ${TEST_LOOPBACK}" fi done } cleanup_linux_loopback() { for TEST_LOOPBACK in ${LOOPBACKS}; do LOOP_DEV="${TEST_LOOPBACK##*/}" DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \ grep "${LOOP_DEV}" | cut -f1) if [ -n "$DM_DEV" ]; then sudo "${DMSETUP}" remove "${DM_DEV}" || echo "Failed to remove: ${DM_DEV}" fi if [ -n "${TEST_LOOPBACK}" ]; then sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" || echo "Failed to remove: ${TEST_LOOPBACK}" fi done } # # Attempt to remove loopback devices and files which where created earlier # by this script to run the test framework. The '-k' option may be passed # to the script to suppress cleanup for debugging purposes. # cleanup() { if [ "$CLEANUP" = "no" ]; then return 0 fi if [ "$LOOPBACK" = "yes" ]; then if [ "$UNAME" = "FreeBSD" ] ; then cleanup_freebsd_loopback else cleanup_linux_loopback fi fi for TEST_FILE in ${FILES}; do rm -f "${TEST_FILE}" >/dev/null 2>&1 done if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then rm -Rf "$STF_PATH" fi } trap cleanup EXIT # # Attempt to remove all testpools (testpool.XXX), unopened dm devices, # loopback devices, and files. This is a useful way to cleanup a previous # test run failure which has left the system in an unknown state. This can # be dangerous and should only be used in a dedicated test environment. # cleanup_all() { TEST_POOLS=$(sudo "$ZPOOL" list -H -o name | grep testpool) if [ "$UNAME" = "FreeBSD" ] ; then TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l) else TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:) fi TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null) msg msg "--- Cleanup ---" msg "Removing pool(s): $(echo "${TEST_POOLS}" | tr '\n' ' ')" for TEST_POOL in $TEST_POOLS; do sudo "$ZPOOL" destroy "${TEST_POOL}" done if [ "$UNAME" != "FreeBSD" ] ; then msg "Removing dm(s): $(sudo "${DMSETUP}" ls | grep loop | tr '\n' ' ')" sudo "${DMSETUP}" remove_all fi msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')" for TEST_LOOPBACK in $TEST_LOOPBACKS; do if [ "$UNAME" = "FreeBSD" ] ; then sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" else sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" fi done msg "Removing files(s): $(echo "${TEST_FILES}" | tr '\n' ' ')" for TEST_FILE in $TEST_FILES; do sudo rm -f "${TEST_FILE}" done } # # Takes a name as the only arguments and looks for the following variations # on that name. If one is found it is returned. # # $RUNFILE_DIR/ # $RUNFILE_DIR/.run # # .run # find_runfile() { NAME=$1 RESULT="" if [ -f "$RUNFILE_DIR/$NAME" ]; then RESULT="$RUNFILE_DIR/$NAME" elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then RESULT="$RUNFILE_DIR/$NAME.run" elif [ -f "$NAME" ]; then RESULT="$NAME" elif [ -f "$NAME.run" ]; then RESULT="$NAME.run" fi echo "$RESULT" } # # Symlink file if it appears under any of the given paths. # create_links() { dir_list="$1" file_list="$2" [ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set" for i in $file_list; do for j in $dir_list; do [ ! -e "$STF_PATH/$i" ] || continue if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then ln -sf "$j/$i" "$STF_PATH/$i" || \ fail "Couldn't link $i" break fi done [ ! -e "$STF_PATH/$i" ] && \ STF_MISSING_BIN="$STF_MISSING_BIN $i" done STF_MISSING_BIN=${STF_MISSING_BIN# } } # # Constrain the path to limit the available binaries to a known set. # When running in-tree a top level ./bin/ directory is created for # convenience, otherwise a temporary directory is used. # constrain_path() { . "$STF_SUITE/include/commands.cfg" # On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils # install to /usr/local/sbin. To avoid testing the wrong utils we # need /usr/local to come before / in the path search order. SYSTEM_DIRS="/usr/local/bin /usr/local/sbin" SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR" if [ "$INTREE" = "yes" ]; then # Constrained path set to ./zfs/bin/ STF_PATH="$BIN_DIR" STF_PATH_REMOVE="no" STF_MISSING_BIN="" if [ ! -d "$STF_PATH" ]; then mkdir "$STF_PATH" chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH" fi # Special case links for standard zfs utilities DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \ ! -name .libs \) -print | tr '\n' ' ')" create_links "$DIRS" "$ZFS_FILES" # Special case links for zfs test suite utilities DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \ ! -name .libs \) -print | tr '\n' ' ')" create_links "$DIRS" "$ZFSTEST_FILES" else # Constrained path set to /var/tmp/constrained_path.* SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX} STF_PATH=$(mktemp -d "$SYSTEMDIR") STF_PATH_REMOVE="yes" STF_MISSING_BIN="" chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH" # Special case links for standard zfs utilities create_links "$SYSTEM_DIRS" "$ZFS_FILES" # Special case links for zfs test suite utilities create_links "$STF_SUITE/bin" "$ZFSTEST_FILES" fi # Standard system utilities SYSTEM_FILES="$SYSTEM_FILES_COMMON" if [ "$UNAME" = "FreeBSD" ] ; then SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD" else SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX" fi create_links "$SYSTEM_DIRS" "$SYSTEM_FILES" # Exceptions ln -fs "$STF_PATH/awk" "$STF_PATH/nawk" if [ "$UNAME" = "Linux" ] ; then ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck" ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs" ln -fs "$STF_PATH/gzip" "$STF_PATH/compress" ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress" ln -fs "$STF_PATH/exportfs" "$STF_PATH/share" ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare" elif [ "$UNAME" = "FreeBSD" ] ; then ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh" fi } # # Output a useful usage message. # usage() { cat << EOF USAGE: $0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER] DESCRIPTION: ZFS Test Suite launch script OPTIONS: -h Show this message -v Verbose zfs-tests.sh output -q Quiet test-runner output -x Remove all testpools, dm, lo, and files (unsafe) -k Disable cleanup after test failure -f Use files only, disables block device tests -S Enable stack tracer (negative performance impact) -c Only create and populate constrained path -R Automatically rerun failing tests -n NFSFILE Use the nfsfile to determine the NFS configuration -I NUM Number of iterations -d DIR Use DIR for files and loopback devices -s SIZE Use vdevs of SIZE (default: 4G) -r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES}) -t PATH Run single test at PATH relative to test suite -T TAGS Comma separated list of tags (default: 'functional') -u USER Run single test as USER (default: root) EXAMPLES: # Run the default (linux) suite of tests and output the configuration used. $0 -v # Run a smaller suite of tests designed to run more quickly. $0 -r linux-fast # Run a single test $0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh # Cleanup a previous run of the test suite prior to testing, run the # default (linux) suite of tests and perform no cleanup on exit. $0 -x EOF } while getopts 'hvqxkfScRn:d:s:r:?t:T:u:I:' OPTION; do case $OPTION in h) usage exit 1 ;; v) VERBOSE="yes" ;; q) QUIET="yes" ;; x) CLEANUPALL="yes" ;; k) CLEANUP="no" ;; f) LOOPBACK="no" ;; S) STACK_TRACER="yes" ;; c) constrain_path exit ;; R) RERUN="yes" ;; n) nfsfile=$OPTARG [ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile" export NFS=1 . "$nfsfile" ;; d) FILEDIR="$OPTARG" ;; I) ITERATIONS="$OPTARG" if [ "$ITERATIONS" -le 0 ]; then fail "Iterations must be greater than 0." fi ;; s) FILESIZE="$OPTARG" ;; r) RUNFILES="$OPTARG" ;; t) if [ -n "$SINGLETEST" ]; then fail "-t can only be provided once." fi SINGLETEST="$OPTARG" ;; T) TAGS="$OPTARG" ;; u) SINGLETESTUSER="$OPTARG" ;; ?) usage exit ;; + *) + ;; esac done shift $((OPTIND-1)) FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"} LOOPBACKS=${LOOPBACKS:-""} if [ -n "$SINGLETEST" ]; then if [ -n "$TAGS" ]; then fail "-t and -T are mutually exclusive." fi RUNFILE_DIR="/var/tmp" RUNFILES="zfs-tests.$$.run" SINGLEQUIET="False" if [ -n "$QUIET" ]; then SINGLEQUIET="True" fi - cat >$RUNFILE_DIR/$RUNFILES << EOF + cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF [DEFAULT] pre = quiet = $SINGLEQUIET pre_user = root user = $SINGLETESTUSER timeout = 600 post_user = root post = outputdir = /var/tmp/test_results EOF SINGLETESTDIR=$(dirname "$SINGLETEST") SINGLETESTFILE=$(basename "$SINGLETEST") SETUPSCRIPT= CLEANUPSCRIPT= if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then SETUPSCRIPT="setup" fi if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then CLEANUPSCRIPT="cleanup" fi - cat >>$RUNFILE_DIR/$RUNFILES << EOF + cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF [$SINGLETESTDIR] tests = ['$SINGLETESTFILE'] pre = $SETUPSCRIPT post = $CLEANUPSCRIPT tags = ['functional'] EOF fi # # Use default tag if none was specified # TAGS=${TAGS:='functional'} # # Attempt to locate the runfiles describing the test workload. # R="" IFS=, for RUNFILE in $RUNFILES; do if [ -n "$RUNFILE" ]; then SAVED_RUNFILE="$RUNFILE" RUNFILE=$(find_runfile "$RUNFILE") [ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE" R="$R,$RUNFILE" fi if [ ! -r "$RUNFILE" ]; then fail "Cannot read runfile: $RUNFILE" fi done unset IFS RUNFILES=${R#,} # # This script should not be run as root. Instead the test user, which may # be a normal user account, needs to be configured such that it can # run commands via sudo passwordlessly. # if [ "$(id -u)" = "0" ]; then fail "This script must not be run as root." fi if [ "$(sudo whoami)" != "root" ]; then fail "Passwordless sudo access required." fi # # Constrain the available binaries to a known set. # constrain_path # # Check if ksh exists # if [ "$UNAME" = "FreeBSD" ]; then sudo ln -fs /usr/local/bin/ksh93 /bin/ksh fi [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh." [ -e "$STF_SUITE/include/default.cfg" ] || fail \ "Missing $STF_SUITE/include/default.cfg file." # # Verify the ZFS module stack is loaded. # if [ "$STACK_TRACER" = "yes" ]; then sudo "${ZFS_SH}" -S >/dev/null 2>&1 else sudo "${ZFS_SH}" >/dev/null 2>&1 fi # # Attempt to cleanup all previous state for a new test run. # if [ "$CLEANUPALL" = "yes" ]; then cleanup_all fi # # By default preserve any existing pools # NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from # space-delimited to newline-delimited. # if [ -z "${KEEP}" ]; then KEEP="$(sudo "$ZPOOL" list -H -o name)" if [ -z "${KEEP}" ]; then KEEP="rpool" fi else KEEP="$(echo "$KEEP" | tr '[:blank:]' '\n')" fi # # NOTE: The following environment variables are undocumented # and should be used for testing purposes only: # # __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists # __ZFS_POOL_RESTRICT - iterate only over the pools it lists # # See libzfs/libzfs_config.c for more information. # if [ "$UNAME" = "FreeBSD" ] ; then __ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')" else __ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')" fi . "$STF_SUITE/include/default.cfg" # # No DISKS have been provided so a basic file or loopback based devices # must be created for the test suite to use. # if [ -z "${DISKS}" ]; then # # If this is a performance run, prevent accidental use of # loopback devices. # [ "$TAGS" = "perf" ] && fail "Running perf tests without disks." # # Create sparse files for the test suite. These may be used # directory or have loopback devices layered on them. # for TEST_FILE in ${FILES}; do [ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}" truncate -s "${FILESIZE}" "${TEST_FILE}" || fail "Failed creating: ${TEST_FILE} ($?)" done # # If requested setup loopback devices backed by the sparse files. # if [ "$LOOPBACK" = "yes" ]; then test -x "$LOSETUP" || fail "$LOSETUP utility must be installed" for TEST_FILE in ${FILES}; do if [ "$UNAME" = "FreeBSD" ] ; then MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}") if [ -z "$MDDEVICE" ] ; then fail "Failed: ${TEST_FILE} -> loopback" fi DISKS="$DISKS $MDDEVICE" LOOPBACKS="$LOOPBACKS $MDDEVICE" else TEST_LOOPBACK=$(sudo "${LOSETUP}" -f) sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" || fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}" BASELOOPBACK="${TEST_LOOPBACK##*/}" DISKS="$DISKS $BASELOOPBACK" LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK" fi done DISKS=${DISKS# } LOOPBACKS=${LOOPBACKS# } else DISKS="$FILES" fi fi # # It may be desirable to test with fewer disks than the default when running # the performance tests, but the functional tests require at least three. # NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}') if [ "$TAGS" != "perf" ]; then [ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)" fi # # Disable SELinux until the ZFS Test Suite has been updated accordingly. # if [ -x "$STF_PATH/setenforce" ]; then sudo setenforce permissive >/dev/null 2>&1 fi # # Enable internal ZFS debug log and clear it. # if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable" sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg" fi msg msg "--- Configuration ---" msg "Runfiles: $RUNFILES" msg "STF_TOOLS: $STF_TOOLS" msg "STF_SUITE: $STF_SUITE" msg "STF_PATH: $STF_PATH" msg "FILEDIR: $FILEDIR" msg "FILES: $FILES" msg "LOOPBACKS: $LOOPBACKS" msg "DISKS: $DISKS" msg "NUM_DISKS: $NUM_DISKS" msg "FILESIZE: $FILESIZE" msg "ITERATIONS: $ITERATIONS" msg "TAGS: $TAGS" msg "STACK_TRACER: $STACK_TRACER" msg "Keep pool(s): $KEEP" msg "Missing util(s): $STF_MISSING_BIN" msg "" export STF_TOOLS export STF_SUITE export STF_PATH export DISKS export FILEDIR export KEEP export __ZFS_POOL_EXCLUDE export TESTFAIL_CALLBACKS export PATH=$STF_PATH if [ "$UNAME" = "FreeBSD" ] ; then mkdir -p "$FILEDIR" || true RESULTS_FILE=$(mktemp -u "${FILEDIR}/zts-results.XXXXXX") REPORT_FILE=$(mktemp -u "${FILEDIR}/zts-report.XXXXXX") else RESULTS_FILE=$(mktemp -u -t zts-results.XXXXXX -p "$FILEDIR") REPORT_FILE=$(mktemp -u -t zts-report.XXXXXX -p "$FILEDIR") fi # # Run all the tests as specified. # msg "${TEST_RUNNER} ${QUIET:+-q}" \ "-c \"${RUNFILES}\"" \ "-T \"${TAGS}\"" \ "-i \"${STF_SUITE}\"" \ "-I \"${ITERATIONS}\"" ${TEST_RUNNER} ${QUIET:+-q} \ -c "${RUNFILES}" \ -T "${TAGS}" \ -i "${STF_SUITE}" \ -I "${ITERATIONS}" \ 2>&1 | tee "$RESULTS_FILE" # # Analyze the results. # ${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE" RESULT=$? if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then MAYBES="$($ZTS_REPORT --list-maybes)" TEMP_RESULTS_FILE=$(mktemp -u -t zts-results-tmp.XXXXX -p "$FILEDIR") TEST_LIST=$(mktemp -u -t test-list.XXXXX -p "$FILEDIR") grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE" for test_name in $MAYBES; do grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST" done ${TEST_RUNNER} ${QUIET:+-q} \ -c "${RUNFILES}" \ -T "${TAGS}" \ -i "${STF_SUITE}" \ -I "${ITERATIONS}" \ -l "${TEST_LIST}" \ 2>&1 | tee "$RESULTS_FILE" # # Analyze the results. # ${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE" RESULT=$? fi cat "$REPORT_FILE" RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE") if [ -d "$RESULTS_DIR" ]; then cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results" fi rm -f "$RESULTS_FILE" "$REPORT_FILE" if [ -n "$SINGLETEST" ]; then rm -f "$RUNFILES" >/dev/null 2>&1 fi -exit ${RESULT} +exit "${RESULT}" diff --git a/scripts/zfs.sh b/scripts/zfs.sh index 940c83ffa28f..0561092a089f 100755 --- a/scripts/zfs.sh +++ b/scripts/zfs.sh @@ -1,291 +1,301 @@ #!/bin/sh # # A simple script to load/unload the ZFS module stack. # BASE_DIR=$(dirname "$0") SCRIPT_COMMON=common.sh if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then . "${BASE_DIR}/${SCRIPT_COMMON}" else echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 fi PROG=zfs.sh VERBOSE="no" UNLOAD="no" LOAD="yes" STACK_TRACER="no" ZED_PIDFILE=${ZED_PIDFILE:-/var/run/zed.pid} LDMOD=${LDMOD:-/sbin/modprobe} KMOD_ZLIB_DEFLATE=${KMOD_ZLIB_DEFLATE:-zlib_deflate} KMOD_ZLIB_INFLATE=${KMOD_ZLIB_INFLATE:-zlib_inflate} KMOD_SPL=${KMOD_SPL:-spl} KMOD_ZAVL=${KMOD_ZAVL:-zavl} KMOD_ZNVPAIR=${KMOD_ZNVPAIR:-znvpair} KMOD_ZUNICODE=${KMOD_ZUNICODE:-zunicode} KMOD_ZCOMMON=${KMOD_ZCOMMON:-zcommon} KMOD_ZLUA=${KMOD_ZLUA:-zlua} KMOD_ICP=${KMOD_ICP:-icp} KMOD_ZFS=${KMOD_ZFS:-zfs} KMOD_FREEBSD=${KMOD_FREEBSD:-openzfs} KMOD_ZZSTD=${KMOD_ZZSTD:-zzstd} usage() { cat << EOF USAGE: $0 [hvudS] [module-options] DESCRIPTION: Load/unload the ZFS module stack. OPTIONS: -h Show this message -v Verbose -r Reload modules -u Unload modules -S Enable kernel stack tracer EOF } while getopts 'hvruS' OPTION; do case $OPTION in h) usage exit 1 ;; v) VERBOSE="yes" ;; r) UNLOAD="yes" LOAD="yes" ;; u) UNLOAD="yes" LOAD="no" ;; S) STACK_TRACER="yes" ;; ?) usage exit ;; + *) + ;; esac done kill_zed() { if [ -f "$ZED_PIDFILE" ]; then PID=$(cat "$ZED_PIDFILE") kill "$PID" fi } check_modules_linux() { LOADED_MODULES="" MISSING_MODULES="" for KMOD in $KMOD_SPL $KMOD_ZAVL $KMOD_ZNVPAIR $KMOD_ZUNICODE $KMOD_ZCOMMON \ $KMOD_ZLUA $KMOD_ZZSTD $KMOD_ICP $KMOD_ZFS; do NAME="${KMOD##*/}" NAME="${NAME%.ko}" if lsmod | grep -E -q "^${NAME}"; then LOADED_MODULES="$LOADED_MODULES\t$NAME\n" fi if ! modinfo "$KMOD" >/dev/null 2>&1; then MISSING_MODULES="$MISSING_MODULES\t${KMOD}\n" fi done if [ -n "$LOADED_MODULES" ]; then printf "Unload the kernel modules by running '%s -u':\n" "$PROG" printf "%b" "$LOADED_MODULES" exit 1 fi if [ -n "$MISSING_MODULES" ]; then printf "The following kernel modules can not be found:\n" printf "%b" "$MISSING_MODULES" exit 1 fi return 0 } load_module_linux() { KMOD=$1 FILE=$(modinfo "$KMOD" | awk '/^filename:/ {print $2}') VERSION=$(modinfo "$KMOD" | awk '/^version:/ {print $2}') if [ "$VERBOSE" = "yes" ]; then echo "Loading: $FILE ($VERSION)" fi if ! $LDMOD "$KMOD" >/dev/null 2>&1; then echo "Failed to load $KMOD" return 1 fi return 0 } load_modules_freebsd() { kldload "$KMOD_FREEBSD" || return 1 if [ "$VERBOSE" = "yes" ]; then echo "Successfully loaded ZFS module stack" fi return 0 } load_modules_linux() { mkdir -p /etc/zfs if modinfo "$KMOD_ZLIB_DEFLATE" >/dev/null 2>&1; then modprobe "$KMOD_ZLIB_DEFLATE" >/dev/null 2>&1 fi if modinfo "$KMOD_ZLIB_INFLATE">/dev/null 2>&1; then modprobe "$KMOD_ZLIB_INFLATE" >/dev/null 2>&1 fi for KMOD in $KMOD_SPL $KMOD_ZAVL $KMOD_ZNVPAIR \ $KMOD_ZUNICODE $KMOD_ZCOMMON $KMOD_ZLUA $KMOD_ZZSTD \ $KMOD_ICP $KMOD_ZFS; do load_module_linux "$KMOD" || return 1 done if [ "$VERBOSE" = "yes" ]; then echo "Successfully loaded ZFS module stack" fi return 0 } unload_module_linux() { KMOD=$1 NAME="${KMOD##*/}" NAME="${NAME%.ko}" FILE=$(modinfo "$KMOD" | awk '/^filename:/ {print $2}') VERSION=$(modinfo "$KMOD" | awk '/^version:/ {print $2}') if [ "$VERBOSE" = "yes" ]; then echo "Unloading: $KMOD ($VERSION)" fi rmmod "$NAME" || echo "Failed to unload $NAME" return 0 } unload_modules_freebsd() { kldunload "$KMOD_FREEBSD" || echo "Failed to unload $KMOD_FREEBSD" if [ "$VERBOSE" = "yes" ]; then echo "Successfully unloaded ZFS module stack" fi return 0 } unload_modules_linux() { for KMOD in $KMOD_ZFS $KMOD_ICP $KMOD_ZZSTD $KMOD_ZLUA $KMOD_ZCOMMON \ $KMOD_ZUNICODE $KMOD_ZNVPAIR $KMOD_ZAVL $KMOD_SPL; do NAME="${KMOD##*/}" NAME="${NAME%.ko}" USE_COUNT=$(lsmod | awk '/^'"${NAME}"'/ {print $3}') if [ "$USE_COUNT" = "0" ] ; then unload_module_linux "$KMOD" || return 1 elif [ "$USE_COUNT" != "" ] ; then echo "Module ${NAME} is still in use!" return 1 fi done if modinfo "$KMOD_ZLIB_DEFLATE" >/dev/null 2>&1; then modprobe -r "$KMOD_ZLIB_DEFLATE" >/dev/null 2>&1 fi if modinfo "$KMOD_ZLIB_INFLATE">/dev/null 2>&1; then modprobe -r "$KMOD_ZLIB_INFLATE" >/dev/null 2>&1 fi if [ "$VERBOSE" = "yes" ]; then echo "Successfully unloaded ZFS module stack" fi return 0 } stack_clear_linux() { STACK_MAX_SIZE=/sys/kernel/debug/tracing/stack_max_size STACK_TRACER_ENABLED=/proc/sys/kernel/stack_tracer_enabled if [ "$STACK_TRACER" = "yes" ] && [ -e "$STACK_MAX_SIZE" ]; then echo 1 >"$STACK_TRACER_ENABLED" echo 0 >"$STACK_MAX_SIZE" fi } stack_check_linux() { STACK_MAX_SIZE=/sys/kernel/debug/tracing/stack_max_size STACK_TRACE=/sys/kernel/debug/tracing/stack_trace STACK_LIMIT=15362 if [ -e "$STACK_MAX_SIZE" ]; then STACK_SIZE=$(cat "$STACK_MAX_SIZE") if [ "$STACK_SIZE" -ge "$STACK_LIMIT" ]; then echo echo "Warning: max stack size $STACK_SIZE bytes" cat "$STACK_TRACE" fi fi } if [ "$(id -u)" != 0 ]; then echo "Must run as root" exit 1 fi UNAME=$(uname -s) if [ "$UNLOAD" = "yes" ]; then kill_zed umount -t zfs -a case $UNAME in FreeBSD) unload_modules_freebsd ;; Linux) stack_check_linux unload_modules_linux ;; + *) + echo "unknown system: $UNAME" >&2 + exit 1 + ;; esac fi if [ "$LOAD" = "yes" ]; then case $UNAME in FreeBSD) load_modules_freebsd ;; Linux) stack_clear_linux check_modules_linux load_modules_linux "$@" udevadm trigger udevadm settle ;; + *) + echo "unknown system: $UNAME" >&2 + exit 1 + ;; esac fi exit 0 diff --git a/scripts/zimport.sh b/scripts/zimport.sh index 03c766cf36c2..14d2813ce268 100755 --- a/scripts/zimport.sh +++ b/scripts/zimport.sh @@ -1,512 +1,512 @@ #!/usr/bin/env bash # # Verify that an assortment of known good reference pools can be imported # using different versions of OpenZFS code. # # By default references pools for the major ZFS implementation will be # checked against the most recent OpenZFS tags and the master development branch. # Alternate tags or branches may be verified with the '-s option. # Passing the keyword "installed" will instruct the script to test whatever # version is installed. # # Preferentially a reference pool is used for all tests. However, if one # does not exist and the pool-tag matches one of the src-tags then a new # reference pool will be created using binaries from that source build. # This is particularly useful when you need to test your changes before # opening a pull request. The keyword 'all' can be used as short hand # refer to all available reference pools. # # New reference pools may be added by placing a bzip2 compressed tarball # of the pool in the scripts/zfs-images directory and then passing # the -p option. To increase the test coverage reference pools # should be collected for all the major ZFS implementations. Having these # pools easily available is also helpful to the developers. # # Care should be taken to run these tests with a kernel supported by all # the listed tags. Otherwise build failure will cause false positives. # # # EXAMPLES: # # The following example will verify the zfs-0.6.2 tag, the master branch, # and the installed zfs version can correctly import the listed pools. # Note there is no reference pool available for master and installed but # because binaries are available one is automatically constructed. The # working directory is also preserved between runs (-k) preventing the # need to rebuild from source for multiple runs. # # zimport.sh -k -f /var/tmp/zimport \ # -s "zfs-0.6.2 master installed" \ # -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed" # # ------------------------ OpenZFS Source Versions ---------------- # zfs-0.6.2 master 0.6.2-175_g36eb554 # ----------------------------------------------------------------- # Clone ZFS Local Local Skip # Build ZFS Pass Pass Skip # ----------------------------------------------------------------- # zevo-1.1.1 Pass Pass Pass # zol-0.6.2 Pass Pass Pass # zol-0.6.2-173 Fail Pass Pass # master Pass Pass Pass # installed Pass Pass Pass # BASE_DIR=$(dirname "$0") SCRIPT_COMMON=common.sh if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then . "${BASE_DIR}/${SCRIPT_COMMON}" else echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 fi PROG=zimport.sh SRC_TAGS="zfs-0.6.5.11 master" POOL_TAGS="all master" POOL_CREATE_OPTIONS= TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX) KEEP="no" VERBOSE="no" COLOR="yes" REPO="https://github.com/openzfs" -IMAGES_DIR="$SCRIPTDIR/zfs-images/" +IMAGES_DIR="${BASE_DIR}/zfs-images/" IMAGES_TAR="https://github.com/openzfs/zfs-images/tarball/master" ERROR=0 CONFIG_LOG="configure.log" CONFIG_OPTIONS=${CONFIG_OPTIONS:-""} MAKE_LOG="make.log" MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"} COLOR_GREEN="\033[0;32m" COLOR_RED="\033[0;31m" COLOR_BROWN="\033[0;33m" COLOR_RESET="\033[0m" usage() { cat << EOF USAGE: zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag] [-f path] [-o options] DESCRIPTION: ZPOOL import verification tests OPTIONS: -h Show this message -v Verbose -c No color -k Keep temporary directory -r Source repository ($REPO) -s ... Verify OpenZFS versions with the listed tags -i Pool image directory -p ... Verify pools created with the listed tags -f Temporary directory to use -o Additional options to pass to 'zpool create' EOF } while getopts 'hvckr:s:i:p:f:o:?' OPTION; do case $OPTION in h) usage exit 1 ;; v) VERBOSE="yes" ;; c) COLOR="no" ;; k) KEEP="yes" ;; r) REPO="$OPTARG" ;; s) SRC_TAGS="$OPTARG" ;; i) IMAGES_DIR="$OPTARG" ;; p) POOL_TAGS="$OPTARG" ;; f) TEST_DIR="$OPTARG" ;; o) POOL_CREATE_OPTIONS="$OPTARG" ;; - ?) + *) usage exit 1 ;; esac done # # Verify the module start is not loaded # if lsmod | grep zfs >/dev/null; then echo "ZFS modules must be unloaded" exit 1 fi # # Create a random directory tree of files and sub-directories to # to act as a copy source for the various regression tests. # populate() { local ROOT=$1 local MAX_DIR_SIZE=$2 local MAX_FILE_SIZE=$3 mkdir -p "$ROOT"/{a,b,c,d,e,f,g}/{h,i} DIRS=$(find "$ROOT") for DIR in $DIRS; do COUNT=$((RANDOM % MAX_DIR_SIZE)) - for _ in $(seq $COUNT); do + for _ in $(seq "$COUNT"); do FILE=$(mktemp -p "$DIR") SIZE=$((RANDOM % MAX_FILE_SIZE)) dd if=/dev/urandom of="$FILE" bs=1k \ count="$SIZE" &>/dev/null done done return 0 } SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX) trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT populate "$SRC_DIR" 10 100 SRC_DIR="$TEST_DIR/src" SRC_DIR_ZFS="$SRC_DIR/zfs" if [ "$COLOR" = "no" ]; then COLOR_GREEN="" COLOR_BROWN="" COLOR_RED="" COLOR_RESET="" fi pass_nonewline() { echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t" } skip_nonewline() { echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t" } fail_nonewline() { echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t" } # # Log a failure message, cleanup, and return an error. # fail() { echo -e "$PROG: $1" >&2 $ZFS_SH -u >/dev/null 2>&1 exit 1 } # # Set several helper variables which are derived from a source tag. # # ZFS_TAG - The passed zfs-x.y.z tag # ZFS_DIR - The zfs directory name # ZFS_URL - The zfs github URL to fetch the tarball # src_set_vars() { local TAG=$1 ZFS_TAG="$TAG" ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG" ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG" if [ "$TAG" = "installed" ]; then ZPOOL_CMD=$(command -v zpool) ZFS_CMD=$(command -v zfs) ZFS_SH="/usr/share/zfs/zfs.sh" else ZPOOL_CMD="./cmd/zpool/zpool" ZFS_CMD="./cmd/zfs/zfs" ZFS_SH="./scripts/zfs.sh" fi } # # Set several helper variables which are derived from a pool name such # as zol-0.6.x, zevo-1.1.1, etc. These refer to example pools from various # ZFS implementations which are used to verify compatibility. # # POOL_TAG - The example pools name in scripts/zfs-images/. # POOL_BZIP - The full path to the example bzip2 compressed pool. # POOL_DIR - The top level test path for this pool. # POOL_DIR_PRISTINE - The directory containing a pristine version of the pool. # POOL_DIR_COPY - The directory containing a working copy of the pool. # POOL_DIR_SRC - Location of a source build if it exists for this pool. # pool_set_vars() { local TAG=$1 POOL_TAG=$TAG POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2 POOL_DIR=$TEST_DIR/pools/$POOL_TAG POOL_DIR_PRISTINE=$POOL_DIR/pristine POOL_DIR_COPY=$POOL_DIR/copy POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}" } # # Construct a non-trivial pool given a specific version of the source. More # interesting pools provide better test coverage so this function should # extended as needed to create more realistic pools. # pool_create() { pool_set_vars "$1" src_set_vars "$1" if [ "$POOL_TAG" != "installed" ]; then cd "$POOL_DIR_SRC" || fail "Failed 'cd $POOL_DIR_SRC'" fi $ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || \ fail "Failed to load kmods" # Create a file vdev RAIDZ pool. truncate -s 1G \ "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ fail "Failed 'truncate -s 1G ...'" # shellcheck disable=SC2086 $ZPOOL_CMD create $POOL_CREATE_OPTIONS "$POOL_TAG" raidz \ "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ fail "Failed '$ZPOOL_CMD create $POOL_CREATE_OPTIONS $POOL_TAG ...'" # Create a pool/fs filesystem with some random contents. $ZFS_CMD create "$POOL_TAG/fs" || \ fail "Failed '$ZFS_CMD create $POOL_TAG/fs'" populate "/$POOL_TAG/fs/" 10 100 # Snapshot that filesystem, clone it, remove the files/dirs, # replace them with new files/dirs. $ZFS_CMD snap "$POOL_TAG/fs@snap" || \ fail "Failed '$ZFS_CMD snap $POOL_TAG/fs@snap'" $ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" || \ fail "Failed '$ZFS_CMD clone $POOL_TAG/fs@snap $POOL_TAG/clone'" # shellcheck disable=SC2086 rm -Rf /$POOL_TAG/clone/* populate "/$POOL_TAG/clone/" 10 100 # Scrub the pool, delay slightly, then export it. It is now # somewhat interesting for testing purposes. $ZPOOL_CMD scrub "$POOL_TAG" || \ fail "Failed '$ZPOOL_CMD scrub $POOL_TAG'" sleep 10 $ZPOOL_CMD export "$POOL_TAG" || \ fail "Failed '$ZPOOL_CMD export $POOL_TAG'" $ZFS_SH -u || fail "Failed to unload kmods" } # If the zfs-images directory doesn't exist fetch a copy from Github then # cache it in the $TEST_DIR and update $IMAGES_DIR. if [ ! -d "$IMAGES_DIR" ]; then IMAGES_DIR="$TEST_DIR/zfs-images" mkdir -p "$IMAGES_DIR" curl -sL "$IMAGES_TAR" | \ tar -xz -C "$IMAGES_DIR" --strip-components=1 || \ fail "Failed to download pool images" fi # Given the available images in the zfs-images directory substitute the # list of available images for the reserved keyword 'all'. for TAG in $POOL_TAGS; do if [ "$TAG" = "all" ]; then ALL_TAGS=$(echo "$IMAGES_DIR"/*.tar.bz2 | \ sed "s|$IMAGES_DIR/||g;s|.tar.bz2||g") NEW_TAGS="$NEW_TAGS $ALL_TAGS" else NEW_TAGS="$NEW_TAGS $TAG" fi done POOL_TAGS="$NEW_TAGS" if [ "$VERBOSE" = "yes" ]; then echo "---------------------------- Options ----------------------------" echo "VERBOSE=$VERBOSE" echo "KEEP=$KEEP" echo "REPO=$REPO" echo "SRC_TAGS=$SRC_TAGS" echo "POOL_TAGS=$POOL_TAGS" echo "PATH=$TEST_DIR" echo "POOL_CREATE_OPTIONS=$POOL_CREATE_OPTIONS" echo fi if [ ! -d "$TEST_DIR" ]; then mkdir -p "$TEST_DIR" fi if [ ! -d "$SRC_DIR" ]; then mkdir -p "$SRC_DIR" fi # Print a header for all tags which are being tested. echo "------------------------ OpenZFS Source Versions ----------------" printf "%-16s" " " for TAG in $SRC_TAGS; do src_set_vars "$TAG" if [ "$TAG" = "installed" ]; then ZFS_VERSION=$(modinfo zfs | awk '/version:/ { print $2; exit }') if [ -n "$ZFS_VERSION" ]; then printf "%-16s" "$ZFS_VERSION" else fail "ZFS is not installed" fi else printf "%-16s" "$TAG" fi done echo -e "\n-----------------------------------------------------------------" # # Attempt to generate the tarball from your local git repository, if that # fails then attempt to download the tarball from Github. # printf "%-16s" "Clone ZFS" for TAG in $SRC_TAGS; do src_set_vars "$TAG" if [ -d "$ZFS_DIR" ]; then skip_nonewline elif [ "$ZFS_TAG" = "installed" ]; then skip_nonewline else cd "$SRC_DIR" || fail "Failed 'cd $SRC_DIR'" if [ ! -d "$SRC_DIR_ZFS" ]; then mkdir -p "$SRC_DIR_ZFS" fi git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \ -o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/null || \ rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" if [ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]; then tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS" rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t" else mkdir -p "$ZFS_DIR" || fail "Failed to create $ZFS_DIR" curl -sL "$ZFS_URL" | tar -xz -C "$ZFS_DIR" \ --strip-components=1 || \ fail "Failed to download $ZFS_URL" echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t" fi fi done printf "\n" # Build the listed tags printf "%-16s" "Build ZFS" for TAG in $SRC_TAGS; do src_set_vars "$TAG" if [ -f "$ZFS_DIR/module/zfs/zfs.ko" ]; then skip_nonewline elif [ "$ZFS_TAG" = "installed" ]; then skip_nonewline else cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" make distclean &>/dev/null ./autogen.sh >>"$CONFIG_LOG" 2>&1 || \ fail "Failed ZFS 'autogen.sh'" # shellcheck disable=SC2086 ./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 || \ fail "Failed ZFS 'configure $CONFIG_OPTIONS'" # shellcheck disable=SC2086 make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || \ fail "Failed ZFS 'make $MAKE_OPTIONS'" pass_nonewline fi done printf "\n" echo "-----------------------------------------------------------------" # Either create a new pool using 'zpool create', or alternately restore an # existing pool from another ZFS implementation for compatibility testing. for TAG in $POOL_TAGS; do pool_set_vars "$TAG" SKIP=0 printf "%-16s" "$POOL_TAG" rm -Rf "$POOL_DIR" mkdir -p "$POOL_DIR_PRISTINE" # Use the existing compressed image if available. if [ -f "$POOL_BZIP" ]; then tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \ --strip-components=1 || \ fail "Failed 'tar -xjf $POOL_BZIP" # Use the installed version to create the pool. elif [ "$TAG" = "installed" ]; then pool_create "$TAG" # A source build is available to create the pool. elif [ -d "$POOL_DIR_SRC" ]; then pool_create "$TAG" else SKIP=1 fi # Verify 'zpool import' works for all listed source versions. for SRC_TAG in $SRC_TAGS; do - if [ $SKIP -eq 1 ]; then + if [ "$SKIP" -eq 1 ]; then skip_nonewline continue fi src_set_vars "$SRC_TAG" if [ "$SRC_TAG" != "installed" ]; then cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" fi $ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY" cp -a --sparse=always "$POOL_DIR_PRISTINE" \ "$POOL_DIR_COPY" || \ fail "Failed to copy $POOL_DIR_PRISTINE to $POOL_DIR_COPY" POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \ awk '/pool:/ { print $2; exit }') if ! $ZPOOL_CMD import -N -d "$POOL_DIR_COPY" "$POOL_NAME" &>/dev/null; then fail_nonewline ERROR=1 else $ZPOOL_CMD export "$POOL_NAME" || \ fail "Failed to export pool" pass_nonewline fi rm -Rf "$POOL_DIR_COPY" $ZFS_SH -u || fail "Failed to unload kmods" done printf "\n" done if [ "$KEEP" = "no" ]; then rm -Rf "$TEST_DIR" fi -exit $ERROR +exit "$ERROR" diff --git a/scripts/zloop.sh b/scripts/zloop.sh index 4a572ebab1fc..1a512657be69 100755 --- a/scripts/zloop.sh +++ b/scripts/zloop.sh @@ -1,339 +1,341 @@ #!/usr/bin/env bash # # CDDL HEADER START # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # CDDL HEADER END # # # Copyright (c) 2015 by Delphix. All rights reserved. # Copyright (C) 2016 Lawrence Livermore National Security, LLC. # Copyright (c) 2017, Intel Corporation. # BASE_DIR=$(dirname "$0") SCRIPT_COMMON=common.sh if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then . "${BASE_DIR}/${SCRIPT_COMMON}" else echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 fi # shellcheck disable=SC2034 PROG=zloop.sh GDB=${GDB:-gdb} DEFAULTWORKDIR=/var/tmp DEFAULTCOREDIR=/var/tmp/zloop function usage { cat >&2 <] [-f ] [-m ] [-s ] [-t ] [-I ] [-- [extra ztest parameters]] This script runs ztest repeatedly with randomized arguments. If a crash is encountered, the ztest logs, any associated vdev files, and core file (if one exists) are moved to the output directory ($DEFAULTCOREDIR by default). Any options after the -- end-of-options marker will be passed to ztest. Options: -c Specify a core dump directory to use. -f Specify working directory for ztest vdev files. -h Print this help message. -l Create 'ztest.core.N' symlink to core directory. -m Max number of core dumps to allow before exiting. -s Size of vdev devices. -t Total time to loop for, in seconds. If not provided, zloop runs forever. -I Max number of iterations to loop before exiting. EOF } function or_die { # shellcheck disable=SC2068 if ! $@; then echo "Command failed: $*" exit 1 fi } case $(uname) in FreeBSD) coreglob="z*.core" ;; Linux) # core file helpers origcorepattern="$(cat /proc/sys/kernel/core_pattern)" coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" if [[ $coreglob = "*" ]]; then echo "Setting core file pattern..." echo "core" > /proc/sys/kernel/core_pattern coreglob="$(grep -E -o '^([^|%[:space:]]*)' \ /proc/sys/kernel/core_pattern)*" fi ;; *) exit 1 ;; esac function core_file { # shellcheck disable=SC2012,SC2086 ls -tr1 $coreglob 2>/dev/null | head -1 } function core_prog { + # shellcheck disable=SC2154 prog=$ZTEST core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \ tr \' ' ') if [[ "$core_id" == *"zdb "* ]]; then + # shellcheck disable=SC2154 prog=$ZDB fi printf "%s" "$prog" } function store_core { core="$(core_file)" if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then df -h "$workdir" >>ztest.out coreid=$(date "+zloop-%y%m%d-%H%M%S") foundcrashes=$((foundcrashes + 1)) # zdb debugging zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest" zdbdebug=$($zdbcmd 2>&1) echo -e "$zdbcmd\n" >>ztest.zdb echo "$zdbdebug" >>ztest.zdb dest=$coredir/$coreid or_die mkdir -p "$dest" or_die mkdir -p "$dest/vdev" if [[ $symlink -ne 0 ]]; then - or_die ln -sf "$dest" ztest.core.$foundcrashes + or_die ln -sf "$dest" "ztest.core.${foundcrashes}" fi echo "*** ztest crash found - moving logs to $dest" or_die mv ztest.history "$dest/" or_die mv ztest.zdb "$dest/" or_die mv ztest.out "$dest/" or_die mv "$workdir/ztest*" "$dest/vdev/" if [[ -e "$workdir/zpool.cache" ]]; then or_die mv "$workdir/zpool.cache" "$dest/vdev/" fi # check for core if [[ -f "$core" ]]; then coreprog=$(core_prog "$core") coredebug=$($GDB --batch --quiet \ -ex "set print thread-events off" \ -ex "printf \"*\n* Backtrace \n*\n\"" \ -ex "bt" \ -ex "printf \"*\n* Libraries \n*\n\"" \ -ex "info sharedlib" \ -ex "printf \"*\n* Threads (full) \n*\n\"" \ -ex "info threads" \ -ex "printf \"*\n* Backtraces \n*\n\"" \ -ex "thread apply all bt" \ -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ -ex "thread apply all bt full" \ -ex "quit" "$coreprog" "$core" 2>&1 | \ grep -v "New LWP") # Dump core + logs to stored directory echo "$coredebug" >>"$dest/ztest.gdb" or_die mv "$core" "$dest/" # Record info in cores logfile echo "*** core @ $coredir/$coreid/$core:" | \ tee -a ztest.cores fi if [[ $coremax -gt 0 ]] && [[ $foundcrashes -ge $coremax ]]; then echo "exiting... max $coremax allowed cores" exit 1 else echo "continuing..." fi fi } # parse arguments # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] coredir=$DEFAULTCOREDIR basedir=$DEFAULTWORKDIR rundir="zloop-run" timeout=0 size="512m" coremax=0 symlink=0 iterations=0 while getopts ":ht:m:I:s:c:f:l" opt; do case $opt in t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;; - I ) [[ $OPTARG ]] && iterations=$OPTARG ;; - s ) [[ $OPTARG ]] && size=$OPTARG ;; - c ) [[ $OPTARG ]] && coredir=$OPTARG ;; - f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; + I ) [[ -n $OPTARG ]] && iterations=$OPTARG ;; + s ) [[ -n $OPTARG ]] && size=$OPTARG ;; + c ) [[ -n $OPTARG ]] && coredir=$OPTARG ;; + f ) [[ -n $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; l ) symlink=1 ;; h ) usage exit 2 ;; * ) echo "Invalid argument: -$OPTARG"; usage exit 1 esac done # pass remaining arguments on to ztest shift $((OPTIND - 1)) # enable core dumps ulimit -c unlimited export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0 if [[ -f "$(core_file)" ]]; then echo -n "There's a core dump here you might want to look at first... " core_file echo exit 1 fi if [[ ! -d $coredir ]]; then echo "core dump directory ($coredir) does not exist, creating it." or_die mkdir -p "$coredir" fi if [[ ! -w $coredir ]]; then echo "core dump directory ($coredir) is not writable." exit 1 fi or_die rm -f ztest.history or_die rm -f ztest.zdb or_die rm -f ztest.cores ztrc=0 # ztest return value foundcrashes=0 # number of crashes found so far starttime=$(date +%s) curtime=$starttime iteration=0 # if no timeout was specified, loop forever. while (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do if (( iterations > 0 )) && (( iteration++ == iterations )); then break fi zopt="-G -VVVVV" # start each run with an empty directory workdir="$basedir/$rundir" or_die rm -rf "$workdir" or_die mkdir "$workdir" # switch between three types of configs # 1/3 basic, 1/3 raidz mix, and 1/3 draid mix choice=$((RANDOM % 3)) # ashift range 9 - 15 align=$(((RANDOM % 2) * 3 + 9)) # randomly use special classes class="special=random" if [[ $choice -eq 0 ]]; then # basic mirror only parity=1 mirrors=2 draid_data=0 draid_spares=0 raid_children=0 vdevs=2 raid_type="raidz" elif [[ $choice -eq 1 ]]; then # fully randomized mirror/raidz (sans dRAID) parity=$(((RANDOM % 3) + 1)) mirrors=$(((RANDOM % 3) * 1)) draid_data=0 draid_spares=0 raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) vdevs=$(((RANDOM % 3) + 3)) raid_type="raidz" else # fully randomized dRAID (sans mirror/raidz) parity=$(((RANDOM % 3) + 1)) mirrors=0 draid_data=$(((RANDOM % 8) + 3)) draid_spares=$(((RANDOM % 2) + parity)) stripe=$((draid_data + parity)) extra=$((draid_spares + (RANDOM % 4))) raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra)) vdevs=$((RANDOM % 3)) raid_type="draid" fi zopt="$zopt -K $raid_type" zopt="$zopt -m $mirrors" zopt="$zopt -r $raid_children" zopt="$zopt -D $draid_data" zopt="$zopt -S $draid_spares" zopt="$zopt -R $parity" zopt="$zopt -v $vdevs" zopt="$zopt -a $align" zopt="$zopt -C $class" zopt="$zopt -s $size" zopt="$zopt -f $workdir" cmd="$ZTEST $zopt $*" desc="$(date '+%m/%d %T') $cmd" echo "$desc" | tee -a ztest.history echo "$desc" >>ztest.out $cmd >>ztest.out 2>&1 ztrc=$? grep -E '===|WARNING' ztest.out >>ztest.history store_core curtime=$(date +%s) done echo "zloop finished, $foundcrashes crashes found" # restore core pattern. case $(uname) in Linux) echo "$origcorepattern" > /proc/sys/kernel/core_pattern ;; *) ;; esac uptime >>ztest.out if [[ $foundcrashes -gt 0 ]]; then exit 1 fi