diff --git a/.github/workflows/zloop.yml b/.github/workflows/zloop.yml index b3679e7f7f20..cf81ad4bcafc 100644 --- a/.github/workflows/zloop.yml +++ b/.github/workflows/zloop.yml @@ -1,67 +1,67 @@ name: zloop on: push: pull_request: jobs: tests: runs-on: ubuntu-latest env: TEST_DIR: /var/tmp/zloop steps: - uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha }} - name: Install dependencies run: | sudo apt-get update sudo apt-get install --yes -qq build-essential autoconf libtool gdb \ git alien fakeroot \ zlib1g-dev uuid-dev libblkid-dev libselinux-dev \ xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \ libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \ libpam0g-dev \ python-dev python-setuptools python-cffi python-packaging \ python3 python3-dev python3-setuptools python3-cffi python3-packaging - name: Autogen.sh run: | sh autogen.sh - name: Configure run: | ./configure --enable-debug --enable-debuginfo - name: Make run: | make --no-print-directory -s pkg-utils pkg-kmod - name: Install run: | sudo dpkg -i *.deb # Update order of directories to search for modules, otherwise # Ubuntu will load kernel-shipped ones. sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf sudo depmod sudo modprobe zfs - name: Tests run: | sudo mkdir -p $TEST_DIR # run for 20 minutes to have a total runner time of 30 minutes - sudo /usr/share/zfs/zloop.sh -t 1200 -l -m1 + sudo /usr/share/zfs/zloop.sh -t 1200 -l -m1 -- -T 120 -P 60 - name: Prepare artifacts if: failure() run: | sudo chmod +r -R $TEST_DIR/ - uses: actions/upload-artifact@v2 if: failure() with: name: Logs path: | /var/tmp/zloop/*/ !/var/tmp/zloop/*/vdev/ if-no-files-found: ignore - uses: actions/upload-artifact@v2 if: failure() with: name: Pool files path: | /var/tmp/zloop/*/vdev/ if-no-files-found: ignore diff --git a/scripts/zloop.sh b/scripts/zloop.sh index 546e7001776d..4a572ebab1fc 100755 --- a/scripts/zloop.sh +++ b/scripts/zloop.sh @@ -1,333 +1,339 @@ #!/usr/bin/env bash # # CDDL HEADER START # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # CDDL HEADER END # # # Copyright (c) 2015 by Delphix. All rights reserved. # Copyright (C) 2016 Lawrence Livermore National Security, LLC. # Copyright (c) 2017, Intel Corporation. # BASE_DIR=$(dirname "$0") SCRIPT_COMMON=common.sh if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then . "${BASE_DIR}/${SCRIPT_COMMON}" else echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 fi # shellcheck disable=SC2034 PROG=zloop.sh GDB=${GDB:-gdb} DEFAULTWORKDIR=/var/tmp DEFAULTCOREDIR=/var/tmp/zloop function usage { - echo -e "\n$0 [-t ] [ -s ] [-c ]" \ - "[ -- [extra ztest parameters]]\n" \ - "\n" \ - " This script runs ztest repeatedly with randomized arguments.\n" \ - " If a crash is encountered, the ztest logs, any associated\n" \ - " vdev files, and core file (if one exists) are moved to the\n" \ - " output directory ($DEFAULTCOREDIR by default). Any options\n" \ - " after the -- end-of-options marker will be passed to ztest.\n" \ - "\n" \ - " Options:\n" \ - " -t Total time to loop for, in seconds. If not provided,\n" \ - " zloop runs forever.\n" \ - " -s Size of vdev devices.\n" \ - " -f Specify working directory for ztest vdev files.\n" \ - " -c Specify a core dump directory to use.\n" \ - " -m Max number of core dumps to allow before exiting.\n" \ - " -l Create 'ztest.core.N' symlink to core directory.\n" \ - " -h Print this help message.\n" \ - "" >&2 + cat >&2 <] [-f ] + [-m ] [-s ] [-t ] + [-I ] [-- [extra ztest parameters]] + + This script runs ztest repeatedly with randomized arguments. + If a crash is encountered, the ztest logs, any associated + vdev files, and core file (if one exists) are moved to the + output directory ($DEFAULTCOREDIR by default). Any options + after the -- end-of-options marker will be passed to ztest. + + Options: + -c Specify a core dump directory to use. + -f Specify working directory for ztest vdev files. + -h Print this help message. + -l Create 'ztest.core.N' symlink to core directory. + -m Max number of core dumps to allow before exiting. + -s Size of vdev devices. + -t Total time to loop for, in seconds. If not provided, + zloop runs forever. + -I Max number of iterations to loop before exiting. + +EOF } function or_die { # shellcheck disable=SC2068 if ! $@; then echo "Command failed: $*" exit 1 fi } case $(uname) in FreeBSD) coreglob="z*.core" ;; Linux) # core file helpers origcorepattern="$(cat /proc/sys/kernel/core_pattern)" coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" if [[ $coreglob = "*" ]]; then echo "Setting core file pattern..." echo "core" > /proc/sys/kernel/core_pattern coreglob="$(grep -E -o '^([^|%[:space:]]*)' \ /proc/sys/kernel/core_pattern)*" fi ;; *) exit 1 ;; esac function core_file { # shellcheck disable=SC2012,SC2086 ls -tr1 $coreglob 2>/dev/null | head -1 } function core_prog { prog=$ZTEST core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \ tr \' ' ') if [[ "$core_id" == *"zdb "* ]]; then prog=$ZDB fi printf "%s" "$prog" } function store_core { core="$(core_file)" if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then df -h "$workdir" >>ztest.out coreid=$(date "+zloop-%y%m%d-%H%M%S") foundcrashes=$((foundcrashes + 1)) # zdb debugging zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest" zdbdebug=$($zdbcmd 2>&1) echo -e "$zdbcmd\n" >>ztest.zdb echo "$zdbdebug" >>ztest.zdb dest=$coredir/$coreid or_die mkdir -p "$dest" or_die mkdir -p "$dest/vdev" if [[ $symlink -ne 0 ]]; then or_die ln -sf "$dest" ztest.core.$foundcrashes fi echo "*** ztest crash found - moving logs to $dest" or_die mv ztest.history "$dest/" or_die mv ztest.zdb "$dest/" or_die mv ztest.out "$dest/" or_die mv "$workdir/ztest*" "$dest/vdev/" if [[ -e "$workdir/zpool.cache" ]]; then or_die mv "$workdir/zpool.cache" "$dest/vdev/" fi # check for core if [[ -f "$core" ]]; then coreprog=$(core_prog "$core") coredebug=$($GDB --batch --quiet \ -ex "set print thread-events off" \ -ex "printf \"*\n* Backtrace \n*\n\"" \ -ex "bt" \ -ex "printf \"*\n* Libraries \n*\n\"" \ -ex "info sharedlib" \ -ex "printf \"*\n* Threads (full) \n*\n\"" \ -ex "info threads" \ -ex "printf \"*\n* Backtraces \n*\n\"" \ -ex "thread apply all bt" \ -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ -ex "thread apply all bt full" \ -ex "quit" "$coreprog" "$core" 2>&1 | \ grep -v "New LWP") # Dump core + logs to stored directory echo "$coredebug" >>"$dest/ztest.gdb" or_die mv "$core" "$dest/" # Record info in cores logfile echo "*** core @ $coredir/$coreid/$core:" | \ tee -a ztest.cores fi if [[ $coremax -gt 0 ]] && [[ $foundcrashes -ge $coremax ]]; then echo "exiting... max $coremax allowed cores" exit 1 else echo "continuing..." fi fi } # parse arguments # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] coredir=$DEFAULTCOREDIR basedir=$DEFAULTWORKDIR rundir="zloop-run" timeout=0 size="512m" coremax=0 symlink=0 -while getopts ":ht:m:s:c:f:l" opt; do +iterations=0 +while getopts ":ht:m:I:s:c:f:l" opt; do case $opt in t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;; + I ) [[ $OPTARG ]] && iterations=$OPTARG ;; s ) [[ $OPTARG ]] && size=$OPTARG ;; c ) [[ $OPTARG ]] && coredir=$OPTARG ;; f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; l ) symlink=1 ;; h ) usage exit 2 ;; * ) echo "Invalid argument: -$OPTARG"; usage exit 1 esac done # pass remaining arguments on to ztest shift $((OPTIND - 1)) # enable core dumps ulimit -c unlimited export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0 if [[ -f "$(core_file)" ]]; then echo -n "There's a core dump here you might want to look at first... " core_file echo exit 1 fi if [[ ! -d $coredir ]]; then echo "core dump directory ($coredir) does not exist, creating it." or_die mkdir -p "$coredir" fi if [[ ! -w $coredir ]]; then echo "core dump directory ($coredir) is not writable." exit 1 fi or_die rm -f ztest.history or_die rm -f ztest.zdb or_die rm -f ztest.cores ztrc=0 # ztest return value foundcrashes=0 # number of crashes found so far starttime=$(date +%s) curtime=$starttime +iteration=0 # if no timeout was specified, loop forever. -while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do +while (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do + if (( iterations > 0 )) && (( iteration++ == iterations )); then + break + fi + zopt="-G -VVVVV" # start each run with an empty directory workdir="$basedir/$rundir" or_die rm -rf "$workdir" or_die mkdir "$workdir" # switch between three types of configs # 1/3 basic, 1/3 raidz mix, and 1/3 draid mix choice=$((RANDOM % 3)) # ashift range 9 - 15 align=$(((RANDOM % 2) * 3 + 9)) # randomly use special classes class="special=random" if [[ $choice -eq 0 ]]; then # basic mirror only parity=1 mirrors=2 draid_data=0 draid_spares=0 raid_children=0 vdevs=2 raid_type="raidz" elif [[ $choice -eq 1 ]]; then # fully randomized mirror/raidz (sans dRAID) parity=$(((RANDOM % 3) + 1)) mirrors=$(((RANDOM % 3) * 1)) draid_data=0 draid_spares=0 raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) vdevs=$(((RANDOM % 3) + 3)) raid_type="raidz" else # fully randomized dRAID (sans mirror/raidz) parity=$(((RANDOM % 3) + 1)) mirrors=0 draid_data=$(((RANDOM % 8) + 3)) draid_spares=$(((RANDOM % 2) + parity)) stripe=$((draid_data + parity)) extra=$((draid_spares + (RANDOM % 4))) raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra)) vdevs=$((RANDOM % 3)) raid_type="draid" fi - # run from 30 to 120 seconds - runtime=$(((RANDOM % 90) + 30)) - passtime=$((RANDOM % (runtime / 3 + 1) + 10)) - zopt="$zopt -K $raid_type" zopt="$zopt -m $mirrors" zopt="$zopt -r $raid_children" zopt="$zopt -D $draid_data" zopt="$zopt -S $draid_spares" zopt="$zopt -R $parity" zopt="$zopt -v $vdevs" zopt="$zopt -a $align" zopt="$zopt -C $class" - zopt="$zopt -T $runtime" - zopt="$zopt -P $passtime" zopt="$zopt -s $size" zopt="$zopt -f $workdir" cmd="$ZTEST $zopt $*" desc="$(date '+%m/%d %T') $cmd" echo "$desc" | tee -a ztest.history echo "$desc" >>ztest.out $cmd >>ztest.out 2>&1 ztrc=$? grep -E '===|WARNING' ztest.out >>ztest.history store_core curtime=$(date +%s) done echo "zloop finished, $foundcrashes crashes found" # restore core pattern. case $(uname) in Linux) echo "$origcorepattern" > /proc/sys/kernel/core_pattern ;; *) ;; esac uptime >>ztest.out if [[ $foundcrashes -gt 0 ]]; then exit 1 fi