diff --git a/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib b/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib index 26ded8720d10..baee8269b1e1 100644 --- a/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib +++ b/tests/zfs-tests/tests/functional/redundancy/redundancy.kshlib @@ -1,374 +1,380 @@ # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # # Copyright (c) 2013, 2016 by Delphix. All rights reserved. # . $STF_SUITE/include/libtest.shlib . $STF_SUITE/tests/functional/redundancy/redundancy.cfg function cleanup { if poolexists $TESTPOOL; then destroy_pool $TESTPOOL fi typeset dir for dir in $TESTDIR $BASEDIR; do if [[ -d $dir ]]; then log_must rm -rf $dir fi done } # # Get random number between min and max number. # # $1 Minimal value # $2 Maximal value # function random { typeset -i min=$1 typeset -i max=$2 typeset -i value while true; do ((value = RANDOM % (max + 1))) if ((value >= min)); then break fi done echo $value } # # Get the number of checksum errors for the pool. # # $1 Pool # function cksum_pool { typeset -i cksum=$(zpool status $1 | awk ' !NF { isvdev = 0 } isvdev { errors += $NF } /CKSUM$/ { isvdev = 1 } END { print errors } ') echo $cksum } # # Record the directories construction and checksum all the files which reside # within the specified pool # # $1 The specified pool # $2 The file which save the record. # function record_data { typeset pool=$1 typeset recordfile=$2 [[ -z $pool ]] && log_fail "No specified pool." [[ -f $recordfile ]] && log_must rm -f $recordfile sync_pool $pool typeset mntpnt mntpnt=$(get_prop mountpoint $pool) log_must eval "du -a $mntpnt > $recordfile 2>&1" # # When the data was damaged, checksum is failing and return 1 # So, will not use log_must # find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1 } # # Create test pool and fill with files and directories. # # $1 pool name # $2 pool type # $3 virtual devices number # function setup_test_env { typeset pool=$1 typeset keyword=$2 typeset -i vdev_cnt=$3 typeset vdevs typeset -i i=0 while (( i < vdev_cnt )); do vdevs="$vdevs $BASEDIR/vdev$i" ((i += 1)) done if [[ ! -d $BASEDIR ]]; then log_must mkdir $BASEDIR fi if poolexists $pool ; then destroy_pool $pool fi log_must truncate -s $MINVDEVSIZE $vdevs log_must zpool create -f -m $TESTDIR $pool $keyword $vdevs log_note "Filling up the filesystem ..." typeset -i ret=0 typeset -i i=0 typeset file=$TESTDIR/file typeset -i limit - (( limit = $(get_prop available $pool) / 4 )) + (( limit = $(get_prop available $pool) / 2 )) while true ; do [[ $(get_prop available $pool) -lt $limit ]] && break file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES ret=$? (( $ret != 0 )) && break (( i = i + 1 )) done record_data $TESTPOOL $PRE_RECORD_FILE } function refill_test_env { log_note "Re-filling the filesystem ..." + typeset pool=$1 typeset -i ret=0 typeset -i i=0 typeset mntpnt mntpnt=$(get_prop mountpoint $pool) typeset file=$mntpnt/file while [[ -e $file.$i ]]; do log_must rm -f $file.$i file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES ret=$? (( $ret != 0 )) && break (( i = i + 1 )) done record_data $TESTPOOL $PRE_RECORD_FILE } # # Check pool status is healthy # # $1 pool # function is_healthy { typeset pool=$1 typeset healthy_output="pool '$pool' is healthy" typeset real_output=$(zpool status -x $pool) if [[ "$real_output" == "$healthy_output" ]]; then return 0 else typeset -i ret zpool status -x $pool | grep "state:" | \ grep "FAULTED" >/dev/null 2>&1 ret=$? (( $ret == 0 )) && return 1 typeset l_scan typeset errnum l_scan=$(zpool status -x $pool | grep "scan:") l_scan=${l_scan##*"with"} errnum=$(echo $l_scan | awk '{print $1}') return $errnum fi } # # Check pool data is valid # # $1 pool # function is_data_valid { typeset pool=$1 + log_must zpool scrub -w $pool + record_data $pool $PST_RECORD_FILE if ! diff $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then + log_must cat $PRE_RECORD_FILE + log_must cat $PST_RECORD_FILE + diff -u $PRE_RECORD_FILE $PST_RECORD_FILE return 1 fi return 0 } # # Get the specified count devices name # # $1 pool name # $2 devices count # function get_vdevs #pool cnt { typeset pool=$1 typeset -i cnt=$2 typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}'| \ - egrep -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|---" | \ + egrep -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|^raidz3$|^draid1.*|^draid2.*|^draid3.*|---" | \ egrep -v "/old$|^$pool$") typeset -i i=0 typeset vdevs while ((i < cnt)); do typeset dev=$(echo $all_devs | awk '{print $1}') eval all_devs=\${all_devs##*$dev} vdevs="$dev $vdevs" ((i += 1)) done echo "$vdevs" } # # Create and replace the same name virtual device files # # $1 pool name # $2-n virtual device files # function replace_missing_devs { typeset pool=$1 shift typeset vdev for vdev in $@; do - log_must gnudd if=/dev/zero of=$vdev \ - bs=1024k count=$(($MINDEVSIZE / (1024 * 1024))) \ - oflag=fdatasync + log_must dd if=/dev/zero of=$vdev \ + bs=1024k count=$((MINVDEVSIZE / (1024 * 1024))) \ + conv=fdatasync log_must zpool replace -wf $pool $vdev $vdev done } # # Damage the pool's virtual device files. # # $1 pool name # $2 Failing devices count # $3 damage vdevs method, if not null, we keep # the label for the vdevs # function damage_devs { typeset pool=$1 typeset -i cnt=$2 typeset label="$3" typeset vdevs - typeset -i bs_count=$((64 * 1024)) + typeset -i bs_count=$(((MINVDEVSIZE / 1024) - 4096)) vdevs=$(get_vdevs $pool $cnt) typeset dev if [[ -n $label ]]; then for dev in $vdevs; do - dd if=/dev/zero of=$dev seek=512 bs=1024 \ + log_must dd if=/dev/zero of=$dev seek=512 bs=1024 \ count=$bs_count conv=notrunc >/dev/null 2>&1 done else for dev in $vdevs; do - dd if=/dev/zero of=$dev bs=1024 count=$bs_count \ - conv=notrunc >/dev/null 2>&1 + log_must dd if=/dev/zero of=$dev bs=1024 \ + count=$bs_count conv=notrunc >/dev/null 2>&1 done fi sync_pool $pool } # # Clear errors in the pool caused by data corruptions # # $1 pool name # function clear_errors { typeset pool=$1 log_must zpool clear $pool if ! is_healthy $pool ; then log_note "$pool should be healthy." return 1 fi if ! is_data_valid $pool ; then log_note "Data should be valid in $pool." return 1 fi return 0 } # # Remove the specified pool's virtual device files # # $1 Pool name # $2 Missing devices count # function remove_devs { typeset pool=$1 typeset -i cnt=$2 typeset vdevs vdevs=$(get_vdevs $pool $cnt) log_must rm -f $vdevs sync_pool $pool } # # Recover the bad or missing device files in the pool # # $1 Pool name # $2 Missing devices count # function recover_bad_missing_devs { typeset pool=$1 typeset -i cnt=$2 typeset vdevs vdevs=$(get_vdevs $pool $cnt) replace_missing_devs $pool $vdevs if ! is_healthy $pool ; then log_note "$pool should be healthy." return 1 fi if ! is_data_valid $pool ; then log_note "Data should be valid in $pool." return 1 fi return 0 } diff --git a/tests/zfs-tests/tests/functional/redundancy/redundancy_draid3.ksh b/tests/zfs-tests/tests/functional/redundancy/redundancy_draid3.ksh index bddd150d0c98..d4c823ed9b37 100755 --- a/tests/zfs-tests/tests/functional/redundancy/redundancy_draid3.ksh +++ b/tests/zfs-tests/tests/functional/redundancy/redundancy_draid3.ksh @@ -1,85 +1,85 @@ #!/bin/ksh -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2007 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # # Copyright (c) 2013 by Delphix. All rights reserved. # Copyright (c) 2020 by Lawrence Livermore National Security, LLC. # . $STF_SUITE/include/libtest.shlib . $STF_SUITE/tests/functional/redundancy/redundancy.kshlib # # DESCRIPTION: # A draid3 pool can withstand 3 devices are failing or missing. # # STRATEGY: # 1. Create N(>5,<6) virtual disk files. # 2. Create draid3 pool based on the virtual disk files. # 3. Fill the filesystem with directories and files. # 4. Record all the files and directories checksum information. -# 5. Damaged at most two of the virtual disk files. +# 5. Damaged at most three of the virtual disk files. # 6. Verify the data is correct to prove draid3 can withstand 3 devices # are failing. # verify_runnable "global" log_assert "Verify draid3 pool can withstand three devices failing." log_onexit cleanup typeset -i cnt=$(random_int_between 5 6) setup_test_env $TESTPOOL draid3 $cnt # # Inject data corruption errors for draid3 pool # for i in 1 2 3; do damage_devs $TESTPOOL $i "label" log_must is_data_valid $TESTPOOL log_must clear_errors $TESTPOOL done # # Inject bad devices errors for draid3 pool # for i in 1 2 3; do damage_devs $TESTPOOL $i log_must is_data_valid $TESTPOOL log_must recover_bad_missing_devs $TESTPOOL $i done # # Inject missing device errors for draid3 pool # for i in 1 2 3; do remove_devs $TESTPOOL $i log_must is_data_valid $TESTPOOL log_must recover_bad_missing_devs $TESTPOOL $i done log_pass "draid3 pool can withstand three devices failing passed." diff --git a/tests/zfs-tests/tests/functional/redundancy/redundancy_stripe.ksh b/tests/zfs-tests/tests/functional/redundancy/redundancy_stripe.ksh index 7ee51051ea1a..b2c4a85febef 100755 --- a/tests/zfs-tests/tests/functional/redundancy/redundancy_stripe.ksh +++ b/tests/zfs-tests/tests/functional/redundancy/redundancy_stripe.ksh @@ -1,62 +1,64 @@ #!/bin/ksh -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2007 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # # Copyright (c) 2013, 2016 by Delphix. All rights reserved. # . $STF_SUITE/include/libtest.shlib . $STF_SUITE/tests/functional/redundancy/redundancy.kshlib # # DESCRIPTION: # Striped pool have no data redundancy. Any device errors will # cause data corruption. # # STRATEGY: # 1. Create N virtual disk file. # 2. Create stripe pool based on the virtual disk files. # 3. Fill the filesystem with directories and files. # 4. Record all the files and directories checksum information. # 5. Damage one of the virtual disk file. # 6. Verify the data is error. # verify_runnable "global" log_assert "Verify striped pool have no data redundancy." log_onexit cleanup typeset -i cnt=$(random_int_between 2 5) setup_test_env $TESTPOOL "" $cnt damage_devs $TESTPOOL 1 "keep_label" log_must zpool scrub -w $TESTPOOL -log_mustnot is_healthy $TESTPOOL +if is_healthy $TESTPOOL ; then + log_fail "$pool should not be healthy." +fi log_pass "Striped pool has no data redundancy as expected."