diff --git a/tests/sys/cddl/zfs/include/libgnop.kshlib b/tests/sys/cddl/zfs/include/libgnop.kshlib index 44809385c075..f4f742fe6929 100644 --- a/tests/sys/cddl/zfs/include/libgnop.kshlib +++ b/tests/sys/cddl/zfs/include/libgnop.kshlib @@ -1,97 +1,103 @@ # vim: filetype=sh # # Copyright (c) 2017 Spectra Logic Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions, and the following disclaimer, # without modification. # 2. Redistributions in binary form must reproduce at minimum a disclaimer # substantially similar to the "NO WARRANTY" disclaimer below # ("Disclaimer") and any redistribution must be conditioned upon # including a substantially similar Disclaimer requirement for further # binary redistribution. # # NO WARRANTY # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGES. # # # # Routines that use gnop(8) to simulate devices that can disappear at any time # # Create a gnop device on top of a real device. Don't use the full extent; use # a portion in the middle so that any ZFS label present on the real device # won't be present on the gnop device and vice versa. function create_gnop { # Name of disk to use, with or without /dev/ typeset disk=$1 # Optional physical path to use typeset physpath=$2 # size of disk in bytes typeset -li disk_size # disk size, rounded down to multiple of 16384 typeset -li disk_size_rounded # size of the nop device in bytes typeset -li nop_size # offset of the beginning of the nop device in bytes typeset -li nop_offset typeset args disk_size=`diskinfo $disk | cut -f 3` # Round it down so the nop device will be 4k-aligned disk_size_rounded=$(( ${disk_size} / 16384 * 16384 )) nop_size=$(( ${disk_size_rounded} / 4 )) nop_offset=${nop_size} args="-s ${nop_size} -o ${nop_offset}" if [ -n "$physpath" ]; then args="$args -z $physpath" fi gnop create ${args} ${disk} } # Create multiple gnop devices function create_gnops { typeset disk for disk in $@; do create_gnop "$disk" || return 1 done return 0 } # Destroy a gnop device. function destroy_gnop { # Name of the underlying (non-gnop) device typeset disk=$1 # Use "-f" so we can destroy a gnop with a consumer (like ZFS) gnop destroy -f ${disk}.nop + + # Wait for it to disappear + for i in `seq 5`; do + gnop status ${disk}.nop >/dev/null 2>/dev/null || break + sleep $i + done } # Destroy multiple gnop devices. Attempt to destroy them all, ignoring errors function destroy_gnops { typeset disk for disk in $@; do destroy_gnop "$disk" done return 0 } diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib b/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib index e9ea036fbbab..8456c5450d2b 100644 --- a/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib +++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib @@ -1,83 +1,96 @@ #!/usr/local/bin/ksh93 -p # vim: filetype=sh # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2013 Spectra Logic. All rights reserved. # Use is subject to license terms. # # Common routines used by multiple zfsd tests function wait_for_pool_dev_state_change { typeset -i timeout=$1 typeset disk=$2 typeset state=$3 log_note "Waiting up to $timeout seconds for $disk to become $state ..." for ((; $timeout > 0; timeout=$timeout-1)); do check_state $TESTPOOL "$disk" "$state" [ $? -eq 0 ] && return $SLEEP 1 done log_must $ZPOOL status $TESTPOOL log_fail "ERROR: Disk $disk not marked as $state in $TESTPOOL" } function wait_for_pool_removal { typeset -i timeout=$1 wait_for_pool_dev_state_change $timeout $REMOVAL_DISK "REMOVED|UNAVAIL" } function wait_until_scrubbed { typeset pool=$1 while is_pool_scrubbing $pool; do log_note "$pool still scrubbing..." $SLEEP 1 done } function corrupt_pool_vdev { typeset pool=$1 typeset vdev=$2 typeset file=$3 + typeset -li start=0 + typeset -li now=0 + typeset -li timeout=60 # do some IO on the pool log_must $DD if=/dev/zero of=$file bs=1024k count=64 $FSYNC $file - # scribble on the underlying file to corrupt the vdev - log_must $DD if=/dev/urandom of=$vdev bs=1024k count=64 conv=notrunc + # ZFS rate limits checksum errors to about 20 per second. So in order + # to ensure that we reach zfsd's threshold, we must alternately + # scribble and scrub. + while (( "$now" - "$start" < "$timeout" )); do + # scribble on the underlying file to corrupt the vdev + log_must $DD if=/dev/urandom of=$vdev bs=1024k count=64 conv=notrunc - # Scrub the pool to detect the corruption - log_must $ZPOOL scrub $pool - wait_until_scrubbed $pool + # Scrub the pool to detect and repair the corruption + log_must $ZPOOL scrub $pool + wait_until_scrubbed $pool + now=`date +%s` + if [ "$start" -eq 0 ]; then + start=`date +%s` + fi + check_state "$pool" "$vdev" DEGRADED && return + $SLEEP 1 + done - # ZFSD can take up to 60 seconds to degrade an array in response to - # errors (though it's usually faster). - wait_for_pool_dev_state_change 60 $vdev DEGRADED + log_must $ZPOOL status "$pool" + log_fail "ERROR: Disk $vdev not marked as DEGRADED in $pool" } diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd_autoreplace_002_pos.ksh b/tests/sys/cddl/zfs/tests/zfsd/zfsd_autoreplace_002_pos.ksh index 2d50c73844a5..6d009a9a8b56 100644 --- a/tests/sys/cddl/zfs/tests/zfsd/zfsd_autoreplace_002_pos.ksh +++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd_autoreplace_002_pos.ksh @@ -1,90 +1,91 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2013 Spectra Logic. All rights reserved. # Use is subject to license terms. # # ident "@(#)hotspare_replace_007_pos.ksh 1.0 12/08/10 SL" # . $STF_SUITE/tests/hotspare/hotspare.kshlib . $STF_SUITE/tests/zfsd/zfsd.kshlib . $STF_SUITE/include/libgnop.kshlib ################################################################################ # # __stc_assertion_start # # ID: zfsd_autoreplace_002_pos # # DESCRIPTION: # In a pool with the autoreplace property set, a vdev will be # replaced by physical path # # STRATEGY: # 1. Create 1 storage pool without hot spares # 2. Remove a vdev # 4. Create a new vdev with the same physical path as the first one # 9. Verify that it does get added to the pool. # # TESTABILITY: explicit # # TEST_AUTOMATION_LEVEL: automated # # CODING STATUS: COMPLETED (2013-02-4) # # __stc_assertion_end # ############################################################################### log_assert "A pool with the autoreplace property will replace disks by physical path" function verify_assertion { wait_for_pool_dev_state_change 20 $NEW_DISK ONLINE } typeset PHYSPATH="some_physical_path" typeset REMOVAL_DISK=$DISK0 typeset REMOVAL_NOP=${DISK0}.nop typeset NEW_DISK=$DISK4 typeset NEW_NOP=${DISK4}.nop typeset OTHER_DISKS="${DISK1} ${DISK2} ${DISK3}" typeset ALLDISKS="${DISK0} ${DISK1} ${DISK2} ${DISK3}" typeset ALLNOPS=${ALLDISKS//~(E)([[:space:]]+|$)/.nop\1} set -A MY_KEYWORDS "mirror" "raidz1" "raidz2" ensure_zfsd_running log_must create_gnops $OTHER_DISKS for keyword in "${MY_KEYWORDS[@]}" ; do log_must create_gnop $REMOVAL_DISK $PHYSPATH log_must create_pool $TESTPOOL $keyword $ALLNOPS log_must $ZPOOL set autoreplace=on $TESTPOOL log_must destroy_gnop $REMOVAL_DISK + log_must wait_for_pool_removal 20 log_must create_gnop $NEW_DISK $PHYSPATH verify_assertion destroy_pool "$TESTPOOL" log_must destroy_gnop $NEW_DISK done log_pass diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd_autoreplace_003_pos.ksh b/tests/sys/cddl/zfs/tests/zfsd/zfsd_autoreplace_003_pos.ksh index e2af801558e0..4eb04d60809e 100644 --- a/tests/sys/cddl/zfs/tests/zfsd/zfsd_autoreplace_003_pos.ksh +++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd_autoreplace_003_pos.ksh @@ -1,100 +1,101 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2014 Spectra Logic. All rights reserved. # Use is subject to license terms. # # . $STF_SUITE/tests/hotspare/hotspare.kshlib . $STF_SUITE/tests/zfsd/zfsd.kshlib . $STF_SUITE/include/libgnop.kshlib ################################################################################ # # __stc_assertion_start # # ID: zfsd_autoreplace_003_pos # # DESCRIPTION: # In a pool with the autoreplace property set, a vdev will be # replaced by physical path even if a spare is already active for that # vdev # # STRATEGY: # 1. Create 1 storage pool with a hot spare # 2. Remove a vdev # 3. Wait for the hotspare to fully resilver # 4. Create a new vdev with the same physical path as the first one # 10. Verify that it does get added to the pool. # 11. Verify that the hotspare gets removed. # # TESTABILITY: explicit # # TEST_AUTOMATION_LEVEL: automated # # CODING STATUS: COMPLETED (2013-05-13) # # __stc_assertion_end # ############################################################################### log_assert "A pool with the autoreplace property will replace disks by physical path, even if a spare is active" function verify_assertion { # Verify that the replacement disk gets added to the pool wait_for_pool_dev_state_change 20 $NEW_DISK ONLINE # Wait for resilvering to complete wait_until_resilvered # Check that the spare is deactivated wait_for_pool_dev_state_change 20 "$SPARE_DISK" "AVAIL" } typeset PHYSPATH="some_physical_path" typeset REMOVAL_DISK=$DISK0 typeset REMOVAL_NOP=${DISK0}.nop typeset NEW_DISK=$DISK3 typeset NEW_NOP=${DISK3}.nop typeset SPARE_DISK=${DISK4} typeset SPARE_NOP=${DISK4}.nop typeset OTHER_DISKS="${DISK1} ${DISK2}" typeset OTHER_NOPS=${OTHER_DISKS//~(E)([[:space:]]+|$)/.nop\1} set -A MY_KEYWORDS "mirror" "raidz1" "raidz2" ensure_zfsd_running log_must create_gnops $OTHER_DISKS $SPARE_DISK for keyword in "${MY_KEYWORDS[@]}" ; do log_must create_gnop $REMOVAL_DISK $PHYSPATH log_must create_pool $TESTPOOL $keyword $REMOVAL_NOP $OTHER_NOPS spare $SPARE_NOP log_must $ZPOOL set autoreplace=on $TESTPOOL log_must destroy_gnop $REMOVAL_DISK + log_must wait_for_pool_removal 20 log_must create_gnop $NEW_DISK $PHYSPATH verify_assertion destroy_pool "$TESTPOOL" log_must destroy_gnop $NEW_DISK done log_pass diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd_fault_001_pos.ksh b/tests/sys/cddl/zfs/tests/zfsd/zfsd_fault_001_pos.ksh index 3e1340b22e56..3456a328e7f9 100644 --- a/tests/sys/cddl/zfs/tests/zfsd/zfsd_fault_001_pos.ksh +++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd_fault_001_pos.ksh @@ -1,162 +1,119 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright (c) 2012,2013 Spectra Logic Corporation. All rights reserved. # Use is subject to license terms. # . $STF_SUITE/include/libtest.kshlib +. $STF_SUITE/include/libgnop.kshlib ################################################################################ # # __stc_assertion_start # # ID: zfsd_fault_001_pos # # DESCRIPTION: # If a vdev experiences IO errors, it will become faulted. # # # STRATEGY: -# 1. Create a storage pool. Only use the da driver (FreeBSD's SCSI disk -# driver) because it has a special interface for simulating IO errors. +# 1. Create a storage pool. Use gnop vdevs so we can inject I/O errors. # 2. Inject IO errors while doing IO to the pool. # 3. Verify that the vdev becomes FAULTED. # 4. ONLINE it and verify that it resilvers and joins the pool. # # TESTABILITY: explicit # # TEST_AUTOMATION_LEVEL: automated # # CODING STATUS: COMPLETED (2012-08-09) # # __stc_assertion_end # ############################################################################### verify_runnable "global" -function cleanup -{ - # Disable error injection, if still active - sysctl kern.cam.da.$TMPDISKNUM.error_inject=0 > /dev/null - - if poolexists $TESTPOOL; then - # We should not get here if the test passed. Print the output - # of zpool status to assist in debugging. - $ZPOOL status - # Clear out artificially generated errors and destroy the pool - $ZPOOL clear $TESTPOOL - destroy_pool $TESTPOOL - fi -} - log_assert "ZFS will fault a vdev that produces IO errors" -log_onexit cleanup ensure_zfsd_running -# Make sure that at least one of the disks is using the da driver, and use -# that disk for inject errors -typeset TMPDISK="" -for d in $DISKS -do - b=`basename $d` - if test ${b%%[0-9]*} == da - then - TMPDISK=$b - TMPDISKNUM=${b##da} - break - fi -done -if test -z $TMPDISK -then - log_unsupported "This test requires at least one disk to use the da driver" -fi +DISK0_NOP=${DISK0}.nop +DISK1_NOP=${DISK1}.nop +log_must create_gnops $DISK0 $DISK1 for type in "raidz" "mirror"; do log_note "Testing raid type $type" # Create a pool on the supplied disks - create_pool $TESTPOOL $type $DISKS + create_pool $TESTPOOL $type "$DISK0_NOP" "$DISK1_NOP" log_must $ZFS create $TESTPOOL/$TESTFS # Cause some IO errors writing to the pool while true; do - # Running zpool status after every dd operation is too slow. - # So we will run several dd's in a row before checking zpool - # status. sync between dd operations to ensure that the disk - # gets IO - for ((i=0; $i<64; i=$i+1)); do - sysctl kern.cam.da.$TMPDISKNUM.error_inject=1 > \ - /dev/null - $DD if=/dev/zero bs=128k count=1 >> \ - /$TESTPOOL/$TESTFS/$TESTFILE 2> /dev/null - $FSYNC /$TESTPOOL/$TESTFS/$TESTFILE - done + log_must gnop configure -e 5 -w 100 "$DISK1_NOP" + $DD if=/dev/zero bs=128k count=1 >> \ + /$TESTPOOL/$TESTFS/$TESTFILE 2> /dev/null + $FSYNC /$TESTPOOL/$TESTFS/$TESTFILE # Check to see if the pool is faulted yet $ZPOOL status $TESTPOOL | grep -q 'state: DEGRADED' if [ $? == 0 ] then log_note "$TESTPOOL got degraded" break fi done log_must check_state $TESTPOOL $TMPDISK "FAULTED" - #find the failed disk guid - typeset FAILED_VDEV=`$ZPOOL status $TESTPOOL | - awk "/^[[:space:]]*$TMPDISK[[:space:]]*FAULTED/ {print \\$1}"` - - # Reattach the failed disk - $ZPOOL online $TESTPOOL $FAILED_VDEV > /dev/null - if [ $? != 0 ]; then - log_fail "Could not reattach $FAILED_VDEV" - fi + # Heal and reattach the failed disk + log_must gnop configure -w 0 "$DISK1_NOP" + log_must $ZPOOL online $TESTPOOL "$DISK1_NOP" # Verify that the pool resilvers and goes to the ONLINE state for (( retries=60; $retries>0; retries=$retries+1 )) do $ZPOOL status $TESTPOOL | egrep -q "scan:.*resilvered" RESILVERED=$? $ZPOOL status $TESTPOOL | egrep -q "state:.*ONLINE" ONLINE=$? if test $RESILVERED -a $ONLINE then break fi $SLEEP 2 done if [ $retries == 0 ] then log_fail "$TESTPOOL never resilvered in the allowed time" fi destroy_pool $TESTPOOL log_must $RM -rf /$TESTPOOL done log_pass diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_001_pos.ksh b/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_001_pos.ksh index dd39d90fd694..a94a3fb7ac42 100644 --- a/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_001_pos.ksh +++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_001_pos.ksh @@ -1,80 +1,81 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2008 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # Copyright 2012,2013 Spectra Logic Corporation. All rights reserved. # Use is subject to license terms. # # Portions taken from: # ident "@(#)replacement_001_pos.ksh 1.4 08/02/27 SMI" # . $STF_SUITE/tests/hotspare/hotspare.kshlib . $STF_SUITE/tests/zfsd/zfsd.kshlib . $STF_SUITE/include/libtest.kshlib . $STF_SUITE/include/libgnop.kshlib log_assert "ZFSD will automatically replace a SAS disk that disappears and reappears in the same location, with the same devname" ensure_zfsd_running set_disks typeset REMOVAL_DISK=$DISK0 typeset REMOVAL_NOP=${DISK0}.nop typeset OTHER_DISKS="${DISK1} ${DISK2}" typeset ALLDISKS="${DISK0} ${DISK1} ${DISK2}" typeset ALLNOPS=${ALLDISKS//~(E)([[:space:]]+|$)/.nop\1} log_must create_gnops $ALLDISKS for type in "raidz" "mirror"; do # Create a pool on the supplied disks create_pool $TESTPOOL $type $ALLNOPS log_must $ZFS create $TESTPOOL/$TESTFS log_must $ZFS set mountpoint=$TESTDIR $TESTPOOL/$TESTFS # Disable the first disk. log_must destroy_gnop $REMOVAL_DISK + log_must wait_for_pool_removal 20 # Write out data to make sure we can do I/O after the disk failure log_must $DD if=/dev/zero of=$TESTDIR/$TESTFILE bs=1m count=1 log_must $FSYNC $TESTDIR/$TESTFILE # Check to make sure ZFS sees the disk as removed wait_for_pool_dev_state_change 20 $REMOVAL_NOP REMOVED # Re-enable the disk log_must create_gnop $REMOVAL_DISK # Disk should auto-join the zpool & be resilvered. wait_for_pool_dev_state_change 20 $REMOVAL_NOP ONLINE wait_until_resilvered $ZPOOL status $TESTPOOL destroy_pool $TESTPOOL log_must $RM -rf /$TESTPOOL done log_pass diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh b/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh index 7f066a3cff21..b6dcfe97dd7b 100755 --- a/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh +++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh @@ -1,611 +1,613 @@ # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2012,2013 Spectra Logic. All rights reserved. # Use is subject to license terms. # atf_test_case zfsd_fault_001_pos cleanup zfsd_fault_001_pos_head() { atf_set "descr" "ZFS will fault a vdev that produces IO errors" - atf_set "require.progs" "ksh93 zfs zpool zfsd" + atf_set "require.progs" "ksh93 gnop zfs zpool zfsd" atf_set "timeout" 300 } zfsd_fault_001_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg + . $(atf_get_srcdir)/../hotspare/hotspare.kshlib + . $(atf_get_srcdir)/../hotspare/hotspare.cfg . $(atf_get_srcdir)/zfsd.cfg verify_disk_count "$DISKS" 2 verify_zfsd_running ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_fault_001_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_fault_001_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_degrade_001_pos cleanup zfsd_degrade_001_pos_head() { atf_set "descr" "ZFS will degrade a vdev that produces checksum errors" atf_set "require.progs" "ksh93 zpool zfsd" atf_set "timeout" 600 } zfsd_degrade_001_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg verify_disk_count "$DISKS" 2 verify_zfsd_running ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_degrade_001_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_degrade_001_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_degrade_002_pos cleanup zfsd_degrade_002_pos_head() { atf_set "descr" "ZFS will degrade a spare that produces checksum errors" atf_set "require.progs" "ksh93 zpool zfsd" atf_set "timeout" 600 } zfsd_degrade_002_pos_body() { atf_expect_fail "https://www.illumos.org/issues/8614 Checksum errors on a mirrored child of a raidz are incorrectly accounted" . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg verify_disk_count "$DISKS" 5 verify_zfsd_running ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_degrade_002_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_degrade_002_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_hotspare_001_pos cleanup zfsd_hotspare_001_pos_head() { atf_set "descr" "An active, damaged spare will be replaced by an available spare" atf_set "require.progs" "ksh93 zpool zfsd" atf_set "timeout" 3600 } zfsd_hotspare_001_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_zfsd_running ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_hotspare_001_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_hotspare_001_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_hotspare_002_pos cleanup zfsd_hotspare_002_pos_head() { atf_set "descr" "If a vdev becomes degraded, the spare will be activated." atf_set "require.progs" "ksh93 zpool zfsd zinject" atf_set "timeout" 3600 } zfsd_hotspare_002_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_zfsd_running ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_hotspare_002_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_hotspare_002_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_hotspare_003_pos cleanup zfsd_hotspare_003_pos_head() { atf_set "descr" "A faulted vdev will be replaced by an available spare" atf_set "require.progs" "ksh93 zpool zfsd zinject" atf_set "timeout" 3600 } zfsd_hotspare_003_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_disk_count "$DISKS" 5 verify_zfsd_running ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_hotspare_003_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_hotspare_003_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_hotspare_004_pos cleanup zfsd_hotspare_004_pos_head() { atf_set "descr" "Removing a disk from a pool results in the spare activating" - atf_set "require.progs" "ksh93 gnop zpool camcontrol zfsd" + atf_set "require.progs" "ksh93 gnop zpool" atf_set "timeout" 3600 } zfsd_hotspare_004_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_disk_count "$DISKS" 5 verify_zfsd_running ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_hotspare_004_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_hotspare_004_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_hotspare_005_pos cleanup zfsd_hotspare_005_pos_head() { atf_set "descr" "A spare that is added to a degraded pool will be activated" atf_set "require.progs" "ksh93 zpool zfsd zinject" atf_set "timeout" 3600 } zfsd_hotspare_005_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_zfsd_running ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_hotspare_005_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_hotspare_005_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_hotspare_006_pos cleanup zfsd_hotspare_006_pos_head() { atf_set "descr" "zfsd will replace two vdevs that fail simultaneously" atf_set "require.progs" "ksh93 zpool zfsd zinject" atf_set "timeout" 3600 } zfsd_hotspare_006_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_zfsd_running ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_hotspare_006_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_hotspare_006_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_hotspare_007_pos cleanup zfsd_hotspare_007_pos_head() { atf_set "descr" "zfsd will swap failed drives at startup" - atf_set "require.progs" "ksh93 gnop zpool camcontrol zfsd" + atf_set "require.progs" "ksh93 gnop zpool" atf_set "timeout" 3600 } zfsd_hotspare_007_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_disk_count "$DISKS" 5 verify_zfsd_running ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_hotspare_007_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_hotspare_007_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_hotspare_008_neg cleanup zfsd_hotspare_008_neg_head() { atf_set "descr" "zfsd will not use newly added spares on replacing vdevs" atf_set "require.progs" "ksh93 zpool zfsd" atf_set "timeout" 3600 } zfsd_hotspare_008_neg_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_disk_count "$DISKS" 4 ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_hotspare_008_neg.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_hotspare_008_neg_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_autoreplace_001_neg cleanup zfsd_autoreplace_001_neg_head() { atf_set "descr" "A pool without autoreplace set will not replace by physical path" - atf_set "require.progs" "ksh93 zpool camcontrol zfsd gnop" + atf_set "require.progs" "ksh93 zpool gnop" atf_set "timeout" 3600 } zfsd_autoreplace_001_neg_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_disk_count "$DISKS" 5 ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_autoreplace_001_neg.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_autoreplace_001_neg_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_autoreplace_002_pos cleanup zfsd_autoreplace_002_pos_head() { atf_set "descr" "A pool with autoreplace set will replace by physical path" atf_set "require.progs" "ksh93 gnop zpool zfsd" atf_set "timeout" 3600 } zfsd_autoreplace_002_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_disk_count "$DISKS" 5 verify_zfsd_running ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_autoreplace_002_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_autoreplace_002_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_autoreplace_003_pos cleanup zfsd_autoreplace_003_pos_head() { atf_set "descr" "A pool with autoreplace set will replace by physical path even if a spare is active" - atf_set "require.progs" "ksh93 zpool camcontrol zfsd gnop" + atf_set "require.progs" "ksh93 zpool gnop" atf_set "timeout" 3600 } zfsd_autoreplace_003_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_disk_count "$DISKS" 5 verify_zfsd_running ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_autoreplace_003_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_autoreplace_003_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_replace_001_pos cleanup zfsd_replace_001_pos_head() { atf_set "descr" "ZFSD will automatically replace a SAS disk that disappears and reappears in the same location, with the same devname" - atf_set "require.progs" "ksh93 zpool camcontrol zfsd zfs gnop" + atf_set "require.progs" "ksh93 zpool zfs gnop" } zfsd_replace_001_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg verify_disk_count "$DISKS" 3 verify_zfsd_running ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_replace_001_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_replace_001_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_replace_002_pos cleanup zfsd_replace_002_pos_head() { atf_set "descr" "zfsd will reactivate a pool after all disks are failed and reappeared" - atf_set "require.progs" "ksh93 zpool camcontrol zfsd zfs" + atf_set "require.progs" "ksh93 zpool zfs" } zfsd_replace_002_pos_body() { atf_expect_fail "Not yet implemented in zfsd" . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg verify_disk_count "$DISKS" 3 verify_zfsd_running ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_replace_002_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_replace_002_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_replace_003_pos cleanup zfsd_replace_003_pos_head() { atf_set "descr" "ZFSD will correctly replace disks that dissapear and reappear with different devnames" - atf_set "require.progs" "ksh93 zpool camcontrol zfsd zfs gnop" + atf_set "require.progs" "ksh93 zpool zfs gnop" } zfsd_replace_003_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_zfsd_running ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_replace_003_pos.ksh if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_replace_003_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/zfsd.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_test_case zfsd_import_001_pos cleanup zfsd_import_001_pos_head() { atf_set "descr" "If a removed drive gets reinserted while the pool is exported, it will detach its spare when imported." atf_set "require.progs" "ksh93 gnop zfsd zpool" atf_set "timeout" 3600 } zfsd_import_001_pos_body() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg verify_disk_count "$DISKS" 5 verify_zfsd_running ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed" ksh93 $(atf_get_srcdir)/zfsd_import_001_pos.ksh || atf_fail "Testcase failed" if [[ $? != 0 ]]; then save_artifacts atf_fail "Testcase failed" fi } zfsd_import_001_pos_cleanup() { . $(atf_get_srcdir)/../../include/default.cfg . $(atf_get_srcdir)/../hotspare/hotspare.kshlib . $(atf_get_srcdir)/../hotspare/hotspare.cfg ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed" } atf_init_test_cases() { atf_add_test_case zfsd_fault_001_pos atf_add_test_case zfsd_degrade_001_pos atf_add_test_case zfsd_degrade_002_pos atf_add_test_case zfsd_hotspare_001_pos atf_add_test_case zfsd_hotspare_002_pos atf_add_test_case zfsd_hotspare_003_pos atf_add_test_case zfsd_hotspare_004_pos atf_add_test_case zfsd_hotspare_005_pos atf_add_test_case zfsd_hotspare_006_pos atf_add_test_case zfsd_hotspare_007_pos atf_add_test_case zfsd_hotspare_008_neg atf_add_test_case zfsd_autoreplace_001_neg atf_add_test_case zfsd_autoreplace_002_pos atf_add_test_case zfsd_autoreplace_003_pos atf_add_test_case zfsd_replace_001_pos atf_add_test_case zfsd_replace_002_pos atf_add_test_case zfsd_replace_003_pos atf_add_test_case zfsd_import_001_pos } save_artifacts() { # If ARTIFACTS_DIR is defined, save test artifacts for # post-mortem analysis if [[ -n $ARTIFACTS_DIR ]]; then TC_ARTIFACTS_DIR=${ARTIFACTS_DIR}/sys/cddl/zfs/tests/zfsd/$(atf_get ident) mkdir -p $TC_ARTIFACTS_DIR cp -a /var/log/zfsd.log* $TC_ARTIFACTS_DIR bzip2 $TC_ARTIFACTS_DIR/zfsd.log fi } verify_zfsd_running() { service zfsd onestatus || \ atf_skip "zfsd(8) must be enabled and running for this test" }