Index: projects/zfsd/head/tests/sys/cddl/zfs/include/libsas.kshlib =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/include/libsas.kshlib (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/include/libsas.kshlib (revision 292374) @@ -1,185 +1,185 @@ # # Copyright (c) 2010 Spectra Logic Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions, and the following disclaimer, # without modification. # 2. Redistributions in binary form must reproduce at minimum a disclaimer # substantially similar to the "NO WARRANTY" disclaimer below # ("Disclaimer") and any redistribution must be conditioned upon # including a substantially similar Disclaimer requirement for further # binary redistribution. # # NO WARRANTY # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGES. # # $Id: //SpectraBSD/stable/tests/sys/cddl/zfs/include/libsas.kshlib#2 $ # $FreeBSD$ # # Get all PHYs for a given expander. # Returns formatting suitable for iteration. function get_all_phys { typeset expander=$1 camcontrol smpphylist $expander -q | awk '{print $1" "$NF}' | \ tr -d '()' | tr ',' ' ' | tr '\n' ',' } # # Given a disk (e.g. /dev/da0 or da0), determine the following: # - Does it exist in CAM? # - Is it attached to an expander and can we find that expander? # Returns two variables: # EXPANDER -- The peripheral we need to talk to to communicate with the # expander that is connected to the given disk. # PHY -- The phy on the expander that we need to talk to. # function find_verify_sas_disk { typeset DISK=${1##*/} typeset i [ ! -c /dev/$DISK ] && log_fail "\"/dev/$DISK\" is not a char device" # Make sure this device exists. An inquiry should always succeed, # even if there is a pending error. log_must camcontrol inquiry $DISK > /dev/null # Pull the list of every peripheral in the system, except for # peripherals that are attached to the disk we're looking for typeset PASSLIST=`camcontrol devlist |awk '{print $NF}' |\ egrep -v ".*$DISK[^[:digit:]]" |tr -d '()' |awk -F, '{print $1}'` typeset FOUND=0 for i in $PASSLIST; do # Make sure this particular device supports SMP. If not, # no big deal, keep going. camcontrol smprg $i > /dev/null 2>&1 [ $? -ne 0 ] && continue # Make sure this particular device is an Enclosure Services # device. That way, we won't wind up removing the device # we're trying to operate on. # XXX this will need to change once CAM is changed to # include SMP targets in the topology. This merely takes # advantage of the fact that most (all?) expanders include # a SES device. camcontrol inquiry $i |grep "Enclosure Services" > /dev/null [ $? -ne 0 ] && continue # For every peripheral, we go through and pull out the # list of devices and their phys that we can see via this # peripheral. IFS="," for j in $(get_all_phys $i); do IFS=", \n" set -A PERIPHLIST $j unset IFS typeset NUMPERIPHS=${#PERIPHLIST[*]} ((k=1)) while [ $k -lt $NUMPERIPHS ]; do if [ "${PERIPHLIST[$k]}" = "$DISK" ]; then # echo "found $DISK PHY = ${PERIPHLIST[0]} on $i" FOUND=1 export EXPANDER=$i export PHY=${PERIPHLIST[0]} break; fi ((k=k+1)) done if [ $FOUND != 0 ]; then break; fi done unset IFS [ $FOUND -ne 0 ] && break done [ $FOUND -eq 0 ] && log_fail "Could not find PHY for disk $DISK" } # # Given an expander and phy number, find the disk device name. # function find_disk_by_phy { typeset EXPANDER=$1 typeset PHY=$2 typeset FOUND=0 unset FOUNDDISK IFS="," for j in $(get_all_phys $EXPANDER); do IFS=", \n" set -A PERIPHLIST $j unset IFS - [ "${PERIPHLIST[0]}" != "$2" ] && continue + [ "${PERIPHLIST[0]}" != "$PHY" ] && continue typeset NUMPERIPHS=${#PERIPHLIST[*]} for ((k=1; $k < $NUMPERIPHS; k=$k + 1)); do ((PSTOP=$NUMPERIPHS-1)) if [ "${PERIPHLIST[$k]%%[0-9]*}" != "pass" ] || \ [ "$k" -eq $PSTOP ]; then export FOUNDDISK=${PERIPHLIST[$k]} break; fi done FOUND=1 break; done } # Given an expander and phy on that expander, disable the phy. # This function will exit (via log_fail) if it can't send the disable # request. function disable_sas_disk { typeset EXPANDER=$1 typeset PHY=$2 # Disable the phy for this particular device log_must camcontrol smppc $EXPANDER -v -p $PHY -o disable } # Given an expander and phy on that expander, enable the phy. # This function will exit (via log_fail) if it can't send the link reset # request. function enable_sas_disk { typeset EXPANDER=$1 typeset PHY=$2 # Send a link reset to bring the device back log_must camcontrol smppc $EXPANDER -p $PHY -o linkreset } function rescan_disks { if [[ -z "$1" ]]; then log_must camcontrol rescan all >/dev/null return fi for device in $(echo $* | sort -u); do log_must camcontrol rescan $device >/dev/null done } Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/hotspare/hotspare.kshlib =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/tests/hotspare/hotspare.kshlib (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/tests/hotspare/hotspare.kshlib (revision 292374) @@ -1,124 +1,125 @@ # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # ident "@(#)hotspare.kshlib 1.6 09/06/22 SMI" # . $STF_SUITE/include/libtest.kshlib . $STF_SUITE/tests/cli_root/zpool_add/zpool_add.kshlib function cleanup_devices_all { $RM -f ${devarray[*]} [ -d "$HOTSPARE_TMPDIR" ] && rmdir $HOTSPARE_TMPDIR return 0 } typeset -a pooldevs typeset -a sparedevs typeset -a logdevs typeset -a keywords=("" "mirror" "raidz" "raidz2") typeset -a devarray function set_devs { mkdir $HOTSPARE_TMPDIR typeset -i i=0 while (( i < $N_DEVARRAY_FILES )) ; do eval devarray[$i]=$HOTSPARE_TMPDIR/file.$i log_must create_vdevs ${devarray[$i]} (( i = i + 1 )) done sparedevs=("${devarray[0]}" "${devarray[1]}") pooldevs=("${devarray[3]}" "${devarray[4]}" "${devarray[5]}") if verify_slog_support ; then logdevs="${devarray[7]}" fi } function partition_cleanup { cleanup_devices_all return 0 } # # $1: keyword, should be "" "mirror" "raidz" "raidz2" # $2: hotspare list, default as $sparedevs # function setup_hotspares # keyword, spares { typeset keyword=$1 shift typeset spares=${@:-${sparedevs[@]}} create_pool "$TESTPOOL" "$keyword" \ ${pooldevs[@]} log_must poolexists "$TESTPOOL" log_must $ZPOOL set autoreplace=on "$TESTPOOL" log_must $ZPOOL add -f "$TESTPOOL" spare $spares log_must iscontained "$TESTPOOL" "$spares" if [[ -n ${logdevs[@]} ]] ; then log_must $ZPOOL add -f "$TESTPOOL" log ${logdevs[@]} log_must iscontained "$TESTPOOL" "${logdevs[@]}" fi } # # $1: the function name that run for all hotspares # $2: hotspare list, default as $sparedevs # function iterate_over_hotspares # function, spares { typeset function=$1 typeset spares=${2:-${sparedevs[@]}} for spare in $spares do $function $spare done } wait_until_resilvered() { typeset -i i=0 typeset -i timeout=60 while [[ $i -lt $timeout ]]; do if is_pool_resilvered $TESTPOOL; then break fi (( i += 1 )) if [[ $i == $timeout ]]; then + $ZPOOL status $TESTPOOL log_fail "Pool didn't resilver in ${timeout} seconds" fi $SLEEP 1 done } Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib (revision 292374) @@ -1,184 +1,185 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2013 Spectra Logic. All rights reserved. # Use is subject to license terms. # # Common routines used by multiple zfsd tests function wait_for_disk_to_reappear { typeset -i timeout=$1 - typeset EXPECTED_DISK=$2 + typeset EXPANDER=$2 + typeset PHY=$3 for ((; $timeout > 0; timeout=$timeout-1)); do find_disk_by_phy $EXPANDER $PHY [ -n "$FOUNDDISK" -a -e "/dev/$FOUNDDISK" ] && return $SLEEP 1 done - log_fail "ERROR: Disk ${EXPECTED_DISK} never reappeared" + log_fail "ERROR: Disk at ${EXPANDER}:${PHY} never reappeared" } function wait_for_pool_dev_state_change { typeset -i timeout=$1 typeset disk=$2 typeset state=$3 log_note "Waiting up to $timeout seconds for $disk to become $state ..." for ((; $timeout > 0; timeout=$timeout-1)); do check_state $TESTPOOL "$disk" "$state" [ $? -eq 0 ] && return $SLEEP 1 done log_must $ZPOOL status $TESTPOOL log_fail "ERROR: Disk $disk not marked as $state in $TESTPOOL" } function wait_for_pool_removal { typeset -i timeout=$1 wait_for_pool_dev_state_change $timeout $REMOVAL_DISK REMOVED } function wait_until_scrubbed { typeset pool=$1 while is_pool_scrubbing $pool; do log_note "$pool still scrubbing..." $SLEEP 1 done } function corrupt_pool_vdev { typeset pool=$1 typeset vdev=$2 typeset file=$3 # do some IO on the pool - log_must $DD if=/dev/zero of=$file bs=1024k count=16 + log_must $DD if=/dev/zero of=$file bs=1024k count=128 # scribble on the underlying file to corrupt the vdev log_must $DD if=/dev/urandom of=$vdev bs=1024k count=64 conv=notrunc # Scrub the pool to detect the corruption $SYNC log_must $ZPOOL scrub $pool wait_until_scrubbed $pool # ZFSD can take up to 60 seconds to degrade an array in response to # errors (though it's usually faster). wait_for_pool_dev_state_change 60 $vdev DEGRADED } # # do_autoreplace # Common code that walks through an autoreplace scenario # Does not verify the final behavior # # $1 spare disk name. Empty if no spare # function do_autoreplace { typeset SPARE_DISK=$1 # Remove a vdev by disabling its SAS phy find_verify_sas_disk $REMOVAL_DISK log_note "Disabling \"$REMOVAL_DISK\" on expander $EXPANDER phy $PHY" disable_sas_disk $EXPANDER $PHY # Check to make sure the disk is gone log_mustnot camcontrol inquiry $REMOVAL_DISK # Check to make sure ZFS sees the disk as removed wait_for_pool_removal 30 if [ -n "$SPARE_DISK" ]; then # Verify that the spare activates for ((timeout=0; $timeout<10; timeout=$timeout+1)); do check_state $TESTPOOL $SPARE_DISK "ONLINE" && break $SLEEP 6 done log_must check_state $TESTPOOL "$SPARE_DISK" "ONLINE" wait_until_resilvered fi # Export the pool # This is to prevent REMOVAL_DISK from being added to the pool when # we reenable its phy log_must $ZPOOL export $TESTPOOL # Reenable the missing dev's SAS phy log_note "Reenabling phy on expander $EXPANDER phy $PHY" enable_sas_disk $EXPANDER $PHY rescan_disks $EXPANDER - wait_for_disk_to_reappear 30 $REMOVAL_DISK + wait_for_disk_to_reappear 30 $EXPANDER $PHY # Erase the missing dev's ZFS label log_must $ZPOOL labelclear -f $( find_disks $FOUNDDISK ) # Disable the missing dev's SAS phy again find_verify_sas_disk $FOUNDDISK log_note "Disabling \"$FOUNDDISK\" on expander $EXPANDER phy $PHY" disable_sas_disk $EXPANDER $PHY # Check to make sure the disk is gone log_mustnot camcontrol inquiry $REMOVAL_DISK # Import the pool log_must $ZPOOL import $TESTPOOL # Wait 5 seconds before enabling the phy so zfsd.log will be easier # to interpret $SLEEP 5 # Reenable the missing dev's SAS phy log_note "Reenabling phy on expander $EXPANDER phy $PHY" enable_sas_disk $EXPANDER $PHY rescan_disks $EXPANDER - wait_for_disk_to_reappear 30 $REMOVAL_DISK + wait_for_disk_to_reappear 30 $EXPANDER $PHY } function autoreplace_cleanup { destroy_pool $TESTPOOL # See if the phy has been disabled, and try to re-enable it if possible. if [ -n "$REMOVAL_DISK" -a -n "$EXPANDER" -a -n "$PHY" ]; then log_note "Renabling ${EXPANDER}:${PHY} for disk ${REMOVAL_DISK}" enable_sas_disk $EXPANDER $PHY rescan_disks $EXPANDER # For debugging purposes, log the partial output of # camcontrol to see if the disk actually came back. out=$(camcontrol smpphylist ${EXPANDER} | ${GREP} "^ *${PHY}") log_note "Expander has: ${out}" fi [[ -e $TESTDIR ]] && log_must $RM -rf $TESTDIR/* partition_cleanup restart_zfsd } Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_degrade_001_pos.ksh =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_degrade_001_pos.ksh (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_degrade_001_pos.ksh (revision 292374) @@ -1,89 +1,90 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright (c) 2012,2013 Spectra Logic Corporation. All rights reserved. # Use is subject to license terms. # # $Id: $ # $FreeBSD$ . $STF_SUITE/include/libtest.kshlib +. $STF_SUITE/tests/zfsd/zfsd.kshlib ################################################################################ # # __stc_assertion_start # # ID: zfsd_degrade_001_pos # # DESCRIPTION: # If a vdev experiences checksum errors, it will become degraded. # # # STRATEGY: # 1. Create a storage pool. Only use the file vdevs because it is easy to # generate checksum errors on them. # 2. Mostly fill the pool with data. # 3. Corrupt it by DDing to the underlying vdev # 4. Verify that the vdev becomes DEGRADED. # 5. ONLINE it and verify that it resilvers and joins the pool. # # TESTABILITY: explicit # # TEST_AUTOMATION_LEVEL: automated # # CODING STATUS: COMPLETED (2012-08-09) # # __stc_assertion_end # ############################################################################### verify_runnable "global" VDEV0=${TMPDIR}/file0.${TESTCASE_ID} VDEV1=${TMPDIR}/file1.${TESTCASE_ID} VDEVS="${VDEV0} ${VDEV1}" TESTFILE=/$TESTPOOL/testfile function cleanup { destroy_pool $TESTPOOL $RM -f $VDEVS } log_assert "ZFS will degrade a vdev that produces checksum errors" log_onexit cleanup log_must create_vdevs $VDEV0 $VDEV1 ensure_zfsd_running for type in "raidz" "mirror"; do log_note "Testing raid type $type" create_pool $TESTPOOL $type ${VDEVS} - corrupt_pool $TESTPOOL $VDEV1 $TESTFILE + corrupt_pool_vdev $TESTPOOL $VDEV1 $TESTFILE destroy_pool $TESTPOOL done cleanup log_pass Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_degrade_002_pos.ksh =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_degrade_002_pos.ksh (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_degrade_002_pos.ksh (revision 292374) @@ -1,101 +1,102 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright (c) 2012-2014 Spectra Logic Corporation. All rights reserved. # Use is subject to license terms. # # $Id: $ # $FreeBSD$ . $STF_SUITE/include/libtest.kshlib +. $STF_SUITE/tests/zfsd/zfsd.kshlib ################################################################################ # # __stc_assertion_start # -# ID: zfsd_degrade_001_pos +# ID: zfsd_degrade_002_pos # # DESCRIPTION: # If an active hotspare experiences checksum errors, it will become degraded. # # # STRATEGY: # 1. Create a storage pool with a hotspare. Only use the file vdevs because # it is easy to generate checksum errors on them. # 2. fault a vdev to active the hotspare # 3. Mostly fill the pool with data. # 4. Corrupt it by DDing to the hotspare's underlying file. # 5. Verify that the hotspare becomes DEGRADED. # 6. ONLINE it and verify that it resilvers and joins the pool. # # TESTABILITY: explicit # # TEST_AUTOMATION_LEVEL: automated # # CODING STATUS: COMPLETED (2014-05-13) # # __stc_assertion_end # ############################################################################### verify_runnable "global" VDEV0=${TMPDIR}/file0.${TESTCASE_ID} VDEV1=${TMPDIR}/file1.${TESTCASE_ID} SPARE_VDEV=${TMPDIR}/file2.${TESTCASE_ID} BASIC_VDEVS="${VDEV0} ${VDEV1}" VDEVS="${BASIC_VDEVS} ${SPARE_VDEV}" TESTFILE=/$TESTPOOL/testfile function cleanup { destroy_pool $TESTPOOL $RM -f $VDEVS } -log_assert "ZFS will degrade a vdev that produces checksum errors" +log_assert "ZFS will degrade a spare vdev that produces checksum errors" log_onexit cleanup ensure_zfsd_running log_must create_vdevs $VDEV0 $VDEV1 $SPARE_VDEV for type in "mirror" "raidz"; do log_note "Testing raid type $type" create_pool $TESTPOOL $type ${BASIC_VDEVS} spare ${SPARE_VDEV} # Activate the hotspare $ZINJECT -d ${VDEV0} -A fault $TESTPOOL # ZFSD can take up to 60 seconds to replace a failed device # (though it's usually faster). wait_for_pool_dev_state_change 60 $SPARE_VDEV INUSE - corrupt_pool $TESTPOOL $SPARE_VDEV $TESTFILE + corrupt_pool_vdev $TESTPOOL $SPARE_VDEV $TESTFILE destroy_pool $TESTPOOL done cleanup log_pass Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_hotspare_004_pos.ksh =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_hotspare_004_pos.ksh (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_hotspare_004_pos.ksh (revision 292374) @@ -1,115 +1,115 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2012 Spectra Logic. All rights reserved. # Use is subject to license terms. # # ident "@(#)hotspare_replace_006_pos.ksh 1.0 12/08/10 SL" # . $STF_SUITE/tests/hotspare/hotspare.kshlib . $STF_SUITE/tests/zfsd/zfsd.kshlib . $STF_SUITE/include/libsas.kshlib ################################################################################ # # __stc_assertion_start # # ID: zfs_hotspare_004_pos # # DESCRIPTION: # If a vdev gets removed from a pool with a spare, the spare will be # activated. # # # STRATEGY: # 1. Create 1 storage pools with hot spares. Use disks instead of files # because they can be removed. # 2. Remove one vdev by turning off its SAS phy. # 3. Verify that the spare is in use. # 4. Reinsert the vdev by enabling its phy # 5. Verify that the vdev gets resilvered and the spare gets removed # # TESTABILITY: explicit # # TEST_AUTOMATION_LEVEL: automated # # CODING STATUS: COMPLETED (2012-08-10) # # __stc_assertion_end # ############################################################################### verify_runnable "global" verify_disk_count "$DISKS" 5 log_assert "Removing a disk from a pool results in the spare activating" log_onexit autoreplace_cleanup function verify_assertion # spare_dev { typeset spare_dev=$1 find_verify_sas_disk $REMOVAL_DISK log_note "Disabling \"$REMOVAL_DISK\" on expander $EXPANDER phy $PHY" disable_sas_disk $EXPANDER $PHY # Check to make sure the disk is gone find_disk_by_phy $EXPANDER $PHY [ -n "$FOUNDDISK" ] && log_fail "Disk \"$REMOVAL_DISK\" was not removed" # Check to make sure ZFS sees the disk as removed wait_for_pool_removal 20 # Check that the spare was activated wait_for_pool_dev_state_change 20 $spare_dev INUSE log_must $ZPOOL status $TESTPOOL # Reenable the missing disk log_note "Reenabling phy on expander $EXPANDER phy $PHY" enable_sas_disk $EXPANDER $PHY - wait_for_disk_to_reappear 20 + wait_for_disk_to_reappear 20 $EXPANDER $PHY # Check that the disk has rejoined the pool & resilvered wait_for_pool_dev_state_change 20 $REMOVAL_DISK ONLINE wait_until_resilvered # Finally, check that the spare deactivated - wait_for_pool_state_change $spare_dev AVAIL + wait_for_pool_dev_state_change 20 $spare_dev AVAIL } typeset REMOVAL_DISK=$DISK0 typeset SDEV=$DISK4 typeset POOLDEVS="$DISK0 $DISK1 $DISK2 $DISK3" set -A MY_KEYWORDS "mirror" "raidz1" "raidz2" ensure_zfsd_running for keyword in "${MY_KEYWORDS[@]}" ; do log_must create_pool $TESTPOOL $keyword $POOLDEVS spare $SDEV log_must poolexists "$TESTPOOL" log_must $ZPOOL set autoreplace=on "$TESTPOOL" iterate_over_hotspares verify_assertion $SDEV destroy_pool "$TESTPOOL" done Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_hotspare_007_pos.ksh =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_hotspare_007_pos.ksh (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_hotspare_007_pos.ksh (revision 292374) @@ -1,109 +1,109 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2012 Spectra Logic. All rights reserved. # Use is subject to license terms. # # ident "@(#)hotspare_replace_006_pos.ksh 1.0 12/08/10 SL" # . $STF_SUITE/tests/hotspare/hotspare.kshlib . $STF_SUITE/tests/zfsd/zfsd.kshlib . $STF_SUITE/include/libsas.kshlib ################################################################################ # # __stc_assertion_start # # ID: zfs_hotspare_007_pos # # DESCRIPTION: # If a vdev gets removed from a pool with a spare while zfsd is shut # down, then the spare will be activated when zfsd restarts # # # STRATEGY: # 1. Create 1 storage pools with hot spares. Use disks instead of files # because they can be removed. # 2. Turn off zfsd # 3. Remove one vdev by turning off its SAS phy. # 4. Restart zfsd # 5. Verify that the spare is in use. # # TESTABILITY: explicit # # TEST_AUTOMATION_LEVEL: automated # # CODING STATUS: COMPLETED (2014-09-17) # # __stc_assertion_end # ############################################################################### verify_runnable "global" verify_disk_count "$DISKS" 5 log_assert "zfsd will spare missing drives on startup" log_onexit autoreplace_cleanup function verify_assertion # spare_dev { typeset spare_dev=$1 find_verify_sas_disk $REMOVAL_DISK stop_zfsd log_note "Disabling \"$REMOVAL_DISK\" on expander $EXPANDER phy $PHY" disable_sas_disk $EXPANDER $PHY # Check to make sure the disk is gone find_disk_by_phy $EXPANDER $PHY [ -n "$FOUNDDISK" ] && log_fail "Disk \"$REMOVAL_DISK\" was not removed" # Check to make sure ZFS sees the disk as removed wait_for_pool_removal 20 restart_zfsd # Check that the spare was activated - wait_for_pool_dev_state_change $spare_dev INUSE + wait_for_pool_dev_state_change 20 $spare_dev INUSE # Reenable the missing disk log_note "Reenabling phy on expander $EXPANDER phy $PHY" enable_sas_disk $EXPANDER $PHY - wait_for_disk_to_reappear 20 + wait_for_disk_to_reappear 20 $EXPANDER $PHY } typeset REMOVAL_DISK=$DISK0 typeset SDEV=$DISK4 typeset POOLDEVS="$DISK0 $DISK1 $DISK2 $DISK3" set -A MY_KEYWORDS "mirror" "raidz1" "raidz2" ensure_zfsd_running for keyword in "${MY_KEYWORDS[@]}" ; do log_must create_pool $TESTPOOL $keyword $POOLDEVS spare $SDEV log_must poolexists "$TESTPOOL" iterate_over_hotspares verify_assertion $SDEV destroy_pool "$TESTPOOL" done Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_import_001_pos.ksh =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_import_001_pos.ksh (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_import_001_pos.ksh (revision 292374) @@ -1,161 +1,161 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2013 Spectra Logic. All rights reserved. # Use is subject to license terms. # # ident "@(#)zfsd_zfsd_002_pos.ksh 1.0 12/08/10 SL" # . $STF_SUITE/tests/hotspare/hotspare.kshlib . $STF_SUITE/tests/zfsd/zfsd.kshlib . $STF_SUITE/include/libsas.kshlib ################################################################################ # # __stc_assertion_start # # ID: zfsd_import_001_pos # # DESCRIPTION: # If a removed drive gets reinserted while the pool is exported, it will -# replace its spare when reinserted. +# replace its spare when reimported. # # This also applies to drives that get reinserted while the machine is # powered off. # # # STRATEGY: # 1. Create 1 storage pools with hot spares. Use disks instead of files # because they can be removed. # 2. Remove one disk by turning off its SAS phy. # 3. Verify that the spare is in use. # 4. Reinsert the vdev by enabling its phy # 5. Verify that the vdev gets resilvered and the spare gets removed # 6. Use additional zpool history data to verify that the pool # finished resilvering _before_ zfsd detached the spare. # # TESTABILITY: explicit # # TEST_AUTOMATION_LEVEL: automated # # CODING STATUS: COMPLETED (2012-08-10) # # __stc_assertion_end # ############################################################################### verify_runnable "global" function verify_assertion # spare_dev { typeset spare_dev=$1 find_verify_sas_disk $REMOVAL_DISK log_note "Disabling \"$REMOVAL_DISK\" on expander $EXPANDER phy $PHY" disable_sas_disk $EXPANDER $PHY # Check to make sure the disk is gone find_disk_by_phy $EXPANDER $PHY [ -n "$FOUNDDISK" ] && log_fail "Disk \"$REMOVAL_DISK\" was not removed" # Check to make sure ZFS sees the disk as removed wait_for_pool_removal 20 # Wait for zfsd to activate the spare wait_for_pool_dev_state_change 20 $spare_dev INUSE log_must $ZPOOL status $TESTPOOL # Export the pool log_must $ZPOOL export $TESTPOOL # Reenable the missing disk log_note "Reenabling phy on expander $EXPANDER phy $PHY" enable_sas_disk $EXPANDER $PHY # Check that the disk has returned - wait_for_disk_to_reappear 20 + wait_for_disk_to_reappear 20 $EXPANDER $PHY # Import the pool log_must $ZPOOL import $TESTPOOL # Check that the disk has rejoined the pool wait_for_pool_dev_state_change 20 $REMOVAL_DISK ONLINE # Check that the pool resilvered while ! is_pool_resilvered $TESTPOOL; do $SLEEP 2 done - log_must $ZPOOL status + log_must $ZPOOL status $TESTPOOL #Finally, check that the spare deactivated wait_for_pool_dev_state_change 20 $spare_dev AVAIL # Verify that the spare was detached after the scrub was complete # Note that resilvers and scrubs are recorded identically in zpool # history $ZPOOL history -i $TESTPOOL | awk ' BEGIN { scrub_txg=0; detach_txg=0 } /scrub done/ { split($6, s, "[:\\]]"); t=s[2]; scrub_txg = scrub_txg > t ? scrub_txg : t } /vdev detach/ { split($6, s, "[:\\]]"); t=s[2]; done_txg = done_txg > t ? done_txg : t } END { print("Scrub completed at txg", scrub_txg); print("Spare detached at txg", detach_txg); exit(detach_txg > scrub_txg) }' [ $? -ne 0 ] && log_fail "The spare detached before the resilver completed" } if ! $(is_physical_device $DISKS) ; then log_unsupported "This directory cannot be run on raw files." fi log_assert "If a removed drive gets reinserted while the pool is exported, \ it will replace its spare when reinserted." log_onexit autoreplace_cleanup ensure_zfsd_running set_devs typeset REMOVAL_DISK=$DISK0 typeset SDEV=$DISK4 typeset POOLDEVS="$DISK0 $DISK1 $DISK2 $DISK3" set -A MY_KEYWORDS "mirror" "raidz1" "raidz2" for keyword in "${MY_KEYWORDS[@]}" ; do log_must create_pool $TESTPOOL $keyword $POOLDEVS spare $SDEV log_must poolexists "$TESTPOOL" iterate_over_hotspares verify_assertion $SDEV destroy_pool "$TESTPOOL" done Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_001_pos.ksh =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_001_pos.ksh (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_001_pos.ksh (revision 292374) @@ -1,93 +1,93 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2008 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # Copyright 2012,2013 Spectra Logic Corporation. All rights reserved. # Use is subject to license terms. # # Portions taken from: # ident "@(#)replacement_001_pos.ksh 1.4 08/02/27 SMI" # # $Id: //SpectraBSD/stable/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_001_pos.ksh#2 $ # $FreeBSD$ . $STF_SUITE/tests/hotspare/hotspare.kshlib . $STF_SUITE/tests/zfsd/zfsd.kshlib . $STF_SUITE/include/libtest.kshlib . $STF_SUITE/include/libsas.kshlib verify_runnable "global" log_assert "Failing a disk from a SAS expander is recognized by ZFS" log_onexit autoreplace_cleanup ensure_zfsd_running child_pids="" set -A TMPDISKS $DISKS typeset REMOVAL_DISK=${TMPDISKS[0]} REMOVAL_DISK=${REMOVAL_DISK##*/} for type in "raidz" "mirror"; do # Create a pool on the supplied disks create_pool $TESTPOOL $type $DISKS log_must $ZFS create $TESTPOOL/$TESTFS log_must $ZFS set mountpoint=$TESTDIR $TESTPOOL/$TESTFS # Find the first disk, get the expander and phy log_note "Looking for expander and phy information for $REMOVAL_DISK" find_verify_sas_disk $REMOVAL_DISK log_note "Disabling \"$REMOVAL_DISK\" on expander $EXPANDER phy $PHY" # Disable the first disk. We have to do this first, because if # there is I/O active to the disable_sas_disk $EXPANDER $PHY # Check to make sure disk is gone. find_disk_by_phy $EXPANDER $PHY [ -n "$FOUNDDISK" ] && log_fail "Disk \"$REMOVAL_DISK\" was not removed" # Write out data to make sure we can do I/O after the disk failure log_must $DD if=/dev/zero of=$TESTDIR/$TESTFILE bs=1m count=512 # Check to make sure ZFS sees the disk as removed wait_for_pool_removal 20 # Re-enable the disk, we don't want to leave it turned off log_note "Re-enabling phy $PHY on expander $EXPANDER" enable_sas_disk $EXPANDER $PHY - wait_for_disk_to_reappear 20 + wait_for_disk_to_reappear 20 $EXPANDER $PHY # Disk should auto-join the zpool & be resilvered. wait_for_pool_dev_state_change 20 $REMOVAL_DISK ONLINE wait_until_resilvered $ZPOOL status $TESTPOOL destroy_pool $TESTPOOL log_must $RM -rf /$TESTPOOL done log_pass Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_003_pos.ksh =================================================================== --- projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_003_pos.ksh (revision 292373) +++ projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_003_pos.ksh (revision 292374) @@ -1,150 +1,154 @@ #!/usr/local/bin/ksh93 -p # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2008 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # Copyright 2012,2013 Spectra Logic Corporation. All rights reserved. # Use is subject to license terms. # # Portions taken from: # ident "@(#)replacement_001_pos.ksh 1.4 08/02/27 SMI" # # $Id: //SpectraBSD/stable/cddl/tools/regression/stc/src/suites/fs/zfs/tests/functional/zfsd/zfsd_replace_003_pos.ksh#1 $ # $FreeBSD$ . $STF_SUITE/include/libtest.kshlib . $STF_SUITE/include/libsas.kshlib +. $STF_SUITE/tests/hotspare/hotspare.kshlib . $STF_SUITE/tests/zfsd/zfsd.kshlib verify_runnable "global" function cleanup { # See if the phy has been disabled, and try to re-enable it if possible. [ -n "$EXPANDER0" -a -n "$PHY0" ] && enable_sas_disk $EXPANDER0 $PHY0 [ -n "$EXPANDER1" -a -n "$PHY1" ] && enable_sas_disk $EXPANDER1 $PHY1 [ -n "$EXPANDER" -a -n "$PHY" ] && enable_sas_disk $EXPANDER $PHY destroy_pool $TESTPOOL [[ -e $TESTDIR ]] && log_must $RM -rf $TESTDIR/* } # arg1: disk devname # Leaves EXPANDER and PHY set appropriately function remove_disk { typeset DISK=$1 # Find the first disk, get the expander and phy log_note "Looking for expander and phy information for $DISK" find_verify_sas_disk $DISK log_note "Disabling \"$DISK\" on expander $EXPANDER phy $PHY" # Disable the first disk. disable_sas_disk $EXPANDER $PHY # Check to make sure disk is gone. find_disk_by_phy $EXPANDER $PHY [ -n "$FOUNDDISK" ] && log_fail "Disk \"$DISK\" was not removed" # Check to make sure ZFS sees the disk as removed - wait_for_pool_removal 20 + wait_for_pool_dev_state_change 20 $DISK REMOVED } # arg1: disk's old devname # arg2: disk's expander's devname # arg3: disk's phy number # arg4: whether the devname must differ after reconnecting function reconnect_disk { typeset DISK=$1 typeset EXPANDER=$2 typeset PHY=$3 - typeset DEVNAME_DIFFERS=$4 # Re-enable the disk, we don't want to leave it turned off log_note "Re-enabling phy $PHY on expander $EXPANDER" enable_sas_disk $EXPANDER $PHY log_note "Checking to see whether disk has reappeared" # Make sure the disk is back in the topology - wait_for_disk_to_reappear 20 + wait_for_disk_to_reappear 20 $EXPANDER $PHY - if [ -n "$DEVNAME_MUST_DIFFER" ]; then - prev_disks=$(find_disks $FOUNDDISK) - cur_disks=$(find_disks $DISK) + prev_disk=$(find_disks $DISK) + cur_disk=$(find_disks $FOUNDDISK) - # If you get this, the test must be fixed to guarantee that - # it will reappear with a different name. - [ "${prev_disk}" = "${cur_disk}" ] && log_unsupported \ - "Disk $DISK reappeared with the same devname." - fi + # If you get this, the test must be fixed to guarantee that + # it will reappear with a different name. + [ "${prev_disk}" = "${cur_disk}" ] && log_unsupported \ + "Disk $DISK reappeared with the same devname." #Disk should have auto-joined the zpool. Verify it's status is online. - wait_for_pool_dev_state_change 20 $DISK ONLINE + wait_for_pool_dev_state_change 20 $FOUNDDISK ONLINE } log_assert "ZFSD will correctly replace disks that disappear and reappear \ with different devnames" # Outline # Create a double-parity pool # Remove two disks by disabling their SAS phys # Reenable the phys in the opposite order # Check that the disks's devnames have swapped # Verify that the pool regains its health log_onexit cleanup ensure_zfsd_running child_pids="" set -A DISKS_ARRAY $DISKS typeset DISK0=${DISKS_ARRAY[0]} typeset DISK1=${DISKS_ARRAY[1]} +if [ ${DISK0##/dev/da} -gt ${DISK1##/dev/da} ]; then + # Swap disks so we'll disable the lowest numbered first + typeset TMP="$DISK1" + DISK1="$DISK0" + DISK0="$TMP" +fi for type in "raidz2" "mirror"; do # Create a pool on the supplied disks create_pool $TESTPOOL $type $DISKS remove_disk $DISK0 typeset EXPANDER0=$EXPANDER typeset PHY0=$PHY remove_disk $DISK1 typeset EXPANDER1=$EXPANDER typeset PHY1=$PHY # Make sure that the pool is degraded $ZPOOL status $TESTPOOL |grep "state:" |grep DEGRADED > /dev/null if [ $? != 0 ]; then log_fail "Pool $TESTPOOL not listed as DEGRADED" fi reconnect_disk $DISK1 $EXPANDER1 $PHY1 reconnect_disk $DISK0 $EXPANDER0 $PHY0 wait_until_resilvered destroy_pool $TESTPOOL log_must $RM -rf /$TESTPOOL done log_pass