Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/hotspare/hotspare_replace_003_neg.ksh
===================================================================
--- projects/zfsd/head/tests/sys/cddl/zfs/tests/hotspare/hotspare_replace_003_neg.ksh	(revision 323117)
+++ projects/zfsd/head/tests/sys/cddl/zfs/tests/hotspare/hotspare_replace_003_neg.ksh	(revision 323118)
@@ -1,176 +1,183 @@
 #!/usr/local/bin/ksh93
 #
 # Copyright (c) 2010 Spectra Logic Corporation
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions, and the following disclaimer,
 #    without modification.
 # 2. Redistributions in binary form must reproduce at minimum a disclaimer       
 #    substantially similar to the "NO WARRANTY" disclaimer below
 #    ("Disclaimer") and any redistribution must be conditioned upon
 #    including a substantially similar Disclaimer requirement for further
 #    binary redistribution.
 #
 # NO WARRANTY
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL     
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGES.
 #
 # $FreeBSD$
 
 . $STF_SUITE/include/libtest.kshlib
 . $STF_SUITE/include/libsas.kshlib
 . $STF_SUITE/tests/hotspare/hotspare.kshlib
 
-# Reproduction script for Rally DE189:
-# https://rally1.rallydev.com/#/9096795496d/detail/defect/13345916506
+# "zpool import" should not show destroyed pools, even if a non-destroyed label
+# is present.
 #
-# To reproduce, from Keith's email:
+# This situation arose when a user activated a spare, removed the spare disk,
+# destroyed the pool, reinserted the spare disk, and then tried to import the
+# pool.  Since the pool was destroyed, nothing should've happened.  But the
+# spare disk had a non-destroyed label, so zpool tried to import it.  A panic
+# ensued.
+#
+# To reproduce:
 # 1.  Create a 3 drive raid.
 # 2.  Disable a drive
 # 3.  Let a Spare take over
 # 4.  Disable that Spare when it is done rebuilding.
 # 5.  Let 2nd spare finish rebuilding
 # 6.  Enable the first spare
 # 7.  Enable the original drive
 # 8.  Destroy the pool.
 
 cleanup() {
 	[[ $DISK0_PHY != 0 ]] && enable_sas_disk $DISK0_EXPANDER $DISK0_PHY
 	[[ $SPARE0_PHY != 0 ]] && enable_sas_disk $SPARE0_EXPANDER $SPARE0_PHY
 	[[ $SPARE1_PHY != 0 ]] && enable_sas_disk $SPARE1_EXPANDER $SPARE1_PHY
 	rescan_disks
 	if poolexists $TESTPOOL; then
 		# Test failed, provide something useful.
 		log_note "For reference, here is the final $TESTPOOL status:"
 		zpool status $TESTPOOL
 		log_must destroy_pool $TESTPOOL
 	fi
+	cleanup_devices ${DISKS}
 }
 
 log_onexit cleanup
 trap cleanup TERM INT
 
 typeset -A MEMBERS
 typeset -A SPARES
 typeset DISK0_EXPANDER=0
 typeset DISK0_PHY=0
 typeset SPARE0_EXPANDER=0
 typeset SPARE0_PHY=0
 typeset SPARE1_EXPANDER=0
 typeset SPARE1_PHY=0
 
 for disk in $DISKS; do
 	if [[ $DISK0_PHY == 0 ]]; then
 		find_verify_sas_disk $disk
 		DISK0_PHY=$PHY
 		DISK0_EXPANDER=$EXPANDER
 		DISK0_NAME=$disk
 		set -A MEMBERS "${MEMBERS[@]}" $disk
 		continue
 	fi
 	if [[ $SPARE0_PHY == 0 ]]; then
 		find_verify_sas_disk $disk
 		SPARE0_PHY=$PHY
 		SPARE0_EXPANDER=$EXPANDER
 		SPARE0_NAME=$disk
 		set -A SPARES "${SPARES[@]}" $disk
 		continue
 	fi
 	if [[ $SPARE1_PHY == 0 ]]; then
 		find_verify_sas_disk $disk
 		SPARE1_PHY=$PHY
 		SPARE1_EXPANDER=$EXPANDER
 		SPARE1_NAME=$disk
 		set -A SPARES "${SPARES[@]}" $disk
 		continue
 	fi
 	# Already filled those positions?  Add disks to the raidz.
 	if [[ ${#DISKS[*]} -lt 3 ]]; then
 		find_verify_sas_disk $disk
 		[[ -z "$DISK1_NAME" ]] && DISK1_NAME=$disk
 		DISK1_LONG_NAME=`find_disks ${DISK1_NAME}`
 		DISK1_SHORT_NAME=${DISK1_LONG_NAME##/dev/}
 		set -A MEMBERS "${MEMBERS[@]}" $disk
 		continue
 	fi
 	break
 done
 
 # Remove labels etc. from all the disks we're about to use.
 poolexists && log_must destroy_pool $TESTPOOL
 cleanup_devices ${MEMBERS[*]} ${SPARES[*]}
 
 log_must $ZPOOL create -f $TESTPOOL raidz1 ${MEMBERS[@]} spare ${SPARES[@]}
 DISK0_GUID=$(get_disk_guid $DISK0_NAME)
 
-log_must disable_sas_disk $DISK0_EXPANDER $DISK0_PHY
+disable_sas_disk $DISK0_EXPANDER $DISK0_PHY
 log_must $ZPOOL replace $TESTPOOL $DISK0_GUID $SPARE0_NAME
 wait_until_resilvered
 SPARE0_GUID=$(get_disk_guid $SPARE0_NAME)
 
-log_must disable_sas_disk $SPARE0_EXPANDER $SPARE0_PHY
+disable_sas_disk $SPARE0_EXPANDER $SPARE0_PHY
 log_must $ZPOOL replace $TESTPOOL $SPARE0_GUID $SPARE1_NAME
 wait_until_resilvered
 
-log_must enable_sas_disk $SPARE0_EXPANDER $SPARE0_PHY
-log_must enable_sas_disk $DISK0_EXPANDER $DISK0_PHY
-log_must rescan_disks
+enable_sas_disk $SPARE0_EXPANDER $SPARE0_PHY
+enable_sas_disk $DISK0_EXPANDER $DISK0_PHY
+rescan_disks
 
 log_must destroy_pool $TESTPOOL
 
 # Screen scrape the 'zpool import' output to ensure that the pool doesn't
 # show up, since it's been destroyed.
 badpoolstate=0
 $ZPOOL import | \
 	while read word1 word2 rest; do
 		echo "$word1 $word2 $rest"
 		case "$word1 $word2" in
 			"pool: $TESTPOOL") (( badpoolstate += 1 )) ;;
 			"$DISK1_LONG_NAME ONLINE") (( badpoolstate += 2 )) ;;
 			"$DISK1_SHORT_NAME ONLINE") (( badpoolstate += 2 )) ;;
 			*) ;;
 		esac
 	done
 
 case $badpoolstate in
 	0)	log_pass ;;
 	1)	log_note "Destroyed pool visible, but probably another pool"
 		log_pass ;;
 	2)	log_fail "One of our disks is visible but pool is not?!" ;;
 	3)	log_fail "Destroyed pool visible!" ;;
 	*)	log_fail "Unexpected output.  Update the test" ;;
 esac
 
 # Bad output looks like this:
 #   pool: testpool.2358
 #     id: 5289863802080396071
 #  state: UNAVAIL
 # status: One or more devices are missing from the system.
 # action: The pool cannot be imported. Attach the missing
 #    devices and try again.
 #   see: http://illumos.org/msg/ZFS-8000-6X
 # config:
 #
 #    testpool.2358              UNAVAIL  missing device
 #      raidz1-0                 DEGRADED
 #        spare-0                UNAVAIL
 #          4564766431272474500  FAULTED  corrupted data
 #          1988887717739330825  FAULTED  corrupted data
 #        da4                    ONLINE
 #        da5                    ONLINE
 #    spares
 #      1988887717739330825
 #      da3
Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/hotspare/hotspare_test.sh
===================================================================
--- projects/zfsd/head/tests/sys/cddl/zfs/tests/hotspare/hotspare_test.sh	(revision 323117)
+++ projects/zfsd/head/tests/sys/cddl/zfs/tests/hotspare/hotspare_test.sh	(revision 323118)
@@ -1,790 +1,789 @@
 # CDDL HEADER START
 #
 # The contents of this file are subject to the terms of the
 # Common Development and Distribution License (the "License").
 # You may not use this file except in compliance with the License.
 #
 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 # or http://www.opensolaris.org/os/licensing.
 # See the License for the specific language governing permissions
 # and limitations under the License.
 #
 # When distributing Covered Code, include this CDDL HEADER in each
 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 # If applicable, add the following below this CDDL HEADER, with the
 # fields enclosed by brackets "[]" replaced with your own identifying
 # information: Portions Copyright [yyyy] [name of copyright owner]
 #
 # CDDL HEADER END
 #
 
 #
 # Copyright 2012 Spectra Logic.  All rights reserved.
 # Use is subject to license terms.
 #
 
 
 atf_test_case hotspare_add_001_pos cleanup
 hotspare_add_001_pos_head()
 {
 	atf_set "descr" "'zpool add <pool> spare <vdev> ...' can add devices to the pool."
 	atf_set "timeout" 3600
 }
 hotspare_add_001_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_add_001_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_add_001_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_add_002_pos cleanup
 hotspare_add_002_pos_head()
 {
 	atf_set "descr" "'zpool add <pool> spare <vdev> ...' can add devices to the pool while it has spare-in device."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_add_002_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_add_002_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_add_002_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_add_003_neg cleanup
 hotspare_add_003_neg_head()
 {
 	atf_set "descr" "'zpool add [-f]' with hot spares should fail with inapplicable scenarios."
 	atf_set "require.config" disks_are_physical
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_add_003_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_add_003_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_add_003_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_add_004_neg cleanup
 hotspare_add_004_neg_head()
 {
 	atf_set "descr" "'zpool add [-f]' will not allow a swap device to be used as a hotspare'"
 	atf_set "require.config" disks_are_physical
 	atf_set "require.progs"  zpool swapon swapoff swapctl
 }
 hotspare_add_004_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_add_004_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_add_004_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_clone_001_pos cleanup
 hotspare_clone_001_pos_head()
 {
 	atf_set "descr" "'zpool detach <pool> <vdev> ...' against hotspare should do no harm to clone."
 	atf_set "require.progs"  zfs zpool sum
 	atf_set "timeout" 3600
 }
 hotspare_clone_001_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_clone_001_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_clone_001_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_clone_002_pos cleanup
 hotspare_clone_002_pos_head()
 {
 	atf_set "descr" "'zpool detach <pool> <vdev> ...' against basic vdev should do no harm to clone."
 	atf_set "require.progs"  zfs zpool sum
 	atf_set "timeout" 3600
 }
 hotspare_clone_002_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_clone_002_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_clone_002_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_create_001_neg cleanup
 hotspare_create_001_neg_head()
 {
 	atf_set "descr" "'zpool create [-f]' with hot spares should be failedwith inapplicable scenarios."
 	atf_set "require.config" disks_are_physical
 	atf_set "require.progs"  dumpadm zpool
 	atf_set "timeout" 3600
 }
 hotspare_create_001_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_create_001_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_create_001_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_detach_001_pos cleanup
 hotspare_detach_001_pos_head()
 {
 	atf_set "descr" "'zpool detach <pool> <vdev> ...' should deactivate the spared-in hot spare device successfully."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_detach_001_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_detach_001_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_detach_001_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_detach_002_pos cleanup
 hotspare_detach_002_pos_head()
 {
 	atf_set "descr" "'zpool detach <pool> <vdev> ...' against a functioning device that have spared should take the hot spare permanently swapping in successfully."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_detach_002_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_detach_002_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_detach_002_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_detach_003_pos cleanup
 hotspare_detach_003_pos_head()
 {
 	atf_set "descr" "'zpool replace <pool> <vdev> <ndev>' against a functioning device that have spared should complete and the hot spare should return to available."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_detach_003_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_detach_003_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_detach_003_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_detach_004_pos cleanup
 hotspare_detach_004_pos_head()
 {
 	atf_set "descr" "'zpool replace <pool> <vdev> <ndev>' against a hot spare device that have been activated should successful while the another dev is a available hot spare."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_detach_004_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_detach_004_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_detach_004_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_detach_005_neg cleanup
 hotspare_detach_005_neg_head()
 {
 	atf_set "descr" "'zpool detach <pool> <vdev>' against a hot spare device that NOT activated should fail and issue an error message."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_detach_005_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_detach_005_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_detach_005_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_export_001_neg cleanup
 hotspare_export_001_neg_head()
 {
 	atf_set "descr" "export pool that using shared hotspares will fail"
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_export_001_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_export_001_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_export_001_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_import_001_pos cleanup
 hotspare_import_001_pos_head()
 {
 	atf_set "descr" "'zpool export/import <pool>' should runs successfully regardless the hotspare is only in list, activated, or offline."
 	atf_set "require.progs"  zpool sum
 	atf_set "timeout" 3600
 }
 hotspare_import_001_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_import_001_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_import_001_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_onoffline_003_neg cleanup
 hotspare_onoffline_003_neg_head()
 {
 	atf_set "descr" "'zpool offline/online <pool> <vdev>' against a hot spare works as expect."
 	atf_set "require.progs"  zpool zdb
 	atf_set "timeout" 3600
 }
 hotspare_onoffline_003_neg_body()
 {
 	atf_expect_fail "This testcase has always failed on FreeBSD"
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_onoffline_003_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_onoffline_003_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_onoffline_004_neg cleanup
 hotspare_onoffline_004_neg_head()
 {
 	atf_set "descr" "'zpool offline/online <pool> <vdev>' against a spared basic vdev during I/O completes."
 	atf_set "require.progs"  zfs zpool zdb
 	atf_set "timeout" 3600
 }
 hotspare_onoffline_004_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_onoffline_004_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_onoffline_004_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_remove_001_pos cleanup
 hotspare_remove_001_pos_head()
 {
 	atf_set "descr" "'zpool remove <pool> <vdev> ...' can remove spare device from the pool."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_remove_001_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_remove_001_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_remove_001_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_remove_002_neg cleanup
 hotspare_remove_002_neg_head()
 {
 	atf_set "descr" "'zpool remove <pool> <vdev> ...' should fail with inapplicable scenarios."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_remove_002_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_remove_002_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_remove_002_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_remove_003_neg cleanup
 hotspare_remove_003_neg_head()
 {
 	atf_set "descr" "Executing 'zpool remove' with bad options fails"
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_remove_003_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_remove_003_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_remove_003_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_remove_004_pos cleanup
 hotspare_remove_004_pos_head()
 {
 	atf_set "descr" "'zpool remove <pool> <vdev> ...' can remove spare device from the pool."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_remove_004_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_remove_004_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_remove_004_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_replace_001_neg cleanup
 hotspare_replace_001_neg_head()
 {
 	atf_set "descr" "'zpool replace <pool> <odev> <ndev>' should fail with inapplicable scenarios."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_replace_001_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_replace_001_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_replace_001_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_replace_002_neg cleanup
 hotspare_replace_002_neg_head()
 {
 	atf_set "descr" "'zpool replace <pool> <odev> <ndev>' should fail while the hot spares smaller than the basic vdev."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_replace_002_neg_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_replace_002_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_replace_002_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_replace_003_neg cleanup
 hotspare_replace_003_neg_head()
 {
 	atf_set "descr" "'zpool replace' of disabled hotspares should result in ignoring them after destroy."
 	atf_set "require.progs"  camcontrol zpool
 	atf_set "timeout" 3600
 }
 hotspare_replace_003_neg_body()
 {
-	atf_expect_fail "Rally: DE189: ZFS config management needs improvement"
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	verify_disk_count "$DISKS" 5
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_replace_003_neg.ksh || atf_fail "Testcase failed"
 }
 hotspare_replace_003_neg_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_scrub_001_pos cleanup
 hotspare_scrub_001_pos_head()
 {
 	atf_set "descr" "'zpool scrub <pool>' should runs successfully regardlessthe hotspare is only in list or activated."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_scrub_001_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_scrub_001_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_scrub_001_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_scrub_002_pos cleanup
 hotspare_scrub_002_pos_head()
 {
 	atf_set "descr" "'zpool scrub' scans spare vdevs"
 	atf_set "require.progs"  zpool
 }
 hotspare_scrub_002_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_scrub_002_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_scrub_002_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_shared_001_pos cleanup
 hotspare_shared_001_pos_head()
 {
 	atf_set "descr" "'zpool add <pool> spare <vdev> ...' can add a disk as a shared spare to multiple pools."
 	atf_set "require.progs"  zpool
 	atf_set "timeout" 3600
 }
 hotspare_shared_001_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	verify_disk_count "$DISKS" 5
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_shared_001_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_shared_001_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_snapshot_001_pos cleanup
 hotspare_snapshot_001_pos_head()
 {
 	atf_set "descr" "'zpool detach <pool> <vdev> ...' against hotspare should do no harm to snapshot."
 	atf_set "require.progs"  zfs zpool sum
 	atf_set "timeout" 3600
 }
 hotspare_snapshot_001_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_snapshot_001_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_snapshot_001_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case hotspare_snapshot_002_pos cleanup
 hotspare_snapshot_002_pos_head()
 {
 	atf_set "descr" "'zpool detach <pool> <vdev> ...' against basic vdev do no harm to snapshot."
 	atf_set "require.progs"  zfs zpool sum
 	atf_set "timeout" 3600
 }
 hotspare_snapshot_002_pos_body()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/hotspare_snapshot_002_pos.ksh || atf_fail "Testcase failed"
 }
 hotspare_snapshot_002_pos_cleanup()
 {
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/hotspare.kshlib
 	. $(atf_get_srcdir)/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_init_test_cases()
 {
 
 	atf_add_test_case hotspare_add_001_pos
 	atf_add_test_case hotspare_add_002_pos
 	atf_add_test_case hotspare_add_003_neg
 	atf_add_test_case hotspare_add_004_neg
 	atf_add_test_case hotspare_clone_001_pos
 	atf_add_test_case hotspare_clone_002_pos
 	atf_add_test_case hotspare_create_001_neg
 	atf_add_test_case hotspare_detach_001_pos
 	atf_add_test_case hotspare_detach_002_pos
 	atf_add_test_case hotspare_detach_003_pos
 	atf_add_test_case hotspare_detach_004_pos
 	atf_add_test_case hotspare_detach_005_neg
 	atf_add_test_case hotspare_export_001_neg
 	atf_add_test_case hotspare_import_001_pos
 	atf_add_test_case hotspare_onoffline_003_neg
 	atf_add_test_case hotspare_onoffline_004_neg
 	atf_add_test_case hotspare_remove_001_pos
 	atf_add_test_case hotspare_remove_002_neg
 	atf_add_test_case hotspare_remove_003_neg
 	atf_add_test_case hotspare_remove_004_pos
 	atf_add_test_case hotspare_replace_001_neg
 	atf_add_test_case hotspare_replace_002_neg
 	atf_add_test_case hotspare_replace_003_neg
 	atf_add_test_case hotspare_scrub_001_pos
 	atf_add_test_case hotspare_scrub_002_pos
 	atf_add_test_case hotspare_shared_001_pos
 	atf_add_test_case hotspare_snapshot_001_pos
 	atf_add_test_case hotspare_snapshot_002_pos
 }