Index: projects/zfsd/head/cddl/sbin/zfsd/zfsd.cc
===================================================================
--- projects/zfsd/head/cddl/sbin/zfsd/zfsd.cc	(revision 292377)
+++ projects/zfsd/head/cddl/sbin/zfsd/zfsd.cc	(revision 292378)
@@ -1,412 +1,447 @@
 /*-
  * Copyright (c) 2011, 2012, 2013 Spectra Logic Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions, and the following disclaimer,
  *    without modification.
  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  *    substantially similar to the "NO WARRANTY" disclaimer below
  *    ("Disclaimer") and any redistribution must be conditioned upon
  *    including a substantially similar Disclaimer requirement for further
  *    binary redistribution.
  *
  * NO WARRANTY
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGES.
  *
  * Authors: Justin T. Gibbs     (Spectra Logic Corporation)
  */
 
 /**
  * \file zfsd.cc
  *
  * The ZFS daemon consumes kernel devctl(4) event data via devd(8)'s
  * unix domain socket in order to react to system changes that impact
  * the function of ZFS storage pools.  The goal of this daemon is to
  * provide similar functionality to the Solaris ZFS Diagnostic Engine
  * (zfs-diagnosis), the Solaris ZFS fault handler (zfs-retire), and
  * the Solaris ZFS vdev insertion agent (zfs-mod sysevent handler).
  */
 
 #include <sys/cdefs.h>
 #include <sys/param.h>
 #include <sys/fs/zfs.h>
 
 #include <err.h>
 #include <libgeom.h>
 #include <libutil.h>
 #include <poll.h>
 #include <syslog.h>
 
 #include <libzfs.h>
 
 #include <list>
 #include <map>
 #include <string>
 
 #include <devctl/guid.h>
 #include <devctl/event.h>
 #include <devctl/event_factory.h>
 #include <devctl/exception.h>
 #include <devctl/consumer.h>
 
 #include "callout.h"
 #include "vdev_iterator.h"
 #include "zfsd_event.h"
 #include "case_file.h"
 #include "vdev.h"
 #include "vdev_iterator.h"
 #include "zfsd.h"
 #include "zfsd_exception.h"
 #include "zpool_list.h"
 
 __FBSDID("$FreeBSD$");
 
 /*================================== Macros ==================================*/
 #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
 
 /*============================ Namespace Control =============================*/
 using DevCtl::Event;
 using DevCtl::EventFactory;
 using DevCtl::EventList;
 
 /*================================ Global Data ===============================*/
 int              g_debug = 0;
 libzfs_handle_t *g_zfsHandle;
 
 /*--------------------------------- ZfsDaemon --------------------------------*/
 //- ZfsDaemon Static Private Data ----------------------------------------------
 ZfsDaemon	    *ZfsDaemon::s_theZfsDaemon;
 bool		     ZfsDaemon::s_logCaseFiles;
 bool		     ZfsDaemon::s_terminateEventLoop;
 char		     ZfsDaemon::s_pidFilePath[] = "/var/run/zfsd.pid";
 pidfh		    *ZfsDaemon::s_pidFH;
 int		     ZfsDaemon::s_signalPipeFD[2];
 bool		     ZfsDaemon::s_systemRescanRequested(false);
 EventFactory::Record ZfsDaemon::s_registryEntries[] =
 {
 	{ Event::NOTIFY, "DEVFS", &DevfsEvent::Builder },
 	{ Event::NOTIFY, "ZFS",   &ZfsEvent::Builder }
 };
 
 //- ZfsDaemon Static Public Methods --------------------------------------------
 ZfsDaemon &
 ZfsDaemon::Get()
 {
 	return (*s_theZfsDaemon);
 }
 
 void
 ZfsDaemon::WakeEventLoop()
 {
 	write(s_signalPipeFD[1], "+", 1);
 }
 
 void
 ZfsDaemon::RequestSystemRescan()
 {
 	s_systemRescanRequested = true;
 	ZfsDaemon::WakeEventLoop();
 }
 
 void
 ZfsDaemon::Run()
 {
 	ZfsDaemon daemon;
 
 	while (s_terminateEventLoop == false) {
 
 		try {
 			daemon.DisconnectFromDevd();
 
 			if (daemon.ConnectToDevd() == false) {
 				sleep(30);
 				continue;
 			}
 
 			daemon.DetectMissedEvents();
 
 			daemon.EventLoop();
 
 		} catch (const DevCtl::Exception &exp) {
 			exp.Log();
 		}
 	}
 
 	daemon.DisconnectFromDevd();
 }
 
 //- ZfsDaemon Private Methods --------------------------------------------------
 ZfsDaemon::ZfsDaemon()
  : Consumer(/*defBuilder*/NULL, s_registryEntries,
 	    NUM_ELEMENTS(s_registryEntries))
 {
 	if (s_theZfsDaemon != NULL)
 		errx(1, "Multiple ZfsDaemon instances created. Exiting");
 
 	s_theZfsDaemon = this;
 
 	if (pipe(s_signalPipeFD) != 0)
 		errx(1, "Unable to allocate signal pipe. Exiting");
 
 	if (fcntl(s_signalPipeFD[0], F_SETFL, O_NONBLOCK) == -1)
 		errx(1, "Unable to set pipe as non-blocking. Exiting");
 
 	if (fcntl(s_signalPipeFD[1], F_SETFL, O_NONBLOCK) == -1)
 		errx(1, "Unable to set pipe as non-blocking. Exiting");
 
 	signal(SIGHUP,  ZfsDaemon::RescanSignalHandler);
 	signal(SIGINFO, ZfsDaemon::InfoSignalHandler);
 	signal(SIGINT,  ZfsDaemon::QuitSignalHandler);
 	signal(SIGTERM, ZfsDaemon::QuitSignalHandler);
 	signal(SIGUSR1, ZfsDaemon::RescanSignalHandler);
 
 	g_zfsHandle = libzfs_init();
 	if (g_zfsHandle == NULL)
 		errx(1, "Unable to initialize ZFS library. Exiting");
 
 	Callout::Init();
 	InitializeSyslog();
 	OpenPIDFile();
 
 	if (g_debug == 0)
 		daemon(0, 0);
 
 	UpdatePIDFile();
 }
 
 ZfsDaemon::~ZfsDaemon()
 {
 	PurgeCaseFiles();
 	ClosePIDFile();
 }
 
 void
 ZfsDaemon::PurgeCaseFiles()
 {
 	CaseFile::PurgeAll();
 }
 
 bool
 ZfsDaemon::VdevAddCaseFile(Vdev &vdev, void *cbArg)
 {
 	if (vdev.State() != VDEV_STATE_HEALTHY)
 		CaseFile::Create(vdev);
 
 	return (/*break early*/false);
 }
 
 void
 ZfsDaemon::BuildCaseFiles()
 {
-	/* Add CaseFiles for vdevs with issues. */
 	ZpoolList zpl;
+	ZpoolList::iterator pool;
 
-	for (ZpoolList::iterator pool = zpl.begin(); pool != zpl.end(); pool++)
+	/* Add CaseFiles for vdevs with issues. */
+	for (pool = zpl.begin(); pool != zpl.end(); pool++)
 		VdevIterator(*pool).Each(VdevAddCaseFile, NULL);
 
 	/* De-serialize any saved cases. */
 	CaseFile::DeSerialize();
+
+	/* Simulate config_sync events to force CaseFile reevaluation */
+	for (pool = zpl.begin(); pool != zpl.end(); pool++) {
+		char evString[160];
+		Event *event;
+		nvlist_t *config;
+		uint64_t poolGUID;
+		const char *poolname;
+
+		poolname = zpool_get_name(*pool);
+		config = zpool_get_config(*pool, NULL);
+		if (config == NULL) {
+			syslog(LOG_ERR, "ZFSDaemon::BuildCaseFiles: Could not "
+			    "find pool config for pool %s", poolname);
+			continue;
+		}
+		if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
+				     &poolGUID) != 0) {
+			syslog(LOG_ERR, "ZFSDaemon::BuildCaseFiles: Could not "
+			    "find pool guid for pool %s", poolname);
+			continue;
+		}
+
+		
+		snprintf(evString, 160, "!system=ZFS subsystem=ZFS "
+		    "type=misc.fs.zfs.config_sync sub_type=synthesized "
+		    "pool_name=%s pool_guid=%lu\n", poolname, poolGUID);
+		event = Event::CreateEvent(GetFactory(), string(evString));
+		if (event != NULL) {
+			event->Process();
+			delete event;
+		}
+	}
 }
 
 void
 ZfsDaemon::RescanSystem()
 {
         struct gmesh	  mesh;
         struct gclass	 *mp;
         struct ggeom	 *gp;
         struct gprovider *pp;
 	int		  result;
 
         /*
 	 * The devctl system doesn't replay events for new consumers
 	 * of the interface.  Emit manufactured DEVFS arrival events
 	 * for any devices that already before we started or during
 	 * periods where we've lost our connection to devd.
          */
 	result = geom_gettree(&mesh);
 	if (result != 0) {
 		syslog(LOG_ERR, "ZfsDaemon::RescanSystem: "
 		       "geom_gettree faild with error %d\n", result);
 		return;
 	}
 
 	const string evStart("!system=DEVFS subsystem=CDEV type=CREATE "
 			     "sub_type=synthesized cdev=");
         LIST_FOREACH(mp, &mesh.lg_class, lg_class) {
                 LIST_FOREACH(gp, &mp->lg_geom, lg_geom) {
                         LIST_FOREACH(pp, &gp->lg_provider, lg_provider) {
 				Event *event;
 
 				string evString(evStart + pp->lg_name + "\n");
 				event = Event::CreateEvent(GetFactory(),
 							   evString);
-				if (event != NULL)
+				if (event != NULL) {
 					if (event->Process())
 						SaveEvent(*event);
 					delete event;
+				}
                         }
                 }
 	}
 	geom_deletetree(&mesh);
 }
 
 void
 ZfsDaemon::DetectMissedEvents()
 {
 	do {
 		PurgeCaseFiles();
 
 		/*
 		 * Discard any events waiting for us.  We don't know
 		 * if they still apply to the current state of the
 		 * system.
 		 */
 		FlushEvents();
 
 		BuildCaseFiles();
 
 		/*
 		 * If the system state has changed durring our
 		 * interrogation, start over.
 		 */
 	} while (s_terminateEventLoop == false && EventsPending());
 
 	RescanSystem();
 }
 
 void
 ZfsDaemon::EventLoop()
 {
 	while (s_terminateEventLoop == false) {
 		struct pollfd fds[2];
 		int	      result;
 
 		if (s_logCaseFiles == true) {
 			EventList::iterator event(m_unconsumedEvents.begin());
 			s_logCaseFiles = false;
 			CaseFile::LogAll();
 			while (event != m_unconsumedEvents.end())
 				(*event++)->Log(LOG_INFO);
 		}
 
 		Callout::ExpireCallouts();
 
 		/* Wait for data. */
 		fds[0].fd      = m_devdSockFD;
 		fds[0].events  = POLLIN;
 		fds[0].revents = 0;
 		fds[1].fd      = s_signalPipeFD[0];
 		fds[1].events  = POLLIN;
 		fds[1].revents = 0;
 		result = poll(fds, NUM_ELEMENTS(fds), /*timeout*/INFTIM);
 		if (result == -1) {
 			if (errno == EINTR)
 				continue;
 			else
 				err(1, "Polling for devd events failed");
 		} else if (result == 0) {
 			errx(1, "Unexpected result of 0 from poll. Exiting");
 		}
 
 		if ((fds[0].revents & POLLIN) != 0)
 			ProcessEvents();
 
 		if ((fds[1].revents & POLLIN) != 0) {
 			static char discardBuf[128];
 
 			/*
 			 * This pipe exists just to close the signal
 			 * race.  Its contents are of no interest to
 			 * us, but we must ensure that future signals
 			 * have space in the pipe to write.
 			 */
 			while (read(s_signalPipeFD[0], discardBuf,
 				    sizeof(discardBuf)) > 0)
 				;
 		}
 
 		if (s_systemRescanRequested == true) {
 			s_systemRescanRequested = false;
 			syslog(LOG_INFO, "System Rescan request processed.");
 			RescanSystem();
 		}
 
 		if ((fds[0].revents & POLLERR) != 0) {
 			syslog(LOG_INFO, "POLLERROR detected on devd socket.");
 			break;
 		}
 
 		if ((fds[0].revents & POLLHUP) != 0) {
 			syslog(LOG_INFO, "POLLHUP detected on devd socket.");
 			break;
 		}
 	}
 }
 //- ZfsDaemon staic Private Methods --------------------------------------------
 void
 ZfsDaemon::InfoSignalHandler(int)
 {
 	s_logCaseFiles = true;
 	ZfsDaemon::WakeEventLoop();
 }
 
 void
 ZfsDaemon::RescanSignalHandler(int)
 {
 	RequestSystemRescan();
 }
 
 void
 ZfsDaemon::QuitSignalHandler(int)
 {
 	s_terminateEventLoop = true;
 	ZfsDaemon::WakeEventLoop();
 }
 
 void
 ZfsDaemon::OpenPIDFile()
 {
 	pid_t otherPID;
 
 	s_pidFH = pidfile_open(s_pidFilePath, 0600, &otherPID);
 	if (s_pidFH == NULL) {
 		if (errno == EEXIST)
 			errx(1, "already running as PID %d. Exiting", otherPID);
 		warn("cannot open PID file");
 	}
 }
 
 void
 ZfsDaemon::UpdatePIDFile()
 {
 	if (s_pidFH != NULL)
 		pidfile_write(s_pidFH);
 }
 
 void
 ZfsDaemon::ClosePIDFile()
 {
 	if (s_pidFH != NULL)
 		pidfile_close(s_pidFH);
 }
 
 void
 ZfsDaemon::InitializeSyslog()
 {
 	openlog("zfsd", LOG_NDELAY, LOG_DAEMON);
 }
 
Index: projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh
===================================================================
--- projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh	(revision 292377)
+++ projects/zfsd/head/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh	(revision 292378)
@@ -1,609 +1,608 @@
 # CDDL HEADER START
 #
 # The contents of this file are subject to the terms of the
 # Common Development and Distribution License (the "License").
 # You may not use this file except in compliance with the License.
 #
 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 # or http://www.opensolaris.org/os/licensing.
 # See the License for the specific language governing permissions
 # and limitations under the License.
 #
 # When distributing Covered Code, include this CDDL HEADER in each
 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 # If applicable, add the following below this CDDL HEADER, with the
 # fields enclosed by brackets "[]" replaced with your own identifying
 # information: Portions Copyright [yyyy] [name of copyright owner]
 #
 # CDDL HEADER END
 #
 
 #
 # Copyright 2012,2013 Spectra Logic.  All rights reserved.
 # Use is subject to license terms.
 #
 
 
 atf_test_case zfsd_fault_001_pos cleanup
 zfsd_fault_001_pos_head()
 {
 	atf_set "descr" "ZFS will fault a vdev that produces IO errors"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_2_disks
 	atf_set "require.progs"  zfs zpool zfsd
 	atf_set "timeout" 300
 }
 zfsd_fault_001_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_fault_001_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_fault_001_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case zfsd_degrade_001_pos cleanup
 zfsd_degrade_001_pos_head()
 {
 	atf_set "descr" "ZFS will degrade a vdev that produces checksum errors"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_2_disks
 	atf_set "require.progs"  mkfile zpool zfsd
 	atf_set "timeout" 300
 }
 zfsd_degrade_001_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_degrade_001_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_degrade_001_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_degrade_002_pos cleanup
 zfsd_degrade_002_pos_head()
 {
 	atf_set "descr" "ZFS will degrade a spare that produces checksum errors"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_3_disks
 	atf_set "require.progs"  mkfile zpool zfsd
 	atf_set "timeout" 300
 }
 zfsd_degrade_002_pos_body()
 {
 	atf_expect_fail "BUG25761 An active spare on a raidz array will incorrectly account its checksum errors"
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_degrade_002_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_degrade_002_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case zfsd_hotspare_001_pos cleanup
 zfsd_hotspare_001_pos_head()
 {
 	atf_set "descr" "An active, damaged spare will be replaced by an available spare"
 	atf_set "require.config" rt_long
 	atf_set "require.progs"  zpool zfsd
 	atf_set "timeout" 3600
 }
 zfsd_hotspare_001_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_hotspare_001_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_hotspare_001_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_hotspare_002_pos cleanup
 zfsd_hotspare_002_pos_head()
 {
 	atf_set "descr" "If a vdev becomes degraded, the spare will be activated."
 	atf_set "require.config" rt_long
 	atf_set "require.progs"  zpool zfsd zinject
 	atf_set "timeout" 3600
 }
 zfsd_hotspare_002_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_hotspare_002_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_hotspare_002_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case zfsd_hotspare_003_pos cleanup
 zfsd_hotspare_003_pos_head()
 {
 	atf_set "descr" "A faulted vdev will be replaced by an available spare"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_5_disks
 	atf_set "require.progs"  zpool zfsd zinject
 	atf_set "timeout" 3600
 }
 zfsd_hotspare_003_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_hotspare_003_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_hotspare_003_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_hotspare_004_pos cleanup
 zfsd_hotspare_004_pos_head()
 {
 	atf_set "descr" "Removing a disk from a pool results in the spare activating"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_5_disks
 	atf_set "require.progs"  zpool camcontrol zfsd
 	atf_set "timeout" 3600
 }
 zfsd_hotspare_004_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_hotspare_004_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_hotspare_004_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_hotspare_005_pos cleanup
 zfsd_hotspare_005_pos_head()
 {
 	atf_set "descr" "A spare that is added to a degraded pool will be activated"
 	atf_set "require.config" rt_long
 	atf_set "require.progs"  zpool zfsd zinject
 	atf_set "timeout" 3600
 }
 zfsd_hotspare_005_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_hotspare_005_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_hotspare_005_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_hotspare_006_pos cleanup
 zfsd_hotspare_006_pos_head()
 {
 	atf_set "descr" "zfsd will replace two vdevs that fail simultaneously"
 	atf_set "require.config" rt_long
 	atf_set "require.progs"  zpool zfsd zinject
 	atf_set "timeout" 3600
 }
 zfsd_hotspare_006_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_hotspare_006_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_hotspare_006_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_hotspare_007_pos cleanup
 zfsd_hotspare_007_pos_head()
 {
 	atf_set "descr" "zfsd will swap failed drives at startup"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_5_disks
 	atf_set "require.progs"  zpool camcontrol zfsd
 	atf_set "timeout" 3600
 }
 zfsd_hotspare_007_pos_body()
 {
-	atf_expect_fail "P3_28731: ZFSD will not replace a vdev that dissappears while power is off"
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_hotspare_007_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_hotspare_007_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_autoreplace_001_neg cleanup
 zfsd_autoreplace_001_neg_head()
 {
 	atf_set "descr" "A pool without autoreplace set will not replace by physical path"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_5_disks
 	atf_set "require.progs"  zpool camcontrol zfsd
 	atf_set "timeout" 3600
 }
 zfsd_autoreplace_001_neg_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_autoreplace_001_neg.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_autoreplace_001_neg_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_autoreplace_002_pos cleanup
 zfsd_autoreplace_002_pos_head()
 {
 	atf_set "descr" "A pool with autoreplace set will replace by physical path"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_5_disks
 	atf_set "require.progs"  zpool camcontrol zfsd
 	atf_set "timeout" 3600
 }
 zfsd_autoreplace_002_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_autoreplace_002_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_autoreplace_002_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_autoreplace_003_pos cleanup
 zfsd_autoreplace_003_pos_head()
 {
 	atf_set "descr" "A pool with autoreplace set will replace by physical path even if a spare is active"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_5_disks
 	atf_set "require.progs"  zpool camcontrol zfsd
 	atf_set "timeout" 3600
 }
 zfsd_autoreplace_003_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_autoreplace_003_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_autoreplace_003_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/hotspare_cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_replace_001_pos cleanup
 zfsd_replace_001_pos_head()
 {
 	atf_set "descr" "ZFSD will automatically replace a SAS disk that dissapears and reappears in the same location, with the same devname"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_2_disks
 	atf_set "require.progs"  zpool camcontrol zfsd zfs
 }
 zfsd_replace_001_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_replace_001_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_replace_001_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 atf_test_case zfsd_replace_002_pos cleanup
 zfsd_replace_002_pos_head()
 {
 	atf_set "descr" "A pool can come back online after all disks have dissapeared and reappeared"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_2_disks
 	atf_set "require.progs"  zpool camcontrol zfsd zfs
 }
 zfsd_replace_002_pos_body()
 {
 	atf_expect_fail "ZFS hangs when an array becomes critical"
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_replace_002_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_replace_002_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_replace_003_pos cleanup
 zfsd_replace_003_pos_head()
 {
 	atf_set "descr" "ZFSD will correctly replace disks that dissapear and reappear with different devnames"
 	atf_set "require.config" rt_long
 	atf_set "require.config" at_least_3_disks
 	atf_set "require.progs"  zpool camcontrol zfsd zfs
 }
 zfsd_replace_003_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_replace_003_pos.ksh
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_replace_003_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/zfsd.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 atf_test_case zfsd_import_001_pos cleanup
 zfsd_import_001_pos_head()
 {
 	atf_set "descr" "If a removed drive gets reinserted while the pool is exported, it will detach its spare when imported."
 	atf_set "require.config" rt_long
 	atf_set "require.progs"  zfsd zpool
 	atf_set "timeout" 3600
 }
 zfsd_import_001_pos_body()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
 	ksh93 $(atf_get_srcdir)/zfsd_import_001_pos.ksh || atf_fail "Testcase failed"
 	if [[ $? != 0 ]]; then
 		save_artifacts
 		atf_fail "Testcase failed"
 	fi
 }
 zfsd_import_001_pos_cleanup()
 {
 	export TESTCASE_ID=$(echo $(atf_get ident) | cksum -o 2 | cut -f 1 -d " ")
 	. $(atf_get_srcdir)/../../include/default.cfg
 	. $(atf_get_srcdir)/../hotspare/hotspare.kshlib
 	. $(atf_get_srcdir)/../hotspare/hotspare.cfg
 
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
 
 
 
 atf_init_test_cases()
 {
 	atf_add_test_case zfsd_fault_001_pos
 	atf_add_test_case zfsd_degrade_001_pos
 	atf_add_test_case zfsd_degrade_002_pos
 	atf_add_test_case zfsd_hotspare_001_pos
 	atf_add_test_case zfsd_hotspare_002_pos
 	atf_add_test_case zfsd_hotspare_003_pos
 	atf_add_test_case zfsd_hotspare_004_pos
 	atf_add_test_case zfsd_hotspare_005_pos
 	atf_add_test_case zfsd_hotspare_006_pos
 	atf_add_test_case zfsd_hotspare_007_pos
 	atf_add_test_case zfsd_autoreplace_001_neg
 	atf_add_test_case zfsd_autoreplace_002_pos
 	atf_add_test_case zfsd_autoreplace_003_pos
 	atf_add_test_case zfsd_replace_001_pos
 	atf_add_test_case zfsd_replace_002_pos
 	atf_add_test_case zfsd_replace_003_pos
 	atf_add_test_case zfsd_import_001_pos
 }
 
 save_artifacts()
 {
 	# If ARTIFACTS_DIR is defined, save test artifacts for
 	# post-mortem analysis
 	if [[ -n $ARTIFACTS_DIR ]]; then
 		TC_ARTIFACTS_DIR=${ARTIFACTS_DIR}/sys/cddl/zfs/tests/zfsd/$(atf_get ident)
 		mkdir -p $TC_ARTIFACTS_DIR
 		cp -a /var/log/zfsd.log* $TC_ARTIFACTS_DIR
 		bzip2 $TC_ARTIFACTS_DIR/zfsd.log
 	fi
 }