Index: user/pho/stress2/misc/graid1.sh
===================================================================
--- user/pho/stress2/misc/graid1.sh	(revision 342043)
+++ user/pho/stress2/misc/graid1.sh	(revision 342044)
@@ -1,80 +1,82 @@
 #!/bin/sh
 
 #
 # Copyright (c) 2014 EMC Corp.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 # No problems seen.
 
 [ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
 
 . ../default.cfg
 
 md1=$mdstart
 md2=$((mdstart + 1))
 md3=$((mdstart + 2))
 
 s=0
 size=1g
 [ $((`sysctl -n hw.usermem` / 1024 / 1024 / 1024)) -le 4 ] &&
     size=512m
 
 for u in $md1 $md2 $md3; do
 	mdconfig -l | grep -q md$u && mdconfig -d -u $u
 	mdconfig -a -t swap -s $size -u $u
 done
 
 gmirror load > /dev/null 2>&1 && unload=1
 old=`sysctl -n kern.geom.mirror.debug`
 sysctl kern.geom.mirror.debug=-1 | grep -q -- -1 ||
     sysctl kern.geom.mirror.debug=$old > /dev/null
 gmirror label -v -b split -s 2048 data /dev/md$md1 /dev/md$md2 \
     /dev/md$md3 > /dev/null || exit 1
+[ "`sysctl -in kern.geom.mirror.launch_mirror_before_timeout`" = "0" ] &&
+    sleep $((`sysctl -n kern.geom.mirror.timeout` + 1))
 [ -c /dev/mirror/data ] || exit 1
 newfs $newfs_flags /dev/mirror/data > /dev/null
 mount /dev/mirror/data $mntpoint
 chmod 777 $mntpoint
 
 export runRUNTIME=10m
 export RUNDIR=$mntpoint/stressX
 
 su $testuser -c 'cd ..; ./run.sh marcus.cfg'
 for i in `jot 6`; do
 	mount | grep -q "on $mntpoint " || break
 	umount $mntpoint && break || sleep 10
 	[ $i -eq 6 ] &&
 	    { echo FATAL; fstat -mf $mntpoint; exit 1; }
 done
 checkfs /dev/mirror/data || s=1
 gmirror stop data || s=2
 gmirror destroy data 2>/dev/null
 [ $unload ] && gmirror unload
 
 for u in $md3 $md2 $md1; do
 	mdconfig -d -u $u || s=3
 done
 exit $s
Index: user/pho/stress2/misc/graid1_10.sh
===================================================================
--- user/pho/stress2/misc/graid1_10.sh	(revision 342043)
+++ user/pho/stress2/misc/graid1_10.sh	(revision 342044)
@@ -1,94 +1,96 @@
 #!/bin/sh
 
 #
 # Copyright (c) 2018 Dell EMC Isilon
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 # Read error handling for synchronization requests
 # Test scenario by Mark Johnston <markj@FreeBSD.org>
 # Fixed by r327779
 
 [ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
 
 . ../default.cfg
 
 md1=$mdstart
 md2=$((mdstart + 1))
 
 s=0
 size=1g
 [ $((`sysctl -n hw.usermem` / 1024 / 1024 / 1024)) -le 4 ] &&
     size=512m
 
 for u in $md1 $md2; do
 	mdconfig -l | grep -q md$u && mdconfig -d -u $u
 	mdconfig -a -t swap -s $size -u $u
 done
 
 gmirror load > /dev/null 2>&1 && unload=1
 old=`sysctl -n kern.geom.mirror.debug`
 sysctl kern.geom.mirror.debug=-1 | grep -q -- -1 ||
     sysctl kern.geom.mirror.debug=$old > /dev/null
 gmirror label -v -b split -s 2048 test /dev/md$md1 /dev/md$md2 \
     > /dev/null || exit 1
+[ "`sysctl -in kern.geom.mirror.launch_mirror_before_timeout`" = "0" ] &&
+    sleep $((`sysctl -n kern.geom.mirror.timeout` + 1))
 [ -c /dev/mirror/test ] || exit 1
 newfs /dev/mirror/test > /dev/null
 mount /dev/mirror/test $mntpoint
 
 gpid=`pgrep -fS "g_mirror test"`
 s=0
 start=`date +%s`
 roid=debug.fail_point.g_mirror_regular_request_read
 while [ $((`date +%s` - start)) -lt 120 ]; do
 	gmirror forget test md$md2 2>/dev/null
 	gmirror remove test md$md2 2>/dev/null
 	sysctl $roid="1%return(5)[pid $gpid]" > /dev/null
 	gmirror insert test md$md2
 	n=0
 	while gmirror status test | grep -q DEGRADED; do
 		sleep 2
 		[ $((n += 1)) -gt 10 ] && { s=1; break; }
 	done
 	sysctl $roid="off" > /dev/null
 done
 [ $s -ne 0 ] && gmirror status test
 
 for i in `jot 12`; do
 	gmirror status test | grep -q SYNCHRONIZING || break
 	sleep 10
 done
 while mount | grep $mntpoint | grep -q /mirror/; do
 	umount $mntpoint || sleep 1
 done
 gmirror stop test || s=2
 gmirror destroy test 2>/dev/null
 [ $unload ] && gmirror unload
 
 for u in $md2 $md1; do
 	mdconfig -d -u $u || s=3
 done
 exit $s
Index: user/pho/stress2/misc/graid1_2.sh
===================================================================
--- user/pho/stress2/misc/graid1_2.sh	(revision 342043)
+++ user/pho/stress2/misc/graid1_2.sh	(revision 342044)
@@ -1,132 +1,134 @@
 #!/bin/sh
 
 #
 # Copyright (c) 2017 Dell EMC Isilon
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 # Test scenario by Mark Johnston <markj@FreeBSD.org>
 # "physwr DL /tmp/graid1_2 /dev/mirror/test" seen.
 # Fixed by r307691.
 
 [ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
 
 . ../default.cfg
 
 dir=/tmp
 odir=`pwd`
 cd $dir
 sed '1,/^EOF/d' < $odir/$0 > $dir/graid1_2.c
 mycc -o graid1_2 -Wall -Wextra -O0 -g graid1_2.c || exit 1
 rm -f graid1_2.c
 cd $odir
 
 gmirror load > /dev/null 2>&1 && unload=1
 [ -c /dev/mirror/test ] && { gmirror stop test; gmirror destroy test; }
 old=`sysctl -n kern.geom.mirror.debug`
 sysctl kern.geom.mirror.debug=-1 | grep -q -- -1 ||
     sysctl kern.geom.mirror.debug=$old > /dev/null
 
 md1=$mdstart
 md2=$((mdstart + 1))
 s=0
 size=$((128 * 1024))
 
 for u in $md1 $md2; do
 	dd if=/dev/zero of=/tmp/graid1_2_di$u bs=$size count=1 2>&1 | \
 	    egrep -v "records|transferred"
 	[ -c /dev/md$u ] && mdconfig -d -u $u
 	mdconfig -a -t vnode -f /tmp/graid1_2_di$u -u $u
 done
 gmirror label test /dev/md$md1 /dev/md$md2 || exit 1
+[ "`sysctl -in kern.geom.mirror.launch_mirror_before_timeout`" = "0" ] &&
+    sleep $((`sysctl -n kern.geom.mirror.timeout` + 1))
 [ -c /dev/mirror/test ] || exit 1
 
 for i in `jot 150`; do /tmp/graid1_2 /dev/mirror/test; done &
 
 sleep 5
 start=`date '+%s'`
 while [ $((`date '+%s'` - start)) -lt 300 ]; do
 	gmirror rebuild test /dev/md$md1
 	sleep 2
 	n=0
 	while ps -lx | grep -v grep | grep graid1_2 | grep -q D; do
 		opid=$pid
 		pid=`pgrep graid1_2`
 		[ -z "$pid" -o "$pid" != "$opid" ] && n=0
 		sleep 1
 		n=$((n + 1))
 		if [ $n -gt 180 ]; then
 			echo FAIL
 			ps -lx | grep -v grep | grep graid1_2 | grep D
 			exit 1
 		fi
 	done
 done
 kill $! 2>/dev/null
 pkill graid1_2
 wait
 
 while mount | grep $mntpoint | grep -q /mirror/; do
 	umount $mntpoint || sleep 1
 done
 gmirror stop test || s=2
 [ $unload ] && gmirror unload
 
 for u in $md2 $md1; do
 	mdconfig -d -u $u || s=4
 done
 rm -f /tmp/graid1_2 /tmp/graid1_2_di*
 exit $s
 EOF
 /* Write last sector on disk */
 #include <err.h>
 #include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <time.h>
 #include <unistd.h>
 
 static char buf[512];
 
 int
 main(int argc __unused, char *argv[])
 {
 	time_t start;
 	int fd;
 
 	if ((fd = open(argv[1], O_RDWR)) == -1)
 		err(1, "open(%s)", argv[1]);
 	start = time(NULL);
 	while (time(NULL) - start < 2) {
 			if (lseek(fd, 254 * sizeof(buf), SEEK_SET) == -1)
 			err(1, "seek");
 		if (write(fd, buf, sizeof(buf)) != sizeof(buf))
 			err(1, "write");
 	}
 	close(fd);
 
 	return (0);
 }
Index: user/pho/stress2/misc/graid1_4.sh
===================================================================
--- user/pho/stress2/misc/graid1_4.sh	(revision 342043)
+++ user/pho/stress2/misc/graid1_4.sh	(revision 342044)
@@ -1,98 +1,100 @@
 #!/bin/sh
 
 #
 # Copyright (c) 2017 Dell EMC Isilon
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 # Non UFS SU gmirror stop -f test.
 
 # Page fault seen:
 # https://people.freebsd.org/~pho/stress/log/graid1_4.txt
 # Fixed in r316867
 
 [ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
 
 . ../default.cfg
 
 rm -f $diskimage*
 need=1024 # MB
 [ `df -k $(dirname $diskimage) | tail -1 | awk '{print int($4 / 1024)'}` -lt \
     $need ] && printf "Need %d MB on %s.\n" $need `dirname $diskimage` && exit
 
 gmirror load > /dev/null 2>&1 && unload=1
 [ -c /dev/mirror/test ] && { gmirror stop test; gmirror destroy test; }
 old=`sysctl -n kern.geom.mirror.debug`
 sysctl kern.geom.mirror.debug=-1 | grep -q -- -1 ||
     sysctl kern.geom.mirror.debug=$old > /dev/null
 
 md1=$mdstart
 md2=$((mdstart + 1))
 
 s=0
 for u in $md1 $md2; do
 	disk="$diskimage.$u"
 	dd if=/dev/zero of=$disk bs=1m count=512 2>&1 | \
 	    egrep -v "records|transferred"
 	[ -c /dev/md$u ] && mdconfig -d -u $u
 	mdconfig -a -t vnode -f $disk -u $u
 done
 
 gmirror label -v -b split -s 2048 test /dev/md$md1 /dev/md$md2 \
     > /dev/null || exit 1
+[ "`sysctl -in kern.geom.mirror.launch_mirror_before_timeout`" = "0" ] &&
+    sleep $((`sysctl -n kern.geom.mirror.timeout` + 1))
 [ -c /dev/mirror/test ] || exit 1
 # Soft Updates issues with removal of backing media
 newfs /dev/mirror/test > /dev/null
 mount /dev/mirror/test $mntpoint
 chmod 777 $mntpoint
 
 export runRUNTIME=10m
 export RUNDIR=$mntpoint/stressX
 
 su $testuser -c 'cd ..; ./run.sh marcus.cfg' &
 
 while kill -0 $! > /dev/null 2>&1; do
 	sleep `jot -r 1 1 5`
 	gmirror remove test md$md2
 	sleep `jot -r 1 1 5`
 	gmirror insert test md$md2
 done
 wait
 
 gmirror stop -f test # Note the stop *before* umount
 
 while mount | grep $mntpoint | grep -q /mirror/; do
 	umount $mntpoint || sleep 1
 done
 gmirror stop test || s=1
 gmirror destroy test 2>/dev/null
 [ $unload ] && gmirror unload
 
 for u in $md2 $md1; do
 	mdconfig -d -u $u || s=3
 done
 rm -f $diskimage*
 exit $s
Index: user/pho/stress2/misc/graid1_5.sh
===================================================================
--- user/pho/stress2/misc/graid1_5.sh	(revision 342043)
+++ user/pho/stress2/misc/graid1_5.sh	(revision 342044)
@@ -1,112 +1,114 @@
 #!/bin/sh
 
 #
 # Copyright (c) 2017 Dell EMC Isilon
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 # Mirror tests with gnop(8) errors introduced in 2 out of three partitions.
 
 # https://people.freebsd.org/~pho/stress/log/graid1_5-2.txt
 # Fixed by r327698
 
 [ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
 
 . ../default.cfg
 
 gmirror load > /dev/null 2>&1 && unload=1
 [ -c /dev/mirror/test ] && { gmirror stop test; gmirror destroy test; }
 old=`sysctl -n kern.geom.mirror.debug`
 sysctl kern.geom.mirror.debug=-1 | grep -q -- -1 ||
 	sysctl kern.geom.mirror.debug=$old > /dev/null
 kldstat | grep -q geom_nop || { gnop load 2>/dev/null || exit 0; }
 gnop status || exit 1
 
 u1=$mdstart
 u2=$((mdstart + 1))
 u3=$((mdstart + 2))
 s=0
 for u in $u1 $u2 $u3; do
 	[ -c /dev/md$u ] && mdconfig -d -u $u
 	mdconfig -a -t swap -s 341m -u $u
 	gpart create -s GPT md$u
 done > /dev/null
 
 set -e
 (
 gpart add -t freebsd-ufs -s 340m md$u1
 gpart add -t freebsd-ufs -s 340m md$u2
 gpart add -t freebsd-ufs -s 340m md$u3
 ) > /dev/null
 gnop create md$u2
 gnop create md$u3
 gmirror label test md${u1}p1 md$u2.nopp1 md$u3.nopp1
+[ "`sysctl -in kern.geom.mirror.launch_mirror_before_timeout`" = "0" ] &&
+    sleep $((`sysctl -n kern.geom.mirror.timeout` + 1))
 [ -c /dev/mirror/test ] || exit 1
 
 newfs /dev/mirror/test > /dev/null
 mount /dev/mirror/test $mntpoint
 set +e
 chmod 777 $mntpoint
 
 export runRUNTIME=5m
 export RUNDIR=$mntpoint/stressX
 rm -rf /tmp/stressX.control
 
 su $testuser -c 'cd ..; ./run.sh marcus.cfg' > /dev/null 2>&1 &
 pid=$!
 
 gnop configure -r 0 -w 1 md$u2.nop
 gnop configure -r 0 -w 1 md$u3.nop
 while kill -0 $pid > /dev/null 2>&1; do
 	if ! gmirror status test | grep -q md$u2.nopp1; then
 		gmirror forget test
 		gmirror remove test md$u2.nopp1 2>/dev/null
 		gmirror insert test md$u2.nopp1 2>/dev/null
 	fi
 	if ! gmirror status test | grep -q md$u3.nopp1; then
 		gmirror forget test
 		gmirror remove test md$u3.nopp1 2>/dev/null
 		gmirror insert test md$u3.nopp1 2>/dev/null
 	fi
 	sleep 1
 done
 wait
 
 while mount | grep $mntpoint | grep -q /mirror/; do
 	umount $mntpoint || sleep 5
 done
 while gmirror status test | grep -q SYNCHRONIZING; do sleep 10; done
 for i in `jot 10`; do
 	gmirror stop test && break || sleep 30
 done
 [ $i -eq 10 ] && s=1
 gmirror destroy test 2>/dev/null
 [ $unload ] && gmirror unload
 
 for u in $u1 $u2 $u3; do
 	mdconfig -d -u $u || s=3
 done
 exit $s
Index: user/pho/stress2/misc/graid1_6.sh
===================================================================
--- user/pho/stress2/misc/graid1_6.sh	(revision 342043)
+++ user/pho/stress2/misc/graid1_6.sh	(revision 342044)
@@ -1,91 +1,93 @@
 #!/bin/sh
 
 #
 # Copyright (c) 2017 Dell EMC Isilon
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 # Variation of graid1_4.sh
 
 [ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
 
 . ../default.cfg
 
 rm -f $diskimage*
 need=1024 # MB
 [ `df -k $(dirname $diskimage) | tail -1 | awk '{print int($4 / 1024)'}` -lt \
     $need ] && printf "Need %d MB on %s.\n" $need `dirname $diskimage` && exit
 
 gmirror load > /dev/null 2>&1 && unload=1
 [ -c /dev/mirror/test ] && { gmirror stop test; gmirror destroy test; }
 old=`sysctl -n kern.geom.mirror.debug`
 sysctl kern.geom.mirror.debug=-1 | grep -q -- -1 ||
     sysctl kern.geom.mirror.debug=$old > /dev/null
 
 md1=$mdstart
 md2=$((mdstart + 1))
 
 s=0
 for u in $md1 $md2; do
 	disk="$diskimage.$u"
 	dd if=/dev/zero of=$disk bs=1m count=512 2>&1 | \
 	    egrep -v "records|transferred"
 	[ -c /dev/md$u ] && mdconfig -d -u $u
 	mdconfig -a -t vnode -f $disk -u $u
 done
 
 gmirror label -v -b split -s 2048 test /dev/md$md1 /dev/md$md2 \
     > /dev/null || exit 1
+[ "`sysctl -in kern.geom.mirror.launch_mirror_before_timeout`" = "0" ] &&
+    sleep $((`sysctl -n kern.geom.mirror.timeout` + 1))
 [ -c /dev/mirror/test ] || exit 1
 newfs $newfs_flags /dev/mirror/test > /dev/null
 mount /dev/mirror/test $mntpoint
 chmod 777 $mntpoint
 
 export runRUNTIME=4m
 export RUNDIR=$mntpoint/stressX
 
 su $testuser -c 'cd ..; ./run.sh marcus.cfg' &
 
 while kill -0 $! > /dev/null 2>&1; do
 	sleep `jot -r 1 1 5`
 	gmirror remove test md$md2
 	sleep `jot -r 1 1 5`
 	gmirror insert test md$md2
 done
 wait
 
 while mount | grep $mntpoint | grep -q /mirror/; do
 	umount $mntpoint || sleep 1
 done
 gmirror stop -f test ||s=1
 gmirror destroy test 2>/dev/null
 [ $unload ] && gmirror unload
 
 for u in $md2 $md1; do
 	mdconfig -d -u $u || s=3
 done
 rm -f $diskimage*
 exit $s
Index: user/pho/stress2/misc/graid1_7.sh
===================================================================
--- user/pho/stress2/misc/graid1_7.sh	(revision 342043)
+++ user/pho/stress2/misc/graid1_7.sh	(revision 342044)
@@ -1,111 +1,113 @@
 #!/bin/sh
 
 #
 # Copyright (c) 2017 Dell EMC Isilon
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 # Component looses it's name:
 # g_dev_taste: make_dev_p() failed
 # (gp->name=gptid/7c598e03-19cb-11e7-b62b-001e6756c168, error=17)
 
 # Deadlock seen:
 # https://people.freebsd.org/~pho/stress/log/graid1_7.txt
 
 [ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
 
 . ../default.cfg
 
 gmirror load > /dev/null 2>&1 && unload=1
 [ -c /dev/mirror/test ] && { gmirror stop test; gmirror destroy test; }
 old=`sysctl -n kern.geom.mirror.debug`
 sysctl kern.geom.mirror.debug=-1 | grep -q -- -1 ||
     sysctl kern.geom.mirror.debug=$old > /dev/null
 
 u1=$mdstart
 s=0
 [ -c /dev/md$u1 ] && mdconfig -d -u $u1
 mdconfig -a -t swap -s 1g -u $u1
 
 set -e
 (
 gpart create -s GPT md$u1
 gpart add -t freebsd-ufs -s 341m md$u1
 gpart add -t freebsd-ufs -s 341m md$u1
 gpart add -t freebsd-ufs -s 341m md$u1
 ) > /dev/null
 gmirror label test md${u1}p1 md${u1}p2 md${u1}p3
+[ "`sysctl -in kern.geom.mirror.launch_mirror_before_timeout`" = "0" ] &&
+    sleep $((`sysctl -n kern.geom.mirror.timeout` + 1))
 [ -c /dev/mirror/test ] || exit 1
 
 newfs /dev/mirror/test > /dev/null
 mount /dev/mirror/test $mntpoint
 set +e
 chmod 777 $mntpoint
 
 export runRUNTIME=5m
 export RUNDIR=$mntpoint/stressX
 rm -rf /tmp/stressX.control
 
 su $testuser -c 'cd ..; ./run.sh io.cfg' > /dev/null 2>&1 &
 pid=$!
 
 sleep 2
 cont=/tmp/graid1_7.cont
 touch $cont
 for i in `jot 8`; do
 	while [ -f $cont ]; do
 		for u in md${u1}p2 md${u1}p3; do
 			gmirror forget test
 			gmirror remove test $u
 			gmirror insert test $u
 			id=`gmirror status test | grep gptid | awk '{print $1}'`
 			if [ $i -eq 1 -a -n "$id" ]; then
 				echo "FAIL Remove component $id"
 				gmirror remove test $id
 			fi
 		done 2>/dev/null
 	done &
 done
 while kill -0 $pid 2>/dev/null; do sleep 5; done
 rm $cont
 wait
 gmirror status test | grep -qw md${u1}p2 || gmirror insert test md${u1}p2
 gmirror status test | grep -qw md${u1}p3 || gmirror insert test md${u1}p3
 while ! gmirror status test | grep -q COMPLETE; do sleep 5; done
 
 while mount | grep $mntpoint | grep -q /mirror/; do
 	umount $mntpoint || sleep 5
 done
 while gmirror status test | grep -q SYNCHRONIZING; do sleep 10; done
 for i in `jot 10`; do
 	gmirror stop test && break || sleep 30
 done
 [ $i -eq 10 ] && s=1
 gmirror destroy test 2>/dev/null
 [ $unload ] && gmirror unload
 
 mdconfig -d -u $mdstart || s=3
 exit $s
Index: user/pho/stress2/misc/graid1_9.sh
===================================================================
--- user/pho/stress2/misc/graid1_9.sh	(revision 342043)
+++ user/pho/stress2/misc/graid1_9.sh	(revision 342044)
@@ -1,114 +1,116 @@
 #!/bin/sh
 
 #
 # Copyright (c) 2018 Dell EMC Isilon
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 # Copy of graid1.sh with fail points added.
 # https://people.freebsd.org/~pho/stress/log/mark015.txt
 # Fixed by r327779
 
 # Problem also seen with non SU:
 # https://people.freebsd.org/~pho/stress/log/numa027.txt
 
 [ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1
 
 . ../default.cfg
 
 md1=$mdstart
 md2=$((mdstart + 1))
 md3=$((mdstart + 2))
 
 s=0
 size=1g
 [ $((`sysctl -n hw.usermem` / 1024 / 1024 / 1024)) -le 4 ] &&
     size=512m
 
 for u in $md1 $md2 $md3; do
 	mdconfig -l | grep -q md$u && mdconfig -d -u $u
 	mdconfig -a -t swap -s $size -u $u
 done
 
 gmirror load > /dev/null 2>&1 && unload=1
 old=`sysctl -n kern.geom.mirror.debug`
 sysctl kern.geom.mirror.debug=-1 | grep -q -- -1 ||
     sysctl kern.geom.mirror.debug=$old > /dev/null
 gmirror label -v -b split -s 2048 test /dev/md$md1 /dev/md$md2 \
     /dev/md$md3 > /dev/null || exit 1
+[ "`sysctl -in kern.geom.mirror.launch_mirror_before_timeout`" = "0" ] &&
+    sleep $((`sysctl -n kern.geom.mirror.timeout` + 1))
 [ -c /dev/mirror/test ] || exit 1
 # Do not use SU as it is more intolerant to FS corruption
 newfs /dev/mirror/test > /dev/null
 mount /dev/mirror/test $mntpoint
 chmod 777 $mntpoint
 
 export runRUNTIME=10m
 export RUNDIR=$mntpoint/stressX
 
 woid=debug.fail_point.g_mirror_regular_request_write
 roid=debug.fail_point.g_mirror_regular_request_read
 gpid=`pgrep -fS "g_mirror test"`
 sysctl $woid="0.0005%return(5)[pid $gpid]" > /dev/null
 sysctl $roid="0.0005%return(5)[pid $gpid]" > /dev/null
 
 su $testuser -c 'cd ..; ./run.sh marcus.cfg' > /dev/null &
 
 while kill -0 $! 2>/dev/null; do
 	if gmirror status test | grep -q DEGRADED; then
 		for i in $md1 $md2 $md3; do
 			if ! gmirror status test | grep -q md$i; then
 				gmirror forget test md$i 2>/dev/null
 				gmirror remove test md$i 2>/dev/null
 				gmirror insert test md$i
 			fi
 		done
 	else
 		sleep 5
 	fi
 done
 wait
 
 sysctl $woid=off > /dev/null
 sysctl $roid=off > /dev/null
 
 for i in `jot 12`; do
 	gmirror status test | grep -q SYNCHRONIZING || break
 	sleep 10
 done
 gmirror status test | grep -q SYNCHRONIZING &&
     { s=1; gmirror status test; }
 while mount | grep $mntpoint | grep -q /mirror/; do
 	umount $mntpoint || sleep 1
 done
 # The FS is most likely corrupted at this point, so do not run fsck(8).
 gmirror stop test || s=2
 gmirror destroy test 2>/dev/null
 [ $unload ] && gmirror unload
 
 for u in $md3 $md2 $md1; do
 	mdconfig -d -u $u || s=3
 done
 exit $s