diff --git a/tools/test/stress2/misc/all.exclude b/tools/test/stress2/misc/all.exclude index 2f4e04473eff..022075e63cb4 100644 --- a/tools/test/stress2/misc/all.exclude +++ b/tools/test/stress2/misc/all.exclude @@ -1,87 +1,94 @@ # List of tests not to run, unless the '-a' option is used with run.sh # Exclude names must start in column 1 backingstore.sh g_vfs_done():md6a[WRITE(offset=...)]error = 28 20111220 backingstore2.sh panic: 43 vncache entries remaining 20111220 backingstore3.sh g_vfs_done():md6a[WRITE(offset=...)]error = 28 20111230 dd.sh CAM stuck in vmwait 20200116 +force13.sh https://people.freebsd.org/~pho/stress/log/log0376.txt 20221113 +force14.sh Waiting for fix 20230319 +force15.sh Waiting for fix 20230319 force4.sh https://people.freebsd.org/~pho/stress/log/log0082.txt 20210328 force7.sh https://people.freebsd.org/~pho/stress/log/log0266.txt 20220207 -force13.sh https://people.freebsd.org/~pho/stress/log/log0376.txt 20221113 +fsck10.sh Waiting for fix 20230319 +fsck11.sh Waiting for fix 20230319 +fsck12.sh Waiting for fix 20230319 fsync.sh panic: Journal overflow 20190208 fuse.sh Memory corruption seen in log file kostik734.txt 20141114 fuse2.sh Deadlock seen 20121129 fuse3.sh Deadlock seen 20141120 getrandom.sh Known DoS issue 20201107 getrandom2.sh Known DoS issue 20200302 gjournal.sh panic: Journal overflow 20190626 gjournal2.sh panic: Journal overflow 20180125 gjournal3.sh panic: Bio not on queue 20171225 gjournal4.sh CAM stuck in vmwait 20180517 +gnop10.sh Waiting for fix 20230319 +gnop13.sh https://people.freebsd.org/~pho/stress/log/log0386.txt 20221113 gnop7.sh Waiting for patch commit 20190820 gnop8.sh Waiting for patch commit 20201214 gnop9.sh Waiting for patch commit 20201214 -gnop13.sh https://people.freebsd.org/~pho/stress/log/log0386.txt 20221113 graid1_8.sh Known issue 20170909 graid1_9.sh panic: Bad effnlink 20180212 ifconfig.sh Bug 253824 20210322 ifconfig2.sh https://people.freebsd.org/~pho/stress/log/log0051.txt 20210210 lockf5.sh Spinning threads seen 20160718 maxvnodes2.sh https://people.freebsd.org/~pho/stress/log/log0083.txt 20210329 memguard.sh https://people.freebsd.org/~pho/stress/log/log0088.txt 20210402 memguard2.sh Waiting for fix commit memguard3.sh Waiting for fix commit mlockall2.sh Unrecoverable OOM killing seen 20190203 mlockall7.sh Needs further investigation 20210123 msetdomain.sh May change policy for random threads to domainset_fixed 20210104 newfs4.sh watchdog fired. newbuf 20190225 nfs10.sh Double fault 20151013 nfs13.sh mount_nfs hangs in mntref 20191007 nfs15lockd.sh panic: Assertion td->td_realucred == td->td_ucred failed ... 20210211 nfs16.sh panic: Failed to register NFS lock locally - error=11 20160608 nullfs28.sh Hang in "mount drain" seen 20220111 oom2.sh Hang in pfault 20180324 overcommit2.sh CAM stuck in vmwait seen 20200112 pageout.sh panic: handle_written_filepage: not started 20190218 pmc8.sh panic: [pmc,2749] (ri21, rc1) waiting too long for pmc to ... 20210621 sctp2.sh panic: Queues are not empty when handling SHUTDOWN-COMPLETE 20210211 sctp3.sh panic: Queues are not empty when handling SHUTDOWN-COMPLETE 20210211 sendfile25.sh WiP 20200611 signal.sh Timing issues. Needs fixing 20171116 snap8.sh https://people.freebsd.org/~pho/stress/log/log0123.txt 20211008 +suj12.sh Waiting for fix 20230319 suj19.sh https://people.freebsd.org/~pho/stress/log/log0378.txt 20221113 suj27.sh https://people.freebsd.org/~pho/stress/log/log0387.txt 20221113 suj36.sh https://people.freebsd.org/~pho/stress/log/log0392.txt 20221114 swapoff2.sh swap_pager_force_pagein: read from swap failed 20171223 swapoff3.sh Excessive OOM killing 20220403 swapoff5.sh log0005.txt, known issue 20210111 systrace.sh WiP 20200227 systrace2.sh WiP 20200227 syzkaller15.sh WiP 20200712 syzkaller16.sh WiP 20210722 syzkaller25.sh WiP 20201116 syzkaller27.sh watchdogd fires 20220512 syzkaller28.sh WiP 20201120 syzkaller50.sh panic: Assertion done != job_total_nbytes failed at ... 20220405 syzkaller54.sh panic: td 0xfffffe014f7193a0 is not suspended 20220527 syzkaller55.sh panic: Counter goes negative 20220525 syzkaller59.sh Page fault 20220625 syzkaller65.sh Waiting for fix 20230318 truss3.sh WiP 20200915 unionfs9.sh https://people.freebsd.org/~pho/stress/log/log0404.txt 20230110 # Test not to run for other reasons: fuzz.sh A know issue marcus3.sh OK, but runs for a long time statfs.sh Not very interesting vunref.sh No problems ever seen vunref2.sh No problems ever seen # Exclude NFS loopback tests nfs2.sh panic: wrong diroffset 20140219 nfs5.sh nfs6.sh nfs11.sh vmwait deadlock 20151004 nullfs8.sh tmpfs18.sh mntref hang seen 20191019 diff --git a/tools/test/stress2/misc/force14.sh b/tools/test/stress2/misc/force14.sh new file mode 100755 index 000000000000..eec850c50891 --- /dev/null +++ b/tools/test/stress2/misc/force14.sh @@ -0,0 +1,119 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2023 Peter Holm +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +# SU+J Snapshot test with a 50G file system + +# "panic: handle_jwork: Unknown type jnewblk" seen: +# https://people.freebsd.org/~pho/stress/log/log0422.txt + +# "panic: flush_pagedep_deps: failed to flush inodedep..." seen: +# https://people.freebsd.org/~pho/stress/log/log0423.txt + +# "panic: softdep_deallocate_dependencies: dangling deps" seen: +# https://people.freebsd.org/~pho/stress/log/log0424.txt + +[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 +. ../default.cfg + +set -u +log=/tmp/force14.sh.log +mount | grep $mntpoint | grep -q /dev/md && umount -f $mntpoint +mdconfig -l | grep -q md$mdstart && mdconfig -d -u $mdstart +truncate -s 50g $diskimage +mdconfig -a -t vnode -f $diskimage -u $mdstart +flags="-j" + +newfs $flags md$mdstart > /dev/null 2>&1 + +# Exclude rename for now due to log0374.txt +export TESTPROGS=`cd ..; find testcases/ -perm -1 -type f | \ + egrep -Ev "/run/|/badcode/|/pty/|/shm/|/socket/|sysctl|tcp|thr|udp|rename"` +export runRUNTIME=3m +export RUNDIR=$mntpoint/stressX +export CTRLDIR=$mntpoint/stressX.control +start=`date +%s` +while [ $((`date +%s` - start)) -lt $((15 * 60)) ]; do + mount /dev/md$mdstart $mntpoint + rm -fr $mntpoint/lost+found + chmod 777 $mntpoint + + su $testuser -c 'cd ..; ./testcases/run/run $TESTPROGS' > \ + /dev/null 2>&1 & + + t=`jot -r 1 60 180` + st=`date +%s` + mkdir -p $mntpoint/.snap + for i in `jot 10`; do + rm -f $mntpoint/.snap/$i + mksnap_ffs $mntpoint $mntpoint/.snap/$i || + { ../tools/killall.sh; break; } + sleep `jot -r 1 1 5` + [ $((`date +%s` - st)) -ge $t ] && break + done & + sleep `jot -r 1 60 180` + while mdconfig -l | grep -q md$mdstart; do + mdconfig -d -u $mdstart -o force || sleep 1 + done + sleep 1 + ../tools/killall.sh + wait + n=0 + while mount | grep -q "on $mntpoint "; do + umount $mntpoint || sleep 1 + [ $((n += 1)) -gt 300 ] && { echo FAIL; exit 1; } + done + mdconfig -a -t vnode -f $diskimage -u $mdstart + c=0 + # Process the journal + fsck_ffs -fy /dev/md$mdstart > $log 2>&1; s=$? + grep 'INTERNAL ERROR: GOT TO reply' $log + for i in `jot 4`; do + [ $i -ne 1 ] && + echo "fsck_ffs -fy /dev/md$mdstart" + fsck_ffs -fy /dev/md$mdstart > $log 2>&1; s=$? + grep 'INTERNAL ERROR: GOT TO reply' $log + grep -q CLEAN $log && c=$((c+=1)) + grep -Eq "WAS MODIFIED" $log || break + done + [ $c -gt 1 ] && + { echo "Note: FS marked clean+modified $c times out of $i fsck runs"; s=101; } + [ $s -ne 0 ] && break + grep -Eq "IS CLEAN|MARKED CLEAN" $log || { s=102; break; } +done +if [ $s -eq 0 ]; then + mount /dev/md$mdstart $mntpoint + cp -R /usr/include $mntpoint + dd if=/dev/zero of=$mntpoint/big bs=1m count=10 status=none + find $mntpoint/* -delete + + umount $mntpoint + mdconfig -d -u $mdstart + rm -f $diskimage $log +fi +exit $s diff --git a/tools/test/stress2/misc/force15.sh b/tools/test/stress2/misc/force15.sh new file mode 100755 index 000000000000..955025cd98c6 --- /dev/null +++ b/tools/test/stress2/misc/force15.sh @@ -0,0 +1,113 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2023 Peter Holm +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +# FFS+SU with snapshots and forced unmounts + +# "panic: flush_pagedep_deps: failed to flush inodedep ..." seen +# https://people.freebsd.org/~pho/stress/log/log0427.txt + +[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 + +prog=$(basename "$0" .sh) +diskimage=/tmp/diskimage +log=/tmp/$prog.log +mdstart=10 +mntpoint=/mnt +newfs_flags="-U" + +set -u +mount | grep $mntpoint | grep -q /dev/md && umount -f $mntpoint +mdconfig -l | grep -q md$mdstart && mdconfig -d -u $mdstart +dd if=/dev/zero of=$diskimage bs=1m count=1k status=none +mdconfig -a -t vnode -f $diskimage -u $mdstart +flags="-U" + +newfs $flags md$mdstart > /dev/null 2>&1 + +start=`date +%s` +while [ $((`date +%s` - start)) -lt $((5 * 60)) ]; do + mount /dev/md$mdstart $mntpoint + rm -fr $mntpoint/lost+found + + jot 10 | xargs -I% mkdir -p $mntpoint/% + n=5000 + for j in `jot 10`; do + ( + jot $n | xargs -P0 -I% touch $mntpoint/$j/% + jot $n | xargs -P0 -I% rm $mntpoint/$j/% + ) > /dev/null 2>&1 & + done + + sleep `jot -r 1 5 20` + t=`jot -r 1 60 180` + st=`date +%s` + mkdir -p $mntpoint/.snap + for i in `jot 10`; do + rm -f $mntpoint/.snap/$i + mksnap_ffs $mntpoint $mntpoint/.snap/$i > /dev/null 2>&1 || break + sleep `jot -r 1 1 5` + [ $((`date +%s` - st)) -ge $t ] && break + done & + sleep `jot -r 1 2 5` + while mdconfig -l | grep -q md$mdstart; do + mdconfig -d -u $mdstart -o force || sleep 1 + done + sleep 1 + wait + n=0 + while mount | grep -q "on $mntpoint "; do + umount $mntpoint || sleep 1 + [ $((n += 1)) -gt 300 ] && { echo FAIL; exit 1; } + done + mdconfig -a -t vnode -f $diskimage -u $mdstart + c=0 + for i in `jot 5`; do + [ $i -ne 1 ] && + echo "$i: fsck_ffs -fy /dev/md$mdstart" + fsck_ffs -fy /dev/md$mdstart > $log 2>&1; s=$? + grep 'INTERNAL ERROR: GOT TO reply' $log + grep -q CLEAN $log && grep -q MODIFIED && c=$((c+=1)) + grep -Eq "WAS MODIFIED" $log || break + done + [ $c -gt 1 ] && + { echo "Note: FS marked clean+modified $c times out of $i fsck runs"; s=101; } + [ $s -ne 0 ] && break + grep -Eq "IS CLEAN|MARKED CLEAN" $log || { s=102; break; } +done +if [ $s -eq 0 ]; then + mount /dev/md$mdstart $mntpoint + cp -R /usr/include $mntpoint + dd if=/dev/zero of=$mntpoint/big bs=1m count=10 status=none + find $mntpoint/* -delete + + umount $mntpoint + mdconfig -d -u $mdstart + rm -f $diskimage $log +fi +exit $s diff --git a/tools/test/stress2/misc/fsck10.sh b/tools/test/stress2/misc/fsck10.sh new file mode 100755 index 000000000000..30479e0dfebb --- /dev/null +++ b/tools/test/stress2/misc/fsck10.sh @@ -0,0 +1,171 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2023 Peter Holm +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +# Variation of fsck9.sh with smaller disk size and newfs options '', 'U' and 'O1' +# fsck_ffs core dump seen + +. ../default.cfg + +[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 + +cc -o /tmp/flip -Wall -Wextra -O2 ../tools/flip.c || exit 1 + +set -e +prog=$(basename "$0" .sh) +u1=$mdstart +u2=$((mdstart + 1)) +mp1=${mntpoint}$u1 +mp2=${mntpoint}$u2 +mkdir -p $mp1 $mp2 +log=$mp1/$prog.sh.log +diskimage=$mp1/$prog.sh.diskimage +backup=/tmp/$prog.sh.diskimage.`date +%Y%m%dT%H%M%S`.gz +cleans=0 +reruns=0 + +# Pick a random newfs flag +newfs_flags=$(echo "" "-U" "-O1" | awk -v N=`jot -r 1 1 3` '{print $N}') +[ $# -eq 1 ] && newfs_flags="$1" # or use script argument +max=$((512 * 1024)) + +mount | grep "on $mp1 " | grep -q /dev/md && umount -f $mp1 +[ -c /dev/md$u1 ] && mdconfig -d -u $u1 +mdconfig -a -t swap -s 1g -u $u1 +newfs $newfs_flags -n /dev/md$u1 > /dev/null +mount /dev/md$u1 $mp1 + +[ -c /dev/md$u2 ] && mdconfig -d -u $u2 +dd if=/dev/zero of=$diskimage bs=$max count=1 status=none +mdconfig -a -t vnode -f $diskimage -u $u2 +backups=`newfs -N $newfs_flags md$u2 | grep -A1 "super-block backups" | \ + tail -1 | sed 's/,//g'` +echo "newfs $newfs_flags -n md$u2" +newfs $newfs_flags -n md$u2 > /dev/null +set +e + +chk() { + local i + + clean=0 + rerun=0 + fsck_ffs -fy $1 > $log 2>&1 + r=$? + if grep -qiE "super-?block.*failed" $log; then + for b in $backups; do + echo "fsck_ffs -b $b -fy $1" + fsck_ffs -b $b -fy $1 > $log 2>&1 + r=$? + grep -qiE "super-?block.*failed" $log || + break + echo "Checking next SB" + done + usedasb=1 + else + usedasb=0 + fi + LANG=C egrep -q "[A-Z][A-Z]" $log && clean=0 + grep -Eq "IS CLEAN|MARKED CLEAN" $log && clean=1 + grep -q RERUN $log && rerun=1 + [ $r -ne 0 -a $clean -eq 1 ] && echo "Exit code $r w/ clean == 1" +} + +cd $mp1 +clean=0 +errors=0 +s=0 +start=`date +%s` +while [ $((`date +%s` - start)) -lt 300 ]; do + mount /dev/md$u2 $mp2 || break + if ! ls -lR $mp2 > /dev/null; then + s=102 + echo "ls failed"; grep "core dumped" /var/log/messages | tail -1 + break + fi + rm -f $mp2/???????? + touch $mp2/`jot -rc 8 a z | tr -d '\n'` + mkdir -p $mp2/dir/dir/dir/dir + echo "abc" > $mp2/dir/dir/dir/dir/f + while mount | grep -q "on $mp2 "; do umount $mp2; done + echo * | grep -q core && break + mdconfig -d -u $u2 + + # Introduce 5 random single bit errors to the file system image + /tmp/flip -n 5 $diskimage + + sync; sleep .1 + if [ `stat -f%z $diskimage` -gt $max ]; then + ls -lh $diskimage + truncate -s $max $diskimage + else + gzip < $diskimage > $backup + fi + fsync $backup + sync; sleep .1 + + for i in `jot 5`; do + [ $i -gt 2 ] && echo "fsck run #$i" + chk $diskimage + [ $rerun -eq 1 ] && { reruns=$((reruns + 1)); continue; } + [ $clean -eq 1 ] && { cleans=$((cleans + 1)); break; } + if [ -f fsck_ffs.core ]; then + tstamp=`date +%Y%m%dT%H%M%S` + gzip < $backup > /tmp/fsck_ffs.core.diskimage.$tstamp + gzip < fsck_ffs.core > /tmp/fsck_ffs.core.$tstamp + break 2 + fi + done + if [ $clean -eq 1 ]; then + fsck_ffs -fy $diskimage > $log 2>&1 + if grep -q MODIFIED $log; then + echo "*** fsck of \"clean\" FS found more issues:" + cat $log + errors=$((errors + 1)) + s=1 + break + fi + fi + [ $clean -ne 1 ] && break + mdconfig -a -t vnode -f $diskimage -u $u2 +done +for i in `jot 5`; do + mount | grep -q "on $mp2 " || break + umount $mp2 && break + sleep 2 +done +mdconfig -l | grep -q $u2 && mdconfig -d -u $u2 + +[ $s -eq 0 ] && rm -f $backup || echo "Preserved $backup due to status code $s" +cd /tmp +for i in `jot 5`; do + umount $mp1 && break + sleep 2 +done +mdconfig -d -u $u1 +rm -f /tmp/flip +exit $s diff --git a/tools/test/stress2/misc/fsck11.sh b/tools/test/stress2/misc/fsck11.sh new file mode 100755 index 000000000000..cb5d1d256be6 --- /dev/null +++ b/tools/test/stress2/misc/fsck11.sh @@ -0,0 +1,170 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2023 Peter Holm +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +# Demonstrate how the "CLEAN" message and the exit code can be misleading + +# "panic: softdep_update_inodeblock inconsistent ip ..." seen: +# https://people.freebsd.org/~pho/stress/log/log0421.txt +# https://people.freebsd.org/~pho/fsck11.sh.diskimage.20230228T064402.gz + +. ../default.cfg + +[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 + +cc -o /tmp/flip -Wall -Wextra -O2 ../tools/flip.c || exit 1 + +set -e +prog=$(basename "$0" .sh) +u1=$mdstart +u2=$((mdstart + 1)) +mp1=${mntpoint}$u1 +mp2=${mntpoint}$u2 +mkdir -p $mp1 $mp2 +log=$mp1/$prog.sh.log +diskimage=$mp1/$prog.sh.diskimage +backup=/tmp/$prog.sh.diskimage.`date +%Y%m%dT%H%M%S`.gz +cleans=0 +reruns=0 + +# Pick a random newfs flag +newfs_flags=$(echo "" "-U" "-O1" | awk -v N=`jot -r 1 1 3` '{print $N}') +[ $# -eq 1 ] && newfs_flags="$1" # or use script argument +max=$((2 * 1024 * 1024)) +[ "$newfs_flags" == "-j" ] && max=$((20 * 1024 * 1024)) # Make room for the journal file + +mount | grep "on $mp1 " | grep -q /dev/md && umount -f $mp1 +[ -c /dev/md$u1 ] && mdconfig -d -u $u1 +mdconfig -a -t swap -s 1g -u $u1 +newfs $newfs_flags -n /dev/md$u1 > /dev/null +mount /dev/md$u1 $mp1 + +[ -c /dev/md$u2 ] && mdconfig -d -u $u2 +dd if=/dev/zero of=$diskimage bs=$max count=1 status=none +mdconfig -a -t vnode -f $diskimage -u $u2 +backups=`newfs -N $newfs_flags md$u2 | grep -A1 "super-block backups" | \ + tail -1 | sed 's/,//g'` +echo "newfs $newfs_flags -n md$u2" +newfs $newfs_flags -n md$u2 > /dev/null +set +e + +chk() { + local i + + clean=0 + rerun=0 + fsck_ffs -fy $1 > $log 2>&1 + r=$? + if grep -qiE "super-?block.*failed" $log; then + for b in $backups; do + echo "fsck_ffs -b $b -fy $1" + fsck_ffs -b $b -fy $1 > $log 2>&1 + r=$? + grep -qiE "super-?block.*failed" $log || + break + echo "Checking next SB" + done + usedasb=1 + else + usedasb=0 + fi + LANG=C egrep -q "[A-Z][A-Z]" $log && clean=0 + grep -Eq "IS CLEAN|MARKED CLEAN" $log && clean=1 + grep -q RERUN $log && rerun=1 + [ $r -ne 0 -a $clean -eq 1 ] && echo "Exit code $r w/ clean == 1" +} + +cd $mp1 +clean=0 +errors=0 +s=0 +start=`date +%s` +while [ $((`date +%s` - start)) -lt 300 ]; do + mount /dev/md$u2 $mp2 || break + if ! ls -lR $mp2 > /dev/null; then + s=102 + echo "ls failed"; grep "core dumped" /var/log/messages | tail -1 + break + fi + + find $mp2 -type f | xargs cat > /dev/null + for j in `jot 9`; do + rm -rf $mp2/$j + mkdir $mp2/$j + jot 10 | xargs -P0 -I% cp /etc/group $mp2/$j/% + done 2>/dev/null + + while mount | grep -q "on $mp2 "; do umount $mp2; done + echo * | grep -q core && break + mdconfig -d -u $u2 + + # Introduce 5 random single bit errors to the file system image + /tmp/flip -n 5 $diskimage + + if [ `stat -f%z $diskimage` -gt $max ]; then + ls -lh $diskimage + truncate -s $max $diskimage + else + gzip < $diskimage > $backup + fi + fsync $backup + sync; sleep 1 + + [ $newfs_flags == "-j" ] && + fsck -fy $diskimage > $log 2>&1 # process the journal file + for i in `jot 5`; do + [ $i -gt 2 ] && echo "fsck run #$i" + chk $diskimage + [ $rerun -eq 1 ] && { reruns=$((reruns + 1)); continue; } + [ $clean -eq 1 ] && { cleans=$((cleans + 1)); break; } + if [ -f fsck_ffs.core ]; then + tstamp=`date +%Y%m%dT%H%M%S` + gzip < $backup > /tmp/fsck_ffs.core.diskimage.$tstamp + gzip < fsck_ffs.core > /tmp/fsck_ffs.core.$tstamp + break 2 + fi + done + [ $clean -ne 1 ] && { s=99; break; } # broken image? + mdconfig -a -t vnode -f $diskimage -u $u2 +done +for i in `jot 5`; do + mount | grep -q "on $mp2 " || break + umount $mp2 && break + sleep 2 +done +mdconfig -l | grep -q $u2 && mdconfig -d -u $u2 + +[ $s -eq 0 ] && rm -f $backup || echo "Preserved $backup due to status code $s" +cd /tmp +for i in `jot 5`; do + umount $mp1 && break + sleep 2 +done +mdconfig -d -u $u1 +rm -f /tmp/flip +exit $s diff --git a/tools/test/stress2/misc/fsck12.sh b/tools/test/stress2/misc/fsck12.sh new file mode 100755 index 000000000000..d2787e962927 --- /dev/null +++ b/tools/test/stress2/misc/fsck12.sh @@ -0,0 +1,177 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2023 Peter Holm +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +# "panic: ffs_copyonwrite: bad copy block" seen: +# Hunt for fsck_ffs's INTERNAL ERROR message + +# https://people.freebsd.org/~pho/stress/log/log0426.txt + +. ../default.cfg + +[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 + +cc -o /tmp/flip -Wall -Wextra -O2 ../tools/flip.c || exit 1 + +set -e +prog=$(basename "$0" .sh) +u1=$mdstart +u2=$((mdstart + 1)) +mp1=${mntpoint}$u1 +mp2=${mntpoint}$u2 +mkdir -p $mp1 $mp2 +log=$mp1/$prog.sh.log +diskimage=$mp1/$prog.sh.diskimage +backup=/tmp/$prog.sh.diskimage.`date +%Y%m%dT%H%M%S`.gz +cleans=0 +reruns=0 + +mount | grep "on $mp1 " | grep -q /dev/md && umount -f $mp1 +[ -c /dev/md$u1 ] && mdconfig -d -u $u1 +mdconfig -a -t swap -s 1g -u $u1 +newfs $newfs_flags -n /dev/md$u1 > /dev/null +mount /dev/md$u1 $mp1 + +newfs_flags='-j' +[ $# -eq 1 ] && newfs_flags="$1" # or use script argument +max=$((2 * 1024 * 1024)) +[ "$newfs_flags" == "-j" ] && max=$((20 * 1024 * 1024)) # Make room for the journal file + +[ -c /dev/md$u2 ] && mdconfig -d -u $u2 +dd if=/dev/zero of=$diskimage bs=$max count=1 status=none +mdconfig -a -t vnode -f $diskimage -u $u2 +backups=`newfs -N $newfs_flags md$u2 | grep -A1 "super-block backups" | \ + tail -1 | sed 's/,//g'` +echo "newfs $newfs_flags md$u2" +newfs $newfs_flags md$u2 > /dev/null +set +e + +chk() { + local i + + clean=0 + rerun=0 + fsck_ffs -fy $1 > $log 2>&1 + r=$? + grep "GOT TO reply" $log && exit 1 + if grep -qiE "super-?block.*failed" $log; then + for b in $backups; do + echo "fsck_ffs -b $b -fy $1" + fsck_ffs -b $b -fy $1 > $log 2>&1 + r=$? + grep "GOT TO reply" $log && exit 1 + grep -qiE "super-?block.*failed" $log || + break + echo "Checking next SB" + done + usedasb=1 + else + usedasb=0 + fi + LANG=C egrep -q "[A-Z][A-Z]" $log && clean=0 + grep -Eq "IS CLEAN|MARKED CLEAN" $log && clean=1 + grep -q RERUN $log && rerun=1 + [ $r -ne 0 -a $clean -eq 1 ] && echo "Exit code $r w/ clean == 1" +} + +cd $mp1 +clean=0 +errors=0 +s=0 +sno=0 +start=`date +%s` +while [ $((`date +%s` - start)) -lt 300 ]; do + mount /dev/md$u2 $mp2 || break + if ! ls -lR $mp2 > /dev/null; then + s=102 + echo "ls failed"; grep "core dumped" /var/log/messages | tail -1 + break + fi + mkdir -p $mp2/d/d/d/d/d + cp /etc/group $mp2/d/d/d/d/d/f + for i in `jot 20`; do + cp -r $mp2/d $i + done + + rm -f $mp2/.snap/$sno + mksnap_ffs $mp2 $mp2/.snap/$sno & + sno=$(((sno + 1) % 10)) + + sleep 1 + rm -rf $mp/? + wait + while mount | grep -q "on $mp2 "; do umount $mp2; done + echo * | grep -q core && break + mdconfig -d -u $u2 + + # Introduce 5 random single bit errors to the file system image + /tmp/flip -n 5 $diskimage + + if [ `stat -f%z $diskimage` -gt $max ]; then + ls -lh $diskimage + truncate -s $max $diskimage + else + gzip < $diskimage > $backup + fi + fsync $backup + sync; sleep 1 + + [ $newfs_flags == "-j" ] && + fsck -fy $diskimage > $log 2>&1 # process the journal file + for i in `jot 5`; do + [ $i -gt 2 ] && echo "fsck run #$i" + chk $diskimage + [ $rerun -eq 1 ] && { reruns=$((reruns + 1)); continue; } + grep -q "MODIFIED" $log && continue # For now, do not trust CLEAN + [ $clean -eq 1 ] && { cleans=$((cleans + 1)); break; } + if [ -f fsck_ffs.core ]; then + tstamp=`date +%Y%m%dT%H%M%S` + gzip < $backup > /tmp/fsck_ffs.core.diskimage.$tstamp.gz + gzip < fsck_ffs.core > /tmp/fsck_ffs.core.$tstamp.gz + break 2 + fi + done + [ $clean -ne 1 ] && { s=99; break; } # broken image? + mdconfig -a -t vnode -f $diskimage -u $u2 +done +for i in `jot 5`; do + mount | grep -q "on $mp2 " || break + umount $mp2 && break + sleep 2 +done +mdconfig -l | grep -q $u2 && mdconfig -d -u $u2 + +[ $s -eq 0 ] && rm -f $backup || echo "Preserved $backup due to status code $s" +cd /tmp +for i in `jot 5`; do + umount $mp1 && break + sleep 2 +done +mdconfig -d -u $u1 +rm -f /tmp/flip +exit $s diff --git a/tools/test/stress2/misc/gnop13.sh b/tools/test/stress2/misc/gnop13.sh new file mode 100755 index 000000000000..45668b9d8e82 --- /dev/null +++ b/tools/test/stress2/misc/gnop13.sh @@ -0,0 +1,138 @@ +#!/bin/sh + +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2023 Peter Holm +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +# Snapshot test +# Variation of force13.sh, but uses "gnop -f destroy" + +# Seen: +# UFS: forcibly unmounting /dev/md10.nop from /mnt +# panic: flush_pagedep_deps: failed to flush inodedep + +[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 +. ../default.cfg + +set -u +log=/tmp/gnop13.sh.log +mount | grep $mntpoint | grep -q /dev/md && umount -f $mntpoint +mdconfig -l | grep -q md$mdstart && mdconfig -d -u $mdstart +truncate -s 10g $diskimage +mdconfig -a -t vnode -f $diskimage -u $mdstart +flags=$newfs_flags +[ `jot -r 1 0 1` -eq 1 ] && flags="-j" + +echo "newfs $flags md$mdstart" +newfs $flags md$mdstart > /dev/null 2>&1 + +# Exclude rename for now due to log0374.txt +export TESTPROGS=`cd ..; find testcases/ -perm -1 -type f | \ + egrep -Ev "/run/|/badcode/|/pty/|/shm/|/socket/|sysctl|tcp|thr|udp|rename"` +export runRUNTIME=3m +export RUNDIR=$mntpoint/stressX +export CTRLDIR=$mntpoint/stressX.control +start=`date +%s` +while [ $((`date +%s` - start)) -lt $((15 * 60)) ]; do + gnop create /dev/md$mdstart || exit 1 + mount /dev/md$mdstart.nop $mntpoint || exit 1 + rm -fr $mntpoint/lost+found + chmod 777 $mntpoint + + su $testuser -c 'cd ..; ./testcases/run/run $TESTPROGS' > \ + /dev/null 2>&1 & + + t=`jot -r 1 60 120` + st=`date +%s` + for i in `jot 10`; do + [ -d $mntpoint/.snap ] || break + rm -f $mntpoint/.snap/$i + echo "mksnap_ffs $mntpoint $mntpoint/.snap/$i" + mksnap_ffs $mntpoint $mntpoint/.snap/$i + sleep `jot -r 1 1 5` + [ $((`date +%s` - st)) -ge $t ] && break + done & + sleep `jot -r 1 60 120` + while [ -c /dev/md$mdstart.nop ]; do + echo "gnop destroy -f /dev/md$mdstart.nop" + gnop destroy -f /dev/md$mdstart.nop || sleep 1 + done + n=0 + st=`date +%s` + while mount | grep -q "on $mntpoint "; do + [ $n -eq 0 ] && /bin/echo -n "Waiting for $mntpoint to force umount ..." + n=$((n + 1)) + sleep 2 + if [ $((`date +%s` - st)) -ge 180 ]; then + echo "Giving up on waiting for umount of $mntpoint" + umount $mntpoint || umount -f $mntpoint + break + fi + done + [ $n -ne 0 ] && echo + + ../tools/killall.sh + wait + mount | grep -q "on $mntpoint " && umount -f $mntpoint + c=0 + # Run fsck minimum two times + for i in `jot 5`; do + fsck_ffs -fy /dev/md$mdstart > $log 2>&1; s=$? + grep -q CLEAN $log && grep -q "MODIFIED" $log && c=$((c+=1)) + grep -Eq "FILE SYSTEM WAS MODIFIED" $log || break + done + [ $c -gt 1 ] && + { echo "Note: FS marked clean+modified $c times out of $i fsck runs"; s=101; } + [ $s -ne 0 ] && break + grep -Eq "IS CLEAN|MARKED CLEAN" $log || { s=102; break; } +done +[ $s -eq 101 ] && s=0 # Ignore CLEANish problem for now +if [ $s -eq 0 ]; then + mount /dev/md$mdstart.nop $mntpoint + cp -R /usr/include $mntpoint + dd if=/dev/zero of=$mntpoint/big bs=1m count=10 status=none + find $mntpoint/* -delete + + # Check the RO snapshots + for f in $mntpoint/.snap/*; do + c=0 + for i in `jot 5`; do + echo "fsck_ffs -fy $f" + fsck_ffs -fy $f > $log 2>&1; s=$? + grep -q CLEAN $log && grep -q "MODIFIED" $log && c=$((c+=1)) + grep -Eq "FILE SYSTEM WAS MODIFIED" $log || break + done + [ $c -gt 1 ] && + { echo "Note: snapshot $i marked clean+modified $c times out of $i fsck runs"; s=201; } + [ $s -ne 0 ] && break + grep -Eq "IS CLEAN|MARKED CLEAN" $log || + { s=202; tail -10 $log; break; } + done + umount $mntpoint + mdconfig -d -u $mdstart + rm -f $diskimage $log +fi +exit $s