diff --git a/.github/workflows/scripts/generate-ci-type.py b/.github/workflows/scripts/generate-ci-type.py index 943aae254469..b49255e8381d 100755 --- a/.github/workflows/scripts/generate-ci-type.py +++ b/.github/workflows/scripts/generate-ci-type.py @@ -1,107 +1,108 @@ #!/usr/bin/env python3 """ Determine the CI type based on the change list and commit message. Prints "quick" if (explicity required by user): - the *last* commit message contains 'ZFS-CI-Type: quick' or if (heuristics): - the files changed are not in the list of specified directories, and - all commit messages do not contain 'ZFS-CI-Type: full' Otherwise prints "full". """ import sys import subprocess import re """ Patterns of files that are not considered to trigger full CI. Note: not using pathlib.Path.match() because it does not support '**' """ FULL_RUN_IGNORE_REGEX = list(map(re.compile, [ r'.*\.md', r'.*\.gitignore' ])) """ Patterns of files that are considered to trigger full CI. """ FULL_RUN_REGEX = list(map(re.compile, [ + r'\.github/workflows/scripts/.*', r'cmd.*', r'configs/.*', r'META', r'.*\.am', r'.*\.m4', r'autogen\.sh', r'configure\.ac', r'copy-builtin', r'contrib', r'etc', r'include', r'lib/.*', r'module/.*', r'scripts/.*', r'tests/.*', r'udev/.*' ])) if __name__ == '__main__': prog = sys.argv[0] if len(sys.argv) != 3: print(f'Usage: {prog} ') sys.exit(1) head, base = sys.argv[1:3] def output_type(type, reason): print(f'{prog}: will run {type} CI: {reason}', file=sys.stderr) print(type) sys.exit(0) # check last (HEAD) commit message last_commit_message_raw = subprocess.run([ 'git', 'show', '-s', '--format=%B', 'HEAD' ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for line in last_commit_message_raw.stdout.decode().splitlines(): if line.strip().lower() == 'zfs-ci-type: quick': output_type('quick', f'explicitly requested by HEAD commit {head}') # check all commit messages all_commit_message_raw = subprocess.run([ 'git', 'show', '-s', '--format=ZFS-CI-Commit: %H%n%B', f'{head}...{base}' ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) all_commit_message = all_commit_message_raw.stdout.decode().splitlines() commit_ref = head for line in all_commit_message: if line.startswith('ZFS-CI-Commit:'): commit_ref = line.lstrip('ZFS-CI-Commit:').rstrip() if line.strip().lower() == 'zfs-ci-type: full': output_type('full', f'explicitly requested by commit {commit_ref}') # check changed files changed_files_raw = subprocess.run([ 'git', 'diff', '--name-only', head, base ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) changed_files = changed_files_raw.stdout.decode().splitlines() for f in changed_files: for r in FULL_RUN_IGNORE_REGEX: if r.match(f): break else: for r in FULL_RUN_REGEX: if r.match(f): output_type( 'full', f'changed file "{f}" matches pattern "{r.pattern}"' ) # catch-all output_type('quick', 'no changed file matches full CI patterns') diff --git a/.github/workflows/scripts/qemu-1-setup.sh b/.github/workflows/scripts/qemu-1-setup.sh index f838da34efff..de29ad1f57b6 100755 --- a/.github/workflows/scripts/qemu-1-setup.sh +++ b/.github/workflows/scripts/qemu-1-setup.sh @@ -1,93 +1,67 @@ #!/usr/bin/env bash ###################################################################### # 1) setup qemu instance on action runner ###################################################################### set -eu # install needed packages export DEBIAN_FRONTEND="noninteractive" sudo apt-get -y update sudo apt-get install -y axel cloud-image-utils daemonize guestfs-tools \ - ksmtuned virt-manager linux-modules-extra-$(uname -r) zfsutils-linux + virt-manager linux-modules-extra-$(uname -r) zfsutils-linux # generate ssh keys rm -f ~/.ssh/id_ed25519 ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N "" -# we expect RAM shortage -cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null -# /etc/ksmtuned.conf - Configuration file for ksmtuned -# https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/virtualization_tuning_and_optimization_guide/chap-ksm -KSM_MONITOR_INTERVAL=60 - -# Millisecond sleep between ksm scans for 16Gb server. -# Smaller servers sleep more, bigger sleep less. -KSM_SLEEP_MSEC=30 - -KSM_NPAGES_BOOST=0 -KSM_NPAGES_DECAY=0 -KSM_NPAGES_MIN=1000 -KSM_NPAGES_MAX=25000 - -KSM_THRES_COEF=80 -KSM_THRES_CONST=8192 - -LOGFILE=/var/log/ksmtuned.log -DEBUG=1 -EOF -sudo systemctl restart ksm -sudo systemctl restart ksmtuned - # not needed sudo systemctl stop docker.socket sudo systemctl stop multipathd.socket # remove default swapfile and /mnt sudo swapoff -a sudo umount -l /mnt DISK="/dev/disk/cloud/azure_resource-part1" sudo sed -e "s|^$DISK.*||g" -i /etc/fstab sudo wipefs -aq $DISK sudo systemctl daemon-reload sudo modprobe loop sudo modprobe zfs # partition the disk as needed DISK="/dev/disk/cloud/azure_resource" sudo sgdisk --zap-all $DISK sudo sgdisk -p \ -n 1:0:+16G -c 1:"swap" \ -n 2:0:0 -c 2:"tests" \ $DISK sync sleep 1 -# swap with same size as RAM +# swap with same size as RAM (16GiB) sudo mkswap $DISK-part1 sudo swapon $DISK-part1 -# 60GB data disk +# JBOD 2xdisk for OpenZFS storage (test vm's) SSD1="$DISK-part2" - -# 10GB data disk on ext4 -sudo fallocate -l 10G /test.ssd1 -SSD2=$(sudo losetup -b 4096 -f /test.ssd1 --show) +sudo fallocate -l 12G /test.ssd2 +SSD2=$(sudo losetup -b 4096 -f /test.ssd2 --show) # adjust zfs module parameter and create pool exec 1>/dev/null ARC_MIN=$((1024*1024*256)) ARC_MAX=$((1024*1024*512)) echo $ARC_MIN | sudo tee /sys/module/zfs/parameters/zfs_arc_min echo $ARC_MAX | sudo tee /sys/module/zfs/parameters/zfs_arc_max echo 1 | sudo tee /sys/module/zfs/parameters/zvol_use_blk_mq -sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 \ - -O relatime=off -O atime=off -O xattr=sa -O compression=lz4 \ - -O mountpoint=/mnt/tests +sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 -O relatime=off \ + -O atime=off -O xattr=sa -O compression=lz4 -O sync=disabled \ + -O redundant_metadata=none -O mountpoint=/mnt/tests # no need for some scheduler for i in /sys/block/s*/queue/scheduler; do - echo "none" | sudo tee $i > /dev/null + echo "none" | sudo tee $i done diff --git a/.github/workflows/scripts/qemu-2-start.sh b/.github/workflows/scripts/qemu-2-start.sh index 43e769f6f98b..53a6ca98ed46 100755 --- a/.github/workflows/scripts/qemu-2-start.sh +++ b/.github/workflows/scripts/qemu-2-start.sh @@ -1,244 +1,259 @@ #!/usr/bin/env bash ###################################################################### # 2) start qemu with some operating system, init via cloud-init ###################################################################### set -eu # short name used in zfs-qemu.yml OS="$1" # OS variant (virt-install --os-variant list) OSv=$OS # compressed with .zst extension REPO="https://github.com/mcmilk/openzfs-freebsd-images" FREEBSD="$REPO/releases/download/v2025-04-13" URLzs="" # Ubuntu mirrors UBMIRROR="https://cloud-images.ubuntu.com" #UBMIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images" #UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images" # default nic model for vm's NIC="virtio" case "$OS" in almalinux8) OSNAME="AlmaLinux 8" URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2" ;; almalinux9) OSNAME="AlmaLinux 9" URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2" ;; archlinux) OSNAME="Archlinux" URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2" ;; centos-stream10) OSNAME="CentOS Stream 10" # TODO: #16903 Overwrite OSv to stream9 for virt-install until it's added to osinfo OSv="centos-stream9" URL="https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2" ;; centos-stream9) OSNAME="CentOS Stream 9" URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2" ;; debian11) OSNAME="Debian 11" URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2" ;; debian12) OSNAME="Debian 12" URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2" ;; fedora40) OSNAME="Fedora 40" OSv="fedora-unknown" URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2" ;; fedora41) OSNAME="Fedora 41" OSv="fedora-unknown" URL="https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2" ;; fedora42) OSNAME="Fedora 42" OSv="fedora-unknown" URL="https://download.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-42-1.1.x86_64.qcow2" ;; freebsd13-4r) OSNAME="FreeBSD 13.4-RELEASE" OSv="freebsd13.0" URLzs="$FREEBSD/amd64-freebsd-13.4-RELEASE.qcow2.zst" BASH="/usr/local/bin/bash" NIC="rtl8139" ;; freebsd13-5r) OSNAME="FreeBSD 13.5-RELEASE" OSv="freebsd13.0" URLzs="$FREEBSD/amd64-freebsd-13.5-RELEASE.qcow2.zst" BASH="/usr/local/bin/bash" NIC="rtl8139" ;; freebsd14-1r) OSNAME="FreeBSD 14.1-RELEASE" OSv="freebsd14.0" URLzs="$FREEBSD/amd64-freebsd-14.1-RELEASE.qcow2.zst" BASH="/usr/local/bin/bash" ;; freebsd14-2r) OSNAME="FreeBSD 14.2-RELEASE" OSv="freebsd14.0" URLzs="$FREEBSD/amd64-freebsd-14.2-RELEASE.qcow2.zst" BASH="/usr/local/bin/bash" ;; freebsd13-5s) OSNAME="FreeBSD 13.5-STABLE" OSv="freebsd13.0" URLzs="$FREEBSD/amd64-freebsd-13.5-STABLE.qcow2.zst" BASH="/usr/local/bin/bash" NIC="rtl8139" ;; freebsd14-2s) OSNAME="FreeBSD 14.2-STABLE" OSv="freebsd14.0" URLzs="$FREEBSD/amd64-freebsd-14.2-STABLE.qcow2.zst" BASH="/usr/local/bin/bash" ;; freebsd15-0c) OSNAME="FreeBSD 15.0-CURRENT" OSv="freebsd14.0" URLzs="$FREEBSD/amd64-freebsd-15.0-CURRENT.qcow2.zst" BASH="/usr/local/bin/bash" ;; tumbleweed) OSNAME="openSUSE Tumbleweed" OSv="opensusetumbleweed" MIRROR="http://opensuse-mirror-gce-us.susecloud.net" URL="$MIRROR/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2" ;; ubuntu20) OSNAME="Ubuntu 20.04" OSv="ubuntu20.04" URL="$UBMIRROR/focal/current/focal-server-cloudimg-amd64.img" ;; ubuntu22) OSNAME="Ubuntu 22.04" OSv="ubuntu22.04" URL="$UBMIRROR/jammy/current/jammy-server-cloudimg-amd64.img" ;; ubuntu24) OSNAME="Ubuntu 24.04" OSv="ubuntu24.04" URL="$UBMIRROR/noble/current/noble-server-cloudimg-amd64.img" ;; *) echo "Wrong value for OS variable!" exit 111 ;; esac # environment file ENV="/var/tmp/env.txt" echo "ENV=$ENV" >> $ENV # result path echo 'RESPATH="/var/tmp/test_results"' >> $ENV -# FreeBSD 13 has problems with: e1000+virtio +# FreeBSD 13 has problems with: e1000 and virtio echo "NIC=$NIC" >> $ENV # freebsd15 -> used in zfs-qemu.yml echo "OS=$OS" >> $ENV # freebsd14.0 -> used for virt-install echo "OSv=\"$OSv\"" >> $ENV # FreeBSD 15 (Current) -> used for summary echo "OSNAME=\"$OSNAME\"" >> $ENV +# default vm count for testings +VMs=2 +echo "VMs=\"$VMs\"" >> $ENV + +# default cpu count for testing vm's +CPU=2 +echo "CPU=\"$CPU\"" >> $ENV + sudo mkdir -p "/mnt/tests" sudo chown -R $(whoami) /mnt/tests # we are downloading via axel, curl and wget are mostly slower and # require more return value checking IMG="/mnt/tests/cloudimg.qcow2" if [ ! -z "$URLzs" ]; then echo "Loading image $URLzs ..." time axel -q -o "$IMG.zst" "$URLzs" zstd -q -d --rm "$IMG.zst" else echo "Loading image $URL ..." time axel -q -o "$IMG" "$URL" fi DISK="/dev/zvol/zpool/openzfs" FORMAT="raw" sudo zfs create -ps -b 64k -V 80g zpool/openzfs while true; do test -b $DISK && break; sleep 1; done echo "Importing VM image to zvol..." sudo qemu-img dd -f qcow2 -O raw if=$IMG of=$DISK bs=4M rm -f $IMG PUBKEY=$(cat ~/.ssh/id_ed25519.pub) cat < /tmp/user-data #cloud-config fqdn: $OS users: - name: root shell: $BASH - name: zfs sudo: ALL=(ALL) NOPASSWD:ALL shell: $BASH ssh_authorized_keys: - $PUBKEY growpart: mode: auto devices: ['/'] ignore_growroot_disabled: false EOF sudo virsh net-update default add ip-dhcp-host \ "" --live --config sudo virt-install \ --os-variant $OSv \ --name "openzfs" \ --cpu host-passthrough \ --virt-type=kvm --hvm \ --vcpus=4,sockets=1 \ --memory $((1024*12)) \ --memballoon model=virtio \ --graphics none \ --network bridge=virbr0,model=$NIC,mac='52:54:00:83:79:00' \ --cloud-init user-data=/tmp/user-data \ --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \ --import --noautoconsole >/dev/null +# enable KSM on Linux +if [ ${OS:0:7} != "freebsd" ]; then + sudo virsh dommemstat --domain "openzfs" --period 5 + sudo virsh node-memory-tune 100 50 1 + echo 1 | sudo tee /sys/kernel/mm/ksm/run > /dev/null +fi + # Give the VMs hostnames so we don't have to refer to them with # hardcoded IP addresses. # # vm0: Initial VM we install dependencies and build ZFS on. # vm1..2 Testing VMs -for i in {0..9} ; do +for ((i=0; i<=VMs; i++)); do echo "192.168.122.1$i vm$i" | sudo tee -a /etc/hosts done # in case the directory isn't there already mkdir -p $HOME/.ssh cat <> $HOME/.ssh/config # no questions please StrictHostKeyChecking no # small timeout, used in while loops later ConnectTimeout 1 EOF diff --git a/.github/workflows/scripts/qemu-5-setup.sh b/.github/workflows/scripts/qemu-5-setup.sh index 5b5bf5d7f7bb..6bf10024a1a6 100755 --- a/.github/workflows/scripts/qemu-5-setup.sh +++ b/.github/workflows/scripts/qemu-5-setup.sh @@ -1,124 +1,116 @@ #!/usr/bin/env bash ###################################################################### # 5) start test machines and load openzfs module ###################################################################### set -eu # read our defined variables source /var/tmp/env.txt # wait for poweroff to succeed PID=$(pidof /usr/bin/qemu-system-x86_64) tail --pid=$PID -f /dev/null sudo virsh undefine openzfs -# default values per test vm: -VMs=2 -CPU=2 - # cpu pinning CPUSET=("0,1" "2,3") case "$OS" in freebsd*) - # FreeBSD can't be optimized via ksmtuned + # FreeBSD needs only 6GiB RAM=6 ;; *) - # Linux can be optimized via ksmtuned + # Linux needs more memory, but can be optimized to share it via KSM RAM=8 ;; esac -# this can be different for each distro -echo "VMs=$VMs" >> $ENV - # create snapshot we can clone later sudo zfs snapshot zpool/openzfs@now # setup the testing vm's PUBKEY=$(cat ~/.ssh/id_ed25519.pub) -for i in $(seq 1 $VMs); do +# start testing VMs +for ((i=1; i<=VMs; i++)); do echo "Creating disk for vm$i..." DISK="/dev/zvol/zpool/vm$i" FORMAT="raw" - sudo zfs clone zpool/openzfs@now zpool/vm$i - sudo zfs create -ps -b 64k -V 80g zpool/vm$i-2 + sudo zfs clone zpool/openzfs@now zpool/vm$i-system + sudo zfs create -ps -b 64k -V 64g zpool/vm$i-tests cat < /tmp/user-data #cloud-config fqdn: vm$i users: - name: root shell: $BASH - name: zfs sudo: ALL=(ALL) NOPASSWD:ALL shell: $BASH ssh_authorized_keys: - $PUBKEY growpart: mode: auto devices: ['/'] ignore_growroot_disabled: false EOF sudo virsh net-update default add ip-dhcp-host \ "" --live --config sudo virt-install \ --os-variant $OSv \ --name "vm$i" \ --cpu host-passthrough \ --virt-type=kvm --hvm \ --vcpus=$CPU,sockets=1 \ --cpuset=${CPUSET[$((i-1))]} \ --memory $((1024*RAM)) \ --memballoon model=virtio \ --graphics none \ --cloud-init user-data=/tmp/user-data \ --network bridge=virbr0,model=$NIC,mac="52:54:00:83:79:0$i" \ - --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \ - --disk $DISK-2,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \ + --disk $DISK-system,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \ + --disk $DISK-tests,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \ --import --noautoconsole >/dev/null done -# check the memory state from time to time +# generate some memory stats cat < cronjob.sh -# $OS exec 1>>/var/tmp/stats.txt exec 2>&1 -echo "*******************************************************" -date +echo "********************************************************************************" uptime free -m -df -h /mnt/tests zfs list EOF + sudo chmod +x cronjob.sh sudo mv -f cronjob.sh /root/cronjob.sh echo '*/5 * * * * /root/cronjob.sh' > crontab.txt sudo crontab crontab.txt rm crontab.txt # check if the machines are okay echo "Waiting for vm's to come up... (${VMs}x CPU=$CPU RAM=$RAM)" -for i in $(seq 1 $VMs); do +for ((i=1; i<=VMs; i++)); do .github/workflows/scripts/qemu-wait-for-vm.sh vm$i done echo "All $VMs VMs are up now." # Save the VM's serial output (ttyS0) to /var/tmp/console.txt # - ttyS0 on the VM corresponds to a local /dev/pty/N entry # - use 'virsh ttyconsole' to lookup the /dev/pty/N entry -for i in $(seq 1 $VMs); do +for ((i=1; i<=VMs; i++)); do mkdir -p $RESPATH/vm$i read "pty" <<< $(sudo virsh ttyconsole vm$i) sudo nohup bash -c "cat $pty > $RESPATH/vm$i/console.txt" & done echo "Console logging for ${VMs}x $OS started." diff --git a/.github/workflows/scripts/qemu-6-tests.sh b/.github/workflows/scripts/qemu-6-tests.sh index 2f023198bbf6..0115f4114604 100755 --- a/.github/workflows/scripts/qemu-6-tests.sh +++ b/.github/workflows/scripts/qemu-6-tests.sh @@ -1,105 +1,110 @@ #!/usr/bin/env bash ###################################################################### # 6) load openzfs module and run the tests # # called on runner: qemu-6-tests.sh # called on qemu-vm: qemu-6-tests.sh $OS $2/$3 ###################################################################### set -eu function prefix() { ID="$1" LINE="$2" CURRENT=$(date +%s) TSSTART=$(cat /tmp/tsstart) DIFF=$((CURRENT-TSSTART)) H=$((DIFF/3600)) DIFF=$((DIFF-(H*3600))) M=$((DIFF/60)) S=$((DIFF-(M*60))) CTR=$(cat /tmp/ctr) echo $LINE| grep -q "^Test[: ]" && CTR=$((CTR+1)) && echo $CTR > /tmp/ctr BASE="$HOME/work/zfs/zfs" COLOR="$BASE/scripts/zfs-tests-color.sh" CLINE=$(echo $LINE| grep "^Test[ :]" | sed -e 's|/usr/local|/usr|g' \ | sed -e 's| /usr/share/zfs/zfs-tests/tests/| |g' | $COLOR) if [ -z "$CLINE" ]; then printf "vm${ID}: %s\n" "$LINE" else # [vm2: 00:15:54 256] Test: functional/checksum/setup (run as root) [00:00] [PASS] printf "[vm${ID}: %02d:%02d:%02d %4d] %s\n" \ "$H" "$M" "$S" "$CTR" "$CLINE" fi } # called directly on the runner if [ -z ${1:-} ]; then cd "/var/tmp" source env.txt SSH=$(which ssh) TESTS='$HOME/zfs/.github/workflows/scripts/qemu-6-tests.sh' echo 0 > /tmp/ctr date "+%s" > /tmp/tsstart - for i in $(seq 1 $VMs); do + for ((i=1; i<=VMs; i++)); do IP="192.168.122.1$i" daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \ $SSH zfs@$IP $TESTS $OS $i $VMs $CI_TYPE # handly line by line and add info prefix stdbuf -oL tail -fq vm${i}log.txt \ | while read -r line; do prefix "$i" "$line"; done & echo $! > vm${i}log.pid # don't mix up the initial --- Configuration --- part sleep 0.13 done # wait for all vm's to finish - for i in $(seq 1 $VMs); do + for ((i=1; i<=VMs; i++)); do tail --pid=$(cat vm${i}.pid) -f /dev/null pid=$(cat vm${i}log.pid) rm -f vm${i}log.pid kill $pid done exit 0 fi # this part runs inside qemu vm export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/sbin:/usr/local/bin" case "$1" in freebsd*) + TDIR="/usr/local/share/zfs" sudo kldstat -n zfs 2>/dev/null && sudo kldunload zfs sudo -E ./zfs/scripts/zfs.sh - TDIR="/usr/local/share/zfs" + sudo mv -f /var/tmp/*.txt /tmp + sudo newfs -U -t -L tmp /dev/vtbd1 >/dev/null + sudo mount -o noatime /dev/vtbd1 /var/tmp + sudo chmod 1777 /var/tmp + sudo mv -f /tmp/*.txt /var/tmp ;; *) # use xfs @ /var/tmp for all distros + TDIR="/usr/share/zfs" + sudo -E modprobe zfs sudo mv -f /var/tmp/*.txt /tmp sudo mkfs.xfs -fq /dev/vdb sudo mount -o noatime /dev/vdb /var/tmp sudo chmod 1777 /var/tmp sudo mv -f /tmp/*.txt /var/tmp - sudo -E modprobe zfs - TDIR="/usr/share/zfs" ;; esac # run functional testings and save exitcode cd /var/tmp TAGS=$2/$3 if [ "$4" == "quick" ]; then export RUNFILES="sanity.run" fi sudo dmesg -c > dmesg-prerun.txt mount > mount.txt df -h > df-prerun.txt $TDIR/zfs-tests.sh -vK -s 3GB -T $TAGS RV=$? df -h > df-postrun.txt echo $RV > tests-exitcode.txt sync exit 0 diff --git a/.github/workflows/scripts/qemu-7-prepare.sh b/.github/workflows/scripts/qemu-7-prepare.sh index 51a3dbe64e5f..98a5c24c2521 100755 --- a/.github/workflows/scripts/qemu-7-prepare.sh +++ b/.github/workflows/scripts/qemu-7-prepare.sh @@ -1,124 +1,124 @@ #!/usr/bin/env bash ###################################################################### # 7) prepare output of the results # - this script pre-creates all needed logfiles for later summary ###################################################################### set -eu # read our defined variables cd /var/tmp source env.txt mkdir -p $RESPATH # check if building the module has failed if [ -z ${VMs:-} ]; then cd $RESPATH echo ":exclamation: ZFS module didn't build successfully :exclamation:" \ | tee summary.txt | tee /tmp/summary.txt cp /var/tmp/*.txt . tar cf /tmp/qemu-$OS.tar -C $RESPATH -h . || true exit 0 fi # build was okay BASE="$HOME/work/zfs/zfs" MERGE="$BASE/.github/workflows/scripts/merge_summary.awk" # catch result files of testings (vm's should be there) -for i in $(seq 1 $VMs); do +for ((i=1; i<=VMs; i++)); do rsync -arL zfs@vm$i:$RESPATH/current $RESPATH/vm$i || true scp zfs@vm$i:"/var/tmp/*.txt" $RESPATH/vm$i || true scp zfs@vm$i:"/var/tmp/*.rpm" $RESPATH/vm$i || true done cp -f /var/tmp/*.txt $RESPATH || true cd $RESPATH # prepare result files for summary -for i in $(seq 1 $VMs); do +for ((i=1; i<=VMs; i++)); do file="vm$i/build-stderr.txt" test -s $file && mv -f $file build-stderr.txt file="vm$i/build-exitcode.txt" test -s $file && mv -f $file build-exitcode.txt file="vm$i/uname.txt" test -s $file && mv -f $file uname.txt file="vm$i/tests-exitcode.txt" if [ ! -s $file ]; then # XXX - add some tests for kernel panic's here # tail -n 80 vm$i/console.txt | grep XYZ echo 1 > $file fi rv=$(cat vm$i/tests-exitcode.txt) test $rv != 0 && touch /tmp/have_failed_tests file="vm$i/current/log" if [ -s $file ]; then cat $file >> log awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }; \ /\[SKIP\]|\[PASS\]/{ show=0; } show' \ $file > /tmp/vm${i}dbg.txt fi file="vm${i}log.txt" fileC="/tmp/vm${i}log.txt" if [ -s $file ]; then cat $file >> summary cat $file | $BASE/scripts/zfs-tests-color.sh > $fileC fi done # create summary of tests if [ -s summary ]; then $MERGE summary | grep -v '^/' > summary.txt $MERGE summary | $BASE/scripts/zfs-tests-color.sh > /tmp/summary.txt rm -f summary else touch summary.txt /tmp/summary.txt fi # create file for debugging if [ -s log ]; then awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }; \ /\[SKIP\]|\[PASS\]/{ show=0; } show' \ log > summary-failure-logs.txt rm -f log else touch summary-failure-logs.txt fi # create debug overview for failed tests cat summary.txt \ | awk '/\(expected PASS\)/{ if ($1!="SKIP") print $2; next; } show' \ | while read t; do cat summary-failure-logs.txt \ | awk '$0~/Test[: ]/{ show=0; } $0~v{ show=1; } show' v="$t" \ > /tmp/fail.txt SIZE=$(stat --printf="%s" /tmp/fail.txt) SIZE=$((SIZE/1024)) # Test Summary: echo "##[group]$t ($SIZE KiB)" >> /tmp/failed.txt cat /tmp/fail.txt | $BASE/scripts/zfs-tests-color.sh >> /tmp/failed.txt echo "##[endgroup]" >> /tmp/failed.txt # Job Summary: echo -e "\n
\n$t ($SIZE KiB)
" >> failed.txt
   cat /tmp/fail.txt >> failed.txt
   echo "
" >> failed.txt done if [ -e /tmp/have_failed_tests ]; then echo ":warning: Some tests failed!" >> failed.txt else echo ":thumbsup: All tests passed." >> failed.txt fi if [ ! -s uname.txt ]; then echo ":interrobang: Panic - where is my uname.txt?" > uname.txt fi # artifact ready now tar cf /tmp/qemu-$OS.tar -C $RESPATH -h . || true diff --git a/.github/workflows/scripts/qemu-8-summary.sh b/.github/workflows/scripts/qemu-8-summary.sh index 26dbab28323b..7d1e16567ab4 100755 --- a/.github/workflows/scripts/qemu-8-summary.sh +++ b/.github/workflows/scripts/qemu-8-summary.sh @@ -1,71 +1,71 @@ #!/usr/bin/env bash ###################################################################### # 8) show colored output of results ###################################################################### set -eu # read our defined variables source /var/tmp/env.txt cd $RESPATH # helper function for showing some content with headline function showfile() { content=$(dd if=$1 bs=1024 count=400k 2>/dev/null) if [ -z "$2" ]; then group1="" group2="" else SIZE=$(stat --printf="%s" "$file") SIZE=$((SIZE/1024)) group1="##[group]$2 ($SIZE KiB)" group2="##[endgroup]" fi cat < tmp$$ $group1 $content $group2 EOF cat tmp$$ rm -f tmp$$ } # overview cat /tmp/summary.txt echo "" if [ -f /tmp/have_failed_tests -a -s /tmp/failed.txt ]; then echo "Debuginfo of failed tests:" cat /tmp/failed.txt echo "" cat /tmp/summary.txt | grep -v '^/' echo "" fi echo -e "\nFull logs for download:\n $1\n" -for i in $(seq 1 $VMs); do +for ((i=1; i<=VMs; i++)); do rv=$(cat vm$i/tests-exitcode.txt) if [ $rv = 0 ]; then vm="vm$i" else vm="vm$i" fi file="vm$i/dmesg-prerun.txt" test -s "$file" && showfile "$file" "$vm: dmesg kernel" file="/tmp/vm${i}log.txt" test -s "$file" && showfile "$file" "$vm: test results" file="vm$i/console.txt" test -s "$file" && showfile "$file" "$vm: serial console" file="/tmp/vm${i}dbg.txt" test -s "$file" && showfile "$file" "$vm: failure logfile" done test -f /tmp/have_failed_tests && exit 1 exit 0