diff --git a/tests/ci/Makefile b/tests/ci/Makefile --- a/tests/ci/Makefile +++ b/tests/ci/Makefile @@ -8,8 +8,9 @@ # Makefile for CI testing. # # User-driven targets: -# ci: Run CI tests. Currently only smoke tests are supported. -# ci-smokeit: Currently same as ci. +# ci: Run CI tests +# ci-smoke: Run smoke tests which is simply booting the image +# ci-full: Run full tests # # Variables affecting the build process: # TARGET/TARGET_ARCH: architecture of built release (default: same as build host) @@ -68,11 +69,26 @@ VMFS?= ufs FORMAT= raw CIIMAGE= ci-${OSRELEASE}-${GITREV}-${KERNCONF}.${FORMAT} +CIDISK?= ${.OBJDIR}/${CIIMAGE} VMSIZE?= 6g -CITYPE?= +CITYPE?= full +CIDISABLED?= +BHYVE_EXTRA_DISK_PARAM?= +META_TAR!=mktemp /tmp/meta.XXXXXX +META_DIR!=mktemp -d /tmp/meta.XXXXXX +META_DIROUT!=mktemp -d /tmp/meta.XXXXXX +DISC_CAM!=truncate -s 128m /tmp/disk-cam +EXTRA_DISK_NUM?=5 +DISK_NUMBERS!=jot - 1 ${EXTRA_DISK_NUM} +BHYVE_EXTRA_DISK_PARAM+=-s 4:0,ahci-hd,/tmp/disk-cam +.for i in ${DISK_NUMBERS} +NEW_DISK!=truncate -s 128m /tmp/disk${i} +BHYVE_EXTRA_DISK_PARAM+=-s $$((${i} + 4)):0,virtio-blk,/tmp/disk${i} +CLEANFILES+=/tmp/disk${i} +.endfor TEST_VM_NAME= ci-${OSRELEASE}-${GITREV}-${KERNCONF} .if ${TOTAL_MEMORY} >= 16 -VM_MEM!=expr ${TOTAL_MEMORY} / 2 +VM_MEM=8 .elif ${TOTAL_MEMORY} >=4 VM_MEM=${TOTAL_MEMORY} .else @@ -117,10 +133,10 @@ METAMODE?=-DWITH_META_MODE .endif -CLEANFILES= ${CIIMAGE} ci.img -CLEANDIRS= ci-buildimage +CLEANFILES+= ${.OBJDIR}/${CIIMAGE} ${.OBJDIR}/ci.img ${META_TAR} +CLEANDIRS+= ${.OBJDIR}/ci-buildimage -portinstall: portinstall-pkg portinstall-qemu portinstall-expect portinstall-${TARGET_ARCH:tl} .PHONY +portinstall: portinstall-pkg portinstall-qemu portinstall-expect portinstall-${TARGET_ARCH:tl} portinstall-xmlstarlet .PHONY portinstall-pkg: .PHONY .if !exists(/usr/local/sbin/pkg-static) @@ -140,12 +156,19 @@ env ASSUME_ALWAYS_YES=yes pkg install lang/expect .endif +portinstall-xmlstarlet: portinstall-pkg .PHONY +.if !exists(/usr/local/bin/xml) + env ASSUME_ALWAYS_YES=yes pkg install textproc/xmlstarlet +.endif + beforeclean: .PHONY - chflags -R noschg . + chflags -R noschg ${.OBJDIR}/${.TARGET} .include clean: beforeclean .PHONY +cleandir: beforeclean .PHONY + ci-buildworld: .PHONY @echo "Building world for ${TARGET_ARCH}" ${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \ @@ -155,7 +178,7 @@ ci-buildkernel: ci-buildworld-${TARGET_ARCH:tl} .PHONY - @echo "Building kenrel for ${TARGET_ARCH"}" + @echo "Building kernel for ${TARGET_ARCH"}" ${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \ ${CROSS_TOOLCHAIN_PARAM} __MAKE_CONF=${MAKECONF} \ SRCCONF=${SRCCONF} buildkernel > ${.CURDIR}/_.${TARGET_ARCH}.${.TARGET} 2>&1 || \ @@ -174,19 +197,28 @@ (echo "${.TARGET} failed, check _.${TARGET_ARCH}.${.TARGET} for details" ; false) touch ${.TARGET} -ci-setsmokevar: .PHONY +ci-set-smoke-var: .PHONY CITYPE=smoke +ci-set-full-var: .PHONY +CITYPE=full + +ci-create-meta: .PHONY + @truncate -s 512M ${META_TAR} + @tar rvf ${META_TAR} -C ${META_DIR} . + ci-runtest: ci-buildimage-${TARGET_ARCH:tl} portinstall .PHONY .if ${MACHINE} == "amd64" && ( ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" ) && ( !defined(USE_QEMU) || empty(USE_QEMU) ) /usr/sbin/bhyvectl --vm=${TEST_VM_NAME} --destroy || true - /usr/sbin/bhyveload -c stdio -m ${VM_MEM_SIZE} -d ${CIIMAGE} ${TEST_VM_NAME} + /usr/sbin/bhyveload -c stdio -m ${VM_MEM_SIZE} -d ${CIDISK} ${TEST_VM_NAME} expect -c "set timeout ${TIMEOUT_EXPECT}; \ spawn /usr/bin/timeout -k 60 ${TIMEOUT_VM} /usr/sbin/bhyve \ -c ${PARALLEL_JOBS} -m ${VM_MEM_SIZE} -A -H -P \ -s 0:0,hostbridge \ -s 1:0,lpc \ - -s 2:0,virtio-blk,${CIIMAGE} \ + -s 2:0,virtio-blk,${CIDISK} \ + -s 3:0,virtio-blk,${META_TAR} \ + ${BHYVE_EXTRA_DISK_PARAM} \ -l com1,stdio \ ${TEST_VM_NAME}; \ expect { eof }" @@ -199,7 +231,8 @@ -nographic \ -no-reboot \ ${QEMU_EXTRA_PARAM} \ - -drive if=none,file=${CIIMAGE},format=raw,id=hd0 \ + -drive if=none,file=${CIDISK},format=raw,id=hd0 \ + -drive if=none,file=${META_TAR},format=raw,id=hd1 \ ${QEMU_DEVICES} .endif @@ -215,8 +248,10 @@ @echo "Error: ${TARGET_ARCH} is not supported on ${TYPE} ${REVISION} ${BRANCH}" .endif -ci-smokeit: ci-setsmokevar ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY +ci-smoke: ci-set-smoke-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY + +ci-full: ci-set-full-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY -ci: ci-smokeit .PHONY +ci: ci-${CITYPE:tl} .PHONY .include "${RELEASEDIR}/Makefile.inc1" diff --git a/tests/ci/tools/ci.conf b/tests/ci/tools/ci.conf --- a/tests/ci/tools/ci.conf +++ b/tests/ci/tools/ci.conf @@ -11,7 +11,7 @@ export VM_RC_LIST="auditd freebsdci" if [ "${CITYPE}" != "smoke" ]; then -export VM_EXTRA_PACKAGES="coreutils devel/py-pytest gdb jq ksh93 net/py-dpkt net/scapy nist-kat nmap perl5 python python3 sudo tcptestsuite" +export VM_EXTRA_PACKAGES="coreutils devel/py-pytest gdb jq ksh93 net/py-dpkt net/scapy nist-kat nmap perl5 python python3 sudo sysutils/porch tcptestsuite" if [ "${TARGET}" = "amd64" ]; then export VM_EXTRA_PACKAGES="${VM_EXTRA_PACKAGES} linux-c7-ltp" @@ -41,6 +41,7 @@ kld_list="" # Load modules needed by tests kld_list="${kld_list} blake2" # sys/opencrypto kld_list="${kld_list} cryptodev" # sys/opencrypto +kld_list="${kld_list} dummymbuf" # sys/netpfil kld_list="${kld_list} fusefs" # sys/fs/fusefs kld_list="${kld_list} ipsec" # sys/netipsec kld_list="${kld_list} mac_portacl" # sys/mac/portacl @@ -64,6 +65,10 @@ cat << EOF >> ${DESTDIR}/etc/rc.conf freebsdci_type="smoke" EOF +elif [ "${CITYPE}" = "full" ]; then +cat << EOF >> ${DESTDIR}/etc/rc.conf +freebsdci_type="full" +EOF fi cat << EOF >> ${DESTDIR}/etc/sysctl.conf kern.cryptodevallowsoft=1 @@ -77,7 +82,6 @@ fdesc /dev/fd fdescfs rw 0 0 EOF mkdir -p ${DESTDIR}/usr/local/etc/rc.d - echo $scriptdir cp -p ${scriptdir}/../../tests/ci/tools/freebsdci ${DESTDIR}/usr/local/etc/rc.d/ touch ${DESTDIR}/firstboot diff --git a/tests/ci/tools/freebsdci b/tests/ci/tools/freebsdci --- a/tests/ci/tools/freebsdci +++ b/tests/ci/tools/freebsdci @@ -26,6 +26,8 @@ . /etc/rc.subr : ${freebsdci_enable:="NO"} +: ${freebsdci_type:="full"} +: ${freebsdci_disable:=""} name="freebsdci" desc="Run FreeBSD CI" @@ -33,10 +35,13 @@ start_cmd="firstboot_ci_run" stop_cmd=":" os_arch=$(uname -p) +tardev=/dev/vtbd1 +metadir=/meta +istar=$(file -s ${tardev} | grep "POSIX tar archive" | wc -l) auto_shutdown() { - # XXX: Currently RISC-V kernels lack the ability to + # NOTE: Currently RISC-V kernels lack the ability to # make qemu exit on shutdown. Reboot instead; # it makes qemu exit too. case "$os_arch" in @@ -49,12 +54,64 @@ esac } +# Functions for specific disable actions +disable_disks_tests() { + echo "Disabling disk-related tests..." + if [ -f /etc/kyua/kyua.conf ]; then + sed -i .bak -E \ + -e 's,test_suites.FreeBSD.(cam|disks),-- &,' \ + /etc/kyua/kyua.conf + fi +} + +disable_dtrace_tests() { + echo "Disabling DTrace-related tests..." + if [ -f /usr/tests/cddl/usr.sbin/dtrace/Kyuafile ]; then + mv /usr/tests/cddl/usr.sbin/dtrace/Kyuafile \ + /usr/tests/cddl/usr.sbin/dtrace/Kyuafile.bak + fi +} + +disable_notyet_tests() { + echo "Disabling not-yet-implemented tests..." + if [ -f /usr/tests/lib/libcasper/services/cap_dns/Kyuafile ]; then + sed -i .bak \ + -e 's/tap_test_program{name="dns_test", }/-- tap_test_program{name="dns_test", }/' \ + /usr/tests/lib/libcasper/services/cap_dns/Kyuafile + fi + + if [ -f /usr/tests/lib/libcasper/services/cap_net/Kyuafile ]; then + sed -i .bak \ + -e 's/atf_test_program{name="net_test", }/-- atf_test_program{name="net_test", }/' \ + /usr/tests/lib/libcasper/services/cap_net/Kyuafile + fi + + if [ -f /usr/local/tests/kyua/utils/signals/Kyuafile ]; then + sed -i .bak \ + -e 's,atf_test_program{name="timer_test"},-- atf_test_program{name="timer_test"},' \ + /usr/local/tests/kyua/utils/signals/Kyuafile + fi + + if [ -f /usr/local/tests/kyua/integration/cmd_about_test ]; then + sed -i '' \ + -e 's,atf_add_test_case topic__authors__installed,#atf_add_test_case topic__authors__installed,' \ + /usr/local/tests/kyua/integration/cmd_about_test + fi +} + +disable_zfs_tests() { + echo "Disabling ZFS-related tests..." + if [ -f /usr/tests/sys/cddl/Kyuafile ]; then + sed -i .bak \ + -e 's,include("zfs/Kyuafile"),-- include("zfs/Kyuafile"),' \ + /usr/tests/sys/cddl/Kyuafile + fi +} + smoke_tests() { echo echo "--------------------------------------------------------------" - echo "BUILD sequence COMPLETED" - echo "IMAGE sequence COMPLETED" echo "BOOT sequence COMPLETED" echo "INITIATING system SHUTDOWN" echo "--------------------------------------------------------------" @@ -62,15 +119,52 @@ full_tests() { - # Currently this is a placeholder. - # This will be used to add the full tests scenario those are run in - # the CI system + if [ -n "${freebsdci_disable}" ]; then + for item in ${freebsdci_disable}; do + case "$item" in + disks) + disable_disks_tests + ;; + dtrace) + disable_dtrace_tests + ;; + notyet) + disable_notyet_tests + ;; + zfs) + disable_zfs_tests + ;; + *) + echo "Unknown disable option: $item" + ;; + esac + done + fi echo echo "--------------------------------------------------------------" - echo "BUILD sequence COMPLETED" - echo "IMAGE sequence COMPLETED" echo "BOOT sequence COMPLETED" echo "TEST sequence STARTED" + if [ "${istar}" -eq 1 ]; then + rm -fr ${metadir} + mkdir -p ${metadir} + tar xvf ${tardev} -C ${metadir} + cd /usr/tests + set +e + kyua test + rc=$? + set -e + if [ ${rc} -ne 0 ] && [ ${rc} -ne 1 ]; then + exit ${rc} + fi + kyua report --verbose --results-filter passed,skipped,xfail,broken,failed --output test-report.txt + kyua report-junit --output=test-report.xml + mv test-report.* /${metadir} + tar cvf ${tardev} -C ${metadir} . + else + echo "ERROR: no device with POSIX tar archive format found." + # Don't shutdown because this is not run in unattended mode + exit 1 + fi echo "TEST sequence COMPLETED" echo "INITIATING system SHUTDOWN" echo "--------------------------------------------------------------"