diff --git a/tests/ci/Makefile b/tests/ci/Makefile index 30ca34a810be..a750b1265169 100644 --- a/tests/ci/Makefile +++ b/tests/ci/Makefile @@ -1,266 +1,307 @@ # SPDX-License-Identifier: BSD-2-Clause # # Copyright (c) 2024 The FreeBSD Foundation # # This software was developed by Cybermancer Infosec # under sponsorship from the FreeBSD Foundation. # # Makefile for CI testing. # # User-driven targets: # ci: Run CI tests # ci-smoke: Run smoke tests which is simply booting the image # ci-full: Run full tests # # Variables affecting the build process: # TARGET/TARGET_ARCH: architecture of built release (default: same as build host) # KERNELCONF: kernel configuration to use # USE_QEMU: Use QEMU for testing rather than bhyve # WORLDDIR?= ${.CURDIR}/../.. RELEASEDIR= ${WORLDDIR}/release MAKECONF?= /dev/null SRCCONF?= /dev/null _MEMORY!=sysctl -n hw.physmem 2>/dev/null PARALLEL_JOBS!=sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null TOTAL_MEMORY!=expr ${_MEMORY} / 1073741824 KERNCONF?= GENERIC LOCALBASE?= /usr/local EXTRA_MAKE_FLAGS?= .if !defined(TARGET) || empty(TARGET) TARGET= ${MACHINE} .endif .if !defined(TARGET_ARCH) || empty(TARGET_ARCH) . if ${TARGET} == ${MACHINE} TARGET_ARCH= ${MACHINE_ARCH} . else TARGET_ARCH= ${TARGET} . endif .endif IMAKE= ${MAKE} TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} .if !defined(CIENV) || empty(CIENV) CIENV= local LOG_TARGET= > ${.CURDIR}/_.${TARGET_ARCH}.${.TARGET} 2>&1 || (echo "${.TARGET} failed, check _.${TARGET_ARCH}.${.TARGET} for details" ; false) .endif .if defined(CROSS_TOOLCHAIN) || !empty(CROSS_TOOLCHAIN) CROSS_TOOLCHAIN_PARAM= "CROSS_TOOLCHAIN=${CROSS_TOOLCHAIN}" .endif # Define OSRELEASE by using newvers.sh .if !defined(OSRELEASE) || empty(OSRELEASE) . for _V in TYPE BRANCH REVISION . if !defined(${_V}) || empty(${_V}) ${_V}!= eval $$(awk '/^${_V}=/{print}' ${.CURDIR}/../../sys/conf/newvers.sh); echo $$${_V} . endif . endfor . for _V in ${TARGET_ARCH} . if !empty(TARGET:M${_V}) OSRELEASE= ${TYPE}-${REVISION}-${BRANCH}-${TARGET} VOLUME_LABEL= ${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-]/_/g}_${TARGET} . else OSRELEASE= ${TYPE}-${REVISION}-${BRANCH}-${TARGET}-${TARGET_ARCH} VOLUME_LABEL= ${REVISION:C/[.-]/_/g}_${BRANCH:C/[.-]/_/g}_${TARGET_ARCH} . endif . endfor .endif .if exists(${.CURDIR}/tools/ci.conf) && !defined(CICONF) CICONF?= ${.CURDIR}/tools/ci.conf .endif SWAPSIZE?= 1g VMFS?= ufs FORMAT= raw CIIMAGE= ci-${OSRELEASE}-${GITREV}-${KERNCONF}.${FORMAT} CIDISK?= ${.OBJDIR}/${CIIMAGE} VMSIZE?= 6g CITYPE?= full KYUA_TEST_FILTERS?= META_TAR!=mktemp /tmp/meta.XXXXXX META_DIR!=mktemp -d /tmp/meta.XXXXXX META_DIROUT!=mktemp -d /tmp/meta.XXXXXX +ENV_FILE= ${META_DIR}/ci-${OSRELEASE}-${GITREV}-${KERNCONF}.env DISC_CAM!=truncate -s 128m /tmp/disk-cam EXTRA_DISK_NUM?=5 DISK_NUMBERS!=jot - 1 ${EXTRA_DISK_NUM} BHYVE_EXTRA_DISK_PARAM?= BHYVE_EXTRA_DISK_PARAM+=-s 4:0,ahci-hd,/tmp/disk-cam .for i in ${DISK_NUMBERS} NEW_DISK!=truncate -s 128m /tmp/disk${i} BHYVE_EXTRA_DISK_PARAM+=-s $$((${i} + 4)):0,virtio-blk,/tmp/disk${i} CLEANFILES+=/tmp/disk${i} .endfor TEST_VM_NAME= ci-${OSRELEASE}-${GITREV}-${KERNCONF} .if ${TOTAL_MEMORY} >= 16 VM_MEM=8 .elif ${TOTAL_MEMORY} >=4 VM_MEM=${TOTAL_MEMORY} .else echo "Please increase the memory to at least 4GB" exit 0 .endif VM_MEM_SIZE?=${VM_MEM}g TIMEOUT_MS?=5400000 TIMEOUT=$$((${TIMEOUT_MS} / 1000)) TIMEOUT_EXPECT=$$((${TIMEOUT} - 60)) TIMEOUT_VM=$$((${TIMEOUT_EXPECT} - 120)) .if exists(${.CURDIR}/Makefile.${TARGET_ARCH}) . include "${.CURDIR}/Makefile.${TARGET_ARCH}" .endif .if ${TARGET_ARCH} != ${MACHINE_ARCH} . if ( ${TARGET_ARCH} != "i386" ) || ( ${MACHINE_ARCH} != "amd64" ) QEMUSTATIC=/usr/local/bin/qemu-${QEMU_ARCH}-static QEMUTGT=portinstall-qemu . endif .endif QEMUTGT?= QEMU_DEVICES?= QEMU_EXTRA_PARAM?= QEMU_MACHINE?=virt QEMUBIN=/usr/local/bin/qemu-system-${QEMU_ARCH} .if ${PARALLEL_JOBS} >= ${QEMU_MAX_CPU_COUNT} QEMU_CPU_COUNT=${QEMU_MAX_CPU_COUNT} .else QEMU_CPU_COUNT=${PARALLEL_JOBS} .endif .if ${VM_MEM} >= ${QEMU_MAX_MEM_SIZE} VM_MEM_SIZE=${QEMU_MAX_MEM_SIZE}g .else VM_MEM_SIZE=${VM_MEM}g .endif VMGUEST!=sysctl -n kern.vm_guest 2>/dev/null || true .if ${VMGUEST} != "none" USE_QEMU?=1 .endif KLDFILEMONISLOADED!=kldload -q -n filemon 2>/dev/null && echo "1" || echo "0" .if ${KLDFILEMONISLOADED} == "1" METAMODE?=-DWITH_META_MODE .endif CLEANFILES+= ${.OBJDIR}/${CIIMAGE} ${.OBJDIR}/ci.img ${META_TAR} IMAGEDIR= ${.OBJDIR}/ci-buildimage CLEANDIRS+= ${IMAGEDIR} portinstall: portinstall-pkg portinstall-qemu portinstall-expect portinstall-${TARGET_ARCH:tl} .PHONY portinstall-pkg: .PHONY .if !exists(/usr/local/sbin/pkg-static) env ASSUME_ALWAYS_YES=yes pkg bootstrap .endif portinstall-qemu: portinstall-pkg .PHONY .if !exists(/usr/local/bin/qemu-${QEMU_ARCH}-static) env ASSUME_ALWAYS_YES=yes pkg install emulators/qemu-user-static .endif .if !exists(${QEMUBIN}) env ASSUME_ALWAYS_YES=yes pkg install emulators/qemu@nox11 .endif portinstall-expect: portinstall-pkg .PHONY .if !exists(/usr/local/bin/expect) env ASSUME_ALWAYS_YES=yes pkg install lang/expect .endif beforeclean: .PHONY chflags -R noschg ${IMAGEDIR} .include clean: beforeclean .PHONY cleandir: beforeclean .PHONY ci-buildworld: .PHONY @echo "Building world for ${TARGET_ARCH}" ${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \ ${CROSS_TOOLCHAIN_PARAM} __MAKE_CONF=${MAKECONF} SRCCONF=${SRCCONF} \ ${EXTRA_MAKE_FLAGS} buildworld ${LOG_TARGET} ci-buildkernel: ci-buildworld-${TARGET_ARCH:tl} .PHONY @echo "Building kernel for ${TARGET_ARCH}" ${IMAKE} -j${PARALLEL_JOBS} -C ${WORLDDIR} ${METAMODE} \ ${CROSS_TOOLCHAIN_PARAM} __MAKE_CONF=${MAKECONF} SRCCONF=${SRCCONF} \ ${EXTRA_MAKE_FLAGS} KERNCONF=${KERNCONF} \ buildkernel ${LOG_TARGET} ci-buildimage: ${QEMUTGT} ci-buildkernel-${TARGET_ARCH:tl} .PHONY @echo "Building ci image for ${TARGET_ARCH}" mkdir -p ${.OBJDIR}/${.TARGET} env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} SWAPSIZE=${SWAPSIZE} \ QEMUSTATIC=${QEMUSTATIC} CITYPE=${CITYPE} \ ${RELEASEDIR}/scripts/mk-vmimage.sh \ -C ${RELEASEDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} -F ${VMFS} \ -i ${.OBJDIR}/ci.img -s ${VMSIZE} -f ${FORMAT} \ -S ${WORLDDIR} -o ${.OBJDIR}/${CIIMAGE} -c ${CICONF} ${LOG_TARGET} touch ${.TARGET} ci-set-smoke-var: .PHONY CITYPE=smoke ci-set-full-var: .PHONY CITYPE=full -ci-create-meta: .PHONY +ci-get-env: .PHONY + @echo "TARGET=${TARGET}" > ${ENV_FILE} + @echo "TARGET_ARCH=${TARGET_ARCH}" >> ${ENV_FILE} + @echo "KERNCONF=${KERNCONF}" >> ${ENV_FILE} + @echo "OSRELEASE=${OSRELEASE}" >> ${ENV_FILE} + @echo "CIIMAGE=${CIIMAGE}" >> ${ENV_FILE} + @echo "CIDISK=${CIDISK}" >> ${ENV_FILE} + @echo "VMSIZE=${VMSIZE}" >> ${ENV_FILE} + @echo "VM_MEM_SIZE=${VM_MEM_SIZE}" >> ${ENV_FILE} + @echo "TIMEOUT=${TIMEOUT}s" >> ${ENV_FILE} + @echo "CITYPE=${CITYPE}" >> ${ENV_FILE} + @echo "KYUA_TEST_FILTERS='${KYUA_TEST_FILTERS}'" >> ${ENV_FILE} + @echo "META_MODE='${METAMODE}'" >> ${ENV_FILE} + @echo "USE_QEMU='${USE_QEMU}'" >> ${ENV_FILE} + @echo "GITREV='${GITREV}'" >> ${ENV_FILE} + @echo "GITBRANCH='${GITBRANCH}'" >> ${ENV_FILE} + @echo "GITCOUNT='${GITCOUNT}'" >> ${ENV_FILE} + @echo "BUILDDATE='${BUILDDATE}'" >> ${ENV_FILE} + @echo "CC='${CC}'" >> ${ENV_FILE} + @echo "CFLAGS='${CFLAGS}'" >> ${ENV_FILE} + @echo "CPP='${CPP}'" >> ${ENV_FILE} + @echo "CXX='${CXX}'" >> ${ENV_FILE} + @echo "CXXFLAGS='${CXXFLAGS}'" >> ${ENV_FILE} + @echo "COMPILER_FEATURES='${COMPILER_FEATURES}'" >> ${ENV_FILE} + @echo "COMPILER_FREEBSD_VERSION='${COMPILER_FREEBSD_VERSION}'" >> ${ENV_FILE} + @echo "COMPILER_TYPE='${COMPILER_TYPE}'" >> ${ENV_FILE} + @echo "COMPILER_VERSION='${COMPILER_VERSION}'" >> ${ENV_FILE} + @echo "HOST_CC='${HOST_CC}'" >> ${ENV_FILE} + @echo "HOST_MACHINE='${HOST_MACHINE}'" >> ${ENV_FILE} + @echo "HOST_OS='${HOST_OS}'" >> ${ENV_FILE} + @echo "HOST_OSMAJOR='${HOST_OSMAJOR}'" >> ${ENV_FILE} + @echo "HOST_OSTYPE='${HOST_OSTYPE}'" >> ${ENV_FILE} + @echo "LD='${LD}'" >> ${ENV_FILE} + @echo "LDFLAGS='${LDFLAGS}'" >> ${ENV_FILE} + @echo "MACHINE='${MACHINE}'" >> ${ENV_FILE} + @echo "MACHINE_ABI='${MACHINE_ABI}'" >> ${ENV_FILE} + @echo "MACHINE_ARCH='${MACHINE_ARCH}'" >> ${ENV_FILE} + @echo "MACHINE_CPU='${MACHINE_CPU}'" >> ${ENV_FILE} + @echo "MACHINE_CPUARCH='${MACHINE_CPUARCH}'" >> ${ENV_FILE} + +ci-create-meta: ci-get-env .PHONY truncate -s 512M ${META_TAR} tar rvf ${META_TAR} -C ${META_DIR} . ci-extract-meta: .PHONY tar xfv ${META_TAR} -C ${META_DIROUT} rm -rf ${META_TAR} ${META_DIR} @echo "Extracted kyua reports to ${META_DIROUT}" ci-runtest: ci-buildimage-${TARGET_ARCH:tl} portinstall .PHONY .if ${MACHINE} == "amd64" && ( ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" ) && ( !defined(USE_QEMU) || empty(USE_QEMU) ) /usr/sbin/bhyvectl --vm=${TEST_VM_NAME} --destroy || true expect -c "set timeout ${TIMEOUT_EXPECT}; \ spawn /usr/bin/timeout -k 5s 30s /usr/sbin/bhyveload \ -c stdio -m ${VM_MEM_SIZE} -d ${CIDISK} ${TEST_VM_NAME}; \ expect { eof }; \ exit [lindex [wait] 3]" expect -c "set timeout ${TIMEOUT_EXPECT}; \ spawn /usr/bin/timeout -k 60 ${TIMEOUT_VM} /usr/sbin/bhyve \ -c ${PARALLEL_JOBS} -m ${VM_MEM_SIZE} -A -H -P \ -s 0:0,hostbridge \ -s 1:0,lpc \ -s 2:0,virtio-blk,${CIDISK} \ -s 3:0,virtio-blk,${META_TAR} \ ${BHYVE_EXTRA_DISK_PARAM} \ -l com1,stdio \ ${TEST_VM_NAME}; \ expect { eof }" /usr/sbin/bhyvectl --vm=${TEST_VM_NAME} --destroy .else timeout -k 60 ${TIMEOUT_VM} ${QEMUBIN} \ -machine ${QEMU_MACHINE} \ -smp ${QEMU_CPU_COUNT} \ -m ${VM_MEM_SIZE} \ -nographic \ -no-reboot \ ${QEMU_EXTRA_PARAM} \ -device virtio-blk,drive=hd0 \ -device virtio-blk,drive=hd1 \ -blockdev driver=raw,node-name=hd0,file.driver=file,file.filename=${CIDISK} \ -blockdev driver=raw,node-name=hd1,file.driver=file,file.filename=${META_TAR} \ ${QEMU_DEVICES} .endif ci-checktarget: .PHONY .if ${TARGET_ARCH} != "aarch64" && \ ${TARGET_ARCH} != "amd64" && \ ${TARGET_ARCH} != "armv7" && \ ${TARGET_ARCH} != "powerpc64" && \ ${TARGET_ARCH} != "powerpc64le" && \ ${TARGET_ARCH} != "riscv64" @false .ERROR: @echo "Error: ${TARGET_ARCH} is not supported on ${TYPE} ${REVISION} ${BRANCH}" .endif ci-smoke: ci-set-smoke-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .PHONY ci-full: ci-set-full-var ci-create-meta ci-checktarget .WAIT ci-runtest-${TARGET_ARCH:tl} .WAIT ci-extract-meta .PHONY ci: ci-${CITYPE:tl} .PHONY .include "${RELEASEDIR}/Makefile.inc1" diff --git a/tests/ci/tools/freebsdci b/tests/ci/tools/freebsdci index 42c565a45055..f71ccc1cad4f 100755 --- a/tests/ci/tools/freebsdci +++ b/tests/ci/tools/freebsdci @@ -1,113 +1,126 @@ #!/bin/sh # # SPDX-License-Identifier: BSD-2-Clause # # Copyright (c) 2024 The FreeBSD Foundation # # This software was developed by Cybermancer Infosec # under sponsorship from the FreeBSD Foundation. # # PROVIDE: freebsdci # REQUIRE: LOGIN FILESYSTEMS # KEYWORD: firstboot # This script is used to run the firstboot CI tests on the first boot of a # FreeBSD image. It is automatically disabled after the first boot. # # The script will run the firstboot CI tests and then shut down the system. # The tests are run in the foreground so that the system can shut down # immediately after the tests are finished. # # Default test types are full and smoke. To run only the smoke tests, set # freebsdci_type="smoke" in /etc/rc.conf.local or /etc/rc.conf. # To run only the full tests, set freebsdci_type="full" in # /etc/rc.conf.local or /etc/rc.conf. . /etc/rc.subr name="freebsdci" desc="Run FreeBSD CI" rcvar=freebsdci_enable start_cmd="firstboot_ci_run" stop_cmd=":" os_arch=$(uname -p) parallelism=$(nproc) tardev=/dev/vtbd1 metadir=/meta istar=$(file -s ${tardev} | grep "POSIX tar archive" | wc -l) load_rc_config $name : ${freebsdci_enable:="NO"} : ${freebsdci_type:="full"} : ${freebsdci_test_filters:=""} PATH="${PATH}:/usr/local/sbin:/usr/local/bin" auto_shutdown() { # NOTE: Currently RISC-V kernels lack the ability to # make qemu exit on shutdown. Reboot instead; # it makes qemu exit too. case "$os_arch" in riscv64) shutdown -r now ;; *) shutdown -p now ;; esac } +set_environment() +{ + if [ "${istar}" -eq 1 ]; then + rm -fr ${metadir} + mkdir -p ${metadir} + tar xvf ${tardev} -C ${metadir} + ci_env_file=$(set -- "${metadir}"/ci-*.env; \ + [ "$#" -eq 1 ] && printf '%s\n' "$1") + if [ -z "${ci_env_file}" ]; then + echo "ci: No CI environment file found in ${metadir}" + else + env >> "${ci_env_file}" + fi + else + echo "ERROR: no device with POSIX tar archive format found." + # Don't shutdown because this is not run in unattended mode + exit 1 + fi + +} + smoke_tests() { echo echo "--------------------------------------------------------------" echo "BOOT sequence COMPLETED" echo "INITIATING system SHUTDOWN" echo "--------------------------------------------------------------" } full_tests() { echo echo "--------------------------------------------------------------" echo "BOOT sequence COMPLETED" echo "TEST sequence STARTED" - if [ "${istar}" -eq 1 ]; then - rm -fr ${metadir} - mkdir -p ${metadir} - tar xvf ${tardev} -C ${metadir} - cd /usr/tests - set +e - kyua \ - -v parallelism=${parallelism} \ - test ${freebsdci_test_filters} - rc=$? - set -e - if [ ${rc} -ne 0 ] && [ ${rc} -ne 1 ]; then - exit ${rc} - fi - kyua report --verbose --results-filter passed,skipped,xfail,broken,failed --output test-report.txt - kyua report-junit --output=test-report.xml - mv test-report.* /${metadir} - tar cvf ${tardev} -C ${metadir} . - else - echo "ERROR: no device with POSIX tar archive format found." - # Don't shutdown because this is not run in unattended mode - exit 1 + cd /usr/tests + set +e + kyua \ + -v parallelism=${parallelism} \ + test ${freebsdci_test_filters} + rc=$? + set -e + if [ ${rc} -ne 0 ] && [ ${rc} -ne 1 ]; then + exit ${rc} fi + kyua report --verbose --results-filter passed,skipped,xfail,broken,failed --output test-report.txt + kyua report-junit --output=test-report.xml + mv test-report.* /${metadir} echo "TEST sequence COMPLETED" echo "INITIATING system SHUTDOWN" echo "--------------------------------------------------------------" } firstboot_ci_run() { + set_environment if [ "$freebsdci_type" = "smoke" ]; then smoke_tests elif [ "$freebsdci_type" = "full" ]; then full_tests fi + tar cvf ${tardev} -C ${metadir} . auto_shutdown } run_rc_command "$1"