diff --git a/share/mk/bsd.opts.mk b/share/mk/bsd.opts.mk --- a/share/mk/bsd.opts.mk +++ b/share/mk/bsd.opts.mk @@ -79,6 +79,7 @@ CTF \ INSTALL_AS_USER \ RETPOLINE \ + RUN_TESTS \ STALE_STAGED \ UBSAN \ UNDEFINED_VERSION \ diff --git a/share/mk/bsd.test.mk b/share/mk/bsd.test.mk --- a/share/mk/bsd.test.mk +++ b/share/mk/bsd.test.mk @@ -91,6 +91,11 @@ .include .endif +.if ${MK_DIRDEPS_BUILD} == "yes" && ${MK_RUN_TESTS} == "yes" +# Run tests as part of the build +.include +.endif + .if !target(realcheck) realcheck: .PHONY @echo "$@ not defined; skipping" diff --git a/share/mk/host.test.mk b/share/mk/host.test.mk new file mode 100644 --- /dev/null +++ b/share/mk/host.test.mk @@ -0,0 +1,65 @@ +# +# For DIRDEPS_BUILD we may want to run tests for "host" +# during the build - and fail the build if any test fails. +# +.if !target(____) +.error ${.PARSEFILE} cannot be included directly. +.endif + +.if ${.MAKE.LEVEL} > 0 && !empty(_TESTS) && ${MACHINE:Nhost*} == "" + +# allow for customization +.-include + +KYUA?= kyua + +# we need to make sure kyua-report can find the results +KYUA_RESULTS?= ${.OBJDIR}/kyua.results +KYUA_ARGS?= --results-file=${KYUA_RESULTS} +KYUA_ENV?= HOME=${KYUA_HOME} TMPDIR=${.OBJDIR} +KYUA_FLAGS?= --config none --loglevel=${KYUA_LOGLEVEL:Uinfo} +KYUA_HOME?= ${OBJTOP} + +.if make(debug*) +KYUA_LOGLEVEL?= debug +.endif + +# some tests have files they need +.if ${${PACKAGE}FILES:U:NKyuafile} != "" +run-tests run-tests.log: link-test-files +link-test-files: ${${PACKAGE}FILES:NKyuafile} + @for f in ${.ALLSRC:N*Kyuafile:M*/*}; do \ + ln -sf $$f .; \ + done +.endif + +# we do not want to stage any of this +RUN_TESTS_LOG= run-tests.log +MK_STAGING= no +META_XTRAS+= ${RUN_TESTS_LOG} + +all run-tests: ${RUN_TESTS_LOG} + +# This is the main event. +# Run kyua-test followed by kyua-report. +# If we have any test failues we want to run kyua-report --verbose +# Also on fail, we rename run-tests.log to run-tests.err so we save the +# output but the target will be out-of-date. +# We prepend ${.OBJDIR}:${.OBJDIR:H}: to PATH seen by kyua +# so tests for things like cat, cp, cmp etc can find the one we just built +# rather than the one from the host. +${RUN_TESTS_LOG}: ${_TESTS} Kyuafile + @( export PATH=${.OBJDIR}:${.OBJDIR:H}:${PATH}; \ + rm -f ${KYUA_RESULTS}; \ + ${KYUA_ENV} ${KYUA} ${KYUA_FLAGS} test ${KYUA_ARGS} -k ${.OBJDIR}/Kyuafile --build-root=${.OBJDIR} && \ + ${KYUA_ENV} ${KYUA} ${KYUA_FLAGS} report ${KYUA_ARGS} ) > ${.TARGET} || \ + { mv ${.TARGET} ${.TARGET:R}.err; \ + ${KYUA_ENV} ${KYUA} ${KYUA_FLAGS} report ${KYUA_ARGS} --verbose --results-filter broken,failed; echo See ${.TARGET:R:tA}.err; \ + exit 1; } + +# make kyua-debug KYUA_DEBUG_ARGS=app:test +kyua-debug: + @(export PATH=${.OBJDIR}:${.OBJDIR:H}:${PATH}; \ + ${KYUA_ENV} ${KYUA} ${KYUA_FLAGS} debug ${KYUA_DEBUG_ARGS}) || true + +.endif diff --git a/share/mk/local.dirdeps.mk b/share/mk/local.dirdeps.mk --- a/share/mk/local.dirdeps.mk +++ b/share/mk/local.dirdeps.mk @@ -250,3 +250,59 @@ KERNEL_NAME:= ${KERNEL_NAME.${DEP_MACHINE}} .-include + +.if ${MK_RUN_TESTS} == "yes" +# some the tests below will not run correctly on the host +# as they require support files not present in .CURDIR +BROKEN_HOST_TESTS += \ + ${BROKEN_HOST_DEP_TESTS} \ + bin/cp/tests \ + lib/atf/libatf-c++/tests \ + lib/atf/libatf-c/tests \ + lib/libarchive/tests \ + lib/libnv/tests \ + libexec/rtld-elf/tests \ + sbin/devd/tests \ + usr.bin/comm/tests \ + usr.bin/du/tests \ + usr.bin/procstat/tests \ + usr.bin/sed/tests \ + usr.bin/tar/tests \ + usr.bin/tr/tests \ + usr.bin/xargs/tests \ + usr.bin/yacc/tests \ + usr.sbin/makefs/tests \ + usr.sbin/rpcbind/tests \ + +# these all have broken dependencies which we choose not to fix for host +BROKEN_HOST_DEP_TESTS += \ + lib/msun/tests \ + usr.bin/cut/tests \ + usr.bin/diff/tests \ + usr.bin/grep/tests \ + usr.bin/gzip/tests \ + usr.bin/printf/tests \ + +.endif + +TESTS_DIR_LIST += tests +# most of the tree only has Makefile.depend +# by default we only want to run host tests for Makefile.depend.host +TESTS_DEPENDFILE_PREFERENCE ?= Makefile.depend.host +EXCLUDE_TESTS_FILTER += . \ + lib/libcasper/services/cap_* \ + +.for t in ${TESTS_DIR_LIST:O:u} +.if !target(${DEP_RELDIR}/$t.test) +.if ${BROKEN_HOST_TESTS:U:M${DEP_RELDIR}/$t} == "" && \ + ${RUN_TESTS_FILTER:U*:@m@${DEP_RELDIR:M$m}@} != "" && \ + ${DEP_RELDIR:${EXCLUDE_TESTS_FILTER:Uno:${M_ListToSkip}}} != "" && \ + ${TESTS_DEPENDFILE_PREFERENCE:@m@${exists(${SRCTOP}/${DEP_RELDIR}/$t/$m):?$m:}@} != "" +# build but not via DIRDEPS to avoid circular dependencies +_build_xtra_dirs += ${SRCTOP}/${DEP_RELDIR}/$t.host +# mark it done +${DEP_RELDIR}/$t.test: +.endif +.endif +.endfor +.endif