Index: Makefile =================================================================== --- Makefile +++ Makefile @@ -21,6 +21,7 @@ # kernel-toolchains - Build kernel-toolchain for all universe targets. # doxygen - Build API documentation of the kernel, needs doxygen. # update - Convenient way to update your source tree(s). +# checkworld - Run test suite on installed world. # check-old - List obsolete directories/files/libraries. # check-old-dirs - List obsolete directories. # check-old-files - List obsolete files. @@ -112,8 +113,8 @@ .else TGTS= all all-man buildenv buildenvvars buildkernel buildworld \ - check-old check-old-dirs check-old-files check-old-libs \ - checkdpadd clean cleandepend cleandir cleanworld \ + check check-old check-old-dirs check-old-files check-old-libs \ + checkdpadd checkworld clean cleandepend cleandir cleanworld \ delete-old delete-old-dirs delete-old-files delete-old-libs \ depend distribute distributekernel distributekernel.debug \ distributeworld distrib-dirs distribution doxygen \ @@ -121,7 +122,7 @@ installkernel.debug packagekernel packageworld \ reinstallkernel reinstallkernel.debug \ installworld kernel-toolchain libraries lint maninstall \ - obj objlink regress rerelease showconfig tags toolchain update \ + obj objlink rerelease showconfig tags toolchain update \ _worldtmp _legacy _bootstrap-tools _cleanobj _obj \ _build-tools _cross-tools _includes _libraries _depend \ build32 builddtb distribute32 install32 xdev xdev-build xdev-install \ Index: Makefile.inc1 =================================================================== --- Makefile.inc1 +++ Makefile.inc1 @@ -37,6 +37,7 @@ # The intended user-driven targets are: # buildworld - rebuild *everything*, including glue to help do upgrades # installworld- install everything built by "buildworld" +# checkworld - run test suite on installed world # doxygen - build API documentation of the kernel # update - convenient way to update your source tree (eg: svn/svnup) # @@ -48,10 +49,10 @@ .error "Both TARGET and TARGET_ARCH must be defined." .endif +LOCALBASE?= /usr/local # Cross toolchain changes must be in effect before bsd.compiler.mk # so that gets the right CC, and pass CROSS_TOOLCHAIN to submakes. .if defined(CROSS_TOOLCHAIN) -LOCALBASE?= /usr/local .include "${LOCALBASE}/share/toolchains/${CROSS_TOOLCHAIN}.mk" CROSSENV+=CROSS_TOOLCHAIN="${CROSS_TOOLCHAIN}" .endif @@ -1284,12 +1285,26 @@ .endif # +# +# checkworld +# +# Run test suite on installed world. +# +checkworld: .PHONY + @if [ ! -x ${LOCALBASE}/bin/kyua ]; then \ + echo "You need kyua (devel/kyua) to run the test suite." | /usr/bin/fmt; \ + exit 1; \ + fi + ${_+_}${LOCALBASE}/bin/kyua test -k ${TESTSBASE}/Kyuafile + +# +# # doxygen # # Build the API documentation with doxygen # doxygen: .PHONY - @if [ ! -x `/usr/bin/which doxygen` ]; then \ + @if [ ! -x ${LOCALBASE}/bin/doxygen ]; then \ echo "You need doxygen (devel/doxygen) to generate the API documentation of the kernel." | /usr/bin/fmt; \ exit 1; \ fi Index: share/man/man7/build.7 =================================================================== --- share/man/man7/build.7 +++ share/man/man7/build.7 @@ -107,6 +107,16 @@ The following list provides the names and actions for the targets supported by the build system: .Bl -tag -width ".Cm cleandepend" +.It Cm check +Run tests for a given subdirectory. +The default directory used is +.Pa ${.OBJDIR} , +but the check directory can be changed with +.Pa ${CHECKDIR} . +.It Cm checkworld +Run the +.Fx +Test Suite on installed world. .It Cm clean Remove any files created during the build process. .It Cm cleandepend @@ -653,6 +663,7 @@ .Xr mergemaster 8 , .Xr portsnap 8 , .Xr reboot 8 , -.Xr shutdown 8 +.Xr shutdown 8 , +.Xr tests 7 .Sh AUTHORS .An Mike W. Meyer Aq Mt mwm@mired.org Index: share/mk/bsd.README =================================================================== --- share/mk/bsd.README +++ share/mk/bsd.README @@ -448,6 +448,17 @@ all: build the test programs. + check: + runs the test programs from CHECKDIR with kyua test. + + The beforecheck and aftercheck targets will be invoked, if + defined, to execute commands before and after the realcheck + target has been executed, respectively. + + The devel/kyua package must be installed before invoking this + target. + + See CHECKDIR for more details. clean: remove the test programs and any object files. cleandir: @@ -466,12 +477,6 @@ run lint on the source files. tags: create a tags file for the source files. - test: - runs the test programs from the object directory; if the - Makefile does not itself define the target test, the - targets beforetest and aftertest may also be used to - cause actions immediately before and after the test - target is executed. It sets/uses the following variables, among many others: @@ -485,6 +490,11 @@ ${TESTSBASE}/${RELDIR:H} , e.g. /usr/tests/bin/ls when included from bin/ls/tests . +CHECKDIR The value of CHECKDIR defaults to TESTSDIR when make check is + run from the top-level directory. Conversely, the value of + CHECKDIR defaults to .OBJDIR when make check is not run from + the top-level directory. + KYUAFILE If 'auto' (the default), generate a Kyuafile out of the test programs defined in the Makefile. If 'yes', then a manually-crafted Kyuafile must be supplied with the @@ -492,6 +502,10 @@ subdirectories providing helper programs or data files only). +LOCALBASE The --prefix for the kyua package. + + The value of LOCALBASE defaults to /usr/local . + ATF_TESTS_C The names of the ATF C test programs to build. ATF_TESTS_CXX The names of the ATF C++ test programs to build. Index: share/mk/bsd.subdir.mk =================================================================== --- share/mk/bsd.subdir.mk +++ share/mk/bsd.subdir.mk @@ -37,10 +37,10 @@ ____: ALL_SUBDIR_TARGETS= all all-man buildconfig buildfiles buildincludes \ - checkdpadd clean cleandepend cleandir cleanilinks \ + check checkdpadd clean cleandepend cleandir cleanilinks \ cleanobj depend distribute files includes installconfig \ installfiles installincludes realinstall lint maninstall \ - manlint obj objlink regress tags \ + manlint obj objlink tags \ ${SUBDIR_TARGETS} # Described above. Index: share/mk/bsd.sys.mk =================================================================== --- share/mk/bsd.sys.mk +++ share/mk/bsd.sys.mk @@ -172,11 +172,11 @@ # or expect to ever be up-to-date. PHONY_NOTMAIN = afterdepend afterinstall all beforedepend beforeinstall \ beforelinking build build-tools buildconfig buildfiles \ - buildincludes checkdpadd clean cleandepend cleandir cleanobj \ - configure depend dependall distclean distribute exe \ + buildincludes check checkdpadd clean cleandepend cleandir \ + cleanobj configure depend dependall distclean distribute exe \ files html includes install installconfig installfiles \ installincludes lint obj objlink objs objwarn realall \ - realdepend realinstall regress subdir-all subdir-depend \ + realdepend realinstall subdir-all subdir-depend \ subdir-install tags whereobj # we don't want ${PROG} to be PHONY Index: share/mk/bsd.test.mk =================================================================== --- share/mk/bsd.test.mk +++ share/mk/bsd.test.mk @@ -10,6 +10,9 @@ ____: +# Path to the prefix of the ATF and Kyua CLI. +LOCALBASE?= /usr/local + # Tests install directory TESTSDIR?= ${TESTSBASE}/${RELDIR:H} @@ -57,11 +60,15 @@ .include .include +# kyua automatically descends directories; only run make check on the +# top-level directory +.if !make(check) .for ts in ${TESTS_SUBDIRS} .if empty(SUBDIR:M${ts}) SUBDIR+= ${ts} .endif .endfor +.endif # it is rare for test cases to have man pages .if !defined(MAN) @@ -76,19 +83,14 @@ .include .endif -.if !target(realtest) -realtest: .PHONY +.if !target(realcheck) +realcheck: .PHONY @echo "$@ not defined; skipping" .endif -test: .PHONY -.ORDER: beforetest realtest -test: beforetest realtest - -.if target(aftertest) -.ORDER: realtest aftertest -test: aftertest -.endif +beforecheck realcheck aftercheck check: .PHONY +.ORDER: beforecheck realcheck aftercheck +check: beforecheck realcheck aftercheck .ifdef PROG # we came here via bsd.progs.mk below Index: share/mk/suite.test.mk =================================================================== --- share/mk/suite.test.mk +++ share/mk/suite.test.mk @@ -45,31 +45,20 @@ # unqualified TEST_METADATA variable. #TEST_METADATA.+= key="value" -# Path to the prefix of the installed Kyua CLI, if any. -# -# If kyua is installed from ports, we automatically define a realtest target -# below to run the tests using this tool. The tools are searched for in the -# hierarchy specified by this variable. -KYUA_PREFIX?= /usr/local - -.if ${KYUAFILE:tl} == "yes" +.if ${KYUAFILE:tl} != "no" FILES+= Kyuafile FILESDIR_Kyuafile= ${TESTSDIR} +CLEANFILES+= Kyuafile Kyuafile.tmp +.endif -CLEANFILES+= Kyuafile.auto Kyuafile.auto.tmp -.elif ${KYUAFILE:tl} == "auto" -FILES+= Kyuafile.auto -FILESDIR_Kyuafile.auto= ${TESTSDIR} -FILESNAME_Kyuafile.auto= Kyuafile - -CLEANFILES+= Kyuafile.auto Kyuafile.auto.tmp +.if ${KYUAFILE:tl} == "auto" .for _T in ${_TESTS} _TEST_METADATA.${_T}= ${TEST_METADATA} ${TEST_METADATA.${_T}} .endfor -.NOPATH: Kyuafile.auto -Kyuafile.auto: Makefile +.NOPATH: Kyuafile +Kyuafile: Makefile @{ \ echo '-- Automatically generated by bsd.test.mk.'; \ echo; \ @@ -77,20 +66,30 @@ echo; \ echo 'test_suite("${TESTSUITE}")'; \ echo; \ - } >Kyuafile.auto.tmp + } > ${.TARGET}.tmp .for _T in ${_TESTS} @echo '${TEST_INTERFACE.${_T}}_test_program{name="${_T}"${_TEST_METADATA.${_T}:C/$/,/:tW:C/^/, /W:C/,$//W}}' \ - >>Kyuafile.auto.tmp + >>${.TARGET}.tmp .endfor .for _T in ${TESTS_SUBDIRS:N.WAIT} - @echo "include(\"${_T}/Kyuafile\")" >>Kyuafile.auto.tmp + @echo "include(\"${_T}/${.TARGET}\")" >>${.TARGET}.tmp .endfor - @mv Kyuafile.auto.tmp Kyuafile.auto + @mv ${.TARGET}.tmp ${.TARGET} .endif -KYUA?= ${KYUA_PREFIX}/bin/kyua -.if exists(${KYUA}) -# Definition of the "make test" target and supporting variables. +CHECKDIR?= ${DESTDIR}${TESTSDIR} + +KYUA= ${LOCALBASE}/bin/kyua +${KYUA}: + @echo + @echo "kyua binary not installed at expected location (${.TARGET})" + @echo + @echo "Please install via pkg install, or specify the path to the kyua" + @echo "package via the \$${LOCALBASE} variable, e.g. " + @echo "LOCALBASE=\"${LOCALBASE}\"" + @false + +# Definition of the "make check" target and supporting variables. # # This target, by necessity, can only work for native builds (i.e. a FreeBSD # host building a release for the same system). The target runs Kyua, which is @@ -99,35 +98,7 @@ # Due to the dependencies of the binaries built by the source tree and how they # are used by tests, it is highly possible for a execution of "make test" to # report bogus results unless the new binaries are put in place. -realtest: .PHONY - @echo "*** WARNING: make test is experimental" - @echo "***" - @echo "*** Using this test does not preclude you from running the tests" - @echo "*** installed in ${TESTSBASE}. This test run may raise false" - @echo "*** positives and/or false negatives." - @echo - @${KYUA} test -k ${DESTDIR}${TESTSDIR}/Kyuafile; \ - result=0; \ - echo; \ - echo "*** Once again, note that "make test" is unsupported."; \ - test $${result} -eq 0 -.endif -beforetest: .PHONY -.if defined(TESTSDIR) -.if ${TESTSDIR} == ${TESTSBASE} -# Forbid running from ${TESTSBASE}. It can cause false positives/negatives and -# it does not cover all the tests (e.g. it misses testing software in external). - @echo "*** Sorry, you cannot use make test from src/tests. Install the" - @echo "*** tests into their final location and run them from ${TESTSBASE}" - @false -.else - @echo "*** Using this test does not preclude you from running the tests" - @echo "*** installed in ${TESTSBASE}. This test run may raise false" - @echo "*** positives and/or false negatives." -.endif -.else - @echo "*** No TESTSDIR defined; nothing to do." - @false -.endif - @echo +realcheck: .PHONY +realcheck: ${KYUA} + @${KYUA} test -k ${CHECKDIR}/Kyuafile Index: share/mk/tap.test.mk =================================================================== --- share/mk/tap.test.mk +++ share/mk/tap.test.mk @@ -26,7 +26,7 @@ TAP_TESTS_SH?= # Perl interpreter to use for test programs written in this language. -TAP_PERL_INTERPRETER?= /usr/local/bin/perl +TAP_PERL_INTERPRETER?= ${LOCALBASE}/bin/perl .if !empty(TAP_TESTS_C) PROGS+= ${TAP_TESTS_C}