Index: Makefile =================================================================== --- Makefile +++ Makefile @@ -609,10 +609,13 @@ # XXX: Passing HOST_OBJTOP into the PATH would allow skipping legacy, # bootstrap-tools, and cross-tools. Need to ensure each tool actually # supports all TARGETS though. +# For now we only pass UNIVERSE_TOOLCHAIN_PATH which will be added at the end +# of STRICTTMPPATH to ensure that the target-specific binaries come first. MAKE_PARAMS_${target}+= \ XCC="${HOST_OBJTOP}/tmp/usr/bin/cc" \ XCXX="${HOST_OBJTOP}/tmp/usr/bin/c++" \ - XCPP="${HOST_OBJTOP}/tmp/usr/bin/cpp" + XCPP="${HOST_OBJTOP}/tmp/usr/bin/cpp" \ + UNIVERSE_TOOLCHAIN_PATH=${HOST_OBJTOP}/tmp/usr/bin .endif .if defined(_need_lld_${target}_${target_arch}) && \ ${_need_lld_${target}_${target_arch}} == "yes" Index: Makefile.inc1 =================================================================== --- Makefile.inc1 +++ Makefile.inc1 @@ -574,8 +574,21 @@ WORLDTMP?= ${OBJTOP}/tmp BPATH= ${CCACHE_WRAPPER_PATH_PFX}${WORLDTMP}/legacy/usr/sbin:${WORLDTMP}/legacy/usr/bin:${WORLDTMP}/legacy/bin XPATH= ${WORLDTMP}/usr/sbin:${WORLDTMP}/usr/bin -STRICTTMPPATH= ${BPATH}:${XPATH} + +# When building we want to find the cross tools before the host tools in tmp/legacy +# We also need to add UNIVERSE_TOOLCHAIN_PATH so that we can find the shared +# toolchain files (clang, lld, etc.) during make universe/tinderbox +STRICTTMPPATH= ${XPATH}:${BPATH}:${UNIVERSE_TOOLCHAIN_PATH} +# Avoid using tools from /usr/bin accidentally since this could cause the build +# to break on other systems that don't have that tool. For now we still allow +# using the old behaviour (inheriting $PATH) if BUILD_WITH_STRICT_TMPPATH is set +# to 0 but this will eventually be removed +BUILD_WITH_STRICT_TMPPATH?=1 +.if ${BUILD_WITH_STRICT_TMPPATH} != 0 +TMPPATH= ${STRICTTMPPATH} +.else TMPPATH= ${STRICTTMPPATH}:${PATH} +.endif # # Avoid running mktemp(1) unless actually needed. @@ -583,7 +596,15 @@ # when in the middle of installing over this system. # .if make(distributeworld) || make(installworld) || make(stageworld) -INSTALLTMP!= mktemp -d -u -t install +.if ${BUILD_WITH_STRICT_TMPPATH} != 0 +MKTEMP=${WORLDTMP}/legacy/usr/bin/mktemp +.if !exists(${MKTEMP}) +.error "mktemp binary doesn't exist in expected location: ${MKTEMP}" +.endif +.else +MKTEMP=mktemp +.endif +INSTALLTMP!= ${MKTEMP} -d -u -t install .endif .if make(stagekernel) || make(distributekernel) @@ -641,6 +662,8 @@ CROSSENV+= ${TARGET_CFLAGS} .endif +BOOTSTRAPPING_OSRELDATE?=${OSRELDATE} + # bootstrap-tools stage BMAKEENV= INSTALL="sh ${.CURDIR}/tools/install.sh" \ TOOLS_PREFIX=${TOOLS_PREFIX_UNDEF:U${WORLDTMP}} \ @@ -652,7 +675,7 @@ OBJTOP='${WORLDTMP}/obj-tools' \ OBJROOT='$${OBJTOP}/' \ MAKEOBJDIRPREFIX= \ - BOOTSTRAPPING=${OSRELDATE} \ + BOOTSTRAPPING=${BOOTSTRAPPING_OSRELDATE} \ BWPHASE=${.TARGET:C,^_,,} \ SSP_CFLAGS= \ MK_HTML=no NO_LINT=yes MK_MAN=no \ @@ -674,7 +697,7 @@ ${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \ TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ DESTDIR= \ - BOOTSTRAPPING=${OSRELDATE} \ + BOOTSTRAPPING=${BOOTSTRAPPING_OSRELDATE} \ BWPHASE=${.TARGET:C,^_,,} \ SSP_CFLAGS= \ -DNO_LINT \ @@ -699,7 +722,7 @@ OBJTOP='${WORLDTMP}/obj-kernel-tools' \ OBJROOT='$${OBJTOP}/' \ MAKEOBJDIRPREFIX= \ - BOOTSTRAPPING=${OSRELDATE} \ + BOOTSTRAPPING=${BOOTSTRAPPING_OSRELDATE} \ SSP_CFLAGS= \ MK_HTML=no -DNO_LINT MK_MAN=no \ -DNO_PIC MK_PROFILE=no -DNO_SHARED \ @@ -868,6 +891,9 @@ DESTDIR_MTREEFLAGS+= -W .endif MTREE?= mtree +.if ${BUILD_WITH_STRICT_TMPPATH} != 0 +MTREE= ${WORLDTMP}/legacy/usr/sbin/mtree +.endif WORLDTMP_MTREE= ${MTREE} ${WORLDTMP_MTREEFLAGS} DESTDIR_MTREE= ${MTREE} ${DESTDIR_MTREEFLAGS} @@ -948,10 +974,15 @@ .if !defined(NO_CLEAN) rm -rf ${WORLDTMP} .else +# Note: for delete-old we need to set $PATH to also include the host $PATH +# since otherwise a partial build with missing symlinks in ${WORLDTMP}/legacy/ +# will fail to run due to missing binaries. $WMAKE sets PATH to only ${TMPPATH} +# so we remove that assingnment from $WMAKE and prepend the new $PATH ${_+_}@if [ -e "${WORLDTMP}" ]; then \ echo ">>> Deleting stale files in build tree..."; \ - cd ${.CURDIR}; ${WMAKE} -DBATCH_DELETE_OLD_FILES _NO_INCLUDE_COMPILERMK=t \ - delete-old delete-old-libs >/dev/null; \ + cd ${.CURDIR}; env PATH=${TMPPATH}:${PATH} ${WMAKE:NPATH=*} \ + _NO_INCLUDE_COMPILERMK=t -DBATCH_DELETE_OLD_FILES delete-old \ + delete-old-libs >/dev/null; \ fi rm -rf ${WORLDTMP}/legacy/usr/include .if ${USING_SYSTEM_COMPILER} == "yes" @@ -972,6 +1003,11 @@ # available on the target system (this happens e.g. when building on non-FreeBSD) cd ${.CURDIR}/tools/build; \ ${MAKE} DIRPRFX=tools/build/ DESTDIR=${WORLDTMP}/legacy installdirs +# In order to build without inheriting $PATH we need to add symlinks to the host +# tools in $WORLDTMP for the tools that we don't build during bootstrap-tools + cd ${.CURDIR}/tools/build; \ + ${MAKE} DIRPRFX=tools/build/ DESTDIR=${WORLDTMP}/legacy host-symlinks + _legacy: @echo @echo "--------------------------------------------------------------" @@ -1967,6 +2003,29 @@ # _bt= _bootstrap-tools +# We want to run the build with only ${WORLDTMP} in $PATH to ensure we don't +# accidentally run tools that are incompatible but happen to be in $PATH. +# This is especially important when building on Linux/MacOS where many of the +# programs used during the build accept different flags or generate different +# output. On those platforms we only symlink the tools known to be compatible +# (e.g. basic utilities such as mkdir) into ${WORLDTMP} and build all others +# from the FreeBSD sources during the bootstrap-tools stage. +# We want to build without the user's $PATH starting in the bootstrap-tools +# phase so the tools used in that phase (ln, cp, etc) must have already been +# linked to $WORLDTMP. The tools are listed in the _host_tools_to_symlink +# variable in tools/build/Makefile and are linked during the legacy phase. +# Since they could be Linux or MacOS binaries, too we must only use flags that +# are portable across operating systems. + +# If BOOTSTRAP_ALL_TOOLS is set we will build all the required tools from the +# current source tree. Otherwise we create a symlink to the version found in +# $PATH during the bootstrap-tools stage. +.if defined(BOOTSTRAP_ALL_TOOLS) +# BOOTSTRAPPING will be set on the command line so we can't override it here. +# Instead set BOOTSTRAPPING_OSRELDATE so that the value 0 is set ${BSARGS} +BOOTSTRAPPING_OSRELDATE:= 0 +.endif + .if ${MK_GAMES} != "no" _strfile= usr.bin/fortune/strfile .endif @@ -1979,12 +2038,21 @@ _vtfontcvt= usr.bin/vtfontcvt .endif +# If we are not building the bootstrap because BOOTSTRAPPING is sufficient +# we symlink the host version to $WORLDTMP instead. By doing this we can also +# detect when a bootstrap tool is being used without the required MK_FOO. +# If you add a new bootstrap tool where we could also use the host version, +# please ensure that you also add a .else case where you add the tool to the +# _bootstrap_tools_links variable. .if ${BOOTSTRAPPING} < 1000033 _m4= usr.bin/m4 _lex= usr.bin/lex - -${_bt}-usr.bin/m4: ${_bt}-lib/libopenbsd -${_bt}-usr.bin/lex: ${_bt}-usr.bin/m4 +# Note: lex needs m4 to build but m4 also depends on lex. However, lex can be +# bootstrapped so we build lex first. +${_bt}-usr.bin/m4: ${_bt}-lib/libopenbsd ${_bt}-usr.bin/yacc ${_bt}-${_lex} +_bt_lex_depend=${_bt}-usr.bin/lex ${_bt}-usr.bin/m4 +.else +_bootstrap_tools_links+=m4 lex .endif # r245440 mtree -N support added @@ -1996,17 +2064,23 @@ ${_bt}-lib/libnetbsd: ${_bt}-lib/libmd ${_bt}-usr.sbin/nmtree: ${_bt}-lib/libnetbsd +.else +_bootstrap_tools_links+=mtree .endif # r246097: log addition login.conf.db, passwd, pwd.db, and spwd.db with cat -l .if ${BOOTSTRAPPING} < 1000027 _cat= bin/cat +.else +_bootstrap_tools_links+=cat .endif # r277259 crunchide: Correct 64-bit section header offset # r281674 crunchide: always include both 32- and 64-bit ELF support .if ${BOOTSTRAPPING} < 1100078 _crunchide= usr.sbin/crunch/crunchide +.else +_bootstrap_tools_links+=crunchide .endif # r285986 crunchen: use STRIPBIN rather than STRIP @@ -2016,12 +2090,16 @@ (${MK_AUTO_OBJ} == "yes" && ${BOOTSTRAPPING} < 1100114) || \ (${MK_META_MODE} == "yes" && ${BOOTSTRAPPING} < 1200006) _crunchgen= usr.sbin/crunch/crunchgen +.else +_bootstrap_tools_links+=crunchgen .endif # r296926 -P keymap search path, MFC to stable/10 in r298297 .if ${BOOTSTRAPPING} < 1003501 || \ (${BOOTSTRAPPING} >= 1100000 && ${BOOTSTRAPPING} < 1100103) _kbdcontrol= usr.sbin/kbdcontrol +.else +_bootstrap_tools_links+=kbdcontrol .endif _yacc= lib/liby \ @@ -2053,6 +2131,10 @@ _dtc= gnu/usr.bin/dtc .endif +.if ${MK_LOCALES} != "no" +_localedef= usr.bin/localedef +.endif + .if ${MK_KERBEROS} != "no" _kerberos5_bootstrap_tools= \ kerberos5/tools/make-roken \ @@ -2063,11 +2145,107 @@ usr.bin/compile_et .ORDER: ${_kerberos5_bootstrap_tools:C/^/${_bt}-/g} +.for _tool in ${_kerberos5_bootstrap_tools} +${_bt}-${_tool}: ${_bt}-usr.bin/yacc ${_bt_lex_depend} +.endfor .endif ${_bt}-usr.bin/mandoc: ${_bt}-lib/libopenbsd -bootstrap-tools: .PHONY +# The tools listed in _basic_bootstrap_tools will generally not be +# bootstrapped unless BOOTSTRAP_ALL_TOOL is set. However, when building on a +# Linux or MacOS host the host versions are incompatible so we need to build +# them from the source tree. Usually the link name will be the same as the subdir, +# but some directories such as grep or test install multiple binaries. In that +# case we use the _basic_bootstrap_tools_multilink variable which is a list of +# subdirectory and comma-separated list of files. +_basic_bootstrap_tools_multilink=usr.bin/grep grep,egrep,fgrep +_basic_bootstrap_tools_multilink+=bin/test test,[ +# bootstrap tools needed by buildworld: +_basic_bootstrap_tools=usr.bin/awk usr.bin/cut bin/expr usr.bin/gencat \ + usr.bin/join usr.bin/mktemp bin/rmdir usr.bin/sed usr.bin/sort \ + usr.bin/truncate usr.bin/tsort +# elf2aout is required for by sparc64 build +_basic_bootstrap_tools+=usr.bin/elf2aout +# file2c is needed by usr.sbin/config: +_basic_bootstrap_tools+=usr.bin/file2c +# uuencode/uudecode needed by share/tabset +_basic_bootstrap_tools+=usr.bin/uuencode usr.bin/uudecode +# xargs is needed by mkioctls +_basic_bootstrap_tools+=usr.bin/xargs +# needed by share/termcap: +_basic_bootstrap_tools+=usr.bin/cap_mkdb +# ldd is needed for installcheck (TODO: just always use /usr/bin/ldd instead?) +_basic_bootstrap_tools+=usr.bin/ldd +# needed by installworld: +_basic_bootstrap_tools+=usr.sbin/services_mkdb usr.sbin/pwd_mkdb +# sysctl+chflags are needed for installkernel: +_basic_bootstrap_tools+=sbin/sysctl bin/chflags + +.if ${MK_AMD} != "no" +# unifdef is only used by usr.sbin/amd/libamu/Makefile +_basic_bootstrap_tools+=usr.bin/unifdef +.endif + +.if ${MK_BOOT} != "no" +_basic_bootstrap_tools+=usr.sbin/btxld +_basic_bootstrap_tools+=bin/dd +# xz/unxz is used by EFI +_basic_bootstrap_tools_multilink+=usr.bin/xz xz,unxz +# md5 is used by boot/beri (and possibly others) +_basic_bootstrap_tools+=sbin/md5 +.if defined(BOOTSTRAP_ALL_TOOLS) +${_bt}-sbin/md5: ${_bt}-lib/libmd +.endif +.endif + +.if ${MK_ZONEINFO} != "no" +_basic_bootstrap_tools+=usr.sbin/zic usr.sbin/tzsetup +.endif + +.if defined(BOOTSTRAP_ALL_TOOLS) +_other_bootstrap_tools+=${_basic_bootstrap_tools} +.for _subdir _links in ${_basic_bootstrap_tools_multilink} +_other_bootstrap_tools+=${_subdir} +.endfor +${_bt}-usr.bin/awk: ${_bt_lex_depend} ${_bt}-usr.bin/yacc +${_bt}-bin/expr: ${_bt_lex_depend} ${_bt}-usr.bin/yacc +# If we are bootstrapping file2c, we have to build it before config: +${_bt}-usr.sbin/config: ${_bt}-usr.bin/file2c ${_bt_lex_depend} +# Note: no symlink to make/bmake in the !BOOTSTRAP_ALL_TOOLS case here since +# the links to make/bmake make links will have already have been created in the +# `make legacy` step. Not adding a link to make is important on non-FreeBSD +# since "make" will usually point to GNU make there. +_other_bootstrap_tools+=usr.bin/bmake +.else +# All tools in _basic_bootstrap_tools have the same name as the subdirectory +# so we can use :T to get the name of the symlinks that we need to create. +_bootstrap_tools_links+=${_basic_bootstrap_tools:T} +.for _subdir _links in ${_basic_bootstrap_tools_multilink} +_bootstrap_tools_links+=${_links:S/,/ /g} +.endfor +.endif # defined(BOOTSTRAP_ALL_TOOLS) + +# Link the tools that we need for building but don't need to bootstrap because +# the host version is known to be compatible into ${WORLDTMP}/legacy +# We do this before building any of the bootstrap tools in case they depend on +# the presence of any of the links (e.g. as m4/lex/awk) +${_bt}-links: .PHONY + +.for _tool in ${_bootstrap_tools_links} +${_bt}-link-${_tool}: .PHONY .MAKE + @if [ ! -e "${WORLDTMP}/legacy/bin/${_tool}" ]; then \ + source_path=`which ${_tool}`; \ + if [ ! -e "$${source_path}" ] ; then \ + echo "Cannot find host tool '${_tool}'"; false; \ + fi; \ + ln -sfnv "$${source_path}" "${WORLDTMP}/legacy/bin/${_tool}"; \ + fi +${_bt}-links: ${_bt}-link-${_tool} +.endfor + + +bootstrap-tools: ${_bt}-links .PHONY # Please document (add comment) why something is in 'bootstrap-tools'. # Try to bound the building of the bootstrap-tool to just the @@ -2087,6 +2265,7 @@ ${_yacc} \ ${_m4} \ ${_lex} \ + ${_other_bootstrap_tools} \ usr.bin/xinstall \ ${_gensnmptree} \ usr.sbin/config \ @@ -2094,11 +2273,14 @@ ${_crunchgen} \ ${_nmtree} \ ${_vtfontcvt} \ - usr.bin/localedef -${_bt}-${_tool}: .PHONY .MAKE + ${_localedef} +${_bt}-${_tool}: ${_bt}-links .PHONY .MAKE ${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \ cd ${.CURDIR}/${_tool}; \ if [ -z "${NO_OBJWALK}" ]; then ${MAKE} DIRPRFX=${_tool}/ obj; fi; \ + if [ "${_tool}" = "usr.bin/lex" ]; then \ + ${MAKE} DIRPRFX=${_tool}/ bootstrap; \ + fi; \ ${MAKE} DIRPRFX=${_tool}/ all; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${WORLDTMP}/legacy install @@ -2229,9 +2411,14 @@ _usb_tools= stand/usb/tools .endif +.if defined(BUILD_WITH_STRICT_TMPPATH) || defined(BOOTSTRAP_ALL_TOOLS) +_ar=usr.bin/ar +.endif + cross-tools: .MAKE .PHONY .for _tool in \ ${LOCAL_XTOOL_DIRS} \ + ${_ar} \ ${_clang_libs} \ ${_clang} \ ${_lld} \ Index: tools/build/Makefile =================================================================== --- tools/build/Makefile +++ tools/build/Makefile @@ -59,15 +59,63 @@ # Needed to build config (since it uses libnv) SYSINCS+= ${SRCTOP}/sys/sys/nv.h ${SRCTOP}/sys/sys/cnv.h +# We want to run the build with only ${WORLDTMP} in $PATH to ensure we don't +# accidentally run tools that are incompatible but happen to be in $PATH. +# This is especially important when building on Linux/MacOS where many of the +# programs used during the build accept different flags or generate different +# output. On those platforms we only symlink the tools known to be compatible +# (e.g. basic utilities such as mkdir) into ${WORLDTMP} and build all others +# from the FreeBSD sources during the bootstrap-tools stage. + +# basic commands: It is fine to use the host version for all of these even on +# Linux/MacOS since we only use flags that are supported by all of them. +_host_tools_to_symlink= basename bzip2 bunzip2 chmod chown cmp comm cp date \ + dirname echo env false find fmt gzip gunzip head hostname id ln ls mkdir \ + mv nice patch rm realpath sh sleep tee touch tr true uname uniq wc which + +# We also need a symlink to the absolute path to the make binary used for +# the toplevel makefile. This is not necessarily the same as `which make` +# since e.g. on Linux and MacOS that will be GNU make. +_make_abs!= which "${MAKE}" +_host_abs_tools_to_symlink= ${_make_abs}:make ${_make_abs}:bmake + +host-symlinks: + @echo "Linking host tools into ${DESTDIR}/bin" +.for _tool in ${_host_tools_to_symlink} + @if [ ! -e "${DESTDIR}/bin/${_tool}" ]; then \ + source_path=`which ${_tool}`; \ + if [ ! -e "$${source_path}" ] ; then \ + echo "Cannot find host tool '${_tool}'"; false; \ + fi; \ + ln -sfnv "$${source_path}" "${DESTDIR}/bin/${_tool}"; \ + fi +.endfor +.for _tool in ${_host_abs_tools_to_symlink} + @source_path="${_tool:S/:/ /:[1]}"; \ + target_path="${DESTDIR}/bin/${_tool:S/:/ /:[2]}"; \ + if [ ! -e "$${target_path}" ] ; then \ + if [ ! -e "$${source_path}" ] ; then \ + echo "Host tool '${src_path}' is missing"; false; \ + fi; \ + ln -sfnv "$${source_path}" "$${target_path}"; \ + fi +.endfor # Create all the directories that are needed during the legacy, bootstrap-tools # and cross-tools stages. We do this here using mkdir since mtree may not exist # yet (this happens if we are crossbuilding from Linux/Mac). installdirs: -.for _dir in bin sbin usr/bin usr/sbin usr/lib usr/include lib/geom lib/casper +.for _dir in bin usr/lib usr/include lib/geom lib/casper mkdir -p "${DESTDIR}/${_dir}" .endfor - +# Link usr/bin, sbin, and usr/sbin to bin so that it doesn't matter whether a +# bootstrap tool was added to WORLTMP with a symlink or by building it in the +# bootstrap-tools phase. We could also overrride BINDIR when building bootstrap +# tools but adding the symlinks is easier and means all tools are also +# in the directory that they are installed to normally. + ln -sf bin ${DESTDIR}/sbin + ln -sf ../bin ${DESTDIR}/usr/bin + ln -sf ../bin ${DESTDIR}/usr/sbin .for _group in ${INCSGROUPS:NINCS} mkdir -p "${DESTDIR}/${${_group}DIR}" .endfor Index: tools/tools/nanobsd/embedded/common =================================================================== --- tools/tools/nanobsd/embedded/common +++ tools/tools/nanobsd/embedded/common @@ -671,4 +671,4 @@ # Set the path to the same path we use for buldworld to use latest mkimg NANO_TARGET=$(cd ${NANO_SRC}; ${NANO_MAKE} TARGET_ARCH=${NANO_ARCH} -V _TARGET) NANO_TMPPATH=$(cd ${NANO_SRC}; ${NANO_MAKE} MK_AUTO_OBJ=no TARGET=${NANO_TARGET} TARGET_ARCH=${NANO_ARCH} -f Makefile.inc1 buildenv -V TMPPATH) -export PATH="${NANO_TMPPATH}" +export PATH="${NANO_TMPPATH}:${PATH}"