Page MenuHomeFreeBSD
Files F110646461

D16815.id49878.diff
No OneTemporary
Actions

D16815.id49878.diff
View Options

Index: Makefile
===================================================================
--- Makefile
+++ Makefile
@@ -610,10 +610,13 @@
# XXX: Passing HOST_OBJTOP into the PATH would allow skipping legacy,
# bootstrap-tools, and cross-tools. Need to ensure each tool actually
# supports all TARGETS though.
+# For now we only pass UNIVERSE_TOOLCHAIN_PATH which will be added at the end
+# of STRICTTMPPATH to ensure that the target-specific binaries come first.
MAKE_PARAMS_${target}+= \
XCC="${HOST_OBJTOP}/tmp/usr/bin/cc" \
XCXX="${HOST_OBJTOP}/tmp/usr/bin/c++" \
- XCPP="${HOST_OBJTOP}/tmp/usr/bin/cpp"
+ XCPP="${HOST_OBJTOP}/tmp/usr/bin/cpp" \
+ UNIVERSE_TOOLCHAIN_PATH=${HOST_OBJTOP}/tmp/usr/bin
.endif
.if defined(_need_lld_${target}_${target_arch}) && \
${_need_lld_${target}_${target_arch}} == "yes"
Index: Makefile.inc1
===================================================================
--- Makefile.inc1
+++ Makefile.inc1
@@ -580,8 +580,21 @@
WORLDTMP?= ${OBJTOP}/tmp
BPATH= ${CCACHE_WRAPPER_PATH_PFX}${WORLDTMP}/legacy/usr/sbin:${WORLDTMP}/legacy/usr/bin:${WORLDTMP}/legacy/bin
XPATH= ${WORLDTMP}/usr/sbin:${WORLDTMP}/usr/bin
-STRICTTMPPATH= ${BPATH}:${XPATH}
+
+# When building we want to find the cross tools before the host tools in tmp/legacy
+# We also need to add UNIVERSE_TOOLCHAIN_PATH so that we can find the shared
+# toolchain files (clang, lld, etc.) during make universe/tinderbox
+STRICTTMPPATH= ${XPATH}:${BPATH}:${UNIVERSE_TOOLCHAIN_PATH}
+# Avoid using tools from /usr/bin accidentally since this could cause the build
+# to break on other systems that don't have that tool. For now we still allow
+# using the old behaviour (inheriting $PATH) if BUILD_WITH_STRICT_TMPPATH is set
+# to 0 but this will eventually be removed
+BUILD_WITH_STRICT_TMPPATH?=1
+.if ${BUILD_WITH_STRICT_TMPPATH} != 0
+TMPPATH= ${STRICTTMPPATH}
+.else
TMPPATH= ${STRICTTMPPATH}:${PATH}
+.endif
#
# Avoid running mktemp(1) unless actually needed.
@@ -589,7 +602,15 @@
# when in the middle of installing over this system.
#
.if make(distributeworld) || make(installworld) || make(stageworld)
-INSTALLTMP!= mktemp -d -u -t install
+.if ${BUILD_WITH_STRICT_TMPPATH} != 0
+MKTEMP=${WORLDTMP}/legacy/usr/bin/mktemp
+.if !exists(${MKTEMP})
+.error "mktemp binary doesn't exist in expected location: ${MKTEMP}"
+.endif
+.else
+MKTEMP=mktemp
+.endif
+INSTALLTMP!= ${MKTEMP} -d -u -t install
.endif
.if make(stagekernel) || make(distributekernel)
@@ -647,6 +668,8 @@
CROSSENV+= ${TARGET_CFLAGS}
.endif
+BOOTSTRAPPING_OSRELDATE?=${OSRELDATE}
+
# bootstrap-tools stage
BMAKEENV= INSTALL="sh ${.CURDIR}/tools/install.sh" \
TOOLS_PREFIX=${TOOLS_PREFIX_UNDEF:U${WORLDTMP}} \
@@ -658,7 +681,7 @@
OBJTOP='${WORLDTMP}/obj-tools' \
OBJROOT='$${OBJTOP}/' \
MAKEOBJDIRPREFIX= \
- BOOTSTRAPPING=${OSRELDATE} \
+ BOOTSTRAPPING=${BOOTSTRAPPING_OSRELDATE} \
BWPHASE=${.TARGET:C,^_,,} \
SSP_CFLAGS= \
MK_HTML=no NO_LINT=yes MK_MAN=no \
@@ -680,7 +703,7 @@
${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \
TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
DESTDIR= \
- BOOTSTRAPPING=${OSRELDATE} \
+ BOOTSTRAPPING=${BOOTSTRAPPING_OSRELDATE} \
BWPHASE=${.TARGET:C,^_,,} \
SSP_CFLAGS= \
-DNO_LINT \
@@ -705,7 +728,7 @@
OBJTOP='${WORLDTMP}/obj-kernel-tools' \
OBJROOT='$${OBJTOP}/' \
MAKEOBJDIRPREFIX= \
- BOOTSTRAPPING=${OSRELDATE} \
+ BOOTSTRAPPING=${BOOTSTRAPPING_OSRELDATE} \
SSP_CFLAGS= \
MK_HTML=no -DNO_LINT MK_MAN=no \
-DNO_PIC MK_PROFILE=no -DNO_SHARED \
@@ -874,6 +897,9 @@
DESTDIR_MTREEFLAGS+= -W
.endif
MTREE?= mtree
+.if ${BUILD_WITH_STRICT_TMPPATH} != 0
+MTREE= ${WORLDTMP}/legacy/usr/sbin/mtree
+.endif
WORLDTMP_MTREE= ${MTREE} ${WORLDTMP_MTREEFLAGS}
DESTDIR_MTREE= ${MTREE} ${DESTDIR_MTREEFLAGS}
@@ -948,10 +974,15 @@
.if !defined(NO_CLEAN)
rm -rf ${WORLDTMP}
.else
+# Note: for delete-old we need to set $PATH to also include the host $PATH
+# since otherwise a partial build with missing symlinks in ${WORLDTMP}/legacy/
+# will fail to run due to missing binaries. $WMAKE sets PATH to only ${TMPPATH}
+# so we remove that assingnment from $WMAKE and prepend the new $PATH
${_+_}@if [ -e "${WORLDTMP}" ]; then \
echo ">>> Deleting stale files in build tree..."; \
- cd ${.CURDIR}; ${WMAKE} -DBATCH_DELETE_OLD_FILES _NO_INCLUDE_COMPILERMK=t \
- delete-old delete-old-libs >/dev/null; \
+ cd ${.CURDIR}; env PATH=${TMPPATH}:${PATH} ${WMAKE:NPATH=*} \
+ _NO_INCLUDE_COMPILERMK=t -DBATCH_DELETE_OLD_FILES delete-old \
+ delete-old-libs >/dev/null; \
fi
rm -rf ${WORLDTMP}/legacy/usr/include
.if ${USING_SYSTEM_COMPILER} == "yes"
@@ -972,6 +1003,11 @@
# available on the target system (this happens e.g. when building on non-FreeBSD)
cd ${.CURDIR}/tools/build; \
${MAKE} DIRPRFX=tools/build/ DESTDIR=${WORLDTMP}/legacy installdirs
+# In order to build without inheriting $PATH we need to add symlinks to the host
+# tools in $WORLDTMP for the tools that we don't build during bootstrap-tools
+ cd ${.CURDIR}/tools/build; \
+ ${MAKE} DIRPRFX=tools/build/ DESTDIR=${WORLDTMP}/legacy host-symlinks
+
_legacy:
@echo
@echo "--------------------------------------------------------------"
@@ -1969,6 +2005,29 @@
#
_bt= _bootstrap-tools
+# We want to run the build with only ${WORLDTMP} in $PATH to ensure we don't
+# accidentally run tools that are incompatible but happen to be in $PATH.
+# This is especially important when building on Linux/MacOS where many of the
+# programs used during the build accept different flags or generate different
+# output. On those platforms we only symlink the tools known to be compatible
+# (e.g. basic utilities such as mkdir) into ${WORLDTMP} and build all others
+# from the FreeBSD sources during the bootstrap-tools stage.
+# We want to build without the user's $PATH starting in the bootstrap-tools
+# phase so the tools used in that phase (ln, cp, etc) must have already been
+# linked to $WORLDTMP. The tools are listed in the _host_tools_to_symlink
+# variable in tools/build/Makefile and are linked during the legacy phase.
+# Since they could be Linux or MacOS binaries, too we must only use flags that
+# are portable across operating systems.
+
+# If BOOTSTRAP_ALL_TOOLS is set we will build all the required tools from the
+# current source tree. Otherwise we create a symlink to the version found in
+# $PATH during the bootstrap-tools stage.
+.if defined(BOOTSTRAP_ALL_TOOLS)
+# BOOTSTRAPPING will be set on the command line so we can't override it here.
+# Instead set BOOTSTRAPPING_OSRELDATE so that the value 0 is set ${BSARGS}
+BOOTSTRAPPING_OSRELDATE:= 0
+.endif
+
.if ${MK_GAMES} != "no"
_strfile= usr.bin/fortune/strfile
.endif
@@ -1981,12 +2040,21 @@
_vtfontcvt= usr.bin/vtfontcvt
.endif
+# If we are not building the bootstrap because BOOTSTRAPPING is sufficient
+# we symlink the host version to $WORLDTMP instead. By doing this we can also
+# detect when a bootstrap tool is being used without the required MK_FOO.
+# If you add a new bootstrap tool where we could also use the host version,
+# please ensure that you also add a .else case where you add the tool to the
+# _bootstrap_tools_links variable.
.if ${BOOTSTRAPPING} < 1000033
_m4= usr.bin/m4
_lex= usr.bin/lex
-
-${_bt}-usr.bin/m4: ${_bt}-lib/libopenbsd
-${_bt}-usr.bin/lex: ${_bt}-usr.bin/m4
+# Note: lex needs m4 to build but m4 also depends on lex. However, lex can be
+# bootstrapped so we build lex first.
+${_bt}-usr.bin/m4: ${_bt}-lib/libopenbsd ${_bt}-usr.bin/yacc ${_bt}-${_lex}
+_bt_lex_depend=${_bt}-usr.bin/lex ${_bt}-usr.bin/m4
+.else
+_bootstrap_tools_links+=m4 lex
.endif
# r245440 mtree -N support added
@@ -1998,17 +2066,23 @@
${_bt}-lib/libnetbsd: ${_bt}-lib/libmd
${_bt}-usr.sbin/nmtree: ${_bt}-lib/libnetbsd
+.else
+_bootstrap_tools_links+=mtree
.endif
# r246097: log addition login.conf.db, passwd, pwd.db, and spwd.db with cat -l
.if ${BOOTSTRAPPING} < 1000027
_cat= bin/cat
+.else
+_bootstrap_tools_links+=cat
.endif
# r277259 crunchide: Correct 64-bit section header offset
# r281674 crunchide: always include both 32- and 64-bit ELF support
.if ${BOOTSTRAPPING} < 1100078
_crunchide= usr.sbin/crunch/crunchide
+.else
+_bootstrap_tools_links+=crunchide
.endif
# r285986 crunchen: use STRIPBIN rather than STRIP
@@ -2018,12 +2092,16 @@
(${MK_AUTO_OBJ} == "yes" && ${BOOTSTRAPPING} < 1100114) || \
(${MK_META_MODE} == "yes" && ${BOOTSTRAPPING} < 1200006)
_crunchgen= usr.sbin/crunch/crunchgen
+.else
+_bootstrap_tools_links+=crunchgen
.endif
# r296926 -P keymap search path, MFC to stable/10 in r298297
.if ${BOOTSTRAPPING} < 1003501 || \
(${BOOTSTRAPPING} >= 1100000 && ${BOOTSTRAPPING} < 1100103)
_kbdcontrol= usr.sbin/kbdcontrol
+.else
+_bootstrap_tools_links+=kbdcontrol
.endif
_yacc= lib/liby \
@@ -2059,6 +2137,10 @@
_dtc= gnu/usr.bin/dtc
.endif
+.if ${MK_LOCALES} != "no"
+_localedef= usr.bin/localedef
+.endif
+
.if ${MK_KERBEROS} != "no"
_kerberos5_bootstrap_tools= \
kerberos5/tools/make-roken \
@@ -2069,11 +2151,107 @@
usr.bin/compile_et
.ORDER: ${_kerberos5_bootstrap_tools:C/^/${_bt}-/g}
+.for _tool in ${_kerberos5_bootstrap_tools}
+${_bt}-${_tool}: ${_bt}-usr.bin/yacc ${_bt_lex_depend}
+.endfor
.endif
${_bt}-usr.bin/mandoc: ${_bt}-lib/libopenbsd
-bootstrap-tools: .PHONY
+# The tools listed in _basic_bootstrap_tools will generally not be
+# bootstrapped unless BOOTSTRAP_ALL_TOOL is set. However, when building on a
+# Linux or MacOS host the host versions are incompatible so we need to build
+# them from the source tree. Usually the link name will be the same as the subdir,
+# but some directories such as grep or test install multiple binaries. In that
+# case we use the _basic_bootstrap_tools_multilink variable which is a list of
+# subdirectory and comma-separated list of files.
+_basic_bootstrap_tools_multilink=usr.bin/grep grep,egrep,fgrep
+_basic_bootstrap_tools_multilink+=bin/test test,[
+# bootstrap tools needed by buildworld:
+_basic_bootstrap_tools=usr.bin/awk usr.bin/cut bin/expr usr.bin/gencat \
+ usr.bin/join usr.bin/mktemp bin/rmdir usr.bin/sed usr.bin/sort \
+ usr.bin/truncate usr.bin/tsort
+# elf2aout is required for by sparc64 build
+_basic_bootstrap_tools+=usr.bin/elf2aout
+# file2c is needed by usr.sbin/config:
+_basic_bootstrap_tools+=usr.bin/file2c
+# uuencode/uudecode needed by share/tabset
+_basic_bootstrap_tools+=usr.bin/uuencode usr.bin/uudecode
+# xargs is needed by mkioctls
+_basic_bootstrap_tools+=usr.bin/xargs
+# needed by share/termcap:
+_basic_bootstrap_tools+=usr.bin/cap_mkdb
+# ldd is needed for installcheck (TODO: just always use /usr/bin/ldd instead?)
+_basic_bootstrap_tools+=usr.bin/ldd
+# needed by installworld:
+_basic_bootstrap_tools+=usr.sbin/services_mkdb usr.sbin/pwd_mkdb
+# sysctl+chflags are needed for installkernel:
+_basic_bootstrap_tools+=sbin/sysctl bin/chflags
+
+.if ${MK_AMD} != "no"
+# unifdef is only used by usr.sbin/amd/libamu/Makefile
+_basic_bootstrap_tools+=usr.bin/unifdef
+.endif
+
+.if ${MK_BOOT} != "no"
+_basic_bootstrap_tools+=usr.sbin/btxld
+_basic_bootstrap_tools+=bin/dd
+# xz/unxz is used by EFI
+_basic_bootstrap_tools_multilink+=usr.bin/xz xz,unxz
+# md5 is used by boot/beri (and possibly others)
+_basic_bootstrap_tools+=sbin/md5
+.if defined(BOOTSTRAP_ALL_TOOLS)
+${_bt}-sbin/md5: ${_bt}-lib/libmd
+.endif
+.endif
+
+.if ${MK_ZONEINFO} != "no"
+_basic_bootstrap_tools+=usr.sbin/zic usr.sbin/tzsetup
+.endif
+
+.if defined(BOOTSTRAP_ALL_TOOLS)
+_other_bootstrap_tools+=${_basic_bootstrap_tools}
+.for _subdir _links in ${_basic_bootstrap_tools_multilink}
+_other_bootstrap_tools+=${_subdir}
+.endfor
+${_bt}-usr.bin/awk: ${_bt_lex_depend} ${_bt}-usr.bin/yacc
+${_bt}-bin/expr: ${_bt_lex_depend} ${_bt}-usr.bin/yacc
+# If we are bootstrapping file2c, we have to build it before config:
+${_bt}-usr.sbin/config: ${_bt}-usr.bin/file2c ${_bt_lex_depend}
+# Note: no symlink to make/bmake in the !BOOTSTRAP_ALL_TOOLS case here since
+# the links to make/bmake make links will have already have been created in the
+# `make legacy` step. Not adding a link to make is important on non-FreeBSD
+# since "make" will usually point to GNU make there.
+_other_bootstrap_tools+=usr.bin/bmake
+.else
+# All tools in _basic_bootstrap_tools have the same name as the subdirectory
+# so we can use :T to get the name of the symlinks that we need to create.
+_bootstrap_tools_links+=${_basic_bootstrap_tools:T}
+.for _subdir _links in ${_basic_bootstrap_tools_multilink}
+_bootstrap_tools_links+=${_links:S/,/ /g}
+.endfor
+.endif # defined(BOOTSTRAP_ALL_TOOLS)
+
+# Link the tools that we need for building but don't need to bootstrap because
+# the host version is known to be compatible into ${WORLDTMP}/legacy
+# We do this before building any of the bootstrap tools in case they depend on
+# the presence of any of the links (e.g. as m4/lex/awk)
+${_bt}-links: .PHONY
+
+.for _tool in ${_bootstrap_tools_links}
+${_bt}-link-${_tool}: .PHONY .MAKE
+ @if [ ! -e "${WORLDTMP}/legacy/bin/${_tool}" ]; then \
+ source_path=`which ${_tool}`; \
+ if [ ! -e "$${source_path}" ] ; then \
+ echo "Cannot find host tool '${_tool}'"; false; \
+ fi; \
+ ln -sfnv "$${source_path}" "${WORLDTMP}/legacy/bin/${_tool}"; \
+ fi
+${_bt}-links: ${_bt}-link-${_tool}
+.endfor
+
+
+bootstrap-tools: ${_bt}-links .PHONY
# Please document (add comment) why something is in 'bootstrap-tools'.
# Try to bound the building of the bootstrap-tool to just the
@@ -2093,6 +2271,7 @@
${_yacc} \
${_m4} \
${_lex} \
+ ${_other_bootstrap_tools} \
usr.bin/xinstall \
${_gensnmptree} \
usr.sbin/config \
@@ -2101,10 +2280,13 @@
${_nmtree} \
${_vtfontcvt} \
${_localedef}
-${_bt}-${_tool}: .PHONY .MAKE
+${_bt}-${_tool}: ${_bt}-links .PHONY .MAKE
${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \
cd ${.CURDIR}/${_tool}; \
if [ -z "${NO_OBJWALK}" ]; then ${MAKE} DIRPRFX=${_tool}/ obj; fi; \
+ if [ "${_tool}" = "usr.bin/lex" ]; then \
+ ${MAKE} DIRPRFX=${_tool}/ bootstrap; \
+ fi; \
${MAKE} DIRPRFX=${_tool}/ all; \
${MAKE} DIRPRFX=${_tool}/ DESTDIR=${WORLDTMP}/legacy install
@@ -2235,9 +2417,14 @@
_usb_tools= stand/usb/tools
.endif
+.if ${BUILD_WITH_STRICT_TMPPATH} != 0 || defined(BOOTSTRAP_ALL_TOOLS)
+_ar=usr.bin/ar
+.endif
+
cross-tools: .MAKE .PHONY
.for _tool in \
${LOCAL_XTOOL_DIRS} \
+ ${_ar} \
${_clang_libs} \
${_clang} \
${_lld} \
Index: tools/build/Makefile
===================================================================
--- tools/build/Makefile
+++ tools/build/Makefile
@@ -59,15 +59,64 @@
# Needed to build config (since it uses libnv)
SYSINCS+= ${SRCTOP}/sys/sys/nv.h ${SRCTOP}/sys/sys/cnv.h
+# We want to run the build with only ${WORLDTMP} in $PATH to ensure we don't
+# accidentally run tools that are incompatible but happen to be in $PATH.
+# This is especially important when building on Linux/MacOS where many of the
+# programs used during the build accept different flags or generate different
+# output. On those platforms we only symlink the tools known to be compatible
+# (e.g. basic utilities such as mkdir) into ${WORLDTMP} and build all others
+# from the FreeBSD sources during the bootstrap-tools stage.
+
+# basic commands: It is fine to use the host version for all of these even on
+# Linux/MacOS since we only use flags that are supported by all of them.
+_host_tools_to_symlink= basename bzip2 bunzip2 chmod chown cmp comm cp date \
+ dirname echo env false find fmt gzip gunzip head hostname id ln ls \
+ mkdir mv nice patch rm realpath sh sleep stat tee touch tr true uname \
+ uniq wc which
+
+# We also need a symlink to the absolute path to the make binary used for
+# the toplevel makefile. This is not necessarily the same as `which make`
+# since e.g. on Linux and MacOS that will be GNU make.
+_make_abs!= which "${MAKE}"
+_host_abs_tools_to_symlink= ${_make_abs}:make ${_make_abs}:bmake
+
+host-symlinks:
+ @echo "Linking host tools into ${DESTDIR}/bin"
+.for _tool in ${_host_tools_to_symlink}
+ @if [ ! -e "${DESTDIR}/bin/${_tool}" ]; then \
+ source_path=`which ${_tool}`; \
+ if [ ! -e "$${source_path}" ] ; then \
+ echo "Cannot find host tool '${_tool}'"; false; \
+ fi; \
+ ln -sfnv "$${source_path}" "${DESTDIR}/bin/${_tool}"; \
+ fi
+.endfor
+.for _tool in ${_host_abs_tools_to_symlink}
+ @source_path="${_tool:S/:/ /:[1]}"; \
+ target_path="${DESTDIR}/bin/${_tool:S/:/ /:[2]}"; \
+ if [ ! -e "$${target_path}" ] ; then \
+ if [ ! -e "$${source_path}" ] ; then \
+ echo "Host tool '${src_path}' is missing"; false; \
+ fi; \
+ ln -sfnv "$${source_path}" "$${target_path}"; \
+ fi
+.endfor
# Create all the directories that are needed during the legacy, bootstrap-tools
# and cross-tools stages. We do this here using mkdir since mtree may not exist
# yet (this happens if we are crossbuilding from Linux/Mac).
installdirs:
-.for _dir in bin sbin usr/bin usr/sbin usr/lib usr/include lib/geom lib/casper
+.for _dir in bin usr/lib usr/include lib/geom lib/casper
mkdir -p "${DESTDIR}/${_dir}"
.endfor
-
+# Link usr/bin, sbin, and usr/sbin to bin so that it doesn't matter whether a
+# bootstrap tool was added to WORLTMP with a symlink or by building it in the
+# bootstrap-tools phase. We could also overrride BINDIR when building bootstrap
+# tools but adding the symlinks is easier and means all tools are also
+# in the directory that they are installed to normally.
+ ln -sf bin ${DESTDIR}/sbin
+ ln -sf ../bin ${DESTDIR}/usr/bin
+ ln -sf ../bin ${DESTDIR}/usr/sbin
.for _group in ${INCSGROUPS:NINCS}
mkdir -p "${DESTDIR}/${${_group}DIR}"
.endfor
Index: tools/tools/nanobsd/embedded/common
===================================================================
--- tools/tools/nanobsd/embedded/common
+++ tools/tools/nanobsd/embedded/common
@@ -671,4 +671,4 @@
# Set the path to the same path we use for buldworld to use latest mkimg
NANO_TARGET=$(cd ${NANO_SRC}; ${NANO_MAKE} TARGET_ARCH=${NANO_ARCH} -V _TARGET)
NANO_TMPPATH=$(cd ${NANO_SRC}; ${NANO_MAKE} MK_AUTO_OBJ=no TARGET=${NANO_TARGET} TARGET_ARCH=${NANO_ARCH} -f Makefile.inc1 buildenv -V TMPPATH)
-export PATH="${NANO_TMPPATH}"
+export PATH="${NANO_TMPPATH}:${PATH}"

File Metadata

Mime Type
text/plain
Expires
Sat, Feb 22, 9:57 AM (1 h, 32 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16769178
Default Alt Text
D16815.id49878.diff (17 KB)

Event Timeline