Index: user/cperciva/freebsd-update-build/scripts/build.subr =================================================================== --- user/cperciva/freebsd-update-build/scripts/build.subr (revision 349168) +++ user/cperciva/freebsd-update-build/scripts/build.subr (revision 349169) @@ -1,1141 +1,1149 @@ #- # Copyright 2006 Colin Percival # All rights reserved # # Redistribution and use in source and binary forms, with or without # modification, are permitted providing that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # $FreeBSD$ # FreeBSD Update build subroutines. # # All the actual work gets done by functions defined here # and optionally overridden by release-specific and/or # release-and-platform-specific functions. # # Sourcing this is the first thing which scripts do. ### Initialization # Read global configuration and set global variables. Run from # readconfig(), so most scripts don't need to call this directly. readglobalconfig () { # Set paths export SCRIPTDIR="`dirname $0`" export BASEDIR="`realpath ${SCRIPTDIR}/..`" export BINDIR=${BASEDIR}/bin export SRCDIR=${BASEDIR}/src export KEYDIR=${BASEDIR}/keys export PRIVKEYDIR=${KEYDIR}/mnt JFLAG=${JFLAG:-"-j$(($(sysctl -n kern.smp.cpus)+1))"} # Read global configuration . ${SCRIPTDIR}/build.conf } # Read configuration and subroutine override files for FreeBSD/$1 $2. readconfig () { # Make sure we have a target platform specified export TARGET=$1 if [ -z "${TARGET}" ]; then echo "Target platform must be set" exit 1 fi # Make sure we have a release specified export REL=$2 if [ -z "${REL}" ]; then echo "Release name must be set" exit 1 fi # Get global configuration and variables. readglobalconfig # Set paths export PATCHDIR=${BASEDIR}/patches/${REL} export WORKDIR=${BASEDIR}/work/${REL}/${TARGET} export PUBDIR=${BASEDIR}/pub/${REL}/${TARGET} export STAGEDIR=${WORKDIR}/stage export TMPDIR=${WORKDIR}/tmp # Default TARGET_ARCH is TARGET. On pc98, this needs to be # overridden in the platform-specific configuration. export TARGET_ARCH=${TARGET} # RELP is REL, unless it is changed to be REL-pX instead. export RELP=${REL} # Make sure the target/release pair make sense. if ! [ -f ${SCRIPTDIR}/${REL}/${TARGET}/build.conf ]; then echo "No configuration available for FreeBSD/${TARGET} ${REL}" exit 1 fi # Read release/platform specific configuration . ${SCRIPTDIR}/${REL}/${TARGET}/build.conf # Read release and release-and-platform specific routines, if any. if [ -f ${SCRIPTDIR}/${REL}/build.subr ]; then . ${SCRIPTDIR}/${REL}/build.subr fi if [ -f ${SCRIPTDIR}/${REL}/${TARGET}/build.subr ]; then . ${SCRIPTDIR}/${REL}/${TARGET}/build.subr fi } ### Utility subroutines # Logging function log () { echo "`date` $1 for FreeBSD/${TARGET} ${RELP}" } # Function for nuking a directory nuke () { umount ${WORKDIR}/$1 2>/dev/null || true rm -rf ${WORKDIR}/$1 2>/dev/null || true if [ -d ${WORKDIR}/$1 ]; then chflags -R noschg ${WORKDIR}/$1 rm -rf ${WORKDIR}/$1 fi } # Nuke a world and delete its associated index. removeworld () { nuke $1 rm -f ${WORKDIR}/$1-index } ### Key handling # Create a small memory disk and mount it on ${PRIVKEYDIR} mountkeydir () { if [ -e ${PRIVKEYDIR}/.snap ]; then echo "Private key directory is already mounted" exit 1 fi KEYMD=`mdconfig -a -t malloc -s 1M -n` newfs /dev/md${KEYMD} > /dev/null mount /dev/md${KEYMD} ${PRIVKEYDIR} chmod 700 ${PRIVKEYDIR} } # Unmount the mini-filesystem containing the unencrypted private key umountkeydir () { if ! [ -e ${PRIVKEYDIR}/.snap ]; then echo "Private key directory is not mounted" exit 1 fi KEYDEV=`df ${PRIVKEYDIR} | tail -1 | awk '{ print $1 }'` case ${KEYDEV} in /dev/md*) ;; *) echo "${PRIVKEYDIR} isn't mounted on a memory disk!" exit 1 ;; esac # unmount umount ${KEYDEV} # Overwrite the memory disk with random garbage before deleting it. # This may prevent the key from lingering in memory; in particular, # overwriting with zeros often won't, due to overly intelligent # memory disk code. dd if=/dev/urandom of=${KEYDEV} count=2048 2>/dev/null mdconfig -d -u `echo ${KEYDEV} | cut -c 8-` } # Generate RSA key makekey () { mountkeydir openssl genrsa -F4 4096 > ${PRIVKEYDIR}/priv.ssl echo echo "Public key fingerprint:" openssl rsa -in ${PRIVKEYDIR}/priv.ssl -pubout 2>/dev/null | sha256 echo } # Make sure an unencrypted key is available checkkey () { if ! [ -f ${PRIVKEYDIR}/priv.ssl ]; then cat <<-EOF You must run mountkey.sh to make the signing key available for use first. EOF exit 1 fi } # Encrypt the key and write it to disk encryptkey () { KEYOWNER=${SUDO_USER:-${USER}} echo "Encrypting signing key for ${KEYOWNER}" openssl enc -aes-256-cbc -md md5 \ -in ${PRIVKEYDIR}/priv.ssl \ -out ${KEYDIR}/priv.ssl-${KEYOWNER} } # Decrypt the key decryptkey () { KEYOWNER=${SUDO_USER:-${USER}} echo "Decrypting signing key for ${KEYOWNER}" openssl enc -d -aes-256-cbc -md md5 \ -in ${KEYDIR}/priv.ssl-${KEYOWNER} \ -out ${PRIVKEYDIR}/priv.ssl } ### Setup subroutines # Make directories needed once for all releases/platforms makeglobaldirs () { mkdir -p ${BINDIR} ${BASEDIR}/logs ${KEYDIR} ${PRIVKEYDIR} chmod 700 ${KEYDIR} ${PRIVKEYDIR} } # Make binaries makebin () { ( cd ${SRCDIR} && make all install clean ) } # Create directories needed for a specific release/platform. makedirs () { mkdir -p ${WORKDIR} ${TMPDIR} \ ${WORKDIR}/oldfiles ${WORKDIR}/oldmeta \ ${STAGEDIR} ${STAGEDIR}/f ${STAGEDIR}/m \ ${STAGEDIR}/bp ${STAGEDIR}/tp ${STAGEDIR}/t \ ${PUBDIR} ${PUBDIR}/f ${PUBDIR}/m \ ${PUBDIR}/bp ${PUBDIR}/tp ${PUBDIR}/t } # Create some empty databases. makedbs () { touch ${WORKDIR}/INDEX-ALL ${WORKDIR}/INDEX-NEW \ ${WORKDIR}/INDEX-OLD ${WORKDIR}/metadb \ ${WORKDIR}/stampvalues ${WORKDIR}/stamplocations \ ${WORKDIR}/stampedfiles ${WORKDIR}/stampedfiles.tgz } # Set up PUBDIR. makepub () { # Create an "all files earlier than this have been uploaded" marker # and wait a second to make sure other files will have timestamps # after (rather than equal to) this. touch ${PUBDIR}/uploaded sleep 1 # Create some files which aren't used by FreeBSD Update but are # useful nonetheless. The Expires directives help caching HTTP # proxies to save bandwidth without being out of date, while # the 'Options -Indexes' directive works around a problem with # Apache and large directory listings (Apache uses the regular # memory allocator to allocate space for each directory entry, # and has no way to tell the OS that it no longer needs the # many MB which it allocated). cat <<-EOF > ${PUBDIR}/.htaccess ExpiresActive on ExpiresDefault "access plus 60 seconds" Options -Indexes EOF echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/f/.htaccess echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/m/.htaccess echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/bp/.htaccess echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/tp/.htaccess echo 'ExpiresDefault "access plus 1 week"' > ${PUBDIR}/t/.htaccess } ### Operation order checking routines # Make sure that the findstamps binary has been built. This more or # less automatically means that the unstamp binary has been built, so # we won't bother checking for that one. checkbins () { if ! [ -f ${BINDIR}/findstamps ]; then cat <<-EOF You must run make.sh to build some binaries before running init.sh. EOF exit 1 fi } # Make sure that init has been run before running diff checkinit () { if ! [ -s ${WORKDIR}/INDEX-ALL ]; then cat <<-EOF You must run init.sh to fetch and extract the release before running diff.sh. EOF exit 1 fi } ### Main functions # Download and verify a release ISO image. fetchiso () { log "Starting fetch" # Figure out where the dvd1 ISO image is RELNUM=${REL%-*} ISO=${FTP}/FreeBSD-${REL}-${TARGET}-dvd1.iso # Fetch the ISO image. We consider the ISO image to be # the One True Release and don't look at the files used # for FTP installs. The FreeBSD 4.7-RELEASE ISO and FTP # files were not identical, but this should never happen # again. if ! [ -f "${WORKDIR}/iso.img" ]; then fetch -o ${WORKDIR}/iso.img -rR ${ISO} 2>&1 fi log "Verifying dvd1 hash" # Check that the downloaded ISO has the correct hash. if ! [ "`sha512 -q ${WORKDIR}/iso.img`" = "${RELH}" ]; then echo "FreeBSD ${REL}-${TARGET}-dvd1.iso has incorrect hash." rm ${WORKDIR}/iso.img return 1 fi } # Extract the released trees and, if appropriate, construct a world (base # plus source code) in which to perform builds. extractiso () { # Create and mount a md(4) attached to the ISO image. ISOMD=`mdconfig -a -t vnode -f ${WORKDIR}/iso.img -n` mkdir -p ${WORKDIR}/iso mount -t cd9660 -o ro,nosuid /dev/md${ISOMD} ${WORKDIR}/iso # Extract the various components into different directories log "Extracting components" for C in ${WORLDPARTS}; do mkdir -p ${WORKDIR}/release/R/trees/world/${C} cat ${WORKDIR}/iso/usr/freebsd-dist/${C}.txz | tar -xpzf - -C ${WORKDIR}/release/R/trees/world/${C} done for C in ${KERNELPARTS}; do mkdir -p ${WORKDIR}/release/R/trees/kernel/${C} cat ${WORKDIR}/iso/usr/freebsd-dist/${C}.txz | tar -xpzf - -C ${WORKDIR}/release/R/trees/kernel/${C} done for C in ${SOURCEPARTS}; do mkdir -p ${WORKDIR}/release/R/trees/src/${C} cat ${WORKDIR}/iso/usr/freebsd-dist/${C}.txz | tar -xpzf - -C ${WORKDIR}/release/R/trees/src/${C} done log "Constructing world+src image" # Create directory for world mkdir ${WORKDIR}/world/ # Extract world and source distributions for C in ${WORLDPARTS}; do cat ${WORKDIR}/iso/usr/freebsd-dist/${C}.txz | tar -xpzf - -C ${WORKDIR}/world/ done for C in ${SOURCEPARTS}; do cat ${WORKDIR}/iso/usr/freebsd-dist/${C}.txz | tar -xpzf - -C ${WORKDIR}/world/ done # build a single tarball of them. tar -czf ${WORKDIR}/../world.tgz -C ${WORKDIR}/world . # clean up nuke world # Unmount and detach the ISO image md(4). umount ${WORKDIR}/iso rmdir ${WORKDIR}/iso mdconfig -d -u ${ISOMD} } # Extract to ${WORKDIR}/$1 a world in which to perform builds. extractworld () { # Clean old world, if necessary (e.g., if a build was interrupted) if [ -d ${WORKDIR}/$1 ]; then log "Removing old world+src" removeworld $1 fi log "Extracting world+src" mkdir -p ${WORKDIR}/$1 mount -t tmpfs tmpfs ${WORKDIR}/$1 tar -xpzf ${WORKDIR}/../world.tgz -C ${WORKDIR}/$1 } # Apply a list of patches stored in $2 to a world in ${WORKDIR}/$1 applypatches () { cd ${WORKDIR}/$1/usr/src while read PATCH; do patch -p0 < ${PATCHDIR}/${PATCH} # Remove ".orig" files created by patch(1) and empty files for file in $(perl -ne \ 'print("$1\n") if /^\+\+\+ (?:b\/|\.\/)?(\S+)(?:\s+[(0-9].*)?$/' < \ ${PATCHDIR}/${PATCH}); do echo ${file}.orig if [ ! -s ${file} ]; then echo ${file} fi done | xargs rm -f done < $2 > ${WORKDIR}/$1-patch.log 2>&1 cd - } # In a world at ${WORKDIR}/$1, edit src/sys/conf/newvers.sh to # 1. Add $2 to the end of the BRANCH= line; and # 2. Revert the RELEASE= line back to what it is in CVS. # Note that the RELEASE= line is modified by src/release/Makefile # at release time. patchnewvers () { sed -i "" -E "s,^(BRANCH=.*)\",\1$2\"," \ ${WORKDIR}/$1/usr/src/sys/conf/newvers.sh sed -i "" -E 's,^(RELEASE=).*,\1"${REVISION}-${BRANCH}",' \ ${WORKDIR}/$1/usr/src/sys/conf/newvers.sh } # Apply "release" patches to a world in ${WORKDIR}/$1. These "release" # patches are special patches which we want to pretend were in the # release all along -- they don't result in newly built files being # distributed, since we're pretending that these patches weren't added # after the release. rpatchworld () { ls ${PATCHDIR} | grep -E '^0-' | cat > ${WORKDIR}/patchlist.tmp applypatches $1 ${WORKDIR}/patchlist.tmp rm ${WORKDIR}/patchlist.tmp # Edit src/sys/conf/newvers.sh patchnewvers $1 "" } # Apply all available patches to a world in ${WORKDIR}/$1 patchworld () { # Apply patches, in order of increasing patch level. The # order matters in case we have multiple security advisories # affecting the same code. log "Patching world+src" ls ${PATCHDIR} | sort -n > ${WORKDIR}/patchlist.tmp applypatches $1 ${WORKDIR}/patchlist.tmp rm ${WORKDIR}/patchlist.tmp # Edit src/sys/conf/newvers.sh patchnewvers $1 "-p${PNUM}" } # Perform a build in ${WORKDIR}/$1 with BRANCH_OVERRIDE set to $2 buildworld () { # We need a devfs inside the jail. Note that we are using a # jail here in order to keep the environment as "clean" as # possible, not for security reasons; we assume that the # original source code plus patches we add personally will # not do anything evil. mount -t devfs devfs ${WORKDIR}/$1/dev # We need to be able to set file flags sysctl security.jail.chflags_allowed=1 >/dev/null # Build stuff. jail ${WORKDIR}/$1 ${BUILDHOSTNAME} 127.1.2.3 \ /usr/bin/env -i PATH=${PATH} RELP=${RELP} \ JFLAG=${JFLAG} \ BRANCH_OVERRIDE=$2 \ XZ_THREADS=16 \ TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ /bin/sh -e <<-"EOF" 2>&1 >${WORKDIR}/$1-build.log # Function for logging what we're doing log () { echo "`date` $1 for FreeBSD/${TARGET} ${RELP}" 1>&2 } # Build the world log "Building world" cd /usr/src && make ${JFLAG} buildworld 2>&1 # Build and kernel log "Building kernel" cd /usr/src && make ${JFLAG} buildkernel 2>&1 # Build and install release images log "Building release" cd /usr/src/release && make release NODVD=y 2>&1 && make install NODVD=y DESTDIR=/R 2>&1 EOF # Put all the components into the right places. log "Moving components into staging area" jail ${WORKDIR}/$1 ${BUILDHOSTNAME} 127.1.2.3 \ /usr/bin/env -i PATH=${PATH} \ WORLDPARTS="${WORLDPARTS}" \ KERNELPARTS="${KERNELPARTS}" \ SOURCEPARTS="${SOURCEPARTS}" \ /bin/sh -e <<-"EOF" 2>&1 >>${WORKDIR}/$1-build.log # Create area for uncompressed components mkdir -p /R/trees # Move world components into place for C in ${WORLDPARTS}; do mkdir -p /R/trees/world/${C} cat /R/ftp/${C}.txz | tar -xpzf - -C /R/trees/world/${C} done # Move kernel components into place for C in ${KERNELPARTS}; do mkdir -p /R/trees/kernel/${C} cat /R/ftp/${C}.txz | tar -xpzf - -C /R/trees/kernel/${C} done # Extract src components into place for C in ${SOURCEPARTS}; do mkdir -p /R/trees/src/${C} cat /R/ftp/${C}.txz | tar -xpzf - -C /R/trees/src/${C} done EOF # Get rid of the devfs we no longer need. umount ${WORKDIR}/$1/dev } # Perform a build in ${WORKDIR}/$1, but with the date set 400 days # into the future. Turn off NTP before we change the date and # turn it back on afterwards. futurebuildworld () { # Turn off ntpd if necessary if /etc/rc.d/ntpd status | grep -q 'is running'; then ntpd_was_running=1 log "Turning off NTP" /etc/rc.d/ntpd stop >/dev/null else ntpd_was_running=0 fi date -n `date -j -v+400d "+%y%m%d%H%M.%S"` >/dev/null buildworld $1 FUTUREBUILD date -n `date -j -v-400d "+%y%m%d%H%M.%S"` >/dev/null # Turn ntpd back on, if appropriate if [ ${ntpd_was_running} = 1 ]; then log "Turning NTP back on" /etc/rc.d/ntpd start >/dev/null fi } # Index ${WORKDIR}/$1/R/trees/ and write to ${WORKDIR}/$1-index. The # index format is described below and the lines are in lexicographical # order. indexfiles () { log "Indexing $1" # Output lines of the form # # comp|scomp|/path/to/file|type|inum|user|group|perm|flags|value # # where the fields have meaning as follows: # comp = component (kernel, world, src) # scomp = subcomponent (generic, smp, base, doc, sys, etc.) # path/to/file = path to the file, symlink, or directory # type = 'f' (file), 'L' (link), or 'd' (directory) # inum = inode number # user, group = owning user and group # perm = permissions, in octal and including setuid bits # flags = file flags, in octal # value = link target (for a symlink), or SHA256 hash (for a file) ( cd ${WORKDIR}/$1/R && find trees -mindepth 2 | grep -vE '/obj$' | while read F; do eval $(stat -s ${F}) - # Remove the uarch flag as that is problematic - st_flags=$(($st_flags & ~04000)) + # In 12.0, we accidentally shipped with everything marked + # with the uarch flag. We needed to preserve the behavior + # for only that release. + if [ -n "$PRESERVE_UARCH" ]; then + # Unconditionally set the uarch flag + st_flags=$(($st_flags | 04000)) + else + # Remove the uarch flag as that is problematic + st_flags=$(($st_flags & ~04000)) + fi st_mode_mid=$(( ($st_mode & 07000) >> 9)) st_mode_low=$(($st_mode & 0777)) if [ -L ${F} ]; then echo -n "${F}|L|" printf "%u|%u|%u|%o%o|%o|" $st_ino $st_uid \ $st_gid $st_mode_mid $st_mode_low $st_flags readlink ${F}; elif [ -f ${F} ]; then echo -n "${F}|f|" printf "%u|%u|%u|%o%o|%o|" $st_ino $st_uid \ $st_gid $st_mode_mid $st_mode_low $st_flags sha256 -q ${F}; elif [ -d ${F} ]; then echo -n "${F}|d|" printf "%u|%u|%u|%o%o|%o|\n" $st_ino $st_uid \ $st_gid $st_mode_mid $st_mode_low $st_flags else echo "Unknown file type: ${F}" \ >/dev/stderr fi done | sed -E 's,trees/([^/]+)/([^/|]+)/?,\1|\2|/,' | sort -k 5,5 -t '|' > ${WORKDIR}/tmp-index ) # By examining the inode numbers, convert the above index # to one containing lines of the form # # comp|scomp|/path/to/file|type|user|group|perm|flags|value|hlink # # where the fields have meanings as above, plus # hlink = (lexicographically) first file to which this file is # hard linked, or blank (if there is no earlier such file). cut -f 3,5 -d '|' ${WORKDIR}/tmp-index | sort -k 1,1 -t '|' | sort -s -u -k 2,2 -t '|' | join -1 2 -2 5 -t '|' - ${WORKDIR}/tmp-index | awk -F \| -v OFS=\| '{ if ($2 == $5) print $3,$4,$5,$6,$7,$8,$9,$10,$11,"" else print $3,$4,$5,$6,$7,$8,$9,$10,$11,$2 }' | sort > ${WORKDIR}/$1-index rm ${WORKDIR}/tmp-index } # Compare release-index and $1-index and warn the user about discrepancies. diffwarn () { # Generate lists of files cut -f 1-3 -d '|' ${WORKDIR}/release-index | sort > ${WORKDIR}/release-index-flist cut -f 1-3 -d '|' ${WORKDIR}/$1-index | sort > ${WORKDIR}/$1-index-flist echo echo "Files built but not released:" comm -13 ${WORKDIR}/release-index-flist ${WORKDIR}/$1-index-flist | tee ${WORKDIR}/tmp-unreleased-flist echo "Files released but not built:" comm -23 ${WORKDIR}/release-index-flist ${WORKDIR}/$1-index-flist echo "Files which differ by more than contents:" cut -f 1-8,10 -d '|' ${WORKDIR}/release-index | sort > ${WORKDIR}/release-index-nohash cut -f 1-8,10 -d '|' ${WORKDIR}/$1-index | sort | comm -23 - ${WORKDIR}/release-index-nohash | cut -f 1-3 -d '|' | sort | comm -23 - ${WORKDIR}/tmp-unreleased-flist echo "Files which differ between release and build:" comm -23 ${WORKDIR}/$1-index ${WORKDIR}/release-index | cut -f 1-3 -d '|' | sort | comm -23 - ${WORKDIR}/tmp-unreleased-flist echo # Clean up rm ${WORKDIR}/$1-index-flist ${WORKDIR}/release-index-flist \ ${WORKDIR}/release-index-nohash ${WORKDIR}/tmp-unreleased-flist } # Compare $1 and $2 to find stamps findstamps () { log "Locating build stamps" # Make sure that the only difference is in file hashes. cut -f 1-8,10 -d '|' < ${WORKDIR}/$1-index \ > ${WORKDIR}/$1-index-nohash cut -f 1-8,10 -d '|' < ${WORKDIR}/$2-index \ > ${WORKDIR}/$2-index-nohash if ! cmp -s ${WORKDIR}/$1-index-nohash \ ${WORKDIR}/$1-index-nohash; then echo -n "Current and future builds differ " echo "by more than just hashes!" exit 1 fi rm ${WORKDIR}/$1-index-nohash ${WORKDIR}/$2-index-nohash # Generate list of files with varying hashes. comm -23 ${WORKDIR}/$1-index ${WORKDIR}/$2-index | cut -f 1-3 -d '|' | sort > ${WORKDIR}/stampedfiles.new # Construct a tarball containing the stamped files. cat ${WORKDIR}/stampedfiles.new | tr -s '|' '/' | tar -czf ${WORKDIR}/stampedfiles.tgz.new \ -C ${WORKDIR}/$1/R/trees -T - # Find stamps! tr '|' ' ' < ${WORKDIR}/stampedfiles.new | while read C SC F; do FP="R/trees/${C}/${SC}${F}" if file -b ${WORKDIR}/$1/${FP} | grep -q "text"; then ${BINDIR}/findstamps -t \ ${WORKDIR}/$1/${FP} ${WORKDIR}/$2/${FP} | lam -s "${C}|${SC}|${F}|t|" - else ${BINDIR}/findstamps \ ${WORKDIR}/$1/${FP} ${WORKDIR}/$2/${FP} | lam -s "${C}|${SC}|${F}|b|" - fi done > ${WORKDIR}/stamplocations.new \ 2> ${WORKDIR}/stampvalues.new } # Compare $1 and release to find differences in non-stamped files. findnonstamps () { # release-filemap contains lines of the form # ${C}|${SC}|${FP}|${RH}|${BH} # meaning that the file ${FP} in the ${C}|${SC} component has # hash ${RH} in the release, but hash ${BH} in the local build. comm -23 ${WORKDIR}/$1-index ${WORKDIR}/release-index | cut -f 1-3 -d '|' | sort | comm -23 - ${WORKDIR}/stampedfiles.new | tr '|' ' ' | while read C SC F; do FP="R/trees/${C}/${SC}${F}" echo "${C}|${SC}|${F}|" sha256 -q ${WORKDIR}/release/${FP} echo "|" sha256 -q ${WORKDIR}/$1/${FP} done | lam - - - - > ${WORKDIR}/release-filemap } # Fixup: the "kernel" kernel is really the "generic" kernel. indexpublish () { sed -E 's,kernel\|kernel,kernel|generic,' } # If a parameter is specified, move $1 to "newworld" and $1-index to # "newworld-index", removing anything previous newworld and index. # From the world "newworld" and associated index "newworld-index", # generate INDEX-*.new, copy data files into ${STAGEDIR}/f/ and # metadata files into ${STAGEDIR}/m/, and generate patches from earlier # data and metadata files into ${STAGEDIR}/bp/ and ${STAGEDIR}/tp/. stageworld () { # If necessary, turn $1 into newworld if ! [ $# -eq 0 ]; then # Clean if necessary if [ -d ${WORKDIR}/newworld ]; then removeworld newworld fi # Move $1 to newworld mkdir -p ${WORKDIR}/newworld tar cf - -C ${WORKDIR}/$1 . | tar xf - -C ${WORKDIR}/newworld/ nuke $1 mv ${WORKDIR}/$1-index ${WORKDIR}/newworld-index fi # Anything in ${STAGEDIR} has been left behind from a # previous build which was nor approved. log "Cleaning staging area" find ${STAGEDIR} -type f | xargs rm log "Preparing to copy files into staging area" # INDEX-ALL.new is newworld-index. Sort the index just in case # it was modified by hand and unsorted at that point. sort ${WORKDIR}/newworld-index > ${WORKDIR}/INDEX-ALL.new # Generate INDEX-NEW.new. Note that when performing an "init" # run, this file will be overwritten later, since the files in # the release aren't "new" to the user (although at this point # they are new files as far as this script is concerned). comm -23 ${WORKDIR}/INDEX-ALL ${WORKDIR}/INDEX-NEW | comm -23 ${WORKDIR}/INDEX-ALL.new - > ${WORKDIR}/INDEX-NEW.new # Generate INDEX-OLD.new. This will contain: # (a) Any lines which were at one point in INDEX-ALL but are no # longer in that file, and # (b) Lines of the form "${C}|${SC}|${F}|${T}||||||" with ${T}="-" # and ${C}, ${SC}, ${F} corresponding to any such tuples in INDEX-ALL # which at some point did not exist in INDEX-ALL (i.e., these lines # represent formerly non-existant files/directories/symlinks). comm -23 ${WORKDIR}/INDEX-ALL ${WORKDIR}/INDEX-ALL.new | sort -u ${WORKDIR}/INDEX-OLD - > ${WORKDIR}/INDEX-OLD.tmp cut -f 1-3 -d '|' < ${WORKDIR}/INDEX-ALL.new | sort > ${WORKDIR}/INDEX-ALL.new.nodes cut -f 1-3 -d '|' < ${WORKDIR}/INDEX-ALL | sort | comm -13 - ${WORKDIR}/INDEX-ALL.new.nodes | lam - -s "|-||||||" | sort -u ${WORKDIR}/INDEX-OLD.tmp - > ${WORKDIR}/INDEX-OLD.new rm ${WORKDIR}/INDEX-OLD.tmp ${WORKDIR}/INDEX-ALL.new.nodes # Copy bits of world into ${STAGEDIR}/f/ log "Copying data files into staging area" comm -23 ${WORKDIR}/INDEX-NEW.new ${WORKDIR}/INDEX-NEW | cut -f 1-4,9 -d '|' | tr '|' ' ' | while read C SC F T H; do # We don't care about directories or symlinks if ! [ ${T} = "f" ]; then continue; fi # Copy the file into the staging area gzip < ${WORKDIR}/newworld/R/trees/${C}/${SC}/${F} \ > ${STAGEDIR}/f/${H}.gz # Generate binary patches for it look "${C}|${SC}|${F}|f|" ${WORKDIR}/INDEX-OLD.new | cut -f 9 -d '|' | while read OH; do gunzip < ${WORKDIR}/oldfiles/${OH}.gz \ > ${TMPDIR}/${OH} bsdiff ${TMPDIR}/${OH} \ ${WORKDIR}/newworld/R/trees/${C}/${SC}/${F} \ ${STAGEDIR}/bp/${OH}-${H} rm ${TMPDIR}/${OH} done done # Special handling of INDEX-(NEW|OLD) for "init" builds: If the # currently existing INDEX-ALL is empty, we're populating our # databases with the release files, none of which belong in the # published list of "new" files; equally, there is no need to # record the fact that formerly nonexistant nodes now exist. if ! [ -s ${WORKDIR}/INDEX-ALL ]; then : > ${WORKDIR}/INDEX-NEW.new : > ${WORKDIR}/INDEX-OLD.new fi # Convert metadata into publishable format and build patches log "Copying metadata files into staging area" for M in INDEX-ALL INDEX-NEW INDEX-OLD; do # Create publishable version indexpublish < ${WORKDIR}/${M}.new \ | fgrep -v 'kernel|generic|/boot/kernel/linker.hints' \ > ${WORKDIR}/${M}.new.pub || true # Copy to staging area H=`sha256 -q ${WORKDIR}/${M}.new.pub` gzip < ${WORKDIR}/${M}.new.pub \ > ${STAGEDIR}/m/${H}.gz # Search for old versions of the same metadata file... grep "^${M}|" ${WORKDIR}/metadb | cut -f 2 -d '|' | while read OH; do # ... and build patches gunzip < ${WORKDIR}/oldmeta/${OH}.gz \ > ${TMPDIR}/${OH} # Identify the lines to remove. Note the for any # "${C}|${SC}|${F}|", we either want to remove # all or none of the lines with that prefix; in # INDEX-ALL and INDEX-NEW, there is at most one # line with any such prefix, and in INDEX-OLD we # never remove lines. comm -23 ${TMPDIR}/${OH} ${WORKDIR}/${M}.new.pub | cut -f 1-3 -d '|' | lam -s '-' - -s '|' > ${TMPDIR}/${OH}-${H} # Identify the lines to add comm -13 ${TMPDIR}/${OH} ${WORKDIR}/${M}.new.pub | lam -s '+' - >> ${TMPDIR}/${OH}-${H} # Move to staging area and clean up. gzip < ${TMPDIR}/${OH}-${H} \ > ${STAGEDIR}/tp/${OH}-${H}.gz rm ${TMPDIR}/${OH} ${TMPDIR}/${OH}-${H} done done # Construct metadata index log "Constructing metadata index and tag" for M in INDEX-ALL INDEX-NEW INDEX-OLD; do echo -n "${M}|" sha256 -q ${WORKDIR}/${M}.new.pub done > ${WORKDIR}/tINDEX.new # Copy metadata index into the right place TH=`sha256 -q ${WORKDIR}/tINDEX.new` cp ${WORKDIR}/tINDEX.new ${STAGEDIR}/t/${TH} # Remove published version of metadata files; we don't need # them any more. for M in INDEX-ALL INDEX-NEW INDEX-OLD; do rm ${WORKDIR}/${M}.new.pub done # Construct tag of the form # freebsd-update|i386|6.1-RELEASE|123|${TH}|${EOL}\n # meaning "FreeBSD/i386 6.1-RELEASE-p123 is described by # the tag file with hash ${TH}; and its EOL time is ${EOL} # seconds after the epoch". If the 4th field is "0", this # is the original release. echo -n 'freebsd-update|' > ${WORKDIR}/tag.new echo -n "${TARGET}|${REL}|" >> ${WORKDIR}/tag.new if [ -f ${WORKDIR}/patchnum.new ]; then echo -n `cat ${WORKDIR}/patchnum.new` >> ${WORKDIR}/tag.new else echo -n "0" >> ${WORKDIR}/tag.new fi echo "|${TH}|${EOL}" >> ${WORKDIR}/tag.new } # Print a list of stamped files and filestamps located, in order to # allow the user to verify that everything is working properly. printstamps () { echo echo "Files found which include build stamps:" cat ${WORKDIR}/stampedfiles.new echo echo "Values of build stamps, excluding library archive headers:" cat ${WORKDIR}/stampvalues.new | grep -vE '.*/[0-9]* *[0-9]{10} 0 0 (0|100644)' } # Print a list of new updates, in order to allow the user to # verify that everything is working properly. printupdates () { echo echo "New updates:" comm -23 ${WORKDIR}/INDEX-NEW.new ${WORKDIR}/INDEX-NEW } # Record the patch number of the build being performed. patchnumber () { # Read old patch number, if any if [ -f ${WORKDIR}/patchnum ]; then OPN=`cat ${WORKDIR}/patchnum` else OPN=0 fi # If the patch number is not specified, add one to the previous if [ -z "$1" ]; then export PNUM=$((${OPN} + 1)) else export PNUM="$1" fi # Patch number must be greater than previous number unless zero if [ ${OPN} -gt 0 ] && ! [ ${PNUM} -gt ${OPN} ]; then echo -n "Patch number (${PNUM}) is not greater" echo " than previous (${OPN})" exit 1 fi # Record the new patch number echo ${PNUM} > ${WORKDIR}/patchnum.new export RELP="${REL}-p${PNUM}" } # Approve the build: Sign it and place the signature and public key # into ${PUBDIR}; update all of our internal databases, and copy # files from staging area into ${PUBDIR}. approve () { # Check that we have a build ready for approval if ! [ -f ${WORKDIR}/tag.new ]; then cat <<-EOF There is no release ready for approval. You probably want to run "init.sh" or "diff.sh". EOF exit 1 fi # Sign the build log "Signing build" openssl rsautl -inkey ${PRIVKEYDIR}/priv.ssl -sign \ < ${WORKDIR}/tag.new \ > ${STAGEDIR}/latest.ssl # Export the public key openssl rsa -in ${PRIVKEYDIR}/priv.ssl -pubout \ > ${STAGEDIR}/pub.ssl 2>/dev/null # Copy files from staging area to old files directories log "Copying files to patch source directories" find ${STAGEDIR}/f/ -type f | xargs -J % cp % ${WORKDIR}/oldfiles find ${STAGEDIR}/m/ -type f | xargs -J % cp % ${WORKDIR}/oldmeta # Copy files into upload staging area. log "Copying files to upload staging area" tar -cf - -C ${STAGEDIR} . | tar -xmf - -C ${PUBDIR} # Back up some databases, just in case log "Updating databases" for X in INDEX-ALL INDEX-NEW INDEX-OLD metadb \ stampedfiles stampedfiles.tgz stamplocations stampvalues; do cp ${WORKDIR}/${X} ${WORKDIR}/${X}.bak done # Update databases for X in INDEX-ALL INDEX-NEW INDEX-OLD stampedfiles \ stampedfiles.tgz stamplocations stampvalues; do mv ${WORKDIR}/${X}.new ${WORKDIR}/${X} done if [ -f ${WORKDIR}/patchnum.new ]; then mv ${WORKDIR}/patchnum.new ${WORKDIR}/patchnum fi cat ${WORKDIR}/tINDEX.new >> ${WORKDIR}/metadb # Clean up stuff we no longer need log "Cleaning staging area" find ${STAGEDIR} -type f | xargs rm rm ${WORKDIR}/tag.new ${WORKDIR}/tINDEX.new } # Edit $1-index to # * Revert binaries to the versions in the most recent shipped build if # they have not changed aside from buildstamps, and # * Revert binaries to the versions shipped in the release if they have # not changed since the first build. unstamp () { log "Reverting changes due to build stamps" # Extract old versions of stamped files mkdir -p ${WORKDIR}/oldstamps tar -xzf ${WORKDIR}/stampedfiles.tgz -C ${WORKDIR}/oldstamps # Compare old and newly built stamped files, and generate lists # of lines to be removed and added to $1-index : > ${WORKDIR}/$1-index-add : > ${WORKDIR}/$1-index-delete tr '|' ' ' < ${WORKDIR}/stampedfiles | while read C SC FP; do # Copy old file into TMPDIR cp ${WORKDIR}/oldstamps/${C}/${SC}/${FP} ${TMPDIR}/old # Copy new file into TMPDIR cp ${WORKDIR}/$1/R/trees/${C}/${SC}/${FP} ${TMPDIR}/new # Remove text stamps grep "${C}|${SC}|${FP}|t|" ${WORKDIR}/stamplocations | cut -f 5 -d '|' | ${BINDIR}/unstamp -t ${TMPDIR}/old ${TMPDIR}/old.1 grep "${C}|${SC}|${FP}|t|" ${WORKDIR}/stamplocations | cut -f 5 -d '|' | ${BINDIR}/unstamp -t ${TMPDIR}/new ${TMPDIR}/new.1 # Remove binary stamps grep "${C}|${SC}|${FP}|b|" ${WORKDIR}/stamplocations | cut -f 5- -d '|' | ${BINDIR}/unstamp ${TMPDIR}/old.1 ${TMPDIR}/old.2 grep "${C}|${SC}|${FP}|b|" ${WORKDIR}/stamplocations | cut -f 5- -d '|' | ${BINDIR}/unstamp ${TMPDIR}/new.1 ${TMPDIR}/new.2 # Compare unstamped files if cmp -s ${TMPDIR}/old.2 ${TMPDIR}/new.2; then look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index | cut -f 1-8 -d '|' | tr -d '\n' >> ${WORKDIR}/$1-index-add echo -n '|' >> ${WORKDIR}/$1-index-add look "${C}|${SC}|${FP}|" ${WORKDIR}/INDEX-ALL | cut -f 9 -d '|' | tr -d '\n' >> ${WORKDIR}/$1-index-add echo -n '|' >> ${WORKDIR}/$1-index-add look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index | cut -f 10 -d '|' >> ${WORKDIR}/$1-index-add look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index \ >> ${WORKDIR}/$1-index-delete fi done # Remove lines from $1-index and add new lines. sort ${WORKDIR}/$1-index-delete | comm -23 ${WORKDIR}/$1-index - | sort - ${WORKDIR}/$1-index-add > ${WORKDIR}/$1-index.tmp mv ${WORKDIR}/$1-index.tmp ${WORKDIR}/$1-index # Repeat the process, only replacing the hashes of locally built # files with the hashes of the released files where appropriate. : > ${WORKDIR}/$1-index-add : > ${WORKDIR}/$1-index-delete tr '|' ' ' < ${WORKDIR}/release-filemap | while read C SC FP RH BH; do if look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index | grep -q "|${BH}|"; then look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index | cut -f 1-8 -d '|' | tr -d '\n' >> ${WORKDIR}/$1-index-add echo -n "|${RH}|" >> ${WORKDIR}/$1-index-add look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index | cut -f 10 -d '|' >> ${WORKDIR}/$1-index-add look "${C}|${SC}|${FP}|" ${WORKDIR}/$1-index \ >> ${WORKDIR}/$1-index-delete fi done sort ${WORKDIR}/$1-index-delete | comm -23 ${WORKDIR}/$1-index - | sort - ${WORKDIR}/$1-index-add > ${WORKDIR}/$1-index.tmp mv ${WORKDIR}/$1-index.tmp ${WORKDIR}/$1-index # Clean up rm ${WORKDIR}/$1-index-add ${WORKDIR}/$1-index-delete rm ${TMPDIR}/old ${TMPDIR}/old.1 ${TMPDIR}/old.2 rm ${TMPDIR}/new ${TMPDIR}/new.1 ${TMPDIR}/new.2 nuke oldstamps } # Upload files to server. Note that we upload latest.ssl last. upload () { log "Uploading files" # Upload files, and make sure the upload was successful tar -cf - --exclude latest.ssl --exclude uploaded \ --newer-mtime-than ${PUBDIR}/uploaded \ -C ${BASEDIR}/pub ${REL}/${TARGET} | ssh -i ${SSHKEY} ${MASTERACCT} \ tar -xmf - -C ${MASTERDIR} if [ $? -ne 0 ]; then exit 1 fi # Upload latest.ssl, and make sure the upload was successful tar -cf - -C ${BASEDIR}/pub ${REL}/${TARGET}/latest.ssl | ssh -i ${SSHKEY} ${MASTERACCT} \ tar -xmf - -C ${MASTERDIR} if [ $? -ne 0 ]; then exit 1 fi # Mark all files prior to now as having been uploaded touch ${PUBDIR}/uploaded }