Index: projects/portbuild/scripts/build =================================================================== --- projects/portbuild/scripts/build (revision 221760) +++ projects/portbuild/scripts/build (revision 221761) @@ -1,599 +1,605 @@ #!/bin/sh # $FreeBSD: ports/Tools/portbuild/scripts/build,v 1.15 2011/03/24 06:24:04 linimon Exp $ # server-side script to handle various commands common to builds # configurable variables pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} # subdirectories to process. yes, this is a hack, but it saves code # duplication. quoted_subdirs="'/src' '/ports' ''" # XXX unused get_latest_snap() { snap=$1 zfs list -rHt snapshot ${snap} | tail -1 | awk '{print $1}' } now() { date +%Y%m%d%H%M%S } do_list() { arch=$1 branch=$2 buildpar=${pbd}/${arch}/${branch}/builds if [ -d ${buildpar} ]; then snaps=$(cd ${buildpar}; ls -1d 2* 2> /dev/null) echo "The following builds are active:" echo ${snaps} if [ -L ${buildpar}/latest -a -d ${buildpar}/latest/ ]; then link=$(readlink ${buildpar}/latest) link=${link%/} link=${link##*/} echo "Latest build is: ${link}" fi else echo "No such build environment ${arch}/${branch}" exit 1 fi } do_create() { arch=$1 branch=$2 buildid=$3 shift 3 archivedir=${pbd}/${arch}/archive # create directory for all build logs buildlogsdir=${archivedir}/buildlogs if [ ! -d ${buildlogsdir} ]; then mkdir -p ${buildlogsdir} || exit 1 chown -R ports-${arch}:portmgr ${archivedir} chmod -R g+w ${archivedir} fi # create directory for all builds for buildenv buildsdir=${pbab}/builds if [ ! -d ${buildsdir} ]; then mkdir -p ${buildsdir} || exit 1 chown -R ports-${arch}:portmgr ${pbab} chmod -R g+w ${pbab} fi # create directory for latest build for buildenv builddir=$(realpath ${buildsdir})/${buildid} if [ -d ${builddir} ]; then echo "Can't create ${builddir}, it already exists" exit 1 fi # create zfs instance for latest build on buildenv mountpoint=${builddir} newfs=${ZFS_VOLUME}/portbuild/${arch}/${buildid} zfs create -o mountpoint=${mountpoint} ${newfs} || exit 1 chown -R ports-${arch}:portmgr ${mountpoint} chmod -R g+w ${mountpoint} # populate ports for latest build on buildenv do_portsupdate_inner ${arch} ${branch} ${buildid} ${builddir} $@ # populate src for latest build on buildenv do_srcupdate_inner ${arch} ${branch} ${buildid} ${builddir} $@ # create the link for building packages for latest build ln -sf ${builddir} ${pbab}/builds/latest # create the links for the webserver errorlogs=${pbd}/errorlogs ln -s ${buildsdir}/latest/bak/errors ${errorlogs}/${arch}-${branch}-previous ln -s ${buildsdir}/latest/bak/logs ${errorlogs}/${arch}-${branch}-previous-logs ln -s ${buildsdir}/latest/errors ${errorlogs}/${arch}-${branch}-latest ln -s ${buildsdir}/latest/logs ${errorlogs}/${arch}-${branch}-latest-logs ln -s ${buildsdir}/latest/bak/packages ${errorlogs}/${arch}-${branch}-packages-previous ln -s ${buildsdir}/latest/packages ${errorlogs}/${arch}-${branch}-packages-latest echo "New build ID is ${buildid}" } do_clone() { arch=$1 branch=$2 buildid=$3 builddir=$4 shift 4 if [ "$#" -gt 0 ]; then newid=$1 shift else newid=$(now) fi tmp=$(realpath ${builddir}) tmp=${tmp%/} newbuilddir="${tmp%/*}/${newid}" oldfs=${ZFS_VOLUME}/portbuild/${arch}/${buildid} newfs=${ZFS_VOLUME}/portbuild/${arch}/${newid} zfs snapshot ${oldfs}@${newid} zfs clone ${oldfs}@${newid} ${newfs} zfs set mountpoint=${newbuilddir} ${newfs} zfs promote ${newfs} if zfs list -H -t filesystem ${oldfs}/ports 2> /dev/null; then portsnap=${oldfs}/ports@${newid} zfs snapshot ${portsnap} zfs clone ${portsnap} ${newfs}/ports zfs promote ${newfs}/ports fi if zfs list -H -t filesystem ${oldfs}/src 2> /dev/null; then srcsnap=${oldfs}/src@${newid} zfs snapshot ${srcsnap} zfs clone ${srcsnap} ${newfs}/src zfs promote ${newfs}/src fi if [ -d ${newbuilddir} ]; then if [ ! -f ${pbab}/builds/previous/.keep ]; then ${pbc}/scripts/build destroy ${arch} ${branch} previous fi rm -f ${pbab}/builds/previous mv ${pbab}/builds/latest ${pbab}/builds/previous ln -sf ${newbuilddir} ${pbab}/builds/latest fi echo "New build ID is ${newid}" } do_portsupdate() { arch=$1 branch=$2 buildid=$3 builddir=$4 shift 4 if [ $# -gt 0 ]; then arg=$1 shift fi destroy_fs ${ZFS_VOLUME}/portbuild/${arch} ${buildid} /ports || exit 1 if [ "${arg}" = "-umount" ]; then return fi echo do_portsupdate_inner ${arch} ${branch} ${buildid} ${builddir} $@ } do_portsupdate_inner() { arch=$1 branch=$2 buildid=$3 builddir=$4 shift 4 echo "================================================" echo "Reimaging ZFS ports tree on ${builddir}/ports" echo "================================================" portsfs=${ZFS_VOLUME}/portbuild/${arch}/${buildid}/ports now=$(now) zfs snapshot ${ZFS_VOLUME}/${SNAP_PORTS_DIRECTORY}/ports@${now} zfs clone ${ZFS_VOLUME}/${SNAP_PORTS_DIRECTORY}/ports@${now} ${portsfs} zfs set mountpoint=${builddir}/ports ${portsfs} } do_srcupdate() { arch=$1 branch=$2 buildid=$3 builddir=$4 shift 4 if [ $# -gt 0 ]; then arg=$1 shift fi destroy_fs ${ZFS_VOLUME}/portbuild/${arch} ${buildid} /src || exit 1 if [ "${arg}" = "-umount" ]; then return fi echo do_srcupdate_inner ${arch} ${branch} ${buildid} ${builddir} $@ } do_srcupdate_inner() { arch=$1 branch=$2 buildid=$3 builddir=$4 shift 4 echo "================================================" echo "Reimaging ZFS src tree on ${builddir}/src" echo "================================================" strippedbranch=${branch%%[-\.]*} srcfs=${ZFS_VOLUME}/portbuild/${arch}/${buildid}/src now=$(now) zfs snapshot ${ZFS_VOLUME}/${SNAP_SRC_DIRECTORY_PREFIX}${strippedbranch}/src@${now} zfs clone ${ZFS_VOLUME}/${SNAP_SRC_DIRECTORY_PREFIX}${strippedbranch}/src@${now} ${srcfs} zfs set mountpoint=${builddir}/src ${srcfs} } cleanup_client() { arch=$1 branch=$2 buildid=$3 mach=$4 arg=$5 # XXX use same exclusion protocol as claim-chroot echo "Started cleaning up ${arch}/${branch} build ID ${buildid} on ${mach}" test -f ${pbd}/${arch}/portbuild.${mach} && . ${pbd}/${arch}/portbuild.${mach} # Kill off builds and clean up chroot ${pbc}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${mach} -nocopy -queue -full echo "Finished cleaning up ${arch}/${branch} build ID ${buildid} on ${mach}" } do_cleanup() { arch=$1 branch=$2 buildid=$3 builddir=$4 arg=$5 shift 5 for i in `cat ${pbd}/${arch}/mlist`; do cleanup_client ${arch} ${branch} ${buildid} ${i} ${arg} & done wait } do_upload() { arch=$1 branch=$2 buildid=$3 builddir=$4 shift 4 echo "Not implemented yet" exit 1 } test_fs() { local fs=$1 zfs list -Ht filesystem ${fs} > /dev/null 2>&1 } get_latest_child() { local fs=$1 # Return the child of this filesystem with lexicographically # highest name # # XXX if a filesystem is cloned into a different prefix # (e.g. different arch) then we may not get the most recent one # but that should not happen. zfs get -H -o name,value origin | grep ${fs} | sort | \ (while read zfs origin; do if [ "${origin%@*}" = "${fs}" ]; then child=${zfs} fi done; echo ${child}) } get_parent() { local fs=$1 # Check whether this filesystem has a parent zfs get -H -o value origin ${fs} | \ (read snap; case "${snap}" in -|${ZFS_VOLUME}/${SNAP_DIRECTORY}/*) ;; *) parent=${snap} ;; esac; echo ${parent}) } destroy_fs() { fs=$1 buildid=$2 subfs=$3 fullfs=${fs}/${buildid}${subfs} if test_fs "${fullfs}"; then # We can destroy a leaf filesystem (having no dependent # clones) with no further effort. However if we are # destroying the root of the clone tree then we have to # promote a child to be the new root. # # XXX In principle we might have to iterate until we end up as # a leaf but I don't know if this can happen. echo "Filesystem ${fullfs}" child=$(get_latest_child ${fullfs}) parent=$(get_parent ${fullfs}) echo "Filesystem has parent ${parent}" if [ -z "${child}" ]; then echo "Filesystem is a leaf" echo else echo "Filesystem has latest child ${child}" # Check whether filesystem is root if [ -z "${parent}" ]; then echo "Filesystem is root; promoting ${child}" zfs promote ${child} parent=$(get_parent ${fullfs}) echo "New parent is ${parent}" echo else echo "Filesystem has parent ${parent} and cannot be destroyed" echo return 1 fi fi # We might have snapshots on the target filesystem, e.g. if it # is both the head and tail of its clone tree. They should be # unreferenced. # We have to grep because zfs list -H returns an error instead of # a null list if no snapshots exist if ! (zfs list -r -H -o name -t snapshot ${fullfs} | grep "^${fullfs}@" | xargs -n 1 zfs destroy); then return 1 fi # The target filesystem should now be unreferenced if ! zfs destroy -f "${fullfs}"; then return 1 fi # Destroy the origin snapshot, which should be unreferenced if [ ! -z "${parent}" ]; then if ! zfs destroy -f ${parent}; then return 1 fi fi fi } do_destroy() { arch=$1 branch=$2 buildid=$3 builddir=$4 shift 4 echo buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 fi # perform sanity check to prevent e.g. "latest" being destroyed # before "previous" for quoted_subdir in $quoted_subdirs; do fullfs=${ZFS_VOLUME}/portbuild/${arch}/${buildid}`echo $quoted_subdir | sed -e "s/'//g"` if test_fs "${fullfs}"; then child=$(get_latest_child ${fullfs}) parent=$(get_parent ${fullfs}) if [ ! -z "${child}" -a ! -z "${parent}" ]; then echo "Filesystem ${fullfs} has parent ${parent} and child ${child}" echo "and thus cannot be destroyed. Run 'build destroy' on the child, first." fi fi done # now ok to do the actual destroy. for quoted_subdir in $quoted_subdirs; do destroy_fs ${ZFS_VOLUME}/portbuild/${arch} ${buildid} `echo $quoted_subdir | sed -e "s/'//g"` || exit 1 done rmdir ${builddir} } # Run a command as root if running as user # Authentication and command validation is taken care of by buildproxy proxy_root() { cmd=$1 arch=$2 branch=$3 buildid=$4 builddir=$5 shift 5 args=$@ id=$(id -u) if [ ${id} != "0" ]; then ${pbc}/scripts/buildproxy-client "build ${cmd} ${arch} ${branch} ${buildid} ${args}" error=$? if [ ${error} -eq 254 ]; then echo "Proxy error" fi else eval "do_${cmd} ${arch} ${branch} ${buildid} ${builddir} ${args}" error=$? fi exit ${error} } # Run a command as the ports-${arch} user if root proxy_user() { cmd=$1 arch=$2 branch=$3 buildid=$4 builddir=$5 shift 5 args=$@ id=$(id -u) if [ ${id} != "0" ]; then eval "do_${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}" error=$? else su ports-${arch} -c "${pbc}/scripts/build ${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}" error=$? fi exit ${error} } usage () { echo "usage: build [] [ ...]" exit 1 } ################## if [ $# -lt 3 ]; then usage fi cmd=$1 arch=$2 branch=$3 shift 3 . ${pbc}/conf/server.conf . ${pbd}/${arch}/portbuild.conf . ${pbc}/scripts/buildenv pbab=${pbd}/${arch}/${branch} validate_env ${arch} ${branch} || exit 1 # Not every command requires a buildid as arg if [ $# -ge 1 ]; then buildid=$1 shift 1 # Most commands require a buildid that is valid on the server. The # exception is "cleanup" which is cleaning up a client build that may # already be destroyed on the server. case "$cmd" in cleanup) # Resolve symlinks but don't bail if the build doesn't exist. newbuildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ ! -z "${newbuildid}" -a "${newbuildid}" != "${buildid}" ]; then echo "Resolved ${buildid} to ${newbuildid}" buildid=${newbuildid} builddir=$(realpath ${pbab}/builds/${buildid}/) # We can't rely on buildenv for this code path fi ;; create) # XXX some way to avoid the latest/previous dance? if [ -z "${buildid}" -o "${buildid}" = "latest" ]; then buildid=$(now)"."`hostname -s` elif [ "${buildid}" = "previous" ]; then echo "Use build clone latest instead" exit 1 else buildid=${buildid%/} fi # We can't rely on buildenv for this code path ;; *) newbuildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${newbuildid}" ]; then echo "Build ID ${buildid} does not exist" exit 1 fi if [ ${newbuildid} != ${buildid} ]; then echo "Resolved ${buildid} to ${newbuildid}" buildid=${newbuildid} fi builddir=$(realpath ${pbab}/builds/${buildid}/) buildenv ${pbd} ${arch} ${branch} ${builddir} ;; esac fi +if [ -n "${buildid}" ]; then + if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf + fi +fi + # Unprivileged commands case "$cmd" in list) do_list ${arch} ${branch} $@ || exit 1 ;; create) if [ -z "${buildid}" ]; then usage fi proxy_root create ${arch} ${branch} ${buildid} $@ || exit 1 ;; clone) if [ -z "${buildid}" ]; then usage fi proxy_root clone ${arch} ${branch} ${buildid} ${builddir} $@ || exit 1 ;; portsupdate) if [ -z "${buildid}" ]; then usage fi proxy_root portsupdate ${arch} ${branch} ${buildid} ${builddir} $@ || exit 1 ;; srcupdate) if [ -z "${buildid}" ]; then usage fi proxy_root srcupdate ${arch} ${branch} ${buildid} ${builddir} $@ || exit 1 ;; cleanup) if [ -z "${buildid}" ]; then usage fi # builddir may be null if cleaning up a destroyed build proxy_user cleanup ${arch} ${branch} ${buildid} "${builddir}" $@ || exit 1 ;; upload) if [ -z "${buildid}" ]; then usage fi proxy_user upload ${arch} ${branch} ${buildid} ${builddir} $@ || exit 1 ;; destroy) if [ -z "${buildid}" ]; then usage fi proxy_root destroy ${arch} ${branch} ${buildid} ${builddir} $@ || exit 1 ;; *) echo "build: invalid command: $cmd" exit 1 ;; esac Index: projects/portbuild/scripts/buildfailure =================================================================== --- projects/portbuild/scripts/buildfailure (revision 221760) +++ projects/portbuild/scripts/buildfailure (revision 221761) @@ -1,65 +1,68 @@ #!/bin/sh # # buildfailure cleanup() { echo "Problem writing new failure file!" rm -f failure.new exit 1 } # configurable variables pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: buildfailure arch branch buildid pkgname" exit 1 } if [ $# -ne 4 ]; then usage fi arch=$1 branch=$2 buildid=$3 pkgname=$4 shift 4 builddir=${pbd}/${arch}/${branch}/builds/${buildid} . ${pbc}/conf/server.conf . ${pbc}/conf/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbc}/scripts/buildenv buildenv ${pbd} ${arch} ${branch} ${builddir} # Don't pick up installed packages from the host export LOCALBASE=/nonexistentlocal index=${PORTSDIR}/${INDEXFILE} portloc=$(grep "^$pkgname|" ${index} | cut -f 2 -d \| | sed s,/usr/ports/,,) pkgbase=$(cd ${PORTSDIR}/${portloc}/ && make -V PKGBASE) cd ${pbd}/${arch}/${branch} entry=$(grep "^${portloc}|" failure) date=$(date +%s) IFS='|' if [ ! -z "$entry" ]; then count=$(echo $entry | cut -f 6 -d \ ) olddate=$(echo $entry | cut -f 4 -d \ ) (grep -v "^${portloc}|" failure > failure.new) || cleanup (echo "${portloc}|${pkgbase}|${pkgname}|${olddate}|${date}|$((${count}+1))" >> failure.new) || cleanup mv failure.new failure else (echo "${portloc}|${pkgbase}|${pkgname}|${date}|${date}|1" >> failure) || cleanup fi link=${pbd}/${arch}/${branch}/latest/${portloc} mkdir -p $(dirname ${link}) errorloc=$(realpath ${builddir}/errors/${pkgname}.log) ln -sf ${errorloc} ${link} Index: projects/portbuild/scripts/buildsuccess =================================================================== --- projects/portbuild/scripts/buildsuccess (revision 221760) +++ projects/portbuild/scripts/buildsuccess (revision 221761) @@ -1,49 +1,52 @@ #!/bin/sh # # buildsuccess # configurable variables pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: buildsuccess arch branch buildid pkgname" exit 1 } if [ $# -ne 4 ]; then usage fi arch=$1 branch=$2 buildid=$3 pkgname=$4 shift 4 builddir=${pbd}/${arch}/${branch}/builds/${buildid} . ${pbc}/conf/server.conf . ${pbc}/conf/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbc}/scripts/buildenv buildenv ${pbd} ${arch} ${branch} ${builddir} # Don't pick up installed packages from the host export LOCALBASE=/nonexistentlocal index=${PORTSDIR}/${INDEXFILE} portloc=$(grep "^$pkgname|" ${index} | cut -f 2 -d \| | sed s,/usr/ports/,,) cd ${pbd}/${arch}/${branch} if grep -q "^${portloc}|" failure; then echo | mail -s "${pkgname} now builds on ${arch} ${branch}" ${mailto} grep -v "^${portloc}|" failure > failure.new mv failure.new failure fi if [ -L ${pbd}/${arch}/${branch}/latest/${portloc} ]; then rm -f ${pbd}/${arch}/${branch}/latest/${portloc} fi Index: projects/portbuild/scripts/claim-chroot =================================================================== --- projects/portbuild/scripts/claim-chroot (revision 221760) +++ projects/portbuild/scripts/claim-chroot (revision 221761) @@ -1,164 +1,167 @@ #!/bin/sh # client-side script to claim a chroot # usage: claim-chroot ${arch} ${branch} ${pkgname} ${buildid} # Care needs to be taken with the output of this script, it cannot # output anything except space-separated pairs of "keyword value". # # Keywords: # chroot : successfully claimed a chroot # setup : we own the rights to setup the build env # wait : someone else is setting up the build env # In case of other error, just exit. # XXX if the setupnode process was a single process invocation we # could use a lockf lock, and be able to tell if the setup process was # still running or died prematurely pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: claim-chroot arch branch buildid" exit 1 } if [ $# -ne 4 ]; then usage fi arch=$1 branch=$2 buildid=$3 pkgname=$4 shift 4 # If client has just rebooted it may not have any files yet if [ ! -f /tmp/.boot_finished ]; then echo "wait boot" exit 1 fi # Do we need to set up the client after cold boot? # # NB: mkdir is being used as an atomic test-and-set operation to # provide mutual exclusion against other callers, since we only want # one of them to perform setup builddir=${pbd}/${arch}/${branch}/builds/${buildid} # Is the build environment populated? Again we only want a single # instance to gain setup rights if not. if (mkdir /tmp/.setup-${buildid} 2> /dev/null); then # The buildenv is not set up, tell the caller to do it echo "setup ${builddir}" exit 1 fi if [ ! -f ${builddir}/.ready ]; then # The buildenv is still being set up echo "wait ${builddir}" exit 1 fi . ${pbd}/${arch}/client.conf . ${pbd}/${arch}/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbd}/${arch}/portbuild.$(hostname) buildroot=${scratchdir} pkgname=${pkgname%.${PKGSUFFIX}} chrootdir=${buildroot}/${branch}/${buildid}/chroot # Perform initial sanity checks # Check squid is running if [ ! -z "${squid_dir}" ]; then /usr/local/sbin/squid -k check 2> /dev/null status=$? if [ "${status}" != "0" ]; then touch ${scratchdir}/.squid /usr/local/etc/rc.d/squid start > /dev/null & echo "error squid" exit 1 else rm -f ${scratchdir}/.squid fi fi # Check for enough disk space df=$(df -k ${scratchdir} | tail -1 | awk '{print $4}') if [ ${df} -lt 102400 ]; then touch ${scratchdir}/.disk echo "error disk" exit 1 else rm -f ${scratchdir}/.disk fi found=0 # Look for pre-existing chroot directories that are populated and unused for dir in ${chrootdir}/*; do if [ -f ${dir}/.ready -o -f ${dir}/.dirty ]; then # Atomically claim the directory mkdir ${dir}/used 2>/dev/null || continue touch ${dir}/used/${pkgname} if [ -f ${dir}/.dirty ]; then /tmp/${buildid}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${dir} 2 >/dev/null 2>/dev/null & continue fi found=1 chroot=${dir} break fi done chrootnum=$$ # If we didn't find a pre-existing directory, create and claim a new one. while [ ${found} != 1 ]; do if [ "${use_zfs}" = "1" ]; then chroot=${chrootdir}/${chrootnum} # XXX deal with failure zfs clone ${scratchdir#/}/${branch}/${buildid}/world@base ${chroot#/} mkdir ${chroot}/used elif [ "${use_md_swap}" = "1" ]; then unit=$(mdconfig -a -t swap -s ${md_size}) if [ -z "${unit}" ]; then echo "error mdconfig" exit 1 fi newfs /dev/${unit} > /dev/null chrootnum=$(echo ${unit} | sed 's,md,,') chroot=${chrootdir}/${chrootnum} mkdir -p ${chroot}/used 2>/dev/null || continue # Need to make sure that used/ is also present after mounting # the fresh md so as to not leave open any races mount -o async /dev/${unit} ${chroot}/used mkdir ${chroot}/used/used touch ${chroot}/used/used/${pkgname} umount -f ${chroot}/used mount -o async /dev/${unit} ${chroot}/ touch ${chroot}/.notready else chrootnum=$(($chrootnum+1)) chroot=${chrootdir}/${chrootnum} mkdir -p ${chrootdir} 2> /dev/null || continue mkdir ${chroot} 2>/dev/null || continue mkdir ${chroot}/used 2>/dev/null || continue touch ${chroot}/.notready fi if [ "${use_tmpfs}" = "1" ]; then mount -t tmpfs -o "size=${tmpfs_size}" foo ${chroot} mkdir ${chroot}/used 2>/dev/null || echo "ERROR: mkdir race" touch ${chroot}/.notready fi touch ${chroot}/used/${pkgname} found=1 done echo "chroot ${chroot}" Index: projects/portbuild/scripts/clean-chroot =================================================================== --- projects/portbuild/scripts/clean-chroot (revision 221760) +++ projects/portbuild/scripts/clean-chroot (revision 221761) @@ -1,124 +1,127 @@ #!/bin/sh # $FreeBSD: ports/Tools/portbuild/scripts/clean-chroot,v 1.18 2010/12/01 02:27:35 linimon Exp $ # client-side script to clean up a chroot kill_procs() { dir=$1 mount=$2 pids="XXX" while [ ! -z "${pids}" ]; do pids=$(fstat -f "${dir}${mount}" | tail +2 | awk '{print $3}' | sort -u) if [ ! -z "${pids}" ]; then echo "Killing off pids in ${dir}" ps -p $pids kill -KILL ${pids} 2> /dev/null sleep 2 fi done } cleanup_mount() { chroot=$1 mount=$2 if [ -d ${chroot}${mount} ]; then mdir=$(fstat -f ${chroot}${mount} | head -2 | tail -1 | awk '{print $5}') if [ "${mdir}" = "MOUNT" ]; then umount -f ${chroot}${mount} || echo "Cleanup of ${chroot}${mount} on $(hostname) failed!" fi if [ "${mdir}" = "${chroot}${mount}" ]; then kill_procs ${chroot} ${mount} umount -f ${chroot}${mount} || echo "Cleanup of ${chroot}${mount} on $(hostname) failed!" fi fi } arch=$1 branch=$2 buildid=$3 chroot=$4 clean=$5 pbd=${PORTBUILD_DATA:-/var/portbuild} . ${pbd}/${arch}/client.conf . ${pbd}/${arch}/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbd}/${arch}/portbuild.$(hostname) # directories to clean cleandirs="${LOCALBASE} /compat /var/db/pkg" if [ ! -d "${chroot}" ]; then exit 0 fi if [ `realpath ${chroot}` = "/" ]; then # Don't spam the root file system if something has gone wrong! exit 1 fi if [ -f ${chroot}/tmp/jail.id ]; then pgrep -lfj `awk '{print $1}' ${chroot}/tmp/jail.id` pkill -j `awk '{print $1}' ${chroot}/tmp/jail.id` fi #umount ${chroot}/proc for i in ${ARCHS_REQUIRING_LINPROCFS}; do if [ ${i} = ${arch} ]; then cleanup_mount ${chroot} /compat/linux/proc break fi done for i in /a/ports /usr/src /dev /root/.ccache; do cleanup_mount ${chroot} ${i} done if [ "${use_zfs}" = "1" ]; then cleanup_mount ${chroot} "" zfs destroy -f ${chroot#/} elif [ "${use_tmpfs}" = "1" -a "${clean}" = "2" ]; then cleanup_mount ${chroot} "" if ! rm -rf ${chroot} >/dev/null 2>&1; then chflags -R noschg ${chroot} >/dev/null 2>&1 rm -rf ${chroot} >/dev/null 2>&1 fi # XXX possible race from cleanup and claim by next build? elif [ "${use_md_swap}" = "1" -a \( "${md_persistent}" != "1" -a "${clean}" -gt "0" \) -o "${clean}" = "2" ]; then cleanup_mount ${chroot} /used > /dev/null 2>&1 cleanup_mount ${chroot} "" mdconfig -d -u $(basename ${chroot}) if ! rm -rf ${chroot} >/dev/null 2>&1; then chflags -R noschg ${chroot} >/dev/null 2>&1 rm -rf ${chroot} >/dev/null 2>&1 fi else if [ "${clean}" = 1 ]; then rm -rf ${chroot}/tmp/* for dir in ${cleandirs}; do if ! rm -rf ${chroot}${dir} >/dev/null 2>&1; then chflags -R noschg ${chroot}${dir} >/dev/null 2>&1 rm -rf ${chroot}${dir} >/dev/null 2>&1 fi done test -x ${chroot}/sbin/ldconfig && chroot ${chroot} /sbin/ldconfig -R for i in ${ARCHS_REQUIRING_AOUT_COMPAT}; do if [ ${i} = ${arch} ]; then test -x ${chroot}/sbin/ldconfig && chroot ${chroot} /sbin/ldconfig -aout -R break fi done rm -rf ${chroot}/var/db/pkg/* rm -rf ${chroot}/used elif [ "${clean}" = 2 ]; then if ! rm -rf ${chroot} >/dev/null 2>&1; then chflags -R noschg ${chroot} >/dev/null 2>&1 rm -rf ${chroot} >/dev/null 2>&1 fi fi fi Index: projects/portbuild/scripts/dopackages =================================================================== --- projects/portbuild/scripts/dopackages (revision 221760) +++ projects/portbuild/scripts/dopackages (revision 221761) @@ -1,787 +1,793 @@ #!/bin/sh # $FreeBSD: ports/Tools/portbuild/scripts/dopackages,v 1.57 2010/12/01 02:30:14 linimon Exp $ # main server-side script to run a package build # configurable variables pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${pbc}/scripts # writable by portmgr umask 002 journalname="journal" usage () { echo "usage: arch branch buildid datestamp [-incremental] [-continue] [-restart] [-nofinish] [-finish] [-nocleanup] [-keep] [-nobuild] [-noindex] [-noduds] [-norestr] [-nochecksubdirs] [-nosrc] [-srcvcs] [-noports] [-portsvcs] [-noplistcheck] [-nodistfiles] [-fetch-original] [-cdrom] [-trybroken] [-target ]" # XXX MCL I think it's going to be too hard to move the create in here, now. echo " -incremental : Start a new incremental build" echo " -continue : Restart an interrupted build, skipping failed ports" echo " -restart : Restart an interrupted build, rebuilding failed ports" echo " -nofinish : Do not post-process upon build completion" echo " -finish : Post-process a completed build" echo " -nocleanup : Do not clean up and deactivate the build once it finishes" echo " -keep : Do not automatically recycle this build" echo " -nobuild : Only do the build preparation steps, do not build packages" echo " -noindex : Do not build the INDEX" echo " -noduds : Do not build the duds file" echo " -nochecksubdirs : Do not check the SUBDIRS" echo " -norestr : Do not build the restricted.sh file" echo " -nosrc : Do not update the src tree" echo " -srcvcs : Update the src tree via CVS, don't use a pre-existing snapshot" echo " -noports : Do not update the ports tree" echo " -portsvcs : Update the ports tree via CVS, don't use a pre-existing snapshot" echo " -noplistcheck : Don't check the plist during the build" echo " -nodistfiles : Don't collect distfiles" echo " -fetch-original : Fetch from original MASTER_SITE" echo " -cdrom : Prepare a build for distribution on CDROM " echo " -trybroken : Try to build BROKEN ports" echo " -target : Build ports listed in file, rather than the whole ports tree" exit 1 } if [ $# -lt 4 ]; then usage fi arch=$1 branch=$2 buildid=$3 datestamp=$4 shift 4 . ${pbc}/conf/server.conf . ${pbc}/conf/common.conf . ${pbc}/scripts/buildenv validate_env ${arch} ${branch} || usage # XXX MCL too early to do this here. buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 fi if [ -f ${pbd}/${arch}/portbuild.conf ]; then . ${pbc}/conf/server.conf . ${pbd}/${arch}/portbuild.conf else usage fi pbab=${pbd}/${arch}/${branch} trap "exit 1" 1 2 3 9 10 11 15 mailexit () { echo | mail -s "$(basename $0) ended for ${arch}-${branch} ${buildid} at $(date)" ${mailto} exit $1 } srctar() { rm -f ${builddir}/src-2*.tbz* tar cfCj ${builddir}/src-${buildid}.tbz ${builddir} src/ 2>/dev/null md5 ${builddir}/src-${buildid}.tbz > ${builddir}/src-${buildid}.tbz.md5 } portstar() { rm -f ${builddir}/ports-2*.tbz* tar cfCj ${builddir}/ports-${buildid}.tbz ${builddir} ports/ 2>/dev/null md5 ${builddir}/ports-${buildid}.tbz > ${builddir}/ports-${buildid}.tbz.md5 } # usage: makeindex pb arch branch builddir [target] # note: can take ~24 minutes! makeindex () { pbc=$1 arch=$2 branch=$3 buildid=$4 builddir=$5 target=$6 cd ${builddir}/ports echo "================================================" echo "generating index" echo "================================================" echo "index generation started at $(date)" ${pbc}/scripts/makeindex ${arch} ${branch} ${buildid} ${target} || return 1 echo "index generation ended at $(date)" echo $(wc -l ${INDEXFILE} | awk '{print $1}') "lines in INDEX" # Save a copy of it for the next build since ports directories may # not be preserved cp ${INDEXFILE} ${builddir}/bak } # usage: checkindex builddir # Perform some sanity checks on the INDEX so we don't blow up later on checkindex () { builddir=$1 cd ${builddir}/ports if [ ! -f ${INDEXFILE} ]; then echo "misssing INDEXFILE ${INDEXFILE} in ${builddir}/ports" return 1 fi if grep -q non-existent ${INDEXFILE}; then echo "errors in INDEX:" grep -n non-existent ${INDEXFILE} return 1 fi if ! awk -F '|' '{if (NF != 13) { error=1; printf("line %d: %s\n", NR, $0)}} END {if (error == 1) exit(1)}' ${INDEXFILE}; then echo "error in INDEX" return 1 fi } # usage: makeduds pb arch branch builddir [target] # note: can take ~21 minutes! makeduds () { pbc=$1 arch=$2 branch=$3 buildid=$4 builddir=$5 target=$6 cd ${builddir}/ports echo "================================================" echo "generating duds" echo "================================================" echo "duds generation started at $(date)" if [ -e ${builddir}/duds ]; then cp -p ${builddir}/duds ${builddir}/duds.old fi if ! ${pbc}/scripts/makeduds ${arch} ${branch} ${buildid} ${target}; then echo "error(s) detected, exiting script at $(date). Failed duds list was:" cat ${builddir}/duds mailexit 1 fi echo "duds generation ended at $(date)" echo $(wc -l ${builddir}/duds | awk '{print $1}') "items in duds" if [ -f ${builddir}/duds.old ]; then echo "duds diff:" diff ${builddir}/duds.old ${builddir}/duds else echo "no previous duds to compare against." fi cp -p ${builddir}/duds ${builddir}/duds.orig } # usage: restrictedlist pb arch branch builddir [target] # note: can take ~25 minutes! restrictedlist () { pbc=$1 arch=$2 branch=$3 buildid=$4 builddir=$5 target=$6 cd ${builddir}/ports echo "================================================" echo "creating restricted list" echo "================================================" echo "restricted list generation started at $(date)" ${pbc}/scripts/makerestr ${arch} ${branch} ${buildid} ${target} || return 1 echo "restricted list generation ended at $(date)" echo $(grep -c '^#' ${builddir}/restricted.sh) "ports in ${builddir}/restricted.sh" } # usage: cdromlist pb arch branch builddir # note: can take ~48 minutes! cdromlist () { pbc=$1 arch=$2 branch=$3 builddir=$4 cd ${builddir}/ports echo "================================================" echo "creating cdrom list" echo "================================================" echo "cdrom list generation started at $(date)" make ECHO_MSG=true clean-for-cdrom-list \ | sed -e "s./usr/ports/distfiles/./distfiles/.g" \ -e "s./usr/ports/./${branch}/.g" \ > ${builddir}/cdrom.sh echo "cdrom list generation ended at $(date)" echo $(grep -c '^#' ${builddir}/cdrom.sh) "ports in ${builddir}/cdrom.sh" } # XXX Should use SHA256 instead, but I'm not sure what consumes this file (if anything) # XXX Should generate these as the packages are copied in, instead of all at once at the end # usage: generatemd5 pb arch branch builddir generatemd5 () { pbc=$1 arch=$2 branch=$3 builddir=$4 echo "started generating CHECKSUM.MD5 at $(date)" cd ${builddir}/packages/All find . -name '*.tbz' | sort | sed -e 's/^..//' | xargs md5 > CHECKSUM.MD5 echo "ended generating CHECKSUM.MD5 at $(date)" } dobuild() { pbc=$1 arch=$2 branch=$3 builddir=$4 echo "================================================" echo "building packages" echo "================================================" echo "started at $(date)" start=$(date +%s) ${pbc}/qmanager/packagebuild ${arch} ${branch} ${buildid} > ${builddir}/${journalname} 2>&1 < /dev/null result=$? if [ $result -ne 0 ]; then echo "ERROR: packagebuild ${arch} ${branch} ${buildid} failed: see ${builddir}/${journalname} for details" fi echo "ended at $(date)" end=$(date +%s) echo "Build took $(date -u -j -r $((end - start)) | awk '{print $4}')" echo $(ls -1 ${builddir}/packages/All | grep tbz | wc -l) "packages built" echo $(wc -l ${PORTSDIR}/${INDEXFILE} | awk '{print $1}') "lines in INDEX" echo $(echo $(du -sk ${builddir}/packages | awk '{print $1}') / 1024 | bc) "MB of packages" echo $(echo $(du -sk ${builddir}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles" cd ${builddir} if grep -qE '(ptimeout|pnohang): killing' ${journalname}; then echo "The following port(s) timed out:" grep -E '(ptimeout|pnohang): killing' ${journalname} | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/' fi } me=$(hostname) starttime=$(date +%s) echo "Subject: $me package building logs" echo echo "Called with arguments: $@" echo "Started at ${starttime}" nobuild=0 noindex=0 noduds=0 nosrc=0 srcvcs=0 noports=0 portsvcs=0 norestr=0 nochecksubdirs=0 noplistcheck=0 cdrom=0 restart=0 cont=0 finish=0 nofinish=0 nodistfiles=0 fetch_orig=0 trybroken=0 incremental=0 keep=0 nocleanup=0 # optional arguments while [ $# -gt 0 ]; do case "x$1" in x-nobuild) nobuild=1 ;; x-noindex) noindex=1 ;; x-noduds) noduds=1 ;; x-cdrom) cdrom=1 ;; x-nosrc) nosrc=1 ;; x-srccvs|x-srcvcs) srcvcs=1 ;; x-noports) noports=1 ;; x-portscvs|x-portsvcs) portsvcs=1 ;; x-norestr) norestr=1 ;; x-nochecksubdirs) nochecksubdirs=1 ;; x-noplistcheck) noplistcheck=1 ;; x-nodistfiles) nodistfiles=1 ;; x-fetch-original) fetch_orig=1 ;; x-trybroken) trybroken=1 ;; x-continue) cont=1 ;; x-restart) restart=1 ;; x-nofinish) nofinish=1 ;; x-finish) nobuild=1 finish=1 ;; x-incremental) incremental=1 ;; x-keep) keep=1 ;; x-nocleanup) nocleanup=1 ;; x-target) shift target=$(realpath $1) ;; *) usage ;; esac shift done if [ "$restart" = 1 -o "$cont" = 1 -o "$finish" = 1 ]; then skipstart=1 else skipstart=0 fi # XXX check for conflict between -noports and -portsvcs etc # We have valid options, start the build if [ "$nodistfiles" = 1 ]; then export NO_DISTFILES=1 fi if [ "$noplistcheck" = 1 ]; then export NOPLISTCHECK=1 fi if [ "$cdrom" = 1 ]; then export FOR_CDROM=1 fi if [ "$fetch_orig" = 1 ]; then export FETCH_ORIGINAL=1 fi if [ "$trybroken" = 1 ]; then export TRYBROKEN=1 fi builddir=${pbab}/builds/${buildid} # bomb out if there are no bindist files if [ ! -f ${builddir}/bindist.tbz -o ! -f ${builddir}/bindist.tbz.md5 ]; then echo "missing bindist.tbz and/or bindist.tbz.md5; exiting script at $(date)." mailexit 1 fi # Start setting up build environment if [ "${skipstart}" -eq 0 ]; then newbuildid=${datestamp} # this is where the latest/previous dance is performed # MCL note 20091109: buildid must exist. For now, use the following # MCL manual command to start new buildenvs, before the first use of # MCL dopackages: "build create arch branch" build clone ${arch} ${branch} ${buildid} ${newbuildid} buildid=${newbuildid} builddir=${pbab}/builds/${buildid} fi # bomb out if build clone failed if [ ! -d ${builddir} ]; then mailexit 1 fi # Set up our environment variables buildenv ${pbd} ${arch} ${branch} ${builddir} # XXX MCL might not return 'latest' ??? echo | mail -s "$(basename $0) started for ${arch}-${branch} ${buildid} at $(date)" ${mailto} # make necessary subdirectories if they don't exist mkdir -p ${builddir}/bak/restricted || mailexit 1 if [ "${keep}" -eq 1 ]; then touch ${builddir}/.keep fi # Mark as active so that it is not automatically cleaned up on the # clients touch ${builddir}/.active # Update link to current logfile created by dopackages.wrapper ln -sf ${pbd}/${arch}/archive/buildlogs/log.${branch}.${datestamp} \ ${builddir}/build.log +# Update build-specific portbuild.conf. +if [ -f ${pbd}/${arch}/${branch}/portbuild.conf ]; then + ln -sf ${pbd}/${arch}/${branch}/portbuild.conf ${builddir}/portbuild.conf + . ${builddir}/portbuild.conf +fi + if [ "$skipstart" = 0 ]; then # Update build if [ "$incremental" = 1 ]; then # Stash a copy of the index since we may be about to replace # it with the ZFS update if [ -f ${PORTSDIR}/${INDEXFILE} ]; then cp ${PORTSDIR}/${INDEXFILE} ${builddir}/bak/${INDEXFILE} fi fi if [ ${noports} -eq 0 ]; then if [ -L ${builddir}/ports -o ${portsvcs} -eq 1 ]; then echo "================================================" echo "updating ${PORTSDIR} from ${VCS}" echo "================================================" cd ${PORTSDIR} updated=$(date '+%Y/%m/%d %H:%M') echo ${updated} > ${builddir}/.updated ${VCS} ${VCS_UPDATE_DATE} "${updated}" # XXX Check for conflicts else # echo "XXX at build portsupdate portsupdate ${arch} ${branch} ${buildid} $@ " build portsupdate ${arch} ${branch} ${buildid} $@ # echo "XXX past build portsupdate portsupdate ${arch} ${branch} ${buildid} $@ " fi else # XXX MCL why??? # XXX rm -f ${builddir}/.updated fi if [ "$incremental" = 1 ]; then if [ -f ${builddir}/bak/${INDEXFILE} ]; then cp ${builddir}/bak/${INDEXFILE} ${PORTSDIR}/${INDEXFILE}.old fi fi # Create tarballs for distributing to clients. Should not cause # much extra delay because we will do this in conjunction with # recursing over the ports tree anyway just below, and might have # just finished vcs updating, so it is likely to be in cache. portstar & if [ ${nosrc} -eq 0 ]; then if [ -L ${builddir}/src -o ${srcvcs} -eq 1 ]; then echo "================================================" echo "updating ${SRC_BASE} from ${VCS}" echo "================================================" cd ${SRC_BASE} if [ -z "${updated}" ]; then # Don't overwrite/create .updated if we didn't set it # with the ports update updated=$(date) fi ${VCS} ${VCS_UPDATE_ARGS} "${updated}" # XXX Check for conflicts else build srcupdate ${arch} ${branch} ${buildid} $@ fi fi srctar & # Begin build preprocess cd ${PORTSDIR} if [ "$nochecksubdirs" = 0 ]; then echo "================================================" echo "running make checksubdirs" echo "================================================" make checksubdirs fi # XXX MCL could background these? # not run in background to check return status if [ "$noindex" = 0 ]; then makeindex ${pbc} ${arch} ${branch} ${buildid} ${builddir} ${target} || mailexit 1 fi checkindex ${builddir} || mailexit 1 if [ "$noduds" = 0 ]; then makeduds ${pbc} ${arch} ${branch} ${buildid} ${builddir} ${target} || mailexit 1 fi wait # for tar creation if [ "$trybroken" = 1 ]; then echo "================================================" echo "pruning stale entries from the failed ports list" echo "================================================" # XXX failure and newfailure are arch/branch-global for now. We # will need to work out how to deal with updates from # concurrent builds though (one build may fail after a more # recent build has fixed the breakage) if [ -f ${pbab}/failure ]; then cp ${pbab}/failure ${builddir}/bak/ fi if [ -f ${pbab}/newfailure ]; then cp ${pbab}/newfailure ${builddir}/bak/ fi lockf -k ${pbab}/failure.lock ${pbc}/scripts/prunefailure ${arch} ${branch} ${builddir} fi # XXX These can happen after build start if [ "$norestr" = 0 ]; then restrictedlist ${pbc} ${arch} ${branch} ${buildid} ${builddir} ${target} & job_restrictedlist=$! fi if [ "$cdrom" = 1 ]; then cdromlist ${pbc} ${arch} ${branch} ${builddir} & job_cdromlist=$! fi cd ${builddir} if [ -d distfiles ]; then mv distfiles .distfiles~ rm -rf .distfiles~ & fi mkdir -p distfiles/ olderrors=$(readlink ${builddir}/errors) oldlogs=$(readlink ${builddir}/logs) # XXX MCL hardcoding of archive/errorlogs newerrors=${pbd}/${arch}/archive/errorlogs/e.${branch}.${buildid} newlogs=${pbd}/${arch}/archive/errorlogs/a.${branch}.${buildid} # Cycle out the previous symlinks rm -f bak/errors rm -f bak/logs if [ -e errors ]; then mv errors bak/ fi if [ -e logs ]; then mv logs bak/ fi # Create new log directories for archival rm -rf ${newerrors} mkdir -p ${newerrors} ln -sf ${newerrors} ${builddir}/errors rm -rf ${newlogs} mkdir -p ${newlogs} ln -sf ${newlogs} ${builddir}/logs echo "error logs in ${newerrors}" if [ -f "${builddir}/.updated" ]; then cp -p ${builddir}/.updated ${newerrors}/.updated cp -p ${builddir}/.updated ${newlogs}/.updated else rm -f ${newerrors}/.updated ${newlogs}/.updated fi cp -p ${builddir}/duds ${newerrors}/duds cp -p ${builddir}/duds ${newlogs}/duds if [ -f "${builddir}/duds.verbose" ]; then cp -p ${builddir}/duds.verbose ${newerrors}/duds.verbose cp -p ${builddir}/duds.verbose ${newlogs}/duds.verbose fi cp -p ${builddir}/ports/${INDEXFILE} ${newerrors}/INDEX cp -p ${builddir}/ports/${INDEXFILE} ${newlogs}/INDEX if [ "$incremental" = 1 ]; then # Copy back in the restricted packages that were saved after the # previous build if [ -d ${builddir}/bak/restricted/ ]; then cd ${builddir}/bak/restricted find . | cpio -dumpl ${builddir} fi cd ${builddir} # Create hardlinks to previous set of logs if [ ! -z "${oldlogs}" -a -d ${oldlogs} ]; then cd ${oldlogs} && find . -name \*.log\* | cpio -dumpl ${newlogs} fi if [ ! -z "${olderrors}" -a -d ${olderrors} ]; then cd ${olderrors} && find . -name \*.log\* | cpio -dumpl ${newerrors} fi # Identify the ports that have changed and thus whose packages # need to be removed before rebuilding cd ${PORTSDIR} if [ -f ${INDEXFILE}.old ]; then cut -f 1,2,3,8,9,11,12,13 -d \| ${INDEXFILE}.old | sort > ${INDEXFILE}.old1 cut -f 1,2,3,8,9,11,12,13 -d \| ${INDEXFILE} | sort > ${INDEXFILE}.1 comm -2 -3 ${INDEXFILE}.old1 ${INDEXFILE}.1 | cut -f 1 -d \| > ${builddir}/.oldports echo "Removing $(wc -l ${builddir}/.oldports | awk '{print $1}') packages in preparation for incremental build" rm ${INDEXFILE}.old1 ${INDEXFILE}.1 cd ${PACKAGES}/All sed "s,$,${PKGSUFFIX}," ${builddir}/.oldports | xargs rm -f # XXX MCL takes an unknown period of time. # XXX MCL return value not checked. ${pbc}/scripts/prunepkgs ${PORTSDIR}/${INDEXFILE} ${PACKAGES} cd ${builddir}/errors/ sed "s,\$,.log," ${builddir}/.oldports | xargs rm -f sed "s,\$,.log.bz2," ${builddir}/.oldports | xargs rm -f cd ${builddir}/logs/ sed 's,$,.log,' ${builddir}/.oldports | xargs rm -f sed 's,$,.log.bz2,' ${builddir}/.oldports | xargs rm -f fi else cd ${builddir} if [ -d packages ]; then # echo "XXX at mv packages .packages~" mv packages .packages~ rm -rf .packages~ & # echo "XXX past mv packages .packages~" fi mkdir -p packages/All fi wait $job_restrictedlist || mailexit 1 wait $job_cdromlist || mailexit 1 fi # if [ "$skipstart" = 0 ] # only need to wait for some tasks, so this is probably redundant. wait if [ "$nobuild" = 0 ]; then cd ${builddir} if [ "$cont" = 1 ]; then find errors/ -name \*.log | sed -e 's,\.log$,,' -e 's,^errors/,,' > duds.errors cat duds duds.errors | sort -u > duds.new mv duds.new duds else cp duds.orig duds fi # Compile ptimeout. /usr/bin/gcc -o ${builddir}/ptimeout -Wall ${pbc}/sources/ptimeout.c dobuild ${pbc} ${arch} ${branch} ${builddir} fi # Clean up temporary duds file if [ "$cont" = 1 ]; then cp duds.orig duds fi cd ${builddir}/packages/All if [ "$nofinish" = 0 ]; then if [ "$norestr" = 0 ]; then # Before deleting restricted packages, save a copy so we don't # have to rebuild them next time ${pbc}/scripts/keeprestr ${arch} ${branch} ${buildid} else rm -rf ${builddir}/bak/restricted/ fi # Always delete restricted packages/distfiles since they're # published on the website echo "deleting restricted ports" sh ${builddir}/restricted.sh if [ "$cdrom" = 1 ]; then echo "deleting cdrom restricted ports" sh ${builddir}/cdrom.sh fi # Remove packages not listed in INDEX ${pbc}/scripts/prunepkgs ${builddir}/ports/${INDEXFILE} ${builddir}/packages fi # XXX Checking for bad packages should be done after the package is uploaded #rm -rf ${builddir}/bad #mkdir -p ${builddir}/bad #echo "checking packages" #for i in *${PKGSUFFIX}; do # if ! ${PKGZIPCMD} -t $i; then # echo "Warning: package $i is bad, moving to ${builddir}/bad" # # the latest link will be left behind... # mv $i ${builddir}/bad # rm ../*/$i # fi #done if [ "$nofinish" = 0 ]; then generatemd5 ${pbc} ${arch} ${branch} ${builddir} & # Remove INDEX entries for packages that do not exist ${pbc}/scripts/chopindex ${builddir}/ports/${INDEXFILE} ${builddir}/packages > ${builddir}/packages/INDEX # Copy UPDATING and MOVED into the packages folder cp ${builddir}/ports/UPDATING ${builddir}/packages/UPDATING cp ${builddir}/ports/MOVED ${builddir}/packages/MOVED for f in INDEX MOVED UPDATING; do bzip2 -k ${builddir}/packages/$f md5 ${builddir}/packages/$f.bz2 > ${builddir}/packages/$f.bz2.md5 sha256 ${builddir}/packages/$f.bz2 > ${builddir}/packages/$f.bz2.sha256 done ls -asFlrt ${builddir}/packages/All > ${builddir}/logs/ls-lrt cp -p ${builddir}/${journalname} ${builddir}/logs echo "================================================" echo "copying distfiles" echo "================================================" echo "started at $(date)" cd ${builddir} ${pbc}/scripts/dodistfiles ${arch} ${branch} ${buildid} # Always delete restricted distfiles echo "deleting restricted distfiles" sh ${builddir}/restricted.sh if [ "$cdrom" = 1 ]; then echo "deleting cdrom restricted distfiles" sh ${builddir}/cdrom.sh fi wait fi if [ "${nocleanup}" -eq 1 ]; then echo "Not cleaning up build, when you are finished be sure to run:" echo " ${pbc}/scripts/build cleanup ${arch} ${branch} ${buildid} -full" else ${pbc}/scripts/build cleanup ${arch} ${branch} ${buildid} -full fi endtime=$(date +%s) echo "================================================" echo "all done at $(date)" echo "entire process took $(date -u -j -r $(($endtime - $starttime)) | awk '{print $4}')" echo "================================================" exit 0 Index: projects/portbuild/scripts/dosetupnode =================================================================== --- projects/portbuild/scripts/dosetupnode (revision 221760) +++ projects/portbuild/scripts/dosetupnode (revision 221761) @@ -1,199 +1,211 @@ #!/bin/sh # $FreeBSD: ports/Tools/portbuild/scripts/dosetupnode,v 1.17 2011/01/23 02:26:14 linimon Exp $ # server-side script to set up an individual client # XXX Use a worker pool that only runs N setups at once to avoid # raping the server. Hard to do in shell? # -norsync|-nocopy : Don't copy files, just clean up builds # # -force : force file copying/extraction even if it appears it is # up-to-date # # NB: branch or buildid might be "-" to specify only to set up the # scripts/ and ${arch}/ directories (e.g. after client reboot) # configurable variables pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} arch=$1 branch=$2 buildid=$3 nodelist=$4 shift 4 . ${pbc}/conf/server.conf if [ -f ${pbd}/${arch}/portbuild.conf ]; then . ${pbd}/${arch}/portbuild.conf else echo "Invalid arch ${arch}" exit 1 fi +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbc}/scripts/buildenv # Check for non-fatal rsync errors checkerror() { error=$? case $error in 0) return 0 ;; 23) echo "Continuing..." return 0 ;; *) echo "Aborting..." return 1 ;; esac } setup() { node=$1 echo "setting up of $node started at $(date)" . ${pbd}/${arch}/portbuild.conf + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf . ${pbd}/${arch}/portbuild.${node} if [ "${buildid}" != "-" ]; then rsync ${rsync_gzip} -e "${ssh_cmd}" -r -l -p --delete ${pbc}/scripts ${pbc}/sources \ ${client_user}@${node}:/tmp/${buildid}/ checkerror $? || (echo "Copying scripts to ${node} failed"; return 1) fi cmdpath=$(cat ${pbc}/scripts/setupnode | ssh -a ${client_user}@${node} 't=$(mktemp -t setupnode); cat >$t; echo $t; chmod 755 $t') case ${cmdpath} in /tmp/*) ;; *) echo "Failed to run dosetupnode on ${host}."; return 1;; esac client_setup="${ssh_cmd} -n ${client_user}@${node} ${cmdpath} ${pbd} ${arch} ${branch} ${buildid} ${scratchdir} \"${portsmd5}\" \"${srcmd5}\" \"${bindistmd5}\"" args="${nocopy} ${force}" ${client_setup} pre-copy ${args} || (echo "pre-copy for ${node} failed"; return 1) if [ "${norsync}" -eq 0 ]; then # client.conf and common.conf can be symlinks outside this dir, so # copy the actual files rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p --delete ${pbd}/${arch}/*.conf \ ${client_user}@${node}:${pbd}/${arch} checkerror $? || (echo "copying *.conf to ${node} failed"; return 1) # portbuild.* can be symlinks outside this dir, so copy the actual # files rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p --delete ${pbd}/${arch}/portbuild* \ ${client_user}@${node}:${pbd}/${arch} checkerror $? || (echo "copying portbuild* files to ${node} failed"; return 1) if [ -f "${pbd}/${arch}/clients/bindist-${node}.tar" ]; then rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p --delete \ ${pbd}/${arch}/clients/bindist-${node}.tar \ ${client_user}@${node}:${pbd}/${arch}/clients/ checkerror $? || (echo "Copying bindist-${node}.tar to ${node} failed"; return 1) else echo "Host customization file not found: ${pbd}/${arch}/clients/bindist-${node}.tar" return 1 fi if [ "${buildid}" != "-" ]; then rsync ${rsync_gzip} -e "${ssh_cmd}" -r -l -p --delete ${pbc}/scripts ${pbc}/sources \ ${client_user}@${node}:${builddir}/ checkerror $? || (echo "Copying scripts to ${node} failed"; return 1) + + if [ -f ${builddir}/portbuild.conf ]; then + rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p --delete ${builddir}/portbuild.conf \ + ${client_user}@${node}:${builddir}/ + checkerror $? || (echo "Copying custom portbuild.conf to ${node} failed"; return 1) + else + echo "No custom portbuild.conf." + fi rsync ${rsync_gzip} -e "${ssh_cmd}" -r -L -p \ ${builddir}/ports-${buildid}.tbz \ ${builddir}/ports-${buildid}.tbz.md5 \ ${builddir}/src-${buildid}.tbz \ ${builddir}/src-${buildid}.tbz.md5 \ ${builddir}/bindist.tbz \ ${builddir}/bindist.tbz.md5 \ ${client_user}@${node}:${builddir}/ checkerror $? || (echo "Copying build tarballs to ${node} failed"; return 1) fi fi ${client_setup} post-copy ${args} || (echo "post-copy for ${node} failed"; return 1) if [ "${queue}" -eq 1 ]; then jobs=$(python ${pbc}/qmanager/qclient jobs | grep "${node}" | grep "${arch}/${branch}/${buildid} package" | awk '{print $1}' | tail +1) for j in ${jobs}; do python ${pbc}/qmanager/qclient release $j done fi if [ "${full}" -eq 1 ]; then ${ssh_cmd} ${client_user}@${node} ${sudo_cmd} rm -rf ${pbd}/${arch}/${branch}/builds/${buildid}/.ready ${pbd}/${arch}/${branch}/builds/${buildid} /tmp/.setup-${buildid} fi echo "setting up of $node ended at $(date)" } pbab=${pbd}/${arch}/${branch} norsync=0 queue=0 full=0 while [ $# -ge 1 ]; do case $1 in -norsync|-nocopy) norsync=1 nocopy=-nocopy ;; -queue) queue=1 ;; -force) force=-force ;; -full) full=1 ;; esac shift done if [ "${norsync}" -eq 0 ]; then if [ "${branch}" != "-" -a "${buildid}" != "-" ]; then buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 fi builddir=${pbab}/builds/${buildid} if [ ! -f ${builddir}/ports-${buildid}.tbz.md5 ]; then echo "ports-${buildid}.tbz.md5 not found" exit 1 else portsmd5=$(awk '{print $4}' ${builddir}/ports-${buildid}.tbz.md5) fi if [ ! -f ${builddir}/src-${buildid}.tbz.md5 ]; then echo "src-${buildid}.tbz.md5 not found" exit 1 else srcmd5=$(awk '{print $4}' ${builddir}/src-${buildid}.tbz.md5) fi if [ ! -f ${builddir}/bindist.tbz.md5 ]; then echo "bindist.tbz.md5 not found" exit 1 else bindistmd5=$(awk '{print $4}' ${builddir}/bindist.tbz.md5) fi fi fi if [ "${nodelist}" = "all" ]; then nodelist=$(cat ${pbd}/${arch}/mlist) fi for node in ${nodelist}; do setup ${node} & done wait Index: projects/portbuild/scripts/makeduds =================================================================== --- projects/portbuild/scripts/makeduds (revision 221760) +++ projects/portbuild/scripts/makeduds (revision 221761) @@ -1,68 +1,71 @@ #!/bin/sh usage () { echo "usage: makeduds arch branch buildid [target]" exit 1 } if [ $# -lt 3 ]; then usage fi # configurable variables pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} arch=$1 branch=$2 buildid=$3 target=$4 builddir=${pbd}/${arch}/${branch}/builds/${buildid} . ${pbc}/conf/server.conf . ${pbc}/conf/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbc}/scripts/buildenv # -j# to make duds DUDSJOBS=4 buildenv ${pbd} ${arch} ${branch} ${builddir} duds=${builddir}/duds index=${PORTSDIR}/${INDEXFILE} unset DISPLAY export __MAKE_SHELL=/rescue/sh export LOCALBASE=/nonexistentlocal export LINUXBASE=/nonexistentlinux export PKG_DBDIR=/nonexistentpkg export PORT_DBDIR=/nonexistentport DUDS_PORTS=. if [ -n "$target" ]; then if [ -f "$target" ]; then DUDS_PORTS=$(cat $target | tr '\n' ' ') else echo "File $target doesn't exist. Exiting." exit 1 fi fi cd ${PORTSDIR} (for i in ${DUDS_PORTS}; do cd ${PORTSDIR}/$i make -j${DUDSJOBS} ignorelist-verbose ECHO_MSG=true done) > ${duds}.verbose 2> /dev/null || exit 1 sort ${duds}.verbose > ${duds}.verbose.tmp mv -f ${duds}.verbose.tmp ${duds}.verbose cut -f 1 -d \| ${duds}.verbose > ${duds} cp ${duds} ${duds}.orig grep -Ff ${duds}.orig ${index} | cut -f 1 -d \| > ${duds}.full cat ${duds} ${duds}.full | sort | uniq -u | sed -e "s@\$@|IGNORE: dependent port@" > ${duds}.full.verbose.tmp cat ${duds}.verbose ${duds}.full.verbose.tmp | sort > ${duds}.full.verbose rm ${duds}.full.verbose.tmp Index: projects/portbuild/scripts/makeindex =================================================================== --- projects/portbuild/scripts/makeindex (revision 221760) +++ projects/portbuild/scripts/makeindex (revision 221761) @@ -1,66 +1,69 @@ #!/bin/sh # usage: $0 arch branch buildid # Don't want to pick up host customizations export INDEX_PRISTINE=1 # Don't give luser advice if it fails export INDEX_QUIET=1 # Concurrency of index build export INDEX_JOBS=6 # For debugging purposes only #export INDEX_VERBOSE=1 pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} usage () { echo "usage: makeindex arch branch buildid [target]" exit 1 } if [ $# -lt 3 ]; then usage fi arch=$1 branch=$2 buildid=$3 target=$4 builddir=${pbd}/${arch}/${branch}/builds/${buildid} . ${pbc}/conf/server.conf . ${pbc}/conf/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbc}/scripts/buildenv # Set up the build env variables buildenv ${pbd} ${arch} ${branch} ${builddir} unset DISPLAY # Don't pick up installed packages from the host export LOCALBASE=/nonexistentlocal if [ -n "$target" ]; then if [ -f "$target" ]; then INDEX_PORTS=$(cat $target | tr '\n' ' ') else echo "File $target doesn't exist. Exiting." exit 1 fi fi cd ${PORTSDIR} make index INDEX_PORTS="${INDEX_PORTS}" if [ ! -e ${INDEXFILE} ]; then echo "makeindex: failed to make ${INDEXFILE}" exit 1 fi # remove extra spaces in dependency list -- this causes problems # Also transform the dummy paths to their canonical locations sed -i '' -e 's/ */ /g' -e 's/| */|/g' -e 's/ *|/|/g' -e "s,${LOCALBASE},/usr/local," ${INDEXFILE} Index: projects/portbuild/scripts/makerestr =================================================================== --- projects/portbuild/scripts/makerestr (revision 221760) +++ projects/portbuild/scripts/makerestr (revision 221761) @@ -1,53 +1,56 @@ #!/bin/sh usage () { echo "usage: makerestr arch branch buildid [target]" exit 1 } if [ $# -lt 3 ]; then usage fi # configurable variables pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} arch=$1 branch=$2 buildid=$3 target=$4 . ${pbc}/conf/server.conf . ${pbc}/conf/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbc}/scripts/buildenv builddir=${pbd}/${arch}/${branch}/builds/${buildid} buildenv ${pbd} ${arch} ${branch} ${builddir} unset DISPLAY export __MAKE_SHELL=/rescue/sh export LOCALBASE=/nonexistentlocal export LINUXBASE=/nonexistentlinux export PKG_DBDIR=/nonexistentpkg export PORT_DBDIR=/nonexistentport RESTR_PORTS=. if [ -n "$target" ]; then if [ -f "$target" ]; then RESTR_PORTS=$(cat $target | tr '\n' ' ') else echo "File $target doesn't exist. Exiting." exit 1 fi fi cd ${PORTSDIR} (for i in ${RESTR_PORTS}; do cd ${PORTSDIR}/$i make -j4 ECHO_MSG=true clean-restricted-list \ | sed -e "s!/usr/ports/packages/!${builddir}/packages/!g" \ -e "s!/usr/ports/!${builddir}/ports/!g" done) > ${builddir}/restricted.sh 2> /dev/null Index: projects/portbuild/scripts/makeworld =================================================================== --- projects/portbuild/scripts/makeworld (revision 221760) +++ projects/portbuild/scripts/makeworld (revision 221761) @@ -1,88 +1,91 @@ #!/bin/sh # # XXX lockfile and interlock with mkbindist to avoid overlapping # builds if [ $# -lt 3 ]; then echo "usage: makeworld arch branch buildid [args]" exit 1 fi arch=$1 branch=$2 buildid=$3 shift 3 pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} builddir=${pbd}/${arch}/${branch}/builds/${buildid} . ${pbc}/conf/server.conf . ${pbc}/conf/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi # NB: we can't use buildenv because it sets ARCH and MACHINE_ARCH that # confuses cross-builds export TARGET_ARCH=${arch} # Workaround needed for zfs - 20090321 erwin export NO_FSCHG=1 client=0 novcs=0 # optional arguments while [ $# -gt 0 ]; do case "$1" in -client) client=1 ;; -nocvs|-novcs) novcs=1 ;; *) args="$1 ${args}" ;; esac shift done # XXX MCL I don't know what this is supposed to do. if [ "$client" = "1" ]; then SRC_BASE=${pbd}/${arch}/src-client shift 1 else SRC_BASE=${builddir}/src export __MAKE_CONF=/dev/null fi cd ${SRC_BASE} if [ "$novcs" = "0" ]; then echo "==> Updating source tree" eval tag=\$SRC_BRANCH_${branch}_TAG ${VCS} ${VCS_UPDATE_TAG} ${tag} || exit $? fi echo "==> Starting make buildworld" make buildworld ${args} || exit $? echo "==> Cleaning up destdir" destdir=${WORLDDIR}/${arch}/${branch} rm -rf ${destdir}/ chflags -R noschg ${destdir}/ rm -rf ${destdir}/ mkdir -p ${destdir} || exit $? echo "==> Starting make installworld" if [ "$client" = "0" ]; then export NEWSPARC_TIMETYPE=__int64_t make installworld DESTDIR=${destdir} || exit $? echo "==> Starting make distribute" make DESTDIR=${destdir} distrib-dirs && \ make DESTDIR=${destdir} distribution || exit $? else echo "==> Not doing installworld of client source tree" fi Index: projects/portbuild/scripts/mkbindist =================================================================== --- projects/portbuild/scripts/mkbindist (revision 221760) +++ projects/portbuild/scripts/mkbindist (revision 221761) @@ -1,85 +1,88 @@ #!/bin/sh # XXX merge with makeworld? usage () { echo "usage: mkbindist " exit 1 } cleandir() { dir=$1 rm -rf ${dir} 2>/dev/null if [ -d ${dir} ]; then chflags -R noschg ${dir} rm -rf ${dir} fi } if [ $# -lt 3 ]; then usage fi arch=$1 branch=$2 buildid=$3 shift 3 pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} . ${pbc}/conf/server.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbc}/scripts/buildenv if ! validate_env ${arch} ${branch}; then echo "Invalid build environment ${arch}/${branch}" exit 1 fi here=${pbd}/${arch}/${branch}/builds/${buildid} if [ ! -d ${here} ]; then echo "Invalid build ID ${buildid}" exit 1 fi tmpdir=${here}/bindist/tmp # Clean up ${tmpdir} cleandir ${tmpdir} mkdir -p ${tmpdir} # Copy the files into the tmpdir from an existing built world destdir=${WORLDDIR}/${arch}/${branch} cd ${destdir}; find -dx . | \ grep -v -E '^./usr/(local|obj|opt|ports|src)' | \ grep -v '^./home' | \ grep -v '^./var/db/pkg' | \ cpio -dump ${tmpdir} cd ${tmpdir} # Customize the tmpdir if [ -s "${here}/bindist/delete" ]; then sed -e "s,^,${tmpdir}," ${here}/bindist/delete | xargs rm -rf fi if [ -s "${here}/bindist/dirlist" ]; then cat "${here}/bindist/dirlist" | xargs mkdir -p fi # XXX MCL seems to be obsoleted by individual files in clients/? if [ -d ${here}/bindist/files ]; then cd ${here}/bindist/files; find -dx . | cpio -dump ${tmpdir} fi # Post-processing of installed world date '+%Y%m%d' > ${tmpdir}/var/db/port.mkversion # Create the tarball tar cfCj ${here}/.bindist.tbz ${tmpdir} . mv -f ${here}/.bindist.tbz ${here}/bindist.tbz md5 ${here}/bindist.tbz > ${here}/bindist.tbz.md5 # Clean up cd ${here} cleandir ${tmpdir} Index: projects/portbuild/scripts/pdispatch =================================================================== --- projects/portbuild/scripts/pdispatch (revision 221760) +++ projects/portbuild/scripts/pdispatch (revision 221761) @@ -1,210 +1,213 @@ #!/bin/sh # $FreeBSD: ports/Tools/portbuild/scripts/pdispatch,v 1.40 2011/01/26 10:41:53 linimon Exp $ # # pdispatch [ ...] # # server-side script to dispatch the job to a host via the ptimeout script. pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} arch=$1 branch=$2 buildid=$3 host=$4 command=$5 shift 5 pbab=${pbd}/${arch}/${branch} . ${pbc}/conf/server.conf . ${pbc}/conf/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbc}/scripts/buildenv timeout=${PDISPATCH_TIMEOUT} loglength=${PDISPATCH_LOGLENGTH} hdrlength=${PDISPATCH_HDRLENGTH} buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid}) if [ -z "${buildid}" ]; then echo "Invalid build ID ${buildid}" exit 1 fi builddir=${pbab}/builds/${buildid} buildenv ${pbd} ${arch} ${branch} ${builddir} # XXX needed still? unset DISPLAY # Allow override by HPN-SSH for performance if [ -z "${ssh_cmd}" ]; then ssh_cmd=ssh fi if [ -z "${scp_cmd}" ]; then scp_cmd=scp fi pkgname=$(basename $1 ${PKGSUFFIX}) if [ -z "${pkgname}" ]; then echo "null packagename" exit 1 fi args=${1+"$@"} flags="" clean=1 if [ "x$NOCLEAN" != "x" ]; then flags="${flags} -noclean" clean=0 fi if [ "x$NO_RESTRICTED" != "x" ]; then flags="${flags} -norestr" fi if [ "x$NOPLISTCHECK" != "x" ]; then flags="${flags} -noplistcheck" fi if [ "x$NO_DISTFILES" = "x" ]; then flags="${flags} -distfiles" fi if [ "x$FETCH_ORIGINAL" != "x" ]; then flags="${flags} -fetch-original" fi if [ "x$TRYBROKEN" != "x" ]; then flags="${flags} -trybroken" fi chroot= . ${pbd}/${arch}/portbuild.conf test -f ${pbd}/${arch}/portbuild.${host} && . ${pbd}/${arch}/portbuild.${host} # Upload scripts/claim-chroot as per-build scripts aren't in place yet. cmdpath=$(cat ${pbc}/scripts/claim-chroot | ssh -a ${client_user}@${host} 't=$(mktemp -t claim-chroot); cat >$t; echo $t; chmod 755 $t') case ${cmdpath} in /tmp/*) ;; *) echo "Failed to scp claim-chroot to ${host}."; exit 254;; esac chrootdata=$(${ssh_cmd} -a -n ${client_user}@${host} ${sudo_cmd} ${cmdpath} ${arch} ${branch} ${buildid} ${pkgname} 2>&1) ${ssh_cmd} -a ${client_user}@${host} "rm -f ${cmdpath}" if [ -z "${chrootdata}" ]; then echo "Failed to claim chroot on ${host}" exit 254 fi case "${chrootdata}" in *${cmdpath}*) # Error executing script, assume system is booting chrootdata="wait boot" ;; esac # echo "Got ${chrootdata} from ${host}" set -- ${chrootdata} if [ $# -ge 2 ]; then case $1 in chroot) chroot=$2 ;; setup) echo "Setting up ${arch}/${branch} build ID ${buildid} on ${host}" # Run in the background so we can potentially # claim a slot on another machine. In # practise I think we often end up trying # again on the same machine though. # Make sure to close stdin/stderr in the child # or make will hang until the child process # exits ${pbc}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${host} > /tmp/setupnode.$$ 2>&1 & exit 253 ;; error) echo "Error reported by ${host}: $2" ;; wait) echo "Waiting for setup of ${host} to finish" ;; esac shift 2 fi if [ -z "${chroot}" ]; then exit 254 fi . ${pbd}/${arch}/portbuild.conf test -f ${pbd}/${arch}/portbuild.${host} && . ${pbd}/${arch}/portbuild.${host} rm -f ${builddir}/logs/${pkgname}.log ${builddir}/logs/${pkgname}.log.bz2 rm -f ${builddir}/errors/${pkgname}.log ${builddir}/errors/${pkgname}.log.bz2 ${builddir}/ptimeout $timeout ${ssh_cmd} -a -n ${client_user}@${host} ${sudo_cmd} ${command} ${arch} ${branch} ${buildid} ${chroot} ${flags} \"$ED\" \"$PD\" \"$FD\" \"$BD\" \"$RD\" ${args} 2>&1 error=$? # Pull in the results of the build from the client ${scp_cmd} ${client_user}@${host}:${chroot}/tmp/${pkgname}.log ${builddir}/logs/${pkgname}.log (${ssh_cmd} -a -n ${client_user}@${host} test -f ${chroot}/tmp/work.tbz ) && ${scp_cmd} ${client_user}@${host}:${chroot}/tmp/work.tbz ${builddir}/wrkdirs/${pkgname}.tbz # XXX Set dirty flag if any of the scp's fail mkdir -p ${builddir}/distfiles/.pbtmp/${pkgname} ${ssh_cmd} -a -n ${client_user}@${host} tar -C ${chroot}/tmp/distfiles --exclude ${chroot}/tmp/distfiles/RESTRICTED -cf - . | \ tar --unlink -C ${builddir}/distfiles/.pbtmp/${pkgname} -xvf - && \ touch ${builddir}/distfiles/.pbtmp/${pkgname}/.done if [ "${error}" = 0 ]; then ${ssh_cmd} -a -n ${client_user}@${host} tar -C ${chroot}/tmp -cf - packages | \ tar --unlink -C ${builddir} -xvf - # XXX why is this needed? test -f ${builddir}/packages/All/${pkgname}${PKGSUFFIX} && \ touch ${builddir}/packages/All/${pkgname}${PKGSUFFIX} if [ -f ${builddir}/errors/${pkgname}.log ]; then rm -f ${builddir}/errors/${pkgname}.log # Force rebuild of html page to remove this package from list touch ${builddir}/errors/.force fi lockf -k ${pbab}/failure.lock ${pbc}/scripts/buildsuccess ${arch} ${branch} ${buildid} ${pkgname} log=${builddir}/logs/$pkgname.log if grep -q "even though it is marked BROKEN" ${log}; then echo | mail -s "${pkgname} BROKEN but built on ${arch} ${branch}" ${mailto} fi if grep -q "^list of .*file" ${log}; then buildlogdir=$(realpath ${builddir}/logs/) baselogdir=$(basename ${buildlogdir}) (sed -e '/^build started/,$d' $log;echo;echo "For the full build log, see"; echo; echo " http://${MASTER_URL}/errorlogs/${arch}-errorlogs/${baselogdir}/$(basename $log)";echo;sed -e '1,/^=== Checking filesystem state/d' $log) | mail -s "${pkgname} pkg-plist errors on ${arch} ${branch}" ${mailto} fi else log=${builddir}/errors/${pkgname}.log ${scp_cmd} ${client_user}@${host}:${chroot}/tmp/${pkgname}.log ${log} result=$? if [ $result -ne 0 ]; then (echo ${chroot}@${host}; ${ssh_cmd} -a -n ${client_user}@${host} ls -laR ${chroot}/tmp) | mail -s "${pkgname} logfile not found" ${mailto} else if ! grep -q "even though it is marked BROKEN" ${log}; then buildlogdir=$(realpath ${builddir}/logs/) baselogdir=$(basename ${buildlogdir}) if [ $(wc -l ${log} | awk '{print $1}') -le $((loglength + hdrlength)) ]; then (echo "You can also find this build log at"; echo; echo " http://${MASTER_URL}/errorlogs/${arch}-errorlogs/${baselogdir}/$(basename $log)";echo;cat ${log}) | mail -s "${pkgname} failed on ${arch} ${branch}" ${mailto} else (echo "Excerpt from the build log at"; echo; echo " http://${MASTER_URL}/errorlogs/${arch}-errorlogs/${baselogdir}/$(basename $log)";echo;sed -e '/^build started/,$d' $log;echo;echo " [... lines trimmed ...]";echo;tail -${loglength} ${log}) | mail -s "${pkgname} failed on ${arch} ${branch}" ${mailto} fi fi lockf -k ${pbab}/failure.lock ${pbc}/scripts/buildfailure ${arch} ${branch} ${buildid} ${pkgname} fi fi ${ssh_cmd} -a -n ${client_user}@${host} ${sudo_cmd} /tmp/${buildid}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${chroot} ${clean} # XXX Set a dirty variable earlier and check here if grep -q "^build of .*ended at" ${builddir}/logs/${pkgname}.log; then exit ${error} else echo "Build of ${pkgname} in ${host}:/${chroot} failed uncleanly" exit 255 fi Index: projects/portbuild/scripts/portbuild =================================================================== --- projects/portbuild/scripts/portbuild (revision 221760) +++ projects/portbuild/scripts/portbuild (revision 221761) @@ -1,345 +1,348 @@ #!/bin/sh # $FreeBSD: ports/Tools/portbuild/scripts/portbuild,v 1.65 2011/01/23 02:39:54 linimon Exp $ # client-side script to do all the work surrounding an individual package # build, and then the package build itself # note: unredirected 'echo' output goes to the journal file # usage: $0 ARCH BRANCH BUILDID CHROOT [-noclean] [-norestr] [-noplistcheck] [-distfiles] [-fetch-original] [-trybroken] PKGNAME.tgz DIRNAME [DEPENDENCY.tgz ...] pbd=${PORTBUILD_DATA:-/var/portbuild} mount_fs() { fs=$1 mntpt=$2 master=$3 if [ ${disconnected} = 1 ]; then mount -t nullfs -r ${fs} ${mntpt} else mount_nfs -o ro -3 -i ${master}:${fs} ${mntpt} fi } copypkg() { pbd=$1 host=$2 from=$3 to=$4 http_proxy=$5 if [ ${host} = $(hostname) ]; then cp ${pbd}/${arch}/${branch}/packages/All/${from} ${to} else if [ ! -z "${http_proxy}" ]; then env HTTP_PROXY=${http_proxy} fetch -m -o ${to} http://${host}/errorlogs/${arch}-${branch}-packages-latest/All/${from} else fetch -m -o ${to} http://${host}/errorlogs/${arch}-${branch}-packages-latest/All/${from} fi fi } bailout() { chroot=$1 clean=$2 error=$3 pkgname=$4 echo -n "$pkgname failed unexpectedly on $(hostname) at " date exit $error } arch=$1 branch=$2 buildid=$3 chroot=$4 shift 4 # Default niceness value nice=0 . ${pbd}/${arch}/client.conf . ${pbd}/${arch}/common.conf # note: should NOT need anything from server.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbd}/${arch}/portbuild.$(hostname) . ${pbd}/scripts/buildenv buildroot=${scratchdir} error=0 clean=1 if [ "x$1" = "x-noclean" ]; then clean=0 shift fi norestr=0 if [ "x$1" = "x-norestr" ]; then norestr=1 # consumed by bsd.port.mk export NO_RESTRICTED=1 shift fi noplistcheck=0 if [ "x$1" = "x-noplistcheck" ]; then noplistcheck=1 # consumed by buildscript directly export NOPLISTCHECK=1 shift fi nodistfiles=1 if [ "x$1" = "x-distfiles" ]; then # consumed by buildscript via make(1) export ALWAYS_KEEP_DISTFILES=1 nodistfiles=0 shift fi if [ "x$1" = "x-fetch-original" ]; then # consumed by buildscript via make(1) export FETCH_ORIGINAL=1 shift fi if [ "x$1" = "x-trybroken" ]; then # consumed by bsd.port.mk export TRYBROKEN=1 shift fi ED=$1 PD=$2 FD=$3 BD=$4 RD=$5 builddir=${pbd}/${arch}/${branch}/builds/${buildid} buildenv.common # Want to use the /etc/make.conf in the chroot unset __MAKE_CONF # set overrides for make.conf export BACKUP_FTP_SITE=${CLIENT_BACKUP_FTP_SITE} pkgname=$(basename $6 ${PKGSUFFIX}) dirname=$7 shift 2 echo $pkgname echo $dirname # set overrides for bsd.port.mk variables export WRKDIRPREFIX=${CLIENT_WRKDIRPREFIX} export DISTDIR=${CLIENT_DISTDIR} export LOCALBASE=${LOCALBASE} export PACKAGES=${CLIENT_PACKAGES_LOCATION} export SRC_BASE=${CLIENT_SRCBASE} # to catch missing dependencies #export DEPENDS_TARGET=/usr/bin/true # don't pass -j, -k etc. to sub-makes unset MAKEFLAGS unset PORTSDIR # wait 2 hours before killing build with no output export BUILD_TIMEOUT=${CLIENT_BUILD_TIMEOUT} # prevent runaway processes ulimit -f ${CLIENT_ULIMIT_F} ulimit -t ${CLIENT_ULIMIT_T} # directories to clean cleandirs="${LOCALBASE} /compat /var/db/pkg" export FTP_TIMEOUT=${CLIENT_FTP_TIMEOUT} export HTTP_TIMEOUT=${CLIENT_HTTP_TIMEOUT} export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${LOCALBASE}/sbin:${LOCALBASE}/bin export MALLOC_OPTIONS=${CLIENT_MALLOC_OPTIONS} echo "building ${pkgname} in ${chroot}" bindist=${buildroot}/${branch}/${buildid}/tarballs/bindist.tar bindistlocal=${buildroot}/${branch}/${buildid}/tarballs/bindist-$(hostname).tar if [ -f ${chroot}/.notready ]; then tar -C ${chroot} -xpf ${bindist} if [ -f ${bindistlocal} ]; then tar -C ${chroot} -xpf ${bindistlocal} fi # to be able to run certain kernel-dependent binaries # inside the chroot area cp -p /rescue/mount /rescue/umount ${chroot}/sbin cp -p /rescue/ps ${chroot}/bin rm ${chroot}/.notready touch ${chroot}/.ready fi if [ "${use_jail}" = "1" ]; then # Figure out jail IP addr chrootpid=$(basename ${chroot}) ipbase=$((${chrootpid}+2)) ip1=$(($ipbase /(256*256))) ip2=$((($ipbase - ($ip1*256*256)) /256)) ip3=$((($ipbase - ($ip1*256*256) - ($ip2*256)))) fi trap "bailout ${chroot} ${clean} ${error} ${pkgname}" 1 2 3 9 10 11 15 rm -rf ${chroot}/tmp/* cd ${chroot}/tmp mkdir -p depends distfiles packages echo "building ${pkgname} on $(hostname)" | tee ${chroot}/tmp/${pkgname}.log echo "in directory ${chroot}" | tee -a ${chroot}/tmp/${pkgname}.log # intentionally set up ${PORTSDIR} with symlink to catch broken ports mkdir -p ${chroot}/a/ports rm -rf ${chroot}/usr/ports # Don't build in a world-writable standard directory because some ports # hardcode this path and try to load things from it at runtime, which is # bad for user security rm -rf ${chroot}/${WRKDIRPREFIX} mkdir -p ${chroot}/${WRKDIRPREFIX} # pick up value from /portbuild.conf if [ ! -z "${ccache_dir}" ]; then mkdir -p ${chroot}/root/.ccache/ if [ "${ccache_dir_nfs}" = "1" ]; then mount_nfs -o rw -T -3 ${ccache_dir} ${chroot}/root/.ccache/ else mount -o rw -t nullfs ${ccache_dir} ${chroot}/root/.ccache/ fi fi mount_fs ${builddir}/ports ${chroot}/a/ports ${CLIENT_NFS_MASTER} ln -sf ../a/ports ${chroot}/usr/ports mkdir -p ${chroot}/usr/src mount_fs ${builddir}/src ${chroot}${CLIENT_SRCBASE} ${CLIENT_NFS_MASTER} # set overrides for uname buildenv.client ${chroot}${CLIENT_SRCBASE} mount -t devfs foo ${chroot}/dev umount -f ${chroot}/compat/linux/proc > /dev/null 2>&1 # just in case... for dir in ${cleandirs}; do if ! rm -rf ${chroot}${dir} >/dev/null 2>&1; then chflags -R noschg ${chroot}${dir} rm -rf ${chroot}${dir} >/dev/null 2>&1 fi done rm -rf ${chroot}/var/db/pkg/* mtree -deU -f ${chroot}/usr/src/etc/mtree/BSD.root.dist -p ${chroot} \ >/dev/null 2>&1 mtree -deU -f ${chroot}/usr/src/etc/mtree/BSD.var.dist -p ${chroot}/var \ >/dev/null 2>&1 mtree -deU -f ${chroot}/usr/src/etc/mtree/BSD.usr.dist -p ${chroot}/usr \ >/dev/null 2>&1 mkdir -p ${chroot}${LOCALBASE} mtree -deU -f ${chroot}/a/ports/Templates/BSD.local.dist -p ${chroot}${LOCALBASE} \ >/dev/null 2>&1 for i in ${ARCHS_REQUIRING_LINPROCFS}; do if [ ${i} = ${arch} ]; then # JDK ports need linprocfs :( mkdir -p ${chroot}/compat/linux/proc mount -t linprocfs linprocfs ${chroot}/compat/linux/proc break fi done _ldconfig_dirs="/lib /usr/lib /usr/lib/compat" ldconfig_dirs="" for i in ${_ldconfig_dirs}; do if [ -d ${chroot}/${i} ]; then ldconfig_dirs="${ldconfig_dirs} ${i}" fi done chroot ${chroot} /sbin/ldconfig ${ldconfig_dirs} for i in ${ARCHS_REQUIRING_AOUT_COMPAT}; do if [ ${i} = ${arch} ]; then chroot ${chroot} /sbin/ldconfig -aout /usr/lib/aout /usr/lib/compat/aout break fi done set x $ED $FD $PD $BD $RD shift 1 while [ $# -gt 0 ]; do # XXX MCL more hard-coding if [ ! -f ${chroot}/tmp/depends/$1 ]; then echo "copying package $1 for ${pkgname}" copypkg ${pbd} ${CLIENT_UPLOAD_HOST} $1 ${chroot}/tmp/depends "${http_proxy}" # Test for copy failure and bail # XXX MCL more hard-coding if [ ! -f ${chroot}/tmp/depends/$1 ]; then echo "ERROR: Couldn't copy $1" | tee -a ${chroot}/tmp/${pkgname}.log bailout ${chroot} ${clean} 255 ${pkgname} fi fi shift done cp -p /tmp/${buildid}/scripts/buildscript ${chroot} cp -p /tmp/${buildid}/sources/pnohang.c ${chroot} # phase 0, compile pnohang chroot ${chroot} /usr/bin/gcc -o /pnohang -Wall /pnohang.c 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log if [ $? -ne 0 ]; then error=255 fi if [ "${error}" = 0 ]; then # phase 1, make checksum # Needs to be chroot not jail so that port can be fetched chroot ${chroot} /buildscript ${dirname} 1 "$ED" "$PD" "$FD" "$BD" "$RD" 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log if [ -f ${chroot}/tmp/status ]; then error=$(cat ${chroot}/tmp/status) else error=255 fi fi if [ "${error}" = 0 ]; then # make checksum succeeded # phase 2, make package ln -sf ${pkgname}.log2 ${chroot}/tmp/make.log if [ "${use_jail}" = 1 ]; then ifconfig lo0 alias 127.${ip1}.${ip2}.${ip3}/32 jail -J ${chroot}/tmp/jail.id ${chroot} jail-${chrootpid} 127.${ip1}.${ip2}.${ip3} /usr/bin/env JAIL_ADDR=127.${ip1}.${ip2}.${ip3} HTTP_PROXY=${http_proxy} /usr/bin/nice -n $nice /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" > ${chroot}/tmp/${pkgname}.log2 2>&1 ifconfig lo0 delete 127.${ip1}.${ip2}.${ip3} else chroot ${chroot} /usr/bin/nice -n ${nice} /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" > ${chroot}/tmp/${pkgname}.log2 2>&1 fi grep pnohang ${chroot}/tmp/${pkgname}.log2 cat ${chroot}/tmp/${pkgname}.log2 >> ${chroot}/tmp/${pkgname}.log rm ${chroot}/tmp/${pkgname}.log2 error=$(cat ${chroot}/tmp/status) fi rm -rf ${chroot}/${WRKDIRPREFIX} # Record build completion time for ganglia echo "${arch} ${branch} ${buildid}" > ${buildroot}/stamp/${pkgname} exit $error Index: projects/portbuild/scripts/prunefailure =================================================================== --- projects/portbuild/scripts/prunefailure (revision 221760) +++ projects/portbuild/scripts/prunefailure (revision 221761) @@ -1,89 +1,92 @@ #!/bin/sh # # Prune the failure files of stale entries # # This must be called via: # # lockf -k ${pbd}/${arch}/${branch}/failure.lock ${pbc}/scripts/prunefailure ${arch} ${branch} ${buildid} # # to avoid racing with any package builds in progress that might try to append to # these files. # configurable variables pbc=${PORTBUILD_CHECKOUT:-/var/portbuild} pbd=${PORTBUILD_DATA:-/var/portbuild} cleanup() { echo "Problem writing new failure file!" rm -f failure.new exit 1 } if [ $# -ne 3 ]; then echo "prunefailure " exit 1 fi arch=$1 branch=$2 buildid=$3 shift 3 . ${pbc}/conf/server.conf . ${pbc}/conf/common.conf . ${pbd}/${arch}/portbuild.conf +if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf +fi . ${pbc}/scripts/buildenv builddir=${pbd}/${arch}/${branch}/builds/${buildid} buildenv ${pbd} ${arch} ${branch} ${builddir} home=${pbd}/${arch}/${branch} cd $home pkgdir=${builddir}/packages/All index=${PORTSDIR}/${INDEXFILE} if [ "`wc -l $index | awk '{print $1}'`" -lt 9000 ]; then echo "INDEX is corrupted, terminating!" exit 1 fi echo "===> Pruning old failure file" rm -f failure.new IFS='|' while read dir name ver olddate date count; do if [ -z "$dir" -o -z "$name" -o -z "$ver" -o -z "$olddate" -o -z "$date" -o -z "$count" ]; then echo Malformed entry "$dir|$name|$ver|$olddate|$date|$count" # Clean up the 'latest error log' symlink rm -f ${pbd}/${arch}/${branch}/latest/${dir} continue fi entry=$(grep "|/usr/ports/$dir|" $index) if [ -z "$entry" ]; then echo $dir not in index rm -f ${pbd}/${arch}/${branch}/latest/${dir} continue fi newver=$(echo $entry | awk '{print $1}') if [ -e "${builddir}/packages/All/$newver${PKGSUFFIX}" ]; then echo "$newver package exists, should not still be here!" rm -f ${pbd}/${arch}/${branch}/latest/${dir} continue fi if grep -qxF $newver ${builddir}/duds.full; then echo "$newver listed in duds, should not be here" rm -f ${pbd}/${arch}/${branch}/latest/${dir} continue fi (echo "$dir|$name|$newver|$olddate|$date|$count" >> $home/failure.new) || cleanup done < $home/failure mv failure.new failure Index: projects/portbuild/scripts/setupnode =================================================================== --- projects/portbuild/scripts/setupnode (revision 221760) +++ projects/portbuild/scripts/setupnode (revision 221761) @@ -1,169 +1,172 @@ #!/bin/sh # # Script run on the clients, to set them up in preparation for building # packages. This includes setting up parts of the /var/portbuild # directory hierarchy, the portbuild script and the bindist.tar file # for populating the build chroots. if [ $# -lt 8 ]; then echo "usage: $0 portbuilddir arch branch buildid tmpdir portsmd5 srcmd5 phase [-nocopy] [-force]" exit 1 fi pbd=$1 arch=$2 branch=$3 buildid=$4 tmpdir=$5 portsmd5=$6 srcmd5=$7 bindistmd5=$8 phase=$9 shift 9 precopy() { # Create directories and symlinks for later population # Timestamp of finished builds mkdir -p ${tmpdir}/stamp/ # Prepare all directories, they will be populated by a rsync # push from the master mkdir -p ${pbd}/${arch}/clients/ if [ "${buildid}" != "-" -a "${branch}" != "-" ]; then if [ ${nocopy} -eq 0 ]; then mkdir -p ${builddir} fi fi } postcopy() { if [ "${buildid}" != "-" -a "${branch}" != "-" ]; then # Extract ports trees and cleanup if [ ${nocopy} -eq 0 ]; then cd ${builddir} || return 1 # Unpack bindist if [ -f bindist.tbz.md5 -a "${force}" -ne 1 ]; then localbindistmd5=$(awk '{print $4}' bindist.tbz.md5) else localbindistmd5=0 fi if [ ${localbindistmd5} != ${bindistmd5} -o ! -f bindist.tar ]; then if [ -f bindist.tar ]; then rm -f bindist.tar fi bzcat bindist.tbz > bindist.tar || return 1 fi # Unpack ports if [ -f ports-${buildid}.tbz.md5 -a "${force}" -ne 1 ]; then localportsmd5=$(awk '{print $4}' ports-${buildid}.tbz.md5) else localportsmd5=0 fi if [ ${localportsmd5} != ${portsmd5} -o ! -d ports ]; then if [ -d ports ]; then mv ports ports~ mkdir ports rm -rf ports~ & fi tar xfj ports-${buildid}.tbz || return 1 fi # Unpack src if [ -f src-${buildid}.tbz.md5 -a "${force}" -ne 1 ]; then localsrcmd5=$(awk '{print $4}' src-${buildid}.tbz.md5) else localsrcmd5=0 fi if [ ${localsrcmd5} != ${srcmd5} -o ! -d src ]; then if [ -d src ]; then mv src src~ mkdir src rm -rf src~ & fi tar xfj src-${buildid}.tbz || return 1 fi touch .ready fi # Clean up the tmpdir # By now the portbuild.conf files are in place so we can source them . ${pbd}/${arch}/portbuild.conf + if [ -f ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf ]; then + . ${pbd}/${arch}/${branch}/builds/${buildid}/portbuild.conf + fi me=$(hostname) if [ -f ${pbd}/${arch}/portbuild.${me} ] ; then . ${pbd}/${arch}/portbuild.${me} fi if [ "${buildid}" != "-" -a "${branch}" != "-" ]; then mkdir -p ${wrkdir}/chroot if [ "${use_zfs}" -eq 1 ]; then zbase=${scratchdir#/} zfs create ${zbase}/${branch} || true zfs create ${zbase}/${branch}/${buildid} || true zfs create ${zbase}/${branch}/${buildid}/world || true zfs create ${zbase}/${branch}/${buildid}/chroot || true tar xfpC ${builddir}/bindist.tar ${scratchdir}/${branch}/${buildid}/world tar xfpC ${pbd}/${arch}/clients/bindist-$(hostname).tar ${scratchdir}/${branch}/${buildid}/world zfs snapshot ${zbase}/${branch}/${buildid}/world@base else mkdir -p ${wrkdir}/tarballs if [ ${nocopy} -eq 0 ]; then ln -sf ${pbab}/builds/${buildid}/bindist.tar ${wrkdir}/tarballs ln -sf ${pbd}/${arch}/clients/bindist-$(hostname).tar ${wrkdir}/tarballs fi fi fi for i in ${wrkdir}/chroot/*; do ${sudo_cmd} /tmp/${buildid}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${i} 2 done wait else # Client may have been waiting for us to set it up, so finish # the job. touch /tmp/.boot_finished fi } if [ "${branch}" != "-" -a "${buildid}" != "-" ]; then pbab=${pbd}/${arch}/${branch} builddir=${pbab}/builds/${buildid} wrkdir=${tmpdir}/${branch}/${buildid} fi nocopy=0 force=0 while [ $# -ge 1 ]; do case $1 in -nocopy) nocopy=1 ;; -force) force=1 ;; esac shift done case ${phase} in pre-copy) precopy ;; post-copy) postcopy ;; *) echo "Invalid phase ${phase}" exit 1 esac