Index: head/Mk/Scripts/check-stagedir.sh =================================================================== --- head/Mk/Scripts/check-stagedir.sh (revision 475360) +++ head/Mk/Scripts/check-stagedir.sh (revision 475361) @@ -1,284 +1,286 @@ #!/bin/sh # ports/Mk/Scripts/check-stagedir.sh - called from ports/Mk/bsd.stage.mk # $FreeBSD$ # # MAINTAINER: portmgr@FreeBSD.org # # This script serves 2 purposes: # 1. Generate a plist # 2. Test a plist for issues: # a. Files in STAGEDIR that are missing from plist # b. Files in plist missing from STAGEDIR # c. Files in plist which are owned by dependencies/MTREEs set -e . ${SCRIPTSDIR}/functions.sh # lists an mtree file's contents, prefixed to dir. listmtree() { # mtreefile prefix { echo '#mtree' sed 's/nochange$//;' $1 } | tar -tf- | sed "s,^,$2/,;s,^$2/\.$,$2,;s,^$,/," } ### PRODUCE MTREE FILE parse_mtree() { { listmtree /etc/mtree/BSD.root.dist "" listmtree /etc/mtree/BSD.usr.dist /usr listmtree /etc/mtree/BSD.var.dist /var # Use MTREE_FILE if specified and it doesn't already # match LOCALBASE if [ -n "${MTREE_FILE}" ]; then if [ "${PREFIX}" != "${LOCALBASE}" -o "${MTREE_FILE}" \ != "${PORTSDIR}/Templates/BSD.local.dist" ]; then listmtree "${MTREE_FILE}" "${PREFIX}" fi fi listmtree "${PORTSDIR}/Templates/BSD.local.dist" "${LOCALBASE}" unset MTREE_FILE # Add LOCALBASE a=${LOCALBASE} while :; do echo ${a} a=${a%/*} [ -z "${a}" ] && break done # Add in PREFIX if this port wants it if [ ${NO_PREFIX_RMDIR} -eq 0 ]; then a=${PREFIX} while :; do echo ${a} a=${a%/*} [ -z "${a}" ] && break done fi } >${WRKDIR}/.mtree } # Sort a directory list by the order of the dfs-sorted file (from find -d -s) sort_dfs() { while read -r dir; do grep "^[0-9]* ${dir}$" ${WRKDIR}/.staged-dirs-dfs-sorted done | sort -n | cut -d ' ' -f2- } # Prepare sed(1) regex for PLIST_SUB_SED/PORTEXAMPLES/OPTIONS/... setup_plist_seds() { ### HANDLE PORTDOCS/PORTEXAMPLES sed_portdocsexamples="/%%DOCSDIR%%/s!^!%%PORTDOCS%%!g; /%%EXAMPLESDIR%%/s!^!%%PORTEXAMPLES%%!g;" if [ ${makeplist} -eq 0 ]; then # echo "=====> Using OPTIONS: ${PORT_OPTIONS}" | /usr/bin/fmt -w 79 | \ # sed -e '2,$s/^/ /' # Handle magical PORT* features for option in DOCS EXAMPLES; do want_option=0 case " ${PORT_OPTIONS} " in *\ ${option}\ *) want_option=1 ;; esac [ ${want_option} -eq 0 ] && \ sed_portdocsexamples="${sed_portdocsexamples} /^%%PORT${option}%%/d;" done unset PORT_OPTIONS fi - sed_plist_sub=$(echo "${PLIST_SUB_SED}" | /bin/sh ${SCRIPTSDIR}/plist_sub_sed_sort.sh) + sed_plist_sub=$(mktemp -t sed_plist_sub) + # We only exit 0 or exit 1 + trap "rm -f ${sed_plist_sub}" EXIT 1 + echo "${PLIST_SUB_SED}" | /bin/sh ${SCRIPTSDIR}/plist_sub_sed_sort.sh ${sed_plist_sub} unset PLIST_SUB_SED # Used for generate_plist - sed_files_gen="s!^${PREFIX}/!!g; ${sed_plist_sub} \ - ${sed_portdocsexamples} /^share\/licenses/d; \ + sed_files_gen="${sed_portdocsexamples} /^share\/licenses/d; \ \#${LOCALBASE}/lib/debug#d;" - sed_dirs_gen="s!^${PREFIX}/!!g; ${sed_plist_sub} s,^,@dir ,; \ + sed_dirs_gen="s,^,@dir ,; \ ${sed_portdocsexamples} \ /^@dir share\/licenses/d;" # These prevent ignoring DOCS/EXAMPLES dirs with sed_portdocsexamples - sed_files="s!^${PREFIX}/!!g; ${sed_plist_sub} /^share\/licenses/d; \ + sed_files="/^share\/licenses/d; \ \#${LOCALBASE}/lib/debug#d;" - sed_dirs="s!^${PREFIX}/!!g; ${sed_plist_sub} s,^,@dir ,; \ + sed_dirs="s,^,@dir ,; \ /^@dir share\/licenses/d;" } # Generate plist from staged files generate_plist() { : >${WRKDIR}/.staged-plist ### HANDLE FILES find ${STAGEDIR} -type f -o -type l | sort | \ sed -e "s,${STAGEDIR},," >${WRKDIR}/.staged-files comm -13 ${WRKDIR}/.plist-files ${WRKDIR}/.staged-files | \ - sed -e "${sed_files_gen}" \ + sed -e "s!^${PREFIX}/!!g;" -f "${sed_plist_sub}" -e "${sed_files_gen}" \ >>${WRKDIR}/.staged-plist || : ### HANDLE DIRS cat ${WRKDIR}/.plist-dirs-unsorted ${WRKDIR}/.mtree \ | sort -u >${WRKDIR}/.traced-dirs find ${STAGEDIR} -type d | sed -e "s,^${STAGEDIR},,;/^$/d" | sort \ >${WRKDIR}/.staged-dirrms-sorted find -s -d ${STAGEDIR}${PREFIX} -type d -empty | sed -e "s,^${STAGEDIR},,;\,^${PREFIX}$,d;/^$/d" \ >${WRKDIR}/.staged-dirs-dfs find -s -d ${STAGEDIR} -type d ! -path "${STAGEDIR}${PREFIX}/*" | sed -e "s,^${STAGEDIR},,;\,^${PREFIX}$,d;/^$/d" \ >>${WRKDIR}/.staged-dirs-dfs sort ${WRKDIR}/.staged-dirs-dfs >${WRKDIR}/.staged-dirs-sorted awk '{print FNR, $0}' ${WRKDIR}/.staged-dirs-dfs \ >${WRKDIR}/.staged-dirs-dfs-sorted # Find all staged dirs and then sort them by depth-first (find -d -s) comm -13 ${WRKDIR}/.traced-dirs ${WRKDIR}/.staged-dirs-sorted \ - | sort_dfs | sed "${sed_dirs_gen}" \ + | sort_dfs | sed -e "s!^${PREFIX}/!!g;" -f "${sed_plist_sub}" -e "${sed_dirs_gen}" \ >>${WRKDIR}/.staged-plist || : } # Check for files in STAGEDIR missing from plist check_orphans_from_plist() { local ret=0 echo "===> Checking for items in STAGEDIR missing from pkg-plist" # Handle whitelisting while read -r path; do case "${path}" in *.bak) ;; *.orig) ;; */.DS_Store) ;; */.cvsignore) ;; */.git/*|'@dir '*/.git) ;; */.gitattributes|*/.gitignore|*/.gitmodules) ;; */.svn/*|'@dir '*/.svn) ;; */.svnignore) ;; */CVS/*|'@dir '*/CVS) ;; */info/dir|info/dir|info/*/dir) ;; share/fonts/*/fonts.dir) ;; share/fonts/*/fonts.scale) ;; share/applications/mimeinfo.cache) ;; share/mime/XMLnamespaces) ;; share/mime/aliases) ;; share/mime/generic-icons) ;; share/mime/globs) ;; share/mime/globs2) ;; share/mime/icons) ;; share/mime/magic) ;; share/mime/mime.cache) ;; share/mime/subclasses) ;; share/mime/treemagic) ;; share/mime/types) ;; share/mime/version) ;; '@dir etc/gconf/gconf.xml.defaults');; *) # An orphan was found, return non-zero status ret=1 echo "Error: Orphaned: ${path}" >&2 ;; esac done < ${WRKDIR}/.staged-plist return ${ret} } # Check for items in plist not in STAGEDIR (pkg lstat(2) errors) check_missing_plist_items() { local ret=0 echo "===> Checking for items in pkg-plist which are not in STAGEDIR" : >${WRKDIR}/.invalid-plist-missing comm -23 ${WRKDIR}/.plist-files-no-comments ${WRKDIR}/.staged-files | \ - sed -e "${sed_files}" \ + sed -e "s!^${PREFIX}/!!g;" -f "${sed_plist_sub}" -e "${sed_files}" \ >>${WRKDIR}/.invalid-plist-missing || : # Look for directories, then sort them by DFS. Must create the dirs # so find -ds can be used to sort them. rm -rf ${WRKDIR}/.missing-dirs > /dev/null 2>&1 || : mkdir ${WRKDIR}/.missing-dirs comm -23 ${WRKDIR}/.plist-dirs-sorted-no-comments \ ${WRKDIR}/.staged-dirrms-sorted > ${WRKDIR}/.missing-plist-dirs # Creates the dirs in WRKDIR/.missing-dirs and ensure spaces are # quoted. sed -e "s,^,${WRKDIR}/.missing-dirs," \ -e 's,^\(.*\)$,"\1",' \ ${WRKDIR}/.missing-plist-dirs | xargs mkdir -p find -d -s ${WRKDIR}/.missing-dirs | \ sed -e "s,^${WRKDIR}/.missing-dirs,," | \ while read -r dir; do \ grep -x "${dir}" ${WRKDIR}/.missing-plist-dirs || :; done | \ - sed "${sed_dirs}" \ + sed -e "s!^${PREFIX}/!!g;" -f "${sed_plist_sub}" -e "${sed_dirs}" \ >>${WRKDIR}/.invalid-plist-missing || : rm -rf ${WRKDIR}/.missing-dirs if [ -s "${WRKDIR}/.invalid-plist-missing" ]; then ret=1 while read -r line; do echo "Error: Missing: ${line}" >&2 done < ${WRKDIR}/.invalid-plist-missing fi return ${ret} } # obtain operating mode from command line ret=0 makeplist=0 case "$1" in checkplist) ;; makeplist) makeplist=1 ;; *) echo >&2 "Usage: $0 {checkplist|makeplist}" ; exit 1 ;; esac # validate environment validate_env STAGEDIR PREFIX LOCALBASE WRKDIR WRKSRC MTREE_FILE \ TMPPLIST PLIST_SUB_SED SCRIPTSDIR PORT_OPTIONS NO_PREFIX_RMDIR [ -n "${DEBUG_MK_SCRIPTS}" -o -n "${DEBUG_MK_SCRIPTS_CHECK_STAGEDIR}" ] && set -x set -u if [ $makeplist = 0 ] ; then echo "===> Parsing plist" parse_plist "${PREFIX}" 1 < ${TMPPLIST} \ 3>${WRKDIR}/.plist-dirs-unsorted \ >${WRKDIR}/.plist-files-unsorted unset TMPPLIST # Create the -no-comments files and trim out @comment from the plists. # This is used for various tests later. sed -e '/^@comment/d' ${WRKDIR}/.plist-dirs-unsorted \ >${WRKDIR}/.plist-dirs-unsorted-no-comments sed -i '' -e 's/^@comment //' ${WRKDIR}/.plist-dirs-unsorted sed -e '/^@comment/d' ${WRKDIR}/.plist-files-unsorted | sort \ >${WRKDIR}/.plist-files-no-comments sed -e 's/^@comment //' ${WRKDIR}/.plist-files-unsorted | sort \ >${WRKDIR}/.plist-files else # generate plist - pretend the plist had been empty : >${WRKDIR}/.plist-dirs-unsorted : >${WRKDIR}/.plist-files echo '/you/have/to/check/what/makeplist/gives/you' fi parse_mtree setup_plist_seds generate_plist # If just making plist, show results and exit successfully. if [ ${makeplist} -eq 1 ]; then cat ${WRKDIR}/.staged-plist exit 0 fi check_orphans_from_plist || ret=1 # Prepare plist-dirs for directory checks sort -u ${WRKDIR}/.plist-dirs-unsorted-no-comments \ >${WRKDIR}/.plist-dirs-sorted-no-comments check_missing_plist_items || ret=1 if [ ${ret} -ne 0 ]; then echo "===> Error: Plist issues found." >&2 if [ "${PREFIX}" != "${LOCALBASE}" ]; then echo "===> Warning: Test was done with PREFIX != LOCALBASE" echo "===> Warning: The port may not be properly installing into PREFIX" fi fi exit ${ret} Index: head/Mk/Scripts/check_leftovers.sh =================================================================== --- head/Mk/Scripts/check_leftovers.sh (revision 475360) +++ head/Mk/Scripts/check_leftovers.sh (revision 475361) @@ -1,171 +1,173 @@ #! /bin/sh # $FreeBSD$ # # MAINTAINER: portmgr@FreeBSD.org # # This script is used by poudriere and tinderbox(soon) as the source-of-truth for # what should be considered a leftover and what is whitelisted. # # !!!! This script's input/output format must remain backwards-compatible. # !!!! If you want to change it, create a new script and have the calling # !!!! scripts use the new one if available. # # Usage: env PORTSDIR=... check_leftovers.sh category/port # stdin: # - missing-file # + new-file # M modified-file reason... # # stdout: # same -/+/M format, but with files substituted, without approved # whitelisted files, and hides any directories already in plist. # # The PLIST_SUB feature can be disabled by setting PLIST_SUB_SED= # in environment. [ -n "${DEBUG_MK_SCRIPTS}" -o -n "${DEBUG_MK_SCRIPTS_CHECK_LEFTOVERS}" ] && set -x origin="$1" [ $# -eq 1 ] || { echo "Must supply ORIGIN as parameter" >&2; exit 1; } [ -n "${PORTSDIR}" ] || { echo "PORTSDIR must be set" >&2; exit 1; } portdir="${PORTSDIR}/${origin}" # PREFIX/LOCALBASE may be set in env or want default from port. if [ -n "${PREFIX}" ]; then PORT_FLAGS="${PORT_FLAGS} PREFIX=${PREFIX}" else PREFIX=$(make -C ${portdir} -VPREFIX) fi if [ -n "${LOCALBASE}" ]; then PORT_FLAGS="${PORT_FLAGS} LOCALBASE=${LOCALBASE}" else LOCALBASE=$(make -C ${portdir} -VLOCALBASE) fi if [ -z "${CCACHE_DIR}" ]; then CCACHE_DIR=$(make -C ${portdir} -VCCACHE_DIR) fi homedirs=$(awk -F: -v users="$(make -C ${portdir} -V USERS|sed -e 's, ,|,g;/^$/d;s,^,^(,;s,$,)$,')" 'users && $1 ~ users {print $9}' ${PORTSDIR}/UIDs|sort -u|sed -e "s|/usr/local|${PREFIX}|"|tr "\n" " ") -plistsub_sed=$(make -C ${portdir} -VPLIST_SUB_SED | /bin/sh ${PORTSDIR}/Mk/Scripts/plist_sub_sed_sort.sh) +plistsub_sed=$(mktemp -t plistsub_sed) +trap "rm -f ${plistsub_sed}" EXIT 1 +make -C ${portdir} -VPLIST_SUB_SED | /bin/sh ${PORTSDIR}/Mk/Scripts/plist_sub_sed_sort.sh ${plistsub_sed} tmpplist=$(make -C ${portdir} -VTMPPLIST) while read -r modtype path extra; do # Ignore everything from these files/directories case "${path}" in ${CCACHE_DIR}/*|\ /compat/linux/proc/*|\ /dev/*|\ /etc/make.conf.bak|\ /proc/*|\ /tmp/*|\ /var/db/pkg/*|\ /var/db/ports/*|\ /var/log/*|\ /var/mail/*|\ /var/run/*|\ /var/tmp/*) continue ;; # fc-cache - skip for now /var/db/fontconfig/*) continue ;; esac ignore_path=0 - sub_path=$(echo "$path" | sed -e "s|^${PREFIX}/||" -e "${plistsub_sed}") + sub_path=$(echo "$path" | sed -e "s|^${PREFIX}/||" -f "${plistsub_sed}") orig_sub_path="${sub_path}" # If this is a directory, use @dir in output is_dir=0 if [ -d "${path}" ]; then is_dir=1 sub_path="@dir ${sub_path}" fi # Handle PORTDOCS/PORTEXAMPLES/etc case "${orig_sub_path}" in %%DOCSDIR%%*) sub_path="%%PORTDOCS%%${sub_path}" ;; %%EXAMPLESDIR%%*) sub_path="%%PORTEXAMPLES%%${sub_path}" ;; esac case $modtype in +) if [ ${is_dir} -eq 1 ]; then # home directory of users created case " ${homedirs} " in *\ ${path}\ *) continue ;; *\ ${path}/*\ *) continue ;; esac # Don't show dirs already in plist (due to parents) grep -qE \ "^@(unexec rmdir \"?(%D/|${PREFIX})?${path#${PREFIX}/}[ \"]|dir(rm|rmtry)? ${path#${PREFIX}/}\$)" \ ${tmpplist} && continue fi # Check absolute paths case "${path}" in # Leave qmail's queue dir alone to not cause lost mail # during upgrades, just as /var/mail is left alone. /var/qmail/queue/*|/var/qmail/queue) continue ;; esac # Check relative/plist paths case "${sub_path}" in # gconftool-2 --makefile-uninstall-rule is unpredictable etc/gconf/gconf.xml.defaults/%gconf-tree*.xml) ;; *) echo "+ ${sub_path}" ;; esac ;; -) # Skip removal of PREFIX and PREFIX/info from # bsd.port.mk for now. # Skip if it is PREFIX and non-LOCALBASE. See misc/kdehier4 # or mail/qmail for examples [ "${path}" = "${PREFIX}" -a "${LOCALBASE}" != "${PREFIX}" ] && ignore_path=1 # The removal of info may be a bug; it's part of BSD.local.dist. # See ports/74691 [ "${sub_path}" = "info" -a "${LOCALBASE}" != "${PREFIX}" ] && ignore_path=1 [ $ignore_path -eq 0 ] && echo "- ${sub_path}" ;; M) # Check relative/plist paths case "${sub_path}" in # gconftool-2 --makefile-uninstall-rule is unpredictable etc/gconf/gconf.xml.defaults/%gconf-tree*.xml) ;; # This is a cache file for gio modules could be modified # for any gio modules lib/gio/modules/giomodule.cache) ;; # removal of info files leaves entry uneasy to cleanup # in info/dir info/dir) ;; */info/dir) ;; # The is pear database cache %%PEARDIR%%/.depdb|%%PEARDIR%%/.filemap) ;; #ls-R files from texmf are often regenerated */ls-R) ;; # Octave packages database, blank lines can be inserted # between pre-install and post-deinstall share/octave/octave_packages) ;; # xmlcatmgr is constantly updating catalog.ports ignore # modification to that file share/xml/catalog.ports) ;; # Ignore ghc's doc index share/doc/ghc-%%GHC_VERSION%%/*) ;; # Ignore ghc's package conf lib/ghc-%%GHC_VERSION%%/package.conf.d/*) ;; # Ignore common system config files /etc/group|\ /etc/make.conf|\ /etc/master.passwd|\ /etc/passwd|\ /etc/pwd.db|\ /etc/shells|\ /etc/spwd.db) ;; *) echo "M ${sub_path#@dir } ${extra}" ;; esac ;; esac done exit 0 Index: head/Mk/Scripts/plist_sub_sed_sort.sh =================================================================== --- head/Mk/Scripts/plist_sub_sed_sort.sh (revision 475360) +++ head/Mk/Scripts/plist_sub_sed_sort.sh (revision 475361) @@ -1,17 +1,17 @@ #! /bin/sh # $FreeBSD$ # # MAINTAINER: portmgr@FreeBSD.org # # PLIST_SUB_SED helper to sort by longest value first. awk '{ while (match($0, /s![^!]*![^!]*!g;/)) { sedp=substr($0, RSTART, RLENGTH) $0=substr($0, RSTART+RLENGTH) split(sedp, a, "!") # Convert \. to . for sorting. gsub(/\\./, ".", a[2]) print length(a[2]), sedp } -}' | sort -rn | awk '{$1=""; print $0}' | paste -s -d ' ' - +}' | sort -rn | awk '{$1=""; print $0}' > $1