Index: ports-mgmt/portsnap/Makefile =================================================================== --- /dev/null +++ ports-mgmt/portsnap/Makefile @@ -0,0 +1,26 @@ +PORTNAME= portsnap +PORTVERSION= 2021.8.15.0 +CATEGORIES= ports-mgmt +MASTER_SITES= # none +DISTFILES= # none + +MAINTAINER= swills@FreeBSD.org +COMMENT= Fetch and extract compressed snapshots of the ports tree + +LICENSE= BSD2CLAUSE + +NO_BUILD= yes +NO_ARCH= yes + +PLIST_FILES= "@sample etc/${PORTNAME}.conf.sample" \ + man/man5/${PORTNAME}.conf.5.gz \ + man/man8/${PORTNAME}.8.gz \ + sbin/${PORTNAME} + +do-install: + ${INSTALL_SCRIPT} ${FILESDIR}/${PORTNAME}.sh ${STAGEDIR}${PREFIX}/sbin/${PORTNAME} + ${INSTALL_MAN} ${FILESDIR}/${PORTNAME}.8 ${STAGEDIR}${MANPREFIX}/man/man8 + ${INSTALL_MAN} ${FILESDIR}/${PORTNAME}.conf.5 ${STAGEDIR}${PREFIX}/man/man5 + ${INSTALL_DATA} ${FILESDIR}/${PORTNAME}.conf ${STAGEDIR}${PREFIX}/etc/${PORTNAME}.conf.sample + +.include Index: ports-mgmt/portsnap/files/portsnap.8 =================================================================== --- /dev/null +++ ports-mgmt/portsnap/files/portsnap.8 @@ -0,0 +1,288 @@ +.\"- +.\" Copyright 2004-2005 Colin Percival +.\" All rights reserved +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted providing that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +.\" POSSIBILITY OF SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd July 1, 2018 +.Dt PORTSNAP 8 +.Os FreeBSD +.Sh NAME +.Nm portsnap +.Nd fetch and extract compressed snapshots of the ports tree +.Sh SYNOPSIS +.Nm +.Op Fl I +.Op Fl d Ar workdir +.Op Fl f Ar conffile +.Op Fl k Ar KEY +.Op Fl l Ar descfile +.Op Fl p Ar portsdir +.Op Fl s Ar server +.Cm command ... +.Op Ar path +.Sh DESCRIPTION +The +.Nm +tool is used to fetch and update compressed snapshots +of the +.Fx +ports tree, and extract and update an +uncompressed ports tree. +.Pp +In a normal update operation, +.Nm +will routinely restore modified files to their unmodified state and +delete unrecognized local files. +.Sh OPTIONS +The following options are supported: +.Bl -tag -width "-f conffile" +.It Fl d Ar workdir +Store working files (e.g.\& downloaded updates) in +.Ar workdir . +(default: +.Pa /var/db/portsnap , +or as given in the configuration file.) +.It Fl f Ar conffile +Read the configuration from +.Ar conffile . +(default: +.Pa /etc/portsnap.conf ) +.It Fl I +For the +.Cm update +command, update INDEX files, but not the rest of the ports tree. +.It Fl k Ar KEY +Expect a public key with given SHA256 hash. +(default: read value from configuration file.) +.It Fl l Ar descfile +Merge the specified local describes file into the INDEX files being +built. +The +.Ar descfile +should be generated by running +.Cm make describe +in each of the local port directories. +.It Fl p Ar portsdir +When extracting or updating an uncompressed snapshot, +operate on the directory +.Ar portsdir . +(default: +.Pa /usr/ports/ , +or as given in the configuration file.) +.It Fl s Ar server +Fetch files from the specified server or server pool. +(default: portsnap.FreeBSD.org, or as given in the +configuration file.) +.It path +For +.Cm extract +command only, operate only on parts of the ports tree starting with +.Ar path . +(e.g.\& +.Nm +.Cm extract +.Ar sysutils/port +would extract sysutils/portsman, sysutils/portsnap, +sysutils/portupgrade, etc.) +.It Fl Fl interactive +override auto-detection of calling process. +Only use this when calling portsnap from an +.Sy interactive, non-terminal application. +(Cron jobs are particularly bad since they cause +load spikes on the Portsnap mirrors.) +.El +.Sh COMMANDS +The +.Cm command +can be any one of the following: +.Bl -tag -width "-f conffile" +.It fetch +Fetch a compressed snapshot of the ports tree, or update +the existing snapshot. +This command should only be used interactively; for +non-interactive use, you should use the +.Cm cron +command. +.It cron +Sleep a random amount of time between 1 and 3600 seconds, +then operate as if the +.Cm fetch +command was specified. +As the name suggests, this command is designed for running +from +.Xr cron 8 ; +the random delay serves to minimize the probability that +a large number of machines will simultaneously attempt to +fetch updates. +.It extract +Extract a ports tree, replacing existing files and directories. +NOTE: This will remove anything occupying the location where +files or directories are being extracted; in particular, any +changes made locally to the ports tree (for example, adding new +patches) will be silently obliterated. +.Pp +Only run this command to initialize your portsnap-maintained +ports tree for the first time, if you wish to start over with +a clean, completely unmodified tree, or if you wish to extract +a specific part of the tree (using the +.Ar path +option). +.It update +Update a ports tree extracted using the +.Cm extract +command. +You must run this command to apply changes to your ports tree +after downloading updates via the +.Cm fetch +or +.Cm cron +commands. +Again, note that in the parts of the ports tree which are being +updated, any local changes or additions will be removed. +.It auto +Run +.Cm fetch +or +.Cm cron +depending on whether stdin is a terminal; then run +.Cm update +or +.Cm extract +depending on whether +.Ar portsdir +exists. +.El +.Sh TIPS +.Bl -bullet +.It +If your clock is set to local time, adding the line +.Pp +.Dl 0 3 * * * root /usr/sbin/portsnap cron +.Pp +to +.Pa /etc/crontab +is a good way to make sure you always have +an up-to-date snapshot of the ports tree available which +can quickly be extracted into +.Pa /usr/ports . +If your clock is set to UTC, please pick a random time other +than 3AM, to avoid overly imposing an uneven load on the +server(s) hosting the snapshots. +.Pp +Note that running +.Nm +.Cm cron +or +.Nm +.Cm fetch +does not apply the changes that were received: they only download +them. +To apply the changes, you must follow these commands with +.Nm +.Cm update . +The +.Nm +.Cm update +command is normally run by hand at a time when you are sure that +no one is manually working in the ports tree. +.It +Running +.Nm +.Cm update +from +.Xr cron 8 +is a bad idea -- if you are ever installing or updating a +port at the time the cron job runs, you will probably end up +in a mess when +.Nm +updates or removes files which are being used by the port +build. +However, running +.Nm +.Fl I +.Cm update +is probably safe, and can be used together with +.Xr pkg-version 8 +to identify installed software which is out of date. +.It +If you wish to use +.Nm +to keep a large number of machines up to date, you may wish +to set up a caching HTTP proxy. +Since +.Nm +uses +.Xr fetch 1 +to download updates, setting the +.Ev HTTP_PROXY +environment variable will direct it to fetch updates from +the given proxy. +This is much more efficient than +.Em mirroring +the files on the portsnap server, since the vast majority +of files are not needed by any particular client. +.El +.Sh PRIVACY NOTICE +As an unavoidable part of its operation, a machine running +.Nm +will make its public IP address and the list of files it fetches +available to the server from which it fetches updates. +Using these it may be possible to recognize a machine over an extended +period of time, determine when it is updated, and identify which +portions of the FreeBSD ports tree, if any, are being ignored using +"REFUSE" directives in +.Pa portsnap.conf . +In addition, the FreeBSD release level is transmitted to the server. +.Pp +Statistical data generated from information collected in this manner +may be published, but only in aggregate and after anonymizing the +individual systems. +.Sh FILES +.Bl -tag -width "/etc/portsnap.conf" +.It Pa /etc/portsnap.conf +Default location of the portsnap configuration file. +.It Pa /var/db/portsnap +Default location where compressed snapshots are stored. +.It Pa /usr/ports +Default location where the ports tree is extracted. +.El +.Sh EXAMPLES +Fetch the snapshots and create the +.Xr ports 7 +tree under +.Pa /usr/ports : +.Dl Nm Ar fetch Ar extract +.Pp +Update the ports tree: +.Dl Nm Ar fetch Ar update +.Sh SEE ALSO +.Xr fetch 1 , +.Xr sha256 1 , +.Xr fetch 3 , +.Xr portsnap.conf 5 , +.Xr pkg 7 , +.Xr pkg-version 8 +.Sh AUTHORS +.An Colin Percival Aq Mt cperciva@FreeBSD.org Index: ports-mgmt/portsnap/files/portsnap.conf =================================================================== --- /dev/null +++ ports-mgmt/portsnap/files/portsnap.conf @@ -0,0 +1,36 @@ +# $FreeBSD$ + +# Default directory where compressed snapshots are stored. +# WORKDIR=/var/db/portsnap + +# Default location of the ports tree (target for "update" and "extract"). +# PORTSDIR=/usr/ports + +# Server or server pool from which to fetch updates. You can change +# this to point at a specific server if you want, but in most cases +# using a "nearby" server won't provide a measurable improvement in +# performance. +SERVERNAME=portsnap.FreeBSD.org + +# Trusted keyprint. Changing this is a Bad Idea unless you've received +# a PGP-signed email from telling you to +# change it and explaining why. +KEYPRINT=9b5feee6d69f170e3dd0a2c8e469ddbd64f13f978f2f3aede40c98633216c330 + +# Example of ignoring parts of the ports tree. If you know that you +# absolutely will not need certain parts of the tree, this will save +# some bandwidth and disk space. See the manual page for more details. +# +# WARNING: Working with an incomplete ports tree is not supported and +# can cause problems due to missing dependencies. If you have REFUSE +# directives and experience problems, remove them and update your tree +# before asking for help on the mailing lists. +# +# REFUSE arabic chinese french german hebrew hungarian japanese +# REFUSE korean polish portuguese russian ukrainian vietnamese + +# List of INDEX files to build and the DESCRIBE file to use for each +#INDEX INDEX-11 DESCRIBE.11 +#INDEX INDEX-12 DESCRIBE.12 +#INDEX INDEX-13 DESCRIBE.13 +INDEX INDEX-14 DESCRIBE.14 Index: ports-mgmt/portsnap/files/portsnap.conf.5 =================================================================== --- /dev/null +++ ports-mgmt/portsnap/files/portsnap.conf.5 @@ -0,0 +1,147 @@ +.\"- +.\" Copyright 2004-2005 Colin Percival +.\" All rights reserved +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted providing that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +.\" POSSIBILITY OF SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd January 30, 2005 +.Dt PORTSNAP.CONF 5 +.Os FreeBSD +.Sh NAME +.Nm portsnap.conf +.Nd configuration file for +.Xr portsnap 8 +.Sh DESCRIPTION +The +.Nm +file controls where +.Xr portsnap 8 +fetches ports tree snapshots from, +which RSA key should be trusted to sign the updates, and what +directories should hold the compressed and live ports trees. +.Pp +A line of the form +.Dl SERVERNAME=portsnap.example.com +specifies the source from which snapshots should be fetched. +This is equivalent to the +.Fl s Ar server +option to +.Xr portsnap 8 , +and will be ignored if the command-line +option is used. +.Pp +A line of the form +.Dl KEYPRINT=0123456789abc ... 456789abcdef +(64 characters in total) +specifies the SHA-256 hash of the OpenSSL public key file +belonging to an RSA keypair which is trusted to sign updates. +This is equivalent to the +.Fl k Ar KEY +option to +.Xr portsnap 8 , +and will be ignored if the command-line +option is used. +.Pp +A line of the form +.Dl WORKDIR=/path/to/workdir +specifies the directory in which portsnap should maintain its compressed +snapshot of the ports tree. +This is equivalent to the +.Fl d Ar workdir +option to +.Xr portsnap 8 , +and will be ignored if the command-line option +is used. +.Pp +A line of the form +.Dl PORTSDIR=/path/to/portstree +specifies the directory in which portsnap will create the live ports +tree from its compressed snapshot via the +.Cm extract +and +.Cm update +commands. +This is equivalent to the +.Fl p Ar portsdir +option to +.Xr portsnap 8 , +and will be ignored if the command-line option +is used. +.Pp +If more than one line of any of the above forms is included in +.Nm +then only the last one will take effect. +.Pp +A line of the form +.Dl INDEX INDEXFILE DESCRIBEFILE +will instruct +.Xr portsnap 8 +that the specified INDEX file is generated from the specified +describe file distributed by the portsnap server. +.Pp +Finally, a line of the form +.Dl REFUSE foo bar +will instruct +.Xr portsnap 8 +to ignore parts of the ports tree with paths starting with +.Ar foo +or +.Ar bar , +which are interpreted as extended regular expressions by +.Xr egrep 1 . +This will result in those parts of the tree not being updated +in the compressed snapshot when the +.Cm fetch +and +.Cm cron +commands are used and not being extracted when the +.Cm extract +command is used (unless a specific +.Ar path +is passed to +.Xr portsnap 8 ) , +and if those parts of the ports tree are present they +will not be updated when the +.Cm update +command is used. +Unlike the other options, the parameters in REFUSE lines +accumulate and all such lines are considered. +.Bf Em +Note that operating with an incomplete ports tree is not +supported and may cause unexpected results. +.Ef +.Pp +Any lines not of the above forms will be ignored. +.Sh FILES +.Bl -tag -width "/etc/portsnap.conf" +.It Pa /etc/portsnap.conf +Default location of the portsnap configuration file. +.El +.Sh SEE ALSO +.Xr egrep 1 , +.Xr fetch 1 , +.Xr sha256 1 , +.Xr portsnap 8 +.Sh AUTHORS +.An Colin Percival Aq Mt cperciva@FreeBSD.org Index: ports-mgmt/portsnap/files/portsnap.sh =================================================================== --- /dev/null +++ ports-mgmt/portsnap/files/portsnap.sh @@ -0,0 +1,1156 @@ +#!/bin/sh + +#- +# SPDX-License-Identifier: BSD-2-Clause-FreeBSD +# +# Copyright 2004-2005 Colin Percival +# All rights reserved +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted providing that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# $FreeBSD$ + +#### Usage function -- called from command-line handling code. + +# Usage instructions. Options not listed: +# --debug -- don't filter output from utilities +# --no-stats -- don't show progress statistics while fetching files +usage() { + cat < serverlist_tried + +# Check that host(1) exists (i.e., that the system wasn't built with the +# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist. + if ! which -s host; then + : > serverlist_full + return 1 + fi + + echo -n "Looking up ${SERVERNAME} mirrors... " + +# Issue the SRV query and pull out the Priority, Weight, and Target fields. +# BIND 9 prints "$name has SRV record ..." while BIND 8 prints +# "$name server selection ..."; we allow either format. + MLIST="_http._tcp.${SERVERNAME}" + host -t srv "${MLIST}" | + sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" | + cut -f 1,2,4 -d ' ' | + sed -e 's/\.$//' | + sort > serverlist_full + +# If no records, give up -- we'll just use the server name we were given. + if [ `wc -l < serverlist_full` -eq 0 ]; then + echo "none found." + return 1 + fi + +# Report how many mirrors we found. + echo `wc -l < serverlist_full` "mirrors found." + +# Generate a random seed for use in picking mirrors. If HTTP_PROXY +# is set, this will be used to generate the seed; otherwise, the seed +# will be random. + if [ -n "${HTTP_PROXY}${http_proxy}" ]; then + RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" | + tr -d 'a-f' | + cut -c 1-9` + else + RANDVALUE=`jot -r 1 0 999999999` + fi +} + +# Pick a mirror. Returns 1 if we have run out of mirrors to try. +fetch_pick_server() { +# Generate a list of not-yet-tried mirrors + sort serverlist_tried | + comm -23 serverlist_full - > serverlist + +# Have we run out of mirrors? + if [ `wc -l < serverlist` -eq 0 ]; then + echo "No mirrors remaining, giving up." + return 1 + fi + +# Find the highest priority level (lowest numeric value). + SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1` + +# Add up the weights of the response lines at that priority level. + SRV_WSUM=0 + while read X; do + case "$X" in + ${SRV_PRIORITY}\ *) + SRV_W=`echo $X | cut -f 2 -d ' '` + SRV_WSUM=$(($SRV_WSUM + $SRV_W)) + ;; + esac + done < serverlist + +# If all the weights are 0, pretend that they are all 1 instead. + if [ ${SRV_WSUM} -eq 0 ]; then + SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l` + SRV_W_ADD=1 + else + SRV_W_ADD=0 + fi + +# Pick a value between 0 and the sum of the weights - 1 + SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}` + +# Read through the list of mirrors and set SERVERNAME. Write the line +# corresponding to the mirror we selected into serverlist_tried so that +# we won't try it again. + while read X; do + case "$X" in + ${SRV_PRIORITY}\ *) + SRV_W=`echo $X | cut -f 2 -d ' '` + SRV_W=$(($SRV_W + $SRV_W_ADD)) + if [ $SRV_RND -lt $SRV_W ]; then + SERVERNAME=`echo $X | cut -f 3 -d ' '` + echo "$X" >> serverlist_tried + break + else + SRV_RND=$(($SRV_RND - $SRV_W)) + fi + ;; + esac + done < serverlist +} + +# Check that we have a public key with an appropriate hash, or +# fetch the key if it doesn't exist. Returns 1 if the key has +# not yet been fetched. +fetch_key() { + if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then + return 0 + fi + + echo -n "Fetching public key from ${SERVERNAME}... " + rm -f pub.ssl + fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \ + 2>${QUIETREDIR} || true + if ! [ -r pub.ssl ]; then + echo "failed." + return 1 + fi + if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then + echo "key has incorrect hash." + rm -f pub.ssl + return 1 + fi + echo "done." +} + +# Fetch a snapshot tag +fetch_tag() { + rm -f snapshot.ssl tag.new + + echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... " + fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \ + 2>${QUIETREDIR} || true + if ! [ -r $1.ssl ]; then + echo "failed." + return 1 + fi + + openssl rsautl -pubin -inkey pub.ssl -verify \ + < $1.ssl > tag.new 2>${QUIETREDIR} || true + rm $1.ssl + + if ! [ `wc -l < tag.new` = 1 ] || + ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then + echo "invalid snapshot tag." + return 1 + fi + + echo "done." + + SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new` + SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new` +} + +# Sanity-check the date on a snapshot tag +fetch_snapshot_tagsanity() { + if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then + echo "Snapshot appears to be more than a year old!" + echo "(Is the system clock correct?)" + echo "Cowardly refusing to proceed any further." + return 1 + fi + if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then + echo -n "Snapshot appears to have been created more than " + echo "one day into the future!" + echo "(Is the system clock correct?)" + echo "Cowardly refusing to proceed any further." + return 1 + fi +} + +# Sanity-check the date on a snapshot update tag +fetch_update_tagsanity() { + fetch_snapshot_tagsanity || return 1 + + if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then + echo -n "Latest snapshot on server is " + echo "older than what we already have!" + echo -n "Cowardly refusing to downgrade from " + date -r ${OLDSNAPSHOTDATE} + echo "to `date -r ${SNAPSHOTDATE}`." + return 1 + fi +} + +# Compare old and new tags; return 1 if update is unnecessary +fetch_update_neededp() { + if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then + echo -n "Latest snapshot on server matches " + echo "what we already have." + echo "No updates needed." + rm tag.new + return 1 + fi + if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then + echo -n "Ports tree hasn't changed since " + echo "last snapshot." + echo "No updates needed." + rm tag.new + return 1 + fi + + return 0 +} + +# Fetch snapshot metadata file +fetch_metadata() { + rm -f ${SNAPSHOTHASH} tINDEX.new + + echo ${NDEBUG} "Fetching snapshot metadata... " + fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} \ + 2>${QUIETREDIR} || return + if [ "`${SHA256} -q ${SNAPSHOTHASH}`" != ${SNAPSHOTHASH} ]; then + echo "snapshot metadata corrupt." + return 1 + fi + mv ${SNAPSHOTHASH} tINDEX.new + echo "done." +} + +# Warn user about bogus metadata +fetch_metadata_freakout() { + echo + echo "Portsnap metadata is correctly signed, but contains" + echo "at least one line which appears bogus." + echo "Cowardly refusing to proceed any further." +} + +# Sanity-check a snapshot metadata file +fetch_metadata_sanity() { + if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then + fetch_metadata_freakout + return 1 + fi + if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then + echo + echo "Portsnap metadata appears bogus." + echo "Cowardly refusing to proceed any further." + return 1 + fi +} + +# Take a list of ${oldhash}|${newhash} and output a list of needed patches +fetch_make_patchlist() { + local IFS='|' + echo "" 1>${QUIETREDIR} + grep -vE "^([0-9a-f]{64})\|\1$" | + while read X Y; do + printf "Processing: $X $Y ...\r" 1>${QUIETREDIR} + if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then continue; fi + echo "${X}|${Y}" + done + echo "" 1>${QUIETREDIR} +} + +# Print user-friendly progress statistics +fetch_progress() { + LNC=0 + while read x; do + LNC=$(($LNC + 1)) + if [ $(($LNC % 10)) = 0 ]; then + echo -n $LNC + elif [ $(($LNC % 2)) = 0 ]; then + echo -n . + fi + done + echo -n " " +} + +pct_fmt() +{ + if [ $TOTAL -gt 0 ]; then + printf " \r" + printf "($1/$2) %02.2f%% " `echo "scale=4;$LNC / $TOTAL * 100"|bc` + fi +} + +fetch_progress_percent() { + TOTAL=$1 + LNC=0 + pct_fmt $LNC $TOTAL + while read x; do + LNC=$(($LNC + 1)) + if [ $(($LNC % 100)) = 0 ]; then + pct_fmt $LNC $TOTAL + elif [ $(($LNC % 10)) = 0 ]; then + echo -n . + fi + done + pct_fmt $LNC $TOTAL + echo " done. " +} + +# Sanity-check an index file +fetch_index_sanity() { + if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new || + fgrep -q "./" INDEX.new; then + fetch_metadata_freakout + return 1 + fi +} + +# Verify a list of files +fetch_snapshot_verify() { + while read F; do + if [ "`gunzip -c < snap/${F}.gz | ${SHA256} -q`" != ${F} ]; then + echo "snapshot corrupt." + return 1 + fi + done + return 0 +} + +# Fetch a snapshot tarball, extract, and verify. +fetch_snapshot() { + while ! fetch_tag snapshot; do + fetch_pick_server || return 1 + done + fetch_snapshot_tagsanity || return 1 + fetch_metadata || return 1 + fetch_metadata_sanity || return 1 + + rm -rf snap/ + +# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will +# probably take a while, so the progrees reports that fetch(1) generates +# will be useful for keeping the users' attention from drifting. + echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:" + fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1 + + echo -n "Extracting snapshot... " + tar -xz --numeric-owner -f ${SNAPSHOTHASH}.tgz snap/ || return 1 + rm ${SNAPSHOTHASH}.tgz + echo "done." + + echo -n "Verifying snapshot integrity... " +# Verify the metadata files + cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1 +# Extract the index + rm -f INDEX.new + gunzip -c < snap/`look INDEX tINDEX.new | + cut -f 2 -d '|'`.gz > INDEX.new + fetch_index_sanity || return 1 +# Verify the snapshot contents + cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1 + cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u | + lam -s 'snap/' - -s '.gz' > files.expected + find snap -mindepth 1 | sort > files.snap + if ! cmp -s files.expected files.snap; then + echo "unexpected files in snapshot." + return 1 + fi + rm files.expected files.snap + echo "done." + +# Move files into their proper locations + rm -f tag INDEX tINDEX + rm -rf files + mv tag.new tag + mv tINDEX.new tINDEX + mv INDEX.new INDEX + mv snap/ files/ + + return 0 +} + +# Update a compressed snapshot +fetch_update() { + rm -f patchlist diff OLD NEW filelist INDEX.new + + OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag` + OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag` + + while ! fetch_tag latest; do + fetch_pick_server || return 1 + done + fetch_update_tagsanity || return 1 + fetch_update_neededp || return 0 + fetch_metadata || return 1 + fetch_metadata_sanity || return 1 + + echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` " + echo "to `date -r ${SNAPSHOTDATE}`." + +# Generate a list of wanted metadata patches + join -t '|' -o 1.2,2.2 tINDEX tINDEX.new | + fetch_make_patchlist > patchlist + +# Attempt to fetch metadata patches + echo -n "Fetching `wc -l < patchlist | tr -d ' '` " + echo ${NDEBUG} "metadata patches.${DDSTATS}" + tr '|' '-' < patchlist | + lam -s "tp/" - -s ".gz" | + xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ + 2>${STATSREDIR} | fetch_progress + echo "done." + +# Attempt to apply metadata patches + echo -n "Applying metadata patches... " + local oldifs="$IFS" IFS='|' + while read X Y; do + if [ ! -f "${X}-${Y}.gz" ]; then continue; fi + gunzip -c < ${X}-${Y}.gz > diff + gunzip -c < files/${X}.gz > OLD + cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp + grep '^\+' diff | cut -c 2- | + sort -k 1,1 -t '|' -m - ptmp > NEW + if [ `${SHA256} -q NEW` = ${Y} ]; then + mv NEW files/${Y} + gzip -n files/${Y} + fi + rm -f diff OLD NEW ${X}-${Y}.gz ptmp + done < patchlist 2>${QUIETREDIR} + IFS="$oldifs" + echo "done." + +# Update metadata without patches + join -t '|' -v 2 tINDEX tINDEX.new | + cut -f 2 -d '|' /dev/stdin patchlist | + while read Y; do + if [ ! -f "files/${Y}.gz" ]; then + echo ${Y} + fi + done > filelist + echo -n "Fetching `wc -l < filelist | tr -d ' '` " + echo ${NDEBUG} "metadata files... " + lam -s "f/" - -s ".gz" < filelist | + xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ + 2>${QUIETREDIR} + + while read Y; do + echo -n "Verifying ${Y}... " 1>${QUIETREDIR} + if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then + mv ${Y}.gz files/${Y}.gz + else + echo "metadata is corrupt." + return 1 + fi + echo "ok." 1>${QUIETREDIR} + done < filelist + echo "done." + +# Extract the index + echo -n "Extracting index... " 1>${QUIETREDIR} + gunzip -c < files/`look INDEX tINDEX.new | + cut -f 2 -d '|'`.gz > INDEX.new + fetch_index_sanity || return 1 + +# If we have decided to refuse certain updates, construct a hybrid index which +# is equal to the old index for parts of the tree which we don't want to +# update, and equal to the new index for parts of the tree which gets updates. +# This means that we should always have a "complete snapshot" of the ports +# tree -- with the caveat that it isn't actually a snapshot. + if [ ! -z "${REFUSE}" ]; then + echo "Refusing to download updates for ${REFUSE}" \ + >${QUIETREDIR} + + grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp + grep -E "${REFUSE}" INDEX | + sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new + rm -f INDEX.tmp + fi + +# Generate a list of wanted ports patches + echo -n "Generating list of wanted patches..." 1>${QUIETREDIR} + join -t '|' -o 1.2,2.2 INDEX INDEX.new | + fetch_make_patchlist > patchlist + echo " done." 1>${QUIETREDIR} + +# Attempt to fetch ports patches + patchcnt=`wc -l < patchlist | tr -d ' '` + echo -n "Fetching $patchcnt " + echo ${NDEBUG} "patches.${DDSTATS}" + echo " " + tr '|' '-' < patchlist | lam -s "bp/" - | + xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ + 2>${STATSREDIR} | fetch_progress_percent $patchcnt + echo "done." + +# Attempt to apply ports patches + PATCHCNT=`wc -l patchlist` + echo "Applying patches... " + local oldifs="$IFS" IFS='|' + I=0 + while read X Y; do + I=$(($I + 1)) + F="${X}-${Y}" + if [ ! -f "${F}" ]; then + XS=${X%[0-9a-f][0-9a-f][0-9a-f][0-9a-f]} + XE=${X#[0-9a-f][0-9a-f][0-9a-f][0-9a-f]} + YS=${Y%[0-9a-f][0-9a-f][0-9a-f][0-9a-f]} + YE=${Y#[0-9a-f][0-9a-f][0-9a-f][0-9a-f]} + F="${X%${XE}}...${X#${XS}}-${Y%${YE}}...${Y#${YS}}" + printf " Skipping ${F} (${I} of ${PATCHCNT}).\r" + continue + fi + echo " Processing ${F}..." 1>${QUIETREDIR} + gunzip -c < files/${X}.gz > OLD + ${BSPATCH} OLD NEW ${X}-${Y} + if [ `${SHA256} -q NEW` = ${Y} ]; then + mv NEW files/${Y} + gzip -n files/${Y} + fi + rm -f diff OLD NEW ${X}-${Y} + done < patchlist 2>${QUIETREDIR} + IFS="$oldifs" + echo "done." + +# Update ports without patches + join -t '|' -v 2 INDEX INDEX.new | + cut -f 2 -d '|' /dev/stdin patchlist | + while read Y; do + if [ ! -f "files/${Y}.gz" ]; then + echo ${Y} + fi + done > filelist + echo -n "Fetching `wc -l < filelist | tr -d ' '` " + echo ${NDEBUG} "new ports or files... " + lam -s "f/" - -s ".gz" < filelist | + xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \ + 2>${QUIETREDIR} + + I=0 + while read Y; do + I=$(($I + 1)) + printf " Processing ${Y} (${I} of ${PATCHCNT}).\r" 1>${QUIETREDIR} + if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then + mv ${Y}.gz files/${Y}.gz + else + echo "snapshot is corrupt." + return 1 + fi + done < filelist + echo "done." + +# Remove files which are no longer needed + cut -f 2 -d '|' tINDEX INDEX | sort -u > oldfiles + cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u | comm -13 - oldfiles | + lam -s "files/" - -s ".gz" | xargs rm -f + rm patchlist filelist oldfiles + +# We're done! + mv INDEX.new INDEX + mv tINDEX.new tINDEX + mv tag.new tag + + return 0 +} + +# Do the actual work involved in "fetch" / "cron". +fetch_run() { + fetch_pick_server_init && fetch_pick_server + + while ! fetch_key; do + fetch_pick_server || return 1 + done + + if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then + fetch_snapshot || return 1 + fi + fetch_update || return 1 +} + +# Build a ports INDEX file +extract_make_index() { + if ! look $1 ${WORKDIR}/tINDEX > /dev/null; then + echo -n "$1 not provided by portsnap server; " + echo "$2 not being generated." + else + gunzip -c < "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX | + cut -f 2 -d '|'`.gz" | + cat - ${LOCALDESC} | + ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2 + fi +} + +# Create INDEX, INDEX-5, INDEX-6 +extract_indices() { + echo -n "Building new INDEX files... " + for PAIR in ${INDEXPAIRS}; do + INDEXFILE=`echo ${PAIR} | cut -f 1 -d '|'` + DESCRIBEFILE=`echo ${PAIR} | cut -f 2 -d '|'` + extract_make_index ${DESCRIBEFILE} ${INDEXFILE} || return 1 + done + echo "done." +} + +# Create .portsnap.INDEX; if we are REFUSEing to touch certain directories, +# merge the values from any exiting .portsnap.INDEX file. +extract_metadata() { + if [ -z "${REFUSE}" ]; then + sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX + elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then + grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \ + > ${PORTSDIR}/.portsnap.INDEX.tmp + grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort | + sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \ + > ${PORTSDIR}/.portsnap.INDEX + rm -f ${PORTSDIR}/.portsnap.INDEX.tmp + else + grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \ + > ${PORTSDIR}/.portsnap.INDEX + fi +} + +# Do the actual work involved in "extract" +extract_run() { + local oldifs="$IFS" IFS='|' + mkdir -p ${PORTSDIR} || return 1 + + if ! + if ! [ -z "${EXTRACTPATH}" ]; then + grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX + elif ! [ -z "${REFUSE}" ]; then + grep -vE "${REFUSE}" ${WORKDIR}/INDEX + else + cat ${WORKDIR}/INDEX + fi | while read FILE HASH; do + echo ${PORTSDIR}/${FILE} + if ! [ -s "${WORKDIR}/files/${HASH}.gz" ]; then + echo "files/${HASH}.gz not found -- snapshot corrupt." + return 1 + fi + case ${FILE} in + */) + rm -rf ${PORTSDIR}/${FILE%/} + mkdir -p ${PORTSDIR}/${FILE} + tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ + -C ${PORTSDIR}/${FILE} + ;; + *) + rm -f ${PORTSDIR}/${FILE} + tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ + -C ${PORTSDIR} ${FILE} + ;; + esac + done; then + return 1 + fi + if [ ! -z "${EXTRACTPATH}" ]; then + return 0 + fi + + IFS="$oldifs" + + extract_metadata + extract_indices +} + +update_run_extract() { + local IFS='|' + +# Install new files + echo "Extracting new files:" + if ! + if ! [ -z "${REFUSE}" ]; then + grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort + else + sort ${WORKDIR}/INDEX + fi | + comm -13 ${PORTSDIR}/.portsnap.INDEX - | + while read FILE HASH; do + echo ${PORTSDIR}/${FILE} + if ! [ -s "${WORKDIR}/files/${HASH}.gz" ]; then + echo "files/${HASH}.gz not found -- snapshot corrupt." + return 1 + fi + case ${FILE} in + */) + mkdir -p ${PORTSDIR}/${FILE} + tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ + -C ${PORTSDIR}/${FILE} + ;; + *) + tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \ + -C ${PORTSDIR} ${FILE} + ;; + esac + done; then + return 1 + fi +} + +# Do the actual work involved in "update" +update_run() { + if ! [ -z "${INDEXONLY}" ]; then + extract_indices >/dev/null || return 1 + return 0 + fi + + if sort ${WORKDIR}/INDEX | + cmp -s ${PORTSDIR}/.portsnap.INDEX -; then + echo "Ports tree is already up to date." + return 0 + fi + +# If we are REFUSEing to touch certain directories, don't remove files +# from those directories (even if they are out of date) + echo -n "Removing old files and directories... " + if ! [ -z "${REFUSE}" ]; then + sort ${WORKDIR}/INDEX | + comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | + grep -vE "${REFUSE}" | + lam -s "${PORTSDIR}/" - | + sed -e 's|/$||' | xargs rm -rf + else + sort ${WORKDIR}/INDEX | + comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' | + lam -s "${PORTSDIR}/" - | + sed -e 's|/$||' | xargs rm -rf + fi + echo "done." + + update_run_extract || return 1 + extract_metadata + extract_indices +} + +#### Main functions -- call parameter-handling and core functions + +# Using the command line, configuration file, and defaults, +# set all the parameters which are needed later. +get_params() { + init_params + parse_cmdline $@ + sanity_conffile + default_conffile + parse_conffile + default_params +} + +# Fetch command. Make sure that we're being called +# interactively, then run fetch_check_params and fetch_run +cmd_fetch() { + if [ "${INTERACTIVE}" != "YES" ]; then + echo -n "`basename $0` fetch should not " + echo "be run non-interactively." + echo "Run `basename $0` cron instead" + exit 1 + fi + fetch_check_params + fetch_run || exit 1 +} + +# Cron command. Make sure the parameters are sensible; wait +# rand(3600) seconds; then fetch updates. While fetching updates, +# send output to a temporary file; only print that file if the +# fetching failed. +cmd_cron() { + fetch_check_params + sleep `jot -r 1 0 3600` + + TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1 + if ! fetch_run >> ${TMPFILE}; then + cat ${TMPFILE} + rm ${TMPFILE} + exit 1 + fi + + rm ${TMPFILE} +} + +# Extract command. Make sure the parameters are sensible, +# then extract the ports tree (or part thereof). +cmd_extract() { + extract_check_params + extract_run || exit 1 +} + +# Update command. Make sure the parameters are sensible, +# then update the ports tree. +cmd_update() { + update_check_params + update_run || exit 1 +} + +# Auto command. Run 'fetch' or 'cron' depending on +# whether stdin is a terminal; then run 'update' or +# 'extract' depending on whether ${PORTSDIR} exists. +cmd_auto() { + if [ "${INTERACTIVE}" = "YES" ]; then + cmd_fetch + else + cmd_cron + fi + if [ -r ${PORTSDIR}/.portsnap.INDEX ]; then + cmd_update + else + cmd_extract + fi +} + +#### Entry point + +# Make sure we find utilities from the base system +export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH} + +# Set LC_ALL in order to avoid problems with character ranges like [A-Z]. +export LC_ALL=C + +get_params $@ +for COMMAND in ${COMMANDS}; do + cmd_${COMMAND} +done Index: ports-mgmt/portsnap/pkg-descr =================================================================== --- /dev/null +++ ports-mgmt/portsnap/pkg-descr @@ -0,0 +1,3 @@ +The portsnap tool is used to fetch and update compressed +snapshots of the FreeBSD ports tree, and extract and update +an uncompressed ports tree.