Index: head/usr.bin/fetch/fetch.1 =================================================================== --- head/usr.bin/fetch/fetch.1 (revision 261233) +++ head/usr.bin/fetch/fetch.1 (revision 261234) @@ -1,417 +1,417 @@ .\"- -.\" Copyright (c) 2000-2013 Dag-Erling Smørgrav +.\" Copyright (c) 2000-2014 Dag-Erling Smørgrav .\" Copyright (c) 2013 Michael Gmelin .\" All rights reserved. .\" Portions Copyright (c) 1999 Massachusetts Institute of Technology; used .\" by permission. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer .\" in this position and unchanged. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" 3. The name of the author may not be used to endorse or promote products .\" derived from this software without specific prior written permission. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR .\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES .\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. .\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, .\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT .\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd July 30, 2013 +.Dd January 28, 2014 .Dt FETCH 1 .Os .Sh NAME .Nm fetch .Nd retrieve a file by Uniform Resource Locator .Sh SYNOPSIS .Nm .Op Fl 146AadFlMmnPpqRrsUv .Op Fl -allow-sslv2 .Op Fl B Ar bytes .Op Fl -bind-address= Ns Ar host .Op Fl -ca-cert= Ns Ar file .Op Fl -ca-path= Ns Ar dir .Op Fl -cert= Ns Ar file .Op Fl -crl= Ns Ar file .Op Fl i Ar file .Op Fl -key= Ns Ar file .Op Fl N Ar file .Op Fl -no-passive .Op Fl -no-proxy= Ns Ar list .Op Fl -no-sslv3 .Op Fl -no-tlsv1 .Op Fl -no-verify-hostname .Op Fl -no-verify-peer .Op Fl o Ar file .Op Fl -referer= Ns Ar URL .Op Fl S Ar bytes .Op Fl T Ar seconds .Op Fl -user-agent= Ns Ar agent-string .Op Fl w Ar seconds .Ar URL ... .Nm .Op Fl 146AadFlMmnPpqRrsUv .Op Fl B Ar bytes .Op Fl -bind-address= Ns Ar host .Op Fl -ca-cert= Ns Ar file .Op Fl -ca-path= Ns Ar dir .Op Fl -cert= Ns Ar file .Op Fl -crl= Ns Ar file .Op Fl i Ar file .Op Fl -key= Ns Ar file .Op Fl N Ar file .Op Fl -no-passive .Op Fl -no-proxy= Ns Ar list .Op Fl -no-sslv3 .Op Fl -no-tlsv1 .Op Fl -no-verify-hostname .Op Fl -no-verify-peer .Op Fl o Ar file .Op Fl -referer= Ns Ar URL .Op Fl S Ar bytes .Op Fl T Ar seconds .Op Fl -user-agent= Ns Ar agent-string .Op Fl w Ar seconds .Fl h Ar host Fl f Ar file Oo Fl c Ar dir Oc .Sh DESCRIPTION The .Nm utility provides a command-line interface to the .Xr fetch 3 library. Its purpose is to retrieve the file(s) pointed to by the URL(s) on the command line. .Pp The following options are available: .Bl -tag -width Fl .It Fl 1 , -one-file Stop and return exit code 0 at the first successfully retrieved file. .It Fl 4 , -ipv4-only Forces .Nm to use IPv4 addresses only. .It Fl 6 , -ipv6-only Forces .Nm to use IPv6 addresses only. .It Fl A , -no-redirect Do not automatically follow ``temporary'' (302) redirects. Some broken Web sites will return a redirect instead of a not-found error when the requested object does not exist. .It Fl a , -retry Automatically retry the transfer upon soft failures. .It Fl -allow-sslv2 [SSL] Allow SSL version 2 when negotiating the connection. .It Fl B Ar bytes , Fl -buffer-size= Ns Ar bytes Specify the read buffer size in bytes. -The default is 4096 bytes. +The default is 16,384 bytes. Attempts to set a buffer size lower than this will be silently ignored. The number of reads actually performed is reported at verbosity level two or higher (see the .Fl v flag). .It Fl -bind-address= Ns Ar host Specifies a hostname or IP address to which sockets used for outgoing connections will be bound. .It Fl c Ar dir The file to retrieve is in directory .Ar dir on the remote host. This option is deprecated and is provided for backward compatibility only. .It Fl -ca-cert= Ns Ar file [SSL] Path to certificate bundle containing trusted CA certificates. If not specified, .Pa /etc/ssl/cert.pem is used. The file may contain multiple CA certificates. The port .Pa security/ca_root_nss is a common source of a current CA bundle. .It Fl -ca-path= Ns Ar dir [SSL] The directory .Ar dir contains trusted CA hashes. .It Fl -cert= Ns Ar file [SSL] .Ar file is a PEM encoded client certificate/key which will be used in client certificate authentication. .It Fl -crl= Ns Ar file [SSL] Points to certificate revocation list .Ar file , which has to be in PEM format and may contain peer certificates that have been revoked. .It Fl d , -direct Use a direct connection even if a proxy is configured. .It Fl F , -force-restart In combination with the .Fl r flag, forces a restart even if the local and remote files have different modification times. Implies .Fl R . .It Fl f Ar file The file to retrieve is named .Ar file on the remote host. This option is deprecated and is provided for backward compatibility only. .It Fl h Ar host The file to retrieve is located on the host .Ar host . This option is deprecated and is provided for backward compatibility only. .It Fl i Ar file , Fl -if-modified-since= Ns Ar file If-Modified-Since mode: the remote file will only be retrieved if it is newer than .Ar file on the local host. (HTTP only) .It Fl -key= Ns Ar file [SSL] .Ar file is a PEM encoded client key that will be used in client certificate authentication in case key and client certificate are stored separately. .It Fl l , -symlink If the target is a file-scheme URL, make a symbolic link to the target rather than trying to copy it. .It Fl M .It Fl m , -mirror Mirror mode: if the file already exists locally and has the same size and modification time as the remote file, it will not be fetched. Note that the .Fl m and .Fl r flags are mutually exclusive. .It Fl N Ar file , Fl -netrc= Ns Ar file Use .Ar file instead of .Pa ~/.netrc to look up login names and passwords for FTP sites. See .Xr ftp 1 for a description of the file format. This feature is experimental. .It Fl n , -no-mtime Do not preserve the modification time of the transferred file. .It Fl -no-passive Forces the FTP code to use active mode. .It Fl -no-proxy= Ns Ar list Either a single asterisk, which disables the use of proxies altogether, or a comma- or whitespace-separated list of hosts for which proxies should not be used. .It Fl -no-sslv3 [SSL] Don't allow SSL version 3 when negotiating the connection. .It Fl -no-tlsv1 [SSL] Don't allow TLS version 1 when negotiating the connection. .It Fl -no-verify-hostname [SSL] Do not verify that the hostname matches the subject of the certificate presented by the server. .It Fl -no-verify-peer [SSL] Do not verify the peer certificate against trusted CAs. .It Fl o Ar file , Fl output= Ns Ar file Set the output file name to .Ar file . By default, a ``pathname'' is extracted from the specified URI, and its basename is used as the name of the output file. A .Ar file argument of .Sq Li \&- indicates that results are to be directed to the standard output. If the .Ar file argument is a directory, fetched file(s) will be placed within the directory, with name(s) selected as in the default behaviour. .It Fl P .It Fl p , -passive Use passive FTP. These flags have no effect, since passive FTP is the default, but are provided for compatibility with earlier versions where active FTP was the default. To force active mode, use the .Fl -no-passive flag or set the .Ev FTP_PASSIVE_MODE environment variable to .Ql NO . .It Fl -referer= Ns Ar URL Specifies the referrer URL to use for HTTP requests. If .Ar URL is set to .Dq auto , the document URL will be used as referrer URL. .It Fl q , -quiet Quiet mode. .It Fl R , -keep-output The output files are precious, and should not be deleted under any circumstances, even if the transfer failed or was incomplete. .It Fl r , -restart Restart a previously interrupted transfer. Note that the .Fl m and .Fl r flags are mutually exclusive. .It Fl S Ar bytes , Fl -require-size= Ns Ar bytes Require the file size reported by the server to match the specified value. If it does not, a message is printed and the file is not fetched. If the server does not support reporting file sizes, this option is ignored and the file is fetched unconditionally. .It Fl s , -print-size Print the size in bytes of each requested file, without fetching it. .It Fl T Ar seconds , Fl -timeout= Ns Ar seconds Set timeout value to .Ar seconds . Overrides the environment variables .Ev FTP_TIMEOUT for FTP transfers or .Ev HTTP_TIMEOUT for HTTP transfers if set. .It Fl U , -passive-portrange-default When using passive FTP, allocate the port for the data connection from the low (default) port range. See .Xr ip 4 for details on how to specify which port range this corresponds to. .It Fl -user-agent= Ns Ar agent-string Specifies the User-Agent string to use for HTTP requests. This can be useful when working with HTTP origin or proxy servers that differentiate between user agents. .It Fl v , -verbose Increase verbosity level. .It Fl w Ar seconds , Fl -retry-delay= Ns Ar seconds When the .Fl a flag is specified, wait this many seconds between successive retries. .El .Pp If .Nm receives a .Dv SIGINFO signal (see the .Cm status argument for .Xr stty 1 ) , the current transfer rate statistics will be written to the standard error output, in the same format as the standard completion message. .Sh ENVIRONMENT .Bl -tag -width HTTP_TIMEOUT .It Ev FTP_TIMEOUT Maximum time, in seconds, to wait before aborting an FTP connection. .It Ev HTTP_TIMEOUT Maximum time, in seconds, to wait before aborting an HTTP connection. .El .Pp See .Xr fetch 3 for a description of additional environment variables, including .Ev FETCH_BIND_ADDRESS , .Ev FTP_LOGIN , .Ev FTP_PASSIVE_MODE , .Ev FTP_PASSWORD , .Ev FTP_PROXY , .Ev ftp_proxy , .Ev HTTP_ACCEPT , .Ev HTTP_AUTH , .Ev HTTP_PROXY , .Ev http_proxy , .Ev HTTP_PROXY_AUTH , .Ev HTTP_REFERER , .Ev HTTP_USER_AGENT , .Ev NETRC , .Ev NO_PROXY , .Ev no_proxy , .Ev SSL_ALLOW_SSL2 , .Ev SSL_CA_CERT_FILE , .Ev SSL_CA_CERT_PATH , .Ev SSL_CLIENT_CERT_FILE , .Ev SSL_CLIENT_KEY_FILE , .Ev SSL_CRL_FILE , .Ev SSL_NO_SSL3 , .Ev SSL_NO_TLS1 , .Ev SSL_NO_VERIFY_HOSTNAME and .Ev SSL_NO_VERIFY_PEER . .Sh EXIT STATUS The .Nm command returns zero on success, or one on failure. If multiple URLs are listed on the command line, .Nm will attempt to retrieve each one of them in turn, and will return zero only if they were all successfully retrieved. .Pp If the .Fl i argument is used and the remote file is not newer than the specified file then the command will still return success, although no file is transferred. .Sh SEE ALSO .Xr fetch 3 .Sh HISTORY The .Nm command appeared in .Fx 2.1.5 . This implementation first appeared in .Fx 4.1 . .Sh AUTHORS .An -nosplit The original implementation of .Nm was done by .An Jean-Marc Zucconi Aq jmz@FreeBSD.org . It was extensively re-worked for .Fx 2.2 by .An Garrett Wollman Aq wollman@FreeBSD.org , and later completely rewritten to use the .Xr fetch 3 library by .An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org and .An Michael Gmelin Aq freebsd@grem.de . .Sh NOTES The .Fl b and .Fl t options are no longer supported and will generate warnings. They were workarounds for bugs in other OSes which this implementation does not trigger. .Pp One cannot both use the .Fl h , .Fl c and .Fl f options and specify URLs on the command line. Index: head/usr.bin/fetch/fetch.c =================================================================== --- head/usr.bin/fetch/fetch.c (revision 261233) +++ head/usr.bin/fetch/fetch.c (revision 261234) @@ -1,1179 +1,1180 @@ /*- * Copyright (c) 2000-2014 Dag-Erling Smørgrav * Copyright (c) 2013 Michael Gmelin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#define MINBUFSIZE 4096 +#define MINBUFSIZE 16384 #define TIMEOUT 120 /* Option flags */ static int A_flag; /* -A: do not follow 302 redirects */ static int a_flag; /* -a: auto retry */ static off_t B_size; /* -B: buffer size */ static int b_flag; /*! -b: workaround TCP bug */ static char *c_dirname; /* -c: remote directory */ static int d_flag; /* -d: direct connection */ static int F_flag; /* -F: restart without checking mtime */ static char *f_filename; /* -f: file to fetch */ static char *h_hostname; /* -h: host to fetch from */ static int i_flag; /* -i: specify file for mtime comparison */ static char *i_filename; /* name of input file */ static int l_flag; /* -l: link rather than copy file: URLs */ static int m_flag; /* -[Mm]: mirror mode */ static char *N_filename; /* -N: netrc file name */ static int n_flag; /* -n: do not preserve modification time */ static int o_flag; /* -o: specify output file */ static int o_directory; /* output file is a directory */ static char *o_filename; /* name of output file */ static int o_stdout; /* output file is stdout */ static int once_flag; /* -1: stop at first successful file */ static int p_flag; /* -[Pp]: use passive FTP */ static int R_flag; /* -R: don't delete partial files */ static int r_flag; /* -r: restart previous transfer */ static off_t S_size; /* -S: require size to match */ static int s_flag; /* -s: show size, don't fetch */ static long T_secs; /* -T: transfer timeout in seconds */ static int t_flag; /*! -t: workaround TCP bug */ static int U_flag; /* -U: do not use high ports */ static int v_level = 1; /* -v: verbosity level */ static int v_tty; /* stdout is a tty */ static pid_t pgrp; /* our process group */ static long w_secs; /* -w: retry delay */ static int family = PF_UNSPEC; /* -[46]: address family to use */ static int sigalrm; /* SIGALRM received */ static int siginfo; /* SIGINFO received */ static int sigint; /* SIGINT received */ static long ftp_timeout = TIMEOUT; /* default timeout for FTP transfers */ static long http_timeout = TIMEOUT;/* default timeout for HTTP transfers */ static char *buf; /* transfer buffer */ enum options { OPTION_BIND_ADDRESS, OPTION_NO_FTP_PASSIVE_MODE, OPTION_HTTP_REFERER, OPTION_HTTP_USER_AGENT, OPTION_NO_PROXY, OPTION_SSL_ALLOW_SSL2, OPTION_SSL_CA_CERT_FILE, OPTION_SSL_CA_CERT_PATH, OPTION_SSL_CLIENT_CERT_FILE, OPTION_SSL_CLIENT_KEY_FILE, OPTION_SSL_CRL_FILE, OPTION_SSL_NO_SSL3, OPTION_SSL_NO_TLS1, OPTION_SSL_NO_VERIFY_HOSTNAME, OPTION_SSL_NO_VERIFY_PEER }; static struct option longopts[] = { /* mapping to single character argument */ { "one-file", no_argument, NULL, '1' }, { "ipv4-only", no_argument, NULL, '4' }, { "ipv6-only", no_argument, NULL, '6' }, { "no-redirect", no_argument, NULL, 'A' }, { "retry", no_argument, NULL, 'a' }, { "buffer-size", required_argument, NULL, 'B' }, /* -c not mapped, since it's deprecated */ { "direct", no_argument, NULL, 'd' }, { "force-restart", no_argument, NULL, 'F' }, /* -f not mapped, since it's deprecated */ /* -h not mapped, since it's deprecated */ { "if-modified-since", required_argument, NULL, 'i' }, { "symlink", no_argument, NULL, 'l' }, /* -M not mapped since it's the same as -m */ { "mirror", no_argument, NULL, 'm' }, { "netrc", required_argument, NULL, 'N' }, { "no-mtime", no_argument, NULL, 'n' }, { "output", required_argument, NULL, 'o' }, /* -P not mapped since it's the same as -p */ { "passive", no_argument, NULL, 'p' }, { "quiet", no_argument, NULL, 'q' }, { "keep-output", no_argument, NULL, 'R' }, { "restart", no_argument, NULL, 'r' }, { "require-size", required_argument, NULL, 'S' }, { "print-size", no_argument, NULL, 's' }, { "timeout", required_argument, NULL, 'T' }, { "passive-portrange-default", no_argument, NULL, 'T' }, { "verbose", no_argument, NULL, 'v' }, { "retry-delay", required_argument, NULL, 'w' }, /* options without a single character equivalent */ { "bind-address", required_argument, NULL, OPTION_BIND_ADDRESS }, { "no-passive", no_argument, NULL, OPTION_NO_FTP_PASSIVE_MODE }, { "referer", required_argument, NULL, OPTION_HTTP_REFERER }, { "user-agent", required_argument, NULL, OPTION_HTTP_USER_AGENT }, { "no-proxy", required_argument, NULL, OPTION_NO_PROXY }, { "allow-sslv2", no_argument, NULL, OPTION_SSL_ALLOW_SSL2 }, { "ca-cert", required_argument, NULL, OPTION_SSL_CA_CERT_FILE }, { "ca-path", required_argument, NULL, OPTION_SSL_CA_CERT_PATH }, { "cert", required_argument, NULL, OPTION_SSL_CLIENT_CERT_FILE }, { "key", required_argument, NULL, OPTION_SSL_CLIENT_KEY_FILE }, { "crl", required_argument, NULL, OPTION_SSL_CRL_FILE }, { "no-sslv3", no_argument, NULL, OPTION_SSL_NO_SSL3 }, { "no-tlsv1", no_argument, NULL, OPTION_SSL_NO_TLS1 }, { "no-verify-hostname", no_argument, NULL, OPTION_SSL_NO_VERIFY_HOSTNAME }, { "no-verify-peer", no_argument, NULL, OPTION_SSL_NO_VERIFY_PEER }, { NULL, 0, NULL, 0 } }; /* * Signal handler */ static void sig_handler(int sig) { switch (sig) { case SIGALRM: sigalrm = 1; break; case SIGINFO: siginfo = 1; break; case SIGINT: sigint = 1; break; } } struct xferstat { char name[64]; struct timeval start; /* start of transfer */ struct timeval last; /* time of last update */ struct timeval last2; /* time of previous last update */ off_t size; /* size of file per HTTP hdr */ off_t offset; /* starting offset in file */ off_t rcvd; /* bytes already received */ off_t lastrcvd; /* bytes received since last update */ }; /* * Compute and display ETA */ static const char * stat_eta(struct xferstat *xs) { static char str[16]; long elapsed, eta; off_t received, expected; elapsed = xs->last.tv_sec - xs->start.tv_sec; received = xs->rcvd - xs->offset; expected = xs->size - xs->rcvd; eta = (long)((double)elapsed * expected / received); if (eta > 3600) snprintf(str, sizeof str, "%02ldh%02ldm", eta / 3600, (eta % 3600) / 60); else if (eta > 0) snprintf(str, sizeof str, "%02ldm%02lds", eta / 60, eta % 60); else snprintf(str, sizeof str, "%02ldm%02lds", elapsed / 60, elapsed % 60); return (str); } /* * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'... */ static const char *prefixes = " kMGTP"; static const char * stat_bytes(off_t bytes) { static char str[16]; const char *prefix = prefixes; while (bytes > 9999 && prefix[1] != '\0') { bytes /= 1024; prefix++; } snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix); return (str); } /* * Compute and display transfer rate */ static const char * stat_bps(struct xferstat *xs) { static char str[16]; double delta, bps; delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6)) - (xs->last2.tv_sec + (xs->last2.tv_usec / 1.e6)); if (delta == 0.0) { snprintf(str, sizeof str, "?? Bps"); } else { bps = (xs->rcvd - xs->lastrcvd) / delta; snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps)); } return (str); } /* * Update the stats display */ static void stat_display(struct xferstat *xs, int force) { struct timeval now; int ctty_pgrp; /* check if we're the foreground process */ if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || (pid_t)ctty_pgrp != pgrp) return; gettimeofday(&now, NULL); if (!force && now.tv_sec <= xs->last.tv_sec) return; xs->last2 = xs->last; xs->last = now; fprintf(stderr, "\r%-46.46s", xs->name); if (xs->size <= 0) { setproctitle("%s [%s]", xs->name, stat_bytes(xs->rcvd)); fprintf(stderr, " %s", stat_bytes(xs->rcvd)); } else { setproctitle("%s [%d%% of %s]", xs->name, (int)((100.0 * xs->rcvd) / xs->size), stat_bytes(xs->size)); fprintf(stderr, "%3d%% of %s", (int)((100.0 * xs->rcvd) / xs->size), stat_bytes(xs->size)); } if (force == 2) { xs->lastrcvd = xs->offset; xs->last2 = xs->start; } fprintf(stderr, " %s", stat_bps(xs)); if ((xs->size > 0 && xs->rcvd > 0 && xs->last.tv_sec >= xs->start.tv_sec + 3) || force == 2) fprintf(stderr, " %s", stat_eta(xs)); xs->lastrcvd = xs->rcvd; } /* * Initialize the transfer statistics */ static void stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) { snprintf(xs->name, sizeof xs->name, "%s", name); gettimeofday(&xs->start, NULL); xs->last.tv_sec = xs->last.tv_usec = 0; xs->size = size; xs->offset = offset; xs->rcvd = offset; xs->lastrcvd = offset; if (v_tty && v_level > 0) stat_display(xs, 1); else if (v_level > 0) fprintf(stderr, "%-46s", xs->name); } /* * Update the transfer statistics */ static void stat_update(struct xferstat *xs, off_t rcvd) { xs->rcvd = rcvd; if (v_tty && v_level > 0) stat_display(xs, 0); } /* * Finalize the transfer statistics */ static void stat_end(struct xferstat *xs) { gettimeofday(&xs->last, NULL); if (v_tty && v_level > 0) { stat_display(xs, 2); putc('\n', stderr); } else if (v_level > 0) { fprintf(stderr, " %s %s\n", stat_bytes(xs->size), stat_bps(xs)); } } /* * Ask the user for authentication details */ static int query_auth(struct url *URL) { struct termios tios; tcflag_t saved_flags; int i, nopwd; fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", URL->scheme, URL->host, URL->port); fprintf(stderr, "Login: "); if (fgets(URL->user, sizeof URL->user, stdin) == NULL) return (-1); for (i = strlen(URL->user); i >= 0; --i) if (URL->user[i] == '\r' || URL->user[i] == '\n') URL->user[i] = '\0'; fprintf(stderr, "Password: "); if (tcgetattr(STDIN_FILENO, &tios) == 0) { saved_flags = tios.c_lflag; tios.c_lflag &= ~ECHO; tios.c_lflag |= ECHONL|ICANON; tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); tios.c_lflag = saved_flags; tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); } else { nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); } if (nopwd) return (-1); for (i = strlen(URL->pwd); i >= 0; --i) if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n') URL->pwd[i] = '\0'; return (0); } /* * Fetch a file */ static int fetch(char *URL, const char *path) { struct url *url; struct url_stat us; struct stat sb, nsb; struct xferstat xs; FILE *f, *of; size_t size, readcnt, wr; off_t count; char flags[8]; const char *slash; char *tmppath; int r; unsigned timeout; char *ptr; f = of = NULL; tmppath = NULL; timeout = 0; *flags = 0; count = 0; /* set verbosity level */ if (v_level > 1) strcat(flags, "v"); if (v_level > 2) fetchDebug = 1; /* parse URL */ url = NULL; if (*URL == '\0') { warnx("empty URL"); goto failure; } if ((url = fetchParseURL(URL)) == NULL) { warnx("%s: parse error", URL); goto failure; } /* if no scheme was specified, take a guess */ if (!*url->scheme) { if (!*url->host) strcpy(url->scheme, SCHEME_FILE); else if (strncasecmp(url->host, "ftp.", 4) == 0) strcpy(url->scheme, SCHEME_FTP); else if (strncasecmp(url->host, "www.", 4) == 0) strcpy(url->scheme, SCHEME_HTTP); } /* common flags */ switch (family) { case PF_INET: strcat(flags, "4"); break; case PF_INET6: strcat(flags, "6"); break; } /* FTP specific flags */ if (strcmp(url->scheme, SCHEME_FTP) == 0) { if (p_flag) strcat(flags, "p"); if (d_flag) strcat(flags, "d"); if (U_flag) strcat(flags, "l"); timeout = T_secs ? T_secs : ftp_timeout; } /* HTTP specific flags */ if (strcmp(url->scheme, SCHEME_HTTP) == 0 || strcmp(url->scheme, SCHEME_HTTPS) == 0) { if (d_flag) strcat(flags, "d"); if (A_flag) strcat(flags, "A"); timeout = T_secs ? T_secs : http_timeout; if (i_flag) { if (stat(i_filename, &sb)) { warn("%s: stat()", i_filename); goto failure; } url->ims_time = sb.st_mtime; strcat(flags, "i"); } } /* set the protocol timeout. */ fetchTimeout = timeout; /* just print size */ if (s_flag) { if (timeout) alarm(timeout); r = fetchStat(url, &us, flags); if (timeout) alarm(0); if (sigalrm || sigint) goto signal; if (r == -1) { warnx("%s", fetchLastErrString); goto failure; } if (us.size == -1) printf("Unknown\n"); else printf("%jd\n", (intmax_t)us.size); goto success; } /* * If the -r flag was specified, we have to compare the local * and remote files, so we should really do a fetchStat() * first, but I know of at least one HTTP server that only * sends the content size in response to GET requests, and * leaves it out of replies to HEAD requests. Also, in the * (frequent) case that the local and remote files match but * the local file is truncated, we have sufficient information * before the compare to issue a correct request. Therefore, * we always issue a GET request as if we were sure the local * file was a truncated copy of the remote file; we can drop * the connection later if we change our minds. */ sb.st_size = -1; if (!o_stdout) { r = stat(path, &sb); if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { url->offset = sb.st_size; } else if (r == -1 || !S_ISREG(sb.st_mode)) { /* * Whatever value sb.st_size has now is either * wrong (if stat(2) failed) or irrelevant (if the * path does not refer to a regular file) */ sb.st_size = -1; } if (r == -1 && errno != ENOENT) { warnx("%s: stat()", path); goto failure; } } /* start the transfer */ if (timeout) alarm(timeout); f = fetchXGet(url, &us, flags); if (timeout) alarm(0); if (sigalrm || sigint) goto signal; if (f == NULL) { warnx("%s: %s", URL, fetchLastErrString); if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0 && fetchLastErrCode == FETCH_OK && strcmp(fetchLastErrString, "Not Modified") == 0) { /* HTTP Not Modified Response, return OK. */ r = 0; goto done; } else goto failure; } if (sigint) goto signal; /* check that size is as expected */ if (S_size) { if (us.size == -1) { warnx("%s: size unknown", URL); } else if (us.size != S_size) { warnx("%s: size mismatch: expected %jd, actual %jd", URL, (intmax_t)S_size, (intmax_t)us.size); goto failure; } } /* symlink instead of copy */ if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { if (symlink(url->doc, path) == -1) { warn("%s: symlink()", path); goto failure; } goto success; } if (us.size == -1 && !o_stdout && v_level > 0) warnx("%s: size of remote file is not known", URL); if (v_level > 1) { if (sb.st_size != -1) fprintf(stderr, "local size / mtime: %jd / %ld\n", (intmax_t)sb.st_size, (long)sb.st_mtime); if (us.size != -1) fprintf(stderr, "remote size / mtime: %jd / %ld\n", (intmax_t)us.size, (long)us.mtime); } /* open output file */ if (o_stdout) { /* output to stdout */ of = stdout; } else if (r_flag && sb.st_size != -1) { /* resume mode, local file exists */ if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { /* no match! have to refetch */ fclose(f); /* if precious, warn the user and give up */ if (R_flag) { warnx("%s: local modification time " "does not match remote", path); goto failure_keep; } } else if (url->offset > sb.st_size) { /* gap between what we asked for and what we got */ warnx("%s: gap in resume mode", URL); fclose(of); of = NULL; /* picked up again later */ } else if (us.size != -1) { if (us.size == sb.st_size) /* nothing to do */ goto success; if (sb.st_size > us.size) { /* local file too long! */ warnx("%s: local file (%jd bytes) is longer " "than remote file (%jd bytes)", path, (intmax_t)sb.st_size, (intmax_t)us.size); goto failure; } /* we got it, open local file */ if ((of = fopen(path, "r+")) == NULL) { warn("%s: fopen()", path); goto failure; } /* check that it didn't move under our feet */ if (fstat(fileno(of), &nsb) == -1) { /* can't happen! */ warn("%s: fstat()", path); goto failure; } if (nsb.st_dev != sb.st_dev || nsb.st_ino != sb.st_ino || nsb.st_size != sb.st_size) { warnx("%s: file has changed", URL); fclose(of); of = NULL; sb = nsb; /* picked up again later */ } } /* seek to where we left off */ if (of != NULL && fseeko(of, url->offset, SEEK_SET) != 0) { warn("%s: fseeko()", path); fclose(of); of = NULL; /* picked up again later */ } } else if (m_flag && sb.st_size != -1) { /* mirror mode, local file exists */ if (sb.st_size == us.size && sb.st_mtime == us.mtime) goto success; } if (of == NULL) { /* * We don't yet have an output file; either this is a * vanilla run with no special flags, or the local and * remote files didn't match. */ if (url->offset > 0) { /* * We tried to restart a transfer, but for * some reason gave up - so we have to restart * from scratch if we want the whole file */ url->offset = 0; if ((f = fetchXGet(url, &us, flags)) == NULL) { warnx("%s: %s", URL, fetchLastErrString); goto failure; } if (sigint) goto signal; } /* construct a temp file name */ if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { if ((slash = strrchr(path, '/')) == NULL) slash = path; else ++slash; asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", (int)(slash - path), path, slash); if (tmppath != NULL) { if (mkstemps(tmppath, strlen(slash) + 1) == -1) { warn("%s: mkstemps()", path); goto failure; } of = fopen(tmppath, "w"); chown(tmppath, sb.st_uid, sb.st_gid); chmod(tmppath, sb.st_mode & ALLPERMS); } } if (of == NULL) of = fopen(path, "w"); if (of == NULL) { warn("%s: open()", path); goto failure; } } count = url->offset; /* start the counter */ stat_start(&xs, path, us.size, count); sigalrm = siginfo = sigint = 0; /* suck in the data */ + setvbuf(f, NULL, _IOFBF, B_size); signal(SIGINFO, sig_handler); while (!sigint) { if (us.size != -1 && us.size - count < B_size && us.size - count >= 0) size = us.size - count; else size = B_size; if (siginfo) { stat_end(&xs); siginfo = 0; } if (size == 0) break; if ((readcnt = fread(buf, 1, size, f)) < size) { if (ferror(f) && errno == EINTR && !sigint) clearerr(f); else if (readcnt == 0) break; } stat_update(&xs, count += readcnt); for (ptr = buf; readcnt > 0; ptr += wr, readcnt -= wr) if ((wr = fwrite(ptr, 1, readcnt, of)) < readcnt) { if (ferror(of) && errno == EINTR && !sigint) clearerr(of); else break; } if (readcnt != 0) break; } if (!sigalrm) sigalrm = ferror(f) && errno == ETIMEDOUT; signal(SIGINFO, SIG_DFL); stat_end(&xs); /* * If the transfer timed out or was interrupted, we still want to * set the mtime in case the file is not removed (-r or -R) and * the user later restarts the transfer. */ signal: /* set mtime of local file */ if (!n_flag && us.mtime && !o_stdout && of != NULL && (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { struct timeval tv[2]; fflush(of); tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); tv[1].tv_sec = (long)us.mtime; tv[0].tv_usec = tv[1].tv_usec = 0; if (utimes(tmppath ? tmppath : path, tv)) warn("%s: utimes()", tmppath ? tmppath : path); } /* timed out or interrupted? */ if (sigalrm) warnx("transfer timed out"); if (sigint) { warnx("transfer interrupted"); goto failure; } /* timeout / interrupt before connection completley established? */ if (f == NULL) goto failure; if (!sigalrm) { /* check the status of our files */ if (ferror(f)) warn("%s", URL); if (ferror(of)) warn("%s", path); if (ferror(f) || ferror(of)) goto failure; } /* did the transfer complete normally? */ if (us.size != -1 && count < us.size) { warnx("%s appears to be truncated: %jd/%jd bytes", path, (intmax_t)count, (intmax_t)us.size); goto failure_keep; } /* * If the transfer timed out and we didn't know how much to * expect, assume the worst (i.e. we didn't get all of it) */ if (sigalrm && us.size == -1) { warnx("%s may be truncated", path); goto failure_keep; } success: r = 0; if (tmppath != NULL && rename(tmppath, path) == -1) { warn("%s: rename()", path); goto failure_keep; } goto done; failure: if (of && of != stdout && !R_flag && !r_flag) if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) unlink(tmppath ? tmppath : path); if (R_flag && tmppath != NULL && sb.st_size == -1) rename(tmppath, path); /* ignore errors here */ failure_keep: r = -1; goto done; done: if (f) fclose(f); if (of && of != stdout) fclose(of); if (url) fetchFreeURL(url); if (tmppath != NULL) free(tmppath); return (r); } static void usage(void) { fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", "usage: fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]", " [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]", " [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]", " [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]", " [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]", " [--user-agent=agent-string] [-w seconds] URL ...", " fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]", " [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]", " [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]", " [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]", " [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]", " [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]"); } /* * Entry point */ int main(int argc, char *argv[]) { struct stat sb; struct sigaction sa; const char *p, *s; char *end, *q; int c, e, r; while ((c = getopt_long(argc, argv, "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:", longopts, NULL)) != -1) switch (c) { case '1': once_flag = 1; break; case '4': family = PF_INET; break; case '6': family = PF_INET6; break; case 'A': A_flag = 1; break; case 'a': a_flag = 1; break; case 'B': B_size = (off_t)strtol(optarg, &end, 10); if (*optarg == '\0' || *end != '\0') errx(1, "invalid buffer size (%s)", optarg); break; case 'b': warnx("warning: the -b option is deprecated"); b_flag = 1; break; case 'c': c_dirname = optarg; break; case 'd': d_flag = 1; break; case 'F': F_flag = 1; break; case 'f': f_filename = optarg; break; case 'H': warnx("the -H option is now implicit, " "use -U to disable"); break; case 'h': h_hostname = optarg; break; case 'i': i_flag = 1; i_filename = optarg; break; case 'l': l_flag = 1; break; case 'o': o_flag = 1; o_filename = optarg; break; case 'M': case 'm': if (r_flag) errx(1, "the -m and -r flags " "are mutually exclusive"); m_flag = 1; break; case 'N': N_filename = optarg; break; case 'n': n_flag = 1; break; case 'P': case 'p': p_flag = 1; break; case 'q': v_level = 0; break; case 'R': R_flag = 1; break; case 'r': if (m_flag) errx(1, "the -m and -r flags " "are mutually exclusive"); r_flag = 1; break; case 'S': S_size = (off_t)strtol(optarg, &end, 10); if (*optarg == '\0' || *end != '\0') errx(1, "invalid size (%s)", optarg); break; case 's': s_flag = 1; break; case 'T': T_secs = strtol(optarg, &end, 10); if (*optarg == '\0' || *end != '\0') errx(1, "invalid timeout (%s)", optarg); break; case 't': t_flag = 1; warnx("warning: the -t option is deprecated"); break; case 'U': U_flag = 1; break; case 'v': v_level++; break; case 'w': a_flag = 1; w_secs = strtol(optarg, &end, 10); if (*optarg == '\0' || *end != '\0') errx(1, "invalid delay (%s)", optarg); break; case OPTION_BIND_ADDRESS: setenv("FETCH_BIND_ADDRESS", optarg, 1); break; case OPTION_NO_FTP_PASSIVE_MODE: setenv("FTP_PASSIVE_MODE", "no", 1); break; case OPTION_HTTP_REFERER: setenv("HTTP_REFERER", optarg, 1); break; case OPTION_HTTP_USER_AGENT: setenv("HTTP_USER_AGENT", optarg, 1); break; case OPTION_NO_PROXY: setenv("NO_PROXY", optarg, 1); break; case OPTION_SSL_ALLOW_SSL2: setenv("SSL_ALLOW_SSL2", "", 1); break; case OPTION_SSL_CA_CERT_FILE: setenv("SSL_CA_CERT_FILE", optarg, 1); break; case OPTION_SSL_CA_CERT_PATH: setenv("SSL_CA_CERT_PATH", optarg, 1); break; case OPTION_SSL_CLIENT_CERT_FILE: setenv("SSL_CLIENT_CERT_FILE", optarg, 1); break; case OPTION_SSL_CLIENT_KEY_FILE: setenv("SSL_CLIENT_KEY_FILE", optarg, 1); break; case OPTION_SSL_CRL_FILE: setenv("SSL_CLIENT_CRL_FILE", optarg, 1); break; case OPTION_SSL_NO_SSL3: setenv("SSL_NO_SSL3", "", 1); break; case OPTION_SSL_NO_TLS1: setenv("SSL_NO_TLS1", "", 1); break; case OPTION_SSL_NO_VERIFY_HOSTNAME: setenv("SSL_NO_VERIFY_HOSTNAME", "", 1); break; case OPTION_SSL_NO_VERIFY_PEER: setenv("SSL_NO_VERIFY_PEER", "", 1); break; default: usage(); exit(1); } argc -= optind; argv += optind; if (h_hostname || f_filename || c_dirname) { if (!h_hostname || !f_filename || argc) { usage(); exit(1); } /* XXX this is a hack. */ if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) errx(1, "invalid hostname"); if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, c_dirname ? c_dirname : "", f_filename) == -1) errx(1, "%s", strerror(ENOMEM)); argc++; } if (!argc) { usage(); exit(1); } /* allocate buffer */ if (B_size < MINBUFSIZE) B_size = MINBUFSIZE; if ((buf = malloc(B_size)) == NULL) errx(1, "%s", strerror(ENOMEM)); /* timeouts */ if ((s = getenv("FTP_TIMEOUT")) != NULL) { ftp_timeout = strtol(s, &end, 10); if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { warnx("FTP_TIMEOUT (%s) is not a positive integer", s); ftp_timeout = 0; } } if ((s = getenv("HTTP_TIMEOUT")) != NULL) { http_timeout = strtol(s, &end, 10); if (*s == '\0' || *end != '\0' || http_timeout < 0) { warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); http_timeout = 0; } } /* signal handling */ sa.sa_flags = 0; sa.sa_handler = sig_handler; sigemptyset(&sa.sa_mask); sigaction(SIGALRM, &sa, NULL); sa.sa_flags = SA_RESETHAND; sigaction(SIGINT, &sa, NULL); fetchRestartCalls = 0; /* output file */ if (o_flag) { if (strcmp(o_filename, "-") == 0) { o_stdout = 1; } else if (stat(o_filename, &sb) == -1) { if (errno == ENOENT) { if (argc > 1) errx(1, "%s is not a directory", o_filename); } else { err(1, "%s", o_filename); } } else { if (sb.st_mode & S_IFDIR) o_directory = 1; } } /* check if output is to a tty (for progress report) */ v_tty = isatty(STDERR_FILENO); if (v_tty) pgrp = getpgrp(); r = 0; /* authentication */ if (v_tty) fetchAuthMethod = query_auth; if (N_filename != NULL) if (setenv("NETRC", N_filename, 1) == -1) err(1, "setenv: cannot set NETRC=%s", N_filename); while (argc) { if ((p = strrchr(*argv, '/')) == NULL) p = *argv; else p++; if (!*p) p = "fetch.out"; fetchLastErrCode = 0; if (o_flag) { if (o_stdout) { e = fetch(*argv, "-"); } else if (o_directory) { asprintf(&q, "%s/%s", o_filename, p); e = fetch(*argv, q); free(q); } else { e = fetch(*argv, o_filename); } } else { e = fetch(*argv, p); } if (sigint) kill(getpid(), SIGINT); if (e == 0 && once_flag) exit(0); if (e) { r = 1; if ((fetchLastErrCode && fetchLastErrCode != FETCH_UNAVAIL && fetchLastErrCode != FETCH_MOVED && fetchLastErrCode != FETCH_URL && fetchLastErrCode != FETCH_RESOLV && fetchLastErrCode != FETCH_UNKNOWN)) { if (w_secs && v_level) fprintf(stderr, "Waiting %ld seconds " "before retrying\n", w_secs); if (w_secs) sleep(w_secs); if (a_flag) continue; } } argc--, argv++; } exit(r); }