Index: head/lib/libfetch/common.c =================================================================== --- head/lib/libfetch/common.c (revision 280629) +++ head/lib/libfetch/common.c (revision 280630) @@ -1,1387 +1,1385 @@ /*- * Copyright (c) 1998-2014 Dag-Erling Smørgrav * Copyright (c) 2013 Michael Gmelin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef WITH_SSL #include #endif #include "fetch.h" #include "common.h" /*** Local data **************************************************************/ /* * Error messages for resolver errors */ static struct fetcherr netdb_errlist[] = { #ifdef EAI_NODATA { EAI_NODATA, FETCH_RESOLV, "Host not found" }, #endif { EAI_AGAIN, FETCH_TEMP, "Transient resolver failure" }, { EAI_FAIL, FETCH_RESOLV, "Non-recoverable resolver failure" }, { EAI_NONAME, FETCH_RESOLV, "No address record" }, { -1, FETCH_UNKNOWN, "Unknown resolver error" } }; /* End-of-Line */ static const char ENDL[2] = "\r\n"; /*** Error-reporting functions ***********************************************/ /* * Map error code to string */ static struct fetcherr * fetch_finderr(struct fetcherr *p, int e) { while (p->num != -1 && p->num != e) p++; return (p); } /* * Set error code */ void fetch_seterr(struct fetcherr *p, int e) { p = fetch_finderr(p, e); fetchLastErrCode = p->cat; snprintf(fetchLastErrString, MAXERRSTRING, "%s", p->string); } /* * Set error code according to errno */ void fetch_syserr(void) { switch (errno) { case 0: fetchLastErrCode = FETCH_OK; break; case EPERM: case EACCES: case EROFS: case EAUTH: case ENEEDAUTH: fetchLastErrCode = FETCH_AUTH; break; case ENOENT: case EISDIR: /* XXX */ fetchLastErrCode = FETCH_UNAVAIL; break; case ENOMEM: fetchLastErrCode = FETCH_MEMORY; break; case EBUSY: case EAGAIN: fetchLastErrCode = FETCH_TEMP; break; case EEXIST: fetchLastErrCode = FETCH_EXISTS; break; case ENOSPC: fetchLastErrCode = FETCH_FULL; break; case EADDRINUSE: case EADDRNOTAVAIL: case ENETDOWN: case ENETUNREACH: case ENETRESET: case EHOSTUNREACH: fetchLastErrCode = FETCH_NETWORK; break; case ECONNABORTED: case ECONNRESET: fetchLastErrCode = FETCH_ABORT; break; case ETIMEDOUT: fetchLastErrCode = FETCH_TIMEOUT; break; case ECONNREFUSED: case EHOSTDOWN: fetchLastErrCode = FETCH_DOWN; break; default: fetchLastErrCode = FETCH_UNKNOWN; } snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno)); } /* * Emit status message */ void fetch_info(const char *fmt, ...) { va_list ap; va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); fputc('\n', stderr); } /*** Network-related utility functions ***************************************/ /* * Return the default port for a scheme */ int fetch_default_port(const char *scheme) { struct servent *se; if ((se = getservbyname(scheme, "tcp")) != NULL) return (ntohs(se->s_port)); if (strcasecmp(scheme, SCHEME_FTP) == 0) return (FTP_DEFAULT_PORT); if (strcasecmp(scheme, SCHEME_HTTP) == 0) return (HTTP_DEFAULT_PORT); return (0); } /* * Return the default proxy port for a scheme */ int fetch_default_proxy_port(const char *scheme) { if (strcasecmp(scheme, SCHEME_FTP) == 0) return (FTP_DEFAULT_PROXY_PORT); if (strcasecmp(scheme, SCHEME_HTTP) == 0) return (HTTP_DEFAULT_PROXY_PORT); return (0); } /* * Create a connection for an existing descriptor. */ conn_t * fetch_reopen(int sd) { conn_t *conn; int opt = 1; /* allocate and fill connection structure */ if ((conn = calloc(1, sizeof(*conn))) == NULL) return (NULL); fcntl(sd, F_SETFD, FD_CLOEXEC); setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE, &opt, sizeof opt); conn->sd = sd; ++conn->ref; return (conn); } /* * Bump a connection's reference count. */ conn_t * fetch_ref(conn_t *conn) { ++conn->ref; return (conn); } /* * Bind a socket to a specific local address */ int fetch_bind(int sd, int af, const char *addr) { struct addrinfo hints, *res, *res0; int err; memset(&hints, 0, sizeof(hints)); hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; if ((err = getaddrinfo(addr, NULL, &hints, &res0)) != 0) return (-1); for (res = res0; res; res = res->ai_next) if (bind(sd, res->ai_addr, res->ai_addrlen) == 0) return (0); return (-1); } /* * Establish a TCP connection to the specified port on the specified host. */ conn_t * fetch_connect(const char *host, int port, int af, int verbose) { conn_t *conn; char pbuf[10]; const char *bindaddr; struct addrinfo hints, *res, *res0; int sd, err; DEBUG(fprintf(stderr, "---> %s:%d\n", host, port)); if (verbose) fetch_info("looking up %s", host); /* look up host name and set up socket address structure */ snprintf(pbuf, sizeof(pbuf), "%d", port); memset(&hints, 0, sizeof(hints)); hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; if ((err = getaddrinfo(host, pbuf, &hints, &res0)) != 0) { netdb_seterr(err); return (NULL); } bindaddr = getenv("FETCH_BIND_ADDRESS"); if (verbose) fetch_info("connecting to %s:%d", host, port); /* try to connect */ for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) { if ((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1) continue; if (bindaddr != NULL && *bindaddr != '\0' && fetch_bind(sd, res->ai_family, bindaddr) != 0) { fetch_info("failed to bind to '%s'", bindaddr); close(sd); continue; } if (connect(sd, res->ai_addr, res->ai_addrlen) == 0 && fcntl(sd, F_SETFL, O_NONBLOCK) == 0) break; close(sd); } freeaddrinfo(res0); if (sd == -1) { fetch_syserr(); return (NULL); } if ((conn = fetch_reopen(sd)) == NULL) { fetch_syserr(); close(sd); } return (conn); } #ifdef WITH_SSL /* * Convert characters A-Z to lowercase (intentionally avoid any locale * specific conversions). */ static char fetch_ssl_tolower(char in) { if (in >= 'A' && in <= 'Z') return (in + 32); else return (in); } /* * isalpha implementation that intentionally avoids any locale specific * conversions. */ static int fetch_ssl_isalpha(char in) { return ((in >= 'A' && in <= 'Z') || (in >= 'a' && in <= 'z')); } /* * Check if passed hostnames a and b are equal. */ static int fetch_ssl_hname_equal(const char *a, size_t alen, const char *b, size_t blen) { size_t i; if (alen != blen) return (0); for (i = 0; i < alen; ++i) { if (fetch_ssl_tolower(a[i]) != fetch_ssl_tolower(b[i])) return (0); } return (1); } /* * Check if domain label is traditional, meaning that only A-Z, a-z, 0-9 * and '-' (hyphen) are allowed. Hyphens have to be surrounded by alpha- * numeric characters. Double hyphens (like they're found in IDN a-labels * 'xn--') are not allowed. Empty labels are invalid. */ static int fetch_ssl_is_trad_domain_label(const char *l, size_t len, int wcok) { size_t i; if (!len || l[0] == '-' || l[len-1] == '-') return (0); for (i = 0; i < len; ++i) { if (!isdigit(l[i]) && !fetch_ssl_isalpha(l[i]) && !(l[i] == '*' && wcok) && !(l[i] == '-' && l[i - 1] != '-')) return (0); } return (1); } /* * Check if host name consists only of numbers. This might indicate an IP * address, which is not a good idea for CN wildcard comparison. */ static int fetch_ssl_hname_is_only_numbers(const char *hostname, size_t len) { size_t i; for (i = 0; i < len; ++i) { if (!((hostname[i] >= '0' && hostname[i] <= '9') || hostname[i] == '.')) return (0); } return (1); } /* * Check if the host name h passed matches the pattern passed in m which * is usually part of subjectAltName or CN of a certificate presented to * the client. This includes wildcard matching. The algorithm is based on * RFC6125, sections 6.4.3 and 7.2, which clarifies RFC2818 and RFC3280. */ static int fetch_ssl_hname_match(const char *h, size_t hlen, const char *m, size_t mlen) { int delta, hdotidx, mdot1idx, wcidx; const char *hdot, *mdot1, *mdot2; const char *wc; /* wildcard */ if (!(h && *h && m && *m)) return (0); if ((wc = strnstr(m, "*", mlen)) == NULL) return (fetch_ssl_hname_equal(h, hlen, m, mlen)); wcidx = wc - m; /* hostname should not be just dots and numbers */ if (fetch_ssl_hname_is_only_numbers(h, hlen)) return (0); /* only one wildcard allowed in pattern */ if (strnstr(wc + 1, "*", mlen - wcidx - 1) != NULL) return (0); /* * there must be at least two more domain labels and * wildcard has to be in the leftmost label (RFC6125) */ mdot1 = strnstr(m, ".", mlen); if (mdot1 == NULL || mdot1 < wc || (mlen - (mdot1 - m)) < 4) return (0); mdot1idx = mdot1 - m; mdot2 = strnstr(mdot1 + 1, ".", mlen - mdot1idx - 1); if (mdot2 == NULL || (mlen - (mdot2 - m)) < 2) return (0); /* hostname must contain a dot and not be the 1st char */ hdot = strnstr(h, ".", hlen); if (hdot == NULL || hdot == h) return (0); hdotidx = hdot - h; /* * host part of hostname must be at least as long as * pattern it's supposed to match */ if (hdotidx < mdot1idx) return (0); /* * don't allow wildcards in non-traditional domain names * (IDN, A-label, U-label...) */ if (!fetch_ssl_is_trad_domain_label(h, hdotidx, 0) || !fetch_ssl_is_trad_domain_label(m, mdot1idx, 1)) return (0); /* match domain part (part after first dot) */ if (!fetch_ssl_hname_equal(hdot, hlen - hdotidx, mdot1, mlen - mdot1idx)) return (0); /* match part left of wildcard */ if (!fetch_ssl_hname_equal(h, wcidx, m, wcidx)) return (0); /* match part right of wildcard */ delta = mdot1idx - wcidx - 1; if (!fetch_ssl_hname_equal(hdot - delta, delta, mdot1 - delta, delta)) return (0); /* all tests succeded, it's a match */ return (1); } /* * Get numeric host address info - returns NULL if host was not an IP * address. The caller is responsible for deallocation using * freeaddrinfo(3). */ static struct addrinfo * fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len) { struct addrinfo hints, *res; char *host; host = (char *)malloc(len + 1); memcpy(host, hostname, len); host[len] = '\0'; memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; hints.ai_flags = AI_NUMERICHOST; /* port is not relevant for this purpose */ getaddrinfo(host, "443", &hints, &res); free(host); return res; } /* * Compare ip address in addrinfo with address passes. */ static int fetch_ssl_ipaddr_match_bin(const struct addrinfo *lhost, const char *rhost, size_t rhostlen) { const void *left; if (lhost->ai_family == AF_INET && rhostlen == 4) { left = (void *)&((struct sockaddr_in*)(void *) lhost->ai_addr)->sin_addr.s_addr; #ifdef INET6 } else if (lhost->ai_family == AF_INET6 && rhostlen == 16) { left = (void *)&((struct sockaddr_in6 *)(void *) lhost->ai_addr)->sin6_addr; #endif } else return (0); return (!memcmp(left, (const void *)rhost, rhostlen) ? 1 : 0); } /* * Compare ip address in addrinfo with host passed. If host is not an IP * address, comparison will fail. */ static int fetch_ssl_ipaddr_match(const struct addrinfo *laddr, const char *r, size_t rlen) { struct addrinfo *raddr; int ret; char *rip; ret = 0; if ((raddr = fetch_ssl_get_numeric_addrinfo(r, rlen)) == NULL) return 0; /* not a numeric host */ if (laddr->ai_family == raddr->ai_family) { if (laddr->ai_family == AF_INET) { rip = (char *)&((struct sockaddr_in *)(void *) raddr->ai_addr)->sin_addr.s_addr; ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 4); #ifdef INET6 } else if (laddr->ai_family == AF_INET6) { rip = (char *)&((struct sockaddr_in6 *)(void *) raddr->ai_addr)->sin6_addr; ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 16); #endif } } freeaddrinfo(raddr); return (ret); } /* * Verify server certificate by subjectAltName. */ static int fetch_ssl_verify_altname(STACK_OF(GENERAL_NAME) *altnames, const char *host, struct addrinfo *ip) { const GENERAL_NAME *name; size_t nslen; int i; const char *ns; for (i = 0; i < sk_GENERAL_NAME_num(altnames); ++i) { #if OPENSSL_VERSION_NUMBER < 0x10000000L /* * This is a workaround, since the following line causes * alignment issues in clang: * name = sk_GENERAL_NAME_value(altnames, i); * OpenSSL explicitly warns not to use those macros * directly, but there isn't much choice (and there * shouldn't be any ill side effects) */ name = (GENERAL_NAME *)SKM_sk_value(void, altnames, i); #else name = sk_GENERAL_NAME_value(altnames, i); #endif ns = (const char *)ASN1_STRING_data(name->d.ia5); nslen = (size_t)ASN1_STRING_length(name->d.ia5); if (name->type == GEN_DNS && ip == NULL && fetch_ssl_hname_match(host, strlen(host), ns, nslen)) return (1); else if (name->type == GEN_IPADD && ip != NULL && fetch_ssl_ipaddr_match_bin(ip, ns, nslen)) return (1); } return (0); } /* * Verify server certificate by CN. */ static int fetch_ssl_verify_cn(X509_NAME *subject, const char *host, struct addrinfo *ip) { ASN1_STRING *namedata; X509_NAME_ENTRY *nameentry; int cnlen, lastpos, loc, ret; unsigned char *cn; ret = 0; lastpos = -1; loc = -1; cn = NULL; /* get most specific CN (last entry in list) and compare */ while ((lastpos = X509_NAME_get_index_by_NID(subject, NID_commonName, lastpos)) != -1) loc = lastpos; if (loc > -1) { nameentry = X509_NAME_get_entry(subject, loc); namedata = X509_NAME_ENTRY_get_data(nameentry); cnlen = ASN1_STRING_to_UTF8(&cn, namedata); if (ip == NULL && fetch_ssl_hname_match(host, strlen(host), cn, cnlen)) ret = 1; else if (ip != NULL && fetch_ssl_ipaddr_match(ip, cn, cnlen)) ret = 1; OPENSSL_free(cn); } return (ret); } /* * Verify that server certificate subjectAltName/CN matches * hostname. First check, if there are alternative subject names. If yes, * those have to match. Only if those don't exist it falls back to * checking the subject's CN. */ static int fetch_ssl_verify_hname(X509 *cert, const char *host) { struct addrinfo *ip; STACK_OF(GENERAL_NAME) *altnames; X509_NAME *subject; int ret; ret = 0; ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host)); altnames = X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL); if (altnames != NULL) { ret = fetch_ssl_verify_altname(altnames, host, ip); } else { subject = X509_get_subject_name(cert); if (subject != NULL) ret = fetch_ssl_verify_cn(subject, host, ip); } if (ip != NULL) freeaddrinfo(ip); if (altnames != NULL) GENERAL_NAMES_free(altnames); return (ret); } /* * Configure transport security layer based on environment. */ static void fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose) { long ssl_ctx_options; - ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_TICKET; - if (getenv("SSL_ALLOW_SSL2") == NULL) - ssl_ctx_options |= SSL_OP_NO_SSLv2; + ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_TICKET; if (getenv("SSL_ALLOW_SSL3") == NULL) ssl_ctx_options |= SSL_OP_NO_SSLv3; if (getenv("SSL_NO_TLS1") != NULL) ssl_ctx_options |= SSL_OP_NO_TLSv1; if (getenv("SSL_NO_TLS1_1") != NULL) ssl_ctx_options |= SSL_OP_NO_TLSv1_1; if (getenv("SSL_NO_TLS1_2") != NULL) ssl_ctx_options |= SSL_OP_NO_TLSv1_2; if (verbose) fetch_info("SSL options: %lx", ssl_ctx_options); SSL_CTX_set_options(ctx, ssl_ctx_options); } /* * Configure peer verification based on environment. */ #define LOCAL_CERT_FILE "/usr/local/etc/ssl/cert.pem" #define BASE_CERT_FILE "/etc/ssl/cert.pem" static int fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose) { X509_LOOKUP *crl_lookup; X509_STORE *crl_store; const char *ca_cert_file, *ca_cert_path, *crl_file; if (getenv("SSL_NO_VERIFY_PEER") == NULL) { ca_cert_file = getenv("SSL_CA_CERT_FILE"); if (ca_cert_file == NULL && access(LOCAL_CERT_FILE, R_OK) == 0) ca_cert_file = LOCAL_CERT_FILE; if (ca_cert_file == NULL) ca_cert_file = BASE_CERT_FILE; ca_cert_path = getenv("SSL_CA_CERT_PATH"); if (verbose) { fetch_info("Peer verification enabled"); if (ca_cert_file != NULL) fetch_info("Using CA cert file: %s", ca_cert_file); if (ca_cert_path != NULL) fetch_info("Using CA cert path: %s", ca_cert_path); } SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, fetch_ssl_cb_verify_crt); SSL_CTX_load_verify_locations(ctx, ca_cert_file, ca_cert_path); if ((crl_file = getenv("SSL_CRL_FILE")) != NULL) { if (verbose) fetch_info("Using CRL file: %s", crl_file); crl_store = SSL_CTX_get_cert_store(ctx); crl_lookup = X509_STORE_add_lookup(crl_store, X509_LOOKUP_file()); if (crl_lookup == NULL || !X509_load_crl_file(crl_lookup, crl_file, X509_FILETYPE_PEM)) { fprintf(stderr, "Could not load CRL file %s\n", crl_file); return (0); } X509_STORE_set_flags(crl_store, X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL); } } return (1); } /* * Configure client certificate based on environment. */ static int fetch_ssl_setup_client_certificate(SSL_CTX *ctx, int verbose) { const char *client_cert_file, *client_key_file; if ((client_cert_file = getenv("SSL_CLIENT_CERT_FILE")) != NULL) { client_key_file = getenv("SSL_CLIENT_KEY_FILE") != NULL ? getenv("SSL_CLIENT_KEY_FILE") : client_cert_file; if (verbose) { fetch_info("Using client cert file: %s", client_cert_file); fetch_info("Using client key file: %s", client_key_file); } if (SSL_CTX_use_certificate_chain_file(ctx, client_cert_file) != 1) { fprintf(stderr, "Could not load client certificate %s\n", client_cert_file); return (0); } if (SSL_CTX_use_PrivateKey_file(ctx, client_key_file, SSL_FILETYPE_PEM) != 1) { fprintf(stderr, "Could not load client key %s\n", client_key_file); return (0); } } return (1); } /* * Callback for SSL certificate verification, this is called on server * cert verification. It takes no decision, but informs the user in case * verification failed. */ int fetch_ssl_cb_verify_crt(int verified, X509_STORE_CTX *ctx) { X509 *crt; X509_NAME *name; char *str; str = NULL; if (!verified) { if ((crt = X509_STORE_CTX_get_current_cert(ctx)) != NULL && (name = X509_get_subject_name(crt)) != NULL) str = X509_NAME_oneline(name, 0, 0); fprintf(stderr, "Certificate verification failed for %s\n", str != NULL ? str : "no relevant certificate"); OPENSSL_free(str); } return (verified); } #endif /* * Enable SSL on a connection. */ int fetch_ssl(conn_t *conn, const struct url *URL, int verbose) { #ifdef WITH_SSL int ret, ssl_err; X509_NAME *name; char *str; /* Init the SSL library and context */ if (!SSL_library_init()){ fprintf(stderr, "SSL library init failed\n"); return (-1); } SSL_load_error_strings(); conn->ssl_meth = SSLv23_client_method(); conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth); SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY); fetch_ssl_setup_transport_layer(conn->ssl_ctx, verbose); if (!fetch_ssl_setup_peer_verification(conn->ssl_ctx, verbose)) return (-1); if (!fetch_ssl_setup_client_certificate(conn->ssl_ctx, verbose)) return (-1); conn->ssl = SSL_new(conn->ssl_ctx); if (conn->ssl == NULL) { fprintf(stderr, "SSL context creation failed\n"); return (-1); } SSL_set_fd(conn->ssl, conn->sd); #if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT) if (!SSL_set_tlsext_host_name(conn->ssl, __DECONST(struct url *, URL)->host)) { fprintf(stderr, "TLS server name indication extension failed for host %s\n", URL->host); return (-1); } #endif while ((ret = SSL_connect(conn->ssl)) == -1) { ssl_err = SSL_get_error(conn->ssl, ret); if (ssl_err != SSL_ERROR_WANT_READ && ssl_err != SSL_ERROR_WANT_WRITE) { ERR_print_errors_fp(stderr); return (-1); } } conn->ssl_cert = SSL_get_peer_certificate(conn->ssl); if (conn->ssl_cert == NULL) { fprintf(stderr, "No server SSL certificate\n"); return (-1); } if (getenv("SSL_NO_VERIFY_HOSTNAME") == NULL) { if (verbose) fetch_info("Verify hostname"); if (!fetch_ssl_verify_hname(conn->ssl_cert, URL->host)) { fprintf(stderr, "SSL certificate subject doesn't match host %s\n", URL->host); return (-1); } } if (verbose) { fetch_info("%s connection established using %s", SSL_get_version(conn->ssl), SSL_get_cipher(conn->ssl)); name = X509_get_subject_name(conn->ssl_cert); str = X509_NAME_oneline(name, 0, 0); fetch_info("Certificate subject: %s", str); OPENSSL_free(str); name = X509_get_issuer_name(conn->ssl_cert); str = X509_NAME_oneline(name, 0, 0); fetch_info("Certificate issuer: %s", str); OPENSSL_free(str); } return (0); #else (void)conn; (void)verbose; fprintf(stderr, "SSL support disabled\n"); return (-1); #endif } #define FETCH_READ_WAIT -2 #define FETCH_READ_ERROR -1 #define FETCH_READ_DONE 0 #ifdef WITH_SSL static ssize_t fetch_ssl_read(SSL *ssl, char *buf, size_t len) { ssize_t rlen; int ssl_err; rlen = SSL_read(ssl, buf, len); if (rlen < 0) { ssl_err = SSL_get_error(ssl, rlen); if (ssl_err == SSL_ERROR_WANT_READ || ssl_err == SSL_ERROR_WANT_WRITE) { return (FETCH_READ_WAIT); } else { ERR_print_errors_fp(stderr); return (FETCH_READ_ERROR); } } return (rlen); } #endif static ssize_t fetch_socket_read(int sd, char *buf, size_t len) { ssize_t rlen; rlen = read(sd, buf, len); if (rlen < 0) { if (errno == EAGAIN || (errno == EINTR && fetchRestartCalls)) return (FETCH_READ_WAIT); else return (FETCH_READ_ERROR); } return (rlen); } /* * Read a character from a connection w/ timeout */ ssize_t fetch_read(conn_t *conn, char *buf, size_t len) { struct timeval now, timeout, delta; struct pollfd pfd; ssize_t rlen; int deltams; if (fetchTimeout > 0) { gettimeofday(&timeout, NULL); timeout.tv_sec += fetchTimeout; } deltams = INFTIM; memset(&pfd, 0, sizeof pfd); pfd.fd = conn->sd; pfd.events = POLLIN | POLLERR; for (;;) { /* * The socket is non-blocking. Instead of the canonical * poll() -> read(), we do the following: * * 1) call read() or SSL_read(). * 2) if we received some data, return it. * 3) if an error occurred, return -1. * 4) if read() or SSL_read() signaled EOF, return. * 5) if we did not receive any data but we're not at EOF, * call poll(). * * In the SSL case, this is necessary because if we * receive a close notification, we have to call * SSL_read() one additional time after we've read * everything we received. * * In the non-SSL case, it may improve performance (very * slightly) when reading small amounts of data. */ #ifdef WITH_SSL if (conn->ssl != NULL) rlen = fetch_ssl_read(conn->ssl, buf, len); else #endif rlen = fetch_socket_read(conn->sd, buf, len); if (rlen >= 0) { break; } else if (rlen == FETCH_READ_ERROR) { fetch_syserr(); return (-1); } // assert(rlen == FETCH_READ_WAIT); if (fetchTimeout > 0) { gettimeofday(&now, NULL); if (!timercmp(&timeout, &now, >)) { errno = ETIMEDOUT; fetch_syserr(); return (-1); } timersub(&timeout, &now, &delta); deltams = delta.tv_sec * 1000 + delta.tv_usec / 1000;; } errno = 0; pfd.revents = 0; if (poll(&pfd, 1, deltams) < 0) { if (errno == EINTR && fetchRestartCalls) continue; fetch_syserr(); return (-1); } } return (rlen); } /* * Read a line of text from a connection w/ timeout */ #define MIN_BUF_SIZE 1024 int fetch_getln(conn_t *conn) { char *tmp; size_t tmpsize; ssize_t len; char c; if (conn->buf == NULL) { if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) { errno = ENOMEM; return (-1); } conn->bufsize = MIN_BUF_SIZE; } conn->buf[0] = '\0'; conn->buflen = 0; do { len = fetch_read(conn, &c, 1); if (len == -1) return (-1); if (len == 0) break; conn->buf[conn->buflen++] = c; if (conn->buflen == conn->bufsize) { tmp = conn->buf; tmpsize = conn->bufsize * 2 + 1; if ((tmp = realloc(tmp, tmpsize)) == NULL) { errno = ENOMEM; return (-1); } conn->buf = tmp; conn->bufsize = tmpsize; } } while (c != '\n'); conn->buf[conn->buflen] = '\0'; DEBUG(fprintf(stderr, "<<< %s", conn->buf)); return (0); } /* * Write to a connection w/ timeout */ ssize_t fetch_write(conn_t *conn, const char *buf, size_t len) { struct iovec iov; iov.iov_base = __DECONST(char *, buf); iov.iov_len = len; return fetch_writev(conn, &iov, 1); } /* * Write a vector to a connection w/ timeout * Note: can modify the iovec. */ ssize_t fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt) { struct timeval now, timeout, delta; struct pollfd pfd; ssize_t wlen, total; int deltams; memset(&pfd, 0, sizeof pfd); if (fetchTimeout) { pfd.fd = conn->sd; pfd.events = POLLOUT | POLLERR; gettimeofday(&timeout, NULL); timeout.tv_sec += fetchTimeout; } total = 0; while (iovcnt > 0) { while (fetchTimeout && pfd.revents == 0) { gettimeofday(&now, NULL); if (!timercmp(&timeout, &now, >)) { errno = ETIMEDOUT; fetch_syserr(); return (-1); } timersub(&timeout, &now, &delta); deltams = delta.tv_sec * 1000 + delta.tv_usec / 1000; errno = 0; pfd.revents = 0; if (poll(&pfd, 1, deltams) < 0) { /* POSIX compliance */ if (errno == EAGAIN) continue; if (errno == EINTR && fetchRestartCalls) continue; return (-1); } } errno = 0; #ifdef WITH_SSL if (conn->ssl != NULL) wlen = SSL_write(conn->ssl, iov->iov_base, iov->iov_len); else #endif wlen = writev(conn->sd, iov, iovcnt); if (wlen == 0) { /* we consider a short write a failure */ /* XXX perhaps we shouldn't in the SSL case */ errno = EPIPE; fetch_syserr(); return (-1); } if (wlen < 0) { if (errno == EINTR && fetchRestartCalls) continue; return (-1); } total += wlen; while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) { wlen -= iov->iov_len; iov++; iovcnt--; } if (iovcnt > 0) { iov->iov_len -= wlen; iov->iov_base = __DECONST(char *, iov->iov_base) + wlen; } } return (total); } /* * Write a line of text to a connection w/ timeout */ int fetch_putln(conn_t *conn, const char *str, size_t len) { struct iovec iov[2]; int ret; DEBUG(fprintf(stderr, ">>> %s\n", str)); iov[0].iov_base = __DECONST(char *, str); iov[0].iov_len = len; iov[1].iov_base = __DECONST(char *, ENDL); iov[1].iov_len = sizeof(ENDL); if (len == 0) ret = fetch_writev(conn, &iov[1], 1); else ret = fetch_writev(conn, iov, 2); if (ret == -1) return (-1); return (0); } /* * Close connection */ int fetch_close(conn_t *conn) { int ret; if (--conn->ref > 0) return (0); #ifdef WITH_SSL if (conn->ssl) { SSL_shutdown(conn->ssl); SSL_set_connect_state(conn->ssl); SSL_free(conn->ssl); conn->ssl = NULL; } if (conn->ssl_ctx) { SSL_CTX_free(conn->ssl_ctx); conn->ssl_ctx = NULL; } if (conn->ssl_cert) { X509_free(conn->ssl_cert); conn->ssl_cert = NULL; } #endif ret = close(conn->sd); free(conn->buf); free(conn); return (ret); } /*** Directory-related utility functions *************************************/ int fetch_add_entry(struct url_ent **p, int *size, int *len, const char *name, struct url_stat *us) { struct url_ent *tmp; if (*p == NULL) { *size = 0; *len = 0; } if (*len >= *size - 1) { tmp = realloc(*p, (*size * 2 + 1) * sizeof(**p)); if (tmp == NULL) { errno = ENOMEM; fetch_syserr(); return (-1); } *size = (*size * 2 + 1); *p = tmp; } tmp = *p + *len; snprintf(tmp->name, PATH_MAX, "%s", name); memcpy(&tmp->stat, us, sizeof(*us)); (*len)++; (++tmp)->name[0] = 0; return (0); } /*** Authentication-related utility functions ********************************/ static const char * fetch_read_word(FILE *f) { static char word[1024]; if (fscanf(f, " %1023s ", word) != 1) return (NULL); return (word); } /* * Get authentication data for a URL from .netrc */ int fetch_netrc_auth(struct url *url) { char fn[PATH_MAX]; const char *word; char *p; FILE *f; if ((p = getenv("NETRC")) != NULL) { if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) { fetch_info("$NETRC specifies a file name " "longer than PATH_MAX"); return (-1); } } else { if ((p = getenv("HOME")) != NULL) { struct passwd *pwd; if ((pwd = getpwuid(getuid())) == NULL || (p = pwd->pw_dir) == NULL) return (-1); } if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn)) return (-1); } if ((f = fopen(fn, "r")) == NULL) return (-1); while ((word = fetch_read_word(f)) != NULL) { if (strcmp(word, "default") == 0) { DEBUG(fetch_info("Using default .netrc settings")); break; } if (strcmp(word, "machine") == 0 && (word = fetch_read_word(f)) != NULL && strcasecmp(word, url->host) == 0) { DEBUG(fetch_info("Using .netrc settings for %s", word)); break; } } if (word == NULL) goto ferr; while ((word = fetch_read_word(f)) != NULL) { if (strcmp(word, "login") == 0) { if ((word = fetch_read_word(f)) == NULL) goto ferr; if (snprintf(url->user, sizeof(url->user), "%s", word) > (int)sizeof(url->user)) { fetch_info("login name in .netrc is too long"); url->user[0] = '\0'; } } else if (strcmp(word, "password") == 0) { if ((word = fetch_read_word(f)) == NULL) goto ferr; if (snprintf(url->pwd, sizeof(url->pwd), "%s", word) > (int)sizeof(url->pwd)) { fetch_info("password in .netrc is too long"); url->pwd[0] = '\0'; } } else if (strcmp(word, "account") == 0) { if ((word = fetch_read_word(f)) == NULL) goto ferr; /* XXX not supported! */ } else { break; } } fclose(f); return (0); ferr: fclose(f); return (-1); } /* * The no_proxy environment variable specifies a set of domains for * which the proxy should not be consulted; the contents is a comma-, * or space-separated list of domain names. A single asterisk will * override all proxy variables and no transactions will be proxied * (for compatability with lynx and curl, see the discussion at * ). */ int fetch_no_proxy_match(const char *host) { const char *no_proxy, *p, *q; size_t h_len, d_len; if ((no_proxy = getenv("NO_PROXY")) == NULL && (no_proxy = getenv("no_proxy")) == NULL) return (0); /* asterisk matches any hostname */ if (strcmp(no_proxy, "*") == 0) return (1); h_len = strlen(host); p = no_proxy; do { /* position p at the beginning of a domain suffix */ while (*p == ',' || isspace((unsigned char)*p)) p++; /* position q at the first separator character */ for (q = p; *q; ++q) if (*q == ',' || isspace((unsigned char)*q)) break; d_len = q - p; if (d_len > 0 && h_len >= d_len && strncasecmp(host + h_len - d_len, p, d_len) == 0) { /* domain name matches */ return (1); } p = q + 1; } while (*q); return (0); } Index: head/lib/libfetch/fetch.3 =================================================================== --- head/lib/libfetch/fetch.3 (revision 280629) +++ head/lib/libfetch/fetch.3 (revision 280630) @@ -1,841 +1,837 @@ .\"- .\" Copyright (c) 1998-2013 Dag-Erling Smørgrav .\" Copyright (c) 2013 Michael Gmelin .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd October 15, 2014 +.Dd March 25, 2015 .Dt FETCH 3 .Os .Sh NAME .Nm fetchMakeURL , .Nm fetchParseURL , .Nm fetchFreeURL , .Nm fetchXGetURL , .Nm fetchGetURL , .Nm fetchPutURL , .Nm fetchStatURL , .Nm fetchListURL , .Nm fetchXGet , .Nm fetchGet , .Nm fetchPut , .Nm fetchStat , .Nm fetchList , .Nm fetchXGetFile , .Nm fetchGetFile , .Nm fetchPutFile , .Nm fetchStatFile , .Nm fetchListFile , .Nm fetchXGetHTTP , .Nm fetchGetHTTP , .Nm fetchPutHTTP , .Nm fetchStatHTTP , .Nm fetchListHTTP , .Nm fetchXGetFTP , .Nm fetchGetFTP , .Nm fetchPutFTP , .Nm fetchStatFTP , .Nm fetchListFTP .Nd file transfer functions .Sh LIBRARY .Lb libfetch .Sh SYNOPSIS .In sys/param.h .In stdio.h .In fetch.h .Ft struct url * .Fn fetchMakeURL "const char *scheme" "const char *host" "int port" "const char *doc" "const char *user" "const char *pwd" .Ft struct url * .Fn fetchParseURL "const char *URL" .Ft void .Fn fetchFreeURL "struct url *u" .Ft FILE * .Fn fetchXGetURL "const char *URL" "struct url_stat *us" "const char *flags" .Ft FILE * .Fn fetchGetURL "const char *URL" "const char *flags" .Ft FILE * .Fn fetchPutURL "const char *URL" "const char *flags" .Ft int .Fn fetchStatURL "const char *URL" "struct url_stat *us" "const char *flags" .Ft struct url_ent * .Fn fetchListURL "const char *URL" "const char *flags" .Ft FILE * .Fn fetchXGet "struct url *u" "struct url_stat *us" "const char *flags" .Ft FILE * .Fn fetchGet "struct url *u" "const char *flags" .Ft FILE * .Fn fetchPut "struct url *u" "const char *flags" .Ft int .Fn fetchStat "struct url *u" "struct url_stat *us" "const char *flags" .Ft struct url_ent * .Fn fetchList "struct url *u" "const char *flags" .Ft FILE * .Fn fetchXGetFile "struct url *u" "struct url_stat *us" "const char *flags" .Ft FILE * .Fn fetchGetFile "struct url *u" "const char *flags" .Ft FILE * .Fn fetchPutFile "struct url *u" "const char *flags" .Ft int .Fn fetchStatFile "struct url *u" "struct url_stat *us" "const char *flags" .Ft struct url_ent * .Fn fetchListFile "struct url *u" "const char *flags" .Ft FILE * .Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "const char *flags" .Ft FILE * .Fn fetchGetHTTP "struct url *u" "const char *flags" .Ft FILE * .Fn fetchPutHTTP "struct url *u" "const char *flags" .Ft int .Fn fetchStatHTTP "struct url *u" "struct url_stat *us" "const char *flags" .Ft struct url_ent * .Fn fetchListHTTP "struct url *u" "const char *flags" .Ft FILE * .Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "const char *flags" .Ft FILE * .Fn fetchGetFTP "struct url *u" "const char *flags" .Ft FILE * .Fn fetchPutFTP "struct url *u" "const char *flags" .Ft int .Fn fetchStatFTP "struct url *u" "struct url_stat *us" "const char *flags" .Ft struct url_ent * .Fn fetchListFTP "struct url *u" "const char *flags" .Sh DESCRIPTION These functions implement a high-level library for retrieving and uploading files using Uniform Resource Locators (URLs). .Pp .Fn fetchParseURL takes a URL in the form of a null-terminated string and splits it into its components function according to the Common Internet Scheme Syntax detailed in RFC1738. A regular expression which produces this syntax is: .Bd -literal :(//((:)?@)?(:)?)?/()? .Ed .Pp If the URL does not seem to begin with a scheme name, the following syntax is assumed: .Bd -literal (((:)?@)?(:)?)?/()? .Ed .Pp Note that some components of the URL are not necessarily relevant to all URL schemes. For instance, the file scheme only needs the and components. .Pp .Fn fetchMakeURL and .Fn fetchParseURL return a pointer to a .Vt url structure, which is defined as follows in .In fetch.h : .Bd -literal #define URL_SCHEMELEN 16 #define URL_USERLEN 256 #define URL_PWDLEN 256 struct url { char scheme[URL_SCHEMELEN+1]; char user[URL_USERLEN+1]; char pwd[URL_PWDLEN+1]; char host[MAXHOSTNAMELEN+1]; int port; char *doc; off_t offset; size_t length; time_t ims_time; }; .Ed .Pp The .Va ims_time field stores the time value for .Li If-Modified-Since HTTP requests. .Pp The pointer returned by .Fn fetchMakeURL or .Fn fetchParseURL should be freed using .Fn fetchFreeURL . .Pp .Fn fetchXGetURL , .Fn fetchGetURL , and .Fn fetchPutURL constitute the recommended interface to the .Nm fetch library. They examine the URL passed to them to determine the transfer method, and call the appropriate lower-level functions to perform the actual transfer. .Fn fetchXGetURL also returns the remote document's metadata in the .Vt url_stat structure pointed to by the .Fa us argument. .Pp The .Fa flags argument is a string of characters which specify transfer options. The meaning of the individual flags is scheme-dependent, and is detailed in the appropriate section below. .Pp .Fn fetchStatURL attempts to obtain the requested document's metadata and fill in the structure pointed to by its second argument. The .Vt url_stat structure is defined as follows in .In fetch.h : .Bd -literal struct url_stat { off_t size; time_t atime; time_t mtime; }; .Ed .Pp If the size could not be obtained from the server, the .Fa size field is set to -1. If the modification time could not be obtained from the server, the .Fa mtime field is set to the epoch. If the access time could not be obtained from the server, the .Fa atime field is set to the modification time. .Pp .Fn fetchListURL attempts to list the contents of the directory pointed to by the URL provided. If successful, it returns a malloced array of .Vt url_ent structures. The .Vt url_ent structure is defined as follows in .In fetch.h : .Bd -literal struct url_ent { char name[PATH_MAX]; struct url_stat stat; }; .Ed .Pp The list is terminated by an entry with an empty name. .Pp The pointer returned by .Fn fetchListURL should be freed using .Fn free . .Pp .Fn fetchXGet , .Fn fetchGet , .Fn fetchPut and .Fn fetchStat are similar to .Fn fetchXGetURL , .Fn fetchGetURL , .Fn fetchPutURL and .Fn fetchStatURL , except that they expect a pre-parsed URL in the form of a pointer to a .Vt struct url rather than a string. .Pp All of the .Fn fetchXGetXXX , .Fn fetchGetXXX and .Fn fetchPutXXX functions return a pointer to a stream which can be used to read or write data from or to the requested document, respectively. Note that although the implementation details of the individual access methods vary, it can generally be assumed that a stream returned by one of the .Fn fetchXGetXXX or .Fn fetchGetXXX functions is read-only, and that a stream returned by one of the .Fn fetchPutXXX functions is write-only. .Sh FILE SCHEME .Fn fetchXGetFile , .Fn fetchGetFile and .Fn fetchPutFile provide access to documents which are files in a locally mounted file system. Only the component of the URL is used. .Pp .Fn fetchXGetFile and .Fn fetchGetFile do not accept any flags. .Pp .Fn fetchPutFile accepts the .Ql a (append to file) flag. If that flag is specified, the data written to the stream returned by .Fn fetchPutFile will be appended to the previous contents of the file, instead of replacing them. .Sh FTP SCHEME .Fn fetchXGetFTP , .Fn fetchGetFTP and .Fn fetchPutFTP implement the FTP protocol as described in RFC959. .Pp If the .Ql P (not passive) flag is specified, an active (rather than passive) connection will be attempted. .Pp The .Ql p flag is supported for compatibility with earlier versions where active connections were the default. It has precedence over the .Ql P flag, so if both are specified, .Nm will use a passive connection. .Pp If the .Ql l (low) flag is specified, data sockets will be allocated in the low (or default) port range instead of the high port range (see .Xr ip 4 ) . .Pp If the .Ql d (direct) flag is specified, .Fn fetchXGetFTP , .Fn fetchGetFTP and .Fn fetchPutFTP will use a direct connection even if a proxy server is defined. .Pp If no user name or password is given, the .Nm fetch library will attempt an anonymous login, with user name "anonymous" and password "anonymous@". .Sh HTTP SCHEME The .Fn fetchXGetHTTP , .Fn fetchGetHTTP and .Fn fetchPutHTTP functions implement the HTTP/1.1 protocol. With a little luck, there is even a chance that they comply with RFC2616 and RFC2617. .Pp If the .Ql d (direct) flag is specified, .Fn fetchXGetHTTP , .Fn fetchGetHTTP and .Fn fetchPutHTTP will use a direct connection even if a proxy server is defined. .Pp If the .Ql i (if-modified-since) flag is specified, and the .Va ims_time field is set in .Vt "struct url" , then .Fn fetchXGetHTTP and .Fn fetchGetHTTP will send a conditional .Li If-Modified-Since HTTP header to only fetch the content if it is newer than .Va ims_time . .Pp Since there seems to be no good way of implementing the HTTP PUT method in a manner consistent with the rest of the .Nm fetch library, .Fn fetchPutHTTP is currently unimplemented. .Sh HTTPS SCHEME Based on HTTP SCHEME. By default the peer is verified using the CA bundle located in .Pa /etc/ssl/cert.pem . The file may contain multiple CA certificates. A common source of a current CA bundle is .Pa \%security/ca_root_nss . .Pp The CA bundle used for peer verification can be changed by setting the environment variables .Ev SSL_CA_CERT_FILE to point to a concatenated bundle of trusted certificates and .Ev SSL_CA_CERT_PATH to point to a directory containing hashes of trusted CAs (see .Xr verify 1 ) . .Pp A certificate revocation list (CRL) can be used by setting the environment variable .Ev SSL_CRL_FILE (see .Xr crl 1 ) . .Pp Peer verification can be disabled by setting the environment variable .Ev SSL_NO_VERIFY_PEER . Note that this also disables CRL checking. .Pp By default the service identity is verified according to the rules detailed in RFC6125 (also known as hostname verification). This feature can be disabled by setting the environment variable .Ev SSL_NO_VERIFY_HOSTNAME . .Pp Client certificate based authentication is supported. The environment variable .Ev SSL_CLIENT_CERT_FILE should be set to point to a file containing key and client certificate to be used in PEM format. In case the key is stored in a separate file, the environment variable .Ev SSL_CLIENT_KEY_FILE can be set to point to the key in PEM format. In case the key uses a password, the user will be prompted on standard input (see .Xr PEM 3 ) . .Pp By default .Nm libfetch allows TLSv1 and newer when negotiating the connecting with the remote peer. You can change this behavior by setting the -.Ev SSL_ALLOW_SSL2 -and .Ev SSL_ALLOW_SSL3 -environment variables to allow SSLv2 and SSLv3, respectively, and +environment variable to allow SSLv3 and .Ev SSL_NO_TLS1 , .Ev SSL_NO_TLS1_1 and .Ev SSL_NO_TLS1_2 to disable TLS 1.0, 1.1 and 1.2 respectively. .Sh AUTHENTICATION Apart from setting the appropriate environment variables and specifying the user name and password in the URL or the .Vt struct url , the calling program has the option of defining an authentication function with the following prototype: .Pp .Ft int .Fn myAuthMethod "struct url *u" .Pp The callback function should fill in the .Fa user and .Fa pwd fields in the provided .Vt struct url and return 0 on success, or any other value to indicate failure. .Pp To register the authentication callback, simply set .Va fetchAuthMethod to point at it. The callback will be used whenever a site requires authentication and the appropriate environment variables are not set. .Pp This interface is experimental and may be subject to change. .Sh RETURN VALUES .Fn fetchParseURL returns a pointer to a .Vt struct url containing the individual components of the URL. If it is unable to allocate memory, or the URL is syntactically incorrect, .Fn fetchParseURL returns a NULL pointer. .Pp The .Fn fetchStat functions return 0 on success and -1 on failure. .Pp All other functions return a stream pointer which may be used to access the requested document, or NULL if an error occurred. .Pp The following error codes are defined in .In fetch.h : .Bl -tag -width 18n .It Bq Er FETCH_ABORT Operation aborted .It Bq Er FETCH_AUTH Authentication failed .It Bq Er FETCH_DOWN Service unavailable .It Bq Er FETCH_EXISTS File exists .It Bq Er FETCH_FULL File system full .It Bq Er FETCH_INFO Informational response .It Bq Er FETCH_MEMORY Insufficient memory .It Bq Er FETCH_MOVED File has moved .It Bq Er FETCH_NETWORK Network error .It Bq Er FETCH_OK No error .It Bq Er FETCH_PROTO Protocol error .It Bq Er FETCH_RESOLV Resolver error .It Bq Er FETCH_SERVER Server error .It Bq Er FETCH_TEMP Temporary error .It Bq Er FETCH_TIMEOUT Operation timed out .It Bq Er FETCH_UNAVAIL File is not available .It Bq Er FETCH_UNKNOWN Unknown error .It Bq Er FETCH_URL Invalid URL .El .Pp The accompanying error message includes a protocol-specific error code and message, e.g.\& "File is not available (404 Not Found)" .Sh ENVIRONMENT .Bl -tag -width ".Ev FETCH_BIND_ADDRESS" .It Ev FETCH_BIND_ADDRESS Specifies a hostname or IP address to which sockets used for outgoing connections will be bound. .It Ev FTP_LOGIN Default FTP login if none was provided in the URL. .It Ev FTP_PASSIVE_MODE If set to .Ql no , forces the FTP code to use active mode. If set to any other value, forces passive mode even if the application requested active mode. .It Ev FTP_PASSWORD Default FTP password if the remote server requests one and none was provided in the URL. .It Ev FTP_PROXY URL of the proxy to use for FTP requests. The document part is ignored. FTP and HTTP proxies are supported; if no scheme is specified, FTP is assumed. If the proxy is an FTP proxy, .Nm libfetch will send .Ql user@host as user name to the proxy, where .Ql user is the real user name, and .Ql host is the name of the FTP server. .Pp If this variable is set to an empty string, no proxy will be used for FTP requests, even if the .Ev HTTP_PROXY variable is set. .It Ev ftp_proxy Same as .Ev FTP_PROXY , for compatibility. .It Ev HTTP_ACCEPT Specifies the value of the .Va Accept header for HTTP requests. If empty, no .Va Accept header is sent. The default is .Dq */* . .It Ev HTTP_AUTH Specifies HTTP authorization parameters as a colon-separated list of items. The first and second item are the authorization scheme and realm respectively; further items are scheme-dependent. Currently, the .Dq basic and .Dq digest authorization methods are supported. .Pp Both methods require two parameters: the user name and password, in that order. .Pp This variable is only used if the server requires authorization and no user name or password was specified in the URL. .It Ev HTTP_PROXY URL of the proxy to use for HTTP requests. The document part is ignored. Only HTTP proxies are supported for HTTP requests. If no port number is specified, the default is 3128. .Pp Note that this proxy will also be used for FTP documents, unless the .Ev FTP_PROXY variable is set. .It Ev http_proxy Same as .Ev HTTP_PROXY , for compatibility. .It Ev HTTP_PROXY_AUTH Specifies authorization parameters for the HTTP proxy in the same format as the .Ev HTTP_AUTH variable. .Pp This variable is used if and only if connected to an HTTP proxy, and is ignored if a user and/or a password were specified in the proxy URL. .It Ev HTTP_REFERER Specifies the referrer URL to use for HTTP requests. If set to .Dq auto , the document URL will be used as referrer URL. .It Ev HTTP_USER_AGENT Specifies the User-Agent string to use for HTTP requests. This can be useful when working with HTTP origin or proxy servers that differentiate between user agents. If defined but empty, no User-Agent header is sent. .It Ev NETRC Specifies a file to use instead of .Pa ~/.netrc to look up login names and passwords for FTP sites. See .Xr ftp 1 for a description of the file format. This feature is experimental. .It Ev NO_PROXY Either a single asterisk, which disables the use of proxies altogether, or a comma- or whitespace-separated list of hosts for which proxies should not be used. .It Ev no_proxy Same as .Ev NO_PROXY , for compatibility. -.It Ev SSL_ALLOW_SSL2 -Allow SSL version 2 when negotiating the connection (not recommended). .It Ev SSL_ALLOW_SSL3 Allow SSL version 3 when negotiating the connection (not recommended). .It Ev SSL_CA_CERT_FILE CA certificate bundle containing trusted CA certificates. Default value: .Pa /etc/ssl/cert.pem . .It Ev SSL_CA_CERT_PATH Path containing trusted CA hashes. .It Ev SSL_CLIENT_CERT_FILE PEM encoded client certificate/key which will be used in client certificate authentication. .It Ev SSL_CLIENT_KEY_FILE PEM encoded client key in case key and client certificate are stored separately. .It Ev SSL_CRL_FILE File containing certificate revocation list. .It Ev SSL_NO_TLS1 Do not allow TLS version 1.0 when negotiating the connection. .It Ev SSL_NO_TLS1_1 Do not allow TLS version 1.1 when negotiating the connection. .It Ev SSL_NO_TLS1_2 Do not allow TLS version 1.2 when negotiating the connection. .It Ev SSL_NO_VERIFY_HOSTNAME If set, do not verify that the hostname matches the subject of the certificate presented by the server. .It Ev SSL_NO_VERIFY_PEER If set, do not verify the peer certificate against trusted CAs. .El .Sh EXAMPLES To access a proxy server on .Pa proxy.example.com port 8080, set the .Ev HTTP_PROXY environment variable in a manner similar to this: .Pp .Dl HTTP_PROXY=http://proxy.example.com:8080 .Pp If the proxy server requires authentication, there are two options available for passing the authentication data. The first method is by using the proxy URL: .Pp .Dl HTTP_PROXY=http://:@proxy.example.com:8080 .Pp The second method is by using the .Ev HTTP_PROXY_AUTH environment variable: .Bd -literal -offset indent HTTP_PROXY=http://proxy.example.com:8080 HTTP_PROXY_AUTH=basic:*:: .Ed .Pp To disable the use of a proxy for an HTTP server running on the local host, define .Ev NO_PROXY as follows: .Bd -literal -offset indent NO_PROXY=localhost,127.0.0.1 .Ed .Pp Access HTTPS website without any certificate verification whatsoever: .Bd -literal -offset indent SSL_NO_VERIFY_PEER=1 SSL_NO_VERIFY_HOSTNAME=1 .Ed .Pp Access HTTPS website using client certificate based authentication and a private CA: .Bd -literal -offset indent SSL_CLIENT_CERT_FILE=/path/to/client.pem SSL_CA_CERT_FILE=/path/to/myca.pem .Ed .Sh SEE ALSO .Xr fetch 1 , .Xr ftpio 3 , .Xr ip 4 .Rs .%A J. Postel .%A J. K. Reynolds .%D October 1985 .%B File Transfer Protocol .%O RFC959 .Re .Rs .%A P. Deutsch .%A A. Emtage .%A A. Marine. .%D May 1994 .%T How to Use Anonymous FTP .%O RFC1635 .Re .Rs .%A T. Berners-Lee .%A L. Masinter .%A M. McCahill .%D December 1994 .%T Uniform Resource Locators (URL) .%O RFC1738 .Re .Rs .%A R. Fielding .%A J. Gettys .%A J. Mogul .%A H. Frystyk .%A L. Masinter .%A P. Leach .%A T. Berners-Lee .%D January 1999 .%B Hypertext Transfer Protocol -- HTTP/1.1 .%O RFC2616 .Re .Rs .%A J. Franks .%A P. Hallam-Baker .%A J. Hostetler .%A S. Lawrence .%A P. Leach .%A A. Luotonen .%A L. Stewart .%D June 1999 .%B HTTP Authentication: Basic and Digest Access Authentication .%O RFC2617 .Re .Sh HISTORY The .Nm fetch library first appeared in .Fx 3.0 . .Sh AUTHORS .An -nosplit The .Nm fetch library was mostly written by .An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org with numerous suggestions and contributions from .An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org , .An Eugene Skepner Aq Mt eu@qub.com , .An Hajimu Umemoto Aq Mt ume@FreeBSD.org , .An Henry Whincup Aq Mt henry@techiebod.com , .An Jukka A. Ukkonen Aq Mt jau@iki.fi , .An Jean-Fran\(,cois Dockes Aq Mt jf@dockes.org , .An Michael Gmelin Aq Mt freebsd@grem.de and others. It replaces the older .Nm ftpio library written by .An Poul-Henning Kamp Aq Mt phk@FreeBSD.org and .An Jordan K. Hubbard Aq Mt jkh@FreeBSD.org . .Pp This manual page was written by .An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org and .An Michael Gmelin Aq Mt freebsd@grem.de . .Sh BUGS Some parts of the library are not yet implemented. The most notable examples of this are .Fn fetchPutHTTP , .Fn fetchListHTTP , .Fn fetchListFTP and FTP proxy support. .Pp There is no way to select a proxy at run-time other than setting the .Ev HTTP_PROXY or .Ev FTP_PROXY environment variables as appropriate. .Pp .Nm libfetch does not understand or obey 305 (Use Proxy) replies. .Pp Error numbers are unique only within a certain context; the error codes used for FTP and HTTP overlap, as do those used for resolver and system errors. For instance, error code 202 means "Command not implemented, superfluous at this site" in an FTP context and "Accepted" in an HTTP context. .Pp .Fn fetchStatFTP does not check that the result of an MDTM command is a valid date. .Pp In case password protected keys are used for client certificate based authentication the user is prompted for the password on each and every fetch operation. .Pp The man page is incomplete, poorly written and produces badly formatted text. .Pp The error reporting mechanism is unsatisfactory. .Pp Some parts of the code are not fully reentrant. Index: head/sys/sys/param.h =================================================================== --- head/sys/sys/param.h (revision 280629) +++ head/sys/sys/param.h (revision 280630) @@ -1,348 +1,348 @@ /*- * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)param.h 8.3 (Berkeley) 4/4/95 * $FreeBSD$ */ #ifndef _SYS_PARAM_H_ #define _SYS_PARAM_H_ #include #define BSD 199506 /* System version (year & month). */ #define BSD4_3 1 #define BSD4_4 1 /* * __FreeBSD_version numbers are documented in the Porter's Handbook. * If you bump the version for any reason, you should update the documentation * there. * Currently this lives here in the doc/ repository: * * head/en_US.ISO8859-1/books/porters-handbook/book.xml * * scheme is: Rxx * 'R' is in the range 0 to 4 if this is a release branch or * x.0-CURRENT before RELENG_*_0 is created, otherwise 'R' is * in the range 5 to 9. */ #undef __FreeBSD_version -#define __FreeBSD_version 1100066 /* Master, propagated to newvers */ +#define __FreeBSD_version 1100067 /* Master, propagated to newvers */ /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, * which by definition is always true on FreeBSD. This macro is also defined * on other systems that use the kernel of FreeBSD, such as GNU/kFreeBSD. * * It is tempting to use this macro in userland code when we want to enable * kernel-specific routines, and in fact it's fine to do this in code that * is part of FreeBSD itself. However, be aware that as presence of this * macro is still not widespread (e.g. older FreeBSD versions, 3rd party * compilers, etc), it is STRONGLY DISCOURAGED to check for this macro in * external applications without also checking for __FreeBSD__ as an * alternative. */ #undef __FreeBSD_kernel__ #define __FreeBSD_kernel__ #ifdef _KERNEL #define P_OSREL_SIGWAIT 700000 #define P_OSREL_SIGSEGV 700004 #define P_OSREL_MAP_ANON 800104 #define P_OSREL_MAP_FSTRICT 1100036 #define P_OSREL_MAJOR(x) ((x) / 100000) #endif #ifndef LOCORE #include #endif /* * Machine-independent constants (some used in following include files). * Redefined constants are from POSIX 1003.1 limits file. * * MAXCOMLEN should be >= sizeof(ac_comm) (see ) */ #include #define MAXCOMLEN 19 /* max command name remembered */ #define MAXINTERP PATH_MAX /* max interpreter file name length */ #define MAXLOGNAME 33 /* max login name length (incl. NUL) */ #define MAXUPRC CHILD_MAX /* max simultaneous processes */ #define NCARGS ARG_MAX /* max bytes for an exec function */ #define NGROUPS (NGROUPS_MAX+1) /* max number groups */ #define NOFILE OPEN_MAX /* max open files per process */ #define NOGROUP 65535 /* marker for empty group set member */ #define MAXHOSTNAMELEN 256 /* max hostname size */ #define SPECNAMELEN 63 /* max length of devicename */ /* More types and definitions used throughout the kernel. */ #ifdef _KERNEL #include #include #ifndef LOCORE #include #include #endif #ifndef FALSE #define FALSE 0 #endif #ifndef TRUE #define TRUE 1 #endif #endif #ifndef _KERNEL /* Signals. */ #include #endif /* Machine type dependent parameters. */ #include #ifndef _KERNEL #include #endif #ifndef DEV_BSHIFT #define DEV_BSHIFT 9 /* log2(DEV_BSIZE) */ #endif #define DEV_BSIZE (1<>PAGE_SHIFT) #endif /* * btodb() is messy and perhaps slow because `bytes' may be an off_t. We * want to shift an unsigned type to avoid sign extension and we don't * want to widen `bytes' unnecessarily. Assume that the result fits in * a daddr_t. */ #ifndef btodb #define btodb(bytes) /* calculates (bytes / DEV_BSIZE) */ \ (sizeof (bytes) > sizeof(long) \ ? (daddr_t)((unsigned long long)(bytes) >> DEV_BSHIFT) \ : (daddr_t)((unsigned long)(bytes) >> DEV_BSHIFT)) #endif #ifndef dbtob #define dbtob(db) /* calculates (db * DEV_BSIZE) */ \ ((off_t)(db) << DEV_BSHIFT) #endif #define PRIMASK 0x0ff #define PCATCH 0x100 /* OR'd with pri for tsleep to check signals */ #define PDROP 0x200 /* OR'd with pri to stop re-entry of interlock mutex */ #define NZERO 0 /* default "nice" */ #define NBBY 8 /* number of bits in a byte */ #define NBPW sizeof(int) /* number of bytes per word (integer) */ #define CMASK 022 /* default file mask: S_IWGRP|S_IWOTH */ #define NODEV (dev_t)(-1) /* non-existent device */ /* * File system parameters and macros. * * MAXBSIZE - Filesystems are made out of blocks of at most MAXBSIZE bytes * per block. MAXBSIZE may be made larger without effecting * any existing filesystems as long as it does not exceed MAXPHYS, * and may be made smaller at the risk of not being able to use * filesystems which require a block size exceeding MAXBSIZE. * * BKVASIZE - Nominal buffer space per buffer, in bytes. BKVASIZE is the * minimum KVM memory reservation the kernel is willing to make. * Filesystems can of course request smaller chunks. Actual * backing memory uses a chunk size of a page (PAGE_SIZE). * * If you make BKVASIZE too small you risk seriously fragmenting * the buffer KVM map which may slow things down a bit. If you * make it too big the kernel will not be able to optimally use * the KVM memory reserved for the buffer cache and will wind * up with too-few buffers. * * The default is 16384, roughly 2x the block size used by a * normal UFS filesystem. */ #define MAXBSIZE 65536 /* must be power of 2 */ #define BKVASIZE 16384 /* must be power of 2 */ #define BKVAMASK (BKVASIZE-1) /* * MAXPATHLEN defines the longest permissible path length after expanding * symbolic links. It is used to allocate a temporary buffer from the buffer * pool in which to do the name expansion, hence should be a power of two, * and must be less than or equal to MAXBSIZE. MAXSYMLINKS defines the * maximum number of symbolic links that may be expanded in a path name. * It should be set high enough to allow all legitimate uses, but halt * infinite loops reasonably quickly. */ #define MAXPATHLEN PATH_MAX #define MAXSYMLINKS 32 /* Bit map related macros. */ #define setbit(a,i) (((unsigned char *)(a))[(i)/NBBY] |= 1<<((i)%NBBY)) #define clrbit(a,i) (((unsigned char *)(a))[(i)/NBBY] &= ~(1<<((i)%NBBY))) #define isset(a,i) \ (((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) #define isclr(a,i) \ ((((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) == 0) /* Macros for counting and rounding. */ #ifndef howmany #define howmany(x, y) (((x)+((y)-1))/(y)) #endif #define nitems(x) (sizeof((x)) / sizeof((x)[0])) #define rounddown(x, y) (((x)/(y))*(y)) #define rounddown2(x, y) ((x)&(~((y)-1))) /* if y is power of two */ #define roundup(x, y) ((((x)+((y)-1))/(y))*(y)) /* to any y */ #define roundup2(x, y) (((x)+((y)-1))&(~((y)-1))) /* if y is powers of two */ #define powerof2(x) ((((x)-1)&(x))==0) /* Macros for min/max. */ #define MIN(a,b) (((a)<(b))?(a):(b)) #define MAX(a,b) (((a)>(b))?(a):(b)) #ifdef _KERNEL /* * Basic byte order function prototypes for non-inline functions. */ #ifndef LOCORE #ifndef _BYTEORDER_PROTOTYPED #define _BYTEORDER_PROTOTYPED __BEGIN_DECLS __uint32_t htonl(__uint32_t); __uint16_t htons(__uint16_t); __uint32_t ntohl(__uint32_t); __uint16_t ntohs(__uint16_t); __END_DECLS #endif #endif #ifndef lint #ifndef _BYTEORDER_FUNC_DEFINED #define _BYTEORDER_FUNC_DEFINED #define htonl(x) __htonl(x) #define htons(x) __htons(x) #define ntohl(x) __ntohl(x) #define ntohs(x) __ntohs(x) #endif /* !_BYTEORDER_FUNC_DEFINED */ #endif /* lint */ #endif /* _KERNEL */ /* * Scale factor for scaled integers used to count %cpu time and load avgs. * * The number of CPU `tick's that map to a unique `%age' can be expressed * by the formula (1 / (2 ^ (FSHIFT - 11))). The maximum load average that * can be calculated (assuming 32 bits) can be closely approximated using * the formula (2 ^ (2 * (16 - FSHIFT))) for (FSHIFT < 15). * * For the scheduler to maintain a 1:1 mapping of CPU `tick' to `%age', * FSHIFT must be at least 11; this gives us a maximum load avg of ~1024. */ #define FSHIFT 11 /* bits to right of fixed binary point */ #define FSCALE (1<> (PAGE_SHIFT - DEV_BSHIFT)) #define ctodb(db) /* calculates pages to devblks */ \ ((db) << (PAGE_SHIFT - DEV_BSHIFT)) /* * Old spelling of __containerof(). */ #define member2struct(s, m, x) \ ((struct s *)(void *)((char *)(x) - offsetof(struct s, m))) /* * Access a variable length array that has been declared as a fixed * length array. */ #define __PAST_END(array, offset) (((__typeof__(*(array)) *)(array))[offset]) #endif /* _SYS_PARAM_H_ */ Index: head/usr.bin/fetch/fetch.1 =================================================================== --- head/usr.bin/fetch/fetch.1 (revision 280629) +++ head/usr.bin/fetch/fetch.1 (revision 280630) @@ -1,417 +1,412 @@ .\"- .\" Copyright (c) 2000-2014 Dag-Erling Smørgrav .\" Copyright (c) 2013 Michael Gmelin .\" All rights reserved. .\" Portions Copyright (c) 1999 Massachusetts Institute of Technology; used .\" by permission. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer .\" in this position and unchanged. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" 3. The name of the author may not be used to endorse or promote products .\" derived from this software without specific prior written permission. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR .\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES .\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. .\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, .\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT .\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd January 28, 2014 +.Dd March 25, 2015 .Dt FETCH 1 .Os .Sh NAME .Nm fetch .Nd retrieve a file by Uniform Resource Locator .Sh SYNOPSIS .Nm .Op Fl 146AadFlMmnPpqRrsUv -.Op Fl -allow-sslv2 .Op Fl B Ar bytes .Op Fl -bind-address= Ns Ar host .Op Fl -ca-cert= Ns Ar file .Op Fl -ca-path= Ns Ar dir .Op Fl -cert= Ns Ar file .Op Fl -crl= Ns Ar file .Op Fl i Ar file .Op Fl -key= Ns Ar file .Op Fl N Ar file .Op Fl -no-passive .Op Fl -no-proxy= Ns Ar list .Op Fl -no-sslv3 .Op Fl -no-tlsv1 .Op Fl -no-verify-hostname .Op Fl -no-verify-peer .Op Fl o Ar file .Op Fl -referer= Ns Ar URL .Op Fl S Ar bytes .Op Fl T Ar seconds .Op Fl -user-agent= Ns Ar agent-string .Op Fl w Ar seconds .Ar URL ... .Nm .Op Fl 146AadFlMmnPpqRrsUv .Op Fl B Ar bytes .Op Fl -bind-address= Ns Ar host .Op Fl -ca-cert= Ns Ar file .Op Fl -ca-path= Ns Ar dir .Op Fl -cert= Ns Ar file .Op Fl -crl= Ns Ar file .Op Fl i Ar file .Op Fl -key= Ns Ar file .Op Fl N Ar file .Op Fl -no-passive .Op Fl -no-proxy= Ns Ar list .Op Fl -no-sslv3 .Op Fl -no-tlsv1 .Op Fl -no-verify-hostname .Op Fl -no-verify-peer .Op Fl o Ar file .Op Fl -referer= Ns Ar URL .Op Fl S Ar bytes .Op Fl T Ar seconds .Op Fl -user-agent= Ns Ar agent-string .Op Fl w Ar seconds .Fl h Ar host Fl f Ar file Oo Fl c Ar dir Oc .Sh DESCRIPTION The .Nm utility provides a command-line interface to the .Xr fetch 3 library. Its purpose is to retrieve the file(s) pointed to by the URL(s) on the command line. .Pp The following options are available: .Bl -tag -width Fl .It Fl 1 , -one-file Stop and return exit code 0 at the first successfully retrieved file. .It Fl 4 , -ipv4-only Forces .Nm to use IPv4 addresses only. .It Fl 6 , -ipv6-only Forces .Nm to use IPv6 addresses only. .It Fl A , -no-redirect Do not automatically follow ``temporary'' (302) redirects. Some broken Web sites will return a redirect instead of a not-found error when the requested object does not exist. .It Fl a , -retry Automatically retry the transfer upon soft failures. -.It Fl -allow-sslv2 -[SSL] -Allow SSL version 2 when negotiating the connection. .It Fl B Ar bytes , Fl -buffer-size= Ns Ar bytes Specify the read buffer size in bytes. The default is 16,384 bytes. Attempts to set a buffer size lower than this will be silently ignored. The number of reads actually performed is reported at verbosity level two or higher (see the .Fl v flag). .It Fl -bind-address= Ns Ar host Specifies a hostname or IP address to which sockets used for outgoing connections will be bound. .It Fl c Ar dir The file to retrieve is in directory .Ar dir on the remote host. This option is deprecated and is provided for backward compatibility only. .It Fl -ca-cert= Ns Ar file [SSL] Path to certificate bundle containing trusted CA certificates. If not specified, .Pa /etc/ssl/cert.pem is used. The file may contain multiple CA certificates. The port .Pa security/ca_root_nss is a common source of a current CA bundle. .It Fl -ca-path= Ns Ar dir [SSL] The directory .Ar dir contains trusted CA hashes. .It Fl -cert= Ns Ar file [SSL] .Ar file is a PEM encoded client certificate/key which will be used in client certificate authentication. .It Fl -crl= Ns Ar file [SSL] Points to certificate revocation list .Ar file , which has to be in PEM format and may contain peer certificates that have been revoked. .It Fl d , -direct Use a direct connection even if a proxy is configured. .It Fl F , -force-restart In combination with the .Fl r flag, forces a restart even if the local and remote files have different modification times. Implies .Fl R . .It Fl f Ar file The file to retrieve is named .Ar file on the remote host. This option is deprecated and is provided for backward compatibility only. .It Fl h Ar host The file to retrieve is located on the host .Ar host . This option is deprecated and is provided for backward compatibility only. .It Fl i Ar file , Fl -if-modified-since= Ns Ar file If-Modified-Since mode: the remote file will only be retrieved if it is newer than .Ar file on the local host. (HTTP only) .It Fl -key= Ns Ar file [SSL] .Ar file is a PEM encoded client key that will be used in client certificate authentication in case key and client certificate are stored separately. .It Fl l , -symlink If the target is a file-scheme URL, make a symbolic link to the target rather than trying to copy it. .It Fl M .It Fl m , -mirror Mirror mode: if the file already exists locally and has the same size and modification time as the remote file, it will not be fetched. Note that the .Fl m and .Fl r flags are mutually exclusive. .It Fl N Ar file , Fl -netrc= Ns Ar file Use .Ar file instead of .Pa ~/.netrc to look up login names and passwords for FTP sites. See .Xr ftp 1 for a description of the file format. This feature is experimental. .It Fl n , -no-mtime Do not preserve the modification time of the transferred file. .It Fl -no-passive Forces the FTP code to use active mode. .It Fl -no-proxy= Ns Ar list Either a single asterisk, which disables the use of proxies altogether, or a comma- or whitespace-separated list of hosts for which proxies should not be used. .It Fl -no-sslv3 [SSL] Don't allow SSL version 3 when negotiating the connection. .It Fl -no-tlsv1 [SSL] Don't allow TLS version 1 when negotiating the connection. .It Fl -no-verify-hostname [SSL] Do not verify that the hostname matches the subject of the certificate presented by the server. .It Fl -no-verify-peer [SSL] Do not verify the peer certificate against trusted CAs. .It Fl o Ar file , Fl output= Ns Ar file Set the output file name to .Ar file . By default, a ``pathname'' is extracted from the specified URI, and its basename is used as the name of the output file. A .Ar file argument of .Sq Li \&- indicates that results are to be directed to the standard output. If the .Ar file argument is a directory, fetched file(s) will be placed within the directory, with name(s) selected as in the default behaviour. .It Fl P .It Fl p , -passive Use passive FTP. These flags have no effect, since passive FTP is the default, but are provided for compatibility with earlier versions where active FTP was the default. To force active mode, use the .Fl -no-passive flag or set the .Ev FTP_PASSIVE_MODE environment variable to .Ql NO . .It Fl -referer= Ns Ar URL Specifies the referrer URL to use for HTTP requests. If .Ar URL is set to .Dq auto , the document URL will be used as referrer URL. .It Fl q , -quiet Quiet mode. .It Fl R , -keep-output The output files are precious, and should not be deleted under any circumstances, even if the transfer failed or was incomplete. .It Fl r , -restart Restart a previously interrupted transfer. Note that the .Fl m and .Fl r flags are mutually exclusive. .It Fl S Ar bytes , Fl -require-size= Ns Ar bytes Require the file size reported by the server to match the specified value. If it does not, a message is printed and the file is not fetched. If the server does not support reporting file sizes, this option is ignored and the file is fetched unconditionally. .It Fl s , -print-size Print the size in bytes of each requested file, without fetching it. .It Fl T Ar seconds , Fl -timeout= Ns Ar seconds Set timeout value to .Ar seconds . Overrides the environment variables .Ev FTP_TIMEOUT for FTP transfers or .Ev HTTP_TIMEOUT for HTTP transfers if set. .It Fl U , -passive-portrange-default When using passive FTP, allocate the port for the data connection from the low (default) port range. See .Xr ip 4 for details on how to specify which port range this corresponds to. .It Fl -user-agent= Ns Ar agent-string Specifies the User-Agent string to use for HTTP requests. This can be useful when working with HTTP origin or proxy servers that differentiate between user agents. .It Fl v , -verbose Increase verbosity level. .It Fl w Ar seconds , Fl -retry-delay= Ns Ar seconds When the .Fl a flag is specified, wait this many seconds between successive retries. .El .Pp If .Nm receives a .Dv SIGINFO signal (see the .Cm status argument for .Xr stty 1 ) , the current transfer rate statistics will be written to the standard error output, in the same format as the standard completion message. .Sh ENVIRONMENT .Bl -tag -width HTTP_TIMEOUT .It Ev FTP_TIMEOUT Maximum time, in seconds, to wait before aborting an FTP connection. .It Ev HTTP_TIMEOUT Maximum time, in seconds, to wait before aborting an HTTP connection. .El .Pp See .Xr fetch 3 for a description of additional environment variables, including .Ev FETCH_BIND_ADDRESS , .Ev FTP_LOGIN , .Ev FTP_PASSIVE_MODE , .Ev FTP_PASSWORD , .Ev FTP_PROXY , .Ev ftp_proxy , .Ev HTTP_ACCEPT , .Ev HTTP_AUTH , .Ev HTTP_PROXY , .Ev http_proxy , .Ev HTTP_PROXY_AUTH , .Ev HTTP_REFERER , .Ev HTTP_USER_AGENT , .Ev NETRC , .Ev NO_PROXY , .Ev no_proxy , -.Ev SSL_ALLOW_SSL2 , .Ev SSL_CA_CERT_FILE , .Ev SSL_CA_CERT_PATH , .Ev SSL_CLIENT_CERT_FILE , .Ev SSL_CLIENT_KEY_FILE , .Ev SSL_CRL_FILE , .Ev SSL_NO_SSL3 , .Ev SSL_NO_TLS1 , .Ev SSL_NO_VERIFY_HOSTNAME and .Ev SSL_NO_VERIFY_PEER . .Sh EXIT STATUS The .Nm command returns zero on success, or one on failure. If multiple URLs are listed on the command line, .Nm will attempt to retrieve each one of them in turn, and will return zero only if they were all successfully retrieved. .Pp If the .Fl i argument is used and the remote file is not newer than the specified file then the command will still return success, although no file is transferred. .Sh SEE ALSO .Xr fetch 3 .Sh HISTORY The .Nm command appeared in .Fx 2.1.5 . This implementation first appeared in .Fx 4.1 . .Sh AUTHORS .An -nosplit The original implementation of .Nm was done by .An Jean-Marc Zucconi Aq Mt jmz@FreeBSD.org . It was extensively re-worked for .Fx 2.2 by .An Garrett Wollman Aq Mt wollman@FreeBSD.org , and later completely rewritten to use the .Xr fetch 3 library by .An Dag-Erling Sm\(/orgrav Aq Mt des@FreeBSD.org and .An Michael Gmelin Aq Mt freebsd@grem.de . .Sh NOTES The .Fl b and .Fl t options are no longer supported and will generate warnings. They were workarounds for bugs in other OSes which this implementation does not trigger. .Pp One cannot both use the .Fl h , .Fl c and .Fl f options and specify URLs on the command line. Index: head/usr.bin/fetch/fetch.c =================================================================== --- head/usr.bin/fetch/fetch.c (revision 280629) +++ head/usr.bin/fetch/fetch.c (revision 280630) @@ -1,1180 +1,1175 @@ /*- * Copyright (c) 2000-2014 Dag-Erling Smørgrav * Copyright (c) 2013 Michael Gmelin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MINBUFSIZE 16384 #define TIMEOUT 120 /* Option flags */ static int A_flag; /* -A: do not follow 302 redirects */ static int a_flag; /* -a: auto retry */ static off_t B_size; /* -B: buffer size */ static int b_flag; /*! -b: workaround TCP bug */ static char *c_dirname; /* -c: remote directory */ static int d_flag; /* -d: direct connection */ static int F_flag; /* -F: restart without checking mtime */ static char *f_filename; /* -f: file to fetch */ static char *h_hostname; /* -h: host to fetch from */ static int i_flag; /* -i: specify file for mtime comparison */ static char *i_filename; /* name of input file */ static int l_flag; /* -l: link rather than copy file: URLs */ static int m_flag; /* -[Mm]: mirror mode */ static char *N_filename; /* -N: netrc file name */ static int n_flag; /* -n: do not preserve modification time */ static int o_flag; /* -o: specify output file */ static int o_directory; /* output file is a directory */ static char *o_filename; /* name of output file */ static int o_stdout; /* output file is stdout */ static int once_flag; /* -1: stop at first successful file */ static int p_flag; /* -[Pp]: use passive FTP */ static int R_flag; /* -R: don't delete partial files */ static int r_flag; /* -r: restart previous transfer */ static off_t S_size; /* -S: require size to match */ static int s_flag; /* -s: show size, don't fetch */ static long T_secs; /* -T: transfer timeout in seconds */ static int t_flag; /*! -t: workaround TCP bug */ static int U_flag; /* -U: do not use high ports */ static int v_level = 1; /* -v: verbosity level */ static int v_tty; /* stdout is a tty */ static pid_t pgrp; /* our process group */ static long w_secs; /* -w: retry delay */ static int family = PF_UNSPEC; /* -[46]: address family to use */ static int sigalrm; /* SIGALRM received */ static int siginfo; /* SIGINFO received */ static int sigint; /* SIGINT received */ static long ftp_timeout = TIMEOUT; /* default timeout for FTP transfers */ static long http_timeout = TIMEOUT;/* default timeout for HTTP transfers */ static char *buf; /* transfer buffer */ enum options { OPTION_BIND_ADDRESS, OPTION_NO_FTP_PASSIVE_MODE, OPTION_HTTP_REFERER, OPTION_HTTP_USER_AGENT, OPTION_NO_PROXY, - OPTION_SSL_ALLOW_SSL2, OPTION_SSL_CA_CERT_FILE, OPTION_SSL_CA_CERT_PATH, OPTION_SSL_CLIENT_CERT_FILE, OPTION_SSL_CLIENT_KEY_FILE, OPTION_SSL_CRL_FILE, OPTION_SSL_NO_SSL3, OPTION_SSL_NO_TLS1, OPTION_SSL_NO_VERIFY_HOSTNAME, OPTION_SSL_NO_VERIFY_PEER }; static struct option longopts[] = { /* mapping to single character argument */ { "one-file", no_argument, NULL, '1' }, { "ipv4-only", no_argument, NULL, '4' }, { "ipv6-only", no_argument, NULL, '6' }, { "no-redirect", no_argument, NULL, 'A' }, { "retry", no_argument, NULL, 'a' }, { "buffer-size", required_argument, NULL, 'B' }, /* -c not mapped, since it's deprecated */ { "direct", no_argument, NULL, 'd' }, { "force-restart", no_argument, NULL, 'F' }, /* -f not mapped, since it's deprecated */ /* -h not mapped, since it's deprecated */ { "if-modified-since", required_argument, NULL, 'i' }, { "symlink", no_argument, NULL, 'l' }, /* -M not mapped since it's the same as -m */ { "mirror", no_argument, NULL, 'm' }, { "netrc", required_argument, NULL, 'N' }, { "no-mtime", no_argument, NULL, 'n' }, { "output", required_argument, NULL, 'o' }, /* -P not mapped since it's the same as -p */ { "passive", no_argument, NULL, 'p' }, { "quiet", no_argument, NULL, 'q' }, { "keep-output", no_argument, NULL, 'R' }, { "restart", no_argument, NULL, 'r' }, { "require-size", required_argument, NULL, 'S' }, { "print-size", no_argument, NULL, 's' }, { "timeout", required_argument, NULL, 'T' }, { "passive-portrange-default", no_argument, NULL, 'T' }, { "verbose", no_argument, NULL, 'v' }, { "retry-delay", required_argument, NULL, 'w' }, /* options without a single character equivalent */ { "bind-address", required_argument, NULL, OPTION_BIND_ADDRESS }, { "no-passive", no_argument, NULL, OPTION_NO_FTP_PASSIVE_MODE }, { "referer", required_argument, NULL, OPTION_HTTP_REFERER }, { "user-agent", required_argument, NULL, OPTION_HTTP_USER_AGENT }, { "no-proxy", required_argument, NULL, OPTION_NO_PROXY }, - { "allow-sslv2", no_argument, NULL, OPTION_SSL_ALLOW_SSL2 }, { "ca-cert", required_argument, NULL, OPTION_SSL_CA_CERT_FILE }, { "ca-path", required_argument, NULL, OPTION_SSL_CA_CERT_PATH }, { "cert", required_argument, NULL, OPTION_SSL_CLIENT_CERT_FILE }, { "key", required_argument, NULL, OPTION_SSL_CLIENT_KEY_FILE }, { "crl", required_argument, NULL, OPTION_SSL_CRL_FILE }, { "no-sslv3", no_argument, NULL, OPTION_SSL_NO_SSL3 }, { "no-tlsv1", no_argument, NULL, OPTION_SSL_NO_TLS1 }, { "no-verify-hostname", no_argument, NULL, OPTION_SSL_NO_VERIFY_HOSTNAME }, { "no-verify-peer", no_argument, NULL, OPTION_SSL_NO_VERIFY_PEER }, { NULL, 0, NULL, 0 } }; /* * Signal handler */ static void sig_handler(int sig) { switch (sig) { case SIGALRM: sigalrm = 1; break; case SIGINFO: siginfo = 1; break; case SIGINT: sigint = 1; break; } } struct xferstat { char name[64]; struct timeval start; /* start of transfer */ struct timeval last; /* time of last update */ struct timeval last2; /* time of previous last update */ off_t size; /* size of file per HTTP hdr */ off_t offset; /* starting offset in file */ off_t rcvd; /* bytes already received */ off_t lastrcvd; /* bytes received since last update */ }; /* * Compute and display ETA */ static const char * stat_eta(struct xferstat *xs) { static char str[16]; long elapsed, eta; off_t received, expected; elapsed = xs->last.tv_sec - xs->start.tv_sec; received = xs->rcvd - xs->offset; expected = xs->size - xs->rcvd; eta = (long)((double)elapsed * expected / received); if (eta > 3600) snprintf(str, sizeof str, "%02ldh%02ldm", eta / 3600, (eta % 3600) / 60); else if (eta > 0) snprintf(str, sizeof str, "%02ldm%02lds", eta / 60, eta % 60); else snprintf(str, sizeof str, "%02ldm%02lds", elapsed / 60, elapsed % 60); return (str); } /* * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'... */ static const char *prefixes = " kMGTP"; static const char * stat_bytes(off_t bytes) { static char str[16]; const char *prefix = prefixes; while (bytes > 9999 && prefix[1] != '\0') { bytes /= 1024; prefix++; } snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix); return (str); } /* * Compute and display transfer rate */ static const char * stat_bps(struct xferstat *xs) { static char str[16]; double delta, bps; delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6)) - (xs->last2.tv_sec + (xs->last2.tv_usec / 1.e6)); if (delta == 0.0) { snprintf(str, sizeof str, "?? Bps"); } else { bps = (xs->rcvd - xs->lastrcvd) / delta; snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps)); } return (str); } /* * Update the stats display */ static void stat_display(struct xferstat *xs, int force) { struct timeval now; int ctty_pgrp; /* check if we're the foreground process */ if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || (pid_t)ctty_pgrp != pgrp) return; gettimeofday(&now, NULL); if (!force && now.tv_sec <= xs->last.tv_sec) return; xs->last2 = xs->last; xs->last = now; fprintf(stderr, "\r%-46.46s", xs->name); if (xs->size <= 0) { setproctitle("%s [%s]", xs->name, stat_bytes(xs->rcvd)); fprintf(stderr, " %s", stat_bytes(xs->rcvd)); } else { setproctitle("%s [%d%% of %s]", xs->name, (int)((100.0 * xs->rcvd) / xs->size), stat_bytes(xs->size)); fprintf(stderr, "%3d%% of %s", (int)((100.0 * xs->rcvd) / xs->size), stat_bytes(xs->size)); } if (force == 2) { xs->lastrcvd = xs->offset; xs->last2 = xs->start; } fprintf(stderr, " %s", stat_bps(xs)); if ((xs->size > 0 && xs->rcvd > 0 && xs->last.tv_sec >= xs->start.tv_sec + 3) || force == 2) fprintf(stderr, " %s", stat_eta(xs)); xs->lastrcvd = xs->rcvd; } /* * Initialize the transfer statistics */ static void stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) { snprintf(xs->name, sizeof xs->name, "%s", name); gettimeofday(&xs->start, NULL); xs->last.tv_sec = xs->last.tv_usec = 0; xs->size = size; xs->offset = offset; xs->rcvd = offset; xs->lastrcvd = offset; if (v_tty && v_level > 0) stat_display(xs, 1); else if (v_level > 0) fprintf(stderr, "%-46s", xs->name); } /* * Update the transfer statistics */ static void stat_update(struct xferstat *xs, off_t rcvd) { xs->rcvd = rcvd; if (v_tty && v_level > 0) stat_display(xs, 0); } /* * Finalize the transfer statistics */ static void stat_end(struct xferstat *xs) { gettimeofday(&xs->last, NULL); if (v_tty && v_level > 0) { stat_display(xs, 2); putc('\n', stderr); } else if (v_level > 0) { fprintf(stderr, " %s %s\n", stat_bytes(xs->size), stat_bps(xs)); } } /* * Ask the user for authentication details */ static int query_auth(struct url *URL) { struct termios tios; tcflag_t saved_flags; int i, nopwd; fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", URL->scheme, URL->host, URL->port); fprintf(stderr, "Login: "); if (fgets(URL->user, sizeof URL->user, stdin) == NULL) return (-1); for (i = strlen(URL->user); i >= 0; --i) if (URL->user[i] == '\r' || URL->user[i] == '\n') URL->user[i] = '\0'; fprintf(stderr, "Password: "); if (tcgetattr(STDIN_FILENO, &tios) == 0) { saved_flags = tios.c_lflag; tios.c_lflag &= ~ECHO; tios.c_lflag |= ECHONL|ICANON; tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); tios.c_lflag = saved_flags; tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); } else { nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); } if (nopwd) return (-1); for (i = strlen(URL->pwd); i >= 0; --i) if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n') URL->pwd[i] = '\0'; return (0); } /* * Fetch a file */ static int fetch(char *URL, const char *path) { struct url *url; struct url_stat us; struct stat sb, nsb; struct xferstat xs; FILE *f, *of; size_t size, readcnt, wr; off_t count; char flags[8]; const char *slash; char *tmppath; int r; unsigned timeout; char *ptr; f = of = NULL; tmppath = NULL; timeout = 0; *flags = 0; count = 0; /* set verbosity level */ if (v_level > 1) strcat(flags, "v"); if (v_level > 2) fetchDebug = 1; /* parse URL */ url = NULL; if (*URL == '\0') { warnx("empty URL"); goto failure; } if ((url = fetchParseURL(URL)) == NULL) { warnx("%s: parse error", URL); goto failure; } /* if no scheme was specified, take a guess */ if (!*url->scheme) { if (!*url->host) strcpy(url->scheme, SCHEME_FILE); else if (strncasecmp(url->host, "ftp.", 4) == 0) strcpy(url->scheme, SCHEME_FTP); else if (strncasecmp(url->host, "www.", 4) == 0) strcpy(url->scheme, SCHEME_HTTP); } /* common flags */ switch (family) { case PF_INET: strcat(flags, "4"); break; case PF_INET6: strcat(flags, "6"); break; } /* FTP specific flags */ if (strcmp(url->scheme, SCHEME_FTP) == 0) { if (p_flag) strcat(flags, "p"); if (d_flag) strcat(flags, "d"); if (U_flag) strcat(flags, "l"); timeout = T_secs ? T_secs : ftp_timeout; } /* HTTP specific flags */ if (strcmp(url->scheme, SCHEME_HTTP) == 0 || strcmp(url->scheme, SCHEME_HTTPS) == 0) { if (d_flag) strcat(flags, "d"); if (A_flag) strcat(flags, "A"); timeout = T_secs ? T_secs : http_timeout; if (i_flag) { if (stat(i_filename, &sb)) { warn("%s: stat()", i_filename); goto failure; } url->ims_time = sb.st_mtime; strcat(flags, "i"); } } /* set the protocol timeout. */ fetchTimeout = timeout; /* just print size */ if (s_flag) { if (timeout) alarm(timeout); r = fetchStat(url, &us, flags); if (timeout) alarm(0); if (sigalrm || sigint) goto signal; if (r == -1) { warnx("%s", fetchLastErrString); goto failure; } if (us.size == -1) printf("Unknown\n"); else printf("%jd\n", (intmax_t)us.size); goto success; } /* * If the -r flag was specified, we have to compare the local * and remote files, so we should really do a fetchStat() * first, but I know of at least one HTTP server that only * sends the content size in response to GET requests, and * leaves it out of replies to HEAD requests. Also, in the * (frequent) case that the local and remote files match but * the local file is truncated, we have sufficient information * before the compare to issue a correct request. Therefore, * we always issue a GET request as if we were sure the local * file was a truncated copy of the remote file; we can drop * the connection later if we change our minds. */ sb.st_size = -1; if (!o_stdout) { r = stat(path, &sb); if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { url->offset = sb.st_size; } else if (r == -1 || !S_ISREG(sb.st_mode)) { /* * Whatever value sb.st_size has now is either * wrong (if stat(2) failed) or irrelevant (if the * path does not refer to a regular file) */ sb.st_size = -1; } if (r == -1 && errno != ENOENT) { warnx("%s: stat()", path); goto failure; } } /* start the transfer */ if (timeout) alarm(timeout); f = fetchXGet(url, &us, flags); if (timeout) alarm(0); if (sigalrm || sigint) goto signal; if (f == NULL) { warnx("%s: %s", URL, fetchLastErrString); if (i_flag && strcmp(url->scheme, SCHEME_HTTP) == 0 && fetchLastErrCode == FETCH_OK && strcmp(fetchLastErrString, "Not Modified") == 0) { /* HTTP Not Modified Response, return OK. */ r = 0; goto done; } else goto failure; } if (sigint) goto signal; /* check that size is as expected */ if (S_size) { if (us.size == -1) { warnx("%s: size unknown", URL); } else if (us.size != S_size) { warnx("%s: size mismatch: expected %jd, actual %jd", URL, (intmax_t)S_size, (intmax_t)us.size); goto failure; } } /* symlink instead of copy */ if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { if (symlink(url->doc, path) == -1) { warn("%s: symlink()", path); goto failure; } goto success; } if (us.size == -1 && !o_stdout && v_level > 0) warnx("%s: size of remote file is not known", URL); if (v_level > 1) { if (sb.st_size != -1) fprintf(stderr, "local size / mtime: %jd / %ld\n", (intmax_t)sb.st_size, (long)sb.st_mtime); if (us.size != -1) fprintf(stderr, "remote size / mtime: %jd / %ld\n", (intmax_t)us.size, (long)us.mtime); } /* open output file */ if (o_stdout) { /* output to stdout */ of = stdout; } else if (r_flag && sb.st_size != -1) { /* resume mode, local file exists */ if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { /* no match! have to refetch */ fclose(f); /* if precious, warn the user and give up */ if (R_flag) { warnx("%s: local modification time " "does not match remote", path); goto failure_keep; } } else if (url->offset > sb.st_size) { /* gap between what we asked for and what we got */ warnx("%s: gap in resume mode", URL); fclose(of); of = NULL; /* picked up again later */ } else if (us.size != -1) { if (us.size == sb.st_size) /* nothing to do */ goto success; if (sb.st_size > us.size) { /* local file too long! */ warnx("%s: local file (%jd bytes) is longer " "than remote file (%jd bytes)", path, (intmax_t)sb.st_size, (intmax_t)us.size); goto failure; } /* we got it, open local file */ if ((of = fopen(path, "r+")) == NULL) { warn("%s: fopen()", path); goto failure; } /* check that it didn't move under our feet */ if (fstat(fileno(of), &nsb) == -1) { /* can't happen! */ warn("%s: fstat()", path); goto failure; } if (nsb.st_dev != sb.st_dev || nsb.st_ino != sb.st_ino || nsb.st_size != sb.st_size) { warnx("%s: file has changed", URL); fclose(of); of = NULL; sb = nsb; /* picked up again later */ } } /* seek to where we left off */ if (of != NULL && fseeko(of, url->offset, SEEK_SET) != 0) { warn("%s: fseeko()", path); fclose(of); of = NULL; /* picked up again later */ } } else if (m_flag && sb.st_size != -1) { /* mirror mode, local file exists */ if (sb.st_size == us.size && sb.st_mtime == us.mtime) goto success; } if (of == NULL) { /* * We don't yet have an output file; either this is a * vanilla run with no special flags, or the local and * remote files didn't match. */ if (url->offset > 0) { /* * We tried to restart a transfer, but for * some reason gave up - so we have to restart * from scratch if we want the whole file */ url->offset = 0; if ((f = fetchXGet(url, &us, flags)) == NULL) { warnx("%s: %s", URL, fetchLastErrString); goto failure; } if (sigint) goto signal; } /* construct a temp file name */ if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { if ((slash = strrchr(path, '/')) == NULL) slash = path; else ++slash; asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s", (int)(slash - path), path, slash); if (tmppath != NULL) { if (mkstemps(tmppath, strlen(slash) + 1) == -1) { warn("%s: mkstemps()", path); goto failure; } of = fopen(tmppath, "w"); chown(tmppath, sb.st_uid, sb.st_gid); chmod(tmppath, sb.st_mode & ALLPERMS); } } if (of == NULL) of = fopen(path, "w"); if (of == NULL) { warn("%s: open()", path); goto failure; } } count = url->offset; /* start the counter */ stat_start(&xs, path, us.size, count); sigalrm = siginfo = sigint = 0; /* suck in the data */ setvbuf(f, NULL, _IOFBF, B_size); signal(SIGINFO, sig_handler); while (!sigint) { if (us.size != -1 && us.size - count < B_size && us.size - count >= 0) size = us.size - count; else size = B_size; if (siginfo) { stat_end(&xs); siginfo = 0; } if (size == 0) break; if ((readcnt = fread(buf, 1, size, f)) < size) { if (ferror(f) && errno == EINTR && !sigint) clearerr(f); else if (readcnt == 0) break; } stat_update(&xs, count += readcnt); for (ptr = buf; readcnt > 0; ptr += wr, readcnt -= wr) if ((wr = fwrite(ptr, 1, readcnt, of)) < readcnt) { if (ferror(of) && errno == EINTR && !sigint) clearerr(of); else break; } if (readcnt != 0) break; } if (!sigalrm) sigalrm = ferror(f) && errno == ETIMEDOUT; signal(SIGINFO, SIG_DFL); stat_end(&xs); /* * If the transfer timed out or was interrupted, we still want to * set the mtime in case the file is not removed (-r or -R) and * the user later restarts the transfer. */ signal: /* set mtime of local file */ if (!n_flag && us.mtime && !o_stdout && of != NULL && (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { struct timeval tv[2]; fflush(of); tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); tv[1].tv_sec = (long)us.mtime; tv[0].tv_usec = tv[1].tv_usec = 0; if (utimes(tmppath ? tmppath : path, tv)) warn("%s: utimes()", tmppath ? tmppath : path); } /* timed out or interrupted? */ if (sigalrm) warnx("transfer timed out"); if (sigint) { warnx("transfer interrupted"); goto failure; } /* timeout / interrupt before connection completley established? */ if (f == NULL) goto failure; if (!sigalrm) { /* check the status of our files */ if (ferror(f)) warn("%s", URL); if (ferror(of)) warn("%s", path); if (ferror(f) || ferror(of)) goto failure; } /* did the transfer complete normally? */ if (us.size != -1 && count < us.size) { warnx("%s appears to be truncated: %jd/%jd bytes", path, (intmax_t)count, (intmax_t)us.size); goto failure_keep; } /* * If the transfer timed out and we didn't know how much to * expect, assume the worst (i.e. we didn't get all of it) */ if (sigalrm && us.size == -1) { warnx("%s may be truncated", path); goto failure_keep; } success: r = 0; if (tmppath != NULL && rename(tmppath, path) == -1) { warn("%s: rename()", path); goto failure_keep; } goto done; failure: if (of && of != stdout && !R_flag && !r_flag) if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) unlink(tmppath ? tmppath : path); if (R_flag && tmppath != NULL && sb.st_size == -1) rename(tmppath, path); /* ignore errors here */ failure_keep: r = -1; goto done; done: if (f) fclose(f); if (of && of != stdout) fclose(of); if (url) fetchFreeURL(url); if (tmppath != NULL) free(tmppath); return (r); } static void usage(void) { fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", -"usage: fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]", -" [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]", -" [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]", -" [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]", -" [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]", +"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]", +" [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]", +" [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]", +" [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]", +" [-o file] [--referer=URL] [-S bytes] [-T seconds]", " [--user-agent=agent-string] [-w seconds] URL ...", -" fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]", -" [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]", -" [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]", -" [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]", -" [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]", +" fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]", +" [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]", +" [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]", +" [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]", +" [-o file] [--referer=URL] [-S bytes] [-T seconds]", " [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]"); } /* * Entry point */ int main(int argc, char *argv[]) { struct stat sb; struct sigaction sa; const char *p, *s; char *end, *q; int c, e, r; while ((c = getopt_long(argc, argv, "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:", longopts, NULL)) != -1) switch (c) { case '1': once_flag = 1; break; case '4': family = PF_INET; break; case '6': family = PF_INET6; break; case 'A': A_flag = 1; break; case 'a': a_flag = 1; break; case 'B': B_size = (off_t)strtol(optarg, &end, 10); if (*optarg == '\0' || *end != '\0') errx(1, "invalid buffer size (%s)", optarg); break; case 'b': warnx("warning: the -b option is deprecated"); b_flag = 1; break; case 'c': c_dirname = optarg; break; case 'd': d_flag = 1; break; case 'F': F_flag = 1; break; case 'f': f_filename = optarg; break; case 'H': warnx("the -H option is now implicit, " "use -U to disable"); break; case 'h': h_hostname = optarg; break; case 'i': i_flag = 1; i_filename = optarg; break; case 'l': l_flag = 1; break; case 'o': o_flag = 1; o_filename = optarg; break; case 'M': case 'm': if (r_flag) errx(1, "the -m and -r flags " "are mutually exclusive"); m_flag = 1; break; case 'N': N_filename = optarg; break; case 'n': n_flag = 1; break; case 'P': case 'p': p_flag = 1; break; case 'q': v_level = 0; break; case 'R': R_flag = 1; break; case 'r': if (m_flag) errx(1, "the -m and -r flags " "are mutually exclusive"); r_flag = 1; break; case 'S': S_size = (off_t)strtol(optarg, &end, 10); if (*optarg == '\0' || *end != '\0') errx(1, "invalid size (%s)", optarg); break; case 's': s_flag = 1; break; case 'T': T_secs = strtol(optarg, &end, 10); if (*optarg == '\0' || *end != '\0') errx(1, "invalid timeout (%s)", optarg); break; case 't': t_flag = 1; warnx("warning: the -t option is deprecated"); break; case 'U': U_flag = 1; break; case 'v': v_level++; break; case 'w': a_flag = 1; w_secs = strtol(optarg, &end, 10); if (*optarg == '\0' || *end != '\0') errx(1, "invalid delay (%s)", optarg); break; case OPTION_BIND_ADDRESS: setenv("FETCH_BIND_ADDRESS", optarg, 1); break; case OPTION_NO_FTP_PASSIVE_MODE: setenv("FTP_PASSIVE_MODE", "no", 1); break; case OPTION_HTTP_REFERER: setenv("HTTP_REFERER", optarg, 1); break; case OPTION_HTTP_USER_AGENT: setenv("HTTP_USER_AGENT", optarg, 1); break; case OPTION_NO_PROXY: setenv("NO_PROXY", optarg, 1); - break; - case OPTION_SSL_ALLOW_SSL2: - setenv("SSL_ALLOW_SSL2", "", 1); break; case OPTION_SSL_CA_CERT_FILE: setenv("SSL_CA_CERT_FILE", optarg, 1); break; case OPTION_SSL_CA_CERT_PATH: setenv("SSL_CA_CERT_PATH", optarg, 1); break; case OPTION_SSL_CLIENT_CERT_FILE: setenv("SSL_CLIENT_CERT_FILE", optarg, 1); break; case OPTION_SSL_CLIENT_KEY_FILE: setenv("SSL_CLIENT_KEY_FILE", optarg, 1); break; case OPTION_SSL_CRL_FILE: setenv("SSL_CLIENT_CRL_FILE", optarg, 1); break; case OPTION_SSL_NO_SSL3: setenv("SSL_NO_SSL3", "", 1); break; case OPTION_SSL_NO_TLS1: setenv("SSL_NO_TLS1", "", 1); break; case OPTION_SSL_NO_VERIFY_HOSTNAME: setenv("SSL_NO_VERIFY_HOSTNAME", "", 1); break; case OPTION_SSL_NO_VERIFY_PEER: setenv("SSL_NO_VERIFY_PEER", "", 1); break; default: usage(); exit(1); } argc -= optind; argv += optind; if (h_hostname || f_filename || c_dirname) { if (!h_hostname || !f_filename || argc) { usage(); exit(1); } /* XXX this is a hack. */ if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) errx(1, "invalid hostname"); if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, c_dirname ? c_dirname : "", f_filename) == -1) errx(1, "%s", strerror(ENOMEM)); argc++; } if (!argc) { usage(); exit(1); } /* allocate buffer */ if (B_size < MINBUFSIZE) B_size = MINBUFSIZE; if ((buf = malloc(B_size)) == NULL) errx(1, "%s", strerror(ENOMEM)); /* timeouts */ if ((s = getenv("FTP_TIMEOUT")) != NULL) { ftp_timeout = strtol(s, &end, 10); if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { warnx("FTP_TIMEOUT (%s) is not a positive integer", s); ftp_timeout = 0; } } if ((s = getenv("HTTP_TIMEOUT")) != NULL) { http_timeout = strtol(s, &end, 10); if (*s == '\0' || *end != '\0' || http_timeout < 0) { warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); http_timeout = 0; } } /* signal handling */ sa.sa_flags = 0; sa.sa_handler = sig_handler; sigemptyset(&sa.sa_mask); sigaction(SIGALRM, &sa, NULL); sa.sa_flags = SA_RESETHAND; sigaction(SIGINT, &sa, NULL); fetchRestartCalls = 0; /* output file */ if (o_flag) { if (strcmp(o_filename, "-") == 0) { o_stdout = 1; } else if (stat(o_filename, &sb) == -1) { if (errno == ENOENT) { if (argc > 1) errx(1, "%s is not a directory", o_filename); } else { err(1, "%s", o_filename); } } else { if (sb.st_mode & S_IFDIR) o_directory = 1; } } /* check if output is to a tty (for progress report) */ v_tty = isatty(STDERR_FILENO); if (v_tty) pgrp = getpgrp(); r = 0; /* authentication */ if (v_tty) fetchAuthMethod = query_auth; if (N_filename != NULL) if (setenv("NETRC", N_filename, 1) == -1) err(1, "setenv: cannot set NETRC=%s", N_filename); while (argc) { if ((p = strrchr(*argv, '/')) == NULL) p = *argv; else p++; if (!*p) p = "fetch.out"; fetchLastErrCode = 0; if (o_flag) { if (o_stdout) { e = fetch(*argv, "-"); } else if (o_directory) { asprintf(&q, "%s/%s", o_filename, p); e = fetch(*argv, q); free(q); } else { e = fetch(*argv, o_filename); } } else { e = fetch(*argv, p); } if (sigint) kill(getpid(), SIGINT); if (e == 0 && once_flag) exit(0); if (e) { r = 1; if ((fetchLastErrCode && fetchLastErrCode != FETCH_UNAVAIL && fetchLastErrCode != FETCH_MOVED && fetchLastErrCode != FETCH_URL && fetchLastErrCode != FETCH_RESOLV && fetchLastErrCode != FETCH_UNKNOWN)) { if (w_secs && v_level) fprintf(stderr, "Waiting %ld seconds " "before retrying\n", w_secs); if (w_secs) sleep(w_secs); if (a_flag) continue; } } argc--, argv++; } exit(r); }