Index: lib/libfetch/common.h =================================================================== --- lib/libfetch/common.h +++ lib/libfetch/common.h @@ -61,6 +61,14 @@ const SSL_METHOD *ssl_meth; /* SSL method */ #endif int ref; /* reference count */ + char scheme[URL_SCHEMELEN+1]; + char user[URL_USERLEN+1]; + char pwd[URL_PWDLEN+1]; + char host[MAXHOSTNAMELEN+1]; + int port; + int af; + int (*close)(conn_t *); + conn_t *next; }; /* Structure used for error message lists */ @@ -116,6 +124,7 @@ struct addrinfo *fetch_resolve(const char *, int, int); int fetch_bind(int, int, const char *); conn_t *fetch_connect(const char *, int, int, int); +conn_t *fetch_connect2(struct url *, int, int); conn_t *fetch_reopen(int); conn_t *fetch_ref(conn_t *); #ifdef WITH_SSL @@ -132,6 +141,8 @@ const char *, struct url_stat *); int fetch_netrc_auth(struct url *url); int fetch_no_proxy_match(const char *); +conn_t *fetch_cache_get(const struct url *, int); +void fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)); #define ftp_seterr(n) fetch_seterr(ftp_errlist, n) #define http_seterr(n) fetch_seterr(http_errlist, n) Index: lib/libfetch/common.c =================================================================== --- lib/libfetch/common.c +++ lib/libfetch/common.c @@ -136,6 +136,10 @@ /* End-of-Line */ static const char ENDL[2] = "\r\n"; +/* Keep alive cached connections */ +static conn_t *connection_cache = NULL; +int cache_global_limit = 0; +int cache_per_host_limit = 0; /*** Error-reporting functions ***********************************************/ @@ -579,7 +583,29 @@ return (2); } +/* + * Establish a TCP connection to the specified url + * and store the url information for later use + */ +conn_t * +fetch_connect2(struct url *u, int af, int verbose) +{ + conn_t *conn = NULL; + conn = fetch_connect(u->host, u->port, af, verbose); + if (conn == NULL) + return (NULL); + + strlcpy(conn->scheme, u->scheme, sizeof(conn->scheme)); + strlcpy(conn->host, u->host, sizeof(conn->host)); + strlcpy(conn->user, u->user, sizeof(conn->user)); + strlcpy(conn->pwd, u->pwd, sizeof(conn->pwd)); + conn->port = u->port; + conn->af = af; + + return (conn); +} + /* * Establish a TCP connection to the specified port on the specified host. */ @@ -1797,4 +1823,90 @@ } while (*q); return (0); +} + +/* + * Initialise the cache with given limits + */ +void +fetchConnectionCacheInit(int global_limit, int per_host_limit) +{ + if (global_limit < 0) + cache_global_limit = INT_MAX; + else if (per_host_limit > global_limit) + cache_global_limit = per_host_limit; + else + cache_global_limit = global_limit; + if (per_host_limit < 0) + cache_per_host_limit = INT_MAX; + else + cache_per_host_limit = per_host_limit; +} + +/* + * Flush cache and free all associated resources. + */ +void +fetchConnectionCacheClose(void) +{ + conn_t *conn; + + while ((conn = connection_cache) != NULL) { + connection_cache = conn->next; + (*conn->close)(conn); + } +} + +/* + * Check connection cache for an existing entry matching + * protocol/host/port/user/password/family + */ +conn_t * +fetch_cache_get(const struct url *url, int af) +{ + conn_t *conn; + + for (conn = connection_cache; conn; conn = conn->next) { + if (conn->port == url->port && + strcmp(conn->scheme, url->scheme) == 0 && + strcmp(conn->host, url->host) == 0 && + strcmp(conn->user, url->user) == 0 && + strcmp(conn->pwd, url->pwd) == 0 && + (conn->af == AF_UNSPEC || af == AF_UNSPEC || + conn->af == af)) { + return (conn); + } + + } + return (NULL); +} + +/* + * Put the connection back into the pool + */ +void +fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)) +{ + conn_t *iter, *last; + int global_count, host_count; + + global_count = host_count = 0; + last = NULL; + for (iter = connection_cache; iter ; last = iter, iter = iter->next) { + ++global_count; + if (strcmp(conn->host, iter->host) == 0) + ++host_count; + if (global_count < cache_global_limit && + host_count < cache_per_host_limit) + continue; + --global_count; + if (last != NULL) + last->next = iter->next; + else + connection_cache = iter->next; + (*iter->close)(iter); + } + conn->close = closecb; + conn->next = connection_cache; + connection_cache = conn; } Index: lib/libfetch/fetch.h =================================================================== --- lib/libfetch/fetch.h +++ lib/libfetch/fetch.h @@ -133,6 +133,10 @@ struct url *fetchParseURL(const char *); void fetchFreeURL(struct url *); +/* Connection caching */ +void fetchConnectionCacheInit(int, int); +void fetchConnectionCacheClose(void); + __END_DECLS /* Authentication */ @@ -152,5 +156,10 @@ /* Extra verbosity */ extern int fetchDebug; + +/* cache / keep-alive */ + +extern int cache_global_limit; +extern int cache_per_host_limit; #endif Index: lib/libfetch/fetch.3 =================================================================== --- lib/libfetch/fetch.3 +++ lib/libfetch/fetch.3 @@ -26,7 +26,7 @@ .\" .\" $FreeBSD$ .\" -.Dd August 28, 2019 +.Dd February 18, 2020 .Dt FETCH 3 .Os .Sh NAME @@ -124,6 +124,10 @@ .Fn fetchStatFTP "struct url *u" "struct url_stat *us" "const char *flags" .Ft struct url_ent * .Fn fetchListFTP "struct url *u" "const char *flags" +.Ft void +.Fn fetchConnectionCacheInit "int global" "int per_host" +.Ft void +.Fn fetchConnectionCacheClose "void" .Sh DESCRIPTION These functions implement a high-level library for retrieving and uploading files using Uniform Resource Locators (URLs). @@ -291,6 +295,16 @@ functions is read-only, and that a stream returned by one of the .Fn fetchPutXXX functions is write-only. +.Pp +.Fn fetchConnectionCacheInit +initialize the size of the cache used for the keep-alive support. +By default keep-alive is disabled, if one of the caches is non 0 then +keep-alive will be enabled. +.Pp +.Fn fetchConnectionCacheClose +will loop over all the cached connections and close them. +.Va -1 +can be used to enable the maximum number of cached connections. .Sh FILE SCHEME .Fn fetchXGetFile , .Fn fetchGetFile Index: lib/libfetch/http.c =================================================================== --- lib/libfetch/http.c +++ lib/libfetch/http.c @@ -131,6 +131,7 @@ { conn_t *conn; /* connection */ int chunked; /* chunked mode */ + int keep_alive; /* keep-alive mode */ char *buf; /* chunk buffer */ size_t bufsize; /* size of chunk buffer */ size_t buflen; /* amount of data currently in buffer */ @@ -316,11 +317,21 @@ http_closefn(void *v) { struct httpio *io = (struct httpio *)v; - int r; + int r, val; - r = fetch_close(io->conn); - if (io->buf) - free(io->buf); + if (io->keep_alive) { + val = 0; + setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NODELAY, &val, + sizeof(val)); + fetch_cache_put(io->conn, fetch_close); + val = 1; + setsockopt(io->conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, + sizeof(val)); + r = 0; + } else { + r = fetch_close(io->conn); + } + free(io->buf); free(io); return (r); } @@ -329,7 +340,7 @@ * Wrap a file descriptor up */ static FILE * -http_funopen(conn_t *conn, int chunked) +http_funopen(conn_t *conn, int chunked, int keep_alive) { struct httpio *io; FILE *f; @@ -340,6 +351,7 @@ } io->conn = conn; io->chunked = chunked; + io->keep_alive = keep_alive; f = funopen(io, http_readfn, http_writefn, NULL, http_closefn); if (f == NULL) { fetch_syserr(); @@ -360,6 +372,7 @@ hdr_error = -1, hdr_end = 0, hdr_unknown = 1, + hdr_connection, hdr_content_length, hdr_content_range, hdr_last_modified, @@ -374,6 +387,7 @@ hdr_t num; const char *name; } hdr_names[] = { + { hdr_connection, "Connection" }, { hdr_content_length, "Content-Length" }, { hdr_content_range, "Content-Range" }, { hdr_last_modified, "Last-Modified" }, @@ -1377,7 +1391,7 @@ * Connect to the correct HTTP server or proxy. */ static conn_t * -http_connect(struct url *URL, struct url *purl, const char *flags) +http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) { struct url *curl; conn_t *conn; @@ -1404,8 +1418,12 @@ curl = (purl != NULL) ? purl : URL; - if ((conn = fetch_connect(curl->host, curl->port, af, verbose)) == NULL) - /* fetch_connect() has already set an error code */ + if ((conn = fetch_cache_get(curl, af)) != NULL) { + *cached = 1; + return (conn); + } + if ((conn = fetch_connect2(curl, af, verbose)) == NULL) + /* fetch_connect2() has already set an error code */ return (NULL); init_http_headerbuf(&headerbuf); if (strcmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) { @@ -1546,7 +1564,7 @@ char hbuf[MAXHOSTNAMELEN + 7], *host; conn_t *conn; struct url *url, *new; - int chunked, direct, ims, noredirect, verbose; + int cached, chunked, direct, ims, keep_alive, noredirect, verbose; int e, i, n, val; off_t offset, clength, length, size; time_t mtime; @@ -1568,6 +1586,7 @@ noredirect = CHECK_FLAG('A'); verbose = CHECK_FLAG('v'); ims = CHECK_FLAG('i'); + keep_alive = 0; if (direct && purl) { fetchFreeURL(purl); @@ -1589,6 +1608,7 @@ length = -1; size = -1; mtime = 0; + cached = 0; /* check port */ if (!url->port) @@ -1603,7 +1623,7 @@ } /* connect to server or proxy */ - if ((conn = http_connect(url, purl, flags)) == NULL) + if ((conn = http_connect(url, purl, flags, &cached)) == NULL) goto ouch; /* append port number only if necessary */ @@ -1724,7 +1744,9 @@ } if (url->offset > 0) http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset); - http_cmd(conn, "Connection: close"); + http_cmd(conn, "Connection: %s", + (cache_global_limit > 0 || cache_per_host_limit > 0) ? + "keep-alive" : "close" ); if (body) { body_len = strlen(body); @@ -1828,6 +1850,10 @@ case hdr_error: http_seterr(HTTP_PROTOCOL_ERROR); goto ouch; + case hdr_connection: + /* XXX too weak? */ + keep_alive = (strcasecmp(p, "keep-alive") == 0); + break; case hdr_content_length: http_parse_length(p, &clength); break; @@ -2002,7 +2028,7 @@ URL->length = clength; /* wrap it up in a FILE */ - if ((f = http_funopen(conn, chunked)) == NULL) { + if ((f = http_funopen(conn, chunked, keep_alive)) == NULL) { fetch_syserr(); goto ouch; }