Page MenuHomeFreeBSD
Files F177531301

D61275.1794644484.diff
No OneTemporary
Actions

Authored By
Unknown
Size
12 KB
Referenced Files
None
Subscribers
None

D61275.1794644484.diff
View Options

diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -51,6 +51,9 @@
# xargs -n1 | sort | uniq -d;
# done
+# 20261106: libfetch symbol versioning, shared library bumped to version 7
+OLD_LIBS+=usr/lib/libfetch.so.6
+
# 20261020: Regenerate trust store
OLD_FILES+=usr/share/certs/trusted/Atos_TrustedRoot_2011.pem
OLD_FILES+=usr/share/certs/trusted/SecureSign_Root_CA12.pem
diff --git a/lib/libfetch/Makefile b/lib/libfetch/Makefile
--- a/lib/libfetch/Makefile
+++ b/lib/libfetch/Makefile
@@ -24,7 +24,10 @@
CFLAGS+= -DFTP_COMBINE_CWDS
-SHLIB_MAJOR= 6
+SHLIB_MAJOR= 7
+
+VERSION_DEF= ${SRCTOP}/lib/libc/Versions.def
+SYMBOL_MAPS= ${.CURDIR}/Symbol.map
ftperr.h: ftp.errors ${.CURDIR}/Makefile
@echo "static struct fetcherr ftp_errlist[] = {" > ${.TARGET}
diff --git a/lib/libfetch/Symbol.map b/lib/libfetch/Symbol.map
new file mode 100644
--- /dev/null
+++ b/lib/libfetch/Symbol.map
@@ -0,0 +1,37 @@
+FBSD_1.8 {
+ fetchAuthMethod;
+ fetchDebug;
+ fetchFreeURL;
+ fetchGet;
+ fetchGetFTP;
+ fetchGetFile;
+ fetchGetHTTP;
+ fetchGetURL;
+ fetchLastErrCode;
+ fetchLastErrString;
+ fetchList;
+ fetchListFTP;
+ fetchListFile;
+ fetchListHTTP;
+ fetchListURL;
+ fetchMakeURL;
+ fetchParseURL;
+ fetchPut;
+ fetchPutFTP;
+ fetchPutFile;
+ fetchPutHTTP;
+ fetchPutURL;
+ fetchReqHTTP;
+ fetchRestartCalls;
+ fetchStat;
+ fetchStatFTP;
+ fetchStatFile;
+ fetchStatHTTP;
+ fetchStatURL;
+ fetchTimeout;
+ fetchXGet;
+ fetchXGetFTP;
+ fetchXGetFile;
+ fetchXGetHTTP;
+ fetchXGetURL;
+};
diff --git a/lib/libfetch/fetch.h b/lib/libfetch/fetch.h
--- a/lib/libfetch/fetch.h
+++ b/lib/libfetch/fetch.h
@@ -36,6 +36,8 @@
#define URL_SCHEMELEN 16
#define URL_USERLEN 256
#define URL_PWDLEN 256
+/* Longest HTTP validator (entity tag or date) kept for If-Range */
+#define FETCH_VALIDATOR_SIZE 256
struct url {
char scheme[URL_SCHEMELEN+1];
@@ -48,6 +50,7 @@
size_t length;
time_t ims_time;
int netrcfd;
+ char validator[FETCH_VALIDATOR_SIZE];
};
struct url_stat {
@@ -87,6 +90,7 @@
#define FETCH_UNKNOWN 17
#define FETCH_URL 18
#define FETCH_VERBOSE 19
+#define FETCH_RESTART 20
__BEGIN_DECLS
diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3
--- a/lib/libfetch/fetch.3
+++ b/lib/libfetch/fetch.3
@@ -24,7 +24,7 @@
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
-.Dd June 27, 2026
+.Dd November 6, 2026
.Dt FETCH 3
.Os
.Sh NAME
@@ -170,15 +170,34 @@
off_t offset;
size_t length;
time_t ims_time;
+ int netrcfd;
+ char validator[FETCH_VALIDATOR_SIZE];
};
.Ed
.Pp
The
+.Va offset
+field requests that the transfer start at the given byte offset rather
+than at the beginning of the document.
+On return, it holds the offset at which the server actually started the
+transfer, which is lower than requested when the server does not
+support, or did not honor, partial transfers.
+.Pp
+The
.Va ims_time
field stores the time value for
.Li If-Modified-Since
HTTP requests.
.Pp
+The
+.Va validator
+field holds the strong entity tag of the document most recently
+retrieved with this
+.Vt "struct url" ,
+or an empty string if the server sent none; see the
+.Sx HTTP SCHEME
+section.
+.Pp
The pointer returned by
.Fn fetchMakeURL
or
@@ -403,6 +422,33 @@
HTTP header to only fetch the content if it is newer than
.Va ims_time .
.Pp
+Every successful HTTP request records the document's strong entity tag
+in the
+.Va validator
+field.
+When the same
+.Vt "struct url"
+is used again with a non-zero
+.Va offset ,
+.Fn fetchXGetHTTP
+and
+.Fn fetchGetHTTP
+send it in an
+.Li If-Range
+HTTP header, so that a server which has changed the document in the
+meantime sends all of it, from offset zero, instead of the tail of a
+different version.
+If the server reports that the range can no longer be served, or sends
+part of a document with a different entity tag, the request fails with
+.Dv FETCH_RESTART
+and should be retried with
+.Va offset
+set to zero.
+The entity tag is that of the document finally retrieved, after any
+redirects; clear
+.Va validator
+before reusing the structure for a different document.
+.Pp
The function
.Fn fetchReqHTTP
can be used to make requests with an arbitrary HTTP verb,
@@ -550,6 +596,8 @@
Unknown error
.It Bq Er FETCH_URL
Invalid URL
+.It Bq Er FETCH_RESTART
+Transfer must be restarted from the beginning
.El
.Pp
The accompanying error message includes a protocol-specific error code
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -384,6 +384,7 @@
hdr_unknown = 1,
hdr_content_length,
hdr_content_range,
+ hdr_etag,
hdr_last_modified,
hdr_location,
hdr_transfer_encoding,
@@ -398,6 +399,7 @@
} hdr_names[] = {
{ hdr_content_length, "Content-Length" },
{ hdr_content_range, "Content-Range" },
+ { hdr_etag, "ETag" },
{ hdr_last_modified, "Last-Modified" },
{ hdr_location, "Location" },
{ hdr_transfer_encoding, "Transfer-Encoding" },
@@ -904,6 +906,27 @@
return (0);
}
+/*
+ * Copy a strong entity tag, the only kind that can validate a range
+ * (RFC 9110, 13.1.5), checking that it can safely be sent back.
+ */
+static int
+http_parse_etag(const char *p, char *etag)
+{
+ size_t len;
+
+ if (p[0] != '"')
+ return (-1);
+ for (len = 1; p[len] != '"'; len++)
+ if ((unsigned char)p[len] < 0x21 || p[len] == 0x7f)
+ return (-1);
+ if (++len >= FETCH_VALIDATOR_SIZE)
+ return (-1);
+ memcpy(etag, p, len);
+ etag[len] = '\0';
+ return (0);
+}
+
/*
* Parse a content-length header
*/
@@ -1593,10 +1616,11 @@
char hbuf[MAXHOSTNAMELEN + 7], *host;
conn_t *conn;
struct url *url, *loc;
- int chunked, direct, ims, noredirect, verbose;
+ int chunked, direct, ifrange, ims, noredirect, verbose;
int e, i, n, val;
off_t offset, clength, length, size;
time_t mtime;
+ char validator[FETCH_VALIDATOR_SIZE];
const char *p;
FILE *f;
hdr_t h;
@@ -1616,6 +1640,9 @@
verbose = CHECK_FLAG('v');
ims = CHECK_FLAG('i');
+ /* a resumed transfer is conditional on the document being unchanged */
+ ifrange = (URL->offset > 0 && URL->validator[0] != '\0');
+
if (direct && purl) {
fetchFreeURL(purl);
purl = NULL;
@@ -1636,6 +1663,7 @@
length = -1;
size = -1;
mtime = 0;
+ validator[0] = '\0';
/* check port */
if (!url->port)
@@ -1769,8 +1797,15 @@
http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER,
getprogname());
}
- if (url->offset > 0)
+ if (url->offset > 0) {
http_cmd(conn, "Range: bytes=%lld-", (long long)url->offset);
+ if (ifrange) {
+ if (verbose)
+ fetch_info("If-Range: %s",
+ URL->validator);
+ http_cmd(conn, "If-Range: %s", URL->validator);
+ }
+ }
http_cmd(conn, "Connection: close");
if (body) {
@@ -1881,6 +1916,10 @@
case hdr_content_range:
http_parse_range(p, &offset, &length, &size);
break;
+ case hdr_etag:
+ if (http_parse_etag(p, validator) != 0)
+ validator[0] = '\0';
+ break;
case hdr_last_modified:
http_parse_mtime(p, &mtime);
break;
@@ -1968,12 +2007,19 @@
/* requested range not satisfiable */
if (conn->err == HTTP_BAD_RANGE) {
- if (url->offset > 0 && url->length == 0) {
+ if (url->offset > 0 && url->length == 0 &&
+ (!ifrange || size == url->offset)) {
/* asked for 0 bytes; fake it */
offset = url->offset;
clength = -1;
conn->err = HTTP_OK;
break;
+ } else if (ifrange) {
+ /* the document has shrunk */
+ fetchLastErrCode = FETCH_RESTART;
+ strlcpy(fetchLastErrString,
+ "range no longer available", MAXERRSTRING);
+ goto ouch;
} else {
http_seterr(conn->err);
goto ouch;
@@ -2016,6 +2062,19 @@
goto ouch;
}
+ /* a conditional partial response must describe the same document */
+ if (ifrange && conn->err == HTTP_PARTIAL) {
+ if (validator[0] == '\0') {
+ strlcpy(validator, URL->validator, sizeof(validator));
+ } else if (strcmp(validator, URL->validator) != 0) {
+ fetchLastErrCode = FETCH_RESTART;
+ strlcpy(fetchLastErrString,
+ "document changed since the transfer started",
+ MAXERRSTRING);
+ goto ouch;
+ }
+ }
+
/* check for inconsistencies */
if (clength != -1 && length != -1 && clength != length) {
http_seterr(HTTP_PROTOCOL_ERROR);
@@ -2044,9 +2103,11 @@
goto ouch;
}
- /* report back real offset and size */
+ /* report back real offset and size, and remember the validator */
URL->offset = offset;
URL->length = clength;
+ if (!HTTP_ERROR(conn->err))
+ strlcpy(URL->validator, validator, sizeof(URL->validator));
/* wrap it up in a FILE */
if ((f = http_funopen(conn, chunked)) == NULL) {
diff --git a/usr.bin/fetch/fetch.1 b/usr.bin/fetch/fetch.1
--- a/usr.bin/fetch/fetch.1
+++ b/usr.bin/fetch/fetch.1
@@ -28,7 +28,7 @@
.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.\"
-.Dd June 27, 2026
+.Dd November 6, 2026
.Dt FETCH 1
.Os
.Sh NAME
@@ -110,6 +110,9 @@
error when the requested object does not exist.
.It Fl a , -retry
Automatically retry the transfer upon soft failures.
+Each retry resumes where the previous attempt stopped.
+Over HTTP, when the server identifies the file with an entity tag, a
+retry resumes only if the file is unchanged, and starts over otherwise.
.It Fl B Ar bytes , Fl -buffer-size= Ns Ar bytes
Specify the read buffer size in bytes.
The default is 16,384 bytes.
diff --git a/usr.bin/fetch/fetch.c b/usr.bin/fetch/fetch.c
--- a/usr.bin/fetch/fetch.c
+++ b/usr.bin/fetch/fetch.c
@@ -430,7 +430,7 @@
char flags[8];
const char *slash;
char *tmppath;
- int r, tries;
+ int r;
unsigned timeout;
int timedout = 0;
char *ptr;
@@ -528,8 +528,6 @@
goto success;
}
- tries = 1;
-again:
r = 0;
/*
* If the -r flag was specified, we have to compare the local
@@ -547,7 +545,7 @@
sb.st_size = -1;
if (!o_stdout) {
r = stat(path, &sb);
- if (r == 0 && (r_flag || tries > 1) && S_ISREG(sb.st_mode)) {
+ if (r == 0 && r_flag && S_ISREG(sb.st_mode)) {
url->offset = sb.st_size;
} else if (r == -1 || !S_ISREG(sb.st_mode)) {
/*
@@ -564,8 +562,15 @@
}
size_prev = sb.st_size;
+again:
/* start the transfer */
f = fetchXGet(url, &us, flags);
+ if (f == NULL && fetchLastErrCode == FETCH_RESTART) {
+ /* the server will not resume from our copy: start over */
+ warnx("%s: %s", URL, fetchLastErrString);
+ url->offset = 0;
+ f = fetchXGet(url, &us, flags);
+ }
if (f == NULL) {
if (i_flag && *is_http && fetchLastErrCode == FETCH_OK &&
strcmp(fetchLastErrString, "Not Modified") == 0) {
@@ -614,14 +619,24 @@
}
/* open output file */
- if (o_stdout) {
+ if (of != NULL) {
+ /* retrying: drop what the server did not resume */
+ if (fflush(of) != 0 ||
+ ftruncate(fileno(of), url->offset) != 0 ||
+ fseeko(of, url->offset, SEEK_SET) != 0) {
+ warn("%s", tmppath ? tmppath : path);
+ goto failure;
+ }
+ size_prev = url->offset;
+ } else if (o_stdout) {
/* output to stdout */
of = stdout;
} else if (r_flag && sb.st_size != -1) {
/* resume mode, local file exists */
- if (!F_flag && us.mtime && sb.st_mtime != us.mtime && tries == 1) {
+ if (!F_flag && us.mtime && sb.st_mtime != us.mtime) {
/* no match! have to refetch */
fclose(f);
+ f = NULL;
/* if precious, warn the user and give up */
if (R_flag) {
warnx("%s: local modification time "
@@ -686,12 +701,14 @@
* remote files didn't match.
*/
- if (url->offset > 0) {
+ if (f == NULL || url->offset > 0) {
/*
* We tried to restart a transfer, but for
* some reason gave up - so we have to restart
* from scratch if we want the whole file
*/
+ if (f != NULL)
+ fclose(f);
url->offset = 0;
if ((f = fetchXGet(url, &us, flags)) == NULL) {
warnx("%s: %s", URL, fetchLastErrString);
@@ -823,10 +840,11 @@
warnx("%s appears to be truncated: %jd/%jd bytes",
path, (intmax_t)count, (intmax_t)us.size);
if(!o_stdout && a_flag && count > size_prev) {
+ /* resume from what we have written */
fclose(f);
+ url->offset = count;
if (w_secs)
sleep(w_secs);
- tries++;
goto again;
}
goto failure_keep;

File Metadata

Mime Type
text/plain
Expires
Sat, Nov 14, 8:21 AM (4 h, 12 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41718851
Default Alt Text
D61275.1794644484.diff (12 KB)

Event Timeline