diff --git a/Makefile.inc1 b/Makefile.inc1 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -2748,6 +2748,7 @@ ${_m4} \ ${_lex} \ ${_other_bootstrap_tools} \ + lib/libutil \ usr.bin/xinstall \ ${_gensnmptree} \ usr.sbin/config \ @@ -2785,6 +2786,12 @@ ${_bt}-usr.bin/xinstall: ${_bt}-lib/libmd ${_bt}-sbin/md5: ${_bt}-lib/libmd .endif +.if target(${_bt}-lib/libutil) +# If we are bootstrapping libutil (e.g. when building on macOS/Linux) add the +# necessary dependencies: +${_bt}-bin/cat: ${_bt}-lib/libutil +${_bt}-usr.bin/xinstall: ${_bt}-lib/libutil +.endif # diff --git a/bin/cat/Makefile b/bin/cat/Makefile --- a/bin/cat/Makefile +++ b/bin/cat/Makefile @@ -4,6 +4,8 @@ PACKAGE=runtime PROG= cat +LIBADD= util + .ifdef BOOTSTRAPPING # For the bootstrap cat we disable all wide char support to allow building # on Linux/macOS diff --git a/bin/cat/cat.c b/bin/cat/cat.c --- a/bin/cat/cat.c +++ b/bin/cat/cat.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -67,9 +68,7 @@ static void scanfiles(char *argv[], int cooked); #ifndef BOOTSTRAP_CAT static void cook_cat(FILE *); -static ssize_t in_kernel_copy(int); #endif -static void raw_cat(int); #ifndef NO_UDOM_SUPPORT static cap_channel_t *capnet; @@ -231,53 +230,49 @@ static void scanfiles(char *argv[], int cooked __unused) { - int fd, i; + int i, rfd, wfd; char *path; + ssize_t done; #ifndef BOOTSTRAP_CAT FILE *fp; #endif i = 0; - fd = -1; + rfd = -1; while ((path = argv[i]) != NULL || i == 0) { if (path == NULL || strcmp(path, "-") == 0) { filename = "stdin"; - fd = STDIN_FILENO; + rfd = STDIN_FILENO; } else { filename = path; - fd = fileargs_open(fa, path); + rfd = fileargs_open(fa, path); #ifndef NO_UDOM_SUPPORT - if (fd < 0 && errno == EOPNOTSUPP) - fd = udom_open(path, O_RDONLY); + if (rfd < 0 && errno == EOPNOTSUPP) + rfd = udom_open(path, O_RDONLY); #endif } - if (fd < 0) { + if (rfd < 0) { warn("%s", path); rval = 1; #ifndef BOOTSTRAP_CAT } else if (cooked) { - if (fd == STDIN_FILENO) + if (rfd == STDIN_FILENO) cook_cat(stdin); else { - fp = fdopen(fd, "r"); + fp = fdopen(rfd, "r"); cook_cat(fp); fclose(fp); } #endif } else { -#ifndef BOOTSTRAP_CAT - if (in_kernel_copy(fd) != 0) { - if (errno == EINVAL || errno == EBADF || - errno == EISDIR) - raw_cat(fd); - else - err(1, "stdout"); - } -#else - raw_cat(fd); -#endif - if (fd != STDIN_FILENO) - close(fd); + wfd = fileno(stdout); + do { + done = fcopydata(rfd, wfd); + } while (done > 0); + if (done < 0) + err(1, "%s", filename); + if (rfd != STDIN_FILENO) + close(rfd); } if (path == NULL) break; @@ -375,62 +370,8 @@ if (ferror(stdout)) err(1, "stdout"); } - -static ssize_t -in_kernel_copy(int rfd) -{ - int wfd; - ssize_t ret; - - wfd = fileno(stdout); - ret = 1; - - while (ret > 0) - ret = copy_file_range(rfd, NULL, wfd, NULL, SSIZE_MAX, 0); - - return (ret); -} #endif /* BOOTSTRAP_CAT */ -static void -raw_cat(int rfd) -{ - long pagesize; - int off, wfd; - ssize_t nr, nw; - static size_t bsize; - static char *buf = NULL; - struct stat sbuf; - - wfd = fileno(stdout); - if (buf == NULL) { - if (fstat(wfd, &sbuf)) - err(1, "stdout"); - if (S_ISREG(sbuf.st_mode)) { - /* If there's plenty of RAM, use a large copy buffer */ - if (sysconf(_SC_PHYS_PAGES) > PHYSPAGES_THRESHOLD) - bsize = MIN(BUFSIZE_MAX, MAXPHYS * 8); - else - bsize = BUFSIZE_SMALL; - } else { - bsize = sbuf.st_blksize; - pagesize = sysconf(_SC_PAGESIZE); - if (pagesize > 0) - bsize = MAX(bsize, (size_t)pagesize); - } - if ((buf = malloc(bsize)) == NULL) - err(1, "malloc() failure of IO buffer"); - } - while ((nr = read(rfd, buf, bsize)) > 0) - for (off = 0; nr; nr -= nw, off += nw) - if ((nw = write(wfd, buf + off, (size_t)nr)) < 0) - err(1, "stdout"); - if (nr < 0) { - warn("%s", filename); - rval = 1; - } -} - #ifndef NO_UDOM_SUPPORT static int diff --git a/bin/cp/Makefile b/bin/cp/Makefile --- a/bin/cp/Makefile +++ b/bin/cp/Makefile @@ -6,6 +6,8 @@ SRCS= cp.c utils.c CFLAGS+= -D_ACL_PRIVATE +LIBADD= util + HAS_TESTS= SUBDIR.${MK_TESTS}= tests diff --git a/bin/cp/utils.c b/bin/cp/utils.c --- a/bin/cp/utils.c +++ b/bin/cp/utils.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -45,66 +46,43 @@ #include "extern.h" -#define cp_pct(x, y) ((y == 0) ? 0 : (int)(100.0 * (x) / (y))) - -/* - * Memory strategy threshold, in pages: if physmem is larger then this, use a - * large buffer. - */ -#define PHYSPAGES_THRESHOLD (32*1024) - -/* Maximum buffer size in bytes - do not allow it to grow larger than this. */ -#define BUFSIZE_MAX (2*1024*1024) - -/* - * Small (default) buffer size in bytes. It's inefficient for this to be - * smaller than MAXPHYS. - */ -#define BUFSIZE_SMALL (MAXPHYS) - /* * Prompt used in -i case. */ #define YESNO "(y/n [n]) " -static ssize_t -copy_fallback(int from_fd, int to_fd) +#define cp_pct(x, y) ((y == 0) ? 0 : (int)(100.0 * (x) / (y))) +static void +copy_file_progress(const char *frompath, const char *topath, + struct stat *fromst, size_t done) { - static char *buf = NULL; - static size_t bufsize; - ssize_t rcount, wresid, wcount = 0; - char *bufp; - - if (buf == NULL) { - if (sysconf(_SC_PHYS_PAGES) > PHYSPAGES_THRESHOLD) - bufsize = MIN(BUFSIZE_MAX, MAXPHYS * 8); - else - bufsize = BUFSIZE_SMALL; - buf = malloc(bufsize); - if (buf == NULL) - err(1, "Not enough memory"); - } - rcount = read(from_fd, buf, bufsize); - if (rcount <= 0) - return (rcount); - for (bufp = buf, wresid = rcount; ; bufp += wcount, wresid -= wcount) { - wcount = write(to_fd, bufp, wresid); - if (wcount <= 0) - break; - if (wcount >= wresid) - break; - } - return (wcount < 0 ? wcount : rcount); + char sdone[5], ssize[5]; + + humanize_number(sdone, sizeof(sdone), (int64_t)done, "", HN_AUTOSCALE, + HN_B | HN_NOSPACE | HN_DECIMAL); + if (S_ISREG(fromst->st_mode)) { + humanize_number(ssize, sizeof(ssize), (int64_t)fromst->st_size, + "", HN_AUTOSCALE, HN_B | HN_NOSPACE | HN_DECIMAL); + (void)fprintf(stderr, "%s -> %s %d%% (%s / %s)\n", frompath, + topath, cp_pct(done, fromst->st_size), sdone, ssize); + } else { + /* + * If this isn't a regular file, we most likely don't have + * its size, so reporting the progress in percents makes no + * sense. + */ + (void)fprintf(stderr, "%s -> %s %s\n", frompath, topath, + sdone); + } } +#undef cp_pct int copy_file(const FTSENT *entp, int dne) { struct stat sb, *fs; - ssize_t wcount; - off_t wtotal; + ssize_t wcount, wtotal; int ch, checkch, from_fd, rval, to_fd; - int use_copy_file_range = 1; fs = entp->fts_statp; from_fd = to_fd = -1; @@ -196,24 +174,12 @@ wtotal = 0; do { - if (use_copy_file_range) { - wcount = copy_file_range(from_fd, NULL, - to_fd, NULL, SSIZE_MAX, 0); - if (wcount < 0 && errno == EINVAL) { - /* probably a non-seekable descriptor */ - use_copy_file_range = 0; - } - } - if (!use_copy_file_range) { - wcount = copy_fallback(from_fd, to_fd); - } + wcount = fcopydata_sig(from_fd, to_fd, &info); wtotal += wcount; if (info) { info = 0; - (void)fprintf(stderr, - "%s -> %s %3d%%\n", - entp->fts_path, to.p_path, - cp_pct(wtotal, fs->st_size)); + copy_file_progress(entp->fts_path, to.p_path, fs, + wtotal); } } while (wcount > 0); if (wcount < 0) { diff --git a/bin/mv/Makefile b/bin/mv/Makefile --- a/bin/mv/Makefile +++ b/bin/mv/Makefile @@ -4,6 +4,8 @@ PACKAGE=runtime PROG= mv +LIBADD= util + HAS_TESTS= SUBDIR.${MK_TESTS}+= tests diff --git a/bin/mv/mv.c b/bin/mv/mv.c --- a/bin/mv/mv.c +++ b/bin/mv/mv.c @@ -44,9 +44,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -57,6 +59,7 @@ #define EXEC_FAILED 127 static int fflg, hflg, iflg, nflg, vflg; +static volatile sig_atomic_t info; static int copy(const char *, const char *); static int do_move(const char *, const char *); @@ -64,6 +67,7 @@ static void usage(void); static void preserve_fd_acls(int source_fd, int dest_fd, const char *source_path, const char *dest_path); +static void siginfo(int sig __unused); int main(int argc, char *argv[]) @@ -104,6 +108,8 @@ if (argc < 2) usage(); + (void)signal(SIGINFO, siginfo); + /* * If the stat on the target fails or the target isn't a directory, * try the move. More than 2 arguments is an error in this case. @@ -258,25 +264,45 @@ fastcopy(from, to, &sb) : copy(from, to)); } +#define cp_pct(x, y) ((y == 0) ? 0 : (int)(100.0 * (x) / (y))) +static void +fastcopy_progress(const char *frompath, const char *topath, + struct stat *fromst, size_t done) +{ + char sdone[5], ssize[5]; + + humanize_number(sdone, sizeof(sdone), (int64_t)done, "", HN_AUTOSCALE, + HN_B | HN_NOSPACE | HN_DECIMAL); + if (S_ISREG(fromst->st_mode)) { + humanize_number(ssize, sizeof(ssize), (int64_t)fromst->st_size, + "", HN_AUTOSCALE, HN_B | HN_NOSPACE | HN_DECIMAL); + (void)fprintf(stderr, "%s -> %s %d%% (%s / %s)\n", frompath, + topath, cp_pct(done, fromst->st_size), sdone, ssize); + } else { + /* + * If this isn't a regular file, we most likely don't have + * its size, so reporting the progress in percents makes no + * sense. + */ + (void)fprintf(stderr, "%s -> %s %s\n", frompath, topath, + sdone); + } +} +#undef cp_pct + static int fastcopy(const char *from, const char *to, struct stat *sbp) { struct timespec ts[2]; - static u_int blen = MAXPHYS; - static char *bp = NULL; mode_t oldmode; - int nread, from_fd, to_fd; + int from_fd, to_fd; + ssize_t wcount, wtotal; struct stat tsb; if ((from_fd = open(from, O_RDONLY, 0)) < 0) { warn("fastcopy: open() failed (from): %s", from); return (1); } - if (bp == NULL && (bp = malloc((size_t)blen)) == NULL) { - warnx("malloc(%u) failed", blen); - (void)close(from_fd); - return (1); - } while ((to_fd = open(to, O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0)) < 0) { if (errno == EEXIST && unlink(to) == 0) @@ -285,14 +311,18 @@ (void)close(from_fd); return (1); } - while ((nread = read(from_fd, bp, (size_t)blen)) > 0) - if (write(to_fd, bp, (size_t)nread) != nread) { - warn("fastcopy: write() failed: %s", to); - goto err; + wtotal = 0; + do { + wcount = fcopydata_sig(from_fd, to_fd, &info); + wtotal += wcount; + if (info) { + info = 0; + fastcopy_progress(from, to, sbp, wtotal); } - if (nread < 0) { - warn("fastcopy: read() failed: %s", from); -err: if (unlink(to)) + } while (wcount > 0); + if (wcount < 0) { + warn("fastcopy: failed: %s -> %s", from, to); + if (unlink(to)) warn("%s: remove", to); (void)close(from_fd); (void)close(to_fd); @@ -490,6 +520,13 @@ acl_free(acl); } +static void +siginfo(int sig __unused) +{ + + info = 1; +} + static void usage(void) { diff --git a/lib/libutil/Makefile b/lib/libutil/Makefile --- a/lib/libutil/Makefile +++ b/lib/libutil/Makefile @@ -10,8 +10,9 @@ LIB= util SHLIB_MAJOR= 9 -SRCS= _secure_path.c auth.c cpuset.c expand_number.c flopen.c fparseln.c \ - getlocalbase.c gr_util.c \ +SRCS= _secure_path.c auth.c cpuset.c expand_number.c \ + fcopydata.c flopen.c fparseln.c \ + getlocalbase.c gr_util.c \ hexdump.c humanize_number.c kinfo_getfile.c \ kinfo_getallproc.c kinfo_getproc.c kinfo_getvmmap.c \ kinfo_getvmobject.c kld.c \ @@ -26,6 +27,9 @@ .if ${MK_INET6_SUPPORT} != "no" CFLAGS+= -DINET6 .endif +.ifdef BOOTSTRAPPING +CFLAGS+= -DBOOTSTRAP_LIBUTIL +.endif CFLAGS+= -I${.CURDIR} -I${SRCTOP}/lib/libc/gen/ @@ -60,9 +64,9 @@ pidfile.3 pidfile_open.3 \ pidfile.3 pidfile_remove.3 \ pidfile.3 pidfile_write.3 -MLINKS+=property.3 property_find.3 property.3 properties_free.3 +MLINKS+=property.3 property_find.3 property.3 properties_free.3 MLINKS+=property.3 properties_read.3 -MLINKS+=pty.3 forkpty.3 pty.3 openpty.3 +MLINKS+=pty.3 forkpty.3 pty.3 openpty.3 MLINKS+=quotafile.3 quota_close.3 \ quotafile.3 quota_fsname.3 \ quotafile.3 quota_open.3 \ diff --git a/lib/libutil/fcopydata.c b/lib/libutil/fcopydata.c new file mode 100644 --- /dev/null +++ b/lib/libutil/fcopydata.c @@ -0,0 +1,155 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Pawel Jakub Dawidek + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include + +#include +#include +#include +#include +#include + +/* + * Memory strategy threshold, in pages: if physmem is larger then this, use a + * large buffer. + */ +#define PHYSPAGES_THRESHOLD (32*1024) + +/* Maximum buffer size in bytes - do not allow it to grow larger than this. */ +#define BUFSIZE_MAX (2*1024*1024) + +/* + * Small (default) buffer size in bytes. It's inefficient for this to be + * smaller than MAXPHYS. + */ +#define BUFSIZE_SMALL (MAXPHYS) + +#include +ssize_t +fcopydata_sig(int infd, int outfd, volatile sig_atomic_t *gotsigp) +{ + unsigned char *buf; + size_t bufsize; + ssize_t done, tdone; + int serrno; + + serrno = errno; + +#ifndef BOOTSTRAP_LIBUTIL + /* + * Let's try copy_file_range(2), which should always work if we are + * dealing with regular files. + */ + done = copy_file_range(infd, NULL, outfd, NULL, SSIZE_MAX, 0); + if (done >= 0) { + errno = serrno; + return (done); + } + /* + * Fall back to read(2)/write(2) in case of the following errors: + * EBADF - outfd might be opened with O_APPEND flag + * EINVAL - we may be operating on non-regular file, + * like a fifo or a device + * EISDIR - file system may support reading directories directly + * when security.bsd.allow_read_dir is set + */ + if (errno != EBADF && errno != EINVAL && errno != EISDIR) { + return (done); + } +#endif + /* + * Probably a non-seekable descriptor, fallback to read/write. + */ + + if (sysconf(_SC_PHYS_PAGES) > PHYSPAGES_THRESHOLD) + bufsize = MIN(BUFSIZE_MAX, MAXPHYS * 8); + else + bufsize = BUFSIZE_SMALL; + buf = malloc(bufsize); + if (buf == NULL) { + return (-1); + } + tdone = 0; + for (;;) { + size_t todo, written; + + done = read(infd, buf, bufsize); + if (done == 0) { + done = tdone; + break; + } else if (done < 0) { + break; + } + + written = 0; + todo = done; + do { + done = write(outfd, buf + written, todo - written); + if (done < 0) { + break; + } else if (done == 0) { + /* This should not happen... */ + assert(todo - written > 0); + errno = EINVAL; + done = -1; + break; + } + written += done; + tdone += done; + assert(written <= todo); + } while (written < todo); + if (done < 0) { + break; + } + assert(written == todo); + + if (todo < bufsize || (gotsigp != NULL && *gotsigp)) { + /* + * We had a short read, so return now. + */ + done = tdone; + break; + } + } + free(buf); + + if (done >= 0) { + errno = serrno; + } + return (done); +} + +ssize_t +fcopydata(int infd, int outfd) +{ + + return (fcopydata_sig(infd, outfd, NULL)); +} diff --git a/lib/libutil/libutil.h b/lib/libutil/libutil.h --- a/lib/libutil/libutil.h +++ b/lib/libutil/libutil.h @@ -43,6 +43,8 @@ #include #include +#include + #ifndef _GID_T_DECLARED typedef __gid_t gid_t; #define _GID_T_DECLARED @@ -63,6 +65,11 @@ #define _SIZE_T_DECLARED #endif +#ifndef _SSIZE_T_DECLARED +typedef __ssize_t ssize_t; +#define _SSIZE_T_DECLARED +#endif + #ifndef _UID_T_DECLARED typedef __uid_t uid_t; #define _UID_T_DECLARED @@ -89,6 +96,8 @@ char *auth_getval(const char *_name); void clean_environment(const char * const *_white, const char * const *_more_white); +ssize_t fcopydata(int _infd, int _outfd); +ssize_t fcopydata_sig(int infd, int outfd, volatile sig_atomic_t *gotsigp); int expand_number(const char *_buf, uint64_t *_num); int extattr_namespace_to_string(int _attrnamespace, char **_string); int extattr_string_to_namespace(const char *_string, int *_attrnamespace); diff --git a/usr.bin/xinstall/Makefile b/usr.bin/xinstall/Makefile --- a/usr.bin/xinstall/Makefile +++ b/usr.bin/xinstall/Makefile @@ -11,15 +11,11 @@ .PATH: ${SRCTOP}/contrib/mtree CFLAGS+= -I${SRCTOP}/contrib/mtree CFLAGS+= -I${SRCTOP}/lib/libnetbsd +CFLAGS+= -I${SRCTOP}/lib/libutil -LIBADD= md +LIBADD= md util CFLAGS+= -DWITH_MD5 -DWITH_RIPEMD160 -.ifdef BOOTSTRAPPING -# For the bootstrap we disable copy_file_range() -CFLAGS+= -DBOOTSTRAP_XINSTALL -.endif - HAS_TESTS= SUBDIR.${MK_TESTS}+= tests diff --git a/usr.bin/xinstall/xinstall.c b/usr.bin/xinstall/xinstall.c --- a/usr.bin/xinstall/xinstall.c +++ b/usr.bin/xinstall/xinstall.c @@ -39,6 +39,7 @@ #include #include #include +#include #ifdef WITH_MD5 #include #endif @@ -1182,9 +1183,7 @@ static size_t bufsize; int nr, nw; int serrno; -#ifndef BOOTSTRAP_XINSTALL ssize_t ret; -#endif DIGEST_CTX ctx; /* Rewind file descriptors. */ @@ -1193,24 +1192,19 @@ if (lseek(to_fd, 0, SEEK_SET) < 0) err(EX_OSERR, "lseek: %s", to_name); -#ifndef BOOTSTRAP_XINSTALL /* Try copy_file_range() if no digest is requested */ if (digesttype == DIGEST_NONE) { do { - ret = copy_file_range(from_fd, NULL, to_fd, NULL, - (size_t)size, 0); + ret = fcopydata(from_fd, to_fd); } while (ret > 0); if (ret == 0) goto done; - if (errno != EINVAL) { - serrno = errno; - (void)unlink(to_name); - errno = serrno; - err(EX_OSERR, "%s", to_name); - } - /* Fall back */ + serrno = errno; + (void)unlink(to_name); + errno = serrno; + err(EX_OSERR, "%s", to_name); } -#endif + digest_init(&ctx); if (buf == NULL) { @@ -1250,9 +1244,7 @@ errno = serrno; err(EX_OSERR, "%s", from_name); } -#ifndef BOOTSTRAP_XINSTALL done: -#endif if (safecopy && fsync(to_fd) == -1) { serrno = errno; (void)unlink(to_name);