Index: head/sbin/recoverdisk/Makefile =================================================================== --- head/sbin/recoverdisk/Makefile (revision 359562) +++ head/sbin/recoverdisk/Makefile (revision 359563) @@ -1,9 +1,13 @@ # $FreeBSD$ PACKAGE=runtime PROG= recoverdisk +LDFLAGS += -lm + +WARNS?= 6 + .include test: ${PROG} ./${PROG} /dev/ad0 Index: head/sbin/recoverdisk/recoverdisk.1 =================================================================== --- head/sbin/recoverdisk/recoverdisk.1 (revision 359562) +++ head/sbin/recoverdisk/recoverdisk.1 (revision 359563) @@ -1,174 +1,183 @@ .\" Copyright (c) 2006 Ulrich Spoerlein .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" .Dd October 1, 2013 .Dt RECOVERDISK 1 .Os .Sh NAME .Nm recoverdisk .Nd recover data from hard disk or optical media .Sh SYNOPSIS .Nm .Op Fl b Ar bigsize .Op Fl r Ar readlist .Op Fl s Ar interval +.Op Fl u Ar pattern +.Op Fl v .Op Fl w Ar writelist .Ar source .Op Ar destination .Sh DESCRIPTION The .Nm utility reads data from the .Ar source file until all blocks could be successfully read. If .Ar destination was specified all data is being written to that file. It starts reading in multiples of the sector size. Whenever a block fails, it is put to the end of the working queue and will be read again, possibly with a smaller read size. .Pp By default it uses block sizes of roughly 1 MB, 32kB, and the native sector size (usually 512 bytes). These figures are adjusted slightly, for devices whose sectorsize is not a power of 2, e.g., audio CDs with a sector size of 2352 bytes. .Pp The options are as follows: .Bl -tag -width indent .It Fl b Ar bigsize The size of reads attempted first. The middle pass is roughly the logarithmic average of the bigsize and the sectorsize. .It Fl r Ar readlist Read the list of blocks and block sizes to read from the specified file. .It Fl s Ar interval How often we should update the writelist file while things go OK. The default is 60 and the unit is "progress messages" so if things go well, this is the same as once per minute. +.It Fl u Ar pattern +By default blocks which encounter read errors will be filled with +the pattern "_UNREAD_" in the output file. This option can be +used to specify another pattern. Nothing gets written if the string +is empty. +.It Fl v +Enables nicer status report using ANSI escapes and UTF-8. .It Fl w Ar writelist Write the list of remaining blocks to read to the specified file if .Nm is aborted via .Dv SIGINT . .El .Pp The .Fl r and .Fl w options can be specified together. Especially, they can point to the same file, which will be updated on abort. .Sh OUTPUT The .Nm utility prints several columns, detailing the progress .Bl -tag -width remaining .It Va start Starting offset of the current block. .It Va size Read size of the current block. .It Va len Length of the current block. .It Va state Is increased for every failed read. .It Va done Number of bytes already read. .It Va remaining Number of bytes remaining. .It Va "% done" Percent complete. .El .Sh EXAMPLES .Bd -literal # recover data from failing hard drive ada3 recoverdisk /dev/ada3 /data/disk.img # clone a hard disk recoverdisk /dev/ada3 /dev/ada4 # read an ISO image from a CD-ROM recoverdisk /dev/cd0 /data/cd.iso # continue reading from a broken CD and update the existing worklist recoverdisk -r worklist -w worklist /dev/cd0 /data/cd.iso # recover a single file from the unreadable media recoverdisk /cdrom/file.avi file.avi # If the disk hangs the system on read-errors try: recoverdisk -b 0 /dev/ada3 /somewhere .Ed .Sh SEE ALSO .Xr dd 1 , .Xr ada 4 , .Xr cam 4 , .Xr cd 4 , .Xr da 4 .Sh HISTORY The .Nm utility first appeared in .Fx 7.0 . .Sh AUTHORS .An -nosplit The original implementation was done by .An Poul-Henning Kamp Aq Mt phk@FreeBSD.org with minor improvements from .An Ulrich Sp\(:orlein Aq Mt uqs@FreeBSD.org . .Pp This manual page was written by .An Ulrich Sp\(:orlein . .Sh BUGS Reading from media where the sectorsize is not a power of 2 will make all 1 MB reads fail. This is due to the DMA reads being split up into blocks of at most 128kB. These reads then fail if the sectorsize is not a divisor of 128kB. When reading a full raw audio CD, this leads to roughly 700 error messages flying by. This is harmless and can be avoided by setting .Fl b to no more than 128kB. .Pp .Nm needs to know about read errors as fast as possible, i.e. retries by lower layers will usually slow down the operation. When using .Xr cam 4 attached drives, you may want to set kern.cam.XX.retry_count to zero, e.g.: .Bd -literal # sysctl kern.cam.ada.retry_count=0 # sysctl kern.cam.cd.retry_count=0 # sysctl kern.cam.da.retry_count=0 .Ed .\".Pp .\"When reading from optical media, a bug in the GEOM framework will .\"prevent it from seeing that the media has been removed. .\"The device can still be opened, but all reads will fail. .\"This is usually harmless, but will send .\".Nm .\"into an infinite loop. Index: head/sbin/recoverdisk/recoverdisk.c =================================================================== --- head/sbin/recoverdisk/recoverdisk.c (revision 359562) +++ head/sbin/recoverdisk/recoverdisk.c (revision 359563) @@ -1,325 +1,620 @@ /*- * SPDX-License-Identifier: Beerware * * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): * wrote this file. As long as you retain this notice you * can do whatever you want with this stuff. If we meet some day, and you think * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * * $FreeBSD$ */ #include #include #include #include +#include #include #include +#include #include #include #include #include #include #include +#include #include #include -static volatile sig_atomic_t aborting = 0; -static size_t bigsize = 1024 * 1024; -static size_t medsize; -static size_t minsize = 512; +/* Safe printf into a fixed-size buffer */ +#define bprintf(buf, fmt, ...) \ + do { \ + int ibprintf; \ + ibprintf = snprintf(buf, sizeof buf, fmt, __VA_ARGS__); \ + assert(ibprintf >= 0 && ibprintf < (int)sizeof buf); \ + } while (0) struct lump { off_t start; off_t len; int state; TAILQ_ENTRY(lump) list; }; +struct period { + time_t t0; + time_t t1; + char str[20]; + off_t bytes_read; + TAILQ_ENTRY(period) list; +}; +TAILQ_HEAD(period_head, period); + +static volatile sig_atomic_t aborting = 0; +static int verbose = 0; +static size_t bigsize = 1024 * 1024; +static size_t medsize; +static size_t minsize = 512; +static off_t tot_size; +static off_t done_size; +static char *input; +static char *wworklist = NULL; +static char *rworklist = NULL; +static const char *unreadable_pattern = "_UNREAD_"; +static const int write_errors_are_fatal = 1; +static int fdr, fdw; + static TAILQ_HEAD(, lump) lumps = TAILQ_HEAD_INITIALIZER(lumps); +static struct period_head minute = TAILQ_HEAD_INITIALIZER(minute); +static struct period_head quarter = TAILQ_HEAD_INITIALIZER(quarter); +static struct period_head hour = TAILQ_HEAD_INITIALIZER(quarter); +static struct period_head day = TAILQ_HEAD_INITIALIZER(quarter); +/**********************************************************************/ + static void +report_good_read2(time_t now, size_t bytes, struct period_head *ph, time_t dt) +{ + struct period *pp; + const char *fmt; + struct tm tm1; + + pp = TAILQ_FIRST(ph); + if (pp == NULL || pp->t1 < now) { + pp = calloc(sizeof *pp, 1L); + assert(pp != NULL); + pp->t0 = (now / dt) * dt; + pp->t1 = (now / dt + 1) * dt; + assert(localtime_r(&pp->t0, &tm1) != NULL); + if (dt < 86400) + fmt = "%H:%M"; + else + fmt = "%d%b"; + assert(strftime(pp->str, sizeof pp->str, fmt, &tm1) != 0); + TAILQ_INSERT_HEAD(ph, pp, list); + } + pp->bytes_read += bytes; +} + +static void +report_good_read(time_t now, size_t bytes) +{ + + report_good_read2(now, bytes, &minute, 60L); + report_good_read2(now, bytes, &quarter, 900L); + report_good_read2(now, bytes, &hour, 3600L); + report_good_read2(now, bytes, &day, 86400L); +} + +static void +report_one_period(const char *period, struct period_head *ph) +{ + struct period *pp; + int n; + + n = 0; + printf("%s \xe2\x94\x82", period); + TAILQ_FOREACH(pp, ph, list) { + if (n == 3) { + TAILQ_REMOVE(ph, pp, list); + free(pp); + break; + } + if (n++) + printf(" \xe2\x94\x82"); + printf(" %s %14jd", pp->str, pp->bytes_read); + } + for (; n < 3; n++) { + printf(" \xe2\x94\x82"); + printf(" %5s %14s", "", ""); + } + printf("\x1b[K\n"); +} + +static void +report_periods(void) +{ + report_one_period("1m ", &minute); + report_one_period("15m", &quarter); + report_one_period("1h ", &hour); + report_one_period("1d ", &day); +} + +/**********************************************************************/ + +static void +set_verbose(void) +{ + struct winsize wsz; + time_t t0; + + if (!isatty(STDIN_FILENO) || ioctl(STDIN_FILENO, TIOCGWINSZ, &wsz)) + return; + printf("\x1b[2J"); + verbose = 1; + t0 = time(NULL); +} + +static void +report_header(int eol) +{ + printf("%13s %7s %13s %5s %13s %13s %9s", + "start", + "size", + "block-len", + "pass", + "done", + "remaining", + "% done"); + if (eol) + printf("\x1b[K"); + putchar('\n'); +} + +#define REPORTWID 79 + +static void +report_hline(const char *how) +{ + int j; + + for (j = 0; j < REPORTWID; j++) { + if (how && (j == 4 || j == 29 || j == 54)) { + printf("%s", how); + } else { + printf("\xe2\x94\x80"); + } + } + printf("\x1b[K\n"); +} + +static off_t hist[REPORTWID]; +static off_t last_done = -1; + +static void +report_histogram(const struct lump *lp) +{ + off_t j, bucket, fp, fe, k, now; + double a; + struct lump *lp2; + + bucket = tot_size / REPORTWID; + if (tot_size > bucket * REPORTWID) + bucket += 1; + if (done_size != last_done) { + memset(hist, 0, sizeof hist); + TAILQ_FOREACH(lp2, &lumps, list) { + fp = lp2->start; + fe = lp2->start + lp2->len; + for (j = fp / bucket; fp < fe; j++) { + k = (j + 1) * bucket; + if (k > fe) + k = fe; + k -= fp; + hist[j] += k; + fp += k; + } + } + last_done = done_size; + } + now = lp->start / bucket; + for (j = 0; j < REPORTWID; j++) { + a = round(8 * (double)hist[j] / bucket); + assert (a >= 0 && a < 9); + if (a == 0 && hist[j]) + a = 1; + if (j == now) + printf("\x1b[31m"); + if (a == 0) { + putchar(' '); + } else { + putchar(0xe2); + putchar(0x96); + putchar(0x80 + (int)a); + } + if (j == now) + printf("\x1b[0m"); + } + putchar('\n'); +} + +static void +report(const struct lump *lp, size_t sz) +{ + struct winsize wsz; + int j; + + assert(lp != NULL); + + if (verbose) { + printf("\x1b[H%s\x1b[K\n", input); + report_header(1); + } else { + putchar('\r'); + } + + printf("%13jd %7zu %13jd %5d %13jd %13jd %9.4f", + (intmax_t)lp->start, + sz, + (intmax_t)lp->len, + lp->state, + (intmax_t)done_size, + (intmax_t)(tot_size - done_size), + 100*(double)done_size/(double)tot_size + ); + + if (verbose) { + printf("\x1b[K\n"); + report_hline(NULL); + report_histogram(lp); + if (TAILQ_EMPTY(&minute)) { + report_hline(NULL); + } else { + report_hline("\xe2\x94\xac"); + report_periods(); + report_hline("\xe2\x94\xb4"); + } + j = ioctl(STDIN_FILENO, TIOCGWINSZ, &wsz); + if (!j) + printf("\x1b[%d;1H", wsz.ws_row); + } + fflush(stdout); +} + +/**********************************************************************/ + +static void new_lump(off_t start, off_t len, int state) { struct lump *lp; lp = malloc(sizeof *lp); if (lp == NULL) err(1, "Malloc failed"); lp->start = start; lp->len = len; lp->state = state; TAILQ_INSERT_TAIL(&lumps, lp, list); } -static struct lump *lp; -static char *wworklist = NULL; -static char *rworklist = NULL; +/********************************************************************** + * Save the worklist if -w was given + */ - -#define PRINT_HEADER \ - printf("%13s %7s %13s %5s %13s %13s %9s\n", \ - "start", "size", "block-len", "state", "done", "remaining", "% done") - -#define PRINT_STATUS(start, i, len, state, d, t) \ - printf("\r%13jd %7zu %13jd %5d %13jd %13jd %9.5f", \ - (intmax_t)start, \ - i, \ - (intmax_t)len, \ - state, \ - (intmax_t)d, \ - (intmax_t)(t - d), \ - 100*(double)d/(double)t) - -/* Save the worklist if -w was given */ static void save_worklist(void) { FILE *file; struct lump *llp; + char buf[PATH_MAX]; + if (fdw >= 0 && fdatasync(fdw)) + err(1, "Write error, probably disk full"); + if (wworklist != NULL) { + bprintf(buf, "%s.tmp", wworklist); (void)fprintf(stderr, "\nSaving worklist ..."); - fflush(stderr); + (void)fflush(stderr); - file = fopen(wworklist, "w"); + file = fopen(buf, "w"); if (file == NULL) - err(1, "Error opening file %s", wworklist); + err(1, "Error opening file %s", buf); TAILQ_FOREACH(llp, &lumps, list) fprintf(file, "%jd %jd %d\n", (intmax_t)llp->start, (intmax_t)llp->len, llp->state); - fclose(file); + (void)fflush(file); + if (ferror(file) || fdatasync(fileno(file)) || fclose(file)) + err(1, "Error writing file %s", buf); + if (rename(buf, wworklist)) + err(1, "Error renaming %s to %s", buf, wworklist); (void)fprintf(stderr, " done.\n"); } } /* Read the worklist if -r was given */ static off_t read_worklist(off_t t) { off_t s, l, d; int state, lines; FILE *file; (void)fprintf(stderr, "Reading worklist ..."); - fflush(stderr); + (void)fflush(stderr); file = fopen(rworklist, "r"); if (file == NULL) err(1, "Error opening file %s", rworklist); lines = 0; d = t; for (;;) { ++lines; if (3 != fscanf(file, "%jd %jd %d\n", &s, &l, &state)) { if (!feof(file)) err(1, "Error parsing file %s at line %d", rworklist, lines); else break; } new_lump(s, l, state); d -= l; } - fclose(file); + if (fclose(file)) + err(1, "Error closing file %s", rworklist); (void)fprintf(stderr, " done.\n"); /* * Return the number of bytes already read * (at least not in worklist). */ return (d); } +/**********************************************************************/ + static void +write_buf(int fd, const void *buf, ssize_t len, off_t where) +{ + ssize_t i; + + i = pwrite(fd, buf, len, where); + if (i == len) + return; + + printf("\nWrite error at %jd/%zu\n\t%s\n", + where, i, strerror(errno)); + save_worklist(); + if (write_errors_are_fatal) + exit(3); +} + +static void +fill_buf(char *buf, ssize_t len, const char *pattern) +{ + ssize_t sz = strlen(pattern); + ssize_t i, j; + + for (i = 0; i < len; i += sz) { + j = len - i; + if (j > sz) + j = sz; + memcpy(buf + i, pattern, j); + } +} + +/**********************************************************************/ + +static void usage(void) { (void)fprintf(stderr, "usage: recoverdisk [-b bigsize] [-r readlist] " "[-s interval] [-w writelist] source [destination]\n"); + /* XXX update */ exit(1); } static void sighandler(__unused int sig) { aborting = 1; } int main(int argc, char * const argv[]) { int ch; - int fdr, fdw; - off_t t, d, start, len; - size_t i, j; - int error, state; - u_char *buf; + size_t sz, j; + int error; + char *buf; u_int sectorsize; off_t stripesize; time_t t1, t2; struct stat sb; u_int n, snapshot = 60; + static struct lump *lp; - while ((ch = getopt(argc, argv, "b:r:w:s:")) != -1) { + while ((ch = getopt(argc, argv, "b:r:w:s:u:v")) != -1) { switch (ch) { case 'b': bigsize = strtoul(optarg, NULL, 0); break; case 'r': rworklist = strdup(optarg); if (rworklist == NULL) err(1, "Cannot allocate enough memory"); break; case 's': snapshot = strtoul(optarg, NULL, 0); break; + case 'u': + unreadable_pattern = optarg; + break; + case 'v': + set_verbose(); + break; case 'w': wworklist = strdup(optarg); if (wworklist == NULL) err(1, "Cannot allocate enough memory"); break; default: usage(); /* NOTREACHED */ } } argc -= optind; argv += optind; if (argc < 1 || argc > 2) usage(); + input = argv[0]; fdr = open(argv[0], O_RDONLY); if (fdr < 0) err(1, "Cannot open read descriptor %s", argv[0]); error = fstat(fdr, &sb); if (error < 0) err(1, "fstat failed"); if (S_ISBLK(sb.st_mode) || S_ISCHR(sb.st_mode)) { error = ioctl(fdr, DIOCGSECTORSIZE, §orsize); if (error < 0) err(1, "DIOCGSECTORSIZE failed"); error = ioctl(fdr, DIOCGSTRIPESIZE, &stripesize); if (error == 0 && stripesize > sectorsize) sectorsize = stripesize; minsize = sectorsize; bigsize = rounddown(bigsize, sectorsize); - error = ioctl(fdr, DIOCGMEDIASIZE, &t); + error = ioctl(fdr, DIOCGMEDIASIZE, &tot_size); if (error < 0) err(1, "DIOCGMEDIASIZE failed"); } else { - t = sb.st_size; + tot_size = sb.st_size; } if (bigsize < minsize) bigsize = minsize; for (ch = 0; (bigsize >> ch) > minsize; ch++) continue; medsize = bigsize >> (ch / 2); medsize = rounddown(medsize, minsize); fprintf(stderr, "Bigsize = %zu, medsize = %zu, minsize = %zu\n", bigsize, medsize, minsize); buf = malloc(bigsize); if (buf == NULL) err(1, "Cannot allocate %zu bytes buffer", bigsize); if (argc > 1) { fdw = open(argv[1], O_WRONLY | O_CREAT, DEFFILEMODE); if (fdw < 0) err(1, "Cannot open write descriptor %s", argv[1]); - if (ftruncate(fdw, t) < 0) + if (ftruncate(fdw, tot_size) < 0) err(1, "Cannot truncate output %s to %jd bytes", - argv[1], (intmax_t)t); + argv[1], (intmax_t)tot_size); } else fdw = -1; if (rworklist != NULL) { - d = read_worklist(t); + done_size = read_worklist(tot_size); } else { - new_lump(0, t, 0); - d = 0; + new_lump(0, tot_size, 0); + done_size = 0; } if (wworklist != NULL) signal(SIGINT, sighandler); - t1 = 0; - start = len = i = state = 0; - PRINT_HEADER; + t1 = time(NULL); + sz = 0; + if (!verbose) + report_header(0); n = 0; for (;;) { lp = TAILQ_FIRST(&lumps); if (lp == NULL) break; - while (lp->len > 0 && !aborting) { - /* These are only copied for printing stats */ - start = lp->start; - len = lp->len; - state = lp->state; + while (lp->len > 0) { - i = MIN(lp->len, (off_t)bigsize); - if (lp->state == 1) - i = MIN(lp->len, (off_t)medsize); - if (lp->state > 1) - i = MIN(lp->len, (off_t)minsize); - time(&t2); + if (lp->state == 0) + sz = MIN(lp->len, (off_t)bigsize); + else if (lp->state == 1) + sz = MIN(lp->len, (off_t)medsize); + else + sz = MIN(lp->len, (off_t)minsize); + assert(sz != 0); + + t2 = time(NULL); if (t1 != t2 || lp->len < (off_t)bigsize) { - PRINT_STATUS(start, i, len, state, d, t); t1 = t2; if (++n == snapshot) { save_worklist(); n = 0; } + report(lp, sz); } - if (i == 0) { - errx(1, "BOGUS i %10jd", (intmax_t)i); - } - fflush(stdout); - j = pread(fdr, buf, i, lp->start); - if (j == i) { - d += i; + + j = pread(fdr, buf, sz, lp->start); +#if 0 +if (!(random() & 0xf)) { + j = -1; + errno = EIO; +} +#endif + if (j == sz) { + done_size += sz; if (fdw >= 0) - j = pwrite(fdw, buf, i, lp->start); - else - j = i; - if (j != i) - printf("\nWrite error at %jd/%zu\n", - lp->start, i); - lp->start += i; - lp->len -= i; + write_buf(fdw, buf, sz, lp->start); + lp->start += sz; + lp->len -= sz; + if (verbose && lp->state > 2) + report_good_read(t2, sz); continue; } - printf("\n%jd %zu failed (%s)\n", - lp->start, i, strerror(errno)); - if (errno == EINVAL) { - printf("read() size too big? Try with -b 131072"); + error = errno; + + printf("%jd %zu %d read error (%s)\n", + lp->start, sz, lp->state, strerror(error)); + if (verbose) + report(lp, sz); + if (error == EINVAL) { + printf("Try with -b 131072 or lower ?\n"); aborting = 1; + break; } - if (errno == ENXIO) + if (error == ENXIO) { + printf("Input device probably detached...\n"); aborting = 1; - new_lump(lp->start, i, lp->state + 1); - lp->start += i; - lp->len -= i; + break; + } + if (fdw >= 0 && strlen(unreadable_pattern)) { + fill_buf(buf, sz, unreadable_pattern); + write_buf(fdw, buf, sz, lp->start); + } + new_lump(lp->start, sz, lp->state + 1); + lp->start += sz; + lp->len -= sz; } - if (aborting) { + if (aborting) save_worklist(); - return (0); - } + if (aborting || !TAILQ_NEXT(lp, list)) + report(lp, sz); + if (aborting) + break; + assert(lp->len == 0); TAILQ_REMOVE(&lumps, lp, list); free(lp); } - PRINT_STATUS(start, i, len, state, d, t); - save_worklist(); - printf("\nCompleted\n"); + printf("%s", aborting ? "Aborted\n" : "Completed\n"); + free(buf); return (0); }