Index: share/man/man4/geom_uzip.4 =================================================================== --- share/man/man4/geom_uzip.4 +++ share/man/man4/geom_uzip.4 @@ -30,7 +30,7 @@ .Os .Sh NAME .Nm geom_uzip -.Nd "GEOM based compressed disk images" +.Nd "GEOM based compressed disk images and partitions" .Sh SYNOPSIS To compile this driver into the kernel, place the following line in your @@ -51,7 +51,7 @@ framework provides support for compressed read only disk images. This allows significant storage savings at the expense of -a little CPU time on each read. +a some CPU time on each read. Data written in the GEOM label area allows .Nm to detect compressed images which have been created with @@ -65,15 +65,53 @@ .Pp The .Nm +is not limited to support +.Xr md 4 +images only. +The image can also be copied to a block device, either read-write such as +disk drive, flash drive or flash card, or read-only such as DVD-ROM. +When that disk partition is probed or media is inserted the appropriate +device node with the +.Pa .uzip +suffix will appear: +.Bd -literal -offset indent +# gpart show da0 +=> 0 7833600 da0 BSD (3.7G) + 0 2097152 1 freebsd-ufs (1.0G) + 2097152 5736448 - free - (2.7G) +# gpart add -t freebsd-ufs -s 1G da0 +da0b added +# dd if=/tmp/20160217_dcomp_zcomp.uzip bs=256k of=/dev/da0b +3190+1 records in +3190+1 records out +836331008 bytes transferred in 111.021489 secs (7533055 bytes/sec) +# fsck -t ffs /dev/da0b.uzip +** /dev/da0b.uzip (NO WRITE) +** Last Mounted on /mnt +** Phase 1 - Check Blocks and Sizes +** Phase 2 - Check Pathnames +** Phase 3 - Check Connectivity +** Phase 4 - Check Reference Counts +** Phase 5 - Check Cyl groups +97455 files, 604242 used, 184741 free (2349 frags, 22799 blocks, + 0.3% fragmentation) +# mount -o ro /dev/da0b.uzip /mnt +# df /dev/da0b.uzip +Filesystem 1K-blocks Used Avail Capacity Mounted on +/dev/da0b.uzip 3155932 2416968 738964 77% /mnt +.Ed +.Pp +The +.Nm device is subsequently used by the .Fx -kernel to access the disk images. +kernel to access the compressed data. The .Nm driver does not allow write operations to the underlying disk image. To check which -.Xr md 4 -devices match a given +.Dq providers +match a given .Nm device: .Bd -literal -offset indent @@ -83,13 +121,46 @@ 1. Name: md1.uzip Mediasize: 22003712 (21M) Sectorsize: 512 - Mode: r1w0e1 Consumers: 1. Name: md1 Mediasize: 9563648 (9.1M) Sectorsize: 512 - Mode: r1w0e1 + +Geom name: da0b.uzip +Providers: +1. Name: da0b.uzip + Mediasize: 3355443200 (3.1G) + Sectorsize: 512 +Consumers: +1. Name: da0b + Mediasize: 1073741824 (1.0G) + Sectorsize: 512 .Ed +.Pp +The +.Nm +allows mounting root file system off compressed preloaded image or disk +partition by setting +.Dv vfs.root.mountfrom +kernel environment parameter. +See +.Xr loader.conf 5 +for details. +.Sh DIAGNOSTICS +Several flags are provided for tracing +.Nm +I/O operations and TOC parsing via the following sysctls. +.Bl -tag -width indent +.It Va kern.geom.uzip.debug +Log level. +Setting it to the value greater than 0 would enable debug logging in key +places of the +.Nm +code. +Supported levels are from 0 (no logging) to 3 (maximum amount of logging). +.It Va kern.geom.uzip.debug_block +Log key operations involving specific compressed cluster number. +.El .Sh SEE ALSO .Xr GEOM 4 , .Xr md 4 , @@ -101,5 +172,12 @@ .Nm driver was written by .An Max Khon Aq Mt fjoe@FreeBSD.org . +The block de-duplication code as well as some +.Nm +driver optimizations have been contributed by +.An Maxim Sobolev Aq Mt sobomax@FreeBSD.org . +The LZMA decompression support (aka CLOOP 3.0) has been added by +.An Aleksandr Rybalko Aq Mt ray@FreeBSD.org . +.Pp This manual page was written by .An Ceri Davies Aq Mt ceri@FreeBSD.org . Index: sys/geom/uzip/g_uzip.h =================================================================== --- /dev/null +++ sys/geom/uzip/g_uzip.h @@ -0,0 +1,20 @@ +extern int g_uzip_debug; +extern int g_uzip_debug_block; + +#if 0 +#undef GEOM_UZIP_DEBUG +#ifdef GEOM_UZIP_DEBUG +#define DPRINTF(lvl, a) printf a +#define DPRINTF_BLK(lvl, cn, a) printf a +#else +#define DPRINTF(lvl, a) if ((lvl) <= g_uzip_debug) { \ + printf a; \ + } +#define DPRINTF_BLK(lvl, cn, a) if ((lvl) <= g_uzip_debug || \ + BLK_IN_RANGE(cn, g_uzip_debug_block, 10)) { + printf a; \ + } +#endif +#endif + +MALLOC_DECLARE(M_GEOM_UZIP); Index: sys/geom/uzip/g_uzip.c =================================================================== --- sys/geom/uzip/g_uzip.c +++ sys/geom/uzip/g_uzip.c @@ -1,6 +1,7 @@ /*- * Copyright (c) 2004 Max Khon * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,19 +39,70 @@ #include #include #include -#include #include +#include +#include +#include -FEATURE(geom_uzip, "GEOM uzip read-only compressed disks support"); +FEATURE(geom_uzip, "GEOM read-only compressed disks support"); + +struct g_uzip_blk { + uint64_t offset; + uint32_t blen; +#define BLEN_UNDEF UINT32_MAX +}; + +#ifndef ABS +#define ABS(a) ((a) < 0 ? -(a) : (a)) +#endif + +#define BLK_IN_RANGE(mcn, bcn, ilen) \ + (((bcn) != BLEN_UNDEF) && ( \ + ((ilen) >= 0 && (mcn >= bcn) && (mcn <= ((intmax_t)(bcn) + (ilen)))) || \ + ((ilen) < 0 && (mcn <= bcn) && (mcn >= ((intmax_t)(bcn) + (ilen)))) \ + )) -#undef GEOM_UZIP_DEBUG #ifdef GEOM_UZIP_DEBUG -#define DPRINTF(a) printf a +# define GEOM_UZIP_DBG_DEFAULT 3 #else -#define DPRINTF(a) +# define GEOM_UZIP_DBG_DEFAULT 0 #endif +#define GUZ_DBG_ERR 1 +#define GUZ_DBG_INFO 2 +#define GUZ_DBG_IO 3 +#define GUZ_DBG_TOC 4 + +SYSCTL_DECL(_kern_geom); +SYSCTL_NODE(_kern_geom, OID_AUTO, uzip, CTLFLAG_RW, 0, "GEOM_UZIP stuff"); +static u_int g_uzip_debug = GEOM_UZIP_DBG_DEFAULT; +SYSCTL_UINT(_kern_geom_uzip, OID_AUTO, debug, CTLFLAG_RWTUN, &g_uzip_debug, 0, + "Debug level"); +static u_int g_uzip_debug_block = BLEN_UNDEF; +SYSCTL_UINT(_kern_geom_uzip, OID_AUTO, debug_block, CTLFLAG_RWTUN, + &g_uzip_debug_block, 0, "Debug operations around specific cluster#"); + +#define DPRINTF(lvl, a) \ + if ((lvl) <= g_uzip_debug) { \ + printf a; \ + } +#define DPRINTF_BLK(lvl, cn, a) \ + if ((lvl) <= g_uzip_debug || \ + BLK_IN_RANGE(cn, g_uzip_debug_block, 8) || \ + BLK_IN_RANGE(cn, g_uzip_debug_block, -8)) { \ + printf a; \ + } +#define DPRINTF_BRNG(lvl, bcn, ecn, a) \ + if (bcn >= ecn) { \ + printf("DPRINTF_BRNG: invalid range (%ju, %ju), BUG BUG " \ + "BUG!\n", (uintmax_t)bcn, (uintmax_t)ecn); \ + } else if (((lvl) <= g_uzip_debug) || \ + BLK_IN_RANGE(g_uzip_debug_block, bcn, \ + (intmax_t)ecn - (intmax_t)bcn)) { \ + printf a; \ + } + static MALLOC_DEFINE(M_GEOM_UZIP, "geom_uzip", "GEOM UZIP data structures"); #define UZIP_CLASS_NAME "UZIP" @@ -68,6 +120,8 @@ #define CLOOP_MAGIC_LEN 128 static char CLOOP_MAGIC_START[] = "#!/bin/sh\n"; +#define CLOOP_COMPR_OFF 0x0b +#define CLOOP_VERSN_OFF (CLOOP_COMPR_OFF + 1) struct cloop_header { char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ @@ -78,13 +132,14 @@ struct g_uzip_softc { uint32_t blksz; /* block size */ uint32_t nblocks; /* number of blocks */ - uint64_t *offsets; + struct g_uzip_blk *toc; /* table of contents */ struct mtx last_mtx; uint32_t last_blk; /* last blk no */ char *last_buf; /* last blk data */ int req_total; /* total requests */ int req_cached; /* cached requests */ + struct g_uzip_dapi *dcp; /* decompressor instance */ }; static void g_uzip_done(struct bio *bp); @@ -94,35 +149,16 @@ { if (gp != NULL) { - DPRINTF(("%s: %d requests, %d cached\n", + DPRINTF(GUZ_DBG_INFO, ("%s: %d requests, %d cached\n", gp->name, sc->req_total, sc->req_cached)); } - if (sc->offsets != NULL) { - free(sc->offsets, M_GEOM_UZIP); - sc->offsets = NULL; - } + sc->dcp->free(sc->dcp); + free(sc->toc, M_GEOM_UZIP); mtx_destroy(&sc->last_mtx); free(sc->last_buf, M_GEOM_UZIP); free(sc, M_GEOM_UZIP); } -static void * -z_alloc(void *nil, u_int type, u_int size) -{ - void *ptr; - - ptr = malloc(type * size, M_GEOM_UZIP, M_NOWAIT); - - return (ptr); -} - -static void -z_free(void *nil, void *ptr) -{ - - free(ptr, M_GEOM_UZIP); -} - static int g_uzip_cached(struct g_geom *gp, struct bio *bp) { @@ -144,8 +180,9 @@ sc->req_cached++; mtx_unlock(&sc->last_mtx); - DPRINTF(("%s/%s: %p: offset=%jd: got %jd bytes from cache\n", - __func__, gp->name, bp, (intmax_t)ofs, (intmax_t)usz)); + DPRINTF(GUZ_DBG_IO, ("%s/%s: %p: offset=%jd: got %jd bytes " + "from cache\n", __func__, gp->name, bp, (intmax_t)ofs, + (intmax_t)usz)); bp->bio_completed += usz; bp->bio_resid -= usz; @@ -160,6 +197,19 @@ return (0); } +#define BLK_ENDS(sc, bi) ((sc)->toc[(bi)].offset + \ + (sc)->toc[(bi)].blen) + +#define BLK_IS_CONT(sc, bi) (BLK_ENDS((sc), (bi) - 1) == \ + (sc)->toc[(bi)].offset) +#define BLK_IS_NIL(sc, bi) ((sc)->toc[(bi)].blen == 0) + +#define TOFF_2_BOFF(sc, pp, bi) ((sc)->toc[(bi)].offset - \ + (sc)->toc[(bi)].offset % (pp)->sectorsize) +#define TLEN_2_BLEN(sc, pp, bp, ei) ((BLK_ENDS((sc), (ei)) - \ + (bp)->bio_offset + (pp)->sectorsize - 1) / \ + (pp)->sectorsize * (pp)->sectorsize) + static int g_uzip_request(struct g_geom *gp, struct bio *bp) { @@ -167,21 +217,14 @@ struct bio *bp2; struct g_consumer *cp; struct g_provider *pp; - off_t ofs; - size_t start_blk, end_blk; + off_t ofs, start_blk_ofs; + size_t i, start_blk, end_blk, zsize; if (g_uzip_cached(gp, bp) != 0) return (1); sc = gp->softc; - bp2 = g_clone_bio(bp); - if (bp2 == NULL) { - g_io_deliver(bp, ENOMEM); - return (1); - } - bp2->bio_done = g_uzip_done; - cp = LIST_FIRST(&gp->consumer); pp = cp->provider; @@ -191,17 +234,54 @@ end_blk = (ofs + bp->bio_resid + sc->blksz - 1) / sc->blksz; KASSERT(end_blk <= sc->nblocks, ("end_blk out of range")); - DPRINTF(("%s/%s: %p: start=%u (%jd), end=%u (%jd)\n", - __func__, gp->name, bp, - (u_int)start_blk, (intmax_t)sc->offsets[start_blk], - (u_int)end_blk, (intmax_t)sc->offsets[end_blk])); + for (; BLK_IS_NIL(sc, start_blk) && start_blk < end_blk; start_blk++) { + /* Fill in any leading Nil blocks */ + start_blk_ofs = ofs % sc->blksz; + zsize = MIN(sc->blksz - start_blk_ofs, bp->bio_resid); + DPRINTF_BLK(GUZ_DBG_IO, start_blk, ("%s/%s: %p/%ju: " + "filling %ju zero bytes\n", __func__, gp->name, gp, + (uintmax_t)bp->bio_completed, (uintmax_t)zsize)); + bzero(bp->bio_data + bp->bio_completed, zsize); + bp->bio_completed += zsize; + bp->bio_resid -= zsize; + ofs += zsize; + } + + if (start_blk == end_blk) { + KASSERT(bp->bio_resid == 0, ("bp->bio_resid is invalid")); + /* + * No non-Nil data is left, complete request immediately. + */ + DPRINTF(GUZ_DBG_IO, ("%s/%s: %p: all done returning %ju " + "bytes\n", __func__, gp->name, gp, + (uintmax_t)bp->bio_completed)); + g_io_deliver(bp, 0); + return (1); + } + + for (i = start_blk + 1; i < end_blk; i++) { + /* Trim discontinuous areas if any */ + if (!BLK_IS_CONT(sc, i)) { + end_blk = i; + break; + } + } - bp2->bio_offset = sc->offsets[start_blk] - - sc->offsets[start_blk] % pp->sectorsize; + DPRINTF_BRNG(GUZ_DBG_IO, start_blk, end_blk, ("%s/%s: %p: " + "start=%u (%ju), end=%u (%ju)\n", __func__, gp->name, bp, + (u_int)start_blk, (uintmax_t)sc->toc[start_blk].offset, + (u_int)end_blk, (uintmax_t)BLK_ENDS(sc, end_blk - 1))); + + bp2 = g_clone_bio(bp); + if (bp2 == NULL) { + g_io_deliver(bp, ENOMEM); + return (1); + } + bp2->bio_done = g_uzip_done; + + bp2->bio_offset = TOFF_2_BOFF(sc, pp, start_blk); while (1) { - bp2->bio_length = sc->offsets[end_blk] - bp2->bio_offset; - bp2->bio_length = (bp2->bio_length + pp->sectorsize - 1) / - pp->sectorsize * pp->sectorsize; + bp2->bio_length = TLEN_2_BLEN(sc, pp, bp2, end_blk - 1); if (bp2->bio_length <= MAXPHYS) break; @@ -215,8 +295,8 @@ return (1); } - DPRINTF(("%s/%s: %p: reading %jd bytes from offset %jd\n", - __func__, gp->name, bp, + DPRINTF_BRNG(GUZ_DBG_IO, start_blk, end_blk, ("%s/%s: %p: " + "reading %jd bytes from offset %jd\n", __func__, gp->name, bp, (intmax_t)bp2->bio_length, (intmax_t)bp2->bio_offset)); g_io_request(bp2, cp); @@ -226,7 +306,6 @@ static void g_uzip_done(struct bio *bp) { - z_stream zs; struct bio *bp2; struct g_provider *pp; struct g_consumer *cp; @@ -234,7 +313,8 @@ struct g_uzip_softc *sc; char *data, *data2; off_t ofs; - size_t blk, blkofs, len, ulen; + size_t blk, blkofs, len, ulen, firstblk; + int err; bp2 = bp->bio_parent; gp = bp2->bio_to->geom; @@ -253,46 +333,47 @@ goto done; } - zs.zalloc = z_alloc; - zs.zfree = z_free; - if (inflateInit(&zs) != Z_OK) { - bp2->bio_error = EILSEQ; - goto done; - } - ofs = bp2->bio_offset + bp2->bio_completed; - blk = ofs / sc->blksz; + firstblk = blk = ofs / sc->blksz; blkofs = ofs % sc->blksz; - data = bp->bio_data + sc->offsets[blk] % pp->sectorsize; + data = bp->bio_data + sc->toc[blk].offset % pp->sectorsize; data2 = bp2->bio_data + bp2->bio_completed; while (bp->bio_completed && bp2->bio_resid) { + if (blk > firstblk && !BLK_IS_CONT(sc, blk)) { + DPRINTF_BLK(GUZ_DBG_IO, blk, ("%s/%s: %p: backref'ed " + "cluster #%u requested, looping around\n", + __func__, gp->name, bp2, (u_int)blk)); + goto done; + } ulen = MIN(sc->blksz - blkofs, bp2->bio_resid); - len = sc->offsets[blk + 1] - sc->offsets[blk]; - DPRINTF(("%s/%s: %p/%ju: data2=%p, ulen=%u, data=%p, len=%u\n", - __func__, gp->name, gp, bp->bio_completed, - data2, (u_int)ulen, data, (u_int)len)); + len = sc->toc[blk].blen; + DPRINTF(GUZ_DBG_IO, ("%s/%s: %p/%ju: data2=%p, ulen=%u, " + "data=%p, len=%u\n", __func__, gp->name, gp, + bp->bio_completed, data2, (u_int)ulen, data, (u_int)len)); if (len == 0) { /* All zero block: no cache update */ bzero(data2, ulen); } else if (len <= bp->bio_completed) { - zs.next_in = data; - zs.avail_in = len; - zs.next_out = sc->last_buf; - zs.avail_out = sc->blksz; mtx_lock(&sc->last_mtx); - if (inflate(&zs, Z_FINISH) != Z_STREAM_END) { + err = sc->dcp->decompress(sc->dcp, gp->name, data, + len, sc->last_buf); + if (err != 0) { sc->last_blk = -1; mtx_unlock(&sc->last_mtx); - inflateEnd(&zs); bp2->bio_error = EILSEQ; + DPRINTF(GUZ_DBG_ERR, ("%s/%s: decompress" + "(%p) failed\n", __func__, gp->name, + sc->dcp)); goto done; } sc->last_blk = blk; memcpy(data2, sc->last_buf + blkofs, ulen); mtx_unlock(&sc->last_mtx); - if (inflateReset(&zs) != Z_OK) { - inflateEnd(&zs); + err = sc->dcp->rewind(sc->dcp, gp->name); + if (err != 0) { bp2->bio_error = EILSEQ; + DPRINTF(GUZ_DBG_ERR, ("%s/%s: rewind(%p) " + "failed\n", __func__, gp->name, sc->dcp)); goto done; } data += len; @@ -307,9 +388,6 @@ blk++; } - if (inflateEnd(&zs) != Z_OK) - bp2->bio_error = EILSEQ; - done: /* Finish processing the request. */ free(bp->bio_data, M_GEOM_UZIP); @@ -330,9 +408,9 @@ pp = bp->bio_to; gp = pp->geom; - DPRINTF(("%s/%s: %p: cmd=%d, offset=%jd, length=%jd, buffer=%p\n", - __func__, gp->name, bp, bp->bio_cmd, (intmax_t)bp->bio_offset, - (intmax_t)bp->bio_length, bp->bio_data)); + DPRINTF(GUZ_DBG_IO, ("%s/%s: %p: cmd=%d, offset=%jd, length=%jd, " + "buffer=%p\n", __func__, gp->name, bp, bp->bio_cmd, + (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length, bp->bio_data)); sc = gp->softc; sc->req_total++; @@ -392,6 +470,79 @@ g_wither_geom(gp, ENXIO); } +static int +g_uzip_parse_toc(struct g_uzip_softc *sc, struct g_provider *pp, + struct g_geom *gp) +{ + uint32_t i, j, backref_to; + uint64_t max_offset; + + max_offset = sc->toc[0].offset - 1; + for (i = 0; i < sc->nblocks; i++) { + DPRINTF_BLK(GUZ_DBG_IO, i, ("%s: cluster #%u " + "sc->toc[i].offset=%ju max_offset=%ju\n", gp->name, + (u_int)i, (uintmax_t)sc->toc[i].offset, + (uintmax_t)max_offset)); + backref_to = BLEN_UNDEF; + if (sc->toc[i].offset < max_offset) { + /* + * For the backref'ed blocks search already parsed + * TOC entries for the matching offset and copy the + * size from matched entry. + */ + for (j = 0; j <= i; j++) { + if (sc->toc[j].offset == sc->toc[i].offset && + !BLK_IS_NIL(sc, j)) { + break; + } + if (j != i) { + continue; + } + DPRINTF(GUZ_DBG_ERR, ("%s: cannot match " + "backref'ed offset at cluster #%u\n", + gp->name, i)); + return (-1); + } + sc->toc[i].blen = sc->toc[j].blen; + backref_to = j; + } else { + /* + * For the "normal blocks" seek forward until we hit + * block whose offset is larger than ours and assume + * it's going to be the next one. + */ + for (j = i + 1; j < sc->nblocks; j++) { + if (sc->toc[j].offset > max_offset) { + break; + } + } + sc->toc[i].blen = sc->toc[j].offset - + sc->toc[i].offset; + if (BLK_ENDS(sc, i) > pp->mediasize) { + DPRINTF(GUZ_DBG_ERR, ("%s: cluster #%u " + "extends past media boundary (%ju > %ju)\n", + gp->name, (u_int)i, + (uintmax_t)BLK_ENDS(sc, i), + (intmax_t)pp->mediasize)); + return (-1); + } + KASSERT(max_offset <= sc->toc[i].offset, ( + "%s: max_offset is incorrect: %ju", + gp->name, (uintmax_t)max_offset)); + max_offset = BLK_ENDS(sc, i) - 1; + } + DPRINTF_BLK(GUZ_DBG_TOC, i, ("%s: cluster #%u, original %u " + "bytes, in %u bytes", gp->name, i, sc->blksz, + sc->toc[i].blen)); + if (backref_to != BLEN_UNDEF) { + DPRINTF_BLK(GUZ_DBG_TOC, i, (" (->#%u)", + (u_int)backref_to)); + } + DPRINTF_BLK(GUZ_DBG_TOC, i, ("\n")); + } + return (0); +} + static struct g_geom * g_uzip_taste(struct g_class *mp, struct g_provider *pp, int flags) { @@ -403,6 +554,10 @@ struct g_geom *gp; struct g_provider *pp2; struct g_uzip_softc *sc; + enum { + GEOM_UZIP = 1, + GEOM_ULZMA + } type; g_trace(G_T_TOPOLOGY, "%s(%s,%s)", __func__, mp->name, pp->name); g_topology_assert(); @@ -422,10 +577,7 @@ if (error == 0) error = g_access(cp, 1, 0, 0); if (error) { - g_detach(cp); - g_destroy_consumer(cp); - g_destroy_geom(gp); - return (NULL); + goto e1; } g_topology_unlock(); @@ -433,22 +585,45 @@ * Read cloop header, look for CLOOP magic, perform * other validity checks. */ - DPRINTF(("%s: media sectorsize %u, mediasize %jd\n", + DPRINTF(GUZ_DBG_INFO, ("%s: media sectorsize %u, mediasize %jd\n", gp->name, pp->sectorsize, (intmax_t)pp->mediasize)); buf = g_read_data(cp, 0, pp->sectorsize, NULL); if (buf == NULL) - goto err; + goto e2; header = (struct cloop_header *) buf; if (strncmp(header->magic, CLOOP_MAGIC_START, sizeof(CLOOP_MAGIC_START) - 1) != 0) { - DPRINTF(("%s: no CLOOP magic\n", gp->name)); - goto err; - } - if (header->magic[0x0b] != 'V' || header->magic[0x0c] < '2') { - DPRINTF(("%s: image version too old\n", gp->name)); - goto err; + DPRINTF(GUZ_DBG_ERR, ("%s: no CLOOP magic\n", gp->name)); + goto e3; } + switch (header->magic[CLOOP_COMPR_OFF]) { + case 'L': + type = GEOM_ULZMA; + if (header->magic[CLOOP_VERSN_OFF] < CLOOP_MAJVER_LZMA) { + DPRINTF(GUZ_DBG_ERR, ("%s: image version too old\n", + gp->name)); + goto e3; + } + DPRINTF(GUZ_DBG_INFO, ("%s: GEOM_UZIP_LZMA image found\n", + gp->name)); + break; + case 'V': + type = GEOM_UZIP; + if (header->magic[CLOOP_VERSN_OFF] < CLOOP_MAJVER_ZLIB) { + DPRINTF(GUZ_DBG_ERR, ("%s: image version too old\n", + gp->name)); + goto e3; + } + DPRINTF(GUZ_DBG_INFO, ("%s: GEOM_UZIP_ZLIB image found\n", + gp->name)); + break; + default: + DPRINTF(GUZ_DBG_ERR, ("%s: unsupported image type\n", + gp->name)); + goto e3; + } + /* * Initialize softc and read offsets. */ @@ -459,7 +634,7 @@ if (sc->blksz % 512 != 0) { printf("%s: block size (%u) should be multiple of 512.\n", gp->name, sc->blksz); - goto err; + goto e4; } if (sc->blksz > MAX_BLKSZ) { printf("%s: block size (%u) should not be larger than %d.\n", @@ -470,15 +645,17 @@ total_offsets * sizeof(uint64_t) > pp->mediasize) { printf("%s: media too small for %u blocks\n", gp->name, sc->nblocks); - goto err; + goto e4; } - sc->offsets = malloc( - total_offsets * sizeof(uint64_t), M_GEOM_UZIP, M_WAITOK); + sc->toc = malloc(total_offsets * sizeof(struct g_uzip_blk), + M_GEOM_UZIP, M_WAITOK | M_ZERO); offsets_read = MIN(total_offsets, (pp->sectorsize - sizeof(*header)) / sizeof(uint64_t)); - for (i = 0; i < offsets_read; i++) - sc->offsets[i] = be64toh(((uint64_t *) (header + 1))[i]); - DPRINTF(("%s: %u offsets in the first sector\n", + for (i = 0; i < offsets_read; i++) { + sc->toc[i].offset = be64toh(((uint64_t *) (header + 1))[i]); + sc->toc[i].blen = BLEN_UNDEF; + } + DPRINTF(GUZ_DBG_INFO, ("%s: %u offsets in the first sector\n", gp->name, offsets_read)); for (blk = 1; offsets_read < total_offsets; blk++) { uint32_t nread; @@ -487,25 +664,49 @@ buf = g_read_data( cp, blk * pp->sectorsize, pp->sectorsize, NULL); if (buf == NULL) - goto err; + goto e5; nread = MIN(total_offsets - offsets_read, pp->sectorsize / sizeof(uint64_t)); - DPRINTF(("%s: %u offsets read from sector %d\n", + DPRINTF(GUZ_DBG_TOC, ("%s: %u offsets read from sector %d\n", gp->name, nread, blk)); for (i = 0; i < nread; i++) { - sc->offsets[offsets_read + i] = + sc->toc[offsets_read + i].offset = be64toh(((uint64_t *) buf)[i]); + sc->toc[offsets_read + i].blen = BLEN_UNDEF; } offsets_read += nread; } free(buf, M_GEOM); - DPRINTF(("%s: done reading offsets\n", gp->name)); + buf = NULL; + offsets_read -= 1; + DPRINTF(GUZ_DBG_INFO, ("%s: done reading %u block offsets from %u " + "sectors\n", gp->name, offsets_read, blk)); + if (sc->nblocks != offsets_read) { + DPRINTF(GUZ_DBG_ERR, ("%s: read %s offsets than expected " + "blocks\n", gp->name, + sc->nblocks < offsets_read ? "more" : "less")); + goto e5; + } + /* Massage TOC (table of contents), make sure it is sound */ + if (g_uzip_parse_toc(sc, pp, gp) != 0) { + DPRINTF(GUZ_DBG_ERR, ("%s: TOC error\n", gp->name)); + goto e5; + } mtx_init(&sc->last_mtx, "geom_uzip cache", NULL, MTX_DEF); sc->last_blk = -1; sc->last_buf = malloc(sc->blksz, M_GEOM_UZIP, M_WAITOK); sc->req_total = 0; sc->req_cached = 0; + if (type == GEOM_UZIP) { + sc->dcp = g_uzip_zlib_ctor(sc->blksz); + } else { + sc->dcp = g_uzip_lzma_ctor(sc->blksz); + } + if (sc->dcp == NULL) { + goto e6; + } + g_topology_lock(); pp2 = g_new_providerf(gp, "%s", gp->name); pp2->sectorsize = 512; @@ -515,22 +716,28 @@ g_error_provider(pp2, 0); g_access(cp, -1, 0, 0); - DPRINTF(("%s: taste ok (%d, %jd), (%d, %d), %x\n", - gp->name, - pp2->sectorsize, (intmax_t)pp2->mediasize, + DPRINTF(GUZ_DBG_INFO, ("%s: taste ok (%d, %jd), (%d, %d), %x\n", + gp->name, pp2->sectorsize, (intmax_t)pp2->mediasize, pp2->stripeoffset, pp2->stripesize, pp2->flags)); - DPRINTF(("%s: %u x %u blocks\n", gp->name, sc->nblocks, sc->blksz)); + DPRINTF(GUZ_DBG_INFO, ("%s: %u x %u blocks\n", gp->name, sc->nblocks, + sc->blksz)); return (gp); -err: - g_topology_lock(); - g_access(cp, -1, 0, 0); - if (buf != NULL) +e6: + free(sc->last_buf, M_GEOM); + mtx_destroy(&sc->last_mtx); +e5: + free(sc->toc, M_GEOM); +e4: + free(gp->softc, M_GEOM_UZIP); +e3: + if (buf != NULL) { free(buf, M_GEOM); - if (gp->softc != NULL) { - g_uzip_softc_free(gp->softc, NULL); - gp->softc = NULL; } +e2: + g_topology_lock(); + g_access(cp, -1, 0, 0); +e1: g_detach(cp); g_destroy_consumer(cp); g_destroy_geom(gp); @@ -547,7 +754,8 @@ g_topology_assert(); if (gp->softc == NULL) { - DPRINTF(("%s(%s): gp->softc == NULL\n", __func__, gp->name)); + DPRINTF(GUZ_DBG_ERR, ("%s(%s): gp->softc == NULL\n", __func__, + gp->name)); return (ENXIO); } Index: sys/geom/uzip/g_uzip_dapi.h =================================================================== --- /dev/null +++ sys/geom/uzip/g_uzip_dapi.h @@ -0,0 +1,15 @@ +#define DEFINE_RAW_METHOD(func, rval, args...) typedef rval (*func##_t)(args) + +struct g_uzip_dapi; + +DEFINE_RAW_METHOD(g_uzip_dapi_decompress, int, struct g_uzip_dapi *, + const char *, void *, size_t, void *); +DEFINE_RAW_METHOD(g_uzip_dapi_free, void, struct g_uzip_dapi *); +DEFINE_RAW_METHOD(g_uzip_dapi_rewind, int, struct g_uzip_dapi *, const char *); + +struct g_uzip_dapi { + g_uzip_dapi_decompress_t decompress; + g_uzip_dapi_free_t free; + g_uzip_dapi_rewind_t rewind; + void *pvt; +}; Index: sys/geom/uzip/g_uzip_lzma.h =================================================================== --- /dev/null +++ sys/geom/uzip/g_uzip_lzma.h @@ -0,0 +1,32 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * Copyright (c) 2010-2012 Aleksandr Rybalko + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#define CLOOP_MAJVER_LZMA '3' + +struct g_uzip_dapi *g_uzip_lzma_ctor(uint32_t); Index: sys/geom/uzip/g_uzip_lzma.c =================================================================== --- /dev/null +++ sys/geom/uzip/g_uzip_lzma.c @@ -0,0 +1,109 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * Copyright (c) 2010-2012 Aleksandr Rybalko + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + +#include + +#include +#include +#include + +struct g_uzip_lzma { + struct g_uzip_dapi pub; + /* XZ decoder structs */ + struct xz_buf b; + struct xz_dec *s; +}; + +static int g_uzip_lzma_nop(struct g_uzip_dapi *, const char *); + +static void +g_uzip_lzma_free(struct g_uzip_dapi *lzpp) +{ + struct g_uzip_lzma *lzp; + + lzp = (struct g_uzip_lzma *)lzpp->pvt; + if (lzp->s != NULL) { + xz_dec_end(lzp->s); + lzp->s = NULL; + } + + free(lzp, M_GEOM_UZIP); +} + +static int +g_uzip_lzma_decompress(struct g_uzip_dapi *lzpp, const char *gp_name, + void *ibp, size_t ilen, void *obp) +{ + struct g_uzip_lzma *lzp; + int err; + + lzp = (struct g_uzip_lzma *)lzpp->pvt; + + lzp->b.in = ibp; + lzp->b.out = obp; + lzp->b.in_pos = lzp->b.out_pos = 0; + lzp->b.in_size = ilen; + lzp->b.out_size = (size_t)-1; + err = (xz_dec_run(lzp->s, &lzp->b) != XZ_STREAM_END) ? 1 : 0; + /* TODO decoder recovery, if needed */ + + return (err); +} + +struct g_uzip_dapi * +g_uzip_lzma_ctor(uint32_t blksz) +{ + struct g_uzip_lzma *lzp; + + lzp = (struct g_uzip_lzma *)malloc(sizeof(struct g_uzip_lzma), + M_GEOM_UZIP, M_WAITOK); + xz_crc32_init(); + lzp->s = xz_dec_init(XZ_SINGLE, 0); + if (lzp->s == NULL) { + goto e1; + } + lzp->pub.decompress = &g_uzip_lzma_decompress; + lzp->pub.free = &g_uzip_lzma_free; + lzp->pub.rewind = &g_uzip_lzma_nop; + lzp->pub.pvt = lzp; + return (&lzp->pub); +e1: + free(lzp, M_GEOM_UZIP); + return (NULL); +} + +static int +g_uzip_lzma_nop(struct g_uzip_dapi *zpp, const char *gp_name) +{ + + return (0); +} Index: sys/geom/uzip/g_uzip_zlib.h =================================================================== --- /dev/null +++ sys/geom/uzip/g_uzip_zlib.h @@ -0,0 +1,33 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +struct g_geom; + +#define CLOOP_MAJVER_ZLIB '2' + +struct g_uzip_dapi *g_uzip_zlib_ctor(uint32_t); Index: sys/geom/uzip/g_uzip_zlib.c =================================================================== --- /dev/null +++ sys/geom/uzip/g_uzip_zlib.c @@ -0,0 +1,135 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + +#include + +#include +#include +#include + +struct g_uzip_zlib { + uint32_t blksz; + struct g_uzip_dapi pub; + /* Zlib decoder structs */ + z_stream zs; +}; + +static void *z_alloc(void *, u_int, u_int); +static void z_free(void *, void *); +static int g_uzip_zlib_rewind(struct g_uzip_dapi *, const char *); + +static void +g_uzip_zlib_free(struct g_uzip_dapi *zpp) +{ + struct g_uzip_zlib *zp; + + zp = (struct g_uzip_zlib *)zpp->pvt; + inflateEnd(&zp->zs); + free(zp, M_GEOM_UZIP); +} + +static int +g_uzip_zlib_decompress(struct g_uzip_dapi *zpp, const char *gp_name, void *ibp, + size_t ilen, void *obp) +{ + int err; + struct g_uzip_zlib *zp; + + zp = (struct g_uzip_zlib *)zpp->pvt; + + zp->zs.next_in = ibp; + zp->zs.avail_in = ilen; + zp->zs.next_out = obp; + zp->zs.avail_out = zp->blksz; + + err = (inflate(&zp->zs, Z_FINISH) != Z_STREAM_END) ? 1 : 0; + if (err != 0) { + printf("%s: UZIP(zlib) inflate() failed\n", gp_name); + } + return (err); +} + +static int +g_uzip_zlib_rewind(struct g_uzip_dapi *zpp, const char *gp_name) +{ + int err; + struct g_uzip_zlib *zp; + + zp = (struct g_uzip_zlib *)zpp->pvt; + + err = 0; + if (inflateReset(&zp->zs) != Z_OK) { + printf("%s: UZIP(zlib) decoder reset failed\n", gp_name); + err = 1; + } + return (err); +} + +struct g_uzip_dapi * +g_uzip_zlib_ctor(uint32_t blksz) +{ + struct g_uzip_zlib *zp; + + zp = (struct g_uzip_zlib *)malloc(sizeof(struct g_uzip_zlib), + M_GEOM_UZIP, M_WAITOK); + zp->zs.zalloc = z_alloc; + zp->zs.zfree = z_free; + if (inflateInit(&zp->zs) != Z_OK) { + goto e1; + } + zp->blksz = blksz; + zp->pub.decompress = &g_uzip_zlib_decompress; + zp->pub.free = &g_uzip_zlib_free; + zp->pub.rewind = &g_uzip_zlib_rewind; + zp->pub.pvt = (void *)zp; + return (&zp->pub); +e1: + free(zp, M_GEOM_UZIP); + return (NULL); +} + +static void * +z_alloc(void *nil, u_int type, u_int size) +{ + void *ptr; + + ptr = malloc(type * size, M_GEOM_UZIP, M_NOWAIT); + + return (ptr); +} + +static void +z_free(void *nil, void *ptr) +{ + + free(ptr, M_GEOM_UZIP); +} Index: sys/modules/geom/geom_uzip/Makefile =================================================================== --- sys/modules/geom/geom_uzip/Makefile +++ sys/modules/geom/geom_uzip/Makefile @@ -3,7 +3,18 @@ .PATH: ${.CURDIR}/../../../geom/uzip ${.CURDIR}/../../../net KMOD= geom_uzip -SRCS= g_uzip.c +SRCS= g_uzip.c g_uzip_zlib.c g_uzip_lzma.c #CFLAGS= -g -DINVARIANT_SUPPORT -DINVARIANTS +.PATH: ${.CURDIR}/../../../contrib/xz-embedded/freebsd/ \ + ${.CURDIR}/../../../contrib/xz-embedded/linux/lib/xz/ \ + ${.CURDIR}/../../../contrib/xz-embedded/linux/include/linux/ \ + ${.CURDIR}/../../../net + +CFLAGS+= -I${.CURDIR}/../../../contrib/xz-embedded/freebsd \ + -I${.CURDIR}/../../../contrib/xz-embedded/linux/lib/xz/ +SRCS+= xz_crc32.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c \ + xz_malloc.c +SRCS+= xz.h xz_config.h xz_lzma2.h xz_malloc.h xz_private.h xz_stream.h + .include Index: usr.bin/mkuzip/Makefile =================================================================== --- usr.bin/mkuzip/Makefile +++ usr.bin/mkuzip/Makefile @@ -2,7 +2,8 @@ PROG= mkuzip MAN= mkuzip.8 +SRCS= mkuzip.c mkuz_blockcache.c mkuz_lzma.c mkuz_zlib.c -LIBADD= z +LIBADD= z md lzma .include Index: usr.bin/mkuzip/mkuz_blockcache.h =================================================================== --- /dev/null +++ usr.bin/mkuzip/mkuz_blockcache.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +struct mkuz_blkcache_hit { + uint64_t offset; + ssize_t len; + uint32_t blkno; +}; + +struct mkuz_blkcache_hit *mkuz_blkcache_regblock(int, uint32_t, off_t, ssize_t, + void *); Index: usr.bin/mkuzip/mkuz_blockcache.c =================================================================== --- /dev/null +++ usr.bin/mkuzip/mkuz_blockcache.c @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +#include +#include +#include +#include + +#if defined(MKUZ_DEBUG) +# include +#endif + +#include "mkuz_blockcache.h" + +struct mkuz_blkcache { + struct mkuz_blkcache_hit hit; + off_t data_offset; + unsigned char digest[16]; + struct mkuz_blkcache *next; +}; + +static struct mkuz_blkcache blkcache; + +struct mkuz_blkcache_hit * +mkuz_blkcache_regblock(int fd, uint32_t blkno, off_t offset, ssize_t len, + void *data) +{ + struct mkuz_blkcache *bcep; + MD5_CTX mcontext; + off_t data_offset; + unsigned char mdigest[16]; + + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset < 0) { + return (NULL); + } + MD5Init(&mcontext); + MD5Update(&mcontext, data, len); + MD5Final(mdigest, &mcontext); + if (blkcache.hit.len == 0) { + bcep = &blkcache; + } else { + for (bcep = &blkcache; bcep != NULL; bcep = bcep->next) { + if (bcep->hit.len != len) + continue; + if (memcmp(mdigest, bcep->digest, sizeof(mdigest)) == 0) { + break; + } + } + if (bcep != NULL) { +#if defined(MKUZ_DEBUG) + printf("cache hit %d, %d, %d\n", (int)bcep->hit.offset, (int)data_offset, (int)len); +#endif + return (&bcep->hit); + } + bcep = malloc(sizeof(struct mkuz_blkcache)); + if (bcep == NULL) + return (NULL); + memset(bcep, '\0', sizeof(struct mkuz_blkcache)); + bcep->next = blkcache.next; + blkcache.next = bcep; + } + memcpy(bcep->digest, mdigest, sizeof(mdigest)); + bcep->data_offset = data_offset; + bcep->hit.offset = offset; + bcep->hit.len = len; + bcep->hit.blkno = blkno; + return (NULL); +} Index: usr.bin/mkuzip/mkuz_lzma.h =================================================================== --- /dev/null +++ usr.bin/mkuzip/mkuz_lzma.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * Copyright (c) 2011 Aleksandr Rybalko + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +/* Format L3.0, since we move to XZ API */ +#define CLOOP_MAGIC_LZMA \ + "#!/bin/sh\n" \ + "#L3.0\n" \ + "n=uncompress\n" \ + "m=geom_$n\n" \ + "(kldstat -m $m 2>&-||kldload $m)>&-&&" \ + "mount_cd9660 /dev/`mdconfig -af $0`.$n $1\n" \ + "exit $?\n" +#define DEFAULT_SUFX_LZMA ".ulzma" + +void *mkuz_lzma_init(uint32_t); +void mkuz_lzma_compress(const char *, uint32_t *); Index: usr.bin/mkuzip/mkuz_lzma.c =================================================================== --- /dev/null +++ usr.bin/mkuzip/mkuz_lzma.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * Copyright (c) 2011 Aleksandr Rybalko + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include + +#include "mkuzip.h" +#include "mkuz_lzma.h" + +#define USED_BLOCKSIZE DEV_BSIZE + +struct mkuz_lzma { + lzma_filter filters[2]; + lzma_options_lzma opt_lzma; + lzma_stream strm; + char *obuf; + uint32_t blksz; +}; + +static struct mkuz_lzma ulzma = {.strm = LZMA_STREAM_INIT}; + +void * +mkuz_lzma_init(uint32_t blksz) +{ + if (blksz % USED_BLOCKSIZE != 0) { + errx(1, "cluster size should be multiple of %d", + USED_BLOCKSIZE); + /* Not reached */ + } + if (blksz > MAXPHYS) { + errx(1, "cluster size is too large"); + /* Not reached */ + } + ulzma.obuf = mkuz_safe_malloc(blksz * 2); + + /* Init lzma encoder */ + if (lzma_lzma_preset(&ulzma.opt_lzma, LZMA_PRESET_DEFAULT)) + errx(1, "Error loading LZMA preset"); + + ulzma.filters[0].id = LZMA_FILTER_LZMA2; + ulzma.filters[0].options = &ulzma.opt_lzma; + ulzma.filters[1].id = LZMA_VLI_UNKNOWN; + + ulzma.blksz = blksz; + + return (ulzma.obuf); +} + +void +mkuz_lzma_compress(const char *ibuf, uint32_t *destlen) +{ + lzma_ret ret; + + ret = lzma_stream_encoder(&ulzma.strm, ulzma.filters, LZMA_CHECK_CRC32); + if (ret != LZMA_OK) { + if (ret == LZMA_MEMLIMIT_ERROR) + errx(1, "can't compress data: LZMA_MEMLIMIT_ERROR"); + + errx(1, "can't compress data: LZMA compressor ERROR"); + } + + ulzma.strm.next_in = ibuf; + ulzma.strm.avail_in = ulzma.blksz; + ulzma.strm.next_out = ulzma.obuf; + ulzma.strm.avail_out = ulzma.blksz * 2; + + ret = lzma_code(&ulzma.strm, LZMA_FINISH); + + if (ret != LZMA_STREAM_END) { + /* Error */ + errx(1, "lzma_code FINISH failed, code=%d, pos(in=%zd, " + "out=%zd)", ret, (ulzma.blksz - ulzma.strm.avail_in), + (ulzma.blksz * 2 - ulzma.strm.avail_out)); + } + + lzma_end(&ulzma.strm); + + *destlen = (ulzma.blksz * 2) - ulzma.strm.avail_out; +} Index: usr.bin/mkuzip/mkuz_zlib.h =================================================================== --- /dev/null +++ usr.bin/mkuzip/mkuz_zlib.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#define DEFAULT_SUFX_ZLIB ".uzip" + +#define CLOOP_MAGIC_ZLIB "#!/bin/sh\n#V2.0 Format\n" \ + "(kldstat -qm g_uzip||kldload geom_uzip)>&-&&" \ + "mount_cd9660 /dev/`mdconfig -af $0`.uzip $1\nexit $?\n" + +void *mkuz_zlib_init(uint32_t); +void mkuz_zlib_compress(const char *, uint32_t *); Index: usr.bin/mkuzip/mkuz_zlib.c =================================================================== --- /dev/null +++ usr.bin/mkuzip/mkuz_zlib.c @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include + +#include "mkuzip.h" +#include "mkuz_zlib.h" + +struct mkuz_zlib { + char *obuf; + uLongf oblen; + uint32_t blksz; +}; + +static struct mkuz_zlib uzip; + +void * +mkuz_zlib_init(uint32_t blksz) +{ + if (blksz % DEV_BSIZE != 0) { + errx(1, "cluster size should be multiple of %d", + DEV_BSIZE); + /* Not reached */ + } + if (compressBound(blksz) > MAXPHYS) { + errx(1, "cluster size is too large"); + /* Not reached */ + } + uzip.oblen = compressBound(blksz); + uzip.obuf = mkuz_safe_malloc(uzip.oblen); + uzip.blksz = blksz; + + return (uzip.obuf); +} + +void +mkuz_zlib_compress(const char *ibuf, uint32_t *destlen) +{ + uLongf destlen_z; + + destlen_z = uzip.oblen; + if (compress2(uzip.obuf, &destlen_z, ibuf, uzip.blksz, + Z_BEST_COMPRESSION) != Z_OK) { + errx(1, "can't compress data: compress2() " + "failed"); + /* Not reached */ + } + + *destlen = (uint32_t)destlen_z; +} Index: usr.bin/mkuzip/mkuzip.h =================================================================== --- /dev/null +++ usr.bin/mkuzip/mkuzip.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +void *mkuz_safe_malloc(size_t); Index: usr.bin/mkuzip/mkuzip.8 =================================================================== --- usr.bin/mkuzip/mkuzip.8 +++ usr.bin/mkuzip/mkuzip.8 @@ -1,9 +1,27 @@ -.\" ---------------------------------------------------------------------------- -.\" "THE BEER-WARE LICENSE" (Revision 42): -.\" wrote this file. As long as you retain this notice you -.\" can do whatever you want with this stuff. If we meet some day, and you think -.\" this stuff is worth it, you can buy me a beer in return. Maxim Sobolev -.\" ---------------------------------------------------------------------------- +.\"- +.\" Copyright (c) 2004-2016 Maxim Sobolev +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. .\" .\" $FreeBSD: head/usr.bin/mkuzip/mkuzip.8 267773 2014-06-23 08:23:05Z bapt $ .\" @@ -39,7 +57,9 @@ An .Ar infile image is split into clusters; each cluster is compressed using -.Xr zlib 3 . +.Xr zlib 3 +or +.Xr lzma 3 . .It The resulting set of compressed clusters along with headers that allow locating each individual cluster is written to the output file. @@ -51,7 +71,23 @@ Name of the output file .Ar outfile . The default is to use the input name with the suffix -.Pa .uzip . +.Pa .uzip +for the +.Xr zlib 3 +compression or +.Pa .ulzma +for the +.Xr lzma 3 . +.It Fl L +Use +.Xr lzma 3 +compression algorithm instead of the default +.Xr zlib 3 . +The +.Xr lzma 3 +provides noticeable better compression levels on the same data set +at the expense of much slower compression speed (10-20x) and somewhat slower +decompression (2-3x). .It Fl s Ar cluster_size Split the image into clusters of .Ar cluster_size @@ -61,6 +97,27 @@ should be a multiple of 512 bytes. .It Fl v Display verbose messages. +.It Fl Z +Disable zero-blocks detection and elimination. +When this option is set, the +.Nm +would compress empty blocks (i.e. clusters that consist of only zero bytes) +just as it would any other block. +When the option is not set, the +.Nm +detects such blocks and skips them from the output. +Setting +.Fl Z +results is slight increase of compressed image size, typically less than 0.1% +of a final size of the compressed image. +.It Fl d +Enable de-duplication. +When the option is enabled the +.Nm +detects identical blocks in the input and replaces each subsequent occurence +of such block with pointer to the very first one in the output. +Setting this option results is moderate decrease of compressed image size, +typically around 3-5% of a final size of the compressed image. .El .Sh NOTES The compression ratio largely depends on the cluster size used. @@ -92,11 +149,27 @@ and automatically mount it using .Xr mount_cd9660 8 on the mount point provided as the first argument to the script. +.Pp +The de-duplication is a +.Fx +specific feature and while it does not require any changes to on-disk +compressed image format, however it did require some matching changes to the +.Xr geom_uzip 4 +to handle resulting images correctly. +It is possible that other implementations may not have the logic necessary +for proper handling of such images and an attempt to attach de-duplicated +images on such systems could result in undefined behaviour including system +crash or data loss. +The same applies to the older versions of the +.Fx +kernel as well. .Sh EXIT STATUS .Ex -std .Sh SEE ALSO .Xr gzip 1 , +.Xr xz 1 , .Xr zlib 3 , +.Xr lzma 3 , .Xr geom 4 , .Xr geom_uzip 4 , .Xr md 4 , Index: usr.bin/mkuzip/mkuzip.c =================================================================== --- usr.bin/mkuzip/mkuzip.c +++ usr.bin/mkuzip/mkuzip.c @@ -1,12 +1,27 @@ /* - * ---------------------------------------------------------------------------- - * "THE BEER-WARE LICENSE" (Revision 42): - * wrote this file. As long as you retain this notice you - * can do whatever you want with this stuff. If we meet some day, and you think - * this stuff is worth it, you can buy me a beer in return. Maxim Sobolev - * ---------------------------------------------------------------------------- + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. * - * $FreeBSD: head/usr.bin/mkuzip/mkuzip.c 221832 2011-05-13 09:55:48Z ru $ + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. * */ @@ -17,7 +32,6 @@ #include #include #include -#include #include #include #include @@ -26,18 +40,44 @@ #include #include -#define CLSTSIZE 16384 -#define DEFAULT_SUFX ".uzip" +#include "mkuzip.h" +#include "mkuz_blockcache.h" +#include "mkuz_zlib.h" +#include "mkuz_lzma.h" + +#define DEFINE_RAW_METHOD(func, rval, args...) typedef rval (*func##_t)(args) + +#define CLOOP_MAGIC_LEN 128 +#define DEFAULT_CLSTSIZE 16384 + +DEFINE_RAW_METHOD(f_init, void *, uint32_t); +DEFINE_RAW_METHOD(f_compress, void, const char *, uint32_t *); + +struct mkuz_format { + const char *magic; + const char *default_sufx; + f_init_t f_init; + f_compress_t f_compress; +}; -#define CLOOP_MAGIC_LEN 128 -static char CLOOP_MAGIC_START[] = "#!/bin/sh\n#V2.0 Format\n" - "(kldstat -qm g_uzip||kldload geom_uzip)>&-&&" - "mount_cd9660 /dev/`mdconfig -af $0`.uzip $1\nexit $?\n"; +static struct mkuz_format uzip_fmt = { + .magic = CLOOP_MAGIC_ZLIB, + .default_sufx = DEFAULT_SUFX_ZLIB, + .f_init = &mkuz_zlib_init, + .f_compress = &mkuz_zlib_compress +}; + +static struct mkuz_format ulzma_fmt = { + .magic = CLOOP_MAGIC_LZMA, + .default_sufx = DEFAULT_SUFX_LZMA, + .f_init = &mkuz_lzma_init, + .f_compress = &mkuz_lzma_compress +}; static char *readblock(int, char *, u_int32_t); static void usage(void); -static void *safe_malloc(size_t); static void cleanup(void); +static int memvcmp(const void *, unsigned char, size_t); static char *cleanfile = NULL; @@ -45,24 +85,28 @@ { char *iname, *oname, *obuf, *ibuf; uint64_t *toc; - int fdr, fdw, i, opt, verbose, tmp; + int fdr, fdw, i, opt, verbose, no_zcomp, tmp, en_dcomp; struct iovec iov[2]; struct stat sb; - uLongf destlen; - uint64_t offset; + uint32_t destlen; + uint64_t offset, last_offset; struct cloop_header { char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ uint32_t blksz; /* block size */ uint32_t nblocks; /* number of blocks */ } hdr; + struct mkuz_blkcache_hit *chit; + const struct mkuz_format *handler; memset(&hdr, 0, sizeof(hdr)); - hdr.blksz = CLSTSIZE; - strcpy(hdr.magic, CLOOP_MAGIC_START); + hdr.blksz = DEFAULT_CLSTSIZE; oname = NULL; verbose = 0; + no_zcomp = 0; + en_dcomp = 0; + handler = &uzip_fmt; - while((opt = getopt(argc, argv, "o:s:v")) != -1) { + while((opt = getopt(argc, argv, "o:s:vZdL")) != -1) { switch(opt) { case 'o': oname = optarg; @@ -75,15 +119,6 @@ optarg); /* Not reached */ } - if (tmp % DEV_BSIZE != 0) { - errx(1, "cluster size should be multiple of %d", - DEV_BSIZE); - /* Not reached */ - } - if (compressBound(tmp) > MAXPHYS) { - errx(1, "cluster size is too large"); - /* Not reached */ - } hdr.blksz = tmp; break; @@ -91,6 +126,18 @@ verbose = 1; break; + case 'Z': + no_zcomp = 1; + break; + + case 'd': + en_dcomp = 1; + break; + + case 'L': + handler = &ulzma_fmt; + break; + default: usage(); /* Not reached */ @@ -104,17 +151,20 @@ /* Not reached */ } + strcpy(hdr.magic, handler->magic); + + obuf = handler->f_init(hdr.blksz); + iname = argv[0]; if (oname == NULL) { - asprintf(&oname, "%s%s", iname, DEFAULT_SUFX); + asprintf(&oname, "%s%s", iname, handler->default_sufx); if (oname == NULL) { err(1, "can't allocate memory"); /* Not reached */ } } - obuf = safe_malloc(compressBound(hdr.blksz)); - ibuf = safe_malloc(hdr.blksz); + ibuf = mkuz_safe_malloc(hdr.blksz); signal(SIGHUP, exit); signal(SIGINT, exit); @@ -152,7 +202,7 @@ "of %d, padding data\n", hdr.blksz); hdr.nblocks++; } - toc = safe_malloc((hdr.nblocks + 1) * sizeof(*toc)); + toc = mkuz_safe_malloc((hdr.nblocks + 1) * sizeof(*toc)); fdw = open(oname, O_WRONLY | O_TRUNC | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); @@ -180,37 +230,69 @@ for(i = 0; i == 0 || ibuf != NULL; i++) { ibuf = readblock(fdr, ibuf, hdr.blksz); if (ibuf != NULL) { - destlen = compressBound(hdr.blksz); - if (compress2(obuf, &destlen, ibuf, hdr.blksz, - Z_BEST_COMPRESSION) != Z_OK) { - errx(1, "can't compress data: compress2() " - "failed"); - /* Not reached */ + if (no_zcomp == 0 && \ + memvcmp(ibuf, '\0', hdr.blksz) != 0) { + /* All zeroes block */ + destlen = 0; + } else { + handler->f_compress(ibuf, &destlen); } - if (verbose != 0) - fprintf(stderr, "cluster #%d, in %u bytes, " - "out %lu bytes\n", i, hdr.blksz, destlen); } else { destlen = DEV_BSIZE - (offset % DEV_BSIZE); memset(obuf, 0, destlen); if (verbose != 0) - fprintf(stderr, "padding data with %lu bytes so " - "that file size is multiple of %d\n", destlen, - DEV_BSIZE); + fprintf(stderr, "padding data with %lu bytes " + "so that file size is multiple of %d\n", + (u_long)destlen, DEV_BSIZE); } - if (write(fdw, obuf, destlen) < 0) { - err(1, "write(%s)", oname); - /* Not reached */ + if (destlen > 0 && en_dcomp != 0) { + chit = mkuz_blkcache_regblock(fdw, i, offset, destlen, + obuf); + /* + * There should be at least one non-empty block + * between us and the backref'ed offset, otherwise + * we won't be able to parse that sequence correctly + * as it would be indistinguishible from another + * empty block. + */ + if (chit != NULL && chit->offset == last_offset) { + chit = NULL; + } + } else { + chit = NULL; + } + if (chit != NULL) { + toc[i] = htobe64(chit->offset); + } else { + if (destlen > 0 && write(fdw, obuf, destlen) < 0) { + err(1, "write(%s)", oname); + /* Not reached */ + } + toc[i] = htobe64(offset); + last_offset = offset; + offset += destlen; + } + if (ibuf != NULL && verbose != 0) { + fprintf(stderr, "cluster #%d, in %u bytes, " + "out len=%lu offset=%lu", i, hdr.blksz, + chit == NULL ? (u_long)destlen : 0, + (u_long)be64toh(toc[i])); + if (chit != NULL) { + fprintf(stderr, " (backref'ed to #%d)", + chit->blkno); + } + fprintf(stderr, "\n"); + } - toc[i] = htobe64(offset); - offset += destlen; } close(fdr); if (verbose != 0) fprintf(stderr, "compressed data to %ju bytes, saved %lld " - "bytes, %.2f%% decrease.\n", offset, (long long)(sb.st_size - offset), - 100.0 * (long long)(sb.st_size - offset) / (float)sb.st_size); + "bytes, %.2f%% decrease.\n", offset, + (long long)(sb.st_size - offset), + 100.0 * (long long)(sb.st_size - offset) / + (float)sb.st_size); /* Convert to big endian */ hdr.blksz = htonl(hdr.blksz); @@ -248,12 +330,13 @@ usage(void) { - fprintf(stderr, "usage: mkuzip [-v] [-o outfile] [-s cluster_size] infile\n"); + fprintf(stderr, "usage: mkuzip [-vZdL] [-o outfile] [-s cluster_size] " + "infile\n"); exit(1); } -static void * -safe_malloc(size_t size) +void * +mkuz_safe_malloc(size_t size) { void *retval; @@ -272,3 +355,12 @@ if (cleanfile != NULL) unlink(cleanfile); } + +static int +memvcmp(const void *memory, unsigned char val, size_t size) +{ + const u_char *mm; + + mm = (const u_char *)memory; + return (*mm == val) && memcmp(mm, mm + 1, size - 1) == 0; +}