diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c index 24b61d9f6d8e..773b9c829acc 100644 --- a/sys/geom/eli/g_eli.c +++ b/sys/geom/eli/g_eli.c @@ -1,1465 +1,1636 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2005-2019 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include +#include #include #include #include #include #include FEATURE(geom_eli, "GEOM crypto module"); MALLOC_DEFINE(M_ELI, "eli data", "GEOM_ELI Data"); SYSCTL_DECL(_kern_geom); SYSCTL_NODE(_kern_geom, OID_AUTO, eli, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "GEOM_ELI stuff"); static int g_eli_version = G_ELI_VERSION; SYSCTL_INT(_kern_geom_eli, OID_AUTO, version, CTLFLAG_RD, &g_eli_version, 0, "GELI version"); int g_eli_debug = 0; SYSCTL_INT(_kern_geom_eli, OID_AUTO, debug, CTLFLAG_RWTUN, &g_eli_debug, 0, "Debug level"); static u_int g_eli_tries = 3; SYSCTL_UINT(_kern_geom_eli, OID_AUTO, tries, CTLFLAG_RWTUN, &g_eli_tries, 0, "Number of tries for entering the passphrase"); static u_int g_eli_visible_passphrase = GETS_NOECHO; SYSCTL_UINT(_kern_geom_eli, OID_AUTO, visible_passphrase, CTLFLAG_RWTUN, &g_eli_visible_passphrase, 0, "Visibility of passphrase prompt (0 = invisible, 1 = visible, 2 = asterisk)"); u_int g_eli_overwrites = G_ELI_OVERWRITES; SYSCTL_UINT(_kern_geom_eli, OID_AUTO, overwrites, CTLFLAG_RWTUN, &g_eli_overwrites, 0, "Number of times on-disk keys should be overwritten when destroying them"); static u_int g_eli_threads = 0; SYSCTL_UINT(_kern_geom_eli, OID_AUTO, threads, CTLFLAG_RWTUN, &g_eli_threads, 0, "Number of threads doing crypto work"); u_int g_eli_batch = 0; SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RWTUN, &g_eli_batch, 0, "Use crypto operations batching"); +static u_int g_eli_minbufs = 16; +static int sysctl_g_eli_minbufs(SYSCTL_HANDLER_ARGS); +SYSCTL_PROC(_kern_geom_eli, OID_AUTO, minbufs, CTLTYPE_UINT | CTLFLAG_RW | + CTLFLAG_MPSAFE, NULL, 0, sysctl_g_eli_minbufs, "IU", + "Number of GELI bufs reserved for swap transactions"); +static struct sx g_eli_umalock; /* Controls changes to UMA zone. */ +SX_SYSINIT(g_eli_umalock, &g_eli_umalock, "GELI UMA"); +static uma_zone_t g_eli_uma = NULL; +static int g_eli_alloc_sz; +static volatile int g_eli_umaoutstanding; +static volatile int g_eli_devs; +static bool g_eli_blocking_malloc = false; +SYSCTL_BOOL(_kern_geom_eli, OID_AUTO, blocking_malloc, CTLFLAG_RWTUN, + &g_eli_blocking_malloc, 0, "Use blocking malloc calls for GELI buffers"); + +/* + * Control the number of reserved entries in the GELI zone. + * If the GELI zone has already been allocated, update the zone. Otherwise, + * simply update the variable for use the next time the zone is created. + */ +static int +sysctl_g_eli_minbufs(SYSCTL_HANDLER_ARGS) +{ + int error; + u_int new; + + new = g_eli_minbufs; + error = sysctl_handle_int(oidp, &new, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + sx_xlock(&g_eli_umalock); + if (g_eli_uma != NULL) { + if (new != g_eli_minbufs) + uma_zone_reserve(g_eli_uma, new); + if (new > g_eli_minbufs) + uma_prealloc(g_eli_uma, new - g_eli_minbufs); + } + if (new != g_eli_minbufs) + g_eli_minbufs = new; + sx_xunlock(&g_eli_umalock); + return (0); +} /* * Passphrase cached during boot, in order to be more user-friendly if * there are multiple providers using the same passphrase. */ static char cached_passphrase[256]; static u_int g_eli_boot_passcache = 1; TUNABLE_INT("kern.geom.eli.boot_passcache", &g_eli_boot_passcache); SYSCTL_UINT(_kern_geom_eli, OID_AUTO, boot_passcache, CTLFLAG_RD, &g_eli_boot_passcache, 0, "Passphrases are cached during boot process for possible reuse"); static void fetch_loader_passphrase(void * dummy) { char * env_passphrase; KASSERT(dynamic_kenv, ("need dynamic kenv")); if ((env_passphrase = kern_getenv("kern.geom.eli.passphrase")) != NULL) { /* Extract passphrase from the environment. */ strlcpy(cached_passphrase, env_passphrase, sizeof(cached_passphrase)); freeenv(env_passphrase); /* Wipe the passphrase from the environment. */ kern_unsetenv("kern.geom.eli.passphrase"); } } SYSINIT(geli_fetch_loader_passphrase, SI_SUB_KMEM + 1, SI_ORDER_ANY, fetch_loader_passphrase, NULL); static void zero_boot_passcache(void) { explicit_bzero(cached_passphrase, sizeof(cached_passphrase)); } static void zero_geli_intake_keys(void) { struct keybuf *keybuf; int i; if ((keybuf = get_keybuf()) != NULL) { /* Scan the key buffer, clear all GELI keys. */ for (i = 0; i < keybuf->kb_nents; i++) { if (keybuf->kb_ents[i].ke_type == KEYBUF_TYPE_GELI) { explicit_bzero(keybuf->kb_ents[i].ke_data, sizeof(keybuf->kb_ents[i].ke_data)); keybuf->kb_ents[i].ke_type = KEYBUF_TYPE_NONE; } } } } static void zero_intake_passcache(void *dummy) { zero_boot_passcache(); zero_geli_intake_keys(); } EVENTHANDLER_DEFINE(mountroot, zero_intake_passcache, NULL, 0); static eventhandler_tag g_eli_pre_sync = NULL; static int g_eli_read_metadata_offset(struct g_class *mp, struct g_provider *pp, off_t offset, struct g_eli_metadata *md); static int g_eli_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp); static void g_eli_init(struct g_class *mp); static void g_eli_fini(struct g_class *mp); static g_taste_t g_eli_taste; static g_dumpconf_t g_eli_dumpconf; struct g_class g_eli_class = { .name = G_ELI_CLASS_NAME, .version = G_VERSION, .ctlreq = g_eli_config, .taste = g_eli_taste, .destroy_geom = g_eli_destroy_geom, .init = g_eli_init, .fini = g_eli_fini }; /* * Code paths: * BIO_READ: * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver * BIO_WRITE: * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver */ /* * EAGAIN from crypto(9) means, that we were probably balanced to another crypto * accelerator or something like this. * The function updates the SID and rerun the operation. */ int g_eli_crypto_rerun(struct cryptop *crp) { struct g_eli_softc *sc; struct g_eli_worker *wr; struct bio *bp; int error; bp = (struct bio *)crp->crp_opaque; sc = bp->bio_to->geom->softc; LIST_FOREACH(wr, &sc->sc_workers, w_next) { - if (wr->w_number == bp->bio_pflags) + if (wr->w_number == G_ELI_WORKER(bp->bio_pflags)) break; } - KASSERT(wr != NULL, ("Invalid worker (%u).", bp->bio_pflags)); + KASSERT(wr != NULL, ("Invalid worker (%u).", + G_ELI_WORKER(bp->bio_pflags))); G_ELI_DEBUG(1, "Rerunning crypto %s request (sid: %p -> %p).", bp->bio_cmd == BIO_READ ? "READ" : "WRITE", wr->w_sid, crp->crp_session); wr->w_sid = crp->crp_session; crp->crp_etype = 0; error = crypto_dispatch(crp); if (error == 0) return (0); G_ELI_DEBUG(1, "%s: crypto_dispatch() returned %d.", __func__, error); crp->crp_etype = error; return (error); } static void g_eli_getattr_done(struct bio *bp) { if (bp->bio_error == 0 && !strcmp(bp->bio_attribute, "GEOM::physpath")) { strlcat(bp->bio_data, "/eli", bp->bio_length); } g_std_done(bp); } /* * The function is called afer reading encrypted data from the provider. * * g_eli_start -> g_eli_crypto_read -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver */ void g_eli_read_done(struct bio *bp) { struct g_eli_softc *sc; struct bio *pbp; G_ELI_LOGREQ(2, bp, "Request done."); pbp = bp->bio_parent; if (pbp->bio_error == 0 && bp->bio_error != 0) pbp->bio_error = bp->bio_error; g_destroy_bio(bp); /* * Do we have all sectors already? */ pbp->bio_inbed++; if (pbp->bio_inbed < pbp->bio_children) return; sc = pbp->bio_to->geom->softc; if (pbp->bio_error != 0) { G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__, pbp->bio_error); pbp->bio_completed = 0; - if (pbp->bio_driver2 != NULL) { - free(pbp->bio_driver2, M_ELI); - pbp->bio_driver2 = NULL; - } + g_eli_free_data(pbp); g_io_deliver(pbp, pbp->bio_error); if (sc != NULL) atomic_subtract_int(&sc->sc_inflight, 1); return; } mtx_lock(&sc->sc_queue_mtx); bioq_insert_tail(&sc->sc_queue, pbp); mtx_unlock(&sc->sc_queue_mtx); wakeup(sc); } /* * The function is called after we encrypt and write data. * * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> G_ELI_WRITE_DONE -> g_io_deliver */ void g_eli_write_done(struct bio *bp) { struct g_eli_softc *sc; struct bio *pbp; G_ELI_LOGREQ(2, bp, "Request done."); pbp = bp->bio_parent; if (pbp->bio_error == 0 && bp->bio_error != 0) pbp->bio_error = bp->bio_error; g_destroy_bio(bp); /* * Do we have all sectors already? */ pbp->bio_inbed++; if (pbp->bio_inbed < pbp->bio_children) return; - free(pbp->bio_driver2, M_ELI); - pbp->bio_driver2 = NULL; + sc = pbp->bio_to->geom->softc; + g_eli_free_data(pbp); if (pbp->bio_error != 0) { G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__, pbp->bio_error); pbp->bio_completed = 0; } else pbp->bio_completed = pbp->bio_length; /* * Write is finished, send it up. */ - sc = pbp->bio_to->geom->softc; g_io_deliver(pbp, pbp->bio_error); if (sc != NULL) atomic_subtract_int(&sc->sc_inflight, 1); } /* * This function should never be called, but GEOM made as it set ->orphan() * method for every geom. */ static void g_eli_orphan_spoil_assert(struct g_consumer *cp) { panic("Function %s() called for %s.", __func__, cp->geom->name); } static void g_eli_orphan(struct g_consumer *cp) { struct g_eli_softc *sc; g_topology_assert(); sc = cp->geom->softc; if (sc == NULL) return; g_eli_destroy(sc, TRUE); } static void g_eli_resize(struct g_consumer *cp) { struct g_eli_softc *sc; struct g_provider *epp, *pp; off_t oldsize; g_topology_assert(); sc = cp->geom->softc; if (sc == NULL) return; if ((sc->sc_flags & G_ELI_FLAG_AUTORESIZE) == 0) { G_ELI_DEBUG(0, "Autoresize is turned off, old size: %jd.", (intmax_t)sc->sc_provsize); return; } pp = cp->provider; if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0) { struct g_eli_metadata md; u_char *sector; int error; sector = NULL; error = g_eli_read_metadata_offset(cp->geom->class, pp, sc->sc_provsize - pp->sectorsize, &md); if (error != 0) { G_ELI_DEBUG(0, "Cannot read metadata from %s (error=%d).", pp->name, error); goto iofail; } md.md_provsize = pp->mediasize; sector = malloc(pp->sectorsize, M_ELI, M_WAITOK | M_ZERO); eli_metadata_encode(&md, sector); error = g_write_data(cp, pp->mediasize - pp->sectorsize, sector, pp->sectorsize); if (error != 0) { G_ELI_DEBUG(0, "Cannot store metadata on %s (error=%d).", pp->name, error); goto iofail; } explicit_bzero(sector, pp->sectorsize); error = g_write_data(cp, sc->sc_provsize - pp->sectorsize, sector, pp->sectorsize); if (error != 0) { G_ELI_DEBUG(0, "Cannot clear old metadata from %s (error=%d).", pp->name, error); goto iofail; } iofail: explicit_bzero(&md, sizeof(md)); zfree(sector, M_ELI); } oldsize = sc->sc_mediasize; sc->sc_mediasize = eli_mediasize(sc, pp->mediasize, pp->sectorsize); g_eli_key_resize(sc); sc->sc_provsize = pp->mediasize; epp = LIST_FIRST(&sc->sc_geom->provider); g_resize_provider(epp, sc->sc_mediasize); G_ELI_DEBUG(0, "Device %s size changed from %jd to %jd.", epp->name, (intmax_t)oldsize, (intmax_t)sc->sc_mediasize); } /* * BIO_READ: * G_ELI_START -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver * BIO_WRITE: * G_ELI_START -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver */ static void g_eli_start(struct bio *bp) { struct g_eli_softc *sc; struct g_consumer *cp; struct bio *cbp; sc = bp->bio_to->geom->softc; KASSERT(sc != NULL, ("Provider's error should be set (error=%d)(device=%s).", bp->bio_to->error, bp->bio_to->name)); G_ELI_LOGREQ(2, bp, "Request received."); switch (bp->bio_cmd) { case BIO_READ: case BIO_WRITE: case BIO_GETATTR: case BIO_FLUSH: case BIO_ZONE: case BIO_SPEEDUP: break; case BIO_DELETE: /* * If the user hasn't set the NODELETE flag, we just pass * it down the stack and let the layers beneath us do (or * not) whatever they do with it. If they have, we * reject it. A possible extension would be an * additional flag to take it as a hint to shred the data * with [multiple?] overwrites. */ if (!(sc->sc_flags & G_ELI_FLAG_NODELETE)) break; default: g_io_deliver(bp, EOPNOTSUPP); return; } cbp = g_clone_bio(bp); if (cbp == NULL) { g_io_deliver(bp, ENOMEM); return; } bp->bio_driver1 = cbp; - bp->bio_pflags = G_ELI_NEW_BIO; + bp->bio_pflags = 0; + G_ELI_SET_NEW_BIO(bp->bio_pflags); switch (bp->bio_cmd) { case BIO_READ: if (!(sc->sc_flags & G_ELI_FLAG_AUTH)) { g_eli_crypto_read(sc, bp, 0); break; } /* FALLTHROUGH */ case BIO_WRITE: mtx_lock(&sc->sc_queue_mtx); bioq_insert_tail(&sc->sc_queue, bp); mtx_unlock(&sc->sc_queue_mtx); wakeup(sc); break; case BIO_GETATTR: case BIO_FLUSH: case BIO_DELETE: case BIO_SPEEDUP: case BIO_ZONE: if (bp->bio_cmd == BIO_GETATTR) cbp->bio_done = g_eli_getattr_done; else cbp->bio_done = g_std_done; cp = LIST_FIRST(&sc->sc_geom->consumer); cbp->bio_to = cp->provider; G_ELI_LOGREQ(2, cbp, "Sending request."); g_io_request(cbp, cp); break; } } static int g_eli_newsession(struct g_eli_worker *wr) { struct g_eli_softc *sc; struct crypto_session_params csp; uint32_t caps; int error, new_crypto; void *key; sc = wr->w_softc; memset(&csp, 0, sizeof(csp)); csp.csp_mode = CSP_MODE_CIPHER; csp.csp_cipher_alg = sc->sc_ealgo; csp.csp_ivlen = g_eli_ivlen(sc->sc_ealgo); csp.csp_cipher_klen = sc->sc_ekeylen / 8; if (sc->sc_ealgo == CRYPTO_AES_XTS) csp.csp_cipher_klen <<= 1; if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0) { key = g_eli_key_hold(sc, 0, LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize); csp.csp_cipher_key = key; } else { key = NULL; csp.csp_cipher_key = sc->sc_ekey; } if (sc->sc_flags & G_ELI_FLAG_AUTH) { csp.csp_mode = CSP_MODE_ETA; csp.csp_auth_alg = sc->sc_aalgo; csp.csp_auth_klen = G_ELI_AUTH_SECKEYLEN; } switch (sc->sc_crypto) { case G_ELI_CRYPTO_SW_ACCEL: case G_ELI_CRYPTO_SW: error = crypto_newsession(&wr->w_sid, &csp, CRYPTOCAP_F_SOFTWARE); break; case G_ELI_CRYPTO_HW: error = crypto_newsession(&wr->w_sid, &csp, CRYPTOCAP_F_HARDWARE); break; case G_ELI_CRYPTO_UNKNOWN: error = crypto_newsession(&wr->w_sid, &csp, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); if (error == 0) { caps = crypto_ses2caps(wr->w_sid); if (caps & CRYPTOCAP_F_HARDWARE) new_crypto = G_ELI_CRYPTO_HW; else if (caps & CRYPTOCAP_F_ACCEL_SOFTWARE) new_crypto = G_ELI_CRYPTO_SW_ACCEL; else new_crypto = G_ELI_CRYPTO_SW; mtx_lock(&sc->sc_queue_mtx); if (sc->sc_crypto == G_ELI_CRYPTO_UNKNOWN) sc->sc_crypto = new_crypto; mtx_unlock(&sc->sc_queue_mtx); } break; default: panic("%s: invalid condition", __func__); } if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0) { if (error) g_eli_key_drop(sc, key); else wr->w_first_key = key; } return (error); } static void g_eli_freesession(struct g_eli_worker *wr) { struct g_eli_softc *sc; crypto_freesession(wr->w_sid); if (wr->w_first_key != NULL) { sc = wr->w_softc; g_eli_key_drop(sc, wr->w_first_key); wr->w_first_key = NULL; } } static void g_eli_cancel(struct g_eli_softc *sc) { struct bio *bp; mtx_assert(&sc->sc_queue_mtx, MA_OWNED); while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL) { - KASSERT(bp->bio_pflags == G_ELI_NEW_BIO, + KASSERT(G_ELI_IS_NEW_BIO(bp->bio_pflags), ("Not new bio when canceling (bp=%p).", bp)); g_io_deliver(bp, ENXIO); } } static struct bio * g_eli_takefirst(struct g_eli_softc *sc) { struct bio *bp; mtx_assert(&sc->sc_queue_mtx, MA_OWNED); if (!(sc->sc_flags & G_ELI_FLAG_SUSPEND)) return (bioq_takefirst(&sc->sc_queue)); /* * Device suspended, so we skip new I/O requests. */ TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) { - if (bp->bio_pflags != G_ELI_NEW_BIO) + if (!G_ELI_IS_NEW_BIO(bp->bio_pflags)) break; } if (bp != NULL) bioq_remove(&sc->sc_queue, bp); return (bp); } /* * This is the main function for kernel worker thread when we don't have * hardware acceleration and we have to do cryptography in software. * Dedicated thread is needed, so we don't slow down g_up/g_down GEOM * threads with crypto work. */ static void g_eli_worker(void *arg) { struct g_eli_softc *sc; struct g_eli_worker *wr; struct bio *bp; int error; wr = arg; sc = wr->w_softc; #ifdef EARLY_AP_STARTUP MPASS(!sc->sc_cpubind || smp_started); #elif defined(SMP) /* Before sched_bind() to a CPU, wait for all CPUs to go on-line. */ if (sc->sc_cpubind) { while (!smp_started) tsleep(wr, 0, "geli:smp", hz / 4); } #endif thread_lock(curthread); sched_prio(curthread, PUSER); if (sc->sc_cpubind) sched_bind(curthread, wr->w_number % mp_ncpus); thread_unlock(curthread); G_ELI_DEBUG(1, "Thread %s started.", curthread->td_proc->p_comm); for (;;) { mtx_lock(&sc->sc_queue_mtx); again: bp = g_eli_takefirst(sc); if (bp == NULL) { if (sc->sc_flags & G_ELI_FLAG_DESTROY) { g_eli_cancel(sc); LIST_REMOVE(wr, w_next); g_eli_freesession(wr); free(wr, M_ELI); G_ELI_DEBUG(1, "Thread %s exiting.", curthread->td_proc->p_comm); wakeup(&sc->sc_workers); mtx_unlock(&sc->sc_queue_mtx); kproc_exit(0); } while (sc->sc_flags & G_ELI_FLAG_SUSPEND) { if (sc->sc_inflight > 0) { G_ELI_DEBUG(0, "inflight=%d", sc->sc_inflight); /* * We still have inflight BIOs, so * sleep and retry. */ msleep(sc, &sc->sc_queue_mtx, PRIBIO, "geli:inf", hz / 5); goto again; } /* * Suspend requested, mark the worker as * suspended and go to sleep. */ if (wr->w_active) { g_eli_freesession(wr); wr->w_active = FALSE; } wakeup(&sc->sc_workers); msleep(sc, &sc->sc_queue_mtx, PRIBIO, "geli:suspend", 0); if (!wr->w_active && !(sc->sc_flags & G_ELI_FLAG_SUSPEND)) { error = g_eli_newsession(wr); KASSERT(error == 0, ("g_eli_newsession() failed on resume (error=%d)", error)); wr->w_active = TRUE; } goto again; } msleep(sc, &sc->sc_queue_mtx, PDROP, "geli:w", 0); continue; } - if (bp->bio_pflags == G_ELI_NEW_BIO) + if (G_ELI_IS_NEW_BIO(bp->bio_pflags)) atomic_add_int(&sc->sc_inflight, 1); mtx_unlock(&sc->sc_queue_mtx); - if (bp->bio_pflags == G_ELI_NEW_BIO) { - bp->bio_pflags = 0; + if (G_ELI_IS_NEW_BIO(bp->bio_pflags)) { + G_ELI_SETWORKER(bp->bio_pflags, 0); if (sc->sc_flags & G_ELI_FLAG_AUTH) { if (bp->bio_cmd == BIO_READ) g_eli_auth_read(sc, bp); else g_eli_auth_run(wr, bp); } else { if (bp->bio_cmd == BIO_READ) g_eli_crypto_read(sc, bp, 1); else g_eli_crypto_run(wr, bp); } } else { if (sc->sc_flags & G_ELI_FLAG_AUTH) g_eli_auth_run(wr, bp); else g_eli_crypto_run(wr, bp); } } } static int g_eli_read_metadata_offset(struct g_class *mp, struct g_provider *pp, off_t offset, struct g_eli_metadata *md) { struct g_geom *gp; struct g_consumer *cp; u_char *buf = NULL; int error; g_topology_assert(); gp = g_new_geomf(mp, "eli:taste"); gp->start = g_eli_start; gp->access = g_std_access; /* * g_eli_read_metadata() is always called from the event thread. * Our geom is created and destroyed in the same event, so there * could be no orphan nor spoil event in the meantime. */ gp->orphan = g_eli_orphan_spoil_assert; gp->spoiled = g_eli_orphan_spoil_assert; cp = g_new_consumer(gp); cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; error = g_attach(cp, pp); if (error != 0) goto end; error = g_access(cp, 1, 0, 0); if (error != 0) goto end; g_topology_unlock(); buf = g_read_data(cp, offset, pp->sectorsize, &error); g_topology_lock(); if (buf == NULL) goto end; error = eli_metadata_decode(buf, md); if (error != 0) goto end; /* Metadata was read and decoded successfully. */ end: if (buf != NULL) g_free(buf); if (cp->provider != NULL) { if (cp->acr == 1) g_access(cp, -1, 0, 0); g_detach(cp); } g_destroy_consumer(cp); g_destroy_geom(gp); return (error); } int g_eli_read_metadata(struct g_class *mp, struct g_provider *pp, struct g_eli_metadata *md) { return (g_eli_read_metadata_offset(mp, pp, pp->mediasize - pp->sectorsize, md)); } /* * The function is called when we had last close on provider and user requested * to close it when this situation occur. */ static void g_eli_last_close(void *arg, int flags __unused) { struct g_geom *gp; char gpname[64]; int error; g_topology_assert(); gp = arg; strlcpy(gpname, gp->name, sizeof(gpname)); error = g_eli_destroy(gp->softc, TRUE); KASSERT(error == 0, ("Cannot detach %s on last close (error=%d).", gpname, error)); G_ELI_DEBUG(0, "Detached %s on last close.", gpname); } int g_eli_access(struct g_provider *pp, int dr, int dw, int de) { struct g_eli_softc *sc; struct g_geom *gp; gp = pp->geom; sc = gp->softc; if (dw > 0) { if (sc->sc_flags & G_ELI_FLAG_RO) { /* Deny write attempts. */ return (EROFS); } /* Someone is opening us for write, we need to remember that. */ sc->sc_flags |= G_ELI_FLAG_WOPEN; return (0); } /* Is this the last close? */ if (pp->acr + dr > 0 || pp->acw + dw > 0 || pp->ace + de > 0) return (0); /* * Automatically detach on last close if requested. */ if ((sc->sc_flags & G_ELI_FLAG_RW_DETACH) || (sc->sc_flags & G_ELI_FLAG_WOPEN)) { g_post_event(g_eli_last_close, gp, M_WAITOK, NULL); } return (0); } static int g_eli_cpu_is_disabled(int cpu) { #ifdef SMP return (CPU_ISSET(cpu, &hlt_cpus_mask)); #else return (0); #endif } +static void +g_eli_init_uma(void) +{ + + atomic_add_int(&g_eli_devs, 1); + sx_xlock(&g_eli_umalock); + if (g_eli_uma == NULL) { + /* + * Calculate the maximum-sized swap buffer we are + * likely to see. + */ + g_eli_alloc_sz = roundup2((PAGE_SIZE + sizeof(int) + + G_ELI_AUTH_SECKEYLEN) * nsw_cluster_max + + sizeof(uintptr_t), PAGE_SIZE); + + /* + * Create the zone, setting UMA_ZONE_NOFREE so we won't + * drain the zone in a memory shortage. + */ + g_eli_uma = uma_zcreate("GELI buffers", g_eli_alloc_sz, + NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, UMA_ZONE_NOFREE); + + /* Reserve and pre-allocate pages, as appropriate. */ + uma_zone_reserve(g_eli_uma, g_eli_minbufs); + uma_prealloc(g_eli_uma, g_eli_minbufs); + } + sx_xunlock(&g_eli_umalock); +} + +/* + * Try to destroy the UMA pool. This will do nothing if there are existing + * GELI devices or existing UMA allocations. + */ +static void +g_eli_destroy_uma(void) +{ + uma_zone_t oldzone; + + sx_xlock(&g_eli_umalock); + /* Ensure we really should be destroying this. */ + if (atomic_load_int(&g_eli_devs) == 0 && + atomic_load_int(&g_eli_umaoutstanding) == 0) { + oldzone = g_eli_uma; + g_eli_uma = NULL; + } else + oldzone = NULL; + sx_xunlock(&g_eli_umalock); + + if (oldzone != NULL) + uma_zdestroy(oldzone); +} + +static void +g_eli_fini_uma(void) +{ + + /* + * If this is the last outstanding GELI device, try to + * destroy the UMA pool. + */ + if (atomic_fetchadd_int(&g_eli_devs, -1) == 1) + g_eli_destroy_uma(); +} + +/* + * Allocate a data buffer. If the size fits within our swap-sized buffers, + * try to allocate a swap-sized buffer from the UMA pool. Otherwise, fall + * back to using malloc. + * + * Swap-related requests are special: they can only use the UMA pool, they + * use M_USE_RESERVE to let them dip farther into system resources, and + * they always use M_NOWAIT to prevent swap operations from deadlocking. + */ +bool +g_eli_alloc_data(struct bio *bp, int sz) +{ + + KASSERT(sz <= g_eli_alloc_sz || (bp->bio_flags & BIO_SWAP) == 0, + ("BIO_SWAP request for %d bytes exceeds the precalculated buffer" + " size (%d)", sz, g_eli_alloc_sz)); + if (sz <= g_eli_alloc_sz) { + bp->bio_driver2 = uma_zalloc(g_eli_uma, M_NOWAIT | + ((bp->bio_flags & BIO_SWAP) != 0 ? M_USE_RESERVE : 0)); + if (bp->bio_driver2 != NULL) { + bp->bio_pflags |= G_ELI_UMA_ALLOC; + atomic_add_int(&g_eli_umaoutstanding, 1); + } + if (bp->bio_driver2 != NULL || (bp->bio_flags & BIO_SWAP) != 0) + return (bp->bio_driver2 != NULL); + } + bp->bio_pflags &= ~(G_ELI_UMA_ALLOC); + bp->bio_driver2 = malloc(sz, M_ELI, g_eli_blocking_malloc ? M_WAITOK : + M_NOWAIT); + return (bp->bio_driver2 != NULL); +} + +/* + * Free a buffer from bp->bio_driver2 which was allocated with + * g_eli_alloc_data(). This function makes sure that the memory is freed + * to the correct place. + * + * Additionally, if this function frees the last outstanding UMA request + * and there are no open GELI devices, this will destroy the UMA pool. + */ +void +g_eli_free_data(struct bio *bp) +{ + + /* + * Mimic the free(9) behavior of allowing a NULL pointer to be + * freed. + */ + if (bp->bio_driver2 == NULL) + return; + + if ((bp->bio_pflags & G_ELI_UMA_ALLOC) != 0) { + uma_zfree(g_eli_uma, bp->bio_driver2); + if (atomic_fetchadd_int(&g_eli_umaoutstanding, -1) == 1 && + atomic_load_int(&g_eli_devs) == 0) + g_eli_destroy_uma(); + } else + free(bp->bio_driver2, M_ELI); + bp->bio_driver2 = NULL; +} + struct g_geom * g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp, const struct g_eli_metadata *md, const u_char *mkey, int nkey) { struct g_eli_softc *sc; struct g_eli_worker *wr; struct g_geom *gp; struct g_provider *pp; struct g_consumer *cp; struct g_geom_alias *gap; u_int i, threads; int dcw, error; G_ELI_DEBUG(1, "Creating device %s%s.", bpp->name, G_ELI_SUFFIX); KASSERT(eli_metadata_crypto_supported(md), ("%s: unsupported crypto for %s", __func__, bpp->name)); gp = g_new_geomf(mp, "%s%s", bpp->name, G_ELI_SUFFIX); sc = malloc(sizeof(*sc), M_ELI, M_WAITOK | M_ZERO); gp->start = g_eli_start; /* * Spoiling can happen even though we have the provider open * exclusively, e.g. through media change events. */ gp->spoiled = g_eli_orphan; gp->orphan = g_eli_orphan; gp->resize = g_eli_resize; gp->dumpconf = g_eli_dumpconf; /* * If detach-on-last-close feature is not enabled and we don't operate * on read-only provider, we can simply use g_std_access(). */ if (md->md_flags & (G_ELI_FLAG_WO_DETACH | G_ELI_FLAG_RO)) gp->access = g_eli_access; else gp->access = g_std_access; eli_metadata_softc(sc, md, bpp->sectorsize, bpp->mediasize); sc->sc_nkey = nkey; gp->softc = sc; sc->sc_geom = gp; bioq_init(&sc->sc_queue); mtx_init(&sc->sc_queue_mtx, "geli:queue", NULL, MTX_DEF); mtx_init(&sc->sc_ekeys_lock, "geli:ekeys", NULL, MTX_DEF); pp = NULL; cp = g_new_consumer(gp); cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; error = g_attach(cp, bpp); if (error != 0) { if (req != NULL) { gctl_error(req, "Cannot attach to %s (error=%d).", bpp->name, error); } else { G_ELI_DEBUG(1, "Cannot attach to %s (error=%d).", bpp->name, error); } goto failed; } /* * Keep provider open all the time, so we can run critical tasks, * like Master Keys deletion, without wondering if we can open * provider or not. * We don't open provider for writing only when user requested read-only * access. */ dcw = (sc->sc_flags & G_ELI_FLAG_RO) ? 0 : 1; error = g_access(cp, 1, dcw, 1); if (error != 0) { if (req != NULL) { gctl_error(req, "Cannot access %s (error=%d).", bpp->name, error); } else { G_ELI_DEBUG(1, "Cannot access %s (error=%d).", bpp->name, error); } goto failed; } /* * Remember the keys in our softc structure. */ g_eli_mkey_propagate(sc, mkey); LIST_INIT(&sc->sc_workers); threads = g_eli_threads; if (threads == 0) threads = mp_ncpus; sc->sc_cpubind = (mp_ncpus > 1 && threads == mp_ncpus); + g_eli_init_uma(); for (i = 0; i < threads; i++) { if (g_eli_cpu_is_disabled(i)) { G_ELI_DEBUG(1, "%s: CPU %u disabled, skipping.", bpp->name, i); continue; } wr = malloc(sizeof(*wr), M_ELI, M_WAITOK | M_ZERO); wr->w_softc = sc; wr->w_number = i; wr->w_active = TRUE; error = g_eli_newsession(wr); if (error != 0) { free(wr, M_ELI); if (req != NULL) { gctl_error(req, "Cannot set up crypto session " "for %s (error=%d).", bpp->name, error); } else { G_ELI_DEBUG(1, "Cannot set up crypto session " "for %s (error=%d).", bpp->name, error); } goto failed; } error = kproc_create(g_eli_worker, wr, &wr->w_proc, 0, 0, "g_eli[%u] %s", i, bpp->name); if (error != 0) { g_eli_freesession(wr); free(wr, M_ELI); if (req != NULL) { gctl_error(req, "Cannot create kernel thread " "for %s (error=%d).", bpp->name, error); } else { G_ELI_DEBUG(1, "Cannot create kernel thread " "for %s (error=%d).", bpp->name, error); } goto failed; } LIST_INSERT_HEAD(&sc->sc_workers, wr, w_next); } /* * Create decrypted provider. */ pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX); pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE; if (CRYPTO_HAS_VMPAGE) { /* * On DMAP architectures we can use unmapped I/O. But don't * use it with data integrity verification. That code hasn't * been written yet. */ if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0) pp->flags |= G_PF_ACCEPT_UNMAPPED; } pp->mediasize = sc->sc_mediasize; pp->sectorsize = sc->sc_sectorsize; LIST_FOREACH(gap, &bpp->aliases, ga_next) g_provider_add_alias(pp, "%s%s", gap->ga_alias, G_ELI_SUFFIX); g_error_provider(pp, 0); G_ELI_DEBUG(0, "Device %s created.", pp->name); G_ELI_DEBUG(0, "Encryption: %s %u", g_eli_algo2str(sc->sc_ealgo), sc->sc_ekeylen); if (sc->sc_flags & G_ELI_FLAG_AUTH) G_ELI_DEBUG(0, " Integrity: %s", g_eli_algo2str(sc->sc_aalgo)); G_ELI_DEBUG(0, " Crypto: %s", sc->sc_crypto == G_ELI_CRYPTO_SW_ACCEL ? "accelerated software" : sc->sc_crypto == G_ELI_CRYPTO_SW ? "software" : "hardware"); return (gp); failed: mtx_lock(&sc->sc_queue_mtx); sc->sc_flags |= G_ELI_FLAG_DESTROY; wakeup(sc); /* * Wait for kernel threads self destruction. */ while (!LIST_EMPTY(&sc->sc_workers)) { msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO, "geli:destroy", 0); } mtx_destroy(&sc->sc_queue_mtx); if (cp->provider != NULL) { if (cp->acr == 1) g_access(cp, -1, -dcw, -1); g_detach(cp); } g_destroy_consumer(cp); g_destroy_geom(gp); g_eli_key_destroy(sc); + g_eli_fini_uma(); zfree(sc, M_ELI); return (NULL); } int g_eli_destroy(struct g_eli_softc *sc, boolean_t force) { struct g_geom *gp; struct g_provider *pp; g_topology_assert(); if (sc == NULL) return (ENXIO); gp = sc->sc_geom; pp = LIST_FIRST(&gp->provider); if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) { if (force) { G_ELI_DEBUG(1, "Device %s is still open, so it " "cannot be definitely removed.", pp->name); sc->sc_flags |= G_ELI_FLAG_RW_DETACH; gp->access = g_eli_access; g_wither_provider(pp, ENXIO); return (EBUSY); } else { G_ELI_DEBUG(1, "Device %s is still open (r%dw%de%d).", pp->name, pp->acr, pp->acw, pp->ace); return (EBUSY); } } mtx_lock(&sc->sc_queue_mtx); sc->sc_flags |= G_ELI_FLAG_DESTROY; wakeup(sc); while (!LIST_EMPTY(&sc->sc_workers)) { msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO, "geli:destroy", 0); } mtx_destroy(&sc->sc_queue_mtx); gp->softc = NULL; g_eli_key_destroy(sc); + g_eli_fini_uma(); zfree(sc, M_ELI); G_ELI_DEBUG(0, "Device %s destroyed.", gp->name); g_wither_geom_close(gp, ENXIO); return (0); } static int g_eli_destroy_geom(struct gctl_req *req __unused, struct g_class *mp __unused, struct g_geom *gp) { struct g_eli_softc *sc; sc = gp->softc; return (g_eli_destroy(sc, FALSE)); } static int g_eli_keyfiles_load(struct hmac_ctx *ctx, const char *provider) { u_char *keyfile, *data; char *file, name[64]; size_t size; int i; for (i = 0; ; i++) { snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i); keyfile = preload_search_by_type(name); if (keyfile == NULL && i == 0) { /* * If there is only one keyfile, allow simpler name. */ snprintf(name, sizeof(name), "%s:geli_keyfile", provider); keyfile = preload_search_by_type(name); } if (keyfile == NULL) return (i); /* Return number of loaded keyfiles. */ data = preload_fetch_addr(keyfile); if (data == NULL) { G_ELI_DEBUG(0, "Cannot find key file data for %s.", name); return (0); } size = preload_fetch_size(keyfile); if (size == 0) { G_ELI_DEBUG(0, "Cannot find key file size for %s.", name); return (0); } file = preload_search_info(keyfile, MODINFO_NAME); if (file == NULL) { G_ELI_DEBUG(0, "Cannot find key file name for %s.", name); return (0); } G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s).", file, provider, name); g_eli_crypto_hmac_update(ctx, data, size); } } static void g_eli_keyfiles_clear(const char *provider) { u_char *keyfile, *data; char name[64]; size_t size; int i; for (i = 0; ; i++) { snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i); keyfile = preload_search_by_type(name); if (keyfile == NULL) return; data = preload_fetch_addr(keyfile); size = preload_fetch_size(keyfile); if (data != NULL && size != 0) explicit_bzero(data, size); } } /* * Tasting is only made on boot. * We detect providers which should be attached before root is mounted. */ static struct g_geom * g_eli_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) { struct g_eli_metadata md; struct g_geom *gp; struct hmac_ctx ctx; char passphrase[256]; u_char key[G_ELI_USERKEYLEN], mkey[G_ELI_DATAIVKEYLEN]; u_int i, nkey, nkeyfiles, tries, showpass; int error; struct keybuf *keybuf; g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name); g_topology_assert(); if (root_mounted() || g_eli_tries == 0) return (NULL); G_ELI_DEBUG(3, "Tasting %s.", pp->name); error = g_eli_read_metadata(mp, pp, &md); if (error != 0) return (NULL); gp = NULL; if (strcmp(md.md_magic, G_ELI_MAGIC) != 0) return (NULL); if (md.md_version > G_ELI_VERSION) { printf("geom_eli.ko module is too old to handle %s.\n", pp->name); return (NULL); } if (md.md_provsize != pp->mediasize) return (NULL); /* Should we attach it on boot? */ if (!(md.md_flags & G_ELI_FLAG_BOOT) && !(md.md_flags & G_ELI_FLAG_GELIBOOT)) return (NULL); if (md.md_keys == 0x00) { G_ELI_DEBUG(0, "No valid keys on %s.", pp->name); return (NULL); } if (!eli_metadata_crypto_supported(&md)) { G_ELI_DEBUG(0, "%s uses invalid or unsupported algorithms\n", pp->name); return (NULL); } if (md.md_iterations == -1) { /* If there is no passphrase, we try only once. */ tries = 1; } else { /* Ask for the passphrase no more than g_eli_tries times. */ tries = g_eli_tries; } if ((keybuf = get_keybuf()) != NULL) { /* Scan the key buffer, try all GELI keys. */ for (i = 0; i < keybuf->kb_nents; i++) { if (keybuf->kb_ents[i].ke_type == KEYBUF_TYPE_GELI) { memcpy(key, keybuf->kb_ents[i].ke_data, sizeof(key)); if (g_eli_mkey_decrypt_any(&md, key, mkey, &nkey) == 0 ) { explicit_bzero(key, sizeof(key)); goto have_key; } } } } for (i = 0; i <= tries; i++) { g_eli_crypto_hmac_init(&ctx, NULL, 0); /* * Load all key files. */ nkeyfiles = g_eli_keyfiles_load(&ctx, pp->name); if (nkeyfiles == 0 && md.md_iterations == -1) { /* * No key files and no passphrase, something is * definitely wrong here. * geli(8) doesn't allow for such situation, so assume * that there was really no passphrase and in that case * key files are no properly defined in loader.conf. */ G_ELI_DEBUG(0, "Found no key files in loader.conf for %s.", pp->name); return (NULL); } /* Ask for the passphrase if defined. */ if (md.md_iterations >= 0) { /* Try first with cached passphrase. */ if (i == 0) { if (!g_eli_boot_passcache) continue; memcpy(passphrase, cached_passphrase, sizeof(passphrase)); } else { printf("Enter passphrase for %s: ", pp->name); showpass = g_eli_visible_passphrase; if ((md.md_flags & G_ELI_FLAG_GELIDISPLAYPASS) != 0) showpass = GETS_ECHOPASS; cngets(passphrase, sizeof(passphrase), showpass); memcpy(cached_passphrase, passphrase, sizeof(passphrase)); } } /* * Prepare Derived-Key from the user passphrase. */ if (md.md_iterations == 0) { g_eli_crypto_hmac_update(&ctx, md.md_salt, sizeof(md.md_salt)); g_eli_crypto_hmac_update(&ctx, passphrase, strlen(passphrase)); explicit_bzero(passphrase, sizeof(passphrase)); } else if (md.md_iterations > 0) { u_char dkey[G_ELI_USERKEYLEN]; pkcs5v2_genkey(dkey, sizeof(dkey), md.md_salt, sizeof(md.md_salt), passphrase, md.md_iterations); explicit_bzero(passphrase, sizeof(passphrase)); g_eli_crypto_hmac_update(&ctx, dkey, sizeof(dkey)); explicit_bzero(dkey, sizeof(dkey)); } g_eli_crypto_hmac_final(&ctx, key, 0); /* * Decrypt Master-Key. */ error = g_eli_mkey_decrypt_any(&md, key, mkey, &nkey); explicit_bzero(key, sizeof(key)); if (error == -1) { if (i == tries) { G_ELI_DEBUG(0, "Wrong key for %s. No tries left.", pp->name); g_eli_keyfiles_clear(pp->name); return (NULL); } if (i > 0) { G_ELI_DEBUG(0, "Wrong key for %s. Tries left: %u.", pp->name, tries - i); } /* Try again. */ continue; } else if (error > 0) { G_ELI_DEBUG(0, "Cannot decrypt Master Key for %s (error=%d).", pp->name, error); g_eli_keyfiles_clear(pp->name); return (NULL); } g_eli_keyfiles_clear(pp->name); G_ELI_DEBUG(1, "Using Master Key %u for %s.", nkey, pp->name); break; } have_key: /* * We have correct key, let's attach provider. */ gp = g_eli_create(NULL, mp, pp, &md, mkey, nkey); explicit_bzero(mkey, sizeof(mkey)); explicit_bzero(&md, sizeof(md)); if (gp == NULL) { G_ELI_DEBUG(0, "Cannot create device %s%s.", pp->name, G_ELI_SUFFIX); return (NULL); } return (gp); } static void g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp) { struct g_eli_softc *sc; g_topology_assert(); sc = gp->softc; if (sc == NULL) return; if (pp != NULL || cp != NULL) return; /* Nothing here. */ sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t)sc->sc_ekeys_total); sbuf_printf(sb, "%s%ju\n", indent, (uintmax_t)sc->sc_ekeys_allocated); sbuf_printf(sb, "%s", indent); if (sc->sc_flags == 0) sbuf_cat(sb, "NONE"); else { int first = 1; #define ADD_FLAG(flag, name) do { \ if (sc->sc_flags & (flag)) { \ if (!first) \ sbuf_cat(sb, ", "); \ else \ first = 0; \ sbuf_cat(sb, name); \ } \ } while (0) ADD_FLAG(G_ELI_FLAG_SUSPEND, "SUSPEND"); ADD_FLAG(G_ELI_FLAG_SINGLE_KEY, "SINGLE-KEY"); ADD_FLAG(G_ELI_FLAG_NATIVE_BYTE_ORDER, "NATIVE-BYTE-ORDER"); ADD_FLAG(G_ELI_FLAG_ONETIME, "ONETIME"); ADD_FLAG(G_ELI_FLAG_BOOT, "BOOT"); ADD_FLAG(G_ELI_FLAG_WO_DETACH, "W-DETACH"); ADD_FLAG(G_ELI_FLAG_RW_DETACH, "RW-DETACH"); ADD_FLAG(G_ELI_FLAG_AUTH, "AUTH"); ADD_FLAG(G_ELI_FLAG_WOPEN, "W-OPEN"); ADD_FLAG(G_ELI_FLAG_DESTROY, "DESTROY"); ADD_FLAG(G_ELI_FLAG_RO, "READ-ONLY"); ADD_FLAG(G_ELI_FLAG_NODELETE, "NODELETE"); ADD_FLAG(G_ELI_FLAG_GELIBOOT, "GELIBOOT"); ADD_FLAG(G_ELI_FLAG_GELIDISPLAYPASS, "GELIDISPLAYPASS"); ADD_FLAG(G_ELI_FLAG_AUTORESIZE, "AUTORESIZE"); #undef ADD_FLAG } sbuf_cat(sb, "\n"); if (!(sc->sc_flags & G_ELI_FLAG_ONETIME)) { sbuf_printf(sb, "%s%u\n", indent, sc->sc_nkey); } sbuf_printf(sb, "%s%u\n", indent, sc->sc_version); sbuf_printf(sb, "%s", indent); switch (sc->sc_crypto) { case G_ELI_CRYPTO_HW: sbuf_cat(sb, "hardware"); break; case G_ELI_CRYPTO_SW: sbuf_cat(sb, "software"); break; case G_ELI_CRYPTO_SW_ACCEL: sbuf_cat(sb, "accelerated software"); break; default: sbuf_cat(sb, "UNKNOWN"); break; } sbuf_cat(sb, "\n"); if (sc->sc_flags & G_ELI_FLAG_AUTH) { sbuf_printf(sb, "%s%s\n", indent, g_eli_algo2str(sc->sc_aalgo)); } sbuf_printf(sb, "%s%u\n", indent, sc->sc_ekeylen); sbuf_printf(sb, "%s%s\n", indent, g_eli_algo2str(sc->sc_ealgo)); sbuf_printf(sb, "%s%s\n", indent, (sc->sc_flags & G_ELI_FLAG_SUSPEND) ? "SUSPENDED" : "ACTIVE"); } static void g_eli_shutdown_pre_sync(void *arg, int howto) { struct g_class *mp; struct g_geom *gp, *gp2; struct g_provider *pp; struct g_eli_softc *sc; int error; mp = arg; g_topology_lock(); LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) { sc = gp->softc; if (sc == NULL) continue; pp = LIST_FIRST(&gp->provider); KASSERT(pp != NULL, ("No provider? gp=%p (%s)", gp, gp->name)); if (pp->acr != 0 || pp->acw != 0 || pp->ace != 0 || SCHEDULER_STOPPED()) { sc->sc_flags |= G_ELI_FLAG_RW_DETACH; gp->access = g_eli_access; } else { error = g_eli_destroy(sc, TRUE); } } g_topology_unlock(); } static void g_eli_init(struct g_class *mp) { g_eli_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync, g_eli_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST); if (g_eli_pre_sync == NULL) G_ELI_DEBUG(0, "Warning! Cannot register shutdown event."); } static void g_eli_fini(struct g_class *mp) { if (g_eli_pre_sync != NULL) EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_eli_pre_sync); } DECLARE_GEOM_CLASS(g_eli_class, g_eli); MODULE_DEPEND(g_eli, crypto, 1, 1, 1); MODULE_VERSION(geom_eli, 0); diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h index d07afe0c8a61..9cd9f1a7f3e6 100644 --- a/sys/geom/eli/g_eli.h +++ b/sys/geom/eli/g_eli.h @@ -1,742 +1,753 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2005-2019 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _G_ELI_H_ #define _G_ELI_H_ #include #include #include #include #include #include #ifdef _KERNEL #include #include #include #include #include #include #else #include #include #include #include #endif #include #include #ifndef _OpenSSL_ #include #endif #define G_ELI_CLASS_NAME "ELI" #define G_ELI_MAGIC "GEOM::ELI" #define G_ELI_SUFFIX ".eli" /* * Version history: * 0 - Initial version number. * 1 - Added data authentication support (md_aalgo field and * G_ELI_FLAG_AUTH flag). * 2 - Added G_ELI_FLAG_READONLY. * 3 - Added 'configure' subcommand. * 4 - IV is generated from offset converted to little-endian * (the G_ELI_FLAG_NATIVE_BYTE_ORDER flag will be set for older versions). * 5 - Added multiple encrypton keys and AES-XTS support. * 6 - Fixed usage of multiple keys for authenticated providers (the * G_ELI_FLAG_FIRST_KEY flag will be set for older versions). * 7 - Encryption keys are now generated from the Data Key and not from the * IV Key (the G_ELI_FLAG_ENC_IVKEY flag will be set for older versions). */ #define G_ELI_VERSION_00 0 #define G_ELI_VERSION_01 1 #define G_ELI_VERSION_02 2 #define G_ELI_VERSION_03 3 #define G_ELI_VERSION_04 4 #define G_ELI_VERSION_05 5 #define G_ELI_VERSION_06 6 #define G_ELI_VERSION_07 7 #define G_ELI_VERSION G_ELI_VERSION_07 /* ON DISK FLAGS. */ /* Use random, onetime keys. */ #define G_ELI_FLAG_ONETIME 0x00000001 /* Ask for the passphrase from the kernel, before mounting root. */ #define G_ELI_FLAG_BOOT 0x00000002 /* Detach on last close, if we were open for writing. */ #define G_ELI_FLAG_WO_DETACH 0x00000004 /* Detach on last close. */ #define G_ELI_FLAG_RW_DETACH 0x00000008 /* Provide data authentication. */ #define G_ELI_FLAG_AUTH 0x00000010 /* Provider is read-only, we should deny all write attempts. */ #define G_ELI_FLAG_RO 0x00000020 /* Don't pass through BIO_DELETE requests. */ #define G_ELI_FLAG_NODELETE 0x00000040 /* This GELI supports GELIBoot */ #define G_ELI_FLAG_GELIBOOT 0x00000080 /* Hide passphrase length in GELIboot. */ #define G_ELI_FLAG_GELIDISPLAYPASS 0x00000100 /* Expand provider automatically. */ #define G_ELI_FLAG_AUTORESIZE 0x00000200 /* RUNTIME FLAGS. */ /* Provider was open for writing. */ #define G_ELI_FLAG_WOPEN 0x00010000 /* Destroy device. */ #define G_ELI_FLAG_DESTROY 0x00020000 /* Provider uses native byte-order for IV generation. */ #define G_ELI_FLAG_NATIVE_BYTE_ORDER 0x00040000 /* Provider uses single encryption key. */ #define G_ELI_FLAG_SINGLE_KEY 0x00080000 /* Device suspended. */ #define G_ELI_FLAG_SUSPEND 0x00100000 /* Provider uses first encryption key. */ #define G_ELI_FLAG_FIRST_KEY 0x00200000 /* Provider uses IV-Key for encryption key generation. */ #define G_ELI_FLAG_ENC_IVKEY 0x00400000 -#define G_ELI_NEW_BIO 255 +/* BIO pflag values. */ +#define G_ELI_WORKER(pflags) ((pflags) & 0xff) +#define G_ELI_MAX_WORKERS 255 +#define G_ELI_NEW_BIO G_ELI_MAX_WORKERS +#define G_ELI_SETWORKER(pflags, w) \ + (pflags) = ((pflags) & 0xff00) | ((w) & 0xff) +#define G_ELI_SET_NEW_BIO(pflags) G_ELI_SETWORKER((pflags), G_ELI_NEW_BIO) +#define G_ELI_IS_NEW_BIO(pflags) (G_ELI_WORKER(pflags) == G_ELI_NEW_BIO) +#define G_ELI_UMA_ALLOC 0x100 /* bio_driver2 alloc came from UMA */ #define SHA512_MDLEN 64 #define G_ELI_AUTH_SECKEYLEN SHA256_DIGEST_LENGTH #define G_ELI_MAXMKEYS 2 #define G_ELI_MAXKEYLEN 64 #define G_ELI_USERKEYLEN G_ELI_MAXKEYLEN #define G_ELI_DATAKEYLEN G_ELI_MAXKEYLEN #define G_ELI_AUTHKEYLEN G_ELI_MAXKEYLEN #define G_ELI_IVKEYLEN G_ELI_MAXKEYLEN #define G_ELI_SALTLEN 64 #define G_ELI_DATAIVKEYLEN (G_ELI_DATAKEYLEN + G_ELI_IVKEYLEN) /* Data-Key, IV-Key, HMAC_SHA512(Derived-Key, Data-Key+IV-Key) */ #define G_ELI_MKEYLEN (G_ELI_DATAIVKEYLEN + SHA512_MDLEN) #define G_ELI_OVERWRITES 5 /* Switch data encryption key every 2^20 blocks. */ #define G_ELI_KEY_SHIFT 20 #define G_ELI_CRYPTO_UNKNOWN 0 #define G_ELI_CRYPTO_HW 1 #define G_ELI_CRYPTO_SW 2 #define G_ELI_CRYPTO_SW_ACCEL 3 #ifdef _KERNEL #if (MAX_KEY_BYTES < G_ELI_DATAIVKEYLEN) #error "MAX_KEY_BYTES is less than G_ELI_DATAKEYLEN" #endif extern int g_eli_debug; extern u_int g_eli_overwrites; extern u_int g_eli_batch; #define G_ELI_DEBUG(lvl, ...) \ _GEOM_DEBUG("GEOM_ELI", g_eli_debug, (lvl), NULL, __VA_ARGS__) #define G_ELI_LOGREQ(lvl, bp, ...) \ _GEOM_DEBUG("GEOM_ELI", g_eli_debug, (lvl), (bp), __VA_ARGS__) struct g_eli_worker { struct g_eli_softc *w_softc; struct proc *w_proc; void *w_first_key; u_int w_number; crypto_session_t w_sid; boolean_t w_active; LIST_ENTRY(g_eli_worker) w_next; }; #endif /* _KERNEL */ struct g_eli_softc { struct g_geom *sc_geom; u_int sc_version; u_int sc_crypto; uint8_t sc_mkey[G_ELI_DATAIVKEYLEN]; uint8_t sc_ekey[G_ELI_DATAKEYLEN]; TAILQ_HEAD(, g_eli_key) sc_ekeys_queue; RB_HEAD(g_eli_key_tree, g_eli_key) sc_ekeys_tree; #ifndef _STANDALONE struct mtx sc_ekeys_lock; #endif uint64_t sc_ekeys_total; uint64_t sc_ekeys_allocated; u_int sc_ealgo; u_int sc_ekeylen; uint8_t sc_akey[G_ELI_AUTHKEYLEN]; u_int sc_aalgo; u_int sc_akeylen; u_int sc_alen; SHA256_CTX sc_akeyctx; uint8_t sc_ivkey[G_ELI_IVKEYLEN]; SHA256_CTX sc_ivctx; int sc_nkey; uint32_t sc_flags; int sc_inflight; off_t sc_mediasize; size_t sc_sectorsize; off_t sc_provsize; u_int sc_bytes_per_sector; u_int sc_data_per_sector; #ifndef _KERNEL int sc_cpubind; #else /* _KERNEL */ boolean_t sc_cpubind; /* Only for software cryptography. */ struct bio_queue_head sc_queue; struct mtx sc_queue_mtx; LIST_HEAD(, g_eli_worker) sc_workers; #endif /* _KERNEL */ }; #define sc_name sc_geom->name #define G_ELI_KEY_MAGIC 0xe11341c struct g_eli_key { /* Key value, must be first in the structure. */ uint8_t gek_key[G_ELI_DATAKEYLEN]; /* Magic. */ int gek_magic; /* Key number. */ uint64_t gek_keyno; /* Reference counter. */ int gek_count; /* Keeps keys sorted by most recent use. */ TAILQ_ENTRY(g_eli_key) gek_next; /* Keeps keys sorted by number. */ RB_ENTRY(g_eli_key) gek_link; }; struct g_eli_metadata { char md_magic[16]; /* Magic value. */ uint32_t md_version; /* Version number. */ uint32_t md_flags; /* Additional flags. */ uint16_t md_ealgo; /* Encryption algorithm. */ uint16_t md_keylen; /* Key length. */ uint16_t md_aalgo; /* Authentication algorithm. */ uint64_t md_provsize; /* Provider's size. */ uint32_t md_sectorsize; /* Sector size. */ uint8_t md_keys; /* Available keys. */ int32_t md_iterations; /* Number of iterations for PKCS#5v2. */ uint8_t md_salt[G_ELI_SALTLEN]; /* Salt. */ /* Encrypted master key (IV-key, Data-key, HMAC). */ uint8_t md_mkeys[G_ELI_MAXMKEYS * G_ELI_MKEYLEN]; u_char md_hash[16]; /* MD5 hash. */ } __packed; #ifndef _OpenSSL_ static __inline void eli_metadata_encode_v0(struct g_eli_metadata *md, u_char **datap) { u_char *p; p = *datap; le32enc(p, md->md_flags); p += sizeof(md->md_flags); le16enc(p, md->md_ealgo); p += sizeof(md->md_ealgo); le16enc(p, md->md_keylen); p += sizeof(md->md_keylen); le64enc(p, md->md_provsize); p += sizeof(md->md_provsize); le32enc(p, md->md_sectorsize); p += sizeof(md->md_sectorsize); *p = md->md_keys; p += sizeof(md->md_keys); le32enc(p, md->md_iterations); p += sizeof(md->md_iterations); bcopy(md->md_salt, p, sizeof(md->md_salt)); p += sizeof(md->md_salt); bcopy(md->md_mkeys, p, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys); *datap = p; } static __inline void eli_metadata_encode_v1v2v3v4v5v6v7(struct g_eli_metadata *md, u_char **datap) { u_char *p; p = *datap; le32enc(p, md->md_flags); p += sizeof(md->md_flags); le16enc(p, md->md_ealgo); p += sizeof(md->md_ealgo); le16enc(p, md->md_keylen); p += sizeof(md->md_keylen); le16enc(p, md->md_aalgo); p += sizeof(md->md_aalgo); le64enc(p, md->md_provsize); p += sizeof(md->md_provsize); le32enc(p, md->md_sectorsize); p += sizeof(md->md_sectorsize); *p = md->md_keys; p += sizeof(md->md_keys); le32enc(p, md->md_iterations); p += sizeof(md->md_iterations); bcopy(md->md_salt, p, sizeof(md->md_salt)); p += sizeof(md->md_salt); bcopy(md->md_mkeys, p, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys); *datap = p; } static __inline void eli_metadata_encode(struct g_eli_metadata *md, u_char *data) { uint32_t hash[4]; MD5_CTX ctx; u_char *p; p = data; bcopy(md->md_magic, p, sizeof(md->md_magic)); p += sizeof(md->md_magic); le32enc(p, md->md_version); p += sizeof(md->md_version); switch (md->md_version) { case G_ELI_VERSION_00: eli_metadata_encode_v0(md, &p); break; case G_ELI_VERSION_01: case G_ELI_VERSION_02: case G_ELI_VERSION_03: case G_ELI_VERSION_04: case G_ELI_VERSION_05: case G_ELI_VERSION_06: case G_ELI_VERSION_07: eli_metadata_encode_v1v2v3v4v5v6v7(md, &p); break; default: #ifdef _KERNEL panic("%s: Unsupported version %u.", __func__, (u_int)md->md_version); #else assert(!"Unsupported metadata version."); #endif } MD5Init(&ctx); MD5Update(&ctx, data, p - data); MD5Final((void *)hash, &ctx); bcopy(hash, md->md_hash, sizeof(md->md_hash)); bcopy(md->md_hash, p, sizeof(md->md_hash)); } static __inline int eli_metadata_decode_v0(const u_char *data, struct g_eli_metadata *md) { uint32_t hash[4]; MD5_CTX ctx; const u_char *p; p = data + sizeof(md->md_magic) + sizeof(md->md_version); md->md_flags = le32dec(p); p += sizeof(md->md_flags); md->md_ealgo = le16dec(p); p += sizeof(md->md_ealgo); md->md_keylen = le16dec(p); p += sizeof(md->md_keylen); md->md_provsize = le64dec(p); p += sizeof(md->md_provsize); md->md_sectorsize = le32dec(p); p += sizeof(md->md_sectorsize); md->md_keys = *p; p += sizeof(md->md_keys); md->md_iterations = le32dec(p); p += sizeof(md->md_iterations); bcopy(p, md->md_salt, sizeof(md->md_salt)); p += sizeof(md->md_salt); bcopy(p, md->md_mkeys, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys); MD5Init(&ctx); MD5Update(&ctx, data, p - data); MD5Final((void *)hash, &ctx); bcopy(hash, md->md_hash, sizeof(md->md_hash)); if (bcmp(md->md_hash, p, 16) != 0) return (EINVAL); return (0); } static __inline int eli_metadata_decode_v1v2v3v4v5v6v7(const u_char *data, struct g_eli_metadata *md) { uint32_t hash[4]; MD5_CTX ctx; const u_char *p; p = data + sizeof(md->md_magic) + sizeof(md->md_version); md->md_flags = le32dec(p); p += sizeof(md->md_flags); md->md_ealgo = le16dec(p); p += sizeof(md->md_ealgo); md->md_keylen = le16dec(p); p += sizeof(md->md_keylen); md->md_aalgo = le16dec(p); p += sizeof(md->md_aalgo); md->md_provsize = le64dec(p); p += sizeof(md->md_provsize); md->md_sectorsize = le32dec(p); p += sizeof(md->md_sectorsize); md->md_keys = *p; p += sizeof(md->md_keys); md->md_iterations = le32dec(p); p += sizeof(md->md_iterations); bcopy(p, md->md_salt, sizeof(md->md_salt)); p += sizeof(md->md_salt); bcopy(p, md->md_mkeys, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys); MD5Init(&ctx); MD5Update(&ctx, data, p - data); MD5Final((void *)hash, &ctx); bcopy(hash, md->md_hash, sizeof(md->md_hash)); if (bcmp(md->md_hash, p, 16) != 0) return (EINVAL); return (0); } static __inline int eli_metadata_decode(const u_char *data, struct g_eli_metadata *md) { int error; bcopy(data, md->md_magic, sizeof(md->md_magic)); if (strcmp(md->md_magic, G_ELI_MAGIC) != 0) return (EINVAL); md->md_version = le32dec(data + sizeof(md->md_magic)); switch (md->md_version) { case G_ELI_VERSION_00: error = eli_metadata_decode_v0(data, md); break; case G_ELI_VERSION_01: case G_ELI_VERSION_02: case G_ELI_VERSION_03: case G_ELI_VERSION_04: case G_ELI_VERSION_05: case G_ELI_VERSION_06: case G_ELI_VERSION_07: error = eli_metadata_decode_v1v2v3v4v5v6v7(data, md); break; default: error = EOPNOTSUPP; break; } return (error); } #endif /* !_OpenSSL */ static __inline u_int g_eli_str2ealgo(const char *name) { if (strcasecmp("null", name) == 0) return (CRYPTO_NULL_CBC); else if (strcasecmp("null-cbc", name) == 0) return (CRYPTO_NULL_CBC); else if (strcasecmp("aes", name) == 0) return (CRYPTO_AES_XTS); else if (strcasecmp("aes-cbc", name) == 0) return (CRYPTO_AES_CBC); else if (strcasecmp("aes-xts", name) == 0) return (CRYPTO_AES_XTS); else if (strcasecmp("camellia", name) == 0) return (CRYPTO_CAMELLIA_CBC); else if (strcasecmp("camellia-cbc", name) == 0) return (CRYPTO_CAMELLIA_CBC); return (CRYPTO_ALGORITHM_MIN - 1); } static __inline u_int g_eli_str2aalgo(const char *name) { if (strcasecmp("hmac/sha1", name) == 0) return (CRYPTO_SHA1_HMAC); else if (strcasecmp("hmac/ripemd160", name) == 0) return (CRYPTO_RIPEMD160_HMAC); else if (strcasecmp("hmac/sha256", name) == 0) return (CRYPTO_SHA2_256_HMAC); else if (strcasecmp("hmac/sha384", name) == 0) return (CRYPTO_SHA2_384_HMAC); else if (strcasecmp("hmac/sha512", name) == 0) return (CRYPTO_SHA2_512_HMAC); return (CRYPTO_ALGORITHM_MIN - 1); } static __inline const char * g_eli_algo2str(u_int algo) { switch (algo) { case CRYPTO_NULL_CBC: return ("NULL"); case CRYPTO_AES_CBC: return ("AES-CBC"); case CRYPTO_AES_XTS: return ("AES-XTS"); case CRYPTO_CAMELLIA_CBC: return ("CAMELLIA-CBC"); case CRYPTO_SHA1_HMAC: return ("HMAC/SHA1"); case CRYPTO_RIPEMD160_HMAC: return ("HMAC/RIPEMD160"); case CRYPTO_SHA2_256_HMAC: return ("HMAC/SHA256"); case CRYPTO_SHA2_384_HMAC: return ("HMAC/SHA384"); case CRYPTO_SHA2_512_HMAC: return ("HMAC/SHA512"); } return ("unknown"); } static __inline void eli_metadata_dump(const struct g_eli_metadata *md) { static const char hex[] = "0123456789abcdef"; char str[sizeof(md->md_mkeys) * 2 + 1]; u_int i; printf(" magic: %s\n", md->md_magic); printf(" version: %u\n", (u_int)md->md_version); printf(" flags: 0x%x\n", (u_int)md->md_flags); printf(" ealgo: %s\n", g_eli_algo2str(md->md_ealgo)); printf(" keylen: %u\n", (u_int)md->md_keylen); if (md->md_flags & G_ELI_FLAG_AUTH) printf(" aalgo: %s\n", g_eli_algo2str(md->md_aalgo)); printf(" provsize: %ju\n", (uintmax_t)md->md_provsize); printf("sectorsize: %u\n", (u_int)md->md_sectorsize); printf(" keys: 0x%02x\n", (u_int)md->md_keys); printf("iterations: %d\n", (int)md->md_iterations); bzero(str, sizeof(str)); for (i = 0; i < sizeof(md->md_salt); i++) { str[i * 2] = hex[md->md_salt[i] >> 4]; str[i * 2 + 1] = hex[md->md_salt[i] & 0x0f]; } printf(" Salt: %s\n", str); bzero(str, sizeof(str)); for (i = 0; i < sizeof(md->md_mkeys); i++) { str[i * 2] = hex[md->md_mkeys[i] >> 4]; str[i * 2 + 1] = hex[md->md_mkeys[i] & 0x0f]; } printf("Master Key: %s\n", str); bzero(str, sizeof(str)); for (i = 0; i < 16; i++) { str[i * 2] = hex[md->md_hash[i] >> 4]; str[i * 2 + 1] = hex[md->md_hash[i] & 0x0f]; } printf(" MD5 hash: %s\n", str); } #ifdef _KERNEL static __inline bool eli_metadata_crypto_supported(const struct g_eli_metadata *md) { switch (md->md_ealgo) { case CRYPTO_NULL_CBC: case CRYPTO_AES_CBC: case CRYPTO_CAMELLIA_CBC: case CRYPTO_AES_XTS: break; default: return (false); } if (md->md_flags & G_ELI_FLAG_AUTH) { switch (md->md_aalgo) { case CRYPTO_SHA1_HMAC: case CRYPTO_RIPEMD160_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: break; default: return (false); } } return (true); } #endif static __inline u_int g_eli_keylen(u_int algo, u_int keylen) { switch (algo) { case CRYPTO_NULL_CBC: if (keylen == 0) keylen = 64 * 8; else { if (keylen > 64 * 8) keylen = 0; } return (keylen); case CRYPTO_AES_CBC: case CRYPTO_CAMELLIA_CBC: switch (keylen) { case 0: return (128); case 128: case 192: case 256: return (keylen); default: return (0); } case CRYPTO_AES_XTS: switch (keylen) { case 0: return (128); case 128: case 256: return (keylen); default: return (0); } default: return (0); } } static __inline u_int g_eli_ivlen(u_int algo) { switch (algo) { case CRYPTO_AES_XTS: return (AES_XTS_IV_LEN); case CRYPTO_AES_CBC: return (AES_BLOCK_LEN); case CRYPTO_CAMELLIA_CBC: return (CAMELLIA_BLOCK_LEN); } return (0); } static __inline u_int g_eli_hashlen(u_int algo) { switch (algo) { case CRYPTO_SHA1_HMAC: return (20); case CRYPTO_RIPEMD160_HMAC: return (20); case CRYPTO_SHA2_256_HMAC: return (32); case CRYPTO_SHA2_384_HMAC: return (48); case CRYPTO_SHA2_512_HMAC: return (64); } return (0); } static __inline off_t eli_mediasize(const struct g_eli_softc *sc, off_t mediasize, u_int sectorsize) { if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0) { mediasize -= sectorsize; } if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0) { mediasize -= (mediasize % sc->sc_sectorsize); } else { mediasize /= sc->sc_bytes_per_sector; mediasize *= sc->sc_sectorsize; } return (mediasize); } static __inline void eli_metadata_softc(struct g_eli_softc *sc, const struct g_eli_metadata *md, u_int sectorsize, off_t mediasize) { sc->sc_version = md->md_version; sc->sc_inflight = 0; sc->sc_crypto = G_ELI_CRYPTO_UNKNOWN; sc->sc_flags = md->md_flags; /* Backward compatibility. */ if (md->md_version < G_ELI_VERSION_04) sc->sc_flags |= G_ELI_FLAG_NATIVE_BYTE_ORDER; if (md->md_version < G_ELI_VERSION_05) sc->sc_flags |= G_ELI_FLAG_SINGLE_KEY; if (md->md_version < G_ELI_VERSION_06 && (sc->sc_flags & G_ELI_FLAG_AUTH) != 0) { sc->sc_flags |= G_ELI_FLAG_FIRST_KEY; } if (md->md_version < G_ELI_VERSION_07) sc->sc_flags |= G_ELI_FLAG_ENC_IVKEY; sc->sc_ealgo = md->md_ealgo; if (sc->sc_flags & G_ELI_FLAG_AUTH) { sc->sc_akeylen = sizeof(sc->sc_akey) * 8; sc->sc_aalgo = md->md_aalgo; sc->sc_alen = g_eli_hashlen(sc->sc_aalgo); sc->sc_data_per_sector = sectorsize - sc->sc_alen; /* * Some hash functions (like SHA1 and RIPEMD160) generates hash * which length is not multiple of 128 bits, but we want data * length to be multiple of 128, so we can encrypt without * padding. The line below rounds down data length to multiple * of 128 bits. */ sc->sc_data_per_sector -= sc->sc_data_per_sector % 16; sc->sc_bytes_per_sector = (md->md_sectorsize - 1) / sc->sc_data_per_sector + 1; sc->sc_bytes_per_sector *= sectorsize; } sc->sc_provsize = mediasize; sc->sc_sectorsize = md->md_sectorsize; sc->sc_mediasize = eli_mediasize(sc, mediasize, sectorsize); sc->sc_ekeylen = md->md_keylen; } #ifdef _KERNEL int g_eli_read_metadata(struct g_class *mp, struct g_provider *pp, struct g_eli_metadata *md); struct g_geom *g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp, const struct g_eli_metadata *md, const u_char *mkey, int nkey); int g_eli_destroy(struct g_eli_softc *sc, boolean_t force); int g_eli_access(struct g_provider *pp, int dr, int dw, int de); void g_eli_config(struct gctl_req *req, struct g_class *mp, const char *verb); void g_eli_read_done(struct bio *bp); void g_eli_write_done(struct bio *bp); int g_eli_crypto_rerun(struct cryptop *crp); +bool g_eli_alloc_data(struct bio *bp, int sz); +void g_eli_free_data(struct bio *bp); + void g_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker); void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp); void g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp); void g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp); #endif void g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv, size_t size); void g_eli_mkey_hmac(unsigned char *mkey, const unsigned char *key); int g_eli_mkey_decrypt(const struct g_eli_metadata *md, const unsigned char *key, unsigned char *mkey, unsigned keyp); int g_eli_mkey_decrypt_any(const struct g_eli_metadata *md, const unsigned char *key, unsigned char *mkey, unsigned *nkeyp); int g_eli_mkey_encrypt(unsigned algo, const unsigned char *key, unsigned keylen, unsigned char *mkey); #ifdef _KERNEL void g_eli_mkey_propagate(struct g_eli_softc *sc, const unsigned char *mkey); #endif int g_eli_crypto_encrypt(u_int algo, u_char *data, size_t datasize, const u_char *key, size_t keysize); int g_eli_crypto_decrypt(u_int algo, u_char *data, size_t datasize, const u_char *key, size_t keysize); struct hmac_ctx { SHA512_CTX innerctx; SHA512_CTX outerctx; }; void g_eli_crypto_hmac_init(struct hmac_ctx *ctx, const char *hkey, size_t hkeylen); void g_eli_crypto_hmac_update(struct hmac_ctx *ctx, const uint8_t *data, size_t datasize); void g_eli_crypto_hmac_final(struct hmac_ctx *ctx, uint8_t *md, size_t mdsize); void g_eli_crypto_hmac(const char *hkey, size_t hkeysize, const uint8_t *data, size_t datasize, uint8_t *md, size_t mdsize); void g_eli_key_fill(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno); #ifdef _KERNEL void g_eli_key_init(struct g_eli_softc *sc); void g_eli_key_destroy(struct g_eli_softc *sc); void g_eli_key_resize(struct g_eli_softc *sc); uint8_t *g_eli_key_hold(struct g_eli_softc *sc, off_t offset, size_t blocksize); void g_eli_key_drop(struct g_eli_softc *sc, uint8_t *rawkey); #endif #endif /* !_G_ELI_H_ */ diff --git a/sys/geom/eli/g_eli_integrity.c b/sys/geom/eli/g_eli_integrity.c index e97924b8df08..17a0912797c1 100644 --- a/sys/geom/eli/g_eli_integrity.c +++ b/sys/geom/eli/g_eli_integrity.c @@ -1,577 +1,590 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2005-2011 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include -#include #include #include #include #include #include #include #include #include #include #include /* * The data layout description when integrity verification is configured. * * One of the most important assumption here is that authenticated data and its * HMAC has to be stored in the same place (namely in the same sector) to make * it work reliable. * The problem is that file systems work only with sectors that are multiple of * 512 bytes and a power of two number. * My idea to implement it is as follows. * Let's store HMAC in sector. This is a must. This leaves us 480 bytes for * data. We can't use that directly (ie. we can't create provider with 480 bytes * sector size). We need another sector from where we take only 32 bytes of data * and we store HMAC of this data as well. This takes two sectors from the * original provider at the input and leaves us one sector of authenticated data * at the output. Not very efficient, but you got the idea. * Now, let's assume, we want to create provider with 4096 bytes sector. * To output 4096 bytes of authenticated data we need 8x480 plus 1x256, so we * need nine 512-bytes sectors at the input to get one 4096-bytes sector at the * output. That's better. With 4096 bytes sector we can use 89% of size of the * original provider. I find it as an acceptable cost. * The reliability comes from the fact, that every HMAC stored inside the sector * is calculated only for the data in the same sector, so its impossible to * write new data and leave old HMAC or vice versa. * * And here is the picture: * * da0: +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+ * |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |256b | * |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data | * +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+ * |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |288 bytes | * +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ |224 unused| * +----------+ * da0.eli: +----+----+----+----+----+----+----+----+----+ * |480b|480b|480b|480b|480b|480b|480b|480b|256b| * +----+----+----+----+----+----+----+----+----+ * | 4096 bytes | * +--------------------------------------------+ * * PS. You can use any sector size with geli(8). My example is using 4kB, * because it's most efficient. For 8kB sectors you need 2 extra sectors, * so the cost is the same as for 4kB sectors. */ /* * Code paths: * BIO_READ: * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> g_eli_auth_read_done -> g_io_deliver * BIO_WRITE: * g_eli_start -> g_eli_auth_run -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver */ -MALLOC_DECLARE(M_ELI); - /* * Here we generate key for HMAC. Every sector has its own HMAC key, so it is * not possible to copy sectors. * We cannot depend on fact, that every sector has its own IV, because different * IV doesn't change HMAC, when we use encrypt-then-authenticate method. */ static void g_eli_auth_keygen(struct g_eli_softc *sc, off_t offset, u_char *key) { SHA256_CTX ctx; /* Copy precalculated SHA256 context. */ bcopy(&sc->sc_akeyctx, &ctx, sizeof(ctx)); SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset)); SHA256_Final(key, &ctx); } /* * The function is called after we read and decrypt data. * * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> G_ELI_AUTH_READ_DONE -> g_io_deliver */ static int g_eli_auth_read_done(struct cryptop *crp) { struct g_eli_softc *sc; struct bio *bp; if (crp->crp_etype == EAGAIN) { if (g_eli_crypto_rerun(crp) == 0) return (0); } bp = (struct bio *)crp->crp_opaque; bp->bio_inbed++; sc = bp->bio_to->geom->softc; if (crp->crp_etype == 0) { bp->bio_completed += crp->crp_payload_length; G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%d completed=%jd).", bp->bio_inbed, bp->bio_children, crp->crp_payload_length, (intmax_t)bp->bio_completed); } else { u_int nsec, decr_secsize, encr_secsize, rel_sec; int *errorp; /* Sectorsize of decrypted provider eg. 4096. */ decr_secsize = bp->bio_to->sectorsize; /* The real sectorsize of encrypted provider, eg. 512. */ encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; /* Number of sectors from decrypted provider, eg. 2. */ nsec = bp->bio_length / decr_secsize; /* Number of sectors from encrypted provider, eg. 18. */ nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; /* Which relative sector this request decrypted. */ rel_sec = ((crp->crp_buf.cb_buf + crp->crp_payload_start) - (char *)bp->bio_driver2) / encr_secsize; errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec + sizeof(int) * rel_sec); *errorp = crp->crp_etype; G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.", bp->bio_inbed, bp->bio_children, crp->crp_etype); if (bp->bio_error == 0 || bp->bio_error == EINTEGRITY) bp->bio_error = crp->crp_etype == EBADMSG ? EINTEGRITY : crp->crp_etype; } if (crp->crp_cipher_key != NULL) g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key)); crypto_freereq(crp); /* * Do we have all sectors already? */ if (bp->bio_inbed < bp->bio_children) return (0); if (bp->bio_error == 0) { u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; u_char *srcdata, *dstdata; /* Sectorsize of decrypted provider eg. 4096. */ decr_secsize = bp->bio_to->sectorsize; /* The real sectorsize of encrypted provider, eg. 512. */ encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; /* Number of data bytes in one encrypted sector, eg. 480. */ data_secsize = sc->sc_data_per_sector; /* Number of sectors from decrypted provider, eg. 2. */ nsec = bp->bio_length / decr_secsize; /* Number of sectors from encrypted provider, eg. 18. */ nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; /* Last sector number in every big sector, eg. 9. */ lsec = sc->sc_bytes_per_sector / encr_secsize; srcdata = bp->bio_driver2; dstdata = bp->bio_data; for (i = 1; i <= nsec; i++) { data_secsize = sc->sc_data_per_sector; if ((i % lsec) == 0) data_secsize = decr_secsize % data_secsize; bcopy(srcdata + sc->sc_alen, dstdata, data_secsize); srcdata += encr_secsize; dstdata += data_secsize; } } else if (bp->bio_error == EINTEGRITY) { u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; int *errorp; off_t coroff, corsize, dstoff; /* Sectorsize of decrypted provider eg. 4096. */ decr_secsize = bp->bio_to->sectorsize; /* The real sectorsize of encrypted provider, eg. 512. */ encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; /* Number of data bytes in one encrypted sector, eg. 480. */ data_secsize = sc->sc_data_per_sector; /* Number of sectors from decrypted provider, eg. 2. */ nsec = bp->bio_length / decr_secsize; /* Number of sectors from encrypted provider, eg. 18. */ nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; /* Last sector number in every big sector, eg. 9. */ lsec = sc->sc_bytes_per_sector / encr_secsize; errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec); coroff = -1; corsize = 0; dstoff = bp->bio_offset; for (i = 1; i <= nsec; i++) { data_secsize = sc->sc_data_per_sector; if ((i % lsec) == 0) data_secsize = decr_secsize % data_secsize; if (errorp[i - 1] == EBADMSG) { /* * Corruption detected, remember the offset if * this is the first corrupted sector and * increase size. */ if (coroff == -1) coroff = dstoff; corsize += data_secsize; } else { /* * No corruption, good. * Report previous corruption if there was one. */ if (coroff != -1) { G_ELI_DEBUG(0, "%s: Failed to authenticate %jd " "bytes of data at offset %jd.", sc->sc_name, (intmax_t)corsize, (intmax_t)coroff); coroff = -1; corsize = 0; } } dstoff += data_secsize; } /* Report previous corruption if there was one. */ if (coroff != -1) { G_ELI_DEBUG(0, "%s: Failed to authenticate %jd " "bytes of data at offset %jd.", sc->sc_name, (intmax_t)corsize, (intmax_t)coroff); } } - free(bp->bio_driver2, M_ELI); - bp->bio_driver2 = NULL; + g_eli_free_data(bp); if (bp->bio_error != 0) { if (bp->bio_error != EINTEGRITY) { G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).", bp->bio_error); } bp->bio_completed = 0; } /* * Read is finished, send it up. */ g_io_deliver(bp, bp->bio_error); atomic_subtract_int(&sc->sc_inflight, 1); return (0); } /* * The function is called after data encryption. * * g_eli_start -> g_eli_auth_run -> G_ELI_AUTH_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver */ static int g_eli_auth_write_done(struct cryptop *crp) { struct g_eli_softc *sc; struct g_consumer *cp; struct bio *bp, *cbp, *cbp2; u_int nsec; if (crp->crp_etype == EAGAIN) { if (g_eli_crypto_rerun(crp) == 0) return (0); } bp = (struct bio *)crp->crp_opaque; bp->bio_inbed++; if (crp->crp_etype == 0) { G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).", bp->bio_inbed, bp->bio_children); } else { G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.", bp->bio_inbed, bp->bio_children, crp->crp_etype); if (bp->bio_error == 0) bp->bio_error = crp->crp_etype; } sc = bp->bio_to->geom->softc; if (crp->crp_cipher_key != NULL) g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key)); crypto_freereq(crp); /* * All sectors are already encrypted? */ if (bp->bio_inbed < bp->bio_children) return (0); if (bp->bio_error != 0) { G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).", bp->bio_error); - free(bp->bio_driver2, M_ELI); - bp->bio_driver2 = NULL; + g_eli_free_data(bp); cbp = bp->bio_driver1; bp->bio_driver1 = NULL; g_destroy_bio(cbp); g_io_deliver(bp, bp->bio_error); atomic_subtract_int(&sc->sc_inflight, 1); return (0); } cp = LIST_FIRST(&sc->sc_geom->consumer); cbp = bp->bio_driver1; bp->bio_driver1 = NULL; cbp->bio_to = cp->provider; cbp->bio_done = g_eli_write_done; /* Number of sectors from decrypted provider, eg. 1. */ nsec = bp->bio_length / bp->bio_to->sectorsize; /* Number of sectors from encrypted provider, eg. 9. */ nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize; cbp->bio_length = cp->provider->sectorsize * nsec; cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector; cbp->bio_data = bp->bio_driver2; /* * We write more than what is requested, so we have to be ready to write * more than maxphys. */ cbp2 = NULL; if (cbp->bio_length > maxphys) { cbp2 = g_duplicate_bio(bp); cbp2->bio_length = cbp->bio_length - maxphys; cbp2->bio_data = cbp->bio_data + maxphys; cbp2->bio_offset = cbp->bio_offset + maxphys; cbp2->bio_to = cp->provider; cbp2->bio_done = g_eli_write_done; cbp->bio_length = maxphys; } /* * Send encrypted data to the provider. */ G_ELI_LOGREQ(2, cbp, "Sending request."); bp->bio_inbed = 0; bp->bio_children = (cbp2 != NULL ? 2 : 1); g_io_request(cbp, cp); if (cbp2 != NULL) { G_ELI_LOGREQ(2, cbp2, "Sending request."); g_io_request(cbp2, cp); } return (0); } void g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp) { struct g_consumer *cp; struct bio *cbp, *cbp2; size_t size; off_t nsec; - bp->bio_pflags = 0; + G_ELI_SETWORKER(bp->bio_pflags, 0); cp = LIST_FIRST(&sc->sc_geom->consumer); cbp = bp->bio_driver1; bp->bio_driver1 = NULL; cbp->bio_to = cp->provider; cbp->bio_done = g_eli_read_done; /* Number of sectors from decrypted provider, eg. 1. */ nsec = bp->bio_length / bp->bio_to->sectorsize; /* Number of sectors from encrypted provider, eg. 9. */ nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize; cbp->bio_length = cp->provider->sectorsize * nsec; size = cbp->bio_length; size += sizeof(int) * nsec; size += G_ELI_AUTH_SECKEYLEN * nsec; cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector; - bp->bio_driver2 = malloc(size, M_ELI, M_WAITOK); + if (!g_eli_alloc_data(bp, size)) { + G_ELI_LOGREQ(0, bp, "Crypto auth read request failed (ENOMEM)"); + g_destroy_bio(cbp); + bp->bio_error = ENOMEM; + g_io_deliver(bp, bp->bio_error); + atomic_subtract_int(&sc->sc_inflight, 1); + return; + } cbp->bio_data = bp->bio_driver2; /* Clear the error array. */ memset((char *)bp->bio_driver2 + cbp->bio_length, 0, sizeof(int) * nsec); /* * We read more than what is requested, so we have to be ready to read * more than maxphys. */ cbp2 = NULL; if (cbp->bio_length > maxphys) { cbp2 = g_duplicate_bio(bp); cbp2->bio_length = cbp->bio_length - maxphys; cbp2->bio_data = cbp->bio_data + maxphys; cbp2->bio_offset = cbp->bio_offset + maxphys; cbp2->bio_to = cp->provider; cbp2->bio_done = g_eli_read_done; cbp->bio_length = maxphys; } /* * Read encrypted data from provider. */ G_ELI_LOGREQ(2, cbp, "Sending request."); g_io_request(cbp, cp); if (cbp2 != NULL) { G_ELI_LOGREQ(2, cbp2, "Sending request."); g_io_request(cbp2, cp); } } /* * This is the main function responsible for cryptography (ie. communication * with crypto(9) subsystem). * * BIO_READ: * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> G_ELI_AUTH_RUN -> g_eli_auth_read_done -> g_io_deliver * BIO_WRITE: * g_eli_start -> G_ELI_AUTH_RUN -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver */ void g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp) { struct g_eli_softc *sc; struct cryptopq crpq; struct cryptop *crp; u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; off_t dstoff; u_char *p, *data, *authkey, *plaindata; int error; bool batch; G_ELI_LOGREQ(3, bp, "%s", __func__); - bp->bio_pflags = wr->w_number; + G_ELI_SETWORKER(bp->bio_pflags, wr->w_number); sc = wr->w_softc; /* Sectorsize of decrypted provider eg. 4096. */ decr_secsize = bp->bio_to->sectorsize; /* The real sectorsize of encrypted provider, eg. 512. */ encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; /* Number of data bytes in one encrypted sector, eg. 480. */ data_secsize = sc->sc_data_per_sector; /* Number of sectors from decrypted provider, eg. 2. */ nsec = bp->bio_length / decr_secsize; /* Number of sectors from encrypted provider, eg. 18. */ nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; /* Last sector number in every big sector, eg. 9. */ lsec = sc->sc_bytes_per_sector / encr_secsize; /* Destination offset, used for IV generation. */ dstoff = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector; plaindata = bp->bio_data; if (bp->bio_cmd == BIO_READ) { data = bp->bio_driver2; p = data + encr_secsize * nsec; p += sizeof(int) * nsec; } else { size_t size; size = encr_secsize * nsec; size += G_ELI_AUTH_SECKEYLEN * nsec; size += sizeof(uintptr_t); /* Space for alignment. */ - data = malloc(size, M_ELI, M_WAITOK); - bp->bio_driver2 = data; + if (!g_eli_alloc_data(bp, size)) { + G_ELI_LOGREQ(0, bp, "Crypto request failed (ENOMEM)"); + if (bp->bio_driver1 != NULL) { + g_destroy_bio(bp->bio_driver1); + bp->bio_driver1 = NULL; + } + bp->bio_error = ENOMEM; + g_io_deliver(bp, bp->bio_error); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); + return; + } + data = bp->bio_driver2; p = data + encr_secsize * nsec; } bp->bio_inbed = 0; bp->bio_children = nsec; #if defined(__mips_n64) || defined(__mips_o64) p = (char *)roundup((uintptr_t)p, sizeof(uintptr_t)); #endif TAILQ_INIT(&crpq); batch = atomic_load_int(&g_eli_batch) != 0; for (i = 1; i <= nsec; i++, dstoff += encr_secsize) { crp = crypto_getreq(wr->w_sid, M_WAITOK); authkey = (u_char *)p; p += G_ELI_AUTH_SECKEYLEN; data_secsize = sc->sc_data_per_sector; if ((i % lsec) == 0) { data_secsize = decr_secsize % data_secsize; /* * Last encrypted sector of each decrypted sector is * only partially filled. */ if (bp->bio_cmd == BIO_WRITE) memset(data + sc->sc_alen + data_secsize, 0, encr_secsize - sc->sc_alen - data_secsize); } else if (data_secsize + sc->sc_alen != encr_secsize) { /* * If the HMAC size is not a multiple of 128 bits, the * per-sector data size is rounded down to ensure that * encryption can be performed without requiring any * padding. In this case, each sector contains unused * bytes. */ if (bp->bio_cmd == BIO_WRITE) memset(data + sc->sc_alen + data_secsize, 0, encr_secsize - sc->sc_alen - data_secsize); } if (bp->bio_cmd == BIO_WRITE) { bcopy(plaindata, data + sc->sc_alen, data_secsize); plaindata += data_secsize; } crypto_use_buf(crp, data, sc->sc_alen + data_secsize); crp->crp_opaque = (void *)bp; data += encr_secsize; crp->crp_flags = CRYPTO_F_CBIFSYNC; if (bp->bio_cmd == BIO_WRITE) { crp->crp_callback = g_eli_auth_write_done; crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST; } else { crp->crp_callback = g_eli_auth_read_done; crp->crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST; } crp->crp_digest_start = 0; crp->crp_payload_start = sc->sc_alen; crp->crp_payload_length = data_secsize; if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0) { crp->crp_cipher_key = g_eli_key_hold(sc, dstoff, encr_secsize); } if (g_eli_ivlen(sc->sc_ealgo) != 0) { crp->crp_flags |= CRYPTO_F_IV_SEPARATE; g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv, sizeof(crp->crp_iv)); } g_eli_auth_keygen(sc, dstoff, authkey); crp->crp_auth_key = authkey; if (batch) { TAILQ_INSERT_TAIL(&crpq, crp, crp_next); } else { error = crypto_dispatch(crp); KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)", error)); } } if (batch) crypto_dispatch_batch(&crpq, 0); } diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c index f4e0416cc828..20a9f09452c5 100644 --- a/sys/geom/eli/g_eli_privacy.c +++ b/sys/geom/eli/g_eli_privacy.c @@ -1,355 +1,362 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2005-2011 Pawel Jakub Dawidek * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include -#include #include #include #include #include #include #include #include #include #include #include /* * Code paths: * BIO_READ: * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver * BIO_WRITE: * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver */ -MALLOC_DECLARE(M_ELI); - /* * Copy data from a (potentially unmapped) bio to a kernelspace buffer. * * The buffer must have at least as much room as bp->bio_length. */ static void g_eli_bio_copyin(struct bio *bp, void *kaddr) { struct uio uio; struct iovec iov[1]; iov[0].iov_base = kaddr; iov[0].iov_len = bp->bio_length; uio.uio_iov = iov; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_resid = bp->bio_length; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_READ; uiomove_fromphys(bp->bio_ma, bp->bio_ma_offset, bp->bio_length, &uio); } /* * The function is called after we read and decrypt data. * * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver */ static int g_eli_crypto_read_done(struct cryptop *crp) { struct g_eli_softc *sc; struct bio *bp; if (crp->crp_etype == EAGAIN) { if (g_eli_crypto_rerun(crp) == 0) return (0); } bp = (struct bio *)crp->crp_opaque; bp->bio_inbed++; if (crp->crp_etype == 0) { G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).", bp->bio_inbed, bp->bio_children); bp->bio_completed += crp->crp_payload_length; } else { G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.", bp->bio_inbed, bp->bio_children, crp->crp_etype); if (bp->bio_error == 0) bp->bio_error = crp->crp_etype; } sc = bp->bio_to->geom->softc; if (sc != NULL && crp->crp_cipher_key != NULL) g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key)); crypto_freereq(crp); /* * Do we have all sectors already? */ if (bp->bio_inbed < bp->bio_children) return (0); if (bp->bio_error != 0) { G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).", bp->bio_error); bp->bio_completed = 0; } /* * Read is finished, send it up. */ g_io_deliver(bp, bp->bio_error); if (sc != NULL) atomic_subtract_int(&sc->sc_inflight, 1); return (0); } /* * The function is called after data encryption. * * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver */ static int g_eli_crypto_write_done(struct cryptop *crp) { struct g_eli_softc *sc; struct g_geom *gp; struct g_consumer *cp; struct bio *bp, *cbp; if (crp->crp_etype == EAGAIN) { if (g_eli_crypto_rerun(crp) == 0) return (0); } bp = (struct bio *)crp->crp_opaque; bp->bio_inbed++; if (crp->crp_etype == 0) { G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).", bp->bio_inbed, bp->bio_children); } else { G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.", bp->bio_inbed, bp->bio_children, crp->crp_etype); if (bp->bio_error == 0) bp->bio_error = crp->crp_etype; } gp = bp->bio_to->geom; sc = gp->softc; if (crp->crp_cipher_key != NULL) g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key)); crypto_freereq(crp); /* * All sectors are already encrypted? */ if (bp->bio_inbed < bp->bio_children) return (0); bp->bio_inbed = 0; bp->bio_children = 1; cbp = bp->bio_driver1; bp->bio_driver1 = NULL; if (bp->bio_error != 0) { G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).", bp->bio_error); - free(bp->bio_driver2, M_ELI); - bp->bio_driver2 = NULL; + g_eli_free_data(bp); g_destroy_bio(cbp); g_io_deliver(bp, bp->bio_error); atomic_subtract_int(&sc->sc_inflight, 1); return (0); } cbp->bio_data = bp->bio_driver2; /* * Clear BIO_UNMAPPED, which was inherited from where we cloned the bio * in g_eli_start, because we manually set bio_data */ cbp->bio_flags &= ~BIO_UNMAPPED; cbp->bio_done = g_eli_write_done; cp = LIST_FIRST(&gp->consumer); cbp->bio_to = cp->provider; G_ELI_LOGREQ(2, cbp, "Sending request."); /* * Send encrypted data to the provider. */ g_io_request(cbp, cp); return (0); } /* * The function is called to read encrypted data. * * g_eli_start -> G_ELI_CRYPTO_READ -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver */ void g_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker) { struct g_consumer *cp; struct bio *cbp; if (!fromworker) { /* * We are not called from the worker thread, so check if * device is suspended. */ mtx_lock(&sc->sc_queue_mtx); if (sc->sc_flags & G_ELI_FLAG_SUSPEND) { /* * If device is suspended, we place the request onto * the queue, so it can be handled after resume. */ G_ELI_DEBUG(0, "device suspended, move onto queue"); bioq_insert_tail(&sc->sc_queue, bp); mtx_unlock(&sc->sc_queue_mtx); wakeup(sc); return; } atomic_add_int(&sc->sc_inflight, 1); mtx_unlock(&sc->sc_queue_mtx); } - bp->bio_pflags = 0; + G_ELI_SETWORKER(bp->bio_pflags, 0); bp->bio_driver2 = NULL; cbp = bp->bio_driver1; cbp->bio_done = g_eli_read_done; cp = LIST_FIRST(&sc->sc_geom->consumer); cbp->bio_to = cp->provider; G_ELI_LOGREQ(2, cbp, "Sending request."); /* * Read encrypted data from provider. */ g_io_request(cbp, cp); } /* * This is the main function responsible for cryptography (ie. communication * with crypto(9) subsystem). * * BIO_READ: * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> G_ELI_CRYPTO_RUN -> g_eli_crypto_read_done -> g_io_deliver * BIO_WRITE: * g_eli_start -> G_ELI_CRYPTO_RUN -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver */ void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp) { struct g_eli_softc *sc; struct cryptopq crpq; struct cryptop *crp; vm_page_t *pages; u_int i, nsec, secsize; off_t dstoff; u_char *data = NULL; int error, pages_offset; bool batch; G_ELI_LOGREQ(3, bp, "%s", __func__); - bp->bio_pflags = wr->w_number; + G_ELI_SETWORKER(bp->bio_pflags, wr->w_number); sc = wr->w_softc; secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize; nsec = bp->bio_length / secsize; bp->bio_inbed = 0; bp->bio_children = nsec; /* * If we write the data we cannot destroy current bio_data content, * so we need to allocate more memory for encrypted data. */ if (bp->bio_cmd == BIO_WRITE) { - data = malloc(bp->bio_length, M_ELI, M_WAITOK); - bp->bio_driver2 = data; + if (!g_eli_alloc_data(bp, bp->bio_length)) { + G_ELI_LOGREQ(0, bp, "Crypto request failed (ENOMEM)."); + if (bp->bio_driver1 != NULL) { + g_destroy_bio(bp->bio_driver1); + bp->bio_driver1 = NULL; + } + bp->bio_error = ENOMEM; + g_io_deliver(bp, bp->bio_error); + if (sc != NULL) + atomic_subtract_int(&sc->sc_inflight, 1); + return; + } + data = bp->bio_driver2; /* * This copy could be eliminated by using crypto's output * buffer, instead of using a single overwriting buffer. */ if ((bp->bio_flags & BIO_UNMAPPED) != 0) g_eli_bio_copyin(bp, data); else bcopy(bp->bio_data, data, bp->bio_length); } else { if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pages = bp->bio_ma; pages_offset = bp->bio_ma_offset; } else { data = bp->bio_data; } } TAILQ_INIT(&crpq); batch = atomic_load_int(&g_eli_batch) != 0; for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) { crp = crypto_getreq(wr->w_sid, M_WAITOK); if (data) { crypto_use_buf(crp, data, secsize); data += secsize; } else { MPASS(pages != NULL); crypto_use_vmpage(crp, pages, secsize, pages_offset); pages_offset += secsize; pages += pages_offset >> PAGE_SHIFT; pages_offset &= PAGE_MASK; } crp->crp_opaque = (void *)bp; if (bp->bio_cmd == BIO_WRITE) { crp->crp_op = CRYPTO_OP_ENCRYPT; crp->crp_callback = g_eli_crypto_write_done; } else /* if (bp->bio_cmd == BIO_READ) */ { crp->crp_op = CRYPTO_OP_DECRYPT; crp->crp_callback = g_eli_crypto_read_done; } crp->crp_flags = CRYPTO_F_CBIFSYNC; crp->crp_payload_start = 0; crp->crp_payload_length = secsize; if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0) { crp->crp_cipher_key = g_eli_key_hold(sc, dstoff, secsize); } if (g_eli_ivlen(sc->sc_ealgo) != 0) { crp->crp_flags |= CRYPTO_F_IV_SEPARATE; g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv, sizeof(crp->crp_iv)); } if (batch) { TAILQ_INSERT_TAIL(&crpq, crp, crp_next); } else { error = crypto_dispatch(crp); KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)", error)); } } if (batch) crypto_dispatch_batch(&crpq, 0); }