diff --git a/sys/geom/geom_vfs.c b/sys/geom/geom_vfs.c --- a/sys/geom/geom_vfs.c +++ b/sys/geom/geom_vfs.c @@ -36,10 +36,12 @@ #include #include #include +#include #include #include #include +#include #include /* @@ -57,8 +59,30 @@ bool sc_orphaned; int sc_enxio_active; int sc_enxio_reported; + long sc_async_writes; + long sc_async_write_limit; }; +SYSCTL_DECL(_kern_geom); + +static long g_vfs_hdd_async_limit = 8; +SYSCTL_LONG(_kern_geom, OID_AUTO, hdd_async_limit, CTLFLAG_RWTUN, + &g_vfs_hdd_async_limit, 0, + "Number of async I/Os allowed concurrently per top-level geom consumer on HDD."); + +static long g_vfs_ssd_async_limit = 8; +SYSCTL_LONG(_kern_geom, OID_AUTO, ssd_async_limit, CTLFLAG_RWTUN, + &g_vfs_ssd_async_limit, 0, + "Number of async I/Os allowed concurrently per top-level geom consumer on SSD."); + +static long g_vfs_writes_throttled = 0; +SYSCTL_LONG(_kern_geom, OID_AUTO, writes_throttled, CTLFLAG_RD, + &g_vfs_writes_throttled, 0, + "Number of async I/Os converted to sync."); + +static void g_vfs_wait_wakeup(bool *); +static void g_vfs_wait_block(bool *); + static struct buf_ops __g_vfs_bufops = { .bop_name = "GEOM_VFS", .bop_write = bufwrite, @@ -166,9 +190,10 @@ else bp->b_error = bip->bio_error; bp->b_resid = bp->b_bcount - bip->bio_completed; - g_destroy_bio(bip); mtx_lock(&sc->sc_mtx); + if (bp->b_iocmd == BIO_WRITE && (bp->b_flags & B_ASYNC) != 0) + sc->sc_async_writes--; destroy = ((--sc->sc_active) == 0 && sc->sc_orphaned); if (destroy) { event = sc->sc_event; @@ -176,18 +201,54 @@ } else event = NULL; mtx_unlock(&sc->sc_mtx); + + if (bip->bio_caller1 != NULL) + g_vfs_wait_wakeup(bip->bio_caller1); + g_destroy_bio(bip); + if (destroy) g_post_event_ep(g_vfs_destroy, cp, event, NULL); bufdone(bp); } +/* + * Rendezvous used to throttle async writes in g_vfs_strategy(): a bool on the + * throttled thread's own stack, used as a wait channel, to avoid racing bp + * destruction. g_vfs_wait_block can't timeout w/o creating a hazard. + */ +static void +g_vfs_wait_wakeup(bool *donep) +{ + struct mtx *mtxp; + + mtxp = mtx_pool_find(mtxpool_sleep, donep); + mtx_lock(mtxp); + *donep = true; + wakeup(donep); + mtx_unlock(mtxp); +} + +static void +g_vfs_wait_block(bool *donep) +{ + struct mtx *mtxp; + + mtxp = mtx_pool_find(mtxpool_sleep, donep); + mtx_lock(mtxp); + while (!*donep) + msleep(donep, mtxp, PRIBIO, "gvfsrun", 0); + mtx_unlock(mtxp); +} + void g_vfs_strategy(struct bufobj *bo, struct buf *bp) { struct g_vfs_softc *sc; struct g_consumer *cp; struct bio *bip; + bool done; + bool throttle; cp = bo->bo_private; sc = cp->geom->softc; @@ -206,6 +267,11 @@ return; } sc->sc_active++; + throttle = false; + if (bp->b_iocmd == BIO_WRITE && (bp->b_flags & B_ASYNC) != 0) { + sc->sc_async_writes++; + throttle = sc->sc_async_writes > sc->sc_async_write_limit; + } mtx_unlock(&sc->sc_mtx); bip = g_alloc_bio(); @@ -221,11 +287,20 @@ bip->bio_flags |= bp->b_ioflags; bip->bio_done = g_vfs_done; bip->bio_caller2 = bp; + if (throttle) { + done = false; + bip->bio_caller1 = &done; + g_vfs_writes_throttled++; + } else { + bip->bio_caller1 = NULL; + } #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING) buf_track(bp, __func__); bip->bio_track_bp = bp; #endif g_io_request(bip, cp); + if (throttle) + g_vfs_wait_block(&done); } static void @@ -271,6 +346,8 @@ struct g_consumer *cp; struct g_vfs_softc *sc; struct bufobj *bo; + uint16_t rate; + long async_limit; int error; g_topology_assert(); @@ -308,6 +385,16 @@ g_wither_geom(gp, ENXIO); return (ENXIO); } + /* + * Throttle based on SSD vs HDD, assume anything that's + * not non-rotating is a HDD. + */ + if (g_getattr("GEOM::rotation_rate", cp, &rate) == 0 && + rate == DISK_RR_NON_ROTATING) + async_limit = g_vfs_hdd_async_limit; + else + async_limit = g_vfs_ssd_async_limit; + sc->sc_async_write_limit = async_limit; vnode_create_disk_vobject(vp, pp->mediasize, curthread); *cpp = cp; cp->private = vp;