Index: sys/cam/cam_iosched.c =================================================================== --- sys/cam/cam_iosched.c +++ sys/cam/cam_iosched.c @@ -1531,6 +1531,41 @@ cam_iosched_queue_work(struct cam_iosched_softc *isc, struct bio *bp) { + /* + * A BIO_SPEEDUP from the uppper layers means that they have a block + * shortage. At the present, this is only sent when we're trying to + * allocate blocks, but have a shortage before giving up. bio_length is + * the size of their shortage. We will complete just enough BIO_DELETEs + * in the queue to satisfy the need. If bio_length is 0, we'll complete + * them all. This allows the scheduler to delay BIO_DELETEs to improve + * read/write performance without worrying about the upper layers. When + * it's possibly a problem, we respond by pretending the BIO_DELETEs + * just worked. We can't do anything about the BIO_DELETEs in the + * hardware, though. We have to wait for them to complete. + */ + if (bp->bio_cmd == BIO_SPEEDUP) { + off_t len; + struct bio *nbp; + + len = 0; + while (bioq_first(&isc->trim_queue) && + (bp->bio_length == 0 || len < bp->bio_length)) { + nbp = bioq_takefirst(&isc->trim_queue); + len += nbp->bio_length; + nbp->bio_error = 0; + biodone(nbp); + } + if (bp->bio_length > 0) { + if (bp->bio_length > len) + bp->bio_resid = bp->bio_length - len; + else + bp->bio_resid = 0; + } + bp->bio_error = 0; + biodone(bp); + return; + } + /* * If we get a BIO_FLUSH, and we're doing delayed BIO_DELETEs then we * set the last tick time to one less than the current ticks minus the @@ -1917,8 +1952,8 @@ db_printf("Trim Q len %d\n", biolen(&isc->trim_queue)); db_printf("read_bias: %d\n", isc->read_bias); db_printf("current_read_bias: %d\n", isc->current_read_bias); - db_printf("Trims active %d\n", isc->pend_trim); - db_printf("Max trims active %d\n", isc->max_trim); + db_printf("Trims active %d\n", isc->pend_trims); + db_printf("Max trims active %d\n", isc->max_trims); } #endif #endif Index: sys/geom/geom.h =================================================================== --- sys/geom/geom.h +++ sys/geom/geom.h @@ -336,6 +336,7 @@ int g_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr); int g_io_zonecmd(struct disk_zone_args *zone_args, struct g_consumer *cp); int g_io_flush(struct g_consumer *cp); +int g_io_speedup(size_t shortage, u_int flags, size_t *resid, struct g_consumer *cp); int g_register_classifier(struct g_classifier_hook *hook); void g_unregister_classifier(struct g_classifier_hook *hook); void g_io_request(struct bio *bp, struct g_consumer *cp); Index: sys/geom/geom_io.c =================================================================== --- sys/geom/geom_io.c +++ sys/geom/geom_io.c @@ -338,6 +338,42 @@ return (error); } +/* + * Send a IBO_SPEEDUP down the stack. This is used to tell the lower layers that + * the upper layers has detected a resource shortage. The lower layers are + * advised to stop delaying I/O that they might be holding for performance + * reasons and to schedule it (non-trims) or complete it successfully (trims) as + * quickly as it can. bio_length is the amount of the shortage. This call + * should be non-blocking. bio_resid is used to communicate back if the lower + * layers couldn't find bio_length worth of I/O to schedule or discard. A length + * of 0 means to do as much as you can (schedule the h/w queues full, discard + * all trims). flags are a hint from the upper layers to the lower layers what + * operation should be done. + */ +int +g_io_speedup(size_t shortage, u_int flags, size_t *resid, struct g_consumer *cp) +{ + struct bio *bp; + int error; + + KASSERT((flags & (BIO_SPEEDUP_TRIM | BIO_SPEEDUP_WRITE)) != 0, + ("Invalid flags passed to g_io_speedup: %#x", flags)); + g_trace(G_T_BIO, "bio_speedup(%s, %zu, %#x)", cp->provider->name, + shortage, flags); + bp = g_new_bio(); + if (bp == NULL) + return (ENOMEM); + bp->bio_cmd = BIO_SPEEDUP; + bp->bio_length = shortage; + bp->bio_done = NULL; + bp->bio_flags |= flags; + g_io_request(bp, cp); + error = biowait(bp, "gflush"); + *resid = bp->bio_resid; + g_destroy_bio(bp); + return (error); +} + int g_io_flush(struct g_consumer *cp) { Index: sys/sys/bio.h =================================================================== --- sys/sys/bio.h +++ sys/sys/bio.h @@ -53,6 +53,7 @@ #define BIO_CMD1 0x07 /* Available for local hacks */ #define BIO_CMD2 0x08 /* Available for local hacks */ #define BIO_ZONE 0x09 /* Zone command */ +#define BIO_SPEEDUP 0x0a /* Upper layers face shortage */ /* bio_flags */ #define BIO_ERROR 0x01 /* An error occurred processing this bio. */ @@ -67,6 +68,9 @@ #define BIO_TRANSIENT_MAPPING 0x20 #define BIO_VLIST 0x40 +#define BIO_SPEEDUP_WRITE 0x4000 /* Resource shortage at upper layers */ +#define BIO_SPEEDUP_TRIM 0x8000 /* Resource shortage at upper layers */ + #ifdef _KERNEL struct disk; struct bio; Index: sys/ufs/ffs/ffs_softdep.c =================================================================== --- sys/ufs/ffs/ffs_softdep.c +++ sys/ufs/ffs/ffs_softdep.c @@ -13327,7 +13327,9 @@ { struct ufsmount *ump; struct mount *mp; + struct g_consumer *cp; long starttime; + size_t resid; ufs2_daddr_t needed; int error, failed_vnode; @@ -13349,6 +13351,7 @@ mp = vp->v_mount; ump = VFSTOUFS(mp); + cp = (struct g_consumer *)ump->um_devvp->v_bufobj.bo_private; mtx_assert(UFS_MTX(ump), MA_OWNED); UFS_UNLOCK(ump); error = ffs_update(vp, 1); @@ -13402,6 +13405,9 @@ } starttime = time_second; retry: + if (resource == FLUSH_BLOCKS_WAIT && + fs->fs_cstotal.cs_nbfree <= needed) + g_io_speedup(needed * fs->fs_bsize, BIO_SPEEDUP_TRIM, &resid, cp); if ((resource == FLUSH_BLOCKS_WAIT && ump->softdep_on_worklist > 0 && fs->fs_cstotal.cs_nbfree <= needed) || (resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 && @@ -13548,6 +13554,8 @@ { struct mount *mp; struct ufsmount *ump; + struct g_consumer *cp; + size_t resid; int error; bool req; @@ -13559,6 +13567,8 @@ return; if (ffs_own_mount(mp) && MOUNTEDSOFTDEP(mp)) { ump = VFSTOUFS(mp); + cp = (struct g_consumer *)ump->um_devvp->v_bufobj.bo_private; + g_io_speedup(0, BIO_SPEEDUP_TRIM, &resid, cp); for (;;) { req = false; ACQUIRE_LOCK(ump);