Index: sys/cam/ata/ata_da.c =================================================================== --- sys/cam/ata/ata_da.c +++ sys/cam/ata/ata_da.c @@ -111,7 +111,8 @@ ADA_FLAG_CAN_NCQ_TRIM = 0x00400000, /* CAN_TRIM also set */ ADA_FLAG_PIM_ATA_EXT = 0x00800000, ADA_FLAG_UNMAPPEDIO = 0x01000000, - ADA_FLAG_ROTATING = 0x02000000 + ADA_FLAG_ROTATING = 0x02000000, + ADA_FLAG_CAN_NCQ_PRIO = 0x04000000 } ada_flags; #define ADA_FLAG_STRING \ "\020" \ @@ -139,7 +140,8 @@ "\027CAN_NCQ_TRIM" \ "\030PIM_ATA_EXT" \ "\031UNMAPPEDIO" \ - "\032ROTATING" + "\032ROTATING" \ + "\033CAN_NCQ_PRIO" typedef enum { ADA_Q_NONE = 0x00, @@ -1723,6 +1725,12 @@ else softc->flags &= ~ADA_FLAG_CAN_NCQ; + if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ_PRIO) && + (softc->flags & ADA_FLAG_CAN_NCQ)) + softc->flags |= ADA_FLAG_CAN_NCQ_PRIO; + else + softc->flags &= ~ADA_FLAG_CAN_NCQ_PRIO; + if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) && (cgd->inq_flags & SID_DMA) && (softc->quirks & ADA_Q_NO_TRIM) == 0) { @@ -2074,6 +2082,9 @@ 0, howmany(ranges, ATA_DSM_BLK_RANGES)); ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM; + if ((bp->bio_flags & BIO_LOWPRIO) == 0 && + (softc->flags & ADA_FLAG_CAN_NCQ_PRIO) && cam_enable_prio) + ataio->cmd.sector_count_exp |= 0x80; ataio->ata_flags |= ATA_FLAG_AUX; ataio->aux = 1; } @@ -2414,6 +2425,10 @@ ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED, lba, count); } + if ((bp->bio_flags & BIO_LOWPRIO) == 0 && + (softc->flags & ADA_FLAG_CAN_NCQ_PRIO) && + cam_enable_prio) + ataio->cmd.sector_count_exp = 0x80; } else if ((softc->flags & ADA_FLAG_CAN_48BIT) && (lba + count >= ATA_MAX_28BIT_LBA || count > 256)) { Index: sys/cam/cam.h =================================================================== --- sys/cam/cam.h +++ sys/cam/cam.h @@ -362,6 +362,7 @@ extern const int num_cam_status_entries; #ifdef _KERNEL extern int cam_sort_io_queues; +extern int cam_enable_prio; #endif union ccb; struct sbuf; Index: sys/cam/cam.c =================================================================== --- sys/cam/cam.c +++ sys/cam/cam.c @@ -118,6 +118,14 @@ int cam_sort_io_queues = CAM_DEFAULT_SORT_IO_QUEUES; SYSCTL_INT(_kern_cam, OID_AUTO, sort_io_queues, CTLFLAG_RWTUN, &cam_sort_io_queues, 0, "Sort IO queues to try and optimise disk access patterns"); + +#ifndef CAM_DEFAULT_ENABLE_PRIO +#define CAM_DEFAULT_ENABLE_PRIO 1 +#endif + +int cam_enable_prio = CAM_DEFAULT_ENABLE_PRIO; +SYSCTL_INT(_kern_cam, OID_AUTO, enable_prio, CTLFLAG_RWTUN, + &cam_enable_prio, 0, "Enable command priority"); #endif void Index: sys/cam/scsi/scsi_cd.c =================================================================== --- sys/cam/scsi/scsi_cd.c +++ sys/cam/scsi/scsi_cd.c @@ -961,6 +961,12 @@ start_ccb->csio.cdb_io.cdb_bytes[11] = 0; start_ccb->csio.cdb_len = 12; } + if (cam_enable_prio) { + if (bp->bio_flags & BIO_LOWPRIO) + start_ccb->csio.priority = 15; + else + start_ccb->csio.priority = 1; + } start_ccb->ccb_h.ccb_state = CD_CCB_BUFFER_IO; LIST_INSERT_HEAD(&softc->pending_ccbs, Index: sys/cam/scsi/scsi_da.c =================================================================== --- sys/cam/scsi/scsi_da.c +++ sys/cam/scsi/scsi_da.c @@ -3457,6 +3457,12 @@ xpt_release_ccb(start_ccb); return; } + if (cam_enable_prio) { + if (bp->bio_flags & BIO_LOWPRIO) + start_ccb->csio.priority = 15; + else + start_ccb->csio.priority = 1; + } start_ccb->ccb_h.ccb_state = DA_CCB_BUFFER_IO; start_ccb->ccb_h.flags |= CAM_UNLOCKED; start_ccb->ccb_h.softtimeout = sbttotv(da_default_softtimeout); Index: sys/contrib/openzfs/module/os/freebsd/zfs/vdev_geom.c =================================================================== --- sys/contrib/openzfs/module/os/freebsd/zfs/vdev_geom.c +++ sys/contrib/openzfs/module/os/freebsd/zfs/vdev_geom.c @@ -1132,6 +1132,11 @@ bp->bio_data = abd_borrow_buf_copy(zio->io_abd, zio->io_size); } + if (zio->io_priority == ZIO_PRIORITY_SCRUB || + zio->io_priority == ZIO_PRIORITY_REMOVAL || + zio->io_priority == ZIO_PRIORITY_INITIALIZING || + zio->io_priority == ZIO_PRIORITY_REBUILD) + bp->bio_flags |= BIO_LOWPRIO; break; case ZIO_TYPE_TRIM: bp->bio_cmd = BIO_DELETE; Index: sys/geom/geom_io.c =================================================================== --- sys/geom/geom_io.c +++ sys/geom/geom_io.c @@ -204,7 +204,7 @@ * Other bio flags are not suitable for cloning. */ bp2->bio_flags = bp->bio_flags & - (BIO_ORDERED | BIO_UNMAPPED | BIO_VLIST); + (BIO_ORDERED | BIO_UNMAPPED | BIO_VLIST | BIO_LOWPRIO); bp2->bio_length = bp->bio_length; bp2->bio_offset = bp->bio_offset; bp2->bio_data = bp->bio_data; Index: sys/geom/mirror/g_mirror.c =================================================================== --- sys/geom/mirror/g_mirror.c +++ sys/geom/mirror/g_mirror.c @@ -2064,6 +2064,7 @@ g_reset_bio(bp); bp->bio_cmd = BIO_READ; + bp->bio_flags = BIO_LOWPRIO; bp->bio_data = data; bp->bio_done = g_mirror_sync_done; bp->bio_from = disk->d_sync.ds_consumer; Index: sys/geom/raid/tr_raid1.c =================================================================== --- sys/geom/raid/tr_raid1.c +++ sys/geom/raid/tr_raid1.c @@ -261,6 +261,7 @@ sd->sd_size - sd->sd_rebuild_pos); bp->bio_data = trs->trso_buffer; bp->bio_cmd = BIO_READ; + bp->bio_flags = BIO_LOWPRIO; bp->bio_cflags = G_RAID_BIO_FLAG_SYNC; bp->bio_caller1 = good_sd; trs->trso_flags |= TR_RAID1_F_DOING_SOME; Index: sys/geom/raid/tr_raid1e.c =================================================================== --- sys/geom/raid/tr_raid1e.c +++ sys/geom/raid/tr_raid1e.c @@ -477,6 +477,7 @@ bp->bio_length = len; bp->bio_data = trs->trso_buffer; bp->bio_cmd = BIO_READ; + bp->bio_flags = BIO_LOWPRIO; bp->bio_cflags = G_RAID_BIO_FLAG_SYNC; bp->bio_caller1 = &vol->v_subdisks[(disk + best) % vol->v_disks_count]; G_RAID_LOGREQ(3, bp, "Queueing rebuild read"); Index: sys/geom/raid3/g_raid3.c =================================================================== --- sys/geom/raid3/g_raid3.c +++ sys/geom/raid3/g_raid3.c @@ -1663,6 +1663,7 @@ bp->bio_offset /= sc->sc_ndisks - 1; bp->bio_length /= sc->sc_ndisks - 1; bp->bio_cmd = BIO_WRITE; + bp->bio_flags = BIO_LOWPRIO; bp->bio_cflags = 0; bp->bio_children = bp->bio_inbed = 0; cp = disk->d_consumer; @@ -1722,6 +1723,7 @@ data = bp->bio_data; g_reset_bio(bp); bp->bio_cmd = BIO_READ; + bp->bio_flags = BIO_LOWPRIO; bp->bio_offset = sync->ds_offset * (sc->sc_ndisks - 1); bp->bio_length = MIN(MAXPHYS, sc->sc_mediasize - bp->bio_offset); sync->ds_offset += bp->bio_length / (sc->sc_ndisks - 1); @@ -2241,6 +2243,7 @@ disk->d_sync.ds_bios[n] = bp; bp->bio_parent = NULL; bp->bio_cmd = BIO_READ; + bp->bio_flags = BIO_LOWPRIO; bp->bio_data = malloc(MAXPHYS, M_RAID3, M_WAITOK); bp->bio_cflags = 0; bp->bio_offset = disk->d_sync.ds_offset * (sc->sc_ndisks - 1); Index: sys/sys/bio.h =================================================================== --- sys/sys/bio.h +++ sys/sys/bio.h @@ -67,8 +67,9 @@ #define BIO_UNMAPPED 0x10 #define BIO_TRANSIENT_MAPPING 0x20 #define BIO_VLIST 0x40 +#define BIO_LOWPRIO 0x80 -#define PRINT_BIO_FLAGS "\20\7vlist\6transient_mapping\5unmapped" \ +#define PRINT_BIO_FLAGS "\20\10lowprio\7vlist\6transient_mapping\5unmapped" \ "\4ordered\3onqueue\2done\1error" #define BIO_SPEEDUP_WRITE 0x4000 /* Resource shortage at upper layers */