Index: sys/cam/ata/ata_da.c =================================================================== --- sys/cam/ata/ata_da.c +++ sys/cam/ata/ata_da.c @@ -1088,13 +1088,8 @@ ata_28bit_cmd(&ccb.ataio, ATA_WRITE_DMA, 0, lba, count); } - xpt_polled_action(&ccb); - - error = adaerror(&ccb, - 0, SF_NO_RECOVERY | SF_NO_RETRY); - if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0) - cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0, - /*reduction*/0, /*timeout*/0, /*getcount_only*/0); + error = cam_periph_runccb(&ccb, adaerror, + 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); if (error != 0) printf("Aborting dump due to I/O error.\n"); @@ -1124,13 +1119,8 @@ ata_48bit_cmd(&ccb.ataio, ATA_FLUSHCACHE48, 0, 0, 0); else ata_28bit_cmd(&ccb.ataio, ATA_FLUSHCACHE, 0, 0, 0); - xpt_polled_action(&ccb); - - error = adaerror(&ccb, - 0, SF_NO_RECOVERY | SF_NO_RETRY); - if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0) - cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0, - /*reduction*/0, /*timeout*/0, /*getcount_only*/0); + error = cam_periph_runccb(&ccb, adaerror, + 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); if (error != 0) xpt_print(periph->path, "Synchronize cache failed\n"); } @@ -3509,32 +3499,9 @@ 0, ada_default_timeout*1000); ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0); - - if (!SCHEDULER_STOPPED()) { - /* - * Not panicing, can just do the normal runccb - * XXX should make cam_periph_runccb work while - * XXX panicing... later - */ - error = cam_periph_runccb((union ccb *)&local_ccb, adaerror, - /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY, - softc->disk->d_devstat); - } else { - /* - * Panicing, so we have to do this by hand: do - * xpt_polled_action to run the request through the SIM, - * extract the error, and if the queue was frozen, - * unfreeze it. cam_periph_runccb takes care of these - * details, but xpt_polled_action doesn't. - */ - xpt_polled_action((union ccb *)&local_ccb); - error = adaerror((union ccb *)&local_ccb, 0, - SF_NO_RECOVERY | SF_NO_RETRY); - if ((local_ccb.ccb_h.status & CAM_DEV_QFRZN) != 0) - cam_release_devq(local_ccb.ccb_h.path, - /*relsim_flags*/0, /*reduction*/0, - /*timeout*/0, /*getcount_only*/0); - } + error = cam_periph_runccb((union ccb *)&local_ccb, adaerror, + /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY, + softc->disk->d_devstat); if (error != 0) xpt_print(periph->path, "Spin-down disk failed\n"); cam_periph_unlock(periph); Index: sys/cam/cam_periph.c =================================================================== --- sys/cam/cam_periph.c +++ sys/cam/cam_periph.c @@ -1160,7 +1160,11 @@ struct bintime *starttime; struct bintime ltime; int error; - + bool sched_stopped; + struct mtx *periph_mtx; + struct cam_periph *periph; + uint32_t timeout = 1; + starttime = NULL; xpt_path_assert(ccb->ccb_h.path, MA_OWNED); KASSERT((ccb->ccb_h.flags & CAM_UNLOCKED) == 0, @@ -1180,21 +1184,47 @@ devstat_start_transaction(ds, starttime); } + sched_stopped = SCHEDULER_STOPPED(); ccb->ccb_h.cbfcnp = cam_periph_done; - xpt_action(ccb); - - do { - cam_periph_ccbwait(ccb); - if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) - error = 0; - else if (error_routine != NULL) { - ccb->ccb_h.cbfcnp = cam_periph_done; - error = (*error_routine)(ccb, camflags, sense_flags); - } else - error = 0; + periph = xpt_path_periph(ccb->ccb_h.path); + periph_mtx = cam_periph_mtx(periph); + + /* + * If we're polling, then we need to ensure that we have ample resources + * in the periph. We also need to drop the periph lock while we're polling. + * cam_periph_error can reschedule the ccb by calling xpt_action and returning + * ERESTART, so we have to effect the polling in the do loop below. + */ + if (sched_stopped) { + mtx_unlock(periph_mtx); + timeout = xpt_poll_setup(ccb); + } + + if (timeout == 0) { + ccb->ccb_h.status = CAM_RESRC_UNAVAIL; + error = EBUSY; + } else { + xpt_action(ccb); + do { + if (!sched_stopped) + cam_periph_ccbwait(ccb); + else { + xpt_pollwait(ccb, timeout); + timeout = ccb->ccb_h.timeout * 10; + } + if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) + error = 0; + else if (error_routine != NULL) { + ccb->ccb_h.cbfcnp = cam_periph_done; + error = (*error_routine)(ccb, camflags, sense_flags); + } else + error = 0; + } while (error == ERESTART); + } + + if (sched_stopped) + mtx_lock(periph_mtx); - } while (error == ERESTART); - if ((ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { cam_release_devq(ccb->ccb_h.path, /* relsim_flags */0, Index: sys/cam/cam_xpt.h =================================================================== --- sys/cam/cam_xpt.h +++ sys/cam/cam_xpt.h @@ -144,6 +144,8 @@ void xpt_release_path(struct cam_path *path); const char * xpt_action_name(uint32_t action); +void xpt_pollwait(union ccb *start_ccb, uint32_t timeout); +uint32_t xpt_poll_setup(union ccb *start_ccb); #endif /* _KERNEL */ Index: sys/cam/cam_xpt.c =================================================================== --- sys/cam/cam_xpt.c +++ sys/cam/cam_xpt.c @@ -3204,8 +3204,8 @@ start_ccb->ccb_h.status)); } -void -xpt_polled_action(union ccb *start_ccb) +uint32_t +xpt_poll_setup(union ccb *start_ccb) { u_int32_t timeout; struct cam_sim *sim; @@ -3219,8 +3219,6 @@ mtx = sim->mtx; dev = start_ccb->ccb_h.path->device; - mtx_unlock(&dev->device_mtx); - /* * Steal an opening so that no other queued requests * can get it before us while we simulate interrupts. @@ -3242,29 +3240,57 @@ dev->ccbq.dev_openings++; mtx_unlock(&devq->send_mtx); - if (timeout != 0) { + return (timeout); +} + +void +xpt_pollwait(union ccb *start_ccb, uint32_t timeout) +{ + struct cam_sim *sim; + struct mtx *mtx; + + sim = start_ccb->ccb_h.path->bus->sim; + mtx = sim->mtx; + + while (--timeout > 0) { + if (mtx) + mtx_lock(mtx); + (*(sim->sim_poll))(sim); + if (mtx) + mtx_unlock(mtx); + camisr_runqueue(); + if ((start_ccb->ccb_h.status & CAM_STATUS_MASK) + != CAM_REQ_INPROG) + break; + DELAY(100); + } + + if (timeout == 0) { + /* + * XXX Is it worth adding a sim_timeout entry + * point so we can attempt recovery? If + * this is only used for dumps, I don't think + * it is. + */ + start_ccb->ccb_h.status = CAM_CMD_TIMEOUT; + } +} + +void +xpt_polled_action(union ccb *start_ccb) +{ + uint32_t timeout; + struct cam_ed *dev; + + timeout = start_ccb->ccb_h.timeout * 10; + dev = start_ccb->ccb_h.path->device; + + mtx_unlock(&dev->device_mtx); + + timeout = xpt_poll_setup(start_ccb); + if (timeout > 0) { xpt_action(start_ccb); - while(--timeout > 0) { - if (mtx) - mtx_lock(mtx); - (*(sim->sim_poll))(sim); - if (mtx) - mtx_unlock(mtx); - camisr_runqueue(); - if ((start_ccb->ccb_h.status & CAM_STATUS_MASK) - != CAM_REQ_INPROG) - break; - DELAY(100); - } - if (timeout == 0) { - /* - * XXX Is it worth adding a sim_timeout entry - * point so we can attempt recovery? If - * this is only used for dumps, I don't think - * it is. - */ - start_ccb->ccb_h.status = CAM_CMD_TIMEOUT; - } + xpt_pollwait(start_ccb, timeout); } else { start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL; } @@ -3306,6 +3332,7 @@ CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n")); + old_priority = pinfo->priority; /* Index: sys/cam/nvme/nvme_da.c =================================================================== --- sys/cam/nvme/nvme_da.c +++ sys/cam/nvme/nvme_da.c @@ -404,17 +404,12 @@ xpt_setup_ccb(&nvmeio.ccb_h, periph->path, CAM_PRIORITY_NORMAL); nvmeio.ccb_h.ccb_state = NDA_CCB_DUMP; nda_nvme_write(softc, &nvmeio, virtual, lba, length, count); - xpt_polled_action((union ccb *)&nvmeio); - - error = cam_periph_error((union ccb *)&nvmeio, - 0, SF_NO_RECOVERY | SF_NO_RETRY); - if ((nvmeio.ccb_h.status & CAM_DEV_QFRZN) != 0) - cam_release_devq(nvmeio.ccb_h.path, /*relsim_flags*/0, - /*reduction*/0, /*timeout*/0, /*getcount_only*/0); + error = cam_periph_runccb((union ccb *)&nvmeio, cam_periph_error, + 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); if (error != 0) - printf("Aborting dump due to I/O error.\n"); - + printf("Aborting dump due to I/O error %d.\n", error); cam_periph_unlock(periph); + return (error); } @@ -423,13 +418,8 @@ nvmeio.ccb_h.ccb_state = NDA_CCB_DUMP; nda_nvme_flush(softc, &nvmeio); - xpt_polled_action((union ccb *)&nvmeio); - - error = cam_periph_error((union ccb *)&nvmeio, - 0, SF_NO_RECOVERY | SF_NO_RETRY); - if ((nvmeio.ccb_h.status & CAM_DEV_QFRZN) != 0) - cam_release_devq(nvmeio.ccb_h.path, /*relsim_flags*/0, - /*reduction*/0, /*timeout*/0, /*getcount_only*/0); + error = cam_periph_runccb((union ccb *)&nvmeio, cam_periph_error, + 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); if (error != 0) xpt_print(periph->path, "flush cmd failed\n"); cam_periph_unlock(periph); Index: sys/cam/scsi/scsi_da.c =================================================================== --- sys/cam/scsi/scsi_da.c +++ sys/cam/scsi/scsi_da.c @@ -1673,13 +1673,8 @@ /*dxfer_len*/length, /*sense_len*/SSD_FULL_SIZE, da_default_timeout * 1000); - xpt_polled_action((union ccb *)&csio); - - error = cam_periph_error((union ccb *)&csio, - 0, SF_NO_RECOVERY | SF_NO_RETRY); - if ((csio.ccb_h.status & CAM_DEV_QFRZN) != 0) - cam_release_devq(csio.ccb_h.path, /*relsim_flags*/0, - /*reduction*/0, /*timeout*/0, /*getcount_only*/0); + error = cam_periph_runccb((union ccb *)&csio, cam_periph_error, + 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); if (error != 0) printf("Aborting dump due to I/O error.\n"); cam_periph_unlock(periph); @@ -1701,13 +1696,8 @@ /*lb_count*/0, SSD_FULL_SIZE, 5 * 1000); - xpt_polled_action((union ccb *)&csio); - - error = cam_periph_error((union ccb *)&csio, - 0, SF_NO_RECOVERY | SF_NO_RETRY | SF_QUIET_IR); - if ((csio.ccb_h.status & CAM_DEV_QFRZN) != 0) - cam_release_devq(csio.ccb_h.path, /*relsim_flags*/0, - /*reduction*/0, /*timeout*/0, /*getcount_only*/0); + error = cam_periph_runccb((union ccb *)&csio, cam_periph_error, + 0, SF_NO_RECOVERY | SF_NO_RETRY, NULL); if (error != 0) xpt_print(periph->path, "Synchronize cache failed\n"); }