Index: sys/cam/nvme/nvme_da.c =================================================================== --- sys/cam/nvme/nvme_da.c +++ sys/cam/nvme/nvme_da.c @@ -986,7 +986,19 @@ out: start_ccb->ccb_h.flags |= CAM_UNLOCKED; softc->outstanding_cmds++; - softc->refcount++; + /* + * Take two references out. One guards against the completion + * routine triggered by start_ccb finishing, triggering a geom + * wither and call to daclose before we can get to the + * cam_periph_lock. The other reference guards against us + * returning, wither + ndaclose because geom falsely thinks all + * its transactions are done (it shouldn't do this, but it's + * better to fail safe in the face of geom bugs and it's easy to + * do with an extra reference here) and then this transaction + * completing. + */ + softc->refcount++; /* For submission */ + softc->refcount++; /* For completion */ cam_periph_unlock(periph); xpt_action(start_ccb); cam_periph_lock(periph); @@ -1059,6 +1071,8 @@ cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb); xpt_release_ccb(done_ccb); ndaschedule(periph); + KASSERT(softc->refcount >= 1, ("ndadone softc %p refcount %d", softc, softc->refcount)); + softc->refcount--; /* Release the periph refcount taken in ndastart() for each CCB. *. cam_periph_unlock(periph); biodone(bp); } else { /* state == NDA_CCB_TRIM */ @@ -1086,6 +1100,8 @@ cam_iosched_bio_complete(softc->cam_iosched, bp1, done_ccb); xpt_release_ccb(done_ccb); ndaschedule(periph); + KASSERT(softc->refcount >= 1, ("ndadone softc %p refcount %d", softc, softc->refcount)); + softc->refcount--; /* Release the periph refcount taken in ndastart() for each CCB. *. cam_periph_unlock(periph); while ((bp2 = TAILQ_FIRST(&queue)) != NULL) { TAILQ_REMOVE(&queue, bp2, bio_queue); @@ -1100,11 +1116,6 @@ biodone(bp2); } } - /* - * Release the periph refcount taken in mdastart() for each CCB. - */ - KASSERT(softc->refcount >= 1, ("ndadone softc %p refcount %d", softc, softc->refcount)); - softc->refcount--; return; } case NDA_CCB_DUMP: