Index: head/sys/dev/nvme/nvme_qpair.c =================================================================== --- head/sys/dev/nvme/nvme_qpair.c +++ head/sys/dev/nvme/nvme_qpair.c @@ -401,9 +401,13 @@ req->retries++; nvme_qpair_submit_tracker(qpair, tr); } else { - if (req->type != NVME_REQUEST_NULL) + if (req->type != NVME_REQUEST_NULL) { + bus_dmamap_sync(qpair->dma_tag_payload, + tr->payload_dma_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(qpair->dma_tag_payload, tr->payload_dma_map); + } nvme_free_request(req); tr->req = NULL; @@ -487,6 +491,8 @@ */ return (false); + bus_dmamap_sync(qpair->dma_tag, qpair->queuemem_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); while (1) { cpl = qpair->cpl[qpair->cq_head]; @@ -828,7 +834,16 @@ if (++qpair->sq_tail == qpair->num_entries) qpair->sq_tail = 0; + bus_dmamap_sync(qpair->dma_tag, qpair->queuemem_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); +#ifndef __powerpc__ + /* + * powerpc's bus_dmamap_sync() already includes a heavyweight sync, but + * no other archs do. + */ wmb(); +#endif + nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl, qpair->sq_tail); @@ -879,6 +894,8 @@ tr->req->cmd.prp2 = 0; } + bus_dmamap_sync(tr->qpair->dma_tag_payload, tr->payload_dma_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); nvme_qpair_submit_tracker(tr->qpair, tr); }