Index: head/sys/dev/nvme/nvme_ctrlr.c =================================================================== --- head/sys/dev/nvme/nvme_ctrlr.c +++ head/sys/dev/nvme/nvme_ctrlr.c @@ -90,19 +90,25 @@ struct nvme_qpair *qpair; uint32_t cap_lo; uint16_t mqes; - int i, error, num_entries, num_trackers; + int i, error, num_entries, num_trackers, max_entries; - num_entries = NVME_IO_ENTRIES; - TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries); - /* - * NVMe spec sets a hard limit of 64K max entries, but - * devices may specify a smaller limit, so we need to check - * the MQES field in the capabilities register. + * NVMe spec sets a hard limit of 64K max entries, but devices may + * specify a smaller limit, so we need to check the MQES field in the + * capabilities register. We have to cap the number of entries to the + * current stride allows for in BAR 0/1, otherwise the remainder entries + * are inaccessable. MQES should reflect this, and this is just a + * fail-safe. */ + max_entries = + (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) / + (1 << (ctrlr->dstrd + 1)); + num_entries = NVME_IO_ENTRIES; + TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries); cap_lo = nvme_mmio_read_4(ctrlr, cap_lo); mqes = NVME_CAP_LO_MQES(cap_lo); num_entries = min(num_entries, mqes + 1); + num_entries = min(num_entries, max_entries); num_trackers = NVME_IO_TRACKERS; TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers); @@ -110,9 +116,9 @@ num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS); num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS); /* - * No need to have more trackers than entries in the submit queue. - * Note also that for a queue size of N, we can only have (N-1) - * commands outstanding, hence the "-1" here. + * No need to have more trackers than entries in the submit queue. Note + * also that for a queue size of N, we can only have (N-1) commands + * outstanding, hence the "-1" here. */ num_trackers = min(num_trackers, (num_entries-1)); @@ -1120,7 +1126,6 @@ uint32_t cap_lo; uint32_t cap_hi; uint32_t to; - uint8_t dstrd; uint8_t mpsmin; int status, timeout_period; @@ -1128,14 +1133,8 @@ mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF); - /* - * Software emulators may set the doorbell stride to something - * other than zero, but this driver is not set up to handle that. - */ cap_hi = nvme_mmio_read_4(ctrlr, cap_hi); - dstrd = NVME_CAP_HI_DSTRD(cap_hi); - if (dstrd != 0) - return (ENXIO); + ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2; mpsmin = NVME_CAP_HI_MPSMIN(cap_hi); ctrlr->min_page_size = 1 << (12 + mpsmin); Index: head/sys/dev/nvme/nvme_private.h =================================================================== --- head/sys/dev/nvme/nvme_private.h +++ head/sys/dev/nvme/nvme_private.h @@ -297,6 +297,9 @@ /** timeout period in seconds */ uint32_t timeout_period; + /** doorbell stride */ + uint32_t dstrd; + struct nvme_qpair adminq; struct nvme_qpair *ioq; Index: head/sys/dev/nvme/nvme_qpair.c =================================================================== --- head/sys/dev/nvme/nvme_qpair.c +++ head/sys/dev/nvme/nvme_qpair.c @@ -622,8 +622,8 @@ qpair->phase = !qpair->phase; /* 3 */ } - nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl, - qpair->cq_head); + bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, + qpair->cq_hdbl_off, qpair->cq_head); } return (done != 0); } @@ -731,8 +731,15 @@ qpair->cpl_bus_addr = queuemem_phys + cmdsz; prpmem_phys = queuemem_phys + cmdsz + cplsz; - qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl); - qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl); + /* + * Calcuate the stride of the doorbell register. Many emulators set this + * value to correspond to a cache line. However, some hardware has set + * it to various small values. + */ + qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[0]) + + (id << (ctrlr->dstrd + 1)); + qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[0]) + + (id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd); TAILQ_INIT(&qpair->free_tr); TAILQ_INIT(&qpair->outstanding_tr); @@ -950,9 +957,8 @@ wmb(); #endif - nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl, - qpair->sq_tail); - + bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, + qpair->sq_tdbl_off, qpair->sq_tail); qpair->num_cmds++; }