Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/nvme/nvme_qpair.c
Show First 20 Lines • Show All 616 Lines • ▼ Show 20 Lines | while (1) { | ||||
* using the atomic store to force strong ordering (at least when | * using the atomic store to force strong ordering (at least when | ||||
* viewed in the aftermath of a panic). | * viewed in the aftermath of a panic). | ||||
*/ | */ | ||||
if (++qpair->cq_head == qpair->num_entries) { /* 1 */ | if (++qpair->cq_head == qpair->num_entries) { /* 1 */ | ||||
atomic_store_rel_int(&qpair->cq_head, 0); /* 2 */ | atomic_store_rel_int(&qpair->cq_head, 0); /* 2 */ | ||||
qpair->phase = !qpair->phase; /* 3 */ | qpair->phase = !qpair->phase; /* 3 */ | ||||
} | } | ||||
nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl, | bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, | ||||
qpair->cq_head); | qpair->cq_hdbl_off, qpair->cq_head); | ||||
scottl: I would have probably stuck with using nvme_mmio_write_4() because it's more compact and easier… | |||||
Done Inline ActionsI'd love to, but it hard-codes a offsetof() its second argument, so I couldn't use it. imp: I'd love to, but it hard-codes a offsetof() its second argument, so I couldn't use it.
| |||||
} | } | ||||
return (done != 0); | return (done != 0); | ||||
} | } | ||||
static void | static void | ||||
nvme_qpair_msix_handler(void *arg) | nvme_qpair_msix_handler(void *arg) | ||||
{ | { | ||||
struct nvme_qpair *qpair = arg; | struct nvme_qpair *qpair = arg; | ||||
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, | ||||
qpair->num_failures = 0; | qpair->num_failures = 0; | ||||
qpair->cmd = (struct nvme_command *)queuemem; | qpair->cmd = (struct nvme_command *)queuemem; | ||||
qpair->cpl = (struct nvme_completion *)(queuemem + cmdsz); | qpair->cpl = (struct nvme_completion *)(queuemem + cmdsz); | ||||
prpmem = (uint8_t *)(queuemem + cmdsz + cplsz); | prpmem = (uint8_t *)(queuemem + cmdsz + cplsz); | ||||
qpair->cmd_bus_addr = queuemem_phys; | qpair->cmd_bus_addr = queuemem_phys; | ||||
qpair->cpl_bus_addr = queuemem_phys + cmdsz; | qpair->cpl_bus_addr = queuemem_phys + cmdsz; | ||||
prpmem_phys = queuemem_phys + cmdsz + cplsz; | prpmem_phys = queuemem_phys + cmdsz + cplsz; | ||||
qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl); | /* | ||||
qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl); | * Calcuate the stride of the doorbell register. Many emulators set this | ||||
* value to correspond to a cache line. However, some hardware has set | |||||
* it to various small values. | |||||
*/ | |||||
qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[0]) + | |||||
(id << (ctrlr->dstrd + 1)); | |||||
qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[0]) + | |||||
(id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd); | |||||
TAILQ_INIT(&qpair->free_tr); | TAILQ_INIT(&qpair->free_tr); | ||||
TAILQ_INIT(&qpair->outstanding_tr); | TAILQ_INIT(&qpair->outstanding_tr); | ||||
STAILQ_INIT(&qpair->queued_req); | STAILQ_INIT(&qpair->queued_req); | ||||
list_phys = prpmem_phys; | list_phys = prpmem_phys; | ||||
prp_list = prpmem; | prp_list = prpmem; | ||||
for (i = 0; i < qpair->num_trackers; i++) { | for (i = 0; i < qpair->num_trackers; i++) { | ||||
▲ Show 20 Lines • Show All 201 Lines • ▼ Show 20 Lines | |||||
#ifndef __powerpc__ | #ifndef __powerpc__ | ||||
/* | /* | ||||
* powerpc's bus_dmamap_sync() already includes a heavyweight sync, but | * powerpc's bus_dmamap_sync() already includes a heavyweight sync, but | ||||
* no other archs do. | * no other archs do. | ||||
*/ | */ | ||||
wmb(); | wmb(); | ||||
#endif | #endif | ||||
nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl, | bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, | ||||
qpair->sq_tail); | qpair->sq_tdbl_off, qpair->sq_tail); | ||||
qpair->num_cmds++; | qpair->num_cmds++; | ||||
} | } | ||||
static void | static void | ||||
nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) | nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) | ||||
{ | { | ||||
struct nvme_tracker *tr = arg; | struct nvme_tracker *tr = arg; | ||||
uint32_t cur_nseg; | uint32_t cur_nseg; | ||||
▲ Show 20 Lines • Show All 301 Lines • Show Last 20 Lines |
I would have probably stuck with using nvme_mmio_write_4() because it's more compact and easier to read, but it's a super minor thing.