Changeset View
Changeset View
Standalone View
Standalone View
head/sys/dev/nvme/nvme_qpair.c
| Show First 20 Lines • Show All 541 Lines • ▼ Show 20 Lines | nvme_qpair_process_completions(struct nvme_qpair *qpair) | ||||
| /* | /* | ||||
| * qpair is not enabled, likely because a controller reset is is in | * qpair is not enabled, likely because a controller reset is is in | ||||
| * progress. Ignore the interrupt - any I/O that was associated with | * progress. Ignore the interrupt - any I/O that was associated with | ||||
| * this interrupt will get retried when the reset is complete. | * this interrupt will get retried when the reset is complete. | ||||
| */ | */ | ||||
| if (!qpair->is_enabled) | if (!qpair->is_enabled) | ||||
| return (false); | return (false); | ||||
| bus_dmamap_sync(qpair->dma_tag, qpair->queuemem_map, | |||||
| BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
| /* | /* | ||||
| * A panic can stop the CPU this routine is running on at any point. If | * A panic can stop the CPU this routine is running on at any point. If | ||||
| * we're called during a panic, complete the sq_head wrap protocol for | * we're called during a panic, complete the sq_head wrap protocol for | ||||
| * the case where we are interrupted just after the increment at 1 | * the case where we are interrupted just after the increment at 1 | ||||
| * below, but before we can reset cq_head to zero at 2. Also cope with | * below, but before we can reset cq_head to zero at 2. Also cope with | ||||
| * the case where we do the zero at 2, but may or may not have done the | * the case where we do the zero at 2, but may or may not have done the | ||||
| * phase adjustment at step 3. The panic machinery flushes all pending | * phase adjustment at step 3. The panic machinery flushes all pending | ||||
| * memory writes, so we can make these strong ordering assumptions | * memory writes, so we can make these strong ordering assumptions | ||||
| Show All 17 Lines | if (qpair->cq_head == qpair->num_entries) { | ||||
| * that it has. This gets us back in sync | * that it has. This gets us back in sync | ||||
| */ | */ | ||||
| cpl = qpair->cpl[qpair->num_entries - 1]; | cpl = qpair->cpl[qpair->num_entries - 1]; | ||||
| nvme_completion_swapbytes(&cpl); | nvme_completion_swapbytes(&cpl); | ||||
| qpair->phase = !NVME_STATUS_GET_P(cpl.status); | qpair->phase = !NVME_STATUS_GET_P(cpl.status); | ||||
| } | } | ||||
| } | } | ||||
| bus_dmamap_sync(qpair->dma_tag, qpair->queuemem_map, | |||||
| BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
| while (1) { | while (1) { | ||||
| cpl = qpair->cpl[qpair->cq_head]; | cpl = qpair->cpl[qpair->cq_head]; | ||||
| /* Convert to host endian */ | /* Convert to host endian */ | ||||
| nvme_completion_swapbytes(&cpl); | nvme_completion_swapbytes(&cpl); | ||||
| if (NVME_STATUS_GET_P(cpl.status) != qpair->phase) | if (NVME_STATUS_GET_P(cpl.status) != qpair->phase) | ||||
| break; | break; | ||||
| ▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Lines | err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), | ||||
| allocsz, 1, allocsz, 0, NULL, NULL, &qpair->dma_tag); | allocsz, 1, allocsz, 0, NULL, NULL, &qpair->dma_tag); | ||||
| if (err != 0) { | if (err != 0) { | ||||
| nvme_printf(ctrlr, "tag create failed %d\n", err); | nvme_printf(ctrlr, "tag create failed %d\n", err); | ||||
| goto out; | goto out; | ||||
| } | } | ||||
| bus_dma_tag_set_domain(qpair->dma_tag, qpair->domain); | bus_dma_tag_set_domain(qpair->dma_tag, qpair->domain); | ||||
| if (bus_dmamem_alloc(qpair->dma_tag, (void **)&queuemem, | if (bus_dmamem_alloc(qpair->dma_tag, (void **)&queuemem, | ||||
| BUS_DMA_NOWAIT, &qpair->queuemem_map)) { | BUS_DMA_COHERENT | BUS_DMA_NOWAIT, &qpair->queuemem_map)) { | ||||
| nvme_printf(ctrlr, "failed to alloc qpair memory\n"); | nvme_printf(ctrlr, "failed to alloc qpair memory\n"); | ||||
| goto out; | goto out; | ||||
| } | } | ||||
| if (bus_dmamap_load(qpair->dma_tag, qpair->queuemem_map, | if (bus_dmamap_load(qpair->dma_tag, qpair->queuemem_map, | ||||
| queuemem, allocsz, nvme_single_map, &queuemem_phys, 0) != 0) { | queuemem, allocsz, nvme_single_map, &queuemem_phys, 0) != 0) { | ||||
| nvme_printf(ctrlr, "failed to load qpair memory\n"); | nvme_printf(ctrlr, "failed to load qpair memory\n"); | ||||
| bus_dmamem_free(qpair->dma_tag, qpair->cmd, | bus_dmamem_free(qpair->dma_tag, qpair->cmd, | ||||
| ▲ Show 20 Lines • Show All 243 Lines • ▼ Show 20 Lines | nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr) | ||||
| /* Copy the command from the tracker to the submission queue. */ | /* Copy the command from the tracker to the submission queue. */ | ||||
| memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); | memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); | ||||
| if (++qpair->sq_tail == qpair->num_entries) | if (++qpair->sq_tail == qpair->num_entries) | ||||
| qpair->sq_tail = 0; | qpair->sq_tail = 0; | ||||
| bus_dmamap_sync(qpair->dma_tag, qpair->queuemem_map, | bus_dmamap_sync(qpair->dma_tag, qpair->queuemem_map, | ||||
| BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||
| #ifndef __powerpc__ | #if !defined( __powerpc__) && !defined( __aarch64__) && !defined( __arm__) | ||||
| /* | /* | ||||
| * powerpc's bus_dmamap_sync() already includes a heavyweight sync, but | * powerpc's bus_dmamap_sync() already includes a heavyweight sync, but | ||||
| * no other archs do. | * no other archs do. | ||||
| */ | */ | ||||
| wmb(); | wmb(); | ||||
| #endif | #endif | ||||
| bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, | bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, | ||||
| ▲ Show 20 Lines • Show All 309 Lines • Show Last 20 Lines | |||||