Changeset View
Changeset View
Standalone View
Standalone View
head/usr.sbin/bhyve/pci_nvme.c
Show First 20 Lines • Show All 111 Lines • ▼ Show 20 Lines | |||||
}; | }; | ||||
enum nvme_cmd_cdw11 { | enum nvme_cmd_cdw11 { | ||||
NVME_CMD_CDW11_PC = 0x0001, | NVME_CMD_CDW11_PC = 0x0001, | ||||
NVME_CMD_CDW11_IEN = 0x0002, | NVME_CMD_CDW11_IEN = 0x0002, | ||||
NVME_CMD_CDW11_IV = 0xFFFF0000, | NVME_CMD_CDW11_IV = 0xFFFF0000, | ||||
}; | }; | ||||
#define NVME_CMD_GET_OPC(opc) \ | |||||
((opc) >> NVME_CMD_OPC_SHIFT & NVME_CMD_OPC_MASK) | |||||
#define NVME_CQ_INTEN 0x01 | #define NVME_CQ_INTEN 0x01 | ||||
#define NVME_CQ_INTCOAL 0x02 | #define NVME_CQ_INTCOAL 0x02 | ||||
struct nvme_completion_queue { | struct nvme_completion_queue { | ||||
struct nvme_completion *qbase; | struct nvme_completion *qbase; | ||||
uint32_t size; | uint32_t size; | ||||
uint16_t tail; /* nvme progress */ | uint16_t tail; /* nvme progress */ | ||||
uint16_t head; /* guest progress */ | uint16_t head; /* guest progress */ | ||||
▲ Show 20 Lines • Show All 756 Lines • ▼ Show 20 Lines | pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value) | ||||
} | } | ||||
DPRINTF(("sqhead %u, tail %u\r\n", sqhead, sq->tail)); | DPRINTF(("sqhead %u, tail %u\r\n", sqhead, sq->tail)); | ||||
while (sqhead != atomic_load_acq_short(&sq->tail)) { | while (sqhead != atomic_load_acq_short(&sq->tail)) { | ||||
cmd = &(sq->qbase)[sqhead]; | cmd = &(sq->qbase)[sqhead]; | ||||
compl.status = 0; | compl.status = 0; | ||||
switch (NVME_CMD_GET_OPC(cmd->opc_fuse)) { | switch (cmd->opc) { | ||||
case NVME_OPC_DELETE_IO_SQ: | case NVME_OPC_DELETE_IO_SQ: | ||||
DPRINTF(("%s command DELETE_IO_SQ\r\n", __func__)); | DPRINTF(("%s command DELETE_IO_SQ\r\n", __func__)); | ||||
do_intr |= nvme_opc_delete_io_sq(sc, cmd, &compl); | do_intr |= nvme_opc_delete_io_sq(sc, cmd, &compl); | ||||
break; | break; | ||||
case NVME_OPC_CREATE_IO_SQ: | case NVME_OPC_CREATE_IO_SQ: | ||||
DPRINTF(("%s command CREATE_IO_SQ\r\n", __func__)); | DPRINTF(("%s command CREATE_IO_SQ\r\n", __func__)); | ||||
do_intr |= nvme_opc_create_io_sq(sc, cmd, &compl); | do_intr |= nvme_opc_create_io_sq(sc, cmd, &compl); | ||||
break; | break; | ||||
Show All 28 Lines | while (sqhead != atomic_load_acq_short(&sq->tail)) { | ||||
case NVME_OPC_ASYNC_EVENT_REQUEST: | case NVME_OPC_ASYNC_EVENT_REQUEST: | ||||
DPRINTF(("%s command ASYNC_EVENT_REQ\r\n", __func__)); | DPRINTF(("%s command ASYNC_EVENT_REQ\r\n", __func__)); | ||||
/* XXX dont care, unhandled for now | /* XXX dont care, unhandled for now | ||||
do_intr |= nvme_opc_async_event_req(sc, cmd, &compl); | do_intr |= nvme_opc_async_event_req(sc, cmd, &compl); | ||||
*/ | */ | ||||
break; | break; | ||||
default: | default: | ||||
WPRINTF(("0x%x command is not implemented\r\n", | WPRINTF(("0x%x command is not implemented\r\n", | ||||
NVME_CMD_GET_OPC(cmd->opc_fuse))); | cmd->opc)); | ||||
} | } | ||||
/* for now skip async event generation */ | /* for now skip async event generation */ | ||||
if (NVME_CMD_GET_OPC(cmd->opc_fuse) != | if (cmd->opc != NVME_OPC_ASYNC_EVENT_REQUEST) { | ||||
NVME_OPC_ASYNC_EVENT_REQUEST) { | |||||
struct nvme_completion *cp; | struct nvme_completion *cp; | ||||
int phase; | int phase; | ||||
cq = &sc->compl_queues[0]; | cq = &sc->compl_queues[0]; | ||||
cp = &(cq->qbase)[cq->tail]; | cp = &(cq->qbase)[cq->tail]; | ||||
cp->sqid = 0; | cp->sqid = 0; | ||||
cp->sqhd = sqhead; | cp->sqhd = sqhead; | ||||
▲ Show 20 Lines • Show All 252 Lines • ▼ Show 20 Lines | while (sqhead != atomic_load_acq_short(&sq->tail)) { | ||||
/* TODO: support scatter gather list handling */ | /* TODO: support scatter gather list handling */ | ||||
cmd = &sq->qbase[sqhead]; | cmd = &sq->qbase[sqhead]; | ||||
sqhead = (sqhead + 1) % sq->size; | sqhead = (sqhead + 1) % sq->size; | ||||
lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10; | lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10; | ||||
if (NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_FLUSH) { | if (cmd->opc == NVME_OPC_FLUSH) { | ||||
pci_nvme_status_genc(&status, NVME_SC_SUCCESS); | pci_nvme_status_genc(&status, NVME_SC_SUCCESS); | ||||
pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, | pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, | ||||
status, 1); | status, 1); | ||||
continue; | continue; | ||||
} else if (NVME_CMD_GET_OPC(cmd->opc_fuse) == 0x08) { | } else if (cmd->opc == 0x08) { | ||||
/* TODO: write zeroes */ | /* TODO: write zeroes */ | ||||
WPRINTF(("%s write zeroes lba 0x%lx blocks %u\r\n", | WPRINTF(("%s write zeroes lba 0x%lx blocks %u\r\n", | ||||
__func__, lba, cmd->cdw12 & 0xFFFF)); | __func__, lba, cmd->cdw12 & 0xFFFF)); | ||||
pci_nvme_status_genc(&status, NVME_SC_SUCCESS); | pci_nvme_status_genc(&status, NVME_SC_SUCCESS); | ||||
pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, | pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0, | ||||
status, 1); | status, 1); | ||||
continue; | continue; | ||||
Show All 12 Lines | while (sqhead != atomic_load_acq_short(&sq->tail)) { | ||||
/* | /* | ||||
* If data starts mid-page and flows into the next page, then | * If data starts mid-page and flows into the next page, then | ||||
* increase page count | * increase page count | ||||
*/ | */ | ||||
DPRINTF(("[h%u:t%u:n%u] %s starting LBA 0x%lx blocks %lu " | DPRINTF(("[h%u:t%u:n%u] %s starting LBA 0x%lx blocks %lu " | ||||
"(%lu-bytes)\r\n", | "(%lu-bytes)\r\n", | ||||
sqhead==0 ? sq->size-1 : sqhead-1, sq->tail, sq->size, | sqhead==0 ? sq->size-1 : sqhead-1, sq->tail, sq->size, | ||||
NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_WRITE ? | cmd->opc == NVME_OPC_WRITE ? | ||||
"WRITE" : "READ", | "WRITE" : "READ", | ||||
lba, nblocks, bytes)); | lba, nblocks, bytes)); | ||||
cmd->prp1 &= ~(0x03UL); | cmd->prp1 &= ~(0x03UL); | ||||
cmd->prp2 &= ~(0x03UL); | cmd->prp2 &= ~(0x03UL); | ||||
DPRINTF((" prp1 0x%lx prp2 0x%lx\r\n", cmd->prp1, cmd->prp2)); | DPRINTF((" prp1 0x%lx prp2 0x%lx\r\n", cmd->prp1, cmd->prp2)); | ||||
size = bytes; | size = bytes; | ||||
lba *= sc->nvstore.sectsz; | lba *= sc->nvstore.sectsz; | ||||
cpsz = PAGE_SIZE - (cmd->prp1 % PAGE_SIZE); | cpsz = PAGE_SIZE - (cmd->prp1 % PAGE_SIZE); | ||||
if (cpsz > bytes) | if (cpsz > bytes) | ||||
cpsz = bytes; | cpsz = bytes; | ||||
if (req != NULL) { | if (req != NULL) { | ||||
req->io_req.br_offset = ((uint64_t)cmd->cdw11 << 32) | | req->io_req.br_offset = ((uint64_t)cmd->cdw11 << 32) | | ||||
cmd->cdw10; | cmd->cdw10; | ||||
req->opc = NVME_CMD_GET_OPC(cmd->opc_fuse); | req->opc = cmd->opc; | ||||
req->cid = cmd->cid; | req->cid = cmd->cid; | ||||
req->nsid = cmd->nsid; | req->nsid = cmd->nsid; | ||||
} | } | ||||
err = pci_nvme_append_iov_req(sc, req, cmd->prp1, cpsz, | err = pci_nvme_append_iov_req(sc, req, cmd->prp1, cpsz, | ||||
NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_WRITE, lba); | cmd->opc == NVME_OPC_WRITE, lba); | ||||
lba += cpsz; | lba += cpsz; | ||||
size -= cpsz; | size -= cpsz; | ||||
if (size == 0) | if (size == 0) | ||||
goto iodone; | goto iodone; | ||||
if (size <= PAGE_SIZE) { | if (size <= PAGE_SIZE) { | ||||
/* prp2 is second (and final) page in transfer */ | /* prp2 is second (and final) page in transfer */ | ||||
err = pci_nvme_append_iov_req(sc, req, cmd->prp2, | err = pci_nvme_append_iov_req(sc, req, cmd->prp2, | ||||
size, | size, | ||||
NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_WRITE, | cmd->opc == NVME_OPC_WRITE, | ||||
lba); | lba); | ||||
} else { | } else { | ||||
uint64_t *prp_list; | uint64_t *prp_list; | ||||
int i; | int i; | ||||
/* prp2 is pointer to a physical region page list */ | /* prp2 is pointer to a physical region page list */ | ||||
prp_list = paddr_guest2host(sc->nsc_pi->pi_vmctx, | prp_list = paddr_guest2host(sc->nsc_pi->pi_vmctx, | ||||
cmd->prp2, PAGE_SIZE); | cmd->prp2, PAGE_SIZE); | ||||
Show All 17 Lines | if (size <= PAGE_SIZE) { | ||||
if (prp_list[i] == 0) { | if (prp_list[i] == 0) { | ||||
WPRINTF(("PRP2[%d] = 0 !!!\r\n", i)); | WPRINTF(("PRP2[%d] = 0 !!!\r\n", i)); | ||||
err = 1; | err = 1; | ||||
break; | break; | ||||
} | } | ||||
err = pci_nvme_append_iov_req(sc, req, | err = pci_nvme_append_iov_req(sc, req, | ||||
prp_list[i], cpsz, | prp_list[i], cpsz, | ||||
NVME_CMD_GET_OPC(cmd->opc_fuse) == | cmd->opc == NVME_OPC_WRITE, lba); | ||||
NVME_OPC_WRITE, lba); | |||||
if (err) | if (err) | ||||
break; | break; | ||||
lba += cpsz; | lba += cpsz; | ||||
size -= cpsz; | size -= cpsz; | ||||
i++; | i++; | ||||
} | } | ||||
} | } | ||||
Show All 14 Lines | iodone: | ||||
if (err) | if (err) | ||||
goto do_error; | goto do_error; | ||||
req->io_req.br_callback = pci_nvme_io_done; | req->io_req.br_callback = pci_nvme_io_done; | ||||
err = 0; | err = 0; | ||||
switch (NVME_CMD_GET_OPC(cmd->opc_fuse)) { | switch (cmd->opc) { | ||||
case NVME_OPC_READ: | case NVME_OPC_READ: | ||||
err = blockif_read(sc->nvstore.ctx, &req->io_req); | err = blockif_read(sc->nvstore.ctx, &req->io_req); | ||||
break; | break; | ||||
case NVME_OPC_WRITE: | case NVME_OPC_WRITE: | ||||
err = blockif_write(sc->nvstore.ctx, &req->io_req); | err = blockif_write(sc->nvstore.ctx, &req->io_req); | ||||
break; | break; | ||||
default: | default: | ||||
WPRINTF(("%s unhandled io command 0x%x\r\n", | WPRINTF(("%s unhandled io command 0x%x\r\n", | ||||
__func__, NVME_CMD_GET_OPC(cmd->opc_fuse))); | __func__, cmd->opc)); | ||||
err = 1; | err = 1; | ||||
} | } | ||||
do_error: | do_error: | ||||
if (err) { | if (err) { | ||||
uint16_t status; | uint16_t status; | ||||
pci_nvme_status_genc(&status, | pci_nvme_status_genc(&status, | ||||
▲ Show 20 Lines • Show All 484 Lines • Show Last 20 Lines |