Differential D16404 Diff 46775 usr.sbin/bhyve/pci_nvme.c

Changeset View

Standalone View

usr.sbin/bhyve/pci_nvme.c

Show First 20 Lines • Show All 111 Lines • ▼ Show 20 Lines
};		};

enum nvme_cmd_cdw11 {		enum nvme_cmd_cdw11 {
NVME_CMD_CDW11_PC = 0x0001,		NVME_CMD_CDW11_PC = 0x0001,
NVME_CMD_CDW11_IEN = 0x0002,		NVME_CMD_CDW11_IEN = 0x0002,
NVME_CMD_CDW11_IV = 0xFFFF0000,		NVME_CMD_CDW11_IV = 0xFFFF0000,
};		};

#define NVME_CMD_GET_OPC(opc) \
((opc) >> NVME_CMD_OPC_SHIFT & NVME_CMD_OPC_MASK)

#define NVME_CQ_INTEN 0x01		#define NVME_CQ_INTEN 0x01
#define NVME_CQ_INTCOAL 0x02		#define NVME_CQ_INTCOAL 0x02

struct nvme_completion_queue {		struct nvme_completion_queue {
struct nvme_completion *qbase;		struct nvme_completion *qbase;
uint32_t size;		uint32_t size;
uint16_t tail; /* nvme progress */		uint16_t tail; /* nvme progress */
uint16_t head; /* guest progress */		uint16_t head; /* guest progress */
▲ Show 20 Lines • Show All 756 Lines • ▼ Show 20 Lines	pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)
}		}

DPRINTF(("sqhead %u, tail %u\r\n", sqhead, sq->tail));		DPRINTF(("sqhead %u, tail %u\r\n", sqhead, sq->tail));

while (sqhead != atomic_load_acq_short(&sq->tail)) {		while (sqhead != atomic_load_acq_short(&sq->tail)) {
cmd = &(sq->qbase)[sqhead];		cmd = &(sq->qbase)[sqhead];
compl.status = 0;		compl.status = 0;

switch (NVME_CMD_GET_OPC(cmd->opc_fuse)) {		switch (cmd->opc) {
case NVME_OPC_DELETE_IO_SQ:		case NVME_OPC_DELETE_IO_SQ:
DPRINTF(("%s command DELETE_IO_SQ\r\n", __func__));		DPRINTF(("%s command DELETE_IO_SQ\r\n", __func__));
do_intr \|= nvme_opc_delete_io_sq(sc, cmd, &compl);		do_intr \|= nvme_opc_delete_io_sq(sc, cmd, &compl);
break;		break;
case NVME_OPC_CREATE_IO_SQ:		case NVME_OPC_CREATE_IO_SQ:
DPRINTF(("%s command CREATE_IO_SQ\r\n", __func__));		DPRINTF(("%s command CREATE_IO_SQ\r\n", __func__));
do_intr \|= nvme_opc_create_io_sq(sc, cmd, &compl);		do_intr \|= nvme_opc_create_io_sq(sc, cmd, &compl);
break;		break;
Show All 28 Lines	while (sqhead != atomic_load_acq_short(&sq->tail)) {
case NVME_OPC_ASYNC_EVENT_REQUEST:		case NVME_OPC_ASYNC_EVENT_REQUEST:
DPRINTF(("%s command ASYNC_EVENT_REQ\r\n", __func__));		DPRINTF(("%s command ASYNC_EVENT_REQ\r\n", __func__));
/* XXX dont care, unhandled for now		/* XXX dont care, unhandled for now
do_intr \|= nvme_opc_async_event_req(sc, cmd, &compl);		do_intr \|= nvme_opc_async_event_req(sc, cmd, &compl);
*/		*/
break;		break;
default:		default:
WPRINTF(("0x%x command is not implemented\r\n",		WPRINTF(("0x%x command is not implemented\r\n",
NVME_CMD_GET_OPC(cmd->opc_fuse)));		cmd->opc));
}		}

/* for now skip async event generation */		/* for now skip async event generation */
if (NVME_CMD_GET_OPC(cmd->opc_fuse) !=		if (cmd->opc != NVME_OPC_ASYNC_EVENT_REQUEST) {
NVME_OPC_ASYNC_EVENT_REQUEST) {
struct nvme_completion *cp;		struct nvme_completion *cp;
int phase;		int phase;

cq = &sc->compl_queues[0];		cq = &sc->compl_queues[0];

cp = &(cq->qbase)[cq->tail];		cp = &(cq->qbase)[cq->tail];
cp->sqid = 0;		cp->sqid = 0;
cp->sqhd = sqhead;		cp->sqhd = sqhead;
▲ Show 20 Lines • Show All 257 Lines • ▼ Show 20 Lines	while (sqhead != atomic_load_acq_short(&sq->tail)) {

/* TODO: support scatter gather list handling */		/* TODO: support scatter gather list handling */

cmd = &sq->qbase[sqhead];		cmd = &sq->qbase[sqhead];
sqhead = (sqhead + 1) % sq->size;		sqhead = (sqhead + 1) % sq->size;

lba = ((uint64_t)cmd->cdw11 << 32) \| cmd->cdw10;		lba = ((uint64_t)cmd->cdw11 << 32) \| cmd->cdw10;

if (NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_FLUSH) {		if (cmd->opc == NVME_OPC_FLUSH) {
pci_nvme_status_genc(&status, NVME_SC_SUCCESS);		pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,		pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
status, 1);		status, 1);

continue;		continue;
} else if (NVME_CMD_GET_OPC(cmd->opc_fuse) == 0x08) {		} else if (cmd->opc == 0x08) {
/* TODO: write zeroes */		/* TODO: write zeroes */
WPRINTF(("%s write zeroes lba 0x%lx blocks %u\r\n",		WPRINTF(("%s write zeroes lba 0x%lx blocks %u\r\n",
__func__, lba, cmd->cdw12 & 0xFFFF));		__func__, lba, cmd->cdw12 & 0xFFFF));
pci_nvme_status_genc(&status, NVME_SC_SUCCESS);		pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,		pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
status, 1);		status, 1);

continue;		continue;
Show All 12 Lines	while (sqhead != atomic_load_acq_short(&sq->tail)) {
/*		/*
* If data starts mid-page and flows into the next page, then		* If data starts mid-page and flows into the next page, then
* increase page count		* increase page count
*/		*/

DPRINTF(("[h%u:t%u:n%u] %s starting LBA 0x%lx blocks %lu "		DPRINTF(("[h%u:t%u:n%u] %s starting LBA 0x%lx blocks %lu "
"(%lu-bytes)\r\n",		"(%lu-bytes)\r\n",
sqhead==0 ? sq->size-1 : sqhead-1, sq->tail, sq->size,		sqhead==0 ? sq->size-1 : sqhead-1, sq->tail, sq->size,
NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_WRITE ?		cmd->opc == NVME_OPC_WRITE ?
"WRITE" : "READ",		"WRITE" : "READ",
lba, nblocks, bytes));		lba, nblocks, bytes));

cmd->prp1 &= ~(0x03UL);		cmd->prp1 &= ~(0x03UL);
cmd->prp2 &= ~(0x03UL);		cmd->prp2 &= ~(0x03UL);

DPRINTF((" prp1 0x%lx prp2 0x%lx\r\n", cmd->prp1, cmd->prp2));		DPRINTF((" prp1 0x%lx prp2 0x%lx\r\n", cmd->prp1, cmd->prp2));

size = bytes;		size = bytes;
lba *= sc->nvstore.sectsz;		lba *= sc->nvstore.sectsz;

cpsz = PAGE_SIZE - (cmd->prp1 % PAGE_SIZE);		cpsz = PAGE_SIZE - (cmd->prp1 % PAGE_SIZE);

if (cpsz > bytes)		if (cpsz > bytes)
cpsz = bytes;		cpsz = bytes;

if (req != NULL) {		if (req != NULL) {
req->io_req.br_offset = ((uint64_t)cmd->cdw11 << 32) \|		req->io_req.br_offset = ((uint64_t)cmd->cdw11 << 32) \|
cmd->cdw10;		cmd->cdw10;
req->opc = NVME_CMD_GET_OPC(cmd->opc_fuse);		req->opc = cmd->opc;
req->cid = cmd->cid;		req->cid = cmd->cid;
req->nsid = cmd->nsid;		req->nsid = cmd->nsid;
}		}

err = pci_nvme_append_iov_req(sc, req, cmd->prp1, cpsz,		err = pci_nvme_append_iov_req(sc, req, cmd->prp1, cpsz,
NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_WRITE, lba);		cmd->opc == NVME_OPC_WRITE, lba);
lba += cpsz;		lba += cpsz;
size -= cpsz;		size -= cpsz;

if (size == 0)		if (size == 0)
goto iodone;		goto iodone;

if (size <= PAGE_SIZE) {		if (size <= PAGE_SIZE) {
/* prp2 is second (and final) page in transfer */		/* prp2 is second (and final) page in transfer */

err = pci_nvme_append_iov_req(sc, req, cmd->prp2,		err = pci_nvme_append_iov_req(sc, req, cmd->prp2,
size,		size,
NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_WRITE,		cmd->opc == NVME_OPC_WRITE,
lba);		lba);
} else {		} else {
uint64_t *prp_list;		uint64_t *prp_list;
int i;		int i;

/* prp2 is pointer to a physical region page list */		/* prp2 is pointer to a physical region page list */
prp_list = paddr_guest2host(sc->nsc_pi->pi_vmctx,		prp_list = paddr_guest2host(sc->nsc_pi->pi_vmctx,
cmd->prp2, PAGE_SIZE);		cmd->prp2, PAGE_SIZE);
Show All 17 Lines	if (size <= PAGE_SIZE) {
if (prp_list[i] == 0) {		if (prp_list[i] == 0) {
WPRINTF(("PRP2[%d] = 0 !!!\r\n", i));		WPRINTF(("PRP2[%d] = 0 !!!\r\n", i));
err = 1;		err = 1;
break;		break;
}		}

err = pci_nvme_append_iov_req(sc, req,		err = pci_nvme_append_iov_req(sc, req,
prp_list[i], cpsz,		prp_list[i], cpsz,
NVME_CMD_GET_OPC(cmd->opc_fuse) ==		cmd->opc == NVME_OPC_WRITE, lba);
NVME_OPC_WRITE, lba);
if (err)		if (err)
break;		break;

lba += cpsz;		lba += cpsz;
size -= cpsz;		size -= cpsz;
i++;		i++;
}		}
}		}
Show All 14 Lines	iodone:


if (err)		if (err)
goto do_error;		goto do_error;

req->io_req.br_callback = pci_nvme_io_done;		req->io_req.br_callback = pci_nvme_io_done;

err = 0;		err = 0;
switch (NVME_CMD_GET_OPC(cmd->opc_fuse)) {		switch (cmd->opc) {
case NVME_OPC_READ:		case NVME_OPC_READ:
err = blockif_read(sc->nvstore.ctx, &req->io_req);		err = blockif_read(sc->nvstore.ctx, &req->io_req);
break;		break;
case NVME_OPC_WRITE:		case NVME_OPC_WRITE:
err = blockif_write(sc->nvstore.ctx, &req->io_req);		err = blockif_write(sc->nvstore.ctx, &req->io_req);
break;		break;
default:		default:
WPRINTF(("%s unhandled io command 0x%x\r\n",		WPRINTF(("%s unhandled io command 0x%x\r\n",
__func__, NVME_CMD_GET_OPC(cmd->opc_fuse)));		__func__, cmd->opc));
err = 1;		err = 1;
}		}

do_error:		do_error:
if (err) {		if (err) {
uint16_t status;		uint16_t status;

pci_nvme_status_genc(&status,		pci_nvme_status_genc(&status,
▲ Show 20 Lines • Show All 478 Lines • Show Last 20 Lines