Differential D24882 Diff 73827 head/usr.sbin/bhyve/pci_nvme.c

Changeset View

Standalone View

head/usr.sbin/bhyve/pci_nvme.c

Show First 20 Lines • Show All 323 Lines • ▼ Show 20 Lines

static __inline void		static __inline void
pci_nvme_status_genc(uint16_t *status, uint16_t code)		pci_nvme_status_genc(uint16_t *status, uint16_t code)
{		{

pci_nvme_status_tc(status, NVME_SCT_GENERIC, code);		pci_nvme_status_tc(status, NVME_SCT_GENERIC, code);
}		}

static __inline void
pci_nvme_toggle_phase(uint16_t *status, int prev)
{

if (prev)
*status &= ~NVME_STATUS_P;
else
*status \|= NVME_STATUS_P;
}

/*		/*
* Initialize the requested number or IO Submission and Completion Queues.		* Initialize the requested number or IO Submission and Completion Queues.
* Admin queues are allocated implicitly.		* Admin queues are allocated implicitly.
*/		*/
static void		static void
pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq)		pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq)
{		{
uint32_t i;		uint32_t i;
▲ Show 20 Lines • Show All 262 Lines • ▼ Show 20 Lines	pci_nvme_init_controller(struct vmctx ctx, struct pci_nvme_softc sc)
DPRINTF("%s mapping Admin-SQ guest 0x%lx, host: %p",		DPRINTF("%s mapping Admin-SQ guest 0x%lx, host: %p",
__func__, sc->regs.asq, sc->submit_queues[0].qbase);		__func__, sc->regs.asq, sc->submit_queues[0].qbase);

acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &		acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &
NVME_AQA_REG_ACQS_MASK) + 1;		NVME_AQA_REG_ACQS_MASK) + 1;
sc->compl_queues[0].size = acqs;		sc->compl_queues[0].size = acqs;
sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq,		sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq,
sizeof(struct nvme_completion) * acqs);		sizeof(struct nvme_completion) * acqs);

DPRINTF("%s mapping Admin-CQ guest 0x%lx, host: %p",		DPRINTF("%s mapping Admin-CQ guest 0x%lx, host: %p",
__func__, sc->regs.acq, sc->compl_queues[0].qbase);		__func__, sc->regs.acq, sc->compl_queues[0].qbase);
}		}

static int		static int
nvme_prp_memcpy(struct vmctx ctx, uint64_t prp1, uint64_t prp2, uint8_t b,		nvme_prp_memcpy(struct vmctx ctx, uint64_t prp1, uint64_t prp2, uint8_t b,
size_t len, enum nvme_copy_dir dir)		size_t len, enum nvme_copy_dir dir)
{		{
Show All 35 Lines	nvme_prp_memcpy(struct vmctx ctx, uint64_t prp1, uint64_t prp2, uint8_t b,
if (dir == NVME_COPY_TO_PRP)		if (dir == NVME_COPY_TO_PRP)
memcpy(p, b, len);		memcpy(p, b, len);
else		else
memcpy(b, p, len);		memcpy(b, p, len);

return (0);		return (0);
}		}

		/*
		* Write a Completion Queue Entry update
		*
		* Write the completion and update the doorbell value
		*/
		static void
		pci_nvme_cq_update(struct pci_nvme_softc *sc,
		struct nvme_completion_queue *cq,
		uint32_t cdw0,
		uint16_t cid,
		uint16_t sqid,
		uint16_t status)
		{
		struct nvme_submission_queue *sq = &sc->submit_queues[sqid];
		struct nvme_completion *cqe;

		assert(cq->qbase != NULL);

		pthread_mutex_lock(&cq->mtx);

		cqe = &cq->qbase[cq->tail];

		/* Flip the phase bit */
		status \|= (cqe->status ^ NVME_STATUS_P) & NVME_STATUS_P_MASK;

		cqe->cdw0 = cdw0;
		cqe->sqhd = sq->head;
		cqe->sqid = sqid;
		cqe->cid = cid;
		cqe->status = status;

		cq->tail++;
		if (cq->tail >= cq->size) {
		cq->tail = 0;
		}

		pthread_mutex_unlock(&cq->mtx);
		}

static int		static int
nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,		nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
struct nvme_completion* compl)		struct nvme_completion* compl)
{		{
uint16_t qid = command->cdw10 & 0xffff;		uint16_t qid = command->cdw10 & 0xffff;

DPRINTF("%s DELETE_IO_SQ %u", __func__, qid);		DPRINTF("%s DELETE_IO_SQ %u", __func__, qid);
if (qid == 0 \|\| qid > sc->num_squeues) {		if (qid == 0 \|\| qid > sc->num_squeues) {
▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines	nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);		pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
return (1);		return (1);
}		}

static int		static int
nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,		nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
struct nvme_completion* compl)		struct nvme_completion* compl)
{		{

if (command->cdw11 & NVME_CMD_CDW11_PC) {		if (command->cdw11 & NVME_CMD_CDW11_PC) {
uint16_t qid = command->cdw10 & 0xffff;		uint16_t qid = command->cdw10 & 0xffff;
struct nvme_completion_queue *ncq;		struct nvme_completion_queue *ncq;

if ((qid == 0) \|\| (qid > sc->num_cqueues)) {		if ((qid == 0) \|\| (qid > sc->num_cqueues)) {
WPRINTF("%s queue index %u > num_cqueues %u",		WPRINTF("%s queue index %u > num_cqueues %u",
__func__, qid, sc->num_cqueues);		__func__, qid, sc->num_cqueues);
pci_nvme_status_tc(&compl->status,		pci_nvme_status_tc(&compl->status,
▲ Show 20 Lines • Show All 418 Lines • ▼ Show 20 Lines	while (sqhead != atomic_load_acq_short(&sq->tail)) {
default:		default:
WPRINTF("0x%x command is not implemented",		WPRINTF("0x%x command is not implemented",
cmd->opc);		cmd->opc);
pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);		pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);
}		}
sqhead = (sqhead + 1) % sq->size;		sqhead = (sqhead + 1) % sq->size;

if (NVME_COMPLETION_VALID(compl)) {		if (NVME_COMPLETION_VALID(compl)) {
struct nvme_completion *cp;		pci_nvme_cq_update(sc, &sc->compl_queues[0],
int phase;		compl.cdw0,
		cmd->cid,
pthread_mutex_lock(&cq->mtx);		0, /* SQID */
		compl.status);
cp = &(cq->qbase)[cq->tail];
cp->cdw0 = compl.cdw0;
cp->sqid = 0;
cp->sqhd = sqhead;
cp->cid = cmd->cid;

phase = NVME_STATUS_GET_P(cp->status);
cp->status = compl.status;
pci_nvme_toggle_phase(&cp->status, phase);

cq->tail = (cq->tail + 1) % cq->size;

pthread_mutex_unlock(&cq->mtx);
}		}
}		}

DPRINTF("setting sqhead %u", sqhead);		DPRINTF("setting sqhead %u", sqhead);
sq->head = sqhead;		sq->head = sqhead;

if (cq->head != cq->tail)		if (cq->head != cq->tail)
pci_generate_msix(sc->nsc_pi, 0);		pci_generate_msix(sc->nsc_pi, 0);
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines
}		}

static void		static void
pci_nvme_set_completion(struct pci_nvme_softc *sc,		pci_nvme_set_completion(struct pci_nvme_softc *sc,
struct nvme_submission_queue *sq, int sqid, uint16_t cid,		struct nvme_submission_queue *sq, int sqid, uint16_t cid,
uint32_t cdw0, uint16_t status)		uint32_t cdw0, uint16_t status)
{		{
struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];		struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];
struct nvme_completion *compl;
int phase;

DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x",		DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x",
__func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),		__func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),
NVME_STATUS_GET_SC(status));		NVME_STATUS_GET_SC(status));

pthread_mutex_lock(&cq->mtx);		pci_nvme_cq_update(sc, cq,
		0, /* CDW0 */
assert(cq->qbase != NULL);		cid,
		sqid,
compl = &cq->qbase[cq->tail];		status);

compl->cdw0 = cdw0;
compl->sqid = sqid;
compl->sqhd = sq->head;
compl->cid = cid;

// toggle phase
phase = NVME_STATUS_GET_P(compl->status);
compl->status = status;
pci_nvme_toggle_phase(&compl->status, phase);

cq->tail = (cq->tail + 1) % cq->size;

pthread_mutex_unlock(&cq->mtx);

if (cq->head != cq->tail) {		if (cq->head != cq->tail) {
if (cq->intr_en & NVME_CQ_INTEN) {		if (cq->intr_en & NVME_CQ_INTEN) {
pci_generate_msix(sc->nsc_pi, cq->intr_vec);		pci_generate_msix(sc->nsc_pi, cq->intr_vec);
} else {		} else {
DPRINTF("%s: CQ%u interrupt disabled",		DPRINTF("%s: CQ%u interrupt disabled",
__func__, sq->cqid);		__func__, sq->cqid);
}		}
▲ Show 20 Lines • Show All 957 Lines • Show Last 20 Lines