Differential D19841 Diff 71924 usr.sbin/bhyve/pci_nvme.c

Changeset View

Standalone View

usr.sbin/bhyve/pci_nvme.c

Show First 20 Lines • Show All 147 Lines • ▼ Show 20 Lines	enum nvme_copy_dir {
NVME_COPY_FROM_PRP,		NVME_COPY_FROM_PRP,
};		};

#define NVME_CQ_INTEN 0x01		#define NVME_CQ_INTEN 0x01
#define NVME_CQ_INTCOAL 0x02		#define NVME_CQ_INTCOAL 0x02

struct nvme_completion_queue {		struct nvme_completion_queue {
struct nvme_completion *qbase;		struct nvme_completion *qbase;
		pthread_mutex_t mtx;
		jhbUnsubmitted Not Done Inline Actions Why did you move this earlier? Was there a padding hole? jhb: Why did you move this earlier? Was there a padding hole?
uint32_t size;		uint32_t size;
uint16_t tail; /* nvme progress */		uint16_t tail; /* nvme progress */
uint16_t head; /* guest progress */		uint16_t head; /* guest progress */
uint16_t intr_vec;		uint16_t intr_vec;
uint32_t intr_en;		uint32_t intr_en;
pthread_mutex_t mtx;
};		};
		jhbUnsubmitted Not Done Inline Actions There looks to be a 16-bit hole between intr_vec and intr_en FWIW. jhb: There looks to be a 16-bit hole between intr_vec and intr_en FWIW.

struct nvme_submission_queue {		struct nvme_submission_queue {
struct nvme_command *qbase;		struct nvme_command *qbase;
		pthread_mutex_t mtx;
		jhbUnsubmitted Not Done Inline Actions It looks like this pthread mutex replaces a home-grown spin lock on the busy flag? jhb: It looks like this pthread mutex replaces a home-grown spin lock on the busy flag?
uint32_t size;		uint32_t size;
uint16_t head; /* nvme progress */		uint16_t head; /* nvme progress */
uint16_t tail; /* guest progress */		uint16_t tail; /* guest progress */
uint16_t cqid; /* completion queue id */		uint16_t cqid; /* completion queue id */
int busy; /* queue is being processed */
int qpriority;		int qpriority;
};		};

enum nvme_storage_type {		enum nvme_storage_type {
NVME_STOR_BLOCKIF = 0,		NVME_STOR_BLOCKIF = 0,
NVME_STOR_RAM = 1,		NVME_STOR_RAM = 1,
};		};

▲ Show 20 Lines • Show All 155 Lines • ▼ Show 20 Lines
{		{

if (prev)		if (prev)
*status &= ~NVME_STATUS_P;		*status &= ~NVME_STATUS_P;
else		else
*status \|= NVME_STATUS_P;		*status \|= NVME_STATUS_P;
}		}

		/*
		* Initialize the requested number or IO Submission and Completion Queues.
		* Admin queues are allocated implicitly.
		*/
static void		static void
		pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq)
		{
		uint32_t i;

		/*
		* Allocate and initialize the Submission Queues
		*/
		if (nsq > NVME_QUEUES) {
		WPRINTF("%s: clamping number of SQ from %u to %u",
		__func__, nsq, NVME_QUEUES);
		impUnsubmitted Not Done Inline Actions this indentation looks wrong :) imp: this indentation looks wrong :)
		nsq = NVME_QUEUES;
		}

		sc->num_squeues = nsq;

		sc->submit_queues = calloc(sc->num_squeues + 1,
		jhbUnsubmitted Not Done Inline Actions Is the +1 for the Admin queue? If so, a comment to that effect might be helpful. jhb: Is the +1 for the Admin queue? If so, a comment to that effect might be helpful.
		sizeof(struct nvme_submission_queue));
		impUnsubmitted Not Done Inline Actions Ditto imp: Ditto
		if (sc->submit_queues == NULL) {
		WPRINTF("%s: SQ allocation failed", __func__);
		sc->num_squeues = 0;
		} else {
		struct nvme_submission_queue *sq = sc->submit_queues;

		for (i = 0; i < sc->num_squeues; i++)
		pthread_mutex_init(&sq[i].mtx, NULL);
		}

		/*
		* Allocate and initialize the Completion Queues
		*/
		if (ncq > NVME_QUEUES) {
		WPRINTF("%s: clamping number of CQ from %u to %u",
		__func__, ncq, NVME_QUEUES);
		ncq = NVME_QUEUES;
		}

		sc->num_cqueues = ncq;

		sc->compl_queues = calloc(sc->num_cqueues + 1,
		sizeof(struct nvme_completion_queue));
		if (sc->compl_queues == NULL) {
		WPRINTF("%s: CQ allocation failed", __func__);
		sc->num_cqueues = 0;
		} else {
		struct nvme_completion_queue *cq = sc->compl_queues;

		for (i = 0; i < sc->num_cqueues; i++)
		pthread_mutex_init(&cq[i].mtx, NULL);
		}
		}

		static void
pci_nvme_init_ctrldata(struct pci_nvme_softc *sc)		pci_nvme_init_ctrldata(struct pci_nvme_softc *sc)
{		{
struct nvme_controller_data *cd = &sc->ctrldata;		struct nvme_controller_data *cd = &sc->ctrldata;

cd->vid = 0xFB5D;		cd->vid = 0xFB5D;
cd->ssvid = 0x0000;		cd->ssvid = 0x0000;

cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' ');		cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' ');
▲ Show 20 Lines • Show All 142 Lines • ▼ Show 20 Lines	pci_nvme_init_logpages(struct pci_nvme_softc *sc)
memset(&sc->err_log, 0, sizeof(sc->err_log));		memset(&sc->err_log, 0, sizeof(sc->err_log));
memset(&sc->health_log, 0, sizeof(sc->health_log));		memset(&sc->health_log, 0, sizeof(sc->health_log));
memset(&sc->fw_log, 0, sizeof(sc->fw_log));		memset(&sc->fw_log, 0, sizeof(sc->fw_log));
}		}

static void		static void
pci_nvme_reset_locked(struct pci_nvme_softc *sc)		pci_nvme_reset_locked(struct pci_nvme_softc *sc)
{		{
		uint32_t i;

DPRINTF("%s", __func__);		DPRINTF("%s", __func__);

sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) \|		sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) \|
(1 << NVME_CAP_LO_REG_CQR_SHIFT) \|		(1 << NVME_CAP_LO_REG_CQR_SHIFT) \|
(60 << NVME_CAP_LO_REG_TO_SHIFT);		(60 << NVME_CAP_LO_REG_TO_SHIFT);

sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT;		sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT;

sc->regs.vs = 0x00010300; /* NVMe v1.3 */		sc->regs.vs = 0x00010300; /* NVMe v1.3 */

sc->regs.cc = 0;		sc->regs.cc = 0;
sc->regs.csts = 0;		sc->regs.csts = 0;

sc->num_cqueues = sc->num_squeues = sc->max_queues;		assert(sc->submit_queues != NULL);
if (sc->submit_queues != NULL) {
for (int i = 0; i < sc->num_squeues + 1; i++) {		for (i = 0; i < sc->num_squeues + 1; i++) {
/*
jhbUnsubmitted Not Done Inline Actions So why is it ok in the new code to reinit the admin queues during reset? Was it actually ok in the old code? jhb: So why is it ok in the new code to reinit the admin queues during reset? Was it actually ok in…
* The Admin Submission Queue is at index 0.
* It must not be changed at reset otherwise the
* emulation will be out of sync with the guest.
*/
if (i != 0) {
sc->submit_queues[i].qbase = NULL;		sc->submit_queues[i].qbase = NULL;
sc->submit_queues[i].size = 0;		sc->submit_queues[i].size = 0;
sc->submit_queues[i].cqid = 0;		sc->submit_queues[i].cqid = 0;
}
sc->submit_queues[i].tail = 0;		sc->submit_queues[i].tail = 0;
sc->submit_queues[i].head = 0;		sc->submit_queues[i].head = 0;
sc->submit_queues[i].busy = 0;
}		}
} else
sc->submit_queues = calloc(sc->num_squeues + 1,
sizeof(struct nvme_submission_queue));

if (sc->compl_queues != NULL) {		assert(sc->compl_queues != NULL);
for (int i = 0; i < sc->num_cqueues + 1; i++) {
/* See Admin Submission Queue note above */		for (i = 0; i < sc->num_cqueues + 1; i++) {
if (i != 0) {
sc->compl_queues[i].qbase = NULL;		sc->compl_queues[i].qbase = NULL;
sc->compl_queues[i].size = 0;		sc->compl_queues[i].size = 0;
}

sc->compl_queues[i].tail = 0;		sc->compl_queues[i].tail = 0;
sc->compl_queues[i].head = 0;		sc->compl_queues[i].head = 0;
}		}
} else {
sc->compl_queues = calloc(sc->num_cqueues + 1,
sizeof(struct nvme_completion_queue));

for (int i = 0; i < sc->num_cqueues + 1; i++)
pthread_mutex_init(&sc->compl_queues[i].mtx, NULL);
}		}
}

static void		static void
pci_nvme_reset(struct pci_nvme_softc *sc)		pci_nvme_reset(struct pci_nvme_softc *sc)
{		{
pthread_mutex_lock(&sc->mtx);		pthread_mutex_lock(&sc->mtx);
pci_nvme_reset_locked(sc);		pci_nvme_reset_locked(sc);
pthread_mutex_unlock(&sc->mtx);		pthread_mutex_unlock(&sc->mtx);
}		}
▲ Show 20 Lines • Show All 525 Lines • ▼ Show 20 Lines	pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)
struct nvme_completion_queue *cq;		struct nvme_completion_queue *cq;
uint16_t sqhead;		uint16_t sqhead;

DPRINTF("%s index %u", __func__, (uint32_t)value);		DPRINTF("%s index %u", __func__, (uint32_t)value);

sq = &sc->submit_queues[0];		sq = &sc->submit_queues[0];
cq = &sc->compl_queues[0];		cq = &sc->compl_queues[0];

sqhead = atomic_load_acq_short(&sq->head);		pthread_mutex_lock(&sq->mtx);

if (atomic_testandset_int(&sq->busy, 1)) {		sqhead = sq->head;
DPRINTF("%s SQ busy, head %u, tail %u",
__func__, sqhead, sq->tail);
return;
}

DPRINTF("sqhead %u, tail %u", sqhead, sq->tail);		DPRINTF("sqhead %u, tail %u", sqhead, sq->tail);

while (sqhead != atomic_load_acq_short(&sq->tail)) {		while (sqhead != atomic_load_acq_short(&sq->tail)) {
cmd = &(sq->qbase)[sqhead];		cmd = &(sq->qbase)[sqhead];
compl.cdw0 = 0;		compl.cdw0 = 0;
compl.status = 0;		compl.status = 0;

switch (cmd->opc) {		switch (cmd->opc) {
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines	default:
pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);		pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);
}		}
sqhead = (sqhead + 1) % sq->size;		sqhead = (sqhead + 1) % sq->size;

if (NVME_COMPLETION_VALID(compl)) {		if (NVME_COMPLETION_VALID(compl)) {
struct nvme_completion *cp;		struct nvme_completion *cp;
int phase;		int phase;

		pthread_mutex_lock(&cq->mtx);

cp = &(cq->qbase)[cq->tail];		cp = &(cq->qbase)[cq->tail];
cp->cdw0 = compl.cdw0;		cp->cdw0 = compl.cdw0;
cp->sqid = 0;		cp->sqid = 0;
cp->sqhd = sqhead;		cp->sqhd = sqhead;
cp->cid = cmd->cid;		cp->cid = cmd->cid;

phase = NVME_STATUS_GET_P(cp->status);		phase = NVME_STATUS_GET_P(cp->status);
cp->status = compl.status;		cp->status = compl.status;
pci_nvme_toggle_phase(&cp->status, phase);		pci_nvme_toggle_phase(&cp->status, phase);

cq->tail = (cq->tail + 1) % cq->size;		cq->tail = (cq->tail + 1) % cq->size;

		pthread_mutex_unlock(&cq->mtx);
}		}
}		}

DPRINTF("setting sqhead %u", sqhead);		DPRINTF("setting sqhead %u", sqhead);
atomic_store_short(&sq->head, sqhead);		sq->head = sqhead;
atomic_store_int(&sq->busy, 0);

if (cq->head != cq->tail)		if (cq->head != cq->tail)
pci_generate_msix(sc->nsc_pi, 0);		pci_generate_msix(sc->nsc_pi, 0);

		pthread_mutex_unlock(&sq->mtx);
		jhbUnsubmitted Not Done Inline Actions You could maybe unlock the mutex before asserting the interrupt. Not sure it will matter in this case if the same thread will answer any doorbell that the interrupt handler in the guest would ring. jhb: You could maybe unlock the mutex before asserting the interrupt. Not sure it will matter in…
}		}

static int		static int
pci_nvme_append_iov_req(struct pci_nvme_softc sc, struct pci_nvme_ioreq req,		pci_nvme_append_iov_req(struct pci_nvme_softc sc, struct pci_nvme_ioreq req,
uint64_t gpaddr, size_t size, int do_write, uint64_t lba)		uint64_t gpaddr, size_t size, int do_write, uint64_t lba)
{		{
int iovidx;		int iovidx;

▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines	else
memcpy(gptr, p, size);		memcpy(gptr, p, size);
}		}
return (0);		return (0);
}		}

static void		static void
pci_nvme_set_completion(struct pci_nvme_softc *sc,		pci_nvme_set_completion(struct pci_nvme_softc *sc,
struct nvme_submission_queue *sq, int sqid, uint16_t cid,		struct nvme_submission_queue *sq, int sqid, uint16_t cid,
uint32_t cdw0, uint16_t status, int ignore_busy)		uint32_t cdw0, uint16_t status)
{		{
struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];		struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];
struct nvme_completion *compl;		struct nvme_completion *compl;
int phase;		int phase;

DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x",		DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x",
__func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),		__func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),
NVME_STATUS_GET_SC(status));		NVME_STATUS_GET_SC(status));

pthread_mutex_lock(&cq->mtx);		pthread_mutex_lock(&cq->mtx);

assert(cq->qbase != NULL);		assert(cq->qbase != NULL);

compl = &cq->qbase[cq->tail];		compl = &cq->qbase[cq->tail];

compl->cdw0 = cdw0;		compl->cdw0 = cdw0;
compl->sqid = sqid;		compl->sqid = sqid;
compl->sqhd = atomic_load_acq_short(&sq->head);		compl->sqhd = sq->head;
compl->cid = cid;		compl->cid = cid;

// toggle phase		// toggle phase
phase = NVME_STATUS_GET_P(compl->status);		phase = NVME_STATUS_GET_P(compl->status);
compl->status = status;		compl->status = status;
pci_nvme_toggle_phase(&compl->status, phase);		pci_nvme_toggle_phase(&compl->status, phase);

cq->tail = (cq->tail + 1) % cq->size;		cq->tail = (cq->tail + 1) % cq->size;
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Lines	pci_nvme_io_done(struct blockif_req *br, int err)
uint16_t code, status;		uint16_t code, status;

DPRINTF("%s error %d %s", __func__, err, strerror(err));		DPRINTF("%s error %d %s", __func__, err, strerror(err));

/* TODO return correct error */		/* TODO return correct error */
code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS;		code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS;
pci_nvme_status_genc(&status, code);		pci_nvme_status_genc(&status, code);

pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, 0, status, 0);		pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, 0, status);
pci_nvme_release_ioreq(req->sc, req);		pci_nvme_release_ioreq(req->sc, req);
}		}

static void		static void
pci_nvme_io_partial(struct blockif_req *br, int err)		pci_nvme_io_partial(struct blockif_req *br, int err)
{		{
struct pci_nvme_ioreq *req = br->br_param;		struct pci_nvme_ioreq *req = br->br_param;

▲ Show 20 Lines • Show All 183 Lines • ▼ Show 20 Lines	if (blockif_delete(sc->nvstore.ctx, &req->io_req)) {
pci_nvme_status_genc(&status,		pci_nvme_status_genc(&status,
NVME_SC_INTERNAL_DEVICE_ERROR);		NVME_SC_INTERNAL_DEVICE_ERROR);
} else		} else
done = false;		done = false;
}		}

if (done) {		if (done) {
pci_nvme_set_completion(sc, req->nvme_sq, req->sqid,		pci_nvme_set_completion(sc, req->nvme_sq, req->sqid,
req->cid, 0, status, 0);		req->cid, 0, status);
pci_nvme_release_ioreq(sc, req);		pci_nvme_release_ioreq(sc, req);
}		}
}		}

static bool		static bool
nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc,		nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc,
struct nvme_command *cmd,		struct nvme_command *cmd,
struct pci_nvme_blockstore *nvstore,		struct pci_nvme_blockstore *nvstore,
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines
{		{
struct nvme_submission_queue *sq;		struct nvme_submission_queue *sq;
uint16_t status;		uint16_t status;
uint16_t sqhead;		uint16_t sqhead;

/* handle all submissions up to sq->tail index */		/* handle all submissions up to sq->tail index */
sq = &sc->submit_queues[idx];		sq = &sc->submit_queues[idx];

if (atomic_testandset_int(&sq->busy, 1)) {		pthread_mutex_lock(&sq->mtx);
DPRINTF("%s sqid %u busy", __func__, idx);
return;
}

sqhead = atomic_load_acq_short(&sq->head);		sqhead = sq->head;

DPRINTF("nvme_handle_io qid %u head %u tail %u cmdlist %p",		DPRINTF("nvme_handle_io qid %u head %u tail %u cmdlist %p",
idx, sqhead, sq->tail, sq->qbase);		idx, sqhead, sq->tail, sq->qbase);

while (sqhead != atomic_load_acq_short(&sq->tail)) {		while (sqhead != atomic_load_acq_short(&sq->tail)) {
struct nvme_command *cmd;		struct nvme_command *cmd;
struct pci_nvme_ioreq *req;		struct pci_nvme_ioreq *req;
uint32_t nsid;		uint32_t nsid;
bool pending;		bool pending;
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines	while (sqhead != atomic_load_acq_short(&sq->tail)) {
default:		default:
WPRINTF("%s unhandled io command 0x%x",		WPRINTF("%s unhandled io command 0x%x",
__func__, cmd->opc);		__func__, cmd->opc);
pci_nvme_status_genc(&status, NVME_SC_INVALID_OPCODE);		pci_nvme_status_genc(&status, NVME_SC_INVALID_OPCODE);
}		}
complete:		complete:
if (!pending) {		if (!pending) {
pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,		pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
status, 1);		status);
if (req != NULL)		if (req != NULL)
pci_nvme_release_ioreq(sc, req);		pci_nvme_release_ioreq(sc, req);
}		}
}		}

atomic_store_short(&sq->head, sqhead);		sq->head = sqhead;
atomic_store_int(&sq->busy, 0);
		pthread_mutex_unlock(&sq->mtx);
}		}

static void		static void
pci_nvme_handle_doorbell(struct vmctx ctx, struct pci_nvme_softc sc,		pci_nvme_handle_doorbell(struct vmctx ctx, struct pci_nvme_softc sc,
uint64_t idx, int is_sq, uint64_t value)		uint64_t idx, int is_sq, uint64_t value)
{		{
DPRINTF("nvme doorbell %lu, %s, val 0x%lx",		DPRINTF("nvme doorbell %lu, %s, val 0x%lx",
idx, is_sq ? "SQ" : "CQ", value & 0xFFFF);		idx, is_sq ? "SQ" : "CQ", value & 0xFFFF);

if (is_sq) {		if (is_sq) {
		if (idx > sc->num_squeues) {
		WPRINTF("%s queue index %lu overflow from "
		"guest (max %u)",
		__func__, idx, sc->num_squeues);
		return;
		}

atomic_store_short(&sc->submit_queues[idx].tail,		atomic_store_short(&sc->submit_queues[idx].tail,
(uint16_t)value);		(uint16_t)value);

if (idx == 0) {		if (idx == 0) {
pci_nvme_handle_admin_cmd(sc, value);		pci_nvme_handle_admin_cmd(sc, value);
} else {		} else {
/* submission queue; handle new entries in SQ */		/* submission queue; handle new entries in SQ */
if (idx > sc->num_squeues) {		if (idx > sc->num_squeues) {
WPRINTF("%s SQ index %lu overflow from "		WPRINTF("%s SQ index %lu overflow from "
"guest (max %u)",		"guest (max %u)",
__func__, idx, sc->num_squeues);		__func__, idx, sc->num_squeues);
return;		return;
}		}
pci_nvme_handle_io_cmd(sc, (uint16_t)idx);		pci_nvme_handle_io_cmd(sc, (uint16_t)idx);
}		}
} else {		} else {
if (idx > sc->num_cqueues) {		if (idx > sc->num_cqueues) {
WPRINTF("%s queue index %lu overflow from "		WPRINTF("%s queue index %lu overflow from "
"guest (max %u)",		"guest (max %u)",
__func__, idx, sc->num_cqueues);		__func__, idx, sc->num_cqueues);
return;		return;
}		}

sc->compl_queues[idx].head = (uint16_t)value;		atomic_store_short(&sc->compl_queues[idx].head,
		(uint16_t)value);
}		}
}		}

static void		static void
pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite)		pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite)
{		{
const char *s = iswrite ? "WRITE" : "READ";		const char *s = iswrite ? "WRITE" : "READ";

▲ Show 20 Lines • Show All 428 Lines • ▼ Show 20 Lines	pci_nvme_init(struct vmctx ctx, struct pci_devinst pi, char *opts)
if (error) {		if (error) {
WPRINTF("%s pci add Express capability failed", __func__);		WPRINTF("%s pci add Express capability failed", __func__);
goto done;		goto done;
}		}

pthread_mutex_init(&sc->mtx, NULL);		pthread_mutex_init(&sc->mtx, NULL);
sem_init(&sc->iosemlock, 0, sc->ioslots);		sem_init(&sc->iosemlock, 0, sc->ioslots);

pci_nvme_reset(sc);		pci_nvme_init_queues(sc, sc->max_queues, sc->max_queues);
/*		/*
* Controller data depends on Namespace data so initialize Namespace		* Controller data depends on Namespace data so initialize Namespace
* data first.		* data first.
*/		*/
pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore);		pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore);
pci_nvme_init_ctrldata(sc);		pci_nvme_init_ctrldata(sc);
pci_nvme_init_logpages(sc);		pci_nvme_init_logpages(sc);

		pci_nvme_reset(sc);

pci_lintr_request(pi);		pci_lintr_request(pi);

done:		done:
return (error);		return (error);
}		}


struct pci_devemu pci_de_nvme = {		struct pci_devemu pci_de_nvme = {
.pe_emu = "nvme",		.pe_emu = "nvme",
.pe_init = pci_nvme_init,		.pe_init = pci_nvme_init,
.pe_barwrite = pci_nvme_write,		.pe_barwrite = pci_nvme_write,
.pe_barread = pci_nvme_read		.pe_barread = pci_nvme_read
};		};
PCI_EMUL_SET(pci_de_nvme);		PCI_EMUL_SET(pci_de_nvme);