Differential D23010 Diff 67204 usr.sbin/bhyve/pci_nvme.c

Changeset View

Standalone View

usr.sbin/bhyve/pci_nvme.c

Show First 20 Lines • Show All 163 Lines • ▼ Show 20 Lines
};		};

enum nvme_storage_type {		enum nvme_storage_type {
NVME_STOR_BLOCKIF = 0,		NVME_STOR_BLOCKIF = 0,
NVME_STOR_RAM = 1,		NVME_STOR_RAM = 1,
};		};

struct pci_nvme_blockstore {		struct pci_nvme_blockstore {
		block_backend_t *be;
enum nvme_storage_type type;		enum nvme_storage_type type;
void *ctx;		void *ctx;
uint64_t size;		uint64_t size;
uint32_t sectsz;		uint32_t sectsz;
uint32_t sectsz_bits;		uint32_t sectsz_bits;
uint64_t eui64;		uint64_t eui64;
};		};

▲ Show 20 Lines • Show All 305 Lines • ▼ Show 20 Lines	for (int i = 0; i < sc->num_squeues + 1; i++) {
sc->submit_queues[i].cqid = 0;		sc->submit_queues[i].cqid = 0;
}		}
sc->submit_queues[i].tail = 0;		sc->submit_queues[i].tail = 0;
sc->submit_queues[i].head = 0;		sc->submit_queues[i].head = 0;
sc->submit_queues[i].busy = 0;		sc->submit_queues[i].busy = 0;
}		}
} else		} else
sc->submit_queues = calloc(sc->num_squeues + 1,		sc->submit_queues = calloc(sc->num_squeues + 1,
sizeof(struct nvme_submission_queue));		sizeof(struct nvme_submission_queue));

if (sc->compl_queues != NULL) {		if (sc->compl_queues != NULL) {
for (int i = 0; i < sc->num_cqueues + 1; i++) {		for (int i = 0; i < sc->num_cqueues + 1; i++) {
/* See Admin Submission Queue note above */		/* See Admin Submission Queue note above */
if (i != 0) {		if (i != 0) {
sc->compl_queues[i].qbase = NULL;		sc->compl_queues[i].qbase = NULL;
sc->compl_queues[i].size = 0;		sc->compl_queues[i].size = 0;
}		}

sc->compl_queues[i].tail = 0;		sc->compl_queues[i].tail = 0;
sc->compl_queues[i].head = 0;		sc->compl_queues[i].head = 0;
}		}
} else {		} else {
sc->compl_queues = calloc(sc->num_cqueues + 1,		sc->compl_queues = calloc(sc->num_cqueues + 1,
sizeof(struct nvme_completion_queue));		sizeof(struct nvme_completion_queue));

for (int i = 0; i < sc->num_cqueues + 1; i++)		for (int i = 0; i < sc->num_cqueues + 1; i++)
pthread_mutex_init(&sc->compl_queues[i].mtx, NULL);		pthread_mutex_init(&sc->compl_queues[i].mtx, NULL);
}		}
}		}

static void		static void
pci_nvme_reset(struct pci_nvme_softc *sc)		pci_nvme_reset(struct pci_nvme_softc *sc)
{		{
pthread_mutex_lock(&sc->mtx);		pthread_mutex_lock(&sc->mtx);
pci_nvme_reset_locked(sc);		pci_nvme_reset_locked(sc);
pthread_mutex_unlock(&sc->mtx);		pthread_mutex_unlock(&sc->mtx);
}		}

static void		static void
pci_nvme_init_controller(struct vmctx ctx, struct pci_nvme_softc sc)		pci_nvme_init_controller(struct vmctx ctx, struct pci_nvme_softc sc)
{		{
uint16_t acqs, asqs;		uint16_t acqs, asqs;

DPRINTF(("%s", __func__));		DPRINTF(("%s", __func__));

asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1;		asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1;
sc->submit_queues[0].size = asqs;		sc->submit_queues[0].size = asqs;
sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq,		sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq,
sizeof(struct nvme_command) * asqs);		sizeof(struct nvme_command) * asqs);

DPRINTF(("%s mapping Admin-SQ guest 0x%lx, host: %p",		DPRINTF(("%s mapping Admin-SQ guest 0x%lx, host: %p",
__func__, sc->regs.asq, sc->submit_queues[0].qbase));		__func__, sc->regs.asq, sc->submit_queues[0].qbase));

acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &		acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &
NVME_AQA_REG_ACQS_MASK) + 1;		NVME_AQA_REG_ACQS_MASK) + 1;
sc->compl_queues[0].size = acqs;		sc->compl_queues[0].size = acqs;
sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq,		sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq,
▲ Show 20 Lines • Show All 624 Lines • ▼ Show 20 Lines	if ((req->prev_gpaddr + req->prev_size) == gpaddr) {
DPRINTF(("large I/O, doing partial req"));		DPRINTF(("large I/O, doing partial req"));

iovidx = 0;		iovidx = 0;
req->io_req.br_iovcnt = 0;		req->io_req.br_iovcnt = 0;

req->io_req.br_callback = pci_nvme_io_partial;		req->io_req.br_callback = pci_nvme_io_partial;

if (!do_write)		if (!do_write)
err = blockif_read(sc->nvstore.ctx,		err = blockbe_read(sc->nvstore.be,
&req->io_req);		&req->io_req);
else		else
err = blockif_write(sc->nvstore.ctx,		err = blockbe_write(sc->nvstore.be,
&req->io_req);		&req->io_req);

/* wait until req completes before cont */		/* wait until req completes before cont */
if (err == 0)		if (err == 0)
pthread_cond_wait(&req->cv, &req->mtx);		pthread_cond_wait(&req->cv, &req->mtx);
}		}
if (iovidx == 0) {		if (iovidx == 0) {
req->io_req.br_offset = lba;		req->io_req.br_offset = lba;
▲ Show 20 Lines • Show All 326 Lines • ▼ Show 20 Lines	iodone:
if (err)		if (err)
goto do_error;		goto do_error;

req->io_req.br_callback = pci_nvme_io_done;		req->io_req.br_callback = pci_nvme_io_done;

err = 0;		err = 0;
switch (cmd->opc) {		switch (cmd->opc) {
case NVME_OPC_READ:		case NVME_OPC_READ:
err = blockif_read(sc->nvstore.ctx, &req->io_req);		err = blockbe_read(sc->nvstore.be, &req->io_req);
break;		break;
case NVME_OPC_WRITE:		case NVME_OPC_WRITE:
err = blockif_write(sc->nvstore.ctx, &req->io_req);		err = blockbe_write(sc->nvstore.be, &req->io_req);
break;		break;
default:		default:
WPRINTF(("%s unhandled io command 0x%x",		WPRINTF(("%s unhandled io command 0x%x",
__func__, cmd->opc));		__func__, cmd->opc));
err = 1;		err = 1;
}		}

do_error:		do_error:
▲ Show 20 Lines • Show All 305 Lines • ▼ Show 20 Lines	pci_nvme_read(struct vmctx ctx, int vcpu, struct pci_devinst pi, int baridx,

return (0);		return (0);
}		}


static int		static int
pci_nvme_parse_opts(struct pci_nvme_softc sc, char opts)		pci_nvme_parse_opts(struct pci_nvme_softc sc, char opts)
{		{
char bident[sizeof("XX:X:X")];		char pci_ident[sizeof("XX:X:X")];
char uopt, xopts, *config;		char uopt, xopts, *config;
uint32_t sectsz;		uint32_t sectsz;
int optidx;		int optidx, res;

sc->max_queues = NVME_QUEUES;		sc->max_queues = NVME_QUEUES;
sc->max_qentries = NVME_MAX_QENTRIES;		sc->max_qentries = NVME_MAX_QENTRIES;
sc->ioslots = NVME_IOSLOTS;		sc->ioslots = NVME_IOSLOTS;
sc->num_squeues = sc->max_queues;		sc->num_squeues = sc->max_queues;
sc->num_cqueues = sc->max_queues;		sc->num_cqueues = sc->max_queues;
sectsz = 0;		sectsz = 0;

Show All 35 Lines	if (!strcmp("maxq", xopts)) {
if (sc->nvstore.ctx == NULL) {		if (sc->nvstore.ctx == NULL) {
perror("Unable to allocate RAM");		perror("Unable to allocate RAM");
free(uopt);		free(uopt);
return (-1);		return (-1);
}		}
} else if (!strcmp("eui64", xopts)) {		} else if (!strcmp("eui64", xopts)) {
sc->nvstore.eui64 = htobe64(strtoull(config, NULL, 0));		sc->nvstore.eui64 = htobe64(strtoull(config, NULL, 0));
} else if (optidx == 0) {		} else if (optidx == 0) {
snprintf(bident, sizeof(bident), "%d:%d",		snprintf(pci_ident, sizeof(pci_ident), "%d:%d",
sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);		sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
sc->nvstore.ctx = blockif_open(xopts, bident);		res = blockbe_open(&(sc->nvstore.be), xopts, pci_ident);
if (sc->nvstore.ctx == NULL) {		if (res != 0) {
perror("Could not open backing file");		perror("Could not open backing file");
free(uopt);		free(uopt);
return (-1);		return (-1);
}		}
		sc->nvstore.ctx = calloc(1, sizeof(struct locblk_ctxt));
		if ( sc->nvstore.ctx == NULL) {
		perror("calloc");
		return(-1);
		}
sc->nvstore.type = NVME_STOR_BLOCKIF;		sc->nvstore.type = NVME_STOR_BLOCKIF;
sc->nvstore.size = blockif_size(sc->nvstore.ctx);		sc->nvstore.size = blockbe_size(sc->nvstore.be);
} else {		} else {
EPRINTLN("Invalid option %s", xopts);		EPRINTLN("Invalid option %s", xopts);
free(uopt);		free(uopt);
return (-1);		return (-1);
}		}

optidx++;		optidx++;
}		}
free(uopt);		free(uopt);

if (sc->nvstore.ctx == NULL \|\| sc->nvstore.size == 0) {		if (sc->nvstore.ctx == NULL \|\| sc->nvstore.size == 0) {
EPRINTLN("backing store not specified");		EPRINTLN("backing store not specified");
return (-1);		return (-1);
}		}
if (sectsz == 512 \|\| sectsz == 4096 \|\| sectsz == 8192)		if (sectsz == 512 \|\| sectsz == 4096 \|\| sectsz == 8192)
sc->nvstore.sectsz = sectsz;		sc->nvstore.sectsz = sectsz;
else if (sc->nvstore.type != NVME_STOR_RAM)		else if (sc->nvstore.type != NVME_STOR_RAM)
sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx);		sc->nvstore.sectsz = blockbe_sectsz(sc->nvstore.be);
for (sc->nvstore.sectsz_bits = 9;		for (sc->nvstore.sectsz_bits = 9;
(1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz;		(1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz;
sc->nvstore.sectsz_bits++);		sc->nvstore.sectsz_bits++);

if (sc->max_queues <= 0 \|\| sc->max_queues > NVME_QUEUES)		if (sc->max_queues <= 0 \|\| sc->max_queues > NVME_QUEUES)
sc->max_queues = NVME_QUEUES;		sc->max_queues = NVME_QUEUES;

if (sc->max_qentries <= 0) {		if (sc->max_qentries <= 0) {
▲ Show 20 Lines • Show All 100 Lines • Show Last 20 Lines