Changeset View
Changeset View
Standalone View
Standalone View
usr.sbin/bhyve/pci_nvme.c
Show First 20 Lines • Show All 163 Lines • ▼ Show 20 Lines | |||||
}; | }; | ||||
enum nvme_storage_type { | enum nvme_storage_type { | ||||
NVME_STOR_BLOCKIF = 0, | NVME_STOR_BLOCKIF = 0, | ||||
NVME_STOR_RAM = 1, | NVME_STOR_RAM = 1, | ||||
}; | }; | ||||
struct pci_nvme_blockstore { | struct pci_nvme_blockstore { | ||||
block_backend_t *be; | |||||
enum nvme_storage_type type; | enum nvme_storage_type type; | ||||
void *ctx; | void *ctx; | ||||
uint64_t size; | uint64_t size; | ||||
uint32_t sectsz; | uint32_t sectsz; | ||||
uint32_t sectsz_bits; | uint32_t sectsz_bits; | ||||
uint64_t eui64; | uint64_t eui64; | ||||
}; | }; | ||||
▲ Show 20 Lines • Show All 305 Lines • ▼ Show 20 Lines | for (int i = 0; i < sc->num_squeues + 1; i++) { | ||||
sc->submit_queues[i].cqid = 0; | sc->submit_queues[i].cqid = 0; | ||||
} | } | ||||
sc->submit_queues[i].tail = 0; | sc->submit_queues[i].tail = 0; | ||||
sc->submit_queues[i].head = 0; | sc->submit_queues[i].head = 0; | ||||
sc->submit_queues[i].busy = 0; | sc->submit_queues[i].busy = 0; | ||||
} | } | ||||
} else | } else | ||||
sc->submit_queues = calloc(sc->num_squeues + 1, | sc->submit_queues = calloc(sc->num_squeues + 1, | ||||
sizeof(struct nvme_submission_queue)); | sizeof(struct nvme_submission_queue)); | ||||
if (sc->compl_queues != NULL) { | if (sc->compl_queues != NULL) { | ||||
for (int i = 0; i < sc->num_cqueues + 1; i++) { | for (int i = 0; i < sc->num_cqueues + 1; i++) { | ||||
/* See Admin Submission Queue note above */ | /* See Admin Submission Queue note above */ | ||||
if (i != 0) { | if (i != 0) { | ||||
sc->compl_queues[i].qbase = NULL; | sc->compl_queues[i].qbase = NULL; | ||||
sc->compl_queues[i].size = 0; | sc->compl_queues[i].size = 0; | ||||
} | } | ||||
sc->compl_queues[i].tail = 0; | sc->compl_queues[i].tail = 0; | ||||
sc->compl_queues[i].head = 0; | sc->compl_queues[i].head = 0; | ||||
} | } | ||||
} else { | } else { | ||||
sc->compl_queues = calloc(sc->num_cqueues + 1, | sc->compl_queues = calloc(sc->num_cqueues + 1, | ||||
sizeof(struct nvme_completion_queue)); | sizeof(struct nvme_completion_queue)); | ||||
for (int i = 0; i < sc->num_cqueues + 1; i++) | for (int i = 0; i < sc->num_cqueues + 1; i++) | ||||
pthread_mutex_init(&sc->compl_queues[i].mtx, NULL); | pthread_mutex_init(&sc->compl_queues[i].mtx, NULL); | ||||
} | } | ||||
} | } | ||||
static void | static void | ||||
pci_nvme_reset(struct pci_nvme_softc *sc) | pci_nvme_reset(struct pci_nvme_softc *sc) | ||||
{ | { | ||||
pthread_mutex_lock(&sc->mtx); | pthread_mutex_lock(&sc->mtx); | ||||
pci_nvme_reset_locked(sc); | pci_nvme_reset_locked(sc); | ||||
pthread_mutex_unlock(&sc->mtx); | pthread_mutex_unlock(&sc->mtx); | ||||
} | } | ||||
static void | static void | ||||
pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc) | pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc) | ||||
{ | { | ||||
uint16_t acqs, asqs; | uint16_t acqs, asqs; | ||||
DPRINTF(("%s", __func__)); | DPRINTF(("%s", __func__)); | ||||
asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1; | asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1; | ||||
sc->submit_queues[0].size = asqs; | sc->submit_queues[0].size = asqs; | ||||
sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq, | sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq, | ||||
sizeof(struct nvme_command) * asqs); | sizeof(struct nvme_command) * asqs); | ||||
DPRINTF(("%s mapping Admin-SQ guest 0x%lx, host: %p", | DPRINTF(("%s mapping Admin-SQ guest 0x%lx, host: %p", | ||||
__func__, sc->regs.asq, sc->submit_queues[0].qbase)); | __func__, sc->regs.asq, sc->submit_queues[0].qbase)); | ||||
acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) & | acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) & | ||||
NVME_AQA_REG_ACQS_MASK) + 1; | NVME_AQA_REG_ACQS_MASK) + 1; | ||||
sc->compl_queues[0].size = acqs; | sc->compl_queues[0].size = acqs; | ||||
sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq, | sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq, | ||||
▲ Show 20 Lines • Show All 624 Lines • ▼ Show 20 Lines | if ((req->prev_gpaddr + req->prev_size) == gpaddr) { | ||||
DPRINTF(("large I/O, doing partial req")); | DPRINTF(("large I/O, doing partial req")); | ||||
iovidx = 0; | iovidx = 0; | ||||
req->io_req.br_iovcnt = 0; | req->io_req.br_iovcnt = 0; | ||||
req->io_req.br_callback = pci_nvme_io_partial; | req->io_req.br_callback = pci_nvme_io_partial; | ||||
if (!do_write) | if (!do_write) | ||||
err = blockif_read(sc->nvstore.ctx, | err = blockbe_read(sc->nvstore.be, | ||||
&req->io_req); | &req->io_req); | ||||
else | else | ||||
err = blockif_write(sc->nvstore.ctx, | err = blockbe_write(sc->nvstore.be, | ||||
&req->io_req); | &req->io_req); | ||||
/* wait until req completes before cont */ | /* wait until req completes before cont */ | ||||
if (err == 0) | if (err == 0) | ||||
pthread_cond_wait(&req->cv, &req->mtx); | pthread_cond_wait(&req->cv, &req->mtx); | ||||
} | } | ||||
if (iovidx == 0) { | if (iovidx == 0) { | ||||
req->io_req.br_offset = lba; | req->io_req.br_offset = lba; | ||||
▲ Show 20 Lines • Show All 326 Lines • ▼ Show 20 Lines | iodone: | ||||
if (err) | if (err) | ||||
goto do_error; | goto do_error; | ||||
req->io_req.br_callback = pci_nvme_io_done; | req->io_req.br_callback = pci_nvme_io_done; | ||||
err = 0; | err = 0; | ||||
switch (cmd->opc) { | switch (cmd->opc) { | ||||
case NVME_OPC_READ: | case NVME_OPC_READ: | ||||
err = blockif_read(sc->nvstore.ctx, &req->io_req); | err = blockbe_read(sc->nvstore.be, &req->io_req); | ||||
break; | break; | ||||
case NVME_OPC_WRITE: | case NVME_OPC_WRITE: | ||||
err = blockif_write(sc->nvstore.ctx, &req->io_req); | err = blockbe_write(sc->nvstore.be, &req->io_req); | ||||
break; | break; | ||||
default: | default: | ||||
WPRINTF(("%s unhandled io command 0x%x", | WPRINTF(("%s unhandled io command 0x%x", | ||||
__func__, cmd->opc)); | __func__, cmd->opc)); | ||||
err = 1; | err = 1; | ||||
} | } | ||||
do_error: | do_error: | ||||
▲ Show 20 Lines • Show All 305 Lines • ▼ Show 20 Lines | pci_nvme_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx, | ||||
return (0); | return (0); | ||||
} | } | ||||
static int | static int | ||||
pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts) | pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts) | ||||
{ | { | ||||
char bident[sizeof("XX:X:X")]; | char pci_ident[sizeof("XX:X:X")]; | ||||
char *uopt, *xopts, *config; | char *uopt, *xopts, *config; | ||||
uint32_t sectsz; | uint32_t sectsz; | ||||
int optidx; | int optidx, res; | ||||
sc->max_queues = NVME_QUEUES; | sc->max_queues = NVME_QUEUES; | ||||
sc->max_qentries = NVME_MAX_QENTRIES; | sc->max_qentries = NVME_MAX_QENTRIES; | ||||
sc->ioslots = NVME_IOSLOTS; | sc->ioslots = NVME_IOSLOTS; | ||||
sc->num_squeues = sc->max_queues; | sc->num_squeues = sc->max_queues; | ||||
sc->num_cqueues = sc->max_queues; | sc->num_cqueues = sc->max_queues; | ||||
sectsz = 0; | sectsz = 0; | ||||
Show All 35 Lines | if (!strcmp("maxq", xopts)) { | ||||
if (sc->nvstore.ctx == NULL) { | if (sc->nvstore.ctx == NULL) { | ||||
perror("Unable to allocate RAM"); | perror("Unable to allocate RAM"); | ||||
free(uopt); | free(uopt); | ||||
return (-1); | return (-1); | ||||
} | } | ||||
} else if (!strcmp("eui64", xopts)) { | } else if (!strcmp("eui64", xopts)) { | ||||
sc->nvstore.eui64 = htobe64(strtoull(config, NULL, 0)); | sc->nvstore.eui64 = htobe64(strtoull(config, NULL, 0)); | ||||
} else if (optidx == 0) { | } else if (optidx == 0) { | ||||
snprintf(bident, sizeof(bident), "%d:%d", | snprintf(pci_ident, sizeof(pci_ident), "%d:%d", | ||||
sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); | sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); | ||||
sc->nvstore.ctx = blockif_open(xopts, bident); | res = blockbe_open(&(sc->nvstore.be), xopts, pci_ident); | ||||
if (sc->nvstore.ctx == NULL) { | if (res != 0) { | ||||
perror("Could not open backing file"); | perror("Could not open backing file"); | ||||
free(uopt); | free(uopt); | ||||
return (-1); | return (-1); | ||||
} | } | ||||
sc->nvstore.ctx = calloc(1, sizeof(struct locblk_ctxt)); | |||||
if ( sc->nvstore.ctx == NULL) { | |||||
perror("calloc"); | |||||
return(-1); | |||||
} | |||||
sc->nvstore.type = NVME_STOR_BLOCKIF; | sc->nvstore.type = NVME_STOR_BLOCKIF; | ||||
sc->nvstore.size = blockif_size(sc->nvstore.ctx); | sc->nvstore.size = blockbe_size(sc->nvstore.be); | ||||
} else { | } else { | ||||
EPRINTLN("Invalid option %s", xopts); | EPRINTLN("Invalid option %s", xopts); | ||||
free(uopt); | free(uopt); | ||||
return (-1); | return (-1); | ||||
} | } | ||||
optidx++; | optidx++; | ||||
} | } | ||||
free(uopt); | free(uopt); | ||||
if (sc->nvstore.ctx == NULL || sc->nvstore.size == 0) { | if (sc->nvstore.ctx == NULL || sc->nvstore.size == 0) { | ||||
EPRINTLN("backing store not specified"); | EPRINTLN("backing store not specified"); | ||||
return (-1); | return (-1); | ||||
} | } | ||||
if (sectsz == 512 || sectsz == 4096 || sectsz == 8192) | if (sectsz == 512 || sectsz == 4096 || sectsz == 8192) | ||||
sc->nvstore.sectsz = sectsz; | sc->nvstore.sectsz = sectsz; | ||||
else if (sc->nvstore.type != NVME_STOR_RAM) | else if (sc->nvstore.type != NVME_STOR_RAM) | ||||
sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx); | sc->nvstore.sectsz = blockbe_sectsz(sc->nvstore.be); | ||||
for (sc->nvstore.sectsz_bits = 9; | for (sc->nvstore.sectsz_bits = 9; | ||||
(1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz; | (1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz; | ||||
sc->nvstore.sectsz_bits++); | sc->nvstore.sectsz_bits++); | ||||
if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES) | if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES) | ||||
sc->max_queues = NVME_QUEUES; | sc->max_queues = NVME_QUEUES; | ||||
if (sc->max_qentries <= 0) { | if (sc->max_qentries <= 0) { | ||||
▲ Show 20 Lines • Show All 100 Lines • Show Last 20 Lines |