Changeset View
Changeset View
Standalone View
Standalone View
usr.sbin/bhyve/pci_nvme.c
Show First 20 Lines • Show All 245 Lines • ▼ Show 20 Lines | struct nvme_feature_obj { | ||||
uint32_t cdw11; | uint32_t cdw11; | ||||
nvme_feature_cb set; | nvme_feature_cb set; | ||||
nvme_feature_cb get; | nvme_feature_cb get; | ||||
bool namespace_specific; | bool namespace_specific; | ||||
}; | }; | ||||
#define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1) | #define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1) | ||||
struct pci_nvme_aer { | |||||
STAILQ_ENTRY(pci_nvme_aer) link; | |||||
uint16_t cid; /* Command ID of the submitted AER */ | |||||
}; | |||||
struct pci_nvme_softc { | struct pci_nvme_softc { | ||||
struct pci_devinst *nsc_pi; | struct pci_devinst *nsc_pi; | ||||
pthread_mutex_t mtx; | pthread_mutex_t mtx; | ||||
struct nvme_registers regs; | struct nvme_registers regs; | ||||
struct nvme_namespace_data nsdata; | struct nvme_namespace_data nsdata; | ||||
Show All 29 Lines | struct pci_nvme_softc { | ||||
/* Accounting for SMART data */ | /* Accounting for SMART data */ | ||||
__uint128_t read_data_units; | __uint128_t read_data_units; | ||||
__uint128_t write_data_units; | __uint128_t write_data_units; | ||||
__uint128_t read_commands; | __uint128_t read_commands; | ||||
__uint128_t write_commands; | __uint128_t write_commands; | ||||
uint32_t read_dunits_remainder; | uint32_t read_dunits_remainder; | ||||
uint32_t write_dunits_remainder; | uint32_t write_dunits_remainder; | ||||
STAILQ_HEAD(, pci_nvme_aer) aer_list; | |||||
uint32_t aer_count; | |||||
}; | }; | ||||
static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *); | static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *); | ||||
static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *); | static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *); | ||||
static void pci_nvme_io_done(struct blockif_req *, int); | static void pci_nvme_io_done(struct blockif_req *, int); | ||||
/* Controller Configuration utils */ | /* Controller Configuration utils */ | ||||
▲ Show 20 Lines • Show All 292 Lines • ▼ Show 20 Lines | pci_nvme_init_features(struct pci_nvme_softc *sc) | ||||
sc->feat[0].get = nvme_feature_invalid_cb; | sc->feat[0].get = nvme_feature_invalid_cb; | ||||
sc->feat[NVME_FEAT_LBA_RANGE_TYPE].namespace_specific = true; | sc->feat[NVME_FEAT_LBA_RANGE_TYPE].namespace_specific = true; | ||||
sc->feat[NVME_FEAT_ERROR_RECOVERY].namespace_specific = true; | sc->feat[NVME_FEAT_ERROR_RECOVERY].namespace_specific = true; | ||||
sc->feat[NVME_FEAT_NUMBER_OF_QUEUES].set = nvme_feature_num_queues; | sc->feat[NVME_FEAT_NUMBER_OF_QUEUES].set = nvme_feature_num_queues; | ||||
} | } | ||||
static void | static void | ||||
pci_nvme_aer_init(struct pci_nvme_softc *sc) | |||||
{ | |||||
STAILQ_INIT(&sc->aer_list); | |||||
sc->aer_count = 0; | |||||
} | |||||
static void | |||||
pci_nvme_aer_destroy(struct pci_nvme_softc *sc) | |||||
{ | |||||
struct pci_nvme_aer *aer = NULL; | |||||
while (!STAILQ_EMPTY(&sc->aer_list)) { | |||||
aer = STAILQ_FIRST(&sc->aer_list); | |||||
STAILQ_REMOVE_HEAD(&sc->aer_list, link); | |||||
free(aer); | |||||
} | |||||
pci_nvme_aer_init(sc); | |||||
} | |||||
static bool | |||||
pci_nvme_aer_available(struct pci_nvme_softc *sc) | |||||
{ | |||||
return (!STAILQ_EMPTY(&sc->aer_list)); | |||||
} | |||||
static bool | |||||
pci_nvme_aer_limit_reached(struct pci_nvme_softc *sc) | |||||
{ | |||||
struct nvme_controller_data *cd = &sc->ctrldata; | |||||
/* AERL is a zero based value while aer_count is one's based */ | |||||
return (sc->aer_count == (cd->aerl + 1)); | |||||
} | |||||
/* | |||||
* Add an Async Event Request | |||||
* | |||||
* Stores an AER to be returned later if the Controller needs to notify the | |||||
* host of an event. | |||||
* Note that while the NVMe spec doesn't require Controllers to return AER's | |||||
* in order, this implementation does preserve the order. | |||||
*/ | |||||
static int | |||||
pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid) | |||||
{ | |||||
struct pci_nvme_aer *aer = NULL; | |||||
if (pci_nvme_aer_limit_reached(sc)) | |||||
return (-1); | |||||
aer = calloc(1, sizeof(struct pci_nvme_aer)); | |||||
if (aer == NULL) | |||||
return (-1); | |||||
sc->aer_count++; | |||||
/* Save the Command ID for use in the completion message */ | |||||
aer->cid = cid; | |||||
STAILQ_INSERT_TAIL(&sc->aer_list, aer, link); | |||||
return (0); | |||||
} | |||||
/* | |||||
* Get an Async Event Request structure | |||||
* | |||||
* Returns a pointer to an AER previously submitted by the host or NULL if | |||||
* no AER's exist. Caller is responsible for freeing the returned struct. | |||||
*/ | |||||
static struct pci_nvme_aer * | |||||
pci_nvme_aer_get(struct pci_nvme_softc *sc) | |||||
{ | |||||
struct pci_nvme_aer *aer = NULL; | |||||
aer = STAILQ_FIRST(&sc->aer_list); | |||||
if (aer != NULL) { | |||||
STAILQ_REMOVE_HEAD(&sc->aer_list, link); | |||||
sc->aer_count--; | |||||
} | |||||
return (aer); | |||||
} | |||||
static void | |||||
pci_nvme_reset_locked(struct pci_nvme_softc *sc) | pci_nvme_reset_locked(struct pci_nvme_softc *sc) | ||||
{ | { | ||||
uint32_t i; | uint32_t i; | ||||
DPRINTF("%s", __func__); | DPRINTF("%s", __func__); | ||||
sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) | | sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) | | ||||
(1 << NVME_CAP_LO_REG_CQR_SHIFT) | | (1 << NVME_CAP_LO_REG_CQR_SHIFT) | | ||||
Show All 21 Lines | pci_nvme_reset_locked(struct pci_nvme_softc *sc) | ||||
for (i = 0; i < sc->num_cqueues + 1; i++) { | for (i = 0; i < sc->num_cqueues + 1; i++) { | ||||
sc->compl_queues[i].qbase = NULL; | sc->compl_queues[i].qbase = NULL; | ||||
sc->compl_queues[i].size = 0; | sc->compl_queues[i].size = 0; | ||||
sc->compl_queues[i].tail = 0; | sc->compl_queues[i].tail = 0; | ||||
sc->compl_queues[i].head = 0; | sc->compl_queues[i].head = 0; | ||||
} | } | ||||
sc->num_q_is_set = false; | sc->num_q_is_set = false; | ||||
pci_nvme_aer_destroy(sc); | |||||
} | } | ||||
static void | static void | ||||
pci_nvme_reset(struct pci_nvme_softc *sc) | pci_nvme_reset(struct pci_nvme_softc *sc) | ||||
{ | { | ||||
pthread_mutex_lock(&sc->mtx); | pthread_mutex_lock(&sc->mtx); | ||||
pci_nvme_reset_locked(sc); | pci_nvme_reset_locked(sc); | ||||
pthread_mutex_unlock(&sc->mtx); | pthread_mutex_unlock(&sc->mtx); | ||||
▲ Show 20 Lines • Show All 719 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static int | static int | ||||
nvme_opc_async_event_req(struct pci_nvme_softc* sc, | nvme_opc_async_event_req(struct pci_nvme_softc* sc, | ||||
struct nvme_command* command, struct nvme_completion* compl) | struct nvme_command* command, struct nvme_completion* compl) | ||||
{ | { | ||||
DPRINTF("%s async event request 0x%x", __func__, command->cdw11); | DPRINTF("%s async event request 0x%x", __func__, command->cdw11); | ||||
/* Don't exceed the Async Event Request Limit (AERL). */ | |||||
if (pci_nvme_aer_limit_reached(sc)) { | |||||
pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, | |||||
NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED); | |||||
return (1); | |||||
} | |||||
if (pci_nvme_aer_add(sc, command->cid)) { | |||||
pci_nvme_status_tc(&compl->status, NVME_SCT_GENERIC, | |||||
NVME_SC_INTERNAL_DEVICE_ERROR); | |||||
return (1); | |||||
} | |||||
/* | /* | ||||
* TODO: raise events when they happen based on the Set Features cmd. | * Raise events when they happen based on the Set Features cmd. | ||||
* These events happen async, so only set completion successful if | * These events happen async, so only set completion successful if | ||||
* there is an event reflective of the request to get event. | * there is an event reflective of the request to get event. | ||||
*/ | */ | ||||
pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, | compl->status = NVME_NO_STATUS; | ||||
NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED); | |||||
return (0); | return (0); | ||||
} | } | ||||
static void | static void | ||||
pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value) | pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value) | ||||
{ | { | ||||
struct nvme_completion compl; | struct nvme_completion compl; | ||||
struct nvme_command *cmd; | struct nvme_command *cmd; | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | case NVME_OPC_SET_FEATURES: | ||||
nvme_opc_set_features(sc, cmd, &compl); | nvme_opc_set_features(sc, cmd, &compl); | ||||
break; | break; | ||||
case NVME_OPC_GET_FEATURES: | case NVME_OPC_GET_FEATURES: | ||||
DPRINTF("%s command GET_FEATURES", __func__); | DPRINTF("%s command GET_FEATURES", __func__); | ||||
nvme_opc_get_features(sc, cmd, &compl); | nvme_opc_get_features(sc, cmd, &compl); | ||||
break; | break; | ||||
case NVME_OPC_ASYNC_EVENT_REQUEST: | case NVME_OPC_ASYNC_EVENT_REQUEST: | ||||
DPRINTF("%s command ASYNC_EVENT_REQ", __func__); | DPRINTF("%s command ASYNC_EVENT_REQ", __func__); | ||||
/* XXX dont care, unhandled for now | |||||
nvme_opc_async_event_req(sc, cmd, &compl); | nvme_opc_async_event_req(sc, cmd, &compl); | ||||
*/ | |||||
compl.status = NVME_NO_STATUS; | |||||
break; | break; | ||||
case NVME_OPC_FORMAT_NVM: | case NVME_OPC_FORMAT_NVM: | ||||
DPRINTF("%s command FORMAT_NVM", __func__); | DPRINTF("%s command FORMAT_NVM", __func__); | ||||
if ((sc->ctrldata.oacs & | if ((sc->ctrldata.oacs & | ||||
(1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT)) == 0) { | (1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT)) == 0) { | ||||
pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); | pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); | ||||
} | } | ||||
compl.status = NVME_NO_STATUS; | compl.status = NVME_NO_STATUS; | ||||
▲ Show 20 Lines • Show All 1,149 Lines • ▼ Show 20 Lines | pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) | ||||
/* | /* | ||||
* Controller data depends on Namespace data so initialize Namespace | * Controller data depends on Namespace data so initialize Namespace | ||||
* data first. | * data first. | ||||
*/ | */ | ||||
pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore); | pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore); | ||||
pci_nvme_init_ctrldata(sc); | pci_nvme_init_ctrldata(sc); | ||||
pci_nvme_init_logpages(sc); | pci_nvme_init_logpages(sc); | ||||
pci_nvme_init_features(sc); | pci_nvme_init_features(sc); | ||||
pci_nvme_aer_init(sc); | |||||
pci_nvme_reset(sc); | pci_nvme_reset(sc); | ||||
pci_lintr_request(pi); | pci_lintr_request(pi); | ||||
done: | done: | ||||
return (error); | return (error); | ||||
} | } | ||||
Show All 9 Lines |