Differential D24896 Diff 73839 head/usr.sbin/bhyve/pci_nvme.c

Changeset View

Standalone View

head/usr.sbin/bhyve/pci_nvme.c

Show First 20 Lines • Show All 245 Lines • ▼ Show 20 Lines	struct nvme_feature_obj {
uint32_t cdw11;		uint32_t cdw11;
nvme_feature_cb set;		nvme_feature_cb set;
nvme_feature_cb get;		nvme_feature_cb get;
bool namespace_specific;		bool namespace_specific;
};		};

#define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1)		#define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1)

		struct pci_nvme_aer {
		STAILQ_ENTRY(pci_nvme_aer) link;
		uint16_t cid; /* Command ID of the submitted AER */
		};

struct pci_nvme_softc {		struct pci_nvme_softc {
struct pci_devinst *nsc_pi;		struct pci_devinst *nsc_pi;

pthread_mutex_t mtx;		pthread_mutex_t mtx;

struct nvme_registers regs;		struct nvme_registers regs;

struct nvme_namespace_data nsdata;		struct nvme_namespace_data nsdata;
Show All 29 Lines	struct pci_nvme_softc {

/* Accounting for SMART data */		/* Accounting for SMART data */
__uint128_t read_data_units;		__uint128_t read_data_units;
__uint128_t write_data_units;		__uint128_t write_data_units;
__uint128_t read_commands;		__uint128_t read_commands;
__uint128_t write_commands;		__uint128_t write_commands;
uint32_t read_dunits_remainder;		uint32_t read_dunits_remainder;
uint32_t write_dunits_remainder;		uint32_t write_dunits_remainder;

		STAILQ_HEAD(, pci_nvme_aer) aer_list;
		uint32_t aer_count;
};		};


static struct pci_nvme_ioreq pci_nvme_get_ioreq(struct pci_nvme_softc );		static struct pci_nvme_ioreq pci_nvme_get_ioreq(struct pci_nvme_softc );
static void pci_nvme_release_ioreq(struct pci_nvme_softc , struct pci_nvme_ioreq );		static void pci_nvme_release_ioreq(struct pci_nvme_softc , struct pci_nvme_ioreq );
static void pci_nvme_io_done(struct blockif_req *, int);		static void pci_nvme_io_done(struct blockif_req *, int);

/* Controller Configuration utils */		/* Controller Configuration utils */
▲ Show 20 Lines • Show All 292 Lines • ▼ Show 20 Lines	pci_nvme_init_features(struct pci_nvme_softc *sc)
sc->feat[0].get = nvme_feature_invalid_cb;		sc->feat[0].get = nvme_feature_invalid_cb;

sc->feat[NVME_FEAT_LBA_RANGE_TYPE].namespace_specific = true;		sc->feat[NVME_FEAT_LBA_RANGE_TYPE].namespace_specific = true;
sc->feat[NVME_FEAT_ERROR_RECOVERY].namespace_specific = true;		sc->feat[NVME_FEAT_ERROR_RECOVERY].namespace_specific = true;
sc->feat[NVME_FEAT_NUMBER_OF_QUEUES].set = nvme_feature_num_queues;		sc->feat[NVME_FEAT_NUMBER_OF_QUEUES].set = nvme_feature_num_queues;
}		}

static void		static void
		pci_nvme_aer_init(struct pci_nvme_softc *sc)
		{

		STAILQ_INIT(&sc->aer_list);
		sc->aer_count = 0;
		}

		static void
		pci_nvme_aer_destroy(struct pci_nvme_softc *sc)
		{
		struct pci_nvme_aer *aer = NULL;

		while (!STAILQ_EMPTY(&sc->aer_list)) {
		aer = STAILQ_FIRST(&sc->aer_list);
		STAILQ_REMOVE_HEAD(&sc->aer_list, link);
		free(aer);
		}

		pci_nvme_aer_init(sc);
		}

		static bool
		pci_nvme_aer_available(struct pci_nvme_softc *sc)
		{

		return (!STAILQ_EMPTY(&sc->aer_list));
		}

		static bool
		pci_nvme_aer_limit_reached(struct pci_nvme_softc *sc)
		{
		struct nvme_controller_data *cd = &sc->ctrldata;

		/* AERL is a zero based value while aer_count is one's based */
		return (sc->aer_count == (cd->aerl + 1));
		}

		/*
		* Add an Async Event Request
		*
		* Stores an AER to be returned later if the Controller needs to notify the
		* host of an event.
		* Note that while the NVMe spec doesn't require Controllers to return AER's
		* in order, this implementation does preserve the order.
		*/
		static int
		pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid)
		{
		struct pci_nvme_aer *aer = NULL;

		if (pci_nvme_aer_limit_reached(sc))
		return (-1);

		aer = calloc(1, sizeof(struct pci_nvme_aer));
		if (aer == NULL)
		return (-1);

		sc->aer_count++;

		/* Save the Command ID for use in the completion message */
		aer->cid = cid;
		STAILQ_INSERT_TAIL(&sc->aer_list, aer, link);

		return (0);
		}

		/*
		* Get an Async Event Request structure
		*
		* Returns a pointer to an AER previously submitted by the host or NULL if
		* no AER's exist. Caller is responsible for freeing the returned struct.
		*/
		static struct pci_nvme_aer *
		pci_nvme_aer_get(struct pci_nvme_softc *sc)
		{
		struct pci_nvme_aer *aer = NULL;

		aer = STAILQ_FIRST(&sc->aer_list);
		if (aer != NULL) {
		STAILQ_REMOVE_HEAD(&sc->aer_list, link);
		sc->aer_count--;
		}

		return (aer);
		}

		static void
pci_nvme_reset_locked(struct pci_nvme_softc *sc)		pci_nvme_reset_locked(struct pci_nvme_softc *sc)
{		{
uint32_t i;		uint32_t i;

DPRINTF("%s", __func__);		DPRINTF("%s", __func__);

sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) \|		sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) \|
(1 << NVME_CAP_LO_REG_CQR_SHIFT) \|		(1 << NVME_CAP_LO_REG_CQR_SHIFT) \|
Show All 21 Lines	pci_nvme_reset_locked(struct pci_nvme_softc *sc)
for (i = 0; i < sc->num_cqueues + 1; i++) {		for (i = 0; i < sc->num_cqueues + 1; i++) {
sc->compl_queues[i].qbase = NULL;		sc->compl_queues[i].qbase = NULL;
sc->compl_queues[i].size = 0;		sc->compl_queues[i].size = 0;
sc->compl_queues[i].tail = 0;		sc->compl_queues[i].tail = 0;
sc->compl_queues[i].head = 0;		sc->compl_queues[i].head = 0;
}		}

sc->num_q_is_set = false;		sc->num_q_is_set = false;

		pci_nvme_aer_destroy(sc);
}		}

static void		static void
pci_nvme_reset(struct pci_nvme_softc *sc)		pci_nvme_reset(struct pci_nvme_softc *sc)
{		{
pthread_mutex_lock(&sc->mtx);		pthread_mutex_lock(&sc->mtx);
pci_nvme_reset_locked(sc);		pci_nvme_reset_locked(sc);
pthread_mutex_unlock(&sc->mtx);		pthread_mutex_unlock(&sc->mtx);
▲ Show 20 Lines • Show All 719 Lines • ▼ Show 20 Lines
}		}

static int		static int
nvme_opc_async_event_req(struct pci_nvme_softc* sc,		nvme_opc_async_event_req(struct pci_nvme_softc* sc,
struct nvme_command* command, struct nvme_completion* compl)		struct nvme_command* command, struct nvme_completion* compl)
{		{
DPRINTF("%s async event request 0x%x", __func__, command->cdw11);		DPRINTF("%s async event request 0x%x", __func__, command->cdw11);

		/* Don't exceed the Async Event Request Limit (AERL). */
		if (pci_nvme_aer_limit_reached(sc)) {
		pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
		NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED);
		return (1);
		}

		if (pci_nvme_aer_add(sc, command->cid)) {
		pci_nvme_status_tc(&compl->status, NVME_SCT_GENERIC,
		NVME_SC_INTERNAL_DEVICE_ERROR);
		return (1);
		}

/*		/*
* TODO: raise events when they happen based on the Set Features cmd.		* Raise events when they happen based on the Set Features cmd.
* These events happen async, so only set completion successful if		* These events happen async, so only set completion successful if
* there is an event reflective of the request to get event.		* there is an event reflective of the request to get event.
*/		*/
pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,		compl->status = NVME_NO_STATUS;
NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED);
return (0);		return (0);
}		}

static void		static void
pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)		pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)
{		{
struct nvme_completion compl;		struct nvme_completion compl;
struct nvme_command *cmd;		struct nvme_command *cmd;
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines	case NVME_OPC_SET_FEATURES:
nvme_opc_set_features(sc, cmd, &compl);		nvme_opc_set_features(sc, cmd, &compl);
break;		break;
case NVME_OPC_GET_FEATURES:		case NVME_OPC_GET_FEATURES:
DPRINTF("%s command GET_FEATURES", __func__);		DPRINTF("%s command GET_FEATURES", __func__);
nvme_opc_get_features(sc, cmd, &compl);		nvme_opc_get_features(sc, cmd, &compl);
break;		break;
case NVME_OPC_ASYNC_EVENT_REQUEST:		case NVME_OPC_ASYNC_EVENT_REQUEST:
DPRINTF("%s command ASYNC_EVENT_REQ", __func__);		DPRINTF("%s command ASYNC_EVENT_REQ", __func__);
/* XXX dont care, unhandled for now
nvme_opc_async_event_req(sc, cmd, &compl);		nvme_opc_async_event_req(sc, cmd, &compl);
*/
compl.status = NVME_NO_STATUS;
break;		break;
case NVME_OPC_FORMAT_NVM:		case NVME_OPC_FORMAT_NVM:
DPRINTF("%s command FORMAT_NVM", __func__);		DPRINTF("%s command FORMAT_NVM", __func__);
if ((sc->ctrldata.oacs &		if ((sc->ctrldata.oacs &
(1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT)) == 0) {		(1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT)) == 0) {
pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);		pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);
}		}
compl.status = NVME_NO_STATUS;		compl.status = NVME_NO_STATUS;
▲ Show 20 Lines • Show All 1,150 Lines • ▼ Show 20 Lines	pci_nvme_init(struct vmctx ctx, struct pci_devinst pi, char *opts)
/*		/*
* Controller data depends on Namespace data so initialize Namespace		* Controller data depends on Namespace data so initialize Namespace
* data first.		* data first.
*/		*/
pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore);		pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore);
pci_nvme_init_ctrldata(sc);		pci_nvme_init_ctrldata(sc);
pci_nvme_init_logpages(sc);		pci_nvme_init_logpages(sc);
pci_nvme_init_features(sc);		pci_nvme_init_features(sc);

		pci_nvme_aer_init(sc);

pci_nvme_reset(sc);		pci_nvme_reset(sc);

pci_lintr_request(pi);		pci_lintr_request(pi);

done:		done:
return (error);		return (error);
}		}
Show All 9 Lines