Differential D52440 Diff 161889 sys/dev/ufshci/ufshci_req_sdb.c

Changeset View

Standalone View

sys/dev/ufshci/ufshci_req_sdb.c

Show All 34 Lines	bus_dmamem_free(req_queue->dma_tag_ucd, req_queue->ucd,
req_queue->ucdmem_map);		req_queue->ucdmem_map);
req_queue->ucd = NULL;		req_queue->ucd = NULL;
}		}

if (req_queue->dma_tag_ucd) {		if (req_queue->dma_tag_ucd) {
bus_dma_tag_destroy(req_queue->dma_tag_ucd);		bus_dma_tag_destroy(req_queue->dma_tag_ucd);
req_queue->dma_tag_ucd = NULL;		req_queue->dma_tag_ucd = NULL;
}		}

		free(req_queue->hwq->ucd_bus_addr, M_UFSHCI);
}		}

static void		static void
ufshci_ucd_map(void arg, bus_dma_segment_t seg, int nseg, int error)		ufshci_ucd_map(void arg, bus_dma_segment_t seg, int nseg, int error)
{		{
struct ufshci_hw_queue *hwq = arg;		struct ufshci_hw_queue *hwq = arg;
int i;		int i;

Show All 18 Lines
ufshci_req_sdb_cmd_desc_construct(struct ufshci_req_queue *req_queue,		ufshci_req_sdb_cmd_desc_construct(struct ufshci_req_queue *req_queue,
uint32_t num_entries, struct ufshci_controller *ctrlr)		uint32_t num_entries, struct ufshci_controller *ctrlr)
{		{
struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];		struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
size_t ucd_allocsz, payload_allocsz;		size_t ucd_allocsz, payload_allocsz;
uint8_t *ucdmem;		uint8_t *ucdmem;
int i, error;		int i, error;

		req_queue->hwq->ucd_bus_addr = malloc(sizeof(bus_addr_t) *
		req_queue->num_trackers,
		M_UFSHCI, M_ZERO \| M_NOWAIT);

/*		/*
* Each component must be page aligned, and individual PRP lists		* Each component must be page aligned, and individual PRP lists
* cannot cross a page boundary.		* cannot cross a page boundary.
*/		*/
ucd_allocsz = num_entries * sizeof(struct ufshci_utp_cmd_desc);		ucd_allocsz = num_entries * sizeof(struct ufshci_utp_cmd_desc);
ucd_allocsz = roundup2(ucd_allocsz, ctrlr->page_size);		ucd_allocsz = roundup2(ucd_allocsz, ctrlr->page_size);
payload_allocsz = num_entries * ctrlr->max_xfer_size;		payload_allocsz = num_entries * ctrlr->max_xfer_size;

▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines
ufshci_req_sdb_construct(struct ufshci_controller *ctrlr,		ufshci_req_sdb_construct(struct ufshci_controller *ctrlr,
struct ufshci_req_queue *req_queue, uint32_t num_entries, bool is_task_mgmt)		struct ufshci_req_queue *req_queue, uint32_t num_entries, bool is_task_mgmt)
{		{
struct ufshci_hw_queue *hwq;		struct ufshci_hw_queue *hwq;
size_t desc_size, alloc_size;		size_t desc_size, alloc_size;
uint64_t queuemem_phys;		uint64_t queuemem_phys;
uint8_t *queuemem;		uint8_t *queuemem;
struct ufshci_tracker *tr;		struct ufshci_tracker *tr;
		const size_t lock_name_len = 32;
		char qlock_name[lock_name_len], recovery_lock_name[lock_name_len];
		char *base;
int i, error;		int i, error;

req_queue->ctrlr = ctrlr;		req_queue->ctrlr = ctrlr;
req_queue->is_task_mgmt = is_task_mgmt;		req_queue->is_task_mgmt = is_task_mgmt;
req_queue->num_entries = num_entries;		req_queue->num_entries = num_entries;
/*		/*
* In Single Doorbell mode, the number of queue entries and the number		* In Single Doorbell mode, the number of queue entries and the number
* of trackers are the same.		* of trackers are the same.
*/		*/
req_queue->num_trackers = num_entries;		req_queue->num_trackers = num_entries;

/* Single Doorbell mode uses only one queue. (UFSHCI_SDB_Q = 0) */		/* Single Doorbell mode uses only one queue. (UFSHCI_SDB_Q = 0) */
req_queue->hwq = malloc(sizeof(struct ufshci_hw_queue), M_UFSHCI,		req_queue->hwq = malloc(sizeof(struct ufshci_hw_queue), M_UFSHCI,
M_ZERO \| M_NOWAIT);		M_ZERO \| M_NOWAIT);
hwq = &req_queue->hwq[UFSHCI_SDB_Q];		hwq = &req_queue->hwq[UFSHCI_SDB_Q];
hwq->num_entries = req_queue->num_entries;		hwq->num_entries = req_queue->num_entries;
hwq->num_trackers = req_queue->num_trackers;		hwq->num_trackers = req_queue->num_trackers;
req_queue->hwq->ucd_bus_addr = malloc(sizeof(bus_addr_t) *		hwq->ctrlr = ctrlr;
req_queue->num_trackers,		hwq->req_queue = req_queue;
M_UFSHCI, M_ZERO \| M_NOWAIT);

mtx_init(&hwq->qlock, "ufshci req_queue lock", NULL, MTX_DEF);		base = is_task_mgmt ? "ufshci utmrq" : "ufshci utrq";
		snprintf(qlock_name, sizeof(qlock_name), "%s #%d lock", base,
		UFSHCI_SDB_Q);
		snprintf(recovery_lock_name, sizeof(recovery_lock_name),
		"%s #%d recovery lock", base, UFSHCI_SDB_Q);

		mtx_init(&hwq->qlock, qlock_name, NULL, MTX_DEF);
		mtx_init(&hwq->recovery_lock, recovery_lock_name, NULL, MTX_DEF);

		callout_init_mtx(&hwq->timer, &hwq->recovery_lock, 0);
		hwq->timer_armed = false;
		hwq->recovery_state = RECOVERY_WAITING;

/*		/*
* Allocate physical memory for request queue (UTP Transfer Request		* Allocate physical memory for request queue (UTP Transfer Request
* Descriptor (UTRD) or UTP Task Management Request Descriptor (UTMRD))		* Descriptor (UTRD) or UTP Task Management Request Descriptor (UTMRD))
* Note: UTRD/UTMRD format is restricted to 1024-byte alignment.		* Note: UTRD/UTMRD format is restricted to 1024-byte alignment.
*/		*/
desc_size = is_task_mgmt ?		desc_size = is_task_mgmt ?
sizeof(struct ufshci_utp_task_mgmt_req_desc) :		sizeof(struct ufshci_utp_task_mgmt_req_desc) :
sizeof(struct ufshci_utp_xfer_req_desc);		sizeof(struct ufshci_utp_xfer_req_desc);
Show All 28 Lines	ufshci_req_sdb_construct(struct ufshci_controller *ctrlr,
hwq->num_failures = 0;		hwq->num_failures = 0;
hwq->req_queue_addr = queuemem_phys;		hwq->req_queue_addr = queuemem_phys;

/* Allocate trackers */		/* Allocate trackers */
hwq->act_tr = malloc_domainset(sizeof(struct ufshci_tracker )		hwq->act_tr = malloc_domainset(sizeof(struct ufshci_tracker )
req_queue->num_entries,		req_queue->num_entries,
M_UFSHCI, DOMAINSET_PREF(req_queue->domain), M_ZERO \| M_WAITOK);		M_UFSHCI, DOMAINSET_PREF(req_queue->domain), M_ZERO \| M_WAITOK);

		TAILQ_INIT(&hwq->free_tr);
		TAILQ_INIT(&hwq->outstanding_tr);

for (i = 0; i < req_queue->num_trackers; i++) {		for (i = 0; i < req_queue->num_trackers; i++) {
tr = malloc_domainset(sizeof(struct ufshci_tracker), M_UFSHCI,		tr = malloc_domainset(sizeof(struct ufshci_tracker), M_UFSHCI,
DOMAINSET_PREF(req_queue->domain), M_ZERO \| M_WAITOK);		DOMAINSET_PREF(req_queue->domain), M_ZERO \| M_WAITOK);

tr->req_queue = req_queue;		tr->req_queue = req_queue;
tr->slot_num = i;		tr->slot_num = i;
tr->slot_state = UFSHCI_SLOT_STATE_FREE;		tr->slot_state = UFSHCI_SLOT_STATE_FREE;
		TAILQ_INSERT_HEAD(&hwq->free_tr, tr, tailq);

hwq->act_tr[i] = tr;		hwq->act_tr[i] = tr;
}		}

if (is_task_mgmt) {		if (is_task_mgmt) {
/* UTP Task Management Request (UTMR) */		/* UTP Task Management Request (UTMR) */
uint32_t utmrlba, utmrlbau;		uint32_t utmrlba, utmrlbau;

Show All 13 Lines	if (is_task_mgmt) {
* Allocate physical memory for the command descriptor.		* Allocate physical memory for the command descriptor.
* UTP Transfer Request (UTR) requires memory for a separate		* UTP Transfer Request (UTR) requires memory for a separate
* command in addition to the queue.		* command in addition to the queue.
*/		*/
if (ufshci_req_sdb_cmd_desc_construct(req_queue, num_entries,		if (ufshci_req_sdb_cmd_desc_construct(req_queue, num_entries,
ctrlr) != 0) {		ctrlr) != 0) {
ufshci_printf(ctrlr,		ufshci_printf(ctrlr,
"failed to construct cmd descriptor memory\n");		"failed to construct cmd descriptor memory\n");
bus_dmamem_free(hwq->dma_tag_queue, hwq->utrd,
hwq->queuemem_map);
goto out;		goto out;
}		}

utrlba = hwq->req_queue_addr & 0xffffffff;		utrlba = hwq->req_queue_addr & 0xffffffff;
utrlbau = hwq->req_queue_addr >> 32;		utrlbau = hwq->req_queue_addr >> 32;
ufshci_mmio_write_4(ctrlr, utrlba, utrlba);		ufshci_mmio_write_4(ctrlr, utrlba, utrlba);
ufshci_mmio_write_4(ctrlr, utrlbau, utrlbau);		ufshci_mmio_write_4(ctrlr, utrlbau, utrlbau);
}		}

return (0);		return (0);
out:		out:
ufshci_req_sdb_destroy(ctrlr, req_queue);		ufshci_req_sdb_destroy(ctrlr, req_queue);
return (ENOMEM);		return (ENOMEM);
}		}

void		void
ufshci_req_sdb_destroy(struct ufshci_controller *ctrlr,		ufshci_req_sdb_destroy(struct ufshci_controller *ctrlr,
struct ufshci_req_queue *req_queue)		struct ufshci_req_queue *req_queue)
{		{
struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];		struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
struct ufshci_tracker *tr;		struct ufshci_tracker *tr;
int i;		int i;

		mtx_lock(&hwq->recovery_lock);
		hwq->timer_armed = false;
		mtx_unlock(&hwq->recovery_lock);
		callout_drain(&hwq->timer);

if (!req_queue->is_task_mgmt)		if (!req_queue->is_task_mgmt)
ufshci_req_sdb_cmd_desc_destroy(&ctrlr->transfer_req_queue);		ufshci_req_sdb_cmd_desc_destroy(&ctrlr->transfer_req_queue);

for (i = 0; i < req_queue->num_trackers; i++) {		for (i = 0; i < req_queue->num_trackers; i++) {
tr = hwq->act_tr[i];		tr = hwq->act_tr[i];
free(tr, M_UFSHCI);		free(tr, M_UFSHCI);
}		}

Show All 9 Lines	if (hwq->utrd != NULL) {
hwq->utrd = NULL;		hwq->utrd = NULL;
}		}

if (hwq->dma_tag_queue) {		if (hwq->dma_tag_queue) {
bus_dma_tag_destroy(hwq->dma_tag_queue);		bus_dma_tag_destroy(hwq->dma_tag_queue);
hwq->dma_tag_queue = NULL;		hwq->dma_tag_queue = NULL;
}		}

		if (mtx_initialized(&hwq->recovery_lock))
		mtx_destroy(&hwq->recovery_lock);
if (mtx_initialized(&hwq->qlock))		if (mtx_initialized(&hwq->qlock))
mtx_destroy(&hwq->qlock);		mtx_destroy(&hwq->qlock);

free(req_queue->hwq->ucd_bus_addr, M_UFSHCI);
free(req_queue->hwq, M_UFSHCI);		free(req_queue->hwq, M_UFSHCI);
}		}

struct ufshci_hw_queue *		struct ufshci_hw_queue *
ufshci_req_sdb_get_hw_queue(struct ufshci_req_queue *req_queue)		ufshci_req_sdb_get_hw_queue(struct ufshci_req_queue *req_queue)
{		{
return &req_queue->hwq[UFSHCI_SDB_Q];		return &req_queue->hwq[UFSHCI_SDB_Q];
}		}

		void
		ufshci_req_sdb_disable(struct ufshci_controller *ctrlr,
		struct ufshci_req_queue *req_queue)
		{
		struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
		struct ufshci_tracker tr, tr_temp;

		mtx_lock(&hwq->recovery_lock);
		mtx_lock(&hwq->qlock);

		if (mtx_initialized(&hwq->recovery_lock))
		mtx_assert(&hwq->recovery_lock, MA_OWNED);
		if (mtx_initialized(&hwq->qlock))
		mtx_assert(&hwq->qlock, MA_OWNED);

		hwq->recovery_state = RECOVERY_WAITING;
		TAILQ_FOREACH_SAFE(tr, &hwq->outstanding_tr, tailq, tr_temp) {
		tr->deadline = SBT_MAX;
		}

		mtx_unlock(&hwq->qlock);
		mtx_unlock(&hwq->recovery_lock);
		}

int		int
ufshci_req_sdb_enable(struct ufshci_controller *ctrlr,		ufshci_req_sdb_enable(struct ufshci_controller *ctrlr,
struct ufshci_req_queue *req_queue)		struct ufshci_req_queue *req_queue)
{		{
		struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];

if (req_queue->is_task_mgmt) {		if (req_queue->is_task_mgmt) {
uint32_t hcs, utmrldbr, utmrlrsr;		uint32_t hcs, utmrldbr, utmrlrsr;

hcs = ufshci_mmio_read_4(ctrlr, hcs);		hcs = ufshci_mmio_read_4(ctrlr, hcs);
if (!(hcs & UFSHCIM(UFSHCI_HCS_REG_UTMRLRDY))) {		if (!(hcs & UFSHCIM(UFSHCI_HCS_REG_UTMRLRDY))) {
ufshci_printf(ctrlr,		ufshci_printf(ctrlr,
"UTP task management request list is not ready\n");		"UTP task management request list is not ready\n");
return (ENXIO);		return (ENXIO);
Show All 35 Lines	if (utrlcnr != 0) {
"Clear the UTP transfer request list notification\n");		"Clear the UTP transfer request list notification\n");
ufshci_mmio_write_4(ctrlr, utrlcnr, utrlcnr);		ufshci_mmio_write_4(ctrlr, utrlcnr, utrlcnr);
}		}

utrlrsr = UFSHCIM(UFSHCI_UTRLRSR_REG_UTRLRSR);		utrlrsr = UFSHCIM(UFSHCI_UTRLRSR_REG_UTRLRSR);
ufshci_mmio_write_4(ctrlr, utrlrsr, utrlrsr);		ufshci_mmio_write_4(ctrlr, utrlrsr, utrlrsr);
}		}

		if (mtx_initialized(&hwq->recovery_lock))
		mtx_assert(&hwq->recovery_lock, MA_OWNED);
		if (mtx_initialized(&hwq->qlock))
		mtx_assert(&hwq->qlock, MA_OWNED);
		KASSERT(!req_queue->ctrlr->is_failed, ("Enabling a failed hwq\n"));

		hwq->recovery_state = RECOVERY_NONE;

return (0);		return (0);
}		}

int		int
ufshci_req_sdb_reserve_slot(struct ufshci_req_queue *req_queue,		ufshci_req_sdb_reserve_slot(struct ufshci_req_queue *req_queue,
struct ufshci_tracker **tr)		struct ufshci_tracker **tr)
{		{
struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];		struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines

bool		bool
ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue)		ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue)
{		{
struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];		struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q];
struct ufshci_tracker *tr;		struct ufshci_tracker *tr;
uint8_t slot;		uint8_t slot;
bool done = false;		bool done = false;

		mtx_assert(&hwq->recovery_lock, MA_OWNED);

hwq->num_intr_handler_calls++;		hwq->num_intr_handler_calls++;

bus_dmamap_sync(hwq->dma_tag_queue, hwq->queuemem_map,		bus_dmamap_sync(hwq->dma_tag_queue, hwq->queuemem_map,
BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);		BUS_DMASYNC_POSTREAD \| BUS_DMASYNC_POSTWRITE);

for (slot = 0; slot < req_queue->num_entries; slot++) {		for (slot = 0; slot < req_queue->num_entries; slot++) {
tr = hwq->act_tr[slot];		tr = hwq->act_tr[slot];
Show All 24 Lines