Page MenuHomeFreeBSD
Files F177284416

D61118.1794083043.diff
No OneTemporary
Actions

Authored By
Unknown
Size
7 KB
Referenced Files
None
Subscribers
None

D61118.1794083043.diff
View Options

diff --git a/sys/dev/ufshci/ufshci_ctrlr.c b/sys/dev/ufshci/ufshci_ctrlr.c
--- a/sys/dev/ufshci/ufshci_ctrlr.c
+++ b/sys/dev/ufshci/ufshci_ctrlr.c
@@ -167,6 +167,15 @@
/* TODO: Configure Background Operations */
+ /*
+ * The device is ready for I/O now, so let the held I/O go out. The
+ * steps below need I/O to flow: a bus scan and the lookup of the
+ * well-known LUN that it creates.
+ */
+ ctrlr->io_ready = true;
+ ufshci_req_queue_resume(&ctrlr->task_mgmt_req_queue);
+ ufshci_req_queue_resume(&ctrlr->transfer_req_queue);
+
/*
* A reset normally arrives after the SIM is attached. But if the
* first start attempt failed early, the reset path runs without a
@@ -359,6 +368,7 @@
int error;
ctrlr->device_init_timeout_in_ms = UFSHCI_DEVICE_INIT_TIMEOUT_MS;
+ ctrlr->io_ready = false;
ctrlr->uic_cmd_timeout_in_ms = UFSHCI_UIC_CMD_TIMEOUT_MS;
ctrlr->dev = dev;
ctrlr->sc_unit = device_get_unit(dev);
@@ -548,6 +558,8 @@
if (ctrlr->taskqueue == NULL)
return;
+ /* I/O waits from here until the reset has brought the device up. */
+ ctrlr->io_ready = false;
taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
}
diff --git a/sys/dev/ufshci/ufshci_private.h b/sys/dev/ufshci/ufshci_private.h
--- a/sys/dev/ufshci/ufshci_private.h
+++ b/sys/dev/ufshci/ufshci_private.h
@@ -180,6 +180,8 @@
TAILQ_HEAD(, ufshci_tracker) free_tr;
TAILQ_HEAD(, ufshci_tracker) outstanding_tr;
+ /* I/O held back while the queue is in recovery (qlock). */
+ STAILQ_HEAD(, ufshci_request) queued_req;
/*
* A Request List using the single doorbell method uses a dedicated
@@ -412,6 +414,12 @@
uint32_t max_rx_lanes;
uint32_t is_failed;
+ /*
+ * The bring-up finished, so I/O may go to the device. Cleared when
+ * a reset is requested. Read under qlock, written without it, so a
+ * stale read costs at most one doorbell that the disable path claims.
+ */
+ bool io_ready;
/* Debug aids: requests left to drop, per I/O and admin. */
uint32_t debug_drop_ios;
@@ -514,6 +522,8 @@
int ufshci_utr_req_queue_enable(struct ufshci_controller *ctrlr);
void ufshci_req_queue_stop_watchdog(struct ufshci_req_queue *req_queue);
void ufshci_req_queue_complete_aborted_hwq(struct ufshci_hw_queue *hwq);
+void ufshci_req_queue_resume(struct ufshci_req_queue *req_queue);
+void ufshci_req_queue_submit_queued(struct ufshci_hw_queue *hwq);
void ufshci_req_queue_fail(struct ufshci_controller *ctrlr,
struct ufshci_req_queue *req_queue);
int ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,
diff --git a/sys/dev/ufshci/ufshci_req_queue.c b/sys/dev/ufshci/ufshci_req_queue.c
--- a/sys/dev/ufshci/ufshci_req_queue.c
+++ b/sys/dev/ufshci/ufshci_req_queue.c
@@ -138,6 +138,17 @@
&ctrlr->transfer_req_queue));
}
+/* Let the held I/O go out, once the controller is ready for I/O. */
+void
+ufshci_req_queue_resume(struct ufshci_req_queue *req_queue)
+{
+ struct ufshci_hw_queue *hwq = req_queue->qops.get_hw_queue(req_queue);
+
+ mtx_lock(&hwq->qlock);
+ ufshci_req_queue_submit_queued(hwq);
+ mtx_unlock(&hwq->qlock);
+}
+
/*
* Stop the watchdog of the hardware queue. The trackers stay in
* place, so a reset that is still running can finish with them.
@@ -220,6 +231,28 @@
ufshci_req_queue_complete_tracker(tr);
}
+/*
+ * Complete a request that never got a tracker, so there is no
+ * descriptor to write the response into. The caller must not hold
+ * the queue lock, because the callback may submit a request.
+ */
+static void
+ufshci_req_queue_manual_complete_request(struct ufshci_request *req,
+ uint8_t ocs, uint8_t rc)
+{
+ struct ufshci_completion cpl;
+
+ memset(&cpl, 0, sizeof(cpl));
+ cpl.size = req->response_size;
+ cpl.response_upiu.header.response = rc;
+ cpl.ocs = ocs;
+
+ if (req->cb_fn)
+ req->cb_fn(req->cb_arg, &cpl, true);
+
+ ufshci_free_request(req);
+}
+
/*
* Complete every tracker the reset claimed. The claim keeps the
* completion scan away, so no tracker completes twice.
@@ -260,6 +293,7 @@
{
struct ufshci_hw_queue *hwq = req_queue->qops.get_hw_queue(req_queue);
struct ufshci_tracker *tr;
+ struct ufshci_request *req;
int i;
if (!mtx_initialized(&hwq->qlock))
@@ -295,6 +329,16 @@
mtx_lock(&hwq->qlock);
}
+ /* The held I/O never reached the hardware, so no slot to clear. */
+ while ((req = STAILQ_FIRST(&hwq->queued_req)) != NULL) {
+ STAILQ_REMOVE_HEAD(&hwq->queued_req, stailq);
+ mtx_unlock(&hwq->qlock);
+ ufshci_req_queue_manual_complete_request(req,
+ UFSHCI_DESC_ABORTED,
+ UFSHCI_RESPONSE_CODE_GENERAL_FAILURE);
+ mtx_lock(&hwq->qlock);
+ }
+
mtx_unlock(&hwq->qlock);
}
@@ -380,6 +424,10 @@
TAILQ_REMOVE(&hwq->outstanding_tr, tr, tailq);
TAILQ_INSERT_HEAD(&hwq->free_tr, tr, tailq);
+
+ /* A slot came free, so the next held I/O can go out. */
+ if (req_queue->ctrlr->io_ready && !req_queue->ctrlr->is_failed)
+ ufshci_req_queue_submit_queued(hwq);
}
mtx_unlock(&tr->hwq->qlock);
@@ -860,14 +908,27 @@
_ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,
struct ufshci_request *req)
{
+ struct ufshci_hw_queue *hwq = req_queue->qops.get_hw_queue(req_queue);
struct ufshci_tracker *tr = NULL;
int error;
- mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED);
+ mtx_assert(&hwq->qlock, MA_OWNED);
if (req_queue->ctrlr->is_failed)
return (ENXIO);
+ /*
+ * The controller is not ready for I/O: a reset was asked for, or
+ * the bring-up has not finished. Hold the I/O until it is. Admin
+ * and task management requests still go out, because the reset
+ * task waits on its own commands and the watchdog sends the aborts.
+ */
+ if (!req->is_admin && !req_queue->is_task_mgmt &&
+ !req_queue->ctrlr->io_ready) {
+ STAILQ_INSERT_TAIL(&hwq->queued_req, req, stailq);
+ return (0);
+ }
+
error = req_queue->qops.reserve_slot(req_queue, &tr,
req->is_admin);
if (error != 0) {
@@ -896,6 +957,38 @@
return (0);
}
+/*
+ * Send the held I/O in order, as far as the free slots go. The rest
+ * waits for the next completion.
+ */
+void
+ufshci_req_queue_submit_queued(struct ufshci_hw_queue *hwq)
+{
+ struct ufshci_req_queue *req_queue = hwq->req_queue;
+ struct ufshci_request *req;
+ struct ufshci_tracker *tr;
+
+ mtx_assert(&hwq->qlock, MA_OWNED);
+
+ while ((req = STAILQ_FIRST(&hwq->queued_req)) != NULL) {
+ /*
+ * A reset can start while this runs. The submit would then put
+ * each request back at the tail, and this loop would never end.
+ */
+ if (!req_queue->ctrlr->io_ready || req_queue->ctrlr->is_failed)
+ break;
+ /* Probe for an I/O slot first, so a full queue stays quiet. */
+ if (req_queue->qops.reserve_slot(req_queue, &tr, false) != 0)
+ break;
+ STAILQ_REMOVE_HEAD(&hwq->queued_req, stailq);
+ if (_ufshci_req_queue_submit_request(req_queue, req) != 0) {
+ /* The failure path completes what is left here. */
+ STAILQ_INSERT_HEAD(&hwq->queued_req, req, stailq);
+ break;
+ }
+ }
+}
+
int
ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,
struct ufshci_request *req)
diff --git a/sys/dev/ufshci/ufshci_req_sdb.c b/sys/dev/ufshci/ufshci_req_sdb.c
--- a/sys/dev/ufshci/ufshci_req_sdb.c
+++ b/sys/dev/ufshci/ufshci_req_sdb.c
@@ -257,6 +257,7 @@
TAILQ_INIT(&hwq->free_tr);
TAILQ_INIT(&hwq->outstanding_tr);
+ STAILQ_INIT(&hwq->queued_req);
for (i = 0; i < req_queue->num_trackers; i++) {
tr = malloc_domainset(sizeof(struct ufshci_tracker), M_UFSHCI,

File Metadata

Mime Type
text/plain
Expires
Sat, Nov 7, 8:24 PM (5 h, 50 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
41554842
Default Alt Text
D61118.1794083043.diff (7 KB)

Event Timeline