Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/ufshci/ufshci_req_queue.c
- This file was added.
| /*- | ||||||||||
| * SPDX-License-Identifier: BSD-2-Clause | ||||||||||
| * | ||||||||||
| * Copyright (c) 2025, Samsung Electronics Co., Ltd. | ||||||||||
| * All rights reserved. | ||||||||||
| * | ||||||||||
| * Redistribution and use in source and binary forms, with or without | ||||||||||
| * modification, are permitted provided that the following conditions | ||||||||||
| * are met: | ||||||||||
| * 1. Redistributions of source code must retain the above copyright | ||||||||||
| * notice, this list of conditions and the following disclaimer. | ||||||||||
| * 2. Redistributions in binary form must reproduce the above copyright | ||||||||||
| * notice, this list of conditions and the following disclaimer in the | ||||||||||
| * documentation and/or other materials provided with the distribution. | ||||||||||
| * | ||||||||||
| * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND | ||||||||||
| * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||||||||||
| * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||||||||||
| * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE | ||||||||||
| * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | ||||||||||
| * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | ||||||||||
| * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | ||||||||||
| * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | ||||||||||
| * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | ||||||||||
| * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | ||||||||||
| * SUCH DAMAGE. | ||||||||||
| */ | ||||||||||
| #include <sys/param.h> | ||||||||||
| #include <sys/bus.h> | ||||||||||
| #include <sys/conf.h> | ||||||||||
| #include <sys/module.h> | ||||||||||
| #include <sys/domainset.h> | ||||||||||
| #include <vm/uma.h> | ||||||||||
| #include <cam/scsi/scsi_all.h> | ||||||||||
| #include "sys/kassert.h" | ||||||||||
| #include "ufshci_private.h" | ||||||||||
| static void | ||||||||||
| ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue, | ||||||||||
| struct ufshci_tracker *tr, enum ufshci_data_direction data_direction); | ||||||||||
| static const struct ufshci_qops sdb_qops = { | ||||||||||
| .construct = ufshci_req_sdb_construct, | ||||||||||
| .destroy = ufshci_req_sdb_destroy, | ||||||||||
| .get_hw_queue = ufshci_req_sdb_get_hw_queue, | ||||||||||
| .enable = ufshci_req_sdb_enable, | ||||||||||
| .reserve_slot = ufshci_req_sdb_reserve_slot, | ||||||||||
| .reserve_admin_slot = ufshci_req_sdb_reserve_slot, | ||||||||||
| .ring_doorbell = ufshci_req_sdb_ring_doorbell, | ||||||||||
| .clear_cpl_ntf = ufshci_req_sdb_clear_cpl_ntf, | ||||||||||
| .process_cpl = ufshci_req_sdb_process_cpl, | ||||||||||
| .get_inflight_io = ufshci_req_sdb_get_inflight_io, | ||||||||||
| }; | ||||||||||
| int | ||||||||||
| ufshci_utm_req_queue_construct(struct ufshci_controller *ctrlr) | ||||||||||
| { | ||||||||||
| struct ufshci_req_queue *req_queue; | ||||||||||
| int error; | ||||||||||
| /* | ||||||||||
| * UTP Task Management Request only supports Legacy Single Doorbell | ||||||||||
| * Queue. | ||||||||||
| */ | ||||||||||
| req_queue = &ctrlr->task_mgmt_req_queue; | ||||||||||
| req_queue->queue_mode = UFSHCI_Q_MODE_SDB; | ||||||||||
| req_queue->qops = sdb_qops; | ||||||||||
| error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTRM_ENTRIES, | ||||||||||
| /*is_task_mgmt*/true); | ||||||||||
| return (error); | ||||||||||
| } | ||||||||||
| void ufshci_utm_req_queue_destroy(struct ufshci_controller *ctrlr) { | ||||||||||
jrtc27Unsubmitted Done Inline Actions
jrtc27: | ||||||||||
| ctrlr->task_mgmt_req_queue.qops.destroy(ctrlr, | ||||||||||
| &ctrlr->task_mgmt_req_queue); | ||||||||||
| } | ||||||||||
| int | ||||||||||
| ufshci_utm_req_queue_enable(struct ufshci_controller *ctrlr) | ||||||||||
| { | ||||||||||
| return (ctrlr->task_mgmt_req_queue.qops.enable(ctrlr, | ||||||||||
| &ctrlr->task_mgmt_req_queue)); | ||||||||||
| } | ||||||||||
| int | ||||||||||
| ufshci_ut_req_queue_construct(struct ufshci_controller *ctrlr) | ||||||||||
| { | ||||||||||
| struct ufshci_req_queue *req_queue; | ||||||||||
| int error; | ||||||||||
| /* | ||||||||||
| * Currently, it does not support MCQ mode, so it should be set to SDB | ||||||||||
| * mode by default. | ||||||||||
| * TODO: Determine queue mode by checking Capability Registers | ||||||||||
| */ | ||||||||||
| req_queue = &ctrlr->transfer_req_queue; | ||||||||||
| req_queue->queue_mode = UFSHCI_Q_MODE_SDB; | ||||||||||
| req_queue->qops = sdb_qops; | ||||||||||
| error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTR_ENTRIES, | ||||||||||
| /*is_task_mgmt*/false); | ||||||||||
| return (error); | ||||||||||
| } | ||||||||||
| void | ||||||||||
| ufshci_ut_req_queue_destroy(struct ufshci_controller *ctrlr) { | ||||||||||
| ctrlr->transfer_req_queue.qops.destroy(ctrlr, | ||||||||||
| &ctrlr->transfer_req_queue); | ||||||||||
| } | ||||||||||
| int | ||||||||||
| ufshci_ut_req_queue_enable(struct ufshci_controller *ctrlr) | ||||||||||
| { | ||||||||||
| return (ctrlr->transfer_req_queue.qops.enable(ctrlr, | ||||||||||
| &ctrlr->transfer_req_queue)); | ||||||||||
| } | ||||||||||
| static bool | ||||||||||
| ufshci_req_queue_req_desc_is_error(struct ufshci_req_queue *req_queue, | ||||||||||
Done Inline ActionsThis seems a rather useless function, and it's a bit unintuitive that a function called ufshci_req_queue_req_desc_is_error will print things jrtc27: This seems a rather useless function, and it's a bit unintuitive that a function called… | ||||||||||
Done Inline ActionsI removed this function. Thanks! jaeyoon: I removed this function. Thanks! | ||||||||||
| struct ufshci_tracker *tr) | ||||||||||
| { | ||||||||||
| struct ufshci_utp_xfer_req_desc *desc; | ||||||||||
| desc = &tr->hwq->utrd[tr->slot_num]; | ||||||||||
| if (desc->overall_command_status != UFSHCI_DESC_SUCCESS) { | ||||||||||
| ufshci_printf(req_queue->ctrlr, "Invalid OCS = 0x%x\n", | ||||||||||
| desc->overall_command_status); | ||||||||||
| return (true); | ||||||||||
| } | ||||||||||
| return (false); | ||||||||||
| } | ||||||||||
| static bool | ||||||||||
| ufshci_req_queue_response_is_error(struct ufshci_req_queue *req_queue, | ||||||||||
| struct ufshci_tracker *tr, union ufshci_reponse_upiu *response) | ||||||||||
| { | ||||||||||
| bool is_error = false; | ||||||||||
| /* Check request descriptor */ | ||||||||||
| if (ufshci_req_queue_req_desc_is_error(req_queue, tr)) { | ||||||||||
| // For debug | ||||||||||
| int error_code, sense_key, asc, ascq; | ||||||||||
| scsi_extract_sense_len((struct scsi_sense_data *)response->cmd_response_upiu.sense_data, | ||||||||||
| response->cmd_response_upiu.sense_data_len, | ||||||||||
Done Inline ActionsSpace. Though I'm not sure about this stream of @s. Also the driver's inconsistent about whether the "For debug" code is enabled or not. jrtc27: Space. Though I'm not sure about this stream of @s. Also the driver's inconsistent about… | ||||||||||
Done Inline ActionsThis is debug code that I need during development. I will be removing it before committing, thank you for your understanding. jaeyoon: This is debug code that I need during development. I will be removing it before committing… | ||||||||||
| &error_code, &sense_key, &asc, &ascq, 1); | ||||||||||
| ufshci_printf(req_queue->ctrlr, "sense_key = 0x%x\n", sense_key); | ||||||||||
| ufshci_printf(req_queue->ctrlr, "asc = 0x%x\n", asc); | ||||||||||
| ufshci_printf(req_queue->ctrlr, "ascq = 0x%x\n", ascq); | ||||||||||
| ufshci_printf(req_queue->ctrlr,"@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"); | ||||||||||
| is_error = true; | ||||||||||
| } | ||||||||||
| /* Check response UPIU header */ | ||||||||||
| if (response->header.response != UFSHCI_RESPONSE_CODE_TARGET_SUCCESS) { | ||||||||||
| ufshci_printf(req_queue->ctrlr, "Invalid response code = 0x%x\n", | ||||||||||
| response->header.response); | ||||||||||
| is_error = true; | ||||||||||
| } | ||||||||||
| return (is_error); | ||||||||||
| } | ||||||||||
| void | ||||||||||
| ufshci_req_queue_complete_tracker(struct ufshci_tracker *tr) | ||||||||||
| { | ||||||||||
| struct ufshci_req_queue *req_queue = tr->req_queue; | ||||||||||
| struct ufshci_request *req = tr->req; | ||||||||||
| struct ufshci_completion cpl; | ||||||||||
| bool retry, error, retriable; | ||||||||||
| mtx_assert(&tr->hwq->qlock, MA_NOTOWNED); | ||||||||||
| bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, | ||||||||||
| BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||||||||
| cpl.size = tr->response_size; | ||||||||||
| memcpy(&cpl.response_upiu, (void*)tr->ucd->response_upiu, cpl.size); | ||||||||||
| error = ufshci_req_queue_response_is_error(req_queue, tr, | ||||||||||
| &cpl.response_upiu); | ||||||||||
| /* TODO: Implement retry */ | ||||||||||
| // retriable = ufshci_completion_is_retry(cpl); | ||||||||||
| retriable = false; | ||||||||||
| retry = error && retriable && req->retries < req_queue->ctrlr->retry_count; | ||||||||||
| if (retry) | ||||||||||
| tr->hwq->num_retries++; | ||||||||||
| if (error && req->retries >= req_queue->ctrlr->retry_count && retriable) | ||||||||||
| tr->hwq->num_failures++; | ||||||||||
| KASSERT(tr->req, ("there is no request assigned to the tracker\n")); | ||||||||||
| KASSERT(cpl.response_upiu.header.task_tag == | ||||||||||
| req->request_upiu.header.task_tag, | ||||||||||
| ("response task_tag does not match request task_tag\n")); | ||||||||||
| if (!retry) { | ||||||||||
| if (req->payload_valid) { | ||||||||||
| bus_dmamap_sync(req_queue->dma_tag_payload, | ||||||||||
| tr->payload_dma_map, | ||||||||||
| BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | ||||||||||
| } | ||||||||||
| /* Copy response from the command descriptor */ | ||||||||||
| if (req->cb_fn) | ||||||||||
| req->cb_fn(req->cb_arg, &cpl, error); | ||||||||||
| } | ||||||||||
| mtx_lock(&tr->hwq->qlock); | ||||||||||
| /* Clear the UTRL Completion Notification register */ | ||||||||||
| req_queue->qops.clear_cpl_ntf(req_queue->ctrlr, tr); | ||||||||||
| if (retry) { | ||||||||||
| req->retries++; | ||||||||||
| ufshci_req_queue_submit_tracker(req_queue, tr, req->data_direction); | ||||||||||
| } else { | ||||||||||
| if (req->payload_valid) { | ||||||||||
| bus_dmamap_unload(req_queue->dma_tag_payload, | ||||||||||
| tr->payload_dma_map); | ||||||||||
| } | ||||||||||
| /* Clear tracker */ | ||||||||||
| ufshci_free_request(req); | ||||||||||
| tr->req = NULL; | ||||||||||
| tr->slot_state = UFSHCI_SLOT_STATE_FREE; | ||||||||||
| } | ||||||||||
| mtx_unlock(&tr->hwq->qlock); | ||||||||||
| } | ||||||||||
| bool | ||||||||||
| ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue) | ||||||||||
| { | ||||||||||
| return (req_queue->qops.process_cpl(req_queue)); | ||||||||||
| } | ||||||||||
| static void | ||||||||||
| ufshci_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) | ||||||||||
| { | ||||||||||
| struct ufshci_tracker *tr = arg; | ||||||||||
| struct ufshci_prdt_entry *prdt_entry; | ||||||||||
| int i; | ||||||||||
| /* | ||||||||||
| * If the mapping operation failed, return immediately. The caller | ||||||||||
| * is responsible for detecting the error status and failing the | ||||||||||
| * tracker manually. | ||||||||||
| */ | ||||||||||
| if (error != 0) { | ||||||||||
| ufshci_printf(tr->req_queue->ctrlr, | ||||||||||
| "Failed to map payload %d\n", error); | ||||||||||
| return; | ||||||||||
| } | ||||||||||
| // For debug | ||||||||||
| // ufshci_printf(tr->req_queue->ctrlr, "ufshci_payload_map nseg=%d\n", nseg); | ||||||||||
| prdt_entry = (struct ufshci_prdt_entry *)tr->ucd->prd_table; | ||||||||||
| tr->prdt_entry_cnt = nseg; | ||||||||||
| for (i = 0; i < nseg; i++) { | ||||||||||
| prdt_entry->data_base_address = htole64(seg[i].ds_addr) & 0xffffffff; | ||||||||||
| prdt_entry->data_base_address_upper = htole64(seg[i].ds_addr) >> 32; | ||||||||||
| prdt_entry->data_byte_count = seg[i].ds_len - 1; | ||||||||||
| // For debug | ||||||||||
| // ufshci_printf(tr->req_queue->ctrlr, | ||||||||||
| // "ufshci_payload_map seg[i].ds_addr=0x%lx\n", seg[i].ds_addr); | ||||||||||
| // ufshci_printf(tr->req_queue->ctrlr, | ||||||||||
| // "ufshci_payload_map seg[i].ds_len=0x%lx\n", seg[i].ds_len); | ||||||||||
| ++prdt_entry; | ||||||||||
| } | ||||||||||
| KASSERT(data_left == 0, ("There is still data left to map.")); | ||||||||||
| bus_dmamap_sync(tr->req_queue->dma_tag_payload, tr->payload_dma_map, | ||||||||||
| BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||||||||
| } | ||||||||||
| static void | ||||||||||
| ufshci_req_queue_prepare_prdt(struct ufshci_tracker *tr) | ||||||||||
| { | ||||||||||
| struct ufshci_request *req = tr->req; | ||||||||||
| struct ufshci_utp_cmd_desc *cmd_desc= tr->ucd; | ||||||||||
| int error; | ||||||||||
| tr->prdt_off = UFSHCI_UTP_XFER_REQ_SIZE + UFSHCI_UTP_XFER_RESP_SIZE; | ||||||||||
| memset(cmd_desc->prd_table, 0, sizeof(cmd_desc->prd_table)); | ||||||||||
| /* Filling PRDT enrties with payload */ | ||||||||||
| error = bus_dmamap_load_mem(tr->req_queue->dma_tag_payload, | ||||||||||
| tr->payload_dma_map, &req->payload, ufshci_payload_map, tr, | ||||||||||
| BUS_DMA_NOWAIT); | ||||||||||
| if (error != 0) { | ||||||||||
| /* | ||||||||||
| * The dmamap operation failed, so we manually fail the | ||||||||||
| * tracker here with DATA_TRANSFER_ERROR status. | ||||||||||
| * | ||||||||||
| * ufshci_req_queue_manual_complete_tracker must not be called | ||||||||||
| * with the req_queue lock held. | ||||||||||
| */ | ||||||||||
| ufshci_printf(tr->req_queue->ctrlr, | ||||||||||
| "bus_dmamap_load_mem returned with error:0x%x!\n", error); | ||||||||||
| // TODO: Implement manual complete | ||||||||||
| // mtx_unlock(&tr->hwq->qlock); | ||||||||||
| // ufshci_req_queue_manual_complete_tracker(tr, NVME_SCT_GENERIC, | ||||||||||
| // NVME_SC_DATA_TRANSFER_ERROR, DO_NOT_RETRY, ERROR_PRINT_ALL); | ||||||||||
| // mtx_lock(&tr->hwq->qlock); | ||||||||||
| } | ||||||||||
| } | ||||||||||
| static void | ||||||||||
| ufshci_req_queue_fill_descriptor(struct ufshci_utp_xfer_req_desc *desc, | ||||||||||
| uint8_t data_direction, const uint64_t paddr, | ||||||||||
| const uint16_t response_off, const uint16_t response_len, | ||||||||||
| const uint16_t prdt_off, const uint16_t prdt_entry_cnt) | ||||||||||
| { | ||||||||||
| uint8_t command_type; | ||||||||||
| /* | ||||||||||
| * Set command type to UFS storage. | ||||||||||
| * The UFS 4.1 spec only defines 'UFS Storage' as a command type. | ||||||||||
| */ | ||||||||||
| command_type = UFSHCI_COMMAND_TYPE_UFS_STORAGE; | ||||||||||
| memset(desc, 0, sizeof(struct ufshci_utp_xfer_req_desc)); | ||||||||||
| desc->interrupt = true; | ||||||||||
| desc->data_direction = data_direction; | ||||||||||
| desc->command_type = command_type; | ||||||||||
| /* Set the initial value to Invalid. */ | ||||||||||
| desc->overall_command_status = UFSHCI_OCS_INVALID; | ||||||||||
| desc->utp_command_descriptor_base_address = (uint32_t)(paddr & 0xffffffff); | ||||||||||
| desc->utp_command_descriptor_base_address_upper = (uint32_t)(paddr >> 32); | ||||||||||
| const uint16_t dword_size = 4; | ||||||||||
| desc->response_upiu_offset = response_off / dword_size; | ||||||||||
| desc->response_upiu_length = response_len / dword_size; | ||||||||||
| desc->prdt_offset = prdt_off / dword_size; | ||||||||||
| desc->prdt_length = prdt_entry_cnt; | ||||||||||
| } | ||||||||||
| /* | ||||||||||
| * Submit the tracker to the hardware. | ||||||||||
| */ | ||||||||||
| static void | ||||||||||
| ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue, | ||||||||||
| struct ufshci_tracker *tr, enum ufshci_data_direction data_direction) | ||||||||||
| { | ||||||||||
| struct ufshci_controller *ctrlr = req_queue->ctrlr; | ||||||||||
| struct ufshci_request *req = tr->req; | ||||||||||
| uint64_t ucd_paddr; | ||||||||||
| uint16_t request_len, response_off, response_len; | ||||||||||
| uint8_t slot_num = tr->slot_num; | ||||||||||
| mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED); | ||||||||||
| /* TODO: Check timeout */ | ||||||||||
| request_len = req->request_size; | ||||||||||
| response_off = UFSHCI_UTP_XFER_REQ_SIZE; | ||||||||||
| response_len = req->response_size; | ||||||||||
| /* Prepare UTP Command Descriptor */ | ||||||||||
| memcpy(tr->ucd, &req->request_upiu, request_len); | ||||||||||
| memset((uint8_t *)tr->ucd + response_off, 0, response_len); | ||||||||||
| /* Prepare PRDT */ | ||||||||||
| if (req->payload_valid) | ||||||||||
| ufshci_req_queue_prepare_prdt(tr); | ||||||||||
| bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, | ||||||||||
| BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||||||||
| /* Prepare UTP Transfer Request Descriptor. */ | ||||||||||
| ucd_paddr = tr->ucd_bus_addr; | ||||||||||
| ufshci_req_queue_fill_descriptor(&tr->hwq->utrd[slot_num], | ||||||||||
| data_direction, ucd_paddr, response_off, response_len, | ||||||||||
| tr->prdt_off, tr->prdt_entry_cnt); | ||||||||||
| bus_dmamap_sync(tr->hwq->dma_tag_queue, tr->hwq->queuemem_map, | ||||||||||
| BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | ||||||||||
| tr->slot_state = UFSHCI_SLOT_STATE_SCHEDULED; | ||||||||||
| /* Ring the doorbell */ | ||||||||||
| req_queue->qops.ring_doorbell(ctrlr, tr); | ||||||||||
| } | ||||||||||
| static int | ||||||||||
| _ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue, | ||||||||||
| struct ufshci_request *req) | ||||||||||
| { | ||||||||||
| struct ufshci_tracker *tr = NULL; | ||||||||||
| int error; | ||||||||||
| mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED); | ||||||||||
| error = req_queue->qops.reserve_slot(req_queue, &tr); | ||||||||||
| if (error != 0) { | ||||||||||
| ufshci_printf(req_queue->ctrlr, "Failed to get tracker"); | ||||||||||
| return (error); | ||||||||||
| } | ||||||||||
| if (tr->slot_state == UFSHCI_SLOT_STATE_RESERVED || | ||||||||||
| tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED) | ||||||||||
| return (EBUSY); | ||||||||||
| tr->slot_state = UFSHCI_SLOT_STATE_RESERVED; | ||||||||||
| tr->response_size = req->response_size; | ||||||||||
| tr->deadline = SBT_MAX; | ||||||||||
| tr->req = req; | ||||||||||
| // For debug | ||||||||||
| // ufshci_printf(req_queue->ctrlr, "_ufshci_req_queue_submit_request, slot=%d\n", tr->slot_num); | ||||||||||
| ufshci_req_queue_submit_tracker(req_queue, tr, req->data_direction); | ||||||||||
| return (0); | ||||||||||
Done Inline ActionsStart of the block, though I'm not sure what value this variable adds jrtc27: Start of the block, though I'm not sure what value this variable adds | ||||||||||
Done Inline ActionsDone. I added a comment. /* Value to convert bytes to dwords */ jaeyoon: Done. I added a comment.
```
/* Value to convert bytes to dwords */
``` | ||||||||||
| } | ||||||||||
| int | ||||||||||
| ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue, | ||||||||||
| struct ufshci_request *req, bool is_admin) | ||||||||||
| { | ||||||||||
| struct ufshci_hw_queue *hwq; | ||||||||||
| uint32_t error; | ||||||||||
| /* TODO: MCQs should use a separate Admin queue. */ | ||||||||||
| hwq = req_queue->qops.get_hw_queue(req_queue); | ||||||||||
| KASSERT(hwq, ("There is no HW queue allocated.")); | ||||||||||
| mtx_lock(&hwq->qlock); | ||||||||||
| error = _ufshci_req_queue_submit_request(req_queue, req); | ||||||||||
| mtx_unlock(&hwq->qlock); | ||||||||||
| return (error); | ||||||||||
| } | ||||||||||