diff --git a/sys/dev/ufshci/ufshci_ctrlr.c b/sys/dev/ufshci/ufshci_ctrlr.c index 36be94b8b8b7..35663b480cfa 100644 --- a/sys/dev/ufshci/ufshci_ctrlr.c +++ b/sys/dev/ufshci/ufshci_ctrlr.c @@ -1,523 +1,612 @@ /*- * Copyright (c) 2025, Samsung Electronics Co., Ltd. * Written by Jaeyoon Choi * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include "ufshci_private.h" #include "ufshci_reg.h" +static void +ufshci_ctrlr_fail(struct ufshci_controller *ctrlr) +{ + ctrlr->is_failed = true; + + ufshci_req_queue_fail(ctrlr, + ctrlr->task_mgmt_req_queue.qops.get_hw_queue( + &ctrlr->task_mgmt_req_queue)); + ufshci_req_queue_fail(ctrlr, + ctrlr->transfer_req_queue.qops.get_hw_queue( + &ctrlr->transfer_req_queue)); +} + +static void +ufshci_ctrlr_start(struct ufshci_controller *ctrlr, bool resetting) +{ + TSENTER(); + + /* + * If `resetting` is true, we are on the reset path. + * Re-enable request queues here because ufshci_ctrlr_reset_task() + * disables them during reset. + */ + if (resetting) { + if (ufshci_utmr_req_queue_enable(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + if (ufshci_utr_req_queue_enable(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + } + + if (ufshci_ctrlr_send_nop(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + + /* Initialize UFS target drvice */ + if (ufshci_dev_init(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + + /* Initialize Reference Clock */ + if (ufshci_dev_init_reference_clock(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + + /* Initialize unipro */ + if (ufshci_dev_init_unipro(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + + /* + * Initialize UIC Power Mode + * QEMU UFS devices do not support unipro and power mode. + */ + if (!(ctrlr->quirks & UFSHCI_QUIRK_IGNORE_UIC_POWER_MODE) && + ufshci_dev_init_uic_power_mode(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + + /* Initialize UFS Power Mode */ + if (ufshci_dev_init_ufs_power_mode(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + + /* Read Controller Descriptor (Device, Geometry) */ + if (ufshci_dev_get_descriptor(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + + if (ufshci_dev_config_write_booster(ctrlr)) { + ufshci_ctrlr_fail(ctrlr); + return; + } + + /* TODO: Configure Write Protect */ + + /* TODO: Configure Background Operations */ + + /* + * If the reset is due to a timeout, it is already attached to the SIM + * and does not need to be attached again. + */ + if (!resetting && ufshci_sim_attach(ctrlr) != 0) { + ufshci_ctrlr_fail(ctrlr); + return; + } + + TSEXIT(); +} + static int -ufshci_ctrlr_enable_host_ctrlr(struct ufshci_controller *ctrlr) +ufshci_ctrlr_disable_host_ctrlr(struct ufshci_controller *ctrlr) { int timeout = ticks + MSEC_2_TICKS(ctrlr->device_init_timeout_in_ms); sbintime_t delta_t = SBT_1US; uint32_t hce; hce = ufshci_mmio_read_4(ctrlr, hce); /* If UFS host controller is already enabled, disable it. */ if (UFSHCIV(UFSHCI_HCE_REG_HCE, hce)) { hce &= ~UFSHCIM(UFSHCI_HCE_REG_HCE); ufshci_mmio_write_4(ctrlr, hce, hce); } + /* Wait for the HCE flag to change */ + while (1) { + hce = ufshci_mmio_read_4(ctrlr, hce); + if (!UFSHCIV(UFSHCI_HCE_REG_HCE, hce)) + break; + if (timeout - ticks < 0) { + ufshci_printf(ctrlr, + "host controller failed to disable " + "within %d ms\n", + ctrlr->device_init_timeout_in_ms); + return (ENXIO); + } + + pause_sbt("ufshci_disable_hce", delta_t, 0, C_PREL(1)); + delta_t = min(SBT_1MS, delta_t * 3 / 2); + } + + return (0); +} + +static int +ufshci_ctrlr_enable_host_ctrlr(struct ufshci_controller *ctrlr) +{ + int timeout = ticks + MSEC_2_TICKS(ctrlr->device_init_timeout_in_ms); + sbintime_t delta_t = SBT_1US; + uint32_t hce; + + hce = ufshci_mmio_read_4(ctrlr, hce); + /* Enable UFS host controller */ hce |= UFSHCIM(UFSHCI_HCE_REG_HCE); ufshci_mmio_write_4(ctrlr, hce, hce); /* * During the controller initialization, the value of the HCE bit is * unstable, so we need to read the HCE value after some time after * initialization is complete. */ - pause_sbt("ufshci_hce", ustosbt(100), 0, C_PREL(1)); + pause_sbt("ufshci_enable_hce", ustosbt(100), 0, C_PREL(1)); /* Wait for the HCE flag to change */ while (1) { hce = ufshci_mmio_read_4(ctrlr, hce); if (UFSHCIV(UFSHCI_HCE_REG_HCE, hce)) break; if (timeout - ticks < 0) { ufshci_printf(ctrlr, "host controller failed to enable " "within %d ms\n", ctrlr->device_init_timeout_in_ms); return (ENXIO); } - pause_sbt("ufshci_hce", delta_t, 0, C_PREL(1)); + pause_sbt("ufshci_enable_hce", delta_t, 0, C_PREL(1)); delta_t = min(SBT_1MS, delta_t * 3 / 2); } return (0); } +static int +ufshci_ctrlr_disable(struct ufshci_controller *ctrlr) +{ + int error; + + /* Disable all interrupts */ + ufshci_mmio_write_4(ctrlr, ie, 0); + + error = ufshci_ctrlr_disable_host_ctrlr(ctrlr); + return (error); +} + +static int +ufshci_ctrlr_enable(struct ufshci_controller *ctrlr) +{ + uint32_t ie, hcs; + int error; + + error = ufshci_ctrlr_enable_host_ctrlr(ctrlr); + if (error) + return (error); + + /* Send DME_LINKSTARTUP command to start the link startup procedure */ + error = ufshci_uic_send_dme_link_startup(ctrlr); + if (error) + return (error); + + /* + * The device_present(UFSHCI_HCS_REG_DP) bit becomes true if the host + * controller has successfully received a Link Startup UIC command + * response and the UFS device has found a physical link to the + * controller. + */ + hcs = ufshci_mmio_read_4(ctrlr, hcs); + if (!UFSHCIV(UFSHCI_HCS_REG_DP, hcs)) { + ufshci_printf(ctrlr, "UFS device not found\n"); + return (ENXIO); + } + + /* Enable additional interrupts by programming the IE register. */ + ie = ufshci_mmio_read_4(ctrlr, ie); + ie |= UFSHCIM(UFSHCI_IE_REG_UTRCE); /* UTR Completion */ + ie |= UFSHCIM(UFSHCI_IE_REG_UEE); /* UIC Error */ + ie |= UFSHCIM(UFSHCI_IE_REG_UTMRCE); /* UTMR Completion */ + ie |= UFSHCIM(UFSHCI_IE_REG_DFEE); /* Device Fatal Error */ + ie |= UFSHCIM(UFSHCI_IE_REG_UTPEE); /* UTP Error */ + ie |= UFSHCIM(UFSHCI_IE_REG_HCFEE); /* Host Ctrlr Fatal Error */ + ie |= UFSHCIM(UFSHCI_IE_REG_SBFEE); /* System Bus Fatal Error */ + ie |= UFSHCIM(UFSHCI_IE_REG_CEFEE); /* Crypto Engine Fatal Error */ + ufshci_mmio_write_4(ctrlr, ie, ie); + + /* TODO: Initialize interrupt Aggregation Control Register (UTRIACR) */ + + return (0); +} + +static int +ufshci_ctrlr_hw_reset(struct ufshci_controller *ctrlr) +{ + int error; + + error = ufshci_ctrlr_disable(ctrlr); + if (error) + return (error); + + error = ufshci_ctrlr_enable(ctrlr); + return (error); +} + +static void +ufshci_ctrlr_reset_task(void *arg, int pending) +{ + struct ufshci_controller *ctrlr = arg; + int error; + + /* Release resources */ + ufshci_utmr_req_queue_disable(ctrlr); + ufshci_utr_req_queue_disable(ctrlr); + + error = ufshci_ctrlr_hw_reset(ctrlr); + if (error) + return (ufshci_ctrlr_fail(ctrlr)); + + ufshci_ctrlr_start(ctrlr, true); +} + int ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev) { - uint32_t ver, cap, hcs, ie, ahit; + uint32_t ver, cap, ahit; uint32_t timeout_period, retry_count; int error; ctrlr->device_init_timeout_in_ms = UFSHCI_DEVICE_INIT_TIMEOUT_MS; ctrlr->uic_cmd_timeout_in_ms = UFSHCI_UIC_CMD_TIMEOUT_MS; ctrlr->dev = dev; ctrlr->sc_unit = device_get_unit(dev); snprintf(ctrlr->sc_name, sizeof(ctrlr->sc_name), "%s", device_get_nameunit(dev)); mtx_init(&ctrlr->sc_mtx, device_get_nameunit(dev), NULL, MTX_DEF | MTX_RECURSE); mtx_init(&ctrlr->uic_cmd_lock, "ufshci ctrlr uic cmd lock", NULL, MTX_DEF); ver = ufshci_mmio_read_4(ctrlr, ver); ctrlr->major_version = UFSHCIV(UFSHCI_VER_REG_MJR, ver); ctrlr->minor_version = UFSHCIV(UFSHCI_VER_REG_MNR, ver); ufshci_printf(ctrlr, "UFSHCI Version: %d.%d\n", ctrlr->major_version, ctrlr->minor_version); /* Read Device Capabilities */ ctrlr->cap = cap = ufshci_mmio_read_4(ctrlr, cap); ctrlr->is_single_db_supported = UFSHCIV(UFSHCI_CAP_REG_LSDBS, cap); /* * TODO: This driver does not yet support multi-queue. * Check the UFSHCI_CAP_REG_MCQS bit in the future to determine if * multi-queue support is available. */ ctrlr->is_mcq_supported = false; if (!(ctrlr->is_single_db_supported == 0 || ctrlr->is_mcq_supported)) return (ENXIO); /* * The maximum transfer size supported by UFSHCI spec is 65535 * 256 KiB * However, we limit the maximum transfer size to 1MiB(256 * 4KiB) for * performance reason. */ ctrlr->page_size = PAGE_SIZE; ctrlr->max_xfer_size = ctrlr->page_size * UFSHCI_MAX_PRDT_ENTRY_COUNT; timeout_period = UFSHCI_DEFAULT_TIMEOUT_PERIOD; TUNABLE_INT_FETCH("hw.ufshci.timeout_period", &timeout_period); timeout_period = min(timeout_period, UFSHCI_MAX_TIMEOUT_PERIOD); timeout_period = max(timeout_period, UFSHCI_MIN_TIMEOUT_PERIOD); ctrlr->timeout_period = timeout_period; retry_count = UFSHCI_DEFAULT_RETRY_COUNT; TUNABLE_INT_FETCH("hw.ufshci.retry_count", &retry_count); ctrlr->retry_count = retry_count; - /* Disable all interrupts */ - ufshci_mmio_write_4(ctrlr, ie, 0); - - /* Enable Host Controller */ - error = ufshci_ctrlr_enable_host_ctrlr(ctrlr); - if (error) - return (error); + ctrlr->enable_aborts = 1; + if (ctrlr->quirks & UFSHCI_QUIRK_NOT_SUPPORT_ABORT_TASK) + ctrlr->enable_aborts = 0; + else + TUNABLE_INT_FETCH("hw.ufshci.enable_aborts", + &ctrlr->enable_aborts); - /* Send DME_LINKSTARTUP command to start the link startup procedure */ - error = ufshci_uic_send_dme_link_startup(ctrlr); + /* Reset the UFSHCI controller */ + error = ufshci_ctrlr_hw_reset(ctrlr); if (error) return (error); /* Read the UECPA register to clear */ ufshci_mmio_read_4(ctrlr, uecpa); /* Diable Auto-hibernate */ ahit = 0; ufshci_mmio_write_4(ctrlr, ahit, ahit); - /* - * The device_present(UFSHCI_HCS_REG_DP) bit becomes true if the host - * controller has successfully received a Link Startup UIC command - * response and the UFS device has found a physical link to the - * controller. - */ - hcs = ufshci_mmio_read_4(ctrlr, hcs); - if (!UFSHCIV(UFSHCI_HCS_REG_DP, hcs)) { - ufshci_printf(ctrlr, "UFS device not found\n"); - return (ENXIO); - } - /* Allocate and initialize UTP Task Management Request List. */ error = ufshci_utmr_req_queue_construct(ctrlr); if (error) return (error); /* Allocate and initialize UTP Transfer Request List or SQ/CQ. */ error = ufshci_utr_req_queue_construct(ctrlr); if (error) return (error); - /* Enable additional interrupts by programming the IE register. */ - ie = ufshci_mmio_read_4(ctrlr, ie); - ie |= UFSHCIM(UFSHCI_IE_REG_UTRCE); /* UTR Completion */ - ie |= UFSHCIM(UFSHCI_IE_REG_UEE); /* UIC Error */ - ie |= UFSHCIM(UFSHCI_IE_REG_UTMRCE); /* UTMR Completion */ - ie |= UFSHCIM(UFSHCI_IE_REG_DFEE); /* Device Fatal Error */ - ie |= UFSHCIM(UFSHCI_IE_REG_UTPEE); /* UTP Error */ - ie |= UFSHCIM(UFSHCI_IE_REG_HCFEE); /* Host Ctrlr Fatal Error */ - ie |= UFSHCIM(UFSHCI_IE_REG_SBFEE); /* System Bus Fatal Error */ - ie |= UFSHCIM(UFSHCI_IE_REG_CEFEE); /* Crypto Engine Fatal Error */ - ufshci_mmio_write_4(ctrlr, ie, ie); - - /* TODO: Initialize interrupt Aggregation Control Register (UTRIACR) */ - /* TODO: Separate IO and Admin slot */ + /* * max_hw_pend_io is the number of slots in the transfer_req_queue. * Reduce num_entries by one to reserve an admin slot. */ ctrlr->max_hw_pend_io = ctrlr->transfer_req_queue.num_entries - 1; + /* Create a thread for the taskqueue. */ + ctrlr->taskqueue = taskqueue_create("ufshci_taskq", M_WAITOK, + taskqueue_thread_enqueue, &ctrlr->taskqueue); + taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "ufshci taskq"); + + TASK_INIT(&ctrlr->reset_task, 0, ufshci_ctrlr_reset_task, ctrlr); + return (0); } void ufshci_ctrlr_destruct(struct ufshci_controller *ctrlr, device_t dev) { if (ctrlr->resource == NULL) goto nores; /* TODO: Flush In-flight IOs */ /* Release resources */ ufshci_utmr_req_queue_destroy(ctrlr); ufshci_utr_req_queue_destroy(ctrlr); if (ctrlr->tag) bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag); if (ctrlr->res) bus_release_resource(ctrlr->dev, SYS_RES_IRQ, rman_get_rid(ctrlr->res), ctrlr->res); mtx_lock(&ctrlr->sc_mtx); ufshci_sim_detach(ctrlr); mtx_unlock(&ctrlr->sc_mtx); bus_release_resource(dev, SYS_RES_MEMORY, ctrlr->resource_id, ctrlr->resource); nores: + KASSERT(!mtx_owned(&ctrlr->uic_cmd_lock), + ("destroying uic_cmd_lock while still owned")); mtx_destroy(&ctrlr->uic_cmd_lock); + + KASSERT(!mtx_owned(&ctrlr->sc_mtx), + ("destroying sc_mtx while still owned")); mtx_destroy(&ctrlr->sc_mtx); return; } -int +void ufshci_ctrlr_reset(struct ufshci_controller *ctrlr) { - uint32_t ie; - int error; - - /* Backup and disable all interrupts */ - ie = ufshci_mmio_read_4(ctrlr, ie); - ufshci_mmio_write_4(ctrlr, ie, 0); - - /* Release resources */ - ufshci_utmr_req_queue_destroy(ctrlr); - ufshci_utr_req_queue_destroy(ctrlr); - - /* Reset Host Controller */ - error = ufshci_ctrlr_enable_host_ctrlr(ctrlr); - if (error) - return (error); - - /* Send DME_LINKSTARTUP command to start the link startup procedure */ - error = ufshci_uic_send_dme_link_startup(ctrlr); - if (error) - return (error); - - /* Enable interrupts */ - ufshci_mmio_write_4(ctrlr, ie, ie); - - /* Allocate and initialize UTP Task Management Request List. */ - error = ufshci_utmr_req_queue_construct(ctrlr); - if (error) - return (error); - - /* Allocate and initialize UTP Transfer Request List or SQ/CQ. */ - error = ufshci_utr_req_queue_construct(ctrlr); - if (error) - return (error); - - return (0); + taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task); } int ufshci_ctrlr_submit_task_mgmt_request(struct ufshci_controller *ctrlr, struct ufshci_request *req) { return ( ufshci_req_queue_submit_request(&ctrlr->task_mgmt_req_queue, req, /*is_admin*/ false)); } int ufshci_ctrlr_submit_admin_request(struct ufshci_controller *ctrlr, struct ufshci_request *req) { return (ufshci_req_queue_submit_request(&ctrlr->transfer_req_queue, req, /*is_admin*/ true)); } int ufshci_ctrlr_submit_io_request(struct ufshci_controller *ctrlr, struct ufshci_request *req) { return (ufshci_req_queue_submit_request(&ctrlr->transfer_req_queue, req, /*is_admin*/ false)); } int ufshci_ctrlr_send_nop(struct ufshci_controller *ctrlr) { struct ufshci_completion_poll_status status; status.done = 0; ufshci_ctrlr_cmd_send_nop(ctrlr, ufshci_completion_poll_cb, &status); ufshci_completion_poll(&status); if (status.error) { ufshci_printf(ctrlr, "ufshci_ctrlr_send_nop failed!\n"); return (ENXIO); } return (0); } -static void -ufshci_ctrlr_fail(struct ufshci_controller *ctrlr, bool admin_also) -{ - printf("ufshci(4): ufshci_ctrlr_fail\n"); - - ctrlr->is_failed = true; - - /* TODO: task_mgmt_req_queue should be handled as fail */ - - ufshci_req_queue_fail(ctrlr, - &ctrlr->transfer_req_queue.hwq[UFSHCI_SDB_Q]); -} - -static void -ufshci_ctrlr_start(struct ufshci_controller *ctrlr) -{ - TSENTER(); - - if (ufshci_ctrlr_send_nop(ctrlr) != 0) { - ufshci_ctrlr_fail(ctrlr, false); - return; - } - - /* Initialize UFS target drvice */ - if (ufshci_dev_init(ctrlr) != 0) { - ufshci_ctrlr_fail(ctrlr, false); - return; - } - - /* Initialize Reference Clock */ - if (ufshci_dev_init_reference_clock(ctrlr) != 0) { - ufshci_ctrlr_fail(ctrlr, false); - return; - } - - /* Initialize unipro */ - if (ufshci_dev_init_unipro(ctrlr) != 0) { - ufshci_ctrlr_fail(ctrlr, false); - return; - } - - /* - * Initialize UIC Power Mode - * QEMU UFS devices do not support unipro and power mode. - */ - if (!(ctrlr->quirks & UFSHCI_QUIRK_IGNORE_UIC_POWER_MODE) && - ufshci_dev_init_uic_power_mode(ctrlr) != 0) { - ufshci_ctrlr_fail(ctrlr, false); - return; - } - - /* Initialize UFS Power Mode */ - if (ufshci_dev_init_ufs_power_mode(ctrlr) != 0) { - ufshci_ctrlr_fail(ctrlr, false); - return; - } - - /* Read Controller Descriptor (Device, Geometry) */ - if (ufshci_dev_get_descriptor(ctrlr) != 0) { - ufshci_ctrlr_fail(ctrlr, false); - return; - } - - if (ufshci_dev_config_write_booster(ctrlr)) { - ufshci_ctrlr_fail(ctrlr, false); - return; - } - - /* TODO: Configure Background Operations */ - - if (ufshci_sim_attach(ctrlr) != 0) { - ufshci_ctrlr_fail(ctrlr, false); - return; - } - - TSEXIT(); -} - void ufshci_ctrlr_start_config_hook(void *arg) { struct ufshci_controller *ctrlr = arg; TSENTER(); if (ufshci_utmr_req_queue_enable(ctrlr) == 0 && ufshci_utr_req_queue_enable(ctrlr) == 0) - ufshci_ctrlr_start(ctrlr); + ufshci_ctrlr_start(ctrlr, false); else - ufshci_ctrlr_fail(ctrlr, false); + ufshci_ctrlr_fail(ctrlr); ufshci_sysctl_initialize_ctrlr(ctrlr); config_intrhook_disestablish(&ctrlr->config_hook); TSEXIT(); } /* * Poll all the queues enabled on the device for completion. */ void ufshci_ctrlr_poll(struct ufshci_controller *ctrlr) { uint32_t is; is = ufshci_mmio_read_4(ctrlr, is); /* UIC error */ if (is & UFSHCIM(UFSHCI_IS_REG_UE)) { uint32_t uecpa, uecdl, uecn, uect, uecdme; /* UECPA for Host UIC Error Code within PHY Adapter Layer */ uecpa = ufshci_mmio_read_4(ctrlr, uecpa); if (uecpa & UFSHCIM(UFSHCI_UECPA_REG_ERR)) { ufshci_printf(ctrlr, "UECPA error code: 0x%x\n", UFSHCIV(UFSHCI_UECPA_REG_EC, uecpa)); } /* UECDL for Host UIC Error Code within Data Link Layer */ uecdl = ufshci_mmio_read_4(ctrlr, uecdl); if (uecdl & UFSHCIM(UFSHCI_UECDL_REG_ERR)) { ufshci_printf(ctrlr, "UECDL error code: 0x%x\n", UFSHCIV(UFSHCI_UECDL_REG_EC, uecdl)); } /* UECN for Host UIC Error Code within Network Layer */ uecn = ufshci_mmio_read_4(ctrlr, uecn); if (uecn & UFSHCIM(UFSHCI_UECN_REG_ERR)) { ufshci_printf(ctrlr, "UECN error code: 0x%x\n", UFSHCIV(UFSHCI_UECN_REG_EC, uecn)); } /* UECT for Host UIC Error Code within Transport Layer */ uect = ufshci_mmio_read_4(ctrlr, uect); if (uect & UFSHCIM(UFSHCI_UECT_REG_ERR)) { ufshci_printf(ctrlr, "UECT error code: 0x%x\n", UFSHCIV(UFSHCI_UECT_REG_EC, uect)); } /* UECDME for Host UIC Error Code within DME subcomponent */ uecdme = ufshci_mmio_read_4(ctrlr, uecdme); if (uecdme & UFSHCIM(UFSHCI_UECDME_REG_ERR)) { ufshci_printf(ctrlr, "UECDME error code: 0x%x\n", UFSHCIV(UFSHCI_UECDME_REG_EC, uecdme)); } ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UE)); } /* Device Fatal Error Status */ if (is & UFSHCIM(UFSHCI_IS_REG_DFES)) { ufshci_printf(ctrlr, "Device fatal error on ISR\n"); ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_DFES)); } /* UTP Error Status */ if (is & UFSHCIM(UFSHCI_IS_REG_UTPES)) { ufshci_printf(ctrlr, "UTP error on ISR\n"); ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTPES)); } /* Host Controller Fatal Error Status */ if (is & UFSHCIM(UFSHCI_IS_REG_HCFES)) { ufshci_printf(ctrlr, "Host controller fatal error on ISR\n"); ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_HCFES)); } /* System Bus Fatal Error Status */ if (is & UFSHCIM(UFSHCI_IS_REG_SBFES)) { ufshci_printf(ctrlr, "System bus fatal error on ISR\n"); ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_SBFES)); } /* Crypto Engine Fatal Error Status */ if (is & UFSHCIM(UFSHCI_IS_REG_CEFES)) { ufshci_printf(ctrlr, "Crypto engine fatal error on ISR\n"); ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_CEFES)); } /* UTP Task Management Request Completion Status */ if (is & UFSHCIM(UFSHCI_IS_REG_UTMRCS)) { ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTMRCS)); ufshci_req_queue_process_completions( &ctrlr->task_mgmt_req_queue); } /* UTP Transfer Request Completion Status */ if (is & UFSHCIM(UFSHCI_IS_REG_UTRCS)) { ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UTRCS)); ufshci_req_queue_process_completions( &ctrlr->transfer_req_queue); } /* MCQ CQ Event Status */ if (is & UFSHCIM(UFSHCI_IS_REG_CQES)) { /* TODO: We need to process completion Queue Pairs */ ufshci_printf(ctrlr, "MCQ completion not yet implemented\n"); ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_CQES)); } } /* * Poll the single-vector interrupt case: num_io_queues will be 1 and * there's only a single vector. While we're polling, we mask further * interrupts in the controller. */ void ufshci_ctrlr_shared_handler(void *arg) { struct ufshci_controller *ctrlr = arg; ufshci_ctrlr_poll(ctrlr); } void ufshci_reg_dump(struct ufshci_controller *ctrlr) { ufshci_printf(ctrlr, "========= UFSHCI Register Dump =========\n"); UFSHCI_DUMP_REG(ctrlr, cap); UFSHCI_DUMP_REG(ctrlr, mcqcap); UFSHCI_DUMP_REG(ctrlr, ver); UFSHCI_DUMP_REG(ctrlr, ext_cap); UFSHCI_DUMP_REG(ctrlr, hcpid); UFSHCI_DUMP_REG(ctrlr, hcmid); UFSHCI_DUMP_REG(ctrlr, ahit); UFSHCI_DUMP_REG(ctrlr, is); UFSHCI_DUMP_REG(ctrlr, ie); UFSHCI_DUMP_REG(ctrlr, hcsext); UFSHCI_DUMP_REG(ctrlr, hcs); UFSHCI_DUMP_REG(ctrlr, hce); UFSHCI_DUMP_REG(ctrlr, uecpa); UFSHCI_DUMP_REG(ctrlr, uecdl); UFSHCI_DUMP_REG(ctrlr, uecn); UFSHCI_DUMP_REG(ctrlr, uect); UFSHCI_DUMP_REG(ctrlr, uecdme); ufshci_printf(ctrlr, "========================================\n"); } diff --git a/sys/dev/ufshci/ufshci_ctrlr_cmd.c b/sys/dev/ufshci/ufshci_ctrlr_cmd.c index 71d163d998af..253f31a93c2e 100644 --- a/sys/dev/ufshci/ufshci_ctrlr_cmd.c +++ b/sys/dev/ufshci/ufshci_ctrlr_cmd.c @@ -1,79 +1,79 @@ /*- * Copyright (c) 2025, Samsung Electronics Co., Ltd. * Written by Jaeyoon Choi * * SPDX-License-Identifier: BSD-2-Clause */ #include "ufshci_private.h" void ufshci_ctrlr_cmd_send_task_mgmt_request(struct ufshci_controller *ctrlr, ufshci_cb_fn_t cb_fn, void *cb_arg, uint8_t function, uint8_t lun, uint8_t task_tag, uint8_t iid) { struct ufshci_request *req; struct ufshci_task_mgmt_request_upiu *upiu; - req = ufshci_allocate_request_vaddr(NULL, 0, M_WAITOK, cb_fn, cb_arg); + req = ufshci_allocate_request_vaddr(NULL, 0, M_NOWAIT, cb_fn, cb_arg); req->request_size = sizeof(struct ufshci_task_mgmt_request_upiu); req->response_size = sizeof(struct ufshci_task_mgmt_response_upiu); upiu = (struct ufshci_task_mgmt_request_upiu *)&req->request_upiu; memset(upiu, 0, req->request_size); upiu->header.trans_type = UFSHCI_UPIU_TRANSACTION_CODE_TASK_MANAGEMENT_REQUEST; upiu->header.lun = lun; upiu->header.ext_iid_or_function = function; upiu->input_param1 = lun; upiu->input_param2 = task_tag; upiu->input_param3 = iid; ufshci_ctrlr_submit_task_mgmt_request(ctrlr, req); } void ufshci_ctrlr_cmd_send_nop(struct ufshci_controller *ctrlr, ufshci_cb_fn_t cb_fn, void *cb_arg) { struct ufshci_request *req; struct ufshci_nop_out_upiu *upiu; req = ufshci_allocate_request_vaddr(NULL, 0, M_WAITOK, cb_fn, cb_arg); req->request_size = sizeof(struct ufshci_nop_out_upiu); req->response_size = sizeof(struct ufshci_nop_in_upiu); upiu = (struct ufshci_nop_out_upiu *)&req->request_upiu; memset(upiu, 0, req->request_size); upiu->header.trans_type = UFSHCI_UPIU_TRANSACTION_CODE_NOP_OUT; ufshci_ctrlr_submit_admin_request(ctrlr, req); } void ufshci_ctrlr_cmd_send_query_request(struct ufshci_controller *ctrlr, ufshci_cb_fn_t cb_fn, void *cb_arg, struct ufshci_query_param param) { struct ufshci_request *req; struct ufshci_query_request_upiu *upiu; req = ufshci_allocate_request_vaddr(NULL, 0, M_WAITOK, cb_fn, cb_arg); req->request_size = sizeof(struct ufshci_query_request_upiu); req->response_size = sizeof(struct ufshci_query_response_upiu); upiu = (struct ufshci_query_request_upiu *)&req->request_upiu; memset(upiu, 0, req->request_size); upiu->header.trans_type = UFSHCI_UPIU_TRANSACTION_CODE_QUERY_REQUEST; upiu->header.ext_iid_or_function = param.function; upiu->opcode = param.opcode; upiu->idn = param.type; upiu->index = param.index; upiu->selector = param.selector; upiu->value_64 = param.value; upiu->length = param.desc_size; ufshci_ctrlr_submit_admin_request(ctrlr, req); } diff --git a/sys/dev/ufshci/ufshci_dev.c b/sys/dev/ufshci/ufshci_dev.c index dd196b1d638b..975468e5156f 100644 --- a/sys/dev/ufshci/ufshci_dev.c +++ b/sys/dev/ufshci/ufshci_dev.c @@ -1,777 +1,776 @@ /*- * Copyright (c) 2025, Samsung Electronics Co., Ltd. * Written by Jaeyoon Choi * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include "ufshci_private.h" #include "ufshci_reg.h" static int ufshci_dev_read_descriptor(struct ufshci_controller *ctrlr, enum ufshci_descriptor_type desc_type, uint8_t index, uint8_t selector, void *desc, size_t desc_size) { struct ufshci_completion_poll_status status; struct ufshci_query_param param; param.function = UFSHCI_QUERY_FUNC_STANDARD_READ_REQUEST; param.opcode = UFSHCI_QUERY_OPCODE_READ_DESCRIPTOR; param.type = desc_type; param.index = index; param.selector = selector; param.value = 0; param.desc_size = desc_size; status.done = 0; ufshci_ctrlr_cmd_send_query_request(ctrlr, ufshci_completion_poll_cb, &status, param); ufshci_completion_poll(&status); if (status.error) { ufshci_printf(ctrlr, "ufshci_dev_read_descriptor failed!\n"); return (ENXIO); } memcpy(desc, status.cpl.response_upiu.query_response_upiu.command_data, desc_size); return (0); } static int ufshci_dev_read_device_descriptor(struct ufshci_controller *ctrlr, struct ufshci_device_descriptor *desc) { return (ufshci_dev_read_descriptor(ctrlr, UFSHCI_DESC_TYPE_DEVICE, 0, 0, desc, sizeof(struct ufshci_device_descriptor))); } static int ufshci_dev_read_geometry_descriptor(struct ufshci_controller *ctrlr, struct ufshci_geometry_descriptor *desc) { return (ufshci_dev_read_descriptor(ctrlr, UFSHCI_DESC_TYPE_GEOMETRY, 0, 0, desc, sizeof(struct ufshci_geometry_descriptor))); } static int ufshci_dev_read_unit_descriptor(struct ufshci_controller *ctrlr, uint8_t lun, struct ufshci_unit_descriptor *desc) { return (ufshci_dev_read_descriptor(ctrlr, UFSHCI_DESC_TYPE_UNIT, lun, 0, desc, sizeof(struct ufshci_unit_descriptor))); } static int ufshci_dev_read_flag(struct ufshci_controller *ctrlr, enum ufshci_flags flag_type, uint8_t *flag) { struct ufshci_completion_poll_status status; struct ufshci_query_param param; param.function = UFSHCI_QUERY_FUNC_STANDARD_READ_REQUEST; param.opcode = UFSHCI_QUERY_OPCODE_READ_FLAG; param.type = flag_type; param.index = 0; param.selector = 0; param.value = 0; status.done = 0; ufshci_ctrlr_cmd_send_query_request(ctrlr, ufshci_completion_poll_cb, &status, param); ufshci_completion_poll(&status); if (status.error) { ufshci_printf(ctrlr, "ufshci_dev_read_flag failed!\n"); return (ENXIO); } *flag = status.cpl.response_upiu.query_response_upiu.flag_value; return (0); } static int ufshci_dev_set_flag(struct ufshci_controller *ctrlr, enum ufshci_flags flag_type) { struct ufshci_completion_poll_status status; struct ufshci_query_param param; param.function = UFSHCI_QUERY_FUNC_STANDARD_WRITE_REQUEST; param.opcode = UFSHCI_QUERY_OPCODE_SET_FLAG; param.type = flag_type; param.index = 0; param.selector = 0; param.value = 0; status.done = 0; ufshci_ctrlr_cmd_send_query_request(ctrlr, ufshci_completion_poll_cb, &status, param); ufshci_completion_poll(&status); if (status.error) { ufshci_printf(ctrlr, "ufshci_dev_set_flag failed!\n"); return (ENXIO); } return (0); } static int ufshci_dev_clear_flag(struct ufshci_controller *ctrlr, enum ufshci_flags flag_type) { struct ufshci_completion_poll_status status; struct ufshci_query_param param; param.function = UFSHCI_QUERY_FUNC_STANDARD_WRITE_REQUEST; param.opcode = UFSHCI_QUERY_OPCODE_CLEAR_FLAG; param.type = flag_type; param.index = 0; param.selector = 0; param.value = 0; status.done = 0; ufshci_ctrlr_cmd_send_query_request(ctrlr, ufshci_completion_poll_cb, &status, param); ufshci_completion_poll(&status); if (status.error) { ufshci_printf(ctrlr, "ufshci_dev_clear_flag failed!\n"); return (ENXIO); } return (0); } static int ufshci_dev_read_attribute(struct ufshci_controller *ctrlr, enum ufshci_attributes attr_type, uint8_t index, uint8_t selector, uint64_t *value) { struct ufshci_completion_poll_status status; struct ufshci_query_param param; param.function = UFSHCI_QUERY_FUNC_STANDARD_READ_REQUEST; param.opcode = UFSHCI_QUERY_OPCODE_READ_ATTRIBUTE; param.type = attr_type; param.index = index; param.selector = selector; param.value = 0; status.done = 0; ufshci_ctrlr_cmd_send_query_request(ctrlr, ufshci_completion_poll_cb, &status, param); ufshci_completion_poll(&status); if (status.error) { ufshci_printf(ctrlr, "ufshci_dev_read_attribute failed!\n"); return (ENXIO); } *value = status.cpl.response_upiu.query_response_upiu.value_64; return (0); } static int ufshci_dev_write_attribute(struct ufshci_controller *ctrlr, enum ufshci_attributes attr_type, uint8_t index, uint8_t selector, uint64_t value) { struct ufshci_completion_poll_status status; struct ufshci_query_param param; param.function = UFSHCI_QUERY_FUNC_STANDARD_WRITE_REQUEST; param.opcode = UFSHCI_QUERY_OPCODE_WRITE_ATTRIBUTE; param.type = attr_type; param.index = index; param.selector = selector; param.value = value; status.done = 0; ufshci_ctrlr_cmd_send_query_request(ctrlr, ufshci_completion_poll_cb, &status, param); ufshci_completion_poll(&status); if (status.error) { ufshci_printf(ctrlr, "ufshci_dev_write_attribute failed!\n"); return (ENXIO); } return (0); } int ufshci_dev_init(struct ufshci_controller *ctrlr) { int timeout = ticks + MSEC_2_TICKS(ctrlr->device_init_timeout_in_ms); sbintime_t delta_t = SBT_1US; uint8_t flag; int error; const uint8_t device_init_completed = 0; error = ufshci_dev_set_flag(ctrlr, UFSHCI_FLAG_F_DEVICE_INIT); if (error) return (error); /* Wait for the UFSHCI_FLAG_F_DEVICE_INIT flag to change */ while (1) { error = ufshci_dev_read_flag(ctrlr, UFSHCI_FLAG_F_DEVICE_INIT, &flag); if (error) return (error); if (flag == device_init_completed) break; if (timeout - ticks < 0) { ufshci_printf(ctrlr, "device init did not become %d " "within %d ms\n", device_init_completed, ctrlr->device_init_timeout_in_ms); return (ENXIO); } pause_sbt("ufshciinit", delta_t, 0, C_PREL(1)); delta_t = min(SBT_1MS, delta_t * 3 / 2); } return (0); } int ufshci_dev_reset(struct ufshci_controller *ctrlr) { if (ufshci_uic_send_dme_endpoint_reset(ctrlr)) return (ENXIO); return (ufshci_dev_init(ctrlr)); } int ufshci_dev_init_reference_clock(struct ufshci_controller *ctrlr) { int error; uint8_t index, selector; index = 0; /* bRefClkFreq is device type attribute */ selector = 0; /* bRefClkFreq is device type attribute */ error = ufshci_dev_write_attribute(ctrlr, UFSHCI_ATTR_B_REF_CLK_FREQ, index, selector, ctrlr->ref_clk); if (error) return (error); return (0); } int ufshci_dev_init_unipro(struct ufshci_controller *ctrlr) { uint32_t pa_granularity, peer_pa_granularity; uint32_t t_activate, pear_t_activate; /* * Unipro Version: * - 7~15 = Above 2.0, 6 = 2.0, 5 = 1.8, 4 = 1.61, 3 = 1.6, 2 = 1.41, * 1 = 1.40, 0 = Reserved */ if (ufshci_uic_send_dme_get(ctrlr, PA_LocalVerInfo, &ctrlr->unipro_version)) return (ENXIO); if (ufshci_uic_send_dme_get(ctrlr, PA_RemoteVerInfo, &ctrlr->ufs_dev.unipro_version)) return (ENXIO); /* * PA_Granularity: Granularity for PA_TActivate and PA_Hibern8Time * - 1=1us, 2=4us, 3=8us, 4=16us, 5=32us, 6=100us */ if (ufshci_uic_send_dme_get(ctrlr, PA_Granularity, &pa_granularity)) return (ENXIO); if (ufshci_uic_send_dme_peer_get(ctrlr, PA_Granularity, &peer_pa_granularity)) return (ENXIO); /* * PA_TActivate: Time to wait before activating a burst in order to * wake-up peer M-RX * UniPro automatically sets timing information such as PA_TActivate * through the PACP_CAP_EXT1_ind command during Link Startup operation. */ if (ufshci_uic_send_dme_get(ctrlr, PA_TActivate, &t_activate)) return (ENXIO); if (ufshci_uic_send_dme_peer_get(ctrlr, PA_TActivate, &pear_t_activate)) return (ENXIO); if (ctrlr->quirks & UFSHCI_QUIRK_LONG_PEER_PA_TACTIVATE) { /* * Intel Lake-field UFSHCI has a quirk. We need to add 200us to * the PEER's PA_TActivate. */ if (pa_granularity == peer_pa_granularity) { pear_t_activate = t_activate + 2; if (ufshci_uic_send_dme_peer_set(ctrlr, PA_TActivate, pear_t_activate)) return (ENXIO); } } return (0); } int ufshci_dev_init_uic_power_mode(struct ufshci_controller *ctrlr) { /* HSSerise: A = 1, B = 2 */ const uint32_t hs_series = 2; /* * TX/RX PWRMode: * - TX[3:0], RX[7:4] * - Fast Mode = 1, Slow Mode = 2, FastAuto Mode = 4, SlowAuto Mode = 5 */ const uint32_t fast_mode = 1; const uint32_t rx_bit_shift = 4; uint32_t power_mode, peer_granularity; /* Update lanes with available TX/RX lanes */ if (ufshci_uic_send_dme_get(ctrlr, PA_AvailTxDataLanes, &ctrlr->max_tx_lanes)) return (ENXIO); if (ufshci_uic_send_dme_get(ctrlr, PA_AvailRxDataLanes, &ctrlr->max_rx_lanes)) return (ENXIO); /* Get max HS-GEAR value */ if (ufshci_uic_send_dme_get(ctrlr, PA_MaxRxHSGear, &ctrlr->max_rx_hs_gear)) return (ENXIO); /* Set the data lane to max */ ctrlr->tx_lanes = ctrlr->max_tx_lanes; ctrlr->rx_lanes = ctrlr->max_rx_lanes; if (ufshci_uic_send_dme_set(ctrlr, PA_ActiveTxDataLanes, ctrlr->tx_lanes)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, PA_ActiveRxDataLanes, ctrlr->rx_lanes)) return (ENXIO); if (ctrlr->quirks & UFSHCI_QUIRK_CHANGE_LANE_AND_GEAR_SEPARATELY) { /* Before changing gears, first change the number of lanes. */ if (ufshci_uic_send_dme_get(ctrlr, PA_PWRMode, &power_mode)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, PA_PWRMode, power_mode)) return (ENXIO); /* Wait for power mode changed. */ if (ufshci_uic_power_mode_ready(ctrlr)) { ufshci_reg_dump(ctrlr); return (ENXIO); } } /* Set HS-GEAR to max gear */ ctrlr->hs_gear = ctrlr->max_rx_hs_gear; if (ufshci_uic_send_dme_set(ctrlr, PA_TxGear, ctrlr->hs_gear)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, PA_RxGear, ctrlr->hs_gear)) return (ENXIO); /* * Set termination * - HS-MODE = ON / LS-MODE = OFF */ if (ufshci_uic_send_dme_set(ctrlr, PA_TxTermination, true)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, PA_RxTermination, true)) return (ENXIO); /* Set HSSerise (A = 1, B = 2) */ if (ufshci_uic_send_dme_set(ctrlr, PA_HSSeries, hs_series)) return (ENXIO); /* Set Timeout values */ if (ufshci_uic_send_dme_set(ctrlr, PA_PWRModeUserData0, DL_FC0ProtectionTimeOutVal_Default)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, PA_PWRModeUserData1, DL_TC0ReplayTimeOutVal_Default)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, PA_PWRModeUserData2, DL_AFC0ReqTimeOutVal_Default)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, PA_PWRModeUserData3, DL_FC0ProtectionTimeOutVal_Default)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, PA_PWRModeUserData4, DL_TC0ReplayTimeOutVal_Default)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, PA_PWRModeUserData5, DL_AFC0ReqTimeOutVal_Default)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, DME_LocalFC0ProtectionTimeOutVal, DL_FC0ProtectionTimeOutVal_Default)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, DME_LocalTC0ReplayTimeOutVal, DL_TC0ReplayTimeOutVal_Default)) return (ENXIO); if (ufshci_uic_send_dme_set(ctrlr, DME_LocalAFC0ReqTimeOutVal, DL_AFC0ReqTimeOutVal_Default)) return (ENXIO); /* Set TX/RX PWRMode */ power_mode = (fast_mode << rx_bit_shift) | fast_mode; if (ufshci_uic_send_dme_set(ctrlr, PA_PWRMode, power_mode)) return (ENXIO); /* Wait for power mode changed. */ if (ufshci_uic_power_mode_ready(ctrlr)) { ufshci_reg_dump(ctrlr); return (ENXIO); } /* Clear 'Power Mode completion status' */ ufshci_mmio_write_4(ctrlr, is, UFSHCIM(UFSHCI_IS_REG_UPMS)); if (ctrlr->quirks & UFSHCI_QUIRK_WAIT_AFTER_POWER_MODE_CHANGE) { /* * Intel Lake-field UFSHCI has a quirk. * We need to wait 1250us and clear dme error. */ pause_sbt("ufshci", ustosbt(1250), 0, C_PREL(1)); /* Test with dme_peer_get to make sure there are no errors. */ if (ufshci_uic_send_dme_peer_get(ctrlr, PA_Granularity, &peer_granularity)) return (ENXIO); } return (0); } int ufshci_dev_init_ufs_power_mode(struct ufshci_controller *ctrlr) { /* TODO: Need to implement */ return (0); } int ufshci_dev_get_descriptor(struct ufshci_controller *ctrlr) { struct ufshci_device *device = &ctrlr->ufs_dev; /* * The kDeviceDensityUnit is defined in the spec as 512. * qTotalRawDeviceCapacity use big-endian byte ordering. */ const uint32_t device_density_unit = 512; uint32_t ver; int error; error = ufshci_dev_read_device_descriptor(ctrlr, &device->dev_desc); if (error) return (error); ver = be16toh(device->dev_desc.wSpecVersion); ufshci_printf(ctrlr, "UFS device spec version %u.%u.%u\n", UFSHCIV(UFSHCI_VER_REG_MJR, ver), UFSHCIV(UFSHCI_VER_REG_MNR, ver), UFSHCIV(UFSHCI_VER_REG_VS, ver)); ufshci_printf(ctrlr, "%u enabled LUNs found\n", device->dev_desc.bNumberLU); error = ufshci_dev_read_geometry_descriptor(ctrlr, &device->geo_desc); if (error) return (error); if (device->geo_desc.bMaxNumberLU == 0) { device->max_lun_count = 8; } else if (device->geo_desc.bMaxNumberLU == 1) { device->max_lun_count = 32; } else { ufshci_printf(ctrlr, "Invalid Geometry Descriptor bMaxNumberLU value=%d\n", device->geo_desc.bMaxNumberLU); return (ENXIO); } ctrlr->max_lun_count = device->max_lun_count; ufshci_printf(ctrlr, "UFS device total size is %lu bytes\n", be64toh(device->geo_desc.qTotalRawDeviceCapacity) * device_density_unit); return (0); } static int ufshci_dev_enable_write_booster(struct ufshci_controller *ctrlr) { struct ufshci_device *dev = &ctrlr->ufs_dev; int error; /* Enable WriteBooster */ error = ufshci_dev_set_flag(ctrlr, UFSHCI_FLAG_F_WRITE_BOOSTER_EN); if (error) { ufshci_printf(ctrlr, "Failed to enable WriteBooster\n"); return (error); } dev->is_wb_enabled = true; /* Enable WriteBooster buffer flush during hibernate */ error = ufshci_dev_set_flag(ctrlr, UFSHCI_FLAG_F_WB_BUFFER_FLUSH_DURING_HIBERNATE); if (error) { ufshci_printf(ctrlr, "Failed to enable WriteBooster buffer flush during hibernate\n"); return (error); } /* Enable WriteBooster buffer flush */ error = ufshci_dev_set_flag(ctrlr, UFSHCI_FLAG_F_WB_BUFFER_FLUSH_EN); if (error) { ufshci_printf(ctrlr, "Failed to enable WriteBooster buffer flush\n"); return (error); } dev->is_wb_flush_enabled = true; return (0); } static int ufshci_dev_disable_write_booster(struct ufshci_controller *ctrlr) { struct ufshci_device *dev = &ctrlr->ufs_dev; int error; /* Disable WriteBooster buffer flush */ error = ufshci_dev_clear_flag(ctrlr, UFSHCI_FLAG_F_WB_BUFFER_FLUSH_EN); if (error) { ufshci_printf(ctrlr, "Failed to disable WriteBooster buffer flush\n"); return (error); } dev->is_wb_flush_enabled = false; /* Disable WriteBooster buffer flush during hibernate */ error = ufshci_dev_clear_flag(ctrlr, UFSHCI_FLAG_F_WB_BUFFER_FLUSH_DURING_HIBERNATE); if (error) { ufshci_printf(ctrlr, "Failed to disable WriteBooster buffer flush during hibernate\n"); return (error); } /* Disable WriteBooster */ error = ufshci_dev_clear_flag(ctrlr, UFSHCI_FLAG_F_WRITE_BOOSTER_EN); if (error) { ufshci_printf(ctrlr, "Failed to disable WriteBooster\n"); return (error); } dev->is_wb_enabled = false; return (0); } static int ufshci_dev_is_write_booster_buffer_life_time_left( struct ufshci_controller *ctrlr, bool *is_life_time_left) { struct ufshci_device *dev = &ctrlr->ufs_dev; uint8_t buffer_lun; uint64_t life_time; uint32_t error; if (dev->wb_buffer_type == UFSHCI_DESC_WB_BUF_TYPE_LU_DEDICATED) buffer_lun = dev->wb_dedicated_lu; else buffer_lun = 0; error = ufshci_dev_read_attribute(ctrlr, UFSHCI_ATTR_B_WB_BUFFER_LIFE_TIME_EST, buffer_lun, 0, &life_time); if (error) return (error); *is_life_time_left = (life_time != UFSHCI_ATTR_WB_LIFE_EXCEEDED); return (0); } /* * This function is not yet in use. It will be used when suspend/resume is * implemented. */ static __unused int ufshci_dev_need_write_booster_buffer_flush(struct ufshci_controller *ctrlr, bool *need_flush) { struct ufshci_device *dev = &ctrlr->ufs_dev; bool is_life_time_left = false; uint64_t available_buffer_size, current_buffer_size; uint8_t buffer_lun; uint32_t error; *need_flush = false; if (!dev->is_wb_enabled) return (0); error = ufshci_dev_is_write_booster_buffer_life_time_left(ctrlr, &is_life_time_left); if (error) return (error); if (!is_life_time_left) return (ufshci_dev_disable_write_booster(ctrlr)); if (dev->wb_buffer_type == UFSHCI_DESC_WB_BUF_TYPE_LU_DEDICATED) buffer_lun = dev->wb_dedicated_lu; else buffer_lun = 0; error = ufshci_dev_read_attribute(ctrlr, UFSHCI_ATTR_B_AVAILABLE_WB_BUFFER_SIZE, buffer_lun, 0, &available_buffer_size); if (error) return (error); switch (dev->wb_user_space_config_option) { case UFSHCI_DESC_WB_BUF_USER_SPACE_REDUCTION: *need_flush = (available_buffer_size <= UFSHCI_ATTR_WB_AVAILABLE_10); break; case UFSHCI_DESC_WB_BUF_PRESERVE_USER_SPACE: /* * In PRESERVE USER SPACE mode, flush should be performed when * the current buffer is greater than 0 and the available buffer * below write_booster_flush_threshold is left. */ error = ufshci_dev_read_attribute(ctrlr, UFSHCI_ATTR_D_CURRENT_WB_BUFFER_SIZE, buffer_lun, 0, ¤t_buffer_size); if (error) return (error); if (current_buffer_size == 0) return (0); *need_flush = (available_buffer_size < dev->write_booster_flush_threshold); break; default: ufshci_printf(ctrlr, "Invalid bWriteBoosterBufferPreserveUserSpaceEn value"); return (EINVAL); } /* * TODO: Need to handle WRITEBOOSTER_FLUSH_NEEDED exception case from * wExceptionEventStatus attribute. */ return (0); } int ufshci_dev_config_write_booster(struct ufshci_controller *ctrlr) { struct ufshci_device *dev = &ctrlr->ufs_dev; uint32_t extended_ufs_feature_support; uint32_t alloc_units; struct ufshci_unit_descriptor unit_desc; uint8_t lun; bool is_life_time_left; uint32_t mega_byte = 1024 * 1024; uint32_t error = 0; extended_ufs_feature_support = be32toh( dev->dev_desc.dExtendedUfsFeaturesSupport); if (!(extended_ufs_feature_support & UFSHCI_DESC_EXT_UFS_FEATURE_WRITE_BOOSTER)) { /* This device does not support Write Booster */ return (0); } if (ufshci_dev_enable_write_booster(ctrlr)) return (0); /* Get WriteBooster buffer parameters */ dev->wb_buffer_type = dev->dev_desc.bWriteBoosterBufferType; dev->wb_user_space_config_option = dev->dev_desc.bWriteBoosterBufferPreserveUserSpaceEn; /* * Find the size of the write buffer. * With LU-dedicated (00h), the WriteBooster buffer is assigned * exclusively to one chosen LU (not one-per-LU), whereas Shared (01h) * uses a single device-wide buffer shared by multiple LUs. */ if (dev->wb_buffer_type == UFSHCI_DESC_WB_BUF_TYPE_SINGLE_SHARED) { alloc_units = be32toh( dev->dev_desc.dNumSharedWriteBoosterBufferAllocUnits); ufshci_printf(ctrlr, "WriteBooster buffer type = Shared, alloc_units=%d\n", alloc_units); } else if (dev->wb_buffer_type == UFSHCI_DESC_WB_BUF_TYPE_LU_DEDICATED) { ufshci_printf(ctrlr, "WriteBooster buffer type = Dedicated\n"); for (lun = 0; lun < ctrlr->max_lun_count; lun++) { /* Find a dedicated buffer using a unit descriptor */ if (ufshci_dev_read_unit_descriptor(ctrlr, lun, &unit_desc)) continue; alloc_units = be32toh( unit_desc.dLUNumWriteBoosterBufferAllocUnits); if (alloc_units) { dev->wb_dedicated_lu = lun; break; } } } else { ufshci_printf(ctrlr, "Not supported WriteBooster buffer type: 0x%x\n", dev->wb_buffer_type); goto out; } if (alloc_units == 0) { ufshci_printf(ctrlr, "The WriteBooster buffer size is zero\n"); goto out; } dev->wb_buffer_size_mb = alloc_units * dev->geo_desc.bAllocationUnitSize * (be32toh(dev->geo_desc.dSegmentSize)) / (mega_byte / UFSHCI_SECTOR_SIZE); /* Set to flush when 40% of the available buffer size remains */ dev->write_booster_flush_threshold = UFSHCI_ATTR_WB_AVAILABLE_40; /* * Check if WriteBooster Buffer lifetime is left. * WriteBooster Buffer lifetime — percent of life used based on P/E * cycles. If "preserve user space" is enabled, writes to normal user * space also consume WB life since the area is shared. */ error = ufshci_dev_is_write_booster_buffer_life_time_left(ctrlr, &is_life_time_left); if (error) goto out; if (!is_life_time_left) { ufshci_printf(ctrlr, "There is no WriteBooster buffer life time left.\n"); goto out; } ufshci_printf(ctrlr, "WriteBooster Enabled\n"); return (0); out: ufshci_dev_disable_write_booster(ctrlr); return (error); } - diff --git a/sys/dev/ufshci/ufshci_pci.c b/sys/dev/ufshci/ufshci_pci.c index d64b7526f713..992026fd4f4d 100644 --- a/sys/dev/ufshci/ufshci_pci.c +++ b/sys/dev/ufshci/ufshci_pci.c @@ -1,261 +1,262 @@ /*- * Copyright (c) 2025, Samsung Electronics Co., Ltd. * Written by Jaeyoon Choi * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include #include #include #include #include #include #include #include "ufshci_private.h" static int ufshci_pci_probe(device_t); static int ufshci_pci_attach(device_t); static int ufshci_pci_detach(device_t); static int ufshci_pci_setup_interrupts(struct ufshci_controller *ctrlr); static device_method_t ufshci_pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ufshci_pci_probe), DEVMETHOD(device_attach, ufshci_pci_attach), DEVMETHOD(device_detach, ufshci_pci_detach), /* TODO: Implement Suspend, Resume */ { 0, 0 } }; static driver_t ufshci_pci_driver = { "ufshci", ufshci_pci_methods, sizeof(struct ufshci_controller), }; DRIVER_MODULE(ufshci, pci, ufshci_pci_driver, 0, 0); static struct _pcsid { uint32_t devid; const char *desc; uint32_t ref_clk; uint32_t quirks; } pci_ids[] = { { 0x131b36, "QEMU UFS Host Controller", UFSHCI_REF_CLK_19_2MHz, - UFSHCI_QUIRK_IGNORE_UIC_POWER_MODE }, + UFSHCI_QUIRK_IGNORE_UIC_POWER_MODE | + UFSHCI_QUIRK_NOT_SUPPORT_ABORT_TASK }, { 0x98fa8086, "Intel Lakefield UFS Host Controller", UFSHCI_REF_CLK_19_2MHz, UFSHCI_QUIRK_LONG_PEER_PA_TACTIVATE | UFSHCI_QUIRK_WAIT_AFTER_POWER_MODE_CHANGE | UFSHCI_QUIRK_CHANGE_LANE_AND_GEAR_SEPARATELY }, { 0x54ff8086, "Intel UFS Host Controller", UFSHCI_REF_CLK_19_2MHz }, { 0x00000000, NULL } }; static int ufshci_pci_probe(device_t device) { struct ufshci_controller *ctrlr = device_get_softc(device); uint32_t devid = pci_get_devid(device); struct _pcsid *ep = pci_ids; while (ep->devid && ep->devid != devid) ++ep; if (ep->devid) { ctrlr->quirks = ep->quirks; ctrlr->ref_clk = ep->ref_clk; } if (ep->desc) { device_set_desc(device, ep->desc); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int ufshci_pci_allocate_bar(struct ufshci_controller *ctrlr) { ctrlr->resource_id = PCIR_BAR(0); ctrlr->resource = bus_alloc_resource_any(ctrlr->dev, SYS_RES_MEMORY, &ctrlr->resource_id, RF_ACTIVE); if (ctrlr->resource == NULL) { ufshci_printf(ctrlr, "unable to allocate pci resource\n"); return (ENOMEM); } ctrlr->bus_tag = rman_get_bustag(ctrlr->resource); ctrlr->bus_handle = rman_get_bushandle(ctrlr->resource); ctrlr->regs = (struct ufshci_registers *)ctrlr->bus_handle; return (0); } static int ufshci_pci_attach(device_t dev) { struct ufshci_controller *ctrlr = device_get_softc(dev); int status; ctrlr->dev = dev; status = ufshci_pci_allocate_bar(ctrlr); if (status != 0) goto bad; pci_enable_busmaster(dev); status = ufshci_pci_setup_interrupts(ctrlr); if (status != 0) goto bad; return (ufshci_attach(dev)); bad: if (ctrlr->resource != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, ctrlr->resource_id, ctrlr->resource); } if (ctrlr->tag) bus_teardown_intr(dev, ctrlr->res, ctrlr->tag); if (ctrlr->res) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(ctrlr->res), ctrlr->res); if (ctrlr->msi_count > 0) pci_release_msi(dev); return (status); } static int ufshci_pci_detach(device_t dev) { struct ufshci_controller *ctrlr = device_get_softc(dev); int error; error = ufshci_detach(dev); if (ctrlr->msi_count > 0) pci_release_msi(dev); pci_disable_busmaster(dev); return (error); } static int ufshci_pci_setup_shared(struct ufshci_controller *ctrlr, int rid) { int error; ctrlr->num_io_queues = 1; ctrlr->rid = rid; ctrlr->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, &ctrlr->rid, RF_SHAREABLE | RF_ACTIVE); if (ctrlr->res == NULL) { ufshci_printf(ctrlr, "unable to allocate shared interrupt\n"); return (ENOMEM); } error = bus_setup_intr(ctrlr->dev, ctrlr->res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, ufshci_ctrlr_shared_handler, ctrlr, &ctrlr->tag); if (error) { ufshci_printf(ctrlr, "unable to setup shared interrupt\n"); return (error); } return (0); } static int ufshci_pci_setup_interrupts(struct ufshci_controller *ctrlr) { device_t dev = ctrlr->dev; int force_intx = 0; int num_io_queues, per_cpu_io_queues, min_cpus_per_ioq; int num_vectors_requested; TUNABLE_INT_FETCH("hw.ufshci.force_intx", &force_intx); if (force_intx) goto intx; if (pci_msix_count(dev) == 0) goto msi; /* * Try to allocate one MSI-X per core for I/O queues, plus one * for admin queue, but accept single shared MSI-X if have to. * Fall back to MSI if can't get any MSI-X. */ /* * TODO: Need to implement MCQ(Multi Circular Queue) * Example: num_io_queues = mp_ncpus; */ num_io_queues = 1; TUNABLE_INT_FETCH("hw.ufshci.num_io_queues", &num_io_queues); if (num_io_queues < 1 || num_io_queues > mp_ncpus) num_io_queues = mp_ncpus; per_cpu_io_queues = 1; TUNABLE_INT_FETCH("hw.ufshci.per_cpu_io_queues", &per_cpu_io_queues); if (per_cpu_io_queues == 0) num_io_queues = 1; min_cpus_per_ioq = smp_threads_per_core; TUNABLE_INT_FETCH("hw.ufshci.min_cpus_per_ioq", &min_cpus_per_ioq); if (min_cpus_per_ioq > 1) { num_io_queues = min(num_io_queues, max(1, mp_ncpus / min_cpus_per_ioq)); } num_io_queues = min(num_io_queues, max(1, pci_msix_count(dev) - 1)); again: if (num_io_queues > vm_ndomains) num_io_queues -= num_io_queues % vm_ndomains; num_vectors_requested = min(num_io_queues + 1, pci_msix_count(dev)); ctrlr->msi_count = num_vectors_requested; if (pci_alloc_msix(dev, &ctrlr->msi_count) != 0) { ufshci_printf(ctrlr, "unable to allocate MSI-X\n"); ctrlr->msi_count = 0; goto msi; } if (ctrlr->msi_count == 1) return (ufshci_pci_setup_shared(ctrlr, 1)); if (ctrlr->msi_count != num_vectors_requested) { pci_release_msi(dev); num_io_queues = ctrlr->msi_count - 1; goto again; } ctrlr->num_io_queues = num_io_queues; return (0); msi: /* * Try to allocate 2 MSIs (admin and I/O queues), but accept single * shared if have to. Fall back to INTx if can't get any MSI. */ ctrlr->msi_count = min(pci_msi_count(dev), 2); if (ctrlr->msi_count > 0) { if (pci_alloc_msi(dev, &ctrlr->msi_count) != 0) { ufshci_printf(ctrlr, "unable to allocate MSI\n"); ctrlr->msi_count = 0; } else if (ctrlr->msi_count == 2) { ctrlr->num_io_queues = 1; return (0); } } intx: return (ufshci_pci_setup_shared(ctrlr, ctrlr->msi_count > 0 ? 1 : 0)); } diff --git a/sys/dev/ufshci/ufshci_private.h b/sys/dev/ufshci/ufshci_private.h index 2e033f84c373..ec388c06e248 100644 --- a/sys/dev/ufshci/ufshci_private.h +++ b/sys/dev/ufshci/ufshci_private.h @@ -1,537 +1,570 @@ /*- * Copyright (c) 2025, Samsung Electronics Co., Ltd. * Written by Jaeyoon Choi * * SPDX-License-Identifier: BSD-2-Clause */ #ifndef __UFSHCI_PRIVATE_H__ #define __UFSHCI_PRIVATE_H__ #ifdef _KERNEL #include #else /* !_KERNEL */ #include #include #endif /* _KERNEL */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ufshci.h" MALLOC_DECLARE(M_UFSHCI); #define UFSHCI_DEVICE_INIT_TIMEOUT_MS (2000) /* in milliseconds */ #define UFSHCI_UIC_CMD_TIMEOUT_MS (500) /* in milliseconds */ #define UFSHCI_DEFAULT_TIMEOUT_PERIOD (10) /* in seconds */ #define UFSHCI_MIN_TIMEOUT_PERIOD (5) /* in seconds */ #define UFSHCI_MAX_TIMEOUT_PERIOD (120) /* in seconds */ #define UFSHCI_DEFAULT_RETRY_COUNT (4) #define UFSHCI_UTR_ENTRIES (32) #define UFSHCI_UTRM_ENTRIES (8) #define UFSHCI_SECTOR_SIZE (512) struct ufshci_controller; struct ufshci_completion_poll_status { struct ufshci_completion cpl; int done; bool error; }; struct ufshci_request { struct ufshci_upiu request_upiu; size_t request_size; size_t response_size; struct memdesc payload; enum ufshci_data_direction data_direction; ufshci_cb_fn_t cb_fn; void *cb_arg; bool is_admin; int32_t retries; bool payload_valid; - bool timeout; bool spare[2]; /* Future use */ STAILQ_ENTRY(ufshci_request) stailq; }; enum ufshci_slot_state { UFSHCI_SLOT_STATE_FREE = 0x0, UFSHCI_SLOT_STATE_RESERVED = 0x1, UFSHCI_SLOT_STATE_SCHEDULED = 0x2, UFSHCI_SLOT_STATE_TIMEOUT = 0x3, UFSHCI_SLOT_STATE_NEED_ERROR_HANDLING = 0x4, }; struct ufshci_tracker { + TAILQ_ENTRY(ufshci_tracker) tailq; struct ufshci_request *req; struct ufshci_req_queue *req_queue; struct ufshci_hw_queue *hwq; uint8_t slot_num; enum ufshci_slot_state slot_state; size_t response_size; sbintime_t deadline; bus_dmamap_t payload_dma_map; uint64_t payload_addr; struct ufshci_utp_cmd_desc *ucd; bus_addr_t ucd_bus_addr; uint16_t prdt_off; uint16_t prdt_entry_cnt; }; enum ufshci_queue_mode { UFSHCI_Q_MODE_SDB = 0x00, /* Single Doorbell Mode*/ UFSHCI_Q_MODE_MCQ = 0x01, /* Multi-Circular Queue Mode*/ }; /* * UFS uses slot-based Single Doorbell (SDB) mode for request submission by * default and additionally supports Multi-Circular Queue (MCQ) in UFS 4.0. To * minimize duplicated code between SDB and MCQ, mode dependent operations are * extracted into ufshci_qops. */ struct ufshci_qops { int (*construct)(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue, uint32_t num_entries, bool is_task_mgmt); void (*destroy)(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue); struct ufshci_hw_queue *(*get_hw_queue)( struct ufshci_req_queue *req_queue); int (*enable)(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue); + void (*disable)(struct ufshci_controller *ctrlr, + struct ufshci_req_queue *req_queue); int (*reserve_slot)(struct ufshci_req_queue *req_queue, struct ufshci_tracker **tr); int (*reserve_admin_slot)(struct ufshci_req_queue *req_queue, struct ufshci_tracker **tr); void (*ring_doorbell)(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); bool (*is_doorbell_cleared)(struct ufshci_controller *ctrlr, uint8_t slot); void (*clear_cpl_ntf)(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); bool (*process_cpl)(struct ufshci_req_queue *req_queue); int (*get_inflight_io)(struct ufshci_controller *ctrlr); }; #define UFSHCI_SDB_Q 0 /* Queue number for a single doorbell queue */ +enum ufshci_recovery { + RECOVERY_NONE = 0, /* Normal operations */ + RECOVERY_WAITING, /* waiting for the reset to complete */ +}; + /* * Generic queue container used by both SDB (fixed 32-slot bitmap) and MCQ * (ring buffer) modes. Fields are shared; some such as sq_head, sq_tail and * cq_head are not used in SDB but used in MCQ. */ struct ufshci_hw_queue { + struct ufshci_controller *ctrlr; + struct ufshci_req_queue *req_queue; uint32_t id; int domain; int cpu; + struct callout timer; /* recovery lock */ + bool timer_armed; /* recovery lock */ + enum ufshci_recovery recovery_state; /* recovery lock */ + union { struct ufshci_utp_xfer_req_desc *utrd; struct ufshci_utp_task_mgmt_req_desc *utmrd; }; bus_dma_tag_t dma_tag_queue; bus_dmamap_t queuemem_map; bus_addr_t req_queue_addr; bus_addr_t *ucd_bus_addr; uint32_t num_entries; uint32_t num_trackers; + TAILQ_HEAD(, ufshci_tracker) free_tr; + TAILQ_HEAD(, ufshci_tracker) outstanding_tr; + /* * A Request List using the single doorbell method uses a dedicated * ufshci_tracker, one per slot. */ struct ufshci_tracker **act_tr; uint32_t sq_head; /* MCQ mode */ uint32_t sq_tail; /* MCQ mode */ uint32_t cq_head; /* MCQ mode */ uint32_t phase; int64_t num_cmds; int64_t num_intr_handler_calls; int64_t num_retries; int64_t num_failures; + /* + * Each lock may be acquired independently. + * When both are required, acquire them in this order to avoid + * deadlocks. (recovery_lock -> qlock) + */ struct mtx_padalign qlock; + struct mtx_padalign recovery_lock; }; struct ufshci_req_queue { struct ufshci_controller *ctrlr; int domain; /* * queue_mode: active transfer scheme * UFSHCI_Q_MODE_SDB – legacy single‑doorbell list * UFSHCI_Q_MODE_MCQ – modern multi‑circular queue (UFSHCI 4.0+) */ enum ufshci_queue_mode queue_mode; uint8_t num_q; struct ufshci_hw_queue *hwq; struct ufshci_qops qops; bool is_task_mgmt; uint32_t num_entries; uint32_t num_trackers; /* Shared DMA resource */ struct ufshci_utp_cmd_desc *ucd; bus_dma_tag_t dma_tag_ucd; bus_dma_tag_t dma_tag_payload; bus_dmamap_t ucdmem_map; }; struct ufshci_device { uint32_t max_lun_count; struct ufshci_device_descriptor dev_desc; struct ufshci_geometry_descriptor geo_desc; uint32_t unipro_version; /* WriteBooster */ bool is_wb_enabled; bool is_wb_flush_enabled; uint32_t wb_buffer_type; uint32_t wb_buffer_size_mb; uint32_t wb_user_space_config_option; uint8_t wb_dedicated_lu; uint32_t write_booster_flush_threshold; }; /* * One of these per allocated device. */ struct ufshci_controller { device_t dev; uint32_t quirks; #define UFSHCI_QUIRK_IGNORE_UIC_POWER_MODE \ 1 /* QEMU does not support UIC POWER MODE */ #define UFSHCI_QUIRK_LONG_PEER_PA_TACTIVATE \ 2 /* Need an additional 200 ms of PA_TActivate */ #define UFSHCI_QUIRK_WAIT_AFTER_POWER_MODE_CHANGE \ 4 /* Need to wait 1250us after power mode change */ #define UFSHCI_QUIRK_CHANGE_LANE_AND_GEAR_SEPARATELY \ 8 /* Need to change the number of lanes before changing HS-GEAR. */ +#define UFSHCI_QUIRK_NOT_SUPPORT_ABORT_TASK \ + 16 /* QEMU does not support Task Management Request */ + uint32_t ref_clk; struct cam_sim *ufshci_sim; struct cam_path *ufshci_path; struct mtx sc_mtx; uint32_t sc_unit; uint8_t sc_name[16]; struct ufshci_device ufs_dev; bus_space_tag_t bus_tag; bus_space_handle_t bus_handle; int resource_id; struct resource *resource; /* Currently, there is no UFSHCI that supports MSI, MSI-X. */ int msi_count; /* Fields for tracking progress during controller initialization. */ struct intr_config_hook config_hook; + struct task reset_task; + struct taskqueue *taskqueue; + /* For shared legacy interrupt. */ int rid; struct resource *res; void *tag; uint32_t major_version; uint32_t minor_version; + uint32_t enable_aborts; + uint32_t num_io_queues; uint32_t max_hw_pend_io; /* Maximum logical unit number */ uint32_t max_lun_count; /* Maximum i/o size in bytes */ uint32_t max_xfer_size; /* Controller capacity */ uint32_t cap; /* Page size and log2(page_size) - 12 that we're currently using */ uint32_t page_size; /* Timeout value on device initialization */ uint32_t device_init_timeout_in_ms; /* Timeout value on UIC command */ uint32_t uic_cmd_timeout_in_ms; /* UTMR/UTR queue timeout period in seconds */ uint32_t timeout_period; /* UTMR/UTR queue retry count */ uint32_t retry_count; /* UFS Host Controller Interface Registers */ struct ufshci_registers *regs; /* UFS Transport Protocol Layer (UTP) */ struct ufshci_req_queue task_mgmt_req_queue; struct ufshci_req_queue transfer_req_queue; bool is_single_db_supported; /* 0 = supported */ bool is_mcq_supported; /* 1 = supported */ /* UFS Interconnect Layer (UIC) */ struct mtx uic_cmd_lock; uint32_t unipro_version; uint8_t hs_gear; uint32_t tx_lanes; uint32_t rx_lanes; uint32_t max_rx_hs_gear; uint32_t max_tx_lanes; uint32_t max_rx_lanes; bool is_failed; }; #define ufshci_mmio_offsetof(reg) offsetof(struct ufshci_registers, reg) #define ufshci_mmio_read_4(sc, reg) \ bus_space_read_4((sc)->bus_tag, (sc)->bus_handle, \ ufshci_mmio_offsetof(reg)) #define ufshci_mmio_write_4(sc, reg, val) \ bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \ ufshci_mmio_offsetof(reg), val) #define ufshci_printf(ctrlr, fmt, args...) \ device_printf(ctrlr->dev, fmt, ##args) /* UFSHCI */ void ufshci_completion_poll_cb(void *arg, const struct ufshci_completion *cpl, bool error); /* SIM */ int ufshci_sim_attach(struct ufshci_controller *ctrlr); void ufshci_sim_detach(struct ufshci_controller *ctrlr); /* Controller */ int ufshci_ctrlr_construct(struct ufshci_controller *ctrlr, device_t dev); void ufshci_ctrlr_destruct(struct ufshci_controller *ctrlr, device_t dev); -int ufshci_ctrlr_reset(struct ufshci_controller *ctrlr); +void ufshci_ctrlr_reset(struct ufshci_controller *ctrlr); /* ctrlr defined as void * to allow use with config_intrhook. */ void ufshci_ctrlr_start_config_hook(void *arg); void ufshci_ctrlr_poll(struct ufshci_controller *ctrlr); int ufshci_ctrlr_submit_task_mgmt_request(struct ufshci_controller *ctrlr, struct ufshci_request *req); int ufshci_ctrlr_submit_admin_request(struct ufshci_controller *ctrlr, struct ufshci_request *req); int ufshci_ctrlr_submit_io_request(struct ufshci_controller *ctrlr, struct ufshci_request *req); int ufshci_ctrlr_send_nop(struct ufshci_controller *ctrlr); void ufshci_reg_dump(struct ufshci_controller *ctrlr); /* Device */ int ufshci_dev_init(struct ufshci_controller *ctrlr); int ufshci_dev_reset(struct ufshci_controller *ctrlr); int ufshci_dev_init_reference_clock(struct ufshci_controller *ctrlr); int ufshci_dev_init_unipro(struct ufshci_controller *ctrlr); int ufshci_dev_init_uic_power_mode(struct ufshci_controller *ctrlr); int ufshci_dev_init_ufs_power_mode(struct ufshci_controller *ctrlr); int ufshci_dev_get_descriptor(struct ufshci_controller *ctrlr); int ufshci_dev_config_write_booster(struct ufshci_controller *ctrlr); /* Controller Command */ void ufshci_ctrlr_cmd_send_task_mgmt_request(struct ufshci_controller *ctrlr, ufshci_cb_fn_t cb_fn, void *cb_arg, uint8_t function, uint8_t lun, uint8_t task_tag, uint8_t iid); void ufshci_ctrlr_cmd_send_nop(struct ufshci_controller *ctrlr, ufshci_cb_fn_t cb_fn, void *cb_arg); void ufshci_ctrlr_cmd_send_query_request(struct ufshci_controller *ctrlr, ufshci_cb_fn_t cb_fn, void *cb_arg, struct ufshci_query_param param); void ufshci_ctrlr_cmd_send_scsi_command(struct ufshci_controller *ctrlr, ufshci_cb_fn_t cb_fn, void *cb_arg, uint8_t *cmd_ptr, uint8_t cmd_len, uint32_t data_len, uint8_t lun, bool is_write); /* Request Queue */ bool ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue); int ufshci_utmr_req_queue_construct(struct ufshci_controller *ctrlr); int ufshci_utr_req_queue_construct(struct ufshci_controller *ctrlr); void ufshci_utmr_req_queue_destroy(struct ufshci_controller *ctrlr); void ufshci_utr_req_queue_destroy(struct ufshci_controller *ctrlr); +void ufshci_utmr_req_queue_disable(struct ufshci_controller *ctrlr); int ufshci_utmr_req_queue_enable(struct ufshci_controller *ctrlr); +void ufshci_utr_req_queue_disable(struct ufshci_controller *ctrlr); int ufshci_utr_req_queue_enable(struct ufshci_controller *ctrlr); void ufshci_req_queue_fail(struct ufshci_controller *ctrlr, struct ufshci_hw_queue *hwq); int ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue, struct ufshci_request *req, bool is_admin); void ufshci_req_queue_complete_tracker(struct ufshci_tracker *tr); /* Request Single Doorbell Queue */ int ufshci_req_sdb_construct(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue, uint32_t num_entries, bool is_task_mgmt); void ufshci_req_sdb_destroy(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue); struct ufshci_hw_queue *ufshci_req_sdb_get_hw_queue( struct ufshci_req_queue *req_queue); +void ufshci_req_sdb_disable(struct ufshci_controller *ctrlr, + struct ufshci_req_queue *req_queue); int ufshci_req_sdb_enable(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue); int ufshci_req_sdb_reserve_slot(struct ufshci_req_queue *req_queue, struct ufshci_tracker **tr); void ufshci_req_sdb_utmr_ring_doorbell(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); void ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); bool ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller *ctrlr, uint8_t slot); bool ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller *ctrlr, uint8_t slot); void ufshci_req_sdb_utmr_clear_cpl_ntf(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); void ufshci_req_sdb_utr_clear_cpl_ntf(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr); bool ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue); int ufshci_req_sdb_get_inflight_io(struct ufshci_controller *ctrlr); /* UIC Command */ int ufshci_uic_power_mode_ready(struct ufshci_controller *ctrlr); int ufshci_uic_cmd_ready(struct ufshci_controller *ctrlr); int ufshci_uic_send_dme_link_startup(struct ufshci_controller *ctrlr); int ufshci_uic_send_dme_get(struct ufshci_controller *ctrlr, uint16_t attribute, uint32_t *return_value); int ufshci_uic_send_dme_set(struct ufshci_controller *ctrlr, uint16_t attribute, uint32_t value); int ufshci_uic_send_dme_peer_get(struct ufshci_controller *ctrlr, uint16_t attribute, uint32_t *return_value); int ufshci_uic_send_dme_peer_set(struct ufshci_controller *ctrlr, uint16_t attribute, uint32_t value); int ufshci_uic_send_dme_endpoint_reset(struct ufshci_controller *ctrlr); /* SYSCTL */ void ufshci_sysctl_initialize_ctrlr(struct ufshci_controller *ctrlr); int ufshci_attach(device_t dev); int ufshci_detach(device_t dev); /* * Wait for a command to complete using the ufshci_completion_poll_cb. Used in * limited contexts where the caller knows it's OK to block briefly while the * command runs. The ISR will run the callback which will set status->done to * true, usually within microseconds. If not, then after one second timeout * handler should reset the controller and abort all outstanding requests * including this polled one. If still not after ten seconds, then something is * wrong with the driver, and panic is the only way to recover. * * Most commands using this interface aren't actual I/O to the drive's media so * complete within a few microseconds. Adaptively spin for one tick to catch the * vast majority of these without waiting for a tick plus scheduling delays. * Since these are on startup, this drastically reduces startup time. */ static __inline void ufshci_completion_poll(struct ufshci_completion_poll_status *status) { int timeout = ticks + 10 * hz; sbintime_t delta_t = SBT_1US; while (!atomic_load_acq_int(&status->done)) { if (timeout - ticks < 0) panic( "UFSHCI polled command failed to complete within 10s."); pause_sbt("ufshci_cpl", delta_t, 0, C_PREL(1)); delta_t = min(SBT_1MS, delta_t * 3 / 2); } } static __inline void ufshci_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) { uint64_t *bus_addr = (uint64_t *)arg; KASSERT(nseg == 1, ("number of segments (%d) is not 1", nseg)); if (error != 0) printf("ufshci_single_map err %d\n", error); *bus_addr = seg[0].ds_addr; } static __inline struct ufshci_request * _ufshci_allocate_request(const int how, ufshci_cb_fn_t cb_fn, void *cb_arg) { struct ufshci_request *req; KASSERT(how == M_WAITOK || how == M_NOWAIT, - ("nvme_allocate_request: invalid how %d", how)); + ("ufshci_allocate_request: invalid how %d", how)); req = malloc(sizeof(*req), M_UFSHCI, how | M_ZERO); if (req != NULL) { req->cb_fn = cb_fn; req->cb_arg = cb_arg; - req->timeout = true; } return (req); } static __inline struct ufshci_request * ufshci_allocate_request_vaddr(void *payload, uint32_t payload_size, const int how, ufshci_cb_fn_t cb_fn, void *cb_arg) { struct ufshci_request *req; req = _ufshci_allocate_request(how, cb_fn, cb_arg); if (req != NULL) { if (payload_size) { req->payload = memdesc_vaddr(payload, payload_size); req->payload_valid = true; } } return (req); } static __inline struct ufshci_request * ufshci_allocate_request_bio(struct bio *bio, const int how, ufshci_cb_fn_t cb_fn, void *cb_arg) { struct ufshci_request *req; req = _ufshci_allocate_request(how, cb_fn, cb_arg); if (req != NULL) { req->payload = memdesc_bio(bio); req->payload_valid = true; } return (req); } #define ufshci_free_request(req) free(req, M_UFSHCI) void ufshci_ctrlr_shared_handler(void *arg); #endif /* __UFSHCI_PRIVATE_H__ */ diff --git a/sys/dev/ufshci/ufshci_req_queue.c b/sys/dev/ufshci/ufshci_req_queue.c index bb6efa6d2ccc..7aa164d00bec 100644 --- a/sys/dev/ufshci/ufshci_req_queue.c +++ b/sys/dev/ufshci/ufshci_req_queue.c @@ -1,533 +1,799 @@ /*- * Copyright (c) 2025, Samsung Electronics Co., Ltd. * Written by Jaeyoon Choi * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include #include #include #include "sys/kassert.h" #include "ufshci_private.h" static void ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue, struct ufshci_tracker *tr, enum ufshci_data_direction data_direction); static const struct ufshci_qops sdb_utmr_qops = { .construct = ufshci_req_sdb_construct, .destroy = ufshci_req_sdb_destroy, .get_hw_queue = ufshci_req_sdb_get_hw_queue, .enable = ufshci_req_sdb_enable, + .disable = ufshci_req_sdb_disable, .reserve_slot = ufshci_req_sdb_reserve_slot, .reserve_admin_slot = ufshci_req_sdb_reserve_slot, .ring_doorbell = ufshci_req_sdb_utmr_ring_doorbell, .is_doorbell_cleared = ufshci_req_sdb_utmr_is_doorbell_cleared, .clear_cpl_ntf = ufshci_req_sdb_utmr_clear_cpl_ntf, .process_cpl = ufshci_req_sdb_process_cpl, .get_inflight_io = ufshci_req_sdb_get_inflight_io, }; static const struct ufshci_qops sdb_utr_qops = { .construct = ufshci_req_sdb_construct, .destroy = ufshci_req_sdb_destroy, .get_hw_queue = ufshci_req_sdb_get_hw_queue, .enable = ufshci_req_sdb_enable, + .disable = ufshci_req_sdb_disable, .reserve_slot = ufshci_req_sdb_reserve_slot, .reserve_admin_slot = ufshci_req_sdb_reserve_slot, .ring_doorbell = ufshci_req_sdb_utr_ring_doorbell, .is_doorbell_cleared = ufshci_req_sdb_utr_is_doorbell_cleared, .clear_cpl_ntf = ufshci_req_sdb_utr_clear_cpl_ntf, .process_cpl = ufshci_req_sdb_process_cpl, .get_inflight_io = ufshci_req_sdb_get_inflight_io, }; int ufshci_utmr_req_queue_construct(struct ufshci_controller *ctrlr) { struct ufshci_req_queue *req_queue; int error; /* * UTP Task Management Request only supports Legacy Single Doorbell * Queue. */ req_queue = &ctrlr->task_mgmt_req_queue; req_queue->queue_mode = UFSHCI_Q_MODE_SDB; req_queue->qops = sdb_utmr_qops; error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTRM_ENTRIES, /*is_task_mgmt*/ true); return (error); } void ufshci_utmr_req_queue_destroy(struct ufshci_controller *ctrlr) { ctrlr->task_mgmt_req_queue.qops.destroy(ctrlr, &ctrlr->task_mgmt_req_queue); } +void +ufshci_utmr_req_queue_disable(struct ufshci_controller *ctrlr) +{ + ctrlr->task_mgmt_req_queue.qops.disable(ctrlr, + &ctrlr->task_mgmt_req_queue); +} + int ufshci_utmr_req_queue_enable(struct ufshci_controller *ctrlr) { return (ctrlr->task_mgmt_req_queue.qops.enable(ctrlr, &ctrlr->task_mgmt_req_queue)); } int ufshci_utr_req_queue_construct(struct ufshci_controller *ctrlr) { struct ufshci_req_queue *req_queue; int error; /* * Currently, it does not support MCQ mode, so it should be set to SDB * mode by default. * TODO: Determine queue mode by checking Capability Registers */ req_queue = &ctrlr->transfer_req_queue; req_queue->queue_mode = UFSHCI_Q_MODE_SDB; req_queue->qops = sdb_utr_qops; error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTR_ENTRIES, /*is_task_mgmt*/ false); return (error); } void ufshci_utr_req_queue_destroy(struct ufshci_controller *ctrlr) { ctrlr->transfer_req_queue.qops.destroy(ctrlr, &ctrlr->transfer_req_queue); } +void +ufshci_utr_req_queue_disable(struct ufshci_controller *ctrlr) +{ + ctrlr->transfer_req_queue.qops.disable(ctrlr, + &ctrlr->transfer_req_queue); +} + int ufshci_utr_req_queue_enable(struct ufshci_controller *ctrlr) { return (ctrlr->transfer_req_queue.qops.enable(ctrlr, &ctrlr->transfer_req_queue)); } static bool ufshci_req_queue_response_is_error(struct ufshci_req_queue *req_queue, uint8_t ocs, union ufshci_reponse_upiu *response) { bool is_error = false; /* Check request descriptor */ if (ocs != UFSHCI_DESC_SUCCESS) { ufshci_printf(req_queue->ctrlr, "Invalid OCS = 0x%x\n", ocs); is_error = true; } /* Check response UPIU header */ if (response->header.response != UFSHCI_RESPONSE_CODE_TARGET_SUCCESS) { ufshci_printf(req_queue->ctrlr, "Invalid response code = 0x%x\n", response->header.response); is_error = true; } return (is_error); } static void ufshci_req_queue_manual_complete_tracker(struct ufshci_tracker *tr, uint8_t ocs, uint8_t rc) { struct ufshci_utp_xfer_req_desc *desc; struct ufshci_upiu_header *resp_header; mtx_assert(&tr->hwq->qlock, MA_NOTOWNED); resp_header = (struct ufshci_upiu_header *)tr->ucd->response_upiu; resp_header->response = rc; desc = &tr->hwq->utrd[tr->slot_num]; desc->overall_command_status = ocs; ufshci_req_queue_complete_tracker(tr); } static void ufshci_req_queue_manual_complete_request(struct ufshci_req_queue *req_queue, struct ufshci_request *req, uint8_t ocs, uint8_t rc) { struct ufshci_completion cpl; bool error; memset(&cpl, 0, sizeof(cpl)); cpl.response_upiu.header.response = rc; error = ufshci_req_queue_response_is_error(req_queue, ocs, &cpl.response_upiu); if (error) { ufshci_printf(req_queue->ctrlr, "Manual complete request error:0x%x", error); } if (req->cb_fn) req->cb_fn(req->cb_arg, &cpl, error); ufshci_free_request(req); } void ufshci_req_queue_fail(struct ufshci_controller *ctrlr, struct ufshci_hw_queue *hwq) { struct ufshci_req_queue *req_queue; struct ufshci_tracker *tr; struct ufshci_request *req; int i; if (!mtx_initialized(&hwq->qlock)) return; mtx_lock(&hwq->qlock); req_queue = &ctrlr->transfer_req_queue; for (i = 0; i < req_queue->num_entries; i++) { tr = hwq->act_tr[i]; req = tr->req; if (tr->slot_state == UFSHCI_SLOT_STATE_RESERVED) { mtx_unlock(&hwq->qlock); ufshci_req_queue_manual_complete_request(req_queue, req, UFSHCI_DESC_ABORTED, UFSHCI_RESPONSE_CODE_GENERAL_FAILURE); mtx_lock(&hwq->qlock); } else if (tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED) { /* * Do not remove the tracker. The abort_tracker path * will do that for us. */ mtx_unlock(&hwq->qlock); ufshci_req_queue_manual_complete_tracker(tr, UFSHCI_DESC_ABORTED, UFSHCI_RESPONSE_CODE_GENERAL_FAILURE); mtx_lock(&hwq->qlock); } } mtx_unlock(&hwq->qlock); } void ufshci_req_queue_complete_tracker(struct ufshci_tracker *tr) { struct ufshci_req_queue *req_queue = tr->req_queue; + struct ufshci_hw_queue *hwq = tr->hwq; struct ufshci_request *req = tr->req; struct ufshci_completion cpl; uint8_t ocs; bool retry, error, retriable; - mtx_assert(&tr->hwq->qlock, MA_NOTOWNED); + mtx_assert(&hwq->qlock, MA_NOTOWNED); /* Copy the response from the Request Descriptor or UTP Command * Descriptor. */ + cpl.size = tr->response_size; if (req_queue->is_task_mgmt) { - cpl.size = tr->response_size; memcpy(&cpl.response_upiu, - (void *)tr->hwq->utmrd[tr->slot_num].response_upiu, - cpl.size); + (void *)hwq->utmrd[tr->slot_num].response_upiu, cpl.size); - ocs = tr->hwq->utmrd[tr->slot_num].overall_command_status; + ocs = hwq->utmrd[tr->slot_num].overall_command_status; } else { bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - cpl.size = tr->response_size; memcpy(&cpl.response_upiu, (void *)tr->ucd->response_upiu, cpl.size); - ocs = tr->hwq->utrd[tr->slot_num].overall_command_status; + ocs = hwq->utrd[tr->slot_num].overall_command_status; } error = ufshci_req_queue_response_is_error(req_queue, ocs, &cpl.response_upiu); /* TODO: Implement retry */ // retriable = ufshci_completion_is_retry(cpl); retriable = false; retry = error && retriable && req->retries < req_queue->ctrlr->retry_count; if (retry) - tr->hwq->num_retries++; + hwq->num_retries++; if (error && req->retries >= req_queue->ctrlr->retry_count && retriable) - tr->hwq->num_failures++; + hwq->num_failures++; KASSERT(tr->req, ("there is no request assigned to the tracker\n")); KASSERT(cpl.response_upiu.header.task_tag == req->request_upiu.header.task_tag, ("response task_tag does not match request task_tag\n")); if (!retry) { if (req->payload_valid) { bus_dmamap_sync(req_queue->dma_tag_payload, tr->payload_dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); } /* Copy response from the command descriptor */ if (req->cb_fn) req->cb_fn(req->cb_arg, &cpl, error); } - mtx_lock(&tr->hwq->qlock); + mtx_lock(&hwq->qlock); /* Clear the UTRL Completion Notification register */ req_queue->qops.clear_cpl_ntf(req_queue->ctrlr, tr); if (retry) { req->retries++; ufshci_req_queue_submit_tracker(req_queue, tr, req->data_direction); } else { if (req->payload_valid) { bus_dmamap_unload(req_queue->dma_tag_payload, tr->payload_dma_map); } /* Clear tracker */ ufshci_free_request(req); tr->req = NULL; tr->slot_state = UFSHCI_SLOT_STATE_FREE; + + TAILQ_REMOVE(&hwq->outstanding_tr, tr, tailq); + TAILQ_INSERT_HEAD(&hwq->free_tr, tr, tailq); } mtx_unlock(&tr->hwq->qlock); } bool ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue) { - return (req_queue->qops.process_cpl(req_queue)); + struct ufshci_hw_queue *hwq; + bool done; + + hwq = req_queue->qops.get_hw_queue(req_queue); + + mtx_lock(&hwq->recovery_lock); + done = req_queue->qops.process_cpl(req_queue); + mtx_unlock(&hwq->recovery_lock); + + return (done); } static void ufshci_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) { struct ufshci_tracker *tr = arg; struct ufshci_prdt_entry *prdt_entry; int i; /* * If the mapping operation failed, return immediately. The caller * is responsible for detecting the error status and failing the * tracker manually. */ if (error != 0) { ufshci_printf(tr->req_queue->ctrlr, "Failed to map payload %d\n", error); return; } prdt_entry = (struct ufshci_prdt_entry *)tr->ucd->prd_table; tr->prdt_entry_cnt = nseg; for (i = 0; i < nseg; i++) { prdt_entry->data_base_address = htole64(seg[i].ds_addr) & 0xffffffff; prdt_entry->data_base_address_upper = htole64(seg[i].ds_addr) >> 32; prdt_entry->data_byte_count = htole32(seg[i].ds_len - 1); ++prdt_entry; } bus_dmamap_sync(tr->req_queue->dma_tag_payload, tr->payload_dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); } static void ufshci_req_queue_prepare_prdt(struct ufshci_tracker *tr) { struct ufshci_request *req = tr->req; struct ufshci_utp_cmd_desc *cmd_desc = tr->ucd; int error; tr->prdt_off = UFSHCI_UTP_XFER_REQ_SIZE + UFSHCI_UTP_XFER_RESP_SIZE; memset(cmd_desc->prd_table, 0, sizeof(cmd_desc->prd_table)); /* Filling PRDT enrties with payload */ error = bus_dmamap_load_mem(tr->req_queue->dma_tag_payload, tr->payload_dma_map, &req->payload, ufshci_payload_map, tr, BUS_DMA_NOWAIT); if (error != 0) { /* * The dmamap operation failed, so we manually fail the * tracker here with UFSHCI_DESC_INVALID_PRDT_ATTRIBUTES. * * ufshci_req_queue_manual_complete_tracker must not be called * with the req_queue lock held. */ ufshci_printf(tr->req_queue->ctrlr, "bus_dmamap_load_mem returned with error:0x%x!\n", error); mtx_unlock(&tr->hwq->qlock); ufshci_req_queue_manual_complete_tracker(tr, UFSHCI_DESC_INVALID_PRDT_ATTRIBUTES, UFSHCI_RESPONSE_CODE_GENERAL_FAILURE); mtx_lock(&tr->hwq->qlock); } } static void ufshci_req_queue_fill_utmr_descriptor( struct ufshci_utp_task_mgmt_req_desc *desc, struct ufshci_request *req) { memset(desc, 0, sizeof(struct ufshci_utp_task_mgmt_req_desc)); desc->interrupt = true; /* Set the initial value to Invalid. */ desc->overall_command_status = UFSHCI_UTMR_OCS_INVALID; memcpy(desc->request_upiu, &req->request_upiu, req->request_size); } static void ufshci_req_queue_fill_utr_descriptor(struct ufshci_utp_xfer_req_desc *desc, uint8_t data_direction, const uint64_t paddr, const uint16_t response_off, const uint16_t response_len, const uint16_t prdt_off, const uint16_t prdt_entry_cnt) { uint8_t command_type; /* Value to convert bytes to dwords */ const uint16_t dword_size = 4; /* * Set command type to UFS storage. * The UFS 4.1 spec only defines 'UFS Storage' as a command type. */ command_type = UFSHCI_COMMAND_TYPE_UFS_STORAGE; memset(desc, 0, sizeof(struct ufshci_utp_xfer_req_desc)); desc->command_type = command_type; desc->data_direction = data_direction; desc->interrupt = true; /* Set the initial value to Invalid. */ desc->overall_command_status = UFSHCI_UTR_OCS_INVALID; desc->utp_command_descriptor_base_address = (uint32_t)(paddr & 0xffffffff); desc->utp_command_descriptor_base_address_upper = (uint32_t)(paddr >> 32); desc->response_upiu_offset = response_off / dword_size; desc->response_upiu_length = response_len / dword_size; desc->prdt_offset = prdt_off / dword_size; desc->prdt_length = prdt_entry_cnt; } +static void +ufshci_req_queue_timeout_recovery(struct ufshci_controller *ctrlr, + struct ufshci_hw_queue *hwq) +{ + /* TODO: Step 2. Logical unit reset */ + /* TODO: Step 3. Target device reset */ + /* TODO: Step 4. Bus reset */ + + /* + * Step 5. All previous commands were timeout. + * Recovery failed, reset the host controller. + */ + ufshci_printf(ctrlr, + "Recovery step 5: Resetting controller due to a timeout.\n"); + hwq->recovery_state = RECOVERY_WAITING; + + ufshci_ctrlr_reset(ctrlr); +} + +static void +ufshci_abort_complete(void *arg, const struct ufshci_completion *status, + bool error) +{ + struct ufshci_tracker *tr = arg; + + /* + * We still need to check the active tracker array, to cover race where + * I/O timed out at same time controller was completing the I/O. An + * abort request always is on the Task Management Request queue, but + * affects either an Task Management Request or an I/O (UTRL) queue, so + * take the appropriate queue lock for the original command's queue, + * since we'll need it to avoid races with the completion code and to + * complete the command manually. + */ + mtx_lock(&tr->hwq->qlock); + if (tr->slot_state != UFSHCI_SLOT_STATE_FREE) { + mtx_unlock(&tr->hwq->qlock); + /* + * An I/O has timed out, and the controller was unable to abort + * it for some reason. And we've not processed a completion for + * it yet. Construct a fake completion status, and then complete + * the I/O's tracker manually. + */ + ufshci_printf(tr->hwq->ctrlr, + "abort task request failed, aborting task manually\n"); + ufshci_req_queue_manual_complete_tracker(tr, + UFSHCI_DESC_ABORTED, UFSHCI_RESPONSE_CODE_GENERAL_FAILURE); + + if ((status->response_upiu.task_mgmt_response_upiu + .output_param1 == + UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_COMPLETE) || + (status->response_upiu.task_mgmt_response_upiu + .output_param1 == + UFSHCI_TASK_MGMT_SERVICE_RESPONSE_FUNCTION_SUCCEEDED)) { + ufshci_printf(tr->hwq->ctrlr, + "Warning: the abort task request completed \ + successfully, but the original task is still incomplete."); + return; + } + + /* Abort Task failed. Perform recovery steps 2-5 */ + ufshci_req_queue_timeout_recovery(tr->hwq->ctrlr, tr->hwq); + } else { + mtx_unlock(&tr->hwq->qlock); + } +} + +static void +ufshci_req_queue_timeout(void *arg) +{ + struct ufshci_hw_queue *hwq = arg; + struct ufshci_controller *ctrlr = hwq->ctrlr; + struct ufshci_tracker *tr; + sbintime_t now; + bool idle = true; + bool fast; + + mtx_assert(&hwq->recovery_lock, MA_OWNED); + + /* + * If the controller is failed, then stop polling. This ensures that any + * failure processing that races with the hwq timeout will fail safely. + */ + if (ctrlr->is_failed) { + ufshci_printf(ctrlr, + "Failed controller, stopping watchdog timeout.\n"); + hwq->timer_armed = false; + return; + } + + /* + * Shutdown condition: We set hwq->timer_armed to false in + * ufshci_req_sdb_destroy before calling callout_drain. When we call + * that, this routine might get called one last time. Exit w/o setting a + * timeout. None of the watchdog stuff needs to be done since we're + * destroying the hwq. + */ + if (!hwq->timer_armed) { + ufshci_printf(ctrlr, + "Timeout fired during ufshci_utr_req_queue_destroy\n"); + return; + } + + switch (hwq->recovery_state) { + case RECOVERY_NONE: + /* + * See if there's any recovery needed. First, do a fast check to + * see if anything could have timed out. If not, then skip + * everything else. + */ + fast = false; + mtx_lock(&hwq->qlock); + now = getsbinuptime(); + TAILQ_FOREACH(tr, &hwq->outstanding_tr, tailq) { + /* + * If the first real transaction is not in timeout, then + * we're done. Otherwise, we try recovery. + */ + idle = false; + if (now <= tr->deadline) + fast = true; + break; + } + mtx_unlock(&hwq->qlock); + if (idle || fast) + break; + + /* + * There's a stale transaction at the start of the queue whose + * deadline has passed. Poll the competions as a last-ditch + * effort in case an interrupt has been missed. + */ + hwq->req_queue->qops.process_cpl(hwq->req_queue); + + /* + * Now that we've run the ISR, re-rheck to see if there's any + * timed out commands and abort them or reset the card if so. + */ + mtx_lock(&hwq->qlock); + idle = true; + TAILQ_FOREACH(tr, &hwq->outstanding_tr, tailq) { + /* + * If we know this tracker hasn't timed out, we also + * know all subsequent ones haven't timed out. The tr + * queue is in submission order and all normal commands + * in a queue have the same timeout (or the timeout was + * changed by the user, but we eventually timeout then). + */ + idle = false; + if (now <= tr->deadline) + break; + + /* + * Timeout recovery is performed in five steps. If + * recovery fails at any step, the process continues to + * the next one: + * next steps: + * Step 1. Abort task + * Step 2. Logical unit reset (TODO) + * Step 3. Target device reset (TODO) + * Step 4. Bus reset (TODO) + * Step 5. Host controller reset + * + * If the timeout occurred in the Task Management + * Request queue, ignore Step 1. + */ + if (ctrlr->enable_aborts && + !hwq->req_queue->is_task_mgmt && + tr->req->cb_fn != ufshci_abort_complete) { + /* + * Step 1. Timeout expired, abort the task. + * + * This isn't an abort command, ask for a + * hardware abort. This goes to the Task + * Management Request queue which will reset the + * task if it times out. + */ + ufshci_printf(ctrlr, + "Recovery step 1: Timeout occurred. aborting the task(%d).\n", + tr->req->request_upiu.header.task_tag); + ufshci_ctrlr_cmd_send_task_mgmt_request(ctrlr, + ufshci_abort_complete, tr, + UFSHCI_TASK_MGMT_FUNCTION_ABORT_TASK, + tr->req->request_upiu.header.lun, + tr->req->request_upiu.header.task_tag, 0); + } else { + /* Recovery Step 2-5 */ + ufshci_req_queue_timeout_recovery(ctrlr, hwq); + idle = false; + break; + } + } + mtx_unlock(&hwq->qlock); + break; + + case RECOVERY_WAITING: + /* + * These messages aren't interesting while we're suspended. We + * put the queues into waiting state while suspending. + * Suspending takes a while, so we'll see these during that time + * and they aren't diagnostic. At other times, they indicate a + * problem that's worth complaining about. + */ + if (!device_is_suspended(ctrlr->dev)) + ufshci_printf(ctrlr, "Waiting for reset to complete\n"); + idle = false; /* We want to keep polling */ + break; + } + + /* + * Rearm the timeout. + */ + if (!idle) { + callout_schedule_sbt(&hwq->timer, SBT_1S / 2, SBT_1S / 2, 0); + } else { + hwq->timer_armed = false; + } +} + /* * Submit the tracker to the hardware. */ static void ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue, struct ufshci_tracker *tr, enum ufshci_data_direction data_direction) { struct ufshci_controller *ctrlr = req_queue->ctrlr; struct ufshci_request *req = tr->req; + struct ufshci_hw_queue *hwq; uint64_t ucd_paddr; uint16_t request_len, response_off, response_len; uint8_t slot_num = tr->slot_num; + int timeout; - mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED); + hwq = req_queue->qops.get_hw_queue(req_queue); + + mtx_assert(&hwq->qlock, MA_OWNED); - /* TODO: Check timeout */ + if (req->cb_fn == ufshci_completion_poll_cb) + timeout = 1; + else + timeout = ctrlr->timeout_period; + tr->deadline = getsbinuptime() + timeout * SBT_1S; + if (!hwq->timer_armed) { + hwq->timer_armed = true; + /* + * It wakes up once every 0.5 seconds to check if the deadline + * has passed. + */ + callout_reset_sbt_on(&hwq->timer, SBT_1S / 2, SBT_1S / 2, + ufshci_req_queue_timeout, hwq, hwq->cpu, 0); + } if (req_queue->is_task_mgmt) { /* Prepare UTP Task Management Request Descriptor. */ ufshci_req_queue_fill_utmr_descriptor(&tr->hwq->utmrd[slot_num], req); } else { request_len = req->request_size; response_off = UFSHCI_UTP_XFER_REQ_SIZE; response_len = req->response_size; /* Prepare UTP Command Descriptor */ memcpy(tr->ucd, &req->request_upiu, request_len); memset((uint8_t *)tr->ucd + response_off, 0, response_len); /* Prepare PRDT */ if (req->payload_valid) ufshci_req_queue_prepare_prdt(tr); /* Prepare UTP Transfer Request Descriptor. */ ucd_paddr = tr->ucd_bus_addr; ufshci_req_queue_fill_utr_descriptor(&tr->hwq->utrd[slot_num], data_direction, ucd_paddr, response_off, response_len, tr->prdt_off, tr->prdt_entry_cnt); bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); } bus_dmamap_sync(tr->hwq->dma_tag_queue, tr->hwq->queuemem_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); tr->slot_state = UFSHCI_SLOT_STATE_SCHEDULED; /* Ring the doorbell */ req_queue->qops.ring_doorbell(ctrlr, tr); } static int _ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue, struct ufshci_request *req) { struct ufshci_tracker *tr = NULL; int error; mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED); error = req_queue->qops.reserve_slot(req_queue, &tr); if (error != 0) { ufshci_printf(req_queue->ctrlr, "Failed to get tracker"); return (error); } KASSERT(tr, ("There is no tracker allocated.")); if (tr->slot_state == UFSHCI_SLOT_STATE_RESERVED || tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED) return (EBUSY); /* Set the task_tag value to slot_num for traceability. */ req->request_upiu.header.task_tag = tr->slot_num; tr->slot_state = UFSHCI_SLOT_STATE_RESERVED; tr->response_size = req->response_size; tr->deadline = SBT_MAX; tr->req = req; + TAILQ_REMOVE(&tr->hwq->free_tr, tr, tailq); + TAILQ_INSERT_TAIL(&tr->hwq->outstanding_tr, tr, tailq); + ufshci_req_queue_submit_tracker(req_queue, tr, req->data_direction); return (0); } int ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue, struct ufshci_request *req, bool is_admin) { struct ufshci_hw_queue *hwq; uint32_t error; /* TODO: MCQs should use a separate Admin queue. */ hwq = req_queue->qops.get_hw_queue(req_queue); KASSERT(hwq, ("There is no HW queue allocated.")); mtx_lock(&hwq->qlock); error = _ufshci_req_queue_submit_request(req_queue, req); mtx_unlock(&hwq->qlock); return (error); } diff --git a/sys/dev/ufshci/ufshci_req_sdb.c b/sys/dev/ufshci/ufshci_req_sdb.c index 834a459d48e3..ca47aa159c5b 100644 --- a/sys/dev/ufshci/ufshci_req_sdb.c +++ b/sys/dev/ufshci/ufshci_req_sdb.c @@ -1,499 +1,562 @@ /*- * Copyright (c) 2025, Samsung Electronics Co., Ltd. * Written by Jaeyoon Choi * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include #include #include "sys/kassert.h" #include "ufshci_private.h" #include "ufshci_reg.h" static void ufshci_req_sdb_cmd_desc_destroy(struct ufshci_req_queue *req_queue) { struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q]; struct ufshci_tracker *tr; int i; for (i = 0; i < req_queue->num_trackers; i++) { tr = hwq->act_tr[i]; bus_dmamap_destroy(req_queue->dma_tag_payload, tr->payload_dma_map); } if (req_queue->ucd) { bus_dmamap_unload(req_queue->dma_tag_ucd, req_queue->ucdmem_map); bus_dmamem_free(req_queue->dma_tag_ucd, req_queue->ucd, req_queue->ucdmem_map); req_queue->ucd = NULL; } if (req_queue->dma_tag_ucd) { bus_dma_tag_destroy(req_queue->dma_tag_ucd); req_queue->dma_tag_ucd = NULL; } + + free(req_queue->hwq->ucd_bus_addr, M_UFSHCI); } static void ufshci_ucd_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) { struct ufshci_hw_queue *hwq = arg; int i; if (error != 0) { printf("ufshci: Failed to map UCD, error = %d\n", error); return; } if (hwq->num_trackers != nseg) { printf( "ufshci: Failed to map UCD, num_trackers = %d, nseg = %d\n", hwq->num_trackers, nseg); return; } for (i = 0; i < nseg; i++) { hwq->ucd_bus_addr[i] = seg[i].ds_addr; } } static int ufshci_req_sdb_cmd_desc_construct(struct ufshci_req_queue *req_queue, uint32_t num_entries, struct ufshci_controller *ctrlr) { struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q]; size_t ucd_allocsz, payload_allocsz; uint8_t *ucdmem; int i, error; + req_queue->hwq->ucd_bus_addr = malloc(sizeof(bus_addr_t) * + req_queue->num_trackers, + M_UFSHCI, M_ZERO | M_NOWAIT); + /* * Each component must be page aligned, and individual PRP lists * cannot cross a page boundary. */ ucd_allocsz = num_entries * sizeof(struct ufshci_utp_cmd_desc); ucd_allocsz = roundup2(ucd_allocsz, ctrlr->page_size); payload_allocsz = num_entries * ctrlr->max_xfer_size; /* * Allocate physical memory for UTP Command Descriptor (UCD) * Note: UFSHCI UCD format is restricted to 128-byte alignment. */ error = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 128, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, ucd_allocsz, howmany(ucd_allocsz, sizeof(struct ufshci_utp_cmd_desc)), sizeof(struct ufshci_utp_cmd_desc), 0, NULL, NULL, &req_queue->dma_tag_ucd); if (error != 0) { ufshci_printf(ctrlr, "request cmd desc tag create failed %d\n", error); goto out; } if (bus_dmamem_alloc(req_queue->dma_tag_ucd, (void **)&ucdmem, BUS_DMA_COHERENT | BUS_DMA_NOWAIT, &req_queue->ucdmem_map)) { ufshci_printf(ctrlr, "failed to allocate cmd desc memory\n"); goto out; } if (bus_dmamap_load(req_queue->dma_tag_ucd, req_queue->ucdmem_map, ucdmem, ucd_allocsz, ufshci_ucd_map, hwq, 0) != 0) { ufshci_printf(ctrlr, "failed to load cmd desc memory\n"); bus_dmamem_free(req_queue->dma_tag_ucd, req_queue->ucd, req_queue->ucdmem_map); goto out; } req_queue->ucd = (struct ufshci_utp_cmd_desc *)ucdmem; /* * Allocate physical memory for PRDT * Note: UFSHCI PRDT format is restricted to 8-byte alignment. */ error = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 8, ctrlr->page_size, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, payload_allocsz, howmany(payload_allocsz, ctrlr->page_size) + 1, ctrlr->page_size, 0, NULL, NULL, &req_queue->dma_tag_payload); if (error != 0) { ufshci_printf(ctrlr, "request prdt tag create failed %d\n", error); goto out; } for (i = 0; i < req_queue->num_trackers; i++) { bus_dmamap_create(req_queue->dma_tag_payload, 0, &hwq->act_tr[i]->payload_dma_map); hwq->act_tr[i]->ucd = (struct ufshci_utp_cmd_desc *)ucdmem; hwq->act_tr[i]->ucd_bus_addr = hwq->ucd_bus_addr[i]; ucdmem += sizeof(struct ufshci_utp_cmd_desc); } return (0); out: ufshci_req_sdb_cmd_desc_destroy(req_queue); return (ENOMEM); } int ufshci_req_sdb_construct(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue, uint32_t num_entries, bool is_task_mgmt) { struct ufshci_hw_queue *hwq; size_t desc_size, alloc_size; uint64_t queuemem_phys; uint8_t *queuemem; struct ufshci_tracker *tr; + const size_t lock_name_len = 32; + char qlock_name[lock_name_len], recovery_lock_name[lock_name_len]; + char *base; int i, error; req_queue->ctrlr = ctrlr; req_queue->is_task_mgmt = is_task_mgmt; req_queue->num_entries = num_entries; /* * In Single Doorbell mode, the number of queue entries and the number * of trackers are the same. */ req_queue->num_trackers = num_entries; /* Single Doorbell mode uses only one queue. (UFSHCI_SDB_Q = 0) */ req_queue->hwq = malloc(sizeof(struct ufshci_hw_queue), M_UFSHCI, M_ZERO | M_NOWAIT); hwq = &req_queue->hwq[UFSHCI_SDB_Q]; hwq->num_entries = req_queue->num_entries; hwq->num_trackers = req_queue->num_trackers; - req_queue->hwq->ucd_bus_addr = malloc(sizeof(bus_addr_t) * - req_queue->num_trackers, - M_UFSHCI, M_ZERO | M_NOWAIT); + hwq->ctrlr = ctrlr; + hwq->req_queue = req_queue; + + base = is_task_mgmt ? "ufshci utmrq" : "ufshci utrq"; + snprintf(qlock_name, sizeof(qlock_name), "%s #%d lock", base, + UFSHCI_SDB_Q); + snprintf(recovery_lock_name, sizeof(recovery_lock_name), + "%s #%d recovery lock", base, UFSHCI_SDB_Q); - mtx_init(&hwq->qlock, "ufshci req_queue lock", NULL, MTX_DEF); + mtx_init(&hwq->qlock, qlock_name, NULL, MTX_DEF); + mtx_init(&hwq->recovery_lock, recovery_lock_name, NULL, MTX_DEF); + + callout_init_mtx(&hwq->timer, &hwq->recovery_lock, 0); + hwq->timer_armed = false; + hwq->recovery_state = RECOVERY_WAITING; /* * Allocate physical memory for request queue (UTP Transfer Request * Descriptor (UTRD) or UTP Task Management Request Descriptor (UTMRD)) * Note: UTRD/UTMRD format is restricted to 1024-byte alignment. */ desc_size = is_task_mgmt ? sizeof(struct ufshci_utp_task_mgmt_req_desc) : sizeof(struct ufshci_utp_xfer_req_desc); alloc_size = num_entries * desc_size; error = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), 1024, ctrlr->page_size, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, alloc_size, 1, alloc_size, 0, NULL, NULL, &hwq->dma_tag_queue); if (error != 0) { ufshci_printf(ctrlr, "request queue tag create failed %d\n", error); goto out; } if (bus_dmamem_alloc(hwq->dma_tag_queue, (void **)&queuemem, BUS_DMA_COHERENT | BUS_DMA_NOWAIT, &hwq->queuemem_map)) { ufshci_printf(ctrlr, "failed to allocate request queue memory\n"); goto out; } if (bus_dmamap_load(hwq->dma_tag_queue, hwq->queuemem_map, queuemem, alloc_size, ufshci_single_map, &queuemem_phys, 0) != 0) { ufshci_printf(ctrlr, "failed to load request queue memory\n"); bus_dmamem_free(hwq->dma_tag_queue, hwq->utrd, hwq->queuemem_map); goto out; } hwq->num_cmds = 0; hwq->num_intr_handler_calls = 0; hwq->num_retries = 0; hwq->num_failures = 0; hwq->req_queue_addr = queuemem_phys; /* Allocate trackers */ hwq->act_tr = malloc_domainset(sizeof(struct ufshci_tracker *) * req_queue->num_entries, M_UFSHCI, DOMAINSET_PREF(req_queue->domain), M_ZERO | M_WAITOK); + TAILQ_INIT(&hwq->free_tr); + TAILQ_INIT(&hwq->outstanding_tr); + for (i = 0; i < req_queue->num_trackers; i++) { tr = malloc_domainset(sizeof(struct ufshci_tracker), M_UFSHCI, DOMAINSET_PREF(req_queue->domain), M_ZERO | M_WAITOK); tr->req_queue = req_queue; tr->slot_num = i; tr->slot_state = UFSHCI_SLOT_STATE_FREE; + TAILQ_INSERT_HEAD(&hwq->free_tr, tr, tailq); hwq->act_tr[i] = tr; } if (is_task_mgmt) { /* UTP Task Management Request (UTMR) */ uint32_t utmrlba, utmrlbau; hwq->utmrd = (struct ufshci_utp_task_mgmt_req_desc *)queuemem; utmrlba = hwq->req_queue_addr & 0xffffffff; utmrlbau = hwq->req_queue_addr >> 32; ufshci_mmio_write_4(ctrlr, utmrlba, utmrlba); ufshci_mmio_write_4(ctrlr, utmrlbau, utmrlbau); } else { /* UTP Transfer Request (UTR) */ uint32_t utrlba, utrlbau; hwq->utrd = (struct ufshci_utp_xfer_req_desc *)queuemem; /* * Allocate physical memory for the command descriptor. * UTP Transfer Request (UTR) requires memory for a separate * command in addition to the queue. */ if (ufshci_req_sdb_cmd_desc_construct(req_queue, num_entries, ctrlr) != 0) { ufshci_printf(ctrlr, "failed to construct cmd descriptor memory\n"); - bus_dmamem_free(hwq->dma_tag_queue, hwq->utrd, - hwq->queuemem_map); goto out; } utrlba = hwq->req_queue_addr & 0xffffffff; utrlbau = hwq->req_queue_addr >> 32; ufshci_mmio_write_4(ctrlr, utrlba, utrlba); ufshci_mmio_write_4(ctrlr, utrlbau, utrlbau); } return (0); out: ufshci_req_sdb_destroy(ctrlr, req_queue); return (ENOMEM); } void ufshci_req_sdb_destroy(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue) { struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q]; struct ufshci_tracker *tr; int i; + mtx_lock(&hwq->recovery_lock); + hwq->timer_armed = false; + mtx_unlock(&hwq->recovery_lock); + callout_drain(&hwq->timer); + if (!req_queue->is_task_mgmt) ufshci_req_sdb_cmd_desc_destroy(&ctrlr->transfer_req_queue); for (i = 0; i < req_queue->num_trackers; i++) { tr = hwq->act_tr[i]; free(tr, M_UFSHCI); } if (hwq->act_tr) { free(hwq->act_tr, M_UFSHCI); hwq->act_tr = NULL; } if (hwq->utrd != NULL) { bus_dmamap_unload(hwq->dma_tag_queue, hwq->queuemem_map); bus_dmamem_free(hwq->dma_tag_queue, hwq->utrd, hwq->queuemem_map); hwq->utrd = NULL; } if (hwq->dma_tag_queue) { bus_dma_tag_destroy(hwq->dma_tag_queue); hwq->dma_tag_queue = NULL; } + if (mtx_initialized(&hwq->recovery_lock)) + mtx_destroy(&hwq->recovery_lock); if (mtx_initialized(&hwq->qlock)) mtx_destroy(&hwq->qlock); - free(req_queue->hwq->ucd_bus_addr, M_UFSHCI); free(req_queue->hwq, M_UFSHCI); } struct ufshci_hw_queue * ufshci_req_sdb_get_hw_queue(struct ufshci_req_queue *req_queue) { return &req_queue->hwq[UFSHCI_SDB_Q]; } +void +ufshci_req_sdb_disable(struct ufshci_controller *ctrlr, + struct ufshci_req_queue *req_queue) +{ + struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q]; + struct ufshci_tracker *tr, *tr_temp; + + mtx_lock(&hwq->recovery_lock); + mtx_lock(&hwq->qlock); + + if (mtx_initialized(&hwq->recovery_lock)) + mtx_assert(&hwq->recovery_lock, MA_OWNED); + if (mtx_initialized(&hwq->qlock)) + mtx_assert(&hwq->qlock, MA_OWNED); + + hwq->recovery_state = RECOVERY_WAITING; + TAILQ_FOREACH_SAFE(tr, &hwq->outstanding_tr, tailq, tr_temp) { + tr->deadline = SBT_MAX; + } + + mtx_unlock(&hwq->qlock); + mtx_unlock(&hwq->recovery_lock); +} + int ufshci_req_sdb_enable(struct ufshci_controller *ctrlr, struct ufshci_req_queue *req_queue) { + struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q]; + if (req_queue->is_task_mgmt) { uint32_t hcs, utmrldbr, utmrlrsr; hcs = ufshci_mmio_read_4(ctrlr, hcs); if (!(hcs & UFSHCIM(UFSHCI_HCS_REG_UTMRLRDY))) { ufshci_printf(ctrlr, "UTP task management request list is not ready\n"); return (ENXIO); } utmrldbr = ufshci_mmio_read_4(ctrlr, utmrldbr); if (utmrldbr != 0) { ufshci_printf(ctrlr, "UTP task management request list door bell is not ready\n"); return (ENXIO); } utmrlrsr = UFSHCIM(UFSHCI_UTMRLRSR_REG_UTMRLRSR); ufshci_mmio_write_4(ctrlr, utmrlrsr, utmrlrsr); } else { uint32_t hcs, utrldbr, utrlcnr, utrlrsr; hcs = ufshci_mmio_read_4(ctrlr, hcs); if (!(hcs & UFSHCIM(UFSHCI_HCS_REG_UTRLRDY))) { ufshci_printf(ctrlr, "UTP transfer request list is not ready\n"); return (ENXIO); } utrldbr = ufshci_mmio_read_4(ctrlr, utrldbr); if (utrldbr != 0) { ufshci_printf(ctrlr, "UTP transfer request list door bell is not ready\n"); ufshci_printf(ctrlr, "Clear the UTP transfer request list door bell\n"); ufshci_mmio_write_4(ctrlr, utrldbr, utrldbr); } utrlcnr = ufshci_mmio_read_4(ctrlr, utrlcnr); if (utrlcnr != 0) { ufshci_printf(ctrlr, "UTP transfer request list notification is not ready\n"); ufshci_printf(ctrlr, "Clear the UTP transfer request list notification\n"); ufshci_mmio_write_4(ctrlr, utrlcnr, utrlcnr); } utrlrsr = UFSHCIM(UFSHCI_UTRLRSR_REG_UTRLRSR); ufshci_mmio_write_4(ctrlr, utrlrsr, utrlrsr); } + if (mtx_initialized(&hwq->recovery_lock)) + mtx_assert(&hwq->recovery_lock, MA_OWNED); + if (mtx_initialized(&hwq->qlock)) + mtx_assert(&hwq->qlock, MA_OWNED); + KASSERT(!req_queue->ctrlr->is_failed, ("Enabling a failed hwq\n")); + + hwq->recovery_state = RECOVERY_NONE; + return (0); } int ufshci_req_sdb_reserve_slot(struct ufshci_req_queue *req_queue, struct ufshci_tracker **tr) { struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q]; uint8_t i; for (i = 0; i < req_queue->num_entries; i++) { if (hwq->act_tr[i]->slot_state == UFSHCI_SLOT_STATE_FREE) { *tr = hwq->act_tr[i]; (*tr)->hwq = hwq; return (0); } } return (EBUSY); } void ufshci_req_sdb_utmr_clear_cpl_ntf(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr) { /* * NOP * UTP Task Management does not have a Completion Notification * Register. */ } void ufshci_req_sdb_utr_clear_cpl_ntf(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr) { uint32_t utrlcnr; utrlcnr = 1 << tr->slot_num; ufshci_mmio_write_4(ctrlr, utrlcnr, utrlcnr); } void ufshci_req_sdb_utmr_ring_doorbell(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr) { uint32_t utmrldbr = 0; utmrldbr |= 1 << tr->slot_num; ufshci_mmio_write_4(ctrlr, utmrldbr, utmrldbr); tr->req_queue->hwq[UFSHCI_SDB_Q].num_cmds++; } void ufshci_req_sdb_utr_ring_doorbell(struct ufshci_controller *ctrlr, struct ufshci_tracker *tr) { uint32_t utrldbr = 0; utrldbr |= 1 << tr->slot_num; ufshci_mmio_write_4(ctrlr, utrldbr, utrldbr); tr->req_queue->hwq[UFSHCI_SDB_Q].num_cmds++; } bool ufshci_req_sdb_utmr_is_doorbell_cleared(struct ufshci_controller *ctrlr, uint8_t slot) { uint32_t utmrldbr; utmrldbr = ufshci_mmio_read_4(ctrlr, utmrldbr); return (!(utmrldbr & (1 << slot))); } bool ufshci_req_sdb_utr_is_doorbell_cleared(struct ufshci_controller *ctrlr, uint8_t slot) { uint32_t utrldbr; utrldbr = ufshci_mmio_read_4(ctrlr, utrldbr); return (!(utrldbr & (1 << slot))); } bool ufshci_req_sdb_process_cpl(struct ufshci_req_queue *req_queue) { struct ufshci_hw_queue *hwq = &req_queue->hwq[UFSHCI_SDB_Q]; struct ufshci_tracker *tr; uint8_t slot; bool done = false; + mtx_assert(&hwq->recovery_lock, MA_OWNED); + hwq->num_intr_handler_calls++; bus_dmamap_sync(hwq->dma_tag_queue, hwq->queuemem_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); for (slot = 0; slot < req_queue->num_entries; slot++) { tr = hwq->act_tr[slot]; KASSERT(tr, ("there is no tracker assigned to the slot")); /* * When the response is delivered from the device, the doorbell * is cleared. */ if (tr->slot_state == UFSHCI_SLOT_STATE_SCHEDULED && req_queue->qops.is_doorbell_cleared(req_queue->ctrlr, slot)) { ufshci_req_queue_complete_tracker(tr); done = true; } } return (done); } int ufshci_req_sdb_get_inflight_io(struct ufshci_controller *ctrlr) { /* TODO: Implement inflight io*/ return (0); } diff --git a/sys/dev/ufshci/ufshci_sim.c b/sys/dev/ufshci/ufshci_sim.c index db24561f4169..828b520614a5 100644 --- a/sys/dev/ufshci/ufshci_sim.c +++ b/sys/dev/ufshci/ufshci_sim.c @@ -1,372 +1,371 @@ /*- * Copyright (c) 2025, Samsung Electronics Co., Ltd. * Written by Jaeyoon Choi * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include #include #include #include #include "ufshci_private.h" #define sim2ctrlr(sim) ((struct ufshci_controller *)cam_sim_softc(sim)) static void ufshci_sim_scsiio_done(void *ccb_arg, const struct ufshci_completion *cpl, bool error) { const uint8_t *sense_data; uint16_t sense_data_max_size; uint16_t sense_data_len; union ccb *ccb = (union ccb *)ccb_arg; /* * Let the periph know the completion, and let it sort out what * it means. Report an error or success based on OCS and UPIU * response code. And We need to copy the sense data to be handled * by the CAM. */ sense_data = cpl->response_upiu.cmd_response_upiu.sense_data; sense_data_max_size = sizeof( cpl->response_upiu.cmd_response_upiu.sense_data); sense_data_len = be16toh( cpl->response_upiu.cmd_response_upiu.sense_data_len); memcpy(&ccb->csio.sense_data, sense_data, min(sense_data_len, sense_data_max_size)); ccb->ccb_h.status &= ~CAM_SIM_QUEUED; if (error) { ccb->ccb_h.status = CAM_REQ_CMP_ERR; xpt_done(ccb); } else { ccb->ccb_h.status = CAM_REQ_CMP; xpt_done_direct(ccb); } } /* * Complete the command as an illegal command with invalid field */ static void ufshci_sim_illegal_request(union ccb *ccb) { scsi_set_sense_data(&ccb->csio.sense_data, /*sense_format*/ SSD_TYPE_NONE, /*current_error*/ 1, /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST, /*asc*/ 0x24, /* 24h/00h INVALID FIELD IN CDB */ /*ascq*/ 0x00, /*extra args*/ SSD_ELEM_NONE); ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID | CAM_DEV_QFRZN; xpt_freeze_devq(ccb->ccb_h.path, 1); xpt_done(ccb); } static void ufshchi_sim_scsiio(struct cam_sim *sim, union ccb *ccb) { struct ccb_scsiio *csio = &ccb->csio; struct ufshci_request *req; void *payload; struct ufshci_cmd_command_upiu *upiu; uint8_t *cdb; uint32_t payload_len; bool is_write; struct ufshci_controller *ctrlr; uint8_t data_direction; int error; /* UFS device cannot process these commands */ if (csio->cdb_io.cdb_bytes[0] == MODE_SENSE_6 || csio->cdb_io.cdb_bytes[0] == MODE_SELECT_6 || csio->cdb_io.cdb_bytes[0] == READ_12 || csio->cdb_io.cdb_bytes[0] == WRITE_12) { ufshci_sim_illegal_request(ccb); return; } ctrlr = sim2ctrlr(sim); payload = csio->data_ptr; payload_len = csio->dxfer_len; is_write = csio->ccb_h.flags & CAM_DIR_OUT; /* TODO: Check other data type */ if ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO) req = ufshci_allocate_request_bio((struct bio *)payload, M_NOWAIT, ufshci_sim_scsiio_done, ccb); else req = ufshci_allocate_request_vaddr(payload, payload_len, M_NOWAIT, ufshci_sim_scsiio_done, ccb); req->request_size = sizeof(struct ufshci_cmd_command_upiu); req->response_size = sizeof(struct ufshci_cmd_response_upiu); switch (ccb->ccb_h.flags & CAM_DIR_MASK) { case CAM_DIR_IN: data_direction = UFSHCI_DATA_DIRECTION_FROM_TGT_TO_SYS; break; case CAM_DIR_OUT: data_direction = UFSHCI_DATA_DIRECTION_FROM_SYS_TO_TGT; break; default: data_direction = UFSHCI_DATA_DIRECTION_NO_DATA_TRANSFER; } req->data_direction = data_direction; upiu = (struct ufshci_cmd_command_upiu *)&req->request_upiu; memset(upiu, 0, req->request_size); upiu->header.trans_type = UFSHCI_UPIU_TRANSACTION_CODE_COMMAND; upiu->header.operational_flags = is_write ? UFSHCI_OPERATIONAL_FLAG_W : UFSHCI_OPERATIONAL_FLAG_R; upiu->header.lun = csio->ccb_h.target_lun; upiu->header.cmd_set_type = UFSHCI_COMMAND_SET_TYPE_SCSI; upiu->expected_data_transfer_length = htobe32(payload_len); ccb->ccb_h.status |= CAM_SIM_QUEUED; if (csio->ccb_h.flags & CAM_CDB_POINTER) cdb = csio->cdb_io.cdb_ptr; else cdb = csio->cdb_io.cdb_bytes; if (cdb == NULL || csio->cdb_len > sizeof(upiu->cdb)) { ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); return; } memcpy(upiu->cdb, cdb, csio->cdb_len); error = ufshci_ctrlr_submit_io_request(ctrlr, req); if (error == EBUSY) { ccb->ccb_h.status = CAM_SCSI_BUSY; xpt_done(ccb); return; } else if (error) { ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); return; } } static uint32_t ufshci_link_kBps(struct ufshci_controller *ctrlr) { uint32_t gear = ctrlr->hs_gear; uint32_t lanes = ctrlr->rx_lanes; /* * per-lane effective bandwidth (KB/s, SI 1 KB = 1000 B) * All HS-Gears use 8b/10b line coding, i.e. 80 % efficiency. * - KB/s per lane = raw-rate(Gbps) × 0.8(8b/10b) / 8(bit) */ static const uint32_t kbps_per_lane[] = { 0, /* unused */ 145920, /* HS-Gear1 : 1459.2 Mbps */ 291840, /* HS-Gear2 : 2918.4 Mbps */ 583680, /* HS-Gear3 : 5836.8 Mbps */ 1167360, /* HS-Gear4 : 11673.6 Mbps */ 2334720 /* HS-Gear5 : 23347.2 Mbps */ }; /* Sanity checks */ if (gear >= nitems(kbps_per_lane)) gear = 0; /* out-of-range -> treat as invalid */ if (lanes == 0 || lanes > 2) lanes = 1; /* UFS spec allows 1–2 data lanes */ return kbps_per_lane[gear] * lanes; } static void ufshci_cam_action(struct cam_sim *sim, union ccb *ccb) { struct ufshci_controller *ctrlr = sim2ctrlr(sim); if (ctrlr == NULL) { ccb->ccb_h.status = CAM_SEL_TIMEOUT; xpt_done(ccb); return; } /* Perform the requested action */ switch (ccb->ccb_h.func_code) { case XPT_SCSI_IO: ufshchi_sim_scsiio(sim, ccb); return; case XPT_PATH_INQ: { struct ccb_pathinq *cpi = &ccb->cpi; cpi->version_num = 1; cpi->hba_inquiry = PI_SDTR_ABLE | PI_TAG_ABLE; cpi->target_sprt = 0; cpi->hba_misc = PIM_UNMAPPED | PIM_NO_6_BYTE; cpi->hba_eng_cnt = 0; cpi->max_target = 0; cpi->max_lun = ctrlr->max_lun_count; cpi->async_flags = 0; cpi->maxio = ctrlr->max_xfer_size; cpi->initiator_id = 1; strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "UFSHCI", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->base_transfer_speed = ufshci_link_kBps(ctrlr); cpi->transport = XPORT_UFSHCI; cpi->transport_version = 1; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_SPC5; ccb->ccb_h.status = CAM_REQ_CMP; break; } case XPT_RESET_BUS: ccb->ccb_h.status = CAM_REQ_CMP; break; case XPT_RESET_DEV: if (ufshci_dev_reset(ctrlr)) ccb->ccb_h.status = CAM_REQ_CMP_ERR; else ccb->ccb_h.status = CAM_REQ_CMP; break; case XPT_ABORT: - /* TODO: Implement Task Management CMD*/ ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; break; case XPT_SET_TRAN_SETTINGS: ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; break; case XPT_GET_TRAN_SETTINGS: { struct ccb_trans_settings *cts; struct ccb_trans_settings_ufshci *ufshcix; cts = &ccb->cts; ufshcix = &cts->xport_specific.ufshci; ufshcix->hs_gear = ctrlr->hs_gear; ufshcix->tx_lanes = ctrlr->tx_lanes; ufshcix->rx_lanes = ctrlr->rx_lanes; ufshcix->max_hs_gear = ctrlr->max_rx_hs_gear; ufshcix->max_tx_lanes = ctrlr->max_tx_lanes; ufshcix->max_rx_lanes = ctrlr->max_rx_lanes; ufshcix->valid = CTS_UFSHCI_VALID_LINK; cts->transport = XPORT_UFSHCI; cts->transport_version = 1; cts->protocol = PROTO_SCSI; cts->protocol_version = SCSI_REV_SPC5; ccb->ccb_h.status = CAM_REQ_CMP; break; } case XPT_CALC_GEOMETRY: cam_calc_geometry(&ccb->ccg, 1); break; case XPT_NOOP: ccb->ccb_h.status = CAM_REQ_CMP; break; default: printf("invalid ccb=%p func=%#x\n", ccb, ccb->ccb_h.func_code); break; } xpt_done(ccb); return; } static void ufshci_cam_poll(struct cam_sim *sim) { struct ufshci_controller *ctrlr = sim2ctrlr(sim); ufshci_ctrlr_poll(ctrlr); } int ufshci_sim_attach(struct ufshci_controller *ctrlr) { device_t dev; struct cam_devq *devq; int max_trans; dev = ctrlr->dev; max_trans = ctrlr->max_hw_pend_io; if ((devq = cam_simq_alloc(max_trans)) == NULL) { printf("Failed to allocate a simq\n"); return (ENOMEM); } ctrlr->ufshci_sim = cam_sim_alloc(ufshci_cam_action, ufshci_cam_poll, "ufshci", ctrlr, device_get_unit(dev), &ctrlr->sc_mtx, max_trans, max_trans, devq); if (ctrlr->ufshci_sim == NULL) { printf("Failed to allocate a sim\n"); cam_simq_free(devq); return (ENOMEM); } mtx_lock(&ctrlr->sc_mtx); if (xpt_bus_register(ctrlr->ufshci_sim, ctrlr->dev, 0) != CAM_SUCCESS) { cam_sim_free(ctrlr->ufshci_sim, /*free_devq*/ TRUE); cam_simq_free(devq); mtx_unlock(&ctrlr->sc_mtx); printf("Failed to create a bus\n"); return (ENOMEM); } if (xpt_create_path(&ctrlr->ufshci_path, /*periph*/ NULL, cam_sim_path(ctrlr->ufshci_sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { xpt_bus_deregister(cam_sim_path(ctrlr->ufshci_sim)); cam_sim_free(ctrlr->ufshci_sim, /*free_devq*/ TRUE); cam_simq_free(devq); mtx_unlock(&ctrlr->sc_mtx); printf("Failed to create a path\n"); return (ENOMEM); } mtx_unlock(&ctrlr->sc_mtx); return (0); } void ufshci_sim_detach(struct ufshci_controller *ctrlr) { int error; if (ctrlr->ufshci_path != NULL) { xpt_free_path(ctrlr->ufshci_path); ctrlr->ufshci_path = NULL; } if (ctrlr->ufshci_sim != NULL) { error = xpt_bus_deregister(cam_sim_path(ctrlr->ufshci_sim)); if (error == 0) { /* accessing the softc is not possible after this */ ctrlr->ufshci_sim->softc = NULL; ufshci_printf(ctrlr, "%s: %s:%d:%d caling " "cam_sim_free sim %p refc %u mtx %p\n", __func__, ctrlr->sc_name, cam_sim_path(ctrlr->ufshci_sim), ctrlr->sc_unit, ctrlr->ufshci_sim, ctrlr->ufshci_sim->refcount, ctrlr->ufshci_sim->mtx); } else { panic("%s: %s: CAM layer is busy: errno %d\n", __func__, ctrlr->sc_name, error); } cam_sim_free(ctrlr->ufshci_sim, /* free_devq */ TRUE); ctrlr->ufshci_sim = NULL; } }