Index: stable/12/sys/cam/nvme/nvme_xpt.c =================================================================== --- stable/12/sys/cam/nvme/nvme_xpt.c (revision 350718) +++ stable/12/sys/cam/nvme/nvme_xpt.c (revision 350719) @@ -1,781 +1,788 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2015 Netflix, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * derived from ata_xpt.c: Copyright (c) 2009 Alexander Motin */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* for xpt_print below */ #include "opt_cam.h" struct nvme_quirk_entry { u_int quirks; #define CAM_QUIRK_MAXTAGS 1 u_int mintags; u_int maxtags; }; /* Not even sure why we need this */ static periph_init_t nvme_probe_periph_init; static struct periph_driver nvme_probe_driver = { nvme_probe_periph_init, "nvme_probe", TAILQ_HEAD_INITIALIZER(nvme_probe_driver.units), /* generation */ 0, CAM_PERIPH_DRV_EARLY }; PERIPHDRIVER_DECLARE(nvme_probe, nvme_probe_driver); typedef enum { NVME_PROBE_IDENTIFY_CD, NVME_PROBE_IDENTIFY_NS, NVME_PROBE_DONE, NVME_PROBE_INVALID } nvme_probe_action; static char *nvme_probe_action_text[] = { "NVME_PROBE_IDENTIFY_CD", "NVME_PROBE_IDENTIFY_NS", "NVME_PROBE_DONE", "NVME_PROBE_INVALID" }; #define NVME_PROBE_SET_ACTION(softc, newaction) \ do { \ char **text; \ text = nvme_probe_action_text; \ CAM_DEBUG((softc)->periph->path, CAM_DEBUG_PROBE, \ ("Probe %s to %s\n", text[(softc)->action], \ text[(newaction)])); \ (softc)->action = (newaction); \ } while(0) typedef enum { NVME_PROBE_NO_ANNOUNCE = 0x04 } nvme_probe_flags; typedef struct { TAILQ_HEAD(, ccb_hdr) request_ccbs; union { struct nvme_controller_data cd; struct nvme_namespace_data ns; }; nvme_probe_action action; nvme_probe_flags flags; int restart; struct cam_periph *periph; } nvme_probe_softc; static struct nvme_quirk_entry nvme_quirk_table[] = { { // { // T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED, // /*vendor*/"*", /*product*/"*", /*revision*/"*" // }, .quirks = 0, .mintags = 0, .maxtags = 0 }, }; static const int nvme_quirk_table_size = sizeof(nvme_quirk_table) / sizeof(*nvme_quirk_table); static cam_status nvme_probe_register(struct cam_periph *periph, void *arg); static void nvme_probe_schedule(struct cam_periph *nvme_probe_periph); static void nvme_probe_start(struct cam_periph *periph, union ccb *start_ccb); static void nvme_probe_done(struct cam_periph *periph, union ccb *done_ccb); static void nvme_probe_cleanup(struct cam_periph *periph); //static void nvme_find_quirk(struct cam_ed *device); static void nvme_scan_lun(struct cam_periph *periph, struct cam_path *path, cam_flags flags, union ccb *ccb); static struct cam_ed * nvme_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id); static void nvme_device_transport(struct cam_path *path); static void nvme_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target, struct cam_ed *device, void *async_arg); static void nvme_action(union ccb *start_ccb); static void nvme_announce_periph(struct cam_periph *periph); static void nvme_proto_announce(struct cam_ed *device); static void nvme_proto_denounce(struct cam_ed *device); static void nvme_proto_debug_out(union ccb *ccb); static struct xpt_xport_ops nvme_xport_ops = { .alloc_device = nvme_alloc_device, .action = nvme_action, .async = nvme_dev_async, .announce = nvme_announce_periph, }; #define NVME_XPT_XPORT(x, X) \ static struct xpt_xport nvme_xport_ ## x = { \ .xport = XPORT_ ## X, \ .name = #x, \ .ops = &nvme_xport_ops, \ }; \ CAM_XPT_XPORT(nvme_xport_ ## x); NVME_XPT_XPORT(nvme, NVME); #undef NVME_XPT_XPORT static struct xpt_proto_ops nvme_proto_ops = { .announce = nvme_proto_announce, .denounce = nvme_proto_denounce, .debug_out = nvme_proto_debug_out, }; static struct xpt_proto nvme_proto = { .proto = PROTO_NVME, .name = "nvme", .ops = &nvme_proto_ops, }; CAM_XPT_PROTO(nvme_proto); static void nvme_probe_periph_init() { } static cam_status nvme_probe_register(struct cam_periph *periph, void *arg) { union ccb *request_ccb; /* CCB representing the probe request */ nvme_probe_softc *softc; request_ccb = (union ccb *)arg; if (request_ccb == NULL) { printf("nvme_probe_register: no probe CCB, " "can't register device\n"); return(CAM_REQ_CMP_ERR); } softc = (nvme_probe_softc *)malloc(sizeof(*softc), M_CAMXPT, M_ZERO | M_NOWAIT); if (softc == NULL) { printf("nvme_probe_register: Unable to probe new device. " "Unable to allocate softc\n"); return(CAM_REQ_CMP_ERR); } TAILQ_INIT(&softc->request_ccbs); TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h, periph_links.tqe); softc->flags = 0; periph->softc = softc; softc->periph = periph; softc->action = NVME_PROBE_INVALID; if (cam_periph_acquire(periph) != 0) return (CAM_REQ_CMP_ERR); CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n")); // nvme_device_transport(periph->path); nvme_probe_schedule(periph); return(CAM_REQ_CMP); } static void nvme_probe_schedule(struct cam_periph *periph) { union ccb *ccb; nvme_probe_softc *softc; softc = (nvme_probe_softc *)periph->softc; ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs); NVME_PROBE_SET_ACTION(softc, NVME_PROBE_IDENTIFY_CD); if (ccb->crcn.flags & CAM_EXPECT_INQ_CHANGE) softc->flags |= NVME_PROBE_NO_ANNOUNCE; else softc->flags &= ~NVME_PROBE_NO_ANNOUNCE; xpt_schedule(periph, CAM_PRIORITY_XPT); } static void nvme_probe_start(struct cam_periph *periph, union ccb *start_ccb) { struct ccb_nvmeio *nvmeio; nvme_probe_softc *softc; struct cam_path *path; lun_id_t lun; CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("nvme_probe_start\n")); softc = (nvme_probe_softc *)periph->softc; path = start_ccb->ccb_h.path; nvmeio = &start_ccb->nvmeio; lun = xpt_path_lun_id(periph->path); if (softc->restart) { softc->restart = 0; NVME_PROBE_SET_ACTION(softc, NVME_PROBE_IDENTIFY_CD); } switch (softc->action) { case NVME_PROBE_IDENTIFY_CD: cam_fill_nvmeadmin(nvmeio, 0, /* retries */ nvme_probe_done, /* cbfcnp */ CAM_DIR_IN, /* flags */ (uint8_t *)&softc->cd, /* data_ptr */ sizeof(softc->cd), /* dxfer_len */ 30 * 1000); /* timeout 30s */ nvme_ns_cmd(nvmeio, NVME_OPC_IDENTIFY, 0, 1, 0, 0, 0, 0, 0); break; case NVME_PROBE_IDENTIFY_NS: cam_fill_nvmeadmin(nvmeio, 0, /* retries */ nvme_probe_done, /* cbfcnp */ CAM_DIR_IN, /* flags */ (uint8_t *)&softc->ns, /* data_ptr */ sizeof(softc->ns), /* dxfer_len */ 30 * 1000); /* timeout 30s */ nvme_ns_cmd(nvmeio, NVME_OPC_IDENTIFY, lun, 0, 0, 0, 0, 0, 0); break; default: panic("nvme_probe_start: invalid action state 0x%x\n", softc->action); } start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE; xpt_action(start_ccb); } static void nvme_probe_done(struct cam_periph *periph, union ccb *done_ccb) { struct nvme_namespace_data *nvme_data; struct nvme_controller_data *nvme_cdata; nvme_probe_softc *softc; struct cam_path *path; cam_status status; u_int32_t priority; int found = 1; CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("nvme_probe_done\n")); softc = (nvme_probe_softc *)periph->softc; path = done_ccb->ccb_h.path; priority = done_ccb->ccb_h.pinfo.priority; if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { if (cam_periph_error(done_ccb, 0, softc->restart ? (SF_NO_RECOVERY | SF_NO_RETRY) : 0 ) == ERESTART) { out: /* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */ cam_release_devq(path, 0, 0, 0, FALSE); return; } if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge the queue */ xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE); } status = done_ccb->ccb_h.status & CAM_STATUS_MASK; /* * If we get to this point, we got an error status back * from the inquiry and the error status doesn't require * automatically retrying the command. Therefore, the * inquiry failed. If we had inquiry information before * for this device, but this latest inquiry command failed, * the device has probably gone away. If this device isn't * already marked unconfigured, notify the peripheral * drivers that this device is no more. */ device_fail: if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0) xpt_async(AC_LOST_DEVICE, path, NULL); NVME_PROBE_SET_ACTION(softc, NVME_PROBE_INVALID); found = 0; goto done; } if (softc->restart) goto done; switch (softc->action) { case NVME_PROBE_IDENTIFY_CD: nvme_controller_data_swapbytes(&softc->cd); nvme_cdata = path->device->nvme_cdata; if (nvme_cdata == NULL) { nvme_cdata = malloc(sizeof(*nvme_cdata), M_CAMXPT, M_NOWAIT); if (nvme_cdata == NULL) { xpt_print(path, "Can't allocate memory"); goto device_fail; } } bcopy(&softc->cd, nvme_cdata, sizeof(*nvme_cdata)); path->device->nvme_cdata = nvme_cdata; // nvme_find_quirk(path->device); nvme_device_transport(path); NVME_PROBE_SET_ACTION(softc, NVME_PROBE_IDENTIFY_NS); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; case NVME_PROBE_IDENTIFY_NS: nvme_namespace_data_swapbytes(&softc->ns); /* Check that the namespace exists. */ if (softc->ns.nsze == 0) goto device_fail; nvme_data = path->device->nvme_data; if (nvme_data == NULL) { nvme_data = malloc(sizeof(*nvme_data), M_CAMXPT, M_NOWAIT); if (nvme_data == NULL) { xpt_print(path, "Can't allocate memory"); goto device_fail; } } bcopy(&softc->ns, nvme_data, sizeof(*nvme_data)); path->device->nvme_data = nvme_data; if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) { path->device->flags &= ~CAM_DEV_UNCONFIGURED; xpt_acquire_device(path->device); done_ccb->ccb_h.func_code = XPT_GDEV_TYPE; xpt_action(done_ccb); xpt_async(AC_FOUND_DEVICE, path, done_ccb); } NVME_PROBE_SET_ACTION(softc, NVME_PROBE_DONE); break; default: panic("nvme_probe_done: invalid action state 0x%x\n", softc->action); } done: if (softc->restart) { softc->restart = 0; xpt_release_ccb(done_ccb); nvme_probe_schedule(periph); goto out; } xpt_release_ccb(done_ccb); CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe completed\n")); while ((done_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs))) { TAILQ_REMOVE(&softc->request_ccbs, &done_ccb->ccb_h, periph_links.tqe); done_ccb->ccb_h.status = found ? CAM_REQ_CMP : CAM_REQ_CMP_ERR; xpt_done(done_ccb); } /* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */ cam_release_devq(path, 0, 0, 0, FALSE); cam_periph_invalidate(periph); cam_periph_release_locked(periph); } static void nvme_probe_cleanup(struct cam_periph *periph) { free(periph->softc, M_CAMXPT); } #if 0 /* XXX should be used, don't delete */ static void nvme_find_quirk(struct cam_ed *device) { struct nvme_quirk_entry *quirk; caddr_t match; match = cam_quirkmatch((caddr_t)&device->nvme_data, (caddr_t)nvme_quirk_table, nvme_quirk_table_size, sizeof(*nvme_quirk_table), nvme_identify_match); if (match == NULL) panic("xpt_find_quirk: device didn't match wildcard entry!!"); quirk = (struct nvme_quirk_entry *)match; device->quirk = quirk; if (quirk->quirks & CAM_QUIRK_MAXTAGS) { device->mintags = quirk->mintags; device->maxtags = quirk->maxtags; } } #endif static void nvme_scan_lun(struct cam_periph *periph, struct cam_path *path, cam_flags flags, union ccb *request_ccb) { struct ccb_pathinq cpi; cam_status status; struct cam_periph *old_periph; int lock; CAM_DEBUG(path, CAM_DEBUG_TRACE, ("nvme_scan_lun\n")); xpt_path_inq(&cpi, path); if (cpi.ccb_h.status != CAM_REQ_CMP) { if (request_ccb != NULL) { request_ccb->ccb_h.status = cpi.ccb_h.status; xpt_done(request_ccb); } return; } if (xpt_path_lun_id(path) == CAM_LUN_WILDCARD) { CAM_DEBUG(path, CAM_DEBUG_TRACE, ("nvme_scan_lun ignoring bus\n")); request_ccb->ccb_h.status = CAM_REQ_CMP; /* XXX signal error ? */ xpt_done(request_ccb); return; } lock = (xpt_path_owned(path) == 0); if (lock) xpt_path_lock(path); if ((old_periph = cam_periph_find(path, "nvme_probe")) != NULL) { if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) { nvme_probe_softc *softc; softc = (nvme_probe_softc *)old_periph->softc; TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h, periph_links.tqe); softc->restart = 1; CAM_DEBUG(path, CAM_DEBUG_TRACE, ("restarting nvme_probe device\n")); } else { request_ccb->ccb_h.status = CAM_REQ_CMP_ERR; CAM_DEBUG(path, CAM_DEBUG_TRACE, ("Failing to restart nvme_probe device\n")); xpt_done(request_ccb); } } else { CAM_DEBUG(path, CAM_DEBUG_TRACE, ("Adding nvme_probe device\n")); status = cam_periph_alloc(nvme_probe_register, NULL, nvme_probe_cleanup, nvme_probe_start, "nvme_probe", CAM_PERIPH_BIO, request_ccb->ccb_h.path, NULL, 0, request_ccb); if (status != CAM_REQ_CMP) { xpt_print(path, "xpt_scan_lun: cam_alloc_periph " "returned an error, can't continue probe\n"); request_ccb->ccb_h.status = status; xpt_done(request_ccb); } } if (lock) xpt_path_unlock(path); } static struct cam_ed * nvme_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) { struct nvme_quirk_entry *quirk; struct cam_ed *device; device = xpt_alloc_device(bus, target, lun_id); if (device == NULL) return (NULL); /* * Take the default quirk entry until we have inquiry * data from nvme and can determine a better quirk to use. */ quirk = &nvme_quirk_table[nvme_quirk_table_size - 1]; device->quirk = (void *)quirk; device->mintags = 0; device->maxtags = 0; device->inq_flags = 0; device->queue_flags = 0; device->device_id = NULL; /* XXX Need to set this somewhere */ device->device_id_len = 0; device->serial_num = NULL; /* XXX Need to set this somewhere */ device->serial_num_len = 0; return (device); } static void nvme_device_transport(struct cam_path *path) { struct ccb_pathinq cpi; struct ccb_trans_settings cts; /* XXX get data from nvme namespace and other info ??? */ /* Get transport information from the SIM */ xpt_path_inq(&cpi, path); path->device->transport = cpi.transport; path->device->transport_version = cpi.transport_version; path->device->protocol = cpi.protocol; path->device->protocol_version = cpi.protocol_version; /* Tell the controller what we think */ xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; cts.transport = path->device->transport; cts.transport_version = path->device->transport_version; cts.protocol = path->device->protocol; cts.protocol_version = path->device->protocol_version; cts.proto_specific.valid = 0; cts.xport_specific.valid = 0; xpt_action((union ccb *)&cts); } static void nvme_dev_advinfo(union ccb *start_ccb) { struct cam_ed *device; struct ccb_dev_advinfo *cdai; off_t amt; start_ccb->ccb_h.status = CAM_REQ_INVALID; device = start_ccb->ccb_h.path->device; cdai = &start_ccb->cdai; switch(cdai->buftype) { case CDAI_TYPE_SCSI_DEVID: if (cdai->flags & CDAI_FLAG_STORE) return; cdai->provsiz = device->device_id_len; if (device->device_id_len == 0) break; amt = device->device_id_len; if (cdai->provsiz > cdai->bufsiz) amt = cdai->bufsiz; memcpy(cdai->buf, device->device_id, amt); break; case CDAI_TYPE_SERIAL_NUM: if (cdai->flags & CDAI_FLAG_STORE) return; cdai->provsiz = device->serial_num_len; if (device->serial_num_len == 0) break; amt = device->serial_num_len; if (cdai->provsiz > cdai->bufsiz) amt = cdai->bufsiz; memcpy(cdai->buf, device->serial_num, amt); break; case CDAI_TYPE_PHYS_PATH: if (cdai->flags & CDAI_FLAG_STORE) { if (device->physpath != NULL) free(device->physpath, M_CAMXPT); device->physpath_len = cdai->bufsiz; /* Clear existing buffer if zero length */ if (cdai->bufsiz == 0) break; device->physpath = malloc(cdai->bufsiz, M_CAMXPT, M_NOWAIT); if (device->physpath == NULL) { start_ccb->ccb_h.status = CAM_REQ_ABORTED; return; } memcpy(device->physpath, cdai->buf, cdai->bufsiz); } else { cdai->provsiz = device->physpath_len; if (device->physpath_len == 0) break; amt = device->physpath_len; if (cdai->provsiz > cdai->bufsiz) amt = cdai->bufsiz; memcpy(cdai->buf, device->physpath, amt); } break; case CDAI_TYPE_NVME_CNTRL: if (cdai->flags & CDAI_FLAG_STORE) return; amt = sizeof(struct nvme_controller_data); cdai->provsiz = amt; if (amt > cdai->bufsiz) amt = cdai->bufsiz; memcpy(cdai->buf, device->nvme_cdata, amt); break; case CDAI_TYPE_NVME_NS: if (cdai->flags & CDAI_FLAG_STORE) return; amt = sizeof(struct nvme_namespace_data); cdai->provsiz = amt; if (amt > cdai->bufsiz) amt = cdai->bufsiz; memcpy(cdai->buf, device->nvme_data, amt); break; default: return; } start_ccb->ccb_h.status = CAM_REQ_CMP; if (cdai->flags & CDAI_FLAG_STORE) { xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path, (void *)(uintptr_t)cdai->buftype); } } static void nvme_action(union ccb *start_ccb) { CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("nvme_action: func= %#x\n", start_ccb->ccb_h.func_code)); switch (start_ccb->ccb_h.func_code) { case XPT_SCAN_BUS: case XPT_SCAN_TGT: case XPT_SCAN_LUN: nvme_scan_lun(start_ccb->ccb_h.path->periph, start_ccb->ccb_h.path, start_ccb->crcn.flags, start_ccb); break; case XPT_DEV_ADVINFO: nvme_dev_advinfo(start_ccb); break; default: xpt_action_default(start_ccb); break; } } /* * Handle any per-device event notifications that require action by the XPT. */ static void nvme_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target, struct cam_ed *device, void *async_arg) { /* * We only need to handle events for real devices. */ if (target->target_id == CAM_TARGET_WILDCARD || device->lun_id == CAM_LUN_WILDCARD) return; if (async_code == AC_LOST_DEVICE && (device->flags & CAM_DEV_UNCONFIGURED) == 0) { device->flags |= CAM_DEV_UNCONFIGURED; xpt_release_device(device); } } static void nvme_announce_periph(struct cam_periph *periph) { struct ccb_pathinq cpi; struct ccb_trans_settings cts; struct cam_path *path = periph->path; struct ccb_trans_settings_nvme *nvmex; + struct sbuf sb; + char buffer[120]; cam_periph_assert(periph, MA_OWNED); /* Ask the SIM for connection details */ xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; xpt_action((union ccb*)&cts); if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) return; nvmex = &cts.xport_specific.nvme; /* Ask the SIM for its base transfer speed */ xpt_path_inq(&cpi, periph->path); - printf("%s%d: nvme version %d.%d x%d (max x%d) lanes PCIe Gen%d (max Gen%d) link", + sbuf_new(&sb, buffer, sizeof(buffer), SBUF_FIXEDLEN); + sbuf_printf(&sb, "%s%d: nvme version %d.%d", periph->periph_name, periph->unit_number, NVME_MAJOR(nvmex->spec), - NVME_MINOR(nvmex->spec), - nvmex->lanes, nvmex->max_lanes, - nvmex->speed, nvmex->max_speed); - printf("\n"); + NVME_MINOR(nvmex->spec)); + if (nvmex->valid & CTS_NVME_VALID_LINK) + sbuf_printf(&sb, " x%d (max x%d) lanes PCIe Gen%d (max Gen%d) link", + nvmex->lanes, nvmex->max_lanes, + nvmex->speed, nvmex->max_speed); + sbuf_printf(&sb, "\n"); + sbuf_finish(&sb); + sbuf_putbuf(&sb); } static void nvme_proto_announce(struct cam_ed *device) { struct sbuf sb; char buffer[120]; sbuf_new(&sb, buffer, sizeof(buffer), SBUF_FIXEDLEN); nvme_print_ident(device->nvme_cdata, device->nvme_data, &sb); sbuf_finish(&sb); sbuf_putbuf(&sb); } static void nvme_proto_denounce(struct cam_ed *device) { nvme_proto_announce(device); } static void nvme_proto_debug_out(union ccb *ccb) { char cdb_str[(sizeof(struct nvme_command) * 3) + 1]; if (ccb->ccb_h.func_code != XPT_NVME_IO && ccb->ccb_h.func_code != XPT_NVME_ADMIN) return; CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_CDB,("%s. NCB: %s\n", nvme_op_string(&ccb->nvmeio.cmd, ccb->ccb_h.func_code == XPT_NVME_ADMIN), nvme_cmd_string(&ccb->nvmeio.cmd, cdb_str, sizeof(cdb_str)))); } Index: stable/12/sys/dev/nvme/nvme_sim.c =================================================================== --- stable/12/sys/dev/nvme/nvme_sim.c (revision 350718) +++ stable/12/sys/dev/nvme/nvme_sim.c (revision 350719) @@ -1,379 +1,383 @@ /*- * Copyright (c) 2016 Netflix, Inc * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "nvme_private.h" #define ccb_accb_ptr spriv_ptr0 #define ccb_ctrlr_ptr spriv_ptr1 static void nvme_sim_action(struct cam_sim *sim, union ccb *ccb); static void nvme_sim_poll(struct cam_sim *sim); #define sim2softc(sim) ((struct nvme_sim_softc *)cam_sim_softc(sim)) #define sim2ctrlr(sim) (sim2softc(sim)->s_ctrlr) struct nvme_sim_softc { struct nvme_controller *s_ctrlr; struct cam_sim *s_sim; struct cam_path *s_path; }; static void nvme_sim_nvmeio_done(void *ccb_arg, const struct nvme_completion *cpl) { union ccb *ccb = (union ccb *)ccb_arg; /* * Let the periph know the completion, and let it sort out what * it means. Make our best guess, though for the status code. */ memcpy(&ccb->nvmeio.cpl, cpl, sizeof(*cpl)); ccb->ccb_h.status &= ~CAM_SIM_QUEUED; if (nvme_completion_is_error(cpl)) { ccb->ccb_h.status = CAM_REQ_CMP_ERR; xpt_done(ccb); } else { ccb->ccb_h.status = CAM_REQ_CMP; xpt_done_direct(ccb); } } static void nvme_sim_nvmeio(struct cam_sim *sim, union ccb *ccb) { struct ccb_nvmeio *nvmeio = &ccb->nvmeio; struct nvme_request *req; void *payload; uint32_t size; struct nvme_controller *ctrlr; ctrlr = sim2ctrlr(sim); payload = nvmeio->data_ptr; size = nvmeio->dxfer_len; /* SG LIST ??? */ if ((nvmeio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO) req = nvme_allocate_request_bio((struct bio *)payload, nvme_sim_nvmeio_done, ccb); else if ((nvmeio->ccb_h.flags & CAM_DATA_SG) == CAM_DATA_SG) req = nvme_allocate_request_ccb(ccb, nvme_sim_nvmeio_done, ccb); else if (payload == NULL) req = nvme_allocate_request_null(nvme_sim_nvmeio_done, ccb); else req = nvme_allocate_request_vaddr(payload, size, nvme_sim_nvmeio_done, ccb); if (req == NULL) { nvmeio->ccb_h.status = CAM_RESRC_UNAVAIL; xpt_done(ccb); return; } ccb->ccb_h.status |= CAM_SIM_QUEUED; memcpy(&req->cmd, &ccb->nvmeio.cmd, sizeof(ccb->nvmeio.cmd)); if (ccb->ccb_h.func_code == XPT_NVME_IO) nvme_ctrlr_submit_io_request(ctrlr, req); else nvme_ctrlr_submit_admin_request(ctrlr, req); } static uint32_t nvme_link_kBps(struct nvme_controller *ctrlr) { uint32_t speed, lanes, link[] = { 1, 250000, 500000, 985000, 1970000 }; uint32_t status; status = pcie_read_config(ctrlr->dev, PCIER_LINK_STA, 2); speed = status & PCIEM_LINK_STA_SPEED; lanes = (status & PCIEM_LINK_STA_WIDTH) >> 4; /* * Failsafe on link speed indicator. If it is insane report the number of * lanes as the speed. Not 100% accurate, but may be diagnostic. */ if (speed >= nitems(link)) speed = 0; return link[speed] * lanes; } static void nvme_sim_action(struct cam_sim *sim, union ccb *ccb) { struct nvme_controller *ctrlr; CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("nvme_sim_action: func= %#x\n", ccb->ccb_h.func_code)); ctrlr = sim2ctrlr(sim); switch (ccb->ccb_h.func_code) { case XPT_CALC_GEOMETRY: /* Calculate Geometry Totally nuts ? XXX */ /* * Only meaningful for old-school SCSI disks since only the SCSI * da driver generates them. Reject all these that slip through. */ /*FALLTHROUGH*/ case XPT_ABORT: /* Abort the specified CCB */ ccb->ccb_h.status = CAM_REQ_INVALID; break; case XPT_SET_TRAN_SETTINGS: /* * NVMe doesn't really have different transfer settings, but * other parts of CAM think failure here is a big deal. */ ccb->ccb_h.status = CAM_REQ_CMP; break; case XPT_PATH_INQ: /* Path routing inquiry */ { struct ccb_pathinq *cpi = &ccb->cpi; device_t dev = ctrlr->dev; /* * NVMe may have multiple LUNs on the same path. Current generation * of NVMe devives support only a single name space. Multiple name * space drives are coming, but it's unclear how we should report * them up the stack. */ cpi->version_num = 1; cpi->hba_inquiry = 0; cpi->target_sprt = 0; cpi->hba_misc = PIM_UNMAPPED | PIM_NOSCAN; cpi->hba_eng_cnt = 0; cpi->max_target = 0; cpi->max_lun = ctrlr->cdata.nn; cpi->maxio = ctrlr->max_xfer_size; cpi->initiator_id = 0; cpi->bus_id = cam_sim_bus(sim); cpi->base_transfer_speed = nvme_link_kBps(ctrlr); strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strlcpy(cpi->hba_vid, "NVMe", HBA_IDLEN); strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->transport = XPORT_NVME; /* XXX XPORT_PCIE ? */ cpi->transport_version = nvme_mmio_read_4(ctrlr, vs); cpi->protocol = PROTO_NVME; cpi->protocol_version = nvme_mmio_read_4(ctrlr, vs); cpi->xport_specific.nvme.nsid = xpt_path_lun_id(ccb->ccb_h.path); cpi->xport_specific.nvme.domain = pci_get_domain(dev); cpi->xport_specific.nvme.bus = pci_get_bus(dev); cpi->xport_specific.nvme.slot = pci_get_slot(dev); cpi->xport_specific.nvme.function = pci_get_function(dev); cpi->xport_specific.nvme.extra = 0; cpi->ccb_h.status = CAM_REQ_CMP; break; } case XPT_GET_TRAN_SETTINGS: /* Get transport settings */ { struct ccb_trans_settings *cts; struct ccb_trans_settings_nvme *nvmep; struct ccb_trans_settings_nvme *nvmex; device_t dev; - uint32_t status, caps; + uint32_t status, caps, flags; dev = ctrlr->dev; cts = &ccb->cts; nvmex = &cts->xport_specific.nvme; nvmep = &cts->proto_specific.nvme; status = pcie_read_config(dev, PCIER_LINK_STA, 2); caps = pcie_read_config(dev, PCIER_LINK_CAP, 2); - nvmex->valid = CTS_NVME_VALID_SPEC | CTS_NVME_VALID_LINK; + flags = pcie_read_config(dev, PCIER_FLAGS, 2); nvmex->spec = nvme_mmio_read_4(ctrlr, vs); - nvmex->speed = status & PCIEM_LINK_STA_SPEED; - nvmex->lanes = (status & PCIEM_LINK_STA_WIDTH) >> 4; - nvmex->max_speed = caps & PCIEM_LINK_CAP_MAX_SPEED; - nvmex->max_lanes = (caps & PCIEM_LINK_CAP_MAX_WIDTH) >> 4; + nvmex->valid = CTS_NVME_VALID_SPEC; + if ((flags & PCIEM_FLAGS_TYPE) == PCIEM_TYPE_ENDPOINT) { + nvmex->valid |= CTS_NVME_VALID_LINK; + nvmex->speed = status & PCIEM_LINK_STA_SPEED; + nvmex->lanes = (status & PCIEM_LINK_STA_WIDTH) >> 4; + nvmex->max_speed = caps & PCIEM_LINK_CAP_MAX_SPEED; + nvmex->max_lanes = (caps & PCIEM_LINK_CAP_MAX_WIDTH) >> 4; + } /* XXX these should be something else maybe ? */ nvmep->valid = 1; nvmep->spec = nvmex->spec; cts->transport = XPORT_NVME; cts->protocol = PROTO_NVME; cts->ccb_h.status = CAM_REQ_CMP; break; } case XPT_TERM_IO: /* Terminate the I/O process */ /* * every driver handles this, but nothing generates it. Assume * it's OK to just say 'that worked'. */ /*FALLTHROUGH*/ case XPT_RESET_DEV: /* Bus Device Reset the specified device */ case XPT_RESET_BUS: /* Reset the specified bus */ /* * NVMe doesn't really support physically resetting the bus. It's part * of the bus scanning dance, so return sucess to tell the process to * proceed. */ ccb->ccb_h.status = CAM_REQ_CMP; break; case XPT_NVME_IO: /* Execute the requested I/O operation */ case XPT_NVME_ADMIN: /* or Admin operation */ nvme_sim_nvmeio(sim, ccb); return; /* no done */ default: ccb->ccb_h.status = CAM_REQ_INVALID; break; } xpt_done(ccb); } static void nvme_sim_poll(struct cam_sim *sim) { nvme_ctrlr_poll(sim2ctrlr(sim)); } static void * nvme_sim_new_controller(struct nvme_controller *ctrlr) { struct nvme_sim_softc *sc; struct cam_devq *devq; int max_trans; max_trans = ctrlr->max_hw_pend_io; devq = cam_simq_alloc(max_trans); if (devq == NULL) return (NULL); sc = malloc(sizeof(*sc), M_NVME, M_ZERO | M_WAITOK); sc->s_ctrlr = ctrlr; sc->s_sim = cam_sim_alloc(nvme_sim_action, nvme_sim_poll, "nvme", sc, device_get_unit(ctrlr->dev), NULL, max_trans, max_trans, devq); if (sc->s_sim == NULL) { printf("Failed to allocate a sim\n"); cam_simq_free(devq); goto err1; } if (xpt_bus_register(sc->s_sim, ctrlr->dev, 0) != CAM_SUCCESS) { printf("Failed to create a bus\n"); goto err2; } if (xpt_create_path(&sc->s_path, /*periph*/NULL, cam_sim_path(sc->s_sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { printf("Failed to create a path\n"); goto err3; } return (sc); err3: xpt_bus_deregister(cam_sim_path(sc->s_sim)); err2: cam_sim_free(sc->s_sim, /*free_devq*/TRUE); err1: free(sc, M_NVME); return (NULL); } static void * nvme_sim_new_ns(struct nvme_namespace *ns, void *sc_arg) { struct nvme_sim_softc *sc = sc_arg; union ccb *ccb; ccb = xpt_alloc_ccb_nowait(); if (ccb == NULL) { printf("unable to alloc CCB for rescan\n"); return (NULL); } if (xpt_create_path(&ccb->ccb_h.path, /*periph*/NULL, cam_sim_path(sc->s_sim), 0, ns->id) != CAM_REQ_CMP) { printf("unable to create path for rescan\n"); xpt_free_ccb(ccb); return (NULL); } xpt_rescan(ccb); return (ns); } static void nvme_sim_controller_fail(void *ctrlr_arg) { struct nvme_sim_softc *sc = ctrlr_arg; xpt_async(AC_LOST_DEVICE, sc->s_path, NULL); xpt_free_path(sc->s_path); xpt_bus_deregister(cam_sim_path(sc->s_sim)); cam_sim_free(sc->s_sim, /*free_devq*/TRUE); free(sc, M_NVME); } struct nvme_consumer *consumer_cookie; static void nvme_sim_init(void) { if (nvme_use_nvd) return; consumer_cookie = nvme_register_consumer(nvme_sim_new_ns, nvme_sim_new_controller, NULL, nvme_sim_controller_fail); } SYSINIT(nvme_sim_register, SI_SUB_DRIVERS, SI_ORDER_ANY, nvme_sim_init, NULL); static void nvme_sim_uninit(void) { if (nvme_use_nvd) return; /* XXX Cleanup */ nvme_unregister_consumer(consumer_cookie); } SYSUNINIT(nvme_sim_unregister, SI_SUB_DRIVERS, SI_ORDER_ANY, nvme_sim_uninit, NULL); Index: stable/12 =================================================================== --- stable/12 (revision 350718) +++ stable/12 (revision 350719) Property changes on: stable/12 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r348786