Index: head/sbin/nvmecontrol/nvmecontrol.c =================================================================== --- head/sbin/nvmecontrol/nvmecontrol.c (revision 249421) +++ head/sbin/nvmecontrol/nvmecontrol.c (revision 249422) @@ -1,655 +1,707 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define DEVLIST_USAGE \ " nvmecontrol devlist\n" #define IDENTIFY_USAGE \ " nvmecontrol identify \n" #define PERFTEST_USAGE \ " nvmecontrol perftest <-n num_threads> <-o read|write>\n" \ " <-s size_in_bytes> <-t time_in_seconds>\n" \ " <-i intr|wait> [-f refthread] [-p]\n" \ " \n" #define RESET_USAGE \ " nvmecontrol reset \n" static void perftest_usage(void); static void usage(void) { fprintf(stderr, "usage:\n"); fprintf(stderr, DEVLIST_USAGE); fprintf(stderr, IDENTIFY_USAGE); fprintf(stderr, RESET_USAGE); fprintf(stderr, PERFTEST_USAGE); exit(EX_USAGE); } static void print_controller_hex(struct nvme_controller_data *cdata, uint32_t length) { uint32_t *p; uint32_t i, j; p = (uint32_t *)cdata; length /= sizeof(uint32_t); for (i = 0; i < length; i+=8) { printf("%03x: ", i*4); for (j = 0; j < 8; j++) printf("%08x ", p[i+j]); printf("\n"); } printf("\n"); } static void print_controller(struct nvme_controller_data *cdata) { printf("Controller Capabilities/Features\n"); printf("================================\n"); printf("Vendor ID: %04x\n", cdata->vid); printf("Subsystem Vendor ID: %04x\n", cdata->ssvid); printf("Serial Number: %s\n", cdata->sn); printf("Model Number: %s\n", cdata->mn); printf("Firmware Version: %s\n", cdata->fr); printf("Recommended Arb Burst: %d\n", cdata->rab); printf("IEEE OUI Identifier: %02x %02x %02x\n", cdata->ieee[0], cdata->ieee[1], cdata->ieee[2]); printf("Multi-Interface Cap: %02x\n", cdata->mic); /* TODO: Use CAP.MPSMIN to determine true memory page size. */ printf("Max Data Transfer Size: "); if (cdata->mdts == 0) printf("Unlimited\n"); else printf("%d\n", PAGE_SIZE * (1 << cdata->mdts)); printf("\n"); printf("Admin Command Set Attributes\n"); printf("============================\n"); printf("Security Send/Receive: %s\n", cdata->oacs.security ? "Supported" : "Not Supported"); printf("Format NVM: %s\n", cdata->oacs.format ? "Supported" : "Not Supported"); printf("Firmware Activate/Download: %s\n", cdata->oacs.firmware ? "Supported" : "Not Supported"); printf("Abort Command Limit: %d\n", cdata->acl+1); printf("Async Event Request Limit: %d\n", cdata->aerl+1); printf("Number of Firmware Slots: "); if (cdata->oacs.firmware != 0) printf("%d\n", cdata->frmw.num_slots); else printf("N/A\n"); printf("Firmware Slot 1 Read-Only: "); if (cdata->oacs.firmware != 0) printf("%s\n", cdata->frmw.slot1_ro ? "Yes" : "No"); else printf("N/A\n"); printf("Per-Namespace SMART Log: %s\n", cdata->lpa.ns_smart ? "Yes" : "No"); printf("Error Log Page Entries: %d\n", cdata->elpe+1); printf("Number of Power States: %d\n", cdata->npss+1); printf("\n"); printf("NVM Command Set Attributes\n"); printf("==========================\n"); printf("Submission Queue Entry Size\n"); printf(" Max: %d\n", 1 << cdata->sqes.max); printf(" Min: %d\n", 1 << cdata->sqes.min); printf("Completion Queue Entry Size\n"); printf(" Max: %d\n", 1 << cdata->cqes.max); printf(" Min: %d\n", 1 << cdata->cqes.min); printf("Number of Namespaces: %d\n", cdata->nn); printf("Compare Command: %s\n", cdata->oncs.compare ? "Supported" : "Not Supported"); printf("Write Uncorrectable Command: %s\n", cdata->oncs.write_unc ? "Supported" : "Not Supported"); printf("Dataset Management Command: %s\n", cdata->oncs.dsm ? "Supported" : "Not Supported"); printf("Volatile Write Cache: %s\n", cdata->vwc.present ? "Present" : "Not Present"); } static void print_namespace_hex(struct nvme_namespace_data *nsdata, uint32_t length) { uint32_t *p; uint32_t i, j; p = (uint32_t *)nsdata; length /= sizeof(uint32_t); for (i = 0; i < length; i+=8) { printf("%03x: ", i*4); for (j = 0; j < 8; j++) printf("%08x ", p[i+j]); printf("\n"); } printf("\n"); } static void print_namespace(struct nvme_namespace_data *nsdata) { uint32_t i; printf("Size (in LBAs): %lld (%lldM)\n", (long long)nsdata->nsze, (long long)nsdata->nsze / 1024 / 1024); printf("Capacity (in LBAs): %lld (%lldM)\n", (long long)nsdata->ncap, (long long)nsdata->ncap / 1024 / 1024); printf("Utilization (in LBAs): %lld (%lldM)\n", (long long)nsdata->nuse, (long long)nsdata->nuse / 1024 / 1024); printf("Thin Provisioning: %s\n", nsdata->nsfeat.thin_prov ? "Supported" : "Not Supported"); printf("Number of LBA Formats: %d\n", nsdata->nlbaf+1); printf("Current LBA Format: LBA Format #%d\n", nsdata->flbas.format); for (i = 0; i <= nsdata->nlbaf; i++) { printf("LBA Format #%d:\n", i); printf(" LBA Data Size: %d\n", 1 << nsdata->lbaf[i].lbads); } } static uint32_t ns_get_sector_size(struct nvme_namespace_data *nsdata) { return (1 << nsdata->lbaf[0].lbads); } +static void +read_controller_data(int fd, struct nvme_controller_data *cdata) +{ + struct nvme_pt_command pt; + memset(&pt, 0, sizeof(pt)); + pt.cmd.opc = NVME_OPC_IDENTIFY; + pt.cmd.cdw10 = 1; + pt.buf = cdata; + pt.len = sizeof(*cdata); + pt.is_read = 1; + + if (ioctl(fd, NVME_PASSTHROUGH_CMD, &pt) < 0) { + printf("Identify request failed. errno=%d (%s)\n", + errno, strerror(errno)); + exit(EX_IOERR); + } + + if (nvme_completion_is_error(&pt.cpl)) { + printf("Passthrough command returned error.\n"); + exit(EX_IOERR); + } +} + static void +read_namespace_data(int fd, int nsid, struct nvme_namespace_data *nsdata) +{ + struct nvme_pt_command pt; + + memset(&pt, 0, sizeof(pt)); + pt.cmd.opc = NVME_OPC_IDENTIFY; + pt.cmd.nsid = nsid; + pt.buf = nsdata; + pt.len = sizeof(*nsdata); + pt.is_read = 1; + + if (ioctl(fd, NVME_PASSTHROUGH_CMD, &pt) < 0) { + printf("Identify request failed. errno=%d (%s)\n", + errno, strerror(errno)); + exit(EX_IOERR); + } + + if (nvme_completion_is_error(&pt.cpl)) { + printf("Passthrough command returned error.\n"); + exit(EX_IOERR); + } +} + +static void devlist(int argc, char *argv[]) { struct nvme_controller_data cdata; struct nvme_namespace_data nsdata; struct stat devstat; char name[64], path[64]; uint32_t i; int ch, ctrlr, exit_code, fd, found; exit_code = EX_OK; while ((ch = getopt(argc, argv, "")) != -1) { switch ((char)ch) { default: usage(); } } ctrlr = -1; found = 0; while (1) { ctrlr++; sprintf(name, "nvme%d", ctrlr); sprintf(path, "/dev/%s", name); if (stat(path, &devstat) != 0) break; found++; fd = open(path, O_RDWR); if (fd < 0) { printf("Could not open %s. errno=%d (%s)\n", path, errno, strerror(errno)); exit_code = EX_NOPERM; continue; } - if (ioctl(fd, NVME_IDENTIFY_CONTROLLER, &cdata) < 0) { - printf("Identify request to %s failed. errno=%d (%s)\n", - path, errno, strerror(errno)); - exit_code = EX_IOERR; - continue; - } - + read_controller_data(fd, &cdata); printf("%6s: %s\n", name, cdata.mn); for (i = 0; i < cdata.nn; i++) { sprintf(name, "nvme%dns%d", ctrlr, i+1); - sprintf(path, "/dev/%s", name); - - fd = open(path, O_RDWR); - if (fd < 0) { - printf("Could not open %s. errno=%d (%s)\n", - path, errno, strerror(errno)); - exit_code = EX_NOPERM; - continue; - } - if (ioctl(fd, NVME_IDENTIFY_NAMESPACE, &nsdata) < 0) { - printf("Identify request to %s failed. " - "errno=%d (%s)\n", path, errno, - strerror(errno)); - exit_code = EX_IOERR; - continue; - } + read_namespace_data(fd, i+1, &nsdata); printf(" %10s (%lldGB)\n", name, nsdata.nsze * (long long)ns_get_sector_size(&nsdata) / 1024 / 1024 / 1024); } } if (found == 0) printf("No NVMe controllers found.\n"); exit(exit_code); } static void identify_ctrlr(int argc, char *argv[]) { struct nvme_controller_data cdata; struct stat devstat; char path[64]; int ch, fd, hexflag = 0, hexlength; int verboseflag = 0; while ((ch = getopt(argc, argv, "vx")) != -1) { switch ((char)ch) { case 'v': verboseflag = 1; break; case 'x': hexflag = 1; break; default: usage(); } } sprintf(path, "/dev/%s", argv[optind]); if (stat(path, &devstat) < 0) { printf("Invalid device node %s. errno=%d (%s)\n", path, errno, strerror(errno)); exit(EX_IOERR); } fd = open(path, O_RDWR); if (fd < 0) { printf("Could not open %s. errno=%d (%s)\n", path, errno, strerror(errno)); exit(EX_NOPERM); } - if (ioctl(fd, NVME_IDENTIFY_CONTROLLER, &cdata) < 0) { - printf("Identify request to %s failed. errno=%d (%s)\n", path, - errno, strerror(errno)); - exit(EX_IOERR); - } + read_controller_data(fd, &cdata); if (hexflag == 1) { if (verboseflag == 1) hexlength = sizeof(struct nvme_controller_data); else hexlength = offsetof(struct nvme_controller_data, reserved5); print_controller_hex(&cdata, hexlength); exit(EX_OK); } if (verboseflag == 1) { printf("-v not currently supported without -x.\n"); usage(); } print_controller(&cdata); exit(EX_OK); } static void identify_ns(int argc, char *argv[]) { struct nvme_namespace_data nsdata; struct stat devstat; char path[64]; - int ch, fd, hexflag = 0, hexlength; + char *nsloc; + int ch, fd, hexflag = 0, hexlength, nsid; int verboseflag = 0; while ((ch = getopt(argc, argv, "vx")) != -1) { switch ((char)ch) { case 'v': verboseflag = 1; break; case 'x': hexflag = 1; break; default: usage(); } } + /* + * Check if the specified device node exists before continuing. + * This is a cleaner check for cases where the correct controller + * is specified, but an invalid namespace on that controller. + */ sprintf(path, "/dev/%s", argv[optind]); + if (stat(path, &devstat) < 0) { + printf("Invalid device node %s. errno=%d (%s)\n", path, errno, + strerror(errno)); + exit(EX_IOERR); + } + nsloc = strstr(argv[optind], "ns"); + if (nsloc == NULL) { + printf("Invalid namepsace %s.\n", argv[optind]); + exit(EX_IOERR); + } + + /* + * Pull the namespace id from the string. +2 skips past the "ns" part + * of the string. + */ + nsid = strtol(nsloc + 2, NULL, 10); + if (nsid == 0 && errno != 0) { + printf("Invalid namespace ID %s.\n", argv[optind]); + exit(EX_IOERR); + } + + /* + * We send IDENTIFY commands to the controller, not the namespace, + * since it is an admin cmd. So the path should only include the + * nvmeX part of the nvmeXnsY string. + */ + sprintf(path, "/dev/"); + strncat(path, argv[optind], nsloc - argv[optind]); if (stat(path, &devstat) < 0) { printf("Invalid device node %s. errno=%d (%s)\n", path, errno, strerror(errno)); exit(EX_IOERR); } fd = open(path, O_RDWR); if (fd < 0) { printf("Could not open %s. errno=%d (%s)\n", path, errno, strerror(errno)); exit(EX_NOPERM); } - if (ioctl(fd, NVME_IDENTIFY_NAMESPACE, &nsdata) < 0) { - printf("Identify request to %s failed. errno=%d (%s)\n", path, - errno, strerror(errno)); - exit(EX_IOERR); - } + read_namespace_data(fd, nsid, &nsdata); if (hexflag == 1) { if (verboseflag == 1) hexlength = sizeof(struct nvme_namespace_data); else hexlength = offsetof(struct nvme_namespace_data, reserved6); print_namespace_hex(&nsdata, hexlength); exit(EX_OK); } if (verboseflag == 1) { printf("-v not currently supported without -x.\n"); usage(); } print_namespace(&nsdata); exit(EX_OK); } static void identify(int argc, char *argv[]) { char *target; if (argc < 2) usage(); while (getopt(argc, argv, "vx") != -1) ; target = argv[optind]; /* Specified device node must have "nvme" in it. */ if (strstr(argv[optind], "nvme") == NULL) { printf("Invalid device node '%s'.\n", argv[optind]); exit(EX_IOERR); } optreset = 1; optind = 1; /* * If device node contains "ns", we consider it a namespace, * otherwise, consider it a controller. */ if (strstr(target, "ns") == NULL) identify_ctrlr(argc, argv); else identify_ns(argc, argv); } static void print_perftest(struct nvme_io_test *io_test, bool perthread) { uint32_t i, io_completed = 0, iops, mbps; for (i = 0; i < io_test->num_threads; i++) io_completed += io_test->io_completed[i]; iops = io_completed/io_test->time; mbps = iops * io_test->size / (1024*1024); printf("Threads: %2d Size: %6d %5s Time: %3d IO/s: %7d MB/s: %4d\n", io_test->num_threads, io_test->size, io_test->opc == NVME_OPC_READ ? "READ" : "WRITE", io_test->time, iops, mbps); if (perthread) for (i = 0; i < io_test->num_threads; i++) printf("\t%3d: %8d IO/s\n", i, io_test->io_completed[i]/io_test->time); exit(1); } static void perftest_usage(void) { fprintf(stderr, "usage:\n"); fprintf(stderr, PERFTEST_USAGE); exit(EX_USAGE); } static void perftest(int argc, char *argv[]) { struct nvme_io_test io_test; int fd; char ch; char *p; const char *name; char path[64]; u_long ioctl_cmd = NVME_IO_TEST; bool nflag, oflag, sflag, tflag; int perthread = 0; nflag = oflag = sflag = tflag = false; name = NULL; memset(&io_test, 0, sizeof(io_test)); while ((ch = getopt(argc, argv, "f:i:n:o:ps:t:")) != -1) { switch (ch) { case 'f': if (!strcmp(optarg, "refthread")) io_test.flags |= NVME_TEST_FLAG_REFTHREAD; break; case 'i': if (!strcmp(optarg, "bio") || !strcmp(optarg, "wait")) ioctl_cmd = NVME_BIO_TEST; else if (!strcmp(optarg, "io") || !strcmp(optarg, "intr")) ioctl_cmd = NVME_IO_TEST; break; case 'n': nflag = true; io_test.num_threads = strtoul(optarg, &p, 0); if (p != NULL && *p != '\0') { fprintf(stderr, "\"%s\" not valid number of threads.\n", optarg); perftest_usage(); } else if (io_test.num_threads == 0 || io_test.num_threads > 128) { fprintf(stderr, "\"%s\" not valid number of threads.\n", optarg); perftest_usage(); } break; case 'o': oflag = true; if (!strcmp(optarg, "read") || !strcmp(optarg, "READ")) io_test.opc = NVME_OPC_READ; else if (!strcmp(optarg, "write") || !strcmp(optarg, "WRITE")) io_test.opc = NVME_OPC_WRITE; else { fprintf(stderr, "\"%s\" not valid opcode.\n", optarg); perftest_usage(); } break; case 'p': perthread = 1; break; case 's': sflag = true; io_test.size = strtoul(optarg, &p, 0); if (p == NULL || *p == '\0' || toupper(*p) == 'B') { // do nothing } else if (toupper(*p) == 'K') { io_test.size *= 1024; } else if (toupper(*p) == 'M') { io_test.size *= 1024 * 1024; } else { fprintf(stderr, "\"%s\" not valid size.\n", optarg); perftest_usage(); } break; case 't': tflag = true; io_test.time = strtoul(optarg, &p, 0); if (p != NULL && *p != '\0') { fprintf(stderr, "\"%s\" not valid time duration.\n", optarg); perftest_usage(); } break; } } name = argv[optind]; if (!nflag || !oflag || !sflag || !tflag || name == NULL) perftest_usage(); sprintf(path, "/dev/%s", name); fd = open(path, O_RDWR); if (fd < 0) { fprintf(stderr, "%s not valid device. errno=%d (%s)\n", path, errno, strerror(errno)); perftest_usage(); } if (ioctl(fd, ioctl_cmd, &io_test) < 0) { fprintf(stderr, "NVME_IO_TEST failed. errno=%d (%s)\n", errno, strerror(errno)); exit(EX_IOERR); } print_perftest(&io_test, perthread); exit(EX_OK); } static void reset_ctrlr(int argc, char *argv[]) { struct stat devstat; char path[64]; int ch, fd; while ((ch = getopt(argc, argv, "")) != -1) { switch ((char)ch) { default: usage(); } } sprintf(path, "/dev/%s", argv[optind]); if (stat(path, &devstat) < 0) { printf("Invalid device node %s. errno=%d (%s)\n", path, errno, strerror(errno)); exit(EX_IOERR); } fd = open(path, O_RDWR); if (fd < 0) { printf("Could not open %s. errno=%d (%s)\n", path, errno, strerror(errno)); exit(EX_NOPERM); } if (ioctl(fd, NVME_RESET_CONTROLLER) < 0) { printf("Reset request to %s failed. errno=%d (%s)\n", path, errno, strerror(errno)); exit(EX_IOERR); } exit(EX_OK); } int main(int argc, char *argv[]) { if (argc < 2) usage(); if (strcmp(argv[1], "devlist") == 0) devlist(argc-1, &argv[1]); else if (strcmp(argv[1], "identify") == 0) identify(argc-1, &argv[1]); else if (strcmp(argv[1], "perftest") == 0) perftest(argc-1, &argv[1]); else if (strcmp(argv[1], "reset") == 0) reset_ctrlr(argc-1, &argv[1]); usage(); return (0); } Index: head/sys/dev/nvme/nvme.h =================================================================== --- head/sys/dev/nvme/nvme.h (revision 249421) +++ head/sys/dev/nvme/nvme.h (revision 249422) @@ -1,861 +1,860 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __NVME_H__ #define __NVME_H__ #ifdef _KERNEL #include #endif -#define NVME_IDENTIFY_CONTROLLER _IOR('n', 0, struct nvme_controller_data) -#define NVME_IDENTIFY_NAMESPACE _IOR('n', 1, struct nvme_namespace_data) -#define NVME_IO_TEST _IOWR('n', 2, struct nvme_io_test) -#define NVME_BIO_TEST _IOWR('n', 4, struct nvme_io_test) -#define NVME_RESET_CONTROLLER _IO('n', 5) -#define NVME_PASSTHROUGH_CMD _IOWR('n', 6, struct nvme_pt_command) +#define NVME_PASSTHROUGH_CMD _IOWR('n', 0, struct nvme_pt_command) +#define NVME_RESET_CONTROLLER _IO('n', 1) + +#define NVME_IO_TEST _IOWR('n', 100, struct nvme_io_test) +#define NVME_BIO_TEST _IOWR('n', 101, struct nvme_io_test) /* * Use to mark a command to apply to all namespaces, or to retrieve global * log pages. */ #define NVME_GLOBAL_NAMESPACE_TAG ((uint32_t)0xFFFFFFFF) union cap_lo_register { uint32_t raw; struct { /** maximum queue entries supported */ uint32_t mqes : 16; /** contiguous queues required */ uint32_t cqr : 1; /** arbitration mechanism supported */ uint32_t ams : 2; uint32_t reserved1 : 5; /** timeout */ uint32_t to : 8; } bits __packed; } __packed; union cap_hi_register { uint32_t raw; struct { /** doorbell stride */ uint32_t dstrd : 4; uint32_t reserved3 : 1; /** command sets supported */ uint32_t css_nvm : 1; uint32_t css_reserved : 3; uint32_t reserved2 : 7; /** memory page size minimum */ uint32_t mpsmin : 4; /** memory page size maximum */ uint32_t mpsmax : 4; uint32_t reserved1 : 8; } bits __packed; } __packed; union cc_register { uint32_t raw; struct { /** enable */ uint32_t en : 1; uint32_t reserved1 : 3; /** i/o command set selected */ uint32_t css : 3; /** memory page size */ uint32_t mps : 4; /** arbitration mechanism selected */ uint32_t ams : 3; /** shutdown notification */ uint32_t shn : 2; /** i/o submission queue entry size */ uint32_t iosqes : 4; /** i/o completion queue entry size */ uint32_t iocqes : 4; uint32_t reserved2 : 8; } bits __packed; } __packed; enum shn_value { NVME_SHN_NORMAL = 0x1, NVME_SHN_ABRUPT = 0x2, }; union csts_register { uint32_t raw; struct { /** ready */ uint32_t rdy : 1; /** controller fatal status */ uint32_t cfs : 1; /** shutdown status */ uint32_t shst : 2; uint32_t reserved1 : 28; } bits __packed; } __packed; enum shst_value { NVME_SHST_NORMAL = 0x0, NVME_SHST_OCCURRING = 0x1, NVME_SHST_COMPLETE = 0x2, }; union aqa_register { uint32_t raw; struct { /** admin submission queue size */ uint32_t asqs : 12; uint32_t reserved1 : 4; /** admin completion queue size */ uint32_t acqs : 12; uint32_t reserved2 : 4; } bits __packed; } __packed; struct nvme_registers { /** controller capabilities */ union cap_lo_register cap_lo; union cap_hi_register cap_hi; uint32_t vs; /* version */ uint32_t intms; /* interrupt mask set */ uint32_t intmc; /* interrupt mask clear */ /** controller configuration */ union cc_register cc; uint32_t reserved1; uint32_t csts; /* controller status */ uint32_t reserved2; /** admin queue attributes */ union aqa_register aqa; uint64_t asq; /* admin submission queue base addr */ uint64_t acq; /* admin completion queue base addr */ uint32_t reserved3[0x3f2]; struct { uint32_t sq_tdbl; /* submission queue tail doorbell */ uint32_t cq_hdbl; /* completion queue head doorbell */ } doorbell[1] __packed; } __packed; struct nvme_command { /* dword 0 */ uint16_t opc : 8; /* opcode */ uint16_t fuse : 2; /* fused operation */ uint16_t rsvd1 : 6; uint16_t cid; /* command identifier */ /* dword 1 */ uint32_t nsid; /* namespace identifier */ /* dword 2-3 */ uint32_t rsvd2; uint32_t rsvd3; /* dword 4-5 */ uint64_t mptr; /* metadata pointer */ /* dword 6-7 */ uint64_t prp1; /* prp entry 1 */ /* dword 8-9 */ uint64_t prp2; /* prp entry 2 */ /* dword 10-15 */ uint32_t cdw10; /* command-specific */ uint32_t cdw11; /* command-specific */ uint32_t cdw12; /* command-specific */ uint32_t cdw13; /* command-specific */ uint32_t cdw14; /* command-specific */ uint32_t cdw15; /* command-specific */ } __packed; struct nvme_status { uint16_t p : 1; /* phase tag */ uint16_t sc : 8; /* status code */ uint16_t sct : 3; /* status code type */ uint16_t rsvd2 : 2; uint16_t m : 1; /* more */ uint16_t dnr : 1; /* do not retry */ } __packed; struct nvme_completion { /* dword 0 */ uint32_t cdw0; /* command-specific */ /* dword 1 */ uint32_t rsvd1; /* dword 2 */ uint16_t sqhd; /* submission queue head pointer */ uint16_t sqid; /* submission queue identifier */ /* dword 3 */ uint16_t cid; /* command identifier */ struct nvme_status status; } __packed; struct nvme_dsm_range { uint32_t attributes; uint32_t length; uint64_t starting_lba; } __packed; /* status code types */ enum nvme_status_code_type { NVME_SCT_GENERIC = 0x0, NVME_SCT_COMMAND_SPECIFIC = 0x1, NVME_SCT_MEDIA_ERROR = 0x2, /* 0x3-0x6 - reserved */ NVME_SCT_VENDOR_SPECIFIC = 0x7, }; /* generic command status codes */ enum nvme_generic_command_status_code { NVME_SC_SUCCESS = 0x00, NVME_SC_INVALID_OPCODE = 0x01, NVME_SC_INVALID_FIELD = 0x02, NVME_SC_COMMAND_ID_CONFLICT = 0x03, NVME_SC_DATA_TRANSFER_ERROR = 0x04, NVME_SC_ABORTED_POWER_LOSS = 0x05, NVME_SC_INTERNAL_DEVICE_ERROR = 0x06, NVME_SC_ABORTED_BY_REQUEST = 0x07, NVME_SC_ABORTED_SQ_DELETION = 0x08, NVME_SC_ABORTED_FAILED_FUSED = 0x09, NVME_SC_ABORTED_MISSING_FUSED = 0x0a, NVME_SC_INVALID_NAMESPACE_OR_FORMAT = 0x0b, NVME_SC_COMMAND_SEQUENCE_ERROR = 0x0c, NVME_SC_LBA_OUT_OF_RANGE = 0x80, NVME_SC_CAPACITY_EXCEEDED = 0x81, NVME_SC_NAMESPACE_NOT_READY = 0x82, }; /* command specific status codes */ enum nvme_command_specific_status_code { NVME_SC_COMPLETION_QUEUE_INVALID = 0x00, NVME_SC_INVALID_QUEUE_IDENTIFIER = 0x01, NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED = 0x02, NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED = 0x03, /* 0x04 - reserved */ NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05, NVME_SC_INVALID_FIRMWARE_SLOT = 0x06, NVME_SC_INVALID_FIRMWARE_IMAGE = 0x07, NVME_SC_INVALID_INTERRUPT_VECTOR = 0x08, NVME_SC_INVALID_LOG_PAGE = 0x09, NVME_SC_INVALID_FORMAT = 0x0a, NVME_SC_FIRMWARE_REQUIRES_RESET = 0x0b, NVME_SC_CONFLICTING_ATTRIBUTES = 0x80, NVME_SC_INVALID_PROTECTION_INFO = 0x81, NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE = 0x82, }; /* media error status codes */ enum nvme_media_error_status_code { NVME_SC_WRITE_FAULTS = 0x80, NVME_SC_UNRECOVERED_READ_ERROR = 0x81, NVME_SC_GUARD_CHECK_ERROR = 0x82, NVME_SC_APPLICATION_TAG_CHECK_ERROR = 0x83, NVME_SC_REFERENCE_TAG_CHECK_ERROR = 0x84, NVME_SC_COMPARE_FAILURE = 0x85, NVME_SC_ACCESS_DENIED = 0x86, }; /* admin opcodes */ enum nvme_admin_opcode { NVME_OPC_DELETE_IO_SQ = 0x00, NVME_OPC_CREATE_IO_SQ = 0x01, NVME_OPC_GET_LOG_PAGE = 0x02, /* 0x03 - reserved */ NVME_OPC_DELETE_IO_CQ = 0x04, NVME_OPC_CREATE_IO_CQ = 0x05, NVME_OPC_IDENTIFY = 0x06, /* 0x07 - reserved */ NVME_OPC_ABORT = 0x08, NVME_OPC_SET_FEATURES = 0x09, NVME_OPC_GET_FEATURES = 0x0a, /* 0x0b - reserved */ NVME_OPC_ASYNC_EVENT_REQUEST = 0x0c, /* 0x0d-0x0f - reserved */ NVME_OPC_FIRMWARE_ACTIVATE = 0x10, NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD = 0x11, NVME_OPC_FORMAT_NVM = 0x80, NVME_OPC_SECURITY_SEND = 0x81, NVME_OPC_SECURITY_RECEIVE = 0x82, }; /* nvme nvm opcodes */ enum nvme_nvm_opcode { NVME_OPC_FLUSH = 0x00, NVME_OPC_WRITE = 0x01, NVME_OPC_READ = 0x02, /* 0x03 - reserved */ NVME_OPC_WRITE_UNCORRECTABLE = 0x04, NVME_OPC_COMPARE = 0x05, /* 0x06-0x07 - reserved */ NVME_OPC_DATASET_MANAGEMENT = 0x09, }; enum nvme_feature { /* 0x00 - reserved */ NVME_FEAT_ARBITRATION = 0x01, NVME_FEAT_POWER_MANAGEMENT = 0x02, NVME_FEAT_LBA_RANGE_TYPE = 0x03, NVME_FEAT_TEMPERATURE_THRESHOLD = 0x04, NVME_FEAT_ERROR_RECOVERY = 0x05, NVME_FEAT_VOLATILE_WRITE_CACHE = 0x06, NVME_FEAT_NUMBER_OF_QUEUES = 0x07, NVME_FEAT_INTERRUPT_COALESCING = 0x08, NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09, NVME_FEAT_WRITE_ATOMICITY = 0x0A, NVME_FEAT_ASYNC_EVENT_CONFIGURATION = 0x0B, /* 0x0C-0x7F - reserved */ NVME_FEAT_SOFTWARE_PROGRESS_MARKER = 0x80, /* 0x81-0xBF - command set specific (reserved) */ /* 0xC0-0xFF - vendor specific */ }; enum nvme_dsm_attribute { NVME_DSM_ATTR_INTEGRAL_READ = 0x1, NVME_DSM_ATTR_INTEGRAL_WRITE = 0x2, NVME_DSM_ATTR_DEALLOCATE = 0x4, }; struct nvme_controller_data { /* bytes 0-255: controller capabilities and features */ /** pci vendor id */ uint16_t vid; /** pci subsystem vendor id */ uint16_t ssvid; /** serial number */ int8_t sn[20]; /** model number */ int8_t mn[40]; /** firmware revision */ uint8_t fr[8]; /** recommended arbitration burst */ uint8_t rab; /** ieee oui identifier */ uint8_t ieee[3]; /** multi-interface capabilities */ uint8_t mic; /** maximum data transfer size */ uint8_t mdts; uint8_t reserved1[178]; /* bytes 256-511: admin command set attributes */ /** optional admin command support */ struct { /* supports security send/receive commands */ uint16_t security : 1; /* supports format nvm command */ uint16_t format : 1; /* supports firmware activate/download commands */ uint16_t firmware : 1; uint16_t oacs_rsvd : 13; } __packed oacs; /** abort command limit */ uint8_t acl; /** asynchronous event request limit */ uint8_t aerl; /** firmware updates */ struct { /* first slot is read-only */ uint8_t slot1_ro : 1; /* number of firmware slots */ uint8_t num_slots : 3; uint8_t frmw_rsvd : 4; } __packed frmw; /** log page attributes */ struct { /* per namespace smart/health log page */ uint8_t ns_smart : 1; uint8_t lpa_rsvd : 7; } __packed lpa; /** error log page entries */ uint8_t elpe; /** number of power states supported */ uint8_t npss; /** admin vendor specific command configuration */ struct { /* admin vendor specific commands use spec format */ uint8_t spec_format : 1; uint8_t avscc_rsvd : 7; } __packed avscc; uint8_t reserved2[247]; /* bytes 512-703: nvm command set attributes */ /** submission queue entry size */ struct { uint8_t min : 4; uint8_t max : 4; } __packed sqes; /** completion queue entry size */ struct { uint8_t min : 4; uint8_t max : 4; } __packed cqes; uint8_t reserved3[2]; /** number of namespaces */ uint32_t nn; /** optional nvm command support */ struct { uint16_t compare : 1; uint16_t write_unc : 1; uint16_t dsm: 1; uint16_t reserved: 13; } __packed oncs; /** fused operation support */ uint16_t fuses; /** format nvm attributes */ uint8_t fna; /** volatile write cache */ struct { uint8_t present : 1; uint8_t reserved : 7; } __packed vwc; /* TODO: flesh out remaining nvm command set attributes */ uint8_t reserved4[178]; /* bytes 704-2047: i/o command set attributes */ uint8_t reserved5[1344]; /* bytes 2048-3071: power state descriptors */ uint8_t reserved6[1024]; /* bytes 3072-4095: vendor specific */ uint8_t reserved7[1024]; } __packed __aligned(4); struct nvme_namespace_data { /** namespace size */ uint64_t nsze; /** namespace capacity */ uint64_t ncap; /** namespace utilization */ uint64_t nuse; /** namespace features */ struct { /** thin provisioning */ uint8_t thin_prov : 1; uint8_t reserved1 : 7; } __packed nsfeat; /** number of lba formats */ uint8_t nlbaf; /** formatted lba size */ struct { uint8_t format : 4; uint8_t extended : 1; uint8_t reserved2 : 3; } __packed flbas; /** metadata capabilities */ struct { /* metadata can be transferred as part of data prp list */ uint8_t extended : 1; /* metadata can be transferred with separate metadata pointer */ uint8_t pointer : 1; uint8_t reserved3 : 6; } __packed mc; /** end-to-end data protection capabilities */ struct { /* protection information type 1 */ uint8_t pit1 : 1; /* protection information type 2 */ uint8_t pit2 : 1; /* protection information type 3 */ uint8_t pit3 : 1; /* first eight bytes of metadata */ uint8_t md_start : 1; /* last eight bytes of metadata */ uint8_t md_end : 1; } __packed dpc; /** end-to-end data protection type settings */ struct { /* protection information type */ uint8_t pit : 3; /* 1 == protection info transferred at start of metadata */ /* 0 == protection info transferred at end of metadata */ uint8_t md_start : 1; uint8_t reserved4 : 4; } __packed dps; uint8_t reserved5[98]; /** lba format support */ struct { /** metadata size */ uint32_t ms : 16; /** lba data size */ uint32_t lbads : 8; /** relative performance */ uint32_t rp : 2; uint32_t reserved6 : 6; } __packed lbaf[16]; uint8_t reserved6[192]; uint8_t vendor_specific[3712]; } __packed __aligned(4); enum nvme_log_page { /* 0x00 - reserved */ NVME_LOG_ERROR = 0x01, NVME_LOG_HEALTH_INFORMATION = 0x02, NVME_LOG_FIRMWARE_SLOT = 0x03, /* 0x04-0x7F - reserved */ /* 0x80-0xBF - I/O command set specific */ /* 0xC0-0xFF - vendor specific */ }; struct nvme_error_information_entry { uint64_t error_count; uint16_t sqid; uint16_t cid; struct nvme_status status; uint16_t error_location; uint64_t lba; uint32_t nsid; uint8_t vendor_specific; uint8_t reserved[35]; } __packed __aligned(4); union nvme_critical_warning_state { uint8_t raw; struct { uint8_t available_spare : 1; uint8_t temperature : 1; uint8_t device_reliability : 1; uint8_t read_only : 1; uint8_t volatile_memory_backup : 1; uint8_t reserved : 3; } __packed bits; } __packed; struct nvme_health_information_page { union nvme_critical_warning_state critical_warning; uint16_t temperature; uint8_t available_spare; uint8_t available_spare_threshold; uint8_t percentage_used; uint8_t reserved[26]; /* * Note that the following are 128-bit values, but are * defined as an array of 2 64-bit values. */ /* Data Units Read is always in 512-byte units. */ uint64_t data_units_read[2]; /* Data Units Written is always in 512-byte units. */ uint64_t data_units_written[2]; /* For NVM command set, this includes Compare commands. */ uint64_t host_read_commands[2]; uint64_t host_write_commands[2]; /* Controller Busy Time is reported in minutes. */ uint64_t controller_busy_time[2]; uint64_t power_cycles[2]; uint64_t power_on_hours[2]; uint64_t unsafe_shutdowns[2]; uint64_t media_errors[2]; uint64_t num_error_info_log_entries[2]; uint8_t reserved2[320]; } __packed __aligned(4); struct nvme_firmware_page { struct { uint8_t slot : 3; /* slot for current FW */ uint8_t reserved : 5; } __packed afi; uint8_t reserved[7]; uint64_t revision[7]; /* revisions for 7 slots */ uint8_t reserved2[448]; } __packed __aligned(4); #define NVME_TEST_MAX_THREADS 128 struct nvme_io_test { enum nvme_nvm_opcode opc; uint32_t size; uint32_t time; /* in seconds */ uint32_t num_threads; uint32_t flags; uint32_t io_completed[NVME_TEST_MAX_THREADS]; }; enum nvme_io_test_flags { /* * Specifies whether dev_refthread/dev_relthread should be * called during NVME_BIO_TEST. Ignored for other test * types. */ NVME_TEST_FLAG_REFTHREAD = 0x1, }; struct nvme_pt_command { /* * cmd is used to specify a passthrough command to a controller or * namespace. * * The following fields from cmd may be specified by the caller: * * opc (opcode) * * nsid (namespace id) - for admin commands only * * cdw10-cdw15 * * Remaining fields must be set to 0 by the caller. */ struct nvme_command cmd; /* * cpl returns completion status for the passthrough command * specified by cmd. * * The following fields will be filled out by the driver, for * consumption by the caller: * * cdw0 * * status (except for phase) * * Remaining fields will be set to 0 by the driver. */ struct nvme_completion cpl; /* buf is the data buffer associated with this passthrough command. */ void * buf; /* * len is the length of the data buffer associated with this * passthrough command. */ uint32_t len; /* * is_read = 1 if the passthrough command will read data into the * supplied buffer. * * is_read = 0 if the passthrough command will write data into the * supplied buffer. */ uint32_t is_read; /* * driver_lock is used by the driver only. It must be set to 0 * by the caller. */ struct mtx * driver_lock; }; #define nvme_completion_is_error(cpl) \ ((cpl)->status.sc != 0 || (cpl)->status.sct != 0) #ifdef _KERNEL struct bio; struct nvme_namespace; struct nvme_controller; struct nvme_consumer; typedef void (*nvme_cb_fn_t)(void *, const struct nvme_completion *); typedef void *(*nvme_cons_ns_fn_t)(struct nvme_namespace *, void *); typedef void *(*nvme_cons_ctrlr_fn_t)(struct nvme_controller *); typedef void (*nvme_cons_async_fn_t)(void *, const struct nvme_completion *, uint32_t, void *, uint32_t); typedef void (*nvme_cons_fail_fn_t)(void *); enum nvme_namespace_flags { NVME_NS_DEALLOCATE_SUPPORTED = 0x1, NVME_NS_FLUSH_SUPPORTED = 0x2, }; int nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr, struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer, int is_admin_cmd); /* Admin functions */ void nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint8_t feature, uint32_t cdw11, void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn, void *cb_arg); void nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint8_t feature, uint32_t cdw11, void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn, void *cb_arg); void nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr, uint8_t log_page, uint32_t nsid, void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn, void *cb_arg); /* NVM I/O functions */ int nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg); int nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp, nvme_cb_fn_t cb_fn, void *cb_arg); int nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg); int nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp, nvme_cb_fn_t cb_fn, void *cb_arg); int nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload, uint8_t num_ranges, nvme_cb_fn_t cb_fn, void *cb_arg); int nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn, void *cb_arg); /* Registration functions */ struct nvme_consumer * nvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn, nvme_cons_async_fn_t async_fn, nvme_cons_fail_fn_t fail_fn); void nvme_unregister_consumer(struct nvme_consumer *consumer); /* Controller helper functions */ device_t nvme_ctrlr_get_device(struct nvme_controller *ctrlr); const struct nvme_controller_data * nvme_ctrlr_get_data(struct nvme_controller *ctrlr); /* Namespace helper functions */ uint32_t nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns); uint32_t nvme_ns_get_sector_size(struct nvme_namespace *ns); uint64_t nvme_ns_get_num_sectors(struct nvme_namespace *ns); uint64_t nvme_ns_get_size(struct nvme_namespace *ns); uint32_t nvme_ns_get_flags(struct nvme_namespace *ns); const char * nvme_ns_get_serial_number(struct nvme_namespace *ns); const char * nvme_ns_get_model_number(struct nvme_namespace *ns); const struct nvme_namespace_data * nvme_ns_get_data(struct nvme_namespace *ns); int nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp, nvme_cb_fn_t cb_fn); #endif /* _KERNEL */ #endif /* __NVME_H__ */ Index: head/sys/dev/nvme/nvme_ctrlr.c =================================================================== --- head/sys/dev/nvme/nvme_ctrlr.c (revision 249421) +++ head/sys/dev/nvme/nvme_ctrlr.c (revision 249422) @@ -1,1227 +1,1205 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include "nvme_private.h" static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr, struct nvme_async_event_request *aer); static int nvme_ctrlr_allocate_bar(struct nvme_controller *ctrlr) { /* Chatham puts the NVMe MMRs behind BAR 2/3, not BAR 0/1. */ if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) ctrlr->resource_id = PCIR_BAR(2); else ctrlr->resource_id = PCIR_BAR(0); ctrlr->resource = bus_alloc_resource(ctrlr->dev, SYS_RES_MEMORY, &ctrlr->resource_id, 0, ~0, 1, RF_ACTIVE); if(ctrlr->resource == NULL) { nvme_printf(ctrlr, "unable to allocate pci resource\n"); return (ENOMEM); } ctrlr->bus_tag = rman_get_bustag(ctrlr->resource); ctrlr->bus_handle = rman_get_bushandle(ctrlr->resource); ctrlr->regs = (struct nvme_registers *)ctrlr->bus_handle; /* * The NVMe spec allows for the MSI-X table to be placed behind * BAR 4/5, separate from the control/doorbell registers. Always * try to map this bar, because it must be mapped prior to calling * pci_alloc_msix(). If the table isn't behind BAR 4/5, * bus_alloc_resource() will just return NULL which is OK. */ ctrlr->bar4_resource_id = PCIR_BAR(4); ctrlr->bar4_resource = bus_alloc_resource(ctrlr->dev, SYS_RES_MEMORY, &ctrlr->bar4_resource_id, 0, ~0, 1, RF_ACTIVE); return (0); } #ifdef CHATHAM2 static int nvme_ctrlr_allocate_chatham_bar(struct nvme_controller *ctrlr) { ctrlr->chatham_resource_id = PCIR_BAR(CHATHAM_CONTROL_BAR); ctrlr->chatham_resource = bus_alloc_resource(ctrlr->dev, SYS_RES_MEMORY, &ctrlr->chatham_resource_id, 0, ~0, 1, RF_ACTIVE); if(ctrlr->chatham_resource == NULL) { nvme_printf(ctrlr, "unable to alloc pci resource\n"); return (ENOMEM); } ctrlr->chatham_bus_tag = rman_get_bustag(ctrlr->chatham_resource); ctrlr->chatham_bus_handle = rman_get_bushandle(ctrlr->chatham_resource); return (0); } static void nvme_ctrlr_setup_chatham(struct nvme_controller *ctrlr) { uint64_t reg1, reg2, reg3; uint64_t temp1, temp2; uint32_t temp3; uint32_t use_flash_timings = 0; DELAY(10000); temp3 = chatham_read_4(ctrlr, 0x8080); device_printf(ctrlr->dev, "Chatham version: 0x%x\n", temp3); ctrlr->chatham_lbas = chatham_read_4(ctrlr, 0x8068) - 0x110; ctrlr->chatham_size = ctrlr->chatham_lbas * 512; device_printf(ctrlr->dev, "Chatham size: %jd\n", (intmax_t)ctrlr->chatham_size); reg1 = reg2 = reg3 = ctrlr->chatham_size - 1; TUNABLE_INT_FETCH("hw.nvme.use_flash_timings", &use_flash_timings); if (use_flash_timings) { device_printf(ctrlr->dev, "Chatham: using flash timings\n"); temp1 = 0x00001b58000007d0LL; temp2 = 0x000000cb00000131LL; } else { device_printf(ctrlr->dev, "Chatham: using DDR timings\n"); temp1 = temp2 = 0x0LL; } chatham_write_8(ctrlr, 0x8000, reg1); chatham_write_8(ctrlr, 0x8008, reg2); chatham_write_8(ctrlr, 0x8010, reg3); chatham_write_8(ctrlr, 0x8020, temp1); temp3 = chatham_read_4(ctrlr, 0x8020); chatham_write_8(ctrlr, 0x8028, temp2); temp3 = chatham_read_4(ctrlr, 0x8028); chatham_write_8(ctrlr, 0x8030, temp1); chatham_write_8(ctrlr, 0x8038, temp2); chatham_write_8(ctrlr, 0x8040, temp1); chatham_write_8(ctrlr, 0x8048, temp2); chatham_write_8(ctrlr, 0x8050, temp1); chatham_write_8(ctrlr, 0x8058, temp2); DELAY(10000); } static void nvme_chatham_populate_cdata(struct nvme_controller *ctrlr) { struct nvme_controller_data *cdata; cdata = &ctrlr->cdata; cdata->vid = 0x8086; cdata->ssvid = 0x2011; /* * Chatham2 puts garbage data in these fields when we * invoke IDENTIFY_CONTROLLER, so we need to re-zero * the fields before calling bcopy(). */ memset(cdata->sn, 0, sizeof(cdata->sn)); memcpy(cdata->sn, "2012", strlen("2012")); memset(cdata->mn, 0, sizeof(cdata->mn)); memcpy(cdata->mn, "CHATHAM2", strlen("CHATHAM2")); memset(cdata->fr, 0, sizeof(cdata->fr)); memcpy(cdata->fr, "0", strlen("0")); cdata->rab = 8; cdata->aerl = 3; cdata->lpa.ns_smart = 1; cdata->sqes.min = 6; cdata->sqes.max = 6; cdata->sqes.min = 4; cdata->sqes.max = 4; cdata->nn = 1; /* Chatham2 doesn't support DSM command */ cdata->oncs.dsm = 0; cdata->vwc.present = 1; } #endif /* CHATHAM2 */ static void nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr) { struct nvme_qpair *qpair; uint32_t num_entries; qpair = &ctrlr->adminq; num_entries = NVME_ADMIN_ENTRIES; TUNABLE_INT_FETCH("hw.nvme.admin_entries", &num_entries); /* * If admin_entries was overridden to an invalid value, revert it * back to our default value. */ if (num_entries < NVME_MIN_ADMIN_ENTRIES || num_entries > NVME_MAX_ADMIN_ENTRIES) { nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d " "specified\n", num_entries); num_entries = NVME_ADMIN_ENTRIES; } /* * The admin queue's max xfer size is treated differently than the * max I/O xfer size. 16KB is sufficient here - maybe even less? */ nvme_qpair_construct(qpair, 0, /* qpair ID */ 0, /* vector */ num_entries, NVME_ADMIN_TRACKERS, 16*1024, /* max xfer size */ ctrlr); } static int nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr) { struct nvme_qpair *qpair; union cap_lo_register cap_lo; int i, num_entries, num_trackers; num_entries = NVME_IO_ENTRIES; TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries); /* * NVMe spec sets a hard limit of 64K max entries, but * devices may specify a smaller limit, so we need to check * the MQES field in the capabilities register. */ cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo); num_entries = min(num_entries, cap_lo.bits.mqes+1); num_trackers = NVME_IO_TRACKERS; TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers); num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS); num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS); /* * No need to have more trackers than entries in the submit queue. * Note also that for a queue size of N, we can only have (N-1) * commands outstanding, hence the "-1" here. */ num_trackers = min(num_trackers, (num_entries-1)); ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE; TUNABLE_INT_FETCH("hw.nvme.max_xfer_size", &ctrlr->max_xfer_size); /* * Check that tunable doesn't specify a size greater than what our * driver supports, and is an even PAGE_SIZE multiple. */ if (ctrlr->max_xfer_size > NVME_MAX_XFER_SIZE || ctrlr->max_xfer_size % PAGE_SIZE) ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE; ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair), M_NVME, M_ZERO | M_WAITOK); for (i = 0; i < ctrlr->num_io_queues; i++) { qpair = &ctrlr->ioq[i]; /* * Admin queue has ID=0. IO queues start at ID=1 - * hence the 'i+1' here. * * For I/O queues, use the controller-wide max_xfer_size * calculated in nvme_attach(). */ nvme_qpair_construct(qpair, i+1, /* qpair ID */ ctrlr->msix_enabled ? i+1 : 0, /* vector */ num_entries, num_trackers, ctrlr->max_xfer_size, ctrlr); if (ctrlr->per_cpu_io_queues) bus_bind_intr(ctrlr->dev, qpair->res, i); } return (0); } static void nvme_ctrlr_fail(struct nvme_controller *ctrlr) { int i; ctrlr->is_failed = TRUE; nvme_qpair_fail(&ctrlr->adminq); for (i = 0; i < ctrlr->num_io_queues; i++) nvme_qpair_fail(&ctrlr->ioq[i]); nvme_notify_fail_consumers(ctrlr); } void nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr, struct nvme_request *req) { mtx_lock(&ctrlr->lock); STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq); mtx_unlock(&ctrlr->lock); taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task); } static void nvme_ctrlr_fail_req_task(void *arg, int pending) { struct nvme_controller *ctrlr = arg; struct nvme_request *req; mtx_lock(&ctrlr->lock); while (!STAILQ_EMPTY(&ctrlr->fail_req)) { req = STAILQ_FIRST(&ctrlr->fail_req); STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq); nvme_qpair_manual_complete_request(req->qpair, req, NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, TRUE); } mtx_unlock(&ctrlr->lock); } static int nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr) { int ms_waited; union cc_register cc; union csts_register csts; cc.raw = nvme_mmio_read_4(ctrlr, cc); csts.raw = nvme_mmio_read_4(ctrlr, csts); if (!cc.bits.en) { nvme_printf(ctrlr, "%s called with cc.en = 0\n", __func__); return (ENXIO); } ms_waited = 0; while (!csts.bits.rdy) { DELAY(1000); if (ms_waited++ > ctrlr->ready_timeout_in_ms) { nvme_printf(ctrlr, "controller did not become ready " "within %d ms\n", ctrlr->ready_timeout_in_ms); return (ENXIO); } csts.raw = nvme_mmio_read_4(ctrlr, csts); } return (0); } static void nvme_ctrlr_disable(struct nvme_controller *ctrlr) { union cc_register cc; union csts_register csts; cc.raw = nvme_mmio_read_4(ctrlr, cc); csts.raw = nvme_mmio_read_4(ctrlr, csts); if (cc.bits.en == 1 && csts.bits.rdy == 0) nvme_ctrlr_wait_for_ready(ctrlr); cc.bits.en = 0; nvme_mmio_write_4(ctrlr, cc, cc.raw); DELAY(5000); } static int nvme_ctrlr_enable(struct nvme_controller *ctrlr) { union cc_register cc; union csts_register csts; union aqa_register aqa; cc.raw = nvme_mmio_read_4(ctrlr, cc); csts.raw = nvme_mmio_read_4(ctrlr, csts); if (cc.bits.en == 1) { if (csts.bits.rdy == 1) return (0); else return (nvme_ctrlr_wait_for_ready(ctrlr)); } nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr); DELAY(5000); nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr); DELAY(5000); aqa.raw = 0; /* acqs and asqs are 0-based. */ aqa.bits.acqs = ctrlr->adminq.num_entries-1; aqa.bits.asqs = ctrlr->adminq.num_entries-1; nvme_mmio_write_4(ctrlr, aqa, aqa.raw); DELAY(5000); cc.bits.en = 1; cc.bits.css = 0; cc.bits.ams = 0; cc.bits.shn = 0; cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */ cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */ /* This evaluates to 0, which is according to spec. */ cc.bits.mps = (PAGE_SIZE >> 13); nvme_mmio_write_4(ctrlr, cc, cc.raw); DELAY(5000); return (nvme_ctrlr_wait_for_ready(ctrlr)); } int nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr) { int i; nvme_admin_qpair_disable(&ctrlr->adminq); for (i = 0; i < ctrlr->num_io_queues; i++) nvme_io_qpair_disable(&ctrlr->ioq[i]); DELAY(100*1000); nvme_ctrlr_disable(ctrlr); return (nvme_ctrlr_enable(ctrlr)); } void nvme_ctrlr_reset(struct nvme_controller *ctrlr) { int cmpset; cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1); if (cmpset == 0 || ctrlr->is_failed) /* * Controller is already resetting or has failed. Return * immediately since there is no need to kick off another * reset in these cases. */ return; taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task); } static int nvme_ctrlr_identify(struct nvme_controller *ctrlr) { struct nvme_completion_poll_status status; status.done = FALSE; nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata, nvme_completion_poll_cb, &status); while (status.done == FALSE) DELAY(5); if (nvme_completion_is_error(&status.cpl)) { nvme_printf(ctrlr, "nvme_identify_controller failed!\n"); return (ENXIO); } #ifdef CHATHAM2 if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) nvme_chatham_populate_cdata(ctrlr); #endif /* * Use MDTS to ensure our default max_xfer_size doesn't exceed what the * controller supports. */ if (ctrlr->cdata.mdts > 0) ctrlr->max_xfer_size = min(ctrlr->max_xfer_size, ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts))); return (0); } static int nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr) { struct nvme_completion_poll_status status; int cq_allocated, i, sq_allocated; status.done = FALSE; nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues, nvme_completion_poll_cb, &status); while (status.done == FALSE) DELAY(5); if (nvme_completion_is_error(&status.cpl)) { nvme_printf(ctrlr, "nvme_set_num_queues failed!\n"); return (ENXIO); } /* * Data in cdw0 is 0-based. * Lower 16-bits indicate number of submission queues allocated. * Upper 16-bits indicate number of completion queues allocated. */ sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1; cq_allocated = (status.cpl.cdw0 >> 16) + 1; /* * Check that the controller was able to allocate the number of * queues we requested. If not, revert to one IO queue pair. */ if (sq_allocated < ctrlr->num_io_queues || cq_allocated < ctrlr->num_io_queues) { /* * Destroy extra IO queue pairs that were created at * controller construction time but are no longer * needed. This will only happen when a controller * supports fewer queues than MSI-X vectors. This * is not the normal case, but does occur with the * Chatham prototype board. */ for (i = 1; i < ctrlr->num_io_queues; i++) nvme_io_qpair_destroy(&ctrlr->ioq[i]); ctrlr->num_io_queues = 1; ctrlr->per_cpu_io_queues = 0; } return (0); } static int nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr) { struct nvme_completion_poll_status status; struct nvme_qpair *qpair; int i; for (i = 0; i < ctrlr->num_io_queues; i++) { qpair = &ctrlr->ioq[i]; status.done = FALSE; nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector, nvme_completion_poll_cb, &status); while (status.done == FALSE) DELAY(5); if (nvme_completion_is_error(&status.cpl)) { nvme_printf(ctrlr, "nvme_create_io_cq failed!\n"); return (ENXIO); } status.done = FALSE; nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair, nvme_completion_poll_cb, &status); while (status.done == FALSE) DELAY(5); if (nvme_completion_is_error(&status.cpl)) { nvme_printf(ctrlr, "nvme_create_io_sq failed!\n"); return (ENXIO); } } return (0); } static int nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr) { struct nvme_namespace *ns; int i, status; for (i = 0; i < ctrlr->cdata.nn; i++) { ns = &ctrlr->ns[i]; status = nvme_ns_construct(ns, i+1, ctrlr); if (status != 0) return (status); } return (0); } static boolean_t is_log_page_id_valid(uint8_t page_id) { switch (page_id) { case NVME_LOG_ERROR: case NVME_LOG_HEALTH_INFORMATION: case NVME_LOG_FIRMWARE_SLOT: return (TRUE); } return (FALSE); } static uint32_t nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id) { uint32_t log_page_size; switch (page_id) { case NVME_LOG_ERROR: log_page_size = min( sizeof(struct nvme_error_information_entry) * ctrlr->cdata.elpe, NVME_MAX_AER_LOG_SIZE); break; case NVME_LOG_HEALTH_INFORMATION: log_page_size = sizeof(struct nvme_health_information_page); break; case NVME_LOG_FIRMWARE_SLOT: log_page_size = sizeof(struct nvme_firmware_page); break; default: log_page_size = 0; break; } return (log_page_size); } static void nvme_ctrlr_async_event_log_page_cb(void *arg, const struct nvme_completion *cpl) { struct nvme_async_event_request *aer = arg; /* * If the log page fetch for some reason completed with an error, * don't pass log page data to the consumers. In practice, this case * should never happen. */ if (nvme_completion_is_error(cpl)) nvme_notify_async_consumers(aer->ctrlr, &aer->cpl, aer->log_page_id, NULL, 0); else /* * Pass the cpl data from the original async event completion, * not the log page fetch. */ nvme_notify_async_consumers(aer->ctrlr, &aer->cpl, aer->log_page_id, aer->log_page_buffer, aer->log_page_size); /* * Repost another asynchronous event request to replace the one * that just completed. */ nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer); } static void nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl) { struct nvme_async_event_request *aer = arg; if (cpl->status.sc == NVME_SC_ABORTED_SQ_DELETION) { /* * This is simulated when controller is being shut down, to * effectively abort outstanding asynchronous event requests * and make sure all memory is freed. Do not repost the * request in this case. */ return; } /* Associated log page is in bits 23:16 of completion entry dw0. */ aer->log_page_id = (cpl->cdw0 & 0xFF0000) >> 16; nvme_printf(aer->ctrlr, "async event occurred (log page id=0x%x)\n", aer->log_page_id); if (is_log_page_id_valid(aer->log_page_id)) { aer->log_page_size = nvme_ctrlr_get_log_page_size(aer->ctrlr, aer->log_page_id); memcpy(&aer->cpl, cpl, sizeof(*cpl)); nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id, NVME_GLOBAL_NAMESPACE_TAG, aer->log_page_buffer, aer->log_page_size, nvme_ctrlr_async_event_log_page_cb, aer); /* Wait to notify consumers until after log page is fetched. */ } else { nvme_notify_async_consumers(aer->ctrlr, cpl, aer->log_page_id, NULL, 0); /* * Repost another asynchronous event request to replace the one * that just completed. */ nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer); } } static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr, struct nvme_async_event_request *aer) { struct nvme_request *req; aer->ctrlr = ctrlr; req = nvme_allocate_request_null(nvme_ctrlr_async_event_cb, aer); aer->req = req; /* * Disable timeout here, since asynchronous event requests should by * nature never be timed out. */ req->timeout = FALSE; req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST; nvme_ctrlr_submit_admin_request(ctrlr, req); } static void nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr) { union nvme_critical_warning_state state; struct nvme_async_event_request *aer; uint32_t i; state.raw = 0xFF; state.bits.reserved = 0; nvme_ctrlr_cmd_set_async_event_config(ctrlr, state, NULL, NULL); /* aerl is a zero-based value, so we need to add 1 here. */ ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1)); /* Chatham doesn't support AERs. */ if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) ctrlr->num_aers = 0; for (i = 0; i < ctrlr->num_aers; i++) { aer = &ctrlr->aer[i]; nvme_ctrlr_construct_and_submit_aer(ctrlr, aer); } } static void nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr) { ctrlr->int_coal_time = 0; TUNABLE_INT_FETCH("hw.nvme.int_coal_time", &ctrlr->int_coal_time); ctrlr->int_coal_threshold = 0; TUNABLE_INT_FETCH("hw.nvme.int_coal_threshold", &ctrlr->int_coal_threshold); nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time, ctrlr->int_coal_threshold, NULL, NULL); } static void nvme_ctrlr_start(void *ctrlr_arg) { struct nvme_controller *ctrlr = ctrlr_arg; int i; nvme_qpair_reset(&ctrlr->adminq); for (i = 0; i < ctrlr->num_io_queues; i++) nvme_qpair_reset(&ctrlr->ioq[i]); nvme_admin_qpair_enable(&ctrlr->adminq); if (nvme_ctrlr_identify(ctrlr) != 0) { nvme_ctrlr_fail(ctrlr); return; } if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) { nvme_ctrlr_fail(ctrlr); return; } if (nvme_ctrlr_create_qpairs(ctrlr) != 0) { nvme_ctrlr_fail(ctrlr); return; } if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) { nvme_ctrlr_fail(ctrlr); return; } nvme_ctrlr_configure_aer(ctrlr); nvme_ctrlr_configure_int_coalescing(ctrlr); for (i = 0; i < ctrlr->num_io_queues; i++) nvme_io_qpair_enable(&ctrlr->ioq[i]); /* * Clear software progress marker to 0, to indicate to pre-boot * software that OS driver load was successful. * * Chatham does not support this feature. */ if (pci_get_devid(ctrlr->dev) != CHATHAM_PCI_ID) nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_SOFTWARE_PROGRESS_MARKER, 0, NULL, 0, NULL, NULL); } void nvme_ctrlr_start_config_hook(void *arg) { struct nvme_controller *ctrlr = arg; nvme_ctrlr_start(ctrlr); config_intrhook_disestablish(&ctrlr->config_hook); } static void nvme_ctrlr_reset_task(void *arg, int pending) { struct nvme_controller *ctrlr = arg; int status; nvme_printf(ctrlr, "resetting controller\n"); status = nvme_ctrlr_hw_reset(ctrlr); /* * Use pause instead of DELAY, so that we yield to any nvme interrupt * handlers on this CPU that were blocked on a qpair lock. We want * all nvme interrupts completed before proceeding with restarting the * controller. * * XXX - any way to guarantee the interrupt handlers have quiesced? */ pause("nvmereset", hz / 10); if (status == 0) nvme_ctrlr_start(ctrlr); else nvme_ctrlr_fail(ctrlr); atomic_cmpset_32(&ctrlr->is_resetting, 1, 0); } static void nvme_ctrlr_intx_handler(void *arg) { struct nvme_controller *ctrlr = arg; nvme_mmio_write_4(ctrlr, intms, 1); nvme_qpair_process_completions(&ctrlr->adminq); if (ctrlr->ioq[0].cpl) nvme_qpair_process_completions(&ctrlr->ioq[0]); nvme_mmio_write_4(ctrlr, intmc, 1); } static int nvme_ctrlr_configure_intx(struct nvme_controller *ctrlr) { ctrlr->num_io_queues = 1; ctrlr->per_cpu_io_queues = 0; ctrlr->rid = 0; ctrlr->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, &ctrlr->rid, RF_SHAREABLE | RF_ACTIVE); if (ctrlr->res == NULL) { nvme_printf(ctrlr, "unable to allocate shared IRQ\n"); return (ENOMEM); } bus_setup_intr(ctrlr->dev, ctrlr->res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_intx_handler, ctrlr, &ctrlr->tag); if (ctrlr->tag == NULL) { nvme_printf(ctrlr, "unable to setup intx handler\n"); return (ENOMEM); } return (0); } static void nvme_pt_done(void *arg, const struct nvme_completion *cpl) { struct nvme_pt_command *pt = arg; bzero(&pt->cpl, sizeof(pt->cpl)); pt->cpl.cdw0 = cpl->cdw0; pt->cpl.status = cpl->status; pt->cpl.status.p = 0; mtx_lock(pt->driver_lock); wakeup(pt); mtx_unlock(pt->driver_lock); } int nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr, struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer, int is_admin_cmd) { struct nvme_request *req; struct mtx *mtx; struct buf *buf = NULL; int ret = 0; if (pt->len > 0) if (is_user_buffer) { /* * Ensure the user buffer is wired for the duration of * this passthrough command. */ PHOLD(curproc); buf = getpbuf(NULL); buf->b_saveaddr = buf->b_data; buf->b_data = pt->buf; buf->b_bufsize = pt->len; buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE; #ifdef NVME_UNMAPPED_BIO_SUPPORT if (vmapbuf(buf, 1) < 0) { #else if (vmapbuf(buf) < 0) { #endif ret = EFAULT; goto err; } req = nvme_allocate_request_vaddr(buf->b_data, pt->len, nvme_pt_done, pt); } else req = nvme_allocate_request_vaddr(pt->buf, pt->len, nvme_pt_done, pt); else req = nvme_allocate_request_null(nvme_pt_done, pt); req->cmd.opc = pt->cmd.opc; req->cmd.cdw10 = pt->cmd.cdw10; req->cmd.cdw11 = pt->cmd.cdw11; req->cmd.cdw12 = pt->cmd.cdw12; req->cmd.cdw13 = pt->cmd.cdw13; req->cmd.cdw14 = pt->cmd.cdw14; req->cmd.cdw15 = pt->cmd.cdw15; req->cmd.nsid = nsid; if (is_admin_cmd) mtx = &ctrlr->lock; else mtx = &ctrlr->ns[nsid-1].lock; mtx_lock(mtx); pt->driver_lock = mtx; if (is_admin_cmd) nvme_ctrlr_submit_admin_request(ctrlr, req); else nvme_ctrlr_submit_io_request(ctrlr, req); mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0); mtx_unlock(mtx); pt->driver_lock = NULL; err: if (buf != NULL) { relpbuf(buf, NULL); PRELE(curproc); } return (ret); } static int nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag, struct thread *td) { - struct nvme_completion_poll_status status; struct nvme_controller *ctrlr; struct nvme_pt_command *pt; ctrlr = cdev->si_drv1; switch (cmd) { - case NVME_IDENTIFY_CONTROLLER: -#ifdef CHATHAM2 - /* - * Don't refresh data on Chatham, since Chatham returns - * garbage on IDENTIFY anyways. - */ - if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) { - memcpy(arg, &ctrlr->cdata, sizeof(ctrlr->cdata)); - break; - } -#endif - /* Refresh data before returning to user. */ - status.done = FALSE; - nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata, - nvme_completion_poll_cb, &status); - while (status.done == FALSE) - DELAY(5); - if (nvme_completion_is_error(&status.cpl)) - return (ENXIO); - memcpy(arg, &ctrlr->cdata, sizeof(ctrlr->cdata)); - break; case NVME_RESET_CONTROLLER: nvme_ctrlr_reset(ctrlr); break; case NVME_PASSTHROUGH_CMD: pt = (struct nvme_pt_command *)arg; return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, pt->cmd.nsid, 1 /* is_user_buffer */, 1 /* is_admin_cmd */)); default: return (ENOTTY); } return (0); } static struct cdevsw nvme_ctrlr_cdevsw = { .d_version = D_VERSION, .d_flags = 0, .d_ioctl = nvme_ctrlr_ioctl }; int nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev) { union cap_lo_register cap_lo; union cap_hi_register cap_hi; int num_vectors, per_cpu_io_queues, status = 0; int timeout_period; ctrlr->dev = dev; mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF); status = nvme_ctrlr_allocate_bar(ctrlr); if (status != 0) return (status); #ifdef CHATHAM2 if (pci_get_devid(dev) == CHATHAM_PCI_ID) { status = nvme_ctrlr_allocate_chatham_bar(ctrlr); if (status != 0) return (status); nvme_ctrlr_setup_chatham(ctrlr); } #endif /* * Software emulators may set the doorbell stride to something * other than zero, but this driver is not set up to handle that. */ cap_hi.raw = nvme_mmio_read_4(ctrlr, cap_hi); if (cap_hi.bits.dstrd != 0) return (ENXIO); ctrlr->min_page_size = 1 << (12 + cap_hi.bits.mpsmin); /* Get ready timeout value from controller, in units of 500ms. */ cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo); ctrlr->ready_timeout_in_ms = cap_lo.bits.to * 500; timeout_period = NVME_DEFAULT_TIMEOUT_PERIOD; TUNABLE_INT_FETCH("hw.nvme.timeout_period", &timeout_period); timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD); timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD); ctrlr->timeout_period = timeout_period; nvme_retry_count = NVME_DEFAULT_RETRY_COUNT; TUNABLE_INT_FETCH("hw.nvme.retry_count", &nvme_retry_count); per_cpu_io_queues = 1; TUNABLE_INT_FETCH("hw.nvme.per_cpu_io_queues", &per_cpu_io_queues); ctrlr->per_cpu_io_queues = per_cpu_io_queues ? TRUE : FALSE; if (ctrlr->per_cpu_io_queues) ctrlr->num_io_queues = mp_ncpus; else ctrlr->num_io_queues = 1; ctrlr->force_intx = 0; TUNABLE_INT_FETCH("hw.nvme.force_intx", &ctrlr->force_intx); ctrlr->enable_aborts = 0; TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts); ctrlr->msix_enabled = 1; if (ctrlr->force_intx) { ctrlr->msix_enabled = 0; goto intx; } /* One vector per IO queue, plus one vector for admin queue. */ num_vectors = ctrlr->num_io_queues + 1; if (pci_msix_count(dev) < num_vectors) { ctrlr->msix_enabled = 0; goto intx; } if (pci_alloc_msix(dev, &num_vectors) != 0) ctrlr->msix_enabled = 0; intx: if (!ctrlr->msix_enabled) nvme_ctrlr_configure_intx(ctrlr); nvme_ctrlr_construct_admin_qpair(ctrlr); status = nvme_ctrlr_construct_io_qpairs(ctrlr); if (status != 0) return (status); ctrlr->cdev = make_dev(&nvme_ctrlr_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "nvme%d", device_get_unit(dev)); if (ctrlr->cdev == NULL) return (ENXIO); ctrlr->cdev->si_drv1 = (void *)ctrlr; ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK, taskqueue_thread_enqueue, &ctrlr->taskqueue); taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "nvme taskq"); ctrlr->is_resetting = 0; TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr); TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr); STAILQ_INIT(&ctrlr->fail_req); ctrlr->is_failed = FALSE; return (0); } void nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev) { int i; nvme_ctrlr_disable(ctrlr); taskqueue_free(ctrlr->taskqueue); for (i = 0; i < NVME_MAX_NAMESPACES; i++) nvme_ns_destruct(&ctrlr->ns[i]); if (ctrlr->cdev) destroy_dev(ctrlr->cdev); for (i = 0; i < ctrlr->num_io_queues; i++) { nvme_io_qpair_destroy(&ctrlr->ioq[i]); } free(ctrlr->ioq, M_NVME); nvme_admin_qpair_destroy(&ctrlr->adminq); if (ctrlr->resource != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, ctrlr->resource_id, ctrlr->resource); } if (ctrlr->bar4_resource != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, ctrlr->bar4_resource_id, ctrlr->bar4_resource); } #ifdef CHATHAM2 if (ctrlr->chatham_resource != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, ctrlr->chatham_resource_id, ctrlr->chatham_resource); } #endif if (ctrlr->tag) bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag); if (ctrlr->res) bus_release_resource(ctrlr->dev, SYS_RES_IRQ, rman_get_rid(ctrlr->res), ctrlr->res); if (ctrlr->msix_enabled) pci_release_msi(dev); } void nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, struct nvme_request *req) { nvme_qpair_submit_request(&ctrlr->adminq, req); } void nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, struct nvme_request *req) { struct nvme_qpair *qpair; if (ctrlr->per_cpu_io_queues) qpair = &ctrlr->ioq[curcpu]; else qpair = &ctrlr->ioq[0]; nvme_qpair_submit_request(qpair, req); } device_t nvme_ctrlr_get_device(struct nvme_controller *ctrlr) { return (ctrlr->dev); } const struct nvme_controller_data * nvme_ctrlr_get_data(struct nvme_controller *ctrlr) { return (&ctrlr->cdata); } Index: head/sys/dev/nvme/nvme_ns.c =================================================================== --- head/sys/dev/nvme/nvme_ns.c (revision 249421) +++ head/sys/dev/nvme/nvme_ns.c (revision 249422) @@ -1,375 +1,353 @@ /*- * Copyright (C) 2012 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include "nvme_private.h" static int nvme_ns_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag, struct thread *td) { - struct nvme_completion_poll_status status; struct nvme_namespace *ns; struct nvme_controller *ctrlr; struct nvme_pt_command *pt; ns = cdev->si_drv1; ctrlr = ns->ctrlr; switch (cmd) { - case NVME_IDENTIFY_NAMESPACE: -#ifdef CHATHAM2 - /* - * Don't refresh data on Chatham, since Chatham returns - * garbage on IDENTIFY anyways. - */ - if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) { - memcpy(arg, &ns->data, sizeof(ns->data)); - break; - } -#endif - /* Refresh data before returning to user. */ - status.done = FALSE; - nvme_ctrlr_cmd_identify_namespace(ctrlr, ns->id, &ns->data, - nvme_completion_poll_cb, &status); - while (status.done == FALSE) - DELAY(5); - if (nvme_completion_is_error(&status.cpl)) - return (ENXIO); - memcpy(arg, &ns->data, sizeof(ns->data)); - break; case NVME_IO_TEST: case NVME_BIO_TEST: nvme_ns_test(ns, cmd, arg); break; case NVME_PASSTHROUGH_CMD: pt = (struct nvme_pt_command *)arg; return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, ns->id, 1 /* is_user_buffer */, 0 /* is_admin_cmd */)); case DIOCGMEDIASIZE: *(off_t *)arg = (off_t)nvme_ns_get_size(ns); break; case DIOCGSECTORSIZE: *(u_int *)arg = nvme_ns_get_sector_size(ns); break; default: return (ENOTTY); } return (0); } static int nvme_ns_open(struct cdev *dev __unused, int flags, int fmt __unused, struct thread *td) { int error = 0; if (flags & FWRITE) error = securelevel_gt(td->td_ucred, 0); return (error); } static int nvme_ns_close(struct cdev *dev __unused, int flags, int fmt __unused, struct thread *td) { return (0); } static void nvme_ns_strategy_done(void *arg, const struct nvme_completion *cpl) { struct bio *bp = arg; /* * TODO: add more extensive translation of NVMe status codes * to different bio error codes (i.e. EIO, EINVAL, etc.) */ if (nvme_completion_is_error(cpl)) { bp->bio_error = EIO; bp->bio_flags |= BIO_ERROR; bp->bio_resid = bp->bio_bcount; } else bp->bio_resid = 0; biodone(bp); } static void nvme_ns_strategy(struct bio *bp) { struct nvme_namespace *ns; int err; ns = bp->bio_dev->si_drv1; err = nvme_ns_bio_process(ns, bp, nvme_ns_strategy_done); if (err) { bp->bio_error = err; bp->bio_flags |= BIO_ERROR; bp->bio_resid = bp->bio_bcount; biodone(bp); } } static struct cdevsw nvme_ns_cdevsw = { .d_version = D_VERSION, #ifdef NVME_UNMAPPED_BIO_SUPPORT .d_flags = D_DISK | D_UNMAPPED_IO, #else .d_flags = D_DISK, #endif .d_read = physread, .d_write = physwrite, .d_open = nvme_ns_open, .d_close = nvme_ns_close, .d_strategy = nvme_ns_strategy, .d_ioctl = nvme_ns_ioctl }; uint32_t nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns) { return ns->ctrlr->max_xfer_size; } uint32_t nvme_ns_get_sector_size(struct nvme_namespace *ns) { return (1 << ns->data.lbaf[0].lbads); } uint64_t nvme_ns_get_num_sectors(struct nvme_namespace *ns) { return (ns->data.nsze); } uint64_t nvme_ns_get_size(struct nvme_namespace *ns) { return (nvme_ns_get_num_sectors(ns) * nvme_ns_get_sector_size(ns)); } uint32_t nvme_ns_get_flags(struct nvme_namespace *ns) { return (ns->flags); } const char * nvme_ns_get_serial_number(struct nvme_namespace *ns) { return ((const char *)ns->ctrlr->cdata.sn); } const char * nvme_ns_get_model_number(struct nvme_namespace *ns) { return ((const char *)ns->ctrlr->cdata.mn); } const struct nvme_namespace_data * nvme_ns_get_data(struct nvme_namespace *ns) { return (&ns->data); } static void nvme_ns_bio_done(void *arg, const struct nvme_completion *status) { struct bio *bp = arg; nvme_cb_fn_t bp_cb_fn; bp_cb_fn = bp->bio_driver1; if (bp->bio_driver2) free(bp->bio_driver2, M_NVME); bp_cb_fn(bp, status); } int nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp, nvme_cb_fn_t cb_fn) { struct nvme_dsm_range *dsm_range; int err; bp->bio_driver1 = cb_fn; switch (bp->bio_cmd) { case BIO_READ: err = nvme_ns_cmd_read_bio(ns, bp, nvme_ns_bio_done, bp); break; case BIO_WRITE: err = nvme_ns_cmd_write_bio(ns, bp, nvme_ns_bio_done, bp); break; case BIO_FLUSH: err = nvme_ns_cmd_flush(ns, nvme_ns_bio_done, bp); break; case BIO_DELETE: dsm_range = malloc(sizeof(struct nvme_dsm_range), M_NVME, M_ZERO | M_WAITOK); dsm_range->length = bp->bio_bcount/nvme_ns_get_sector_size(ns); dsm_range->starting_lba = bp->bio_offset/nvme_ns_get_sector_size(ns); bp->bio_driver2 = dsm_range; err = nvme_ns_cmd_deallocate(ns, dsm_range, 1, nvme_ns_bio_done, bp); if (err != 0) free(dsm_range, M_NVME); break; default: err = EIO; break; } return (err); } #ifdef CHATHAM2 static void nvme_ns_populate_chatham_data(struct nvme_namespace *ns) { struct nvme_controller *ctrlr; struct nvme_namespace_data *nsdata; ctrlr = ns->ctrlr; nsdata = &ns->data; nsdata->nsze = ctrlr->chatham_lbas; nsdata->ncap = ctrlr->chatham_lbas; nsdata->nuse = ctrlr->chatham_lbas; /* Chatham2 doesn't support thin provisioning. */ nsdata->nsfeat.thin_prov = 0; /* Set LBA size to 512 bytes. */ nsdata->lbaf[0].lbads = 9; } #endif /* CHATHAM2 */ int nvme_ns_construct(struct nvme_namespace *ns, uint16_t id, struct nvme_controller *ctrlr) { struct nvme_completion_poll_status status; ns->ctrlr = ctrlr; ns->id = id; /* * Namespaces are reconstructed after a controller reset, so check * to make sure we only call mtx_init once on each mtx. * * TODO: Move this somewhere where it gets called at controller * construction time, which is not invoked as part of each * controller reset. */ if (!mtx_initialized(&ns->lock)) mtx_init(&ns->lock, "nvme ns lock", NULL, MTX_DEF); #ifdef CHATHAM2 if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) nvme_ns_populate_chatham_data(ns); else { #endif status.done = FALSE; nvme_ctrlr_cmd_identify_namespace(ctrlr, id, &ns->data, nvme_completion_poll_cb, &status); while (status.done == FALSE) DELAY(5); if (nvme_completion_is_error(&status.cpl)) { nvme_printf(ctrlr, "nvme_identify_namespace failed\n"); return (ENXIO); } #ifdef CHATHAM2 } #endif if (ctrlr->cdata.oncs.dsm) ns->flags |= NVME_NS_DEALLOCATE_SUPPORTED; if (ctrlr->cdata.vwc.present) ns->flags |= NVME_NS_FLUSH_SUPPORTED; /* * cdev may have already been created, if we are reconstructing the * namespace after a controller-level reset. */ if (ns->cdev != NULL) return (0); /* * MAKEDEV_ETERNAL was added in r210923, for cdevs that will never * be destroyed. This avoids refcounting on the cdev object. * That should be OK case here, as long as we're not supporting PCIe * surprise removal nor namespace deletion. */ #ifdef MAKEDEV_ETERNAL_KLD ns->cdev = make_dev_credf(MAKEDEV_ETERNAL_KLD, &nvme_ns_cdevsw, 0, NULL, UID_ROOT, GID_WHEEL, 0600, "nvme%dns%d", device_get_unit(ctrlr->dev), ns->id); #else ns->cdev = make_dev_credf(0, &nvme_ns_cdevsw, 0, NULL, UID_ROOT, GID_WHEEL, 0600, "nvme%dns%d", device_get_unit(ctrlr->dev), ns->id); #endif if (ns->cdev != NULL) ns->cdev->si_drv1 = ns; return (0); } void nvme_ns_destruct(struct nvme_namespace *ns) { if (ns->cdev != NULL) destroy_dev(ns->cdev); }