diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8 --- a/usr.sbin/bhyve/bhyve.8 +++ b/usr.sbin/bhyve/bhyve.8 @@ -1,4 +1,5 @@ .\" Copyright (c) 2013 Peter Grehan +.\" Copyright 2025 Hans Rosenfeld .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -22,7 +23,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd August 21, 2024 +.Dd October 20, 2025 .Dt BHYVE 8 .Os .Sh NAME @@ -626,10 +627,45 @@ .Bl -bullet .Sm off .It -.Pa /dev/cam/ctl Oo Ar pp Cm \&. Ar vp Oc Oo Cm \&, Ar scsi-device-options Oc +.Oo Cm target Ns = Ns Oo ID : Oc Ar /dev/cam/ctl Oo Ar pp Cm \&. Ar vp Oc Oc +.Oo Cm \&, Ar scsi-device-options Oc .Sm on .El .Pp +Multiple +.Pa target +parameters may be specified, each configuring a different +.Ar path +as distinct SCSI target. +If the +.Pa target +.Ar ID +is not explicitly configured for a +.Pa target , +the +.Pa target +will be assigned the next sequential +.Ar ID +following the highest +.Pa target +.Ar ID +used at that point, or 0 if it is the first target configured. +All +.Pa target +.Ar ID Ns s +must be unique per instance. +The +.Ar path +must point to a valid CAM target layer +.Po CTL +.Pc +device node. +If no +.Pa target +is configured, a single default target backed by +.Sy /dev/cam/ctl +will be created. +.Pp The .Ar scsi-device-options are: diff --git a/usr.sbin/bhyve/bhyve_config.5 b/usr.sbin/bhyve/bhyve_config.5 --- a/usr.sbin/bhyve/bhyve_config.5 +++ b/usr.sbin/bhyve/bhyve_config.5 @@ -1,6 +1,7 @@ .\" SPDX-License-Identifier: BSD-2-Clause .\" .\" Copyright (c) 2021 John H. Baldwin +.\" Copyright 2025 Hans Rosenfeld .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions @@ -23,7 +24,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd August 21, 2024 +.Dd October 20, 2025 .Dt BHYVE_CONFIG 5 .Os .Sh NAME @@ -751,14 +752,33 @@ The largest supported MTU advertised to the guest. .El .Ss VirtIO SCSI Settings -.Bl -column "Name" "integer" "Default" +.Bl -column "target" "integer" "/dev/cam/ctl" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description -.It Va dev Ta path Ta Ta -The path of a CAM target layer (CTL) device to export: -.Pa /dev/cam/ctl Ns Oo Ar pp . Ns Ar vp Oc . .It Va iid Ta integer Ta 0 Ta Initiator ID to use when sending requests to the CTL port. +.It Va target Ta Oo Va ID : Oc Ns path Ta Sy /dev/cam/ctl Ta +The path of a CAM target layer (CTL) device to use: +.Pa /dev/cam/ctl Ns Oo Ar pp . Ns Ar vp Oc +Optionally, a numeric target +.Ar ID +in the range from 0 to 255 may be specified before the +.Ar path , +separated by a colon. .El +.Pp +The +.Va target +variable may be specified multiple times with different +.Ar path +arguments to configure multiple distinct SCSI targets. +If not explicitly configured, the target +.Ar ID Ns s +will be assigned sequentially beginning with the highest target +.Ar ID +configured so far, or 0 for the first target configured. +The target +.Ar ID Ns s +must be unique within each virtio-scsi instance. .Sh SEE ALSO .Xr expand_number 3 , .Xr getaddrinfo 3 , diff --git a/usr.sbin/bhyve/pci_virtio_scsi.c b/usr.sbin/bhyve/pci_virtio_scsi.c --- a/usr.sbin/bhyve/pci_virtio_scsi.c +++ b/usr.sbin/bhyve/pci_virtio_scsi.c @@ -77,7 +77,7 @@ (sizeof(struct pci_vtscsi_req_cmd_wr) + _sc->vss_config.sense_size) #define VIRTIO_SCSI_MAX_CHANNEL 0 -#define VIRTIO_SCSI_MAX_TARGET 0 +#define VIRTIO_SCSI_MAX_TARGET 255 #define VIRTIO_SCSI_MAX_LUN 16383 #define VIRTIO_SCSI_F_INOUT (1 << 0) @@ -140,6 +140,12 @@ STAILQ_ENTRY(pci_vtscsi_request) vsr_link; }; +struct pci_vtscsi_target { + int vst_target; + int vst_fd; + int vst_max_sectors; +}; + /* * Per-device softc */ @@ -151,7 +157,9 @@ int vss_iid; int vss_ctl_fd; uint32_t vss_features; + int vss_num_target; struct pci_vtscsi_config vss_config; + struct pci_vtscsi_target *vss_targets; }; #define VIRTIO_SCSI_T_TMF 0 @@ -242,8 +250,12 @@ static int pci_vtscsi_cfgread(void *, int, int, uint32_t *); static int pci_vtscsi_cfgwrite(void *, int, int, uint32_t); -static inline bool pci_vtscsi_check_lun(const uint8_t *); -static inline int pci_vtscsi_get_lun(const uint8_t *); +static inline bool pci_vtscsi_check_lun(struct pci_vtscsi_softc *, + const uint8_t *); +static inline int pci_vtscsi_get_target(struct pci_vtscsi_softc *, + const uint8_t *); +static inline int pci_vtscsi_get_lun(struct pci_vtscsi_softc *, + const uint8_t *); static void pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *, size_t); static void pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *, @@ -262,13 +274,15 @@ struct vqueue_info *); static void pci_vtscsi_return_request(struct pci_vtscsi_queue *, struct pci_vtscsi_request *, int); -static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *, +static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); static void pci_vtscsi_controlq_notify(void *, struct vqueue_info *); static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *); static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *); -static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, + +static int pci_vtscsi_add_target_config(nvlist_t *, const char *, int); +static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, struct pci_vtscsi_queue *, int); static int pci_vtscsi_init(struct pci_devinst *, nvlist_t *); @@ -289,10 +303,12 @@ struct pci_vtscsi_worker *worker = (struct pci_vtscsi_worker *)arg; struct pci_vtscsi_queue *q = worker->vsw_queue; struct pci_vtscsi_softc *sc = q->vsq_sc; - int iolen; for (;;) { struct pci_vtscsi_request *req; + int target; + int iolen; + int fd; pthread_mutex_lock(&q->vsq_rmtx); @@ -307,11 +323,15 @@ req = pci_vtscsi_get_request(&q->vsq_requests); pthread_mutex_unlock(&q->vsq_rmtx); - DPRINTF("I/O request lun %d, data_niov_in %d, data_niov_out %d", - pci_vtscsi_get_lun(req->vsr_cmd_rd->lun), + target = pci_vtscsi_get_target(sc, req->vsr_cmd_rd->lun); + fd = sc->vss_targets[target].vst_fd; + + DPRINTF("I/O request tgt %d, lun %d, data_niov_in %d, " + "data_niov_out %d", target, + pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun), req->vsr_data_niov_in, req->vsr_data_niov_out); - iolen = pci_vtscsi_request_handle(sc, req); + iolen = pci_vtscsi_request_handle(sc, fd, req); pci_vtscsi_return_request(q, req, iolen); } @@ -339,7 +359,7 @@ .sense_size = 96, .cdb_size = 32, .max_channel = VIRTIO_SCSI_MAX_CHANNEL, - .max_target = VIRTIO_SCSI_MAX_TARGET, + .max_target = MAX(1, sc->vss_num_target) - 1, .max_lun = VIRTIO_SCSI_MAX_LUN }; } @@ -387,7 +407,6 @@ * or: Flat Space Addressing: LUN (0-16383) * Level 3 and 4: not used, MBZ * - * Currently, we only support Target 0. * * Alternatively, the first level may contain an extended LUN address to select * the REPORT_LUNS well-known logical unit: @@ -408,7 +427,7 @@ * well-known logical unit. */ static inline bool -pci_vtscsi_check_lun(const uint8_t *lun) +pci_vtscsi_check_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) { if (lun[0] == 0xC1) return (false); @@ -416,7 +435,13 @@ if (lun[0] != 0x01) return (false); - if (lun[1] != 0x00) + if (lun[1] >= sc->vss_num_target) + return (false); + + if (lun[1] != sc->vss_targets[lun[1]].vst_target) + return (false); + + if (sc->vss_targets[lun[1]].vst_fd < 0) return (false); if (lun[2] != 0x00 && (lun[2] & 0xc0) != 0x40) @@ -429,10 +454,24 @@ } static inline int -pci_vtscsi_get_lun(const uint8_t *lun) +pci_vtscsi_get_target(struct pci_vtscsi_softc *sc, const uint8_t *lun) +{ + assert(lun[0] == 0x01); + assert(lun[1] < sc->vss_num_target); + assert(lun[1] == sc->vss_targets[lun[1]].vst_target); + assert(sc->vss_targets[lun[1]].vst_fd >= 0); + assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); + + return (lun[1]); +} + +static inline int +pci_vtscsi_get_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) { assert(lun[0] == 0x01); - assert(lun[1] == 0x00); + assert(lun[1] < sc->vss_num_target); + assert(lun[1] == sc->vss_targets[lun[1]].vst_target); + assert(sc->vss_targets[lun[1]].vst_fd >= 0); assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); return (((lun[2] << 8) | lun[3]) & 0x3fff); @@ -475,9 +514,11 @@ struct pci_vtscsi_ctrl_tmf *tmf) { union ctl_io *io; + int target; int err; + int fd; - if (pci_vtscsi_check_lun(tmf->lun) == false) { + if (pci_vtscsi_check_lun(sc, tmf->lun) == false) { DPRINTF("TMF request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-" "%.2hhx%.2hhx-%.2hhx%.2hhx", tmf->lun[0], tmf->lun[1], tmf->lun[2], tmf->lun[3], tmf->lun[4], tmf->lun[5], @@ -487,6 +528,10 @@ return; } + target = pci_vtscsi_get_target(sc, tmf->lun); + + fd = sc->vss_targets[target].vst_fd; + io = ctl_scsi_alloc_io(sc->vss_iid); if (io == NULL) { WPRINTF("failed to allocate ctl_io: err=%d (%s)", @@ -500,7 +545,7 @@ io->io_hdr.io_type = CTL_IO_TASK; io->io_hdr.nexus.initid = sc->vss_iid; - io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(tmf->lun); + io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(sc, tmf->lun); io->taskio.tag_type = CTL_TAG_SIMPLE; io->taskio.tag_num = tmf->id; io->io_hdr.flags |= CTL_FLAG_USER_TAG; @@ -547,7 +592,7 @@ sbuf_delete(sb); } - err = ioctl(sc->vss_ctl_fd, CTL_IO, io); + err = ioctl(fd, CTL_IO, io); if (err != 0) WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); @@ -714,7 +759,7 @@ (void **)&req->vsr_cmd_rd) == VTSCSI_IN_HEADER_LEN(q->vsq_sc)); /* Make sure this request addresses a valid LUN. */ - if (pci_vtscsi_check_lun(req->vsr_cmd_rd->lun) == false) { + if (pci_vtscsi_check_lun(sc, req->vsr_cmd_rd->lun) == false) { DPRINTF("I/O request to invalid LUN " "%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx", req->vsr_cmd_rd->lun[0], req->vsr_cmd_rd->lun[1], @@ -770,7 +815,7 @@ } static int -pci_vtscsi_request_handle(struct pci_vtscsi_softc *sc, +pci_vtscsi_request_handle(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_request *req) { union ctl_io *io = req->vsr_ctl_io; @@ -779,7 +824,8 @@ int err, nxferred; io->io_hdr.nexus.initid = sc->vss_iid; - io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(req->vsr_cmd_rd->lun); + io->io_hdr.nexus.targ_lun = + pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun); io->io_hdr.io_type = CTL_IO_SCSI; @@ -830,7 +876,7 @@ sbuf_delete(sb); } - err = ioctl(sc->vss_ctl_fd, CTL_IO, io); + err = ioctl(fd, CTL_IO, io); if (err != 0) { WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; @@ -939,30 +985,201 @@ return (0); } +/* + * Create a target config node, return target id. If the target number isn't + * given as part of the path argument, use last_id + 1. + */ +static int +pci_vtscsi_add_target_config(nvlist_t *nvl, const char *path, int last_id) +{ + long target; + char tmp[4]; + + if (path == NULL) { + EPRINTLN("target path must be specified"); + return (-1); + } + + if (path[0] != '/') { + char *endptr; + + target = strtoul(path, &endptr, 10); + if (endptr == path || endptr[0] != ':' || target < 0) + return (-1); + + path = endptr + 1; + } else { + target = last_id + 1; + } + + if (target > VIRTIO_SCSI_MAX_TARGET) { + EPRINTLN("max target (%d) reached, can't add another", + VIRTIO_SCSI_MAX_TARGET); + return (-1); + } + + snprintf(tmp, sizeof(tmp), "%ld", target); + set_config_value_node(nvl, tmp, path); + + return (target); +} + +/* + * The following forms are accepted for legacy config options to configure a + * single target: + * + * (0) -s B:D:F,virtio-scsi + * (1) -s B:D:F,virtio-scsi, + * (2) -s B:D:F,virtio-scsi,,,... + * (3) -s B:D:F,virtio-scsi,,... + * (4) -s B:D:F,virtio-scsi, + * + * To configure multiple targets, the following form is accepted: + * (5) -s B:D:F,virtio-scsi,[target=,...] + */ static int pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts) { - char *cp, *devname; + int last_id = -1; + char *config, *tofree, *name, *value; + nvlist_t *targets; + size_t n; + + /* Make sure no one accidentally sets "dev" anymore. */ + (void) create_relative_config_node(nvl, "dev"); + + targets = create_relative_config_node(nvl, "target"); - if (opts == NULL) + /* Handle legacy form (0). */ + if (opts == NULL) { + pci_vtscsi_add_target_config(targets, "/dev/cam/ctl", 0); return (0); + } + + n = strcspn(opts, ",="); + + /* Handle legacy form (1) and (2). */ + if (opts[n] == ',' || opts[n] == '\0') { + char *tmp = strndup(opts, n); - cp = strchr(opts, ','); - if (cp == NULL) { - set_config_value_node(nvl, "dev", opts); + last_id = pci_vtscsi_add_target_config(targets, tmp, last_id); + free(tmp); + + if (last_id < 0) + return (-1); + + opts += n; + if (opts[0] == ',' && opts[1] != '\0') + opts++; + } + + /* If this was form (1), we're done. */ + if (opts[0] == '\0') return (0); + + /* + * For form (2), (3), (4), and (5), parse the remaining options. + * + * Contrary to other options, multiple target= options create a new + * target for each such option. + * + * For compatibility reasons we also accept dev= options for + * targets. + */ + config = tofree = strdup(opts); + while ((name = strsep(&config, ",")) != NULL) { + value = strchr(name, '='); + if (value != NULL) + *value++ = '\0'; + + if (strcmp(name, "dev") == 0 || strcmp(name, "target") == 0) { + int new_id = pci_vtscsi_add_target_config(targets, + value, last_id); + + if (new_id < 0) { + free(tofree); + return (-1); + } + + if (new_id > last_id) + last_id = new_id; + + } else if (value != NULL) { + set_config_value_node(nvl, name, value); + } else { + set_config_bool_node(nvl, name, true); + } + } + + free(tofree); + return (0); +} + +static int +pci_vtscsi_count_targets(const char *prefix __unused, + const nvlist_t *parent __unused, const char *name, int type, void *arg) +{ + struct pci_vtscsi_softc *sc = arg; + char *endptr; + long target; + + if (type != NV_TYPE_STRING) { + EPRINTLN("invalid target \"%s\" type: not a string", name); + errno = EINVAL; + return (-1); } - devname = strndup(opts, cp - opts); - set_config_value_node(nvl, "dev", devname); - free(devname); - return (pci_parse_legacy_config(nvl, cp + 1)); + + target = strtol(name, &endptr, 10); + assert(endptr[0] == '\0'); + assert(target >= 0); + assert(target <= VIRTIO_SCSI_MAX_TARGET); + + if (target >= sc->vss_num_target) + sc->vss_num_target = target + 1; + + return (0); +} + +static int +pci_vtscsi_init_target(const char *prefix __unused, const nvlist_t *parent, + const char *name, int type, void *arg) +{ + struct pci_vtscsi_softc *sc = arg; + const char *value; + char *endptr; + long target; + + assert(type == NV_TYPE_STRING); + + /* + * Get the numeric value of the target id from 'name'. + */ + target = strtol(name, &endptr, 10); + assert(endptr[0] == '\0'); + assert(target >= 0); + assert(target <= VIRTIO_SCSI_MAX_TARGET); + assert(target <= sc->vss_num_target); + sc->vss_targets[target].vst_target = target; + + /* + * 'value' contains the CTL device node path of this target. + */ + value = nvlist_get_string(parent, name); + sc->vss_targets[target].vst_fd = open(value, O_RDWR); + if (sc->vss_targets[target].vst_fd < 0) { + EPRINTLN("cannot open target %ld at %s: %s", target, value, + strerror(errno)); + return (-1); + } + + return (0); } static int pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtscsi_softc *sc; - const char *devname, *value; + const char *value; int err; int i; @@ -978,19 +1195,42 @@ if (value != NULL) { if (pci_emul_add_boot_device(pi, atoi(value))) { EPRINTLN("Invalid bootindex %d", atoi(value)); - free(sc); - return (-1); + errno = EINVAL; + goto fail; } } - devname = get_config_value_node(nvl, "dev"); - if (devname == NULL) - devname = "/dev/cam/ctl"; - sc->vss_ctl_fd = open(devname, O_RDWR); - if (sc->vss_ctl_fd < 0) { - WPRINTF("cannot open %s: %s", devname, strerror(errno)); - free(sc); - return (1); + nvl = find_relative_config_node(nvl, "target"); + if (nvl != NULL) { + err = walk_config_nodes("", nvl, sc, pci_vtscsi_count_targets); + if (err != 0) + goto fail; + } + + if (sc->vss_num_target > 0) { + sc->vss_targets = malloc(sc->vss_num_target * + sizeof(struct pci_vtscsi_target)); + memset(sc->vss_targets, -1, sc->vss_num_target * + sizeof(struct pci_vtscsi_target)); + + if (sc->vss_targets == NULL) { + EPRINTLN("can't allocate space for %d targets", + sc->vss_num_target); + goto fail; + } + + err = walk_config_nodes("", nvl, sc, pci_vtscsi_init_target); + if (err != 0) + goto fail; + } + + /* + * All targets should be open now and have a valid fd. + */ + for (i = 0; i < sc->vss_num_target; i++) { + if (sc->vss_targets[i].vst_target == i && + sc->vss_targets[i].vst_fd < 0) + goto fail; } pthread_mutex_init(&sc->vss_mtx, NULL); @@ -1025,8 +1265,10 @@ sc->vss_vq[i].vq_notify = pci_vtscsi_requestq_notify; err = pci_vtscsi_init_queue(sc, &sc->vss_queues[i - 2], i - 2); - if (err != 0) - return (err); + if (err != 0) { + free(sc->vss_targets); + goto fail; + } } /* initialize config space */ @@ -1036,11 +1278,19 @@ pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_SCSI); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR); - if (vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix())) - return (1); + err = vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix()); + if (err != 0) { + free(sc->vss_targets); + goto fail; + } + vi_set_io_bar(&sc->vss_vs, 0); return (0); + +fail: + free(sc); + return (-1); }