diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8 --- a/usr.sbin/bhyve/bhyve.8 +++ b/usr.sbin/bhyve/bhyve.8 @@ -1,4 +1,5 @@ .\" Copyright (c) 2013 Peter Grehan +.\" Copyright 2025 Hans Rosenfeld .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -22,7 +23,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd August 21, 2024 +.Dd October 20, 2025 .Dt BHYVE 8 .Os .Sh NAME @@ -626,10 +627,28 @@ .Bl -bullet .Sm off .It -.Pa /dev/cam/ctl Oo Ar pp Cm \&. Ar vp Oc Oo Cm \&, Ar scsi-device-options Oc +.Oo Cm target Ns = Ns Ar /dev/cam/ctl Oo Ar pp Cm \&. Ar vp Oc Oc +.Oo Cm \&, Ar scsi-device-options Oc .Sm on .El .Pp +Multiple +.Pa target +parameters may be specified, each configuring a different +.Ar path +as distinct SCSI target. +The +.Ar path +must point to a valid CAM target layer +.Po CTL +.Pc +device node. +If no +.Pa target +is configured, a single default target backed by +.Sy /dev/cam/ctl +will be created. +.Pp The .Ar scsi-device-options are: diff --git a/usr.sbin/bhyve/bhyve_config.5 b/usr.sbin/bhyve/bhyve_config.5 --- a/usr.sbin/bhyve/bhyve_config.5 +++ b/usr.sbin/bhyve/bhyve_config.5 @@ -1,6 +1,7 @@ .\" SPDX-License-Identifier: BSD-2-Clause .\" .\" Copyright (c) 2021 John H. Baldwin +.\" Copyright 2025 Hans Rosenfeld .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions @@ -23,7 +24,7 @@ .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" -.Dd August 21, 2024 +.Dd October 20, 2025 .Dt BHYVE_CONFIG 5 .Os .Sh NAME @@ -751,14 +752,20 @@ The largest supported MTU advertised to the guest. .El .Ss VirtIO SCSI Settings -.Bl -column "Name" "integer" "Default" +.Bl -column "target" "integer" "/dev/cam/ctl" .It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description -.It Va dev Ta path Ta Ta -The path of a CAM target layer (CTL) device to export: -.Pa /dev/cam/ctl Ns Oo Ar pp . Ns Ar vp Oc . .It Va iid Ta integer Ta 0 Ta Initiator ID to use when sending requests to the CTL port. +.It Va target Ta path Ta Sy /dev/cam/ctl Ta +The path of a CAM target layer (CTL) device to use: +.Pa /dev/cam/ctl Ns Oo Ar pp . Ns Ar vp Oc . .El +.Pp +The +.Va target +variable may be specified multiple times with different +.Ar path +arguments to configure multiple distinct SCSI targets. .Sh SEE ALSO .Xr expand_number 3 , .Xr getaddrinfo 3 , diff --git a/usr.sbin/bhyve/pci_virtio_scsi.c b/usr.sbin/bhyve/pci_virtio_scsi.c --- a/usr.sbin/bhyve/pci_virtio_scsi.c +++ b/usr.sbin/bhyve/pci_virtio_scsi.c @@ -77,7 +77,7 @@ (sizeof(struct pci_vtscsi_req_cmd_wr) + _sc->vss_config.sense_size) #define VIRTIO_SCSI_MAX_CHANNEL 0 -#define VIRTIO_SCSI_MAX_TARGET 0 +#define VIRTIO_SCSI_MAX_TARGET 255 #define VIRTIO_SCSI_MAX_LUN 16383 #define VIRTIO_SCSI_F_INOUT (1 << 0) @@ -140,6 +140,12 @@ STAILQ_ENTRY(pci_vtscsi_request) vsr_link; }; +struct pci_vtscsi_target { + int vst_target; + int vst_fd; + int vst_max_sectors; +}; + /* * Per-device softc */ @@ -151,7 +157,9 @@ int vss_iid; int vss_ctl_fd; uint32_t vss_features; + int vss_num_target; struct pci_vtscsi_config vss_config; + struct pci_vtscsi_target *vss_targets; }; #define VIRTIO_SCSI_T_TMF 0 @@ -242,8 +250,12 @@ static int pci_vtscsi_cfgread(void *, int, int, uint32_t *); static int pci_vtscsi_cfgwrite(void *, int, int, uint32_t); -static inline bool pci_vtscsi_check_lun(const uint8_t *); -static inline int pci_vtscsi_get_lun(const uint8_t *); +static inline bool pci_vtscsi_check_lun(struct pci_vtscsi_softc *, + const uint8_t *); +static inline int pci_vtscsi_get_target(struct pci_vtscsi_softc *, + const uint8_t *); +static inline int pci_vtscsi_get_lun(struct pci_vtscsi_softc *, + const uint8_t *); static void pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *, size_t); static void pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *, @@ -262,13 +274,15 @@ struct vqueue_info *); static void pci_vtscsi_return_request(struct pci_vtscsi_queue *, struct pci_vtscsi_request *, int); -static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *, +static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); static void pci_vtscsi_controlq_notify(void *, struct vqueue_info *); static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *); static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *); -static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, + +static int pci_vtscsi_add_target_config(nvlist_t *, const char *); +static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, struct pci_vtscsi_queue *, int); static int pci_vtscsi_init(struct pci_devinst *, nvlist_t *); @@ -289,10 +303,12 @@ struct pci_vtscsi_worker *worker = (struct pci_vtscsi_worker *)arg; struct pci_vtscsi_queue *q = worker->vsw_queue; struct pci_vtscsi_softc *sc = q->vsq_sc; - int iolen; for (;;) { struct pci_vtscsi_request *req; + int target; + int iolen; + int fd; pthread_mutex_lock(&q->vsq_rmtx); @@ -307,11 +323,15 @@ req = pci_vtscsi_get_request(&q->vsq_requests); pthread_mutex_unlock(&q->vsq_rmtx); - DPRINTF("I/O request lun %d, data_niov_in %d, data_niov_out %d", - pci_vtscsi_get_lun(req->vsr_cmd_rd->lun), + target = pci_vtscsi_get_target(sc, req->vsr_cmd_rd->lun); + fd = sc->vss_targets[target].vst_fd; + + DPRINTF("I/O request tgt %d, lun %d, data_niov_in %d, " + "data_niov_out %d", target, + pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun), req->vsr_data_niov_in, req->vsr_data_niov_out); - iolen = pci_vtscsi_request_handle(sc, req); + iolen = pci_vtscsi_request_handle(sc, fd, req); pci_vtscsi_return_request(q, req, iolen); } @@ -339,7 +359,7 @@ .sense_size = 96, .cdb_size = 32, .max_channel = VIRTIO_SCSI_MAX_CHANNEL, - .max_target = VIRTIO_SCSI_MAX_TARGET, + .max_target = MAX(1, sc->vss_num_target) - 1, .max_lun = VIRTIO_SCSI_MAX_LUN }; } @@ -372,11 +392,11 @@ /* * Check that the given LUN address conforms to the virtio spec, does not - * address a target other than 0, and especially does not address the - * REPORT_LUNS well-known logical unit. + * address an unknown target, and especially does not address the REPORT_LUNS + * well-known logical unit. */ static inline bool -pci_vtscsi_check_lun(const uint8_t *lun) +pci_vtscsi_check_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) { /* * The virtio spec says that we SHOULD implement the REPORT_LUNS well- @@ -389,8 +409,8 @@ if (lun[0] != 0x01) return (false); - /* Next comes the target. We currently only support target 0. */ - if (lun[1] != 0x00) + /* Next comes the target. */ + if (lun[1] >= sc->vss_num_target) return (false); /* @@ -417,10 +437,20 @@ } static inline int -pci_vtscsi_get_lun(const uint8_t *lun) +pci_vtscsi_get_target(struct pci_vtscsi_softc *sc, const uint8_t *lun) +{ + assert(lun[0] == 0x01); + assert(lun[1] < sc->vss_num_target); + assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); + + return (lun[1]); +} + +static inline int +pci_vtscsi_get_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) { assert(lun[0] == 0x01); - assert(lun[1] == 0x00); + assert(lun[1] < sc->vss_num_target); assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); return (((lun[2] << 8) | lun[3]) & 0x3fff); @@ -463,9 +493,11 @@ struct pci_vtscsi_ctrl_tmf *tmf) { union ctl_io *io; + int target; int err; + int fd; - if (pci_vtscsi_check_lun(tmf->lun) == false) { + if (pci_vtscsi_check_lun(sc, tmf->lun) == false) { WPRINTF("TMF request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-" "%.2hhx%.2hhx-%.2hhx%.2hhx", tmf->lun[0], tmf->lun[1], tmf->lun[2], tmf->lun[3], tmf->lun[4], tmf->lun[5], @@ -475,6 +507,10 @@ return; } + target = pci_vtscsi_get_target(sc, tmf->lun); + + fd = sc->vss_targets[target].vst_fd; + io = ctl_scsi_alloc_io(sc->vss_iid); if (io == NULL) { WPRINTF("failed to allocate ctl_io: err=%d (%s)", @@ -488,7 +524,7 @@ io->io_hdr.io_type = CTL_IO_TASK; io->io_hdr.nexus.initid = sc->vss_iid; - io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(tmf->lun); + io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(sc, tmf->lun); io->taskio.tag_type = CTL_TAG_SIMPLE; io->taskio.tag_num = tmf->id; io->io_hdr.flags |= CTL_FLAG_USER_TAG; @@ -535,7 +571,7 @@ sbuf_delete(sb); } - err = ioctl(sc->vss_ctl_fd, CTL_IO, io); + err = ioctl(fd, CTL_IO, io); if (err != 0) WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); @@ -702,7 +738,7 @@ (void **)&req->vsr_cmd_rd) == VTSCSI_IN_HEADER_LEN(q->vsq_sc)); /* Make sure this request addresses a valid LUN. */ - if (pci_vtscsi_check_lun(req->vsr_cmd_rd->lun) == false) { + if (pci_vtscsi_check_lun(sc, req->vsr_cmd_rd->lun) == false) { WPRINTF("I/O request to invalid LUN " "%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx", req->vsr_cmd_rd->lun[0], req->vsr_cmd_rd->lun[1], @@ -758,7 +794,7 @@ } static int -pci_vtscsi_request_handle(struct pci_vtscsi_softc *sc, +pci_vtscsi_request_handle(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_request *req) { union ctl_io *io = req->vsr_ctl_io; @@ -767,7 +803,8 @@ int err, nxferred; io->io_hdr.nexus.initid = sc->vss_iid; - io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(req->vsr_cmd_rd->lun); + io->io_hdr.nexus.targ_lun = + pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun); io->io_hdr.io_type = CTL_IO_SCSI; @@ -818,7 +855,7 @@ sbuf_delete(sb); } - err = ioctl(sc->vss_ctl_fd, CTL_IO, io); + err = ioctl(fd, CTL_IO, io); if (err != 0) { WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; @@ -927,30 +964,176 @@ return (0); } +/* + * Create a target config node, return target id. The target ids are set + * sequentially beginning with 0. + */ +static int +pci_vtscsi_add_target_config(nvlist_t *nvl, const char *path) +{ + static int target = 0; + char tmp[4]; + + if (path == NULL) { + EPRINTLN("target path must be specified"); + return (-1); + } + + if (target > VIRTIO_SCSI_MAX_TARGET) { + EPRINTLN("max target (%d) reached, can't add another", + VIRTIO_SCSI_MAX_TARGET); + return (-1); + } + + snprintf(tmp, sizeof(tmp), "%d", target); + set_config_value_node(nvl, tmp, path); + + return (target++); +} + +/* + * The following forms are accepted for legacy config options to configure a + * single target: + * + * (0) -s B:D:F,virtio-scsi + * (1) -s B:D:F,virtio-scsi, + * (2) -s B:D:F,virtio-scsi,,,... + * (3) -s B:D:F,virtio-scsi,,... + * (4) -s B:D:F,virtio-scsi, + * + * To configure multiple targets, the following form is accepted: + * -s B:D:F,virtio-scsi,[target=,...] + */ static int pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts) { - char *cp, *devname; + int last_id = -1; + char *config, *tofree, *name, *value; + nvlist_t *targets; + size_t n; + + /* Make sure no one accidentally sets "dev" anymore. */ + (void) create_relative_config_node(nvl, "dev"); - if (opts == NULL) + targets = create_relative_config_node(nvl, "target"); + + /* Handle legacy form (0). */ + if (opts == NULL) { + pci_vtscsi_add_target_config(targets, "/dev/cam/ctl"); return (0); + } + + n = strcspn(opts, ",="); - cp = strchr(opts, ','); - if (cp == NULL) { - set_config_value_node(nvl, "dev", opts); + /* Handle legacy form (1) and (2). */ + if (opts[n] == ',' || opts[n] == '\0') { + char *tmp = strndup(opts, n); + + last_id = pci_vtscsi_add_target_config(targets, tmp); + free(tmp); + + if (last_id < 0) + return (-1); + + opts += n; + if (opts[0] == ',' && opts[1] != '\0') + opts++; + } + + /* If this was form (1), we're done. */ + if (opts[0] == '\0') return (0); + + /* + * For form (2), (3), and (4), parse the remaining options. + * + * Contrary to other options, multiple target= options create a new + * target for each such option. + * + * For compatibility reasons we also accept dev= options for + * targets. + */ + config = tofree = strdup(opts); + while ((name = strsep(&config, ",")) != NULL) { + value = strchr(name, '='); + if (value != NULL) + *value++ = '\0'; + + if (strcmp(name, "dev") == 0 || strcmp(name, "target") == 0) { + last_id = pci_vtscsi_add_target_config(targets, value); + if (last_id < 0) { + free(tofree); + return (-1); + } + } else if (value != NULL) { + set_config_value_node(nvl, name, value); + } else { + set_config_bool_node(nvl, name, true); + } } - devname = strndup(opts, cp - opts); - set_config_value_node(nvl, "dev", devname); - free(devname); - return (pci_parse_legacy_config(nvl, cp + 1)); + + free(tofree); + return (0); +} + +static int +pci_vtscsi_count_targets(const char *prefix __unused, + const nvlist_t *parent __unused, const char *name __unused, int type, + void *arg) +{ + struct pci_vtscsi_softc *sc = arg; + + if (type != NV_TYPE_STRING) { + EPRINTLN("invalid target \"%s\" type: not a string", name); + errno = EINVAL; + return (-1); + } + + sc->vss_num_target++; + + return (0); +} + +static int +pci_vtscsi_init_target(const char *prefix __unused, const nvlist_t *parent, + const char *name, int type, void *arg) +{ + struct pci_vtscsi_softc *sc = arg; + const char *value; + char *endptr; + long target; + + assert(type == NV_TYPE_STRING); + + /* + * Get the numeric value of the target id from 'name'. + */ + target = strtol(name, &endptr, 10); + assert(endptr[0] == '\0'); + assert(target >= 0); + assert(target <= VIRTIO_SCSI_MAX_TARGET); + assert(target <= sc->vss_num_target); + sc->vss_targets[target].vst_target = target; + + /* + * 'value' contains the CTL device node path of this target. + */ + value = nvlist_get_string(parent, name); + sc->vss_targets[target].vst_fd = open(value, O_RDWR); + if (sc->vss_targets[target].vst_fd < 0) { + EPRINTLN("cannot open target %ld at %s: %s", target, value, + strerror(errno)); + return (-1); + } + + return (0); } static int pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtscsi_softc *sc; - const char *devname, *value; + const char *value; int err; int i; @@ -966,21 +1149,39 @@ if (value != NULL) { if (pci_emul_add_boot_device(pi, atoi(value))) { EPRINTLN("Invalid bootindex %d", atoi(value)); - free(sc); - return (-1); + errno = EINVAL; + goto fail; } } - devname = get_config_value_node(nvl, "dev"); - if (devname == NULL) - devname = "/dev/cam/ctl"; - sc->vss_ctl_fd = open(devname, O_RDWR); - if (sc->vss_ctl_fd < 0) { - WPRINTF("cannot open %s: %s", devname, strerror(errno)); - free(sc); - return (1); + nvl = find_relative_config_node(nvl, "target"); + if (nvl != NULL) { + err = walk_config_nodes("", nvl, sc, pci_vtscsi_count_targets); + if (err != 0) + goto fail; + } + + if (sc->vss_num_target > 0) { + sc->vss_targets = calloc(sc->vss_num_target, + sizeof(struct pci_vtscsi_target)); + + if (sc->vss_targets == NULL) { + EPRINTLN("can't allocate space for %d targets", + sc->vss_num_target); + goto fail; + } + + err = walk_config_nodes("", nvl, sc, pci_vtscsi_init_target); + if (err != 0) + goto fail; } + /* + * All targets should be open now and have a valid fd. + */ + for (i = 0; i < sc->vss_num_target; i++) + assert(sc->vss_targets[i].vst_fd >= 0); + pthread_mutex_init(&sc->vss_mtx, NULL); vi_softc_linkup(&sc->vss_vs, &vtscsi_vi_consts, sc, pi, sc->vss_vq); @@ -1013,8 +1214,10 @@ sc->vss_vq[i].vq_notify = pci_vtscsi_requestq_notify; err = pci_vtscsi_init_queue(sc, &sc->vss_queues[i - 2], i - 2); - if (err != 0) - return (err); + if (err != 0) { + free(sc->vss_targets); + goto fail; + } } /* initialize config space */ @@ -1024,11 +1227,19 @@ pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_SCSI); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR); - if (vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix())) - return (1); + err = vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix()); + if (err != 0) { + free(sc->vss_targets); + goto fail; + } + vi_set_io_bar(&sc->vss_vs, 0); return (0); + +fail: + free(sc); + return (-1); }