Page MenuHomeFreeBSD
Files F153043236

D54073.id167548.diff
No OneTemporary
Actions

D54073.id167548.diff
View Options

diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8
--- a/usr.sbin/bhyve/bhyve.8
+++ b/usr.sbin/bhyve/bhyve.8
@@ -683,6 +683,18 @@
A fwcfg file is used to specify the bootorder.
The guest firmware may ignore or doesn't support this fwcfg file.
In that case, this feature doesn't work as expected.
+.It Li ctl_ringsz= Ns Ar ringsz
+The ring size to use for the control queue.
+.It Li evt_ringsz= Ns Ar ringsz
+The ring size to use for the event queue.
+.It Li req_ringsz= Ns Ar ringsz
+The ring size to use for each I/O request queues.
+.It Li num_queues= Ns Ar num
+The number of I/O request queues to use.
+.It Li seg_max= Ns Ar num
+The maximum number of segments allowed in a single command.
+.It Li thr_per_q= Ns Ar num
+The number of parallel request processing threads per I/O request queue.
.El
.Pp
The
diff --git a/usr.sbin/bhyve/bhyve_config.5 b/usr.sbin/bhyve/bhyve_config.5
--- a/usr.sbin/bhyve/bhyve_config.5
+++ b/usr.sbin/bhyve/bhyve_config.5
@@ -752,10 +752,22 @@
The largest supported MTU advertised to the guest.
.El
.Ss VirtIO SCSI Settings
-.Bl -column "backend" "string" "/dev/cam/ctl"
+.Bl -column "num_queues" "Format" "Default"
.It Sy Name Ta Sy Format Ta Sy Default Ta Sy Description
.It Va backend Ta string Ta ctl Ta
The virtio-scsi backend to use (case-insensitive).
+.It Va ctl_ringsz Ta integer Ta 64 Ta
+The ring size of the control queue.
+.It Va evt_ringsz Ta integer Ta 64 Ta
+The ring size of the event queue.
+.It Va req_ringsz Ta integer Ta 64 Ta
+The ring size of each I/O request queue.
+.It Va num_queues Ta integer Ta 1 Ta
+The number of I/O request queues.
+.It Va seg_max Ta integer Ta 64 Ta
+The maximum number of segments allowed in a single command.
+.It Va thr_per_q Ta integer Ta 16 Ta
+The number of parallel request processing threads per I/O request queue.
.It Va target Ta Oo Va ID : Oc Ns path Ta /dev/cam/ctl Ta
The backend
.Ar path
diff --git a/usr.sbin/bhyve/pci_virtio_scsi.h b/usr.sbin/bhyve/pci_virtio_scsi.h
--- a/usr.sbin/bhyve/pci_virtio_scsi.h
+++ b/usr.sbin/bhyve/pci_virtio_scsi.h
@@ -47,12 +47,15 @@
#define VIRTIO_SCSI_F_HOTPLUG (1 << 1)
#define VIRTIO_SCSI_F_CHANGE (1 << 2)
-/* Limits which we set. These should really be configurable. */
-#define VTSCSI_RINGSZ 64
-#define VTSCSI_REQUESTQ 1
-#define VTSCSI_THR_PER_Q 16
-#define VTSCSI_MAXQ (VTSCSI_REQUESTQ + 2)
-#define VTSCSI_MAXSEG 64
+/* Default limits which we set. Some of these are configurable. */
+#define VTSCSI_DEF_RINGSZ 64
+#define VTSCSI_MIN_RINGSZ 4
+#define VTSCSI_DEF_THR_PER_Q 16
+#define VTSCSI_MIN_THR_PER_Q 1
+#define VTSCSI_DEF_MAXSEG 64
+#define VTSCSI_MIN_MAXSEG 3
+#define VTSCSI_DEF_REQUESTQ 1
+#define VTSCSI_MIN_REQUESTQ 1
/*
* Device-specific config space registers
@@ -84,13 +87,13 @@
/*
* I/O request state
*
- * Each pci_vtscsi_queue has VTSCSI_RINGSZ pci_vtscsi_request structures
- * pre-allocated on vsq_free_requests. For each I/O request coming in on
- * the I/O virtqueue, the request queue handler takes pci_vtscsi_request
- * off vsq_free_requests, fills in the data from the I/O virtqueue, puts
- * it on vsq_requests, and signals vsq_cv.
+ * Each pci_vtscsi_queue has configurable number of pci_vtscsi_request
+ * structures pre-allocated on vsq_free_requests. For each I/O request
+ * coming in on the I/O virtqueue, the request queue handler takes
+ * pci_vtscsi_request off vsq_free_requests, fills in the data from the
+ * I/O virtqueue, puts it on vsq_requests, and signals vsq_cv.
*
- * There are VTSCSI_THR_PER_Q worker threads for each pci_vtscsi_queue
+ * Each pci_vtscsi_queue will have a configurable number of worker threads,
* which wait on vsq_cv. When signalled, they repeatedly take a single
* pci_vtscsi_request off vsq_requests and hand it to the backend, which
* processes it synchronously. After completion, the pci_vtscsi_request
@@ -121,7 +124,7 @@
struct pci_vtscsi_request {
struct pci_vtscsi_queue *vsr_queue;
- struct iovec vsr_iov[VTSCSI_MAXSEG + 2];
+ struct iovec *vsr_iov;
struct iovec *vsr_iov_in;
struct iovec *vsr_iov_out;
struct iovec *vsr_data_iov_in;
@@ -151,11 +154,17 @@
*/
struct pci_vtscsi_softc {
struct virtio_softc vss_vs;
- struct vqueue_info vss_vq[VTSCSI_MAXQ];
- struct pci_vtscsi_queue vss_queues[VTSCSI_REQUESTQ];
+ struct virtio_consts vss_vi_consts;
+ struct vqueue_info *vss_vq;
+ struct pci_vtscsi_queue *vss_queues;
pthread_mutex_t vss_mtx;
uint32_t vss_features;
int vss_num_target;
+ uint32_t vss_ctl_ringsz;
+ uint32_t vss_evt_ringsz;
+ uint32_t vss_req_ringsz;
+ uint32_t vss_thr_per_q;
+ struct pci_vtscsi_config vss_default_config;
struct pci_vtscsi_config vss_config;
struct pci_vtscsi_target *vss_targets;
struct pci_vtscsi_backend *vss_backend;
diff --git a/usr.sbin/bhyve/pci_virtio_scsi.c b/usr.sbin/bhyve/pci_virtio_scsi.c
--- a/usr.sbin/bhyve/pci_virtio_scsi.c
+++ b/usr.sbin/bhyve/pci_virtio_scsi.c
@@ -119,7 +119,7 @@
static struct virtio_consts vtscsi_vi_consts = {
.vc_name = "vtscsi",
- .vc_nvq = VTSCSI_MAXQ,
+ .vc_nvq = VTSCSI_DEF_REQUESTQ + 2,
.vc_cfgsize = sizeof(struct pci_vtscsi_config),
.vc_reset = pci_vtscsi_reset,
.vc_cfgread = pci_vtscsi_cfgread,
@@ -128,6 +128,20 @@
.vc_hv_caps = VIRTIO_RING_F_INDIRECT_DESC,
};
+static const struct pci_vtscsi_config vtscsi_config = {
+ .num_queues = VTSCSI_DEF_REQUESTQ,
+ /* Leave room for the request and the response. */
+ .seg_max = VTSCSI_DEF_MAXSEG - 2,
+ .max_sectors = 0,
+ .cmd_per_lun = 1,
+ .event_info_size = sizeof(struct pci_vtscsi_event),
+ .sense_size = 96,
+ .cdb_size = 32,
+ .max_channel = VIRTIO_SCSI_MAX_CHANNEL,
+ .max_target = VIRTIO_SCSI_MAX_TARGET,
+ .max_lun = VIRTIO_SCSI_MAX_LUN
+};
+
int pci_vtscsi_debug = 0;
@@ -198,19 +212,9 @@
vi_reset_dev(&sc->vss_vs);
/* initialize config structure */
- sc->vss_config = (struct pci_vtscsi_config){
- .num_queues = VTSCSI_REQUESTQ,
- /* Leave room for the request and the response. */
- .seg_max = VTSCSI_MAXSEG - 2,
- .max_sectors = 0, /* overridden by backend reset() */
- .cmd_per_lun = 1,
- .event_info_size = sizeof(struct pci_vtscsi_event),
- .sense_size = 96,
- .cdb_size = 32,
- .max_channel = VIRTIO_SCSI_MAX_CHANNEL,
- .max_target = MAX(1, sc->vss_num_target) - 1,
- .max_lun = VIRTIO_SCSI_MAX_LUN
- };
+ sc->vss_config = sc->vss_default_config;
+
+ sc->vss_config.max_target = MAX(1, sc->vss_num_target) - 1;
sc->vss_backend->vsb_reset(sc);
}
@@ -527,7 +531,7 @@
* must guarantee to adhere to the ordering requirements for any TMF
* function which aborts tasks.
*/
- for (int i = 0; i < VTSCSI_REQUESTQ; i++) {
+ for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) {
struct pci_vtscsi_queue *q = &sc->vss_queues[i];
pthread_mutex_lock(&q->vsq_rmtx);
@@ -568,20 +572,21 @@
}
} else {
pci_vtscsi_walk_t ret = PCI_VTSCSI_WALK_CONTINUE;
- int i;
+ uint32_t i;
- for (i = 0;
- i < VTSCSI_REQUESTQ && ret != PCI_VTSCSI_WALK_STOP;
- i++) {
+ for (i = 0; i < sc->vss_config.num_queues; i++) {
struct pci_vtscsi_queue *q = &sc->vss_queues[i];
ret = pci_vtscsi_walk_request_queue(q,
pci_vtscsi_tmf_handler_cb[tmf->subtype], tmf);
+
+ if (ret == PCI_VTSCSI_WALK_STOP)
+ break;
}
}
/* Unlock the request queues before we return. */
- for (int i = 0; i < VTSCSI_REQUESTQ; i++) {
+ for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) {
struct pci_vtscsi_queue *q = &sc->vss_queues[i];
pthread_mutex_unlock(&q->vsq_rmtx);
@@ -654,6 +659,15 @@
if (req == NULL)
goto alloc_fail;
+ /*
+ * Allocate 4 extra segs:
+ * - 2 for the input and output headers
+ * - another 2 for split_iov()
+ */
+ req->vsr_iov = calloc(sc->vss_config.seg_max + 4, sizeof(struct iovec));
+ if (req->vsr_iov == NULL)
+ goto alloc_fail;
+
req->vsr_cmd_rd = calloc(1, VTSCSI_IN_HEADER_LEN(sc));
if (req->vsr_cmd_rd == NULL)
goto alloc_fail;
@@ -686,6 +700,8 @@
free(req->vsr_cmd_rd);
if (req->vsr_cmd_wr != NULL)
free(req->vsr_cmd_wr);
+ if (req->vsr_iov != NULL)
+ free(req->vsr_iov);
free(req);
}
@@ -723,8 +739,8 @@
assert(req != NULL);
pthread_mutex_unlock(&q->vsq_fmtx);
- n = vq_getchain(vq, req->vsr_iov, VTSCSI_MAXSEG, &vireq);
- assert(n >= 1 && n <= VTSCSI_MAXSEG);
+ n = vq_getchain(vq, req->vsr_iov, sc->vss_config.seg_max + 2, &vireq);
+ assert(n >= 1 && n <= (int)sc->vss_config.seg_max + 2);
req->vsr_idx = vireq.idx;
req->vsr_queue = q;
@@ -823,6 +839,7 @@
struct pci_vtscsi_request *req, int iolen)
{
struct pci_vtscsi_softc *sc = q->vsq_sc;
+ void *iov = req->vsr_iov;
void *cmd_rd = req->vsr_cmd_rd;
void *cmd_wr = req->vsr_cmd_wr;
void *backend = req->vsr_backend;
@@ -836,10 +853,12 @@
sc->vss_backend->vsb_req_clear(backend);
+ memset(iov, 0, sizeof(struct iovec) * sc->vss_config.seg_max + 4);
memset(cmd_rd, 0, VTSCSI_IN_HEADER_LEN(q->vsq_sc));
memset(cmd_wr, 0, VTSCSI_OUT_HEADER_LEN(q->vsq_sc));
memset(req, 0, sizeof(struct pci_vtscsi_request));
+ req->vsr_iov = iov;
req->vsr_cmd_rd = cmd_rd;
req->vsr_cmd_wr = cmd_wr;
req->vsr_backend = backend;
@@ -864,8 +883,12 @@
static void
pci_vtscsi_controlq_notify(void *vsc, struct vqueue_info *vq)
{
+ /*
+ * Control requests shouldn't require more than one iov. Allow for
+ * VTSCSI_MIN_MAXSEG, which is 3.
+ */
+ struct iovec iov[VTSCSI_MIN_MAXSEG];
struct pci_vtscsi_softc *sc;
- struct iovec iov[VTSCSI_MAXSEG];
struct vi_req req;
void *buf = NULL;
size_t bufsize;
@@ -874,8 +897,8 @@
sc = vsc;
while (vq_has_descs(vq)) {
- n = vq_getchain(vq, iov, VTSCSI_MAXSEG, &req);
- assert(n >= 1 && n <= VTSCSI_MAXSEG);
+ n = vq_getchain(vq, iov, VTSCSI_MIN_MAXSEG, &req);
+ assert(n >= 1 && n <= VTSCSI_MIN_MAXSEG);
bufsize = iov_to_buf(iov, n, &buf);
pci_vtscsi_control_handle(sc, buf, bufsize);
@@ -909,7 +932,7 @@
struct pci_vtscsi_queue *queue, int num)
{
char tname[MAXCOMLEN + 1];
- int i;
+ uint32_t i;
queue->vsq_sc = sc;
queue->vsq_vq = &sc->vss_vq[num + 2];
@@ -922,7 +945,7 @@
STAILQ_INIT(&queue->vsq_free_requests);
LIST_INIT(&queue->vsq_workers);
- for (i = 0; i < VTSCSI_RINGSZ; i++) {
+ for (i = 0; i < sc->vss_req_ringsz; i++) {
struct pci_vtscsi_request *req;
req = pci_vtscsi_alloc_request(sc);
@@ -932,7 +955,7 @@
pci_vtscsi_put_request(&queue->vsq_free_requests, req);
}
- for (i = 0; i < VTSCSI_THR_PER_Q; i++) {
+ for (i = 0; i < sc->vss_thr_per_q; i++) {
struct pci_vtscsi_worker *worker;
worker = calloc(1, sizeof(struct pci_vtscsi_worker));
if (worker == NULL)
@@ -1142,12 +1165,38 @@
return (ret);
}
+static int
+pci_vtscsi_get_config_num(nvlist_t *nvl, const char *name, uint32_t lim_lo,
+ uint32_t lim_hi, uint32_t *res)
+{
+ const char *value;
+ char *endptr;
+ unsigned long val;
+
+ value = get_config_value_node(nvl, name);
+ if (value == NULL)
+ return (0);
+
+ errno = 0;
+ val = strtoul(value, &endptr, 0);
+ if (errno != 0 || endptr[0] != '\0' ||
+ val < lim_lo || val > lim_hi) {
+ EPRINTLN("Invalid value for %s: %s", name, value);
+ errno = EINVAL;
+ return (-1);
+ }
+
+ *res = (uint32_t)val;
+ return (0);
+}
+
static int
pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl)
{
struct pci_vtscsi_softc *sc;
struct pci_vtscsi_backend *backend, **vbpp;
const char *value;
+ uint32_t val;
int err;
int i;
@@ -1155,6 +1204,13 @@
if (sc == NULL)
return (-1);
+ sc->vss_vi_consts = vtscsi_vi_consts;
+ sc->vss_ctl_ringsz = VTSCSI_DEF_RINGSZ;
+ sc->vss_evt_ringsz = VTSCSI_DEF_RINGSZ;
+ sc->vss_req_ringsz = VTSCSI_DEF_RINGSZ;
+ sc->vss_thr_per_q = VTSCSI_DEF_THR_PER_Q;
+ sc->vss_default_config = vtscsi_config;
+
value = get_config_value_node(nvl, "bootindex");
if (value != NULL) {
if (pci_emul_add_boot_device(pi, atoi(value))) {
@@ -1164,6 +1220,60 @@
}
}
+ val = vtscsi_config.seg_max;
+ if (pci_vtscsi_get_config_num(nvl, "seg_max", VTSCSI_MIN_MAXSEG,
+ INT32_MAX, &val) != 0)
+ goto fail;
+ sc->vss_default_config.seg_max = val;
+
+ val = vtscsi_config.num_queues;
+ if (pci_vtscsi_get_config_num(nvl, "num_queues", VTSCSI_MIN_REQUESTQ,
+ INT32_MAX - 2, &val) != 0)
+ goto fail;
+ sc->vss_default_config.num_queues = val;
+
+ /*
+ * num_queues is only the number of request queues, but nvq must
+ * account for the control and event queues.
+ */
+ sc->vss_vi_consts.vc_nvq = val + 2;
+
+ /*
+ * Allocate queues early, so that they're there for the call to
+ * vi_softc_linkup().
+ */
+ sc->vss_vq = calloc(sc->vss_vi_consts.vc_nvq,
+ sizeof(struct vqueue_info));
+ if (sc->vss_vq == NULL) {
+ EPRINTLN("can't allocate space for %d virtqueues",
+ sc->vss_vi_consts.vc_nvq);
+ goto fail;
+ }
+
+ sc->vss_queues = calloc(sc->vss_default_config.num_queues,
+ sizeof(struct pci_vtscsi_queue));
+ if (sc->vss_queues == NULL) {
+ EPRINTLN("can't allocate space for %d request queues",
+ sc->vss_config.num_queues);
+ goto fail;
+ }
+
+ if (pci_vtscsi_get_config_num(nvl, "ctl_ringsz", VTSCSI_MIN_RINGSZ,
+ INT32_MAX, &sc->vss_ctl_ringsz) != 0)
+ goto fail;
+
+ if (pci_vtscsi_get_config_num(nvl, "evt_ringsz", VTSCSI_MIN_RINGSZ,
+ INT32_MAX, &sc->vss_evt_ringsz) != 0)
+ goto fail;
+
+ if (pci_vtscsi_get_config_num(nvl, "req_ringsz", VTSCSI_MIN_RINGSZ,
+ INT32_MAX, &sc->vss_req_ringsz) != 0)
+ goto fail;
+
+ if (pci_vtscsi_get_config_num(nvl, "thr_per_q", VTSCSI_MIN_THR_PER_Q,
+ INT32_MAX, &sc->vss_thr_per_q) != 0)
+ goto fail;
+
value = get_config_value_node(nvl, "backend");
if (value == NULL) {
if (SET_COUNT(pci_vtscsi_backend_set) == 0) {
@@ -1228,7 +1338,7 @@
pthread_mutex_init(&sc->vss_mtx, NULL);
- vi_softc_linkup(&sc->vss_vs, &vtscsi_vi_consts, sc, pi, sc->vss_vq);
+ vi_softc_linkup(&sc->vss_vs, &sc->vss_vi_consts, sc, pi, sc->vss_vq);
sc->vss_vs.vs_mtx = &sc->vss_mtx;
/*
@@ -1244,17 +1354,18 @@
pci_vtscsi_reset(sc);
pthread_mutex_unlock(&sc->vss_mtx);
- /* controlq */
- sc->vss_vq[0].vq_qsize = VTSCSI_RINGSZ;
+ /* virtqueue 0: control queue */
+ sc->vss_vq[0].vq_qsize = sc->vss_ctl_ringsz;
sc->vss_vq[0].vq_notify = pci_vtscsi_controlq_notify;
- /* eventq */
- sc->vss_vq[1].vq_qsize = VTSCSI_RINGSZ;
+ /* virtqueue 1: event queue */
+ sc->vss_vq[1].vq_qsize = sc->vss_evt_ringsz;
sc->vss_vq[1].vq_notify = pci_vtscsi_eventq_notify;
- /* request queues */
- for (i = 2; i < VTSCSI_MAXQ; i++) {
- sc->vss_vq[i].vq_qsize = VTSCSI_RINGSZ;
+ /* virtqueue 2-n: request queues */
+
+ for (i = 2; i < sc->vss_vi_consts.vc_nvq; i++) {
+ sc->vss_vq[i].vq_qsize = sc->vss_req_ringsz;
sc->vss_vq[i].vq_notify = pci_vtscsi_requestq_notify;
err = pci_vtscsi_init_queue(sc, &sc->vss_queues[i - 2], i - 2);

File Metadata

Mime Type
text/plain
Expires
Sun, Apr 19, 7:40 PM (22 h, 9 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
31793535
Default Alt Text
D54073.id167548.diff (14 KB)

Event Timeline