Page MenuHomeFreeBSD
Files F163266922

D57364.id179021.diff
No OneTemporary
Actions

D57364.id179021.diff
View Options

diff --git a/sys/dev/virtio/virtio_config.h b/sys/dev/virtio/virtio_config.h
--- a/sys/dev/virtio/virtio_config.h
+++ b/sys/dev/virtio/virtio_config.h
@@ -77,12 +77,22 @@
*/
#define VIRTIO_F_IOMMU_PLATFORM (1ULL << 33)
+/*
+ * VIRTIO_F_RING_PACKED (bit 34): when negotiated, the device and driver
+ * use a packed virtqueue instead of the split layout. A packed ring
+ * combines the descriptor table, available ring, and used ring into a
+ * single array of struct vring_packed_desc, with ownership indicated by
+ * the VRING_PACKED_DESC_F_AVAIL and VRING_PACKED_DESC_F_USED flag bits
+ * rather than separate ring indices.
+ */
+#define VIRTIO_F_RING_PACKED (1ULL << 34)
+
/*
* Some VirtIO feature bits (currently bits 28 through 34) are
* reserved for the transport being used (eg. virtio_ring), the
* rest are per-device feature bits.
*/
#define VIRTIO_TRANSPORT_F_START 28
-#define VIRTIO_TRANSPORT_F_END 34
+#define VIRTIO_TRANSPORT_F_END 35
#endif /* _VIRTIO_CONFIG_H_ */
diff --git a/sys/dev/virtio/virtio_ring.h b/sys/dev/virtio/virtio_ring.h
--- a/sys/dev/virtio/virtio_ring.h
+++ b/sys/dev/virtio/virtio_ring.h
@@ -40,6 +40,43 @@
/* This means the buffer contains a list of buffer descriptors. */
#define VRING_DESC_F_INDIRECT 4
+/*
+ * Packed virtqueue descriptor flags (VIRTIO_F_RING_PACKED).
+ *
+ * AVAIL and USED are the ownership bits used in place of the separate
+ * avail/used rings from split virtqueues:
+ *
+ * Driver makes descriptor available: AVAIL = driver_wrap, USED = !driver_wrap
+ * Device marks descriptor as used: AVAIL = USED = device_wrap
+ *
+ * A descriptor is available when (AVAIL != USED).
+ */
+#define VRING_PACKED_DESC_F_AVAIL (1 << 7)
+#define VRING_PACKED_DESC_F_USED (1 << 15)
+
+/* Packed virtqueue descriptor (16 bytes, same size as split). */
+struct vring_packed_desc {
+ uint64_t addr; /* buffer address (guest-physical) */
+ uint32_t len; /* buffer length */
+ uint16_t id; /* buffer id (chain head identifier) */
+ uint16_t flags; /* VRING_DESC_F_* | VRING_PACKED_DESC_F_* */
+};
+
+/*
+ * Packed virtqueue event suppression structure.
+ * Two of these follow the descriptor ring in guest memory:
+ * offset [qsize * 16 + 0]: device event (written by device, read by driver)
+ * offset [qsize * 16 + 4]: driver event (written by driver, read by device)
+ */
+struct vring_packed_desc_event {
+ uint16_t off_wrap; /* descriptor index and ring wrap counter */
+ uint16_t flags; /* VRING_PACKED_EVENT_FLAG_* */
+};
+
+#define VRING_PACKED_EVENT_FLAG_ENABLE 0 /* always interrupt */
+#define VRING_PACKED_EVENT_FLAG_DISABLE 1 /* never interrupt */
+#define VRING_PACKED_EVENT_FLAG_DESC 2 /* interrupt at descriptor */
+
/* The Host uses this in used->flags to advise the Guest: don't kick me
* when you add a buffer. It's unreliable, so it's simply an
* optimization. Guest will still kick if it's out of buffers. */
diff --git a/usr.sbin/bhyve/virtio.h b/usr.sbin/bhyve/virtio.h
--- a/usr.sbin/bhyve/virtio.h
+++ b/usr.sbin/bhyve/virtio.h
@@ -349,10 +349,26 @@
uint64_t vq_avail_addr; /* available ring GPA */
uint64_t vq_used_addr; /* used ring GPA */
- struct vring_desc *vq_desc; /* descriptor array */
- struct vring_avail *vq_avail; /* the "avail" ring */
- struct vring_used *vq_used; /* the "used" ring */
+ struct vring_desc *vq_desc; /* split: descriptor array */
+ struct vring_avail *vq_avail; /* split: the "avail" ring */
+ struct vring_used *vq_used; /* split: the "used" ring */
+ /*
+ * Packed virtqueue state (VIRTIO_F_RING_PACKED).
+ *
+ * The single packed ring lives at vq_desc_addr; vq_avail_addr and
+ * vq_used_addr are unused. Wrap counters begin at 1 and toggle
+ * each time the ring index wraps from (qsize-1) back to 0.
+ *
+ * vq_last_avail -- ring index of the next descriptor to consume
+ * vq_next_used -- ring index of the chain head being returned
+ * (set by vq_relchain_prepare, used by _publish)
+ * vq_save_used -- previous vq_next_used, for interrupt decisions
+ */
+ bool vq_packed; /* true: packed ring format in use */
+ uint8_t vq_avail_wrap; /* device's copy of driver ring wrap counter */
+ uint8_t vq_used_wrap; /* device ring wrap counter */
+ struct vring_packed_desc *vq_packed_desc; /* packed ring host pointer */
};
/* as noted above, these are sort of backwards, name-wise */
#define VQ_AVAIL_EVENT_IDX(vq) \
@@ -377,9 +393,19 @@
static inline int
vq_has_descs(struct vqueue_info *vq)
{
-
- return (vq_ring_ready(vq) && vq->vq_last_avail !=
- vq->vq_avail->idx);
+ uint16_t flags;
+
+ if (!vq_ring_ready(vq))
+ return (0);
+ if (vq->vq_packed) {
+ /*
+ * A packed descriptor is available when AVAIL==avail_wrap
+ * and USED!=avail_wrap, which simplifies to AVAIL!=USED.
+ */
+ flags = vq->vq_packed_desc[vq->vq_last_avail].flags;
+ return (((flags >> 7) & 1) != ((flags >> 15) & 1));
+ }
+ return (vq->vq_last_avail != vq->vq_avail->idx);
}
/*
@@ -416,11 +442,13 @@
vq_kick_enable(struct vqueue_info *vq)
{
- vq->vq_used->flags &= ~VRING_USED_F_NO_NOTIFY;
+ if (!vq->vq_packed)
+ vq->vq_used->flags &= ~VRING_USED_F_NO_NOTIFY;
/*
* Full memory barrier to make sure the store to vq_used->flags
- * happens before the load from vq_avail->idx, which results from a
- * subsequent call to vq_has_descs().
+ * (split) or the packed ring (packed) happens before the load from
+ * vq_avail->idx / packed flags, which results from a subsequent
+ * call to vq_has_descs().
*/
atomic_thread_fence_seq_cst();
}
@@ -429,7 +457,8 @@
vq_kick_disable(struct vqueue_info *vq)
{
- vq->vq_used->flags |= VRING_USED_F_NO_NOTIFY;
+ if (!vq->vq_packed)
+ vq->vq_used->flags |= VRING_USED_F_NO_NOTIFY;
}
struct iovec;
diff --git a/usr.sbin/bhyve/virtio.c b/usr.sbin/bhyve/virtio.c
--- a/usr.sbin/bhyve/virtio.c
+++ b/usr.sbin/bhyve/virtio.c
@@ -77,7 +77,7 @@
uint64_t caps = vs->vs_vc->vc_hv_caps;
if (caps & VIRTIO_F_VERSION_1)
- caps |= VIRTIO_F_IOMMU_PLATFORM;
+ caps |= VIRTIO_F_IOMMU_PLATFORM | VIRTIO_F_RING_PACKED;
return (caps);
}
@@ -136,6 +136,10 @@
vq->vq_avail_addr = 0;
vq->vq_used_addr = 0;
vq->vq_msix_idx = VIRTIO_MSI_NO_VECTOR;
+ vq->vq_packed = false;
+ vq->vq_avail_wrap = 0;
+ vq->vq_used_wrap = 0;
+ vq->vq_packed_desc = NULL;
/* Restore queue size to device maximum */
if (vq->vq_max_qsize != 0)
vq->vq_qsize = vq->vq_max_qsize;
@@ -254,6 +258,13 @@
/*
* Initialize a virtqueue from 64-bit modern addresses (called when the
* driver sets queue_enable = 1 in common_cfg).
+ *
+ * For split virtqueues, maps the three ring regions.
+ * For packed virtqueues (VIRTIO_F_RING_PACKED), maps the single descriptor
+ * ring plus the two trailing event-suppression structures.
+ *
+ * Ring position counters (vq_last_avail, vq_next_used, vq_save_used) are
+ * NOT reset here; they are managed by vi_reset_dev() and snapshot restore.
*/
static void
vi_vq_init_modern(struct virtio_softc *vs, struct vqueue_info *vq)
@@ -261,18 +272,40 @@
struct vmctx *ctx = vs->vs_pi->pi_vmctx;
size_t desc_size, avail_size, used_size;
- desc_size = (size_t)vq->vq_qsize * sizeof(struct vring_desc);
- avail_size = (2 + (size_t)vq->vq_qsize + 1) * sizeof(uint16_t);
- used_size = (2 + 2 * (size_t)vq->vq_qsize + 1) * sizeof(uint16_t);
-
- vq->vq_desc = paddr_guest2host(ctx, vq->vq_desc_addr, desc_size);
- vq->vq_avail = paddr_guest2host(ctx, vq->vq_avail_addr, avail_size);
- vq->vq_used = paddr_guest2host(ctx, vq->vq_used_addr, used_size);
+ if (vs->vs_negotiated_caps & VIRTIO_F_RING_PACKED) {
+ /*
+ * Packed ring: a single array of vring_packed_desc followed
+ * by two vring_packed_desc_event structures (device event,
+ * then driver event).
+ */
+ size_t ring_size = (size_t)vq->vq_qsize *
+ sizeof(struct vring_packed_desc) +
+ 2 * sizeof(struct vring_packed_desc_event);
+
+ vq->vq_packed = true;
+ vq->vq_avail_wrap = 1;
+ vq->vq_used_wrap = 1;
+ vq->vq_packed_desc = paddr_guest2host(ctx,
+ vq->vq_desc_addr, ring_size);
+ vq->vq_desc = NULL;
+ vq->vq_avail = NULL;
+ vq->vq_used = NULL;
+ } else {
+ desc_size = (size_t)vq->vq_qsize * sizeof(struct vring_desc);
+ avail_size = (2 + (size_t)vq->vq_qsize + 1) * sizeof(uint16_t);
+ used_size = (2 + 2 * (size_t)vq->vq_qsize + 1) *
+ sizeof(uint16_t);
+
+ vq->vq_packed = false;
+ vq->vq_desc = paddr_guest2host(ctx, vq->vq_desc_addr,
+ desc_size);
+ vq->vq_avail = paddr_guest2host(ctx, vq->vq_avail_addr,
+ avail_size);
+ vq->vq_used = paddr_guest2host(ctx, vq->vq_used_addr,
+ used_size);
+ }
vq->vq_flags = VQ_ALLOC;
- vq->vq_last_avail = 0;
- vq->vq_next_used = 0;
- vq->vq_save_used = 0;
}
/*
@@ -470,9 +503,18 @@
vq = &vs->vs_queues[vs->vs_curq];
if ((uint16_t)value == 1) {
vq->vq_enabled = true;
- if (vq->vq_desc_addr != 0 && vq->vq_avail_addr != 0 &&
- vq->vq_used_addr != 0)
+ /*
+ * Packed queues use only desc_addr (the single ring);
+ * split queues require all three region addresses.
+ */
+ if (vs->vs_negotiated_caps & VIRTIO_F_RING_PACKED) {
+ if (vq->vq_desc_addr != 0)
+ vi_vq_init_modern(vs, vq);
+ } else if (vq->vq_desc_addr != 0 &&
+ vq->vq_avail_addr != 0 &&
+ vq->vq_used_addr != 0) {
vi_vq_init_modern(vs, vq);
+ }
} else {
vq->vq_enabled = false;
vq->vq_flags &= ~VQ_ALLOC;
@@ -753,6 +795,142 @@
}
#define VQ_MAX_DESCRIPTORS 512 /* see below */
+/*
+ * Helper inline for vq_getchain_packed(): record the i'th packed descriptor.
+ * Packed descriptors have id/flags swapped relative to split, so a separate
+ * helper is needed.
+ */
+static inline void
+_vq_record_packed(int i, struct vring_packed_desc *vd, struct vmctx *ctx,
+ struct iovec *iov, int n_iov, struct vi_req *reqp)
+{
+ uint32_t len;
+ uint64_t addr;
+
+ if (i >= n_iov)
+ return;
+ len = atomic_load_32(&vd->len);
+ addr = atomic_load_64(&vd->addr);
+ iov[i].iov_len = len;
+ iov[i].iov_base = paddr_guest2host(ctx, addr, len);
+ if ((vd->flags & VRING_DESC_F_WRITE) == 0)
+ reqp->readable++;
+ else
+ reqp->writable++;
+}
+
+/*
+ * Packed-virtqueue variant of vq_getchain().
+ *
+ * Walks the chain of packed descriptors starting at vq->vq_last_avail,
+ * advancing vq_last_avail (and toggling vq_avail_wrap on wrap) for each
+ * descriptor consumed. req.idx is set to the ring index of the chain head
+ * for use by vq_relchain_prepare().
+ *
+ * Indirect descriptors use the split-ring layout (per spec).
+ */
+static int
+vq_getchain_packed(struct vqueue_info *vq, struct iovec *iov, int niov,
+ struct vi_req *reqp)
+{
+ struct virtio_softc *vs;
+ struct vmctx *ctx;
+ const char *name;
+ struct vring_packed_desc *pd;
+ struct vring_desc *vindir, *vp;
+ struct vi_req req;
+ uint16_t flags, head;
+ u_int n_indir, next_indir;
+ int i;
+
+ vs = vq->vq_vs;
+ ctx = vs->vs_pi->pi_vmctx;
+ name = vs->vs_vc->vc_name;
+ memset(&req, 0, sizeof(req));
+
+ head = vq->vq_last_avail;
+ pd = &vq->vq_packed_desc[head];
+ flags = atomic_load_acq_16(&pd->flags);
+
+ /* Descriptor is available when AVAIL==avail_wrap AND USED!=avail_wrap */
+ if (((flags >> 7) & 1) != vq->vq_avail_wrap ||
+ ((flags >> 15) & 1) == vq->vq_avail_wrap)
+ return (0);
+
+ req.idx = head;
+ i = 0;
+
+ for (;;) {
+ if (i >= VQ_MAX_DESCRIPTORS) {
+ EPRINTLN("%s: packed chain too long (>= %d), "
+ "driver confused?", name, VQ_MAX_DESCRIPTORS);
+ return (-1);
+ }
+
+ pd = &vq->vq_packed_desc[vq->vq_last_avail];
+ flags = atomic_load_acq_16(&pd->flags);
+
+ if (flags & VRING_DESC_F_INDIRECT) {
+ uint32_t ilen = atomic_load_32(&pd->len);
+ uint64_t iaddr = atomic_load_64(&pd->addr);
+
+ n_indir = ilen / sizeof(struct vring_desc);
+ if ((ilen & (sizeof(struct vring_desc) - 1)) ||
+ n_indir == 0) {
+ EPRINTLN("%s: packed indirect invalid len 0x%x, "
+ "driver confused?", name, ilen);
+ return (-1);
+ }
+ if ((vs->vs_negotiated_caps &
+ VIRTIO_RING_F_INDIRECT_DESC) == 0) {
+ EPRINTLN("%s: packed descriptor has forbidden "
+ "INDIRECT flag, driver confused?", name);
+ return (-1);
+ }
+ vindir = paddr_guest2host(ctx, iaddr, ilen);
+ next_indir = 0;
+ for (;;) {
+ vp = &vindir[next_indir];
+ if (vp->flags & VRING_DESC_F_INDIRECT) {
+ EPRINTLN("%s: indirect desc has INDIRECT "
+ "flag, driver confused?", name);
+ return (-1);
+ }
+ _vq_record(i, vp, ctx, iov, niov, &req);
+ if (++i > VQ_MAX_DESCRIPTORS) {
+ EPRINTLN("%s: packed chain too long "
+ "(indirect), driver confused?",
+ name);
+ return (-1);
+ }
+ if (!(vp->flags & VRING_DESC_F_NEXT))
+ break;
+ next_indir = vp->next;
+ if (next_indir >= n_indir) {
+ EPRINTLN("%s: indirect index out of "
+ "range, driver confused?", name);
+ return (-1);
+ }
+ }
+ } else {
+ _vq_record_packed(i, pd, ctx, iov, niov, &req);
+ i++;
+ }
+
+ /* Advance ring position; toggle wrap counter on wrap. */
+ if (++vq->vq_last_avail >= vq->vq_qsize) {
+ vq->vq_last_avail = 0;
+ vq->vq_avail_wrap ^= 1;
+ }
+
+ if (!(flags & VRING_DESC_F_NEXT))
+ break;
+ }
+
+ *reqp = req;
+ return (i);
+}
+
/*
* Examine the chain of descriptors starting at the "next one" to
* make sure that they describe a sensible request. If so, return
@@ -804,6 +982,10 @@
name = vs->vs_vc->vc_name;
memset(&req, 0, sizeof(req));
+ /* Packed virtqueues use a completely different ring layout. */
+ if (vq->vq_packed)
+ return (vq_getchain_packed(vq, iov, niov, reqp));
+
/*
* Note: it's the responsibility of the guest not to
* update vq->vq_avail->idx until all of the descriptors
@@ -927,6 +1109,16 @@
vq_retchains(struct vqueue_info *vq, uint16_t n_chains)
{
+ /*
+ * Packed queues cannot safely rewind vq_last_avail: the wrap counter
+ * would also need to be reversed, which requires knowing the original
+ * chain lengths. This path is only reached on error; warn and leak.
+ */
+ if (vq->vq_packed) {
+ EPRINTLN("%s: vq_retchains not supported for packed queues",
+ vq->vq_vs->vs_vc->vc_name);
+ return;
+ }
vq->vq_last_avail -= n_chains;
}
@@ -935,10 +1127,25 @@
{
struct vring_used *vuh;
struct vring_used_elem *vue;
+ struct vring_packed_desc *pd;
uint16_t mask;
+ if (vq->vq_packed) {
+ /*
+ * For packed queues, write len to the chain-head descriptor.
+ * The id field already has the correct value from the driver.
+ * The flags (which transfer ownership back to the driver) are
+ * written by vq_relchain_publish() with a release barrier.
+ * vq_next_used saves idx so publish() can find the descriptor.
+ */
+ pd = &vq->vq_packed_desc[idx];
+ pd->len = iolen;
+ vq->vq_next_used = idx;
+ return;
+ }
+
/*
- * Notes:
+ * Split queue:
* - mask is N-1 where N is a power of 2 so computes x % N
* - vuh points to the "used" data shared with guest
* - vue points to the "used" ring entry we want to update
@@ -954,10 +1161,41 @@
void
vq_relchain_publish(struct vqueue_info *vq)
{
+ struct vring_packed_desc *pd;
+ uint16_t head, used_flags;
+
+ if (vq->vq_packed) {
+ /*
+ * Packed queue: publish the used descriptor by writing the
+ * ownership flags with a release barrier so the guest sees
+ * the id/len writes from vq_relchain_prepare() first.
+ *
+ * Ownership transfer: AVAIL = USED = vq_used_wrap.
+ *
+ * After publishing, advance vq_next_used to vq_last_avail
+ * (the position after the chain) and flip vq_used_wrap if
+ * the ring wrapped during this chain.
+ */
+ head = vq->vq_next_used;
+ pd = &vq->vq_packed_desc[head];
+
+ used_flags = vq->vq_used_wrap ?
+ (VRING_PACKED_DESC_F_AVAIL | VRING_PACKED_DESC_F_USED) : 0;
+
+ atomic_thread_fence_rel();
+ atomic_store_16(&pd->flags, used_flags);
+
+ /* Flip device wrap counter if the chain crossed position 0. */
+ if (vq->vq_last_avail <= head)
+ vq->vq_used_wrap ^= 1;
+ vq->vq_next_used = vq->vq_last_avail;
+ return;
+ }
+
/*
- * Ensure the used descriptor is visible before updating the index.
- * This is necessary on ISAs with memory ordering less strict than x86
- * (and even on x86 to act as a compiler barrier).
+ * Split queue: ensure the used descriptor is visible before updating
+ * the index. This is necessary on ISAs with memory ordering less
+ * strict than x86 (and even on x86 to act as a compiler barrier).
*/
atomic_thread_fence_rel();
vq->vq_used->idx = vq->vq_next_used;
@@ -1009,6 +1247,36 @@
* entire avail was processed, we need to interrupt always.
*/
vs = vq->vq_vs;
+
+ if (vq->vq_packed) {
+ /*
+ * Packed queue interrupt decision.
+ *
+ * Read the driver event suppression structure that sits
+ * immediately after the device event suppression structure
+ * (i.e., at byte offset qsize*16 + 4 from the ring start).
+ * If the driver has set DISABLE, suppress the interrupt.
+ * Otherwise always deliver one if we produced anything.
+ */
+ struct vring_packed_desc_event *drv_evt;
+ uint16_t cur_used;
+
+ drv_evt = (struct vring_packed_desc_event *)
+ ((uint8_t *)vq->vq_packed_desc +
+ (size_t)vq->vq_qsize * sizeof(struct vring_packed_desc) +
+ sizeof(struct vring_packed_desc_event));
+
+ cur_used = vq->vq_next_used;
+ atomic_thread_fence_seq_cst();
+
+ if (cur_used != vq->vq_save_used || used_all_avail) {
+ vq->vq_save_used = cur_used;
+ if (drv_evt->flags != VRING_PACKED_EVENT_FLAG_DISABLE)
+ vq_interrupt(vs, vq);
+ }
+ return;
+ }
+
old_idx = vq->vq_save_used;
vq->vq_save_used = new_idx = vq->vq_used->idx;
@@ -1483,9 +1751,19 @@
* Re-establish host-side ring pointers from the saved
* GPA addresses on restore; on save the pointers are
* already valid.
+ *
+ * vi_vq_init_modern() also sets vq_avail_wrap and
+ * vq_used_wrap to 1 (initial state), so save/restore
+ * them AFTER the call so the snapshot values win.
*/
if (meta->op == VM_SNAPSHOT_RESTORE)
vi_vq_init_modern(vs, vq);
+
+ /* Packed-queue wrap counters (no-ops for split). */
+ SNAPSHOT_VAR_OR_LEAVE(vq->vq_avail_wrap, meta, ret,
+ done);
+ SNAPSHOT_VAR_OR_LEAVE(vq->vq_used_wrap, meta, ret,
+ done);
} else {
addr_size = vq->vq_qsize * sizeof(struct vring_desc);
SNAPSHOT_GUEST2HOST_ADDR_OR_LEAVE(ctx, vq->vq_desc,
@@ -1501,8 +1779,15 @@
addr_size, false, meta, ret, done);
}
- SNAPSHOT_BUF_OR_LEAVE(vq->vq_desc,
- vring_size_aligned(vq->vq_qsize), meta, ret, done);
+ if (vq->vq_packed) {
+ SNAPSHOT_BUF_OR_LEAVE(vq->vq_packed_desc,
+ (size_t)vq->vq_qsize *
+ sizeof(struct vring_packed_desc),
+ meta, ret, done);
+ } else {
+ SNAPSHOT_BUF_OR_LEAVE(vq->vq_desc,
+ vring_size_aligned(vq->vq_qsize), meta, ret, done);
+ }
}
done:

File Metadata

Mime Type
text/plain
Expires
Wed, Jul 22, 2:29 PM (18 h, 13 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35367234
Default Alt Text
D57364.id179021.diff (18 KB)

Event Timeline