Index: ObsoleteFiles.inc
===================================================================
--- ObsoleteFiles.inc
+++ ObsoleteFiles.inc
@@ -52,6 +52,9 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 2022xxxx: libvmmapi version bump
+OLD_LIBS+=usr/lib/libvmmapi.so.5
+
 # 20220721: libnv version bumps
 OLD_LIBS+=lib/libnv.so.0
 
Index: lib/libvmmapi/Makefile
===================================================================
--- lib/libvmmapi/Makefile
+++ lib/libvmmapi/Makefile
@@ -2,6 +2,7 @@
 
 PACKAGE=lib${LIB}
 LIB=	vmmapi
+SHLIB_MAJOR= 6
 SRCS=	vmmapi.c vmmapi_freebsd.c
 INCS=	vmmapi.h
 
Index: lib/libvmmapi/vmmapi.h
===================================================================
--- lib/libvmmapi/vmmapi.h
+++ lib/libvmmapi/vmmapi.h
@@ -3,6 +3,10 @@
  *
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Ararat River
+ * Consulting, LLC under sponsorship of the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -43,7 +47,7 @@
  * API version for out-of-tree consumers like grub-bhyve for making compile
  * time decisions.
  */
-#define	VMMAPI_VERSION	0103	/* 2 digit major followed by 2 digit minor */
+#define	VMMAPI_VERSION	0104	/* 2 digit major followed by 2 digit minor */
 
 struct iovec;
 struct vmctx;
@@ -181,8 +185,9 @@
 			  int val);
 int	vm_assign_pptdev(struct vmctx *ctx, int bus, int slot, int func);
 int	vm_unassign_pptdev(struct vmctx *ctx, int bus, int slot, int func);
-int	vm_map_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
-			   vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
+int	vm_map_pptdev_mmio_bar(struct vmctx *ctx, int bus, int slot, int func,
+			       vm_paddr_t gpa, size_t len, int bar,
+			       size_t offset);
 int	vm_unmap_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
 			     vm_paddr_t gpa, size_t len);
 int	vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, int bus, int slot,
Index: lib/libvmmapi/vmmapi.c
===================================================================
--- lib/libvmmapi/vmmapi.c
+++ lib/libvmmapi/vmmapi.c
@@ -3,6 +3,10 @@
  *
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Ararat River
+ * Consulting, LLC under sponsorship of the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -986,20 +990,21 @@
 }
 
 int
-vm_map_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
-		   vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
+vm_map_pptdev_mmio_bar(struct vmctx *ctx, int bus, int slot, int func,
+		       vm_paddr_t gpa, size_t len, int bar, size_t offset)
 {
-	struct vm_pptdev_mmio pptmmio;
+	struct vm_pptdev_mmio_bar pptmmio;
 
 	bzero(&pptmmio, sizeof(pptmmio));
 	pptmmio.bus = bus;
 	pptmmio.slot = slot;
 	pptmmio.func = func;
-	pptmmio.gpa = gpa;
+	pptmmio.bar = bar;
+	pptmmio.offset = offset;
 	pptmmio.len = len;
-	pptmmio.hpa = hpa;
+	pptmmio.gpa = gpa;
 
-	return (ioctl(ctx->fd, VM_MAP_PPTDEV_MMIO, &pptmmio));
+	return (ioctl(ctx->fd, VM_MAP_PPTDEV_MMIO_BAR, &pptmmio));
 }
 
 int
@@ -1752,7 +1757,7 @@
 	    VM_IOAPIC_PULSE_IRQ, VM_IOAPIC_PINCOUNT, VM_ISA_ASSERT_IRQ,
 	    VM_ISA_DEASSERT_IRQ, VM_ISA_PULSE_IRQ, VM_ISA_SET_IRQ_TRIGGER,
 	    VM_SET_CAPABILITY, VM_GET_CAPABILITY, VM_BIND_PPTDEV,
-	    VM_UNBIND_PPTDEV, VM_MAP_PPTDEV_MMIO, VM_PPTDEV_MSI,
+	    VM_UNBIND_PPTDEV, VM_MAP_PPTDEV_MMIO_BAR, VM_PPTDEV_MSI,
 	    VM_PPTDEV_MSIX, VM_UNMAP_PPTDEV_MMIO, VM_PPTDEV_DISABLE_MSIX,
 	    VM_INJECT_NMI, VM_STATS, VM_STAT_DESC,
 	    VM_SET_X2APIC_STATE, VM_GET_X2APIC_STATE,
Index: sys/amd64/include/vmm_dev.h
===================================================================
--- sys/amd64/include/vmm_dev.h
+++ sys/amd64/include/vmm_dev.h
@@ -3,6 +3,10 @@
  *
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Ararat River
+ * Consulting, LLC under sponsorship of the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -146,6 +150,16 @@
 	size_t		len;
 };
 
+struct vm_pptdev_mmio_bar {
+	int		bus;
+	int		slot;
+	int		func;
+	int		bar;
+	size_t		offset;
+	size_t		len;
+	vm_paddr_t	gpa;
+};
+
 struct vm_pptdev_msi {
 	int		vcpu;
 	int		bus;
@@ -310,6 +324,7 @@
 	IOCNUM_PPTDEV_MSIX = 44,
 	IOCNUM_PPTDEV_DISABLE_MSIX = 45,
 	IOCNUM_UNMAP_PPTDEV_MMIO = 46,
+	IOCNUM_MAP_PPTDEV_MMIO_BAR = 47,
 
 	/* statistics */
 	IOCNUM_VM_STATS = 50, 
@@ -428,6 +443,8 @@
 	_IOW('v', IOCNUM_PPTDEV_DISABLE_MSIX, struct vm_pptdev)
 #define	VM_UNMAP_PPTDEV_MMIO \
 	_IOW('v', IOCNUM_UNMAP_PPTDEV_MMIO, struct vm_pptdev_mmio)
+#define	VM_MAP_PPTDEV_MMIO_BAR \
+	_IOW('v', IOCNUM_MAP_PPTDEV_MMIO_BAR, struct vm_pptdev_mmio_bar)
 #define VM_INJECT_NMI \
 	_IOW('v', IOCNUM_INJECT_NMI, struct vm_nmi)
 #define	VM_STATS \
Index: sys/amd64/vmm/io/ppt.h
===================================================================
--- sys/amd64/vmm/io/ppt.h
+++ sys/amd64/vmm/io/ppt.h
@@ -3,6 +3,10 @@
  *
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Ararat River
+ * Consulting, LLC under sponsorship of the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -34,6 +38,8 @@
 int	ppt_unassign_all(struct vm *vm);
 int	ppt_map_mmio(struct vm *vm, int bus, int slot, int func,
 		     vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
+int	ppt_map_mmio_bar(struct vm *vm, int bus, int slot, int func,
+			 vm_paddr_t gpa, size_t len, int bar, size_t offset);
 int	ppt_unmap_mmio(struct vm *vm, int bus, int slot, int func,
 		       vm_paddr_t gpa, size_t len);
 int	ppt_setup_msi(struct vm *vm, int vcpu, int bus, int slot, int func,
Index: sys/amd64/vmm/io/ppt.c
===================================================================
--- sys/amd64/vmm/io/ppt.c
+++ sys/amd64/vmm/io/ppt.c
@@ -3,6 +3,10 @@
  *
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Ararat River
+ * Consulting, LLC under sponsorship of the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -89,6 +93,7 @@
 	struct vm	*vm;			/* owner of this device */
 	TAILQ_ENTRY(pptdev)	next;
 	struct pptseg mmio[MAX_MMIOSEGS];
+	struct resource *bar_res[PCIR_MAX_BAR_0];
 	struct {
 		int	num_msgs;		/* guest state */
 
@@ -101,8 +106,6 @@
 	struct {
 		int num_msgs;
 		int startrid;
-		int msix_table_rid;
-		int msix_pba_rid;
 		struct resource *msix_table_res;
 		struct resource *msix_pba_res;
 		struct resource **res;
@@ -174,6 +177,7 @@
 ppt_detach(device_t dev)
 {
 	struct pptdev *ppt;
+	int i;
 
 	ppt = device_get_softc(dev);
 
@@ -182,6 +186,12 @@
 	num_pptdevs--;
 	TAILQ_REMOVE(&pptdev_list, ppt, next);
 	pci_disable_busmaster(dev);
+
+	for (i = 0; i < PCIR_MAX_BAR_0; i++) {
+		if (ppt->bar_res[i] != NULL)
+			bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(i),
+			    ppt->bar_res[i]);
+	}
 	iommu_add_device(iommu_host_domain(), pci_get_rid(dev));
 
 	return (0);
@@ -306,21 +316,6 @@
 
 	pci_release_msi(ppt->dev);
 
-	if (ppt->msix.msix_table_res) {
-		bus_release_resource(ppt->dev, SYS_RES_MEMORY, 
-				     ppt->msix.msix_table_rid,
-				     ppt->msix.msix_table_res);
-		ppt->msix.msix_table_res = NULL;
-		ppt->msix.msix_table_rid = 0;
-	}
-	if (ppt->msix.msix_pba_res) {
-		bus_release_resource(ppt->dev, SYS_RES_MEMORY, 
-				     ppt->msix.msix_pba_rid,
-				     ppt->msix.msix_pba_res);
-		ppt->msix.msix_pba_res = NULL;
-		ppt->msix.msix_pba_rid = 0;
-	}
-
 	ppt->msix.num_msgs = 0;
 }
 
@@ -345,6 +340,24 @@
 	return (num);
 }
 
+static struct resource *
+ppt_map_bar(struct pptdev *ppt, int rid)
+{
+	int idx;
+
+	if (rid < PCIR_BAR(0) || rid >= PCIR_BAR(PCIR_MAX_BAR_0) ||
+	    rid % 4 != 0)
+		return (NULL);
+
+	idx = PCI_RID2BAR(rid);
+	if (ppt->bar_res[idx] != NULL)
+		return (ppt->bar_res[idx]);
+
+	ppt->bar_res[idx] = bus_alloc_resource_any(ppt->dev, SYS_RES_MEMORY,
+	    &rid, RF_ACTIVE);
+	return (ppt->bar_res[idx]);
+}
+
 bool
 ppt_is_mmio(struct vm *vm, vm_paddr_t gpa)
 {
@@ -439,6 +452,7 @@
 	return (0);
 }
 
+#ifdef COMPAT_FREEBSD13
 int
 ppt_map_mmio(struct vm *vm, int bus, int slot, int func,
 	     vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
@@ -464,6 +478,45 @@
 	}
 	return (ENOSPC);
 }
+#endif
+
+int
+ppt_map_mmio_bar(struct vm *vm, int bus, int slot, int func,
+		 vm_paddr_t gpa, size_t len, int bar, size_t offset)
+{
+	struct resource *res;
+	vm_paddr_t hpa;
+	int i, error;
+	struct pptseg *seg;
+	struct pptdev *ppt;
+
+	if (len == 0 || offset + len < offset)
+		return (EINVAL);
+
+	error = ppt_find(vm, bus, slot, func, &ppt);
+	if (error)
+		return (error);
+
+	res = ppt_map_bar(ppt, bar);
+	if (res == NULL)
+		return (ENXIO);
+	if (offset + len > rman_get_size(res))
+		return (EINVAL);
+	hpa = rman_get_start(res) + offset;
+
+	for (i = 0; i < MAX_MMIOSEGS; i++) {
+		seg = &ppt->mmio[i];
+		if (seg->len == 0) {
+			error = vm_map_mmio(vm, gpa, len, hpa);
+			if (error == 0) {
+				seg->gpa = gpa;
+				seg->len = len;
+			}
+			return (error);
+		}
+	}
+	return (ENOSPC);
+}
 
 int
 ppt_unmap_mmio(struct vm *vm, int bus, int slot, int func,
@@ -656,27 +709,21 @@
 					  M_WAITOK | M_ZERO);
 		ppt->msix.arg = malloc(arg_size, M_PPTMSIX, M_WAITOK | M_ZERO);
 
-		rid = dinfo->cfg.msix.msix_table_bar;
-		ppt->msix.msix_table_res = bus_alloc_resource_any(ppt->dev,
-					       SYS_RES_MEMORY, &rid, RF_ACTIVE);
-
+		ppt->msix.msix_table_res = ppt_map_bar(ppt,
+		    dinfo->cfg.msix.msix_table_bar);
 		if (ppt->msix.msix_table_res == NULL) {
 			ppt_teardown_msix(ppt);
 			return (ENOSPC);
 		}
-		ppt->msix.msix_table_rid = rid;
 
 		if (dinfo->cfg.msix.msix_table_bar !=
 		    dinfo->cfg.msix.msix_pba_bar) {
-			rid = dinfo->cfg.msix.msix_pba_bar;
-			ppt->msix.msix_pba_res = bus_alloc_resource_any(
-			    ppt->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
-
+			ppt->msix.msix_pba_res = ppt_map_bar(ppt,
+			    dinfo->cfg.msix.msix_pba_bar);
 			if (ppt->msix.msix_pba_res == NULL) {
 				ppt_teardown_msix(ppt);
 				return (ENOSPC);
 			}
-			ppt->msix.msix_pba_rid = rid;
 		}
 
 		alloced = numvec;
Index: sys/amd64/vmm/vmm_dev.c
===================================================================
--- sys/amd64/vmm/vmm_dev.c
+++ sys/amd64/vmm/vmm_dev.c
@@ -3,6 +3,10 @@
  *
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Ararat River
+ * Consulting, LLC under sponsorship of the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -385,6 +389,7 @@
 	struct vm_capability *vmcap;
 	struct vm_pptdev *pptdev;
 	struct vm_pptdev_mmio *pptmmio;
+	struct vm_pptdev_mmio_bar *pptmmio_bar;
 	struct vm_pptdev_msi *pptmsi;
 	struct vm_pptdev_msix *pptmsix;
 	struct vm_nmi *vmnmi;
@@ -457,7 +462,10 @@
 		state_changed = 1;
 		break;
 
+#ifdef COMPAT_FREEBSD13
 	case VM_MAP_PPTDEV_MMIO:
+#endif
+	case VM_MAP_PPTDEV_MMIO_BAR:
 	case VM_UNMAP_PPTDEV_MMIO:
 	case VM_BIND_PPTDEV:
 	case VM_UNBIND_PPTDEV:
@@ -554,12 +562,21 @@
 		error = ppt_disable_msix(sc->vm, pptdev->bus, pptdev->slot,
 					 pptdev->func);
 		break;
+#ifdef COMPAT_FREEBSD13
 	case VM_MAP_PPTDEV_MMIO:
 		pptmmio = (struct vm_pptdev_mmio *)data;
 		error = ppt_map_mmio(sc->vm, pptmmio->bus, pptmmio->slot,
 				     pptmmio->func, pptmmio->gpa, pptmmio->len,
 				     pptmmio->hpa);
 		break;
+#endif
+	case VM_MAP_PPTDEV_MMIO_BAR:
+		pptmmio_bar = (struct vm_pptdev_mmio_bar *)data;
+		error = ppt_map_mmio_bar(sc->vm, pptmmio_bar->bus,
+					 pptmmio_bar->slot, pptmmio_bar->func,
+					 pptmmio_bar->gpa, pptmmio_bar->len,
+					 pptmmio_bar->bar, pptmmio_bar->offset);
+		break;
 	case VM_UNMAP_PPTDEV_MMIO:
 		pptmmio = (struct vm_pptdev_mmio *)data;
 		error = ppt_unmap_mmio(sc->vm, pptmmio->bus, pptmmio->slot,
Index: usr.sbin/bhyve/pci_passthru.c
===================================================================
--- usr.sbin/bhyve/pci_passthru.c
+++ usr.sbin/bhyve/pci_passthru.c
@@ -3,6 +3,10 @@
  *
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
+ * Copyright (c) 2022 The FreeBSD Foundation
+ *
+ * Portions of this software were developed by Ararat River
+ * Consulting, LLC under sponsorship of the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -1023,11 +1027,11 @@
 						 table_offset) != 0)
 				warnx("pci_passthru: unmap_pptdev_mmio failed");
 		} else {
-			if (vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus,
-					       sc->psc_sel.pc_dev,
-					       sc->psc_sel.pc_func, address,
-					       table_offset,
-					       sc->psc_bar[baridx].addr) != 0)
+			if (vm_map_pptdev_mmio_bar(ctx, sc->psc_sel.pc_bus,
+						   sc->psc_sel.pc_dev,
+						   sc->psc_sel.pc_func, address,
+						   table_offset,
+						   PCIR_BAR(baridx), 0) != 0)
 				warnx("pci_passthru: map_pptdev_mmio failed");
 		}
 	}
@@ -1044,12 +1048,12 @@
 						 remaining) != 0)
 				warnx("pci_passthru: unmap_pptdev_mmio failed");
 		} else {
-			if (vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus,
-					       sc->psc_sel.pc_dev,
-					       sc->psc_sel.pc_func, address,
-					       remaining,
-					       sc->psc_bar[baridx].addr +
-					       table_offset + table_size) != 0)
+			if (vm_map_pptdev_mmio_bar(ctx, sc->psc_sel.pc_bus,
+						   sc->psc_sel.pc_dev,
+						   sc->psc_sel.pc_func, address,
+						   remaining, PCIR_BAR(baridx),
+						   table_offset + table_size) !=
+			    0)
 				warnx("pci_passthru: map_pptdev_mmio failed");
 		}
 	}
@@ -1069,11 +1073,11 @@
 					 sc->psc_bar[baridx].size) != 0)
 			warnx("pci_passthru: unmap_pptdev_mmio failed");
 	} else {
-		if (vm_map_pptdev_mmio(ctx, sc->psc_sel.pc_bus,
-				       sc->psc_sel.pc_dev,
-				       sc->psc_sel.pc_func, address,
-				       sc->psc_bar[baridx].size,
-				       sc->psc_bar[baridx].addr) != 0)
+		if (vm_map_pptdev_mmio_bar(ctx, sc->psc_sel.pc_bus,
+					   sc->psc_sel.pc_dev,
+					   sc->psc_sel.pc_func, address,
+					   sc->psc_bar[baridx].size,
+					   PCIR_BAR(baridx), 0) != 0)
 			warnx("pci_passthru: map_pptdev_mmio failed");
 	}
 }