D59481.diff
No OneTemporary
Actions

Size

49 KB

Referenced Files

None

Subscribers

None

D59481.diff
View Options

	diff --git a/lib/libsys/mlock.2 b/lib/libsys/mlock.2
	--- a/lib/libsys/mlock.2
	+++ b/lib/libsys/mlock.2
	@@ -84,6 +84,17 @@
	Locked mappings are not inherited by the child process after a
	.Xr fork 2 .
	.Pp
	+Special device mappings explicitly exempted by their pager are skipped by
	+.Fn mlock
	+and acquire no user wire reference.
	+This includes LinuxKPI managed-device mappings marked as IO or PFN mappings.
	+The residency and address-translation guarantees above do not apply to these
	+mappings; their driver controls their backing lifetime.
	+Ordinary mappings retain their existing locking behavior.
	+Existing privilege, address-range, protection and resource-limit checks still
	+apply, including conservative accounting of the requested virtual range.
	+Thus a request containing an exempt mapping can still fail those checks.
	+.Pp
	Since physical memory is a potentially scarce resource, processes are
	limited in how much they can lock down.
	The amount of memory that a single process can
	@@ -108,7 +119,8 @@
	.Sh RETURN VALUES
	.Rv -std
	.Pp
	-If the call succeeds, all pages in the range become locked (unlocked);
	+If the call succeeds, all non-exempt pages in the range become locked
	+(unlocked);
	otherwise the locked status of all pages in the range remains unchanged.
	.Sh ERRORS
	The
	@@ -172,5 +184,3 @@
	Hence two distinct locked mappings of the same physical page counts as
	2 pages aginst the system limit, and also against the per-process limit
	if both mappings belong to the same physical map.
	-.Pp
	-The per-process resource limit is not currently supported.
	diff --git a/lib/libsys/mlockall.2 b/lib/libsys/mlockall.2
	--- a/lib/libsys/mlockall.2
	+++ b/lib/libsys/mlockall.2
	@@ -62,6 +62,19 @@
	cause resource limits to be exceeded.
	.El
	.Pp
	+Special device mappings explicitly exempted by their pager are skipped for
	+both
	+.Dv MCL_CURRENT
	+and
	+.Dv MCL_FUTURE .
	+This includes LinuxKPI managed-device mappings marked as IO or PFN mappings.
	+These mappings acquire no user wire reference and are not covered by the
	+residency guarantee; their driver controls their backing lifetime.
	+Existing resource-limit checks and conservative accounting of the requested
	+virtual range still apply.
	+See
	+.Xr mlock 2 .
	+.Pp
	Since physical memory is a potentially scarce resource, processes are
	limited in how much they can lock down.
	A single process can lock the minimum of a system-wide
	@@ -91,7 +104,8 @@
	call will not be locked.
	.Sh RETURN VALUES
	A return value of 0 indicates that the call
	-succeeded and all pages in the range have either been locked or unlocked.
	+succeeded and all non-exempt pages in the range have either been locked or
	+unlocked.
	A return value of \-1 indicates an error occurred and the locked
	status of all pages in the range remains unchanged.
	In this case, the global location
	diff --git a/sys/compat/linuxkpi/common/include/linux/mm.h b/sys/compat/linuxkpi/common/include/linux/mm.h
	--- a/sys/compat/linuxkpi/common/include/linux/mm.h
	+++ b/sys/compat/linuxkpi/common/include/linux/mm.h
	@@ -125,6 +125,7 @@
	int *vm_pfn_pcount;
	vm_object_t vm_obj;
	vm_map_t vm_cached_map;
	+ void *vm_pfn_state;
	TAILQ_ENTRY(vm_area_struct) vm_entry;
	};

	@@ -140,6 +141,25 @@
	struct vm_area_struct *vma;
	};

	+int lkpi_vma_pfn_init(struct vm_area_struct *vma);
	+int lkpi_vma_pfn_begin(struct vm_area_struct vma, uint64_t invalidation_seq);
	+bool lkpi_vma_pfn_unchanged(struct vm_area_struct *vma,
	+ uint64_t invalidation_seq);
	+bool lkpi_vma_pfn_handoff_valid(struct vm_area_struct *vma);
	+bool lkpi_vma_pfn_lock(struct vm_area_struct *vma);
	+void lkpi_vma_pfn_end(struct vm_area_struct *vma);
	+vm_page_t lkpi_vma_pfn_take_page(struct vm_area_struct *vma,
	+ vm_object_t object, vm_pindex_t pindex);
	+void lkpi_vma_pfn_abort(struct vm_area_struct *vma, vm_object_t object);
	+void lkpi_vma_pfn_done(struct vm_area_struct *vma, vm_object_t object);
	+bool lkpi_vma_pfn_invalidate_begin(struct vm_area_struct *vma);
	+bool lkpi_vma_pfn_unmap_begin(struct vm_area_struct *vma);
	+void lkpi_vma_pfn_unmap(struct vm_area_struct *vma);
	+void lkpi_vma_pfn_unmap_end(struct vm_area_struct *vma);
	+void lkpi_vma_pfn_invalidate_end(struct vm_area_struct *vma);
	+void lkpi_vma_pfn_fini(struct vm_area_struct *vma);
	+void linux_cdev_pager_free_pages(vm_object_t object);
	+
	struct vm_operations_struct {
	void (open) (struct vm_area_struct );
	void (close) (struct vm_area_struct );
	diff --git a/sys/compat/linuxkpi/common/src/linux_compat.c b/sys/compat/linuxkpi/common/src/linux_compat.c
	--- a/sys/compat/linuxkpi/common/src/linux_compat.c
	+++ b/sys/compat/linuxkpi/common/src/linux_compat.c
	@@ -507,6 +507,9 @@
	vm_prot_t max_prot, vm_pindex_t first, vm_pindex_t last)
	{
	struct vm_area_struct *vmap;
	+ bool pfn_started;
	+ bool retry;
	+ uint64_t invalidation_seq;
	int err;

	/* get VM area structure */
	@@ -514,14 +517,26 @@
	MPASS(vmap != NULL);
	MPASS(vmap->vm_private_data == vm_obj->handle);

	+retry_fault:
	VM_OBJECT_WUNLOCK(vm_obj);
	-
	linux_set_current(curthread);
	-
	down_write(&vmap->vm_mm->mmap_sem);
	- if (unlikely(vmap->vm_ops == NULL)) {
	- err = VM_FAULT_SIGBUS;
	+ pfn_started = false;
	+ retry = false;
	+ err = lkpi_vma_pfn_begin(vmap, &invalidation_seq);
	+ if (err != 0) {
	+ err = err == ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
	} else {
	+ pfn_started = true;
	+ /* The previous populate may still be completing its handoff. */
	+ atomic_store_rel_ptr((volatile uintptr_t *)&vmap->vm_obj,
	+ (uintptr_t)vm_obj);
	+ vmap->vm_pfn_count = 0;
	+ vmap->vm_pfn_pcount = &vmap->vm_pfn_count;
	+ }
	+ if (err == 0 && unlikely(vmap->vm_ops == NULL)) {
	+ err = VM_FAULT_SIGBUS;
	+ } else if (err == 0) {
	struct vm_fault vmf;

	/* fill out VM fault structure */
	@@ -531,33 +546,39 @@
	vmf.page = NULL;
	vmf.vma = vmap;

	- vmap->vm_pfn_count = 0;
	- vmap->vm_pfn_pcount = &vmap->vm_pfn_count;
	- vmap->vm_obj = vm_obj;
	-
	err = vmap->vm_ops->fault(&vmf);

	- while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE) {
	+ while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE &&
	+ lkpi_vma_pfn_unchanged(vmap, invalidation_seq)) {
	kern_yield(PRI_USER);
	err = vmap->vm_ops->fault(&vmf);
	}
	+ retry = !lkpi_vma_pfn_unchanged(vmap, invalidation_seq);
	}

	/* translate return code */
	- switch (err) {
	+ switch (retry ? VM_FAULT_RETRY : err) {
	+ case VM_FAULT_RETRY:
	+ err = VM_PAGER_ERROR;
	+ break;
	case VM_FAULT_OOM:
	err = VM_PAGER_AGAIN;
	break;
	case VM_FAULT_SIGBUS:
	- err = VM_PAGER_BAD;
	+ err = VM_PAGER_OUT_OF_BOUNDS;
	break;
	case VM_FAULT_NOPAGE:
	/*
	- * By contract the fault handler will return having
	- * busied all the pages itself. If pidx is already
	- * found in the object, it will simply xbusy the first
	- * page and return with vm_pfn_count set to 1.
	+ * By contract, the fault handler returns with every
	+ * populated page exclusively busied. A page is either
	+ * installed in the pager object or handed to the VM by
	+ * cdev_pg_populate_take_page(). The two representations
	+ * cannot be mixed in one populated range.
	*/
	+ if (!lkpi_vma_pfn_handoff_valid(vmap)) {
	+ err = VM_PAGER_ERROR;
	+ break;
	+ }
	*first = vmap->vm_pfn_first;
	last = first + vmap->vm_pfn_count - 1;
	err = VM_PAGER_OK;
	@@ -568,9 +589,39 @@
	}
	up_write(&vmap->vm_mm->mmap_sem);
	VM_OBJECT_WLOCK(vm_obj);
	+ if (err != VM_PAGER_OK && pfn_started) {
	+ if (vmap->vm_pfn_count != 0)
	+ lkpi_vma_pfn_abort(vmap, vm_obj);
	+ lkpi_vma_pfn_end(vmap);
	+ }
	+ if (retry)
	+ goto retry_fault;
	return (err);
	}

	+static vm_page_t
	+linux_cdev_pager_populate_take_page(vm_object_t vm_obj, vm_pindex_t pidx)
	+{
	+ struct vm_area_struct *vmap;
	+
	+ vmap = linux_cdev_handle_find(vm_obj->handle);
	+ MPASS(vmap != NULL);
	+ MPASS(vmap->vm_private_data == vm_obj->handle);
	+ return (lkpi_vma_pfn_take_page(vmap, vm_obj, pidx));
	+}
	+
	+static void
	+linux_cdev_pager_populate_done(vm_object_t vm_obj)
	+{
	+ struct vm_area_struct *vmap;
	+
	+ vmap = linux_cdev_handle_find(vm_obj->handle);
	+ MPASS(vmap != NULL);
	+ MPASS(vmap->vm_private_data == vm_obj->handle);
	+ lkpi_vma_pfn_done(vmap, vm_obj);
	+ lkpi_vma_pfn_end(vmap);
	+}
	+
	static struct rwlock linux_vma_lock;
	static TAILQ_HEAD(, vm_area_struct) linux_vma_head =
	TAILQ_HEAD_INITIALIZER(linux_vma_head);
	@@ -620,6 +671,22 @@
	return (0);
	}

	+static int
	+linux_cdev_mgtdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
	+ vm_ooffset_t foff, struct ucred cred, u_short color)
	+{
	+ struct vm_area_struct *vmap;
	+ int error;
	+
	+ vmap = linux_cdev_handle_find(handle);
	+ MPASS(vmap != NULL);
	+ error = lkpi_vma_pfn_init(vmap);
	+ if (error != 0)
	+ return (error);
	+ *color = 0;
	+ return (0);
	+}
	+
	static void
	linux_cdev_pager_dtor(void *handle)
	{
	@@ -636,6 +703,7 @@
	linux_cdev_handle_remove(vmap);

	down_write(&vmap->vm_mm->mmap_sem);
	+ lkpi_vma_pfn_fini(vmap);
	vm_ops = vmap->vm_ops;
	if (likely(vm_ops != NULL))
	vm_ops->close(vmap);
	@@ -644,11 +712,25 @@
	linux_cdev_handle_free(vmap);
	}

	+static bool
	+linux_cdev_pager_mlock_skip(void *handle)
	+{
	+ struct vm_area_struct *vmap;
	+
	+ vmap = linux_cdev_handle_find(handle);
	+ MPASS(vmap != NULL);
	+ /* Linux does not mlock special IO/PFN mappings. */
	+ return ((vmap->vm_flags & (VM_IO \| VM_PFNMAP)) != 0);
	+}
	+
	static struct cdev_pager_ops linux_cdev_pager_ops[2] = {
	{
	/* OBJT_MGTDEVICE */
	.cdev_pg_populate = linux_cdev_pager_populate,
	- .cdev_pg_ctor = linux_cdev_pager_ctor,
	+ .cdev_pg_populate_take_page = linux_cdev_pager_populate_take_page,
	+ .cdev_pg_populate_done = linux_cdev_pager_populate_done,
	+ .cdev_pg_mlock_skip = linux_cdev_pager_mlock_skip,
	+ .cdev_pg_ctor = linux_cdev_mgtdev_pager_ctor,
	.cdev_pg_dtor = linux_cdev_pager_dtor
	},
	{
	@@ -659,6 +741,34 @@
	},
	};

	+void
	+linux_cdev_pager_free_pages(vm_object_t object)
	+{
	+ struct vm_area_struct *vmap;
	+ bool invalidating;
	+
	+ vmap = linux_cdev_handle_find(object->handle);
	+ if (vmap == NULL) {
	+ cdev_mgtdev_pager_free_pages(object);
	+ return;
	+ }
	+ invalidating = lkpi_vma_pfn_invalidate_begin(vmap);
	+ if (!invalidating) {
	+ cdev_mgtdev_pager_free_pages(object);
	+ return;
	+ }
	+ /*
	+ * Mark invalidation before removing either representation. A fault
	+ * which has entered the driver but not yet selected a page will reject
	+ * its result. A page selected earlier remains xbusy until
	+ * vm_fault_populate() installs its PTE, so either removal pass waits and
	+ * revokes that PTE before invalidation completes.
	+ */
	+ cdev_mgtdev_pager_free_pages(object);
	+ lkpi_vma_pfn_unmap(vmap);
	+ lkpi_vma_pfn_invalidate_end(vmap);
	+}
	+
	int
	zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
	unsigned long size)
	@@ -666,16 +776,49 @@
	struct pctrie_iter pages;
	vm_object_t obj;
	vm_page_t m;
	+ vm_pindex_t first, last;
	+ bool unmapping;
	+ bool pfn_locked;
	+ int error;

	- obj = vma->vm_obj;
	- if (obj == NULL \|\| (obj->flags & OBJ_UNMANAGED) != 0)
	- return (-ENOTSUP);
	- VM_OBJECT_RLOCK(obj);
	- vm_page_iter_limit_init(&pages, obj, OFF_TO_IDX(address + size));
	- VM_RADIX_FOREACH_FROM(m, &pages, OFF_TO_IDX(address))
	+ pfn_locked = lkpi_vma_pfn_lock(vma);
	+ obj = (void *)atomic_load_acq_ptr(
	+ (volatile uintptr_t *)&vma->vm_obj);
	+ if (obj == NULL \|\| (obj->flags & OBJ_UNMANAGED) != 0) {
	+ error = -ENOTSUP;
	+ goto out;
	+ }
	+ if (size == 0) {
	+ error = 0;
	+ goto out;
	+ }
	+ if (offset_in_page(address) != 0 \|\| offset_in_page(size) != 0 \|\|
	+ address < vma->vm_start \|\| address >= vma->vm_end \|\|
	+ size > vma->vm_end - address) {
	+ error = -EINVAL;
	+ goto out;
	+ }
	+ first = OFF_TO_IDX(address - vma->vm_start);
	+ last = OFF_TO_IDX(address + size - vma->vm_start);
	+ VM_OBJECT_WLOCK(obj);
	+ if (vma->vm_pfn_count != 0)
	+ lkpi_vma_pfn_abort(vma, obj);
	+ VM_OBJECT_WUNLOCK(obj);
	+ unmapping = lkpi_vma_pfn_unmap_begin(vma);
	+ if (unmapping)
	+ lkpi_vma_pfn_unmap(vma);
	+ VM_OBJECT_WLOCK(obj);
	+ vm_page_iter_limit_init(&pages, obj, last);
	+ VM_RADIX_FOREACH_FROM(m, &pages, first)
	pmap_remove_all(m);
	- VM_OBJECT_RUNLOCK(obj);
	- return (0);
	+ VM_OBJECT_WUNLOCK(obj);
	+ if (unmapping)
	+ lkpi_vma_pfn_unmap_end(vma);
	+ error = 0;
	+out:
	+ if (pfn_locked)
	+ lkpi_vma_pfn_end(vma);
	+ return (error);
	}

	void
	diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c
	--- a/sys/compat/linuxkpi/common/src/linux_page.c
	+++ b/sys/compat/linuxkpi/common/src/linux_page.c
	@@ -31,9 +31,11 @@
	#include <sys/malloc.h>
	#include <sys/kernel.h>
	#include <sys/sysctl.h>
	+#include <sys/condvar.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/rwlock.h>
	+#include <sys/sx.h>
	#include <sys/proc.h>
	#include <sys/sched.h>
	#include <sys/memrange.h>
	@@ -65,6 +67,7 @@
	#include <linux/idr.h>
	#include <linux/io.h>
	#include <linux/io-mapping.h>
	+#include <linux/slab.h>

	#ifdef __i386__
	DEFINE_IDR(mtrr_idr);
	@@ -505,22 +508,552 @@
	kfree(vmmap);
	}

	+struct lkpi_vma_pfn_object {
	+ TAILQ_ENTRY(lkpi_vma_pfn_object) link;
	+ vm_object_t object;
	+ vm_pindex_t first;
	+ vm_pindex_t last;
	+};
	+
	+#define LKPI_VMA_PFN_CHUNK_PAGES 64
	+
	+struct lkpi_vma_pfn_chunk {
	+ TAILQ_ENTRY(lkpi_vma_pfn_chunk) link;
	+ vm_pindex_t first;
	+ unsigned int count;
	+ unsigned int remaining;
	+ vm_page_t pages[LKPI_VMA_PFN_CHUNK_PAGES];
	+};
	+
	+struct lkpi_vma_pfn_state {
	+ TAILQ_HEAD(, lkpi_vma_pfn_object) objects;
	+ TAILQ_HEAD(, lkpi_vma_pfn_chunk) page_chunks;
	+ struct mtx objects_lock;
	+ struct cv invalidation_cv;
	+ struct sx populate_lock;
	+ struct sx unmap_lock;
	+ struct lkpi_vma_pfn_chunk *last_chunk;
	+ vm_pindex_t npages;
	+ vm_pindex_t pending;
	+ uint64_t invalidation_seq;
	+};
	+
	+static struct lkpi_vma_pfn_state *
	+lkpi_vma_pfn_get_state(struct vm_area_struct *vma)
	+{
	+
	+ return ((void *)atomic_load_acq_ptr(
	+ (volatile uintptr_t *)&vma->vm_pfn_state));
	+}
	+
	+int
	+lkpi_vma_pfn_init(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+ vm_pindex_t npages;
	+
	+ MPASS(lkpi_vma_pfn_get_state(vma) == NULL);
	+ npages = vma_pages(vma);
	+ if (npages == 0)
	+ return (EINVAL);
	+ state = kzalloc(sizeof(*state), GFP_KERNEL);
	+ if (state == NULL)
	+ return (ENOMEM);
	+ TAILQ_INIT(&state->objects);
	+ TAILQ_INIT(&state->page_chunks);
	+ mtx_init(&state->objects_lock, "lkpi pfn objects", NULL, MTX_DEF);
	+ cv_init(&state->invalidation_cv, "lkpi pfn invalidate");
	+ sx_init(&state->populate_lock, "lkpi pfn populate");
	+ sx_init(&state->unmap_lock, "lkpi pfn unmap");
	+ state->npages = npages;
	+ atomic_store_rel_ptr((volatile uintptr_t *)&vma->vm_pfn_state,
	+ (uintptr_t)state);
	+ return (0);
	+}
	+
	+int
	+lkpi_vma_pfn_begin(struct vm_area_struct vma, uint64_t invalidation_seq)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ if (state == NULL)
	+ return (EINVAL);
	+ sx_xlock(&state->populate_lock);
	+ if (state->pending != 0 \|\| !TAILQ_EMPTY(&state->page_chunks)) {
	+ sx_xunlock(&state->populate_lock);
	+ return (EBUSY);
	+ }
	+ mtx_lock(&state->objects_lock);
	+ while ((state->invalidation_seq & 1) != 0)
	+ cv_wait(&state->invalidation_cv, &state->objects_lock);
	+ *invalidation_seq = state->invalidation_seq;
	+ mtx_unlock(&state->objects_lock);
	+ return (0);
	+}
	+
	+bool
	+lkpi_vma_pfn_unchanged(struct vm_area_struct *vma,
	+ uint64_t invalidation_seq)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+ bool unchanged;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ if (state == NULL)
	+ return (false);
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ mtx_lock(&state->objects_lock);
	+ unchanged = state->invalidation_seq == invalidation_seq;
	+ mtx_unlock(&state->objects_lock);
	+ return (unchanged);
	+}
	+
	+bool
	+lkpi_vma_pfn_handoff_valid(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ MPASS(state != NULL);
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ MPASS(vma->vm_pfn_count > 0);
	+ return (state->pending == 0 \|\|
	+ state->pending == (vm_pindex_t)vma->vm_pfn_count);
	+}
	+
	+bool
	+lkpi_vma_pfn_lock(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ if (state == NULL \|\| sx_xlocked(&state->populate_lock))
	+ return (false);
	+ sx_xlock(&state->populate_lock);
	+ return (true);
	+}
	+
	+void
	+lkpi_vma_pfn_end(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ MPASS(state != NULL);
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ MPASS(state->pending == 0);
	+ sx_xunlock(&state->populate_lock);
	+}
	+
	+static bool
	+lkpi_vma_pfn_object_is_tracked_locked(struct lkpi_vma_pfn_state *state,
	+ vm_object_t object)
	+{
	+ struct lkpi_vma_pfn_object *entry;
	+
	+ mtx_assert(&state->objects_lock, MA_OWNED);
	+ TAILQ_FOREACH(entry, &state->objects, link) {
	+ if (entry->object == object)
	+ return (true);
	+ }
	+ return (false);
	+}
	+
	+static bool
	+lkpi_vma_pfn_object_is_tracked(struct lkpi_vma_pfn_state *state,
	+ vm_object_t object)
	+{
	+ bool tracked;
	+
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ mtx_lock(&state->objects_lock);
	+ tracked = lkpi_vma_pfn_object_is_tracked_locked(state, object);
	+ mtx_unlock(&state->objects_lock);
	+ return (tracked);
	+}
	+
	+static void
	+lkpi_vma_pfn_drop_object_ref(vm_object_t locked_object,
	+ vm_object_t referenced_object)
	+{
	+
	+ VM_OBJECT_ASSERT_WLOCKED(locked_object);
	+ VM_OBJECT_WUNLOCK(locked_object);
	+ vm_object_deallocate(referenced_object);
	+ VM_OBJECT_WLOCK(locked_object);
	+}
	+
	+static int
	+lkpi_vma_pfn_track_object(struct lkpi_vma_pfn_state *state,
	+ vm_object_t object, vm_pindex_t pindex, bool have_reference,
	+ bool *reference_consumed)
	+{
	+ struct lkpi_vma_pfn_object *entry;
	+
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ *reference_consumed = false;
	+ mtx_lock(&state->objects_lock);
	+ TAILQ_FOREACH(entry, &state->objects, link) {
	+ if (entry->object == object) {
	+ entry->first = MIN(entry->first, pindex);
	+ entry->last = MAX(entry->last, pindex);
	+ mtx_unlock(&state->objects_lock);
	+ return (0);
	+ }
	+ }
	+ mtx_unlock(&state->objects_lock);
	+ if (!have_reference)
	+ return (ESTALE);
	+ entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
	+ if (entry == NULL) {
	+ return (ENOMEM);
	+ }
	+ entry->object = object;
	+ entry->first = pindex;
	+ entry->last = pindex;
	+ mtx_lock(&state->objects_lock);
	+ KASSERT(!lkpi_vma_pfn_object_is_tracked_locked(state, object),
	+ ("%s: duplicate object %p", __func__, object));
	+ TAILQ_INSERT_TAIL(&state->objects, entry, link);
	+ mtx_unlock(&state->objects_lock);
	+ *reference_consumed = true;
	+ return (0);
	+}
	+
	+static int
	+lkpi_vma_pfn_store_page(struct lkpi_vma_pfn_state *state,
	+ vm_pindex_t pindex, vm_page_t page)
	+{
	+ struct lkpi_vma_pfn_chunk *chunk;
	+
	+ MPASS(page != NULL);
	+ MPASS(vm_page_xbusied(page));
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ MPASS(pindex < state->npages);
	+ MPASS(state->pending < state->npages);
	+ chunk = state->last_chunk;
	+ if (chunk == NULL \|\| chunk->count == nitems(chunk->pages) \|\|
	+ pindex != chunk->first + chunk->count) {
	+ chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
	+ if (chunk == NULL)
	+ return (ENOMEM);
	+ chunk->first = pindex;
	+ TAILQ_INSERT_TAIL(&state->page_chunks, chunk, link);
	+ state->last_chunk = chunk;
	+ }
	+ MPASS(chunk->pages[chunk->count] == NULL);
	+ chunk->pages[chunk->count++] = page;
	+ chunk->remaining++;
	+ state->pending++;
	+ return (0);
	+}
	+
	+static bool
	+lkpi_vma_pfn_page_is_selected(struct vm_area_struct *vma, vm_page_t page)
	+{
	+ struct lkpi_vma_pfn_chunk *chunk;
	+ struct lkpi_vma_pfn_state *state;
	+ unsigned int slot;
	+
	+ VM_OBJECT_ASSERT_WLOCKED(vma->vm_obj);
	+ state = lkpi_vma_pfn_get_state(vma);
	+ MPASS(state != NULL);
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ TAILQ_FOREACH(chunk, &state->page_chunks, link) {
	+ for (slot = 0; slot < chunk->count; slot++) {
	+ if (chunk->pages[slot] == page)
	+ return (true);
	+ }
	+ }
	+ return (atomic_load_ptr(&page->object) == vma->vm_obj &&
	+ page->pindex >= vma->vm_pfn_first &&
	+ page->pindex - vma->vm_pfn_first <
	+ (vm_pindex_t)vma->vm_pfn_count);
	+}
	+
	+vm_page_t
	+lkpi_vma_pfn_take_page(struct vm_area_struct *vma, vm_object_t object,
	+ vm_pindex_t pindex)
	+{
	+ struct lkpi_vma_pfn_chunk *chunk;
	+ struct lkpi_vma_pfn_state *state;
	+ vm_page_t page;
	+ unsigned int slot;
	+
	+ VM_OBJECT_ASSERT_WLOCKED(object);
	+ state = lkpi_vma_pfn_get_state(vma);
	+ if (state == NULL \|\| pindex >= state->npages)
	+ return (NULL);
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ TAILQ_FOREACH(chunk, &state->page_chunks, link) {
	+ if (pindex < chunk->first)
	+ break;
	+ if (pindex - chunk->first >= chunk->count)
	+ continue;
	+ slot = pindex - chunk->first;
	+ page = chunk->pages[slot];
	+ if (page == NULL)
	+ return (NULL);
	+ chunk->pages[slot] = NULL;
	+ MPASS(chunk->remaining > 0 && state->pending > 0);
	+ chunk->remaining--;
	+ state->pending--;
	+ if (chunk->remaining == 0) {
	+ TAILQ_REMOVE(&state->page_chunks, chunk, link);
	+ if (state->last_chunk == chunk)
	+ state->last_chunk = NULL;
	+ kfree(chunk);
	+ }
	+ return (page);
	+ }
	+ return (NULL);
	+}
	+
	+void
	+lkpi_vma_pfn_abort(struct vm_area_struct *vma, vm_object_t object)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+ vm_page_t page;
	+ vm_pindex_t count, pindex;
	+
	+ VM_OBJECT_ASSERT_WLOCKED(object);
	+ state = lkpi_vma_pfn_get_state(vma);
	+ if (state != NULL)
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ count = vma->vm_pfn_count;
	+ for (pindex = vma->vm_pfn_first; count != 0;
	+ count--, pindex++) {
	+ page = lkpi_vma_pfn_take_page(vma, object, pindex);
	+ if (page == NULL)
	+ page = vm_page_lookup(object, pindex);
	+ if (page != NULL) {
	+ vm_page_deactivate(page);
	+ vm_page_xunbusy(page);
	+ }
	+ }
	+ vma->vm_pfn_count = 0;
	+ KASSERT(state == NULL \|\| state->pending == 0,
	+ ("%s: %ju pages remain pending", __func__,
	+ state == NULL ? 0 : (uintmax_t)state->pending));
	+}
	+
	+void
	+lkpi_vma_pfn_done(struct vm_area_struct *vma, vm_object_t object)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+
	+ VM_OBJECT_ASSERT_WLOCKED(object);
	+ state = lkpi_vma_pfn_get_state(vma);
	+ MPASS(state != NULL);
	+ sx_assert(&state->populate_lock, SA_XLOCKED);
	+ MPASS(state->pending == 0 &&
	+ TAILQ_EMPTY(&state->page_chunks));
	+ vma->vm_pfn_count = 0;
	+}
	+
	+bool
	+lkpi_vma_pfn_unmap_begin(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ if (state == NULL)
	+ return (false);
	+ sx_xlock(&state->unmap_lock);
	+ return (true);
	+}
	+
	+void
	+lkpi_vma_pfn_unmap_end(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ MPASS(state != NULL);
	+ sx_assert(&state->unmap_lock, SA_XLOCKED);
	+ sx_xunlock(&state->unmap_lock);
	+}
	+
	+bool
	+lkpi_vma_pfn_invalidate_begin(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+
	+ if (!lkpi_vma_pfn_unmap_begin(vma))
	+ return (false);
	+ state = lkpi_vma_pfn_get_state(vma);
	+ MPASS(state != NULL);
	+ mtx_lock(&state->objects_lock);
	+ MPASS((state->invalidation_seq & 1) == 0);
	+ state->invalidation_seq++;
	+ mtx_unlock(&state->objects_lock);
	+ return (true);
	+}
	+
	+static void
	+lkpi_vma_pfn_unmap_object(vm_object_t object, vm_pindex_t first,
	+ vm_pindex_t last)
	+{
	+ struct pctrie_iter pages;
	+ vm_page_t page;
	+
	+ vm_page_iter_init(&pages, object);
	+ VM_OBJECT_WLOCK(object);
	+retry:
	+ KASSERT(pctrie_iter_is_reset(&pages),
	+ ("%s: pctrie iterator is not reset", __func__));
	+ VM_RADIX_FOREACH_FROM(page, &pages, first) {
	+ if (page->pindex > last)
	+ break;
	+ /*
	+ * Busy even an apparently unmapped page. A concurrent fault can
	+ * hold it busy before installing its PTE; skipping it here would
	+ * let that PTE appear after this invalidation pass.
	+ */
	+ if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
	+ pctrie_iter_reset(&pages);
	+ goto retry;
	+ }
	+ if (pmap_page_is_mapped(page))
	+ pmap_remove_all(page);
	+ vm_page_xunbusy(page);
	+ }
	+ VM_OBJECT_WUNLOCK(object);
	+}
	+
	+void
	+lkpi_vma_pfn_unmap(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_object *entry;
	+ struct lkpi_vma_pfn_state *state;
	+ vm_object_t object;
	+ vm_pindex_t first, last;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ if (state == NULL)
	+ return;
	+ sx_assert(&state->unmap_lock, SA_XLOCKED);
	+ mtx_lock(&state->objects_lock);
	+ entry = TAILQ_FIRST(&state->objects);
	+ while (entry != NULL) {
	+ /* Entries and their object references live until VMA teardown. */
	+ object = entry->object;
	+ first = entry->first;
	+ last = entry->last;
	+ entry = TAILQ_NEXT(entry, link);
	+ mtx_unlock(&state->objects_lock);
	+ lkpi_vma_pfn_unmap_object(object, first, last);
	+ mtx_lock(&state->objects_lock);
	+ }
	+ mtx_unlock(&state->objects_lock);
	+}
	+
	+void
	+lkpi_vma_pfn_invalidate_end(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_state *state;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ MPASS(state != NULL);
	+ sx_assert(&state->unmap_lock, SA_XLOCKED);
	+ mtx_lock(&state->objects_lock);
	+ MPASS((state->invalidation_seq & 1) != 0);
	+ state->invalidation_seq++;
	+ cv_broadcast(&state->invalidation_cv);
	+ mtx_unlock(&state->objects_lock);
	+ lkpi_vma_pfn_unmap_end(vma);
	+}
	+
	+void
	+lkpi_vma_pfn_fini(struct vm_area_struct *vma)
	+{
	+ struct lkpi_vma_pfn_chunk *chunk;
	+ struct lkpi_vma_pfn_object *entry;
	+ struct lkpi_vma_pfn_state *state;
	+ vm_page_t page;
	+ unsigned int slot;
	+
	+ state = lkpi_vma_pfn_get_state(vma);
	+ if (state == NULL)
	+ return;
	+ sx_assert(&state->populate_lock, SA_UNLOCKED);
	+ mtx_lock(&state->objects_lock);
	+ MPASS((state->invalidation_seq & 1) == 0);
	+ mtx_unlock(&state->objects_lock);
	+ atomic_store_rel_ptr((volatile uintptr_t *)&vma->vm_pfn_state, 0);
	+ while ((chunk = TAILQ_FIRST(&state->page_chunks)) != NULL) {
	+ TAILQ_REMOVE(&state->page_chunks, chunk, link);
	+ for (slot = 0; slot < chunk->count; slot++) {
	+ page = chunk->pages[slot];
	+ if (page != NULL) {
	+ MPASS(state->pending > 0);
	+ state->pending--;
	+ vm_page_deactivate(page);
	+ vm_page_xunbusy(page);
	+ }
	+ }
	+ kfree(chunk);
	+ }
	+ state->last_chunk = NULL;
	+ MPASS(state->pending == 0);
	+ sx_xlock(&state->unmap_lock);
	+ for (;;) {
	+ mtx_lock(&state->objects_lock);
	+ entry = TAILQ_FIRST(&state->objects);
	+ if (entry != NULL)
	+ TAILQ_REMOVE(&state->objects, entry, link);
	+ mtx_unlock(&state->objects_lock);
	+ if (entry == NULL)
	+ break;
	+ vm_object_deallocate(entry->object);
	+ kfree(entry);
	+ }
	+ sx_xunlock(&state->unmap_lock);
	+ sx_destroy(&state->unmap_lock);
	+ sx_destroy(&state->populate_lock);
	+ cv_destroy(&state->invalidation_cv);
	+ mtx_destroy(&state->objects_lock);
	+ kfree(state);
	+}
	+
	vm_fault_t
	lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr,
	unsigned long pfn, pgprot_t prot)
	{
	+ struct lkpi_vma_pfn_state *state;
	struct pctrie_iter pages;
	vm_object_t vm_obj = vma->vm_obj;
	vm_object_t tmp_obj;
	vm_page_t page;
	vm_pindex_t pindex;
	+ vm_memattr_t memattr;
	+ bool have_reference, reference_consumed, tracked;
	+ int error;

	VM_OBJECT_ASSERT_WLOCKED(vm_obj);
	+ if (offset_in_page(addr) != 0 \|\| addr < vma->vm_start \|\|
	+ addr >= vma->vm_end \|\| (vm_pindex_t)pfn != pfn \|\|
	+ OFF_TO_IDX(IDX_TO_OFF((vm_pindex_t)pfn)) !=
	+ (vm_pindex_t)pfn)
	+ return (VM_FAULT_SIGBUS);
	+ state = lkpi_vma_pfn_get_state(vma);
	+ if (state == NULL)
	+ return (VM_FAULT_SIGBUS);
	+ if (vma->vm_pfn_count < 0 \|\| vma->vm_pfn_count == INT_MAX)
	+ return (VM_FAULT_SIGBUS);
	vm_page_iter_init(&pages, vm_obj);
	pindex = OFF_TO_IDX(addr - vma->vm_start);
	- if (vma->vm_pfn_count == 0)
	+ if (pindex >= state->npages)
	+ return (VM_FAULT_SIGBUS);
	+ if (vma->vm_pfn_count != 0 &&
	+ pindex != vma->vm_pfn_first + vma->vm_pfn_count)
	+ return (VM_FAULT_SIGBUS);
	+ if (vma->vm_pfn_count == 0) {
	vma->vm_pfn_first = pindex;
	- MPASS(pindex <= OFF_TO_IDX(vma->vm_end));
	+ }
	+ MPASS(pindex < OFF_TO_IDX(vma->vm_end));
	+ memattr = pgprot2cachemode(prot);

	retry:
	page = vm_page_grab_iter(vm_obj, pindex, VM_ALLOC_NOCREAT, &pages);
	@@ -528,12 +1061,172 @@
	page = PHYS_TO_VM_PAGE(IDX_TO_OFF(pfn));
	if (page == NULL)
	return (VM_FAULT_SIGBUS);
	- if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
	+ tmp_obj = atomic_load_ptr(&page->object);
	+ have_reference = false;
	+ tracked = tmp_obj != NULL && tmp_obj != vm_obj &&
	+ lkpi_vma_pfn_object_is_tracked(state, tmp_obj);
	+ if (tmp_obj != NULL && tmp_obj != vm_obj && !tracked) {
	+ /*
	+ * VM object locks are type-stable. Lock and revalidate the
	+ * source before taking the reference that will protect this VMA.
	+ */
	+ VM_OBJECT_WUNLOCK(vm_obj);
	+ VM_OBJECT_WLOCK(tmp_obj);
	+ if (page->object != tmp_obj \|\|
	+ (tmp_obj->flags & OBJ_DEAD) != 0) {
	+ VM_OBJECT_WUNLOCK(tmp_obj);
	+ VM_OBJECT_WLOCK(vm_obj);
	+ pctrie_iter_reset(&pages);
	+ goto retry;
	+ }
	+ vm_object_reference_locked(tmp_obj);
	+ VM_OBJECT_WUNLOCK(tmp_obj);
	+ VM_OBJECT_WLOCK(vm_obj);
	+ if (vm_page_lookup(vm_obj, pindex) != NULL \|\|
	+ atomic_load_ptr(&page->object) != tmp_obj) {
	+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	+ pctrie_iter_reset(&pages);
	+ goto retry;
	+ }
	+ have_reference = true;
	+ }
	+ if (!vm_page_tryxbusy(page)) {
	+ /*
	+ * A selected page stays xbusy until this transaction is
	+ * consumed or aborted. Refuse an alias rather than wait
	+ * for busy ownership that this transaction must release.
	+ */
	+ if (lkpi_vma_pfn_page_is_selected(vma, page)) {
	+ if (have_reference)
	+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	+ return (VM_FAULT_SIGBUS);
	+ }
	+ if (have_reference)
	+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	+ else {
	+ VM_OBJECT_WUNLOCK(vm_obj);
	+ kern_yield(PRI_USER);
	+ VM_OBJECT_WLOCK(vm_obj);
	+ }
	pctrie_iter_reset(&pages);
	goto retry;
	}
	+ if (page->object != tmp_obj) {
	+ vm_page_xunbusy(page);
	+ if (have_reference)
	+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	+ pctrie_iter_reset(&pages);
	+ goto retry;
	+ }
	+ /*
	+ * Linux installs a special PFN PTE without moving a managed page
	+ * out of its backing object. Preserve that ownership for shmem
	+ * pages and hand the xbusy page directly to vm_fault_populate().
	+ */
	+ if (tmp_obj != NULL && tmp_obj != vm_obj &&
	+ tmp_obj->type == OBJT_SWAP &&
	+ (page->oflags & VPO_UNMANAGED) == 0) {
	+ if (!vm_page_all_valid(page)) {
	+ vm_page_xunbusy(page);
	+ if (have_reference)
	+ lkpi_vma_pfn_drop_object_ref(vm_obj,
	+ tmp_obj);
	+ return (VM_FAULT_SIGBUS);
	+ }
	+ if (pmap_page_get_memattr(page) != memattr &&
	+ pmap_page_is_mapped(page)) {
	+ vm_page_xunbusy(page);
	+ if (have_reference)
	+ lkpi_vma_pfn_drop_object_ref(vm_obj,
	+ tmp_obj);
	+ return (VM_FAULT_SIGBUS);
	+ }
	+ if (pmap_page_get_memattr(page) != memattr) {
	+ vm_page_xunbusy(page);
	+ VM_OBJECT_WUNLOCK(vm_obj);
	+ VM_OBJECT_WLOCK(tmp_obj);
	+ if (page->object != tmp_obj \|\|
	+ !vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
	+ VM_OBJECT_WUNLOCK(tmp_obj);
	+ if (have_reference)
	+ vm_object_deallocate(tmp_obj);
	+ VM_OBJECT_WLOCK(vm_obj);
	+ pctrie_iter_reset(&pages);
	+ goto retry;
	+ }
	+ if (page->object != tmp_obj \|\|
	+ page->object->type != OBJT_SWAP \|\|
	+ (page->oflags & VPO_UNMANAGED) != 0 \|\|
	+ !vm_page_all_valid(page)) {
	+ vm_page_xunbusy(page);
	+ VM_OBJECT_WUNLOCK(tmp_obj);
	+ if (have_reference)
	+ vm_object_deallocate(tmp_obj);
	+ VM_OBJECT_WLOCK(vm_obj);
	+ return (VM_FAULT_SIGBUS);
	+ }
	+ if (pmap_page_get_memattr(page) != memattr &&
	+ pmap_page_is_mapped(page)) {
	+ vm_page_xunbusy(page);
	+ VM_OBJECT_WUNLOCK(tmp_obj);
	+ if (have_reference)
	+ vm_object_deallocate(tmp_obj);
	+ VM_OBJECT_WLOCK(vm_obj);
	+ return (VM_FAULT_SIGBUS);
	+ }
	+ if (pmap_page_get_memattr(page) != memattr)
	+ pmap_page_set_memattr(page, memattr);
	+ VM_OBJECT_WUNLOCK(tmp_obj);
	+ VM_OBJECT_WLOCK(vm_obj);
	+ if (vm_page_lookup(vm_obj, pindex) != NULL) {
	+ vm_page_xunbusy(page);
	+ if (have_reference)
	+ lkpi_vma_pfn_drop_object_ref(vm_obj,
	+ tmp_obj);
	+ pctrie_iter_reset(&pages);
	+ goto retry;
	+ }
	+ }
	+ MPASS(page->object == tmp_obj);
	+ error = lkpi_vma_pfn_track_object(state, tmp_obj,
	+ page->pindex, have_reference, &reference_consumed);
	+ if (error != 0) {
	+ vm_page_xunbusy(page);
	+ if (have_reference)
	+ lkpi_vma_pfn_drop_object_ref(vm_obj,
	+ tmp_obj);
	+ return (error == ENOMEM ? VM_FAULT_OOM :
	+ VM_FAULT_SIGBUS);
	+ }
	+ if (have_reference && !reference_consumed) {
	+ /*
	+ * populate_lock normally makes this impossible. Retry
	+ * instead of leaking the redundant reference if another
	+ * producer ever gains access to the tracking list.
	+ */
	+ vm_page_xunbusy(page);
	+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	+ pctrie_iter_reset(&pages);
	+ goto retry;
	+ }
	+ error = lkpi_vma_pfn_store_page(state, pindex, page);
	+ if (error != 0) {
	+ vm_page_xunbusy(page);
	+ return (VM_FAULT_OOM);
	+ }
	+ vma->vm_pfn_count++;
	+ return (VM_FAULT_NOPAGE);
	+ }
	if (page->object != NULL) {
	- tmp_obj = page->object;
	+ if (tracked) {
	+ vm_page_xunbusy(page);
	+ return (VM_FAULT_SIGBUS);
	+ }
	+ if (tmp_obj == vm_obj) {
	+ vm_object_reference_locked(tmp_obj);
	+ have_reference = true;
	+ }
	+ MPASS(have_reference);
	vm_page_xunbusy(page);
	VM_OBJECT_WUNLOCK(vm_obj);
	VM_OBJECT_WLOCK(tmp_obj);
	@@ -541,12 +1234,18 @@
	vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
	KASSERT(page->object == tmp_obj,
	("page has changed identity"));
	- KASSERT((page->oflags & VPO_UNMANAGED) == 0,
	- ("page does not belong to shmem"));
	+ if ((page->oflags & VPO_UNMANAGED) != 0) {
	+ vm_page_xunbusy(page);
	+ VM_OBJECT_WUNLOCK(tmp_obj);
	+ vm_object_deallocate(tmp_obj);
	+ VM_OBJECT_WLOCK(vm_obj);
	+ return (VM_FAULT_SIGBUS);
	+ }
	vm_pager_page_unswapped(page);
	if (pmap_page_is_mapped(page)) {
	vm_page_xunbusy(page);
	VM_OBJECT_WUNLOCK(tmp_obj);
	+ vm_object_deallocate(tmp_obj);
	printf("%s: page rename failed: page "
	"is mapped\n", __func__);
	VM_OBJECT_WLOCK(vm_obj);
	@@ -555,6 +1254,7 @@
	vm_page_remove(page);
	}
	VM_OBJECT_WUNLOCK(tmp_obj);
	+ vm_object_deallocate(tmp_obj);
	pctrie_iter_reset(&pages);
	VM_OBJECT_WLOCK(vm_obj);
	goto retry;
	@@ -565,7 +1265,16 @@
	}
	vm_page_valid(page);
	}
	- pmap_page_set_memattr(page, pgprot2cachemode(prot));
	+ if (VM_PAGE_TO_PHYS(page) != IDX_TO_OFF(pfn)) {
	+ vm_page_xunbusy(page);
	+ return (VM_FAULT_SIGBUS);
	+ }
	+ if (pmap_page_get_memattr(page) != memattr &&
	+ pmap_page_is_mapped(page)) {
	+ vm_page_xunbusy(page);
	+ return (VM_FAULT_SIGBUS);
	+ }
	+ pmap_page_set_memattr(page, memattr);
	vma->vm_pfn_count++;

	return (VM_FAULT_NOPAGE);
	@@ -576,14 +1285,25 @@
	unsigned long start_pfn, unsigned long size, pgprot_t prot)
	{
	vm_object_t vm_obj;
	- unsigned long addr, pfn;
	+ unsigned long addr, end_addr, npages, pfn;
	int err = 0;

	vm_obj = vma->vm_obj;
	+ if (size == 0)
	+ return (0);
	+ if (offset_in_page(start_addr) != 0 \|\| offset_in_page(size) != 0 \|\|
	+ start_addr < vma->vm_start \|\| start_addr >= vma->vm_end \|\|
	+ size > vma->vm_end - start_addr)
	+ return (-EINVAL);
	+ npages = size >> PAGE_SHIFT;
	+ if ((vm_pindex_t)start_pfn != start_pfn \|\|
	+ npages - 1 > ULONG_MAX - start_pfn)
	+ return (-EINVAL);
	+ end_addr = start_addr + size;

	VM_OBJECT_WLOCK(vm_obj);
	for (addr = start_addr, pfn = start_pfn;
	- addr < start_addr + size;
	+ addr != end_addr;
	addr += PAGE_SIZE) {
	vm_fault_t ret;
	retry:
	@@ -606,8 +1326,7 @@
	VM_OBJECT_WUNLOCK(vm_obj);

	if (unlikely(err)) {
	- zap_vma_ptes(vma, start_addr,
	- (pfn - start_pfn) << PAGE_SHIFT);
	+ zap_vma_ptes(vma, start_addr, addr - start_addr);
	return (err);
	}

	@@ -643,7 +1362,7 @@

	devobj = cdev_pager_lookup(obj);
	if (devobj != NULL) {
	- cdev_mgtdev_pager_free_pages(devobj);
	+ linux_cdev_pager_free_pages(devobj);
	vm_object_deallocate(devobj);
	}
	}
	diff --git a/sys/sys/param.h b/sys/sys/param.h
	--- a/sys/sys/param.h
	+++ b/sys/sys/param.h
	@@ -74,7 +74,7 @@
	* cannot include sys/param.h and should only be updated here.
	*/
	#undef __FreeBSD_version
	-#define __FreeBSD_version 1600024
	+#define __FreeBSD_version 1600025

	/*
	* __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
	diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c
	--- a/sys/vm/device_pager.c
	+++ b/sys/vm/device_pager.c
	@@ -66,6 +66,9 @@
	static void dev_pager_free_page(vm_object_t object, vm_page_t m);
	static int dev_pager_populate(vm_object_t object, vm_pindex_t pidx,
	int fault_type, vm_prot_t, vm_pindex_t first, vm_pindex_t last);
	+static vm_page_t dev_pager_populate_take_page(vm_object_t object,
	+ vm_pindex_t pidx);
	+static void dev_pager_populate_done(vm_object_t object);

	/* list of device pager objects */
	static struct pagerlst dev_pager_object_list;
	@@ -90,6 +93,8 @@
	.pgo_putpages = dev_pager_putpages,
	.pgo_haspage = dev_pager_haspage,
	.pgo_populate = dev_pager_populate,
	+ .pgo_populate_take_page = dev_pager_populate_take_page,
	+ .pgo_populate_done = dev_pager_populate_done,
	};

	static int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
	@@ -146,6 +151,12 @@
	return (NULL);
	KASSERT(tp == OBJT_MGTDEVICE \|\| ops->cdev_pg_populate == NULL,
	("populate on unmanaged device pager"));
	+ KASSERT((ops->cdev_pg_populate_take_page == NULL) ==
	+ (ops->cdev_pg_populate_done == NULL),
	+ ("incomplete populate handoff methods"));
	+ KASSERT(ops->cdev_pg_populate_take_page == NULL \|\|
	+ ops->cdev_pg_populate != NULL,
	+ ("populate handoff without populate method"));

	/*
	* Offset should be page aligned.
	@@ -232,6 +243,9 @@
	object = NULL;
	mtx_lock(&dev_pager_mtx);
	} else {
	+ if (ops->cdev_pg_mlock_skip != NULL &&
	+ ops->cdev_pg_mlock_skip(handle))
	+ vm_object_set_flag(object, OBJ_NOMLOCK);
	mtx_lock(&dev_pager_mtx);
	object->flags \|= OBJ_COLORED;
	object->pg_color = color;
	@@ -410,6 +424,26 @@
	fault_type, max_prot, first, last));
	}

	+static vm_page_t
	+dev_pager_populate_take_page(vm_object_t object, vm_pindex_t pidx)
	+{
	+
	+ VM_OBJECT_ASSERT_WLOCKED(object);
	+ if (object->un_pager.devp.ops->cdev_pg_populate_take_page == NULL)
	+ return (NULL);
	+ return (object->un_pager.devp.ops->cdev_pg_populate_take_page(
	+ object, pidx));
	+}
	+
	+static void
	+dev_pager_populate_done(vm_object_t object)
	+{
	+
	+ VM_OBJECT_ASSERT_WLOCKED(object);
	+ if (object->un_pager.devp.ops->cdev_pg_populate_done != NULL)
	+ object->un_pager.devp.ops->cdev_pg_populate_done(object);
	+}
	+
	static int
	old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot,
	vm_page_t *mres)
	diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
	--- a/sys/vm/vm_fault.c
	+++ b/sys/vm/vm_fault.c
	@@ -163,6 +163,7 @@
	FAULT_FAILURE, /* Return failure to user. */
	FAULT_CONTINUE, /* Continue faulting. */
	FAULT_RESTART, /* Restart fault. */
	+ FAULT_OOM, /* Retry after waiting for memory. */
	FAULT_OUT_OF_BOUNDS, /* Invalid address for pager. */
	FAULT_HARD, /* Performed I/O. */
	FAULT_SOFT, /* Found valid page. */
	@@ -484,8 +485,9 @@

	/*
	* Check each page to ensure that the pager is obeying the
	- * interface: the page must be installed in the object, fully
	- * valid, and exclusively busied.
	+ * interface: the page must be fully valid and exclusively busied.
	+ * Most populated pages are installed in the pager object, but a pager
	+ * may explicitly return a page owned by another object.
	*/
	MPASS(m != NULL);
	MPASS(vm_page_all_valid(m));
	@@ -498,26 +500,39 @@
	{
	struct pctrie_iter pages;
	vm_page_t m;
	+ vm_pindex_t pidx;

	VM_OBJECT_ASSERT_WLOCKED(object);
	MPASS(first <= last);
	- vm_page_iter_limit_init(&pages, object, last + 1);
	- VM_RADIX_FORALL_FROM(m, &pages, first) {
	+ m = vm_pager_populate_take_page(object, first);
	+ if (m == NULL) {
	+ vm_page_iter_limit_init(&pages, object, last + 1);
	+ VM_RADIX_FORALL_FROM(m, &pages, first) {
	+ vm_fault_populate_check_page(m);
	+ vm_page_deactivate(m);
	+ vm_page_xunbusy(m);
	+ }
	+ KASSERT(pages.index == last,
	+ ("%s: Object %p first %#jx last %#jx index %#jx",
	+ __func__, object, (uintmax_t)first, (uintmax_t)last,
	+ (uintmax_t)pages.index));
	+ return;
	+ }
	+ for (pidx = first;; pidx++) {
	vm_fault_populate_check_page(m);
	vm_page_deactivate(m);
	vm_page_xunbusy(m);
	+ if (pidx == last)
	+ break;
	+ m = vm_pager_populate_take_page(object, pidx + 1);
	}
	- KASSERT(pages.index == last,
	- ("%s: Object %p first %#jx last %#jx index %#jx",
	- __func__, object, (uintmax_t)first, (uintmax_t)last,
	- (uintmax_t)pages.index));
	}

	static enum fault_status
	vm_fault_populate(struct faultstate *fs)
	{
	vm_offset_t vaddr;
	- vm_page_t m;
	+ vm_page_t external_hold, m;
	vm_pindex_t map_first, map_last, pager_first, pager_last, pidx;
	int bdry_idx, i, npages, psind, rv;
	enum fault_status res;
	@@ -534,6 +549,7 @@
	vm_fault_unlock_vp(fs);

	res = FAULT_SUCCESS;
	+ external_hold = NULL;

	/*
	* Call the pager (driver) populate() method.
	@@ -558,6 +574,10 @@
	return (FAULT_RESTART);
	return (FAULT_CONTINUE);
	}
	+ if (rv == VM_PAGER_OUT_OF_BOUNDS)
	+ return (FAULT_OUT_OF_BOUNDS);
	+ if (rv == VM_PAGER_AGAIN)
	+ return (FAULT_OOM);
	if (rv != VM_PAGER_OK)
	return (FAULT_FAILURE); /* AKA SIGSEGV */

	@@ -574,10 +594,23 @@
	vm_fault_populate_cleanup(fs->first_object, pager_first,
	pager_last);
	} else {
	- m = vm_page_lookup(fs->first_object, pager_first);
	- if (m != fs->m)
	+ m = vm_pager_populate_take_page(fs->first_object,
	+ pager_first);
	+ if (m != NULL) {
	+ vm_fault_populate_check_page(m);
	+ vm_page_deactivate(m);
	vm_page_xunbusy(m);
	+ if (pager_first < pager_last)
	+ vm_fault_populate_cleanup(
	+ fs->first_object, pager_first + 1,
	+ pager_last);
	+ } else {
	+ m = vm_page_lookup(fs->first_object, pager_first);
	+ if (m != NULL && m != fs->m)
	+ vm_page_xunbusy(m);
	+ }
	}
	+ vm_pager_populate_done(fs->first_object);
	return (FAULT_RESTART);
	}

	@@ -590,6 +623,18 @@
	if (bdry_idx != 0) {
	KASSERT(PMAP_HAS_LARGEPAGES,
	("missing pmap support for large pages"));
	+ m = vm_pager_populate_take_page(fs->first_object,
	+ pager_first);
	+ if (m != NULL) {
	+ vm_fault_populate_check_page(m);
	+ vm_page_deactivate(m);
	+ vm_page_xunbusy(m);
	+ if (pager_first < pager_last)
	+ vm_fault_populate_cleanup(fs->first_object,
	+ pager_first + 1, pager_last);
	+ res = FAULT_FAILURE;
	+ goto out;
	+ }
	m = vm_page_lookup(fs->first_object, pager_first);
	vm_fault_populate_check_page(m);
	VM_OBJECT_WUNLOCK(fs->first_object);
	@@ -645,8 +690,44 @@
	for (pidx = pager_first; pidx <= pager_last; pidx += npages) {
	bool writeable;

	- m = vm_page_lookup(fs->first_object, pidx);
	vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset;
	+ m = vm_pager_populate_take_page(fs->first_object, pidx);
	+ if (m != NULL) {
	+ npages = 1;
	+ vm_fault_populate_check_page(m);
	+ if (fs->wired \|\|
	+ (fs->fault_flags & VM_FAULT_WIRE) != 0) {
	+ vm_page_deactivate(m);
	+ vm_page_xunbusy(m);
	+ if (pidx < pager_last)
	+ vm_fault_populate_cleanup(fs->first_object,
	+ pidx + 1, pager_last);
	+ res = FAULT_FAILURE;
	+ goto out;
	+ }
	+ vm_fault_dirty(fs, m);
	+ VM_OBJECT_WUNLOCK(fs->first_object);
	+ rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot,
	+ fs->fault_type, 0);
	+ VM_OBJECT_WLOCK(fs->first_object);
	+ if (rv != KERN_SUCCESS) {
	+ vm_page_deactivate(m);
	+ vm_page_xunbusy(m);
	+ if (pidx < pager_last)
	+ vm_fault_populate_cleanup(fs->first_object,
	+ pidx + 1, pager_last);
	+ res = FAULT_FAILURE;
	+ goto out;
	+ }
	+ vm_page_activate(m);
	+ if (fs->m_hold != NULL && pidx == fs->first_pindex) {
	+ vm_page_wire(m);
	+ external_hold = m;
	+ }
	+ vm_page_xunbusy(m);
	+ continue;
	+ }
	+ m = vm_page_lookup(fs->first_object, pidx);
	KASSERT(m != NULL && m->pindex == pidx,
	("%s: pindex mismatch", __func__));
	psind = m->psind;
	@@ -713,6 +794,14 @@
	}
	}
	out:
	+ /* Publish the held page only after the entire handoff succeeds. */
	+ if (external_hold != NULL) {
	+ if (res == FAULT_SUCCESS)
	+ *fs->m_hold = external_hold;
	+ else
	+ vm_page_unwire(external_hold, PQ_INACTIVE);
	+ }
	+ vm_pager_populate_done(fs->first_object);
	curthread->td_ru.ru_majflt++;
	return (res);
	}
	@@ -1325,9 +1414,17 @@
	switch (res) {
	case FAULT_SUCCESS:
	case FAULT_FAILURE:
	+ case FAULT_OUT_OF_BOUNDS:
	case FAULT_RESTART:
	vm_fault_unlock_and_deallocate(fs);
	return (res);
	+ case FAULT_OOM:
	+ dset = fs->object->domain.dr_policy;
	+ if (dset == NULL)
	+ dset = curthread->td_domain.dr_policy;
	+ if (vm_fault_allocate_oom(fs))
	+ vm_waitpfault(dset, vm_pfault_oom_wait * hz);
	+ return (FAULT_RESTART);
	case FAULT_CONTINUE:
	pctrie_iter_reset(pages);
	/*
	diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
	--- a/sys/vm/vm_map.c
	+++ b/sys/vm/vm_map.c
	@@ -3624,6 +3624,17 @@
	rv = KERN_INVALID_ADDRESS;
	goto done;
	}
	+ } else if (user_wire &&
	+ (entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
	+ entry->object.vm_object != NULL &&
	+ (entry->object.vm_object->flags & OBJ_NOMLOCK) != 0) {
	+ /*
	+ * The pager owns residency for this special device mapping.
	+ * Retain range/hole checks and transition cleanup, but do not
	+ * claim to pin its pages or acquire a user wire reference.
	+ * Kernel wiring must still satisfy the ordinary contract.
	+ */
	+ entry->eflags \|= MAP_ENTRY_WIRE_SKIPPED;
	} else if (entry->wired_count == 0) {
	entry->wired_count++;

	diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
	--- a/sys/vm/vm_object.h
	+++ b/sys/vm/vm_object.h
	@@ -201,6 +201,7 @@
	#define OBJ_PAGERPRIV2 0x00008000 /* Pager private */
	#define OBJ_SYSVSHM 0x00010000 /* SysV SHM */
	#define OBJ_POSIXSHM 0x00020000 /* Posix SHM */
	+#define OBJ_NOMLOCK 0x00040000 /* (c) skip userspace memory locking */

	/*
	* Helpers to perform conversion between vm_object page indexes and offsets.
	diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h
	--- a/sys/vm/vm_pager.h
	+++ b/sys/vm/vm_pager.h
	@@ -58,6 +58,8 @@
	typedef boolean_t pgo_haspage_t(vm_object_t, vm_pindex_t, int , int );
	typedef int pgo_populate_t(vm_object_t, vm_pindex_t, int, vm_prot_t,
	vm_pindex_t , vm_pindex_t );
	+typedef vm_page_t pgo_populate_take_page_t(vm_object_t, vm_pindex_t);
	+typedef void pgo_populate_done_t(vm_object_t);
	typedef void pgo_pageunswapped_t(vm_page_t);
	typedef void pgo_writecount_t(vm_object_t, vm_offset_t, vm_offset_t);
	typedef void pgo_set_writeable_dirty_t(vm_object_t);
	@@ -90,6 +92,8 @@
	pgo_page_inserted_t *pgo_page_inserted;
	pgo_page_removed_t *pgo_page_removed;
	pgo_can_alloc_page_t *pgo_can_alloc_page;
	+ pgo_populate_take_page_t *pgo_populate_take_page;
	+ pgo_populate_done_t *pgo_populate_done;
	};

	extern const struct pagerops defaultpagerops;
	@@ -109,6 +113,7 @@
	* PEND operations was initiated but not completed
	* ERROR error while accessing data that is in range and exists
	* AGAIN temporary resource shortage prevented operation from happening
	+ * OUT_OF_BOUNDS pager-specific address is invalid and should raise SIGBUS
	*/
	#define VM_PAGER_OK 0
	#define VM_PAGER_BAD 1
	@@ -116,6 +121,7 @@
	#define VM_PAGER_PEND 3
	#define VM_PAGER_ERROR 4
	#define VM_PAGER_AGAIN 5
	+#define VM_PAGER_OUT_OF_BOUNDS 6

	#define VM_PAGER_PUT_SYNC 0x0001
	#define VM_PAGER_PUT_INVAL 0x0002
	@@ -186,6 +192,33 @@
	fault_type, max_prot, first, last));
	}

	+/*
	+ * Take an xbusy page populated by the pager without inserting it into the
	+ * pager object. The caller assumes responsibility for releasing the busy
	+ * state. A populate operation must not mix such pages with pages installed
	+ * in the pager object. Most pagers populate only object-resident pages and
	+ * leave this method unset.
	+ */
	+static __inline vm_page_t
	+vm_pager_populate_take_page(vm_object_t object, vm_pindex_t pidx)
	+{
	+ pgo_populate_take_page_t *method;
	+
	+ method = pagertab[object->type]->pgo_populate_take_page;
	+ return (method == NULL ? NULL : method(object, pidx));
	+}
	+
	+/* Notify the pager that all pages returned by populate() were consumed. */
	+static __inline void
	+vm_pager_populate_done(vm_object_t object)
	+{
	+ pgo_populate_done_t *method;
	+
	+ method = pagertab[object->type]->pgo_populate_done;
	+ if (method != NULL)
	+ method(object);
	+}
	+
	/*
	* vm_pager_page_unswapped
	*
	@@ -294,6 +327,17 @@
	vm_ooffset_t foff, struct ucred cred, u_short color);
	void (cdev_pg_dtor)(void handle);
	void (cdev_pg_path)(void handle, char *path, size_t len);
	+ vm_page_t (*cdev_pg_populate_take_page)(vm_object_t vm_obj,
	+ vm_pindex_t pidx);
	+ void (*cdev_pg_populate_done)(vm_object_t vm_obj);
	+ /*
	+ * Optional special-mapping policy, queried after successful construction
	+ * and before publishing the object. True exempts this object's mappings
	+ * from userspace memory locking, but does not permit kernel wiring to
	+ * succeed without actually wiring pages. The result is immutable for
	+ * the object's lifetime; NULL preserves ordinary memory-locking policy.
	+ */
	+ bool (cdev_pg_mlock_skip)(void handle);
	};

	vm_object_t cdev_pager_allocate(void *handle, enum obj_type tp,
	diff --git a/sys/vm/vm_pager.c b/sys/vm/vm_pager.c
	--- a/sys/vm/vm_pager.c
	+++ b/sys/vm/vm_pager.c
	@@ -401,6 +401,10 @@
	int res;

	mtx_lock(&pagertab_lock);
	+ MPASS((ops->pgo_populate_take_page == NULL) ==
	+ (ops->pgo_populate_done == NULL));
	+ MPASS(ops->pgo_populate_take_page == NULL \|\|
	+ ops->pgo_populate != NULL);
	MPASS(base_type == -1 \|\|
	(base_type >= OBJT_SWAP && base_type < nitems(pagertab)));
	for (res = OBJT_FIRST_DYN; res < nitems(pagertab); res++) {
	@@ -423,7 +427,6 @@
	FIX(getpages_async);
	FIX(putpages);
	FIX(haspage);
	- FIX(populate);
	FIX(pageunswapped);
	FIX(update_writecount);
	FIX(release_writecount);
	@@ -435,6 +438,17 @@
	FIX(page_removed);
	FIX(can_alloc_page);
	#undef FIX
	+ /* The populate handoff methods form one indivisible contract. */
	+ if (ops->pgo_populate == NULL) {
	+ MPASS(ops->pgo_populate_take_page == NULL);
	+ MPASS(ops->pgo_populate_done == NULL);
	+ ops->pgo_populate =
	+ pagertab[base_type]->pgo_populate;
	+ ops->pgo_populate_take_page =
	+ pagertab[base_type]->pgo_populate_take_page;
	+ ops->pgo_populate_done =
	+ pagertab[base_type]->pgo_populate_done;
	+ }
	}
	pagertab[res] = ops; /* XXXKIB should be rel, but acq is too much */
	mtx_unlock(&pagertab_lock);

File Metadata

Mime Type: text/plain
Expires: Wed, Sep 9, 7:50 AM (12 h, 17 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 38524336
Default Alt Text: D59481.diff (49 KB)

D59481.diffNo OneTemporaryActions

D59481.diffView Options

File Metadata

Event Timeline

D59481.diff
No OneTemporary
Actions

D59481.diff
View Options