Page MenuHomeFreeBSD
Files F170920053

D59481.diff
No OneTemporary
Actions

D59481.diff
View Options

diff --git a/lib/libsys/mlock.2 b/lib/libsys/mlock.2
--- a/lib/libsys/mlock.2
+++ b/lib/libsys/mlock.2
@@ -25,7 +25,7 @@
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
-.Dd May 13, 2019
+.Dd September 6, 2026
.Dt MLOCK 2
.Os
.Sh NAME
@@ -84,6 +84,17 @@
Locked mappings are not inherited by the child process after a
.Xr fork 2 .
.Pp
+Special device mappings explicitly exempted by their pager are skipped by
+.Fn mlock
+and acquire no user wire reference.
+This includes LinuxKPI managed-device mappings marked as IO or PFN mappings.
+The residency and address-translation guarantees above do not apply to these
+mappings; their driver controls their backing lifetime.
+Ordinary mappings retain their existing locking behavior.
+Existing privilege, address-range, protection and resource-limit checks still
+apply, including conservative accounting of the requested virtual range.
+Thus a request containing an exempt mapping can still fail those checks.
+.Pp
Since physical memory is a potentially scarce resource, processes are
limited in how much they can lock down.
The amount of memory that a single process can
@@ -108,7 +119,8 @@
.Sh RETURN VALUES
.Rv -std
.Pp
-If the call succeeds, all pages in the range become locked (unlocked);
+If the call succeeds, all non-exempt pages in the range become locked
+(unlocked);
otherwise the locked status of all pages in the range remains unchanged.
.Sh ERRORS
The
@@ -172,5 +184,3 @@
Hence two distinct locked mappings of the same physical page counts as
2 pages aginst the system limit, and also against the per-process limit
if both mappings belong to the same physical map.
-.Pp
-The per-process resource limit is not currently supported.
diff --git a/lib/libsys/mlockall.2 b/lib/libsys/mlockall.2
--- a/lib/libsys/mlockall.2
+++ b/lib/libsys/mlockall.2
@@ -28,7 +28,7 @@
.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
-.Dd May 13, 2019
+.Dd September 6, 2026
.Dt MLOCKALL 2
.Os
.Sh NAME
@@ -62,6 +62,19 @@
cause resource limits to be exceeded.
.El
.Pp
+Special device mappings explicitly exempted by their pager are skipped for
+both
+.Dv MCL_CURRENT
+and
+.Dv MCL_FUTURE .
+This includes LinuxKPI managed-device mappings marked as IO or PFN mappings.
+These mappings acquire no user wire reference and are not covered by the
+residency guarantee; their driver controls their backing lifetime.
+Existing resource-limit checks and conservative accounting of the requested
+virtual range still apply.
+See
+.Xr mlock 2 .
+.Pp
Since physical memory is a potentially scarce resource, processes are
limited in how much they can lock down.
A single process can lock the minimum of a system-wide
@@ -91,7 +104,8 @@
call will not be locked.
.Sh RETURN VALUES
A return value of 0 indicates that the call
-succeeded and all pages in the range have either been locked or unlocked.
+succeeded and all non-exempt pages in the range have either been locked or
+unlocked.
A return value of \-1 indicates an error occurred and the locked
status of all pages in the range remains unchanged.
In this case, the global location
diff --git a/sys/compat/linuxkpi/common/include/linux/mm.h b/sys/compat/linuxkpi/common/include/linux/mm.h
--- a/sys/compat/linuxkpi/common/include/linux/mm.h
+++ b/sys/compat/linuxkpi/common/include/linux/mm.h
@@ -125,6 +125,7 @@
int *vm_pfn_pcount;
vm_object_t vm_obj;
vm_map_t vm_cached_map;
+ void *vm_pfn_state;
TAILQ_ENTRY(vm_area_struct) vm_entry;
};
@@ -140,6 +141,25 @@
struct vm_area_struct *vma;
};
+int lkpi_vma_pfn_init(struct vm_area_struct *vma);
+int lkpi_vma_pfn_begin(struct vm_area_struct *vma, uint64_t *invalidation_seq);
+bool lkpi_vma_pfn_unchanged(struct vm_area_struct *vma,
+ uint64_t invalidation_seq);
+bool lkpi_vma_pfn_handoff_valid(struct vm_area_struct *vma);
+bool lkpi_vma_pfn_lock(struct vm_area_struct *vma);
+void lkpi_vma_pfn_end(struct vm_area_struct *vma);
+vm_page_t lkpi_vma_pfn_take_page(struct vm_area_struct *vma,
+ vm_object_t object, vm_pindex_t pindex);
+void lkpi_vma_pfn_abort(struct vm_area_struct *vma, vm_object_t object);
+void lkpi_vma_pfn_done(struct vm_area_struct *vma, vm_object_t object);
+bool lkpi_vma_pfn_invalidate_begin(struct vm_area_struct *vma);
+bool lkpi_vma_pfn_unmap_begin(struct vm_area_struct *vma);
+void lkpi_vma_pfn_unmap(struct vm_area_struct *vma);
+void lkpi_vma_pfn_unmap_end(struct vm_area_struct *vma);
+void lkpi_vma_pfn_invalidate_end(struct vm_area_struct *vma);
+void lkpi_vma_pfn_fini(struct vm_area_struct *vma);
+void linux_cdev_pager_free_pages(vm_object_t object);
+
struct vm_operations_struct {
void (*open) (struct vm_area_struct *);
void (*close) (struct vm_area_struct *);
diff --git a/sys/compat/linuxkpi/common/src/linux_compat.c b/sys/compat/linuxkpi/common/src/linux_compat.c
--- a/sys/compat/linuxkpi/common/src/linux_compat.c
+++ b/sys/compat/linuxkpi/common/src/linux_compat.c
@@ -507,6 +507,9 @@
vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last)
{
struct vm_area_struct *vmap;
+ bool pfn_started;
+ bool retry;
+ uint64_t invalidation_seq;
int err;
/* get VM area structure */
@@ -514,14 +517,26 @@
MPASS(vmap != NULL);
MPASS(vmap->vm_private_data == vm_obj->handle);
+retry_fault:
VM_OBJECT_WUNLOCK(vm_obj);
-
linux_set_current(curthread);
-
down_write(&vmap->vm_mm->mmap_sem);
- if (unlikely(vmap->vm_ops == NULL)) {
- err = VM_FAULT_SIGBUS;
+ pfn_started = false;
+ retry = false;
+ err = lkpi_vma_pfn_begin(vmap, &invalidation_seq);
+ if (err != 0) {
+ err = err == ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS;
} else {
+ pfn_started = true;
+ /* The previous populate may still be completing its handoff. */
+ atomic_store_rel_ptr((volatile uintptr_t *)&vmap->vm_obj,
+ (uintptr_t)vm_obj);
+ vmap->vm_pfn_count = 0;
+ vmap->vm_pfn_pcount = &vmap->vm_pfn_count;
+ }
+ if (err == 0 && unlikely(vmap->vm_ops == NULL)) {
+ err = VM_FAULT_SIGBUS;
+ } else if (err == 0) {
struct vm_fault vmf;
/* fill out VM fault structure */
@@ -531,33 +546,39 @@
vmf.page = NULL;
vmf.vma = vmap;
- vmap->vm_pfn_count = 0;
- vmap->vm_pfn_pcount = &vmap->vm_pfn_count;
- vmap->vm_obj = vm_obj;
-
err = vmap->vm_ops->fault(&vmf);
- while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE) {
+ while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE &&
+ lkpi_vma_pfn_unchanged(vmap, invalidation_seq)) {
kern_yield(PRI_USER);
err = vmap->vm_ops->fault(&vmf);
}
+ retry = !lkpi_vma_pfn_unchanged(vmap, invalidation_seq);
}
/* translate return code */
- switch (err) {
+ switch (retry ? VM_FAULT_RETRY : err) {
+ case VM_FAULT_RETRY:
+ err = VM_PAGER_ERROR;
+ break;
case VM_FAULT_OOM:
err = VM_PAGER_AGAIN;
break;
case VM_FAULT_SIGBUS:
- err = VM_PAGER_BAD;
+ err = VM_PAGER_OUT_OF_BOUNDS;
break;
case VM_FAULT_NOPAGE:
/*
- * By contract the fault handler will return having
- * busied all the pages itself. If pidx is already
- * found in the object, it will simply xbusy the first
- * page and return with vm_pfn_count set to 1.
+ * By contract, the fault handler returns with every
+ * populated page exclusively busied. A page is either
+ * installed in the pager object or handed to the VM by
+ * cdev_pg_populate_take_page(). The two representations
+ * cannot be mixed in one populated range.
*/
+ if (!lkpi_vma_pfn_handoff_valid(vmap)) {
+ err = VM_PAGER_ERROR;
+ break;
+ }
*first = vmap->vm_pfn_first;
*last = *first + vmap->vm_pfn_count - 1;
err = VM_PAGER_OK;
@@ -568,9 +589,39 @@
}
up_write(&vmap->vm_mm->mmap_sem);
VM_OBJECT_WLOCK(vm_obj);
+ if (err != VM_PAGER_OK && pfn_started) {
+ if (vmap->vm_pfn_count != 0)
+ lkpi_vma_pfn_abort(vmap, vm_obj);
+ lkpi_vma_pfn_end(vmap);
+ }
+ if (retry)
+ goto retry_fault;
return (err);
}
+static vm_page_t
+linux_cdev_pager_populate_take_page(vm_object_t vm_obj, vm_pindex_t pidx)
+{
+ struct vm_area_struct *vmap;
+
+ vmap = linux_cdev_handle_find(vm_obj->handle);
+ MPASS(vmap != NULL);
+ MPASS(vmap->vm_private_data == vm_obj->handle);
+ return (lkpi_vma_pfn_take_page(vmap, vm_obj, pidx));
+}
+
+static void
+linux_cdev_pager_populate_done(vm_object_t vm_obj)
+{
+ struct vm_area_struct *vmap;
+
+ vmap = linux_cdev_handle_find(vm_obj->handle);
+ MPASS(vmap != NULL);
+ MPASS(vmap->vm_private_data == vm_obj->handle);
+ lkpi_vma_pfn_done(vmap, vm_obj);
+ lkpi_vma_pfn_end(vmap);
+}
+
static struct rwlock linux_vma_lock;
static TAILQ_HEAD(, vm_area_struct) linux_vma_head =
TAILQ_HEAD_INITIALIZER(linux_vma_head);
@@ -620,6 +671,22 @@
return (0);
}
+static int
+linux_cdev_mgtdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
+ vm_ooffset_t foff, struct ucred *cred, u_short *color)
+{
+ struct vm_area_struct *vmap;
+ int error;
+
+ vmap = linux_cdev_handle_find(handle);
+ MPASS(vmap != NULL);
+ error = lkpi_vma_pfn_init(vmap);
+ if (error != 0)
+ return (error);
+ *color = 0;
+ return (0);
+}
+
static void
linux_cdev_pager_dtor(void *handle)
{
@@ -636,6 +703,7 @@
linux_cdev_handle_remove(vmap);
down_write(&vmap->vm_mm->mmap_sem);
+ lkpi_vma_pfn_fini(vmap);
vm_ops = vmap->vm_ops;
if (likely(vm_ops != NULL))
vm_ops->close(vmap);
@@ -644,11 +712,25 @@
linux_cdev_handle_free(vmap);
}
+static bool
+linux_cdev_pager_mlock_skip(void *handle)
+{
+ struct vm_area_struct *vmap;
+
+ vmap = linux_cdev_handle_find(handle);
+ MPASS(vmap != NULL);
+ /* Linux does not mlock special IO/PFN mappings. */
+ return ((vmap->vm_flags & (VM_IO | VM_PFNMAP)) != 0);
+}
+
static struct cdev_pager_ops linux_cdev_pager_ops[2] = {
{
/* OBJT_MGTDEVICE */
.cdev_pg_populate = linux_cdev_pager_populate,
- .cdev_pg_ctor = linux_cdev_pager_ctor,
+ .cdev_pg_populate_take_page = linux_cdev_pager_populate_take_page,
+ .cdev_pg_populate_done = linux_cdev_pager_populate_done,
+ .cdev_pg_mlock_skip = linux_cdev_pager_mlock_skip,
+ .cdev_pg_ctor = linux_cdev_mgtdev_pager_ctor,
.cdev_pg_dtor = linux_cdev_pager_dtor
},
{
@@ -659,6 +741,34 @@
},
};
+void
+linux_cdev_pager_free_pages(vm_object_t object)
+{
+ struct vm_area_struct *vmap;
+ bool invalidating;
+
+ vmap = linux_cdev_handle_find(object->handle);
+ if (vmap == NULL) {
+ cdev_mgtdev_pager_free_pages(object);
+ return;
+ }
+ invalidating = lkpi_vma_pfn_invalidate_begin(vmap);
+ if (!invalidating) {
+ cdev_mgtdev_pager_free_pages(object);
+ return;
+ }
+ /*
+ * Mark invalidation before removing either representation. A fault
+ * which has entered the driver but not yet selected a page will reject
+ * its result. A page selected earlier remains xbusy until
+ * vm_fault_populate() installs its PTE, so either removal pass waits and
+ * revokes that PTE before invalidation completes.
+ */
+ cdev_mgtdev_pager_free_pages(object);
+ lkpi_vma_pfn_unmap(vmap);
+ lkpi_vma_pfn_invalidate_end(vmap);
+}
+
int
zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
unsigned long size)
@@ -666,16 +776,49 @@
struct pctrie_iter pages;
vm_object_t obj;
vm_page_t m;
+ vm_pindex_t first, last;
+ bool unmapping;
+ bool pfn_locked;
+ int error;
- obj = vma->vm_obj;
- if (obj == NULL || (obj->flags & OBJ_UNMANAGED) != 0)
- return (-ENOTSUP);
- VM_OBJECT_RLOCK(obj);
- vm_page_iter_limit_init(&pages, obj, OFF_TO_IDX(address + size));
- VM_RADIX_FOREACH_FROM(m, &pages, OFF_TO_IDX(address))
+ pfn_locked = lkpi_vma_pfn_lock(vma);
+ obj = (void *)atomic_load_acq_ptr(
+ (volatile uintptr_t *)&vma->vm_obj);
+ if (obj == NULL || (obj->flags & OBJ_UNMANAGED) != 0) {
+ error = -ENOTSUP;
+ goto out;
+ }
+ if (size == 0) {
+ error = 0;
+ goto out;
+ }
+ if (offset_in_page(address) != 0 || offset_in_page(size) != 0 ||
+ address < vma->vm_start || address >= vma->vm_end ||
+ size > vma->vm_end - address) {
+ error = -EINVAL;
+ goto out;
+ }
+ first = OFF_TO_IDX(address - vma->vm_start);
+ last = OFF_TO_IDX(address + size - vma->vm_start);
+ VM_OBJECT_WLOCK(obj);
+ if (vma->vm_pfn_count != 0)
+ lkpi_vma_pfn_abort(vma, obj);
+ VM_OBJECT_WUNLOCK(obj);
+ unmapping = lkpi_vma_pfn_unmap_begin(vma);
+ if (unmapping)
+ lkpi_vma_pfn_unmap(vma);
+ VM_OBJECT_WLOCK(obj);
+ vm_page_iter_limit_init(&pages, obj, last);
+ VM_RADIX_FOREACH_FROM(m, &pages, first)
pmap_remove_all(m);
- VM_OBJECT_RUNLOCK(obj);
- return (0);
+ VM_OBJECT_WUNLOCK(obj);
+ if (unmapping)
+ lkpi_vma_pfn_unmap_end(vma);
+ error = 0;
+out:
+ if (pfn_locked)
+ lkpi_vma_pfn_end(vma);
+ return (error);
}
void
diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c
--- a/sys/compat/linuxkpi/common/src/linux_page.c
+++ b/sys/compat/linuxkpi/common/src/linux_page.c
@@ -31,9 +31,11 @@
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/sysctl.h>
+#include <sys/condvar.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/rwlock.h>
+#include <sys/sx.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/memrange.h>
@@ -65,6 +67,7 @@
#include <linux/idr.h>
#include <linux/io.h>
#include <linux/io-mapping.h>
+#include <linux/slab.h>
#ifdef __i386__
DEFINE_IDR(mtrr_idr);
@@ -505,22 +508,552 @@
kfree(vmmap);
}
+struct lkpi_vma_pfn_object {
+ TAILQ_ENTRY(lkpi_vma_pfn_object) link;
+ vm_object_t object;
+ vm_pindex_t first;
+ vm_pindex_t last;
+};
+
+#define LKPI_VMA_PFN_CHUNK_PAGES 64
+
+struct lkpi_vma_pfn_chunk {
+ TAILQ_ENTRY(lkpi_vma_pfn_chunk) link;
+ vm_pindex_t first;
+ unsigned int count;
+ unsigned int remaining;
+ vm_page_t pages[LKPI_VMA_PFN_CHUNK_PAGES];
+};
+
+struct lkpi_vma_pfn_state {
+ TAILQ_HEAD(, lkpi_vma_pfn_object) objects;
+ TAILQ_HEAD(, lkpi_vma_pfn_chunk) page_chunks;
+ struct mtx objects_lock;
+ struct cv invalidation_cv;
+ struct sx populate_lock;
+ struct sx unmap_lock;
+ struct lkpi_vma_pfn_chunk *last_chunk;
+ vm_pindex_t npages;
+ vm_pindex_t pending;
+ uint64_t invalidation_seq;
+};
+
+static struct lkpi_vma_pfn_state *
+lkpi_vma_pfn_get_state(struct vm_area_struct *vma)
+{
+
+ return ((void *)atomic_load_acq_ptr(
+ (volatile uintptr_t *)&vma->vm_pfn_state));
+}
+
+int
+lkpi_vma_pfn_init(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_state *state;
+ vm_pindex_t npages;
+
+ MPASS(lkpi_vma_pfn_get_state(vma) == NULL);
+ npages = vma_pages(vma);
+ if (npages == 0)
+ return (EINVAL);
+ state = kzalloc(sizeof(*state), GFP_KERNEL);
+ if (state == NULL)
+ return (ENOMEM);
+ TAILQ_INIT(&state->objects);
+ TAILQ_INIT(&state->page_chunks);
+ mtx_init(&state->objects_lock, "lkpi pfn objects", NULL, MTX_DEF);
+ cv_init(&state->invalidation_cv, "lkpi pfn invalidate");
+ sx_init(&state->populate_lock, "lkpi pfn populate");
+ sx_init(&state->unmap_lock, "lkpi pfn unmap");
+ state->npages = npages;
+ atomic_store_rel_ptr((volatile uintptr_t *)&vma->vm_pfn_state,
+ (uintptr_t)state);
+ return (0);
+}
+
+int
+lkpi_vma_pfn_begin(struct vm_area_struct *vma, uint64_t *invalidation_seq)
+{
+ struct lkpi_vma_pfn_state *state;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ if (state == NULL)
+ return (EINVAL);
+ sx_xlock(&state->populate_lock);
+ if (state->pending != 0 || !TAILQ_EMPTY(&state->page_chunks)) {
+ sx_xunlock(&state->populate_lock);
+ return (EBUSY);
+ }
+ mtx_lock(&state->objects_lock);
+ while ((state->invalidation_seq & 1) != 0)
+ cv_wait(&state->invalidation_cv, &state->objects_lock);
+ *invalidation_seq = state->invalidation_seq;
+ mtx_unlock(&state->objects_lock);
+ return (0);
+}
+
+bool
+lkpi_vma_pfn_unchanged(struct vm_area_struct *vma,
+ uint64_t invalidation_seq)
+{
+ struct lkpi_vma_pfn_state *state;
+ bool unchanged;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ if (state == NULL)
+ return (false);
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ mtx_lock(&state->objects_lock);
+ unchanged = state->invalidation_seq == invalidation_seq;
+ mtx_unlock(&state->objects_lock);
+ return (unchanged);
+}
+
+bool
+lkpi_vma_pfn_handoff_valid(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_state *state;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ MPASS(state != NULL);
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ MPASS(vma->vm_pfn_count > 0);
+ return (state->pending == 0 ||
+ state->pending == (vm_pindex_t)vma->vm_pfn_count);
+}
+
+bool
+lkpi_vma_pfn_lock(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_state *state;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ if (state == NULL || sx_xlocked(&state->populate_lock))
+ return (false);
+ sx_xlock(&state->populate_lock);
+ return (true);
+}
+
+void
+lkpi_vma_pfn_end(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_state *state;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ MPASS(state != NULL);
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ MPASS(state->pending == 0);
+ sx_xunlock(&state->populate_lock);
+}
+
+static bool
+lkpi_vma_pfn_object_is_tracked_locked(struct lkpi_vma_pfn_state *state,
+ vm_object_t object)
+{
+ struct lkpi_vma_pfn_object *entry;
+
+ mtx_assert(&state->objects_lock, MA_OWNED);
+ TAILQ_FOREACH(entry, &state->objects, link) {
+ if (entry->object == object)
+ return (true);
+ }
+ return (false);
+}
+
+static bool
+lkpi_vma_pfn_object_is_tracked(struct lkpi_vma_pfn_state *state,
+ vm_object_t object)
+{
+ bool tracked;
+
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ mtx_lock(&state->objects_lock);
+ tracked = lkpi_vma_pfn_object_is_tracked_locked(state, object);
+ mtx_unlock(&state->objects_lock);
+ return (tracked);
+}
+
+static void
+lkpi_vma_pfn_drop_object_ref(vm_object_t locked_object,
+ vm_object_t referenced_object)
+{
+
+ VM_OBJECT_ASSERT_WLOCKED(locked_object);
+ VM_OBJECT_WUNLOCK(locked_object);
+ vm_object_deallocate(referenced_object);
+ VM_OBJECT_WLOCK(locked_object);
+}
+
+static int
+lkpi_vma_pfn_track_object(struct lkpi_vma_pfn_state *state,
+ vm_object_t object, vm_pindex_t pindex, bool have_reference,
+ bool *reference_consumed)
+{
+ struct lkpi_vma_pfn_object *entry;
+
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ *reference_consumed = false;
+ mtx_lock(&state->objects_lock);
+ TAILQ_FOREACH(entry, &state->objects, link) {
+ if (entry->object == object) {
+ entry->first = MIN(entry->first, pindex);
+ entry->last = MAX(entry->last, pindex);
+ mtx_unlock(&state->objects_lock);
+ return (0);
+ }
+ }
+ mtx_unlock(&state->objects_lock);
+ if (!have_reference)
+ return (ESTALE);
+ entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
+ if (entry == NULL) {
+ return (ENOMEM);
+ }
+ entry->object = object;
+ entry->first = pindex;
+ entry->last = pindex;
+ mtx_lock(&state->objects_lock);
+ KASSERT(!lkpi_vma_pfn_object_is_tracked_locked(state, object),
+ ("%s: duplicate object %p", __func__, object));
+ TAILQ_INSERT_TAIL(&state->objects, entry, link);
+ mtx_unlock(&state->objects_lock);
+ *reference_consumed = true;
+ return (0);
+}
+
+static int
+lkpi_vma_pfn_store_page(struct lkpi_vma_pfn_state *state,
+ vm_pindex_t pindex, vm_page_t page)
+{
+ struct lkpi_vma_pfn_chunk *chunk;
+
+ MPASS(page != NULL);
+ MPASS(vm_page_xbusied(page));
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ MPASS(pindex < state->npages);
+ MPASS(state->pending < state->npages);
+ chunk = state->last_chunk;
+ if (chunk == NULL || chunk->count == nitems(chunk->pages) ||
+ pindex != chunk->first + chunk->count) {
+ chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
+ if (chunk == NULL)
+ return (ENOMEM);
+ chunk->first = pindex;
+ TAILQ_INSERT_TAIL(&state->page_chunks, chunk, link);
+ state->last_chunk = chunk;
+ }
+ MPASS(chunk->pages[chunk->count] == NULL);
+ chunk->pages[chunk->count++] = page;
+ chunk->remaining++;
+ state->pending++;
+ return (0);
+}
+
+static bool
+lkpi_vma_pfn_page_is_selected(struct vm_area_struct *vma, vm_page_t page)
+{
+ struct lkpi_vma_pfn_chunk *chunk;
+ struct lkpi_vma_pfn_state *state;
+ unsigned int slot;
+
+ VM_OBJECT_ASSERT_WLOCKED(vma->vm_obj);
+ state = lkpi_vma_pfn_get_state(vma);
+ MPASS(state != NULL);
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ TAILQ_FOREACH(chunk, &state->page_chunks, link) {
+ for (slot = 0; slot < chunk->count; slot++) {
+ if (chunk->pages[slot] == page)
+ return (true);
+ }
+ }
+ return (atomic_load_ptr(&page->object) == vma->vm_obj &&
+ page->pindex >= vma->vm_pfn_first &&
+ page->pindex - vma->vm_pfn_first <
+ (vm_pindex_t)vma->vm_pfn_count);
+}
+
+vm_page_t
+lkpi_vma_pfn_take_page(struct vm_area_struct *vma, vm_object_t object,
+ vm_pindex_t pindex)
+{
+ struct lkpi_vma_pfn_chunk *chunk;
+ struct lkpi_vma_pfn_state *state;
+ vm_page_t page;
+ unsigned int slot;
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ state = lkpi_vma_pfn_get_state(vma);
+ if (state == NULL || pindex >= state->npages)
+ return (NULL);
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ TAILQ_FOREACH(chunk, &state->page_chunks, link) {
+ if (pindex < chunk->first)
+ break;
+ if (pindex - chunk->first >= chunk->count)
+ continue;
+ slot = pindex - chunk->first;
+ page = chunk->pages[slot];
+ if (page == NULL)
+ return (NULL);
+ chunk->pages[slot] = NULL;
+ MPASS(chunk->remaining > 0 && state->pending > 0);
+ chunk->remaining--;
+ state->pending--;
+ if (chunk->remaining == 0) {
+ TAILQ_REMOVE(&state->page_chunks, chunk, link);
+ if (state->last_chunk == chunk)
+ state->last_chunk = NULL;
+ kfree(chunk);
+ }
+ return (page);
+ }
+ return (NULL);
+}
+
+void
+lkpi_vma_pfn_abort(struct vm_area_struct *vma, vm_object_t object)
+{
+ struct lkpi_vma_pfn_state *state;
+ vm_page_t page;
+ vm_pindex_t count, pindex;
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ state = lkpi_vma_pfn_get_state(vma);
+ if (state != NULL)
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ count = vma->vm_pfn_count;
+ for (pindex = vma->vm_pfn_first; count != 0;
+ count--, pindex++) {
+ page = lkpi_vma_pfn_take_page(vma, object, pindex);
+ if (page == NULL)
+ page = vm_page_lookup(object, pindex);
+ if (page != NULL) {
+ vm_page_deactivate(page);
+ vm_page_xunbusy(page);
+ }
+ }
+ vma->vm_pfn_count = 0;
+ KASSERT(state == NULL || state->pending == 0,
+ ("%s: %ju pages remain pending", __func__,
+ state == NULL ? 0 : (uintmax_t)state->pending));
+}
+
+void
+lkpi_vma_pfn_done(struct vm_area_struct *vma, vm_object_t object)
+{
+ struct lkpi_vma_pfn_state *state;
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ state = lkpi_vma_pfn_get_state(vma);
+ MPASS(state != NULL);
+ sx_assert(&state->populate_lock, SA_XLOCKED);
+ MPASS(state->pending == 0 &&
+ TAILQ_EMPTY(&state->page_chunks));
+ vma->vm_pfn_count = 0;
+}
+
+bool
+lkpi_vma_pfn_unmap_begin(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_state *state;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ if (state == NULL)
+ return (false);
+ sx_xlock(&state->unmap_lock);
+ return (true);
+}
+
+void
+lkpi_vma_pfn_unmap_end(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_state *state;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ MPASS(state != NULL);
+ sx_assert(&state->unmap_lock, SA_XLOCKED);
+ sx_xunlock(&state->unmap_lock);
+}
+
+bool
+lkpi_vma_pfn_invalidate_begin(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_state *state;
+
+ if (!lkpi_vma_pfn_unmap_begin(vma))
+ return (false);
+ state = lkpi_vma_pfn_get_state(vma);
+ MPASS(state != NULL);
+ mtx_lock(&state->objects_lock);
+ MPASS((state->invalidation_seq & 1) == 0);
+ state->invalidation_seq++;
+ mtx_unlock(&state->objects_lock);
+ return (true);
+}
+
+static void
+lkpi_vma_pfn_unmap_object(vm_object_t object, vm_pindex_t first,
+ vm_pindex_t last)
+{
+ struct pctrie_iter pages;
+ vm_page_t page;
+
+ vm_page_iter_init(&pages, object);
+ VM_OBJECT_WLOCK(object);
+retry:
+ KASSERT(pctrie_iter_is_reset(&pages),
+ ("%s: pctrie iterator is not reset", __func__));
+ VM_RADIX_FOREACH_FROM(page, &pages, first) {
+ if (page->pindex > last)
+ break;
+ /*
+ * Busy even an apparently unmapped page. A concurrent fault can
+ * hold it busy before installing its PTE; skipping it here would
+ * let that PTE appear after this invalidation pass.
+ */
+ if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
+ pctrie_iter_reset(&pages);
+ goto retry;
+ }
+ if (pmap_page_is_mapped(page))
+ pmap_remove_all(page);
+ vm_page_xunbusy(page);
+ }
+ VM_OBJECT_WUNLOCK(object);
+}
+
+void
+lkpi_vma_pfn_unmap(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_object *entry;
+ struct lkpi_vma_pfn_state *state;
+ vm_object_t object;
+ vm_pindex_t first, last;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ if (state == NULL)
+ return;
+ sx_assert(&state->unmap_lock, SA_XLOCKED);
+ mtx_lock(&state->objects_lock);
+ entry = TAILQ_FIRST(&state->objects);
+ while (entry != NULL) {
+ /* Entries and their object references live until VMA teardown. */
+ object = entry->object;
+ first = entry->first;
+ last = entry->last;
+ entry = TAILQ_NEXT(entry, link);
+ mtx_unlock(&state->objects_lock);
+ lkpi_vma_pfn_unmap_object(object, first, last);
+ mtx_lock(&state->objects_lock);
+ }
+ mtx_unlock(&state->objects_lock);
+}
+
+void
+lkpi_vma_pfn_invalidate_end(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_state *state;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ MPASS(state != NULL);
+ sx_assert(&state->unmap_lock, SA_XLOCKED);
+ mtx_lock(&state->objects_lock);
+ MPASS((state->invalidation_seq & 1) != 0);
+ state->invalidation_seq++;
+ cv_broadcast(&state->invalidation_cv);
+ mtx_unlock(&state->objects_lock);
+ lkpi_vma_pfn_unmap_end(vma);
+}
+
+void
+lkpi_vma_pfn_fini(struct vm_area_struct *vma)
+{
+ struct lkpi_vma_pfn_chunk *chunk;
+ struct lkpi_vma_pfn_object *entry;
+ struct lkpi_vma_pfn_state *state;
+ vm_page_t page;
+ unsigned int slot;
+
+ state = lkpi_vma_pfn_get_state(vma);
+ if (state == NULL)
+ return;
+ sx_assert(&state->populate_lock, SA_UNLOCKED);
+ mtx_lock(&state->objects_lock);
+ MPASS((state->invalidation_seq & 1) == 0);
+ mtx_unlock(&state->objects_lock);
+ atomic_store_rel_ptr((volatile uintptr_t *)&vma->vm_pfn_state, 0);
+ while ((chunk = TAILQ_FIRST(&state->page_chunks)) != NULL) {
+ TAILQ_REMOVE(&state->page_chunks, chunk, link);
+ for (slot = 0; slot < chunk->count; slot++) {
+ page = chunk->pages[slot];
+ if (page != NULL) {
+ MPASS(state->pending > 0);
+ state->pending--;
+ vm_page_deactivate(page);
+ vm_page_xunbusy(page);
+ }
+ }
+ kfree(chunk);
+ }
+ state->last_chunk = NULL;
+ MPASS(state->pending == 0);
+ sx_xlock(&state->unmap_lock);
+ for (;;) {
+ mtx_lock(&state->objects_lock);
+ entry = TAILQ_FIRST(&state->objects);
+ if (entry != NULL)
+ TAILQ_REMOVE(&state->objects, entry, link);
+ mtx_unlock(&state->objects_lock);
+ if (entry == NULL)
+ break;
+ vm_object_deallocate(entry->object);
+ kfree(entry);
+ }
+ sx_xunlock(&state->unmap_lock);
+ sx_destroy(&state->unmap_lock);
+ sx_destroy(&state->populate_lock);
+ cv_destroy(&state->invalidation_cv);
+ mtx_destroy(&state->objects_lock);
+ kfree(state);
+}
+
vm_fault_t
lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, pgprot_t prot)
{
+ struct lkpi_vma_pfn_state *state;
struct pctrie_iter pages;
vm_object_t vm_obj = vma->vm_obj;
vm_object_t tmp_obj;
vm_page_t page;
vm_pindex_t pindex;
+ vm_memattr_t memattr;
+ bool have_reference, reference_consumed, tracked;
+ int error;
VM_OBJECT_ASSERT_WLOCKED(vm_obj);
+ if (offset_in_page(addr) != 0 || addr < vma->vm_start ||
+ addr >= vma->vm_end || (vm_pindex_t)pfn != pfn ||
+ OFF_TO_IDX(IDX_TO_OFF((vm_pindex_t)pfn)) !=
+ (vm_pindex_t)pfn)
+ return (VM_FAULT_SIGBUS);
+ state = lkpi_vma_pfn_get_state(vma);
+ if (state == NULL)
+ return (VM_FAULT_SIGBUS);
+ if (vma->vm_pfn_count < 0 || vma->vm_pfn_count == INT_MAX)
+ return (VM_FAULT_SIGBUS);
vm_page_iter_init(&pages, vm_obj);
pindex = OFF_TO_IDX(addr - vma->vm_start);
- if (vma->vm_pfn_count == 0)
+ if (pindex >= state->npages)
+ return (VM_FAULT_SIGBUS);
+ if (vma->vm_pfn_count != 0 &&
+ pindex != vma->vm_pfn_first + vma->vm_pfn_count)
+ return (VM_FAULT_SIGBUS);
+ if (vma->vm_pfn_count == 0) {
vma->vm_pfn_first = pindex;
- MPASS(pindex <= OFF_TO_IDX(vma->vm_end));
+ }
+ MPASS(pindex < OFF_TO_IDX(vma->vm_end));
+ memattr = pgprot2cachemode(prot);
retry:
page = vm_page_grab_iter(vm_obj, pindex, VM_ALLOC_NOCREAT, &pages);
@@ -528,12 +1061,172 @@
page = PHYS_TO_VM_PAGE(IDX_TO_OFF(pfn));
if (page == NULL)
return (VM_FAULT_SIGBUS);
- if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
+ tmp_obj = atomic_load_ptr(&page->object);
+ have_reference = false;
+ tracked = tmp_obj != NULL && tmp_obj != vm_obj &&
+ lkpi_vma_pfn_object_is_tracked(state, tmp_obj);
+ if (tmp_obj != NULL && tmp_obj != vm_obj && !tracked) {
+ /*
+ * VM object locks are type-stable. Lock and revalidate the
+ * source before taking the reference that will protect this VMA.
+ */
+ VM_OBJECT_WUNLOCK(vm_obj);
+ VM_OBJECT_WLOCK(tmp_obj);
+ if (page->object != tmp_obj ||
+ (tmp_obj->flags & OBJ_DEAD) != 0) {
+ VM_OBJECT_WUNLOCK(tmp_obj);
+ VM_OBJECT_WLOCK(vm_obj);
+ pctrie_iter_reset(&pages);
+ goto retry;
+ }
+ vm_object_reference_locked(tmp_obj);
+ VM_OBJECT_WUNLOCK(tmp_obj);
+ VM_OBJECT_WLOCK(vm_obj);
+ if (vm_page_lookup(vm_obj, pindex) != NULL ||
+ atomic_load_ptr(&page->object) != tmp_obj) {
+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
+ pctrie_iter_reset(&pages);
+ goto retry;
+ }
+ have_reference = true;
+ }
+ if (!vm_page_tryxbusy(page)) {
+ /*
+ * A selected page stays xbusy until this transaction is
+ * consumed or aborted. Refuse an alias rather than wait
+ * for busy ownership that this transaction must release.
+ */
+ if (lkpi_vma_pfn_page_is_selected(vma, page)) {
+ if (have_reference)
+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
+ return (VM_FAULT_SIGBUS);
+ }
+ if (have_reference)
+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
+ else {
+ VM_OBJECT_WUNLOCK(vm_obj);
+ kern_yield(PRI_USER);
+ VM_OBJECT_WLOCK(vm_obj);
+ }
pctrie_iter_reset(&pages);
goto retry;
}
+ if (page->object != tmp_obj) {
+ vm_page_xunbusy(page);
+ if (have_reference)
+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
+ pctrie_iter_reset(&pages);
+ goto retry;
+ }
+ /*
+ * Linux installs a special PFN PTE without moving a managed page
+ * out of its backing object. Preserve that ownership for shmem
+ * pages and hand the xbusy page directly to vm_fault_populate().
+ */
+ if (tmp_obj != NULL && tmp_obj != vm_obj &&
+ tmp_obj->type == OBJT_SWAP &&
+ (page->oflags & VPO_UNMANAGED) == 0) {
+ if (!vm_page_all_valid(page)) {
+ vm_page_xunbusy(page);
+ if (have_reference)
+ lkpi_vma_pfn_drop_object_ref(vm_obj,
+ tmp_obj);
+ return (VM_FAULT_SIGBUS);
+ }
+ if (pmap_page_get_memattr(page) != memattr &&
+ pmap_page_is_mapped(page)) {
+ vm_page_xunbusy(page);
+ if (have_reference)
+ lkpi_vma_pfn_drop_object_ref(vm_obj,
+ tmp_obj);
+ return (VM_FAULT_SIGBUS);
+ }
+ if (pmap_page_get_memattr(page) != memattr) {
+ vm_page_xunbusy(page);
+ VM_OBJECT_WUNLOCK(vm_obj);
+ VM_OBJECT_WLOCK(tmp_obj);
+ if (page->object != tmp_obj ||
+ !vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
+ VM_OBJECT_WUNLOCK(tmp_obj);
+ if (have_reference)
+ vm_object_deallocate(tmp_obj);
+ VM_OBJECT_WLOCK(vm_obj);
+ pctrie_iter_reset(&pages);
+ goto retry;
+ }
+ if (page->object != tmp_obj ||
+ page->object->type != OBJT_SWAP ||
+ (page->oflags & VPO_UNMANAGED) != 0 ||
+ !vm_page_all_valid(page)) {
+ vm_page_xunbusy(page);
+ VM_OBJECT_WUNLOCK(tmp_obj);
+ if (have_reference)
+ vm_object_deallocate(tmp_obj);
+ VM_OBJECT_WLOCK(vm_obj);
+ return (VM_FAULT_SIGBUS);
+ }
+ if (pmap_page_get_memattr(page) != memattr &&
+ pmap_page_is_mapped(page)) {
+ vm_page_xunbusy(page);
+ VM_OBJECT_WUNLOCK(tmp_obj);
+ if (have_reference)
+ vm_object_deallocate(tmp_obj);
+ VM_OBJECT_WLOCK(vm_obj);
+ return (VM_FAULT_SIGBUS);
+ }
+ if (pmap_page_get_memattr(page) != memattr)
+ pmap_page_set_memattr(page, memattr);
+ VM_OBJECT_WUNLOCK(tmp_obj);
+ VM_OBJECT_WLOCK(vm_obj);
+ if (vm_page_lookup(vm_obj, pindex) != NULL) {
+ vm_page_xunbusy(page);
+ if (have_reference)
+ lkpi_vma_pfn_drop_object_ref(vm_obj,
+ tmp_obj);
+ pctrie_iter_reset(&pages);
+ goto retry;
+ }
+ }
+ MPASS(page->object == tmp_obj);
+ error = lkpi_vma_pfn_track_object(state, tmp_obj,
+ page->pindex, have_reference, &reference_consumed);
+ if (error != 0) {
+ vm_page_xunbusy(page);
+ if (have_reference)
+ lkpi_vma_pfn_drop_object_ref(vm_obj,
+ tmp_obj);
+ return (error == ENOMEM ? VM_FAULT_OOM :
+ VM_FAULT_SIGBUS);
+ }
+ if (have_reference && !reference_consumed) {
+ /*
+ * populate_lock normally makes this impossible. Retry
+ * instead of leaking the redundant reference if another
+ * producer ever gains access to the tracking list.
+ */
+ vm_page_xunbusy(page);
+ lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
+ pctrie_iter_reset(&pages);
+ goto retry;
+ }
+ error = lkpi_vma_pfn_store_page(state, pindex, page);
+ if (error != 0) {
+ vm_page_xunbusy(page);
+ return (VM_FAULT_OOM);
+ }
+ vma->vm_pfn_count++;
+ return (VM_FAULT_NOPAGE);
+ }
if (page->object != NULL) {
- tmp_obj = page->object;
+ if (tracked) {
+ vm_page_xunbusy(page);
+ return (VM_FAULT_SIGBUS);
+ }
+ if (tmp_obj == vm_obj) {
+ vm_object_reference_locked(tmp_obj);
+ have_reference = true;
+ }
+ MPASS(have_reference);
vm_page_xunbusy(page);
VM_OBJECT_WUNLOCK(vm_obj);
VM_OBJECT_WLOCK(tmp_obj);
@@ -541,12 +1234,18 @@
vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
KASSERT(page->object == tmp_obj,
("page has changed identity"));
- KASSERT((page->oflags & VPO_UNMANAGED) == 0,
- ("page does not belong to shmem"));
+ if ((page->oflags & VPO_UNMANAGED) != 0) {
+ vm_page_xunbusy(page);
+ VM_OBJECT_WUNLOCK(tmp_obj);
+ vm_object_deallocate(tmp_obj);
+ VM_OBJECT_WLOCK(vm_obj);
+ return (VM_FAULT_SIGBUS);
+ }
vm_pager_page_unswapped(page);
if (pmap_page_is_mapped(page)) {
vm_page_xunbusy(page);
VM_OBJECT_WUNLOCK(tmp_obj);
+ vm_object_deallocate(tmp_obj);
printf("%s: page rename failed: page "
"is mapped\n", __func__);
VM_OBJECT_WLOCK(vm_obj);
@@ -555,6 +1254,7 @@
vm_page_remove(page);
}
VM_OBJECT_WUNLOCK(tmp_obj);
+ vm_object_deallocate(tmp_obj);
pctrie_iter_reset(&pages);
VM_OBJECT_WLOCK(vm_obj);
goto retry;
@@ -565,7 +1265,16 @@
}
vm_page_valid(page);
}
- pmap_page_set_memattr(page, pgprot2cachemode(prot));
+ if (VM_PAGE_TO_PHYS(page) != IDX_TO_OFF(pfn)) {
+ vm_page_xunbusy(page);
+ return (VM_FAULT_SIGBUS);
+ }
+ if (pmap_page_get_memattr(page) != memattr &&
+ pmap_page_is_mapped(page)) {
+ vm_page_xunbusy(page);
+ return (VM_FAULT_SIGBUS);
+ }
+ pmap_page_set_memattr(page, memattr);
vma->vm_pfn_count++;
return (VM_FAULT_NOPAGE);
@@ -576,14 +1285,25 @@
unsigned long start_pfn, unsigned long size, pgprot_t prot)
{
vm_object_t vm_obj;
- unsigned long addr, pfn;
+ unsigned long addr, end_addr, npages, pfn;
int err = 0;
vm_obj = vma->vm_obj;
+ if (size == 0)
+ return (0);
+ if (offset_in_page(start_addr) != 0 || offset_in_page(size) != 0 ||
+ start_addr < vma->vm_start || start_addr >= vma->vm_end ||
+ size > vma->vm_end - start_addr)
+ return (-EINVAL);
+ npages = size >> PAGE_SHIFT;
+ if ((vm_pindex_t)start_pfn != start_pfn ||
+ npages - 1 > ULONG_MAX - start_pfn)
+ return (-EINVAL);
+ end_addr = start_addr + size;
VM_OBJECT_WLOCK(vm_obj);
for (addr = start_addr, pfn = start_pfn;
- addr < start_addr + size;
+ addr != end_addr;
addr += PAGE_SIZE) {
vm_fault_t ret;
retry:
@@ -606,8 +1326,7 @@
VM_OBJECT_WUNLOCK(vm_obj);
if (unlikely(err)) {
- zap_vma_ptes(vma, start_addr,
- (pfn - start_pfn) << PAGE_SHIFT);
+ zap_vma_ptes(vma, start_addr, addr - start_addr);
return (err);
}
@@ -643,7 +1362,7 @@
devobj = cdev_pager_lookup(obj);
if (devobj != NULL) {
- cdev_mgtdev_pager_free_pages(devobj);
+ linux_cdev_pager_free_pages(devobj);
vm_object_deallocate(devobj);
}
}
diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c
--- a/sys/vm/device_pager.c
+++ b/sys/vm/device_pager.c
@@ -66,6 +66,9 @@
static void dev_pager_free_page(vm_object_t object, vm_page_t m);
static int dev_pager_populate(vm_object_t object, vm_pindex_t pidx,
int fault_type, vm_prot_t, vm_pindex_t *first, vm_pindex_t *last);
+static vm_page_t dev_pager_populate_take_page(vm_object_t object,
+ vm_pindex_t pidx);
+static void dev_pager_populate_done(vm_object_t object);
/* list of device pager objects */
static struct pagerlst dev_pager_object_list;
@@ -90,6 +93,8 @@
.pgo_putpages = dev_pager_putpages,
.pgo_haspage = dev_pager_haspage,
.pgo_populate = dev_pager_populate,
+ .pgo_populate_take_page = dev_pager_populate_take_page,
+ .pgo_populate_done = dev_pager_populate_done,
};
static int old_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
@@ -146,6 +151,12 @@
return (NULL);
KASSERT(tp == OBJT_MGTDEVICE || ops->cdev_pg_populate == NULL,
("populate on unmanaged device pager"));
+ KASSERT((ops->cdev_pg_populate_take_page == NULL) ==
+ (ops->cdev_pg_populate_done == NULL),
+ ("incomplete populate handoff methods"));
+ KASSERT(ops->cdev_pg_populate_take_page == NULL ||
+ ops->cdev_pg_populate != NULL,
+ ("populate handoff without populate method"));
/*
* Offset should be page aligned.
@@ -232,6 +243,9 @@
object = NULL;
mtx_lock(&dev_pager_mtx);
} else {
+ if (ops->cdev_pg_mlock_skip != NULL &&
+ ops->cdev_pg_mlock_skip(handle))
+ vm_object_set_flag(object, OBJ_NOMLOCK);
mtx_lock(&dev_pager_mtx);
object->flags |= OBJ_COLORED;
object->pg_color = color;
@@ -410,6 +424,26 @@
fault_type, max_prot, first, last));
}
+static vm_page_t
+dev_pager_populate_take_page(vm_object_t object, vm_pindex_t pidx)
+{
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ if (object->un_pager.devp.ops->cdev_pg_populate_take_page == NULL)
+ return (NULL);
+ return (object->un_pager.devp.ops->cdev_pg_populate_take_page(
+ object, pidx));
+}
+
+static void
+dev_pager_populate_done(vm_object_t object)
+{
+
+ VM_OBJECT_ASSERT_WLOCKED(object);
+ if (object->un_pager.devp.ops->cdev_pg_populate_done != NULL)
+ object->un_pager.devp.ops->cdev_pg_populate_done(object);
+}
+
static int
old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot,
vm_page_t *mres)
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -163,6 +163,7 @@
FAULT_FAILURE, /* Return failure to user. */
FAULT_CONTINUE, /* Continue faulting. */
FAULT_RESTART, /* Restart fault. */
+ FAULT_OOM, /* Retry after waiting for memory. */
FAULT_OUT_OF_BOUNDS, /* Invalid address for pager. */
FAULT_HARD, /* Performed I/O. */
FAULT_SOFT, /* Found valid page. */
@@ -484,8 +485,9 @@
/*
* Check each page to ensure that the pager is obeying the
- * interface: the page must be installed in the object, fully
- * valid, and exclusively busied.
+ * interface: the page must be fully valid and exclusively busied.
+ * Most populated pages are installed in the pager object, but a pager
+ * may explicitly return a page owned by another object.
*/
MPASS(m != NULL);
MPASS(vm_page_all_valid(m));
@@ -498,26 +500,39 @@
{
struct pctrie_iter pages;
vm_page_t m;
+ vm_pindex_t pidx;
VM_OBJECT_ASSERT_WLOCKED(object);
MPASS(first <= last);
- vm_page_iter_limit_init(&pages, object, last + 1);
- VM_RADIX_FORALL_FROM(m, &pages, first) {
+ m = vm_pager_populate_take_page(object, first);
+ if (m == NULL) {
+ vm_page_iter_limit_init(&pages, object, last + 1);
+ VM_RADIX_FORALL_FROM(m, &pages, first) {
+ vm_fault_populate_check_page(m);
+ vm_page_deactivate(m);
+ vm_page_xunbusy(m);
+ }
+ KASSERT(pages.index == last,
+ ("%s: Object %p first %#jx last %#jx index %#jx",
+ __func__, object, (uintmax_t)first, (uintmax_t)last,
+ (uintmax_t)pages.index));
+ return;
+ }
+ for (pidx = first;; pidx++) {
vm_fault_populate_check_page(m);
vm_page_deactivate(m);
vm_page_xunbusy(m);
+ if (pidx == last)
+ break;
+ m = vm_pager_populate_take_page(object, pidx + 1);
}
- KASSERT(pages.index == last,
- ("%s: Object %p first %#jx last %#jx index %#jx",
- __func__, object, (uintmax_t)first, (uintmax_t)last,
- (uintmax_t)pages.index));
}
static enum fault_status
vm_fault_populate(struct faultstate *fs)
{
vm_offset_t vaddr;
- vm_page_t m;
+ vm_page_t external_hold, m;
vm_pindex_t map_first, map_last, pager_first, pager_last, pidx;
int bdry_idx, i, npages, psind, rv;
enum fault_status res;
@@ -534,6 +549,7 @@
vm_fault_unlock_vp(fs);
res = FAULT_SUCCESS;
+ external_hold = NULL;
/*
* Call the pager (driver) populate() method.
@@ -558,6 +574,10 @@
return (FAULT_RESTART);
return (FAULT_CONTINUE);
}
+ if (rv == VM_PAGER_OUT_OF_BOUNDS)
+ return (FAULT_OUT_OF_BOUNDS);
+ if (rv == VM_PAGER_AGAIN)
+ return (FAULT_OOM);
if (rv != VM_PAGER_OK)
return (FAULT_FAILURE); /* AKA SIGSEGV */
@@ -574,10 +594,23 @@
vm_fault_populate_cleanup(fs->first_object, pager_first,
pager_last);
} else {
- m = vm_page_lookup(fs->first_object, pager_first);
- if (m != fs->m)
+ m = vm_pager_populate_take_page(fs->first_object,
+ pager_first);
+ if (m != NULL) {
+ vm_fault_populate_check_page(m);
+ vm_page_deactivate(m);
vm_page_xunbusy(m);
+ if (pager_first < pager_last)
+ vm_fault_populate_cleanup(
+ fs->first_object, pager_first + 1,
+ pager_last);
+ } else {
+ m = vm_page_lookup(fs->first_object, pager_first);
+ if (m != NULL && m != fs->m)
+ vm_page_xunbusy(m);
+ }
}
+ vm_pager_populate_done(fs->first_object);
return (FAULT_RESTART);
}
@@ -590,6 +623,18 @@
if (bdry_idx != 0) {
KASSERT(PMAP_HAS_LARGEPAGES,
("missing pmap support for large pages"));
+ m = vm_pager_populate_take_page(fs->first_object,
+ pager_first);
+ if (m != NULL) {
+ vm_fault_populate_check_page(m);
+ vm_page_deactivate(m);
+ vm_page_xunbusy(m);
+ if (pager_first < pager_last)
+ vm_fault_populate_cleanup(fs->first_object,
+ pager_first + 1, pager_last);
+ res = FAULT_FAILURE;
+ goto out;
+ }
m = vm_page_lookup(fs->first_object, pager_first);
vm_fault_populate_check_page(m);
VM_OBJECT_WUNLOCK(fs->first_object);
@@ -645,8 +690,44 @@
for (pidx = pager_first; pidx <= pager_last; pidx += npages) {
bool writeable;
- m = vm_page_lookup(fs->first_object, pidx);
vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset;
+ m = vm_pager_populate_take_page(fs->first_object, pidx);
+ if (m != NULL) {
+ npages = 1;
+ vm_fault_populate_check_page(m);
+ if (fs->wired ||
+ (fs->fault_flags & VM_FAULT_WIRE) != 0) {
+ vm_page_deactivate(m);
+ vm_page_xunbusy(m);
+ if (pidx < pager_last)
+ vm_fault_populate_cleanup(fs->first_object,
+ pidx + 1, pager_last);
+ res = FAULT_FAILURE;
+ goto out;
+ }
+ vm_fault_dirty(fs, m);
+ VM_OBJECT_WUNLOCK(fs->first_object);
+ rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot,
+ fs->fault_type, 0);
+ VM_OBJECT_WLOCK(fs->first_object);
+ if (rv != KERN_SUCCESS) {
+ vm_page_deactivate(m);
+ vm_page_xunbusy(m);
+ if (pidx < pager_last)
+ vm_fault_populate_cleanup(fs->first_object,
+ pidx + 1, pager_last);
+ res = FAULT_FAILURE;
+ goto out;
+ }
+ vm_page_activate(m);
+ if (fs->m_hold != NULL && pidx == fs->first_pindex) {
+ vm_page_wire(m);
+ external_hold = m;
+ }
+ vm_page_xunbusy(m);
+ continue;
+ }
+ m = vm_page_lookup(fs->first_object, pidx);
KASSERT(m != NULL && m->pindex == pidx,
("%s: pindex mismatch", __func__));
psind = m->psind;
@@ -713,6 +794,14 @@
}
}
out:
+ /* Publish the held page only after the entire handoff succeeds. */
+ if (external_hold != NULL) {
+ if (res == FAULT_SUCCESS)
+ *fs->m_hold = external_hold;
+ else
+ vm_page_unwire(external_hold, PQ_INACTIVE);
+ }
+ vm_pager_populate_done(fs->first_object);
curthread->td_ru.ru_majflt++;
return (res);
}
@@ -1325,9 +1414,17 @@
switch (res) {
case FAULT_SUCCESS:
case FAULT_FAILURE:
+ case FAULT_OUT_OF_BOUNDS:
case FAULT_RESTART:
vm_fault_unlock_and_deallocate(fs);
return (res);
+ case FAULT_OOM:
+ dset = fs->object->domain.dr_policy;
+ if (dset == NULL)
+ dset = curthread->td_domain.dr_policy;
+ if (vm_fault_allocate_oom(fs))
+ vm_waitpfault(dset, vm_pfault_oom_wait * hz);
+ return (FAULT_RESTART);
case FAULT_CONTINUE:
pctrie_iter_reset(pages);
/*
diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -3624,6 +3624,17 @@
rv = KERN_INVALID_ADDRESS;
goto done;
}
+ } else if (user_wire &&
+ (entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
+ entry->object.vm_object != NULL &&
+ (entry->object.vm_object->flags & OBJ_NOMLOCK) != 0) {
+ /*
+ * The pager owns residency for this special device mapping.
+ * Retain range/hole checks and transition cleanup, but do not
+ * claim to pin its pages or acquire a user wire reference.
+ * Kernel wiring must still satisfy the ordinary contract.
+ */
+ entry->eflags |= MAP_ENTRY_WIRE_SKIPPED;
} else if (entry->wired_count == 0) {
entry->wired_count++;
diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
--- a/sys/vm/vm_object.h
+++ b/sys/vm/vm_object.h
@@ -201,6 +201,7 @@
#define OBJ_PAGERPRIV2 0x00008000 /* Pager private */
#define OBJ_SYSVSHM 0x00010000 /* SysV SHM */
#define OBJ_POSIXSHM 0x00020000 /* Posix SHM */
+#define OBJ_NOMLOCK 0x00040000 /* (c) skip userspace memory locking */
/*
* Helpers to perform conversion between vm_object page indexes and offsets.
diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h
--- a/sys/vm/vm_pager.h
+++ b/sys/vm/vm_pager.h
@@ -58,6 +58,8 @@
typedef boolean_t pgo_haspage_t(vm_object_t, vm_pindex_t, int *, int *);
typedef int pgo_populate_t(vm_object_t, vm_pindex_t, int, vm_prot_t,
vm_pindex_t *, vm_pindex_t *);
+typedef vm_page_t pgo_populate_take_page_t(vm_object_t, vm_pindex_t);
+typedef void pgo_populate_done_t(vm_object_t);
typedef void pgo_pageunswapped_t(vm_page_t);
typedef void pgo_writecount_t(vm_object_t, vm_offset_t, vm_offset_t);
typedef void pgo_set_writeable_dirty_t(vm_object_t);
@@ -90,6 +92,8 @@
pgo_page_inserted_t *pgo_page_inserted;
pgo_page_removed_t *pgo_page_removed;
pgo_can_alloc_page_t *pgo_can_alloc_page;
+ pgo_populate_take_page_t *pgo_populate_take_page;
+ pgo_populate_done_t *pgo_populate_done;
};
extern const struct pagerops defaultpagerops;
@@ -109,6 +113,7 @@
* PEND operations was initiated but not completed
* ERROR error while accessing data that is in range and exists
* AGAIN temporary resource shortage prevented operation from happening
+ * OUT_OF_BOUNDS pager-specific address is invalid and should raise SIGBUS
*/
#define VM_PAGER_OK 0
#define VM_PAGER_BAD 1
@@ -116,6 +121,7 @@
#define VM_PAGER_PEND 3
#define VM_PAGER_ERROR 4
#define VM_PAGER_AGAIN 5
+#define VM_PAGER_OUT_OF_BOUNDS 6
#define VM_PAGER_PUT_SYNC 0x0001
#define VM_PAGER_PUT_INVAL 0x0002
@@ -186,6 +192,33 @@
fault_type, max_prot, first, last));
}
+/*
+ * Take an xbusy page populated by the pager without inserting it into the
+ * pager object. The caller assumes responsibility for releasing the busy
+ * state. A populate operation must not mix such pages with pages installed
+ * in the pager object. Most pagers populate only object-resident pages and
+ * leave this method unset.
+ */
+static __inline vm_page_t
+vm_pager_populate_take_page(vm_object_t object, vm_pindex_t pidx)
+{
+ pgo_populate_take_page_t *method;
+
+ method = pagertab[object->type]->pgo_populate_take_page;
+ return (method == NULL ? NULL : method(object, pidx));
+}
+
+/* Notify the pager that all pages returned by populate() were consumed. */
+static __inline void
+vm_pager_populate_done(vm_object_t object)
+{
+ pgo_populate_done_t *method;
+
+ method = pagertab[object->type]->pgo_populate_done;
+ if (method != NULL)
+ method(object);
+}
+
/*
* vm_pager_page_unswapped
*
@@ -294,6 +327,17 @@
vm_ooffset_t foff, struct ucred *cred, u_short *color);
void (*cdev_pg_dtor)(void *handle);
void (*cdev_pg_path)(void *handle, char *path, size_t len);
+ vm_page_t (*cdev_pg_populate_take_page)(vm_object_t vm_obj,
+ vm_pindex_t pidx);
+ void (*cdev_pg_populate_done)(vm_object_t vm_obj);
+ /*
+ * Optional special-mapping policy, queried after successful construction
+ * and before publishing the object. True exempts this object's mappings
+ * from userspace memory locking, but does not permit kernel wiring to
+ * succeed without actually wiring pages. The result is immutable for
+ * the object's lifetime; NULL preserves ordinary memory-locking policy.
+ */
+ bool (*cdev_pg_mlock_skip)(void *handle);
};
vm_object_t cdev_pager_allocate(void *handle, enum obj_type tp,
diff --git a/sys/vm/vm_pager.c b/sys/vm/vm_pager.c
--- a/sys/vm/vm_pager.c
+++ b/sys/vm/vm_pager.c
@@ -401,6 +401,10 @@
int res;
mtx_lock(&pagertab_lock);
+ MPASS((ops->pgo_populate_take_page == NULL) ==
+ (ops->pgo_populate_done == NULL));
+ MPASS(ops->pgo_populate_take_page == NULL ||
+ ops->pgo_populate != NULL);
MPASS(base_type == -1 ||
(base_type >= OBJT_SWAP && base_type < nitems(pagertab)));
for (res = OBJT_FIRST_DYN; res < nitems(pagertab); res++) {
@@ -423,7 +427,6 @@
FIX(getpages_async);
FIX(putpages);
FIX(haspage);
- FIX(populate);
FIX(pageunswapped);
FIX(update_writecount);
FIX(release_writecount);
@@ -435,6 +438,17 @@
FIX(page_removed);
FIX(can_alloc_page);
#undef FIX
+ /* The populate handoff methods form one indivisible contract. */
+ if (ops->pgo_populate == NULL) {
+ MPASS(ops->pgo_populate_take_page == NULL);
+ MPASS(ops->pgo_populate_done == NULL);
+ ops->pgo_populate =
+ pagertab[base_type]->pgo_populate;
+ ops->pgo_populate_take_page =
+ pagertab[base_type]->pgo_populate_take_page;
+ ops->pgo_populate_done =
+ pagertab[base_type]->pgo_populate_done;
+ }
}
pagertab[res] = ops; /* XXXKIB should be rel, but acq is too much */
mtx_unlock(&pagertab_lock);

File Metadata

Mime Type
text/plain
Expires
Tue, Sep 8, 2:38 PM (6 h, 3 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
38520730
Default Alt Text
D59481.diff (49 KB)

Event Timeline