linuxkpi-vmap-D59481-coordination-full-context.patch
Public
Actions

Authored By

	oleglelchuk_gmail.com
	Tue, Sep 29, 3:38 PM

Size

49 KB

Referenced Files

None

Subscribers

None

linuxkpi-vmap-D59481-coordination-full-context.patch
View Options

	diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c
	index 90ba1606fd69ff52b39f3095cdaa2987aad40dd1..16498f2f04e5c6152b2ed834a75bec4956f43a4c 100644
	--- a/sys/compat/linuxkpi/common/src/linux_page.c
	+++ b/sys/compat/linuxkpi/common/src/linux_page.c
	@@ -1,1828 +1,1843 @@
	/*-
	* Copyright (c) 2010 Isilon Systems, Inc.
	* Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io)
	* Copyright (c) 2017 Mellanox Technologies, Ltd.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	* notice unmodified, this list of conditions, and the following
	* disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in the
	* documentation and/or other materials provided with the distribution.
	*
	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	#include <sys/param.h>
	#include <sys/systm.h>
	#include <sys/malloc.h>
	#include <sys/kernel.h>
	#include <sys/sysctl.h>
	#include <sys/lock.h>
	#include <sys/mutex.h>
	#include <sys/rwlock.h>
	#include <sys/sx.h>
	#include <sys/tree.h>
	#include <sys/proc.h>
	#include <sys/sched.h>
	#include <sys/memrange.h>

	#include <machine/bus.h>

	#include <vm/vm.h>
	#include <vm/pmap.h>
	#include <vm/vm_param.h>
	#include <vm/vm_kern.h>
	#include <vm/vm_object.h>
	#include <vm/vm_map.h>
	#include <vm/vm_page.h>
	#include <vm/vm_pageout.h>
	#include <vm/vm_pager.h>
	#include <vm/vm_radix.h>
	#include <vm/vm_reserv.h>
	#include <vm/vm_extern.h>

	#include <vm/uma.h>
	#include <vm/uma_int.h>

	/*
	* One pager-owned wire per physical page, independent of driver wiring.
	* Generate the native tree before Linux headers redefine RB_ROOT.
	*/
	struct lkpi_vma_pfn_pin {
	RB_ENTRY(lkpi_vma_pfn_pin) link;
	vm_page_t m;
	};

	RB_HEAD(lkpi_vma_pfn_pins, lkpi_vma_pfn_pin);

	static int
	lkpi_vma_pfn_pin_cmp(struct lkpi_vma_pfn_pin *a,
	struct lkpi_vma_pfn_pin *b)
	{

	return ((uintptr_t)a->m < (uintptr_t)b->m ? -1 :
	(uintptr_t)a->m > (uintptr_t)b->m);
	}

	RB_GENERATE_STATIC(lkpi_vma_pfn_pins, lkpi_vma_pfn_pin, link,
	lkpi_vma_pfn_pin_cmp);

	#include <linux/gfp.h>
	#include <linux/mm.h>
	#include <linux/preempt.h>
	#include <linux/fs.h>
	#include <linux/shmem_fs.h>
	#include <linux/kernel.h>
	#include <linux/idr.h>
	#include <linux/io.h>
	#include <linux/io-mapping.h>
	#include <linux/device.h>
	#include <linux/slab.h>

	#ifdef __i386__
	DEFINE_IDR(mtrr_idr);
	static MALLOC_DEFINE(M_LKMTRR, "idr", "Linux MTRR compat");
	extern int pat_works;
	#endif

	void
	si_meminfo(struct sysinfo *si)
	{
	si->totalram = physmem;
	si->freeram = vm_free_count();
	si->totalhigh = 0;
	si->freehigh = 0;
	si->mem_unit = PAGE_SIZE;
	}

	void *
	linux_page_address(const struct page *page)
	{

	if (page->object != kernel_object) {
	return (PMAP_HAS_DMAP ? PHYS_TO_DMAP(page_to_phys(page)) :
	NULL);
	}
	return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS +
	IDX_TO_OFF(page->pindex)));
	}

	struct page *
	linux_alloc_pages(gfp_t flags, unsigned int order)
	{
	struct page *page;

	if (PMAP_HAS_DMAP) {
	unsigned long npages = 1UL << order;
	int req = VM_ALLOC_WIRED;

	if ((flags & M_ZERO) != 0)
	req \|= VM_ALLOC_ZERO;

	if (order == 0 && (flags & GFP_DMA32) == 0) {
	page = vm_page_alloc_noobj(req);
	if (page == NULL)
	return (NULL);
	} else {
	vm_paddr_t pmax = (flags & GFP_DMA32) ?
	BUS_SPACE_MAXADDR_32BIT : BUS_SPACE_MAXADDR;

	if ((flags & __GFP_NORETRY) != 0)
	req \|= VM_ALLOC_NORECLAIM;

	retry:
	if ((flags & __GFP_THISNODE) != 0) {
	int curdomain = PCPU_GET(domain);
	page = vm_page_alloc_noobj_contig_domain(
	curdomain, req, npages, 0, pmax,
	PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
	} else {
	page = vm_page_alloc_noobj_contig(
	req, npages, 0, pmax,
	PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
	}

	if (page == NULL) {
	if ((flags & (M_WAITOK \| __GFP_NORETRY \| __GFP_THISNODE)) ==
	M_WAITOK) {
	int err = vm_page_reclaim_contig(req,
	npages, 0, pmax, PAGE_SIZE, 0);
	if (err == ENOMEM)
	vm_wait(NULL);
	else if (err != 0)
	return (NULL);
	flags &= ~M_WAITOK;
	goto retry;
	}
	return (NULL);
	}
	}
	} else {
	vm_offset_t vaddr;

	vaddr = linux_alloc_kmem(flags, order);
	if (vaddr == 0)
	return (NULL);

	page = virt_to_page((void *)vaddr);

	KASSERT(vaddr == (vm_offset_t)page_address(page),
	("Page address mismatch"));
	}

	return (page);
	}

	static void
	_linux_free_kmem(vm_offset_t addr, unsigned int order)
	{
	size_t size = ((size_t)PAGE_SIZE) << order;

	kmem_free((void *)addr, size);
	}

	void
	linux_free_pages(struct page *page, unsigned int order)
	{
	if (PMAP_HAS_DMAP) {
	unsigned long npages = 1UL << order;
	unsigned long x;

	for (x = 0; x != npages; x++) {
	vm_page_t pgo = page + x;

	/*
	* The "free page" function is used in several
	* contexts.
	*
	* Some pages are allocated by `linux_alloc_pages()`
	* above, but not all of them are. For instance in the
	* DRM drivers, some pages come from
	* `shmem_read_mapping_page_gfp()`.
	*
	* That's why we need to check if the page is managed
	* or not here.
	*/
	if ((pgo->oflags & VPO_UNMANAGED) == 0) {
	vm_page_unwire(pgo, PQ_ACTIVE);
	} else {
	if (vm_page_unwire_noq(pgo))
	vm_page_free(pgo);
	}
	}
	} else {
	vm_offset_t vaddr;

	vaddr = (vm_offset_t)page_address(page);

	_linux_free_kmem(vaddr, order);
	}
	}

	void
	linux_release_pages(release_pages_arg arg, int nr)
	{
	int i;

	CTASSERT(offsetof(struct folio, page) == 0);

	for (i = 0; i < nr; i++)
	__free_page(arg.pages[i]);
	}

	vm_offset_t
	linux_alloc_kmem(gfp_t flags, unsigned int order)
	{
	size_t size = ((size_t)PAGE_SIZE) << order;
	void *addr;

	addr = kmem_alloc_contig(size, flags & GFP_NATIVE_MASK, 0,
	((flags & GFP_DMA32) == 0) ? -1UL : BUS_SPACE_MAXADDR_32BIT,
	PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);

	return ((vm_offset_t)addr);
	}

	void
	linux_free_kmem(vm_offset_t addr, unsigned int order)
	{
	KASSERT((addr & ~PAGE_MASK) == 0,
	("%s: addr %p is not page aligned", __func__, (void *)addr));

	if (addr >= VM_MIN_KERNEL_ADDRESS && addr < VM_MAX_KERNEL_ADDRESS) {
	_linux_free_kmem(addr, order);
	} else {
	vm_page_t page;

	page = DMAP_TO_VM_PAGE(addr);
	linux_free_pages(page, order);
	}
	}

	static int
	linux_get_user_pages_internal(vm_map_t map, unsigned long start, int nr_pages,
	int write, struct page **pages)
	{
	vm_prot_t prot;
	size_t len;
	int count;

	prot = write ? (VM_PROT_READ \| VM_PROT_WRITE) : VM_PROT_READ;
	len = ptoa((vm_offset_t)nr_pages);
	count = vm_fault_quick_hold_pages(map, start, len, prot, pages, nr_pages);
	return (count == -1 ? -EFAULT : nr_pages);
	}

	int
	__get_user_pages_fast(unsigned long start, int nr_pages, int write,
	struct page **pages)
	{
	vm_map_t map;
	vm_page_t *mp;
	vm_offset_t va;
	vm_offset_t end;
	vm_prot_t prot;
	int count;

	if (nr_pages == 0 \|\| in_interrupt())
	return (0);

	MPASS(pages != NULL);
	map = &curthread->td_proc->p_vmspace->vm_map;
	end = start + ptoa((vm_offset_t)nr_pages);
	if (!vm_map_range_valid(map, start, end))
	return (-EINVAL);
	prot = write ? (VM_PROT_READ \| VM_PROT_WRITE) : VM_PROT_READ;
	for (count = 0, mp = pages, va = start; va < end;
	mp++, va += PAGE_SIZE, count++) {
	*mp = pmap_extract_and_hold(map->pmap, va, prot);
	if (*mp == NULL)
	break;

	if ((prot & VM_PROT_WRITE) != 0 &&
	(*mp)->dirty != VM_PAGE_BITS_ALL) {
	/*
	* Explicitly dirty the physical page. Otherwise, the
	* caller's changes may go unnoticed because they are
	* performed through an unmanaged mapping or by a DMA
	* operation.
	*
	* The object lock is not held here.
	* See vm_page_clear_dirty_mask().
	*/
	vm_page_dirty(*mp);
	}
	}
	return (count);
	}

	long
	get_user_pages_remote(struct task_struct task, struct mm_struct mm,
	unsigned long start, unsigned long nr_pages, unsigned int gup_flags,
	struct page pages, struct vm_area_struct vmas)
	{
	vm_map_t map;

	map = &task->task_thread->td_proc->p_vmspace->vm_map;
	return (linux_get_user_pages_internal(map, start, nr_pages,
	!!(gup_flags & FOLL_WRITE), pages));
	}

	long
	lkpi_get_user_pages(unsigned long start, unsigned long nr_pages,
	unsigned int gup_flags, struct page **pages)
	{
	vm_map_t map;

	map = &curthread->td_proc->p_vmspace->vm_map;
	return (linux_get_user_pages_internal(map, start, nr_pages,
	!!(gup_flags & FOLL_WRITE), pages));
	}

	/*
	* Hash of vmmap addresses. This is infrequently accessed and does not
	* need to be particularly large. This is done because we must store the
	* caller's idea of the map size to properly unmap.
	*/
	/*
	* vmap() owns a cache-attribute reference for each page it maps. Compatible
	* overlapping vmaps must keep that attribute until their last unmap. The
	* caller continues to own the page lifetime, as required by the vmap API.
	*/
	struct vmmap_attr {
	LIST_ENTRY(vmmap_attr) link;
	vm_page_t page;
	unsigned int refs;
	vm_memattr_t attr;
	vm_memattr_t saved_attr;
	};

	struct vmmap {
	LIST_ENTRY(vmmap) vm_next;
	void *vm_addr;
	unsigned long vm_size;
	struct vmmap_attr **vm_attrs;
	};

	struct vmmaphd {
	struct vmmap *lh_first;
	};
	#define VMMAP_HASH_SIZE 64
	#define VMMAP_HASH_MASK (VMMAP_HASH_SIZE - 1)
	#define VM_HASH(addr) ((uintptr_t)(addr) >> PAGE_SHIFT) & VMMAP_HASH_MASK
	static struct vmmaphd vmmaphead[VMMAP_HASH_SIZE];
	static struct mtx vmmaplock;
	#define VMMAP_ATTR_HASH_SIZE 1024
	static struct {
	struct vmmap_attr *lh_first;
	} vmmap_attrs[VMMAP_ATTR_HASH_SIZE];
	#define VMMAP_ATTR_HASH(m) \
	(atop(VM_PAGE_TO_PHYS(m)) & (VMMAP_ATTR_HASH_SIZE - 1))

	static struct vmmap_attr *
	vmmap_attr_find(vm_page_t page)
	{
	struct vmmap_attr *a;

	mtx_assert(&vmmaplock, MA_OWNED);
	LIST_FOREACH(a, &vmmap_attrs[VMMAP_ATTR_HASH(page)], link) {
	if (a->page == page)
	return (a);
	}
	return (NULL);
	}

	static void
	vmmap_attr_put(struct vmmap_attr *a)
	{
	vm_page_t page;

	mtx_assert(&vmmaplock, MA_OWNED);
	MPASS(a->refs != 0);
	if (--a->refs != 0)
	return;
	page = a->page;
	/* The kernel PTEs must have been removed before changing the DMAP. */
	if (a->saved_attr != a->attr &&
	pmap_page_get_memattr(page) == a->attr &&
	!pmap_page_is_mapped(page))
	pmap_page_set_memattr(page, a->saved_attr);
	LIST_REMOVE(a, link);
	kfree(a);
	}

	static bool
	vmmap_attr_get_pages(struct page **pages, unsigned int count, int prot,
	struct vmmap_attr **attrs)
	{
	struct vmmap_attr *a;
	vm_memattr_t attr, requested;
	unsigned int i;

	requested = pgprot2cachemode(prot);
	mtx_lock(&vmmaplock);
	for (i = 0; i < count; i++) {
	/* Default mappings retain a page's existing cache attribute. */
	attr = requested == VM_MEMATTR_DEFAULT ?
	pmap_page_get_memattr(pages[i]) : requested;
	a = vmmap_attr_find(pages[i]);
	if (a != NULL) {
	if (a->attr != attr \|\| a->refs == UINT_MAX)
	goto fail;
	a->refs++;
	} else {
	/* Do not create conflicting aliases of existing mappings. */
	if (pmap_page_get_memattr(pages[i]) != attr &&
	(pages[i]->object == kernel_object \|\|
	pmap_page_is_mapped(pages[i])))
	goto fail;
	a = kmalloc(sizeof(*a), GFP_ATOMIC);
	if (a == NULL)
	goto fail;
	a->page = pages[i];
	a->refs = 1;
	a->attr = attr;
	a->saved_attr = pmap_page_get_memattr(pages[i]);
	LIST_INSERT_HEAD(&vmmap_attrs[VMMAP_ATTR_HASH(pages[i])],
	a, link);
	if (a->saved_attr != attr)
	pmap_page_set_memattr(pages[i], attr);
	}
	attrs[i] = a;
	}
	mtx_unlock(&vmmaplock);
	return (true);
	fail:
	while (i != 0)
	vmmap_attr_put(attrs[--i]);
	mtx_unlock(&vmmaplock);
	return (false);
	}

	+/* Caller owns xbusy and the backing-object reference. */
	+static void
	+vmmap_restore_page_memattr(vm_page_t page, vm_memattr_t attr)
	+{
	+ struct vmmap_attr *a;
	+
	+ mtx_lock(&vmmaplock);
	+ a = vmmap_attr_find(page);
	+ if (a != NULL)
	+ a->saved_attr = attr; /* Defer until the last kernel alias is gone. */
	+ else
	+ pmap_page_set_memattr(page, attr);
	+ mtx_unlock(&vmmaplock);
	+}
	+
	int
	is_vmalloc_addr(const void *addr)
	{
	struct vmmap *vmmap;

	mtx_lock(&vmmaplock);
	LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next)
	if (addr == vmmap->vm_addr)
	break;
	mtx_unlock(&vmmaplock);
	if (vmmap != NULL)
	return (1);

	return (vtoslab((vm_offset_t)addr & ~UMA_SLAB_MASK) != NULL);
	}

	static struct vmmap *
	vmmap_add(void *addr, unsigned long size)
	{
	struct vmmap *vmmap;

	vmmap = kmalloc(sizeof(*vmmap), GFP_KERNEL);
	mtx_lock(&vmmaplock);
	vmmap->vm_size = size;
	vmmap->vm_addr = addr;
	vmmap->vm_attrs = NULL;
	LIST_INSERT_HEAD(&vmmaphead[VM_HASH(addr)], vmmap, vm_next);
	mtx_unlock(&vmmaplock);
	return (vmmap);
	}

	static struct vmmap *
	vmmap_remove(void *addr)
	{
	struct vmmap *vmmap;

	mtx_lock(&vmmaplock);
	LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next)
	if (vmmap->vm_addr == addr)
	break;
	if (vmmap)
	LIST_REMOVE(vmmap, vm_next);
	mtx_unlock(&vmmaplock);

	return (vmmap);
	}

	#if defined(__i386__) \|\| defined(__amd64__) \|\| defined(__powerpc__) \|\| defined(__aarch64__) \|\| defined(__riscv)
	void *
	_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr)
	{
	void *addr;

	addr = pmap_mapdev_attr(phys_addr, size, attr);
	if (addr == NULL)
	return (NULL);
	vmmap_add(addr, size);

	return (addr);
	}
	#endif

	void
	iounmap(void *addr)
	{
	struct vmmap *vmmap;

	vmmap = vmmap_remove(addr);
	if (vmmap == NULL)
	return;
	#if defined(__i386__) \|\| defined(__amd64__) \|\| defined(__powerpc__) \|\| defined(__aarch64__) \|\| defined(__riscv)
	pmap_unmapdev(addr, vmmap->vm_size);
	#endif
	kfree(vmmap);
	}

	static void
	lkpi_devm_memremap_unmap(struct device dev, void p)
	{
	void **dr = p;

	memunmap(*dr);
	}

	void *
	linuxkpi_devm_memremap(struct device *dev, resource_size_t offset, size_t size,
	unsigned long flags)
	{
	void *dr, addr;

	dr = devres_alloc(lkpi_devm_memremap_unmap, sizeof(*dr), GFP_KERNEL);
	if (dr == NULL)
	return (ERR_PTR(-ENOMEM));
	addr = memremap(offset, size, flags);
	if (addr != NULL) {
	*dr = addr;
	devres_add(dev, dr);
	} else {
	addr = ERR_PTR(-ENXIO);
	devres_free(dr);
	}

	return (addr);
	}

	void *
	vmap(struct page **pages, unsigned int count, unsigned long flags, int prot)
	{
	struct vmmap_attr **attrs;
	struct vmmap *vmmap;
	void *off;
	size_t size;

	size = (size_t)count * PAGE_SIZE;
	if (count == 0 \|\| size / PAGE_SIZE != count)
	return (NULL);
	off = kva_alloc(size);
	if (off == NULL)
	return (NULL);
	attrs = kcalloc(count, sizeof(*attrs), GFP_KERNEL);
	if (attrs == NULL)
	goto fail;

	/*
	* pmap_qenter() uses each page's memattr, not the prot argument.
	* Apply the requested attribute before installing kernel PTEs;
	* pmap_page_set_memattr() also maintains the direct-map alias.
	*/
	if (!vmmap_attr_get_pages(pages, count, prot, attrs))
	goto fail;
	pmap_qenter(off, pages, count);
	vmmap = vmmap_add(off, size);
	vmmap->vm_attrs = attrs;
	return (off);
	fail:
	kfree(attrs);
	kva_free(off, size);
	return (NULL);
	}

	#define VMAP_MAX_CHUNK_SIZE (65536U / sizeof(struct vm_page)) /* KMEM_ZMAX */

	void *
	linuxkpi_vmap_pfn(unsigned long *pfns, unsigned int count, int prot)
	{
	vm_page_t m, *ma, fma;
	void *off;
	char *coff;
	vm_paddr_t pa;
	vm_memattr_t attr;
	size_t size;
	unsigned int i, c, chunk;

	size = ptoa(count);
	off = kva_alloc(size);
	if (off == NULL)
	return (NULL);
	vmmap_add(off, size);

	chunk = MIN(count, VMAP_MAX_CHUNK_SIZE);
	attr = pgprot2cachemode(prot);
	ma = malloc(chunk * sizeof(vm_page_t), M_TEMP, M_WAITOK \| M_ZERO);
	fma = NULL;
	c = 0;
	coff = off;
	for (i = 0; i < count; i++) {
	pa = IDX_TO_OFF(pfns[i]);
	m = PHYS_TO_VM_PAGE(pa);
	if (m == NULL) {
	if (fma == NULL)
	fma = malloc(chunk * sizeof(struct vm_page),
	M_TEMP, M_WAITOK \| M_ZERO);
	m = fma + c;
	vm_page_initfake(m, pa, attr);
	} else {
	pmap_page_set_memattr(m, attr);
	}
	ma[c] = m;
	c++;
	if (c == chunk \|\| i == count - 1) {
	pmap_qenter(coff, ma, c);
	if (i == count - 1)
	break;
	coff += ptoa(c);
	c = 0;
	memset(ma, 0, chunk * sizeof(vm_page_t));
	if (fma != NULL)
	memset(fma, 0, chunk * sizeof(struct vm_page));
	}
	}
	free(fma, M_TEMP);
	free(ma, M_TEMP);

	return (off);
	}

	void
	vunmap(void *addr)
	{
	struct vmmap *vmmap;
	unsigned int i, count;

	vmmap = vmmap_remove(addr);
	if (vmmap == NULL)
	return;
	count = vmmap->vm_size / PAGE_SIZE;
	/* Remove every alias owned by this vmap before restoring attributes. */
	pmap_qremove(addr, count);
	if (vmmap->vm_attrs != NULL) {
	mtx_lock(&vmmaplock);
	for (i = 0; i < count; i++)
	vmmap_attr_put(vmmap->vm_attrs[i]);
	mtx_unlock(&vmmaplock);
	kfree(vmmap->vm_attrs);
	}
	kva_free(addr, vmmap->vm_size);
	kfree(vmmap);
	}

	struct lkpi_vma_pfn_object {
	TAILQ_ENTRY(lkpi_vma_pfn_object) link;
	vm_object_t object;
	};

	#define LKPI_VMA_PFN_CHUNK_PAGES 64

	struct lkpi_vma_pfn_chunk {
	TAILQ_ENTRY(lkpi_vma_pfn_chunk) link;
	vm_pindex_t first;
	unsigned int count;
	unsigned int remaining;
	vm_page_t pages[LKPI_VMA_PFN_CHUNK_PAGES];
	};

	struct lkpi_vma_pfn_state {
	TAILQ_HEAD(, lkpi_vma_pfn_object) objects;
	TAILQ_HEAD(, lkpi_vma_pfn_chunk) page_chunks;
	struct lkpi_vma_pfn_pins pins;
	struct mtx objects_lock;
	struct sx populate_lock;
	struct sx unmap_lock;
	struct lkpi_vma_pfn_chunk *last_chunk;
	vm_pindex_t npages;
	vm_pindex_t pending;
	uint64_t invalidation_seq;
	uint64_t populate_seq;
	int error;
	};

	static struct lkpi_vma_pfn_state *
	lkpi_vma_pfn_get_state(struct vm_area_struct *vma)
	{

	return ((void *)atomic_load_acq_ptr(
	(volatile uintptr_t *)&vma->vm_pfn_state));
	}

	int
	lkpi_vma_pfn_init(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;
	vm_pindex_t npages;

	MPASS(lkpi_vma_pfn_get_state(vma) == NULL);
	npages = vma_pages(vma);
	if (npages == 0)
	return (EINVAL);
	state = kzalloc(sizeof(*state), GFP_KERNEL);
	if (state == NULL)
	return (ENOMEM);
	TAILQ_INIT(&state->objects);
	TAILQ_INIT(&state->page_chunks);
	RB_INIT(&state->pins);
	mtx_init(&state->objects_lock, "lkpi pfn objects", NULL, MTX_DEF);
	sx_init(&state->populate_lock, "lkpi pfn populate");
	sx_init(&state->unmap_lock, "lkpi pfn unmap");
	state->npages = npages;
	atomic_store_rel_ptr((volatile uintptr_t *)&vma->vm_pfn_state,
	(uintptr_t)state);
	return (0);
	}

	int
	lkpi_vma_pfn_begin(struct vm_area_struct vma, uint64_t invalidation_seq)
	{
	struct lkpi_vma_pfn_state *state;

	state = lkpi_vma_pfn_get_state(vma);
	if (state == NULL)
	return (EINVAL);
	/* Never wait for a previous handoff while holding mmap_sem. */
	if (!sx_try_xlock(&state->populate_lock))
	return (EAGAIN);
	if (state->pending != 0 \|\| !TAILQ_EMPTY(&state->page_chunks)) {
	sx_xunlock(&state->populate_lock);
	return (EBUSY);
	}
	mtx_lock(&state->objects_lock);
	if ((state->invalidation_seq & 1) != 0) {
	mtx_unlock(&state->objects_lock);
	sx_xunlock(&state->populate_lock);
	return (EAGAIN);
	}
	state->populate_seq = state->invalidation_seq;
	*invalidation_seq = state->populate_seq;
	state->error = 0;
	mtx_unlock(&state->objects_lock);
	return (0);
	}

	/*
	* Driver remappers may retry VM_FAULT_OOM internally while retaining their
	* locks and our earlier busy pages. Stop the remapper with an error bit,
	* retaining the real cause for linux_cdev_pager_populate() to translate after
	* the driver has unwound. In particular, VM_FAULT_RETRY alone is not an
	* error bit and would be mistaken for a successful insertion by remap_sg().
	*/
	static vm_fault_t
	lkpi_vma_pfn_fail(struct lkpi_vma_pfn_state *state, int error)
	{

	sx_assert(&state->populate_lock, SA_XLOCKED);
	if (state->error == 0)
	state->error = error;
	return (VM_FAULT_SIGBUS);
	}

	int
	lkpi_vma_pfn_error(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;

	state = lkpi_vma_pfn_get_state(vma);
	MPASS(state != NULL);
	sx_assert(&state->populate_lock, SA_XLOCKED);
	return (state->error);
	}

	bool
	lkpi_vma_pfn_unchanged(struct vm_area_struct *vma,
	uint64_t invalidation_seq)
	{
	struct lkpi_vma_pfn_state *state;
	bool unchanged;

	state = lkpi_vma_pfn_get_state(vma);
	if (state == NULL)
	return (false);
	sx_assert(&state->populate_lock, SA_XLOCKED);
	mtx_lock(&state->objects_lock);
	unchanged = state->invalidation_seq == invalidation_seq;
	mtx_unlock(&state->objects_lock);
	return (unchanged);
	}

	bool
	lkpi_vma_pfn_handoff_valid(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;

	state = lkpi_vma_pfn_get_state(vma);
	MPASS(state != NULL);
	sx_assert(&state->populate_lock, SA_XLOCKED);
	return (vma->vm_pfn_count > 0 && (state->pending == 0 \|\|
	state->pending == (vm_pindex_t)vma->vm_pfn_count));
	}

	bool
	lkpi_vma_pfn_lock(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;

	state = lkpi_vma_pfn_get_state(vma);
	if (state == NULL \|\| sx_xlocked(&state->populate_lock))
	return (false);
	sx_xlock(&state->populate_lock);
	return (true);
	}

	void
	lkpi_vma_pfn_end(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;

	state = lkpi_vma_pfn_get_state(vma);
	MPASS(state != NULL);
	sx_assert(&state->populate_lock, SA_XLOCKED);
	MPASS(state->pending == 0);
	sx_xunlock(&state->populate_lock);
	}

	static bool
	lkpi_vma_pfn_object_is_tracked_locked(struct lkpi_vma_pfn_state *state,
	vm_object_t object)
	{
	struct lkpi_vma_pfn_object *entry;

	mtx_assert(&state->objects_lock, MA_OWNED);
	TAILQ_FOREACH(entry, &state->objects, link) {
	if (entry->object == object)
	return (true);
	}
	return (false);
	}

	static bool
	lkpi_vma_pfn_object_is_tracked(struct lkpi_vma_pfn_state *state,
	vm_object_t object)
	{
	bool tracked;

	sx_assert(&state->populate_lock, SA_XLOCKED);
	mtx_lock(&state->objects_lock);
	tracked = lkpi_vma_pfn_object_is_tracked_locked(state, object);
	mtx_unlock(&state->objects_lock);
	return (tracked);
	}

	static void
	lkpi_vma_pfn_drop_object_ref(vm_object_t locked_object,
	vm_object_t referenced_object)
	{

	VM_OBJECT_ASSERT_WLOCKED(locked_object);
	VM_OBJECT_WUNLOCK(locked_object);
	vm_object_deallocate(referenced_object);
	VM_OBJECT_WLOCK(locked_object);
	}

	static int
	lkpi_vma_pfn_track_object(struct lkpi_vma_pfn_state *state,
	vm_object_t object, bool have_reference,
	bool *reference_consumed)
	{
	struct lkpi_vma_pfn_object *entry;

	sx_assert(&state->populate_lock, SA_XLOCKED);
	*reference_consumed = false;
	mtx_lock(&state->objects_lock);
	TAILQ_FOREACH(entry, &state->objects, link) {
	if (entry->object == object) {
	mtx_unlock(&state->objects_lock);
	return (0);
	}
	}
	mtx_unlock(&state->objects_lock);
	if (!have_reference)
	return (ESTALE);
	entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
	if (entry == NULL) {
	return (ENOMEM);
	}
	entry->object = object;
	mtx_lock(&state->objects_lock);
	KASSERT(!lkpi_vma_pfn_object_is_tracked_locked(state, object),
	("%s: duplicate object %p", __func__, object));
	TAILQ_INSERT_TAIL(&state->objects, entry, link);
	mtx_unlock(&state->objects_lock);
	*reference_consumed = true;
	return (0);
	}

	/*
	* Publish the pin while holding the same lock that starts invalidation.
	* Reject a transaction spanning invalidation even when its pass has ended:
	* no new pin may appear behind the invalidator's completed traversal.
	*/
	static int
	lkpi_vma_pfn_pin_page(struct lkpi_vma_pfn_state *state, vm_page_t page)
	{
	struct lkpi_vma_pfn_pin key, *pin;

	MPASS(vm_page_xbusied(page));
	sx_assert(&state->populate_lock, SA_XLOCKED);
	key.m = page;
	mtx_lock(&state->objects_lock);
	if (state->invalidation_seq != state->populate_seq) {
	mtx_unlock(&state->objects_lock);
	return (ESTALE);
	}
	if (RB_FIND(lkpi_vma_pfn_pins, &state->pins, &key) != NULL) {
	mtx_unlock(&state->objects_lock);
	return (0);
	}
	mtx_unlock(&state->objects_lock);
	pin = kzalloc(sizeof(*pin), GFP_ATOMIC);
	if (pin == NULL)
	return (ENOMEM);
	pin->m = page;
	mtx_lock(&state->objects_lock);
	if (state->invalidation_seq != state->populate_seq) {
	mtx_unlock(&state->objects_lock);
	kfree(pin);
	return (ESTALE);
	}
	/* populate_lock serializes insertions; invalidation can only remove. */
	vm_page_wire(page);
	RB_INSERT(lkpi_vma_pfn_pins, &state->pins, pin);
	mtx_unlock(&state->objects_lock);
	return (0);
	}

	static int
	lkpi_vma_pfn_store_page(struct lkpi_vma_pfn_state *state,
	vm_pindex_t pindex, vm_page_t page)
	{
	struct lkpi_vma_pfn_chunk *chunk;

	MPASS(page != NULL);
	MPASS(vm_page_xbusied(page));
	sx_assert(&state->populate_lock, SA_XLOCKED);
	MPASS(pindex < state->npages);
	MPASS(state->pending < state->npages);
	chunk = state->last_chunk;
	if (chunk == NULL \|\| chunk->count == nitems(chunk->pages) \|\|
	pindex != chunk->first + chunk->count) {
	chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
	if (chunk == NULL)
	return (ENOMEM);
	chunk->first = pindex;
	TAILQ_INSERT_TAIL(&state->page_chunks, chunk, link);
	state->last_chunk = chunk;
	}
	MPASS(chunk->pages[chunk->count] == NULL);
	chunk->pages[chunk->count++] = page;
	chunk->remaining++;
	state->pending++;
	return (0);
	}

	static bool
	lkpi_vma_pfn_page_is_selected(struct vm_area_struct *vma, vm_page_t page)
	{
	struct lkpi_vma_pfn_chunk *chunk;
	struct lkpi_vma_pfn_state *state;
	unsigned int slot;

	VM_OBJECT_ASSERT_WLOCKED(vma->vm_obj);
	state = lkpi_vma_pfn_get_state(vma);
	MPASS(state != NULL);
	sx_assert(&state->populate_lock, SA_XLOCKED);
	TAILQ_FOREACH(chunk, &state->page_chunks, link) {
	for (slot = 0; slot < chunk->count; slot++) {
	if (chunk->pages[slot] == page)
	return (true);
	}
	}
	return (atomic_load_ptr(&page->object) == vma->vm_obj &&
	page->pindex >= vma->vm_pfn_first &&
	page->pindex - vma->vm_pfn_first <
	(vm_pindex_t)vma->vm_pfn_count);
	}

	/*
	* A preserved shmem page must regain its backing object's cache attribute
	* before the last protecting wire is released. Removing its PTEs alone does
	* not undo the direct-map attribute installed by the PFN fault. Leaving a
	* WC page on a default-attribute swap object violates the reclaim contract.
	*
	* The caller owns xbusy and retains the backing object. Read-only fast
	* faults can map an xbusy page, so its object must also be write locked
	* across the unmapped check and attribute change. Drop the caller's pager
	* lock, if different, rather than nesting VM object locks. The page's busy
	* ownership and the populate transaction survive that lock drop.
	*
	* Return whether the page was unmapped under that lock. In particular, a
	* caller must not drop its pin on the strength of an earlier unlocked check.
	*/
	static bool
	lkpi_vma_pfn_restore_memattr(vm_page_t page, vm_object_t locked_object)
	{
	vm_object_t object;
	vm_memattr_t memattr;
	bool relock, unmapped;

	MPASS(vm_page_xbusied(page));
	if (locked_object != NULL)
	VM_OBJECT_ASSERT_WLOCKED(locked_object);
	object = page->object;
	relock = object != NULL && object != locked_object;
	if (relock) {
	if (locked_object != NULL)
	VM_OBJECT_WUNLOCK(locked_object);
	VM_OBJECT_WLOCK(object);
	}
	unmapped = !pmap_page_is_mapped(page);
	if (unmapped && (page->oflags & VPO_UNMANAGED) == 0 &&
	(object == NULL \|\| object->type == OBJT_SWAP)) {
	memattr = object == NULL ? VM_MEMATTR_DEFAULT : object->memattr;
	if (pmap_page_get_memattr(page) != memattr)
	- pmap_page_set_memattr(page, memattr);
	+ vmmap_restore_page_memattr(page, memattr);
	}
	if (relock) {
	VM_OBJECT_WUNLOCK(object);
	if (locked_object != NULL)
	VM_OBJECT_WLOCK(locked_object);
	}
	return (unmapped);
	}

	/*
	* Complete both successful and discarded external-page handoffs. A live
	* mapping retains its pin. An unmapped page can be restored and released.
	* If invalidation has detached the pin already, that thread owns its wire.
	*/
	void
	lkpi_vma_pfn_release_page(struct vm_area_struct *vma, vm_page_t page)
	{
	struct lkpi_vma_pfn_pin key, *pin;
	struct lkpi_vma_pfn_state *state;

	MPASS(vm_page_xbusied(page));
	state = lkpi_vma_pfn_get_state(vma);
	MPASS(state != NULL);
	sx_assert(&state->populate_lock, SA_XLOCKED);
	pin = NULL;
	if (lkpi_vma_pfn_restore_memattr(page, vma->vm_obj)) {
	key.m = page;
	mtx_lock(&state->objects_lock);
	pin = RB_FIND(lkpi_vma_pfn_pins, &state->pins, &key);
	if (pin != NULL)
	RB_REMOVE(lkpi_vma_pfn_pins, &state->pins, pin);
	mtx_unlock(&state->objects_lock);
	}
	vm_page_xunbusy(page);
	if (pin != NULL) {
	/* The wire keeps even an objectless page alive until this drop. */
	vm_page_unwire(page, PQ_INACTIVE);
	kfree(pin);
	}
	}

	vm_page_t
	lkpi_vma_pfn_take_page(struct vm_area_struct *vma, vm_object_t object,
	vm_pindex_t pindex)
	{
	struct lkpi_vma_pfn_chunk *chunk;
	struct lkpi_vma_pfn_state *state;
	vm_page_t page;
	unsigned int slot;

	VM_OBJECT_ASSERT_WLOCKED(object);
	state = lkpi_vma_pfn_get_state(vma);
	if (state == NULL \|\| pindex >= state->npages)
	return (NULL);
	sx_assert(&state->populate_lock, SA_XLOCKED);
	TAILQ_FOREACH(chunk, &state->page_chunks, link) {
	if (pindex < chunk->first)
	break;
	if (pindex - chunk->first >= chunk->count)
	continue;
	slot = pindex - chunk->first;
	page = chunk->pages[slot];
	if (page == NULL)
	return (NULL);
	chunk->pages[slot] = NULL;
	MPASS(chunk->remaining > 0 && state->pending > 0);
	chunk->remaining--;
	state->pending--;
	if (chunk->remaining == 0) {
	TAILQ_REMOVE(&state->page_chunks, chunk, link);
	if (state->last_chunk == chunk)
	state->last_chunk = NULL;
	kfree(chunk);
	}
	return (page);
	}
	return (NULL);
	}

	void
	lkpi_vma_pfn_abort(struct vm_area_struct *vma, vm_object_t object)
	{
	struct lkpi_vma_pfn_state *state;
	vm_page_t page;
	vm_pindex_t count, pindex;

	VM_OBJECT_ASSERT_WLOCKED(object);
	state = lkpi_vma_pfn_get_state(vma);
	if (state != NULL)
	sx_assert(&state->populate_lock, SA_XLOCKED);
	count = vma->vm_pfn_count;
	for (pindex = vma->vm_pfn_first; count != 0;
	count--, pindex++) {
	page = lkpi_vma_pfn_take_page(vma, object, pindex);
	if (page == NULL)
	page = vm_page_lookup(object, pindex);
	if (page != NULL) {
	vm_page_deactivate(page);
	lkpi_vma_pfn_release_page(vma, page);
	}
	}
	vma->vm_pfn_count = 0;
	KASSERT(state == NULL \|\| state->pending == 0,
	("%s: %ju pages remain pending", __func__,
	state == NULL ? 0 : (uintmax_t)state->pending));
	}

	void
	lkpi_vma_pfn_done(struct vm_area_struct *vma, vm_object_t object)
	{
	struct lkpi_vma_pfn_state *state;

	VM_OBJECT_ASSERT_WLOCKED(object);
	state = lkpi_vma_pfn_get_state(vma);
	MPASS(state != NULL);
	sx_assert(&state->populate_lock, SA_XLOCKED);
	MPASS(state->pending == 0 &&
	TAILQ_EMPTY(&state->page_chunks));
	vma->vm_pfn_count = 0;
	}

	bool
	lkpi_vma_pfn_unmap_begin(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;

	state = lkpi_vma_pfn_get_state(vma);
	if (state == NULL)
	return (false);
	sx_xlock(&state->unmap_lock);
	return (true);
	}

	void
	lkpi_vma_pfn_unmap_end(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;

	state = lkpi_vma_pfn_get_state(vma);
	MPASS(state != NULL);
	sx_assert(&state->unmap_lock, SA_XLOCKED);
	sx_xunlock(&state->unmap_lock);
	}

	bool
	lkpi_vma_pfn_invalidate_begin(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;

	if (!lkpi_vma_pfn_unmap_begin(vma))
	return (false);
	state = lkpi_vma_pfn_get_state(vma);
	MPASS(state != NULL);
	mtx_lock(&state->objects_lock);
	MPASS((state->invalidation_seq & 1) == 0);
	state->invalidation_seq++;
	mtx_unlock(&state->objects_lock);
	return (true);
	}

	/*
	* Pins, not historical owner intervals, identify the pages to revoke. In
	* particular, the extra wire survives OBJ_DEAD making cdev_pager_lookup()
	* miss this pager while its destructor is still waiting to run.
	*/
	static void
	lkpi_vma_pfn_unmap_pins(struct lkpi_vma_pfn_state *state)
	{
	struct lkpi_vma_pfn_pin *pin;
	vm_object_t object;
	vm_page_t page;

	sx_assert(&state->unmap_lock, SA_XLOCKED);
	for (;;) {
	mtx_lock(&state->objects_lock);
	pin = RB_MIN(lkpi_vma_pfn_pins, &state->pins);
	if (pin != NULL)
	RB_REMOVE(lkpi_vma_pfn_pins, &state->pins, pin);
	mtx_unlock(&state->objects_lock);
	if (pin == NULL)
	break;
	page = pin->m;
	/* Our wire permits waiting without owning the page's object lock. */
	while (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL))
	continue;
	/* Exclude read-only fast faults until the attribute is restored. */
	object = page->object;
	if (object != NULL)
	VM_OBJECT_WLOCK(object);
	pmap_remove_all(page);
	lkpi_vma_pfn_restore_memattr(page, object);
	vm_page_xunbusy(page);
	if (object != NULL)
	VM_OBJECT_WUNLOCK(object);
	vm_page_unwire(page, PQ_INACTIVE);
	kfree(pin);
	}
	}

	void
	lkpi_vma_pfn_unmap(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;

	state = lkpi_vma_pfn_get_state(vma);
	if (state != NULL)
	lkpi_vma_pfn_unmap_pins(state);
	}

	void
	lkpi_vma_pfn_invalidate_end(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_state *state;

	state = lkpi_vma_pfn_get_state(vma);
	MPASS(state != NULL);
	sx_assert(&state->unmap_lock, SA_XLOCKED);
	mtx_lock(&state->objects_lock);
	MPASS((state->invalidation_seq & 1) != 0);
	state->invalidation_seq++;
	mtx_unlock(&state->objects_lock);
	lkpi_vma_pfn_unmap_end(vma);
	}

	void
	lkpi_vma_pfn_fini(struct vm_area_struct *vma)
	{
	struct lkpi_vma_pfn_chunk *chunk;
	struct lkpi_vma_pfn_object *entry;
	struct lkpi_vma_pfn_state *state;
	vm_page_t page;
	unsigned int slot;

	state = lkpi_vma_pfn_get_state(vma);
	if (state == NULL)
	return;
	sx_assert(&state->populate_lock, SA_UNLOCKED);
	mtx_lock(&state->objects_lock);
	MPASS((state->invalidation_seq & 1) == 0);
	mtx_unlock(&state->objects_lock);
	atomic_store_rel_ptr((volatile uintptr_t *)&vma->vm_pfn_state, 0);
	while ((chunk = TAILQ_FIRST(&state->page_chunks)) != NULL) {
	TAILQ_REMOVE(&state->page_chunks, chunk, link);
	for (slot = 0; slot < chunk->count; slot++) {
	page = chunk->pages[slot];
	if (page != NULL) {
	MPASS(state->pending > 0);
	state->pending--;
	lkpi_vma_pfn_restore_memattr(page, NULL);
	vm_page_deactivate(page);
	vm_page_xunbusy(page);
	}
	}
	kfree(chunk);
	}
	state->last_chunk = NULL;
	MPASS(state->pending == 0);
	sx_xlock(&state->unmap_lock);
	lkpi_vma_pfn_unmap_pins(state);
	for (;;) {
	mtx_lock(&state->objects_lock);
	entry = TAILQ_FIRST(&state->objects);
	if (entry != NULL)
	TAILQ_REMOVE(&state->objects, entry, link);
	mtx_unlock(&state->objects_lock);
	if (entry == NULL)
	break;
	vm_object_deallocate(entry->object);
	kfree(entry);
	}
	sx_xunlock(&state->unmap_lock);
	sx_destroy(&state->unmap_lock);
	sx_destroy(&state->populate_lock);
	mtx_destroy(&state->objects_lock);
	kfree(state);
	}

	vm_fault_t
	lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr,
	unsigned long pfn, pgprot_t prot)
	{
	struct lkpi_vma_pfn_state *state;
	struct pctrie_iter pages;
	vm_object_t vm_obj = vma->vm_obj;
	vm_object_t tmp_obj;
	vm_page_t page;
	vm_pindex_t pindex;
	vm_memattr_t memattr;
	bool have_reference, reference_consumed, tracked;
	int error;

	if (addr < vma->vm_start \|\| addr >= vma->vm_end)
	return (VM_FAULT_SIGBUS);

	VM_OBJECT_ASSERT_WLOCKED(vm_obj);
	if (offset_in_page(addr) != 0 \|\| (vm_pindex_t)pfn != pfn \|\|
	OFF_TO_IDX(IDX_TO_OFF((vm_pindex_t)pfn)) !=
	(vm_pindex_t)pfn)
	return (VM_FAULT_SIGBUS);
	state = lkpi_vma_pfn_get_state(vma);
	if (state == NULL)
	return (VM_FAULT_SIGBUS);
	if (state->error != 0)
	return (VM_FAULT_SIGBUS);
	/* Reused VMAs can grow; chunks are sparse, not a fixed-size array. */
	sx_assert(&state->populate_lock, SA_XLOCKED);
	state->npages = vma_pages(vma);
	if (vma->vm_pfn_count < 0 \|\| vma->vm_pfn_count == INT_MAX)
	return (lkpi_vma_pfn_fail(state, EINVAL));
	vm_page_iter_init(&pages, vm_obj);
	pindex = OFF_TO_IDX(addr - vma->vm_start);
	if (pindex >= state->npages)
	return (lkpi_vma_pfn_fail(state, EINVAL));
	if (vma->vm_pfn_count != 0 &&
	pindex != vma->vm_pfn_first + vma->vm_pfn_count)
	return (lkpi_vma_pfn_fail(state, EINVAL));
	if (vma->vm_pfn_count == 0) {
	vma->vm_pfn_first = pindex;
	}
	MPASS(pindex < OFF_TO_IDX(vma->vm_end));
	memattr = pgprot2cachemode(prot);

	retry:
	page = vm_page_grab_iter(vm_obj, pindex,
	VM_ALLOC_NOCREAT \| VM_ALLOC_NOWAIT, &pages);
	if (page == NULL) {
	if (vm_page_lookup(vm_obj, pindex) != NULL)
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	page = PHYS_TO_VM_PAGE(IDX_TO_OFF(pfn));
	if (page == NULL)
	return (lkpi_vma_pfn_fail(state, EINVAL));
	tmp_obj = atomic_load_ptr(&page->object);
	have_reference = false;
	tracked = tmp_obj != NULL && tmp_obj != vm_obj &&
	lkpi_vma_pfn_object_is_tracked(state, tmp_obj);
	if (tmp_obj != NULL && tmp_obj != vm_obj && !tracked) {
	/*
	* VM object locks are type-stable. Lock and revalidate the
	* source before taking the reference that will protect this VMA.
	*/
	VM_OBJECT_WUNLOCK(vm_obj);
	if (!VM_OBJECT_TRYWLOCK(tmp_obj)) {
	VM_OBJECT_WLOCK(vm_obj);
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	}
	if (page->object != tmp_obj \|\|
	(tmp_obj->flags & OBJ_DEAD) != 0) {
	VM_OBJECT_WUNLOCK(tmp_obj);
	VM_OBJECT_WLOCK(vm_obj);
	pctrie_iter_reset(&pages);
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	}
	vm_object_reference_locked(tmp_obj);
	VM_OBJECT_WUNLOCK(tmp_obj);
	VM_OBJECT_WLOCK(vm_obj);
	if (vm_page_lookup(vm_obj, pindex) != NULL \|\|
	atomic_load_ptr(&page->object) != tmp_obj) {
	lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	pctrie_iter_reset(&pages);
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	}
	have_reference = true;
	}
	if (!vm_page_tryxbusy(page)) {
	/*
	* A selected page stays xbusy until this transaction is
	* consumed or aborted. Refuse an alias rather than wait
	* for busy ownership that this transaction must release.
	*/
	if (lkpi_vma_pfn_page_is_selected(vma, page)) {
	if (have_reference)
	lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	return (lkpi_vma_pfn_fail(state, EINVAL));
	}
	if (have_reference)
	lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	/* Drop the whole batch before waiting for another owner. */
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	}
	if (page->object != tmp_obj) {
	vm_page_xunbusy(page);
	if (have_reference)
	lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	pctrie_iter_reset(&pages);
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	}
	/*
	* Linux installs a special PFN PTE without moving a managed page
	* out of its backing object. Preserve that ownership for shmem
	* pages and hand the xbusy page directly to vm_fault_populate().
	*/
	if (tmp_obj != NULL && tmp_obj != vm_obj &&
	tmp_obj->type == OBJT_SWAP &&
	(page->oflags & VPO_UNMANAGED) == 0) {
	/* The driver must pin non-default mappings until invalidation. */
	if (!vm_page_all_valid(page) \|\|
	(memattr != tmp_obj->memattr && !vm_page_wired(page)) \|\|
	(pmap_page_get_memattr(page) != memattr &&
	pmap_page_is_mapped(page))) {
	vm_page_xunbusy(page);
	if (have_reference)
	lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	return (lkpi_vma_pfn_fail(state, EINVAL));
	}
	error = lkpi_vma_pfn_track_object(state, tmp_obj,
	have_reference, &reference_consumed);
	if (error != 0 \|\| (have_reference && !reference_consumed)) {
	vm_page_xunbusy(page);
	if (have_reference)
	lkpi_vma_pfn_drop_object_ref(vm_obj, tmp_obj);
	return (lkpi_vma_pfn_fail(state,
	error == ENOMEM ? ENOMEM : EAGAIN));
	}
	/*
	* Publish our pin before changing the cache attribute. A
	* read-only backing-object fault can create an alias even
	* while this page is xbusy. If a later allocation fails,
	* that alias may prevent restoration and must retain a pin.
	* Keep xbusy across the object-lock switch so invalidation
	* cannot consume the pin and then miss our attribute change.
	*/
	error = lkpi_vma_pfn_pin_page(state, page);
	if (error != 0) {
	vm_page_xunbusy(page);
	return (lkpi_vma_pfn_fail(state,
	error == ESTALE ? EAGAIN : error));
	}
	if (pmap_page_get_memattr(page) != memattr) {
	VM_OBJECT_WUNLOCK(vm_obj);
	if (!VM_OBJECT_TRYWLOCK(tmp_obj)) {
	VM_OBJECT_WLOCK(vm_obj);
	lkpi_vma_pfn_release_page(vma, page);
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	}
	MPASS(page->object == tmp_obj);
	if (pmap_page_get_memattr(page) != memattr &&
	pmap_page_is_mapped(page)) {
	VM_OBJECT_WUNLOCK(tmp_obj);
	VM_OBJECT_WLOCK(vm_obj);
	lkpi_vma_pfn_release_page(vma, page);
	return (lkpi_vma_pfn_fail(state, EINVAL));
	}
	if (pmap_page_get_memattr(page) != memattr)
	pmap_page_set_memattr(page, memattr);
	VM_OBJECT_WUNLOCK(tmp_obj);
	VM_OBJECT_WLOCK(vm_obj);
	if (vm_page_lookup(vm_obj, pindex) != NULL) {
	lkpi_vma_pfn_release_page(vma, page);
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	}
	}
	error = lkpi_vma_pfn_store_page(state, pindex, page);
	if (error != 0) {
	lkpi_vma_pfn_release_page(vma, page);
	return (lkpi_vma_pfn_fail(state, error));
	}
	vma->vm_pfn_count++;
	return (VM_FAULT_NOPAGE);
	}
	if (page->object != NULL) {
	if (tracked) {
	vm_page_xunbusy(page);
	return (lkpi_vma_pfn_fail(state, EINVAL));
	}
	if (tmp_obj == vm_obj) {
	vm_object_reference_locked(tmp_obj);
	have_reference = true;
	}
	MPASS(have_reference);
	vm_page_xunbusy(page);
	VM_OBJECT_WUNLOCK(vm_obj);
	if (!VM_OBJECT_TRYWLOCK(tmp_obj)) {
	vm_object_deallocate(tmp_obj);
	VM_OBJECT_WLOCK(vm_obj);
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	}
	if (page->object == tmp_obj && vm_page_tryxbusy(page)) {
	KASSERT(page->object == tmp_obj,
	("page has changed identity"));
	if ((page->oflags & VPO_UNMANAGED) != 0 \|\|
	!vm_page_wired(page)) {
	vm_page_xunbusy(page);
	VM_OBJECT_WUNLOCK(tmp_obj);
	vm_object_deallocate(tmp_obj);
	VM_OBJECT_WLOCK(vm_obj);
	return (lkpi_vma_pfn_fail(state, EINVAL));
	}
	if (pmap_page_is_mapped(page)) {
	vm_page_xunbusy(page);
	VM_OBJECT_WUNLOCK(tmp_obj);
	vm_object_deallocate(tmp_obj);
	VM_OBJECT_WLOCK(vm_obj);
	return (lkpi_vma_pfn_fail(state, EINVAL));
	}
	vm_pager_page_unswapped(page);
	vm_page_remove(page);
	} else {
	VM_OBJECT_WUNLOCK(tmp_obj);
	vm_object_deallocate(tmp_obj);
	VM_OBJECT_WLOCK(vm_obj);
	return (lkpi_vma_pfn_fail(state, EAGAIN));
	}
	VM_OBJECT_WUNLOCK(tmp_obj);
	vm_object_deallocate(tmp_obj);
	pctrie_iter_reset(&pages);
	VM_OBJECT_WLOCK(vm_obj);
	goto retry;
	}
	if (vm_page_iter_insert(page, vm_obj, pindex, &pages) != 0) {
	vm_page_xunbusy(page);
	return (lkpi_vma_pfn_fail(state, ENOMEM));
	}
	vm_page_valid(page);
	}
	if (!vm_page_all_valid(page) \|\|
	(page->oflags & VPO_UNMANAGED) != 0 \|\|
	VM_PAGE_TO_PHYS(page) != IDX_TO_OFF(pfn)) {
	vm_page_xunbusy(page);
	return (lkpi_vma_pfn_fail(state, EINVAL));
	}
	if (pmap_page_get_memattr(page) != memattr &&
	pmap_page_is_mapped(page)) {
	vm_page_xunbusy(page);
	return (lkpi_vma_pfn_fail(state, EINVAL));
	}
	pmap_page_set_memattr(page, memattr);
	vma->vm_pfn_count++;

	return (VM_FAULT_NOPAGE);
	}

	int
	lkpi_remap_pfn_range(struct vm_area_struct *vma, unsigned long start_addr,
	unsigned long start_pfn, unsigned long size, pgprot_t prot)
	{
	vm_object_t vm_obj;
	unsigned long addr, end_addr, npages, pfn;
	int err = 0;

	vm_obj = vma->vm_obj;
	if (size == 0)
	return (0);
	if (offset_in_page(start_addr) != 0 \|\| offset_in_page(size) != 0 \|\|
	start_addr < vma->vm_start \|\| start_addr >= vma->vm_end \|\|
	size > vma->vm_end - start_addr)
	return (-EINVAL);
	npages = size >> PAGE_SHIFT;
	if ((vm_pindex_t)start_pfn != start_pfn \|\|
	npages - 1 > ULONG_MAX - start_pfn)
	return (-EINVAL);
	end_addr = start_addr + size;

	VM_OBJECT_WLOCK(vm_obj);
	for (addr = start_addr, pfn = start_pfn;
	addr != end_addr;
	addr += PAGE_SIZE) {
	vm_fault_t ret;
	ret = lkpi_vmf_insert_pfn_prot_locked(vma, addr, pfn, prot);

	if ((ret & VM_FAULT_OOM) != 0) {
	err = -ENOMEM;
	break;
	}

	if ((ret & VM_FAULT_ERROR) != 0) {
	err = -EFAULT;
	break;
	}

	pfn++;
	}
	VM_OBJECT_WUNLOCK(vm_obj);

	if (unlikely(err)) {
	zap_vma_ptes(vma, start_addr, addr - start_addr);
	return (err);
	}

	return (0);
	}

	int
	lkpi_io_mapping_map_user(struct io_mapping *iomap,
	struct vm_area_struct *vma, unsigned long addr,
	unsigned long pfn, unsigned long size)
	{
	pgprot_t prot;
	int ret;

	prot = cachemode2protval(iomap->attr);
	ret = lkpi_remap_pfn_range(vma, addr, pfn, size, prot);

	return (ret);
	}

	/*
	* Although FreeBSD version of unmap_mapping_range has semantics and types of
	* parameters compatible with Linux version, the values passed in are different
	* @obj should match to vm_private_data field of vm_area_struct returned by
	* mmap file operation handler, see linux_file_mmap_single() sources
	* @holelen should match to size of area to be munmapped.
	*/
	void
	lkpi_unmap_mapping_range(void *obj, loff_t const holebegin __unused,
	loff_t const holelen __unused, int even_cows __unused)
	{
	vm_object_t devobj;

	devobj = cdev_pager_lookup(obj);
	if (devobj != NULL) {
	linux_cdev_pager_free_pages(devobj);
	vm_object_deallocate(devobj);
	}
	}

	int
	lkpi_arch_phys_wc_add(unsigned long base, unsigned long size)
	{
	#ifdef __i386__
	struct mem_range_desc *mrdesc;
	int error, id, act;

	/* If PAT is available, do nothing */
	if (pat_works)
	return (0);

	mrdesc = malloc(sizeof(*mrdesc), M_LKMTRR, M_WAITOK);
	mrdesc->mr_base = base;
	mrdesc->mr_len = size;
	mrdesc->mr_flags = MDF_WRITECOMBINE;
	strlcpy(mrdesc->mr_owner, "drm", sizeof(mrdesc->mr_owner));
	act = MEMRANGE_SET_UPDATE;
	error = mem_range_attr_set(mrdesc, &act);
	if (error == 0) {
	error = idr_get_new(&mtrr_idr, mrdesc, &id);
	MPASS(idr_find(&mtrr_idr, id) == mrdesc);
	if (error != 0) {
	act = MEMRANGE_SET_REMOVE;
	mem_range_attr_set(mrdesc, &act);
	}
	}
	if (error != 0) {
	free(mrdesc, M_LKMTRR);
	pr_warn(
	"Failed to add WC MTRR for [%p-%p]: %d; "
	"performance may suffer\n",
	(void )base, (void )(base + size - 1), error);
	} else
	pr_warn("Successfully added WC MTRR for [%p-%p]\n",
	(void )base, (void )(base + size - 1));

	return (error != 0 ? -error : id + __MTRR_ID_BASE);
	#else
	return (0);
	#endif
	}

	void
	lkpi_arch_phys_wc_del(int reg)
	{
	#ifdef __i386__
	struct mem_range_desc *mrdesc;
	int act;

	/* Check if arch_phys_wc_add() failed. */
	if (reg < __MTRR_ID_BASE)
	return;

	mrdesc = idr_find(&mtrr_idr, reg - __MTRR_ID_BASE);
	MPASS(mrdesc != NULL);
	idr_remove(&mtrr_idr, reg - __MTRR_ID_BASE);
	act = MEMRANGE_SET_REMOVE;
	mem_range_attr_set(mrdesc, &act);
	free(mrdesc, M_LKMTRR);
	#endif
	}

	int
	lkpi_set_pages_attr(struct page *page, int numpages, vm_memattr_t ma)
	{
	while (numpages-- > 0) {
	/*
	* pmap_page_set_memattr() would only update the DMAP mapping
	* if it's a normal page, leaving the kernel map untouched.
	*/
	MPASS(page->object != kernel_object);

	/*
	* pmap_page_set_memattr() sets page->md.pat_mode, which is
	* crucial for future userspace mappings.
	*/
	pmap_page_set_memattr(page, ma);
	page++;
	}

	return (0);
	}

	/*
	* This is a highly simplified version of the Linux page_frag_cache.
	* We only support up-to 1 single page as fragment size and we will
	* always return a full page. This may be wasteful on small objects
	* but the only known consumer (mt76) is either asking for a half-page
	* or a full page. If this was to become a problem we can implement
	* a more elaborate version.
	*/
	void *
	linuxkpi_page_frag_alloc(struct page_frag_cache *pfc,
	size_t fragsz, gfp_t gfp)
	{
	struct page *pages;

	if (fragsz == 0)
	return (NULL);

	KASSERT(fragsz <= PAGE_SIZE, ("%s: fragsz %zu > PAGE_SIZE not yet "
	"supported", __func__, fragsz));

	pages = alloc_pages(gfp, flsl(howmany(fragsz, PAGE_SIZE) - 1));
	if (pages == NULL)
	return (NULL);
	pfc->va = linux_page_address(pages);

	/* Passed in as "count" to __page_frag_cache_drain(). Unused by us. */
	pfc->pagecnt_bias = 0;

	return (pfc->va);
	}

	void
	linuxkpi_page_frag_free(void *addr)
	{
	struct page *page;

	page = virt_to_page(addr);
	linux_free_pages(page, 0);
	}

	void
	linuxkpi__page_frag_cache_drain(struct page *page, size_t count __unused)
	{

	linux_free_pages(page, 0);
	}

	static void
	lkpi_page_init(void *arg)
	{
	int i;

	mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF);
	for (i = 0; i < VMMAP_HASH_SIZE; i++)
	LIST_INIT(&vmmaphead[i]);
	}
	SYSINIT(lkpi_page, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_page_init, NULL);

	static void
	lkpi_page_uninit(void *arg)
	{
	mtx_destroy(&vmmaplock);
	}
	SYSUNINIT(lkpi_page, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_page_uninit, NULL);

File Metadata

Mime Type: text/x-diff
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 39953355
Default Alt Text: linuxkpi-vmap-D59481-coordination-full-context.patch (49 KB)

linuxkpi-vmap-D59481-coordination-full-context.patchPublicActions

linuxkpi-vmap-D59481-coordination-full-context.patchView Options

File Metadata

Event Timeline

linuxkpi-vmap-D59481-coordination-full-context.patch
Public
Actions

linuxkpi-vmap-D59481-coordination-full-context.patch
View Options