diff --git a/sys/compat/linuxkpi/common/include/linux/highmem.h b/sys/compat/linuxkpi/common/include/linux/highmem.h --- a/sys/compat/linuxkpi/common/include/linux/highmem.h +++ b/sys/compat/linuxkpi/common/include/linux/highmem.h @@ -65,7 +65,11 @@ return (PHYS_TO_DMAP(page_to_phys(page))); } else { sched_pin(); +#ifdef PAGE_IS_LKPI_PAGE + sf = sf_buf_alloc(page->vm_page, SFB_NOWAIT | SFB_CPUPRIVATE); +#else sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); +#endif if (sf == NULL) { sched_unpin(); return (NULL); @@ -80,8 +84,13 @@ vm_memattr_t attr = pgprot2cachemode(prot); if (attr != VM_MEMATTR_DEFAULT) { +#ifdef PAGE_IS_LKPI_PAGE + page->vm_page->flags |= PG_FICTITIOUS; + pmap_page_set_memattr(page->vm_page, attr); +#else page->flags |= PG_FICTITIOUS; pmap_page_set_memattr(page, attr); +#endif } return (kmap(page)); } @@ -126,7 +135,11 @@ if (!PMAP_HAS_DMAP) { /* lookup SF buffer in list */ +#ifdef PAGE_IS_LKPI_PAGE + sf = sf_buf_alloc(page->vm_page, SFB_NOWAIT | SFB_CPUPRIVATE); +#else sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); +#endif /* double-free */ sf_buf_free(sf); diff --git a/sys/compat/linuxkpi/common/include/linux/mm.h b/sys/compat/linuxkpi/common/include/linux/mm.h --- a/sys/compat/linuxkpi/common/include/linux/mm.h +++ b/sys/compat/linuxkpi/common/include/linux/mm.h @@ -269,19 +269,31 @@ static inline void set_page_dirty(struct page *page) { +#ifdef PAGE_IS_LKPI_PAGE + vm_page_dirty(page->vm_page); +#else vm_page_dirty(page); +#endif } static inline void mark_page_accessed(struct page *page) { +#ifdef PAGE_IS_LKPI_PAGE + vm_page_reference(page->vm_page); +#else vm_page_reference(page); +#endif } static inline void get_page(struct page *page) { +#ifdef PAGE_IS_LKPI_PAGE + vm_page_wire(page->vm_page); +#else vm_page_wire(page); +#endif } static inline void @@ -375,7 +387,15 @@ #define unpin_user_page(page) put_page(page) #define unpin_user_pages(pages, npages) release_pages(pages, npages) +#ifdef PAGE_IS_LKPI_PAGE +static inline void +copy_highpage(struct page *to, struct page *from) +{ + pmap_copy_page(from->vm_page, to->vm_page); +} +#else #define copy_highpage(to, from) pmap_copy_page(from, to) +#endif static inline pgprot_t vm_get_page_prot(unsigned long vm_flags) @@ -398,23 +418,37 @@ static inline struct page * vmalloc_to_page(const void *addr) { + struct vm_page *m; vm_paddr_t paddr; paddr = pmap_kextract((vm_offset_t)addr); - return (PHYS_TO_VM_PAGE(paddr)); + m = PHYS_TO_VM_PAGE(paddr); +#ifdef PAGE_IS_LKPI_PAGE + return (lkpi_vm_page_to_page(m)); +#else + return (m); +#endif } static inline int trylock_page(struct page *page) { +#ifdef PAGE_IS_LKPI_PAGE + return (vm_page_tryxbusy(page->vm_page)); +#else return (vm_page_tryxbusy(page)); +#endif } static inline void unlock_page(struct page *page) { +#ifdef PAGE_IS_LKPI_PAGE + vm_page_xunbusy(page->vm_page); +#else vm_page_xunbusy(page); +#endif } extern int is_vmalloc_addr(const void *addr); diff --git a/sys/compat/linuxkpi/common/include/linux/page.h b/sys/compat/linuxkpi/common/include/linux/page.h --- a/sys/compat/linuxkpi/common/include/linux/page.h +++ b/sys/compat/linuxkpi/common/include/linux/page.h @@ -48,7 +48,24 @@ typedef unsigned long linux_pgd_t; typedef unsigned long pgprot_t; +#ifdef __not_yet__ +#define PAGE_IS_LKPI_PAGE +#endif +#ifdef PAGE_IS_LKPI_PAGE +struct page { + /* Native FreeBSD vm_page; SHOULD stay first. */ + struct vm_page *vm_page; + + /* Linux fields. */ + struct page_pool *pp; + dma_addr_t dma_addr; + + uintptr_t private; /* drm-kmod ttm-pool */ + struct list_head lru; +}; +#else #define page vm_page +#endif #define LINUXKPI_PROT_VALID (1 << 3) #define LINUXKPI_CACHE_MODE_SHIFT 4 @@ -72,10 +89,25 @@ return (VM_MEMATTR_DEFAULT); } +#ifdef PAGE_IS_LKPI_PAGE +/* Internal */ +void lkpi_page_set_vm_page(struct page *, struct vm_page *); +struct page *lkpi_vm_page_to_page(struct vm_page *); + +/* Public */ +struct page *linuxkpi_virt_to_page(const void *); +struct page *linuxkpi_pfn_to_page(unsigned long); + +#define virt_to_page(_v) linuxkpi_virt_to_page(_v) +#define page_to_pfn(_page) atop(VM_PAGE_TO_PHYS((_page)->vm_page)) +#define pfn_to_page(_pfn) linuxkpi_pfn_to_page(_pfn) +#define page_to_phys(_page) VM_PAGE_TO_PHYS((_page)->vm_page) +#else /* !PAGE_IS_LKPI_PAGE */ #define virt_to_page(x) PHYS_TO_VM_PAGE(vtophys(x)) #define page_to_pfn(pp) atop(VM_PAGE_TO_PHYS(pp)) #define pfn_to_page(pfn) PHYS_TO_VM_PAGE(ptoa(pfn)) #define page_to_phys(page) VM_PAGE_TO_PHYS(page) +#endif /* PAGE_IS_LKPI_PAGE */ #define page_to_virt(page) linux_page_address(page) #define nth_page(page,n) pfn_to_page(page_to_pfn(page) + (n)) diff --git a/sys/compat/linuxkpi/common/include/linux/scatterlist.h b/sys/compat/linuxkpi/common/include/linux/scatterlist.h --- a/sys/compat/linuxkpi/common/include/linux/scatterlist.h +++ b/sys/compat/linuxkpi/common/include/linux/scatterlist.h @@ -602,7 +602,11 @@ len = min(piter.sg->length - skip, buflen); page = sg_page_iter_page(&piter); +#ifdef PAGE_IS_LKPI_PAGE + sf = sf_buf_alloc(page->vm_page, SFB_CPUPRIVATE | SFB_NOWAIT); +#else sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); +#endif if (sf == NULL) break; p = (char *)sf_buf_kva(sf) + piter.sg_pgoffset + skip; @@ -657,7 +661,11 @@ page = sg_page_iter_page(&iter); if (!PMAP_HAS_DMAP) { +#ifdef PAGE_IS_LKPI_PAGE + sf = sf_buf_alloc(page->vm_page, SFB_CPUPRIVATE | SFB_NOWAIT); +#else sf = sf_buf_alloc(page, SFB_CPUPRIVATE | SFB_NOWAIT); +#endif if (sf == NULL) break; vaddr = sf_buf_kva(sf); diff --git a/sys/compat/linuxkpi/common/src/linux_page.c b/sys/compat/linuxkpi/common/src/linux_page.c --- a/sys/compat/linuxkpi/common/src/linux_page.c +++ b/sys/compat/linuxkpi/common/src/linux_page.c @@ -82,16 +82,47 @@ si->mem_unit = PAGE_SIZE; } +#ifdef PAGE_IS_LKPI_PAGE +void +lkpi_page_set_vm_page(struct page *page, struct vm_page *m) +{ + + KASSERT(page->vm_page == NULL || page->vm_page == m, + ("%s: page %p page->vm_page %p != NULL/vm_page %p; " + "pp %p, dma_addr %#jx, private %#jx, lru %sempty\n", + __func__, page, page->vm_page, m, + page->pp, (uintmax_t)page->dma_addr, (uintmax_t)page->private, + list_empty(&page->lru) ? "" : "not ")); + + page->vm_page = m; +} + +static void +lkpi_page_unset_vm_page(struct page *page) +{ + KASSERT(page->vm_page != NULL, ("%s: page %p page->vm_page %p is NULL\n", + __func__, page, page->vm_page)); + + memset(page, 0, sizeof(*page)); +} +#endif + void * linux_page_address(const struct page *page) { + const struct vm_page *m; - if (page->object != kernel_object) { +#ifdef PAGE_IS_LKPI_PAGE + m = page->vm_page; +#else + m = page; +#endif + if (m->object != kernel_object) { return (PMAP_HAS_DMAP ? PHYS_TO_DMAP(page_to_phys(page)) : NULL); } return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS + - IDX_TO_OFF(page->pindex))); + IDX_TO_OFF(m->pindex))); } struct page * @@ -100,15 +131,19 @@ struct page *page; if (PMAP_HAS_DMAP) { + struct vm_page *m; unsigned long npages = 1UL << order; +#ifdef PAGE_IS_LKPI_PAGE + unsigned long x; +#endif int req = VM_ALLOC_WIRED; if ((flags & M_ZERO) != 0) req |= VM_ALLOC_ZERO; if (order == 0 && (flags & GFP_DMA32) == 0) { - page = vm_page_alloc_noobj(req); - if (page == NULL) + m = vm_page_alloc_noobj(req); + if (m == NULL) return (NULL); } else { vm_paddr_t pmax = (flags & GFP_DMA32) ? @@ -120,16 +155,16 @@ retry: if ((flags & __GFP_THISNODE) != 0) { int curdomain = PCPU_GET(domain); - page = vm_page_alloc_noobj_contig_domain( + m = vm_page_alloc_noobj_contig_domain( curdomain, req, npages, 0, pmax, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); } else { - page = vm_page_alloc_noobj_contig( + m = vm_page_alloc_noobj_contig( req, npages, 0, pmax, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); } - if (page == NULL) { + if (m == NULL) { if ((flags & (M_WAITOK | __GFP_NORETRY | __GFP_THISNODE)) == M_WAITOK) { int err = vm_page_reclaim_contig(req, @@ -144,6 +179,13 @@ return (NULL); } } +#ifdef PAGE_IS_LKPI_PAGE + page = lkpi_vm_page_to_page(m); + for (x = 0; x < npages; x++) + lkpi_page_set_vm_page(page + x, m + x); +#else + page = m; +#endif } else { vm_offset_t vaddr; @@ -164,6 +206,15 @@ _linux_free_kmem(vm_offset_t addr, unsigned int order) { size_t size = ((size_t)PAGE_SIZE) << order; +#ifdef PAGE_IS_LKPI_PAGE + unsigned long npages = 1UL << order; + unsigned long x; + struct page *page; + + page = virt_to_page((void *)addr); + for (x = 0; x != npages; x++) + lkpi_page_unset_vm_page(page + x); +#endif kmem_free((void *)addr, size); } @@ -172,11 +223,17 @@ linux_free_pages(struct page *page, unsigned int order) { if (PMAP_HAS_DMAP) { + struct vm_page *m; unsigned long npages = 1UL << order; unsigned long x; +#ifdef PAGE_IS_LKPI_PAGE + m = page->vm_page; +#else + m = page; +#endif for (x = 0; x != npages; x++) { - vm_page_t pgo = page + x; + vm_page_t pgo = m + x; /* * The "free page" function is used in several @@ -192,16 +249,23 @@ */ if ((pgo->oflags & VPO_UNMANAGED) == 0) { vm_page_unwire(pgo, PQ_ACTIVE); +#ifdef PAGE_IS_LKPI_PAGE + if (!vm_page_wired(pgo)) + lkpi_page_unset_vm_page(page + x); +#endif } else { - if (vm_page_unwire_noq(pgo)) + if (vm_page_unwire_noq(pgo)) { +#ifdef PAGE_IS_LKPI_PAGE + lkpi_page_unset_vm_page(page + x); +#endif vm_page_free(pgo); + } } } } else { vm_offset_t vaddr; vaddr = (vm_offset_t)page_address(page); - _linux_free_kmem(vaddr, order); } } @@ -220,6 +284,11 @@ vm_offset_t linux_alloc_kmem(gfp_t flags, unsigned int order) { +#ifdef PAGE_IS_LKPI_PAGE + struct page *page; + struct vm_page *m; + unsigned long x; +#endif size_t size = ((size_t)PAGE_SIZE) << order; void *addr; @@ -227,6 +296,20 @@ ((flags & GFP_DMA32) == 0) ? -1UL : BUS_SPACE_MAXADDR_32BIT, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); +#ifdef PAGE_IS_LKPI_PAGE + if (addr == 0) + return ((vm_offset_t)addr); + + page = virt_to_page((void *)addr); + m = PHYS_TO_VM_PAGE(vtophys(addr)); + x = 1UL << order; + while (x-- > 0) + lkpi_page_set_vm_page(page + x, m + x); + + KASSERT(addr == page_address(page), + ("Page address mismatch")); +#endif + return ((vm_offset_t)addr); } @@ -239,9 +322,15 @@ if (addr >= VM_MIN_KERNEL_ADDRESS && addr < VM_MAX_KERNEL_ADDRESS) { _linux_free_kmem(addr, order); } else { - vm_page_t page; + struct page *page; + vm_page_t mp; - page = DMAP_TO_VM_PAGE(addr); + mp = DMAP_TO_VM_PAGE(addr); +#ifdef PAGE_IS_LKPI_PAGE + page = lkpi_vm_page_to_page(mp); +#else + page = mp; +#endif linux_free_pages(page, order); } } @@ -250,13 +339,26 @@ linux_get_user_pages_internal(vm_map_t map, unsigned long start, int nr_pages, int write, struct page **pages) { + struct vm_page **mpp; vm_prot_t prot; size_t len; - int count; + int count, i; + + mpp = malloc(sizeof(*mpp) * nr_pages, M_KMALLOC, M_NOWAIT|M_ZERO); + if (mpp == NULL) + return (-ENOMEM); + + for (i = 0; i < nr_pages; i++) +#ifdef PAGE_IS_LKPI_PAGE + mpp[i] = pages[i]->vm_page; +#else + mpp[i] = pages[i]; +#endif prot = write ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ; len = ptoa((vm_offset_t)nr_pages); - count = vm_fault_quick_hold_pages(map, start, len, prot, pages, nr_pages); + count = vm_fault_quick_hold_pages(map, start, len, prot, mpp, nr_pages); + free(mpp, M_KMALLOC); return (count == -1 ? -EFAULT : nr_pages); } @@ -264,12 +366,13 @@ __get_user_pages_fast(unsigned long start, int nr_pages, int write, struct page **pages) { + struct vm_page **mpp; vm_map_t map; vm_page_t *mp; vm_offset_t va; vm_offset_t end; vm_prot_t prot; - int count; + int count, i; if (nr_pages == 0 || in_interrupt()) return (0); @@ -280,7 +383,19 @@ if (!vm_map_range_valid(map, start, end)) return (-EINVAL); prot = write ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ; - for (count = 0, mp = pages, va = start; va < end; + + mpp = malloc(sizeof(*mpp) * nr_pages, M_KMALLOC, M_NOWAIT|M_ZERO); + if (mpp == NULL) + return (-ENOMEM); + + for (i = 0; i < nr_pages; i++) +#ifdef PAGE_IS_LKPI_PAGE + mpp[i] = pages[i]->vm_page; +#else + mpp[i] = pages[i]; +#endif + mp = mpp; + for (count = 0, va = start; va < end; mp++, va += PAGE_SIZE, count++) { *mp = pmap_extract_and_hold(map->pmap, va, prot); if (*mp == NULL) @@ -300,6 +415,8 @@ vm_page_dirty(*mp); } } + free(mpp, M_KMALLOC); + return (count); } @@ -423,15 +540,33 @@ void * vmap(struct page **pages, unsigned int count, unsigned long flags, int prot) { + struct vm_page **mpp; void *off; size_t size; + int i; + + mpp = malloc(sizeof(*mpp) * count, M_KMALLOC, M_NOWAIT|M_ZERO); + if (mpp == NULL) + return (NULL); size = count * PAGE_SIZE; off = kva_alloc(size); - if (off == NULL) + if (off == NULL) { + free(mpp, M_KMALLOC); return (NULL); + } + + for (i = 0; i < count; i++) +#ifdef PAGE_IS_LKPI_PAGE + mpp[i] = pages[i]->vm_page; +#else + mpp[i] = pages[i]; +#endif + vmmap_add(off, size); - pmap_qenter(off, pages, count); + pmap_qenter(off, mpp, count); + + free(mpp, M_KMALLOC); return (off); } @@ -713,18 +848,26 @@ int lkpi_set_pages_attr(struct page *page, int numpages, vm_memattr_t ma) { + struct vm_page *m; + while (numpages-- > 0) { +#ifdef PAGE_IS_LKPI_PAGE + m = page->vm_page; +#else + m = page; +#endif /* * pmap_page_set_memattr() would only update the DMAP mapping * if it's a normal page, leaving the kernel map untouched. */ - MPASS(page->object != kernel_object); + MPASS(m->object != kernel_object); /* * pmap_page_set_memattr() sets page->md.pat_mode, which is * crucial for future userspace mappings. */ - pmap_page_set_memattr(page, ma); + pmap_page_set_memattr(m, ma); + page++; } @@ -778,20 +921,116 @@ linux_free_pages(page, 0); } +#ifdef PAGE_IS_LKPI_PAGE +/* -------------------------------------------------------------------------- */ +/* + * Mimic vm_page_array[] or rather vm_phys_segs[] and similar logic dependent on + * PMAP_HAS_PAGE_ARRAY and VM_PHYSSEG_SPARSE or VM_PHYSSEG_DENSE for struct page + * essentially creating a shadow structure to keep LinuxKPI specific information + * along native pages. + * Note: we currently just allocate a full set of vm_phys_nsegs copies ignoring + * any sparseness or denseness. + */ + +static MALLOC_DEFINE(M_LKPI_PAGE, "lkpipage", "LinuxKPI page array"); +static struct page __read_mostly *lkpi_phy_segs[VM_PHYSSEG_MAX]; + +struct page * +lkpi_vm_page_to_page(struct vm_page *m) +{ + struct vm_phys_seg *seg; + vm_paddr_t pa; + struct page *p; + + MPASS(m != NULL); + + seg = &vm_phys_segs[m->segind]; + pa = VM_PAGE_TO_PHYS(m); + if (pa < seg->start) + return (NULL); + p = &lkpi_phy_segs[m->segind][atop(pa - seg->start)]; + return (p); +} + +struct page * +linuxkpi_virt_to_page(const void *v) +{ + struct vm_page *m; + + m = PHYS_TO_VM_PAGE(vtophys(v)); + return (lkpi_vm_page_to_page(m)); +} + +/* pfn (page-frame number) to page. */ +struct page * +linuxkpi_pfn_to_page(unsigned long pfn) +{ + struct vm_page *m; + + m = PHYS_TO_VM_PAGE(ptoa(pfn)); + return (lkpi_vm_page_to_page(m)); +} + +/* -------------------------------------------------------------------------- */ +#endif + static void -lkpi_page_init(void *arg) +lkpi_page_init(void *arg __unused) { int i; mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF); for (i = 0; i < VMMAP_HASH_SIZE; i++) LIST_INIT(&vmmaphead[i]); + +#ifdef PAGE_IS_LKPI_PAGE + for (i = 0; i < vm_phys_nsegs; i++) { + struct vm_phys_seg *seg; + size_t n; + + seg = &vm_phys_segs[i]; + n = atop(seg->end - seg->start); + + lkpi_phy_segs[i] = malloc(n * sizeof(struct page), + M_LKPI_PAGE, M_WAITOK | M_ZERO); + } +#endif } -SYSINIT(lkpi_page, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_page_init, NULL); +SYSINIT(lkpi_page, SI_SUB_VM_CONF+1, SI_ORDER_ANY, lkpi_page_init, NULL); static void lkpi_page_uninit(void *arg) { mtx_destroy(&vmmaplock); + +#ifdef PAGE_IS_LKPI_PAGE + size_t leaked; + int i; + + leaked = 0; + for (i = 0; i < vm_phys_nsegs; i++) { + struct vm_phys_seg *seg; + struct page *page, *p; + size_t n, l; + + l = 0; + seg = &vm_phys_segs[i]; + n = atop(seg->end - seg->start); + + page = lkpi_phy_segs[i]; + for (size_t j = 0; j < n; j++) { + p = page + j; + if (p->vm_page != NULL) + l++; + } + if (l > 0) + printf("%s: segment %d leaked %zu vm_pages\n", __func__, i, l); + leaked += l; + + free(lkpi_phy_segs[i], M_LKPI_PAGE); + } + if (leaked > 0) + printf("%s: total leaked vm_pages: %zu\n", __func__, leaked); +#endif } -SYSUNINIT(lkpi_page, SI_SUB_DRIVERS, SI_ORDER_SECOND, lkpi_page_uninit, NULL); +SYSUNINIT(lkpi_page, SI_SUB_VM_CONF+1, SI_ORDER_ANY, lkpi_page_uninit, NULL); diff --git a/sys/compat/linuxkpi/common/src/linux_shmemfs.c b/sys/compat/linuxkpi/common/src/linux_shmemfs.c --- a/sys/compat/linuxkpi/common/src/linux_shmemfs.c +++ b/sys/compat/linuxkpi/common/src/linux_shmemfs.c @@ -44,6 +44,7 @@ struct page * linux_shmem_read_mapping_page_gfp(vm_object_t obj, int pindex, gfp_t gfp) { + struct vm_page *m; struct page *page; int rv; @@ -51,11 +52,17 @@ panic("GFP_NOWAIT is unimplemented"); VM_OBJECT_WLOCK(obj); - rv = vm_page_grab_valid(&page, obj, pindex, VM_ALLOC_NORMAL | + rv = vm_page_grab_valid(&m, obj, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); VM_OBJECT_WUNLOCK(obj); if (rv != VM_PAGER_OK) return (ERR_PTR(-EINVAL)); +#ifdef PAGE_IS_LKPI_PAGE + page = lkpi_vm_page_to_page(m); + lkpi_page_set_vm_page(page, m); +#else + page = m; +#endif return (page); } diff --git a/sys/dev/iser/iser_initiator.c b/sys/dev/iser/iser_initiator.c --- a/sys/dev/iser/iser_initiator.c +++ b/sys/dev/iser/iser_initiator.c @@ -304,6 +304,7 @@ int i; size_t len, tlen; int offset; + struct page *page; tlen = bp->bio_bcount; offset = bp->bio_ma_offset; @@ -311,7 +312,18 @@ for (i = 0; 0 < tlen; i++, tlen -= len) { sg = &data_buf->sgl[i]; len = min(PAGE_SIZE - offset, tlen); - sg_set_page(sg, bp->bio_ma[i], len, offset); + /* + * This is a historic LinuxKPI accident from the times when + * Linux 'struct page' was aliased to 'struct vm_page'. + * The sg_* interface was never meant to take a + * FreeBSD native vm page *. + */ +#ifdef PAGE_IS_LKPI_PAGE + page = lkpi_vm_page_to_page(bp->bio_ma[i]); +#else + page = bp->bio_ma[i]; +#endif + sg_set_page(sg, page, len, offset); offset = 0; }