Index: head/sys/compat/linuxkpi/common/include/linux/mm_types.h =================================================================== --- head/sys/compat/linuxkpi/common/include/linux/mm_types.h (revision 319317) +++ head/sys/compat/linuxkpi/common/include/linux/mm_types.h (revision 319318) @@ -1,71 +1,69 @@ /*- * Copyright (c) 2017 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _LINUX_MM_TYPES_H_ #define _LINUX_MM_TYPES_H_ #include #include #include #include #include struct vm_area_struct; struct task_struct; -struct vmspace; struct mm_struct { struct vm_area_struct *mmap; - struct vmspace *vmspace; atomic_t mm_count; atomic_t mm_users; size_t pinned_vm; struct rw_semaphore mmap_sem; }; extern void linux_mm_dtor(struct mm_struct *mm); static inline void mmdrop(struct mm_struct *mm) { if (__predict_false(atomic_dec_and_test(&mm->mm_count))) linux_mm_dtor(mm); } static inline void mmput(struct mm_struct *mm) { if (__predict_false(atomic_dec_and_test(&mm->mm_users))) mmdrop(mm); } extern struct mm_struct *linux_get_task_mm(struct task_struct *); #define get_task_mm(task) linux_get_task_mm(task) #endif /* _LINUX_MM_TYPES_H_ */ Index: head/sys/compat/linuxkpi/common/src/linux_current.c =================================================================== --- head/sys/compat/linuxkpi/common/src/linux_current.c (revision 319317) +++ head/sys/compat/linuxkpi/common/src/linux_current.c (revision 319318) @@ -1,231 +1,228 @@ /*- * Copyright (c) 2017 Hans Petter Selasky * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include static eventhandler_tag linuxkpi_thread_dtor_tag; static MALLOC_DEFINE(M_LINUX_CURRENT, "linuxcurrent", "LinuxKPI task structure"); int linux_alloc_current(struct thread *td, int flags) { struct proc *proc; struct thread *td_other; struct task_struct *ts; struct task_struct *ts_other; struct mm_struct *mm; struct mm_struct *mm_other; MPASS(td->td_lkpi_task == NULL); ts = malloc(sizeof(*ts), M_LINUX_CURRENT, flags | M_ZERO); if (ts == NULL) return (ENOMEM); mm = malloc(sizeof(*mm), M_LINUX_CURRENT, flags | M_ZERO); if (mm == NULL) { free(ts, M_LINUX_CURRENT); return (ENOMEM); } /* setup new task structure */ atomic_set(&ts->kthread_flags, 0); ts->task_thread = td; ts->comm = td->td_name; ts->pid = td->td_tid; atomic_set(&ts->usage, 1); ts->state = TASK_RUNNING; proc = td->td_proc; /* check if another thread already has a mm_struct */ PROC_LOCK(proc); FOREACH_THREAD_IN_PROC(proc, td_other) { ts_other = td_other->td_lkpi_task; if (ts_other == NULL) continue; mm_other = ts_other->mm; if (mm_other == NULL) continue; /* try to share other mm_struct */ if (atomic_inc_not_zero(&mm_other->mm_users)) { /* set mm_struct pointer */ ts->mm = mm_other; break; } } /* use allocated mm_struct as a fallback */ if (ts->mm == NULL) { /* setup new mm_struct */ init_rwsem(&mm->mmap_sem); atomic_set(&mm->mm_count, 1); atomic_set(&mm->mm_users, 1); - mm->vmspace = vmspace_acquire_ref(proc); /* set mm_struct pointer */ ts->mm = mm; /* clear pointer to not free memory */ mm = NULL; } /* store pointer to task struct */ td->td_lkpi_task = ts; PROC_UNLOCK(proc); /* free mm_struct pointer, if any */ free(mm, M_LINUX_CURRENT); return (0); } struct mm_struct * linux_get_task_mm(struct task_struct *task) { struct mm_struct *mm; mm = task->mm; - if (mm != NULL && mm->vmspace != NULL) { + if (mm != NULL) { atomic_inc(&mm->mm_users); return (mm); } return (NULL); } void linux_mm_dtor(struct mm_struct *mm) { - if (mm->vmspace != NULL) - vmspace_free(mm->vmspace); free(mm, M_LINUX_CURRENT); } void linux_free_current(struct task_struct *ts) { mmput(ts->mm); free(ts, M_LINUX_CURRENT); } static void linuxkpi_thread_dtor(void *arg __unused, struct thread *td) { struct task_struct *ts; ts = td->td_lkpi_task; if (ts == NULL) return; td->td_lkpi_task = NULL; put_task_struct(ts); } struct task_struct * linux_pid_task(pid_t pid) { struct thread *td; struct proc *p; /* try to find corresponding thread */ td = tdfind(pid, -1); if (td != NULL) { struct task_struct *ts = td->td_lkpi_task; PROC_UNLOCK(td->td_proc); return (ts); } /* try to find corresponding procedure */ p = pfind(pid); if (p != NULL) { FOREACH_THREAD_IN_PROC(p, td) { struct task_struct *ts = td->td_lkpi_task; if (ts != NULL) { PROC_UNLOCK(p); return (ts); } } PROC_UNLOCK(p); } return (NULL); } struct task_struct * linux_get_pid_task(pid_t pid) { struct thread *td; struct proc *p; /* try to find corresponding thread */ td = tdfind(pid, -1); if (td != NULL) { struct task_struct *ts = td->td_lkpi_task; if (ts != NULL) get_task_struct(ts); PROC_UNLOCK(td->td_proc); return (ts); } /* try to find corresponding procedure */ p = pfind(pid); if (p != NULL) { FOREACH_THREAD_IN_PROC(p, td) { struct task_struct *ts = td->td_lkpi_task; if (ts != NULL) { get_task_struct(ts); PROC_UNLOCK(p); return (ts); } } PROC_UNLOCK(p); } return (NULL); } static void linux_current_init(void *arg __unused) { linuxkpi_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, linuxkpi_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); } SYSINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, linux_current_init, NULL); static void linux_current_uninit(void *arg __unused) { EVENTHANDLER_DEREGISTER(thread_dtor, linuxkpi_thread_dtor_tag); } SYSUNINIT(linux_current, SI_SUB_EVENTHANDLER, SI_ORDER_SECOND, linux_current_uninit, NULL); Index: head/sys/compat/linuxkpi/common/src/linux_page.c =================================================================== --- head/sys/compat/linuxkpi/common/src/linux_page.c (revision 319317) +++ head/sys/compat/linuxkpi/common/src/linux_page.c (revision 319318) @@ -1,291 +1,291 @@ /*- * Copyright (c) 2010 Isilon Systems, Inc. * Copyright (c) 2016 Matt Macy (mmacy@nextbsd.org) * Copyright (c) 2017 Mellanox Technologies, Ltd. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__amd64__) || defined(__aarch64__) || defined(__riscv__) #define LINUXKPI_HAVE_DMAP #else #undef LINUXKPI_HAVE_DMAP #endif void * linux_page_address(struct page *page) { if (page->object != kmem_object && page->object != kernel_object) { #ifdef LINUXKPI_HAVE_DMAP return ((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(page))); #else return (NULL); #endif } return ((void *)(uintptr_t)(VM_MIN_KERNEL_ADDRESS + IDX_TO_OFF(page->pindex))); } vm_page_t linux_alloc_pages(gfp_t flags, unsigned int order) { #ifdef LINUXKPI_HAVE_DMAP unsigned long npages = 1UL << order; int req = (flags & M_ZERO) ? (VM_ALLOC_ZERO | VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL) : (VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL); vm_page_t page; if (order == 0 && (flags & GFP_DMA32) == 0) { page = vm_page_alloc(NULL, 0, req); if (page == NULL) return (NULL); } else { vm_paddr_t pmax = (flags & GFP_DMA32) ? BUS_SPACE_MAXADDR_32BIT : BUS_SPACE_MAXADDR; retry: page = vm_page_alloc_contig(NULL, 0, req, npages, 0, pmax, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); if (page == NULL) { if (flags & M_WAITOK) { if (!vm_page_reclaim_contig(req, npages, 0, pmax, PAGE_SIZE, 0)) { VM_WAIT; } flags &= ~M_WAITOK; goto retry; } return (NULL); } } if (flags & M_ZERO) { unsigned long x; for (x = 0; x != npages; x++) { vm_page_t pgo = page + x; if ((pgo->flags & PG_ZERO) == 0) pmap_zero_page(pgo); } } #else vm_offset_t vaddr; vm_page_t page; vaddr = linux_alloc_kmem(flags, order); if (vaddr == 0) return (NULL); page = PHYS_TO_VM_PAGE(vtophys((void *)vaddr)); KASSERT(vaddr == (vm_offset_t)page_address(page), ("Page address mismatch")); #endif return (page); } void linux_free_pages(vm_page_t page, unsigned int order) { #ifdef LINUXKPI_HAVE_DMAP unsigned long npages = 1UL << order; unsigned long x; for (x = 0; x != npages; x++) { vm_page_t pgo = page + x; vm_page_lock(pgo); vm_page_free(pgo); vm_page_unlock(pgo); } #else vm_offset_t vaddr; vaddr = (vm_offset_t)page_address(page); linux_free_kmem(vaddr, order); #endif } vm_offset_t linux_alloc_kmem(gfp_t flags, unsigned int order) { size_t size = ((size_t)PAGE_SIZE) << order; vm_offset_t addr; if ((flags & GFP_DMA32) == 0) { addr = kmem_malloc(kmem_arena, size, flags & GFP_NATIVE_MASK); } else { addr = kmem_alloc_contig(kmem_arena, size, flags & GFP_NATIVE_MASK, 0, BUS_SPACE_MAXADDR_32BIT, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); } return (addr); } void linux_free_kmem(vm_offset_t addr, unsigned int order) { size_t size = ((size_t)PAGE_SIZE) << order; kmem_free(kmem_arena, addr, size); } static int linux_get_user_pages_internal(vm_map_t map, unsigned long start, int nr_pages, int write, struct page **pages) { vm_prot_t prot; size_t len; int count; int i; prot = write ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ; len = ((size_t)nr_pages) << PAGE_SHIFT; count = vm_fault_quick_hold_pages(map, start, len, prot, pages, nr_pages); if (count == -1) return (-EFAULT); for (i = 0; i != nr_pages; i++) { struct page *pg = pages[i]; vm_page_lock(pg); vm_page_wire(pg); vm_page_unlock(pg); } return (nr_pages); } int __get_user_pages_fast(unsigned long start, int nr_pages, int write, struct page **pages) { vm_map_t map; vm_page_t *mp; vm_offset_t va; vm_offset_t end; vm_prot_t prot; int count; if (nr_pages == 0 || in_interrupt()) return (0); MPASS(pages != NULL); va = start; map = &curthread->td_proc->p_vmspace->vm_map; end = start + (((size_t)nr_pages) << PAGE_SHIFT); if (start < vm_map_min(map) || end > vm_map_max(map)) return (-EINVAL); prot = write ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ; for (count = 0, mp = pages, va = start; va < end; mp++, va += PAGE_SIZE, count++) { *mp = pmap_extract_and_hold(map->pmap, va, prot); if (*mp == NULL) break; vm_page_lock(*mp); vm_page_wire(*mp); vm_page_unlock(*mp); if ((prot & VM_PROT_WRITE) != 0 && (*mp)->dirty != VM_PAGE_BITS_ALL) { /* * Explicitly dirty the physical page. Otherwise, the * caller's changes may go unnoticed because they are * performed through an unmanaged mapping or by a DMA * operation. * * The object lock is not held here. * See vm_page_clear_dirty_mask(). */ vm_page_dirty(*mp); } } return (count); } long get_user_pages_remote(struct task_struct *task, struct mm_struct *mm, unsigned long start, unsigned long nr_pages, int gup_flags, struct page **pages, struct vm_area_struct **vmas) { vm_map_t map; - map = &mm->vmspace->vm_map; + map = &task->task_thread->td_proc->p_vmspace->vm_map; return (linux_get_user_pages_internal(map, start, nr_pages, !!(gup_flags & FOLL_WRITE), pages)); } long get_user_pages(unsigned long start, unsigned long nr_pages, int gup_flags, struct page **pages, struct vm_area_struct **vmas) { vm_map_t map; map = &curthread->td_proc->p_vmspace->vm_map; return (linux_get_user_pages_internal(map, start, nr_pages, !!(gup_flags & FOLL_WRITE), pages)); } int is_vmalloc_addr(const void *addr) { return (vtoslab((vm_offset_t)addr & ~UMA_SLAB_MASK) != NULL); }