Index: sys/amd64/amd64/pmap.c =================================================================== --- sys/amd64/amd64/pmap.c +++ sys/amd64/amd64/pmap.c @@ -8356,10 +8356,12 @@ } /* - * perform the pmap work for mincore + * Perform the pmap work for mincore(2). If the page is not both referenced and + * modified by this pmap, returns its physical address so that the caller can + * find other mappings. */ int -pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) +pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) { pd_entry_t *pdep; pt_entry_t pte, PG_A, PG_M, PG_RW, PG_V; @@ -8372,7 +8374,6 @@ PG_RW = pmap_rw_bit(pmap); PMAP_LOCK(pmap); -retry: pdep = pmap_pde(pmap, addr); if (pdep != NULL && (*pdep & PG_V)) { if (*pdep & PG_PS) { @@ -8401,11 +8402,8 @@ if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { - /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ - if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) - goto retry; - } else - PA_UNLOCK_COND(*locked_pa); + *pap = pa; + } PMAP_UNLOCK(pmap); return (val); } Index: sys/arm/arm/pmap-v4.c =================================================================== --- sys/arm/arm/pmap-v4.c +++ sys/arm/arm/pmap-v4.c @@ -4149,10 +4149,12 @@ /* - * perform the pmap work for mincore + * Perform the pmap work for mincore(2). If the page is not both referenced and + * modified by this pmap, returns its physical address so that the caller can + * find other mappings. */ int -pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) +pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) { struct l2_bucket *l2b; pt_entry_t *ptep, pte; @@ -4162,17 +4164,16 @@ boolean_t managed; PMAP_LOCK(pmap); -retry: l2b = pmap_get_l2_bucket(pmap, addr); if (l2b == NULL) { - val = 0; - goto out; + PMAP_UNLOCK(pmap); + return (0); } ptep = &l2b->l2b_kva[l2pte_index(addr)]; pte = *ptep; if (!l2pte_valid(pte)) { - val = 0; - goto out; + PMAP_UNLOCK(pmap); + return (0); } val = MINCORE_INCORE; if (pte & L2_S_PROT_W) @@ -4199,12 +4200,8 @@ } if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { - /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ - if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) - goto retry; - } else -out: - PA_UNLOCK_COND(*locked_pa); + *pap = pa; + } PMAP_UNLOCK(pmap); return (val); } Index: sys/arm/arm/pmap-v6.c =================================================================== --- sys/arm/arm/pmap-v6.c +++ sys/arm/arm/pmap-v6.c @@ -6233,10 +6233,12 @@ } /* - * Perform the pmap work for mincore. + * Perform the pmap work for mincore(2). If the page is not both referenced and + * modified by this pmap, returns its physical address so that the caller can + * find other mappings. */ int -pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) +pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) { pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; @@ -6245,7 +6247,6 @@ int val; PMAP_LOCK(pmap); -retry: pte1p = pmap_pte1(pmap, addr); pte1 = pte1_load(pte1p); if (pte1_is_section(pte1)) { @@ -6273,11 +6274,8 @@ } if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { - /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ - if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) - goto retry; - } else - PA_UNLOCK_COND(*locked_pa); + *pap = pa; + } PMAP_UNLOCK(pmap); return (val); } Index: sys/arm64/arm64/pmap.c =================================================================== --- sys/arm64/arm64/pmap.c +++ sys/arm64/arm64/pmap.c @@ -5624,10 +5624,12 @@ } /* - * perform the pmap work for mincore + * Perform the pmap work for mincore(2). If the page is not both referenced and + * modified by this pmap, returns its physical address so that the caller can + * find other mappings. */ int -pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) +pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) { pt_entry_t *pte, tpte; vm_paddr_t mask, pa; @@ -5635,8 +5637,6 @@ bool managed; PMAP_LOCK(pmap); -retry: - val = 0; pte = pmap_pte(pmap, addr, &lvl); if (pte != NULL) { tpte = pmap_load(pte); @@ -5666,18 +5666,16 @@ val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; pa = (tpte & ~ATTR_MASK) | (addr & mask); - } else + } else { managed = false; + val = 0; + } if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { - /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ - if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) - goto retry; - } else - PA_UNLOCK_COND(*locked_pa); + *pap = pa; + } PMAP_UNLOCK(pmap); - return (val); } Index: sys/i386/i386/pmap.c =================================================================== --- sys/i386/i386/pmap.c +++ sys/i386/i386/pmap.c @@ -5715,10 +5715,12 @@ } /* - * perform the pmap work for mincore + * Perform the pmap work for mincore(2). If the page is not both referenced and + * modified by this pmap, returns its physical address so that the caller can + * find other mappings. */ static int -__CONCAT(PMTYPE, mincore)(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) +__CONCAT(PMTYPE, mincore)(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) { pd_entry_t pde; pt_entry_t pte; @@ -5726,7 +5728,6 @@ int val; PMAP_LOCK(pmap); -retry: pde = *pmap_pde(pmap, addr); if (pde != 0) { if ((pde & PG_PS) != 0) { @@ -5755,11 +5756,8 @@ if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { - /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ - if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) - goto retry; - } else - PA_UNLOCK_COND(*locked_pa); + *pap = pa; + } PMAP_UNLOCK(pmap); return (val); } Index: sys/i386/i386/pmap_base.c =================================================================== --- sys/i386/i386/pmap_base.c +++ sys/i386/i386/pmap_base.c @@ -632,10 +632,10 @@ } int -pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) +pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) { - return (pmap_methods_ptr->pm_mincore(pmap, addr, locked_pa)); + return (pmap_methods_ptr->pm_mincore(pmap, addr, pap)); } void Index: sys/kern/kern_proc.c =================================================================== --- sys/kern/kern_proc.c +++ sys/kern/kern_proc.c @@ -2363,7 +2363,7 @@ vm_object_t obj, tobj; vm_page_t m, m_adv; vm_offset_t addr; - vm_paddr_t locked_pa; + vm_paddr_t pa; vm_pindex_t pi, pi_adv, pindex; *super = false; @@ -2371,7 +2371,7 @@ if (vmmap_skip_res_cnt) return; - locked_pa = 0; + pa = 0; obj = entry->object.vm_object; addr = entry->start; m_adv = NULL; @@ -2401,8 +2401,7 @@ m_adv = NULL; if (m->psind != 0 && addr + pagesizes[1] <= entry->end && (addr & (pagesizes[1] - 1)) == 0 && - (pmap_mincore(map->pmap, addr, &locked_pa) & - MINCORE_SUPER) != 0) { + (pmap_mincore(map->pmap, addr, &pa) & MINCORE_SUPER) != 0) { *super = true; pi_adv = atop(pagesizes[1]); } else { @@ -2418,7 +2417,6 @@ *resident_count += pi_adv; next:; } - PA_UNLOCK_COND(locked_pa); } /* Index: sys/mips/mips/pmap.c =================================================================== --- sys/mips/mips/pmap.c +++ sys/mips/mips/pmap.c @@ -3206,10 +3206,12 @@ } /* - * perform the pmap work for mincore + * Perform the pmap work for mincore(2). If the page is not both referenced and + * modified by this pmap, returns its physical address so that the caller can + * find other mappings. */ int -pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) +pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) { pt_entry_t *ptep, pte; vm_paddr_t pa; @@ -3217,12 +3219,11 @@ int val; PMAP_LOCK(pmap); -retry: ptep = pmap_pte(pmap, addr); pte = (ptep != NULL) ? *ptep : 0; if (!pte_test(&pte, PTE_V)) { - val = 0; - goto out; + PMAP_UNLOCK(pmap); + return (0); } val = MINCORE_INCORE; if (pte_test(&pte, PTE_D)) @@ -3242,12 +3243,8 @@ if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && pte_test(&pte, PTE_MANAGED)) { - /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ - if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) - goto retry; - } else -out: - PA_UNLOCK_COND(*locked_pa); + *pap = pa; + } PMAP_UNLOCK(pmap); return (val); } Index: sys/powerpc/booke/pmap.c =================================================================== --- sys/powerpc/booke/pmap.c +++ sys/powerpc/booke/pmap.c @@ -3574,7 +3574,7 @@ */ static int mmu_booke_mincore(mmu_t mmu, pmap_t pmap, vm_offset_t addr, - vm_paddr_t *locked_pa) + vm_paddr_t *pap) { /* XXX: this should be implemented at some point */ Index: sys/powerpc/powerpc/mmu_if.m =================================================================== --- sys/powerpc/powerpc/mmu_if.m +++ sys/powerpc/powerpc/mmu_if.m @@ -91,7 +91,7 @@ } static int mmu_null_mincore(mmu_t mmu, pmap_t pmap, vm_offset_t addr, - vm_paddr_t *locked_pa) + vm_paddr_t *pap) { return (0); } @@ -663,7 +663,7 @@ * * @param _pmap physical map * @param _addr page virtual address - * @param _locked_pa page physical address + * @param _pa page physical address * * @retval 0 no result * @retval non-zero mincore(2) flag values @@ -672,7 +672,7 @@ mmu_t _mmu; pmap_t _pmap; vm_offset_t _addr; - vm_paddr_t *_locked_pa; + vm_paddr_t *_pap; } DEFAULT mmu_null_mincore; Index: sys/powerpc/powerpc/pmap_dispatch.c =================================================================== --- sys/powerpc/powerpc/pmap_dispatch.c +++ sys/powerpc/powerpc/pmap_dispatch.c @@ -381,11 +381,11 @@ } int -pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) +pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) { CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); - return (MMU_MINCORE(mmu_obj, pmap, addr, locked_pa)); + return (MMU_MINCORE(mmu_obj, pmap, addr, pap)); } void Index: sys/riscv/riscv/pmap.c =================================================================== --- sys/riscv/riscv/pmap.c +++ sys/riscv/riscv/pmap.c @@ -4226,10 +4226,12 @@ } /* - * perform the pmap work for mincore + * Perform the pmap work for mincore(2). If the page is not both referenced and + * modified by this pmap, returns its physical address so that the caller can + * find other mappings. */ int -pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) +pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) { pt_entry_t *l2, *l3, tpte; vm_paddr_t pa; @@ -4237,10 +4239,6 @@ bool managed; PMAP_LOCK(pmap); -retry: - managed = false; - val = 0; - l2 = pmap_l2(pmap, addr); if (l2 != NULL && ((tpte = pmap_load(l2)) & PTE_V) != 0) { if ((tpte & PTE_RWX) != 0) { @@ -4249,8 +4247,10 @@ } else { l3 = pmap_l2_to_l3(l2, addr); tpte = pmap_load(l3); - if ((tpte & PTE_V) == 0) - goto done; + if ((tpte & PTE_V) == 0) { + PMAP_UNLOCK(pmap); + return (0); + } pa = PTE_TO_PHYS(tpte) | (addr & L3_OFFSET); val = MINCORE_INCORE; } @@ -4260,16 +4260,14 @@ if ((tpte & PTE_A) != 0) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; managed = (tpte & PTE_SW_MANAGED) == PTE_SW_MANAGED; + } else { + managed = false; + val = 0; } - -done: if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { - /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ - if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) - goto retry; - } else - PA_UNLOCK_COND(*locked_pa); + *pap = pa; + } PMAP_UNLOCK(pmap); return (val); } Index: sys/sparc64/sparc64/pmap.c =================================================================== --- sys/sparc64/sparc64/pmap.c +++ sys/sparc64/sparc64/pmap.c @@ -2249,7 +2249,7 @@ } int -pmap_mincore(pmap_t pm, vm_offset_t addr, vm_paddr_t *locked_pa) +pmap_mincore(pmap_t pm, vm_offset_t addr, vm_paddr_t *pap) { /* TODO; */ Index: sys/vm/pmap.h =================================================================== --- sys/vm/pmap.h +++ sys/vm/pmap.h @@ -144,8 +144,7 @@ boolean_t pmap_is_referenced(vm_page_t m); boolean_t pmap_is_valid_memattr(pmap_t, vm_memattr_t); vm_offset_t pmap_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int); -int pmap_mincore(pmap_t pmap, vm_offset_t addr, - vm_paddr_t *locked_pa); +int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap); void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size); boolean_t pmap_page_exists_quick(pmap_t pmap, vm_page_t m); Index: sys/vm/vm_mmap.c =================================================================== --- sys/vm/vm_mmap.c +++ sys/vm/vm_mmap.c @@ -779,28 +779,23 @@ int kern_mincore(struct thread *td, uintptr_t addr0, size_t len, char *vec) { - vm_offset_t addr, first_addr; - vm_offset_t end, cend; pmap_t pmap; vm_map_t map; - int error = 0; - int vecindex, lastvecindex; - vm_map_entry_t current; - vm_map_entry_t entry; + vm_map_entry_t current, entry; vm_object_t object; - vm_paddr_t locked_pa; + vm_offset_t addr, cend, end, first_addr; + vm_paddr_t pa; vm_page_t m; vm_pindex_t pindex; - int mincoreinfo; + int error, lastvecindex, mincoreinfo, vecindex; unsigned int timestamp; - boolean_t locked; /* * Make sure that the addresses presented are valid for user * mode. */ first_addr = addr = trunc_page(addr0); - end = addr + (vm_size_t)round_page(len); + end = round_page(addr0 + len); map = &td->td_proc->p_vmspace->vm_map; if (end > vm_map_max(map) || end < addr) return (ENOMEM); @@ -836,7 +831,7 @@ * ignore submaps (for now) or null objects */ if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) || - current->object.vm_object == NULL) + current->object.vm_object == NULL) continue; /* @@ -849,50 +844,45 @@ if (cend > end) cend = end; - /* - * scan this entry one page at a time - */ - while (addr < cend) { + for (; addr < cend; addr += PAGE_SIZE) { /* * Check pmap first, it is likely faster, also * it can provide info as to whether we are the * one referencing or modifying the page. */ - object = NULL; - locked_pa = 0; - retry: m = NULL; - mincoreinfo = pmap_mincore(pmap, addr, &locked_pa); + object = NULL; +retry: + pa = 0; + mincoreinfo = pmap_mincore(pmap, addr, &pa); if (mincore_mapped) { /* * We only care about this pmap's * mapping of the page, if any. */ - if (locked_pa != 0) { - vm_page_unlock(PHYS_TO_VM_PAGE( - locked_pa)); - } - } else if (locked_pa != 0) { + ; + } else if (pa != 0) { /* * The page is mapped by this process but not * both accessed and modified. It is also * managed. Acquire the object lock so that - * other mappings might be examined. + * other mappings might be examined. The page's + * identity may change at any point before its + * object lock is acquired, so re-validate if + * necessary. */ - m = PHYS_TO_VM_PAGE(locked_pa); - if (m->object != object) { + m = PHYS_TO_VM_PAGE(pa); + while (object == NULL || m->object != object) { if (object != NULL) VM_OBJECT_WUNLOCK(object); - object = m->object; - locked = VM_OBJECT_TRYWLOCK(object); - vm_page_unlock(m); - if (!locked) { - VM_OBJECT_WLOCK(object); - vm_page_lock(m); + object = (vm_object_t)atomic_load_ptr( + &m->object); + if (object == NULL) goto retry; - } - } else - vm_page_unlock(m); + VM_OBJECT_WLOCK(object); + } + if (pa != pmap_extract(pmap, addr)) + goto retry; KASSERT(m->valid == VM_PAGE_BITS_ALL, ("mincore: page %p is mapped but invalid", m)); @@ -922,11 +912,14 @@ } } if (m != NULL) { - /* Examine other mappings to the page. */ + VM_OBJECT_ASSERT_WLOCKED(m->object); + + /* Examine other mappings of the page. */ if (m->dirty == 0 && pmap_is_modified(m)) vm_page_dirty(m); if (m->dirty != 0) mincoreinfo |= MINCORE_MODIFIED_OTHER; + /* * The first test for PGA_REFERENCED is an * optimization. The second test is @@ -985,7 +978,6 @@ goto RestartScan; lastvecindex = vecindex; - addr += PAGE_SIZE; } } Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h +++ sys/vm/vm_page.h @@ -581,7 +581,6 @@ void vm_page_launder(vm_page_t m); vm_page_t vm_page_lookup (vm_object_t, vm_pindex_t); vm_page_t vm_page_next(vm_page_t m); -int vm_page_pa_tryrelock(pmap_t, vm_paddr_t, vm_paddr_t *); void vm_page_pqbatch_drain(void); void vm_page_pqbatch_submit(vm_page_t m, uint8_t queue); vm_page_t vm_page_prev(vm_page_t m); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -168,10 +168,6 @@ &boot_pages, 0, "number of pages allocated for bootstrapping the VM system"); -static int pa_tryrelock_restart; -SYSCTL_INT(_vm, OID_AUTO, tryrelock_restart, CTLFLAG_RD, - &pa_tryrelock_restart, 0, "Number of tryrelock restarts"); - static TAILQ_HEAD(, vm_page) blacklist_head; static int sysctl_vm_page_blacklist(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vm, OID_AUTO, page_blacklist, CTLTYPE_STRING | CTLFLAG_RD | @@ -250,34 +246,6 @@ #endif #endif -/* - * Try to acquire a physical address lock while a pmap is locked. If we - * fail to trylock we unlock and lock the pmap directly and cache the - * locked pa in *locked. The caller should then restart their loop in case - * the virtual to physical mapping has changed. - */ -int -vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) -{ - vm_paddr_t lockpa; - - lockpa = *locked; - *locked = pa; - if (lockpa) { - PA_LOCK_ASSERT(lockpa, MA_OWNED); - if (PA_LOCKPTR(pa) == PA_LOCKPTR(lockpa)) - return (0); - PA_UNLOCK(lockpa); - } - if (PA_TRYLOCK(pa)) - return (0); - PMAP_UNLOCK(pmap); - atomic_add_int(&pa_tryrelock_restart, 1); - PA_LOCK(pa); - PMAP_LOCK(pmap); - return (EAGAIN); -} - /* * vm_set_page_size: *