Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
- This file was added.
Property | Old Value | New Value |
---|---|---|
svn:eol-style | null | native \ No newline at end of property |
svn:keywords | null | FreeBSD=%H \ No newline at end of property |
svn:mime-type | null | text/plain \ No newline at end of property |
/*- | |||||
* Copyright (c) 1991 Regents of the University of California. | |||||
* All rights reserved. | |||||
* Copyright (c) 1994 John S. Dyson | |||||
* All rights reserved. | |||||
* Copyright (c) 1994 David Greenman | |||||
* All rights reserved. | |||||
* Copyright (c) 2003 Peter Wemm | |||||
* All rights reserved. | |||||
* Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu> | |||||
* All rights reserved. | |||||
* Copyright (c) 2014 Andrew Turner | |||||
* All rights reserved. | |||||
* Copyright (c) 2014 The FreeBSD Foundation | |||||
* All rights reserved. | |||||
* | |||||
emaste: What is left from other archs? | |||||
Not Done Inline ActionsQuite a lot. Most of my changes are due to the differences in pagetable format, the logic is still similar. andrew: Quite a lot. Most of my changes are due to the differences in pagetable format, the logic is… | |||||
* This code is derived from software contributed to Berkeley by | |||||
* the Systems Programming Group of the University of Utah Computer | |||||
* Science Department and William Jolitz of UUNET Technologies Inc. | |||||
* | |||||
* This software was developed by Andrew Turner under sponsorship from | |||||
* the FreeBSD Foundation. | |||||
emasteUnsubmitted Not Done Inline ActionsIf applied to existing code should be the "Portions of" text. emaste: If applied to existing code should be the "Portions of" text. | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | |||||
* modification, are permitted provided that the following conditions | |||||
* are met: | |||||
* 1. Redistributions of source code must retain the above copyright | |||||
* notice, this list of conditions and the following disclaimer. | |||||
* 2. Redistributions in binary form must reproduce the above copyright | |||||
* notice, this list of conditions and the following disclaimer in the | |||||
* documentation and/or other materials provided with the distribution. | |||||
* 3. All advertising materials mentioning features or use of this software | |||||
* must display the following acknowledgement: | |||||
* This product includes software developed by the University of | |||||
* California, Berkeley and its contributors. | |||||
* 4. Neither the name of the University nor the names of its contributors | |||||
* may be used to endorse or promote products derived from this software | |||||
* without specific prior written permission. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | |||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | |||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |||||
* SUCH DAMAGE. | |||||
* | |||||
* from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 | |||||
*/ | |||||
/*- | |||||
* Copyright (c) 2003 Networks Associates Technology, Inc. | |||||
* All rights reserved. | |||||
* | |||||
* This software was developed for the FreeBSD Project by Jake Burkholder, | |||||
* Safeport Network Services, and Network Associates Laboratories, the | |||||
* Security Research Division of Network Associates, Inc. under | |||||
* DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA | |||||
* CHATS research program. | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | |||||
* modification, are permitted provided that the following conditions | |||||
* are met: | |||||
* 1. Redistributions of source code must retain the above copyright | |||||
* notice, this list of conditions and the following disclaimer. | |||||
* 2. Redistributions in binary form must reproduce the above copyright | |||||
* notice, this list of conditions and the following disclaimer in the | |||||
* documentation and/or other materials provided with the distribution. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND | |||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE | |||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | |||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | |||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | |||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | |||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |||||
* SUCH DAMAGE. | |||||
*/ | |||||
#define AMD64_NPT_AWARE | |||||
#include <sys/cdefs.h> | |||||
__FBSDID("$FreeBSD$"); | |||||
/* | |||||
* Manages physical address maps. | |||||
* | |||||
* Since the information managed by this module is | |||||
* also stored by the logical address mapping module, | |||||
* this module may throw away valid virtual-to-physical | |||||
* mappings at almost any time. However, invalidations | |||||
* of virtual-to-physical mappings must be done as | |||||
* requested. | |||||
* | |||||
* In order to cope with hardware architectures which | |||||
* make virtual-to-physical map invalidates expensive, | |||||
* this module may delay invalidate or reduced protection | |||||
* operations until such time as they are actually | |||||
* necessary. This module is given full information as | |||||
* to which processors are currently using which maps, | |||||
* and to when physical maps must be made correct. | |||||
*/ | |||||
#include <sys/param.h> | |||||
#include <sys/bus.h> | |||||
#include <sys/systm.h> | |||||
#include <sys/kernel.h> | |||||
#include <sys/ktr.h> | |||||
#include <sys/lock.h> | |||||
#include <sys/malloc.h> | |||||
#include <sys/mman.h> | |||||
#include <sys/msgbuf.h> | |||||
#include <sys/mutex.h> | |||||
#include <sys/proc.h> | |||||
#include <sys/rwlock.h> | |||||
#include <sys/sx.h> | |||||
#include <sys/vmem.h> | |||||
#include <sys/vmmeter.h> | |||||
#include <sys/sched.h> | |||||
#include <sys/sysctl.h> | |||||
#include <sys/_unrhdr.h> | |||||
#include <sys/smp.h> | |||||
#include <vm/vm.h> | |||||
#include <vm/vm_param.h> | |||||
#include <vm/vm_kern.h> | |||||
#include <vm/vm_page.h> | |||||
#include <vm/vm_map.h> | |||||
#include <vm/vm_object.h> | |||||
#include <vm/vm_extern.h> | |||||
#include <vm/vm_pageout.h> | |||||
#include <vm/vm_pager.h> | |||||
#include <vm/vm_radix.h> | |||||
#include <vm/vm_reserv.h> | |||||
#include <vm/uma.h> | |||||
#include <machine/machdep.h> | |||||
#include <machine/md_var.h> | |||||
#include <machine/pcb.h> | |||||
#define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t))) | |||||
#define NUPDE (NPDEPG * NPDEPG) | |||||
#define NUSERPGTBLS (NUPDE + NPDEPG) | |||||
#if !defined(DIAGNOSTIC) | |||||
#ifdef __GNUC_GNU_INLINE__ | |||||
#define PMAP_INLINE __attribute__((__gnu_inline__)) inline | |||||
#else | |||||
#define PMAP_INLINE extern inline | |||||
#endif | |||||
#else | |||||
#define PMAP_INLINE | |||||
#endif | |||||
/* | |||||
* These are configured by the mair_el1 register. This is set up in locore.S | |||||
*/ | |||||
#define DEVICE_MEMORY 0 | |||||
#define UNCACHED_MEMORY 1 | |||||
#define CACHED_MEMORY 2 | |||||
#ifdef PV_STATS | |||||
#define PV_STAT(x) do { x ; } while (0) | |||||
#else | |||||
#define PV_STAT(x) do { } while (0) | |||||
#endif | |||||
#define pmap_l2_pindex(v) ((v) >> L2_SHIFT) | |||||
#define NPV_LIST_LOCKS MAXCPU | |||||
#define PHYS_TO_PV_LIST_LOCK(pa) \ | |||||
(&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS]) | |||||
#define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ | |||||
struct rwlock **_lockp = (lockp); \ | |||||
struct rwlock *_new_lock; \ | |||||
\ | |||||
_new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ | |||||
if (_new_lock != *_lockp) { \ | |||||
if (*_lockp != NULL) \ | |||||
rw_wunlock(*_lockp); \ | |||||
*_lockp = _new_lock; \ | |||||
rw_wlock(*_lockp); \ | |||||
} \ | |||||
} while (0) | |||||
#define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ | |||||
CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) | |||||
#define RELEASE_PV_LIST_LOCK(lockp) do { \ | |||||
struct rwlock **_lockp = (lockp); \ | |||||
\ | |||||
if (*_lockp != NULL) { \ | |||||
rw_wunlock(*_lockp); \ | |||||
*_lockp = NULL; \ | |||||
} \ | |||||
} while (0) | |||||
#define VM_PAGE_TO_PV_LIST_LOCK(m) \ | |||||
PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) | |||||
struct pmap kernel_pmap_store; | |||||
vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ | |||||
vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ | |||||
vm_offset_t kernel_vm_end = 0; | |||||
struct msgbuf *msgbufp = NULL; | |||||
static struct rwlock_padalign pvh_global_lock; | |||||
/* | |||||
* Data for the pv entry allocation mechanism | |||||
*/ | |||||
static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); | |||||
static struct mtx pv_chunks_mutex; | |||||
static struct rwlock pv_list_locks[NPV_LIST_LOCKS]; | |||||
static void free_pv_chunk(struct pv_chunk *pc); | |||||
static void free_pv_entry(pmap_t pmap, pv_entry_t pv); | |||||
static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); | |||||
static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); | |||||
static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); | |||||
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, | |||||
vm_offset_t va); | |||||
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, | |||||
vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); | |||||
static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva, | |||||
pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); | |||||
static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, | |||||
vm_page_t m, struct rwlock **lockp); | |||||
static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, | |||||
struct rwlock **lockp); | |||||
static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, | |||||
struct spglist *free); | |||||
static int pmap_unuse_l3(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); | |||||
/********************/ | |||||
/* Inline functions */ | |||||
/********************/ | |||||
static __inline void | |||||
pagecopy(void *s, void *d) | |||||
{ | |||||
memcpy(d, s, PAGE_SIZE); | |||||
} | |||||
static __inline void | |||||
pagezero(void *p) | |||||
{ | |||||
bzero(p, PAGE_SIZE); | |||||
} | |||||
#define pmap_l1_index(va) (((va) >> L1_SHIFT) & Ln_ADDR_MASK) | |||||
#define pmap_l2_index(va) (((va) >> L2_SHIFT) & Ln_ADDR_MASK) | |||||
#define pmap_l3_index(va) (((va) >> L3_SHIFT) & Ln_ADDR_MASK) | |||||
static __inline pd_entry_t * | |||||
pmap_l1(pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
return (&pmap->pm_l1[pmap_l1_index(va)]); | |||||
} | |||||
static __inline pd_entry_t * | |||||
pmap_l1_to_l2(pd_entry_t *l1, vm_offset_t va) | |||||
{ | |||||
pd_entry_t *l2; | |||||
l2 = (pd_entry_t *)PHYS_TO_DMAP(*l1 & ~ATTR_MASK); | |||||
return (&l2[pmap_l2_index(va)]); | |||||
} | |||||
static __inline pd_entry_t * | |||||
pmap_l2(pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
pd_entry_t *l1; | |||||
l1 = pmap_l1(pmap, va); | |||||
if ((*l1 & ATTR_DESCR_MASK) != L1_TABLE) | |||||
return (NULL); | |||||
return (pmap_l1_to_l2(l1, va)); | |||||
} | |||||
static __inline pt_entry_t * | |||||
pmap_l2_to_l3(pd_entry_t *l2, vm_offset_t va) | |||||
{ | |||||
pt_entry_t *l3; | |||||
l3 = (pd_entry_t *)PHYS_TO_DMAP(*l2 & ~ATTR_MASK); | |||||
return (&l3[pmap_l3_index(va)]); | |||||
} | |||||
static __inline pt_entry_t * | |||||
pmap_l3(pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
pd_entry_t *l2; | |||||
l2 = pmap_l2(pmap, va); | |||||
if (l2 == NULL || (*l2 & ATTR_DESCR_MASK) != L2_TABLE) | |||||
return (NULL); | |||||
return (pmap_l2_to_l3(l2, va)); | |||||
} | |||||
/* | |||||
* These load the old table data and store the new value. | |||||
* They need to be atomic as the System MMU may write to the table at | |||||
* the same time as the CPU. | |||||
*/ | |||||
#define pmap_load_store(table, entry) atomic_swap_64(table, entry) | |||||
#define pmap_set(table, mask) atomic_set_64(table, mask) | |||||
#define pmap_load_clear(table) atomic_swap_64(table, 0) | |||||
#define pmap_load(table) (*table) | |||||
static __inline int | |||||
pmap_is_current(pmap_t pmap) | |||||
{ | |||||
return ((pmap == pmap_kernel()) || | |||||
(pmap == curthread->td_proc->p_vmspace->vm_map.pmap)); | |||||
} | |||||
static __inline int | |||||
pmap_l3_valid(pt_entry_t l3) | |||||
{ | |||||
return ((l3 & ATTR_DESCR_MASK) == L3_PAGE); | |||||
} | |||||
static __inline int | |||||
pmap_l3_valid_cacheable(pt_entry_t l3) | |||||
{ | |||||
return (((l3 & ATTR_DESCR_MASK) == L3_PAGE) && | |||||
((l3 & ATTR_IDX_MASK) == ATTR_IDX(CACHED_MEMORY))); | |||||
} | |||||
#define PTE_SYNC(pte) cpu_dcache_wb_range((vm_offset_t)pte, sizeof(*pte)) | |||||
/* | |||||
* Checks if the page is dirty. We currently lack proper tracking of this on | |||||
* arm64 so for now assume is a page mapped as rw was accessed it is. | |||||
*/ | |||||
static inline int | |||||
pmap_page_dirty(pt_entry_t pte) | |||||
{ | |||||
return ((pte & (ATTR_AF | ATTR_AP_RW_BIT)) == | |||||
(ATTR_AF | ATTR_AP(ATTR_AP_RW))); | |||||
} | |||||
static __inline void | |||||
pmap_resident_count_inc(pmap_t pmap, int count) | |||||
{ | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
pmap->pm_stats.resident_count += count; | |||||
} | |||||
static __inline void | |||||
pmap_resident_count_dec(pmap_t pmap, int count) | |||||
{ | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
KASSERT(pmap->pm_stats.resident_count >= count, | |||||
("pmap %p resident count underflow %ld %d", pmap, | |||||
pmap->pm_stats.resident_count, count)); | |||||
pmap->pm_stats.resident_count -= count; | |||||
} | |||||
static pt_entry_t * | |||||
pmap_early_page_idx(vm_offset_t l1pt, vm_offset_t va, u_int *l1_slot, | |||||
u_int *l2_slot) | |||||
{ | |||||
pt_entry_t *l2; | |||||
pd_entry_t *l1; | |||||
l1 = (pd_entry_t *)l1pt; | |||||
*l1_slot = (va >> L1_SHIFT) & Ln_ADDR_MASK; | |||||
/* Check locore has used a table L1 map */ | |||||
KASSERT((l1[*l1_slot] & ATTR_DESCR_MASK) == L1_TABLE, | |||||
("Invalid bootstrap L1 table")); | |||||
/* Find the address of the L2 table */ | |||||
l2 = (pt_entry_t *)init_pt_va; | |||||
*l2_slot = pmap_l2_index(va); | |||||
return (l2); | |||||
} | |||||
static vm_paddr_t | |||||
pmap_early_vtophys(vm_offset_t l1pt, vm_offset_t va) | |||||
{ | |||||
u_int l1_slot, l2_slot; | |||||
pt_entry_t *l2; | |||||
l2 = pmap_early_page_idx(l1pt, va, &l1_slot, &l2_slot); | |||||
return ((l2[l2_slot] & ~ATTR_MASK) + (va & L2_OFFSET)); | |||||
} | |||||
static void | |||||
pmap_bootstrap_dmap(vm_offset_t l1pt) | |||||
{ | |||||
vm_offset_t va; | |||||
vm_paddr_t pa; | |||||
pd_entry_t *l1; | |||||
u_int l1_slot; | |||||
va = DMAP_MIN_ADDRESS; | |||||
l1 = (pd_entry_t *)l1pt; | |||||
l1_slot = pmap_l1_index(DMAP_MIN_ADDRESS); | |||||
for (pa = 0; va < DMAP_MAX_ADDRESS; | |||||
pa += L1_SIZE, va += L1_SIZE, l1_slot++) { | |||||
KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index")); | |||||
/* | |||||
* TODO: Turn the cache on here when we have cache | |||||
* flushing code. | |||||
*/ | |||||
pmap_load_store(&l1[l1_slot], | |||||
(pa & ~L1_OFFSET) | ATTR_AF | L1_BLOCK | | |||||
ATTR_IDX(CACHED_MEMORY)); | |||||
} | |||||
cpu_dcache_wb_range((vm_offset_t)l1, PAGE_SIZE); | |||||
cpu_tlb_flushID(); | |||||
} | |||||
static vm_offset_t | |||||
pmap_bootstrap_l2(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l2_start) | |||||
{ | |||||
vm_offset_t l2pt; | |||||
vm_paddr_t pa; | |||||
pd_entry_t *l1; | |||||
u_int l1_slot; | |||||
KASSERT((va & L1_OFFSET) == 0, ("Invalid virtual address")); | |||||
l1 = (pd_entry_t *)l1pt; | |||||
l1_slot = pmap_l1_index(va); | |||||
l2pt = l2_start; | |||||
for (; va < VM_MAX_KERNEL_ADDRESS; l1_slot++, va += L1_SIZE) { | |||||
KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index")); | |||||
pa = pmap_early_vtophys(l1pt, l2pt); | |||||
pmap_load_store(&l1[l1_slot], | |||||
(pa & ~Ln_TABLE_MASK) | L1_TABLE); | |||||
l2pt += PAGE_SIZE; | |||||
} | |||||
/* Clean the L2 page table */ | |||||
memset((void *)l2_start, 0, l2pt - l2_start); | |||||
cpu_dcache_wb_range(l2_start, l2pt - l2_start); | |||||
/* Flush the l1 table to ram */ | |||||
cpu_dcache_wb_range((vm_offset_t)l1, PAGE_SIZE); | |||||
return l2pt; | |||||
} | |||||
static vm_offset_t | |||||
pmap_bootstrap_l3(vm_offset_t l1pt, vm_offset_t va, vm_offset_t l3_start) | |||||
{ | |||||
vm_offset_t l2pt, l3pt; | |||||
vm_paddr_t pa; | |||||
pd_entry_t *l2; | |||||
u_int l2_slot; | |||||
KASSERT((va & L2_OFFSET) == 0, ("Invalid virtual address")); | |||||
l2 = pmap_l2(kernel_pmap, va); | |||||
l2 = (pd_entry_t *)((uintptr_t)l2 & ~(PAGE_SIZE - 1)); | |||||
l2pt = (vm_offset_t)l2; | |||||
l2_slot = pmap_l2_index(va); | |||||
l3pt = l3_start; | |||||
for (; va < VM_MAX_KERNEL_ADDRESS; l2_slot++, va += L2_SIZE) { | |||||
KASSERT(l2_slot < Ln_ENTRIES, ("Invalid L2 index")); | |||||
pa = pmap_early_vtophys(l1pt, l3pt); | |||||
pmap_load_store(&l2[l2_slot], | |||||
(pa & ~Ln_TABLE_MASK) | L2_TABLE); | |||||
l3pt += PAGE_SIZE; | |||||
} | |||||
/* Clean the L2 page table */ | |||||
memset((void *)l3_start, 0, l3pt - l3_start); | |||||
cpu_dcache_wb_range(l3_start, l3pt - l3_start); | |||||
cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE); | |||||
return l3pt; | |||||
} | |||||
/* | |||||
* Bootstrap the system enough to run with virtual memory. | |||||
*/ | |||||
void | |||||
pmap_bootstrap(vm_offset_t l1pt, vm_paddr_t kernstart, vm_size_t kernlen) | |||||
{ | |||||
u_int l1_slot, l2_slot, avail_slot, map_slot, used_map_slot; | |||||
uint64_t kern_delta; | |||||
pt_entry_t *l2; | |||||
vm_offset_t va, freemempos; | |||||
vm_offset_t dpcpu, msgbufpv; | |||||
vm_paddr_t pa; | |||||
kern_delta = KERNBASE - kernstart; | |||||
physmem = 0; | |||||
printf("pmap_bootstrap %lx %lx %lx\n", l1pt, kernstart, kernlen); | |||||
printf("%lx\n", l1pt); | |||||
printf("%lx\n", (KERNBASE >> L1_SHIFT) & Ln_ADDR_MASK); | |||||
/* Set this early so we can use the pagetable walking functions */ | |||||
kernel_pmap_store.pm_l1 = (pd_entry_t *)l1pt; | |||||
PMAP_LOCK_INIT(kernel_pmap); | |||||
/* | |||||
* Initialize the global pv list lock. | |||||
*/ | |||||
rw_init(&pvh_global_lock, "pmap pv global"); | |||||
/* Create a direct map region early so we can use it for pa -> va */ | |||||
pmap_bootstrap_dmap(l1pt); | |||||
va = KERNBASE; | |||||
pa = KERNBASE - kern_delta; | |||||
/* | |||||
* Start to initialise phys_avail by copying from physmap | |||||
* up to the physical address KERNBASE points at. | |||||
*/ | |||||
map_slot = avail_slot = 0; | |||||
for (; map_slot < (physmap_idx * 2); map_slot += 2) { | |||||
if (physmap[map_slot] == physmap[map_slot + 1]) | |||||
continue; | |||||
if (physmap[map_slot] <= pa && | |||||
physmap[map_slot + 1] > pa) | |||||
break; | |||||
phys_avail[avail_slot] = physmap[map_slot]; | |||||
phys_avail[avail_slot + 1] = physmap[map_slot + 1]; | |||||
physmem += (phys_avail[avail_slot + 1] - | |||||
phys_avail[avail_slot]) >> PAGE_SHIFT; | |||||
avail_slot += 2; | |||||
} | |||||
/* Add the memory before the kernel */ | |||||
if (physmap[avail_slot] < pa) { | |||||
phys_avail[avail_slot] = physmap[map_slot]; | |||||
phys_avail[avail_slot + 1] = pa; | |||||
physmem += (phys_avail[avail_slot + 1] - | |||||
phys_avail[avail_slot]) >> PAGE_SHIFT; | |||||
avail_slot += 2; | |||||
} | |||||
used_map_slot = map_slot; | |||||
/* | |||||
* Read the page table to find out what is already mapped. | |||||
* This assumes we have mapped a block of memory from KERNBASE | |||||
* using a single L1 entry. | |||||
*/ | |||||
l2 = pmap_early_page_idx(l1pt, KERNBASE, &l1_slot, &l2_slot); | |||||
/* Sanity check the index, KERNBASE should be the first VA */ | |||||
KASSERT(l2_slot == 0, ("The L2 index is non-zero")); | |||||
/* Find how many pages we have mapped */ | |||||
for (; l2_slot < Ln_ENTRIES; l2_slot++) { | |||||
if ((l2[l2_slot] & ATTR_DESCR_MASK) == 0) | |||||
break; | |||||
/* Check locore used L2 blocks */ | |||||
KASSERT((l2[l2_slot] & ATTR_DESCR_MASK) == L2_BLOCK, | |||||
("Invalid bootstrap L2 table")); | |||||
KASSERT((l2[l2_slot] & ~ATTR_MASK) == pa, | |||||
("Incorrect PA in L2 table")); | |||||
va += L2_SIZE; | |||||
pa += L2_SIZE; | |||||
} | |||||
va = roundup2(va, L1_SIZE); | |||||
freemempos = KERNBASE + kernlen; | |||||
freemempos = roundup2(freemempos, PAGE_SIZE); | |||||
/* Create the l2 tables up to VM_MAX_KERNEL_ADDRESS */ | |||||
freemempos = pmap_bootstrap_l2(l1pt, va, freemempos); | |||||
/* And the l3 tables for the early devmap */ | |||||
freemempos = pmap_bootstrap_l3(l1pt, | |||||
VM_MAX_KERNEL_ADDRESS - L2_SIZE, freemempos); | |||||
cpu_tlb_flushID(); | |||||
#define alloc_pages(var, np) \ | |||||
(var) = freemempos; \ | |||||
freemempos += (np * PAGE_SIZE); \ | |||||
memset((char *)(var), 0, ((np) * PAGE_SIZE)); | |||||
/* Allocate dynamic per-cpu area. */ | |||||
alloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); | |||||
dpcpu_init((void *)dpcpu, 0); | |||||
/* Allocate memory for the msgbuf, e.g. for /sbin/dmesg */ | |||||
alloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); | |||||
msgbufp = (void *)msgbufpv; | |||||
virtual_avail = roundup2(freemempos, L1_SIZE); | |||||
virtual_end = VM_MAX_KERNEL_ADDRESS - L2_SIZE; | |||||
kernel_vm_end = virtual_avail; | |||||
pa = pmap_early_vtophys(l1pt, freemempos); | |||||
/* Finish initialising physmap */ | |||||
map_slot = used_map_slot; | |||||
for (; avail_slot < (PHYS_AVAIL_SIZE - 2) && | |||||
map_slot < (physmap_idx * 2); map_slot += 2) { | |||||
if (physmap[map_slot] == physmap[map_slot + 1]) | |||||
continue; | |||||
/* Have we used the current range? */ | |||||
if (physmap[map_slot + 1] <= pa) | |||||
continue; | |||||
/* Do we need to split the entry? */ | |||||
if (physmap[map_slot] < pa) { | |||||
phys_avail[avail_slot] = pa; | |||||
phys_avail[avail_slot + 1] = physmap[map_slot + 1]; | |||||
} else { | |||||
phys_avail[avail_slot] = physmap[map_slot]; | |||||
phys_avail[avail_slot + 1] = physmap[map_slot + 1]; | |||||
} | |||||
physmem += (phys_avail[avail_slot + 1] - | |||||
phys_avail[avail_slot]) >> PAGE_SHIFT; | |||||
avail_slot += 2; | |||||
} | |||||
phys_avail[avail_slot] = 0; | |||||
phys_avail[avail_slot + 1] = 0; | |||||
/* | |||||
* Maxmem isn't the "maximum memory", it's one larger than the | |||||
* highest page of the physical address space. It should be | |||||
* called something like "Maxphyspage". | |||||
*/ | |||||
Maxmem = atop(phys_avail[avail_slot - 1]); | |||||
cpu_tlb_flushID(); | |||||
} | |||||
/* | |||||
* Initialize a vm_page's machine-dependent fields. | |||||
*/ | |||||
void | |||||
pmap_page_init(vm_page_t m) | |||||
{ | |||||
TAILQ_INIT(&m->md.pv_list); | |||||
m->md.pv_memattr = VM_MEMATTR_WRITE_BACK; | |||||
} | |||||
/* | |||||
* Initialize the pmap module. | |||||
* Called by vm_init, to initialize any structures that the pmap | |||||
* system needs to map virtual memory. | |||||
*/ | |||||
void | |||||
pmap_init(void) | |||||
{ | |||||
int i; | |||||
/* | |||||
* Initialize the pv chunk list mutex. | |||||
*/ | |||||
mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); | |||||
/* | |||||
* Initialize the pool of pv list locks. | |||||
*/ | |||||
for (i = 0; i < NPV_LIST_LOCKS; i++) | |||||
rw_init(&pv_list_locks[i], "pmap pv list"); | |||||
} | |||||
/* | |||||
* Normal, non-SMP, invalidation functions. | |||||
* We inline these within pmap.c for speed. | |||||
*/ | |||||
PMAP_INLINE void | |||||
pmap_invalidate_page(pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
sched_pin(); | |||||
__asm __volatile( | |||||
"dsb sy \n" | |||||
"tlbi vaae1is, %0 \n" | |||||
"dsb sy \n" | |||||
"isb \n" | |||||
: : "r"(va >> PAGE_SHIFT)); | |||||
sched_unpin(); | |||||
} | |||||
PMAP_INLINE void | |||||
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | |||||
{ | |||||
vm_offset_t addr; | |||||
sched_pin(); | |||||
sva >>= PAGE_SHIFT; | |||||
eva >>= PAGE_SHIFT; | |||||
__asm __volatile("dsb sy"); | |||||
for (addr = sva; addr < eva; addr++) { | |||||
__asm __volatile( | |||||
"tlbi vaae1is, %0" : : "r"(addr)); | |||||
} | |||||
__asm __volatile( | |||||
"dsb sy \n" | |||||
"isb \n"); | |||||
sched_unpin(); | |||||
} | |||||
PMAP_INLINE void | |||||
pmap_invalidate_all(pmap_t pmap) | |||||
{ | |||||
sched_pin(); | |||||
__asm __volatile( | |||||
"dsb sy \n" | |||||
"tlbi vmalle1is \n" | |||||
"dsb sy \n" | |||||
"isb \n"); | |||||
sched_unpin(); | |||||
} | |||||
/* | |||||
* Routine: pmap_extract | |||||
* Function: | |||||
* Extract the physical page address associated | |||||
* with the given map/virtual_address pair. | |||||
*/ | |||||
vm_paddr_t | |||||
pmap_extract(pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
pd_entry_t *l2p, l2; | |||||
pt_entry_t *l3p, l3; | |||||
vm_paddr_t pa; | |||||
pa = 0; | |||||
PMAP_LOCK(pmap); | |||||
/* | |||||
* Start with the l2 tabel. We are unable to allocate | |||||
* pages in the l1 table. | |||||
*/ | |||||
l2p = pmap_l2(pmap, va); | |||||
if (l2p != NULL) { | |||||
l2 = *l2p; | |||||
if ((l2 & ATTR_DESCR_MASK) == L2_TABLE) { | |||||
l3p = pmap_l2_to_l3(l2p, va); | |||||
if (l3p != NULL) { | |||||
l3 = *l3p; | |||||
if ((l3 & ATTR_DESCR_MASK) == L3_PAGE) | |||||
pa = (l3 & ~ATTR_MASK) | | |||||
(va & L3_OFFSET); | |||||
} | |||||
} else if ((l2 & ATTR_DESCR_MASK) == L2_BLOCK) | |||||
pa = (l2 & ~ATTR_MASK) | (va & L2_OFFSET); | |||||
} | |||||
PMAP_UNLOCK(pmap); | |||||
return (pa); | |||||
} | |||||
/* | |||||
* Routine: pmap_extract_and_hold | |||||
* Function: | |||||
* Atomically extract and hold the physical page | |||||
* with the given pmap and virtual address pair | |||||
* if that mapping permits the given protection. | |||||
*/ | |||||
vm_page_t | |||||
pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) | |||||
{ | |||||
pt_entry_t *l3p, l3; | |||||
vm_paddr_t pa; | |||||
vm_page_t m; | |||||
pa = 0; | |||||
m = NULL; | |||||
PMAP_LOCK(pmap); | |||||
retry: | |||||
l3p = pmap_l3(pmap, va); | |||||
if (l3p != NULL && (l3 = *l3p) != 0) { | |||||
if (((l3 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) || | |||||
((prot & VM_PROT_WRITE) == 0)) { | |||||
if (vm_page_pa_tryrelock(pmap, l3 & ~ATTR_MASK, &pa)) | |||||
goto retry; | |||||
m = PHYS_TO_VM_PAGE(l3 & ~ATTR_MASK); | |||||
vm_page_hold(m); | |||||
} | |||||
} | |||||
PA_UNLOCK_COND(pa); | |||||
PMAP_UNLOCK(pmap); | |||||
return (m); | |||||
} | |||||
vm_paddr_t | |||||
pmap_kextract(vm_offset_t va) | |||||
{ | |||||
pd_entry_t *l2; | |||||
pt_entry_t *l3; | |||||
vm_paddr_t pa; | |||||
if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { | |||||
pa = DMAP_TO_PHYS(va); | |||||
} else { | |||||
l2 = pmap_l2(kernel_pmap, va); | |||||
if (l2 == NULL) | |||||
panic("pmap_kextract: No l2"); | |||||
if ((*l2 & ATTR_DESCR_MASK) == L2_BLOCK) | |||||
return ((*l2 & ~ATTR_MASK) | (va & L2_OFFSET)); | |||||
l3 = pmap_l2_to_l3(l2, va); | |||||
if (l3 == NULL) | |||||
panic("pmap_kextract: No l3..."); | |||||
pa = (*l3 & ~ATTR_MASK) | (va & PAGE_MASK); | |||||
} | |||||
return (pa); | |||||
} | |||||
/*************************************************** | |||||
* Low level mapping routines..... | |||||
***************************************************/ | |||||
void | |||||
pmap_kenter_device(vm_offset_t va, vm_paddr_t pa) | |||||
{ | |||||
pt_entry_t *l3; | |||||
KASSERT((pa & L3_OFFSET) == 0, | |||||
("pmap_kenter_device: Invalid physical address")); | |||||
KASSERT((va & L3_OFFSET) == 0, | |||||
("pmap_kenter_device: Invalid virtual address")); | |||||
l3 = pmap_l3(kernel_pmap, va); | |||||
KASSERT(l3 != NULL, ("Invalid page table, va: 0x%lx", va)); | |||||
pmap_load_store(l3, (pa & ~L3_OFFSET) | ATTR_AF | L3_PAGE | | |||||
ATTR_IDX(DEVICE_MEMORY)); | |||||
PTE_SYNC(l3); | |||||
} | |||||
/* | |||||
* Remove a page from the kernel pagetables. | |||||
* Note: not SMP coherent. | |||||
*/ | |||||
PMAP_INLINE void | |||||
pmap_kremove(vm_offset_t va) | |||||
{ | |||||
pt_entry_t *l3; | |||||
l3 = pmap_l3(kernel_pmap, va); | |||||
KASSERT(l3 != NULL, ("pmap_kremove: Invalid address")); | |||||
if (pmap_l3_valid_cacheable(pmap_load(l3))) | |||||
cpu_dcache_wb_range(va, L3_SIZE); | |||||
pmap_load_clear(l3); | |||||
PTE_SYNC(l3); | |||||
} | |||||
/* | |||||
* Used to map a range of physical addresses into kernel | |||||
* virtual address space. | |||||
* | |||||
* The value passed in '*virt' is a suggested virtual address for | |||||
* the mapping. Architectures which can support a direct-mapped | |||||
* physical to virtual region can return the appropriate address | |||||
* within that region, leaving '*virt' unchanged. Other | |||||
* architectures should map the pages starting at '*virt' and | |||||
* update '*virt' with the first usable address after the mapped | |||||
* region. | |||||
*/ | |||||
vm_offset_t | |||||
pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) | |||||
{ | |||||
return PHYS_TO_DMAP(start); | |||||
} | |||||
/* | |||||
* Add a list of wired pages to the kva | |||||
* this routine is only used for temporary | |||||
* kernel mappings that do not need to have | |||||
* page modification or references recorded. | |||||
* Note that old mappings are simply written | |||||
* over. The page *must* be wired. | |||||
* Note: SMP coherent. Uses a ranged shootdown IPI. | |||||
*/ | |||||
void | |||||
pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) | |||||
{ | |||||
pt_entry_t *l3, pa; | |||||
vm_offset_t va; | |||||
vm_page_t m; | |||||
int i; | |||||
va = sva; | |||||
for (i = 0; i < count; i++) { | |||||
m = ma[i]; | |||||
pa = VM_PAGE_TO_PHYS(m) | ATTR_AF | | |||||
ATTR_IDX(m->md.pv_memattr) | ATTR_AP(ATTR_AP_RW) | L3_PAGE; | |||||
l3 = pmap_l3(kernel_pmap, va); | |||||
pmap_load_store(l3, pa); | |||||
PTE_SYNC(l3); | |||||
va += L3_SIZE; | |||||
} | |||||
} | |||||
/* | |||||
* This routine tears out page mappings from the | |||||
* kernel -- it is meant only for temporary mappings. | |||||
* Note: SMP coherent. Uses a ranged shootdown IPI. | |||||
*/ | |||||
void | |||||
pmap_qremove(vm_offset_t sva, int count) | |||||
{ | |||||
vm_offset_t va; | |||||
va = sva; | |||||
while (count-- > 0) { | |||||
KASSERT(va >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", va)); | |||||
pmap_kremove(va); | |||||
va += PAGE_SIZE; | |||||
} | |||||
pmap_invalidate_range(kernel_pmap, sva, va); | |||||
} | |||||
/*************************************************** | |||||
* Page table page management routines..... | |||||
***************************************************/ | |||||
static __inline void | |||||
pmap_free_zero_pages(struct spglist *free) | |||||
{ | |||||
vm_page_t m; | |||||
while ((m = SLIST_FIRST(free)) != NULL) { | |||||
SLIST_REMOVE_HEAD(free, plinks.s.ss); | |||||
/* Preserve the page's PG_ZERO setting. */ | |||||
vm_page_free_toq(m); | |||||
} | |||||
} | |||||
/* | |||||
* Schedule the specified unused page table page to be freed. Specifically, | |||||
* add the page to the specified list of pages that will be released to the | |||||
* physical memory manager after the TLB has been updated. | |||||
*/ | |||||
static __inline void | |||||
pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, | |||||
boolean_t set_PG_ZERO) | |||||
{ | |||||
if (set_PG_ZERO) | |||||
m->flags |= PG_ZERO; | |||||
else | |||||
m->flags &= ~PG_ZERO; | |||||
SLIST_INSERT_HEAD(free, m, plinks.s.ss); | |||||
} | |||||
/* | |||||
* Decrements a page table page's wire count, which is used to record the | |||||
* number of valid page table entries within the page. If the wire count | |||||
* drops to zero, then the page table page is unmapped. Returns TRUE if the | |||||
* page table page was unmapped and FALSE otherwise. | |||||
*/ | |||||
static inline boolean_t | |||||
pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | |||||
{ | |||||
--m->wire_count; | |||||
if (m->wire_count == 0) { | |||||
_pmap_unwire_l3(pmap, va, m, free); | |||||
return (TRUE); | |||||
} else | |||||
return (FALSE); | |||||
} | |||||
static void | |||||
_pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) | |||||
{ | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
/* | |||||
* unmap the page table page | |||||
*/ | |||||
if (m->pindex >= NUPDE) { | |||||
/* PD page */ | |||||
pd_entry_t *l1; | |||||
l1 = pmap_l1(pmap, va); | |||||
pmap_load_clear(l1); | |||||
PTE_SYNC(l1); | |||||
} else { | |||||
/* PTE page */ | |||||
pd_entry_t *l2; | |||||
l2 = pmap_l2(pmap, va); | |||||
pmap_load_clear(l2); | |||||
PTE_SYNC(l2); | |||||
} | |||||
pmap_resident_count_dec(pmap, 1); | |||||
if (m->pindex < NUPDE) { | |||||
/* We just released a PT, unhold the matching PD */ | |||||
vm_page_t pdpg; | |||||
pdpg = PHYS_TO_VM_PAGE(*pmap_l1(pmap, va) & ~ATTR_MASK); | |||||
pmap_unwire_l3(pmap, va, pdpg, free); | |||||
} | |||||
/* | |||||
* This is a release store so that the ordinary store unmapping | |||||
* the page table page is globally performed before TLB shoot- | |||||
* down is begun. | |||||
*/ | |||||
atomic_subtract_rel_int(&vm_cnt.v_wire_count, 1); | |||||
/* | |||||
* Put page on a list so that it is released after | |||||
* *ALL* TLB shootdown is done | |||||
*/ | |||||
pmap_add_delayed_free_list(m, free, TRUE); | |||||
} | |||||
/* | |||||
* After removing an l3 entry, this routine is used to | |||||
* conditionally free the page, and manage the hold/wire counts. | |||||
*/ | |||||
static int | |||||
pmap_unuse_l3(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | |||||
struct spglist *free) | |||||
{ | |||||
vm_page_t mpte; | |||||
if (va >= VM_MAXUSER_ADDRESS) | |||||
return (0); | |||||
KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); | |||||
mpte = PHYS_TO_VM_PAGE(ptepde & ~ATTR_MASK); | |||||
return (pmap_unwire_l3(pmap, va, mpte, free)); | |||||
} | |||||
void | |||||
pmap_pinit0(pmap_t pmap) | |||||
{ | |||||
PMAP_LOCK_INIT(pmap); | |||||
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | |||||
pmap->pm_l1 = kernel_pmap->pm_l1; | |||||
} | |||||
int | |||||
pmap_pinit(pmap_t pmap) | |||||
{ | |||||
vm_paddr_t l1phys; | |||||
vm_page_t l1pt; | |||||
/* | |||||
* allocate the l1 page | |||||
*/ | |||||
while ((l1pt = vm_page_alloc(NULL, 0xdeadbeef, VM_ALLOC_NORMAL | | |||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) | |||||
VM_WAIT; | |||||
l1phys = VM_PAGE_TO_PHYS(l1pt); | |||||
pmap->pm_l1 = (pd_entry_t *)PHYS_TO_DMAP(l1phys); | |||||
if ((l1pt->flags & PG_ZERO) == 0) | |||||
pagezero(pmap->pm_l1); | |||||
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | |||||
return (1); | |||||
} | |||||
/* | |||||
* This routine is called if the desired page table page does not exist. | |||||
* | |||||
* If page table page allocation fails, this routine may sleep before | |||||
* returning NULL. It sleeps only if a lock pointer was given. | |||||
* | |||||
* Note: If a page allocation fails at page table level two or three, | |||||
* one or two pages may be held during the wait, only to be released | |||||
* afterwards. This conservative approach is easily argued to avoid | |||||
* race conditions. | |||||
*/ | |||||
static vm_page_t | |||||
_pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) | |||||
{ | |||||
vm_page_t m, /*pdppg, */pdpg; | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
/* | |||||
* Allocate a page table page. | |||||
*/ | |||||
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | |||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { | |||||
if (lockp != NULL) { | |||||
RELEASE_PV_LIST_LOCK(lockp); | |||||
PMAP_UNLOCK(pmap); | |||||
rw_runlock(&pvh_global_lock); | |||||
VM_WAIT; | |||||
rw_rlock(&pvh_global_lock); | |||||
PMAP_LOCK(pmap); | |||||
} | |||||
/* | |||||
* Indicate the need to retry. While waiting, the page table | |||||
* page may have been allocated. | |||||
*/ | |||||
return (NULL); | |||||
} | |||||
if ((m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m); | |||||
/* | |||||
* Map the pagetable page into the process address space, if | |||||
* it isn't already there. | |||||
*/ | |||||
if (ptepindex >= NUPDE) { | |||||
pd_entry_t *l1; | |||||
vm_pindex_t l1index; | |||||
l1index = ptepindex - NUPDE; | |||||
l1 = &pmap->pm_l1[l1index]; | |||||
pmap_load_store(l1, VM_PAGE_TO_PHYS(m) | L1_TABLE); | |||||
PTE_SYNC(l1); | |||||
} else { | |||||
vm_pindex_t l1index; | |||||
pd_entry_t *l1, *l2; | |||||
l1index = ptepindex >> (L1_SHIFT - L2_SHIFT); | |||||
l1 = &pmap->pm_l1[l1index]; | |||||
if (*l1 == 0) { | |||||
/* recurse for allocating page dir */ | |||||
if (_pmap_alloc_l3(pmap, NUPDE + l1index, | |||||
lockp) == NULL) { | |||||
--m->wire_count; | |||||
atomic_subtract_int(&vm_cnt.v_wire_count, 1); | |||||
vm_page_free_zero(m); | |||||
return (NULL); | |||||
} | |||||
} else { | |||||
pdpg = PHYS_TO_VM_PAGE(*l1 & ~ATTR_MASK); | |||||
pdpg->wire_count++; | |||||
} | |||||
l2 = (pd_entry_t *)PHYS_TO_DMAP(*l1 & ~ATTR_MASK); | |||||
l2 = &l2[ptepindex & Ln_ADDR_MASK]; | |||||
pmap_load_store(l2, VM_PAGE_TO_PHYS(m) | ATTR_AF | | |||||
ATTR_IDX(CACHED_MEMORY) | L2_TABLE); | |||||
PTE_SYNC(l2); | |||||
} | |||||
pmap_resident_count_inc(pmap, 1); | |||||
return (m); | |||||
} | |||||
static vm_page_t | |||||
pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) | |||||
{ | |||||
vm_pindex_t ptepindex; | |||||
pd_entry_t *l2; | |||||
vm_page_t m; | |||||
/* | |||||
* Calculate pagetable page index | |||||
*/ | |||||
ptepindex = pmap_l2_pindex(va); | |||||
retry: | |||||
/* | |||||
* Get the page directory entry | |||||
*/ | |||||
l2 = pmap_l2(pmap, va); | |||||
/* | |||||
* If the page table page is mapped, we just increment the | |||||
* hold count, and activate it. | |||||
*/ | |||||
if (l2 != NULL && *l2 != 0) { | |||||
m = PHYS_TO_VM_PAGE(*l2 & ~ATTR_MASK); | |||||
m->wire_count++; | |||||
} else { | |||||
/* | |||||
* Here if the pte page isn't mapped, or if it has been | |||||
* deallocated. | |||||
*/ | |||||
m = _pmap_alloc_l3(pmap, ptepindex, lockp); | |||||
if (m == NULL && lockp != NULL) | |||||
goto retry; | |||||
} | |||||
/* | |||||
* XXXARM64: I'm not sure why we need this but it fixes a crash | |||||
* when running things from a shell script. | |||||
*/ | |||||
pmap_invalidate_all(pmap); | |||||
return (m); | |||||
} | |||||
/*************************************************** | |||||
* Pmap allocation/deallocation routines. | |||||
***************************************************/ | |||||
/* | |||||
* Release any resources held by the given physical map. | |||||
* Called when a pmap initialized by pmap_pinit is being released. | |||||
* Should only be called if the map contains no valid mappings. | |||||
*/ | |||||
void | |||||
pmap_release(pmap_t pmap) | |||||
{ | |||||
vm_page_t m; | |||||
KASSERT(pmap->pm_stats.resident_count == 0, | |||||
("pmap_release: pmap resident count %ld != 0", | |||||
pmap->pm_stats.resident_count)); | |||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_l1)); | |||||
m->wire_count--; | |||||
atomic_subtract_int(&vm_cnt.v_wire_count, 1); | |||||
vm_page_free_zero(m); | |||||
} | |||||
#if 0 | |||||
static int | |||||
kvm_size(SYSCTL_HANDLER_ARGS) | |||||
{ | |||||
unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; | |||||
return sysctl_handle_long(oidp, &ksize, 0, req); | |||||
} | |||||
SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, | |||||
0, 0, kvm_size, "LU", "Size of KVM"); | |||||
static int | |||||
kvm_free(SYSCTL_HANDLER_ARGS) | |||||
{ | |||||
unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; | |||||
return sysctl_handle_long(oidp, &kfree, 0, req); | |||||
} | |||||
SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, | |||||
0, 0, kvm_free, "LU", "Amount of KVM free"); | |||||
#endif /* 0 */ | |||||
/* | |||||
* grow the number of kernel page table entries, if needed | |||||
*/ | |||||
void | |||||
pmap_growkernel(vm_offset_t addr) | |||||
{ | |||||
vm_paddr_t paddr; | |||||
vm_page_t nkpg; | |||||
pd_entry_t *l1, *l2; | |||||
mtx_assert(&kernel_map->system_mtx, MA_OWNED); | |||||
addr = roundup2(addr, L2_SIZE); | |||||
if (addr - 1 >= kernel_map->max_offset) | |||||
addr = kernel_map->max_offset; | |||||
while (kernel_vm_end < addr) { | |||||
l1 = pmap_l1(kernel_pmap, kernel_vm_end); | |||||
if (*l1 == 0) { | |||||
/* We need a new PDP entry */ | |||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT, | |||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | | |||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO); | |||||
if (nkpg == NULL) | |||||
panic("pmap_growkernel: no memory to grow kernel"); | |||||
if ((nkpg->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(nkpg); | |||||
paddr = VM_PAGE_TO_PHYS(nkpg); | |||||
pmap_load_store(l1, paddr | L1_TABLE); | |||||
PTE_SYNC(l1); | |||||
continue; /* try again */ | |||||
} | |||||
l2 = pmap_l1_to_l2(l1, kernel_vm_end); | |||||
if ((*l2 & ATTR_AF) != 0) { | |||||
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | |||||
if (kernel_vm_end - 1 >= kernel_map->max_offset) { | |||||
kernel_vm_end = kernel_map->max_offset; | |||||
break; | |||||
} | |||||
continue; | |||||
} | |||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT, | |||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | |||||
VM_ALLOC_ZERO); | |||||
if (nkpg == NULL) | |||||
panic("pmap_growkernel: no memory to grow kernel"); | |||||
if ((nkpg->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(nkpg); | |||||
paddr = VM_PAGE_TO_PHYS(nkpg); | |||||
pmap_load_store(l2, paddr | L2_TABLE); | |||||
PTE_SYNC(l2); | |||||
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | |||||
if (kernel_vm_end - 1 >= kernel_map->max_offset) { | |||||
kernel_vm_end = kernel_map->max_offset; | |||||
break; | |||||
} | |||||
} | |||||
} | |||||
/*************************************************** | |||||
* page management routines. | |||||
***************************************************/ | |||||
CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); | |||||
CTASSERT(_NPCM == 3); | |||||
CTASSERT(_NPCPV == 168); | |||||
static __inline struct pv_chunk * | |||||
pv_to_chunk(pv_entry_t pv) | |||||
{ | |||||
return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); | |||||
} | |||||
#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) | |||||
#define PC_FREE0 0xfffffffffffffffful | |||||
#define PC_FREE1 0xfffffffffffffffful | |||||
#define PC_FREE2 0x000000fffffffffful | |||||
static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; | |||||
#if 0 | |||||
#ifdef PV_STATS | |||||
static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; | |||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, | |||||
"Current number of pv entry chunks"); | |||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, | |||||
"Current number of pv entry chunks allocated"); | |||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, | |||||
"Current number of pv entry chunks frees"); | |||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, | |||||
"Number of times tried to get a chunk page but failed."); | |||||
static long pv_entry_frees, pv_entry_allocs, pv_entry_count; | |||||
static int pv_entry_spare; | |||||
SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, | |||||
"Current number of pv entry frees"); | |||||
SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, | |||||
"Current number of pv entry allocs"); | |||||
SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, | |||||
"Current number of pv entries"); | |||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, | |||||
"Current number of spare pv entries"); | |||||
#endif | |||||
#endif /* 0 */ | |||||
/* | |||||
* We are in a serious low memory condition. Resort to | |||||
* drastic measures to free some pages so we can allocate | |||||
* another pv entry chunk. | |||||
* | |||||
* Returns NULL if PV entries were reclaimed from the specified pmap. | |||||
* | |||||
* We do not, however, unmap 2mpages because subsequent accesses will | |||||
* allocate per-page pv entries until repromotion occurs, thereby | |||||
* exacerbating the shortage of free pv entries. | |||||
*/ | |||||
static vm_page_t | |||||
reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) | |||||
{ | |||||
panic("reclaim_pv_chunk"); | |||||
} | |||||
/* | |||||
* free the pv_entry back to the free list | |||||
*/ | |||||
static void | |||||
free_pv_entry(pmap_t pmap, pv_entry_t pv) | |||||
{ | |||||
struct pv_chunk *pc; | |||||
int idx, field, bit; | |||||
rw_assert(&pvh_global_lock, RA_LOCKED); | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
PV_STAT(atomic_add_long(&pv_entry_frees, 1)); | |||||
PV_STAT(atomic_add_int(&pv_entry_spare, 1)); | |||||
PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); | |||||
pc = pv_to_chunk(pv); | |||||
idx = pv - &pc->pc_pventry[0]; | |||||
field = idx / 64; | |||||
bit = idx % 64; | |||||
pc->pc_map[field] |= 1ul << bit; | |||||
if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 || | |||||
pc->pc_map[2] != PC_FREE2) { | |||||
/* 98% of the time, pc is already at the head of the list. */ | |||||
if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { | |||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | |||||
TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); | |||||
} | |||||
return; | |||||
} | |||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | |||||
free_pv_chunk(pc); | |||||
} | |||||
static void | |||||
free_pv_chunk(struct pv_chunk *pc) | |||||
{ | |||||
vm_page_t m; | |||||
mtx_lock(&pv_chunks_mutex); | |||||
TAILQ_REMOVE(&pv_chunks, pc, pc_lru); | |||||
mtx_unlock(&pv_chunks_mutex); | |||||
PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); | |||||
PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); | |||||
PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); | |||||
/* entire chunk is free, return it */ | |||||
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); | |||||
#if 0 /* TODO: For minidump */ | |||||
dump_drop_page(m->phys_addr); | |||||
#endif | |||||
vm_page_unwire(m, PQ_INACTIVE); | |||||
vm_page_free(m); | |||||
} | |||||
/* | |||||
* Returns a new PV entry, allocating a new PV chunk from the system when | |||||
* needed. If this PV chunk allocation fails and a PV list lock pointer was | |||||
* given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is | |||||
* returned. | |||||
* | |||||
* The given PV list lock may be released. | |||||
*/ | |||||
static pv_entry_t | |||||
get_pv_entry(pmap_t pmap, struct rwlock **lockp) | |||||
{ | |||||
int bit, field; | |||||
pv_entry_t pv; | |||||
struct pv_chunk *pc; | |||||
vm_page_t m; | |||||
rw_assert(&pvh_global_lock, RA_LOCKED); | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); | |||||
retry: | |||||
pc = TAILQ_FIRST(&pmap->pm_pvchunk); | |||||
if (pc != NULL) { | |||||
for (field = 0; field < _NPCM; field++) { | |||||
if (pc->pc_map[field]) { | |||||
bit = ffsl(pc->pc_map[field]) - 1; | |||||
break; | |||||
} | |||||
} | |||||
if (field < _NPCM) { | |||||
pv = &pc->pc_pventry[field * 64 + bit]; | |||||
pc->pc_map[field] &= ~(1ul << bit); | |||||
/* If this was the last item, move it to tail */ | |||||
if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && | |||||
pc->pc_map[2] == 0) { | |||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | |||||
TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, | |||||
pc_list); | |||||
} | |||||
PV_STAT(atomic_add_long(&pv_entry_count, 1)); | |||||
PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); | |||||
return (pv); | |||||
} | |||||
} | |||||
/* No free items, allocate another chunk */ | |||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | |||||
VM_ALLOC_WIRED); | |||||
if (m == NULL) { | |||||
if (lockp == NULL) { | |||||
PV_STAT(pc_chunk_tryfail++); | |||||
return (NULL); | |||||
} | |||||
m = reclaim_pv_chunk(pmap, lockp); | |||||
if (m == NULL) | |||||
goto retry; | |||||
} | |||||
PV_STAT(atomic_add_int(&pc_chunk_count, 1)); | |||||
PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); | |||||
#if 0 /* TODO: This is for minidump */ | |||||
dump_add_page(m->phys_addr); | |||||
#endif | |||||
pc = (void *)PHYS_TO_DMAP(m->phys_addr); | |||||
pc->pc_pmap = pmap; | |||||
pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ | |||||
pc->pc_map[1] = PC_FREE1; | |||||
pc->pc_map[2] = PC_FREE2; | |||||
mtx_lock(&pv_chunks_mutex); | |||||
TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); | |||||
mtx_unlock(&pv_chunks_mutex); | |||||
pv = &pc->pc_pventry[0]; | |||||
TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); | |||||
PV_STAT(atomic_add_long(&pv_entry_count, 1)); | |||||
PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); | |||||
return (pv); | |||||
} | |||||
/* | |||||
* First find and then remove the pv entry for the specified pmap and virtual | |||||
* address from the specified pv list. Returns the pv entry if found and NULL | |||||
* otherwise. This operation can be performed on pv lists for either 4KB or | |||||
* 2MB page mappings. | |||||
*/ | |||||
static __inline pv_entry_t | |||||
pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
pv_entry_t pv; | |||||
rw_assert(&pvh_global_lock, RA_LOCKED); | |||||
TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { | |||||
if (pmap == PV_PMAP(pv) && va == pv->pv_va) { | |||||
TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); | |||||
pvh->pv_gen++; | |||||
break; | |||||
} | |||||
} | |||||
return (pv); | |||||
} | |||||
/* | |||||
* First find and then destroy the pv entry for the specified pmap and virtual | |||||
* address. This operation can be performed on pv lists for either 4KB or 2MB | |||||
* page mappings. | |||||
*/ | |||||
static void | |||||
pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) | |||||
{ | |||||
pv_entry_t pv; | |||||
pv = pmap_pvh_remove(pvh, pmap, va); | |||||
KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); | |||||
free_pv_entry(pmap, pv); | |||||
} | |||||
/* | |||||
* Conditionally create the PV entry for a 4KB page mapping if the required | |||||
* memory can be allocated without resorting to reclamation. | |||||
*/ | |||||
static boolean_t | |||||
pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, | |||||
struct rwlock **lockp) | |||||
{ | |||||
pv_entry_t pv; | |||||
rw_assert(&pvh_global_lock, RA_LOCKED); | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
/* Pass NULL instead of the lock pointer to disable reclamation. */ | |||||
if ((pv = get_pv_entry(pmap, NULL)) != NULL) { | |||||
pv->pv_va = va; | |||||
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); | |||||
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | |||||
m->md.pv_gen++; | |||||
return (TRUE); | |||||
} else | |||||
return (FALSE); | |||||
} | |||||
/* | |||||
* pmap_remove_l3: do the things to unmap a page in a process | |||||
*/ | |||||
static int | |||||
pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va, | |||||
pd_entry_t l2e, struct spglist *free, struct rwlock **lockp) | |||||
{ | |||||
pt_entry_t old_l3; | |||||
vm_page_t m; | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(pmap_load(l3))) | |||||
cpu_dcache_wb_range(va, L3_SIZE); | |||||
old_l3 = pmap_load_clear(l3); | |||||
PTE_SYNC(l3); | |||||
if (old_l3 & ATTR_SW_WIRED) | |||||
pmap->pm_stats.wired_count -= 1; | |||||
pmap_resident_count_dec(pmap, 1); | |||||
if (old_l3 & ATTR_SW_MANAGED) { | |||||
m = PHYS_TO_VM_PAGE(old_l3 & ~ATTR_MASK); | |||||
if (pmap_page_dirty(old_l3)) | |||||
vm_page_dirty(m); | |||||
if (old_l3 & ATTR_AF) | |||||
vm_page_aflag_set(m, PGA_REFERENCED); | |||||
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); | |||||
pmap_pvh_free(&m->md, pmap, va); | |||||
} | |||||
return (pmap_unuse_l3(pmap, va, l2e, free)); | |||||
} | |||||
/* | |||||
* Remove the given range of addresses from the specified map. | |||||
* | |||||
* It is assumed that the start and end are properly | |||||
* rounded to the page size. | |||||
*/ | |||||
void | |||||
pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | |||||
{ | |||||
struct rwlock *lock; | |||||
vm_offset_t va, va_next; | |||||
pd_entry_t *l1, *l2; | |||||
pt_entry_t l3_paddr, *l3; | |||||
struct spglist free; | |||||
int anyvalid; | |||||
/* | |||||
* Perform an unsynchronized read. This is, however, safe. | |||||
*/ | |||||
if (pmap->pm_stats.resident_count == 0) | |||||
return; | |||||
anyvalid = 0; | |||||
SLIST_INIT(&free); | |||||
rw_rlock(&pvh_global_lock); | |||||
PMAP_LOCK(pmap); | |||||
lock = NULL; | |||||
for (; sva < eva; sva = va_next) { | |||||
if (pmap->pm_stats.resident_count == 0) | |||||
break; | |||||
l1 = pmap_l1(pmap, sva); | |||||
if (*l1 == 0) { | |||||
va_next = (sva + L1_SIZE) & ~L1_OFFSET; | |||||
if (va_next < sva) | |||||
va_next = eva; | |||||
continue; | |||||
} | |||||
/* | |||||
* Calculate index for next page table. | |||||
*/ | |||||
va_next = (sva + L2_SIZE) & ~L2_OFFSET; | |||||
if (va_next < sva) | |||||
va_next = eva; | |||||
l2 = pmap_l1_to_l2(l1, sva); | |||||
if (l2 == NULL) | |||||
continue; | |||||
l3_paddr = *l2; | |||||
/* | |||||
* Weed out invalid mappings. | |||||
*/ | |||||
if ((l3_paddr & ATTR_DESCR_MASK) != L2_TABLE) | |||||
continue; | |||||
/* | |||||
* Limit our scan to either the end of the va represented | |||||
* by the current page table page, or to the end of the | |||||
* range being removed. | |||||
*/ | |||||
if (va_next > eva) | |||||
va_next = eva; | |||||
va = va_next; | |||||
for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, | |||||
sva += L3_SIZE) { | |||||
if (l3 == NULL) | |||||
panic("l3 == NULL"); | |||||
if (*l3 == 0) { | |||||
if (va != va_next) { | |||||
pmap_invalidate_range(pmap, va, sva); | |||||
va = va_next; | |||||
} | |||||
continue; | |||||
} | |||||
if (va == va_next) | |||||
va = sva; | |||||
if (pmap_remove_l3(pmap, l3, sva, l3_paddr, &free, | |||||
&lock)) { | |||||
sva += L3_SIZE; | |||||
break; | |||||
} | |||||
} | |||||
if (va != va_next) | |||||
pmap_invalidate_range(pmap, va, sva); | |||||
} | |||||
if (lock != NULL) | |||||
rw_wunlock(lock); | |||||
if (anyvalid) | |||||
pmap_invalidate_all(pmap); | |||||
rw_runlock(&pvh_global_lock); | |||||
PMAP_UNLOCK(pmap); | |||||
pmap_free_zero_pages(&free); | |||||
} | |||||
/* | |||||
* Routine: pmap_remove_all | |||||
* Function: | |||||
* Removes this physical page from | |||||
* all physical maps in which it resides. | |||||
* Reflects back modify bits to the pager. | |||||
* | |||||
* Notes: | |||||
* Original versions of this routine were very | |||||
* inefficient because they iteratively called | |||||
* pmap_remove (slow...) | |||||
*/ | |||||
void | |||||
pmap_remove_all(vm_page_t m) | |||||
{ | |||||
pv_entry_t pv; | |||||
pmap_t pmap; | |||||
pt_entry_t *l3, tl3; | |||||
pd_entry_t *l2; | |||||
struct spglist free; | |||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | |||||
("pmap_remove_all: page %p is not managed", m)); | |||||
SLIST_INIT(&free); | |||||
rw_wlock(&pvh_global_lock); | |||||
while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { | |||||
pmap = PV_PMAP(pv); | |||||
PMAP_LOCK(pmap); | |||||
pmap_resident_count_dec(pmap, 1); | |||||
l2 = pmap_l2(pmap, pv->pv_va); | |||||
KASSERT((*l2 & ATTR_DESCR_MASK) == L2_TABLE, | |||||
("pmap_remove_all: found a table when expecting " | |||||
"a block in %p's pv list", m)); | |||||
l3 = pmap_l2_to_l3(l2, pv->pv_va); | |||||
if (pmap_is_current(pmap) && | |||||
pmap_l3_valid_cacheable(pmap_load(l3))) | |||||
cpu_dcache_wb_range(pv->pv_va, L3_SIZE); | |||||
tl3 = pmap_load_clear(l3); | |||||
PTE_SYNC(l3); | |||||
if (tl3 & ATTR_SW_WIRED) | |||||
pmap->pm_stats.wired_count--; | |||||
if ((tl3 & ATTR_AF) != 0) | |||||
vm_page_aflag_set(m, PGA_REFERENCED); | |||||
/* | |||||
* Update the vm_page_t clean and reference bits. | |||||
*/ | |||||
if (pmap_page_dirty(tl3)) | |||||
vm_page_dirty(m); | |||||
pmap_unuse_l3(pmap, pv->pv_va, *l2, &free); | |||||
pmap_invalidate_page(pmap, pv->pv_va); | |||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | |||||
m->md.pv_gen++; | |||||
free_pv_entry(pmap, pv); | |||||
PMAP_UNLOCK(pmap); | |||||
} | |||||
vm_page_aflag_clear(m, PGA_WRITEABLE); | |||||
rw_wunlock(&pvh_global_lock); | |||||
pmap_free_zero_pages(&free); | |||||
} | |||||
/* | |||||
* Set the physical protection on the | |||||
* specified range of this map as requested. | |||||
*/ | |||||
void | |||||
pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) | |||||
{ | |||||
vm_offset_t va, va_next; | |||||
pd_entry_t *l1, *l2; | |||||
pt_entry_t *l3p, l3; | |||||
if ((prot & VM_PROT_READ) == VM_PROT_NONE) { | |||||
pmap_remove(pmap, sva, eva); | |||||
return; | |||||
} | |||||
if ((prot & VM_PROT_WRITE) == VM_PROT_WRITE) | |||||
return; | |||||
PMAP_LOCK(pmap); | |||||
for (; sva < eva; sva = va_next) { | |||||
l1 = pmap_l1(pmap, sva); | |||||
if (*l1 == 0) { | |||||
va_next = (sva + L1_SIZE) & ~L1_OFFSET; | |||||
if (va_next < sva) | |||||
va_next = eva; | |||||
continue; | |||||
} | |||||
va_next = (sva + L2_SIZE) & ~L2_OFFSET; | |||||
if (va_next < sva) | |||||
va_next = eva; | |||||
l2 = pmap_l1_to_l2(l1, sva); | |||||
if (l2 == NULL || (*l2 & ATTR_DESCR_MASK) != L2_TABLE) | |||||
continue; | |||||
if (va_next > eva) | |||||
va_next = eva; | |||||
va = va_next; | |||||
for (l3p = pmap_l2_to_l3(l2, sva); sva != va_next; l3p++, | |||||
sva += L3_SIZE) { | |||||
l3 = pmap_load(l3p); | |||||
if (pmap_l3_valid(l3)) { | |||||
pmap_set(l3p, ATTR_AP(ATTR_AP_RO)); | |||||
PTE_SYNC(l3p); | |||||
} | |||||
} | |||||
} | |||||
PMAP_UNLOCK(pmap); | |||||
/* TODO: Only invalidate entries we are touching */ | |||||
pmap_invalidate_all(pmap); | |||||
} | |||||
/* | |||||
* Insert the given physical page (p) at | |||||
* the specified virtual address (v) in the | |||||
* target physical map with the protection requested. | |||||
* | |||||
* If specified, the page will be wired down, meaning | |||||
* that the related pte can not be reclaimed. | |||||
* | |||||
* NB: This is the only routine which MAY NOT lazy-evaluate | |||||
* or lose information. That is, this routine must actually | |||||
* insert this page into the given map NOW. | |||||
*/ | |||||
int | |||||
pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | |||||
u_int flags, int8_t psind __unused) | |||||
{ | |||||
struct rwlock *lock; | |||||
pd_entry_t *l1, *l2; | |||||
pt_entry_t new_l3, orig_l3; | |||||
pt_entry_t *l3; | |||||
pv_entry_t pv; | |||||
vm_paddr_t opa, pa, l2_pa, l3_pa; | |||||
vm_page_t mpte, om, l2_m, l3_m; | |||||
boolean_t nosleep; | |||||
va = trunc_page(va); | |||||
if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) | |||||
VM_OBJECT_ASSERT_LOCKED(m->object); | |||||
pa = VM_PAGE_TO_PHYS(m); | |||||
new_l3 = (pt_entry_t)(pa | ATTR_AF | L3_PAGE); | |||||
if ((prot & VM_PROT_WRITE) == 0) | |||||
new_l3 |= ATTR_AP(ATTR_AP_RO); | |||||
if ((flags & PMAP_ENTER_WIRED) != 0) | |||||
new_l3 |= ATTR_SW_WIRED; | |||||
if ((va >> 63) == 0) | |||||
new_l3 |= ATTR_AP(ATTR_AP_USER); | |||||
new_l3 |= ATTR_IDX(m->md.pv_memattr); | |||||
mpte = NULL; | |||||
lock = NULL; | |||||
rw_rlock(&pvh_global_lock); | |||||
PMAP_LOCK(pmap); | |||||
if (va < VM_MAXUSER_ADDRESS) { | |||||
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | |||||
mpte = pmap_alloc_l3(pmap, va, nosleep ? NULL : &lock); | |||||
if (mpte == NULL && nosleep) { | |||||
if (lock != NULL) | |||||
rw_wunlock(lock); | |||||
rw_runlock(&pvh_global_lock); | |||||
PMAP_UNLOCK(pmap); | |||||
return (KERN_RESOURCE_SHORTAGE); | |||||
} | |||||
l3 = pmap_l3(pmap, va); | |||||
} else { | |||||
l3 = pmap_l3(pmap, va); | |||||
/* TODO: This is not optimal, but should mostly work */ | |||||
if (l3 == NULL) { | |||||
l2 = pmap_l2(pmap, va); | |||||
if (l2 == NULL) { | |||||
l2_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | |||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | |||||
VM_ALLOC_ZERO); | |||||
if (l2_m == NULL) | |||||
panic("pmap_enter: l2 pte_m == NULL"); | |||||
if ((l2_m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(l2_m); | |||||
l2_pa = VM_PAGE_TO_PHYS(l2_m); | |||||
l1 = pmap_l1(pmap, va); | |||||
pmap_load_store(l1, l2_pa | L1_TABLE); | |||||
PTE_SYNC(l1); | |||||
l2 = pmap_l1_to_l2(l1, va); | |||||
} | |||||
KASSERT(l2 != NULL, | |||||
("No l2 table after allocating one")); | |||||
l3_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | |||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); | |||||
if (l3_m == NULL) | |||||
panic("pmap_enter: l3 pte_m == NULL"); | |||||
if ((l3_m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(l3_m); | |||||
l3_pa = VM_PAGE_TO_PHYS(l3_m); | |||||
pmap_load_store(l2, l3_pa | L2_TABLE); | |||||
PTE_SYNC(l2); | |||||
l3 = pmap_l2_to_l3(l2, va); | |||||
} | |||||
} | |||||
om = NULL; | |||||
orig_l3 = pmap_load(l3); | |||||
opa = orig_l3 & ~ATTR_MASK; | |||||
/* | |||||
* Is the specified virtual address already mapped? | |||||
*/ | |||||
if (pmap_l3_valid(orig_l3)) { | |||||
/* | |||||
* Wiring change, just update stats. We don't worry about | |||||
* wiring PT pages as they remain resident as long as there | |||||
* are valid mappings in them. Hence, if a user page is wired, | |||||
* the PT page will be also. | |||||
*/ | |||||
if ((flags & PMAP_ENTER_WIRED) != 0 && | |||||
(orig_l3 & ATTR_SW_WIRED) == 0) | |||||
pmap->pm_stats.wired_count++; | |||||
else if ((flags & PMAP_ENTER_WIRED) == 0 && | |||||
(orig_l3 & ATTR_SW_WIRED) != 0) | |||||
pmap->pm_stats.wired_count--; | |||||
/* | |||||
* Remove the extra PT page reference. | |||||
*/ | |||||
if (mpte != NULL) { | |||||
mpte->wire_count--; | |||||
KASSERT(mpte->wire_count > 0, | |||||
("pmap_enter: missing reference to page table page," | |||||
" va: 0x%lx", va)); | |||||
} | |||||
/* | |||||
* Has the physical page changed? | |||||
*/ | |||||
if (opa == pa) { | |||||
/* | |||||
* No, might be a protection or wiring change. | |||||
*/ | |||||
if ((orig_l3 & ATTR_SW_MANAGED) != 0) { | |||||
new_l3 |= ATTR_SW_MANAGED; | |||||
if ((new_l3 & ATTR_AP(ATTR_AP_RW)) == | |||||
ATTR_AP(ATTR_AP_RW)) { | |||||
vm_page_aflag_set(m, PGA_WRITEABLE); | |||||
} | |||||
} | |||||
goto validate; | |||||
} | |||||
/* Flush the cache, there might be uncommitted data in it */ | |||||
if (pmap_is_current(pmap) && pmap_l3_valid_cacheable(orig_l3)) | |||||
cpu_dcache_wb_range(va, L3_SIZE); | |||||
} else { | |||||
/* | |||||
* Increment the counters. | |||||
*/ | |||||
if ((new_l3 & ATTR_SW_WIRED) != 0) | |||||
pmap->pm_stats.wired_count++; | |||||
pmap_resident_count_inc(pmap, 1); | |||||
} | |||||
/* | |||||
* Enter on the PV list if part of our managed memory. | |||||
*/ | |||||
if ((m->oflags & VPO_UNMANAGED) == 0) { | |||||
new_l3 |= ATTR_SW_MANAGED; | |||||
pv = get_pv_entry(pmap, &lock); | |||||
pv->pv_va = va; | |||||
CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); | |||||
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | |||||
m->md.pv_gen++; | |||||
if ((new_l3 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) | |||||
vm_page_aflag_set(m, PGA_WRITEABLE); | |||||
} | |||||
/* | |||||
* Update the L3 entry. | |||||
*/ | |||||
if (orig_l3 != 0) { | |||||
validate: | |||||
orig_l3 = pmap_load_store(l3, new_l3); | |||||
PTE_SYNC(l3); | |||||
opa = orig_l3 & ~ATTR_MASK; | |||||
if (opa != pa) { | |||||
if ((orig_l3 & ATTR_SW_MANAGED) != 0) { | |||||
om = PHYS_TO_VM_PAGE(opa); | |||||
if (pmap_page_dirty(orig_l3)) | |||||
vm_page_dirty(om); | |||||
if ((orig_l3 & ATTR_AF) != 0) | |||||
vm_page_aflag_set(om, PGA_REFERENCED); | |||||
CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); | |||||
pmap_pvh_free(&om->md, pmap, va); | |||||
} | |||||
} else if (pmap_page_dirty(orig_l3)) { | |||||
if ((orig_l3 & ATTR_SW_MANAGED) != 0) | |||||
vm_page_dirty(m); | |||||
} | |||||
if ((orig_l3 & ATTR_AF) != 0) | |||||
pmap_invalidate_page(pmap, va); | |||||
} else { | |||||
pmap_load_store(l3, new_l3); | |||||
PTE_SYNC(l3); | |||||
} | |||||
if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap)) | |||||
cpu_icache_sync_range(va, PAGE_SIZE); | |||||
if (lock != NULL) | |||||
rw_wunlock(lock); | |||||
rw_runlock(&pvh_global_lock); | |||||
PMAP_UNLOCK(pmap); | |||||
return (KERN_SUCCESS); | |||||
} | |||||
/* | |||||
* Maps a sequence of resident pages belonging to the same object. | |||||
* The sequence begins with the given page m_start. This page is | |||||
* mapped at the given virtual address start. Each subsequent page is | |||||
* mapped at a virtual address that is offset from start by the same | |||||
* amount as the page is offset from m_start within the object. The | |||||
* last page in the sequence is the page with the largest offset from | |||||
* m_start that can be mapped at a virtual address less than the given | |||||
* virtual address end. Not every virtual page between start and end | |||||
* is mapped; only those for which a resident page exists with the | |||||
* corresponding offset from m_start are mapped. | |||||
*/ | |||||
void | |||||
pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, | |||||
vm_page_t m_start, vm_prot_t prot) | |||||
{ | |||||
struct rwlock *lock; | |||||
vm_offset_t va; | |||||
vm_page_t m, mpte; | |||||
vm_pindex_t diff, psize; | |||||
VM_OBJECT_ASSERT_LOCKED(m_start->object); | |||||
psize = atop(end - start); | |||||
mpte = NULL; | |||||
m = m_start; | |||||
lock = NULL; | |||||
rw_rlock(&pvh_global_lock); | |||||
PMAP_LOCK(pmap); | |||||
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { | |||||
va = start + ptoa(diff); | |||||
mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte, &lock); | |||||
m = TAILQ_NEXT(m, listq); | |||||
} | |||||
if (lock != NULL) | |||||
rw_wunlock(lock); | |||||
rw_runlock(&pvh_global_lock); | |||||
PMAP_UNLOCK(pmap); | |||||
} | |||||
/* | |||||
* this code makes some *MAJOR* assumptions: | |||||
* 1. Current pmap & pmap exists. | |||||
* 2. Not wired. | |||||
* 3. Read access. | |||||
* 4. No page table pages. | |||||
* but is *MUCH* faster than pmap_enter... | |||||
*/ | |||||
void | |||||
pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) | |||||
{ | |||||
struct rwlock *lock; | |||||
lock = NULL; | |||||
rw_rlock(&pvh_global_lock); | |||||
PMAP_LOCK(pmap); | |||||
(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); | |||||
if (lock != NULL) | |||||
rw_wunlock(lock); | |||||
rw_runlock(&pvh_global_lock); | |||||
PMAP_UNLOCK(pmap); | |||||
} | |||||
static vm_page_t | |||||
pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, | |||||
vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp) | |||||
{ | |||||
struct spglist free; | |||||
pd_entry_t *l2; | |||||
pt_entry_t *l3; | |||||
vm_paddr_t pa; | |||||
KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || | |||||
(m->oflags & VPO_UNMANAGED) != 0, | |||||
("pmap_enter_quick_locked: managed mapping within the clean submap")); | |||||
rw_assert(&pvh_global_lock, RA_LOCKED); | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
/* | |||||
* In the case that a page table page is not | |||||
* resident, we are creating it here. | |||||
*/ | |||||
if (va < VM_MAXUSER_ADDRESS) { | |||||
vm_pindex_t l2pindex; | |||||
/* | |||||
* Calculate pagetable page index | |||||
*/ | |||||
l2pindex = pmap_l2_pindex(va); | |||||
if (mpte && (mpte->pindex == l2pindex)) { | |||||
mpte->wire_count++; | |||||
} else { | |||||
/* | |||||
* Get the l2 entry | |||||
*/ | |||||
l2 = pmap_l2(pmap, va); | |||||
/* | |||||
* If the page table page is mapped, we just increment | |||||
* the hold count, and activate it. Otherwise, we | |||||
* attempt to allocate a page table page. If this | |||||
* attempt fails, we don't retry. Instead, we give up. | |||||
*/ | |||||
if (l2 != NULL && *l2 != 0) { | |||||
mpte = PHYS_TO_VM_PAGE(*l2 & ~ATTR_MASK); | |||||
mpte->wire_count++; | |||||
} else { | |||||
/* | |||||
* Pass NULL instead of the PV list lock | |||||
* pointer, because we don't intend to sleep. | |||||
*/ | |||||
mpte = _pmap_alloc_l3(pmap, l2pindex, NULL); | |||||
if (mpte == NULL) | |||||
return (mpte); | |||||
} | |||||
} | |||||
l3 = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); | |||||
l3 = &l3[pmap_l3_index(va)]; | |||||
} else { | |||||
mpte = NULL; | |||||
l3 = pmap_l3(kernel_pmap, va); | |||||
} | |||||
if (l3 == NULL) | |||||
panic("pmap_enter_quick_locked: No l3"); | |||||
if (*l3) { | |||||
if (mpte != NULL) { | |||||
mpte->wire_count--; | |||||
mpte = NULL; | |||||
} | |||||
return (mpte); | |||||
} | |||||
/* | |||||
* Enter on the PV list if part of our managed memory. | |||||
*/ | |||||
if ((m->oflags & VPO_UNMANAGED) == 0 && | |||||
!pmap_try_insert_pv_entry(pmap, va, m, lockp)) { | |||||
if (mpte != NULL) { | |||||
SLIST_INIT(&free); | |||||
if (pmap_unwire_l3(pmap, va, mpte, &free)) { | |||||
pmap_invalidate_page(pmap, va); | |||||
pmap_free_zero_pages(&free); | |||||
} | |||||
mpte = NULL; | |||||
} | |||||
return (mpte); | |||||
} | |||||
/* | |||||
* Increment counters | |||||
*/ | |||||
pmap_resident_count_inc(pmap, 1); | |||||
pa = VM_PAGE_TO_PHYS(m) | ATTR_AF | ATTR_IDX(m->md.pv_memattr) | | |||||
ATTR_AP(ATTR_AP_RW) | L3_PAGE; | |||||
/* | |||||
* Now validate mapping with RO protection | |||||
*/ | |||||
if ((m->oflags & VPO_UNMANAGED) == 0) | |||||
pa |= ATTR_SW_MANAGED; | |||||
pmap_load_store(l3, pa); | |||||
PTE_SYNC(l3); | |||||
pmap_invalidate_page(pmap, va); | |||||
return (mpte); | |||||
} | |||||
/* | |||||
* This code maps large physical mmap regions into the | |||||
* processor address space. Note that some shortcuts | |||||
* are taken, but the code works. | |||||
*/ | |||||
void | |||||
pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, | |||||
vm_pindex_t pindex, vm_size_t size) | |||||
{ | |||||
panic("pmap_object_init_pt"); | |||||
} | |||||
/* | |||||
* Clear the wired attribute from the mappings for the specified range of | |||||
* addresses in the given pmap. Every valid mapping within that range | |||||
* must have the wired attribute set. In contrast, invalid mappings | |||||
* cannot have the wired attribute set, so they are ignored. | |||||
* | |||||
* The wired attribute of the page table entry is not a hardware feature, | |||||
* so there is no need to invalidate any TLB entries. | |||||
*/ | |||||
void | |||||
pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | |||||
{ | |||||
vm_offset_t va_next; | |||||
pd_entry_t *l1, *l2; | |||||
pt_entry_t *l3; | |||||
boolean_t pv_lists_locked; | |||||
pv_lists_locked = FALSE; | |||||
PMAP_LOCK(pmap); | |||||
for (; sva < eva; sva = va_next) { | |||||
l1 = pmap_l1(pmap, sva); | |||||
if (*l1 == 0) { | |||||
va_next = (sva + L1_SIZE) & ~L1_OFFSET; | |||||
if (va_next < sva) | |||||
va_next = eva; | |||||
continue; | |||||
} | |||||
va_next = (sva + L2_SIZE) & ~L2_OFFSET; | |||||
if (va_next < sva) | |||||
va_next = eva; | |||||
l2 = pmap_l1_to_l2(l1, sva); | |||||
if (*l2 == 0) | |||||
continue; | |||||
if (va_next > eva) | |||||
va_next = eva; | |||||
for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, | |||||
sva += L3_SIZE) { | |||||
if (*l3 == 0) | |||||
continue; | |||||
if ((*l3 & ATTR_SW_WIRED) == 0) | |||||
panic("pmap_unwire: l3 %#jx is missing " | |||||
"ATTR_SW_WIRED", (uintmax_t)*l3); | |||||
/* | |||||
* PG_W must be cleared atomically. Although the pmap | |||||
* lock synchronizes access to PG_W, another processor | |||||
* could be setting PG_M and/or PG_A concurrently. | |||||
*/ | |||||
atomic_clear_long(l3, ATTR_SW_WIRED); | |||||
pmap->pm_stats.wired_count--; | |||||
} | |||||
} | |||||
if (pv_lists_locked) | |||||
rw_runlock(&pvh_global_lock); | |||||
PMAP_UNLOCK(pmap); | |||||
} | |||||
/* | |||||
* Copy the range specified by src_addr/len | |||||
* from the source map to the range dst_addr/len | |||||
* in the destination map. | |||||
* | |||||
* This routine is only advisory and need not do anything. | |||||
*/ | |||||
void | |||||
pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, | |||||
vm_offset_t src_addr) | |||||
{ | |||||
} | |||||
/* | |||||
* pmap_zero_page zeros the specified hardware page by mapping | |||||
* the page into KVM and using bzero to clear its contents. | |||||
*/ | |||||
void | |||||
pmap_zero_page(vm_page_t m) | |||||
{ | |||||
vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); | |||||
pagezero((void *)va); | |||||
} | |||||
/* | |||||
* pmap_zero_page_area zeros the specified hardware page by mapping | |||||
* the page into KVM and using bzero to clear its contents. | |||||
* | |||||
* off and size may not cover an area beyond a single hardware page. | |||||
*/ | |||||
void | |||||
pmap_zero_page_area(vm_page_t m, int off, int size) | |||||
{ | |||||
vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); | |||||
if (off == 0 && size == PAGE_SIZE) | |||||
pagezero((void *)va); | |||||
else | |||||
bzero((char *)va + off, size); | |||||
} | |||||
/* | |||||
* pmap_zero_page_idle zeros the specified hardware page by mapping | |||||
* the page into KVM and using bzero to clear its contents. This | |||||
* is intended to be called from the vm_pagezero process only and | |||||
* outside of Giant. | |||||
*/ | |||||
void | |||||
pmap_zero_page_idle(vm_page_t m) | |||||
{ | |||||
vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); | |||||
pagezero((void *)va); | |||||
} | |||||
/* | |||||
* pmap_copy_page copies the specified (machine independent) | |||||
* page by mapping the page into virtual memory and using | |||||
* bcopy to copy the page, one machine dependent page at a | |||||
* time. | |||||
*/ | |||||
void | |||||
pmap_copy_page(vm_page_t msrc, vm_page_t mdst) | |||||
{ | |||||
vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); | |||||
vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); | |||||
pagecopy((void *)src, (void *)dst); | |||||
} | |||||
int unmapped_buf_allowed = 1; | |||||
void | |||||
pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], | |||||
vm_offset_t b_offset, int xfersize) | |||||
{ | |||||
void *a_cp, *b_cp; | |||||
vm_page_t m_a, m_b; | |||||
vm_paddr_t p_a, p_b; | |||||
vm_offset_t a_pg_offset, b_pg_offset; | |||||
int cnt; | |||||
while (xfersize > 0) { | |||||
a_pg_offset = a_offset & PAGE_MASK; | |||||
m_a = ma[a_offset >> PAGE_SHIFT]; | |||||
p_a = m_a->phys_addr; | |||||
b_pg_offset = b_offset & PAGE_MASK; | |||||
m_b = mb[b_offset >> PAGE_SHIFT]; | |||||
p_b = m_b->phys_addr; | |||||
cnt = min(xfersize, PAGE_SIZE - a_pg_offset); | |||||
cnt = min(cnt, PAGE_SIZE - b_pg_offset); | |||||
if (__predict_false(!PHYS_IN_DMAP(p_a))) { | |||||
panic("!DMAP a %lx", p_a); | |||||
} else { | |||||
a_cp = (char *)PHYS_TO_DMAP(p_a) + a_pg_offset; | |||||
} | |||||
if (__predict_false(!PHYS_IN_DMAP(p_b))) { | |||||
panic("!DMAP b %lx", p_b); | |||||
} else { | |||||
b_cp = (char *)PHYS_TO_DMAP(p_b) + b_pg_offset; | |||||
} | |||||
bcopy(a_cp, b_cp, cnt); | |||||
a_offset += cnt; | |||||
b_offset += cnt; | |||||
xfersize -= cnt; | |||||
} | |||||
} | |||||
/* | |||||
* Returns true if the pmap's pv is one of the first | |||||
* 16 pvs linked to from this page. This count may | |||||
* be changed upwards or downwards in the future; it | |||||
* is only necessary that true be returned for a small | |||||
* subset of pmaps for proper page aging. | |||||
*/ | |||||
boolean_t | |||||
pmap_page_exists_quick(pmap_t pmap, vm_page_t m) | |||||
{ | |||||
struct rwlock *lock; | |||||
pv_entry_t pv; | |||||
int loops = 0; | |||||
boolean_t rv; | |||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | |||||
("pmap_page_exists_quick: page %p is not managed", m)); | |||||
rv = FALSE; | |||||
rw_rlock(&pvh_global_lock); | |||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | |||||
rw_rlock(lock); | |||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | |||||
if (PV_PMAP(pv) == pmap) { | |||||
rv = TRUE; | |||||
break; | |||||
} | |||||
loops++; | |||||
if (loops >= 16) | |||||
break; | |||||
} | |||||
rw_runlock(lock); | |||||
rw_runlock(&pvh_global_lock); | |||||
return (rv); | |||||
} | |||||
/* | |||||
* pmap_page_wired_mappings: | |||||
* | |||||
* Return the number of managed mappings to the given physical page | |||||
* that are wired. | |||||
*/ | |||||
int | |||||
pmap_page_wired_mappings(vm_page_t m) | |||||
{ | |||||
struct rwlock *lock; | |||||
pmap_t pmap; | |||||
pt_entry_t *l3; | |||||
pv_entry_t pv; | |||||
int count, md_gen; | |||||
if ((m->oflags & VPO_UNMANAGED) != 0) | |||||
return (0); | |||||
rw_rlock(&pvh_global_lock); | |||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | |||||
rw_rlock(lock); | |||||
restart: | |||||
count = 0; | |||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | |||||
pmap = PV_PMAP(pv); | |||||
if (!PMAP_TRYLOCK(pmap)) { | |||||
md_gen = m->md.pv_gen; | |||||
rw_runlock(lock); | |||||
PMAP_LOCK(pmap); | |||||
rw_rlock(lock); | |||||
if (md_gen != m->md.pv_gen) { | |||||
PMAP_UNLOCK(pmap); | |||||
goto restart; | |||||
} | |||||
} | |||||
l3 = pmap_l3(pmap, pv->pv_va); | |||||
if (l3 != NULL && (*l3 & ATTR_SW_WIRED) != 0) | |||||
count++; | |||||
PMAP_UNLOCK(pmap); | |||||
} | |||||
rw_runlock(lock); | |||||
rw_runlock(&pvh_global_lock); | |||||
return (count); | |||||
} | |||||
/* | |||||
* Destroy all managed, non-wired mappings in the given user-space | |||||
* pmap. This pmap cannot be active on any processor besides the | |||||
* caller. | |||||
* | |||||
* This function cannot be applied to the kernel pmap. Moreover, it | |||||
* is not intended for general use. It is only to be used during | |||||
* process termination. Consequently, it can be implemented in ways | |||||
* that make it faster than pmap_remove(). First, it can more quickly | |||||
* destroy mappings by iterating over the pmap's collection of PV | |||||
* entries, rather than searching the page table. Second, it doesn't | |||||
* have to test and clear the page table entries atomically, because | |||||
* no processor is currently accessing the user address space. In | |||||
* particular, a page table entry's dirty bit won't change state once | |||||
* this function starts. | |||||
*/ | |||||
void | |||||
pmap_remove_pages(pmap_t pmap) | |||||
{ | |||||
pd_entry_t ptepde, *l2; | |||||
pt_entry_t *l3, tl3; | |||||
struct spglist free; | |||||
vm_page_t m; | |||||
pv_entry_t pv; | |||||
struct pv_chunk *pc, *npc; | |||||
struct rwlock *lock; | |||||
int64_t bit; | |||||
uint64_t inuse, bitmask; | |||||
int allfree, field, freed, idx; | |||||
vm_paddr_t pa; | |||||
lock = NULL; | |||||
SLIST_INIT(&free); | |||||
rw_rlock(&pvh_global_lock); | |||||
PMAP_LOCK(pmap); | |||||
TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { | |||||
allfree = 1; | |||||
freed = 0; | |||||
for (field = 0; field < _NPCM; field++) { | |||||
inuse = ~pc->pc_map[field] & pc_freemask[field]; | |||||
while (inuse != 0) { | |||||
bit = ffsl(inuse) - 1; | |||||
bitmask = 1UL << bit; | |||||
idx = field * 64 + bit; | |||||
pv = &pc->pc_pventry[idx]; | |||||
inuse &= ~bitmask; | |||||
l2 = pmap_l2(pmap, pv->pv_va); | |||||
ptepde = pmap_load(l2); | |||||
l3 = pmap_l2_to_l3(l2, pv->pv_va); | |||||
tl3 = pmap_load(l3); | |||||
/* | |||||
* We cannot remove wired pages from a process' mapping at this time | |||||
*/ | |||||
if (tl3 & ATTR_SW_WIRED) { | |||||
allfree = 0; | |||||
continue; | |||||
} | |||||
pa = tl3 & ~ATTR_MASK; | |||||
m = PHYS_TO_VM_PAGE(pa); | |||||
KASSERT(m->phys_addr == pa, | |||||
("vm_page_t %p phys_addr mismatch %016jx %016jx", | |||||
m, (uintmax_t)m->phys_addr, | |||||
(uintmax_t)tl3)); | |||||
KASSERT((m->flags & PG_FICTITIOUS) != 0 || | |||||
m < &vm_page_array[vm_page_array_size], | |||||
("pmap_remove_pages: bad l3 %#jx", | |||||
(uintmax_t)tl3)); | |||||
if (pmap_is_current(pmap) && | |||||
pmap_l3_valid_cacheable(pmap_load(l3))) | |||||
cpu_dcache_wb_range(pv->pv_va, L3_SIZE); | |||||
pmap_load_clear(l3); | |||||
PTE_SYNC(l3); | |||||
/* | |||||
* Update the vm_page_t clean/reference bits. | |||||
*/ | |||||
if ((tl3 & ATTR_AP_RW_BIT) == | |||||
ATTR_AP(ATTR_AP_RW)) | |||||
vm_page_dirty(m); | |||||
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); | |||||
/* Mark free */ | |||||
pc->pc_map[field] |= bitmask; | |||||
pmap_resident_count_dec(pmap, 1); | |||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | |||||
m->md.pv_gen++; | |||||
pmap_unuse_l3(pmap, pv->pv_va, ptepde, &free); | |||||
freed++; | |||||
} | |||||
} | |||||
PV_STAT(atomic_add_long(&pv_entry_frees, freed)); | |||||
PV_STAT(atomic_add_int(&pv_entry_spare, freed)); | |||||
PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); | |||||
if (allfree) { | |||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | |||||
free_pv_chunk(pc); | |||||
} | |||||
} | |||||
pmap_invalidate_all(pmap); | |||||
if (lock != NULL) | |||||
rw_wunlock(lock); | |||||
rw_runlock(&pvh_global_lock); | |||||
PMAP_UNLOCK(pmap); | |||||
pmap_free_zero_pages(&free); | |||||
} | |||||
/* | |||||
* This is used to check if a page has been accessed or modified. As we | |||||
* don't have a bit to see if it has been modified we have to assume it | |||||
* has been if the page is read/write. | |||||
*/ | |||||
static boolean_t | |||||
pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) | |||||
{ | |||||
struct rwlock *lock; | |||||
pv_entry_t pv; | |||||
pt_entry_t *l3, mask, value; | |||||
pmap_t pmap; | |||||
int md_gen; | |||||
boolean_t rv; | |||||
rv = FALSE; | |||||
rw_rlock(&pvh_global_lock); | |||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | |||||
rw_rlock(lock); | |||||
restart: | |||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | |||||
pmap = PV_PMAP(pv); | |||||
if (!PMAP_TRYLOCK(pmap)) { | |||||
md_gen = m->md.pv_gen; | |||||
rw_runlock(lock); | |||||
PMAP_LOCK(pmap); | |||||
rw_rlock(lock); | |||||
if (md_gen != m->md.pv_gen) { | |||||
PMAP_UNLOCK(pmap); | |||||
goto restart; | |||||
} | |||||
} | |||||
l3 = pmap_l3(pmap, pv->pv_va); | |||||
mask = 0; | |||||
value = 0; | |||||
if (modified) { | |||||
mask |= ATTR_AP_RW_BIT; | |||||
value |= ATTR_AP(ATTR_AP_RW); | |||||
} | |||||
if (accessed) { | |||||
mask |= ATTR_AF | ATTR_DESCR_MASK; | |||||
value |= ATTR_AF | L3_PAGE; | |||||
} | |||||
rv = (pmap_load(l3) & mask) == value; | |||||
PMAP_UNLOCK(pmap); | |||||
if (rv) | |||||
goto out; | |||||
} | |||||
out: | |||||
rw_runlock(lock); | |||||
rw_runlock(&pvh_global_lock); | |||||
return (rv); | |||||
} | |||||
/* | |||||
* pmap_is_modified: | |||||
* | |||||
* Return whether or not the specified physical page was modified | |||||
* in any physical maps. | |||||
*/ | |||||
boolean_t | |||||
pmap_is_modified(vm_page_t m) | |||||
{ | |||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | |||||
("pmap_is_modified: page %p is not managed", m)); | |||||
/* | |||||
* If the page is not exclusive busied, then PGA_WRITEABLE cannot be | |||||
* concurrently set while the object is locked. Thus, if PGA_WRITEABLE | |||||
* is clear, no PTEs can have PG_M set. | |||||
*/ | |||||
VM_OBJECT_ASSERT_WLOCKED(m->object); | |||||
if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) | |||||
return (FALSE); | |||||
return (pmap_page_test_mappings(m, FALSE, TRUE)); | |||||
} | |||||
/* | |||||
* pmap_is_prefaultable: | |||||
* | |||||
* Return whether or not the specified virtual address is eligible | |||||
* for prefault. | |||||
*/ | |||||
boolean_t | |||||
pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) | |||||
{ | |||||
pt_entry_t *l3; | |||||
boolean_t rv; | |||||
rv = FALSE; | |||||
PMAP_LOCK(pmap); | |||||
l3 = pmap_l3(pmap, addr); | |||||
if (l3 != NULL && *l3 != 0) { | |||||
rv = TRUE; | |||||
} | |||||
PMAP_UNLOCK(pmap); | |||||
return (rv); | |||||
} | |||||
/* | |||||
* pmap_is_referenced: | |||||
* | |||||
* Return whether or not the specified physical page was referenced | |||||
* in any physical maps. | |||||
*/ | |||||
boolean_t | |||||
pmap_is_referenced(vm_page_t m) | |||||
{ | |||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | |||||
("pmap_is_referenced: page %p is not managed", m)); | |||||
return (pmap_page_test_mappings(m, TRUE, FALSE)); | |||||
} | |||||
/* | |||||
* Clear the write and modified bits in each of the given page's mappings. | |||||
*/ | |||||
void | |||||
pmap_remove_write(vm_page_t m) | |||||
{ | |||||
pmap_t pmap; | |||||
struct rwlock *lock; | |||||
pv_entry_t pv; | |||||
pt_entry_t *l3, oldl3; | |||||
int md_gen; | |||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | |||||
("pmap_remove_write: page %p is not managed", m)); | |||||
/* | |||||
* If the page is not exclusive busied, then PGA_WRITEABLE cannot be | |||||
* set by another thread while the object is locked. Thus, | |||||
* if PGA_WRITEABLE is clear, no page table entries need updating. | |||||
*/ | |||||
VM_OBJECT_ASSERT_WLOCKED(m->object); | |||||
if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) | |||||
return; | |||||
rw_rlock(&pvh_global_lock); | |||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | |||||
retry_pv_loop: | |||||
rw_wlock(lock); | |||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | |||||
pmap = PV_PMAP(pv); | |||||
if (!PMAP_TRYLOCK(pmap)) { | |||||
md_gen = m->md.pv_gen; | |||||
rw_wunlock(lock); | |||||
PMAP_LOCK(pmap); | |||||
rw_wlock(lock); | |||||
if (md_gen != m->md.pv_gen) { | |||||
PMAP_UNLOCK(pmap); | |||||
rw_wunlock(lock); | |||||
goto retry_pv_loop; | |||||
} | |||||
} | |||||
l3 = pmap_l3(pmap, pv->pv_va); | |||||
retry: | |||||
oldl3 = *l3; | |||||
if ((oldl3 & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) { | |||||
if (!atomic_cmpset_long(l3, oldl3, | |||||
oldl3 | ATTR_AP(ATTR_AP_RO))) | |||||
goto retry; | |||||
if ((oldl3 & ATTR_AF) != 0) | |||||
vm_page_dirty(m); | |||||
pmap_invalidate_page(pmap, pv->pv_va); | |||||
} | |||||
PMAP_UNLOCK(pmap); | |||||
} | |||||
rw_wunlock(lock); | |||||
vm_page_aflag_clear(m, PGA_WRITEABLE); | |||||
rw_runlock(&pvh_global_lock); | |||||
} | |||||
static __inline boolean_t | |||||
safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte) | |||||
{ | |||||
return (FALSE); | |||||
} | |||||
#define PMAP_TS_REFERENCED_MAX 5 | |||||
/* | |||||
* pmap_ts_referenced: | |||||
* | |||||
* Return a count of reference bits for a page, clearing those bits. | |||||
* It is not necessary for every reference bit to be cleared, but it | |||||
* is necessary that 0 only be returned when there are truly no | |||||
* reference bits set. | |||||
* | |||||
* XXX: The exact number of bits to check and clear is a matter that | |||||
* should be tested and standardized at some point in the future for | |||||
* optimal aging of shared pages. | |||||
*/ | |||||
int | |||||
pmap_ts_referenced(vm_page_t m) | |||||
{ | |||||
pv_entry_t pv, pvf; | |||||
pmap_t pmap; | |||||
struct rwlock *lock; | |||||
pd_entry_t *l2; | |||||
pt_entry_t *l3; | |||||
vm_paddr_t pa; | |||||
int cleared, md_gen, not_cleared; | |||||
struct spglist free; | |||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | |||||
("pmap_ts_referenced: page %p is not managed", m)); | |||||
SLIST_INIT(&free); | |||||
cleared = 0; | |||||
pa = VM_PAGE_TO_PHYS(m); | |||||
lock = PHYS_TO_PV_LIST_LOCK(pa); | |||||
rw_rlock(&pvh_global_lock); | |||||
rw_wlock(lock); | |||||
retry: | |||||
not_cleared = 0; | |||||
if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) | |||||
goto out; | |||||
pv = pvf; | |||||
do { | |||||
if (pvf == NULL) | |||||
pvf = pv; | |||||
pmap = PV_PMAP(pv); | |||||
if (!PMAP_TRYLOCK(pmap)) { | |||||
md_gen = m->md.pv_gen; | |||||
rw_wunlock(lock); | |||||
PMAP_LOCK(pmap); | |||||
rw_wlock(lock); | |||||
if (md_gen != m->md.pv_gen) { | |||||
PMAP_UNLOCK(pmap); | |||||
goto retry; | |||||
} | |||||
} | |||||
l2 = pmap_l2(pmap, pv->pv_va); | |||||
KASSERT((*l2 & ATTR_DESCR_MASK) == L2_TABLE, | |||||
("pmap_ts_referenced: found an invalid l2 table")); | |||||
l3 = pmap_l2_to_l3(l2, pv->pv_va); | |||||
if ((*l3 & ATTR_AF) != 0) { | |||||
if (safe_to_clear_referenced(pmap, *l3)) { | |||||
/* | |||||
* TODO: We don't handle the access flag | |||||
* at all. We need to be able to set it in | |||||
* the exception handler. | |||||
*/ | |||||
panic("TODO: safe_to_clear_referenced\n"); | |||||
} else if ((*l3 & ATTR_SW_WIRED) == 0) { | |||||
/* | |||||
* Wired pages cannot be paged out so | |||||
* doing accessed bit emulation for | |||||
* them is wasted effort. We do the | |||||
* hard work for unwired pages only. | |||||
*/ | |||||
pmap_remove_l3(pmap, l3, pv->pv_va, | |||||
*l2, &free, &lock); | |||||
pmap_invalidate_page(pmap, pv->pv_va); | |||||
cleared++; | |||||
if (pvf == pv) | |||||
pvf = NULL; | |||||
pv = NULL; | |||||
KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), | |||||
("inconsistent pv lock %p %p for page %p", | |||||
lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); | |||||
} else | |||||
not_cleared++; | |||||
} | |||||
PMAP_UNLOCK(pmap); | |||||
/* Rotate the PV list if it has more than one entry. */ | |||||
if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { | |||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | |||||
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | |||||
m->md.pv_gen++; | |||||
} | |||||
} while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + | |||||
not_cleared < PMAP_TS_REFERENCED_MAX); | |||||
out: | |||||
rw_wunlock(lock); | |||||
rw_runlock(&pvh_global_lock); | |||||
pmap_free_zero_pages(&free); | |||||
return (cleared + not_cleared); | |||||
} | |||||
/* | |||||
* Apply the given advice to the specified range of addresses within the | |||||
* given pmap. Depending on the advice, clear the referenced and/or | |||||
* modified flags in each mapping and set the mapped page's dirty field. | |||||
*/ | |||||
void | |||||
pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) | |||||
{ | |||||
} | |||||
/* | |||||
* Clear the modify bits on the specified physical page. | |||||
*/ | |||||
void | |||||
pmap_clear_modify(vm_page_t m) | |||||
{ | |||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | |||||
("pmap_clear_modify: page %p is not managed", m)); | |||||
VM_OBJECT_ASSERT_WLOCKED(m->object); | |||||
KASSERT(!vm_page_xbusied(m), | |||||
("pmap_clear_modify: page %p is exclusive busied", m)); | |||||
/* | |||||
* If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. | |||||
* If the object containing the page is locked and the page is not | |||||
* exclusive busied, then PGA_WRITEABLE cannot be concurrently set. | |||||
*/ | |||||
if ((m->aflags & PGA_WRITEABLE) == 0) | |||||
return; | |||||
panic("pmap_clear_modify"); | |||||
} | |||||
/* | |||||
* Sets the memory attribute for the specified page. | |||||
*/ | |||||
void | |||||
pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) | |||||
{ | |||||
panic("pmap_page_set_memattr"); | |||||
} | |||||
/* | |||||
* perform the pmap work for mincore | |||||
*/ | |||||
int | |||||
pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) | |||||
{ | |||||
panic("pmap_mincore"); | |||||
} | |||||
void | |||||
pmap_activate(struct thread *td) | |||||
{ | |||||
pmap_t pmap; | |||||
critical_enter(); | |||||
pmap = vmspace_pmap(td->td_proc->p_vmspace); | |||||
td->td_pcb->pcb_l1addr = vtophys(pmap->pm_l1); | |||||
__asm __volatile("msr ttbr0_el1, %0" : : "r"(td->td_pcb->pcb_l1addr)); | |||||
critical_exit(); | |||||
} | |||||
void | |||||
pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) | |||||
{ | |||||
panic("pmap_sync_icache"); | |||||
} | |||||
/* | |||||
* Increase the starting virtual address of the given mapping if a | |||||
* different alignment might result in more superpage mappings. | |||||
*/ | |||||
void | |||||
pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, | |||||
vm_offset_t *addr, vm_size_t size) | |||||
{ | |||||
} | |||||
/** | |||||
* Get the kernel virtual address of a set of physical pages. If there are | |||||
* physical addresses not covered by the DMAP perform a transient mapping | |||||
* that will be removed when calling pmap_unmap_io_transient. | |||||
* | |||||
* \param page The pages the caller wishes to obtain the virtual | |||||
* address on the kernel memory map. | |||||
* \param vaddr On return contains the kernel virtual memory address | |||||
* of the pages passed in the page parameter. | |||||
* \param count Number of pages passed in. | |||||
* \param can_fault TRUE if the thread using the mapped pages can take | |||||
* page faults, FALSE otherwise. | |||||
* | |||||
* \returns TRUE if the caller must call pmap_unmap_io_transient when | |||||
* finished or FALSE otherwise. | |||||
* | |||||
*/ | |||||
boolean_t | |||||
pmap_map_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, | |||||
boolean_t can_fault) | |||||
{ | |||||
vm_paddr_t paddr; | |||||
boolean_t needs_mapping; | |||||
int error, i; | |||||
/* | |||||
* Allocate any KVA space that we need, this is done in a separate | |||||
* loop to prevent calling vmem_alloc while pinned. | |||||
*/ | |||||
needs_mapping = FALSE; | |||||
for (i = 0; i < count; i++) { | |||||
paddr = VM_PAGE_TO_PHYS(page[i]); | |||||
if (__predict_false(paddr >= DMAP_MAX_PHYSADDR)) { | |||||
error = vmem_alloc(kernel_arena, PAGE_SIZE, | |||||
M_BESTFIT | M_WAITOK, &vaddr[i]); | |||||
KASSERT(error == 0, ("vmem_alloc failed: %d", error)); | |||||
needs_mapping = TRUE; | |||||
} else { | |||||
vaddr[i] = PHYS_TO_DMAP(paddr); | |||||
} | |||||
} | |||||
/* Exit early if everything is covered by the DMAP */ | |||||
if (!needs_mapping) | |||||
return (FALSE); | |||||
/* | |||||
* NB: The sequence of updating a page table followed by accesses | |||||
* to the corresponding pages used in the !DMAP case is subject to | |||||
* the situation described in the "AMD64 Architecture Programmer's | |||||
* Manual Volume 2: System Programming" rev. 3.23, "7.3.1 Special | |||||
* Coherency Considerations". Therefore, issuing the INVLPG right | |||||
* after modifying the PTE bits is crucial. | |||||
*/ | |||||
if (!can_fault) | |||||
sched_pin(); | |||||
for (i = 0; i < count; i++) { | |||||
paddr = VM_PAGE_TO_PHYS(page[i]); | |||||
if (paddr >= DMAP_MAX_PHYSADDR) { | |||||
panic( | |||||
"pmap_map_io_transient: TODO: Map out of DMAP data"); | |||||
} | |||||
} | |||||
return (needs_mapping); | |||||
} | |||||
void | |||||
pmap_unmap_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, | |||||
boolean_t can_fault) | |||||
{ | |||||
vm_paddr_t paddr; | |||||
int i; | |||||
if (!can_fault) | |||||
sched_unpin(); | |||||
for (i = 0; i < count; i++) { | |||||
paddr = VM_PAGE_TO_PHYS(page[i]); | |||||
if (paddr >= DMAP_MAX_PHYSADDR) { | |||||
panic("pmap_unmap_io_transient: TODO: Unmap data"); | |||||
} | |||||
} | |||||
} |
What is left from other archs?