Changeset View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 223 Lines • ▼ Show 20 Lines | |||||
/* This code assumes all L1 DMAP entries will be used */ | /* This code assumes all L1 DMAP entries will be used */ | ||||
CTASSERT((DMAP_MIN_ADDRESS & ~L0_OFFSET) == DMAP_MIN_ADDRESS); | CTASSERT((DMAP_MIN_ADDRESS & ~L0_OFFSET) == DMAP_MIN_ADDRESS); | ||||
CTASSERT((DMAP_MAX_ADDRESS & ~L0_OFFSET) == DMAP_MAX_ADDRESS); | CTASSERT((DMAP_MAX_ADDRESS & ~L0_OFFSET) == DMAP_MAX_ADDRESS); | ||||
#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) | #define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) | ||||
extern pt_entry_t pagetable_dmap[]; | extern pt_entry_t pagetable_dmap[]; | ||||
static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); | |||||
static int superpages_enabled = 1; | |||||
SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, | |||||
CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &superpages_enabled, 0, | |||||
"Are large page mappings enabled?"); | |||||
/* | /* | ||||
* Data for the pv entry allocation mechanism | * Data for the pv entry allocation mechanism | ||||
*/ | */ | ||||
static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); | static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); | ||||
static struct mtx pv_chunks_mutex; | static struct mtx pv_chunks_mutex; | ||||
static struct rwlock pv_list_locks[NPV_LIST_LOCKS]; | static struct rwlock pv_list_locks[NPV_LIST_LOCKS]; | ||||
static void free_pv_chunk(struct pv_chunk *pc); | static void free_pv_chunk(struct pv_chunk *pc); | ||||
static void free_pv_entry(pmap_t pmap, pv_entry_t pv); | static void free_pv_entry(pmap_t pmap, pv_entry_t pv); | ||||
static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); | static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); | ||||
static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); | static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); | ||||
static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); | static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); | ||||
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, | static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, | ||||
vm_offset_t va); | vm_offset_t va); | ||||
static int pmap_change_attr(vm_offset_t va, vm_size_t size, int mode); | |||||
static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode); | |||||
static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va); | |||||
static pt_entry_t *pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va); | |||||
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, | static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, | ||||
vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); | vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); | ||||
static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva, | static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva, | ||||
pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); | pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); | ||||
static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, | static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, | ||||
vm_page_t m, struct rwlock **lockp); | vm_page_t m, struct rwlock **lockp); | ||||
static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, | static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, | ||||
▲ Show 20 Lines • Show All 163 Lines • ▼ Show 20 Lines | pmap_pte(pmap_t pmap, vm_offset_t va, int *level) | ||||
*level = 3; | *level = 3; | ||||
l3 = pmap_l2_to_l3(l2, va); | l3 = pmap_l2_to_l3(l2, va); | ||||
if ((pmap_load(l3) & ATTR_DESCR_MASK) != L3_PAGE) | if ((pmap_load(l3) & ATTR_DESCR_MASK) != L3_PAGE) | ||||
return (NULL); | return (NULL); | ||||
return (l3); | return (l3); | ||||
} | } | ||||
static inline bool | |||||
pmap_superpages_enabled(void) | |||||
{ | |||||
return (superpages_enabled != 0); | |||||
} | |||||
bool | bool | ||||
pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l0, pd_entry_t **l1, | pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l0, pd_entry_t **l1, | ||||
pd_entry_t **l2, pt_entry_t **l3) | pd_entry_t **l2, pt_entry_t **l3) | ||||
{ | { | ||||
pd_entry_t *l0p, *l1p, *l2p; | pd_entry_t *l0p, *l1p, *l2p; | ||||
if (pmap->pm_l0 == NULL) | if (pmap->pm_l0 == NULL) | ||||
return (false); | return (false); | ||||
▲ Show 20 Lines • Show All 401 Lines • ▼ Show 20 Lines | |||||
* system needs to map virtual memory. | * system needs to map virtual memory. | ||||
*/ | */ | ||||
void | void | ||||
pmap_init(void) | pmap_init(void) | ||||
{ | { | ||||
int i; | int i; | ||||
/* | /* | ||||
* Are large page mappings enabled? | |||||
*/ | |||||
TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled); | |||||
kib: On amd64/i386, the same knob is called vm.pmap.pg_ps_enabled. Not sure if it is required to… | |||||
Not Done Inline ActionsIt looked like this was named after the flag in the pagetables. On armv6 is seems to be vm.pmap.sp_enabled. I don't have an opinion on the correct name. andrew: It looked like this was named after the flag in the pagetables. On armv6 is seems to be `vm. | |||||
/* | |||||
* Initialize the pv chunk list mutex. | * Initialize the pv chunk list mutex. | ||||
*/ | */ | ||||
mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); | mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); | ||||
/* | /* | ||||
* Initialize the pool of pv list locks. | * Initialize the pool of pv list locks. | ||||
*/ | */ | ||||
for (i = 0; i < NPV_LIST_LOCKS; i++) | for (i = 0; i < NPV_LIST_LOCKS; i++) | ||||
▲ Show 20 Lines • Show All 1,149 Lines • ▼ Show 20 Lines | if (va_next < sva) | ||||
va_next = eva; | va_next = eva; | ||||
l2 = pmap_l1_to_l2(l1, sva); | l2 = pmap_l1_to_l2(l1, sva); | ||||
if (l2 == NULL) | if (l2 == NULL) | ||||
continue; | continue; | ||||
l3_paddr = pmap_load(l2); | l3_paddr = pmap_load(l2); | ||||
if ((l3_paddr & ATTR_DESCR_MASK) == L2_BLOCK) { | |||||
KASSERT((l3_paddr & ATTR_SW_MANAGED) == 0, | |||||
("%s: TODO: Demote managed pages", __func__)); | |||||
pmap_demote_l2(pmap, l2, sva & ~L2_OFFSET); | |||||
l3_paddr = pmap_load(l2); | |||||
} | |||||
/* | /* | ||||
* Weed out invalid mappings. | * Weed out invalid mappings. | ||||
*/ | */ | ||||
if ((l3_paddr & ATTR_DESCR_MASK) != L2_TABLE) | if ((l3_paddr & ATTR_DESCR_MASK) != L2_TABLE) | ||||
continue; | continue; | ||||
/* | /* | ||||
* Limit our scan to either the end of the va represented | * Limit our scan to either the end of the va represented | ||||
▲ Show 20 Lines • Show All 178 Lines • ▼ Show 20 Lines | pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) | ||||
} | } | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
/* TODO: Only invalidate entries we are touching */ | /* TODO: Only invalidate entries we are touching */ | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
} | } | ||||
/* | /* | ||||
* Performs a break-before-make update of a pmap entry. This is needed when | |||||
* either promoting or demoting pages to ensure the TLB doesn't get into an | |||||
* inconsistent state. | |||||
*/ | |||||
static void | |||||
pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_entry_t newpte, | |||||
vm_offset_t va) | |||||
{ | |||||
register_t intr; | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
/* | |||||
* Ensure we don't get switched out with the page table in an | |||||
* inconsistent state. We also need to ensure no interrupts fire | |||||
* as they may make use of an address we are about to invalidate. | |||||
*/ | |||||
intr = intr_disable(); | |||||
critical_enter(); | |||||
/* Clear the old mapping */ | |||||
pmap_load_clear(pte); | |||||
PTE_SYNC(pte); | |||||
pmap_invalidate_page(pmap, va); | |||||
/* Create the new mapping */ | |||||
pmap_load_store(pte, newpte); | |||||
PTE_SYNC(pte); | |||||
critical_exit(); | |||||
intr_restore(intr); | |||||
} | |||||
/* | |||||
* Tries to promote the 512, contiguous 4KB page mappings that are within a | |||||
* single level 2 table entry to a single 2MB page mapping. For promotion | |||||
* to occur, two conditions must be met: (1) the 4KB page mappings must map | |||||
* aligned, contiguous physical memory and (2) the 4KB page mappings must have | |||||
* identical characteristics. | |||||
*/ | |||||
static void | |||||
pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, | |||||
struct rwlock **lockp) | |||||
{ | |||||
pt_entry_t *firstl3, *l3, newl2, oldl3, pa; | |||||
register_t intr; | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
firstl3 = (pt_entry_t *)PHYS_TO_DMAP(pmap_load(l2) & ~ATTR_MASK); | |||||
newl2 = pmap_load(firstl3); | |||||
/* Ignore managed pages for now */ | |||||
if ((newl2 & ATTR_SW_MANAGED) != 0) | |||||
return; | |||||
/* Check the alingment is valid */ | |||||
if (((newl2 & ~ATTR_MASK) & L2_OFFSET) != 0) | |||||
return; | |||||
pa = newl2 + L2_SIZE - PAGE_SIZE; | |||||
for (l3 = firstl3 + NL3PG - 1; l3 > firstl3; l3--) { | |||||
oldl3 = pmap_load(l3); | |||||
if (oldl3 != pa) | |||||
return; | |||||
pa -= PAGE_SIZE; | |||||
} | |||||
newl2 &= ~ATTR_DESCR_MASK; | |||||
newl2 |= L2_BLOCK; | |||||
/* | |||||
* Ensure we don't get switched out with the page table in an | |||||
* inconsistent state. We also need to ensure no interrupts fire | |||||
* as they may make use of an address we are about to invalidate. | |||||
*/ | |||||
intr = intr_disable(); | |||||
critical_enter(); | |||||
/* Clear the old mapping */ | |||||
pmap_load_clear(l2); | |||||
PTE_SYNC(l2); | |||||
pmap_invalidate_range(pmap, rounddown2(va, L2_SIZE), | |||||
roundup2(va, L2_SIZE)); | |||||
/* Create the new mapping */ | |||||
pmap_load_store(l2, newl2); | |||||
PTE_SYNC(l2); | |||||
critical_exit(); | |||||
intr_restore(intr); | |||||
} | |||||
/* | |||||
* Insert the given physical page (p) at | * Insert the given physical page (p) at | ||||
* the specified virtual address (v) in the | * the specified virtual address (v) in the | ||||
* target physical map with the protection requested. | * target physical map with the protection requested. | ||||
* | * | ||||
* If specified, the page will be wired down, meaning | * If specified, the page will be wired down, meaning | ||||
* that the related pte can not be reclaimed. | * that the related pte can not be reclaimed. | ||||
* | * | ||||
* NB: This is the only routine which MAY NOT lazy-evaluate | * NB: This is the only routine which MAY NOT lazy-evaluate | ||||
▲ Show 20 Lines • Show All 190 Lines • ▼ Show 20 Lines | pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
} | } | ||||
/* | /* | ||||
* Update the L3 entry. | * Update the L3 entry. | ||||
*/ | */ | ||||
if (orig_l3 != 0) { | if (orig_l3 != 0) { | ||||
validate: | validate: | ||||
orig_l3 = pmap_load_store(l3, new_l3); | orig_l3 = pmap_load_store(l3, new_l3); | ||||
PTE_SYNC(l3); | |||||
opa = orig_l3 & ~ATTR_MASK; | opa = orig_l3 & ~ATTR_MASK; | ||||
if (opa != pa) { | if (opa != pa) { | ||||
if ((orig_l3 & ATTR_SW_MANAGED) != 0) { | if ((orig_l3 & ATTR_SW_MANAGED) != 0) { | ||||
om = PHYS_TO_VM_PAGE(opa); | om = PHYS_TO_VM_PAGE(opa); | ||||
if (pmap_page_dirty(orig_l3)) | if (pmap_page_dirty(orig_l3)) | ||||
vm_page_dirty(om); | vm_page_dirty(om); | ||||
if ((orig_l3 & ATTR_AF) != 0) | if ((orig_l3 & ATTR_AF) != 0) | ||||
vm_page_aflag_set(om, PGA_REFERENCED); | vm_page_aflag_set(om, PGA_REFERENCED); | ||||
CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); | CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); | ||||
pmap_pvh_free(&om->md, pmap, va); | pmap_pvh_free(&om->md, pmap, va); | ||||
} | } | ||||
} else if (pmap_page_dirty(orig_l3)) { | } else if (pmap_page_dirty(orig_l3)) { | ||||
if ((orig_l3 & ATTR_SW_MANAGED) != 0) | if ((orig_l3 & ATTR_SW_MANAGED) != 0) | ||||
vm_page_dirty(m); | vm_page_dirty(m); | ||||
} | } | ||||
} else { | } else { | ||||
pmap_load_store(l3, new_l3); | pmap_load_store(l3, new_l3); | ||||
PTE_SYNC(l3); | |||||
} | } | ||||
PTE_SYNC(l3); | |||||
pmap_invalidate_page(pmap, va); | pmap_invalidate_page(pmap, va); | ||||
if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap)) | if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap)) | ||||
cpu_icache_sync_range(va, PAGE_SIZE); | cpu_icache_sync_range(va, PAGE_SIZE); | ||||
/* XXX: Not yet, not all demotions are handled */ | |||||
#if 0 | |||||
if ((mpte == NULL || mpte->wire_count == NL3PG) && | |||||
pmap_superpages_enabled() && (m->flags & PG_FICTITIOUS) == 0 && | |||||
vm_reserv_level_iffullpop(m) == 0) { | |||||
KASSERT(lvl == 2, ("Invalid pde level %d", lvl)); | |||||
pmap_promote_l2(pmap, pde, va, &lock); | |||||
} | |||||
#endif | |||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
return (KERN_SUCCESS); | return (KERN_SUCCESS); | ||||
} | } | ||||
/* | /* | ||||
* Maps a sequence of resident pages belonging to the same object. | * Maps a sequence of resident pages belonging to the same object. | ||||
▲ Show 20 Lines • Show All 899 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
void | void | ||||
pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) | pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) | ||||
{ | { | ||||
m->md.pv_memattr = ma; | m->md.pv_memattr = ma; | ||||
/* | /* | ||||
* ARM64TODO: Implement the below (from the amd64 pmap) | |||||
* If "m" is a normal page, update its direct mapping. This update | * If "m" is a normal page, update its direct mapping. This update | ||||
* can be relied upon to perform any cache operations that are | * can be relied upon to perform any cache operations that are | ||||
* required for data coherence. | * required for data coherence. | ||||
*/ | */ | ||||
if ((m->flags & PG_FICTITIOUS) == 0 && | if ((m->flags & PG_FICTITIOUS) == 0 && | ||||
PHYS_IN_DMAP(VM_PAGE_TO_PHYS(m))) | pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), PAGE_SIZE, | ||||
panic("ARM64TODO: pmap_page_set_memattr"); | m->md.pv_memattr) != 0) | ||||
panic("memory attribute change on the direct map failed"); | |||||
} | } | ||||
Not Done Inline ActionsThe given page can be mapped by multiple times, but attributes are changed only in DMAP. ARM doesn't allows multiple mapping with different page attributes. mmel: The given page can be mapped by multiple times, but attributes are changed only in DMAP. ARM… | |||||
Not Done Inline ActionsHow would this happen? My understanding is the VM code will only call this with unmapped pages. andrew: How would this happen? My understanding is the VM code will only call this with unmapped pages. | |||||
/* | /* | ||||
* Changes the specified virtual address range's memory type to that given by | |||||
* the parameter "mode". The specified virtual address range must be | |||||
* completely contained within either the direct map or the kernel map. If | |||||
* the virtual address range is contained within the kernel map, then the | |||||
* memory type for each of the corresponding ranges of the direct map is also | |||||
* changed. (The corresponding ranges of the direct map are those ranges that | |||||
* map the same physical pages as the specified virtual address range.) These | |||||
* changes to the direct map are necessary because Intel describes the | |||||
* behavior of their processors as "undefined" if two or more mappings to the | |||||
* same physical page have different memory types. | |||||
* | |||||
* Returns zero if the change completed successfully, and either EINVAL or | |||||
* ENOMEM if the change failed. Specifically, EINVAL is returned if some part | |||||
* of the virtual address range was not mapped, and ENOMEM is returned if | |||||
* there was insufficient memory available to complete the change. In the | |||||
* latter case, the memory type may have been changed on some part of the | |||||
* virtual address range or the direct map. | |||||
*/ | |||||
static int | |||||
pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) | |||||
{ | |||||
int error; | |||||
PMAP_LOCK(kernel_pmap); | |||||
error = pmap_change_attr_locked(va, size, mode); | |||||
PMAP_UNLOCK(kernel_pmap); | |||||
return (error); | |||||
} | |||||
static int | |||||
pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode) | |||||
{ | |||||
vm_offset_t base, offset, tmpva; | |||||
pt_entry_t l3, *pte, *newpte; | |||||
int lvl; | |||||
PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); | |||||
base = trunc_page(va); | |||||
offset = va & PAGE_MASK; | |||||
size = round_page(offset + size); | |||||
if (!VIRT_IN_DMAP(base)) | |||||
return (EINVAL); | |||||
for (tmpva = base; tmpva < base + size; ) { | |||||
pte = pmap_pte(kernel_pmap, va, &lvl); | |||||
if (pte == NULL) | |||||
return (EINVAL); | |||||
if ((pmap_load(pte) & ATTR_IDX_MASK) == ATTR_IDX(mode)) { | |||||
/* | |||||
* We already have the correct attribute, | |||||
* ignore this entry. | |||||
*/ | |||||
switch (lvl) { | |||||
default: | |||||
panic("Invalid DMAP table level: %d\n", lvl); | |||||
case 1: | |||||
tmpva = (tmpva & ~L1_OFFSET) + L1_SIZE; | |||||
break; | |||||
case 2: | |||||
tmpva = (tmpva & ~L2_OFFSET) + L2_SIZE; | |||||
break; | |||||
case 3: | |||||
tmpva += PAGE_SIZE; | |||||
break; | |||||
} | |||||
} else { | |||||
/* | |||||
* Split the entry to an level 3 table, then | |||||
* set the new attribute. | |||||
*/ | |||||
switch (lvl) { | |||||
default: | |||||
panic("Invalid DMAP table level: %d\n", lvl); | |||||
case 1: | |||||
newpte = pmap_demote_l1(kernel_pmap, pte, | |||||
tmpva & ~L1_OFFSET); | |||||
if (newpte == NULL) | |||||
return (EINVAL); | |||||
pte = pmap_l1_to_l2(pte, tmpva); | |||||
case 2: | |||||
newpte = pmap_demote_l2(kernel_pmap, pte, | |||||
tmpva & ~L2_OFFSET); | |||||
if (newpte == NULL) | |||||
return (EINVAL); | |||||
pte = pmap_l2_to_l3(pte, tmpva); | |||||
case 3: | |||||
/* Update the entry */ | |||||
l3 = pmap_load(pte); | |||||
l3 &= ~ATTR_IDX_MASK; | |||||
l3 |= ATTR_IDX(mode); | |||||
pmap_update_entry(kernel_pmap, pte, l3, tmpva); | |||||
/* | |||||
* If moving to a non-cacheable entry flush | |||||
* the cache. | |||||
*/ | |||||
Not Done Inline ActionsWhy idcache ? mmel: Why idcache ? | |||||
if (mode == VM_MEMATTR_UNCACHEABLE) | |||||
cpu_dcache_wbinv_range(tmpva, L3_SIZE); | |||||
break; | |||||
} | |||||
Not Done Inline ActionsIsn't tmpva just unmapped? mmel: Isn't tmpva just unmapped? | |||||
Not Done Inline ActionsNo, it is the virtual address we are currently changing. It will have been mapped in pmap_demote_l2. andrew: No, it is the virtual address we are currently changing. It will have been mapped in… | |||||
tmpva += PAGE_SIZE; | |||||
} | |||||
} | |||||
return (0); | |||||
} | |||||
/* | |||||
* Create an L2 table to map all addresses within an L1 mapping. | |||||
*/ | |||||
static pt_entry_t * | |||||
pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va) | |||||
{ | |||||
pt_entry_t *l2, newl2, oldl1; | |||||
vm_offset_t tmpl1; | |||||
vm_paddr_t l2phys, phys; | |||||
vm_page_t ml2; | |||||
int i; | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
oldl1 = pmap_load(l1); | |||||
KASSERT((oldl1 & ATTR_DESCR_MASK) == L1_BLOCK, | |||||
("pmap_demote_l1: Demoting a non-block entry")); | |||||
KASSERT((va & L1_OFFSET) == 0, | |||||
("pmap_demote_l1: Invalid virtual address %#lx", va)); | |||||
tmpl1 = 0; | |||||
if (va <= (vm_offset_t)l1 && va + L1_SIZE > (vm_offset_t)l1) { | |||||
tmpl1 = kva_alloc(PAGE_SIZE); | |||||
if (tmpl1 == 0) | |||||
return (NULL); | |||||
} | |||||
if ((ml2 = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT | | |||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { | |||||
CTR2(KTR_PMAP, "pmap_demote_l1: failure for va %#lx" | |||||
" in pmap %p", va, pmap); | |||||
return (NULL); | |||||
} | |||||
l2phys = VM_PAGE_TO_PHYS(ml2); | |||||
l2 = (pt_entry_t *)PHYS_TO_DMAP(l2phys); | |||||
/* Address the range points at */ | |||||
phys = oldl1 & ~ATTR_MASK; | |||||
/* The attributed from the old l1 table to be copied */ | |||||
newl2 = oldl1 & ATTR_MASK; | |||||
/* Create the new entries */ | |||||
for (i = 0; i < Ln_ENTRIES; i++) { | |||||
l2[i] = newl2 | phys; | |||||
phys += L2_SIZE; | |||||
} | |||||
cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE); | |||||
if (tmpl1 != 0) { | |||||
pmap_kenter(tmpl1, PAGE_SIZE, | |||||
DMAP_TO_PHYS((vm_offset_t)l1) & ~L3_OFFSET, CACHED_MEMORY); | |||||
Not Done Inline ActionsNew entries are not synced. mmel: New entries are not synced. | |||||
l1 = (pt_entry_t *)(tmpl1 + ((vm_offset_t)l1 & PAGE_MASK)); | |||||
} | |||||
pmap_update_entry(pmap, l1, l2phys | L1_TABLE, va); | |||||
if (tmpl1 != 0) { | |||||
pmap_kremove(tmpl1); | |||||
kva_free(tmpl1, PAGE_SIZE); | |||||
} | |||||
return (l2); | |||||
} | |||||
/* | |||||
* Create an L3 table to map all addresses within an L2 mapping. | |||||
*/ | |||||
static pt_entry_t * | |||||
pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va) | |||||
{ | |||||
pt_entry_t *l3, newl3, oldl2; | |||||
vm_offset_t tmpl2; | |||||
vm_paddr_t l3phys, phys; | |||||
vm_page_t ml3; | |||||
int i; | |||||
Not Done Inline ActionsThis invalidate is superfluous, unmapped TLB entries cannot be cached. mmel: This invalidate is superfluous, unmapped TLB entries cannot be cached. | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | |||||
oldl2 = pmap_load(l2); | |||||
KASSERT((oldl2 & ATTR_DESCR_MASK) == L2_BLOCK, | |||||
("pmap_demote_l2: Demoting a non-block entry")); | |||||
KASSERT((va & L2_OFFSET) == 0, | |||||
("pmap_demote_l2: Invalid virtual address %#lx", va)); | |||||
KASSERT((oldl2 & ATTR_SW_MANAGED) == 0, | |||||
("pmap_demote_l2: TODO: Demote managed pages")); | |||||
tmpl2 = 0; | |||||
if (va <= (vm_offset_t)l2 && va + L2_SIZE > (vm_offset_t)l2) { | |||||
tmpl2 = kva_alloc(PAGE_SIZE); | |||||
if (tmpl2 == 0) | |||||
return (NULL); | |||||
} | |||||
if ((ml3 = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT | | |||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { | |||||
CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" | |||||
" in pmap %p", va, pmap); | |||||
return (NULL); | |||||
} | |||||
l3phys = VM_PAGE_TO_PHYS(ml3); | |||||
l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); | |||||
/* Address the range points at */ | |||||
phys = oldl2 & ~ATTR_MASK; | |||||
/* The attributed from the old l2 table to be copied */ | |||||
newl3 = (oldl2 & (ATTR_MASK & ~ATTR_DESCR_MASK)) | L3_PAGE; | |||||
/* Create the new entries */ | |||||
for (i = 0; i < Ln_ENTRIES; i++) { | |||||
l3[i] = newl3 | phys; | |||||
phys += L3_SIZE; | |||||
} | |||||
cpu_dcache_wb_range((vm_offset_t)l3, PAGE_SIZE); | |||||
if (tmpl2 != 0) { | |||||
pmap_kenter(tmpl2, PAGE_SIZE, | |||||
DMAP_TO_PHYS((vm_offset_t)l2) & ~L3_OFFSET, CACHED_MEMORY); | |||||
l2 = (pt_entry_t *)(tmpl2 + ((vm_offset_t)l2 & PAGE_MASK)); | |||||
} | |||||
pmap_update_entry(pmap, l2, l3phys | L2_TABLE, va); | |||||
if (tmpl2 != 0) { | |||||
pmap_kremove(tmpl2); | |||||
kva_free(tmpl2, PAGE_SIZE); | |||||
} | |||||
return (l3); | |||||
} | |||||
/* | |||||
* perform the pmap work for mincore | * perform the pmap work for mincore | ||||
*/ | */ | ||||
int | int | ||||
pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) | pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) | ||||
{ | { | ||||
pd_entry_t *l1p, l1; | pd_entry_t *l1p, l1; | ||||
pd_entry_t *l2p, l2; | pd_entry_t *l2p, l2; | ||||
pt_entry_t *l3p, l3; | pt_entry_t *l3p, l3; | ||||
▲ Show 20 Lines • Show All 111 Lines • ▼ Show 20 Lines | while (sz != 0) { | ||||
/* Move to the next page */ | /* Move to the next page */ | ||||
sz -= len; | sz -= len; | ||||
va += len; | va += len; | ||||
/* Set the length for the next iteration */ | /* Set the length for the next iteration */ | ||||
len = imin(PAGE_SIZE, sz); | len = imin(PAGE_SIZE, sz); | ||||
} | } | ||||
} | } | ||||
} | |||||
int | |||||
pmap_fault(pmap_t pmap, uint64_t esr, uint64_t far) | |||||
{ | |||||
#ifdef SMP | |||||
uint64_t par; | |||||
#endif | |||||
switch (ESR_ELx_EXCEPTION(esr)) { | |||||
case EXCP_DATA_ABORT_L: | |||||
case EXCP_DATA_ABORT: | |||||
break; | |||||
default: | |||||
return (KERN_FAILURE); | |||||
} | |||||
#ifdef SMP | |||||
PMAP_LOCK(pmap); | |||||
switch (esr & ISS_DATA_DFSC_MASK) { | |||||
case ISS_DATA_DFSC_TF_L0: | |||||
case ISS_DATA_DFSC_TF_L1: | |||||
case ISS_DATA_DFSC_TF_L2: | |||||
case ISS_DATA_DFSC_TF_L3: | |||||
/* Ask the MMU to check the address */ | |||||
if (pmap == kernel_pmap) | |||||
par = arm64_address_translate_s1e1r(far); | |||||
else | |||||
par = arm64_address_translate_s1e0r(far); | |||||
/* | |||||
* If the translation was successful the address was invalid | |||||
* due to a break-before-make sequence. We can unlock and | |||||
* return success to the trap handler. | |||||
*/ | |||||
if (PAR_SUCCESS(par)) { | |||||
PMAP_UNLOCK(pmap); | |||||
Not Done Inline ActionsI remember that I asked the same question when reviewing the similar arm changes (break-before-make). Could transient page fault occur recursively ? In other words, could you get the fault while the thread owns pmap lock ? E.g., might it happen that interrupt handler accesses the DMAP mapping which is being promoted or demoted by the same CPU and interrupted ? kib: I remember that I asked the same question when reviewing the similar arm changes (break-before… | |||||
Not Done Inline ActionsI can change the critical section to a spin lock to disable interrupts. I don't expect any other exception to be triggered within the break-before-make section, and any that do would indicate a bug in the break-before-make section. andrew: I can change the critical section to a spin lock to disable interrupts. I don't expect any… | |||||
return (KERN_SUCCESS); | |||||
} | |||||
break; | |||||
default: | |||||
break; | |||||
} | |||||
PMAP_UNLOCK(pmap); | |||||
#endif | |||||
return (KERN_FAILURE); | |||||
} | } | ||||
/* | /* | ||||
* Increase the starting virtual address of the given mapping if a | * Increase the starting virtual address of the given mapping if a | ||||
* different alignment might result in more superpage mappings. | * different alignment might result in more superpage mappings. | ||||
*/ | */ | ||||
void | void | ||||
pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, | pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, | ||||
▲ Show 20 Lines • Show All 79 Lines • Show Last 20 Lines |
On amd64/i386, the same knob is called vm.pmap.pg_ps_enabled. Not sure if it is required to have consistent name there, but at least this is something to consider.