Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 377 Lines • ▼ Show 20 Lines | |||||||||
static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); | static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); | ||||||||
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, | static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, | ||||||||
vm_offset_t va); | vm_offset_t va); | ||||||||
static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte); | static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte); | ||||||||
static bool pmap_activate_int(pmap_t pmap); | static bool pmap_activate_int(pmap_t pmap); | ||||||||
static void pmap_alloc_asid(pmap_t pmap); | static void pmap_alloc_asid(pmap_t pmap); | ||||||||
static int pmap_change_props_locked(vm_offset_t va, vm_size_t size, | static int pmap_change_props_locked(vm_offset_t va, vm_size_t size, | ||||||||
vm_prot_t prot, int mode); | vm_prot_t prot, int mode, bool skip_unmapped); | ||||||||
static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va); | static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va); | ||||||||
static pt_entry_t *pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, | static pt_entry_t *pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, | ||||||||
vm_offset_t va, struct rwlock **lockp); | vm_offset_t va, struct rwlock **lockp); | ||||||||
static pt_entry_t *pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va); | static pt_entry_t *pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va); | ||||||||
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, | static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, | ||||||||
vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); | vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); | ||||||||
static int pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, | static int pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, | ||||||||
u_int flags, vm_page_t m, struct rwlock **lockp); | u_int flags, vm_page_t m, struct rwlock **lockp); | ||||||||
▲ Show 20 Lines • Show All 152 Lines • ▼ Show 20 Lines | pmap_pde(pmap_t pmap, vm_offset_t va, int *level) | ||||||||
*level = 2; | *level = 2; | ||||||||
return (l2); | return (l2); | ||||||||
} | } | ||||||||
/* | /* | ||||||||
* Returns the lowest valid pte block or table entry for a given virtual | * Returns the lowest valid pte block or table entry for a given virtual | ||||||||
* address. If there are no valid entries return NULL and set the level to | * address. If there are no valid entries return NULL and set the level to | ||||||||
* the first invalid level. | * the first invalid level. | ||||||||
markj: The last sentence of this comment isn't accurate now. | |||||||||
*/ | */ | ||||||||
static __inline pt_entry_t * | static __inline pt_entry_t * | ||||||||
pmap_pte(pmap_t pmap, vm_offset_t va, int *level) | pmap_pte(pmap_t pmap, vm_offset_t va, int *level) | ||||||||
Not Done Inline ActionsThere is a very similar function in arm64/iommu/iommu_pmap.c that still has the old behaviour w.r.t. setting *level when a PTE is missing. I think it would be better to keep them consistent. markj: There is a very similar function in arm64/iommu/iommu_pmap.c that still has the old behaviour w. | |||||||||
Done Inline ActionsI think I was confused and the code was correct, however it should start with pmap_l0 so we can skip over the l0 entry if it's unmapped. andrew: I think I was confused and the code was correct, however it should start with `pmap_l0` so we… | |||||||||
Not Done Inline ActionsI also think that the code was correct, and that that the proposed changes to pmap_pte() should be undone. alc: I also think that the code was correct, and that that the proposed changes to pmap_pte() should… | |||||||||
{ | { | ||||||||
pd_entry_t *l0, *l1, *l2, desc; | pd_entry_t *l0, *l1, *l2, desc; | ||||||||
pt_entry_t *l3; | pt_entry_t *l3; | ||||||||
l0 = pmap_l0(pmap, va); | l0 = pmap_l0(pmap, va); | ||||||||
desc = pmap_load(l0) & ATTR_DESCR_MASK; | desc = pmap_load(l0) & ATTR_DESCR_MASK; | ||||||||
if (desc != L0_TABLE) { | if (desc != L0_TABLE) { | ||||||||
*level = 0; | *level = 0; | ||||||||
Show All 15 Lines | pmap_pte(pmap_t pmap, vm_offset_t va, int *level) | ||||||||
l2 = pmap_l1_to_l2(l1, va); | l2 = pmap_l1_to_l2(l1, va); | ||||||||
desc = pmap_load(l2) & ATTR_DESCR_MASK; | desc = pmap_load(l2) & ATTR_DESCR_MASK; | ||||||||
if (desc == L2_BLOCK) { | if (desc == L2_BLOCK) { | ||||||||
*level = 2; | *level = 2; | ||||||||
return (l2); | return (l2); | ||||||||
} | } | ||||||||
if (desc != L2_TABLE) { | if (desc != L2_TABLE) { | ||||||||
*level = 2; | *level = 2; | ||||||||
Not Done Inline ActionsThe assertion lvl == 3 in pmap_qremove() doesn't catch some erroneous cases now. markj: The assertion `lvl == 3` in pmap_qremove() doesn't catch some erroneous cases now. | |||||||||
Done Inline ActionsShould be fixed in D33509 andrew: Should be fixed in D33509 | |||||||||
return (NULL); | return (NULL); | ||||||||
} | } | ||||||||
*level = 3; | *level = 3; | ||||||||
l3 = pmap_l2_to_l3(l2, va); | l3 = pmap_l2_to_l3(l2, va); | ||||||||
if ((pmap_load(l3) & ATTR_DESCR_MASK) != L3_PAGE) | if ((pmap_load(l3) & ATTR_DESCR_MASK) != L3_PAGE) | ||||||||
return (NULL); | return (NULL); | ||||||||
▲ Show 20 Lines • Show All 5,344 Lines • ▼ Show 20 Lines | |||||||||
* virtual address range or the direct map. | * virtual address range or the direct map. | ||||||||
*/ | */ | ||||||||
int | int | ||||||||
pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) | pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) | ||||||||
{ | { | ||||||||
int error; | int error; | ||||||||
PMAP_LOCK(kernel_pmap); | PMAP_LOCK(kernel_pmap); | ||||||||
error = pmap_change_props_locked(va, size, PROT_NONE, mode); | error = pmap_change_props_locked(va, size, PROT_NONE, mode, false); | ||||||||
PMAP_UNLOCK(kernel_pmap); | PMAP_UNLOCK(kernel_pmap); | ||||||||
return (error); | return (error); | ||||||||
} | } | ||||||||
/* | /* | ||||||||
* Changes the specified virtual address range's protections to those | * Changes the specified virtual address range's protections to those | ||||||||
* specified by "prot". Like pmap_change_attr(), protections for aliases | * specified by "prot". Like pmap_change_attr(), protections for aliases | ||||||||
* in the direct map are updated as well. Protections on aliasing mappings may | * in the direct map are updated as well. Protections on aliasing mappings may | ||||||||
* be a subset of the requested protections; for example, mappings in the direct | * be a subset of the requested protections; for example, mappings in the direct | ||||||||
* map are never executable. | * map are never executable. | ||||||||
*/ | */ | ||||||||
int | int | ||||||||
pmap_change_prot(vm_offset_t va, vm_size_t size, vm_prot_t prot) | pmap_change_prot(vm_offset_t va, vm_size_t size, vm_prot_t prot) | ||||||||
{ | { | ||||||||
int error; | int error; | ||||||||
/* Only supported within the kernel map. */ | /* Only supported within the kernel map. */ | ||||||||
if (va < VM_MIN_KERNEL_ADDRESS) | if (va < VM_MIN_KERNEL_ADDRESS) | ||||||||
return (EINVAL); | return (EINVAL); | ||||||||
PMAP_LOCK(kernel_pmap); | PMAP_LOCK(kernel_pmap); | ||||||||
error = pmap_change_props_locked(va, size, prot, -1); | error = pmap_change_props_locked(va, size, prot, -1, false); | ||||||||
PMAP_UNLOCK(kernel_pmap); | PMAP_UNLOCK(kernel_pmap); | ||||||||
return (error); | return (error); | ||||||||
} | } | ||||||||
static int | static int | ||||||||
pmap_change_props_locked(vm_offset_t va, vm_size_t size, vm_prot_t prot, | pmap_change_props_locked(vm_offset_t va, vm_size_t size, vm_prot_t prot, | ||||||||
int mode) | int mode, bool skip_unmapped) | ||||||||
{ | { | ||||||||
vm_offset_t base, offset, tmpva; | vm_offset_t base, offset, tmpva; | ||||||||
vm_size_t pte_size; | vm_size_t pte_size; | ||||||||
vm_paddr_t pa; | vm_paddr_t pa; | ||||||||
pt_entry_t pte, *ptep, *newpte; | pt_entry_t pte, *ptep, *newpte; | ||||||||
pt_entry_t bits, mask; | pt_entry_t bits, mask; | ||||||||
int lvl, rv; | int lvl, rv; | ||||||||
Show All 35 Lines | if ((prot & VM_PROT_EXECUTE) == 0) { | ||||||||
bits |= ATTR_S1_PXN; | bits |= ATTR_S1_PXN; | ||||||||
} | } | ||||||||
bits |= ATTR_S1_UXN; | bits |= ATTR_S1_UXN; | ||||||||
mask |= ATTR_S1_AP_MASK | ATTR_S1_XN; | mask |= ATTR_S1_AP_MASK | ATTR_S1_XN; | ||||||||
} | } | ||||||||
for (tmpva = base; tmpva < base + size; ) { | for (tmpva = base; tmpva < base + size; ) { | ||||||||
ptep = pmap_pte(kernel_pmap, tmpva, &lvl); | ptep = pmap_pte(kernel_pmap, tmpva, &lvl); | ||||||||
if (ptep == NULL) | if (ptep == NULL && !skip_unmapped) { | ||||||||
return (EINVAL); | return (EINVAL); | ||||||||
} else if ((ptep == NULL && skip_unmapped) || | |||||||||
if ((pmap_load(ptep) & mask) == bits) { | (pmap_load(ptep) & mask) == bits) { | ||||||||
/* | /* | ||||||||
* We already have the correct attribute, | * We already have the correct attribute or there | ||||||||
* ignore this entry. | * is no memory mapped at this address and we are | ||||||||
Done Inline Actions
alc: | |||||||||
* skipping unmapped memory. | |||||||||
*/ | */ | ||||||||
switch (lvl) { | switch (lvl) { | ||||||||
default: | default: | ||||||||
panic("Invalid DMAP table level: %d\n", lvl); | panic("Invalid DMAP table level: %d\n", lvl); | ||||||||
case 1: | case 1: | ||||||||
tmpva = (tmpva & ~L1_OFFSET) + L1_SIZE; | tmpva = (tmpva & ~L1_OFFSET) + L1_SIZE; | ||||||||
break; | break; | ||||||||
case 2: | case 2: | ||||||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | if (ptep == NULL && !skip_unmapped) { | ||||||||
pa = pte & ~ATTR_MASK; | pa = pte & ~ATTR_MASK; | ||||||||
if (!VIRT_IN_DMAP(tmpva) && PHYS_IN_DMAP(pa)) { | if (!VIRT_IN_DMAP(tmpva) && PHYS_IN_DMAP(pa)) { | ||||||||
/* | /* | ||||||||
* Keep the DMAP memory in sync. | * Keep the DMAP memory in sync. | ||||||||
*/ | */ | ||||||||
rv = pmap_change_props_locked( | rv = pmap_change_props_locked( | ||||||||
PHYS_TO_DMAP(pa), pte_size, | PHYS_TO_DMAP(pa), pte_size, | ||||||||
prot, mode); | prot, mode, true); | ||||||||
if (rv != 0) | if (rv != 0) | ||||||||
return (rv); | return (rv); | ||||||||
} | } | ||||||||
/* | /* | ||||||||
* If moving to a non-cacheable entry flush | * If moving to a non-cacheable entry flush | ||||||||
* the cache. | * the cache. | ||||||||
*/ | */ | ||||||||
▲ Show 20 Lines • Show All 1,097 Lines • Show Last 20 Lines |
The last sentence of this comment isn't accurate now.