Changeset View
Standalone View
sys/amd64/amd64/pmap.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 1,270 Lines • ▼ Show 20 Lines | static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, | static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, | ||||
vm_page_t m, struct rwlock **lockp); | vm_page_t m, struct rwlock **lockp); | ||||
static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | ||||
pd_entry_t newpde); | pd_entry_t newpde); | ||||
static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde); | static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde); | ||||
static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, | static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, | ||||
struct rwlock **lockp, vm_offset_t va); | struct rwlock **lockp, vm_offset_t va, bool first_level); | ||||
static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, | static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, | static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
struct spglist *free); | struct spglist *free); | ||||
static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); | static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); | ||||
▲ Show 20 Lines • Show All 2,874 Lines • ▼ Show 20 Lines | pmap_allocpte_getpml4(pmap_t pmap, struct rwlock **lockp, vm_offset_t va, | ||||
if (!pmap_is_la57(pmap)) | if (!pmap_is_la57(pmap)) | ||||
return (&pmap->pm_pmltop[pmap_pml4e_index(va)]); | return (&pmap->pm_pmltop[pmap_pml4e_index(va)]); | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
pml5index = pmap_pml5e_index(va); | pml5index = pmap_pml5e_index(va); | ||||
pml5 = &pmap->pm_pmltop[pml5index]; | pml5 = &pmap->pm_pmltop[pml5index]; | ||||
if ((*pml5 & PG_V) == 0) { | if ((*pml5 & PG_V) == 0) { | ||||
if (_pmap_allocpte(pmap, pmap_pml5e_pindex(va), lockp, va) == | if (_pmap_allocpte(pmap, pmap_pml5e_pindex(va), lockp, va, | ||||
NULL) | false) == NULL) | ||||
return (NULL); | return (NULL); | ||||
allocated = true; | allocated = true; | ||||
} else { | } else { | ||||
allocated = false; | allocated = false; | ||||
} | } | ||||
pml4 = (pml4_entry_t *)PHYS_TO_DMAP(*pml5 & PG_FRAME); | pml4 = (pml4_entry_t *)PHYS_TO_DMAP(*pml5 & PG_FRAME); | ||||
pml4 = &pml4[pmap_pml4e_index(va)]; | pml4 = &pml4[pmap_pml4e_index(va)]; | ||||
if ((*pml4 & PG_V) == 0) { | if ((*pml4 & PG_V) == 0) { | ||||
Show All 19 Lines | pmap_allocpte_getpdp(pmap_t pmap, struct rwlock **lockp, vm_offset_t va, | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
pml4 = pmap_allocpte_getpml4(pmap, lockp, va, false); | pml4 = pmap_allocpte_getpml4(pmap, lockp, va, false); | ||||
if (pml4 == NULL) | if (pml4 == NULL) | ||||
return (NULL); | return (NULL); | ||||
if ((*pml4 & PG_V) == 0) { | if ((*pml4 & PG_V) == 0) { | ||||
/* Have to allocate a new pdp, recurse */ | /* Have to allocate a new pdp, recurse */ | ||||
if (_pmap_allocpte(pmap, pmap_pml4e_pindex(va), lockp, va) == | if (_pmap_allocpte(pmap, pmap_pml4e_pindex(va), lockp, va, | ||||
NULL) { | false) == NULL) { | ||||
if (pmap_is_la57(pmap)) | if (pmap_is_la57(pmap)) | ||||
pmap_allocpte_free_unref(pmap, va, | pmap_allocpte_free_unref(pmap, va, | ||||
pmap_pml5e(pmap, va)); | pmap_pml5e(pmap, va)); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
allocated = true; | allocated = true; | ||||
} else { | } else { | ||||
allocated = false; | allocated = false; | ||||
Show All 9 Lines | pmap_allocpte_getpdp(pmap_t pmap, struct rwlock **lockp, vm_offset_t va, | ||||
} | } | ||||
return (pdp); | return (pdp); | ||||
} | } | ||||
/* | /* | ||||
* This routine is called if the desired page table page does not exist. | * This routine is called if the desired page table page does not exist. | ||||
* | * | ||||
* If page table page allocation fails, this routine may sleep before | * If page table page allocation fails, this routine may sleep before | ||||
* returning NULL. It sleeps only if a lock pointer was given. | * returning NULL. It sleeps only if a lock pointer was given. Sleep | ||||
* occurs right before returning from the first recursion level, which | |||||
* is indicated by the first_level parameter. This way, we never sleep | |||||
markj: The comment is stale now. | |||||
Done Inline ActionsI also moved first two paragraphs to the pmap_allocpte_alloc() herald. It is more logical. kib: I also moved first two paragraphs to the pmap_allocpte_alloc() herald. It is more logical. | |||||
* with a page table page having ref_count == 0, otherwise the page could | |||||
* be freed under us. | |||||
* | * | ||||
* Note: If a page allocation fails at page table level two, three, or four, | |||||
* up to three pages may be held during the wait, only to be released | |||||
* afterwards. This conservative approach is easily argued to avoid | |||||
* race conditions. | |||||
* | |||||
* The ptepindexes, i.e. page indices, of the page table pages encountered | * The ptepindexes, i.e. page indices, of the page table pages encountered | ||||
* while translating virtual address va are defined as follows: | * while translating virtual address va are defined as follows: | ||||
* - for the page table page (last level), | * - for the page table page (last level), | ||||
* ptepindex = pmap_pde_pindex(va) = va >> PDRSHIFT, | * ptepindex = pmap_pde_pindex(va) = va >> PDRSHIFT, | ||||
* in other words, it is just the index of the PDE that maps the page | * in other words, it is just the index of the PDE that maps the page | ||||
* table page. | * table page. | ||||
* - for the page directory page, | * - for the page directory page, | ||||
* ptepindex = NUPDE (number of userland PD entries) + | * ptepindex = NUPDE (number of userland PD entries) + | ||||
Show All 18 Lines | |||||
* ptepindexes are calculated as if the paging structures were 5-level | * ptepindexes are calculated as if the paging structures were 5-level | ||||
* regardless of the actual mode of operation. | * regardless of the actual mode of operation. | ||||
* | * | ||||
* The root page at PML4/PML5 does not participate in this indexing scheme, | * The root page at PML4/PML5 does not participate in this indexing scheme, | ||||
* since it is statically allocated by pmap_pinit() and not by _pmap_allocpte(). | * since it is statically allocated by pmap_pinit() and not by _pmap_allocpte(). | ||||
*/ | */ | ||||
static vm_page_t | static vm_page_t | ||||
_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp, | _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp, | ||||
vm_offset_t va __unused) | vm_offset_t va, bool first_level) | ||||
Done Inline ActionsWhy is va labeled __unused? alc: Why is `va` labeled `__unused`? | |||||
Done Inline ActionsRight now it should be not. It came from some intermediate version of LA57 patch where va was only used to improve asserts. kib: Right now it should be not. It came from some intermediate version of LA57 patch where va was… | |||||
Done Inline ActionsWhy not split it into two functions, one for handling the base case of the recursion (try to alloc and sleep if that fails), and a second recursive function that never sleeps? markj: Why not split it into two functions, one for handling the base case of the recursion (try to… | |||||
Done Inline ActionsDone, with the note that alloc cannot go into non-recursive part, it must occur in the recursive helper when needed. kib: Done, with the note that alloc cannot go into non-recursive part, it must occur in the… | |||||
Done Inline ActionsI would call it _pmap_allocpte_nosleep() or __pmap_allocpte(), to make it clearer that it is an internal helper for _pmap_allocpte(). markj: I would call it _pmap_allocpte_nosleep() or __pmap_allocpte(), to make it clearer that it is an… | |||||
Done Inline ActionsI renamed it to pmap_allocpte_alloc(). I quite dislike the name that pretends on belonging to the language implementation namespace. kib: I renamed it to pmap_allocpte_alloc(). I quite dislike the name that pretends on belonging to… | |||||
{ | { | ||||
vm_pindex_t pml5index, pml4index; | vm_pindex_t pml5index, pml4index; | ||||
pml5_entry_t *pml5, *pml5u; | pml5_entry_t *pml5, *pml5u; | ||||
pml4_entry_t *pml4, *pml4u; | pml4_entry_t *pml4, *pml4u; | ||||
pdp_entry_t *pdp; | pdp_entry_t *pdp; | ||||
pd_entry_t *pd; | pd_entry_t *pd; | ||||
vm_page_t m, pdpg; | vm_page_t m, pdpg; | ||||
pt_entry_t PG_A, PG_M, PG_RW, PG_V; | pt_entry_t PG_A, PG_M, PG_RW, PG_V; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PG_A = pmap_accessed_bit(pmap); | PG_A = pmap_accessed_bit(pmap); | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
/* | /* | ||||
* Allocate a page table page. | * Allocate a page table page. | ||||
*/ | */ | ||||
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) | ||||
if (lockp != NULL) { | goto wait; | ||||
RELEASE_PV_LIST_LOCK(lockp); | |||||
PMAP_UNLOCK(pmap); | |||||
PMAP_ASSERT_NOT_IN_DI(); | |||||
vm_wait(NULL); | |||||
PMAP_LOCK(pmap); | |||||
} | |||||
/* | |||||
* Indicate the need to retry. While waiting, the page table | |||||
* page may have been allocated. | |||||
*/ | |||||
return (NULL); | |||||
} | |||||
if ((m->flags & PG_ZERO) == 0) | if ((m->flags & PG_ZERO) == 0) | ||||
pmap_zero_page(m); | pmap_zero_page(m); | ||||
/* | /* | ||||
* Map the pagetable page into the process address space, if | * Map the pagetable page into the process address space, if | ||||
* it isn't already there. | * it isn't already there. | ||||
*/ | */ | ||||
if (ptepindex >= NUPDE + NUPDPE + NUPML4E) { | if (ptepindex >= NUPDE + NUPDPE + NUPML4E) { | ||||
Show All 15 Lines | if (ptepindex >= NUPDE + NUPDPE + NUPML4E) { | ||||
} | } | ||||
} else if (ptepindex >= NUPDE + NUPDPE) { | } else if (ptepindex >= NUPDE + NUPDPE) { | ||||
pml4index = pmap_pml4e_index(va); | pml4index = pmap_pml4e_index(va); | ||||
/* Wire up a new PDPE page */ | /* Wire up a new PDPE page */ | ||||
pml4 = pmap_allocpte_getpml4(pmap, lockp, va, true); | pml4 = pmap_allocpte_getpml4(pmap, lockp, va, true); | ||||
if (pml4 == NULL) { | if (pml4 == NULL) { | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | goto wait; | ||||
} | } | ||||
KASSERT((*pml4 & PG_V) == 0, | KASSERT((*pml4 & PG_V) == 0, | ||||
("pmap %p va %#lx pml4 %#lx", pmap, va, *pml4)); | ("pmap %p va %#lx pml4 %#lx", pmap, va, *pml4)); | ||||
*pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | ||||
if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL && | if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL && | ||||
pml4index < NUPML4E) { | pml4index < NUPML4E) { | ||||
/* | /* | ||||
Show All 11 Lines | if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL && | ||||
PG_A | PG_M; | PG_A | PG_M; | ||||
} | } | ||||
} else if (ptepindex >= NUPDE) { | } else if (ptepindex >= NUPDE) { | ||||
/* Wire up a new PDE page */ | /* Wire up a new PDE page */ | ||||
pdp = pmap_allocpte_getpdp(pmap, lockp, va, true); | pdp = pmap_allocpte_getpdp(pmap, lockp, va, true); | ||||
if (pdp == NULL) { | if (pdp == NULL) { | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | goto wait; | ||||
} | } | ||||
KASSERT((*pdp & PG_V) == 0, | KASSERT((*pdp & PG_V) == 0, | ||||
("pmap %p va %#lx pdp %#lx", pmap, va, *pdp)); | ("pmap %p va %#lx pdp %#lx", pmap, va, *pdp)); | ||||
*pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | ||||
} else { | } else { | ||||
/* Wire up a new PTE page */ | /* Wire up a new PTE page */ | ||||
pdp = pmap_allocpte_getpdp(pmap, lockp, va, false); | pdp = pmap_allocpte_getpdp(pmap, lockp, va, false); | ||||
if (pdp == NULL) { | if (pdp == NULL) { | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | goto wait; | ||||
} | } | ||||
if ((*pdp & PG_V) == 0) { | if ((*pdp & PG_V) == 0) { | ||||
/* Have to allocate a new pd, recurse */ | /* Have to allocate a new pd, recurse */ | ||||
if (_pmap_allocpte(pmap, pmap_pdpe_pindex(va), | if (_pmap_allocpte(pmap, pmap_pdpe_pindex(va), | ||||
lockp, va) == NULL) { | lockp, va, false) == NULL) { | ||||
pmap_allocpte_free_unref(pmap, va, | pmap_allocpte_free_unref(pmap, va, | ||||
pmap_pml4e(pmap, va)); | pmap_pml4e(pmap, va)); | ||||
vm_page_unwire_noq(m); | vm_page_unwire_noq(m); | ||||
vm_page_free_zero(m); | vm_page_free_zero(m); | ||||
return (NULL); | goto wait; | ||||
} | } | ||||
} else { | } else { | ||||
/* Add reference to the pd page */ | /* Add reference to the pd page */ | ||||
pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME); | pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME); | ||||
pdpg->ref_count++; | pdpg->ref_count++; | ||||
} | } | ||||
pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME); | pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME); | ||||
/* Now we know where the page directory page is */ | /* Now we know where the page directory page is */ | ||||
pd = &pd[pmap_pde_index(va)]; | pd = &pd[pmap_pde_index(va)]; | ||||
KASSERT((*pd & PG_V) == 0, | KASSERT((*pd & PG_V) == 0, | ||||
("pmap %p va %#lx pd %#lx", pmap, va, *pd)); | ("pmap %p va %#lx pd %#lx", pmap, va, *pd)); | ||||
*pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; | ||||
} | } | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
return (m); | return (m); | ||||
wait: | |||||
if (first_level && lockp != NULL) { | |||||
RELEASE_PV_LIST_LOCK(lockp); | |||||
PMAP_UNLOCK(pmap); | |||||
PMAP_ASSERT_NOT_IN_DI(); | |||||
vm_wait(NULL); | |||||
PMAP_LOCK(pmap); | |||||
Done Inline Actionss/from/to/ markj: s/from/to/ | |||||
} | } | ||||
/* | |||||
* Indicate the need to retry. While waiting, the page table | |||||
* page may have been allocated. | |||||
*/ | |||||
return (NULL); | |||||
} | |||||
static pd_entry_t * | static pd_entry_t * | ||||
pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | ||||
struct rwlock **lockp) | struct rwlock **lockp) | ||||
{ | { | ||||
pdp_entry_t *pdpe, PG_V; | pdp_entry_t *pdpe, PG_V; | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
vm_page_t pdpg; | vm_page_t pdpg; | ||||
vm_pindex_t pdpindex; | vm_pindex_t pdpindex; | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
retry: | retry: | ||||
pdpe = pmap_pdpe(pmap, va); | pdpe = pmap_pdpe(pmap, va); | ||||
if (pdpe != NULL && (*pdpe & PG_V) != 0) { | if (pdpe != NULL && (*pdpe & PG_V) != 0) { | ||||
pde = pmap_pdpe_to_pde(pdpe, va); | pde = pmap_pdpe_to_pde(pdpe, va); | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (va < VM_MAXUSER_ADDRESS) { | ||||
/* Add a reference to the pd page. */ | /* Add a reference to the pd page. */ | ||||
pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); | pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); | ||||
pdpg->ref_count++; | pdpg->ref_count++; | ||||
} else | } else | ||||
pdpg = NULL; | pdpg = NULL; | ||||
} else if (va < VM_MAXUSER_ADDRESS) { | } else if (va < VM_MAXUSER_ADDRESS) { | ||||
/* Allocate a pd page. */ | /* Allocate a pd page. */ | ||||
pdpindex = pmap_pde_pindex(va) >> NPDPEPGSHIFT; | pdpindex = pmap_pde_pindex(va) >> NPDPEPGSHIFT; | ||||
pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp, va); | pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp, va, true); | ||||
if (pdpg == NULL) { | if (pdpg == NULL) { | ||||
if (lockp != NULL) | if (lockp != NULL) | ||||
goto retry; | goto retry; | ||||
else | else | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); | pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); | ||||
pde = &pde[pmap_pde_index(va)]; | pde = &pde[pmap_pde_index(va)]; | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | retry: | ||||
if (pd != NULL && (*pd & PG_V) != 0) { | if (pd != NULL && (*pd & PG_V) != 0) { | ||||
m = PHYS_TO_VM_PAGE(*pd & PG_FRAME); | m = PHYS_TO_VM_PAGE(*pd & PG_FRAME); | ||||
m->ref_count++; | m->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Here if the pte page isn't mapped, or if it has been | * Here if the pte page isn't mapped, or if it has been | ||||
* deallocated. | * deallocated. | ||||
*/ | */ | ||||
m = _pmap_allocpte(pmap, ptepindex, lockp, va); | m = _pmap_allocpte(pmap, ptepindex, lockp, va, true); | ||||
if (m == NULL && lockp != NULL) | if (m == NULL && lockp != NULL) | ||||
goto retry; | goto retry; | ||||
} | } | ||||
return (m); | return (m); | ||||
} | } | ||||
/*************************************************** | /*************************************************** | ||||
* Pmap allocation/deallocation routines. | * Pmap allocation/deallocation routines. | ||||
▲ Show 20 Lines • Show All 2,003 Lines • ▼ Show 20 Lines | restart: | ||||
pten = newpte; | pten = newpte; | ||||
if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) | if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) | ||||
pten |= pmap_pkru_get(pmap, va); | pten |= pmap_pkru_get(pmap, va); | ||||
if (psind == 2) { /* 1G */ | if (psind == 2) { /* 1G */ | ||||
pml4e = pmap_pml4e(pmap, va); | pml4e = pmap_pml4e(pmap, va); | ||||
if (pml4e == NULL || (*pml4e & PG_V) == 0) { | if (pml4e == NULL || (*pml4e & PG_V) == 0) { | ||||
mp = _pmap_allocpte(pmap, pmap_pml4e_pindex(va), | mp = _pmap_allocpte(pmap, pmap_pml4e_pindex(va), | ||||
NULL, va); | NULL, va, true); | ||||
if (mp == NULL) | if (mp == NULL) | ||||
goto allocf; | goto allocf; | ||||
pdpe = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); | pdpe = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); | ||||
pdpe = &pdpe[pmap_pdpe_index(va)]; | pdpe = &pdpe[pmap_pdpe_index(va)]; | ||||
origpte = *pdpe; | origpte = *pdpe; | ||||
MPASS(origpte == 0); | MPASS(origpte == 0); | ||||
} else { | } else { | ||||
pdpe = pmap_pml4e_to_pdpe(pml4e, va); | pdpe = pmap_pml4e_to_pdpe(pml4e, va); | ||||
KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va)); | KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va)); | ||||
origpte = *pdpe; | origpte = *pdpe; | ||||
if ((origpte & PG_V) == 0) { | if ((origpte & PG_V) == 0) { | ||||
mp = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME); | mp = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME); | ||||
mp->ref_count++; | mp->ref_count++; | ||||
} | } | ||||
} | } | ||||
*pdpe = pten; | *pdpe = pten; | ||||
} else /* (psind == 1) */ { /* 2M */ | } else /* (psind == 1) */ { /* 2M */ | ||||
pde = pmap_pde(pmap, va); | pde = pmap_pde(pmap, va); | ||||
if (pde == NULL) { | if (pde == NULL) { | ||||
mp = _pmap_allocpte(pmap, pmap_pdpe_pindex(va), | mp = _pmap_allocpte(pmap, pmap_pdpe_pindex(va), | ||||
NULL, va); | NULL, va, true); | ||||
if (mp == NULL) | if (mp == NULL) | ||||
goto allocf; | goto allocf; | ||||
pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); | pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); | ||||
pde = &pde[pmap_pde_index(va)]; | pde = &pde[pmap_pde_index(va)]; | ||||
origpte = *pde; | origpte = *pde; | ||||
MPASS(origpte == 0); | MPASS(origpte == 0); | ||||
} else { | } else { | ||||
origpte = *pde; | origpte = *pde; | ||||
▲ Show 20 Lines • Show All 138 Lines • ▼ Show 20 Lines | if (pde != NULL && (*pde & PG_V) != 0 && ((*pde & PG_PS) == 0 || | ||||
} | } | ||||
} else if (va < VM_MAXUSER_ADDRESS) { | } else if (va < VM_MAXUSER_ADDRESS) { | ||||
/* | /* | ||||
* Here if the pte page isn't mapped, or if it has been | * Here if the pte page isn't mapped, or if it has been | ||||
* deallocated. | * deallocated. | ||||
*/ | */ | ||||
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | ||||
mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), | mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), | ||||
nosleep ? NULL : &lock, va); | nosleep ? NULL : &lock, va, true); | ||||
if (mpte == NULL && nosleep) { | if (mpte == NULL && nosleep) { | ||||
rv = KERN_RESOURCE_SHORTAGE; | rv = KERN_RESOURCE_SHORTAGE; | ||||
goto out; | goto out; | ||||
} | } | ||||
goto retry; | goto retry; | ||||
} else | } else | ||||
panic("pmap_enter: invalid page directory va=%#lx", va); | panic("pmap_enter: invalid page directory va=%#lx", va); | ||||
▲ Show 20 Lines • Show All 467 Lines • ▼ Show 20 Lines | if (mpte && (mpte->pindex == ptepindex)) { | ||||
mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME); | mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME); | ||||
mpte->ref_count++; | mpte->ref_count++; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Pass NULL instead of the PV list lock | * Pass NULL instead of the PV list lock | ||||
* pointer, because we don't intend to sleep. | * pointer, because we don't intend to sleep. | ||||
*/ | */ | ||||
mpte = _pmap_allocpte(pmap, ptepindex, NULL, | mpte = _pmap_allocpte(pmap, ptepindex, NULL, | ||||
va); | va, true); | ||||
if (mpte == NULL) | if (mpte == NULL) | ||||
return (mpte); | return (mpte); | ||||
} | } | ||||
} | } | ||||
pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); | pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); | ||||
pte = &pte[pmap_pte_index(va)]; | pte = &pte[pmap_pte_index(va)]; | ||||
} else { | } else { | ||||
mpte = NULL; | mpte = NULL; | ||||
▲ Show 20 Lines • Show All 312 Lines • ▼ Show 20 Lines | if ((*pdpe & PG_PS) != 0) { | ||||
"pdpe %#lx sva %#lx eva %#lx va_next %#lx", | "pdpe %#lx sva %#lx eva %#lx va_next %#lx", | ||||
*pdpe, addr, end_addr, va_next)); | *pdpe, addr, end_addr, va_next)); | ||||
MPASS((addr & PDPMASK) == 0); | MPASS((addr & PDPMASK) == 0); | ||||
MPASS((*pdpe & PG_MANAGED) == 0); | MPASS((*pdpe & PG_MANAGED) == 0); | ||||
srcptepaddr = *pdpe; | srcptepaddr = *pdpe; | ||||
pdpe = pmap_pdpe(dst_pmap, addr); | pdpe = pmap_pdpe(dst_pmap, addr); | ||||
if (pdpe == NULL) { | if (pdpe == NULL) { | ||||
if (_pmap_allocpte(dst_pmap, | if (_pmap_allocpte(dst_pmap, | ||||
pmap_pml4e_pindex(addr), NULL, addr) == | pmap_pml4e_pindex(addr), NULL, addr, | ||||
NULL) | true) == NULL) | ||||
break; | break; | ||||
pdpe = pmap_pdpe(dst_pmap, addr); | pdpe = pmap_pdpe(dst_pmap, addr); | ||||
} else { | } else { | ||||
pml4e = pmap_pml4e(dst_pmap, addr); | pml4e = pmap_pml4e(dst_pmap, addr); | ||||
dst_pdpg = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME); | dst_pdpg = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME); | ||||
dst_pdpg->ref_count++; | dst_pdpg->ref_count++; | ||||
} | } | ||||
KASSERT(*pdpe == 0, | KASSERT(*pdpe == 0, | ||||
▲ Show 20 Lines • Show All 3,922 Lines • Show Last 20 Lines |
The comment is stale now.