Changeset View
Changeset View
Standalone View
Standalone View
sys/i386/i386/pmap.c
Show First 20 Lines • Show All 2,079 Lines • ▼ Show 20 Lines | #endif | ||||
} | } | ||||
KASSERT(vm_radix_is_empty(&pmap->pm_root), | KASSERT(vm_radix_is_empty(&pmap->pm_root), | ||||
("pmap_pinit: pmap has reserved page table page(s)")); | ("pmap_pinit: pmap has reserved page table page(s)")); | ||||
/* | /* | ||||
* allocate the page directory page(s) | * allocate the page directory page(s) | ||||
*/ | */ | ||||
for (i = 0; i < NPGPTD; i++) { | for (i = 0; i < NPGPTD; i++) { | ||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | m = vm_page_alloc_noobj(VM_ALLOC_NORMAL | VM_ALLOC_WIRED | | ||||
kib: Eliminate m while there? | |||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); | VM_ALLOC_ZERO | VM_ALLOC_WAITOK); | ||||
pmap->pm_ptdpg[i] = m; | pmap->pm_ptdpg[i] = m; | ||||
#ifdef PMAP_PAE_COMP | #ifdef PMAP_PAE_COMP | ||||
pmap->pm_pdpt[i] = VM_PAGE_TO_PHYS(m) | PG_V; | pmap->pm_pdpt[i] = VM_PAGE_TO_PHYS(m) | PG_V; | ||||
#endif | #endif | ||||
} | } | ||||
pmap_qenter((vm_offset_t)pmap->pm_pdir, pmap->pm_ptdpg, NPGPTD); | pmap_qenter((vm_offset_t)pmap->pm_pdir, pmap->pm_ptdpg, NPGPTD); | ||||
#ifdef PMAP_PAE_COMP | #ifdef PMAP_PAE_COMP | ||||
if ((cpu_feature & CPUID_PAT) == 0) { | if ((cpu_feature & CPUID_PAT) == 0) { | ||||
pmap_invalidate_cache_range( | pmap_invalidate_cache_range( | ||||
trunc_page((vm_offset_t)pmap->pm_pdpt), | trunc_page((vm_offset_t)pmap->pm_pdpt), | ||||
round_page((vm_offset_t)pmap->pm_pdpt + | round_page((vm_offset_t)pmap->pm_pdpt + | ||||
NPGPTD * sizeof(pdpt_entry_t))); | NPGPTD * sizeof(pdpt_entry_t))); | ||||
} | } | ||||
#endif | #endif | ||||
for (i = 0; i < NPGPTD; i++) | |||||
if ((pmap->pm_ptdpg[i]->flags & PG_ZERO) == 0) | |||||
pagezero(pmap->pm_pdir + (i * NPDEPG)); | |||||
/* Install the trampoline mapping. */ | /* Install the trampoline mapping. */ | ||||
pmap->pm_pdir[TRPTDI] = PTD[TRPTDI]; | pmap->pm_pdir[TRPTDI] = PTD[TRPTDI]; | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
TAILQ_INIT(&pmap->pm_pvchunk); | TAILQ_INIT(&pmap->pm_pvchunk); | ||||
bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | ||||
return (1); | return (1); | ||||
} | } | ||||
/* | /* | ||||
* this routine is called if the page table page is not | * this routine is called if the page table page is not | ||||
* mapped correctly. | * mapped correctly. | ||||
*/ | */ | ||||
static vm_page_t | static vm_page_t | ||||
_pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags) | _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags) | ||||
{ | { | ||||
vm_paddr_t ptepa; | vm_paddr_t ptepa; | ||||
vm_page_t m; | vm_page_t m; | ||||
/* | /* | ||||
* Allocate a page table page. | * Allocate a page table page. | ||||
*/ | */ | ||||
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { | |||||
if ((flags & PMAP_ENTER_NOSLEEP) == 0) { | if ((flags & PMAP_ENTER_NOSLEEP) == 0) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
rw_wunlock(&pvh_global_lock); | rw_wunlock(&pvh_global_lock); | ||||
vm_wait(NULL); | vm_wait(NULL); | ||||
rw_wlock(&pvh_global_lock); | rw_wlock(&pvh_global_lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
} | } | ||||
/* | /* | ||||
* Indicate the need to retry. While waiting, the page table | * Indicate the need to retry. While waiting, the page table | ||||
* page may have been allocated. | * page may have been allocated. | ||||
*/ | */ | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
if ((m->flags & PG_ZERO) == 0) | m->pindex = ptepindex; | ||||
pmap_zero_page(m); | |||||
/* | /* | ||||
* Map the pagetable page into the process address space, if | * Map the pagetable page into the process address space, if | ||||
* it isn't already there. | * it isn't already there. | ||||
*/ | */ | ||||
pmap->pm_stats.resident_count++; | pmap->pm_stats.resident_count++; | ||||
▲ Show 20 Lines • Show All 107 Lines • ▼ Show 20 Lines | if (pdir_pde(PTD, kernel_vm_end)) { | ||||
kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; | kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
break; | break; | ||||
} | } | ||||
continue; | continue; | ||||
} | } | ||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDRSHIFT, | nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | |||||
VM_ALLOC_ZERO); | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
nkpg->pindex = kernel_vm_end >> PDRSHIFT; | |||||
nkpt++; | nkpt++; | ||||
if ((nkpg->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(nkpg); | |||||
ptppaddr = VM_PAGE_TO_PHYS(nkpg); | ptppaddr = VM_PAGE_TO_PHYS(nkpg); | ||||
newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M); | newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M); | ||||
pdir_pde(KPTD, kernel_vm_end) = newpdir; | pdir_pde(KPTD, kernel_vm_end) = newpdir; | ||||
pmap_kenter_pde(kernel_vm_end, newpdir); | pmap_kenter_pde(kernel_vm_end, newpdir); | ||||
kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; | kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
▲ Show 20 Lines • Show All 278 Lines • ▼ Show 20 Lines | if (field < _NPCM) { | ||||
return (pv); | return (pv); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Access to the ptelist "pv_vafree" is synchronized by the pvh | * Access to the ptelist "pv_vafree" is synchronized by the pvh | ||||
* global lock. If "pv_vafree" is currently non-empty, it will | * global lock. If "pv_vafree" is currently non-empty, it will | ||||
* remain non-empty until pmap_ptelist_alloc() completes. | * remain non-empty until pmap_ptelist_alloc() completes. | ||||
*/ | */ | ||||
if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | if (pv_vafree == 0 || (m = vm_page_alloc_noobj(VM_ALLOC_NORMAL | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { | VM_ALLOC_WIRED)) == NULL) { | ||||
if (try) { | if (try) { | ||||
pv_entry_count--; | pv_entry_count--; | ||||
PV_STAT(pc_chunk_tryfail++); | PV_STAT(pc_chunk_tryfail++); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
m = pmap_pv_reclaim(pmap); | m = pmap_pv_reclaim(pmap); | ||||
if (m == NULL) | if (m == NULL) | ||||
goto retry; | goto retry; | ||||
▲ Show 20 Lines • Show All 215 Lines • ▼ Show 20 Lines | KASSERT((oldpde & PG_W) == 0, | ||||
("pmap_demote_pde: page table page for a wired mapping" | ("pmap_demote_pde: page table page for a wired mapping" | ||||
" is missing")); | " is missing")); | ||||
/* | /* | ||||
* Invalidate the 2- or 4MB page mapping and return | * Invalidate the 2- or 4MB page mapping and return | ||||
* "failure" if the mapping was never accessed or the | * "failure" if the mapping was never accessed or the | ||||
* allocation of the new page table page fails. | * allocation of the new page table page fails. | ||||
*/ | */ | ||||
if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, | if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj( | ||||
va >> PDRSHIFT, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | | VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) { | ||||
VM_ALLOC_WIRED)) == NULL) { | |||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
sva = trunc_4mpage(va); | sva = trunc_4mpage(va); | ||||
pmap_remove_pde(pmap, pde, sva, &free); | pmap_remove_pde(pmap, pde, sva, &free); | ||||
if ((oldpde & PG_G) == 0) | if ((oldpde & PG_G) == 0) | ||||
pmap_invalidate_pde_page(pmap, sva, oldpde); | pmap_invalidate_pde_page(pmap, sva, oldpde); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x" | CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
mpte->pindex = va >> PDRSHIFT; | |||||
if (pmap != kernel_pmap) { | if (pmap != kernel_pmap) { | ||||
mpte->ref_count = NPTEPG; | mpte->ref_count = NPTEPG; | ||||
pmap->pm_stats.resident_count++; | pmap->pm_stats.resident_count++; | ||||
} | } | ||||
} | } | ||||
mptepa = VM_PAGE_TO_PHYS(mpte); | mptepa = VM_PAGE_TO_PHYS(mpte); | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 3,077 Lines • ▼ Show 20 Lines | if (prev_addr + size < prev_addr || prev_addr + size < size || | ||||
return (ENOMEM); | return (ENOMEM); | ||||
addr = prev_addr + size; | addr = prev_addr + size; | ||||
if (atomic_fcmpset_int(&pmap_trm_arena_last, &prev_addr, addr)) | if (atomic_fcmpset_int(&pmap_trm_arena_last, &prev_addr, addr)) | ||||
break; | break; | ||||
} | } | ||||
prev_addr += trm_guard; | prev_addr += trm_guard; | ||||
trm_pte = PTmap + atop(prev_addr); | trm_pte = PTmap + atop(prev_addr); | ||||
for (af = prev_addr; af < addr; af += PAGE_SIZE) { | for (af = prev_addr; af < addr; af += PAGE_SIZE) { | ||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_NOBUSY | | m = vm_page_alloc_noobj(VM_ALLOC_NOBUSY | VM_ALLOC_NORMAL | | ||||
VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_WAITOK); | VM_ALLOC_WIRED | VM_ALLOC_WAITOK); | ||||
pte_store(&trm_pte[atop(af - prev_addr)], VM_PAGE_TO_PHYS(m) | | pte_store(&trm_pte[atop(af - prev_addr)], VM_PAGE_TO_PHYS(m) | | ||||
PG_M | PG_A | PG_RW | PG_V | pgeflag | | PG_M | PG_A | PG_RW | PG_V | pgeflag | | ||||
pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, FALSE)); | pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, FALSE)); | ||||
} | } | ||||
*addrp = prev_addr; | *addrp = prev_addr; | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
pmap_init_trm(void) | pmap_init_trm(void) | ||||
{ | { | ||||
vm_page_t pd_m; | vm_page_t pd_m; | ||||
TUNABLE_INT_FETCH("machdep.trm_guard", &trm_guard); | TUNABLE_INT_FETCH("machdep.trm_guard", &trm_guard); | ||||
if ((trm_guard & PAGE_MASK) != 0) | if ((trm_guard & PAGE_MASK) != 0) | ||||
trm_guard = 0; | trm_guard = 0; | ||||
pmap_trm_arena = vmem_create("i386trampoline", 0, 0, 1, 0, M_WAITOK); | pmap_trm_arena = vmem_create("i386trampoline", 0, 0, 1, 0, M_WAITOK); | ||||
vmem_set_import(pmap_trm_arena, pmap_trm_import, NULL, NULL, PAGE_SIZE); | vmem_set_import(pmap_trm_arena, pmap_trm_import, NULL, NULL, PAGE_SIZE); | ||||
pd_m = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_NOBUSY | | pd_m = vm_page_alloc_noobj(VM_ALLOC_NOBUSY | VM_ALLOC_NORMAL | | ||||
VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_WAITOK | VM_ALLOC_ZERO); | VM_ALLOC_WIRED | VM_ALLOC_WAITOK | VM_ALLOC_ZERO); | ||||
if ((pd_m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(pd_m); | |||||
PTD[TRPTDI] = VM_PAGE_TO_PHYS(pd_m) | PG_M | PG_A | PG_RW | PG_V | | PTD[TRPTDI] = VM_PAGE_TO_PHYS(pd_m) | PG_M | PG_A | PG_RW | PG_V | | ||||
pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, TRUE); | pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, TRUE); | ||||
} | } | ||||
static void * | static void * | ||||
__CONCAT(PMTYPE, trm_alloc)(size_t size, int flags) | __CONCAT(PMTYPE, trm_alloc)(size_t size, int flags) | ||||
{ | { | ||||
vmem_addr_t res; | vmem_addr_t res; | ||||
▲ Show 20 Lines • Show All 463 Lines • Show Last 20 Lines |
Eliminate m while there?