Changeset View
Changeset View
Standalone View
Standalone View
sys/riscv/riscv/pmap.c
Show First 20 Lines • Show All 1,214 Lines • ▼ Show 20 Lines | |||||
pmap_pinit(pmap_t pmap) | pmap_pinit(pmap_t pmap) | ||||
{ | { | ||||
vm_paddr_t l1phys; | vm_paddr_t l1phys; | ||||
vm_page_t l1pt; | vm_page_t l1pt; | ||||
/* | /* | ||||
* allocate the l1 page | * allocate the l1 page | ||||
*/ | */ | ||||
while ((l1pt = vm_page_alloc(NULL, 0xdeadbeef, VM_ALLOC_NORMAL | | l1pt = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) | VM_ALLOC_WAITOK); | ||||
vm_wait(NULL); | |||||
l1phys = VM_PAGE_TO_PHYS(l1pt); | l1phys = VM_PAGE_TO_PHYS(l1pt); | ||||
pmap->pm_l1 = (pd_entry_t *)PHYS_TO_DMAP(l1phys); | pmap->pm_l1 = (pd_entry_t *)PHYS_TO_DMAP(l1phys); | ||||
pmap->pm_satp = SATP_MODE_SV39 | (l1phys >> PAGE_SHIFT); | pmap->pm_satp = SATP_MODE_SV39 | (l1phys >> PAGE_SHIFT); | ||||
if ((l1pt->flags & PG_ZERO) == 0) | |||||
pagezero(pmap->pm_l1); | |||||
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
/* Install kernel pagetables */ | /* Install kernel pagetables */ | ||||
memcpy(pmap->pm_l1, kernel_pmap->pm_l1, PAGE_SIZE); | memcpy(pmap->pm_l1, kernel_pmap->pm_l1, PAGE_SIZE); | ||||
/* Add to the list of all user pmaps */ | /* Add to the list of all user pmaps */ | ||||
Show All 25 Lines | _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) | ||||
vm_paddr_t phys; | vm_paddr_t phys; | ||||
pn_t pn; | pn_t pn; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
/* | /* | ||||
* Allocate a page table page. | * Allocate a page table page. | ||||
*/ | */ | ||||
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO); | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { | if (m == NULL) { | ||||
if (lockp != NULL) { | if (lockp != NULL) { | ||||
RELEASE_PV_LIST_LOCK(lockp); | RELEASE_PV_LIST_LOCK(lockp); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
rw_runlock(&pvh_global_lock); | rw_runlock(&pvh_global_lock); | ||||
vm_wait(NULL); | vm_wait(NULL); | ||||
rw_rlock(&pvh_global_lock); | rw_rlock(&pvh_global_lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
} | } | ||||
/* | /* | ||||
* Indicate the need to retry. While waiting, the page table | * Indicate the need to retry. While waiting, the page table | ||||
* page may have been allocated. | * page may have been allocated. | ||||
*/ | */ | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
m->pindex = ptepindex; | |||||
if ((m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m); | |||||
/* | /* | ||||
* Map the pagetable page into the process address space, if | * Map the pagetable page into the process address space, if | ||||
* it isn't already there. | * it isn't already there. | ||||
*/ | */ | ||||
if (ptepindex >= NUL2E) { | if (ptepindex >= NUL2E) { | ||||
pd_entry_t *l1; | pd_entry_t *l1; | ||||
vm_pindex_t l1index; | vm_pindex_t l1index; | ||||
▲ Show 20 Lines • Show All 176 Lines • ▼ Show 20 Lines | pmap_growkernel(vm_offset_t addr) | ||||
addr = roundup2(addr, L2_SIZE); | addr = roundup2(addr, L2_SIZE); | ||||
if (addr - 1 >= vm_map_max(kernel_map)) | if (addr - 1 >= vm_map_max(kernel_map)) | ||||
addr = vm_map_max(kernel_map); | addr = vm_map_max(kernel_map); | ||||
while (kernel_vm_end < addr) { | while (kernel_vm_end < addr) { | ||||
l1 = pmap_l1(kernel_pmap, kernel_vm_end); | l1 = pmap_l1(kernel_pmap, kernel_vm_end); | ||||
if (pmap_load(l1) == 0) { | if (pmap_load(l1) == 0) { | ||||
/* We need a new PDP entry */ | /* We need a new PDP entry */ | ||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT, | nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | | |||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO); | VM_ALLOC_WIRED | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) | nkpg->pindex = kernel_vm_end >> L1_SHIFT; | ||||
pmap_zero_page(nkpg); | |||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
pn = (paddr / PAGE_SIZE); | pn = (paddr / PAGE_SIZE); | ||||
entry = (PTE_V); | entry = (PTE_V); | ||||
entry |= (pn << PTE_PPN0_S); | entry |= (pn << PTE_PPN0_S); | ||||
pmap_store(l1, entry); | pmap_store(l1, entry); | ||||
pmap_distribute_l1(kernel_pmap, | pmap_distribute_l1(kernel_pmap, | ||||
pmap_l1_index(kernel_vm_end), entry); | pmap_l1_index(kernel_vm_end), entry); | ||||
continue; /* try again */ | continue; /* try again */ | ||||
} | } | ||||
l2 = pmap_l1_to_l2(l1, kernel_vm_end); | l2 = pmap_l1_to_l2(l1, kernel_vm_end); | ||||
if ((pmap_load(l2) & PTE_V) != 0 && | if ((pmap_load(l2) & PTE_V) != 0 && | ||||
(pmap_load(l2) & PTE_RWX) == 0) { | (pmap_load(l2) & PTE_RWX) == 0) { | ||||
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
break; | break; | ||||
} | } | ||||
continue; | continue; | ||||
} | } | ||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT, | nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | |||||
VM_ALLOC_ZERO); | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) { | nkpg->pindex = kernel_vm_end >> L2_SHIFT; | ||||
pmap_zero_page(nkpg); | |||||
} | |||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
pn = (paddr / PAGE_SIZE); | pn = (paddr / PAGE_SIZE); | ||||
entry = (PTE_V); | entry = (PTE_V); | ||||
entry |= (pn << PTE_PPN0_S); | entry |= (pn << PTE_PPN0_S); | ||||
pmap_store(l2, entry); | pmap_store(l2, entry); | ||||
pmap_invalidate_page(kernel_pmap, kernel_vm_end); | pmap_invalidate_page(kernel_pmap, kernel_vm_end); | ||||
▲ Show 20 Lines • Show All 163 Lines • ▼ Show 20 Lines | if (field < _NPCM) { | ||||
pc_list); | pc_list); | ||||
} | } | ||||
PV_STAT(atomic_add_long(&pv_entry_count, 1)); | PV_STAT(atomic_add_long(&pv_entry_count, 1)); | ||||
PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); | PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); | ||||
return (pv); | return (pv); | ||||
} | } | ||||
} | } | ||||
/* No free items, allocate another chunk */ | /* No free items, allocate another chunk */ | ||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | m = vm_page_alloc_noobj(VM_ALLOC_WIRED); | ||||
VM_ALLOC_WIRED); | |||||
if (m == NULL) { | if (m == NULL) { | ||||
if (lockp == NULL) { | if (lockp == NULL) { | ||||
PV_STAT(pc_chunk_tryfail++); | PV_STAT(pc_chunk_tryfail++); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
m = reclaim_pv_chunk(pmap, lockp); | m = reclaim_pv_chunk(pmap, lockp); | ||||
if (m == NULL) | if (m == NULL) | ||||
goto retry; | goto retry; | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | bit_count((bitstr_t *)pc->pc_map, 0, | ||||
sizeof(pc->pc_map) * NBBY, &free); | sizeof(pc->pc_map) * NBBY, &free); | ||||
if (free == 0) | if (free == 0) | ||||
break; | break; | ||||
avail += free; | avail += free; | ||||
if (avail >= needed) | if (avail >= needed) | ||||
break; | break; | ||||
} | } | ||||
for (reclaimed = false; avail < needed; avail += _NPCPV) { | for (reclaimed = false; avail < needed; avail += _NPCPV) { | ||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | m = vm_page_alloc_noobj(VM_ALLOC_WIRED); | ||||
VM_ALLOC_WIRED); | |||||
if (m == NULL) { | if (m == NULL) { | ||||
m = reclaim_pv_chunk(pmap, lockp); | m = reclaim_pv_chunk(pmap, lockp); | ||||
if (m == NULL) | if (m == NULL) | ||||
goto retry; | goto retry; | ||||
reclaimed = true; | reclaimed = true; | ||||
} | } | ||||
/* XXX PV STATS */ | /* XXX PV STATS */ | ||||
#if 0 | #if 0 | ||||
▲ Show 20 Lines • Show All 702 Lines • ▼ Show 20 Lines | pmap_demote_l2_locked(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
oldl2 = pmap_load(l2); | oldl2 = pmap_load(l2); | ||||
KASSERT((oldl2 & PTE_RWX) != 0, | KASSERT((oldl2 & PTE_RWX) != 0, | ||||
("pmap_demote_l2_locked: oldl2 is not a leaf entry")); | ("pmap_demote_l2_locked: oldl2 is not a leaf entry")); | ||||
if ((oldl2 & PTE_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == | if ((oldl2 & PTE_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == | ||||
NULL) { | NULL) { | ||||
if ((oldl2 & PTE_A) == 0 || (mpte = vm_page_alloc(NULL, | if ((oldl2 & PTE_A) == 0 || (mpte = vm_page_alloc_noobj( | ||||
pmap_l2_pindex(va), (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : | (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : 0) | | ||||
VM_ALLOC_NORMAL) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == | VM_ALLOC_WIRED)) == NULL) { | ||||
NULL) { | |||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
(void)pmap_remove_l2(pmap, l2, va & ~L2_OFFSET, | (void)pmap_remove_l2(pmap, l2, va & ~L2_OFFSET, | ||||
pmap_load(pmap_l1(pmap, va)), &free, lockp); | pmap_load(pmap_l1(pmap, va)), &free, lockp); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
CTR2(KTR_PMAP, "pmap_demote_l2_locked: " | CTR2(KTR_PMAP, "pmap_demote_l2_locked: " | ||||
"failure for va %#lx in pmap %p", va, pmap); | "failure for va %#lx in pmap %p", va, pmap); | ||||
return (false); | return (false); | ||||
} | } | ||||
mpte->pindex = pmap_l2_pindex(va); | |||||
if (va < VM_MAXUSER_ADDRESS) { | if (va < VM_MAXUSER_ADDRESS) { | ||||
mpte->ref_count = Ln_ENTRIES; | mpte->ref_count = Ln_ENTRIES; | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
} | } | ||||
} | } | ||||
mptepa = VM_PAGE_TO_PHYS(mpte); | mptepa = VM_PAGE_TO_PHYS(mpte); | ||||
firstl3 = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | firstl3 = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | ||||
newl2 = ((mptepa / PAGE_SIZE) << PTE_PPN0_S) | PTE_V; | newl2 = ((mptepa / PAGE_SIZE) << PTE_PPN0_S) | PTE_V; | ||||
▲ Show 20 Lines • Show All 235 Lines • ▼ Show 20 Lines | if (mpte == NULL && nosleep) { | ||||
return (KERN_RESOURCE_SHORTAGE); | return (KERN_RESOURCE_SHORTAGE); | ||||
} | } | ||||
l3 = pmap_l3(pmap, va); | l3 = pmap_l3(pmap, va); | ||||
} else { | } else { | ||||
l3 = pmap_l3(pmap, va); | l3 = pmap_l3(pmap, va); | ||||
/* TODO: This is not optimal, but should mostly work */ | /* TODO: This is not optimal, but should mostly work */ | ||||
if (l3 == NULL) { | if (l3 == NULL) { | ||||
if (l2 == NULL) { | if (l2 == NULL) { | ||||
l2_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | l2_m = vm_page_alloc_noobj(VM_ALLOC_WIRED | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | |||||
VM_ALLOC_ZERO); | VM_ALLOC_ZERO); | ||||
if (l2_m == NULL) | if (l2_m == NULL) | ||||
panic("pmap_enter: l2 pte_m == NULL"); | panic("pmap_enter: l2 pte_m == NULL"); | ||||
if ((l2_m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(l2_m); | |||||
l2_pa = VM_PAGE_TO_PHYS(l2_m); | l2_pa = VM_PAGE_TO_PHYS(l2_m); | ||||
l2_pn = (l2_pa / PAGE_SIZE); | l2_pn = (l2_pa / PAGE_SIZE); | ||||
l1 = pmap_l1(pmap, va); | l1 = pmap_l1(pmap, va); | ||||
entry = (PTE_V); | entry = (PTE_V); | ||||
entry |= (l2_pn << PTE_PPN0_S); | entry |= (l2_pn << PTE_PPN0_S); | ||||
pmap_store(l1, entry); | pmap_store(l1, entry); | ||||
pmap_distribute_l1(pmap, pmap_l1_index(va), entry); | pmap_distribute_l1(pmap, pmap_l1_index(va), entry); | ||||
l2 = pmap_l1_to_l2(l1, va); | l2 = pmap_l1_to_l2(l1, va); | ||||
} | } | ||||
l3_m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | l3_m = vm_page_alloc_noobj(VM_ALLOC_WIRED | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); | VM_ALLOC_ZERO); | ||||
if (l3_m == NULL) | if (l3_m == NULL) | ||||
panic("pmap_enter: l3 pte_m == NULL"); | panic("pmap_enter: l3 pte_m == NULL"); | ||||
if ((l3_m->flags & PG_ZERO) == 0) | if ((l3_m->flags & PG_ZERO) == 0) | ||||
pmap_zero_page(l3_m); | pmap_zero_page(l3_m); | ||||
l3_pa = VM_PAGE_TO_PHYS(l3_m); | l3_pa = VM_PAGE_TO_PHYS(l3_m); | ||||
l3_pn = (l3_pa / PAGE_SIZE); | l3_pn = (l3_pa / PAGE_SIZE); | ||||
entry = (PTE_V); | entry = (PTE_V); | ||||
▲ Show 20 Lines • Show All 2,011 Lines • Show Last 20 Lines |