Changeset View
Changeset View
Standalone View
Standalone View
sys/riscv/riscv/pmap.c
Show First 20 Lines • Show All 222 Lines • ▼ Show 20 Lines | if (*_lockp != NULL) { \ | ||||
rw_wunlock(*_lockp); \ | rw_wunlock(*_lockp); \ | ||||
*_lockp = NULL; \ | *_lockp = NULL; \ | ||||
} \ | } \ | ||||
} while (0) | } while (0) | ||||
#define VM_PAGE_TO_PV_LIST_LOCK(m) \ | #define VM_PAGE_TO_PV_LIST_LOCK(m) \ | ||||
PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) | PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) | ||||
static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | |||||
"VM/pmap parameters"); | |||||
/* The list of all the user pmaps */ | /* The list of all the user pmaps */ | ||||
LIST_HEAD(pmaplist, pmap); | LIST_HEAD(pmaplist, pmap); | ||||
static struct pmaplist allpmaps = LIST_HEAD_INITIALIZER(); | static struct pmaplist allpmaps = LIST_HEAD_INITIALIZER(); | ||||
enum pmap_mode __read_frequently pmap_mode = PMAP_MODE_SV39; | enum pmap_mode __read_frequently pmap_mode = PMAP_MODE_SV39; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, mode, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, | |||||
&pmap_mode, 0, | |||||
"translation mode, 0 = SV39, 1 = SV48"); | |||||
struct pmap kernel_pmap_store; | struct pmap kernel_pmap_store; | ||||
vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ | vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ | ||||
vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ | vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ | ||||
vm_offset_t kernel_vm_end = 0; | vm_offset_t kernel_vm_end = 0; | ||||
vm_paddr_t dmap_phys_base; /* The start of the dmap region */ | vm_paddr_t dmap_phys_base; /* The start of the dmap region */ | ||||
vm_paddr_t dmap_phys_max; /* The limit of the dmap region */ | vm_paddr_t dmap_phys_max; /* The limit of the dmap region */ | ||||
vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */ | vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */ | ||||
/* This code assumes all L1 DMAP entries will be used */ | /* This code assumes all L1 DMAP entries will be used */ | ||||
CTASSERT((DMAP_MIN_ADDRESS & ~L1_OFFSET) == DMAP_MIN_ADDRESS); | CTASSERT((DMAP_MIN_ADDRESS & ~L1_OFFSET) == DMAP_MIN_ADDRESS); | ||||
CTASSERT((DMAP_MAX_ADDRESS & ~L1_OFFSET) == DMAP_MAX_ADDRESS); | CTASSERT((DMAP_MAX_ADDRESS & ~L1_OFFSET) == DMAP_MAX_ADDRESS); | ||||
static struct rwlock_padalign pvh_global_lock; | static struct rwlock_padalign pvh_global_lock; | ||||
static struct mtx_padalign allpmaps_lock; | static struct mtx_padalign allpmaps_lock; | ||||
static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | |||||
"VM/pmap parameters"); | |||||
static int superpages_enabled = 1; | static int superpages_enabled = 1; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, | SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, | ||||
CTLFLAG_RDTUN, &superpages_enabled, 0, | CTLFLAG_RDTUN, &superpages_enabled, 0, | ||||
"Enable support for transparent superpages"); | "Enable support for transparent superpages"); | ||||
static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, | ||||
"2MB page mapping counters"); | "2MB page mapping counters"); | ||||
▲ Show 20 Lines • Show All 347 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
/* | /* | ||||
* Bootstrap the system enough to run with virtual memory. | * Bootstrap the system enough to run with virtual memory. | ||||
*/ | */ | ||||
void | void | ||||
pmap_bootstrap(vm_offset_t l1pt, vm_paddr_t kernstart, vm_size_t kernlen) | pmap_bootstrap(vm_offset_t l1pt, vm_paddr_t kernstart, vm_size_t kernlen) | ||||
{ | { | ||||
u_int l1_slot, l2_slot; | uint64_t satp; | ||||
vm_offset_t freemempos; | vm_offset_t dpcpu, freemempos, l0pv, msgbufpv; | ||||
vm_offset_t dpcpu, msgbufpv; | vm_paddr_t l0pa, l1pa, max_pa, min_pa, pa; | ||||
vm_paddr_t max_pa, min_pa, pa; | pd_entry_t *l0p; | ||||
pt_entry_t *l2p; | pt_entry_t *l2p; | ||||
int i; | u_int l1_slot, l2_slot; | ||||
int i, mode; | |||||
printf("pmap_bootstrap %lx %lx %lx\n", l1pt, kernstart, kernlen); | printf("pmap_bootstrap %lx %lx %lx\n", l1pt, kernstart, kernlen); | ||||
/* Set this early so we can use the pagetable walking functions */ | /* Set this early so we can use the pagetable walking functions */ | ||||
kernel_pmap_store.pm_top = (pd_entry_t *)l1pt; | kernel_pmap_store.pm_top = (pd_entry_t *)l1pt; | ||||
PMAP_LOCK_INIT(kernel_pmap); | PMAP_LOCK_INIT(kernel_pmap); | ||||
rw_init(&pvh_global_lock, "pmap pv global"); | rw_init(&pvh_global_lock, "pmap pv global"); | ||||
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | pmap_bootstrap(vm_offset_t l1pt, vm_paddr_t kernstart, vm_size_t kernlen) | ||||
sfence_vma(); | sfence_vma(); | ||||
#define alloc_pages(var, np) \ | #define alloc_pages(var, np) \ | ||||
(var) = freemempos; \ | (var) = freemempos; \ | ||||
freemempos += (np * PAGE_SIZE); \ | freemempos += (np * PAGE_SIZE); \ | ||||
memset((char *)(var), 0, ((np) * PAGE_SIZE)); | memset((char *)(var), 0, ((np) * PAGE_SIZE)); | ||||
mode = 0; | |||||
TUNABLE_INT_FETCH("vm.pmap.mode", &mode); | |||||
if (mode == PMAP_MODE_SV48) { | |||||
/* | |||||
* Enable SV48 mode: allocate an L0 page and set SV48 mode in | |||||
* SATP. If the implementation does not provide SV48 mode, | |||||
* the mode read back from the (WARL) SATP register will be | |||||
* unchanged, and we continue in SV39 mode. | |||||
*/ | |||||
alloc_pages(l0pv, 1); | |||||
l0p = (void *)l0pv; | |||||
l1pa = pmap_early_vtophys(l1pt, l1pt); | |||||
l0p[pmap_l0_index(KERNBASE)] = PTE_V | PTE_A | PTE_D | | |||||
((l1pa >> PAGE_SHIFT) << PTE_PPN0_S); | |||||
l0pa = pmap_early_vtophys(l1pt, l0pv); | |||||
csr_write(satp, (l0pa >> PAGE_SHIFT) | SATP_MODE_SV48); | |||||
satp = csr_read(satp); | |||||
if ((satp & SATP_MODE_M) == SATP_MODE_SV48) { | |||||
pmap_mode = PMAP_MODE_SV48; | |||||
kernel_pmap_store.pm_top = l0p; | |||||
} else { | |||||
/* Mode didn't change, give the page back. */ | |||||
freemempos -= PAGE_SIZE; | |||||
} | |||||
} | |||||
/* Allocate dynamic per-cpu area. */ | /* Allocate dynamic per-cpu area. */ | ||||
alloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); | alloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); | ||||
dpcpu_init((void *)dpcpu, 0); | dpcpu_init((void *)dpcpu, 0); | ||||
/* Allocate memory for the msgbuf, e.g. for /sbin/dmesg */ | /* Allocate memory for the msgbuf, e.g. for /sbin/dmesg */ | ||||
alloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); | alloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); | ||||
msgbufp = (void *)msgbufpv; | msgbufp = (void *)msgbufpv; | ||||
▲ Show 20 Lines • Show All 557 Lines • ▼ Show 20 Lines | pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | ||||
if (va >= VM_MAXUSER_ADDRESS) | if (va >= VM_MAXUSER_ADDRESS) | ||||
return (0); | return (0); | ||||
KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); | KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); | ||||
mpte = PHYS_TO_VM_PAGE(PTE_TO_PHYS(ptepde)); | mpte = PHYS_TO_VM_PAGE(PTE_TO_PHYS(ptepde)); | ||||
return (pmap_unwire_ptp(pmap, va, mpte, free)); | return (pmap_unwire_ptp(pmap, va, mpte, free)); | ||||
} | } | ||||
static uint64_t | |||||
pmap_satp_mode(void) | |||||
{ | |||||
return (pmap_mode == PMAP_MODE_SV39 ? SATP_MODE_SV39 : SATP_MODE_SV48); | |||||
} | |||||
void | void | ||||
pmap_pinit0(pmap_t pmap) | pmap_pinit0(pmap_t pmap) | ||||
{ | { | ||||
PMAP_LOCK_INIT(pmap); | PMAP_LOCK_INIT(pmap); | ||||
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | ||||
pmap->pm_top = kernel_pmap->pm_top; | pmap->pm_top = kernel_pmap->pm_top; | ||||
pmap->pm_satp = SATP_MODE_SV39 | (vtophys(pmap->pm_top) >> PAGE_SHIFT); | pmap->pm_satp = pmap_satp_mode() | | ||||
(vtophys(pmap->pm_top) >> PAGE_SHIFT); | |||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
pmap_activate_boot(pmap); | pmap_activate_boot(pmap); | ||||
} | } | ||||
int | int | ||||
pmap_pinit(pmap_t pmap) | pmap_pinit(pmap_t pmap) | ||||
{ | { | ||||
vm_paddr_t topphys; | vm_paddr_t topphys; | ||||
vm_page_t mtop; | vm_page_t mtop; | ||||
size_t i; | size_t i; | ||||
mtop = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO | | mtop = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO | | ||||
VM_ALLOC_WAITOK); | VM_ALLOC_WAITOK); | ||||
topphys = VM_PAGE_TO_PHYS(mtop); | topphys = VM_PAGE_TO_PHYS(mtop); | ||||
pmap->pm_top = (pd_entry_t *)PHYS_TO_DMAP(topphys); | pmap->pm_top = (pd_entry_t *)PHYS_TO_DMAP(topphys); | ||||
pmap->pm_satp = SATP_MODE_SV39 | (topphys >> PAGE_SHIFT); | pmap->pm_satp = pmap_satp_mode() | (topphys >> PAGE_SHIFT); | ||||
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
if (pmap_mode == PMAP_MODE_SV39) { | if (pmap_mode == PMAP_MODE_SV39) { | ||||
/* | /* | ||||
* Copy L1 entries from the kernel pmap. This must be done with | * Copy L1 entries from the kernel pmap. This must be done with | ||||
▲ Show 20 Lines • Show All 3,652 Lines • Show Last 20 Lines |