Changeset View
Changeset View
Standalone View
Standalone View
head/sys/powerpc/pseries/mmu_phyp.c
Show First 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | static void mphyp_bootstrap(vm_offset_t kernelstart, | ||||
vm_offset_t kernelend); | vm_offset_t kernelend); | ||||
static void mphyp_cpu_bootstrap(int ap); | static void mphyp_cpu_bootstrap(int ap); | ||||
static void *mphyp_dump_pmap(void *ctx, void *buf, | static void *mphyp_dump_pmap(void *ctx, void *buf, | ||||
u_long *nbytes); | u_long *nbytes); | ||||
static int64_t mphyp_pte_synch(struct pvo_entry *pvo); | static int64_t mphyp_pte_synch(struct pvo_entry *pvo); | ||||
static int64_t mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit); | static int64_t mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit); | ||||
static int64_t mphyp_pte_unset(struct pvo_entry *pvo); | static int64_t mphyp_pte_unset(struct pvo_entry *pvo); | ||||
static int64_t mphyp_pte_insert(struct pvo_entry *pvo); | static int64_t mphyp_pte_insert(struct pvo_entry *pvo); | ||||
static int64_t mphyp_pte_unset_sp(struct pvo_entry *pvo); | |||||
static int64_t mphyp_pte_insert_sp(struct pvo_entry *pvo); | |||||
static int64_t mphyp_pte_replace_sp(struct pvo_entry *pvo); | |||||
static struct pmap_funcs mphyp_methods = { | static struct pmap_funcs mphyp_methods = { | ||||
.install = mphyp_install, | .install = mphyp_install, | ||||
.bootstrap = mphyp_bootstrap, | .bootstrap = mphyp_bootstrap, | ||||
.cpu_bootstrap = mphyp_cpu_bootstrap, | .cpu_bootstrap = mphyp_cpu_bootstrap, | ||||
.dumpsys_dump_pmap = mphyp_dump_pmap, | .dumpsys_dump_pmap = mphyp_dump_pmap, | ||||
}; | }; | ||||
static struct moea64_funcs mmu_phyp_funcs = { | static struct moea64_funcs mmu_phyp_funcs = { | ||||
.pte_synch = mphyp_pte_synch, | .pte_synch = mphyp_pte_synch, | ||||
.pte_clear = mphyp_pte_clear, | .pte_clear = mphyp_pte_clear, | ||||
.pte_unset = mphyp_pte_unset, | .pte_unset = mphyp_pte_unset, | ||||
.pte_insert = mphyp_pte_insert, | .pte_insert = mphyp_pte_insert, | ||||
.pte_unset_sp = mphyp_pte_unset_sp, | |||||
.pte_insert_sp = mphyp_pte_insert_sp, | |||||
.pte_replace_sp = mphyp_pte_replace_sp, | |||||
}; | }; | ||||
MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, oea64_mmu); | MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, oea64_mmu); | ||||
static int brokenkvm = 0; | static int brokenkvm = 0; | ||||
static void | static void | ||||
print_kvm_bug_warning(void *data) | print_kvm_bug_warning(void *data) | ||||
Show All 24 Lines | mphyp_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend) | ||||
char buf[8]; | char buf[8]; | ||||
uint32_t prop[2]; | uint32_t prop[2]; | ||||
uint32_t nptlp, shift = 0, slb_encoding = 0; | uint32_t nptlp, shift = 0, slb_encoding = 0; | ||||
uint32_t lp_size, lp_encoding; | uint32_t lp_size, lp_encoding; | ||||
struct lpte old; | struct lpte old; | ||||
uint64_t vsid; | uint64_t vsid; | ||||
phandle_t dev, node, root; | phandle_t dev, node, root; | ||||
int idx, len, res; | int idx, len, res; | ||||
bool has_lp; | |||||
rm_init(&mphyp_eviction_lock, "pte eviction"); | rm_init(&mphyp_eviction_lock, "pte eviction"); | ||||
moea64_early_bootstrap(kernelstart, kernelend); | moea64_early_bootstrap(kernelstart, kernelend); | ||||
root = OF_peer(0); | root = OF_peer(0); | ||||
dev = OF_child(root); | dev = OF_child(root); | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | if (len > 0) { | ||||
* We have to use a variable length array on the stack | * We have to use a variable length array on the stack | ||||
* since we have very limited stack space. | * since we have very limited stack space. | ||||
*/ | */ | ||||
pcell_t arr[len/sizeof(cell_t)]; | pcell_t arr[len/sizeof(cell_t)]; | ||||
res = OF_getencprop(node, "ibm,segment-page-sizes", arr, | res = OF_getencprop(node, "ibm,segment-page-sizes", arr, | ||||
sizeof(arr)); | sizeof(arr)); | ||||
len /= 4; | len /= 4; | ||||
idx = 0; | idx = 0; | ||||
has_lp = false; | |||||
while (len > 0) { | while (len > 0) { | ||||
shift = arr[idx]; | shift = arr[idx]; | ||||
slb_encoding = arr[idx + 1]; | slb_encoding = arr[idx + 1]; | ||||
nptlp = arr[idx + 2]; | nptlp = arr[idx + 2]; | ||||
dprintf0("Segment Page Size: " | dprintf0("Segment Page Size: " | ||||
"%uKB, slb_enc=0x%X: {size, encoding}[%u] =", | "%uKB, slb_enc=0x%X: {size, encoding}[%u] =", | ||||
shift > 10? 1 << (shift-10) : 0, | shift > 10? 1 << (shift-10) : 0, | ||||
slb_encoding, nptlp); | slb_encoding, nptlp); | ||||
idx += 3; | idx += 3; | ||||
len -= 3; | len -= 3; | ||||
while (len > 0 && nptlp) { | while (len > 0 && nptlp) { | ||||
lp_size = arr[idx]; | lp_size = arr[idx]; | ||||
lp_encoding = arr[idx+1]; | lp_encoding = arr[idx+1]; | ||||
dprintf(" {%uKB, 0x%X}", | dprintf(" {%uKB, 0x%X}", | ||||
lp_size > 10? 1 << (lp_size-10) : 0, | lp_size > 10? 1 << (lp_size-10) : 0, | ||||
lp_encoding); | lp_encoding); | ||||
if (slb_encoding == SLBV_L && lp_encoding == 0) | if (slb_encoding == SLBV_L && lp_encoding == 0) | ||||
break; | has_lp = true; | ||||
if (slb_encoding == SLB_PGSZ_4K_4K && | |||||
lp_encoding == LP_4K_16M) | |||||
moea64_has_lp_4k_16m = true; | |||||
idx += 2; | idx += 2; | ||||
len -= 2; | len -= 2; | ||||
nptlp--; | nptlp--; | ||||
} | } | ||||
dprintf("\n"); | dprintf("\n"); | ||||
if (nptlp && slb_encoding == SLBV_L && lp_encoding == 0) | if (has_lp && moea64_has_lp_4k_16m) | ||||
break; | break; | ||||
} | } | ||||
if (len > 0) { | if (has_lp) { | ||||
moea64_large_page_shift = shift; | moea64_large_page_shift = shift; | ||||
moea64_large_page_size = 1ULL << lp_size; | moea64_large_page_size = 1ULL << lp_size; | ||||
moea64_large_page_mask = moea64_large_page_size - 1; | moea64_large_page_mask = moea64_large_page_size - 1; | ||||
hw_direct_map = 1; | hw_direct_map = 1; | ||||
printf(MMU_PHYP_ID | printf(MMU_PHYP_ID | ||||
"Support for hugepages of %uKB detected\n", | "Support for hugepages of %uKB detected\n", | ||||
moea64_large_page_shift > 10? | moea64_large_page_shift > 10? | ||||
1 << (moea64_large_page_shift-10) : 0); | 1 << (moea64_large_page_shift-10) : 0); | ||||
▲ Show 20 Lines • Show All 145 Lines • ▼ Show 20 Lines | mphyp_pte_spillable_ident(uintptr_t ptegbase, struct lpte *to_evict) | ||||
/* Start at a random slot */ | /* Start at a random slot */ | ||||
i = mftb() % 8; | i = mftb() % 8; | ||||
k = -1; | k = -1; | ||||
for (j = 0; j < 8; j++) { | for (j = 0; j < 8; j++) { | ||||
slot = ptegbase + (i + j) % 8; | slot = ptegbase + (i + j) % 8; | ||||
phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, | phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, | ||||
&pt.pte_lo, &junk); | &pt.pte_lo, &junk); | ||||
if (pt.pte_hi & LPTE_WIRED) | if ((pt.pte_hi & (LPTE_WIRED | LPTE_BIG)) != 0) | ||||
continue; | continue; | ||||
/* This is a candidate, so remember it */ | /* This is a candidate, so remember it */ | ||||
k = slot; | k = slot; | ||||
/* Try to get a page that has not been used lately */ | /* Try to get a page that has not been used lately */ | ||||
if (!(pt.pte_hi & LPTE_VALID) || !(pt.pte_lo & LPTE_REF)) { | if (!(pt.pte_hi & LPTE_VALID) || !(pt.pte_lo & LPTE_REF)) { | ||||
memcpy(to_evict, &pt, sizeof(struct lpte)); | memcpy(to_evict, &pt, sizeof(struct lpte)); | ||||
return (k); | return (k); | ||||
} | } | ||||
} | } | ||||
if (k == -1) | if (k == -1) | ||||
return (k); | return (k); | ||||
phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi, | phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi, | ||||
&to_evict->pte_lo, &junk); | &to_evict->pte_lo, &junk); | ||||
return (k); | return (k); | ||||
} | } | ||||
static int64_t | static __inline int64_t | ||||
mphyp_pte_insert(struct pvo_entry *pvo) | mphyp_pte_insert_locked(struct pvo_entry *pvo, struct lpte *pte) | ||||
{ | { | ||||
struct rm_priotracker track; | struct lpte evicted; | ||||
uint64_t index, junk; | |||||
int64_t result; | int64_t result; | ||||
struct lpte evicted, pte; | |||||
uint64_t index, junk, lastptelo; | |||||
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); | |||||
/* Initialize PTE */ | |||||
moea64_pte_from_pvo(pvo, &pte); | |||||
evicted.pte_hi = 0; | |||||
/* Make sure further insertion is locked out during evictions */ | |||||
rm_rlock(&mphyp_eviction_lock, &track); | |||||
/* | /* | ||||
* First try primary hash. | * First try primary hash. | ||||
*/ | */ | ||||
pvo->pvo_pte.slot &= ~7UL; /* Base slot address */ | pvo->pvo_pte.slot &= ~7UL; /* Base slot address */ | ||||
result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte.pte_hi, | result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte->pte_hi, | ||||
pte.pte_lo, &index, &evicted.pte_lo, &junk); | pte->pte_lo, &index, &evicted.pte_lo, &junk); | ||||
if (result == H_SUCCESS) { | if (result == H_SUCCESS) { | ||||
rm_runlock(&mphyp_eviction_lock, &track); | |||||
pvo->pvo_pte.slot = index; | pvo->pvo_pte.slot = index; | ||||
return (0); | return (0); | ||||
} | } | ||||
KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld " | KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld " | ||||
"(ptegidx: %#zx/%#lx, PTE %#lx/%#lx", result, pvo->pvo_pte.slot, | "(ptegidx: %#zx/%#lx, PTE %#lx/%#lx", result, pvo->pvo_pte.slot, | ||||
moea64_pteg_count, pte.pte_hi, pte.pte_lo)); | moea64_pteg_count, pte->pte_hi, pte->pte_lo)); | ||||
/* | /* | ||||
* Next try secondary hash. | * Next try secondary hash. | ||||
*/ | */ | ||||
pvo->pvo_vaddr ^= PVO_HID; | pvo->pvo_vaddr ^= PVO_HID; | ||||
pte.pte_hi ^= LPTE_HID; | pte->pte_hi ^= LPTE_HID; | ||||
pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); | pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); | ||||
result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, | result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, | ||||
pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk); | pte->pte_hi, pte->pte_lo, &index, &evicted.pte_lo, &junk); | ||||
if (result == H_SUCCESS) { | if (result == H_SUCCESS) { | ||||
rm_runlock(&mphyp_eviction_lock, &track); | |||||
pvo->pvo_pte.slot = index; | pvo->pvo_pte.slot = index; | ||||
return (0); | return (0); | ||||
} | } | ||||
KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld", | KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld", | ||||
result)); | result)); | ||||
/* | return (-1); | ||||
* Out of luck. Find a PTE to sacrifice. | } | ||||
*/ | |||||
/* Lock out all insertions for a bit */ | |||||
rm_runlock(&mphyp_eviction_lock, &track); | |||||
rm_wlock(&mphyp_eviction_lock); | |||||
static __inline int64_t | |||||
mphyp_pte_evict_and_insert_locked(struct pvo_entry *pvo, struct lpte *pte) | |||||
{ | |||||
struct lpte evicted; | |||||
uint64_t index, junk, lastptelo; | |||||
int64_t result; | |||||
evicted.pte_hi = 0; | |||||
index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); | index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); | ||||
if (index == -1L) { | if (index == -1L) { | ||||
/* Try other hash table? */ | /* Try other hash table? */ | ||||
pvo->pvo_vaddr ^= PVO_HID; | pvo->pvo_vaddr ^= PVO_HID; | ||||
pte.pte_hi ^= LPTE_HID; | pte->pte_hi ^= LPTE_HID; | ||||
pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); | pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); | ||||
index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); | index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); | ||||
} | } | ||||
if (index == -1L) { | if (index == -1L) { | ||||
/* No freeable slots in either PTEG? We're hosed. */ | /* No freeable slots in either PTEG? We're hosed. */ | ||||
rm_wunlock(&mphyp_eviction_lock); | rm_wunlock(&mphyp_eviction_lock); | ||||
panic("mphyp_pte_insert: overflow"); | panic("mphyp_pte_insert: overflow"); | ||||
return (-1); | return (-1); | ||||
} | } | ||||
/* Victim acquired: update page before waving goodbye */ | /* Victim acquired: update page before waving goodbye */ | ||||
if (evicted.pte_hi & LPTE_VALID) { | if (evicted.pte_hi & LPTE_VALID) { | ||||
result = phyp_pft_hcall(H_REMOVE, H_AVPN, index, | result = phyp_pft_hcall(H_REMOVE, H_AVPN, index, | ||||
evicted.pte_hi & LPTE_AVPN_MASK, 0, &junk, &lastptelo, | evicted.pte_hi & LPTE_AVPN_MASK, 0, &junk, &lastptelo, | ||||
&junk); | &junk); | ||||
STAT_MOEA64(moea64_pte_overflow++); | STAT_MOEA64(moea64_pte_overflow++); | ||||
KASSERT(result == H_SUCCESS || result == H_NOT_FOUND, | KASSERT(result == H_SUCCESS || result == H_NOT_FOUND, | ||||
("Error evicting page: %d", (int)result)); | ("Error evicting page: %d", (int)result)); | ||||
} | } | ||||
/* | /* | ||||
* Set the new PTE. | * Set the new PTE. | ||||
*/ | */ | ||||
result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte.pte_hi, | result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte->pte_hi, | ||||
pte.pte_lo, &index, &evicted.pte_lo, &junk); | pte->pte_lo, &index, &evicted.pte_lo, &junk); | ||||
rm_wunlock(&mphyp_eviction_lock); /* All clear */ | |||||
pvo->pvo_pte.slot = index; | pvo->pvo_pte.slot = index; | ||||
if (result == H_SUCCESS) | if (result == H_SUCCESS) | ||||
return (0); | return (0); | ||||
rm_wunlock(&mphyp_eviction_lock); | |||||
panic("Page replacement error: %ld", result); | panic("Page replacement error: %ld", result); | ||||
return (result); | return (result); | ||||
} | } | ||||
static int64_t | |||||
mphyp_pte_insert(struct pvo_entry *pvo) | |||||
{ | |||||
struct rm_priotracker track; | |||||
int64_t ret; | |||||
struct lpte pte; | |||||
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); | |||||
/* Initialize PTE */ | |||||
moea64_pte_from_pvo(pvo, &pte); | |||||
/* Make sure further insertion is locked out during evictions */ | |||||
rm_rlock(&mphyp_eviction_lock, &track); | |||||
ret = mphyp_pte_insert_locked(pvo, &pte); | |||||
rm_runlock(&mphyp_eviction_lock, &track); | |||||
if (ret == -1) { | |||||
/* | |||||
* Out of luck. Find a PTE to sacrifice. | |||||
*/ | |||||
/* Lock out all insertions for a bit */ | |||||
rm_wlock(&mphyp_eviction_lock); | |||||
ret = mphyp_pte_evict_and_insert_locked(pvo, &pte); | |||||
rm_wunlock(&mphyp_eviction_lock); /* All clear */ | |||||
} | |||||
return (ret); | |||||
} | |||||
static void * | static void * | ||||
mphyp_dump_pmap(void *ctx, void *buf, u_long *nbytes) | mphyp_dump_pmap(void *ctx, void *buf, u_long *nbytes) | ||||
{ | { | ||||
struct dump_context *dctx; | struct dump_context *dctx; | ||||
struct lpte p, *pbuf; | struct lpte p, *pbuf; | ||||
int bufidx; | int bufidx; | ||||
uint64_t junk; | uint64_t junk; | ||||
u_long ptex, ptex_end; | u_long ptex, ptex_end; | ||||
Show All 12 Lines | mphyp_dump_pmap(void *ctx, void *buf, u_long *nbytes) | ||||
for (; ptex < ptex_end; ptex++) { | for (; ptex < ptex_end; ptex++) { | ||||
phyp_pft_hcall(H_READ, 0, ptex, 0, 0, | phyp_pft_hcall(H_READ, 0, ptex, 0, 0, | ||||
&p.pte_hi, &p.pte_lo, &junk); | &p.pte_hi, &p.pte_lo, &junk); | ||||
pbuf[bufidx++] = p; | pbuf[bufidx++] = p; | ||||
} | } | ||||
dctx->ptex = ptex; | dctx->ptex = ptex; | ||||
return (buf); | return (buf); | ||||
} | |||||
static int64_t | |||||
mphyp_pte_unset_sp(struct pvo_entry *pvo) | |||||
{ | |||||
struct lpte pte; | |||||
uint64_t junk, refchg; | |||||
int err; | |||||
vm_offset_t eva; | |||||
pmap_t pm; | |||||
pm = pvo->pvo_pmap; | |||||
PMAP_LOCK_ASSERT(pm, MA_OWNED); | |||||
KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0, | |||||
("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo))); | |||||
refchg = 0; | |||||
eva = PVO_VADDR(pvo) + HPT_SP_SIZE; | |||||
for (; pvo != NULL && PVO_VADDR(pvo) < eva; | |||||
pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { | |||||
moea64_pte_from_pvo(pvo, &pte); | |||||
err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot, | |||||
pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo, | |||||
&junk); | |||||
KASSERT(err == H_SUCCESS || err == H_NOT_FOUND, | |||||
("Error removing page: %d", err)); | |||||
if (err == H_NOT_FOUND) | |||||
STAT_MOEA64(moea64_pte_overflow--); | |||||
refchg |= pte.pte_lo & (LPTE_REF | LPTE_CHG); | |||||
} | |||||
return (refchg); | |||||
} | |||||
static int64_t | |||||
mphyp_pte_insert_sp(struct pvo_entry *pvo) | |||||
{ | |||||
struct rm_priotracker track; | |||||
int64_t ret; | |||||
struct lpte pte; | |||||
vm_offset_t eva; | |||||
pmap_t pm; | |||||
pm = pvo->pvo_pmap; | |||||
PMAP_LOCK_ASSERT(pm, MA_OWNED); | |||||
KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0, | |||||
("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo))); | |||||
eva = PVO_VADDR(pvo) + HPT_SP_SIZE; | |||||
/* Make sure further insertion is locked out during evictions */ | |||||
rm_rlock(&mphyp_eviction_lock, &track); | |||||
for (; pvo != NULL && PVO_VADDR(pvo) < eva; | |||||
pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { | |||||
/* Initialize PTE */ | |||||
moea64_pte_from_pvo(pvo, &pte); | |||||
ret = mphyp_pte_insert_locked(pvo, &pte); | |||||
if (ret == -1) { | |||||
/* | |||||
* Out of luck. Find a PTE to sacrifice. | |||||
*/ | |||||
/* Lock out all insertions for a bit */ | |||||
rm_runlock(&mphyp_eviction_lock, &track); | |||||
rm_wlock(&mphyp_eviction_lock); | |||||
mphyp_pte_evict_and_insert_locked(pvo, &pte); | |||||
rm_wunlock(&mphyp_eviction_lock); /* All clear */ | |||||
rm_rlock(&mphyp_eviction_lock, &track); | |||||
} | |||||
} | |||||
rm_runlock(&mphyp_eviction_lock, &track); | |||||
return (0); | |||||
} | |||||
static int64_t | |||||
mphyp_pte_replace_sp(struct pvo_entry *pvo) | |||||
{ | |||||
int64_t refchg; | |||||
refchg = mphyp_pte_unset_sp(pvo); | |||||
mphyp_pte_insert_sp(pvo); | |||||
return (refchg); | |||||
} | } |