Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 2,443 Lines • ▼ Show 20 Lines | pmap_growkernel(vm_offset_t addr) | ||||
} | } | ||||
} | } | ||||
/*************************************************** | /*************************************************** | ||||
* page management routines. | * page management routines. | ||||
***************************************************/ | ***************************************************/ | ||||
CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); | CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); | ||||
#if PAGE_SIZE == PAGE_SIZE_4K | |||||
CTASSERT(_NPCM == 3); | |||||
CTASSERT(_NPCPV == 168); | |||||
#else | |||||
CTASSERT(_NPCM == 11); | |||||
CTASSERT(_NPCPV == 677); | |||||
#endif | |||||
static __inline struct pv_chunk * | static __inline struct pv_chunk * | ||||
pv_to_chunk(pv_entry_t pv) | pv_to_chunk(pv_entry_t pv) | ||||
{ | { | ||||
return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); | return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); | ||||
} | } | ||||
#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) | #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) | ||||
#define PC_FREEN 0xfffffffffffffffful | #define PC_FREEN 0xfffffffffffffffful | ||||
#if _NPCM == 3 | #define PC_FREEL ((1ul << (_NPCPV % 64)) - 1) | ||||
jhb: I think this also makes it a bit more obvious how the value is derived compared to the previous… | |||||
Done Inline Actions+1 for defining _NPCM in terms of _NPCPV. markj: +1 for defining _NPCM in terms of _NPCPV. | |||||
#define PC_FREEL 0x000000fffffffffful | |||||
#elif _NPCM == 11 | |||||
#define PC_FREEL 0x0000001ffffffffful | |||||
#endif | |||||
#if _NPCM == 3 | #if _NPCM == 3 | ||||
#define PC_IS_FREE(pc) ((pc)->pc_map[0] == PC_FREEN && \ | #define PC_IS_FREE(pc) ((pc)->pc_map[0] == PC_FREEN && \ | ||||
(pc)->pc_map[1] == PC_FREEN && (pc)->pc_map[2] == PC_FREEL) | (pc)->pc_map[1] == PC_FREEN && (pc)->pc_map[2] == PC_FREEL) | ||||
#else | #else | ||||
#define PC_IS_FREE(pc) \ | #define PC_IS_FREE(pc) \ | ||||
(memcmp((pc)->pc_map, pc_freemask, sizeof(pc_freemask)) == 0) | (memcmp((pc)->pc_map, pc_freemask, sizeof(pc_freemask)) == 0) | ||||
#endif | #endif | ||||
static const uint64_t pc_freemask[] = { PC_FREEN, PC_FREEN, | static const uint64_t pc_freemask[] = { PC_FREEN, PC_FREEN, | ||||
Done Inline ActionsAfter my other followup commit to reimplement PC_IS_FREE, the definition of pc_freemask is the only place that needs #if's now. It's a bit messier in CheriBSD and it might be nice to just initialize it during pmap_bootstrap() or the like with a simple loop. Here's what it is in CheriBSD: static const uint64_t pc_freemask[] = { PC_FREEN, #if _NPCM > 2 PC_FREEN, #endif #if _NPCM > 3 PC_FREEN, PC_FREEN, PC_FREEN, #endif #if _NPCM > 6 PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, #endif PC_FREEL }; jhb: After my other followup commit to reimplement PC_IS_FREE, the definition of `pc_freemask` is… | |||||
#if _NPCM > 3 | #if _NPCM > 3 | ||||
PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, | PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, PC_FREEN, | ||||
PC_FREEN, | PC_FREEN, | ||||
#endif | #endif | ||||
PC_FREEL | PC_FREEL | ||||
}; | }; | ||||
CTASSERT(nitems(pc_freemask) == _NPCM); | CTASSERT(nitems(pc_freemask) == _NPCM); | ||||
static __inline bool | |||||
pc_is_full(struct pv_chunk *pc) | |||||
{ | |||||
for (u_int i = 0; i < _NPCM; i++) | |||||
if (pc->pc_map[i] != 0) | |||||
Done Inline ActionsThis unrolls to a series of cbnz jhb: This unrolls to a series of cbnz | |||||
return (false); | |||||
return (true); | |||||
} | |||||
#ifdef PV_STATS | #ifdef PV_STATS | ||||
static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; | static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, | SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, | ||||
"Current number of pv entry chunks"); | "Current number of pv entry chunks"); | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, | SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, | ||||
"Current number of pv entry chunks allocated"); | "Current number of pv entry chunks allocated"); | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, | SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, | ||||
▲ Show 20 Lines • Show All 278 Lines • ▼ Show 20 Lines | for (field = 0; field < _NPCM; field++) { | ||||
bit = ffsl(pc->pc_map[field]) - 1; | bit = ffsl(pc->pc_map[field]) - 1; | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
if (field < _NPCM) { | if (field < _NPCM) { | ||||
pv = &pc->pc_pventry[field * 64 + bit]; | pv = &pc->pc_pventry[field * 64 + bit]; | ||||
pc->pc_map[field] &= ~(1ul << bit); | pc->pc_map[field] &= ~(1ul << bit); | ||||
/* If this was the last item, move it to tail */ | /* If this was the last item, move it to tail */ | ||||
if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && | if (pc_is_full(pc)) { | ||||
Not Done Inline ActionsHmm, I think this test could be implemented more efficiently: if (field == _NPCM - 1 && pc->pc_map[_NPCM - 1] == 0) That is, take advantage of the fact that we've already scanned the bitmap looking for a set bit. This diff seems fine as-is though, I can write a follow-up patch. markj: Hmm, I think this test could be implemented more efficiently:
```
if (field == _NPCM - 1 && pc… | |||||
Done Inline ActionsI agree that's a useful optimization and that it should be a separate followup. jhb: I agree that's a useful optimization and that it should be a separate followup. | |||||
pc->pc_map[2] == 0) { | |||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | ||||
TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, | TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, | ||||
pc_list); | pc_list); | ||||
} | } | ||||
PV_STAT(atomic_add_long(&pv_entry_count, 1)); | PV_STAT(atomic_add_long(&pv_entry_count, 1)); | ||||
PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); | PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); | ||||
return (pv); | return (pv); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 150 Lines • ▼ Show 20 Lines | pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | ||||
m = PHYS_TO_VM_PAGE(pa); | m = PHYS_TO_VM_PAGE(pa); | ||||
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
/* Instantiate the remaining Ln_ENTRIES - 1 pv entries. */ | /* Instantiate the remaining Ln_ENTRIES - 1 pv entries. */ | ||||
PV_STAT(atomic_add_long(&pv_entry_allocs, Ln_ENTRIES - 1)); | PV_STAT(atomic_add_long(&pv_entry_allocs, Ln_ENTRIES - 1)); | ||||
va_last = va + L2_SIZE - PAGE_SIZE; | va_last = va + L2_SIZE - PAGE_SIZE; | ||||
for (;;) { | for (;;) { | ||||
pc = TAILQ_FIRST(&pmap->pm_pvchunk); | pc = TAILQ_FIRST(&pmap->pm_pvchunk); | ||||
KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 || | KASSERT(!pc_is_full(pc), ("pmap_pv_demote_l2: missing spare")); | ||||
pc->pc_map[2] != 0, ("pmap_pv_demote_l2: missing spare")); | |||||
for (field = 0; field < _NPCM; field++) { | for (field = 0; field < _NPCM; field++) { | ||||
while (pc->pc_map[field]) { | while (pc->pc_map[field]) { | ||||
bit = ffsl(pc->pc_map[field]) - 1; | bit = ffsl(pc->pc_map[field]) - 1; | ||||
pc->pc_map[field] &= ~(1ul << bit); | pc->pc_map[field] &= ~(1ul << bit); | ||||
pv = &pc->pc_pventry[field * 64 + bit]; | pv = &pc->pc_pventry[field * 64 + bit]; | ||||
va += PAGE_SIZE; | va += PAGE_SIZE; | ||||
pv->pv_va = va; | pv->pv_va = va; | ||||
m++; | m++; | ||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | KASSERT((m->oflags & VPO_UNMANAGED) == 0, | ||||
("pmap_pv_demote_l2: page %p is not managed", m)); | ("pmap_pv_demote_l2: page %p is not managed", m)); | ||||
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
if (va == va_last) | if (va == va_last) | ||||
goto out; | goto out; | ||||
} | } | ||||
} | } | ||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | ||||
TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); | TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); | ||||
} | } | ||||
out: | out: | ||||
if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) { | if (pc_is_full(pc)) { | ||||
Not Done Inline ActionsI think a similar optimization can be applied here. markj: I think a similar optimization can be applied here. | |||||
Not Done Inline ActionsOops, no, not in this case. markj: Oops, no, not in this case. | |||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | ||||
TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); | TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); | ||||
} | } | ||||
PV_STAT(atomic_add_long(&pv_entry_count, Ln_ENTRIES - 1)); | PV_STAT(atomic_add_long(&pv_entry_count, Ln_ENTRIES - 1)); | ||||
PV_STAT(atomic_subtract_int(&pv_entry_spare, Ln_ENTRIES - 1)); | PV_STAT(atomic_subtract_int(&pv_entry_spare, Ln_ENTRIES - 1)); | ||||
} | } | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 4,559 Lines • Show Last 20 Lines |
I think this also makes it a bit more obvious how the value is derived compared to the previous versions. Possibly we could even define _NPCM as howmany(_NPCPV, 64) which would be in a similar vein.
In CheriBSD purecap kernels have different values for _NPCPV so we end up with 4 values here otherwise.