diff --git a/share/man/man7/arch.7 b/share/man/man7/arch.7 --- a/share/man/man7/arch.7 +++ b/share/man/man7/arch.7 @@ -218,7 +218,7 @@ .Ss Page Size .Bl -column -offset indent "Architecture" "Page Sizes" .It Sy Architecture Ta Sy Page Sizes -.It aarch64 Ta 4K, 2M, 1G +.It aarch64 Ta 4K, 64K, 2M, 1G .It amd64 Ta 4K, 2M, 1G .It armv7 Ta 4K, 1M .It i386 Ta 4K, 2M (PAE), 4M diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -1631,11 +1631,14 @@ if (superpages_enabled) { KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, ("pmap_init: can't assign to pagesizes[1]")); - pagesizes[1] = L2_SIZE; + pagesizes[1] = L3C_SIZE; + KASSERT(MAXPAGESIZES > 2 && pagesizes[2] == 0, + ("pmap_init: can't assign to pagesizes[2]")); + pagesizes[2] = L2_SIZE; if (L1_BLOCKS_SUPPORTED) { - KASSERT(MAXPAGESIZES > 2 && pagesizes[2] == 0, - ("pmap_init: can't assign to pagesizes[2]")); - pagesizes[2] = L1_SIZE; + KASSERT(MAXPAGESIZES > 3 && pagesizes[3] == 0, + ("pmap_init: can't assign to pagesizes[3]")); + pagesizes[3] = L1_SIZE; } } @@ -4959,7 +4962,7 @@ pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t pte, int flags, int psind) { - pd_entry_t *l0p, *l1p, *l2p, newpte, origpte; + pd_entry_t *l0p, *l1p, *l2p, *l3p, newpte, origpte, *tl3p; vm_page_t mp; PMAP_LOCK_ASSERT(pmap, MA_OWNED); @@ -4973,9 +4976,11 @@ newpte = pte; if (!pmap_bti_same(pmap, va, va + pagesizes[psind], &newpte)) return (KERN_PROTECTION_FAILURE); - if (psind == 2) { + if (psind == 3) { PMAP_ASSERT_L1_BLOCKS_SUPPORTED; + KASSERT(pagesizes[psind] == L1_SIZE, + ("pagesizes[%d] != L1_SIZE", psind)); l0p = pmap_l0(pmap, va); if ((pmap_load(l0p) & ATTR_DESCR_VALID) == 0) { mp = _pmap_alloc_l3(pmap, pmap_l0_pindex(va), NULL); @@ -5005,7 +5010,9 @@ ("va %#lx changing 1G phys page l1 %#lx newpte %#lx", va, origpte, newpte)); pmap_store(l1p, newpte); - } else /* (psind == 1) */ { + } else if (psind == 2) { + KASSERT(pagesizes[psind] == L2_SIZE, + ("pagesizes[%d] != L2_SIZE", psind)); l2p = pmap_l2(pmap, va); if (l2p == NULL) { mp = _pmap_alloc_l3(pmap, pmap_l1_pindex(va), NULL); @@ -5034,6 +5041,40 @@ ("va %#lx changing 2M phys page l2 %#lx newpte %#lx", va, origpte, newpte)); pmap_store(l2p, newpte); + } else /* (psind == 1) */ { + KASSERT(pagesizes[psind] == L3C_SIZE, + ("pagesizes[%d] != L3C_SIZE", psind)); + l2p = pmap_l2(pmap, va); + if (l2p == NULL || (pmap_load(l2p) & ATTR_DESCR_VALID) == 0) { + mp = _pmap_alloc_l3(pmap, pmap_l2_pindex(va), NULL); + if (mp == NULL) { + if ((flags & PMAP_ENTER_NOSLEEP) != 0) + return (KERN_RESOURCE_SHORTAGE); + PMAP_UNLOCK(pmap); + vm_wait(NULL); + PMAP_LOCK(pmap); + goto restart; + } + mp->ref_count += L3C_ENTRIES - 1; + l3p = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); + l3p = &l3p[pmap_l3_index(va)]; + } else { + l3p = pmap_l2_to_l3(l2p, va); + if ((pmap_load(l3p) & ATTR_DESCR_VALID) == 0) { + mp = PTE_TO_VM_PAGE(pmap_load(l2p)); + mp->ref_count += L3C_ENTRIES; + } + } + for (tl3p = l3p; tl3p < &l3p[L3C_ENTRIES]; tl3p++) { + origpte = pmap_load(tl3p); + KASSERT((origpte & ATTR_DESCR_VALID) == 0 || + ((origpte & ATTR_CONTIGUOUS) != 0 && + PTE_TO_PHYS(origpte) == PTE_TO_PHYS(newpte)), + ("va %#lx changing 64K phys page l3 %#lx newpte %#lx", + va, origpte, newpte)); + pmap_store(tl3p, newpte); + newpte += L3_SIZE; + } } dsb(ishst); @@ -5072,7 +5113,7 @@ vm_paddr_t opa, pa; vm_page_t mpte, om; bool nosleep; - int lvl, rv; + int full_lvl, lvl, rv; KASSERT(ADDR_IS_CANONICAL(va), ("%s: Address not in canonical form: %lx", __func__, va)); @@ -5128,24 +5169,47 @@ if ((flags & PMAP_ENTER_LARGEPAGE) != 0) { KASSERT((m->oflags & VPO_UNMANAGED) != 0, ("managed largepage va %#lx flags %#x", va, flags)); - new_l3 &= ~L3_PAGE; - if (psind == 2) { + if (psind == 3) { PMAP_ASSERT_L1_BLOCKS_SUPPORTED; + new_l3 &= ~L3_PAGE; new_l3 |= L1_BLOCK; - } else /* (psind == 1) */ + } else if (psind == 2) { + new_l3 &= ~L3_PAGE; new_l3 |= L2_BLOCK; + } else /* (psind == 1) */ + new_l3 |= ATTR_CONTIGUOUS; rv = pmap_enter_largepage(pmap, va, new_l3, flags, psind); goto out; } - if (psind == 1) { + if (psind == 2) { /* Assert the required virtual and physical alignment. */ KASSERT((va & L2_OFFSET) == 0, ("pmap_enter: va unaligned")); - KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); + KASSERT(m->psind > 1, ("pmap_enter: m->psind < psind")); rv = pmap_enter_l2(pmap, va, (new_l3 & ~L3_PAGE) | L2_BLOCK, flags, m, &lock); goto out; } mpte = NULL; + if (psind == 1) { + KASSERT((va & L3C_OFFSET) == 0, ("pmap_enter: va unaligned")); + KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); + rv = pmap_enter_l3c(pmap, va, new_l3 | ATTR_CONTIGUOUS, flags, + m, &mpte, &lock); +#if VM_NRESERVLEVEL > 0 + /* + * Attempt L2 promotion, if both the PTP and a level 1 + * reservation are fully populated. + */ + if (rv == KERN_SUCCESS && + (mpte == NULL || mpte->ref_count == NL3PG) && + (m->flags & PG_FICTITIOUS) == 0 && + vm_reserv_level_iffullpop(m) == 1) { + pde = pmap_l2(pmap, va); + (void)pmap_promote_l2(pmap, pde, va, mpte, &lock); + } +#endif + goto out; + } /* * In the case that a page table page is not @@ -5365,15 +5429,14 @@ * are aligned with each other and an underlying reservation has the * neighboring L3 pages allocated. The first condition is simply an * optimization that recognizes some eventual promotion failures early - * at a lower run-time cost. Then, if both the page table page and - * the reservation are fully populated, attempt L2 promotion. + * at a lower run-time cost. Then, if both a level 1 reservation and + * the PTP are fully populated, attempt L2 promotion. */ if ((va & L3C_OFFSET) == (pa & L3C_OFFSET) && (m->flags & PG_FICTITIOUS) == 0 && - vm_reserv_is_populated(m, L3C_ENTRIES) && + (full_lvl = vm_reserv_level_iffullpop(m)) >= 0 && pmap_promote_l3c(pmap, l3, va) && - (mpte == NULL || mpte->ref_count == NL3PG) && - vm_reserv_level_iffullpop(m) == 0) + full_lvl == 1 && (mpte == NULL || mpte->ref_count == NL3PG)) (void)pmap_promote_l2(pmap, pde, va, mpte, &lock); #endif @@ -5667,6 +5730,8 @@ ("pmap_enter_l3c: va is not aligned")); KASSERT(!VA_IS_CLEANMAP(va) || (l3e & ATTR_SW_MANAGED) == 0, ("pmap_enter_l3c: managed mapping within the clean submap")); + KASSERT((l3e & ATTR_CONTIGUOUS) != 0, + ("pmap_enter_l3c: l3e is missing ATTR_CONTIGUOUS")); /* * If the L3 PTP is not resident, we attempt to create it here. @@ -5873,14 +5938,12 @@ while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { va = start + ptoa(diff); if ((va & L2_OFFSET) == 0 && va + L2_SIZE <= end && - m->psind == 1 && pmap_ps_enabled(pmap) && + m->psind == 2 && pmap_ps_enabled(pmap) && ((rv = pmap_enter_l2_rx(pmap, va, m, prot, &lock)) == KERN_SUCCESS || rv == KERN_NO_SPACE)) m = &m[L2_SIZE / PAGE_SIZE - 1]; else if ((va & L3C_OFFSET) == 0 && va + L3C_SIZE <= end && - (VM_PAGE_TO_PHYS(m) & L3C_OFFSET) == 0 && - vm_reserv_is_populated(m, L3C_ENTRIES) && - pmap_ps_enabled(pmap) && + m->psind >= 1 && pmap_ps_enabled(pmap) && ((rv = pmap_enter_l3c_rx(pmap, va, m, &mpte, prot, &lock)) == KERN_SUCCESS || rv == KERN_NO_SPACE)) m = &m[L3C_ENTRIES - 1]; @@ -5932,7 +5995,7 @@ { pt_entry_t *l1, *l2, *l3, l3_val; vm_paddr_t pa; - int lvl; + int full_lvl, lvl; KASSERT(!VA_IS_CLEANMAP(va) || (m->oflags & VPO_UNMANAGED) != 0, @@ -6063,18 +6126,17 @@ * are aligned with each other and an underlying reservation has the * neighboring L3 pages allocated. The first condition is simply an * optimization that recognizes some eventual promotion failures early - * at a lower run-time cost. Then, attempt L2 promotion, if both the - * PTP and the reservation are fully populated. + * at a lower run-time cost. Then, attempt L2 promotion, if both a + * level 1 reservation and the PTP are fully populated. */ if ((prot & VM_PROT_NO_PROMOTE) == 0 && (va & L3C_OFFSET) == (pa & L3C_OFFSET) && (m->flags & PG_FICTITIOUS) == 0 && - vm_reserv_is_populated(m, L3C_ENTRIES) && + (full_lvl = vm_reserv_level_iffullpop(m)) >= 0 && pmap_promote_l3c(pmap, l3, va) && - (mpte == NULL || mpte->ref_count == NL3PG) && - vm_reserv_level_iffullpop(m) == 0) { + full_lvl == 1 && (mpte == NULL || mpte->ref_count == NL3PG)) { if (l2 == NULL) - l2 = pmap_pde(pmap, va, &lvl); + l2 = pmap_l2(pmap, va); /* * If promotion succeeds, then the next call to this function @@ -8566,7 +8628,7 @@ { pt_entry_t *pte, tpte; vm_paddr_t mask, pa; - int lvl, val; + int lvl, psind, val; bool managed; PMAP_ASSERT_STAGE1(pmap); @@ -8578,21 +8640,22 @@ switch (lvl) { case 3: mask = L3_OFFSET; + psind = (tpte & ATTR_CONTIGUOUS) != 0 ? 1 : 0; break; case 2: mask = L2_OFFSET; + psind = 2; break; case 1: mask = L1_OFFSET; + psind = 3; break; default: panic("pmap_mincore: invalid level %d", lvl); } managed = (tpte & ATTR_SW_MANAGED) != 0; - val = MINCORE_INCORE; - if (lvl != 3) - val |= MINCORE_PSIND(3 - lvl); + val = MINCORE_INCORE | MINCORE_PSIND(psind); if ((managed && pmap_pte_dirty(pmap, tpte)) || (!managed && (tpte & ATTR_S1_AP_RW_BIT) == ATTR_S1_AP(ATTR_S1_AP_RW))) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; @@ -9128,18 +9191,37 @@ { vm_offset_t superpage_offset; - if (size < L2_SIZE) + if (size < L3C_SIZE) return; if (object != NULL && (object->flags & OBJ_COLORED) != 0) offset += ptoa(object->pg_color); + + /* + * Considering the object's physical alignment, is the mapping large + * enough to encompass an L2 (2MB/32MB) superpage ... + */ superpage_offset = offset & L2_OFFSET; - if (size - ((L2_SIZE - superpage_offset) & L2_OFFSET) < L2_SIZE || - (*addr & L2_OFFSET) == superpage_offset) + if (size - ((L2_SIZE - superpage_offset) & L2_OFFSET) >= L2_SIZE) { + /* + * If the virtual and physical alignments differ, then + * increase the virtual address so that the alignments match. + */ + if ((*addr & L2_OFFSET) < superpage_offset) + *addr = (*addr & ~L2_OFFSET) + superpage_offset; + else if ((*addr & L2_OFFSET) > superpage_offset) + *addr = ((*addr + L2_OFFSET) & ~L2_OFFSET) + + superpage_offset; return; - if ((*addr & L2_OFFSET) < superpage_offset) - *addr = (*addr & ~L2_OFFSET) + superpage_offset; - else - *addr = ((*addr + L2_OFFSET) & ~L2_OFFSET) + superpage_offset; + } + /* ... or an L3C (64KB/2MB) superpage? */ + superpage_offset = offset & L3C_OFFSET; + if (size - ((L3C_SIZE - superpage_offset) & L3C_OFFSET) >= L3C_SIZE) { + if ((*addr & L3C_OFFSET) < superpage_offset) + *addr = (*addr & ~L3C_OFFSET) + superpage_offset; + else if ((*addr & L3C_OFFSET) > superpage_offset) + *addr = ((*addr + L3C_OFFSET) & ~L3C_OFFSET) + + superpage_offset; + } } /** diff --git a/sys/arm64/include/param.h b/sys/arm64/include/param.h --- a/sys/arm64/include/param.h +++ b/sys/arm64/include/param.h @@ -97,7 +97,7 @@ #define PAGE_SIZE (1 << PAGE_SHIFT) #define PAGE_MASK (PAGE_SIZE - 1) -#define MAXPAGESIZES 3 /* maximum number of supported page sizes */ +#define MAXPAGESIZES 4 /* maximum number of supported page sizes */ #ifndef KSTACK_PAGES #if defined(KASAN) || defined(KMSAN) diff --git a/sys/arm64/include/vmparam.h b/sys/arm64/include/vmparam.h --- a/sys/arm64/include/vmparam.h +++ b/sys/arm64/include/vmparam.h @@ -114,25 +114,34 @@ #endif /* - * Enable superpage reservations: 1 level. + * Enable superpage reservations: 2 levels. */ #ifndef VM_NRESERVLEVEL -#define VM_NRESERVLEVEL 1 +#define VM_NRESERVLEVEL 2 #endif /* - * Level 0 reservations consist of 512 pages when PAGE_SIZE is 4KB, and - * 2048 pages when PAGE_SIZE is 16KB. + * Level 0 reservations consist of 16 pages when PAGE_SIZE is 4KB, and 128 + * pages when PAGE_SIZE is 16KB. Level 1 reservations consist of 32 64KB + * pages when PAGE_SIZE is 4KB, and 16 2M pages when PAGE_SIZE is 16KB. */ -#ifndef VM_LEVEL_0_ORDER #if PAGE_SIZE == PAGE_SIZE_4K -#define VM_LEVEL_0_ORDER 9 +#ifndef VM_LEVEL_0_ORDER +#define VM_LEVEL_0_ORDER 4 +#endif +#ifndef VM_LEVEL_1_ORDER +#define VM_LEVEL_1_ORDER 5 +#endif #elif PAGE_SIZE == PAGE_SIZE_16K -#define VM_LEVEL_0_ORDER 11 +#ifndef VM_LEVEL_0_ORDER +#define VM_LEVEL_0_ORDER 7 +#endif +#ifndef VM_LEVEL_1_ORDER +#define VM_LEVEL_1_ORDER 4 +#endif #else #error Unsupported page size #endif -#endif /** * Address space layout. diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -1360,8 +1360,12 @@ if ((map->flags & MAP_ASLR) != 0) { maxv1 = maxv / 2 + addr / 2; error = __CONCAT(rnd_, __elfN(base))(map, addr, maxv1, - (MAXPAGESIZES > 1 && pagesizes[1] != 0) ? - pagesizes[1] : pagesizes[0], &anon_loc); +#if VM_NRESERVLEVEL > 0 + pagesizes[VM_NRESERVLEVEL] != 0 ? + /* Align anon_loc to the largest superpage size. */ + pagesizes[VM_NRESERVLEVEL] : +#endif + pagesizes[0], &anon_loc); if (error != 0) goto ret; map->anon_loc = anon_loc; diff --git a/sys/kern/kern_mib.c b/sys/kern/kern_mib.c --- a/sys/kern/kern_mib.c +++ b/sys/kern/kern_mib.c @@ -58,6 +58,8 @@ #include #include +#include + SYSCTL_ROOT_NODE(0, sysctl, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "Sysctl internal magic"); SYSCTL_ROOT_NODE(CTL_KERN, kern, CTLFLAG_RW | CTLFLAG_CAPRD | CTLFLAG_MPSAFE, 0, @@ -242,7 +244,11 @@ SYSCTL_LONG(_hw, OID_AUTO, availpages, CTLFLAG_RD, &physmem, 0, "Amount of physical memory (in pages)"); -u_long pagesizes[MAXPAGESIZES] = { PAGE_SIZE }; +#if VM_NRESERVLEVEL > 0 +_Static_assert(MAXPAGESIZES > VM_NRESERVLEVEL, "MAXPAGESIZES is too small"); +#endif + +u_long __read_mostly pagesizes[MAXPAGESIZES] = { PAGE_SIZE }; static int sysctl_hw_pagesizes(SYSCTL_HANDLER_ARGS) diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -2542,6 +2542,7 @@ vm_offset_t addr; vm_paddr_t pa; vm_pindex_t pi, pi_adv, pindex; + int incore; *super = false; *resident_count = 0; @@ -2577,10 +2578,15 @@ } m_adv = NULL; if (m->psind != 0 && addr + pagesizes[1] <= entry->end && - (addr & (pagesizes[1] - 1)) == 0 && - (pmap_mincore(map->pmap, addr, &pa) & MINCORE_SUPER) != 0) { + (addr & (pagesizes[1] - 1)) == 0 && (incore = + pmap_mincore(map->pmap, addr, &pa) & MINCORE_SUPER) != 0) { *super = true; - pi_adv = atop(pagesizes[1]); + /* + * The virtual page might be smaller than the physical + * page, so we use the page size reported by the pmap + * rather than m->psind. + */ + pi_adv = atop(pagesizes[incore >> MINCORE_PSIND_SHIFT]); } else { /* * We do not test the found page on validity. diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c --- a/sys/kern/uipc_shm.c +++ b/sys/kern/uipc_shm.c @@ -1589,9 +1589,20 @@ if (align == 0) { align = pagesizes[shmfd->shm_lp_psind]; } else if (align == MAP_ALIGNED_SUPER) { - if (shmfd->shm_lp_psind != 1) + /* + * MAP_ALIGNED_SUPER is only supported on superpage sizes, + * i.e., [1, VM_NRESERVLEVEL]. shmfd->shm_lp_psind < 1 is + * handled above. + */ + if ( +#if VM_NRESERVLEVEL > 0 + shmfd->shm_lp_psind > VM_NRESERVLEVEL +#else + shmfd->shm_lp_psind > 1 +#endif + ) return (EINVAL); - align = pagesizes[1]; + align = pagesizes[shmfd->shm_lp_psind]; } else { align >>= MAP_ALIGNMENT_SHIFT; align = 1ULL << align; diff --git a/sys/sys/mman.h b/sys/sys/mman.h --- a/sys/sys/mman.h +++ b/sys/sys/mman.h @@ -175,7 +175,9 @@ #define MINCORE_REFERENCED_OTHER 0x8 /* Page has been referenced */ #define MINCORE_MODIFIED_OTHER 0x10 /* Page has been modified */ #define MINCORE_SUPER 0x60 /* Page is a "super" page */ -#define MINCORE_PSIND(i) (((i) << 5) & MINCORE_SUPER) /* Page size */ +#define MINCORE_PSIND_SHIFT 5 +#define MINCORE_PSIND(i) (((i) << MINCORE_PSIND_SHIFT) & MINCORE_SUPER) + /* Page size */ /* * Anonymous object constant for shm_open(). diff --git a/sys/vm/vm_domainset.c b/sys/vm/vm_domainset.c --- a/sys/vm/vm_domainset.c +++ b/sys/vm/vm_domainset.c @@ -77,6 +77,9 @@ * reservation boundary. */ pindex += obj->pg_color; +#if VM_NRESERVLEVEL > 1 + pindex >>= VM_LEVEL_1_ORDER; +#endif pindex >>= VM_LEVEL_0_ORDER; } else #endif diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -380,12 +380,10 @@ psind = 0; #if VM_NRESERVLEVEL > 0 if ((m->flags & PG_FICTITIOUS) == 0 && - (m_super = vm_reserv_to_superpage(m)) != NULL && - rounddown2(vaddr, pagesizes[m_super->psind]) >= fs->entry->start && - roundup2(vaddr + 1, pagesizes[m_super->psind]) <= fs->entry->end && - (vaddr & (pagesizes[m_super->psind] - 1)) == (VM_PAGE_TO_PHYS(m) & - (pagesizes[m_super->psind] - 1)) && - pmap_ps_enabled(fs->map->pmap)) { + (m_super = vm_reserv_to_superpage(m)) != NULL) { + psind = m_super->psind; + KASSERT(psind > 0, + ("psind %d of m_super %p < 1", psind, m_super)); flags = PS_ALL_VALID; if ((fs->prot & VM_PROT_WRITE) != 0) { /* @@ -398,9 +396,23 @@ if ((fs->first_object->flags & OBJ_UNMANAGED) == 0) flags |= PS_ALL_DIRTY; } - if (vm_page_ps_test(m_super, flags, m)) { + while (rounddown2(vaddr, pagesizes[psind]) < fs->entry->start || + roundup2(vaddr + 1, pagesizes[psind]) > fs->entry->end || + (vaddr & (pagesizes[psind] - 1)) != + (VM_PAGE_TO_PHYS(m) & (pagesizes[psind] - 1)) || + !vm_page_ps_test(m_super, psind, flags, m) || + !pmap_ps_enabled(fs->map->pmap)) { + psind--; + if (psind == 0) + break; + m_super += rounddown2(m - m_super, + atop(pagesizes[psind])); + KASSERT(m_super->psind >= psind, + ("psind %d of m_super %p < %d", m_super->psind, + m_super, psind)); + } + if (psind > 0) { m_map = m_super; - psind = m_super->psind; vaddr = rounddown2(vaddr, pagesizes[psind]); /* Preset the modified bit for dirty superpages. */ if ((flags & PS_ALL_DIRTY) != 0) @@ -615,10 +627,10 @@ vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset; psind = m->psind; - if (psind > 0 && ((vaddr & (pagesizes[psind] - 1)) != 0 || + while (psind > 0 && ((vaddr & (pagesizes[psind] - 1)) != 0 || pidx + OFF_TO_IDX(pagesizes[psind]) - 1 > pager_last || !pmap_ps_enabled(fs->map->pmap))) - psind = 0; + psind--; npages = atop(pagesizes[psind]); for (i = 0; i < npages; i++) { diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c --- a/sys/vm/vm_glue.c +++ b/sys/vm/vm_glue.c @@ -106,7 +106,10 @@ #include -#if VM_NRESERVLEVEL > 0 +#if VM_NRESERVLEVEL > 1 +#define KVA_KSTACK_QUANTUM_SHIFT (VM_LEVEL_1_ORDER + VM_LEVEL_0_ORDER + \ + PAGE_SHIFT) +#elif VM_NRESERVLEVEL > 0 #define KVA_KSTACK_QUANTUM_SHIFT (VM_LEVEL_0_ORDER + PAGE_SHIFT) #else #define KVA_KSTACK_QUANTUM_SHIFT (8 + PAGE_SHIFT) diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -120,7 +120,10 @@ #endif "Max kernel address"); -#if VM_NRESERVLEVEL > 0 +#if VM_NRESERVLEVEL > 1 +#define KVA_QUANTUM_SHIFT (VM_LEVEL_1_ORDER + VM_LEVEL_0_ORDER + \ + PAGE_SHIFT) +#elif VM_NRESERVLEVEL > 0 #define KVA_QUANTUM_SHIFT (VM_LEVEL_0_ORDER + PAGE_SHIFT) #else /* On non-superpage architectures we want large import sizes. */ diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -1993,8 +1993,15 @@ return (result); } +#if VM_NRESERVLEVEL == 1 static const int aslr_pages_rnd_64[2] = {0x1000, 0x10}; static const int aslr_pages_rnd_32[2] = {0x100, 0x4}; +#elif VM_NRESERVLEVEL == 2 +static const int aslr_pages_rnd_64[3] = {0x1000, 0x1000, 0x10}; +static const int aslr_pages_rnd_32[3] = {0x100, 0x100, 0x4}; +#else +#error "Unsupported VM_NRESERVLEVEL" +#endif static int cluster_anon = 1; SYSCTL_INT(_vm, OID_AUTO, cluster_anon, CTLFLAG_RW, @@ -2190,9 +2197,23 @@ * Find space for allocation, including * gap needed for later randomization. */ - pidx = MAXPAGESIZES > 1 && pagesizes[1] != 0 && - (find_space == VMFS_SUPER_SPACE || find_space == - VMFS_OPTIMAL_SPACE) ? 1 : 0; + pidx = 0; +#if VM_NRESERVLEVEL > 0 + if ((find_space == VMFS_SUPER_SPACE || + find_space == VMFS_OPTIMAL_SPACE) && + pagesizes[VM_NRESERVLEVEL] != 0) { + /* + * Do not pointlessly increase the space that + * is requested from vm_map_findspace(). + * pmap_align_superpage() will only change a + * mapping's alignment if that mapping is at + * least a superpage in size. + */ + pidx = VM_NRESERVLEVEL; + while (pidx > 0 && length < pagesizes[pidx]) + pidx--; + } +#endif gap = vm_map_max(map) > MAP_32BIT_MAX_ADDR && (max_addr == 0 || max_addr > MAP_32BIT_MAX_ADDR) ? aslr_pages_rnd_64[pidx] : aslr_pages_rnd_32[pidx]; @@ -2656,6 +2677,7 @@ vm_offset_t start; vm_page_t p, p_start; vm_pindex_t mask, psize, threshold, tmpidx; + int psind; if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL) return; @@ -2710,13 +2732,17 @@ p_start = p; } /* Jump ahead if a superpage mapping is possible. */ - if (p->psind > 0 && ((addr + ptoa(tmpidx)) & - (pagesizes[p->psind] - 1)) == 0) { - mask = atop(pagesizes[p->psind]) - 1; - if (tmpidx + mask < psize && - vm_page_ps_test(p, PS_ALL_VALID, NULL)) { - p += mask; - threshold += mask; + for (psind = p->psind; psind > 0; psind--) { + if (((addr + ptoa(tmpidx)) & + (pagesizes[psind] - 1)) == 0) { + mask = atop(pagesizes[psind]) - 1; + if (tmpidx + mask < psize && + vm_page_ps_test(p, psind, + PS_ALL_VALID, NULL)) { + p += mask; + threshold += mask; + break; + } } } } else if (p_start != NULL) { diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -657,7 +657,7 @@ bool vm_page_pqstate_commit(vm_page_t m, vm_page_astate_t *old, vm_page_astate_t new); vm_page_t vm_page_prev(vm_page_t m); -bool vm_page_ps_test(vm_page_t m, int flags, vm_page_t skip_m); +bool vm_page_ps_test(vm_page_t m, int psind, int flags, vm_page_t skip_m); void vm_page_putfake(vm_page_t m); void vm_page_readahead_finish(vm_page_t m); int vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -5562,7 +5562,7 @@ * (super)page and false otherwise. */ bool -vm_page_ps_test(vm_page_t m, int flags, vm_page_t skip_m) +vm_page_ps_test(vm_page_t m, int psind, int flags, vm_page_t skip_m) { vm_object_t object; int i, npages; @@ -5571,7 +5571,9 @@ if (skip_m != NULL && skip_m->object != object) return (false); VM_OBJECT_ASSERT_LOCKED(object); - npages = atop(pagesizes[m->psind]); + KASSERT(psind <= m->psind, + ("psind %d > psind %d of m %p", psind, m->psind, m)); + npages = atop(pagesizes[psind]); /* * The physically contiguous pages that make up a superpage, i.e., a diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c --- a/sys/vm/vm_reserv.c +++ b/sys/vm/vm_reserv.c @@ -77,6 +77,29 @@ #if VM_NRESERVLEVEL > 0 +/* + * Temporarily simulate two-level reservations. Effectively, VM_LEVEL_0_* is + * level 1, and VM_SUBLEVEL_0_* is level 0. + */ +#if VM_NRESERVLEVEL == 2 +#undef VM_NRESERVLEVEL +#define VM_NRESERVLEVEL 1 +#if VM_LEVEL_0_ORDER == 4 +#undef VM_LEVEL_0_ORDER +#define VM_LEVEL_0_ORDER (4 + VM_LEVEL_1_ORDER) +#define VM_SUBLEVEL_0_NPAGES (1 << 4) +#elif VM_LEVEL_0_ORDER == 7 +#undef VM_LEVEL_0_ORDER +#define VM_LEVEL_0_ORDER (7 + VM_LEVEL_1_ORDER) +#define VM_SUBLEVEL_0_NPAGES (1 << 7) +#else +#error "Unsupported level 0 reservation size" +#endif +#define VM_LEVEL_0_PSIND 2 +#else +#define VM_LEVEL_0_PSIND 1 +#endif + #ifndef VM_LEVEL_0_ORDER_MAX #define VM_LEVEL_0_ORDER_MAX VM_LEVEL_0_ORDER #endif @@ -381,6 +404,27 @@ vm_reserv_object_unlock(object); } +#ifdef VM_SUBLEVEL_0_NPAGES +static inline bool +vm_reserv_is_sublevel_full(vm_reserv_t rv, int index) +{ + _Static_assert(VM_SUBLEVEL_0_NPAGES == 16 || + VM_SUBLEVEL_0_NPAGES == 128, + "vm_reserv_is_sublevel_full: unsupported VM_SUBLEVEL_0_NPAGES"); + /* An equivalent bit_ntest() compiles to more instructions. */ + switch (VM_SUBLEVEL_0_NPAGES) { + case 16: + return (((uint16_t *)rv->popmap)[index / 16] == UINT16_MAX); + case 128: + index = rounddown2(index, 128) / 64; + return (((uint64_t *)rv->popmap)[index] == UINT64_MAX && + ((uint64_t *)rv->popmap)[index + 1] == UINT64_MAX); + default: + __unreachable(); + } +} +#endif + /* * Reduces the given reservation's population count. If the population count * becomes zero, the reservation is destroyed. Additionally, moves the @@ -406,11 +450,15 @@ ("vm_reserv_depopulate: reserv %p's domain is corrupted %d", rv, rv->domain)); if (rv->popcnt == VM_LEVEL_0_NPAGES) { - KASSERT(rv->pages->psind == 1, + KASSERT(rv->pages->psind == VM_LEVEL_0_PSIND, ("vm_reserv_depopulate: reserv %p is already demoted", rv)); - rv->pages->psind = 0; + rv->pages->psind = VM_LEVEL_0_PSIND - 1; } +#ifdef VM_SUBLEVEL_0_NPAGES + if (vm_reserv_is_sublevel_full(rv, index)) + rv->pages[rounddown2(index, VM_SUBLEVEL_0_NPAGES)].psind = 0; +#endif bit_clear(rv->popmap, index); rv->popcnt--; if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP || @@ -522,12 +570,17 @@ index)); KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, ("vm_reserv_populate: reserv %p is already full", rv)); - KASSERT(rv->pages->psind == 0, + KASSERT(rv->pages->psind >= 0 && + rv->pages->psind < VM_LEVEL_0_PSIND, ("vm_reserv_populate: reserv %p is already promoted", rv)); KASSERT(rv->domain < vm_ndomains, ("vm_reserv_populate: reserv %p's domain is corrupted %d", rv, rv->domain)); bit_set(rv->popmap, index); +#ifdef VM_SUBLEVEL_0_NPAGES + if (vm_reserv_is_sublevel_full(rv, index)) + rv->pages[rounddown2(index, VM_SUBLEVEL_0_NPAGES)].psind = 1; +#endif rv->popcnt++; if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP && rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES) @@ -542,10 +595,10 @@ rv->inpartpopq = TRUE; TAILQ_INSERT_TAIL(&vm_rvd[rv->domain].partpop, rv, partpopq); } else { - KASSERT(rv->pages->psind == 0, + KASSERT(rv->pages->psind == VM_LEVEL_0_PSIND - 1, ("vm_reserv_populate: reserv %p is already promoted", rv)); - rv->pages->psind = 1; + rv->pages->psind = VM_LEVEL_0_PSIND; } vm_reserv_domain_unlock(rv->domain); } @@ -889,13 +942,18 @@ static void vm_reserv_break(vm_reserv_t rv) { + vm_page_t m; int hi, lo, pos; vm_reserv_assert_locked(rv); CTR5(KTR_VM, "%s: rv %p object %p popcnt %d inpartpop %d", __FUNCTION__, rv, rv->object, rv->popcnt, rv->inpartpopq); vm_reserv_remove(rv); - rv->pages->psind = 0; + m = rv->pages; +#ifdef VM_SUBLEVEL_0_NPAGES + for (; m < rv->pages + VM_LEVEL_0_NPAGES; m += VM_SUBLEVEL_0_NPAGES) +#endif + m->psind = 0; hi = lo = -1; pos = 0; for (;;) { @@ -1089,7 +1147,11 @@ vm_reserv_t rv; rv = vm_reserv_from_page(m); +#ifdef VM_SUBLEVEL_0_NPAGES + return (rv->object != NULL ? 1 : -1); +#else return (rv->object != NULL ? 0 : -1); +#endif } /* @@ -1102,7 +1164,15 @@ vm_reserv_t rv; rv = vm_reserv_from_page(m); - return (rv->popcnt == VM_LEVEL_0_NPAGES ? 0 : -1); + if (rv->popcnt == VM_LEVEL_0_NPAGES) { +#ifdef VM_SUBLEVEL_0_NPAGES + return (1); + } else if (rv->pages != NULL && + vm_reserv_is_sublevel_full(rv, m - rv->pages)) { +#endif + return (0); + } + return (-1); } /* @@ -1357,6 +1427,10 @@ switch (level) { case 0: +#ifdef VM_SUBLEVEL_0_NPAGES + return (VM_SUBLEVEL_0_NPAGES * PAGE_SIZE); + case 1: +#endif return (VM_LEVEL_0_SIZE); case -1: return (PAGE_SIZE); @@ -1432,12 +1506,16 @@ VM_OBJECT_ASSERT_LOCKED(m->object); rv = vm_reserv_from_page(m); - if (rv->object == m->object && rv->popcnt == VM_LEVEL_0_NPAGES) - m = rv->pages; - else - m = NULL; - - return (m); + if (rv->object == m->object) { + if (rv->popcnt == VM_LEVEL_0_NPAGES) + return (rv->pages); +#ifdef VM_SUBLEVEL_0_NPAGES + if (vm_reserv_is_sublevel_full(rv, m - rv->pages)) + return (rv->pages + rounddown2(m - rv->pages, + VM_SUBLEVEL_0_NPAGES)); +#endif + } + return (NULL); } #endif /* VM_NRESERVLEVEL > 0 */