Differential D26976 Diff 79613 sys/arm64/arm64/pmap.c

Changeset View

Standalone View

sys/arm64/arm64/pmap.c

Show First 20 Lines • Show All 401 Lines • ▼ Show 20 Lines
static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex,		static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex,
struct rwlock **lockp);		struct rwlock **lockp);

static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m,		static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m,
struct spglist *free);		struct spglist *free);
static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *);		static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *);
static __inline vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);		static __inline vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);

		static uint64_t pa_range_bits = 0;

/*		/*
* These load the old table data and store the new value.		* These load the old table data and store the new value.
* They need to be atomic as the System MMU may write to the table at		* They need to be atomic as the System MMU may write to the table at
* the same time as the CPU.		* the same time as the CPU.
*/		*/
#define pmap_clear(table) atomic_store_64(table, 0)		#define pmap_clear(table) atomic_store_64(table, 0)
#define pmap_clear_bits(table, bits) atomic_clear_64(table, bits)		#define pmap_clear_bits(table, bits) atomic_clear_64(table, bits)
#define pmap_load(table) (*table)		#define pmap_load(table) (*table)
#define pmap_load_clear(table) atomic_swap_64(table, 0)		#define pmap_load_clear(table) atomic_swap_64(table, 0)
#define pmap_load_store(table, entry) atomic_swap_64(table, entry)		#define pmap_load_store(table, entry) atomic_swap_64(table, entry)
#define pmap_set_bits(table, bits) atomic_set_64(table, bits)		#define pmap_set_bits(table, bits) atomic_set_64(table, bits)
#define pmap_store(table, entry) atomic_store_64(table, entry)		#define pmap_store(table, entry) atomic_store_64(table, entry)

/********************/		/********************/
/* Inline functions */		/* Inline functions */
/********************/		/********************/

static __inline void		static __inline void
pagecopy(void s, void d)		pagecopy(void s, void d)
{		{

memcpy(d, s, PAGE_SIZE);		memcpy(d, s, PAGE_SIZE);
}		}

		#define pmap_l0_index(va) (((va) >> L0_SHIFT) & L0_ADDR_MASK)
		#define pmap_l1_index(va) (((va) >> L1_SHIFT) & Ln_ADDR_MASK)
		#define pmap_l2_index(va) (((va) >> L2_SHIFT) & Ln_ADDR_MASK)
		#define pmap_l3_index(va) (((va) >> L3_SHIFT) & Ln_ADDR_MASK)

		#define STAGE2_L1_ADDR_MASK ((1UL << (pa_range_bits - L1_SHIFT)) - 1)
		#define pmap_stage2_l1_index(va) (((va) >> L1_SHIFT) & STAGE2_L1_ADDR_MASK)

static __inline pd_entry_t *		static __inline pd_entry_t *
pmap_l0(pmap_t pmap, vm_offset_t va)		pmap_l0(pmap_t pmap, vm_offset_t va)
{		{
		KASSERT(pmap->pm_stage != PM_STAGE2,
		("Level 0 table is invalid for PM_STAGE2 pmap"));

return (&pmap->pm_l0[pmap_l0_index(va)]);		return (&pmap->pm_l0[pmap_l0_index(va)]);
}		}

static __inline pd_entry_t *		static __inline pd_entry_t *
pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va)		pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va)
{		{
pd_entry_t *l1;		pd_entry_t *l1;

l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK);		l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK);
return (&l1[pmap_l1_index(va)]);		return (&l1[pmap_l1_index(va)]);
}		}

static __inline pd_entry_t *		static __inline pd_entry_t *
pmap_l1(pmap_t pmap, vm_offset_t va)		pmap_l1(pmap_t pmap, vm_offset_t va)
{		{
		if (pmap->pm_stage == PM_STAGE2)
		return (&pmap->pm_l0[pmap_stage2_l1_index(va)]);

pd_entry_t *l0;		pd_entry_t *l0;

l0 = pmap_l0(pmap, va);		l0 = pmap_l0(pmap, va);
if ((pmap_load(l0) & ATTR_DESCR_MASK) != L0_TABLE)		if ((pmap_load(l0) & ATTR_DESCR_MASK) != L0_TABLE)
return (NULL);		return (NULL);

return (pmap_l0_to_l1(l0, va));		return (pmap_l0_to_l1(l0, va));
}		}

		static __inline vm_page_t
		pmap_l1pg(pmap_t pmap, vm_offset_t va)
		{
		if (pmap->pm_stage == PM_STAGE1) {
		pd_entry_t *l0, tl0;

		l0 = pmap_l0(pmap, va);
		tl0 = pmap_load(l0);

		return (PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK));
		} else {
		vm_paddr_t pa, pa_offset;

		/*
		* The offset will be the bits
		* [pa_range_bits-1:L0_SHIFT]
		*/
		va = va & ((1 << pa_range_bits) - 1);
		pa_offset = va >> L0_SHIFT;
		pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_l0) + \
		(pa_offset << PAGE_SHIFT);

		return (PHYS_TO_VM_PAGE(pa));
		}
		}

static __inline pd_entry_t *		static __inline pd_entry_t *
pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va)		pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va)
{		{
pd_entry_t l1, *l2p;		pd_entry_t l1, *l2p;

l1 = pmap_load(l1p);		l1 = pmap_load(l1p);

/*		/*
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines
* Returns the lowest valid pde for a given virtual address.		* Returns the lowest valid pde for a given virtual address.
* The next level may or may not point to a valid page or block.		* The next level may or may not point to a valid page or block.
*/		*/
static __inline pd_entry_t *		static __inline pd_entry_t *
pmap_pde(pmap_t pmap, vm_offset_t va, int *level)		pmap_pde(pmap_t pmap, vm_offset_t va, int *level)
{		{
pd_entry_t l0, l1, *l2, desc;		pd_entry_t l0, l1, *l2, desc;

		if (pmap->pm_stage == PM_STAGE1) {
l0 = pmap_l0(pmap, va);		l0 = pmap_l0(pmap, va);
desc = pmap_load(l0) & ATTR_DESCR_MASK;		desc = pmap_load(l0) & ATTR_DESCR_MASK;
if (desc != L0_TABLE) {		if (desc != L0_TABLE) {
*level = -1;		*level = -1;
return (NULL);		return (NULL);
}		}

l1 = pmap_l0_to_l1(l0, va);		l1 = pmap_l0_to_l1(l0, va);
desc = pmap_load(l1) & ATTR_DESCR_MASK;		desc = pmap_load(l1) & ATTR_DESCR_MASK;
if (desc != L1_TABLE) {		if (desc != L1_TABLE) {
*level = 0;		*level = 0;
return (l0);		return (l0);
}		}
		} else {
		l1 = pmap_l1(pmap, va);
		desc = pmap_load(l1) & ATTR_DESCR_MASK;
		if (desc != L1_TABLE) {
		/* For PM_STAGE2 mappings the first level is level 1 */
		*level = -1;
		return (NULL);
		}
		}

l2 = pmap_l1_to_l2(l1, va);		l2 = pmap_l1_to_l2(l1, va);
desc = pmap_load(l2) & ATTR_DESCR_MASK;		desc = pmap_load(l2) & ATTR_DESCR_MASK;
if (desc != L2_TABLE) {		if (desc != L2_TABLE) {
*level = 1;		*level = 1;
return (l1);		return (l1);
}		}

▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines
pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t l0, pd_entry_t l1,		pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t l0, pd_entry_t l1,
pd_entry_t l2, pt_entry_t l3)		pd_entry_t l2, pt_entry_t l3)
{		{
pd_entry_t l0p, l1p, *l2p;		pd_entry_t l0p, l1p, *l2p;

if (pmap->pm_l0 == NULL)		if (pmap->pm_l0 == NULL)
return (false);		return (false);

		if (pmap->pm_stage == PM_STAGE1) {
l0p = pmap_l0(pmap, va);		l0p = pmap_l0(pmap, va);
*l0 = l0p;		*l0 = l0p;

if ((pmap_load(l0p) & ATTR_DESCR_MASK) != L0_TABLE)		if ((pmap_load(l0p) & ATTR_DESCR_MASK) != L0_TABLE)
return (false);		return (false);

l1p = pmap_l0_to_l1(l0p, va);		l1p = pmap_l0_to_l1(l0p, va);
		} else {
		*l0 = NULL;
		l1p = pmap_l1(pmap, va);
		}
*l1 = l1p;		*l1 = l1p;

if ((pmap_load(l1p) & ATTR_DESCR_MASK) == L1_BLOCK) {		if ((pmap_load(l1p) & ATTR_DESCR_MASK) == L1_BLOCK) {
*l2 = NULL;		*l2 = NULL;
*l3 = NULL;		*l3 = NULL;
return (true);		return (true);
}		}

▲ Show 20 Lines • Show All 318 Lines • ▼ Show 20 Lines

/*		/*
* Bootstrap the system enough to run with virtual memory.		* Bootstrap the system enough to run with virtual memory.
*/		*/
void		void
pmap_bootstrap(vm_offset_t l0pt, vm_offset_t l1pt, vm_paddr_t kernstart,		pmap_bootstrap(vm_offset_t l0pt, vm_offset_t l1pt, vm_paddr_t kernstart,
vm_size_t kernlen)		vm_size_t kernlen)
{		{
		uint64_t id_aa64mmfr0_el1;
vm_offset_t freemempos;		vm_offset_t freemempos;
vm_offset_t dpcpu, msgbufpv;		vm_offset_t dpcpu, msgbufpv;
vm_paddr_t start_pa, pa, min_pa;		vm_paddr_t start_pa, pa, min_pa;
uint64_t kern_delta;		uint64_t kern_delta;
int i;		int i;

/* Verify that the ASID is set through TTBR0. */		/* Verify that the ASID is set through TTBR0. */
KASSERT((READ_SPECIALREG(tcr_el1) & TCR_A1) == 0,		KASSERT((READ_SPECIALREG(tcr_el1) & TCR_A1) == 0,
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines	#define alloc_pages(var, np) \
virtual_avail = roundup2(virtual_avail, L1_SIZE);		virtual_avail = roundup2(virtual_avail, L1_SIZE);
virtual_end = VM_MAX_KERNEL_ADDRESS - (PMAP_MAPDEV_EARLY_SIZE);		virtual_end = VM_MAX_KERNEL_ADDRESS - (PMAP_MAPDEV_EARLY_SIZE);
kernel_vm_end = virtual_avail;		kernel_vm_end = virtual_avail;

pa = pmap_early_vtophys(l1pt, freemempos);		pa = pmap_early_vtophys(l1pt, freemempos);

physmem_exclude_region(start_pa, pa - start_pa, EXFLAG_NOALLOC);		physmem_exclude_region(start_pa, pa - start_pa, EXFLAG_NOALLOC);

		id_aa64mmfr0_el1 = READ_SPECIALREG(id_aa64mmfr0_el1);
		switch (ID_AA64MMFR0_PARange_VAL(id_aa64mmfr0_el1)) {
		case ID_AA64MMFR0_PARange_4G:
		pa_range_bits = 32;
		break;
		case ID_AA64MMFR0_PARange_64G:
		pa_range_bits = 36;
		break;
		case ID_AA64MMFR0_PARange_1T:
		pa_range_bits = 40;
		break;
		case ID_AA64MMFR0_PARange_4T:
		pa_range_bits = 42;
		break;
		case ID_AA64MMFR0_PARange_16T:
		pa_range_bits = 44;
		break;
		case ID_AA64MMFR0_PARange_256T:
		pa_range_bits = 48;
		break;
		default:
		/*
		* Unknown PA range bits, will lead to a panic if a stage 2
		* pmap starting at level 1 is created.
		*/
		pa_range_bits = 0;
		break;
		}

cpu_tlb_flushID();		cpu_tlb_flushID();
}		}

/*		/*
* Initialize a vm_page's machine-dependent fields.		* Initialize a vm_page's machine-dependent fields.
*/		*/
void		void
pmap_page_init(vm_page_t m)		pmap_page_init(vm_page_t m)
▲ Show 20 Lines • Show All 567 Lines • ▼ Show 20 Lines
{		{

PMAP_LOCK_ASSERT(pmap, MA_OWNED);		PMAP_LOCK_ASSERT(pmap, MA_OWNED);
/*		/*
* unmap the page table page		* unmap the page table page
*/		*/
if (m->pindex >= (NUL2E + NUL1E)) {		if (m->pindex >= (NUL2E + NUL1E)) {
/* l1 page */		/* l1 page */
		if (pmap->pm_stage == PM_STAGE1) {
pd_entry_t *l0;		pd_entry_t *l0;

l0 = pmap_l0(pmap, va);		l0 = pmap_l0(pmap, va);
pmap_clear(l0);		pmap_clear(l0);
		}
} else if (m->pindex >= NUL2E) {		} else if (m->pindex >= NUL2E) {
/* l2 page */		/* l2 page */
pd_entry_t *l1;		pd_entry_t *l1;

l1 = pmap_l1(pmap, va);		l1 = pmap_l1(pmap, va);
pmap_clear(l1);		pmap_clear(l1);
} else {		} else {
/* l3 page */		/* l3 page */
Show All 9 Lines	if (m->pindex < NUL2E) {
vm_page_t l2pg;		vm_page_t l2pg;

l1 = pmap_l1(pmap, va);		l1 = pmap_l1(pmap, va);
tl1 = pmap_load(l1);		tl1 = pmap_load(l1);
l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);		l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
pmap_unwire_l3(pmap, va, l2pg, free);		pmap_unwire_l3(pmap, va, l2pg, free);
} else if (m->pindex < (NUL2E + NUL1E)) {		} else if (m->pindex < (NUL2E + NUL1E)) {
/* We just released an l2, unhold the matching l1 */		/* We just released an l2, unhold the matching l1 */
pd_entry_t *l0, tl0;
vm_page_t l1pg;		vm_page_t l1pg;
		pd_entry_t *l0, tl0;

		if (pmap->pm_stage == PM_STAGE1) {
l0 = pmap_l0(pmap, va);		l0 = pmap_l0(pmap, va);
tl0 = pmap_load(l0);		tl0 = pmap_load(l0);
l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);		l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);
		} else {
		l1pg = pmap_l1pg(pmap, va);
		}
pmap_unwire_l3(pmap, va, l1pg, free);		pmap_unwire_l3(pmap, va, l1pg, free);
}		}
pmap_invalidate_page(pmap, va);		pmap_invalidate_page(pmap, va);

/*		/*
* Put page on a list so that it is released after		* Put page on a list so that it is released after
* ALL TLB shootdown is done		* ALL TLB shootdown is done
*/		*/
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines	pmap_pinit0(pmap_t pmap)
PCPU_SET(curpmap, pmap);		PCPU_SET(curpmap, pmap);
}		}

int		int
pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage, int levels)		pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage, int levels)
{		{
vm_page_t m;		vm_page_t m;

		KASSERT((stage == PM_STAGE1 \|\| stage == PM_STAGE2),
		("Invalid pmap stage %d", stage));
		KASSERT(!((stage == PM_STAGE2) && (pa_range_bits == 0)),
		("Unknown PARange bits"));

/*		/*
* allocate the l0 page		* allocate the l0 page
*/		*/
		if (stage == PM_STAGE1) {
while ((m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \|		while ((m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \|
VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED \| VM_ALLOC_ZERO)) == NULL)		VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED \| VM_ALLOC_ZERO)) == NULL)
vm_wait(NULL);		vm_wait(NULL);
		} else {
		uint64_t npages;
		uint64_t alignment;

		if (pa_range_bits <= L0_SHIFT) {
		/*
		* The level 1 translation table is not larger than a
		* PM_STAGE1 level 1 table, use only one page.
		*/
		npages = 1;
		alignment = PAGE_SIZE;
		} else {
		/*
		* The level 1 translation table is larger than a
		* regular PM_STAGE1 level 1 table, for every x bits
		* that is larger we need 2^x pages and the table must
		* be aligned at a 2^(x + 12) boundary.
		*
		* See Table D5-25 and Example D4-5 from the DDI0487B
		* ARMv8 Architecture Manual for more information.
		*/
		npages = 1 << (pa_range_bits - L0_SHIFT);
		alignment = 1 << (PAGE_SHIFT + pa_range_bits - L0_SHIFT);
		}
		while ((m = vm_page_alloc_contig(NULL, 0, VM_ALLOC_NORMAL \|
		VM_ALLOC_NOOBJ \| VM_ALLOC_WIRED \| VM_ALLOC_ZERO,
		npages, DMAP_MIN_PHYSADDR, DMAP_MAX_PHYSADDR,
		alignment, 0, VM_MEMATTR_DEFAULT)) == NULL)
		vm_wait(NULL);
		}

pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m);		pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m);
pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr);		pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr);

if ((m->flags & PG_ZERO) == 0)		if ((m->flags & PG_ZERO) == 0)
pagezero(pmap->pm_l0);		pagezero(pmap->pm_l0);

pmap->pm_root.rt_root = 0;		pmap->pm_root.rt_root = 0;
		pmap->pm_stage = stage;
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));		bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
pmap->pm_cookie = COOKIE_FROM(-1, INT_MAX);		pmap->pm_cookie = COOKIE_FROM(-1, INT_MAX);

MPASS(levels == 3 \|\| levels == 4);		MPASS(levels == 3 \|\| levels == 4);
pmap->pm_levels = levels;		pmap->pm_levels = levels;
pmap->pm_stage = stage;		pmap->pm_stage = stage;
switch (stage) {		switch (stage) {
case PM_STAGE1:		case PM_STAGE1:
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines	if (ptepindex >= (NUL2E + NUL1E)) {
l0 = &pmap->pm_l0[l0index];		l0 = &pmap->pm_l0[l0index];
pmap_store(l0, VM_PAGE_TO_PHYS(m) \| L0_TABLE);		pmap_store(l0, VM_PAGE_TO_PHYS(m) \| L0_TABLE);
} else if (ptepindex >= NUL2E) {		} else if (ptepindex >= NUL2E) {
vm_pindex_t l0index, l1index;		vm_pindex_t l0index, l1index;
pd_entry_t l0, l1;		pd_entry_t l0, l1;
pd_entry_t tl0;		pd_entry_t tl0;

l1index = ptepindex - NUL2E;		l1index = ptepindex - NUL2E;
		if (pmap->pm_stage == PM_STAGE1) {
l0index = l1index >> L0_ENTRIES_SHIFT;		l0index = l1index >> L0_ENTRIES_SHIFT;

l0 = &pmap->pm_l0[l0index];		l0 = &pmap->pm_l0[l0index];
tl0 = pmap_load(l0);		tl0 = pmap_load(l0);
if (tl0 == 0) {		if (tl0 == 0) {
/* recurse for allocating page dir */		/* recurse for allocating page dir */
if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index,		if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index,
lockp) == NULL) {		lockp) == NULL) {
vm_page_unwire_noq(m);		vm_page_unwire_noq(m);
vm_page_free_zero(m);		vm_page_free_zero(m);
return (NULL);		return (NULL);
}		}
} else {		} else {
l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);		l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);
l1pg->ref_count++;		l1pg->ref_count++;
}		}

l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK);		l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK);
l1 = &l1[ptepindex & Ln_ADDR_MASK];		l1 = &l1[ptepindex & Ln_ADDR_MASK];
		} else {
		l1pg = pmap_l1pg(pmap, l1index);
		l1pg->ref_count++;
		l1 = &pmap->pm_l0[l1index & STAGE2_L1_ADDR_MASK];
		}
pmap_store(l1, VM_PAGE_TO_PHYS(m) \| L1_TABLE);		pmap_store(l1, VM_PAGE_TO_PHYS(m) \| L1_TABLE);
} else {		} else {
vm_pindex_t l0index, l1index;		vm_pindex_t l0index, l1index;
pd_entry_t l0, l1, *l2;		pd_entry_t l0, l1, *l2;
pd_entry_t tl0, tl1;		pd_entry_t tl0, tl1;

l1index = ptepindex >> Ln_ENTRIES_SHIFT;		l1index = ptepindex >> Ln_ENTRIES_SHIFT;
		if (pmap->pm_stage == PM_STAGE1) {
l0index = l1index >> L0_ENTRIES_SHIFT;		l0index = l1index >> L0_ENTRIES_SHIFT;

l0 = &pmap->pm_l0[l0index];		l0 = &pmap->pm_l0[l0index];
tl0 = pmap_load(l0);		tl0 = pmap_load(l0);
if (tl0 == 0) {		if (tl0 == 0) {
/* recurse for allocating page dir */		/* recurse for allocating page dir */
if (_pmap_alloc_l3(pmap, NUL2E + l1index,		if (_pmap_alloc_l3(pmap, NUL2E + l1index,
lockp) == NULL) {		lockp) == NULL) {
vm_page_unwire_noq(m);		vm_page_unwire_noq(m);
vm_page_free_zero(m);		vm_page_free_zero(m);
return (NULL);		return (NULL);
}		}
tl0 = pmap_load(l0);		tl0 = pmap_load(l0);
l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);		l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
l1 = &l1[l1index & Ln_ADDR_MASK];		l1 = &l1[l1index & Ln_ADDR_MASK];
} else {		} else {
l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);		l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
l1 = &l1[l1index & Ln_ADDR_MASK];		l1 = &l1[l1index & Ln_ADDR_MASK];
tl1 = pmap_load(l1);		tl1 = pmap_load(l1);
if (tl1 == 0) {		if (tl1 == 0) {
/* recurse for allocating page dir */		/* recurse for allocating page dir */
if (_pmap_alloc_l3(pmap, NUL2E + l1index,		if (_pmap_alloc_l3(pmap, NUL2E + l1index,
lockp) == NULL) {		lockp) == NULL) {
vm_page_unwire_noq(m);		vm_page_unwire_noq(m);
vm_page_free_zero(m);		vm_page_free_zero(m);
return (NULL);		return (NULL);
}		}
} else {		} else {
l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);		l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
l2pg->ref_count++;		l2pg->ref_count++;
}		}
}		}
		} else {
		l1 = &pmap->pm_l0[l1index & STAGE2_L1_ADDR_MASK];
		tl1 = pmap_load(l1);
		if (tl1 == 0) {
		/* recurse for allocating page dir */
		if (_pmap_alloc_l3(pmap, NUL2E + l1index,
		lockp) == NULL) {
		vm_page_unwire_noq(m);
		vm_page_free_zero(m);
		return (NULL);
		}
		} else {
		l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
		l2pg->ref_count++;
		}
		}

l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK);		l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK);
l2 = &l2[ptepindex & Ln_ADDR_MASK];		l2 = &l2[ptepindex & Ln_ADDR_MASK];
pmap_store(l2, VM_PAGE_TO_PHYS(m) \| L2_TABLE);		pmap_store(l2, VM_PAGE_TO_PHYS(m) \| L2_TABLE);
}		}

pmap_resident_count_inc(pmap, 1);		pmap_resident_count_inc(pmap, 1);

▲ Show 20 Lines • Show All 159 Lines • ▼ Show 20 Lines	if (COOKIE_TO_EPOCH(pmap->pm_cookie) == set->asid_epoch) {
KASSERT(asid >= ASID_FIRST_AVAILABLE &&		KASSERT(asid >= ASID_FIRST_AVAILABLE &&
asid < set->asid_set_size,		asid < set->asid_set_size,
("pmap_release: pmap cookie has out-of-range asid"));		("pmap_release: pmap cookie has out-of-range asid"));
bit_clear(set->asid_set, asid);		bit_clear(set->asid_set, asid);
}		}
mtx_unlock_spin(&set->asid_set_mutex);		mtx_unlock_spin(&set->asid_set_mutex);
}		}

		if (pmap->pm_stage == PM_STAGE1) {
m = PHYS_TO_VM_PAGE(pmap->pm_l0_paddr);		m = PHYS_TO_VM_PAGE(pmap->pm_l0_paddr);
vm_page_unwire_noq(m);		vm_page_unwire_noq(m);
vm_page_free_zero(m);		vm_page_free_zero(m);
		} else {
		uint64_t i, page_cnt;
		vm_paddr_t pa;

		if (pa_range_bits < L0_SHIFT)
		page_cnt = 1;
		else
		page_cnt = 1 << (pa_range_bits - L0_SHIFT);

		pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_l0);
		for (i = 0; i < page_cnt; i++) {
		m = PHYS_TO_VM_PAGE(pa);
		vm_page_unwire_noq(m);
		vm_page_free_zero(m);
		pa += PAGE_SIZE;
}		}
		}
		}

static int		static int
kvm_size(SYSCTL_HANDLER_ARGS)		kvm_size(SYSCTL_HANDLER_ARGS)
{		{
unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;		unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;

return sysctl_handle_long(oidp, &ksize, 0, req);		return sysctl_handle_long(oidp, &ksize, 0, req);
}		}
▲ Show 20 Lines • Show All 898 Lines • ▼ Show 20 Lines	pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)

PMAP_LOCK(pmap);		PMAP_LOCK(pmap);

lock = NULL;		lock = NULL;
for (; sva < eva; sva = va_next) {		for (; sva < eva; sva = va_next) {
if (pmap->pm_stats.resident_count == 0)		if (pmap->pm_stats.resident_count == 0)
break;		break;

		if (pmap->pm_stage == PM_STAGE1) {
l0 = pmap_l0(pmap, sva);		l0 = pmap_l0(pmap, sva);
if (pmap_load(l0) == 0) {		if (pmap_load(l0) == 0) {
va_next = (sva + L0_SIZE) & ~L0_OFFSET;		va_next = (sva + L0_SIZE) & ~L0_OFFSET;
if (va_next < sva)		if (va_next < sva)
va_next = eva;		va_next = eva;
continue;		continue;
}		}
		}

va_next = (sva + L1_SIZE) & ~L1_OFFSET;		va_next = (sva + L1_SIZE) & ~L1_OFFSET;
if (va_next < sva)		if (va_next < sva)
va_next = eva;		va_next = eva;
l1 = pmap_l0_to_l1(l0, sva);		l1 = pmap_l0_to_l1(l0, sva);
if (pmap_load(l1) == 0)		if (pmap_load(l1) == 0)
continue;		continue;
if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) {		if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) {
▲ Show 20 Lines • Show All 836 Lines • ▼ Show 20 Lines	if ((flags & PMAP_ENTER_WIRED) != 0)
new_l3 \|= ATTR_SW_WIRED;		new_l3 \|= ATTR_SW_WIRED;
if (pmap->pm_stage == PM_STAGE1) {		if (pmap->pm_stage == PM_STAGE1) {
if (va < VM_MAXUSER_ADDRESS)		if (va < VM_MAXUSER_ADDRESS)
new_l3 \|= ATTR_S1_AP(ATTR_S1_AP_USER) \| ATTR_S1_PXN;		new_l3 \|= ATTR_S1_AP(ATTR_S1_AP_USER) \| ATTR_S1_PXN;
else		else
new_l3 \|= ATTR_S1_UXN;		new_l3 \|= ATTR_S1_UXN;
if (pmap != kernel_pmap)		if (pmap != kernel_pmap)
new_l3 \|= ATTR_S1_nG;		new_l3 \|= ATTR_S1_nG;
} else {
/*
* Clear the access flag on executable mappings, this will be
* set later when the page is accessed. The fault handler is
* required to invalidate the I-cache.
*
* TODO: Switch to the valid flag to allow hardware management
* of the access flag. Much of the pmap code assumes the
* valid flag is set and fails to destroy the old page tables
* correctly if it is clear.
*/
if (prot & VM_PROT_EXECUTE)
new_l3 &= ~ATTR_AF;
}
if ((m->oflags & VPO_UNMANAGED) == 0) {		if ((m->oflags & VPO_UNMANAGED) == 0) {
new_l3 \|= ATTR_SW_MANAGED;		new_l3 \|= ATTR_SW_MANAGED;
if ((prot & VM_PROT_WRITE) != 0) {		if ((prot & VM_PROT_WRITE) != 0) {
new_l3 \|= ATTR_SW_DBM;		new_l3 \|= ATTR_SW_DBM;
if ((flags & VM_PROT_WRITE) == 0) {		if ((flags & VM_PROT_WRITE) == 0)
if (pmap->pm_stage == PM_STAGE1)
new_l3 \|= ATTR_S1_AP(ATTR_S1_AP_RO);		new_l3 \|= ATTR_S1_AP(ATTR_S1_AP_RO);
else
new_l3 &=
~ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE);
}		}
}		}
		} else {
		new_l3 = (pd_entry_t)(pa \| ATTR_ST2_DEFAULT \| L3_PAGE);
}		}
		if ((flags & PMAP_ENTER_WIRED) != 0)
		new_l3 \|= ATTR_SW_WIRED;

CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa);		CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa);

lock = NULL;		lock = NULL;
PMAP_LOCK(pmap);		PMAP_LOCK(pmap);
if ((flags & PMAP_ENTER_LARGEPAGE) != 0) {		if ((flags & PMAP_ENTER_LARGEPAGE) != 0) {
KASSERT((m->oflags & VPO_UNMANAGED) != 0,		KASSERT((m->oflags & VPO_UNMANAGED) != 0,
("managed largepage va %#lx flags %#x", va, flags));		("managed largepage va %#lx flags %#x", va, flags));
Show All 37 Lines	if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK &&
mpte = PHYS_TO_VM_PAGE(		mpte = PHYS_TO_VM_PAGE(
pmap_load(l2) & ~ATTR_MASK);		pmap_load(l2) & ~ATTR_MASK);
mpte->ref_count++;		mpte->ref_count++;
}		}
goto havel3;		goto havel3;
}		}
/* We need to allocate an L3 table. */		/* We need to allocate an L3 table. */
}		}

if (va < VM_MAXUSER_ADDRESS) {		if (va < VM_MAXUSER_ADDRESS) {
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;		nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;

/*		/*
* We use _pmap_alloc_l3() instead of pmap_alloc_l3() in order		* We use _pmap_alloc_l3() instead of pmap_alloc_l3() in order
* to handle the possibility that a superpage mapping for "va"		* to handle the possibility that a superpage mapping for "va"
* was created while we slept.		* was created while we slept.
*/		*/
▲ Show 20 Lines • Show All 3,221 Lines • Show Last 20 Lines