Differential D20812 Diff 59293 head/sys/amd64/amd64/pmap.c

Changeset View

Standalone View

head/sys/amd64/amd64/pmap.c

This file is larger than 256 KB, so syntax highlighting is disabled by default.

Show First 20 Lines • Show All 6,339 Lines • ▼ Show 20 Lines

/*		/*
* Copy the range specified by src_addr/len		* Copy the range specified by src_addr/len
* from the source map to the range dst_addr/len		* from the source map to the range dst_addr/len
* in the destination map.		* in the destination map.
*		*
* This routine is only advisory and need not do anything.		* This routine is only advisory and need not do anything.
*/		*/

void		void
pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,		pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
vm_offset_t src_addr)		vm_offset_t src_addr)
{		{
struct rwlock *lock;		struct rwlock *lock;
struct spglist free;		struct spglist free;
vm_offset_t addr;		pml4_entry_t *pml4e;
vm_offset_t end_addr = src_addr + len;		pdp_entry_t *pdpe;
vm_offset_t va_next;		pd_entry_t *pde, srcptepaddr;
		pt_entry_t dst_pte, PG_A, PG_M, PG_V, ptetemp, src_pte;
		vm_offset_t addr, end_addr, va_next;
vm_page_t dst_pdpg, dstmpte, srcmpte;		vm_page_t dst_pdpg, dstmpte, srcmpte;
pt_entry_t PG_A, PG_M, PG_V;

if (dst_addr != src_addr)		if (dst_addr != src_addr)
return;		return;

if (dst_pmap->pm_type != src_pmap->pm_type)		if (dst_pmap->pm_type != src_pmap->pm_type)
return;		return;

/*		/*
* EPT page table entries that require emulation of A/D bits are		* EPT page table entries that require emulation of A/D bits are
* sensitive to clearing the PG_A bit (aka EPT_PG_READ). Although		* sensitive to clearing the PG_A bit (aka EPT_PG_READ). Although
* we clear PG_M (aka EPT_PG_WRITE) concomitantly, the PG_U bit		* we clear PG_M (aka EPT_PG_WRITE) concomitantly, the PG_U bit
* (aka EPT_PG_EXECUTE) could still be set. Since some EPT		* (aka EPT_PG_EXECUTE) could still be set. Since some EPT
* implementations flag an EPT misconfiguration for exec-only		* implementations flag an EPT misconfiguration for exec-only
* mappings we skip this function entirely for emulated pmaps.		* mappings we skip this function entirely for emulated pmaps.
*/		*/
if (pmap_emulate_ad_bits(dst_pmap))		if (pmap_emulate_ad_bits(dst_pmap))
return;		return;

		end_addr = src_addr + len;
lock = NULL;		lock = NULL;
if (dst_pmap < src_pmap) {		if (dst_pmap < src_pmap) {
PMAP_LOCK(dst_pmap);		PMAP_LOCK(dst_pmap);
PMAP_LOCK(src_pmap);		PMAP_LOCK(src_pmap);
} else {		} else {
PMAP_LOCK(src_pmap);		PMAP_LOCK(src_pmap);
PMAP_LOCK(dst_pmap);		PMAP_LOCK(dst_pmap);
}		}

PG_A = pmap_accessed_bit(dst_pmap);		PG_A = pmap_accessed_bit(dst_pmap);
PG_M = pmap_modified_bit(dst_pmap);		PG_M = pmap_modified_bit(dst_pmap);
PG_V = pmap_valid_bit(dst_pmap);		PG_V = pmap_valid_bit(dst_pmap);

for (addr = src_addr; addr < end_addr; addr = va_next) {		for (addr = src_addr; addr < end_addr; addr = va_next) {
pt_entry_t src_pte, dst_pte;
pml4_entry_t *pml4e;
pdp_entry_t *pdpe;
pd_entry_t srcptepaddr, *pde;

KASSERT(addr < UPT_MIN_ADDRESS,		KASSERT(addr < UPT_MIN_ADDRESS,
("pmap_copy: invalid to pmap_copy page tables"));		("pmap_copy: invalid to pmap_copy page tables"));

pml4e = pmap_pml4e(src_pmap, addr);		pml4e = pmap_pml4e(src_pmap, addr);
if ((*pml4e & PG_V) == 0) {		if ((*pml4e & PG_V) == 0) {
va_next = (addr + NBPML4) & ~PML4MASK;		va_next = (addr + NBPML4) & ~PML4MASK;
if (va_next < addr)		if (va_next < addr)
va_next = end_addr;		va_next = end_addr;
Show All 25 Lines	if (srcptepaddr & PG_PS) {
break;		break;
pde = (pd_entry_t *)		pde = (pd_entry_t *)
PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg));		PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg));
pde = &pde[pmap_pde_index(addr)];		pde = &pde[pmap_pde_index(addr)];
if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 \|\|		if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 \|\|
pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr,		pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr,
PMAP_ENTER_NORECLAIM, &lock))) {		PMAP_ENTER_NORECLAIM, &lock))) {
*pde = srcptepaddr & ~PG_W;		*pde = srcptepaddr & ~PG_W;
pmap_resident_count_inc(dst_pmap, NBPDR / PAGE_SIZE);		pmap_resident_count_inc(dst_pmap, NBPDR /
		PAGE_SIZE);
atomic_add_long(&pmap_pde_mappings, 1);		atomic_add_long(&pmap_pde_mappings, 1);
} else		} else
dst_pdpg->wire_count--;		dst_pdpg->wire_count--;
continue;		continue;
}		}

srcptepaddr &= PG_FRAME;		srcptepaddr &= PG_FRAME;
srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);		srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
KASSERT(srcmpte->wire_count > 0,		KASSERT(srcmpte->wire_count > 0,
("pmap_copy: source page table page is unused"));		("pmap_copy: source page table page is unused"));

if (va_next > end_addr)		if (va_next > end_addr)
va_next = end_addr;		va_next = end_addr;

src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr);		src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr);
src_pte = &src_pte[pmap_pte_index(addr)];		src_pte = &src_pte[pmap_pte_index(addr)];
dstmpte = NULL;		dstmpte = NULL;
while (addr < va_next) {		for (; addr < va_next; addr += PAGE_SIZE, src_pte++) {
pt_entry_t ptetemp;
ptetemp = *src_pte;		ptetemp = *src_pte;

/*		/*
* we only virtual copy managed pages		* We only virtual copy managed pages.
*/		*/
if ((ptetemp & PG_MANAGED) != 0) {		if ((ptetemp & PG_MANAGED) == 0)
if (dstmpte != NULL &&		continue;
dstmpte->pindex == pmap_pde_pindex(addr))
		if (dstmpte != NULL) {
		KASSERT(dstmpte->pindex ==
		pmap_pde_pindex(addr),
		("dstmpte pindex/addr mismatch"));
dstmpte->wire_count++;		dstmpte->wire_count++;
else if ((dstmpte = pmap_allocpte(dst_pmap,		} else if ((dstmpte = pmap_allocpte(dst_pmap, addr,
addr, NULL)) == NULL)		NULL)) == NULL)
goto out;		goto out;
dst_pte = (pt_entry_t *)		dst_pte = (pt_entry_t *)
PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));		PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
dst_pte = &dst_pte[pmap_pte_index(addr)];		dst_pte = &dst_pte[pmap_pte_index(addr)];
if (*dst_pte == 0 &&		if (*dst_pte == 0 &&
pmap_try_insert_pv_entry(dst_pmap, addr,		pmap_try_insert_pv_entry(dst_pmap, addr,
PHYS_TO_VM_PAGE(ptetemp & PG_FRAME),		PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), &lock)) {
&lock)) {
/*		/*
* Clear the wired, modified, and		* Clear the wired, modified, and accessed
* accessed (referenced) bits		* (referenced) bits during the copy.
* during the copy.
*/		*/
*dst_pte = ptetemp & ~(PG_W \| PG_M \|		*dst_pte = ptetemp & ~(PG_W \| PG_M \| PG_A);
PG_A);
pmap_resident_count_inc(dst_pmap, 1);		pmap_resident_count_inc(dst_pmap, 1);
} else {		} else {
SLIST_INIT(&free);		SLIST_INIT(&free);
if (pmap_unwire_ptp(dst_pmap, addr,		if (pmap_unwire_ptp(dst_pmap, addr, dstmpte,
dstmpte, &free)) {		&free)) {
/*		/*
* Although "addr" is not		* Although "addr" is not mapped,
* mapped, paging-structure		* paging-structure caches could
* caches could nonetheless		* nonetheless have entries that refer
* have entries that refer to		* to the freed page table pages.
* the freed page table pages.
* Invalidate those entries.		* Invalidate those entries.
*/		*/
pmap_invalidate_page(dst_pmap,		pmap_invalidate_page(dst_pmap, addr);
addr);		vm_page_free_pages_toq(&free, true);
vm_page_free_pages_toq(&free,
true);
}		}
goto out;		goto out;
}		}
		/* Have we copied all of the valid mappings? */
if (dstmpte->wire_count >= srcmpte->wire_count)		if (dstmpte->wire_count >= srcmpte->wire_count)
break;		break;
}
addr += PAGE_SIZE;
src_pte++;
}		}
}		}
out:		out:
if (lock != NULL)		if (lock != NULL)
rw_wunlock(lock);		rw_wunlock(lock);
PMAP_UNLOCK(src_pmap);		PMAP_UNLOCK(src_pmap);
PMAP_UNLOCK(dst_pmap);		PMAP_UNLOCK(dst_pmap);
}		}
▲ Show 20 Lines • Show All 3,441 Lines • Show Last 20 Lines