Changeset View
Standalone View
sys/amd64/amd64/pmap.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 115 Lines • ▼ Show 20 Lines | |||||
#include <sys/bus.h> | #include <sys/bus.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/ktr.h> | #include <sys/ktr.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/mman.h> | #include <sys/mman.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/obm.h> | |||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/rangeset.h> | #include <sys/rangeset.h> | ||||
#include <sys/rwlock.h> | #include <sys/rwlock.h> | ||||
#include <sys/sbuf.h> | #include <sys/sbuf.h> | ||||
#include <sys/sx.h> | #include <sys/sx.h> | ||||
#include <sys/turnstile.h> | #include <sys/turnstile.h> | ||||
#include <sys/vmem.h> | #include <sys/vmem.h> | ||||
#include <sys/vmmeter.h> | #include <sys/vmmeter.h> | ||||
Show All 38 Lines | |||||
#else | #else | ||||
#define PMAP_MEMDOM 1 | #define PMAP_MEMDOM 1 | ||||
#endif | #endif | ||||
static __inline boolean_t | static __inline boolean_t | ||||
pmap_type_guest(pmap_t pmap) | pmap_type_guest(pmap_t pmap) | ||||
{ | { | ||||
return ((pmap->pm_type == PT_EPT) || (pmap->pm_type == PT_RVI)); | return ((pmap->pm_type == PT_EPT) || (pmap->pm_type == PT_RVI)); | ||||
} | } | ||||
alc: Empty messages? | |||||
static __inline boolean_t | static __inline boolean_t | ||||
pmap_emulate_ad_bits(pmap_t pmap) | pmap_emulate_ad_bits(pmap_t pmap) | ||||
{ | { | ||||
return ((pmap->pm_flags & PMAP_EMULATE_AD_BITS) != 0); | return ((pmap->pm_flags & PMAP_EMULATE_AD_BITS) != 0); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 123 Lines • ▼ Show 20 Lines | |||||
#define PMAP_INLINE __attribute__((__gnu_inline__)) inline | #define PMAP_INLINE __attribute__((__gnu_inline__)) inline | ||||
#else | #else | ||||
#define PMAP_INLINE extern inline | #define PMAP_INLINE extern inline | ||||
#endif | #endif | ||||
#else | #else | ||||
#define PMAP_INLINE | #define PMAP_INLINE | ||||
#endif | #endif | ||||
#ifdef PV_STATS | #ifdef PV_STATS | ||||
Done Inline ActionsCan't it be static? markj: Can't it be static? | |||||
#define PV_STAT(x) do { x ; } while (0) | #define PV_STAT(x) do { x ; } while (0) | ||||
#else | #else | ||||
#define PV_STAT(x) do { } while (0) | #define PV_STAT(x) do { } while (0) | ||||
#endif | #endif | ||||
#undef pa_index | #undef pa_index | ||||
#define pa_index(pa) ({ \ | #define pa_index(pa) ({ \ | ||||
KASSERT((pa) <= vm_phys_segs[vm_phys_nsegs - 1].end, \ | KASSERT((pa) <= vm_phys_segs[vm_phys_nsegs - 1].end, \ | ||||
("address %lx beyond the last segment", (pa))); \ | ("address %lx beyond the last segment", (pa))); \ | ||||
(pa) >> PDRSHIFT; \ | (pa) >> PDRSHIFT; \ | ||||
}) | }) | ||||
#ifdef NUMA | #ifdef NUMA | ||||
#define pa_to_pmdp(pa) (&pv_table[pa_index(pa)]) | #define pa_to_pmdp(pa) (&pv_table[pa_index(pa)]) | ||||
#define pa_to_pvh(pa) (&(pa_to_pmdp(pa)->pv_page)) | #define pa_to_pvh(pa) (&(pa_to_pmdp(pa)->pv_page)) | ||||
#define PHYS_TO_PV_LIST_LOCK(pa) ({ \ | |||||
struct rwlock *_lock; \ | |||||
if (__predict_false((pa) > pmap_last_pa)) \ | |||||
_lock = &pv_dummy_large.pv_lock; \ | |||||
else \ | |||||
_lock = &(pa_to_pmdp(pa)->pv_lock); \ | |||||
_lock; \ | |||||
}) | |||||
#else | #else | ||||
#define pa_to_pvh(pa) (&pv_table[pa_index(pa)]) | #define pa_to_pvh(pa) (&pv_table[pa_index(pa)]) | ||||
#define NPV_LIST_LOCKS MAXCPU | #define NPV_LIST_LOCKS MAXCPU | ||||
#define PHYS_TO_PV_LIST_LOCK(pa) \ | |||||
(&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS]) | |||||
#endif | #endif | ||||
#define PHYS_TO_PV_LIST_LOCK(pa) PHYS_TO_VM_PAGE(pa) | |||||
#define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ | #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ | ||||
struct rwlock **_lockp = (lockp); \ | PVLL **_lockp = (lockp); \ | ||||
struct rwlock *_new_lock; \ | PVLL *_new_lock; \ | ||||
\ | \ | ||||
_new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ | _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ | ||||
if (_new_lock != *_lockp) { \ | if (_new_lock != *_lockp) { \ | ||||
if (*_lockp != NULL) \ | if (*_lockp != NULL) \ | ||||
rw_wunlock(*_lockp); \ | pmap_pv_list_unlock(*_lockp); \ | ||||
if (_new_lock == NULL) \ | |||||
_new_lock = &pv_fake_page; \ | |||||
Done Inline ActionsIs this actually the correct place for this "if" statement? Suppose that *lockp is NULL, because this execution of CHANGE_PV_LIST_LOCK_TO_PHYS() is the first to acquire a lock, and that PHYS_TO_VM_PAGE(pa) returns NULL. _new_lock != *_lockp will be false, and so we won't enter this block and acquire the lock associated with pv_fake_page. Am I missing something? alc: Is this actually the correct place for this "if" statement? Suppose that *lockp is NULL… | |||||
*_lockp = _new_lock; \ | *_lockp = _new_lock; \ | ||||
rw_wlock(*_lockp); \ | pmap_pv_list_lock(*_lockp); \ | ||||
} \ | } \ | ||||
Done Inline ActionsThis could be shortened to: vm_page_t _m; _m = PHYS_TO_VM_PAGE(pa); if (_m == NULL) _m = &pv_fake_page; CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, _m); alc: This could be shortened to:
```
vm_page_t _m;
_m = PHYS_TO_VM_PAGE(pa);
if (_m == NULL)… | |||||
} while (0) | } while (0) | ||||
#define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ | #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) do { \ | ||||
CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) | PVLL **_lockp = (lockp); \ | ||||
\ | |||||
if (m != *_lockp) { \ | |||||
if (*_lockp != NULL) \ | |||||
pmap_pv_list_unlock(*_lockp); \ | |||||
*_lockp = m; \ | |||||
pmap_pv_list_lock(m); \ | |||||
} \ | |||||
} while (0) | |||||
#define RELEASE_PV_LIST_LOCK(lockp) do { \ | #define RELEASE_PV_LIST_LOCK(lockp) do { \ | ||||
struct rwlock **_lockp = (lockp); \ | PVLL **_lockp = (lockp); \ | ||||
\ | \ | ||||
if (*_lockp != NULL) { \ | if (*_lockp != NULL) { \ | ||||
rw_wunlock(*_lockp); \ | pmap_pv_list_unlock(*_lockp); \ | ||||
*_lockp = NULL; \ | *_lockp = NULL; \ | ||||
} \ | } \ | ||||
} while (0) | } while (0) | ||||
#define VM_PAGE_TO_PV_LIST_LOCK(m) \ | #define VM_PAGE_TO_PV_LIST_LOCK(m) (m) | ||||
PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) | |||||
struct pmap kernel_pmap_store; | struct pmap kernel_pmap_store; | ||||
vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ | vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ | ||||
vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ | vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ | ||||
int nkpt; | int nkpt; | ||||
SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, | SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, | ||||
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | |||||
struct pv_chunks_list { | struct pv_chunks_list { | ||||
struct mtx pvc_lock; | struct mtx pvc_lock; | ||||
TAILQ_HEAD(pch, pv_chunk) pvc_list; | TAILQ_HEAD(pch, pv_chunk) pvc_list; | ||||
int active_reclaims; | int active_reclaims; | ||||
} __aligned(CACHE_LINE_SIZE); | } __aligned(CACHE_LINE_SIZE); | ||||
struct pv_chunks_list __exclusive_cache_line pv_chunks[PMAP_MEMDOM]; | struct pv_chunks_list __exclusive_cache_line pv_chunks[PMAP_MEMDOM]; | ||||
typedef struct vm_page PVLL; | |||||
#ifdef NUMA | #ifdef NUMA | ||||
struct pmap_large_md_page { | struct pmap_large_md_page { | ||||
struct rwlock pv_lock; | struct lock_object lo; | ||||
uintptr_t pad; | |||||
struct md_page pv_page; | struct md_page pv_page; | ||||
struct md_page pv_page; | |||||
markjUnsubmitted Done Inline ActionsDuplicate field? markj: Duplicate field? | |||||
kibAuthorUnsubmitted Done Inline ActionsIndeed, double-patched while moved the patch around and then did not tested NUMA. kib: Indeed, double-patched while moved the patch around and then did not tested NUMA. | |||||
u_long pv_invl_gen; | u_long pv_invl_gen; | ||||
}; | }; | ||||
/* | |||||
* We strongly depend on the size being a power of two, so the assert | |||||
Done Inline Actions-> "We only depend ... alc: -> "We only depend ... | |||||
* is overzealous. However, should the struct be resized to a | |||||
Done Inline ActionsElsewhere there are two spaces after a period. alc: Elsewhere there are two spaces after a period. | |||||
* different power of two, the code below needs to be revisited. | |||||
*/ | |||||
_Static_assert(sizeof(struct pmap_large_md_page) == 64, "pmap_large_md_page"); | |||||
__exclusive_cache_line static struct pmap_large_md_page pv_dummy_large; | __exclusive_cache_line static struct pmap_large_md_page pv_dummy_large; | ||||
#define pv_dummy pv_dummy_large.pv_page | #define pv_dummy pv_dummy_large.pv_page | ||||
__read_mostly static struct pmap_large_md_page *pv_table; | __read_mostly static struct pmap_large_md_page *pv_table; | ||||
__read_mostly vm_paddr_t pmap_last_pa; | __read_mostly vm_paddr_t pmap_last_pa; | ||||
static struct lock_object * | |||||
pv_list_lock_object(vm_paddr_t pa) | |||||
{ | |||||
if (__predict_false((pa) > pmap_last_pa)) \ | |||||
Done Inline ActionsExtra parens. markj: Extra parens. | |||||
return (&pv_dummy_large.lo); | |||||
return (&(pa_to_pmdp(pa)->lo)); | |||||
Done Inline ActionsExtra parens. markj: Extra parens. | |||||
} | |||||
#else | #else | ||||
static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS]; | static struct lock_object __exclusive_cache_line pv_lo[NPV_LIST_LOCKS]; | ||||
static u_long pv_invl_gen[NPV_LIST_LOCKS]; | static u_long pv_invl_gen[NPV_LIST_LOCKS]; | ||||
static struct md_page *pv_table; | static struct md_page *pv_table; | ||||
static struct md_page pv_dummy; | static struct md_page pv_dummy; | ||||
static struct lock_object * | |||||
pv_list_lock_object(vm_paddr_t pa) | |||||
{ | |||||
return (&pv_lo[pa_index(pa) % NPV_LIST_LOCKS]); | |||||
} | |||||
#endif | #endif | ||||
__read_mostly static struct vm_page pv_fake_page; | |||||
static void | |||||
pmap_pv_list_lock(vm_page_t m) | |||||
{ | |||||
obm_lock(&m->md.pv_lock, pv_list_lock_object(VM_PAGE_TO_PHYS(m))); | |||||
} | |||||
static void | |||||
pmap_pv_list_unlock(vm_page_t m) | |||||
{ | |||||
obm_unlock(&m->md.pv_lock, pv_list_lock_object(VM_PAGE_TO_PHYS(m))); | |||||
} | |||||
/* | /* | ||||
* Locks all pv lists for 4k pages constituing the superpage that | |||||
Done Inline ActionsTypo, "constituting". markj: Typo, "constituting". | |||||
* contains the passed page. The page's pv list must be locked. | |||||
Done Inline ActionsIt might not be locked, depending on pte_locked. markj: It might not be locked, depending on `pte_locked`. | |||||
Done Inline Actions"indicates whether the PV llst for m is already locked." alc: "indicates whether the PV llst for m is already locked." | |||||
* Returns false if trylock failed and the page's pv list was unlocked | |||||
Done Inline ActionsPerhaps, "if the initial trylock failed". markj: Perhaps, "if the initial trylock failed". | |||||
Done Inline ActionsThis sentence, specifically, the part that says, "... is locked according to ...", doesn't really explain anything to me. alc: This sentence, specifically, the part that says, "... is locked according to ...", doesn't… | |||||
* in the process, which typically means that the caller must restart. | |||||
*/ | |||||
static bool | |||||
Done Inline Actions"in the process" of what? alc: "in the process" of what? | |||||
pmap_pv_list_lock_pde1(vm_page_t m, bool pte_locked) | |||||
{ | |||||
vm_page_t mt, sm; | |||||
struct lock_object *lo; | |||||
int i; | |||||
bool ret; | |||||
obm_assert_locked(&m->md.pv_lock); | |||||
Done Inline ActionsI don't see how this assertion is true when pmap_pv_list_pde() is called with lockp == NULL. markj: I don't see how this assertion is true when pmap_pv_list_pde() is called with lockp == NULL. | |||||
Done Inline ActionsYes, practically we always demote or promote with the pv list lock held. Also I think it is safe to assume that PHYS_TO_PAGE(pa) in pmap_pv_list_lock_pde() never returns NULL, since we can be only called for managed pages. kib: Yes, practically we always demote or promote with the pv list lock held. Also I think it is… | |||||
Done Inline ActionsElsewhere, e.g., promotion, I've written this as atop(VM_PAGE_TO_PHYS(m) & PG_FRAME & PDRMASK)) so that there is only one arithmetic operation at run-time. alc: Elsewhere, e.g., promotion, I've written this as `atop(VM_PAGE_TO_PHYS(m) & PG_FRAME &… | |||||
sm = m - ((VM_PAGE_TO_PHYS(m) - (VM_PAGE_TO_PHYS(m) & | |||||
PG_PS_FRAME)) >> PAGE_SHIFT); | |||||
Done Inline ActionsUse atop()? markj: Use atop()? | |||||
lo = pv_list_lock_object(VM_PAGE_TO_PHYS(m)); | |||||
if (!pte_locked) { | |||||
ret = true; | |||||
goto alllocks; | |||||
} | |||||
/* | |||||
Done Inline ActionsSince this is iterating at 4KB granularity, NPTEPG would seem more appropriate. alc: Since this is iterating at 4KB granularity, NPTEPG would seem more appropriate. | |||||
* Fast attempt. If we either own or can get the pv list lock | |||||
* of the first page in the superpage, all other owners must | |||||
* release their locks without waiting for us. | |||||
*/ | |||||
if (m == sm || obm_trylock(&sm->md.pv_lock)) { | |||||
for (i = 1, mt = sm + 1; i < NPDEPG; i++, mt++) { | |||||
if (m != mt) | |||||
obm_lock(&mt->md.pv_lock, lo); | |||||
} | |||||
return (true); | |||||
Done Inline ActionsSince this is iterating at 4KB granularity, NPTEPG would seem more appropriate. alc: Since this is iterating at 4KB granularity, NPTEPG would seem more appropriate. | |||||
} | |||||
obm_unlock(&m->md.pv_lock, lo); | |||||
ret = false; | |||||
alllocks: | |||||
for (i = 0, mt = sm; i < NPDEPG; i++, mt++) { | |||||
obm_lock(&mt->md.pv_lock, lo); | |||||
} | |||||
return (ret); | |||||
markjUnsubmitted Done Inline ActionsThe return value appears to be unused. markj: The return value appears to be unused. | |||||
} | |||||
/* | |||||
Done Inline Actions"page'" -> "page's" alc: "page'" -> "page's" | |||||
* If *lockp points to one of the ordinary pages from the superpage we | |||||
* are demoting or promoting, then we keep this page' pv list locked | |||||
Done Inline Actions"... all run ..."? alc: "... all run ..."? | |||||
* after pmap_pv_list_unlock_pde(). Otherwise, we just unlock whatever | |||||
* was locked, and unlock all run on pmap_pv_list_unlock_pde(). | |||||
Done Inline ActionsDoesn't this comment describe the next function and not the one that immediately follows? alc: Doesn't this comment describe the next function and not the one that immediately follows? | |||||
*/ | |||||
static void | |||||
pmap_pv_list_lock_pde(vm_paddr_t pa, PVLL **lockp) | |||||
{ | |||||
vm_page_t m; | |||||
m = PHYS_TO_VM_PAGE(pa); | |||||
if (m == NULL) | |||||
m = &pv_fake_page; | |||||
if (*lockp == NULL) { | |||||
pmap_pv_list_lock_pde1(m, false); | |||||
return; | |||||
} | |||||
if ((VM_PAGE_TO_PHYS(*lockp) & PG_PS_FRAME) != (pa & PG_PS_FRAME)) { | |||||
pmap_pv_list_unlock(*lockp); | |||||
*lockp = NULL; | |||||
pmap_pv_list_lock_pde1(m, false); | |||||
return; | |||||
} | |||||
pmap_pv_list_lock_pde1(*lockp, true); | |||||
Done Inline ActionsWhy is it a separate subroutine? There is only one caller. markj: Why is it a separate subroutine? There is only one caller. | |||||
} | |||||
static void | |||||
pmap_pv_list_unlock_pde1(vm_page_t m, bool pte_locked) | |||||
{ | |||||
vm_page_t mt, sm; | |||||
struct lock_object *lo; | |||||
int i; | |||||
sm = m - ((VM_PAGE_TO_PHYS(m) - (VM_PAGE_TO_PHYS(m) & | |||||
PG_PS_FRAME)) >> PAGE_SHIFT); | |||||
lo = pv_list_lock_object(VM_PAGE_TO_PHYS(m)); | |||||
obm_assert_locked(&m->md.pv_lock); | |||||
obm_assert_locked(&sm->md.pv_lock); | |||||
for (i = 0, mt = sm; i < NPDEPG; i++, mt++) { | |||||
if (!pte_locked || mt != m) | |||||
obm_unlock(&mt->md.pv_lock, lo); | |||||
} | |||||
} | |||||
Done Inline ActionsElsewhere, e.g., promotion, I've written this as atop(VM_PAGE_TO_PHYS(m) & PG_FRAME & PDRMASK)). alc: Elsewhere, e.g., promotion, I've written this as `atop(VM_PAGE_TO_PHYS(m) & PG_FRAME &… | |||||
static void | |||||
pmap_pv_list_unlock_pde(vm_paddr_t pa, PVLL **lockp) | |||||
Done Inline ActionsAlthough it makes no functional difference, I think that this would more correctly be NPTEPG. alc: Although it makes no functional difference, I think that this would more correctly be NPTEPG. | |||||
{ | |||||
vm_page_t m; | |||||
bool pte_locked; | |||||
m = *lockp; | |||||
pte_locked = m != NULL; | |||||
if (!pte_locked) { | |||||
m = PHYS_TO_VM_PAGE(pa); | |||||
if (m == NULL) | |||||
m = &pv_fake_page; | |||||
} | |||||
pmap_pv_list_unlock_pde1(m, pte_locked); | |||||
} | |||||
/* | |||||
* All those kernel PT submaps that BSD is so fond of | * All those kernel PT submaps that BSD is so fond of | ||||
*/ | */ | ||||
pt_entry_t *CMAP1 = NULL; | pt_entry_t *CMAP1 = NULL; | ||||
caddr_t CADDR1 = 0; | caddr_t CADDR1 = 0; | ||||
static vm_offset_t qframe = 0; | static vm_offset_t qframe = 0; | ||||
static struct mtx qframe_mtx; | static struct mtx qframe_mtx; | ||||
static int pmap_flags = PMAP_PDE_SUPERPAGE; /* flags for x86 pmaps */ | static int pmap_flags = PMAP_PDE_SUPERPAGE; /* flags for x86 pmaps */ | ||||
▲ Show 20 Lines • Show All 667 Lines • ▼ Show 20 Lines | |||||
* This forces a caller of pmap_delayed_invl_wait() to block until | * This forces a caller of pmap_delayed_invl_wait() to block until | ||||
* current thread calls pmap_delayed_invl_finish(). | * current thread calls pmap_delayed_invl_finish(). | ||||
*/ | */ | ||||
static void | static void | ||||
pmap_delayed_invl_page(vm_page_t m) | pmap_delayed_invl_page(vm_page_t m) | ||||
{ | { | ||||
u_long gen, *m_gen; | u_long gen, *m_gen; | ||||
rw_assert(VM_PAGE_TO_PV_LIST_LOCK(m), RA_WLOCKED); | //XXXKIB rw_assert(VM_PAGE_TO_PV_LIST_LOCK(m), RA_WLOCKED); | ||||
Done Inline ActionsI don't quite see what the problem is here. markj: I don't quite see what the problem is here. | |||||
Done Inline ActionsIn pmap_remove_pde(), only first 4k page from the superpage run has its pv list locked. kib: In pmap_remove_pde(), only first 4k page from the superpage run has its pv list locked. | |||||
Done Inline ActionsAnd after I thought about it, I believe it should be just fixed. kib: And after I thought about it, I believe it should be just fixed. | |||||
gen = curthread->td_md.md_invl_gen.gen; | gen = curthread->td_md.md_invl_gen.gen; | ||||
if (gen == 0) | if (gen == 0) | ||||
return; | return; | ||||
m_gen = pmap_delayed_invl_genp(m); | m_gen = pmap_delayed_invl_genp(m); | ||||
if (*m_gen < gen) | if (*m_gen < gen) | ||||
*m_gen = gen; | *m_gen = gen; | ||||
} | } | ||||
Show All 16 Lines | |||||
#define MAPDEV_SETATTR 0x00000002 /* Modify existing attrs. */ | #define MAPDEV_SETATTR 0x00000002 /* Modify existing attrs. */ | ||||
#define MAPDEV_ASSERTVALID 0x00000004 /* Assert mapping validity. */ | #define MAPDEV_ASSERTVALID 0x00000004 /* Assert mapping validity. */ | ||||
TAILQ_HEAD(pv_chunklist, pv_chunk); | TAILQ_HEAD(pv_chunklist, pv_chunk); | ||||
static void free_pv_chunk(struct pv_chunk *pc); | static void free_pv_chunk(struct pv_chunk *pc); | ||||
static void free_pv_chunk_batch(struct pv_chunklist *batch); | static void free_pv_chunk_batch(struct pv_chunklist *batch); | ||||
static void free_pv_entry(pmap_t pmap, pv_entry_t pv); | static void free_pv_entry(pmap_t pmap, pv_entry_t pv); | ||||
static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); | static pv_entry_t get_pv_entry(pmap_t pmap, PVLL **lockp); | ||||
static int popcnt_pc_map_pq(uint64_t *map); | static int popcnt_pc_map_pq(uint64_t *map); | ||||
static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); | static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, PVLL **lockp); | ||||
static void reserve_pv_entries(pmap_t pmap, int needed, | static void reserve_pv_entries(pmap_t pmap, int needed, PVLL **lockp); | ||||
struct rwlock **lockp); | |||||
static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | ||||
struct rwlock **lockp); | PVLL **lockp); | ||||
static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, | static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, | ||||
u_int flags, struct rwlock **lockp); | u_int flags, PVLL **lockp); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | ||||
struct rwlock **lockp); | PVLL **lockp); | ||||
#endif | #endif | ||||
static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); | static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); | ||||
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, | static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, | ||||
vm_offset_t va); | vm_offset_t va); | ||||
static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte); | static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte); | ||||
static int pmap_change_props_locked(vm_offset_t va, vm_size_t size, | static int pmap_change_props_locked(vm_offset_t va, vm_size_t size, | ||||
vm_prot_t prot, int mode, int flags); | vm_prot_t prot, int mode, int flags); | ||||
static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); | static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); | ||||
static boolean_t pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, | static boolean_t pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, | ||||
vm_offset_t va, struct rwlock **lockp); | vm_offset_t va, PVLL **lockp); | ||||
static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, | static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, | ||||
vm_offset_t va); | vm_offset_t va); | ||||
static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, | static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
vm_prot_t prot, struct rwlock **lockp); | vm_prot_t prot, PVLL **lockp); | ||||
static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, | static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, | ||||
u_int flags, vm_page_t m, struct rwlock **lockp); | u_int flags, vm_page_t m, PVLL **lockp); | ||||
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, | static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, | ||||
vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); | vm_page_t m, vm_prot_t prot, vm_page_t mpte, PVLL **lockp); | ||||
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); | static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); | ||||
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted); | static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted); | ||||
static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, | static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, | ||||
vm_offset_t eva); | vm_offset_t eva); | ||||
static void pmap_invalidate_cache_range_all(vm_offset_t sva, | static void pmap_invalidate_cache_range_all(vm_offset_t sva, | ||||
vm_offset_t eva); | vm_offset_t eva); | ||||
static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, | static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, | ||||
pd_entry_t pde); | pd_entry_t pde); | ||||
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); | static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); | ||||
static vm_page_t pmap_large_map_getptp_unlocked(void); | static vm_page_t pmap_large_map_getptp_unlocked(void); | ||||
static vm_paddr_t pmap_large_map_kextract(vm_offset_t va); | static vm_paddr_t pmap_large_map_kextract(vm_offset_t va); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, | static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, | ||||
struct rwlock **lockp); | PVLL **lockp); | ||||
#endif | #endif | ||||
static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, | static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, | ||||
vm_prot_t prot); | vm_prot_t prot); | ||||
static void pmap_pte_props(pt_entry_t *pte, u_long bits, u_long mask); | static void pmap_pte_props(pt_entry_t *pte, u_long bits, u_long mask); | ||||
static void pmap_pti_add_kva_locked(vm_offset_t sva, vm_offset_t eva, | static void pmap_pti_add_kva_locked(vm_offset_t sva, vm_offset_t eva, | ||||
bool exec); | bool exec); | ||||
static pdp_entry_t *pmap_pti_pdpe(vm_offset_t va); | static pdp_entry_t *pmap_pti_pdpe(vm_offset_t va); | ||||
static pd_entry_t *pmap_pti_pde(vm_offset_t va); | static pd_entry_t *pmap_pti_pde(vm_offset_t va); | ||||
static void pmap_pti_wire_pte(void *pte); | static void pmap_pti_wire_pte(void *pte); | ||||
static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, | static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, | ||||
struct spglist *free, struct rwlock **lockp); | struct spglist *free, PVLL **lockp); | ||||
static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, | static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, | ||||
pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); | pd_entry_t ptepde, struct spglist *free, PVLL **lockp); | ||||
static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); | static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); | ||||
static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | ||||
struct spglist *free); | struct spglist *free); | ||||
static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, | static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, | ||||
pd_entry_t *pde, struct spglist *free, | pd_entry_t *pde, struct spglist *free, PVLL **lockp); | ||||
struct rwlock **lockp); | |||||
static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, | static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, | ||||
vm_page_t m, struct rwlock **lockp); | vm_page_t m, PVLL **lockp); | ||||
static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | ||||
pd_entry_t newpde); | pd_entry_t newpde); | ||||
static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde); | static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde); | ||||
static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, | static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, | ||||
struct rwlock **lockp); | PVLL **lockp); | ||||
static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | ||||
struct rwlock **lockp); | PVLL **lockp); | ||||
static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, | static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, | ||||
struct rwlock **lockp); | PVLL **lockp); | ||||
static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, | static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
struct spglist *free); | struct spglist *free); | ||||
static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); | static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); | ||||
/********************/ | /********************/ | ||||
/* Inline functions */ | /* Inline functions */ | ||||
/********************/ | /********************/ | ||||
▲ Show 20 Lines • Show All 618 Lines • ▼ Show 20 Lines | |||||
* Initialize a vm_page's machine-dependent fields. | * Initialize a vm_page's machine-dependent fields. | ||||
*/ | */ | ||||
void | void | ||||
pmap_page_init(vm_page_t m) | pmap_page_init(vm_page_t m) | ||||
{ | { | ||||
TAILQ_INIT(&m->md.pv_list); | TAILQ_INIT(&m->md.pv_list); | ||||
m->md.pat_mode = PAT_WRITE_BACK; | m->md.pat_mode = PAT_WRITE_BACK; | ||||
obm_init(&m->md.pv_lock); | |||||
} | } | ||||
static int pmap_allow_2m_x_ept; | static int pmap_allow_2m_x_ept; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, allow_2m_x_ept, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, | SYSCTL_INT(_vm_pmap, OID_AUTO, allow_2m_x_ept, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, | ||||
&pmap_allow_2m_x_ept, 0, | &pmap_allow_2m_x_ept, 0, | ||||
"Allow executable superpage mappings in EPT"); | "Allow executable superpage mappings in EPT"); | ||||
void | void | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | |||||
pmap_init_pv_table(void) | pmap_init_pv_table(void) | ||||
{ | { | ||||
struct pmap_large_md_page *pvd; | struct pmap_large_md_page *pvd; | ||||
vm_size_t s; | vm_size_t s; | ||||
long start, end, highest, pv_npg; | long start, end, highest, pv_npg; | ||||
int domain, i, j, pages; | int domain, i, j, pages; | ||||
/* | /* | ||||
* We strongly depend on the size being a power of two, so the assert | |||||
* is overzealous. However, should the struct be resized to a | |||||
* different power of two, the code below needs to be revisited. | |||||
*/ | |||||
CTASSERT((sizeof(*pvd) == 64)); | |||||
/* | |||||
* Calculate the size of the array. | * Calculate the size of the array. | ||||
*/ | */ | ||||
pmap_last_pa = vm_phys_segs[vm_phys_nsegs - 1].end; | pmap_last_pa = vm_phys_segs[vm_phys_nsegs - 1].end; | ||||
pv_npg = howmany(pmap_last_pa, NBPDR); | pv_npg = howmany(pmap_last_pa, NBPDR); | ||||
s = (vm_size_t)pv_npg * sizeof(struct pmap_large_md_page); | s = (vm_size_t)pv_npg * sizeof(struct pmap_large_md_page); | ||||
s = round_page(s); | s = round_page(s); | ||||
pv_table = (struct pmap_large_md_page *)kva_alloc(s); | pv_table = (struct pmap_large_md_page *)kva_alloc(s); | ||||
if (pv_table == NULL) | if (pv_table == NULL) | ||||
Show All 17 Lines | for (i = 0; i < vm_phys_nsegs; i++) { | ||||
pages = end - start + 1; | pages = end - start + 1; | ||||
s = round_page(pages * sizeof(*pvd)); | s = round_page(pages * sizeof(*pvd)); | ||||
highest = start + (s / sizeof(*pvd)) - 1; | highest = start + (s / sizeof(*pvd)) - 1; | ||||
for (j = 0; j < s; j += PAGE_SIZE) { | for (j = 0; j < s; j += PAGE_SIZE) { | ||||
vm_page_t m = vm_page_alloc_domain(NULL, 0, | vm_page_t m = vm_page_alloc_domain(NULL, 0, | ||||
domain, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ); | domain, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ); | ||||
if (m == NULL) | if (m == NULL) | ||||
panic("vm_page_alloc_domain failed for %lx\n", (vm_offset_t)pvd + j); | panic("vm_page_alloc_domain failed for %lx\n", | ||||
(vm_offset_t)pvd + j); | |||||
pmap_qenter((vm_offset_t)pvd + j, &m, 1); | pmap_qenter((vm_offset_t)pvd + j, &m, 1); | ||||
} | } | ||||
for (j = 0; j < s / sizeof(*pvd); j++) { | for (j = 0; j < s / sizeof(*pvd); j++) { | ||||
rw_init_flags(&pvd->pv_lock, "pmap pv list", RW_NEW); | obm_init_lo(&pvd->lo, "pmap pv list"); | ||||
TAILQ_INIT(&pvd->pv_page.pv_list); | TAILQ_INIT(&pvd->pv_page.pv_list); | ||||
pvd->pv_page.pv_gen = 0; | pvd->pv_page.pv_gen = 0; | ||||
pvd->pv_page.pat_mode = 0; | pvd->pv_page.pat_mode = 0; | ||||
pvd->pv_invl_gen = 0; | pvd->pv_invl_gen = 0; | ||||
pvd++; | pvd++; | ||||
} | } | ||||
} | } | ||||
pvd = &pv_dummy_large; | pvd = &pv_dummy_large; | ||||
rw_init_flags(&pvd->pv_lock, "pmap pv list dummy", RW_NEW); | obm_init_lo(&pvd->lo, "pmap pv list dummy"); | ||||
TAILQ_INIT(&pvd->pv_page.pv_list); | TAILQ_INIT(&pvd->pv_page.pv_list); | ||||
pmap_page_init(&pv_fake_page); | |||||
pv_fake_page.phys_addr = pmap_last_pa + PAGE_SIZE; | |||||
Done Inline ActionsPerhaps add a comment explaining that this assignment is required by pv_list_lock_object(), ditto in the #else case below. markj: Perhaps add a comment explaining that this assignment is required by pv_list_lock_object()… | |||||
pvd->pv_page.pv_gen = 0; | pvd->pv_page.pv_gen = 0; | ||||
Done Inline ActionsTypo, "fake" markj: Typo, "fake" | |||||
Done Inline ActionsThis is actually 'make' (used to make all physical addresses work) kib: This is actually 'make' (`used to make all physical addresses work`) | |||||
Done Inline ActionsThe 'm' is missing from the word "make". It's currently "ake". alc: The 'm' is missing from the word "make". It's currently "ake". | |||||
pvd->pv_page.pat_mode = 0; | pvd->pv_page.pat_mode = 0; | ||||
pvd->pv_invl_gen = 0; | pvd->pv_invl_gen = 0; | ||||
} | } | ||||
#else | #else | ||||
Done Inline Actions"Initialize pv_fake page, which is used to make pv_list locking work for physical addresses not covered by vm_page_array[]. In particular, it is needed by pv_list_lock_object() and CHANGE_PV_LIST_LOCK_TO_PHYS(). alc: "Initialize pv_fake page, which is used to make pv_list locking work for physical addresses not… | |||||
static void | static void | ||||
pmap_init_pv_table(void) | pmap_init_pv_table(void) | ||||
{ | { | ||||
vm_size_t s; | vm_size_t s; | ||||
long i, pv_npg; | long i, pv_npg; | ||||
/* | /* | ||||
* Initialize the pool of pv list locks. | * Initialize the pool of pv list locks. | ||||
*/ | */ | ||||
for (i = 0; i < NPV_LIST_LOCKS; i++) | for (i = 0; i < NPV_LIST_LOCKS; i++) | ||||
rw_init(&pv_list_locks[i], "pmap pv list"); | obm_init_lo(&pv_lo[i], "pmap pv list"); | ||||
/* | /* | ||||
* Calculate the size of the pv head table for superpages. | * Calculate the size of the pv head table for superpages. | ||||
*/ | */ | ||||
pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, NBPDR); | pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, NBPDR); | ||||
/* | /* | ||||
* Allocate memory for the pv head table for superpages. | * Allocate memory for the pv head table for superpages. | ||||
*/ | */ | ||||
s = (vm_size_t)pv_npg * sizeof(struct md_page); | s = (vm_size_t)pv_npg * sizeof(struct md_page); | ||||
s = round_page(s); | s = round_page(s); | ||||
pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO); | pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO); | ||||
for (i = 0; i < pv_npg; i++) | for (i = 0; i < pv_npg; i++) | ||||
TAILQ_INIT(&pv_table[i].pv_list); | TAILQ_INIT(&pv_table[i].pv_list); | ||||
TAILQ_INIT(&pv_dummy.pv_list); | TAILQ_INIT(&pv_dummy.pv_list); | ||||
pmap_page_init(&pv_fake_page); | |||||
pv_fake_page.phys_addr = vm_phys_segs[vm_phys_nsegs - 1].end + PAGE_SIZE; | |||||
Done Inline ActionsPerhaps add a comment explaining that this assignment is required by pv_list_lock_object(). markj: Perhaps add a comment explaining that this assignment is required by pv_list_lock_object(). | |||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* Initialize the pmap module. | * Initialize the pmap module. | ||||
* Called by vm_init, to initialize any structures that the pmap | * Called by vm_init, to initialize any structures that the pmap | ||||
* system needs to map virtual memory. | * system needs to map virtual memory. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 92 Lines • ▼ Show 20 Lines | pmap_init(void) | ||||
* Initialize pv chunk lists. | * Initialize pv chunk lists. | ||||
*/ | */ | ||||
for (i = 0; i < PMAP_MEMDOM; i++) { | for (i = 0; i < PMAP_MEMDOM; i++) { | ||||
mtx_init(&pv_chunks[i].pvc_lock, "pmap pv chunk list", NULL, MTX_DEF); | mtx_init(&pv_chunks[i].pvc_lock, "pmap pv chunk list", NULL, MTX_DEF); | ||||
TAILQ_INIT(&pv_chunks[i].pvc_list); | TAILQ_INIT(&pv_chunks[i].pvc_list); | ||||
} | } | ||||
pmap_init_pv_table(); | pmap_init_pv_table(); | ||||
pmap_initialized = 1; | pmap_initialized = 1; | ||||
Done Inline ActionsThese must be initialized in pmap_bootstrap(). markj: These must be initialized in pmap_bootstrap(). | |||||
for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { | for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { | ||||
ppim = pmap_preinit_mapping + i; | ppim = pmap_preinit_mapping + i; | ||||
if (ppim->va == 0) | if (ppim->va == 0) | ||||
continue; | continue; | ||||
/* Make the direct map consistent */ | /* Make the direct map consistent */ | ||||
if (ppim->pa < dmaplimit && ppim->pa + ppim->sz <= dmaplimit) { | if (ppim->pa < dmaplimit && ppim->pa + ppim->sz <= dmaplimit) { | ||||
(void)pmap_change_attr(PHYS_TO_DMAP(ppim->pa), | (void)pmap_change_attr(PHYS_TO_DMAP(ppim->pa), | ||||
ppim->sz, ppim->mode); | ppim->sz, ppim->mode); | ||||
▲ Show 20 Lines • Show All 1,461 Lines • ▼ Show 20 Lines | if (pmap_unwire_ptp(pmap, va, mpte, &free)) { | ||||
pmap_invalidate_page(pmap, va); | pmap_invalidate_page(pmap, va); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
} | } | ||||
void | void | ||||
pmap_pinit0(pmap_t pmap) | pmap_pinit0(pmap_t pmap) | ||||
{ | { | ||||
struct proc *p; | struct proc *p; | ||||
Done Inline ActionsRedundant parens. markj: Redundant parens. | |||||
struct thread *td; | struct thread *td; | ||||
int i; | int i; | ||||
Done Inline ActionsI would add a comment explaining that the pmap is never removed from the list, relying on type-stability of the vmspace zone. markj: I would add a comment explaining that the pmap is never removed from the list, relying on type… | |||||
PMAP_LOCK_INIT(pmap); | PMAP_LOCK_INIT(pmap); | ||||
Done Inline Actions"Add the pmap to the global list of pmaps, which is used during pv chunk reclamation. The pmap is ..." alc: "Add the pmap to the global list of pmaps, which is used during pv chunk reclamation. The pmap… | |||||
Done Inline Actions"pmap is" is duplicated. alc: "pmap is" is duplicated. | |||||
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); | pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); | ||||
pmap->pm_pml4u = NULL; | pmap->pm_pml4u = NULL; | ||||
pmap->pm_cr3 = KPML4phys; | pmap->pm_cr3 = KPML4phys; | ||||
/* hack to keep pmap_pti_pcid_invalidate() alive */ | /* hack to keep pmap_pti_pcid_invalidate() alive */ | ||||
pmap->pm_ucr3 = PMAP_NO_CR3; | pmap->pm_ucr3 = PMAP_NO_CR3; | ||||
pmap->pm_root.rt_root = 0; | pmap->pm_root.rt_root = 0; | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
TAILQ_INIT(&pmap->pm_pvchunk); | TAILQ_INIT(&pmap->pm_pvchunk); | ||||
bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | ||||
pmap->pm_flags = pmap_flags; | pmap->pm_flags = pmap_flags; | ||||
CPU_FOREACH(i) { | CPU_FOREACH(i) { | ||||
pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN + 1; | pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN + 1; | ||||
pmap->pm_pcids[i].pm_gen = 1; | pmap->pm_pcids[i].pm_gen = 1; | ||||
} | } | ||||
pmap_activate_boot(pmap); | pmap_activate_boot(pmap); | ||||
td = curthread; | td = curthread; | ||||
Done Inline Actionspmap_release() also needs to be modified to remove the queue entry. markj: pmap_release() also needs to be modified to remove the queue entry. | |||||
Done Inline ActionsI left the released pmap on the list on purpose. I want to be able to walk the list in reclaim function without adding a marker for pmap, because pmap placeholder would be rather large and I do not want allocate it when low on chunks (it cannot be on stack due to size). This is why I check for the pmap resident page count after obtaining pmap lock, to quickly skip such pmaps. kib: I left the released pmap on the list on purpose. I want to be able to walk the list in reclaim… | |||||
Done Inline ActionsBut then we cannot unconditionally enqueue here, since the pmap may already reside on the list. markj: But then we cannot unconditionally enqueue here, since the pmap may already reside on the list. | |||||
Done Inline ActionsCertainly we must not enqueue here in pinit0, since PMAP_LOCK_INIT() above takes care of it. markj: Certainly we must not enqueue here in pinit0, since PMAP_LOCK_INIT() above takes care of it. | |||||
if (pti) { | if (pti) { | ||||
p = td->td_proc; | p = td->td_proc; | ||||
PROC_LOCK(p); | PROC_LOCK(p); | ||||
p->p_md.md_flags |= P_MD_KPTI; | p->p_md.md_flags |= P_MD_KPTI; | ||||
PROC_UNLOCK(p); | PROC_UNLOCK(p); | ||||
} | } | ||||
pmap_thread_init_invl_gen(td); | pmap_thread_init_invl_gen(td); | ||||
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines | pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) | ||||
pmap->pm_root.rt_root = 0; | pmap->pm_root.rt_root = 0; | ||||
CPU_ZERO(&pmap->pm_active); | CPU_ZERO(&pmap->pm_active); | ||||
TAILQ_INIT(&pmap->pm_pvchunk); | TAILQ_INIT(&pmap->pm_pvchunk); | ||||
bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | bzero(&pmap->pm_stats, sizeof pmap->pm_stats); | ||||
pmap->pm_flags = flags; | pmap->pm_flags = flags; | ||||
pmap->pm_eptgen = 0; | pmap->pm_eptgen = 0; | ||||
return (1); | return (1); | ||||
} | } | ||||
Done Inline ActionsI still do not understand how this can work. pmap_pinit() is called every time a vmspace is allocated. The insertion has to be done in a vmspace zone init method instead. markj: I still do not understand how this can work. pmap_pinit() is called every time a vmspace is… | |||||
Done Inline ActionsOops, this is slightly out-of-date version of the patch. Look for pmap_lock_init(), but I uploaded the patch where I still did not removed this chunk kib: Oops, this is slightly out-of-date version of the patch. Look for pmap_lock_init(), but I… | |||||
int | int | ||||
pmap_pinit(pmap_t pmap) | pmap_pinit(pmap_t pmap) | ||||
{ | { | ||||
return (pmap_pinit_type(pmap, PT_X86, pmap_flags)); | return (pmap_pinit_type(pmap, PT_X86, pmap_flags)); | ||||
} | } | ||||
/* | /* | ||||
* This routine is called if the desired page table page does not exist. | * This routine is called if the desired page table page does not exist. | ||||
* | * | ||||
* If page table page allocation fails, this routine may sleep before | * If page table page allocation fails, this routine may sleep before | ||||
* returning NULL. It sleeps only if a lock pointer was given. | * returning NULL. It sleeps only if a lock pointer was given. | ||||
* | * | ||||
* Note: If a page allocation fails at page table level two or three, | * Note: If a page allocation fails at page table level two or three, | ||||
* one or two pages may be held during the wait, only to be released | * one or two pages may be held during the wait, only to be released | ||||
* afterwards. This conservative approach is easily argued to avoid | * afterwards. This conservative approach is easily argued to avoid | ||||
* race conditions. | * race conditions. | ||||
*/ | */ | ||||
static vm_page_t | static vm_page_t | ||||
_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) | _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, PVLL **lockp) | ||||
{ | { | ||||
vm_page_t m, pdppg, pdpg; | vm_page_t m, pdppg, pdpg; | ||||
pt_entry_t PG_A, PG_M, PG_RW, PG_V; | pt_entry_t PG_A, PG_M, PG_RW, PG_V; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PG_A = pmap_accessed_bit(pmap); | PG_A = pmap_accessed_bit(pmap); | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines | _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, PVLL **lockp) | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
return (m); | return (m); | ||||
} | } | ||||
static pd_entry_t * | static pd_entry_t * | ||||
pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, | ||||
struct rwlock **lockp) | PVLL **lockp) | ||||
{ | { | ||||
pdp_entry_t *pdpe, PG_V; | pdp_entry_t *pdpe, PG_V; | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
vm_page_t pdpg; | vm_page_t pdpg; | ||||
vm_pindex_t pdpindex; | vm_pindex_t pdpindex; | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
Show All 22 Lines | retry: | ||||
} else | } else | ||||
panic("pmap_alloc_pde: missing page table page for va %#lx", | panic("pmap_alloc_pde: missing page table page for va %#lx", | ||||
va); | va); | ||||
*pdpgp = pdpg; | *pdpgp = pdpg; | ||||
return (pde); | return (pde); | ||||
} | } | ||||
static vm_page_t | static vm_page_t | ||||
pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) | pmap_allocpte(pmap_t pmap, vm_offset_t va, PVLL **lockp) | ||||
{ | { | ||||
vm_pindex_t ptepindex; | vm_pindex_t ptepindex; | ||||
pd_entry_t *pd, PG_V; | pd_entry_t *pd, PG_V; | ||||
vm_page_t m; | vm_page_t m; | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 242 Lines • ▼ Show 20 Lines | |||||
pv_to_chunk(pv_entry_t pv) | pv_to_chunk(pv_entry_t pv) | ||||
{ | { | ||||
return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); | return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); | ||||
} | } | ||||
#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) | #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) | ||||
#define PC_FREE0 0xfffffffffffffffful | #define PC_FREE0 0xfffffffffffffffful | ||||
#define PC_FREE1 0xfffffffffffffffful | #define PC_FREE1 0xfffffffffffffffful | ||||
#define PC_FREE2 0x000000fffffffffful | #define PC_FREE2 0x000000fffffffffful | ||||
Done Inline ActionsWhy move these given that their use is localized to the code that follows? alc: Why move these given that their use is localized to the code that follows? | |||||
Done Inline ActionsI added assert in pmap_init() that uses the constants, and that comes 2K lines before this place. So I moved defines to some logical place earlier. kib: I added assert in pmap_init() that uses the constants, and that comes 2K lines before this… | |||||
static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; | static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; | ||||
#ifdef PV_STATS | #ifdef PV_STATS | ||||
static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; | static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, | SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, | ||||
"Current number of pv entry chunks"); | "Current number of pv entry chunks"); | ||||
SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, | SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, | ||||
Show All 36 Lines | |||||
* | * | ||||
* Returns NULL if PV entries were reclaimed from the specified pmap. | * Returns NULL if PV entries were reclaimed from the specified pmap. | ||||
* | * | ||||
* We do not, however, unmap 2mpages because subsequent accesses will | * We do not, however, unmap 2mpages because subsequent accesses will | ||||
* allocate per-page pv entries until repromotion occurs, thereby | * allocate per-page pv entries until repromotion occurs, thereby | ||||
* exacerbating the shortage of free pv entries. | * exacerbating the shortage of free pv entries. | ||||
*/ | */ | ||||
static vm_page_t | static vm_page_t | ||||
reclaim_pv_chunk_domain(pmap_t locked_pmap, struct rwlock **lockp, int domain) | reclaim_pv_chunk_domain(pmap_t locked_pmap, PVLL **lockp, int domain) | ||||
{ | { | ||||
struct pv_chunks_list *pvc; | struct pv_chunks_list *pvc; | ||||
struct pv_chunk *pc, *pc_marker, *pc_marker_end; | struct pv_chunk *pc, *pc_marker, *pc_marker_end; | ||||
struct pv_chunk_header pc_marker_b, pc_marker_end_b; | struct pv_chunk_header pc_marker_b, pc_marker_end_b; | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
pmap_t next_pmap, pmap; | pmap_t next_pmap, pmap; | ||||
pt_entry_t *pte, tpte; | pt_entry_t *pte, tpte; | ||||
▲ Show 20 Lines • Show All 183 Lines • ▼ Show 20 Lines | if (m_pc == NULL && !SLIST_EMPTY(&free)) { | ||||
/* Recycle a freed page table page. */ | /* Recycle a freed page table page. */ | ||||
m_pc->ref_count = 1; | m_pc->ref_count = 1; | ||||
} | } | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
return (m_pc); | return (m_pc); | ||||
} | } | ||||
static vm_page_t | static vm_page_t | ||||
reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) | reclaim_pv_chunk(pmap_t locked_pmap, PVLL **lockp) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
int i, domain; | int i, domain; | ||||
Done Inline Actions... otherwise, returns a free page to be used for a PV chunk. markj: ... otherwise, returns a free page to be used for a PV chunk. | |||||
domain = PCPU_GET(domain); | domain = PCPU_GET(domain); | ||||
for (i = 0; i < vm_ndomains; i++) { | for (i = 0; i < vm_ndomains; i++) { | ||||
m = reclaim_pv_chunk_domain(locked_pmap, lockp, domain); | m = reclaim_pv_chunk_domain(locked_pmap, lockp, domain); | ||||
if (m != NULL) | if (m != NULL) | ||||
break; | break; | ||||
domain = (domain + 1) % vm_ndomains; | domain = (domain + 1) % vm_ndomains; | ||||
} | } | ||||
Done Inline ActionsSuppose we set next_pmap = pmap = TAILQ_FIRST(&all_pmaps) in the first iteration, and before the second iteration begins, pmap is rotated to the end of the list by a concurrent reclaim_pv_chunk() call. Won't it cause the function to return early? I guess it does not matter too much since all callers will retry infinitely, but it seems worthy of a comment. markj: Suppose we set `next_pmap = pmap = TAILQ_FIRST(&all_pmaps)` in the first iteration, and before… | |||||
Done Inline ActionsThis continue needs pmap = next_pmap; before it. kib: This continue needs `pmap = next_pmap;` before it. | |||||
Done Inline ActionsThis check is racy, add a comment to that effect? markj: This check is racy, add a comment to that effect? | |||||
return (m); | return (m); | ||||
Done Inline ActionsDon't we need to lock pmap before this call? markj: Don't we need to lock `pmap` before this call? | |||||
Done Inline ActionsI think that this call is not needed, at all. The call to leave_pmap() after reclaim_handle_pmap() is enough. And then, it is easier to read code if I inline reclaim_leave_pmap(). kib: I think that this call is not needed, at all. The call to leave_pmap() after… | |||||
} | } | ||||
/* | /* | ||||
* free the pv_entry back to the free list | * free the pv_entry back to the free list | ||||
*/ | */ | ||||
static void | static void | ||||
free_pv_entry(pmap_t pmap, pv_entry_t pv) | free_pv_entry(pmap_t pmap, pv_entry_t pv) | ||||
{ | { | ||||
struct pv_chunk *pc; | struct pv_chunk *pc; | ||||
int idx, field, bit; | int idx, field, bit; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PV_STAT(atomic_add_long(&pv_entry_frees, 1)); | PV_STAT(atomic_add_long(&pv_entry_frees, 1)); | ||||
PV_STAT(atomic_add_int(&pv_entry_spare, 1)); | PV_STAT(atomic_add_int(&pv_entry_spare, 1)); | ||||
PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); | PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); | ||||
pc = pv_to_chunk(pv); | pc = pv_to_chunk(pv); | ||||
idx = pv - &pc->pc_pventry[0]; | idx = pv - &pc->pc_pventry[0]; | ||||
field = idx / 64; | field = idx / 64; | ||||
bit = idx % 64; | bit = idx % 64; | ||||
pc->pc_map[field] |= 1ul << bit; | pc->pc_map[field] |= 1ul << bit; | ||||
if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 || | if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 || | ||||
Done Inline ActionsWhy do we invalidate if resident_count == 0? markj: Why do we invalidate if resident_count == 0? | |||||
Done Inline ActionsWe potentially would exit di region on the next loop iteration. We must invalidate then, even if this function did not changed any valid mapping. kib: We potentially would exit di region on the next loop iteration. We must invalidate then, even… | |||||
pc->pc_map[2] != PC_FREE2) { | pc->pc_map[2] != PC_FREE2) { | ||||
/* 98% of the time, pc is already at the head of the list. */ | /* 98% of the time, pc is already at the head of the list. */ | ||||
if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { | if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { | ||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | ||||
TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); | TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); | ||||
} | } | ||||
return; | return; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | |||||
* Returns a new PV entry, allocating a new PV chunk from the system when | * Returns a new PV entry, allocating a new PV chunk from the system when | ||||
* needed. If this PV chunk allocation fails and a PV list lock pointer was | * needed. If this PV chunk allocation fails and a PV list lock pointer was | ||||
* given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is | * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is | ||||
* returned. | * returned. | ||||
* | * | ||||
* The given PV list lock may be released. | * The given PV list lock may be released. | ||||
*/ | */ | ||||
static pv_entry_t | static pv_entry_t | ||||
get_pv_entry(pmap_t pmap, struct rwlock **lockp) | get_pv_entry(pmap_t pmap, PVLL **lockp) | ||||
{ | { | ||||
struct pv_chunks_list *pvc; | struct pv_chunks_list *pvc; | ||||
int bit, field; | int bit, field; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
struct pv_chunk *pc; | struct pv_chunk *pc; | ||||
vm_page_t m; | vm_page_t m; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Ensure that the number of spare PV entries in the specified pmap meets or | * Ensure that the number of spare PV entries in the specified pmap meets or | ||||
* exceeds the given count, "needed". | * exceeds the given count, "needed". | ||||
* | * | ||||
* The given PV list lock may be released. | * The given PV list lock may be released. | ||||
*/ | */ | ||||
static void | static void | ||||
reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) | reserve_pv_entries(pmap_t pmap, int needed, PVLL **lockp) | ||||
{ | { | ||||
struct pv_chunks_list *pvc; | struct pv_chunks_list *pvc; | ||||
struct pch new_tail[PMAP_MEMDOM]; | struct pch new_tail[PMAP_MEMDOM]; | ||||
struct pv_chunk *pc; | struct pv_chunk *pc; | ||||
vm_page_t m; | vm_page_t m; | ||||
int avail, free, i; | int avail, free, i; | ||||
bool reclaimed; | bool reclaimed; | ||||
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* After demotion from a 2MB page mapping to 512 4KB page mappings, | * After demotion from a 2MB page mapping to 512 4KB page mappings, | ||||
* destroy the pv entry for the 2MB page mapping and reinstantiate the pv | * destroy the pv entry for the 2MB page mapping and reinstantiate the pv | ||||
* entries for each of the 4KB page mappings. | * entries for each of the 4KB page mappings. | ||||
*/ | */ | ||||
static void | static void | ||||
pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | ||||
struct rwlock **lockp) | PVLL **lockp) | ||||
Done Inline ActionsIs lockp still used? alc: Is `lockp` still used? | |||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
struct pv_chunk *pc; | struct pv_chunk *pc; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
vm_offset_t va_last; | vm_offset_t va_last; | ||||
vm_page_t m; | vm_page_t m; | ||||
int bit, field; | int bit, field; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
KASSERT((pa & PDRMASK) == 0, | KASSERT((pa & PDRMASK) == 0, | ||||
("pmap_pv_demote_pde: pa is not 2mpage aligned")); | ("pmap_pv_demote_pde: pa is not 2mpage aligned")); | ||||
CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); | |||||
/* | /* | ||||
* Transfer the 2mpage's pv entry for this mapping to the first | * Transfer the 2mpage's pv entry for this mapping to the first | ||||
* page's pv list. Once this transfer begins, the pv list lock | * page's pv list. Once this transfer begins, the pv list lock | ||||
* must not be released until the last pv entry is reinstantiated. | * must not be released until the last pv entry is reinstantiated. | ||||
*/ | */ | ||||
pvh = pa_to_pvh(pa); | pvh = pa_to_pvh(pa); | ||||
va = trunc_2mpage(va); | va = trunc_2mpage(va); | ||||
Show All 40 Lines | |||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
/* | /* | ||||
* After promotion from 512 4KB page mappings to a single 2MB page mapping, | * After promotion from 512 4KB page mappings to a single 2MB page mapping, | ||||
* replace the many pv entries for the 4KB page mappings by a single pv entry | * replace the many pv entries for the 4KB page mappings by a single pv entry | ||||
* for the 2MB page mapping. | * for the 2MB page mapping. | ||||
*/ | */ | ||||
static void | static void | ||||
pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | ||||
struct rwlock **lockp) | PVLL **lockp) | ||||
Done Inline ActionsIs lockp still used? alc: Is `lockp` still used? | |||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
vm_offset_t va_last; | vm_offset_t va_last; | ||||
vm_page_t m; | vm_page_t m; | ||||
KASSERT((pa & PDRMASK) == 0, | KASSERT((pa & PDRMASK) == 0, | ||||
("pmap_pv_promote_pde: pa is not 2mpage aligned")); | ("pmap_pv_promote_pde: pa is not 2mpage aligned")); | ||||
CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); | |||||
/* | /* | ||||
* Transfer the first page's pv entry for this mapping to the 2mpage's | * Transfer the first page's pv entry for this mapping to the 2mpage's | ||||
* pv list. Aside from avoiding the cost of a call to get_pv_entry(), | * pv list. Aside from avoiding the cost of a call to get_pv_entry(), | ||||
* a transfer avoids the possibility that get_pv_entry() calls | * a transfer avoids the possibility that get_pv_entry() calls | ||||
* reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the | * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the | ||||
* mappings that is being promoted. | * mappings that is being promoted. | ||||
*/ | */ | ||||
Show All 30 Lines | |||||
} | } | ||||
/* | /* | ||||
* Conditionally create the PV entry for a 4KB page mapping if the required | * Conditionally create the PV entry for a 4KB page mapping if the required | ||||
* memory can be allocated without resorting to reclamation. | * memory can be allocated without resorting to reclamation. | ||||
*/ | */ | ||||
static boolean_t | static boolean_t | ||||
pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, | pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
struct rwlock **lockp) | PVLL **lockp) | ||||
{ | { | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
/* Pass NULL instead of the lock pointer to disable reclamation. */ | /* Pass NULL instead of the lock pointer to disable reclamation. */ | ||||
if ((pv = get_pv_entry(pmap, NULL)) != NULL) { | if ((pv = get_pv_entry(pmap, NULL)) != NULL) { | ||||
pv->pv_va = va; | pv->pv_va = va; | ||||
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); | CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); | ||||
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
return (TRUE); | return (TRUE); | ||||
} else | } else | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
/* | /* | ||||
* Create the PV entry for a 2MB page mapping. Always returns true unless the | * Create the PV entry for a 2MB page mapping. Always returns true unless the | ||||
* flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns | * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns | ||||
* false if the PV entry cannot be allocated without resorting to reclamation. | * false if the PV entry cannot be allocated without resorting to reclamation. | ||||
*/ | */ | ||||
static bool | static bool | ||||
pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags, | pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags, | ||||
struct rwlock **lockp) | PVLL **lockp) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
vm_paddr_t pa; | vm_paddr_t pa; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
/* Pass NULL instead of the lock pointer to disable reclamation. */ | /* Pass NULL instead of the lock pointer to disable reclamation. */ | ||||
if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? | if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? | ||||
Show All 24 Lines | |||||
/* | /* | ||||
* Tries to demote a 2MB page mapping. If demotion fails, the 2MB page | * Tries to demote a 2MB page mapping. If demotion fails, the 2MB page | ||||
* mapping is invalidated. | * mapping is invalidated. | ||||
*/ | */ | ||||
static boolean_t | static boolean_t | ||||
pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) | pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
boolean_t rv; | boolean_t rv; | ||||
lock = NULL; | lock = NULL; | ||||
rv = pmap_demote_pde_locked(pmap, pde, va, &lock); | rv = pmap_demote_pde_locked(pmap, pde, va, &lock); | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
static void | static void | ||||
pmap_demote_pde_check(pt_entry_t *firstpte __unused, pt_entry_t newpte __unused) | pmap_demote_pde_check(pt_entry_t *firstpte __unused, pt_entry_t newpte __unused) | ||||
{ | { | ||||
#ifdef INVARIANTS | #ifdef INVARIANTS | ||||
#ifdef DIAGNOSTIC | #ifdef DIAGNOSTIC | ||||
Show All 16 Lines | KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME), | ||||
("pmap_demote_pde: firstpte and newpte map different physical" | ("pmap_demote_pde: firstpte and newpte map different physical" | ||||
" addresses")); | " addresses")); | ||||
#endif | #endif | ||||
#endif | #endif | ||||
} | } | ||||
static void | static void | ||||
pmap_demote_pde_abort(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | pmap_demote_pde_abort(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | ||||
pd_entry_t oldpde, struct rwlock **lockp) | pd_entry_t oldpde, PVLL **lockp) | ||||
{ | { | ||||
struct spglist free; | struct spglist free; | ||||
vm_offset_t sva; | vm_offset_t sva; | ||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
sva = trunc_2mpage(va); | sva = trunc_2mpage(va); | ||||
pmap_remove_pde(pmap, pde, sva, &free, lockp); | pmap_remove_pde(pmap, pde, sva, &free, lockp); | ||||
if ((oldpde & pmap_global_bit(pmap)) == 0) | if ((oldpde & pmap_global_bit(pmap)) == 0) | ||||
pmap_invalidate_pde_page(pmap, sva, oldpde); | pmap_invalidate_pde_page(pmap, sva, oldpde); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx in pmap %p", | CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx in pmap %p", | ||||
va, pmap); | va, pmap); | ||||
} | } | ||||
static boolean_t | static boolean_t | ||||
pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, | pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, | ||||
struct rwlock **lockp) | PVLL **lockp) | ||||
{ | { | ||||
pd_entry_t newpde, oldpde; | pd_entry_t newpde, oldpde; | ||||
pt_entry_t *firstpte, newpte; | pt_entry_t *firstpte, newpte; | ||||
pt_entry_t PG_A, PG_G, PG_M, PG_PKU_MASK, PG_RW, PG_V; | pt_entry_t PG_A, PG_G, PG_M, PG_PKU_MASK, PG_RW, PG_V; | ||||
vm_paddr_t mptepa; | vm_paddr_t mptepa; | ||||
vm_page_t mpte; | vm_page_t mpte; | ||||
int PG_PTE_CACHE; | int PG_PTE_CACHE; | ||||
bool in_kernel; | bool in_kernel; | ||||
▲ Show 20 Lines • Show All 117 Lines • ▼ Show 20 Lines | pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, | ||||
* Invalidate a stale recursive mapping of the page table page. | * Invalidate a stale recursive mapping of the page table page. | ||||
*/ | */ | ||||
if (in_kernel) | if (in_kernel) | ||||
pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va)); | pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va)); | ||||
/* | /* | ||||
* Demote the PV entry. | * Demote the PV entry. | ||||
*/ | */ | ||||
if ((oldpde & PG_MANAGED) != 0) | if ((oldpde & PG_MANAGED) != 0) { | ||||
pmap_pv_list_lock_pde(oldpde & PG_PS_FRAME, lockp); | |||||
pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp); | pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp); | ||||
pmap_pv_list_unlock_pde(oldpde & PG_PS_FRAME, lockp); | |||||
} | |||||
atomic_add_long(&pmap_pde_demotions, 1); | atomic_add_long(&pmap_pde_demotions, 1); | ||||
CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx in pmap %p", | CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx in pmap %p", | ||||
va, pmap); | va, pmap); | ||||
return (TRUE); | return (TRUE); | ||||
} | } | ||||
/* | /* | ||||
Show All 36 Lines | pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) | ||||
pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va)); | pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va)); | ||||
} | } | ||||
/* | /* | ||||
* pmap_remove_pde: do the things to unmap a superpage in a process | * pmap_remove_pde: do the things to unmap a superpage in a process | ||||
*/ | */ | ||||
static int | static int | ||||
pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, | pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, | ||||
struct spglist *free, struct rwlock **lockp) | struct spglist *free, PVLL **lockp) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pd_entry_t oldpde; | pd_entry_t oldpde; | ||||
vm_offset_t eva, va; | vm_offset_t eva, va; | ||||
vm_page_t m, mpte; | vm_page_t m, mpte; | ||||
pt_entry_t PG_G, PG_A, PG_M, PG_RW; | pt_entry_t PG_G, PG_A, PG_M, PG_RW; | ||||
PG_G = pmap_global_bit(pmap); | PG_G = pmap_global_bit(pmap); | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, | ||||
return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free)); | return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free)); | ||||
} | } | ||||
/* | /* | ||||
* pmap_remove_pte: do the things to unmap a page in a process | * pmap_remove_pte: do the things to unmap a page in a process | ||||
*/ | */ | ||||
static int | static int | ||||
pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, | pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, | ||||
pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp) | pd_entry_t ptepde, struct spglist *free, PVLL **lockp) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pt_entry_t oldpte, PG_A, PG_M, PG_RW; | pt_entry_t oldpte, PG_A, PG_M, PG_RW; | ||||
vm_page_t m; | vm_page_t m; | ||||
PG_A = pmap_accessed_bit(pmap); | PG_A = pmap_accessed_bit(pmap); | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
Show All 24 Lines | |||||
/* | /* | ||||
* Remove a single page from a process address space | * Remove a single page from a process address space | ||||
*/ | */ | ||||
static void | static void | ||||
pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, | ||||
struct spglist *free) | struct spglist *free) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
pt_entry_t *pte, PG_V; | pt_entry_t *pte, PG_V; | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
if ((*pde & PG_V) == 0) | if ((*pde & PG_V) == 0) | ||||
return; | return; | ||||
pte = pmap_pde_to_pte(pde, va); | pte = pmap_pde_to_pte(pde, va); | ||||
if ((*pte & PG_V) == 0) | if ((*pte & PG_V) == 0) | ||||
return; | return; | ||||
lock = NULL; | lock = NULL; | ||||
pmap_remove_pte(pmap, pte, va, *pde, free, &lock); | pmap_remove_pte(pmap, pte, va, *pde, free, &lock); | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
pmap_invalidate_page(pmap, va); | pmap_invalidate_page(pmap, va); | ||||
} | } | ||||
/* | /* | ||||
* Removes the specified range of addresses from the page table page. | * Removes the specified range of addresses from the page table page. | ||||
*/ | */ | ||||
static bool | static bool | ||||
pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, | pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, | ||||
pd_entry_t *pde, struct spglist *free, struct rwlock **lockp) | pd_entry_t *pde, struct spglist *free, PVLL **lockp) | ||||
{ | { | ||||
pt_entry_t PG_G, *pte; | pt_entry_t PG_G, *pte; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
bool anyvalid; | bool anyvalid; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PG_G = pmap_global_bit(pmap); | PG_G = pmap_global_bit(pmap); | ||||
anyvalid = false; | anyvalid = false; | ||||
Show All 25 Lines | |||||
* Remove the given range of addresses from the specified map. | * Remove the given range of addresses from the specified map. | ||||
* | * | ||||
* It is assumed that the start and end are properly | * It is assumed that the start and end are properly | ||||
* rounded to the page size. | * rounded to the page size. | ||||
*/ | */ | ||||
void | void | ||||
pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
vm_offset_t va_next; | vm_offset_t va_next; | ||||
pml4_entry_t *pml4e; | pml4_entry_t *pml4e; | ||||
pdp_entry_t *pdpe; | pdp_entry_t *pdpe; | ||||
pd_entry_t ptpaddr, *pde; | pd_entry_t ptpaddr, *pde; | ||||
pt_entry_t PG_G, PG_V; | pt_entry_t PG_G, PG_V; | ||||
struct spglist free; | struct spglist free; | ||||
int anyvalid; | int anyvalid; | ||||
▲ Show 20 Lines • Show All 96 Lines • ▼ Show 20 Lines | for (; sva < eva; sva = va_next) { | ||||
*/ | */ | ||||
if (va_next > eva) | if (va_next > eva) | ||||
va_next = eva; | va_next = eva; | ||||
if (pmap_remove_ptes(pmap, sva, va_next, pde, &free, &lock)) | if (pmap_remove_ptes(pmap, sva, va_next, pde, &free, &lock)) | ||||
anyvalid = 1; | anyvalid = 1; | ||||
} | } | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
out: | out: | ||||
if (anyvalid) | if (anyvalid) | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
pmap_delayed_invl_finish(); | pmap_delayed_invl_finish(); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
Show All 11 Lines | |||||
*/ | */ | ||||
void | void | ||||
pmap_remove_all(vm_page_t m) | pmap_remove_all(vm_page_t m) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
pmap_t pmap; | pmap_t pmap; | ||||
struct rwlock *lock; | PVLL *lock; | ||||
pt_entry_t *pte, tpte, PG_A, PG_M, PG_RW; | pt_entry_t *pte, tpte, PG_A, PG_M, PG_RW; | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
struct spglist free; | struct spglist free; | ||||
int pvh_gen, md_gen; | int pvh_gen, md_gen; | ||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | KASSERT((m->oflags & VPO_UNMANAGED) == 0, | ||||
("pmap_remove_all: page %p is not managed", m)); | ("pmap_remove_all: page %p is not managed", m)); | ||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | lock = VM_PAGE_TO_PV_LIST_LOCK(m); | ||||
pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : | pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : | ||||
pa_to_pvh(VM_PAGE_TO_PHYS(m)); | pa_to_pvh(VM_PAGE_TO_PHYS(m)); | ||||
retry: | retry: | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { | while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
if (pvh_gen != pvh->pv_gen) { | if (pvh_gen != pvh->pv_gen) { | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto retry; | goto retry; | ||||
} | } | ||||
} | } | ||||
va = pv->pv_va; | va = pv->pv_va; | ||||
pde = pmap_pde(pmap, va); | pde = pmap_pde(pmap, va); | ||||
(void)pmap_demote_pde_locked(pmap, pde, va, &lock); | (void)pmap_demote_pde_locked(pmap, pde, va, &lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { | while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { | if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto retry; | goto retry; | ||||
} | } | ||||
} | } | ||||
PG_A = pmap_accessed_bit(pmap); | PG_A = pmap_accessed_bit(pmap); | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
pmap_resident_count_dec(pmap, 1); | pmap_resident_count_dec(pmap, 1); | ||||
Show All 15 Lines | while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { | ||||
pmap_unuse_pt(pmap, pv->pv_va, *pde, &free); | pmap_unuse_pt(pmap, pv->pv_va, *pde, &free); | ||||
pmap_invalidate_page(pmap, pv->pv_va); | pmap_invalidate_page(pmap, pv->pv_va); | ||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
free_pv_entry(pmap, pv); | free_pv_entry(pmap, pv); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
vm_page_aflag_clear(m, PGA_WRITEABLE); | vm_page_aflag_clear(m, PGA_WRITEABLE); | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
pmap_delayed_invl_wait(m); | pmap_delayed_invl_wait(m); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
/* | /* | ||||
* pmap_protect_pde: do the things to protect a 2mpage in a process | * pmap_protect_pde: do the things to protect a 2mpage in a process | ||||
*/ | */ | ||||
static boolean_t | static boolean_t | ||||
▲ Show 20 Lines • Show All 199 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Tries to promote the 512, contiguous 4KB page mappings that are within a | * Tries to promote the 512, contiguous 4KB page mappings that are within a | ||||
* single page table page (PTP) to a single 2MB page mapping. For promotion | * single page table page (PTP) to a single 2MB page mapping. For promotion | ||||
* to occur, two conditions must be met: (1) the 4KB page mappings must map | * to occur, two conditions must be met: (1) the 4KB page mappings must map | ||||
* aligned, contiguous physical memory and (2) the 4KB page mappings must have | * aligned, contiguous physical memory and (2) the 4KB page mappings must have | ||||
* identical characteristics. | * identical characteristics. | ||||
*/ | */ | ||||
static void | static void | ||||
pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, | pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, PVLL **lockp) | ||||
struct rwlock **lockp) | |||||
{ | { | ||||
pd_entry_t newpde; | pd_entry_t newpde; | ||||
pt_entry_t *firstpte, oldpte, pa, *pte; | pt_entry_t *firstpte, oldpte, pa, *pte; | ||||
pt_entry_t PG_G, PG_A, PG_M, PG_RW, PG_V, PG_PKU_MASK; | pt_entry_t PG_G, PG_A, PG_M, PG_RW, PG_V, PG_PKU_MASK; | ||||
vm_page_t mpte; | vm_page_t mpte; | ||||
int PG_PTE_CACHE; | int PG_PTE_CACHE; | ||||
PG_A = pmap_accessed_bit(pmap); | PG_A = pmap_accessed_bit(pmap); | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | CTR2(KTR_PMAP, | ||||
"pmap_promote_pde: failure for va %#lx in pmap %p", va, | "pmap_promote_pde: failure for va %#lx in pmap %p", va, | ||||
pmap); | pmap); | ||||
return; | return; | ||||
} | } | ||||
/* | /* | ||||
* Promote the pv entries. | * Promote the pv entries. | ||||
*/ | */ | ||||
if ((newpde & PG_MANAGED) != 0) | if ((newpde & PG_MANAGED) != 0) { | ||||
pmap_pv_list_lock_pde(newpde & PG_PS_FRAME, lockp); | |||||
pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp); | pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp); | ||||
pmap_pv_list_unlock_pde(newpde & PG_PS_FRAME, lockp); | |||||
} | |||||
/* | /* | ||||
* Propagate the PAT index to its proper position. | * Propagate the PAT index to its proper position. | ||||
*/ | */ | ||||
newpde = pmap_swap_pat(pmap, newpde); | newpde = pmap_swap_pat(pmap, newpde); | ||||
/* | /* | ||||
* Map the superpage. | * Map the superpage. | ||||
Show All 24 Lines | |||||
* When destroying both a page table and PV entry, this function | * When destroying both a page table and PV entry, this function | ||||
* performs the TLB invalidation before releasing the PV list | * performs the TLB invalidation before releasing the PV list | ||||
* lock, so we do not need pmap_delayed_invl_page() calls here. | * lock, so we do not need pmap_delayed_invl_page() calls here. | ||||
*/ | */ | ||||
int | int | ||||
pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
u_int flags, int8_t psind) | u_int flags, int8_t psind) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
pt_entry_t *pte, PG_G, PG_A, PG_M, PG_RW, PG_V; | pt_entry_t *pte, PG_G, PG_A, PG_M, PG_RW, PG_V; | ||||
pt_entry_t newpte, origpte; | pt_entry_t newpte, origpte; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
vm_paddr_t opa, pa; | vm_paddr_t opa, pa; | ||||
vm_page_t mpte, om; | vm_page_t mpte, om; | ||||
int rv; | int rv; | ||||
boolean_t nosleep; | boolean_t nosleep; | ||||
▲ Show 20 Lines • Show All 189 Lines • ▼ Show 20 Lines | retry: | ||||
/* | /* | ||||
* Enter on the PV list if part of our managed memory. | * Enter on the PV list if part of our managed memory. | ||||
*/ | */ | ||||
if ((newpte & PG_MANAGED) != 0) { | if ((newpte & PG_MANAGED) != 0) { | ||||
if (pv == NULL) { | if (pv == NULL) { | ||||
pv = get_pv_entry(pmap, &lock); | pv = get_pv_entry(pmap, &lock); | ||||
pv->pv_va = va; | pv->pv_va = va; | ||||
} | } | ||||
CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); | CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); | ||||
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
if ((newpte & PG_RW) != 0) | if ((newpte & PG_RW) != 0) | ||||
vm_page_aflag_set(m, PGA_WRITEABLE); | vm_page_aflag_set(m, PGA_WRITEABLE); | ||||
} | } | ||||
/* | /* | ||||
* Update the PTE. | * Update the PTE. | ||||
Show All 36 Lines | if ((mpte == NULL || mpte->ref_count == NPTEPG) && | ||||
(m->flags & PG_FICTITIOUS) == 0 && | (m->flags & PG_FICTITIOUS) == 0 && | ||||
vm_reserv_level_iffullpop(m) == 0) | vm_reserv_level_iffullpop(m) == 0) | ||||
pmap_promote_pde(pmap, pde, va, &lock); | pmap_promote_pde(pmap, pde, va, &lock); | ||||
#endif | #endif | ||||
rv = KERN_SUCCESS; | rv = KERN_SUCCESS; | ||||
out: | out: | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
* Tries to create a read- and/or execute-only 2MB page mapping. Returns true | * Tries to create a read- and/or execute-only 2MB page mapping. Returns true | ||||
* if successful. Returns false if (1) a page table page cannot be allocated | * if successful. Returns false if (1) a page table page cannot be allocated | ||||
* without sleeping, (2) a mapping already exists at the specified virtual | * without sleeping, (2) a mapping already exists at the specified virtual | ||||
* address, or (3) a PV entry cannot be allocated without reclaiming another | * address, or (3) a PV entry cannot be allocated without reclaiming another | ||||
* PV entry. | * PV entry. | ||||
*/ | */ | ||||
static bool | static bool | ||||
pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
struct rwlock **lockp) | PVLL **lockp) | ||||
{ | { | ||||
pd_entry_t newpde; | pd_entry_t newpde; | ||||
pt_entry_t PG_V; | pt_entry_t PG_V; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(pmap, m->md.pat_mode, 1) | | newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(pmap, m->md.pat_mode, 1) | | ||||
PG_PS | PG_V; | PG_PS | PG_V; | ||||
Show All 34 Lines | |||||
* KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table | * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table | ||||
* page allocation failed. Returns KERN_RESOURCE_SHORTAGE if | * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if | ||||
* PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. | * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. | ||||
* | * | ||||
* The parameter "m" is only used when creating a managed, writeable mapping. | * The parameter "m" is only used when creating a managed, writeable mapping. | ||||
*/ | */ | ||||
static int | static int | ||||
pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags, | pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags, | ||||
vm_page_t m, struct rwlock **lockp) | vm_page_t m, PVLL **lockp) | ||||
{ | { | ||||
struct spglist free; | struct spglist free; | ||||
pd_entry_t oldpde, *pde; | pd_entry_t oldpde, *pde; | ||||
pt_entry_t PG_G, PG_RW, PG_V; | pt_entry_t PG_G, PG_RW, PG_V; | ||||
vm_page_t mt, pdpg; | vm_page_t mt, pdpg; | ||||
KASSERT(pmap == kernel_pmap || (newpde & PG_W) == 0, | KASSERT(pmap == kernel_pmap || (newpde & PG_W) == 0, | ||||
("pmap_enter_pde: cannot create wired user mapping")); | ("pmap_enter_pde: cannot create wired user mapping")); | ||||
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | |||||
* virtual address end. Not every virtual page between start and end | * virtual address end. Not every virtual page between start and end | ||||
* is mapped; only those for which a resident page exists with the | * is mapped; only those for which a resident page exists with the | ||||
* corresponding offset from m_start are mapped. | * corresponding offset from m_start are mapped. | ||||
*/ | */ | ||||
void | void | ||||
pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, | pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, | ||||
vm_page_t m_start, vm_prot_t prot) | vm_page_t m_start, vm_prot_t prot) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
vm_page_t m, mpte; | vm_page_t m, mpte; | ||||
vm_pindex_t diff, psize; | vm_pindex_t diff, psize; | ||||
VM_OBJECT_ASSERT_LOCKED(m_start->object); | VM_OBJECT_ASSERT_LOCKED(m_start->object); | ||||
psize = atop(end - start); | psize = atop(end - start); | ||||
mpte = NULL; | mpte = NULL; | ||||
m = m_start; | m = m_start; | ||||
lock = NULL; | lock = NULL; | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { | while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { | ||||
va = start + ptoa(diff); | va = start + ptoa(diff); | ||||
if ((va & PDRMASK) == 0 && va + NBPDR <= end && | if ((va & PDRMASK) == 0 && va + NBPDR <= end && | ||||
m->psind == 1 && pmap_ps_enabled(pmap) && | m->psind == 1 && pmap_ps_enabled(pmap) && | ||||
pmap_allow_2m_x_page(pmap, (prot & VM_PROT_EXECUTE) != 0) && | pmap_allow_2m_x_page(pmap, (prot & VM_PROT_EXECUTE) != 0) && | ||||
pmap_enter_2mpage(pmap, va, m, prot, &lock)) | pmap_enter_2mpage(pmap, va, m, prot, &lock)) | ||||
m = &m[NBPDR / PAGE_SIZE - 1]; | m = &m[NBPDR / PAGE_SIZE - 1]; | ||||
else | else | ||||
mpte = pmap_enter_quick_locked(pmap, va, m, prot, | mpte = pmap_enter_quick_locked(pmap, va, m, prot, | ||||
mpte, &lock); | mpte, &lock); | ||||
m = TAILQ_NEXT(m, listq); | m = TAILQ_NEXT(m, listq); | ||||
} | } | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
/* | /* | ||||
* this code makes some *MAJOR* assumptions: | * this code makes some *MAJOR* assumptions: | ||||
* 1. Current pmap & pmap exists. | * 1. Current pmap & pmap exists. | ||||
* 2. Not wired. | * 2. Not wired. | ||||
* 3. Read access. | * 3. Read access. | ||||
* 4. No page table pages. | * 4. No page table pages. | ||||
* but is *MUCH* faster than pmap_enter... | * but is *MUCH* faster than pmap_enter... | ||||
*/ | */ | ||||
void | void | ||||
pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) | pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
lock = NULL; | lock = NULL; | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
(void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); | (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
static vm_page_t | static vm_page_t | ||||
pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, | pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp) | vm_prot_t prot, vm_page_t mpte, PVLL **lockp) | ||||
{ | { | ||||
pt_entry_t newpte, *pte, PG_V; | pt_entry_t newpte, *pte, PG_V; | ||||
KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || | KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || | ||||
(m->oflags & VPO_UNMANAGED) != 0, | (m->oflags & VPO_UNMANAGED) != 0, | ||||
("pmap_enter_quick_locked: managed mapping within the clean submap")); | ("pmap_enter_quick_locked: managed mapping within the clean submap")); | ||||
PG_V = pmap_valid_bit(pmap); | PG_V = pmap_valid_bit(pmap); | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
▲ Show 20 Lines • Show All 277 Lines • ▼ Show 20 Lines | |||||
* in the destination map. | * in the destination map. | ||||
* | * | ||||
* This routine is only advisory and need not do anything. | * This routine is only advisory and need not do anything. | ||||
*/ | */ | ||||
void | void | ||||
pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, | pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, | ||||
vm_offset_t src_addr) | vm_offset_t src_addr) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
pml4_entry_t *pml4e; | pml4_entry_t *pml4e; | ||||
pdp_entry_t *pdpe; | pdp_entry_t *pdpe; | ||||
pd_entry_t *pde, srcptepaddr; | pd_entry_t *pde, srcptepaddr; | ||||
pt_entry_t *dst_pte, PG_A, PG_M, PG_V, ptetemp, *src_pte; | pt_entry_t *dst_pte, PG_A, PG_M, PG_V, ptetemp, *src_pte; | ||||
vm_offset_t addr, end_addr, va_next; | vm_offset_t addr, end_addr, va_next; | ||||
vm_page_t dst_pdpg, dstmpte, srcmpte; | vm_page_t dst_pdpg, dstmpte, srcmpte; | ||||
if (dst_addr != src_addr) | if (dst_addr != src_addr) | ||||
▲ Show 20 Lines • Show All 120 Lines • ▼ Show 20 Lines | for (; addr < va_next; addr += PAGE_SIZE, src_pte++) { | ||||
} | } | ||||
/* Have we copied all of the valid mappings? */ | /* Have we copied all of the valid mappings? */ | ||||
if (dstmpte->ref_count >= srcmpte->ref_count) | if (dstmpte->ref_count >= srcmpte->ref_count) | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
out: | out: | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_UNLOCK(src_pmap); | PMAP_UNLOCK(src_pmap); | ||||
PMAP_UNLOCK(dst_pmap); | PMAP_UNLOCK(dst_pmap); | ||||
} | } | ||||
int | int | ||||
pmap_vmspace_copy(pmap_t dst_pmap, pmap_t src_pmap) | pmap_vmspace_copy(pmap_t dst_pmap, pmap_t src_pmap) | ||||
{ | { | ||||
int error; | int error; | ||||
▲ Show 20 Lines • Show All 98 Lines • ▼ Show 20 Lines | |||||
* be changed upwards or downwards in the future; it | * be changed upwards or downwards in the future; it | ||||
* is only necessary that true be returned for a small | * is only necessary that true be returned for a small | ||||
* subset of pmaps for proper page aging. | * subset of pmaps for proper page aging. | ||||
*/ | */ | ||||
boolean_t | boolean_t | ||||
pmap_page_exists_quick(pmap_t pmap, vm_page_t m) | pmap_page_exists_quick(pmap_t pmap, vm_page_t m) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
struct rwlock *lock; | PVLL *lock; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
int loops = 0; | int loops = 0; | ||||
boolean_t rv; | boolean_t rv; | ||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | KASSERT((m->oflags & VPO_UNMANAGED) == 0, | ||||
("pmap_page_exists_quick: page %p is not managed", m)); | ("pmap_page_exists_quick: page %p is not managed", m)); | ||||
rv = FALSE; | rv = FALSE; | ||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | lock = VM_PAGE_TO_PV_LIST_LOCK(m); | ||||
rw_rlock(lock); | pmap_pv_list_lock(lock); | ||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | ||||
if (PV_PMAP(pv) == pmap) { | if (PV_PMAP(pv) == pmap) { | ||||
rv = TRUE; | rv = TRUE; | ||||
break; | break; | ||||
} | } | ||||
loops++; | loops++; | ||||
if (loops >= 16) | if (loops >= 16) | ||||
break; | break; | ||||
} | } | ||||
if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { | if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { | ||||
pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); | pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); | ||||
TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { | TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { | ||||
if (PV_PMAP(pv) == pmap) { | if (PV_PMAP(pv) == pmap) { | ||||
rv = TRUE; | rv = TRUE; | ||||
break; | break; | ||||
} | } | ||||
loops++; | loops++; | ||||
if (loops >= 16) | if (loops >= 16) | ||||
break; | break; | ||||
} | } | ||||
} | } | ||||
rw_runlock(lock); | pmap_pv_list_unlock(lock); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
* pmap_page_wired_mappings: | * pmap_page_wired_mappings: | ||||
* | * | ||||
* Return the number of managed mappings to the given physical page | * Return the number of managed mappings to the given physical page | ||||
* that are wired. | * that are wired. | ||||
*/ | */ | ||||
int | int | ||||
pmap_page_wired_mappings(vm_page_t m) | pmap_page_wired_mappings(vm_page_t m) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pmap_t pmap; | pmap_t pmap; | ||||
pt_entry_t *pte; | pt_entry_t *pte; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
int count, md_gen, pvh_gen; | int count, md_gen, pvh_gen; | ||||
if ((m->oflags & VPO_UNMANAGED) != 0) | if ((m->oflags & VPO_UNMANAGED) != 0) | ||||
return (0); | return (0); | ||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | lock = VM_PAGE_TO_PV_LIST_LOCK(m); | ||||
rw_rlock(lock); | pmap_pv_list_lock(lock); | ||||
restart: | restart: | ||||
count = 0; | count = 0; | ||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
rw_runlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_rlock(lock); | pmap_pv_list_lock(lock); | ||||
if (md_gen != m->md.pv_gen) { | if (md_gen != m->md.pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto restart; | goto restart; | ||||
} | } | ||||
} | } | ||||
pte = pmap_pte(pmap, pv->pv_va); | pte = pmap_pte(pmap, pv->pv_va); | ||||
if ((*pte & PG_W) != 0) | if ((*pte & PG_W) != 0) | ||||
count++; | count++; | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
if ((m->flags & PG_FICTITIOUS) == 0) { | if ((m->flags & PG_FICTITIOUS) == 0) { | ||||
pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); | pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); | ||||
TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { | TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
rw_runlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_rlock(lock); | pmap_pv_list_lock(lock); | ||||
if (md_gen != m->md.pv_gen || | if (md_gen != m->md.pv_gen || | ||||
pvh_gen != pvh->pv_gen) { | pvh_gen != pvh->pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto restart; | goto restart; | ||||
} | } | ||||
} | } | ||||
pte = pmap_pde(pmap, pv->pv_va); | pte = pmap_pde(pmap, pv->pv_va); | ||||
if ((*pte & PG_W) != 0) | if ((*pte & PG_W) != 0) | ||||
count++; | count++; | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
} | } | ||||
rw_runlock(lock); | pmap_pv_list_unlock(lock); | ||||
return (count); | return (count); | ||||
} | } | ||||
/* | /* | ||||
* Returns TRUE if the given page is mapped individually or as part of | * Returns TRUE if the given page is mapped individually or as part of | ||||
* a 2mpage. Otherwise, returns FALSE. | * a 2mpage. Otherwise, returns FALSE. | ||||
*/ | */ | ||||
boolean_t | boolean_t | ||||
pmap_page_is_mapped(vm_page_t m) | pmap_page_is_mapped(vm_page_t m) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
boolean_t rv; | boolean_t rv; | ||||
if ((m->oflags & VPO_UNMANAGED) != 0) | if ((m->oflags & VPO_UNMANAGED) != 0) | ||||
return (FALSE); | return (FALSE); | ||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | lock = VM_PAGE_TO_PV_LIST_LOCK(m); | ||||
rw_rlock(lock); | pmap_pv_list_lock(lock); | ||||
rv = !TAILQ_EMPTY(&m->md.pv_list) || | rv = !TAILQ_EMPTY(&m->md.pv_list) || | ||||
((m->flags & PG_FICTITIOUS) == 0 && | ((m->flags & PG_FICTITIOUS) == 0 && | ||||
!TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); | !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); | ||||
rw_runlock(lock); | pmap_pv_list_unlock(lock); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
* Destroy all managed, non-wired mappings in the given user-space | * Destroy all managed, non-wired mappings in the given user-space | ||||
* pmap. This pmap cannot be active on any processor besides the | * pmap. This pmap cannot be active on any processor besides the | ||||
* caller. | * caller. | ||||
* | * | ||||
Show All 24 Lines | pmap_remove_pages(pmap_t pmap) | ||||
pt_entry_t *pte, tpte; | pt_entry_t *pte, tpte; | ||||
pt_entry_t PG_M, PG_RW, PG_V; | pt_entry_t PG_M, PG_RW, PG_V; | ||||
struct spglist free; | struct spglist free; | ||||
struct pv_chunklist free_chunks[PMAP_MEMDOM]; | struct pv_chunklist free_chunks[PMAP_MEMDOM]; | ||||
vm_page_t m, mpte, mt; | vm_page_t m, mpte, mt; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
struct pv_chunk *pc, *npc; | struct pv_chunk *pc, *npc; | ||||
struct rwlock *lock; | PVLL *lock; | ||||
int64_t bit; | int64_t bit; | ||||
uint64_t inuse, bitmask; | uint64_t inuse, bitmask; | ||||
int allfree, field, freed, i, idx; | int allfree, field, freed, i, idx; | ||||
boolean_t superpage; | boolean_t superpage; | ||||
vm_paddr_t pa; | vm_paddr_t pa; | ||||
/* | /* | ||||
* Assert that the given pmap is only active on the current | * Assert that the given pmap is only active on the current | ||||
▲ Show 20 Lines • Show All 147 Lines • ▼ Show 20 Lines | */ | ||||
PV_STAT(atomic_add_int(&pv_entry_spare, freed)); | PV_STAT(atomic_add_int(&pv_entry_spare, freed)); | ||||
PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); | PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); | ||||
if (allfree) { | if (allfree) { | ||||
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); | ||||
TAILQ_INSERT_TAIL(&free_chunks[pc_to_domain(pc)], pc, pc_list); | TAILQ_INSERT_TAIL(&free_chunks[pc_to_domain(pc)], pc, pc_list); | ||||
} | } | ||||
} | } | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
pmap_invalidate_all(pmap); | pmap_invalidate_all(pmap); | ||||
pmap_pkru_deassign_all(pmap); | pmap_pkru_deassign_all(pmap); | ||||
free_pv_chunk_batch((struct pv_chunklist *)&free_chunks); | free_pv_chunk_batch((struct pv_chunklist *)&free_chunks); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
} | } | ||||
static boolean_t | static boolean_t | ||||
pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) | pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pt_entry_t *pte, mask; | pt_entry_t *pte, mask; | ||||
pt_entry_t PG_A, PG_M, PG_RW, PG_V; | pt_entry_t PG_A, PG_M, PG_RW, PG_V; | ||||
pmap_t pmap; | pmap_t pmap; | ||||
int md_gen, pvh_gen; | int md_gen, pvh_gen; | ||||
boolean_t rv; | boolean_t rv; | ||||
rv = FALSE; | rv = FALSE; | ||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | lock = VM_PAGE_TO_PV_LIST_LOCK(m); | ||||
rw_rlock(lock); | pmap_pv_list_lock(lock); | ||||
restart: | restart: | ||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
rw_runlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_rlock(lock); | pmap_pv_list_lock(lock); | ||||
if (md_gen != m->md.pv_gen) { | if (md_gen != m->md.pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto restart; | goto restart; | ||||
} | } | ||||
} | } | ||||
pte = pmap_pte(pmap, pv->pv_va); | pte = pmap_pte(pmap, pv->pv_va); | ||||
mask = 0; | mask = 0; | ||||
if (modified) { | if (modified) { | ||||
Show All 13 Lines | restart: | ||||
} | } | ||||
if ((m->flags & PG_FICTITIOUS) == 0) { | if ((m->flags & PG_FICTITIOUS) == 0) { | ||||
pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); | pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); | ||||
TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { | TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
rw_runlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_rlock(lock); | pmap_pv_list_lock(lock); | ||||
if (md_gen != m->md.pv_gen || | if (md_gen != m->md.pv_gen || | ||||
pvh_gen != pvh->pv_gen) { | pvh_gen != pvh->pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto restart; | goto restart; | ||||
} | } | ||||
} | } | ||||
pte = pmap_pde(pmap, pv->pv_va); | pte = pmap_pde(pmap, pv->pv_va); | ||||
mask = 0; | mask = 0; | ||||
Show All 9 Lines | TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { | ||||
} | } | ||||
rv = (*pte & mask) == mask; | rv = (*pte & mask) == mask; | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
if (rv) | if (rv) | ||||
goto out; | goto out; | ||||
} | } | ||||
} | } | ||||
out: | out: | ||||
rw_runlock(lock); | pmap_pv_list_unlock(lock); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
* pmap_is_modified: | * pmap_is_modified: | ||||
* | * | ||||
* Return whether or not the specified physical page was modified | * Return whether or not the specified physical page was modified | ||||
* in any physical maps. | * in any physical maps. | ||||
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Clear the write and modified bits in each of the given page's mappings. | * Clear the write and modified bits in each of the given page's mappings. | ||||
*/ | */ | ||||
void | void | ||||
pmap_remove_write(vm_page_t m) | pmap_remove_write(vm_page_t m) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pmap_t pmap; | pmap_t pmap; | ||||
struct rwlock *lock; | PVLL *lock; | ||||
pv_entry_t next_pv, pv; | pv_entry_t next_pv, pv; | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
pt_entry_t oldpte, *pte, PG_M, PG_RW; | pt_entry_t oldpte, *pte, PG_M, PG_RW; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
int pvh_gen, md_gen; | int pvh_gen, md_gen; | ||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | KASSERT((m->oflags & VPO_UNMANAGED) == 0, | ||||
("pmap_remove_write: page %p is not managed", m)); | ("pmap_remove_write: page %p is not managed", m)); | ||||
vm_page_assert_busied(m); | vm_page_assert_busied(m); | ||||
if (!pmap_page_is_write_mapped(m)) | if (!pmap_page_is_write_mapped(m)) | ||||
return; | return; | ||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | lock = VM_PAGE_TO_PV_LIST_LOCK(m); | ||||
Done Inline ActionsWouldn't it be simpler to lock the base page of the superpage when removing large mappings, here and in pmap_remove_all()? Then switch to the page's PV list lock for small mappings below. markj: Wouldn't it be simpler to lock the base page of the superpage when removing large mappings… | |||||
Done Inline ActionsI am not sure, due to 'then switch to the small mappings lock''. Having superpage mapping is quite rare, actually. As result we typically would need to unlock and relock quite often. kib: I am not sure, due to 'then switch to the small mappings lock''. Having superpage mapping is… | |||||
pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : | pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : | ||||
pa_to_pvh(VM_PAGE_TO_PHYS(m)); | pa_to_pvh(VM_PAGE_TO_PHYS(m)); | ||||
retry_pv_loop: | retry_pv_loop: | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { | TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
if (pvh_gen != pvh->pv_gen) { | if (pvh_gen != pvh->pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
goto retry_pv_loop; | goto retry_pv_loop; | ||||
} | } | ||||
} | } | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
va = pv->pv_va; | va = pv->pv_va; | ||||
pde = pmap_pde(pmap, va); | pde = pmap_pde(pmap, va); | ||||
if ((*pde & PG_RW) != 0) | if ((*pde & PG_RW) != 0) | ||||
(void)pmap_demote_pde_locked(pmap, pde, va, &lock); | (void)pmap_demote_pde_locked(pmap, pde, va, &lock); | ||||
markjUnsubmitted Done Inline ActionsNow, pmap_demote_pde_locked() may release the lock if pmap_pv_list_lock_pde1() fails to lock the first page in the superpage (i.e., the fast attempt fails). Don't we need to restart the loop in that case? BTW I think there is a rare existing bug here, since pmap_demote_pde_locked() may also release the lock when reclaiming PV entries. In particular, I think the assertion below can fail. markj: Now, pmap_demote_pde_locked() may release the lock if pmap_pv_list_lock_pde1() fails to lock… | |||||
KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), | KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), | ||||
("inconsistent pv lock %p %p for page %p", | ("inconsistent pv lock %p %p for page %p", | ||||
lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); | lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
if (pvh_gen != pvh->pv_gen || | if (pvh_gen != pvh->pv_gen || | ||||
md_gen != m->md.pv_gen) { | md_gen != m->md.pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
goto retry_pv_loop; | goto retry_pv_loop; | ||||
} | } | ||||
} | } | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
pde = pmap_pde(pmap, pv->pv_va); | pde = pmap_pde(pmap, pv->pv_va); | ||||
KASSERT((*pde & PG_PS) == 0, | KASSERT((*pde & PG_PS) == 0, | ||||
("pmap_remove_write: found a 2mpage in page %p's pv list", | ("pmap_remove_write: found a 2mpage in page %p's pv list", | ||||
m)); | m)); | ||||
pte = pmap_pde_to_pte(pde, pv->pv_va); | pte = pmap_pde_to_pte(pde, pv->pv_va); | ||||
retry: | retry: | ||||
oldpte = *pte; | oldpte = *pte; | ||||
if (oldpte & PG_RW) { | if (oldpte & PG_RW) { | ||||
if (!atomic_cmpset_long(pte, oldpte, oldpte & | if (!atomic_cmpset_long(pte, oldpte, oldpte & | ||||
~(PG_RW | PG_M))) | ~(PG_RW | PG_M))) | ||||
goto retry; | goto retry; | ||||
if ((oldpte & PG_M) != 0) | if ((oldpte & PG_M) != 0) | ||||
vm_page_dirty(m); | vm_page_dirty(m); | ||||
pmap_invalidate_page(pmap, pv->pv_va); | pmap_invalidate_page(pmap, pv->pv_va); | ||||
} | } | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
vm_page_aflag_clear(m, PGA_WRITEABLE); | vm_page_aflag_clear(m, PGA_WRITEABLE); | ||||
pmap_delayed_invl_wait(m); | pmap_delayed_invl_wait(m); | ||||
} | } | ||||
static __inline boolean_t | static __inline boolean_t | ||||
safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte) | safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | |||||
* released. | * released. | ||||
*/ | */ | ||||
int | int | ||||
pmap_ts_referenced(vm_page_t m) | pmap_ts_referenced(vm_page_t m) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pv_entry_t pv, pvf; | pv_entry_t pv, pvf; | ||||
pmap_t pmap; | pmap_t pmap; | ||||
struct rwlock *lock; | PVLL *lock; | ||||
pd_entry_t oldpde, *pde; | pd_entry_t oldpde, *pde; | ||||
pt_entry_t *pte, PG_A, PG_M, PG_RW; | pt_entry_t *pte, PG_A, PG_M, PG_RW; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
vm_paddr_t pa; | vm_paddr_t pa; | ||||
int cleared, md_gen, not_cleared, pvh_gen; | int cleared, md_gen, not_cleared, pvh_gen; | ||||
struct spglist free; | struct spglist free; | ||||
boolean_t demoted; | boolean_t demoted; | ||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | KASSERT((m->oflags & VPO_UNMANAGED) == 0, | ||||
("pmap_ts_referenced: page %p is not managed", m)); | ("pmap_ts_referenced: page %p is not managed", m)); | ||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
cleared = 0; | cleared = 0; | ||||
pa = VM_PAGE_TO_PHYS(m); | pa = VM_PAGE_TO_PHYS(m); | ||||
lock = PHYS_TO_PV_LIST_LOCK(pa); | lock = PHYS_TO_PV_LIST_LOCK(pa); | ||||
pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); | pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
retry: | retry: | ||||
not_cleared = 0; | not_cleared = 0; | ||||
if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) | if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) | ||||
goto small_mappings; | goto small_mappings; | ||||
pv = pvf; | pv = pvf; | ||||
do { | do { | ||||
if (pvf == NULL) | if (pvf == NULL) | ||||
pvf = pv; | pvf = pv; | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
if (pvh_gen != pvh->pv_gen) { | if (pvh_gen != pvh->pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto retry; | goto retry; | ||||
} | } | ||||
} | } | ||||
PG_A = pmap_accessed_bit(pmap); | PG_A = pmap_accessed_bit(pmap); | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | if ((oldpde & PG_A) != 0) { | ||||
va += VM_PAGE_TO_PHYS(m) - (oldpde & | va += VM_PAGE_TO_PHYS(m) - (oldpde & | ||||
PG_PS_FRAME); | PG_PS_FRAME); | ||||
pte = pmap_pde_to_pte(pde, va); | pte = pmap_pde_to_pte(pde, va); | ||||
pmap_remove_pte(pmap, pte, va, *pde, | pmap_remove_pte(pmap, pte, va, *pde, | ||||
NULL, &lock); | NULL, &lock); | ||||
pmap_invalidate_page(pmap, va); | pmap_invalidate_page(pmap, va); | ||||
} else | } else | ||||
demoted = TRUE; | demoted = TRUE; | ||||
Done Inline ActionsHuh, there is some problem with the patch generation, I believe this line is stray. kib: Huh, there is some problem with the patch generation, I believe this line is stray. | |||||
if (demoted) { | if (demoted) { | ||||
Done Inline ActionsI think we need to retry in this case too. markj: I think we need to retry in this case too. | |||||
/* | /* | ||||
* The superpage mapping was removed | * The superpage mapping was removed | ||||
* entirely and therefore 'pv' is no | * entirely and therefore 'pv' is no | ||||
* longer valid. | * longer valid. | ||||
*/ | */ | ||||
if (pvf == pv) | if (pvf == pv) | ||||
pvf = NULL; | pvf = NULL; | ||||
pv = NULL; | pv = NULL; | ||||
Show All 21 Lines | small_mappings: | ||||
pv = pvf; | pv = pvf; | ||||
do { | do { | ||||
if (pvf == NULL) | if (pvf == NULL) | ||||
pvf = pv; | pvf = pv; | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { | if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto retry; | goto retry; | ||||
} | } | ||||
} | } | ||||
PG_A = pmap_accessed_bit(pmap); | PG_A = pmap_accessed_bit(pmap); | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
Show All 34 Lines | do { | ||||
if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { | if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { | ||||
TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); | ||||
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); | ||||
m->md.pv_gen++; | m->md.pv_gen++; | ||||
} | } | ||||
} while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + | } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + | ||||
not_cleared < PMAP_TS_REFERENCED_MAX); | not_cleared < PMAP_TS_REFERENCED_MAX); | ||||
out: | out: | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
return (cleared + not_cleared); | return (cleared + not_cleared); | ||||
} | } | ||||
/* | /* | ||||
* Apply the given advice to the specified range of addresses within the | * Apply the given advice to the specified range of addresses within the | ||||
* given pmap. Depending on the advice, clear the referenced and/or | * given pmap. Depending on the advice, clear the referenced and/or | ||||
* modified flags in each mapping and set the mapped page's dirty field. | * modified flags in each mapping and set the mapped page's dirty field. | ||||
*/ | */ | ||||
void | void | ||||
pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) | pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) | ||||
{ | { | ||||
struct rwlock *lock; | PVLL *lock; | ||||
pml4_entry_t *pml4e; | pml4_entry_t *pml4e; | ||||
pdp_entry_t *pdpe; | pdp_entry_t *pdpe; | ||||
pd_entry_t oldpde, *pde; | pd_entry_t oldpde, *pde; | ||||
pt_entry_t *pte, PG_A, PG_G, PG_M, PG_RW, PG_V; | pt_entry_t *pte, PG_A, PG_G, PG_M, PG_RW, PG_V; | ||||
vm_offset_t va, va_next; | vm_offset_t va, va_next; | ||||
vm_page_t m; | vm_page_t m; | ||||
bool anychanged; | bool anychanged; | ||||
Show All 40 Lines | for (; sva < eva; sva = va_next) { | ||||
if ((oldpde & PG_V) == 0) | if ((oldpde & PG_V) == 0) | ||||
continue; | continue; | ||||
else if ((oldpde & PG_PS) != 0) { | else if ((oldpde & PG_PS) != 0) { | ||||
if ((oldpde & PG_MANAGED) == 0) | if ((oldpde & PG_MANAGED) == 0) | ||||
continue; | continue; | ||||
lock = NULL; | lock = NULL; | ||||
if (!pmap_demote_pde_locked(pmap, pde, sva, &lock)) { | if (!pmap_demote_pde_locked(pmap, pde, sva, &lock)) { | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
/* | /* | ||||
* The large page mapping was destroyed. | * The large page mapping was destroyed. | ||||
*/ | */ | ||||
continue; | continue; | ||||
} | } | ||||
/* | /* | ||||
Show All 15 Lines | else if ((oldpde & PG_PS) != 0) { | ||||
pte = pmap_pde_to_pte(pde, va); | pte = pmap_pde_to_pte(pde, va); | ||||
KASSERT((*pte & PG_V) != 0, | KASSERT((*pte & PG_V) != 0, | ||||
("pmap_advise: invalid PTE")); | ("pmap_advise: invalid PTE")); | ||||
pmap_remove_pte(pmap, pte, va, *pde, NULL, | pmap_remove_pte(pmap, pte, va, *pde, NULL, | ||||
&lock); | &lock); | ||||
anychanged = true; | anychanged = true; | ||||
} | } | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
} | } | ||||
if (va_next > eva) | if (va_next > eva) | ||||
va_next = eva; | va_next = eva; | ||||
va = va_next; | va = va_next; | ||||
for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, | for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, | ||||
sva += PAGE_SIZE) { | sva += PAGE_SIZE) { | ||||
if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) | if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) | ||||
goto maybe_invlrng; | goto maybe_invlrng; | ||||
Show All 40 Lines | |||||
void | void | ||||
pmap_clear_modify(vm_page_t m) | pmap_clear_modify(vm_page_t m) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
pmap_t pmap; | pmap_t pmap; | ||||
pv_entry_t next_pv, pv; | pv_entry_t next_pv, pv; | ||||
pd_entry_t oldpde, *pde; | pd_entry_t oldpde, *pde; | ||||
pt_entry_t *pte, PG_M, PG_RW; | pt_entry_t *pte, PG_M, PG_RW; | ||||
struct rwlock *lock; | PVLL *lock; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
int md_gen, pvh_gen; | int md_gen, pvh_gen; | ||||
KASSERT((m->oflags & VPO_UNMANAGED) == 0, | KASSERT((m->oflags & VPO_UNMANAGED) == 0, | ||||
("pmap_clear_modify: page %p is not managed", m)); | ("pmap_clear_modify: page %p is not managed", m)); | ||||
vm_page_assert_busied(m); | vm_page_assert_busied(m); | ||||
if (!pmap_page_is_write_mapped(m)) | if (!pmap_page_is_write_mapped(m)) | ||||
return; | return; | ||||
pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : | pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : | ||||
pa_to_pvh(VM_PAGE_TO_PHYS(m)); | pa_to_pvh(VM_PAGE_TO_PHYS(m)); | ||||
lock = VM_PAGE_TO_PV_LIST_LOCK(m); | lock = VM_PAGE_TO_PV_LIST_LOCK(m); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
restart: | restart: | ||||
TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { | TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
if (pvh_gen != pvh->pv_gen) { | if (pvh_gen != pvh->pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto restart; | goto restart; | ||||
} | } | ||||
} | } | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
va = pv->pv_va; | va = pv->pv_va; | ||||
Show All 15 Lines | TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { | ||||
} | } | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { | ||||
pmap = PV_PMAP(pv); | pmap = PV_PMAP(pv); | ||||
if (!PMAP_TRYLOCK(pmap)) { | if (!PMAP_TRYLOCK(pmap)) { | ||||
md_gen = m->md.pv_gen; | md_gen = m->md.pv_gen; | ||||
pvh_gen = pvh->pv_gen; | pvh_gen = pvh->pv_gen; | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
rw_wlock(lock); | pmap_pv_list_lock(lock); | ||||
if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { | if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
goto restart; | goto restart; | ||||
} | } | ||||
} | } | ||||
PG_M = pmap_modified_bit(pmap); | PG_M = pmap_modified_bit(pmap); | ||||
PG_RW = pmap_rw_bit(pmap); | PG_RW = pmap_rw_bit(pmap); | ||||
pde = pmap_pde(pmap, pv->pv_va); | pde = pmap_pde(pmap, pv->pv_va); | ||||
KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found" | KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found" | ||||
" a 2mpage in page %p's pv list", m)); | " a 2mpage in page %p's pv list", m)); | ||||
pte = pmap_pde_to_pte(pde, pv->pv_va); | pte = pmap_pde_to_pte(pde, pv->pv_va); | ||||
if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { | if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { | ||||
atomic_clear_long(pte, PG_M); | atomic_clear_long(pte, PG_M); | ||||
pmap_invalidate_page(pmap, pv->pv_va); | pmap_invalidate_page(pmap, pv->pv_va); | ||||
} | } | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
} | } | ||||
/* | /* | ||||
* Miscellaneous support routines follow | * Miscellaneous support routines follow | ||||
*/ | */ | ||||
/* Adjust the properties for a leaf page table entry. */ | /* Adjust the properties for a leaf page table entry. */ | ||||
static __inline void | static __inline void | ||||
▲ Show 20 Lines • Show All 932 Lines • ▼ Show 20 Lines | |||||
SYSCTL_ULONG(_vm_pmap, OID_AUTO, ad_emulation_superpage_promotions, CTLFLAG_RW, | SYSCTL_ULONG(_vm_pmap, OID_AUTO, ad_emulation_superpage_promotions, CTLFLAG_RW, | ||||
&ad_emulation_superpage_promotions, 0, NULL); | &ad_emulation_superpage_promotions, 0, NULL); | ||||
#endif /* INVARIANTS */ | #endif /* INVARIANTS */ | ||||
int | int | ||||
pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype) | pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype) | ||||
{ | { | ||||
int rv; | int rv; | ||||
struct rwlock *lock; | PVLL *lock; | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
vm_page_t m, mpte; | vm_page_t m, mpte; | ||||
#endif | #endif | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
pt_entry_t *pte, PG_A, PG_M, PG_RW, PG_V; | pt_entry_t *pte, PG_A, PG_M, PG_RW, PG_V; | ||||
KASSERT(ftype == VM_PROT_READ || ftype == VM_PROT_WRITE, | KASSERT(ftype == VM_PROT_READ || ftype == VM_PROT_WRITE, | ||||
("pmap_emulate_accessed_dirty: invalid fault type %d", ftype)); | ("pmap_emulate_accessed_dirty: invalid fault type %d", ftype)); | ||||
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | #ifdef INVARIANTS | ||||
if (ftype == VM_PROT_WRITE) | if (ftype == VM_PROT_WRITE) | ||||
atomic_add_long(&num_dirty_emulations, 1); | atomic_add_long(&num_dirty_emulations, 1); | ||||
else | else | ||||
atomic_add_long(&num_accessed_emulations, 1); | atomic_add_long(&num_accessed_emulations, 1); | ||||
#endif | #endif | ||||
rv = 0; /* success */ | rv = 0; /* success */ | ||||
done: | done: | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | pmap_pv_list_unlock(lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
void | void | ||||
pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num) | pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num) | ||||
{ | { | ||||
pml4_entry_t *pml4; | pml4_entry_t *pml4; | ||||
▲ Show 20 Lines • Show All 1,547 Lines • Show Last 20 Lines |
Empty messages?