Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/pmap.c
Show First 20 Lines • Show All 633 Lines • ▼ Show 20 Lines | |||||
static caddr_t crashdumpmap; | static caddr_t crashdumpmap; | ||||
/* | /* | ||||
* Internal flags for pmap_enter()'s helper functions. | * Internal flags for pmap_enter()'s helper functions. | ||||
*/ | */ | ||||
#define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ | #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ | ||||
#define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ | #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ | ||||
/* | |||||
* Internal flags for pmap_mapdev_internal() and | |||||
* pmap_change_attr_locked(). | |||||
*/ | |||||
#define MAPDEV_FLUSHCACHE 0x0000001 /* Flush cache after mapping. */ | |||||
#define MAPDEV_SETATTR 0x0000002 /* Modify existing attrs. */ | |||||
static void free_pv_chunk(struct pv_chunk *pc); | static void free_pv_chunk(struct pv_chunk *pc); | ||||
static void free_pv_entry(pmap_t pmap, pv_entry_t pv); | static void free_pv_entry(pmap_t pmap, pv_entry_t pv); | ||||
static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); | static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); | ||||
static int popcnt_pc_map_pq(uint64_t *map); | static int popcnt_pc_map_pq(uint64_t *map); | ||||
static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); | static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); | ||||
static void reserve_pv_entries(pmap_t pmap, int needed, | static void reserve_pv_entries(pmap_t pmap, int needed, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, | static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, | ||||
u_int flags, struct rwlock **lockp); | u_int flags, struct rwlock **lockp); | ||||
#if VM_NRESERVLEVEL > 0 | #if VM_NRESERVLEVEL > 0 | ||||
static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, | ||||
struct rwlock **lockp); | struct rwlock **lockp); | ||||
#endif | #endif | ||||
static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); | static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); | ||||
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, | static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, | ||||
vm_offset_t va); | vm_offset_t va); | ||||
static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, | static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, | ||||
bool noflush); | int flags); | ||||
static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); | static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); | ||||
static boolean_t pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, | static boolean_t pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, | ||||
vm_offset_t va, struct rwlock **lockp); | vm_offset_t va, struct rwlock **lockp); | ||||
static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, | static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, | ||||
vm_offset_t va); | vm_offset_t va); | ||||
static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, | static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
vm_prot_t prot, struct rwlock **lockp); | vm_prot_t prot, struct rwlock **lockp); | ||||
static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, | static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, | ||||
▲ Show 20 Lines • Show All 6,505 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Map a set of physical memory pages into the kernel virtual | * Map a set of physical memory pages into the kernel virtual | ||||
* address space. Return a pointer to where it is mapped. This | * address space. Return a pointer to where it is mapped. This | ||||
* routine is intended to be used for mapping device memory, | * routine is intended to be used for mapping device memory, | ||||
* NOT real memory. | * NOT real memory. | ||||
*/ | */ | ||||
static void * | static void * | ||||
pmap_mapdev_internal(vm_paddr_t pa, vm_size_t size, int mode, bool noflush) | pmap_mapdev_internal(vm_paddr_t pa, vm_size_t size, int mode, int flags) | ||||
{ | { | ||||
struct pmap_preinit_mapping *ppim; | struct pmap_preinit_mapping *ppim; | ||||
vm_offset_t va, offset; | vm_offset_t va, offset; | ||||
vm_size_t tmpsize; | vm_size_t tmpsize; | ||||
int i; | int i; | ||||
offset = pa & PAGE_MASK; | offset = pa & PAGE_MASK; | ||||
size = round_page(offset + size); | size = round_page(offset + size); | ||||
Show All 17 Lines | if (va == 0) | ||||
panic("%s: too many preinit mappings", __func__); | panic("%s: too many preinit mappings", __func__); | ||||
} else { | } else { | ||||
/* | /* | ||||
* If we have a preinit mapping, re-use it. | * If we have a preinit mapping, re-use it. | ||||
*/ | */ | ||||
for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { | for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { | ||||
ppim = pmap_preinit_mapping + i; | ppim = pmap_preinit_mapping + i; | ||||
if (ppim->pa == pa && ppim->sz == size && | if (ppim->pa == pa && ppim->sz == size && | ||||
ppim->mode == mode) | (ppim->mode == mode || | ||||
(flags & MAPDEV_SETATTR) == 0)) | |||||
return ((void *)(ppim->va + offset)); | return ((void *)(ppim->va + offset)); | ||||
} | } | ||||
/* | /* | ||||
* If the specified range of physical addresses fits within | * If the specified range of physical addresses fits within | ||||
* the direct map window, use the direct map. | * the direct map window, use the direct map. | ||||
*/ | */ | ||||
if (pa < dmaplimit && pa + size <= dmaplimit) { | if (pa < dmaplimit && pa + size <= dmaplimit) { | ||||
va = PHYS_TO_DMAP(pa); | va = PHYS_TO_DMAP(pa); | ||||
if ((flags & MAPDEV_SETATTR) != 0) { | |||||
PMAP_LOCK(kernel_pmap); | PMAP_LOCK(kernel_pmap); | ||||
i = pmap_change_attr_locked(va, size, mode, noflush); | i = pmap_change_attr_locked(va, size, mode, flags); | ||||
PMAP_UNLOCK(kernel_pmap); | PMAP_UNLOCK(kernel_pmap); | ||||
} else | |||||
i = 0; | |||||
kib: != 0 | |||||
Done Inline ActionsI really do not like that particular interpretation of that rule for single-bit flags. Testing a single-bit flag always looks like a boolean to me. jhb: I really do not like that particular interpretation of that rule for single-bit flags. Testing… | |||||
if (!i) | if (!i) | ||||
return ((void *)(va + offset)); | return ((void *)(va + offset)); | ||||
} | } | ||||
va = kva_alloc(size); | va = kva_alloc(size); | ||||
if (va == 0) | if (va == 0) | ||||
panic("%s: Couldn't allocate KVA", __func__); | panic("%s: Couldn't allocate KVA", __func__); | ||||
} | } | ||||
for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) | for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) | ||||
pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode); | pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode); | ||||
pmap_invalidate_range(kernel_pmap, va, va + tmpsize); | pmap_invalidate_range(kernel_pmap, va, va + tmpsize); | ||||
if (!noflush) | if ((flags & MAPDEV_FLUSHCACHE) != 0) | ||||
pmap_invalidate_cache_range(va, va + tmpsize); | pmap_invalidate_cache_range(va, va + tmpsize); | ||||
return ((void *)(va + offset)); | return ((void *)(va + offset)); | ||||
} | } | ||||
void * | void * | ||||
pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) | pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) | ||||
{ | { | ||||
return (pmap_mapdev_internal(pa, size, mode, false)); | return (pmap_mapdev_internal(pa, size, mode, MAPDEV_FLUSHCACHE | | ||||
MAPDEV_SETATTR)); | |||||
} | } | ||||
void * | void * | ||||
pmap_mapdev(vm_paddr_t pa, vm_size_t size) | pmap_mapdev(vm_paddr_t pa, vm_size_t size) | ||||
{ | { | ||||
return (pmap_mapdev_internal(pa, size, PAT_UNCACHEABLE, false)); | return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE)); | ||||
} | } | ||||
void * | void * | ||||
pmap_mapdev_pciecfg(vm_paddr_t pa, vm_size_t size) | pmap_mapdev_pciecfg(vm_paddr_t pa, vm_size_t size) | ||||
{ | { | ||||
return (pmap_mapdev_internal(pa, size, PAT_UNCACHEABLE, true)); | return (pmap_mapdev_internal(pa, size, PAT_UNCACHEABLE, | ||||
MAPDEV_SETATTR)); | |||||
} | } | ||||
void * | void * | ||||
pmap_mapbios(vm_paddr_t pa, vm_size_t size) | pmap_mapbios(vm_paddr_t pa, vm_size_t size) | ||||
{ | { | ||||
return (pmap_mapdev_internal(pa, size, PAT_WRITE_BACK, false)); | return (pmap_mapdev_internal(pa, size, PAT_WRITE_BACK, | ||||
MAPDEV_FLUSHCACHE)); | |||||
} | } | ||||
void | void | ||||
pmap_unmapdev(vm_offset_t va, vm_size_t size) | pmap_unmapdev(vm_offset_t va, vm_size_t size) | ||||
{ | { | ||||
struct pmap_preinit_mapping *ppim; | struct pmap_preinit_mapping *ppim; | ||||
vm_offset_t offset; | vm_offset_t offset; | ||||
int i; | int i; | ||||
▲ Show 20 Lines • Show All 122 Lines • ▼ Show 20 Lines | |||||
* virtual address range or the direct map. | * virtual address range or the direct map. | ||||
*/ | */ | ||||
int | int | ||||
pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) | pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) | ||||
{ | { | ||||
int error; | int error; | ||||
PMAP_LOCK(kernel_pmap); | PMAP_LOCK(kernel_pmap); | ||||
error = pmap_change_attr_locked(va, size, mode, false); | error = pmap_change_attr_locked(va, size, mode, MAPDEV_FLUSHCACHE); | ||||
PMAP_UNLOCK(kernel_pmap); | PMAP_UNLOCK(kernel_pmap); | ||||
return (error); | return (error); | ||||
} | } | ||||
static int | static int | ||||
pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool noflush) | pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, int flags) | ||||
{ | { | ||||
vm_offset_t base, offset, tmpva; | vm_offset_t base, offset, tmpva; | ||||
vm_paddr_t pa_start, pa_end, pa_end1; | vm_paddr_t pa_start, pa_end, pa_end1; | ||||
pdp_entry_t *pdpe; | pdp_entry_t *pdpe; | ||||
pd_entry_t *pde; | pd_entry_t *pde; | ||||
pt_entry_t *pte; | pt_entry_t *pte; | ||||
int cache_bits_pte, cache_bits_pde, error; | int cache_bits_pte, cache_bits_pde, error; | ||||
boolean_t changed; | boolean_t changed; | ||||
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines | if (*pdpe & PG_PS) { | ||||
pa_start = *pdpe & PG_PS_FRAME; | pa_start = *pdpe & PG_PS_FRAME; | ||||
pa_end = pa_start + NBPDP; | pa_end = pa_start + NBPDP; | ||||
} else if (pa_end == (*pdpe & PG_PS_FRAME)) | } else if (pa_end == (*pdpe & PG_PS_FRAME)) | ||||
pa_end += NBPDP; | pa_end += NBPDP; | ||||
else { | else { | ||||
/* Run ended, update direct map. */ | /* Run ended, update direct map. */ | ||||
error = pmap_change_attr_locked( | error = pmap_change_attr_locked( | ||||
PHYS_TO_DMAP(pa_start), | PHYS_TO_DMAP(pa_start), | ||||
pa_end - pa_start, mode, noflush); | pa_end - pa_start, mode, flags); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
/* Start physical address run. */ | /* Start physical address run. */ | ||||
pa_start = *pdpe & PG_PS_FRAME; | pa_start = *pdpe & PG_PS_FRAME; | ||||
pa_end = pa_start + NBPDP; | pa_end = pa_start + NBPDP; | ||||
} | } | ||||
} | } | ||||
tmpva = trunc_1gpage(tmpva) + NBPDP; | tmpva = trunc_1gpage(tmpva) + NBPDP; | ||||
Show All 13 Lines | if (*pde & PG_PS) { | ||||
pa_start = *pde & PG_PS_FRAME; | pa_start = *pde & PG_PS_FRAME; | ||||
pa_end = pa_start + NBPDR; | pa_end = pa_start + NBPDR; | ||||
} else if (pa_end == (*pde & PG_PS_FRAME)) | } else if (pa_end == (*pde & PG_PS_FRAME)) | ||||
pa_end += NBPDR; | pa_end += NBPDR; | ||||
else { | else { | ||||
/* Run ended, update direct map. */ | /* Run ended, update direct map. */ | ||||
error = pmap_change_attr_locked( | error = pmap_change_attr_locked( | ||||
PHYS_TO_DMAP(pa_start), | PHYS_TO_DMAP(pa_start), | ||||
pa_end - pa_start, mode, noflush); | pa_end - pa_start, mode, flags); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
/* Start physical address run. */ | /* Start physical address run. */ | ||||
pa_start = *pde & PG_PS_FRAME; | pa_start = *pde & PG_PS_FRAME; | ||||
pa_end = pa_start + NBPDR; | pa_end = pa_start + NBPDR; | ||||
} | } | ||||
} | } | ||||
tmpva = trunc_2mpage(tmpva) + NBPDR; | tmpva = trunc_2mpage(tmpva) + NBPDR; | ||||
Show All 11 Lines | if (*pde & PG_PS) { | ||||
pa_start = *pte & PG_FRAME; | pa_start = *pte & PG_FRAME; | ||||
pa_end = pa_start + PAGE_SIZE; | pa_end = pa_start + PAGE_SIZE; | ||||
} else if (pa_end == (*pte & PG_FRAME)) | } else if (pa_end == (*pte & PG_FRAME)) | ||||
pa_end += PAGE_SIZE; | pa_end += PAGE_SIZE; | ||||
else { | else { | ||||
/* Run ended, update direct map. */ | /* Run ended, update direct map. */ | ||||
error = pmap_change_attr_locked( | error = pmap_change_attr_locked( | ||||
PHYS_TO_DMAP(pa_start), | PHYS_TO_DMAP(pa_start), | ||||
pa_end - pa_start, mode, noflush); | pa_end - pa_start, mode, flags); | ||||
if (error != 0) | if (error != 0) | ||||
break; | break; | ||||
/* Start physical address run. */ | /* Start physical address run. */ | ||||
pa_start = *pte & PG_FRAME; | pa_start = *pte & PG_FRAME; | ||||
pa_end = pa_start + PAGE_SIZE; | pa_end = pa_start + PAGE_SIZE; | ||||
} | } | ||||
} | } | ||||
tmpva += PAGE_SIZE; | tmpva += PAGE_SIZE; | ||||
} | } | ||||
} | } | ||||
if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { | if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { | ||||
pa_end1 = MIN(pa_end, dmaplimit); | pa_end1 = MIN(pa_end, dmaplimit); | ||||
if (pa_start != pa_end1) | if (pa_start != pa_end1) | ||||
error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), | error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), | ||||
pa_end1 - pa_start, mode, noflush); | pa_end1 - pa_start, mode, flags); | ||||
} | } | ||||
/* | /* | ||||
* Flush CPU caches if required to make sure any data isn't cached that | * Flush CPU caches if required to make sure any data isn't cached that | ||||
* shouldn't be, etc. | * shouldn't be, etc. | ||||
*/ | */ | ||||
if (changed) { | if (changed) { | ||||
pmap_invalidate_range(kernel_pmap, base, tmpva); | pmap_invalidate_range(kernel_pmap, base, tmpva); | ||||
if (!noflush) | if ((flags & MAPDEV_FLUSHCACHE) != 0) | ||||
pmap_invalidate_cache_range(base, tmpva); | pmap_invalidate_cache_range(base, tmpva); | ||||
} | } | ||||
return (error); | return (error); | ||||
} | } | ||||
/* | /* | ||||
* Demotes any mapping within the direct map region that covers more than the | * Demotes any mapping within the direct map region that covers more than the | ||||
* specified range of physical addresses. This range's size must be a power | * specified range of physical addresses. This range's size must be a power | ||||
▲ Show 20 Lines • Show All 1,767 Lines • Show Last 20 Lines |
!= 0