Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_fault.c
Show First 20 Lines • Show All 418 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static int | static int | ||||
vm_fault_populate(struct faultstate *fs) | vm_fault_populate(struct faultstate *fs) | ||||
{ | { | ||||
vm_offset_t vaddr; | vm_offset_t vaddr; | ||||
vm_page_t m; | vm_page_t m; | ||||
vm_pindex_t map_first, map_last, pager_first, pager_last, pidx; | vm_pindex_t map_first, map_last, pager_first, pager_last, pidx; | ||||
int i, npages, psind, rv; | int bdry_idx, i, npages, psind, rv; | ||||
MPASS(fs->object == fs->first_object); | MPASS(fs->object == fs->first_object); | ||||
VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | VM_OBJECT_ASSERT_WLOCKED(fs->first_object); | ||||
MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); | MPASS(blockcount_read(&fs->first_object->paging_in_progress) > 0); | ||||
MPASS(fs->first_object->backing_object == NULL); | MPASS(fs->first_object->backing_object == NULL); | ||||
MPASS(fs->lookup_still_valid); | MPASS(fs->lookup_still_valid); | ||||
pager_first = OFF_TO_IDX(fs->entry->offset); | pager_first = OFF_TO_IDX(fs->entry->offset); | ||||
Show All 28 Lines | vm_fault_populate(struct faultstate *fs) | ||||
/* Ensure that the driver is obeying the interface. */ | /* Ensure that the driver is obeying the interface. */ | ||||
MPASS(pager_first <= pager_last); | MPASS(pager_first <= pager_last); | ||||
MPASS(fs->first_pindex <= pager_last); | MPASS(fs->first_pindex <= pager_last); | ||||
MPASS(fs->first_pindex >= pager_first); | MPASS(fs->first_pindex >= pager_first); | ||||
MPASS(pager_last < fs->first_object->size); | MPASS(pager_last < fs->first_object->size); | ||||
vm_fault_restore_map_lock(fs); | vm_fault_restore_map_lock(fs); | ||||
bdry_idx = (fs->entry->eflags & MAP_ENTRY_SPLIT_BOUNDARY_MASK) >> | |||||
MAP_ENTRY_SPLIT_BOUNDARY_SHIFT; | |||||
if (fs->map->timestamp != fs->map_generation) { | if (fs->map->timestamp != fs->map_generation) { | ||||
if (bdry_idx == 0) { | |||||
vm_fault_populate_cleanup(fs->first_object, pager_first, | vm_fault_populate_cleanup(fs->first_object, pager_first, | ||||
pager_last); | pager_last); | ||||
} else { | |||||
m = vm_page_lookup(fs->first_object, pager_first); | |||||
vm_page_xunbusy(m); | |||||
} | |||||
return (KERN_RESTART); | return (KERN_RESTART); | ||||
} | } | ||||
/* | /* | ||||
* The map is unchanged after our last unlock. Process the fault. | * The map is unchanged after our last unlock. Process the fault. | ||||
* | * | ||||
* First, the special case of largepage mappings, where | |||||
* populate only busies the first page in superpage run. | |||||
*/ | |||||
if (bdry_idx != 0) { | |||||
m = vm_page_lookup(fs->first_object, pager_first); | |||||
vm_fault_populate_check_page(m); | |||||
VM_OBJECT_WUNLOCK(fs->first_object); | |||||
vaddr = fs->entry->start + IDX_TO_OFF(pager_first) - | |||||
fs->entry->offset; | |||||
/* assert alignment for entry */ | |||||
KASSERT((vaddr & (pagesizes[bdry_idx] - 1)) == 0, | |||||
("unaligned superpage start %#jx pager_first %#jx offset %#jx vaddr %#jx", | |||||
(uintmax_t)fs->entry->start, (uintmax_t)pager_first, | |||||
(uintmax_t)fs->entry->offset, (uintmax_t)vaddr)); | |||||
KASSERT((VM_PAGE_TO_PHYS(m) & (pagesizes[bdry_idx] - 1)) == 0, | |||||
("unaligned superpage m %p %#jx", m, | |||||
(uintmax_t)VM_PAGE_TO_PHYS(m))); | |||||
rv = pmap_enter(fs->map->pmap, vaddr, m, fs->prot, | |||||
fs->fault_type | PMAP_ENTER_LARGEPAGE, bdry_idx); | |||||
markj: We are not passing PMAP_ENTER_WIRED even if the fault is from mlock(). More on that below. | |||||
VM_OBJECT_WLOCK(fs->first_object); | |||||
vm_page_xunbusy(m); | |||||
goto out; | |||||
} | |||||
/* | |||||
* The range [pager_first, pager_last] that is given to the | * The range [pager_first, pager_last] that is given to the | ||||
* pager is only a hint. The pager may populate any range | * pager is only a hint. The pager may populate any range | ||||
* within the object that includes the requested page index. | * within the object that includes the requested page index. | ||||
* In case the pager expanded the range, clip it to fit into | * In case the pager expanded the range, clip it to fit into | ||||
* the map entry. | * the map entry. | ||||
*/ | */ | ||||
map_first = OFF_TO_IDX(fs->entry->offset); | map_first = OFF_TO_IDX(fs->entry->offset); | ||||
if (map_first > pager_first) { | if (map_first > pager_first) { | ||||
vm_fault_populate_cleanup(fs->first_object, pager_first, | vm_fault_populate_cleanup(fs->first_object, pager_first, | ||||
map_first - 1); | map_first - 1); | ||||
pager_first = map_first; | pager_first = map_first; | ||||
} | } | ||||
map_last = map_first + atop(fs->entry->end - fs->entry->start) - 1; | map_last = map_first + atop(fs->entry->end - fs->entry->start) - 1; | ||||
if (map_last < pager_last) { | if (map_last < pager_last) { | ||||
vm_fault_populate_cleanup(fs->first_object, map_last + 1, | vm_fault_populate_cleanup(fs->first_object, map_last + 1, | ||||
pager_last); | pager_last); | ||||
pager_last = map_last; | pager_last = map_last; | ||||
} | } | ||||
for (pidx = pager_first, m = vm_page_lookup(fs->first_object, pidx); | for (pidx = pager_first, m = vm_page_lookup(fs->first_object, pidx); | ||||
Done Inline Actions/* should be on its own line. markj: `/*` should be on its own line. | |||||
pidx <= pager_last; | pidx <= pager_last; | ||||
pidx += npages, m = vm_page_next(&m[npages - 1])) { | pidx += npages, m = vm_page_next(&m[npages - 1])) { | ||||
vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset; | vaddr = fs->entry->start + IDX_TO_OFF(pidx) - fs->entry->offset; | ||||
#if defined(__aarch64__) || defined(__amd64__) || (defined(__arm__) && \ | #if defined(__aarch64__) || defined(__amd64__) || (defined(__arm__) && \ | ||||
__ARM_ARCH >= 6) || defined(__i386__) || defined(__riscv) | __ARM_ARCH >= 6) || defined(__i386__) || defined(__riscv) | ||||
psind = m->psind; | psind = m->psind; | ||||
if (psind > 0 && ((vaddr & (pagesizes[psind] - 1)) != 0 || | if (psind > 0 && ((vaddr & (pagesizes[psind] - 1)) != 0 || | ||||
pidx + OFF_TO_IDX(pagesizes[psind]) - 1 > pager_last || | pidx + OFF_TO_IDX(pagesizes[psind]) - 1 > pager_last || | ||||
Show All 30 Lines | for (i = 0; i < npages; i++) { | ||||
vm_page_activate(&m[i]); | vm_page_activate(&m[i]); | ||||
if (fs->m_hold != NULL && m[i].pindex == fs->first_pindex) { | if (fs->m_hold != NULL && m[i].pindex == fs->first_pindex) { | ||||
(*fs->m_hold) = &m[i]; | (*fs->m_hold) = &m[i]; | ||||
vm_page_wire(&m[i]); | vm_page_wire(&m[i]); | ||||
} | } | ||||
vm_page_xunbusy(&m[i]); | vm_page_xunbusy(&m[i]); | ||||
} | } | ||||
} | } | ||||
out: | |||||
curthread->td_ru.ru_majflt++; | curthread->td_ru.ru_majflt++; | ||||
return (KERN_SUCCESS); | return (KERN_SUCCESS); | ||||
} | } | ||||
static int prot_fault_translation; | static int prot_fault_translation; | ||||
SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RWTUN, | SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RWTUN, | ||||
&prot_fault_translation, 0, | &prot_fault_translation, 0, | ||||
"Control signal to deliver on protection fault"); | "Control signal to deliver on protection fault"); | ||||
▲ Show 20 Lines • Show All 1,461 Lines • Show Last 20 Lines |
We are not passing PMAP_ENTER_WIRED even if the fault is from mlock(). More on that below.