Page Menu
Home
FreeBSD
Search
Configure Global Search
Log In
Files
F168162639
D4444.id11249.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Size
36 KB
Referenced Files
None
Subscribers
None
D4444.id11249.diff
View Options
Index: dev/drm2/ttm/ttm_bo.c
===================================================================
--- dev/drm2/ttm/ttm_bo.c
+++ dev/drm2/ttm/ttm_bo.c
@@ -1488,21 +1488,21 @@
struct ttm_bo_global_ref *bo_ref =
container_of(ref, struct ttm_bo_global_ref, ref);
struct ttm_bo_global *glob = ref->object;
- int ret;
+ int req, ret;
int tries;
sx_init(&glob->device_list_mutex, "ttmdlm");
mtx_init(&glob->lru_lock, "ttmlru", NULL, MTX_DEF);
glob->mem_glob = bo_ref->mem_glob;
+ req = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ;
tries = 0;
retry:
- glob->dummy_read_page = vm_page_alloc_contig(NULL, 0,
- VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ,
+ glob->dummy_read_page = vm_page_alloc_contig(NULL, 0, req,
1, 0, VM_MAX_ADDRESS, PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE);
if (unlikely(glob->dummy_read_page == NULL)) {
- if (tries < 1) {
- vm_pageout_grow_cache(tries, 0, VM_MAX_ADDRESS);
+ if (tries < 1 && vm_page_reclaim_contig(req, 1,
+ 0, VM_MAX_ADDRESS, PAGE_SIZE, 0)) {
tries++;
goto retry;
}
Index: dev/drm2/ttm/ttm_page_alloc.c
===================================================================
--- dev/drm2/ttm/ttm_page_alloc.c
+++ dev/drm2/ttm/ttm_page_alloc.c
@@ -166,13 +166,9 @@
PAGE_SIZE, 0, memattr);
if (p != NULL || tries > 2)
return (p);
-
- /*
- * Before growing the cache see if this is just a normal
- * memory shortage.
- */
- VM_WAIT;
- vm_pageout_grow_cache(tries, 0, 0xffffffff);
+ if (!vm_page_reclaim_contig(req, 1, 0, 0xffffffff,
+ PAGE_SIZE, 0))
+ VM_WAIT;
}
}
Index: mips/include/pmap.h
===================================================================
--- mips/include/pmap.h
+++ mips/include/pmap.h
@@ -178,7 +178,6 @@
void pmap_kenter_temporary_free(vm_paddr_t pa);
void pmap_flush_pvcache(vm_page_t m);
int pmap_emulate_modified(pmap_t pmap, vm_offset_t va);
-void pmap_grow_direct_page_cache(void);
void pmap_page_set_memattr(vm_page_t, vm_memattr_t);
#endif /* _KERNEL */
Index: mips/mips/pmap.c
===================================================================
--- mips/mips/pmap.c
+++ mips/mips/pmap.c
@@ -166,6 +166,7 @@
static vm_page_t pmap_alloc_direct_page(unsigned int index, int req);
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte);
+static void pmap_grow_direct_page(int req);
static int pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va,
pd_entry_t pde);
static void pmap_remove_page(struct pmap *pmap, vm_offset_t va);
@@ -1040,14 +1041,16 @@
bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
}
-void
-pmap_grow_direct_page_cache()
+static void
+pmap_grow_direct_page(int req)
{
#ifdef __mips_n64
VM_WAIT;
#else
- vm_pageout_grow_cache(3, 0, MIPS_KSEG0_LARGEST_PHYS);
+ if (!vm_page_reclaim_contig(req, 1, 0, MIPS_KSEG0_LARGEST_PHYS,
+ PAGE_SIZE, 0))
+ VM_WAIT;
#endif
}
@@ -1077,13 +1080,15 @@
{
vm_offset_t ptdva;
vm_page_t ptdpg;
- int i;
+ int i, req_class;
/*
* allocate the page directory page
*/
- while ((ptdpg = pmap_alloc_direct_page(NUSERPGTBLS, VM_ALLOC_NORMAL)) == NULL)
- pmap_grow_direct_page_cache();
+ req_class = VM_ALLOC_NORMAL;
+ while ((ptdpg = pmap_alloc_direct_page(NUSERPGTBLS, req_class)) ==
+ NULL)
+ pmap_grow_direct_page(req_class);
ptdva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(ptdpg));
pmap->pm_segtab = (pd_entry_t *)ptdva;
@@ -1107,15 +1112,17 @@
{
vm_offset_t pageva;
vm_page_t m;
+ int req_class;
/*
* Find or fabricate a new pagetable page
*/
- if ((m = pmap_alloc_direct_page(ptepindex, VM_ALLOC_NORMAL)) == NULL) {
+ req_class = VM_ALLOC_NORMAL;
+ if ((m = pmap_alloc_direct_page(ptepindex, req_class)) == NULL) {
if ((flags & PMAP_ENTER_NOSLEEP) == 0) {
PMAP_UNLOCK(pmap);
rw_wunlock(&pvh_global_lock);
- pmap_grow_direct_page_cache();
+ pmap_grow_direct_page(req_class);
rw_wlock(&pvh_global_lock);
PMAP_LOCK(pmap);
}
@@ -1241,9 +1248,10 @@
vm_page_t nkpg;
pd_entry_t *pde, *pdpe;
pt_entry_t *pte;
- int i;
+ int i, req_class;
mtx_assert(&kernel_map->system_mtx, MA_OWNED);
+ req_class = VM_ALLOC_INTERRUPT;
addr = roundup2(addr, NBSEG);
if (addr - 1 >= kernel_map->max_offset)
addr = kernel_map->max_offset;
@@ -1252,7 +1260,7 @@
#ifdef __mips_n64
if (*pdpe == 0) {
/* new intermediate page table entry */
- nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT);
+ nkpg = pmap_alloc_direct_page(nkpt, req_class);
if (nkpg == NULL)
panic("pmap_growkernel: no memory to grow kernel");
*pdpe = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg));
@@ -1272,8 +1280,13 @@
/*
* This index is bogus, but out of the way
*/
- nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT);
- if (!nkpg)
+ nkpg = pmap_alloc_direct_page(nkpt, req_class);
+#ifndef __mips_n64
+ if (nkpg == NULL && vm_page_reclaim_contig(req_class, 1,
+ 0, MIPS_KSEG0_LARGEST_PHYS, PAGE_SIZE, 0))
+ nkpg = pmap_alloc_direct_page(nkpt, req_class);
+#endif
+ if (nkpg == NULL)
panic("pmap_growkernel: no memory to grow kernel");
nkpt++;
*pde = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg));
Index: mips/mips/uma_machdep.c
===================================================================
--- mips/mips/uma_machdep.c
+++ mips/mips/uma_machdep.c
@@ -53,11 +53,16 @@
for (;;) {
m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, pflags);
+#ifndef __mips_n64
+ if (m == NULL && vm_page_reclaim_contig(pflags, 1,
+ 0, MIPS_KSEG0_LARGEST_PHYS, PAGE_SIZE, 0))
+ continue;
+#endif
if (m == NULL) {
if (wait & M_NOWAIT)
return (NULL);
else
- pmap_grow_direct_page_cache();
+ VM_WAIT;
} else
break;
}
Index: vm/vm_kern.c
===================================================================
--- vm/vm_kern.c
+++ vm/vm_kern.c
@@ -181,7 +181,10 @@
if (m == NULL) {
VM_OBJECT_WUNLOCK(object);
if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
- vm_pageout_grow_cache(tries, low, high);
+ if (!vm_page_reclaim_contig(pflags, 1,
+ low, high, PAGE_SIZE, 0) &&
+ (flags & M_WAITOK) != 0)
+ VM_WAIT;
VM_OBJECT_WLOCK(object);
tries++;
goto retry;
@@ -217,6 +220,7 @@
vm_offset_t addr, tmp;
vm_ooffset_t offset;
vm_page_t end_m, m;
+ u_long npages;
int pflags, tries;
size = round_page(size);
@@ -224,15 +228,18 @@
return (0);
offset = addr - VM_MIN_KERNEL_ADDRESS;
pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
+ npages = atop(size);
VM_OBJECT_WLOCK(object);
tries = 0;
retry:
m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags,
- atop(size), low, high, alignment, boundary, memattr);
+ npages, low, high, alignment, boundary, memattr);
if (m == NULL) {
VM_OBJECT_WUNLOCK(object);
if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) {
- vm_pageout_grow_cache(tries, low, high);
+ if (!vm_page_reclaim_contig(pflags, npages, low, high,
+ alignment, boundary) && (flags & M_WAITOK) != 0)
+ VM_WAIT;
VM_OBJECT_WLOCK(object);
tries++;
goto retry;
@@ -240,7 +247,7 @@
vmem_free(vmem, addr, size);
return (0);
}
- end_m = m + atop(size);
+ end_m = m + npages;
tmp = addr;
for (; m < end_m; m++) {
if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0)
@@ -528,6 +535,36 @@
vm_map_unlock(m);
}
+/*
+ * XXX
+ */
+static int
+debug_vm_compact(SYSCTL_HANDLER_ARGS)
+{
+ vm_offset_t va;
+ vm_paddr_t pa;
+ int error, i;
+
+ i = 0;
+ error = sysctl_handle_int(oidp, &i, 0, req);
+ if (error != 0)
+ return (error);
+ if (i != 0) {
+ pa = 0;
+ va = kmem_alloc_contig(kernel_arena, 1024 * PAGE_SIZE, M_WAITOK,
+ 0, 0x7fffffff, 1024 * PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
+ if (va != 0) {
+ pa = pmap_kextract(va);
+ kmem_free(kernel_arena, va, 1024 * PAGE_SIZE);
+ }
+ printf("va: %#jx -> pa: %#jx\n", (uintmax_t)va, (uintmax_t)pa);
+ }
+ return (0);
+}
+
+SYSCTL_PROC(_debug, OID_AUTO, vm_compact, CTLTYPE_INT | CTLFLAG_RW, 0, 0,
+ debug_vm_compact, "I", "set to trigger vm_compact event");
+
#ifdef DIAGNOSTIC
/*
* Allow userspace to directly trigger the VM drain routine for testing
Index: vm/vm_page.h
===================================================================
--- vm/vm_page.h
+++ vm/vm_page.h
@@ -474,6 +474,8 @@
boolean_t vm_page_ps_is_valid(vm_page_t m);
void vm_page_putfake(vm_page_t m);
void vm_page_readahead_finish(vm_page_t m);
+bool vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low,
+ vm_paddr_t high, u_long alignment, vm_paddr_t boundary);
void vm_page_reference(vm_page_t m);
void vm_page_remove (vm_page_t);
int vm_page_rename (vm_page_t, vm_object_t, vm_pindex_t);
@@ -482,6 +484,8 @@
void vm_page_requeue(vm_page_t m);
void vm_page_requeue_locked(vm_page_t m);
int vm_page_sbusied(vm_page_t m);
+vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start,
+ vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options);
void vm_page_set_valid_range(vm_page_t m, int base, int size);
int vm_page_sleep_if_busy(vm_page_t m, const char *msg);
vm_offset_t vm_page_startup(vm_offset_t vaddr);
Index: vm/vm_page.c
===================================================================
--- vm/vm_page.c
+++ vm/vm_page.c
@@ -158,11 +158,14 @@
static void vm_page_cache_turn_free(vm_page_t m);
static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
static void vm_page_enqueue(uint8_t queue, vm_page_t m);
+static void vm_page_free_wakeup(void);
static void vm_page_init_fakepg(void *dummy);
static int vm_page_insert_after(vm_page_t m, vm_object_t object,
vm_pindex_t pindex, vm_page_t mpred);
static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
vm_page_t mpred);
+static int vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run,
+ vm_paddr_t high);
SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init_fakepg, NULL);
@@ -2103,7 +2106,591 @@
return (m);
}
+#define VPSC_ANY 0 /* No restrictions. */
+#define VPSC_NORESERV 1 /* Skip reservations; implies VPSC_NOSUPER. */
+#define VPSC_NOSUPER 2 /* Skip superpages. */
+
/*
+ * vm_page_scan_contig:
+ *
+ * Scan vm_page_array[] between the specified entries "m_start" and
+ * "m_end" for a contiguous set of physical pages that satisfy the
+ * specified conditions, and return the lowest page in the set. The
+ * specified "alignment" determines the alignment of the lowest physical
+ * page in the set. If the specified "boundary" is non-zero, then the
+ * set of physical pages cannot span a physical address that is a
+ * multiple of "boundary".
+ *
+ * "m_end" is never dereferenced, so it need not point to a vm_page
+ * structure within vm_page_array[].
+ *
+ * "npages" must be greater than zero. "m_start" and "m_end" must not
+ * span a hole (or discontiguity) in the physical address space. Both
+ * "alignment" and "boundary" must be a power of two.
+ */
+vm_page_t
+vm_page_scan_contig(u_long npages, vm_page_t m_start, vm_page_t m_end,
+ u_long alignment, vm_paddr_t boundary, int options)
+{
+ struct mtx *m_mtx, *new_mtx;
+ vm_object_t object;
+ vm_paddr_t pa;
+ vm_page_t m, m_run;
+#if VM_NRESERVLEVEL > 0
+ int level;
+#endif
+ int m_inc, order, run_ext, run_len;
+
+ KASSERT(npages > 0, ("npages is 0"));
+ KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
+ KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
+ m_run = NULL;
+ run_len = 0;
+ m_mtx = NULL;
+ for (m = m_start; m < m_end && run_len < npages; m += m_inc) {
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0,
+ ("page %p is PG_FICTITIOUS or PG_MARKER", m));
+
+ /*
+ * If the current run is empty, check the page's physical
+ * address against the end, alignment, and boundary
+ * conditions. If the page doesn't satisfy these conditions,
+ * either terminate the scan or advance to the next page that
+ * satisfies the failed condition.
+ */
+ if (run_len == 0) {
+ KASSERT(m_run == NULL, ("m_run != NULL"));
+ if (m + npages > m_end)
+ break;
+ pa = VM_PAGE_TO_PHYS(m);
+ if ((pa & (alignment - 1)) != 0) {
+ m_inc = atop(roundup2(pa, alignment) - pa);
+ continue;
+ }
+ if (((pa ^ (pa + ptoa(npages) - 1)) & ~(boundary -
+ 1)) != 0) {
+ m_inc = atop(roundup2(pa, boundary) - pa);
+ continue;/*XXX*/
+ }
+ } else
+ KASSERT(m_run != NULL, ("m_run == NULL"));
+
+ /*
+ * Avoid releasing and reacquiring the same page lock.
+ */
+ new_mtx = vm_page_lockptr(m);
+ if (m_mtx != new_mtx) {
+ if (m_mtx != NULL)
+ mtx_unlock(m_mtx);
+ m_mtx = new_mtx;
+ mtx_lock(m_mtx);
+ }
+ m_inc = 1;
+retry:
+ if (m->wire_count != 0 || m->hold_count != 0)
+ run_ext = 0;
+#if VM_NRESERVLEVEL > 0
+ else if ((level = vm_reserv_level(m)) >= 0 &&
+ (options & VPSC_NORESERV) != 0) {
+ run_ext = 0;
+ /* Advance to the end of the reservation. */
+ pa = VM_PAGE_TO_PHYS(m);
+ m_inc = atop(roundup2(pa + 1, vm_reserv_size(level)) -
+ pa);
+ }
+#endif
+ else if ((object = m->object) != NULL) {
+ /*
+ * The page is considered eligible for relocation if
+ * and only if it could be laundered or reclaimed by
+ * the page daemon.
+ */
+ if (!VM_OBJECT_TRYRLOCK(object)) {
+ mtx_unlock(m_mtx);
+ VM_OBJECT_RLOCK(object);
+ mtx_lock(m_mtx);
+ if (m->object != object) {
+ /*
+ * The page may have been freed.
+ */
+ VM_OBJECT_RUNLOCK(object);
+ goto retry;
+ } else if (m->wire_count != 0 ||
+ m->hold_count != 0) {
+ run_ext = 0;
+ goto unlock;
+ }
+ }
+ KASSERT((m->flags & PG_UNHOLDFREE) == 0,
+ ("page %p is PG_UNHOLDFREE", m));
+ /* Don't care: PG_NODUMP, PG_WINATCFLS, PG_ZERO. */
+ if (object->type != OBJT_DEFAULT &&
+ object->type != OBJT_SWAP &&
+ object->type != OBJT_VNODE)
+ run_ext = 0;
+ else if ((m->flags & PG_CACHED) != 0 ||
+ m != vm_page_lookup(object, m->pindex)) {
+ /*
+ * The page is cached or recently converted
+ * from cached to free.
+ */
+#if VM_NRESERVLEVEL > 0
+ if (level >= 0) {
+ /*
+ * The page is reserved. Extend the
+ * current run by one page.
+ */
+ run_ext = 1;
+ } else
+#endif
+ if ((order = m->order) < VM_NFREEORDER) {
+ /*
+ * The page is enqueued in the
+ * physical memory allocator's cache/
+ * free page queues. Moreover, it is
+ * the first page in a power-of-two-
+ * sized set of contiguous cache/free
+ * pages. Add the entire set to the
+ * current run, and jump ahead.
+ */
+ run_ext = 1 << order;
+ m_inc = 1 << order;
+ } else
+ run_ext = 0;
+#if VM_NRESERVLEVEL > 0
+ } else if ((options & VPSC_NOSUPER) != 0 &&
+ (level = vm_reserv_level_iffullpop(m)) >= 0) {
+ run_ext = 0;
+ /* Advance to the end of the superpage. */
+ pa = VM_PAGE_TO_PHYS(m);
+ m_inc = atop(roundup2(pa + 1,
+ vm_reserv_size(level)) - pa);
+#endif
+ } else if (object->memattr == VM_MEMATTR_DEFAULT &&
+ m->queue != PQ_NONE && !vm_page_busied(m)) {
+ /*
+ * The page is allocated but eligible for
+ * relocation. Extend the current run by one
+ * page.
+ */
+ KASSERT(pmap_page_get_memattr(m) ==
+ VM_MEMATTR_DEFAULT,
+ ("page %p has an unexpected memattr", m));
+ KASSERT((m->oflags & (VPO_SWAPINPROG |
+ VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0,
+ ("page %p has unexpected oflags", m));
+ /* Don't care: VPO_NOSYNC. */
+ run_ext = 1;
+ } else
+ run_ext = 0;
+unlock:
+ VM_OBJECT_RUNLOCK(object);
+#if VM_NRESERVLEVEL > 0
+ } else if (level >= 0) {
+ /*
+ * The page is reserved but not yet allocated. In
+ * other words, it is still cached or free. Extend
+ * the current run by one page.
+ */
+ run_ext = 1;
+#endif
+ } else if ((order = m->order) < VM_NFREEORDER) {
+ /*
+ * The page is enqueued in the physical memory
+ * allocator's cache/free page queues. Moreover, it
+ * is the first page in a contiguous set of cache/free
+ * pages. Add the entire set to the current run, and
+ * jump ahead.
+ */
+ run_ext = 1 << order;
+ m_inc = 1 << order;
+ } else {
+ /*
+ * Skip the page: (1) It is enqueued in the physical
+ * memory allocator's cache/free page queues.
+ * However, it is not the first page in a contiguous
+ * set of cache/free pages. This case rarely occurs
+ * because the scan is performed in ascending
+ * order. (2) It is not reserved, and it is
+ * transitioning from free to allocated. Conversely,
+ * the transition from allocated to free for managed
+ * pages is blocked by the page lock. (3) It is
+ * allocated but not contained by an object and not
+ * wired, e.g., allocated by Xen's balloon driver.
+ */
+ run_ext = 0;
+ }
+
+ /*
+ * Extend or reset the current run of pages.
+ */
+ if (run_ext > 0) {
+ if (run_len == 0)
+ m_run = m;
+ run_len += run_ext;
+ } else {
+ if (run_len > 0) {
+ m_run = NULL;
+ run_len = 0;
+ }
+ }
+ }
+ if (m_mtx != NULL)
+ mtx_unlock(m_mtx);
+ if (run_len >= npages)
+ return (m_run);
+ return (NULL);
+}
+
+/*
+ * vm_page_reclaim_run:
+ *
+ * Try to relocate the allocated virtual pages within the specified run
+ * of physical pages. Free the physical pages underlying the relocated
+ * virtual pages. Returns 0 if every physical page within the run is
+ * either already free or successfully relocated. Otherwise, returns a
+ * non-zero value indicating why the relocation of a virtual page was
+ * unsuccessful.
+ *
+ * "req_class" must be an allocation class.
+ */
+static int
+vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run,
+ vm_paddr_t high)
+{
+ struct mtx *m_mtx, *new_mtx;
+ struct spglist free;
+ vm_object_t object;
+ vm_paddr_t pa;
+ vm_page_t m, m_end, m_new, m_tmp;
+ int error, order, req;
+// long case0 = 0, case2 = 0, case3 = 0;/*XXX*/
+
+ KASSERT((req_class & VM_ALLOC_CLASS_MASK) == req_class,
+ ("req_class is not an allocation class"));
+ SLIST_INIT(&free);
+ error = 0;
+ m = m_run;
+ m_end = m_run + npages;
+ m_mtx = NULL;
+ for (; error == 0 && m < m_end; m++) {
+ KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0,
+ ("page %p is PG_FICTITIOUS or PG_MARKER", m));
+
+ /*
+ * Avoid releasing and reacquiring the same page lock.
+ */
+ new_mtx = vm_page_lockptr(m);
+ if (m_mtx != new_mtx) {
+ if (m_mtx != NULL)
+ mtx_unlock(m_mtx);
+ m_mtx = new_mtx;
+ mtx_lock(m_mtx);
+ }
+retry:
+ if (m->wire_count != 0 || m->hold_count != 0)
+ error = EBUSY;
+ else if ((object = m->object) != NULL) {
+ /*
+ * The page is relocated if and only if it could be
+ * laundered or reclaimed by the page daemon.
+ */
+ if (!VM_OBJECT_TRYWLOCK(object)) {
+ mtx_unlock(m_mtx);
+ VM_OBJECT_WLOCK(object);
+ mtx_lock(m_mtx);
+ if (m->object != object) {
+ /*
+ * The page may have been freed.
+ */
+ VM_OBJECT_WUNLOCK(object);
+ goto retry;
+ } else if (m->wire_count != 0 ||
+ m->hold_count != 0) {
+ error = EBUSY;
+ goto unlock;
+ }
+ }
+ KASSERT((m->flags & PG_UNHOLDFREE) == 0,
+ ("page %p is PG_UNHOLDFREE", m));
+ /* Don't care: PG_NODUMP, PG_WINATCFLS, PG_ZERO. */
+ if (object->type != OBJT_DEFAULT &&
+ object->type != OBJT_SWAP &&
+ object->type != OBJT_VNODE)
+ error = EINVAL;
+ else if ((m->flags & PG_CACHED) != 0 ||
+ m != vm_page_lookup(object, m->pindex)) {
+ /*
+ * The page is cached or recently converted
+ * from cached to free.
+ */
+ VM_OBJECT_WUNLOCK(object);
+ goto cached;
+ } else if (object->memattr != VM_MEMATTR_DEFAULT)
+ error = EINVAL;
+ else if (m->queue != PQ_NONE && !vm_page_busied(m)) {
+ KASSERT(pmap_page_get_memattr(m) ==
+ VM_MEMATTR_DEFAULT,
+ ("page %p has an unexpected memattr", m));
+ KASSERT((m->oflags & (VPO_SWAPINPROG |
+ VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0,
+ ("page %p has unexpected oflags", m));
+ /* Don't care: VPO_NOSYNC. */
+ if (m->valid != 0) {
+ /*
+ * First, try to allocate a new page
+ * that is above "high". Failing
+ * that, try to allocate a new page
+ * that is below "m_run". Allocate
+ * the new page between the end of
+ * "m_run" and "high" only as a last
+ * resort.
+ */
+ req = req_class | VM_ALLOC_NOOBJ;
+ if ((m->flags & PG_NODUMP) != 0)
+ req |= VM_ALLOC_NODUMP;
+ if (trunc_page(high) !=
+ ~(vm_paddr_t)PAGE_MASK) {
+ m_new = vm_page_alloc_contig(
+ NULL, 0, req, 1,
+ round_page(high),
+ ~(vm_paddr_t)0,
+ PAGE_SIZE, 0,
+ VM_MEMATTR_DEFAULT);
+ } else
+ m_new = NULL;
+ if (m_new == NULL) {
+ pa = VM_PAGE_TO_PHYS(m_run);
+ m_new = vm_page_alloc_contig(
+ NULL, 0, req, 1,
+ 0, pa - 1, PAGE_SIZE, 0,
+ VM_MEMATTR_DEFAULT);
+ }
+ if (m_new == NULL) {
+ pa += ptoa(npages);
+ m_new = vm_page_alloc_contig(
+ NULL, 0, req, 1,
+ pa, high, PAGE_SIZE, 0,
+ VM_MEMATTR_DEFAULT);
+ }
+ if (m_new == NULL) {
+ error = ENOMEM;
+ goto unlock;
+ }
+ KASSERT(m_new->wire_count == 0,
+ ("page %p is wired", m));
+
+ /*
+ * Replace "m" with the new page. For
+ * vm_page_replace(), "m" must be busy
+ * and dequeued. Finally, change "m"
+ * as if vm_page_free() was called.
+ */
+ if (object->ref_count != 0)
+ pmap_remove_all(m);
+ m_new->aflags = m->aflags;
+ KASSERT(m_new->oflags == VPO_UNMANAGED,
+ ("page %p is managed", m));
+ m_new->oflags = m->oflags & VPO_NOSYNC;
+ pmap_copy_page(m, m_new);
+ m_new->valid = m->valid;
+ m_new->dirty = m->dirty;
+ m->flags &= ~PG_ZERO;
+ vm_page_xbusy(m);
+ vm_page_remque(m);
+ m_tmp = vm_page_replace(m_new, object,
+ m->pindex);
+ KASSERT(m_tmp == m,
+ ("replaced page %p, not page %p",
+ m_tmp, m));
+ m->valid = 0;
+ vm_page_undirty(m);
+
+ /*
+ * The new page must be deactivated
+ * before the object is unlocked.
+ */
+ new_mtx = vm_page_lockptr(m_new);
+ if (m_mtx != new_mtx) {
+ mtx_unlock(m_mtx);
+ m_mtx = new_mtx;
+ mtx_lock(m_mtx);
+ }
+ vm_page_deactivate(m_new);
+ } else {
+ m->flags &= ~PG_ZERO;
+ vm_page_remque(m);
+ vm_page_remove(m);
+ KASSERT(m->dirty == 0,
+ ("page %p is dirty", m));
+ }
+ SLIST_INSERT_HEAD(&free, m, plinks.s.ss);
+// case2++;/*XXX*/
+ } else
+ error = EBUSY;
+unlock:
+ VM_OBJECT_WUNLOCK(object);
+ } else {
+cached:
+ mtx_lock(&vm_page_queue_free_mtx);
+ order = m->order;
+ if (order < VM_NFREEORDER) {
+ /*
+ * The page is enqueued in the physical memory
+ * allocator's cache/free page queues.
+ * Moreover, it is the first page in a power-
+ * of-two-sized set of contiguous cache/free
+ * pages. Jump ahead to the last page within
+ * the set, and continue from there.
+ */
+ m += (1 << order) - 1;
+// case0 += 1 << order;/*XXX*/
+ }
+#if VM_NRESERVLEVEL > 0
+ else if (vm_reserv_is_page_free(m)) {
+ order = 0;
+// case3++;/*XXX*/
+ }
+#endif
+ mtx_unlock(&vm_page_queue_free_mtx);
+ if (order == VM_NFREEORDER)
+ error = EINVAL;
+ }
+ }
+ if (m_mtx != NULL)
+ mtx_unlock(m_mtx);
+ if ((m = SLIST_FIRST(&free)) != NULL) {
+ mtx_lock(&vm_page_queue_free_mtx);
+ do {
+ SLIST_REMOVE_HEAD(&free, plinks.s.ss);
+ vm_phys_freecnt_adj(m, 1);
+#if VM_NRESERVLEVEL > 0
+ if (!vm_reserv_free_page(m))
+#else
+ if (true)
+#endif
+ vm_phys_free_pages(m, 0);
+ } while ((m = SLIST_FIRST(&free)) != NULL);
+ vm_page_zero_idle_wakeup();
+ vm_page_free_wakeup();
+ mtx_unlock(&vm_page_queue_free_mtx);
+ }
+// order = m_run->order;
+// printf("error: %d, case0: %ld, case2: %ld, case3: %ld, order: %d\n",
+// error, case0, case2, case3, order); /*XXX*/
+ return (error);
+}
+
+#define NRUNS 16
+
+CTASSERT(powerof2(NRUNS));
+
+#define RUN_INDEX(count) ((count) & (NRUNS - 1))
+
+#define MIN_RECLAIM 8
+
+/*
+ * vm_page_reclaim_contig:
+ *
+ * Reclaim physical memory contiguity by relocating virtual pages. XXX
+ *
+ * The caller must always specify an allocation class through "req".
+ *
+ * allocation classes:
+ * VM_ALLOC_NORMAL normal process request
+ * VM_ALLOC_SYSTEM system *really* needs a page
+ * VM_ALLOC_INTERRUPT interrupt time request
+ *
+ * The optional allocation flags are ignored.
+ *
+ * "npages" must be greater than zero. Both "alignment" and "boundary"
+ * must be a power of two.
+ */
+bool
+vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high,
+ u_long alignment, vm_paddr_t boundary)
+{
+ vm_paddr_t curr_low;
+ vm_page_t m_run, m_runs[NRUNS];
+ u_long count, reclaimed;
+ int error, i, options, req_class;
+
+ KASSERT(npages > 0, ("npages is 0"));
+ KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
+ KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
+ req_class = req & VM_ALLOC_CLASS_MASK;
+
+ /*
+ * The page daemon is allowed to dig deeper into the free page list.
+ */
+ if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
+ req_class = VM_ALLOC_SYSTEM;
+
+ /*
+ * Return if the number of cached and free pages cannot satisfy the
+ * requested allocation.
+ */
+ count = vm_cnt.v_free_count + vm_cnt.v_cache_count;
+ if (count < npages + vm_cnt.v_free_reserved || (count < npages +
+ vm_cnt.v_interrupt_free_min && req_class == VM_ALLOC_SYSTEM) ||
+ (count < npages && req_class == VM_ALLOC_INTERRUPT))
+ return (false);
+
+ /*
+ * Scan up to three times, relaxing the restrictions ("options") on
+ * the reclamation of reservations and superpages each time.
+ */
+ for (options = VPSC_NORESERV;;) {
+ /*
+ * Find the highest runs that satisfy the given constraints
+ * and restrictions, and record them in "m_runs".
+ */
+ curr_low = low;
+ count = 0;
+ for (;;) {
+ m_run = vm_phys_scan_contig(npages, curr_low, high,
+ alignment, boundary, options);
+ if (m_run == NULL)
+ break;
+ curr_low = VM_PAGE_TO_PHYS(m_run) + ptoa(npages);
+ m_runs[RUN_INDEX(count)] = m_run;
+ count++;
+ }
+
+ /*
+ * Reclaim the highest runs in LIFO (descending) order until
+ * the number of reclaimed pages, "reclaimed", is at least
+ * MIN_RECLAIM. Reset "reclaimed" each time because each
+ * reclamation is idempotent, and runs will (likely) recur
+ * from one scan to the next as restrictions are relaxed.
+ */
+ reclaimed = 0;
+ for (i = 0; count > 0 && i < NRUNS; i++) {
+ count--;
+ m_run = m_runs[RUN_INDEX(count)];
+ error = vm_page_reclaim_run(req_class, npages, m_run,
+ high);
+ if (error == 0) {
+ reclaimed += npages;
+ if (reclaimed >= MIN_RECLAIM)
+ return (true);
+ }
+ }
+
+ /*
+ * Either relax the restrictions on the next scan or return if
+ * the last scan had no restrictions.
+ */
+ if (options == VPSC_NORESERV)
+ options = VPSC_NOSUPER;
+ else if (options == VPSC_NOSUPER)
+ options = VPSC_ANY;
+ else if (options == VPSC_ANY)
+ return (reclaimed != 0);
+ }
+}
+
+/*
* vm_wait: (also see VM_WAIT macro)
*
* Sleep until free pages are available for allocation.
Index: vm/vm_pageout.h
===================================================================
--- vm/vm_pageout.h
+++ vm/vm_pageout.h
@@ -101,7 +101,6 @@
#ifdef _KERNEL
int vm_pageout_flush(vm_page_t *, int, int, int, int *, boolean_t *);
-void vm_pageout_grow_cache(int, vm_paddr_t, vm_paddr_t);
void vm_pageout_oom(int shortage);
#endif
#endif /* _VM_VM_PAGEOUT_H_ */
Index: vm/vm_pageout.c
===================================================================
--- vm/vm_pageout.c
+++ vm/vm_pageout.c
@@ -237,8 +237,6 @@
CTLFLAG_RW, &vm_page_max_wired, 0, "System-wide limit to wired page count");
static boolean_t vm_pageout_fallback_object_lock(vm_page_t, vm_page_t *);
-static boolean_t vm_pageout_launder(struct vm_pagequeue *pq, int, vm_paddr_t,
- vm_paddr_t);
#if !defined(NO_SWAPPING)
static void vm_pageout_map_deactivate_pages(vm_map_t, long);
static void vm_pageout_object_deactivate_pages(pmap_t, vm_object_t, long);
@@ -595,170 +593,6 @@
return (numpagedout);
}
-static boolean_t
-vm_pageout_launder(struct vm_pagequeue *pq, int tries, vm_paddr_t low,
- vm_paddr_t high)
-{
- struct mount *mp;
- struct vnode *vp;
- vm_object_t object;
- vm_paddr_t pa;
- vm_page_t m, m_tmp, next;
- int lockmode;
-
- vm_pagequeue_lock(pq);
- TAILQ_FOREACH_SAFE(m, &pq->pq_pl, plinks.q, next) {
- if ((m->flags & PG_MARKER) != 0)
- continue;
- pa = VM_PAGE_TO_PHYS(m);
- if (pa < low || pa + PAGE_SIZE > high)
- continue;
- if (!vm_pageout_page_lock(m, &next) || m->hold_count != 0) {
- vm_page_unlock(m);
- continue;
- }
- object = m->object;
- if ((!VM_OBJECT_TRYWLOCK(object) &&
- (!vm_pageout_fallback_object_lock(m, &next) ||
- m->hold_count != 0)) || vm_page_busied(m)) {
- vm_page_unlock(m);
- VM_OBJECT_WUNLOCK(object);
- continue;
- }
- vm_page_test_dirty(m);
- if (m->dirty == 0 && object->ref_count != 0)
- pmap_remove_all(m);
- if (m->dirty != 0) {
- vm_page_unlock(m);
- if (tries == 0 || (object->flags & OBJ_DEAD) != 0) {
- VM_OBJECT_WUNLOCK(object);
- continue;
- }
- if (object->type == OBJT_VNODE) {
- vm_pagequeue_unlock(pq);
- vp = object->handle;
- vm_object_reference_locked(object);
- VM_OBJECT_WUNLOCK(object);
- (void)vn_start_write(vp, &mp, V_WAIT);
- lockmode = MNT_SHARED_WRITES(vp->v_mount) ?
- LK_SHARED : LK_EXCLUSIVE;
- vn_lock(vp, lockmode | LK_RETRY);
- VM_OBJECT_WLOCK(object);
- vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
- VM_OBJECT_WUNLOCK(object);
- VOP_UNLOCK(vp, 0);
- vm_object_deallocate(object);
- vn_finished_write(mp);
- return (TRUE);
- } else if (object->type == OBJT_SWAP ||
- object->type == OBJT_DEFAULT) {
- vm_pagequeue_unlock(pq);
- m_tmp = m;
- vm_pageout_flush(&m_tmp, 1, VM_PAGER_PUT_SYNC,
- 0, NULL, NULL);
- VM_OBJECT_WUNLOCK(object);
- return (TRUE);
- }
- } else {
- /*
- * Dequeue here to prevent lock recursion in
- * vm_page_cache().
- */
- vm_page_dequeue_locked(m);
- vm_page_cache(m);
- vm_page_unlock(m);
- }
- VM_OBJECT_WUNLOCK(object);
- }
- vm_pagequeue_unlock(pq);
- return (FALSE);
-}
-
-/*
- * Increase the number of cached pages. The specified value, "tries",
- * determines which categories of pages are cached:
- *
- * 0: All clean, inactive pages within the specified physical address range
- * are cached. Will not sleep.
- * 1: The vm_lowmem handlers are called. All inactive pages within
- * the specified physical address range are cached. May sleep.
- * 2: The vm_lowmem handlers are called. All inactive and active pages
- * within the specified physical address range are cached. May sleep.
- */
-void
-vm_pageout_grow_cache(int tries, vm_paddr_t low, vm_paddr_t high)
-{
- int actl, actmax, inactl, inactmax, dom, initial_dom;
- static int start_dom = 0;
-
- if (tries > 0) {
- /*
- * Decrease registered cache sizes. The vm_lowmem handlers
- * may acquire locks and/or sleep, so they can only be invoked
- * when "tries" is greater than zero.
- */
- SDT_PROBE0(vm, , , vm__lowmem_cache);
- EVENTHANDLER_INVOKE(vm_lowmem, 0);
-
- /*
- * We do this explicitly after the caches have been drained
- * above.
- */
- uma_reclaim();
- }
-
- /*
- * Make the next scan start on the next domain.
- */
- initial_dom = atomic_fetchadd_int(&start_dom, 1) % vm_ndomains;
-
- inactl = 0;
- inactmax = vm_cnt.v_inactive_count;
- actl = 0;
- actmax = tries < 2 ? 0 : vm_cnt.v_active_count;
- dom = initial_dom;
-
- /*
- * Scan domains in round-robin order, first inactive queues,
- * then active. Since domain usually owns large physically
- * contiguous chunk of memory, it makes sense to completely
- * exhaust one domain before switching to next, while growing
- * the pool of contiguous physical pages.
- *
- * Do not even start launder a domain which cannot contain
- * the specified address range, as indicated by segments
- * constituting the domain.
- */
-again_inact:
- if (inactl < inactmax) {
- if (vm_phys_domain_intersects(vm_dom[dom].vmd_segs,
- low, high) &&
- vm_pageout_launder(&vm_dom[dom].vmd_pagequeues[PQ_INACTIVE],
- tries, low, high)) {
- inactl++;
- goto again_inact;
- }
- if (++dom == vm_ndomains)
- dom = 0;
- if (dom != initial_dom)
- goto again_inact;
- }
-again_act:
- if (actl < actmax) {
- if (vm_phys_domain_intersects(vm_dom[dom].vmd_segs,
- low, high) &&
- vm_pageout_launder(&vm_dom[dom].vmd_pagequeues[PQ_ACTIVE],
- tries, low, high)) {
- actl++;
- goto again_act;
- }
- if (++dom == vm_ndomains)
- dom = 0;
- if (dom != initial_dom)
- goto again_act;
- }
-}
-
#if !defined(NO_SWAPPING)
/*
* vm_pageout_object_deactivate_pages
Index: vm/vm_phys.h
===================================================================
--- vm/vm_phys.h
+++ vm/vm_phys.h
@@ -84,6 +84,8 @@
void vm_phys_free_pages(vm_page_t m, int order);
void vm_phys_init(void);
vm_page_t vm_phys_paddr_to_vm_page(vm_paddr_t pa);
+vm_page_t vm_phys_scan_contig(u_long npages, vm_paddr_t low, vm_paddr_t high,
+ u_long alignment, vm_paddr_t boundary, int options);
void vm_phys_set_pool(int pool, vm_page_t m, int order);
boolean_t vm_phys_unfree_page(vm_page_t m);
boolean_t vm_phys_zero_pages_idle(void);
Index: vm/vm_phys.c
===================================================================
--- vm/vm_phys.c
+++ vm/vm_phys.c
@@ -1163,6 +1163,56 @@
}
/*
+ * Scan physical memory between the specified addresses "low" and "high" for a
+ * contiguous set of physical pages that satisfy the specified conditions, and
+ * return the lowest page in the set. The specified "alignment" determines
+ * the alignment of the lowest physical page in the set. If the specified
+ * "boundary" is non-zero, then the set of physical pages cannot span a
+ * physical address that is a multiple of "boundary".
+ *
+ * "npages" must be greater than zero. Both "alignment" and "boundary" must
+ * be a power of two.
+ */
+vm_page_t
+vm_phys_scan_contig(u_long npages, vm_paddr_t low, vm_paddr_t high,
+ u_long alignment, vm_paddr_t boundary, int options)
+{
+ vm_paddr_t pa_end;
+ vm_page_t m_end, m_run, m_start;
+ struct vm_phys_seg *seg;
+ int segind;
+
+ KASSERT(npages > 0, ("npages is 0"));
+ KASSERT(powerof2(alignment), ("alignment is not a power of 2"));
+ KASSERT(powerof2(boundary), ("boundary is not a power of 2"));
+ if (low >= high)
+ return (NULL);
+ for (segind = 0; segind < vm_phys_nsegs; segind++) {
+ seg = &vm_phys_segs[segind];
+ if (seg->start >= high)
+ break;
+ if (low >= seg->end)
+ continue;
+ if (low <= seg->start)
+ m_start = seg->first_page;
+ else
+ m_start = &seg->first_page[atop(low - seg->start)];
+ if (high < seg->end)
+ pa_end = high;
+ else
+ pa_end = seg->end;
+ if (pa_end - VM_PAGE_TO_PHYS(m_start) < ptoa(npages))
+ continue;
+ m_end = &seg->first_page[atop(pa_end - seg->start)];
+ m_run = vm_page_scan_contig(npages, m_start, m_end,
+ alignment, boundary, options);
+ if (m_run != NULL)
+ return (m_run);
+ }
+ return (NULL);
+}
+
+/*
* Set the pool for a contiguous, power of two-sized set of physical pages.
*/
void
Index: vm/vm_reserv.h
===================================================================
--- vm/vm_reserv.h
+++ vm/vm_reserv.h
@@ -53,6 +53,8 @@
void vm_reserv_break_all(vm_object_t object);
boolean_t vm_reserv_free_page(vm_page_t m);
void vm_reserv_init(void);
+bool vm_reserv_is_page_free(vm_page_t m);
+int vm_reserv_level(vm_page_t m);
int vm_reserv_level_iffullpop(vm_page_t m);
boolean_t vm_reserv_reactivate_page(vm_page_t m);
boolean_t vm_reserv_reclaim_contig(u_long npages, vm_paddr_t low,
@@ -60,6 +62,7 @@
boolean_t vm_reserv_reclaim_inactive(void);
void vm_reserv_rename(vm_page_t m, vm_object_t new_object,
vm_object_t old_object, vm_pindex_t old_object_offset);
+int vm_reserv_size(int level);
vm_paddr_t vm_reserv_startup(vm_offset_t *vaddr, vm_paddr_t end,
vm_paddr_t high_water);
Index: vm/vm_reserv.c
===================================================================
--- vm/vm_reserv.c
+++ vm/vm_reserv.c
@@ -866,6 +866,35 @@
}
/*
+ * Returns true if the given page belongs to a reservation and that page is
+ * free. Otherwise, returns false.
+ */
+bool
+vm_reserv_is_page_free(vm_page_t m)
+{
+ vm_reserv_t rv;
+
+ mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
+ rv = vm_reserv_from_page(m);
+ if (rv->object == NULL)
+ return (false);
+ return (popmap_is_clear(rv->popmap, m - rv->pages));
+}
+
+/*
+ * If the given page belongs to a reservation, returns the level of that
+ * reservation. Otherwise, returns -1.
+ */
+int
+vm_reserv_level(vm_page_t m)
+{
+ vm_reserv_t rv;
+
+ rv = vm_reserv_from_page(m);
+ return (rv->object != NULL ? 0 : -1);
+}
+
+/*
* Returns a reservation level if the given page belongs to a fully-populated
* reservation and -1 otherwise.
*/
@@ -1076,6 +1105,23 @@
}
/*
+ * Returns the size (in bytes) of a reservation of the specified level.
+ */
+int
+vm_reserv_size(int level)
+{
+
+ switch (level) {
+ case 0:
+ return (VM_LEVEL_0_SIZE);
+ case -1:
+ return (PAGE_SIZE);
+ default:
+ return (0);
+ }
+}
+
+/*
* Allocates the virtual and physical memory required by the reservation
* management system's data structures, in particular, the reservation array.
*/
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Thu, Aug 27, 4:14 PM (15 m, 4 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
37400603
Default Alt Text
D4444.id11249.diff (36 KB)
Attached To
Mode
D4444: Handle contiguous memory allocation failures through page relocation.
Attached
Detach File
Event Timeline
Log In to Comment