D46620.id143361.diff
No OneTemporary
Actions

Size

13 KB

Referenced Files

None

Subscribers

None

D46620.id143361.diff
View Options

	Index: sys/fs/tmpfs/tmpfs_vnops.c
	===================================================================
	--- sys/fs/tmpfs/tmpfs_vnops.c
	+++ sys/fs/tmpfs/tmpfs_vnops.c
	@@ -48,6 +48,7 @@
	#include <sys/lock.h>
	#include <sys/mount.h>
	#include <sys/namei.h>
	+#include <sys/pctrie.h>
	#include <sys/priv.h>
	#include <sys/proc.h>
	#include <sys/rwlock.h>
	@@ -2098,11 +2099,13 @@
	static off_t
	tmpfs_seek_data_locked(vm_object_t obj, off_t noff)
	{
	+ struct pctrie_iter blks;
	vm_page_t m;
	vm_pindex_t p, p_m, p_swp;

	p = OFF_TO_IDX(noff);
	m = vm_page_find_least(obj, p);
	+ swblk_iter_init(&blks, obj);

	/*
	* Microoptimize the most common case for SEEK_DATA, where
	@@ -2111,7 +2114,7 @@
	if (m != NULL && vm_page_any_valid(m) && m->pindex == p)
	return (noff);

	- p_swp = swap_pager_find_least(obj, p);
	+ p_swp = swap_pager_find_least(&blks, p);
	if (p_swp == p)
	return (noff);

	@@ -2139,9 +2142,11 @@
	static off_t
	tmpfs_seek_hole_locked(vm_object_t obj, off_t noff)
	{
	+ struct pctrie_iter blks;
	vm_page_t m;
	vm_pindex_t p, p_swp;

	+ swblk_iter_init(&blks, obj);
	for (;; noff = tmpfs_seek_next(noff)) {
	/*
	* Walk over the largest sequential run of the valid pages.
	@@ -2156,7 +2161,7 @@
	* there is a hole in the swap at the same place.
	*/
	p = OFF_TO_IDX(noff);
	- p_swp = swap_pager_find_least(obj, p);
	+ p_swp = swap_pager_find_least(&blks, p);
	if (p_swp != p) {
	noff = IDX_TO_OFF(p);
	break;
	Index: sys/vm/swap_pager.h
	===================================================================
	--- sys/vm/swap_pager.h
	+++ sys/vm/swap_pager.h
	@@ -40,6 +40,7 @@
	#include <sys/_types.h>

	struct buf;
	+struct pctrie_iter;
	struct swdevt;
	struct thread;
	typedef void sw_strategy_t(struct buf , struct swdevt );
	@@ -74,7 +75,8 @@
	struct xswdev;
	int swap_dev_info(int name, struct xswdev xs, char devname, size_t len);
	void swap_pager_copy(vm_object_t, vm_object_t, vm_pindex_t, int);
	-vm_pindex_t swap_pager_find_least(vm_object_t object, vm_pindex_t pindex);
	+void swblk_iter_init(struct pctrie_iter *blks, vm_object_t object);
	+vm_pindex_t swap_pager_find_least(struct pctrie_iter *blks, vm_pindex_t pindex);
	void swap_pager_freespace(vm_object_t object, vm_pindex_t start,
	vm_size_t size, vm_size_t *freed);
	void swap_pager_swap_init(void);
	Index: sys/vm/swap_pager.c
	===================================================================
	--- sys/vm/swap_pager.c
	+++ sys/vm/swap_pager.c
	@@ -531,39 +531,47 @@

	PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free);

	-static struct swblk *
	-swblk_lookup(vm_object_t object, vm_pindex_t pindex)
	+void
	+swblk_iter_init(struct pctrie_iter *blks, vm_object_t object)
	{
	- return (SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
	- rounddown(pindex, SWAP_META_PAGES)));
	+ VM_OBJECT_ASSERT_LOCKED(object);
	+ MPASS((object->flags & OBJ_SWAP) != 0);
	+ pctrie_iter_init(blks, &object->un_pager.swp.swp_blks);
	+}
	+
	+static void
	+swblk_iter_limit_init(struct pctrie_iter *blks, vm_object_t object,
	+ vm_pindex_t limit)
	+{
	+ VM_OBJECT_ASSERT_LOCKED(object);
	+ MPASS((object->flags & OBJ_SWAP) != 0);
	+ pctrie_iter_limit_init(blks, &object->un_pager.swp.swp_blks, limit);
	}

	static struct swblk *
	-swblk_start(vm_object_t object, vm_pindex_t pindex)
	+swblk_lookup(vm_object_t object, vm_pindex_t pindex)
	{
	- return (SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
	+ return (SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
	rounddown(pindex, SWAP_META_PAGES)));
	}

	static struct swblk *
	-swblk_next(vm_object_t object, struct swblk *sb)
	+swblk_start(struct pctrie_iter *blks, vm_pindex_t pindex)
	{
	- return (swblk_start(object, sb->p + SWAP_META_PAGES));
	+ return (SWAP_PCTRIE_ITER_LOOKUP_GE(blks,
	+ rounddown(pindex, SWAP_META_PAGES)));
	}

	static struct swblk *
	-swblk_start_limit(vm_object_t object, vm_pindex_t pindex, vm_pindex_t limit)
	+swblk_next(struct pctrie_iter *blks)
	{
	- struct swblk *sb = swblk_start(object, pindex);
	- if (sb != NULL && sb->p < limit)
	- return (sb);
	- return (NULL);
	+ return (SWAP_PCTRIE_ITER_JUMP_GE(blks, SWAP_META_PAGES));
	}

	-static struct swblk *
	-swblk_next_limit(vm_object_t object, struct swblk *sb, vm_pindex_t limit)
	+static void
	+swblk_remove(struct pctrie_iter *blks)
	{
	- return (swblk_start_limit(object, sb->p + SWAP_META_PAGES, limit));
	+ SWAP_PCTRIE_ITER_REMOVE(blks);
	}

	static void
	@@ -1827,6 +1835,7 @@
	u_long
	swap_pager_swapped_pages(vm_object_t object)
	{
	+ struct pctrie_iter blks;
	struct swblk *sb;
	u_long res;
	int i;
	@@ -1837,8 +1846,8 @@
	return (0);

	res = 0;
	- for (sb = swblk_start(object, 0); sb != NULL;
	- sb = swblk_next(object, sb)) {
	+ swblk_iter_init(&blks, object);
	+ for (sb = swblk_start(&blks, 0); sb != NULL; sb = swblk_next(&blks)) {
	for (i = 0; i < SWAP_META_PAGES; i++) {
	if (sb->d[i] != SWAPBLK_NONE)
	res++;
	@@ -1856,6 +1865,7 @@
	static void
	swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object)
	{
	+ struct pctrie_iter blks;
	struct page_range range;
	struct swblk *sb;
	vm_page_t m;
	@@ -1869,34 +1879,21 @@
	("%s: Object not swappable", __func__));

	pi = 0;
	- i = 0;
	swp_pager_init_freerange(&range);
	- for (;;) {
	- if (i == 0 && (object->flags & OBJ_DEAD) != 0) {
	- /*
	- * Make sure that pending writes finish before
	- * returning.
	- */
	- vm_object_pip_wait(object, "swpoff");
	- swp_pager_meta_free_all(object);
	- break;
	- }
	-
	+restart:
	+ swblk_iter_init(&blks, object);
	+ sb = swblk_start(&blks, pi);
	+reloop:
	+ for (i = 0, sb_empty = true, m = NULL;
	+ sb != NULL && !(i == 0 && (object->flags & OBJ_DEAD) != 0); i++) {
	if (i == SWAP_META_PAGES) {
	- pi = sb->p + SWAP_META_PAGES;
	+ pi = blks.index + SWAP_META_PAGES;
	if (sb_empty) {
	- swblk_lookup_remove(object, sb);
	+ swblk_remove(&blks);
	uma_zfree(swblk_zone, sb);
	}
	- i = 0;
	- }
	-
	- if (i == 0) {
	- sb = swblk_start(object, pi);
	- if (sb == NULL)
	- break;
	- sb_empty = true;
	- m = NULL;
	+ sb = swblk_next(&blks);
	+ goto reloop;
	}

	/* Skip an invalid block. */
	@@ -1905,7 +1902,6 @@
	if (blk != SWAPBLK_NONE)
	sb_empty = false;
	m = NULL;
	- i++;
	continue;
	}

	@@ -1914,13 +1910,12 @@
	* page has pending operations, sleep and restart the scan.
	*/
	m = m != NULL ? vm_page_next(m) :
	- vm_page_lookup(object, sb->p + i);
	+ vm_page_lookup(object, blks.index + i);
	if (m != NULL && (m->oflags & VPO_SWAPINPROG) != 0) {
	m->oflags \|= VPO_SWAPSLEEP;
	VM_OBJECT_SLEEP(object, &object->handle, PSWP, "swpoff",
	0);
	- i = 0; /* Restart scan after object lock dropped. */
	- continue;
	+ goto restart;
	}

	/*
	@@ -1929,20 +1924,21 @@
	*/
	if (m != NULL && vm_page_all_valid(m)) {
	swp_pager_force_dirty(&range, m, &sb->d[i]);
	- i++;
	continue;
	}

	/* Is there a page we can acquire or allocate? */
	if (m == NULL) {
	- m = vm_page_alloc(object, sb->p + i,
	+ m = vm_page_alloc(object, blks.index + i,
	VM_ALLOC_NORMAL \| VM_ALLOC_WAITFAIL);
	} else if (!vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL))
	m = NULL;

	/* If no page available, repeat this iteration. */
	- if (m == NULL)
	+ if (m == NULL) {
	+ i--;
	continue;
	+ }

	/* Get the page from swap, mark it dirty, restart the scan. */
	vm_object_pip_add(object, 1);
	@@ -1956,7 +1952,14 @@
	("%s: Page %p not all valid", __func__, m));
	swp_pager_force_dirty(&range, m, &sb->d[i]);
	vm_page_xunbusy(m);
	- i = 0; /* Restart scan after object lock dropped. */
	+ goto restart;
	+ }
	+ if (i == 0 && (object->flags & OBJ_DEAD) != 0) {
	+ /*
	+ * Make sure that pending writes finish before returning.
	+ */
	+ vm_object_pip_wait(object, "swpoff");
	+ swp_pager_meta_free_all(object);
	}
	swp_pager_freeswapspace(&range);
	}
	@@ -2197,13 +2200,12 @@
	swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject,
	vm_pindex_t pindex, vm_pindex_t count)
	{
	+ struct pctrie_iter blks;
	struct page_range range;
	struct swblk *sb;
	daddr_t blk, d[SWAP_META_PAGES];
	- vm_pindex_t offset, last;
	+ vm_pindex_t last;
	int d_mask, i, limit, start;
	- _Static_assert(8 * sizeof(d_mask) >= SWAP_META_PAGES,
	- "d_mask not big enough");

	VM_OBJECT_ASSERT_WLOCKED(srcobject);
	VM_OBJECT_ASSERT_WLOCKED(dstobject);
	@@ -2213,20 +2215,16 @@

	swp_pager_init_freerange(&range);
	d_mask = 0;
	- offset = pindex;
	last = pindex + count;
	- sb = swblk_start_limit(srcobject, pindex, last);
	- start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0;
	- for (; sb != NULL;
	- sb = swblk_start_limit(srcobject, pindex, last), start = 0) {
	- pindex = sb->p;
	- MPASS(d_mask == 0);
	- limit = MIN(last - pindex, SWAP_META_PAGES);
	+ swblk_iter_limit_init(&blks, srcobject, last);
	+ for (sb = swblk_start(&blks, pindex), start = pindex % SWAP_META_PAGES;
	+ sb != NULL; sb = swblk_next(&blks), start = 0) {
	+ limit = MIN(last - blks.index, SWAP_META_PAGES);
	for (i = start; i < limit; i++) {
	if (sb->d[i] == SWAPBLK_NONE)
	continue;
	blk = swp_pager_meta_build(dstobject,
	- pindex + i - offset, sb->d[i], true);
	+ blks.index + i - pindex, sb->d[i], true);
	if (blk == sb->d[i]) {
	/*
	* Failed memory allocation stopped transfer;
	@@ -2241,7 +2239,7 @@
	}
	if (swp_pager_swblk_empty(sb, 0, start) &&
	swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
	- swblk_lookup_remove(srcobject, sb);
	+ swblk_remove(&blks);
	uma_zfree(swblk_zone, sb);
	}
	if (d_mask != 0) {
	@@ -2250,12 +2248,12 @@
	do {
	i = ffs(d_mask) - 1;
	swp_pager_meta_build(dstobject,
	- pindex + i - offset, d[i], false);
	+ blks.index + i - pindex, d[i], false);
	d_mask &= ~(1 << i);
	} while (d_mask != 0);
	VM_OBJECT_WLOCK(srcobject);
	+ pctrie_iter_reset(&blks);
	}
	- pindex += SWAP_META_PAGES;
	}
	swp_pager_freeswapspace(&range);
	}
	@@ -2274,6 +2272,7 @@
	swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count,
	vm_size_t *freed)
	{
	+ struct pctrie_iter blks;
	struct page_range range;
	struct swblk *sb;
	vm_page_t m;
	@@ -2290,28 +2289,27 @@

	swp_pager_init_freerange(&range);
	last = pindex + count;
	- sb = swblk_start_limit(object, pindex, last);
	- start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0;
	- for (; sb != NULL;
	- sb = swblk_start_limit(object, pindex, last), start = 0) {
	- limit = MIN(last - sb->p, SWAP_META_PAGES);
	+ swblk_iter_limit_init(&blks, object, last);
	+ for (sb = swblk_start(&blks, pindex), start = pindex % SWAP_META_PAGES;
	+ sb != NULL; sb = swblk_next(&blks), start = 0) {
	+ limit = MIN(last - blks.index, SWAP_META_PAGES);
	for (i = start; i < limit; i++) {
	if (sb->d[i] == SWAPBLK_NONE)
	continue;
	swp_pager_update_freerange(&range, sb->d[i]);
	if (freed != NULL) {
	- m = (m != NULL && m->pindex == sb->p + i - 1) ?
	+ m = (m != NULL &&
	+ m->pindex == blks.index + i - 1) ?
	vm_page_next(m) :
	- vm_page_lookup(object, sb->p + i);
	+ vm_page_lookup(object, blks.index + i);
	if (m == NULL \|\| vm_page_none_valid(m))
	fc++;
	}
	sb->d[i] = SWAPBLK_NONE;
	}
	- pindex = sb->p + SWAP_META_PAGES;
	if (swp_pager_swblk_empty(sb, 0, start) &&
	swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
	- swblk_lookup_remove(object, sb);
	+ swblk_remove(&blks);
	uma_zfree(swblk_zone, sb);
	}
	}
	@@ -2389,24 +2387,24 @@
	* if are no allocated swap blocks for the object after the requested pindex.
	*/
	vm_pindex_t
	-swap_pager_find_least(vm_object_t object, vm_pindex_t pindex)
	+swap_pager_find_least(struct pctrie_iter *blks, vm_pindex_t pindex)
	{
	struct swblk *sb;
	int i;

	- if ((sb = swblk_start(object, pindex)) == NULL)
	+ if ((sb = swblk_start(blks, pindex)) == NULL)
	return (OBJ_MAX_SIZE);
	- if (sb->p < pindex) {
	+ if (blks->index < pindex) {
	for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) {
	if (sb->d[i] != SWAPBLK_NONE)
	- return (sb->p + i);
	+ return (blks->index + i);
	}
	- if ((sb = swblk_next(object, sb)) == NULL)
	+ if ((sb = swblk_next(blks)) == NULL)
	return (OBJ_MAX_SIZE);
	}
	for (i = 0; i < SWAP_META_PAGES; i++) {
	if (sb->d[i] != SWAPBLK_NONE)
	- return (sb->p + i);
	+ return (blks->index + i);
	}

	/*
	@@ -2844,13 +2842,14 @@
	long
	vmspace_swap_count(struct vmspace *vmspace)
	{
	+ struct pctrie_iter blks;
	vm_map_t map;
	vm_map_entry_t cur;
	vm_object_t object;
	struct swblk *sb;
	vm_pindex_t e, pi;
	long count;
	- int i;
	+ int i, limit, start;

	map = &vmspace->vm_map;
	count = 0;
	@@ -2866,11 +2865,12 @@
	goto unlock;
	pi = OFF_TO_IDX(cur->offset);
	e = pi + OFF_TO_IDX(cur->end - cur->start);
	- for (sb = swblk_start_limit(object, pi, e);
	- sb != NULL; sb = swblk_next_limit(object, sb, e)) {
	- for (i = 0; i < SWAP_META_PAGES; i++) {
	- if (sb->p + i < e &&
	- sb->d[i] != SWAPBLK_NONE)
	+ swblk_iter_limit_init(&blks, object, e);
	+ for (sb = swblk_start(&blks, pi), start = pi % SWAP_META_PAGES;
	+ sb != NULL; sb = swblk_next(&blks), start = 0) {
	+ limit = MIN(e - blks.index, SWAP_META_PAGES);
	+ for (i = start; i < limit; i++) {
	+ if (sb->d[i] != SWAPBLK_NONE)
	count++;
	}
	}
	Index: sys/vm/vm_object.c
	===================================================================
	--- sys/vm/vm_object.c
	+++ sys/vm/vm_object.c
	@@ -1681,6 +1681,7 @@
	static bool
	vm_object_scan_all_shadowed(vm_object_t object)
	{
	+ struct pctrie_iter blks;
	vm_object_t backing_object;
	vm_page_t p, pp;
	vm_pindex_t backing_offset_index, new_pindex, pi, ps;
	@@ -1695,7 +1696,8 @@

	pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
	p = vm_page_find_least(backing_object, pi);
	- ps = swap_pager_find_least(backing_object, pi);
	+ swblk_iter_init(&blks, backing_object);
	+ ps = swap_pager_find_least(&blks, pi);

	/*
	* Only check pages inside the parent object's range and
	@@ -1705,7 +1707,7 @@
	if (p != NULL && p->pindex < pi)
	p = TAILQ_NEXT(p, listq);
	if (ps < pi)
	- ps = swap_pager_find_least(backing_object, pi);
	+ ps = swap_pager_find_least(&blks, pi);
	if (p == NULL && ps >= backing_object->size)
	break;
	else if (p == NULL)

File Metadata

Mime Type: text/plain
Expires: Mon, Dec 29, 11:35 AM (3 h, 40 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 27358706
Default Alt Text: D46620.id143361.diff (13 KB)

D46620.id143361.diffNo OneTemporaryActions

D46620.id143361.diffView Options

File Metadata

Event Timeline

D46620.id143361.diff
No OneTemporary
Actions

D46620.id143361.diff
View Options