Page MenuHomeFreeBSD
Files F165321673

D46620.id143173.diff
No OneTemporary
Actions

D46620.id143173.diff
View Options

Index: sys/fs/tmpfs/tmpfs_vnops.c
===================================================================
--- sys/fs/tmpfs/tmpfs_vnops.c
+++ sys/fs/tmpfs/tmpfs_vnops.c
@@ -48,6 +48,7 @@
#include <sys/lock.h>
#include <sys/mount.h>
#include <sys/namei.h>
+#include <sys/pctrie.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/rwlock.h>
@@ -2098,11 +2099,13 @@
static off_t
tmpfs_seek_data_locked(vm_object_t obj, off_t noff)
{
+ struct pctrie_iter blks;
vm_page_t m;
vm_pindex_t p, p_m, p_swp;
p = OFF_TO_IDX(noff);
m = vm_page_find_least(obj, p);
+ swblk_iter_init(&blks, obj);
/*
* Microoptimize the most common case for SEEK_DATA, where
@@ -2111,7 +2114,7 @@
if (m != NULL && vm_page_any_valid(m) && m->pindex == p)
return (noff);
- p_swp = swap_pager_find_least(obj, p);
+ p_swp = swap_pager_find_least(&blks, p);
if (p_swp == p)
return (noff);
@@ -2139,9 +2142,11 @@
static off_t
tmpfs_seek_hole_locked(vm_object_t obj, off_t noff)
{
+ struct pctrie_iter blks;
vm_page_t m;
vm_pindex_t p, p_swp;
+ swblk_iter_init(&blks, obj);
for (;; noff = tmpfs_seek_next(noff)) {
/*
* Walk over the largest sequential run of the valid pages.
@@ -2156,7 +2161,7 @@
* there is a hole in the swap at the same place.
*/
p = OFF_TO_IDX(noff);
- p_swp = swap_pager_find_least(obj, p);
+ p_swp = swap_pager_find_least(&blks, p);
if (p_swp != p) {
noff = IDX_TO_OFF(p);
break;
Index: sys/vm/swap_pager.h
===================================================================
--- sys/vm/swap_pager.h
+++ sys/vm/swap_pager.h
@@ -40,6 +40,7 @@
#include <sys/_types.h>
struct buf;
+struct pctrie_iter;
struct swdevt;
struct thread;
typedef void sw_strategy_t(struct buf *, struct swdevt *);
@@ -74,7 +75,8 @@
struct xswdev;
int swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len);
void swap_pager_copy(vm_object_t, vm_object_t, vm_pindex_t, int);
-vm_pindex_t swap_pager_find_least(vm_object_t object, vm_pindex_t pindex);
+void swblk_iter_init(struct pctrie_iter *blks, vm_object_t object);
+vm_pindex_t swap_pager_find_least(struct pctrie_iter *blks, vm_pindex_t pindex);
void swap_pager_freespace(vm_object_t object, vm_pindex_t start,
vm_size_t size, vm_size_t *freed);
void swap_pager_swap_init(void);
Index: sys/vm/swap_pager.c
===================================================================
--- sys/vm/swap_pager.c
+++ sys/vm/swap_pager.c
@@ -531,6 +531,23 @@
PCTRIE_DEFINE(SWAP, swblk, p, swblk_trie_alloc, swblk_trie_free);
+void
+swblk_iter_init(struct pctrie_iter *blks, vm_object_t object)
+{
+ VM_OBJECT_ASSERT_LOCKED(object);
+ MPASS((object->flags & OBJ_SWAP) != 0);
+ pctrie_iter_init(blks, &object->un_pager.swp.swp_blks);
+}
+
+static void
+swblk_iter_limit_init(struct pctrie_iter *blks, vm_object_t object,
+ vm_pindex_t limit)
+{
+ VM_OBJECT_ASSERT_LOCKED(object);
+ MPASS((object->flags & OBJ_SWAP) != 0);
+ pctrie_iter_limit_init(blks, &object->un_pager.swp.swp_blks, limit);
+}
+
static struct swblk *
swblk_lookup(vm_object_t object, vm_pindex_t pindex)
{
@@ -539,31 +556,28 @@
}
static struct swblk *
-swblk_start(vm_object_t object, vm_pindex_t pindex)
+swblk_start(struct pctrie_iter *blks, vm_pindex_t pindex)
{
- return (SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
+ return (SWAP_PCTRIE_ITER_LOOKUP_GE(blks,
rounddown(pindex, SWAP_META_PAGES)));
}
static struct swblk *
-swblk_next(vm_object_t object, struct swblk *sb)
+swblk_next(struct pctrie_iter *blks)
{
- return (swblk_start(object, sb->p + SWAP_META_PAGES));
+ return (SWAP_PCTRIE_ITER_JUMP_GE(blks, SWAP_META_PAGES));
}
static struct swblk *
-swblk_start_limit(vm_object_t object, vm_pindex_t pindex, vm_pindex_t limit)
+swblk_restart(struct pctrie_iter *blks, vm_pindex_t pindex)
{
- struct swblk *sb = swblk_start(object, pindex);
- if (sb != NULL && sb->p < limit)
- return (sb);
- return (NULL);
+ return (SWAP_PCTRIE_ITER_LOOKUP(blks, pindex));
}
-static struct swblk *
-swblk_next_limit(vm_object_t object, struct swblk *sb, vm_pindex_t limit)
+static void
+swblk_remove(struct pctrie_iter *blks)
{
- return (swblk_start_limit(object, sb->p + SWAP_META_PAGES, limit));
+ SWAP_PCTRIE_ITER_REMOVE(blks);
}
static void
@@ -1827,6 +1841,7 @@
u_long
swap_pager_swapped_pages(vm_object_t object)
{
+ struct pctrie_iter blks;
struct swblk *sb;
u_long res;
int i;
@@ -1837,8 +1852,8 @@
return (0);
res = 0;
- for (sb = swblk_start(object, 0); sb != NULL;
- sb = swblk_next(object, sb)) {
+ swblk_iter_init(&blks, object);
+ for (sb = swblk_start(&blks, 0); sb != NULL; sb = swblk_next(&blks)) {
for (i = 0; i < SWAP_META_PAGES; i++) {
if (sb->d[i] != SWAPBLK_NONE)
res++;
@@ -1856,6 +1871,7 @@
static void
swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object)
{
+ struct pctrie_iter blks;
struct page_range range;
struct swblk *sb;
vm_page_t m;
@@ -1872,27 +1888,28 @@
i = 0;
swp_pager_init_freerange(&range);
for (;;) {
- if (i == 0 && (object->flags & OBJ_DEAD) != 0) {
- /*
- * Make sure that pending writes finish before
- * returning.
- */
- vm_object_pip_wait(object, "swpoff");
- swp_pager_meta_free_all(object);
- break;
- }
-
- if (i == SWAP_META_PAGES) {
+ if (i == 0) {
+ if ((object->flags & OBJ_DEAD) != 0) {
+ /*
+ * Make sure that pending writes finish before
+ * returning.
+ */
+ vm_object_pip_wait(object, "swpoff");
+ swp_pager_meta_free_all(object);
+ break;
+ }
+ swblk_iter_init(&blks, object);
+ sb = swblk_start(&blks, pi);
+ } else if (i == SWAP_META_PAGES) {
pi = sb->p + SWAP_META_PAGES;
if (sb_empty) {
- swblk_lookup_remove(object, sb);
+ swblk_remove(&blks);
uma_zfree(swblk_zone, sb);
}
+ sb = swblk_next(&blks);
i = 0;
}
-
if (i == 0) {
- sb = swblk_start(object, pi);
if (sb == NULL)
break;
sb_empty = true;
@@ -2197,11 +2214,13 @@
swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject,
vm_pindex_t pindex, vm_pindex_t count)
{
+ struct pctrie_iter blks;
struct page_range range;
struct swblk *sb;
daddr_t blk;
- vm_pindex_t offset, last;
+ vm_pindex_t last;
int i, limit, start;
+ bool reset;
VM_OBJECT_ASSERT_WLOCKED(srcobject);
VM_OBJECT_ASSERT_WLOCKED(dstobject);
@@ -2210,18 +2229,18 @@
return;
swp_pager_init_freerange(&range);
- offset = pindex;
last = pindex + count;
- sb = swblk_start_limit(srcobject, pindex, last);
+ swblk_iter_limit_init(&blks, srcobject, last);
+ reset = false;
+ sb = swblk_start(&blks, pindex);
start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0;
- for (; sb != NULL;
- sb = swblk_start_limit(srcobject, pindex, last), start = 0) {
+ for (; sb != NULL; sb = swblk_next(&blks), start = 0) {
limit = MIN(last - sb->p, SWAP_META_PAGES);
for (i = start; i < limit; i++) {
if (sb->d[i] == SWAPBLK_NONE)
continue;
blk = swp_pager_meta_build(dstobject,
- sb->p + i - offset, sb->d[i], true);
+ sb->p + i - pindex, sb->d[i], true);
if (blk == sb->d[i]) {
/*
* Destination has no swapblk and is not
@@ -2231,16 +2250,22 @@
*/
VM_OBJECT_WUNLOCK(srcobject);
swp_pager_meta_build(dstobject,
- sb->p + i - offset, sb->d[i], false);
+ sb->p + i - pindex, sb->d[i], false);
VM_OBJECT_WLOCK(srcobject);
+ reset = true;
} else if (blk != SWAPBLK_NONE)
swp_pager_update_freerange(&range, sb->d[i]);
sb->d[i] = SWAPBLK_NONE;
}
- pindex = sb->p + SWAP_META_PAGES;
+ if (reset) {
+ /* Rebuild search path after losing object lock. */
+ reset = false;
+ swblk_iter_limit_init(&blks, srcobject, last);
+ swblk_restart(&blks, sb->p);
+ }
if (swp_pager_swblk_empty(sb, 0, start) &&
swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
- swblk_lookup_remove(srcobject, sb);
+ swblk_remove(&blks);
uma_zfree(swblk_zone, sb);
}
}
@@ -2261,6 +2286,7 @@
swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count,
vm_size_t *freed)
{
+ struct pctrie_iter blks;
struct page_range range;
struct swblk *sb;
vm_page_t m;
@@ -2277,10 +2303,10 @@
swp_pager_init_freerange(&range);
last = pindex + count;
- sb = swblk_start_limit(object, pindex, last);
+ swblk_iter_limit_init(&blks, object, last);
+ sb = swblk_start(&blks, pindex);
start = (sb != NULL && sb->p < pindex) ? pindex - sb->p : 0;
- for (; sb != NULL;
- sb = swblk_start_limit(object, pindex, last), start = 0) {
+ for (; sb != NULL; sb = swblk_next(&blks), start = 0) {
limit = MIN(last - sb->p, SWAP_META_PAGES);
for (i = start; i < limit; i++) {
if (sb->d[i] == SWAPBLK_NONE)
@@ -2295,10 +2321,9 @@
}
sb->d[i] = SWAPBLK_NONE;
}
- pindex = sb->p + SWAP_META_PAGES;
if (swp_pager_swblk_empty(sb, 0, start) &&
swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
- swblk_lookup_remove(object, sb);
+ swblk_remove(&blks);
uma_zfree(swblk_zone, sb);
}
}
@@ -2376,19 +2401,19 @@
* if are no allocated swap blocks for the object after the requested pindex.
*/
vm_pindex_t
-swap_pager_find_least(vm_object_t object, vm_pindex_t pindex)
+swap_pager_find_least(struct pctrie_iter *blks, vm_pindex_t pindex)
{
struct swblk *sb;
int i;
- if ((sb = swblk_start(object, pindex)) == NULL)
+ if ((sb = swblk_start(blks, pindex)) == NULL)
return (OBJ_MAX_SIZE);
if (sb->p < pindex) {
for (i = pindex % SWAP_META_PAGES; i < SWAP_META_PAGES; i++) {
if (sb->d[i] != SWAPBLK_NONE)
return (sb->p + i);
}
- if ((sb = swblk_next(object, sb)) == NULL)
+ if ((sb = swblk_next(blks)) == NULL)
return (OBJ_MAX_SIZE);
}
for (i = 0; i < SWAP_META_PAGES; i++) {
@@ -2831,6 +2856,7 @@
long
vmspace_swap_count(struct vmspace *vmspace)
{
+ struct pctrie_iter blks;
vm_map_t map;
vm_map_entry_t cur;
vm_object_t object;
@@ -2853,8 +2879,9 @@
goto unlock;
pi = OFF_TO_IDX(cur->offset);
e = pi + OFF_TO_IDX(cur->end - cur->start);
- for (sb = swblk_start_limit(object, pi, e);
- sb != NULL; sb = swblk_next_limit(object, sb, e)) {
+ swblk_iter_limit_init(&blks, object, e);
+ for (sb = swblk_start(&blks, pi);
+ sb != NULL; sb = swblk_next(&blks)) {
for (i = 0; i < SWAP_META_PAGES; i++) {
if (sb->p + i < e &&
sb->d[i] != SWAPBLK_NONE)
Index: sys/vm/vm_object.c
===================================================================
--- sys/vm/vm_object.c
+++ sys/vm/vm_object.c
@@ -1681,6 +1681,7 @@
static bool
vm_object_scan_all_shadowed(vm_object_t object)
{
+ struct pctrie_iter blks;
vm_object_t backing_object;
vm_page_t p, pp;
vm_pindex_t backing_offset_index, new_pindex, pi, ps;
@@ -1695,7 +1696,8 @@
pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
p = vm_page_find_least(backing_object, pi);
- ps = swap_pager_find_least(backing_object, pi);
+ swblk_iter_init(&blks, backing_object);
+ ps = swap_pager_find_least(&blks, pi);
/*
* Only check pages inside the parent object's range and
@@ -1705,7 +1707,7 @@
if (p != NULL && p->pindex < pi)
p = TAILQ_NEXT(p, listq);
if (ps < pi)
- ps = swap_pager_find_least(backing_object, pi);
+ ps = swap_pager_find_least(&blks, pi);
if (p == NULL && ps >= backing_object->size)
break;
else if (p == NULL)

File Metadata

Mime Type
text/plain
Expires
Sat, Aug 8, 3:29 PM (14 h, 24 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
36237772
Default Alt Text
D46620.id143173.diff (11 KB)

Event Timeline