Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -421,11 +421,13 @@ * Metadata functions */ static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); +static daddr_t swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex0, + int *before, int *after); static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, vm_pindex_t pindex, vm_pindex_t count); static void swp_pager_meta_free_all(vm_object_t); -static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); +static daddr_t swp_pager_meta_remove(vm_object_t, vm_pindex_t); static void swp_pager_init_freerange(daddr_t *start, daddr_t *num) @@ -941,7 +943,8 @@ { daddr_t dstaddr; - if (swp_pager_meta_ctl(dstobject, pindex, 0) != SWAPBLK_NONE) { + if (swp_pager_meta_lookup(dstobject, pindex, NULL, NULL) + != SWAPBLK_NONE) { /* Caller should destroy the source block. */ return (false); } @@ -1045,15 +1048,15 @@ swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after) { - daddr_t blk, blk0; - int i; + daddr_t blk0; - VM_OBJECT_ASSERT_LOCKED(object); + if (before) + *before = SWB_NPAGES - 1; + if (after) + *after = SWB_NPAGES - 1; - /* - * do we have good backing store at the requested index ? - */ - blk0 = swp_pager_meta_ctl(object, pindex, 0); + VM_OBJECT_ASSERT_LOCKED(object); + blk0 = swp_pager_meta_lookup(object, pindex, before, after); if (blk0 == SWAPBLK_NONE) { if (before) *before = 0; @@ -1062,31 +1065,6 @@ return (FALSE); } - /* - * find backwards-looking contiguous good backing store - */ - if (before != NULL) { - for (i = 1; i < SWB_NPAGES; i++) { - if (i > pindex) - break; - blk = swp_pager_meta_ctl(object, pindex - i, 0); - if (blk != blk0 - i) - break; - } - *before = i - 1; - } - - /* - * find forward-looking contiguous good backing store - */ - if (after != NULL) { - for (i = 1; i < SWB_NPAGES; i++) { - blk = swp_pager_meta_ctl(object, pindex + i, 0); - if (blk != blk0 + i) - break; - } - *after = i - 1; - } return (TRUE); } @@ -1114,7 +1092,7 @@ { daddr_t srcaddr; - srcaddr = swp_pager_meta_ctl(m->object, m->pindex, SWM_POP); + srcaddr = swp_pager_meta_remove(m->object, m->pindex); if (srcaddr != SWAPBLK_NONE) swp_pager_freeswapspace(srcaddr, 1); } @@ -1210,7 +1188,7 @@ vm_object_pip_add(object, count); pindex = bm->pindex; - blk = swp_pager_meta_ctl(object, pindex, 0); + blk = swp_pager_meta_lookup(object, pindex, NULL, NULL); KASSERT(blk != SWAPBLK_NONE, ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); @@ -2101,28 +2079,127 @@ } /* - * SWP_PAGER_METACTL() - misc control of swap meta data. + * SWP_PAGER_META_LOOKUP() - lookup good backing store for + * the requested page and optionally the number of continuous blocks. * - * This routine is capable of looking up, or removing swapblk - * assignments in the swap meta data. It returns the swapblk being - * looked-up, popped, or SWAPBLK_NONE if the block was invalid. + * This routine looks up swapblk assignment in the swap meta data. + * In addition, if before and/or after are provided with a limit value, + * it looks for the number of continuous blocks backward and forward. * + * This returns SWAPBLK_NONE if swblk at pindex isn't associated to + * swap meta data; *before and *after are not modified for this case. + * Otherwise, return the swap address of pindex. If *before and/or + * *after are provided with positive numbers, the number of continuous + * blocks before and/or after up to provided limits are searched and + * returned. + * + * Input - "object" and "pindex0" + * Input/Output - "before" and "after" - the number of continuous blocks + * to look for as input and its actual size as output. + */ +static daddr_t +swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex0, int *before, + int *after) +{ + struct swblk *sb, *sbb; + vm_pindex_t pindex, pindexb; + daddr_t r0; + int i, i0, end, offset; + + VM_OBJECT_ASSERT_LOCKED(object); + /* + * The meta data only exists if the object is OBJT_SWAP + * and even then might not be allocated yet. + */ + if (object->type != OBJT_SWAP) + return (SWAPBLK_NONE); + + i0 = pindex0 % SWAP_META_PAGES; + pindex = rounddown(pindex0, SWAP_META_PAGES); + + sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, pindex); + if (sb == NULL || sb->d[i0] == SWAPBLK_NONE) + return (SWAPBLK_NONE); + r0 = sb->d[i0]; + + if (before) { + if (*before <= 0 || pindex0 == 0) { + *before = 0; + goto finished_backward; + } + offset = 1; + pindexb = pindex; + sbb = sb; + if (pindex0 < *before) /* some pindexes are less than 0. */ + end = pindex0; + else + end = *before; + i = i0 - 1; + do { + for (; i >= 0; --i, ++offset) { + if (r0 - offset != sbb->d[i]) { + *before = offset - 1; + goto finished_backward; + } + if (offset == end) { + *before = offset; + goto finished_backward; + } + } + pindexb -= SWAP_META_PAGES; + i = SWAP_META_PAGES - 1; + } while ((sbb = SWAP_PCTRIE_LOOKUP( + &object->un_pager.swp.swp_blks, pindexb)) != NULL); + *before = offset - 1; + } +finished_backward: + if (!after) + return (r0); + if (*after <= 0) { + *after = 0; + return (r0); + } + + offset = 1; + end = *after; /* how do I check overflow? */ + i = i0 + 1; + do { + for (; i < SWAP_META_PAGES; ++i, ++offset) { + if (r0 + offset != sb->d[i]) { + *after = offset - 1; + return (r0); + } + if (offset == end) { + *after = offset; + return (r0); + } + } + pindex += SWAP_META_PAGES; + i = 0; + } while ((sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, + pindex)) != NULL); + *after = offset - 1; + return (r0); +} + +/* + * SWP_PAGER_META_REMOVE() - remove swap meta data. + * + * This routine is capable of removing swapblk assignments in the + * swap meta data. It returns the swapblk being removed or + * SWAPBLK_NONE if the block was invalid. + * * When acting on a busy resident page and paging is in progress, we * have to wait until paging is complete but otherwise can act on the * busy page. - * - * SWM_POP remove from meta data but do not free it */ static daddr_t -swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags) +swp_pager_meta_remove(vm_object_t object, vm_pindex_t pindex) { struct swblk *sb; daddr_t r1; - if ((flags & SWM_POP) != 0) - VM_OBJECT_ASSERT_WLOCKED(object); - else - VM_OBJECT_ASSERT_LOCKED(object); + VM_OBJECT_ASSERT_LOCKED(object); /* * The meta data only exists if the object is OBJT_SWAP @@ -2138,13 +2215,11 @@ r1 = sb->d[pindex % SWAP_META_PAGES]; if (r1 == SWAPBLK_NONE) return (SWAPBLK_NONE); - if ((flags & SWM_POP) != 0) { - sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE; - if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { - SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, - rounddown(pindex, SWAP_META_PAGES)); - uma_zfree(swblk_zone, sb); - } + sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE; + if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) { + SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, + rounddown(pindex, SWAP_META_PAGES)); + uma_zfree(swblk_zone, sb); } return (r1); }