Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -438,7 +438,8 @@ static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, vm_pindex_t pindex, vm_pindex_t count); static void swp_pager_meta_free_all(vm_object_t); -static daddr_t swp_pager_meta_lookup(vm_object_t, vm_pindex_t); +static daddr_t swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex0, + int *before, int *after); static void swp_pager_init_freerange(daddr_t *start, daddr_t *num) @@ -958,7 +959,8 @@ KASSERT(srcobject->type == OBJT_SWAP, ("%s: Srcobject not swappable", __func__)); if (dstobject->type == OBJT_SWAP && - swp_pager_meta_lookup(dstobject, pindex) != SWAPBLK_NONE) { + swp_pager_meta_lookup(dstobject, pindex, NULL, NULL) != + SWAPBLK_NONE) { /* Caller should destroy the source block. */ return (false); } @@ -1055,51 +1057,19 @@ swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after) { - daddr_t blk, blk0; - int i; + daddr_t blk0; + if (before != NULL) + *before = SWB_NPAGES - 1; + if (after != NULL) + *after = SWB_NPAGES - 1; + VM_OBJECT_ASSERT_LOCKED(object); KASSERT(object->type == OBJT_SWAP, ("%s: object not swappable", __func__)); - /* - * do we have good backing store at the requested index ? - */ - blk0 = swp_pager_meta_lookup(object, pindex); - if (blk0 == SWAPBLK_NONE) { - if (before) - *before = 0; - if (after) - *after = 0; - return (FALSE); - } - - /* - * find backwards-looking contiguous good backing store - */ - if (before != NULL) { - for (i = 1; i < SWB_NPAGES; i++) { - if (i > pindex) - break; - blk = swp_pager_meta_lookup(object, pindex - i); - if (blk != blk0 - i) - break; - } - *before = i - 1; - } - - /* - * find forward-looking contiguous good backing store - */ - if (after != NULL) { - for (i = 1; i < SWB_NPAGES; i++) { - blk = swp_pager_meta_lookup(object, pindex + i); - if (blk != blk0 + i) - break; - } - *after = i - 1; - } - return (TRUE); + blk0 = swp_pager_meta_lookup(object, pindex, before, after); + return (blk0 != SWAPBLK_NONE); } /* @@ -1268,7 +1238,7 @@ vm_object_pip_add(object, count); pindex = bm->pindex; - blk = swp_pager_meta_lookup(object, pindex); + blk = swp_pager_meta_lookup(object, pindex, NULL, NULL); KASSERT(blk != SWAPBLK_NONE, ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex)); @@ -1920,6 +1890,64 @@ } /* + * SWP_SWBLK_FORWARD_SEARCH() - looks for the number of continuous blocks + * + * Given "swblk" and an "index" to it with the address of the + * "index", the function looks forward for the number of continuous + * blocks up to "max_seek". + */ +static int +swp_swblk_forward_search(vm_object_t object, struct swblk *sb, + int index, daddr_t expected_addr, int max_seek) +{ + int seek = 0; + + VM_OBJECT_ASSERT_LOCKED(object); + + do { + for (; index < SWAP_META_PAGES; ++index) { + if (expected_addr++ != sb->d[index]) + return (seek); + ++seek; + if (seek >= max_seek) + return (seek); + } + index = 0; + } while ((sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, + sb->p + SWAP_META_PAGES)) != NULL); + return (seek); +} + +/* + * SWP_SWBLK_BACKWARD_SEARCH() - looks for the number of continuous blocks + * + * Given "swblk" and an "index" to it with the address of the + * "index", the function looks backward for the number of continuous + * blocks up to "max_seek". + */ +static int +swp_swblk_backward_search(vm_object_t object, struct swblk *sb, + int index, daddr_t expected_addr, int max_seek) +{ + int seek = 0; + + VM_OBJECT_ASSERT_LOCKED(object); + + do { + for (; index >= 0; --index) { + if (expected_addr-- != sb->d[index]) + return (seek); + ++seek; + if (seek >= max_seek) + return (seek); + } + index = SWAP_META_PAGES - 1; + } while ((sb = SWAP_PCTRIE_LOOKUP( &object->un_pager.swp.swp_blks, + sb->p - SWAP_META_PAGES)) != NULL); + return (seek); +} + +/* * SWP_PAGER_FREE_EMPTY_SWBLK() - frees if a block is free * * Nothing is done if the block is still in use. @@ -2162,20 +2190,30 @@ } /* - * SWP_PAGER_METACTL() - misc control of swap meta data. + * SWP_PAGER_META_LOOKUP() - lookup good backing store for + * the requested page and optionally the number of continuous blocks. * - * This routine is capable of looking up, or removing swapblk - * assignments in the swap meta data. It returns the swapblk being - * looked-up, popped, or SWAPBLK_NONE if the block was invalid. + * This routine looks up swapblk assignment in the swap meta data. + * In addition, if before and/or after are provided with a limit value, + * it looks for the number of continuous blocks backward and forward. * - * When acting on a busy resident page and paging is in progress, we - * have to wait until paging is complete but otherwise can act on the - * busy page. + * This returns SWAPBLK_NONE if swblk at pindex isn't associated to + * swap meta data; *before and *after are not modified for this case. + * Otherwise, return the swap address of pindex. If *before and/or + * *after are provided with positive numbers, the number of continuous + * blocks before and/or after up to provided limits are searched and + * returned. + * + * Input - "object" and "pindex0" + * Input/Output - "before" and "after" - the number of continuous blocks + * to look for as input and its actual size as output. */ static daddr_t -swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex) +swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex, int *before, + int *after) { struct swblk *sb; + int i, end; VM_OBJECT_ASSERT_LOCKED(object); @@ -2186,11 +2224,44 @@ KASSERT(object->type == OBJT_SWAP, ("Lookup object not swappable")); + i = pindex % SWAP_META_PAGES; + sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rounddown(pindex, SWAP_META_PAGES)); - if (sb == NULL) + if (sb == NULL || sb->d[i] == SWAPBLK_NONE) { + if (before != NULL) + *before = 0; + if (after != NULL) + *after = 0; return (SWAPBLK_NONE); - return (sb->d[pindex % SWAP_META_PAGES]); + } + + if (before != NULL) { + if (*before <= 0 || pindex == 0) { + *before = 0; + } else { + if (pindex < *before) + end = pindex; + else + end = *before; + *before = swp_swblk_backward_search(object, sb, i - 1, + sb->d[i] - 1, end); + } + } + + if (after != NULL) { + if (*after <= 0) + *after = 0; + else { + if (pindex + *after < pindex) + end = UINT64_MAX - pindex; + else + end = *after; + *after = swp_swblk_forward_search(object, sb, i + 1, + sb->d[i] + 1, end); + } + } + return (sb->d[i]); } /*