Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -486,7 +486,8 @@ /* * Metadata functions */ -static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); +static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t, + bool); static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t, vm_size_t *); static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, @@ -1031,7 +1032,7 @@ } for (j = 0; j < n; ++j) { addr = swp_pager_meta_build(object, - start + i + j, blk + j); + start + i + j, blk + j, false); if (addr != SWAPBLK_NONE) swp_pager_update_freerange(&range, addr); } @@ -1041,35 +1042,6 @@ return (0); } -static bool -swp_pager_xfer_source(vm_object_t srcobject, vm_object_t dstobject, - vm_pindex_t pindex, daddr_t addr) -{ - daddr_t dstaddr __diagused; - - KASSERT((srcobject->flags & OBJ_SWAP) != 0, - ("%s: srcobject not swappable", __func__)); - KASSERT((dstobject->flags & OBJ_SWAP) != 0, - ("%s: dstobject not swappable", __func__)); - - if (swp_pager_meta_lookup(dstobject, pindex) != SWAPBLK_NONE) { - /* Caller should destroy the source block. */ - return (false); - } - - /* - * Destination has no swapblk and is not resident, transfer source. - * swp_pager_meta_build() can sleep. - */ - VM_OBJECT_WUNLOCK(srcobject); - dstaddr = swp_pager_meta_build(dstobject, pindex, addr); - KASSERT(dstaddr == SWAPBLK_NONE, - ("Unexpected destination swapblk")); - VM_OBJECT_WLOCK(srcobject); - - return (true); -} - /* * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager * and destroy the source. @@ -1548,7 +1520,7 @@ mreq = ma[i + j]; vm_page_aflag_clear(mreq, PGA_SWAP_FREE); addr = swp_pager_meta_build(mreq->object, mreq->pindex, - blk + j); + blk + j, false); if (addr != SWAPBLK_NONE) swp_pager_update_freerange(&range, addr); MPASS(mreq->dirty == VM_PAGE_BITS_ALL); @@ -2052,106 +2024,104 @@ uma_zfree(swblk_zone, sb); } } - + +/* + * SWP_PAGER_ALLOC_WAIT() - wait before retrying allocation from a zone. + */ +static void +swp_pager_alloc_wait(vm_object_t object, uma_zone_t zone, + volatile int *zone_exhausted, const char *zone_name, const char *pause_msg) +{ + VM_OBJECT_WUNLOCK(object); + if (uma_zone_exhausted(zone)) { + if (atomic_cmpset_int(zone_exhausted, 0, 1)) + printf("swap %s zone exhausted, " + "increase kern.maxswzone\n", zone_name); + vm_pageout_oom(VM_OOM_SWAPZ); + pause(pause_msg, 10); + } else + uma_zwait(zone); + VM_OBJECT_WLOCK(object); +} + /* * SWP_PAGER_META_BUILD() - add swap block to swap meta data for object * - * The specified swapblk is added to the object's swap metadata. If - * the swapblk is not valid, it is freed instead. Any previously - * assigned swapblk is returned. + * Try to add the specified swapblk to the object's swap metadata. If + * nowait and memory allocation fails, return the specified swapblk + * immediately to indicate failure. If nowait, add the specified swapblk + * only if there is no previously assigned swapblk. Return the previously + * assigned swapblk, if any. */ static daddr_t -swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) +swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk, + bool nowait) { static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted; struct swblk *sb, *sb1; vm_pindex_t modpi, rdpi; daddr_t prev_swapblk; - int error, i; + int i; VM_OBJECT_ASSERT_WLOCKED(object); rdpi = rounddown(pindex, SWAP_META_PAGES); sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); - if (sb == NULL) { - if (swapblk == SWAPBLK_NONE) - return (SWAPBLK_NONE); - for (;;) { - sb = uma_zalloc(swblk_zone, M_NOWAIT | (curproc == - pageproc ? M_USE_RESERVE : 0)); - if (sb != NULL) { - sb->p = rdpi; - for (i = 0; i < SWAP_META_PAGES; i++) - sb->d[i] = SWAPBLK_NONE; - if (atomic_cmpset_int(&swblk_zone_exhausted, - 1, 0)) - printf("swblk zone ok\n"); - break; - } - VM_OBJECT_WUNLOCK(object); - if (uma_zone_exhausted(swblk_zone)) { - if (atomic_cmpset_int(&swblk_zone_exhausted, - 0, 1)) - printf("swap blk zone exhausted, " - "increase kern.maxswzone\n"); - vm_pageout_oom(VM_OOM_SWAPZ); - pause("swzonxb", 10); - } else - uma_zwait(swblk_zone); - VM_OBJECT_WLOCK(object); - sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, - rdpi); - if (sb != NULL) - /* - * Somebody swapped out a nearby page, - * allocating swblk at the rdpi index, - * while we dropped the object lock. - */ - goto allocated; + if (sb != NULL) + goto allocated; + if (swapblk == SWAPBLK_NONE) + return (SWAPBLK_NONE); + while ((sb = uma_zalloc(swblk_zone, + M_NOWAIT | (curproc == pageproc ? M_USE_RESERVE : 0))) == NULL) { + if (nowait) + return (swapblk); + swp_pager_alloc_wait(object, swblk_zone, + &swblk_zone_exhausted, "blk", "swzonxb"); + sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); + if (sb != NULL) + /* + * Somebody swapped out a nearby page, allocating swblk + * at the rdpi index, while we dropped the object lock. + */ + goto allocated; + } + sb->p = rdpi; + for (i = 0; i < SWAP_META_PAGES; i++) + sb->d[i] = SWAPBLK_NONE; + if (atomic_cmpset_int(&swblk_zone_exhausted, 1, 0)) + printf("swblk zone ok\n"); + while (SWAP_PCTRIE_INSERT(&object->un_pager.swp.swp_blks, sb) != 0) { + if (nowait) { + uma_zfree(swblk_zone, sb); + return (swapblk); } - for (;;) { - error = SWAP_PCTRIE_INSERT( - &object->un_pager.swp.swp_blks, sb); - if (error == 0) { - if (atomic_cmpset_int(&swpctrie_zone_exhausted, - 1, 0)) - printf("swpctrie zone ok\n"); - break; - } - VM_OBJECT_WUNLOCK(object); - if (uma_zone_exhausted(swpctrie_zone)) { - if (atomic_cmpset_int(&swpctrie_zone_exhausted, - 0, 1)) - printf("swap pctrie zone exhausted, " - "increase kern.maxswzone\n"); - vm_pageout_oom(VM_OOM_SWAPZ); - pause("swzonxp", 10); - } else - uma_zwait(swpctrie_zone); - VM_OBJECT_WLOCK(object); - sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, - rdpi); - if (sb1 != NULL) { - uma_zfree(swblk_zone, sb); - sb = sb1; - goto allocated; - } + swp_pager_alloc_wait(object, swpctrie_zone, + &swpctrie_zone_exhausted, "pctrie", "swzonxp"); + sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi); + if (sb1 != NULL) { + uma_zfree(swblk_zone, sb); + sb = sb1; + goto allocated; } } + if (atomic_cmpset_int(&swpctrie_zone_exhausted, 1, 0)) + printf("swpctrie zone ok\n"); allocated: MPASS(sb->p == rdpi); modpi = pindex % SWAP_META_PAGES; /* Return prior contents of metadata. */ prev_swapblk = sb->d[modpi]; - /* Enter block into metadata. */ - sb->d[modpi] = swapblk; + if (!nowait || prev_swapblk == SWAPBLK_NONE) { + /* Enter block into metadata. */ + sb->d[modpi] = swapblk; - /* - * Free the swblk if we end up with the empty page run. - */ - if (swapblk == SWAPBLK_NONE) - swp_pager_free_empty_swblk(object, sb); + /* + * Free the swblk if we end up with the empty page run. + */ + if (swapblk == SWAPBLK_NONE) + swp_pager_free_empty_swblk(object, sb); + } return (prev_swapblk); } @@ -2168,6 +2138,7 @@ { struct page_range range; struct swblk *sb; + daddr_t blk; vm_page_t m; vm_pindex_t offset, last; vm_size_t mc; @@ -2193,12 +2164,25 @@ limit = last - sb->p < SWAP_META_PAGES ? last - sb->p : SWAP_META_PAGES; for (i = start; i < limit; i++) { - if (sb->d[i] == SWAPBLK_NONE) + blk = sb->d[i]; + if (blk == SWAPBLK_NONE) continue; if (dstobject == NULL || - !swp_pager_xfer_source(srcobject, dstobject, - sb->p + i - offset, sb->d[i])) { + (blk = swp_pager_meta_build(dstobject, + sb->p + i - offset, blk, true), + blk != sb->d[i] && blk != SWAPBLK_NONE)) swp_pager_update_freerange(&range, sb->d[i]); + else if (blk == sb->d[i]) { + /* + * Destination has no swapblk and is not + * resident, so transfer source. + * swp_pager_meta_build() failed memory + * allocation already, likely to sleep in retry. + */ + VM_OBJECT_WUNLOCK(srcobject); + swp_pager_meta_build(dstobject, + sb->p + i - offset, blk, false); + VM_OBJECT_WLOCK(srcobject); } if (moved != NULL) { if (m != NULL && m->pindex != pindex + i - 1)