Index: sys/kern/subr_blist.c =================================================================== --- sys/kern/subr_blist.c +++ sys/kern/subr_blist.c @@ -295,7 +295,7 @@ * not be allocated. */ daddr_t -blist_alloc(blist_t bl, int *count, int maxcount) +blist_alloc(blist_t bl, daddr_t *io_cursor, int *count, int maxcount) { daddr_t blk, cursor; @@ -310,14 +310,14 @@ * non-zero. When the cursor is zero, an allocation failure will * stop further iterations. */ - for (cursor = bl->bl_cursor;; cursor = 0) { + for (cursor = *io_cursor;; cursor = 0) { blk = blst_meta_alloc(bl->bl_root, cursor, count, maxcount, bl->bl_radix); if (blk != SWAPBLK_NONE) { bl->bl_avail -= *count; - bl->bl_cursor = blk + *count; - if (bl->bl_cursor == bl->bl_blocks) - bl->bl_cursor = 0; + *io_cursor = blk + *count; + if (*io_cursor == bl->bl_blocks) + *io_cursor = 0; return (blk); } if (cursor == 0) @@ -404,8 +404,7 @@ void blist_print(blist_t bl) { - printf("BLIST avail = %jd, cursor = %08jx {\n", - (uintmax_t)bl->bl_avail, (uintmax_t)bl->bl_cursor); + printf("BLIST avail = %jd {\n", (uintmax_t)bl->bl_avail); if (bl->bl_root->bm_bitmap != 0) blst_radix_print(bl->bl_root, 0, bl->bl_radix, 4); Index: sys/sys/blist.h =================================================================== --- sys/sys/blist.h +++ sys/sys/blist.h @@ -81,7 +81,6 @@ daddr_t bl_blocks; /* area of coverage */ daddr_t bl_avail; /* # available blocks */ u_daddr_t bl_radix; /* coverage radix */ - daddr_t bl_cursor; /* next-fit search starts at */ blmeta_t bl_root[1]; /* root of radix tree */ } *blist_t; @@ -92,7 +91,7 @@ struct sbuf; -daddr_t blist_alloc(blist_t blist, int *count, int maxcount); +daddr_t blist_alloc(blist_t blist, daddr_t *cursor, int *count, int maxcount); daddr_t blist_avail(blist_t blist); blist_t blist_create(daddr_t blocks, int flags); void blist_destroy(blist_t blist); Index: sys/vm/swap_pager.h =================================================================== --- sys/vm/swap_pager.h +++ sys/vm/swap_pager.h @@ -65,6 +65,8 @@ swblk_t sw_first; swblk_t sw_end; struct blist *sw_blist; + daddr_t sw_cursor; /* next-fit search starts at */ + daddr_t sw_trimmer; /* where to look for free space to trim */ TAILQ_ENTRY(swdevt) sw_list; sw_strategy_t *sw_strategy; sw_close_t *sw_close; Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -702,6 +702,85 @@ object->type = OBJT_DEAD; } +static void +swp_pager_async_trimdone(struct buf *bp) +{ + struct swdevt *sp; + daddr_t blk; + int npages; + + sp = (struct swdevt *)bp->b_fsprivate1; + blk = bp->b_blkno; + npages = bp->b_npages; + uma_zfree(swwbuf_zone, bp); + + mtx_lock(&swbuf_mtx); + if (++nsw_wcount_async == 1) + wakeup(&nsw_wcount_async); + mtx_unlock(&swbuf_mtx); + + mtx_lock(&sw_dev_mtx); + blk -= sp->sw_first; + blist_free(sp->sw_blist, blk, npages); + mtx_unlock(&sw_dev_mtx); +} + +static struct buf * +swapdev_trim(struct swdevt *sp) +{ + struct buf *bp; + daddr_t blk; + u_long nblks; + int npages, mpages; + + /* Quit if the cursor is too far behind the trimmer. */ + mtx_lock(&sw_dev_mtx); + nblks = sp->sw_nblks; + blk = sp->sw_trimmer; + if ((blk + nblks - sp->sw_cursor) % nblks > nblks / 5) { + mtx_unlock(&sw_dev_mtx); + return (NULL); + } + + /* Grab a lot of free space allocated long ago. */ + npages = 1; + mpages = 10000; + blk = blist_alloc(sp->sw_blist, &sp->sw_trimmer, &npages, mpages); + + /* Quit if there's nothing free between the trimmer and cursor. */ + if (blk == SWAPBLK_NONE) { + mtx_unlock(&sw_dev_mtx); + return (NULL); + } + if ((blk + nblks - sp->sw_cursor) % nblks > nblks / 5) { + /* The trim allocation jumped the cursor. Bail out. */ + blist_free(sp->sw_blist, blk, npages); + mtx_unlock(&sw_dev_mtx); + return (NULL); + } + blk += sp->sw_first; + mtx_unlock(&sw_dev_mtx); + mtx_lock(&swbuf_mtx); + while (nsw_wcount_async == 0) + msleep(&nsw_wcount_async, &swbuf_mtx, PVM, + "swbufa", 0); + nsw_wcount_async--; + mtx_unlock(&swbuf_mtx); + bp = uma_zalloc(swwbuf_zone, M_WAITOK); + bp->b_flags = B_ASYNC; + bp->b_iocmd = BIO_DELETE; + bp->b_rcred = crhold(thread0.td_ucred); + bp->b_wcred = crhold(thread0.td_ucred); + bp->b_bcount = PAGE_SIZE * npages; + bp->b_bufsize = PAGE_SIZE * npages; + bp->b_blkno = blk; + bp->b_npages = npages; + bp->b_iodone = swp_pager_async_trimdone; + bp->b_fsprivate1 = sp; + BUF_KERNPROC(bp); + return (bp); +} + /************************************************************************ * SWAP PAGER BITMAP ROUTINES * ************************************************************************/ @@ -726,6 +805,7 @@ { daddr_t blk; struct swdevt *sp; + struct buf *bp; int mpages, npages; blk = SWAPBLK_NONE; @@ -737,7 +817,8 @@ if (sp == NULL) sp = TAILQ_FIRST(&swtailq); if ((sp->sw_flags & SW_CLOSING) == 0) - blk = blist_alloc(sp->sw_blist, &npages, mpages); + blk = blist_alloc(sp->sw_blist, &sp->sw_cursor, + &npages, mpages); if (blk != SWAPBLK_NONE) break; sp = TAILQ_NEXT(sp, sw_list); @@ -765,6 +846,13 @@ swdevhd = NULL; } mtx_unlock(&sw_dev_mtx); + + if (blk != SWAPBLK_NONE) { + bp = swapdev_trim(sp); + if (bp != NULL) + swp_pager_strategy(bp); + } + return (blk); } @@ -899,18 +987,31 @@ swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size) { daddr_t addr, blk, n_free, s_free; - int i, j, n; + int i, j, k, n; + struct { + daddr_t blk; + int nblks; + } mem[size]; + for (i = k = 0; i < size; i += n, k++) { + n = size - i; + blk = swp_pager_getswapspace(&n, 1); + if (blk == SWAPBLK_NONE) + break; + mem[k].blk = blk; + mem[k].nblks = n; + } + if (i < size) { + while (k-- > 0) + swp_pager_freeswapspace(mem[k].blk, mem[k].nblks); + return (-1); + } + swp_pager_init_freerange(&s_free, &n_free); VM_OBJECT_WLOCK(object); - for (i = 0; i < size; i += n) { - n = size - i; - blk = swp_pager_getswapspace(&n, 1); - if (blk == SWAPBLK_NONE) { - swp_pager_meta_free(object, start, i); - VM_OBJECT_WUNLOCK(object); - return (-1); - } + for (i = k = 0; i < size; i += n, k++) { + blk = mem[k].blk; + n = mem[k].blks; for (j = 0; j < n; ++j) { addr = swp_pager_meta_build(object, start + i + j, blk + j); @@ -2310,6 +2411,8 @@ sp->sw_flags = flags; sp->sw_blist = blist_create(nblks, M_WAITOK); + sp->sw_cursor = 0; + sp->sw_trimmer = 0; /* * Do not free the first two block in order to avoid overwriting * any bsd label at the front of the partition