Index: sys/kern/subr_blist.c =================================================================== --- sys/kern/subr_blist.c +++ sys/kern/subr_blist.c @@ -295,7 +295,7 @@ * not be allocated. */ daddr_t -blist_alloc(blist_t bl, int *count, int maxcount) +blist_alloc(blist_t bl, daddr_t *io_cursor, int *count, int maxcount) { daddr_t blk, cursor; @@ -310,14 +310,14 @@ * non-zero. When the cursor is zero, an allocation failure will * stop further iterations. */ - for (cursor = bl->bl_cursor;; cursor = 0) { + for (cursor = *io_cursor;; cursor = 0) { blk = blst_meta_alloc(bl->bl_root, cursor, count, maxcount, bl->bl_radix); if (blk != SWAPBLK_NONE) { bl->bl_avail -= *count; - bl->bl_cursor = blk + *count; - if (bl->bl_cursor == bl->bl_blocks) - bl->bl_cursor = 0; + *io_cursor = blk + *count; + if (*io_cursor == bl->bl_blocks) + *io_cursor = 0; return (blk); } if (cursor == 0) @@ -404,8 +404,7 @@ void blist_print(blist_t bl) { - printf("BLIST avail = %jd, cursor = %08jx {\n", - (uintmax_t)bl->bl_avail, (uintmax_t)bl->bl_cursor); + printf("BLIST avail = %jd {\n", (uintmax_t)bl->bl_avail); if (bl->bl_root->bm_bitmap != 0) blst_radix_print(bl->bl_root, 0, bl->bl_radix, 4); Index: sys/sys/blist.h =================================================================== --- sys/sys/blist.h +++ sys/sys/blist.h @@ -81,7 +81,6 @@ daddr_t bl_blocks; /* area of coverage */ daddr_t bl_avail; /* # available blocks */ u_daddr_t bl_radix; /* coverage radix */ - daddr_t bl_cursor; /* next-fit search starts at */ blmeta_t bl_root[1]; /* root of radix tree */ } *blist_t; @@ -92,7 +91,7 @@ struct sbuf; -daddr_t blist_alloc(blist_t blist, int *count, int maxcount); +daddr_t blist_alloc(blist_t blist, daddr_t *cursor, int *count, int maxcount); daddr_t blist_avail(blist_t blist); blist_t blist_create(daddr_t blocks, int flags); void blist_destroy(blist_t blist); Index: sys/vm/swap_pager.h =================================================================== --- sys/vm/swap_pager.h +++ sys/vm/swap_pager.h @@ -65,6 +65,8 @@ swblk_t sw_first; swblk_t sw_end; struct blist *sw_blist; + daddr_t sw_cursor; /* next-fit search starts at */ + daddr_t sw_trimmer; /* where to look for free space to trim */ TAILQ_ENTRY(swdevt) sw_list; sw_strategy_t *sw_strategy; sw_close_t *sw_close; Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -702,6 +702,119 @@ object->type = OBJT_DEAD; } +#ifdef JUST_LOGGING_TRIM +static void +swp_pager_async_trimdone(struct buf *bp) +{ + struct swdevt *sp; + daddr_t blk; + int npages; + + sp = (struct swdevt *)bp->b_fsprivate1; + blk = bp->b_blkno; + npages = bp->b_npages; + uma_zfree(swwbuf_zone, bp); + + mtx_lock(&swbuf_mtx); + if (++nsw_wcount_async == 1) + wakeup(&nsw_wcount_async); + mtx_unlock(&swbuf_mtx); + + mtx_lock(&sw_dev_mtx); + blk -= sp->sw_first; + blist_free(sp->sw_blist, blk, npages); + mtx_unlock(&sw_dev_mtx); +} +#endif + +static int trimzone_numer = 1; +static int trimzone_denom = 5; +static int min_trim_alloc = 1; +static int max_trim_alloc = 10000; +/* + * The 'trim zone' is the small range of addresses just ahead of the cursor + * likely to be allocated soon. When the trimmer falls into the trim zone, + * we allocate blocks with the trimmer. + */ +static bool +swapdev_in_trim_zone(daddr_t start, daddr_t end, daddr_t nblks) +{ + return ((end + nblks - start) % nblks / trimzone_numer < + nblks / trimzone_denom); +} +#define JUST_LOGGING_TRIM + +static struct buf * +swapdev_trim(struct swdevt *sp, int alloc_npages) +{ +#ifndef JUST_LOGGING_TRIM + struct buf *bp; +#endif + daddr_t blk; + u_long nblks; + int npages; + + /* Quit if the cursor is too far behind the trimmer. */ + mtx_lock(&sw_dev_mtx); + nblks = sp->sw_nblks; + if (!swapdev_in_trim_zone(sp->sw_cursor, sp->sw_trimmer, nblks)) { + mtx_unlock(&sw_dev_mtx); + return (NULL); + } + + /* Grab a (hopefully) lot of free space allocated long ago. */ + npages = min_trim_alloc; + blk = blist_alloc(sp->sw_blist, &sp->sw_trimmer, + &npages, max_trim_alloc); + + /* Quit if there's nothing free. */ + if (blk == SWAPBLK_NONE) { + mtx_unlock(&sw_dev_mtx); + return (NULL); + } + if (swapdev_in_trim_zone(sp->sw_cursor, sp->sw_trimmer, nblks)) { + /* + * Trim allocation is too close to the cursor. Either we've + * wrapped around and jumped the cursor, or we've found too + * little free space close to the cursor to move the trimmer + * out of the trim zone. Bail out. + * + */ + blist_free(sp->sw_blist, blk, npages); + mtx_unlock(&sw_dev_mtx); + return (NULL); + } + blk += sp->sw_first; +#ifdef JUST_LOGGING_TRIM + blist_free(sp->sw_blist, blk, npages); + mtx_unlock(&sw_dev_mtx); + CTR4(KTR_SPARE5, "swapdev_trim: cursor %p alloc %d trimmer %p trimsize %d", + (void*)sp->sw_cursor, alloc_npages, (void*)sp->sw_trimmer, nblks); + return (NULL); +#else + mtx_unlock(&sw_dev_mtx); + mtx_lock(&swbuf_mtx); + while (nsw_wcount_async == 0) + msleep(&nsw_wcount_async, &swbuf_mtx, PVM, + "swbufa", 0); + nsw_wcount_async--; + mtx_unlock(&swbuf_mtx); + bp = uma_zalloc(swwbuf_zone, M_WAITOK); + bp->b_flags = B_ASYNC; + bp->b_iocmd = BIO_DELETE; + bp->b_rcred = crhold(thread0.td_ucred); + bp->b_wcred = crhold(thread0.td_ucred); + bp->b_bcount = PAGE_SIZE * npages; + bp->b_bufsize = PAGE_SIZE * npages; + bp->b_blkno = blk; + bp->b_npages = npages; + bp->b_iodone = swp_pager_async_trimdone; + bp->b_fsprivate1 = sp; + BUF_KERNPROC(bp); + return (bp); +#endif +} + /************************************************************************ * SWAP PAGER BITMAP ROUTINES * ************************************************************************/ @@ -726,6 +839,7 @@ { daddr_t blk; struct swdevt *sp; + struct buf *bp; int mpages, npages; blk = SWAPBLK_NONE; @@ -737,7 +851,8 @@ if (sp == NULL) sp = TAILQ_FIRST(&swtailq); if ((sp->sw_flags & SW_CLOSING) == 0) - blk = blist_alloc(sp->sw_blist, &npages, mpages); + blk = blist_alloc(sp->sw_blist, &sp->sw_cursor, + &npages, mpages); if (blk != SWAPBLK_NONE) break; sp = TAILQ_NEXT(sp, sw_list); @@ -765,6 +880,15 @@ swdevhd = NULL; } mtx_unlock(&sw_dev_mtx); + + if (blk != SWAPBLK_NONE) { + bp = swapdev_trim(sp, npages); +#ifndef JUST_LOGGING_TRIM + if (bp != NULL) + swp_pager_strategy(bp); +#endif + } + return (blk); } @@ -2310,6 +2434,8 @@ sp->sw_flags = flags; sp->sw_blist = blist_create(nblks, M_WAITOK); + sp->sw_cursor = 0; + sp->sw_trimmer = 0; /* * Do not free the first two block in order to avoid overwriting * any bsd label at the front of the partition