Index: sys/kern/subr_blist.c =================================================================== --- sys/kern/subr_blist.c +++ sys/kern/subr_blist.c @@ -130,6 +130,8 @@ /* * static support functions */ +static int blst_next_leaf_alloc(blmeta_t *scan, daddr_t blk, int count, + bool exact); static daddr_t blst_leaf_alloc(blmeta_t *scan, daddr_t blk, int count); static daddr_t blst_meta_alloc(blmeta_t *scan, daddr_t cursor, daddr_t count, u_daddr_t radix); @@ -193,7 +195,6 @@ /* * Use binary search, or a faster method, to find the 1 bit in a u_daddr_t. - * Assumes that the argument has only one bit set. */ static inline int bitpos(u_daddr_t mask) @@ -210,7 +211,7 @@ hi = BLIST_BMAP_RADIX; while (lo + 1 < hi) { mid = (lo + hi) >> 1; - if ((mask >> mid) != 0) + if ((mask & (((u_daddr_t)1 << mid) - 1)) == 0) lo = mid; else hi = mid; @@ -312,6 +313,71 @@ } } +/* + * blist_grow_alloc() - reserve more space in the block bitmap. Return the + * number of blocks allocated to extend the last + * allocated contiguous region. + */ +daddr_t +blist_grow_alloc(blist_t bl, daddr_t count) +{ + daddr_t blk; + blmeta_t *scan; + u_daddr_t mask, radix; + int digit, hi, lo; + + if (count == 0) + return (0); + if (bl->bl_cursor == 0) { + /* There's no previous allocation to extend. */ + return (0); + } + if (count > BLIST_MAX_ALLOC) + panic("grow allocation too large"); + scan = bl->bl_root; + blk = bl->bl_cursor; + radix = bl->bl_radix; + while (radix > BLIST_BMAP_RADIX) { + radix /= BLIST_META_RADIX; + digit = (blk / radix) & BLIST_META_MASK; + scan += 1 + digit * radix_to_skip(radix); + } + lo = blk & BLIST_BMAP_MASK; + mask = ~(scan->bm_bitmap | bitrange(0, lo)); + if (mask != 0) { + /* Count the 1 bits starting at position lo. */ + hi = bitpos(mask); + if (count >= hi - lo) + count = hi - lo; + else + hi = lo + count; + } else if (count <= BLIST_BMAP_RADIX - lo) + hi = lo + count; + else { + /* Allocate the 1 bits starting the next leaf. */ + count -= BLIST_BMAP_RADIX - lo; + count = blst_next_leaf_alloc(scan, blk, count, false); + count += BLIST_BMAP_RADIX - lo; + hi = BLIST_BMAP_RADIX; + } + /* Clear the allocated bits from this leaf. */ + scan->bm_bitmap &= ~bitrange(lo, hi - lo); + /* Fix parent bitmaps */ + while (scan->bm_bitmap == 0) { + digit = (blk / radix) & BLIST_META_MASK; + scan -= 1 + digit * radix_to_skip(radix); + scan->bm_bitmap ^= (u_daddr_t)1 << digit; + if (scan == bl->bl_root) + break; + radix *= BLIST_META_RADIX; + } + bl->bl_cursor += count; + if (bl->bl_cursor == bl->bl_blocks) + bl->bl_cursor = 0; + bl->bl_avail -= count; + return (count); +} + /* * blist_avail() - return the number of free blocks. */ @@ -603,10 +669,9 @@ * common ancestor to mark any subtrees made completely empty. */ static int -blst_next_leaf_alloc(blmeta_t *scan, daddr_t blk, int count) +blst_next_leaf_alloc(blmeta_t *scan, daddr_t blk, int count, bool exact) { blmeta_t *next; - daddr_t skip; u_daddr_t radix; int digit; @@ -618,27 +683,32 @@ next++; radix *= BLIST_META_RADIX; } - if (((next->bm_bitmap + 1) & ~((u_daddr_t)-1 << count)) != 0) { - /* - * The next leaf doesn't have enough free blocks at the - * beginning to complete the spanning allocation. - */ - return (ENOMEM); - } + if (!exact) { + if (~next->bm_bitmap != 0) + count = ummin(bitpos(~next->bm_bitmap), count); + } else if (((next->bm_bitmap + 1) & ~((u_daddr_t)-1 << count)) != 0) { + /* + * The next leaf doesn't have enough free blocks at the + * beginning to complete the spanning allocation. + */ + return (0); + } /* Clear the first 'count' bits in the next leaf to allocate. */ next->bm_bitmap &= (u_daddr_t)-1 << count; /* * Update bitmaps of next-ancestors, up to least common ancestor. */ - skip = radix_to_skip(radix); - while (radix != BLIST_BMAP_RADIX && next->bm_bitmap == 0) { - (--next)->bm_bitmap ^= 1; - radix /= BLIST_META_RADIX; - } - if (next->bm_bitmap == 0) - scan[-digit * skip].bm_bitmap ^= (u_daddr_t)1 << digit; - return (0); + while (next->bm_bitmap == 0) { + if (--next == scan) { + next[-digit * radix_to_skip(radix)].bm_bitmap ^= + (u_daddr_t)1 << digit; + break; + } + next->bm_bitmap ^= 1; + } + + return (count); } /* @@ -720,9 +790,11 @@ if (hi > BLIST_BMAP_RADIX) { /* * An allocation within this leaf is impossible, so a successful - * allocation depends on the next leaf providing some of the blocks. + * allocation depends on the next leaf providing some of the + * blocks. */ - if (blst_next_leaf_alloc(scan, blk, hi - BLIST_BMAP_RADIX) != 0) + if (blst_next_leaf_alloc(scan, blk, hi - BLIST_BMAP_RADIX, true) + == 0) /* * The hint cannot be updated, because the same * allocation request could be satisfied later, by this @@ -1109,6 +1181,14 @@ printf("?\n"); } break; + case 'x': + if (sscanf(buf + 1, "%lld", &count) == 1) { + daddr_t size = blist_grow_alloc(bl, count); + printf(" R=%08llx\n", (long long)size); + } else { + printf("?\n"); + } + break; case 'f': if (sscanf(buf + 1, "%llx %lld", &da, &count) == 2) { blist_free(bl, da, count); @@ -1130,16 +1210,22 @@ "p -print\n" "s -stats\n" "a %d -allocate\n" + "x %d -extend\n" "f %x %d -free\n" "l %x %d -fill\n" "r %d -resize\n" - "h/? -help" + "h/? -help\n" + "q -quit" ); break; + case 'q': + break; default: printf("?\n"); break; } + if (buf[0] == 'q') + break; } return(0); } Index: sys/sys/blist.h =================================================================== --- sys/sys/blist.h +++ sys/sys/blist.h @@ -34,6 +34,7 @@ * blist = blist_create(blocks, flags) * (void) blist_destroy(blist) * blkno = blist_alloc(blist, count) + * count += blist_grow_alloc(blist, grow_count) * (void) blist_free(blist, blkno, count) * nblks = blist_fill(blist, blkno, count) * (void) blist_resize(&blist, count, freeextra, flags) @@ -69,11 +70,11 @@ #define SWAPBLK_NONE ((daddr_t)((u_daddr_t)SWAPBLK_MASK + 1))/* flag */ /* - * Both blmeta and bmu_bitmap MUST be a power of 2 in size. + * Both blmeta and bm_bitmap MUST be a power of 2 in size. */ typedef struct blmeta { - u_daddr_t bm_bitmap; /* bitmap if we are a leaf */ + u_daddr_t bm_bitmap; /* bitmap marking unfilled block sets*/ daddr_t bm_bighint; /* biggest contiguous block hint*/ } blmeta_t; @@ -93,6 +94,7 @@ struct sbuf; daddr_t blist_alloc(blist_t blist, daddr_t count); +daddr_t blist_grow_alloc(blist_t blist, daddr_t count); daddr_t blist_avail(blist_t blist); blist_t blist_create(daddr_t blocks, int flags); void blist_destroy(blist_t blist); Index: sys/vm/swap_pager.c =================================================================== --- sys/vm/swap_pager.c +++ sys/vm/swap_pager.c @@ -407,7 +407,7 @@ * Swap bitmap functions */ static void swp_pager_freeswapspace(daddr_t blk, daddr_t npages); -static daddr_t swp_pager_getswapspace(int npages); +static daddr_t swp_pager_getswapspace(int *npages, int minpages); /* * Metadata functions @@ -708,9 +708,10 @@ /* * SWP_PAGER_GETSWAPSPACE() - allocate raw swap space * - * Allocate swap for the requested number of pages. The starting - * swap block number (a page index) is returned or SWAPBLK_NONE - * if the allocation failed. + * Allocate swap for up to the requested number of pages, and at + * least a minimum number of pages. The starting swap block number + * (a page index) is returned or SWAPBLK_NONE if the allocation + * failed. * * Also has the side effect of advising that somebody made a mistake * when they configured swap and didn't configure enough. @@ -720,38 +721,51 @@ * We allocate in round-robin fashion from the configured devices. */ static daddr_t -swp_pager_getswapspace(int npages) +swp_pager_getswapspace(int *io_npages, int minpages) { daddr_t blk; struct swdevt *sp; - int i; + int mpages, npages; blk = SWAPBLK_NONE; + npages = mpages = *io_npages; mtx_lock(&sw_dev_mtx); + if (TAILQ_EMPTY(&swtailq)) { + mtx_unlock(&sw_dev_mtx); + return (blk); + } sp = swdevhd; - for (i = 0; i < nswapdev; i++) { + while (npages >= minpages) { if (sp == NULL) sp = TAILQ_FIRST(&swtailq); - if (!(sp->sw_flags & SW_CLOSING)) { + if ((sp->sw_flags & SW_CLOSING) == 0) blk = blist_alloc(sp->sw_blist, npages); - if (blk != SWAPBLK_NONE) { - blk += sp->sw_first; - sp->sw_used += npages; - swap_pager_avail -= npages; - swp_sizecheck(); - swdevhd = TAILQ_NEXT(sp, sw_list); - goto done; - } + if (blk != SWAPBLK_NONE) { + npages += blist_grow_alloc(sp->sw_blist, + mpages - npages); + *io_npages = npages; + blk += sp->sw_first; + sp->sw_used += npages; + swap_pager_avail -= npages; + swp_sizecheck(); + swdevhd = TAILQ_NEXT(sp, sw_list); + break; } sp = TAILQ_NEXT(sp, sw_list); + if (swdevhd == sp) { + mpages = npages - 1; + npages >>= 1; + } } - if (swap_pager_full != 2) { - printf("swap_pager_getswapspace(%d): failed\n", npages); - swap_pager_full = 2; - swap_pager_almost_full = 1; + if (blk == SWAPBLK_NONE) { + if (swap_pager_full != 2) { + printf("swp_pager_getswapspace(%d): failed\n", + *io_npages); + swap_pager_full = 2; + swap_pager_almost_full = 1; + } + swdevhd = NULL; } - swdevhd = NULL; -done: mtx_unlock(&sw_dev_mtx); return (blk); } @@ -886,35 +900,30 @@ int swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size) { - int n = 0; - daddr_t blk = SWAPBLK_NONE; + int i, n; + daddr_t blk; vm_pindex_t beg = start; /* save start index */ daddr_t addr, n_free, s_free; swp_pager_init_freerange(&s_free, &n_free); VM_OBJECT_WLOCK(object); - while (size) { - if (n == 0) { - n = BLIST_MAX_ALLOC; - while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) { - n >>= 1; - if (n == 0) { - swp_pager_meta_free(object, beg, start - beg); - VM_OBJECT_WUNLOCK(object); - return (-1); - } - } + while (beg + size > start) { + n = min(BLIST_MAX_ALLOC, beg + size - start); + blk = swp_pager_getswapspace(&n, 1); + if (blk == SWAPBLK_NONE) { + swp_pager_meta_free(object, beg, start - beg); + VM_OBJECT_WUNLOCK(object); + return (-1); } - addr = swp_pager_meta_build(object, start, blk); - if (addr != SWAPBLK_NONE) + for (i = 0; i < n; ++i) { + addr = swp_pager_meta_build(object, start + i, blk + i); + if (addr == SWAPBLK_NONE) + continue; swp_pager_update_freerange(&s_free, &n_free, addr); - --size; - ++start; - ++blk; - --n; + } + start += n; } swp_pager_freeswapspace(s_free, n_free); - swp_pager_meta_free(object, start, n); VM_OBJECT_WUNLOCK(object); return (0); } @@ -1384,18 +1393,8 @@ n = min(BLIST_MAX_ALLOC, count - i); n = min(n, nsw_cluster_max); - /* - * Get biggest block of swap we can. If we fail, fall - * back and try to allocate a smaller block. Don't go - * overboard trying to allocate space if it would overly - * fragment swap. - */ - while ( - (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE && - n > 4 - ) { - n >>= 1; - } + /* Get a block of swap of size up to size n. */ + blk = swp_pager_getswapspace(&n, 2); if (blk == SWAPBLK_NONE) { for (j = 0; j < n; ++j) rtvals[i+j] = VM_PAGER_FAIL;