Index: sys/kern/subr_blist.c =================================================================== --- sys/kern/subr_blist.c +++ sys/kern/subr_blist.c @@ -190,6 +190,14 @@ ((u_daddr_t)-1 >> (BLIST_BMAP_RADIX - (n + count)))); } +/* + * Return the argument with all but the least significant 1 bit cleared. + */ +static inline u_addr_t +leastbit(u_addr_t val) +{ + return (val & -val); +} /* * Use binary search, or a faster method, to find the 1 bit in a u_daddr_t. @@ -569,7 +577,7 @@ if (gap_stats_counting(stats)) diff ^= 1; while (diff != 0) { - bit = diff & -diff; + bit = leastbit(diff); update_gap_stats(stats, i + bitpos(bit)); diff ^= bit; } @@ -644,8 +652,7 @@ /* * BLST_LEAF_ALLOC() - allocate at a leaf in the radix tree (a bitmap). * - * This is the core of the allocator and is optimized for the - * BLIST_BMAP_RADIX block allocation case. Otherwise, execution + * This is the core of the allocator. Execution * time is proportional to log2(count) + bitpos time. */ static daddr_t @@ -658,7 +665,17 @@ count1 = count - 1; num_shifts = fls(count1); mask = scan->bm_bitmap; - while ((-mask & ~mask) != 0 && num_shifts > 0) { + while (num_shifts > 0) { + if (leastbit(mask) == -mask) { + /* + * Anding mask with right-shifted versions of itself + * won't change mask, so there's no point in more + * iteration, and growing range1 further will just + * make hinting less effective. + */ + break; + } + /* * If bit i is set in mask, then bits in [i, i+range1] are set * in scan->bm_bitmap. The value of range1 is equal to @@ -666,9 +683,9 @@ * while preserving these invariants. The updates to mask leave * fewer bits set, but each bit that remains set represents a * longer string of consecutive bits set in scan->bm_bitmap. - * If more updates to mask cannot clear more bits, because mask - * is partitioned with all 0 bits preceding all 1 bits, the loop - * terminates immediately. + * The length of the 1-strings in scan->bm_bitmap represented + * by a bit set in mask doubles, or one-less-than-doubles, with + * each iteration. */ num_shifts--; range_ext = range1 + ((count1 >> num_shifts) & 1); @@ -781,7 +798,7 @@ * Examine the nonempty subtree associated with each bit set in mask. */ do { - bit = mask & -mask; + bit = leastbit(mask); digit = bitpos(bit); i = 1 + digit * skip; if (count <= scan[i].bm_bighint) { @@ -1023,7 +1040,7 @@ mask = scan->bm_bitmap; /* Examine the nonempty subtree associated with each bit set in mask */ do { - bit = mask & -mask; + bit = leastbit(mask); digit = bitpos(bit); blst_radix_print(&scan[1 + digit * skip], blk + digit * radix, radix, tab);