Changeset View
Changeset View
Standalone View
Standalone View
sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c
Show All 27 Lines | |||||
#include <sys/zfs_context.h> | #include <sys/zfs_context.h> | ||||
#include <sys/spa.h> | #include <sys/spa.h> | ||||
#include <sys/dmu.h> | #include <sys/dmu.h> | ||||
#include <sys/dnode.h> | #include <sys/dnode.h> | ||||
#include <sys/zio.h> | #include <sys/zio.h> | ||||
#include <sys/range_tree.h> | #include <sys/range_tree.h> | ||||
/* | |||||
* Range trees are tree-based data structures that can be used to | |||||
* track free space or generally any space allocation information. | |||||
* A range tree keeps track of individual segments and automatically | |||||
* provides facilities such as adjacent extent merging and extent | |||||
* splitting in response to range add/remove requests. | |||||
* | |||||
* A range tree starts out completely empty, with no segments in it. | |||||
* Adding an allocation via range_tree_add to the range tree can either: | |||||
* 1) create a new extent | |||||
* 2) extend an adjacent extent | |||||
* 3) merge two adjacent extents | |||||
* Conversely, removing an allocation via range_tree_remove can: | |||||
* 1) completely remove an extent | |||||
* 2) shorten an extent (if the allocation was near one of its ends) | |||||
* 3) split an extent into two extents, in effect punching a hole | |||||
* | |||||
* A range tree is also capable of 'bridging' gaps when adding | |||||
* allocations. This is useful for cases when close proximity of | |||||
* allocations is an important detail that needs to be represented | |||||
* in the range tree. See range_tree_set_gap(). The default behavior | |||||
* is not to bridge gaps (i.e. the maximum allowed gap size is 0). | |||||
* | |||||
* In order to traverse a range tree, use either the range_tree_walk() | |||||
* or range_tree_vacate() functions. | |||||
* | |||||
* To obtain more accurate information on individual segment | |||||
* operations that the range tree performs "under the hood", you can | |||||
* specify a set of callbacks by passing a range_tree_ops_t structure | |||||
* to the range_tree_create function. Any callbacks that are non-NULL | |||||
* are then called at the appropriate times. | |||||
* | |||||
* The range tree code also supports a special variant of range trees | |||||
* that can bridge small gaps between segments. This kind of tree is used | |||||
* by the dsl scanning code to group I/Os into mostly sequential chunks to | |||||
* optimize disk performance. The code here attempts to do this with as | |||||
* little memory and computational overhead as possible. One limitation of | |||||
* this implementation is that segments of range trees with gaps can only | |||||
* support removing complete segments. | |||||
*/ | |||||
kmem_cache_t *range_seg_cache; | kmem_cache_t *range_seg_cache; | ||||
/* Generic ops for managing an AVL tree alongside a range tree */ | |||||
struct range_tree_ops rt_avl_ops = { | |||||
.rtop_create = rt_avl_create, | |||||
.rtop_destroy = rt_avl_destroy, | |||||
.rtop_add = rt_avl_add, | |||||
.rtop_remove = rt_avl_remove, | |||||
.rtop_vacate = rt_avl_vacate, | |||||
}; | |||||
void | void | ||||
range_tree_init(void) | range_tree_init(void) | ||||
{ | { | ||||
ASSERT(range_seg_cache == NULL); | ASSERT(range_seg_cache == NULL); | ||||
range_seg_cache = kmem_cache_create("range_seg_cache", | range_seg_cache = kmem_cache_create("range_seg_cache", | ||||
sizeof (range_seg_t), 0, NULL, NULL, NULL, NULL, NULL, 0); | sizeof (range_seg_t), 0, NULL, NULL, NULL, NULL, NULL, 0); | ||||
} | } | ||||
Show All 24 Lines | for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) { | ||||
if (hist[i] != rt->rt_histogram[i]) { | if (hist[i] != rt->rt_histogram[i]) { | ||||
zfs_dbgmsg("i=%d, hist=%p, hist=%llu, rt_hist=%llu", | zfs_dbgmsg("i=%d, hist=%p, hist=%llu, rt_hist=%llu", | ||||
i, hist, hist[i], rt->rt_histogram[i]); | i, hist, hist[i], rt->rt_histogram[i]); | ||||
} | } | ||||
VERIFY3U(hist[i], ==, rt->rt_histogram[i]); | VERIFY3U(hist[i], ==, rt->rt_histogram[i]); | ||||
} | } | ||||
} | } | ||||
/* | |||||
* Changes out the lock used by the range tree. Useful when you are moving | |||||
* the range tree between containing structures without having to recreate | |||||
* it. Both the old and new locks must be held by the caller. | |||||
*/ | |||||
void | |||||
range_tree_set_lock(range_tree_t *rt, kmutex_t *lp) | |||||
{ | |||||
ASSERT(MUTEX_HELD(rt->rt_lock) && MUTEX_HELD(lp)); | |||||
rt->rt_lock = lp; | |||||
} | |||||
static void | static void | ||||
range_tree_stat_incr(range_tree_t *rt, range_seg_t *rs) | range_tree_stat_incr(range_tree_t *rt, range_seg_t *rs) | ||||
{ | { | ||||
uint64_t size = rs->rs_end - rs->rs_start; | uint64_t size = rs->rs_end - rs->rs_start; | ||||
int idx = highbit64(size) - 1; | int idx = highbit64(size) - 1; | ||||
ASSERT(size != 0); | ASSERT(size != 0); | ||||
ASSERT3U(idx, <, | ASSERT3U(idx, <, | ||||
Show All 18 Lines | |||||
} | } | ||||
/* | /* | ||||
* NOTE: caller is responsible for all locking. | * NOTE: caller is responsible for all locking. | ||||
*/ | */ | ||||
static int | static int | ||||
range_tree_seg_compare(const void *x1, const void *x2) | range_tree_seg_compare(const void *x1, const void *x2) | ||||
{ | { | ||||
const range_seg_t *r1 = x1; | const range_seg_t *r1 = (const range_seg_t *)x1; | ||||
const range_seg_t *r2 = x2; | const range_seg_t *r2 = (const range_seg_t *)x2; | ||||
if (r1->rs_start < r2->rs_start) { | ASSERT3U(r1->rs_start, <=, r1->rs_end); | ||||
if (r1->rs_end > r2->rs_start) | ASSERT3U(r2->rs_start, <=, r2->rs_end); | ||||
return (0); | |||||
return (-1); | return ((r1->rs_start >= r2->rs_end) - (r1->rs_end <= r2->rs_start)); | ||||
} | } | ||||
if (r1->rs_start > r2->rs_start) { | |||||
if (r1->rs_start < r2->rs_end) | |||||
return (0); | |||||
return (1); | |||||
} | |||||
return (0); | |||||
} | |||||
range_tree_t * | range_tree_t * | ||||
range_tree_create(range_tree_ops_t *ops, void *arg) | range_tree_create_impl(range_tree_ops_t *ops, void *arg, | ||||
int (*avl_compare) (const void *, const void *), kmutex_t *lp, uint64_t gap) | |||||
{ | { | ||||
range_tree_t *rt; | range_tree_t *rt = kmem_zalloc(sizeof (range_tree_t), KM_SLEEP); | ||||
rt = kmem_zalloc(sizeof (range_tree_t), KM_SLEEP); | |||||
avl_create(&rt->rt_root, range_tree_seg_compare, | avl_create(&rt->rt_root, range_tree_seg_compare, | ||||
sizeof (range_seg_t), offsetof(range_seg_t, rs_node)); | sizeof (range_seg_t), offsetof(range_seg_t, rs_node)); | ||||
rt->rt_lock = lp; | |||||
rt->rt_ops = ops; | rt->rt_ops = ops; | ||||
rt->rt_arg = arg; | rt->rt_arg = arg; | ||||
rt->rt_gap = gap; | |||||
rt->rt_avl_compare = avl_compare; | |||||
if (rt->rt_ops != NULL) | if (rt->rt_ops != NULL && rt->rt_ops->rtop_create != NULL) | ||||
rt->rt_ops->rtop_create(rt, rt->rt_arg); | rt->rt_ops->rtop_create(rt, rt->rt_arg); | ||||
return (rt); | return (rt); | ||||
} | } | ||||
range_tree_t * | |||||
range_tree_create(range_tree_ops_t *ops, void *arg, kmutex_t *lp) | |||||
{ | |||||
return (range_tree_create_impl(ops, arg, NULL, lp, 0)); | |||||
} | |||||
void | void | ||||
range_tree_destroy(range_tree_t *rt) | range_tree_destroy(range_tree_t *rt) | ||||
{ | { | ||||
VERIFY0(rt->rt_space); | VERIFY0(rt->rt_space); | ||||
if (rt->rt_ops != NULL) | if (rt->rt_ops != NULL && rt->rt_ops->rtop_destroy != NULL) | ||||
rt->rt_ops->rtop_destroy(rt, rt->rt_arg); | rt->rt_ops->rtop_destroy(rt, rt->rt_arg); | ||||
avl_destroy(&rt->rt_root); | avl_destroy(&rt->rt_root); | ||||
kmem_free(rt, sizeof (*rt)); | kmem_free(rt, sizeof (*rt)); | ||||
} | } | ||||
void | void | ||||
range_tree_add(void *arg, uint64_t start, uint64_t size) | range_tree_adjust_fill(range_tree_t *rt, range_seg_t *rs, int64_t delta) | ||||
{ | { | ||||
ASSERT(MUTEX_HELD(rt->rt_lock)); | |||||
ASSERT3U(rs->rs_fill + delta, !=, 0); | |||||
ASSERT3U(rs->rs_fill + delta, <=, rs->rs_end - rs->rs_start); | |||||
if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) | |||||
rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg); | |||||
rs->rs_fill += delta; | |||||
if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL) | |||||
rt->rt_ops->rtop_add(rt, rs, rt->rt_arg); | |||||
} | |||||
static void | |||||
range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill) | |||||
{ | |||||
range_tree_t *rt = arg; | range_tree_t *rt = arg; | ||||
avl_index_t where; | avl_index_t where; | ||||
range_seg_t rsearch, *rs_before, *rs_after, *rs; | range_seg_t rsearch, *rs_before, *rs_after, *rs; | ||||
uint64_t end = start + size; | uint64_t end = start + size, gap = rt->rt_gap; | ||||
uint64_t bridge_size = 0; | |||||
boolean_t merge_before, merge_after; | boolean_t merge_before, merge_after; | ||||
VERIFY(size != 0); | ASSERT3U(size, !=, 0); | ||||
ASSERT3U(fill, <=, size); | |||||
rsearch.rs_start = start; | rsearch.rs_start = start; | ||||
rsearch.rs_end = end; | rsearch.rs_end = end; | ||||
rs = avl_find(&rt->rt_root, &rsearch, &where); | rs = avl_find(&rt->rt_root, &rsearch, &where); | ||||
if (rs != NULL && rs->rs_start <= start && rs->rs_end >= end) { | if (gap == 0 && rs != NULL && | ||||
rs->rs_start <= start && rs->rs_end >= end) { | |||||
zfs_panic_recover("zfs: allocating allocated segment" | zfs_panic_recover("zfs: allocating allocated segment" | ||||
"(offset=%llu size=%llu)\n", | "(offset=%llu size=%llu) of (offset=%llu size=%llu)\n", | ||||
(longlong_t)start, (longlong_t)size); | (longlong_t)start, (longlong_t)size, | ||||
(longlong_t)rs->rs_start, | |||||
(longlong_t)rs->rs_end - rs->rs_start); | |||||
return; | return; | ||||
} | } | ||||
/* Make sure we don't overlap with either of our neighbors */ | /* | ||||
VERIFY(rs == NULL); | * If this is a gap-supporting range tree, it is possible that we | ||||
* are inserting into an existing segment. In this case simply | |||||
* bump the fill count and call the remove / add callbacks. If the | |||||
* new range will extend an existing segment, we remove the | |||||
* existing one, apply the new extent to it and re-insert it using | |||||
* the normal code paths. | |||||
*/ | |||||
if (rs != NULL) { | |||||
ASSERT3U(gap, !=, 0); | |||||
if (rs->rs_start <= start && rs->rs_end >= end) { | |||||
range_tree_adjust_fill(rt, rs, fill); | |||||
return; | |||||
} | |||||
avl_remove(&rt->rt_root, rs); | |||||
if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) | |||||
rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg); | |||||
range_tree_stat_decr(rt, rs); | |||||
rt->rt_space -= rs->rs_end - rs->rs_start; | |||||
fill += rs->rs_fill; | |||||
start = MIN(start, rs->rs_start); | |||||
end = MAX(end, rs->rs_end); | |||||
size = end - start; | |||||
range_tree_add_impl(rt, start, size, fill); | |||||
kmem_cache_free(range_seg_cache, rs); | |||||
return; | |||||
} | |||||
ASSERT3P(rs, ==, NULL); | |||||
/* | |||||
* Determine whether or not we will have to merge with our neighbors. | |||||
* If gap != 0, we might need to merge with our neighbors even if we | |||||
* aren't directly touching. | |||||
*/ | |||||
rs_before = avl_nearest(&rt->rt_root, where, AVL_BEFORE); | rs_before = avl_nearest(&rt->rt_root, where, AVL_BEFORE); | ||||
rs_after = avl_nearest(&rt->rt_root, where, AVL_AFTER); | rs_after = avl_nearest(&rt->rt_root, where, AVL_AFTER); | ||||
merge_before = (rs_before != NULL && rs_before->rs_end == start); | merge_before = (rs_before != NULL && rs_before->rs_end >= start - gap); | ||||
merge_after = (rs_after != NULL && rs_after->rs_start == end); | merge_after = (rs_after != NULL && rs_after->rs_start <= end + gap); | ||||
if (merge_before && gap != 0) | |||||
bridge_size += start - rs_before->rs_end; | |||||
if (merge_after && gap != 0) | |||||
bridge_size += rs_after->rs_start - end; | |||||
if (merge_before && merge_after) { | if (merge_before && merge_after) { | ||||
avl_remove(&rt->rt_root, rs_before); | avl_remove(&rt->rt_root, rs_before); | ||||
if (rt->rt_ops != NULL) { | if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) { | ||||
rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg); | rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg); | ||||
rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg); | rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg); | ||||
} | } | ||||
range_tree_stat_decr(rt, rs_before); | range_tree_stat_decr(rt, rs_before); | ||||
range_tree_stat_decr(rt, rs_after); | range_tree_stat_decr(rt, rs_after); | ||||
rs_after->rs_fill += rs_before->rs_fill + fill; | |||||
rs_after->rs_start = rs_before->rs_start; | rs_after->rs_start = rs_before->rs_start; | ||||
kmem_cache_free(range_seg_cache, rs_before); | kmem_cache_free(range_seg_cache, rs_before); | ||||
rs = rs_after; | rs = rs_after; | ||||
} else if (merge_before) { | } else if (merge_before) { | ||||
if (rt->rt_ops != NULL) | if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) | ||||
rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg); | rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg); | ||||
range_tree_stat_decr(rt, rs_before); | range_tree_stat_decr(rt, rs_before); | ||||
rs_before->rs_fill += fill; | |||||
rs_before->rs_end = end; | rs_before->rs_end = end; | ||||
rs = rs_before; | rs = rs_before; | ||||
} else if (merge_after) { | } else if (merge_after) { | ||||
if (rt->rt_ops != NULL) | if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) | ||||
rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg); | rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg); | ||||
range_tree_stat_decr(rt, rs_after); | range_tree_stat_decr(rt, rs_after); | ||||
rs_after->rs_fill += fill; | |||||
rs_after->rs_start = start; | rs_after->rs_start = start; | ||||
rs = rs_after; | rs = rs_after; | ||||
} else { | } else { | ||||
rs = kmem_cache_alloc(range_seg_cache, KM_SLEEP); | rs = kmem_cache_alloc(range_seg_cache, KM_SLEEP); | ||||
rs->rs_fill = fill; | |||||
rs->rs_start = start; | rs->rs_start = start; | ||||
rs->rs_end = end; | rs->rs_end = end; | ||||
avl_insert(&rt->rt_root, rs, where); | avl_insert(&rt->rt_root, rs, where); | ||||
} | } | ||||
if (rt->rt_ops != NULL) | if (gap != 0) | ||||
ASSERT3U(rs->rs_fill, <=, rs->rs_end - rs->rs_start); | |||||
else | |||||
ASSERT3U(rs->rs_fill, ==, rs->rs_end - rs->rs_start); | |||||
if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL) | |||||
rt->rt_ops->rtop_add(rt, rs, rt->rt_arg); | rt->rt_ops->rtop_add(rt, rs, rt->rt_arg); | ||||
range_tree_stat_incr(rt, rs); | range_tree_stat_incr(rt, rs); | ||||
rt->rt_space += size; | rt->rt_space += size + bridge_size; | ||||
} | } | ||||
void | void | ||||
range_tree_remove(void *arg, uint64_t start, uint64_t size) | range_tree_add(void *arg, uint64_t start, uint64_t size) | ||||
{ | { | ||||
range_tree_t *rt = arg; | range_tree_add_impl(arg, start, size, size); | ||||
} | |||||
static void | |||||
range_tree_remove_impl(range_tree_t *rt, uint64_t start, uint64_t size, | |||||
boolean_t do_fill) | |||||
{ | |||||
avl_index_t where; | avl_index_t where; | ||||
range_seg_t rsearch, *rs, *newseg; | range_seg_t rsearch, *rs, *newseg; | ||||
uint64_t end = start + size; | uint64_t end = start + size; | ||||
boolean_t left_over, right_over; | boolean_t left_over, right_over; | ||||
VERIFY3U(size, !=, 0); | VERIFY3U(size, !=, 0); | ||||
VERIFY3U(size, <=, rt->rt_space); | VERIFY3U(size, <=, rt->rt_space); | ||||
rsearch.rs_start = start; | rsearch.rs_start = start; | ||||
rsearch.rs_end = end; | rsearch.rs_end = end; | ||||
rs = avl_find(&rt->rt_root, &rsearch, &where); | rs = avl_find(&rt->rt_root, &rsearch, &where); | ||||
/* Make sure we completely overlap with someone */ | /* Make sure we completely overlap with someone */ | ||||
if (rs == NULL) { | if (rs == NULL) { | ||||
zfs_panic_recover("zfs: freeing free segment " | zfs_panic_recover("zfs: freeing free segment " | ||||
"(offset=%llu size=%llu)", | "(offset=%llu size=%llu)", | ||||
(longlong_t)start, (longlong_t)size); | (longlong_t)start, (longlong_t)size); | ||||
return; | return; | ||||
} | } | ||||
/* | |||||
* Range trees with gap support must only remove complete segments | |||||
* from the tree. This allows us to maintain accurate fill accounting | |||||
* and to ensure that bridged sections are not leaked. If we need to | |||||
* remove less than the full segment, we can only adjust the fill count. | |||||
*/ | |||||
if (rt->rt_gap != 0) { | |||||
if (do_fill) { | |||||
if (rs->rs_fill == size) { | |||||
start = rs->rs_start; | |||||
end = rs->rs_end; | |||||
size = end - start; | |||||
} else { | |||||
range_tree_adjust_fill(rt, rs, -size); | |||||
return; | |||||
} | |||||
} else if (rs->rs_start != start || rs->rs_end != end) { | |||||
zfs_panic_recover("zfs: freeing partial segment of " | |||||
"gap tree (offset=%llu size=%llu) of " | |||||
"(offset=%llu size=%llu)", | |||||
(longlong_t)start, (longlong_t)size, | |||||
(longlong_t)rs->rs_start, | |||||
(longlong_t)rs->rs_end - rs->rs_start); | |||||
return; | |||||
} | |||||
} | |||||
VERIFY3U(rs->rs_start, <=, start); | VERIFY3U(rs->rs_start, <=, start); | ||||
VERIFY3U(rs->rs_end, >=, end); | VERIFY3U(rs->rs_end, >=, end); | ||||
left_over = (rs->rs_start != start); | left_over = (rs->rs_start != start); | ||||
right_over = (rs->rs_end != end); | right_over = (rs->rs_end != end); | ||||
range_tree_stat_decr(rt, rs); | range_tree_stat_decr(rt, rs); | ||||
if (rt->rt_ops != NULL) | if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) | ||||
rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg); | rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg); | ||||
if (left_over && right_over) { | if (left_over && right_over) { | ||||
newseg = kmem_cache_alloc(range_seg_cache, KM_SLEEP); | newseg = kmem_cache_alloc(range_seg_cache, KM_SLEEP); | ||||
newseg->rs_start = end; | newseg->rs_start = end; | ||||
newseg->rs_end = rs->rs_end; | newseg->rs_end = rs->rs_end; | ||||
newseg->rs_fill = newseg->rs_end - newseg->rs_start; | |||||
range_tree_stat_incr(rt, newseg); | range_tree_stat_incr(rt, newseg); | ||||
rs->rs_end = start; | rs->rs_end = start; | ||||
avl_insert_here(&rt->rt_root, newseg, rs, AVL_AFTER); | avl_insert_here(&rt->rt_root, newseg, rs, AVL_AFTER); | ||||
if (rt->rt_ops != NULL) | if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL) | ||||
rt->rt_ops->rtop_add(rt, newseg, rt->rt_arg); | rt->rt_ops->rtop_add(rt, newseg, rt->rt_arg); | ||||
} else if (left_over) { | } else if (left_over) { | ||||
rs->rs_end = start; | rs->rs_end = start; | ||||
} else if (right_over) { | } else if (right_over) { | ||||
rs->rs_start = end; | rs->rs_start = end; | ||||
} else { | } else { | ||||
avl_remove(&rt->rt_root, rs); | avl_remove(&rt->rt_root, rs); | ||||
kmem_cache_free(range_seg_cache, rs); | kmem_cache_free(range_seg_cache, rs); | ||||
rs = NULL; | rs = NULL; | ||||
} | } | ||||
if (rs != NULL) { | if (rs != NULL) { | ||||
/* | |||||
* The fill of the leftover segment will always be equal to | |||||
* the size, since we do not support removing partial segments | |||||
* of range trees with gaps. | |||||
*/ | |||||
rs->rs_fill = rs->rs_end - rs->rs_start; | |||||
range_tree_stat_incr(rt, rs); | range_tree_stat_incr(rt, rs); | ||||
if (rt->rt_ops != NULL) | if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL) | ||||
rt->rt_ops->rtop_add(rt, rs, rt->rt_arg); | rt->rt_ops->rtop_add(rt, rs, rt->rt_arg); | ||||
} | } | ||||
rt->rt_space -= size; | rt->rt_space -= size; | ||||
} | } | ||||
void | |||||
range_tree_remove(void *arg, uint64_t start, uint64_t size) | |||||
{ | |||||
range_tree_remove_impl(arg, start, size, B_FALSE); | |||||
} | |||||
void | |||||
range_tree_remove_fill(range_tree_t *rt, uint64_t start, uint64_t size) | |||||
{ | |||||
range_tree_remove_impl(rt, start, size, B_TRUE); | |||||
} | |||||
void | |||||
range_tree_resize_segment(range_tree_t *rt, range_seg_t *rs, | |||||
uint64_t newstart, uint64_t newsize) | |||||
{ | |||||
int64_t delta = newsize - (rs->rs_end - rs->rs_start); | |||||
ASSERT(MUTEX_HELD(rt->rt_lock)); | |||||
range_tree_stat_decr(rt, rs); | |||||
if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) | |||||
rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg); | |||||
rs->rs_start = newstart; | |||||
rs->rs_end = newstart + newsize; | |||||
range_tree_stat_incr(rt, rs); | |||||
if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL) | |||||
rt->rt_ops->rtop_add(rt, rs, rt->rt_arg); | |||||
rt->rt_space += delta; | |||||
} | |||||
static range_seg_t * | static range_seg_t * | ||||
range_tree_find_impl(range_tree_t *rt, uint64_t start, uint64_t size) | range_tree_find_impl(range_tree_t *rt, uint64_t start, uint64_t size) | ||||
{ | { | ||||
avl_index_t where; | avl_index_t where; | ||||
range_seg_t rsearch; | range_seg_t rsearch; | ||||
uint64_t end = start + size; | uint64_t end = start + size; | ||||
VERIFY(size != 0); | VERIFY(size != 0); | ||||
rsearch.rs_start = start; | rsearch.rs_start = start; | ||||
rsearch.rs_end = end; | rsearch.rs_end = end; | ||||
return (avl_find(&rt->rt_root, &rsearch, &where)); | return (avl_find(&rt->rt_root, &rsearch, &where)); | ||||
} | } | ||||
static range_seg_t * | range_seg_t * | ||||
range_tree_find(range_tree_t *rt, uint64_t start, uint64_t size) | range_tree_find(range_tree_t *rt, uint64_t start, uint64_t size) | ||||
{ | { | ||||
range_seg_t *rs = range_tree_find_impl(rt, start, size); | range_seg_t *rs = range_tree_find_impl(rt, start, size); | ||||
if (rs != NULL && rs->rs_start <= start && rs->rs_end >= start + size) | if (rs != NULL && rs->rs_start <= start && rs->rs_end >= start + size) | ||||
return (rs); | return (rs); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
range_tree_vacate(range_tree_t *rt, range_tree_func_t *func, void *arg) | range_tree_vacate(range_tree_t *rt, range_tree_func_t *func, void *arg) | ||||
{ | { | ||||
range_seg_t *rs; | range_seg_t *rs; | ||||
void *cookie = NULL; | void *cookie = NULL; | ||||
if (rt->rt_ops != NULL) | if (rt->rt_ops != NULL && rt->rt_ops->rtop_vacate != NULL) | ||||
rt->rt_ops->rtop_vacate(rt, rt->rt_arg); | rt->rt_ops->rtop_vacate(rt, rt->rt_arg); | ||||
while ((rs = avl_destroy_nodes(&rt->rt_root, &cookie)) != NULL) { | while ((rs = avl_destroy_nodes(&rt->rt_root, &cookie)) != NULL) { | ||||
if (func != NULL) | if (func != NULL) | ||||
func(arg, rs->rs_start, rs->rs_end - rs->rs_start); | func(arg, rs->rs_start, rs->rs_end - rs->rs_start); | ||||
kmem_cache_free(range_seg_cache, rs); | kmem_cache_free(range_seg_cache, rs); | ||||
} | } | ||||
bzero(rt->rt_histogram, sizeof (rt->rt_histogram)); | bzero(rt->rt_histogram, sizeof (rt->rt_histogram)); | ||||
rt->rt_space = 0; | rt->rt_space = 0; | ||||
} | } | ||||
void | void | ||||
range_tree_walk(range_tree_t *rt, range_tree_func_t *func, void *arg) | range_tree_walk(range_tree_t *rt, range_tree_func_t *func, void *arg) | ||||
{ | { | ||||
range_seg_t *rs; | range_seg_t *rs; | ||||
for (rs = avl_first(&rt->rt_root); rs; rs = AVL_NEXT(&rt->rt_root, rs)) | for (rs = avl_first(&rt->rt_root); rs; rs = AVL_NEXT(&rt->rt_root, rs)) | ||||
func(arg, rs->rs_start, rs->rs_end - rs->rs_start); | func(arg, rs->rs_start, rs->rs_end - rs->rs_start); | ||||
} | } | ||||
range_seg_t * | |||||
range_tree_first(range_tree_t *rt) | |||||
{ | |||||
ASSERT(MUTEX_HELD(rt->rt_lock)); | |||||
return (avl_first(&rt->rt_root)); | |||||
} | |||||
uint64_t | uint64_t | ||||
range_tree_space(range_tree_t *rt) | range_tree_space(range_tree_t *rt) | ||||
{ | { | ||||
return (rt->rt_space); | return (rt->rt_space); | ||||
} | |||||
/* Generic range tree functions for maintaining segments in an AVL tree. */ | |||||
void | |||||
rt_avl_create(range_tree_t *rt, void *arg) | |||||
{ | |||||
avl_tree_t *tree = arg; | |||||
avl_create(tree, rt->rt_avl_compare, sizeof (range_seg_t), | |||||
offsetof(range_seg_t, rs_pp_node)); | |||||
} | |||||
void | |||||
rt_avl_destroy(range_tree_t *rt, void *arg) | |||||
{ | |||||
avl_tree_t *tree = arg; | |||||
ASSERT0(avl_numnodes(tree)); | |||||
avl_destroy(tree); | |||||
} | |||||
void | |||||
rt_avl_add(range_tree_t *rt, range_seg_t *rs, void *arg) | |||||
{ | |||||
avl_tree_t *tree = arg; | |||||
avl_add(tree, rs); | |||||
} | |||||
void | |||||
rt_avl_remove(range_tree_t *rt, range_seg_t *rs, void *arg) | |||||
{ | |||||
avl_tree_t *tree = arg; | |||||
avl_remove(tree, rs); | |||||
} | |||||
void | |||||
rt_avl_vacate(range_tree_t *rt, void *arg) | |||||
{ | |||||
/* | |||||
* Normally one would walk the tree freeing nodes along the way. | |||||
* Since the nodes are shared with the range trees we can avoid | |||||
* walking all nodes and just reinitialize the avl tree. The nodes | |||||
* will be freed by the range tree, so we don't want to free them here. | |||||
*/ | |||||
rt_avl_create(rt, arg); | |||||
} | } | ||||
boolean_t | boolean_t | ||||
range_tree_is_empty(range_tree_t *rt) | range_tree_is_empty(range_tree_t *rt) | ||||
{ | { | ||||
ASSERT(rt != NULL); | ASSERT(rt != NULL); | ||||
return (range_tree_space(rt) == 0); | return (range_tree_space(rt) == 0); | ||||
} | } |