Index: sys/compat/linuxkpi/common/include/linux/rbtree.h =================================================================== --- sys/compat/linuxkpi/common/include/linux/rbtree.h +++ sys/compat/linuxkpi/common/include/linux/rbtree.h @@ -74,8 +74,11 @@ #define RB_EMPTY_NODE(node) (RB_PARENT(node, __entry) == node) #define RB_CLEAR_NODE(node) RB_SET_PARENT(node, node, __entry) -#define rb_insert_color(node, root) \ - linux_root_RB_INSERT_COLOR((struct linux_root *)(root), (node)) +#define rb_insert_color(node, root) do { \ + if (rb_parent(node)) \ + linux_root_RB_INSERT_COLOR((struct linux_root *)(root), \ + rb_parent(node), (node)); \ +} while (0) #define rb_erase(node, root) \ linux_root_RB_REMOVE((struct linux_root *)(root), (node)) #define rb_next(node) RB_NEXT(linux_root, NULL, (node)) @@ -132,7 +135,8 @@ struct rb_root *root) { - RB_SWAP_CHILD((struct linux_root *)root, victim, new, __entry); + RB_SWAP_CHILD((struct linux_root *)root, rb_parent(victim), + victim, new, __entry); if (victim->rb_left) RB_SET_PARENT(victim->rb_left, new, __entry); if (victim->rb_right) @@ -144,7 +148,9 @@ rb_insert_color_cached(struct rb_node *node, struct rb_root_cached *root, bool leftmost) { - linux_root_RB_INSERT_COLOR((struct linux_root *)&root->rb_root, node); + if (rb_parent(node)) + linux_root_RB_INSERT_COLOR((struct linux_root *)&root->rb_root, + rb_parent(node), node); if (leftmost) root->rb_leftmost = node; } Index: sys/sys/tree.h =================================================================== --- sys/sys/tree.h +++ sys/sys/tree.h @@ -341,8 +341,6 @@ #define RB_FLIP_LEFT(elm, field) (RB_BITS(elm, field) ^= RB_RED_L) #define RB_FLIP_RIGHT(elm, field) (RB_BITS(elm, field) ^= RB_RED_R) #define RB_FLIP_ALL(elm, field) (RB_BITS(elm, field) ^= RB_RED_MASK) -#define RB_RED_LEFT(elm, field) ((RB_BITS(elm, field) & RB_RED_L) != 0) -#define RB_RED_RIGHT(elm, field) ((RB_BITS(elm, field) & RB_RED_R) != 0) #define RB_PARENT(elm, field) ((__typeof(RB_UP(elm, field))) \ (RB_BITS(elm, field) & ~RB_RED_MASK)) #define RB_ROOT(head) (head)->rbh_root @@ -358,10 +356,14 @@ RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL; \ } while (/*CONSTCOND*/ 0) +#define RB_RED_LEFT(elm, field) ((RB_BITS(elm, field) & RB_RED_L) != 0) +#define RB_RED_RIGHT(elm, field) ((RB_BITS(elm, field) & RB_RED_R) != 0) #define RB_COLOR(elm, field) (RB_PARENT(elm, field) == NULL ? 0 : \ RB_LEFT(RB_PARENT(elm, field), field) == elm ? \ RB_RED_LEFT(RB_PARENT(elm, field), field) : \ RB_RED_RIGHT(RB_PARENT(elm, field), field)) +#undef RB_RED_LEFT +#undef RB_RED_RIGHT /* * Something to be invoked in a loop at the root of every modified subtree, @@ -378,31 +380,27 @@ } while ((rb_update_tmp = RB_PARENT(rb_update_tmp, field)) != NULL); \ } while (0) -#define RB_SWAP_CHILD(head, out, in, field) do { \ - if (RB_PARENT(out, field) == NULL) \ +#define RB_SWAP_CHILD(head, parent, out, in, field) do { \ + if ((parent) == NULL) \ RB_ROOT(head) = (in); \ - else if ((out) == RB_LEFT(RB_PARENT(out, field), field)) \ - RB_LEFT(RB_PARENT(out, field), field) = (in); \ + else if ((out) == RB_LEFT(parent, field)) \ + RB_LEFT(parent, field) = (in); \ else \ - RB_RIGHT(RB_PARENT(out, field), field) = (in); \ + RB_RIGHT(parent, field) = (in); \ } while (/*CONSTCOND*/ 0) -#define RB_ROTATE_LEFT(head, elm, tmp, field) do { \ +#define RB_ROTATE_LEFT(elm, tmp, field) do { \ if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field)) != NULL) { \ RB_SET_PARENT(RB_RIGHT(elm, field), elm, field); \ } \ - RB_SET_PARENT(tmp, RB_PARENT(elm, field), field); \ - RB_SWAP_CHILD(head, elm, tmp, field); \ RB_LEFT(tmp, field) = (elm); \ RB_SET_PARENT(elm, tmp, field); \ } while (/*CONSTCOND*/ 0) -#define RB_ROTATE_RIGHT(head, elm, tmp, field) do { \ +#define RB_ROTATE_RIGHT(elm, tmp, field) do { \ if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field)) != NULL) { \ RB_SET_PARENT(RB_LEFT(elm, field), elm, field); \ } \ - RB_SET_PARENT(tmp, RB_PARENT(elm, field), field); \ - RB_SWAP_CHILD(head, elm, tmp, field); \ RB_RIGHT(tmp, field) = (elm); \ RB_SET_PARENT(elm, tmp, field); \ } while (/*CONSTCOND*/ 0) @@ -424,7 +422,8 @@ RB_PROTOTYPE_MINMAX(name, type, attr); \ RB_PROTOTYPE_REINSERT(name, type, attr); #define RB_PROTOTYPE_INSERT_COLOR(name, type, attr) \ - attr void name##_RB_INSERT_COLOR(struct name *, struct type *) + attr void name##_RB_INSERT_COLOR(struct name *, \ + struct type *, struct type *) #define RB_PROTOTYPE_REMOVE_COLOR(name, type, attr) \ attr void name##_RB_REMOVE_COLOR(struct name *, \ struct type *, struct type *) @@ -466,7 +465,8 @@ #define RB_GENERATE_INSERT_COLOR(name, type, field, attr) \ attr void \ -name##_RB_INSERT_COLOR(struct name *head, struct type *elm) \ +name##_RB_INSERT_COLOR(struct name *head, \ + struct type *parent, struct type *elm) \ { \ /* \ * Initially, elm is a leaf. Either its parent was previously \ @@ -479,69 +479,69 @@ * when a value has been assigned to 'child' in the previous \ * one. \ */ \ - struct type *child, *parent; \ - while ((parent = RB_PARENT(elm, field)) != NULL) { \ + struct type *child; \ + __uintptr_t gp_bits; \ + do { \ + gp_bits = RB_BITS(parent, field); \ if (RB_LEFT(parent, field) == elm) { \ - if (RB_RED_LEFT(parent, field)) { \ + if (gp_bits & RB_RED_L) { \ RB_FLIP_LEFT(parent, field); \ return; \ } \ RB_FLIP_RIGHT(parent, field); \ - if (RB_RED_RIGHT(parent, field)) { \ - child = elm; \ - elm = parent; \ + if (!(gp_bits & RB_RED_MASK)) \ continue; \ - } \ - if (RB_RED_RIGHT(elm, field)) \ + if (RB_BITS(elm, field) & RB_RED_R) \ child = elm; \ else { \ /* coverity[uninit_use] */ \ - RB_ROTATE_LEFT(head, elm, child, field);\ - if (RB_RED_RIGHT(child, field)) \ + RB_ROTATE_LEFT(elm, child, field); \ + __uintptr_t red = RB_BITS(child, field); \ + if (red & RB_RED_R) \ RB_FLIP_LEFT(parent, field); \ - if (RB_RED_LEFT(child, field)) \ + if (red & RB_RED_L) \ RB_FLIP_ALL(elm, field); \ else \ RB_FLIP_LEFT(elm, field); \ - if ((RB_BITS(child, field) & \ - RB_RED_MASK) == 0) \ + if (!(red & RB_RED_MASK)) \ elm = child; \ } \ - RB_ROTATE_RIGHT(head, parent, child, field); \ + RB_ROTATE_RIGHT(parent, child, field); \ } else { \ - if (RB_RED_RIGHT(parent, field)) { \ + if (gp_bits & RB_RED_R) { \ RB_FLIP_RIGHT(parent, field); \ return; \ } \ RB_FLIP_LEFT(parent, field); \ - if (RB_RED_LEFT(parent, field)) { \ - child = elm; \ - elm = parent; \ + if (!(gp_bits & RB_RED_MASK)) \ continue; \ - } \ - if (RB_RED_LEFT(elm, field)) \ + if (RB_BITS(elm, field) & RB_RED_L) \ child = elm; \ else { \ /* coverity[uninit_use] */ \ - RB_ROTATE_RIGHT(head, elm, child, field);\ - if (RB_RED_LEFT(child, field)) \ + RB_ROTATE_RIGHT(elm, child, field); \ + __uintptr_t red = RB_BITS(child, field); \ + if (red & RB_RED_L) \ RB_FLIP_RIGHT(parent, field); \ - if (RB_RED_RIGHT(child, field)) \ + if (red & RB_RED_R) \ RB_FLIP_ALL(elm, field); \ else \ RB_FLIP_RIGHT(elm, field); \ - if ((RB_BITS(child, field) & \ - RB_RED_MASK) == 0) \ + if (!(red & RB_RED_MASK)) \ elm = child; \ } \ - RB_ROTATE_LEFT(head, parent, child, field); \ + RB_ROTATE_LEFT(parent, child, field); \ } \ - RB_BITS(child, field) &= ~RB_RED_MASK; \ + gp_bits &= ~RB_RED_MASK; \ + RB_UP(child, field) = (struct type *)gp_bits; \ + RB_SWAP_CHILD(head, (struct type *)gp_bits, \ + parent, child, field); \ if (elm != child) \ RB_AUGMENT(elm); \ RB_AUGMENT(parent); \ break; \ - } \ + } while (child = elm, elm = parent, \ + (parent = (struct type *)gp_bits) != NULL); \ } #ifndef RB_STRICT_HST @@ -561,39 +561,39 @@ struct type *parent, struct type *elm) \ { \ struct type *sib; \ + __uintptr_t gp_bits; \ if (RB_LEFT(parent, field) == elm && \ RB_RIGHT(parent, field) == elm) { \ - RB_BITS(parent, field) &= ~RB_RED_MASK; \ elm = parent; \ - parent = RB_PARENT(elm, field); \ - if (parent == NULL) \ + RB_BITS(elm, field) &= ~RB_RED_MASK; \ + if ((parent = RB_UP(elm, field)) == NULL) \ return; \ } \ do { \ + gp_bits = RB_BITS(parent, field); \ if (RB_LEFT(parent, field) == elm) { \ - if (!RB_RED_LEFT(parent, field)) { \ + if (!(gp_bits & RB_RED_L)) { \ RB_FLIP_LEFT(parent, field); \ return; \ } \ - if (RB_RED_RIGHT(parent, field)) { \ + if (gp_bits & RB_RED_R) { \ RB_FLIP_RIGHT(parent, field); \ - elm = parent; \ continue; \ } \ sib = RB_RIGHT(parent, field); \ switch (RB_BITS(sib, field) & RB_RED_MASK) { \ case RB_RED_MASK: \ RB_FLIP_ALL(sib, field); \ - elm = parent; \ continue; \ case RB_RED_R: \ elm = RB_LEFT(sib, field); \ - RB_ROTATE_RIGHT(head, sib, elm, field); \ - if (RB_RED_LEFT(elm, field)) \ + RB_ROTATE_RIGHT(sib, elm, field); \ + __uintptr_t red = RB_BITS(elm, field); \ + if (red & RB_RED_L) \ RB_FLIP_ALL(parent, field); \ else \ RB_FLIP_LEFT(parent, field); \ - if (RB_RED_RIGHT(elm, field)) \ + if (red & RB_RED_R) \ RB_FLIP_ALL(sib, field); \ else \ RB_FLIP_RIGHT(sib, field); \ @@ -613,31 +613,30 @@ elm = sib; \ break; \ } \ - RB_ROTATE_LEFT(head, parent, elm, field); \ + RB_ROTATE_LEFT(parent, elm, field); \ } else { \ - if (!RB_RED_RIGHT(parent, field)) { \ + if (!(gp_bits & RB_RED_R)) { \ RB_FLIP_RIGHT(parent, field); \ return; \ } \ - if (RB_RED_LEFT(parent, field)) { \ + if (gp_bits & RB_RED_L) { \ RB_FLIP_LEFT(parent, field); \ - elm = parent; \ continue; \ } \ sib = RB_LEFT(parent, field); \ switch (RB_BITS(sib, field) & RB_RED_MASK) { \ case RB_RED_MASK: \ RB_FLIP_ALL(sib, field); \ - elm = parent; \ continue; \ case RB_RED_L: \ elm = RB_RIGHT(sib, field); \ - RB_ROTATE_LEFT(head, sib, elm, field); \ - if (RB_RED_RIGHT(elm, field)) \ + RB_ROTATE_LEFT(sib, elm, field); \ + __uintptr_t red = RB_BITS(elm, field); \ + if (red & RB_RED_R) \ RB_FLIP_ALL(parent, field); \ else \ RB_FLIP_RIGHT(parent, field); \ - if (RB_RED_LEFT(elm, field)) \ + if (red & RB_RED_L) \ RB_FLIP_ALL(sib, field); \ else \ RB_FLIP_LEFT(sib, field); \ @@ -657,47 +656,52 @@ elm = sib; \ break; \ } \ - RB_ROTATE_RIGHT(head, parent, elm, field); \ + RB_ROTATE_RIGHT(parent, elm, field); \ } \ + gp_bits &= ~RB_RED_MASK; \ + RB_SET_PARENT(elm, (struct type *)gp_bits, field); \ + RB_SWAP_CHILD(head, (struct type *)gp_bits, \ + parent, elm, field); \ if (sib != elm) \ RB_AUGMENT(sib); \ break; \ - } while ((parent = RB_PARENT(elm, field)) != NULL); \ + } while (gp_bits &= ~RB_RED_MASK, elm = parent, \ + (parent = (struct type *)gp_bits) != NULL); \ } #define RB_GENERATE_REMOVE(name, type, field, attr) \ attr struct type * \ -name##_RB_REMOVE(struct name *head, struct type *elm) \ +name##_RB_REMOVE(struct name *head, struct type *old) \ { \ - struct type *child, *old, *parent, *right; \ + struct type *child, *elm, *old_parent, *old_up, *parent; \ \ - old = elm; \ - parent = RB_PARENT(elm, field); \ - right = RB_RIGHT(elm, field); \ - if (RB_LEFT(elm, field) == NULL) \ - elm = child = right; \ - else if (right == NULL) \ - elm = child = RB_LEFT(elm, field); \ - else { \ - if ((child = RB_LEFT(right, field)) == NULL) { \ - child = RB_RIGHT(right, field); \ - RB_RIGHT(old, field) = child; \ - parent = elm = right; \ - } else { \ - do \ - elm = child; \ - while ((child = RB_LEFT(elm, field)) != NULL); \ - child = RB_RIGHT(elm, field); \ + old_up = RB_UP(old, field); \ + old_parent = RB_PARENT(old, field); \ + elm = RB_RIGHT(old, field); \ + child = RB_LEFT(old, field); \ + if (elm == NULL || child == NULL) { \ + if (elm == NULL) \ + elm = child; \ + child = elm; \ + parent = old_parent; \ + } else { \ + parent = elm; \ + while (RB_LEFT(elm, field) != NULL) \ + elm = RB_LEFT(elm, field); \ + RB_SET_PARENT(child, elm, field); \ + RB_LEFT(elm, field) = child; \ + child = RB_RIGHT(elm, field); \ + if (parent != elm) { \ + RB_SET_PARENT(parent, elm, field); \ + RB_RIGHT(elm, field) = parent; \ parent = RB_PARENT(elm, field); \ RB_LEFT(parent, field) = child; \ - RB_SET_PARENT(RB_RIGHT(old, field), elm, field);\ } \ - RB_SET_PARENT(RB_LEFT(old, field), elm, field); \ - elm->field = old->field; \ + RB_UP(elm, field) = old_up; \ } \ - RB_SWAP_CHILD(head, old, elm, field); \ + RB_SWAP_CHILD(head, old_parent, old, elm, field); \ if (child != NULL) \ - RB_SET_PARENT(child, parent, field); \ + RB_UP(child, field) = parent; \ if (parent != NULL) { \ name##_RB_REMOVE_COLOR(head, parent, child); \ if (parent == elm && RB_LEFT(parent, field) == NULL) \ @@ -720,20 +724,22 @@ parent = tmp; \ comp = (cmp)(elm, parent); \ if (comp < 0) \ - tmp = RB_LEFT(tmp, field); \ + tmp = RB_LEFT(parent, field); \ else if (comp > 0) \ - tmp = RB_RIGHT(tmp, field); \ + tmp = RB_RIGHT(parent, field); \ else \ - return (tmp); \ + return (parent); \ } \ RB_SET(elm, parent, field); \ if (parent == NULL) \ RB_ROOT(head) = elm; \ - else if (comp < 0) \ - RB_LEFT(parent, field) = elm; \ - else \ - RB_RIGHT(parent, field) = elm; \ - name##_RB_INSERT_COLOR(head, elm); \ + else { \ + if (comp < 0) \ + RB_LEFT(parent, field) = elm; \ + else \ + RB_RIGHT(parent, field) = elm; \ + name##_RB_INSERT_COLOR(head, parent, elm); \ + } \ RB_UPDATE_AUGMENT(elm, field); \ return (NULL); \ }