rbtree: move augmented rbtree functionality to rbtree_augmented.h
Provide rb_insert_augmented() and rb_erase_augmented() through a new
rbtree_augmented.h include file. rb_erase_augmented() is defined there as
an __always_inline function, in order to allow inlining of augmented
rbtree callbacks into it. Since this generates a relatively large
function, each augmented rbtree user should make sure to have a single
call site.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
diff --git a/lib/rbtree.c b/lib/rbtree.c
index c0088ca..4f56a11 100644
--- a/lib/rbtree.c
+++ b/lib/rbtree.c
@@ -21,7 +21,7 @@
linux/lib/rbtree.c
*/
-#include <linux/rbtree.h>
+#include <linux/rbtree_augmented.h>
#include <linux/export.h>
/*
@@ -44,52 +44,16 @@
* parentheses and have some accompanying text comment.
*/
-#define RB_RED 0
-#define RB_BLACK 1
-
-#define __rb_parent(pc) ((struct rb_node *)(pc & ~3))
-
-#define __rb_color(pc) ((pc) & 1)
-#define __rb_is_black(pc) __rb_color(pc)
-#define __rb_is_red(pc) (!__rb_color(pc))
-#define rb_color(rb) __rb_color((rb)->__rb_parent_color)
-#define rb_is_red(rb) __rb_is_red((rb)->__rb_parent_color)
-#define rb_is_black(rb) __rb_is_black((rb)->__rb_parent_color)
-
static inline void rb_set_black(struct rb_node *rb)
{
rb->__rb_parent_color |= RB_BLACK;
}
-static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p)
-{
- rb->__rb_parent_color = rb_color(rb) | (unsigned long)p;
-}
-
-static inline void rb_set_parent_color(struct rb_node *rb,
- struct rb_node *p, int color)
-{
- rb->__rb_parent_color = (unsigned long)p | color;
-}
-
static inline struct rb_node *rb_red_parent(struct rb_node *red)
{
return (struct rb_node *)red->__rb_parent_color;
}
-static inline void
-__rb_change_child(struct rb_node *old, struct rb_node *new,
- struct rb_node *parent, struct rb_root *root)
-{
- if (parent) {
- if (parent->rb_left == old)
- parent->rb_left = new;
- else
- parent->rb_right = new;
- } else
- root->rb_node = new;
-}
-
/*
* Helper function for rotations:
* - old's parent and color get assigned to new
@@ -230,9 +194,9 @@
}
}
-static __always_inline void
+__always_inline void
__rb_erase_color(struct rb_node *parent, struct rb_root *root,
- const struct rb_augment_callbacks *augment)
+ void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
{
struct rb_node *node = NULL, *sibling, *tmp1, *tmp2;
@@ -261,7 +225,7 @@
rb_set_parent_color(tmp1, parent, RB_BLACK);
__rb_rotate_set_parents(parent, sibling, root,
RB_RED);
- augment->rotate(parent, sibling);
+ augment_rotate(parent, sibling);
sibling = tmp1;
}
tmp1 = sibling->rb_right;
@@ -313,7 +277,7 @@
if (tmp1)
rb_set_parent_color(tmp1, sibling,
RB_BLACK);
- augment->rotate(sibling, tmp2);
+ augment_rotate(sibling, tmp2);
tmp1 = sibling;
sibling = tmp2;
}
@@ -336,7 +300,7 @@
rb_set_parent(tmp2, parent);
__rb_rotate_set_parents(parent, sibling, root,
RB_BLACK);
- augment->rotate(parent, sibling);
+ augment_rotate(parent, sibling);
break;
} else {
sibling = parent->rb_left;
@@ -347,7 +311,7 @@
rb_set_parent_color(tmp1, parent, RB_BLACK);
__rb_rotate_set_parents(parent, sibling, root,
RB_RED);
- augment->rotate(parent, sibling);
+ augment_rotate(parent, sibling);
sibling = tmp1;
}
tmp1 = sibling->rb_left;
@@ -374,7 +338,7 @@
if (tmp1)
rb_set_parent_color(tmp1, sibling,
RB_BLACK);
- augment->rotate(sibling, tmp2);
+ augment_rotate(sibling, tmp2);
tmp1 = sibling;
sibling = tmp2;
}
@@ -386,109 +350,12 @@
rb_set_parent(tmp2, parent);
__rb_rotate_set_parents(parent, sibling, root,
RB_BLACK);
- augment->rotate(parent, sibling);
+ augment_rotate(parent, sibling);
break;
}
}
}
-
-static __always_inline void
-__rb_erase(struct rb_node *node, struct rb_root *root,
- const struct rb_augment_callbacks *augment)
-{
- struct rb_node *child = node->rb_right, *tmp = node->rb_left;
- struct rb_node *parent, *rebalance;
- unsigned long pc;
-
- if (!tmp) {
- /*
- * Case 1: node to erase has no more than 1 child (easy!)
- *
- * Note that if there is one child it must be red due to 5)
- * and node must be black due to 4). We adjust colors locally
- * so as to bypass __rb_erase_color() later on.
- */
- pc = node->__rb_parent_color;
- parent = __rb_parent(pc);
- __rb_change_child(node, child, parent, root);
- if (child) {
- child->__rb_parent_color = pc;
- rebalance = NULL;
- } else
- rebalance = __rb_is_black(pc) ? parent : NULL;
- tmp = parent;
- } else if (!child) {
- /* Still case 1, but this time the child is node->rb_left */
- tmp->__rb_parent_color = pc = node->__rb_parent_color;
- parent = __rb_parent(pc);
- __rb_change_child(node, tmp, parent, root);
- rebalance = NULL;
- tmp = parent;
- } else {
- struct rb_node *successor = child, *child2;
- tmp = child->rb_left;
- if (!tmp) {
- /*
- * Case 2: node's successor is its right child
- *
- * (n) (s)
- * / \ / \
- * (x) (s) -> (x) (c)
- * \
- * (c)
- */
- parent = successor;
- child2 = successor->rb_right;
- augment->copy(node, successor);
- } else {
- /*
- * Case 3: node's successor is leftmost under
- * node's right child subtree
- *
- * (n) (s)
- * / \ / \
- * (x) (y) -> (x) (y)
- * / /
- * (p) (p)
- * / /
- * (s) (c)
- * \
- * (c)
- */
- do {
- parent = successor;
- successor = tmp;
- tmp = tmp->rb_left;
- } while (tmp);
- parent->rb_left = child2 = successor->rb_right;
- successor->rb_right = child;
- rb_set_parent(child, successor);
- augment->copy(node, successor);
- augment->propagate(parent, successor);
- }
-
- successor->rb_left = tmp = node->rb_left;
- rb_set_parent(tmp, successor);
-
- pc = node->__rb_parent_color;
- tmp = __rb_parent(pc);
- __rb_change_child(node, successor, tmp, root);
- if (child2) {
- successor->__rb_parent_color = pc;
- rb_set_parent_color(child2, parent, RB_BLACK);
- rebalance = NULL;
- } else {
- unsigned long pc2 = successor->__rb_parent_color;
- successor->__rb_parent_color = pc;
- rebalance = __rb_is_black(pc2) ? parent : NULL;
- }
- tmp = successor;
- }
-
- augment->propagate(tmp, NULL);
- if (rebalance)
- __rb_erase_color(rebalance, root, augment);
-}
+EXPORT_SYMBOL(__rb_erase_color);
/*
* Non-augmented rbtree manipulation functions.
@@ -513,7 +380,7 @@
void rb_erase(struct rb_node *node, struct rb_root *root)
{
- __rb_erase(node, root, &dummy_callbacks);
+ rb_erase_augmented(node, root, &dummy_callbacks);
}
EXPORT_SYMBOL(rb_erase);
@@ -531,13 +398,6 @@
}
EXPORT_SYMBOL(__rb_insert_augmented);
-void rb_erase_augmented(struct rb_node *node, struct rb_root *root,
- const struct rb_augment_callbacks *augment)
-{
- __rb_erase(node, root, augment);
-}
-EXPORT_SYMBOL(rb_erase_augmented);
-
/*
* This function returns the first node (in sort order) of the tree.
*/