33 #define RB_EMPTY_NODE(node)  \  argument
34 	((node)->__rb_parent_color == (unsigned long)(node))
35 #define RB_CLEAR_NODE(node)  \  argument
36 	((node)->__rb_parent_color = (unsigned long)(node))
53 /* Fast replacement of a single node without remove/rebalance/add/rebalance */
59 static inline void rb_link_node(struct rb_node *node, struct rb_node *parent,  in rb_link_node()  argument
62 	node->__rb_parent_color = (unsigned long)parent;  in rb_link_node()
63 	node->rb_left = node->rb_right = NULL;  in rb_link_node()
65 	*rb_link = node;  in rb_link_node()
68 static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent,  in rb_link_node_rcu()  argument
71 	node->__rb_parent_color = (unsigned long)parent;  in rb_link_node_rcu()
72 	node->rb_left = node->rb_right = NULL;  in rb_link_node_rcu()
74 	rcu_assign_pointer(*rb_link, node);  in rb_link_node_rcu()
108 static inline void rb_insert_color_cached(struct rb_node *node,  in rb_insert_color_cached()  argument
113 		root->rb_leftmost = node;  in rb_insert_color_cached()
114 	rb_insert_color(node, &root->rb_root);  in rb_insert_color_cached()
119 rb_erase_cached(struct rb_node *node, struct rb_root_cached *root)  in rb_erase_cached()  argument
123 	if (root->rb_leftmost == node)  in rb_erase_cached()
124 		leftmost = root->rb_leftmost = rb_next(node);  in rb_erase_cached()
126 	rb_erase(node, &root->rb_root);  in rb_erase_cached()
157  * rb_add_cached() - insert @node into the leftmost cached tree @tree
158  * @node: node to insert
159  * @tree: leftmost cached tree to insert @node into
160  * @less: operator defining the (partial) node order
162  * Returns @node when it is the new leftmost, or NULL.
165 rb_add_cached(struct rb_node *node, struct rb_root_cached *tree,  in rb_add_cached()  argument
174 		if (less(node, parent)) {  in rb_add_cached()
182 	rb_link_node(node, parent, link);  in rb_add_cached()
183 	rb_insert_color_cached(node, tree, leftmost);  in rb_add_cached()
185 	return leftmost ? node : NULL;  in rb_add_cached()
189  * rb_add() - insert @node into @tree
190  * @node: node to insert
191  * @tree: tree to insert @node into
192  * @less: operator defining the (partial) node order
195 rb_add(struct rb_node *node, struct rb_root *tree,  in rb_add()  argument
203 		if (less(node, parent))  in rb_add()
209 	rb_link_node(node, parent, link);  in rb_add()
210 	rb_insert_color(node, tree);  in rb_add()
214  * rb_find_add() - find equivalent @node in @tree, or add @node
215  * @node: node to look-for / insert
217  * @cmp: operator defining the node order
219  * Returns the rb_node matching @node, or NULL when no match is found and @node
223 rb_find_add(struct rb_node *node, struct rb_root *tree,  in rb_find_add()  argument
232 		c = cmp(node, parent);  in rb_find_add()
242 	rb_link_node(node, parent, link);  in rb_find_add()
243 	rb_insert_color(node, tree);  in rb_find_add()
248  * rb_find_add_rcu() - find equivalent @node in @tree, or add @node
249  * @node: node to look-for / insert
251  * @cmp: operator defining the node order
255  * Returns the rb_node matching @node, or NULL when no match is found and @node
259 rb_find_add_rcu(struct rb_node *node, struct rb_root *tree,  in rb_find_add_rcu()  argument
268 		c = cmp(node, parent);  in rb_find_add_rcu()
278 	rb_link_node_rcu(node, parent, link);  in rb_find_add_rcu()
279 	rb_insert_color(node, tree);  in rb_find_add_rcu()
287  * @cmp: operator defining the node order
295 	struct rb_node *node = tree->rb_node;  in rb_find()  local
297 	while (node) {  in rb_find()
298 		int c = cmp(key, node);  in rb_find()
301 			node = node->rb_left;  in rb_find()
303 			node = node->rb_right;  in rb_find()
305 			return node;  in rb_find()
315  * @cmp: operator defining the node order
326 	struct rb_node *node = tree->rb_node;  in rb_find_rcu()  local
328 	while (node) {  in rb_find_rcu()
329 		int c = cmp(key, node);  in rb_find_rcu()
332 			node = rcu_dereference_raw(node->rb_left);  in rb_find_rcu()
334 			node = rcu_dereference_raw(node->rb_right);  in rb_find_rcu()
336 			return node;  in rb_find_rcu()
346  * @cmp: operator defining node order
348  * Returns the leftmost node matching @key, or NULL.
354 	struct rb_node *node = tree->rb_node;  in rb_find_first()  local
357 	while (node) {  in rb_find_first()
358 		int c = cmp(key, node);  in rb_find_first()
362 				match = node;  in rb_find_first()
363 			node = node->rb_left;  in rb_find_first()
365 			node = node->rb_right;  in rb_find_first()
376  * @cmp: operator defining node order
378  * Returns the next node matching @key, or NULL.
381 rb_next_match(const void *key, struct rb_node *node,  in rb_next_match()  argument
384 	node = rb_next(node);  in rb_next_match()
385 	if (node && cmp(key, node))  in rb_next_match()
386 		node = NULL;  in rb_next_match()
387 	return node;  in rb_next_match()
392  * @node: iterator
395  * @cmp: operator defining node order
397 #define rb_for_each(node, key, tree, cmp) \  argument
398 	for ((node) = rb_find_first((key), (tree), (cmp)); \
399 	     (node); (node) = rb_next_match((key), (node), (cmp)))