8  * RCU-protected list version
10 #include <linux/list.h>
15  * @list: list to be initialized
18  * cleanup tasks, when readers have no access to the list being initialized.
19  * However, if the list being initialized is visible to readers, you
22 static inline void INIT_LIST_HEAD_RCU(struct list_head *list)  in INIT_LIST_HEAD_RCU()  argument
24 	WRITE_ONCE(list->next, list);  in INIT_LIST_HEAD_RCU()
25 	WRITE_ONCE(list->prev, list);  in INIT_LIST_HEAD_RCU()
32 #define list_next_rcu(list)	(*((struct list_head __rcu **)(&(list)->next)))  argument
35  * list_tail_rcu - returns the prev pointer of the head of the list
36  * @head: the head of the list
38  * Note: This should only be used with the list header, and even then
40  * list header.
45  * Check during list traversal that we are within an RCU reader
55 			 "RCU-list traversed in non-reader section!");	\
61 		"RCU-list traversed without holding the required lock!");\
73  * This is only for internal list manipulation where we know
89  * list_add_rcu - add a new entry to rcu-protected list
91  * @head: list head to add it after
98  * with another list-mutation primitive, such as list_add_rcu()
99  * or list_del_rcu(), running on this same list.
101  * the _rcu list-traversal primitives, such as
110  * list_add_tail_rcu - add a new entry to rcu-protected list
112  * @head: list head to add it before
119  * with another list-mutation primitive, such as list_add_tail_rcu()
120  * or list_del_rcu(), running on this same list.
122  * the _rcu list-traversal primitives, such as
132  * list_del_rcu - deletes entry from list without re-initialization
133  * @entry: the element to delete from the list.
140  * pointers that may still be used for walking the list.
144  * with another list-mutation primitive, such as list_del_rcu()
145  * or list_add_rcu(), running on this same list.
147  * the _rcu list-traversal primitives, such as
162  * hlist_del_init_rcu - deletes entry from hash list with re-initialization
163  * @n: the element to delete from the hash list.
167  * must know if the list entry is still hashed or already unhashed.
170  * that may still be used for walking the hash list and we can only
176  * list-mutation primitive, such as hlist_add_head_rcu() or
177  * hlist_del_rcu(), running on this same list.  However, it is
178  * perfectly legal to run concurrently with the _rcu list-traversal
211  * __list_splice_init_rcu - join an RCU-protected list into an existing list.
212  * @list:	the RCU-protected list to splice
213  * @prev:	points to the last element of the existing list
214  * @next:	points to the first element of the existing list
217  * The list pointed to by @prev and @next can be RCU-read traversed
223  * any other updates to the existing list.  In principle, it is possible to
224  * modify the list as soon as sync() begins execution. If this sort of thing
229 static inline void __list_splice_init_rcu(struct list_head *list,  in __list_splice_init_rcu()  argument
234 	struct list_head *first = list->next;  in __list_splice_init_rcu()
235 	struct list_head *last = list->prev;  in __list_splice_init_rcu()
238 	 * "first" and "last" tracking list, so initialize it.  RCU readers  in __list_splice_init_rcu()
239 	 * have access to this list, so we must use INIT_LIST_HEAD_RCU()  in __list_splice_init_rcu()
243 	INIT_LIST_HEAD_RCU(list);  in __list_splice_init_rcu()
246 	 * At this point, the list body still points to the source list.  in __list_splice_init_rcu()
247 	 * Wait for any readers to finish using the list before splicing  in __list_splice_init_rcu()
248 	 * the list body into the new list.  Any new readers will see  in __list_splice_init_rcu()
249 	 * an empty list.  in __list_splice_init_rcu()
257 	 * Readers are finished with the source list, so perform splice.  in __list_splice_init_rcu()
258 	 * The order is important if the new list is global and accessible  in __list_splice_init_rcu()
271  * list_splice_init_rcu - splice an RCU-protected list into an existing list,
273  * @list:	the RCU-protected list to splice
274  * @head:	the place in the existing list to splice the first list into
277 static inline void list_splice_init_rcu(struct list_head *list,  in list_splice_init_rcu()  argument
281 	if (!list_empty(list))  in list_splice_init_rcu()
282 		__list_splice_init_rcu(list, head, head->next, sync);  in list_splice_init_rcu()
286  * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
287  *                             list, designed for queues.
288  * @list:	the RCU-protected list to splice
289  * @head:	the place in the existing list to splice the first list into
292 static inline void list_splice_tail_init_rcu(struct list_head *list,  in list_splice_tail_init_rcu()  argument
296 	if (!list_empty(list))  in list_splice_tail_init_rcu()
297 		__list_splice_init_rcu(list, head->prev, head, sync);  in list_splice_tail_init_rcu()
306  * This primitive may safely run concurrently with the _rcu list-mutation
322  * The list might be non-empty when list_empty_rcu() checks it, but it
332  * list head.  However, it neither dereferences this pointer nor provides
343  * list_first_or_null_rcu - get the first element from a list
344  * @ptr:        the list head to take the element from.
348  * Note that if the list is empty, it returns NULL.
350  * This primitive may safely run concurrently with the _rcu list-mutation
361  * list_next_or_null_rcu - get the next element from a list
362  * @head:	the head for the list.
363  * @ptr:        the list head to take the next element from.
367  * Note that if the ptr is at the end of the list, NULL is returned.
369  * This primitive may safely run concurrently with the _rcu list-mutation
382  * list_for_each_entry_rcu	-	iterate over rcu list of given type
384  * @head:	the head for your list.
388  * This list-traversal primitive may safely run concurrently with
389  * the _rcu list-mutation primitives such as list_add_rcu()
399  * list_for_each_entry_srcu	-	iterate over rcu list of given type
401  * @head:	the head for your list.
403  * @cond:	lockdep expression for the lock required to traverse the list.
405  * This list-traversal primitive may safely run concurrently with
406  * the _rcu list-mutation primitives such as list_add_rcu()
424  * list-mutation primitives such as list_add_rcu(), but requires some
427  * cannot be invoked.  Another example is when items are added to the list,
434  * list_for_each_entry_lockless - iterate over rcu list of given type
436  * @head:	the head for your list.
440  * list-mutation primitives such as list_add_rcu(), but requires some
443  * cannot be invoked.  Another example is when items are added to the list,
452  * list_for_each_entry_continue_rcu - continue iteration over list of given type
454  * @head:	the head for your list.
457  * Continue to iterate over list of given type, continuing after
458  * the current position which must have been in the list when the RCU read
461  * previous walk of the list in the same RCU read-side critical section, or
463  * to keep the node alive *and* in the list.
475  * list_for_each_entry_from_rcu - iterate over a list from current point
477  * @head:	the head for your list.
480  * Iterate over the tail of a list starting from a given position,
481  * which must have been in the list when the RCU read lock was taken.
483  * previous walk of the list in the same RCU read-side critical section, or
485  * to keep the node alive *and* in the list.
496  * hlist_del_rcu - deletes entry from hash list without re-initialization
497  * @n: the element to delete from the hash list.
504  * pointers that may still be used for walking the hash list.
508  * with another list-mutation primitive, such as hlist_add_head_rcu()
509  * or hlist_del_rcu(), running on this same list.
511  * the _rcu list-traversal primitives, such as
572  * @n: the element to add to the hash list.
573  * @h: the list to add to.
581  * with another list-mutation primitive, such as hlist_add_head_rcu()
582  * or hlist_del_rcu(), running on this same list.
584  * the _rcu list-traversal primitives, such as
587  * list-traversal primitive must be guarded by rcu_read_lock().
603  * @n: the element to add to the hash list.
604  * @h: the list to add to.
612  * with another list-mutation primitive, such as hlist_add_head_rcu()
613  * or hlist_del_rcu(), running on this same list.
615  * the _rcu list-traversal primitives, such as
618  * list-traversal primitive must be guarded by rcu_read_lock().
640  * @n: the new element to add to the hash list.
649  * with another list-mutation primitive, such as hlist_add_head_rcu()
650  * or hlist_del_rcu(), running on this same list.
652  * the _rcu list-traversal primitives, such as
667  * @n: the new element to add to the hash list.
676  * with another list-mutation primitive, such as hlist_add_head_rcu()
677  * or hlist_del_rcu(), running on this same list.
679  * the _rcu list-traversal primitives, such as
699  * hlist_for_each_entry_rcu - iterate over rcu list of given type
701  * @head:	the head for your list.
705  * This list-traversal primitive may safely run concurrently with
706  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
718  * hlist_for_each_entry_srcu - iterate over rcu list of given type
720  * @head:	the head for your list.
722  * @cond:	lockdep expression for the lock required to traverse the list.
724  * This list-traversal primitive may safely run concurrently with
725  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
739  * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
741  * @head:	the head for your list.
744  * This list-traversal primitive may safely run concurrently with
745  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
759  * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
761  * @head:	the head for your list.
764  * This list-traversal primitive may safely run concurrently with
765  * the _rcu list-mutation primitives such as hlist_add_head_rcu()