Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
rculist.h
Go to the documentation of this file.
1 #ifndef _LINUX_RCULIST_H
2 #define _LINUX_RCULIST_H
3 
4 #ifdef __KERNEL__
5 
6 /*
7  * RCU-protected list version
8  */
9 #include <linux/list.h>
10 #include <linux/rcupdate.h>
11 
12 /*
13  * Why is there no list_empty_rcu()? Because list_empty() serves this
14  * purpose. The list_empty() function fetches the RCU-protected pointer
15  * and compares it to the address of the list head, but neither dereferences
16  * this pointer itself nor provides this pointer to the caller. Therefore,
17  * it is not necessary to use rcu_dereference(), so that list_empty() can
18  * be used anywhere you would want to use a list_empty_rcu().
19  */
20 
21 /*
22  * return the ->next pointer of a list_head in an rcu safe
23  * way, we must not access it directly
24  */
25 #define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next)))
26 
27 /*
28  * Insert a new entry between two known consecutive entries.
29  *
30  * This is only for internal list manipulation where we know
31  * the prev/next entries already!
32  */
33 #ifndef CONFIG_DEBUG_LIST
34 static inline void __list_add_rcu(struct list_head *new,
35  struct list_head *prev, struct list_head *next)
36 {
37  new->next = next;
38  new->prev = prev;
39  rcu_assign_pointer(list_next_rcu(prev), new);
40  next->prev = new;
41 }
42 #else
43 extern void __list_add_rcu(struct list_head *new,
44  struct list_head *prev, struct list_head *next);
45 #endif
46 
63 static inline void list_add_rcu(struct list_head *new, struct list_head *head)
64 {
65  __list_add_rcu(new, head, head->next);
66 }
67 
84 static inline void list_add_tail_rcu(struct list_head *new,
85  struct list_head *head)
86 {
87  __list_add_rcu(new, head->prev, head);
88 }
89 
114 static inline void list_del_rcu(struct list_head *entry)
115 {
116  __list_del_entry(entry);
117  entry->prev = LIST_POISON2;
118 }
119 
140 static inline void hlist_del_init_rcu(struct hlist_node *n)
141 {
142  if (!hlist_unhashed(n)) {
143  __hlist_del(n);
144  n->pprev = NULL;
145  }
146 }
147 
156 static inline void list_replace_rcu(struct list_head *old,
157  struct list_head *new)
158 {
159  new->next = old->next;
160  new->prev = old->prev;
161  rcu_assign_pointer(list_next_rcu(new->prev), new);
162  new->next->prev = new;
163  old->prev = LIST_POISON2;
164 }
165 
183 static inline void list_splice_init_rcu(struct list_head *list,
184  struct list_head *head,
185  void (*sync)(void))
186 {
187  struct list_head *first = list->next;
188  struct list_head *last = list->prev;
189  struct list_head *at = head->next;
190 
191  if (list_empty(list))
192  return;
193 
194  /* "first" and "last" tracking list, so initialize it. */
195 
196  INIT_LIST_HEAD(list);
197 
198  /*
199  * At this point, the list body still points to the source list.
200  * Wait for any readers to finish using the list before splicing
201  * the list body into the new list. Any new readers will see
202  * an empty list.
203  */
204 
205  sync();
206 
207  /*
208  * Readers are finished with the source list, so perform splice.
209  * The order is important if the new list is global and accessible
210  * to concurrent RCU readers. Note that RCU readers are not
211  * permitted to traverse the prev pointers without excluding
212  * this function.
213  */
214 
215  last->next = at;
216  rcu_assign_pointer(list_next_rcu(head), first);
217  first->prev = head;
218  at->prev = last;
219 }
220 
230 #define list_entry_rcu(ptr, type, member) \
231  ({typeof (*ptr) __rcu *__ptr = (typeof (*ptr) __rcu __force *)ptr; \
232  container_of((typeof(ptr))rcu_dereference_raw(__ptr), type, member); \
233  })
234 
268 #define list_first_or_null_rcu(ptr, type, member) \
269  ({struct list_head *__ptr = (ptr); \
270  struct list_head __rcu *__next = list_next_rcu(__ptr); \
271  likely(__ptr != __next) ? container_of(__next, type, member) : NULL; \
272  })
273 
284 #define list_for_each_entry_rcu(pos, head, member) \
285  for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \
286  &pos->member != (head); \
287  pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
288 
289 
301 #define list_for_each_continue_rcu(pos, head) \
302  for ((pos) = rcu_dereference_raw(list_next_rcu(pos)); \
303  (pos) != (head); \
304  (pos) = rcu_dereference_raw(list_next_rcu(pos)))
305 
315 #define list_for_each_entry_continue_rcu(pos, head, member) \
316  for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
317  &pos->member != (head); \
318  pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
319 
339 static inline void hlist_del_rcu(struct hlist_node *n)
340 {
341  __hlist_del(n);
342  n->pprev = LIST_POISON2;
343 }
344 
352 static inline void hlist_replace_rcu(struct hlist_node *old,
353  struct hlist_node *new)
354 {
355  struct hlist_node *next = old->next;
356 
357  new->next = next;
358  new->pprev = old->pprev;
359  rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
360  if (next)
361  new->next->pprev = &new->next;
362  old->pprev = LIST_POISON2;
363 }
364 
365 /*
366  * return the first or the next element in an RCU protected hlist
367  */
368 #define hlist_first_rcu(head) (*((struct hlist_node __rcu **)(&(head)->first)))
369 #define hlist_next_rcu(node) (*((struct hlist_node __rcu **)(&(node)->next)))
370 #define hlist_pprev_rcu(node) (*((struct hlist_node __rcu **)((node)->pprev)))
371 
391 static inline void hlist_add_head_rcu(struct hlist_node *n,
392  struct hlist_head *h)
393 {
394  struct hlist_node *first = h->first;
395 
396  n->next = first;
397  n->pprev = &h->first;
398  rcu_assign_pointer(hlist_first_rcu(h), n);
399  if (first)
400  first->pprev = &n->next;
401 }
402 
421 static inline void hlist_add_before_rcu(struct hlist_node *n,
422  struct hlist_node *next)
423 {
424  n->pprev = next->pprev;
425  n->next = next;
426  rcu_assign_pointer(hlist_pprev_rcu(n), n);
427  next->pprev = &n->next;
428 }
429 
448 static inline void hlist_add_after_rcu(struct hlist_node *prev,
449  struct hlist_node *n)
450 {
451  n->next = prev->next;
452  n->pprev = &prev->next;
453  rcu_assign_pointer(hlist_next_rcu(prev), n);
454  if (n->next)
455  n->next->pprev = &n->next;
456 }
457 
458 #define __hlist_for_each_rcu(pos, head) \
459  for (pos = rcu_dereference(hlist_first_rcu(head)); \
460  pos; \
461  pos = rcu_dereference(hlist_next_rcu(pos)))
462 
474 #define hlist_for_each_entry_rcu(tpos, pos, head, member) \
475  for (pos = rcu_dereference_raw(hlist_first_rcu(head)); \
476  pos && \
477  ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
478  pos = rcu_dereference_raw(hlist_next_rcu(pos)))
479 
491 #define hlist_for_each_entry_rcu_bh(tpos, pos, head, member) \
492  for (pos = rcu_dereference_bh((head)->first); \
493  pos && \
494  ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
495  pos = rcu_dereference_bh(pos->next))
496 
503 #define hlist_for_each_entry_continue_rcu(tpos, pos, member) \
504  for (pos = rcu_dereference((pos)->next); \
505  pos && \
506  ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
507  pos = rcu_dereference(pos->next))
508 
515 #define hlist_for_each_entry_continue_rcu_bh(tpos, pos, member) \
516  for (pos = rcu_dereference_bh((pos)->next); \
517  pos && \
518  ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
519  pos = rcu_dereference_bh(pos->next))
520 
521 
522 #endif /* __KERNEL__ */
523 #endif