Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
Macros | Typedefs | Functions
rcupdate.h File Reference
#include <linux/types.h>
#include <linux/cache.h>
#include <linux/spinlock.h>
#include <linux/threads.h>
#include <linux/cpumask.h>
#include <linux/seqlock.h>
#include <linux/lockdep.h>
#include <linux/completion.h>
#include <linux/debugobjects.h>
#include <linux/bug.h>
#include <linux/compiler.h>

Go to the source code of this file.

Macros

#define do_trace_rcu_torture_read(rcutorturename, rhp)   do { } while (0)
 
#define UINT_CMP_GE(a, b)   (UINT_MAX / 2 >= (a) - (b))
 
#define UINT_CMP_LT(a, b)   (UINT_MAX / 2 < (a) - (b))
 
#define ULONG_CMP_GE(a, b)   (ULONG_MAX / 2 >= (a) - (b))
 
#define ULONG_CMP_LT(a, b)   (ULONG_MAX / 2 < (a) - (b))
 
#define call_rcu   call_rcu_sched
 
#define RCU_NONIDLE(a)
 
#define rcu_lock_acquire(a)   do { } while (0)
 
#define rcu_lock_release(a)   do { } while (0)
 
#define rcu_lockdep_assert(c, s)   do { } while (0)
 
#define rcu_sleep_check()   do { } while (0)
 
#define rcu_dereference_sparse(p, space)
 
#define __rcu_access_pointer(p, space)
 
#define __rcu_dereference_check(p, c, space)
 
#define __rcu_dereference_protected(p, c, space)
 
#define __rcu_access_index(p, space)
 
#define __rcu_dereference_index_check(p, c)
 
#define __rcu_assign_pointer(p, v, space)
 
#define rcu_access_pointer(p)   __rcu_access_pointer((p), __rcu)
 
#define rcu_dereference_check(p, c)   __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)
 
#define rcu_dereference_bh_check(p, c)   __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)
 
#define rcu_dereference_sched_check(p, c)
 
#define rcu_dereference_raw(p)   rcu_dereference_check(p, 1) /*@@@ needed? @@@*/
 
#define rcu_access_index(p)   __rcu_access_index((p), __rcu)
 
#define rcu_dereference_index_check(p, c)   __rcu_dereference_index_check((p), (c))
 
#define rcu_dereference_protected(p, c)   __rcu_dereference_protected((p), (c), __rcu)
 
#define rcu_dereference(p)   rcu_dereference_check(p, 0)
 
#define rcu_dereference_bh(p)   rcu_dereference_bh_check(p, 0)
 
#define rcu_dereference_sched(p)   rcu_dereference_sched_check(p, 0)
 
#define rcu_assign_pointer(p, v)   __rcu_assign_pointer((p), (v), __rcu)
 
#define RCU_INIT_POINTER(p, v)
 
#define RCU_POINTER_INITIALIZER(p, v)   .p = (typeof(*v) __force __rcu *)(v)
 
#define __is_kfree_rcu_offset(offset)   ((offset) < 4096)
 
#define __kfree_rcu(head, offset)
 
#define kfree_rcu(ptr, rcu_head)   __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
 

Typedefs

typedef void call_rcu_func_t (struct rcu_head *head, void(*func)(struct rcu_head *head))
 

Functions

void call_rcu_bh (struct rcu_head *head, void(*func)(struct rcu_head *head))
 
void call_rcu_sched (struct rcu_head *head, void(*func)(struct rcu_head *rcu))
 
void synchronize_sched (void)
 
void rcu_sched_qs (int cpu)
 
void rcu_bh_qs (int cpu)
 
void rcu_check_callbacks (int cpu, int user)
 
void rcu_idle_enter (void)
 
void rcu_idle_exit (void)
 
void rcu_irq_enter (void)
 
void rcu_irq_exit (void)
 
void exit_rcu (void)
 
void wait_rcu_gp (call_rcu_func_t crf)
 

Macro Definition Documentation

#define __is_kfree_rcu_offset (   offset)    ((offset) < 4096)

Definition at line 930 of file rcupdate.h.

#define __kfree_rcu (   head,
  offset 
)
Value:
do { \
BUILD_BUG_ON(!__is_kfree_rcu_offset(offset)); \
kfree_call_rcu(head, (void (*)(struct rcu_head *))(unsigned long)(offset)); \
} while (0)

Definition at line 935 of file rcupdate.h.

#define __rcu_access_index (   p,
  space 
)
Value:
({ \
typeof(p) _________p1 = ACCESS_ONCE(p); \
(_________p1); \
})

Definition at line 508 of file rcupdate.h.

#define __rcu_access_pointer (   p,
  space 
)
Value:
({ \
typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
rcu_dereference_sparse(p, space); \
((typeof(*p) __force __kernel *)(_________p1)); \
})

Definition at line 485 of file rcupdate.h.

#define __rcu_assign_pointer (   p,
  v,
  space 
)
Value:
do { \
smp_wmb(); \
(p) = (typeof(*v) __force space *)(v); \
} while (0)

Definition at line 523 of file rcupdate.h.

#define __rcu_dereference_check (   p,
  c,
  space 
)
Value:
({ \
typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
rcu_lockdep_assert(c, "suspicious rcu_dereference_check()" \
" usage"); \
rcu_dereference_sparse(p, space); \
smp_read_barrier_depends(); \
((typeof(*p) __force __kernel *)(_________p1)); \
})

Definition at line 491 of file rcupdate.h.

#define __rcu_dereference_index_check (   p,
  c 
)
Value:
({ \
typeof(p) _________p1 = ACCESS_ONCE(p); \
"suspicious rcu_dereference_index_check()" \
" usage"); \
(_________p1); \
})

Definition at line 514 of file rcupdate.h.

#define __rcu_dereference_protected (   p,
  c,
  space 
)
Value:
({ \
rcu_lockdep_assert(c, "suspicious rcu_dereference_protected()" \
" usage"); \
rcu_dereference_sparse(p, space); \
((typeof(*p) __force __kernel *)(p)); \
})

Definition at line 500 of file rcupdate.h.

#define call_rcu   call_rcu_sched

Definition at line 100 of file rcupdate.h.

#define do_trace_rcu_torture_read (   rcutorturename,
  rhp 
)    do { } while (0)

Definition at line 68 of file rcupdate.h.

#define kfree_rcu (   ptr,
  rcu_head 
)    __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))

kfree_rcu() - kfree an object after a grace period. : pointer to kfree : the name of the struct rcu_head within the type of .

Many rcu callbacks functions just call kfree() on the base structure. These functions are trivial, but their size adds up, and furthermore when they are used in a kernel module, that module must invoke the high-latency rcu_barrier() function at module-unload time.

The kfree_rcu() function handles this issue. Rather than encoding a function address in the embedded rcu_head structure, kfree_rcu() instead encodes the offset of the rcu_head structure within the base structure. Because the functions are not allowed in the low-order 4096 bytes of kernel virtual memory, offsets up to 4095 bytes can be accommodated. If the offset is larger than 4095 bytes, a compile-time error will be generated in __kfree_rcu(). If this error is triggered, you can either fall back to use of call_rcu() or rearrange the structure to position the rcu_head structure into the first 4096 bytes.

Note that the allowable offset might decrease in the future, for example, to allow something like kmem_cache_free_rcu().

The BUILD_BUG_ON check must not involve any function calls, hence the checks are done in macros here.

Definition at line 967 of file rcupdate.h.

#define rcu_access_index (   p)    __rcu_access_index((p), __rcu)

rcu_access_index() - fetch RCU index with no dereferencing : The index to read

Return the value of the specified RCU-protected index, but omit the smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful when the value of this index is accessed, but the index is not dereferenced, for example, when testing an RCU-protected index against -1. Although rcu_access_index() may also be used in cases where update-side locks prevent the value of the index from changing, you should instead use rcu_dereference_index_protected() for this use case.

Definition at line 622 of file rcupdate.h.

#define rcu_access_pointer (   p)    __rcu_access_pointer((p), __rcu)

rcu_access_pointer() - fetch RCU pointer with no dereferencing : The pointer to read

Return the value of the specified RCU-protected pointer, but omit the smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful when the value of this pointer is accessed, but the pointer is not dereferenced, for example, when testing an RCU-protected pointer against NULL. Although rcu_access_pointer() may also be used in cases where update-side locks prevent the value of the pointer from changing, you should instead use rcu_dereference_protected() for this use case.

It is also permissible to use rcu_access_pointer() when read-side access to the pointer was removed at least one grace period ago, as is the case in the context of the RCU callback that is freeing up the data, or after a synchronize_rcu() returns. This can be useful when tearing down multi-linked structures after a grace period has elapsed.

Definition at line 549 of file rcupdate.h.

#define rcu_assign_pointer (   p,
  v 
)    __rcu_assign_pointer((p), (v), __rcu)

rcu_assign_pointer() - assign to RCU-protected pointer : pointer to assign to : value to assign (publish)

Assigns the specified value to the specified RCU-protected pointer, ensuring that any concurrent RCU readers will see any prior initialization.

Inserts memory barriers on architectures that require them (which is most of them), and also prevents the compiler from reordering the code that initializes the structure after the pointer assignment. More importantly, this call documents which pointers will be dereferenced by RCU read-side code.

In some special cases, you may use RCU_INIT_POINTER() instead of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due to the fact that it does not constrain either the CPU or the compiler. That said, using RCU_INIT_POINTER() when you should have used rcu_assign_pointer() is a very bad thing that results in impossible-to-diagnose memory corruption. So please be careful. See the RCU_INIT_POINTER() comment header for details.

Definition at line 878 of file rcupdate.h.

#define rcu_dereference (   p)    rcu_dereference_check(p, 0)

rcu_dereference() - fetch RCU-protected pointer for dereferencing : The pointer to read, prior to dereferencing

This is a simple wrapper around rcu_dereference_check().

Definition at line 672 of file rcupdate.h.

#define rcu_dereference_bh (   p)    rcu_dereference_bh_check(p, 0)

rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing : The pointer to read, prior to dereferencing

Makes rcu_dereference_check() do the dirty work.

Definition at line 680 of file rcupdate.h.

#define rcu_dereference_bh_check (   p,
  c 
)    __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)

rcu_dereference_bh_check() - rcu_dereference_bh with debug checking : The pointer to read, prior to dereferencing : The conditions under which the dereference will take place

This is the RCU-bh counterpart to rcu_dereference_check().

Definition at line 594 of file rcupdate.h.

#define rcu_dereference_check (   p,
  c 
)    __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)

rcu_dereference_check() - rcu_dereference with debug checking : The pointer to read, prior to dereferencing : The conditions under which the dereference will take place

Do an rcu_dereference(), but check that the conditions under which the dereference will take place are correct. Typically the conditions indicate the various locking conditions that should be held at that point. The check should return true if the conditions are satisfied. An implicit check for being in an RCU read-side critical section (rcu_read_lock()) is included.

For example:

bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));

could be used to indicate to lockdep that foo->bar may only be dereferenced if either rcu_read_lock() is held, or that the lock required to replace the bar struct at foo->bar is held.

Note that the list of conditions may also include indications of when a lock need not be held, for example during initialisation or destruction of the target struct:

bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) || atomic_read(&foo->usage) == 0);

Inserts memory barriers on architectures that require them (currently only the Alpha), prevents the compiler from refetching (and from merging fetches), and, more importantly, documents exactly which pointers are protected by RCU and checks that the pointer is annotated as __rcu.

Definition at line 584 of file rcupdate.h.

#define rcu_dereference_index_check (   p,
  c 
)    __rcu_dereference_index_check((p), (c))

rcu_dereference_index_check() - rcu_dereference for indices with debug checking : The pointer to read, prior to dereferencing : The conditions under which the dereference will take place

Similar to rcu_dereference_check(), but omits the sparse checking. This allows rcu_dereference_index_check() to be used on integers, which can then be used as array indices. Attempting to use rcu_dereference_check() on an integer will give compiler warnings because the sparse address-space mechanism relies on dereferencing the RCU-protected pointer. Dereferencing integers is not something that even gcc will put up with.

Note that this function does not implicitly check for RCU read-side critical sections. If this function gains lots of uses, it might make sense to provide versions for each flavor of RCU, but it does not make sense as of early 2010.

Definition at line 642 of file rcupdate.h.

#define rcu_dereference_protected (   p,
  c 
)    __rcu_dereference_protected((p), (c), __rcu)

rcu_dereference_protected() - fetch RCU pointer when updates prevented : The pointer to read, prior to dereferencing : The conditions under which the dereference will take place

Return the value of the specified RCU-protected pointer, but omit both the smp_read_barrier_depends() and the ACCESS_ONCE(). This is useful in cases where update-side locks prevent the value of the pointer from changing. Please note that this primitive does -not- prevent the compiler from repeating this reference or combining it with other references, so it should not be used without protection of appropriate locks.

This function is only for update-side use. Using this function when protected only by rcu_read_lock() will result in infrequent but very ugly failures.

Definition at line 662 of file rcupdate.h.

#define rcu_dereference_raw (   p)    rcu_dereference_check(p, 1) /*@@@ needed? @@@*/

Definition at line 608 of file rcupdate.h.

#define rcu_dereference_sched (   p)    rcu_dereference_sched_check(p, 0)

rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing : The pointer to read, prior to dereferencing

Makes rcu_dereference_check() do the dirty work.

Definition at line 688 of file rcupdate.h.

#define rcu_dereference_sched_check (   p,
  c 
)
Value:
__rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \

rcu_dereference_sched_check() - rcu_dereference_sched with debug checking : The pointer to read, prior to dereferencing : The conditions under which the dereference will take place

This is the RCU-sched counterpart to rcu_dereference_check().

Definition at line 604 of file rcupdate.h.

#define rcu_dereference_sparse (   p,
  space 
)

Definition at line 482 of file rcupdate.h.

#define RCU_INIT_POINTER (   p,
  v 
)
Value:
do { \
p = (typeof(*v) __force __rcu *)(v); \
} while (0)

RCU_INIT_POINTER() - initialize an RCU protected pointer

Initialize an RCU-protected pointer in special cases where readers do not need ordering constraints on the CPU or the compiler. These special cases are:

  1. This use of RCU_INIT_POINTER() is NULLing out the pointer -or-
  2. The caller has taken whatever steps are required to prevent RCU readers from concurrently accessing this pointer -or-
  3. The referenced data structure has already been exposed to readers either at compile time or via rcu_assign_pointer() -and- a. You have not made -any- reader-visible changes to this structure since then -or- b. It is OK for readers accessing this structure from its new location to see the old state of the structure. (For example, the changes were to statistical counters or to other state where exact synchronization is not required.)

Failure to follow these rules governing use of RCU_INIT_POINTER() will result in impossible-to-diagnose memory corruption. As in the structures will look OK in crash dumps, but any concurrent RCU readers might see pre-initialized values of the referenced data structure. So please be very careful how you use RCU_INIT_POINTER()!!!

If you are creating an RCU-protected linked structure that is accessed by a single external-to-structure RCU-protected pointer, then you may use RCU_INIT_POINTER() to initialize the internal RCU-protected pointers, but you must use rcu_assign_pointer() to initialize the external-to-structure pointer -after- you have completely initialized the reader-accessible portions of the linked structure.

Definition at line 913 of file rcupdate.h.

#define rcu_lock_acquire (   a)    do { } while (0)

Definition at line 394 of file rcupdate.h.

#define rcu_lock_release (   a)    do { } while (0)

Definition at line 395 of file rcupdate.h.

#define rcu_lockdep_assert (   c,
  s 
)    do { } while (0)

Definition at line 464 of file rcupdate.h.

#define RCU_NONIDLE (   a)
Value:
do { \
rcu_irq_enter(); \
do { a; } while (0); \
rcu_irq_exit(); \
} while (0)

RCU_NONIDLE - Indicate idle-loop code that needs RCU readers : Code that RCU needs to pay attention to.

RCU, RCU-bh, and RCU-sched read-side critical sections are forbidden in the inner idle loop, that is, between the rcu_idle_enter() and the rcu_idle_exit() – RCU will happily ignore any such read-side critical sections. However, things like powertop need tracepoints in the inner idle loop.

This macro provides the way out: RCU_NONIDLE(do_something_with_RCU()) will tell RCU that it needs to pay attending, invoke its argument (in this example, a call to the do_something_with_RCU() function), and then tell RCU to go back to ignoring this CPU. It is permissible to nest RCU_NONIDLE() wrappers, but the nesting level is currently quite limited. If deeper nesting is required, it will be necessary to adjust DYNTICK_TASK_NESTING_VALUE accordingly.

Definition at line 229 of file rcupdate.h.

#define RCU_POINTER_INITIALIZER (   p,
  v 
)    .p = (typeof(*v) __force __rcu *)(v)

RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer

GCC-style initialization for an RCU-protected pointer in a structure field.

Definition at line 923 of file rcupdate.h.

#define rcu_sleep_check ( )    do { } while (0)

Definition at line 465 of file rcupdate.h.

#define UINT_CMP_GE (   a,
  b 
)    (UINT_MAX / 2 >= (a) - (b))

Definition at line 72 of file rcupdate.h.

#define UINT_CMP_LT (   a,
  b 
)    (UINT_MAX / 2 < (a) - (b))

Definition at line 73 of file rcupdate.h.

#define ULONG_CMP_GE (   a,
  b 
)    (ULONG_MAX / 2 >= (a) - (b))

Definition at line 74 of file rcupdate.h.

#define ULONG_CMP_LT (   a,
  b 
)    (ULONG_MAX / 2 < (a) - (b))

Definition at line 75 of file rcupdate.h.

Typedef Documentation

typedef void call_rcu_func_t(struct rcu_head *head, void(*func)(struct rcu_head *head))

Definition at line 241 of file rcupdate.h.

Function Documentation

void call_rcu_bh ( struct rcu_head head,
void(*)(struct rcu_head *head func 
)

call_rcu_bh() - Queue an RCU for invocation after a quicker grace period. : structure to be used for queueing the RCU updates. : actual callback function to be invoked after the grace period

The callback function will be invoked some time after a full grace period elapses, in other words after all currently executing RCU read-side critical sections have completed. call_rcu_bh() assumes that the read-side critical sections end on completion of a softirq handler. This means that read-side critical sections in process context must not be interrupted by softirqs. This interface is to be used when most of the read-side critical sections are in softirq context. RCU read-side critical sections are delimited by :

  • rcu_read_lock() and rcu_read_unlock(), if in interrupt context. OR
  • rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context. These may be nested.
void call_rcu_sched ( struct rcu_head head,
void(*)(struct rcu_head *rcu)  func 
)

call_rcu_sched() - Queue an RCU for invocation after sched grace period. : structure to be used for queueing the RCU updates. : actual callback function to be invoked after the grace period

The callback function will be invoked some time after a full grace period elapses, in other words after all currently executing RCU read-side critical sections have completed. call_rcu_sched() assumes that the read-side critical sections end on enabling of preemption or on voluntary preemption. RCU read-side critical sections are delimited by :

  • rcu_read_lock_sched() and rcu_read_unlock_sched(), OR anything that disables preemption. These may be nested.

Definition at line 368 of file rcutiny.c.

void exit_rcu ( void  )

Definition at line 124 of file rcupdate.c.

void rcu_bh_qs ( int  cpu)

Definition at line 236 of file rcutiny.c.

void rcu_check_callbacks ( int  cpu,
int  user 
)

Definition at line 252 of file rcutiny.c.

void rcu_idle_enter ( void  )

rcu_idle_enter - inform RCU that current CPU is entering idle

Enter idle mode, in other words, -leave- the mode in which RCU read-side critical sections can occur. (Though RCU read-side critical sections can occur in irq handlers in idle, a possibility handled by irq_enter() and irq_exit().)

We crowbar the ->dynticks_nesting field to zero to allow for the possibility of usermode upcalls having messed up our count of interrupt nesting level during the prior busy period.

Definition at line 87 of file rcutiny.c.

void rcu_idle_exit ( void  )

rcu_idle_exit - inform RCU that current CPU is leaving idle

Exit idle mode, in other words, -enter- the mode in which RCU read-side critical sections can occur.

We crowbar the ->dynticks_nesting field to DYNTICK_TASK_NEST to allow for the possibility of usermode upcalls messing up our count of interrupt nesting level during the busy period that is just now starting.

Definition at line 144 of file rcutiny.c.

void rcu_irq_enter ( void  )

rcu_irq_enter - inform RCU that current CPU is entering irq away from idle

Enter an interrupt handler, which might possibly result in exiting idle mode, in other words, entering the mode in which read-side critical sections can occur.

Note that the Linux kernel is fully capable of entering an interrupt handler that it never exits, for example when doing upcalls to user mode! This code assumes that the idle loop never does upcalls to user mode. If your architecture does do upcalls from the idle loop (or does anything else that results in unbalanced calls to the irq_enter() and irq_exit() functions), RCU will give you what you deserve, good and hard. But very infrequently and irreproducibly.

Use things like work queues to work around this limitation.

You have been warned.

Definition at line 164 of file rcutiny.c.

void rcu_irq_exit ( void  )

rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle

Exit from an interrupt handler, which might possibly result in entering idle mode, in other words, leaving the mode in which read-side critical sections can occur.

This code assumes that the idle loop never does anything that might result in unbalanced calls to irq_enter() and irq_exit(). If your architecture violates this assumption, RCU will give you what you deserve, good and hard. But very infrequently and irreproducibly.

Use things like work queues to work around this limitation.

You have been warned.

Definition at line 107 of file rcutiny.c.

void rcu_sched_qs ( int  cpu)

Definition at line 222 of file rcutiny.c.

void synchronize_sched ( void  )

synchronize_sched - wait until an rcu-sched grace period has elapsed.

Control will return to the caller some time after a full rcu-sched grace period has elapsed, in other words after all currently executing rcu-sched read-side critical sections have completed. These read-side critical sections are delimited by rcu_read_lock_sched() and rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(), local_irq_disable(), and so on may be used in place of rcu_read_lock_sched().

This means that all preempt_disable code sequences, including NMI and hardware-interrupt handlers, in progress on entry will have completed before this primitive returns. However, this does not guarantee that softirq handlers will have completed, since in some kernels, these handlers can run in process context, and can block.

This primitive provides the guarantees made by the (now removed) synchronize_kernel() API. In contrast, synchronize_rcu() only guarantees that rcu_read_lock() sections will have completed. In "classic RCU", these two guarantees happen to be one and the same, but can differ in realtime RCU implementations.

Definition at line 333 of file rcutiny.c.

void wait_rcu_gp ( call_rcu_func_t  crf)

Definition at line 202 of file rcupdate.c.