|
Linux Kernel
3.7.1
|
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
|
Linux Kernel
3.7.1
|
#include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/nmi.h>#include <linux/atomic.h>#include <linux/bitops.h>#include <linux/export.h>#include <linux/completion.h>#include <linux/moduleparam.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/cpu.h>#include <linux/mutex.h>#include <linux/time.h>#include <linux/kernel_stat.h>#include <linux/wait.h>#include <linux/kthread.h>#include <linux/prefetch.h>#include <linux/delay.h>#include <linux/stop_machine.h>#include <linux/random.h>#include "rcutree.h"#include <trace/events/rcu.h>#include "rcu.h"#include "rcutree_plugin.h"Go to the source code of this file.
Macros | |
| #define | RCU_STATE_INITIALIZER(sname, cr) |
Variables | |
| struct rcu_state | rcu_sched_state |
| struct rcu_state | rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh, call_rcu_bh) |
| int rcu_num_lvls | __read_mostly = RCU_NUM_LVLS |
| unsigned long | rcutorture_testseq |
| unsigned long | rcutorture_vernum |
| #define RCU_STATE_INITIALIZER | ( | sname, | |
| cr | |||
| ) |
call_rcu_sched() - Queue an RCU for invocation after sched grace period. : structure to be used for queueing the RCU updates. : actual callback function to be invoked after the grace period
The callback function will be invoked some time after a full grace period elapses, in other words after all currently executing RCU read-side critical sections have completed. call_rcu_sched() assumes that the read-side critical sections end on enabling of preemption or on voluntary preemption. RCU read-side critical sections are delimited by :
| DEFINE_PER_CPU | ( | struct rcu_dynticks | , |
| rcu_dynticks | |||
| ) |
| early_initcall | ( | rcu_spawn_gp_kthread | ) |
| EXPORT_SYMBOL | ( | rcu_is_cpu_idle | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_scheduler_active | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_note_context_switch | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_batches_completed_sched | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_batches_completed_bh | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_bh_force_quiescent_state | ) |
| EXPORT_SYMBOL_GPL | ( | rcutorture_record_test_transition | ) |
| EXPORT_SYMBOL_GPL | ( | rcutorture_record_progress | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_sched_force_quiescent_state | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_idle_enter | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_idle_exit | ) |
| EXPORT_SYMBOL_GPL | ( | call_rcu_sched | ) |
| EXPORT_SYMBOL_GPL | ( | call_rcu_bh | ) |
| EXPORT_SYMBOL_GPL | ( | synchronize_sched | ) |
| EXPORT_SYMBOL_GPL | ( | synchronize_rcu_bh | ) |
| EXPORT_SYMBOL_GPL | ( | synchronize_sched_expedited | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_barrier_bh | ) |
| EXPORT_SYMBOL_GPL | ( | rcu_barrier_sched | ) |
| LIST_HEAD | ( | rcu_struct_flavors | ) |
| module_param | ( | rcu_fanout_leaf | , |
| int | , | ||
| 0444 | |||
| ) |
| module_param | ( | blimit | , |
| long | , | ||
| 0444 | |||
| ) |
| module_param | ( | qhimark | , |
| long | , | ||
| 0444 | |||
| ) |
| module_param | ( | qlowmark | , |
| long | , | ||
| 0444 | |||
| ) |
| module_param | ( | rcu_cpu_stall_suppress | , |
| int | , | ||
| 0644 | |||
| ) |
| module_param | ( | rcu_cpu_stall_timeout | , |
| int | , | ||
| 0644 | |||
| ) |
| module_param | ( | jiffies_till_first_fqs | , |
| ulong | , | ||
| 0644 | |||
| ) |
| module_param | ( | jiffies_till_next_fqs | , |
| ulong | , | ||
| 0644 | |||
| ) |
rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
rcu_idle_enter - inform RCU that current CPU is entering idle
Enter idle mode, in other words, -leave- the mode in which RCU read-side critical sections can occur. (Though RCU read-side critical sections can occur in irq handlers in idle, a possibility handled by irq_enter() and irq_exit().)
We crowbar the ->dynticks_nesting field to zero to allow for the possibility of usermode upcalls having messed up our count of interrupt nesting level during the prior busy period.
rcu_idle_exit - inform RCU that current CPU is leaving idle
Exit idle mode, in other words, -enter- the mode in which RCU read-side critical sections can occur.
We crowbar the ->dynticks_nesting field to DYNTICK_TASK_NEST to allow for the possibility of usermode upcalls messing up our count of interrupt nesting level during the busy period that is just now starting.
rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
Enter an interrupt handler, which might possibly result in exiting idle mode, in other words, entering the mode in which read-side critical sections can occur.
Note that the Linux kernel is fully capable of entering an interrupt handler that it never exits, for example when doing upcalls to user mode! This code assumes that the idle loop never does upcalls to user mode. If your architecture does do upcalls from the idle loop (or does anything else that results in unbalanced calls to the irq_enter() and irq_exit() functions), RCU will give you what you deserve, good and hard. But very infrequently and irreproducibly.
Use things like work queues to work around this limitation.
You have been warned.
rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
Exit from an interrupt handler, which might possibly result in entering idle mode, in other words, leaving the mode in which read-side critical sections can occur.
This code assumes that the idle loop never does anything that might result in unbalanced calls to irq_enter() and irq_exit(). If your architecture violates this assumption, RCU will give you what you deserve, good and hard. But very infrequently and irreproducibly.
Use things like work queues to work around this limitation.
You have been warned.
synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed.
Control will return to the caller some time after a full rcu_bh grace period has elapsed, in other words after all currently executing rcu_bh read-side critical sections have completed. RCU read-side critical sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(), and may be nested.
synchronize_sched - wait until an rcu-sched grace period has elapsed.
Control will return to the caller some time after a full rcu-sched grace period has elapsed, in other words after all currently executing rcu-sched read-side critical sections have completed. These read-side critical sections are delimited by rcu_read_lock_sched() and rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(), local_irq_disable(), and so on may be used in place of rcu_read_lock_sched().
This means that all preempt_disable code sequences, including NMI and hardware-interrupt handlers, in progress on entry will have completed before this primitive returns. However, this does not guarantee that softirq handlers will have completed, since in some kernels, these handlers can run in process context, and can block.
This primitive provides the guarantees made by the (now removed) synchronize_kernel() API. In contrast, synchronize_rcu() only guarantees that rcu_read_lock() sections will have completed. In "classic RCU", these two guarantees happen to be one and the same, but can differ in realtime RCU implementations.
synchronize_sched_expedited - Brute-force RCU-sched grace period
Wait for an RCU-sched grace period to elapse, but use a "big hammer" approach to force the grace period to end quickly. This consumes significant time on all CPUs and is unfriendly to real-time workloads, so is thus not recommended for any sort of common-case code. In fact, if you are using synchronize_sched_expedited() in a loop, please restructure your code to batch your updates, and then use a single synchronize_sched() instead.
Note that it is illegal to call this function while holding any lock that is acquired by a CPU-hotplug notifier. And yes, it is also illegal to call this function from a CPU-hotplug notifier. Failing to observe these restriction will result in deadlock.
This implementation can be thought of as an application of ticket locking to RCU, with sync_sched_expedited_started and sync_sched_expedited_done taking on the roles of the halves of the ticket-lock word. Each task atomically increments sync_sched_expedited_started upon entry, snapshotting the old value, then attempts to stop all the CPUs. If this succeeds, then each CPU will have executed a context switch, resulting in an RCU-sched grace period. We are then done, so we use atomic_cmpxchg() to update sync_sched_expedited_done to match our snapshot – but only if someone else has not already advanced past our snapshot.
On the other hand, if try_stop_cpus() fails, we check the value of sync_sched_expedited_done. If it has advanced past our initial snapshot, then someone else must have forced a grace period some time after we took our snapshot. In this case, our work is done for us, and we can simply return. Otherwise, we try again, but keep our initial snapshot for purposes of checking for someone doing our work for us.
If we fail too many times in a row, we fall back to synchronize_sched().
| int rcu_cpu_stall_timeout __read_mostly = RCU_NUM_LVLS |
| struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh, call_rcu_bh) |
1.8.2