Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
percpu-rwsem.h
Go to the documentation of this file.
1 #ifndef _LINUX_PERCPU_RWSEM_H
2 #define _LINUX_PERCPU_RWSEM_H
3 
4 #include <linux/mutex.h>
5 #include <linux/percpu.h>
6 #include <linux/rcupdate.h>
7 #include <linux/delay.h>
8 
10  unsigned __percpu *counters;
11  bool locked;
12  struct mutex mtx;
13 };
14 
15 #define light_mb() barrier()
16 #define heavy_mb() synchronize_sched_expedited()
17 
18 static inline void percpu_down_read(struct percpu_rw_semaphore *p)
19 {
20  rcu_read_lock_sched();
21  if (unlikely(p->locked)) {
22  rcu_read_unlock_sched();
23  mutex_lock(&p->mtx);
25  mutex_unlock(&p->mtx);
26  return;
27  }
29  rcu_read_unlock_sched();
30  light_mb(); /* A, between read of p->locked and read of data, paired with D */
31 }
32 
33 static inline void percpu_up_read(struct percpu_rw_semaphore *p)
34 {
35  light_mb(); /* B, between read of the data and write to p->counter, paired with C */
37 }
38 
39 static inline unsigned __percpu_count(unsigned __percpu *counters)
40 {
41  unsigned total = 0;
42  int cpu;
43 
45  total += ACCESS_ONCE(*per_cpu_ptr(counters, cpu));
46 
47  return total;
48 }
49 
50 static inline void percpu_down_write(struct percpu_rw_semaphore *p)
51 {
52  mutex_lock(&p->mtx);
53  p->locked = true;
54  synchronize_sched_expedited(); /* make sure that all readers exit the rcu_read_lock_sched region */
55  while (__percpu_count(p->counters))
56  msleep(1);
57  heavy_mb(); /* C, between read of p->counter and write to data, paired with B */
58 }
59 
60 static inline void percpu_up_write(struct percpu_rw_semaphore *p)
61 {
62  heavy_mb(); /* D, between write to data and write to p->locked, paired with A */
63  p->locked = false;
64  mutex_unlock(&p->mtx);
65 }
66 
67 static inline int percpu_init_rwsem(struct percpu_rw_semaphore *p)
68 {
69  p->counters = alloc_percpu(unsigned);
70  if (unlikely(!p->counters))
71  return -ENOMEM;
72  p->locked = false;
73  mutex_init(&p->mtx);
74  return 0;
75 }
76 
77 static inline void percpu_free_rwsem(struct percpu_rw_semaphore *p)
78 {
80  p->counters = NULL; /* catch use after free bugs */
81 }
82 
83 #endif