Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
cmpxchg.h
Go to the documentation of this file.
1 /*
2  * Copyright IBM Corp. 1999, 2011
3  *
4  * Author(s): Martin Schwidefsky <[email protected]>,
5  */
6 
7 #ifndef __ASM_CMPXCHG_H
8 #define __ASM_CMPXCHG_H
9 
10 #include <linux/mmdebug.h>
11 #include <linux/types.h>
12 #include <linux/bug.h>
13 
14 extern void __xchg_called_with_bad_pointer(void);
15 
16 static inline unsigned long __xchg(unsigned long x, void *ptr, int size)
17 {
18  unsigned long addr, old;
19  int shift;
20 
21  switch (size) {
22  case 1:
23  addr = (unsigned long) ptr;
24  shift = (3 ^ (addr & 3)) << 3;
25  addr ^= addr & 3;
26  asm volatile(
27  " l %0,%4\n"
28  "0: lr 0,%0\n"
29  " nr 0,%3\n"
30  " or 0,%2\n"
31  " cs %0,0,%4\n"
32  " jl 0b\n"
33  : "=&d" (old), "=Q" (*(int *) addr)
34  : "d" ((x & 0xff) << shift), "d" (~(0xff << shift)),
35  "Q" (*(int *) addr) : "memory", "cc", "0");
36  return old >> shift;
37  case 2:
38  addr = (unsigned long) ptr;
39  shift = (2 ^ (addr & 2)) << 3;
40  addr ^= addr & 2;
41  asm volatile(
42  " l %0,%4\n"
43  "0: lr 0,%0\n"
44  " nr 0,%3\n"
45  " or 0,%2\n"
46  " cs %0,0,%4\n"
47  " jl 0b\n"
48  : "=&d" (old), "=Q" (*(int *) addr)
49  : "d" ((x & 0xffff) << shift), "d" (~(0xffff << shift)),
50  "Q" (*(int *) addr) : "memory", "cc", "0");
51  return old >> shift;
52  case 4:
53  asm volatile(
54  " l %0,%3\n"
55  "0: cs %0,%2,%3\n"
56  " jl 0b\n"
57  : "=&d" (old), "=Q" (*(int *) ptr)
58  : "d" (x), "Q" (*(int *) ptr)
59  : "memory", "cc");
60  return old;
61 #ifdef CONFIG_64BIT
62  case 8:
63  asm volatile(
64  " lg %0,%3\n"
65  "0: csg %0,%2,%3\n"
66  " jl 0b\n"
67  : "=&d" (old), "=m" (*(long *) ptr)
68  : "d" (x), "Q" (*(long *) ptr)
69  : "memory", "cc");
70  return old;
71 #endif /* CONFIG_64BIT */
72  }
73  __xchg_called_with_bad_pointer();
74  return x;
75 }
76 
77 #define xchg(ptr, x) \
78 ({ \
79  __typeof__(*(ptr)) __ret; \
80  __ret = (__typeof__(*(ptr))) \
81  __xchg((unsigned long)(x), (void *)(ptr), sizeof(*(ptr)));\
82  __ret; \
83 })
84 
85 /*
86  * Atomic compare and exchange. Compare OLD with MEM, if identical,
87  * store NEW in MEM. Return the initial value in MEM. Success is
88  * indicated by comparing RETURN with OLD.
89  */
90 
91 #define __HAVE_ARCH_CMPXCHG
92 
93 extern void __cmpxchg_called_with_bad_pointer(void);
94 
95 static inline unsigned long __cmpxchg(void *ptr, unsigned long old,
96  unsigned long new, int size)
97 {
98  unsigned long addr, prev, tmp;
99  int shift;
100 
101  switch (size) {
102  case 1:
103  addr = (unsigned long) ptr;
104  shift = (3 ^ (addr & 3)) << 3;
105  addr ^= addr & 3;
106  asm volatile(
107  " l %0,%2\n"
108  "0: nr %0,%5\n"
109  " lr %1,%0\n"
110  " or %0,%3\n"
111  " or %1,%4\n"
112  " cs %0,%1,%2\n"
113  " jnl 1f\n"
114  " xr %1,%0\n"
115  " nr %1,%5\n"
116  " jnz 0b\n"
117  "1:"
118  : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr)
119  : "d" ((old & 0xff) << shift),
120  "d" ((new & 0xff) << shift),
121  "d" (~(0xff << shift))
122  : "memory", "cc");
123  return prev >> shift;
124  case 2:
125  addr = (unsigned long) ptr;
126  shift = (2 ^ (addr & 2)) << 3;
127  addr ^= addr & 2;
128  asm volatile(
129  " l %0,%2\n"
130  "0: nr %0,%5\n"
131  " lr %1,%0\n"
132  " or %0,%3\n"
133  " or %1,%4\n"
134  " cs %0,%1,%2\n"
135  " jnl 1f\n"
136  " xr %1,%0\n"
137  " nr %1,%5\n"
138  " jnz 0b\n"
139  "1:"
140  : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr)
141  : "d" ((old & 0xffff) << shift),
142  "d" ((new & 0xffff) << shift),
143  "d" (~(0xffff << shift))
144  : "memory", "cc");
145  return prev >> shift;
146  case 4:
147  asm volatile(
148  " cs %0,%3,%1\n"
149  : "=&d" (prev), "=Q" (*(int *) ptr)
150  : "0" (old), "d" (new), "Q" (*(int *) ptr)
151  : "memory", "cc");
152  return prev;
153 #ifdef CONFIG_64BIT
154  case 8:
155  asm volatile(
156  " csg %0,%3,%1\n"
157  : "=&d" (prev), "=Q" (*(long *) ptr)
158  : "0" (old), "d" (new), "Q" (*(long *) ptr)
159  : "memory", "cc");
160  return prev;
161 #endif /* CONFIG_64BIT */
162  }
163  __cmpxchg_called_with_bad_pointer();
164  return old;
165 }
166 
167 #define cmpxchg(ptr, o, n) \
168 ({ \
169  __typeof__(*(ptr)) __ret; \
170  __ret = (__typeof__(*(ptr))) \
171  __cmpxchg((ptr), (unsigned long)(o), (unsigned long)(n), \
172  sizeof(*(ptr))); \
173  __ret; \
174 })
175 
176 #ifdef CONFIG_64BIT
177 #define cmpxchg64(ptr, o, n) \
178 ({ \
179  cmpxchg((ptr), (o), (n)); \
180 })
181 #else /* CONFIG_64BIT */
182 static inline unsigned long long __cmpxchg64(void *ptr,
183  unsigned long long old,
184  unsigned long long new)
185 {
186  register_pair rp_old = {.pair = old};
187  register_pair rp_new = {.pair = new};
188 
189  asm volatile(
190  " cds %0,%2,%1"
191  : "+&d" (rp_old), "=Q" (ptr)
192  : "d" (rp_new), "Q" (ptr)
193  : "memory", "cc");
194  return rp_old.pair;
195 }
196 
197 #define cmpxchg64(ptr, o, n) \
198 ({ \
199  __typeof__(*(ptr)) __ret; \
200  __ret = (__typeof__(*(ptr))) \
201  __cmpxchg64((ptr), \
202  (unsigned long long)(o), \
203  (unsigned long long)(n)); \
204  __ret; \
205 })
206 #endif /* CONFIG_64BIT */
207 
208 #define __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, insn) \
209 ({ \
210  register __typeof__(*(p1)) __old1 asm("2") = (o1); \
211  register __typeof__(*(p2)) __old2 asm("3") = (o2); \
212  register __typeof__(*(p1)) __new1 asm("4") = (n1); \
213  register __typeof__(*(p2)) __new2 asm("5") = (n2); \
214  int cc; \
215  asm volatile( \
216  insn " %[old],%[new],%[ptr]\n" \
217  " ipm %[cc]\n" \
218  " srl %[cc],28" \
219  : [cc] "=d" (cc), [old] "+d" (__old1), "+d" (__old2) \
220  : [new] "d" (__new1), "d" (__new2), \
221  [ptr] "Q" (*(p1)), "Q" (*(p2)) \
222  : "memory", "cc"); \
223  !cc; \
224 })
225 
226 #define __cmpxchg_double_4(p1, p2, o1, o2, n1, n2) \
227  __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cds")
228 
229 #define __cmpxchg_double_8(p1, p2, o1, o2, n1, n2) \
230  __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cdsg")
231 
232 extern void __cmpxchg_double_called_with_bad_pointer(void);
233 
234 #define __cmpxchg_double(p1, p2, o1, o2, n1, n2) \
235 ({ \
236  int __ret; \
237  switch (sizeof(*(p1))) { \
238  case 4: \
239  __ret = __cmpxchg_double_4(p1, p2, o1, o2, n1, n2); \
240  break; \
241  case 8: \
242  __ret = __cmpxchg_double_8(p1, p2, o1, o2, n1, n2); \
243  break; \
244  default: \
245  __cmpxchg_double_called_with_bad_pointer(); \
246  } \
247  __ret; \
248 })
249 
250 #define cmpxchg_double(p1, p2, o1, o2, n1, n2) \
251 ({ \
252  __typeof__(p1) __p1 = (p1); \
253  __typeof__(p2) __p2 = (p2); \
254  int __ret; \
255  BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
256  BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
257  VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\
258  if (sizeof(long) == 4) \
259  __ret = __cmpxchg_double_4(__p1, __p2, o1, o2, n1, n2); \
260  else \
261  __ret = __cmpxchg_double_8(__p1, __p2, o1, o2, n1, n2); \
262  __ret; \
263 })
264 
265 #define system_has_cmpxchg_double() 1
266 
267 #include <asm-generic/cmpxchg-local.h>
268 
269 static inline unsigned long __cmpxchg_local(void *ptr,
270  unsigned long old,
271  unsigned long new, int size)
272 {
273  switch (size) {
274  case 1:
275  case 2:
276  case 4:
277 #ifdef CONFIG_64BIT
278  case 8:
279 #endif
280  return __cmpxchg(ptr, old, new, size);
281  default:
282  return __cmpxchg_local_generic(ptr, old, new, size);
283  }
284 
285  return old;
286 }
287 
288 /*
289  * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make
290  * them available.
291  */
292 #define cmpxchg_local(ptr, o, n) \
293 ({ \
294  __typeof__(*(ptr)) __ret; \
295  __ret = (__typeof__(*(ptr))) \
296  __cmpxchg_local((ptr), (unsigned long)(o), \
297  (unsigned long)(n), sizeof(*(ptr))); \
298  __ret; \
299 })
300 
301 #define cmpxchg64_local(ptr, o, n) cmpxchg64((ptr), (o), (n))
302 
303 #endif /* __ASM_CMPXCHG_H */
Generated on Thu Jan 10 2013 12:49:17 for Linux Kernel by   doxygen 1.8.2