Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
atomic.h
Go to the documentation of this file.
1 /*
2  * Atomic operations that C can't guarantee us. Useful for
3  * resource counting etc..
4  *
5  * But use these as seldom as possible since they are much more slower
6  * than regular operations.
7  *
8  * This file is subject to the terms and conditions of the GNU General Public
9  * License. See the file "COPYING" in the main directory of this archive
10  * for more details.
11  *
12  * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle
13  */
14 #ifndef _ASM_ATOMIC_H
15 #define _ASM_ATOMIC_H
16 
17 #include <linux/irqflags.h>
18 #include <linux/types.h>
19 #include <asm/barrier.h>
20 #include <asm/cpu-features.h>
21 #include <asm/cmpxchg.h>
22 #include <asm/war.h>
23 
24 #define ATOMIC_INIT(i) { (i) }
25 
26 /*
27  * atomic_read - read atomic variable
28  * @v: pointer of type atomic_t
29  *
30  * Atomically reads the value of @v.
31  */
32 #define atomic_read(v) (*(volatile int *)&(v)->counter)
33 
34 /*
35  * atomic_set - set atomic variable
36  * @v: pointer of type atomic_t
37  * @i: required value
38  *
39  * Atomically sets the value of @v to @i.
40  */
41 #define atomic_set(v, i) ((v)->counter = (i))
42 
43 /*
44  * atomic_add - add integer to atomic variable
45  * @i: integer value to add
46  * @v: pointer of type atomic_t
47  *
48  * Atomically adds @i to @v.
49  */
50 static __inline__ void atomic_add(int i, atomic_t * v)
51 {
52  if (kernel_uses_llsc && R10000_LLSC_WAR) {
53  int temp;
54 
55  __asm__ __volatile__(
56  " .set mips3 \n"
57  "1: ll %0, %1 # atomic_add \n"
58  " addu %0, %2 \n"
59  " sc %0, %1 \n"
60  " beqzl %0, 1b \n"
61  " .set mips0 \n"
62  : "=&r" (temp), "+m" (v->counter)
63  : "Ir" (i));
64  } else if (kernel_uses_llsc) {
65  int temp;
66 
67  do {
68  __asm__ __volatile__(
69  " .set mips3 \n"
70  " ll %0, %1 # atomic_add \n"
71  " addu %0, %2 \n"
72  " sc %0, %1 \n"
73  " .set mips0 \n"
74  : "=&r" (temp), "+m" (v->counter)
75  : "Ir" (i));
76  } while (unlikely(!temp));
77  } else {
78  unsigned long flags;
79 
80  raw_local_irq_save(flags);
81  v->counter += i;
82  raw_local_irq_restore(flags);
83  }
84 }
85 
86 /*
87  * atomic_sub - subtract the atomic variable
88  * @i: integer value to subtract
89  * @v: pointer of type atomic_t
90  *
91  * Atomically subtracts @i from @v.
92  */
93 static __inline__ void atomic_sub(int i, atomic_t * v)
94 {
95  if (kernel_uses_llsc && R10000_LLSC_WAR) {
96  int temp;
97 
98  __asm__ __volatile__(
99  " .set mips3 \n"
100  "1: ll %0, %1 # atomic_sub \n"
101  " subu %0, %2 \n"
102  " sc %0, %1 \n"
103  " beqzl %0, 1b \n"
104  " .set mips0 \n"
105  : "=&r" (temp), "+m" (v->counter)
106  : "Ir" (i));
107  } else if (kernel_uses_llsc) {
108  int temp;
109 
110  do {
111  __asm__ __volatile__(
112  " .set mips3 \n"
113  " ll %0, %1 # atomic_sub \n"
114  " subu %0, %2 \n"
115  " sc %0, %1 \n"
116  " .set mips0 \n"
117  : "=&r" (temp), "+m" (v->counter)
118  : "Ir" (i));
119  } while (unlikely(!temp));
120  } else {
121  unsigned long flags;
122 
123  raw_local_irq_save(flags);
124  v->counter -= i;
125  raw_local_irq_restore(flags);
126  }
127 }
128 
129 /*
130  * Same as above, but return the result value
131  */
132 static __inline__ int atomic_add_return(int i, atomic_t * v)
133 {
134  int result;
135 
136  smp_mb__before_llsc();
137 
138  if (kernel_uses_llsc && R10000_LLSC_WAR) {
139  int temp;
140 
141  __asm__ __volatile__(
142  " .set mips3 \n"
143  "1: ll %1, %2 # atomic_add_return \n"
144  " addu %0, %1, %3 \n"
145  " sc %0, %2 \n"
146  " beqzl %0, 1b \n"
147  " addu %0, %1, %3 \n"
148  " .set mips0 \n"
149  : "=&r" (result), "=&r" (temp), "+m" (v->counter)
150  : "Ir" (i));
151  } else if (kernel_uses_llsc) {
152  int temp;
153 
154  do {
155  __asm__ __volatile__(
156  " .set mips3 \n"
157  " ll %1, %2 # atomic_add_return \n"
158  " addu %0, %1, %3 \n"
159  " sc %0, %2 \n"
160  " .set mips0 \n"
161  : "=&r" (result), "=&r" (temp), "+m" (v->counter)
162  : "Ir" (i));
163  } while (unlikely(!result));
164 
165  result = temp + i;
166  } else {
167  unsigned long flags;
168 
169  raw_local_irq_save(flags);
170  result = v->counter;
171  result += i;
172  v->counter = result;
173  raw_local_irq_restore(flags);
174  }
175 
176  smp_llsc_mb();
177 
178  return result;
179 }
180 
181 static __inline__ int atomic_sub_return(int i, atomic_t * v)
182 {
183  int result;
184 
185  smp_mb__before_llsc();
186 
187  if (kernel_uses_llsc && R10000_LLSC_WAR) {
188  int temp;
189 
190  __asm__ __volatile__(
191  " .set mips3 \n"
192  "1: ll %1, %2 # atomic_sub_return \n"
193  " subu %0, %1, %3 \n"
194  " sc %0, %2 \n"
195  " beqzl %0, 1b \n"
196  " subu %0, %1, %3 \n"
197  " .set mips0 \n"
198  : "=&r" (result), "=&r" (temp), "=m" (v->counter)
199  : "Ir" (i), "m" (v->counter)
200  : "memory");
201 
202  result = temp - i;
203  } else if (kernel_uses_llsc) {
204  int temp;
205 
206  do {
207  __asm__ __volatile__(
208  " .set mips3 \n"
209  " ll %1, %2 # atomic_sub_return \n"
210  " subu %0, %1, %3 \n"
211  " sc %0, %2 \n"
212  " .set mips0 \n"
213  : "=&r" (result), "=&r" (temp), "+m" (v->counter)
214  : "Ir" (i));
215  } while (unlikely(!result));
216 
217  result = temp - i;
218  } else {
219  unsigned long flags;
220 
221  raw_local_irq_save(flags);
222  result = v->counter;
223  result -= i;
224  v->counter = result;
225  raw_local_irq_restore(flags);
226  }
227 
228  smp_llsc_mb();
229 
230  return result;
231 }
232 
233 /*
234  * atomic_sub_if_positive - conditionally subtract integer from atomic variable
235  * @i: integer value to subtract
236  * @v: pointer of type atomic_t
237  *
238  * Atomically test @v and subtract @i if @v is greater or equal than @i.
239  * The function returns the old value of @v minus @i.
240  */
241 static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
242 {
243  int result;
244 
245  smp_mb__before_llsc();
246 
247  if (kernel_uses_llsc && R10000_LLSC_WAR) {
248  int temp;
249 
250  __asm__ __volatile__(
251  " .set mips3 \n"
252  "1: ll %1, %2 # atomic_sub_if_positive\n"
253  " subu %0, %1, %3 \n"
254  " bltz %0, 1f \n"
255  " sc %0, %2 \n"
256  " .set noreorder \n"
257  " beqzl %0, 1b \n"
258  " subu %0, %1, %3 \n"
259  " .set reorder \n"
260  "1: \n"
261  " .set mips0 \n"
262  : "=&r" (result), "=&r" (temp), "+m" (v->counter)
263  : "Ir" (i), "m" (v->counter)
264  : "memory");
265  } else if (kernel_uses_llsc) {
266  int temp;
267 
268  __asm__ __volatile__(
269  " .set mips3 \n"
270  "1: ll %1, %2 # atomic_sub_if_positive\n"
271  " subu %0, %1, %3 \n"
272  " bltz %0, 1f \n"
273  " sc %0, %2 \n"
274  " .set noreorder \n"
275  " beqz %0, 1b \n"
276  " subu %0, %1, %3 \n"
277  " .set reorder \n"
278  "1: \n"
279  " .set mips0 \n"
280  : "=&r" (result), "=&r" (temp), "+m" (v->counter)
281  : "Ir" (i));
282  } else {
283  unsigned long flags;
284 
285  raw_local_irq_save(flags);
286  result = v->counter;
287  result -= i;
288  if (result >= 0)
289  v->counter = result;
290  raw_local_irq_restore(flags);
291  }
292 
293  smp_llsc_mb();
294 
295  return result;
296 }
297 
298 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
299 #define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
300 
310 static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u)
311 {
312  int c, old;
313  c = atomic_read(v);
314  for (;;) {
315  if (unlikely(c == (u)))
316  break;
317  old = atomic_cmpxchg((v), c, c + (a));
318  if (likely(old == c))
319  break;
320  c = old;
321  }
322  return c;
323 }
324 
325 #define atomic_dec_return(v) atomic_sub_return(1, (v))
326 #define atomic_inc_return(v) atomic_add_return(1, (v))
327 
328 /*
329  * atomic_sub_and_test - subtract value from variable and test result
330  * @i: integer value to subtract
331  * @v: pointer of type atomic_t
332  *
333  * Atomically subtracts @i from @v and returns
334  * true if the result is zero, or false for all
335  * other cases.
336  */
337 #define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
338 
339 /*
340  * atomic_inc_and_test - increment and test
341  * @v: pointer of type atomic_t
342  *
343  * Atomically increments @v by 1
344  * and returns true if the result is zero, or false for all
345  * other cases.
346  */
347 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
348 
349 /*
350  * atomic_dec_and_test - decrement by 1 and test
351  * @v: pointer of type atomic_t
352  *
353  * Atomically decrements @v by 1 and
354  * returns true if the result is 0, or false for all other
355  * cases.
356  */
357 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
358 
359 /*
360  * atomic_dec_if_positive - decrement by 1 if old value positive
361  * @v: pointer of type atomic_t
362  */
363 #define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v)
364 
365 /*
366  * atomic_inc - increment atomic variable
367  * @v: pointer of type atomic_t
368  *
369  * Atomically increments @v by 1.
370  */
371 #define atomic_inc(v) atomic_add(1, (v))
372 
373 /*
374  * atomic_dec - decrement and test
375  * @v: pointer of type atomic_t
376  *
377  * Atomically decrements @v by 1.
378  */
379 #define atomic_dec(v) atomic_sub(1, (v))
380 
381 /*
382  * atomic_add_negative - add and test if negative
383  * @v: pointer of type atomic_t
384  * @i: integer value to add
385  *
386  * Atomically adds @i to @v and returns true
387  * if the result is negative, or false when
388  * result is greater than or equal to zero.
389  */
390 #define atomic_add_negative(i, v) (atomic_add_return(i, (v)) < 0)
391 
392 #ifdef CONFIG_64BIT
393 
394 #define ATOMIC64_INIT(i) { (i) }
395 
396 /*
397  * atomic64_read - read atomic variable
398  * @v: pointer of type atomic64_t
399  *
400  */
401 #define atomic64_read(v) (*(volatile long *)&(v)->counter)
402 
403 /*
404  * atomic64_set - set atomic variable
405  * @v: pointer of type atomic64_t
406  * @i: required value
407  */
408 #define atomic64_set(v, i) ((v)->counter = (i))
409 
410 /*
411  * atomic64_add - add integer to atomic variable
412  * @i: integer value to add
413  * @v: pointer of type atomic64_t
414  *
415  * Atomically adds @i to @v.
416  */
417 static __inline__ void atomic64_add(long i, atomic64_t * v)
418 {
419  if (kernel_uses_llsc && R10000_LLSC_WAR) {
420  long temp;
421 
422  __asm__ __volatile__(
423  " .set mips3 \n"
424  "1: lld %0, %1 # atomic64_add \n"
425  " daddu %0, %2 \n"
426  " scd %0, %1 \n"
427  " beqzl %0, 1b \n"
428  " .set mips0 \n"
429  : "=&r" (temp), "+m" (v->counter)
430  : "Ir" (i));
431  } else if (kernel_uses_llsc) {
432  long temp;
433 
434  do {
435  __asm__ __volatile__(
436  " .set mips3 \n"
437  " lld %0, %1 # atomic64_add \n"
438  " daddu %0, %2 \n"
439  " scd %0, %1 \n"
440  " .set mips0 \n"
441  : "=&r" (temp), "+m" (v->counter)
442  : "Ir" (i));
443  } while (unlikely(!temp));
444  } else {
445  unsigned long flags;
446 
447  raw_local_irq_save(flags);
448  v->counter += i;
449  raw_local_irq_restore(flags);
450  }
451 }
452 
453 /*
454  * atomic64_sub - subtract the atomic variable
455  * @i: integer value to subtract
456  * @v: pointer of type atomic64_t
457  *
458  * Atomically subtracts @i from @v.
459  */
460 static __inline__ void atomic64_sub(long i, atomic64_t * v)
461 {
462  if (kernel_uses_llsc && R10000_LLSC_WAR) {
463  long temp;
464 
465  __asm__ __volatile__(
466  " .set mips3 \n"
467  "1: lld %0, %1 # atomic64_sub \n"
468  " dsubu %0, %2 \n"
469  " scd %0, %1 \n"
470  " beqzl %0, 1b \n"
471  " .set mips0 \n"
472  : "=&r" (temp), "+m" (v->counter)
473  : "Ir" (i));
474  } else if (kernel_uses_llsc) {
475  long temp;
476 
477  do {
478  __asm__ __volatile__(
479  " .set mips3 \n"
480  " lld %0, %1 # atomic64_sub \n"
481  " dsubu %0, %2 \n"
482  " scd %0, %1 \n"
483  " .set mips0 \n"
484  : "=&r" (temp), "+m" (v->counter)
485  : "Ir" (i));
486  } while (unlikely(!temp));
487  } else {
488  unsigned long flags;
489 
490  raw_local_irq_save(flags);
491  v->counter -= i;
492  raw_local_irq_restore(flags);
493  }
494 }
495 
496 /*
497  * Same as above, but return the result value
498  */
499 static __inline__ long atomic64_add_return(long i, atomic64_t * v)
500 {
501  long result;
502 
503  smp_mb__before_llsc();
504 
505  if (kernel_uses_llsc && R10000_LLSC_WAR) {
506  long temp;
507 
508  __asm__ __volatile__(
509  " .set mips3 \n"
510  "1: lld %1, %2 # atomic64_add_return \n"
511  " daddu %0, %1, %3 \n"
512  " scd %0, %2 \n"
513  " beqzl %0, 1b \n"
514  " daddu %0, %1, %3 \n"
515  " .set mips0 \n"
516  : "=&r" (result), "=&r" (temp), "+m" (v->counter)
517  : "Ir" (i));
518  } else if (kernel_uses_llsc) {
519  long temp;
520 
521  do {
522  __asm__ __volatile__(
523  " .set mips3 \n"
524  " lld %1, %2 # atomic64_add_return \n"
525  " daddu %0, %1, %3 \n"
526  " scd %0, %2 \n"
527  " .set mips0 \n"
528  : "=&r" (result), "=&r" (temp), "=m" (v->counter)
529  : "Ir" (i), "m" (v->counter)
530  : "memory");
531  } while (unlikely(!result));
532 
533  result = temp + i;
534  } else {
535  unsigned long flags;
536 
537  raw_local_irq_save(flags);
538  result = v->counter;
539  result += i;
540  v->counter = result;
541  raw_local_irq_restore(flags);
542  }
543 
544  smp_llsc_mb();
545 
546  return result;
547 }
548 
549 static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
550 {
551  long result;
552 
553  smp_mb__before_llsc();
554 
555  if (kernel_uses_llsc && R10000_LLSC_WAR) {
556  long temp;
557 
558  __asm__ __volatile__(
559  " .set mips3 \n"
560  "1: lld %1, %2 # atomic64_sub_return \n"
561  " dsubu %0, %1, %3 \n"
562  " scd %0, %2 \n"
563  " beqzl %0, 1b \n"
564  " dsubu %0, %1, %3 \n"
565  " .set mips0 \n"
566  : "=&r" (result), "=&r" (temp), "=m" (v->counter)
567  : "Ir" (i), "m" (v->counter)
568  : "memory");
569  } else if (kernel_uses_llsc) {
570  long temp;
571 
572  do {
573  __asm__ __volatile__(
574  " .set mips3 \n"
575  " lld %1, %2 # atomic64_sub_return \n"
576  " dsubu %0, %1, %3 \n"
577  " scd %0, %2 \n"
578  " .set mips0 \n"
579  : "=&r" (result), "=&r" (temp), "=m" (v->counter)
580  : "Ir" (i), "m" (v->counter)
581  : "memory");
582  } while (unlikely(!result));
583 
584  result = temp - i;
585  } else {
586  unsigned long flags;
587 
588  raw_local_irq_save(flags);
589  result = v->counter;
590  result -= i;
591  v->counter = result;
592  raw_local_irq_restore(flags);
593  }
594 
595  smp_llsc_mb();
596 
597  return result;
598 }
599 
600 /*
601  * atomic64_sub_if_positive - conditionally subtract integer from atomic variable
602  * @i: integer value to subtract
603  * @v: pointer of type atomic64_t
604  *
605  * Atomically test @v and subtract @i if @v is greater or equal than @i.
606  * The function returns the old value of @v minus @i.
607  */
608 static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
609 {
610  long result;
611 
612  smp_mb__before_llsc();
613 
614  if (kernel_uses_llsc && R10000_LLSC_WAR) {
615  long temp;
616 
617  __asm__ __volatile__(
618  " .set mips3 \n"
619  "1: lld %1, %2 # atomic64_sub_if_positive\n"
620  " dsubu %0, %1, %3 \n"
621  " bltz %0, 1f \n"
622  " scd %0, %2 \n"
623  " .set noreorder \n"
624  " beqzl %0, 1b \n"
625  " dsubu %0, %1, %3 \n"
626  " .set reorder \n"
627  "1: \n"
628  " .set mips0 \n"
629  : "=&r" (result), "=&r" (temp), "=m" (v->counter)
630  : "Ir" (i), "m" (v->counter)
631  : "memory");
632  } else if (kernel_uses_llsc) {
633  long temp;
634 
635  __asm__ __volatile__(
636  " .set mips3 \n"
637  "1: lld %1, %2 # atomic64_sub_if_positive\n"
638  " dsubu %0, %1, %3 \n"
639  " bltz %0, 1f \n"
640  " scd %0, %2 \n"
641  " .set noreorder \n"
642  " beqz %0, 1b \n"
643  " dsubu %0, %1, %3 \n"
644  " .set reorder \n"
645  "1: \n"
646  " .set mips0 \n"
647  : "=&r" (result), "=&r" (temp), "+m" (v->counter)
648  : "Ir" (i));
649  } else {
650  unsigned long flags;
651 
652  raw_local_irq_save(flags);
653  result = v->counter;
654  result -= i;
655  if (result >= 0)
656  v->counter = result;
657  raw_local_irq_restore(flags);
658  }
659 
660  smp_llsc_mb();
661 
662  return result;
663 }
664 
665 #define atomic64_cmpxchg(v, o, n) \
666  ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
667 #define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
668 
678 static __inline__ int atomic64_add_unless(atomic64_t *v, long a, long u)
679 {
680  long c, old;
681  c = atomic64_read(v);
682  for (;;) {
683  if (unlikely(c == (u)))
684  break;
685  old = atomic64_cmpxchg((v), c, c + (a));
686  if (likely(old == c))
687  break;
688  c = old;
689  }
690  return c != (u);
691 }
692 
693 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
694 
695 #define atomic64_dec_return(v) atomic64_sub_return(1, (v))
696 #define atomic64_inc_return(v) atomic64_add_return(1, (v))
697 
698 /*
699  * atomic64_sub_and_test - subtract value from variable and test result
700  * @i: integer value to subtract
701  * @v: pointer of type atomic64_t
702  *
703  * Atomically subtracts @i from @v and returns
704  * true if the result is zero, or false for all
705  * other cases.
706  */
707 #define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
708 
709 /*
710  * atomic64_inc_and_test - increment and test
711  * @v: pointer of type atomic64_t
712  *
713  * Atomically increments @v by 1
714  * and returns true if the result is zero, or false for all
715  * other cases.
716  */
717 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
718 
719 /*
720  * atomic64_dec_and_test - decrement by 1 and test
721  * @v: pointer of type atomic64_t
722  *
723  * Atomically decrements @v by 1 and
724  * returns true if the result is 0, or false for all other
725  * cases.
726  */
727 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
728 
729 /*
730  * atomic64_dec_if_positive - decrement by 1 if old value positive
731  * @v: pointer of type atomic64_t
732  */
733 #define atomic64_dec_if_positive(v) atomic64_sub_if_positive(1, v)
734 
735 /*
736  * atomic64_inc - increment atomic variable
737  * @v: pointer of type atomic64_t
738  *
739  * Atomically increments @v by 1.
740  */
741 #define atomic64_inc(v) atomic64_add(1, (v))
742 
743 /*
744  * atomic64_dec - decrement and test
745  * @v: pointer of type atomic64_t
746  *
747  * Atomically decrements @v by 1.
748  */
749 #define atomic64_dec(v) atomic64_sub(1, (v))
750 
751 /*
752  * atomic64_add_negative - add and test if negative
753  * @v: pointer of type atomic64_t
754  * @i: integer value to add
755  *
756  * Atomically adds @i to @v and returns true
757  * if the result is negative, or false when
758  * result is greater than or equal to zero.
759  */
760 #define atomic64_add_negative(i, v) (atomic64_add_return(i, (v)) < 0)
761 
762 #endif /* CONFIG_64BIT */
763 
764 /*
765  * atomic*_return operations are serializing but not the non-*_return
766  * versions.
767  */
768 #define smp_mb__before_atomic_dec() smp_mb__before_llsc()
769 #define smp_mb__after_atomic_dec() smp_llsc_mb()
770 #define smp_mb__before_atomic_inc() smp_mb__before_llsc()
771 #define smp_mb__after_atomic_inc() smp_llsc_mb()
772 
773 #endif /* _ASM_ATOMIC_H */
Generated on Thu Jan 10 2013 12:49:07 for Linux Kernel by   doxygen 1.8.2