Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
spinlock.h
Go to the documentation of this file.
1 /*
2  * This file is subject to the terms and conditions of the GNU General Public
3  * License. See the file "COPYING" in the main directory of this archive
4  * for more details.
5  *
6  * Copyright (C) 1999, 2000, 06 Ralf Baechle ([email protected])
7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8  */
9 #ifndef _ASM_SPINLOCK_H
10 #define _ASM_SPINLOCK_H
11 
12 #include <linux/compiler.h>
13 
14 #include <asm/barrier.h>
15 #include <asm/war.h>
16 
17 /*
18  * Your basic SMP spinlocks, allowing only a single CPU anywhere
19  *
20  * Simple spin lock operations. There are two variants, one clears IRQ's
21  * on the local processor, one does not.
22  *
23  * These are fair FIFO ticket locks
24  *
25  * (the type definitions are in asm/spinlock_types.h)
26  */
27 
28 
29 /*
30  * Ticket locks are conceptually two parts, one indicating the current head of
31  * the queue, and the other indicating the current tail. The lock is acquired
32  * by atomically noting the tail and incrementing it by one (thus adding
33  * ourself to the queue and noting our position), then waiting until the head
34  * becomes equal to the the initial value of the tail.
35  */
36 
37 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
38 {
39  u32 counters = ACCESS_ONCE(lock->lock);
40 
41  return ((counters >> 16) ^ counters) & 0xffff;
42 }
43 
44 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
45 #define arch_spin_unlock_wait(x) \
46  while (arch_spin_is_locked(x)) { cpu_relax(); }
47 
48 static inline int arch_spin_is_contended(arch_spinlock_t *lock)
49 {
50  u32 counters = ACCESS_ONCE(lock->lock);
51 
52  return (((counters >> 16) - counters) & 0xffff) > 1;
53 }
54 #define arch_spin_is_contended arch_spin_is_contended
55 
56 static inline void arch_spin_lock(arch_spinlock_t *lock)
57 {
58  int my_ticket;
59  int tmp;
60  int inc = 0x10000;
61 
62  if (R10000_LLSC_WAR) {
63  __asm__ __volatile__ (
64  " .set push # arch_spin_lock \n"
65  " .set noreorder \n"
66  " \n"
67  "1: ll %[ticket], %[ticket_ptr] \n"
68  " addu %[my_ticket], %[ticket], %[inc] \n"
69  " sc %[my_ticket], %[ticket_ptr] \n"
70  " beqzl %[my_ticket], 1b \n"
71  " nop \n"
72  " srl %[my_ticket], %[ticket], 16 \n"
73  " andi %[ticket], %[ticket], 0xffff \n"
74  " andi %[my_ticket], %[my_ticket], 0xffff \n"
75  " bne %[ticket], %[my_ticket], 4f \n"
76  " subu %[ticket], %[my_ticket], %[ticket] \n"
77  "2: \n"
78  " .subsection 2 \n"
79  "4: andi %[ticket], %[ticket], 0xffff \n"
80  " sll %[ticket], 5 \n"
81  " \n"
82  "6: bnez %[ticket], 6b \n"
83  " subu %[ticket], 1 \n"
84  " \n"
85  " lhu %[ticket], %[serving_now_ptr] \n"
86  " beq %[ticket], %[my_ticket], 2b \n"
87  " subu %[ticket], %[my_ticket], %[ticket] \n"
88  " b 4b \n"
89  " subu %[ticket], %[ticket], 1 \n"
90  " .previous \n"
91  " .set pop \n"
92  : [ticket_ptr] "+m" (lock->lock),
93  [serving_now_ptr] "+m" (lock->h.serving_now),
94  [ticket] "=&r" (tmp),
95  [my_ticket] "=&r" (my_ticket)
96  : [inc] "r" (inc));
97  } else {
98  __asm__ __volatile__ (
99  " .set push # arch_spin_lock \n"
100  " .set noreorder \n"
101  " \n"
102  "1: ll %[ticket], %[ticket_ptr] \n"
103  " addu %[my_ticket], %[ticket], %[inc] \n"
104  " sc %[my_ticket], %[ticket_ptr] \n"
105  " beqz %[my_ticket], 1b \n"
106  " srl %[my_ticket], %[ticket], 16 \n"
107  " andi %[ticket], %[ticket], 0xffff \n"
108  " andi %[my_ticket], %[my_ticket], 0xffff \n"
109  " bne %[ticket], %[my_ticket], 4f \n"
110  " subu %[ticket], %[my_ticket], %[ticket] \n"
111  "2: \n"
112  " .subsection 2 \n"
113  "4: andi %[ticket], %[ticket], 0x1fff \n"
114  " sll %[ticket], 5 \n"
115  " \n"
116  "6: bnez %[ticket], 6b \n"
117  " subu %[ticket], 1 \n"
118  " \n"
119  " lhu %[ticket], %[serving_now_ptr] \n"
120  " beq %[ticket], %[my_ticket], 2b \n"
121  " subu %[ticket], %[my_ticket], %[ticket] \n"
122  " b 4b \n"
123  " subu %[ticket], %[ticket], 1 \n"
124  " .previous \n"
125  " .set pop \n"
126  : [ticket_ptr] "+m" (lock->lock),
127  [serving_now_ptr] "+m" (lock->h.serving_now),
128  [ticket] "=&r" (tmp),
129  [my_ticket] "=&r" (my_ticket)
130  : [inc] "r" (inc));
131  }
132 
133  smp_llsc_mb();
134 }
135 
136 static inline void arch_spin_unlock(arch_spinlock_t *lock)
137 {
138  unsigned int serving_now = lock->h.serving_now + 1;
139  wmb();
140  lock->h.serving_now = (u16)serving_now;
141  nudge_writes();
142 }
143 
144 static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
145 {
146  int tmp, tmp2, tmp3;
147  int inc = 0x10000;
148 
149  if (R10000_LLSC_WAR) {
150  __asm__ __volatile__ (
151  " .set push # arch_spin_trylock \n"
152  " .set noreorder \n"
153  " \n"
154  "1: ll %[ticket], %[ticket_ptr] \n"
155  " srl %[my_ticket], %[ticket], 16 \n"
156  " andi %[my_ticket], %[my_ticket], 0xffff \n"
157  " andi %[now_serving], %[ticket], 0xffff \n"
158  " bne %[my_ticket], %[now_serving], 3f \n"
159  " addu %[ticket], %[ticket], %[inc] \n"
160  " sc %[ticket], %[ticket_ptr] \n"
161  " beqzl %[ticket], 1b \n"
162  " li %[ticket], 1 \n"
163  "2: \n"
164  " .subsection 2 \n"
165  "3: b 2b \n"
166  " li %[ticket], 0 \n"
167  " .previous \n"
168  " .set pop \n"
169  : [ticket_ptr] "+m" (lock->lock),
170  [ticket] "=&r" (tmp),
171  [my_ticket] "=&r" (tmp2),
172  [now_serving] "=&r" (tmp3)
173  : [inc] "r" (inc));
174  } else {
175  __asm__ __volatile__ (
176  " .set push # arch_spin_trylock \n"
177  " .set noreorder \n"
178  " \n"
179  "1: ll %[ticket], %[ticket_ptr] \n"
180  " srl %[my_ticket], %[ticket], 16 \n"
181  " andi %[my_ticket], %[my_ticket], 0xffff \n"
182  " andi %[now_serving], %[ticket], 0xffff \n"
183  " bne %[my_ticket], %[now_serving], 3f \n"
184  " addu %[ticket], %[ticket], %[inc] \n"
185  " sc %[ticket], %[ticket_ptr] \n"
186  " beqz %[ticket], 1b \n"
187  " li %[ticket], 1 \n"
188  "2: \n"
189  " .subsection 2 \n"
190  "3: b 2b \n"
191  " li %[ticket], 0 \n"
192  " .previous \n"
193  " .set pop \n"
194  : [ticket_ptr] "+m" (lock->lock),
195  [ticket] "=&r" (tmp),
196  [my_ticket] "=&r" (tmp2),
197  [now_serving] "=&r" (tmp3)
198  : [inc] "r" (inc));
199  }
200 
201  smp_llsc_mb();
202 
203  return tmp;
204 }
205 
206 /*
207  * Read-write spinlocks, allowing multiple readers but only one writer.
208  *
209  * NOTE! it is quite common to have readers in interrupts but no interrupt
210  * writers. For those circumstances we can "mix" irq-safe locks - any writer
211  * needs to get a irq-safe write-lock, but readers can get non-irqsafe
212  * read-locks.
213  */
214 
215 /*
216  * read_can_lock - would read_trylock() succeed?
217  * @lock: the rwlock in question.
218  */
219 #define arch_read_can_lock(rw) ((rw)->lock >= 0)
220 
221 /*
222  * write_can_lock - would write_trylock() succeed?
223  * @lock: the rwlock in question.
224  */
225 #define arch_write_can_lock(rw) (!(rw)->lock)
226 
227 static inline void arch_read_lock(arch_rwlock_t *rw)
228 {
229  unsigned int tmp;
230 
231  if (R10000_LLSC_WAR) {
232  __asm__ __volatile__(
233  " .set noreorder # arch_read_lock \n"
234  "1: ll %1, %2 \n"
235  " bltz %1, 1b \n"
236  " addu %1, 1 \n"
237  " sc %1, %0 \n"
238  " beqzl %1, 1b \n"
239  " nop \n"
240  " .set reorder \n"
241  : "=m" (rw->lock), "=&r" (tmp)
242  : "m" (rw->lock)
243  : "memory");
244  } else {
245  __asm__ __volatile__(
246  " .set noreorder # arch_read_lock \n"
247  "1: ll %1, %2 \n"
248  " bltz %1, 3f \n"
249  " addu %1, 1 \n"
250  "2: sc %1, %0 \n"
251  " beqz %1, 1b \n"
252  " nop \n"
253  " .subsection 2 \n"
254  "3: ll %1, %2 \n"
255  " bltz %1, 3b \n"
256  " addu %1, 1 \n"
257  " b 2b \n"
258  " nop \n"
259  " .previous \n"
260  " .set reorder \n"
261  : "=m" (rw->lock), "=&r" (tmp)
262  : "m" (rw->lock)
263  : "memory");
264  }
265 
266  smp_llsc_mb();
267 }
268 
269 /* Note the use of sub, not subu which will make the kernel die with an
270  overflow exception if we ever try to unlock an rwlock that is already
271  unlocked or is being held by a writer. */
272 static inline void arch_read_unlock(arch_rwlock_t *rw)
273 {
274  unsigned int tmp;
275 
276  smp_mb__before_llsc();
277 
278  if (R10000_LLSC_WAR) {
279  __asm__ __volatile__(
280  "1: ll %1, %2 # arch_read_unlock \n"
281  " sub %1, 1 \n"
282  " sc %1, %0 \n"
283  " beqzl %1, 1b \n"
284  : "=m" (rw->lock), "=&r" (tmp)
285  : "m" (rw->lock)
286  : "memory");
287  } else {
288  __asm__ __volatile__(
289  " .set noreorder # arch_read_unlock \n"
290  "1: ll %1, %2 \n"
291  " sub %1, 1 \n"
292  " sc %1, %0 \n"
293  " beqz %1, 2f \n"
294  " nop \n"
295  " .subsection 2 \n"
296  "2: b 1b \n"
297  " nop \n"
298  " .previous \n"
299  " .set reorder \n"
300  : "=m" (rw->lock), "=&r" (tmp)
301  : "m" (rw->lock)
302  : "memory");
303  }
304 }
305 
306 static inline void arch_write_lock(arch_rwlock_t *rw)
307 {
308  unsigned int tmp;
309 
310  if (R10000_LLSC_WAR) {
311  __asm__ __volatile__(
312  " .set noreorder # arch_write_lock \n"
313  "1: ll %1, %2 \n"
314  " bnez %1, 1b \n"
315  " lui %1, 0x8000 \n"
316  " sc %1, %0 \n"
317  " beqzl %1, 1b \n"
318  " nop \n"
319  " .set reorder \n"
320  : "=m" (rw->lock), "=&r" (tmp)
321  : "m" (rw->lock)
322  : "memory");
323  } else {
324  __asm__ __volatile__(
325  " .set noreorder # arch_write_lock \n"
326  "1: ll %1, %2 \n"
327  " bnez %1, 3f \n"
328  " lui %1, 0x8000 \n"
329  "2: sc %1, %0 \n"
330  " beqz %1, 3f \n"
331  " nop \n"
332  " .subsection 2 \n"
333  "3: ll %1, %2 \n"
334  " bnez %1, 3b \n"
335  " lui %1, 0x8000 \n"
336  " b 2b \n"
337  " nop \n"
338  " .previous \n"
339  " .set reorder \n"
340  : "=m" (rw->lock), "=&r" (tmp)
341  : "m" (rw->lock)
342  : "memory");
343  }
344 
345  smp_llsc_mb();
346 }
347 
348 static inline void arch_write_unlock(arch_rwlock_t *rw)
349 {
350  smp_mb();
351 
352  __asm__ __volatile__(
353  " # arch_write_unlock \n"
354  " sw $0, %0 \n"
355  : "=m" (rw->lock)
356  : "m" (rw->lock)
357  : "memory");
358 }
359 
360 static inline int arch_read_trylock(arch_rwlock_t *rw)
361 {
362  unsigned int tmp;
363  int ret;
364 
365  if (R10000_LLSC_WAR) {
366  __asm__ __volatile__(
367  " .set noreorder # arch_read_trylock \n"
368  " li %2, 0 \n"
369  "1: ll %1, %3 \n"
370  " bltz %1, 2f \n"
371  " addu %1, 1 \n"
372  " sc %1, %0 \n"
373  " .set reorder \n"
374  " beqzl %1, 1b \n"
375  " nop \n"
376  __WEAK_LLSC_MB
377  " li %2, 1 \n"
378  "2: \n"
379  : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
380  : "m" (rw->lock)
381  : "memory");
382  } else {
383  __asm__ __volatile__(
384  " .set noreorder # arch_read_trylock \n"
385  " li %2, 0 \n"
386  "1: ll %1, %3 \n"
387  " bltz %1, 2f \n"
388  " addu %1, 1 \n"
389  " sc %1, %0 \n"
390  " beqz %1, 1b \n"
391  " nop \n"
392  " .set reorder \n"
393  __WEAK_LLSC_MB
394  " li %2, 1 \n"
395  "2: \n"
396  : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
397  : "m" (rw->lock)
398  : "memory");
399  }
400 
401  return ret;
402 }
403 
404 static inline int arch_write_trylock(arch_rwlock_t *rw)
405 {
406  unsigned int tmp;
407  int ret;
408 
409  if (R10000_LLSC_WAR) {
410  __asm__ __volatile__(
411  " .set noreorder # arch_write_trylock \n"
412  " li %2, 0 \n"
413  "1: ll %1, %3 \n"
414  " bnez %1, 2f \n"
415  " lui %1, 0x8000 \n"
416  " sc %1, %0 \n"
417  " beqzl %1, 1b \n"
418  " nop \n"
419  __WEAK_LLSC_MB
420  " li %2, 1 \n"
421  " .set reorder \n"
422  "2: \n"
423  : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
424  : "m" (rw->lock)
425  : "memory");
426  } else {
427  __asm__ __volatile__(
428  " .set noreorder # arch_write_trylock \n"
429  " li %2, 0 \n"
430  "1: ll %1, %3 \n"
431  " bnez %1, 2f \n"
432  " lui %1, 0x8000 \n"
433  " sc %1, %0 \n"
434  " beqz %1, 3f \n"
435  " li %2, 1 \n"
436  "2: \n"
437  __WEAK_LLSC_MB
438  " .subsection 2 \n"
439  "3: b 1b \n"
440  " li %2, 0 \n"
441  " .previous \n"
442  " .set reorder \n"
443  : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
444  : "m" (rw->lock)
445  : "memory");
446  }
447 
448  return ret;
449 }
450 
451 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
452 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
453 
454 #define arch_spin_relax(lock) cpu_relax()
455 #define arch_read_relax(lock) cpu_relax()
456 #define arch_write_relax(lock) cpu_relax()
457 
458 #endif /* _ASM_SPINLOCK_H */
Generated on Thu Jan 10 2013 12:50:34 for Linux Kernel by   doxygen 1.8.2