Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
percpu.h
Go to the documentation of this file.
1 #ifndef __LINUX_PERCPU_H
2 #define __LINUX_PERCPU_H
3 
4 #include <linux/preempt.h>
5 #include <linux/smp.h>
6 #include <linux/cpumask.h>
7 #include <linux/pfn.h>
8 #include <linux/init.h>
9 
10 #include <asm/percpu.h>
11 
12 /* enough to cover all DEFINE_PER_CPUs in modules */
13 #ifdef CONFIG_MODULES
14 #define PERCPU_MODULE_RESERVE (8 << 10)
15 #else
16 #define PERCPU_MODULE_RESERVE 0
17 #endif
18 
19 #ifndef PERCPU_ENOUGH_ROOM
20 #define PERCPU_ENOUGH_ROOM \
21  (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
22  PERCPU_MODULE_RESERVE)
23 #endif
24 
25 /*
26  * Must be an lvalue. Since @var must be a simple identifier,
27  * we force a syntax error here if it isn't.
28  */
29 #define get_cpu_var(var) (*({ \
30  preempt_disable(); \
31  &__get_cpu_var(var); }))
32 
33 /*
34  * The weird & is necessary because sparse considers (void)(var) to be
35  * a direct dereference of percpu variable (var).
36  */
37 #define put_cpu_var(var) do { \
38  (void)&(var); \
39  preempt_enable(); \
40 } while (0)
41 
42 #define get_cpu_ptr(var) ({ \
43  preempt_disable(); \
44  this_cpu_ptr(var); })
45 
46 #define put_cpu_ptr(var) do { \
47  (void)(var); \
48  preempt_enable(); \
49 } while (0)
50 
51 /* minimum unit size, also is the maximum supported allocation size */
52 #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)
53 
54 /*
55  * Percpu allocator can serve percpu allocations before slab is
56  * initialized which allows slab to depend on the percpu allocator.
57  * The following two parameters decide how much resource to
58  * preallocate for this. Keep PERCPU_DYNAMIC_RESERVE equal to or
59  * larger than PERCPU_DYNAMIC_EARLY_SIZE.
60  */
61 #define PERCPU_DYNAMIC_EARLY_SLOTS 128
62 #define PERCPU_DYNAMIC_EARLY_SIZE (12 << 10)
63 
64 /*
65  * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
66  * back on the first chunk for dynamic percpu allocation if arch is
67  * manually allocating and mapping it for faster access (as a part of
68  * large page mapping for example).
69  *
70  * The following values give between one and two pages of free space
71  * after typical minimal boot (2-way SMP, single disk and NIC) with
72  * both defconfig and a distro config on x86_64 and 32. More
73  * intelligent way to determine this would be nice.
74  */
75 #if BITS_PER_LONG > 32
76 #define PERCPU_DYNAMIC_RESERVE (20 << 10)
77 #else
78 #define PERCPU_DYNAMIC_RESERVE (12 << 10)
79 #endif
80 
81 extern void *pcpu_base_addr;
82 extern const unsigned long *pcpu_unit_offsets;
83 
84 struct pcpu_group_info {
85  int nr_units; /* aligned # of units */
86  unsigned long base_offset; /* base address offset */
87  unsigned int *cpu_map; /* unit->cpu map, empty
88  * entries contain NR_CPUS */
89 };
90 
91 struct pcpu_alloc_info {
92  size_t static_size;
93  size_t reserved_size;
94  size_t dyn_size;
95  size_t unit_size;
96  size_t atom_size;
97  size_t alloc_size;
98  size_t __ai_size; /* internal, don't use */
99  int nr_groups; /* 0 if grouping unnecessary */
100  struct pcpu_group_info groups[];
101 };
102 
103 enum pcpu_fc {
104  PCPU_FC_AUTO,
105  PCPU_FC_EMBED,
106  PCPU_FC_PAGE,
107 
108  PCPU_FC_NR,
109 };
110 extern const char * const pcpu_fc_names[PCPU_FC_NR];
111 
112 extern enum pcpu_fc pcpu_chosen_fc;
113 
114 typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size,
115  size_t align);
116 typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size);
117 typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);
118 typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to);
119 
120 extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
121  int nr_units);
122 extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai);
123 
124 extern int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
125  void *base_addr);
126 
127 #ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK
128 extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
129  size_t atom_size,
130  pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
131  pcpu_fc_alloc_fn_t alloc_fn,
132  pcpu_fc_free_fn_t free_fn);
133 #endif
134 
135 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
136 extern int __init pcpu_page_first_chunk(size_t reserved_size,
137  pcpu_fc_alloc_fn_t alloc_fn,
138  pcpu_fc_free_fn_t free_fn,
139  pcpu_fc_populate_pte_fn_t populate_pte_fn);
140 #endif
141 
142 /*
143  * Use this to get to a cpu's version of the per-cpu object
144  * dynamically allocated. Non-atomic access to the current CPU's
145  * version should probably be combined with get_cpu()/put_cpu().
146  */
147 #ifdef CONFIG_SMP
148 #define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
149 #else
150 #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })
151 #endif
152 
153 extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
154 extern bool is_kernel_percpu_address(unsigned long addr);
155 
156 #if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA)
157 extern void __init setup_per_cpu_areas(void);
158 #endif
159 extern void __init percpu_init_late(void);
160 
161 extern void __percpu *__alloc_percpu(size_t size, size_t align);
162 extern void free_percpu(void __percpu *__pdata);
163 extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
164 
165 #define alloc_percpu(type) \
166  (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
167 
168 /*
169  * Branching function to split up a function into a set of functions that
170  * are called for different scalar sizes of the objects handled.
171  */
172 
173 extern void __bad_size_call_parameter(void);
174 
175 #define __pcpu_size_call_return(stem, variable) \
176 ({ typeof(variable) pscr_ret__; \
177  __verify_pcpu_ptr(&(variable)); \
178  switch(sizeof(variable)) { \
179  case 1: pscr_ret__ = stem##1(variable);break; \
180  case 2: pscr_ret__ = stem##2(variable);break; \
181  case 4: pscr_ret__ = stem##4(variable);break; \
182  case 8: pscr_ret__ = stem##8(variable);break; \
183  default: \
184  __bad_size_call_parameter();break; \
185  } \
186  pscr_ret__; \
187 })
188 
189 #define __pcpu_size_call_return2(stem, variable, ...) \
190 ({ \
191  typeof(variable) pscr2_ret__; \
192  __verify_pcpu_ptr(&(variable)); \
193  switch(sizeof(variable)) { \
194  case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
195  case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
196  case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
197  case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
198  default: \
199  __bad_size_call_parameter(); break; \
200  } \
201  pscr2_ret__; \
202 })
203 
204 /*
205  * Special handling for cmpxchg_double. cmpxchg_double is passed two
206  * percpu variables. The first has to be aligned to a double word
207  * boundary and the second has to follow directly thereafter.
208  * We enforce this on all architectures even if they don't support
209  * a double cmpxchg instruction, since it's a cheap requirement, and it
210  * avoids breaking the requirement for architectures with the instruction.
211  */
212 #define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
213 ({ \
214  bool pdcrb_ret__; \
215  __verify_pcpu_ptr(&pcp1); \
216  BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
217  VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \
218  VM_BUG_ON((unsigned long)(&pcp2) != \
219  (unsigned long)(&pcp1) + sizeof(pcp1)); \
220  switch(sizeof(pcp1)) { \
221  case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
222  case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
223  case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
224  case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
225  default: \
226  __bad_size_call_parameter(); break; \
227  } \
228  pdcrb_ret__; \
229 })
230 
231 #define __pcpu_size_call(stem, variable, ...) \
232 do { \
233  __verify_pcpu_ptr(&(variable)); \
234  switch(sizeof(variable)) { \
235  case 1: stem##1(variable, __VA_ARGS__);break; \
236  case 2: stem##2(variable, __VA_ARGS__);break; \
237  case 4: stem##4(variable, __VA_ARGS__);break; \
238  case 8: stem##8(variable, __VA_ARGS__);break; \
239  default: \
240  __bad_size_call_parameter();break; \
241  } \
242 } while (0)
243 
244 /*
245  * Optimized manipulation for memory allocated through the per cpu
246  * allocator or for addresses of per cpu variables.
247  *
248  * These operation guarantee exclusivity of access for other operations
249  * on the *same* processor. The assumption is that per cpu data is only
250  * accessed by a single processor instance (the current one).
251  *
252  * The first group is used for accesses that must be done in a
253  * preemption safe way since we know that the context is not preempt
254  * safe. Interrupts may occur. If the interrupt modifies the variable
255  * too then RMW actions will not be reliable.
256  *
257  * The arch code can provide optimized functions in two ways:
258  *
259  * 1. Override the function completely. F.e. define this_cpu_add().
260  * The arch must then ensure that the various scalar format passed
261  * are handled correctly.
262  *
263  * 2. Provide functions for certain scalar sizes. F.e. provide
264  * this_cpu_add_2() to provide per cpu atomic operations for 2 byte
265  * sized RMW actions. If arch code does not provide operations for
266  * a scalar size then the fallback in the generic code will be
267  * used.
268  */
269 
270 #define _this_cpu_generic_read(pcp) \
271 ({ typeof(pcp) ret__; \
272  preempt_disable(); \
273  ret__ = *this_cpu_ptr(&(pcp)); \
274  preempt_enable(); \
275  ret__; \
276 })
277 
278 #ifndef this_cpu_read
279 # ifndef this_cpu_read_1
280 # define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
281 # endif
282 # ifndef this_cpu_read_2
283 # define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
284 # endif
285 # ifndef this_cpu_read_4
286 # define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
287 # endif
288 # ifndef this_cpu_read_8
289 # define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
290 # endif
291 # define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
292 #endif
293 
294 #define _this_cpu_generic_to_op(pcp, val, op) \
295 do { \
296  unsigned long flags; \
297  raw_local_irq_save(flags); \
298  *__this_cpu_ptr(&(pcp)) op val; \
299  raw_local_irq_restore(flags); \
300 } while (0)
301 
302 #ifndef this_cpu_write
303 # ifndef this_cpu_write_1
304 # define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
305 # endif
306 # ifndef this_cpu_write_2
307 # define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
308 # endif
309 # ifndef this_cpu_write_4
310 # define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
311 # endif
312 # ifndef this_cpu_write_8
313 # define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
314 # endif
315 # define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
316 #endif
317 
318 #ifndef this_cpu_add
319 # ifndef this_cpu_add_1
320 # define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
321 # endif
322 # ifndef this_cpu_add_2
323 # define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
324 # endif
325 # ifndef this_cpu_add_4
326 # define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
327 # endif
328 # ifndef this_cpu_add_8
329 # define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
330 # endif
331 # define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
332 #endif
333 
334 #ifndef this_cpu_sub
335 # define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(val))
336 #endif
337 
338 #ifndef this_cpu_inc
339 # define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
340 #endif
341 
342 #ifndef this_cpu_dec
343 # define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
344 #endif
345 
346 #ifndef this_cpu_and
347 # ifndef this_cpu_and_1
348 # define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
349 # endif
350 # ifndef this_cpu_and_2
351 # define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
352 # endif
353 # ifndef this_cpu_and_4
354 # define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
355 # endif
356 # ifndef this_cpu_and_8
357 # define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
358 # endif
359 # define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
360 #endif
361 
362 #ifndef this_cpu_or
363 # ifndef this_cpu_or_1
364 # define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
365 # endif
366 # ifndef this_cpu_or_2
367 # define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
368 # endif
369 # ifndef this_cpu_or_4
370 # define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
371 # endif
372 # ifndef this_cpu_or_8
373 # define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
374 # endif
375 # define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
376 #endif
377 
378 #ifndef this_cpu_xor
379 # ifndef this_cpu_xor_1
380 # define this_cpu_xor_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
381 # endif
382 # ifndef this_cpu_xor_2
383 # define this_cpu_xor_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
384 # endif
385 # ifndef this_cpu_xor_4
386 # define this_cpu_xor_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
387 # endif
388 # ifndef this_cpu_xor_8
389 # define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
390 # endif
391 # define this_cpu_xor(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
392 #endif
393 
394 #define _this_cpu_generic_add_return(pcp, val) \
395 ({ \
396  typeof(pcp) ret__; \
397  unsigned long flags; \
398  raw_local_irq_save(flags); \
399  __this_cpu_add(pcp, val); \
400  ret__ = __this_cpu_read(pcp); \
401  raw_local_irq_restore(flags); \
402  ret__; \
403 })
404 
405 #ifndef this_cpu_add_return
406 # ifndef this_cpu_add_return_1
407 # define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val)
408 # endif
409 # ifndef this_cpu_add_return_2
410 # define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val)
411 # endif
412 # ifndef this_cpu_add_return_4
413 # define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val)
414 # endif
415 # ifndef this_cpu_add_return_8
416 # define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val)
417 # endif
418 # define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
419 #endif
420 
421 #define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(val))
422 #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
423 #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
424 
425 #define _this_cpu_generic_xchg(pcp, nval) \
426 ({ typeof(pcp) ret__; \
427  unsigned long flags; \
428  raw_local_irq_save(flags); \
429  ret__ = __this_cpu_read(pcp); \
430  __this_cpu_write(pcp, nval); \
431  raw_local_irq_restore(flags); \
432  ret__; \
433 })
434 
435 #ifndef this_cpu_xchg
436 # ifndef this_cpu_xchg_1
437 # define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
438 # endif
439 # ifndef this_cpu_xchg_2
440 # define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
441 # endif
442 # ifndef this_cpu_xchg_4
443 # define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
444 # endif
445 # ifndef this_cpu_xchg_8
446 # define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
447 # endif
448 # define this_cpu_xchg(pcp, nval) \
449  __pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval)
450 #endif
451 
452 #define _this_cpu_generic_cmpxchg(pcp, oval, nval) \
453 ({ \
454  typeof(pcp) ret__; \
455  unsigned long flags; \
456  raw_local_irq_save(flags); \
457  ret__ = __this_cpu_read(pcp); \
458  if (ret__ == (oval)) \
459  __this_cpu_write(pcp, nval); \
460  raw_local_irq_restore(flags); \
461  ret__; \
462 })
463 
464 #ifndef this_cpu_cmpxchg
465 # ifndef this_cpu_cmpxchg_1
466 # define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
467 # endif
468 # ifndef this_cpu_cmpxchg_2
469 # define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
470 # endif
471 # ifndef this_cpu_cmpxchg_4
472 # define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
473 # endif
474 # ifndef this_cpu_cmpxchg_8
475 # define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
476 # endif
477 # define this_cpu_cmpxchg(pcp, oval, nval) \
478  __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
479 #endif
480 
481 /*
482  * cmpxchg_double replaces two adjacent scalars at once. The first
483  * two parameters are per cpu variables which have to be of the same
484  * size. A truth value is returned to indicate success or failure
485  * (since a double register result is difficult to handle). There is
486  * very limited hardware support for these operations, so only certain
487  * sizes may work.
488  */
489 #define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
490 ({ \
491  int ret__; \
492  unsigned long flags; \
493  raw_local_irq_save(flags); \
494  ret__ = __this_cpu_generic_cmpxchg_double(pcp1, pcp2, \
495  oval1, oval2, nval1, nval2); \
496  raw_local_irq_restore(flags); \
497  ret__; \
498 })
499 
500 #ifndef this_cpu_cmpxchg_double
501 # ifndef this_cpu_cmpxchg_double_1
502 # define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
503  _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
504 # endif
505 # ifndef this_cpu_cmpxchg_double_2
506 # define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
507  _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
508 # endif
509 # ifndef this_cpu_cmpxchg_double_4
510 # define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
511  _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
512 # endif
513 # ifndef this_cpu_cmpxchg_double_8
514 # define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
515  _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
516 # endif
517 # define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
518  __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
519 #endif
520 
521 /*
522  * Generic percpu operations for context that are safe from preemption/interrupts.
523  * Either we do not care about races or the caller has the
524  * responsibility of handling preemption/interrupt issues. Arch code can still
525  * override these instructions since the arch per cpu code may be more
526  * efficient and may actually get race freeness for free (that is the
527  * case for x86 for example).
528  *
529  * If there is no other protection through preempt disable and/or
530  * disabling interupts then one of these RMW operations can show unexpected
531  * behavior because the execution thread was rescheduled on another processor
532  * or an interrupt occurred and the same percpu variable was modified from
533  * the interrupt context.
534  */
535 #ifndef __this_cpu_read
536 # ifndef __this_cpu_read_1
537 # define __this_cpu_read_1(pcp) (*__this_cpu_ptr(&(pcp)))
538 # endif
539 # ifndef __this_cpu_read_2
540 # define __this_cpu_read_2(pcp) (*__this_cpu_ptr(&(pcp)))
541 # endif
542 # ifndef __this_cpu_read_4
543 # define __this_cpu_read_4(pcp) (*__this_cpu_ptr(&(pcp)))
544 # endif
545 # ifndef __this_cpu_read_8
546 # define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp)))
547 # endif
548 # define __this_cpu_read(pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp))
549 #endif
550 
551 #define __this_cpu_generic_to_op(pcp, val, op) \
552 do { \
553  *__this_cpu_ptr(&(pcp)) op val; \
554 } while (0)
555 
556 #ifndef __this_cpu_write
557 # ifndef __this_cpu_write_1
558 # define __this_cpu_write_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
559 # endif
560 # ifndef __this_cpu_write_2
561 # define __this_cpu_write_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
562 # endif
563 # ifndef __this_cpu_write_4
564 # define __this_cpu_write_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
565 # endif
566 # ifndef __this_cpu_write_8
567 # define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
568 # endif
569 # define __this_cpu_write(pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val))
570 #endif
571 
572 #ifndef __this_cpu_add
573 # ifndef __this_cpu_add_1
574 # define __this_cpu_add_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
575 # endif
576 # ifndef __this_cpu_add_2
577 # define __this_cpu_add_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
578 # endif
579 # ifndef __this_cpu_add_4
580 # define __this_cpu_add_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
581 # endif
582 # ifndef __this_cpu_add_8
583 # define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
584 # endif
585 # define __this_cpu_add(pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val))
586 #endif
587 
588 #ifndef __this_cpu_sub
589 # define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(val))
590 #endif
591 
592 #ifndef __this_cpu_inc
593 # define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
594 #endif
595 
596 #ifndef __this_cpu_dec
597 # define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
598 #endif
599 
600 #ifndef __this_cpu_and
601 # ifndef __this_cpu_and_1
602 # define __this_cpu_and_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
603 # endif
604 # ifndef __this_cpu_and_2
605 # define __this_cpu_and_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
606 # endif
607 # ifndef __this_cpu_and_4
608 # define __this_cpu_and_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
609 # endif
610 # ifndef __this_cpu_and_8
611 # define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
612 # endif
613 # define __this_cpu_and(pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val))
614 #endif
615 
616 #ifndef __this_cpu_or
617 # ifndef __this_cpu_or_1
618 # define __this_cpu_or_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
619 # endif
620 # ifndef __this_cpu_or_2
621 # define __this_cpu_or_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
622 # endif
623 # ifndef __this_cpu_or_4
624 # define __this_cpu_or_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
625 # endif
626 # ifndef __this_cpu_or_8
627 # define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
628 # endif
629 # define __this_cpu_or(pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val))
630 #endif
631 
632 #ifndef __this_cpu_xor
633 # ifndef __this_cpu_xor_1
634 # define __this_cpu_xor_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
635 # endif
636 # ifndef __this_cpu_xor_2
637 # define __this_cpu_xor_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
638 # endif
639 # ifndef __this_cpu_xor_4
640 # define __this_cpu_xor_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
641 # endif
642 # ifndef __this_cpu_xor_8
643 # define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
644 # endif
645 # define __this_cpu_xor(pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val))
646 #endif
647 
648 #define __this_cpu_generic_add_return(pcp, val) \
649 ({ \
650  __this_cpu_add(pcp, val); \
651  __this_cpu_read(pcp); \
652 })
653 
654 #ifndef __this_cpu_add_return
655 # ifndef __this_cpu_add_return_1
656 # define __this_cpu_add_return_1(pcp, val) __this_cpu_generic_add_return(pcp, val)
657 # endif
658 # ifndef __this_cpu_add_return_2
659 # define __this_cpu_add_return_2(pcp, val) __this_cpu_generic_add_return(pcp, val)
660 # endif
661 # ifndef __this_cpu_add_return_4
662 # define __this_cpu_add_return_4(pcp, val) __this_cpu_generic_add_return(pcp, val)
663 # endif
664 # ifndef __this_cpu_add_return_8
665 # define __this_cpu_add_return_8(pcp, val) __this_cpu_generic_add_return(pcp, val)
666 # endif
667 # define __this_cpu_add_return(pcp, val) \
668  __pcpu_size_call_return2(__this_cpu_add_return_, pcp, val)
669 #endif
670 
671 #define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(val))
672 #define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
673 #define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
674 
675 #define __this_cpu_generic_xchg(pcp, nval) \
676 ({ typeof(pcp) ret__; \
677  ret__ = __this_cpu_read(pcp); \
678  __this_cpu_write(pcp, nval); \
679  ret__; \
680 })
681 
682 #ifndef __this_cpu_xchg
683 # ifndef __this_cpu_xchg_1
684 # define __this_cpu_xchg_1(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
685 # endif
686 # ifndef __this_cpu_xchg_2
687 # define __this_cpu_xchg_2(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
688 # endif
689 # ifndef __this_cpu_xchg_4
690 # define __this_cpu_xchg_4(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
691 # endif
692 # ifndef __this_cpu_xchg_8
693 # define __this_cpu_xchg_8(pcp, nval) __this_cpu_generic_xchg(pcp, nval)
694 # endif
695 # define __this_cpu_xchg(pcp, nval) \
696  __pcpu_size_call_return2(__this_cpu_xchg_, (pcp), nval)
697 #endif
698 
699 #define __this_cpu_generic_cmpxchg(pcp, oval, nval) \
700 ({ \
701  typeof(pcp) ret__; \
702  ret__ = __this_cpu_read(pcp); \
703  if (ret__ == (oval)) \
704  __this_cpu_write(pcp, nval); \
705  ret__; \
706 })
707 
708 #ifndef __this_cpu_cmpxchg
709 # ifndef __this_cpu_cmpxchg_1
710 # define __this_cpu_cmpxchg_1(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
711 # endif
712 # ifndef __this_cpu_cmpxchg_2
713 # define __this_cpu_cmpxchg_2(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
714 # endif
715 # ifndef __this_cpu_cmpxchg_4
716 # define __this_cpu_cmpxchg_4(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
717 # endif
718 # ifndef __this_cpu_cmpxchg_8
719 # define __this_cpu_cmpxchg_8(pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)
720 # endif
721 # define __this_cpu_cmpxchg(pcp, oval, nval) \
722  __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval)
723 #endif
724 
725 #define __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
726 ({ \
727  int __ret = 0; \
728  if (__this_cpu_read(pcp1) == (oval1) && \
729  __this_cpu_read(pcp2) == (oval2)) { \
730  __this_cpu_write(pcp1, (nval1)); \
731  __this_cpu_write(pcp2, (nval2)); \
732  __ret = 1; \
733  } \
734  (__ret); \
735 })
736 
737 #ifndef __this_cpu_cmpxchg_double
738 # ifndef __this_cpu_cmpxchg_double_1
739 # define __this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
740  __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
741 # endif
742 # ifndef __this_cpu_cmpxchg_double_2
743 # define __this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
744  __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
745 # endif
746 # ifndef __this_cpu_cmpxchg_double_4
747 # define __this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
748  __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
749 # endif
750 # ifndef __this_cpu_cmpxchg_double_8
751 # define __this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
752  __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
753 # endif
754 # define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
755  __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
756 #endif
757 
758 #endif /* __LINUX_PERCPU_H */
Generated on Thu Jan 10 2013 12:50:10 for Linux Kernel by   doxygen 1.8.2