Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
perf_event.h
Go to the documentation of this file.
1 /*
2  * Performance events x86 architecture header
3  *
4  * Copyright (C) 2008 Thomas Gleixner <[email protected]>
5  * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  * Copyright (C) 2009 Jaswinder Singh Rajput
7  * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8  * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <[email protected]>
9  * Copyright (C) 2009 Intel Corporation, <[email protected]>
10  * Copyright (C) 2009 Google, Inc., Stephane Eranian
11  *
12  * For licencing details see kernel-base/COPYING
13  */
14 
15 #include <linux/perf_event.h>
16 
17 #if 0
18 #undef wrmsrl
19 #define wrmsrl(msr, val) \
20 do { \
21  unsigned int _msr = (msr); \
22  u64 _val = (val); \
23  trace_printk("wrmsrl(%x, %Lx)\n", (unsigned int)(_msr), \
24  (unsigned long long)(_val)); \
25  native_write_msr((_msr), (u32)(_val), (u32)(_val >> 32)); \
26 } while (0)
27 #endif
28 
29 /*
30  * | NHM/WSM | SNB |
31  * register -------------------------------
32  * | HT | no HT | HT | no HT |
33  *-----------------------------------------
34  * offcore | core | core | cpu | core |
35  * lbr_sel | core | core | cpu | core |
36  * ld_lat | cpu | core | cpu | core |
37  *-----------------------------------------
38  *
39  * Given that there is a small number of shared regs,
40  * we can pre-allocate their slot in the per-cpu
41  * per-core reg tables.
42  */
44  EXTRA_REG_NONE = -1, /* not used */
45 
46  EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */
47  EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */
48  EXTRA_REG_LBR = 2, /* lbr_select */
49 
50  EXTRA_REG_MAX /* number of entries needed */
51 };
52 
54  union {
57  };
60  int weight;
61  int overlap;
62 };
63 
64 struct amd_nb {
65  int nb_id; /* NorthBridge id */
66  int refcnt; /* reference count */
69 };
70 
71 /* The maximal number of PEBS events: */
72 #define MAX_PEBS_EVENTS 8
73 
74 /*
75  * A debug store configuration.
76  *
77  * We only support architectures that use 64bit fields.
78  */
79 struct debug_store {
89 };
90 
91 /*
92  * Per register state.
93  */
94 struct er_account {
95  raw_spinlock_t lock; /* per-core: protect structure */
96  u64 config; /* extra MSR config */
97  u64 reg; /* extra MSR number */
98  atomic_t ref; /* reference count */
99 };
100 
101 /*
102  * Per core/cpu state
103  *
104  * Used to coordinate shared registers between HT threads or
105  * among events on a single PMU.
106  */
109  int refcnt; /* per-core: #HT threads */
110  unsigned core_id; /* per-core: core id */
111 };
112 
113 #define MAX_LBR_ENTRIES 16
114 
115 struct cpu_hw_events {
116  /*
117  * Generic x86 PMC bits
118  */
119  struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
120  unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
122  int enabled;
123 
124  int n_events;
125  int n_added;
126  int n_txn;
127  int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
129  struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
130 
131  unsigned int group_flag;
132  int is_fake;
133 
134  /*
135  * Intel DebugStore bits
136  */
137  struct debug_store *ds;
139 
140  /*
141  * Intel LBR bits
142  */
144  void *lbr_context;
149 
150  /*
151  * Intel host/guest exclude bits
152  */
155  struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX];
156 
157  /*
158  * manage shared (per-core, per-cpu) registers
159  * used on Intel NHM/WSM/SNB
160  */
162 
163  /*
164  * AMD specific bits
165  */
166  struct amd_nb *amd_nb;
167  /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
169 
171 };
172 
173 #define __EVENT_CONSTRAINT(c, n, m, w, o) {\
174  { .idxmsk64 = (n) }, \
175  .code = (c), \
176  .cmask = (m), \
177  .weight = (w), \
178  .overlap = (o), \
179 }
180 
181 #define EVENT_CONSTRAINT(c, n, m) \
182  __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0)
183 
184 /*
185  * The overlap flag marks event constraints with overlapping counter
186  * masks. This is the case if the counter mask of such an event is not
187  * a subset of any other counter mask of a constraint with an equal or
188  * higher weight, e.g.:
189  *
190  * c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
191  * c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
192  * c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
193  *
194  * The event scheduler may not select the correct counter in the first
195  * cycle because it needs to know which subsequent events will be
196  * scheduled. It may fail to schedule the events then. So we set the
197  * overlap flag for such constraints to give the scheduler a hint which
198  * events to select for counter rescheduling.
199  *
200  * Care must be taken as the rescheduling algorithm is O(n!) which
201  * will increase scheduling cycles for an over-commited system
202  * dramatically. The number of such EVENT_CONSTRAINT_OVERLAP() macros
203  * and its counter masks must be kept at a minimum.
204  */
205 #define EVENT_CONSTRAINT_OVERLAP(c, n, m) \
206  __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1)
207 
208 /*
209  * Constraint on the Event code.
210  */
211 #define INTEL_EVENT_CONSTRAINT(c, n) \
212  EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
213 
214 /*
215  * Constraint on the Event code + UMask + fixed-mask
216  *
217  * filter mask to validate fixed counter events.
218  * the following filters disqualify for fixed counters:
219  * - inv
220  * - edge
221  * - cnt-mask
222  * The other filters are supported by fixed counters.
223  * The any-thread option is supported starting with v3.
224  */
225 #define FIXED_EVENT_CONSTRAINT(c, n) \
226  EVENT_CONSTRAINT(c, (1ULL << (32+n)), X86_RAW_EVENT_MASK)
227 
228 /*
229  * Constraint on the Event code + UMask
230  */
231 #define INTEL_UEVENT_CONSTRAINT(c, n) \
232  EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
233 
234 #define EVENT_CONSTRAINT_END \
235  EVENT_CONSTRAINT(0, 0, 0)
236 
237 #define for_each_event_constraint(e, c) \
238  for ((e) = (c); (e)->weight; (e)++)
239 
240 /*
241  * Extra registers for specific events.
242  *
243  * Some events need large masks and require external MSRs.
244  * Those extra MSRs end up being shared for all events on
245  * a PMU and sometimes between PMU of sibling HT threads.
246  * In either case, the kernel needs to handle conflicting
247  * accesses to those extra, shared, regs. The data structure
248  * to manage those registers is stored in cpu_hw_event.
249  */
250 struct extra_reg {
251  unsigned int event;
252  unsigned int msr;
255  int idx; /* per_xxx->regs[] reg index */
256 };
257 
258 #define EVENT_EXTRA_REG(e, ms, m, vm, i) { \
259  .event = (e), \
260  .msr = (ms), \
261  .config_mask = (m), \
262  .valid_mask = (vm), \
263  .idx = EXTRA_REG_##i \
264  }
265 
266 #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \
267  EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
268 
269 #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
270 
272  struct {
278  };
280 };
281 
285 };
286 
288  struct {
289  u64 event:8,
290  umask:8,
291  usr:1,
292  os:1,
293  edge:1,
294  pc:1,
295  interrupt:1,
296  __reserved1:1,
297  en:1,
298  inv:1,
299  cmask:8,
300  event2:4,
301  __reserved2:4,
302  go:1,
303  ho:1;
304  } bits;
306 };
307 
308 #define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
309 
310 /*
311  * struct x86_pmu - generic x86 pmu
312  */
313 struct x86_pmu {
314  /*
315  * Generic x86 PMC bits
316  */
317  const char *name;
318  int version;
319  int (*handle_irq)(struct pt_regs *);
321  void (*enable_all)(int added);
322  void (*enable)(struct perf_event *);
323  void (*disable)(struct perf_event *);
325  int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
326  unsigned eventsel;
327  unsigned perfctr;
334  union {
335  unsigned long events_maskl;
337  };
339  int apic;
341  struct event_constraint *
342  (*get_event_constraints)(struct cpu_hw_events *cpuc,
343  struct perf_event *event);
344 
346  struct perf_event *event);
350 
351  /*
352  * sysfs attrs
353  */
356 
357  /*
358  * CPU Hotplug hooks
359  */
360  int (*cpu_prepare)(int cpu);
362  void (*cpu_dying)(int cpu);
363  void (*cpu_dead)(int cpu);
364 
367 
368  /*
369  * Intel Arch Perfmon v2+
370  */
373 
374  /*
375  * Intel DebugStore bits
376  */
377  unsigned int bts :1,
378  bts_active :1,
379  pebs :1,
380  pebs_active :1,
381  pebs_broken :1;
387 
388  /*
389  * Intel LBR
390  */
391  unsigned long lbr_tos, lbr_from, lbr_to; /* MSR base regs */
392  int lbr_nr; /* hardware stack size */
393  u64 lbr_sel_mask; /* LBR_SELECT valid bits */
394  const int *lbr_sel_map; /* lbr_select mappings */
395 
396  /*
397  * Extra registers for events
398  */
400  unsigned int er_flags;
401 
402  /*
403  * Intel host/guest support (KVM)
404  */
405  struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr);
406 };
407 
408 #define x86_add_quirk(func_) \
409 do { \
410  static struct x86_pmu_quirk __quirk __initdata = { \
411  .func = func_, \
412  }; \
413  __quirk.next = x86_pmu.quirks; \
414  x86_pmu.quirks = &__quirk; \
415 } while (0)
416 
417 #define ERF_NO_HT_SHARING 1
418 #define ERF_HAS_RSP_1 2
419 
420 extern struct x86_pmu x86_pmu __read_mostly;
421 
423 
425 
426 /*
427  * Generalized hw caching related hw_event table, filled
428  * in on a per model basis. A value of 0 means
429  * 'not supported', -1 means 'hw_event makes no sense on
430  * this CPU', any other value means the raw hw_event
431  * ID.
432  */
433 
434 #define C(x) PERF_COUNT_HW_CACHE_##x
435 
444 
446 
447 static inline int x86_pmu_addr_offset(int index)
448 {
449  int offset;
450 
451  /* offset = X86_FEATURE_PERFCTR_CORE ? index << 1 : index */
453  "shll $1, %%eax",
455  "=a" (offset),
456  "a" (index));
457 
458  return offset;
459 }
460 
461 static inline unsigned int x86_pmu_config_addr(int index)
462 {
463  return x86_pmu.eventsel + x86_pmu_addr_offset(index);
464 }
465 
466 static inline unsigned int x86_pmu_event_addr(int index)
467 {
468  return x86_pmu.perfctr + x86_pmu_addr_offset(index);
469 }
470 
471 int x86_setup_perfctr(struct perf_event *event);
472 
473 int x86_pmu_hw_config(struct perf_event *event);
474 
475 void x86_pmu_disable_all(void);
476 
477 static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
478  u64 enable_mask)
479 {
481 
482  if (hwc->extra_reg.reg)
483  wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
484  wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
485 }
486 
487 void x86_pmu_enable_all(int added);
488 
489 int perf_assign_events(struct event_constraint **constraints, int n,
490  int wmin, int wmax, int *assign);
491 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
492 
493 void x86_pmu_stop(struct perf_event *event, int flags);
494 
495 static inline void x86_pmu_disable_event(struct perf_event *event)
496 {
497  struct hw_perf_event *hwc = &event->hw;
498 
499  wrmsrl(hwc->config_base, hwc->config);
500 }
501 
503 
504 int x86_pmu_handle_irq(struct pt_regs *regs);
505 
506 extern struct event_constraint emptyconstraint;
507 
508 extern struct event_constraint unconstrained;
509 
510 static inline bool kernel_ip(unsigned long ip)
511 {
512 #ifdef CONFIG_X86_32
513  return ip > PAGE_OFFSET;
514 #else
515  return (long)ip < 0;
516 #endif
517 }
518 
519 /*
520  * Not all PMUs provide the right context information to place the reported IP
521  * into full context. Specifically segment registers are typically not
522  * supplied.
523  *
524  * Assuming the address is a linear address (it is for IBS), we fake the CS and
525  * vm86 mode using the known zero-based code segment and 'fix up' the registers
526  * to reflect this.
527  *
528  * Intel PEBS/LBR appear to typically provide the effective address, nothing
529  * much we can do about that but pray and treat it like a linear address.
530  */
531 static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
532 {
533  regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
534  if (regs->flags & X86_VM_MASK)
535  regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
536  regs->ip = ip;
537 }
538 
539 #ifdef CONFIG_CPU_SUP_AMD
540 
541 int amd_pmu_init(void);
542 
543 #else /* CONFIG_CPU_SUP_AMD */
544 
545 static inline int amd_pmu_init(void)
546 {
547  return 0;
548 }
549 
550 #endif /* CONFIG_CPU_SUP_AMD */
551 
552 #ifdef CONFIG_CPU_SUP_INTEL
553 
555 
556 struct event_constraint *
558 
560 
561 int intel_pmu_init(void);
562 
563 void init_debug_store_on_cpu(int cpu);
564 
565 void fini_debug_store_on_cpu(int cpu);
566 
567 void release_ds_buffers(void);
568 
569 void reserve_ds_buffers(void);
570 
571 extern struct event_constraint bts_constraint;
572 
574 
575 void intel_pmu_disable_bts(void);
576 
578 
580 
582 
584 
586 
588 
590 
592 
594 
596 
597 void intel_pmu_pebs_enable_all(void);
598 
599 void intel_pmu_pebs_disable_all(void);
600 
601 void intel_ds_init(void);
602 
603 void intel_pmu_lbr_reset(void);
604 
606 
608 
609 void intel_pmu_lbr_enable_all(void);
610 
611 void intel_pmu_lbr_disable_all(void);
612 
613 void intel_pmu_lbr_read(void);
614 
615 void intel_pmu_lbr_init_core(void);
616 
617 void intel_pmu_lbr_init_nhm(void);
618 
619 void intel_pmu_lbr_init_atom(void);
620 
621 void intel_pmu_lbr_init_snb(void);
622 
624 
625 int p4_pmu_init(void);
626 
627 int p6_pmu_init(void);
628 
629 int knc_pmu_init(void);
630 
631 #else /* CONFIG_CPU_SUP_INTEL */
632 
633 static inline void reserve_ds_buffers(void)
634 {
635 }
636 
637 static inline void release_ds_buffers(void)
638 {
639 }
640 
641 static inline int intel_pmu_init(void)
642 {
643  return 0;
644 }
645 
646 static inline struct intel_shared_regs *allocate_shared_regs(int cpu)
647 {
648  return NULL;
649 }
650 
651 #endif /* CONFIG_CPU_SUP_INTEL */