Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
op_model_amd.c
Go to the documentation of this file.
1 /*
2  * @file op_model_amd.c
3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
4  *
5  * @remark Copyright 2002-2009 OProfile authors
6  * @remark Read the file COPYING
7  *
8  * @author John Levon
9  * @author Philippe Elie
10  * @author Graydon Hoare
11  * @author Robert Richter <[email protected]>
12  * @author Barry Kasindorf <[email protected]>
13  * @author Jason Yeh <[email protected]>
14  * @author Suravee Suthikulpanit <[email protected]>
15  */
16 
17 #include <linux/oprofile.h>
18 #include <linux/device.h>
19 #include <linux/pci.h>
20 #include <linux/percpu.h>
21 
22 #include <asm/ptrace.h>
23 #include <asm/msr.h>
24 #include <asm/nmi.h>
25 #include <asm/apic.h>
26 #include <asm/processor.h>
27 #include <asm/cpufeature.h>
28 
29 #include "op_x86_model.h"
30 #include "op_counter.h"
31 
32 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
33 #define NUM_VIRT_COUNTERS 32
34 #else
35 #define NUM_VIRT_COUNTERS 0
36 #endif
37 
38 #define OP_EVENT_MASK 0x0FFF
39 #define OP_CTR_OVERFLOW (1ULL<<31)
40 
41 #define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
42 
43 static int num_counters;
44 static unsigned long reset_value[OP_MAX_COUNTER];
45 
46 #define IBS_FETCH_SIZE 6
47 #define IBS_OP_SIZE 12
48 
49 static u32 ibs_caps;
50 
51 struct ibs_config {
52  unsigned long op_enabled;
53  unsigned long fetch_enabled;
54  unsigned long max_cnt_fetch;
55  unsigned long max_cnt_op;
56  unsigned long rand_en;
57  unsigned long dispatched_ops;
58  unsigned long branch_target;
59 };
60 
61 struct ibs_state {
62  u64 ibs_op_ctl;
63  int branch_target;
64  unsigned long sample_size;
65 };
66 
67 static struct ibs_config ibs_config;
68 static struct ibs_state ibs_state;
69 
70 /*
71  * IBS randomization macros
72  */
73 #define IBS_RANDOM_BITS 12
74 #define IBS_RANDOM_MASK ((1ULL << IBS_RANDOM_BITS) - 1)
75 #define IBS_RANDOM_MAXCNT_OFFSET (1ULL << (IBS_RANDOM_BITS - 5))
76 
77 /*
78  * 16-bit Linear Feedback Shift Register (LFSR)
79  *
80  * 16 14 13 11
81  * Feedback polynomial = X + X + X + X + 1
82  */
83 static unsigned int lfsr_random(void)
84 {
85  static unsigned int lfsr_value = 0xF00D;
86  unsigned int bit;
87 
88  /* Compute next bit to shift in */
89  bit = ((lfsr_value >> 0) ^
90  (lfsr_value >> 2) ^
91  (lfsr_value >> 3) ^
92  (lfsr_value >> 5)) & 0x0001;
93 
94  /* Advance to next register value */
95  lfsr_value = (lfsr_value >> 1) | (bit << 15);
96 
97  return lfsr_value;
98 }
99 
100 /*
101  * IBS software randomization
102  *
103  * The IBS periodic op counter is randomized in software. The lower 12
104  * bits of the 20 bit counter are randomized. IbsOpCurCnt is
105  * initialized with a 12 bit random value.
106  */
107 static inline u64 op_amd_randomize_ibs_op(u64 val)
108 {
109  unsigned int random = lfsr_random();
110 
111  if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
112  /*
113  * Work around if the hw can not write to IbsOpCurCnt
114  *
115  * Randomize the lower 8 bits of the 16 bit
116  * IbsOpMaxCnt [15:0] value in the range of -128 to
117  * +127 by adding/subtracting an offset to the
118  * maximum count (IbsOpMaxCnt).
119  *
120  * To avoid over or underflows and protect upper bits
121  * starting at bit 16, the initial value for
122  * IbsOpMaxCnt must fit in the range from 0x0081 to
123  * 0xff80.
124  */
125  val += (s8)(random >> 4);
126  else
127  val |= (u64)(random & IBS_RANDOM_MASK) << 32;
128 
129  return val;
130 }
131 
132 static inline void
133 op_amd_handle_ibs(struct pt_regs * const regs,
134  struct op_msrs const * const msrs)
135 {
136  u64 val, ctl;
137  struct op_entry entry;
138 
139  if (!ibs_caps)
140  return;
141 
142  if (ibs_config.fetch_enabled) {
143  rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
144  if (ctl & IBS_FETCH_VAL) {
145  rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
146  oprofile_write_reserve(&entry, regs, val,
147  IBS_FETCH_CODE, IBS_FETCH_SIZE);
148  oprofile_add_data64(&entry, val);
149  oprofile_add_data64(&entry, ctl);
150  rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
151  oprofile_add_data64(&entry, val);
152  oprofile_write_commit(&entry);
153 
154  /* reenable the IRQ */
155  ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT);
156  ctl |= IBS_FETCH_ENABLE;
157  wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
158  }
159  }
160 
161  if (ibs_config.op_enabled) {
162  rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
163  if (ctl & IBS_OP_VAL) {
164  rdmsrl(MSR_AMD64_IBSOPRIP, val);
165  oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE,
166  ibs_state.sample_size);
167  oprofile_add_data64(&entry, val);
168  rdmsrl(MSR_AMD64_IBSOPDATA, val);
169  oprofile_add_data64(&entry, val);
170  rdmsrl(MSR_AMD64_IBSOPDATA2, val);
171  oprofile_add_data64(&entry, val);
172  rdmsrl(MSR_AMD64_IBSOPDATA3, val);
173  oprofile_add_data64(&entry, val);
174  rdmsrl(MSR_AMD64_IBSDCLINAD, val);
175  oprofile_add_data64(&entry, val);
176  rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
177  oprofile_add_data64(&entry, val);
178  if (ibs_state.branch_target) {
179  rdmsrl(MSR_AMD64_IBSBRTARGET, val);
180  oprofile_add_data(&entry, (unsigned long)val);
181  }
182  oprofile_write_commit(&entry);
183 
184  /* reenable the IRQ */
185  ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
186  wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
187  }
188  }
189 }
190 
191 static inline void op_amd_start_ibs(void)
192 {
193  u64 val;
194 
195  if (!ibs_caps)
196  return;
197 
198  memset(&ibs_state, 0, sizeof(ibs_state));
199 
200  /*
201  * Note: Since the max count settings may out of range we
202  * write back the actual used values so that userland can read
203  * it.
204  */
205 
206  if (ibs_config.fetch_enabled) {
207  val = ibs_config.max_cnt_fetch >> 4;
208  val = min(val, IBS_FETCH_MAX_CNT);
209  ibs_config.max_cnt_fetch = val << 4;
210  val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
211  val |= IBS_FETCH_ENABLE;
212  wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
213  }
214 
215  if (ibs_config.op_enabled) {
216  val = ibs_config.max_cnt_op >> 4;
217  if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
218  /*
219  * IbsOpCurCnt not supported. See
220  * op_amd_randomize_ibs_op() for details.
221  */
222  val = clamp(val, 0x0081ULL, 0xFF80ULL);
223  ibs_config.max_cnt_op = val << 4;
224  } else {
225  /*
226  * The start value is randomized with a
227  * positive offset, we need to compensate it
228  * with the half of the randomized range. Also
229  * avoid underflows.
230  */
231  val += IBS_RANDOM_MAXCNT_OFFSET;
232  if (ibs_caps & IBS_CAPS_OPCNTEXT)
233  val = min(val, IBS_OP_MAX_CNT_EXT);
234  else
235  val = min(val, IBS_OP_MAX_CNT);
236  ibs_config.max_cnt_op =
237  (val - IBS_RANDOM_MAXCNT_OFFSET) << 4;
238  }
239  val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT);
240  val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
241  val |= IBS_OP_ENABLE;
242  ibs_state.ibs_op_ctl = val;
243  ibs_state.sample_size = IBS_OP_SIZE;
244  if (ibs_config.branch_target) {
245  ibs_state.branch_target = 1;
246  ibs_state.sample_size++;
247  }
248  val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
249  wrmsrl(MSR_AMD64_IBSOPCTL, val);
250  }
251 }
252 
253 static void op_amd_stop_ibs(void)
254 {
255  if (!ibs_caps)
256  return;
257 
258  if (ibs_config.fetch_enabled)
259  /* clear max count and enable */
260  wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
261 
262  if (ibs_config.op_enabled)
263  /* clear max count and enable */
264  wrmsrl(MSR_AMD64_IBSOPCTL, 0);
265 }
266 
267 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
268 
269 static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
270  struct op_msrs const * const msrs)
271 {
272  u64 val;
273  int i;
274 
275  /* enable active counters */
276  for (i = 0; i < num_counters; ++i) {
277  int virt = op_x86_phys_to_virt(i);
278  if (!reset_value[virt])
279  continue;
280  rdmsrl(msrs->controls[i].addr, val);
281  val &= model->reserved;
282  val |= op_x86_get_ctrl(model, &counter_config[virt]);
283  wrmsrl(msrs->controls[i].addr, val);
284  }
285 }
286 
287 #endif
288 
289 /* functions for op_amd_spec */
290 
291 static void op_amd_shutdown(struct op_msrs const * const msrs)
292 {
293  int i;
294 
295  for (i = 0; i < num_counters; ++i) {
296  if (!msrs->counters[i].addr)
297  continue;
298  release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
299  release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
300  }
301 }
302 
303 static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
304 {
305  int i;
306 
307  for (i = 0; i < num_counters; i++) {
308  if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
309  goto fail;
310  if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
311  release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
312  goto fail;
313  }
314  /* both registers must be reserved */
315  if (num_counters == AMD64_NUM_COUNTERS_CORE) {
316  msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1);
317  msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1);
318  } else {
319  msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
320  msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
321  }
322  continue;
323  fail:
324  if (!counter_config[i].enabled)
325  continue;
326  op_x86_warn_reserved(i);
327  op_amd_shutdown(msrs);
328  return -EBUSY;
329  }
330 
331  return 0;
332 }
333 
334 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
335  struct op_msrs const * const msrs)
336 {
337  u64 val;
338  int i;
339 
340  /* setup reset_value */
341  for (i = 0; i < OP_MAX_COUNTER; ++i) {
342  if (counter_config[i].enabled
343  && msrs->counters[op_x86_virt_to_phys(i)].addr)
344  reset_value[i] = counter_config[i].count;
345  else
346  reset_value[i] = 0;
347  }
348 
349  /* clear all counters */
350  for (i = 0; i < num_counters; ++i) {
351  if (!msrs->controls[i].addr)
352  continue;
353  rdmsrl(msrs->controls[i].addr, val);
354  if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
355  op_x86_warn_in_use(i);
356  val &= model->reserved;
357  wrmsrl(msrs->controls[i].addr, val);
358  /*
359  * avoid a false detection of ctr overflows in NMI
360  * handler
361  */
362  wrmsrl(msrs->counters[i].addr, -1LL);
363  }
364 
365  /* enable active counters */
366  for (i = 0; i < num_counters; ++i) {
367  int virt = op_x86_phys_to_virt(i);
368  if (!reset_value[virt])
369  continue;
370 
371  /* setup counter registers */
372  wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
373 
374  /* setup control registers */
375  rdmsrl(msrs->controls[i].addr, val);
376  val &= model->reserved;
377  val |= op_x86_get_ctrl(model, &counter_config[virt]);
378  wrmsrl(msrs->controls[i].addr, val);
379  }
380 }
381 
382 static int op_amd_check_ctrs(struct pt_regs * const regs,
383  struct op_msrs const * const msrs)
384 {
385  u64 val;
386  int i;
387 
388  for (i = 0; i < num_counters; ++i) {
389  int virt = op_x86_phys_to_virt(i);
390  if (!reset_value[virt])
391  continue;
392  rdmsrl(msrs->counters[i].addr, val);
393  /* bit is clear if overflowed: */
394  if (val & OP_CTR_OVERFLOW)
395  continue;
396  oprofile_add_sample(regs, virt);
397  wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
398  }
399 
400  op_amd_handle_ibs(regs, msrs);
401 
402  /* See op_model_ppro.c */
403  return 1;
404 }
405 
406 static void op_amd_start(struct op_msrs const * const msrs)
407 {
408  u64 val;
409  int i;
410 
411  for (i = 0; i < num_counters; ++i) {
412  if (!reset_value[op_x86_phys_to_virt(i)])
413  continue;
414  rdmsrl(msrs->controls[i].addr, val);
415  val |= ARCH_PERFMON_EVENTSEL_ENABLE;
416  wrmsrl(msrs->controls[i].addr, val);
417  }
418 
419  op_amd_start_ibs();
420 }
421 
422 static void op_amd_stop(struct op_msrs const * const msrs)
423 {
424  u64 val;
425  int i;
426 
427  /*
428  * Subtle: stop on all counters to avoid race with setting our
429  * pm callback
430  */
431  for (i = 0; i < num_counters; ++i) {
432  if (!reset_value[op_x86_phys_to_virt(i)])
433  continue;
434  rdmsrl(msrs->controls[i].addr, val);
435  val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
436  wrmsrl(msrs->controls[i].addr, val);
437  }
438 
439  op_amd_stop_ibs();
440 }
441 
442 /*
443  * check and reserve APIC extended interrupt LVT offset for IBS if
444  * available
445  */
446 
447 static void init_ibs(void)
448 {
449  ibs_caps = get_ibs_caps();
450 
451  if (!ibs_caps)
452  return;
453 
454  printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps);
455 }
456 
457 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
458 
459 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
460 {
461  struct dentry *dir;
462  int ret = 0;
463 
464  /* architecture specific files */
465  if (create_arch_files)
466  ret = create_arch_files(sb, root);
467 
468  if (ret)
469  return ret;
470 
471  if (!ibs_caps)
472  return ret;
473 
474  /* model specific files */
475 
476  /* setup some reasonable defaults */
477  memset(&ibs_config, 0, sizeof(ibs_config));
478  ibs_config.max_cnt_fetch = 250000;
479  ibs_config.max_cnt_op = 250000;
480 
481  if (ibs_caps & IBS_CAPS_FETCHSAM) {
482  dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
483  oprofilefs_create_ulong(sb, dir, "enable",
484  &ibs_config.fetch_enabled);
485  oprofilefs_create_ulong(sb, dir, "max_count",
486  &ibs_config.max_cnt_fetch);
487  oprofilefs_create_ulong(sb, dir, "rand_enable",
488  &ibs_config.rand_en);
489  }
490 
491  if (ibs_caps & IBS_CAPS_OPSAM) {
492  dir = oprofilefs_mkdir(sb, root, "ibs_op");
493  oprofilefs_create_ulong(sb, dir, "enable",
494  &ibs_config.op_enabled);
495  oprofilefs_create_ulong(sb, dir, "max_count",
496  &ibs_config.max_cnt_op);
497  if (ibs_caps & IBS_CAPS_OPCNT)
498  oprofilefs_create_ulong(sb, dir, "dispatched_ops",
499  &ibs_config.dispatched_ops);
500  if (ibs_caps & IBS_CAPS_BRNTRGT)
501  oprofilefs_create_ulong(sb, dir, "branch_target",
502  &ibs_config.branch_target);
503  }
504 
505  return 0;
506 }
507 
508 struct op_x86_model_spec op_amd_spec;
509 
510 static int op_amd_init(struct oprofile_operations *ops)
511 {
512  init_ibs();
513  create_arch_files = ops->create_files;
514  ops->create_files = setup_ibs_files;
515 
516  if (boot_cpu_data.x86 == 0x15) {
517  num_counters = AMD64_NUM_COUNTERS_CORE;
518  } else {
519  num_counters = AMD64_NUM_COUNTERS;
520  }
521 
522  op_amd_spec.num_counters = num_counters;
523  op_amd_spec.num_controls = num_counters;
524  op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
525 
526  return 0;
527 }
528 
529 struct op_x86_model_spec op_amd_spec = {
530  /* num_counters/num_controls filled in at runtime */
531  .reserved = MSR_AMD_EVENTSEL_RESERVED,
532  .event_mask = OP_EVENT_MASK,
533  .init = op_amd_init,
534  .fill_in_addresses = &op_amd_fill_in_addresses,
535  .setup_ctrs = &op_amd_setup_ctrs,
536  .check_ctrs = &op_amd_check_ctrs,
537  .start = &op_amd_start,
538  .stop = &op_amd_stop,
539  .shutdown = &op_amd_shutdown,
540 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
541  .switch_ctrl = &op_mux_switch_ctrl,
542 #endif
543 };
Generated on Thu Jan 10 2013 13:21:33 for Linux Kernel by   doxygen 1.8.2