Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
op_model_cell.c
Go to the documentation of this file.
1 /*
2  * Cell Broadband Engine OProfile Support
3  *
4  * (C) Copyright IBM Corporation 2006
5  *
6  * Author: David Erb ([email protected])
7  * Modifications:
8  * Carl Love <[email protected]>
9  * Maynard Johnson <[email protected]>
10  *
11  * This program is free software; you can redistribute it and/or
12  * modify it under the terms of the GNU General Public License
13  * as published by the Free Software Foundation; either version
14  * 2 of the License, or (at your option) any later version.
15  */
16 
17 #include <linux/cpufreq.h>
18 #include <linux/delay.h>
19 #include <linux/init.h>
20 #include <linux/jiffies.h>
21 #include <linux/kthread.h>
22 #include <linux/oprofile.h>
23 #include <linux/percpu.h>
24 #include <linux/smp.h>
25 #include <linux/spinlock.h>
26 #include <linux/timer.h>
27 #include <asm/cell-pmu.h>
28 #include <asm/cputable.h>
29 #include <asm/firmware.h>
30 #include <asm/io.h>
31 #include <asm/oprofile_impl.h>
32 #include <asm/processor.h>
33 #include <asm/prom.h>
34 #include <asm/ptrace.h>
35 #include <asm/reg.h>
36 #include <asm/rtas.h>
37 #include <asm/cell-regs.h>
38 
39 #include "../platforms/cell/interrupt.h"
40 #include "cell/pr_util.h"
41 
42 #define PPU_PROFILING 0
43 #define SPU_PROFILING_CYCLES 1
44 #define SPU_PROFILING_EVENTS 2
45 
46 #define SPU_EVENT_NUM_START 4100
47 #define SPU_EVENT_NUM_STOP 4399
48 #define SPU_PROFILE_EVENT_ADDR 4363 /* spu, address trace, decimal */
49 #define SPU_PROFILE_EVENT_ADDR_MASK_A 0x146 /* sub unit set to zero */
50 #define SPU_PROFILE_EVENT_ADDR_MASK_B 0x186 /* sub unit set to zero */
51 
52 #define NUM_SPUS_PER_NODE 8
53 #define SPU_CYCLES_EVENT_NUM 2 /* event number for SPU_CYCLES */
54 
55 #define PPU_CYCLES_EVENT_NUM 1 /* event number for CYCLES */
56 #define PPU_CYCLES_GRP_NUM 1 /* special group number for identifying
57  * PPU_CYCLES event
58  */
59 #define CBE_COUNT_ALL_CYCLES 0x42800000 /* PPU cycle event specifier */
60 
61 #define NUM_THREADS 2 /* number of physical threads in
62  * physical processor
63  */
64 #define NUM_DEBUG_BUS_WORDS 4
65 #define NUM_INPUT_BUS_WORDS 2
66 
67 #define MAX_SPU_COUNT 0xFFFFFF /* maximum 24 bit LFSR value */
68 
69 /* Minimum HW interval timer setting to send value to trace buffer is 10 cycle.
70  * To configure counter to send value every N cycles set counter to
71  * 2^32 - 1 - N.
72  */
73 #define NUM_INTERVAL_CYC 0xFFFFFFFF - 10
74 
75 /*
76  * spu_cycle_reset is the number of cycles between samples.
77  * This variable is used for SPU profiling and should ONLY be set
78  * at the beginning of cell_reg_setup; otherwise, it's read-only.
79  */
80 static unsigned int spu_cycle_reset;
81 static unsigned int profiling_mode;
82 static int spu_evnt_phys_spu_indx;
83 
85  unsigned long vcntr;
86  unsigned long evnts;
87  unsigned long masks;
88  unsigned long enabled;
89 };
90 
91 /*
92  * ibm,cbe-perftools rtas parameters
93  */
94 struct pm_signal {
95  u16 cpu; /* Processor to modify */
96  u16 sub_unit; /* hw subunit this applies to (if applicable)*/
97  short int signal_group; /* Signal Group to Enable/Disable */
98  u8 bus_word; /* Enable/Disable on this Trace/Trigger/Event
99  * Bus Word(s) (bitmask)
100  */
101  u8 bit; /* Trigger/Event bit (if applicable) */
102 };
103 
104 /*
105  * rtas call arguments
106  */
107 enum {
111 
112  PASSTHRU_IGNORE = 0,
115 };
117 struct pm_cntrl {
121  u16 freeze;
122  u16 count_mode;
125 };
127 static struct {
132 } pm_regs;
134 #define GET_SUB_UNIT(x) ((x & 0x0000f000) >> 12)
135 #define GET_BUS_WORD(x) ((x & 0x000000f0) >> 4)
136 #define GET_BUS_TYPE(x) ((x & 0x00000300) >> 8)
137 #define GET_POLARITY(x) ((x & 0x00000002) >> 1)
138 #define GET_COUNT_CYCLES(x) (x & 0x00000001)
139 #define GET_INPUT_CONTROL(x) ((x & 0x00000004) >> 2)
140 
141 static DEFINE_PER_CPU(unsigned long[NR_PHYS_CTRS], pmc_values);
142 static unsigned long spu_pm_cnt[MAX_NUMNODES * NUM_SPUS_PER_NODE];
143 static struct pmc_cntrl_data pmc_cntrl[NUM_THREADS][NR_PHYS_CTRS];
144 
145 /*
146  * The CELL profiling code makes rtas calls to setup the debug bus to
147  * route the performance signals. Additionally, SPU profiling requires
148  * a second rtas call to setup the hardware to capture the SPU PCs.
149  * The EIO error value is returned if the token lookups or the rtas
150  * call fail. The EIO error number is the best choice of the existing
151  * error numbers. The probability of rtas related error is very low. But
152  * by returning EIO and printing additional information to dmsg the user
153  * will know that OProfile did not start and dmesg will tell them why.
154  * OProfile does not support returning errors on Stop. Not a huge issue
155  * since failure to reset the debug bus or stop the SPU PC collection is
156  * not a fatel issue. Chances are if the Stop failed, Start doesn't work
157  * either.
158  */
159 
160 /*
161  * Interpetation of hdw_thread:
162  * 0 - even virtual cpus 0, 2, 4,...
163  * 1 - odd virtual cpus 1, 3, 5, ...
164  *
165  * FIXME: this is strictly wrong, we need to clean this up in a number
166  * of places. It works for now. -arnd
167  */
168 static u32 hdw_thread;
169 
170 static u32 virt_cntr_inter_mask;
171 static struct timer_list timer_virt_cntr;
172 static struct timer_list timer_spu_event_swap;
173 
174 /*
175  * pm_signal needs to be global since it is initialized in
176  * cell_reg_setup at the time when the necessary information
177  * is available.
178  */
179 static struct pm_signal pm_signal[NR_PHYS_CTRS];
180 static int pm_rtas_token; /* token for debug bus setup call */
181 static int spu_rtas_token; /* token for SPU cycle profiling */
182 
183 static u32 reset_value[NR_PHYS_CTRS];
184 static int num_counters;
185 static int oprofile_running;
186 static DEFINE_SPINLOCK(cntr_lock);
187 
188 static u32 ctr_enabled;
189 
190 static unsigned char input_bus[NUM_INPUT_BUS_WORDS];
191 
192 /*
193  * Firmware interface functions
194  */
195 static int
196 rtas_ibm_cbe_perftools(int subfunc, int passthru,
197  void *address, unsigned long length)
198 {
199  u64 paddr = __pa(address);
200 
201  return rtas_call(pm_rtas_token, 5, 1, NULL, subfunc,
202  passthru, paddr >> 32, paddr & 0xffffffff, length);
203 }
204 
205 static void pm_rtas_reset_signals(u32 node)
206 {
207  int ret;
208  struct pm_signal pm_signal_local;
209 
210  /*
211  * The debug bus is being set to the passthru disable state.
212  * However, the FW still expects atleast one legal signal routing
213  * entry or it will return an error on the arguments. If we don't
214  * supply a valid entry, we must ignore all return values. Ignoring
215  * all return values means we might miss an error we should be
216  * concerned about.
217  */
218 
219  /* fw expects physical cpu #. */
220  pm_signal_local.cpu = node;
221  pm_signal_local.signal_group = 21;
222  pm_signal_local.bus_word = 1;
223  pm_signal_local.sub_unit = 0;
224  pm_signal_local.bit = 0;
225 
226  ret = rtas_ibm_cbe_perftools(SUBFUNC_RESET, PASSTHRU_DISABLE,
227  &pm_signal_local,
228  sizeof(struct pm_signal));
229 
230  if (unlikely(ret))
231  /*
232  * Not a fatal error. For Oprofile stop, the oprofile
233  * functions do not support returning an error for
234  * failure to stop OProfile.
235  */
236  printk(KERN_WARNING "%s: rtas returned: %d\n",
237  __func__, ret);
238 }
239 
240 static int pm_rtas_activate_signals(u32 node, u32 count)
241 {
242  int ret;
243  int i, j;
244  struct pm_signal pm_signal_local[NR_PHYS_CTRS];
245 
246  /*
247  * There is no debug setup required for the cycles event.
248  * Note that only events in the same group can be used.
249  * Otherwise, there will be conflicts in correctly routing
250  * the signals on the debug bus. It is the responsibility
251  * of the OProfile user tool to check the events are in
252  * the same group.
253  */
254  i = 0;
255  for (j = 0; j < count; j++) {
257 
258  /* fw expects physical cpu # */
259  pm_signal_local[i].cpu = node;
260  pm_signal_local[i].signal_group
262  pm_signal_local[i].bus_word = pm_signal[j].bus_word;
263  pm_signal_local[i].sub_unit = pm_signal[j].sub_unit;
264  pm_signal_local[i].bit = pm_signal[j].bit;
265  i++;
266  }
267  }
268 
269  if (i != 0) {
270  ret = rtas_ibm_cbe_perftools(SUBFUNC_ACTIVATE, PASSTHRU_ENABLE,
271  pm_signal_local,
272  i * sizeof(struct pm_signal));
273 
274  if (unlikely(ret)) {
275  printk(KERN_WARNING "%s: rtas returned: %d\n",
276  __func__, ret);
277  return -EIO;
278  }
279  }
280 
281  return 0;
282 }
283 
284 /*
285  * PM Signal functions
286  */
287 static void set_pm_event(u32 ctr, int event, u32 unit_mask)
288 {
289  struct pm_signal *p;
290  u32 signal_bit;
291  u32 bus_word, bus_type, count_cycles, polarity, input_control;
292  int j, i;
293 
294  if (event == PPU_CYCLES_EVENT_NUM) {
295  /* Special Event: Count all cpu cycles */
296  pm_regs.pm07_cntrl[ctr] = CBE_COUNT_ALL_CYCLES;
297  p = &(pm_signal[ctr]);
299  p->bus_word = 1;
300  p->sub_unit = 0;
301  p->bit = 0;
302  goto out;
303  } else {
304  pm_regs.pm07_cntrl[ctr] = 0;
305  }
306 
307  bus_word = GET_BUS_WORD(unit_mask);
308  bus_type = GET_BUS_TYPE(unit_mask);
309  count_cycles = GET_COUNT_CYCLES(unit_mask);
310  polarity = GET_POLARITY(unit_mask);
311  input_control = GET_INPUT_CONTROL(unit_mask);
312  signal_bit = (event % 100);
313 
314  p = &(pm_signal[ctr]);
315 
316  p->signal_group = event / 100;
317  p->bus_word = bus_word;
318  p->sub_unit = GET_SUB_UNIT(unit_mask);
319 
320  pm_regs.pm07_cntrl[ctr] = 0;
321  pm_regs.pm07_cntrl[ctr] |= PM07_CTR_COUNT_CYCLES(count_cycles);
322  pm_regs.pm07_cntrl[ctr] |= PM07_CTR_POLARITY(polarity);
323  pm_regs.pm07_cntrl[ctr] |= PM07_CTR_INPUT_CONTROL(input_control);
324 
325  /*
326  * Some of the islands signal selection is based on 64 bit words.
327  * The debug bus words are 32 bits, the input words to the performance
328  * counters are defined as 32 bits. Need to convert the 64 bit island
329  * specification to the appropriate 32 input bit and bus word for the
330  * performance counter event selection. See the CELL Performance
331  * monitoring signals manual and the Perf cntr hardware descriptions
332  * for the details.
333  */
334  if (input_control == 0) {
335  if (signal_bit > 31) {
336  signal_bit -= 32;
337  if (bus_word == 0x3)
338  bus_word = 0x2;
339  else if (bus_word == 0xc)
340  bus_word = 0x8;
341  }
342 
343  if ((bus_type == 0) && p->signal_group >= 60)
344  bus_type = 2;
345  if ((bus_type == 1) && p->signal_group >= 50)
346  bus_type = 0;
347 
348  pm_regs.pm07_cntrl[ctr] |= PM07_CTR_INPUT_MUX(signal_bit);
349  } else {
350  pm_regs.pm07_cntrl[ctr] = 0;
351  p->bit = signal_bit;
352  }
353 
354  for (i = 0; i < NUM_DEBUG_BUS_WORDS; i++) {
355  if (bus_word & (1 << i)) {
356  pm_regs.debug_bus_control |=
357  (bus_type << (30 - (2 * i)));
358 
359  for (j = 0; j < NUM_INPUT_BUS_WORDS; j++) {
360  if (input_bus[j] == 0xff) {
361  input_bus[j] = i;
362  pm_regs.group_control |=
363  (i << (30 - (2 * j)));
364 
365  break;
366  }
367  }
368  }
369  }
370 out:
371  ;
372 }
373 
374 static void write_pm_cntrl(int cpu)
375 {
376  /*
377  * Oprofile will use 32 bit counters, set bits 7:10 to 0
378  * pmregs.pm_cntrl is a global
379  */
380 
381  u32 val = 0;
382  if (pm_regs.pm_cntrl.enable == 1)
383  val |= CBE_PM_ENABLE_PERF_MON;
384 
385  if (pm_regs.pm_cntrl.stop_at_max == 1)
386  val |= CBE_PM_STOP_AT_MAX;
387 
388  if (pm_regs.pm_cntrl.trace_mode != 0)
389  val |= CBE_PM_TRACE_MODE_SET(pm_regs.pm_cntrl.trace_mode);
390 
391  if (pm_regs.pm_cntrl.trace_buf_ovflw == 1)
392  val |= CBE_PM_TRACE_BUF_OVFLW(pm_regs.pm_cntrl.trace_buf_ovflw);
393  if (pm_regs.pm_cntrl.freeze == 1)
394  val |= CBE_PM_FREEZE_ALL_CTRS;
395 
396  val |= CBE_PM_SPU_ADDR_TRACE_SET(pm_regs.pm_cntrl.spu_addr_trace);
397 
398  /*
399  * Routine set_count_mode must be called previously to set
400  * the count mode based on the user selection of user and kernel.
401  */
402  val |= CBE_PM_COUNT_MODE_SET(pm_regs.pm_cntrl.count_mode);
403  cbe_write_pm(cpu, pm_control, val);
404 }
405 
406 static inline void
407 set_count_mode(u32 kernel, u32 user)
408 {
409  /*
410  * The user must specify user and kernel if they want them. If
411  * neither is specified, OProfile will count in hypervisor mode.
412  * pm_regs.pm_cntrl is a global
413  */
414  if (kernel) {
415  if (user)
416  pm_regs.pm_cntrl.count_mode = CBE_COUNT_ALL_MODES;
417  else
418  pm_regs.pm_cntrl.count_mode =
420  } else {
421  if (user)
422  pm_regs.pm_cntrl.count_mode = CBE_COUNT_PROBLEM_MODE;
423  else
424  pm_regs.pm_cntrl.count_mode =
426  }
427 }
428 
429 static inline void enable_ctr(u32 cpu, u32 ctr, u32 *pm07_cntrl)
430 {
431 
432  pm07_cntrl[ctr] |= CBE_PM_CTR_ENABLE;
433  cbe_write_pm07_control(cpu, ctr, pm07_cntrl[ctr]);
434 }
435 
436 /*
437  * Oprofile is expected to collect data on all CPUs simultaneously.
438  * However, there is one set of performance counters per node. There are
439  * two hardware threads or virtual CPUs on each node. Hence, OProfile must
440  * multiplex in time the performance counter collection on the two virtual
441  * CPUs. The multiplexing of the performance counters is done by this
442  * virtual counter routine.
443  *
444  * The pmc_values used below is defined as 'per-cpu' but its use is
445  * more akin to 'per-node'. We need to store two sets of counter
446  * values per node -- one for the previous run and one for the next.
447  * The per-cpu[NR_PHYS_CTRS] gives us the storage we need. Each odd/even
448  * pair of per-cpu arrays is used for storing the previous and next
449  * pmc values for a given node.
450  * NOTE: We use the per-cpu variable to improve cache performance.
451  *
452  * This routine will alternate loading the virtual counters for
453  * virtual CPUs
454  */
455 static void cell_virtual_cntr(unsigned long data)
456 {
457  int i, prev_hdw_thread, next_hdw_thread;
458  u32 cpu;
459  unsigned long flags;
460 
461  /*
462  * Make sure that the interrupt_hander and the virt counter are
463  * not both playing with the counters on the same node.
464  */
465 
466  spin_lock_irqsave(&cntr_lock, flags);
467 
468  prev_hdw_thread = hdw_thread;
469 
470  /* switch the cpu handling the interrupts */
471  hdw_thread = 1 ^ hdw_thread;
472  next_hdw_thread = hdw_thread;
473 
474  pm_regs.group_control = 0;
475  pm_regs.debug_bus_control = 0;
476 
477  for (i = 0; i < NUM_INPUT_BUS_WORDS; i++)
478  input_bus[i] = 0xff;
479 
480  /*
481  * There are some per thread events. Must do the
482  * set event, for the thread that is being started
483  */
484  for (i = 0; i < num_counters; i++)
485  set_pm_event(i,
486  pmc_cntrl[next_hdw_thread][i].evnts,
487  pmc_cntrl[next_hdw_thread][i].masks);
488 
489  /*
490  * The following is done only once per each node, but
491  * we need cpu #, not node #, to pass to the cbe_xxx functions.
492  */
493  for_each_online_cpu(cpu) {
494  if (cbe_get_hw_thread_id(cpu))
495  continue;
496 
497  /*
498  * stop counters, save counter values, restore counts
499  * for previous thread
500  */
501  cbe_disable_pm(cpu);
503  for (i = 0; i < num_counters; i++) {
504  per_cpu(pmc_values, cpu + prev_hdw_thread)[i]
505  = cbe_read_ctr(cpu, i);
506 
507  if (per_cpu(pmc_values, cpu + next_hdw_thread)[i]
508  == 0xFFFFFFFF)
509  /* If the cntr value is 0xffffffff, we must
510  * reset that to 0xfffffff0 when the current
511  * thread is restarted. This will generate a
512  * new interrupt and make sure that we never
513  * restore the counters to the max value. If
514  * the counters were restored to the max value,
515  * they do not increment and no interrupts are
516  * generated. Hence no more samples will be
517  * collected on that cpu.
518  */
519  cbe_write_ctr(cpu, i, 0xFFFFFFF0);
520  else
521  cbe_write_ctr(cpu, i,
522  per_cpu(pmc_values,
523  cpu +
524  next_hdw_thread)[i]);
525  }
526 
527  /*
528  * Switch to the other thread. Change the interrupt
529  * and control regs to be scheduled on the CPU
530  * corresponding to the thread to execute.
531  */
532  for (i = 0; i < num_counters; i++) {
533  if (pmc_cntrl[next_hdw_thread][i].enabled) {
534  /*
535  * There are some per thread events.
536  * Must do the set event, enable_cntr
537  * for each cpu.
538  */
539  enable_ctr(cpu, i,
540  pm_regs.pm07_cntrl);
541  } else {
542  cbe_write_pm07_control(cpu, i, 0);
543  }
544  }
545 
546  /* Enable interrupts on the CPU thread that is starting */
547  cbe_enable_pm_interrupts(cpu, next_hdw_thread,
548  virt_cntr_inter_mask);
549  cbe_enable_pm(cpu);
550  }
551 
552  spin_unlock_irqrestore(&cntr_lock, flags);
553 
554  mod_timer(&timer_virt_cntr, jiffies + HZ / 10);
555 }
556 
557 static void start_virt_cntrs(void)
558 {
559  init_timer(&timer_virt_cntr);
560  timer_virt_cntr.function = cell_virtual_cntr;
561  timer_virt_cntr.data = 0UL;
562  timer_virt_cntr.expires = jiffies + HZ / 10;
563  add_timer(&timer_virt_cntr);
564 }
565 
566 static int cell_reg_setup_spu_cycles(struct op_counter_config *ctr,
567  struct op_system_config *sys, int num_ctrs)
568 {
569  spu_cycle_reset = ctr[0].count;
570 
571  /*
572  * Each node will need to make the rtas call to start
573  * and stop SPU profiling. Get the token once and store it.
574  */
575  spu_rtas_token = rtas_token("ibm,cbe-spu-perftools");
576 
577  if (unlikely(spu_rtas_token == RTAS_UNKNOWN_SERVICE)) {
579  "%s: rtas token ibm,cbe-spu-perftools unknown\n",
580  __func__);
581  return -EIO;
582  }
583  return 0;
584 }
585 
586 /* Unfortunately, the hardware will only support event profiling
587  * on one SPU per node at a time. Therefore, we must time slice
588  * the profiling across all SPUs in the node. Note, we do this
589  * in parallel for each node. The following routine is called
590  * periodically based on kernel timer to switch which SPU is
591  * being monitored in a round robbin fashion.
592  */
593 static void spu_evnt_swap(unsigned long data)
594 {
595  int node;
596  int cur_phys_spu, nxt_phys_spu, cur_spu_evnt_phys_spu_indx;
597  unsigned long flags;
598  int cpu;
599  int ret;
600  u32 interrupt_mask;
601 
602 
603  /* enable interrupts on cntr 0 */
604  interrupt_mask = CBE_PM_CTR_OVERFLOW_INTR(0);
605 
606  hdw_thread = 0;
607 
608  /* Make sure spu event interrupt handler and spu event swap
609  * don't access the counters simultaneously.
610  */
611  spin_lock_irqsave(&cntr_lock, flags);
612 
613  cur_spu_evnt_phys_spu_indx = spu_evnt_phys_spu_indx;
614 
615  if (++(spu_evnt_phys_spu_indx) == NUM_SPUS_PER_NODE)
616  spu_evnt_phys_spu_indx = 0;
617 
618  pm_signal[0].sub_unit = spu_evnt_phys_spu_indx;
619  pm_signal[1].sub_unit = spu_evnt_phys_spu_indx;
620  pm_signal[2].sub_unit = spu_evnt_phys_spu_indx;
621 
622  /* switch the SPU being profiled on each node */
623  for_each_online_cpu(cpu) {
624  if (cbe_get_hw_thread_id(cpu))
625  continue;
626 
627  node = cbe_cpu_to_node(cpu);
628  cur_phys_spu = (node * NUM_SPUS_PER_NODE)
629  + cur_spu_evnt_phys_spu_indx;
630  nxt_phys_spu = (node * NUM_SPUS_PER_NODE)
631  + spu_evnt_phys_spu_indx;
632 
633  /*
634  * stop counters, save counter values, restore counts
635  * for previous physical SPU
636  */
637  cbe_disable_pm(cpu);
639 
640  spu_pm_cnt[cur_phys_spu]
641  = cbe_read_ctr(cpu, 0);
642 
643  /* restore previous count for the next spu to sample */
644  /* NOTE, hardware issue, counter will not start if the
645  * counter value is at max (0xFFFFFFFF).
646  */
647  if (spu_pm_cnt[nxt_phys_spu] >= 0xFFFFFFFF)
648  cbe_write_ctr(cpu, 0, 0xFFFFFFF0);
649  else
650  cbe_write_ctr(cpu, 0, spu_pm_cnt[nxt_phys_spu]);
651 
652  pm_rtas_reset_signals(cbe_cpu_to_node(cpu));
653 
654  /* setup the debug bus measure the one event and
655  * the two events to route the next SPU's PC on
656  * the debug bus
657  */
658  ret = pm_rtas_activate_signals(cbe_cpu_to_node(cpu), 3);
659  if (ret)
660  printk(KERN_ERR "%s: pm_rtas_activate_signals failed, "
661  "SPU event swap\n", __func__);
662 
663  /* clear the trace buffer, don't want to take PC for
664  * previous SPU*/
665  cbe_write_pm(cpu, trace_address, 0);
666 
667  enable_ctr(cpu, 0, pm_regs.pm07_cntrl);
668 
669  /* Enable interrupts on the CPU thread that is starting */
670  cbe_enable_pm_interrupts(cpu, hdw_thread,
671  interrupt_mask);
672  cbe_enable_pm(cpu);
673  }
674 
675  spin_unlock_irqrestore(&cntr_lock, flags);
676 
677  /* swap approximately every 0.1 seconds */
678  mod_timer(&timer_spu_event_swap, jiffies + HZ / 25);
679 }
680 
681 static void start_spu_event_swap(void)
682 {
683  init_timer(&timer_spu_event_swap);
684  timer_spu_event_swap.function = spu_evnt_swap;
685  timer_spu_event_swap.data = 0UL;
686  timer_spu_event_swap.expires = jiffies + HZ / 25;
687  add_timer(&timer_spu_event_swap);
688 }
689 
690 static int cell_reg_setup_spu_events(struct op_counter_config *ctr,
691  struct op_system_config *sys, int num_ctrs)
692 {
693  int i;
694 
695  /* routine is called once for all nodes */
696 
697  spu_evnt_phys_spu_indx = 0;
698  /*
699  * For all events except PPU CYCLEs, each node will need to make
700  * the rtas cbe-perftools call to setup and reset the debug bus.
701  * Make the token lookup call once and store it in the global
702  * variable pm_rtas_token.
703  */
704  pm_rtas_token = rtas_token("ibm,cbe-perftools");
705 
706  if (unlikely(pm_rtas_token == RTAS_UNKNOWN_SERVICE)) {
708  "%s: rtas token ibm,cbe-perftools unknown\n",
709  __func__);
710  return -EIO;
711  }
712 
713  /* setup the pm_control register settings,
714  * settings will be written per node by the
715  * cell_cpu_setup() function.
716  */
717  pm_regs.pm_cntrl.trace_buf_ovflw = 1;
718 
719  /* Use the occurrence trace mode to have SPU PC saved
720  * to the trace buffer. Occurrence data in trace buffer
721  * is not used. Bit 2 must be set to store SPU addresses.
722  */
723  pm_regs.pm_cntrl.trace_mode = 2;
724 
725  pm_regs.pm_cntrl.spu_addr_trace = 0x1; /* using debug bus
726  event 2 & 3 */
727 
728  /* setup the debug bus event array with the SPU PC routing events.
729  * Note, pm_signal[0] will be filled in by set_pm_event() call below.
730  */
734  pm_signal[1].sub_unit = spu_evnt_phys_spu_indx;
735 
739  pm_signal[2].sub_unit = spu_evnt_phys_spu_indx;
740 
741  /* Set the user selected spu event to profile on,
742  * note, only one SPU profiling event is supported
743  */
744  num_counters = 1; /* Only support one SPU event at a time */
745  set_pm_event(0, ctr[0].event, ctr[0].unit_mask);
746 
747  reset_value[0] = 0xFFFFFFFF - ctr[0].count;
748 
749  /* global, used by cell_cpu_setup */
750  ctr_enabled |= 1;
751 
752  /* Initialize the count for each SPU to the reset value */
753  for (i=0; i < MAX_NUMNODES * NUM_SPUS_PER_NODE; i++)
754  spu_pm_cnt[i] = reset_value[0];
755 
756  return 0;
757 }
758 
759 static int cell_reg_setup_ppu(struct op_counter_config *ctr,
760  struct op_system_config *sys, int num_ctrs)
761 {
762  /* routine is called once for all nodes */
763  int i, j, cpu;
764 
765  num_counters = num_ctrs;
766 
767  if (unlikely(num_ctrs > NR_PHYS_CTRS)) {
769  "%s: Oprofile, number of specified events " \
770  "exceeds number of physical counters\n",
771  __func__);
772  return -EIO;
773  }
774 
775  set_count_mode(sys->enable_kernel, sys->enable_user);
776 
777  /* Setup the thread 0 events */
778  for (i = 0; i < num_ctrs; ++i) {
779 
780  pmc_cntrl[0][i].evnts = ctr[i].event;
781  pmc_cntrl[0][i].masks = ctr[i].unit_mask;
782  pmc_cntrl[0][i].enabled = ctr[i].enabled;
783  pmc_cntrl[0][i].vcntr = i;
784 
786  per_cpu(pmc_values, j)[i] = 0;
787  }
788 
789  /*
790  * Setup the thread 1 events, map the thread 0 event to the
791  * equivalent thread 1 event.
792  */
793  for (i = 0; i < num_ctrs; ++i) {
794  if ((ctr[i].event >= 2100) && (ctr[i].event <= 2111))
795  pmc_cntrl[1][i].evnts = ctr[i].event + 19;
796  else if (ctr[i].event == 2203)
797  pmc_cntrl[1][i].evnts = ctr[i].event;
798  else if ((ctr[i].event >= 2200) && (ctr[i].event <= 2215))
799  pmc_cntrl[1][i].evnts = ctr[i].event + 16;
800  else
801  pmc_cntrl[1][i].evnts = ctr[i].event;
802 
803  pmc_cntrl[1][i].masks = ctr[i].unit_mask;
804  pmc_cntrl[1][i].enabled = ctr[i].enabled;
805  pmc_cntrl[1][i].vcntr = i;
806  }
807 
808  for (i = 0; i < NUM_INPUT_BUS_WORDS; i++)
809  input_bus[i] = 0xff;
810 
811  /*
812  * Our counters count up, and "count" refers to
813  * how much before the next interrupt, and we interrupt
814  * on overflow. So we calculate the starting value
815  * which will give us "count" until overflow.
816  * Then we set the events on the enabled counters.
817  */
818  for (i = 0; i < num_counters; ++i) {
819  /* start with virtual counter set 0 */
820  if (pmc_cntrl[0][i].enabled) {
821  /* Using 32bit counters, reset max - count */
822  reset_value[i] = 0xFFFFFFFF - ctr[i].count;
823  set_pm_event(i,
824  pmc_cntrl[0][i].evnts,
825  pmc_cntrl[0][i].masks);
826 
827  /* global, used by cell_cpu_setup */
828  ctr_enabled |= (1 << i);
829  }
830  }
831 
832  /* initialize the previous counts for the virtual cntrs */
834  for (i = 0; i < num_counters; ++i) {
835  per_cpu(pmc_values, cpu)[i] = reset_value[i];
836  }
837 
838  return 0;
839 }
840 
841 
842 /* This function is called once for all cpus combined */
843 static int cell_reg_setup(struct op_counter_config *ctr,
844  struct op_system_config *sys, int num_ctrs)
845 {
846  int ret=0;
847  spu_cycle_reset = 0;
848 
849  /* initialize the spu_arr_trace value, will be reset if
850  * doing spu event profiling.
851  */
852  pm_regs.group_control = 0;
853  pm_regs.debug_bus_control = 0;
854  pm_regs.pm_cntrl.stop_at_max = 1;
855  pm_regs.pm_cntrl.trace_mode = 0;
856  pm_regs.pm_cntrl.freeze = 1;
857  pm_regs.pm_cntrl.trace_buf_ovflw = 0;
858  pm_regs.pm_cntrl.spu_addr_trace = 0;
859 
860  /*
861  * For all events except PPU CYCLEs, each node will need to make
862  * the rtas cbe-perftools call to setup and reset the debug bus.
863  * Make the token lookup call once and store it in the global
864  * variable pm_rtas_token.
865  */
866  pm_rtas_token = rtas_token("ibm,cbe-perftools");
867 
868  if (unlikely(pm_rtas_token == RTAS_UNKNOWN_SERVICE)) {
870  "%s: rtas token ibm,cbe-perftools unknown\n",
871  __func__);
872  return -EIO;
873  }
874 
875  if (ctr[0].event == SPU_CYCLES_EVENT_NUM) {
876  profiling_mode = SPU_PROFILING_CYCLES;
877  ret = cell_reg_setup_spu_cycles(ctr, sys, num_ctrs);
878  } else if ((ctr[0].event >= SPU_EVENT_NUM_START) &&
879  (ctr[0].event <= SPU_EVENT_NUM_STOP)) {
880  profiling_mode = SPU_PROFILING_EVENTS;
881  spu_cycle_reset = ctr[0].count;
882 
883  /* for SPU event profiling, need to setup the
884  * pm_signal array with the events to route the
885  * SPU PC before making the FW call. Note, only
886  * one SPU event for profiling can be specified
887  * at a time.
888  */
889  cell_reg_setup_spu_events(ctr, sys, num_ctrs);
890  } else {
891  profiling_mode = PPU_PROFILING;
892  ret = cell_reg_setup_ppu(ctr, sys, num_ctrs);
893  }
894 
895  return ret;
896 }
897 
898 
899 
900 /* This function is called once for each cpu */
901 static int cell_cpu_setup(struct op_counter_config *cntr)
902 {
903  u32 cpu = smp_processor_id();
904  u32 num_enabled = 0;
905  int i;
906  int ret;
907 
908  /* Cycle based SPU profiling does not use the performance
909  * counters. The trace array is configured to collect
910  * the data.
911  */
912  if (profiling_mode == SPU_PROFILING_CYCLES)
913  return 0;
914 
915  /* There is one performance monitor per processor chip (i.e. node),
916  * so we only need to perform this function once per node.
917  */
918  if (cbe_get_hw_thread_id(cpu))
919  return 0;
920 
921  /* Stop all counters */
922  cbe_disable_pm(cpu);
924 
925  cbe_write_pm(cpu, pm_start_stop, 0);
926  cbe_write_pm(cpu, group_control, pm_regs.group_control);
927  cbe_write_pm(cpu, debug_bus_control, pm_regs.debug_bus_control);
928  write_pm_cntrl(cpu);
929 
930  for (i = 0; i < num_counters; ++i) {
931  if (ctr_enabled & (1 << i)) {
932  pm_signal[num_enabled].cpu = cbe_cpu_to_node(cpu);
933  num_enabled++;
934  }
935  }
936 
937  /*
938  * The pm_rtas_activate_signals will return -EIO if the FW
939  * call failed.
940  */
941  if (profiling_mode == SPU_PROFILING_EVENTS) {
942  /* For SPU event profiling also need to setup the
943  * pm interval timer
944  */
945  ret = pm_rtas_activate_signals(cbe_cpu_to_node(cpu),
946  num_enabled+2);
947  /* store PC from debug bus to Trace buffer as often
948  * as possible (every 10 cycles)
949  */
951  return ret;
952  } else
953  return pm_rtas_activate_signals(cbe_cpu_to_node(cpu),
954  num_enabled);
955 }
956 
957 #define ENTRIES 303
958 #define MAXLFSR 0xFFFFFF
959 
960 /* precomputed table of 24 bit LFSR values */
961 static int initial_lfsr[] = {
962  8221349, 12579195, 5379618, 10097839, 7512963, 7519310, 3955098, 10753424,
963  15507573, 7458917, 285419, 2641121, 9780088, 3915503, 6668768, 1548716,
964  4885000, 8774424, 9650099, 2044357, 2304411, 9326253, 10332526, 4421547,
965  3440748, 10179459, 13332843, 10375561, 1313462, 8375100, 5198480, 6071392,
966  9341783, 1526887, 3985002, 1439429, 13923762, 7010104, 11969769, 4547026,
967  2040072, 4025602, 3437678, 7939992, 11444177, 4496094, 9803157, 10745556,
968  3671780, 4257846, 5662259, 13196905, 3237343, 12077182, 16222879, 7587769,
969  14706824, 2184640, 12591135, 10420257, 7406075, 3648978, 11042541, 15906893,
970  11914928, 4732944, 10695697, 12928164, 11980531, 4430912, 11939291, 2917017,
971  6119256, 4172004, 9373765, 8410071, 14788383, 5047459, 5474428, 1737756,
972  15967514, 13351758, 6691285, 8034329, 2856544, 14394753, 11310160, 12149558,
973  7487528, 7542781, 15668898, 12525138, 12790975, 3707933, 9106617, 1965401,
974  16219109, 12801644, 2443203, 4909502, 8762329, 3120803, 6360315, 9309720,
975  15164599, 10844842, 4456529, 6667610, 14924259, 884312, 6234963, 3326042,
976  15973422, 13919464, 5272099, 6414643, 3909029, 2764324, 5237926, 4774955,
977  10445906, 4955302, 5203726, 10798229, 11443419, 2303395, 333836, 9646934,
978  3464726, 4159182, 568492, 995747, 10318756, 13299332, 4836017, 8237783,
979  3878992, 2581665, 11394667, 5672745, 14412947, 3159169, 9094251, 16467278,
980  8671392, 15230076, 4843545, 7009238, 15504095, 1494895, 9627886, 14485051,
981  8304291, 252817, 12421642, 16085736, 4774072, 2456177, 4160695, 15409741,
982  4902868, 5793091, 13162925, 16039714, 782255, 11347835, 14884586, 366972,
983  16308990, 11913488, 13390465, 2958444, 10340278, 1177858, 1319431, 10426302,
984  2868597, 126119, 5784857, 5245324, 10903900, 16436004, 3389013, 1742384,
985  14674502, 10279218, 8536112, 10364279, 6877778, 14051163, 1025130, 6072469,
986  1988305, 8354440, 8216060, 16342977, 13112639, 3976679, 5913576, 8816697,
987  6879995, 14043764, 3339515, 9364420, 15808858, 12261651, 2141560, 5636398,
988  10345425, 10414756, 781725, 6155650, 4746914, 5078683, 7469001, 6799140,
989  10156444, 9667150, 10116470, 4133858, 2121972, 1124204, 1003577, 1611214,
990  14304602, 16221850, 13878465, 13577744, 3629235, 8772583, 10881308, 2410386,
991  7300044, 5378855, 9301235, 12755149, 4977682, 8083074, 10327581, 6395087,
992  9155434, 15501696, 7514362, 14520507, 15808945, 3244584, 4741962, 9658130,
993  14336147, 8654727, 7969093, 15759799, 14029445, 5038459, 9894848, 8659300,
994  13699287, 8834306, 10712885, 14753895, 10410465, 3373251, 309501, 9561475,
995  5526688, 14647426, 14209836, 5339224, 207299, 14069911, 8722990, 2290950,
996  3258216, 12505185, 6007317, 9218111, 14661019, 10537428, 11731949, 9027003,
997  6641507, 9490160, 200241, 9720425, 16277895, 10816638, 1554761, 10431375,
998  7467528, 6790302, 3429078, 14633753, 14428997, 11463204, 3576212, 2003426,
999  6123687, 820520, 9992513, 15784513, 5778891, 6428165, 8388607
1000 };
1001 
1002 /*
1003  * The hardware uses an LFSR counting sequence to determine when to capture
1004  * the SPU PCs. An LFSR sequence is like a puesdo random number sequence
1005  * where each number occurs once in the sequence but the sequence is not in
1006  * numerical order. The SPU PC capture is done when the LFSR sequence reaches
1007  * the last value in the sequence. Hence the user specified value N
1008  * corresponds to the LFSR number that is N from the end of the sequence.
1009  *
1010  * To avoid the time to compute the LFSR, a lookup table is used. The 24 bit
1011  * LFSR sequence is broken into four ranges. The spacing of the precomputed
1012  * values is adjusted in each range so the error between the user specifed
1013  * number (N) of events between samples and the actual number of events based
1014  * on the precomputed value will be les then about 6.2%. Note, if the user
1015  * specifies N < 2^16, the LFSR value that is 2^16 from the end will be used.
1016  * This is to prevent the loss of samples because the trace buffer is full.
1017  *
1018  * User specified N Step between Index in
1019  * precomputed values precomputed
1020  * table
1021  * 0 to 2^16-1 ---- 0
1022  * 2^16 to 2^16+2^19-1 2^12 1 to 128
1023  * 2^16+2^19 to 2^16+2^19+2^22-1 2^15 129 to 256
1024  * 2^16+2^19+2^22 to 2^24-1 2^18 257 to 302
1025  *
1026  *
1027  * For example, the LFSR values in the second range are computed for 2^16,
1028  * 2^16+2^12, ... , 2^19-2^16, 2^19 and stored in the table at indicies
1029  * 1, 2,..., 127, 128.
1030  *
1031  * The 24 bit LFSR value for the nth number in the sequence can be
1032  * calculated using the following code:
1033  *
1034  * #define size 24
1035  * int calculate_lfsr(int n)
1036  * {
1037  * int i;
1038  * unsigned int newlfsr0;
1039  * unsigned int lfsr = 0xFFFFFF;
1040  * unsigned int howmany = n;
1041  *
1042  * for (i = 2; i < howmany + 2; i++) {
1043  * newlfsr0 = (((lfsr >> (size - 1 - 0)) & 1) ^
1044  * ((lfsr >> (size - 1 - 1)) & 1) ^
1045  * (((lfsr >> (size - 1 - 6)) & 1) ^
1046  * ((lfsr >> (size - 1 - 23)) & 1)));
1047  *
1048  * lfsr >>= 1;
1049  * lfsr = lfsr | (newlfsr0 << (size - 1));
1050  * }
1051  * return lfsr;
1052  * }
1053  */
1054 
1055 #define V2_16 (0x1 << 16)
1056 #define V2_19 (0x1 << 19)
1057 #define V2_22 (0x1 << 22)
1058 
1059 static int calculate_lfsr(int n)
1060 {
1061  /*
1062  * The ranges and steps are in powers of 2 so the calculations
1063  * can be done using shifts rather then divide.
1064  */
1065  int index;
1066 
1067  if ((n >> 16) == 0)
1068  index = 0;
1069  else if (((n - V2_16) >> 19) == 0)
1070  index = ((n - V2_16) >> 12) + 1;
1071  else if (((n - V2_16 - V2_19) >> 22) == 0)
1072  index = ((n - V2_16 - V2_19) >> 15 ) + 1 + 128;
1073  else if (((n - V2_16 - V2_19 - V2_22) >> 24) == 0)
1074  index = ((n - V2_16 - V2_19 - V2_22) >> 18 ) + 1 + 256;
1075  else
1076  index = ENTRIES-1;
1077 
1078  /* make sure index is valid */
1079  if ((index >= ENTRIES) || (index < 0))
1080  index = ENTRIES-1;
1081 
1082  return initial_lfsr[index];
1083 }
1084 
1085 static int pm_rtas_activate_spu_profiling(u32 node)
1086 {
1087  int ret, i;
1088  struct pm_signal pm_signal_local[NUM_SPUS_PER_NODE];
1089 
1090  /*
1091  * Set up the rtas call to configure the debug bus to
1092  * route the SPU PCs. Setup the pm_signal for each SPU
1093  */
1094  for (i = 0; i < ARRAY_SIZE(pm_signal_local); i++) {
1095  pm_signal_local[i].cpu = node;
1096  pm_signal_local[i].signal_group = 41;
1097  /* spu i on word (i/2) */
1098  pm_signal_local[i].bus_word = 1 << i / 2;
1099  /* spu i */
1100  pm_signal_local[i].sub_unit = i;
1101  pm_signal_local[i].bit = 63;
1102  }
1103 
1104  ret = rtas_ibm_cbe_perftools(SUBFUNC_ACTIVATE,
1105  PASSTHRU_ENABLE, pm_signal_local,
1106  (ARRAY_SIZE(pm_signal_local)
1107  * sizeof(struct pm_signal)));
1108 
1109  if (unlikely(ret)) {
1110  printk(KERN_WARNING "%s: rtas returned: %d\n",
1111  __func__, ret);
1112  return -EIO;
1113  }
1114 
1115  return 0;
1116 }
1117 
1118 #ifdef CONFIG_CPU_FREQ
1119 static int
1120 oprof_cpufreq_notify(struct notifier_block *nb, unsigned long val, void *data)
1121 {
1122  int ret = 0;
1123  struct cpufreq_freqs *frq = data;
1124  if ((val == CPUFREQ_PRECHANGE && frq->old < frq->new) ||
1125  (val == CPUFREQ_POSTCHANGE && frq->old > frq->new) ||
1126  (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE))
1127  set_spu_profiling_frequency(frq->new, spu_cycle_reset);
1128  return ret;
1129 }
1130 
1131 static struct notifier_block cpu_freq_notifier_block = {
1132  .notifier_call = oprof_cpufreq_notify
1133 };
1134 #endif
1135 
1136 /*
1137  * Note the generic OProfile stop calls do not support returning
1138  * an error on stop. Hence, will not return an error if the FW
1139  * calls fail on stop. Failure to reset the debug bus is not an issue.
1140  * Failure to disable the SPU profiling is not an issue. The FW calls
1141  * to enable the performance counters and debug bus will work even if
1142  * the hardware was not cleanly reset.
1143  */
1144 static void cell_global_stop_spu_cycles(void)
1145 {
1146  int subfunc, rtn_value;
1147  unsigned int lfsr_value;
1148  int cpu;
1149 
1150  oprofile_running = 0;
1151  smp_wmb();
1152 
1153 #ifdef CONFIG_CPU_FREQ
1154  cpufreq_unregister_notifier(&cpu_freq_notifier_block,
1156 #endif
1157 
1158  for_each_online_cpu(cpu) {
1159  if (cbe_get_hw_thread_id(cpu))
1160  continue;
1161 
1162  subfunc = 3; /*
1163  * 2 - activate SPU tracing,
1164  * 3 - deactivate
1165  */
1166  lfsr_value = 0x8f100000;
1167 
1168  rtn_value = rtas_call(spu_rtas_token, 3, 1, NULL,
1169  subfunc, cbe_cpu_to_node(cpu),
1170  lfsr_value);
1171 
1172  if (unlikely(rtn_value != 0)) {
1174  "%s: rtas call ibm,cbe-spu-perftools " \
1175  "failed, return = %d\n",
1176  __func__, rtn_value);
1177  }
1178 
1179  /* Deactivate the signals */
1180  pm_rtas_reset_signals(cbe_cpu_to_node(cpu));
1181  }
1182 
1184 }
1185 
1186 static void cell_global_stop_spu_events(void)
1187 {
1188  int cpu;
1189  oprofile_running = 0;
1190 
1192  smp_wmb();
1193 
1194  for_each_online_cpu(cpu) {
1195  if (cbe_get_hw_thread_id(cpu))
1196  continue;
1197 
1199  /* Stop the counters */
1200  cbe_disable_pm(cpu);
1201  cbe_write_pm07_control(cpu, 0, 0);
1202 
1203  /* Deactivate the signals */
1204  pm_rtas_reset_signals(cbe_cpu_to_node(cpu));
1205 
1206  /* Deactivate interrupts */
1208  }
1209  del_timer_sync(&timer_spu_event_swap);
1210 }
1211 
1212 static void cell_global_stop_ppu(void)
1213 {
1214  int cpu;
1215 
1216  /*
1217  * This routine will be called once for the system.
1218  * There is one performance monitor per node, so we
1219  * only need to perform this function once per node.
1220  */
1221  del_timer_sync(&timer_virt_cntr);
1222  oprofile_running = 0;
1223  smp_wmb();
1224 
1225  for_each_online_cpu(cpu) {
1226  if (cbe_get_hw_thread_id(cpu))
1227  continue;
1228 
1230  /* Stop the counters */
1231  cbe_disable_pm(cpu);
1232 
1233  /* Deactivate the signals */
1234  pm_rtas_reset_signals(cbe_cpu_to_node(cpu));
1235 
1236  /* Deactivate interrupts */
1238  }
1239 }
1240 
1241 static void cell_global_stop(void)
1242 {
1243  if (profiling_mode == PPU_PROFILING)
1244  cell_global_stop_ppu();
1245  else if (profiling_mode == SPU_PROFILING_EVENTS)
1246  cell_global_stop_spu_events();
1247  else
1248  cell_global_stop_spu_cycles();
1249 }
1250 
1251 static int cell_global_start_spu_cycles(struct op_counter_config *ctr)
1252 {
1253  int subfunc;
1254  unsigned int lfsr_value;
1255  int cpu;
1256  int ret;
1257  int rtas_error;
1258  unsigned int cpu_khzfreq = 0;
1259 
1260  /* The SPU profiling uses time-based profiling based on
1261  * cpu frequency, so if configured with the CPU_FREQ
1262  * option, we should detect frequency changes and react
1263  * accordingly.
1264  */
1265 #ifdef CONFIG_CPU_FREQ
1266  ret = cpufreq_register_notifier(&cpu_freq_notifier_block,
1268  if (ret < 0)
1269  /* this is not a fatal error */
1270  printk(KERN_ERR "CPU freq change registration failed: %d\n",
1271  ret);
1272 
1273  else
1274  cpu_khzfreq = cpufreq_quick_get(smp_processor_id());
1275 #endif
1276 
1277  set_spu_profiling_frequency(cpu_khzfreq, spu_cycle_reset);
1278 
1279  for_each_online_cpu(cpu) {
1280  if (cbe_get_hw_thread_id(cpu))
1281  continue;
1282 
1283  /*
1284  * Setup SPU cycle-based profiling.
1285  * Set perf_mon_control bit 0 to a zero before
1286  * enabling spu collection hardware.
1287  */
1288  cbe_write_pm(cpu, pm_control, 0);
1289 
1290  if (spu_cycle_reset > MAX_SPU_COUNT)
1291  /* use largest possible value */
1292  lfsr_value = calculate_lfsr(MAX_SPU_COUNT-1);
1293  else
1294  lfsr_value = calculate_lfsr(spu_cycle_reset);
1295 
1296  /* must use a non zero value. Zero disables data collection. */
1297  if (lfsr_value == 0)
1298  lfsr_value = calculate_lfsr(1);
1299 
1300  lfsr_value = lfsr_value << 8; /* shift lfsr to correct
1301  * register location
1302  */
1303 
1304  /* debug bus setup */
1305  ret = pm_rtas_activate_spu_profiling(cbe_cpu_to_node(cpu));
1306 
1307  if (unlikely(ret)) {
1308  rtas_error = ret;
1309  goto out;
1310  }
1311 
1312 
1313  subfunc = 2; /* 2 - activate SPU tracing, 3 - deactivate */
1314 
1315  /* start profiling */
1316  ret = rtas_call(spu_rtas_token, 3, 1, NULL, subfunc,
1317  cbe_cpu_to_node(cpu), lfsr_value);
1318 
1319  if (unlikely(ret != 0)) {
1321  "%s: rtas call ibm,cbe-spu-perftools failed, " \
1322  "return = %d\n", __func__, ret);
1323  rtas_error = -EIO;
1324  goto out;
1325  }
1326  }
1327 
1328  rtas_error = start_spu_profiling_cycles(spu_cycle_reset);
1329  if (rtas_error)
1330  goto out_stop;
1331 
1332  oprofile_running = 1;
1333  return 0;
1334 
1335 out_stop:
1336  cell_global_stop_spu_cycles(); /* clean up the PMU/debug bus */
1337 out:
1338  return rtas_error;
1339 }
1340 
1341 static int cell_global_start_spu_events(struct op_counter_config *ctr)
1342 {
1343  int cpu;
1344  u32 interrupt_mask = 0;
1345  int rtn = 0;
1346 
1347  hdw_thread = 0;
1348 
1349  /* spu event profiling, uses the performance counters to generate
1350  * an interrupt. The hardware is setup to store the SPU program
1351  * counter into the trace array. The occurrence mode is used to
1352  * enable storing data to the trace buffer. The bits are set
1353  * to send/store the SPU address in the trace buffer. The debug
1354  * bus must be setup to route the SPU program counter onto the
1355  * debug bus. The occurrence data in the trace buffer is not used.
1356  */
1357 
1358  /* This routine gets called once for the system.
1359  * There is one performance monitor per node, so we
1360  * only need to perform this function once per node.
1361  */
1362 
1363  for_each_online_cpu(cpu) {
1364  if (cbe_get_hw_thread_id(cpu))
1365  continue;
1366 
1367  /*
1368  * Setup SPU event-based profiling.
1369  * Set perf_mon_control bit 0 to a zero before
1370  * enabling spu collection hardware.
1371  *
1372  * Only support one SPU event on one SPU per node.
1373  */
1374  if (ctr_enabled & 1) {
1375  cbe_write_ctr(cpu, 0, reset_value[0]);
1376  enable_ctr(cpu, 0, pm_regs.pm07_cntrl);
1377  interrupt_mask |=
1379  } else {
1380  /* Disable counter */
1381  cbe_write_pm07_control(cpu, 0, 0);
1382  }
1383 
1385  cbe_enable_pm_interrupts(cpu, hdw_thread, interrupt_mask);
1386  cbe_enable_pm(cpu);
1387 
1388  /* clear the trace buffer */
1389  cbe_write_pm(cpu, trace_address, 0);
1390  }
1391 
1392  /* Start the timer to time slice collecting the event profile
1393  * on each of the SPUs. Note, can collect profile on one SPU
1394  * per node at a time.
1395  */
1396  start_spu_event_swap();
1398  oprofile_running = 1;
1399  smp_wmb();
1400 
1401  return rtn;
1402 }
1403 
1404 static int cell_global_start_ppu(struct op_counter_config *ctr)
1405 {
1406  u32 cpu, i;
1407  u32 interrupt_mask = 0;
1408 
1409  /* This routine gets called once for the system.
1410  * There is one performance monitor per node, so we
1411  * only need to perform this function once per node.
1412  */
1413  for_each_online_cpu(cpu) {
1414  if (cbe_get_hw_thread_id(cpu))
1415  continue;
1416 
1417  interrupt_mask = 0;
1418 
1419  for (i = 0; i < num_counters; ++i) {
1420  if (ctr_enabled & (1 << i)) {
1421  cbe_write_ctr(cpu, i, reset_value[i]);
1422  enable_ctr(cpu, i, pm_regs.pm07_cntrl);
1423  interrupt_mask |= CBE_PM_CTR_OVERFLOW_INTR(i);
1424  } else {
1425  /* Disable counter */
1426  cbe_write_pm07_control(cpu, i, 0);
1427  }
1428  }
1429 
1431  cbe_enable_pm_interrupts(cpu, hdw_thread, interrupt_mask);
1432  cbe_enable_pm(cpu);
1433  }
1434 
1435  virt_cntr_inter_mask = interrupt_mask;
1436  oprofile_running = 1;
1437  smp_wmb();
1438 
1439  /*
1440  * NOTE: start_virt_cntrs will result in cell_virtual_cntr() being
1441  * executed which manipulates the PMU. We start the "virtual counter"
1442  * here so that we do not need to synchronize access to the PMU in
1443  * the above for-loop.
1444  */
1445  start_virt_cntrs();
1446 
1447  return 0;
1448 }
1449 
1450 static int cell_global_start(struct op_counter_config *ctr)
1451 {
1452  if (profiling_mode == SPU_PROFILING_CYCLES)
1453  return cell_global_start_spu_cycles(ctr);
1454  else if (profiling_mode == SPU_PROFILING_EVENTS)
1455  return cell_global_start_spu_events(ctr);
1456  else
1457  return cell_global_start_ppu(ctr);
1458 }
1459 
1460 
1461 /* The SPU interrupt handler
1462  *
1463  * SPU event profiling works as follows:
1464  * The pm_signal[0] holds the one SPU event to be measured. It is routed on
1465  * the debug bus using word 0 or 1. The value of pm_signal[1] and
1466  * pm_signal[2] contain the necessary events to route the SPU program
1467  * counter for the selected SPU onto the debug bus using words 2 and 3.
1468  * The pm_interval register is setup to write the SPU PC value into the
1469  * trace buffer at the maximum rate possible. The trace buffer is configured
1470  * to store the PCs, wrapping when it is full. The performance counter is
1471  * initialized to the max hardware count minus the number of events, N, between
1472  * samples. Once the N events have occurred, a HW counter overflow occurs
1473  * causing the generation of a HW counter interrupt which also stops the
1474  * writing of the SPU PC values to the trace buffer. Hence the last PC
1475  * written to the trace buffer is the SPU PC that we want. Unfortunately,
1476  * we have to read from the beginning of the trace buffer to get to the
1477  * last value written. We just hope the PPU has nothing better to do then
1478  * service this interrupt. The PC for the specific SPU being profiled is
1479  * extracted from the trace buffer processed and stored. The trace buffer
1480  * is cleared, interrupts are cleared, the counter is reset to max - N.
1481  * A kernel timer is used to periodically call the routine spu_evnt_swap()
1482  * to switch to the next physical SPU in the node to profile in round robbin
1483  * order. This way data is collected for all SPUs on the node. It does mean
1484  * that we need to use a relatively small value of N to ensure enough samples
1485  * on each SPU are collected each SPU is being profiled 1/8 of the time.
1486  * It may also be necessary to use a longer sample collection period.
1487  */
1488 static void cell_handle_interrupt_spu(struct pt_regs *regs,
1489  struct op_counter_config *ctr)
1490 {
1491  u32 cpu, cpu_tmp;
1492  u64 trace_entry;
1493  u32 interrupt_mask;
1494  u64 trace_buffer[2];
1495  u64 last_trace_buffer;
1496  u32 sample;
1497  u32 trace_addr;
1498  unsigned long sample_array_lock_flags;
1499  int spu_num;
1500  unsigned long flags;
1501 
1502  /* Make sure spu event interrupt handler and spu event swap
1503  * don't access the counters simultaneously.
1504  */
1505  cpu = smp_processor_id();
1506  spin_lock_irqsave(&cntr_lock, flags);
1507 
1508  cpu_tmp = cpu;
1509  cbe_disable_pm(cpu);
1510 
1511  interrupt_mask = cbe_get_and_clear_pm_interrupts(cpu);
1512 
1513  sample = 0xABCDEF;
1514  trace_entry = 0xfedcba;
1515  last_trace_buffer = 0xdeadbeaf;
1516 
1517  if ((oprofile_running == 1) && (interrupt_mask != 0)) {
1518  /* disable writes to trace buff */
1519  cbe_write_pm(cpu, pm_interval, 0);
1520 
1521  /* only have one perf cntr being used, cntr 0 */
1522  if ((interrupt_mask & CBE_PM_CTR_OVERFLOW_INTR(0))
1523  && ctr[0].enabled)
1524  /* The SPU PC values will be read
1525  * from the trace buffer, reset counter
1526  */
1527 
1528  cbe_write_ctr(cpu, 0, reset_value[0]);
1529 
1530  trace_addr = cbe_read_pm(cpu, trace_address);
1531 
1532  while (!(trace_addr & CBE_PM_TRACE_BUF_EMPTY)) {
1533  /* There is data in the trace buffer to process
1534  * Read the buffer until you get to the last
1535  * entry. This is the value we want.
1536  */
1537 
1538  cbe_read_trace_buffer(cpu, trace_buffer);
1539  trace_addr = cbe_read_pm(cpu, trace_address);
1540  }
1541 
1542  /* SPU Address 16 bit count format for 128 bit
1543  * HW trace buffer is used for the SPU PC storage
1544  * HDR bits 0:15
1545  * SPU Addr 0 bits 16:31
1546  * SPU Addr 1 bits 32:47
1547  * unused bits 48:127
1548  *
1549  * HDR: bit4 = 1 SPU Address 0 valid
1550  * HDR: bit5 = 1 SPU Address 1 valid
1551  * - unfortunately, the valid bits don't seem to work
1552  *
1553  * Note trace_buffer[0] holds bits 0:63 of the HW
1554  * trace buffer, trace_buffer[1] holds bits 64:127
1555  */
1556 
1557  trace_entry = trace_buffer[0]
1558  & 0x00000000FFFF0000;
1559 
1560  /* only top 16 of the 18 bit SPU PC address
1561  * is stored in trace buffer, hence shift right
1562  * by 16 -2 bits */
1563  sample = trace_entry >> 14;
1564  last_trace_buffer = trace_buffer[0];
1565 
1566  spu_num = spu_evnt_phys_spu_indx
1568 
1569  /* make sure only one process at a time is calling
1570  * spu_sync_buffer()
1571  */
1572  spin_lock_irqsave(&oprof_spu_smpl_arry_lck,
1573  sample_array_lock_flags);
1574  spu_sync_buffer(spu_num, &sample, 1);
1575  spin_unlock_irqrestore(&oprof_spu_smpl_arry_lck,
1576  sample_array_lock_flags);
1577 
1578  smp_wmb(); /* insure spu event buffer updates are written
1579  * don't want events intermingled... */
1580 
1581  /* The counters were frozen by the interrupt.
1582  * Reenable the interrupt and restart the counters.
1583  */
1585  cbe_enable_pm_interrupts(cpu, hdw_thread,
1586  virt_cntr_inter_mask);
1587 
1588  /* clear the trace buffer, re-enable writes to trace buff */
1589  cbe_write_pm(cpu, trace_address, 0);
1591 
1592  /* The writes to the various performance counters only writes
1593  * to a latch. The new values (interrupt setting bits, reset
1594  * counter value etc.) are not copied to the actual registers
1595  * until the performance monitor is enabled. In order to get
1596  * this to work as desired, the performance monitor needs to
1597  * be disabled while writing to the latches. This is a
1598  * HW design issue.
1599  */
1600  write_pm_cntrl(cpu);
1601  cbe_enable_pm(cpu);
1602  }
1603  spin_unlock_irqrestore(&cntr_lock, flags);
1604 }
1605 
1606 static void cell_handle_interrupt_ppu(struct pt_regs *regs,
1607  struct op_counter_config *ctr)
1608 {
1609  u32 cpu;
1610  u64 pc;
1611  int is_kernel;
1612  unsigned long flags = 0;
1613  u32 interrupt_mask;
1614  int i;
1615 
1616  cpu = smp_processor_id();
1617 
1618  /*
1619  * Need to make sure the interrupt handler and the virt counter
1620  * routine are not running at the same time. See the
1621  * cell_virtual_cntr() routine for additional comments.
1622  */
1623  spin_lock_irqsave(&cntr_lock, flags);
1624 
1625  /*
1626  * Need to disable and reenable the performance counters
1627  * to get the desired behavior from the hardware. This
1628  * is hardware specific.
1629  */
1630 
1631  cbe_disable_pm(cpu);
1632 
1633  interrupt_mask = cbe_get_and_clear_pm_interrupts(cpu);
1634 
1635  /*
1636  * If the interrupt mask has been cleared, then the virt cntr
1637  * has cleared the interrupt. When the thread that generated
1638  * the interrupt is restored, the data count will be restored to
1639  * 0xffffff0 to cause the interrupt to be regenerated.
1640  */
1641 
1642  if ((oprofile_running == 1) && (interrupt_mask != 0)) {
1643  pc = regs->nip;
1644  is_kernel = is_kernel_addr(pc);
1645 
1646  for (i = 0; i < num_counters; ++i) {
1647  if ((interrupt_mask & CBE_PM_CTR_OVERFLOW_INTR(i))
1648  && ctr[i].enabled) {
1649  oprofile_add_ext_sample(pc, regs, i, is_kernel);
1650  cbe_write_ctr(cpu, i, reset_value[i]);
1651  }
1652  }
1653 
1654  /*
1655  * The counters were frozen by the interrupt.
1656  * Reenable the interrupt and restart the counters.
1657  * If there was a race between the interrupt handler and
1658  * the virtual counter routine. The virtual counter
1659  * routine may have cleared the interrupts. Hence must
1660  * use the virt_cntr_inter_mask to re-enable the interrupts.
1661  */
1662  cbe_enable_pm_interrupts(cpu, hdw_thread,
1663  virt_cntr_inter_mask);
1664 
1665  /*
1666  * The writes to the various performance counters only writes
1667  * to a latch. The new values (interrupt setting bits, reset
1668  * counter value etc.) are not copied to the actual registers
1669  * until the performance monitor is enabled. In order to get
1670  * this to work as desired, the performance monitor needs to
1671  * be disabled while writing to the latches. This is a
1672  * HW design issue.
1673  */
1674  cbe_enable_pm(cpu);
1675  }
1676  spin_unlock_irqrestore(&cntr_lock, flags);
1677 }
1678 
1679 static void cell_handle_interrupt(struct pt_regs *regs,
1680  struct op_counter_config *ctr)
1681 {
1682  if (profiling_mode == PPU_PROFILING)
1683  cell_handle_interrupt_ppu(regs, ctr);
1684  else
1685  cell_handle_interrupt_spu(regs, ctr);
1686 }
1687 
1688 /*
1689  * This function is called from the generic OProfile
1690  * driver. When profiling PPUs, we need to do the
1691  * generic sync start; otherwise, do spu_sync_start.
1692  */
1693 static int cell_sync_start(void)
1694 {
1695  if ((profiling_mode == SPU_PROFILING_CYCLES) ||
1696  (profiling_mode == SPU_PROFILING_EVENTS))
1697  return spu_sync_start();
1698  else
1699  return DO_GENERIC_SYNC;
1700 }
1701 
1702 static int cell_sync_stop(void)
1703 {
1704  if ((profiling_mode == SPU_PROFILING_CYCLES) ||
1705  (profiling_mode == SPU_PROFILING_EVENTS))
1706  return spu_sync_stop();
1707  else
1708  return 1;
1709 }
1710 
1711 struct op_powerpc_model op_model_cell = {
1712  .reg_setup = cell_reg_setup,
1713  .cpu_setup = cell_cpu_setup,
1714  .global_start = cell_global_start,
1715  .global_stop = cell_global_stop,
1716  .sync_start = cell_sync_start,
1717  .sync_stop = cell_sync_stop,
1718  .handle_interrupt = cell_handle_interrupt,
1719 };