Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
perf_event_v7.c
Go to the documentation of this file.
1 /*
2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
3  *
4  * ARMv7 support: Jean Pihet <[email protected]>
5  * 2010 (c) MontaVista Software, LLC.
6  *
7  * Copied from ARMv6 code, with the low level code inspired
8  * by the ARMv7 Oprofile code.
9  *
10  * Cortex-A8 has up to 4 configurable performance counters and
11  * a single cycle counter.
12  * Cortex-A9 has up to 31 configurable performance counters and
13  * a single cycle counter.
14  *
15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
16  * counter and all 4 performance counters together can be reset separately.
17  */
18 
19 #ifdef CONFIG_CPU_V7
20 
21 static struct arm_pmu armv7pmu;
22 
23 /*
24  * Common ARMv7 event types
25  *
26  * Note: An implementation may not be able to count all of these events
27  * but the encodings are considered to be `reserved' in the case that
28  * they are not available.
29  */
30 enum armv7_perf_types {
31  ARMV7_PERFCTR_PMNC_SW_INCR = 0x00,
32  ARMV7_PERFCTR_L1_ICACHE_REFILL = 0x01,
33  ARMV7_PERFCTR_ITLB_REFILL = 0x02,
34  ARMV7_PERFCTR_L1_DCACHE_REFILL = 0x03,
35  ARMV7_PERFCTR_L1_DCACHE_ACCESS = 0x04,
36  ARMV7_PERFCTR_DTLB_REFILL = 0x05,
37  ARMV7_PERFCTR_MEM_READ = 0x06,
38  ARMV7_PERFCTR_MEM_WRITE = 0x07,
39  ARMV7_PERFCTR_INSTR_EXECUTED = 0x08,
40  ARMV7_PERFCTR_EXC_TAKEN = 0x09,
41  ARMV7_PERFCTR_EXC_EXECUTED = 0x0A,
42  ARMV7_PERFCTR_CID_WRITE = 0x0B,
43 
44  /*
45  * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
46  * It counts:
47  * - all (taken) branch instructions,
48  * - instructions that explicitly write the PC,
49  * - exception generating instructions.
50  */
51  ARMV7_PERFCTR_PC_WRITE = 0x0C,
52  ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D,
53  ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E,
54  ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS = 0x0F,
55  ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10,
56  ARMV7_PERFCTR_CLOCK_CYCLES = 0x11,
57  ARMV7_PERFCTR_PC_BRANCH_PRED = 0x12,
58 
59  /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
60  ARMV7_PERFCTR_MEM_ACCESS = 0x13,
61  ARMV7_PERFCTR_L1_ICACHE_ACCESS = 0x14,
62  ARMV7_PERFCTR_L1_DCACHE_WB = 0x15,
63  ARMV7_PERFCTR_L2_CACHE_ACCESS = 0x16,
64  ARMV7_PERFCTR_L2_CACHE_REFILL = 0x17,
65  ARMV7_PERFCTR_L2_CACHE_WB = 0x18,
66  ARMV7_PERFCTR_BUS_ACCESS = 0x19,
67  ARMV7_PERFCTR_MEM_ERROR = 0x1A,
68  ARMV7_PERFCTR_INSTR_SPEC = 0x1B,
69  ARMV7_PERFCTR_TTBR_WRITE = 0x1C,
70  ARMV7_PERFCTR_BUS_CYCLES = 0x1D,
71 
72  ARMV7_PERFCTR_CPU_CYCLES = 0xFF
73 };
74 
75 /* ARMv7 Cortex-A8 specific event types */
76 enum armv7_a8_perf_types {
77  ARMV7_A8_PERFCTR_L2_CACHE_ACCESS = 0x43,
78  ARMV7_A8_PERFCTR_L2_CACHE_REFILL = 0x44,
79  ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS = 0x50,
80  ARMV7_A8_PERFCTR_STALL_ISIDE = 0x56,
81 };
82 
83 /* ARMv7 Cortex-A9 specific event types */
84 enum armv7_a9_perf_types {
85  ARMV7_A9_PERFCTR_INSTR_CORE_RENAME = 0x68,
86  ARMV7_A9_PERFCTR_STALL_ICACHE = 0x60,
87  ARMV7_A9_PERFCTR_STALL_DISPATCH = 0x66,
88 };
89 
90 /* ARMv7 Cortex-A5 specific event types */
91 enum armv7_a5_perf_types {
92  ARMV7_A5_PERFCTR_PREFETCH_LINEFILL = 0xc2,
93  ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP = 0xc3,
94 };
95 
96 /* ARMv7 Cortex-A15 specific event types */
97 enum armv7_a15_perf_types {
98  ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ = 0x40,
99  ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE = 0x41,
100  ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ = 0x42,
101  ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE = 0x43,
102 
103  ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ = 0x4C,
104  ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE = 0x4D,
105 
106  ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ = 0x50,
107  ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE = 0x51,
108  ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ = 0x52,
109  ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE = 0x53,
110 
111  ARMV7_A15_PERFCTR_PC_WRITE_SPEC = 0x76,
112 };
113 
114 /*
115  * Cortex-A8 HW events mapping
116  *
117  * The hardware events that we support. We do support cache operations but
118  * we have harvard caches and no way to combine instruction and data
119  * accesses/misses in hardware.
120  */
121 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
122  [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
123  [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
124  [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
125  [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
126  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
127  [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
128  [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
129  [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
130  [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED,
131 };
132 
133 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
134  [PERF_COUNT_HW_CACHE_OP_MAX]
135  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
136  [C(L1D)] = {
137  /*
138  * The performance counters don't differentiate between read
139  * and write accesses/misses so this isn't strictly correct,
140  * but it's the best we can do. Writes and reads get
141  * combined.
142  */
143  [C(OP_READ)] = {
144  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
145  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
146  },
147  [C(OP_WRITE)] = {
148  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
149  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
150  },
151  [C(OP_PREFETCH)] = {
152  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
153  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
154  },
155  },
156  [C(L1I)] = {
157  [C(OP_READ)] = {
158  [C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
159  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
160  },
161  [C(OP_WRITE)] = {
162  [C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
163  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
164  },
165  [C(OP_PREFETCH)] = {
166  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
167  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
168  },
169  },
170  [C(LL)] = {
171  [C(OP_READ)] = {
172  [C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
173  [C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
174  },
175  [C(OP_WRITE)] = {
176  [C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
177  [C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
178  },
179  [C(OP_PREFETCH)] = {
180  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
181  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
182  },
183  },
184  [C(DTLB)] = {
185  [C(OP_READ)] = {
186  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
187  [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
188  },
189  [C(OP_WRITE)] = {
190  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
191  [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
192  },
193  [C(OP_PREFETCH)] = {
194  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
195  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
196  },
197  },
198  [C(ITLB)] = {
199  [C(OP_READ)] = {
200  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
201  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
202  },
203  [C(OP_WRITE)] = {
204  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
205  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
206  },
207  [C(OP_PREFETCH)] = {
208  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
209  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
210  },
211  },
212  [C(BPU)] = {
213  [C(OP_READ)] = {
214  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
215  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
216  },
217  [C(OP_WRITE)] = {
218  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
219  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
220  },
221  [C(OP_PREFETCH)] = {
222  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
223  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
224  },
225  },
226  [C(NODE)] = {
227  [C(OP_READ)] = {
228  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
229  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
230  },
231  [C(OP_WRITE)] = {
232  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
233  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
234  },
235  [C(OP_PREFETCH)] = {
236  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
237  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
238  },
239  },
240 };
241 
242 /*
243  * Cortex-A9 HW events mapping
244  */
245 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
246  [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
247  [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
248  [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
249  [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
250  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
251  [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
252  [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
253  [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
254  [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH,
255 };
256 
257 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
258  [PERF_COUNT_HW_CACHE_OP_MAX]
259  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
260  [C(L1D)] = {
261  /*
262  * The performance counters don't differentiate between read
263  * and write accesses/misses so this isn't strictly correct,
264  * but it's the best we can do. Writes and reads get
265  * combined.
266  */
267  [C(OP_READ)] = {
268  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
269  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
270  },
271  [C(OP_WRITE)] = {
272  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
273  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
274  },
275  [C(OP_PREFETCH)] = {
276  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
277  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
278  },
279  },
280  [C(L1I)] = {
281  [C(OP_READ)] = {
282  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
283  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
284  },
285  [C(OP_WRITE)] = {
286  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
287  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
288  },
289  [C(OP_PREFETCH)] = {
290  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
291  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
292  },
293  },
294  [C(LL)] = {
295  [C(OP_READ)] = {
296  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
297  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
298  },
299  [C(OP_WRITE)] = {
300  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
301  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
302  },
303  [C(OP_PREFETCH)] = {
304  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
305  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
306  },
307  },
308  [C(DTLB)] = {
309  [C(OP_READ)] = {
310  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
311  [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
312  },
313  [C(OP_WRITE)] = {
314  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
315  [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
316  },
317  [C(OP_PREFETCH)] = {
318  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
319  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
320  },
321  },
322  [C(ITLB)] = {
323  [C(OP_READ)] = {
324  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
325  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
326  },
327  [C(OP_WRITE)] = {
328  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
329  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
330  },
331  [C(OP_PREFETCH)] = {
332  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
333  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
334  },
335  },
336  [C(BPU)] = {
337  [C(OP_READ)] = {
338  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
339  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
340  },
341  [C(OP_WRITE)] = {
342  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
343  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
344  },
345  [C(OP_PREFETCH)] = {
346  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
347  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
348  },
349  },
350  [C(NODE)] = {
351  [C(OP_READ)] = {
352  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
353  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
354  },
355  [C(OP_WRITE)] = {
356  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
357  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
358  },
359  [C(OP_PREFETCH)] = {
360  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
361  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
362  },
363  },
364 };
365 
366 /*
367  * Cortex-A5 HW events mapping
368  */
369 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
370  [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
371  [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
372  [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
373  [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
374  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
375  [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
376  [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED,
377  [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
378  [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED,
379 };
380 
381 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
382  [PERF_COUNT_HW_CACHE_OP_MAX]
383  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
384  [C(L1D)] = {
385  [C(OP_READ)] = {
386  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
387  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
388  },
389  [C(OP_WRITE)] = {
390  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
391  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
392  },
393  [C(OP_PREFETCH)] = {
394  [C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
395  [C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
396  },
397  },
398  [C(L1I)] = {
399  [C(OP_READ)] = {
400  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
401  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
402  },
403  [C(OP_WRITE)] = {
404  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
405  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
406  },
407  /*
408  * The prefetch counters don't differentiate between the I
409  * side and the D side.
410  */
411  [C(OP_PREFETCH)] = {
412  [C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
413  [C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
414  },
415  },
416  [C(LL)] = {
417  [C(OP_READ)] = {
418  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
419  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
420  },
421  [C(OP_WRITE)] = {
422  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
423  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
424  },
425  [C(OP_PREFETCH)] = {
426  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
427  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
428  },
429  },
430  [C(DTLB)] = {
431  [C(OP_READ)] = {
432  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
433  [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
434  },
435  [C(OP_WRITE)] = {
436  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
437  [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
438  },
439  [C(OP_PREFETCH)] = {
440  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
441  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
442  },
443  },
444  [C(ITLB)] = {
445  [C(OP_READ)] = {
446  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
447  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
448  },
449  [C(OP_WRITE)] = {
450  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
451  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
452  },
453  [C(OP_PREFETCH)] = {
454  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
455  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
456  },
457  },
458  [C(BPU)] = {
459  [C(OP_READ)] = {
460  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
461  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
462  },
463  [C(OP_WRITE)] = {
464  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
465  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
466  },
467  [C(OP_PREFETCH)] = {
468  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
469  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
470  },
471  },
472  [C(NODE)] = {
473  [C(OP_READ)] = {
474  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
475  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
476  },
477  [C(OP_WRITE)] = {
478  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
479  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
480  },
481  [C(OP_PREFETCH)] = {
482  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
483  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
484  },
485  },
486 };
487 
488 /*
489  * Cortex-A15 HW events mapping
490  */
491 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
492  [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
493  [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
494  [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
495  [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
496  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
497  [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
498  [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
499  [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
500  [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED,
501 };
502 
503 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
504  [PERF_COUNT_HW_CACHE_OP_MAX]
505  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
506  [C(L1D)] = {
507  [C(OP_READ)] = {
508  [C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
509  [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
510  },
511  [C(OP_WRITE)] = {
512  [C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
513  [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
514  },
515  [C(OP_PREFETCH)] = {
516  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
517  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
518  },
519  },
520  [C(L1I)] = {
521  /*
522  * Not all performance counters differentiate between read
523  * and write accesses/misses so we're not always strictly
524  * correct, but it's the best we can do. Writes and reads get
525  * combined in these cases.
526  */
527  [C(OP_READ)] = {
528  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
529  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
530  },
531  [C(OP_WRITE)] = {
532  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
533  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
534  },
535  [C(OP_PREFETCH)] = {
536  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
537  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
538  },
539  },
540  [C(LL)] = {
541  [C(OP_READ)] = {
542  [C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
543  [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
544  },
545  [C(OP_WRITE)] = {
546  [C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
547  [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
548  },
549  [C(OP_PREFETCH)] = {
550  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
551  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
552  },
553  },
554  [C(DTLB)] = {
555  [C(OP_READ)] = {
556  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
557  [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
558  },
559  [C(OP_WRITE)] = {
560  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
561  [C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
562  },
563  [C(OP_PREFETCH)] = {
564  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
565  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
566  },
567  },
568  [C(ITLB)] = {
569  [C(OP_READ)] = {
570  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
571  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
572  },
573  [C(OP_WRITE)] = {
574  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
575  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
576  },
577  [C(OP_PREFETCH)] = {
578  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
579  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
580  },
581  },
582  [C(BPU)] = {
583  [C(OP_READ)] = {
584  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
585  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
586  },
587  [C(OP_WRITE)] = {
588  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
589  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
590  },
591  [C(OP_PREFETCH)] = {
592  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
593  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
594  },
595  },
596  [C(NODE)] = {
597  [C(OP_READ)] = {
598  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
599  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
600  },
601  [C(OP_WRITE)] = {
602  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
603  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
604  },
605  [C(OP_PREFETCH)] = {
606  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
607  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
608  },
609  },
610 };
611 
612 /*
613  * Cortex-A7 HW events mapping
614  */
615 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
616  [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES,
617  [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED,
618  [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
619  [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
620  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
621  [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
622  [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES,
623  [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
624  [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED,
625 };
626 
627 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
628  [PERF_COUNT_HW_CACHE_OP_MAX]
629  [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
630  [C(L1D)] = {
631  /*
632  * The performance counters don't differentiate between read
633  * and write accesses/misses so this isn't strictly correct,
634  * but it's the best we can do. Writes and reads get
635  * combined.
636  */
637  [C(OP_READ)] = {
638  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
639  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
640  },
641  [C(OP_WRITE)] = {
642  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
643  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
644  },
645  [C(OP_PREFETCH)] = {
646  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
647  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
648  },
649  },
650  [C(L1I)] = {
651  [C(OP_READ)] = {
652  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
653  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
654  },
655  [C(OP_WRITE)] = {
656  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
657  [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL,
658  },
659  [C(OP_PREFETCH)] = {
660  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
661  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
662  },
663  },
664  [C(LL)] = {
665  [C(OP_READ)] = {
666  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
667  [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
668  },
669  [C(OP_WRITE)] = {
670  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS,
671  [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL,
672  },
673  [C(OP_PREFETCH)] = {
674  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
675  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
676  },
677  },
678  [C(DTLB)] = {
679  [C(OP_READ)] = {
680  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
681  [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
682  },
683  [C(OP_WRITE)] = {
684  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
685  [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL,
686  },
687  [C(OP_PREFETCH)] = {
688  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
689  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
690  },
691  },
692  [C(ITLB)] = {
693  [C(OP_READ)] = {
694  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
695  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
696  },
697  [C(OP_WRITE)] = {
698  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
699  [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL,
700  },
701  [C(OP_PREFETCH)] = {
702  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
703  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
704  },
705  },
706  [C(BPU)] = {
707  [C(OP_READ)] = {
708  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
709  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
710  },
711  [C(OP_WRITE)] = {
712  [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
713  [C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
714  },
715  [C(OP_PREFETCH)] = {
716  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
717  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
718  },
719  },
720  [C(NODE)] = {
721  [C(OP_READ)] = {
722  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
723  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
724  },
725  [C(OP_WRITE)] = {
726  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
727  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
728  },
729  [C(OP_PREFETCH)] = {
730  [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED,
731  [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED,
732  },
733  },
734 };
735 
736 /*
737  * Perf Events' indices
738  */
739 #define ARMV7_IDX_CYCLE_COUNTER 0
740 #define ARMV7_IDX_COUNTER0 1
741 #define ARMV7_IDX_COUNTER_LAST (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
742 
743 #define ARMV7_MAX_COUNTERS 32
744 #define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1)
745 
746 /*
747  * ARMv7 low level PMNC access
748  */
749 
750 /*
751  * Perf Event to low level counters mapping
752  */
753 #define ARMV7_IDX_TO_COUNTER(x) \
754  (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
755 
756 /*
757  * Per-CPU PMNC: config reg
758  */
759 #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */
760 #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */
761 #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */
762 #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */
763 #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */
764 #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/
765 #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */
766 #define ARMV7_PMNC_N_MASK 0x1f
767 #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */
768 
769 /*
770  * FLAG: counters overflow flag status reg
771  */
772 #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */
773 #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK
774 
775 /*
776  * PMXEVTYPER: Event selection reg
777  */
778 #define ARMV7_EVTYPE_MASK 0xc00000ff /* Mask for writable bits */
779 #define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */
780 
781 /*
782  * Event filters for PMUv2
783  */
784 #define ARMV7_EXCLUDE_PL1 (1 << 31)
785 #define ARMV7_EXCLUDE_USER (1 << 30)
786 #define ARMV7_INCLUDE_HYP (1 << 27)
787 
788 static inline u32 armv7_pmnc_read(void)
789 {
790  u32 val;
791  asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
792  return val;
793 }
794 
795 static inline void armv7_pmnc_write(u32 val)
796 {
797  val &= ARMV7_PMNC_MASK;
798  isb();
799  asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
800 }
801 
802 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
803 {
804  return pmnc & ARMV7_OVERFLOWED_MASK;
805 }
806 
807 static inline int armv7_pmnc_counter_valid(int idx)
808 {
809  return idx >= ARMV7_IDX_CYCLE_COUNTER && idx <= ARMV7_IDX_COUNTER_LAST;
810 }
811 
812 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
813 {
814  int ret = 0;
815  u32 counter;
816 
817  if (!armv7_pmnc_counter_valid(idx)) {
818  pr_err("CPU%u checking wrong counter %d overflow status\n",
819  smp_processor_id(), idx);
820  } else {
821  counter = ARMV7_IDX_TO_COUNTER(idx);
822  ret = pmnc & BIT(counter);
823  }
824 
825  return ret;
826 }
827 
828 static inline int armv7_pmnc_select_counter(int idx)
829 {
830  u32 counter;
831 
832  if (!armv7_pmnc_counter_valid(idx)) {
833  pr_err("CPU%u selecting wrong PMNC counter %d\n",
834  smp_processor_id(), idx);
835  return -EINVAL;
836  }
837 
838  counter = ARMV7_IDX_TO_COUNTER(idx);
839  asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
840  isb();
841 
842  return idx;
843 }
844 
845 static inline u32 armv7pmu_read_counter(int idx)
846 {
847  u32 value = 0;
848 
849  if (!armv7_pmnc_counter_valid(idx))
850  pr_err("CPU%u reading wrong counter %d\n",
851  smp_processor_id(), idx);
852  else if (idx == ARMV7_IDX_CYCLE_COUNTER)
853  asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
854  else if (armv7_pmnc_select_counter(idx) == idx)
855  asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
856 
857  return value;
858 }
859 
860 static inline void armv7pmu_write_counter(int idx, u32 value)
861 {
862  if (!armv7_pmnc_counter_valid(idx))
863  pr_err("CPU%u writing wrong counter %d\n",
864  smp_processor_id(), idx);
865  else if (idx == ARMV7_IDX_CYCLE_COUNTER)
866  asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
867  else if (armv7_pmnc_select_counter(idx) == idx)
868  asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
869 }
870 
871 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
872 {
873  if (armv7_pmnc_select_counter(idx) == idx) {
874  val &= ARMV7_EVTYPE_MASK;
875  asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
876  }
877 }
878 
879 static inline int armv7_pmnc_enable_counter(int idx)
880 {
881  u32 counter;
882 
883  if (!armv7_pmnc_counter_valid(idx)) {
884  pr_err("CPU%u enabling wrong PMNC counter %d\n",
885  smp_processor_id(), idx);
886  return -EINVAL;
887  }
888 
889  counter = ARMV7_IDX_TO_COUNTER(idx);
890  asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
891  return idx;
892 }
893 
894 static inline int armv7_pmnc_disable_counter(int idx)
895 {
896  u32 counter;
897 
898  if (!armv7_pmnc_counter_valid(idx)) {
899  pr_err("CPU%u disabling wrong PMNC counter %d\n",
900  smp_processor_id(), idx);
901  return -EINVAL;
902  }
903 
904  counter = ARMV7_IDX_TO_COUNTER(idx);
905  asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
906  return idx;
907 }
908 
909 static inline int armv7_pmnc_enable_intens(int idx)
910 {
911  u32 counter;
912 
913  if (!armv7_pmnc_counter_valid(idx)) {
914  pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
915  smp_processor_id(), idx);
916  return -EINVAL;
917  }
918 
919  counter = ARMV7_IDX_TO_COUNTER(idx);
920  asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
921  return idx;
922 }
923 
924 static inline int armv7_pmnc_disable_intens(int idx)
925 {
926  u32 counter;
927 
928  if (!armv7_pmnc_counter_valid(idx)) {
929  pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
930  smp_processor_id(), idx);
931  return -EINVAL;
932  }
933 
934  counter = ARMV7_IDX_TO_COUNTER(idx);
935  asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
936  isb();
937  /* Clear the overflow flag in case an interrupt is pending. */
938  asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
939  isb();
940 
941  return idx;
942 }
943 
944 static inline u32 armv7_pmnc_getreset_flags(void)
945 {
946  u32 val;
947 
948  /* Read */
949  asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
950 
951  /* Write to clear flags */
952  val &= ARMV7_FLAG_MASK;
953  asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
954 
955  return val;
956 }
957 
958 #ifdef DEBUG
959 static void armv7_pmnc_dump_regs(void)
960 {
961  u32 val;
962  unsigned int cnt;
963 
964  printk(KERN_INFO "PMNC registers dump:\n");
965 
966  asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
967  printk(KERN_INFO "PMNC =0x%08x\n", val);
968 
969  asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
970  printk(KERN_INFO "CNTENS=0x%08x\n", val);
971 
972  asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
973  printk(KERN_INFO "INTENS=0x%08x\n", val);
974 
975  asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
976  printk(KERN_INFO "FLAGS =0x%08x\n", val);
977 
978  asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
979  printk(KERN_INFO "SELECT=0x%08x\n", val);
980 
981  asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
982  printk(KERN_INFO "CCNT =0x%08x\n", val);
983 
984  for (cnt = ARMV7_IDX_COUNTER0; cnt <= ARMV7_IDX_COUNTER_LAST; cnt++) {
985  armv7_pmnc_select_counter(cnt);
986  asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
987  printk(KERN_INFO "CNT[%d] count =0x%08x\n",
988  ARMV7_IDX_TO_COUNTER(cnt), val);
989  asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
990  printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
991  ARMV7_IDX_TO_COUNTER(cnt), val);
992  }
993 }
994 #endif
995 
996 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
997 {
998  unsigned long flags;
999  struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1000 
1001  /*
1002  * Enable counter and interrupt, and set the counter to count
1003  * the event that we're interested in.
1004  */
1005  raw_spin_lock_irqsave(&events->pmu_lock, flags);
1006 
1007  /*
1008  * Disable counter
1009  */
1010  armv7_pmnc_disable_counter(idx);
1011 
1012  /*
1013  * Set event (if destined for PMNx counters)
1014  * We only need to set the event for the cycle counter if we
1015  * have the ability to perform event filtering.
1016  */
1017  if (armv7pmu.set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
1018  armv7_pmnc_write_evtsel(idx, hwc->config_base);
1019 
1020  /*
1021  * Enable interrupt for this counter
1022  */
1023  armv7_pmnc_enable_intens(idx);
1024 
1025  /*
1026  * Enable counter
1027  */
1028  armv7_pmnc_enable_counter(idx);
1029 
1030  raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1031 }
1032 
1033 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
1034 {
1035  unsigned long flags;
1036  struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1037 
1038  /*
1039  * Disable counter and interrupt
1040  */
1041  raw_spin_lock_irqsave(&events->pmu_lock, flags);
1042 
1043  /*
1044  * Disable counter
1045  */
1046  armv7_pmnc_disable_counter(idx);
1047 
1048  /*
1049  * Disable interrupt for this counter
1050  */
1051  armv7_pmnc_disable_intens(idx);
1052 
1053  raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1054 }
1055 
1056 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
1057 {
1058  u32 pmnc;
1059  struct perf_sample_data data;
1060  struct pmu_hw_events *cpuc;
1061  struct pt_regs *regs;
1062  int idx;
1063 
1064  /*
1065  * Get and reset the IRQ flags
1066  */
1067  pmnc = armv7_pmnc_getreset_flags();
1068 
1069  /*
1070  * Did an overflow occur?
1071  */
1072  if (!armv7_pmnc_has_overflowed(pmnc))
1073  return IRQ_NONE;
1074 
1075  /*
1076  * Handle the counter(s) overflow(s)
1077  */
1078  regs = get_irq_regs();
1079 
1080  cpuc = &__get_cpu_var(cpu_hw_events);
1081  for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
1082  struct perf_event *event = cpuc->events[idx];
1083  struct hw_perf_event *hwc;
1084 
1085  /* Ignore if we don't have an event. */
1086  if (!event)
1087  continue;
1088 
1089  /*
1090  * We have a single interrupt for all counters. Check that
1091  * each counter has overflowed before we process it.
1092  */
1093  if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
1094  continue;
1095 
1096  hwc = &event->hw;
1097  armpmu_event_update(event, hwc, idx);
1098  perf_sample_data_init(&data, 0, hwc->last_period);
1099  if (!armpmu_event_set_period(event, hwc, idx))
1100  continue;
1101 
1102  if (perf_event_overflow(event, &data, regs))
1103  cpu_pmu->disable(hwc, idx);
1104  }
1105 
1106  /*
1107  * Handle the pending perf events.
1108  *
1109  * Note: this call *must* be run with interrupts disabled. For
1110  * platforms that can have the PMU interrupts raised as an NMI, this
1111  * will not work.
1112  */
1113  irq_work_run();
1114 
1115  return IRQ_HANDLED;
1116 }
1117 
1118 static void armv7pmu_start(void)
1119 {
1120  unsigned long flags;
1121  struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1122 
1123  raw_spin_lock_irqsave(&events->pmu_lock, flags);
1124  /* Enable all counters */
1125  armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
1126  raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1127 }
1128 
1129 static void armv7pmu_stop(void)
1130 {
1131  unsigned long flags;
1132  struct pmu_hw_events *events = cpu_pmu->get_hw_events();
1133 
1134  raw_spin_lock_irqsave(&events->pmu_lock, flags);
1135  /* Disable all counters */
1136  armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
1137  raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1138 }
1139 
1140 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
1141  struct hw_perf_event *event)
1142 {
1143  int idx;
1144  unsigned long evtype = event->config_base & ARMV7_EVTYPE_EVENT;
1145 
1146  /* Always place a cycle counter into the cycle counter. */
1147  if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
1148  if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
1149  return -EAGAIN;
1150 
1151  return ARMV7_IDX_CYCLE_COUNTER;
1152  }
1153 
1154  /*
1155  * For anything other than a cycle counter, try and use
1156  * the events counters
1157  */
1158  for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1159  if (!test_and_set_bit(idx, cpuc->used_mask))
1160  return idx;
1161  }
1162 
1163  /* The counters are all in use. */
1164  return -EAGAIN;
1165 }
1166 
1167 /*
1168  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1169  */
1170 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
1171  struct perf_event_attr *attr)
1172 {
1173  unsigned long config_base = 0;
1174 
1175  if (attr->exclude_idle)
1176  return -EPERM;
1177  if (attr->exclude_user)
1178  config_base |= ARMV7_EXCLUDE_USER;
1179  if (attr->exclude_kernel)
1180  config_base |= ARMV7_EXCLUDE_PL1;
1181  if (!attr->exclude_hv)
1182  config_base |= ARMV7_INCLUDE_HYP;
1183 
1184  /*
1185  * Install the filter into config_base as this is used to
1186  * construct the event type.
1187  */
1188  event->config_base = config_base;
1189 
1190  return 0;
1191 }
1192 
1193 static void armv7pmu_reset(void *info)
1194 {
1195  u32 idx, nb_cnt = cpu_pmu->num_events;
1196 
1197  /* The counter and interrupt enable registers are unknown at reset. */
1198  for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx)
1199  armv7pmu_disable_event(NULL, idx);
1200 
1201  /* Initialize & Reset PMNC: C and P bits */
1202  armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
1203 }
1204 
1205 static int armv7_a8_map_event(struct perf_event *event)
1206 {
1207  return armpmu_map_event(event, &armv7_a8_perf_map,
1208  &armv7_a8_perf_cache_map, 0xFF);
1209 }
1210 
1211 static int armv7_a9_map_event(struct perf_event *event)
1212 {
1213  return armpmu_map_event(event, &armv7_a9_perf_map,
1214  &armv7_a9_perf_cache_map, 0xFF);
1215 }
1216 
1217 static int armv7_a5_map_event(struct perf_event *event)
1218 {
1219  return armpmu_map_event(event, &armv7_a5_perf_map,
1220  &armv7_a5_perf_cache_map, 0xFF);
1221 }
1222 
1223 static int armv7_a15_map_event(struct perf_event *event)
1224 {
1225  return armpmu_map_event(event, &armv7_a15_perf_map,
1226  &armv7_a15_perf_cache_map, 0xFF);
1227 }
1228 
1229 static int armv7_a7_map_event(struct perf_event *event)
1230 {
1231  return armpmu_map_event(event, &armv7_a7_perf_map,
1232  &armv7_a7_perf_cache_map, 0xFF);
1233 }
1234 
1235 static struct arm_pmu armv7pmu = {
1236  .handle_irq = armv7pmu_handle_irq,
1237  .enable = armv7pmu_enable_event,
1238  .disable = armv7pmu_disable_event,
1239  .read_counter = armv7pmu_read_counter,
1240  .write_counter = armv7pmu_write_counter,
1241  .get_event_idx = armv7pmu_get_event_idx,
1242  .start = armv7pmu_start,
1243  .stop = armv7pmu_stop,
1244  .reset = armv7pmu_reset,
1245  .max_period = (1LLU << 32) - 1,
1246 };
1247 
1248 static u32 __devinit armv7_read_num_pmnc_events(void)
1249 {
1250  u32 nb_cnt;
1251 
1252  /* Read the nb of CNTx counters supported from PMNC */
1253  nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1254 
1255  /* Add the CPU cycles counter and return */
1256  return nb_cnt + 1;
1257 }
1258 
1259 static struct arm_pmu *__devinit armv7_a8_pmu_init(void)
1260 {
1261  armv7pmu.name = "ARMv7 Cortex-A8";
1262  armv7pmu.map_event = armv7_a8_map_event;
1263  armv7pmu.num_events = armv7_read_num_pmnc_events();
1264  return &armv7pmu;
1265 }
1266 
1267 static struct arm_pmu *__devinit armv7_a9_pmu_init(void)
1268 {
1269  armv7pmu.name = "ARMv7 Cortex-A9";
1270  armv7pmu.map_event = armv7_a9_map_event;
1271  armv7pmu.num_events = armv7_read_num_pmnc_events();
1272  return &armv7pmu;
1273 }
1274 
1275 static struct arm_pmu *__devinit armv7_a5_pmu_init(void)
1276 {
1277  armv7pmu.name = "ARMv7 Cortex-A5";
1278  armv7pmu.map_event = armv7_a5_map_event;
1279  armv7pmu.num_events = armv7_read_num_pmnc_events();
1280  return &armv7pmu;
1281 }
1282 
1283 static struct arm_pmu *__devinit armv7_a15_pmu_init(void)
1284 {
1285  armv7pmu.name = "ARMv7 Cortex-A15";
1286  armv7pmu.map_event = armv7_a15_map_event;
1287  armv7pmu.num_events = armv7_read_num_pmnc_events();
1288  armv7pmu.set_event_filter = armv7pmu_set_event_filter;
1289  return &armv7pmu;
1290 }
1291 
1292 static struct arm_pmu *__devinit armv7_a7_pmu_init(void)
1293 {
1294  armv7pmu.name = "ARMv7 Cortex-A7";
1295  armv7pmu.map_event = armv7_a7_map_event;
1296  armv7pmu.num_events = armv7_read_num_pmnc_events();
1297  armv7pmu.set_event_filter = armv7pmu_set_event_filter;
1298  return &armv7pmu;
1299 }
1300 #else
1301 static struct arm_pmu *__devinit armv7_a8_pmu_init(void)
1302 {
1303  return NULL;
1304 }
1305 
1306 static struct arm_pmu *__devinit armv7_a9_pmu_init(void)
1307 {
1308  return NULL;
1309 }
1310 
1311 static struct arm_pmu *__devinit armv7_a5_pmu_init(void)
1312 {
1313  return NULL;
1314 }
1315 
1316 static struct arm_pmu *__devinit armv7_a15_pmu_init(void)
1317 {
1318  return NULL;
1319 }
1320 
1321 static struct arm_pmu *__devinit armv7_a7_pmu_init(void)
1322 {
1323  return NULL;
1324 }
1325 #endif /* CONFIG_CPU_V7 */
Generated on Thu Jan 10 2013 12:56:20 for Linux Kernel by   doxygen 1.8.2