Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
core.c
Go to the documentation of this file.
1 /*
2  * Performance events core code:
3  *
4  * Copyright (C) 2008 Thomas Gleixner <[email protected]>
5  * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6  * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <[email protected]>
7  * Copyright © 2009 Paul Mackerras, IBM Corp. <[email protected]>
8  *
9  * For licensing details see kernel-base/COPYING
10  */
11 
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/idr.h>
17 #include <linux/file.h>
18 #include <linux/poll.h>
19 #include <linux/slab.h>
20 #include <linux/hash.h>
21 #include <linux/sysfs.h>
22 #include <linux/dcache.h>
23 #include <linux/percpu.h>
24 #include <linux/ptrace.h>
25 #include <linux/reboot.h>
26 #include <linux/vmstat.h>
27 #include <linux/device.h>
28 #include <linux/export.h>
29 #include <linux/vmalloc.h>
30 #include <linux/hardirq.h>
31 #include <linux/rculist.h>
32 #include <linux/uaccess.h>
33 #include <linux/syscalls.h>
34 #include <linux/anon_inodes.h>
35 #include <linux/kernel_stat.h>
36 #include <linux/perf_event.h>
37 #include <linux/ftrace_event.h>
38 #include <linux/hw_breakpoint.h>
39 #include <linux/mm_types.h>
40 
41 #include "internal.h"
42 
43 #include <asm/irq_regs.h>
44 
46  struct task_struct *p;
47  int (*func)(void *info);
48  void *info;
49  int ret;
50 };
51 
52 static void remote_function(void *data)
53 {
54  struct remote_function_call *tfc = data;
55  struct task_struct *p = tfc->p;
56 
57  if (p) {
58  tfc->ret = -EAGAIN;
59  if (task_cpu(p) != smp_processor_id() || !task_curr(p))
60  return;
61  }
62 
63  tfc->ret = tfc->func(tfc->info);
64 }
65 
79 static int
80 task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
81 {
82  struct remote_function_call data = {
83  .p = p,
84  .func = func,
85  .info = info,
86  .ret = -ESRCH, /* No such (running) process */
87  };
88 
89  if (task_curr(p))
90  smp_call_function_single(task_cpu(p), remote_function, &data, 1);
91 
92  return data.ret;
93 }
94 
104 static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
105 {
106  struct remote_function_call data = {
107  .p = NULL,
108  .func = func,
109  .info = info,
110  .ret = -ENXIO, /* No such CPU */
111  };
112 
113  smp_call_function_single(cpu, remote_function, &data, 1);
114 
115  return data.ret;
116 }
117 
118 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
119  PERF_FLAG_FD_OUTPUT |\
120  PERF_FLAG_PID_CGROUP)
121 
122 /*
123  * branch priv levels that need permission checks
124  */
125 #define PERF_SAMPLE_BRANCH_PERM_PLM \
126  (PERF_SAMPLE_BRANCH_KERNEL |\
127  PERF_SAMPLE_BRANCH_HV)
128 
133 };
134 
135 /*
136  * perf_sched_events : >0 events exist
137  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
138  */
139 struct static_key_deferred perf_sched_events __read_mostly;
140 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
141 static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
142 
143 static atomic_t nr_mmap_events __read_mostly;
144 static atomic_t nr_comm_events __read_mostly;
145 static atomic_t nr_task_events __read_mostly;
146 
147 static LIST_HEAD(pmus);
148 static DEFINE_MUTEX(pmus_lock);
149 static struct srcu_struct pmus_srcu;
150 
151 /*
152  * perf event paranoia level:
153  * -1 - not paranoid at all
154  * 0 - disallow raw tracepoint access for unpriv
155  * 1 - disallow cpu events for unpriv
156  * 2 - disallow kernel profiling for unpriv
157  */
158 int sysctl_perf_event_paranoid __read_mostly = 1;
159 
160 /* Minimum for 512 kiB + 1 user control page */
161 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
162 
163 /*
164  * max perf event sample rate
165  */
166 #define DEFAULT_MAX_SAMPLE_RATE 100000
167 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
168 static int max_samples_per_tick __read_mostly =
170 
172  void __user *buffer, size_t *lenp,
173  loff_t *ppos)
174 {
175  int ret = proc_dointvec(table, write, buffer, lenp, ppos);
176 
177  if (ret || !write)
178  return ret;
179 
180  max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
181 
182  return 0;
183 }
184 
185 static atomic64_t perf_event_id;
186 
187 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
188  enum event_type_t event_type);
189 
190 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
192  struct task_struct *task);
193 
194 static void update_context_time(struct perf_event_context *ctx);
195 static u64 perf_event_time(struct perf_event *event);
196 
197 static void ring_buffer_attach(struct perf_event *event,
198  struct ring_buffer *rb);
199 
201 
202 extern __weak const char *perf_pmu_name(void)
203 {
204  return "pmu";
205 }
206 
207 static inline u64 perf_clock(void)
208 {
209  return local_clock();
210 }
211 
212 static inline struct perf_cpu_context *
213 __get_cpu_context(struct perf_event_context *ctx)
214 {
215  return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
216 }
217 
218 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
219  struct perf_event_context *ctx)
220 {
221  raw_spin_lock(&cpuctx->ctx.lock);
222  if (ctx)
223  raw_spin_lock(&ctx->lock);
224 }
225 
226 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
227  struct perf_event_context *ctx)
228 {
229  if (ctx)
230  raw_spin_unlock(&ctx->lock);
231  raw_spin_unlock(&cpuctx->ctx.lock);
232 }
233 
234 #ifdef CONFIG_CGROUP_PERF
235 
236 /*
237  * Must ensure cgroup is pinned (css_get) before calling
238  * this function. In other words, we cannot call this function
239  * if there is no cgroup event for the current CPU context.
240  */
241 static inline struct perf_cgroup *
242 perf_cgroup_from_task(struct task_struct *task)
243 {
244  return container_of(task_subsys_state(task, perf_subsys_id),
245  struct perf_cgroup, css);
246 }
247 
248 static inline bool
249 perf_cgroup_match(struct perf_event *event)
250 {
251  struct perf_event_context *ctx = event->ctx;
252  struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
253 
254  return !event->cgrp || event->cgrp == cpuctx->cgrp;
255 }
256 
257 static inline bool perf_tryget_cgroup(struct perf_event *event)
258 {
259  return css_tryget(&event->cgrp->css);
260 }
261 
262 static inline void perf_put_cgroup(struct perf_event *event)
263 {
264  css_put(&event->cgrp->css);
265 }
266 
267 static inline void perf_detach_cgroup(struct perf_event *event)
268 {
269  perf_put_cgroup(event);
270  event->cgrp = NULL;
271 }
272 
273 static inline int is_cgroup_event(struct perf_event *event)
274 {
275  return event->cgrp != NULL;
276 }
277 
278 static inline u64 perf_cgroup_event_time(struct perf_event *event)
279 {
280  struct perf_cgroup_info *t;
281 
282  t = per_cpu_ptr(event->cgrp->info, event->cpu);
283  return t->time;
284 }
285 
286 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
287 {
288  struct perf_cgroup_info *info;
289  u64 now;
290 
291  now = perf_clock();
292 
293  info = this_cpu_ptr(cgrp->info);
294 
295  info->time += now - info->timestamp;
296  info->timestamp = now;
297 }
298 
299 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
300 {
301  struct perf_cgroup *cgrp_out = cpuctx->cgrp;
302  if (cgrp_out)
303  __update_cgrp_time(cgrp_out);
304 }
305 
306 static inline void update_cgrp_time_from_event(struct perf_event *event)
307 {
308  struct perf_cgroup *cgrp;
309 
310  /*
311  * ensure we access cgroup data only when needed and
312  * when we know the cgroup is pinned (css_get)
313  */
314  if (!is_cgroup_event(event))
315  return;
316 
317  cgrp = perf_cgroup_from_task(current);
318  /*
319  * Do not update time when cgroup is not active
320  */
321  if (cgrp == event->cgrp)
322  __update_cgrp_time(event->cgrp);
323 }
324 
325 static inline void
326 perf_cgroup_set_timestamp(struct task_struct *task,
327  struct perf_event_context *ctx)
328 {
329  struct perf_cgroup *cgrp;
330  struct perf_cgroup_info *info;
331 
332  /*
333  * ctx->lock held by caller
334  * ensure we do not access cgroup data
335  * unless we have the cgroup pinned (css_get)
336  */
337  if (!task || !ctx->nr_cgroups)
338  return;
339 
340  cgrp = perf_cgroup_from_task(task);
341  info = this_cpu_ptr(cgrp->info);
342  info->timestamp = ctx->timestamp;
343 }
344 
345 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */
346 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */
347 
348 /*
349  * reschedule events based on the cgroup constraint of task.
350  *
351  * mode SWOUT : schedule out everything
352  * mode SWIN : schedule in based on cgroup for next
353  */
354 void perf_cgroup_switch(struct task_struct *task, int mode)
355 {
356  struct perf_cpu_context *cpuctx;
357  struct pmu *pmu;
358  unsigned long flags;
359 
360  /*
361  * disable interrupts to avoid geting nr_cgroup
362  * changes via __perf_event_disable(). Also
363  * avoids preemption.
364  */
365  local_irq_save(flags);
366 
367  /*
368  * we reschedule only in the presence of cgroup
369  * constrained events.
370  */
371  rcu_read_lock();
372 
373  list_for_each_entry_rcu(pmu, &pmus, entry) {
374  cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
375  if (cpuctx->unique_pmu != pmu)
376  continue; /* ensure we process each cpuctx once */
377 
378  /*
379  * perf_cgroup_events says at least one
380  * context on this CPU has cgroup events.
381  *
382  * ctx->nr_cgroups reports the number of cgroup
383  * events for a context.
384  */
385  if (cpuctx->ctx.nr_cgroups > 0) {
386  perf_ctx_lock(cpuctx, cpuctx->task_ctx);
387  perf_pmu_disable(cpuctx->ctx.pmu);
388 
389  if (mode & PERF_CGROUP_SWOUT) {
390  cpu_ctx_sched_out(cpuctx, EVENT_ALL);
391  /*
392  * must not be done before ctxswout due
393  * to event_filter_match() in event_sched_out()
394  */
395  cpuctx->cgrp = NULL;
396  }
397 
398  if (mode & PERF_CGROUP_SWIN) {
399  WARN_ON_ONCE(cpuctx->cgrp);
400  /*
401  * set cgrp before ctxsw in to allow
402  * event_filter_match() to not have to pass
403  * task around
404  */
405  cpuctx->cgrp = perf_cgroup_from_task(task);
406  cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
407  }
408  perf_pmu_enable(cpuctx->ctx.pmu);
409  perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
410  }
411  }
412 
413  rcu_read_unlock();
414 
415  local_irq_restore(flags);
416 }
417 
418 static inline void perf_cgroup_sched_out(struct task_struct *task,
419  struct task_struct *next)
420 {
421  struct perf_cgroup *cgrp1;
422  struct perf_cgroup *cgrp2 = NULL;
423 
424  /*
425  * we come here when we know perf_cgroup_events > 0
426  */
427  cgrp1 = perf_cgroup_from_task(task);
428 
429  /*
430  * next is NULL when called from perf_event_enable_on_exec()
431  * that will systematically cause a cgroup_switch()
432  */
433  if (next)
434  cgrp2 = perf_cgroup_from_task(next);
435 
436  /*
437  * only schedule out current cgroup events if we know
438  * that we are switching to a different cgroup. Otherwise,
439  * do no touch the cgroup events.
440  */
441  if (cgrp1 != cgrp2)
442  perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
443 }
444 
445 static inline void perf_cgroup_sched_in(struct task_struct *prev,
446  struct task_struct *task)
447 {
448  struct perf_cgroup *cgrp1;
449  struct perf_cgroup *cgrp2 = NULL;
450 
451  /*
452  * we come here when we know perf_cgroup_events > 0
453  */
454  cgrp1 = perf_cgroup_from_task(task);
455 
456  /* prev can never be NULL */
457  cgrp2 = perf_cgroup_from_task(prev);
458 
459  /*
460  * only need to schedule in cgroup events if we are changing
461  * cgroup during ctxsw. Cgroup events were not scheduled
462  * out of ctxsw out if that was not the case.
463  */
464  if (cgrp1 != cgrp2)
465  perf_cgroup_switch(task, PERF_CGROUP_SWIN);
466 }
467 
468 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
469  struct perf_event_attr *attr,
470  struct perf_event *group_leader)
471 {
472  struct perf_cgroup *cgrp;
473  struct cgroup_subsys_state *css;
474  struct fd f = fdget(fd);
475  int ret = 0;
476 
477  if (!f.file)
478  return -EBADF;
479 
480  css = cgroup_css_from_dir(f.file, perf_subsys_id);
481  if (IS_ERR(css)) {
482  ret = PTR_ERR(css);
483  goto out;
484  }
485 
486  cgrp = container_of(css, struct perf_cgroup, css);
487  event->cgrp = cgrp;
488 
489  /* must be done before we fput() the file */
490  if (!perf_tryget_cgroup(event)) {
491  event->cgrp = NULL;
492  ret = -ENOENT;
493  goto out;
494  }
495 
496  /*
497  * all events in a group must monitor
498  * the same cgroup because a task belongs
499  * to only one perf cgroup at a time
500  */
501  if (group_leader && group_leader->cgrp != cgrp) {
502  perf_detach_cgroup(event);
503  ret = -EINVAL;
504  }
505 out:
506  fdput(f);
507  return ret;
508 }
509 
510 static inline void
511 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
512 {
513  struct perf_cgroup_info *t;
514  t = per_cpu_ptr(event->cgrp->info, event->cpu);
515  event->shadow_ctx_time = now - t->timestamp;
516 }
517 
518 static inline void
519 perf_cgroup_defer_enabled(struct perf_event *event)
520 {
521  /*
522  * when the current task's perf cgroup does not match
523  * the event's, we need to remember to call the
524  * perf_mark_enable() function the first time a task with
525  * a matching perf cgroup is scheduled in.
526  */
527  if (is_cgroup_event(event) && !perf_cgroup_match(event))
528  event->cgrp_defer_enabled = 1;
529 }
530 
531 static inline void
532 perf_cgroup_mark_enabled(struct perf_event *event,
533  struct perf_event_context *ctx)
534 {
535  struct perf_event *sub;
536  u64 tstamp = perf_event_time(event);
537 
538  if (!event->cgrp_defer_enabled)
539  return;
540 
541  event->cgrp_defer_enabled = 0;
542 
543  event->tstamp_enabled = tstamp - event->total_time_enabled;
544  list_for_each_entry(sub, &event->sibling_list, group_entry) {
545  if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
546  sub->tstamp_enabled = tstamp - sub->total_time_enabled;
547  sub->cgrp_defer_enabled = 0;
548  }
549  }
550 }
551 #else /* !CONFIG_CGROUP_PERF */
552 
553 static inline bool
554 perf_cgroup_match(struct perf_event *event)
555 {
556  return true;
557 }
558 
559 static inline void perf_detach_cgroup(struct perf_event *event)
560 {}
561 
562 static inline int is_cgroup_event(struct perf_event *event)
563 {
564  return 0;
565 }
566 
567 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
568 {
569  return 0;
570 }
571 
572 static inline void update_cgrp_time_from_event(struct perf_event *event)
573 {
574 }
575 
576 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
577 {
578 }
579 
580 static inline void perf_cgroup_sched_out(struct task_struct *task,
581  struct task_struct *next)
582 {
583 }
584 
585 static inline void perf_cgroup_sched_in(struct task_struct *prev,
586  struct task_struct *task)
587 {
588 }
589 
590 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
591  struct perf_event_attr *attr,
592  struct perf_event *group_leader)
593 {
594  return -EINVAL;
595 }
596 
597 static inline void
598 perf_cgroup_set_timestamp(struct task_struct *task,
599  struct perf_event_context *ctx)
600 {
601 }
602 
603 void
604 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
605 {
606 }
607 
608 static inline void
609 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
610 {
611 }
612 
613 static inline u64 perf_cgroup_event_time(struct perf_event *event)
614 {
615  return 0;
616 }
617 
618 static inline void
619 perf_cgroup_defer_enabled(struct perf_event *event)
620 {
621 }
622 
623 static inline void
624 perf_cgroup_mark_enabled(struct perf_event *event,
625  struct perf_event_context *ctx)
626 {
627 }
628 #endif
629 
630 void perf_pmu_disable(struct pmu *pmu)
631 {
632  int *count = this_cpu_ptr(pmu->pmu_disable_count);
633  if (!(*count)++)
634  pmu->pmu_disable(pmu);
635 }
636 
637 void perf_pmu_enable(struct pmu *pmu)
638 {
639  int *count = this_cpu_ptr(pmu->pmu_disable_count);
640  if (!--(*count))
641  pmu->pmu_enable(pmu);
642 }
643 
644 static DEFINE_PER_CPU(struct list_head, rotation_list);
645 
646 /*
647  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
648  * because they're strictly cpu affine and rotate_start is called with IRQs
649  * disabled, while rotate_context is called from IRQ context.
650  */
651 static void perf_pmu_rotate_start(struct pmu *pmu)
652 {
653  struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
654  struct list_head *head = &__get_cpu_var(rotation_list);
655 
657 
658  if (list_empty(&cpuctx->rotation_list))
659  list_add(&cpuctx->rotation_list, head);
660 }
661 
662 static void get_ctx(struct perf_event_context *ctx)
663 {
665 }
666 
667 static void put_ctx(struct perf_event_context *ctx)
668 {
669  if (atomic_dec_and_test(&ctx->refcount)) {
670  if (ctx->parent_ctx)
671  put_ctx(ctx->parent_ctx);
672  if (ctx->task)
673  put_task_struct(ctx->task);
674  kfree_rcu(ctx, rcu_head);
675  }
676 }
677 
678 static void unclone_ctx(struct perf_event_context *ctx)
679 {
680  if (ctx->parent_ctx) {
681  put_ctx(ctx->parent_ctx);
682  ctx->parent_ctx = NULL;
683  }
684 }
685 
686 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
687 {
688  /*
689  * only top level events have the pid namespace they were created in
690  */
691  if (event->parent)
692  event = event->parent;
693 
694  return task_tgid_nr_ns(p, event->ns);
695 }
696 
697 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
698 {
699  /*
700  * only top level events have the pid namespace they were created in
701  */
702  if (event->parent)
703  event = event->parent;
704 
705  return task_pid_nr_ns(p, event->ns);
706 }
707 
708 /*
709  * If we inherit events we want to return the parent event id
710  * to userspace.
711  */
712 static u64 primary_event_id(struct perf_event *event)
713 {
714  u64 id = event->id;
715 
716  if (event->parent)
717  id = event->parent->id;
718 
719  return id;
720 }
721 
722 /*
723  * Get the perf_event_context for a task and lock it.
724  * This has to cope with with the fact that until it is locked,
725  * the context could get moved to another task.
726  */
727 static struct perf_event_context *
728 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
729 {
730  struct perf_event_context *ctx;
731 
732  rcu_read_lock();
733 retry:
734  ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
735  if (ctx) {
736  /*
737  * If this context is a clone of another, it might
738  * get swapped for another underneath us by
739  * perf_event_task_sched_out, though the
740  * rcu_read_lock() protects us from any context
741  * getting freed. Lock the context and check if it
742  * got swapped before we could get the lock, and retry
743  * if so. If we locked the right context, then it
744  * can't get swapped on us any more.
745  */
746  raw_spin_lock_irqsave(&ctx->lock, *flags);
747  if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
748  raw_spin_unlock_irqrestore(&ctx->lock, *flags);
749  goto retry;
750  }
751 
752  if (!atomic_inc_not_zero(&ctx->refcount)) {
753  raw_spin_unlock_irqrestore(&ctx->lock, *flags);
754  ctx = NULL;
755  }
756  }
757  rcu_read_unlock();
758  return ctx;
759 }
760 
761 /*
762  * Get the context for a task and increment its pin_count so it
763  * can't get swapped to another task. This also increments its
764  * reference count so that the context can't get freed.
765  */
766 static struct perf_event_context *
767 perf_pin_task_context(struct task_struct *task, int ctxn)
768 {
769  struct perf_event_context *ctx;
770  unsigned long flags;
771 
772  ctx = perf_lock_task_context(task, ctxn, &flags);
773  if (ctx) {
774  ++ctx->pin_count;
775  raw_spin_unlock_irqrestore(&ctx->lock, flags);
776  }
777  return ctx;
778 }
779 
780 static void perf_unpin_context(struct perf_event_context *ctx)
781 {
782  unsigned long flags;
783 
784  raw_spin_lock_irqsave(&ctx->lock, flags);
785  --ctx->pin_count;
786  raw_spin_unlock_irqrestore(&ctx->lock, flags);
787 }
788 
789 /*
790  * Update the record of the current time in a context.
791  */
792 static void update_context_time(struct perf_event_context *ctx)
793 {
794  u64 now = perf_clock();
795 
796  ctx->time += now - ctx->timestamp;
797  ctx->timestamp = now;
798 }
799 
800 static u64 perf_event_time(struct perf_event *event)
801 {
802  struct perf_event_context *ctx = event->ctx;
803 
804  if (is_cgroup_event(event))
805  return perf_cgroup_event_time(event);
806 
807  return ctx ? ctx->time : 0;
808 }
809 
810 /*
811  * Update the total_time_enabled and total_time_running fields for a event.
812  * The caller of this function needs to hold the ctx->lock.
813  */
814 static void update_event_times(struct perf_event *event)
815 {
816  struct perf_event_context *ctx = event->ctx;
817  u64 run_end;
818 
819  if (event->state < PERF_EVENT_STATE_INACTIVE ||
820  event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
821  return;
822  /*
823  * in cgroup mode, time_enabled represents
824  * the time the event was enabled AND active
825  * tasks were in the monitored cgroup. This is
826  * independent of the activity of the context as
827  * there may be a mix of cgroup and non-cgroup events.
828  *
829  * That is why we treat cgroup events differently
830  * here.
831  */
832  if (is_cgroup_event(event))
833  run_end = perf_cgroup_event_time(event);
834  else if (ctx->is_active)
835  run_end = ctx->time;
836  else
837  run_end = event->tstamp_stopped;
838 
839  event->total_time_enabled = run_end - event->tstamp_enabled;
840 
841  if (event->state == PERF_EVENT_STATE_INACTIVE)
842  run_end = event->tstamp_stopped;
843  else
844  run_end = perf_event_time(event);
845 
846  event->total_time_running = run_end - event->tstamp_running;
847 
848 }
849 
850 /*
851  * Update total_time_enabled and total_time_running for all events in a group.
852  */
853 static void update_group_times(struct perf_event *leader)
854 {
855  struct perf_event *event;
856 
857  update_event_times(leader);
858  list_for_each_entry(event, &leader->sibling_list, group_entry)
859  update_event_times(event);
860 }
861 
862 static struct list_head *
863 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
864 {
865  if (event->attr.pinned)
866  return &ctx->pinned_groups;
867  else
868  return &ctx->flexible_groups;
869 }
870 
871 /*
872  * Add a event from the lists for its context.
873  * Must be called with ctx->mutex and ctx->lock held.
874  */
875 static void
876 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
877 {
878  WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
879  event->attach_state |= PERF_ATTACH_CONTEXT;
880 
881  /*
882  * If we're a stand alone event or group leader, we go to the context
883  * list, group events are kept attached to the group so that
884  * perf_group_detach can, at all times, locate all siblings.
885  */
886  if (event->group_leader == event) {
887  struct list_head *list;
888 
889  if (is_software_event(event))
890  event->group_flags |= PERF_GROUP_SOFTWARE;
891 
892  list = ctx_group_list(event, ctx);
893  list_add_tail(&event->group_entry, list);
894  }
895 
896  if (is_cgroup_event(event))
897  ctx->nr_cgroups++;
898 
899  if (has_branch_stack(event))
900  ctx->nr_branch_stack++;
901 
902  list_add_rcu(&event->event_entry, &ctx->event_list);
903  if (!ctx->nr_events)
904  perf_pmu_rotate_start(ctx->pmu);
905  ctx->nr_events++;
906  if (event->attr.inherit_stat)
907  ctx->nr_stat++;
908 }
909 
910 /*
911  * Called at perf_event creation and when events are attached/detached from a
912  * group.
913  */
914 static void perf_event__read_size(struct perf_event *event)
915 {
916  int entry = sizeof(u64); /* value */
917  int size = 0;
918  int nr = 1;
919 
920  if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
921  size += sizeof(u64);
922 
923  if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
924  size += sizeof(u64);
925 
926  if (event->attr.read_format & PERF_FORMAT_ID)
927  entry += sizeof(u64);
928 
929  if (event->attr.read_format & PERF_FORMAT_GROUP) {
930  nr += event->group_leader->nr_siblings;
931  size += sizeof(u64);
932  }
933 
934  size += entry * nr;
935  event->read_size = size;
936 }
937 
938 static void perf_event__header_size(struct perf_event *event)
939 {
940  struct perf_sample_data *data;
941  u64 sample_type = event->attr.sample_type;
942  u16 size = 0;
943 
944  perf_event__read_size(event);
945 
946  if (sample_type & PERF_SAMPLE_IP)
947  size += sizeof(data->ip);
948 
949  if (sample_type & PERF_SAMPLE_ADDR)
950  size += sizeof(data->addr);
951 
952  if (sample_type & PERF_SAMPLE_PERIOD)
953  size += sizeof(data->period);
954 
955  if (sample_type & PERF_SAMPLE_READ)
956  size += event->read_size;
957 
958  event->header_size = size;
959 }
960 
961 static void perf_event__id_header_size(struct perf_event *event)
962 {
963  struct perf_sample_data *data;
964  u64 sample_type = event->attr.sample_type;
965  u16 size = 0;
966 
967  if (sample_type & PERF_SAMPLE_TID)
968  size += sizeof(data->tid_entry);
969 
970  if (sample_type & PERF_SAMPLE_TIME)
971  size += sizeof(data->time);
972 
973  if (sample_type & PERF_SAMPLE_ID)
974  size += sizeof(data->id);
975 
976  if (sample_type & PERF_SAMPLE_STREAM_ID)
977  size += sizeof(data->stream_id);
978 
979  if (sample_type & PERF_SAMPLE_CPU)
980  size += sizeof(data->cpu_entry);
981 
982  event->id_header_size = size;
983 }
984 
985 static void perf_group_attach(struct perf_event *event)
986 {
987  struct perf_event *group_leader = event->group_leader, *pos;
988 
989  /*
990  * We can have double attach due to group movement in perf_event_open.
991  */
992  if (event->attach_state & PERF_ATTACH_GROUP)
993  return;
994 
995  event->attach_state |= PERF_ATTACH_GROUP;
996 
997  if (group_leader == event)
998  return;
999 
1000  if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
1001  !is_software_event(event))
1002  group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1003 
1004  list_add_tail(&event->group_entry, &group_leader->sibling_list);
1005  group_leader->nr_siblings++;
1006 
1007  perf_event__header_size(group_leader);
1008 
1009  list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1010  perf_event__header_size(pos);
1011 }
1012 
1013 /*
1014  * Remove a event from the lists for its context.
1015  * Must be called with ctx->mutex and ctx->lock held.
1016  */
1017 static void
1018 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1019 {
1020  struct perf_cpu_context *cpuctx;
1021  /*
1022  * We can have double detach due to exit/hot-unplug + close.
1023  */
1024  if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1025  return;
1026 
1027  event->attach_state &= ~PERF_ATTACH_CONTEXT;
1028 
1029  if (is_cgroup_event(event)) {
1030  ctx->nr_cgroups--;
1031  cpuctx = __get_cpu_context(ctx);
1032  /*
1033  * if there are no more cgroup events
1034  * then cler cgrp to avoid stale pointer
1035  * in update_cgrp_time_from_cpuctx()
1036  */
1037  if (!ctx->nr_cgroups)
1038  cpuctx->cgrp = NULL;
1039  }
1040 
1041  if (has_branch_stack(event))
1042  ctx->nr_branch_stack--;
1043 
1044  ctx->nr_events--;
1045  if (event->attr.inherit_stat)
1046  ctx->nr_stat--;
1047 
1048  list_del_rcu(&event->event_entry);
1049 
1050  if (event->group_leader == event)
1051  list_del_init(&event->group_entry);
1052 
1053  update_group_times(event);
1054 
1055  /*
1056  * If event was in error state, then keep it
1057  * that way, otherwise bogus counts will be
1058  * returned on read(). The only way to get out
1059  * of error state is by explicit re-enabling
1060  * of the event
1061  */
1062  if (event->state > PERF_EVENT_STATE_OFF)
1063  event->state = PERF_EVENT_STATE_OFF;
1064 }
1065 
1066 static void perf_group_detach(struct perf_event *event)
1067 {
1068  struct perf_event *sibling, *tmp;
1069  struct list_head *list = NULL;
1070 
1071  /*
1072  * We can have double detach due to exit/hot-unplug + close.
1073  */
1074  if (!(event->attach_state & PERF_ATTACH_GROUP))
1075  return;
1076 
1077  event->attach_state &= ~PERF_ATTACH_GROUP;
1078 
1079  /*
1080  * If this is a sibling, remove it from its group.
1081  */
1082  if (event->group_leader != event) {
1083  list_del_init(&event->group_entry);
1084  event->group_leader->nr_siblings--;
1085  goto out;
1086  }
1087 
1088  if (!list_empty(&event->group_entry))
1089  list = &event->group_entry;
1090 
1091  /*
1092  * If this was a group event with sibling events then
1093  * upgrade the siblings to singleton events by adding them
1094  * to whatever list we are on.
1095  */
1096  list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1097  if (list)
1098  list_move_tail(&sibling->group_entry, list);
1099  sibling->group_leader = sibling;
1100 
1101  /* Inherit group flags from the previous leader */
1102  sibling->group_flags = event->group_flags;
1103  }
1104 
1105 out:
1106  perf_event__header_size(event->group_leader);
1107 
1108  list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1109  perf_event__header_size(tmp);
1110 }
1111 
1112 static inline int
1113 event_filter_match(struct perf_event *event)
1114 {
1115  return (event->cpu == -1 || event->cpu == smp_processor_id())
1116  && perf_cgroup_match(event);
1117 }
1118 
1119 static void
1120 event_sched_out(struct perf_event *event,
1121  struct perf_cpu_context *cpuctx,
1122  struct perf_event_context *ctx)
1123 {
1124  u64 tstamp = perf_event_time(event);
1125  u64 delta;
1126  /*
1127  * An event which could not be activated because of
1128  * filter mismatch still needs to have its timings
1129  * maintained, otherwise bogus information is return
1130  * via read() for time_enabled, time_running:
1131  */
1132  if (event->state == PERF_EVENT_STATE_INACTIVE
1133  && !event_filter_match(event)) {
1134  delta = tstamp - event->tstamp_stopped;
1135  event->tstamp_running += delta;
1136  event->tstamp_stopped = tstamp;
1137  }
1138 
1139  if (event->state != PERF_EVENT_STATE_ACTIVE)
1140  return;
1141 
1142  event->state = PERF_EVENT_STATE_INACTIVE;
1143  if (event->pending_disable) {
1144  event->pending_disable = 0;
1145  event->state = PERF_EVENT_STATE_OFF;
1146  }
1147  event->tstamp_stopped = tstamp;
1148  event->pmu->del(event, 0);
1149  event->oncpu = -1;
1150 
1151  if (!is_software_event(event))
1152  cpuctx->active_oncpu--;
1153  ctx->nr_active--;
1154  if (event->attr.freq && event->attr.sample_freq)
1155  ctx->nr_freq--;
1156  if (event->attr.exclusive || !cpuctx->active_oncpu)
1157  cpuctx->exclusive = 0;
1158 }
1159 
1160 static void
1161 group_sched_out(struct perf_event *group_event,
1162  struct perf_cpu_context *cpuctx,
1163  struct perf_event_context *ctx)
1164 {
1165  struct perf_event *event;
1166  int state = group_event->state;
1167 
1168  event_sched_out(group_event, cpuctx, ctx);
1169 
1170  /*
1171  * Schedule out siblings (if any):
1172  */
1173  list_for_each_entry(event, &group_event->sibling_list, group_entry)
1174  event_sched_out(event, cpuctx, ctx);
1175 
1176  if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1177  cpuctx->exclusive = 0;
1178 }
1179 
1180 /*
1181  * Cross CPU call to remove a performance event
1182  *
1183  * We disable the event on the hardware level first. After that we
1184  * remove it from the context list.
1185  */
1186 static int __perf_remove_from_context(void *info)
1187 {
1188  struct perf_event *event = info;
1189  struct perf_event_context *ctx = event->ctx;
1190  struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1191 
1192  raw_spin_lock(&ctx->lock);
1193  event_sched_out(event, cpuctx, ctx);
1194  list_del_event(event, ctx);
1195  if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1196  ctx->is_active = 0;
1197  cpuctx->task_ctx = NULL;
1198  }
1199  raw_spin_unlock(&ctx->lock);
1200 
1201  return 0;
1202 }
1203 
1204 
1205 /*
1206  * Remove the event from a task's (or a CPU's) list of events.
1207  *
1208  * CPU events are removed with a smp call. For task events we only
1209  * call when the task is on a CPU.
1210  *
1211  * If event->ctx is a cloned context, callers must make sure that
1212  * every task struct that event->ctx->task could possibly point to
1213  * remains valid. This is OK when called from perf_release since
1214  * that only calls us on the top-level context, which can't be a clone.
1215  * When called from perf_event_exit_task, it's OK because the
1216  * context has been detached from its task.
1217  */
1218 static void perf_remove_from_context(struct perf_event *event)
1219 {
1220  struct perf_event_context *ctx = event->ctx;
1221  struct task_struct *task = ctx->task;
1222 
1223  lockdep_assert_held(&ctx->mutex);
1224 
1225  if (!task) {
1226  /*
1227  * Per cpu events are removed via an smp call and
1228  * the removal is always successful.
1229  */
1230  cpu_function_call(event->cpu, __perf_remove_from_context, event);
1231  return;
1232  }
1233 
1234 retry:
1235  if (!task_function_call(task, __perf_remove_from_context, event))
1236  return;
1237 
1238  raw_spin_lock_irq(&ctx->lock);
1239  /*
1240  * If we failed to find a running task, but find the context active now
1241  * that we've acquired the ctx->lock, retry.
1242  */
1243  if (ctx->is_active) {
1244  raw_spin_unlock_irq(&ctx->lock);
1245  goto retry;
1246  }
1247 
1248  /*
1249  * Since the task isn't running, its safe to remove the event, us
1250  * holding the ctx->lock ensures the task won't get scheduled in.
1251  */
1252  list_del_event(event, ctx);
1253  raw_spin_unlock_irq(&ctx->lock);
1254 }
1255 
1256 /*
1257  * Cross CPU call to disable a performance event
1258  */
1259 int __perf_event_disable(void *info)
1260 {
1261  struct perf_event *event = info;
1262  struct perf_event_context *ctx = event->ctx;
1263  struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1264 
1265  /*
1266  * If this is a per-task event, need to check whether this
1267  * event's task is the current task on this cpu.
1268  *
1269  * Can trigger due to concurrent perf_event_context_sched_out()
1270  * flipping contexts around.
1271  */
1272  if (ctx->task && cpuctx->task_ctx != ctx)
1273  return -EINVAL;
1274 
1275  raw_spin_lock(&ctx->lock);
1276 
1277  /*
1278  * If the event is on, turn it off.
1279  * If it is in error state, leave it in error state.
1280  */
1281  if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1282  update_context_time(ctx);
1283  update_cgrp_time_from_event(event);
1284  update_group_times(event);
1285  if (event == event->group_leader)
1286  group_sched_out(event, cpuctx, ctx);
1287  else
1288  event_sched_out(event, cpuctx, ctx);
1289  event->state = PERF_EVENT_STATE_OFF;
1290  }
1291 
1292  raw_spin_unlock(&ctx->lock);
1293 
1294  return 0;
1295 }
1296 
1297 /*
1298  * Disable a event.
1299  *
1300  * If event->ctx is a cloned context, callers must make sure that
1301  * every task struct that event->ctx->task could possibly point to
1302  * remains valid. This condition is satisifed when called through
1303  * perf_event_for_each_child or perf_event_for_each because they
1304  * hold the top-level event's child_mutex, so any descendant that
1305  * goes to exit will block in sync_child_event.
1306  * When called from perf_pending_event it's OK because event->ctx
1307  * is the current context on this CPU and preemption is disabled,
1308  * hence we can't get into perf_event_task_sched_out for this context.
1309  */
1310 void perf_event_disable(struct perf_event *event)
1311 {
1312  struct perf_event_context *ctx = event->ctx;
1313  struct task_struct *task = ctx->task;
1314 
1315  if (!task) {
1316  /*
1317  * Disable the event on the cpu that it's on
1318  */
1319  cpu_function_call(event->cpu, __perf_event_disable, event);
1320  return;
1321  }
1322 
1323 retry:
1324  if (!task_function_call(task, __perf_event_disable, event))
1325  return;
1326 
1327  raw_spin_lock_irq(&ctx->lock);
1328  /*
1329  * If the event is still active, we need to retry the cross-call.
1330  */
1331  if (event->state == PERF_EVENT_STATE_ACTIVE) {
1332  raw_spin_unlock_irq(&ctx->lock);
1333  /*
1334  * Reload the task pointer, it might have been changed by
1335  * a concurrent perf_event_context_sched_out().
1336  */
1337  task = ctx->task;
1338  goto retry;
1339  }
1340 
1341  /*
1342  * Since we have the lock this context can't be scheduled
1343  * in, so we can change the state safely.
1344  */
1345  if (event->state == PERF_EVENT_STATE_INACTIVE) {
1346  update_group_times(event);
1347  event->state = PERF_EVENT_STATE_OFF;
1348  }
1349  raw_spin_unlock_irq(&ctx->lock);
1350 }
1352 
1353 static void perf_set_shadow_time(struct perf_event *event,
1354  struct perf_event_context *ctx,
1355  u64 tstamp)
1356 {
1357  /*
1358  * use the correct time source for the time snapshot
1359  *
1360  * We could get by without this by leveraging the
1361  * fact that to get to this function, the caller
1362  * has most likely already called update_context_time()
1363  * and update_cgrp_time_xx() and thus both timestamp
1364  * are identical (or very close). Given that tstamp is,
1365  * already adjusted for cgroup, we could say that:
1366  * tstamp - ctx->timestamp
1367  * is equivalent to
1368  * tstamp - cgrp->timestamp.
1369  *
1370  * Then, in perf_output_read(), the calculation would
1371  * work with no changes because:
1372  * - event is guaranteed scheduled in
1373  * - no scheduled out in between
1374  * - thus the timestamp would be the same
1375  *
1376  * But this is a bit hairy.
1377  *
1378  * So instead, we have an explicit cgroup call to remain
1379  * within the time time source all along. We believe it
1380  * is cleaner and simpler to understand.
1381  */
1382  if (is_cgroup_event(event))
1383  perf_cgroup_set_shadow_time(event, tstamp);
1384  else
1385  event->shadow_ctx_time = tstamp - ctx->timestamp;
1386 }
1387 
1388 #define MAX_INTERRUPTS (~0ULL)
1389 
1390 static void perf_log_throttle(struct perf_event *event, int enable);
1391 
1392 static int
1393 event_sched_in(struct perf_event *event,
1394  struct perf_cpu_context *cpuctx,
1395  struct perf_event_context *ctx)
1396 {
1397  u64 tstamp = perf_event_time(event);
1398 
1399  if (event->state <= PERF_EVENT_STATE_OFF)
1400  return 0;
1401 
1402  event->state = PERF_EVENT_STATE_ACTIVE;
1403  event->oncpu = smp_processor_id();
1404 
1405  /*
1406  * Unthrottle events, since we scheduled we might have missed several
1407  * ticks already, also for a heavily scheduling task there is little
1408  * guarantee it'll get a tick in a timely manner.
1409  */
1410  if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1411  perf_log_throttle(event, 1);
1412  event->hw.interrupts = 0;
1413  }
1414 
1415  /*
1416  * The new state must be visible before we turn it on in the hardware:
1417  */
1418  smp_wmb();
1419 
1420  if (event->pmu->add(event, PERF_EF_START)) {
1421  event->state = PERF_EVENT_STATE_INACTIVE;
1422  event->oncpu = -1;
1423  return -EAGAIN;
1424  }
1425 
1426  event->tstamp_running += tstamp - event->tstamp_stopped;
1427 
1428  perf_set_shadow_time(event, ctx, tstamp);
1429 
1430  if (!is_software_event(event))
1431  cpuctx->active_oncpu++;
1432  ctx->nr_active++;
1433  if (event->attr.freq && event->attr.sample_freq)
1434  ctx->nr_freq++;
1435 
1436  if (event->attr.exclusive)
1437  cpuctx->exclusive = 1;
1438 
1439  return 0;
1440 }
1441 
1442 static int
1443 group_sched_in(struct perf_event *group_event,
1444  struct perf_cpu_context *cpuctx,
1445  struct perf_event_context *ctx)
1446 {
1447  struct perf_event *event, *partial_group = NULL;
1448  struct pmu *pmu = group_event->pmu;
1449  u64 now = ctx->time;
1450  bool simulate = false;
1451 
1452  if (group_event->state == PERF_EVENT_STATE_OFF)
1453  return 0;
1454 
1455  pmu->start_txn(pmu);
1456 
1457  if (event_sched_in(group_event, cpuctx, ctx)) {
1458  pmu->cancel_txn(pmu);
1459  return -EAGAIN;
1460  }
1461 
1462  /*
1463  * Schedule in siblings as one group (if any):
1464  */
1465  list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1466  if (event_sched_in(event, cpuctx, ctx)) {
1467  partial_group = event;
1468  goto group_error;
1469  }
1470  }
1471 
1472  if (!pmu->commit_txn(pmu))
1473  return 0;
1474 
1475 group_error:
1476  /*
1477  * Groups can be scheduled in as one unit only, so undo any
1478  * partial group before returning:
1479  * The events up to the failed event are scheduled out normally,
1480  * tstamp_stopped will be updated.
1481  *
1482  * The failed events and the remaining siblings need to have
1483  * their timings updated as if they had gone thru event_sched_in()
1484  * and event_sched_out(). This is required to get consistent timings
1485  * across the group. This also takes care of the case where the group
1486  * could never be scheduled by ensuring tstamp_stopped is set to mark
1487  * the time the event was actually stopped, such that time delta
1488  * calculation in update_event_times() is correct.
1489  */
1490  list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1491  if (event == partial_group)
1492  simulate = true;
1493 
1494  if (simulate) {
1495  event->tstamp_running += now - event->tstamp_stopped;
1496  event->tstamp_stopped = now;
1497  } else {
1498  event_sched_out(event, cpuctx, ctx);
1499  }
1500  }
1501  event_sched_out(group_event, cpuctx, ctx);
1502 
1503  pmu->cancel_txn(pmu);
1504 
1505  return -EAGAIN;
1506 }
1507 
1508 /*
1509  * Work out whether we can put this event group on the CPU now.
1510  */
1511 static int group_can_go_on(struct perf_event *event,
1512  struct perf_cpu_context *cpuctx,
1513  int can_add_hw)
1514 {
1515  /*
1516  * Groups consisting entirely of software events can always go on.
1517  */
1518  if (event->group_flags & PERF_GROUP_SOFTWARE)
1519  return 1;
1520  /*
1521  * If an exclusive group is already on, no other hardware
1522  * events can go on.
1523  */
1524  if (cpuctx->exclusive)
1525  return 0;
1526  /*
1527  * If this group is exclusive and there are already
1528  * events on the CPU, it can't go on.
1529  */
1530  if (event->attr.exclusive && cpuctx->active_oncpu)
1531  return 0;
1532  /*
1533  * Otherwise, try to add it if all previous groups were able
1534  * to go on.
1535  */
1536  return can_add_hw;
1537 }
1538 
1539 static void add_event_to_ctx(struct perf_event *event,
1540  struct perf_event_context *ctx)
1541 {
1542  u64 tstamp = perf_event_time(event);
1543 
1544  list_add_event(event, ctx);
1545  perf_group_attach(event);
1546  event->tstamp_enabled = tstamp;
1547  event->tstamp_running = tstamp;
1548  event->tstamp_stopped = tstamp;
1549 }
1550 
1551 static void task_ctx_sched_out(struct perf_event_context *ctx);
1552 static void
1553 ctx_sched_in(struct perf_event_context *ctx,
1554  struct perf_cpu_context *cpuctx,
1555  enum event_type_t event_type,
1556  struct task_struct *task);
1557 
1558 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1559  struct perf_event_context *ctx,
1560  struct task_struct *task)
1561 {
1562  cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1563  if (ctx)
1564  ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1565  cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1566  if (ctx)
1567  ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1568 }
1569 
1570 /*
1571  * Cross CPU call to install and enable a performance event
1572  *
1573  * Must be called with ctx->mutex held
1574  */
1575 static int __perf_install_in_context(void *info)
1576 {
1577  struct perf_event *event = info;
1578  struct perf_event_context *ctx = event->ctx;
1579  struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1580  struct perf_event_context *task_ctx = cpuctx->task_ctx;
1581  struct task_struct *task = current;
1582 
1583  perf_ctx_lock(cpuctx, task_ctx);
1584  perf_pmu_disable(cpuctx->ctx.pmu);
1585 
1586  /*
1587  * If there was an active task_ctx schedule it out.
1588  */
1589  if (task_ctx)
1590  task_ctx_sched_out(task_ctx);
1591 
1592  /*
1593  * If the context we're installing events in is not the
1594  * active task_ctx, flip them.
1595  */
1596  if (ctx->task && task_ctx != ctx) {
1597  if (task_ctx)
1598  raw_spin_unlock(&task_ctx->lock);
1599  raw_spin_lock(&ctx->lock);
1600  task_ctx = ctx;
1601  }
1602 
1603  if (task_ctx) {
1604  cpuctx->task_ctx = task_ctx;
1605  task = task_ctx->task;
1606  }
1607 
1608  cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1609 
1610  update_context_time(ctx);
1611  /*
1612  * update cgrp time only if current cgrp
1613  * matches event->cgrp. Must be done before
1614  * calling add_event_to_ctx()
1615  */
1616  update_cgrp_time_from_event(event);
1617 
1618  add_event_to_ctx(event, ctx);
1619 
1620  /*
1621  * Schedule everything back in
1622  */
1623  perf_event_sched_in(cpuctx, task_ctx, task);
1624 
1625  perf_pmu_enable(cpuctx->ctx.pmu);
1626  perf_ctx_unlock(cpuctx, task_ctx);
1627 
1628  return 0;
1629 }
1630 
1631 /*
1632  * Attach a performance event to a context
1633  *
1634  * First we add the event to the list with the hardware enable bit
1635  * in event->hw_config cleared.
1636  *
1637  * If the event is attached to a task which is on a CPU we use a smp
1638  * call to enable it in the task context. The task might have been
1639  * scheduled away, but we check this in the smp call again.
1640  */
1641 static void
1642 perf_install_in_context(struct perf_event_context *ctx,
1643  struct perf_event *event,
1644  int cpu)
1645 {
1646  struct task_struct *task = ctx->task;
1647 
1648  lockdep_assert_held(&ctx->mutex);
1649 
1650  event->ctx = ctx;
1651  if (event->cpu != -1)
1652  event->cpu = cpu;
1653 
1654  if (!task) {
1655  /*
1656  * Per cpu events are installed via an smp call and
1657  * the install is always successful.
1658  */
1659  cpu_function_call(cpu, __perf_install_in_context, event);
1660  return;
1661  }
1662 
1663 retry:
1664  if (!task_function_call(task, __perf_install_in_context, event))
1665  return;
1666 
1667  raw_spin_lock_irq(&ctx->lock);
1668  /*
1669  * If we failed to find a running task, but find the context active now
1670  * that we've acquired the ctx->lock, retry.
1671  */
1672  if (ctx->is_active) {
1673  raw_spin_unlock_irq(&ctx->lock);
1674  goto retry;
1675  }
1676 
1677  /*
1678  * Since the task isn't running, its safe to add the event, us holding
1679  * the ctx->lock ensures the task won't get scheduled in.
1680  */
1681  add_event_to_ctx(event, ctx);
1682  raw_spin_unlock_irq(&ctx->lock);
1683 }
1684 
1685 /*
1686  * Put a event into inactive state and update time fields.
1687  * Enabling the leader of a group effectively enables all
1688  * the group members that aren't explicitly disabled, so we
1689  * have to update their ->tstamp_enabled also.
1690  * Note: this works for group members as well as group leaders
1691  * since the non-leader members' sibling_lists will be empty.
1692  */
1693 static void __perf_event_mark_enabled(struct perf_event *event)
1694 {
1695  struct perf_event *sub;
1696  u64 tstamp = perf_event_time(event);
1697 
1698  event->state = PERF_EVENT_STATE_INACTIVE;
1699  event->tstamp_enabled = tstamp - event->total_time_enabled;
1700  list_for_each_entry(sub, &event->sibling_list, group_entry) {
1701  if (sub->state >= PERF_EVENT_STATE_INACTIVE)
1702  sub->tstamp_enabled = tstamp - sub->total_time_enabled;
1703  }
1704 }
1705 
1706 /*
1707  * Cross CPU call to enable a performance event
1708  */
1709 static int __perf_event_enable(void *info)
1710 {
1711  struct perf_event *event = info;
1712  struct perf_event_context *ctx = event->ctx;
1713  struct perf_event *leader = event->group_leader;
1714  struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1715  int err;
1716 
1717  if (WARN_ON_ONCE(!ctx->is_active))
1718  return -EINVAL;
1719 
1720  raw_spin_lock(&ctx->lock);
1721  update_context_time(ctx);
1722 
1723  if (event->state >= PERF_EVENT_STATE_INACTIVE)
1724  goto unlock;
1725 
1726  /*
1727  * set current task's cgroup time reference point
1728  */
1729  perf_cgroup_set_timestamp(current, ctx);
1730 
1731  __perf_event_mark_enabled(event);
1732 
1733  if (!event_filter_match(event)) {
1734  if (is_cgroup_event(event))
1735  perf_cgroup_defer_enabled(event);
1736  goto unlock;
1737  }
1738 
1739  /*
1740  * If the event is in a group and isn't the group leader,
1741  * then don't put it on unless the group is on.
1742  */
1743  if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1744  goto unlock;
1745 
1746  if (!group_can_go_on(event, cpuctx, 1)) {
1747  err = -EEXIST;
1748  } else {
1749  if (event == leader)
1750  err = group_sched_in(event, cpuctx, ctx);
1751  else
1752  err = event_sched_in(event, cpuctx, ctx);
1753  }
1754 
1755  if (err) {
1756  /*
1757  * If this event can't go on and it's part of a
1758  * group, then the whole group has to come off.
1759  */
1760  if (leader != event)
1761  group_sched_out(leader, cpuctx, ctx);
1762  if (leader->attr.pinned) {
1763  update_group_times(leader);
1764  leader->state = PERF_EVENT_STATE_ERROR;
1765  }
1766  }
1767 
1768 unlock:
1769  raw_spin_unlock(&ctx->lock);
1770 
1771  return 0;
1772 }
1773 
1774 /*
1775  * Enable a event.
1776  *
1777  * If event->ctx is a cloned context, callers must make sure that
1778  * every task struct that event->ctx->task could possibly point to
1779  * remains valid. This condition is satisfied when called through
1780  * perf_event_for_each_child or perf_event_for_each as described
1781  * for perf_event_disable.
1782  */
1783 void perf_event_enable(struct perf_event *event)
1784 {
1785  struct perf_event_context *ctx = event->ctx;
1786  struct task_struct *task = ctx->task;
1787 
1788  if (!task) {
1789  /*
1790  * Enable the event on the cpu that it's on
1791  */
1792  cpu_function_call(event->cpu, __perf_event_enable, event);
1793  return;
1794  }
1795 
1796  raw_spin_lock_irq(&ctx->lock);
1797  if (event->state >= PERF_EVENT_STATE_INACTIVE)
1798  goto out;
1799 
1800  /*
1801  * If the event is in error state, clear that first.
1802  * That way, if we see the event in error state below, we
1803  * know that it has gone back into error state, as distinct
1804  * from the task having been scheduled away before the
1805  * cross-call arrived.
1806  */
1807  if (event->state == PERF_EVENT_STATE_ERROR)
1808  event->state = PERF_EVENT_STATE_OFF;
1809 
1810 retry:
1811  if (!ctx->is_active) {
1812  __perf_event_mark_enabled(event);
1813  goto out;
1814  }
1815 
1816  raw_spin_unlock_irq(&ctx->lock);
1817 
1818  if (!task_function_call(task, __perf_event_enable, event))
1819  return;
1820 
1821  raw_spin_lock_irq(&ctx->lock);
1822 
1823  /*
1824  * If the context is active and the event is still off,
1825  * we need to retry the cross-call.
1826  */
1827  if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
1828  /*
1829  * task could have been flipped by a concurrent
1830  * perf_event_context_sched_out()
1831  */
1832  task = ctx->task;
1833  goto retry;
1834  }
1835 
1836 out:
1837  raw_spin_unlock_irq(&ctx->lock);
1838 }
1840 
1841 int perf_event_refresh(struct perf_event *event, int refresh)
1842 {
1843  /*
1844  * not supported on inherited events
1845  */
1846  if (event->attr.inherit || !is_sampling_event(event))
1847  return -EINVAL;
1848 
1849  atomic_add(refresh, &event->event_limit);
1850  perf_event_enable(event);
1851 
1852  return 0;
1853 }
1855 
1856 static void ctx_sched_out(struct perf_event_context *ctx,
1857  struct perf_cpu_context *cpuctx,
1858  enum event_type_t event_type)
1859 {
1860  struct perf_event *event;
1861  int is_active = ctx->is_active;
1862 
1863  ctx->is_active &= ~event_type;
1864  if (likely(!ctx->nr_events))
1865  return;
1866 
1867  update_context_time(ctx);
1868  update_cgrp_time_from_cpuctx(cpuctx);
1869  if (!ctx->nr_active)
1870  return;
1871 
1872  perf_pmu_disable(ctx->pmu);
1873  if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
1874  list_for_each_entry(event, &ctx->pinned_groups, group_entry)
1875  group_sched_out(event, cpuctx, ctx);
1876  }
1877 
1878  if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
1879  list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1880  group_sched_out(event, cpuctx, ctx);
1881  }
1882  perf_pmu_enable(ctx->pmu);
1883 }
1884 
1885 /*
1886  * Test whether two contexts are equivalent, i.e. whether they
1887  * have both been cloned from the same version of the same context
1888  * and they both have the same number of enabled events.
1889  * If the number of enabled events is the same, then the set
1890  * of enabled events should be the same, because these are both
1891  * inherited contexts, therefore we can't access individual events
1892  * in them directly with an fd; we can only enable/disable all
1893  * events via prctl, or enable/disable all events in a family
1894  * via ioctl, which will have the same effect on both contexts.
1895  */
1896 static int context_equiv(struct perf_event_context *ctx1,
1897  struct perf_event_context *ctx2)
1898 {
1899  return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1900  && ctx1->parent_gen == ctx2->parent_gen
1901  && !ctx1->pin_count && !ctx2->pin_count;
1902 }
1903 
1904 static void __perf_event_sync_stat(struct perf_event *event,
1905  struct perf_event *next_event)
1906 {
1907  u64 value;
1908 
1909  if (!event->attr.inherit_stat)
1910  return;
1911 
1912  /*
1913  * Update the event value, we cannot use perf_event_read()
1914  * because we're in the middle of a context switch and have IRQs
1915  * disabled, which upsets smp_call_function_single(), however
1916  * we know the event must be on the current CPU, therefore we
1917  * don't need to use it.
1918  */
1919  switch (event->state) {
1921  event->pmu->read(event);
1922  /* fall-through */
1923 
1925  update_event_times(event);
1926  break;
1927 
1928  default:
1929  break;
1930  }
1931 
1932  /*
1933  * In order to keep per-task stats reliable we need to flip the event
1934  * values when we flip the contexts.
1935  */
1936  value = local64_read(&next_event->count);
1937  value = local64_xchg(&event->count, value);
1938  local64_set(&next_event->count, value);
1939 
1940  swap(event->total_time_enabled, next_event->total_time_enabled);
1941  swap(event->total_time_running, next_event->total_time_running);
1942 
1943  /*
1944  * Since we swizzled the values, update the user visible data too.
1945  */
1947  perf_event_update_userpage(next_event);
1948 }
1949 
1950 #define list_next_entry(pos, member) \
1951  list_entry(pos->member.next, typeof(*pos), member)
1952 
1953 static void perf_event_sync_stat(struct perf_event_context *ctx,
1954  struct perf_event_context *next_ctx)
1955 {
1956  struct perf_event *event, *next_event;
1957 
1958  if (!ctx->nr_stat)
1959  return;
1960 
1961  update_context_time(ctx);
1962 
1963  event = list_first_entry(&ctx->event_list,
1964  struct perf_event, event_entry);
1965 
1966  next_event = list_first_entry(&next_ctx->event_list,
1967  struct perf_event, event_entry);
1968 
1969  while (&event->event_entry != &ctx->event_list &&
1970  &next_event->event_entry != &next_ctx->event_list) {
1971 
1972  __perf_event_sync_stat(event, next_event);
1973 
1974  event = list_next_entry(event, event_entry);
1975  next_event = list_next_entry(next_event, event_entry);
1976  }
1977 }
1978 
1979 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
1980  struct task_struct *next)
1981 {
1982  struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1983  struct perf_event_context *next_ctx;
1984  struct perf_event_context *parent;
1985  struct perf_cpu_context *cpuctx;
1986  int do_switch = 1;
1987 
1988  if (likely(!ctx))
1989  return;
1990 
1991  cpuctx = __get_cpu_context(ctx);
1992  if (!cpuctx->task_ctx)
1993  return;
1994 
1995  rcu_read_lock();
1996  parent = rcu_dereference(ctx->parent_ctx);
1997  next_ctx = next->perf_event_ctxp[ctxn];
1998  if (parent && next_ctx &&
1999  rcu_dereference(next_ctx->parent_ctx) == parent) {
2000  /*
2001  * Looks like the two contexts are clones, so we might be
2002  * able to optimize the context switch. We lock both
2003  * contexts and check that they are clones under the
2004  * lock (including re-checking that neither has been
2005  * uncloned in the meantime). It doesn't matter which
2006  * order we take the locks because no other cpu could
2007  * be trying to lock both of these tasks.
2008  */
2009  raw_spin_lock(&ctx->lock);
2011  if (context_equiv(ctx, next_ctx)) {
2012  /*
2013  * XXX do we need a memory barrier of sorts
2014  * wrt to rcu_dereference() of perf_event_ctxp
2015  */
2016  task->perf_event_ctxp[ctxn] = next_ctx;
2017  next->perf_event_ctxp[ctxn] = ctx;
2018  ctx->task = next;
2019  next_ctx->task = task;
2020  do_switch = 0;
2021 
2022  perf_event_sync_stat(ctx, next_ctx);
2023  }
2024  raw_spin_unlock(&next_ctx->lock);
2025  raw_spin_unlock(&ctx->lock);
2026  }
2027  rcu_read_unlock();
2028 
2029  if (do_switch) {
2030  raw_spin_lock(&ctx->lock);
2031  ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2032  cpuctx->task_ctx = NULL;
2033  raw_spin_unlock(&ctx->lock);
2034  }
2035 }
2036 
2037 #define for_each_task_context_nr(ctxn) \
2038  for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2039 
2040 /*
2041  * Called from scheduler to remove the events of the current task,
2042  * with interrupts disabled.
2043  *
2044  * We stop each event and update the event value in event->count.
2045  *
2046  * This does not protect us against NMI, but disable()
2047  * sets the disabled bit in the control field of event _before_
2048  * accessing the event control register. If a NMI hits, then it will
2049  * not restart the event.
2050  */
2052  struct task_struct *next)
2053 {
2054  int ctxn;
2055 
2057  perf_event_context_sched_out(task, ctxn, next);
2058 
2059  /*
2060  * if cgroup events exist on this CPU, then we need
2061  * to check if we have to switch out PMU state.
2062  * cgroup event are system-wide mode only
2063  */
2064  if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2065  perf_cgroup_sched_out(task, next);
2066 }
2067 
2068 static void task_ctx_sched_out(struct perf_event_context *ctx)
2069 {
2070  struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2071 
2072  if (!cpuctx->task_ctx)
2073  return;
2074 
2075  if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2076  return;
2077 
2078  ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2079  cpuctx->task_ctx = NULL;
2080 }
2081 
2082 /*
2083  * Called with IRQs disabled
2084  */
2085 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2086  enum event_type_t event_type)
2087 {
2088  ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2089 }
2090 
2091 static void
2092 ctx_pinned_sched_in(struct perf_event_context *ctx,
2093  struct perf_cpu_context *cpuctx)
2094 {
2095  struct perf_event *event;
2096 
2097  list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2098  if (event->state <= PERF_EVENT_STATE_OFF)
2099  continue;
2100  if (!event_filter_match(event))
2101  continue;
2102 
2103  /* may need to reset tstamp_enabled */
2104  if (is_cgroup_event(event))
2105  perf_cgroup_mark_enabled(event, ctx);
2106 
2107  if (group_can_go_on(event, cpuctx, 1))
2108  group_sched_in(event, cpuctx, ctx);
2109 
2110  /*
2111  * If this pinned group hasn't been scheduled,
2112  * put it in error state.
2113  */
2114  if (event->state == PERF_EVENT_STATE_INACTIVE) {
2115  update_group_times(event);
2116  event->state = PERF_EVENT_STATE_ERROR;
2117  }
2118  }
2119 }
2120 
2121 static void
2122 ctx_flexible_sched_in(struct perf_event_context *ctx,
2123  struct perf_cpu_context *cpuctx)
2124 {
2125  struct perf_event *event;
2126  int can_add_hw = 1;
2127 
2128  list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2129  /* Ignore events in OFF or ERROR state */
2130  if (event->state <= PERF_EVENT_STATE_OFF)
2131  continue;
2132  /*
2133  * Listen to the 'cpu' scheduling filter constraint
2134  * of events:
2135  */
2136  if (!event_filter_match(event))
2137  continue;
2138 
2139  /* may need to reset tstamp_enabled */
2140  if (is_cgroup_event(event))
2141  perf_cgroup_mark_enabled(event, ctx);
2142 
2143  if (group_can_go_on(event, cpuctx, can_add_hw)) {
2144  if (group_sched_in(event, cpuctx, ctx))
2145  can_add_hw = 0;
2146  }
2147  }
2148 }
2149 
2150 static void
2151 ctx_sched_in(struct perf_event_context *ctx,
2152  struct perf_cpu_context *cpuctx,
2153  enum event_type_t event_type,
2154  struct task_struct *task)
2155 {
2156  u64 now;
2157  int is_active = ctx->is_active;
2158 
2159  ctx->is_active |= event_type;
2160  if (likely(!ctx->nr_events))
2161  return;
2162 
2163  now = perf_clock();
2164  ctx->timestamp = now;
2165  perf_cgroup_set_timestamp(task, ctx);
2166  /*
2167  * First go through the list and put on any pinned groups
2168  * in order to give them the best chance of going on.
2169  */
2170  if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2171  ctx_pinned_sched_in(ctx, cpuctx);
2172 
2173  /* Then walk through the lower prio flexible groups */
2174  if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2175  ctx_flexible_sched_in(ctx, cpuctx);
2176 }
2177 
2178 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2179  enum event_type_t event_type,
2180  struct task_struct *task)
2181 {
2182  struct perf_event_context *ctx = &cpuctx->ctx;
2183 
2184  ctx_sched_in(ctx, cpuctx, event_type, task);
2185 }
2186 
2187 static void perf_event_context_sched_in(struct perf_event_context *ctx,
2188  struct task_struct *task)
2189 {
2190  struct perf_cpu_context *cpuctx;
2191 
2192  cpuctx = __get_cpu_context(ctx);
2193  if (cpuctx->task_ctx == ctx)
2194  return;
2195 
2196  perf_ctx_lock(cpuctx, ctx);
2197  perf_pmu_disable(ctx->pmu);
2198  /*
2199  * We want to keep the following priority order:
2200  * cpu pinned (that don't need to move), task pinned,
2201  * cpu flexible, task flexible.
2202  */
2203  cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2204 
2205  if (ctx->nr_events)
2206  cpuctx->task_ctx = ctx;
2207 
2208  perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2209 
2210  perf_pmu_enable(ctx->pmu);
2211  perf_ctx_unlock(cpuctx, ctx);
2212 
2213  /*
2214  * Since these rotations are per-cpu, we need to ensure the
2215  * cpu-context we got scheduled on is actually rotating.
2216  */
2217  perf_pmu_rotate_start(ctx->pmu);
2218 }
2219 
2220 /*
2221  * When sampling the branck stack in system-wide, it may be necessary
2222  * to flush the stack on context switch. This happens when the branch
2223  * stack does not tag its entries with the pid of the current task.
2224  * Otherwise it becomes impossible to associate a branch entry with a
2225  * task. This ambiguity is more likely to appear when the branch stack
2226  * supports priv level filtering and the user sets it to monitor only
2227  * at the user level (which could be a useful measurement in system-wide
2228  * mode). In that case, the risk is high of having a branch stack with
2229  * branch from multiple tasks. Flushing may mean dropping the existing
2230  * entries or stashing them somewhere in the PMU specific code layer.
2231  *
2232  * This function provides the context switch callback to the lower code
2233  * layer. It is invoked ONLY when there is at least one system-wide context
2234  * with at least one active event using taken branch sampling.
2235  */
2236 static void perf_branch_stack_sched_in(struct task_struct *prev,
2237  struct task_struct *task)
2238 {
2239  struct perf_cpu_context *cpuctx;
2240  struct pmu *pmu;
2241  unsigned long flags;
2242 
2243  /* no need to flush branch stack if not changing task */
2244  if (prev == task)
2245  return;
2246 
2247  local_irq_save(flags);
2248 
2249  rcu_read_lock();
2250 
2251  list_for_each_entry_rcu(pmu, &pmus, entry) {
2252  cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2253 
2254  /*
2255  * check if the context has at least one
2256  * event using PERF_SAMPLE_BRANCH_STACK
2257  */
2258  if (cpuctx->ctx.nr_branch_stack > 0
2259  && pmu->flush_branch_stack) {
2260 
2261  pmu = cpuctx->ctx.pmu;
2262 
2263  perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2264 
2265  perf_pmu_disable(pmu);
2266 
2267  pmu->flush_branch_stack();
2268 
2269  perf_pmu_enable(pmu);
2270 
2271  perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2272  }
2273  }
2274 
2275  rcu_read_unlock();
2276 
2277  local_irq_restore(flags);
2278 }
2279 
2280 /*
2281  * Called from scheduler to add the events of the current task
2282  * with interrupts disabled.
2283  *
2284  * We restore the event value and then enable it.
2285  *
2286  * This does not protect us against NMI, but enable()
2287  * sets the enabled bit in the control field of event _before_
2288  * accessing the event control register. If a NMI hits, then it will
2289  * keep the event running.
2290  */
2292  struct task_struct *task)
2293 {
2294  struct perf_event_context *ctx;
2295  int ctxn;
2296 
2297  for_each_task_context_nr(ctxn) {
2298  ctx = task->perf_event_ctxp[ctxn];
2299  if (likely(!ctx))
2300  continue;
2301 
2302  perf_event_context_sched_in(ctx, task);
2303  }
2304  /*
2305  * if cgroup events exist on this CPU, then we need
2306  * to check if we have to switch in PMU state.
2307  * cgroup event are system-wide mode only
2308  */
2309  if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2310  perf_cgroup_sched_in(prev, task);
2311 
2312  /* check for system-wide branch_stack events */
2313  if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
2314  perf_branch_stack_sched_in(prev, task);
2315 }
2316 
2317 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2318 {
2319  u64 frequency = event->attr.sample_freq;
2320  u64 sec = NSEC_PER_SEC;
2321  u64 divisor, dividend;
2322 
2323  int count_fls, nsec_fls, frequency_fls, sec_fls;
2324 
2325  count_fls = fls64(count);
2326  nsec_fls = fls64(nsec);
2327  frequency_fls = fls64(frequency);
2328  sec_fls = 30;
2329 
2330  /*
2331  * We got @count in @nsec, with a target of sample_freq HZ
2332  * the target period becomes:
2333  *
2334  * @count * 10^9
2335  * period = -------------------
2336  * @nsec * sample_freq
2337  *
2338  */
2339 
2340  /*
2341  * Reduce accuracy by one bit such that @a and @b converge
2342  * to a similar magnitude.
2343  */
2344 #define REDUCE_FLS(a, b) \
2345 do { \
2346  if (a##_fls > b##_fls) { \
2347  a >>= 1; \
2348  a##_fls--; \
2349  } else { \
2350  b >>= 1; \
2351  b##_fls--; \
2352  } \
2353 } while (0)
2354 
2355  /*
2356  * Reduce accuracy until either term fits in a u64, then proceed with
2357  * the other, so that finally we can do a u64/u64 division.
2358  */
2359  while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2360  REDUCE_FLS(nsec, frequency);
2361  REDUCE_FLS(sec, count);
2362  }
2363 
2364  if (count_fls + sec_fls > 64) {
2365  divisor = nsec * frequency;
2366 
2367  while (count_fls + sec_fls > 64) {
2368  REDUCE_FLS(count, sec);
2369  divisor >>= 1;
2370  }
2371 
2372  dividend = count * sec;
2373  } else {
2374  dividend = count * sec;
2375 
2376  while (nsec_fls + frequency_fls > 64) {
2377  REDUCE_FLS(nsec, frequency);
2378  dividend >>= 1;
2379  }
2380 
2381  divisor = nsec * frequency;
2382  }
2383 
2384  if (!divisor)
2385  return dividend;
2386 
2387  return div64_u64(dividend, divisor);
2388 }
2389 
2390 static DEFINE_PER_CPU(int, perf_throttled_count);
2391 static DEFINE_PER_CPU(u64, perf_throttled_seq);
2392 
2393 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2394 {
2395  struct hw_perf_event *hwc = &event->hw;
2396  s64 period, sample_period;
2397  s64 delta;
2398 
2399  period = perf_calculate_period(event, nsec, count);
2400 
2401  delta = (s64)(period - hwc->sample_period);
2402  delta = (delta + 7) / 8; /* low pass filter */
2403 
2404  sample_period = hwc->sample_period + delta;
2405 
2406  if (!sample_period)
2407  sample_period = 1;
2408 
2409  hwc->sample_period = sample_period;
2410 
2411  if (local64_read(&hwc->period_left) > 8*sample_period) {
2412  if (disable)
2413  event->pmu->stop(event, PERF_EF_UPDATE);
2414 
2415  local64_set(&hwc->period_left, 0);
2416 
2417  if (disable)
2418  event->pmu->start(event, PERF_EF_RELOAD);
2419  }
2420 }
2421 
2422 /*
2423  * combine freq adjustment with unthrottling to avoid two passes over the
2424  * events. At the same time, make sure, having freq events does not change
2425  * the rate of unthrottling as that would introduce bias.
2426  */
2427 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2428  int needs_unthr)
2429 {
2430  struct perf_event *event;
2431  struct hw_perf_event *hwc;
2432  u64 now, period = TICK_NSEC;
2433  s64 delta;
2434 
2435  /*
2436  * only need to iterate over all events iff:
2437  * - context have events in frequency mode (needs freq adjust)
2438  * - there are events to unthrottle on this cpu
2439  */
2440  if (!(ctx->nr_freq || needs_unthr))
2441  return;
2442 
2443  raw_spin_lock(&ctx->lock);
2444  perf_pmu_disable(ctx->pmu);
2445 
2446  list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2447  if (event->state != PERF_EVENT_STATE_ACTIVE)
2448  continue;
2449 
2450  if (!event_filter_match(event))
2451  continue;
2452 
2453  hwc = &event->hw;
2454 
2455  if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
2456  hwc->interrupts = 0;
2457  perf_log_throttle(event, 1);
2458  event->pmu->start(event, 0);
2459  }
2460 
2461  if (!event->attr.freq || !event->attr.sample_freq)
2462  continue;
2463 
2464  /*
2465  * stop the event and update event->count
2466  */
2467  event->pmu->stop(event, PERF_EF_UPDATE);
2468 
2469  now = local64_read(&event->count);
2470  delta = now - hwc->freq_count_stamp;
2471  hwc->freq_count_stamp = now;
2472 
2473  /*
2474  * restart the event
2475  * reload only if value has changed
2476  * we have stopped the event so tell that
2477  * to perf_adjust_period() to avoid stopping it
2478  * twice.
2479  */
2480  if (delta > 0)
2481  perf_adjust_period(event, period, delta, false);
2482 
2483  event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
2484  }
2485 
2486  perf_pmu_enable(ctx->pmu);
2487  raw_spin_unlock(&ctx->lock);
2488 }
2489 
2490 /*
2491  * Round-robin a context's events:
2492  */
2493 static void rotate_ctx(struct perf_event_context *ctx)
2494 {
2495  /*
2496  * Rotate the first entry last of non-pinned groups. Rotation might be
2497  * disabled by the inheritance code.
2498  */
2499  if (!ctx->rotate_disable)
2500  list_rotate_left(&ctx->flexible_groups);
2501 }
2502 
2503 /*
2504  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2505  * because they're strictly cpu affine and rotate_start is called with IRQs
2506  * disabled, while rotate_context is called from IRQ context.
2507  */
2508 static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2509 {
2510  struct perf_event_context *ctx = NULL;
2511  int rotate = 0, remove = 1;
2512 
2513  if (cpuctx->ctx.nr_events) {
2514  remove = 0;
2515  if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2516  rotate = 1;
2517  }
2518 
2519  ctx = cpuctx->task_ctx;
2520  if (ctx && ctx->nr_events) {
2521  remove = 0;
2522  if (ctx->nr_events != ctx->nr_active)
2523  rotate = 1;
2524  }
2525 
2526  if (!rotate)
2527  goto done;
2528 
2529  perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2530  perf_pmu_disable(cpuctx->ctx.pmu);
2531 
2532  cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2533  if (ctx)
2534  ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2535 
2536  rotate_ctx(&cpuctx->ctx);
2537  if (ctx)
2538  rotate_ctx(ctx);
2539 
2540  perf_event_sched_in(cpuctx, ctx, current);
2541 
2542  perf_pmu_enable(cpuctx->ctx.pmu);
2543  perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2544 done:
2545  if (remove)
2546  list_del_init(&cpuctx->rotation_list);
2547 }
2548 
2550 {
2551  struct list_head *head = &__get_cpu_var(rotation_list);
2552  struct perf_cpu_context *cpuctx, *tmp;
2553  struct perf_event_context *ctx;
2554  int throttled;
2555 
2556  WARN_ON(!irqs_disabled());
2557 
2558  __this_cpu_inc(perf_throttled_seq);
2559  throttled = __this_cpu_xchg(perf_throttled_count, 0);
2560 
2561  list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2562  ctx = &cpuctx->ctx;
2563  perf_adjust_freq_unthr_context(ctx, throttled);
2564 
2565  ctx = cpuctx->task_ctx;
2566  if (ctx)
2567  perf_adjust_freq_unthr_context(ctx, throttled);
2568 
2569  if (cpuctx->jiffies_interval == 1 ||
2570  !(jiffies % cpuctx->jiffies_interval))
2571  perf_rotate_context(cpuctx);
2572  }
2573 }
2574 
2575 static int event_enable_on_exec(struct perf_event *event,
2576  struct perf_event_context *ctx)
2577 {
2578  if (!event->attr.enable_on_exec)
2579  return 0;
2580 
2581  event->attr.enable_on_exec = 0;
2582  if (event->state >= PERF_EVENT_STATE_INACTIVE)
2583  return 0;
2584 
2585  __perf_event_mark_enabled(event);
2586 
2587  return 1;
2588 }
2589 
2590 /*
2591  * Enable all of a task's events that have been marked enable-on-exec.
2592  * This expects task == current.
2593  */
2594 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2595 {
2596  struct perf_event *event;
2597  unsigned long flags;
2598  int enabled = 0;
2599  int ret;
2600 
2601  local_irq_save(flags);
2602  if (!ctx || !ctx->nr_events)
2603  goto out;
2604 
2605  /*
2606  * We must ctxsw out cgroup events to avoid conflict
2607  * when invoking perf_task_event_sched_in() later on
2608  * in this function. Otherwise we end up trying to
2609  * ctxswin cgroup events which are already scheduled
2610  * in.
2611  */
2612  perf_cgroup_sched_out(current, NULL);
2613 
2614  raw_spin_lock(&ctx->lock);
2615  task_ctx_sched_out(ctx);
2616 
2617  list_for_each_entry(event, &ctx->event_list, event_entry) {
2618  ret = event_enable_on_exec(event, ctx);
2619  if (ret)
2620  enabled = 1;
2621  }
2622 
2623  /*
2624  * Unclone this context if we enabled any event.
2625  */
2626  if (enabled)
2627  unclone_ctx(ctx);
2628 
2629  raw_spin_unlock(&ctx->lock);
2630 
2631  /*
2632  * Also calls ctxswin for cgroup events, if any:
2633  */
2634  perf_event_context_sched_in(ctx, ctx->task);
2635 out:
2636  local_irq_restore(flags);
2637 }
2638 
2639 /*
2640  * Cross CPU call to read the hardware event
2641  */
2642 static void __perf_event_read(void *info)
2643 {
2644  struct perf_event *event = info;
2645  struct perf_event_context *ctx = event->ctx;
2646  struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2647 
2648  /*
2649  * If this is a task context, we need to check whether it is
2650  * the current task context of this cpu. If not it has been
2651  * scheduled out before the smp call arrived. In that case
2652  * event->count would have been updated to a recent sample
2653  * when the event was scheduled out.
2654  */
2655  if (ctx->task && cpuctx->task_ctx != ctx)
2656  return;
2657 
2658  raw_spin_lock(&ctx->lock);
2659  if (ctx->is_active) {
2660  update_context_time(ctx);
2661  update_cgrp_time_from_event(event);
2662  }
2663  update_event_times(event);
2664  if (event->state == PERF_EVENT_STATE_ACTIVE)
2665  event->pmu->read(event);
2666  raw_spin_unlock(&ctx->lock);
2667 }
2668 
2669 static inline u64 perf_event_count(struct perf_event *event)
2670 {
2671  return local64_read(&event->count) + atomic64_read(&event->child_count);
2672 }
2673 
2674 static u64 perf_event_read(struct perf_event *event)
2675 {
2676  /*
2677  * If event is enabled and currently active on a CPU, update the
2678  * value in the event structure:
2679  */
2680  if (event->state == PERF_EVENT_STATE_ACTIVE) {
2681  smp_call_function_single(event->oncpu,
2682  __perf_event_read, event, 1);
2683  } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
2684  struct perf_event_context *ctx = event->ctx;
2685  unsigned long flags;
2686 
2687  raw_spin_lock_irqsave(&ctx->lock, flags);
2688  /*
2689  * may read while context is not active
2690  * (e.g., thread is blocked), in that case
2691  * we cannot update context time
2692  */
2693  if (ctx->is_active) {
2694  update_context_time(ctx);
2695  update_cgrp_time_from_event(event);
2696  }
2697  update_event_times(event);
2698  raw_spin_unlock_irqrestore(&ctx->lock, flags);
2699  }
2700 
2701  return perf_event_count(event);
2702 }
2703 
2704 /*
2705  * Initialize the perf_event context in a task_struct:
2706  */
2707 static void __perf_event_init_context(struct perf_event_context *ctx)
2708 {
2709  raw_spin_lock_init(&ctx->lock);
2710  mutex_init(&ctx->mutex);
2711  INIT_LIST_HEAD(&ctx->pinned_groups);
2712  INIT_LIST_HEAD(&ctx->flexible_groups);
2713  INIT_LIST_HEAD(&ctx->event_list);
2714  atomic_set(&ctx->refcount, 1);
2715 }
2716 
2717 static struct perf_event_context *
2718 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
2719 {
2720  struct perf_event_context *ctx;
2721 
2722  ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2723  if (!ctx)
2724  return NULL;
2725 
2726  __perf_event_init_context(ctx);
2727  if (task) {
2728  ctx->task = task;
2729  get_task_struct(task);
2730  }
2731  ctx->pmu = pmu;
2732 
2733  return ctx;
2734 }
2735 
2736 static struct task_struct *
2737 find_lively_task_by_vpid(pid_t vpid)
2738 {
2739  struct task_struct *task;
2740  int err;
2741 
2742  rcu_read_lock();
2743  if (!vpid)
2744  task = current;
2745  else
2746  task = find_task_by_vpid(vpid);
2747  if (task)
2748  get_task_struct(task);
2749  rcu_read_unlock();
2750 
2751  if (!task)
2752  return ERR_PTR(-ESRCH);
2753 
2754  /* Reuse ptrace permission checks for now. */
2755  err = -EACCES;
2756  if (!ptrace_may_access(task, PTRACE_MODE_READ))
2757  goto errout;
2758 
2759  return task;
2760 errout:
2761  put_task_struct(task);
2762  return ERR_PTR(err);
2763 
2764 }
2765 
2766 /*
2767  * Returns a matching context with refcount and pincount.
2768  */
2769 static struct perf_event_context *
2770 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
2771 {
2772  struct perf_event_context *ctx;
2773  struct perf_cpu_context *cpuctx;
2774  unsigned long flags;
2775  int ctxn, err;
2776 
2777  if (!task) {
2778  /* Must be root to operate on a CPU event: */
2779  if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2780  return ERR_PTR(-EACCES);
2781 
2782  /*
2783  * We could be clever and allow to attach a event to an
2784  * offline CPU and activate it when the CPU comes up, but
2785  * that's for later.
2786  */
2787  if (!cpu_online(cpu))
2788  return ERR_PTR(-ENODEV);
2789 
2790  cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2791  ctx = &cpuctx->ctx;
2792  get_ctx(ctx);
2793  ++ctx->pin_count;
2794 
2795  return ctx;
2796  }
2797 
2798  err = -EINVAL;
2799  ctxn = pmu->task_ctx_nr;
2800  if (ctxn < 0)
2801  goto errout;
2802 
2803 retry:
2804  ctx = perf_lock_task_context(task, ctxn, &flags);
2805  if (ctx) {
2806  unclone_ctx(ctx);
2807  ++ctx->pin_count;
2808  raw_spin_unlock_irqrestore(&ctx->lock, flags);
2809  } else {
2810  ctx = alloc_perf_context(pmu, task);
2811  err = -ENOMEM;
2812  if (!ctx)
2813  goto errout;
2814 
2815  err = 0;
2816  mutex_lock(&task->perf_event_mutex);
2817  /*
2818  * If it has already passed perf_event_exit_task().
2819  * we must see PF_EXITING, it takes this mutex too.
2820  */
2821  if (task->flags & PF_EXITING)
2822  err = -ESRCH;
2823  else if (task->perf_event_ctxp[ctxn])
2824  err = -EAGAIN;
2825  else {
2826  get_ctx(ctx);
2827  ++ctx->pin_count;
2828  rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2829  }
2830  mutex_unlock(&task->perf_event_mutex);
2831 
2832  if (unlikely(err)) {
2833  put_ctx(ctx);
2834 
2835  if (err == -EAGAIN)
2836  goto retry;
2837  goto errout;
2838  }
2839  }
2840 
2841  return ctx;
2842 
2843 errout:
2844  return ERR_PTR(err);
2845 }
2846 
2847 static void perf_event_free_filter(struct perf_event *event);
2848 
2849 static void free_event_rcu(struct rcu_head *head)
2850 {
2851  struct perf_event *event;
2852 
2853  event = container_of(head, struct perf_event, rcu_head);
2854  if (event->ns)
2855  put_pid_ns(event->ns);
2856  perf_event_free_filter(event);
2857  kfree(event);
2858 }
2859 
2860 static void ring_buffer_put(struct ring_buffer *rb);
2861 
2862 static void free_event(struct perf_event *event)
2863 {
2864  irq_work_sync(&event->pending);
2865 
2866  if (!event->parent) {
2867  if (event->attach_state & PERF_ATTACH_TASK)
2868  static_key_slow_dec_deferred(&perf_sched_events);
2869  if (event->attr.mmap || event->attr.mmap_data)
2870  atomic_dec(&nr_mmap_events);
2871  if (event->attr.comm)
2872  atomic_dec(&nr_comm_events);
2873  if (event->attr.task)
2874  atomic_dec(&nr_task_events);
2875  if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
2877  if (is_cgroup_event(event)) {
2878  atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
2879  static_key_slow_dec_deferred(&perf_sched_events);
2880  }
2881 
2882  if (has_branch_stack(event)) {
2883  static_key_slow_dec_deferred(&perf_sched_events);
2884  /* is system-wide event */
2885  if (!(event->attach_state & PERF_ATTACH_TASK))
2886  atomic_dec(&per_cpu(perf_branch_stack_events,
2887  event->cpu));
2888  }
2889  }
2890 
2891  if (event->rb) {
2892  ring_buffer_put(event->rb);
2893  event->rb = NULL;
2894  }
2895 
2896  if (is_cgroup_event(event))
2897  perf_detach_cgroup(event);
2898 
2899  if (event->destroy)
2900  event->destroy(event);
2901 
2902  if (event->ctx)
2903  put_ctx(event->ctx);
2904 
2905  call_rcu(&event->rcu_head, free_event_rcu);
2906 }
2907 
2909 {
2910  struct perf_event_context *ctx = event->ctx;
2911 
2912  WARN_ON_ONCE(ctx->parent_ctx);
2913  /*
2914  * There are two ways this annotation is useful:
2915  *
2916  * 1) there is a lock recursion from perf_event_exit_task
2917  * see the comment there.
2918  *
2919  * 2) there is a lock-inversion with mmap_sem through
2920  * perf_event_read_group(), which takes faults while
2921  * holding ctx->mutex, however this is called after
2922  * the last filedesc died, so there is no possibility
2923  * to trigger the AB-BA case.
2924  */
2926  raw_spin_lock_irq(&ctx->lock);
2927  perf_group_detach(event);
2928  raw_spin_unlock_irq(&ctx->lock);
2929  perf_remove_from_context(event);
2930  mutex_unlock(&ctx->mutex);
2931 
2932  free_event(event);
2933 
2934  return 0;
2935 }
2937 
2938 /*
2939  * Called when the last reference to the file is gone.
2940  */
2941 static void put_event(struct perf_event *event)
2942 {
2943  struct task_struct *owner;
2944 
2945  if (!atomic_long_dec_and_test(&event->refcount))
2946  return;
2947 
2948  rcu_read_lock();
2949  owner = ACCESS_ONCE(event->owner);
2950  /*
2951  * Matches the smp_wmb() in perf_event_exit_task(). If we observe
2952  * !owner it means the list deletion is complete and we can indeed
2953  * free this event, otherwise we need to serialize on
2954  * owner->perf_event_mutex.
2955  */
2957  if (owner) {
2958  /*
2959  * Since delayed_put_task_struct() also drops the last
2960  * task reference we can safely take a new reference
2961  * while holding the rcu_read_lock().
2962  */
2963  get_task_struct(owner);
2964  }
2965  rcu_read_unlock();
2966 
2967  if (owner) {
2968  mutex_lock(&owner->perf_event_mutex);
2969  /*
2970  * We have to re-check the event->owner field, if it is cleared
2971  * we raced with perf_event_exit_task(), acquiring the mutex
2972  * ensured they're done, and we can proceed with freeing the
2973  * event.
2974  */
2975  if (event->owner)
2976  list_del_init(&event->owner_entry);
2977  mutex_unlock(&owner->perf_event_mutex);
2978  put_task_struct(owner);
2979  }
2980 
2982 }
2983 
2984 static int perf_release(struct inode *inode, struct file *file)
2985 {
2986  put_event(file->private_data);
2987  return 0;
2988 }
2989 
2990 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2991 {
2992  struct perf_event *child;
2993  u64 total = 0;
2994 
2995  *enabled = 0;
2996  *running = 0;
2997 
2998  mutex_lock(&event->child_mutex);
2999  total += perf_event_read(event);
3000  *enabled += event->total_time_enabled +
3001  atomic64_read(&event->child_total_time_enabled);
3002  *running += event->total_time_running +
3003  atomic64_read(&event->child_total_time_running);
3004 
3005  list_for_each_entry(child, &event->child_list, child_list) {
3006  total += perf_event_read(child);
3007  *enabled += child->total_time_enabled;
3008  *running += child->total_time_running;
3009  }
3010  mutex_unlock(&event->child_mutex);
3011 
3012  return total;
3013 }
3015 
3016 static int perf_event_read_group(struct perf_event *event,
3017  u64 read_format, char __user *buf)
3018 {
3019  struct perf_event *leader = event->group_leader, *sub;
3020  int n = 0, size = 0, ret = -EFAULT;
3021  struct perf_event_context *ctx = leader->ctx;
3022  u64 values[5];
3024 
3025  mutex_lock(&ctx->mutex);
3026  count = perf_event_read_value(leader, &enabled, &running);
3027 
3028  values[n++] = 1 + leader->nr_siblings;
3029  if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3030  values[n++] = enabled;
3031  if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3032  values[n++] = running;
3033  values[n++] = count;
3034  if (read_format & PERF_FORMAT_ID)
3035  values[n++] = primary_event_id(leader);
3036 
3037  size = n * sizeof(u64);
3038 
3039  if (copy_to_user(buf, values, size))
3040  goto unlock;
3041 
3042  ret = size;
3043 
3044  list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3045  n = 0;
3046 
3047  values[n++] = perf_event_read_value(sub, &enabled, &running);
3048  if (read_format & PERF_FORMAT_ID)
3049  values[n++] = primary_event_id(sub);
3050 
3051  size = n * sizeof(u64);
3052 
3053  if (copy_to_user(buf + ret, values, size)) {
3054  ret = -EFAULT;
3055  goto unlock;
3056  }
3057 
3058  ret += size;
3059  }
3060 unlock:
3061  mutex_unlock(&ctx->mutex);
3062 
3063  return ret;
3064 }
3065 
3066 static int perf_event_read_one(struct perf_event *event,
3067  u64 read_format, char __user *buf)
3068 {
3069  u64 enabled, running;
3070  u64 values[4];
3071  int n = 0;
3072 
3073  values[n++] = perf_event_read_value(event, &enabled, &running);
3074  if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3075  values[n++] = enabled;
3076  if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3077  values[n++] = running;
3078  if (read_format & PERF_FORMAT_ID)
3079  values[n++] = primary_event_id(event);
3080 
3081  if (copy_to_user(buf, values, n * sizeof(u64)))
3082  return -EFAULT;
3083 
3084  return n * sizeof(u64);
3085 }
3086 
3087 /*
3088  * Read the performance event - simple non blocking version for now
3089  */
3090 static ssize_t
3091 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3092 {
3093  u64 read_format = event->attr.read_format;
3094  int ret;
3095 
3096  /*
3097  * Return end-of-file for a read on a event that is in
3098  * error state (i.e. because it was pinned but it couldn't be
3099  * scheduled on to the CPU at some point).
3100  */
3101  if (event->state == PERF_EVENT_STATE_ERROR)
3102  return 0;
3103 
3104  if (count < event->read_size)
3105  return -ENOSPC;
3106 
3107  WARN_ON_ONCE(event->ctx->parent_ctx);
3108  if (read_format & PERF_FORMAT_GROUP)
3109  ret = perf_event_read_group(event, read_format, buf);
3110  else
3111  ret = perf_event_read_one(event, read_format, buf);
3112 
3113  return ret;
3114 }
3115 
3116 static ssize_t
3117 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3118 {
3119  struct perf_event *event = file->private_data;
3120 
3121  return perf_read_hw(event, buf, count);
3122 }
3123 
3124 static unsigned int perf_poll(struct file *file, poll_table *wait)
3125 {
3126  struct perf_event *event = file->private_data;
3127  struct ring_buffer *rb;
3128  unsigned int events = POLL_HUP;
3129 
3130  /*
3131  * Race between perf_event_set_output() and perf_poll(): perf_poll()
3132  * grabs the rb reference but perf_event_set_output() overrides it.
3133  * Here is the timeline for two threads T1, T2:
3134  * t0: T1, rb = rcu_dereference(event->rb)
3135  * t1: T2, old_rb = event->rb
3136  * t2: T2, event->rb = new rb
3137  * t3: T2, ring_buffer_detach(old_rb)
3138  * t4: T1, ring_buffer_attach(rb1)
3139  * t5: T1, poll_wait(event->waitq)
3140  *
3141  * To avoid this problem, we grab mmap_mutex in perf_poll()
3142  * thereby ensuring that the assignment of the new ring buffer
3143  * and the detachment of the old buffer appear atomic to perf_poll()
3144  */
3145  mutex_lock(&event->mmap_mutex);
3146 
3147  rcu_read_lock();
3148  rb = rcu_dereference(event->rb);
3149  if (rb) {
3150  ring_buffer_attach(event, rb);
3151  events = atomic_xchg(&rb->poll, 0);
3152  }
3153  rcu_read_unlock();
3154 
3155  mutex_unlock(&event->mmap_mutex);
3156 
3157  poll_wait(file, &event->waitq, wait);
3158 
3159  return events;
3160 }
3161 
3162 static void perf_event_reset(struct perf_event *event)
3163 {
3164  (void)perf_event_read(event);
3165  local64_set(&event->count, 0);
3167 }
3168 
3169 /*
3170  * Holding the top-level event's child_mutex means that any
3171  * descendant process that has inherited this event will block
3172  * in sync_child_event if it goes to exit, thus satisfying the
3173  * task existence requirements of perf_event_enable/disable.
3174  */
3175 static void perf_event_for_each_child(struct perf_event *event,
3176  void (*func)(struct perf_event *))
3177 {
3178  struct perf_event *child;
3179 
3180  WARN_ON_ONCE(event->ctx->parent_ctx);
3181  mutex_lock(&event->child_mutex);
3182  func(event);
3183  list_for_each_entry(child, &event->child_list, child_list)
3184  func(child);
3185  mutex_unlock(&event->child_mutex);
3186 }
3187 
3188 static void perf_event_for_each(struct perf_event *event,
3189  void (*func)(struct perf_event *))
3190 {
3191  struct perf_event_context *ctx = event->ctx;
3192  struct perf_event *sibling;
3193 
3194  WARN_ON_ONCE(ctx->parent_ctx);
3195  mutex_lock(&ctx->mutex);
3196  event = event->group_leader;
3197 
3198  perf_event_for_each_child(event, func);
3199  list_for_each_entry(sibling, &event->sibling_list, group_entry)
3200  perf_event_for_each_child(sibling, func);
3201  mutex_unlock(&ctx->mutex);
3202 }
3203 
3204 static int perf_event_period(struct perf_event *event, u64 __user *arg)
3205 {
3206  struct perf_event_context *ctx = event->ctx;
3207  int ret = 0;
3208  u64 value;
3209 
3210  if (!is_sampling_event(event))
3211  return -EINVAL;
3212 
3213  if (copy_from_user(&value, arg, sizeof(value)))
3214  return -EFAULT;
3215 
3216  if (!value)
3217  return -EINVAL;
3218 
3219  raw_spin_lock_irq(&ctx->lock);
3220  if (event->attr.freq) {
3221  if (value > sysctl_perf_event_sample_rate) {
3222  ret = -EINVAL;
3223  goto unlock;
3224  }
3225 
3226  event->attr.sample_freq = value;
3227  } else {
3228  event->attr.sample_period = value;
3229  event->hw.sample_period = value;
3230  }
3231 unlock:
3232  raw_spin_unlock_irq(&ctx->lock);
3233 
3234  return ret;
3235 }
3236 
3237 static const struct file_operations perf_fops;
3238 
3239 static inline int perf_fget_light(int fd, struct fd *p)
3240 {
3241  struct fd f = fdget(fd);
3242  if (!f.file)
3243  return -EBADF;
3244 
3245  if (f.file->f_op != &perf_fops) {
3246  fdput(f);
3247  return -EBADF;
3248  }
3249  *p = f;
3250  return 0;
3251 }
3252 
3253 static int perf_event_set_output(struct perf_event *event,
3254  struct perf_event *output_event);
3255 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3256 
3257 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3258 {
3259  struct perf_event *event = file->private_data;
3260  void (*func)(struct perf_event *);
3261  u32 flags = arg;
3262 
3263  switch (cmd) {
3264  case PERF_EVENT_IOC_ENABLE:
3265  func = perf_event_enable;
3266  break;
3268  func = perf_event_disable;
3269  break;
3270  case PERF_EVENT_IOC_RESET:
3271  func = perf_event_reset;
3272  break;
3273 
3275  return perf_event_refresh(event, arg);
3276 
3277  case PERF_EVENT_IOC_PERIOD:
3278  return perf_event_period(event, (u64 __user *)arg);
3279 
3281  {
3282  int ret;
3283  if (arg != -1) {
3284  struct perf_event *output_event;
3285  struct fd output;
3286  ret = perf_fget_light(arg, &output);
3287  if (ret)
3288  return ret;
3289  output_event = output.file->private_data;
3290  ret = perf_event_set_output(event, output_event);
3291  fdput(output);
3292  } else {
3293  ret = perf_event_set_output(event, NULL);
3294  }
3295  return ret;
3296  }
3297 
3299  return perf_event_set_filter(event, (void __user *)arg);
3300 
3301  default:
3302  return -ENOTTY;
3303  }
3304 
3305  if (flags & PERF_IOC_FLAG_GROUP)
3306  perf_event_for_each(event, func);
3307  else
3308  perf_event_for_each_child(event, func);
3309 
3310  return 0;
3311 }
3312 
3314 {
3315  struct perf_event *event;
3316 
3317  mutex_lock(&current->perf_event_mutex);
3318  list_for_each_entry(event, &current->perf_event_list, owner_entry)
3319  perf_event_for_each_child(event, perf_event_enable);
3320  mutex_unlock(&current->perf_event_mutex);
3321 
3322  return 0;
3323 }
3324 
3326 {
3327  struct perf_event *event;
3328 
3329  mutex_lock(&current->perf_event_mutex);
3330  list_for_each_entry(event, &current->perf_event_list, owner_entry)
3331  perf_event_for_each_child(event, perf_event_disable);
3332  mutex_unlock(&current->perf_event_mutex);
3333 
3334  return 0;
3335 }
3336 
3337 static int perf_event_index(struct perf_event *event)
3338 {
3339  if (event->hw.state & PERF_HES_STOPPED)
3340  return 0;
3341 
3342  if (event->state != PERF_EVENT_STATE_ACTIVE)
3343  return 0;
3344 
3345  return event->pmu->event_idx(event);
3346 }
3347 
3348 static void calc_timer_values(struct perf_event *event,
3349  u64 *now,
3350  u64 *enabled,
3351  u64 *running)
3352 {
3353  u64 ctx_time;
3354 
3355  *now = perf_clock();
3356  ctx_time = event->shadow_ctx_time + *now;
3357  *enabled = ctx_time - event->tstamp_enabled;
3358  *running = ctx_time - event->tstamp_running;
3359 }
3360 
3362 {
3363 }
3364 
3365 /*
3366  * Callers need to ensure there can be no nesting of this function, otherwise
3367  * the seqlock logic goes bad. We can not serialize this because the arch
3368  * code calls this from NMI context.
3369  */
3370 void perf_event_update_userpage(struct perf_event *event)
3371 {
3372  struct perf_event_mmap_page *userpg;
3373  struct ring_buffer *rb;
3374  u64 enabled, running, now;
3375 
3376  rcu_read_lock();
3377  /*
3378  * compute total_time_enabled, total_time_running
3379  * based on snapshot values taken when the event
3380  * was last scheduled in.
3381  *
3382  * we cannot simply called update_context_time()
3383  * because of locking issue as we can be called in
3384  * NMI context
3385  */
3386  calc_timer_values(event, &now, &enabled, &running);
3387  rb = rcu_dereference(event->rb);
3388  if (!rb)
3389  goto unlock;
3390 
3391  userpg = rb->user_page;
3392 
3393  /*
3394  * Disable preemption so as to not let the corresponding user-space
3395  * spin too long if we get preempted.
3396  */
3397  preempt_disable();
3398  ++userpg->lock;
3399  barrier();
3400  userpg->index = perf_event_index(event);
3401  userpg->offset = perf_event_count(event);
3402  if (userpg->index)
3403  userpg->offset -= local64_read(&event->hw.prev_count);
3404 
3405  userpg->time_enabled = enabled +
3406  atomic64_read(&event->child_total_time_enabled);
3407 
3408  userpg->time_running = running +
3409  atomic64_read(&event->child_total_time_running);
3410 
3411  arch_perf_update_userpage(userpg, now);
3412 
3413  barrier();
3414  ++userpg->lock;
3415  preempt_enable();
3416 unlock:
3417  rcu_read_unlock();
3418 }
3419 
3420 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3421 {
3422  struct perf_event *event = vma->vm_file->private_data;
3423  struct ring_buffer *rb;
3424  int ret = VM_FAULT_SIGBUS;
3425 
3426  if (vmf->flags & FAULT_FLAG_MKWRITE) {
3427  if (vmf->pgoff == 0)
3428  ret = 0;
3429  return ret;
3430  }
3431 
3432  rcu_read_lock();
3433  rb = rcu_dereference(event->rb);
3434  if (!rb)
3435  goto unlock;
3436 
3437  if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
3438  goto unlock;
3439 
3440  vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
3441  if (!vmf->page)
3442  goto unlock;
3443 
3444  get_page(vmf->page);
3445  vmf->page->mapping = vma->vm_file->f_mapping;
3446  vmf->page->index = vmf->pgoff;
3447 
3448  ret = 0;
3449 unlock:
3450  rcu_read_unlock();
3451 
3452  return ret;
3453 }
3454 
3455 static void ring_buffer_attach(struct perf_event *event,
3456  struct ring_buffer *rb)
3457 {
3458  unsigned long flags;
3459 
3460  if (!list_empty(&event->rb_entry))
3461  return;
3462 
3463  spin_lock_irqsave(&rb->event_lock, flags);
3464  if (!list_empty(&event->rb_entry))
3465  goto unlock;
3466 
3467  list_add(&event->rb_entry, &rb->event_list);
3468 unlock:
3469  spin_unlock_irqrestore(&rb->event_lock, flags);
3470 }
3471 
3472 static void ring_buffer_detach(struct perf_event *event,
3473  struct ring_buffer *rb)
3474 {
3475  unsigned long flags;
3476 
3477  if (list_empty(&event->rb_entry))
3478  return;
3479 
3480  spin_lock_irqsave(&rb->event_lock, flags);
3481  list_del_init(&event->rb_entry);
3482  wake_up_all(&event->waitq);
3483  spin_unlock_irqrestore(&rb->event_lock, flags);
3484 }
3485 
3486 static void ring_buffer_wakeup(struct perf_event *event)
3487 {
3488  struct ring_buffer *rb;
3489 
3490  rcu_read_lock();
3491  rb = rcu_dereference(event->rb);
3492  if (!rb)
3493  goto unlock;
3494 
3495  list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
3496  wake_up_all(&event->waitq);
3497 
3498 unlock:
3499  rcu_read_unlock();
3500 }
3501 
3502 static void rb_free_rcu(struct rcu_head *rcu_head)
3503 {
3504  struct ring_buffer *rb;
3505 
3506  rb = container_of(rcu_head, struct ring_buffer, rcu_head);
3507  rb_free(rb);
3508 }
3509 
3510 static struct ring_buffer *ring_buffer_get(struct perf_event *event)
3511 {
3512  struct ring_buffer *rb;
3513 
3514  rcu_read_lock();
3515  rb = rcu_dereference(event->rb);
3516  if (rb) {
3517  if (!atomic_inc_not_zero(&rb->refcount))
3518  rb = NULL;
3519  }
3520  rcu_read_unlock();
3521 
3522  return rb;
3523 }
3524 
3525 static void ring_buffer_put(struct ring_buffer *rb)
3526 {
3527  struct perf_event *event, *n;
3528  unsigned long flags;
3529 
3530  if (!atomic_dec_and_test(&rb->refcount))
3531  return;
3532 
3533  spin_lock_irqsave(&rb->event_lock, flags);
3534  list_for_each_entry_safe(event, n, &rb->event_list, rb_entry) {
3535  list_del_init(&event->rb_entry);
3536  wake_up_all(&event->waitq);
3537  }
3538  spin_unlock_irqrestore(&rb->event_lock, flags);
3539 
3540  call_rcu(&rb->rcu_head, rb_free_rcu);
3541 }
3542 
3543 static void perf_mmap_open(struct vm_area_struct *vma)
3544 {
3545  struct perf_event *event = vma->vm_file->private_data;
3546 
3547  atomic_inc(&event->mmap_count);
3548 }
3549 
3550 static void perf_mmap_close(struct vm_area_struct *vma)
3551 {
3552  struct perf_event *event = vma->vm_file->private_data;
3553 
3554  if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3555  unsigned long size = perf_data_size(event->rb);
3556  struct user_struct *user = event->mmap_user;
3557  struct ring_buffer *rb = event->rb;
3558 
3559  atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3560  vma->vm_mm->pinned_vm -= event->mmap_locked;
3561  rcu_assign_pointer(event->rb, NULL);
3562  ring_buffer_detach(event, rb);
3563  mutex_unlock(&event->mmap_mutex);
3564 
3565  ring_buffer_put(rb);
3566  free_uid(user);
3567  }
3568 }
3569 
3570 static const struct vm_operations_struct perf_mmap_vmops = {
3571  .open = perf_mmap_open,
3572  .close = perf_mmap_close,
3573  .fault = perf_mmap_fault,
3574  .page_mkwrite = perf_mmap_fault,
3575 };
3576 
3577 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3578 {
3579  struct perf_event *event = file->private_data;
3580  unsigned long user_locked, user_lock_limit;
3581  struct user_struct *user = current_user();
3582  unsigned long locked, lock_limit;
3583  struct ring_buffer *rb;
3584  unsigned long vma_size;
3585  unsigned long nr_pages;
3586  long user_extra, extra;
3587  int ret = 0, flags = 0;
3588 
3589  /*
3590  * Don't allow mmap() of inherited per-task counters. This would
3591  * create a performance issue due to all children writing to the
3592  * same rb.
3593  */
3594  if (event->cpu == -1 && event->attr.inherit)
3595  return -EINVAL;
3596 
3597  if (!(vma->vm_flags & VM_SHARED))
3598  return -EINVAL;
3599 
3600  vma_size = vma->vm_end - vma->vm_start;
3601  nr_pages = (vma_size / PAGE_SIZE) - 1;
3602 
3603  /*
3604  * If we have rb pages ensure they're a power-of-two number, so we
3605  * can do bitmasks instead of modulo.
3606  */
3607  if (nr_pages != 0 && !is_power_of_2(nr_pages))
3608  return -EINVAL;
3609 
3610  if (vma_size != PAGE_SIZE * (1 + nr_pages))
3611  return -EINVAL;
3612 
3613  if (vma->vm_pgoff != 0)
3614  return -EINVAL;
3615 
3616  WARN_ON_ONCE(event->ctx->parent_ctx);
3617  mutex_lock(&event->mmap_mutex);
3618  if (event->rb) {
3619  if (event->rb->nr_pages == nr_pages)
3620  atomic_inc(&event->rb->refcount);
3621  else
3622  ret = -EINVAL;
3623  goto unlock;
3624  }
3625 
3626  user_extra = nr_pages + 1;
3627  user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3628 
3629  /*
3630  * Increase the limit linearly with more CPUs:
3631  */
3632  user_lock_limit *= num_online_cpus();
3633 
3634  user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3635 
3636  extra = 0;
3637  if (user_locked > user_lock_limit)
3638  extra = user_locked - user_lock_limit;
3639 
3640  lock_limit = rlimit(RLIMIT_MEMLOCK);
3641  lock_limit >>= PAGE_SHIFT;
3642  locked = vma->vm_mm->pinned_vm + extra;
3643 
3644  if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3645  !capable(CAP_IPC_LOCK)) {
3646  ret = -EPERM;
3647  goto unlock;
3648  }
3649 
3650  WARN_ON(event->rb);
3651 
3652  if (vma->vm_flags & VM_WRITE)
3653  flags |= RING_BUFFER_WRITABLE;
3654 
3655  rb = rb_alloc(nr_pages,
3656  event->attr.watermark ? event->attr.wakeup_watermark : 0,
3657  event->cpu, flags);
3658 
3659  if (!rb) {
3660  ret = -ENOMEM;
3661  goto unlock;
3662  }
3663  rcu_assign_pointer(event->rb, rb);
3664 
3665  atomic_long_add(user_extra, &user->locked_vm);
3666  event->mmap_locked = extra;
3667  event->mmap_user = get_current_user();
3668  vma->vm_mm->pinned_vm += event->mmap_locked;
3669 
3671 
3672 unlock:
3673  if (!ret)
3674  atomic_inc(&event->mmap_count);
3675  mutex_unlock(&event->mmap_mutex);
3676 
3677  vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
3678  vma->vm_ops = &perf_mmap_vmops;
3679 
3680  return ret;
3681 }
3682 
3683 static int perf_fasync(int fd, struct file *filp, int on)
3684 {
3685  struct inode *inode = filp->f_path.dentry->d_inode;
3686  struct perf_event *event = filp->private_data;
3687  int retval;
3688 
3689  mutex_lock(&inode->i_mutex);
3690  retval = fasync_helper(fd, filp, on, &event->fasync);
3691  mutex_unlock(&inode->i_mutex);
3692 
3693  if (retval < 0)
3694  return retval;
3695 
3696  return 0;
3697 }
3698 
3699 static const struct file_operations perf_fops = {
3700  .llseek = no_llseek,
3701  .release = perf_release,
3702  .read = perf_read,
3703  .poll = perf_poll,
3704  .unlocked_ioctl = perf_ioctl,
3705  .compat_ioctl = perf_ioctl,
3706  .mmap = perf_mmap,
3707  .fasync = perf_fasync,
3708 };
3709 
3710 /*
3711  * Perf event wakeup
3712  *
3713  * If there's data, ensure we set the poll() state and publish everything
3714  * to user-space before waking everybody up.
3715  */
3716 
3717 void perf_event_wakeup(struct perf_event *event)
3718 {
3719  ring_buffer_wakeup(event);
3720 
3721  if (event->pending_kill) {
3722  kill_fasync(&event->fasync, SIGIO, event->pending_kill);
3723  event->pending_kill = 0;
3724  }
3725 }
3726 
3727 static void perf_pending_event(struct irq_work *entry)
3728 {
3729  struct perf_event *event = container_of(entry,
3730  struct perf_event, pending);
3731 
3732  if (event->pending_disable) {
3733  event->pending_disable = 0;
3734  __perf_event_disable(event);
3735  }
3736 
3737  if (event->pending_wakeup) {
3738  event->pending_wakeup = 0;
3739  perf_event_wakeup(event);
3740  }
3741 }
3742 
3743 /*
3744  * We assume there is only KVM supporting the callbacks.
3745  * Later on, we might change it to a list if there is
3746  * another virtualization implementation supporting the callbacks.
3747  */
3749 
3751 {
3752  perf_guest_cbs = cbs;
3753  return 0;
3754 }
3756 
3758 {
3759  perf_guest_cbs = NULL;
3760  return 0;
3761 }
3763 
3764 static void
3765 perf_output_sample_regs(struct perf_output_handle *handle,
3766  struct pt_regs *regs, u64 mask)
3767 {
3768  int bit;
3769 
3770  for_each_set_bit(bit, (const unsigned long *) &mask,
3771  sizeof(mask) * BITS_PER_BYTE) {
3772  u64 val;
3773 
3774  val = perf_reg_value(regs, bit);
3775  perf_output_put(handle, val);
3776  }
3777 }
3778 
3779 static void perf_sample_regs_user(struct perf_regs_user *regs_user,
3780  struct pt_regs *regs)
3781 {
3782  if (!user_mode(regs)) {
3783  if (current->mm)
3784  regs = task_pt_regs(current);
3785  else
3786  regs = NULL;
3787  }
3788 
3789  if (regs) {
3790  regs_user->regs = regs;
3791  regs_user->abi = perf_reg_abi(current);
3792  }
3793 }
3794 
3795 /*
3796  * Get remaining task size from user stack pointer.
3797  *
3798  * It'd be better to take stack vma map and limit this more
3799  * precisly, but there's no way to get it safely under interrupt,
3800  * so using TASK_SIZE as limit.
3801  */
3802 static u64 perf_ustack_task_size(struct pt_regs *regs)
3803 {
3804  unsigned long addr = perf_user_stack_pointer(regs);
3805 
3806  if (!addr || addr >= TASK_SIZE)
3807  return 0;
3808 
3809  return TASK_SIZE - addr;
3810 }
3811 
3812 static u16
3813 perf_sample_ustack_size(u16 stack_size, u16 header_size,
3814  struct pt_regs *regs)
3815 {
3816  u64 task_size;
3817 
3818  /* No regs, no stack pointer, no dump. */
3819  if (!regs)
3820  return 0;
3821 
3822  /*
3823  * Check if we fit in with the requested stack size into the:
3824  * - TASK_SIZE
3825  * If we don't, we limit the size to the TASK_SIZE.
3826  *
3827  * - remaining sample size
3828  * If we don't, we customize the stack size to
3829  * fit in to the remaining sample size.
3830  */
3831 
3832  task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
3833  stack_size = min(stack_size, (u16) task_size);
3834 
3835  /* Current header size plus static size and dynamic size. */
3836  header_size += 2 * sizeof(u64);
3837 
3838  /* Do we fit in with the current stack dump size? */
3839  if ((u16) (header_size + stack_size) < header_size) {
3840  /*
3841  * If we overflow the maximum size for the sample,
3842  * we customize the stack dump size to fit in.
3843  */
3844  stack_size = USHRT_MAX - header_size - sizeof(u64);
3845  stack_size = round_up(stack_size, sizeof(u64));
3846  }
3847 
3848  return stack_size;
3849 }
3850 
3851 static void
3852 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
3853  struct pt_regs *regs)
3854 {
3855  /* Case of a kernel thread, nothing to dump */
3856  if (!regs) {
3857  u64 size = 0;
3858  perf_output_put(handle, size);
3859  } else {
3860  unsigned long sp;
3861  unsigned int rem;
3862  u64 dyn_size;
3863 
3864  /*
3865  * We dump:
3866  * static size
3867  * - the size requested by user or the best one we can fit
3868  * in to the sample max size
3869  * data
3870  * - user stack dump data
3871  * dynamic size
3872  * - the actual dumped size
3873  */
3874 
3875  /* Static size. */
3876  perf_output_put(handle, dump_size);
3877 
3878  /* Data. */
3879  sp = perf_user_stack_pointer(regs);
3880  rem = __output_copy_user(handle, (void *) sp, dump_size);
3881  dyn_size = dump_size - rem;
3882 
3883  perf_output_skip(handle, rem);
3884 
3885  /* Dynamic size. */
3886  perf_output_put(handle, dyn_size);
3887  }
3888 }
3889 
3890 static void __perf_event_header__init_id(struct perf_event_header *header,
3891  struct perf_sample_data *data,
3892  struct perf_event *event)
3893 {
3894  u64 sample_type = event->attr.sample_type;
3895 
3896  data->type = sample_type;
3897  header->size += event->id_header_size;
3898 
3899  if (sample_type & PERF_SAMPLE_TID) {
3900  /* namespace issues */
3901  data->tid_entry.pid = perf_event_pid(event, current);
3902  data->tid_entry.tid = perf_event_tid(event, current);
3903  }
3904 
3905  if (sample_type & PERF_SAMPLE_TIME)
3906  data->time = perf_clock();
3907 
3908  if (sample_type & PERF_SAMPLE_ID)
3909  data->id = primary_event_id(event);
3910 
3911  if (sample_type & PERF_SAMPLE_STREAM_ID)
3912  data->stream_id = event->id;
3913 
3914  if (sample_type & PERF_SAMPLE_CPU) {
3915  data->cpu_entry.cpu = raw_smp_processor_id();
3916  data->cpu_entry.reserved = 0;
3917  }
3918 }
3919 
3921  struct perf_sample_data *data,
3922  struct perf_event *event)
3923 {
3924  if (event->attr.sample_id_all)
3925  __perf_event_header__init_id(header, data, event);
3926 }
3927 
3928 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
3929  struct perf_sample_data *data)
3930 {
3931  u64 sample_type = data->type;
3932 
3933  if (sample_type & PERF_SAMPLE_TID)
3934  perf_output_put(handle, data->tid_entry);
3935 
3936  if (sample_type & PERF_SAMPLE_TIME)
3937  perf_output_put(handle, data->time);
3938 
3939  if (sample_type & PERF_SAMPLE_ID)
3940  perf_output_put(handle, data->id);
3941 
3942  if (sample_type & PERF_SAMPLE_STREAM_ID)
3943  perf_output_put(handle, data->stream_id);
3944 
3945  if (sample_type & PERF_SAMPLE_CPU)
3946  perf_output_put(handle, data->cpu_entry);
3947 }
3948 
3949 void perf_event__output_id_sample(struct perf_event *event,
3950  struct perf_output_handle *handle,
3951  struct perf_sample_data *sample)
3952 {
3953  if (event->attr.sample_id_all)
3954  __perf_event__output_id_sample(handle, sample);
3955 }
3956 
3957 static void perf_output_read_one(struct perf_output_handle *handle,
3958  struct perf_event *event,
3959  u64 enabled, u64 running)
3960 {
3961  u64 read_format = event->attr.read_format;
3962  u64 values[4];
3963  int n = 0;
3964 
3965  values[n++] = perf_event_count(event);
3966  if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3967  values[n++] = enabled +
3968  atomic64_read(&event->child_total_time_enabled);
3969  }
3970  if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3971  values[n++] = running +
3972  atomic64_read(&event->child_total_time_running);
3973  }
3974  if (read_format & PERF_FORMAT_ID)
3975  values[n++] = primary_event_id(event);
3976 
3977  __output_copy(handle, values, n * sizeof(u64));
3978 }
3979 
3980 /*
3981  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3982  */
3983 static void perf_output_read_group(struct perf_output_handle *handle,
3984  struct perf_event *event,
3985  u64 enabled, u64 running)
3986 {
3987  struct perf_event *leader = event->group_leader, *sub;
3988  u64 read_format = event->attr.read_format;
3989  u64 values[5];
3990  int n = 0;
3991 
3992  values[n++] = 1 + leader->nr_siblings;
3993 
3994  if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3995  values[n++] = enabled;
3996 
3997  if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3998  values[n++] = running;
3999 
4000  if (leader != event)
4001  leader->pmu->read(leader);
4002 
4003  values[n++] = perf_event_count(leader);
4004  if (read_format & PERF_FORMAT_ID)
4005  values[n++] = primary_event_id(leader);
4006 
4007  __output_copy(handle, values, n * sizeof(u64));
4008 
4009  list_for_each_entry(sub, &leader->sibling_list, group_entry) {
4010  n = 0;
4011 
4012  if (sub != event)
4013  sub->pmu->read(sub);
4014 
4015  values[n++] = perf_event_count(sub);
4016  if (read_format & PERF_FORMAT_ID)
4017  values[n++] = primary_event_id(sub);
4018 
4019  __output_copy(handle, values, n * sizeof(u64));
4020  }
4021 }
4022 
4023 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
4024  PERF_FORMAT_TOTAL_TIME_RUNNING)
4025 
4026 static void perf_output_read(struct perf_output_handle *handle,
4027  struct perf_event *event)
4028 {
4029  u64 enabled = 0, running = 0, now;
4030  u64 read_format = event->attr.read_format;
4031 
4032  /*
4033  * compute total_time_enabled, total_time_running
4034  * based on snapshot values taken when the event
4035  * was last scheduled in.
4036  *
4037  * we cannot simply called update_context_time()
4038  * because of locking issue as we are called in
4039  * NMI context
4040  */
4041  if (read_format & PERF_FORMAT_TOTAL_TIMES)
4042  calc_timer_values(event, &now, &enabled, &running);
4043 
4044  if (event->attr.read_format & PERF_FORMAT_GROUP)
4045  perf_output_read_group(handle, event, enabled, running);
4046  else
4047  perf_output_read_one(handle, event, enabled, running);
4048 }
4049 
4051  struct perf_event_header *header,
4052  struct perf_sample_data *data,
4053  struct perf_event *event)
4054 {
4055  u64 sample_type = data->type;
4056 
4057  perf_output_put(handle, *header);
4058 
4059  if (sample_type & PERF_SAMPLE_IP)
4060  perf_output_put(handle, data->ip);
4061 
4062  if (sample_type & PERF_SAMPLE_TID)
4063  perf_output_put(handle, data->tid_entry);
4064 
4065  if (sample_type & PERF_SAMPLE_TIME)
4066  perf_output_put(handle, data->time);
4067 
4068  if (sample_type & PERF_SAMPLE_ADDR)
4069  perf_output_put(handle, data->addr);
4070 
4071  if (sample_type & PERF_SAMPLE_ID)
4072  perf_output_put(handle, data->id);
4073 
4074  if (sample_type & PERF_SAMPLE_STREAM_ID)
4075  perf_output_put(handle, data->stream_id);
4076 
4077  if (sample_type & PERF_SAMPLE_CPU)
4078  perf_output_put(handle, data->cpu_entry);
4079 
4080  if (sample_type & PERF_SAMPLE_PERIOD)
4081  perf_output_put(handle, data->period);
4082 
4083  if (sample_type & PERF_SAMPLE_READ)
4084  perf_output_read(handle, event);
4085 
4086  if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4087  if (data->callchain) {
4088  int size = 1;
4089 
4090  if (data->callchain)
4091  size += data->callchain->nr;
4092 
4093  size *= sizeof(u64);
4094 
4095  __output_copy(handle, data->callchain, size);
4096  } else {
4097  u64 nr = 0;
4098  perf_output_put(handle, nr);
4099  }
4100  }
4101 
4102  if (sample_type & PERF_SAMPLE_RAW) {
4103  if (data->raw) {
4104  perf_output_put(handle, data->raw->size);
4105  __output_copy(handle, data->raw->data,
4106  data->raw->size);
4107  } else {
4108  struct {
4109  u32 size;
4110  u32 data;
4111  } raw = {
4112  .size = sizeof(u32),
4113  .data = 0,
4114  };
4115  perf_output_put(handle, raw);
4116  }
4117  }
4118 
4119  if (!event->attr.watermark) {
4120  int wakeup_events = event->attr.wakeup_events;
4121 
4122  if (wakeup_events) {
4123  struct ring_buffer *rb = handle->rb;
4124  int events = local_inc_return(&rb->events);
4125 
4126  if (events >= wakeup_events) {
4127  local_sub(wakeup_events, &rb->events);
4128  local_inc(&rb->wakeup);
4129  }
4130  }
4131  }
4132 
4133  if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4134  if (data->br_stack) {
4135  size_t size;
4136 
4137  size = data->br_stack->nr
4138  * sizeof(struct perf_branch_entry);
4139 
4140  perf_output_put(handle, data->br_stack->nr);
4141  perf_output_copy(handle, data->br_stack->entries, size);
4142  } else {
4143  /*
4144  * we always store at least the value of nr
4145  */
4146  u64 nr = 0;
4147  perf_output_put(handle, nr);
4148  }
4149  }
4150 
4151  if (sample_type & PERF_SAMPLE_REGS_USER) {
4152  u64 abi = data->regs_user.abi;
4153 
4154  /*
4155  * If there are no regs to dump, notice it through
4156  * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
4157  */
4158  perf_output_put(handle, abi);
4159 
4160  if (abi) {
4161  u64 mask = event->attr.sample_regs_user;
4162  perf_output_sample_regs(handle,
4163  data->regs_user.regs,
4164  mask);
4165  }
4166  }
4167 
4168  if (sample_type & PERF_SAMPLE_STACK_USER)
4169  perf_output_sample_ustack(handle,
4170  data->stack_user_size,
4171  data->regs_user.regs);
4172 }
4173 
4175  struct perf_sample_data *data,
4176  struct perf_event *event,
4177  struct pt_regs *regs)
4178 {
4179  u64 sample_type = event->attr.sample_type;
4180 
4181  header->type = PERF_RECORD_SAMPLE;
4182  header->size = sizeof(*header) + event->header_size;
4183 
4184  header->misc = 0;
4185  header->misc |= perf_misc_flags(regs);
4186 
4187  __perf_event_header__init_id(header, data, event);
4188 
4189  if (sample_type & PERF_SAMPLE_IP)
4190  data->ip = perf_instruction_pointer(regs);
4191 
4192  if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4193  int size = 1;
4194 
4195  data->callchain = perf_callchain(event, regs);
4196 
4197  if (data->callchain)
4198  size += data->callchain->nr;
4199 
4200  header->size += size * sizeof(u64);
4201  }
4202 
4203  if (sample_type & PERF_SAMPLE_RAW) {
4204  int size = sizeof(u32);
4205 
4206  if (data->raw)
4207  size += data->raw->size;
4208  else
4209  size += sizeof(u32);
4210 
4211  WARN_ON_ONCE(size & (sizeof(u64)-1));
4212  header->size += size;
4213  }
4214 
4215  if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4216  int size = sizeof(u64); /* nr */
4217  if (data->br_stack) {
4218  size += data->br_stack->nr
4219  * sizeof(struct perf_branch_entry);
4220  }
4221  header->size += size;
4222  }
4223 
4224  if (sample_type & PERF_SAMPLE_REGS_USER) {
4225  /* regs dump ABI info */
4226  int size = sizeof(u64);
4227 
4228  perf_sample_regs_user(&data->regs_user, regs);
4229 
4230  if (data->regs_user.regs) {
4231  u64 mask = event->attr.sample_regs_user;
4232  size += hweight64(mask) * sizeof(u64);
4233  }
4234 
4235  header->size += size;
4236  }
4237 
4238  if (sample_type & PERF_SAMPLE_STACK_USER) {
4239  /*
4240  * Either we need PERF_SAMPLE_STACK_USER bit to be allways
4241  * processed as the last one or have additional check added
4242  * in case new sample type is added, because we could eat
4243  * up the rest of the sample size.
4244  */
4245  struct perf_regs_user *uregs = &data->regs_user;
4246  u16 stack_size = event->attr.sample_stack_user;
4247  u16 size = sizeof(u64);
4248 
4249  if (!uregs->abi)
4250  perf_sample_regs_user(uregs, regs);
4251 
4252  stack_size = perf_sample_ustack_size(stack_size, header->size,
4253  uregs->regs);
4254 
4255  /*
4256  * If there is something to dump, add space for the dump
4257  * itself and for the field that tells the dynamic size,
4258  * which is how many have been actually dumped.
4259  */
4260  if (stack_size)
4261  size += sizeof(u64) + stack_size;
4262 
4263  data->stack_user_size = stack_size;
4264  header->size += size;
4265  }
4266 }
4267 
4268 static void perf_event_output(struct perf_event *event,
4269  struct perf_sample_data *data,
4270  struct pt_regs *regs)
4271 {
4272  struct perf_output_handle handle;
4273  struct perf_event_header header;
4274 
4275  /* protect the callchain buffers */
4276  rcu_read_lock();
4277 
4278  perf_prepare_sample(&header, data, event, regs);
4279 
4280  if (perf_output_begin(&handle, event, header.size))
4281  goto exit;
4282 
4283  perf_output_sample(&handle, &header, data, event);
4284 
4285  perf_output_end(&handle);
4286 
4287 exit:
4288  rcu_read_unlock();
4289 }
4290 
4291 /*
4292  * read event_id
4293  */
4294 
4296  struct perf_event_header header;
4297 
4300 };
4301 
4302 static void
4303 perf_event_read_event(struct perf_event *event,
4304  struct task_struct *task)
4305 {
4306  struct perf_output_handle handle;
4307  struct perf_sample_data sample;
4308  struct perf_read_event read_event = {
4309  .header = {
4310  .type = PERF_RECORD_READ,
4311  .misc = 0,
4312  .size = sizeof(read_event) + event->read_size,
4313  },
4314  .pid = perf_event_pid(event, task),
4315  .tid = perf_event_tid(event, task),
4316  };
4317  int ret;
4318 
4319  perf_event_header__init_id(&read_event.header, &sample, event);
4320  ret = perf_output_begin(&handle, event, read_event.header.size);
4321  if (ret)
4322  return;
4323 
4324  perf_output_put(&handle, read_event);
4325  perf_output_read(&handle, event);
4326  perf_event__output_id_sample(event, &handle, &sample);
4327 
4328  perf_output_end(&handle);
4329 }
4330 
4331 /*
4332  * task tracking -- fork/exit
4333  *
4334  * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
4335  */
4336 
4340 
4341  struct {
4342  struct perf_event_header header;
4343 
4349  } event_id;
4350 };
4351 
4352 static void perf_event_task_output(struct perf_event *event,
4353  struct perf_task_event *task_event)
4354 {
4355  struct perf_output_handle handle;
4356  struct perf_sample_data sample;
4357  struct task_struct *task = task_event->task;
4358  int ret, size = task_event->event_id.header.size;
4359 
4360  perf_event_header__init_id(&task_event->event_id.header, &sample, event);
4361 
4362  ret = perf_output_begin(&handle, event,
4363  task_event->event_id.header.size);
4364  if (ret)
4365  goto out;
4366 
4367  task_event->event_id.pid = perf_event_pid(event, task);
4368  task_event->event_id.ppid = perf_event_pid(event, current);
4369 
4370  task_event->event_id.tid = perf_event_tid(event, task);
4371  task_event->event_id.ptid = perf_event_tid(event, current);
4372 
4373  perf_output_put(&handle, task_event->event_id);
4374 
4375  perf_event__output_id_sample(event, &handle, &sample);
4376 
4377  perf_output_end(&handle);
4378 out:
4379  task_event->event_id.header.size = size;
4380 }
4381 
4382 static int perf_event_task_match(struct perf_event *event)
4383 {
4384  if (event->state < PERF_EVENT_STATE_INACTIVE)
4385  return 0;
4386 
4387  if (!event_filter_match(event))
4388  return 0;
4389 
4390  if (event->attr.comm || event->attr.mmap ||
4391  event->attr.mmap_data || event->attr.task)
4392  return 1;
4393 
4394  return 0;
4395 }
4396 
4397 static void perf_event_task_ctx(struct perf_event_context *ctx,
4398  struct perf_task_event *task_event)
4399 {
4400  struct perf_event *event;
4401 
4402  list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4403  if (perf_event_task_match(event))
4404  perf_event_task_output(event, task_event);
4405  }
4406 }
4407 
4408 static void perf_event_task_event(struct perf_task_event *task_event)
4409 {
4410  struct perf_cpu_context *cpuctx;
4411  struct perf_event_context *ctx;
4412  struct pmu *pmu;
4413  int ctxn;
4414 
4415  rcu_read_lock();
4416  list_for_each_entry_rcu(pmu, &pmus, entry) {
4417  cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4418  if (cpuctx->unique_pmu != pmu)
4419  goto next;
4420  perf_event_task_ctx(&cpuctx->ctx, task_event);
4421 
4422  ctx = task_event->task_ctx;
4423  if (!ctx) {
4424  ctxn = pmu->task_ctx_nr;
4425  if (ctxn < 0)
4426  goto next;
4427  ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4428  }
4429  if (ctx)
4430  perf_event_task_ctx(ctx, task_event);
4431 next:
4433  }
4434  rcu_read_unlock();
4435 }
4436 
4437 static void perf_event_task(struct task_struct *task,
4438  struct perf_event_context *task_ctx,
4439  int new)
4440 {
4441  struct perf_task_event task_event;
4442 
4443  if (!atomic_read(&nr_comm_events) &&
4444  !atomic_read(&nr_mmap_events) &&
4445  !atomic_read(&nr_task_events))
4446  return;
4447 
4448  task_event = (struct perf_task_event){
4449  .task = task,
4450  .task_ctx = task_ctx,
4451  .event_id = {
4452  .header = {
4453  .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4454  .misc = 0,
4455  .size = sizeof(task_event.event_id),
4456  },
4457  /* .pid */
4458  /* .ppid */
4459  /* .tid */
4460  /* .ptid */
4461  .time = perf_clock(),
4462  },
4463  };
4464 
4465  perf_event_task_event(&task_event);
4466 }
4467 
4468 void perf_event_fork(struct task_struct *task)
4469 {
4470  perf_event_task(task, NULL, 1);
4471 }
4472 
4473 /*
4474  * comm tracking
4475  */
4476 
4479  char *comm;
4481 
4482  struct {
4483  struct perf_event_header header;
4484 
4487  } event_id;
4488 };
4489 
4490 static void perf_event_comm_output(struct perf_event *event,
4491  struct perf_comm_event *comm_event)
4492 {
4493  struct perf_output_handle handle;
4494  struct perf_sample_data sample;
4495  int size = comm_event->event_id.header.size;
4496  int ret;
4497 
4498  perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4499  ret = perf_output_begin(&handle, event,
4500  comm_event->event_id.header.size);
4501 
4502  if (ret)
4503  goto out;
4504 
4505  comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
4506  comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4507 
4508  perf_output_put(&handle, comm_event->event_id);
4509  __output_copy(&handle, comm_event->comm,
4510  comm_event->comm_size);
4511 
4512  perf_event__output_id_sample(event, &handle, &sample);
4513 
4514  perf_output_end(&handle);
4515 out:
4516  comm_event->event_id.header.size = size;
4517 }
4518 
4519 static int perf_event_comm_match(struct perf_event *event)
4520 {
4521  if (event->state < PERF_EVENT_STATE_INACTIVE)
4522  return 0;
4523 
4524  if (!event_filter_match(event))
4525  return 0;
4526 
4527  if (event->attr.comm)
4528  return 1;
4529 
4530  return 0;
4531 }
4532 
4533 static void perf_event_comm_ctx(struct perf_event_context *ctx,
4534  struct perf_comm_event *comm_event)
4535 {
4536  struct perf_event *event;
4537 
4538  list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4539  if (perf_event_comm_match(event))
4540  perf_event_comm_output(event, comm_event);
4541  }
4542 }
4543 
4544 static void perf_event_comm_event(struct perf_comm_event *comm_event)
4545 {
4546  struct perf_cpu_context *cpuctx;
4547  struct perf_event_context *ctx;
4548  char comm[TASK_COMM_LEN];
4549  unsigned int size;
4550  struct pmu *pmu;
4551  int ctxn;
4552 
4553  memset(comm, 0, sizeof(comm));
4554  strlcpy(comm, comm_event->task->comm, sizeof(comm));
4555  size = ALIGN(strlen(comm)+1, sizeof(u64));
4556 
4557  comm_event->comm = comm;
4558  comm_event->comm_size = size;
4559 
4560  comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4561  rcu_read_lock();
4562  list_for_each_entry_rcu(pmu, &pmus, entry) {
4563  cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4564  if (cpuctx->unique_pmu != pmu)
4565  goto next;
4566  perf_event_comm_ctx(&cpuctx->ctx, comm_event);
4567 
4568  ctxn = pmu->task_ctx_nr;
4569  if (ctxn < 0)
4570  goto next;
4571 
4572  ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4573  if (ctx)
4574  perf_event_comm_ctx(ctx, comm_event);
4575 next:
4577  }
4578  rcu_read_unlock();
4579 }
4580 
4581 void perf_event_comm(struct task_struct *task)
4582 {
4583  struct perf_comm_event comm_event;
4584  struct perf_event_context *ctx;
4585  int ctxn;
4586 
4587  for_each_task_context_nr(ctxn) {
4588  ctx = task->perf_event_ctxp[ctxn];
4589  if (!ctx)
4590  continue;
4591 
4592  perf_event_enable_on_exec(ctx);
4593  }
4594 
4595  if (!atomic_read(&nr_comm_events))
4596  return;
4597 
4598  comm_event = (struct perf_comm_event){
4599  .task = task,
4600  /* .comm */
4601  /* .comm_size */
4602  .event_id = {
4603  .header = {
4604  .type = PERF_RECORD_COMM,
4605  .misc = 0,
4606  /* .size */
4607  },
4608  /* .pid */
4609  /* .tid */
4610  },
4611  };
4612 
4613  perf_event_comm_event(&comm_event);
4614 }
4615 
4616 /*
4617  * mmap tracking
4618  */
4619 
4622 
4623  const char *file_name;
4625 
4626  struct {
4627  struct perf_event_header header;
4628 
4634  } event_id;
4635 };
4636 
4637 static void perf_event_mmap_output(struct perf_event *event,
4638  struct perf_mmap_event *mmap_event)
4639 {
4640  struct perf_output_handle handle;
4641  struct perf_sample_data sample;
4642  int size = mmap_event->event_id.header.size;
4643  int ret;
4644 
4645  perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4646  ret = perf_output_begin(&handle, event,
4647  mmap_event->event_id.header.size);
4648  if (ret)
4649  goto out;
4650 
4651  mmap_event->event_id.pid = perf_event_pid(event, current);
4652  mmap_event->event_id.tid = perf_event_tid(event, current);
4653 
4654  perf_output_put(&handle, mmap_event->event_id);
4655  __output_copy(&handle, mmap_event->file_name,
4656  mmap_event->file_size);
4657 
4658  perf_event__output_id_sample(event, &handle, &sample);
4659 
4660  perf_output_end(&handle);
4661 out:
4662  mmap_event->event_id.header.size = size;
4663 }
4664 
4665 static int perf_event_mmap_match(struct perf_event *event,
4666  struct perf_mmap_event *mmap_event,
4667  int executable)
4668 {
4669  if (event->state < PERF_EVENT_STATE_INACTIVE)
4670  return 0;
4671 
4672  if (!event_filter_match(event))
4673  return 0;
4674 
4675  if ((!executable && event->attr.mmap_data) ||
4676  (executable && event->attr.mmap))
4677  return 1;
4678 
4679  return 0;
4680 }
4681 
4682 static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4683  struct perf_mmap_event *mmap_event,
4684  int executable)
4685 {
4686  struct perf_event *event;
4687 
4688  list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4689  if (perf_event_mmap_match(event, mmap_event, executable))
4690  perf_event_mmap_output(event, mmap_event);
4691  }
4692 }
4693 
4694 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4695 {
4696  struct perf_cpu_context *cpuctx;
4697  struct perf_event_context *ctx;
4698  struct vm_area_struct *vma = mmap_event->vma;
4699  struct file *file = vma->vm_file;
4700  unsigned int size;
4701  char tmp[16];
4702  char *buf = NULL;
4703  const char *name;
4704  struct pmu *pmu;
4705  int ctxn;
4706 
4707  memset(tmp, 0, sizeof(tmp));
4708 
4709  if (file) {
4710  /*
4711  * d_path works from the end of the rb backwards, so we
4712  * need to add enough zero bytes after the string to handle
4713  * the 64bit alignment we do later.
4714  */
4715  buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4716  if (!buf) {
4717  name = strncpy(tmp, "//enomem", sizeof(tmp));
4718  goto got_name;
4719  }
4720  name = d_path(&file->f_path, buf, PATH_MAX);
4721  if (IS_ERR(name)) {
4722  name = strncpy(tmp, "//toolong", sizeof(tmp));
4723  goto got_name;
4724  }
4725  } else {
4726  if (arch_vma_name(mmap_event->vma)) {
4727  name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4728  sizeof(tmp));
4729  goto got_name;
4730  }
4731 
4732  if (!vma->vm_mm) {
4733  name = strncpy(tmp, "[vdso]", sizeof(tmp));
4734  goto got_name;
4735  } else if (vma->vm_start <= vma->vm_mm->start_brk &&
4736  vma->vm_end >= vma->vm_mm->brk) {
4737  name = strncpy(tmp, "[heap]", sizeof(tmp));
4738  goto got_name;
4739  } else if (vma->vm_start <= vma->vm_mm->start_stack &&
4740  vma->vm_end >= vma->vm_mm->start_stack) {
4741  name = strncpy(tmp, "[stack]", sizeof(tmp));
4742  goto got_name;
4743  }
4744 
4745  name = strncpy(tmp, "//anon", sizeof(tmp));
4746  goto got_name;
4747  }
4748 
4749 got_name:
4750  size = ALIGN(strlen(name)+1, sizeof(u64));
4751 
4752  mmap_event->file_name = name;
4753  mmap_event->file_size = size;
4754 
4755  mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4756 
4757  rcu_read_lock();
4758  list_for_each_entry_rcu(pmu, &pmus, entry) {
4759  cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4760  if (cpuctx->unique_pmu != pmu)
4761  goto next;
4762  perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
4763  vma->vm_flags & VM_EXEC);
4764 
4765  ctxn = pmu->task_ctx_nr;
4766  if (ctxn < 0)
4767  goto next;
4768 
4769  ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4770  if (ctx) {
4771  perf_event_mmap_ctx(ctx, mmap_event,
4772  vma->vm_flags & VM_EXEC);
4773  }
4774 next:
4776  }
4777  rcu_read_unlock();
4778 
4779  kfree(buf);
4780 }
4781 
4783 {
4784  struct perf_mmap_event mmap_event;
4785 
4786  if (!atomic_read(&nr_mmap_events))
4787  return;
4788 
4789  mmap_event = (struct perf_mmap_event){
4790  .vma = vma,
4791  /* .file_name */
4792  /* .file_size */
4793  .event_id = {
4794  .header = {
4795  .type = PERF_RECORD_MMAP,
4796  .misc = PERF_RECORD_MISC_USER,
4797  /* .size */
4798  },
4799  /* .pid */
4800  /* .tid */
4801  .start = vma->vm_start,
4802  .len = vma->vm_end - vma->vm_start,
4803  .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
4804  },
4805  };
4806 
4807  perf_event_mmap_event(&mmap_event);
4808 }
4809 
4810 /*
4811  * IRQ throttle logging
4812  */
4813 
4814 static void perf_log_throttle(struct perf_event *event, int enable)
4815 {
4816  struct perf_output_handle handle;
4817  struct perf_sample_data sample;
4818  int ret;
4819 
4820  struct {
4821  struct perf_event_header header;
4822  u64 time;
4823  u64 id;
4824  u64 stream_id;
4825  } throttle_event = {
4826  .header = {
4827  .type = PERF_RECORD_THROTTLE,
4828  .misc = 0,
4829  .size = sizeof(throttle_event),
4830  },
4831  .time = perf_clock(),
4832  .id = primary_event_id(event),
4833  .stream_id = event->id,
4834  };
4835 
4836  if (enable)
4838 
4840 
4841  ret = perf_output_begin(&handle, event,
4842  throttle_event.header.size);
4843  if (ret)
4844  return;
4845 
4846  perf_output_put(&handle, throttle_event);
4847  perf_event__output_id_sample(event, &handle, &sample);
4848  perf_output_end(&handle);
4849 }
4850 
4851 /*
4852  * Generic event overflow handling, sampling.
4853  */
4854 
4855 static int __perf_event_overflow(struct perf_event *event,
4856  int throttle, struct perf_sample_data *data,
4857  struct pt_regs *regs)
4858 {
4859  int events = atomic_read(&event->event_limit);
4860  struct hw_perf_event *hwc = &event->hw;
4861  u64 seq;
4862  int ret = 0;
4863 
4864  /*
4865  * Non-sampling counters might still use the PMI to fold short
4866  * hardware counters, ignore those.
4867  */
4868  if (unlikely(!is_sampling_event(event)))
4869  return 0;
4870 
4871  seq = __this_cpu_read(perf_throttled_seq);
4872  if (seq != hwc->interrupts_seq) {
4873  hwc->interrupts_seq = seq;
4874  hwc->interrupts = 1;
4875  } else {
4876  hwc->interrupts++;
4877  if (unlikely(throttle
4878  && hwc->interrupts >= max_samples_per_tick)) {
4879  __this_cpu_inc(perf_throttled_count);
4880  hwc->interrupts = MAX_INTERRUPTS;
4881  perf_log_throttle(event, 0);
4882  ret = 1;
4883  }
4884  }
4885 
4886  if (event->attr.freq) {
4887  u64 now = perf_clock();
4888  s64 delta = now - hwc->freq_time_stamp;
4889 
4890  hwc->freq_time_stamp = now;
4891 
4892  if (delta > 0 && delta < 2*TICK_NSEC)
4893  perf_adjust_period(event, delta, hwc->last_period, true);
4894  }
4895 
4896  /*
4897  * XXX event_limit might not quite work as expected on inherited
4898  * events
4899  */
4900 
4901  event->pending_kill = POLL_IN;
4902  if (events && atomic_dec_and_test(&event->event_limit)) {
4903  ret = 1;
4904  event->pending_kill = POLL_HUP;
4905  event->pending_disable = 1;
4906  irq_work_queue(&event->pending);
4907  }
4908 
4909  if (event->overflow_handler)
4910  event->overflow_handler(event, data, regs);
4911  else
4912  perf_event_output(event, data, regs);
4913 
4914  if (event->fasync && event->pending_kill) {
4915  event->pending_wakeup = 1;
4916  irq_work_queue(&event->pending);
4917  }
4918 
4919  return ret;
4920 }
4921 
4922 int perf_event_overflow(struct perf_event *event,
4923  struct perf_sample_data *data,
4924  struct pt_regs *regs)
4925 {
4926  return __perf_event_overflow(event, 1, data, regs);
4927 }
4928 
4929 /*
4930  * Generic software event infrastructure
4931  */
4932 
4935  struct mutex hlist_mutex;
4937 
4938  /* Recursion avoidance in each contexts */
4940 };
4941 
4943 
4944 /*
4945  * We directly increment event->count and keep a second value in
4946  * event->hw.period_left to count intervals. This period event
4947  * is kept in the range [-sample_period, 0] so that we can use the
4948  * sign as trigger.
4949  */
4950 
4951 static u64 perf_swevent_set_period(struct perf_event *event)
4952 {
4953  struct hw_perf_event *hwc = &event->hw;
4954  u64 period = hwc->last_period;
4955  u64 nr, offset;
4956  s64 old, val;
4957 
4958  hwc->last_period = hwc->sample_period;
4959 
4960 again:
4961  old = val = local64_read(&hwc->period_left);
4962  if (val < 0)
4963  return 0;
4964 
4965  nr = div64_u64(period + val, period);
4966  offset = nr * period;
4967  val -= offset;
4968  if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4969  goto again;
4970 
4971  return nr;
4972 }
4973 
4974 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4975  struct perf_sample_data *data,
4976  struct pt_regs *regs)
4977 {
4978  struct hw_perf_event *hwc = &event->hw;
4979  int throttle = 0;
4980 
4981  if (!overflow)
4982  overflow = perf_swevent_set_period(event);
4983 
4984  if (hwc->interrupts == MAX_INTERRUPTS)
4985  return;
4986 
4987  for (; overflow; overflow--) {
4988  if (__perf_event_overflow(event, throttle,
4989  data, regs)) {
4990  /*
4991  * We inhibit the overflow from happening when
4992  * hwc->interrupts == MAX_INTERRUPTS.
4993  */
4994  break;
4995  }
4996  throttle = 1;
4997  }
4998 }
4999 
5000 static void perf_swevent_event(struct perf_event *event, u64 nr,
5001  struct perf_sample_data *data,
5002  struct pt_regs *regs)
5003 {
5004  struct hw_perf_event *hwc = &event->hw;
5005 
5006  local64_add(nr, &event->count);
5007 
5008  if (!regs)
5009  return;
5010 
5011  if (!is_sampling_event(event))
5012  return;
5013 
5014  if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
5015  data->period = nr;
5016  return perf_swevent_overflow(event, 1, data, regs);
5017  } else
5018  data->period = event->hw.last_period;
5019 
5020  if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
5021  return perf_swevent_overflow(event, 1, data, regs);
5022 
5023  if (local64_add_negative(nr, &hwc->period_left))
5024  return;
5025 
5026  perf_swevent_overflow(event, 0, data, regs);
5027 }
5028 
5029 static int perf_exclude_event(struct perf_event *event,
5030  struct pt_regs *regs)
5031 {
5032  if (event->hw.state & PERF_HES_STOPPED)
5033  return 1;
5034 
5035  if (regs) {
5036  if (event->attr.exclude_user && user_mode(regs))
5037  return 1;
5038 
5039  if (event->attr.exclude_kernel && !user_mode(regs))
5040  return 1;
5041  }
5042 
5043  return 0;
5044 }
5045 
5046 static int perf_swevent_match(struct perf_event *event,
5047  enum perf_type_id type,
5048  u32 event_id,
5049  struct perf_sample_data *data,
5050  struct pt_regs *regs)
5051 {
5052  if (event->attr.type != type)
5053  return 0;
5054 
5055  if (event->attr.config != event_id)
5056  return 0;
5057 
5058  if (perf_exclude_event(event, regs))
5059  return 0;
5060 
5061  return 1;
5062 }
5063 
5064 static inline u64 swevent_hash(u64 type, u32 event_id)
5065 {
5066  u64 val = event_id | (type << 32);
5067 
5068  return hash_64(val, SWEVENT_HLIST_BITS);
5069 }
5070 
5071 static inline struct hlist_head *
5072 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
5073 {
5074  u64 hash = swevent_hash(type, event_id);
5075 
5076  return &hlist->heads[hash];
5077 }
5078 
5079 /* For the read side: events when they trigger */
5080 static inline struct hlist_head *
5081 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
5082 {
5083  struct swevent_hlist *hlist;
5084 
5085  hlist = rcu_dereference(swhash->swevent_hlist);
5086  if (!hlist)
5087  return NULL;
5088 
5089  return __find_swevent_head(hlist, type, event_id);
5090 }
5091 
5092 /* For the event head insertion and removal in the hlist */
5093 static inline struct hlist_head *
5094 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
5095 {
5096  struct swevent_hlist *hlist;
5097  u32 event_id = event->attr.config;
5098  u64 type = event->attr.type;
5099 
5100  /*
5101  * Event scheduling is always serialized against hlist allocation
5102  * and release. Which makes the protected version suitable here.
5103  * The context lock guarantees that.
5104  */
5105  hlist = rcu_dereference_protected(swhash->swevent_hlist,
5106  lockdep_is_held(&event->ctx->lock));
5107  if (!hlist)
5108  return NULL;
5109 
5110  return __find_swevent_head(hlist, type, event_id);
5111 }
5112 
5113 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
5114  u64 nr,
5115  struct perf_sample_data *data,
5116  struct pt_regs *regs)
5117 {
5118  struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5119  struct perf_event *event;
5120  struct hlist_node *node;
5121  struct hlist_head *head;
5122 
5123  rcu_read_lock();
5124  head = find_swevent_head_rcu(swhash, type, event_id);
5125  if (!head)
5126  goto end;
5127 
5128  hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5129  if (perf_swevent_match(event, type, event_id, data, regs))
5130  perf_swevent_event(event, nr, data, regs);
5131  }
5132 end:
5133  rcu_read_unlock();
5134 }
5135 
5137 {
5138  struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5139 
5140  return get_recursion_context(swhash->recursion);
5141 }
5143 
5145 {
5146  struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5147 
5148  put_recursion_context(swhash->recursion, rctx);
5149 }
5150 
5151 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
5152 {
5153  struct perf_sample_data data;
5154  int rctx;
5155 
5158  if (rctx < 0)
5159  return;
5160 
5161  perf_sample_data_init(&data, addr, 0);
5162 
5163  do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
5164 
5167 }
5168 
5169 static void perf_swevent_read(struct perf_event *event)
5170 {
5171 }
5172 
5173 static int perf_swevent_add(struct perf_event *event, int flags)
5174 {
5175  struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5176  struct hw_perf_event *hwc = &event->hw;
5177  struct hlist_head *head;
5178 
5179  if (is_sampling_event(event)) {
5180  hwc->last_period = hwc->sample_period;
5181  perf_swevent_set_period(event);
5182  }
5183 
5184  hwc->state = !(flags & PERF_EF_START);
5185 
5186  head = find_swevent_head(swhash, event);
5187  if (WARN_ON_ONCE(!head))
5188  return -EINVAL;
5189 
5190  hlist_add_head_rcu(&event->hlist_entry, head);
5191 
5192  return 0;
5193 }
5194 
5195 static void perf_swevent_del(struct perf_event *event, int flags)
5196 {
5197  hlist_del_rcu(&event->hlist_entry);
5198 }
5199 
5200 static void perf_swevent_start(struct perf_event *event, int flags)
5201 {
5202  event->hw.state = 0;
5203 }
5204 
5205 static void perf_swevent_stop(struct perf_event *event, int flags)
5206 {
5207  event->hw.state = PERF_HES_STOPPED;
5208 }
5209 
5210 /* Deref the hlist from the update side */
5211 static inline struct swevent_hlist *
5212 swevent_hlist_deref(struct swevent_htable *swhash)
5213 {
5215  lockdep_is_held(&swhash->hlist_mutex));
5216 }
5217 
5218 static void swevent_hlist_release(struct swevent_htable *swhash)
5219 {
5220  struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5221 
5222  if (!hlist)
5223  return;
5224 
5226  kfree_rcu(hlist, rcu_head);
5227 }
5228 
5229 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
5230 {
5231  struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5232 
5233  mutex_lock(&swhash->hlist_mutex);
5234 
5235  if (!--swhash->hlist_refcount)
5236  swevent_hlist_release(swhash);
5237 
5238  mutex_unlock(&swhash->hlist_mutex);
5239 }
5240 
5241 static void swevent_hlist_put(struct perf_event *event)
5242 {
5243  int cpu;
5244 
5245  if (event->cpu != -1) {
5246  swevent_hlist_put_cpu(event, event->cpu);
5247  return;
5248  }
5249 
5251  swevent_hlist_put_cpu(event, cpu);
5252 }
5253 
5254 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
5255 {
5256  struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5257  int err = 0;
5258 
5259  mutex_lock(&swhash->hlist_mutex);
5260 
5261  if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
5262  struct swevent_hlist *hlist;
5263 
5264  hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5265  if (!hlist) {
5266  err = -ENOMEM;
5267  goto exit;
5268  }
5269  rcu_assign_pointer(swhash->swevent_hlist, hlist);
5270  }
5271  swhash->hlist_refcount++;
5272 exit:
5273  mutex_unlock(&swhash->hlist_mutex);
5274 
5275  return err;
5276 }
5277 
5278 static int swevent_hlist_get(struct perf_event *event)
5279 {
5280  int err;
5281  int cpu, failed_cpu;
5282 
5283  if (event->cpu != -1)
5284  return swevent_hlist_get_cpu(event, event->cpu);
5285 
5286  get_online_cpus();
5287  for_each_possible_cpu(cpu) {
5288  err = swevent_hlist_get_cpu(event, cpu);
5289  if (err) {
5290  failed_cpu = cpu;
5291  goto fail;
5292  }
5293  }
5294  put_online_cpus();
5295 
5296  return 0;
5297 fail:
5298  for_each_possible_cpu(cpu) {
5299  if (cpu == failed_cpu)
5300  break;
5301  swevent_hlist_put_cpu(event, cpu);
5302  }
5303 
5304  put_online_cpus();
5305  return err;
5306 }
5307 
5309 
5310 static void sw_perf_event_destroy(struct perf_event *event)
5311 {
5312  u64 event_id = event->attr.config;
5313 
5314  WARN_ON(event->parent);
5315 
5316  static_key_slow_dec(&perf_swevent_enabled[event_id]);
5317  swevent_hlist_put(event);
5318 }
5319 
5320 static int perf_swevent_init(struct perf_event *event)
5321 {
5322  int event_id = event->attr.config;
5323 
5324  if (event->attr.type != PERF_TYPE_SOFTWARE)
5325  return -ENOENT;
5326 
5327  /*
5328  * no branch sampling for software events
5329  */
5330  if (has_branch_stack(event))
5331  return -EOPNOTSUPP;
5332 
5333  switch (event_id) {
5336  return -ENOENT;
5337 
5338  default:
5339  break;
5340  }
5341 
5342  if (event_id >= PERF_COUNT_SW_MAX)
5343  return -ENOENT;
5344 
5345  if (!event->parent) {
5346  int err;
5347 
5348  err = swevent_hlist_get(event);
5349  if (err)
5350  return err;
5351 
5352  static_key_slow_inc(&perf_swevent_enabled[event_id]);
5353  event->destroy = sw_perf_event_destroy;
5354  }
5355 
5356  return 0;
5357 }
5358 
5359 static int perf_swevent_event_idx(struct perf_event *event)
5360 {
5361  return 0;
5362 }
5363 
5364 static struct pmu perf_swevent = {
5365  .task_ctx_nr = perf_sw_context,
5366 
5367  .event_init = perf_swevent_init,
5368  .add = perf_swevent_add,
5369  .del = perf_swevent_del,
5370  .start = perf_swevent_start,
5371  .stop = perf_swevent_stop,
5372  .read = perf_swevent_read,
5373 
5374  .event_idx = perf_swevent_event_idx,
5375 };
5376 
5377 #ifdef CONFIG_EVENT_TRACING
5378 
5379 static int perf_tp_filter_match(struct perf_event *event,
5380  struct perf_sample_data *data)
5381 {
5382  void *record = data->raw->data;
5383 
5384  if (likely(!event->filter) || filter_match_preds(event->filter, record))
5385  return 1;
5386  return 0;
5387 }
5388 
5389 static int perf_tp_event_match(struct perf_event *event,
5390  struct perf_sample_data *data,
5391  struct pt_regs *regs)
5392 {
5393  if (event->hw.state & PERF_HES_STOPPED)
5394  return 0;
5395  /*
5396  * All tracepoints are from kernel-space.
5397  */
5398  if (event->attr.exclude_kernel)
5399  return 0;
5400 
5401  if (!perf_tp_filter_match(event, data))
5402  return 0;
5403 
5404  return 1;
5405 }
5406 
5407 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5408  struct pt_regs *regs, struct hlist_head *head, int rctx,
5409  struct task_struct *task)
5410 {
5411  struct perf_sample_data data;
5412  struct perf_event *event;
5413  struct hlist_node *node;
5414 
5415  struct perf_raw_record raw = {
5416  .size = entry_size,
5417  .data = record,
5418  };
5419 
5420  perf_sample_data_init(&data, addr, 0);
5421  data.raw = &raw;
5422 
5423  hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
5424  if (perf_tp_event_match(event, &data, regs))
5425  perf_swevent_event(event, count, &data, regs);
5426  }
5427 
5428  /*
5429  * If we got specified a target task, also iterate its context and
5430  * deliver this event there too.
5431  */
5432  if (task && task != current) {
5433  struct perf_event_context *ctx;
5434  struct trace_entry *entry = record;
5435 
5436  rcu_read_lock();
5437  ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
5438  if (!ctx)
5439  goto unlock;
5440 
5441  list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
5442  if (event->attr.type != PERF_TYPE_TRACEPOINT)
5443  continue;
5444  if (event->attr.config != entry->type)
5445  continue;
5446  if (perf_tp_event_match(event, &data, regs))
5447  perf_swevent_event(event, count, &data, regs);
5448  }
5449 unlock:
5450  rcu_read_unlock();
5451  }
5452 
5454 }
5455 EXPORT_SYMBOL_GPL(perf_tp_event);
5456 
5457 static void tp_perf_event_destroy(struct perf_event *event)
5458 {
5459  perf_trace_destroy(event);
5460 }
5461 
5462 static int perf_tp_event_init(struct perf_event *event)
5463 {
5464  int err;
5465 
5466  if (event->attr.type != PERF_TYPE_TRACEPOINT)
5467  return -ENOENT;
5468 
5469  /*
5470  * no branch sampling for tracepoint events
5471  */
5472  if (has_branch_stack(event))
5473  return -EOPNOTSUPP;
5474 
5475  err = perf_trace_init(event);
5476  if (err)
5477  return err;
5478 
5479  event->destroy = tp_perf_event_destroy;
5480 
5481  return 0;
5482 }
5483 
5484 static struct pmu perf_tracepoint = {
5486 
5487  .event_init = perf_tp_event_init,
5488  .add = perf_trace_add,
5489  .del = perf_trace_del,
5490  .start = perf_swevent_start,
5491  .stop = perf_swevent_stop,
5492  .read = perf_swevent_read,
5493 
5494  .event_idx = perf_swevent_event_idx,
5495 };
5496 
5497 static inline void perf_tp_register(void)
5498 {
5499  perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
5500 }
5501 
5502 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5503 {
5504  char *filter_str;
5505  int ret;
5506 
5507  if (event->attr.type != PERF_TYPE_TRACEPOINT)
5508  return -EINVAL;
5509 
5510  filter_str = strndup_user(arg, PAGE_SIZE);
5511  if (IS_ERR(filter_str))
5512  return PTR_ERR(filter_str);
5513 
5514  ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
5515 
5516  kfree(filter_str);
5517  return ret;
5518 }
5519 
5520 static void perf_event_free_filter(struct perf_event *event)
5521 {
5522  ftrace_profile_free_filter(event);
5523 }
5524 
5525 #else
5526 
5527 static inline void perf_tp_register(void)
5528 {
5529 }
5530 
5531 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
5532 {
5533  return -ENOENT;
5534 }
5535 
5536 static void perf_event_free_filter(struct perf_event *event)
5537 {
5538 }
5539 
5540 #endif /* CONFIG_EVENT_TRACING */
5541 
5542 #ifdef CONFIG_HAVE_HW_BREAKPOINT
5543 void perf_bp_event(struct perf_event *bp, void *data)
5544 {
5545  struct perf_sample_data sample;
5546  struct pt_regs *regs = data;
5547 
5548  perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
5549 
5550  if (!bp->hw.state && !perf_exclude_event(bp, regs))
5551  perf_swevent_event(bp, 1, &sample, regs);
5552 }
5553 #endif
5554 
5555 /*
5556  * hrtimer based swevent callback
5557  */
5558 
5559 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5560 {
5561  enum hrtimer_restart ret = HRTIMER_RESTART;
5562  struct perf_sample_data data;
5563  struct pt_regs *regs;
5564  struct perf_event *event;
5565  u64 period;
5566 
5567  event = container_of(hrtimer, struct perf_event, hw.hrtimer);
5568 
5569  if (event->state != PERF_EVENT_STATE_ACTIVE)
5570  return HRTIMER_NORESTART;
5571 
5572  event->pmu->read(event);
5573 
5574  perf_sample_data_init(&data, 0, event->hw.last_period);
5575  regs = get_irq_regs();
5576 
5577  if (regs && !perf_exclude_event(event, regs)) {
5578  if (!(event->attr.exclude_idle && is_idle_task(current)))
5579  if (__perf_event_overflow(event, 1, &data, regs))
5580  ret = HRTIMER_NORESTART;
5581  }
5582 
5583  period = max_t(u64, 10000, event->hw.sample_period);
5584  hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5585 
5586  return ret;
5587 }
5588 
5589 static void perf_swevent_start_hrtimer(struct perf_event *event)
5590 {
5591  struct hw_perf_event *hwc = &event->hw;
5592  s64 period;
5593 
5594  if (!is_sampling_event(event))
5595  return;
5596 
5597  period = local64_read(&hwc->period_left);
5598  if (period) {
5599  if (period < 0)
5600  period = 10000;
5601 
5602  local64_set(&hwc->period_left, 0);
5603  } else {
5604  period = max_t(u64, 10000, hwc->sample_period);
5605  }
5606  __hrtimer_start_range_ns(&hwc->hrtimer,
5607  ns_to_ktime(period), 0,
5609 }
5610 
5611 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5612 {
5613  struct hw_perf_event *hwc = &event->hw;
5614 
5615  if (is_sampling_event(event)) {
5616  ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
5617  local64_set(&hwc->period_left, ktime_to_ns(remaining));
5618 
5619  hrtimer_cancel(&hwc->hrtimer);
5620  }
5621 }
5622 
5623 static void perf_swevent_init_hrtimer(struct perf_event *event)
5624 {
5625  struct hw_perf_event *hwc = &event->hw;
5626 
5627  if (!is_sampling_event(event))
5628  return;
5629 
5631  hwc->hrtimer.function = perf_swevent_hrtimer;
5632 
5633  /*
5634  * Since hrtimers have a fixed rate, we can do a static freq->period
5635  * mapping and avoid the whole period adjust feedback stuff.
5636  */
5637  if (event->attr.freq) {
5638  long freq = event->attr.sample_freq;
5639 
5640  event->attr.sample_period = NSEC_PER_SEC / freq;
5641  hwc->sample_period = event->attr.sample_period;
5642  local64_set(&hwc->period_left, hwc->sample_period);
5643  event->attr.freq = 0;
5644  }
5645 }
5646 
5647 /*
5648  * Software event: cpu wall time clock
5649  */
5650 
5651 static void cpu_clock_event_update(struct perf_event *event)
5652 {
5653  s64 prev;
5654  u64 now;
5655 
5656  now = local_clock();
5657  prev = local64_xchg(&event->hw.prev_count, now);
5658  local64_add(now - prev, &event->count);
5659 }
5660 
5661 static void cpu_clock_event_start(struct perf_event *event, int flags)
5662 {
5663  local64_set(&event->hw.prev_count, local_clock());
5664  perf_swevent_start_hrtimer(event);
5665 }
5666 
5667 static void cpu_clock_event_stop(struct perf_event *event, int flags)
5668 {
5669  perf_swevent_cancel_hrtimer(event);
5670  cpu_clock_event_update(event);
5671 }
5672 
5673 static int cpu_clock_event_add(struct perf_event *event, int flags)
5674 {
5675  if (flags & PERF_EF_START)
5676  cpu_clock_event_start(event, flags);
5677 
5678  return 0;
5679 }
5680 
5681 static void cpu_clock_event_del(struct perf_event *event, int flags)
5682 {
5683  cpu_clock_event_stop(event, flags);
5684 }
5685 
5686 static void cpu_clock_event_read(struct perf_event *event)
5687 {
5688  cpu_clock_event_update(event);
5689 }
5690 
5691 static int cpu_clock_event_init(struct perf_event *event)
5692 {
5693  if (event->attr.type != PERF_TYPE_SOFTWARE)
5694  return -ENOENT;
5695 
5696  if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
5697  return -ENOENT;
5698 
5699  /*
5700  * no branch sampling for software events
5701  */
5702  if (has_branch_stack(event))
5703  return -EOPNOTSUPP;
5704 
5705  perf_swevent_init_hrtimer(event);
5706 
5707  return 0;
5708 }
5709 
5710 static struct pmu perf_cpu_clock = {
5711  .task_ctx_nr = perf_sw_context,
5712 
5713  .event_init = cpu_clock_event_init,
5714  .add = cpu_clock_event_add,
5715  .del = cpu_clock_event_del,
5716  .start = cpu_clock_event_start,
5717  .stop = cpu_clock_event_stop,
5718  .read = cpu_clock_event_read,
5719 
5720  .event_idx = perf_swevent_event_idx,
5721 };
5722 
5723 /*
5724  * Software event: task time clock
5725  */
5726 
5727 static void task_clock_event_update(struct perf_event *event, u64 now)
5728 {
5729  u64 prev;
5730  s64 delta;
5731 
5732  prev = local64_xchg(&event->hw.prev_count, now);
5733  delta = now - prev;
5734  local64_add(delta, &event->count);
5735 }
5736 
5737 static void task_clock_event_start(struct perf_event *event, int flags)
5738 {
5739  local64_set(&event->hw.prev_count, event->ctx->time);
5740  perf_swevent_start_hrtimer(event);
5741 }
5742 
5743 static void task_clock_event_stop(struct perf_event *event, int flags)
5744 {
5745  perf_swevent_cancel_hrtimer(event);
5746  task_clock_event_update(event, event->ctx->time);
5747 }
5748 
5749 static int task_clock_event_add(struct perf_event *event, int flags)
5750 {
5751  if (flags & PERF_EF_START)
5752  task_clock_event_start(event, flags);
5753 
5754  return 0;
5755 }
5756 
5757 static void task_clock_event_del(struct perf_event *event, int flags)
5758 {
5759  task_clock_event_stop(event, PERF_EF_UPDATE);
5760 }
5761 
5762 static void task_clock_event_read(struct perf_event *event)
5763 {
5764  u64 now = perf_clock();
5765  u64 delta = now - event->ctx->timestamp;
5766  u64 time = event->ctx->time + delta;
5767 
5768  task_clock_event_update(event, time);
5769 }
5770 
5771 static int task_clock_event_init(struct perf_event *event)
5772 {
5773  if (event->attr.type != PERF_TYPE_SOFTWARE)
5774  return -ENOENT;
5775 
5776  if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
5777  return -ENOENT;
5778 
5779  /*
5780  * no branch sampling for software events
5781  */
5782  if (has_branch_stack(event))
5783  return -EOPNOTSUPP;
5784 
5785  perf_swevent_init_hrtimer(event);
5786 
5787  return 0;
5788 }
5789 
5790 static struct pmu perf_task_clock = {
5791  .task_ctx_nr = perf_sw_context,
5792 
5793  .event_init = task_clock_event_init,
5794  .add = task_clock_event_add,
5795  .del = task_clock_event_del,
5796  .start = task_clock_event_start,
5797  .stop = task_clock_event_stop,
5798  .read = task_clock_event_read,
5799 
5800  .event_idx = perf_swevent_event_idx,
5801 };
5802 
5803 static void perf_pmu_nop_void(struct pmu *pmu)
5804 {
5805 }
5806 
5807 static int perf_pmu_nop_int(struct pmu *pmu)
5808 {
5809  return 0;
5810 }
5811 
5812 static void perf_pmu_start_txn(struct pmu *pmu)
5813 {
5814  perf_pmu_disable(pmu);
5815 }
5816 
5817 static int perf_pmu_commit_txn(struct pmu *pmu)
5818 {
5819  perf_pmu_enable(pmu);
5820  return 0;
5821 }
5822 
5823 static void perf_pmu_cancel_txn(struct pmu *pmu)
5824 {
5825  perf_pmu_enable(pmu);
5826 }
5827 
5828 static int perf_event_idx_default(struct perf_event *event)
5829 {
5830  return event->hw.idx + 1;
5831 }
5832 
5833 /*
5834  * Ensures all contexts with the same task_ctx_nr have the same
5835  * pmu_cpu_context too.
5836  */
5837 static void *find_pmu_context(int ctxn)
5838 {
5839  struct pmu *pmu;
5840 
5841  if (ctxn < 0)
5842  return NULL;
5843 
5844  list_for_each_entry(pmu, &pmus, entry) {
5845  if (pmu->task_ctx_nr == ctxn)
5846  return pmu->pmu_cpu_context;
5847  }
5848 
5849  return NULL;
5850 }
5851 
5852 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5853 {
5854  int cpu;
5855 
5856  for_each_possible_cpu(cpu) {
5857  struct perf_cpu_context *cpuctx;
5858 
5859  cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5860 
5861  if (cpuctx->unique_pmu == old_pmu)
5862  cpuctx->unique_pmu = pmu;
5863  }
5864 }
5865 
5866 static void free_pmu_context(struct pmu *pmu)
5867 {
5868  struct pmu *i;
5869 
5870  mutex_lock(&pmus_lock);
5871  /*
5872  * Like a real lame refcount.
5873  */
5874  list_for_each_entry(i, &pmus, entry) {
5875  if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
5876  update_pmu_context(i, pmu);
5877  goto out;
5878  }
5879  }
5880 
5882 out:
5883  mutex_unlock(&pmus_lock);
5884 }
5885 static struct idr pmu_idr;
5886 
5887 static ssize_t
5888 type_show(struct device *dev, struct device_attribute *attr, char *page)
5889 {
5890  struct pmu *pmu = dev_get_drvdata(dev);
5891 
5892  return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
5893 }
5894 
5895 static struct device_attribute pmu_dev_attrs[] = {
5896  __ATTR_RO(type),
5897  __ATTR_NULL,
5898 };
5899 
5900 static int pmu_bus_running;
5901 static struct bus_type pmu_bus = {
5902  .name = "event_source",
5903  .dev_attrs = pmu_dev_attrs,
5904 };
5905 
5906 static void pmu_dev_release(struct device *dev)
5907 {
5908  kfree(dev);
5909 }
5910 
5911 static int pmu_dev_alloc(struct pmu *pmu)
5912 {
5913  int ret = -ENOMEM;
5914 
5915  pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
5916  if (!pmu->dev)
5917  goto out;
5918 
5919  pmu->dev->groups = pmu->attr_groups;
5920  device_initialize(pmu->dev);
5921  ret = dev_set_name(pmu->dev, "%s", pmu->name);
5922  if (ret)
5923  goto free_dev;
5924 
5925  dev_set_drvdata(pmu->dev, pmu);
5926  pmu->dev->bus = &pmu_bus;
5927  pmu->dev->release = pmu_dev_release;
5928  ret = device_add(pmu->dev);
5929  if (ret)
5930  goto free_dev;
5931 
5932 out:
5933  return ret;
5934 
5935 free_dev:
5936  put_device(pmu->dev);
5937  goto out;
5938 }
5939 
5940 static struct lock_class_key cpuctx_mutex;
5941 static struct lock_class_key cpuctx_lock;
5942 
5943 int perf_pmu_register(struct pmu *pmu, char *name, int type)
5944 {
5945  int cpu, ret;
5946 
5947  mutex_lock(&pmus_lock);
5948  ret = -ENOMEM;
5949  pmu->pmu_disable_count = alloc_percpu(int);
5950  if (!pmu->pmu_disable_count)
5951  goto unlock;
5952 
5953  pmu->type = -1;
5954  if (!name)
5955  goto skip_type;
5956  pmu->name = name;
5957 
5958  if (type < 0) {
5959  int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
5960  if (!err)
5961  goto free_pdc;
5962 
5963  err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
5964  if (err) {
5965  ret = err;
5966  goto free_pdc;
5967  }
5968  }
5969  pmu->type = type;
5970 
5971  if (pmu_bus_running) {
5972  ret = pmu_dev_alloc(pmu);
5973  if (ret)
5974  goto free_idr;
5975  }
5976 
5977 skip_type:
5978  pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
5979  if (pmu->pmu_cpu_context)
5980  goto got_cpu_context;
5981 
5983  if (!pmu->pmu_cpu_context)
5984  goto free_dev;
5985 
5986  for_each_possible_cpu(cpu) {
5987  struct perf_cpu_context *cpuctx;
5988 
5989  cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5990  __perf_event_init_context(&cpuctx->ctx);
5991  lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
5992  lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
5993  cpuctx->ctx.type = cpu_context;
5994  cpuctx->ctx.pmu = pmu;
5995  cpuctx->jiffies_interval = 1;
5996  INIT_LIST_HEAD(&cpuctx->rotation_list);
5997  cpuctx->unique_pmu = pmu;
5998  }
5999 
6000 got_cpu_context:
6001  if (!pmu->start_txn) {
6002  if (pmu->pmu_enable) {
6003  /*
6004  * If we have pmu_enable/pmu_disable calls, install
6005  * transaction stubs that use that to try and batch
6006  * hardware accesses.
6007  */
6008  pmu->start_txn = perf_pmu_start_txn;
6009  pmu->commit_txn = perf_pmu_commit_txn;
6010  pmu->cancel_txn = perf_pmu_cancel_txn;
6011  } else {
6012  pmu->start_txn = perf_pmu_nop_void;
6013  pmu->commit_txn = perf_pmu_nop_int;
6014  pmu->cancel_txn = perf_pmu_nop_void;
6015  }
6016  }
6017 
6018  if (!pmu->pmu_enable) {
6019  pmu->pmu_enable = perf_pmu_nop_void;
6020  pmu->pmu_disable = perf_pmu_nop_void;
6021  }
6022 
6023  if (!pmu->event_idx)
6024  pmu->event_idx = perf_event_idx_default;
6025 
6026  list_add_rcu(&pmu->entry, &pmus);
6027  ret = 0;
6028 unlock:
6029  mutex_unlock(&pmus_lock);
6030 
6031  return ret;
6032 
6033 free_dev:
6034  device_del(pmu->dev);
6035  put_device(pmu->dev);
6036 
6037 free_idr:
6038  if (pmu->type >= PERF_TYPE_MAX)
6039  idr_remove(&pmu_idr, pmu->type);
6040 
6041 free_pdc:
6043  goto unlock;
6044 }
6045 
6046 void perf_pmu_unregister(struct pmu *pmu)
6047 {
6048  mutex_lock(&pmus_lock);
6049  list_del_rcu(&pmu->entry);
6050  mutex_unlock(&pmus_lock);
6051 
6052  /*
6053  * We dereference the pmu list under both SRCU and regular RCU, so
6054  * synchronize against both of those.
6055  */
6056  synchronize_srcu(&pmus_srcu);
6057  synchronize_rcu();
6058 
6060  if (pmu->type >= PERF_TYPE_MAX)
6061  idr_remove(&pmu_idr, pmu->type);
6062  device_del(pmu->dev);
6063  put_device(pmu->dev);
6064  free_pmu_context(pmu);
6065 }
6066 
6067 struct pmu *perf_init_event(struct perf_event *event)
6068 {
6069  struct pmu *pmu = NULL;
6070  int idx;
6071  int ret;
6072 
6073  idx = srcu_read_lock(&pmus_srcu);
6074 
6075  rcu_read_lock();
6076  pmu = idr_find(&pmu_idr, event->attr.type);
6077  rcu_read_unlock();
6078  if (pmu) {
6079  event->pmu = pmu;
6080  ret = pmu->event_init(event);
6081  if (ret)
6082  pmu = ERR_PTR(ret);
6083  goto unlock;
6084  }
6085 
6086  list_for_each_entry_rcu(pmu, &pmus, entry) {
6087  event->pmu = pmu;
6088  ret = pmu->event_init(event);
6089  if (!ret)
6090  goto unlock;
6091 
6092  if (ret != -ENOENT) {
6093  pmu = ERR_PTR(ret);
6094  goto unlock;
6095  }
6096  }
6097  pmu = ERR_PTR(-ENOENT);
6098 unlock:
6099  srcu_read_unlock(&pmus_srcu, idx);
6100 
6101  return pmu;
6102 }
6103 
6104 /*
6105  * Allocate and initialize a event structure
6106  */
6107 static struct perf_event *
6108 perf_event_alloc(struct perf_event_attr *attr, int cpu,
6109  struct task_struct *task,
6110  struct perf_event *group_leader,
6111  struct perf_event *parent_event,
6112  perf_overflow_handler_t overflow_handler,
6113  void *context)
6114 {
6115  struct pmu *pmu;
6116  struct perf_event *event;
6117  struct hw_perf_event *hwc;
6118  long err;
6119 
6120  if ((unsigned)cpu >= nr_cpu_ids) {
6121  if (!task || cpu != -1)
6122  return ERR_PTR(-EINVAL);
6123  }
6124 
6125  event = kzalloc(sizeof(*event), GFP_KERNEL);
6126  if (!event)
6127  return ERR_PTR(-ENOMEM);
6128 
6129  /*
6130  * Single events are their own group leaders, with an
6131  * empty sibling list:
6132  */
6133  if (!group_leader)
6134  group_leader = event;
6135 
6136  mutex_init(&event->child_mutex);
6137  INIT_LIST_HEAD(&event->child_list);
6138 
6139  INIT_LIST_HEAD(&event->group_entry);
6140  INIT_LIST_HEAD(&event->event_entry);
6141  INIT_LIST_HEAD(&event->sibling_list);
6142  INIT_LIST_HEAD(&event->rb_entry);
6143 
6144  init_waitqueue_head(&event->waitq);
6145  init_irq_work(&event->pending, perf_pending_event);
6146 
6147  mutex_init(&event->mmap_mutex);
6148 
6149  atomic_long_set(&event->refcount, 1);
6150  event->cpu = cpu;
6151  event->attr = *attr;
6152  event->group_leader = group_leader;
6153  event->pmu = NULL;
6154  event->oncpu = -1;
6155 
6156  event->parent = parent_event;
6157 
6158  event->ns = get_pid_ns(current->nsproxy->pid_ns);
6159  event->id = atomic64_inc_return(&perf_event_id);
6160 
6161  event->state = PERF_EVENT_STATE_INACTIVE;
6162 
6163  if (task) {
6164  event->attach_state = PERF_ATTACH_TASK;
6165 #ifdef CONFIG_HAVE_HW_BREAKPOINT
6166  /*
6167  * hw_breakpoint is a bit difficult here..
6168  */
6169  if (attr->type == PERF_TYPE_BREAKPOINT)
6170  event->hw.bp_target = task;
6171 #endif
6172  }
6173 
6174  if (!overflow_handler && parent_event) {
6175  overflow_handler = parent_event->overflow_handler;
6176  context = parent_event->overflow_handler_context;
6177  }
6178 
6179  event->overflow_handler = overflow_handler;
6180  event->overflow_handler_context = context;
6181 
6182  if (attr->disabled)
6183  event->state = PERF_EVENT_STATE_OFF;
6184 
6185  pmu = NULL;
6186 
6187  hwc = &event->hw;
6188  hwc->sample_period = attr->sample_period;
6189  if (attr->freq && attr->sample_freq)
6190  hwc->sample_period = 1;
6191  hwc->last_period = hwc->sample_period;
6192 
6193  local64_set(&hwc->period_left, hwc->sample_period);
6194 
6195  /*
6196  * we currently do not support PERF_FORMAT_GROUP on inherited events
6197  */
6198  if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
6199  goto done;
6200 
6201  pmu = perf_init_event(event);
6202 
6203 done:
6204  err = 0;
6205  if (!pmu)
6206  err = -EINVAL;
6207  else if (IS_ERR(pmu))
6208  err = PTR_ERR(pmu);
6209 
6210  if (err) {
6211  if (event->ns)
6212  put_pid_ns(event->ns);
6213  kfree(event);
6214  return ERR_PTR(err);
6215  }
6216 
6217  if (!event->parent) {
6218  if (event->attach_state & PERF_ATTACH_TASK)
6219  static_key_slow_inc(&perf_sched_events.key);
6220  if (event->attr.mmap || event->attr.mmap_data)
6221  atomic_inc(&nr_mmap_events);
6222  if (event->attr.comm)
6223  atomic_inc(&nr_comm_events);
6224  if (event->attr.task)
6225  atomic_inc(&nr_task_events);
6226  if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
6227  err = get_callchain_buffers();
6228  if (err) {
6229  free_event(event);
6230  return ERR_PTR(err);
6231  }
6232  }
6233  if (has_branch_stack(event)) {
6234  static_key_slow_inc(&perf_sched_events.key);
6235  if (!(event->attach_state & PERF_ATTACH_TASK))
6236  atomic_inc(&per_cpu(perf_branch_stack_events,
6237  event->cpu));
6238  }
6239  }
6240 
6241  return event;
6242 }
6243 
6244 static int perf_copy_attr(struct perf_event_attr __user *uattr,
6245  struct perf_event_attr *attr)
6246 {
6247  u32 size;
6248  int ret;
6249 
6251  return -EFAULT;
6252 
6253  /*
6254  * zero the full structure, so that a short copy will be nice.
6255  */
6256  memset(attr, 0, sizeof(*attr));
6257 
6258  ret = get_user(size, &uattr->size);
6259  if (ret)
6260  return ret;
6261 
6262  if (size > PAGE_SIZE) /* silly large */
6263  goto err_size;
6264 
6265  if (!size) /* abi compat */
6266  size = PERF_ATTR_SIZE_VER0;
6267 
6268  if (size < PERF_ATTR_SIZE_VER0)
6269  goto err_size;
6270 
6271  /*
6272  * If we're handed a bigger struct than we know of,
6273  * ensure all the unknown bits are 0 - i.e. new
6274  * user-space does not rely on any kernel feature
6275  * extensions we dont know about yet.
6276  */
6277  if (size > sizeof(*attr)) {
6278  unsigned char __user *addr;
6279  unsigned char __user *end;
6280  unsigned char val;
6281 
6282  addr = (void __user *)uattr + sizeof(*attr);
6283  end = (void __user *)uattr + size;
6284 
6285  for (; addr < end; addr++) {
6286  ret = get_user(val, addr);
6287  if (ret)
6288  return ret;
6289  if (val)
6290  goto err_size;
6291  }
6292  size = sizeof(*attr);
6293  }
6294 
6295  ret = copy_from_user(attr, uattr, size);
6296  if (ret)
6297  return -EFAULT;
6298 
6299  if (attr->__reserved_1)
6300  return -EINVAL;
6301 
6302  if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
6303  return -EINVAL;
6304 
6305  if (attr->read_format & ~(PERF_FORMAT_MAX-1))
6306  return -EINVAL;
6307 
6308  if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
6309  u64 mask = attr->branch_sample_type;
6310 
6311  /* only using defined bits */
6312  if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
6313  return -EINVAL;
6314 
6315  /* at least one branch bit must be set */
6316  if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
6317  return -EINVAL;
6318 
6319  /* kernel level capture: check permissions */
6320  if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
6321  && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6322  return -EACCES;
6323 
6324  /* propagate priv level, when not set for branch */
6325  if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
6326 
6327  /* exclude_kernel checked on syscall entry */
6328  if (!attr->exclude_kernel)
6329  mask |= PERF_SAMPLE_BRANCH_KERNEL;
6330 
6331  if (!attr->exclude_user)
6332  mask |= PERF_SAMPLE_BRANCH_USER;
6333 
6334  if (!attr->exclude_hv)
6335  mask |= PERF_SAMPLE_BRANCH_HV;
6336  /*
6337  * adjust user setting (for HW filter setup)
6338  */
6339  attr->branch_sample_type = mask;
6340  }
6341  }
6342 
6343  if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
6344  ret = perf_reg_validate(attr->sample_regs_user);
6345  if (ret)
6346  return ret;
6347  }
6348 
6349  if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
6350  if (!arch_perf_have_user_stack_dump())
6351  return -ENOSYS;
6352 
6353  /*
6354  * We have __u32 type for the size, but so far
6355  * we can only use __u16 as maximum due to the
6356  * __u16 sample size limit.
6357  */
6358  if (attr->sample_stack_user >= USHRT_MAX)
6359  ret = -EINVAL;
6360  else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
6361  ret = -EINVAL;
6362  }
6363 
6364 out:
6365  return ret;
6366 
6367 err_size:
6368  put_user(sizeof(*attr), &uattr->size);
6369  ret = -E2BIG;
6370  goto out;
6371 }
6372 
6373 static int
6374 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
6375 {
6376  struct ring_buffer *rb = NULL, *old_rb = NULL;
6377  int ret = -EINVAL;
6378 
6379  if (!output_event)
6380  goto set;
6381 
6382  /* don't allow circular references */
6383  if (event == output_event)
6384  goto out;
6385 
6386  /*
6387  * Don't allow cross-cpu buffers
6388  */
6389  if (output_event->cpu != event->cpu)
6390  goto out;
6391 
6392  /*
6393  * If its not a per-cpu rb, it must be the same task.
6394  */
6395  if (output_event->cpu == -1 && output_event->ctx != event->ctx)
6396  goto out;
6397 
6398 set:
6399  mutex_lock(&event->mmap_mutex);
6400  /* Can't redirect output if we've got an active mmap() */
6401  if (atomic_read(&event->mmap_count))
6402  goto unlock;
6403 
6404  if (output_event) {
6405  /* get the rb we want to redirect to */
6406  rb = ring_buffer_get(output_event);
6407  if (!rb)
6408  goto unlock;
6409  }
6410 
6411  old_rb = event->rb;
6412  rcu_assign_pointer(event->rb, rb);
6413  if (old_rb)
6414  ring_buffer_detach(event, old_rb);
6415  ret = 0;
6416 unlock:
6417  mutex_unlock(&event->mmap_mutex);
6418 
6419  if (old_rb)
6420  ring_buffer_put(old_rb);
6421 out:
6422  return ret;
6423 }
6424 
6433 SYSCALL_DEFINE5(perf_event_open,
6434  struct perf_event_attr __user *, attr_uptr,
6435  pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
6436 {
6437  struct perf_event *group_leader = NULL, *output_event = NULL;
6438  struct perf_event *event, *sibling;
6439  struct perf_event_attr attr;
6440  struct perf_event_context *ctx;
6441  struct file *event_file = NULL;
6442  struct fd group = {NULL, 0};
6443  struct task_struct *task = NULL;
6444  struct pmu *pmu;
6445  int event_fd;
6446  int move_group = 0;
6447  int err;
6448 
6449  /* for future expandability... */
6450  if (flags & ~PERF_FLAG_ALL)
6451  return -EINVAL;
6452 
6453  err = perf_copy_attr(attr_uptr, &attr);
6454  if (err)
6455  return err;
6456 
6457  if (!attr.exclude_kernel) {
6458  if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
6459  return -EACCES;
6460  }
6461 
6462  if (attr.freq) {
6463  if (attr.sample_freq > sysctl_perf_event_sample_rate)
6464  return -EINVAL;
6465  }
6466 
6467  /*
6468  * In cgroup mode, the pid argument is used to pass the fd
6469  * opened to the cgroup directory in cgroupfs. The cpu argument
6470  * designates the cpu on which to monitor threads from that
6471  * cgroup.
6472  */
6473  if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
6474  return -EINVAL;
6475 
6476  event_fd = get_unused_fd();
6477  if (event_fd < 0)
6478  return event_fd;
6479 
6480  if (group_fd != -1) {
6481  err = perf_fget_light(group_fd, &group);
6482  if (err)
6483  goto err_fd;
6484  group_leader = group.file->private_data;
6485  if (flags & PERF_FLAG_FD_OUTPUT)
6486  output_event = group_leader;
6487  if (flags & PERF_FLAG_FD_NO_GROUP)
6488  group_leader = NULL;
6489  }
6490 
6491  if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
6492  task = find_lively_task_by_vpid(pid);
6493  if (IS_ERR(task)) {
6494  err = PTR_ERR(task);
6495  goto err_group_fd;
6496  }
6497  }
6498 
6499  get_online_cpus();
6500 
6501  event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
6502  NULL, NULL);
6503  if (IS_ERR(event)) {
6504  err = PTR_ERR(event);
6505  goto err_task;
6506  }
6507 
6508  if (flags & PERF_FLAG_PID_CGROUP) {
6509  err = perf_cgroup_connect(pid, event, &attr, group_leader);
6510  if (err)
6511  goto err_alloc;
6512  /*
6513  * one more event:
6514  * - that has cgroup constraint on event->cpu
6515  * - that may need work on context switch
6516  */
6517  atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
6518  static_key_slow_inc(&perf_sched_events.key);
6519  }
6520 
6521  /*
6522  * Special case software events and allow them to be part of
6523  * any hardware group.
6524  */
6525  pmu = event->pmu;
6526 
6527  if (group_leader &&
6528  (is_software_event(event) != is_software_event(group_leader))) {
6529  if (is_software_event(event)) {
6530  /*
6531  * If event and group_leader are not both a software
6532  * event, and event is, then group leader is not.
6533  *
6534  * Allow the addition of software events to !software
6535  * groups, this is safe because software events never
6536  * fail to schedule.
6537  */
6538  pmu = group_leader->pmu;
6539  } else if (is_software_event(group_leader) &&
6540  (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
6541  /*
6542  * In case the group is a pure software group, and we
6543  * try to add a hardware event, move the whole group to
6544  * the hardware context.
6545  */
6546  move_group = 1;
6547  }
6548  }
6549 
6550  /*
6551  * Get the target context (task or percpu):
6552  */
6553  ctx = find_get_context(pmu, task, event->cpu);
6554  if (IS_ERR(ctx)) {
6555  err = PTR_ERR(ctx);
6556  goto err_alloc;
6557  }
6558 
6559  if (task) {
6560  put_task_struct(task);
6561  task = NULL;
6562  }
6563 
6564  /*
6565  * Look up the group leader (we will attach this event to it):
6566  */
6567  if (group_leader) {
6568  err = -EINVAL;
6569 
6570  /*
6571  * Do not allow a recursive hierarchy (this new sibling
6572  * becoming part of another group-sibling):
6573  */
6574  if (group_leader->group_leader != group_leader)
6575  goto err_context;
6576  /*
6577  * Do not allow to attach to a group in a different
6578  * task or CPU context:
6579  */
6580  if (move_group) {
6581  if (group_leader->ctx->type != ctx->type)
6582  goto err_context;
6583  } else {
6584  if (group_leader->ctx != ctx)
6585  goto err_context;
6586  }
6587 
6588  /*
6589  * Only a group leader can be exclusive or pinned
6590  */
6591  if (attr.exclusive || attr.pinned)
6592  goto err_context;
6593  }
6594 
6595  if (output_event) {
6596  err = perf_event_set_output(event, output_event);
6597  if (err)
6598  goto err_context;
6599  }
6600 
6601  event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
6602  if (IS_ERR(event_file)) {
6603  err = PTR_ERR(event_file);
6604  goto err_context;
6605  }
6606 
6607  if (move_group) {
6608  struct perf_event_context *gctx = group_leader->ctx;
6609 
6610  mutex_lock(&gctx->mutex);
6611  perf_remove_from_context(group_leader);
6612  list_for_each_entry(sibling, &group_leader->sibling_list,
6613  group_entry) {
6614  perf_remove_from_context(sibling);
6615  put_ctx(gctx);
6616  }
6617  mutex_unlock(&gctx->mutex);
6618  put_ctx(gctx);
6619  }
6620 
6621  WARN_ON_ONCE(ctx->parent_ctx);
6622  mutex_lock(&ctx->mutex);
6623 
6624  if (move_group) {
6625  synchronize_rcu();
6626  perf_install_in_context(ctx, group_leader, event->cpu);
6627  get_ctx(ctx);
6628  list_for_each_entry(sibling, &group_leader->sibling_list,
6629  group_entry) {
6630  perf_install_in_context(ctx, sibling, event->cpu);
6631  get_ctx(ctx);
6632  }
6633  }
6634 
6635  perf_install_in_context(ctx, event, event->cpu);
6636  ++ctx->generation;
6637  perf_unpin_context(ctx);
6638  mutex_unlock(&ctx->mutex);
6639 
6640  put_online_cpus();
6641 
6642  event->owner = current;
6643 
6644  mutex_lock(&current->perf_event_mutex);
6645  list_add_tail(&event->owner_entry, &current->perf_event_list);
6646  mutex_unlock(&current->perf_event_mutex);
6647 
6648  /*
6649  * Precalculate sample_data sizes
6650  */
6651  perf_event__header_size(event);
6652  perf_event__id_header_size(event);
6653 
6654  /*
6655  * Drop the reference on the group_event after placing the
6656  * new event on the sibling_list. This ensures destruction
6657  * of the group leader will find the pointer to itself in
6658  * perf_group_detach().
6659  */
6660  fdput(group);
6661  fd_install(event_fd, event_file);
6662  return event_fd;
6663 
6664 err_context:
6665  perf_unpin_context(ctx);
6666  put_ctx(ctx);
6667 err_alloc:
6668  free_event(event);
6669 err_task:
6670  put_online_cpus();
6671  if (task)
6672  put_task_struct(task);
6673 err_group_fd:
6674  fdput(group);
6675 err_fd:
6676  put_unused_fd(event_fd);
6677  return err;
6678 }
6679 
6687 struct perf_event *
6689  struct task_struct *task,
6690  perf_overflow_handler_t overflow_handler,
6691  void *context)
6692 {
6693  struct perf_event_context *ctx;
6694  struct perf_event *event;
6695  int err;
6696 
6697  /*
6698  * Get the target context (task or percpu):
6699  */
6700 
6701  event = perf_event_alloc(attr, cpu, task, NULL, NULL,
6702  overflow_handler, context);
6703  if (IS_ERR(event)) {
6704  err = PTR_ERR(event);
6705  goto err;
6706  }
6707 
6708  ctx = find_get_context(event->pmu, task, cpu);
6709  if (IS_ERR(ctx)) {
6710  err = PTR_ERR(ctx);
6711  goto err_free;
6712  }
6713 
6714  WARN_ON_ONCE(ctx->parent_ctx);
6715  mutex_lock(&ctx->mutex);
6716  perf_install_in_context(ctx, event, cpu);
6717  ++ctx->generation;
6718  perf_unpin_context(ctx);
6719  mutex_unlock(&ctx->mutex);
6720 
6721  return event;
6722 
6723 err_free:
6724  free_event(event);
6725 err:
6726  return ERR_PTR(err);
6727 }
6729 
6730 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
6731 {
6732  struct perf_event_context *src_ctx;
6733  struct perf_event_context *dst_ctx;
6734  struct perf_event *event, *tmp;
6735  LIST_HEAD(events);
6736 
6737  src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
6738  dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
6739 
6740  mutex_lock(&src_ctx->mutex);
6741  list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
6742  event_entry) {
6743  perf_remove_from_context(event);
6744  put_ctx(src_ctx);
6745  list_add(&event->event_entry, &events);
6746  }
6747  mutex_unlock(&src_ctx->mutex);
6748 
6749  synchronize_rcu();
6750 
6751  mutex_lock(&dst_ctx->mutex);
6752  list_for_each_entry_safe(event, tmp, &events, event_entry) {
6753  list_del(&event->event_entry);
6754  if (event->state >= PERF_EVENT_STATE_OFF)
6755  event->state = PERF_EVENT_STATE_INACTIVE;
6756  perf_install_in_context(dst_ctx, event, dst_cpu);
6757  get_ctx(dst_ctx);
6758  }
6759  mutex_unlock(&dst_ctx->mutex);
6760 }
6762 
6763 static void sync_child_event(struct perf_event *child_event,
6764  struct task_struct *child)
6765 {
6766  struct perf_event *parent_event = child_event->parent;
6767  u64 child_val;
6768 
6769  if (child_event->attr.inherit_stat)
6770  perf_event_read_event(child_event, child);
6771 
6772  child_val = perf_event_count(child_event);
6773 
6774  /*
6775  * Add back the child's count to the parent's count:
6776  */
6777  atomic64_add(child_val, &parent_event->child_count);
6778  atomic64_add(child_event->total_time_enabled,
6779  &parent_event->child_total_time_enabled);
6780  atomic64_add(child_event->total_time_running,
6781  &parent_event->child_total_time_running);
6782 
6783  /*
6784  * Remove this event from the parent's list
6785  */
6786  WARN_ON_ONCE(parent_event->ctx->parent_ctx);
6787  mutex_lock(&parent_event->child_mutex);
6788  list_del_init(&child_event->child_list);
6789  mutex_unlock(&parent_event->child_mutex);
6790 
6791  /*
6792  * Release the parent event, if this was the last
6793  * reference to it.
6794  */
6795  put_event(parent_event);
6796 }
6797 
6798 static void
6799 __perf_event_exit_task(struct perf_event *child_event,
6800  struct perf_event_context *child_ctx,
6801  struct task_struct *child)
6802 {
6803  if (child_event->parent) {
6804  raw_spin_lock_irq(&child_ctx->lock);
6805  perf_group_detach(child_event);
6806  raw_spin_unlock_irq(&child_ctx->lock);
6807  }
6808 
6809  perf_remove_from_context(child_event);
6810 
6811  /*
6812  * It can happen that the parent exits first, and has events
6813  * that are still around due to the child reference. These
6814  * events need to be zapped.
6815  */
6816  if (child_event->parent) {
6817  sync_child_event(child_event, child);
6818  free_event(child_event);
6819  }
6820 }
6821 
6822 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6823 {
6824  struct perf_event *child_event, *tmp;
6825  struct perf_event_context *child_ctx;
6826  unsigned long flags;
6827 
6828  if (likely(!child->perf_event_ctxp[ctxn])) {
6829  perf_event_task(child, NULL, 0);
6830  return;
6831  }
6832 
6833  local_irq_save(flags);
6834  /*
6835  * We can't reschedule here because interrupts are disabled,
6836  * and either child is current or it is a task that can't be
6837  * scheduled, so we are now safe from rescheduling changing
6838  * our context.
6839  */
6840  child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6841 
6842  /*
6843  * Take the context lock here so that if find_get_context is
6844  * reading child->perf_event_ctxp, we wait until it has
6845  * incremented the context's refcount before we do put_ctx below.
6846  */
6847  raw_spin_lock(&child_ctx->lock);
6848  task_ctx_sched_out(child_ctx);
6849  child->perf_event_ctxp[ctxn] = NULL;
6850  /*
6851  * If this context is a clone; unclone it so it can't get
6852  * swapped to another process while we're removing all
6853  * the events from it.
6854  */
6855  unclone_ctx(child_ctx);
6856  update_context_time(child_ctx);
6857  raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6858 
6859  /*
6860  * Report the task dead after unscheduling the events so that we
6861  * won't get any samples after PERF_RECORD_EXIT. We can however still
6862  * get a few PERF_RECORD_READ events.
6863  */
6864  perf_event_task(child, child_ctx, 0);
6865 
6866  /*
6867  * We can recurse on the same lock type through:
6868  *
6869  * __perf_event_exit_task()
6870  * sync_child_event()
6871  * put_event()
6872  * mutex_lock(&ctx->mutex)
6873  *
6874  * But since its the parent context it won't be the same instance.
6875  */
6876  mutex_lock(&child_ctx->mutex);
6877 
6878 again:
6879  list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
6880  group_entry)
6881  __perf_event_exit_task(child_event, child_ctx, child);
6882 
6883  list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
6884  group_entry)
6885  __perf_event_exit_task(child_event, child_ctx, child);
6886 
6887  /*
6888  * If the last event was a group event, it will have appended all
6889  * its siblings to the list, but we obtained 'tmp' before that which
6890  * will still point to the list head terminating the iteration.
6891  */
6892  if (!list_empty(&child_ctx->pinned_groups) ||
6893  !list_empty(&child_ctx->flexible_groups))
6894  goto again;
6895 
6896  mutex_unlock(&child_ctx->mutex);
6897 
6898  put_ctx(child_ctx);
6899 }
6900 
6901 /*
6902  * When a child task exits, feed back event values to parent events.
6903  */
6905 {
6906  struct perf_event *event, *tmp;
6907  int ctxn;
6908 
6909  mutex_lock(&child->perf_event_mutex);
6910  list_for_each_entry_safe(event, tmp, &child->perf_event_list,
6911  owner_entry) {
6912  list_del_init(&event->owner_entry);
6913 
6914  /*
6915  * Ensure the list deletion is visible before we clear
6916  * the owner, closes a race against perf_release() where
6917  * we need to serialize on the owner->perf_event_mutex.
6918  */
6919  smp_wmb();
6920  event->owner = NULL;
6921  }
6922  mutex_unlock(&child->perf_event_mutex);
6923 
6925  perf_event_exit_task_context(child, ctxn);
6926 }
6927 
6928 static void perf_free_event(struct perf_event *event,
6929  struct perf_event_context *ctx)
6930 {
6931  struct perf_event *parent = event->parent;
6932 
6933  if (WARN_ON_ONCE(!parent))
6934  return;
6935 
6936  mutex_lock(&parent->child_mutex);
6937  list_del_init(&event->child_list);
6938  mutex_unlock(&parent->child_mutex);
6939 
6940  put_event(parent);
6941 
6942  perf_group_detach(event);
6943  list_del_event(event, ctx);
6944  free_event(event);
6945 }
6946 
6947 /*
6948  * free an unexposed, unused context as created by inheritance by
6949  * perf_event_init_task below, used by fork() in case of fail.
6950  */
6952 {
6953  struct perf_event_context *ctx;
6954  struct perf_event *event, *tmp;
6955  int ctxn;
6956 
6957  for_each_task_context_nr(ctxn) {
6958  ctx = task->perf_event_ctxp[ctxn];
6959  if (!ctx)
6960  continue;
6961 
6962  mutex_lock(&ctx->mutex);
6963 again:
6964  list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
6965  group_entry)
6966  perf_free_event(event, ctx);
6967 
6968  list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
6969  group_entry)
6970  perf_free_event(event, ctx);
6971 
6972  if (!list_empty(&ctx->pinned_groups) ||
6973  !list_empty(&ctx->flexible_groups))
6974  goto again;
6975 
6976  mutex_unlock(&ctx->mutex);
6977 
6978  put_ctx(ctx);
6979  }
6980 }
6981 
6983 {
6984  int ctxn;
6985 
6987  WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
6988 }
6989 
6990 /*
6991  * inherit a event from parent task to child task:
6992  */
6993 static struct perf_event *
6994 inherit_event(struct perf_event *parent_event,
6995  struct task_struct *parent,
6996  struct perf_event_context *parent_ctx,
6997  struct task_struct *child,
6998  struct perf_event *group_leader,
6999  struct perf_event_context *child_ctx)
7000 {
7001  struct perf_event *child_event;
7002  unsigned long flags;
7003 
7004  /*
7005  * Instead of creating recursive hierarchies of events,
7006  * we link inherited events back to the original parent,
7007  * which has a filp for sure, which we use as the reference
7008  * count:
7009  */
7010  if (parent_event->parent)
7011  parent_event = parent_event->parent;
7012 
7013  child_event = perf_event_alloc(&parent_event->attr,
7014  parent_event->cpu,
7015  child,
7016  group_leader, parent_event,
7017  NULL, NULL);
7018  if (IS_ERR(child_event))
7019  return child_event;
7020 
7021  if (!atomic_long_inc_not_zero(&parent_event->refcount)) {
7022  free_event(child_event);
7023  return NULL;
7024  }
7025 
7026  get_ctx(child_ctx);
7027 
7028  /*
7029  * Make the child state follow the state of the parent event,
7030  * not its attr.disabled bit. We hold the parent's mutex,
7031  * so we won't race with perf_event_{en, dis}able_family.
7032  */
7033  if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
7034  child_event->state = PERF_EVENT_STATE_INACTIVE;
7035  else
7036  child_event->state = PERF_EVENT_STATE_OFF;
7037 
7038  if (parent_event->attr.freq) {
7039  u64 sample_period = parent_event->hw.sample_period;
7040  struct hw_perf_event *hwc = &child_event->hw;
7041 
7042  hwc->sample_period = sample_period;
7043  hwc->last_period = sample_period;
7044 
7045  local64_set(&hwc->period_left, sample_period);
7046  }
7047 
7048  child_event->ctx = child_ctx;
7049  child_event->overflow_handler = parent_event->overflow_handler;
7050  child_event->overflow_handler_context
7051  = parent_event->overflow_handler_context;
7052 
7053  /*
7054  * Precalculate sample_data sizes
7055  */
7056  perf_event__header_size(child_event);
7057  perf_event__id_header_size(child_event);
7058 
7059  /*
7060  * Link it up in the child's context:
7061  */
7062  raw_spin_lock_irqsave(&child_ctx->lock, flags);
7063  add_event_to_ctx(child_event, child_ctx);
7064  raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
7065 
7066  /*
7067  * Link this into the parent event's child list
7068  */
7069  WARN_ON_ONCE(parent_event->ctx->parent_ctx);
7070  mutex_lock(&parent_event->child_mutex);
7071  list_add_tail(&child_event->child_list, &parent_event->child_list);
7072  mutex_unlock(&parent_event->child_mutex);
7073 
7074  return child_event;
7075 }
7076 
7077 static int inherit_group(struct perf_event *parent_event,
7078  struct task_struct *parent,
7079  struct perf_event_context *parent_ctx,
7080  struct task_struct *child,
7081  struct perf_event_context *child_ctx)
7082 {
7083  struct perf_event *leader;
7084  struct perf_event *sub;
7085  struct perf_event *child_ctr;
7086 
7087  leader = inherit_event(parent_event, parent, parent_ctx,
7088  child, NULL, child_ctx);
7089  if (IS_ERR(leader))
7090  return PTR_ERR(leader);
7091  list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
7092  child_ctr = inherit_event(sub, parent, parent_ctx,
7093  child, leader, child_ctx);
7094  if (IS_ERR(child_ctr))
7095  return PTR_ERR(child_ctr);
7096  }
7097  return 0;
7098 }
7099 
7100 static int
7101 inherit_task_group(struct perf_event *event, struct task_struct *parent,
7102  struct perf_event_context *parent_ctx,
7103  struct task_struct *child, int ctxn,
7104  int *inherited_all)
7105 {
7106  int ret;
7107  struct perf_event_context *child_ctx;
7108 
7109  if (!event->attr.inherit) {
7110  *inherited_all = 0;
7111  return 0;
7112  }
7113 
7114  child_ctx = child->perf_event_ctxp[ctxn];
7115  if (!child_ctx) {
7116  /*
7117  * This is executed from the parent task context, so
7118  * inherit events that have been marked for cloning.
7119  * First allocate and initialize a context for the
7120  * child.
7121  */
7122 
7123  child_ctx = alloc_perf_context(event->pmu, child);
7124  if (!child_ctx)
7125  return -ENOMEM;
7126 
7127  child->perf_event_ctxp[ctxn] = child_ctx;
7128  }
7129 
7130  ret = inherit_group(event, parent, parent_ctx,
7131  child, child_ctx);
7132 
7133  if (ret)
7134  *inherited_all = 0;
7135 
7136  return ret;
7137 }
7138 
7139 /*
7140  * Initialize the perf_event context in task_struct
7141  */
7142 int perf_event_init_context(struct task_struct *child, int ctxn)
7143 {
7144  struct perf_event_context *child_ctx, *parent_ctx;
7145  struct perf_event_context *cloned_ctx;
7146  struct perf_event *event;
7147  struct task_struct *parent = current;
7148  int inherited_all = 1;
7149  unsigned long flags;
7150  int ret = 0;
7151 
7152  if (likely(!parent->perf_event_ctxp[ctxn]))
7153  return 0;
7154 
7155  /*
7156  * If the parent's context is a clone, pin it so it won't get
7157  * swapped under us.
7158  */
7159  parent_ctx = perf_pin_task_context(parent, ctxn);
7160 
7161  /*
7162  * No need to check if parent_ctx != NULL here; since we saw
7163  * it non-NULL earlier, the only reason for it to become NULL
7164  * is if we exit, and since we're currently in the middle of
7165  * a fork we can't be exiting at the same time.
7166  */
7167 
7168  /*
7169  * Lock the parent list. No need to lock the child - not PID
7170  * hashed yet and not running, so nobody can access it.
7171  */
7172  mutex_lock(&parent_ctx->mutex);
7173 
7174  /*
7175  * We dont have to disable NMIs - we are only looking at
7176  * the list, not manipulating it:
7177  */
7178  list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
7179  ret = inherit_task_group(event, parent, parent_ctx,
7180  child, ctxn, &inherited_all);
7181  if (ret)
7182  break;
7183  }
7184 
7185  /*
7186  * We can't hold ctx->lock when iterating the ->flexible_group list due
7187  * to allocations, but we need to prevent rotation because
7188  * rotate_ctx() will change the list from interrupt context.
7189  */
7190  raw_spin_lock_irqsave(&parent_ctx->lock, flags);
7191  parent_ctx->rotate_disable = 1;
7192  raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
7193 
7194  list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
7195  ret = inherit_task_group(event, parent, parent_ctx,
7196  child, ctxn, &inherited_all);
7197  if (ret)
7198  break;
7199  }
7200 
7201  raw_spin_lock_irqsave(&parent_ctx->lock, flags);
7202  parent_ctx->rotate_disable = 0;
7203 
7204  child_ctx = child->perf_event_ctxp[ctxn];
7205 
7206  if (child_ctx && inherited_all) {
7207  /*
7208  * Mark the child context as a clone of the parent
7209  * context, or of whatever the parent is a clone of.
7210  *
7211  * Note that if the parent is a clone, the holding of
7212  * parent_ctx->lock avoids it from being uncloned.
7213  */
7214  cloned_ctx = parent_ctx->parent_ctx;
7215  if (cloned_ctx) {
7216  child_ctx->parent_ctx = cloned_ctx;
7217  child_ctx->parent_gen = parent_ctx->parent_gen;
7218  } else {
7219  child_ctx->parent_ctx = parent_ctx;
7220  child_ctx->parent_gen = parent_ctx->generation;
7221  }
7222  get_ctx(child_ctx->parent_ctx);
7223  }
7224 
7225  raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
7226  mutex_unlock(&parent_ctx->mutex);
7227 
7228  perf_unpin_context(parent_ctx);
7229  put_ctx(parent_ctx);
7230 
7231  return ret;
7232 }
7233 
7234 /*
7235  * Initialize the perf_event context in task_struct
7236  */
7238 {
7239  int ctxn, ret;
7240 
7241  memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
7242  mutex_init(&child->perf_event_mutex);
7243  INIT_LIST_HEAD(&child->perf_event_list);
7244 
7245  for_each_task_context_nr(ctxn) {
7246  ret = perf_event_init_context(child, ctxn);
7247  if (ret)
7248  return ret;
7249  }
7250 
7251  return 0;
7252 }
7253 
7254 static void __init perf_event_init_all_cpus(void)
7255 {
7256  struct swevent_htable *swhash;
7257  int cpu;
7258 
7259  for_each_possible_cpu(cpu) {
7260  swhash = &per_cpu(swevent_htable, cpu);
7261  mutex_init(&swhash->hlist_mutex);
7262  INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
7263  }
7264 }
7265 
7266 static void __cpuinit perf_event_init_cpu(int cpu)
7267 {
7268  struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
7269 
7270  mutex_lock(&swhash->hlist_mutex);
7271  if (swhash->hlist_refcount > 0) {
7272  struct swevent_hlist *hlist;
7273 
7274  hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
7275  WARN_ON(!hlist);
7276  rcu_assign_pointer(swhash->swevent_hlist, hlist);
7277  }
7278  mutex_unlock(&swhash->hlist_mutex);
7279 }
7280 
7281 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
7282 static void perf_pmu_rotate_stop(struct pmu *pmu)
7283 {
7284  struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
7285 
7286  WARN_ON(!irqs_disabled());
7287 
7288  list_del_init(&cpuctx->rotation_list);
7289 }
7290 
7291 static void __perf_event_exit_context(void *__info)
7292 {
7293  struct perf_event_context *ctx = __info;
7294  struct perf_event *event, *tmp;
7295 
7296  perf_pmu_rotate_stop(ctx->pmu);
7297 
7298  list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
7299  __perf_remove_from_context(event);
7300  list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
7301  __perf_remove_from_context(event);
7302 }
7303 
7304 static void perf_event_exit_cpu_context(int cpu)
7305 {
7306  struct perf_event_context *ctx;
7307  struct pmu *pmu;
7308  int idx;
7309 
7310  idx = srcu_read_lock(&pmus_srcu);
7311  list_for_each_entry_rcu(pmu, &pmus, entry) {
7312  ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
7313 
7314  mutex_lock(&ctx->mutex);
7315  smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
7316  mutex_unlock(&ctx->mutex);
7317  }
7318  srcu_read_unlock(&pmus_srcu, idx);
7319 }
7320 
7321 static void perf_event_exit_cpu(int cpu)
7322 {
7323  struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
7324 
7325  mutex_lock(&swhash->hlist_mutex);
7326  swevent_hlist_release(swhash);
7327  mutex_unlock(&swhash->hlist_mutex);
7328 
7329  perf_event_exit_cpu_context(cpu);
7330 }
7331 #else
7332 static inline void perf_event_exit_cpu(int cpu) { }
7333 #endif
7334 
7335 static int
7336 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
7337 {
7338  int cpu;
7339 
7340  for_each_online_cpu(cpu)
7341  perf_event_exit_cpu(cpu);
7342 
7343  return NOTIFY_OK;
7344 }
7345 
7346 /*
7347  * Run the perf reboot notifier at the very last possible moment so that
7348  * the generic watchdog code runs as long as possible.
7349  */
7350 static struct notifier_block perf_reboot_notifier = {
7351  .notifier_call = perf_reboot,
7352  .priority = INT_MIN,
7353 };
7354 
7355 static int __cpuinit
7356 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
7357 {
7358  unsigned int cpu = (long)hcpu;
7359 
7360  switch (action & ~CPU_TASKS_FROZEN) {
7361 
7362  case CPU_UP_PREPARE:
7363  case CPU_DOWN_FAILED:
7364  perf_event_init_cpu(cpu);
7365  break;
7366 
7367  case CPU_UP_CANCELED:
7368  case CPU_DOWN_PREPARE:
7369  perf_event_exit_cpu(cpu);
7370  break;
7371 
7372  default:
7373  break;
7374  }
7375 
7376  return NOTIFY_OK;
7377 }
7378 
7380 {
7381  int ret;
7382 
7383  idr_init(&pmu_idr);
7384 
7385  perf_event_init_all_cpus();
7386  init_srcu_struct(&pmus_srcu);
7387  perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
7388  perf_pmu_register(&perf_cpu_clock, NULL, -1);
7389  perf_pmu_register(&perf_task_clock, NULL, -1);
7390  perf_tp_register();
7391  perf_cpu_notifier(perf_cpu_notify);
7392  register_reboot_notifier(&perf_reboot_notifier);
7393 
7394  ret = init_hw_breakpoint();
7395  WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
7396 
7397  /* do not patch jump label more than once per second */
7398  jump_label_rate_limit(&perf_sched_events, HZ);
7399 
7400  /*
7401  * Build time assertion that we keep the data_head at the intended
7402  * location. IOW, validation we got the __reserved[] size right.
7403  */
7404  BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
7405  != 1024);
7406 }
7407 
7408 static int __init perf_event_sysfs_init(void)
7409 {
7410  struct pmu *pmu;
7411  int ret;
7412 
7413  mutex_lock(&pmus_lock);
7414 
7415  ret = bus_register(&pmu_bus);
7416  if (ret)
7417  goto unlock;
7418 
7419  list_for_each_entry(pmu, &pmus, entry) {
7420  if (!pmu->name || pmu->type < 0)
7421  continue;
7422 
7423  ret = pmu_dev_alloc(pmu);
7424  WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
7425  }
7426  pmu_bus_running = 1;
7427  ret = 0;
7428 
7429 unlock:
7430  mutex_unlock(&pmus_lock);
7431 
7432  return ret;
7433 }
7434 device_initcall(perf_event_sysfs_init);
7435 
7436 #ifdef CONFIG_CGROUP_PERF
7437 static struct cgroup_subsys_state *perf_cgroup_create(struct cgroup *cont)
7438 {
7439  struct perf_cgroup *jc;
7440 
7441  jc = kzalloc(sizeof(*jc), GFP_KERNEL);
7442  if (!jc)
7443  return ERR_PTR(-ENOMEM);
7444 
7445  jc->info = alloc_percpu(struct perf_cgroup_info);
7446  if (!jc->info) {
7447  kfree(jc);
7448  return ERR_PTR(-ENOMEM);
7449  }
7450 
7451  return &jc->css;
7452 }
7453 
7454 static void perf_cgroup_destroy(struct cgroup *cont)
7455 {
7456  struct perf_cgroup *jc;
7457  jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
7458  struct perf_cgroup, css);
7459  free_percpu(jc->info);
7460  kfree(jc);
7461 }
7462 
7463 static int __perf_cgroup_move(void *info)
7464 {
7465  struct task_struct *task = info;
7466  perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
7467  return 0;
7468 }
7469 
7470 static void perf_cgroup_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
7471 {
7472  struct task_struct *task;
7473 
7474  cgroup_taskset_for_each(task, cgrp, tset)
7475  task_function_call(task, __perf_cgroup_move, task);
7476 }
7477 
7478 static void perf_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
7479  struct task_struct *task)
7480 {
7481  /*
7482  * cgroup_exit() is called in the copy_process() failure path.
7483  * Ignore this case since the task hasn't ran yet, this avoids
7484  * trying to poke a half freed task state from generic code.
7485  */
7486  if (!(task->flags & PF_EXITING))
7487  return;
7488 
7489  task_function_call(task, __perf_cgroup_move, task);
7490 }
7491 
7492 struct cgroup_subsys perf_subsys = {
7493  .name = "perf_event",
7494  .subsys_id = perf_subsys_id,
7495  .create = perf_cgroup_create,
7496  .destroy = perf_cgroup_destroy,
7497  .exit = perf_cgroup_exit,
7498  .attach = perf_cgroup_attach,
7499 
7500  /*
7501  * perf_event cgroup doesn't handle nesting correctly.
7502  * ctx->nr_cgroups adjustments should be propagated through the
7503  * cgroup hierarchy. Fix it and remove the following.
7504  */
7505  .broken_hierarchy = true,
7506 };
7507 #endif /* CONFIG_CGROUP_PERF */