Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
ftrace.c
Go to the documentation of this file.
1 /*
2  * Infrastructure for profiling code inserted by 'gcc -pg'.
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
5  * Copyright (C) 2004-2008 Ingo Molnar <[email protected]>
6  *
7  * Originally ported from the -rt patch by:
8  * Copyright (C) 2007 Arnaldo Carvalho de Melo <[email protected]>
9  *
10  * Based on code in the latency_tracer, that is:
11  *
12  * Copyright (C) 2004-2006 Ingo Molnar
13  * Copyright (C) 2004 William Lee Irwin III
14  */
15 
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/bsearch.h>
26 #include <linux/module.h>
27 #include <linux/ftrace.h>
28 #include <linux/sysctl.h>
29 #include <linux/slab.h>
30 #include <linux/ctype.h>
31 #include <linux/sort.h>
32 #include <linux/list.h>
33 #include <linux/hash.h>
34 #include <linux/rcupdate.h>
35 
36 #include <trace/events/sched.h>
37 
38 #include <asm/setup.h>
39 
40 #include "trace_output.h"
41 #include "trace_stat.h"
42 
43 #define FTRACE_WARN_ON(cond) \
44  ({ \
45  int ___r = cond; \
46  if (WARN_ON(___r)) \
47  ftrace_kill(); \
48  ___r; \
49  })
50 
51 #define FTRACE_WARN_ON_ONCE(cond) \
52  ({ \
53  int ___r = cond; \
54  if (WARN_ON_ONCE(___r)) \
55  ftrace_kill(); \
56  ___r; \
57  })
58 
59 /* hash bits for specific function selection */
60 #define FTRACE_HASH_BITS 7
61 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
62 #define FTRACE_HASH_DEFAULT_BITS 10
63 #define FTRACE_HASH_MAX_BITS 12
64 
65 #define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL)
66 
67 static struct ftrace_ops ftrace_list_end __read_mostly = {
68  .func = ftrace_stub,
69  .flags = FTRACE_OPS_FL_RECURSION_SAFE,
70 };
71 
72 /* ftrace_enabled is a method to turn ftrace on or off */
73 int ftrace_enabled __read_mostly;
74 static int last_ftrace_enabled;
75 
76 /* Quick disabling of function tracer. */
77 int function_trace_stop __read_mostly;
78 
79 /* Current function tracing op */
80 struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
81 
82 /* List for set_ftrace_pid's pids. */
83 LIST_HEAD(ftrace_pids);
84 struct ftrace_pid {
85  struct list_head list;
86  struct pid *pid;
87 };
88 
89 /*
90  * ftrace_disabled is set when an anomaly is discovered.
91  * ftrace_disabled is much stronger than ftrace_enabled.
92  */
93 static int ftrace_disabled __read_mostly;
94 
95 static DEFINE_MUTEX(ftrace_lock);
96 
97 static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
98 static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;
99 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
100 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
101 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
102 static struct ftrace_ops global_ops;
103 static struct ftrace_ops control_ops;
104 
105 #if ARCH_SUPPORTS_FTRACE_OPS
106 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
107  struct ftrace_ops *op, struct pt_regs *regs);
108 #else
109 /* See comment below, where ftrace_ops_list_func is defined */
110 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
111 #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
112 #endif
113 
120 {
121  struct ftrace_ops *ops;
122  int cnt = 0;
123 
124  mutex_lock(&ftrace_lock);
125 
126  for (ops = ftrace_ops_list;
127  ops != &ftrace_list_end; ops = ops->next)
128  cnt++;
129 
130  mutex_unlock(&ftrace_lock);
131 
132  return cnt;
133 }
134 
135 /*
136  * Traverse the ftrace_global_list, invoking all entries. The reason that we
137  * can use rcu_dereference_raw() is that elements removed from this list
138  * are simply leaked, so there is no need to interact with a grace-period
139  * mechanism. The rcu_dereference_raw() calls are needed to handle
140  * concurrent insertions into the ftrace_global_list.
141  *
142  * Silly Alpha and silly pointer-speculation compiler optimizations!
143  */
144 static void
145 ftrace_global_list_func(unsigned long ip, unsigned long parent_ip,
146  struct ftrace_ops *op, struct pt_regs *regs)
147 {
149  return;
150 
152  op = rcu_dereference_raw(ftrace_global_list); /*see above*/
153  while (op != &ftrace_list_end) {
154  op->func(ip, parent_ip, op, regs);
155  op = rcu_dereference_raw(op->next); /*see above*/
156  };
158 }
159 
160 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
161  struct ftrace_ops *op, struct pt_regs *regs)
162 {
163  if (!test_tsk_trace_trace(current))
164  return;
165 
166  ftrace_pid_function(ip, parent_ip, op, regs);
167 }
168 
169 static void set_ftrace_pid_function(ftrace_func_t func)
170 {
171  /* do not set ftrace_pid_function to itself! */
172  if (func != ftrace_pid_func)
173  ftrace_pid_function = func;
174 }
175 
183 {
184  ftrace_trace_function = ftrace_stub;
185  ftrace_pid_function = ftrace_stub;
186 }
187 
188 static void control_ops_disable_all(struct ftrace_ops *ops)
189 {
190  int cpu;
191 
193  *per_cpu_ptr(ops->disabled, cpu) = 1;
194 }
195 
196 static int control_ops_alloc(struct ftrace_ops *ops)
197 {
198  int __percpu *disabled;
199 
200  disabled = alloc_percpu(int);
201  if (!disabled)
202  return -ENOMEM;
203 
204  ops->disabled = disabled;
205  control_ops_disable_all(ops);
206  return 0;
207 }
208 
209 static void control_ops_free(struct ftrace_ops *ops)
210 {
211  free_percpu(ops->disabled);
212 }
213 
214 static void update_global_ops(void)
215 {
216  ftrace_func_t func;
217 
218  /*
219  * If there's only one function registered, then call that
220  * function directly. Otherwise, we need to iterate over the
221  * registered callers.
222  */
223  if (ftrace_global_list == &ftrace_list_end ||
224  ftrace_global_list->next == &ftrace_list_end)
225  func = ftrace_global_list->func;
226  else
227  func = ftrace_global_list_func;
228 
229  /* If we filter on pids, update to use the pid function */
230  if (!list_empty(&ftrace_pids)) {
231  set_ftrace_pid_function(func);
232  func = ftrace_pid_func;
233  }
234 
235  global_ops.func = func;
236 }
237 
238 static void update_ftrace_function(void)
239 {
240  ftrace_func_t func;
241 
242  update_global_ops();
243 
244  /*
245  * If we are at the end of the list and this ops is
246  * recursion safe and not dynamic and the arch supports passing ops,
247  * then have the mcount trampoline call the function directly.
248  */
249  if (ftrace_ops_list == &ftrace_list_end ||
250  (ftrace_ops_list->next == &ftrace_list_end &&
251  !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC) &&
252  (ftrace_ops_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) &&
254  /* Set the ftrace_ops that the arch callback uses */
255  if (ftrace_ops_list == &global_ops)
256  function_trace_op = ftrace_global_list;
257  else
258  function_trace_op = ftrace_ops_list;
259  func = ftrace_ops_list->func;
260  } else {
261  /* Just use the default ftrace_ops */
262  function_trace_op = &ftrace_list_end;
263  func = ftrace_ops_list_func;
264  }
265 
266  ftrace_trace_function = func;
267 }
268 
269 static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
270 {
271  ops->next = *list;
272  /*
273  * We are entering ops into the list but another
274  * CPU might be walking that list. We need to make sure
275  * the ops->next pointer is valid before another CPU sees
276  * the ops pointer included into the list.
277  */
278  rcu_assign_pointer(*list, ops);
279 }
280 
281 static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
282 {
283  struct ftrace_ops **p;
284 
285  /*
286  * If we are removing the last function, then simply point
287  * to the ftrace_stub.
288  */
289  if (*list == ops && ops->next == &ftrace_list_end) {
290  *list = &ftrace_list_end;
291  return 0;
292  }
293 
294  for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
295  if (*p == ops)
296  break;
297 
298  if (*p != ops)
299  return -1;
300 
301  *p = (*p)->next;
302  return 0;
303 }
304 
305 static void add_ftrace_list_ops(struct ftrace_ops **list,
306  struct ftrace_ops *main_ops,
307  struct ftrace_ops *ops)
308 {
309  int first = *list == &ftrace_list_end;
310  add_ftrace_ops(list, ops);
311  if (first)
312  add_ftrace_ops(&ftrace_ops_list, main_ops);
313 }
314 
315 static int remove_ftrace_list_ops(struct ftrace_ops **list,
316  struct ftrace_ops *main_ops,
317  struct ftrace_ops *ops)
318 {
319  int ret = remove_ftrace_ops(list, ops);
320  if (!ret && *list == &ftrace_list_end)
321  ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);
322  return ret;
323 }
324 
325 static int __register_ftrace_function(struct ftrace_ops *ops)
326 {
327  if (unlikely(ftrace_disabled))
328  return -ENODEV;
329 
330  if (FTRACE_WARN_ON(ops == &global_ops))
331  return -EINVAL;
332 
333  if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
334  return -EBUSY;
335 
336  /* We don't support both control and global flags set. */
337  if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK)
338  return -EINVAL;
339 
340 #ifndef ARCH_SUPPORTS_FTRACE_SAVE_REGS
341  /*
342  * If the ftrace_ops specifies SAVE_REGS, then it only can be used
343  * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
344  * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
345  */
346  if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
347  !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
348  return -EINVAL;
349 
350  if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
351  ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
352 #endif
353 
354  if (!core_kernel_data((unsigned long)ops))
355  ops->flags |= FTRACE_OPS_FL_DYNAMIC;
356 
357  if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
358  add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops);
359  ops->flags |= FTRACE_OPS_FL_ENABLED;
360  } else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
361  if (control_ops_alloc(ops))
362  return -ENOMEM;
363  add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);
364  } else
365  add_ftrace_ops(&ftrace_ops_list, ops);
366 
367  if (ftrace_enabled)
368  update_ftrace_function();
369 
370  return 0;
371 }
372 
373 static int __unregister_ftrace_function(struct ftrace_ops *ops)
374 {
375  int ret;
376 
377  if (ftrace_disabled)
378  return -ENODEV;
379 
380  if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
381  return -EBUSY;
382 
383  if (FTRACE_WARN_ON(ops == &global_ops))
384  return -EINVAL;
385 
386  if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
387  ret = remove_ftrace_list_ops(&ftrace_global_list,
388  &global_ops, ops);
389  if (!ret)
390  ops->flags &= ~FTRACE_OPS_FL_ENABLED;
391  } else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
392  ret = remove_ftrace_list_ops(&ftrace_control_list,
393  &control_ops, ops);
394  if (!ret) {
395  /*
396  * The ftrace_ops is now removed from the list,
397  * so there'll be no new users. We must ensure
398  * all current users are done before we free
399  * the control data.
400  */
402  control_ops_free(ops);
403  }
404  } else
405  ret = remove_ftrace_ops(&ftrace_ops_list, ops);
406 
407  if (ret < 0)
408  return ret;
409 
410  if (ftrace_enabled)
411  update_ftrace_function();
412 
413  /*
414  * Dynamic ops may be freed, we must make sure that all
415  * callers are done before leaving this function.
416  */
417  if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
419 
420  return 0;
421 }
422 
423 static void ftrace_update_pid_func(void)
424 {
425  /* Only do something if we are tracing something */
426  if (ftrace_trace_function == ftrace_stub)
427  return;
428 
429  update_ftrace_function();
430 }
431 
432 #ifdef CONFIG_FUNCTION_PROFILER
433 struct ftrace_profile {
434  struct hlist_node node;
435  unsigned long ip;
436  unsigned long counter;
437 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
438  unsigned long long time;
439  unsigned long long time_squared;
440 #endif
441 };
442 
443 struct ftrace_profile_page {
444  struct ftrace_profile_page *next;
445  unsigned long index;
446  struct ftrace_profile records[];
447 };
448 
449 struct ftrace_profile_stat {
450  atomic_t disabled;
451  struct hlist_head *hash;
452  struct ftrace_profile_page *pages;
453  struct ftrace_profile_page *start;
454  struct tracer_stat stat;
455 };
456 
457 #define PROFILE_RECORDS_SIZE \
458  (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
459 
460 #define PROFILES_PER_PAGE \
461  (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
462 
463 static int ftrace_profile_bits __read_mostly;
464 static int ftrace_profile_enabled __read_mostly;
465 
466 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
467 static DEFINE_MUTEX(ftrace_profile_lock);
468 
469 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
470 
471 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
472 
473 static void *
474 function_stat_next(void *v, int idx)
475 {
476  struct ftrace_profile *rec = v;
477  struct ftrace_profile_page *pg;
478 
479  pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
480 
481  again:
482  if (idx != 0)
483  rec++;
484 
485  if ((void *)rec >= (void *)&pg->records[pg->index]) {
486  pg = pg->next;
487  if (!pg)
488  return NULL;
489  rec = &pg->records[0];
490  if (!rec->counter)
491  goto again;
492  }
493 
494  return rec;
495 }
496 
497 static void *function_stat_start(struct tracer_stat *trace)
498 {
499  struct ftrace_profile_stat *stat =
500  container_of(trace, struct ftrace_profile_stat, stat);
501 
502  if (!stat || !stat->start)
503  return NULL;
504 
505  return function_stat_next(&stat->start->records[0], 0);
506 }
507 
508 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
509 /* function graph compares on total time */
510 static int function_stat_cmp(void *p1, void *p2)
511 {
512  struct ftrace_profile *a = p1;
513  struct ftrace_profile *b = p2;
514 
515  if (a->time < b->time)
516  return -1;
517  if (a->time > b->time)
518  return 1;
519  else
520  return 0;
521 }
522 #else
523 /* not function graph compares against hits */
524 static int function_stat_cmp(void *p1, void *p2)
525 {
526  struct ftrace_profile *a = p1;
527  struct ftrace_profile *b = p2;
528 
529  if (a->counter < b->counter)
530  return -1;
531  if (a->counter > b->counter)
532  return 1;
533  else
534  return 0;
535 }
536 #endif
537 
538 static int function_stat_headers(struct seq_file *m)
539 {
540 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
541  seq_printf(m, " Function "
542  "Hit Time Avg s^2\n"
543  " -------- "
544  "--- ---- --- ---\n");
545 #else
546  seq_printf(m, " Function Hit\n"
547  " -------- ---\n");
548 #endif
549  return 0;
550 }
551 
552 static int function_stat_show(struct seq_file *m, void *v)
553 {
554  struct ftrace_profile *rec = v;
555  char str[KSYM_SYMBOL_LEN];
556  int ret = 0;
557 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
558  static struct trace_seq s;
559  unsigned long long avg;
560  unsigned long long stddev;
561 #endif
562  mutex_lock(&ftrace_profile_lock);
563 
564  /* we raced with function_profile_reset() */
565  if (unlikely(rec->counter == 0)) {
566  ret = -EBUSY;
567  goto out;
568  }
569 
570  kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
571  seq_printf(m, " %-30.30s %10lu", str, rec->counter);
572 
573 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
574  seq_printf(m, " ");
575  avg = rec->time;
576  do_div(avg, rec->counter);
577 
578  /* Sample standard deviation (s^2) */
579  if (rec->counter <= 1)
580  stddev = 0;
581  else {
582  stddev = rec->time_squared - rec->counter * avg * avg;
583  /*
584  * Divide only 1000 for ns^2 -> us^2 conversion.
585  * trace_print_graph_duration will divide 1000 again.
586  */
587  do_div(stddev, (rec->counter - 1) * 1000);
588  }
589 
590  trace_seq_init(&s);
591  trace_print_graph_duration(rec->time, &s);
592  trace_seq_puts(&s, " ");
594  trace_seq_puts(&s, " ");
595  trace_print_graph_duration(stddev, &s);
596  trace_print_seq(m, &s);
597 #endif
598  seq_putc(m, '\n');
599 out:
600  mutex_unlock(&ftrace_profile_lock);
601 
602  return ret;
603 }
604 
605 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
606 {
607  struct ftrace_profile_page *pg;
608 
609  pg = stat->pages = stat->start;
610 
611  while (pg) {
612  memset(pg->records, 0, PROFILE_RECORDS_SIZE);
613  pg->index = 0;
614  pg = pg->next;
615  }
616 
617  memset(stat->hash, 0,
618  FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
619 }
620 
621 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
622 {
623  struct ftrace_profile_page *pg;
624  int functions;
625  int pages;
626  int i;
627 
628  /* If we already allocated, do nothing */
629  if (stat->pages)
630  return 0;
631 
632  stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
633  if (!stat->pages)
634  return -ENOMEM;
635 
636 #ifdef CONFIG_DYNAMIC_FTRACE
637  functions = ftrace_update_tot_cnt;
638 #else
639  /*
640  * We do not know the number of functions that exist because
641  * dynamic tracing is what counts them. With past experience
642  * we have around 20K functions. That should be more than enough.
643  * It is highly unlikely we will execute every function in
644  * the kernel.
645  */
646  functions = 20000;
647 #endif
648 
649  pg = stat->start = stat->pages;
650 
651  pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
652 
653  for (i = 0; i < pages; i++) {
654  pg->next = (void *)get_zeroed_page(GFP_KERNEL);
655  if (!pg->next)
656  goto out_free;
657  pg = pg->next;
658  }
659 
660  return 0;
661 
662  out_free:
663  pg = stat->start;
664  while (pg) {
665  unsigned long tmp = (unsigned long)pg;
666 
667  pg = pg->next;
668  free_page(tmp);
669  }
670 
671  free_page((unsigned long)stat->pages);
672  stat->pages = NULL;
673  stat->start = NULL;
674 
675  return -ENOMEM;
676 }
677 
678 static int ftrace_profile_init_cpu(int cpu)
679 {
680  struct ftrace_profile_stat *stat;
681  int size;
682 
683  stat = &per_cpu(ftrace_profile_stats, cpu);
684 
685  if (stat->hash) {
686  /* If the profile is already created, simply reset it */
687  ftrace_profile_reset(stat);
688  return 0;
689  }
690 
691  /*
692  * We are profiling all functions, but usually only a few thousand
693  * functions are hit. We'll make a hash of 1024 items.
694  */
695  size = FTRACE_PROFILE_HASH_SIZE;
696 
697  stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
698 
699  if (!stat->hash)
700  return -ENOMEM;
701 
702  if (!ftrace_profile_bits) {
703  size--;
704 
705  for (; size; size >>= 1)
706  ftrace_profile_bits++;
707  }
708 
709  /* Preallocate the function profiling pages */
710  if (ftrace_profile_pages_init(stat) < 0) {
711  kfree(stat->hash);
712  stat->hash = NULL;
713  return -ENOMEM;
714  }
715 
716  return 0;
717 }
718 
719 static int ftrace_profile_init(void)
720 {
721  int cpu;
722  int ret = 0;
723 
724  for_each_online_cpu(cpu) {
725  ret = ftrace_profile_init_cpu(cpu);
726  if (ret)
727  break;
728  }
729 
730  return ret;
731 }
732 
733 /* interrupts must be disabled */
734 static struct ftrace_profile *
735 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
736 {
737  struct ftrace_profile *rec;
738  struct hlist_head *hhd;
739  struct hlist_node *n;
740  unsigned long key;
741 
742  key = hash_long(ip, ftrace_profile_bits);
743  hhd = &stat->hash[key];
744 
745  if (hlist_empty(hhd))
746  return NULL;
747 
748  hlist_for_each_entry_rcu(rec, n, hhd, node) {
749  if (rec->ip == ip)
750  return rec;
751  }
752 
753  return NULL;
754 }
755 
756 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
757  struct ftrace_profile *rec)
758 {
759  unsigned long key;
760 
761  key = hash_long(rec->ip, ftrace_profile_bits);
762  hlist_add_head_rcu(&rec->node, &stat->hash[key]);
763 }
764 
765 /*
766  * The memory is already allocated, this simply finds a new record to use.
767  */
768 static struct ftrace_profile *
769 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
770 {
771  struct ftrace_profile *rec = NULL;
772 
773  /* prevent recursion (from NMIs) */
774  if (atomic_inc_return(&stat->disabled) != 1)
775  goto out;
776 
777  /*
778  * Try to find the function again since an NMI
779  * could have added it
780  */
781  rec = ftrace_find_profiled_func(stat, ip);
782  if (rec)
783  goto out;
784 
785  if (stat->pages->index == PROFILES_PER_PAGE) {
786  if (!stat->pages->next)
787  goto out;
788  stat->pages = stat->pages->next;
789  }
790 
791  rec = &stat->pages->records[stat->pages->index++];
792  rec->ip = ip;
793  ftrace_add_profile(stat, rec);
794 
795  out:
796  atomic_dec(&stat->disabled);
797 
798  return rec;
799 }
800 
801 static void
802 function_profile_call(unsigned long ip, unsigned long parent_ip,
803  struct ftrace_ops *ops, struct pt_regs *regs)
804 {
805  struct ftrace_profile_stat *stat;
806  struct ftrace_profile *rec;
807  unsigned long flags;
808 
809  if (!ftrace_profile_enabled)
810  return;
811 
812  local_irq_save(flags);
813 
814  stat = &__get_cpu_var(ftrace_profile_stats);
815  if (!stat->hash || !ftrace_profile_enabled)
816  goto out;
817 
818  rec = ftrace_find_profiled_func(stat, ip);
819  if (!rec) {
820  rec = ftrace_profile_alloc(stat, ip);
821  if (!rec)
822  goto out;
823  }
824 
825  rec->counter++;
826  out:
827  local_irq_restore(flags);
828 }
829 
830 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
831 static int profile_graph_entry(struct ftrace_graph_ent *trace)
832 {
833  function_profile_call(trace->func, 0, NULL, NULL);
834  return 1;
835 }
836 
837 static void profile_graph_return(struct ftrace_graph_ret *trace)
838 {
839  struct ftrace_profile_stat *stat;
840  unsigned long long calltime;
841  struct ftrace_profile *rec;
842  unsigned long flags;
843 
844  local_irq_save(flags);
845  stat = &__get_cpu_var(ftrace_profile_stats);
846  if (!stat->hash || !ftrace_profile_enabled)
847  goto out;
848 
849  /* If the calltime was zero'd ignore it */
850  if (!trace->calltime)
851  goto out;
852 
853  calltime = trace->rettime - trace->calltime;
854 
856  int index;
857 
858  index = trace->depth;
859 
860  /* Append this call time to the parent time to subtract */
861  if (index)
862  current->ret_stack[index - 1].subtime += calltime;
863 
864  if (current->ret_stack[index].subtime < calltime)
865  calltime -= current->ret_stack[index].subtime;
866  else
867  calltime = 0;
868  }
869 
870  rec = ftrace_find_profiled_func(stat, trace->func);
871  if (rec) {
872  rec->time += calltime;
873  rec->time_squared += calltime * calltime;
874  }
875 
876  out:
877  local_irq_restore(flags);
878 }
879 
880 static int register_ftrace_profiler(void)
881 {
882  return register_ftrace_graph(&profile_graph_return,
883  &profile_graph_entry);
884 }
885 
886 static void unregister_ftrace_profiler(void)
887 {
888  unregister_ftrace_graph();
889 }
890 #else
891 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
892  .func = function_profile_call,
893  .flags = FTRACE_OPS_FL_RECURSION_SAFE,
894 };
895 
896 static int register_ftrace_profiler(void)
897 {
898  return register_ftrace_function(&ftrace_profile_ops);
899 }
900 
901 static void unregister_ftrace_profiler(void)
902 {
903  unregister_ftrace_function(&ftrace_profile_ops);
904 }
905 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
906 
907 static ssize_t
908 ftrace_profile_write(struct file *filp, const char __user *ubuf,
909  size_t cnt, loff_t *ppos)
910 {
911  unsigned long val;
912  int ret;
913 
914  ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
915  if (ret)
916  return ret;
917 
918  val = !!val;
919 
920  mutex_lock(&ftrace_profile_lock);
921  if (ftrace_profile_enabled ^ val) {
922  if (val) {
923  ret = ftrace_profile_init();
924  if (ret < 0) {
925  cnt = ret;
926  goto out;
927  }
928 
929  ret = register_ftrace_profiler();
930  if (ret < 0) {
931  cnt = ret;
932  goto out;
933  }
934  ftrace_profile_enabled = 1;
935  } else {
936  ftrace_profile_enabled = 0;
937  /*
938  * unregister_ftrace_profiler calls stop_machine
939  * so this acts like an synchronize_sched.
940  */
941  unregister_ftrace_profiler();
942  }
943  }
944  out:
945  mutex_unlock(&ftrace_profile_lock);
946 
947  *ppos += cnt;
948 
949  return cnt;
950 }
951 
952 static ssize_t
953 ftrace_profile_read(struct file *filp, char __user *ubuf,
954  size_t cnt, loff_t *ppos)
955 {
956  char buf[64]; /* big enough to hold a number */
957  int r;
958 
959  r = sprintf(buf, "%u\n", ftrace_profile_enabled);
960  return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
961 }
962 
963 static const struct file_operations ftrace_profile_fops = {
965  .read = ftrace_profile_read,
966  .write = ftrace_profile_write,
967  .llseek = default_llseek,
968 };
969 
970 /* used to initialize the real stat files */
971 static struct tracer_stat function_stats __initdata = {
972  .name = "functions",
973  .stat_start = function_stat_start,
974  .stat_next = function_stat_next,
975  .stat_cmp = function_stat_cmp,
976  .stat_headers = function_stat_headers,
977  .stat_show = function_stat_show
978 };
979 
980 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
981 {
982  struct ftrace_profile_stat *stat;
983  struct dentry *entry;
984  char *name;
985  int ret;
986  int cpu;
987 
988  for_each_possible_cpu(cpu) {
989  stat = &per_cpu(ftrace_profile_stats, cpu);
990 
991  /* allocate enough for function name + cpu number */
992  name = kmalloc(32, GFP_KERNEL);
993  if (!name) {
994  /*
995  * The files created are permanent, if something happens
996  * we still do not free memory.
997  */
998  WARN(1,
999  "Could not allocate stat file for cpu %d\n",
1000  cpu);
1001  return;
1002  }
1003  stat->stat = function_stats;
1004  snprintf(name, 32, "function%d", cpu);
1005  stat->stat.name = name;
1006  ret = register_stat_tracer(&stat->stat);
1007  if (ret) {
1008  WARN(1,
1009  "Could not register function stat for cpu %d\n",
1010  cpu);
1011  kfree(name);
1012  return;
1013  }
1014  }
1015 
1016  entry = debugfs_create_file("function_profile_enabled", 0644,
1017  d_tracer, NULL, &ftrace_profile_fops);
1018  if (!entry)
1019  pr_warning("Could not create debugfs "
1020  "'function_profile_enabled' entry\n");
1021 }
1022 
1023 #else /* CONFIG_FUNCTION_PROFILER */
1024 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
1025 {
1026 }
1027 #endif /* CONFIG_FUNCTION_PROFILER */
1028 
1029 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
1030 
1031 #ifdef CONFIG_DYNAMIC_FTRACE
1032 
1033 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
1034 # error Dynamic ftrace depends on MCOUNT_RECORD
1035 #endif
1036 
1037 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
1038 
1039 struct ftrace_func_probe {
1040  struct hlist_node node;
1041  struct ftrace_probe_ops *ops;
1042  unsigned long flags;
1043  unsigned long ip;
1044  void *data;
1045  struct rcu_head rcu;
1046 };
1047 
1048 struct ftrace_func_entry {
1049  struct hlist_node hlist;
1050  unsigned long ip;
1051 };
1052 
1053 struct ftrace_hash {
1054  unsigned long size_bits;
1055  struct hlist_head *buckets;
1056  unsigned long count;
1057  struct rcu_head rcu;
1058 };
1059 
1060 /*
1061  * We make these constant because no one should touch them,
1062  * but they are used as the default "empty hash", to avoid allocating
1063  * it all the time. These are in a read only section such that if
1064  * anyone does try to modify it, it will cause an exception.
1065  */
1066 static const struct hlist_head empty_buckets[1];
1067 static const struct ftrace_hash empty_hash = {
1068  .buckets = (struct hlist_head *)empty_buckets,
1069 };
1070 #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
1071 
1072 static struct ftrace_ops global_ops = {
1073  .func = ftrace_stub,
1074  .notrace_hash = EMPTY_HASH,
1075  .filter_hash = EMPTY_HASH,
1076  .flags = FTRACE_OPS_FL_RECURSION_SAFE,
1077 };
1078 
1079 static DEFINE_MUTEX(ftrace_regex_lock);
1080 
1081 struct ftrace_page {
1082  struct ftrace_page *next;
1083  struct dyn_ftrace *records;
1084  int index;
1085  int size;
1086 };
1087 
1088 static struct ftrace_page *ftrace_new_pgs;
1089 
1090 #define ENTRY_SIZE sizeof(struct dyn_ftrace)
1091 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1092 
1093 /* estimate from running different kernels */
1094 #define NR_TO_INIT 10000
1095 
1096 static struct ftrace_page *ftrace_pages_start;
1097 static struct ftrace_page *ftrace_pages;
1098 
1099 static bool ftrace_hash_empty(struct ftrace_hash *hash)
1100 {
1101  return !hash || !hash->count;
1102 }
1103 
1104 static struct ftrace_func_entry *
1105 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1106 {
1107  unsigned long key;
1108  struct ftrace_func_entry *entry;
1109  struct hlist_head *hhd;
1110  struct hlist_node *n;
1111 
1112  if (ftrace_hash_empty(hash))
1113  return NULL;
1114 
1115  if (hash->size_bits > 0)
1116  key = hash_long(ip, hash->size_bits);
1117  else
1118  key = 0;
1119 
1120  hhd = &hash->buckets[key];
1121 
1122  hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
1123  if (entry->ip == ip)
1124  return entry;
1125  }
1126  return NULL;
1127 }
1128 
1129 static void __add_hash_entry(struct ftrace_hash *hash,
1130  struct ftrace_func_entry *entry)
1131 {
1132  struct hlist_head *hhd;
1133  unsigned long key;
1134 
1135  if (hash->size_bits)
1136  key = hash_long(entry->ip, hash->size_bits);
1137  else
1138  key = 0;
1139 
1140  hhd = &hash->buckets[key];
1141  hlist_add_head(&entry->hlist, hhd);
1142  hash->count++;
1143 }
1144 
1145 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1146 {
1147  struct ftrace_func_entry *entry;
1148 
1149  entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1150  if (!entry)
1151  return -ENOMEM;
1152 
1153  entry->ip = ip;
1154  __add_hash_entry(hash, entry);
1155 
1156  return 0;
1157 }
1158 
1159 static void
1160 free_hash_entry(struct ftrace_hash *hash,
1161  struct ftrace_func_entry *entry)
1162 {
1163  hlist_del(&entry->hlist);
1164  kfree(entry);
1165  hash->count--;
1166 }
1167 
1168 static void
1169 remove_hash_entry(struct ftrace_hash *hash,
1170  struct ftrace_func_entry *entry)
1171 {
1172  hlist_del(&entry->hlist);
1173  hash->count--;
1174 }
1175 
1176 static void ftrace_hash_clear(struct ftrace_hash *hash)
1177 {
1178  struct hlist_head *hhd;
1179  struct hlist_node *tp, *tn;
1180  struct ftrace_func_entry *entry;
1181  int size = 1 << hash->size_bits;
1182  int i;
1183 
1184  if (!hash->count)
1185  return;
1186 
1187  for (i = 0; i < size; i++) {
1188  hhd = &hash->buckets[i];
1189  hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
1190  free_hash_entry(hash, entry);
1191  }
1192  FTRACE_WARN_ON(hash->count);
1193 }
1194 
1195 static void free_ftrace_hash(struct ftrace_hash *hash)
1196 {
1197  if (!hash || hash == EMPTY_HASH)
1198  return;
1199  ftrace_hash_clear(hash);
1200  kfree(hash->buckets);
1201  kfree(hash);
1202 }
1203 
1204 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1205 {
1206  struct ftrace_hash *hash;
1207 
1208  hash = container_of(rcu, struct ftrace_hash, rcu);
1209  free_ftrace_hash(hash);
1210 }
1211 
1212 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1213 {
1214  if (!hash || hash == EMPTY_HASH)
1215  return;
1216  call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1217 }
1218 
1219 void ftrace_free_filter(struct ftrace_ops *ops)
1220 {
1221  free_ftrace_hash(ops->filter_hash);
1222  free_ftrace_hash(ops->notrace_hash);
1223 }
1224 
1225 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1226 {
1227  struct ftrace_hash *hash;
1228  int size;
1229 
1230  hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1231  if (!hash)
1232  return NULL;
1233 
1234  size = 1 << size_bits;
1235  hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1236 
1237  if (!hash->buckets) {
1238  kfree(hash);
1239  return NULL;
1240  }
1241 
1242  hash->size_bits = size_bits;
1243 
1244  return hash;
1245 }
1246 
1247 static struct ftrace_hash *
1248 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1249 {
1250  struct ftrace_func_entry *entry;
1251  struct ftrace_hash *new_hash;
1252  struct hlist_node *tp;
1253  int size;
1254  int ret;
1255  int i;
1256 
1257  new_hash = alloc_ftrace_hash(size_bits);
1258  if (!new_hash)
1259  return NULL;
1260 
1261  /* Empty hash? */
1262  if (ftrace_hash_empty(hash))
1263  return new_hash;
1264 
1265  size = 1 << hash->size_bits;
1266  for (i = 0; i < size; i++) {
1267  hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
1268  ret = add_hash_entry(new_hash, entry->ip);
1269  if (ret < 0)
1270  goto free_hash;
1271  }
1272  }
1273 
1274  FTRACE_WARN_ON(new_hash->count != hash->count);
1275 
1276  return new_hash;
1277 
1278  free_hash:
1279  free_ftrace_hash(new_hash);
1280  return NULL;
1281 }
1282 
1283 static void
1284 ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
1285 static void
1286 ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
1287 
1288 static int
1289 ftrace_hash_move(struct ftrace_ops *ops, int enable,
1290  struct ftrace_hash **dst, struct ftrace_hash *src)
1291 {
1292  struct ftrace_func_entry *entry;
1293  struct hlist_node *tp, *tn;
1294  struct hlist_head *hhd;
1295  struct ftrace_hash *old_hash;
1296  struct ftrace_hash *new_hash;
1297  unsigned long key;
1298  int size = src->count;
1299  int bits = 0;
1300  int ret;
1301  int i;
1302 
1303  /*
1304  * Remove the current set, update the hash and add
1305  * them back.
1306  */
1307  ftrace_hash_rec_disable(ops, enable);
1308 
1309  /*
1310  * If the new source is empty, just free dst and assign it
1311  * the empty_hash.
1312  */
1313  if (!src->count) {
1314  free_ftrace_hash_rcu(*dst);
1315  rcu_assign_pointer(*dst, EMPTY_HASH);
1316  /* still need to update the function records */
1317  ret = 0;
1318  goto out;
1319  }
1320 
1321  /*
1322  * Make the hash size about 1/2 the # found
1323  */
1324  for (size /= 2; size; size >>= 1)
1325  bits++;
1326 
1327  /* Don't allocate too much */
1328  if (bits > FTRACE_HASH_MAX_BITS)
1329  bits = FTRACE_HASH_MAX_BITS;
1330 
1331  ret = -ENOMEM;
1332  new_hash = alloc_ftrace_hash(bits);
1333  if (!new_hash)
1334  goto out;
1335 
1336  size = 1 << src->size_bits;
1337  for (i = 0; i < size; i++) {
1338  hhd = &src->buckets[i];
1339  hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
1340  if (bits > 0)
1341  key = hash_long(entry->ip, bits);
1342  else
1343  key = 0;
1344  remove_hash_entry(src, entry);
1345  __add_hash_entry(new_hash, entry);
1346  }
1347  }
1348 
1349  old_hash = *dst;
1350  rcu_assign_pointer(*dst, new_hash);
1351  free_ftrace_hash_rcu(old_hash);
1352 
1353  ret = 0;
1354  out:
1355  /*
1356  * Enable regardless of ret:
1357  * On success, we enable the new hash.
1358  * On failure, we re-enable the original hash.
1359  */
1360  ftrace_hash_rec_enable(ops, enable);
1361 
1362  return ret;
1363 }
1364 
1365 /*
1366  * Test the hashes for this ops to see if we want to call
1367  * the ops->func or not.
1368  *
1369  * It's a match if the ip is in the ops->filter_hash or
1370  * the filter_hash does not exist or is empty,
1371  * AND
1372  * the ip is not in the ops->notrace_hash.
1373  *
1374  * This needs to be called with preemption disabled as
1375  * the hashes are freed with call_rcu_sched().
1376  */
1377 static int
1378 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
1379 {
1380  struct ftrace_hash *filter_hash;
1381  struct ftrace_hash *notrace_hash;
1382  int ret;
1383 
1384  filter_hash = rcu_dereference_raw(ops->filter_hash);
1385  notrace_hash = rcu_dereference_raw(ops->notrace_hash);
1386 
1387  if ((ftrace_hash_empty(filter_hash) ||
1388  ftrace_lookup_ip(filter_hash, ip)) &&
1389  (ftrace_hash_empty(notrace_hash) ||
1390  !ftrace_lookup_ip(notrace_hash, ip)))
1391  ret = 1;
1392  else
1393  ret = 0;
1394 
1395  return ret;
1396 }
1397 
1398 /*
1399  * This is a double for. Do not use 'break' to break out of the loop,
1400  * you must use a goto.
1401  */
1402 #define do_for_each_ftrace_rec(pg, rec) \
1403  for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1404  int _____i; \
1405  for (_____i = 0; _____i < pg->index; _____i++) { \
1406  rec = &pg->records[_____i];
1407 
1408 #define while_for_each_ftrace_rec() \
1409  } \
1410  }
1411 
1412 
1413 static int ftrace_cmp_recs(const void *a, const void *b)
1414 {
1415  const struct dyn_ftrace *key = a;
1416  const struct dyn_ftrace *rec = b;
1417 
1418  if (key->flags < rec->ip)
1419  return -1;
1420  if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1421  return 1;
1422  return 0;
1423 }
1424 
1425 static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1426 {
1427  struct ftrace_page *pg;
1428  struct dyn_ftrace *rec;
1429  struct dyn_ftrace key;
1430 
1431  key.ip = start;
1432  key.flags = end; /* overload flags, as it is unsigned long */
1433 
1434  for (pg = ftrace_pages_start; pg; pg = pg->next) {
1435  if (end < pg->records[0].ip ||
1436  start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1437  continue;
1438  rec = bsearch(&key, pg->records, pg->index,
1439  sizeof(struct dyn_ftrace),
1440  ftrace_cmp_recs);
1441  if (rec)
1442  return rec->ip;
1443  }
1444 
1445  return 0;
1446 }
1447 
1457 unsigned long ftrace_location(unsigned long ip)
1458 {
1459  return ftrace_location_range(ip, ip);
1460 }
1461 
1472 int ftrace_text_reserved(void *start, void *end)
1473 {
1474  unsigned long ret;
1475 
1476  ret = ftrace_location_range((unsigned long)start,
1477  (unsigned long)end);
1478 
1479  return (int)!!ret;
1480 }
1481 
1482 static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1483  int filter_hash,
1484  bool inc)
1485 {
1486  struct ftrace_hash *hash;
1487  struct ftrace_hash *other_hash;
1488  struct ftrace_page *pg;
1489  struct dyn_ftrace *rec;
1490  int count = 0;
1491  int all = 0;
1492 
1493  /* Only update if the ops has been registered */
1494  if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1495  return;
1496 
1497  /*
1498  * In the filter_hash case:
1499  * If the count is zero, we update all records.
1500  * Otherwise we just update the items in the hash.
1501  *
1502  * In the notrace_hash case:
1503  * We enable the update in the hash.
1504  * As disabling notrace means enabling the tracing,
1505  * and enabling notrace means disabling, the inc variable
1506  * gets inversed.
1507  */
1508  if (filter_hash) {
1509  hash = ops->filter_hash;
1510  other_hash = ops->notrace_hash;
1511  if (ftrace_hash_empty(hash))
1512  all = 1;
1513  } else {
1514  inc = !inc;
1515  hash = ops->notrace_hash;
1516  other_hash = ops->filter_hash;
1517  /*
1518  * If the notrace hash has no items,
1519  * then there's nothing to do.
1520  */
1521  if (ftrace_hash_empty(hash))
1522  return;
1523  }
1524 
1525  do_for_each_ftrace_rec(pg, rec) {
1526  int in_other_hash = 0;
1527  int in_hash = 0;
1528  int match = 0;
1529 
1530  if (all) {
1531  /*
1532  * Only the filter_hash affects all records.
1533  * Update if the record is not in the notrace hash.
1534  */
1535  if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1536  match = 1;
1537  } else {
1538  in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1539  in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1540 
1541  /*
1542  *
1543  */
1544  if (filter_hash && in_hash && !in_other_hash)
1545  match = 1;
1546  else if (!filter_hash && in_hash &&
1547  (in_other_hash || ftrace_hash_empty(other_hash)))
1548  match = 1;
1549  }
1550  if (!match)
1551  continue;
1552 
1553  if (inc) {
1554  rec->flags++;
1555  if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1556  return;
1557  /*
1558  * If any ops wants regs saved for this function
1559  * then all ops will get saved regs.
1560  */
1561  if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1562  rec->flags |= FTRACE_FL_REGS;
1563  } else {
1564  if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1565  return;
1566  rec->flags--;
1567  }
1568  count++;
1569  /* Shortcut, if we handled all records, we are done. */
1570  if (!all && count == hash->count)
1571  return;
1572  } while_for_each_ftrace_rec();
1573 }
1574 
1575 static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1576  int filter_hash)
1577 {
1578  __ftrace_hash_rec_update(ops, filter_hash, 0);
1579 }
1580 
1581 static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1582  int filter_hash)
1583 {
1584  __ftrace_hash_rec_update(ops, filter_hash, 1);
1585 }
1586 
1587 static void print_ip_ins(const char *fmt, unsigned char *p)
1588 {
1589  int i;
1590 
1591  printk(KERN_CONT "%s", fmt);
1592 
1593  for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1594  printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1595 }
1596 
1609 void ftrace_bug(int failed, unsigned long ip)
1610 {
1611  switch (failed) {
1612  case -EFAULT:
1614  pr_info("ftrace faulted on modifying ");
1615  print_ip_sym(ip);
1616  break;
1617  case -EINVAL:
1619  pr_info("ftrace failed to modify ");
1620  print_ip_sym(ip);
1621  print_ip_ins(" actual: ", (unsigned char *)ip);
1622  printk(KERN_CONT "\n");
1623  break;
1624  case -EPERM:
1626  pr_info("ftrace faulted on writing ");
1627  print_ip_sym(ip);
1628  break;
1629  default:
1631  pr_info("ftrace faulted on unknown error ");
1632  print_ip_sym(ip);
1633  }
1634 }
1635 
1636 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
1637 {
1638  unsigned long flag = 0UL;
1639 
1640  /*
1641  * If we are updating calls:
1642  *
1643  * If the record has a ref count, then we need to enable it
1644  * because someone is using it.
1645  *
1646  * Otherwise we make sure its disabled.
1647  *
1648  * If we are disabling calls, then disable all records that
1649  * are enabled.
1650  */
1651  if (enable && (rec->flags & ~FTRACE_FL_MASK))
1652  flag = FTRACE_FL_ENABLED;
1653 
1654  /*
1655  * If enabling and the REGS flag does not match the REGS_EN, then
1656  * do not ignore this record. Set flags to fail the compare against
1657  * ENABLED.
1658  */
1659  if (flag &&
1660  (!(rec->flags & FTRACE_FL_REGS) != !(rec->flags & FTRACE_FL_REGS_EN)))
1661  flag |= FTRACE_FL_REGS;
1662 
1663  /* If the state of this record hasn't changed, then do nothing */
1664  if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1665  return FTRACE_UPDATE_IGNORE;
1666 
1667  if (flag) {
1668  /* Save off if rec is being enabled (for return value) */
1669  flag ^= rec->flags & FTRACE_FL_ENABLED;
1670 
1671  if (update) {
1672  rec->flags |= FTRACE_FL_ENABLED;
1673  if (flag & FTRACE_FL_REGS) {
1674  if (rec->flags & FTRACE_FL_REGS)
1675  rec->flags |= FTRACE_FL_REGS_EN;
1676  else
1677  rec->flags &= ~FTRACE_FL_REGS_EN;
1678  }
1679  }
1680 
1681  /*
1682  * If this record is being updated from a nop, then
1683  * return UPDATE_MAKE_CALL.
1684  * Otherwise, if the EN flag is set, then return
1685  * UPDATE_MODIFY_CALL_REGS to tell the caller to convert
1686  * from the non-save regs, to a save regs function.
1687  * Otherwise,
1688  * return UPDATE_MODIFY_CALL to tell the caller to convert
1689  * from the save regs, to a non-save regs function.
1690  */
1691  if (flag & FTRACE_FL_ENABLED)
1692  return FTRACE_UPDATE_MAKE_CALL;
1693  else if (rec->flags & FTRACE_FL_REGS_EN)
1694  return FTRACE_UPDATE_MODIFY_CALL_REGS;
1695  else
1696  return FTRACE_UPDATE_MODIFY_CALL;
1697  }
1698 
1699  if (update) {
1700  /* If there's no more users, clear all flags */
1701  if (!(rec->flags & ~FTRACE_FL_MASK))
1702  rec->flags = 0;
1703  else
1704  /* Just disable the record (keep REGS state) */
1705  rec->flags &= ~FTRACE_FL_ENABLED;
1706  }
1707 
1708  return FTRACE_UPDATE_MAKE_NOP;
1709 }
1710 
1719 int ftrace_update_record(struct dyn_ftrace *rec, int enable)
1720 {
1721  return ftrace_check_record(rec, enable, 1);
1722 }
1723 
1733 int ftrace_test_record(struct dyn_ftrace *rec, int enable)
1734 {
1735  return ftrace_check_record(rec, enable, 0);
1736 }
1737 
1738 static int
1739 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1740 {
1741  unsigned long ftrace_old_addr;
1742  unsigned long ftrace_addr;
1743  int ret;
1744 
1745  ret = ftrace_update_record(rec, enable);
1746 
1747  if (rec->flags & FTRACE_FL_REGS)
1748  ftrace_addr = (unsigned long)FTRACE_REGS_ADDR;
1749  else
1750  ftrace_addr = (unsigned long)FTRACE_ADDR;
1751 
1752  switch (ret) {
1753  case FTRACE_UPDATE_IGNORE:
1754  return 0;
1755 
1756  case FTRACE_UPDATE_MAKE_CALL:
1757  return ftrace_make_call(rec, ftrace_addr);
1758 
1759  case FTRACE_UPDATE_MAKE_NOP:
1760  return ftrace_make_nop(NULL, rec, ftrace_addr);
1761 
1762  case FTRACE_UPDATE_MODIFY_CALL_REGS:
1763  case FTRACE_UPDATE_MODIFY_CALL:
1764  if (rec->flags & FTRACE_FL_REGS)
1765  ftrace_old_addr = (unsigned long)FTRACE_ADDR;
1766  else
1767  ftrace_old_addr = (unsigned long)FTRACE_REGS_ADDR;
1768 
1769  return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
1770  }
1771 
1772  return -1; /* unknow ftrace bug */
1773 }
1774 
1775 void __weak ftrace_replace_code(int enable)
1776 {
1777  struct dyn_ftrace *rec;
1778  struct ftrace_page *pg;
1779  int failed;
1780 
1781  if (unlikely(ftrace_disabled))
1782  return;
1783 
1784  do_for_each_ftrace_rec(pg, rec) {
1785  failed = __ftrace_replace_code(rec, enable);
1786  if (failed) {
1787  ftrace_bug(failed, rec->ip);
1788  /* Stop processing */
1789  return;
1790  }
1791  } while_for_each_ftrace_rec();
1792 }
1793 
1794 struct ftrace_rec_iter {
1795  struct ftrace_page *pg;
1796  int index;
1797 };
1798 
1808 struct ftrace_rec_iter *ftrace_rec_iter_start(void)
1809 {
1810  /*
1811  * We only use a single iterator.
1812  * Protected by the ftrace_lock mutex.
1813  */
1814  static struct ftrace_rec_iter ftrace_rec_iter;
1815  struct ftrace_rec_iter *iter = &ftrace_rec_iter;
1816 
1817  iter->pg = ftrace_pages_start;
1818  iter->index = 0;
1819 
1820  /* Could have empty pages */
1821  while (iter->pg && !iter->pg->index)
1822  iter->pg = iter->pg->next;
1823 
1824  if (!iter->pg)
1825  return NULL;
1826 
1827  return iter;
1828 }
1829 
1836 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
1837 {
1838  iter->index++;
1839 
1840  if (iter->index >= iter->pg->index) {
1841  iter->pg = iter->pg->next;
1842  iter->index = 0;
1843 
1844  /* Could have empty pages */
1845  while (iter->pg && !iter->pg->index)
1846  iter->pg = iter->pg->next;
1847  }
1848 
1849  if (!iter->pg)
1850  return NULL;
1851 
1852  return iter;
1853 }
1854 
1861 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
1862 {
1863  return &iter->pg->records[iter->index];
1864 }
1865 
1866 static int
1867 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1868 {
1869  unsigned long ip;
1870  int ret;
1871 
1872  ip = rec->ip;
1873 
1874  if (unlikely(ftrace_disabled))
1875  return 0;
1876 
1877  ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1878  if (ret) {
1879  ftrace_bug(ret, ip);
1880  return 0;
1881  }
1882  return 1;
1883 }
1884 
1885 /*
1886  * archs can override this function if they must do something
1887  * before the modifying code is performed.
1888  */
1889 int __weak ftrace_arch_code_modify_prepare(void)
1890 {
1891  return 0;
1892 }
1893 
1894 /*
1895  * archs can override this function if they must do something
1896  * after the modifying code is performed.
1897  */
1898 int __weak ftrace_arch_code_modify_post_process(void)
1899 {
1900  return 0;
1901 }
1902 
1903 void ftrace_modify_all_code(int command)
1904 {
1905  if (command & FTRACE_UPDATE_CALLS)
1906  ftrace_replace_code(1);
1907  else if (command & FTRACE_DISABLE_CALLS)
1908  ftrace_replace_code(0);
1909 
1910  if (command & FTRACE_UPDATE_TRACE_FUNC)
1911  ftrace_update_ftrace_func(ftrace_trace_function);
1912 
1913  if (command & FTRACE_START_FUNC_RET)
1914  ftrace_enable_ftrace_graph_caller();
1915  else if (command & FTRACE_STOP_FUNC_RET)
1916  ftrace_disable_ftrace_graph_caller();
1917 }
1918 
1919 static int __ftrace_modify_code(void *data)
1920 {
1921  int *command = data;
1922 
1923  ftrace_modify_all_code(*command);
1924 
1925  return 0;
1926 }
1927 
1935 void ftrace_run_stop_machine(int command)
1936 {
1937  stop_machine(__ftrace_modify_code, &command, NULL);
1938 }
1939 
1947 void __weak arch_ftrace_update_code(int command)
1948 {
1949  ftrace_run_stop_machine(command);
1950 }
1951 
1952 static void ftrace_run_update_code(int command)
1953 {
1954  int ret;
1955 
1956  ret = ftrace_arch_code_modify_prepare();
1957  FTRACE_WARN_ON(ret);
1958  if (ret)
1959  return;
1960  /*
1961  * Do not call function tracer while we update the code.
1962  * We are in stop machine.
1963  */
1964  function_trace_stop++;
1965 
1966  /*
1967  * By default we use stop_machine() to modify the code.
1968  * But archs can do what ever they want as long as it
1969  * is safe. The stop_machine() is the safest, but also
1970  * produces the most overhead.
1971  */
1972  arch_ftrace_update_code(command);
1973 
1974  function_trace_stop--;
1975 
1976  ret = ftrace_arch_code_modify_post_process();
1977  FTRACE_WARN_ON(ret);
1978 }
1979 
1980 static ftrace_func_t saved_ftrace_func;
1981 static int ftrace_start_up;
1982 static int global_start_up;
1983 
1984 static void ftrace_startup_enable(int command)
1985 {
1986  if (saved_ftrace_func != ftrace_trace_function) {
1987  saved_ftrace_func = ftrace_trace_function;
1988  command |= FTRACE_UPDATE_TRACE_FUNC;
1989  }
1990 
1991  if (!command || !ftrace_enabled)
1992  return;
1993 
1994  ftrace_run_update_code(command);
1995 }
1996 
1997 static int ftrace_startup(struct ftrace_ops *ops, int command)
1998 {
1999  bool hash_enable = true;
2000 
2001  if (unlikely(ftrace_disabled))
2002  return -ENODEV;
2003 
2004  ftrace_start_up++;
2005  command |= FTRACE_UPDATE_CALLS;
2006 
2007  /* ops marked global share the filter hashes */
2008  if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
2009  ops = &global_ops;
2010  /* Don't update hash if global is already set */
2011  if (global_start_up)
2012  hash_enable = false;
2013  global_start_up++;
2014  }
2015 
2016  ops->flags |= FTRACE_OPS_FL_ENABLED;
2017  if (hash_enable)
2018  ftrace_hash_rec_enable(ops, 1);
2019 
2020  ftrace_startup_enable(command);
2021 
2022  return 0;
2023 }
2024 
2025 static void ftrace_shutdown(struct ftrace_ops *ops, int command)
2026 {
2027  bool hash_disable = true;
2028 
2029  if (unlikely(ftrace_disabled))
2030  return;
2031 
2032  ftrace_start_up--;
2033  /*
2034  * Just warn in case of unbalance, no need to kill ftrace, it's not
2035  * critical but the ftrace_call callers may be never nopped again after
2036  * further ftrace uses.
2037  */
2038  WARN_ON_ONCE(ftrace_start_up < 0);
2039 
2040  if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
2041  ops = &global_ops;
2042  global_start_up--;
2043  WARN_ON_ONCE(global_start_up < 0);
2044  /* Don't update hash if global still has users */
2045  if (global_start_up) {
2046  WARN_ON_ONCE(!ftrace_start_up);
2047  hash_disable = false;
2048  }
2049  }
2050 
2051  if (hash_disable)
2052  ftrace_hash_rec_disable(ops, 1);
2053 
2054  if (ops != &global_ops || !global_start_up)
2055  ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2056 
2057  command |= FTRACE_UPDATE_CALLS;
2058 
2059  if (saved_ftrace_func != ftrace_trace_function) {
2060  saved_ftrace_func = ftrace_trace_function;
2061  command |= FTRACE_UPDATE_TRACE_FUNC;
2062  }
2063 
2064  if (!command || !ftrace_enabled)
2065  return;
2066 
2067  ftrace_run_update_code(command);
2068 }
2069 
2070 static void ftrace_startup_sysctl(void)
2071 {
2072  if (unlikely(ftrace_disabled))
2073  return;
2074 
2075  /* Force update next time */
2076  saved_ftrace_func = NULL;
2077  /* ftrace_start_up is true if we want ftrace running */
2078  if (ftrace_start_up)
2079  ftrace_run_update_code(FTRACE_UPDATE_CALLS);
2080 }
2081 
2082 static void ftrace_shutdown_sysctl(void)
2083 {
2084  if (unlikely(ftrace_disabled))
2085  return;
2086 
2087  /* ftrace_start_up is true if ftrace is running */
2088  if (ftrace_start_up)
2089  ftrace_run_update_code(FTRACE_DISABLE_CALLS);
2090 }
2091 
2092 static cycle_t ftrace_update_time;
2093 static unsigned long ftrace_update_cnt;
2094 unsigned long ftrace_update_tot_cnt;
2095 
2096 static int ops_traces_mod(struct ftrace_ops *ops)
2097 {
2098  struct ftrace_hash *hash;
2099 
2100  hash = ops->filter_hash;
2101  return ftrace_hash_empty(hash);
2102 }
2103 
2104 static int ftrace_update_code(struct module *mod)
2105 {
2106  struct ftrace_page *pg;
2107  struct dyn_ftrace *p;
2108  cycle_t start, stop;
2109  unsigned long ref = 0;
2110  int i;
2111 
2112  /*
2113  * When adding a module, we need to check if tracers are
2114  * currently enabled and if they are set to trace all functions.
2115  * If they are, we need to enable the module functions as well
2116  * as update the reference counts for those function records.
2117  */
2118  if (mod) {
2119  struct ftrace_ops *ops;
2120 
2121  for (ops = ftrace_ops_list;
2122  ops != &ftrace_list_end; ops = ops->next) {
2123  if (ops->flags & FTRACE_OPS_FL_ENABLED &&
2124  ops_traces_mod(ops))
2125  ref++;
2126  }
2127  }
2128 
2129  start = ftrace_now(raw_smp_processor_id());
2130  ftrace_update_cnt = 0;
2131 
2132  for (pg = ftrace_new_pgs; pg; pg = pg->next) {
2133 
2134  for (i = 0; i < pg->index; i++) {
2135  /* If something went wrong, bail without enabling anything */
2136  if (unlikely(ftrace_disabled))
2137  return -1;
2138 
2139  p = &pg->records[i];
2140  p->flags = ref;
2141 
2142  /*
2143  * Do the initial record conversion from mcount jump
2144  * to the NOP instructions.
2145  */
2146  if (!ftrace_code_disable(mod, p))
2147  break;
2148 
2149  ftrace_update_cnt++;
2150 
2151  /*
2152  * If the tracing is enabled, go ahead and enable the record.
2153  *
2154  * The reason not to enable the record immediatelly is the
2155  * inherent check of ftrace_make_nop/ftrace_make_call for
2156  * correct previous instructions. Making first the NOP
2157  * conversion puts the module to the correct state, thus
2158  * passing the ftrace_make_call check.
2159  */
2160  if (ftrace_start_up && ref) {
2161  int failed = __ftrace_replace_code(p, 1);
2162  if (failed)
2163  ftrace_bug(failed, p->ip);
2164  }
2165  }
2166  }
2167 
2168  ftrace_new_pgs = NULL;
2169 
2170  stop = ftrace_now(raw_smp_processor_id());
2171  ftrace_update_time = stop - start;
2172  ftrace_update_tot_cnt += ftrace_update_cnt;
2173 
2174  return 0;
2175 }
2176 
2177 static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2178 {
2179  int order;
2180  int cnt;
2181 
2182  if (WARN_ON(!count))
2183  return -EINVAL;
2184 
2185  order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2186 
2187  /*
2188  * We want to fill as much as possible. No more than a page
2189  * may be empty.
2190  */
2191  while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
2192  order--;
2193 
2194  again:
2195  pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2196 
2197  if (!pg->records) {
2198  /* if we can't allocate this size, try something smaller */
2199  if (!order)
2200  return -ENOMEM;
2201  order >>= 1;
2202  goto again;
2203  }
2204 
2205  cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
2206  pg->size = cnt;
2207 
2208  if (cnt > count)
2209  cnt = count;
2210 
2211  return cnt;
2212 }
2213 
2214 static struct ftrace_page *
2215 ftrace_allocate_pages(unsigned long num_to_init)
2216 {
2217  struct ftrace_page *start_pg;
2218  struct ftrace_page *pg;
2219  int order;
2220  int cnt;
2221 
2222  if (!num_to_init)
2223  return 0;
2224 
2225  start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
2226  if (!pg)
2227  return NULL;
2228 
2229  /*
2230  * Try to allocate as much as possible in one continues
2231  * location that fills in all of the space. We want to
2232  * waste as little space as possible.
2233  */
2234  for (;;) {
2235  cnt = ftrace_allocate_records(pg, num_to_init);
2236  if (cnt < 0)
2237  goto free_pages;
2238 
2239  num_to_init -= cnt;
2240  if (!num_to_init)
2241  break;
2242 
2243  pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
2244  if (!pg->next)
2245  goto free_pages;
2246 
2247  pg = pg->next;
2248  }
2249 
2250  return start_pg;
2251 
2252  free_pages:
2253  while (start_pg) {
2254  order = get_count_order(pg->size / ENTRIES_PER_PAGE);
2255  free_pages((unsigned long)pg->records, order);
2256  start_pg = pg->next;
2257  kfree(pg);
2258  pg = start_pg;
2259  }
2260  pr_info("ftrace: FAILED to allocate memory for functions\n");
2261  return NULL;
2262 }
2263 
2264 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
2265 {
2266  int cnt;
2267 
2268  if (!num_to_init) {
2269  pr_info("ftrace: No functions to be traced?\n");
2270  return -1;
2271  }
2272 
2273  cnt = num_to_init / ENTRIES_PER_PAGE;
2274  pr_info("ftrace: allocating %ld entries in %d pages\n",
2275  num_to_init, cnt + 1);
2276 
2277  return 0;
2278 }
2279 
2280 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
2281 
2282 struct ftrace_iterator {
2283  loff_t pos;
2284  loff_t func_pos;
2285  struct ftrace_page *pg;
2286  struct dyn_ftrace *func;
2287  struct ftrace_func_probe *probe;
2288  struct trace_parser parser;
2289  struct ftrace_hash *hash;
2290  struct ftrace_ops *ops;
2291  int hidx;
2292  int idx;
2293  unsigned flags;
2294 };
2295 
2296 static void *
2297 t_hash_next(struct seq_file *m, loff_t *pos)
2298 {
2299  struct ftrace_iterator *iter = m->private;
2300  struct hlist_node *hnd = NULL;
2301  struct hlist_head *hhd;
2302 
2303  (*pos)++;
2304  iter->pos = *pos;
2305 
2306  if (iter->probe)
2307  hnd = &iter->probe->node;
2308  retry:
2309  if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
2310  return NULL;
2311 
2312  hhd = &ftrace_func_hash[iter->hidx];
2313 
2314  if (hlist_empty(hhd)) {
2315  iter->hidx++;
2316  hnd = NULL;
2317  goto retry;
2318  }
2319 
2320  if (!hnd)
2321  hnd = hhd->first;
2322  else {
2323  hnd = hnd->next;
2324  if (!hnd) {
2325  iter->hidx++;
2326  goto retry;
2327  }
2328  }
2329 
2330  if (WARN_ON_ONCE(!hnd))
2331  return NULL;
2332 
2333  iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
2334 
2335  return iter;
2336 }
2337 
2338 static void *t_hash_start(struct seq_file *m, loff_t *pos)
2339 {
2340  struct ftrace_iterator *iter = m->private;
2341  void *p = NULL;
2342  loff_t l;
2343 
2344  if (!(iter->flags & FTRACE_ITER_DO_HASH))
2345  return NULL;
2346 
2347  if (iter->func_pos > *pos)
2348  return NULL;
2349 
2350  iter->hidx = 0;
2351  for (l = 0; l <= (*pos - iter->func_pos); ) {
2352  p = t_hash_next(m, &l);
2353  if (!p)
2354  break;
2355  }
2356  if (!p)
2357  return NULL;
2358 
2359  /* Only set this if we have an item */
2360  iter->flags |= FTRACE_ITER_HASH;
2361 
2362  return iter;
2363 }
2364 
2365 static int
2366 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
2367 {
2368  struct ftrace_func_probe *rec;
2369 
2370  rec = iter->probe;
2371  if (WARN_ON_ONCE(!rec))
2372  return -EIO;
2373 
2374  if (rec->ops->print)
2375  return rec->ops->print(m, rec->ip, rec->ops, rec->data);
2376 
2377  seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
2378 
2379  if (rec->data)
2380  seq_printf(m, ":%p", rec->data);
2381  seq_putc(m, '\n');
2382 
2383  return 0;
2384 }
2385 
2386 static void *
2387 t_next(struct seq_file *m, void *v, loff_t *pos)
2388 {
2389  struct ftrace_iterator *iter = m->private;
2390  struct ftrace_ops *ops = iter->ops;
2391  struct dyn_ftrace *rec = NULL;
2392 
2393  if (unlikely(ftrace_disabled))
2394  return NULL;
2395 
2396  if (iter->flags & FTRACE_ITER_HASH)
2397  return t_hash_next(m, pos);
2398 
2399  (*pos)++;
2400  iter->pos = iter->func_pos = *pos;
2401 
2402  if (iter->flags & FTRACE_ITER_PRINTALL)
2403  return t_hash_start(m, pos);
2404 
2405  retry:
2406  if (iter->idx >= iter->pg->index) {
2407  if (iter->pg->next) {
2408  iter->pg = iter->pg->next;
2409  iter->idx = 0;
2410  goto retry;
2411  }
2412  } else {
2413  rec = &iter->pg->records[iter->idx++];
2414  if (((iter->flags & FTRACE_ITER_FILTER) &&
2415  !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
2416 
2417  ((iter->flags & FTRACE_ITER_NOTRACE) &&
2418  !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
2419 
2420  ((iter->flags & FTRACE_ITER_ENABLED) &&
2421  !(rec->flags & ~FTRACE_FL_MASK))) {
2422 
2423  rec = NULL;
2424  goto retry;
2425  }
2426  }
2427 
2428  if (!rec)
2429  return t_hash_start(m, pos);
2430 
2431  iter->func = rec;
2432 
2433  return iter;
2434 }
2435 
2436 static void reset_iter_read(struct ftrace_iterator *iter)
2437 {
2438  iter->pos = 0;
2439  iter->func_pos = 0;
2440  iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
2441 }
2442 
2443 static void *t_start(struct seq_file *m, loff_t *pos)
2444 {
2445  struct ftrace_iterator *iter = m->private;
2446  struct ftrace_ops *ops = iter->ops;
2447  void *p = NULL;
2448  loff_t l;
2449 
2450  mutex_lock(&ftrace_lock);
2451 
2452  if (unlikely(ftrace_disabled))
2453  return NULL;
2454 
2455  /*
2456  * If an lseek was done, then reset and start from beginning.
2457  */
2458  if (*pos < iter->pos)
2459  reset_iter_read(iter);
2460 
2461  /*
2462  * For set_ftrace_filter reading, if we have the filter
2463  * off, we can short cut and just print out that all
2464  * functions are enabled.
2465  */
2466  if (iter->flags & FTRACE_ITER_FILTER &&
2467  ftrace_hash_empty(ops->filter_hash)) {
2468  if (*pos > 0)
2469  return t_hash_start(m, pos);
2470  iter->flags |= FTRACE_ITER_PRINTALL;
2471  /* reset in case of seek/pread */
2472  iter->flags &= ~FTRACE_ITER_HASH;
2473  return iter;
2474  }
2475 
2476  if (iter->flags & FTRACE_ITER_HASH)
2477  return t_hash_start(m, pos);
2478 
2479  /*
2480  * Unfortunately, we need to restart at ftrace_pages_start
2481  * every time we let go of the ftrace_mutex. This is because
2482  * those pointers can change without the lock.
2483  */
2484  iter->pg = ftrace_pages_start;
2485  iter->idx = 0;
2486  for (l = 0; l <= *pos; ) {
2487  p = t_next(m, p, &l);
2488  if (!p)
2489  break;
2490  }
2491 
2492  if (!p)
2493  return t_hash_start(m, pos);
2494 
2495  return iter;
2496 }
2497 
2498 static void t_stop(struct seq_file *m, void *p)
2499 {
2500  mutex_unlock(&ftrace_lock);
2501 }
2502 
2503 static int t_show(struct seq_file *m, void *v)
2504 {
2505  struct ftrace_iterator *iter = m->private;
2506  struct dyn_ftrace *rec;
2507 
2508  if (iter->flags & FTRACE_ITER_HASH)
2509  return t_hash_show(m, iter);
2510 
2511  if (iter->flags & FTRACE_ITER_PRINTALL) {
2512  seq_printf(m, "#### all functions enabled ####\n");
2513  return 0;
2514  }
2515 
2516  rec = iter->func;
2517 
2518  if (!rec)
2519  return 0;
2520 
2521  seq_printf(m, "%ps", (void *)rec->ip);
2522  if (iter->flags & FTRACE_ITER_ENABLED)
2523  seq_printf(m, " (%ld)%s",
2524  rec->flags & ~FTRACE_FL_MASK,
2525  rec->flags & FTRACE_FL_REGS ? " R" : "");
2526  seq_printf(m, "\n");
2527 
2528  return 0;
2529 }
2530 
2531 static const struct seq_operations show_ftrace_seq_ops = {
2532  .start = t_start,
2533  .next = t_next,
2534  .stop = t_stop,
2535  .show = t_show,
2536 };
2537 
2538 static int
2539 ftrace_avail_open(struct inode *inode, struct file *file)
2540 {
2541  struct ftrace_iterator *iter;
2542 
2543  if (unlikely(ftrace_disabled))
2544  return -ENODEV;
2545 
2546  iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2547  if (iter) {
2548  iter->pg = ftrace_pages_start;
2549  iter->ops = &global_ops;
2550  }
2551 
2552  return iter ? 0 : -ENOMEM;
2553 }
2554 
2555 static int
2556 ftrace_enabled_open(struct inode *inode, struct file *file)
2557 {
2558  struct ftrace_iterator *iter;
2559 
2560  if (unlikely(ftrace_disabled))
2561  return -ENODEV;
2562 
2563  iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2564  if (iter) {
2565  iter->pg = ftrace_pages_start;
2566  iter->flags = FTRACE_ITER_ENABLED;
2567  iter->ops = &global_ops;
2568  }
2569 
2570  return iter ? 0 : -ENOMEM;
2571 }
2572 
2573 static void ftrace_filter_reset(struct ftrace_hash *hash)
2574 {
2575  mutex_lock(&ftrace_lock);
2576  ftrace_hash_clear(hash);
2577  mutex_unlock(&ftrace_lock);
2578 }
2579 
2596 int
2597 ftrace_regex_open(struct ftrace_ops *ops, int flag,
2598  struct inode *inode, struct file *file)
2599 {
2600  struct ftrace_iterator *iter;
2601  struct ftrace_hash *hash;
2602  int ret = 0;
2603 
2604  if (unlikely(ftrace_disabled))
2605  return -ENODEV;
2606 
2607  iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2608  if (!iter)
2609  return -ENOMEM;
2610 
2611  if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
2612  kfree(iter);
2613  return -ENOMEM;
2614  }
2615 
2616  if (flag & FTRACE_ITER_NOTRACE)
2617  hash = ops->notrace_hash;
2618  else
2619  hash = ops->filter_hash;
2620 
2621  iter->ops = ops;
2622  iter->flags = flag;
2623 
2624  if (file->f_mode & FMODE_WRITE) {
2625  mutex_lock(&ftrace_lock);
2626  iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
2627  mutex_unlock(&ftrace_lock);
2628 
2629  if (!iter->hash) {
2630  trace_parser_put(&iter->parser);
2631  kfree(iter);
2632  return -ENOMEM;
2633  }
2634  }
2635 
2636  mutex_lock(&ftrace_regex_lock);
2637 
2638  if ((file->f_mode & FMODE_WRITE) &&
2639  (file->f_flags & O_TRUNC))
2640  ftrace_filter_reset(iter->hash);
2641 
2642  if (file->f_mode & FMODE_READ) {
2643  iter->pg = ftrace_pages_start;
2644 
2645  ret = seq_open(file, &show_ftrace_seq_ops);
2646  if (!ret) {
2647  struct seq_file *m = file->private_data;
2648  m->private = iter;
2649  } else {
2650  /* Failed */
2651  free_ftrace_hash(iter->hash);
2652  trace_parser_put(&iter->parser);
2653  kfree(iter);
2654  }
2655  } else
2656  file->private_data = iter;
2657  mutex_unlock(&ftrace_regex_lock);
2658 
2659  return ret;
2660 }
2661 
2662 static int
2663 ftrace_filter_open(struct inode *inode, struct file *file)
2664 {
2665  return ftrace_regex_open(&global_ops,
2666  FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
2667  inode, file);
2668 }
2669 
2670 static int
2671 ftrace_notrace_open(struct inode *inode, struct file *file)
2672 {
2673  return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
2674  inode, file);
2675 }
2676 
2677 loff_t
2678 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
2679 {
2680  loff_t ret;
2681 
2682  if (file->f_mode & FMODE_READ)
2683  ret = seq_lseek(file, offset, origin);
2684  else
2685  file->f_pos = ret = 1;
2686 
2687  return ret;
2688 }
2689 
2690 static int ftrace_match(char *str, char *regex, int len, int type)
2691 {
2692  int matched = 0;
2693  int slen;
2694 
2695  switch (type) {
2696  case MATCH_FULL:
2697  if (strcmp(str, regex) == 0)
2698  matched = 1;
2699  break;
2700  case MATCH_FRONT_ONLY:
2701  if (strncmp(str, regex, len) == 0)
2702  matched = 1;
2703  break;
2704  case MATCH_MIDDLE_ONLY:
2705  if (strstr(str, regex))
2706  matched = 1;
2707  break;
2708  case MATCH_END_ONLY:
2709  slen = strlen(str);
2710  if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
2711  matched = 1;
2712  break;
2713  }
2714 
2715  return matched;
2716 }
2717 
2718 static int
2719 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2720 {
2721  struct ftrace_func_entry *entry;
2722  int ret = 0;
2723 
2724  entry = ftrace_lookup_ip(hash, rec->ip);
2725  if (not) {
2726  /* Do nothing if it doesn't exist */
2727  if (!entry)
2728  return 0;
2729 
2730  free_hash_entry(hash, entry);
2731  } else {
2732  /* Do nothing if it exists */
2733  if (entry)
2734  return 0;
2735 
2736  ret = add_hash_entry(hash, rec->ip);
2737  }
2738  return ret;
2739 }
2740 
2741 static int
2742 ftrace_match_record(struct dyn_ftrace *rec, char *mod,
2743  char *regex, int len, int type)
2744 {
2745  char str[KSYM_SYMBOL_LEN];
2746  char *modname;
2747 
2748  kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
2749 
2750  if (mod) {
2751  /* module lookup requires matching the module */
2752  if (!modname || strcmp(modname, mod))
2753  return 0;
2754 
2755  /* blank search means to match all funcs in the mod */
2756  if (!len)
2757  return 1;
2758  }
2759 
2760  return ftrace_match(str, regex, len, type);
2761 }
2762 
2763 static int
2764 match_records(struct ftrace_hash *hash, char *buff,
2765  int len, char *mod, int not)
2766 {
2767  unsigned search_len = 0;
2768  struct ftrace_page *pg;
2769  struct dyn_ftrace *rec;
2770  int type = MATCH_FULL;
2771  char *search = buff;
2772  int found = 0;
2773  int ret;
2774 
2775  if (len) {
2776  type = filter_parse_regex(buff, len, &search, &not);
2777  search_len = strlen(search);
2778  }
2779 
2780  mutex_lock(&ftrace_lock);
2781 
2782  if (unlikely(ftrace_disabled))
2783  goto out_unlock;
2784 
2785  do_for_each_ftrace_rec(pg, rec) {
2786  if (ftrace_match_record(rec, mod, search, search_len, type)) {
2787  ret = enter_record(hash, rec, not);
2788  if (ret < 0) {
2789  found = ret;
2790  goto out_unlock;
2791  }
2792  found = 1;
2793  }
2794  } while_for_each_ftrace_rec();
2795  out_unlock:
2796  mutex_unlock(&ftrace_lock);
2797 
2798  return found;
2799 }
2800 
2801 static int
2802 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
2803 {
2804  return match_records(hash, buff, len, NULL, 0);
2805 }
2806 
2807 static int
2808 ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
2809 {
2810  int not = 0;
2811 
2812  /* blank or '*' mean the same */
2813  if (strcmp(buff, "*") == 0)
2814  buff[0] = 0;
2815 
2816  /* handle the case of 'dont filter this module' */
2817  if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
2818  buff[0] = 0;
2819  not = 1;
2820  }
2821 
2822  return match_records(hash, buff, strlen(buff), mod, not);
2823 }
2824 
2825 /*
2826  * We register the module command as a template to show others how
2827  * to register the a command as well.
2828  */
2829 
2830 static int
2831 ftrace_mod_callback(struct ftrace_hash *hash,
2832  char *func, char *cmd, char *param, int enable)
2833 {
2834  char *mod;
2835  int ret = -EINVAL;
2836 
2837  /*
2838  * cmd == 'mod' because we only registered this func
2839  * for the 'mod' ftrace_func_command.
2840  * But if you register one func with multiple commands,
2841  * you can tell which command was used by the cmd
2842  * parameter.
2843  */
2844 
2845  /* we must have a module name */
2846  if (!param)
2847  return ret;
2848 
2849  mod = strsep(&param, ":");
2850  if (!strlen(mod))
2851  return ret;
2852 
2853  ret = ftrace_match_module_records(hash, func, mod);
2854  if (!ret)
2855  ret = -EINVAL;
2856  if (ret < 0)
2857  return ret;
2858 
2859  return 0;
2860 }
2861 
2862 static struct ftrace_func_command ftrace_mod_cmd = {
2863  .name = "mod",
2864  .func = ftrace_mod_callback,
2865 };
2866 
2867 static int __init ftrace_mod_cmd_init(void)
2868 {
2869  return register_ftrace_command(&ftrace_mod_cmd);
2870 }
2871 device_initcall(ftrace_mod_cmd_init);
2872 
2873 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
2874  struct ftrace_ops *op, struct pt_regs *pt_regs)
2875 {
2876  struct ftrace_func_probe *entry;
2877  struct hlist_head *hhd;
2878  struct hlist_node *n;
2879  unsigned long key;
2880 
2881  key = hash_long(ip, FTRACE_HASH_BITS);
2882 
2883  hhd = &ftrace_func_hash[key];
2884 
2885  if (hlist_empty(hhd))
2886  return;
2887 
2888  /*
2889  * Disable preemption for these calls to prevent a RCU grace
2890  * period. This syncs the hash iteration and freeing of items
2891  * on the hash. rcu_read_lock is too dangerous here.
2892  */
2894  hlist_for_each_entry_rcu(entry, n, hhd, node) {
2895  if (entry->ip == ip)
2896  entry->ops->func(ip, parent_ip, &entry->data);
2897  }
2899 }
2900 
2901 static struct ftrace_ops trace_probe_ops __read_mostly =
2902 {
2903  .func = function_trace_probe_call,
2904 };
2905 
2906 static int ftrace_probe_registered;
2907 
2908 static void __enable_ftrace_function_probe(void)
2909 {
2910  int ret;
2911  int i;
2912 
2913  if (ftrace_probe_registered)
2914  return;
2915 
2916  for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2917  struct hlist_head *hhd = &ftrace_func_hash[i];
2918  if (hhd->first)
2919  break;
2920  }
2921  /* Nothing registered? */
2922  if (i == FTRACE_FUNC_HASHSIZE)
2923  return;
2924 
2925  ret = __register_ftrace_function(&trace_probe_ops);
2926  if (!ret)
2927  ret = ftrace_startup(&trace_probe_ops, 0);
2928 
2929  ftrace_probe_registered = 1;
2930 }
2931 
2932 static void __disable_ftrace_function_probe(void)
2933 {
2934  int ret;
2935  int i;
2936 
2937  if (!ftrace_probe_registered)
2938  return;
2939 
2940  for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2941  struct hlist_head *hhd = &ftrace_func_hash[i];
2942  if (hhd->first)
2943  return;
2944  }
2945 
2946  /* no more funcs left */
2947  ret = __unregister_ftrace_function(&trace_probe_ops);
2948  if (!ret)
2949  ftrace_shutdown(&trace_probe_ops, 0);
2950 
2951  ftrace_probe_registered = 0;
2952 }
2953 
2954 
2955 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2956 {
2957  struct ftrace_func_probe *entry =
2958  container_of(rhp, struct ftrace_func_probe, rcu);
2959 
2960  if (entry->ops->free)
2961  entry->ops->free(&entry->data);
2962  kfree(entry);
2963 }
2964 
2965 
2966 int
2967 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2968  void *data)
2969 {
2970  struct ftrace_func_probe *entry;
2971  struct ftrace_page *pg;
2972  struct dyn_ftrace *rec;
2973  int type, len, not;
2974  unsigned long key;
2975  int count = 0;
2976  char *search;
2977 
2978  type = filter_parse_regex(glob, strlen(glob), &search, &not);
2979  len = strlen(search);
2980 
2981  /* we do not support '!' for function probes */
2982  if (WARN_ON(not))
2983  return -EINVAL;
2984 
2985  mutex_lock(&ftrace_lock);
2986 
2987  if (unlikely(ftrace_disabled))
2988  goto out_unlock;
2989 
2990  do_for_each_ftrace_rec(pg, rec) {
2991 
2992  if (!ftrace_match_record(rec, NULL, search, len, type))
2993  continue;
2994 
2995  entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2996  if (!entry) {
2997  /* If we did not process any, then return error */
2998  if (!count)
2999  count = -ENOMEM;
3000  goto out_unlock;
3001  }
3002 
3003  count++;
3004 
3005  entry->data = data;
3006 
3007  /*
3008  * The caller might want to do something special
3009  * for each function we find. We call the callback
3010  * to give the caller an opportunity to do so.
3011  */
3012  if (ops->callback) {
3013  if (ops->callback(rec->ip, &entry->data) < 0) {
3014  /* caller does not like this func */
3015  kfree(entry);
3016  continue;
3017  }
3018  }
3019 
3020  entry->ops = ops;
3021  entry->ip = rec->ip;
3022 
3023  key = hash_long(entry->ip, FTRACE_HASH_BITS);
3024  hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
3025 
3026  } while_for_each_ftrace_rec();
3027  __enable_ftrace_function_probe();
3028 
3029  out_unlock:
3030  mutex_unlock(&ftrace_lock);
3031 
3032  return count;
3033 }
3034 
3035 enum {
3036  PROBE_TEST_FUNC = 1,
3037  PROBE_TEST_DATA = 2
3038 };
3039 
3040 static void
3041 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3042  void *data, int flags)
3043 {
3044  struct ftrace_func_probe *entry;
3045  struct hlist_node *n, *tmp;
3046  char str[KSYM_SYMBOL_LEN];
3047  int type = MATCH_FULL;
3048  int i, len = 0;
3049  char *search;
3050 
3051  if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
3052  glob = NULL;
3053  else if (glob) {
3054  int not;
3055 
3056  type = filter_parse_regex(glob, strlen(glob), &search, &not);
3057  len = strlen(search);
3058 
3059  /* we do not support '!' for function probes */
3060  if (WARN_ON(not))
3061  return;
3062  }
3063 
3064  mutex_lock(&ftrace_lock);
3065  for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3066  struct hlist_head *hhd = &ftrace_func_hash[i];
3067 
3068  hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
3069 
3070  /* break up if statements for readability */
3071  if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
3072  continue;
3073 
3074  if ((flags & PROBE_TEST_DATA) && entry->data != data)
3075  continue;
3076 
3077  /* do this last, since it is the most expensive */
3078  if (glob) {
3079  kallsyms_lookup(entry->ip, NULL, NULL,
3080  NULL, str);
3081  if (!ftrace_match(str, glob, len, type))
3082  continue;
3083  }
3084 
3085  hlist_del(&entry->node);
3086  call_rcu(&entry->rcu, ftrace_free_entry_rcu);
3087  }
3088  }
3089  __disable_ftrace_function_probe();
3090  mutex_unlock(&ftrace_lock);
3091 }
3092 
3093 void
3094 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3095  void *data)
3096 {
3097  __unregister_ftrace_function_probe(glob, ops, data,
3098  PROBE_TEST_FUNC | PROBE_TEST_DATA);
3099 }
3100 
3101 void
3102 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
3103 {
3104  __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
3105 }
3106 
3107 void unregister_ftrace_function_probe_all(char *glob)
3108 {
3109  __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
3110 }
3111 
3112 static LIST_HEAD(ftrace_commands);
3113 static DEFINE_MUTEX(ftrace_cmd_mutex);
3114 
3115 int register_ftrace_command(struct ftrace_func_command *cmd)
3116 {
3117  struct ftrace_func_command *p;
3118  int ret = 0;
3119 
3120  mutex_lock(&ftrace_cmd_mutex);
3121  list_for_each_entry(p, &ftrace_commands, list) {
3122  if (strcmp(cmd->name, p->name) == 0) {
3123  ret = -EBUSY;
3124  goto out_unlock;
3125  }
3126  }
3127  list_add(&cmd->list, &ftrace_commands);
3128  out_unlock:
3129  mutex_unlock(&ftrace_cmd_mutex);
3130 
3131  return ret;
3132 }
3133 
3134 int unregister_ftrace_command(struct ftrace_func_command *cmd)
3135 {
3136  struct ftrace_func_command *p, *n;
3137  int ret = -ENODEV;
3138 
3139  mutex_lock(&ftrace_cmd_mutex);
3140  list_for_each_entry_safe(p, n, &ftrace_commands, list) {
3141  if (strcmp(cmd->name, p->name) == 0) {
3142  ret = 0;
3143  list_del_init(&p->list);
3144  goto out_unlock;
3145  }
3146  }
3147  out_unlock:
3148  mutex_unlock(&ftrace_cmd_mutex);
3149 
3150  return ret;
3151 }
3152 
3153 static int ftrace_process_regex(struct ftrace_hash *hash,
3154  char *buff, int len, int enable)
3155 {
3156  char *func, *command, *next = buff;
3157  struct ftrace_func_command *p;
3158  int ret = -EINVAL;
3159 
3160  func = strsep(&next, ":");
3161 
3162  if (!next) {
3163  ret = ftrace_match_records(hash, func, len);
3164  if (!ret)
3165  ret = -EINVAL;
3166  if (ret < 0)
3167  return ret;
3168  return 0;
3169  }
3170 
3171  /* command found */
3172 
3173  command = strsep(&next, ":");
3174 
3175  mutex_lock(&ftrace_cmd_mutex);
3176  list_for_each_entry(p, &ftrace_commands, list) {
3177  if (strcmp(p->name, command) == 0) {
3178  ret = p->func(hash, func, command, next, enable);
3179  goto out_unlock;
3180  }
3181  }
3182  out_unlock:
3183  mutex_unlock(&ftrace_cmd_mutex);
3184 
3185  return ret;
3186 }
3187 
3188 static ssize_t
3189 ftrace_regex_write(struct file *file, const char __user *ubuf,
3190  size_t cnt, loff_t *ppos, int enable)
3191 {
3192  struct ftrace_iterator *iter;
3193  struct trace_parser *parser;
3194  ssize_t ret, read;
3195 
3196  if (!cnt)
3197  return 0;
3198 
3199  mutex_lock(&ftrace_regex_lock);
3200 
3201  ret = -ENODEV;
3202  if (unlikely(ftrace_disabled))
3203  goto out_unlock;
3204 
3205  if (file->f_mode & FMODE_READ) {
3206  struct seq_file *m = file->private_data;
3207  iter = m->private;
3208  } else
3209  iter = file->private_data;
3210 
3211  parser = &iter->parser;
3212  read = trace_get_user(parser, ubuf, cnt, ppos);
3213 
3214  if (read >= 0 && trace_parser_loaded(parser) &&
3215  !trace_parser_cont(parser)) {
3216  ret = ftrace_process_regex(iter->hash, parser->buffer,
3217  parser->idx, enable);
3218  trace_parser_clear(parser);
3219  if (ret)
3220  goto out_unlock;
3221  }
3222 
3223  ret = read;
3224 out_unlock:
3225  mutex_unlock(&ftrace_regex_lock);
3226 
3227  return ret;
3228 }
3229 
3230 ssize_t
3231 ftrace_filter_write(struct file *file, const char __user *ubuf,
3232  size_t cnt, loff_t *ppos)
3233 {
3234  return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
3235 }
3236 
3237 ssize_t
3238 ftrace_notrace_write(struct file *file, const char __user *ubuf,
3239  size_t cnt, loff_t *ppos)
3240 {
3241  return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
3242 }
3243 
3244 static int
3245 ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
3246 {
3247  struct ftrace_func_entry *entry;
3248 
3249  if (!ftrace_location(ip))
3250  return -EINVAL;
3251 
3252  if (remove) {
3253  entry = ftrace_lookup_ip(hash, ip);
3254  if (!entry)
3255  return -ENOENT;
3256  free_hash_entry(hash, entry);
3257  return 0;
3258  }
3259 
3260  return add_hash_entry(hash, ip);
3261 }
3262 
3263 static int
3264 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
3265  unsigned long ip, int remove, int reset, int enable)
3266 {
3267  struct ftrace_hash **orig_hash;
3268  struct ftrace_hash *hash;
3269  int ret;
3270 
3271  /* All global ops uses the global ops filters */
3272  if (ops->flags & FTRACE_OPS_FL_GLOBAL)
3273  ops = &global_ops;
3274 
3275  if (unlikely(ftrace_disabled))
3276  return -ENODEV;
3277 
3278  if (enable)
3279  orig_hash = &ops->filter_hash;
3280  else
3281  orig_hash = &ops->notrace_hash;
3282 
3283  hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3284  if (!hash)
3285  return -ENOMEM;
3286 
3287  mutex_lock(&ftrace_regex_lock);
3288  if (reset)
3289  ftrace_filter_reset(hash);
3290  if (buf && !ftrace_match_records(hash, buf, len)) {
3291  ret = -EINVAL;
3292  goto out_regex_unlock;
3293  }
3294  if (ip) {
3295  ret = ftrace_match_addr(hash, ip, remove);
3296  if (ret < 0)
3297  goto out_regex_unlock;
3298  }
3299 
3300  mutex_lock(&ftrace_lock);
3301  ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3302  if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
3303  && ftrace_enabled)
3304  ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3305 
3306  mutex_unlock(&ftrace_lock);
3307 
3308  out_regex_unlock:
3309  mutex_unlock(&ftrace_regex_lock);
3310 
3311  free_ftrace_hash(hash);
3312  return ret;
3313 }
3314 
3315 static int
3316 ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
3317  int reset, int enable)
3318 {
3319  return ftrace_set_hash(ops, 0, 0, ip, remove, reset, enable);
3320 }
3321 
3332 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
3333  int remove, int reset)
3334 {
3335  return ftrace_set_addr(ops, ip, remove, reset, 1);
3336 }
3338 
3339 static int
3340 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
3341  int reset, int enable)
3342 {
3343  return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
3344 }
3345 
3356 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
3357  int len, int reset)
3358 {
3359  return ftrace_set_regex(ops, buf, len, reset, 1);
3360 }
3362 
3374 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
3375  int len, int reset)
3376 {
3377  return ftrace_set_regex(ops, buf, len, reset, 0);
3378 }
3390 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
3391 {
3392  ftrace_set_regex(&global_ops, buf, len, reset, 1);
3393 }
3394 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
3395 
3407 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
3408 {
3409  ftrace_set_regex(&global_ops, buf, len, reset, 0);
3410 }
3411 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
3412 
3413 /*
3414  * command line interface to allow users to set filters on boot up.
3415  */
3416 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
3417 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
3418 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
3419 
3420 static int __init set_ftrace_notrace(char *str)
3421 {
3422  strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
3423  return 1;
3424 }
3425 __setup("ftrace_notrace=", set_ftrace_notrace);
3426 
3427 static int __init set_ftrace_filter(char *str)
3428 {
3429  strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
3430  return 1;
3431 }
3432 __setup("ftrace_filter=", set_ftrace_filter);
3433 
3434 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3435 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
3436 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
3437 
3438 static int __init set_graph_function(char *str)
3439 {
3440  strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
3441  return 1;
3442 }
3443 __setup("ftrace_graph_filter=", set_graph_function);
3444 
3445 static void __init set_ftrace_early_graph(char *buf)
3446 {
3447  int ret;
3448  char *func;
3449 
3450  while (buf) {
3451  func = strsep(&buf, ",");
3452  /* we allow only one expression at a time */
3453  ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3454  func);
3455  if (ret)
3456  printk(KERN_DEBUG "ftrace: function %s not "
3457  "traceable\n", func);
3458  }
3459 }
3460 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3461 
3462 void __init
3463 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
3464 {
3465  char *func;
3466 
3467  while (buf) {
3468  func = strsep(&buf, ",");
3469  ftrace_set_regex(ops, func, strlen(func), 0, enable);
3470  }
3471 }
3472 
3473 static void __init set_ftrace_early_filters(void)
3474 {
3475  if (ftrace_filter_buf[0])
3476  ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
3477  if (ftrace_notrace_buf[0])
3478  ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
3479 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3480  if (ftrace_graph_buf[0])
3481  set_ftrace_early_graph(ftrace_graph_buf);
3482 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3483 }
3484 
3485 int ftrace_regex_release(struct inode *inode, struct file *file)
3486 {
3487  struct seq_file *m = (struct seq_file *)file->private_data;
3488  struct ftrace_iterator *iter;
3489  struct ftrace_hash **orig_hash;
3490  struct trace_parser *parser;
3491  int filter_hash;
3492  int ret;
3493 
3494  mutex_lock(&ftrace_regex_lock);
3495  if (file->f_mode & FMODE_READ) {
3496  iter = m->private;
3497 
3498  seq_release(inode, file);
3499  } else
3500  iter = file->private_data;
3501 
3502  parser = &iter->parser;
3503  if (trace_parser_loaded(parser)) {
3504  parser->buffer[parser->idx] = 0;
3505  ftrace_match_records(iter->hash, parser->buffer, parser->idx);
3506  }
3507 
3508  trace_parser_put(parser);
3509 
3510  if (file->f_mode & FMODE_WRITE) {
3511  filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3512 
3513  if (filter_hash)
3514  orig_hash = &iter->ops->filter_hash;
3515  else
3516  orig_hash = &iter->ops->notrace_hash;
3517 
3518  mutex_lock(&ftrace_lock);
3519  ret = ftrace_hash_move(iter->ops, filter_hash,
3520  orig_hash, iter->hash);
3521  if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
3522  && ftrace_enabled)
3523  ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3524 
3525  mutex_unlock(&ftrace_lock);
3526  }
3527  free_ftrace_hash(iter->hash);
3528  kfree(iter);
3529 
3530  mutex_unlock(&ftrace_regex_lock);
3531  return 0;
3532 }
3533 
3534 static const struct file_operations ftrace_avail_fops = {
3535  .open = ftrace_avail_open,
3536  .read = seq_read,
3537  .llseek = seq_lseek,
3538  .release = seq_release_private,
3539 };
3540 
3541 static const struct file_operations ftrace_enabled_fops = {
3542  .open = ftrace_enabled_open,
3543  .read = seq_read,
3544  .llseek = seq_lseek,
3545  .release = seq_release_private,
3546 };
3547 
3548 static const struct file_operations ftrace_filter_fops = {
3549  .open = ftrace_filter_open,
3550  .read = seq_read,
3551  .write = ftrace_filter_write,
3552  .llseek = ftrace_regex_lseek,
3553  .release = ftrace_regex_release,
3554 };
3555 
3556 static const struct file_operations ftrace_notrace_fops = {
3557  .open = ftrace_notrace_open,
3558  .read = seq_read,
3559  .write = ftrace_notrace_write,
3560  .llseek = ftrace_regex_lseek,
3561  .release = ftrace_regex_release,
3562 };
3563 
3564 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3565 
3566 static DEFINE_MUTEX(graph_lock);
3567 
3568 int ftrace_graph_count;
3569 int ftrace_graph_filter_enabled;
3570 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3571 
3572 static void *
3573 __g_next(struct seq_file *m, loff_t *pos)
3574 {
3575  if (*pos >= ftrace_graph_count)
3576  return NULL;
3577  return &ftrace_graph_funcs[*pos];
3578 }
3579 
3580 static void *
3581 g_next(struct seq_file *m, void *v, loff_t *pos)
3582 {
3583  (*pos)++;
3584  return __g_next(m, pos);
3585 }
3586 
3587 static void *g_start(struct seq_file *m, loff_t *pos)
3588 {
3589  mutex_lock(&graph_lock);
3590 
3591  /* Nothing, tell g_show to print all functions are enabled */
3592  if (!ftrace_graph_filter_enabled && !*pos)
3593  return (void *)1;
3594 
3595  return __g_next(m, pos);
3596 }
3597 
3598 static void g_stop(struct seq_file *m, void *p)
3599 {
3600  mutex_unlock(&graph_lock);
3601 }
3602 
3603 static int g_show(struct seq_file *m, void *v)
3604 {
3605  unsigned long *ptr = v;
3606 
3607  if (!ptr)
3608  return 0;
3609 
3610  if (ptr == (unsigned long *)1) {
3611  seq_printf(m, "#### all functions enabled ####\n");
3612  return 0;
3613  }
3614 
3615  seq_printf(m, "%ps\n", (void *)*ptr);
3616 
3617  return 0;
3618 }
3619 
3620 static const struct seq_operations ftrace_graph_seq_ops = {
3621  .start = g_start,
3622  .next = g_next,
3623  .stop = g_stop,
3624  .show = g_show,
3625 };
3626 
3627 static int
3628 ftrace_graph_open(struct inode *inode, struct file *file)
3629 {
3630  int ret = 0;
3631 
3632  if (unlikely(ftrace_disabled))
3633  return -ENODEV;
3634 
3635  mutex_lock(&graph_lock);
3636  if ((file->f_mode & FMODE_WRITE) &&
3637  (file->f_flags & O_TRUNC)) {
3638  ftrace_graph_filter_enabled = 0;
3639  ftrace_graph_count = 0;
3640  memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
3641  }
3642  mutex_unlock(&graph_lock);
3643 
3644  if (file->f_mode & FMODE_READ)
3645  ret = seq_open(file, &ftrace_graph_seq_ops);
3646 
3647  return ret;
3648 }
3649 
3650 static int
3651 ftrace_graph_release(struct inode *inode, struct file *file)
3652 {
3653  if (file->f_mode & FMODE_READ)
3654  seq_release(inode, file);
3655  return 0;
3656 }
3657 
3658 static int
3659 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
3660 {
3661  struct dyn_ftrace *rec;
3662  struct ftrace_page *pg;
3663  int search_len;
3664  int fail = 1;
3665  int type, not;
3666  char *search;
3667  bool exists;
3668  int i;
3669 
3670  /* decode regex */
3671  type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
3672  if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
3673  return -EBUSY;
3674 
3675  search_len = strlen(search);
3676 
3677  mutex_lock(&ftrace_lock);
3678 
3679  if (unlikely(ftrace_disabled)) {
3680  mutex_unlock(&ftrace_lock);
3681  return -ENODEV;
3682  }
3683 
3684  do_for_each_ftrace_rec(pg, rec) {
3685 
3686  if (ftrace_match_record(rec, NULL, search, search_len, type)) {
3687  /* if it is in the array */
3688  exists = false;
3689  for (i = 0; i < *idx; i++) {
3690  if (array[i] == rec->ip) {
3691  exists = true;
3692  break;
3693  }
3694  }
3695 
3696  if (!not) {
3697  fail = 0;
3698  if (!exists) {
3699  array[(*idx)++] = rec->ip;
3700  if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
3701  goto out;
3702  }
3703  } else {
3704  if (exists) {
3705  array[i] = array[--(*idx)];
3706  array[*idx] = 0;
3707  fail = 0;
3708  }
3709  }
3710  }
3711  } while_for_each_ftrace_rec();
3712 out:
3713  mutex_unlock(&ftrace_lock);
3714 
3715  if (fail)
3716  return -EINVAL;
3717 
3718  ftrace_graph_filter_enabled = 1;
3719  return 0;
3720 }
3721 
3722 static ssize_t
3723 ftrace_graph_write(struct file *file, const char __user *ubuf,
3724  size_t cnt, loff_t *ppos)
3725 {
3726  struct trace_parser parser;
3727  ssize_t read, ret;
3728 
3729  if (!cnt)
3730  return 0;
3731 
3732  mutex_lock(&graph_lock);
3733 
3734  if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
3735  ret = -ENOMEM;
3736  goto out_unlock;
3737  }
3738 
3739  read = trace_get_user(&parser, ubuf, cnt, ppos);
3740 
3741  if (read >= 0 && trace_parser_loaded((&parser))) {
3742  parser.buffer[parser.idx] = 0;
3743 
3744  /* we allow only one expression at a time */
3745  ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3746  parser.buffer);
3747  if (ret)
3748  goto out_free;
3749  }
3750 
3751  ret = read;
3752 
3753 out_free:
3754  trace_parser_put(&parser);
3755 out_unlock:
3756  mutex_unlock(&graph_lock);
3757 
3758  return ret;
3759 }
3760 
3761 static const struct file_operations ftrace_graph_fops = {
3762  .open = ftrace_graph_open,
3763  .read = seq_read,
3764  .write = ftrace_graph_write,
3765  .release = ftrace_graph_release,
3766  .llseek = seq_lseek,
3767 };
3768 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3769 
3770 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
3771 {
3772 
3773  trace_create_file("available_filter_functions", 0444,
3774  d_tracer, NULL, &ftrace_avail_fops);
3775 
3776  trace_create_file("enabled_functions", 0444,
3777  d_tracer, NULL, &ftrace_enabled_fops);
3778 
3779  trace_create_file("set_ftrace_filter", 0644, d_tracer,
3780  NULL, &ftrace_filter_fops);
3781 
3782  trace_create_file("set_ftrace_notrace", 0644, d_tracer,
3783  NULL, &ftrace_notrace_fops);
3784 
3785 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3786  trace_create_file("set_graph_function", 0444, d_tracer,
3787  NULL,
3788  &ftrace_graph_fops);
3789 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3790 
3791  return 0;
3792 }
3793 
3794 static int ftrace_cmp_ips(const void *a, const void *b)
3795 {
3796  const unsigned long *ipa = a;
3797  const unsigned long *ipb = b;
3798 
3799  if (*ipa > *ipb)
3800  return 1;
3801  if (*ipa < *ipb)
3802  return -1;
3803  return 0;
3804 }
3805 
3806 static void ftrace_swap_ips(void *a, void *b, int size)
3807 {
3808  unsigned long *ipa = a;
3809  unsigned long *ipb = b;
3810  unsigned long t;
3811 
3812  t = *ipa;
3813  *ipa = *ipb;
3814  *ipb = t;
3815 }
3816 
3817 static int ftrace_process_locs(struct module *mod,
3818  unsigned long *start,
3819  unsigned long *end)
3820 {
3821  struct ftrace_page *start_pg;
3822  struct ftrace_page *pg;
3823  struct dyn_ftrace *rec;
3824  unsigned long count;
3825  unsigned long *p;
3826  unsigned long addr;
3827  unsigned long flags = 0; /* Shut up gcc */
3828  int ret = -ENOMEM;
3829 
3830  count = end - start;
3831 
3832  if (!count)
3833  return 0;
3834 
3835  sort(start, count, sizeof(*start),
3836  ftrace_cmp_ips, ftrace_swap_ips);
3837 
3838  start_pg = ftrace_allocate_pages(count);
3839  if (!start_pg)
3840  return -ENOMEM;
3841 
3842  mutex_lock(&ftrace_lock);
3843 
3844  /*
3845  * Core and each module needs their own pages, as
3846  * modules will free them when they are removed.
3847  * Force a new page to be allocated for modules.
3848  */
3849  if (!mod) {
3850  WARN_ON(ftrace_pages || ftrace_pages_start);
3851  /* First initialization */
3852  ftrace_pages = ftrace_pages_start = start_pg;
3853  } else {
3854  if (!ftrace_pages)
3855  goto out;
3856 
3857  if (WARN_ON(ftrace_pages->next)) {
3858  /* Hmm, we have free pages? */
3859  while (ftrace_pages->next)
3860  ftrace_pages = ftrace_pages->next;
3861  }
3862 
3863  ftrace_pages->next = start_pg;
3864  }
3865 
3866  p = start;
3867  pg = start_pg;
3868  while (p < end) {
3869  addr = ftrace_call_adjust(*p++);
3870  /*
3871  * Some architecture linkers will pad between
3872  * the different mcount_loc sections of different
3873  * object files to satisfy alignments.
3874  * Skip any NULL pointers.
3875  */
3876  if (!addr)
3877  continue;
3878 
3879  if (pg->index == pg->size) {
3880  /* We should have allocated enough */
3881  if (WARN_ON(!pg->next))
3882  break;
3883  pg = pg->next;
3884  }
3885 
3886  rec = &pg->records[pg->index++];
3887  rec->ip = addr;
3888  }
3889 
3890  /* We should have used all pages */
3891  WARN_ON(pg->next);
3892 
3893  /* Assign the last page to ftrace_pages */
3894  ftrace_pages = pg;
3895 
3896  /* These new locations need to be initialized */
3897  ftrace_new_pgs = start_pg;
3898 
3899  /*
3900  * We only need to disable interrupts on start up
3901  * because we are modifying code that an interrupt
3902  * may execute, and the modification is not atomic.
3903  * But for modules, nothing runs the code we modify
3904  * until we are finished with it, and there's no
3905  * reason to cause large interrupt latencies while we do it.
3906  */
3907  if (!mod)
3908  local_irq_save(flags);
3909  ftrace_update_code(mod);
3910  if (!mod)
3911  local_irq_restore(flags);
3912  ret = 0;
3913  out:
3914  mutex_unlock(&ftrace_lock);
3915 
3916  return ret;
3917 }
3918 
3919 #ifdef CONFIG_MODULES
3920 
3921 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
3922 
3923 void ftrace_release_mod(struct module *mod)
3924 {
3925  struct dyn_ftrace *rec;
3926  struct ftrace_page **last_pg;
3927  struct ftrace_page *pg;
3928  int order;
3929 
3930  mutex_lock(&ftrace_lock);
3931 
3932  if (ftrace_disabled)
3933  goto out_unlock;
3934 
3935  /*
3936  * Each module has its own ftrace_pages, remove
3937  * them from the list.
3938  */
3939  last_pg = &ftrace_pages_start;
3940  for (pg = ftrace_pages_start; pg; pg = *last_pg) {
3941  rec = &pg->records[0];
3942  if (within_module_core(rec->ip, mod)) {
3943  /*
3944  * As core pages are first, the first
3945  * page should never be a module page.
3946  */
3947  if (WARN_ON(pg == ftrace_pages_start))
3948  goto out_unlock;
3949 
3950  /* Check if we are deleting the last page */
3951  if (pg == ftrace_pages)
3952  ftrace_pages = next_to_ftrace_page(last_pg);
3953 
3954  *last_pg = pg->next;
3955  order = get_count_order(pg->size / ENTRIES_PER_PAGE);
3956  free_pages((unsigned long)pg->records, order);
3957  kfree(pg);
3958  } else
3959  last_pg = &pg->next;
3960  }
3961  out_unlock:
3962  mutex_unlock(&ftrace_lock);
3963 }
3964 
3965 static void ftrace_init_module(struct module *mod,
3966  unsigned long *start, unsigned long *end)
3967 {
3968  if (ftrace_disabled || start == end)
3969  return;
3970  ftrace_process_locs(mod, start, end);
3971 }
3972 
3973 static int ftrace_module_notify(struct notifier_block *self,
3974  unsigned long val, void *data)
3975 {
3976  struct module *mod = data;
3977 
3978  switch (val) {
3979  case MODULE_STATE_COMING:
3980  ftrace_init_module(mod, mod->ftrace_callsites,
3981  mod->ftrace_callsites +
3982  mod->num_ftrace_callsites);
3983  break;
3984  case MODULE_STATE_GOING:
3985  ftrace_release_mod(mod);
3986  break;
3987  }
3988 
3989  return 0;
3990 }
3991 #else
3992 static int ftrace_module_notify(struct notifier_block *self,
3993  unsigned long val, void *data)
3994 {
3995  return 0;
3996 }
3997 #endif /* CONFIG_MODULES */
3998 
3999 struct notifier_block ftrace_module_nb = {
4000  .notifier_call = ftrace_module_notify,
4001  .priority = 0,
4002 };
4003 
4004 extern unsigned long __start_mcount_loc[];
4005 extern unsigned long __stop_mcount_loc[];
4006 
4007 void __init ftrace_init(void)
4008 {
4009  unsigned long count, addr, flags;
4010  int ret;
4011 
4012  /* Keep the ftrace pointer to the stub */
4013  addr = (unsigned long)ftrace_stub;
4014 
4015  local_irq_save(flags);
4016  ftrace_dyn_arch_init(&addr);
4017  local_irq_restore(flags);
4018 
4019  /* ftrace_dyn_arch_init places the return code in addr */
4020  if (addr)
4021  goto failed;
4022 
4023  count = __stop_mcount_loc - __start_mcount_loc;
4024 
4025  ret = ftrace_dyn_table_alloc(count);
4026  if (ret)
4027  goto failed;
4028 
4029  last_ftrace_enabled = ftrace_enabled = 1;
4030 
4031  ret = ftrace_process_locs(NULL,
4032  __start_mcount_loc,
4033  __stop_mcount_loc);
4034 
4035  ret = register_module_notifier(&ftrace_module_nb);
4036  if (ret)
4037  pr_warning("Failed to register trace ftrace module notifier\n");
4038 
4039  set_ftrace_early_filters();
4040 
4041  return;
4042  failed:
4043  ftrace_disabled = 1;
4044 }
4045 
4046 #else
4047 
4048 static struct ftrace_ops global_ops = {
4049  .func = ftrace_stub,
4050  .flags = FTRACE_OPS_FL_RECURSION_SAFE,
4051 };
4052 
4053 static int __init ftrace_nodyn_init(void)
4054 {
4055  ftrace_enabled = 1;
4056  return 0;
4057 }
4058 device_initcall(ftrace_nodyn_init);
4059 
4060 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
4061 static inline void ftrace_startup_enable(int command) { }
4062 /* Keep as macros so we do not need to define the commands */
4063 # define ftrace_startup(ops, command) \
4064  ({ \
4065  (ops)->flags |= FTRACE_OPS_FL_ENABLED; \
4066  0; \
4067  })
4068 # define ftrace_shutdown(ops, command) do { } while (0)
4069 # define ftrace_startup_sysctl() do { } while (0)
4070 # define ftrace_shutdown_sysctl() do { } while (0)
4071 
4072 static inline int
4073 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
4074 {
4075  return 1;
4076 }
4077 
4078 #endif /* CONFIG_DYNAMIC_FTRACE */
4079 
4080 static void
4081 ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip,
4082  struct ftrace_ops *op, struct pt_regs *regs)
4083 {
4085  return;
4086 
4087  /*
4088  * Some of the ops may be dynamically allocated,
4089  * they must be freed after a synchronize_sched().
4090  */
4093  op = rcu_dereference_raw(ftrace_control_list);
4094  while (op != &ftrace_list_end) {
4095  if (!ftrace_function_local_disabled(op) &&
4096  ftrace_ops_test(op, ip))
4097  op->func(ip, parent_ip, op, regs);
4098 
4099  op = rcu_dereference_raw(op->next);
4100  };
4103 }
4104 
4105 static struct ftrace_ops control_ops = {
4106  .func = ftrace_ops_control_func,
4107  .flags = FTRACE_OPS_FL_RECURSION_SAFE,
4108 };
4109 
4110 static inline void
4111 __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
4112  struct ftrace_ops *ignored, struct pt_regs *regs)
4113 {
4114  struct ftrace_ops *op;
4115 
4116  if (function_trace_stop)
4117  return;
4118 
4120  return;
4121 
4123  /*
4124  * Some of the ops may be dynamically allocated,
4125  * they must be freed after a synchronize_sched().
4126  */
4128  op = rcu_dereference_raw(ftrace_ops_list);
4129  while (op != &ftrace_list_end) {
4130  if (ftrace_ops_test(op, ip))
4131  op->func(ip, parent_ip, op, regs);
4132  op = rcu_dereference_raw(op->next);
4133  };
4136 }
4137 
4138 /*
4139  * Some archs only support passing ip and parent_ip. Even though
4140  * the list function ignores the op parameter, we do not want any
4141  * C side effects, where a function is called without the caller
4142  * sending a third parameter.
4143  * Archs are to support both the regs and ftrace_ops at the same time.
4144  * If they support ftrace_ops, it is assumed they support regs.
4145  * If call backs want to use regs, they must either check for regs
4146  * being NULL, or ARCH_SUPPORTS_FTRACE_SAVE_REGS.
4147  * Note, ARCH_SUPPORT_SAVE_REGS expects a full regs to be saved.
4148  * An architecture can pass partial regs with ftrace_ops and still
4149  * set the ARCH_SUPPORT_FTARCE_OPS.
4150  */
4151 #if ARCH_SUPPORTS_FTRACE_OPS
4152 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
4153  struct ftrace_ops *op, struct pt_regs *regs)
4154 {
4155  __ftrace_ops_list_func(ip, parent_ip, NULL, regs);
4156 }
4157 #else
4158 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
4159 {
4160  __ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
4161 }
4162 #endif
4163 
4164 static void clear_ftrace_swapper(void)
4165 {
4166  struct task_struct *p;
4167  int cpu;
4168 
4169  get_online_cpus();
4170  for_each_online_cpu(cpu) {
4171  p = idle_task(cpu);
4172  clear_tsk_trace_trace(p);
4173  }
4174  put_online_cpus();
4175 }
4176 
4177 static void set_ftrace_swapper(void)
4178 {
4179  struct task_struct *p;
4180  int cpu;
4181 
4182  get_online_cpus();
4183  for_each_online_cpu(cpu) {
4184  p = idle_task(cpu);
4185  set_tsk_trace_trace(p);
4186  }
4187  put_online_cpus();
4188 }
4189 
4190 static void clear_ftrace_pid(struct pid *pid)
4191 {
4192  struct task_struct *p;
4193 
4194  rcu_read_lock();
4195  do_each_pid_task(pid, PIDTYPE_PID, p) {
4196  clear_tsk_trace_trace(p);
4197  } while_each_pid_task(pid, PIDTYPE_PID, p);
4198  rcu_read_unlock();
4199 
4200  put_pid(pid);
4201 }
4202 
4203 static void set_ftrace_pid(struct pid *pid)
4204 {
4205  struct task_struct *p;
4206 
4207  rcu_read_lock();
4208  do_each_pid_task(pid, PIDTYPE_PID, p) {
4209  set_tsk_trace_trace(p);
4210  } while_each_pid_task(pid, PIDTYPE_PID, p);
4211  rcu_read_unlock();
4212 }
4213 
4214 static void clear_ftrace_pid_task(struct pid *pid)
4215 {
4216  if (pid == ftrace_swapper_pid)
4217  clear_ftrace_swapper();
4218  else
4219  clear_ftrace_pid(pid);
4220 }
4221 
4222 static void set_ftrace_pid_task(struct pid *pid)
4223 {
4224  if (pid == ftrace_swapper_pid)
4225  set_ftrace_swapper();
4226  else
4227  set_ftrace_pid(pid);
4228 }
4229 
4230 static int ftrace_pid_add(int p)
4231 {
4232  struct pid *pid;
4233  struct ftrace_pid *fpid;
4234  int ret = -EINVAL;
4235 
4236  mutex_lock(&ftrace_lock);
4237 
4238  if (!p)
4239  pid = ftrace_swapper_pid;
4240  else
4241  pid = find_get_pid(p);
4242 
4243  if (!pid)
4244  goto out;
4245 
4246  ret = 0;
4247 
4248  list_for_each_entry(fpid, &ftrace_pids, list)
4249  if (fpid->pid == pid)
4250  goto out_put;
4251 
4252  ret = -ENOMEM;
4253 
4254  fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
4255  if (!fpid)
4256  goto out_put;
4257 
4258  list_add(&fpid->list, &ftrace_pids);
4259  fpid->pid = pid;
4260 
4261  set_ftrace_pid_task(pid);
4262 
4263  ftrace_update_pid_func();
4264  ftrace_startup_enable(0);
4265 
4266  mutex_unlock(&ftrace_lock);
4267  return 0;
4268 
4269 out_put:
4270  if (pid != ftrace_swapper_pid)
4271  put_pid(pid);
4272 
4273 out:
4274  mutex_unlock(&ftrace_lock);
4275  return ret;
4276 }
4277 
4278 static void ftrace_pid_reset(void)
4279 {
4280  struct ftrace_pid *fpid, *safe;
4281 
4282  mutex_lock(&ftrace_lock);
4283  list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
4284  struct pid *pid = fpid->pid;
4285 
4286  clear_ftrace_pid_task(pid);
4287 
4288  list_del(&fpid->list);
4289  kfree(fpid);
4290  }
4291 
4292  ftrace_update_pid_func();
4293  ftrace_startup_enable(0);
4294 
4295  mutex_unlock(&ftrace_lock);
4296 }
4297 
4298 static void *fpid_start(struct seq_file *m, loff_t *pos)
4299 {
4300  mutex_lock(&ftrace_lock);
4301 
4302  if (list_empty(&ftrace_pids) && (!*pos))
4303  return (void *) 1;
4304 
4305  return seq_list_start(&ftrace_pids, *pos);
4306 }
4307 
4308 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
4309 {
4310  if (v == (void *)1)
4311  return NULL;
4312 
4313  return seq_list_next(v, &ftrace_pids, pos);
4314 }
4315 
4316 static void fpid_stop(struct seq_file *m, void *p)
4317 {
4318  mutex_unlock(&ftrace_lock);
4319 }
4320 
4321 static int fpid_show(struct seq_file *m, void *v)
4322 {
4323  const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
4324 
4325  if (v == (void *)1) {
4326  seq_printf(m, "no pid\n");
4327  return 0;
4328  }
4329 
4330  if (fpid->pid == ftrace_swapper_pid)
4331  seq_printf(m, "swapper tasks\n");
4332  else
4333  seq_printf(m, "%u\n", pid_vnr(fpid->pid));
4334 
4335  return 0;
4336 }
4337 
4338 static const struct seq_operations ftrace_pid_sops = {
4339  .start = fpid_start,
4340  .next = fpid_next,
4341  .stop = fpid_stop,
4342  .show = fpid_show,
4343 };
4344 
4345 static int
4346 ftrace_pid_open(struct inode *inode, struct file *file)
4347 {
4348  int ret = 0;
4349 
4350  if ((file->f_mode & FMODE_WRITE) &&
4351  (file->f_flags & O_TRUNC))
4352  ftrace_pid_reset();
4353 
4354  if (file->f_mode & FMODE_READ)
4355  ret = seq_open(file, &ftrace_pid_sops);
4356 
4357  return ret;
4358 }
4359 
4360 static ssize_t
4361 ftrace_pid_write(struct file *filp, const char __user *ubuf,
4362  size_t cnt, loff_t *ppos)
4363 {
4364  char buf[64], *tmp;
4365  long val;
4366  int ret;
4367 
4368  if (cnt >= sizeof(buf))
4369  return -EINVAL;
4370 
4371  if (copy_from_user(&buf, ubuf, cnt))
4372  return -EFAULT;
4373 
4374  buf[cnt] = 0;
4375 
4376  /*
4377  * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
4378  * to clean the filter quietly.
4379  */
4380  tmp = strstrip(buf);
4381  if (strlen(tmp) == 0)
4382  return 1;
4383 
4384  ret = strict_strtol(tmp, 10, &val);
4385  if (ret < 0)
4386  return ret;
4387 
4388  ret = ftrace_pid_add(val);
4389 
4390  return ret ? ret : cnt;
4391 }
4392 
4393 static int
4394 ftrace_pid_release(struct inode *inode, struct file *file)
4395 {
4396  if (file->f_mode & FMODE_READ)
4397  seq_release(inode, file);
4398 
4399  return 0;
4400 }
4401 
4402 static const struct file_operations ftrace_pid_fops = {
4403  .open = ftrace_pid_open,
4404  .write = ftrace_pid_write,
4405  .read = seq_read,
4406  .llseek = seq_lseek,
4407  .release = ftrace_pid_release,
4408 };
4409 
4410 static __init int ftrace_init_debugfs(void)
4411 {
4412  struct dentry *d_tracer;
4413 
4414  d_tracer = tracing_init_dentry();
4415  if (!d_tracer)
4416  return 0;
4417 
4418  ftrace_init_dyn_debugfs(d_tracer);
4419 
4420  trace_create_file("set_ftrace_pid", 0644, d_tracer,
4421  NULL, &ftrace_pid_fops);
4422 
4423  ftrace_profile_debugfs(d_tracer);
4424 
4425  return 0;
4426 }
4427 fs_initcall(ftrace_init_debugfs);
4428 
4436 void ftrace_kill(void)
4437 {
4438  ftrace_disabled = 1;
4439  ftrace_enabled = 0;
4441 }
4442 
4447 {
4448  return ftrace_disabled;
4449 }
4450 
4462 int register_ftrace_function(struct ftrace_ops *ops)
4463 {
4464  int ret = -1;
4465 
4466  mutex_lock(&ftrace_lock);
4467 
4468  ret = __register_ftrace_function(ops);
4469  if (!ret)
4470  ret = ftrace_startup(ops, 0);
4471 
4472  mutex_unlock(&ftrace_lock);
4473 
4474  return ret;
4475 }
4477 
4484 int unregister_ftrace_function(struct ftrace_ops *ops)
4485 {
4486  int ret;
4487 
4488  mutex_lock(&ftrace_lock);
4489  ret = __unregister_ftrace_function(ops);
4490  if (!ret)
4491  ftrace_shutdown(ops, 0);
4492  mutex_unlock(&ftrace_lock);
4493 
4494  return ret;
4495 }
4497 
4498 int
4500  void __user *buffer, size_t *lenp,
4501  loff_t *ppos)
4502 {
4503  int ret = -ENODEV;
4504 
4505  mutex_lock(&ftrace_lock);
4506 
4507  if (unlikely(ftrace_disabled))
4508  goto out;
4509 
4510  ret = proc_dointvec(table, write, buffer, lenp, ppos);
4511 
4512  if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
4513  goto out;
4514 
4515  last_ftrace_enabled = !!ftrace_enabled;
4516 
4517  if (ftrace_enabled) {
4518 
4520 
4521  /* we are starting ftrace again */
4522  if (ftrace_ops_list != &ftrace_list_end) {
4523  if (ftrace_ops_list->next == &ftrace_list_end)
4524  ftrace_trace_function = ftrace_ops_list->func;
4525  else
4526  ftrace_trace_function = ftrace_ops_list_func;
4527  }
4528 
4529  } else {
4530  /* stopping ftrace calls (just send to ftrace_stub) */
4531  ftrace_trace_function = ftrace_stub;
4532 
4534  }
4535 
4536  out:
4537  mutex_unlock(&ftrace_lock);
4538  return ret;
4539 }
4540 
4541 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4542 
4543 static int ftrace_graph_active;
4544 static struct notifier_block ftrace_suspend_notifier;
4545 
4546 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
4547 {
4548  return 0;
4549 }
4550 
4551 /* The callbacks that hook a function */
4552 trace_func_graph_ret_t ftrace_graph_return =
4553  (trace_func_graph_ret_t)ftrace_stub;
4554 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
4555 
4556 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
4557 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
4558 {
4559  int i;
4560  int ret = 0;
4561  unsigned long flags;
4562  int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
4563  struct task_struct *g, *t;
4564 
4565  for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
4566  ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
4567  * sizeof(struct ftrace_ret_stack),
4568  GFP_KERNEL);
4569  if (!ret_stack_list[i]) {
4570  start = 0;
4571  end = i;
4572  ret = -ENOMEM;
4573  goto free;
4574  }
4575  }
4576 
4577  read_lock_irqsave(&tasklist_lock, flags);
4578  do_each_thread(g, t) {
4579  if (start == end) {
4580  ret = -EAGAIN;
4581  goto unlock;
4582  }
4583 
4584  if (t->ret_stack == NULL) {
4585  atomic_set(&t->tracing_graph_pause, 0);
4586  atomic_set(&t->trace_overrun, 0);
4587  t->curr_ret_stack = -1;
4588  /* Make sure the tasks see the -1 first: */
4589  smp_wmb();
4590  t->ret_stack = ret_stack_list[start++];
4591  }
4592  } while_each_thread(g, t);
4593 
4594 unlock:
4595  read_unlock_irqrestore(&tasklist_lock, flags);
4596 free:
4597  for (i = start; i < end; i++)
4598  kfree(ret_stack_list[i]);
4599  return ret;
4600 }
4601 
4602 static void
4603 ftrace_graph_probe_sched_switch(void *ignore,
4604  struct task_struct *prev, struct task_struct *next)
4605 {
4606  unsigned long long timestamp;
4607  int index;
4608 
4609  /*
4610  * Does the user want to count the time a function was asleep.
4611  * If so, do not update the time stamps.
4612  */
4614  return;
4615 
4616  timestamp = trace_clock_local();
4617 
4618  prev->ftrace_timestamp = timestamp;
4619 
4620  /* only process tasks that we timestamped */
4621  if (!next->ftrace_timestamp)
4622  return;
4623 
4624  /*
4625  * Update all the counters in next to make up for the
4626  * time next was sleeping.
4627  */
4628  timestamp -= next->ftrace_timestamp;
4629 
4630  for (index = next->curr_ret_stack; index >= 0; index--)
4631  next->ret_stack[index].calltime += timestamp;
4632 }
4633 
4634 /* Allocate a return stack for each task */
4635 static int start_graph_tracing(void)
4636 {
4637  struct ftrace_ret_stack **ret_stack_list;
4638  int ret, cpu;
4639 
4640  ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
4641  sizeof(struct ftrace_ret_stack *),
4642  GFP_KERNEL);
4643 
4644  if (!ret_stack_list)
4645  return -ENOMEM;
4646 
4647  /* The cpu_boot init_task->ret_stack will never be freed */
4648  for_each_online_cpu(cpu) {
4649  if (!idle_task(cpu)->ret_stack)
4650  ftrace_graph_init_idle_task(idle_task(cpu), cpu);
4651  }
4652 
4653  do {
4654  ret = alloc_retstack_tasklist(ret_stack_list);
4655  } while (ret == -EAGAIN);
4656 
4657  if (!ret) {
4658  ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4659  if (ret)
4660  pr_info("ftrace_graph: Couldn't activate tracepoint"
4661  " probe to kernel_sched_switch\n");
4662  }
4663 
4664  kfree(ret_stack_list);
4665  return ret;
4666 }
4667 
4668 /*
4669  * Hibernation protection.
4670  * The state of the current task is too much unstable during
4671  * suspend/restore to disk. We want to protect against that.
4672  */
4673 static int
4674 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
4675  void *unused)
4676 {
4677  switch (state) {
4679  pause_graph_tracing();
4680  break;
4681 
4682  case PM_POST_HIBERNATION:
4683  unpause_graph_tracing();
4684  break;
4685  }
4686  return NOTIFY_DONE;
4687 }
4688 
4689 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
4690  trace_func_graph_ent_t entryfunc)
4691 {
4692  int ret = 0;
4693 
4694  mutex_lock(&ftrace_lock);
4695 
4696  /* we currently allow only one tracer registered at a time */
4697  if (ftrace_graph_active) {
4698  ret = -EBUSY;
4699  goto out;
4700  }
4701 
4702  ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
4703  register_pm_notifier(&ftrace_suspend_notifier);
4704 
4705  ftrace_graph_active++;
4706  ret = start_graph_tracing();
4707  if (ret) {
4708  ftrace_graph_active--;
4709  goto out;
4710  }
4711 
4712  ftrace_graph_return = retfunc;
4713  ftrace_graph_entry = entryfunc;
4714 
4715  ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
4716 
4717 out:
4718  mutex_unlock(&ftrace_lock);
4719  return ret;
4720 }
4721 
4722 void unregister_ftrace_graph(void)
4723 {
4724  mutex_lock(&ftrace_lock);
4725 
4726  if (unlikely(!ftrace_graph_active))
4727  goto out;
4728 
4729  ftrace_graph_active--;
4730  ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
4731  ftrace_graph_entry = ftrace_graph_entry_stub;
4732  ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
4733  unregister_pm_notifier(&ftrace_suspend_notifier);
4734  unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4735 
4736  out:
4737  mutex_unlock(&ftrace_lock);
4738 }
4739 
4740 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
4741 
4742 static void
4743 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
4744 {
4745  atomic_set(&t->tracing_graph_pause, 0);
4746  atomic_set(&t->trace_overrun, 0);
4747  t->ftrace_timestamp = 0;
4748  /* make curr_ret_stack visible before we add the ret_stack */
4749  smp_wmb();
4750  t->ret_stack = ret_stack;
4751 }
4752 
4753 /*
4754  * Allocate a return stack for the idle task. May be the first
4755  * time through, or it may be done by CPU hotplug online.
4756  */
4757 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
4758 {
4759  t->curr_ret_stack = -1;
4760  /*
4761  * The idle task has no parent, it either has its own
4762  * stack or no stack at all.
4763  */
4764  if (t->ret_stack)
4765  WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
4766 
4767  if (ftrace_graph_active) {
4768  struct ftrace_ret_stack *ret_stack;
4769 
4770  ret_stack = per_cpu(idle_ret_stack, cpu);
4771  if (!ret_stack) {
4772  ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4773  * sizeof(struct ftrace_ret_stack),
4774  GFP_KERNEL);
4775  if (!ret_stack)
4776  return;
4777  per_cpu(idle_ret_stack, cpu) = ret_stack;
4778  }
4779  graph_init_task(t, ret_stack);
4780  }
4781 }
4782 
4783 /* Allocate a return stack for newly created task */
4784 void ftrace_graph_init_task(struct task_struct *t)
4785 {
4786  /* Make sure we do not use the parent ret_stack */
4787  t->ret_stack = NULL;
4788  t->curr_ret_stack = -1;
4789 
4790  if (ftrace_graph_active) {
4791  struct ftrace_ret_stack *ret_stack;
4792 
4793  ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4794  * sizeof(struct ftrace_ret_stack),
4795  GFP_KERNEL);
4796  if (!ret_stack)
4797  return;
4798  graph_init_task(t, ret_stack);
4799  }
4800 }
4801 
4802 void ftrace_graph_exit_task(struct task_struct *t)
4803 {
4804  struct ftrace_ret_stack *ret_stack = t->ret_stack;
4805 
4806  t->ret_stack = NULL;
4807  /* NULL must become visible to IRQs before we free it: */
4808  barrier();
4809 
4810  kfree(ret_stack);
4811 }
4812 
4813 void ftrace_graph_stop(void)
4814 {
4815  ftrace_stop();
4816 }
4817 #endif