Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
tlb.h
Go to the documentation of this file.
1 /* include/asm-generic/tlb.h
2  *
3  * Generic TLB shootdown code
4  *
5  * Copyright 2001 Red Hat, Inc.
6  * Based on code from mm/memory.c Copyright Linus Torvalds and others.
7  *
8  * Copyright 2011 Red Hat, Inc., Peter Zijlstra <[email protected]>
9  *
10  * This program is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU General Public License
12  * as published by the Free Software Foundation; either version
13  * 2 of the License, or (at your option) any later version.
14  */
15 #ifndef _ASM_GENERIC__TLB_H
16 #define _ASM_GENERIC__TLB_H
17 
18 #include <linux/swap.h>
19 #include <asm/pgalloc.h>
20 #include <asm/tlbflush.h>
21 
22 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
23 /*
24  * Semi RCU freeing of the page directories.
25  *
26  * This is needed by some architectures to implement software pagetable walkers.
27  *
28  * gup_fast() and other software pagetable walkers do a lockless page-table
29  * walk and therefore needs some synchronization with the freeing of the page
30  * directories. The chosen means to accomplish that is by disabling IRQs over
31  * the walk.
32  *
33  * Architectures that use IPIs to flush TLBs will then automagically DTRT,
34  * since we unlink the page, flush TLBs, free the page. Since the disabling of
35  * IRQs delays the completion of the TLB flush we can never observe an already
36  * freed page.
37  *
38  * Architectures that do not have this (PPC) need to delay the freeing by some
39  * other means, this is that means.
40  *
41  * What we do is batch the freed directory pages (tables) and RCU free them.
42  * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
43  * holds off grace periods.
44  *
45  * However, in order to batch these pages we need to allocate storage, this
46  * allocation is deep inside the MM code and can thus easily fail on memory
47  * pressure. To guarantee progress we fall back to single table freeing, see
48  * the implementation of tlb_remove_table_one().
49  *
50  */
51 struct mmu_table_batch {
52  struct rcu_head rcu;
53  unsigned int nr;
54  void *tables[0];
55 };
56 
57 #define MAX_TABLE_BATCH \
58  ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
59 
60 extern void tlb_table_flush(struct mmu_gather *tlb);
61 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
62 
63 #endif
64 
65 /*
66  * If we can't allocate a page to make a big batch of page pointers
67  * to work on, then just handle a few from the on-stack structure.
68  */
69 #define MMU_GATHER_BUNDLE 8
70 
73  unsigned int nr;
74  unsigned int max;
75  struct page *pages[0];
76 };
77 
78 #define MAX_GATHER_BATCH \
79  ((PAGE_SIZE - sizeof(struct mmu_gather_batch)) / sizeof(void *))
80 
81 /* struct mmu_gather is an opaque type used by the mm code for passing around
82  * any data needed by arch specific code for tlb_remove_page.
83  */
84 struct mmu_gather {
85  struct mm_struct *mm;
86 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
87  struct mmu_table_batch *batch;
88 #endif
89  unsigned long start;
90  unsigned long end;
91  unsigned int need_flush : 1, /* Did free PTEs */
92  fast_mode : 1; /* No batching */
93 
94  unsigned int fullmm;
95 
99 };
100 
101 #define HAVE_GENERIC_MMU_GATHER
102 
103 static inline int tlb_fast_mode(struct mmu_gather *tlb)
104 {
105 #ifdef CONFIG_SMP
106  return tlb->fast_mode;
107 #else
108  /*
109  * For UP we don't need to worry about TLB flush
110  * and page free order so much..
111  */
112  return 1;
113 #endif
114 }
115 
116 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm);
117 void tlb_flush_mmu(struct mmu_gather *tlb);
118 void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
119  unsigned long end);
120 int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
121 
122 /* tlb_remove_page
123  * Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when
124  * required.
125  */
126 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
127 {
128  if (!__tlb_remove_page(tlb, page))
129  tlb_flush_mmu(tlb);
130 }
131 
139 #define tlb_remove_tlb_entry(tlb, ptep, address) \
140  do { \
141  tlb->need_flush = 1; \
142  __tlb_remove_tlb_entry(tlb, ptep, address); \
143  } while (0)
144 
149 #ifndef __tlb_remove_pmd_tlb_entry
150 #define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0)
151 #endif
152 
153 #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \
154  do { \
155  tlb->need_flush = 1; \
156  __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
157  } while (0)
158 
159 #define pte_free_tlb(tlb, ptep, address) \
160  do { \
161  tlb->need_flush = 1; \
162  __pte_free_tlb(tlb, ptep, address); \
163  } while (0)
164 
165 #ifndef __ARCH_HAS_4LEVEL_HACK
166 #define pud_free_tlb(tlb, pudp, address) \
167  do { \
168  tlb->need_flush = 1; \
169  __pud_free_tlb(tlb, pudp, address); \
170  } while (0)
171 #endif
172 
173 #define pmd_free_tlb(tlb, pmdp, address) \
174  do { \
175  tlb->need_flush = 1; \
176  __pmd_free_tlb(tlb, pmdp, address); \
177  } while (0)
178 
179 #define tlb_migrate_finish(mm) do {} while (0)
180 
181 #endif /* _ASM_GENERIC__TLB_H */