Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
r4kcache.h
Go to the documentation of this file.
1 /*
2  * This file is subject to the terms and conditions of the GNU General Public
3  * License. See the file "COPYING" in the main directory of this archive
4  * for more details.
5  *
6  * Inline assembly cache operations.
7  *
8  * Copyright (C) 1996 David S. Miller ([email protected])
9  * Copyright (C) 1997 - 2002 Ralf Baechle ([email protected])
10  * Copyright (C) 2004 Ralf Baechle ([email protected])
11  */
12 #ifndef _ASM_R4KCACHE_H
13 #define _ASM_R4KCACHE_H
14 
15 #include <asm/asm.h>
16 #include <asm/cacheops.h>
17 #include <asm/cpu-features.h>
18 #include <asm/mipsmtregs.h>
19 
20 /*
21  * This macro return a properly sign-extended address suitable as base address
22  * for indexed cache operations. Two issues here:
23  *
24  * - The MIPS32 and MIPS64 specs permit an implementation to directly derive
25  * the index bits from the virtual address. This breaks with tradition
26  * set by the R4000. To keep unpleasant surprises from happening we pick
27  * an address in KSEG0 / CKSEG0.
28  * - We need a properly sign extended address for 64-bit code. To get away
29  * without ifdefs we let the compiler do it by a type cast.
30  */
31 #define INDEX_BASE CKSEG0
32 
33 #define cache_op(op,addr) \
34  __asm__ __volatile__( \
35  " .set push \n" \
36  " .set noreorder \n" \
37  " .set mips3\n\t \n" \
38  " cache %0, %1 \n" \
39  " .set pop \n" \
40  : \
41  : "i" (op), "R" (*(unsigned char *)(addr)))
42 
43 #ifdef CONFIG_MIPS_MT
44 /*
45  * Temporary hacks for SMTC debug. Optionally force single-threaded
46  * execution during I-cache flushes.
47  */
48 
49 #define PROTECT_CACHE_FLUSHES 1
50 
51 #ifdef PROTECT_CACHE_FLUSHES
52 
53 extern int mt_protiflush;
54 extern int mt_protdflush;
55 extern void mt_cflush_lockdown(void);
56 extern void mt_cflush_release(void);
57 
58 #define BEGIN_MT_IPROT \
59  unsigned long flags = 0; \
60  unsigned long mtflags = 0; \
61  if(mt_protiflush) { \
62  local_irq_save(flags); \
63  ehb(); \
64  mtflags = dvpe(); \
65  mt_cflush_lockdown(); \
66  }
67 
68 #define END_MT_IPROT \
69  if(mt_protiflush) { \
70  mt_cflush_release(); \
71  evpe(mtflags); \
72  local_irq_restore(flags); \
73  }
74 
75 #define BEGIN_MT_DPROT \
76  unsigned long flags = 0; \
77  unsigned long mtflags = 0; \
78  if(mt_protdflush) { \
79  local_irq_save(flags); \
80  ehb(); \
81  mtflags = dvpe(); \
82  mt_cflush_lockdown(); \
83  }
84 
85 #define END_MT_DPROT \
86  if(mt_protdflush) { \
87  mt_cflush_release(); \
88  evpe(mtflags); \
89  local_irq_restore(flags); \
90  }
91 
92 #else
93 
94 #define BEGIN_MT_IPROT
95 #define BEGIN_MT_DPROT
96 #define END_MT_IPROT
97 #define END_MT_DPROT
98 
99 #endif /* PROTECT_CACHE_FLUSHES */
100 
101 #define __iflush_prologue \
102  unsigned long redundance; \
103  extern int mt_n_iflushes; \
104  BEGIN_MT_IPROT \
105  for (redundance = 0; redundance < mt_n_iflushes; redundance++) {
106 
107 #define __iflush_epilogue \
108  END_MT_IPROT \
109  }
110 
111 #define __dflush_prologue \
112  unsigned long redundance; \
113  extern int mt_n_dflushes; \
114  BEGIN_MT_DPROT \
115  for (redundance = 0; redundance < mt_n_dflushes; redundance++) {
116 
117 #define __dflush_epilogue \
118  END_MT_DPROT \
119  }
120 
121 #define __inv_dflush_prologue __dflush_prologue
122 #define __inv_dflush_epilogue __dflush_epilogue
123 #define __sflush_prologue {
124 #define __sflush_epilogue }
125 #define __inv_sflush_prologue __sflush_prologue
126 #define __inv_sflush_epilogue __sflush_epilogue
127 
128 #else /* CONFIG_MIPS_MT */
129 
130 #define __iflush_prologue {
131 #define __iflush_epilogue }
132 #define __dflush_prologue {
133 #define __dflush_epilogue }
134 #define __inv_dflush_prologue {
135 #define __inv_dflush_epilogue }
136 #define __sflush_prologue {
137 #define __sflush_epilogue }
138 #define __inv_sflush_prologue {
139 #define __inv_sflush_epilogue }
140 
141 #endif /* CONFIG_MIPS_MT */
142 
143 static inline void flush_icache_line_indexed(unsigned long addr)
144 {
145  __iflush_prologue
146  cache_op(Index_Invalidate_I, addr);
147  __iflush_epilogue
148 }
149 
150 static inline void flush_dcache_line_indexed(unsigned long addr)
151 {
152  __dflush_prologue
153  cache_op(Index_Writeback_Inv_D, addr);
154  __dflush_epilogue
155 }
156 
157 static inline void flush_scache_line_indexed(unsigned long addr)
158 {
159  cache_op(Index_Writeback_Inv_SD, addr);
160 }
161 
162 static inline void flush_icache_line(unsigned long addr)
163 {
164  __iflush_prologue
165  cache_op(Hit_Invalidate_I, addr);
166  __iflush_epilogue
167 }
168 
169 static inline void flush_dcache_line(unsigned long addr)
170 {
171  __dflush_prologue
172  cache_op(Hit_Writeback_Inv_D, addr);
173  __dflush_epilogue
174 }
175 
176 static inline void invalidate_dcache_line(unsigned long addr)
177 {
178  __dflush_prologue
179  cache_op(Hit_Invalidate_D, addr);
180  __dflush_epilogue
181 }
182 
183 static inline void invalidate_scache_line(unsigned long addr)
184 {
185  cache_op(Hit_Invalidate_SD, addr);
186 }
187 
188 static inline void flush_scache_line(unsigned long addr)
189 {
190  cache_op(Hit_Writeback_Inv_SD, addr);
191 }
192 
193 #define protected_cache_op(op,addr) \
194  __asm__ __volatile__( \
195  " .set push \n" \
196  " .set noreorder \n" \
197  " .set mips3 \n" \
198  "1: cache %0, (%1) \n" \
199  "2: .set pop \n" \
200  " .section __ex_table,\"a\" \n" \
201  " "STR(PTR)" 1b, 2b \n" \
202  " .previous" \
203  : \
204  : "i" (op), "r" (addr))
205 
206 /*
207  * The next two are for badland addresses like signal trampolines.
208  */
209 static inline void protected_flush_icache_line(unsigned long addr)
210 {
211  protected_cache_op(Hit_Invalidate_I, addr);
212 }
213 
214 /*
215  * R10000 / R12000 hazard - these processors don't support the Hit_Writeback_D
216  * cacheop so we use Hit_Writeback_Inv_D which is supported by all R4000-style
217  * caches. We're talking about one cacheline unnecessarily getting invalidated
218  * here so the penalty isn't overly hard.
219  */
220 static inline void protected_writeback_dcache_line(unsigned long addr)
221 {
222  protected_cache_op(Hit_Writeback_Inv_D, addr);
223 }
224 
225 static inline void protected_writeback_scache_line(unsigned long addr)
226 {
227  protected_cache_op(Hit_Writeback_Inv_SD, addr);
228 }
229 
230 /*
231  * This one is RM7000-specific
232  */
233 static inline void invalidate_tcache_page(unsigned long addr)
234 {
235  cache_op(Page_Invalidate_T, addr);
236 }
237 
238 #define cache16_unroll32(base,op) \
239  __asm__ __volatile__( \
240  " .set push \n" \
241  " .set noreorder \n" \
242  " .set mips3 \n" \
243  " cache %1, 0x000(%0); cache %1, 0x010(%0) \n" \
244  " cache %1, 0x020(%0); cache %1, 0x030(%0) \n" \
245  " cache %1, 0x040(%0); cache %1, 0x050(%0) \n" \
246  " cache %1, 0x060(%0); cache %1, 0x070(%0) \n" \
247  " cache %1, 0x080(%0); cache %1, 0x090(%0) \n" \
248  " cache %1, 0x0a0(%0); cache %1, 0x0b0(%0) \n" \
249  " cache %1, 0x0c0(%0); cache %1, 0x0d0(%0) \n" \
250  " cache %1, 0x0e0(%0); cache %1, 0x0f0(%0) \n" \
251  " cache %1, 0x100(%0); cache %1, 0x110(%0) \n" \
252  " cache %1, 0x120(%0); cache %1, 0x130(%0) \n" \
253  " cache %1, 0x140(%0); cache %1, 0x150(%0) \n" \
254  " cache %1, 0x160(%0); cache %1, 0x170(%0) \n" \
255  " cache %1, 0x180(%0); cache %1, 0x190(%0) \n" \
256  " cache %1, 0x1a0(%0); cache %1, 0x1b0(%0) \n" \
257  " cache %1, 0x1c0(%0); cache %1, 0x1d0(%0) \n" \
258  " cache %1, 0x1e0(%0); cache %1, 0x1f0(%0) \n" \
259  " .set pop \n" \
260  : \
261  : "r" (base), \
262  "i" (op));
263 
264 #define cache32_unroll32(base,op) \
265  __asm__ __volatile__( \
266  " .set push \n" \
267  " .set noreorder \n" \
268  " .set mips3 \n" \
269  " cache %1, 0x000(%0); cache %1, 0x020(%0) \n" \
270  " cache %1, 0x040(%0); cache %1, 0x060(%0) \n" \
271  " cache %1, 0x080(%0); cache %1, 0x0a0(%0) \n" \
272  " cache %1, 0x0c0(%0); cache %1, 0x0e0(%0) \n" \
273  " cache %1, 0x100(%0); cache %1, 0x120(%0) \n" \
274  " cache %1, 0x140(%0); cache %1, 0x160(%0) \n" \
275  " cache %1, 0x180(%0); cache %1, 0x1a0(%0) \n" \
276  " cache %1, 0x1c0(%0); cache %1, 0x1e0(%0) \n" \
277  " cache %1, 0x200(%0); cache %1, 0x220(%0) \n" \
278  " cache %1, 0x240(%0); cache %1, 0x260(%0) \n" \
279  " cache %1, 0x280(%0); cache %1, 0x2a0(%0) \n" \
280  " cache %1, 0x2c0(%0); cache %1, 0x2e0(%0) \n" \
281  " cache %1, 0x300(%0); cache %1, 0x320(%0) \n" \
282  " cache %1, 0x340(%0); cache %1, 0x360(%0) \n" \
283  " cache %1, 0x380(%0); cache %1, 0x3a0(%0) \n" \
284  " cache %1, 0x3c0(%0); cache %1, 0x3e0(%0) \n" \
285  " .set pop \n" \
286  : \
287  : "r" (base), \
288  "i" (op));
289 
290 #define cache64_unroll32(base,op) \
291  __asm__ __volatile__( \
292  " .set push \n" \
293  " .set noreorder \n" \
294  " .set mips3 \n" \
295  " cache %1, 0x000(%0); cache %1, 0x040(%0) \n" \
296  " cache %1, 0x080(%0); cache %1, 0x0c0(%0) \n" \
297  " cache %1, 0x100(%0); cache %1, 0x140(%0) \n" \
298  " cache %1, 0x180(%0); cache %1, 0x1c0(%0) \n" \
299  " cache %1, 0x200(%0); cache %1, 0x240(%0) \n" \
300  " cache %1, 0x280(%0); cache %1, 0x2c0(%0) \n" \
301  " cache %1, 0x300(%0); cache %1, 0x340(%0) \n" \
302  " cache %1, 0x380(%0); cache %1, 0x3c0(%0) \n" \
303  " cache %1, 0x400(%0); cache %1, 0x440(%0) \n" \
304  " cache %1, 0x480(%0); cache %1, 0x4c0(%0) \n" \
305  " cache %1, 0x500(%0); cache %1, 0x540(%0) \n" \
306  " cache %1, 0x580(%0); cache %1, 0x5c0(%0) \n" \
307  " cache %1, 0x600(%0); cache %1, 0x640(%0) \n" \
308  " cache %1, 0x680(%0); cache %1, 0x6c0(%0) \n" \
309  " cache %1, 0x700(%0); cache %1, 0x740(%0) \n" \
310  " cache %1, 0x780(%0); cache %1, 0x7c0(%0) \n" \
311  " .set pop \n" \
312  : \
313  : "r" (base), \
314  "i" (op));
315 
316 #define cache128_unroll32(base,op) \
317  __asm__ __volatile__( \
318  " .set push \n" \
319  " .set noreorder \n" \
320  " .set mips3 \n" \
321  " cache %1, 0x000(%0); cache %1, 0x080(%0) \n" \
322  " cache %1, 0x100(%0); cache %1, 0x180(%0) \n" \
323  " cache %1, 0x200(%0); cache %1, 0x280(%0) \n" \
324  " cache %1, 0x300(%0); cache %1, 0x380(%0) \n" \
325  " cache %1, 0x400(%0); cache %1, 0x480(%0) \n" \
326  " cache %1, 0x500(%0); cache %1, 0x580(%0) \n" \
327  " cache %1, 0x600(%0); cache %1, 0x680(%0) \n" \
328  " cache %1, 0x700(%0); cache %1, 0x780(%0) \n" \
329  " cache %1, 0x800(%0); cache %1, 0x880(%0) \n" \
330  " cache %1, 0x900(%0); cache %1, 0x980(%0) \n" \
331  " cache %1, 0xa00(%0); cache %1, 0xa80(%0) \n" \
332  " cache %1, 0xb00(%0); cache %1, 0xb80(%0) \n" \
333  " cache %1, 0xc00(%0); cache %1, 0xc80(%0) \n" \
334  " cache %1, 0xd00(%0); cache %1, 0xd80(%0) \n" \
335  " cache %1, 0xe00(%0); cache %1, 0xe80(%0) \n" \
336  " cache %1, 0xf00(%0); cache %1, 0xf80(%0) \n" \
337  " .set pop \n" \
338  : \
339  : "r" (base), \
340  "i" (op));
341 
342 /* build blast_xxx, blast_xxx_page, blast_xxx_page_indexed */
343 #define __BUILD_BLAST_CACHE(pfx, desc, indexop, hitop, lsize) \
344 static inline void blast_##pfx##cache##lsize(void) \
345 { \
346  unsigned long start = INDEX_BASE; \
347  unsigned long end = start + current_cpu_data.desc.waysize; \
348  unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \
349  unsigned long ws_end = current_cpu_data.desc.ways << \
350  current_cpu_data.desc.waybit; \
351  unsigned long ws, addr; \
352  \
353  __##pfx##flush_prologue \
354  \
355  for (ws = 0; ws < ws_end; ws += ws_inc) \
356  for (addr = start; addr < end; addr += lsize * 32) \
357  cache##lsize##_unroll32(addr|ws, indexop); \
358  \
359  __##pfx##flush_epilogue \
360 } \
361  \
362 static inline void blast_##pfx##cache##lsize##_page(unsigned long page) \
363 { \
364  unsigned long start = page; \
365  unsigned long end = page + PAGE_SIZE; \
366  \
367  __##pfx##flush_prologue \
368  \
369  do { \
370  cache##lsize##_unroll32(start, hitop); \
371  start += lsize * 32; \
372  } while (start < end); \
373  \
374  __##pfx##flush_epilogue \
375 } \
376  \
377 static inline void blast_##pfx##cache##lsize##_page_indexed(unsigned long page) \
378 { \
379  unsigned long indexmask = current_cpu_data.desc.waysize - 1; \
380  unsigned long start = INDEX_BASE + (page & indexmask); \
381  unsigned long end = start + PAGE_SIZE; \
382  unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \
383  unsigned long ws_end = current_cpu_data.desc.ways << \
384  current_cpu_data.desc.waybit; \
385  unsigned long ws, addr; \
386  \
387  __##pfx##flush_prologue \
388  \
389  for (ws = 0; ws < ws_end; ws += ws_inc) \
390  for (addr = start; addr < end; addr += lsize * 32) \
391  cache##lsize##_unroll32(addr|ws, indexop); \
392  \
393  __##pfx##flush_epilogue \
394 }
395 
396 __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 16)
397 __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16)
398 __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 16)
399 __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 32)
400 __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32)
401 __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 32)
402 __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 64)
403 __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64)
404 __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 64)
405 __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 128)
406 
407 __BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 16)
408 __BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 32)
409 __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 16)
410 __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 32)
411 __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 64)
412 __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 128)
413 
414 /* build blast_xxx_range, protected_blast_xxx_range */
415 #define __BUILD_BLAST_CACHE_RANGE(pfx, desc, hitop, prot) \
416 static inline void prot##blast_##pfx##cache##_range(unsigned long start, \
417  unsigned long end) \
418 { \
419  unsigned long lsize = cpu_##desc##_line_size(); \
420  unsigned long addr = start & ~(lsize - 1); \
421  unsigned long aend = (end - 1) & ~(lsize - 1); \
422  \
423  __##pfx##flush_prologue \
424  \
425  while (1) { \
426  prot##cache_op(hitop, addr); \
427  if (addr == aend) \
428  break; \
429  addr += lsize; \
430  } \
431  \
432  __##pfx##flush_epilogue \
433 }
434 
435 __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, protected_)
436 __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, protected_)
437 __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, protected_)
438 __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, )
439 __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, )
440 /* blast_inv_dcache_range */
441 __BUILD_BLAST_CACHE_RANGE(inv_d, dcache, Hit_Invalidate_D, )
442 __BUILD_BLAST_CACHE_RANGE(inv_s, scache, Hit_Invalidate_SD, )
443 
444 #endif /* _ASM_R4KCACHE_H */
Generated on Thu Jan 10 2013 13:10:51 for Linux Kernel by   doxygen 1.8.2