Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
tlb.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
3  * Licensed under the GPL
4  */
5 
6 #include <linux/mm.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <asm/pgtable.h>
10 #include <asm/tlbflush.h>
11 #include <as-layout.h>
12 #include <mem_user.h>
13 #include <os.h>
14 #include <skas.h>
15 
17  struct host_vm_op {
18  enum { NONE, MMAP, MUNMAP, MPROTECT } type;
19  union {
20  struct {
21  unsigned long addr;
22  unsigned long len;
23  unsigned int prot;
24  int fd;
26  } mmap;
27  struct {
28  unsigned long addr;
29  unsigned long len;
30  } munmap;
31  struct {
32  unsigned long addr;
33  unsigned long len;
34  unsigned int prot;
35  } mprotect;
36  } u;
37  } ops[1];
38  int index;
39  struct mm_id *id;
40  void *data;
41  int force;
42 };
43 
44 #define INIT_HVC(mm, force) \
45  ((struct host_vm_change) \
46  { .ops = { { .type = NONE } }, \
47  .id = &mm->context.id, \
48  .data = NULL, \
49  .index = 0, \
50  .force = force })
51 
52 static int do_ops(struct host_vm_change *hvc, int end,
53  int finished)
54 {
55  struct host_vm_op *op;
56  int i, ret = 0;
57 
58  for (i = 0; i < end && !ret; i++) {
59  op = &hvc->ops[i];
60  switch (op->type) {
61  case MMAP:
62  ret = map(hvc->id, op->u.mmap.addr, op->u.mmap.len,
63  op->u.mmap.prot, op->u.mmap.fd,
64  op->u.mmap.offset, finished, &hvc->data);
65  break;
66  case MUNMAP:
67  ret = unmap(hvc->id, op->u.munmap.addr,
68  op->u.munmap.len, finished, &hvc->data);
69  break;
70  case MPROTECT:
71  ret = protect(hvc->id, op->u.mprotect.addr,
72  op->u.mprotect.len, op->u.mprotect.prot,
73  finished, &hvc->data);
74  break;
75  default:
76  printk(KERN_ERR "Unknown op type %d in do_ops\n",
77  op->type);
78  BUG();
79  break;
80  }
81  }
82 
83  return ret;
84 }
85 
86 static int add_mmap(unsigned long virt, unsigned long phys, unsigned long len,
87  unsigned int prot, struct host_vm_change *hvc)
88 {
89  __u64 offset;
90  struct host_vm_op *last;
91  int fd, ret = 0;
92 
93  fd = phys_mapping(phys, &offset);
94  if (hvc->index != 0) {
95  last = &hvc->ops[hvc->index - 1];
96  if ((last->type == MMAP) &&
97  (last->u.mmap.addr + last->u.mmap.len == virt) &&
98  (last->u.mmap.prot == prot) && (last->u.mmap.fd == fd) &&
99  (last->u.mmap.offset + last->u.mmap.len == offset)) {
100  last->u.mmap.len += len;
101  return 0;
102  }
103  }
104 
105  if (hvc->index == ARRAY_SIZE(hvc->ops)) {
106  ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
107  hvc->index = 0;
108  }
109 
110  hvc->ops[hvc->index++] = ((struct host_vm_op)
111  { .type = MMAP,
112  .u = { .mmap = { .addr = virt,
113  .len = len,
114  .prot = prot,
115  .fd = fd,
116  .offset = offset }
117  } });
118  return ret;
119 }
120 
121 static int add_munmap(unsigned long addr, unsigned long len,
122  struct host_vm_change *hvc)
123 {
124  struct host_vm_op *last;
125  int ret = 0;
126 
127  if (hvc->index != 0) {
128  last = &hvc->ops[hvc->index - 1];
129  if ((last->type == MUNMAP) &&
130  (last->u.munmap.addr + last->u.mmap.len == addr)) {
131  last->u.munmap.len += len;
132  return 0;
133  }
134  }
135 
136  if (hvc->index == ARRAY_SIZE(hvc->ops)) {
137  ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
138  hvc->index = 0;
139  }
140 
141  hvc->ops[hvc->index++] = ((struct host_vm_op)
142  { .type = MUNMAP,
143  .u = { .munmap = { .addr = addr,
144  .len = len } } });
145  return ret;
146 }
147 
148 static int add_mprotect(unsigned long addr, unsigned long len,
149  unsigned int prot, struct host_vm_change *hvc)
150 {
151  struct host_vm_op *last;
152  int ret = 0;
153 
154  if (hvc->index != 0) {
155  last = &hvc->ops[hvc->index - 1];
156  if ((last->type == MPROTECT) &&
157  (last->u.mprotect.addr + last->u.mprotect.len == addr) &&
158  (last->u.mprotect.prot == prot)) {
159  last->u.mprotect.len += len;
160  return 0;
161  }
162  }
163 
164  if (hvc->index == ARRAY_SIZE(hvc->ops)) {
165  ret = do_ops(hvc, ARRAY_SIZE(hvc->ops), 0);
166  hvc->index = 0;
167  }
168 
169  hvc->ops[hvc->index++] = ((struct host_vm_op)
170  { .type = MPROTECT,
171  .u = { .mprotect = { .addr = addr,
172  .len = len,
173  .prot = prot } } });
174  return ret;
175 }
176 
177 #define ADD_ROUND(n, inc) (((n) + (inc)) & ~((inc) - 1))
178 
179 static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
180  unsigned long end,
181  struct host_vm_change *hvc)
182 {
183  pte_t *pte;
184  int r, w, x, prot, ret = 0;
185 
186  pte = pte_offset_kernel(pmd, addr);
187  do {
188  if ((addr >= STUB_START) && (addr < STUB_END))
189  continue;
190 
191  r = pte_read(*pte);
192  w = pte_write(*pte);
193  x = pte_exec(*pte);
194  if (!pte_young(*pte)) {
195  r = 0;
196  w = 0;
197  } else if (!pte_dirty(*pte))
198  w = 0;
199 
200  prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
201  (x ? UM_PROT_EXEC : 0));
202  if (hvc->force || pte_newpage(*pte)) {
203  if (pte_present(*pte))
204  ret = add_mmap(addr, pte_val(*pte) & PAGE_MASK,
205  PAGE_SIZE, prot, hvc);
206  else
207  ret = add_munmap(addr, PAGE_SIZE, hvc);
208  } else if (pte_newprot(*pte))
209  ret = add_mprotect(addr, PAGE_SIZE, prot, hvc);
210  *pte = pte_mkuptodate(*pte);
211  } while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret));
212  return ret;
213 }
214 
215 static inline int update_pmd_range(pud_t *pud, unsigned long addr,
216  unsigned long end,
217  struct host_vm_change *hvc)
218 {
219  pmd_t *pmd;
220  unsigned long next;
221  int ret = 0;
222 
223  pmd = pmd_offset(pud, addr);
224  do {
225  next = pmd_addr_end(addr, end);
226  if (!pmd_present(*pmd)) {
227  if (hvc->force || pmd_newpage(*pmd)) {
228  ret = add_munmap(addr, next - addr, hvc);
229  pmd_mkuptodate(*pmd);
230  }
231  }
232  else ret = update_pte_range(pmd, addr, next, hvc);
233  } while (pmd++, addr = next, ((addr < end) && !ret));
234  return ret;
235 }
236 
237 static inline int update_pud_range(pgd_t *pgd, unsigned long addr,
238  unsigned long end,
239  struct host_vm_change *hvc)
240 {
241  pud_t *pud;
242  unsigned long next;
243  int ret = 0;
244 
245  pud = pud_offset(pgd, addr);
246  do {
247  next = pud_addr_end(addr, end);
248  if (!pud_present(*pud)) {
249  if (hvc->force || pud_newpage(*pud)) {
250  ret = add_munmap(addr, next - addr, hvc);
251  pud_mkuptodate(*pud);
252  }
253  }
254  else ret = update_pmd_range(pud, addr, next, hvc);
255  } while (pud++, addr = next, ((addr < end) && !ret));
256  return ret;
257 }
258 
259 void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
260  unsigned long end_addr, int force)
261 {
262  pgd_t *pgd;
263  struct host_vm_change hvc;
264  unsigned long addr = start_addr, next;
265  int ret = 0;
266 
267  hvc = INIT_HVC(mm, force);
268  pgd = pgd_offset(mm, addr);
269  do {
270  next = pgd_addr_end(addr, end_addr);
271  if (!pgd_present(*pgd)) {
272  if (force || pgd_newpage(*pgd)) {
273  ret = add_munmap(addr, next - addr, &hvc);
274  pgd_mkuptodate(*pgd);
275  }
276  }
277  else ret = update_pud_range(pgd, addr, next, &hvc);
278  } while (pgd++, addr = next, ((addr < end_addr) && !ret));
279 
280  if (!ret)
281  ret = do_ops(&hvc, hvc.index, 1);
282 
283  /* This is not an else because ret is modified above */
284  if (ret) {
285  printk(KERN_ERR "fix_range_common: failed, killing current "
286  "process\n");
288  }
289 }
290 
291 static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
292 {
293  struct mm_struct *mm;
294  pgd_t *pgd;
295  pud_t *pud;
296  pmd_t *pmd;
297  pte_t *pte;
298  unsigned long addr, last;
299  int updated = 0, err;
300 
301  mm = &init_mm;
302  for (addr = start; addr < end;) {
303  pgd = pgd_offset(mm, addr);
304  if (!pgd_present(*pgd)) {
305  last = ADD_ROUND(addr, PGDIR_SIZE);
306  if (last > end)
307  last = end;
308  if (pgd_newpage(*pgd)) {
309  updated = 1;
310  err = os_unmap_memory((void *) addr,
311  last - addr);
312  if (err < 0)
313  panic("munmap failed, errno = %d\n",
314  -err);
315  }
316  addr = last;
317  continue;
318  }
319 
320  pud = pud_offset(pgd, addr);
321  if (!pud_present(*pud)) {
322  last = ADD_ROUND(addr, PUD_SIZE);
323  if (last > end)
324  last = end;
325  if (pud_newpage(*pud)) {
326  updated = 1;
327  err = os_unmap_memory((void *) addr,
328  last - addr);
329  if (err < 0)
330  panic("munmap failed, errno = %d\n",
331  -err);
332  }
333  addr = last;
334  continue;
335  }
336 
337  pmd = pmd_offset(pud, addr);
338  if (!pmd_present(*pmd)) {
339  last = ADD_ROUND(addr, PMD_SIZE);
340  if (last > end)
341  last = end;
342  if (pmd_newpage(*pmd)) {
343  updated = 1;
344  err = os_unmap_memory((void *) addr,
345  last - addr);
346  if (err < 0)
347  panic("munmap failed, errno = %d\n",
348  -err);
349  }
350  addr = last;
351  continue;
352  }
353 
354  pte = pte_offset_kernel(pmd, addr);
355  if (!pte_present(*pte) || pte_newpage(*pte)) {
356  updated = 1;
357  err = os_unmap_memory((void *) addr,
358  PAGE_SIZE);
359  if (err < 0)
360  panic("munmap failed, errno = %d\n",
361  -err);
362  if (pte_present(*pte))
363  map_memory(addr,
364  pte_val(*pte) & PAGE_MASK,
365  PAGE_SIZE, 1, 1, 1);
366  }
367  else if (pte_newprot(*pte)) {
368  updated = 1;
369  os_protect_memory((void *) addr, PAGE_SIZE, 1, 1, 1);
370  }
371  addr += PAGE_SIZE;
372  }
373  return updated;
374 }
375 
376 void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
377 {
378  pgd_t *pgd;
379  pud_t *pud;
380  pmd_t *pmd;
381  pte_t *pte;
382  struct mm_struct *mm = vma->vm_mm;
383  void *flush = NULL;
384  int r, w, x, prot, err = 0;
385  struct mm_id *mm_id;
386 
387  address &= PAGE_MASK;
388  pgd = pgd_offset(mm, address);
389  if (!pgd_present(*pgd))
390  goto kill;
391 
392  pud = pud_offset(pgd, address);
393  if (!pud_present(*pud))
394  goto kill;
395 
396  pmd = pmd_offset(pud, address);
397  if (!pmd_present(*pmd))
398  goto kill;
399 
400  pte = pte_offset_kernel(pmd, address);
401 
402  r = pte_read(*pte);
403  w = pte_write(*pte);
404  x = pte_exec(*pte);
405  if (!pte_young(*pte)) {
406  r = 0;
407  w = 0;
408  } else if (!pte_dirty(*pte)) {
409  w = 0;
410  }
411 
412  mm_id = &mm->context.id;
413  prot = ((r ? UM_PROT_READ : 0) | (w ? UM_PROT_WRITE : 0) |
414  (x ? UM_PROT_EXEC : 0));
415  if (pte_newpage(*pte)) {
416  if (pte_present(*pte)) {
417  unsigned long long offset;
418  int fd;
419 
420  fd = phys_mapping(pte_val(*pte) & PAGE_MASK, &offset);
421  err = map(mm_id, address, PAGE_SIZE, prot, fd, offset,
422  1, &flush);
423  }
424  else err = unmap(mm_id, address, PAGE_SIZE, 1, &flush);
425  }
426  else if (pte_newprot(*pte))
427  err = protect(mm_id, address, PAGE_SIZE, prot, 1, &flush);
428 
429  if (err)
430  goto kill;
431 
432  *pte = pte_mkuptodate(*pte);
433 
434  return;
435 
436 kill:
437  printk(KERN_ERR "Failed to flush page for address 0x%lx\n", address);
439 }
440 
441 pgd_t *pgd_offset_proc(struct mm_struct *mm, unsigned long address)
442 {
443  return pgd_offset(mm, address);
444 }
445 
446 pud_t *pud_offset_proc(pgd_t *pgd, unsigned long address)
447 {
448  return pud_offset(pgd, address);
449 }
450 
451 pmd_t *pmd_offset_proc(pud_t *pud, unsigned long address)
452 {
453  return pmd_offset(pud, address);
454 }
455 
456 pte_t *pte_offset_proc(pmd_t *pmd, unsigned long address)
457 {
458  return pte_offset_kernel(pmd, address);
459 }
460 
461 pte_t *addr_pte(struct task_struct *task, unsigned long addr)
462 {
463  pgd_t *pgd = pgd_offset(task->mm, addr);
464  pud_t *pud = pud_offset(pgd, addr);
465  pmd_t *pmd = pmd_offset(pud, addr);
466 
467  return pte_offset_map(pmd, addr);
468 }
469 
470 void flush_tlb_all(void)
471 {
472  flush_tlb_mm(current->mm);
473 }
474 
475 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
476 {
477  flush_tlb_kernel_range_common(start, end);
478 }
479 
481 {
482  flush_tlb_kernel_range_common(start_vm, end_vm);
483 }
484 
485 void __flush_tlb_one(unsigned long addr)
486 {
487  flush_tlb_kernel_range_common(addr, addr + PAGE_SIZE);
488 }
489 
490 static void fix_range(struct mm_struct *mm, unsigned long start_addr,
491  unsigned long end_addr, int force)
492 {
493  fix_range_common(mm, start_addr, end_addr, force);
494 }
495 
496 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
497  unsigned long end)
498 {
499  if (vma->vm_mm == NULL)
500  flush_tlb_kernel_range_common(start, end);
501  else fix_range(vma->vm_mm, start, end, 0);
502 }
504 
505 void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
506  unsigned long end)
507 {
508  /*
509  * Don't bother flushing if this address space is about to be
510  * destroyed.
511  */
512  if (atomic_read(&mm->mm_users) == 0)
513  return;
514 
515  fix_range(mm, start, end, 0);
516 }
517 
518 void flush_tlb_mm(struct mm_struct *mm)
519 {
520  struct vm_area_struct *vma = mm->mmap;
521 
522  while (vma != NULL) {
523  fix_range(mm, vma->vm_start, vma->vm_end, 0);
524  vma = vma->vm_next;
525  }
526 }
527 
528 void force_flush_all(void)
529 {
530  struct mm_struct *mm = current->mm;
531  struct vm_area_struct *vma = mm->mmap;
532 
533  while (vma != NULL) {
534  fix_range(mm, vma->vm_start, vma->vm_end, 1);
535  vma = vma->vm_next;
536  }
537 }