Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
fault.c
Go to the documentation of this file.
1 /*
2  * Copyright 2010 Tilera Corporation. All Rights Reserved.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation, version 2.
7  *
8  * This program is distributed in the hope that it will be useful, but
9  * WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11  * NON INFRINGEMENT. See the GNU General Public License for
12  * more details.
13  *
14  * From i386 code copyright (C) 1995 Linus Torvalds
15  */
16 
17 #include <linux/signal.h>
18 #include <linux/sched.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/string.h>
22 #include <linux/types.h>
23 #include <linux/ptrace.h>
24 #include <linux/mman.h>
25 #include <linux/mm.h>
26 #include <linux/smp.h>
27 #include <linux/interrupt.h>
28 #include <linux/init.h>
29 #include <linux/tty.h>
30 #include <linux/vt_kern.h> /* For unblank_screen() */
31 #include <linux/highmem.h>
32 #include <linux/module.h>
33 #include <linux/kprobes.h>
34 #include <linux/hugetlb.h>
35 #include <linux/syscalls.h>
36 #include <linux/uaccess.h>
37 
38 #include <asm/pgalloc.h>
39 #include <asm/sections.h>
40 #include <asm/traps.h>
41 #include <asm/syscalls.h>
42 
43 #include <arch/interrupts.h>
44 
45 static noinline void force_sig_info_fault(const char *type, int si_signo,
46  int si_code, unsigned long address,
47  int fault_num,
48  struct task_struct *tsk,
49  struct pt_regs *regs)
50 {
52 
53  if (unlikely(tsk->pid < 2)) {
54  panic("Signal %d (code %d) at %#lx sent to %s!",
55  si_signo, si_code & 0xffff, address,
56  is_idle_task(tsk) ? "the idle task" : "init");
57  }
58 
59  info.si_signo = si_signo;
60  info.si_errno = 0;
61  info.si_code = si_code;
62  info.si_addr = (void __user *)address;
63  info.si_trapno = fault_num;
64  trace_unhandled_signal(type, regs, address, si_signo);
65  force_sig_info(si_signo, &info, tsk);
66 }
67 
68 #ifndef __tilegx__
69 /*
70  * Synthesize the fault a PL0 process would get by doing a word-load of
71  * an unaligned address or a high kernel address.
72  */
73 SYSCALL_DEFINE2(cmpxchg_badaddr, unsigned long, address,
74  struct pt_regs *, regs)
75 {
76  if (address >= PAGE_OFFSET)
77  force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR,
78  address, INT_DTLB_MISS, current, regs);
79  else
80  force_sig_info_fault("atomic alignment fault", SIGBUS,
81  BUS_ADRALN, address,
82  INT_UNALIGN_DATA, current, regs);
83 
84  /*
85  * Adjust pc to point at the actual instruction, which is unusual
86  * for syscalls normally, but is appropriate when we are claiming
87  * that a syscall swint1 caused a page fault or bus error.
88  */
89  regs->pc -= 8;
90 
91  /*
92  * Mark this as a caller-save interrupt, like a normal page fault,
93  * so that when we go through the signal handler path we will
94  * properly restore r0, r1, and r2 for the signal handler arguments.
95  */
97 
98  return 0;
99 }
100 #endif
101 
102 static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
103 {
104  unsigned index = pgd_index(address);
105  pgd_t *pgd_k;
106  pud_t *pud, *pud_k;
107  pmd_t *pmd, *pmd_k;
108 
109  pgd += index;
110  pgd_k = init_mm.pgd + index;
111 
112  if (!pgd_present(*pgd_k))
113  return NULL;
114 
115  pud = pud_offset(pgd, address);
116  pud_k = pud_offset(pgd_k, address);
117  if (!pud_present(*pud_k))
118  return NULL;
119 
120  pmd = pmd_offset(pud, address);
121  pmd_k = pmd_offset(pud_k, address);
122  if (!pmd_present(*pmd_k))
123  return NULL;
124  if (!pmd_present(*pmd)) {
125  set_pmd(pmd, *pmd_k);
127  } else
128  BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k));
129  return pmd_k;
130 }
131 
132 /*
133  * Handle a fault on the vmalloc area.
134  */
135 static inline int vmalloc_fault(pgd_t *pgd, unsigned long address)
136 {
137  pmd_t *pmd_k;
138  pte_t *pte_k;
139 
140  /* Make sure we are in vmalloc area */
141  if (!(address >= VMALLOC_START && address < VMALLOC_END))
142  return -1;
143 
144  /*
145  * Synchronize this task's top level page-table
146  * with the 'reference' page table.
147  */
148  pmd_k = vmalloc_sync_one(pgd, address);
149  if (!pmd_k)
150  return -1;
151  if (pmd_huge(*pmd_k))
152  return 0; /* support TILE huge_vmap() API */
153  pte_k = pte_offset_kernel(pmd_k, address);
154  if (!pte_present(*pte_k))
155  return -1;
156  return 0;
157 }
158 
159 /* Wait until this PTE has completed migration. */
160 static void wait_for_migration(pte_t *pte)
161 {
162  if (pte_migrating(*pte)) {
163  /*
164  * Wait until the migrater fixes up this pte.
165  * We scale the loop count by the clock rate so we'll wait for
166  * a few seconds here.
167  */
168  int retries = 0;
169  int bound = get_clock_rate();
170  while (pte_migrating(*pte)) {
171  barrier();
172  if (++retries > bound)
173  panic("Hit migrating PTE (%#llx) and"
174  " page PFN %#lx still migrating",
175  pte->val, pte_pfn(*pte));
176  }
177  }
178 }
179 
180 /*
181  * It's not generally safe to use "current" to get the page table pointer,
182  * since we might be running an oprofile interrupt in the middle of a
183  * task switch.
184  */
185 static pgd_t *get_current_pgd(void)
186 {
188  unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT;
189  struct page *pgd_page = pfn_to_page(pgd_pfn);
190  BUG_ON(PageHighMem(pgd_page));
191  return (pgd_t *) __va(ctx.page_table);
192 }
193 
194 /*
195  * We can receive a page fault from a migrating PTE at any time.
196  * Handle it by just waiting until the fault resolves.
197  *
198  * It's also possible to get a migrating kernel PTE that resolves
199  * itself during the downcall from hypervisor to Linux. We just check
200  * here to see if the PTE seems valid, and if so we retry it.
201  *
202  * NOTE! We MUST NOT take any locks for this case. We may be in an
203  * interrupt or a critical region, and must do as little as possible.
204  * Similarly, we can't use atomic ops here, since we may be handling a
205  * fault caused by an atomic op access.
206  *
207  * If we find a migrating PTE while we're in an NMI context, and we're
208  * at a PC that has a registered exception handler, we don't wait,
209  * since this thread may (e.g.) have been interrupted while migrating
210  * its own stack, which would then cause us to self-deadlock.
211  */
212 static int handle_migrating_pte(pgd_t *pgd, int fault_num,
213  unsigned long address, unsigned long pc,
214  int is_kernel_mode, int write)
215 {
216  pud_t *pud;
217  pmd_t *pmd;
218  pte_t *pte;
219  pte_t pteval;
220 
221  if (pgd_addr_invalid(address))
222  return 0;
223 
224  pgd += pgd_index(address);
225  pud = pud_offset(pgd, address);
226  if (!pud || !pud_present(*pud))
227  return 0;
228  pmd = pmd_offset(pud, address);
229  if (!pmd || !pmd_present(*pmd))
230  return 0;
231  pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) :
232  pte_offset_kernel(pmd, address);
233  pteval = *pte;
234  if (pte_migrating(pteval)) {
235  if (in_nmi() && search_exception_tables(pc))
236  return 0;
237  wait_for_migration(pte);
238  return 1;
239  }
240 
241  if (!is_kernel_mode || !pte_present(pteval))
242  return 0;
243  if (fault_num == INT_ITLB_MISS) {
244  if (pte_exec(pteval))
245  return 1;
246  } else if (write) {
247  if (pte_write(pteval))
248  return 1;
249  } else {
250  if (pte_read(pteval))
251  return 1;
252  }
253 
254  return 0;
255 }
256 
257 /*
258  * This routine is responsible for faulting in user pages.
259  * It passes the work off to one of the appropriate routines.
260  * It returns true if the fault was successfully handled.
261  */
262 static int handle_page_fault(struct pt_regs *regs,
263  int fault_num,
264  int is_page_fault,
265  unsigned long address,
266  int write)
267 {
268  struct task_struct *tsk;
269  struct mm_struct *mm;
270  struct vm_area_struct *vma;
271  unsigned long stack_offset;
272  int fault;
273  int si_code;
274  int is_kernel_mode;
275  pgd_t *pgd;
276  unsigned int flags;
277 
278  /* on TILE, protection faults are always writes */
279  if (!is_page_fault)
280  write = 1;
281 
282  flags = (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
283  (write ? FAULT_FLAG_WRITE : 0));
284 
285  is_kernel_mode = (EX1_PL(regs->ex1) != USER_PL);
286 
287  tsk = validate_current();
288 
289  /*
290  * Check to see if we might be overwriting the stack, and bail
291  * out if so. The page fault code is a relatively likely
292  * place to get trapped in an infinite regress, and once we
293  * overwrite the whole stack, it becomes very hard to recover.
294  */
295  stack_offset = stack_pointer & (THREAD_SIZE-1);
296  if (stack_offset < THREAD_SIZE / 8) {
297  pr_alert("Potential stack overrun: sp %#lx\n",
298  stack_pointer);
299  show_regs(regs);
300  pr_alert("Killing current process %d/%s\n",
301  tsk->pid, tsk->comm);
303  }
304 
305  /*
306  * Early on, we need to check for migrating PTE entries;
307  * see homecache.c. If we find a migrating PTE, we wait until
308  * the backing page claims to be done migrating, then we proceed.
309  * For kernel PTEs, we rewrite the PTE and return and retry.
310  * Otherwise, we treat the fault like a normal "no PTE" fault,
311  * rather than trying to patch up the existing PTE.
312  */
313  pgd = get_current_pgd();
314  if (handle_migrating_pte(pgd, fault_num, address, regs->pc,
315  is_kernel_mode, write))
316  return 1;
317 
318  si_code = SEGV_MAPERR;
319 
320  /*
321  * We fault-in kernel-space virtual memory on-demand. The
322  * 'reference' page table is init_mm.pgd.
323  *
324  * NOTE! We MUST NOT take any locks for this case. We may
325  * be in an interrupt or a critical region, and should
326  * only copy the information from the master page table,
327  * nothing more.
328  *
329  * This verifies that the fault happens in kernel space
330  * and that the fault was not a protection fault.
331  */
332  if (unlikely(address >= TASK_SIZE &&
333  !is_arch_mappable_range(address, 0))) {
334  if (is_kernel_mode && is_page_fault &&
335  vmalloc_fault(pgd, address) >= 0)
336  return 1;
337  /*
338  * Don't take the mm semaphore here. If we fixup a prefetch
339  * fault we could otherwise deadlock.
340  */
341  mm = NULL; /* happy compiler */
342  vma = NULL;
343  goto bad_area_nosemaphore;
344  }
345 
346  /*
347  * If we're trying to touch user-space addresses, we must
348  * be either at PL0, or else with interrupts enabled in the
349  * kernel, so either way we can re-enable interrupts here
350  * unless we are doing atomic access to user space with
351  * interrupts disabled.
352  */
353  if (!(regs->flags & PT_FLAGS_DISABLE_IRQ))
355 
356  mm = tsk->mm;
357 
358  /*
359  * If we're in an interrupt, have no user context or are running in an
360  * atomic region then we must not take the fault.
361  */
362  if (in_atomic() || !mm) {
363  vma = NULL; /* happy compiler */
364  goto bad_area_nosemaphore;
365  }
366 
367  /*
368  * When running in the kernel we expect faults to occur only to
369  * addresses in user space. All other faults represent errors in the
370  * kernel and should generate an OOPS. Unfortunately, in the case of an
371  * erroneous fault occurring in a code path which already holds mmap_sem
372  * we will deadlock attempting to validate the fault against the
373  * address space. Luckily the kernel only validly references user
374  * space from well defined areas of code, which are listed in the
375  * exceptions table.
376  *
377  * As the vast majority of faults will be valid we will only perform
378  * the source reference check when there is a possibility of a deadlock.
379  * Attempt to lock the address space, if we cannot we then validate the
380  * source. If this is invalid we can skip the address space check,
381  * thus avoiding the deadlock.
382  */
383  if (!down_read_trylock(&mm->mmap_sem)) {
384  if (is_kernel_mode &&
385  !search_exception_tables(regs->pc)) {
386  vma = NULL; /* happy compiler */
387  goto bad_area_nosemaphore;
388  }
389 
390 retry:
391  down_read(&mm->mmap_sem);
392  }
393 
394  vma = find_vma(mm, address);
395  if (!vma)
396  goto bad_area;
397  if (vma->vm_start <= address)
398  goto good_area;
399  if (!(vma->vm_flags & VM_GROWSDOWN))
400  goto bad_area;
401  if (regs->sp < PAGE_OFFSET) {
402  /*
403  * accessing the stack below sp is always a bug.
404  */
405  if (address < regs->sp)
406  goto bad_area;
407  }
408  if (expand_stack(vma, address))
409  goto bad_area;
410 
411 /*
412  * Ok, we have a good vm_area for this memory access, so
413  * we can handle it..
414  */
415 good_area:
416  si_code = SEGV_ACCERR;
417  if (fault_num == INT_ITLB_MISS) {
418  if (!(vma->vm_flags & VM_EXEC))
419  goto bad_area;
420  } else if (write) {
421 #ifdef TEST_VERIFY_AREA
422  if (!is_page_fault && regs->cs == KERNEL_CS)
423  pr_err("WP fault at "REGFMT"\n", regs->eip);
424 #endif
425  if (!(vma->vm_flags & VM_WRITE))
426  goto bad_area;
427  } else {
428  if (!is_page_fault || !(vma->vm_flags & VM_READ))
429  goto bad_area;
430  }
431 
432  survive:
433  /*
434  * If for any reason at all we couldn't handle the fault,
435  * make sure we exit gracefully rather than endlessly redo
436  * the fault.
437  */
438  fault = handle_mm_fault(mm, vma, address, flags);
439 
440  if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
441  return 0;
442 
443  if (unlikely(fault & VM_FAULT_ERROR)) {
444  if (fault & VM_FAULT_OOM)
445  goto out_of_memory;
446  else if (fault & VM_FAULT_SIGBUS)
447  goto do_sigbus;
448  BUG();
449  }
450  if (flags & FAULT_FLAG_ALLOW_RETRY) {
451  if (fault & VM_FAULT_MAJOR)
452  tsk->maj_flt++;
453  else
454  tsk->min_flt++;
455  if (fault & VM_FAULT_RETRY) {
456  flags &= ~FAULT_FLAG_ALLOW_RETRY;
457  flags |= FAULT_FLAG_TRIED;
458 
459  /*
460  * No need to up_read(&mm->mmap_sem) as we would
461  * have already released it in __lock_page_or_retry
462  * in mm/filemap.c.
463  */
464  goto retry;
465  }
466  }
467 
468 #if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
469  /*
470  * If this was an asynchronous fault,
471  * restart the appropriate engine.
472  */
473  switch (fault_num) {
474 #if CHIP_HAS_TILE_DMA()
475  case INT_DMATLB_MISS:
477  case INT_DMATLB_ACCESS:
479  __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
480  break;
481 #endif
482 #if CHIP_HAS_SN_PROC()
483  case INT_SNITLB_MISS:
485  __insn_mtspr(SPR_SNCTL,
486  __insn_mfspr(SPR_SNCTL) &
488  break;
489 #endif
490  }
491 #endif
492 
493  up_read(&mm->mmap_sem);
494  return 1;
495 
496 /*
497  * Something tried to access memory that isn't in our memory map..
498  * Fix it, but check if it's kernel or user first..
499  */
500 bad_area:
501  up_read(&mm->mmap_sem);
502 
503 bad_area_nosemaphore:
504  /* User mode accesses just cause a SIGSEGV */
505  if (!is_kernel_mode) {
506  /*
507  * It's possible to have interrupts off here.
508  */
510 
511  force_sig_info_fault("segfault", SIGSEGV, si_code, address,
512  fault_num, tsk, regs);
513  return 0;
514  }
515 
516 no_context:
517  /* Are we prepared to handle this kernel fault? */
518  if (fixup_exception(regs))
519  return 0;
520 
521 /*
522  * Oops. The kernel tried to access some bad page. We'll have to
523  * terminate things with extreme prejudice.
524  */
525 
526  bust_spinlocks(1);
527 
528  /* FIXME: no lookup_address() yet */
529 #ifdef SUPPORT_LOOKUP_ADDRESS
530  if (fault_num == INT_ITLB_MISS) {
531  pte_t *pte = lookup_address(address);
532 
533  if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
534  pr_crit("kernel tried to execute"
535  " non-executable page - exploit attempt?"
536  " (uid: %d)\n", current->uid);
537  }
538 #endif
539  if (address < PAGE_SIZE)
540  pr_alert("Unable to handle kernel NULL pointer dereference\n");
541  else
542  pr_alert("Unable to handle kernel paging request\n");
543  pr_alert(" at virtual address "REGFMT", pc "REGFMT"\n",
544  address, regs->pc);
545 
546  show_regs(regs);
547 
548  if (unlikely(tsk->pid < 2)) {
549  panic("Kernel page fault running %s!",
550  is_idle_task(tsk) ? "the idle task" : "init");
551  }
552 
553  /*
554  * More FIXME: we should probably copy the i386 here and
555  * implement a generic die() routine. Not today.
556  */
557 #ifdef SUPPORT_DIE
558  die("Oops", regs);
559 #endif
560  bust_spinlocks(1);
561 
563 
564 /*
565  * We ran out of memory, or some other thing happened to us that made
566  * us unable to handle the page fault gracefully.
567  */
569  up_read(&mm->mmap_sem);
570  if (is_global_init(tsk)) {
571  yield();
572  down_read(&mm->mmap_sem);
573  goto survive;
574  }
575  pr_alert("VM: killing process %s\n", tsk->comm);
576  if (!is_kernel_mode)
578  goto no_context;
579 
580 do_sigbus:
581  up_read(&mm->mmap_sem);
582 
583  /* Kernel mode? Handle exceptions or die */
584  if (is_kernel_mode)
585  goto no_context;
586 
587  force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address,
588  fault_num, tsk, regs);
589  return 0;
590 }
591 
592 #ifndef __tilegx__
593 
594 /* We must release ICS before panicking or we won't get anywhere. */
595 #define ics_panic(fmt, ...) do { \
596  __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \
597  panic(fmt, __VA_ARGS__); \
598 } while (0)
599 
600 /*
601  * When we take an ITLB or DTLB fault or access violation in the
602  * supervisor while the critical section bit is set, the hypervisor is
603  * reluctant to write new values into the EX_CONTEXT_K_x registers,
604  * since that might indicate we have not yet squirreled the SPR
605  * contents away and can thus safely take a recursive interrupt.
606  * Accordingly, the hypervisor passes us the PC via SYSTEM_SAVE_K_2.
607  *
608  * Note that this routine is called before homecache_tlb_defer_enter(),
609  * which means that we can properly unlock any atomics that might
610  * be used there (good), but also means we must be very sensitive
611  * to not touch any data structures that might be located in memory
612  * that could migrate, as we could be entering the kernel on a dataplane
613  * cpu that has been deferring kernel TLB updates. This means, for
614  * example, that we can't migrate init_mm or its pgd.
615  */
617  unsigned long address,
618  unsigned long info)
619 {
620  unsigned long pc = info & ~1;
621  int write = info & 1;
622  pgd_t *pgd = get_current_pgd();
623 
624  /* Retval is 1 at first since we will handle the fault fully. */
625  struct intvec_state state = {
627  };
628 
629  /* Validate that we are plausibly in the right routine. */
630  if ((pc & 0x7) != 0 || pc < PAGE_OFFSET ||
631  (fault_num != INT_DTLB_MISS &&
632  fault_num != INT_DTLB_ACCESS)) {
633  unsigned long old_pc = regs->pc;
634  regs->pc = pc;
635  ics_panic("Bad ICS page fault args:"
636  " old PC %#lx, fault %d/%d at %#lx\n",
637  old_pc, fault_num, write, address);
638  }
639 
640  /* We might be faulting on a vmalloc page, so check that first. */
641  if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0)
642  return state;
643 
644  /*
645  * If we faulted with ICS set in sys_cmpxchg, we are providing
646  * a user syscall service that should generate a signal on
647  * fault. We didn't set up a kernel stack on initial entry to
648  * sys_cmpxchg, but instead had one set up by the fault, which
649  * (because sys_cmpxchg never releases ICS) came to us via the
650  * SYSTEM_SAVE_K_2 mechanism, and thus EX_CONTEXT_K_[01] are
651  * still referencing the original user code. We release the
652  * atomic lock and rewrite pt_regs so that it appears that we
653  * came from user-space directly, and after we finish the
654  * fault we'll go back to user space and re-issue the swint.
655  * This way the backtrace information is correct if we need to
656  * emit a stack dump at any point while handling this.
657  *
658  * Must match register use in sys_cmpxchg().
659  */
660  if (pc >= (unsigned long) sys_cmpxchg &&
661  pc < (unsigned long) __sys_cmpxchg_end) {
662 #ifdef CONFIG_SMP
663  /* Don't unlock before we could have locked. */
664  if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) {
665  int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
666  __atomic_fault_unlock(lock_ptr);
667  }
668 #endif
669  regs->sp = regs->regs[27];
670  }
671 
672  /*
673  * We can also fault in the atomic assembly, in which
674  * case we use the exception table to do the first-level fixup.
675  * We may re-fixup again in the real fault handler if it
676  * turns out the faulting address is just bad, and not,
677  * for example, migrating.
678  */
679  else if (pc >= (unsigned long) __start_atomic_asm_code &&
680  pc < (unsigned long) __end_atomic_asm_code) {
681  const struct exception_table_entry *fixup;
682 #ifdef CONFIG_SMP
683  /* Unlock the atomic lock. */
684  int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
685  __atomic_fault_unlock(lock_ptr);
686 #endif
687  fixup = search_exception_tables(pc);
688  if (!fixup)
689  ics_panic("ICS atomic fault not in table:"
690  " PC %#lx, fault %d", pc, fault_num);
691  regs->pc = fixup->fixup;
692  regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0);
693  }
694 
695  /*
696  * Now that we have released the atomic lock (if necessary),
697  * it's safe to spin if the PTE that caused the fault was migrating.
698  */
699  if (fault_num == INT_DTLB_ACCESS)
700  write = 1;
701  if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write))
702  return state;
703 
704  /* Return zero so that we continue on with normal fault handling. */
705  state.retval = 0;
706  return state;
707 }
708 
709 #endif /* !__tilegx__ */
710 
711 /*
712  * This routine handles page faults. It determines the address, and the
713  * problem, and then passes it handle_page_fault() for normal DTLB and
714  * ITLB issues, and for DMA or SN processor faults when we are in user
715  * space. For the latter, if we're in kernel mode, we just save the
716  * interrupt away appropriately and return immediately. We can't do
717  * page faults for user code while in kernel mode.
718  */
719 void do_page_fault(struct pt_regs *regs, int fault_num,
720  unsigned long address, unsigned long write)
721 {
722  int is_page_fault;
723 
724  /* This case should have been handled by do_page_fault_ics(). */
725  BUG_ON(write & ~1);
726 
727 #if CHIP_HAS_TILE_DMA()
728  /*
729  * If it's a DMA fault, suspend the transfer while we're
730  * handling the miss; we'll restart after it's handled. If we
731  * don't suspend, it's possible that this process could swap
732  * out and back in, and restart the engine since the DMA is
733  * still 'running'.
734  */
735  if (fault_num == INT_DMATLB_MISS ||
736  fault_num == INT_DMATLB_ACCESS ||
737  fault_num == INT_DMATLB_MISS_DWNCL ||
738  fault_num == INT_DMATLB_ACCESS_DWNCL) {
739  __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
740  while (__insn_mfspr(SPR_DMA_USER_STATUS) &
742  ;
743  }
744 #endif
745 
746  /* Validate fault num and decide if this is a first-time page fault. */
747  switch (fault_num) {
748  case INT_ITLB_MISS:
749  case INT_DTLB_MISS:
750 #if CHIP_HAS_TILE_DMA()
751  case INT_DMATLB_MISS:
753 #endif
754 #if CHIP_HAS_SN_PROC()
755  case INT_SNITLB_MISS:
757 #endif
758  is_page_fault = 1;
759  break;
760 
761  case INT_DTLB_ACCESS:
762 #if CHIP_HAS_TILE_DMA()
763  case INT_DMATLB_ACCESS:
765 #endif
766  is_page_fault = 0;
767  break;
768 
769  default:
770  panic("Bad fault number %d in do_page_fault", fault_num);
771  }
772 
773 #if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
774  if (EX1_PL(regs->ex1) != USER_PL) {
775  struct async_tlb *async;
776  switch (fault_num) {
777 #if CHIP_HAS_TILE_DMA()
778  case INT_DMATLB_MISS:
779  case INT_DMATLB_ACCESS:
782  async = &current->thread.dma_async_tlb;
783  break;
784 #endif
785 #if CHIP_HAS_SN_PROC()
786  case INT_SNITLB_MISS:
788  async = &current->thread.sn_async_tlb;
789  break;
790 #endif
791  default:
792  async = NULL;
793  }
794  if (async) {
795 
796  /*
797  * No vmalloc check required, so we can allow
798  * interrupts immediately at this point.
799  */
801 
802  set_thread_flag(TIF_ASYNC_TLB);
803  if (async->fault_num != 0) {
804  panic("Second async fault %d;"
805  " old fault was %d (%#lx/%ld)",
806  fault_num, async->fault_num,
807  address, write);
808  }
809  BUG_ON(fault_num == 0);
810  async->fault_num = fault_num;
811  async->is_fault = is_page_fault;
812  async->is_write = write;
813  async->address = address;
814  return;
815  }
816  }
817 #endif
818 
819  handle_page_fault(regs, fault_num, is_page_fault, address, write);
820 }
821 
822 
823 #if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
824 /*
825  * Check an async_tlb structure to see if a deferred fault is waiting,
826  * and if so pass it to the page-fault code.
827  */
828 static void handle_async_page_fault(struct pt_regs *regs,
829  struct async_tlb *async)
830 {
831  if (async->fault_num) {
832  /*
833  * Clear async->fault_num before calling the page-fault
834  * handler so that if we re-interrupt before returning
835  * from the function we have somewhere to put the
836  * information from the new interrupt.
837  */
838  int fault_num = async->fault_num;
839  async->fault_num = 0;
840  handle_page_fault(regs, fault_num, async->is_fault,
841  async->address, async->is_write);
842  }
843 }
844 
845 /*
846  * This routine effectively re-issues asynchronous page faults
847  * when we are returning to user space.
848  */
849 void do_async_page_fault(struct pt_regs *regs)
850 {
851  /*
852  * Clear thread flag early. If we re-interrupt while processing
853  * code here, we will reset it and recall this routine before
854  * returning to user space.
855  */
856  clear_thread_flag(TIF_ASYNC_TLB);
857 
858 #if CHIP_HAS_TILE_DMA()
859  handle_async_page_fault(regs, &current->thread.dma_async_tlb);
860 #endif
861 #if CHIP_HAS_SN_PROC()
862  handle_async_page_fault(regs, &current->thread.sn_async_tlb);
863 #endif
864 }
865 #endif /* CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC() */
866 
867 
869 {
870 #ifdef __tilegx__
871  /* Currently all L1 kernel pmd's are static and shared. */
873 #else
874  /*
875  * Note that races in the updates of insync and start aren't
876  * problematic: insync can only get set bits added, and updates to
877  * start are only improving performance (without affecting correctness
878  * if undone).
879  */
880  static DECLARE_BITMAP(insync, PTRS_PER_PGD);
881  static unsigned long start = PAGE_OFFSET;
882  unsigned long address;
883 
885  for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) {
886  if (!test_bit(pgd_index(address), insync)) {
887  unsigned long flags;
888  struct list_head *pos;
889 
890  spin_lock_irqsave(&pgd_lock, flags);
891  list_for_each(pos, &pgd_list)
892  if (!vmalloc_sync_one(list_to_pgd(pos),
893  address)) {
894  /* Must be at first entry in list. */
895  BUG_ON(pos != pgd_list.next);
896  break;
897  }
898  spin_unlock_irqrestore(&pgd_lock, flags);
899  if (pos != pgd_list.next)
900  set_bit(pgd_index(address), insync);
901  }
902  if (address == start && test_bit(pgd_index(address), insync))
903  start = address + PGDIR_SIZE;
904  }
905 #endif
906 }