Linux Kernel  3.7.1
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
unaligned_64.c
Go to the documentation of this file.
1 /*
2  * unaligned.c: Unaligned load/store trap handling with special
3  * cases for the kernel to do them more quickly.
4  *
5  * Copyright (C) 1996,2008 David S. Miller ([email protected])
6  * Copyright (C) 1996,1997 Jakub Jelinek ([email protected])
7  */
8 
9 
10 #include <linux/jiffies.h>
11 #include <linux/kernel.h>
12 #include <linux/sched.h>
13 #include <linux/mm.h>
14 #include <linux/module.h>
15 #include <asm/asi.h>
16 #include <asm/ptrace.h>
17 #include <asm/pstate.h>
18 #include <asm/processor.h>
19 #include <asm/uaccess.h>
20 #include <linux/smp.h>
21 #include <linux/bitops.h>
22 #include <linux/perf_event.h>
23 #include <linux/ratelimit.h>
24 #include <asm/fpumacro.h>
25 #include <asm/cacheflush.h>
26 
27 enum direction {
28  load, /* ld, ldd, ldh, ldsh */
29  store, /* st, std, sth, stsh */
30  both, /* Swap, ldstub, cas, ... */
31  fpld,
32  fpst,
33  invalid,
34 };
35 
36 static inline enum direction decode_direction(unsigned int insn)
37 {
38  unsigned long tmp = (insn >> 21) & 1;
39 
40  if (!tmp)
41  return load;
42  else {
43  switch ((insn>>19)&0xf) {
44  case 15: /* swap* */
45  return both;
46  default:
47  return store;
48  }
49  }
50 }
51 
52 /* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */
53 static inline int decode_access_size(struct pt_regs *regs, unsigned int insn)
54 {
55  unsigned int tmp;
56 
57  tmp = ((insn >> 19) & 0xf);
58  if (tmp == 11 || tmp == 14) /* ldx/stx */
59  return 8;
60  tmp &= 3;
61  if (!tmp)
62  return 4;
63  else if (tmp == 3)
64  return 16; /* ldd/std - Although it is actually 8 */
65  else if (tmp == 2)
66  return 2;
67  else {
68  printk("Impossible unaligned trap. insn=%08x\n", insn);
69  die_if_kernel("Byte sized unaligned access?!?!", regs);
70 
71  /* GCC should never warn that control reaches the end
72  * of this function without returning a value because
73  * die_if_kernel() is marked with attribute 'noreturn'.
74  * Alas, some versions do...
75  */
76 
77  return 0;
78  }
79 }
80 
81 static inline int decode_asi(unsigned int insn, struct pt_regs *regs)
82 {
83  if (insn & 0x800000) {
84  if (insn & 0x2000)
85  return (unsigned char)(regs->tstate >> 24); /* %asi */
86  else
87  return (unsigned char)(insn >> 5); /* imm_asi */
88  } else
89  return ASI_P;
90 }
91 
92 /* 0x400000 = signed, 0 = unsigned */
93 static inline int decode_signedness(unsigned int insn)
94 {
95  return (insn & 0x400000);
96 }
97 
98 static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2,
99  unsigned int rd, int from_kernel)
100 {
101  if (rs2 >= 16 || rs1 >= 16 || rd >= 16) {
102  if (from_kernel != 0)
103  __asm__ __volatile__("flushw");
104  else
105  flushw_user();
106  }
107 }
108 
109 static inline long sign_extend_imm13(long imm)
110 {
111  return imm << 51 >> 51;
112 }
113 
114 static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs)
115 {
116  unsigned long value, fp;
117 
118  if (reg < 16)
119  return (!reg ? 0 : regs->u_regs[reg]);
120 
121  fp = regs->u_regs[UREG_FP];
122 
123  if (regs->tstate & TSTATE_PRIV) {
124  struct reg_window *win;
125  win = (struct reg_window *)(fp + STACK_BIAS);
126  value = win->locals[reg - 16];
127  } else if (!test_thread_64bit_stack(fp)) {
128  struct reg_window32 __user *win32;
129  win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp));
130  get_user(value, &win32->locals[reg - 16]);
131  } else {
132  struct reg_window __user *win;
133  win = (struct reg_window __user *)(fp + STACK_BIAS);
134  get_user(value, &win->locals[reg - 16]);
135  }
136  return value;
137 }
138 
139 static unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs)
140 {
141  unsigned long fp;
142 
143  if (reg < 16)
144  return &regs->u_regs[reg];
145 
146  fp = regs->u_regs[UREG_FP];
147 
148  if (regs->tstate & TSTATE_PRIV) {
149  struct reg_window *win;
150  win = (struct reg_window *)(fp + STACK_BIAS);
151  return &win->locals[reg - 16];
152  } else if (!test_thread_64bit_stack(fp)) {
153  struct reg_window32 *win32;
154  win32 = (struct reg_window32 *)((unsigned long)((u32)fp));
155  return (unsigned long *)&win32->locals[reg - 16];
156  } else {
157  struct reg_window *win;
158  win = (struct reg_window *)(fp + STACK_BIAS);
159  return &win->locals[reg - 16];
160  }
161 }
162 
163 unsigned long compute_effective_address(struct pt_regs *regs,
164  unsigned int insn, unsigned int rd)
165 {
166  unsigned int rs1 = (insn >> 14) & 0x1f;
167  unsigned int rs2 = insn & 0x1f;
168  int from_kernel = (regs->tstate & TSTATE_PRIV) != 0;
169 
170  if (insn & 0x2000) {
171  maybe_flush_windows(rs1, 0, rd, from_kernel);
172  return (fetch_reg(rs1, regs) + sign_extend_imm13(insn));
173  } else {
174  maybe_flush_windows(rs1, rs2, rd, from_kernel);
175  return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs));
176  }
177 }
178 
179 /* This is just to make gcc think die_if_kernel does return... */
180 static void __used unaligned_panic(char *str, struct pt_regs *regs)
181 {
182  die_if_kernel(str, regs);
183 }
184 
185 extern int do_int_load(unsigned long *dest_reg, int size,
186  unsigned long *saddr, int is_signed, int asi);
187 
188 extern int __do_int_store(unsigned long *dst_addr, int size,
189  unsigned long src_val, int asi);
190 
191 static inline int do_int_store(int reg_num, int size, unsigned long *dst_addr,
192  struct pt_regs *regs, int asi, int orig_asi)
193 {
194  unsigned long zero = 0;
195  unsigned long *src_val_p = &zero;
196  unsigned long src_val;
197 
198  if (size == 16) {
199  size = 8;
200  zero = (((long)(reg_num ?
201  (unsigned)fetch_reg(reg_num, regs) : 0)) << 32) |
202  (unsigned)fetch_reg(reg_num + 1, regs);
203  } else if (reg_num) {
204  src_val_p = fetch_reg_addr(reg_num, regs);
205  }
206  src_val = *src_val_p;
207  if (unlikely(asi != orig_asi)) {
208  switch (size) {
209  case 2:
210  src_val = swab16(src_val);
211  break;
212  case 4:
213  src_val = swab32(src_val);
214  break;
215  case 8:
216  src_val = swab64(src_val);
217  break;
218  case 16:
219  default:
220  BUG();
221  break;
222  }
223  }
224  return __do_int_store(dst_addr, size, src_val, asi);
225 }
226 
227 static inline void advance(struct pt_regs *regs)
228 {
229  regs->tpc = regs->tnpc;
230  regs->tnpc += 4;
231  if (test_thread_flag(TIF_32BIT)) {
232  regs->tpc &= 0xffffffff;
233  regs->tnpc &= 0xffffffff;
234  }
235 }
236 
237 static inline int floating_point_load_or_store_p(unsigned int insn)
238 {
239  return (insn >> 24) & 1;
240 }
241 
242 static inline int ok_for_kernel(unsigned int insn)
243 {
244  return !floating_point_load_or_store_p(insn);
245 }
246 
247 static void kernel_mna_trap_fault(int fixup_tstate_asi)
248 {
249  struct pt_regs *regs = current_thread_info()->kern_una_regs;
250  unsigned int insn = current_thread_info()->kern_una_insn;
251  const struct exception_table_entry *entry;
252 
253  entry = search_exception_tables(regs->tpc);
254  if (!entry) {
255  unsigned long address;
256 
257  address = compute_effective_address(regs, insn,
258  ((insn >> 25) & 0x1f));
259  if (address < PAGE_SIZE) {
260  printk(KERN_ALERT "Unable to handle kernel NULL "
261  "pointer dereference in mna handler");
262  } else
263  printk(KERN_ALERT "Unable to handle kernel paging "
264  "request in mna handler");
265  printk(KERN_ALERT " at virtual address %016lx\n",address);
266  printk(KERN_ALERT "current->{active_,}mm->context = %016lx\n",
267  (current->mm ? CTX_HWBITS(current->mm->context) :
268  CTX_HWBITS(current->active_mm->context)));
269  printk(KERN_ALERT "current->{active_,}mm->pgd = %016lx\n",
270  (current->mm ? (unsigned long) current->mm->pgd :
271  (unsigned long) current->active_mm->pgd));
272  die_if_kernel("Oops", regs);
273  /* Not reached */
274  }
275  regs->tpc = entry->fixup;
276  regs->tnpc = regs->tpc + 4;
277 
278  if (fixup_tstate_asi) {
279  regs->tstate &= ~TSTATE_ASI;
280  regs->tstate |= (ASI_AIUS << 24UL);
281  }
282 }
283 
284 static void log_unaligned(struct pt_regs *regs)
285 {
286  static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5);
287 
288  if (__ratelimit(&ratelimit)) {
289  printk("Kernel unaligned access at TPC[%lx] %pS\n",
290  regs->tpc, (void *) regs->tpc);
291  }
292 }
293 
294 asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn)
295 {
296  enum direction dir = decode_direction(insn);
297  int size = decode_access_size(regs, insn);
298  int orig_asi, asi;
299 
300  current_thread_info()->kern_una_regs = regs;
301  current_thread_info()->kern_una_insn = insn;
302 
303  orig_asi = asi = decode_asi(insn, regs);
304 
305  /* If this is a {get,put}_user() on an unaligned userspace pointer,
306  * just signal a fault and do not log the event.
307  */
308  if (asi == ASI_AIUS) {
309  kernel_mna_trap_fault(0);
310  return;
311  }
312 
313  log_unaligned(regs);
314 
315  if (!ok_for_kernel(insn) || dir == both) {
316  printk("Unsupported unaligned load/store trap for kernel "
317  "at <%016lx>.\n", regs->tpc);
318  unaligned_panic("Kernel does fpu/atomic "
319  "unaligned load/store.", regs);
320 
321  kernel_mna_trap_fault(0);
322  } else {
323  unsigned long addr, *reg_addr;
324  int err;
325 
326  addr = compute_effective_address(regs, insn,
327  ((insn >> 25) & 0x1f));
328  perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
329  switch (asi) {
330  case ASI_NL:
331  case ASI_AIUPL:
332  case ASI_AIUSL:
333  case ASI_PL:
334  case ASI_SL:
335  case ASI_PNFL:
336  case ASI_SNFL:
337  asi &= ~0x08;
338  break;
339  }
340  switch (dir) {
341  case load:
342  reg_addr = fetch_reg_addr(((insn>>25)&0x1f), regs);
343  err = do_int_load(reg_addr, size,
344  (unsigned long *) addr,
345  decode_signedness(insn), asi);
346  if (likely(!err) && unlikely(asi != orig_asi)) {
347  unsigned long val_in = *reg_addr;
348  switch (size) {
349  case 2:
350  val_in = swab16(val_in);
351  break;
352  case 4:
353  val_in = swab32(val_in);
354  break;
355  case 8:
356  val_in = swab64(val_in);
357  break;
358  case 16:
359  default:
360  BUG();
361  break;
362  }
363  *reg_addr = val_in;
364  }
365  break;
366 
367  case store:
368  err = do_int_store(((insn>>25)&0x1f), size,
369  (unsigned long *) addr, regs,
370  asi, orig_asi);
371  break;
372 
373  default:
374  panic("Impossible kernel unaligned trap.");
375  /* Not reached... */
376  }
377  if (unlikely(err))
378  kernel_mna_trap_fault(1);
379  else
380  advance(regs);
381  }
382 }
383 
384 int handle_popc(u32 insn, struct pt_regs *regs)
385 {
386  int from_kernel = (regs->tstate & TSTATE_PRIV) != 0;
387  int ret, rd = ((insn >> 25) & 0x1f);
388  u64 value;
389 
390  perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
391  if (insn & 0x2000) {
392  maybe_flush_windows(0, 0, rd, from_kernel);
393  value = sign_extend_imm13(insn);
394  } else {
395  maybe_flush_windows(0, insn & 0x1f, rd, from_kernel);
396  value = fetch_reg(insn & 0x1f, regs);
397  }
398  ret = hweight64(value);
399  if (rd < 16) {
400  if (rd)
401  regs->u_regs[rd] = ret;
402  } else {
403  unsigned long fp = regs->u_regs[UREG_FP];
404 
405  if (!test_thread_64bit_stack(fp)) {
406  struct reg_window32 __user *win32;
407  win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp));
408  put_user(ret, &win32->locals[rd - 16]);
409  } else {
410  struct reg_window __user *win;
411  win = (struct reg_window __user *)(fp + STACK_BIAS);
412  put_user(ret, &win->locals[rd - 16]);
413  }
414  }
415  advance(regs);
416  return 1;
417 }
418 
419 extern void do_fpother(struct pt_regs *regs);
420 extern void do_privact(struct pt_regs *regs);
421 extern void spitfire_data_access_exception(struct pt_regs *regs,
422  unsigned long sfsr,
423  unsigned long sfar);
424 extern void sun4v_data_access_exception(struct pt_regs *regs,
425  unsigned long addr,
426  unsigned long type_ctx);
427 
428 int handle_ldf_stq(u32 insn, struct pt_regs *regs)
429 {
430  unsigned long addr = compute_effective_address(regs, insn, 0);
431  int freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
432  struct fpustate *f = FPUSTATE;
433  int asi = decode_asi(insn, regs);
434  int flag = (freg < 32) ? FPRS_DL : FPRS_DU;
435 
436  perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
437 
438  save_and_clear_fpu();
439  current_thread_info()->xfsr[0] &= ~0x1c000;
440  if (freg & 3) {
441  current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */;
442  do_fpother(regs);
443  return 0;
444  }
445  if (insn & 0x200000) {
446  /* STQ */
447  u64 first = 0, second = 0;
448 
449  if (current_thread_info()->fpsaved[0] & flag) {
450  first = *(u64 *)&f->regs[freg];
451  second = *(u64 *)&f->regs[freg+2];
452  }
453  if (asi < 0x80) {
454  do_privact(regs);
455  return 1;
456  }
457  switch (asi) {
458  case ASI_P:
459  case ASI_S: break;
460  case ASI_PL:
461  case ASI_SL:
462  {
463  /* Need to convert endians */
464  u64 tmp = __swab64p(&first);
465 
466  first = __swab64p(&second);
467  second = tmp;
468  break;
469  }
470  default:
471  if (tlb_type == hypervisor)
472  sun4v_data_access_exception(regs, addr, 0);
473  else
474  spitfire_data_access_exception(regs, 0, addr);
475  return 1;
476  }
477  if (put_user (first >> 32, (u32 __user *)addr) ||
478  __put_user ((u32)first, (u32 __user *)(addr + 4)) ||
479  __put_user (second >> 32, (u32 __user *)(addr + 8)) ||
480  __put_user ((u32)second, (u32 __user *)(addr + 12))) {
481  if (tlb_type == hypervisor)
482  sun4v_data_access_exception(regs, addr, 0);
483  else
484  spitfire_data_access_exception(regs, 0, addr);
485  return 1;
486  }
487  } else {
488  /* LDF, LDDF, LDQF */
489  u32 data[4] __attribute__ ((aligned(8)));
490  int size, i;
491  int err;
492 
493  if (asi < 0x80) {
494  do_privact(regs);
495  return 1;
496  } else if (asi > ASI_SNFL) {
497  if (tlb_type == hypervisor)
498  sun4v_data_access_exception(regs, addr, 0);
499  else
500  spitfire_data_access_exception(regs, 0, addr);
501  return 1;
502  }
503  switch (insn & 0x180000) {
504  case 0x000000: size = 1; break;
505  case 0x100000: size = 4; break;
506  default: size = 2; break;
507  }
508  for (i = 0; i < size; i++)
509  data[i] = 0;
510 
511  err = get_user (data[0], (u32 __user *) addr);
512  if (!err) {
513  for (i = 1; i < size; i++)
514  err |= __get_user (data[i], (u32 __user *)(addr + 4*i));
515  }
516  if (err && !(asi & 0x2 /* NF */)) {
517  if (tlb_type == hypervisor)
518  sun4v_data_access_exception(regs, addr, 0);
519  else
520  spitfire_data_access_exception(regs, 0, addr);
521  return 1;
522  }
523  if (asi & 0x8) /* Little */ {
524  u64 tmp;
525 
526  switch (size) {
527  case 1: data[0] = le32_to_cpup(data + 0); break;
528  default:*(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 0));
529  break;
530  case 4: tmp = le64_to_cpup((u64 *)(data + 0));
531  *(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 2));
532  *(u64 *)(data + 2) = tmp;
533  break;
534  }
535  }
536  if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) {
537  current_thread_info()->fpsaved[0] = FPRS_FEF;
538  current_thread_info()->gsr[0] = 0;
539  }
540  if (!(current_thread_info()->fpsaved[0] & flag)) {
541  if (freg < 32)
542  memset(f->regs, 0, 32*sizeof(u32));
543  else
544  memset(f->regs+32, 0, 32*sizeof(u32));
545  }
546  memcpy(f->regs + freg, data, size * 4);
547  current_thread_info()->fpsaved[0] |= flag;
548  }
549  advance(regs);
550  return 1;
551 }
552 
553 void handle_ld_nf(u32 insn, struct pt_regs *regs)
554 {
555  int rd = ((insn >> 25) & 0x1f);
556  int from_kernel = (regs->tstate & TSTATE_PRIV) != 0;
557  unsigned long *reg;
558 
559  perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
560 
561  maybe_flush_windows(0, 0, rd, from_kernel);
562  reg = fetch_reg_addr(rd, regs);
563  if (from_kernel || rd < 16) {
564  reg[0] = 0;
565  if ((insn & 0x780000) == 0x180000)
566  reg[1] = 0;
567  } else if (!test_thread_64bit_stack(regs->u_regs[UREG_FP])) {
568  put_user(0, (int __user *) reg);
569  if ((insn & 0x780000) == 0x180000)
570  put_user(0, ((int __user *) reg) + 1);
571  } else {
572  put_user(0, (unsigned long __user *) reg);
573  if ((insn & 0x780000) == 0x180000)
574  put_user(0, (unsigned long __user *) reg + 1);
575  }
576  advance(regs);
577 }
578 
579 void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
580 {
581  unsigned long pc = regs->tpc;
582  unsigned long tstate = regs->tstate;
583  u32 insn;
584  u64 value;
585  u8 freg;
586  int flag;
587  struct fpustate *f = FPUSTATE;
588 
589  if (tstate & TSTATE_PRIV)
590  die_if_kernel("lddfmna from kernel", regs);
591  perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar);
592  if (test_thread_flag(TIF_32BIT))
593  pc = (u32)pc;
594  if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
595  int asi = decode_asi(insn, regs);
596  u32 first, second;
597  int err;
598 
599  if ((asi > ASI_SNFL) ||
600  (asi < ASI_P))
601  goto daex;
602  first = second = 0;
603  err = get_user(first, (u32 __user *)sfar);
604  if (!err)
605  err = get_user(second, (u32 __user *)(sfar + 4));
606  if (err) {
607  if (!(asi & 0x2))
608  goto daex;
609  first = second = 0;
610  }
611  save_and_clear_fpu();
612  freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
613  value = (((u64)first) << 32) | second;
614  if (asi & 0x8) /* Little */
615  value = __swab64p(&value);
616  flag = (freg < 32) ? FPRS_DL : FPRS_DU;
617  if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) {
618  current_thread_info()->fpsaved[0] = FPRS_FEF;
619  current_thread_info()->gsr[0] = 0;
620  }
621  if (!(current_thread_info()->fpsaved[0] & flag)) {
622  if (freg < 32)
623  memset(f->regs, 0, 32*sizeof(u32));
624  else
625  memset(f->regs+32, 0, 32*sizeof(u32));
626  }
627  *(u64 *)(f->regs + freg) = value;
628  current_thread_info()->fpsaved[0] |= flag;
629  } else {
630 daex:
631  if (tlb_type == hypervisor)
632  sun4v_data_access_exception(regs, sfar, sfsr);
633  else
634  spitfire_data_access_exception(regs, sfsr, sfar);
635  return;
636  }
637  advance(regs);
638 }
639 
640 void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
641 {
642  unsigned long pc = regs->tpc;
643  unsigned long tstate = regs->tstate;
644  u32 insn;
645  u64 value;
646  u8 freg;
647  int flag;
648  struct fpustate *f = FPUSTATE;
649 
650  if (tstate & TSTATE_PRIV)
651  die_if_kernel("stdfmna from kernel", regs);
652  perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar);
653  if (test_thread_flag(TIF_32BIT))
654  pc = (u32)pc;
655  if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
656  int asi = decode_asi(insn, regs);
657  freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
658  value = 0;
659  flag = (freg < 32) ? FPRS_DL : FPRS_DU;
660  if ((asi > ASI_SNFL) ||
661  (asi < ASI_P))
662  goto daex;
663  save_and_clear_fpu();
664  if (current_thread_info()->fpsaved[0] & flag)
665  value = *(u64 *)&f->regs[freg];
666  switch (asi) {
667  case ASI_P:
668  case ASI_S: break;
669  case ASI_PL:
670  case ASI_SL:
671  value = __swab64p(&value); break;
672  default: goto daex;
673  }
674  if (put_user (value >> 32, (u32 __user *) sfar) ||
675  __put_user ((u32)value, (u32 __user *)(sfar + 4)))
676  goto daex;
677  } else {
678 daex:
679  if (tlb_type == hypervisor)
680  sun4v_data_access_exception(regs, sfar, sfsr);
681  else
682  spitfire_data_access_exception(regs, sfsr, sfar);
683  return;
684  }
685  advance(regs);
686 }
Generated on Thu Jan 10 2013 13:18:42 for Linux Kernel by   doxygen 1.8.2