43 #include <linux/ptrace.h>
44 #include <linux/string.h>
45 #include <linux/slab.h>
48 #include <linux/module.h>
53 #include <asm/cacheflush.h>
55 #include <asm/pgtable.h>
56 #include <asm/uaccess.h>
68 #define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
70 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
71 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
72 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
73 (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \
74 (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \
83 static volatile u32 twobyte_is_boostable[256 / 32] = {
86 W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) |
87 W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ,
88 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) |
89 W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ,
90 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) |
91 W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ,
92 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) |
93 W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) ,
94 W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) |
95 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) ,
96 W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) |
97 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) ,
98 W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) |
99 W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) ,
100 W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) |
101 W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0)
117 struct __arch_relative_insn {
122 insn = (
struct __arch_relative_insn *)from;
123 insn->raddr = (
s32)((
long)(to) - ((
long)(
from) + 5));
147 while (inat_is_legacy_prefix(attr)) {
152 if (inat_is_rex_prefix(attr))
173 opcode = *(opcodes++);
176 if (opcode == 0x0f) {
180 (
unsigned long *)twobyte_is_boostable);
183 switch (opcode & 0xf0) {
189 if (0x63 < opcode && opcode < 0x67)
192 return (opcode != 0x62 && opcode != 0x67);
197 return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
200 return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
203 return ((opcode & 0x04) || opcode == 0xea);
205 if ((opcode & 0x0c) == 0 && opcode != 0xf1)
208 return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
211 if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
214 return (opcode != 0x2e && opcode != 0x9a);
243 return (
unsigned long)
buf;
259 return __recover_probed_insn(buf, addr);
274 while (addr < paddr) {
284 kernel_insn_init(&insn, (
void *)__addr);
296 return (addr == paddr);
305 insn = skip_prefixes(insn);
341 kernel_insn_init(&insn, dest);
357 disp = (
u8 *) dest + insn_offset_displacement(&insn);
358 *(
s32 *) disp = (
s32) newdisp;
374 p->
ainsn.boostable = 0;
376 p->
ainsn.boostable = -1;
384 if (alternatives_text_reserved(p->
addr, p->
addr))
387 if (!can_probe((
unsigned long)p->
addr))
390 p->
ainsn.insn = get_insn_slot();
410 free_insn_slot(p->
ainsn.insn, (p->
ainsn.boostable == 1));
437 if (is_IF_modifier(p->
ainsn.insn))
444 unsigned long debugctl = get_debugctlmsr();
447 update_debugctlmsr(debugctl);
454 unsigned long debugctl = get_debugctlmsr();
457 update_debugctlmsr(debugctl);
478 #if !defined(CONFIG_PREEMPT)
482 reset_current_kprobe();
494 save_previous_kprobe(kcb);
495 set_current_kprobe(p, regs, kcb);
507 regs->ip = (
unsigned long)p->
ainsn.insn;
519 case KPROBE_HIT_SSDONE:
522 setup_singlestep(p, regs, kcb, 1);
544 #ifdef KPROBES_CAN_USE_FTRACE
580 kcb = get_kprobe_ctlblk();
584 if (kprobe_running()) {
585 if (reenter_kprobe(p, regs, kcb))
588 set_current_kprobe(p, regs, kcb);
600 setup_singlestep(p, regs, kcb, 0);
613 regs->ip = (
unsigned long)addr;
616 }
else if (kprobe_running()) {
619 #ifdef KPROBES_CAN_USE_FTRACE
620 if (kprobe_ftrace(p)) {
621 skip_singlestep(p, regs, kcb);
625 setup_singlestep(p, regs, kcb, 0);
641 ".global kretprobe_trampoline\n"
642 "kretprobe_trampoline: \n"
649 " call trampoline_handler\n"
651 " movq %rax, 152(%rsp)\n"
658 " call trampoline_handler\n"
660 " movl 56(%esp), %edx\n"
661 " movl %edx, 52(%esp)\n"
663 " movl %eax, 56(%esp)\n"
678 unsigned long flags, orig_ret_address = 0;
691 regs->ip = trampoline_address;
692 regs->orig_ax = ~0
UL;
714 if (orig_ret_address != trampoline_address)
723 kretprobe_assert(ri, orig_ret_address, trampoline_address);
732 if (ri->
rp && ri->
rp->handler) {
736 ri->
rp->handler(ri, regs);
742 if (orig_ret_address != trampoline_address)
754 hlist_del(&ri->
hlist);
757 return (
void *)orig_ret_address;
791 unsigned long copy_ip = (
unsigned long)p->
ainsn.insn;
792 unsigned long orig_ip = (
unsigned long)p->
addr;
796 insn = skip_prefixes(insn);
811 p->
ainsn.boostable = 1;
814 *tos = orig_ip + (*tos - copy_ip);
818 *tos = orig_ip + (*tos - copy_ip);
822 if ((insn[1] & 0x30) == 0x10) {
828 *tos = orig_ip + (*tos - copy_ip);
830 }
else if (((insn[1] & 0x31) == 0x20) ||
831 ((insn[1] & 0x31) == 0x21)) {
836 p->
ainsn.boostable = 1;
843 if (p->
ainsn.boostable == 0) {
844 if ((regs->ip > copy_ip) &&
851 (
void *)orig_ip + (regs->ip - copy_ip));
852 p->
ainsn.boostable = 1;
854 p->
ainsn.boostable = -1;
858 regs->ip += orig_ip - copy_ip;
876 resume_execution(cur, regs, kcb);
886 restore_previous_kprobe(kcb);
889 reset_current_kprobe();
906 struct kprobe *cur = kprobe_running();
919 regs->ip = (
unsigned long)cur->
addr;
922 restore_previous_kprobe(kcb);
924 reset_current_kprobe();
928 case KPROBE_HIT_SSDONE:
971 int ret = NOTIFY_DONE;
973 if (args->
regs && user_mode_vm(args->
regs))
982 if (post_kprobe_handler(args->
regs)) {
987 (*(
unsigned long *)ERR_PTR(args->
err)) &= ~
DR_STEP;
1037 #ifdef CONFIG_X86_64
1038 " xchg %%rbx,%%rsp \n"
1040 " xchgl %%ebx,%%esp \n"
1043 " .globl jprobe_return_end\n"
1044 " jprobe_return_end: \n"
1052 u8 *addr = (
u8 *) (regs->ip - 1);
1060 "current sp %p does not match saved sp %p\n",
1078 #ifdef KPROBES_CAN_USE_FTRACE
1080 void __kprobes kprobe_ftrace_handler(
unsigned long ip,
unsigned long parent_ip,
1081 struct ftrace_ops *ops,
struct pt_regs *regs)
1085 unsigned long flags;
1091 if (
unlikely(!p) || kprobe_disabled(p))
1094 kcb = get_kprobe_ctlblk();
1095 if (kprobe_running()) {
1104 skip_singlestep(p, regs, kcb);
1117 p->
ainsn.boostable = -1;