Lines Matching refs:p
381 struct kprobe *p; in get_kprobe() local
384 hlist_for_each_entry_rcu(p, head, hlist, in get_kprobe()
386 if (p->addr == addr) in get_kprobe()
387 return p; in get_kprobe()
394 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
397 static inline bool kprobe_aggrprobe(struct kprobe *p) in kprobe_aggrprobe() argument
399 return p->pre_handler == aggr_pre_handler; in kprobe_aggrprobe()
403 static inline bool kprobe_unused(struct kprobe *p) in kprobe_unused() argument
405 return kprobe_aggrprobe(p) && kprobe_disabled(p) && in kprobe_unused()
406 list_empty(&p->list); in kprobe_unused()
410 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p) in copy_kprobe() argument
412 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t)); in copy_kprobe()
413 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn)); in copy_kprobe()
424 void opt_pre_handler(struct kprobe *p, struct pt_regs *regs) in opt_pre_handler() argument
428 list_for_each_entry_rcu(kp, &p->list, list) { in opt_pre_handler()
439 static void free_aggr_kprobe(struct kprobe *p) in free_aggr_kprobe() argument
443 op = container_of(p, struct optimized_kprobe, kp); in free_aggr_kprobe()
445 arch_remove_kprobe(p); in free_aggr_kprobe()
450 static inline int kprobe_optready(struct kprobe *p) in kprobe_optready() argument
454 if (kprobe_aggrprobe(p)) { in kprobe_optready()
455 op = container_of(p, struct optimized_kprobe, kp); in kprobe_optready()
463 bool kprobe_disarmed(struct kprobe *p) in kprobe_disarmed() argument
468 if (!kprobe_aggrprobe(p)) in kprobe_disarmed()
469 return kprobe_disabled(p); in kprobe_disarmed()
471 op = container_of(p, struct optimized_kprobe, kp); in kprobe_disarmed()
473 return kprobe_disabled(p) && list_empty(&op->list); in kprobe_disarmed()
477 static bool kprobe_queued(struct kprobe *p) in kprobe_queued() argument
481 if (kprobe_aggrprobe(p)) { in kprobe_queued()
482 op = container_of(p, struct optimized_kprobe, kp); in kprobe_queued()
496 struct kprobe *p = NULL; in get_optimized_kprobe() local
500 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) in get_optimized_kprobe()
501 p = get_kprobe(addr - i); in get_optimized_kprobe()
503 if (p && kprobe_optready(p)) { in get_optimized_kprobe()
504 op = container_of(p, struct optimized_kprobe, kp); in get_optimized_kprobe()
506 return p; in get_optimized_kprobe()
680 static void optimize_kprobe(struct kprobe *p) in optimize_kprobe() argument
685 if (!kprobe_optready(p) || !kprobes_allow_optimization || in optimize_kprobe()
686 (kprobe_disabled(p) || kprobes_all_disarmed)) in optimize_kprobe()
690 if (p->post_handler) in optimize_kprobe()
693 op = container_of(p, struct optimized_kprobe, kp); in optimize_kprobe()
729 static void unoptimize_kprobe(struct kprobe *p, bool force) in unoptimize_kprobe() argument
733 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p)) in unoptimize_kprobe()
736 op = container_of(p, struct optimized_kprobe, kp); in unoptimize_kprobe()
737 if (!kprobe_optimized(p)) in unoptimize_kprobe()
791 static void kill_optimized_kprobe(struct kprobe *p) in kill_optimized_kprobe() argument
795 op = container_of(p, struct optimized_kprobe, kp); in kill_optimized_kprobe()
801 if (kprobe_unused(p)) { in kill_optimized_kprobe()
816 void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p) in __prepare_optimized_kprobe() argument
818 if (!kprobe_ftrace(p)) in __prepare_optimized_kprobe()
819 arch_prepare_optimized_kprobe(op, p); in __prepare_optimized_kprobe()
823 static void prepare_optimized_kprobe(struct kprobe *p) in prepare_optimized_kprobe() argument
827 op = container_of(p, struct optimized_kprobe, kp); in prepare_optimized_kprobe()
828 __prepare_optimized_kprobe(op, p); in prepare_optimized_kprobe()
832 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) in alloc_aggr_kprobe() argument
841 op->kp.addr = p->addr; in alloc_aggr_kprobe()
842 __prepare_optimized_kprobe(op, p); in alloc_aggr_kprobe()
847 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
853 static void try_to_optimize_kprobe(struct kprobe *p) in try_to_optimize_kprobe() argument
859 if (kprobe_ftrace(p)) in try_to_optimize_kprobe()
867 ap = alloc_aggr_kprobe(p); in try_to_optimize_kprobe()
879 init_aggr_kprobe(ap, p); in try_to_optimize_kprobe()
886 struct kprobe *p; in optimize_all_kprobes() local
898 hlist_for_each_entry(p, head, hlist) in optimize_all_kprobes()
899 if (!kprobe_disabled(p)) in optimize_all_kprobes()
900 optimize_kprobe(p); in optimize_all_kprobes()
910 struct kprobe *p; in unoptimize_all_kprobes() local
922 hlist_for_each_entry(p, head, hlist) { in unoptimize_all_kprobes()
923 if (!kprobe_disabled(p)) in unoptimize_all_kprobes()
924 unoptimize_kprobe(p, false); in unoptimize_all_kprobes()
972 static void __arm_kprobe(struct kprobe *p) in __arm_kprobe() argument
979 _p = get_optimized_kprobe(p->addr); in __arm_kprobe()
984 arch_arm_kprobe(p); in __arm_kprobe()
985 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ in __arm_kprobe()
989 static void __disarm_kprobe(struct kprobe *p, bool reopt) in __disarm_kprobe() argument
996 unoptimize_kprobe(p, kprobes_all_disarmed); in __disarm_kprobe()
998 if (!kprobe_queued(p)) { in __disarm_kprobe()
999 arch_disarm_kprobe(p); in __disarm_kprobe()
1001 _p = get_optimized_kprobe(p->addr); in __disarm_kprobe()
1015 #define optimize_kprobe(p) do {} while (0) argument
1016 #define unoptimize_kprobe(p, f) do {} while (0) argument
1017 #define kill_optimized_kprobe(p) do {} while (0) argument
1018 #define prepare_optimized_kprobe(p) do {} while (0) argument
1019 #define try_to_optimize_kprobe(p) do {} while (0) argument
1020 #define __arm_kprobe(p) arch_arm_kprobe(p) argument
1021 #define __disarm_kprobe(p, o) arch_disarm_kprobe(p) argument
1022 #define kprobe_disarmed(p) kprobe_disabled(p) argument
1038 static void free_aggr_kprobe(struct kprobe *p) in free_aggr_kprobe() argument
1040 arch_remove_kprobe(p); in free_aggr_kprobe()
1041 kfree(p); in free_aggr_kprobe()
1044 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) in alloc_aggr_kprobe() argument
1065 static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, in __arm_kprobe_ftrace() argument
1072 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0); in __arm_kprobe_ftrace()
1073 if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret)) in __arm_kprobe_ftrace()
1083 ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); in __arm_kprobe_ftrace()
1092 static int arm_kprobe_ftrace(struct kprobe *p) in arm_kprobe_ftrace() argument
1094 bool ipmodify = (p->post_handler != NULL); in arm_kprobe_ftrace()
1096 return __arm_kprobe_ftrace(p, in arm_kprobe_ftrace()
1101 static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, in __disarm_kprobe_ftrace() argument
1116 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); in __disarm_kprobe_ftrace()
1118 p->addr, ret); in __disarm_kprobe_ftrace()
1122 static int disarm_kprobe_ftrace(struct kprobe *p) in disarm_kprobe_ftrace() argument
1124 bool ipmodify = (p->post_handler != NULL); in disarm_kprobe_ftrace()
1126 return __disarm_kprobe_ftrace(p, in disarm_kprobe_ftrace()
1136 static inline int arm_kprobe_ftrace(struct kprobe *p) in arm_kprobe_ftrace() argument
1141 static inline int disarm_kprobe_ftrace(struct kprobe *p) in disarm_kprobe_ftrace() argument
1147 static int prepare_kprobe(struct kprobe *p) in prepare_kprobe() argument
1150 if (kprobe_ftrace(p)) in prepare_kprobe()
1151 return arch_prepare_kprobe_ftrace(p); in prepare_kprobe()
1153 return arch_prepare_kprobe(p); in prepare_kprobe()
1182 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) in aggr_pre_handler() argument
1186 list_for_each_entry_rcu(kp, &p->list, list) { in aggr_pre_handler()
1198 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, in aggr_post_handler() argument
1203 list_for_each_entry_rcu(kp, &p->list, list) { in aggr_post_handler()
1214 void kprobes_inc_nmissed_count(struct kprobe *p) in kprobes_inc_nmissed_count() argument
1218 if (!kprobe_aggrprobe(p)) { in kprobes_inc_nmissed_count()
1219 p->nmissed++; in kprobes_inc_nmissed_count()
1221 list_for_each_entry_rcu(kp, &p->list, list) in kprobes_inc_nmissed_count()
1248 static int add_new_kprobe(struct kprobe *ap, struct kprobe *p) in add_new_kprobe() argument
1250 if (p->post_handler) in add_new_kprobe()
1253 list_add_rcu(&p->list, &ap->list); in add_new_kprobe()
1254 if (p->post_handler && !ap->post_handler) in add_new_kprobe()
1264 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p) in init_aggr_kprobe() argument
1267 copy_kprobe(p, ap); in init_aggr_kprobe()
1269 ap->addr = p->addr; in init_aggr_kprobe()
1270 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED; in init_aggr_kprobe()
1273 if (p->post_handler && !kprobe_gone(p)) in init_aggr_kprobe()
1279 list_add_rcu(&p->list, &ap->list); in init_aggr_kprobe()
1280 hlist_replace_rcu(&p->hlist, &ap->hlist); in init_aggr_kprobe()
1286 static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p) in register_aggr_kprobe() argument
1337 copy_kprobe(ap, p); in register_aggr_kprobe()
1338 ret = add_new_kprobe(ap, p); in register_aggr_kprobe()
1341 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) { in register_aggr_kprobe()
1348 list_del_rcu(&p->list); in register_aggr_kprobe()
1382 char symname[KSYM_NAME_LEN], *p; in within_kprobe_blacklist() local
1389 p = strchr(symname, '.'); in within_kprobe_blacklist()
1390 if (!p) in within_kprobe_blacklist()
1392 *p = '\0'; in within_kprobe_blacklist()
1469 static kprobe_opcode_t *kprobe_addr(struct kprobe *p) in kprobe_addr() argument
1473 return _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry); in kprobe_addr()
1480 static struct kprobe *__get_valid_kprobe(struct kprobe *p) in __get_valid_kprobe() argument
1486 ap = get_kprobe(p->addr); in __get_valid_kprobe()
1490 if (p == ap) in __get_valid_kprobe()
1494 if (list_p == p) in __get_valid_kprobe()
1505 static inline int warn_kprobe_rereg(struct kprobe *p) in warn_kprobe_rereg() argument
1509 if (WARN_ON_ONCE(__get_valid_kprobe(p))) in warn_kprobe_rereg()
1515 static int check_ftrace_location(struct kprobe *p) in check_ftrace_location() argument
1517 unsigned long addr = (unsigned long)p->addr; in check_ftrace_location()
1521 p->flags |= KPROBE_FLAG_FTRACE; in check_ftrace_location()
1540 static int check_kprobe_address_safe(struct kprobe *p, in check_kprobe_address_safe() argument
1545 ret = check_ftrace_location(p); in check_kprobe_address_safe()
1553 if (!core_kernel_text((unsigned long) p->addr)) { in check_kprobe_address_safe()
1555 *probed_mod = __module_text_address((unsigned long) p->addr); in check_kprobe_address_safe()
1567 if (in_gate_area_no_mm((unsigned long) p->addr) || in check_kprobe_address_safe()
1568 within_kprobe_blacklist((unsigned long) p->addr) || in check_kprobe_address_safe()
1569 jump_label_text_reserved(p->addr, p->addr) || in check_kprobe_address_safe()
1570 static_call_text_reserved(p->addr, p->addr) || in check_kprobe_address_safe()
1571 find_bug((unsigned long)p->addr) || in check_kprobe_address_safe()
1572 is_cfi_preamble_symbol((unsigned long)p->addr)) { in check_kprobe_address_safe()
1583 if (within_module_init((unsigned long)p->addr, *probed_mod) && in check_kprobe_address_safe()
1593 static int __register_kprobe(struct kprobe *p) in __register_kprobe() argument
1600 old_p = get_kprobe(p->addr); in __register_kprobe()
1603 return register_aggr_kprobe(old_p, p); in __register_kprobe()
1608 ret = prepare_kprobe(p); in __register_kprobe()
1613 INIT_HLIST_NODE(&p->hlist); in __register_kprobe()
1614 hlist_add_head_rcu(&p->hlist, in __register_kprobe()
1615 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); in __register_kprobe()
1617 if (!kprobes_all_disarmed && !kprobe_disabled(p)) { in __register_kprobe()
1618 ret = arm_kprobe(p); in __register_kprobe()
1620 hlist_del_rcu(&p->hlist); in __register_kprobe()
1626 try_to_optimize_kprobe(p); in __register_kprobe()
1630 int register_kprobe(struct kprobe *p) in register_kprobe() argument
1638 addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry); in register_kprobe()
1641 p->addr = addr; in register_kprobe()
1643 ret = warn_kprobe_rereg(p); in register_kprobe()
1648 p->flags &= KPROBE_FLAG_DISABLED; in register_kprobe()
1650 p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY; in register_kprobe()
1651 p->nmissed = 0; in register_kprobe()
1652 INIT_LIST_HEAD(&p->list); in register_kprobe()
1654 ret = check_kprobe_address_safe(p, &probed_mod); in register_kprobe()
1658 ret = __register_kprobe(p); in register_kprobe()
1685 static struct kprobe *__disable_kprobe(struct kprobe *p) in __disable_kprobe() argument
1693 orig_p = __get_valid_kprobe(p); in __disable_kprobe()
1697 if (kprobe_disabled(p)) in __disable_kprobe()
1701 if (p != orig_p) in __disable_kprobe()
1702 p->flags |= KPROBE_FLAG_DISABLED; in __disable_kprobe()
1705 if (p == orig_p || aggr_kprobe_disabled(orig_p)) { in __disable_kprobe()
1715 p->flags &= ~KPROBE_FLAG_DISABLED; in __disable_kprobe()
1728 static int __unregister_kprobe_top(struct kprobe *p) in __unregister_kprobe_top() argument
1733 ap = __disable_kprobe(p); in __unregister_kprobe_top()
1737 WARN_ON(ap != p && !kprobe_aggrprobe(ap)); in __unregister_kprobe_top()
1744 if (ap == p || in __unregister_kprobe_top()
1755 if (p->post_handler && !kprobe_gone(p)) { in __unregister_kprobe_top()
1757 if ((list_p != p) && (list_p->post_handler)) in __unregister_kprobe_top()
1776 list_del_rcu(&p->list); in __unregister_kprobe_top()
1787 static void __unregister_kprobe_bottom(struct kprobe *p) in __unregister_kprobe_bottom() argument
1791 if (list_empty(&p->list)) in __unregister_kprobe_bottom()
1793 arch_remove_kprobe(p); in __unregister_kprobe_bottom()
1794 else if (list_is_singular(&p->list)) { in __unregister_kprobe_bottom()
1796 ap = list_entry(p->list.next, struct kprobe, list); in __unregister_kprobe_bottom()
1797 list_del(&p->list); in __unregister_kprobe_bottom()
1821 void unregister_kprobe(struct kprobe *p) in unregister_kprobe() argument
1823 unregister_kprobes(&p, 1); in unregister_kprobe()
2070 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) in NOKPROBE_SYMBOL()
2072 struct kretprobe *rp = container_of(p, struct kretprobe, kp); in NOKPROBE_SYMBOL()
2099 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) in pre_handler_kretprobe() argument
2101 struct kretprobe *rp = container_of(p, struct kretprobe, kp); in pre_handler_kretprobe()
2116 rethook_hook(rhn, regs, kprobe_ftrace(p)); in pre_handler_kretprobe()
2323 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) in pre_handler_kretprobe() argument
2332 static void kill_kprobe(struct kprobe *p) in kill_kprobe() argument
2343 if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed) in kill_kprobe()
2344 disarm_kprobe_ftrace(p); in kill_kprobe()
2346 p->flags |= KPROBE_FLAG_GONE; in kill_kprobe()
2347 if (kprobe_aggrprobe(p)) { in kill_kprobe()
2352 list_for_each_entry(kp, &p->list, list) in kill_kprobe()
2354 p->post_handler = NULL; in kill_kprobe()
2355 kill_optimized_kprobe(p); in kill_kprobe()
2361 arch_remove_kprobe(p); in kill_kprobe()
2367 struct kprobe *p; in disable_kprobe() local
2372 p = __disable_kprobe(kp); in disable_kprobe()
2374 return IS_ERR(p) ? PTR_ERR(p) : 0; in disable_kprobe()
2382 struct kprobe *p; in enable_kprobe() local
2387 p = __get_valid_kprobe(kp); in enable_kprobe()
2388 if (unlikely(p == NULL)) in enable_kprobe()
2395 if (p != kp) in enable_kprobe()
2398 if (!kprobes_all_disarmed && kprobe_disabled(p)) { in enable_kprobe()
2399 p->flags &= ~KPROBE_FLAG_DISABLED; in enable_kprobe()
2400 ret = arm_kprobe(p); in enable_kprobe()
2402 p->flags |= KPROBE_FLAG_DISABLED; in enable_kprobe()
2403 if (p != kp) in enable_kprobe()
2590 struct kprobe *p; in kprobes_module_callback() local
2610 hlist_for_each_entry(p, head, hlist) in kprobes_module_callback()
2611 if (within_module_init((unsigned long)p->addr, mod) || in kprobes_module_callback()
2613 within_module_core((unsigned long)p->addr, mod))) { in kprobes_module_callback()
2625 kill_kprobe(p); in kprobes_module_callback()
2654 struct kprobe *p; in kprobe_free_init_mem() local
2662 hlist_for_each_entry(p, head, hlist) { in kprobe_free_init_mem()
2663 if (start <= (void *)p->addr && (void *)p->addr < end) in kprobe_free_init_mem()
2664 kill_kprobe(p); in kprobe_free_init_mem()
2730 static void report_probe(struct seq_file *pi, struct kprobe *p, in report_probe() argument
2734 void *addr = p->addr; in report_probe()
2736 if (p->pre_handler == pre_handler_kretprobe) in report_probe()
2750 addr, kprobe_type, p->addr); in report_probe()
2753 pp = p; in report_probe()
2755 (kprobe_gone(p) ? "[GONE]" : ""), in report_probe()
2756 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""), in report_probe()
2782 struct kprobe *p, *kp; in show_kprobe_addr() local
2790 hlist_for_each_entry_rcu(p, head, hlist) { in show_kprobe_addr()
2791 sym = kallsyms_lookup((unsigned long)p->addr, NULL, in show_kprobe_addr()
2793 if (kprobe_aggrprobe(p)) { in show_kprobe_addr()
2794 list_for_each_entry_rcu(kp, &p->list, list) in show_kprobe_addr()
2795 report_probe(pi, kp, sym, offset, modname, p); in show_kprobe_addr()
2797 report_probe(pi, p, sym, offset, modname, NULL); in show_kprobe_addr()
2858 struct kprobe *p; in arm_all_kprobes() local
2878 hlist_for_each_entry(p, head, hlist) { in arm_all_kprobes()
2879 if (!kprobe_disabled(p)) { in arm_all_kprobes()
2880 err = arm_kprobe(p); in arm_all_kprobes()
2902 struct kprobe *p; in disarm_all_kprobes() local
2917 hlist_for_each_entry(p, head, hlist) { in disarm_all_kprobes()
2918 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) { in disarm_all_kprobes()
2919 err = disarm_kprobe(p, false); in disarm_all_kprobes()