Lines Matching refs:prog
107 bool bpf_prog_has_trampoline(const struct bpf_prog *prog) in bpf_prog_has_trampoline() argument
109 enum bpf_attach_type eatype = prog->expected_attach_type; in bpf_prog_has_trampoline()
110 enum bpf_prog_type ptype = prog->type; in bpf_prog_has_trampoline()
251 *ip_arg |= link->link.prog->call_get_func_ip; in bpf_trampoline_get_progs()
509 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) in bpf_attach_type_to_tramp() argument
511 switch (prog->expected_attach_type) { in bpf_attach_type_to_tramp()
519 if (!prog->aux->attach_func_proto->type) in bpf_attach_type_to_tramp()
558 kind = bpf_attach_type_to_tramp(link->link.prog); in __bpf_trampoline_link_prog()
575 tr->extension_prog = link->link.prog; in __bpf_trampoline_link_prog()
577 link->link.prog->bpf_func); in __bpf_trampoline_link_prog()
585 if (link_exiting->link.prog != link->link.prog) in __bpf_trampoline_link_prog()
620 kind = bpf_attach_type_to_tramp(link->link.prog); in __bpf_trampoline_unlink_prog()
675 static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog, in cgroup_shim_alloc() argument
697 p->aux->attach_func_proto = prog->aux->attach_func_proto; in cgroup_shim_alloc()
698 p->aux->attach_btf_id = prog->aux->attach_btf_id; in cgroup_shim_alloc()
699 p->aux->attach_btf = prog->aux->attach_btf; in cgroup_shim_alloc()
719 struct bpf_prog *p = link->link.prog; in cgroup_shim_find()
729 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, in bpf_trampoline_link_cgroup_shim() argument
740 err = bpf_check_attach_target(NULL, prog, NULL, in bpf_trampoline_link_cgroup_shim()
741 prog->aux->attach_btf_id, in bpf_trampoline_link_cgroup_shim()
746 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, in bpf_trampoline_link_cgroup_shim()
747 prog->aux->attach_btf_id); in bpf_trampoline_link_cgroup_shim()
749 bpf_lsm_find_cgroup_shim(prog, &bpf_func); in bpf_trampoline_link_cgroup_shim()
768 shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype, attach_type); in bpf_trampoline_link_cgroup_shim()
796 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) in bpf_trampoline_unlink_cgroup_shim() argument
803 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, in bpf_trampoline_unlink_cgroup_shim()
804 prog->aux->attach_btf_id); in bpf_trampoline_unlink_cgroup_shim()
806 bpf_lsm_find_cgroup_shim(prog, &bpf_func); in bpf_trampoline_unlink_cgroup_shim()
899 static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx) in __bpf_prog_enter_recur() argument
907 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) { in __bpf_prog_enter_recur()
908 bpf_prog_inc_misses_counter(prog); in __bpf_prog_enter_recur()
909 if (prog->aux->recursion_detected) in __bpf_prog_enter_recur()
910 prog->aux->recursion_detected(prog); in __bpf_prog_enter_recur()
916 static void notrace __update_prog_stats(struct bpf_prog *prog, u64 start) in __update_prog_stats() argument
930 stats = this_cpu_ptr(prog->stats); in __update_prog_stats()
937 static __always_inline void notrace update_prog_stats(struct bpf_prog *prog, in update_prog_stats() argument
941 __update_prog_stats(prog, start); in update_prog_stats()
944 static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start, in __bpf_prog_exit_recur() argument
950 update_prog_stats(prog, start); in __bpf_prog_exit_recur()
951 this_cpu_dec(*(prog->active)); in __bpf_prog_exit_recur()
956 static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog, in __bpf_prog_enter_lsm_cgroup() argument
971 static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start, in __bpf_prog_exit_lsm_cgroup() argument
981 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, in __bpf_prog_enter_sleepable_recur() argument
990 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) { in __bpf_prog_enter_sleepable_recur()
991 bpf_prog_inc_misses_counter(prog); in __bpf_prog_enter_sleepable_recur()
992 if (prog->aux->recursion_detected) in __bpf_prog_enter_sleepable_recur()
993 prog->aux->recursion_detected(prog); in __bpf_prog_enter_sleepable_recur()
999 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start, in __bpf_prog_exit_sleepable_recur() argument
1004 update_prog_stats(prog, start); in __bpf_prog_exit_sleepable_recur()
1005 this_cpu_dec(*(prog->active)); in __bpf_prog_exit_sleepable_recur()
1010 static u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, in __bpf_prog_enter_sleepable() argument
1022 static void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start, in __bpf_prog_exit_sleepable() argument
1027 update_prog_stats(prog, start); in __bpf_prog_exit_sleepable()
1032 static u64 notrace __bpf_prog_enter(struct bpf_prog *prog, in __bpf_prog_enter() argument
1044 static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, in __bpf_prog_exit() argument
1050 update_prog_stats(prog, start); in __bpf_prog_exit()
1065 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog) in bpf_trampoline_enter() argument
1067 bool sleepable = prog->sleepable; in bpf_trampoline_enter()
1069 if (bpf_prog_check_recur(prog)) in bpf_trampoline_enter()
1073 if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM && in bpf_trampoline_enter()
1074 prog->expected_attach_type == BPF_LSM_CGROUP) in bpf_trampoline_enter()
1080 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog) in bpf_trampoline_exit() argument
1082 bool sleepable = prog->sleepable; in bpf_trampoline_exit()
1084 if (bpf_prog_check_recur(prog)) in bpf_trampoline_exit()
1088 if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM && in bpf_trampoline_exit()
1089 prog->expected_attach_type == BPF_LSM_CGROUP) in bpf_trampoline_exit()