Lines Matching refs:prog
2238 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) in find_prog_type() argument
2249 if (!bpf_prog_is_offloaded(prog->aux)) in find_prog_type()
2250 prog->aux->ops = ops; in find_prog_type()
2252 prog->aux->ops = &bpf_offload_prog_ops; in find_prog_type()
2253 prog->type = type; in find_prog_type()
2268 static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op) in bpf_audit_prog() argument
2283 prog->aux->id, bpf_audit_str[op]); in bpf_audit_prog()
2287 static int bpf_prog_alloc_id(struct bpf_prog *prog) in bpf_prog_alloc_id() argument
2293 id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); in bpf_prog_alloc_id()
2295 prog->aux->id = id; in bpf_prog_alloc_id()
2306 void bpf_prog_free_id(struct bpf_prog *prog) in bpf_prog_free_id() argument
2315 if (!prog->aux->id) in bpf_prog_free_id()
2319 idr_remove(&prog_idr, prog->aux->id); in bpf_prog_free_id()
2320 prog->aux->id = 0; in bpf_prog_free_id()
2331 security_bpf_prog_free(aux->prog); in __bpf_prog_put_rcu()
2332 bpf_prog_free(aux->prog); in __bpf_prog_put_rcu()
2335 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) in __bpf_prog_put_noref() argument
2337 bpf_prog_kallsyms_del_all(prog); in __bpf_prog_put_noref()
2338 btf_put(prog->aux->btf); in __bpf_prog_put_noref()
2339 module_put(prog->aux->mod); in __bpf_prog_put_noref()
2340 kvfree(prog->aux->jited_linfo); in __bpf_prog_put_noref()
2341 kvfree(prog->aux->linfo); in __bpf_prog_put_noref()
2342 kfree(prog->aux->kfunc_tab); in __bpf_prog_put_noref()
2343 kfree(prog->aux->ctx_arg_info); in __bpf_prog_put_noref()
2344 if (prog->aux->attach_btf) in __bpf_prog_put_noref()
2345 btf_put(prog->aux->attach_btf); in __bpf_prog_put_noref()
2348 if (prog->sleepable) in __bpf_prog_put_noref()
2349 call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); in __bpf_prog_put_noref()
2351 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); in __bpf_prog_put_noref()
2353 __bpf_prog_put_rcu(&prog->aux->rcu); in __bpf_prog_put_noref()
2360 struct bpf_prog *prog; in bpf_prog_put_deferred() local
2363 prog = aux->prog; in bpf_prog_put_deferred()
2364 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0); in bpf_prog_put_deferred()
2365 bpf_audit_prog(prog, BPF_AUDIT_UNLOAD); in bpf_prog_put_deferred()
2366 bpf_prog_free_id(prog); in bpf_prog_put_deferred()
2367 __bpf_prog_put_noref(prog, true); in bpf_prog_put_deferred()
2370 static void __bpf_prog_put(struct bpf_prog *prog) in __bpf_prog_put() argument
2372 struct bpf_prog_aux *aux = prog->aux; in __bpf_prog_put()
2384 void bpf_prog_put(struct bpf_prog *prog) in bpf_prog_put() argument
2386 __bpf_prog_put(prog); in bpf_prog_put()
2392 struct bpf_prog *prog = filp->private_data; in bpf_prog_release() local
2394 bpf_prog_put(prog); in bpf_prog_release()
2404 void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog) in bpf_prog_inc_misses_counter() argument
2409 stats = this_cpu_ptr(prog->stats); in bpf_prog_inc_misses_counter()
2415 static void bpf_prog_get_stats(const struct bpf_prog *prog, in bpf_prog_get_stats() argument
2426 st = per_cpu_ptr(prog->stats, cpu); in bpf_prog_get_stats()
2445 const struct bpf_prog *prog = filp->private_data; in bpf_prog_show_fdinfo() local
2446 char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; in bpf_prog_show_fdinfo()
2449 bpf_prog_get_stats(prog, &stats); in bpf_prog_show_fdinfo()
2450 bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); in bpf_prog_show_fdinfo()
2461 prog->type, in bpf_prog_show_fdinfo()
2462 prog->jited, in bpf_prog_show_fdinfo()
2464 prog->pages * 1ULL << PAGE_SHIFT, in bpf_prog_show_fdinfo()
2465 prog->aux->id, in bpf_prog_show_fdinfo()
2469 prog->aux->verified_insns); in bpf_prog_show_fdinfo()
2482 int bpf_prog_new_fd(struct bpf_prog *prog) in bpf_prog_new_fd() argument
2486 ret = security_bpf_prog(prog); in bpf_prog_new_fd()
2490 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, in bpf_prog_new_fd()
2494 void bpf_prog_add(struct bpf_prog *prog, int i) in bpf_prog_add() argument
2496 atomic64_add(i, &prog->aux->refcnt); in bpf_prog_add()
2500 void bpf_prog_sub(struct bpf_prog *prog, int i) in bpf_prog_sub() argument
2507 WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); in bpf_prog_sub()
2511 void bpf_prog_inc(struct bpf_prog *prog) in bpf_prog_inc() argument
2513 atomic64_inc(&prog->aux->refcnt); in bpf_prog_inc()
2518 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) in bpf_prog_inc_not_zero() argument
2522 refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); in bpf_prog_inc_not_zero()
2527 return prog; in bpf_prog_inc_not_zero()
2531 bool bpf_prog_get_ok(struct bpf_prog *prog, in bpf_prog_get_ok() argument
2538 if (prog->type != *attach_type) in bpf_prog_get_ok()
2540 if (bpf_prog_is_offloaded(prog->aux) && !attach_drv) in bpf_prog_get_ok()
2550 struct bpf_prog *prog; in __bpf_prog_get() local
2557 prog = fd_file(f)->private_data; in __bpf_prog_get()
2558 if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) in __bpf_prog_get()
2561 bpf_prog_inc(prog); in __bpf_prog_get()
2562 return prog; in __bpf_prog_get()
2770 struct bpf_prog *prog, *dst_prog = NULL; in bpf_prog_load() local
2889 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); in bpf_prog_load()
2890 if (!prog) { in bpf_prog_load()
2899 prog->expected_attach_type = attr->expected_attach_type; in bpf_prog_load()
2900 prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE); in bpf_prog_load()
2901 prog->aux->attach_btf = attach_btf; in bpf_prog_load()
2902 prog->aux->attach_btf_id = attr->attach_btf_id; in bpf_prog_load()
2903 prog->aux->dst_prog = dst_prog; in bpf_prog_load()
2904 prog->aux->dev_bound = !!attr->prog_ifindex; in bpf_prog_load()
2905 prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; in bpf_prog_load()
2908 prog->aux->token = token; in bpf_prog_load()
2911 prog->aux->user = get_current_user(); in bpf_prog_load()
2912 prog->len = attr->insn_cnt; in bpf_prog_load()
2915 if (copy_from_bpfptr(prog->insns, in bpf_prog_load()
2917 bpf_prog_insn_size(prog)) != 0) in bpf_prog_load()
2927 prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; in bpf_prog_load()
2929 prog->orig_prog = NULL; in bpf_prog_load()
2930 prog->jited = 0; in bpf_prog_load()
2932 atomic64_set(&prog->aux->refcnt, 1); in bpf_prog_load()
2934 if (bpf_prog_is_dev_bound(prog->aux)) { in bpf_prog_load()
2935 err = bpf_prog_dev_bound_init(prog, attr); in bpf_prog_load()
2942 err = bpf_prog_dev_bound_inherit(prog, dst_prog); in bpf_prog_load()
2960 prog->aux->attach_tracing_prog = true; in bpf_prog_load()
2964 err = find_prog_type(type, prog); in bpf_prog_load()
2968 prog->aux->load_time = ktime_get_boottime_ns(); in bpf_prog_load()
2969 err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, in bpf_prog_load()
2974 err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel); in bpf_prog_load()
2979 err = bpf_check(&prog, attr, uattr, uattr_size); in bpf_prog_load()
2983 prog = bpf_prog_select_runtime(prog, &err); in bpf_prog_load()
2987 err = bpf_prog_alloc_id(prog); in bpf_prog_load()
3005 bpf_prog_kallsyms_add(prog); in bpf_prog_load()
3006 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); in bpf_prog_load()
3007 bpf_audit_prog(prog, BPF_AUDIT_LOAD); in bpf_prog_load()
3009 err = bpf_prog_new_fd(prog); in bpf_prog_load()
3011 bpf_prog_put(prog); in bpf_prog_load()
3019 __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); in bpf_prog_load()
3023 security_bpf_prog_free(prog); in bpf_prog_load()
3025 free_uid(prog->aux->user); in bpf_prog_load()
3026 if (prog->aux->attach_btf) in bpf_prog_load()
3027 btf_put(prog->aux->attach_btf); in bpf_prog_load()
3028 bpf_prog_free(prog); in bpf_prog_load()
3078 const struct bpf_link_ops *ops, struct bpf_prog *prog, in bpf_link_init_sleepable() argument
3087 link->prog = prog; in bpf_link_init_sleepable()
3092 const struct bpf_link_ops *ops, struct bpf_prog *prog, in bpf_link_init() argument
3095 bpf_link_init_sleepable(link, type, ops, prog, attach_type, false); in bpf_link_init()
3119 primer->link->prog = NULL; in bpf_link_cleanup()
3133 if (link->prog) in bpf_link_dealloc()
3134 bpf_prog_put(link->prog); in bpf_link_dealloc()
3165 if (link->prog) in bpf_link_free()
3176 if (link->sleepable || (link->prog && link->prog->sleepable)) in bpf_link_free()
3235 const struct bpf_prog *prog = link->prog; in bpf_link_show_fdinfo() local
3237 char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; in bpf_link_show_fdinfo()
3254 if (prog) { in bpf_link_show_fdinfo()
3255 bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); in bpf_link_show_fdinfo()
3260 prog->aux->id); in bpf_link_show_fdinfo()
3452 static int bpf_tracing_prog_attach(struct bpf_prog *prog, in bpf_tracing_prog_attach() argument
3465 switch (prog->type) { in bpf_tracing_prog_attach()
3467 if (prog->expected_attach_type != BPF_TRACE_FENTRY && in bpf_tracing_prog_attach()
3468 prog->expected_attach_type != BPF_TRACE_FEXIT && in bpf_tracing_prog_attach()
3469 prog->expected_attach_type != BPF_MODIFY_RETURN) { in bpf_tracing_prog_attach()
3475 if (prog->expected_attach_type != 0) { in bpf_tracing_prog_attach()
3481 if (prog->expected_attach_type != BPF_LSM_MAC) { in bpf_tracing_prog_attach()
3503 if (prog->type != BPF_PROG_TYPE_EXT) { in bpf_tracing_prog_attach()
3524 &bpf_tracing_link_lops, prog, attach_type); in bpf_tracing_prog_attach()
3528 mutex_lock(&prog->aux->dst_mutex); in bpf_tracing_prog_attach()
3552 if (!prog->aux->dst_trampoline && !tgt_prog) { in bpf_tracing_prog_attach()
3559 if (prog->type != BPF_PROG_TYPE_TRACING && in bpf_tracing_prog_attach()
3560 prog->type != BPF_PROG_TYPE_LSM) { in bpf_tracing_prog_attach()
3565 if (!prog->aux->attach_btf) { in bpf_tracing_prog_attach()
3569 btf_id = prog->aux->attach_btf_id; in bpf_tracing_prog_attach()
3570 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id); in bpf_tracing_prog_attach()
3573 if (!prog->aux->dst_trampoline || in bpf_tracing_prog_attach()
3574 (key && key != prog->aux->dst_trampoline->key)) { in bpf_tracing_prog_attach()
3581 err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id, in bpf_tracing_prog_attach()
3587 module_put(prog->aux->mod); in bpf_tracing_prog_attach()
3588 prog->aux->mod = tgt_info.tgt_mod; in bpf_tracing_prog_attach()
3604 tr = prog->aux->dst_trampoline; in bpf_tracing_prog_attach()
3605 tgt_prog = prog->aux->dst_prog; in bpf_tracing_prog_attach()
3626 if (prog->aux->dst_prog && in bpf_tracing_prog_attach()
3627 (tgt_prog_fd || tr != prog->aux->dst_trampoline)) in bpf_tracing_prog_attach()
3629 bpf_prog_put(prog->aux->dst_prog); in bpf_tracing_prog_attach()
3630 if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline) in bpf_tracing_prog_attach()
3632 bpf_trampoline_put(prog->aux->dst_trampoline); in bpf_tracing_prog_attach()
3634 prog->aux->dst_prog = NULL; in bpf_tracing_prog_attach()
3635 prog->aux->dst_trampoline = NULL; in bpf_tracing_prog_attach()
3636 mutex_unlock(&prog->aux->dst_mutex); in bpf_tracing_prog_attach()
3640 if (tr && tr != prog->aux->dst_trampoline) in bpf_tracing_prog_attach()
3642 mutex_unlock(&prog->aux->dst_mutex); in bpf_tracing_prog_attach()
3955 switch (event->prog->type) { in bpf_perf_link_fill_link_info()
4023 switch (event->prog->type) { in bpf_perf_link_show_fdinfo()
4042 static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) in bpf_perf_link_attach() argument
4062 bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog, in bpf_perf_link_attach()
4073 err = perf_event_set_bpf_prog(event, prog, attr->link_create.perf_event.bpf_cookie); in bpf_perf_link_attach()
4079 bpf_prog_inc(prog); in bpf_perf_link_attach()
4088 static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) in bpf_perf_link_attach() argument
4094 static int bpf_raw_tp_link_attach(struct bpf_prog *prog, in bpf_raw_tp_link_attach() argument
4105 switch (prog->type) { in bpf_raw_tp_link_attach()
4114 if (prog->type == BPF_PROG_TYPE_TRACING && in bpf_raw_tp_link_attach()
4115 prog->expected_attach_type == BPF_TRACE_RAW_TP) { in bpf_raw_tp_link_attach()
4116 tp_name = prog->aux->attach_func_name; in bpf_raw_tp_link_attach()
4119 return bpf_tracing_prog_attach(prog, 0, 0, 0, attach_type); in bpf_raw_tp_link_attach()
4141 &bpf_raw_tp_link_lops, prog, attach_type, in bpf_raw_tp_link_attach()
4169 struct bpf_prog *prog; in bpf_raw_tracepoint_open() local
4177 prog = bpf_prog_get(attr->raw_tracepoint.prog_fd); in bpf_raw_tracepoint_open()
4178 if (IS_ERR(prog)) in bpf_raw_tracepoint_open()
4179 return PTR_ERR(prog); in bpf_raw_tracepoint_open()
4183 fd = bpf_raw_tp_link_attach(prog, tp_name, cookie, prog->expected_attach_type); in bpf_raw_tracepoint_open()
4185 bpf_prog_put(prog); in bpf_raw_tracepoint_open()
4262 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, in bpf_prog_attach_check_attach_type() argument
4267 switch (prog->type) { in bpf_prog_attach_check_attach_type()
4272 return attach_type == prog->expected_attach_type ? 0 : -EINVAL; in bpf_prog_attach_check_attach_type()
4274 if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN)) in bpf_prog_attach_check_attach_type()
4281 if (prog->type != ptype) in bpf_prog_attach_check_attach_type()
4284 return prog->enforce_expected_attach_type && in bpf_prog_attach_check_attach_type()
4285 prog->expected_attach_type != attach_type ? in bpf_prog_attach_check_attach_type()
4299 if (prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI && in bpf_prog_attach_check_attach_type()
4302 if (prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION && in bpf_prog_attach_check_attach_type()
4305 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI && in bpf_prog_attach_check_attach_type()
4308 if (prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION && in bpf_prog_attach_check_attach_type()
4327 if (ptype == BPF_PROG_TYPE_UNSPEC || ptype != prog->type) in bpf_prog_attach_check_attach_type()
4370 struct bpf_prog *prog; in bpf_prog_attach() local
4393 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); in bpf_prog_attach()
4394 if (IS_ERR(prog)) in bpf_prog_attach()
4395 return PTR_ERR(prog); in bpf_prog_attach()
4397 if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { in bpf_prog_attach()
4398 bpf_prog_put(prog); in bpf_prog_attach()
4402 if (is_cgroup_prog_type(ptype, prog->expected_attach_type, true)) { in bpf_prog_attach()
4403 ret = cgroup_bpf_prog_attach(attr, ptype, prog); in bpf_prog_attach()
4410 ret = sock_map_get_from_fd(attr, prog); in bpf_prog_attach()
4413 ret = lirc_prog_attach(attr, prog); in bpf_prog_attach()
4416 ret = netns_bpf_prog_attach(attr, prog); in bpf_prog_attach()
4421 ret = tcx_prog_attach(attr, prog); in bpf_prog_attach()
4423 ret = netkit_prog_attach(attr, prog); in bpf_prog_attach()
4430 bpf_prog_put(prog); in bpf_prog_attach()
4438 struct bpf_prog *prog = NULL; in bpf_prog_detach() local
4452 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); in bpf_prog_detach()
4453 if (IS_ERR(prog)) in bpf_prog_detach()
4454 return PTR_ERR(prog); in bpf_prog_detach()
4489 ret = tcx_prog_detach(attr, prog); in bpf_prog_detach()
4491 ret = netkit_prog_detach(attr, prog); in bpf_prog_detach()
4497 if (prog) in bpf_prog_detach()
4498 bpf_prog_put(prog); in bpf_prog_detach()
4571 struct bpf_prog *prog; in bpf_prog_test_run() local
4585 prog = bpf_prog_get(attr->test.prog_fd); in bpf_prog_test_run()
4586 if (IS_ERR(prog)) in bpf_prog_test_run()
4587 return PTR_ERR(prog); in bpf_prog_test_run()
4589 if (prog->aux->ops->test_run) in bpf_prog_test_run()
4590 ret = prog->aux->ops->test_run(prog, attr, uattr); in bpf_prog_test_run()
4592 bpf_prog_put(prog); in bpf_prog_test_run()
4645 struct bpf_prog *prog; in bpf_prog_get_curr_or_next() local
4649 prog = idr_get_next(&prog_idr, id); in bpf_prog_get_curr_or_next()
4650 if (prog) { in bpf_prog_get_curr_or_next()
4651 prog = bpf_prog_inc_not_zero(prog); in bpf_prog_get_curr_or_next()
4652 if (IS_ERR(prog)) { in bpf_prog_get_curr_or_next()
4659 return prog; in bpf_prog_get_curr_or_next()
4666 struct bpf_prog *prog; in bpf_prog_by_id() local
4672 prog = idr_find(&prog_idr, id); in bpf_prog_by_id()
4673 if (prog) in bpf_prog_by_id()
4674 prog = bpf_prog_inc_not_zero(prog); in bpf_prog_by_id()
4676 prog = ERR_PTR(-ENOENT); in bpf_prog_by_id()
4678 return prog; in bpf_prog_by_id()
4683 struct bpf_prog *prog; in bpf_prog_get_fd_by_id() local
4693 prog = bpf_prog_by_id(id); in bpf_prog_get_fd_by_id()
4694 if (IS_ERR(prog)) in bpf_prog_get_fd_by_id()
4695 return PTR_ERR(prog); in bpf_prog_get_fd_by_id()
4697 fd = bpf_prog_new_fd(prog); in bpf_prog_get_fd_by_id()
4699 bpf_prog_put(prog); in bpf_prog_get_fd_by_id()
4742 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, in bpf_map_from_imm() argument
4749 mutex_lock(&prog->aux->used_maps_mutex); in bpf_map_from_imm()
4750 for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { in bpf_map_from_imm()
4751 map = prog->aux->used_maps[i]; in bpf_map_from_imm()
4766 mutex_unlock(&prog->aux->used_maps_mutex); in bpf_map_from_imm()
4770 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, in bpf_insn_prepare_dump() argument
4780 insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), in bpf_insn_prepare_dump()
4785 for (i = 0; i < prog->len; i++) { in bpf_insn_prepare_dump()
4816 map = bpf_map_from_imm(prog, imm, &off, &type); in bpf_insn_prepare_dump()
4860 struct bpf_prog *prog, in bpf_prog_get_info_by_fd() argument
4865 struct btf *attach_btf = bpf_prog_get_target_btf(prog); in bpf_prog_get_info_by_fd()
4882 info.type = prog->type; in bpf_prog_get_info_by_fd()
4883 info.id = prog->aux->id; in bpf_prog_get_info_by_fd()
4884 info.load_time = prog->aux->load_time; in bpf_prog_get_info_by_fd()
4886 prog->aux->user->uid); in bpf_prog_get_info_by_fd()
4887 info.gpl_compatible = prog->gpl_compatible; in bpf_prog_get_info_by_fd()
4889 memcpy(info.tag, prog->tag, sizeof(prog->tag)); in bpf_prog_get_info_by_fd()
4890 memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); in bpf_prog_get_info_by_fd()
4892 mutex_lock(&prog->aux->used_maps_mutex); in bpf_prog_get_info_by_fd()
4894 info.nr_map_ids = prog->aux->used_map_cnt; in bpf_prog_get_info_by_fd()
4901 if (put_user(prog->aux->used_maps[i]->id, in bpf_prog_get_info_by_fd()
4903 mutex_unlock(&prog->aux->used_maps_mutex); in bpf_prog_get_info_by_fd()
4907 mutex_unlock(&prog->aux->used_maps_mutex); in bpf_prog_get_info_by_fd()
4913 bpf_prog_get_stats(prog, &stats); in bpf_prog_get_info_by_fd()
4918 info.verified_insns = prog->aux->verified_insns; in bpf_prog_get_info_by_fd()
4919 if (prog->aux->btf) in bpf_prog_get_info_by_fd()
4920 info.btf_id = btf_obj_id(prog->aux->btf); in bpf_prog_get_info_by_fd()
4934 info.xlated_prog_len = bpf_prog_insn_size(prog); in bpf_prog_get_info_by_fd()
4939 if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) { in bpf_prog_get_info_by_fd()
4943 insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); in bpf_prog_get_info_by_fd()
4954 if (bpf_prog_is_offloaded(prog->aux)) { in bpf_prog_get_info_by_fd()
4955 err = bpf_prog_offload_info_fill(&info, prog); in bpf_prog_get_info_by_fd()
4966 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
4970 for (i = 0; i < prog->aux->func_cnt; i++) in bpf_prog_get_info_by_fd()
4971 info.jited_prog_len += prog->aux->func[i]->jited_len; in bpf_prog_get_info_by_fd()
4973 info.jited_prog_len = prog->jited_len; in bpf_prog_get_info_by_fd()
4984 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
4989 for (i = 0; i < prog->aux->func_cnt; i++) { in bpf_prog_get_info_by_fd()
4990 len = prog->aux->func[i]->jited_len; in bpf_prog_get_info_by_fd()
4992 img = (u8 *) prog->aux->func[i]->bpf_func; in bpf_prog_get_info_by_fd()
5001 if (copy_to_user(uinsns, prog->bpf_func, ulen)) in bpf_prog_get_info_by_fd()
5010 info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; in bpf_prog_get_info_by_fd()
5022 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
5025 prog->aux->func[i]->bpf_func; in bpf_prog_get_info_by_fd()
5031 ksym_addr = (unsigned long) prog->bpf_func; in bpf_prog_get_info_by_fd()
5041 info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; in bpf_prog_get_info_by_fd()
5050 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
5053 prog->aux->func[i]->jited_len; in bpf_prog_get_info_by_fd()
5058 func_len = prog->jited_len; in bpf_prog_get_info_by_fd()
5067 info.attach_btf_id = prog->aux->attach_btf_id; in bpf_prog_get_info_by_fd()
5072 info.nr_func_info = prog->aux->func_info_cnt; in bpf_prog_get_info_by_fd()
5078 if (copy_to_user(user_finfo, prog->aux->func_info, in bpf_prog_get_info_by_fd()
5084 info.nr_line_info = prog->aux->nr_linfo; in bpf_prog_get_info_by_fd()
5090 if (copy_to_user(user_linfo, prog->aux->linfo, in bpf_prog_get_info_by_fd()
5096 if (prog->aux->jited_linfo) in bpf_prog_get_info_by_fd()
5097 info.nr_jited_line_info = prog->aux->nr_linfo; in bpf_prog_get_info_by_fd()
5109 line_addr = (unsigned long)prog->aux->jited_linfo[i]; in bpf_prog_get_info_by_fd()
5119 info.nr_prog_tags = prog->aux->func_cnt ? : 1; in bpf_prog_get_info_by_fd()
5126 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
5129 prog->aux->func[i]->tag, in bpf_prog_get_info_by_fd()
5135 prog->tag, BPF_TAG_SIZE)) in bpf_prog_get_info_by_fd()
5232 if (link->prog) in bpf_link_get_info_by_fd()
5233 info.prog_id = link->prog->aux->id; in bpf_link_get_info_by_fd()
5435 raw_tp->link.prog->aux->id, in bpf_task_fd_query()
5526 struct bpf_prog *prog; in link_create() local
5535 prog = bpf_prog_get(attr->link_create.prog_fd); in link_create()
5536 if (IS_ERR(prog)) in link_create()
5537 return PTR_ERR(prog); in link_create()
5539 ret = bpf_prog_attach_check_attach_type(prog, in link_create()
5544 switch (prog->type) { in link_create()
5552 ret = cgroup_bpf_link_attach(attr, prog); in link_create()
5555 ret = bpf_tracing_prog_attach(prog, in link_create()
5563 if (attr->link_create.attach_type != prog->expected_attach_type) { in link_create()
5567 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) in link_create()
5568 ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie, in link_create()
5570 else if (prog->expected_attach_type == BPF_TRACE_ITER) in link_create()
5571 ret = bpf_iter_link_attach(attr, uattr, prog); in link_create()
5572 else if (prog->expected_attach_type == BPF_LSM_CGROUP) in link_create()
5573 ret = cgroup_bpf_link_attach(attr, prog); in link_create()
5575 ret = bpf_tracing_prog_attach(prog, in link_create()
5583 ret = netns_bpf_link_create(attr, prog); in link_create()
5587 ret = sock_map_link_create(attr, prog); in link_create()
5591 ret = bpf_xdp_link_attach(attr, prog); in link_create()
5596 ret = tcx_link_attach(attr, prog); in link_create()
5598 ret = netkit_link_attach(attr, prog); in link_create()
5601 ret = bpf_nf_link_attach(attr, prog); in link_create()
5606 ret = bpf_perf_link_attach(attr, prog); in link_create()
5610 ret = bpf_perf_link_attach(attr, prog); in link_create()
5613 ret = bpf_kprobe_multi_link_attach(attr, prog); in link_create()
5616 ret = bpf_uprobe_multi_link_attach(attr, prog); in link_create()
5624 bpf_prog_put(prog); in link_create()
5891 struct bpf_prog *prog; in bpf_prog_bind_map() local
5902 prog = bpf_prog_get(attr->prog_bind_map.prog_fd); in bpf_prog_bind_map()
5903 if (IS_ERR(prog)) in bpf_prog_bind_map()
5904 return PTR_ERR(prog); in bpf_prog_bind_map()
5912 mutex_lock(&prog->aux->used_maps_mutex); in bpf_prog_bind_map()
5914 used_maps_old = prog->aux->used_maps; in bpf_prog_bind_map()
5916 for (i = 0; i < prog->aux->used_map_cnt; i++) in bpf_prog_bind_map()
5922 used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1, in bpf_prog_bind_map()
5933 if (prog->sleepable) in bpf_prog_bind_map()
5936 sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); in bpf_prog_bind_map()
5937 used_maps_new[prog->aux->used_map_cnt] = map; in bpf_prog_bind_map()
5939 prog->aux->used_map_cnt++; in bpf_prog_bind_map()
5940 prog->aux->used_maps = used_maps_new; in bpf_prog_bind_map()
5945 mutex_unlock(&prog->aux->used_maps_mutex); in bpf_prog_bind_map()
5950 bpf_prog_put(prog); in bpf_prog_bind_map()
5974 struct bpf_prog *prog; in prog_stream_read() local
5980 prog = bpf_prog_get(attr->prog_stream_read.prog_fd); in prog_stream_read()
5981 if (IS_ERR(prog)) in prog_stream_read()
5982 return PTR_ERR(prog); in prog_stream_read()
5984 ret = bpf_prog_stream_read(prog, attr->prog_stream_read.stream_id, buf, len); in prog_stream_read()
5985 bpf_prog_put(prog); in prog_stream_read()
6144 const struct bpf_prog *prog, in syscall_prog_is_valid_access() argument
6183 struct bpf_prog * __maybe_unused prog; in kern_sys_bpf() local
6194 prog = bpf_prog_get_type(attr->test.prog_fd, BPF_PROG_TYPE_SYSCALL); in kern_sys_bpf()
6195 if (IS_ERR(prog)) in kern_sys_bpf()
6196 return PTR_ERR(prog); in kern_sys_bpf()
6198 if (attr->test.ctx_size_in < prog->aux->max_ctx_offset || in kern_sys_bpf()
6200 bpf_prog_put(prog); in kern_sys_bpf()
6205 if (!__bpf_prog_enter_sleepable_recur(prog, &run_ctx)) { in kern_sys_bpf()
6207 __bpf_prog_exit_sleepable_recur(prog, 0, &run_ctx); in kern_sys_bpf()
6208 bpf_prog_put(prog); in kern_sys_bpf()
6211 attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in); in kern_sys_bpf()
6212 __bpf_prog_exit_sleepable_recur(prog, 0 /* bpf_prog_run does runtime stats */, in kern_sys_bpf()
6214 bpf_prog_put(prog); in kern_sys_bpf()
6233 tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) in tracing_prog_func_proto() argument
6235 return bpf_base_func_proto(func_id, prog); in tracing_prog_func_proto()
6283 syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) in syscall_prog_func_proto() argument
6287 return !bpf_token_capable(prog->aux->token, CAP_PERFMON) in syscall_prog_func_proto()
6296 return tracing_prog_func_proto(func_id, prog); in syscall_prog_func_proto()