Lines Matching refs:aux
2249 if (!bpf_prog_is_offloaded(prog->aux)) in find_prog_type()
2250 prog->aux->ops = ops; in find_prog_type()
2252 prog->aux->ops = &bpf_offload_prog_ops; in find_prog_type()
2283 prog->aux->id, bpf_audit_str[op]); in bpf_audit_prog()
2295 prog->aux->id = id; in bpf_prog_alloc_id()
2315 if (!prog->aux->id) in bpf_prog_free_id()
2319 idr_remove(&prog_idr, prog->aux->id); in bpf_prog_free_id()
2320 prog->aux->id = 0; in bpf_prog_free_id()
2326 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); in __bpf_prog_put_rcu() local
2328 kvfree(aux->func_info); in __bpf_prog_put_rcu()
2329 kfree(aux->func_info_aux); in __bpf_prog_put_rcu()
2330 free_uid(aux->user); in __bpf_prog_put_rcu()
2331 security_bpf_prog_free(aux->prog); in __bpf_prog_put_rcu()
2332 bpf_prog_free(aux->prog); in __bpf_prog_put_rcu()
2338 btf_put(prog->aux->btf); in __bpf_prog_put_noref()
2339 module_put(prog->aux->mod); in __bpf_prog_put_noref()
2340 kvfree(prog->aux->jited_linfo); in __bpf_prog_put_noref()
2341 kvfree(prog->aux->linfo); in __bpf_prog_put_noref()
2342 kfree(prog->aux->kfunc_tab); in __bpf_prog_put_noref()
2343 kfree(prog->aux->ctx_arg_info); in __bpf_prog_put_noref()
2344 if (prog->aux->attach_btf) in __bpf_prog_put_noref()
2345 btf_put(prog->aux->attach_btf); in __bpf_prog_put_noref()
2349 call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); in __bpf_prog_put_noref()
2351 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); in __bpf_prog_put_noref()
2353 __bpf_prog_put_rcu(&prog->aux->rcu); in __bpf_prog_put_noref()
2359 struct bpf_prog_aux *aux; in bpf_prog_put_deferred() local
2362 aux = container_of(work, struct bpf_prog_aux, work); in bpf_prog_put_deferred()
2363 prog = aux->prog; in bpf_prog_put_deferred()
2372 struct bpf_prog_aux *aux = prog->aux; in __bpf_prog_put() local
2374 if (atomic64_dec_and_test(&aux->refcnt)) { in __bpf_prog_put()
2376 INIT_WORK(&aux->work, bpf_prog_put_deferred); in __bpf_prog_put()
2377 schedule_work(&aux->work); in __bpf_prog_put()
2379 bpf_prog_put_deferred(&aux->work); in __bpf_prog_put()
2465 prog->aux->id, in bpf_prog_show_fdinfo()
2469 prog->aux->verified_insns); in bpf_prog_show_fdinfo()
2496 atomic64_add(i, &prog->aux->refcnt); in bpf_prog_add()
2507 WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0); in bpf_prog_sub()
2513 atomic64_inc(&prog->aux->refcnt); in bpf_prog_inc()
2522 refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0); in bpf_prog_inc_not_zero()
2540 if (bpf_prog_is_offloaded(prog->aux) && !attach_drv) in bpf_prog_get_ok()
2901 prog->aux->attach_btf = attach_btf; in bpf_prog_load()
2902 prog->aux->attach_btf_id = attr->attach_btf_id; in bpf_prog_load()
2903 prog->aux->dst_prog = dst_prog; in bpf_prog_load()
2904 prog->aux->dev_bound = !!attr->prog_ifindex; in bpf_prog_load()
2905 prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; in bpf_prog_load()
2908 prog->aux->token = token; in bpf_prog_load()
2911 prog->aux->user = get_current_user(); in bpf_prog_load()
2932 atomic64_set(&prog->aux->refcnt, 1); in bpf_prog_load()
2934 if (bpf_prog_is_dev_bound(prog->aux)) { in bpf_prog_load()
2941 bpf_prog_is_dev_bound(dst_prog->aux)) { in bpf_prog_load()
2960 prog->aux->attach_tracing_prog = true; in bpf_prog_load()
2968 prog->aux->load_time = ktime_get_boottime_ns(); in bpf_prog_load()
2969 err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, in bpf_prog_load()
3019 __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); in bpf_prog_load()
3025 free_uid(prog->aux->user); in bpf_prog_load()
3026 if (prog->aux->attach_btf) in bpf_prog_load()
3027 btf_put(prog->aux->attach_btf); in bpf_prog_load()
3260 prog->aux->id); in bpf_link_show_fdinfo()
3528 mutex_lock(&prog->aux->dst_mutex); in bpf_tracing_prog_attach()
3552 if (!prog->aux->dst_trampoline && !tgt_prog) { in bpf_tracing_prog_attach()
3565 if (!prog->aux->attach_btf) { in bpf_tracing_prog_attach()
3569 btf_id = prog->aux->attach_btf_id; in bpf_tracing_prog_attach()
3570 key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf, btf_id); in bpf_tracing_prog_attach()
3573 if (!prog->aux->dst_trampoline || in bpf_tracing_prog_attach()
3574 (key && key != prog->aux->dst_trampoline->key)) { in bpf_tracing_prog_attach()
3587 module_put(prog->aux->mod); in bpf_tracing_prog_attach()
3588 prog->aux->mod = tgt_info.tgt_mod; in bpf_tracing_prog_attach()
3604 tr = prog->aux->dst_trampoline; in bpf_tracing_prog_attach()
3605 tgt_prog = prog->aux->dst_prog; in bpf_tracing_prog_attach()
3626 if (prog->aux->dst_prog && in bpf_tracing_prog_attach()
3627 (tgt_prog_fd || tr != prog->aux->dst_trampoline)) in bpf_tracing_prog_attach()
3629 bpf_prog_put(prog->aux->dst_prog); in bpf_tracing_prog_attach()
3630 if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline) in bpf_tracing_prog_attach()
3632 bpf_trampoline_put(prog->aux->dst_trampoline); in bpf_tracing_prog_attach()
3634 prog->aux->dst_prog = NULL; in bpf_tracing_prog_attach()
3635 prog->aux->dst_trampoline = NULL; in bpf_tracing_prog_attach()
3636 mutex_unlock(&prog->aux->dst_mutex); in bpf_tracing_prog_attach()
3640 if (tr && tr != prog->aux->dst_trampoline) in bpf_tracing_prog_attach()
3642 mutex_unlock(&prog->aux->dst_mutex); in bpf_tracing_prog_attach()
4116 tp_name = prog->aux->attach_func_name; in bpf_raw_tp_link_attach()
4274 if (!bpf_token_capable(prog->aux->token, CAP_NET_ADMIN)) in bpf_prog_attach_check_attach_type()
4589 if (prog->aux->ops->test_run) in bpf_prog_test_run()
4590 ret = prog->aux->ops->test_run(prog, attr, uattr); in bpf_prog_test_run()
4749 mutex_lock(&prog->aux->used_maps_mutex); in bpf_map_from_imm()
4750 for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { in bpf_map_from_imm()
4751 map = prog->aux->used_maps[i]; in bpf_map_from_imm()
4766 mutex_unlock(&prog->aux->used_maps_mutex); in bpf_map_from_imm()
4883 info.id = prog->aux->id; in bpf_prog_get_info_by_fd()
4884 info.load_time = prog->aux->load_time; in bpf_prog_get_info_by_fd()
4886 prog->aux->user->uid); in bpf_prog_get_info_by_fd()
4890 memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); in bpf_prog_get_info_by_fd()
4892 mutex_lock(&prog->aux->used_maps_mutex); in bpf_prog_get_info_by_fd()
4894 info.nr_map_ids = prog->aux->used_map_cnt; in bpf_prog_get_info_by_fd()
4901 if (put_user(prog->aux->used_maps[i]->id, in bpf_prog_get_info_by_fd()
4903 mutex_unlock(&prog->aux->used_maps_mutex); in bpf_prog_get_info_by_fd()
4907 mutex_unlock(&prog->aux->used_maps_mutex); in bpf_prog_get_info_by_fd()
4918 info.verified_insns = prog->aux->verified_insns; in bpf_prog_get_info_by_fd()
4919 if (prog->aux->btf) in bpf_prog_get_info_by_fd()
4920 info.btf_id = btf_obj_id(prog->aux->btf); in bpf_prog_get_info_by_fd()
4954 if (bpf_prog_is_offloaded(prog->aux)) { in bpf_prog_get_info_by_fd()
4966 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
4970 for (i = 0; i < prog->aux->func_cnt; i++) in bpf_prog_get_info_by_fd()
4971 info.jited_prog_len += prog->aux->func[i]->jited_len; in bpf_prog_get_info_by_fd()
4984 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
4989 for (i = 0; i < prog->aux->func_cnt; i++) { in bpf_prog_get_info_by_fd()
4990 len = prog->aux->func[i]->jited_len; in bpf_prog_get_info_by_fd()
4992 img = (u8 *) prog->aux->func[i]->bpf_func; in bpf_prog_get_info_by_fd()
5010 info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; in bpf_prog_get_info_by_fd()
5022 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
5025 prog->aux->func[i]->bpf_func; in bpf_prog_get_info_by_fd()
5041 info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; in bpf_prog_get_info_by_fd()
5050 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
5053 prog->aux->func[i]->jited_len; in bpf_prog_get_info_by_fd()
5067 info.attach_btf_id = prog->aux->attach_btf_id; in bpf_prog_get_info_by_fd()
5072 info.nr_func_info = prog->aux->func_info_cnt; in bpf_prog_get_info_by_fd()
5078 if (copy_to_user(user_finfo, prog->aux->func_info, in bpf_prog_get_info_by_fd()
5084 info.nr_line_info = prog->aux->nr_linfo; in bpf_prog_get_info_by_fd()
5090 if (copy_to_user(user_linfo, prog->aux->linfo, in bpf_prog_get_info_by_fd()
5096 if (prog->aux->jited_linfo) in bpf_prog_get_info_by_fd()
5097 info.nr_jited_line_info = prog->aux->nr_linfo; in bpf_prog_get_info_by_fd()
5109 line_addr = (unsigned long)prog->aux->jited_linfo[i]; in bpf_prog_get_info_by_fd()
5119 info.nr_prog_tags = prog->aux->func_cnt ? : 1; in bpf_prog_get_info_by_fd()
5126 if (prog->aux->func_cnt) { in bpf_prog_get_info_by_fd()
5129 prog->aux->func[i]->tag, in bpf_prog_get_info_by_fd()
5233 info.prog_id = link->prog->aux->id; in bpf_link_get_info_by_fd()
5435 raw_tp->link.prog->aux->id, in bpf_task_fd_query()
5912 mutex_lock(&prog->aux->used_maps_mutex); in bpf_prog_bind_map()
5914 used_maps_old = prog->aux->used_maps; in bpf_prog_bind_map()
5916 for (i = 0; i < prog->aux->used_map_cnt; i++) in bpf_prog_bind_map()
5922 used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1, in bpf_prog_bind_map()
5936 sizeof(used_maps_old[0]) * prog->aux->used_map_cnt); in bpf_prog_bind_map()
5937 used_maps_new[prog->aux->used_map_cnt] = map; in bpf_prog_bind_map()
5939 prog->aux->used_map_cnt++; in bpf_prog_bind_map()
5940 prog->aux->used_maps = used_maps_new; in bpf_prog_bind_map()
5945 mutex_unlock(&prog->aux->used_maps_mutex); in bpf_prog_bind_map()
6198 if (attr->test.ctx_size_in < prog->aux->max_ctx_offset || in kern_sys_bpf()
6287 return !bpf_token_capable(prog->aux->token, CAP_PERFMON) in syscall_prog_func_proto()