Lines Matching refs:aux
216 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) in bpf_map_ptr_poisoned() argument
218 return aux->map_ptr_state.poison; in bpf_map_ptr_poisoned()
221 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) in bpf_map_ptr_unpriv() argument
223 return aux->map_ptr_state.unpriv; in bpf_map_ptr_unpriv()
226 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, in bpf_map_ptr_store() argument
230 unpriv |= bpf_map_ptr_unpriv(aux); in bpf_map_ptr_store()
231 aux->map_ptr_state.unpriv = unpriv; in bpf_map_ptr_store()
232 aux->map_ptr_state.poison = poison; in bpf_map_ptr_store()
233 aux->map_ptr_state.map_ptr = map; in bpf_map_ptr_store()
236 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) in bpf_map_key_poisoned() argument
238 return aux->map_key_state & BPF_MAP_KEY_POISON; in bpf_map_key_poisoned()
241 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) in bpf_map_key_unseen() argument
243 return !(aux->map_key_state & BPF_MAP_KEY_SEEN); in bpf_map_key_unseen()
246 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) in bpf_map_key_immediate() argument
248 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); in bpf_map_key_immediate()
251 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) in bpf_map_key_store() argument
253 bool poisoned = bpf_map_key_poisoned(aux); in bpf_map_key_store()
255 aux->map_key_state = state | BPF_MAP_KEY_SEEN | in bpf_map_key_store()
435 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux; in subprog_is_global() local
437 return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; in subprog_is_global()
444 if (!env->prog->aux->func_info) in subprog_name()
447 info = &env->prog->aux->func_info[subprog]; in subprog_name()
448 return btf_type_name(env->prog->aux->btf, info->type_id); in subprog_name()
3017 struct bpf_prog_aux *aux = env->prog->aux; in bpf_find_exception_callback_insn_off() local
3018 struct btf *btf = aux->btf; in bpf_find_exception_callback_insn_off()
3025 if (!aux->func_info_cnt) in bpf_find_exception_callback_insn_off()
3027 main_btf_id = aux->func_info[0].type_id; in bpf_find_exception_callback_insn_off()
3058 for (i = 0; i < aux->func_info_cnt; i++) { in bpf_find_exception_callback_insn_off()
3059 if (aux->func_info[i].type_id != id) in bpf_find_exception_callback_insn_off()
3061 ret = aux->func_info[i].insn_off; in bpf_find_exception_callback_insn_off()
3135 tab = prog->aux->kfunc_tab; in find_kfunc_desc()
3163 tab = env->prog->aux->kfunc_btf_tab; in __find_kfunc_desc_btf()
3258 prog_aux = env->prog->aux; in add_kfunc_call()
3388 tab = prog->aux->kfunc_tab; in sort_kfunc_descs_by_imm_off()
3398 return !!prog->aux->kfunc_tab; in bpf_prog_has_kfunc_call()
3412 tab = prog->aux->kfunc_tab; in bpf_jit_find_kfunc_model()
5086 struct bpf_insn_aux_data *aux = env->insn_aux_data; in check_fastcall_stack_contract() local
5089 if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern) in check_fastcall_stack_contract()
5101 aux[i].fastcall_spills_num = 0; in check_fastcall_stack_contract()
5102 aux[i].fastcall_pattern = 0; in check_fastcall_stack_contract()
6221 env->prog->aux->max_pkt_offset = in check_packet_access()
6222 max_t(u32, env->prog->aux->max_pkt_offset, in check_packet_access()
6252 if (env->prog->aux->max_ctx_offset < off + size) in check_ctx_access()
6253 env->prog->aux->max_ctx_offset = off + size; in check_ctx_access()
6554 if (prog->aux->priv_stack_requested || bpf_prog_check_recur(prog)) in bpf_enable_priv_stack()
6727 env->prog->aux->tail_call_reachable = true; in check_max_stack_depth_subprog()
6778 env->prog->aux->jits_use_priv_stack = true; in check_max_stack_depth()
6833 if (off + size > env->prog->aux->max_tp_access) in check_tp_buffer_access()
6834 env->prog->aux->max_tp_access = off + size; in check_tp_buffer_access()
7760 max_access = &env->prog->aux->max_rdonly_access; in check_mem_access()
7762 max_access = &env->prog->aux->max_rdwr_access; in check_mem_access()
8217 max_access = &env->prog->aux->max_rdonly_access; in check_helper_mem_access()
8219 max_access = &env->prog->aux->max_rdwr_access; in check_helper_mem_access()
10558 struct btf *btf = prog->aux->btf; in btf_check_subprog_call()
10562 if (!prog->aux->func_info) in btf_check_subprog_call()
10565 btf_id = prog->aux->func_info[subprog].type_id; in btf_check_subprog_call()
10569 if (prog->aux->func_info_aux[subprog].unreliable) in btf_check_subprog_call()
10578 prog->aux->func_info_aux[subprog].unreliable = true; in btf_check_subprog_call()
11099 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_map() local
11132 if (!aux->map_ptr_state.map_ptr) in record_func_map()
11133 bpf_map_ptr_store(aux, meta->map_ptr, in record_func_map()
11135 else if (aux->map_ptr_state.map_ptr != meta->map_ptr) in record_func_map()
11136 bpf_map_ptr_store(aux, meta->map_ptr, in record_func_map()
11145 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_key() local
11163 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); in record_func_key()
11170 if (bpf_map_key_unseen(aux)) in record_func_key()
11171 bpf_map_key_store(aux, val); in record_func_key()
11172 else if (!bpf_map_key_poisoned(aux) && in record_func_key()
11173 bpf_map_key_immediate(aux) != val) in record_func_key()
11174 bpf_map_key_store(aux, BPF_MAP_KEY_POISON); in record_func_key()
11586 if (!env->prog->aux->attach_func_proto->type) { in check_helper_call()
13593 ret_btf = env->prog->aux->btf; in check_special_kfunc()
14173 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, in update_alu_sanitation_state() argument
14179 if (aux->alu_state && in update_alu_sanitation_state()
14180 (aux->alu_state != alu_state || in update_alu_sanitation_state()
14181 aux->alu_limit != alu_limit)) in update_alu_sanitation_state()
14185 aux->alu_state = alu_state; in update_alu_sanitation_state()
14186 aux->alu_limit = alu_limit; in update_alu_sanitation_state()
14193 struct bpf_insn_aux_data *aux = cur_aux(env); in sanitize_val_alu() local
14198 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); in sanitize_val_alu()
14207 struct bpf_insn_aux_data aux; member
14240 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; in sanitize_ptr_alu() local
14278 alu_state = info->aux.alu_state; in sanitize_ptr_alu()
14279 alu_limit = abs(info->aux.alu_limit - alu_limit); in sanitize_ptr_alu()
14293 err = update_alu_sanitation_state(aux, alu_state, alu_limit); in sanitize_ptr_alu()
15420 struct bpf_insn_aux_data *aux = cur_aux(env); in adjust_reg_min_max_vals() local
15427 aux->needs_zext = true; in adjust_reg_min_max_vals()
15601 if (!env->prog->aux->arena) { in check_alu_op()
16866 struct bpf_insn_aux_data *aux = cur_aux(env); in check_ld_imm() local
16901 dst_reg->type = aux->btf_var.reg_type; in check_ld_imm()
16904 dst_reg->mem_size = aux->btf_var.mem_size; in check_ld_imm()
16907 dst_reg->btf = aux->btf_var.btf; in check_ld_imm()
16908 dst_reg->btf_id = aux->btf_var.btf_id; in check_ld_imm()
16918 struct bpf_prog_aux *aux = env->prog->aux; in check_ld_imm() local
16922 if (!aux->func_info) { in check_ld_imm()
16926 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) { in check_ld_imm()
16936 map = env->used_maps[aux->map_index]; in check_ld_imm()
16946 dst_reg->off = aux->map_off; in check_ld_imm()
17078 if (!prog->aux->attach_func_proto->type) in check_return_code()
17082 if (!prog->aux->attach_func_proto->type) in check_return_code()
17093 ret_type = btf_type_resolve_ptr(prog->aux->attach_btf, in check_return_code()
17094 prog->aux->attach_func_proto->type, in check_return_code()
17164 if (!env->prog->aux->attach_btf_id) in check_return_code()
17206 } else if (!env->prog->aux->attach_func_proto->type) { in check_return_code()
17246 !prog->aux->attach_func_proto->type) in check_return_code()
17910 env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data; in check_cfg()
17911 env->prog->aux->might_sleep = env->subprog_info[0].might_sleep; in check_cfg()
17971 btf = prog->aux->btf; in check_btf_func_early()
18034 prog->aux->func_info = krecord; in check_btf_func_early()
18035 prog->aux->func_info_cnt = nfuncs; in check_btf_func_early()
18071 btf = prog->aux->btf; in check_btf_func()
18075 krecord = prog->aux->func_info; in check_btf_func()
18110 prog->aux->func_info_aux = info_aux; in check_btf_func()
18120 struct bpf_prog_aux *aux = env->prog->aux; in adjust_btf_func() local
18123 if (!aux->func_info) in adjust_btf_func()
18128 aux->func_info[i].insn_off = env->subprog_info[i].start; in adjust_btf_func()
18167 btf = prog->aux->btf; in check_btf_line()
18249 prog->aux->linfo = linfo; in check_btf_line()
18250 prog->aux->nr_linfo = nr_linfo; in check_btf_line()
18269 const struct btf *btf = prog->aux->btf; in check_core_relo()
18351 env->prog->aux->btf = btf; in check_btf_info_early()
20077 if (bpf_prog_is_offloaded(env->prog->aux)) { in do_check()
20220 struct bpf_insn_aux_data *aux, in __check_pseudo_btf_id() argument
20255 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; in __check_pseudo_btf_id()
20256 aux->btf_var.mem_size = 0; in __check_pseudo_btf_id()
20274 aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU; in __check_pseudo_btf_id()
20275 aux->btf_var.btf = btf; in __check_pseudo_btf_id()
20276 aux->btf_var.btf_id = type; in __check_pseudo_btf_id()
20290 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY; in __check_pseudo_btf_id()
20291 aux->btf_var.mem_size = tsize; in __check_pseudo_btf_id()
20293 aux->btf_var.reg_type = PTR_TO_BTF_ID; in __check_pseudo_btf_id()
20294 aux->btf_var.btf = btf; in __check_pseudo_btf_id()
20295 aux->btf_var.btf_id = type; in __check_pseudo_btf_id()
20303 struct bpf_insn_aux_data *aux) in check_pseudo_btf_id() argument
20326 err = __check_pseudo_btf_id(env, insn, aux, btf); in check_pseudo_btf_id()
20397 if ((bpf_prog_is_offloaded(prog->aux) || bpf_map_is_offloaded(map)) && in check_map_prog_compatibility()
20435 bpf_cgroup_storage_assign(env->prog->aux, map)) { in check_map_prog_compatibility()
20441 if (env->prog->aux->arena) { in check_map_prog_compatibility()
20457 env->prog->aux->arena = (void *)map; in check_map_prog_compatibility()
20458 if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { in check_map_prog_compatibility()
20545 struct bpf_insn_aux_data *aux; in resolve_pseudo_ldimm64() local
20563 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
20564 err = check_pseudo_btf_id(env, insn, aux); in resolve_pseudo_ldimm64()
20571 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
20572 aux->ptr_type = PTR_TO_FUNC; in resolve_pseudo_ldimm64()
20615 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
20616 aux->map_index = map_idx; in resolve_pseudo_ldimm64()
20641 aux->map_off = off; in resolve_pseudo_ldimm64()
20671 __bpf_free_used_maps(env->prog->aux, env->used_maps, in release_maps()
20749 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; in adjust_poke_descs()
20750 int i, sz = prog->aux->size_poke_tab; in adjust_poke_descs()
20847 struct bpf_prog_aux *aux = env->prog->aux; in adjust_subprog_starts_after_remove() local
20859 if (aux->func_info) { in adjust_subprog_starts_after_remove()
20860 move = aux->func_info_cnt - j; in adjust_subprog_starts_after_remove()
20862 memmove(aux->func_info + i, in adjust_subprog_starts_after_remove()
20863 aux->func_info + j, in adjust_subprog_starts_after_remove()
20864 sizeof(*aux->func_info) * move); in adjust_subprog_starts_after_remove()
20865 aux->func_info_cnt -= j - i; in adjust_subprog_starts_after_remove()
20890 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo_after_remove()
20894 linfo = prog->aux->linfo; in bpf_adj_linfo_after_remove()
20924 prog->aux->nr_linfo -= l_cnt; in bpf_adj_linfo_after_remove()
20925 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo_after_remove()
20953 if (bpf_prog_is_offloaded(env->prog->aux)) in verifier_remove_insns()
21034 if (bpf_prog_is_offloaded(env->prog->aux)) in opt_hard_wire_dead_code_branches()
21100 struct bpf_insn_aux_data *aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32() local
21118 if (!aux[adj_idx].zext_dst) { in opt_subreg_zext_lo32_rnd_hi32()
21143 aux[adj_idx].ptr_type == PTR_TO_CTX) in opt_subreg_zext_lo32_rnd_hi32()
21187 aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
21266 if (bpf_prog_is_offloaded(env->prog->aux)) in convert_ctx_accesses()
21320 env->prog->aux->num_exentries++; in convert_ctx_accesses()
21398 env->prog->aux->num_exentries++; in convert_ctx_accesses()
21407 env->prog->aux->num_exentries++; in convert_ctx_accesses()
21574 func[i]->aux->func_idx = i; in jit_subprogs()
21576 func[i]->aux->btf = prog->aux->btf; in jit_subprogs()
21577 func[i]->aux->func_info = prog->aux->func_info; in jit_subprogs()
21578 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt; in jit_subprogs()
21579 func[i]->aux->poke_tab = prog->aux->poke_tab; in jit_subprogs()
21580 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab; in jit_subprogs()
21582 for (j = 0; j < prog->aux->size_poke_tab; j++) { in jit_subprogs()
21585 poke = &prog->aux->poke_tab[j]; in jit_subprogs()
21588 poke->aux = func[i]->aux; in jit_subprogs()
21591 func[i]->aux->name[0] = 'F'; in jit_subprogs()
21592 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; in jit_subprogs()
21594 func[i]->aux->jits_use_priv_stack = true; in jit_subprogs()
21598 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab; in jit_subprogs()
21599 func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab; in jit_subprogs()
21600 func[i]->aux->linfo = prog->aux->linfo; in jit_subprogs()
21601 func[i]->aux->nr_linfo = prog->aux->nr_linfo; in jit_subprogs()
21602 func[i]->aux->jited_linfo = prog->aux->jited_linfo; in jit_subprogs()
21603 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; in jit_subprogs()
21604 func[i]->aux->arena = prog->aux->arena; in jit_subprogs()
21621 func[i]->aux->num_exentries = num_exentries; in jit_subprogs()
21622 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; in jit_subprogs()
21623 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; in jit_subprogs()
21624 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; in jit_subprogs()
21625 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; in jit_subprogs()
21627 func[i]->aux->exception_boundary = env->seen_exception; in jit_subprogs()
21666 func[i]->aux->func = func; in jit_subprogs()
21667 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; in jit_subprogs()
21668 func[i]->aux->real_func_cnt = env->subprog_cnt; in jit_subprogs()
21715 prog->aux->extable = func[0]->aux->extable; in jit_subprogs()
21716 prog->aux->num_exentries = func[0]->aux->num_exentries; in jit_subprogs()
21717 prog->aux->func = func; in jit_subprogs()
21718 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; in jit_subprogs()
21719 prog->aux->real_func_cnt = env->subprog_cnt; in jit_subprogs()
21720 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; in jit_subprogs()
21721 prog->aux->exception_boundary = func[0]->aux->exception_boundary; in jit_subprogs()
21729 for (i = 0; i < prog->aux->size_poke_tab; i++) { in jit_subprogs()
21730 map_ptr = prog->aux->poke_tab[i].tail_call.map; in jit_subprogs()
21731 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); in jit_subprogs()
21740 func[i]->aux->poke_tab = NULL; in jit_subprogs()
21769 !bpf_prog_is_offloaded(env->prog->aux)) { in fixup_call_args()
21781 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { in fixup_call_args()
21965 struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) }; in fixup_kfunc_call()
22015 struct bpf_insn_aux_data *aux; in do_misc_fixups() local
22044 (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { in do_misc_fixups()
22237 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
22238 if (!aux->alu_state || in do_misc_fixups()
22239 aux->alu_state == BPF_ALU_NON_POINTER) in do_misc_fixups()
22242 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; in do_misc_fixups()
22243 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == in do_misc_fixups()
22245 isimm = aux->alu_state & BPF_ALU_IMMEDIATE; in do_misc_fixups()
22249 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); in do_misc_fixups()
22253 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); in do_misc_fixups()
22386 prog->aux->stack_depth = MAX_BPF_STACK; in do_misc_fixups()
22387 prog->aux->max_pkt_offset = MAX_PACKET_OFF; in do_misc_fixups()
22397 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
22400 !bpf_map_key_poisoned(aux) && in do_misc_fixups()
22401 !bpf_map_ptr_poisoned(aux) && in do_misc_fixups()
22402 !bpf_map_ptr_unpriv(aux)) { in do_misc_fixups()
22405 .tail_call.map = aux->map_ptr_state.map_ptr, in do_misc_fixups()
22406 .tail_call.key = bpf_map_key_immediate(aux), in do_misc_fixups()
22420 if (!bpf_map_ptr_unpriv(aux)) in do_misc_fixups()
22429 if (bpf_map_ptr_poisoned(aux)) { in do_misc_fixups()
22434 map_ptr = aux->map_ptr_state.map_ptr; in do_misc_fixups()
22468 BPF_LD_IMM64(BPF_REG_3, (long)prog->aux), in do_misc_fixups()
22538 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
22539 if (bpf_map_ptr_poisoned(aux)) in do_misc_fixups()
22542 map_ptr = aux->map_ptr_state.map_ptr; in do_misc_fixups()
22863 env->prog->aux->stack_depth = subprogs[0].stack_depth; in do_misc_fixups()
22905 for (i = 0; i < prog->aux->size_poke_tab; i++) { in do_misc_fixups()
22906 map_ptr = prog->aux->poke_tab[i].tail_call.map; in do_misc_fixups()
22914 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); in do_misc_fixups()
23058 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in optimize_bpf_loop()
23069 struct bpf_insn_aux_data *aux = env->insn_aux_data; in remove_fastcall_spills_fills() local
23077 if (aux[i].fastcall_spills_num > 0) { in remove_fastcall_spills_fills()
23078 spills_num = aux[i].fastcall_spills_num; in remove_fastcall_spills_fills()
23144 struct bpf_prog_aux *aux = env->prog->aux; in do_check_common() local
23242 if (env->prog->aux->func_info_aux) { in do_check_common()
23245 env->prog->aux->func_info_aux[0].unreliable = true; in do_check_common()
23255 for (i = 0; i < aux->ctx_arg_info_size; i++) in do_check_common()
23256 aux->ctx_arg_info[i].ref_obj_id = aux->ctx_arg_info[i].refcounted ? in do_check_common()
23290 struct bpf_prog_aux *aux = env->prog->aux; in do_check_subprogs() local
23294 if (!aux->func_info) in do_check_subprogs()
23345 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in do_check_main()
23377 prog->aux->ctx_arg_info = kmemdup_array(info, cnt, sizeof(*info), GFP_KERNEL_ACCOUNT); in bpf_prog_ctx_arg_info_init()
23378 prog->aux->ctx_arg_info_size = cnt; in bpf_prog_ctx_arg_info_init()
23380 return prog->aux->ctx_arg_info ? 0 : -ENOMEM; in bpf_prog_ctx_arg_info_init()
23401 if (!prog->aux->attach_btf_id) in check_struct_ops_btf_id()
23404 btf = prog->aux->attach_btf; in check_struct_ops_btf_id()
23415 btf_id = prog->aux->attach_btf_id; in check_struct_ops_btf_id()
23460 if (prog->aux->priv_stack_requested && !bpf_jit_supports_private_stack()) { in check_struct_ops_btf_id()
23482 prog->aux->st_ops = st_ops; in check_struct_ops_btf_id()
23483 prog->aux->attach_st_ops_member_off = member_off; in check_struct_ops_btf_id()
23485 prog->aux->attach_func_proto = func_proto; in check_struct_ops_btf_id()
23486 prog->aux->attach_func_name = mname; in check_struct_ops_btf_id()
23547 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; in bpf_check_attach_target()
23564 struct bpf_prog_aux *aux = tgt_prog->aux; in bpf_check_attach_target() local
23568 if (bpf_prog_is_dev_bound(prog->aux) && in bpf_check_attach_target()
23574 for (i = 0; i < aux->func_info_cnt; i++) in bpf_check_attach_target()
23575 if (aux->func_info[i].type_id == btf_id) { in bpf_check_attach_target()
23583 if (aux->func && aux->func[subprog]->aux->exception_cb) { in bpf_check_attach_target()
23589 conservative = aux->func_info_aux[subprog].unreliable; in bpf_check_attach_target()
23601 tgt_changes_pkt_data = aux->func in bpf_check_attach_target()
23602 ? aux->func[subprog]->aux->changes_pkt_data in bpf_check_attach_target()
23603 : aux->changes_pkt_data; in bpf_check_attach_target()
23604 if (prog->aux->changes_pkt_data && !tgt_changes_pkt_data) { in bpf_check_attach_target()
23610 tgt_might_sleep = aux->func in bpf_check_attach_target()
23611 ? aux->func[subprog]->aux->might_sleep in bpf_check_attach_target()
23612 : aux->might_sleep; in bpf_check_attach_target()
23613 if (prog->aux->might_sleep && !tgt_might_sleep) { in bpf_check_attach_target()
23624 if (aux->attach_tracing_prog) { in bpf_check_attach_target()
23756 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && in bpf_check_attach_target()
23757 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || in bpf_check_attach_target()
23758 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) in bpf_check_attach_target()
23772 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; in bpf_check_attach_target()
23922 struct bpf_prog *tgt_prog = prog->aux->dst_prog; in check_attach_btf_id()
23924 u32 btf_id = prog->aux->attach_btf_id; in check_attach_btf_id()
23964 prog->aux->attach_func_proto = tgt_info.tgt_type; in check_attach_btf_id()
23965 prog->aux->attach_func_name = tgt_info.tgt_name; in check_attach_btf_id()
23966 prog->aux->mod = tgt_info.tgt_mod; in check_attach_btf_id()
23969 prog->aux->saved_dst_prog_type = tgt_prog->type; in check_attach_btf_id()
23970 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; in check_attach_btf_id()
23974 prog->aux->attach_btf_trace = true; in check_attach_btf_id()
23997 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); in check_attach_btf_id()
24002 if (tgt_prog && tgt_prog->aux->tail_call_reachable) in check_attach_btf_id()
24005 prog->aux->dst_trampoline = tr; in check_attach_btf_id()
24399 struct bpf_insn_aux_data *aux = env->insn_aux_data; in compute_scc() local
24547 aux[t].scc = next_scc_id; in compute_scc()
24602 env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); in bpf_check()
24603 env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); in bpf_check()
24604 env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); in bpf_check()
24605 env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); in bpf_check()
24606 env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); in bpf_check()
24677 if (bpf_prog_is_offloaded(env->prog->aux)) { in bpf_check()
24706 if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) in bpf_check()
24747 if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) { in bpf_check()
24749 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret in bpf_check()
24758 env->prog->aux->verified_insns = env->insn_processed; in bpf_check()
24777 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, in bpf_check()
24781 if (!env->prog->aux->used_maps) { in bpf_check()
24786 memcpy(env->prog->aux->used_maps, env->used_maps, in bpf_check()
24788 env->prog->aux->used_map_cnt = env->used_map_cnt; in bpf_check()
24792 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, in bpf_check()
24795 if (!env->prog->aux->used_btfs) { in bpf_check()
24800 memcpy(env->prog->aux->used_btfs, env->used_btfs, in bpf_check()
24802 env->prog->aux->used_btf_cnt = env->used_btf_cnt; in bpf_check()
24814 if (!env->prog->aux->used_maps) in bpf_check()
24819 if (!env->prog->aux->used_btfs) in bpf_check()