Lines Matching refs:subprog
421 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog) in subprog_is_global() argument
425 return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; in subprog_is_global()
428 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) in subprog_name() argument
435 info = &env->prog->aux->func_info[subprog]; in subprog_name()
439 static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) in mark_subprog_exc_cb() argument
441 struct bpf_subprog_info *info = subprog_info(env, subprog); in mark_subprog_exc_cb()
448 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog) in subprog_is_exc_cb() argument
450 return subprog_info(env, subprog)->is_exception_cb; in subprog_is_exc_cb()
2470 int subprog, bool is_sleepable) in push_async_cb() argument
2504 subprog /* subprog number within this prog */); in push_async_cb()
2969 struct bpf_subprog_info *subprog = env->subprog_info; in add_subprog_and_kfunc() local
3021 subprog[env->subprog_cnt].start = insn_cnt; in add_subprog_and_kfunc()
3025 verbose(env, "func#%d @%d\n", i, subprog[i].start); in add_subprog_and_kfunc()
3033 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs() local
3038 subprog_start = subprog[cur_subprog].start; in check_subprogs()
3039 subprog_end = subprog[cur_subprog + 1].start; in check_subprogs()
3046 subprog[cur_subprog].has_tail_call = true; in check_subprogs()
3047 subprog[cur_subprog].tail_call_reachable = true; in check_subprogs()
3051 subprog[cur_subprog].has_ld_abs = true; in check_subprogs()
3079 subprog_end = subprog[cur_subprog + 1].start; in check_subprogs()
3885 int subprog_insn_idx, subprog; in backtrack_insn() local
3888 subprog = find_subprog(env, subprog_insn_idx); in backtrack_insn()
3889 if (subprog < 0) in backtrack_insn()
3892 if (subprog_is_global(env, subprog)) { in backtrack_insn()
4589 struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; in check_fastcall_stack_contract() local
4593 if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern) in check_fastcall_stack_contract()
4600 subprog->fastcall_stack_off = S16_MIN; in check_fastcall_stack_contract()
4604 for (i = subprog->start; i < (subprog + 1)->start; ++i) { in check_fastcall_stack_contract()
6010 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth_subprog() local
6018 i = subprog[idx].start; in check_max_stack_depth_subprog()
6039 if (idx && subprog[idx].has_tail_call && depth >= 256) { in check_max_stack_depth_subprog()
6045 depth += round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6052 subprog_end = subprog[idx + 1].start; in check_max_stack_depth_subprog()
6061 if (subprog[idx].is_cb) in check_max_stack_depth_subprog()
6064 if (subprog[ret_prog[c]].is_cb) { in check_max_stack_depth_subprog()
6091 if (subprog[sidx].is_async_cb) { in check_max_stack_depth_subprog()
6092 if (subprog[sidx].has_tail_call) { in check_max_stack_depth_subprog()
6099 if (subprog[sidx].is_exception_cb) { in check_max_stack_depth_subprog()
6107 if (subprog[idx].has_tail_call) in check_max_stack_depth_subprog()
6125 if (subprog[ret_prog[j]].is_exception_cb) { in check_max_stack_depth_subprog()
6129 subprog[ret_prog[j]].tail_call_reachable = true; in check_max_stack_depth_subprog()
6131 if (subprog[0].tail_call_reachable) in check_max_stack_depth_subprog()
6139 depth -= round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6166 int start = idx + insn->imm + 1, subprog; in get_callee_stack_depth() local
6168 subprog = find_subprog(env, start); in get_callee_stack_depth()
6169 if (subprog < 0) { in get_callee_stack_depth()
6174 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
9518 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite, in setup_func_entry() argument
9551 subprog /* subprog number within this prog */); in setup_func_entry()
9569 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, in btf_check_func_arg_match() argument
9573 struct bpf_subprog_info *sub = subprog_info(env, subprog); in btf_check_func_arg_match()
9578 ret = btf_prepare_func_args(env, subprog); in btf_check_func_arg_match()
9665 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, in btf_check_subprog_call() argument
9676 btf_id = prog->aux->func_info[subprog].type_id; in btf_check_subprog_call()
9680 if (prog->aux->func_info_aux[subprog].unreliable) in btf_check_subprog_call()
9683 err = btf_check_func_arg_match(env, subprog, btf, regs); in btf_check_subprog_call()
9689 prog->aux->func_info_aux[subprog].unreliable = true; in btf_check_subprog_call()
9694 int insn_idx, int subprog, in push_callback_call() argument
9702 err = btf_check_subprog_call(env, subprog, caller->regs); in push_callback_call()
9710 env->subprog_info[subprog].is_cb = true; in push_callback_call()
9727 env->subprog_info[subprog].is_async_cb = true; in push_callback_call()
9728 async_cb = push_async_cb(env, env->subprog_info[subprog].start, in push_callback_call()
9729 insn_idx, subprog, in push_callback_call()
9747 callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false); in push_callback_call()
9751 err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb, in push_callback_call()
9767 int err, subprog, target_insn; in check_func_call() local
9770 subprog = find_subprog(env, target_insn); in check_func_call()
9771 if (subprog < 0) { in check_func_call()
9777 err = btf_check_subprog_call(env, subprog, caller->regs); in check_func_call()
9780 if (subprog_is_global(env, subprog)) { in check_func_call()
9781 const char *sub_name = subprog_name(env, subprog); in check_func_call()
9799 subprog, sub_name); in check_func_call()
9804 subprog, sub_name); in check_func_call()
9806 subprog_aux(env, subprog)->called = true; in check_func_call()
9820 err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); in check_func_call()
9827 *insn_idx = env->subprog_info[subprog].start - 1; in check_func_call()
16259 struct bpf_subprog_info *subprog, in mark_fastcall_pattern_for_call() argument
16357 subprog->keep_fastcall_stack = 1; in mark_fastcall_pattern_for_call()
16358 subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off); in mark_fastcall_pattern_for_call()
16363 struct bpf_subprog_info *subprog = env->subprog_info; in mark_fastcall_patterns() local
16368 for (s = 0; s < env->subprog_cnt; ++s, ++subprog) { in mark_fastcall_patterns()
16371 for (i = subprog->start; i < (subprog + 1)->start; ++i) { in mark_fastcall_patterns()
16379 for (i = subprog->start; i < (subprog + 1)->start; ++i) { in mark_fastcall_patterns()
16383 mark_fastcall_pattern_for_call(env, subprog, i, lowest_off); in mark_fastcall_patterns()
19947 int i, j, subprog_start, subprog_end = 0, len, subprog; in jit_subprogs() local
19964 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
19965 if (subprog < 0) { in jit_subprogs()
19973 insn->off = subprog; in jit_subprogs()
20090 subprog = insn->off; in jit_subprogs()
20091 insn[0].imm = (u32)(long)func[subprog]->bpf_func; in jit_subprogs()
20092 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; in jit_subprogs()
20097 subprog = insn->off; in jit_subprogs()
20098 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); in jit_subprogs()
20154 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
20155 insn->imm = subprog; in jit_subprogs()
21445 struct bpf_subprog_info *subprog = env->subprog_info; in remove_fastcall_spills_fills() local
21463 if ((subprog + 1)->start == i + 1) { in remove_fastcall_spills_fills()
21464 if (modified && !subprog->keep_fastcall_stack) in remove_fastcall_spills_fills()
21465 subprog->stack_depth = -subprog->fastcall_stack_off; in remove_fastcall_spills_fills()
21466 subprog++; in remove_fastcall_spills_fills()
21504 static int do_check_common(struct bpf_verifier_env *env, int subprog) in do_check_common() argument
21507 struct bpf_subprog_info *sub = subprog_info(env, subprog); in do_check_common()
21530 subprog); in do_check_common()
21531 state->first_insn_idx = env->subprog_info[subprog].start; in do_check_common()
21535 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { in do_check_common()
21536 const char *sub_name = subprog_name(env, subprog); in do_check_common()
21540 verbose(env, "Validating %s() func#%d...\n", sub_name, subprog); in do_check_common()
21541 ret = btf_prepare_func_args(env, subprog); in do_check_common()
21545 if (subprog_is_exc_cb(env, subprog)) { in do_check_common()
21867 int ret = 0, subprog = -1, i; in bpf_check_attach_target() local
21906 subprog = i; in bpf_check_attach_target()
21909 if (subprog == -1) { in bpf_check_attach_target()
21913 if (aux->func && aux->func[subprog]->aux->exception_cb) { in bpf_check_attach_target()
21919 conservative = aux->func_info_aux[subprog].unreliable; in bpf_check_attach_target()
22082 if (subprog == 0) in bpf_check_attach_target()
22085 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; in bpf_check_attach_target()