Lines Matching refs:prog
122 fp->aux->prog = fp; in bpf_prog_alloc_no_stats()
147 struct bpf_prog *prog; in bpf_prog_alloc() local
150 prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags); in bpf_prog_alloc()
151 if (!prog) in bpf_prog_alloc()
154 prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); in bpf_prog_alloc()
155 if (!prog->stats) { in bpf_prog_alloc()
156 free_percpu(prog->active); in bpf_prog_alloc()
157 kfree(prog->aux); in bpf_prog_alloc()
158 vfree(prog); in bpf_prog_alloc()
165 pstats = per_cpu_ptr(prog->stats, cpu); in bpf_prog_alloc()
168 return prog; in bpf_prog_alloc()
172 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) in bpf_prog_alloc_jited_linfo() argument
174 if (!prog->aux->nr_linfo || !prog->jit_requested) in bpf_prog_alloc_jited_linfo()
177 prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo, in bpf_prog_alloc_jited_linfo()
178 sizeof(*prog->aux->jited_linfo), in bpf_prog_alloc_jited_linfo()
180 if (!prog->aux->jited_linfo) in bpf_prog_alloc_jited_linfo()
186 void bpf_prog_jit_attempt_done(struct bpf_prog *prog) in bpf_prog_jit_attempt_done() argument
188 if (prog->aux->jited_linfo && in bpf_prog_jit_attempt_done()
189 (!prog->jited || !prog->aux->jited_linfo[0])) { in bpf_prog_jit_attempt_done()
190 kvfree(prog->aux->jited_linfo); in bpf_prog_jit_attempt_done()
191 prog->aux->jited_linfo = NULL; in bpf_prog_jit_attempt_done()
194 kfree(prog->aux->kfunc_tab); in bpf_prog_jit_attempt_done()
195 prog->aux->kfunc_tab = NULL; in bpf_prog_jit_attempt_done()
222 void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, in bpf_prog_fill_jited_linfo() argument
229 if (!prog->aux->jited_linfo || prog->aux->func_idx > prog->aux->func_cnt) in bpf_prog_fill_jited_linfo()
233 linfo_idx = prog->aux->linfo_idx; in bpf_prog_fill_jited_linfo()
234 linfo = &prog->aux->linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
236 insn_end = insn_start + prog->len; in bpf_prog_fill_jited_linfo()
238 jited_linfo = &prog->aux->jited_linfo[linfo_idx]; in bpf_prog_fill_jited_linfo()
239 jited_linfo[0] = prog->bpf_func; in bpf_prog_fill_jited_linfo()
241 nr_linfo = prog->aux->nr_linfo - linfo_idx; in bpf_prog_fill_jited_linfo()
247 jited_linfo[i] = prog->bpf_func + in bpf_prog_fill_jited_linfo()
267 fp->aux->prog = fp; in bpf_prog_realloc()
416 static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old, in bpf_adj_branches() argument
419 u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0); in bpf_adj_branches()
420 struct bpf_insn *insn = prog->insnsi; in bpf_adj_branches()
432 insn = prog->insnsi + end_old; in bpf_adj_branches()
463 static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) in bpf_adj_linfo() argument
468 nr_linfo = prog->aux->nr_linfo; in bpf_adj_linfo()
472 linfo = prog->aux->linfo; in bpf_adj_linfo()
483 struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, in bpf_patch_insn_single() argument
493 memcpy(prog->insnsi + off, patch, sizeof(*patch)); in bpf_patch_insn_single()
494 return prog; in bpf_patch_insn_single()
497 insn_adj_cnt = prog->len + insn_delta; in bpf_patch_insn_single()
505 (err = bpf_adj_branches(prog, off, off + 1, off + len, true))) in bpf_patch_insn_single()
512 prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt), in bpf_patch_insn_single()
544 int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) in bpf_remove_insns() argument
551 memmove(prog->insnsi + off, prog->insnsi + off + cnt, in bpf_remove_insns()
552 sizeof(struct bpf_insn) * (prog->len - off - cnt)); in bpf_remove_insns()
553 prog->len -= cnt; in bpf_remove_insns()
555 err = bpf_adj_branches(prog, off, off + cnt, off, false); in bpf_remove_insns()
583 bpf_prog_ksym_set_addr(struct bpf_prog *prog) in bpf_prog_ksym_set_addr() argument
585 WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog)); in bpf_prog_ksym_set_addr()
587 prog->aux->ksym.start = (unsigned long) prog->bpf_func; in bpf_prog_ksym_set_addr()
588 prog->aux->ksym.end = prog->aux->ksym.start + prog->jited_len; in bpf_prog_ksym_set_addr()
592 bpf_prog_ksym_set_name(struct bpf_prog *prog) in bpf_prog_ksym_set_name() argument
594 char *sym = prog->aux->ksym.name; in bpf_prog_ksym_set_name()
600 sizeof(prog->tag) * 2 + in bpf_prog_ksym_set_name()
608 sizeof(prog->aux->name) > KSYM_NAME_LEN); in bpf_prog_ksym_set_name()
611 sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); in bpf_prog_ksym_set_name()
614 if (prog->aux->func_info_cnt && prog->aux->func_idx < prog->aux->func_info_cnt) { in bpf_prog_ksym_set_name()
615 type = btf_type_by_id(prog->aux->btf, in bpf_prog_ksym_set_name()
616 prog->aux->func_info[prog->aux->func_idx].type_id); in bpf_prog_ksym_set_name()
617 func_name = btf_name_by_offset(prog->aux->btf, type->name_off); in bpf_prog_ksym_set_name()
622 if (prog->aux->name[0]) in bpf_prog_ksym_set_name()
623 snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); in bpf_prog_ksym_set_name()
705 fp->aux->ksym.prog = true; in bpf_prog_kallsyms_add()
790 return ksym && ksym->prog ? in bpf_prog_ksym_find()
791 container_of(ksym, struct bpf_prog_aux, ksym)->prog : in bpf_prog_ksym_find()
798 struct bpf_prog *prog; in search_bpf_extables() local
801 prog = bpf_prog_ksym_find(addr); in search_bpf_extables()
802 if (!prog) in search_bpf_extables()
804 if (!prog->aux->num_exentries) in search_bpf_extables()
807 e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr); in search_bpf_extables()
841 int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, in bpf_jit_add_poke_descriptor() argument
844 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; in bpf_jit_add_poke_descriptor()
846 u32 slot = prog->aux->size_poke_tab; in bpf_jit_add_poke_descriptor()
869 prog->aux->size_poke_tab = size; in bpf_jit_add_poke_descriptor()
870 prog->aux->poke_tab = tab; in bpf_jit_add_poke_descriptor()
1260 int bpf_jit_get_func_addr(const struct bpf_prog *prog, in bpf_jit_get_func_addr() argument
1277 else if (prog->aux->func && in bpf_jit_get_func_addr()
1278 off >= 0 && off < prog->aux->real_func_cnt) in bpf_jit_get_func_addr()
1279 addr = (u8 *)prog->aux->func[off]->bpf_func; in bpf_jit_get_func_addr()
1284 err = bpf_get_kfunc_addr(prog, insn->imm, insn->off, &addr); in bpf_jit_get_func_addr()
1300 const char *bpf_jit_get_prog_name(struct bpf_prog *prog) in bpf_jit_get_prog_name() argument
1302 if (prog->aux->ksym.prog) in bpf_jit_get_prog_name()
1303 return prog->aux->ksym.name; in bpf_jit_get_prog_name()
1304 return prog->aux->name; in bpf_jit_get_prog_name()
1481 fp->aux->prog = fp; in bpf_jit_prog_release_other()
1485 struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) in bpf_jit_blind_constants() argument
1493 if (!prog->blinding_requested || prog->blinded) in bpf_jit_blind_constants()
1494 return prog; in bpf_jit_blind_constants()
1496 clone = bpf_prog_clone_create(prog, GFP_USER); in bpf_jit_blind_constants()
1533 bpf_jit_prog_release_other(prog, clone); in bpf_jit_blind_constants()
2046 struct bpf_prog *prog; in ___bpf_prog_run() local
2057 prog = READ_ONCE(array->ptrs[index]); in ___bpf_prog_run()
2058 if (!prog) in ___bpf_prog_run()
2066 insn = prog->insnsi; in ___bpf_prog_run()
2565 struct bpf_prog prog; member
2567 .prog = {
2622 for (item = array->items; item->prog; item++) in bpf_prog_array_length()
2623 if (item->prog != &dummy_bpf_prog.prog) in bpf_prog_array_length()
2632 for (item = array->items; item->prog; item++) in bpf_prog_array_is_empty()
2633 if (item->prog != &dummy_bpf_prog.prog) in bpf_prog_array_is_empty()
2645 for (item = array->items; item->prog; item++) { in bpf_prog_array_copy_core()
2646 if (item->prog == &dummy_bpf_prog.prog) in bpf_prog_array_copy_core()
2648 prog_ids[i] = item->prog->aux->id; in bpf_prog_array_copy_core()
2655 return !!(item->prog); in bpf_prog_array_copy_core()
2689 for (item = array->items; item->prog; item++) in bpf_prog_array_delete_safe()
2690 if (item->prog == old_prog) { in bpf_prog_array_delete_safe()
2691 WRITE_ONCE(item->prog, &dummy_bpf_prog.prog); in bpf_prog_array_delete_safe()
2713 return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog); in bpf_prog_array_delete_safe_at()
2732 struct bpf_prog *prog) in bpf_prog_array_update_at() argument
2739 for (item = array->items; item->prog; item++) { in bpf_prog_array_update_at()
2740 if (item->prog == &dummy_bpf_prog.prog) in bpf_prog_array_update_at()
2743 WRITE_ONCE(item->prog, prog); in bpf_prog_array_update_at()
2767 for (; existing->prog; existing++) { in bpf_prog_array_copy()
2768 if (existing->prog == exclude_prog) { in bpf_prog_array_copy()
2772 if (existing->prog != &dummy_bpf_prog.prog) in bpf_prog_array_copy()
2774 if (existing->prog == include_prog) in bpf_prog_array_copy()
2802 for (; existing->prog; existing++) { in bpf_prog_array_copy()
2803 if (existing->prog == exclude_prog || in bpf_prog_array_copy()
2804 existing->prog == &dummy_bpf_prog.prog) in bpf_prog_array_copy()
2807 new->prog = existing->prog; in bpf_prog_array_copy()
2813 new->prog = include_prog; in bpf_prog_array_copy()
2817 new->prog = NULL; in bpf_prog_array_copy()
2849 sleepable = aux->prog->sleepable; in __bpf_free_used_maps()
2895 bpf_prog_stream_free(aux->prog); in bpf_prog_free_deferred()
2904 bpf_prog_dev_bound_destroy(aux->prog); in bpf_prog_free_deferred()
2906 if (aux->prog->has_callchain_buf) in bpf_prog_free_deferred()
2921 bpf_prog_unlock_free(aux->prog); in bpf_prog_free_deferred()
2923 bpf_jit_free(aux->prog); in bpf_prog_free_deferred()
3039 struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) in bpf_int_jit_compile() argument
3041 return prog; in bpf_int_jit_compile()
3047 void __weak bpf_jit_compile(struct bpf_prog *prog) in bpf_jit_compile() argument
3198 struct bpf_prog *prog; in bpf_prog_report_may_goto_violation() local
3200 prog = bpf_prog_find_from_stack(); in bpf_prog_report_may_goto_violation()
3201 if (!prog) in bpf_prog_report_may_goto_violation()
3203 bpf_stream_stage(ss, prog, BPF_STDERR, ({ in bpf_prog_report_may_goto_violation()
3263 int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, in bpf_prog_get_file_line() argument
3272 btf = prog->aux->btf; in bpf_prog_get_file_line()
3273 linfo = prog->aux->linfo; in bpf_prog_get_file_line()
3274 jited_linfo = prog->aux->jited_linfo; in bpf_prog_get_file_line()
3278 len = prog->aux->func ? prog->aux->func[prog->aux->func_idx]->len : prog->len; in bpf_prog_get_file_line()
3280 linfo = &prog->aux->linfo[prog->aux->linfo_idx]; in bpf_prog_get_file_line()
3281 jited_linfo = &prog->aux->jited_linfo[prog->aux->linfo_idx]; in bpf_prog_get_file_line()
3285 nr_linfo = prog->aux->nr_linfo - prog->aux->linfo_idx; in bpf_prog_get_file_line()
3309 struct bpf_prog *prog; member
3315 struct bpf_prog *prog; in find_from_stack_cb() local
3323 prog = bpf_prog_ksym_find(ip); in find_from_stack_cb()
3325 if (!prog) in find_from_stack_cb()
3327 if (bpf_is_subprog(prog)) in find_from_stack_cb()
3329 ctxp->prog = prog; in find_from_stack_cb()
3338 return ctx.prog; in bpf_prog_find_from_stack()