Lines Matching refs:prog
33 do { prog = emit_code(prog, bytes, len); } while (0)
248 u8 *prog = *pprog; in push_callee_regs() local
258 *pprog = prog; in push_callee_regs()
263 u8 *prog = *pprog; in pop_callee_regs() local
273 *pprog = prog; in pop_callee_regs()
284 u8 *prog = *pprog; in emit_prologue() local
289 memcpy(prog, x86_nops[5], X86_PATCH_SIZE); in emit_prologue()
290 prog += X86_PATCH_SIZE; in emit_prologue()
304 *pprog = prog; in emit_prologue()
309 u8 *prog = *pprog; in emit_patch() local
318 *pprog = prog; in emit_patch()
339 u8 *prog; in __bpf_arch_text_poke() local
344 prog = old_insn; in __bpf_arch_text_poke()
346 emit_call(&prog, old_addr, ip) : in __bpf_arch_text_poke()
347 emit_jump(&prog, old_addr, ip); in __bpf_arch_text_poke()
354 prog = new_insn; in __bpf_arch_text_poke()
356 emit_call(&prog, new_addr, ip) : in __bpf_arch_text_poke()
357 emit_jump(&prog, new_addr, ip); in __bpf_arch_text_poke()
394 u8 *prog = *pprog; in emit_indirect_jump() local
401 emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); in emit_indirect_jump()
406 *pprog = prog; in emit_indirect_jump()
428 u8 *prog = *pprog, *start = *pprog; in emit_bpf_tail_call_indirect() local
445 offset = ctx->tail_call_indirect_label - (prog + 2 - start); in emit_bpf_tail_call_indirect()
455 offset = ctx->tail_call_indirect_label - (prog + 2 - start); in emit_bpf_tail_call_indirect()
470 offset = ctx->tail_call_indirect_label - (prog + 2 - start); in emit_bpf_tail_call_indirect()
473 pop_callee_regs(&prog, callee_regs_used); in emit_bpf_tail_call_indirect()
490 emit_indirect_jump(&prog, 1 /* rcx */, ip + (prog - start)); in emit_bpf_tail_call_indirect()
493 ctx->tail_call_indirect_label = prog - start; in emit_bpf_tail_call_indirect()
494 *pprog = prog; in emit_bpf_tail_call_indirect()
503 u8 *prog = *pprog, *start = *pprog; in emit_bpf_tail_call_direct() local
513 offset = ctx->tail_call_direct_label - (prog + 2 - start); in emit_bpf_tail_call_direct()
518 poke->tailcall_bypass = ip + (prog - start); in emit_bpf_tail_call_direct()
523 emit_jump(&prog, (u8 *)poke->tailcall_target + X86_PATCH_SIZE, in emit_bpf_tail_call_direct()
526 pop_callee_regs(&prog, callee_regs_used); in emit_bpf_tail_call_direct()
531 memcpy(prog, x86_nops[5], X86_PATCH_SIZE); in emit_bpf_tail_call_direct()
532 prog += X86_PATCH_SIZE; in emit_bpf_tail_call_direct()
535 ctx->tail_call_direct_label = prog - start; in emit_bpf_tail_call_direct()
537 *pprog = prog; in emit_bpf_tail_call_direct()
540 static void bpf_tail_call_direct_fixup(struct bpf_prog *prog) in bpf_tail_call_direct_fixup() argument
547 for (i = 0; i < prog->aux->size_poke_tab; i++) { in bpf_tail_call_direct_fixup()
548 poke = &prog->aux->poke_tab[i]; in bpf_tail_call_direct_fixup()
549 if (poke->aux && poke->aux != prog->aux) in bpf_tail_call_direct_fixup()
589 u8 *prog = *pprog; in emit_mov_imm32() local
623 *pprog = prog; in emit_mov_imm32()
629 u8 *prog = *pprog; in emit_mov_imm64() local
638 emit_mov_imm32(&prog, false, dst_reg, imm32_lo); in emit_mov_imm64()
646 *pprog = prog; in emit_mov_imm64()
651 u8 *prog = *pprog; in emit_mov_reg() local
663 *pprog = prog; in emit_mov_reg()
669 u8 *prog = *pprog; in emit_insn_suffix() local
683 *pprog = prog; in emit_insn_suffix()
691 u8 *prog = *pprog; in maybe_emit_mod() local
697 *pprog = prog; in maybe_emit_mod()
705 u8 *prog = *pprog; in maybe_emit_1mod() local
711 *pprog = prog; in maybe_emit_1mod()
717 u8 *prog = *pprog; in emit_ldx() local
740 emit_insn_suffix(&prog, src_reg, dst_reg, off); in emit_ldx()
741 *pprog = prog; in emit_ldx()
747 u8 *prog = *pprog; in emit_stx() local
774 emit_insn_suffix(&prog, dst_reg, src_reg, off); in emit_stx()
775 *pprog = prog; in emit_stx()
781 u8 *prog = *pprog; in emit_atomic() local
785 maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW); in emit_atomic()
814 emit_insn_suffix(&prog, dst_reg, src_reg, off); in emit_atomic()
816 *pprog = prog; in emit_atomic()
851 u8 *prog = *pprog; in emit_nops() local
865 *pprog = prog; in emit_nops()
868 #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
882 u8 *prog = temp; in do_jit() local
891 emit_prologue(&prog, bpf_prog->aux->stack_depth, in do_jit()
894 push_callee_regs(&prog, callee_regs_used); in do_jit()
896 ilen = prog - temp; in do_jit()
901 prog = temp; in do_jit()
926 maybe_emit_mod(&prog, dst_reg, src_reg, in do_jit()
934 emit_mov_reg(&prog, in do_jit()
942 maybe_emit_1mod(&prog, dst_reg, in do_jit()
957 maybe_emit_1mod(&prog, dst_reg, in do_jit()
997 emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64, in do_jit()
1002 emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm); in do_jit()
1038 emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg); in do_jit()
1047 maybe_emit_1mod(&prog, src_reg, is64); in do_jit()
1053 emit_mov_reg(&prog, is64, dst_reg, BPF_REG_3); in do_jit()
1057 emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0); in do_jit()
1068 maybe_emit_mod(&prog, dst_reg, dst_reg, in do_jit()
1084 maybe_emit_mod(&prog, src_reg, dst_reg, in do_jit()
1098 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1130 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1242 emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off); in do_jit()
1275 maybe_emit_mod(&prog, src_reg, AUX_REG, true); in do_jit()
1279 end_of_jmp1 = prog; in do_jit()
1282 emit_mov_reg(&prog, true, AUX_REG, src_reg); in do_jit()
1284 maybe_emit_1mod(&prog, AUX_REG, true); in do_jit()
1291 end_of_jmp2 = prog; in do_jit()
1294 emit_mov_imm32(&prog, false, dst_reg, 0); in do_jit()
1301 start_of_ldx = prog; in do_jit()
1304 emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off); in do_jit()
1311 start_of_ldx[-1] = prog - start_of_ldx; in do_jit()
1343 ex->fixup = (prog - start_of_ldx) | (reg2pt_regs[dst_reg] << 8); in do_jit()
1363 emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); in do_jit()
1369 branch_target = prog; in do_jit()
1371 emit_ldx(&prog, BPF_SIZE(insn->code), in do_jit()
1377 emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); in do_jit()
1378 maybe_emit_mod(&prog, AUX_REG, real_src_reg, is64); in do_jit()
1382 err = emit_atomic(&prog, BPF_CMPXCHG, in do_jit()
1392 EMIT2(X86_JNE, -(prog - branch_target) - 2); in do_jit()
1394 emit_mov_reg(&prog, is64, real_src_reg, BPF_REG_0); in do_jit()
1396 emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); in do_jit()
1400 err = emit_atomic(&prog, insn->imm, dst_reg, src_reg, in do_jit()
1412 if (!imm32 || emit_call(&prog, func, image + addrs[i - 1] + 7)) in do_jit()
1415 if (!imm32 || emit_call(&prog, func, image + addrs[i - 1])) in do_jit()
1423 &prog, image + addrs[i - 1], in do_jit()
1428 emit_bpf_tail_call_indirect(&prog, in do_jit()
1457 maybe_emit_mod(&prog, dst_reg, src_reg, in do_jit()
1465 maybe_emit_mod(&prog, dst_reg, src_reg, in do_jit()
1473 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1500 maybe_emit_mod(&prog, dst_reg, dst_reg, in do_jit()
1507 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1583 emit_nops(&prog, nops); in do_jit()
1629 emit_nops(&prog, nops); in do_jit()
1654 emit_nops(&prog, INSN_SZ_DIFF - 2); in do_jit()
1673 pop_callee_regs(&prog, callee_regs_used); in do_jit()
1689 ilen = prog - temp; in do_jit()
1713 prog = temp; in do_jit()
1723 static void save_regs(const struct btf_func_model *m, u8 **prog, int nr_args, in save_regs() argument
1733 emit_stx(prog, bytes_to_bpf_size(m->arg_size[i]), in save_regs()
1739 static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, in restore_regs() argument
1750 emit_ldx(prog, bytes_to_bpf_size(m->arg_size[i]), in restore_regs()
1759 u8 *prog = *pprog; in invoke_bpf_prog() local
1763 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); in invoke_bpf_prog()
1764 if (emit_call(&prog, in invoke_bpf_prog()
1766 __bpf_prog_enter, prog)) in invoke_bpf_prog()
1769 emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0); in invoke_bpf_prog()
1776 jmp_insn = prog; in invoke_bpf_prog()
1777 emit_nops(&prog, 2); in invoke_bpf_prog()
1783 emit_mov_imm64(&prog, BPF_REG_2, in invoke_bpf_prog()
1787 if (emit_call(&prog, p->bpf_func, prog)) in invoke_bpf_prog()
1799 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); in invoke_bpf_prog()
1803 jmp_insn[1] = prog - jmp_insn - 2; in invoke_bpf_prog()
1806 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); in invoke_bpf_prog()
1808 emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); in invoke_bpf_prog()
1809 if (emit_call(&prog, in invoke_bpf_prog()
1811 __bpf_prog_exit, prog)) in invoke_bpf_prog()
1814 *pprog = prog; in invoke_bpf_prog()
1820 u8 *target, *prog = *pprog; in emit_align() local
1822 target = PTR_ALIGN(prog, align); in emit_align()
1823 if (target != prog) in emit_align()
1824 emit_nops(&prog, target - prog); in emit_align()
1826 *pprog = prog; in emit_align()
1831 u8 *prog = *pprog; in emit_cond_near_jump() local
1840 *pprog = prog; in emit_cond_near_jump()
1849 u8 *prog = *pprog; in invoke_bpf() local
1852 if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, in invoke_bpf()
1856 *pprog = prog; in invoke_bpf()
1864 u8 *prog = *pprog; in invoke_bpf_mod_ret() local
1870 emit_mov_imm32(&prog, false, BPF_REG_0, 0); in invoke_bpf_mod_ret()
1871 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); in invoke_bpf_mod_ret()
1873 if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true)) in invoke_bpf_mod_ret()
1888 branches[i] = prog; in invoke_bpf_mod_ret()
1889 emit_nops(&prog, 4 + 2); in invoke_bpf_mod_ret()
1892 *pprog = prog; in invoke_bpf_mod_ret()
1984 u8 *prog; in arch_prepare_bpf_trampoline() local
2008 prog = image; in arch_prepare_bpf_trampoline()
2021 emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8); in arch_prepare_bpf_trampoline()
2023 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -stack_size); in arch_prepare_bpf_trampoline()
2031 save_regs(m, &prog, nr_args, stack_size); in arch_prepare_bpf_trampoline()
2035 emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); in arch_prepare_bpf_trampoline()
2036 if (emit_call(&prog, __bpf_tramp_enter, prog)) { in arch_prepare_bpf_trampoline()
2043 if (invoke_bpf(m, &prog, fentry, stack_size, in arch_prepare_bpf_trampoline()
2053 if (invoke_bpf_mod_ret(m, &prog, fmod_ret, stack_size, in arch_prepare_bpf_trampoline()
2061 restore_regs(m, &prog, nr_args, stack_size); in arch_prepare_bpf_trampoline()
2064 if (emit_call(&prog, orig_call, prog)) { in arch_prepare_bpf_trampoline()
2069 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); in arch_prepare_bpf_trampoline()
2070 im->ip_after_call = prog; in arch_prepare_bpf_trampoline()
2071 memcpy(prog, x86_nops[5], X86_PATCH_SIZE); in arch_prepare_bpf_trampoline()
2072 prog += X86_PATCH_SIZE; in arch_prepare_bpf_trampoline()
2081 emit_align(&prog, 16); in arch_prepare_bpf_trampoline()
2086 emit_cond_near_jump(&branches[i], prog, branches[i], in arch_prepare_bpf_trampoline()
2091 if (invoke_bpf(m, &prog, fexit, stack_size, false)) { in arch_prepare_bpf_trampoline()
2097 restore_regs(m, &prog, nr_args, stack_size); in arch_prepare_bpf_trampoline()
2104 im->ip_epilogue = prog; in arch_prepare_bpf_trampoline()
2106 emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); in arch_prepare_bpf_trampoline()
2107 if (emit_call(&prog, __bpf_tramp_exit, prog)) { in arch_prepare_bpf_trampoline()
2114 emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8); in arch_prepare_bpf_trampoline()
2123 if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) { in arch_prepare_bpf_trampoline()
2127 ret = prog - (u8 *)image; in arch_prepare_bpf_trampoline()
2136 u8 *jg_reloc, *prog = *pprog; in emit_bpf_dispatcher() local
2149 err = emit_cond_near_jump(&prog, /* je func */ in emit_bpf_dispatcher()
2150 (void *)progs[a], prog, in emit_bpf_dispatcher()
2155 emit_indirect_jump(&prog, 2 /* rdx */, prog); in emit_bpf_dispatcher()
2157 *pprog = prog; in emit_bpf_dispatcher()
2177 jg_reloc = prog; in emit_bpf_dispatcher()
2179 err = emit_bpf_dispatcher(&prog, a, a + pivot, /* emit lower_part */ in emit_bpf_dispatcher()
2189 emit_align(&prog, 16); in emit_bpf_dispatcher()
2190 jg_offset = prog - jg_reloc; in emit_bpf_dispatcher()
2193 err = emit_bpf_dispatcher(&prog, a + pivot + 1, /* emit upper_part */ in emit_bpf_dispatcher()
2198 *pprog = prog; in emit_bpf_dispatcher()
2216 u8 *prog = image; in arch_prepare_bpf_dispatcher() local
2219 return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs); in arch_prepare_bpf_dispatcher()
2233 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) in bpf_int_jit_compile() argument
2236 struct bpf_prog *tmp, *orig_prog = prog; in bpf_int_jit_compile()
2248 if (!prog->jit_requested) in bpf_int_jit_compile()
2251 tmp = bpf_jit_blind_constants(prog); in bpf_int_jit_compile()
2258 if (tmp != prog) { in bpf_int_jit_compile()
2260 prog = tmp; in bpf_int_jit_compile()
2263 jit_data = prog->aux->jit_data; in bpf_int_jit_compile()
2267 prog = orig_prog; in bpf_int_jit_compile()
2270 prog->aux->jit_data = jit_data; in bpf_int_jit_compile()
2282 addrs = kvmalloc_array(prog->len + 1, sizeof(*addrs), GFP_KERNEL); in bpf_int_jit_compile()
2284 prog = orig_prog; in bpf_int_jit_compile()
2292 for (proglen = 0, i = 0; i <= prog->len; i++) { in bpf_int_jit_compile()
2308 proglen = do_jit(prog, addrs, image, oldproglen, &ctx, padding); in bpf_int_jit_compile()
2314 prog = orig_prog; in bpf_int_jit_compile()
2333 u32 extable_size = prog->aux->num_exentries * in bpf_int_jit_compile()
2340 prog = orig_prog; in bpf_int_jit_compile()
2343 prog->aux->extable = (void *) image + roundup(proglen, align); in bpf_int_jit_compile()
2350 bpf_jit_dump(prog->len, proglen, pass + 1, image); in bpf_int_jit_compile()
2353 if (!prog->is_func || extra_pass) { in bpf_int_jit_compile()
2354 bpf_tail_call_direct_fixup(prog); in bpf_int_jit_compile()
2363 prog->bpf_func = (void *)image; in bpf_int_jit_compile()
2364 prog->jited = 1; in bpf_int_jit_compile()
2365 prog->jited_len = proglen; in bpf_int_jit_compile()
2367 prog = orig_prog; in bpf_int_jit_compile()
2370 if (!image || !prog->is_func || extra_pass) { in bpf_int_jit_compile()
2372 bpf_prog_fill_jited_linfo(prog, addrs + 1); in bpf_int_jit_compile()
2376 prog->aux->jit_data = NULL; in bpf_int_jit_compile()
2380 bpf_jit_prog_release_other(prog, prog == orig_prog ? in bpf_int_jit_compile()
2382 return prog; in bpf_int_jit_compile()