Lines Matching refs:ctx
42 static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) in prepare_bpf_tail_call_cnt() argument
44 const struct bpf_prog *prog = ctx->prog; in prepare_bpf_tail_call_cnt()
64 emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); in prepare_bpf_tail_call_cnt()
67 emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4); in prepare_bpf_tail_call_cnt()
73 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); in prepare_bpf_tail_call_cnt()
76 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset); in prepare_bpf_tail_call_cnt()
78 emit_uncond_jmp(ctx, 2); in prepare_bpf_tail_call_cnt()
84 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); in prepare_bpf_tail_call_cnt()
87 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); in prepare_bpf_tail_call_cnt()
92 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); in prepare_bpf_tail_call_cnt()
125 static void build_prologue(struct jit_ctx *ctx) in build_prologue() argument
128 const struct bpf_prog *prog = ctx->prog; in build_prologue()
131 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); in build_prologue()
144 emit_insn(ctx, nop); in build_prologue()
152 emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0); in build_prologue()
154 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); in build_prologue()
157 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); in build_prologue()
160 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); in build_prologue()
163 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); in build_prologue()
166 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); in build_prologue()
169 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); in build_prologue()
172 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); in build_prologue()
175 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); in build_prologue()
178 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); in build_prologue()
180 prepare_bpf_tail_call_cnt(ctx, &store_offset); in build_prologue()
182 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); in build_prologue()
185 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); in build_prologue()
187 ctx->stack_size = stack_adjust; in build_prologue()
190 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) in __build_epilogue() argument
192 int stack_adjust = ctx->stack_size; in __build_epilogue()
196 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
199 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
202 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
205 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
208 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
211 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
214 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
217 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
224 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
227 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); in __build_epilogue()
229 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); in __build_epilogue()
233 emit_insn(ctx, addiw, LOONGARCH_GPR_A0, regmap[BPF_REG_0], 0); in __build_epilogue()
235 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); in __build_epilogue()
241 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 6); in __build_epilogue()
245 static void build_epilogue(struct jit_ctx *ctx) in build_epilogue() argument
247 __build_epilogue(ctx, false); in build_epilogue()
260 static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) in emit_bpf_tail_call() argument
263 int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); in emit_bpf_tail_call()
269 const int idx0 = ctx->idx; in emit_bpf_tail_call()
271 #define cur_offset (ctx->idx - idx0) in emit_bpf_tail_call()
282 tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; in emit_bpf_tail_call()
284 emit_insn(ctx, ldwu, t1, a1, off); in emit_bpf_tail_call()
286 if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) in emit_bpf_tail_call()
293 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); in emit_bpf_tail_call()
294 emit_insn(ctx, ldd, t3, REG_TCC, 0); in emit_bpf_tail_call()
295 emit_insn(ctx, addid, t3, t3, 1); in emit_bpf_tail_call()
296 emit_insn(ctx, std, t3, REG_TCC, 0); in emit_bpf_tail_call()
297 emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); in emit_bpf_tail_call()
298 if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0) in emit_bpf_tail_call()
306 emit_insn(ctx, alsld, t2, a2, a1, 2); in emit_bpf_tail_call()
308 emit_insn(ctx, ldd, t2, t2, off); in emit_bpf_tail_call()
310 if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) in emit_bpf_tail_call()
315 emit_insn(ctx, ldd, t3, t2, off); in emit_bpf_tail_call()
316 __build_epilogue(ctx, true); in emit_bpf_tail_call()
327 static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) in emit_atomic() argument
339 move_imm(ctx, t1, off, false); in emit_atomic()
340 emit_insn(ctx, addd, t1, dst, t1); in emit_atomic()
341 move_reg(ctx, t3, src); in emit_atomic()
347 emit_insn(ctx, amaddd, t2, t1, src); in emit_atomic()
349 emit_insn(ctx, amaddw, t2, t1, src); in emit_atomic()
353 emit_insn(ctx, amandd, t2, t1, src); in emit_atomic()
355 emit_insn(ctx, amandw, t2, t1, src); in emit_atomic()
359 emit_insn(ctx, amord, t2, t1, src); in emit_atomic()
361 emit_insn(ctx, amorw, t2, t1, src); in emit_atomic()
365 emit_insn(ctx, amxord, t2, t1, src); in emit_atomic()
367 emit_insn(ctx, amxorw, t2, t1, src); in emit_atomic()
372 emit_insn(ctx, amaddd, src, t1, t3); in emit_atomic()
374 emit_insn(ctx, amaddw, src, t1, t3); in emit_atomic()
375 emit_zext_32(ctx, src, true); in emit_atomic()
380 emit_insn(ctx, amandd, src, t1, t3); in emit_atomic()
382 emit_insn(ctx, amandw, src, t1, t3); in emit_atomic()
383 emit_zext_32(ctx, src, true); in emit_atomic()
388 emit_insn(ctx, amord, src, t1, t3); in emit_atomic()
390 emit_insn(ctx, amorw, src, t1, t3); in emit_atomic()
391 emit_zext_32(ctx, src, true); in emit_atomic()
396 emit_insn(ctx, amxord, src, t1, t3); in emit_atomic()
398 emit_insn(ctx, amxorw, src, t1, t3); in emit_atomic()
399 emit_zext_32(ctx, src, true); in emit_atomic()
405 emit_insn(ctx, amswapd, src, t1, t3); in emit_atomic()
407 emit_insn(ctx, amswapw, src, t1, t3); in emit_atomic()
408 emit_zext_32(ctx, src, true); in emit_atomic()
413 move_reg(ctx, t2, r0); in emit_atomic()
415 emit_insn(ctx, lld, r0, t1, 0); in emit_atomic()
416 emit_insn(ctx, bne, t2, r0, 4); in emit_atomic()
417 move_reg(ctx, t3, src); in emit_atomic()
418 emit_insn(ctx, scd, t3, t1, 0); in emit_atomic()
419 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4); in emit_atomic()
421 emit_insn(ctx, llw, r0, t1, 0); in emit_atomic()
422 emit_zext_32(ctx, t2, true); in emit_atomic()
423 emit_zext_32(ctx, r0, true); in emit_atomic()
424 emit_insn(ctx, bne, t2, r0, 4); in emit_atomic()
425 move_reg(ctx, t3, src); in emit_atomic()
426 emit_insn(ctx, scw, t3, t1, 0); in emit_atomic()
427 emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); in emit_atomic()
428 emit_zext_32(ctx, r0, true); in emit_atomic()
457 struct jit_ctx *ctx, in add_exception_handler() argument
464 if (!ctx->image || !ctx->prog->aux->extable) in add_exception_handler()
471 if (WARN_ON_ONCE(ctx->num_exentries >= ctx->prog->aux->num_exentries)) in add_exception_handler()
474 ex = &ctx->prog->aux->extable[ctx->num_exentries]; in add_exception_handler()
475 pc = (unsigned long)&ctx->image[ctx->idx - 1]; in add_exception_handler()
498 ctx->num_exentries++; in add_exception_handler()
503 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) in build_insn() argument
508 int i = insn - ctx->prog->insnsi; in build_insn()
526 move_reg(ctx, dst, src); in build_insn()
527 emit_zext_32(ctx, dst, is32); in build_insn()
530 move_reg(ctx, t1, src); in build_insn()
531 emit_insn(ctx, extwb, dst, t1); in build_insn()
532 emit_zext_32(ctx, dst, is32); in build_insn()
535 move_reg(ctx, t1, src); in build_insn()
536 emit_insn(ctx, extwh, dst, t1); in build_insn()
537 emit_zext_32(ctx, dst, is32); in build_insn()
540 emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO); in build_insn()
548 move_imm(ctx, dst, imm, is32); in build_insn()
554 emit_insn(ctx, addd, dst, dst, src); in build_insn()
555 emit_zext_32(ctx, dst, is32); in build_insn()
562 emit_insn(ctx, addid, dst, dst, imm); in build_insn()
564 move_imm(ctx, t1, imm, is32); in build_insn()
565 emit_insn(ctx, addd, dst, dst, t1); in build_insn()
567 emit_zext_32(ctx, dst, is32); in build_insn()
573 emit_insn(ctx, subd, dst, dst, src); in build_insn()
574 emit_zext_32(ctx, dst, is32); in build_insn()
581 emit_insn(ctx, addid, dst, dst, -imm); in build_insn()
583 move_imm(ctx, t1, imm, is32); in build_insn()
584 emit_insn(ctx, subd, dst, dst, t1); in build_insn()
586 emit_zext_32(ctx, dst, is32); in build_insn()
592 emit_insn(ctx, muld, dst, dst, src); in build_insn()
593 emit_zext_32(ctx, dst, is32); in build_insn()
599 move_imm(ctx, t1, imm, is32); in build_insn()
600 emit_insn(ctx, muld, dst, dst, t1); in build_insn()
601 emit_zext_32(ctx, dst, is32); in build_insn()
608 emit_zext_32(ctx, dst, is32); in build_insn()
609 move_reg(ctx, t1, src); in build_insn()
610 emit_zext_32(ctx, t1, is32); in build_insn()
611 emit_insn(ctx, divdu, dst, dst, t1); in build_insn()
612 emit_zext_32(ctx, dst, is32); in build_insn()
614 emit_sext_32(ctx, dst, is32); in build_insn()
615 move_reg(ctx, t1, src); in build_insn()
616 emit_sext_32(ctx, t1, is32); in build_insn()
617 emit_insn(ctx, divd, dst, dst, t1); in build_insn()
618 emit_sext_32(ctx, dst, is32); in build_insn()
626 move_imm(ctx, t1, imm, is32); in build_insn()
627 emit_zext_32(ctx, dst, is32); in build_insn()
628 emit_insn(ctx, divdu, dst, dst, t1); in build_insn()
629 emit_zext_32(ctx, dst, is32); in build_insn()
631 move_imm(ctx, t1, imm, false); in build_insn()
632 emit_sext_32(ctx, t1, is32); in build_insn()
633 emit_sext_32(ctx, dst, is32); in build_insn()
634 emit_insn(ctx, divd, dst, dst, t1); in build_insn()
635 emit_sext_32(ctx, dst, is32); in build_insn()
643 emit_zext_32(ctx, dst, is32); in build_insn()
644 move_reg(ctx, t1, src); in build_insn()
645 emit_zext_32(ctx, t1, is32); in build_insn()
646 emit_insn(ctx, moddu, dst, dst, t1); in build_insn()
647 emit_zext_32(ctx, dst, is32); in build_insn()
649 emit_sext_32(ctx, dst, is32); in build_insn()
650 move_reg(ctx, t1, src); in build_insn()
651 emit_sext_32(ctx, t1, is32); in build_insn()
652 emit_insn(ctx, modd, dst, dst, t1); in build_insn()
653 emit_sext_32(ctx, dst, is32); in build_insn()
661 move_imm(ctx, t1, imm, is32); in build_insn()
662 emit_zext_32(ctx, dst, is32); in build_insn()
663 emit_insn(ctx, moddu, dst, dst, t1); in build_insn()
664 emit_zext_32(ctx, dst, is32); in build_insn()
666 move_imm(ctx, t1, imm, false); in build_insn()
667 emit_sext_32(ctx, t1, is32); in build_insn()
668 emit_sext_32(ctx, dst, is32); in build_insn()
669 emit_insn(ctx, modd, dst, dst, t1); in build_insn()
670 emit_sext_32(ctx, dst, is32); in build_insn()
677 move_imm(ctx, t1, imm, is32); in build_insn()
678 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); in build_insn()
679 emit_zext_32(ctx, dst, is32); in build_insn()
685 emit_insn(ctx, and, dst, dst, src); in build_insn()
686 emit_zext_32(ctx, dst, is32); in build_insn()
693 emit_insn(ctx, andi, dst, dst, imm); in build_insn()
695 move_imm(ctx, t1, imm, is32); in build_insn()
696 emit_insn(ctx, and, dst, dst, t1); in build_insn()
698 emit_zext_32(ctx, dst, is32); in build_insn()
704 emit_insn(ctx, or, dst, dst, src); in build_insn()
705 emit_zext_32(ctx, dst, is32); in build_insn()
712 emit_insn(ctx, ori, dst, dst, imm); in build_insn()
714 move_imm(ctx, t1, imm, is32); in build_insn()
715 emit_insn(ctx, or, dst, dst, t1); in build_insn()
717 emit_zext_32(ctx, dst, is32); in build_insn()
723 emit_insn(ctx, xor, dst, dst, src); in build_insn()
724 emit_zext_32(ctx, dst, is32); in build_insn()
731 emit_insn(ctx, xori, dst, dst, imm); in build_insn()
733 move_imm(ctx, t1, imm, is32); in build_insn()
734 emit_insn(ctx, xor, dst, dst, t1); in build_insn()
736 emit_zext_32(ctx, dst, is32); in build_insn()
741 emit_insn(ctx, sllw, dst, dst, src); in build_insn()
742 emit_zext_32(ctx, dst, is32); in build_insn()
746 emit_insn(ctx, slld, dst, dst, src); in build_insn()
751 emit_insn(ctx, slliw, dst, dst, imm); in build_insn()
752 emit_zext_32(ctx, dst, is32); in build_insn()
756 emit_insn(ctx, sllid, dst, dst, imm); in build_insn()
761 emit_insn(ctx, srlw, dst, dst, src); in build_insn()
762 emit_zext_32(ctx, dst, is32); in build_insn()
766 emit_insn(ctx, srld, dst, dst, src); in build_insn()
771 emit_insn(ctx, srliw, dst, dst, imm); in build_insn()
772 emit_zext_32(ctx, dst, is32); in build_insn()
776 emit_insn(ctx, srlid, dst, dst, imm); in build_insn()
781 emit_insn(ctx, sraw, dst, dst, src); in build_insn()
782 emit_zext_32(ctx, dst, is32); in build_insn()
786 emit_insn(ctx, srad, dst, dst, src); in build_insn()
791 emit_insn(ctx, sraiw, dst, dst, imm); in build_insn()
792 emit_zext_32(ctx, dst, is32); in build_insn()
796 emit_insn(ctx, sraid, dst, dst, imm); in build_insn()
804 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); in build_insn()
808 emit_zext_32(ctx, dst, is32); in build_insn()
820 emit_insn(ctx, revb2h, dst, dst); in build_insn()
822 emit_insn(ctx, bstrpickd, dst, dst, 15, 0); in build_insn()
825 emit_insn(ctx, revb2w, dst, dst); in build_insn()
827 emit_zext_32(ctx, dst, true); in build_insn()
830 emit_insn(ctx, revbd, dst, dst); in build_insn()
856 jmp_offset = bpf2la_offset(i, off, ctx); in build_insn()
857 move_reg(ctx, t1, dst); in build_insn()
858 move_reg(ctx, t2, src); in build_insn()
860 emit_sext_32(ctx, t1, is32); in build_insn()
861 emit_sext_32(ctx, t2, is32); in build_insn()
863 emit_zext_32(ctx, t1, is32); in build_insn()
864 emit_zext_32(ctx, t2, is32); in build_insn()
866 if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0) in build_insn()
891 jmp_offset = bpf2la_offset(i, off, ctx); in build_insn()
893 move_imm(ctx, t1, imm, false); in build_insn()
899 move_reg(ctx, t2, dst); in build_insn()
901 emit_sext_32(ctx, tm, is32); in build_insn()
902 emit_sext_32(ctx, t2, is32); in build_insn()
904 emit_zext_32(ctx, tm, is32); in build_insn()
905 emit_zext_32(ctx, t2, is32); in build_insn()
907 if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0) in build_insn()
914 jmp_offset = bpf2la_offset(i, off, ctx); in build_insn()
915 emit_insn(ctx, and, t1, dst, src); in build_insn()
916 emit_zext_32(ctx, t1, is32); in build_insn()
917 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) in build_insn()
924 jmp_offset = bpf2la_offset(i, off, ctx); in build_insn()
925 move_imm(ctx, t1, imm, is32); in build_insn()
926 emit_insn(ctx, and, t1, dst, t1); in build_insn()
927 emit_zext_32(ctx, t1, is32); in build_insn()
928 if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) in build_insn()
936 jmp_offset = bpf2la_offset(i, off, ctx); in build_insn()
938 jmp_offset = bpf2la_offset(i, imm, ctx); in build_insn()
939 if (emit_uncond_jmp(ctx, jmp_offset) < 0) in build_insn()
945 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, in build_insn()
951 tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); in build_insn()
952 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); in build_insn()
955 move_addr(ctx, t1, func_addr); in build_insn()
956 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); in build_insn()
959 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); in build_insn()
965 if (emit_bpf_tail_call(ctx, i) < 0) in build_insn()
971 if (i == ctx->prog->len - 1) in build_insn()
974 jmp_offset = epilogue_offset(ctx); in build_insn()
975 if (emit_uncond_jmp(ctx, jmp_offset) < 0) in build_insn()
985 move_addr(ctx, dst, imm64); in build_insn()
987 move_imm(ctx, dst, imm64, is32); in build_insn()
1013 emit_insn(ctx, ldb, dst, src, off); in build_insn()
1015 emit_insn(ctx, ldbu, dst, src, off); in build_insn()
1017 move_imm(ctx, t1, off, is32); in build_insn()
1019 emit_insn(ctx, ldxb, dst, src, t1); in build_insn()
1021 emit_insn(ctx, ldxbu, dst, src, t1); in build_insn()
1027 emit_insn(ctx, ldh, dst, src, off); in build_insn()
1029 emit_insn(ctx, ldhu, dst, src, off); in build_insn()
1031 move_imm(ctx, t1, off, is32); in build_insn()
1033 emit_insn(ctx, ldxh, dst, src, t1); in build_insn()
1035 emit_insn(ctx, ldxhu, dst, src, t1); in build_insn()
1041 emit_insn(ctx, ldw, dst, src, off); in build_insn()
1043 emit_insn(ctx, ldwu, dst, src, off); in build_insn()
1045 move_imm(ctx, t1, off, is32); in build_insn()
1047 emit_insn(ctx, ldxw, dst, src, t1); in build_insn()
1049 emit_insn(ctx, ldxwu, dst, src, t1); in build_insn()
1053 move_imm(ctx, t1, off, is32); in build_insn()
1054 emit_insn(ctx, ldxd, dst, src, t1); in build_insn()
1058 ret = add_exception_handler(insn, ctx, dst); in build_insn()
1070 move_imm(ctx, t1, imm, is32); in build_insn()
1072 emit_insn(ctx, stb, t1, dst, off); in build_insn()
1074 move_imm(ctx, t2, off, is32); in build_insn()
1075 emit_insn(ctx, stxb, t1, dst, t2); in build_insn()
1079 move_imm(ctx, t1, imm, is32); in build_insn()
1081 emit_insn(ctx, sth, t1, dst, off); in build_insn()
1083 move_imm(ctx, t2, off, is32); in build_insn()
1084 emit_insn(ctx, stxh, t1, dst, t2); in build_insn()
1088 move_imm(ctx, t1, imm, is32); in build_insn()
1090 emit_insn(ctx, stw, t1, dst, off); in build_insn()
1092 emit_insn(ctx, stptrw, t1, dst, off); in build_insn()
1094 move_imm(ctx, t2, off, is32); in build_insn()
1095 emit_insn(ctx, stxw, t1, dst, t2); in build_insn()
1099 move_imm(ctx, t1, imm, is32); in build_insn()
1101 emit_insn(ctx, std, t1, dst, off); in build_insn()
1103 emit_insn(ctx, stptrd, t1, dst, off); in build_insn()
1105 move_imm(ctx, t2, off, is32); in build_insn()
1106 emit_insn(ctx, stxd, t1, dst, t2); in build_insn()
1120 emit_insn(ctx, stb, src, dst, off); in build_insn()
1122 move_imm(ctx, t1, off, is32); in build_insn()
1123 emit_insn(ctx, stxb, src, dst, t1); in build_insn()
1128 emit_insn(ctx, sth, src, dst, off); in build_insn()
1130 move_imm(ctx, t1, off, is32); in build_insn()
1131 emit_insn(ctx, stxh, src, dst, t1); in build_insn()
1136 emit_insn(ctx, stw, src, dst, off); in build_insn()
1138 emit_insn(ctx, stptrw, src, dst, off); in build_insn()
1140 move_imm(ctx, t1, off, is32); in build_insn()
1141 emit_insn(ctx, stxw, src, dst, t1); in build_insn()
1146 emit_insn(ctx, std, src, dst, off); in build_insn()
1148 emit_insn(ctx, stptrd, src, dst, off); in build_insn()
1150 move_imm(ctx, t1, off, is32); in build_insn()
1151 emit_insn(ctx, stxd, src, dst, t1); in build_insn()
1159 emit_atomic(insn, ctx); in build_insn()
1178 static int build_body(struct jit_ctx *ctx, bool extra_pass) in build_body() argument
1181 const struct bpf_prog *prog = ctx->prog; in build_body()
1187 if (ctx->image == NULL) in build_body()
1188 ctx->offset[i] = ctx->idx; in build_body()
1190 ret = build_insn(insn, ctx, extra_pass); in build_body()
1193 if (ctx->image == NULL) in build_body()
1194 ctx->offset[i] = ctx->idx; in build_body()
1201 if (ctx->image == NULL) in build_body()
1202 ctx->offset[i] = ctx->idx; in build_body()
1217 static int validate_code(struct jit_ctx *ctx) in validate_code() argument
1222 for (i = 0; i < ctx->idx; i++) { in validate_code()
1223 insn = ctx->image[i]; in validate_code()
1232 static int validate_ctx(struct jit_ctx *ctx) in validate_ctx() argument
1234 if (validate_code(ctx)) in validate_ctx()
1237 if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries)) in validate_ctx()
1243 static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target) in emit_jump_and_link() argument
1250 move_imm(ctx, LOONGARCH_GPR_T1, target, false); in emit_jump_and_link()
1251 emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0); in emit_jump_and_link()
1259 struct jit_ctx ctx; in emit_jump_or_nops() local
1261 ctx.idx = 0; in emit_jump_or_nops()
1262 ctx.image = (union loongarch_instruction *)insns; in emit_jump_or_nops()
1266 emit_insn((&ctx), nop); in emit_jump_or_nops()
1270 return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO, (u64)target); in emit_jump_or_nops()
1273 static int emit_call(struct jit_ctx *ctx, u64 addr) in emit_call() argument
1275 return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr); in emit_call()
1343 static void store_args(struct jit_ctx *ctx, int nargs, int args_off) in store_args() argument
1348 emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); in store_args()
1353 static void restore_args(struct jit_ctx *ctx, int nargs, int args_off) in restore_args() argument
1358 emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); in restore_args()
1363 static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, in invoke_bpf_prog() argument
1372 move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false); in invoke_bpf_prog()
1373 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); in invoke_bpf_prog()
1375 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); in invoke_bpf_prog()
1379 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); in invoke_bpf_prog()
1381 emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off); in invoke_bpf_prog()
1382 ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p)); in invoke_bpf_prog()
1387 move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0); in invoke_bpf_prog()
1393 branch = (u32 *)ctx->image + ctx->idx; in invoke_bpf_prog()
1395 emit_insn(ctx, nop); in invoke_bpf_prog()
1398 emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off); in invoke_bpf_prog()
1400 move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false); in invoke_bpf_prog()
1401 ret = emit_call(ctx, (const u64)p->bpf_func); in invoke_bpf_prog()
1406 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); in invoke_bpf_prog()
1407 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); in invoke_bpf_prog()
1411 if (ctx->image) { in invoke_bpf_prog()
1412 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch; in invoke_bpf_prog()
1417 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); in invoke_bpf_prog()
1419 move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1); in invoke_bpf_prog()
1421 emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); in invoke_bpf_prog()
1422 ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p)); in invoke_bpf_prog()
1427 static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl, in invoke_bpf_mod_ret() argument
1432 emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); in invoke_bpf_mod_ret()
1434 invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, run_ctx_off, true); in invoke_bpf_mod_ret()
1435 emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off); in invoke_bpf_mod_ret()
1436 branches[i] = (u32 *)ctx->image + ctx->idx; in invoke_bpf_mod_ret()
1437 emit_insn(ctx, nop); in invoke_bpf_mod_ret()
1451 static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, in __arch_prepare_bpf_trampoline() argument
1548 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); in __arch_prepare_bpf_trampoline()
1549 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); in __arch_prepare_bpf_trampoline()
1550 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); in __arch_prepare_bpf_trampoline()
1551 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); in __arch_prepare_bpf_trampoline()
1559 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16); in __arch_prepare_bpf_trampoline()
1560 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); in __arch_prepare_bpf_trampoline()
1561 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); in __arch_prepare_bpf_trampoline()
1562 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16); in __arch_prepare_bpf_trampoline()
1565 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); in __arch_prepare_bpf_trampoline()
1566 emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); in __arch_prepare_bpf_trampoline()
1567 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); in __arch_prepare_bpf_trampoline()
1568 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); in __arch_prepare_bpf_trampoline()
1572 emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); in __arch_prepare_bpf_trampoline()
1575 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); in __arch_prepare_bpf_trampoline()
1579 move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false); in __arch_prepare_bpf_trampoline()
1580 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off); in __arch_prepare_bpf_trampoline()
1584 move_imm(ctx, LOONGARCH_GPR_T1, nargs, false); in __arch_prepare_bpf_trampoline()
1585 emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off); in __arch_prepare_bpf_trampoline()
1587 store_args(ctx, nargs, args_off); in __arch_prepare_bpf_trampoline()
1598 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false); in __arch_prepare_bpf_trampoline()
1599 ret = emit_call(ctx, (const u64)__bpf_tramp_enter); in __arch_prepare_bpf_trampoline()
1605 ret = invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off, in __arch_prepare_bpf_trampoline()
1615 invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, run_ctx_off, branches); in __arch_prepare_bpf_trampoline()
1619 restore_args(ctx, m->nr_args, args_off); in __arch_prepare_bpf_trampoline()
1622 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); in __arch_prepare_bpf_trampoline()
1624 ret = emit_call(ctx, (const u64)orig_call); in __arch_prepare_bpf_trampoline()
1627 emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); in __arch_prepare_bpf_trampoline()
1628 emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); in __arch_prepare_bpf_trampoline()
1629 im->ip_after_call = ctx->ro_image + ctx->idx; in __arch_prepare_bpf_trampoline()
1632 emit_insn(ctx, nop); in __arch_prepare_bpf_trampoline()
1635 for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) { in __arch_prepare_bpf_trampoline()
1636 int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i]; in __arch_prepare_bpf_trampoline()
1641 ret = invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, run_ctx_off, false); in __arch_prepare_bpf_trampoline()
1647 im->ip_epilogue = ctx->ro_image + ctx->idx; in __arch_prepare_bpf_trampoline()
1648 move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false); in __arch_prepare_bpf_trampoline()
1649 ret = emit_call(ctx, (const u64)__bpf_tramp_exit); in __arch_prepare_bpf_trampoline()
1655 restore_args(ctx, m->nr_args, args_off); in __arch_prepare_bpf_trampoline()
1658 emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); in __arch_prepare_bpf_trampoline()
1659 emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); in __arch_prepare_bpf_trampoline()
1662 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); in __arch_prepare_bpf_trampoline()
1665 emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); in __arch_prepare_bpf_trampoline()
1669 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); in __arch_prepare_bpf_trampoline()
1670 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); in __arch_prepare_bpf_trampoline()
1671 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); in __arch_prepare_bpf_trampoline()
1673 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); in __arch_prepare_bpf_trampoline()
1676 emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); in __arch_prepare_bpf_trampoline()
1677 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); in __arch_prepare_bpf_trampoline()
1678 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); in __arch_prepare_bpf_trampoline()
1680 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); in __arch_prepare_bpf_trampoline()
1681 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); in __arch_prepare_bpf_trampoline()
1682 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); in __arch_prepare_bpf_trampoline()
1686 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); in __arch_prepare_bpf_trampoline()
1689 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); in __arch_prepare_bpf_trampoline()
1692 ret = ctx->idx; in __arch_prepare_bpf_trampoline()
1705 struct jit_ctx ctx; in arch_prepare_bpf_trampoline() local
1712 ctx.image = (union loongarch_instruction *)image; in arch_prepare_bpf_trampoline()
1713 ctx.ro_image = (union loongarch_instruction *)ro_image; in arch_prepare_bpf_trampoline()
1714 ctx.idx = 0; in arch_prepare_bpf_trampoline()
1717 ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags); in arch_prepare_bpf_trampoline()
1718 if (ret > 0 && validate_code(&ctx) < 0) { in arch_prepare_bpf_trampoline()
1739 struct jit_ctx ctx; in arch_bpf_trampoline_size() local
1742 ctx.image = NULL; in arch_bpf_trampoline_size()
1743 ctx.idx = 0; in arch_bpf_trampoline_size()
1745 ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags); in arch_bpf_trampoline_size()
1756 struct jit_ctx ctx; in bpf_int_jit_compile() local
1791 if (jit_data->ctx.offset) { in bpf_int_jit_compile()
1792 ctx = jit_data->ctx; in bpf_int_jit_compile()
1796 prog_size = sizeof(u32) * ctx.idx; in bpf_int_jit_compile()
1800 memset(&ctx, 0, sizeof(ctx)); in bpf_int_jit_compile()
1801 ctx.prog = prog; in bpf_int_jit_compile()
1803 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); in bpf_int_jit_compile()
1804 if (ctx.offset == NULL) { in bpf_int_jit_compile()
1810 build_prologue(&ctx); in bpf_int_jit_compile()
1811 if (build_body(&ctx, extra_pass)) { in bpf_int_jit_compile()
1815 ctx.epilogue_offset = ctx.idx; in bpf_int_jit_compile()
1816 build_epilogue(&ctx); in bpf_int_jit_compile()
1825 prog_size = sizeof(u32) * ctx.idx; in bpf_int_jit_compile()
1836 ctx.image = (union loongarch_instruction *)image_ptr; in bpf_int_jit_compile()
1841 ctx.idx = 0; in bpf_int_jit_compile()
1842 ctx.num_exentries = 0; in bpf_int_jit_compile()
1844 build_prologue(&ctx); in bpf_int_jit_compile()
1845 if (build_body(&ctx, extra_pass)) { in bpf_int_jit_compile()
1850 build_epilogue(&ctx); in bpf_int_jit_compile()
1853 if (validate_ctx(&ctx)) { in bpf_int_jit_compile()
1861 bpf_jit_dump(prog->len, prog_size, 2, ctx.image); in bpf_int_jit_compile()
1864 flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx)); in bpf_int_jit_compile()
1869 if (extra_pass && ctx.idx != jit_data->ctx.idx) { in bpf_int_jit_compile()
1871 ctx.idx, jit_data->ctx.idx); in bpf_int_jit_compile()
1881 jit_data->ctx = ctx; in bpf_int_jit_compile()
1887 prog->bpf_func = (void *)ctx.image; in bpf_int_jit_compile()
1894 ctx.offset[i] *= LOONGARCH_INSN_SIZE; in bpf_int_jit_compile()
1895 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); in bpf_int_jit_compile()
1898 kvfree(ctx.offset); in bpf_int_jit_compile()