Lines Matching refs:ctx
127 int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) in push_regs() argument
135 emit(ctx, sw, reg, depth, MIPS_R_SP); in push_regs()
137 emit(ctx, sd, reg, depth, MIPS_R_SP); in push_regs()
142 ctx->stack_used = max((int)ctx->stack_used, depth); in push_regs()
150 int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) in pop_regs() argument
158 emit(ctx, lw, reg, depth, MIPS_R_SP); in pop_regs()
160 emit(ctx, ld, reg, depth, MIPS_R_SP); in pop_regs()
169 int get_target(struct jit_context *ctx, u32 loc) in get_target() argument
171 u32 index = INDEX(ctx->descriptors[loc]); in get_target()
172 unsigned long pc = (unsigned long)&ctx->target[ctx->jit_index]; in get_target()
173 unsigned long addr = (unsigned long)&ctx->target[index]; in get_target()
175 if (!ctx->target) in get_target()
185 int get_offset(const struct jit_context *ctx, int off) in get_offset() argument
187 return (INDEX(ctx->descriptors[ctx->bpf_index + off]) - in get_offset()
188 ctx->jit_index - 1) * sizeof(u32); in get_offset()
192 void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm) in emit_mov_i() argument
195 emit(ctx, addiu, dst, MIPS_R_ZERO, imm); in emit_mov_i()
197 emit(ctx, lui, dst, (s16)((u32)imm >> 16)); in emit_mov_i()
198 emit(ctx, ori, dst, dst, (u16)(imm & 0xffff)); in emit_mov_i()
200 clobber_reg(ctx, dst); in emit_mov_i()
204 void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src) in emit_mov_r() argument
206 emit(ctx, ori, dst, src, 0); in emit_mov_r()
207 clobber_reg(ctx, dst); in emit_mov_r()
298 void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) in emit_alu_i() argument
303 emit(ctx, subu, dst, MIPS_R_ZERO, dst); in emit_alu_i()
307 emit(ctx, andi, dst, dst, (u16)imm); in emit_alu_i()
311 emit(ctx, ori, dst, dst, (u16)imm); in emit_alu_i()
315 emit(ctx, xori, dst, dst, (u16)imm); in emit_alu_i()
319 emit(ctx, sll, dst, dst, imm); in emit_alu_i()
323 emit(ctx, srl, dst, dst, imm); in emit_alu_i()
327 emit(ctx, sra, dst, dst, imm); in emit_alu_i()
331 emit(ctx, addiu, dst, dst, imm); in emit_alu_i()
335 emit(ctx, addiu, dst, dst, -imm); in emit_alu_i()
338 clobber_reg(ctx, dst); in emit_alu_i()
342 void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) in emit_alu_r() argument
347 emit(ctx, and, dst, dst, src); in emit_alu_r()
351 emit(ctx, or, dst, dst, src); in emit_alu_r()
355 emit(ctx, xor, dst, dst, src); in emit_alu_r()
359 emit(ctx, sllv, dst, dst, src); in emit_alu_r()
363 emit(ctx, srlv, dst, dst, src); in emit_alu_r()
367 emit(ctx, srav, dst, dst, src); in emit_alu_r()
371 emit(ctx, addu, dst, dst, src); in emit_alu_r()
375 emit(ctx, subu, dst, dst, src); in emit_alu_r()
380 emit(ctx, mul, dst, dst, src); in emit_alu_r()
382 emit(ctx, multu, dst, src); in emit_alu_r()
383 emit(ctx, mflo, dst); in emit_alu_r()
389 emit(ctx, divu_r6, dst, dst, src); in emit_alu_r()
391 emit(ctx, divu, dst, src); in emit_alu_r()
392 emit(ctx, mflo, dst); in emit_alu_r()
398 emit(ctx, modu, dst, dst, src); in emit_alu_r()
400 emit(ctx, divu, dst, src); in emit_alu_r()
401 emit(ctx, mfhi, dst); in emit_alu_r()
405 clobber_reg(ctx, dst); in emit_alu_r()
409 void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) in emit_atomic_r() argument
411 LLSC_sync(ctx); in emit_atomic_r()
412 emit(ctx, ll, MIPS_R_T9, off, dst); in emit_atomic_r()
416 emit(ctx, addu, MIPS_R_T8, MIPS_R_T9, src); in emit_atomic_r()
420 emit(ctx, and, MIPS_R_T8, MIPS_R_T9, src); in emit_atomic_r()
424 emit(ctx, or, MIPS_R_T8, MIPS_R_T9, src); in emit_atomic_r()
428 emit(ctx, xor, MIPS_R_T8, MIPS_R_T9, src); in emit_atomic_r()
431 emit(ctx, move, MIPS_R_T8, src); in emit_atomic_r()
434 emit(ctx, sc, MIPS_R_T8, off, dst); in emit_atomic_r()
435 emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset); in emit_atomic_r()
436 emit(ctx, nop); /* Delay slot */ in emit_atomic_r()
439 emit(ctx, move, src, MIPS_R_T9); in emit_atomic_r()
440 clobber_reg(ctx, src); in emit_atomic_r()
445 void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) in emit_cmpxchg_r() argument
447 LLSC_sync(ctx); in emit_cmpxchg_r()
448 emit(ctx, ll, MIPS_R_T9, off, dst); in emit_cmpxchg_r()
449 emit(ctx, bne, MIPS_R_T9, res, 12); in emit_cmpxchg_r()
450 emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ in emit_cmpxchg_r()
451 emit(ctx, sc, MIPS_R_T8, off, dst); in emit_cmpxchg_r()
452 emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset); in emit_cmpxchg_r()
453 emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ in emit_cmpxchg_r()
454 clobber_reg(ctx, res); in emit_cmpxchg_r()
458 void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width) in emit_bswap_r() argument
467 emit(ctx, wsbh, dst, dst); in emit_bswap_r()
468 emit(ctx, rotr, dst, dst, 16); in emit_bswap_r()
470 emit(ctx, sll, tmp, dst, 16); /* tmp = dst << 16 */ in emit_bswap_r()
471 emit(ctx, srl, dst, dst, 16); /* dst = dst >> 16 */ in emit_bswap_r()
472 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ in emit_bswap_r()
474 emit(ctx, lui, msk, 0xff); /* msk = 0x00ff0000 */ in emit_bswap_r()
475 emit(ctx, ori, msk, msk, 0xff); /* msk = msk | 0xff */ in emit_bswap_r()
477 emit(ctx, and, tmp, dst, msk); /* tmp = dst & msk */ in emit_bswap_r()
478 emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ in emit_bswap_r()
479 emit(ctx, srl, dst, dst, 8); /* dst = dst >> 8 */ in emit_bswap_r()
480 emit(ctx, and, dst, dst, msk); /* dst = dst & msk */ in emit_bswap_r()
481 emit(ctx, or, dst, dst, tmp); /* reg = dst | tmp */ in emit_bswap_r()
487 emit(ctx, wsbh, dst, dst); in emit_bswap_r()
488 emit(ctx, andi, dst, dst, 0xffff); in emit_bswap_r()
490 emit(ctx, andi, tmp, dst, 0xff00); /* t = d & 0xff00 */ in emit_bswap_r()
491 emit(ctx, srl, tmp, tmp, 8); /* t = t >> 8 */ in emit_bswap_r()
492 emit(ctx, andi, dst, dst, 0x00ff); /* d = d & 0x00ff */ in emit_bswap_r()
493 emit(ctx, sll, dst, dst, 8); /* d = d << 8 */ in emit_bswap_r()
494 emit(ctx, or, dst, dst, tmp); /* d = d | t */ in emit_bswap_r()
498 clobber_reg(ctx, dst); in emit_bswap_r()
553 static void setup_jmp(struct jit_context *ctx, u8 bpf_op, in setup_jmp() argument
556 u32 *descp = &ctx->descriptors[ctx->bpf_index]; in setup_jmp()
580 int target = ctx->bpf_index + bpf_off + 1; in setup_jmp()
581 int origin = ctx->bpf_index + 1; in setup_jmp()
583 offset = (INDEX(ctx->descriptors[target]) - in setup_jmp()
584 INDEX(ctx->descriptors[origin]) + 1) * sizeof(u32); in setup_jmp()
595 ctx->changes += !CONVERTED(*descp); in setup_jmp()
605 void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, in setup_jmp_i() argument
646 setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); in setup_jmp_i()
650 void setup_jmp_r(struct jit_context *ctx, bool same_reg, in setup_jmp_r() argument
673 setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); in setup_jmp_r()
677 int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off) in finish_jmp() argument
681 emit(ctx, nop); in finish_jmp()
686 if (CONVERTED(ctx->descriptors[ctx->bpf_index])) { in finish_jmp()
687 int target = get_target(ctx, ctx->bpf_index + bpf_off + 1); in finish_jmp()
691 emit(ctx, j, target); in finish_jmp()
692 emit(ctx, nop); in finish_jmp()
698 void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op) in emit_jmp_i() argument
706 emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); in emit_jmp_i()
707 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
711 emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); in emit_jmp_i()
712 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
716 emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); in emit_jmp_i()
717 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
721 emit(ctx, sltiu, MIPS_R_T9, dst, imm); in emit_jmp_i()
722 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
726 emit(ctx, sltiu, MIPS_R_T9, dst, imm); in emit_jmp_i()
727 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
731 emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); in emit_jmp_i()
732 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
736 emit(ctx, slti, MIPS_R_T9, dst, imm + 1); in emit_jmp_i()
737 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
741 emit(ctx, slti, MIPS_R_T9, dst, imm); in emit_jmp_i()
742 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
746 emit(ctx, slti, MIPS_R_T9, dst, imm); in emit_jmp_i()
747 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
751 emit(ctx, slti, MIPS_R_T9, dst, imm + 1); in emit_jmp_i()
752 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
758 void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op) in emit_jmp_r() argument
766 emit(ctx, beq, dst, src, off); in emit_jmp_r()
770 emit(ctx, bne, dst, src, off); in emit_jmp_r()
774 emit(ctx, and, MIPS_R_T9, dst, src); in emit_jmp_r()
775 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
779 emit(ctx, and, MIPS_R_T9, dst, src); in emit_jmp_r()
780 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
784 emit(ctx, sltu, MIPS_R_T9, src, dst); in emit_jmp_r()
785 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
789 emit(ctx, sltu, MIPS_R_T9, dst, src); in emit_jmp_r()
790 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
794 emit(ctx, sltu, MIPS_R_T9, dst, src); in emit_jmp_r()
795 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
799 emit(ctx, sltu, MIPS_R_T9, src, dst); in emit_jmp_r()
800 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
804 emit(ctx, slt, MIPS_R_T9, src, dst); in emit_jmp_r()
805 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
809 emit(ctx, slt, MIPS_R_T9, dst, src); in emit_jmp_r()
810 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
814 emit(ctx, slt, MIPS_R_T9, dst, src); in emit_jmp_r()
815 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
819 emit(ctx, slt, MIPS_R_T9, src, dst); in emit_jmp_r()
820 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
826 int emit_ja(struct jit_context *ctx, s16 off) in emit_ja() argument
828 int target = get_target(ctx, ctx->bpf_index + off + 1); in emit_ja()
832 emit(ctx, j, target); in emit_ja()
833 emit(ctx, nop); in emit_ja()
838 int emit_exit(struct jit_context *ctx) in emit_exit() argument
840 int target = get_target(ctx, ctx->program->len); in emit_exit()
844 emit(ctx, j, target); in emit_exit()
845 emit(ctx, nop); in emit_exit()
850 static int build_body(struct jit_context *ctx) in build_body() argument
852 const struct bpf_prog *prog = ctx->program; in build_body()
855 ctx->stack_used = 0; in build_body()
858 u32 *descp = &ctx->descriptors[i]; in build_body()
861 access_reg(ctx, insn->src_reg); in build_body()
862 access_reg(ctx, insn->dst_reg); in build_body()
864 ctx->bpf_index = i; in build_body()
865 if (ctx->target == NULL) { in build_body()
866 ctx->changes += INDEX(*descp) != ctx->jit_index; in build_body()
868 *descp |= ctx->jit_index; in build_body()
871 ret = build_insn(insn, ctx); in build_body()
877 if (ctx->target == NULL) in build_body()
878 descp[1] = ctx->jit_index; in build_body()
883 ctx->descriptors[prog->len] = ctx->jit_index; in build_body()
888 static void set_convert_flag(struct jit_context *ctx, bool enable) in set_convert_flag() argument
890 const struct bpf_prog *prog = ctx->program; in set_convert_flag()
895 ctx->descriptors[i] = INDEX(ctx->descriptors[i]) | flag; in set_convert_flag()
916 struct jit_context ctx; in bpf_int_jit_compile() local
942 memset(&ctx, 0, sizeof(ctx)); in bpf_int_jit_compile()
943 ctx.program = prog; in bpf_int_jit_compile()
949 ctx.descriptors = kcalloc(prog->len + 1, sizeof(*ctx.descriptors), in bpf_int_jit_compile()
951 if (ctx.descriptors == NULL) in bpf_int_jit_compile()
955 if (build_body(&ctx) < 0) in bpf_int_jit_compile()
967 ctx.jit_index = 0; in bpf_int_jit_compile()
968 build_prologue(&ctx); in bpf_int_jit_compile()
969 tmp_idx = ctx.jit_index; in bpf_int_jit_compile()
973 ctx.jit_index = tmp_idx; in bpf_int_jit_compile()
974 ctx.changes = 0; in bpf_int_jit_compile()
976 set_convert_flag(&ctx, true); in bpf_int_jit_compile()
977 if (build_body(&ctx) < 0) in bpf_int_jit_compile()
979 } while (ctx.changes > 0 && --tries > 0); in bpf_int_jit_compile()
981 if (WARN_ONCE(ctx.changes > 0, "JIT offsets failed to converge")) in bpf_int_jit_compile()
984 build_epilogue(&ctx, MIPS_R_RA); in bpf_int_jit_compile()
987 image_size = sizeof(u32) * ctx.jit_index; in bpf_int_jit_compile()
998 ctx.target = (u32 *)image_ptr; in bpf_int_jit_compile()
999 ctx.jit_index = 0; in bpf_int_jit_compile()
1005 build_prologue(&ctx); in bpf_int_jit_compile()
1006 if (build_body(&ctx) < 0) in bpf_int_jit_compile()
1008 build_epilogue(&ctx, MIPS_R_RA); in bpf_int_jit_compile()
1011 set_convert_flag(&ctx, false); in bpf_int_jit_compile()
1012 bpf_prog_fill_jited_linfo(prog, &ctx.descriptors[1]); in bpf_int_jit_compile()
1018 (unsigned long)&ctx.target[ctx.jit_index]); in bpf_int_jit_compile()
1021 bpf_jit_dump(prog->len, image_size, 2, ctx.target); in bpf_int_jit_compile()
1023 prog->bpf_func = (void *)ctx.target; in bpf_int_jit_compile()
1031 kfree(ctx.descriptors); in bpf_int_jit_compile()