Lines Matching refs:ctx
127 int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) in push_regs() argument
135 emit(ctx, sw, reg, depth, MIPS_R_SP); in push_regs()
137 emit(ctx, sd, reg, depth, MIPS_R_SP); in push_regs()
142 ctx->stack_used = max((int)ctx->stack_used, depth); in push_regs()
150 int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth) in pop_regs() argument
158 emit(ctx, lw, reg, depth, MIPS_R_SP); in pop_regs()
160 emit(ctx, ld, reg, depth, MIPS_R_SP); in pop_regs()
169 int get_target(struct jit_context *ctx, u32 loc) in get_target() argument
171 u32 index = INDEX(ctx->descriptors[loc]); in get_target()
172 unsigned long pc = (unsigned long)&ctx->target[ctx->jit_index]; in get_target()
173 unsigned long addr = (unsigned long)&ctx->target[index]; in get_target()
175 if (!ctx->target) in get_target()
185 int get_offset(const struct jit_context *ctx, int off) in get_offset() argument
187 return (INDEX(ctx->descriptors[ctx->bpf_index + off]) - in get_offset()
188 ctx->jit_index - 1) * sizeof(u32); in get_offset()
192 void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm) in emit_mov_i() argument
195 emit(ctx, addiu, dst, MIPS_R_ZERO, imm); in emit_mov_i()
197 emit(ctx, lui, dst, (s16)((u32)imm >> 16)); in emit_mov_i()
198 emit(ctx, ori, dst, dst, (u16)(imm & 0xffff)); in emit_mov_i()
200 clobber_reg(ctx, dst); in emit_mov_i()
204 void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src) in emit_mov_r() argument
206 emit(ctx, ori, dst, src, 0); in emit_mov_r()
207 clobber_reg(ctx, dst); in emit_mov_r()
294 void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op) in emit_alu_i() argument
299 emit(ctx, subu, dst, MIPS_R_ZERO, dst); in emit_alu_i()
303 emit(ctx, andi, dst, dst, (u16)imm); in emit_alu_i()
307 emit(ctx, ori, dst, dst, (u16)imm); in emit_alu_i()
311 emit(ctx, xori, dst, dst, (u16)imm); in emit_alu_i()
315 emit(ctx, sll, dst, dst, imm); in emit_alu_i()
319 emit(ctx, srl, dst, dst, imm); in emit_alu_i()
323 emit(ctx, sra, dst, dst, imm); in emit_alu_i()
327 emit(ctx, addiu, dst, dst, imm); in emit_alu_i()
331 emit(ctx, addiu, dst, dst, -imm); in emit_alu_i()
334 clobber_reg(ctx, dst); in emit_alu_i()
338 void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op) in emit_alu_r() argument
343 emit(ctx, and, dst, dst, src); in emit_alu_r()
347 emit(ctx, or, dst, dst, src); in emit_alu_r()
351 emit(ctx, xor, dst, dst, src); in emit_alu_r()
355 emit(ctx, sllv, dst, dst, src); in emit_alu_r()
359 emit(ctx, srlv, dst, dst, src); in emit_alu_r()
363 emit(ctx, srav, dst, dst, src); in emit_alu_r()
367 emit(ctx, addu, dst, dst, src); in emit_alu_r()
371 emit(ctx, subu, dst, dst, src); in emit_alu_r()
376 emit(ctx, mul, dst, dst, src); in emit_alu_r()
378 emit(ctx, multu, dst, src); in emit_alu_r()
379 emit(ctx, mflo, dst); in emit_alu_r()
385 emit(ctx, divu_r6, dst, dst, src); in emit_alu_r()
387 emit(ctx, divu, dst, src); in emit_alu_r()
388 emit(ctx, mflo, dst); in emit_alu_r()
394 emit(ctx, modu, dst, dst, src); in emit_alu_r()
396 emit(ctx, divu, dst, src); in emit_alu_r()
397 emit(ctx, mfhi, dst); in emit_alu_r()
401 clobber_reg(ctx, dst); in emit_alu_r()
405 void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code) in emit_atomic_r() argument
407 LLSC_sync(ctx); in emit_atomic_r()
408 emit(ctx, ll, MIPS_R_T9, off, dst); in emit_atomic_r()
412 emit(ctx, addu, MIPS_R_T8, MIPS_R_T9, src); in emit_atomic_r()
416 emit(ctx, and, MIPS_R_T8, MIPS_R_T9, src); in emit_atomic_r()
420 emit(ctx, or, MIPS_R_T8, MIPS_R_T9, src); in emit_atomic_r()
424 emit(ctx, xor, MIPS_R_T8, MIPS_R_T9, src); in emit_atomic_r()
427 emit(ctx, move, MIPS_R_T8, src); in emit_atomic_r()
430 emit(ctx, sc, MIPS_R_T8, off, dst); in emit_atomic_r()
431 emit(ctx, LLSC_beqz, MIPS_R_T8, -16 - LLSC_offset); in emit_atomic_r()
432 emit(ctx, nop); /* Delay slot */ in emit_atomic_r()
435 emit(ctx, move, src, MIPS_R_T9); in emit_atomic_r()
436 clobber_reg(ctx, src); in emit_atomic_r()
441 void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off) in emit_cmpxchg_r() argument
443 LLSC_sync(ctx); in emit_cmpxchg_r()
444 emit(ctx, ll, MIPS_R_T9, off, dst); in emit_cmpxchg_r()
445 emit(ctx, bne, MIPS_R_T9, res, 12); in emit_cmpxchg_r()
446 emit(ctx, move, MIPS_R_T8, src); /* Delay slot */ in emit_cmpxchg_r()
447 emit(ctx, sc, MIPS_R_T8, off, dst); in emit_cmpxchg_r()
448 emit(ctx, LLSC_beqz, MIPS_R_T8, -20 - LLSC_offset); in emit_cmpxchg_r()
449 emit(ctx, move, res, MIPS_R_T9); /* Delay slot */ in emit_cmpxchg_r()
450 clobber_reg(ctx, res); in emit_cmpxchg_r()
454 void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width) in emit_bswap_r() argument
463 emit(ctx, wsbh, dst, dst); in emit_bswap_r()
464 emit(ctx, rotr, dst, dst, 16); in emit_bswap_r()
466 emit(ctx, sll, tmp, dst, 16); /* tmp = dst << 16 */ in emit_bswap_r()
467 emit(ctx, srl, dst, dst, 16); /* dst = dst >> 16 */ in emit_bswap_r()
468 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ in emit_bswap_r()
470 emit(ctx, lui, msk, 0xff); /* msk = 0x00ff0000 */ in emit_bswap_r()
471 emit(ctx, ori, msk, msk, 0xff); /* msk = msk | 0xff */ in emit_bswap_r()
473 emit(ctx, and, tmp, dst, msk); /* tmp = dst & msk */ in emit_bswap_r()
474 emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ in emit_bswap_r()
475 emit(ctx, srl, dst, dst, 8); /* dst = dst >> 8 */ in emit_bswap_r()
476 emit(ctx, and, dst, dst, msk); /* dst = dst & msk */ in emit_bswap_r()
477 emit(ctx, or, dst, dst, tmp); /* reg = dst | tmp */ in emit_bswap_r()
483 emit(ctx, wsbh, dst, dst); in emit_bswap_r()
484 emit(ctx, andi, dst, dst, 0xffff); in emit_bswap_r()
486 emit(ctx, andi, tmp, dst, 0xff00); /* t = d & 0xff00 */ in emit_bswap_r()
487 emit(ctx, srl, tmp, tmp, 8); /* t = t >> 8 */ in emit_bswap_r()
488 emit(ctx, andi, dst, dst, 0x00ff); /* d = d & 0x00ff */ in emit_bswap_r()
489 emit(ctx, sll, dst, dst, 8); /* d = d << 8 */ in emit_bswap_r()
490 emit(ctx, or, dst, dst, tmp); /* d = d | t */ in emit_bswap_r()
494 clobber_reg(ctx, dst); in emit_bswap_r()
549 static void setup_jmp(struct jit_context *ctx, u8 bpf_op, in setup_jmp() argument
552 u32 *descp = &ctx->descriptors[ctx->bpf_index]; in setup_jmp()
576 int target = ctx->bpf_index + bpf_off + 1; in setup_jmp()
577 int origin = ctx->bpf_index + 1; in setup_jmp()
579 offset = (INDEX(ctx->descriptors[target]) - in setup_jmp()
580 INDEX(ctx->descriptors[origin]) + 1) * sizeof(u32); in setup_jmp()
591 ctx->changes += !CONVERTED(*descp); in setup_jmp()
601 void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, in setup_jmp_i() argument
642 setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); in setup_jmp_i()
646 void setup_jmp_r(struct jit_context *ctx, bool same_reg, in setup_jmp_r() argument
669 setup_jmp(ctx, bpf_op, bpf_off, jit_op, jit_off); in setup_jmp_r()
673 int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off) in finish_jmp() argument
677 emit(ctx, nop); in finish_jmp()
682 if (CONVERTED(ctx->descriptors[ctx->bpf_index])) { in finish_jmp()
683 int target = get_target(ctx, ctx->bpf_index + bpf_off + 1); in finish_jmp()
687 emit(ctx, j, target); in finish_jmp()
688 emit(ctx, nop); in finish_jmp()
694 void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op) in emit_jmp_i() argument
702 emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); in emit_jmp_i()
703 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
707 emit(ctx, andi, MIPS_R_T9, dst, (u16)imm); in emit_jmp_i()
708 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
712 emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); in emit_jmp_i()
713 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
717 emit(ctx, sltiu, MIPS_R_T9, dst, imm); in emit_jmp_i()
718 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
722 emit(ctx, sltiu, MIPS_R_T9, dst, imm); in emit_jmp_i()
723 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
727 emit(ctx, sltiu, MIPS_R_T9, dst, imm + 1); in emit_jmp_i()
728 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
732 emit(ctx, slti, MIPS_R_T9, dst, imm + 1); in emit_jmp_i()
733 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
737 emit(ctx, slti, MIPS_R_T9, dst, imm); in emit_jmp_i()
738 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_i()
742 emit(ctx, slti, MIPS_R_T9, dst, imm); in emit_jmp_i()
743 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
747 emit(ctx, slti, MIPS_R_T9, dst, imm + 1); in emit_jmp_i()
748 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_i()
754 void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op) in emit_jmp_r() argument
762 emit(ctx, beq, dst, src, off); in emit_jmp_r()
766 emit(ctx, bne, dst, src, off); in emit_jmp_r()
770 emit(ctx, and, MIPS_R_T9, dst, src); in emit_jmp_r()
771 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
775 emit(ctx, and, MIPS_R_T9, dst, src); in emit_jmp_r()
776 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
780 emit(ctx, sltu, MIPS_R_T9, src, dst); in emit_jmp_r()
781 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
785 emit(ctx, sltu, MIPS_R_T9, dst, src); in emit_jmp_r()
786 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
790 emit(ctx, sltu, MIPS_R_T9, dst, src); in emit_jmp_r()
791 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
795 emit(ctx, sltu, MIPS_R_T9, src, dst); in emit_jmp_r()
796 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
800 emit(ctx, slt, MIPS_R_T9, src, dst); in emit_jmp_r()
801 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
805 emit(ctx, slt, MIPS_R_T9, dst, src); in emit_jmp_r()
806 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
810 emit(ctx, slt, MIPS_R_T9, dst, src); in emit_jmp_r()
811 emit(ctx, bnez, MIPS_R_T9, off); in emit_jmp_r()
815 emit(ctx, slt, MIPS_R_T9, src, dst); in emit_jmp_r()
816 emit(ctx, beqz, MIPS_R_T9, off); in emit_jmp_r()
822 int emit_ja(struct jit_context *ctx, s16 off) in emit_ja() argument
824 int target = get_target(ctx, ctx->bpf_index + off + 1); in emit_ja()
828 emit(ctx, j, target); in emit_ja()
829 emit(ctx, nop); in emit_ja()
834 int emit_exit(struct jit_context *ctx) in emit_exit() argument
836 int target = get_target(ctx, ctx->program->len); in emit_exit()
840 emit(ctx, j, target); in emit_exit()
841 emit(ctx, nop); in emit_exit()
846 static int build_body(struct jit_context *ctx) in build_body() argument
848 const struct bpf_prog *prog = ctx->program; in build_body()
851 ctx->stack_used = 0; in build_body()
854 u32 *descp = &ctx->descriptors[i]; in build_body()
857 access_reg(ctx, insn->src_reg); in build_body()
858 access_reg(ctx, insn->dst_reg); in build_body()
860 ctx->bpf_index = i; in build_body()
861 if (ctx->target == NULL) { in build_body()
862 ctx->changes += INDEX(*descp) != ctx->jit_index; in build_body()
864 *descp |= ctx->jit_index; in build_body()
867 ret = build_insn(insn, ctx); in build_body()
873 if (ctx->target == NULL) in build_body()
874 descp[1] = ctx->jit_index; in build_body()
879 ctx->descriptors[prog->len] = ctx->jit_index; in build_body()
884 static void set_convert_flag(struct jit_context *ctx, bool enable) in set_convert_flag() argument
886 const struct bpf_prog *prog = ctx->program; in set_convert_flag()
891 ctx->descriptors[i] = INDEX(ctx->descriptors[i]) | flag; in set_convert_flag()
912 struct jit_context ctx; in bpf_int_jit_compile() local
938 memset(&ctx, 0, sizeof(ctx)); in bpf_int_jit_compile()
939 ctx.program = prog; in bpf_int_jit_compile()
945 ctx.descriptors = kcalloc(prog->len + 1, sizeof(*ctx.descriptors), in bpf_int_jit_compile()
947 if (ctx.descriptors == NULL) in bpf_int_jit_compile()
951 if (build_body(&ctx) < 0) in bpf_int_jit_compile()
963 ctx.jit_index = 0; in bpf_int_jit_compile()
964 build_prologue(&ctx); in bpf_int_jit_compile()
965 tmp_idx = ctx.jit_index; in bpf_int_jit_compile()
969 ctx.jit_index = tmp_idx; in bpf_int_jit_compile()
970 ctx.changes = 0; in bpf_int_jit_compile()
972 set_convert_flag(&ctx, true); in bpf_int_jit_compile()
973 if (build_body(&ctx) < 0) in bpf_int_jit_compile()
975 } while (ctx.changes > 0 && --tries > 0); in bpf_int_jit_compile()
977 if (WARN_ONCE(ctx.changes > 0, "JIT offsets failed to converge")) in bpf_int_jit_compile()
980 build_epilogue(&ctx, MIPS_R_RA); in bpf_int_jit_compile()
983 image_size = sizeof(u32) * ctx.jit_index; in bpf_int_jit_compile()
994 ctx.target = (u32 *)image_ptr; in bpf_int_jit_compile()
995 ctx.jit_index = 0; in bpf_int_jit_compile()
1001 build_prologue(&ctx); in bpf_int_jit_compile()
1002 if (build_body(&ctx) < 0) in bpf_int_jit_compile()
1004 build_epilogue(&ctx, MIPS_R_RA); in bpf_int_jit_compile()
1007 set_convert_flag(&ctx, false); in bpf_int_jit_compile()
1008 bpf_prog_fill_jited_linfo(prog, &ctx.descriptors[1]); in bpf_int_jit_compile()
1013 (unsigned long)&ctx.target[ctx.jit_index]); in bpf_int_jit_compile()
1016 bpf_jit_dump(prog->len, image_size, 2, ctx.target); in bpf_int_jit_compile()
1018 prog->bpf_func = (void *)ctx.target; in bpf_int_jit_compile()
1026 kfree(ctx.descriptors); in bpf_int_jit_compile()