Lines Matching refs:ctx
167 static void clobber_reg64(struct jit_context *ctx, const u8 reg[]) in clobber_reg64() argument
169 clobber_reg(ctx, reg[0]); in clobber_reg64()
170 clobber_reg(ctx, reg[1]); in clobber_reg64()
174 static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm) in emit_mov_se_i64() argument
176 emit_mov_i(ctx, lo(dst), imm); in emit_mov_se_i64()
178 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); in emit_mov_se_i64()
180 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_mov_se_i64()
181 clobber_reg64(ctx, dst); in emit_mov_se_i64()
185 static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) in emit_zext_ver() argument
187 if (!ctx->program->aux->verifier_zext) { in emit_zext_ver()
188 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_zext_ver()
189 clobber_reg(ctx, hi(dst)); in emit_zext_ver()
194 static void emit_load_delay(struct jit_context *ctx) in emit_load_delay() argument
197 emit(ctx, nop); in emit_load_delay()
201 static void emit_alu_i64(struct jit_context *ctx, in emit_alu_i64() argument
223 emit_mov_i(ctx, src, imm); in emit_alu_i64()
228 emit(ctx, addu, lo(dst), lo(dst), src); in emit_alu_i64()
229 emit(ctx, sltu, MIPS_R_T9, lo(dst), src); in emit_alu_i64()
230 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); in emit_alu_i64()
232 emit(ctx, addiu, hi(dst), hi(dst), -1); in emit_alu_i64()
236 emit(ctx, sltu, MIPS_R_T9, lo(dst), src); in emit_alu_i64()
237 emit(ctx, subu, lo(dst), lo(dst), src); in emit_alu_i64()
238 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); in emit_alu_i64()
240 emit(ctx, addiu, hi(dst), hi(dst), 1); in emit_alu_i64()
244 emit(ctx, or, lo(dst), lo(dst), src); in emit_alu_i64()
246 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); in emit_alu_i64()
250 emit(ctx, and, lo(dst), lo(dst), src); in emit_alu_i64()
252 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_alu_i64()
256 emit(ctx, xor, lo(dst), lo(dst), src); in emit_alu_i64()
258 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); in emit_alu_i64()
259 emit(ctx, addiu, hi(dst), hi(dst), -1); in emit_alu_i64()
263 clobber_reg64(ctx, dst); in emit_alu_i64()
267 static void emit_alu_r64(struct jit_context *ctx, in emit_alu_r64() argument
274 emit(ctx, srl, MIPS_R_T9, lo(dst), 31); in emit_alu_r64()
275 emit(ctx, addu, lo(dst), lo(dst), lo(dst)); in emit_alu_r64()
277 emit(ctx, addu, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
278 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); in emit_alu_r64()
280 emit(ctx, addu, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
281 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); in emit_alu_r64()
285 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); in emit_alu_r64()
286 emit(ctx, subu, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
287 emit(ctx, subu, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
288 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); in emit_alu_r64()
292 emit(ctx, or, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
293 emit(ctx, or, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
297 emit(ctx, and, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
298 emit(ctx, and, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
302 emit(ctx, xor, lo(dst), lo(dst), lo(src)); in emit_alu_r64()
303 emit(ctx, xor, hi(dst), hi(dst), hi(src)); in emit_alu_r64()
306 clobber_reg64(ctx, dst); in emit_alu_r64()
310 static void emit_neg_i64(struct jit_context *ctx, const u8 dst[]) in emit_neg_i64() argument
312 emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst)); in emit_neg_i64()
313 emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst)); in emit_neg_i64()
314 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); in emit_neg_i64()
315 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); in emit_neg_i64()
317 clobber_reg64(ctx, dst); in emit_neg_i64()
321 static void emit_shift_i64(struct jit_context *ctx, in emit_shift_i64() argument
328 emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm); in emit_shift_i64()
329 emit(ctx, sll, lo(dst), lo(dst), imm); in emit_shift_i64()
330 emit(ctx, sll, hi(dst), hi(dst), imm); in emit_shift_i64()
331 emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9); in emit_shift_i64()
333 emit(ctx, sll, hi(dst), lo(dst), imm - 32); in emit_shift_i64()
334 emit(ctx, move, lo(dst), MIPS_R_ZERO); in emit_shift_i64()
340 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); in emit_shift_i64()
341 emit(ctx, srl, lo(dst), lo(dst), imm); in emit_shift_i64()
342 emit(ctx, srl, hi(dst), hi(dst), imm); in emit_shift_i64()
343 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); in emit_shift_i64()
345 emit(ctx, srl, lo(dst), hi(dst), imm - 32); in emit_shift_i64()
346 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_shift_i64()
352 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); in emit_shift_i64()
353 emit(ctx, srl, lo(dst), lo(dst), imm); in emit_shift_i64()
354 emit(ctx, sra, hi(dst), hi(dst), imm); in emit_shift_i64()
355 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); in emit_shift_i64()
357 emit(ctx, sra, lo(dst), hi(dst), imm - 32); in emit_shift_i64()
358 emit(ctx, sra, hi(dst), hi(dst), 31); in emit_shift_i64()
362 clobber_reg64(ctx, dst); in emit_shift_i64()
366 static void emit_shift_r64(struct jit_context *ctx, in emit_shift_r64() argument
372 emit(ctx, andi, t1, src, 32); /* t1 = src & 32 */ in emit_shift_r64()
373 emit(ctx, beqz, t1, 16); /* PC += 16 if t1 == 0 */ in emit_shift_r64()
374 emit(ctx, nor, t2, src, MIPS_R_ZERO); /* t2 = ~src (delay slot) */ in emit_shift_r64()
380 emit(ctx, sllv, hi(dst), lo(dst), src); /* dh = dl << src */ in emit_shift_r64()
381 emit(ctx, move, lo(dst), MIPS_R_ZERO); /* dl = 0 */ in emit_shift_r64()
382 emit(ctx, b, 20); /* PC += 20 */ in emit_shift_r64()
384 emit(ctx, srl, t1, lo(dst), 1); /* t1 = dl >> 1 */ in emit_shift_r64()
385 emit(ctx, srlv, t1, t1, t2); /* t1 = t1 >> t2 */ in emit_shift_r64()
386 emit(ctx, sllv, lo(dst), lo(dst), src); /* dl = dl << src */ in emit_shift_r64()
387 emit(ctx, sllv, hi(dst), hi(dst), src); /* dh = dh << src */ in emit_shift_r64()
388 emit(ctx, or, hi(dst), hi(dst), t1); /* dh = dh | t1 */ in emit_shift_r64()
393 emit(ctx, srlv, lo(dst), hi(dst), src); /* dl = dh >> src */ in emit_shift_r64()
394 emit(ctx, move, hi(dst), MIPS_R_ZERO); /* dh = 0 */ in emit_shift_r64()
395 emit(ctx, b, 20); /* PC += 20 */ in emit_shift_r64()
397 emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ in emit_shift_r64()
398 emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ in emit_shift_r64()
399 emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >> src */ in emit_shift_r64()
400 emit(ctx, srlv, hi(dst), hi(dst), src); /* dh = dh >> src */ in emit_shift_r64()
401 emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ in emit_shift_r64()
406 emit(ctx, srav, lo(dst), hi(dst), src); /* dl = dh >>a src */ in emit_shift_r64()
407 emit(ctx, sra, hi(dst), hi(dst), 31); /* dh = dh >>a 31 */ in emit_shift_r64()
408 emit(ctx, b, 20); /* PC += 20 */ in emit_shift_r64()
410 emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ in emit_shift_r64()
411 emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ in emit_shift_r64()
412 emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >>a src */ in emit_shift_r64()
413 emit(ctx, srav, hi(dst), hi(dst), src); /* dh = dh >> src */ in emit_shift_r64()
414 emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ in emit_shift_r64()
419 clobber_reg64(ctx, dst); in emit_shift_r64()
423 static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm) in emit_mul_i64() argument
434 emit_neg_i64(ctx, dst); in emit_mul_i64()
437 emit_mov_r(ctx, lo(dst), MIPS_R_ZERO); in emit_mul_i64()
438 emit_mov_r(ctx, hi(dst), MIPS_R_ZERO); in emit_mul_i64()
443 emit_mov_i(ctx, src, imm); in emit_mul_i64()
445 emit(ctx, mul, hi(dst), hi(dst), src); in emit_mul_i64()
447 emit(ctx, multu, hi(dst), src); in emit_mul_i64()
448 emit(ctx, mflo, hi(dst)); in emit_mul_i64()
453 emit(ctx, subu, hi(dst), hi(dst), lo(dst)); in emit_mul_i64()
458 emit(ctx, muhu, tmp, lo(dst), src); in emit_mul_i64()
459 emit(ctx, mulu, lo(dst), lo(dst), src); in emit_mul_i64()
461 emit(ctx, multu, lo(dst), src); in emit_mul_i64()
462 emit(ctx, mflo, lo(dst)); in emit_mul_i64()
463 emit(ctx, mfhi, tmp); in emit_mul_i64()
467 emit(ctx, addu, hi(dst), hi(dst), tmp); in emit_mul_i64()
468 clobber_reg64(ctx, dst); in emit_mul_i64()
474 static void emit_mul_r64(struct jit_context *ctx, in emit_mul_r64() argument
482 emit(ctx, mul, acc, hi(dst), lo(src)); in emit_mul_r64()
484 emit(ctx, multu, hi(dst), lo(src)); in emit_mul_r64()
485 emit(ctx, mflo, acc); in emit_mul_r64()
490 emit(ctx, mul, tmp, lo(dst), hi(src)); in emit_mul_r64()
492 emit(ctx, multu, lo(dst), hi(src)); in emit_mul_r64()
493 emit(ctx, mflo, tmp); in emit_mul_r64()
497 emit(ctx, addu, acc, acc, tmp); in emit_mul_r64()
502 emit(ctx, muhu, tmp, lo(dst), lo(src)); in emit_mul_r64()
503 emit(ctx, mulu, lo(dst), lo(dst), lo(src)); in emit_mul_r64()
505 emit(ctx, multu, lo(dst), lo(src)); in emit_mul_r64()
506 emit(ctx, mflo, lo(dst)); in emit_mul_r64()
507 emit(ctx, mfhi, tmp); in emit_mul_r64()
511 emit(ctx, addu, hi(dst), acc, tmp); in emit_mul_r64()
512 clobber_reg64(ctx, dst); in emit_mul_r64()
525 static void emit_divmod_r64(struct jit_context *ctx, in emit_divmod_r64() argument
535 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_divmod_r64()
540 emit(ctx, move, MIPS_R_T9, src[k]); in emit_divmod_r64()
541 emit(ctx, move, r1[k], dst[k]); in emit_divmod_r64()
542 emit(ctx, move, r2[k], MIPS_R_T9); in emit_divmod_r64()
556 emit_mov_i(ctx, MIPS_R_T9, addr); in emit_divmod_r64()
557 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); in emit_divmod_r64()
558 emit(ctx, nop); /* Delay slot */ in emit_divmod_r64()
561 emit(ctx, move, dst[0], r0[0]); in emit_divmod_r64()
562 emit(ctx, move, dst[1], r0[1]); in emit_divmod_r64()
566 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_divmod_r64()
568 emit_load_delay(ctx); in emit_divmod_r64()
570 clobber_reg64(ctx, dst); in emit_divmod_r64()
571 clobber_reg(ctx, MIPS_R_V0); in emit_divmod_r64()
572 clobber_reg(ctx, MIPS_R_V1); in emit_divmod_r64()
573 clobber_reg(ctx, MIPS_R_RA); in emit_divmod_r64()
577 static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask) in emit_swap8_r() argument
581 emit(ctx, and, tmp, src, mask); /* tmp = src & 0x00ff00ff */ in emit_swap8_r()
582 emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ in emit_swap8_r()
583 emit(ctx, srl, dst, src, 8); /* dst = src >> 8 */ in emit_swap8_r()
584 emit(ctx, and, dst, dst, mask); /* dst = dst & 0x00ff00ff */ in emit_swap8_r()
585 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ in emit_swap8_r()
589 static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src) in emit_swap16_r() argument
593 emit(ctx, sll, tmp, src, 16); /* tmp = src << 16 */ in emit_swap16_r()
594 emit(ctx, srl, dst, src, 16); /* dst = src >> 16 */ in emit_swap16_r()
595 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ in emit_swap16_r()
599 static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width) in emit_bswap_r64() argument
607 emit(ctx, rotr, tmp, hi(dst), 16); in emit_bswap_r64()
608 emit(ctx, rotr, hi(dst), lo(dst), 16); in emit_bswap_r64()
609 emit(ctx, wsbh, lo(dst), tmp); in emit_bswap_r64()
610 emit(ctx, wsbh, hi(dst), hi(dst)); in emit_bswap_r64()
612 emit_swap16_r(ctx, tmp, lo(dst)); in emit_bswap_r64()
613 emit_swap16_r(ctx, lo(dst), hi(dst)); in emit_bswap_r64()
614 emit(ctx, move, hi(dst), tmp); in emit_bswap_r64()
616 emit(ctx, lui, tmp, 0xff); /* tmp = 0x00ff0000 */ in emit_bswap_r64()
617 emit(ctx, ori, tmp, tmp, 0xff); /* tmp = 0x00ff00ff */ in emit_bswap_r64()
618 emit_swap8_r(ctx, lo(dst), lo(dst), tmp); in emit_bswap_r64()
619 emit_swap8_r(ctx, hi(dst), hi(dst), tmp); in emit_bswap_r64()
626 emit_bswap_r(ctx, lo(dst), width); in emit_bswap_r64()
627 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_bswap_r64()
630 clobber_reg64(ctx, dst); in emit_bswap_r64()
634 static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width) in emit_trunc_r64() argument
641 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_trunc_r64()
642 clobber_reg(ctx, hi(dst)); in emit_trunc_r64()
646 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_trunc_r64()
647 emit(ctx, andi, lo(dst), lo(dst), 0xffff); in emit_trunc_r64()
648 clobber_reg64(ctx, dst); in emit_trunc_r64()
654 static void emit_ldx(struct jit_context *ctx, in emit_ldx() argument
660 emit(ctx, lbu, lo(dst), off, src); in emit_ldx()
661 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_ldx()
665 emit(ctx, lhu, lo(dst), off, src); in emit_ldx()
666 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_ldx()
670 emit(ctx, lw, lo(dst), off, src); in emit_ldx()
671 emit(ctx, move, hi(dst), MIPS_R_ZERO); in emit_ldx()
676 emit(ctx, lw, dst[0], off + 4, src); in emit_ldx()
677 emit(ctx, lw, dst[1], off, src); in emit_ldx()
679 emit(ctx, lw, dst[1], off, src); in emit_ldx()
680 emit(ctx, lw, dst[0], off + 4, src); in emit_ldx()
682 emit_load_delay(ctx); in emit_ldx()
685 clobber_reg64(ctx, dst); in emit_ldx()
689 static void emit_stx(struct jit_context *ctx, in emit_stx() argument
695 emit(ctx, sb, lo(src), off, dst); in emit_stx()
699 emit(ctx, sh, lo(src), off, dst); in emit_stx()
703 emit(ctx, sw, lo(src), off, dst); in emit_stx()
707 emit(ctx, sw, src[1], off, dst); in emit_stx()
708 emit(ctx, sw, src[0], off + 4, dst); in emit_stx()
714 static void emit_atomic_r32(struct jit_context *ctx, in emit_atomic_r32() argument
721 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_atomic_r32()
727 emit(ctx, move, MIPS_R_T9, dst); in emit_atomic_r32()
729 emit(ctx, move, MIPS_R_A1, src); in emit_atomic_r32()
730 emit(ctx, addiu, MIPS_R_A0, MIPS_R_T9, off); in emit_atomic_r32()
732 emit(ctx, move, MIPS_R_A0, src); in emit_atomic_r32()
733 emit(ctx, addiu, MIPS_R_A1, MIPS_R_T9, off); in emit_atomic_r32()
772 emit_mov_i(ctx, MIPS_R_T9, addr); in emit_atomic_r32()
773 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); in emit_atomic_r32()
774 emit(ctx, nop); /* Delay slot */ in emit_atomic_r32()
778 emit(ctx, move, src, MIPS_R_V0); in emit_atomic_r32()
780 clobber_reg(ctx, src); in emit_atomic_r32()
784 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_atomic_r32()
786 emit_load_delay(ctx); in emit_atomic_r32()
787 clobber_reg(ctx, MIPS_R_RA); in emit_atomic_r32()
797 static void emit_atomic_r64(struct jit_context *ctx, in emit_atomic_r64() argument
806 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_atomic_r64()
812 emit(ctx, move, MIPS_R_T9, dst); in emit_atomic_r64()
813 emit(ctx, move, r1[0], src[0]); in emit_atomic_r64()
814 emit(ctx, move, r1[1], src[1]); in emit_atomic_r64()
815 emit(ctx, addiu, MIPS_R_A2, MIPS_R_T9, off); in emit_atomic_r64()
853 emit_mov_i(ctx, MIPS_R_T9, addr); in emit_atomic_r64()
854 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); in emit_atomic_r64()
855 emit(ctx, nop); /* Delay slot */ in emit_atomic_r64()
859 emit(ctx, move, lo(src), lo(r0)); in emit_atomic_r64()
860 emit(ctx, move, hi(src), hi(r0)); in emit_atomic_r64()
862 clobber_reg64(ctx, src); in emit_atomic_r64()
866 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_atomic_r64()
868 emit_load_delay(ctx); in emit_atomic_r64()
869 clobber_reg(ctx, MIPS_R_RA); in emit_atomic_r64()
873 static void emit_cmpxchg_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off) in emit_cmpxchg_r32() argument
878 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_cmpxchg_r32()
885 emit(ctx, addiu, MIPS_R_T9, dst, off); in emit_cmpxchg_r32()
886 emit(ctx, move, MIPS_R_T8, src); in emit_cmpxchg_r32()
887 emit(ctx, move, MIPS_R_A1, lo(r0)); in emit_cmpxchg_r32()
888 emit(ctx, move, MIPS_R_A0, MIPS_R_T9); in emit_cmpxchg_r32()
889 emit(ctx, move, MIPS_R_A2, MIPS_R_T8); in emit_cmpxchg_r32()
892 emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic_cmpxchg); in emit_cmpxchg_r32()
893 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); in emit_cmpxchg_r32()
894 emit(ctx, nop); /* Delay slot */ in emit_cmpxchg_r32()
897 emit(ctx, move, lo(r0), MIPS_R_V0); in emit_cmpxchg_r32()
900 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_cmpxchg_r32()
902 emit_load_delay(ctx); in emit_cmpxchg_r32()
903 clobber_reg(ctx, MIPS_R_V0); in emit_cmpxchg_r32()
904 clobber_reg(ctx, MIPS_R_V1); in emit_cmpxchg_r32()
905 clobber_reg(ctx, MIPS_R_RA); in emit_cmpxchg_r32()
909 static void emit_cmpxchg_r64(struct jit_context *ctx, in emit_cmpxchg_r64() argument
916 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_cmpxchg_r64()
923 push_regs(ctx, BIT(src[0]) | BIT(src[1]), 0, JIT_RESERVED_STACK); in emit_cmpxchg_r64()
924 emit(ctx, addiu, MIPS_R_T9, dst, off); in emit_cmpxchg_r64()
925 emit(ctx, move, r2[0], r0[0]); in emit_cmpxchg_r64()
926 emit(ctx, move, r2[1], r0[1]); in emit_cmpxchg_r64()
927 emit(ctx, move, MIPS_R_A0, MIPS_R_T9); in emit_cmpxchg_r64()
930 emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic64_cmpxchg); in emit_cmpxchg_r64()
931 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); in emit_cmpxchg_r64()
932 emit(ctx, nop); /* Delay slot */ in emit_cmpxchg_r64()
935 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, in emit_cmpxchg_r64()
937 emit_load_delay(ctx); in emit_cmpxchg_r64()
938 clobber_reg(ctx, MIPS_R_V0); in emit_cmpxchg_r64()
939 clobber_reg(ctx, MIPS_R_V1); in emit_cmpxchg_r64()
940 clobber_reg(ctx, MIPS_R_RA); in emit_cmpxchg_r64()
947 static void emit_movz_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) in emit_movz_r() argument
950 emit(ctx, movz, rd, rs, rt); /* rd = rt ? rd : rs */ in emit_movz_r()
953 emit(ctx, seleqz, rs, rs, rt); /* rs = 0 if rt == 0 */ in emit_movz_r()
954 emit(ctx, selnez, rd, rd, rt); /* rd = 0 if rt != 0 */ in emit_movz_r()
956 emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ in emit_movz_r()
958 emit(ctx, bnez, rt, 8); /* PC += 8 if rd != 0 */ in emit_movz_r()
959 emit(ctx, nop); /* +0: delay slot */ in emit_movz_r()
960 emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ in emit_movz_r()
962 clobber_reg(ctx, rd); in emit_movz_r()
963 clobber_reg(ctx, rs); in emit_movz_r()
970 static void emit_movn_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) in emit_movn_r() argument
973 emit(ctx, movn, rd, rs, rt); /* rd = rt ? rs : rd */ in emit_movn_r()
976 emit(ctx, selnez, rs, rs, rt); /* rs = 0 if rt == 0 */ in emit_movn_r()
977 emit(ctx, seleqz, rd, rd, rt); /* rd = 0 if rt != 0 */ in emit_movn_r()
979 emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ in emit_movn_r()
981 emit(ctx, beqz, rt, 8); /* PC += 8 if rd == 0 */ in emit_movn_r()
982 emit(ctx, nop); /* +0: delay slot */ in emit_movn_r()
983 emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ in emit_movn_r()
985 clobber_reg(ctx, rd); in emit_movn_r()
986 clobber_reg(ctx, rs); in emit_movn_r()
990 static void emit_sltiu_r64(struct jit_context *ctx, u8 rd, in emit_sltiu_r64() argument
996 emit_mov_i(ctx, rd, imm); /* rd = imm */ in emit_sltiu_r64()
997 emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ in emit_sltiu_r64()
998 emit(ctx, sltiu, tmp, hi(rs), -1); /* tmp = rsh < ~0U */ in emit_sltiu_r64()
999 emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ in emit_sltiu_r64()
1002 emit_mov_i(ctx, rd, (s32)imm); /* rd = imm */ in emit_sltiu_r64()
1003 emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ in emit_sltiu_r64()
1005 emit(ctx, sltiu, rd, lo(rs), imm); /* rd = rsl < imm */ in emit_sltiu_r64()
1007 emit_movn_r(ctx, rd, MIPS_R_ZERO, hi(rs)); /* rd = 0 if rsh */ in emit_sltiu_r64()
1012 static void emit_sltu_r64(struct jit_context *ctx, u8 rd, in emit_sltu_r64() argument
1017 emit(ctx, sltu, rd, lo(rs), lo(rt)); /* rd = rsl < rtl */ in emit_sltu_r64()
1018 emit(ctx, subu, tmp, hi(rs), hi(rt)); /* tmp = rsh - rth */ in emit_sltu_r64()
1019 emit_movn_r(ctx, rd, MIPS_R_ZERO, tmp); /* rd = 0 if tmp != 0 */ in emit_sltu_r64()
1020 emit(ctx, sltu, tmp, hi(rs), hi(rt)); /* tmp = rsh < rth */ in emit_sltu_r64()
1021 emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ in emit_sltu_r64()
1025 static void emit_slti_r64(struct jit_context *ctx, u8 rd, in emit_slti_r64() argument
1036 emit_mov_i(ctx, rd, (s32)imm); in emit_slti_r64()
1037 emit(ctx, sltu, t1, lo(rs), rd); /* t1 = rsl <u imm */ in emit_slti_r64()
1038 emit(ctx, sltu, t2, rd, lo(rs)); /* t2 = imm <u rsl */ in emit_slti_r64()
1039 emit(ctx, srl, rd, hi(rs), 31); /* rd = rsh >> 31 */ in emit_slti_r64()
1041 emit_movz_r(ctx, t1, t2, rd); /* t1 = rd ? t1 : t2 */ in emit_slti_r64()
1043 emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ in emit_slti_r64()
1050 emit(ctx, addiu, rd, hi(rs), 1); /* rd = rsh + 1 */ in emit_slti_r64()
1055 emit_movn_r(ctx, t1, MIPS_R_ZERO, cmp); /* t1 = 0 if cmp != 0 */ in emit_slti_r64()
1062 emit(ctx, slti, rd, hi(rs), imm < 0 ? -1 : 0); /* rd = rsh < hi(imm) */ in emit_slti_r64()
1063 emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ in emit_slti_r64()
1067 static void emit_slt_r64(struct jit_context *ctx, u8 rd, in emit_slt_r64() argument
1079 emit(ctx, sltu, t1, lo(rs), lo(rt)); /* t1 = rsl <u rtl */ in emit_slt_r64()
1080 emit(ctx, sltu, t2, lo(rt), lo(rs)); /* t2 = rtl <u rsl */ in emit_slt_r64()
1081 emit(ctx, xor, t3, hi(rs), hi(rt)); /* t3 = rlh ^ rth */ in emit_slt_r64()
1082 emit(ctx, srl, rd, t3, 31); /* rd = t3 >> 31 */ in emit_slt_r64()
1083 emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ in emit_slt_r64()
1084 emit_movn_r(ctx, t1, MIPS_R_ZERO, t3); /* t1 = 0 if t3 != 0 */ in emit_slt_r64()
1087 emit(ctx, slt, rd, hi(rs), hi(rt)); /* rd = rsh <s rth */ in emit_slt_r64()
1088 emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ in emit_slt_r64()
1092 static void emit_jmp_i64(struct jit_context *ctx, in emit_jmp_i64() argument
1106 emit(ctx, addiu, tmp, lo(dst), -imm); in emit_jmp_i64()
1108 emit(ctx, xori, tmp, lo(dst), imm); in emit_jmp_i64()
1110 emit_mov_i(ctx, tmp, imm); in emit_jmp_i64()
1111 emit(ctx, xor, tmp, lo(dst), tmp); in emit_jmp_i64()
1114 emit(ctx, addu, MIPS_R_T9, hi(dst), 1); in emit_jmp_i64()
1115 emit(ctx, or, tmp, tmp, MIPS_R_T9); in emit_jmp_i64()
1117 emit(ctx, or, tmp, tmp, hi(dst)); in emit_jmp_i64()
1120 emit(ctx, beqz, tmp, off); in emit_jmp_i64()
1122 emit(ctx, bnez, tmp, off); in emit_jmp_i64()
1129 emit(ctx, andi, tmp, lo(dst), imm); in emit_jmp_i64()
1131 emit_mov_i(ctx, tmp, imm); in emit_jmp_i64()
1132 emit(ctx, and, tmp, lo(dst), tmp); in emit_jmp_i64()
1135 emit(ctx, or, tmp, tmp, hi(dst)); in emit_jmp_i64()
1137 emit(ctx, bnez, tmp, off); in emit_jmp_i64()
1139 emit(ctx, beqz, tmp, off); in emit_jmp_i64()
1143 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); in emit_jmp_i64()
1144 emit(ctx, beqz, tmp, off); in emit_jmp_i64()
1148 emit_sltiu_r64(ctx, tmp, dst, imm); in emit_jmp_i64()
1149 emit(ctx, beqz, tmp, off); in emit_jmp_i64()
1153 emit_sltiu_r64(ctx, tmp, dst, imm); in emit_jmp_i64()
1154 emit(ctx, bnez, tmp, off); in emit_jmp_i64()
1158 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); in emit_jmp_i64()
1159 emit(ctx, bnez, tmp, off); in emit_jmp_i64()
1163 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); in emit_jmp_i64()
1164 emit(ctx, beqz, tmp, off); in emit_jmp_i64()
1168 emit_slti_r64(ctx, tmp, dst, imm); in emit_jmp_i64()
1169 emit(ctx, beqz, tmp, off); in emit_jmp_i64()
1173 emit_slti_r64(ctx, tmp, dst, imm); in emit_jmp_i64()
1174 emit(ctx, bnez, tmp, off); in emit_jmp_i64()
1178 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); in emit_jmp_i64()
1179 emit(ctx, bnez, tmp, off); in emit_jmp_i64()
1185 static void emit_jmp_r64(struct jit_context *ctx, in emit_jmp_r64() argument
1199 emit(ctx, subu, t1, lo(dst), lo(src)); in emit_jmp_r64()
1200 emit(ctx, subu, t2, hi(dst), hi(src)); in emit_jmp_r64()
1201 emit(ctx, or, t1, t1, t2); in emit_jmp_r64()
1203 emit(ctx, beqz, t1, off); in emit_jmp_r64()
1205 emit(ctx, bnez, t1, off); in emit_jmp_r64()
1211 emit(ctx, and, t1, lo(dst), lo(src)); in emit_jmp_r64()
1212 emit(ctx, and, t2, hi(dst), hi(src)); in emit_jmp_r64()
1213 emit(ctx, or, t1, t1, t2); in emit_jmp_r64()
1215 emit(ctx, bnez, t1, off); in emit_jmp_r64()
1217 emit(ctx, beqz, t1, off); in emit_jmp_r64()
1221 emit_sltu_r64(ctx, t1, src, dst); in emit_jmp_r64()
1222 emit(ctx, bnez, t1, off); in emit_jmp_r64()
1226 emit_sltu_r64(ctx, t1, dst, src); in emit_jmp_r64()
1227 emit(ctx, beqz, t1, off); in emit_jmp_r64()
1231 emit_sltu_r64(ctx, t1, dst, src); in emit_jmp_r64()
1232 emit(ctx, bnez, t1, off); in emit_jmp_r64()
1236 emit_sltu_r64(ctx, t1, src, dst); in emit_jmp_r64()
1237 emit(ctx, beqz, t1, off); in emit_jmp_r64()
1241 emit_slt_r64(ctx, t1, src, dst); in emit_jmp_r64()
1242 emit(ctx, bnez, t1, off); in emit_jmp_r64()
1246 emit_slt_r64(ctx, t1, dst, src); in emit_jmp_r64()
1247 emit(ctx, beqz, t1, off); in emit_jmp_r64()
1251 emit_slt_r64(ctx, t1, dst, src); in emit_jmp_r64()
1252 emit(ctx, bnez, t1, off); in emit_jmp_r64()
1256 emit_slt_r64(ctx, t1, src, dst); in emit_jmp_r64()
1257 emit(ctx, beqz, t1, off); in emit_jmp_r64()
1263 static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) in emit_call() argument
1269 if (bpf_jit_get_func_addr(ctx->program, insn, false, in emit_call()
1276 push_regs(ctx, JIT_STACK_REGS, 0, JIT_RESERVED_STACK); in emit_call()
1279 emit_mov_i(ctx, MIPS_R_T9, addr); in emit_call()
1280 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); in emit_call()
1281 emit(ctx, nop); /* Delay slot */ in emit_call()
1283 clobber_reg(ctx, MIPS_R_RA); in emit_call()
1284 clobber_reg(ctx, MIPS_R_V0); in emit_call()
1285 clobber_reg(ctx, MIPS_R_V1); in emit_call()
1290 static int emit_tail_call(struct jit_context *ctx) in emit_tail_call() argument
1310 emit(ctx, lw, t1, off, ary); /* t1 = ary->map.max_entries*/ in emit_tail_call()
1311 emit_load_delay(ctx); /* Load delay slot */ in emit_tail_call()
1312 emit(ctx, sltu, t1, ind, t1); /* t1 = ind < t1 */ in emit_tail_call()
1313 emit(ctx, beqz, t1, get_offset(ctx, 1)); /* PC += off(1) if t1 == 0 */ in emit_tail_call()
1316 emit(ctx, lw, t2, ctx->stack_size, MIPS_R_SP); /* t2 = *(SP + size) */ in emit_tail_call()
1317 emit_load_delay(ctx); /* Load delay slot */ in emit_tail_call()
1318 emit(ctx, blez, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 <= 0 */ in emit_tail_call()
1319 emit(ctx, addiu, t2, t2, -1); /* t2-- (delay slot) */ in emit_tail_call()
1320 emit(ctx, sw, t2, ctx->stack_size, MIPS_R_SP); /* *(SP + size) = t2 */ in emit_tail_call()
1326 emit(ctx, sll, t1, ind, 2); /* t1 = ind << 2 */ in emit_tail_call()
1327 emit(ctx, addu, t1, t1, ary); /* t1 += ary */ in emit_tail_call()
1328 emit(ctx, lw, t2, off, t1); /* t2 = *(t1 + off) */ in emit_tail_call()
1329 emit_load_delay(ctx); /* Load delay slot */ in emit_tail_call()
1332 emit(ctx, beqz, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 == 0 */ in emit_tail_call()
1333 emit(ctx, nop); /* Delay slot */ in emit_tail_call()
1339 emit(ctx, lw, t1, off, t2); /* t1 = *(t2 + off) */ in emit_tail_call()
1340 emit_load_delay(ctx); /* Load delay slot */ in emit_tail_call()
1341 emit(ctx, addiu, t1, t1, JIT_TCALL_SKIP); /* t1 += skip (8 or 12) */ in emit_tail_call()
1344 build_epilogue(ctx, t1); in emit_tail_call()
1373 void build_prologue(struct jit_context *ctx) in build_prologue() argument
1392 emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, MAX_TAIL_CALL_CNT); in build_prologue()
1393 emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP); in build_prologue()
1401 emit(ctx, move, lo(r1), MIPS_R_A0); in build_prologue()
1407 emit(ctx, move, hi(r1), MIPS_R_ZERO); in build_prologue()
1410 if (ctx->accessed & BIT(BPF_REG_FP)) in build_prologue()
1411 clobber_reg64(ctx, fp); in build_prologue()
1414 saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u32); in build_prologue()
1418 locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); in build_prologue()
1426 reserved = ctx->stack_used; in build_prologue()
1430 emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, -stack); in build_prologue()
1433 push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); in build_prologue()
1436 if (ctx->accessed & BIT(BPF_REG_FP)) in build_prologue()
1437 emit(ctx, addiu, lo(fp), MIPS_R_SP, stack - saved); in build_prologue()
1439 ctx->saved_size = saved; in build_prologue()
1440 ctx->stack_size = stack; in build_prologue()
1444 void build_epilogue(struct jit_context *ctx, int dest_reg) in build_epilogue() argument
1447 pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, in build_epilogue()
1448 ctx->stack_size - ctx->saved_size); in build_epilogue()
1454 emit(ctx, move, MIPS_R_V0, MIPS_R_V1); in build_epilogue()
1458 emit(ctx, jr, dest_reg); in build_epilogue()
1459 emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); in build_epilogue()
1463 int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) in build_insn() argument
1479 emit_mov_i(ctx, lo(dst), imm); in build_insn()
1480 emit_zext_ver(ctx, dst); in build_insn()
1486 emit_mov_i(ctx, hi(dst), 0); in build_insn()
1488 emit_mov_r(ctx, lo(dst), lo(src)); in build_insn()
1489 emit_zext_ver(ctx, dst); in build_insn()
1494 emit_alu_i(ctx, lo(dst), 0, BPF_NEG); in build_insn()
1495 emit_zext_ver(ctx, dst); in build_insn()
1520 emit_mov_i(ctx, MIPS_R_T6, imm); in build_insn()
1521 emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code)); in build_insn()
1523 emit_alu_i(ctx, lo(dst), val, alu); in build_insn()
1525 emit_zext_ver(ctx, dst); in build_insn()
1549 emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code)); in build_insn()
1550 emit_zext_ver(ctx, dst); in build_insn()
1554 emit_mov_se_i64(ctx, dst, imm); in build_insn()
1558 emit_mov_r(ctx, lo(dst), lo(src)); in build_insn()
1559 emit_mov_r(ctx, hi(dst), hi(src)); in build_insn()
1563 emit_neg_i64(ctx, dst); in build_insn()
1567 emit_alu_i64(ctx, dst, imm, BPF_OP(code)); in build_insn()
1578 emit_alu_i64(ctx, dst, imm, BPF_OP(code)); in build_insn()
1587 emit_shift_i64(ctx, dst, imm, BPF_OP(code)); in build_insn()
1591 emit_mul_i64(ctx, dst, imm); in build_insn()
1601 emit_mov_se_i64(ctx, tmp, imm); in build_insn()
1602 emit_divmod_r64(ctx, dst, tmp, BPF_OP(code)); in build_insn()
1614 emit_alu_r64(ctx, dst, src, BPF_OP(code)); in build_insn()
1622 emit_shift_r64(ctx, dst, lo(src), BPF_OP(code)); in build_insn()
1626 emit_mul_r64(ctx, dst, src); in build_insn()
1632 emit_divmod_r64(ctx, dst, src, BPF_OP(code)); in build_insn()
1645 emit_bswap_r64(ctx, dst, imm); in build_insn()
1647 emit_trunc_r64(ctx, dst, imm); in build_insn()
1651 emit_mov_i(ctx, lo(dst), imm); in build_insn()
1652 emit_mov_i(ctx, hi(dst), insn[1].imm); in build_insn()
1659 emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code)); in build_insn()
1669 emit_mov_se_i64(ctx, tmp, imm); in build_insn()
1674 emit_mov_i(ctx, lo(tmp), imm); in build_insn()
1677 emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code)); in build_insn()
1684 emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code)); in build_insn()
1702 emit_atomic_r(ctx, lo(dst), lo(src), off, imm); in build_insn()
1704 emit_atomic_r32(ctx, lo(dst), lo(src), in build_insn()
1707 emit_zext_ver(ctx, src); in build_insn()
1711 emit_cmpxchg_r(ctx, lo(dst), lo(src), in build_insn()
1714 emit_cmpxchg_r32(ctx, lo(dst), lo(src), off); in build_insn()
1733 emit_atomic_r64(ctx, lo(dst), src, off, imm); in build_insn()
1736 emit_cmpxchg_r64(ctx, lo(dst), src, off); in build_insn()
1766 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); in build_insn()
1767 emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp); in build_insn()
1768 if (finish_jmp(ctx, jmp, off) < 0) in build_insn()
1795 setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); in build_insn()
1797 emit_jmp_i(ctx, lo(dst), imm, rel, jmp); in build_insn()
1800 emit_mov_i(ctx, MIPS_R_T6, imm); in build_insn()
1801 emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp); in build_insn()
1803 if (finish_jmp(ctx, jmp, off) < 0) in build_insn()
1830 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); in build_insn()
1831 emit_jmp_r64(ctx, dst, src, rel, jmp); in build_insn()
1832 if (finish_jmp(ctx, jmp, off) < 0) in build_insn()
1859 setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); in build_insn()
1860 emit_jmp_i64(ctx, dst, imm, rel, jmp); in build_insn()
1861 if (finish_jmp(ctx, jmp, off) < 0) in build_insn()
1868 if (emit_ja(ctx, off) < 0) in build_insn()
1873 if (emit_tail_call(ctx) < 0) in build_insn()
1878 if (emit_call(ctx, insn) < 0) in build_insn()
1887 if (ctx->bpf_index == ctx->program->len - 1) in build_insn()
1889 if (emit_exit(ctx) < 0) in build_insn()
1902 ctx->bpf_index, code); in build_insn()