Lines Matching refs:ctx
57 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) in bpf_to_rv_reg() argument
68 __set_bit(reg, &ctx->flags); in bpf_to_rv_reg()
73 static bool seen_reg(int reg, struct rv_jit_context *ctx) in seen_reg() argument
83 return test_bit(reg, &ctx->flags); in seen_reg()
88 static void mark_fp(struct rv_jit_context *ctx) in mark_fp() argument
90 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); in mark_fp()
93 static void mark_call(struct rv_jit_context *ctx) in mark_call() argument
95 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in mark_call()
98 static bool seen_call(struct rv_jit_context *ctx) in seen_call() argument
100 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in seen_call()
103 static void mark_tail_call(struct rv_jit_context *ctx) in mark_tail_call() argument
105 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in mark_tail_call()
108 static bool seen_tail_call(struct rv_jit_context *ctx) in seen_tail_call() argument
110 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in seen_tail_call()
113 static u8 rv_tail_call_reg(struct rv_jit_context *ctx) in rv_tail_call_reg() argument
115 mark_tail_call(ctx); in rv_tail_call_reg()
117 if (seen_call(ctx)) { in rv_tail_call_reg()
118 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); in rv_tail_call_reg()
139 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) in emit_imm() argument
160 emit_lui(rd, upper, ctx); in emit_imm()
163 emit_li(rd, lower, ctx); in emit_imm()
167 emit_addiw(rd, rd, lower, ctx); in emit_imm()
175 emit_imm(rd, upper, ctx); in emit_imm()
177 emit_slli(rd, rd, shift, ctx); in emit_imm()
179 emit_addi(rd, rd, lower, ctx); in emit_imm()
182 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) in __build_epilogue() argument
184 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; in __build_epilogue()
186 if (seen_reg(RV_REG_RA, ctx)) { in __build_epilogue()
187 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); in __build_epilogue()
190 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); in __build_epilogue()
192 if (seen_reg(RV_REG_S1, ctx)) { in __build_epilogue()
193 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); in __build_epilogue()
196 if (seen_reg(RV_REG_S2, ctx)) { in __build_epilogue()
197 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); in __build_epilogue()
200 if (seen_reg(RV_REG_S3, ctx)) { in __build_epilogue()
201 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); in __build_epilogue()
204 if (seen_reg(RV_REG_S4, ctx)) { in __build_epilogue()
205 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); in __build_epilogue()
208 if (seen_reg(RV_REG_S5, ctx)) { in __build_epilogue()
209 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); in __build_epilogue()
212 if (seen_reg(RV_REG_S6, ctx)) { in __build_epilogue()
213 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); in __build_epilogue()
217 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); in __build_epilogue()
220 emit_mv(RV_REG_A0, RV_REG_A5, ctx); in __build_epilogue()
223 ctx); in __build_epilogue()
227 struct rv_jit_context *ctx) in emit_bcc() argument
231 emit(rv_beq(rd, rs, rvoff >> 1), ctx); in emit_bcc()
234 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); in emit_bcc()
237 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); in emit_bcc()
240 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); in emit_bcc()
243 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); in emit_bcc()
246 emit(rv_bne(rd, rs, rvoff >> 1), ctx); in emit_bcc()
249 emit(rv_blt(rs, rd, rvoff >> 1), ctx); in emit_bcc()
252 emit(rv_blt(rd, rs, rvoff >> 1), ctx); in emit_bcc()
255 emit(rv_bge(rd, rs, rvoff >> 1), ctx); in emit_bcc()
258 emit(rv_bge(rs, rd, rvoff >> 1), ctx); in emit_bcc()
263 struct rv_jit_context *ctx) in emit_branch() argument
268 emit_bcc(cond, rd, rs, rvoff, ctx); in emit_branch()
285 emit_bcc(cond, rd, rs, 8, ctx); in emit_branch()
286 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); in emit_branch()
296 emit_bcc(cond, rd, rs, 12, ctx); in emit_branch()
297 emit(rv_auipc(RV_REG_T1, upper), ctx); in emit_branch()
298 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); in emit_branch()
301 static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) in emit_zext_32() argument
303 emit_slli(reg, reg, 32, ctx); in emit_zext_32()
304 emit_srli(reg, reg, 32, ctx); in emit_zext_32()
307 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) in emit_bpf_tail_call() argument
309 int tc_ninsn, off, start_insn = ctx->ninsns; in emit_bpf_tail_call()
310 u8 tcc = rv_tail_call_reg(ctx); in emit_bpf_tail_call()
319 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : in emit_bpf_tail_call()
320 ctx->offset[0]; in emit_bpf_tail_call()
321 emit_zext_32(RV_REG_A2, ctx); in emit_bpf_tail_call()
326 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); in emit_bpf_tail_call()
327 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
328 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); in emit_bpf_tail_call()
333 emit_addi(RV_REG_T1, tcc, -1, ctx); in emit_bpf_tail_call()
334 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
335 emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); in emit_bpf_tail_call()
341 emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); in emit_bpf_tail_call()
342 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); in emit_bpf_tail_call()
346 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); in emit_bpf_tail_call()
347 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
348 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); in emit_bpf_tail_call()
354 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); in emit_bpf_tail_call()
355 emit_mv(RV_REG_TCC, RV_REG_T1, ctx); in emit_bpf_tail_call()
356 __build_epilogue(true, ctx); in emit_bpf_tail_call()
361 struct rv_jit_context *ctx) in init_regs() argument
372 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); in init_regs()
378 *rs = bpf_to_rv_reg(insn->src_reg, ctx); in init_regs()
381 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) in emit_zext_32_rd_rs() argument
383 emit_mv(RV_REG_T2, *rd, ctx); in emit_zext_32_rd_rs()
384 emit_zext_32(RV_REG_T2, ctx); in emit_zext_32_rd_rs()
385 emit_mv(RV_REG_T1, *rs, ctx); in emit_zext_32_rd_rs()
386 emit_zext_32(RV_REG_T1, ctx); in emit_zext_32_rd_rs()
391 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) in emit_sext_32_rd_rs() argument
393 emit_addiw(RV_REG_T2, *rd, 0, ctx); in emit_sext_32_rd_rs()
394 emit_addiw(RV_REG_T1, *rs, 0, ctx); in emit_sext_32_rd_rs()
399 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) in emit_zext_32_rd_t1() argument
401 emit_mv(RV_REG_T2, *rd, ctx); in emit_zext_32_rd_t1()
402 emit_zext_32(RV_REG_T2, ctx); in emit_zext_32_rd_t1()
403 emit_zext_32(RV_REG_T1, ctx); in emit_zext_32_rd_t1()
407 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) in emit_sext_32_rd() argument
409 emit_addiw(RV_REG_T2, *rd, 0, ctx); in emit_sext_32_rd()
414 struct rv_jit_context *ctx) in emit_jump_and_link() argument
419 emit(rv_jal(rd, rvoff >> 1), ctx); in emit_jump_and_link()
424 emit(rv_auipc(RV_REG_T1, upper), ctx); in emit_jump_and_link()
425 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); in emit_jump_and_link()
439 static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) in emit_call() argument
446 if (addr && ctx->insns) { in emit_call()
447 ip = (u64)(long)(ctx->insns + ctx->ninsns); in emit_call()
451 ret = emit_jump_and_link(RV_REG_RA, off, !fixed, ctx); in emit_call()
454 rd = bpf_to_rv_reg(BPF_REG_0, ctx); in emit_call()
455 emit_mv(rd, RV_REG_A0, ctx); in emit_call()
478 struct rv_jit_context *ctx, in add_exception_handler() argument
485 if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM) in add_exception_handler()
488 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) in add_exception_handler()
491 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) in add_exception_handler()
497 ex = &ctx->prog->aux->extable[ctx->nexentries]; in add_exception_handler()
498 pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len]; in add_exception_handler()
520 ctx->nexentries++; in add_exception_handler()
524 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, in bpf_jit_emit_insn() argument
529 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; in bpf_jit_emit_insn()
530 struct bpf_prog_aux *aux = ctx->prog->aux; in bpf_jit_emit_insn()
535 init_regs(&rd, &rs, insn, ctx); in bpf_jit_emit_insn()
543 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
546 emit_mv(rd, rs, ctx); in bpf_jit_emit_insn()
548 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
554 emit_add(rd, rd, rs, ctx); in bpf_jit_emit_insn()
556 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
561 emit_sub(rd, rd, rs, ctx); in bpf_jit_emit_insn()
563 emit_subw(rd, rd, rs, ctx); in bpf_jit_emit_insn()
566 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
570 emit_and(rd, rd, rs, ctx); in bpf_jit_emit_insn()
572 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
576 emit_or(rd, rd, rs, ctx); in bpf_jit_emit_insn()
578 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
582 emit_xor(rd, rd, rs, ctx); in bpf_jit_emit_insn()
584 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
588 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
590 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
594 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
596 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
600 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
602 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
606 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
608 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
612 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
614 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
618 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); in bpf_jit_emit_insn()
620 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
626 emit_sub(rd, RV_REG_ZERO, rd, ctx); in bpf_jit_emit_insn()
628 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
635 emit_slli(rd, rd, 48, ctx); in bpf_jit_emit_insn()
636 emit_srli(rd, rd, 48, ctx); in bpf_jit_emit_insn()
640 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
649 emit_li(RV_REG_T2, 0, ctx); in bpf_jit_emit_insn()
651 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
652 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
653 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
654 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
658 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
659 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
660 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
661 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
663 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
664 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
665 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
666 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
670 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
671 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
672 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
673 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
675 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
676 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
677 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
678 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
680 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
681 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
682 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
683 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
685 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
686 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
687 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); in bpf_jit_emit_insn()
688 emit_srli(rd, rd, 8, ctx); in bpf_jit_emit_insn()
690 emit_andi(RV_REG_T1, rd, 0xff, ctx); in bpf_jit_emit_insn()
691 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); in bpf_jit_emit_insn()
693 emit_mv(rd, RV_REG_T2, ctx); in bpf_jit_emit_insn()
699 emit_imm(rd, imm, ctx); in bpf_jit_emit_insn()
701 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
708 emit_addi(rd, rd, imm, ctx); in bpf_jit_emit_insn()
710 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
711 emit_add(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
714 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
719 emit_addi(rd, rd, -imm, ctx); in bpf_jit_emit_insn()
721 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
722 emit_sub(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
725 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
730 emit_andi(rd, rd, imm, ctx); in bpf_jit_emit_insn()
732 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
733 emit_and(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
736 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
741 emit(rv_ori(rd, rd, imm), ctx); in bpf_jit_emit_insn()
743 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
744 emit_or(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
747 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
752 emit(rv_xori(rd, rd, imm), ctx); in bpf_jit_emit_insn()
754 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
755 emit_xor(rd, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
758 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
762 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
764 rv_mulw(rd, rd, RV_REG_T1), ctx); in bpf_jit_emit_insn()
766 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
770 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
772 rv_divuw(rd, rd, RV_REG_T1), ctx); in bpf_jit_emit_insn()
774 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
778 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
780 rv_remuw(rd, rd, RV_REG_T1), ctx); in bpf_jit_emit_insn()
782 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
786 emit_slli(rd, rd, imm, ctx); in bpf_jit_emit_insn()
789 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
794 emit_srli(rd, rd, imm, ctx); in bpf_jit_emit_insn()
796 emit(rv_srliw(rd, rd, imm), ctx); in bpf_jit_emit_insn()
799 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
804 emit_srai(rd, rd, imm, ctx); in bpf_jit_emit_insn()
806 emit(rv_sraiw(rd, rd, imm), ctx); in bpf_jit_emit_insn()
809 emit_zext_32(rd, ctx); in bpf_jit_emit_insn()
814 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
815 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); in bpf_jit_emit_insn()
843 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
845 s = ctx->ninsns; in bpf_jit_emit_insn()
847 emit_sext_32_rd_rs(&rd, &rs, ctx); in bpf_jit_emit_insn()
849 emit_zext_32_rd_rs(&rd, &rs, ctx); in bpf_jit_emit_insn()
850 e = ctx->ninsns; in bpf_jit_emit_insn()
859 emit_and(RV_REG_T1, rd, rs, ctx); in bpf_jit_emit_insn()
861 ctx); in bpf_jit_emit_insn()
863 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); in bpf_jit_emit_insn()
888 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
889 s = ctx->ninsns; in bpf_jit_emit_insn()
891 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
899 emit_sext_32_rd(&rd, ctx); in bpf_jit_emit_insn()
901 emit_zext_32_rd_t1(&rd, ctx); in bpf_jit_emit_insn()
903 e = ctx->ninsns; in bpf_jit_emit_insn()
907 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); in bpf_jit_emit_insn()
912 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
913 s = ctx->ninsns; in bpf_jit_emit_insn()
915 emit_andi(RV_REG_T1, rd, imm, ctx); in bpf_jit_emit_insn()
917 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
918 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); in bpf_jit_emit_insn()
925 emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); in bpf_jit_emit_insn()
926 e = ctx->ninsns; in bpf_jit_emit_insn()
928 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); in bpf_jit_emit_insn()
937 mark_call(ctx); in bpf_jit_emit_insn()
938 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, in bpf_jit_emit_insn()
942 ret = emit_call(fixed, addr, ctx); in bpf_jit_emit_insn()
949 if (emit_bpf_tail_call(i, ctx)) in bpf_jit_emit_insn()
955 if (i == ctx->prog->len - 1) in bpf_jit_emit_insn()
958 rvoff = epilogue_offset(ctx); in bpf_jit_emit_insn()
959 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); in bpf_jit_emit_insn()
971 emit_imm(rd, imm64, ctx); in bpf_jit_emit_insn()
990 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
991 emit(rv_lbu(rd, off, rs), ctx); in bpf_jit_emit_insn()
992 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
996 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
997 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
998 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
999 emit(rv_lbu(rd, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
1000 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1006 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1007 emit(rv_lhu(rd, off, rs), ctx); in bpf_jit_emit_insn()
1008 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1012 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1013 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
1014 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1015 emit(rv_lhu(rd, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
1016 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1022 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1023 emit(rv_lwu(rd, off, rs), ctx); in bpf_jit_emit_insn()
1024 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1028 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1029 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
1030 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1031 emit(rv_lwu(rd, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
1032 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1038 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1039 emit_ld(rd, off, rs, ctx); in bpf_jit_emit_insn()
1040 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1044 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1045 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); in bpf_jit_emit_insn()
1046 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1047 emit_ld(rd, 0, RV_REG_T1, ctx); in bpf_jit_emit_insn()
1048 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1052 ret = add_exception_handler(insn, ctx, rd, insn_len); in bpf_jit_emit_insn()
1063 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
1065 emit(rv_sb(rd, off, RV_REG_T1), ctx); in bpf_jit_emit_insn()
1069 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
1070 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); in bpf_jit_emit_insn()
1071 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
1075 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
1077 emit(rv_sh(rd, off, RV_REG_T1), ctx); in bpf_jit_emit_insn()
1081 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
1082 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); in bpf_jit_emit_insn()
1083 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); in bpf_jit_emit_insn()
1086 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
1088 emit_sw(rd, off, RV_REG_T1, ctx); in bpf_jit_emit_insn()
1092 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
1093 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); in bpf_jit_emit_insn()
1094 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); in bpf_jit_emit_insn()
1097 emit_imm(RV_REG_T1, imm, ctx); in bpf_jit_emit_insn()
1099 emit_sd(rd, off, RV_REG_T1, ctx); in bpf_jit_emit_insn()
1103 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
1104 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); in bpf_jit_emit_insn()
1105 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); in bpf_jit_emit_insn()
1111 emit(rv_sb(rd, off, rs), ctx); in bpf_jit_emit_insn()
1115 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1116 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1117 emit(rv_sb(RV_REG_T1, 0, rs), ctx); in bpf_jit_emit_insn()
1121 emit(rv_sh(rd, off, rs), ctx); in bpf_jit_emit_insn()
1125 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1126 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1127 emit(rv_sh(RV_REG_T1, 0, rs), ctx); in bpf_jit_emit_insn()
1131 emit_sw(rd, off, rs, ctx); in bpf_jit_emit_insn()
1135 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1136 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1137 emit_sw(RV_REG_T1, 0, rs, ctx); in bpf_jit_emit_insn()
1141 emit_sd(rd, off, rs, ctx); in bpf_jit_emit_insn()
1145 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1146 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1147 emit_sd(RV_REG_T1, 0, rs, ctx); in bpf_jit_emit_insn()
1163 emit_addi(RV_REG_T1, rd, off, ctx); in bpf_jit_emit_insn()
1165 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1166 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); in bpf_jit_emit_insn()
1174 rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx); in bpf_jit_emit_insn()
1184 void bpf_jit_build_prologue(struct rv_jit_context *ctx) in bpf_jit_build_prologue() argument
1188 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); in bpf_jit_build_prologue()
1190 mark_fp(ctx); in bpf_jit_build_prologue()
1192 if (seen_reg(RV_REG_RA, ctx)) in bpf_jit_build_prologue()
1195 if (seen_reg(RV_REG_S1, ctx)) in bpf_jit_build_prologue()
1197 if (seen_reg(RV_REG_S2, ctx)) in bpf_jit_build_prologue()
1199 if (seen_reg(RV_REG_S3, ctx)) in bpf_jit_build_prologue()
1201 if (seen_reg(RV_REG_S4, ctx)) in bpf_jit_build_prologue()
1203 if (seen_reg(RV_REG_S5, ctx)) in bpf_jit_build_prologue()
1205 if (seen_reg(RV_REG_S6, ctx)) in bpf_jit_build_prologue()
1217 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); in bpf_jit_build_prologue()
1219 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); in bpf_jit_build_prologue()
1221 if (seen_reg(RV_REG_RA, ctx)) { in bpf_jit_build_prologue()
1222 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); in bpf_jit_build_prologue()
1225 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); in bpf_jit_build_prologue()
1227 if (seen_reg(RV_REG_S1, ctx)) { in bpf_jit_build_prologue()
1228 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); in bpf_jit_build_prologue()
1231 if (seen_reg(RV_REG_S2, ctx)) { in bpf_jit_build_prologue()
1232 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); in bpf_jit_build_prologue()
1235 if (seen_reg(RV_REG_S3, ctx)) { in bpf_jit_build_prologue()
1236 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); in bpf_jit_build_prologue()
1239 if (seen_reg(RV_REG_S4, ctx)) { in bpf_jit_build_prologue()
1240 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); in bpf_jit_build_prologue()
1243 if (seen_reg(RV_REG_S5, ctx)) { in bpf_jit_build_prologue()
1244 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); in bpf_jit_build_prologue()
1247 if (seen_reg(RV_REG_S6, ctx)) { in bpf_jit_build_prologue()
1248 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); in bpf_jit_build_prologue()
1252 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); in bpf_jit_build_prologue()
1255 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); in bpf_jit_build_prologue()
1260 if (seen_tail_call(ctx) && seen_call(ctx)) in bpf_jit_build_prologue()
1261 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); in bpf_jit_build_prologue()
1263 ctx->stack_size = stack_adjust; in bpf_jit_build_prologue()
1266 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) in bpf_jit_build_epilogue() argument
1268 __build_epilogue(false, ctx); in bpf_jit_build_epilogue()