Lines Matching refs:dst_reg
2697 return insn->dst_reg; in insn_def_regno()
2704 int dst_reg = insn_def_regno(insn); in insn_has_def32() local
2706 if (dst_reg == -1) in insn_has_def32()
2709 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
2854 u32 dreg = 1u << insn->dst_reg; in backtrack_insn()
2930 if (insn->dst_reg != BPF_REG_FP) in backtrack_insn()
3494 u32 dst_reg = insn->dst_reg; in check_stack_write_fixed_off() local
3535 if (dst_reg != BPF_REG_FP) { in check_stack_write_fixed_off()
5571 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic()
5594 if (is_ctx_reg(env, insn->dst_reg) || in check_atomic()
5595 is_pkt_reg(env, insn->dst_reg) || in check_atomic()
5596 is_flow_key_reg(env, insn->dst_reg) || in check_atomic()
5597 is_sk_reg(env, insn->dst_reg)) { in check_atomic()
5599 insn->dst_reg, in check_atomic()
5600 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic()
5624 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5627 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5634 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
10239 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
10241 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
10252 struct bpf_reg_state *dst_reg, in sanitize_ptr_alu() argument
10260 bool ptr_is_dst_reg = ptr_reg == dst_reg; in sanitize_ptr_alu()
10334 tmp = *dst_reg; in sanitize_ptr_alu()
10335 copy_register_state(dst_reg, ptr_reg); in sanitize_ptr_alu()
10340 *dst_reg = tmp; in sanitize_ptr_alu()
10360 const struct bpf_reg_state *dst_reg) in sanitize_err() argument
10364 u32 dst = insn->dst_reg, src = insn->src_reg; in sanitize_err()
10369 off_reg == dst_reg ? dst : src, err); in sanitize_err()
10373 off_reg == dst_reg ? src : dst, err); in sanitize_err()
10432 const struct bpf_reg_state *dst_reg) in sanitize_check_bounds() argument
10434 u32 dst = insn->dst_reg; in sanitize_check_bounds()
10442 switch (dst_reg->type) { in sanitize_check_bounds()
10444 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
10445 dst_reg->off + dst_reg->var_off.value)) in sanitize_check_bounds()
10449 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
10474 struct bpf_reg_state *regs = state->regs, *dst_reg; in adjust_ptr_min_max_vals() local
10482 u32 dst = insn->dst_reg; in adjust_ptr_min_max_vals()
10485 dst_reg = ®s[dst]; in adjust_ptr_min_max_vals()
10492 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
10499 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
10536 dst_reg->type = ptr_reg->type; in adjust_ptr_min_max_vals()
10537 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
10544 __mark_reg32_unbounded(dst_reg); in adjust_ptr_min_max_vals()
10547 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
10550 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
10561 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
10562 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
10563 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
10564 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
10565 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
10566 dst_reg->off = ptr_reg->off + smin_val; in adjust_ptr_min_max_vals()
10567 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
10581 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
10582 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
10584 dst_reg->smin_value = smin_ptr + smin_val; in adjust_ptr_min_max_vals()
10585 dst_reg->smax_value = smax_ptr + smax_val; in adjust_ptr_min_max_vals()
10589 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
10590 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
10592 dst_reg->umin_value = umin_ptr + umin_val; in adjust_ptr_min_max_vals()
10593 dst_reg->umax_value = umax_ptr + umax_val; in adjust_ptr_min_max_vals()
10595 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
10596 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
10597 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
10599 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
10601 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
10605 if (dst_reg == off_reg) { in adjust_ptr_min_max_vals()
10623 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
10624 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
10625 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
10626 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
10627 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
10628 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
10629 dst_reg->off = ptr_reg->off - smin_val; in adjust_ptr_min_max_vals()
10630 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
10639 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
10640 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
10642 dst_reg->smin_value = smin_ptr - smax_val; in adjust_ptr_min_max_vals()
10643 dst_reg->smax_value = smax_ptr - smin_val; in adjust_ptr_min_max_vals()
10647 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
10648 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
10651 dst_reg->umin_value = umin_ptr - umax_val; in adjust_ptr_min_max_vals()
10652 dst_reg->umax_value = umax_ptr - umin_val; in adjust_ptr_min_max_vals()
10654 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
10655 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
10656 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
10658 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
10661 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
10678 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
10680 reg_bounds_sync(dst_reg); in adjust_ptr_min_max_vals()
10681 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
10684 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
10687 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
10693 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, in scalar32_min_max_add() argument
10701 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || in scalar32_min_max_add()
10702 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { in scalar32_min_max_add()
10703 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_add()
10704 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_add()
10706 dst_reg->s32_min_value += smin_val; in scalar32_min_max_add()
10707 dst_reg->s32_max_value += smax_val; in scalar32_min_max_add()
10709 if (dst_reg->u32_min_value + umin_val < umin_val || in scalar32_min_max_add()
10710 dst_reg->u32_max_value + umax_val < umax_val) { in scalar32_min_max_add()
10711 dst_reg->u32_min_value = 0; in scalar32_min_max_add()
10712 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_add()
10714 dst_reg->u32_min_value += umin_val; in scalar32_min_max_add()
10715 dst_reg->u32_max_value += umax_val; in scalar32_min_max_add()
10719 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, in scalar_min_max_add() argument
10727 if (signed_add_overflows(dst_reg->smin_value, smin_val) || in scalar_min_max_add()
10728 signed_add_overflows(dst_reg->smax_value, smax_val)) { in scalar_min_max_add()
10729 dst_reg->smin_value = S64_MIN; in scalar_min_max_add()
10730 dst_reg->smax_value = S64_MAX; in scalar_min_max_add()
10732 dst_reg->smin_value += smin_val; in scalar_min_max_add()
10733 dst_reg->smax_value += smax_val; in scalar_min_max_add()
10735 if (dst_reg->umin_value + umin_val < umin_val || in scalar_min_max_add()
10736 dst_reg->umax_value + umax_val < umax_val) { in scalar_min_max_add()
10737 dst_reg->umin_value = 0; in scalar_min_max_add()
10738 dst_reg->umax_value = U64_MAX; in scalar_min_max_add()
10740 dst_reg->umin_value += umin_val; in scalar_min_max_add()
10741 dst_reg->umax_value += umax_val; in scalar_min_max_add()
10745 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, in scalar32_min_max_sub() argument
10753 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || in scalar32_min_max_sub()
10754 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { in scalar32_min_max_sub()
10756 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_sub()
10757 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_sub()
10759 dst_reg->s32_min_value -= smax_val; in scalar32_min_max_sub()
10760 dst_reg->s32_max_value -= smin_val; in scalar32_min_max_sub()
10762 if (dst_reg->u32_min_value < umax_val) { in scalar32_min_max_sub()
10764 dst_reg->u32_min_value = 0; in scalar32_min_max_sub()
10765 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_sub()
10768 dst_reg->u32_min_value -= umax_val; in scalar32_min_max_sub()
10769 dst_reg->u32_max_value -= umin_val; in scalar32_min_max_sub()
10773 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, in scalar_min_max_sub() argument
10781 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || in scalar_min_max_sub()
10782 signed_sub_overflows(dst_reg->smax_value, smin_val)) { in scalar_min_max_sub()
10784 dst_reg->smin_value = S64_MIN; in scalar_min_max_sub()
10785 dst_reg->smax_value = S64_MAX; in scalar_min_max_sub()
10787 dst_reg->smin_value -= smax_val; in scalar_min_max_sub()
10788 dst_reg->smax_value -= smin_val; in scalar_min_max_sub()
10790 if (dst_reg->umin_value < umax_val) { in scalar_min_max_sub()
10792 dst_reg->umin_value = 0; in scalar_min_max_sub()
10793 dst_reg->umax_value = U64_MAX; in scalar_min_max_sub()
10796 dst_reg->umin_value -= umax_val; in scalar_min_max_sub()
10797 dst_reg->umax_value -= umin_val; in scalar_min_max_sub()
10801 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, in scalar32_min_max_mul() argument
10808 if (smin_val < 0 || dst_reg->s32_min_value < 0) { in scalar32_min_max_mul()
10810 __mark_reg32_unbounded(dst_reg); in scalar32_min_max_mul()
10816 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { in scalar32_min_max_mul()
10818 __mark_reg32_unbounded(dst_reg); in scalar32_min_max_mul()
10821 dst_reg->u32_min_value *= umin_val; in scalar32_min_max_mul()
10822 dst_reg->u32_max_value *= umax_val; in scalar32_min_max_mul()
10823 if (dst_reg->u32_max_value > S32_MAX) { in scalar32_min_max_mul()
10825 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_mul()
10826 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_mul()
10828 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_mul()
10829 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_mul()
10833 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, in scalar_min_max_mul() argument
10840 if (smin_val < 0 || dst_reg->smin_value < 0) { in scalar_min_max_mul()
10842 __mark_reg64_unbounded(dst_reg); in scalar_min_max_mul()
10848 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { in scalar_min_max_mul()
10850 __mark_reg64_unbounded(dst_reg); in scalar_min_max_mul()
10853 dst_reg->umin_value *= umin_val; in scalar_min_max_mul()
10854 dst_reg->umax_value *= umax_val; in scalar_min_max_mul()
10855 if (dst_reg->umax_value > S64_MAX) { in scalar_min_max_mul()
10857 dst_reg->smin_value = S64_MIN; in scalar_min_max_mul()
10858 dst_reg->smax_value = S64_MAX; in scalar_min_max_mul()
10860 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_mul()
10861 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_mul()
10865 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, in scalar32_min_max_and() argument
10869 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_and()
10870 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_and()
10875 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_and()
10882 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_and()
10883 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); in scalar32_min_max_and()
10884 if (dst_reg->s32_min_value < 0 || smin_val < 0) { in scalar32_min_max_and()
10888 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_and()
10889 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_and()
10894 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_and()
10895 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_and()
10899 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, in scalar_min_max_and() argument
10903 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_and()
10908 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_and()
10915 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_and()
10916 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); in scalar_min_max_and()
10917 if (dst_reg->smin_value < 0 || smin_val < 0) { in scalar_min_max_and()
10921 dst_reg->smin_value = S64_MIN; in scalar_min_max_and()
10922 dst_reg->smax_value = S64_MAX; in scalar_min_max_and()
10927 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_and()
10928 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_and()
10931 __update_reg_bounds(dst_reg); in scalar_min_max_and()
10934 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, in scalar32_min_max_or() argument
10938 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_or()
10939 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_or()
10944 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_or()
10951 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); in scalar32_min_max_or()
10952 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_or()
10953 if (dst_reg->s32_min_value < 0 || smin_val < 0) { in scalar32_min_max_or()
10957 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_or()
10958 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_or()
10963 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_or()
10964 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_or()
10968 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, in scalar_min_max_or() argument
10972 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_or()
10977 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_or()
10984 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); in scalar_min_max_or()
10985 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_or()
10986 if (dst_reg->smin_value < 0 || smin_val < 0) { in scalar_min_max_or()
10990 dst_reg->smin_value = S64_MIN; in scalar_min_max_or()
10991 dst_reg->smax_value = S64_MAX; in scalar_min_max_or()
10996 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_or()
10997 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_or()
11000 __update_reg_bounds(dst_reg); in scalar_min_max_or()
11003 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, in scalar32_min_max_xor() argument
11007 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_xor()
11008 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_xor()
11012 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_xor()
11017 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_xor()
11018 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_xor()
11020 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { in scalar32_min_max_xor()
11024 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_xor()
11025 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_xor()
11027 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_xor()
11028 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_xor()
11032 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, in scalar_min_max_xor() argument
11036 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_xor()
11041 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_xor()
11046 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_xor()
11047 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_xor()
11049 if (dst_reg->smin_value >= 0 && smin_val >= 0) { in scalar_min_max_xor()
11053 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_xor()
11054 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_xor()
11056 dst_reg->smin_value = S64_MIN; in scalar_min_max_xor()
11057 dst_reg->smax_value = S64_MAX; in scalar_min_max_xor()
11060 __update_reg_bounds(dst_reg); in scalar_min_max_xor()
11063 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar32_min_max_lsh() argument
11069 dst_reg->s32_min_value = S32_MIN; in __scalar32_min_max_lsh()
11070 dst_reg->s32_max_value = S32_MAX; in __scalar32_min_max_lsh()
11072 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { in __scalar32_min_max_lsh()
11073 dst_reg->u32_min_value = 0; in __scalar32_min_max_lsh()
11074 dst_reg->u32_max_value = U32_MAX; in __scalar32_min_max_lsh()
11076 dst_reg->u32_min_value <<= umin_val; in __scalar32_min_max_lsh()
11077 dst_reg->u32_max_value <<= umax_val; in __scalar32_min_max_lsh()
11081 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_lsh() argument
11087 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_lsh()
11089 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar32_min_max_lsh()
11090 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); in scalar32_min_max_lsh()
11095 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_lsh()
11096 __update_reg32_bounds(dst_reg); in scalar32_min_max_lsh()
11099 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar64_min_max_lsh() argument
11109 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) in __scalar64_min_max_lsh()
11110 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; in __scalar64_min_max_lsh()
11112 dst_reg->smax_value = S64_MAX; in __scalar64_min_max_lsh()
11114 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) in __scalar64_min_max_lsh()
11115 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; in __scalar64_min_max_lsh()
11117 dst_reg->smin_value = S64_MIN; in __scalar64_min_max_lsh()
11120 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { in __scalar64_min_max_lsh()
11121 dst_reg->umin_value = 0; in __scalar64_min_max_lsh()
11122 dst_reg->umax_value = U64_MAX; in __scalar64_min_max_lsh()
11124 dst_reg->umin_value <<= umin_val; in __scalar64_min_max_lsh()
11125 dst_reg->umax_value <<= umax_val; in __scalar64_min_max_lsh()
11129 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar_min_max_lsh() argument
11136 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
11137 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
11139 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); in scalar_min_max_lsh()
11141 __update_reg_bounds(dst_reg); in scalar_min_max_lsh()
11144 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_rsh() argument
11147 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_rsh()
11165 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_rsh()
11166 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_rsh()
11168 dst_reg->var_off = tnum_rshift(subreg, umin_val); in scalar32_min_max_rsh()
11169 dst_reg->u32_min_value >>= umax_val; in scalar32_min_max_rsh()
11170 dst_reg->u32_max_value >>= umin_val; in scalar32_min_max_rsh()
11172 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_rsh()
11173 __update_reg32_bounds(dst_reg); in scalar32_min_max_rsh()
11176 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar_min_max_rsh() argument
11196 dst_reg->smin_value = S64_MIN; in scalar_min_max_rsh()
11197 dst_reg->smax_value = S64_MAX; in scalar_min_max_rsh()
11198 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); in scalar_min_max_rsh()
11199 dst_reg->umin_value >>= umax_val; in scalar_min_max_rsh()
11200 dst_reg->umax_value >>= umin_val; in scalar_min_max_rsh()
11206 __mark_reg32_unbounded(dst_reg); in scalar_min_max_rsh()
11207 __update_reg_bounds(dst_reg); in scalar_min_max_rsh()
11210 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_arsh() argument
11218 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); in scalar32_min_max_arsh()
11219 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); in scalar32_min_max_arsh()
11221 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); in scalar32_min_max_arsh()
11226 dst_reg->u32_min_value = 0; in scalar32_min_max_arsh()
11227 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_arsh()
11229 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_arsh()
11230 __update_reg32_bounds(dst_reg); in scalar32_min_max_arsh()
11233 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar_min_max_arsh() argument
11241 dst_reg->smin_value >>= umin_val; in scalar_min_max_arsh()
11242 dst_reg->smax_value >>= umin_val; in scalar_min_max_arsh()
11244 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); in scalar_min_max_arsh()
11249 dst_reg->umin_value = 0; in scalar_min_max_arsh()
11250 dst_reg->umax_value = U64_MAX; in scalar_min_max_arsh()
11256 __mark_reg32_unbounded(dst_reg); in scalar_min_max_arsh()
11257 __update_reg_bounds(dst_reg); in scalar_min_max_arsh()
11266 struct bpf_reg_state *dst_reg, in adjust_scalar_min_max_vals() argument
11298 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
11309 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
11316 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
11342 scalar32_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11343 scalar_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11344 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
11347 scalar32_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11348 scalar_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11349 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
11352 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
11353 scalar32_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11354 scalar_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11357 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
11358 scalar32_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11359 scalar_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11362 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
11363 scalar32_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11364 scalar_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11367 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
11368 scalar32_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11369 scalar_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11376 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
11380 scalar32_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11382 scalar_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11389 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
11393 scalar32_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11395 scalar_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11402 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
11406 scalar32_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11408 scalar_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
11411 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
11417 zext_32_to_64(dst_reg); in adjust_scalar_min_max_vals()
11418 reg_bounds_sync(dst_reg); in adjust_scalar_min_max_vals()
11430 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; in adjust_reg_min_max_vals() local
11435 dst_reg = ®s[insn->dst_reg]; in adjust_reg_min_max_vals()
11437 if (dst_reg->type != SCALAR_VALUE) in adjust_reg_min_max_vals()
11438 ptr_reg = dst_reg; in adjust_reg_min_max_vals()
11443 dst_reg->id = 0; in adjust_reg_min_max_vals()
11447 if (dst_reg->type != SCALAR_VALUE) { in adjust_reg_min_max_vals()
11453 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
11457 insn->dst_reg, in adjust_reg_min_max_vals()
11465 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
11469 src_reg, dst_reg); in adjust_reg_min_max_vals()
11477 dst_reg, src_reg); in adjust_reg_min_max_vals()
11478 } else if (dst_reg->precise) { in adjust_reg_min_max_vals()
11507 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
11535 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
11539 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
11541 insn->dst_reg); in check_alu_op()
11546 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
11570 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
11576 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; in check_alu_op() local
11588 copy_register_state(dst_reg, src_reg); in check_alu_op()
11589 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
11590 dst_reg->subreg_def = DEF_NOT_SUBREG; in check_alu_op()
11599 copy_register_state(dst_reg, src_reg); in check_alu_op()
11604 dst_reg->id = 0; in check_alu_op()
11605 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
11606 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
11609 insn->dst_reg); in check_alu_op()
11611 zext_32_to_64(dst_reg); in check_alu_op()
11612 reg_bounds_sync(dst_reg); in check_alu_op()
11619 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
11620 regs[insn->dst_reg].type = SCALAR_VALUE; in check_alu_op()
11622 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
11625 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
11653 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
11674 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
11685 struct bpf_reg_state *dst_reg, in find_good_pkt_pointers() argument
11693 if (dst_reg->off < 0 || in find_good_pkt_pointers()
11694 (dst_reg->off == 0 && range_right_open)) in find_good_pkt_pointers()
11698 if (dst_reg->umax_value > MAX_PACKET_OFF || in find_good_pkt_pointers()
11699 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) in find_good_pkt_pointers()
11705 new_range = dst_reg->off; in find_good_pkt_pointers()
11757 if (reg->type == type && reg->id == dst_reg->id) in find_good_pkt_pointers()
11965 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, in is_pkt_ptr_branch_taken() argument
11972 pkt = dst_reg; in is_pkt_ptr_branch_taken()
11973 } else if (dst_reg->type == PTR_TO_PACKET_END) { in is_pkt_ptr_branch_taken()
12185 struct bpf_reg_state *dst_reg) in __reg_combine_min_max() argument
12187 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, in __reg_combine_min_max()
12188 dst_reg->umin_value); in __reg_combine_min_max()
12189 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, in __reg_combine_min_max()
12190 dst_reg->umax_value); in __reg_combine_min_max()
12191 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, in __reg_combine_min_max()
12192 dst_reg->smin_value); in __reg_combine_min_max()
12193 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, in __reg_combine_min_max()
12194 dst_reg->smax_value); in __reg_combine_min_max()
12195 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, in __reg_combine_min_max()
12196 dst_reg->var_off); in __reg_combine_min_max()
12198 reg_bounds_sync(dst_reg); in __reg_combine_min_max()
12288 struct bpf_reg_state *dst_reg, in try_match_pkt_pointers() argument
12302 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
12304 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
12307 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
12308 dst_reg->type, false); in try_match_pkt_pointers()
12309 mark_pkt_end(other_branch, insn->dst_reg, true); in try_match_pkt_pointers()
12310 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
12312 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
12323 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
12325 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
12328 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
12329 dst_reg->type, true); in try_match_pkt_pointers()
12330 mark_pkt_end(this_branch, insn->dst_reg, false); in try_match_pkt_pointers()
12331 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
12333 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
12344 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
12346 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
12349 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
12350 dst_reg->type, true); in try_match_pkt_pointers()
12351 mark_pkt_end(other_branch, insn->dst_reg, false); in try_match_pkt_pointers()
12352 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
12354 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
12365 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
12367 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
12370 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
12371 dst_reg->type, false); in try_match_pkt_pointers()
12372 mark_pkt_end(this_branch, insn->dst_reg, true); in try_match_pkt_pointers()
12373 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
12375 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
12410 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; in check_cond_jmp_op() local
12448 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
12452 dst_reg = ®s[insn->dst_reg]; in check_cond_jmp_op()
12456 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); in check_cond_jmp_op()
12459 pred = is_branch_taken(dst_reg, in check_cond_jmp_op()
12465 pred = is_branch_taken(dst_reg, in check_cond_jmp_op()
12469 } else if (reg_is_pkt_pointer_any(dst_reg) && in check_cond_jmp_op()
12472 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); in check_cond_jmp_op()
12479 if (!__is_pointer_value(false, dst_reg)) in check_cond_jmp_op()
12480 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
12528 if (dst_reg->type == SCALAR_VALUE && in check_cond_jmp_op()
12533 reg_set_min_max(&other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
12534 dst_reg, in check_cond_jmp_op()
12538 else if (tnum_is_const(dst_reg->var_off) || in check_cond_jmp_op()
12540 tnum_is_const(tnum_subreg(dst_reg->var_off)))) in check_cond_jmp_op()
12543 dst_reg->var_off.value, in check_cond_jmp_op()
12544 tnum_subreg(dst_reg->var_off).value, in check_cond_jmp_op()
12550 &other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
12551 src_reg, dst_reg, opcode); in check_cond_jmp_op()
12559 } else if (dst_reg->type == SCALAR_VALUE) { in check_cond_jmp_op()
12560 reg_set_min_max(&other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
12561 dst_reg, insn->imm, (u32)insn->imm, in check_cond_jmp_op()
12565 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && in check_cond_jmp_op()
12566 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { in check_cond_jmp_op()
12567 find_equal_scalars(this_branch, dst_reg); in check_cond_jmp_op()
12568 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); in check_cond_jmp_op()
12584 __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && in check_cond_jmp_op()
12585 type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && in check_cond_jmp_op()
12587 base_type(dst_reg->type) != PTR_TO_BTF_ID) { in check_cond_jmp_op()
12604 mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); in check_cond_jmp_op()
12614 type_may_be_null(dst_reg->type)) { in check_cond_jmp_op()
12618 mark_ptr_or_null_regs(this_branch, insn->dst_reg, in check_cond_jmp_op()
12620 mark_ptr_or_null_regs(other_branch, insn->dst_reg, in check_cond_jmp_op()
12622 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], in check_cond_jmp_op()
12624 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
12626 insn->dst_reg); in check_cond_jmp_op()
12639 struct bpf_reg_state *dst_reg; in check_ld_imm() local
12652 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
12656 dst_reg = ®s[insn->dst_reg]; in check_ld_imm()
12660 dst_reg->type = SCALAR_VALUE; in check_ld_imm()
12661 __mark_reg_known(®s[insn->dst_reg], imm); in check_ld_imm()
12669 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
12672 dst_reg->type = aux->btf_var.reg_type; in check_ld_imm()
12673 switch (base_type(dst_reg->type)) { in check_ld_imm()
12675 dst_reg->mem_size = aux->btf_var.mem_size; in check_ld_imm()
12678 dst_reg->btf = aux->btf_var.btf; in check_ld_imm()
12679 dst_reg->btf_id = aux->btf_var.btf_id; in check_ld_imm()
12702 dst_reg->type = PTR_TO_FUNC; in check_ld_imm()
12703 dst_reg->subprogno = subprogno; in check_ld_imm()
12708 dst_reg->map_ptr = map; in check_ld_imm()
12712 dst_reg->type = PTR_TO_MAP_VALUE; in check_ld_imm()
12713 dst_reg->off = aux->map_off; in check_ld_imm()
12718 dst_reg->type = CONST_PTR_TO_MAP; in check_ld_imm()
12771 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || in check_ld_abs()
14621 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
14632 BPF_READ, insn->dst_reg, false); in do_check()
14678 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
14682 dst_reg_type = regs[insn->dst_reg].type; in do_check()
14685 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
14707 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
14711 if (is_ctx_reg(env, insn->dst_reg)) { in do_check()
14713 insn->dst_reg, in do_check()
14714 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in do_check()
14719 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
14736 insn->dst_reg != BPF_REG_0 || in do_check()
14763 insn->dst_reg != BPF_REG_0 || in do_check()
14776 insn->dst_reg != BPF_REG_0 || in do_check()
15152 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || in resolve_pseudo_ldimm64()
15744 rnd_hi32_patch[3].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
15773 zext_patch[1].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
15959 insn->dst_reg, in convert_ctx_accesses()
15961 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
15966 insn->dst_reg, in convert_ctx_accesses()
15968 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
16360 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), in do_misc_fixups()
16371 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), in do_misc_fixups()
16427 off_reg = issrc ? insn->src_reg : insn->dst_reg; in do_misc_fixups()
16441 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); in do_misc_fixups()