1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Common functionality for RV32 and RV64 BPF JIT compilers
4 *
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
6 *
7 */
8
9 #include <linux/bpf.h>
10 #include <linux/filter.h>
11 #include "bpf_jit.h"
12
13 /* Number of iterations to try until offsets converge. */
14 #define NR_JIT_ITERATIONS 32
15
build_body(struct rv_jit_context * ctx,bool extra_pass,int * offset)16 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
17 {
18 const struct bpf_prog *prog = ctx->prog;
19 int i;
20
21 for (i = 0; i < prog->len; i++) {
22 const struct bpf_insn *insn = &prog->insnsi[i];
23 int ret;
24
25 ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
26 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
27 if (ret > 0)
28 i++;
29 if (offset)
30 offset[i] = ctx->ninsns;
31 if (ret < 0)
32 return ret;
33 }
34 return 0;
35 }
36
bpf_jit_needs_zext(void)37 bool bpf_jit_needs_zext(void)
38 {
39 return true;
40 }
41
bpf_int_jit_compile(struct bpf_prog * prog)42 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
43 {
44 unsigned int prog_size = 0, extable_size = 0;
45 bool tmp_blinded = false, extra_pass = false;
46 struct bpf_prog *tmp, *orig_prog = prog;
47 int pass = 0, prev_ninsns = 0, prologue_len, i;
48 struct rv_jit_data *jit_data;
49 struct rv_jit_context *ctx;
50
51 if (!prog->jit_requested)
52 return orig_prog;
53
54 tmp = bpf_jit_blind_constants(prog);
55 if (IS_ERR(tmp))
56 return orig_prog;
57 if (tmp != prog) {
58 tmp_blinded = true;
59 prog = tmp;
60 }
61
62 jit_data = prog->aux->jit_data;
63 if (!jit_data) {
64 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
65 if (!jit_data) {
66 prog = orig_prog;
67 goto out;
68 }
69 prog->aux->jit_data = jit_data;
70 }
71
72 ctx = &jit_data->ctx;
73
74 if (ctx->offset) {
75 extra_pass = true;
76 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
77 goto skip_init_ctx;
78 }
79
80 ctx->prog = prog;
81 ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
82 if (!ctx->offset) {
83 prog = orig_prog;
84 goto out_offset;
85 }
86 for (i = 0; i < prog->len; i++) {
87 prev_ninsns += 32;
88 ctx->offset[i] = prev_ninsns;
89 }
90
91 for (i = 0; i < NR_JIT_ITERATIONS; i++) {
92 pass++;
93 ctx->ninsns = 0;
94 if (build_body(ctx, extra_pass, ctx->offset)) {
95 prog = orig_prog;
96 goto out_offset;
97 }
98 ctx->body_len = ctx->ninsns;
99 bpf_jit_build_prologue(ctx);
100 ctx->epilogue_offset = ctx->ninsns;
101 bpf_jit_build_epilogue(ctx);
102
103 if (ctx->ninsns == prev_ninsns) {
104 if (jit_data->header)
105 break;
106 /* obtain the actual image size */
107 extable_size = prog->aux->num_exentries *
108 sizeof(struct exception_table_entry);
109 prog_size = sizeof(*ctx->insns) * ctx->ninsns;
110
111 jit_data->header =
112 bpf_jit_binary_alloc(prog_size + extable_size,
113 &jit_data->image,
114 sizeof(u32),
115 bpf_fill_ill_insns);
116 if (!jit_data->header) {
117 prog = orig_prog;
118 goto out_offset;
119 }
120
121 ctx->insns = (u16 *)jit_data->image;
122 /*
123 * Now, when the image is allocated, the image can
124 * potentially shrink more (auipc/jalr -> jal).
125 */
126 }
127 prev_ninsns = ctx->ninsns;
128 }
129
130 if (i == NR_JIT_ITERATIONS) {
131 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
132 if (jit_data->header)
133 bpf_jit_binary_free(jit_data->header);
134 prog = orig_prog;
135 goto out_offset;
136 }
137
138 if (extable_size)
139 prog->aux->extable = (void *)ctx->insns + prog_size;
140
141 skip_init_ctx:
142 pass++;
143 ctx->ninsns = 0;
144 ctx->nexentries = 0;
145
146 bpf_jit_build_prologue(ctx);
147 if (build_body(ctx, extra_pass, NULL)) {
148 bpf_jit_binary_free(jit_data->header);
149 prog = orig_prog;
150 goto out_offset;
151 }
152 bpf_jit_build_epilogue(ctx);
153
154 if (bpf_jit_enable > 1)
155 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
156
157 prog->bpf_func = (void *)ctx->insns;
158 prog->jited = 1;
159 prog->jited_len = prog_size;
160
161 bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
162
163 if (!prog->is_func || extra_pass) {
164 bpf_jit_binary_lock_ro(jit_data->header);
165 prologue_len = ctx->epilogue_offset - ctx->body_len;
166 for (i = 0; i < prog->len; i++)
167 ctx->offset[i] = ninsns_rvoff(prologue_len +
168 ctx->offset[i]);
169 bpf_prog_fill_jited_linfo(prog, ctx->offset);
170 out_offset:
171 kfree(ctx->offset);
172 kfree(jit_data);
173 prog->aux->jit_data = NULL;
174 }
175 out:
176
177 if (tmp_blinded)
178 bpf_jit_prog_release_other(prog, prog == orig_prog ?
179 tmp : orig_prog);
180 return prog;
181 }
182
bpf_jit_alloc_exec_limit(void)183 u64 bpf_jit_alloc_exec_limit(void)
184 {
185 return BPF_JIT_REGION_SIZE;
186 }
187
bpf_jit_alloc_exec(unsigned long size)188 void *bpf_jit_alloc_exec(unsigned long size)
189 {
190 return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
191 BPF_JIT_REGION_END, GFP_KERNEL,
192 PAGE_KERNEL, 0, NUMA_NO_NODE,
193 __builtin_return_address(0));
194 }
195
bpf_jit_free_exec(void * addr)196 void bpf_jit_free_exec(void *addr)
197 {
198 return vfree(addr);
199 }
200