1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2019 Intel Corporation. */
3 
4 #include <linux/hash.h>
5 #include <linux/bpf.h>
6 #include <linux/filter.h>
7 #include <linux/static_call.h>
8 
9 /* The BPF dispatcher is a multiway branch code generator. The
10  * dispatcher is a mechanism to avoid the performance penalty of an
11  * indirect call, which is expensive when retpolines are enabled. A
12  * dispatch client registers a BPF program into the dispatcher, and if
13  * there is available room in the dispatcher a direct call to the BPF
14  * program will be generated. All calls to the BPF programs called via
15  * the dispatcher will then be a direct call, instead of an
16  * indirect. The dispatcher hijacks a trampoline function it via the
17  * __fentry__ of the trampoline. The trampoline function has the
18  * following signature:
19  *
20  * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
21  *                         unsigned int (*bpf_func)(const void *,
22  *                                                  const struct bpf_insn *));
23  */
24 
bpf_dispatcher_find_prog(struct bpf_dispatcher * d,struct bpf_prog * prog)25 static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
26 	struct bpf_dispatcher *d, struct bpf_prog *prog)
27 {
28 	int i;
29 
30 	for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
31 		if (prog == d->progs[i].prog)
32 			return &d->progs[i];
33 	}
34 	return NULL;
35 }
36 
bpf_dispatcher_find_free(struct bpf_dispatcher * d)37 static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
38 	struct bpf_dispatcher *d)
39 {
40 	return bpf_dispatcher_find_prog(d, NULL);
41 }
42 
bpf_dispatcher_add_prog(struct bpf_dispatcher * d,struct bpf_prog * prog)43 static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
44 				    struct bpf_prog *prog)
45 {
46 	struct bpf_dispatcher_prog *entry;
47 
48 	if (!prog)
49 		return false;
50 
51 	entry = bpf_dispatcher_find_prog(d, prog);
52 	if (entry) {
53 		refcount_inc(&entry->users);
54 		return false;
55 	}
56 
57 	entry = bpf_dispatcher_find_free(d);
58 	if (!entry)
59 		return false;
60 
61 	bpf_prog_inc(prog);
62 	entry->prog = prog;
63 	refcount_set(&entry->users, 1);
64 	d->num_progs++;
65 	return true;
66 }
67 
bpf_dispatcher_remove_prog(struct bpf_dispatcher * d,struct bpf_prog * prog)68 static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
69 				       struct bpf_prog *prog)
70 {
71 	struct bpf_dispatcher_prog *entry;
72 
73 	if (!prog)
74 		return false;
75 
76 	entry = bpf_dispatcher_find_prog(d, prog);
77 	if (!entry)
78 		return false;
79 
80 	if (refcount_dec_and_test(&entry->users)) {
81 		entry->prog = NULL;
82 		bpf_prog_put(prog);
83 		d->num_progs--;
84 		return true;
85 	}
86 	return false;
87 }
88 
arch_prepare_bpf_dispatcher(void * image,void * buf,s64 * funcs,int num_funcs)89 int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs)
90 {
91 	return -ENOTSUPP;
92 }
93 
bpf_dispatcher_prepare(struct bpf_dispatcher * d,void * image,void * buf)94 static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf)
95 {
96 	s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
97 	int i;
98 
99 	for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
100 		if (d->progs[i].prog)
101 			*ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
102 	}
103 	return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs);
104 }
105 
bpf_dispatcher_update(struct bpf_dispatcher * d,int prev_num_progs)106 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
107 {
108 	void *new, *tmp;
109 	u32 noff = 0;
110 
111 	if (prev_num_progs)
112 		noff = d->image_off ^ (PAGE_SIZE / 2);
113 
114 	new = d->num_progs ? d->image + noff : NULL;
115 	tmp = d->num_progs ? d->rw_image + noff : NULL;
116 	if (new) {
117 		/* Prepare the dispatcher in d->rw_image. Then use
118 		 * bpf_arch_text_copy to update d->image, which is RO+X.
119 		 */
120 		if (bpf_dispatcher_prepare(d, new, tmp))
121 			return;
122 		if (IS_ERR(bpf_arch_text_copy(new, tmp, PAGE_SIZE / 2)))
123 			return;
124 	}
125 
126 	__BPF_DISPATCHER_UPDATE(d, new ?: (void *)&bpf_dispatcher_nop_func);
127 
128 	/* Make sure all the callers executing the previous/old half of the
129 	 * image leave it, so following update call can modify it safely.
130 	 */
131 	synchronize_rcu();
132 
133 	if (new)
134 		d->image_off = noff;
135 }
136 
bpf_dispatcher_change_prog(struct bpf_dispatcher * d,struct bpf_prog * from,struct bpf_prog * to)137 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
138 				struct bpf_prog *to)
139 {
140 	bool changed = false;
141 	int prev_num_progs;
142 
143 	if (from == to)
144 		return;
145 
146 	mutex_lock(&d->mutex);
147 	if (!d->image) {
148 		d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);
149 		if (!d->image)
150 			goto out;
151 		d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);
152 		if (!d->rw_image) {
153 			u32 size = PAGE_SIZE;
154 
155 			bpf_arch_text_copy(d->image, &size, sizeof(size));
156 			bpf_prog_pack_free((struct bpf_binary_header *)d->image);
157 			d->image = NULL;
158 			goto out;
159 		}
160 		bpf_image_ksym_add(d->image, &d->ksym);
161 	}
162 
163 	prev_num_progs = d->num_progs;
164 	changed |= bpf_dispatcher_remove_prog(d, from);
165 	changed |= bpf_dispatcher_add_prog(d, to);
166 
167 	if (!changed)
168 		goto out;
169 
170 	bpf_dispatcher_update(d, prev_num_progs);
171 out:
172 	mutex_unlock(&d->mutex);
173 }
174