1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Unstable Conntrack Helpers for XDP and TC-BPF hook
3 *
4 * These are called from the XDP and SCHED_CLS BPF programs. Note that it is
5 * allowed to break compatibility for these functions since the interface they
6 * are exposed through to BPF programs is explicitly unstable.
7 */
8
9 #include <linux/bpf_verifier.h>
10 #include <linux/bpf.h>
11 #include <linux/btf.h>
12 #include <linux/filter.h>
13 #include <linux/mutex.h>
14 #include <linux/types.h>
15 #include <linux/btf_ids.h>
16 #include <linux/net_namespace.h>
17 #include <net/netfilter/nf_conntrack_bpf.h>
18 #include <net/netfilter/nf_conntrack_core.h>
19
20 /* bpf_ct_opts - Options for CT lookup helpers
21 *
22 * Members:
23 * @netns_id - Specify the network namespace for lookup
24 * Values:
25 * BPF_F_CURRENT_NETNS (-1)
26 * Use namespace associated with ctx (xdp_md, __sk_buff)
27 * [0, S32_MAX]
28 * Network Namespace ID
29 * @error - Out parameter, set for any errors encountered
30 * Values:
31 * -EINVAL - Passed NULL for bpf_tuple pointer
32 * -EINVAL - opts->reserved is not 0
33 * -EINVAL - netns_id is less than -1
34 * -EINVAL - opts__sz isn't NF_BPF_CT_OPTS_SZ (12)
35 * -EPROTO - l4proto isn't one of IPPROTO_TCP or IPPROTO_UDP
36 * -ENONET - No network namespace found for netns_id
37 * -ENOENT - Conntrack lookup could not find entry for tuple
38 * -EAFNOSUPPORT - tuple__sz isn't one of sizeof(tuple->ipv4)
39 * or sizeof(tuple->ipv6)
40 * @l4proto - Layer 4 protocol
41 * Values:
42 * IPPROTO_TCP, IPPROTO_UDP
43 * @dir: - connection tracking tuple direction.
44 * @reserved - Reserved member, will be reused for more options in future
45 * Values:
46 * 0
47 */
48 struct bpf_ct_opts {
49 s32 netns_id;
50 s32 error;
51 u8 l4proto;
52 u8 dir;
53 u8 reserved[2];
54 };
55
56 enum {
57 NF_BPF_CT_OPTS_SZ = 12,
58 };
59
bpf_nf_ct_tuple_parse(struct bpf_sock_tuple * bpf_tuple,u32 tuple_len,u8 protonum,u8 dir,struct nf_conntrack_tuple * tuple)60 static int bpf_nf_ct_tuple_parse(struct bpf_sock_tuple *bpf_tuple,
61 u32 tuple_len, u8 protonum, u8 dir,
62 struct nf_conntrack_tuple *tuple)
63 {
64 union nf_inet_addr *src = dir ? &tuple->dst.u3 : &tuple->src.u3;
65 union nf_inet_addr *dst = dir ? &tuple->src.u3 : &tuple->dst.u3;
66 union nf_conntrack_man_proto *sport = dir ? (void *)&tuple->dst.u
67 : &tuple->src.u;
68 union nf_conntrack_man_proto *dport = dir ? &tuple->src.u
69 : (void *)&tuple->dst.u;
70
71 if (unlikely(protonum != IPPROTO_TCP && protonum != IPPROTO_UDP))
72 return -EPROTO;
73
74 memset(tuple, 0, sizeof(*tuple));
75
76 switch (tuple_len) {
77 case sizeof(bpf_tuple->ipv4):
78 tuple->src.l3num = AF_INET;
79 src->ip = bpf_tuple->ipv4.saddr;
80 sport->tcp.port = bpf_tuple->ipv4.sport;
81 dst->ip = bpf_tuple->ipv4.daddr;
82 dport->tcp.port = bpf_tuple->ipv4.dport;
83 break;
84 case sizeof(bpf_tuple->ipv6):
85 tuple->src.l3num = AF_INET6;
86 memcpy(src->ip6, bpf_tuple->ipv6.saddr, sizeof(bpf_tuple->ipv6.saddr));
87 sport->tcp.port = bpf_tuple->ipv6.sport;
88 memcpy(dst->ip6, bpf_tuple->ipv6.daddr, sizeof(bpf_tuple->ipv6.daddr));
89 dport->tcp.port = bpf_tuple->ipv6.dport;
90 break;
91 default:
92 return -EAFNOSUPPORT;
93 }
94 tuple->dst.protonum = protonum;
95 tuple->dst.dir = dir;
96
97 return 0;
98 }
99
100 static struct nf_conn *
__bpf_nf_ct_alloc_entry(struct net * net,struct bpf_sock_tuple * bpf_tuple,u32 tuple_len,struct bpf_ct_opts * opts,u32 opts_len,u32 timeout)101 __bpf_nf_ct_alloc_entry(struct net *net, struct bpf_sock_tuple *bpf_tuple,
102 u32 tuple_len, struct bpf_ct_opts *opts, u32 opts_len,
103 u32 timeout)
104 {
105 struct nf_conntrack_tuple otuple, rtuple;
106 struct nf_conn *ct;
107 int err;
108
109 if (!opts || !bpf_tuple || opts->reserved[0] || opts->reserved[1] ||
110 opts_len != NF_BPF_CT_OPTS_SZ)
111 return ERR_PTR(-EINVAL);
112
113 if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS))
114 return ERR_PTR(-EINVAL);
115
116 err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto,
117 IP_CT_DIR_ORIGINAL, &otuple);
118 if (err < 0)
119 return ERR_PTR(err);
120
121 err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto,
122 IP_CT_DIR_REPLY, &rtuple);
123 if (err < 0)
124 return ERR_PTR(err);
125
126 if (opts->netns_id >= 0) {
127 net = get_net_ns_by_id(net, opts->netns_id);
128 if (unlikely(!net))
129 return ERR_PTR(-ENONET);
130 }
131
132 ct = nf_conntrack_alloc(net, &nf_ct_zone_dflt, &otuple, &rtuple,
133 GFP_ATOMIC);
134 if (IS_ERR(ct))
135 goto out;
136
137 memset(&ct->proto, 0, sizeof(ct->proto));
138 __nf_ct_set_timeout(ct, timeout * HZ);
139
140 out:
141 if (opts->netns_id >= 0)
142 put_net(net);
143
144 return ct;
145 }
146
__bpf_nf_ct_lookup(struct net * net,struct bpf_sock_tuple * bpf_tuple,u32 tuple_len,struct bpf_ct_opts * opts,u32 opts_len)147 static struct nf_conn *__bpf_nf_ct_lookup(struct net *net,
148 struct bpf_sock_tuple *bpf_tuple,
149 u32 tuple_len, struct bpf_ct_opts *opts,
150 u32 opts_len)
151 {
152 struct nf_conntrack_tuple_hash *hash;
153 struct nf_conntrack_tuple tuple;
154 struct nf_conn *ct;
155 int err;
156
157 if (!opts || !bpf_tuple || opts->reserved[0] || opts->reserved[1] ||
158 opts_len != NF_BPF_CT_OPTS_SZ)
159 return ERR_PTR(-EINVAL);
160 if (unlikely(opts->l4proto != IPPROTO_TCP && opts->l4proto != IPPROTO_UDP))
161 return ERR_PTR(-EPROTO);
162 if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS))
163 return ERR_PTR(-EINVAL);
164
165 err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto,
166 IP_CT_DIR_ORIGINAL, &tuple);
167 if (err < 0)
168 return ERR_PTR(err);
169
170 if (opts->netns_id >= 0) {
171 net = get_net_ns_by_id(net, opts->netns_id);
172 if (unlikely(!net))
173 return ERR_PTR(-ENONET);
174 }
175
176 hash = nf_conntrack_find_get(net, &nf_ct_zone_dflt, &tuple);
177 if (opts->netns_id >= 0)
178 put_net(net);
179 if (!hash)
180 return ERR_PTR(-ENOENT);
181
182 ct = nf_ct_tuplehash_to_ctrack(hash);
183 opts->dir = NF_CT_DIRECTION(hash);
184
185 return ct;
186 }
187
188 BTF_ID_LIST(btf_nf_conn_ids)
BTF_ID(struct,nf_conn)189 BTF_ID(struct, nf_conn)
190 BTF_ID(struct, nf_conn___init)
191
192 /* Check writes into `struct nf_conn` */
193 static int _nf_conntrack_btf_struct_access(struct bpf_verifier_log *log,
194 const struct bpf_reg_state *reg,
195 int off, int size, enum bpf_access_type atype,
196 u32 *next_btf_id, enum bpf_type_flag *flag)
197 {
198 const struct btf_type *ncit, *nct, *t;
199 size_t end;
200
201 ncit = btf_type_by_id(reg->btf, btf_nf_conn_ids[1]);
202 nct = btf_type_by_id(reg->btf, btf_nf_conn_ids[0]);
203 t = btf_type_by_id(reg->btf, reg->btf_id);
204 if (t != nct && t != ncit) {
205 bpf_log(log, "only read is supported\n");
206 return -EACCES;
207 }
208
209 /* `struct nf_conn` and `struct nf_conn___init` have the same layout
210 * so we are safe to simply merge offset checks here
211 */
212 switch (off) {
213 #if defined(CONFIG_NF_CONNTRACK_MARK)
214 case offsetof(struct nf_conn, mark):
215 end = offsetofend(struct nf_conn, mark);
216 break;
217 #endif
218 default:
219 bpf_log(log, "no write support to nf_conn at off %d\n", off);
220 return -EACCES;
221 }
222
223 if (off + size > end) {
224 bpf_log(log,
225 "write access at off %d with size %d beyond the member of nf_conn ended at %zu\n",
226 off, size, end);
227 return -EACCES;
228 }
229
230 return 0;
231 }
232
233 __diag_push();
234 __diag_ignore_all("-Wmissing-prototypes",
235 "Global functions as their definitions will be in nf_conntrack BTF");
236
237 /* bpf_xdp_ct_alloc - Allocate a new CT entry
238 *
239 * Parameters:
240 * @xdp_ctx - Pointer to ctx (xdp_md) in XDP program
241 * Cannot be NULL
242 * @bpf_tuple - Pointer to memory representing the tuple to look up
243 * Cannot be NULL
244 * @tuple__sz - Length of the tuple structure
245 * Must be one of sizeof(bpf_tuple->ipv4) or
246 * sizeof(bpf_tuple->ipv6)
247 * @opts - Additional options for allocation (documented above)
248 * Cannot be NULL
249 * @opts__sz - Length of the bpf_ct_opts structure
250 * Must be NF_BPF_CT_OPTS_SZ (12)
251 */
252 __bpf_kfunc struct nf_conn___init *
bpf_xdp_ct_alloc(struct xdp_md * xdp_ctx,struct bpf_sock_tuple * bpf_tuple,u32 tuple__sz,struct bpf_ct_opts * opts,u32 opts__sz)253 bpf_xdp_ct_alloc(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple,
254 u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz)
255 {
256 struct xdp_buff *ctx = (struct xdp_buff *)xdp_ctx;
257 struct nf_conn *nfct;
258
259 nfct = __bpf_nf_ct_alloc_entry(dev_net(ctx->rxq->dev), bpf_tuple, tuple__sz,
260 opts, opts__sz, 10);
261 if (IS_ERR(nfct)) {
262 if (opts)
263 opts->error = PTR_ERR(nfct);
264 return NULL;
265 }
266
267 return (struct nf_conn___init *)nfct;
268 }
269
270 /* bpf_xdp_ct_lookup - Lookup CT entry for the given tuple, and acquire a
271 * reference to it
272 *
273 * Parameters:
274 * @xdp_ctx - Pointer to ctx (xdp_md) in XDP program
275 * Cannot be NULL
276 * @bpf_tuple - Pointer to memory representing the tuple to look up
277 * Cannot be NULL
278 * @tuple__sz - Length of the tuple structure
279 * Must be one of sizeof(bpf_tuple->ipv4) or
280 * sizeof(bpf_tuple->ipv6)
281 * @opts - Additional options for lookup (documented above)
282 * Cannot be NULL
283 * @opts__sz - Length of the bpf_ct_opts structure
284 * Must be NF_BPF_CT_OPTS_SZ (12)
285 */
286 __bpf_kfunc struct nf_conn *
bpf_xdp_ct_lookup(struct xdp_md * xdp_ctx,struct bpf_sock_tuple * bpf_tuple,u32 tuple__sz,struct bpf_ct_opts * opts,u32 opts__sz)287 bpf_xdp_ct_lookup(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple,
288 u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz)
289 {
290 struct xdp_buff *ctx = (struct xdp_buff *)xdp_ctx;
291 struct net *caller_net;
292 struct nf_conn *nfct;
293
294 caller_net = dev_net(ctx->rxq->dev);
295 nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts, opts__sz);
296 if (IS_ERR(nfct)) {
297 if (opts)
298 opts->error = PTR_ERR(nfct);
299 return NULL;
300 }
301 return nfct;
302 }
303
304 /* bpf_skb_ct_alloc - Allocate a new CT entry
305 *
306 * Parameters:
307 * @skb_ctx - Pointer to ctx (__sk_buff) in TC program
308 * Cannot be NULL
309 * @bpf_tuple - Pointer to memory representing the tuple to look up
310 * Cannot be NULL
311 * @tuple__sz - Length of the tuple structure
312 * Must be one of sizeof(bpf_tuple->ipv4) or
313 * sizeof(bpf_tuple->ipv6)
314 * @opts - Additional options for allocation (documented above)
315 * Cannot be NULL
316 * @opts__sz - Length of the bpf_ct_opts structure
317 * Must be NF_BPF_CT_OPTS_SZ (12)
318 */
319 __bpf_kfunc struct nf_conn___init *
bpf_skb_ct_alloc(struct __sk_buff * skb_ctx,struct bpf_sock_tuple * bpf_tuple,u32 tuple__sz,struct bpf_ct_opts * opts,u32 opts__sz)320 bpf_skb_ct_alloc(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple,
321 u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz)
322 {
323 struct sk_buff *skb = (struct sk_buff *)skb_ctx;
324 struct nf_conn *nfct;
325 struct net *net;
326
327 net = skb->dev ? dev_net(skb->dev) : sock_net(skb->sk);
328 nfct = __bpf_nf_ct_alloc_entry(net, bpf_tuple, tuple__sz, opts, opts__sz, 10);
329 if (IS_ERR(nfct)) {
330 if (opts)
331 opts->error = PTR_ERR(nfct);
332 return NULL;
333 }
334
335 return (struct nf_conn___init *)nfct;
336 }
337
338 /* bpf_skb_ct_lookup - Lookup CT entry for the given tuple, and acquire a
339 * reference to it
340 *
341 * Parameters:
342 * @skb_ctx - Pointer to ctx (__sk_buff) in TC program
343 * Cannot be NULL
344 * @bpf_tuple - Pointer to memory representing the tuple to look up
345 * Cannot be NULL
346 * @tuple__sz - Length of the tuple structure
347 * Must be one of sizeof(bpf_tuple->ipv4) or
348 * sizeof(bpf_tuple->ipv6)
349 * @opts - Additional options for lookup (documented above)
350 * Cannot be NULL
351 * @opts__sz - Length of the bpf_ct_opts structure
352 * Must be NF_BPF_CT_OPTS_SZ (12)
353 */
354 __bpf_kfunc struct nf_conn *
bpf_skb_ct_lookup(struct __sk_buff * skb_ctx,struct bpf_sock_tuple * bpf_tuple,u32 tuple__sz,struct bpf_ct_opts * opts,u32 opts__sz)355 bpf_skb_ct_lookup(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple,
356 u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz)
357 {
358 struct sk_buff *skb = (struct sk_buff *)skb_ctx;
359 struct net *caller_net;
360 struct nf_conn *nfct;
361
362 caller_net = skb->dev ? dev_net(skb->dev) : sock_net(skb->sk);
363 nfct = __bpf_nf_ct_lookup(caller_net, bpf_tuple, tuple__sz, opts, opts__sz);
364 if (IS_ERR(nfct)) {
365 if (opts)
366 opts->error = PTR_ERR(nfct);
367 return NULL;
368 }
369 return nfct;
370 }
371
372 /* bpf_ct_insert_entry - Add the provided entry into a CT map
373 *
374 * This must be invoked for referenced PTR_TO_BTF_ID.
375 *
376 * @nfct - Pointer to referenced nf_conn___init object, obtained
377 * using bpf_xdp_ct_alloc or bpf_skb_ct_alloc.
378 */
bpf_ct_insert_entry(struct nf_conn___init * nfct_i)379 __bpf_kfunc struct nf_conn *bpf_ct_insert_entry(struct nf_conn___init *nfct_i)
380 {
381 struct nf_conn *nfct = (struct nf_conn *)nfct_i;
382 int err;
383
384 err = nf_conntrack_hash_check_insert(nfct);
385 if (err < 0) {
386 nf_conntrack_free(nfct);
387 return NULL;
388 }
389 return nfct;
390 }
391
392 /* bpf_ct_release - Release acquired nf_conn object
393 *
394 * This must be invoked for referenced PTR_TO_BTF_ID, and the verifier rejects
395 * the program if any references remain in the program in all of the explored
396 * states.
397 *
398 * Parameters:
399 * @nf_conn - Pointer to referenced nf_conn object, obtained using
400 * bpf_xdp_ct_lookup or bpf_skb_ct_lookup.
401 */
bpf_ct_release(struct nf_conn * nfct)402 __bpf_kfunc void bpf_ct_release(struct nf_conn *nfct)
403 {
404 if (!nfct)
405 return;
406 nf_ct_put(nfct);
407 }
408
409 /* bpf_ct_set_timeout - Set timeout of allocated nf_conn
410 *
411 * Sets the default timeout of newly allocated nf_conn before insertion.
412 * This helper must be invoked for refcounted pointer to nf_conn___init.
413 *
414 * Parameters:
415 * @nfct - Pointer to referenced nf_conn object, obtained using
416 * bpf_xdp_ct_alloc or bpf_skb_ct_alloc.
417 * @timeout - Timeout in msecs.
418 */
bpf_ct_set_timeout(struct nf_conn___init * nfct,u32 timeout)419 __bpf_kfunc void bpf_ct_set_timeout(struct nf_conn___init *nfct, u32 timeout)
420 {
421 __nf_ct_set_timeout((struct nf_conn *)nfct, msecs_to_jiffies(timeout));
422 }
423
424 /* bpf_ct_change_timeout - Change timeout of inserted nf_conn
425 *
426 * Change timeout associated of the inserted or looked up nf_conn.
427 * This helper must be invoked for refcounted pointer to nf_conn.
428 *
429 * Parameters:
430 * @nfct - Pointer to referenced nf_conn object, obtained using
431 * bpf_ct_insert_entry, bpf_xdp_ct_lookup, or bpf_skb_ct_lookup.
432 * @timeout - New timeout in msecs.
433 */
bpf_ct_change_timeout(struct nf_conn * nfct,u32 timeout)434 __bpf_kfunc int bpf_ct_change_timeout(struct nf_conn *nfct, u32 timeout)
435 {
436 return __nf_ct_change_timeout(nfct, msecs_to_jiffies(timeout));
437 }
438
439 /* bpf_ct_set_status - Set status field of allocated nf_conn
440 *
441 * Set the status field of the newly allocated nf_conn before insertion.
442 * This must be invoked for referenced PTR_TO_BTF_ID to nf_conn___init.
443 *
444 * Parameters:
445 * @nfct - Pointer to referenced nf_conn object, obtained using
446 * bpf_xdp_ct_alloc or bpf_skb_ct_alloc.
447 * @status - New status value.
448 */
bpf_ct_set_status(const struct nf_conn___init * nfct,u32 status)449 __bpf_kfunc int bpf_ct_set_status(const struct nf_conn___init *nfct, u32 status)
450 {
451 return nf_ct_change_status_common((struct nf_conn *)nfct, status);
452 }
453
454 /* bpf_ct_change_status - Change status of inserted nf_conn
455 *
456 * Change the status field of the provided connection tracking entry.
457 * This must be invoked for referenced PTR_TO_BTF_ID to nf_conn.
458 *
459 * Parameters:
460 * @nfct - Pointer to referenced nf_conn object, obtained using
461 * bpf_ct_insert_entry, bpf_xdp_ct_lookup or bpf_skb_ct_lookup.
462 * @status - New status value.
463 */
bpf_ct_change_status(struct nf_conn * nfct,u32 status)464 __bpf_kfunc int bpf_ct_change_status(struct nf_conn *nfct, u32 status)
465 {
466 return nf_ct_change_status_common(nfct, status);
467 }
468
469 __diag_pop()
470
471 BTF_SET8_START(nf_ct_kfunc_set)
472 BTF_ID_FLAGS(func, bpf_xdp_ct_alloc, KF_ACQUIRE | KF_RET_NULL)
473 BTF_ID_FLAGS(func, bpf_xdp_ct_lookup, KF_ACQUIRE | KF_RET_NULL)
474 BTF_ID_FLAGS(func, bpf_skb_ct_alloc, KF_ACQUIRE | KF_RET_NULL)
475 BTF_ID_FLAGS(func, bpf_skb_ct_lookup, KF_ACQUIRE | KF_RET_NULL)
476 BTF_ID_FLAGS(func, bpf_ct_insert_entry, KF_ACQUIRE | KF_RET_NULL | KF_RELEASE)
477 BTF_ID_FLAGS(func, bpf_ct_release, KF_RELEASE)
478 BTF_ID_FLAGS(func, bpf_ct_set_timeout, KF_TRUSTED_ARGS)
479 BTF_ID_FLAGS(func, bpf_ct_change_timeout, KF_TRUSTED_ARGS)
480 BTF_ID_FLAGS(func, bpf_ct_set_status, KF_TRUSTED_ARGS)
481 BTF_ID_FLAGS(func, bpf_ct_change_status, KF_TRUSTED_ARGS)
482 BTF_SET8_END(nf_ct_kfunc_set)
483
484 static const struct btf_kfunc_id_set nf_conntrack_kfunc_set = {
485 .owner = THIS_MODULE,
486 .set = &nf_ct_kfunc_set,
487 };
488
register_nf_conntrack_bpf(void)489 int register_nf_conntrack_bpf(void)
490 {
491 int ret;
492
493 ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &nf_conntrack_kfunc_set);
494 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_kfunc_set);
495 if (!ret) {
496 mutex_lock(&nf_conn_btf_access_lock);
497 nfct_btf_struct_access = _nf_conntrack_btf_struct_access;
498 mutex_unlock(&nf_conn_btf_access_lock);
499 }
500
501 return ret;
502 }
503
cleanup_nf_conntrack_bpf(void)504 void cleanup_nf_conntrack_bpf(void)
505 {
506 mutex_lock(&nf_conn_btf_access_lock);
507 nfct_btf_struct_access = NULL;
508 mutex_unlock(&nf_conn_btf_access_lock);
509 }
510