1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <signal.h>
15 #include <fcntl.h>
16 #include <sys/wait.h>
17 #include <time.h>
18 #include <sched.h>
19
20 #include <sys/time.h>
21 #include <sys/resource.h>
22 #include <sys/types.h>
23 #include <sys/sendfile.h>
24
25 #include <linux/netlink.h>
26 #include <linux/socket.h>
27 #include <linux/sock_diag.h>
28 #include <linux/bpf.h>
29 #include <linux/if_link.h>
30 #include <linux/tls.h>
31 #include <assert.h>
32 #include <libgen.h>
33
34 #include <getopt.h>
35
36 #include <bpf/bpf.h>
37 #include <bpf/libbpf.h>
38
39 #include "bpf_util.h"
40 #include "bpf_rlimit.h"
41 #include "cgroup_helpers.h"
42
43 int running;
44 static void running_handler(int a);
45
46 #ifndef TCP_ULP
47 # define TCP_ULP 31
48 #endif
49 #ifndef SOL_TLS
50 # define SOL_TLS 282
51 #endif
52
53 /* randomly selected ports for testing on lo */
54 #define S1_PORT 10000
55 #define S2_PORT 10001
56
57 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
58 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59 #define CG_PATH "/sockmap"
60
61 /* global sockets */
62 int s1, s2, c1, c2, p1, p2;
63 int test_cnt;
64 int passed;
65 int failed;
66 int map_fd[9];
67 struct bpf_map *maps[9];
68 int prog_fd[11];
69
70 int txmsg_pass;
71 int txmsg_redir;
72 int txmsg_drop;
73 int txmsg_apply;
74 int txmsg_cork;
75 int txmsg_start;
76 int txmsg_end;
77 int txmsg_start_push;
78 int txmsg_end_push;
79 int txmsg_start_pop;
80 int txmsg_pop;
81 int txmsg_ingress;
82 int txmsg_redir_skb;
83 int txmsg_ktls_skb;
84 int txmsg_ktls_skb_drop;
85 int txmsg_ktls_skb_redir;
86 int ktls;
87 int peek_flag;
88 int skb_use_parser;
89 int txmsg_omit_skb_parser;
90
91 static const struct option long_options[] = {
92 {"help", no_argument, NULL, 'h' },
93 {"cgroup", required_argument, NULL, 'c' },
94 {"rate", required_argument, NULL, 'r' },
95 {"verbose", optional_argument, NULL, 'v' },
96 {"iov_count", required_argument, NULL, 'i' },
97 {"length", required_argument, NULL, 'l' },
98 {"test", required_argument, NULL, 't' },
99 {"data_test", no_argument, NULL, 'd' },
100 {"txmsg", no_argument, &txmsg_pass, 1 },
101 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
102 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
103 {"txmsg_apply", required_argument, NULL, 'a'},
104 {"txmsg_cork", required_argument, NULL, 'k'},
105 {"txmsg_start", required_argument, NULL, 's'},
106 {"txmsg_end", required_argument, NULL, 'e'},
107 {"txmsg_start_push", required_argument, NULL, 'p'},
108 {"txmsg_end_push", required_argument, NULL, 'q'},
109 {"txmsg_start_pop", required_argument, NULL, 'w'},
110 {"txmsg_pop", required_argument, NULL, 'x'},
111 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
112 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
113 {"ktls", no_argument, &ktls, 1 },
114 {"peek", no_argument, &peek_flag, 1 },
115 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
116 {"whitelist", required_argument, NULL, 'n' },
117 {"blacklist", required_argument, NULL, 'b' },
118 {0, 0, NULL, 0 }
119 };
120
121 struct test_env {
122 const char *type;
123 const char *subtest;
124 const char *prepend;
125
126 int test_num;
127 int subtest_num;
128
129 int succ_cnt;
130 int fail_cnt;
131 int fail_last;
132 };
133
134 struct test_env env;
135
136 struct sockmap_options {
137 int verbose;
138 bool base;
139 bool sendpage;
140 bool data_test;
141 bool drop_expected;
142 bool check_recved_len;
143 int iov_count;
144 int iov_length;
145 int rate;
146 char *map;
147 char *whitelist;
148 char *blacklist;
149 char *prepend;
150 };
151
152 struct _test {
153 char *title;
154 void (*tester)(int cg_fd, struct sockmap_options *opt);
155 };
156
test_start(void)157 static void test_start(void)
158 {
159 env.subtest_num++;
160 }
161
test_fail(void)162 static void test_fail(void)
163 {
164 env.fail_cnt++;
165 }
166
test_pass(void)167 static void test_pass(void)
168 {
169 env.succ_cnt++;
170 }
171
test_reset(void)172 static void test_reset(void)
173 {
174 txmsg_start = txmsg_end = 0;
175 txmsg_start_pop = txmsg_pop = 0;
176 txmsg_start_push = txmsg_end_push = 0;
177 txmsg_pass = txmsg_drop = txmsg_redir = 0;
178 txmsg_apply = txmsg_cork = 0;
179 txmsg_ingress = txmsg_redir_skb = 0;
180 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
181 txmsg_omit_skb_parser = 0;
182 skb_use_parser = 0;
183 }
184
test_start_subtest(const struct _test * t,struct sockmap_options * o)185 static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
186 {
187 env.type = o->map;
188 env.subtest = t->title;
189 env.prepend = o->prepend;
190 env.test_num++;
191 env.subtest_num = 0;
192 env.fail_last = env.fail_cnt;
193 test_reset();
194 return 0;
195 }
196
test_end_subtest(void)197 static void test_end_subtest(void)
198 {
199 int error = env.fail_cnt - env.fail_last;
200 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
201
202 if (!error)
203 test_pass();
204
205 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
206 env.test_num, env.subtest_num,
207 !type ? "sockmap" : "sockhash",
208 env.prepend ? : "",
209 env.subtest, error ? "FAIL" : "OK");
210 }
211
test_print_results(void)212 static void test_print_results(void)
213 {
214 fprintf(stdout, "Pass: %d Fail: %d\n",
215 env.succ_cnt, env.fail_cnt);
216 }
217
usage(char * argv[])218 static void usage(char *argv[])
219 {
220 int i;
221
222 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
223 printf(" options:\n");
224 for (i = 0; long_options[i].name != 0; i++) {
225 printf(" --%-12s", long_options[i].name);
226 if (long_options[i].flag != NULL)
227 printf(" flag (internal value:%d)\n",
228 *long_options[i].flag);
229 else
230 printf(" -%c\n", long_options[i].val);
231 }
232 printf("\n");
233 }
234
sock_to_string(int s)235 char *sock_to_string(int s)
236 {
237 if (s == c1)
238 return "client1";
239 else if (s == c2)
240 return "client2";
241 else if (s == s1)
242 return "server1";
243 else if (s == s2)
244 return "server2";
245 else if (s == p1)
246 return "peer1";
247 else if (s == p2)
248 return "peer2";
249 else
250 return "unknown";
251 }
252
sockmap_init_ktls(int verbose,int s)253 static int sockmap_init_ktls(int verbose, int s)
254 {
255 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
256 .info = {
257 .version = TLS_1_2_VERSION,
258 .cipher_type = TLS_CIPHER_AES_GCM_128,
259 },
260 };
261 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
262 .info = {
263 .version = TLS_1_2_VERSION,
264 .cipher_type = TLS_CIPHER_AES_GCM_128,
265 },
266 };
267 int so_buf = 6553500;
268 int err;
269
270 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
271 if (err) {
272 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
273 return -EINVAL;
274 }
275 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
276 if (err) {
277 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
278 return -EINVAL;
279 }
280 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
281 if (err) {
282 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
283 return -EINVAL;
284 }
285 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
286 if (err) {
287 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
288 return -EINVAL;
289 }
290 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
291 if (err) {
292 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
293 return -EINVAL;
294 }
295
296 if (verbose)
297 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
298 return 0;
299 }
sockmap_init_sockets(int verbose)300 static int sockmap_init_sockets(int verbose)
301 {
302 int i, err, one = 1;
303 struct sockaddr_in addr;
304 int *fds[4] = {&s1, &s2, &c1, &c2};
305
306 s1 = s2 = p1 = p2 = c1 = c2 = 0;
307
308 /* Init sockets */
309 for (i = 0; i < 4; i++) {
310 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
311 if (*fds[i] < 0) {
312 perror("socket s1 failed()");
313 return errno;
314 }
315 }
316
317 /* Allow reuse */
318 for (i = 0; i < 2; i++) {
319 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
320 (char *)&one, sizeof(one));
321 if (err) {
322 perror("setsockopt failed()");
323 return errno;
324 }
325 }
326
327 /* Non-blocking sockets */
328 for (i = 0; i < 2; i++) {
329 err = ioctl(*fds[i], FIONBIO, (char *)&one);
330 if (err < 0) {
331 perror("ioctl s1 failed()");
332 return errno;
333 }
334 }
335
336 /* Bind server sockets */
337 memset(&addr, 0, sizeof(struct sockaddr_in));
338 addr.sin_family = AF_INET;
339 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
340
341 addr.sin_port = htons(S1_PORT);
342 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
343 if (err < 0) {
344 perror("bind s1 failed()");
345 return errno;
346 }
347
348 addr.sin_port = htons(S2_PORT);
349 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
350 if (err < 0) {
351 perror("bind s2 failed()");
352 return errno;
353 }
354
355 /* Listen server sockets */
356 addr.sin_port = htons(S1_PORT);
357 err = listen(s1, 32);
358 if (err < 0) {
359 perror("listen s1 failed()");
360 return errno;
361 }
362
363 addr.sin_port = htons(S2_PORT);
364 err = listen(s2, 32);
365 if (err < 0) {
366 perror("listen s1 failed()");
367 return errno;
368 }
369
370 /* Initiate Connect */
371 addr.sin_port = htons(S1_PORT);
372 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
373 if (err < 0 && errno != EINPROGRESS) {
374 perror("connect c1 failed()");
375 return errno;
376 }
377
378 addr.sin_port = htons(S2_PORT);
379 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
380 if (err < 0 && errno != EINPROGRESS) {
381 perror("connect c2 failed()");
382 return errno;
383 } else if (err < 0) {
384 err = 0;
385 }
386
387 /* Accept Connecrtions */
388 p1 = accept(s1, NULL, NULL);
389 if (p1 < 0) {
390 perror("accept s1 failed()");
391 return errno;
392 }
393
394 p2 = accept(s2, NULL, NULL);
395 if (p2 < 0) {
396 perror("accept s1 failed()");
397 return errno;
398 }
399
400 if (verbose > 1) {
401 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
402 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
403 c1, s1, c2, s2);
404 }
405 return 0;
406 }
407
408 struct msg_stats {
409 size_t bytes_sent;
410 size_t bytes_recvd;
411 struct timespec start;
412 struct timespec end;
413 };
414
msg_loop_sendpage(int fd,int iov_length,int cnt,struct msg_stats * s,struct sockmap_options * opt)415 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
416 struct msg_stats *s,
417 struct sockmap_options *opt)
418 {
419 bool drop = opt->drop_expected;
420 unsigned char k = 0;
421 FILE *file;
422 int i, fp;
423
424 file = tmpfile();
425 if (!file) {
426 perror("create file for sendpage");
427 return 1;
428 }
429 for (i = 0; i < iov_length * cnt; i++, k++)
430 fwrite(&k, sizeof(char), 1, file);
431 fflush(file);
432 fseek(file, 0, SEEK_SET);
433
434 fp = fileno(file);
435
436 clock_gettime(CLOCK_MONOTONIC, &s->start);
437 for (i = 0; i < cnt; i++) {
438 int sent;
439
440 errno = 0;
441 sent = sendfile(fd, fp, NULL, iov_length);
442
443 if (!drop && sent < 0) {
444 perror("sendpage loop error");
445 fclose(file);
446 return sent;
447 } else if (drop && sent >= 0) {
448 printf("sendpage loop error expected: %i errno %i\n",
449 sent, errno);
450 fclose(file);
451 return -EIO;
452 }
453
454 if (sent > 0)
455 s->bytes_sent += sent;
456 }
457 clock_gettime(CLOCK_MONOTONIC, &s->end);
458 fclose(file);
459 return 0;
460 }
461
msg_free_iov(struct msghdr * msg)462 static void msg_free_iov(struct msghdr *msg)
463 {
464 int i;
465
466 for (i = 0; i < msg->msg_iovlen; i++)
467 free(msg->msg_iov[i].iov_base);
468 free(msg->msg_iov);
469 msg->msg_iov = NULL;
470 msg->msg_iovlen = 0;
471 }
472
msg_alloc_iov(struct msghdr * msg,int iov_count,int iov_length,bool data,bool xmit)473 static int msg_alloc_iov(struct msghdr *msg,
474 int iov_count, int iov_length,
475 bool data, bool xmit)
476 {
477 unsigned char k = 0;
478 struct iovec *iov;
479 int i;
480
481 iov = calloc(iov_count, sizeof(struct iovec));
482 if (!iov)
483 return errno;
484
485 for (i = 0; i < iov_count; i++) {
486 unsigned char *d = calloc(iov_length, sizeof(char));
487
488 if (!d) {
489 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
490 goto unwind_iov;
491 }
492 iov[i].iov_base = d;
493 iov[i].iov_len = iov_length;
494
495 if (data && xmit) {
496 int j;
497
498 for (j = 0; j < iov_length; j++)
499 d[j] = k++;
500 }
501 }
502
503 msg->msg_iov = iov;
504 msg->msg_iovlen = iov_count;
505
506 return 0;
507 unwind_iov:
508 for (i--; i >= 0 ; i--)
509 free(msg->msg_iov[i].iov_base);
510 return -ENOMEM;
511 }
512
msg_verify_data(struct msghdr * msg,int size,int chunk_sz)513 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
514 {
515 int i, j = 0, bytes_cnt = 0;
516 unsigned char k = 0;
517
518 for (i = 0; i < msg->msg_iovlen; i++) {
519 unsigned char *d = msg->msg_iov[i].iov_base;
520
521 /* Special case test for skb ingress + ktls */
522 if (i == 0 && txmsg_ktls_skb) {
523 if (msg->msg_iov[i].iov_len < 4)
524 return -EIO;
525 if (memcmp(d, "PASS", 4) != 0) {
526 fprintf(stderr,
527 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
528 i, 0, d[0], d[1], d[2], d[3]);
529 return -EIO;
530 }
531 j = 4; /* advance index past PASS header */
532 }
533
534 for (; j < msg->msg_iov[i].iov_len && size; j++) {
535 if (d[j] != k++) {
536 fprintf(stderr,
537 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
538 i, j, d[j], k - 1, d[j+1], k);
539 return -EIO;
540 }
541 bytes_cnt++;
542 if (bytes_cnt == chunk_sz) {
543 k = 0;
544 bytes_cnt = 0;
545 }
546 size--;
547 }
548 }
549 return 0;
550 }
551
msg_loop(int fd,int iov_count,int iov_length,int cnt,struct msg_stats * s,bool tx,struct sockmap_options * opt)552 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
553 struct msg_stats *s, bool tx,
554 struct sockmap_options *opt)
555 {
556 struct msghdr msg = {0}, msg_peek = {0};
557 int err, i, flags = MSG_NOSIGNAL;
558 bool drop = opt->drop_expected;
559 bool data = opt->data_test;
560 int iov_alloc_length = iov_length;
561
562 if (!tx && opt->check_recved_len)
563 iov_alloc_length *= 2;
564
565 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
566 if (err)
567 goto out_errno;
568 if (peek_flag) {
569 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
570 if (err)
571 goto out_errno;
572 }
573
574 if (tx) {
575 clock_gettime(CLOCK_MONOTONIC, &s->start);
576 for (i = 0; i < cnt; i++) {
577 int sent;
578
579 errno = 0;
580 sent = sendmsg(fd, &msg, flags);
581
582 if (!drop && sent < 0) {
583 perror("sendmsg loop error");
584 goto out_errno;
585 } else if (drop && sent >= 0) {
586 fprintf(stderr,
587 "sendmsg loop error expected: %i errno %i\n",
588 sent, errno);
589 errno = -EIO;
590 goto out_errno;
591 }
592 if (sent > 0)
593 s->bytes_sent += sent;
594 }
595 clock_gettime(CLOCK_MONOTONIC, &s->end);
596 } else {
597 int slct, recvp = 0, recv, max_fd = fd;
598 float total_bytes, txmsg_pop_total;
599 int fd_flags = O_NONBLOCK;
600 struct timeval timeout;
601 fd_set w;
602
603 fcntl(fd, fd_flags);
604 /* Account for pop bytes noting each iteration of apply will
605 * call msg_pop_data helper so we need to account for this
606 * by calculating the number of apply iterations. Note user
607 * of the tool can create cases where no data is sent by
608 * manipulating pop/push/pull/etc. For example txmsg_apply 1
609 * with txmsg_pop 1 will try to apply 1B at a time but each
610 * iteration will then pop 1B so no data will ever be sent.
611 * This is really only useful for testing edge cases in code
612 * paths.
613 */
614 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
615 if (txmsg_apply)
616 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
617 else
618 txmsg_pop_total = txmsg_pop * cnt;
619 total_bytes -= txmsg_pop_total;
620 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
621 if (err < 0)
622 perror("recv start time");
623 while (s->bytes_recvd < total_bytes) {
624 if (txmsg_cork) {
625 timeout.tv_sec = 0;
626 timeout.tv_usec = 300000;
627 } else {
628 timeout.tv_sec = 3;
629 timeout.tv_usec = 0;
630 }
631
632 /* FD sets */
633 FD_ZERO(&w);
634 FD_SET(fd, &w);
635
636 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
637 if (slct == -1) {
638 perror("select()");
639 clock_gettime(CLOCK_MONOTONIC, &s->end);
640 goto out_errno;
641 } else if (!slct) {
642 if (opt->verbose)
643 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
644 errno = -EIO;
645 clock_gettime(CLOCK_MONOTONIC, &s->end);
646 goto out_errno;
647 }
648
649 errno = 0;
650 if (peek_flag) {
651 flags |= MSG_PEEK;
652 recvp = recvmsg(fd, &msg_peek, flags);
653 if (recvp < 0) {
654 if (errno != EWOULDBLOCK) {
655 clock_gettime(CLOCK_MONOTONIC, &s->end);
656 goto out_errno;
657 }
658 }
659 flags = 0;
660 }
661
662 recv = recvmsg(fd, &msg, flags);
663 if (recv < 0) {
664 if (errno != EWOULDBLOCK) {
665 clock_gettime(CLOCK_MONOTONIC, &s->end);
666 perror("recv failed()");
667 goto out_errno;
668 }
669 }
670
671 s->bytes_recvd += recv;
672
673 if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
674 errno = EMSGSIZE;
675 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
676 s->bytes_recvd, total_bytes);
677 goto out_errno;
678 }
679
680 if (data) {
681 int chunk_sz = opt->sendpage ?
682 iov_length * cnt :
683 iov_length * iov_count;
684
685 errno = msg_verify_data(&msg, recv, chunk_sz);
686 if (errno) {
687 perror("data verify msg failed");
688 goto out_errno;
689 }
690 if (recvp) {
691 errno = msg_verify_data(&msg_peek,
692 recvp,
693 chunk_sz);
694 if (errno) {
695 perror("data verify msg_peek failed");
696 goto out_errno;
697 }
698 }
699 }
700 }
701 clock_gettime(CLOCK_MONOTONIC, &s->end);
702 }
703
704 msg_free_iov(&msg);
705 msg_free_iov(&msg_peek);
706 return err;
707 out_errno:
708 msg_free_iov(&msg);
709 msg_free_iov(&msg_peek);
710 return errno;
711 }
712
713 static float giga = 1000000000;
714
sentBps(struct msg_stats s)715 static inline float sentBps(struct msg_stats s)
716 {
717 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
718 }
719
recvdBps(struct msg_stats s)720 static inline float recvdBps(struct msg_stats s)
721 {
722 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
723 }
724
sendmsg_test(struct sockmap_options * opt)725 static int sendmsg_test(struct sockmap_options *opt)
726 {
727 float sent_Bps = 0, recvd_Bps = 0;
728 int rx_fd, txpid, rxpid, err = 0;
729 struct msg_stats s = {0};
730 int iov_count = opt->iov_count;
731 int iov_buf = opt->iov_length;
732 int rx_status, tx_status;
733 int cnt = opt->rate;
734
735 errno = 0;
736
737 if (opt->base)
738 rx_fd = p1;
739 else
740 rx_fd = p2;
741
742 if (ktls) {
743 /* Redirecting into non-TLS socket which sends into a TLS
744 * socket is not a valid test. So in this case lets not
745 * enable kTLS but still run the test.
746 */
747 if (!txmsg_redir || txmsg_ingress) {
748 err = sockmap_init_ktls(opt->verbose, rx_fd);
749 if (err)
750 return err;
751 }
752 err = sockmap_init_ktls(opt->verbose, c1);
753 if (err)
754 return err;
755 }
756
757 rxpid = fork();
758 if (rxpid == 0) {
759 if (txmsg_pop || txmsg_start_pop)
760 iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
761 if (opt->drop_expected || txmsg_ktls_skb_drop)
762 _exit(0);
763
764 if (!iov_buf) /* zero bytes sent case */
765 _exit(0);
766
767 if (opt->sendpage)
768 iov_count = 1;
769 err = msg_loop(rx_fd, iov_count, iov_buf,
770 cnt, &s, false, opt);
771 if (opt->verbose > 1)
772 fprintf(stderr,
773 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
774 iov_count, iov_buf, cnt, err);
775 if (s.end.tv_sec - s.start.tv_sec) {
776 sent_Bps = sentBps(s);
777 recvd_Bps = recvdBps(s);
778 }
779 if (opt->verbose > 1)
780 fprintf(stdout,
781 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
782 s.bytes_sent, sent_Bps, sent_Bps/giga,
783 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
784 peek_flag ? "(peek_msg)" : "");
785 if (err && txmsg_cork)
786 err = 0;
787 exit(err ? 1 : 0);
788 } else if (rxpid == -1) {
789 perror("msg_loop_rx");
790 return errno;
791 }
792
793 txpid = fork();
794 if (txpid == 0) {
795 if (opt->sendpage)
796 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
797 else
798 err = msg_loop(c1, iov_count, iov_buf,
799 cnt, &s, true, opt);
800
801 if (err)
802 fprintf(stderr,
803 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
804 iov_count, iov_buf, cnt, err);
805 if (s.end.tv_sec - s.start.tv_sec) {
806 sent_Bps = sentBps(s);
807 recvd_Bps = recvdBps(s);
808 }
809 if (opt->verbose > 1)
810 fprintf(stdout,
811 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
812 s.bytes_sent, sent_Bps, sent_Bps/giga,
813 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
814 exit(err ? 1 : 0);
815 } else if (txpid == -1) {
816 perror("msg_loop_tx");
817 return errno;
818 }
819
820 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
821 assert(waitpid(txpid, &tx_status, 0) == txpid);
822 if (WIFEXITED(rx_status)) {
823 err = WEXITSTATUS(rx_status);
824 if (err) {
825 fprintf(stderr, "rx thread exited with err %d.\n", err);
826 goto out;
827 }
828 }
829 if (WIFEXITED(tx_status)) {
830 err = WEXITSTATUS(tx_status);
831 if (err)
832 fprintf(stderr, "tx thread exited with err %d.\n", err);
833 }
834 out:
835 return err;
836 }
837
forever_ping_pong(int rate,struct sockmap_options * opt)838 static int forever_ping_pong(int rate, struct sockmap_options *opt)
839 {
840 struct timeval timeout;
841 char buf[1024] = {0};
842 int sc;
843
844 timeout.tv_sec = 10;
845 timeout.tv_usec = 0;
846
847 /* Ping/Pong data from client to server */
848 sc = send(c1, buf, sizeof(buf), 0);
849 if (sc < 0) {
850 perror("send failed()");
851 return sc;
852 }
853
854 do {
855 int s, rc, i, max_fd = p2;
856 fd_set w;
857
858 /* FD sets */
859 FD_ZERO(&w);
860 FD_SET(c1, &w);
861 FD_SET(c2, &w);
862 FD_SET(p1, &w);
863 FD_SET(p2, &w);
864
865 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
866 if (s == -1) {
867 perror("select()");
868 break;
869 } else if (!s) {
870 fprintf(stderr, "unexpected timeout\n");
871 break;
872 }
873
874 for (i = 0; i <= max_fd && s > 0; ++i) {
875 if (!FD_ISSET(i, &w))
876 continue;
877
878 s--;
879
880 rc = recv(i, buf, sizeof(buf), 0);
881 if (rc < 0) {
882 if (errno != EWOULDBLOCK) {
883 perror("recv failed()");
884 return rc;
885 }
886 }
887
888 if (rc == 0) {
889 close(i);
890 break;
891 }
892
893 sc = send(i, buf, rc, 0);
894 if (sc < 0) {
895 perror("send failed()");
896 return sc;
897 }
898 }
899
900 if (rate)
901 sleep(rate);
902
903 if (opt->verbose) {
904 printf(".");
905 fflush(stdout);
906
907 }
908 } while (running);
909
910 return 0;
911 }
912
913 enum {
914 SELFTESTS,
915 PING_PONG,
916 SENDMSG,
917 BASE,
918 BASE_SENDPAGE,
919 SENDPAGE,
920 };
921
run_options(struct sockmap_options * options,int cg_fd,int test)922 static int run_options(struct sockmap_options *options, int cg_fd, int test)
923 {
924 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
925
926 /* If base test skip BPF setup */
927 if (test == BASE || test == BASE_SENDPAGE)
928 goto run;
929
930 /* Attach programs to sockmap */
931 if (!txmsg_omit_skb_parser) {
932 err = bpf_prog_attach(prog_fd[0], map_fd[0],
933 BPF_SK_SKB_STREAM_PARSER, 0);
934 if (err) {
935 fprintf(stderr,
936 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
937 prog_fd[0], map_fd[0], err, strerror(errno));
938 return err;
939 }
940 }
941
942 err = bpf_prog_attach(prog_fd[1], map_fd[0],
943 BPF_SK_SKB_STREAM_VERDICT, 0);
944 if (err) {
945 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
946 err, strerror(errno));
947 return err;
948 }
949
950 /* Attach programs to TLS sockmap */
951 if (txmsg_ktls_skb) {
952 if (!txmsg_omit_skb_parser) {
953 err = bpf_prog_attach(prog_fd[0], map_fd[8],
954 BPF_SK_SKB_STREAM_PARSER, 0);
955 if (err) {
956 fprintf(stderr,
957 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
958 prog_fd[0], map_fd[8], err, strerror(errno));
959 return err;
960 }
961 }
962
963 err = bpf_prog_attach(prog_fd[2], map_fd[8],
964 BPF_SK_SKB_STREAM_VERDICT, 0);
965 if (err) {
966 fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
967 err, strerror(errno));
968 return err;
969 }
970 }
971
972 /* Attach to cgroups */
973 err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
974 if (err) {
975 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
976 err, strerror(errno));
977 return err;
978 }
979
980 run:
981 err = sockmap_init_sockets(options->verbose);
982 if (err) {
983 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
984 goto out;
985 }
986
987 /* Attach txmsg program to sockmap */
988 if (txmsg_pass)
989 tx_prog_fd = prog_fd[4];
990 else if (txmsg_redir)
991 tx_prog_fd = prog_fd[5];
992 else if (txmsg_apply)
993 tx_prog_fd = prog_fd[6];
994 else if (txmsg_cork)
995 tx_prog_fd = prog_fd[7];
996 else if (txmsg_drop)
997 tx_prog_fd = prog_fd[8];
998 else
999 tx_prog_fd = 0;
1000
1001 if (tx_prog_fd) {
1002 int redir_fd, i = 0;
1003
1004 err = bpf_prog_attach(tx_prog_fd,
1005 map_fd[1], BPF_SK_MSG_VERDICT, 0);
1006 if (err) {
1007 fprintf(stderr,
1008 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1009 err, strerror(errno));
1010 goto out;
1011 }
1012
1013 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1014 if (err) {
1015 fprintf(stderr,
1016 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1017 err, strerror(errno));
1018 goto out;
1019 }
1020
1021 if (txmsg_redir)
1022 redir_fd = c2;
1023 else
1024 redir_fd = c1;
1025
1026 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1027 if (err) {
1028 fprintf(stderr,
1029 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1030 err, strerror(errno));
1031 goto out;
1032 }
1033
1034 if (txmsg_apply) {
1035 err = bpf_map_update_elem(map_fd[3],
1036 &i, &txmsg_apply, BPF_ANY);
1037 if (err) {
1038 fprintf(stderr,
1039 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1040 err, strerror(errno));
1041 goto out;
1042 }
1043 }
1044
1045 if (txmsg_cork) {
1046 err = bpf_map_update_elem(map_fd[4],
1047 &i, &txmsg_cork, BPF_ANY);
1048 if (err) {
1049 fprintf(stderr,
1050 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1051 err, strerror(errno));
1052 goto out;
1053 }
1054 }
1055
1056 if (txmsg_start) {
1057 err = bpf_map_update_elem(map_fd[5],
1058 &i, &txmsg_start, BPF_ANY);
1059 if (err) {
1060 fprintf(stderr,
1061 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1062 err, strerror(errno));
1063 goto out;
1064 }
1065 }
1066
1067 if (txmsg_end) {
1068 i = 1;
1069 err = bpf_map_update_elem(map_fd[5],
1070 &i, &txmsg_end, BPF_ANY);
1071 if (err) {
1072 fprintf(stderr,
1073 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1074 err, strerror(errno));
1075 goto out;
1076 }
1077 }
1078
1079 if (txmsg_start_push) {
1080 i = 2;
1081 err = bpf_map_update_elem(map_fd[5],
1082 &i, &txmsg_start_push, BPF_ANY);
1083 if (err) {
1084 fprintf(stderr,
1085 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1086 err, strerror(errno));
1087 goto out;
1088 }
1089 }
1090
1091 if (txmsg_end_push) {
1092 i = 3;
1093 err = bpf_map_update_elem(map_fd[5],
1094 &i, &txmsg_end_push, BPF_ANY);
1095 if (err) {
1096 fprintf(stderr,
1097 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1098 txmsg_end_push, i, err, strerror(errno));
1099 goto out;
1100 }
1101 }
1102
1103 if (txmsg_start_pop) {
1104 i = 4;
1105 err = bpf_map_update_elem(map_fd[5],
1106 &i, &txmsg_start_pop, BPF_ANY);
1107 if (err) {
1108 fprintf(stderr,
1109 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1110 txmsg_start_pop, i, err, strerror(errno));
1111 goto out;
1112 }
1113 } else {
1114 i = 4;
1115 bpf_map_update_elem(map_fd[5],
1116 &i, &txmsg_start_pop, BPF_ANY);
1117 }
1118
1119 if (txmsg_pop) {
1120 i = 5;
1121 err = bpf_map_update_elem(map_fd[5],
1122 &i, &txmsg_pop, BPF_ANY);
1123 if (err) {
1124 fprintf(stderr,
1125 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1126 txmsg_pop, i, err, strerror(errno));
1127 goto out;
1128 }
1129 } else {
1130 i = 5;
1131 bpf_map_update_elem(map_fd[5],
1132 &i, &txmsg_pop, BPF_ANY);
1133
1134 }
1135
1136 if (txmsg_ingress) {
1137 int in = BPF_F_INGRESS;
1138
1139 i = 0;
1140 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1141 if (err) {
1142 fprintf(stderr,
1143 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1144 err, strerror(errno));
1145 }
1146 i = 1;
1147 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1148 if (err) {
1149 fprintf(stderr,
1150 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1151 err, strerror(errno));
1152 }
1153 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1154 if (err) {
1155 fprintf(stderr,
1156 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1157 err, strerror(errno));
1158 }
1159
1160 i = 2;
1161 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1162 if (err) {
1163 fprintf(stderr,
1164 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1165 err, strerror(errno));
1166 }
1167 }
1168
1169 if (txmsg_ktls_skb) {
1170 int ingress = BPF_F_INGRESS;
1171
1172 i = 0;
1173 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1174 if (err) {
1175 fprintf(stderr,
1176 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1177 err, strerror(errno));
1178 }
1179
1180 if (txmsg_ktls_skb_redir) {
1181 i = 1;
1182 err = bpf_map_update_elem(map_fd[7],
1183 &i, &ingress, BPF_ANY);
1184 if (err) {
1185 fprintf(stderr,
1186 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1187 err, strerror(errno));
1188 }
1189 }
1190
1191 if (txmsg_ktls_skb_drop) {
1192 i = 1;
1193 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1194 }
1195 }
1196
1197 if (txmsg_redir_skb) {
1198 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1199 p2 : p1;
1200 int ingress = BPF_F_INGRESS;
1201
1202 i = 0;
1203 err = bpf_map_update_elem(map_fd[7],
1204 &i, &ingress, BPF_ANY);
1205 if (err) {
1206 fprintf(stderr,
1207 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1208 err, strerror(errno));
1209 }
1210
1211 i = 3;
1212 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1213 if (err) {
1214 fprintf(stderr,
1215 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1216 err, strerror(errno));
1217 }
1218 }
1219 }
1220
1221 if (skb_use_parser) {
1222 i = 2;
1223 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1224 }
1225
1226 if (txmsg_drop)
1227 options->drop_expected = true;
1228
1229 if (test == PING_PONG)
1230 err = forever_ping_pong(options->rate, options);
1231 else if (test == SENDMSG) {
1232 options->base = false;
1233 options->sendpage = false;
1234 err = sendmsg_test(options);
1235 } else if (test == SENDPAGE) {
1236 options->base = false;
1237 options->sendpage = true;
1238 err = sendmsg_test(options);
1239 } else if (test == BASE) {
1240 options->base = true;
1241 options->sendpage = false;
1242 err = sendmsg_test(options);
1243 } else if (test == BASE_SENDPAGE) {
1244 options->base = true;
1245 options->sendpage = true;
1246 err = sendmsg_test(options);
1247 } else
1248 fprintf(stderr, "unknown test\n");
1249 out:
1250 /* Detatch and zero all the maps */
1251 bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1252 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1253 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1254 bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1255 bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1256
1257 if (tx_prog_fd >= 0)
1258 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1259
1260 for (i = 0; i < 8; i++) {
1261 key = next_key = 0;
1262 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1263 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1264 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1265 key = next_key;
1266 }
1267 }
1268
1269 close(s1);
1270 close(s2);
1271 close(p1);
1272 close(p2);
1273 close(c1);
1274 close(c2);
1275 return err;
1276 }
1277
test_to_str(int test)1278 static char *test_to_str(int test)
1279 {
1280 switch (test) {
1281 case SENDMSG:
1282 return "sendmsg";
1283 case SENDPAGE:
1284 return "sendpage";
1285 }
1286 return "unknown";
1287 }
1288
append_str(char * dst,const char * src,size_t dst_cap)1289 static void append_str(char *dst, const char *src, size_t dst_cap)
1290 {
1291 size_t avail = dst_cap - strlen(dst);
1292
1293 if (avail <= 1) /* just zero byte could be written */
1294 return;
1295
1296 strncat(dst, src, avail - 1); /* strncat() adds + 1 for zero byte */
1297 }
1298
1299 #define OPTSTRING 60
test_options(char * options)1300 static void test_options(char *options)
1301 {
1302 char tstr[OPTSTRING];
1303
1304 memset(options, 0, OPTSTRING);
1305
1306 if (txmsg_pass)
1307 append_str(options, "pass,", OPTSTRING);
1308 if (txmsg_redir)
1309 append_str(options, "redir,", OPTSTRING);
1310 if (txmsg_drop)
1311 append_str(options, "drop,", OPTSTRING);
1312 if (txmsg_apply) {
1313 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1314 append_str(options, tstr, OPTSTRING);
1315 }
1316 if (txmsg_cork) {
1317 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1318 append_str(options, tstr, OPTSTRING);
1319 }
1320 if (txmsg_start) {
1321 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1322 append_str(options, tstr, OPTSTRING);
1323 }
1324 if (txmsg_end) {
1325 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1326 append_str(options, tstr, OPTSTRING);
1327 }
1328 if (txmsg_start_pop) {
1329 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1330 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1331 append_str(options, tstr, OPTSTRING);
1332 }
1333 if (txmsg_ingress)
1334 append_str(options, "ingress,", OPTSTRING);
1335 if (txmsg_redir_skb)
1336 append_str(options, "redir_skb,", OPTSTRING);
1337 if (txmsg_ktls_skb)
1338 append_str(options, "ktls_skb,", OPTSTRING);
1339 if (ktls)
1340 append_str(options, "ktls,", OPTSTRING);
1341 if (peek_flag)
1342 append_str(options, "peek,", OPTSTRING);
1343 }
1344
__test_exec(int cgrp,int test,struct sockmap_options * opt)1345 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1346 {
1347 char *options = calloc(OPTSTRING, sizeof(char));
1348 int err;
1349
1350 if (test == SENDPAGE)
1351 opt->sendpage = true;
1352 else
1353 opt->sendpage = false;
1354
1355 if (txmsg_drop)
1356 opt->drop_expected = true;
1357 else
1358 opt->drop_expected = false;
1359
1360 test_options(options);
1361
1362 if (opt->verbose) {
1363 fprintf(stdout,
1364 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1365 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1366 test_to_str(test), options);
1367 fflush(stdout);
1368 }
1369 err = run_options(opt, cgrp, test);
1370 if (opt->verbose)
1371 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1372 test_cnt++;
1373 !err ? passed++ : failed++;
1374 free(options);
1375 return err;
1376 }
1377
test_exec(int cgrp,struct sockmap_options * opt)1378 static void test_exec(int cgrp, struct sockmap_options *opt)
1379 {
1380 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1381 int err;
1382
1383 if (type == 0) {
1384 test_start();
1385 err = __test_exec(cgrp, SENDMSG, opt);
1386 if (err)
1387 test_fail();
1388 } else {
1389 test_start();
1390 err = __test_exec(cgrp, SENDPAGE, opt);
1391 if (err)
1392 test_fail();
1393 }
1394 }
1395
test_send_one(struct sockmap_options * opt,int cgrp)1396 static void test_send_one(struct sockmap_options *opt, int cgrp)
1397 {
1398 opt->iov_length = 1;
1399 opt->iov_count = 1;
1400 opt->rate = 1;
1401 test_exec(cgrp, opt);
1402
1403 opt->iov_length = 1;
1404 opt->iov_count = 1024;
1405 opt->rate = 1;
1406 test_exec(cgrp, opt);
1407
1408 opt->iov_length = 1024;
1409 opt->iov_count = 1;
1410 opt->rate = 1;
1411 test_exec(cgrp, opt);
1412
1413 }
1414
test_send_many(struct sockmap_options * opt,int cgrp)1415 static void test_send_many(struct sockmap_options *opt, int cgrp)
1416 {
1417 opt->iov_length = 3;
1418 opt->iov_count = 1;
1419 opt->rate = 512;
1420 test_exec(cgrp, opt);
1421
1422 opt->rate = 100;
1423 opt->iov_count = 1;
1424 opt->iov_length = 5;
1425 test_exec(cgrp, opt);
1426 }
1427
test_send_large(struct sockmap_options * opt,int cgrp)1428 static void test_send_large(struct sockmap_options *opt, int cgrp)
1429 {
1430 opt->iov_length = 256;
1431 opt->iov_count = 1024;
1432 opt->rate = 2;
1433 test_exec(cgrp, opt);
1434 }
1435
test_send(struct sockmap_options * opt,int cgrp)1436 static void test_send(struct sockmap_options *opt, int cgrp)
1437 {
1438 test_send_one(opt, cgrp);
1439 test_send_many(opt, cgrp);
1440 test_send_large(opt, cgrp);
1441 sched_yield();
1442 }
1443
test_txmsg_pass(int cgrp,struct sockmap_options * opt)1444 static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1445 {
1446 /* Test small and large iov_count values with pass/redir/apply/cork */
1447 txmsg_pass = 1;
1448 test_send(opt, cgrp);
1449 }
1450
test_txmsg_redir(int cgrp,struct sockmap_options * opt)1451 static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1452 {
1453 txmsg_redir = 1;
1454 test_send(opt, cgrp);
1455 }
1456
test_txmsg_drop(int cgrp,struct sockmap_options * opt)1457 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1458 {
1459 txmsg_drop = 1;
1460 test_send(opt, cgrp);
1461 }
1462
test_txmsg_ingress_redir(int cgrp,struct sockmap_options * opt)1463 static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1464 {
1465 txmsg_pass = txmsg_drop = 0;
1466 txmsg_ingress = txmsg_redir = 1;
1467 test_send(opt, cgrp);
1468 }
1469
test_txmsg_skb(int cgrp,struct sockmap_options * opt)1470 static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1471 {
1472 bool data = opt->data_test;
1473 int k = ktls;
1474
1475 opt->data_test = true;
1476 ktls = 1;
1477
1478 txmsg_pass = txmsg_drop = 0;
1479 txmsg_ingress = txmsg_redir = 0;
1480 txmsg_ktls_skb = 1;
1481 txmsg_pass = 1;
1482
1483 /* Using data verification so ensure iov layout is
1484 * expected from test receiver side. e.g. has enough
1485 * bytes to write test code.
1486 */
1487 opt->iov_length = 100;
1488 opt->iov_count = 1;
1489 opt->rate = 1;
1490 test_exec(cgrp, opt);
1491
1492 txmsg_ktls_skb_drop = 1;
1493 test_exec(cgrp, opt);
1494
1495 txmsg_ktls_skb_drop = 0;
1496 txmsg_ktls_skb_redir = 1;
1497 test_exec(cgrp, opt);
1498 txmsg_ktls_skb_redir = 0;
1499
1500 /* Tests that omit skb_parser */
1501 txmsg_omit_skb_parser = 1;
1502 ktls = 0;
1503 txmsg_ktls_skb = 0;
1504 test_exec(cgrp, opt);
1505
1506 txmsg_ktls_skb_drop = 1;
1507 test_exec(cgrp, opt);
1508 txmsg_ktls_skb_drop = 0;
1509
1510 txmsg_ktls_skb_redir = 1;
1511 test_exec(cgrp, opt);
1512
1513 ktls = 1;
1514 test_exec(cgrp, opt);
1515 txmsg_omit_skb_parser = 0;
1516
1517 opt->data_test = data;
1518 ktls = k;
1519 }
1520
1521 /* Test cork with hung data. This tests poor usage patterns where
1522 * cork can leave data on the ring if user program is buggy and
1523 * doesn't flush them somehow. They do take some time however
1524 * because they wait for a timeout. Test pass, redir and cork with
1525 * apply logic. Use cork size of 4097 with send_large to avoid
1526 * aligning cork size with send size.
1527 */
test_txmsg_cork_hangs(int cgrp,struct sockmap_options * opt)1528 static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1529 {
1530 txmsg_pass = 1;
1531 txmsg_redir = 0;
1532 txmsg_cork = 4097;
1533 txmsg_apply = 4097;
1534 test_send_large(opt, cgrp);
1535
1536 txmsg_pass = 0;
1537 txmsg_redir = 1;
1538 txmsg_apply = 0;
1539 txmsg_cork = 4097;
1540 test_send_large(opt, cgrp);
1541
1542 txmsg_pass = 0;
1543 txmsg_redir = 1;
1544 txmsg_apply = 4097;
1545 txmsg_cork = 4097;
1546 test_send_large(opt, cgrp);
1547 }
1548
test_txmsg_pull(int cgrp,struct sockmap_options * opt)1549 static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1550 {
1551 /* Test basic start/end */
1552 txmsg_start = 1;
1553 txmsg_end = 2;
1554 test_send(opt, cgrp);
1555
1556 /* Test >4k pull */
1557 txmsg_start = 4096;
1558 txmsg_end = 9182;
1559 test_send_large(opt, cgrp);
1560
1561 /* Test pull + redirect */
1562 txmsg_redir = 0;
1563 txmsg_start = 1;
1564 txmsg_end = 2;
1565 test_send(opt, cgrp);
1566
1567 /* Test pull + cork */
1568 txmsg_redir = 0;
1569 txmsg_cork = 512;
1570 txmsg_start = 1;
1571 txmsg_end = 2;
1572 test_send_many(opt, cgrp);
1573
1574 /* Test pull + cork + redirect */
1575 txmsg_redir = 1;
1576 txmsg_cork = 512;
1577 txmsg_start = 1;
1578 txmsg_end = 2;
1579 test_send_many(opt, cgrp);
1580 }
1581
test_txmsg_pop(int cgrp,struct sockmap_options * opt)1582 static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1583 {
1584 /* Test basic pop */
1585 txmsg_start_pop = 1;
1586 txmsg_pop = 2;
1587 test_send_many(opt, cgrp);
1588
1589 /* Test pop with >4k */
1590 txmsg_start_pop = 4096;
1591 txmsg_pop = 4096;
1592 test_send_large(opt, cgrp);
1593
1594 /* Test pop + redirect */
1595 txmsg_redir = 1;
1596 txmsg_start_pop = 1;
1597 txmsg_pop = 2;
1598 test_send_many(opt, cgrp);
1599
1600 /* Test pop + cork */
1601 txmsg_redir = 0;
1602 txmsg_cork = 512;
1603 txmsg_start_pop = 1;
1604 txmsg_pop = 2;
1605 test_send_many(opt, cgrp);
1606
1607 /* Test pop + redirect + cork */
1608 txmsg_redir = 1;
1609 txmsg_cork = 4;
1610 txmsg_start_pop = 1;
1611 txmsg_pop = 2;
1612 test_send_many(opt, cgrp);
1613 }
1614
test_txmsg_push(int cgrp,struct sockmap_options * opt)1615 static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1616 {
1617 /* Test basic push */
1618 txmsg_start_push = 1;
1619 txmsg_end_push = 1;
1620 test_send(opt, cgrp);
1621
1622 /* Test push 4kB >4k */
1623 txmsg_start_push = 4096;
1624 txmsg_end_push = 4096;
1625 test_send_large(opt, cgrp);
1626
1627 /* Test push + redirect */
1628 txmsg_redir = 1;
1629 txmsg_start_push = 1;
1630 txmsg_end_push = 2;
1631 test_send_many(opt, cgrp);
1632
1633 /* Test push + cork */
1634 txmsg_redir = 0;
1635 txmsg_cork = 512;
1636 txmsg_start_push = 1;
1637 txmsg_end_push = 2;
1638 test_send_many(opt, cgrp);
1639 }
1640
test_txmsg_push_pop(int cgrp,struct sockmap_options * opt)1641 static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1642 {
1643 txmsg_start_push = 1;
1644 txmsg_end_push = 10;
1645 txmsg_start_pop = 5;
1646 txmsg_pop = 4;
1647 test_send_large(opt, cgrp);
1648 }
1649
test_txmsg_apply(int cgrp,struct sockmap_options * opt)1650 static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1651 {
1652 txmsg_pass = 1;
1653 txmsg_redir = 0;
1654 txmsg_apply = 1;
1655 txmsg_cork = 0;
1656 test_send_one(opt, cgrp);
1657
1658 txmsg_pass = 0;
1659 txmsg_redir = 1;
1660 txmsg_apply = 1;
1661 txmsg_cork = 0;
1662 test_send_one(opt, cgrp);
1663
1664 txmsg_pass = 1;
1665 txmsg_redir = 0;
1666 txmsg_apply = 1024;
1667 txmsg_cork = 0;
1668 test_send_large(opt, cgrp);
1669
1670 txmsg_pass = 0;
1671 txmsg_redir = 1;
1672 txmsg_apply = 1024;
1673 txmsg_cork = 0;
1674 test_send_large(opt, cgrp);
1675 }
1676
test_txmsg_cork(int cgrp,struct sockmap_options * opt)1677 static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1678 {
1679 txmsg_pass = 1;
1680 txmsg_redir = 0;
1681 txmsg_apply = 0;
1682 txmsg_cork = 1;
1683 test_send(opt, cgrp);
1684
1685 txmsg_pass = 1;
1686 txmsg_redir = 0;
1687 txmsg_apply = 1;
1688 txmsg_cork = 1;
1689 test_send(opt, cgrp);
1690 }
1691
test_txmsg_ingress_parser(int cgrp,struct sockmap_options * opt)1692 static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1693 {
1694 txmsg_pass = 1;
1695 skb_use_parser = 512;
1696 if (ktls == 1)
1697 skb_use_parser = 570;
1698 opt->iov_length = 256;
1699 opt->iov_count = 1;
1700 opt->rate = 2;
1701 test_exec(cgrp, opt);
1702 }
1703
test_txmsg_ingress_parser2(int cgrp,struct sockmap_options * opt)1704 static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1705 {
1706 if (ktls == 1)
1707 return;
1708 skb_use_parser = 10;
1709 opt->iov_length = 20;
1710 opt->iov_count = 1;
1711 opt->rate = 1;
1712 opt->check_recved_len = true;
1713 test_exec(cgrp, opt);
1714 opt->check_recved_len = false;
1715 }
1716
1717 char *map_names[] = {
1718 "sock_map",
1719 "sock_map_txmsg",
1720 "sock_map_redir",
1721 "sock_apply_bytes",
1722 "sock_cork_bytes",
1723 "sock_bytes",
1724 "sock_redir_flags",
1725 "sock_skb_opts",
1726 "tls_sock_map",
1727 };
1728
1729 int prog_attach_type[] = {
1730 BPF_SK_SKB_STREAM_PARSER,
1731 BPF_SK_SKB_STREAM_VERDICT,
1732 BPF_SK_SKB_STREAM_VERDICT,
1733 BPF_CGROUP_SOCK_OPS,
1734 BPF_SK_MSG_VERDICT,
1735 BPF_SK_MSG_VERDICT,
1736 BPF_SK_MSG_VERDICT,
1737 BPF_SK_MSG_VERDICT,
1738 BPF_SK_MSG_VERDICT,
1739 BPF_SK_MSG_VERDICT,
1740 BPF_SK_MSG_VERDICT,
1741 };
1742
1743 int prog_type[] = {
1744 BPF_PROG_TYPE_SK_SKB,
1745 BPF_PROG_TYPE_SK_SKB,
1746 BPF_PROG_TYPE_SK_SKB,
1747 BPF_PROG_TYPE_SOCK_OPS,
1748 BPF_PROG_TYPE_SK_MSG,
1749 BPF_PROG_TYPE_SK_MSG,
1750 BPF_PROG_TYPE_SK_MSG,
1751 BPF_PROG_TYPE_SK_MSG,
1752 BPF_PROG_TYPE_SK_MSG,
1753 BPF_PROG_TYPE_SK_MSG,
1754 BPF_PROG_TYPE_SK_MSG,
1755 };
1756
populate_progs(char * bpf_file)1757 static int populate_progs(char *bpf_file)
1758 {
1759 struct bpf_program *prog;
1760 struct bpf_object *obj;
1761 int i = 0;
1762 long err;
1763
1764 obj = bpf_object__open(bpf_file);
1765 err = libbpf_get_error(obj);
1766 if (err) {
1767 char err_buf[256];
1768
1769 libbpf_strerror(err, err_buf, sizeof(err_buf));
1770 printf("Unable to load eBPF objects in file '%s' : %s\n",
1771 bpf_file, err_buf);
1772 return -1;
1773 }
1774
1775 bpf_object__for_each_program(prog, obj) {
1776 bpf_program__set_type(prog, prog_type[i]);
1777 bpf_program__set_expected_attach_type(prog,
1778 prog_attach_type[i]);
1779 i++;
1780 }
1781
1782 i = bpf_object__load(obj);
1783 i = 0;
1784 bpf_object__for_each_program(prog, obj) {
1785 prog_fd[i] = bpf_program__fd(prog);
1786 i++;
1787 }
1788
1789 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1790 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1791 map_fd[i] = bpf_map__fd(maps[i]);
1792 if (map_fd[i] < 0) {
1793 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1794 map_fd[i], strerror(errno));
1795 return -1;
1796 }
1797 }
1798
1799 return 0;
1800 }
1801
1802 struct _test test[] = {
1803 {"txmsg test passthrough", test_txmsg_pass},
1804 {"txmsg test redirect", test_txmsg_redir},
1805 {"txmsg test drop", test_txmsg_drop},
1806 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1807 {"txmsg test skb", test_txmsg_skb},
1808 {"txmsg test apply", test_txmsg_apply},
1809 {"txmsg test cork", test_txmsg_cork},
1810 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1811 {"txmsg test push_data", test_txmsg_push},
1812 {"txmsg test pull-data", test_txmsg_pull},
1813 {"txmsg test pop-data", test_txmsg_pop},
1814 {"txmsg test push/pop data", test_txmsg_push_pop},
1815 {"txmsg test ingress parser", test_txmsg_ingress_parser},
1816 {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1817 };
1818
check_whitelist(struct _test * t,struct sockmap_options * opt)1819 static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1820 {
1821 char *entry, *ptr;
1822
1823 if (!opt->whitelist)
1824 return 0;
1825 ptr = strdup(opt->whitelist);
1826 if (!ptr)
1827 return -ENOMEM;
1828 entry = strtok(ptr, ",");
1829 while (entry) {
1830 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1831 strstr(opt->map, entry) != 0 ||
1832 strstr(t->title, entry) != 0)
1833 return 0;
1834 entry = strtok(NULL, ",");
1835 }
1836 return -EINVAL;
1837 }
1838
check_blacklist(struct _test * t,struct sockmap_options * opt)1839 static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1840 {
1841 char *entry, *ptr;
1842
1843 if (!opt->blacklist)
1844 return -EINVAL;
1845 ptr = strdup(opt->blacklist);
1846 if (!ptr)
1847 return -ENOMEM;
1848 entry = strtok(ptr, ",");
1849 while (entry) {
1850 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1851 strstr(opt->map, entry) != 0 ||
1852 strstr(t->title, entry) != 0)
1853 return 0;
1854 entry = strtok(NULL, ",");
1855 }
1856 return -EINVAL;
1857 }
1858
__test_selftests(int cg_fd,struct sockmap_options * opt)1859 static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1860 {
1861 int i, err;
1862
1863 err = populate_progs(opt->map);
1864 if (err < 0) {
1865 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1866 return err;
1867 }
1868
1869 /* Tests basic commands and APIs */
1870 for (i = 0; i < sizeof(test)/sizeof(struct _test); i++) {
1871 struct _test t = test[i];
1872
1873 if (check_whitelist(&t, opt) != 0)
1874 continue;
1875 if (check_blacklist(&t, opt) == 0)
1876 continue;
1877
1878 test_start_subtest(&t, opt);
1879 t.tester(cg_fd, opt);
1880 test_end_subtest();
1881 }
1882
1883 return err;
1884 }
1885
test_selftests_sockmap(int cg_fd,struct sockmap_options * opt)1886 static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1887 {
1888 opt->map = BPF_SOCKMAP_FILENAME;
1889 __test_selftests(cg_fd, opt);
1890 }
1891
test_selftests_sockhash(int cg_fd,struct sockmap_options * opt)1892 static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1893 {
1894 opt->map = BPF_SOCKHASH_FILENAME;
1895 __test_selftests(cg_fd, opt);
1896 }
1897
test_selftests_ktls(int cg_fd,struct sockmap_options * opt)1898 static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1899 {
1900 opt->map = BPF_SOCKHASH_FILENAME;
1901 opt->prepend = "ktls";
1902 ktls = 1;
1903 __test_selftests(cg_fd, opt);
1904 ktls = 0;
1905 }
1906
test_selftest(int cg_fd,struct sockmap_options * opt)1907 static int test_selftest(int cg_fd, struct sockmap_options *opt)
1908 {
1909
1910 test_selftests_sockmap(cg_fd, opt);
1911 test_selftests_sockhash(cg_fd, opt);
1912 test_selftests_ktls(cg_fd, opt);
1913 test_print_results();
1914 return 0;
1915 }
1916
main(int argc,char ** argv)1917 int main(int argc, char **argv)
1918 {
1919 int iov_count = 1, length = 1024, rate = 1;
1920 struct sockmap_options options = {0};
1921 int opt, longindex, err, cg_fd = 0;
1922 char *bpf_file = BPF_SOCKMAP_FILENAME;
1923 int test = SELFTESTS;
1924 bool cg_created = 0;
1925
1926 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1927 long_options, &longindex)) != -1) {
1928 switch (opt) {
1929 case 's':
1930 txmsg_start = atoi(optarg);
1931 break;
1932 case 'e':
1933 txmsg_end = atoi(optarg);
1934 break;
1935 case 'p':
1936 txmsg_start_push = atoi(optarg);
1937 break;
1938 case 'q':
1939 txmsg_end_push = atoi(optarg);
1940 break;
1941 case 'w':
1942 txmsg_start_pop = atoi(optarg);
1943 break;
1944 case 'x':
1945 txmsg_pop = atoi(optarg);
1946 break;
1947 case 'a':
1948 txmsg_apply = atoi(optarg);
1949 break;
1950 case 'k':
1951 txmsg_cork = atoi(optarg);
1952 break;
1953 case 'c':
1954 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1955 if (cg_fd < 0) {
1956 fprintf(stderr,
1957 "ERROR: (%i) open cg path failed: %s\n",
1958 cg_fd, optarg);
1959 return cg_fd;
1960 }
1961 break;
1962 case 'r':
1963 rate = atoi(optarg);
1964 break;
1965 case 'v':
1966 options.verbose = 1;
1967 if (optarg)
1968 options.verbose = atoi(optarg);
1969 break;
1970 case 'i':
1971 iov_count = atoi(optarg);
1972 break;
1973 case 'l':
1974 length = atoi(optarg);
1975 break;
1976 case 'd':
1977 options.data_test = true;
1978 break;
1979 case 't':
1980 if (strcmp(optarg, "ping") == 0) {
1981 test = PING_PONG;
1982 } else if (strcmp(optarg, "sendmsg") == 0) {
1983 test = SENDMSG;
1984 } else if (strcmp(optarg, "base") == 0) {
1985 test = BASE;
1986 } else if (strcmp(optarg, "base_sendpage") == 0) {
1987 test = BASE_SENDPAGE;
1988 } else if (strcmp(optarg, "sendpage") == 0) {
1989 test = SENDPAGE;
1990 } else {
1991 usage(argv);
1992 return -1;
1993 }
1994 break;
1995 case 'n':
1996 options.whitelist = strdup(optarg);
1997 if (!options.whitelist)
1998 return -ENOMEM;
1999 break;
2000 case 'b':
2001 options.blacklist = strdup(optarg);
2002 if (!options.blacklist)
2003 return -ENOMEM;
2004 case 0:
2005 break;
2006 case 'h':
2007 default:
2008 usage(argv);
2009 return -1;
2010 }
2011 }
2012
2013 if (!cg_fd) {
2014 cg_fd = cgroup_setup_and_join(CG_PATH);
2015 if (cg_fd < 0)
2016 return cg_fd;
2017 cg_created = 1;
2018 }
2019
2020 if (test == SELFTESTS) {
2021 err = test_selftest(cg_fd, &options);
2022 goto out;
2023 }
2024
2025 err = populate_progs(bpf_file);
2026 if (err) {
2027 fprintf(stderr, "populate program: (%s) %s\n",
2028 bpf_file, strerror(errno));
2029 return 1;
2030 }
2031 running = 1;
2032
2033 /* catch SIGINT */
2034 signal(SIGINT, running_handler);
2035
2036 options.iov_count = iov_count;
2037 options.iov_length = length;
2038 options.rate = rate;
2039
2040 err = run_options(&options, cg_fd, test);
2041 out:
2042 if (options.whitelist)
2043 free(options.whitelist);
2044 if (options.blacklist)
2045 free(options.blacklist);
2046 if (cg_created)
2047 cleanup_cgroup_environment();
2048 close(cg_fd);
2049 return err;
2050 }
2051
running_handler(int a)2052 void running_handler(int a)
2053 {
2054 running = 0;
2055 }
2056