1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <sys/mman.h>
4 #include <stdbool.h>
5 #include <time.h>
6 #include <string.h>
7 #include <numa.h>
8 #include <unistd.h>
9 #include <fcntl.h>
10 #include <stdint.h>
11 #include <err.h>
12
13 #include "../kselftest.h"
14 #include "../../../../include/vdso/time64.h"
15
16 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
17 #define KSM_FP(s) (KSM_SYSFS_PATH s)
18 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120
19 #define KSM_PAGE_COUNT_DEFAULT 10l
20 #define KSM_PROT_STR_DEFAULT "rw"
21 #define KSM_USE_ZERO_PAGES_DEFAULT false
22 #define KSM_MERGE_ACROSS_NODES_DEFAULT true
23 #define MB (1ul << 20)
24
25 #define PAGE_SHIFT 12
26 #define HPAGE_SHIFT 21
27
28 #define PAGE_SIZE (1 << PAGE_SHIFT)
29 #define HPAGE_SIZE (1 << HPAGE_SHIFT)
30
31 #define PAGEMAP_PRESENT(ent) (((ent) & (1ull << 63)) != 0)
32 #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1))
33
34 struct ksm_sysfs {
35 unsigned long max_page_sharing;
36 unsigned long merge_across_nodes;
37 unsigned long pages_to_scan;
38 unsigned long run;
39 unsigned long sleep_millisecs;
40 unsigned long stable_node_chains_prune_millisecs;
41 unsigned long use_zero_pages;
42 };
43
44 enum ksm_test_name {
45 CHECK_KSM_MERGE,
46 CHECK_KSM_UNMERGE,
47 CHECK_KSM_ZERO_PAGE_MERGE,
48 CHECK_KSM_NUMA_MERGE,
49 KSM_MERGE_TIME,
50 KSM_MERGE_TIME_HUGE_PAGES,
51 KSM_COW_TIME
52 };
53
ksm_write_sysfs(const char * file_path,unsigned long val)54 static int ksm_write_sysfs(const char *file_path, unsigned long val)
55 {
56 FILE *f = fopen(file_path, "w");
57
58 if (!f) {
59 fprintf(stderr, "f %s\n", file_path);
60 perror("fopen");
61 return 1;
62 }
63 if (fprintf(f, "%lu", val) < 0) {
64 perror("fprintf");
65 return 1;
66 }
67 fclose(f);
68
69 return 0;
70 }
71
ksm_read_sysfs(const char * file_path,unsigned long * val)72 static int ksm_read_sysfs(const char *file_path, unsigned long *val)
73 {
74 FILE *f = fopen(file_path, "r");
75
76 if (!f) {
77 fprintf(stderr, "f %s\n", file_path);
78 perror("fopen");
79 return 1;
80 }
81 if (fscanf(f, "%lu", val) != 1) {
82 perror("fscanf");
83 return 1;
84 }
85 fclose(f);
86
87 return 0;
88 }
89
str_to_prot(char * prot_str)90 static int str_to_prot(char *prot_str)
91 {
92 int prot = 0;
93
94 if ((strchr(prot_str, 'r')) != NULL)
95 prot |= PROT_READ;
96 if ((strchr(prot_str, 'w')) != NULL)
97 prot |= PROT_WRITE;
98 if ((strchr(prot_str, 'x')) != NULL)
99 prot |= PROT_EXEC;
100
101 return prot;
102 }
103
print_help(void)104 static void print_help(void)
105 {
106 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n"
107 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n");
108
109 printf("Supported <test type>:\n"
110 " -M (page merging)\n"
111 " -Z (zero pages merging)\n"
112 " -N (merging of pages in different NUMA nodes)\n"
113 " -U (page unmerging)\n"
114 " -P evaluate merging time and speed.\n"
115 " For this test, the size of duplicated memory area (in MiB)\n"
116 " must be provided using -s option\n"
117 " -H evaluate merging time and speed of area allocated mostly with huge pages\n"
118 " For this test, the size of duplicated memory area (in MiB)\n"
119 " must be provided using -s option\n"
120 " -C evaluate the time required to break COW of merged pages.\n\n");
121
122 printf(" -a: specify the access protections of pages.\n"
123 " <prot> must be of the form [rwx].\n"
124 " Default: %s\n", KSM_PROT_STR_DEFAULT);
125 printf(" -p: specify the number of pages to test.\n"
126 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT);
127 printf(" -l: limit the maximum running time (in seconds) for a test.\n"
128 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT);
129 printf(" -z: change use_zero_pages tunable\n"
130 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT);
131 printf(" -m: change merge_across_nodes tunable\n"
132 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT);
133 printf(" -s: the size of duplicated memory area (in MiB)\n");
134
135 exit(0);
136 }
137
allocate_memory(void * ptr,int prot,int mapping,char data,size_t map_size)138 static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size)
139 {
140 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
141
142 if (!map_ptr) {
143 perror("mmap");
144 return NULL;
145 }
146 memset(map_ptr, data, map_size);
147 if (mprotect(map_ptr, map_size, prot)) {
148 perror("mprotect");
149 munmap(map_ptr, map_size);
150 return NULL;
151 }
152
153 return map_ptr;
154 }
155
ksm_do_scan(int scan_count,struct timespec start_time,int timeout)156 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout)
157 {
158 struct timespec cur_time;
159 unsigned long cur_scan, init_scan;
160
161 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan))
162 return 1;
163 cur_scan = init_scan;
164
165 while (cur_scan < init_scan + scan_count) {
166 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan))
167 return 1;
168 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) {
169 perror("clock_gettime");
170 return 1;
171 }
172 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) {
173 printf("Scan time limit exceeded\n");
174 return 1;
175 }
176 }
177
178 return 0;
179 }
180
ksm_merge_pages(void * addr,size_t size,struct timespec start_time,int timeout)181 static int ksm_merge_pages(void *addr, size_t size, struct timespec start_time, int timeout)
182 {
183 if (madvise(addr, size, MADV_MERGEABLE)) {
184 perror("madvise");
185 return 1;
186 }
187 if (ksm_write_sysfs(KSM_FP("run"), 1))
188 return 1;
189
190 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */
191 if (ksm_do_scan(2, start_time, timeout))
192 return 1;
193
194 return 0;
195 }
196
assert_ksm_pages_count(long dupl_page_count)197 static bool assert_ksm_pages_count(long dupl_page_count)
198 {
199 unsigned long max_page_sharing, pages_sharing, pages_shared;
200
201 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) ||
202 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) ||
203 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing))
204 return false;
205
206 /*
207 * Since there must be at least 2 pages for merging and 1 page can be
208 * shared with the limited number of pages (max_page_sharing), sometimes
209 * there are 'leftover' pages that cannot be merged. For example, if there
210 * are 11 pages and max_page_sharing = 10, then only 10 pages will be
211 * merged and the 11th page won't be affected. As a result, when the number
212 * of duplicate pages is divided by max_page_sharing and the remainder is 1,
213 * pages_shared and pages_sharing values will be equal between dupl_page_count
214 * and dupl_page_count - 1.
215 */
216 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) {
217 if (pages_shared == dupl_page_count / max_page_sharing &&
218 pages_sharing == pages_shared * (max_page_sharing - 1))
219 return true;
220 } else {
221 if (pages_shared == (dupl_page_count / max_page_sharing + 1) &&
222 pages_sharing == dupl_page_count - pages_shared)
223 return true;
224 }
225
226 return false;
227 }
228
ksm_save_def(struct ksm_sysfs * ksm_sysfs)229 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
230 {
231 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
232 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
233 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
234 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
235 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
236 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
237 &ksm_sysfs->stable_node_chains_prune_millisecs) ||
238 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages))
239 return 1;
240
241 return 0;
242 }
243
ksm_restore(struct ksm_sysfs * ksm_sysfs)244 static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
245 {
246 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
247 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
248 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
249 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
250 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
251 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"),
252 ksm_sysfs->stable_node_chains_prune_millisecs) ||
253 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages))
254 return 1;
255
256 return 0;
257 }
258
check_ksm_merge(int mapping,int prot,long page_count,int timeout,size_t page_size)259 static int check_ksm_merge(int mapping, int prot, long page_count, int timeout, size_t page_size)
260 {
261 void *map_ptr;
262 struct timespec start_time;
263
264 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
265 perror("clock_gettime");
266 return KSFT_FAIL;
267 }
268
269 /* fill pages with the same data and merge them */
270 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
271 if (!map_ptr)
272 return KSFT_FAIL;
273
274 if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
275 goto err_out;
276
277 /* verify that the right number of pages are merged */
278 if (assert_ksm_pages_count(page_count)) {
279 printf("OK\n");
280 munmap(map_ptr, page_size * page_count);
281 return KSFT_PASS;
282 }
283
284 err_out:
285 printf("Not OK\n");
286 munmap(map_ptr, page_size * page_count);
287 return KSFT_FAIL;
288 }
289
check_ksm_unmerge(int mapping,int prot,int timeout,size_t page_size)290 static int check_ksm_unmerge(int mapping, int prot, int timeout, size_t page_size)
291 {
292 void *map_ptr;
293 struct timespec start_time;
294 int page_count = 2;
295
296 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
297 perror("clock_gettime");
298 return KSFT_FAIL;
299 }
300
301 /* fill pages with the same data and merge them */
302 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
303 if (!map_ptr)
304 return KSFT_FAIL;
305
306 if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
307 goto err_out;
308
309 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */
310 memset(map_ptr, '-', 1);
311 memset(map_ptr + page_size, '+', 1);
312
313 /* get at least 1 scan, so KSM can detect that the pages were modified */
314 if (ksm_do_scan(1, start_time, timeout))
315 goto err_out;
316
317 /* check that unmerging was successful and 0 pages are currently merged */
318 if (assert_ksm_pages_count(0)) {
319 printf("OK\n");
320 munmap(map_ptr, page_size * page_count);
321 return KSFT_PASS;
322 }
323
324 err_out:
325 printf("Not OK\n");
326 munmap(map_ptr, page_size * page_count);
327 return KSFT_FAIL;
328 }
329
check_ksm_zero_page_merge(int mapping,int prot,long page_count,int timeout,bool use_zero_pages,size_t page_size)330 static int check_ksm_zero_page_merge(int mapping, int prot, long page_count, int timeout,
331 bool use_zero_pages, size_t page_size)
332 {
333 void *map_ptr;
334 struct timespec start_time;
335
336 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
337 perror("clock_gettime");
338 return KSFT_FAIL;
339 }
340
341 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages))
342 return KSFT_FAIL;
343
344 /* fill pages with zero and try to merge them */
345 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count);
346 if (!map_ptr)
347 return KSFT_FAIL;
348
349 if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
350 goto err_out;
351
352 /*
353 * verify that the right number of pages are merged:
354 * 1) if use_zero_pages is set to 1, empty pages are merged
355 * with the kernel zero page instead of with each other;
356 * 2) if use_zero_pages is set to 0, empty pages are not treated specially
357 * and merged as usual.
358 */
359 if (use_zero_pages && !assert_ksm_pages_count(0))
360 goto err_out;
361 else if (!use_zero_pages && !assert_ksm_pages_count(page_count))
362 goto err_out;
363
364 printf("OK\n");
365 munmap(map_ptr, page_size * page_count);
366 return KSFT_PASS;
367
368 err_out:
369 printf("Not OK\n");
370 munmap(map_ptr, page_size * page_count);
371 return KSFT_FAIL;
372 }
373
get_next_mem_node(int node)374 static int get_next_mem_node(int node)
375 {
376
377 long node_size;
378 int mem_node = 0;
379 int i, max_node = numa_max_node();
380
381 for (i = node + 1; i <= max_node + node; i++) {
382 mem_node = i % (max_node + 1);
383 node_size = numa_node_size(mem_node, NULL);
384 if (node_size > 0)
385 break;
386 }
387 return mem_node;
388 }
389
get_first_mem_node(void)390 static int get_first_mem_node(void)
391 {
392 return get_next_mem_node(numa_max_node());
393 }
394
check_ksm_numa_merge(int mapping,int prot,int timeout,bool merge_across_nodes,size_t page_size)395 static int check_ksm_numa_merge(int mapping, int prot, int timeout, bool merge_across_nodes,
396 size_t page_size)
397 {
398 void *numa1_map_ptr, *numa2_map_ptr;
399 struct timespec start_time;
400 int page_count = 2;
401 int first_node;
402
403 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
404 perror("clock_gettime");
405 return KSFT_FAIL;
406 }
407
408 if (numa_available() < 0) {
409 perror("NUMA support not enabled");
410 return KSFT_SKIP;
411 }
412 if (numa_num_configured_nodes() <= 1) {
413 printf("At least 2 NUMA nodes must be available\n");
414 return KSFT_SKIP;
415 }
416 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
417 return KSFT_FAIL;
418
419 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
420 first_node = get_first_mem_node();
421 numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
422 numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
423 if (!numa1_map_ptr || !numa2_map_ptr) {
424 perror("numa_alloc_onnode");
425 return KSFT_FAIL;
426 }
427
428 memset(numa1_map_ptr, '*', page_size);
429 memset(numa2_map_ptr, '*', page_size);
430
431 /* try to merge the pages */
432 if (ksm_merge_pages(numa1_map_ptr, page_size, start_time, timeout) ||
433 ksm_merge_pages(numa2_map_ptr, page_size, start_time, timeout))
434 goto err_out;
435
436 /*
437 * verify that the right number of pages are merged:
438 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged;
439 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is
440 * only 1 unique page in each node and they can't be shared.
441 */
442 if (merge_across_nodes && !assert_ksm_pages_count(page_count))
443 goto err_out;
444 else if (!merge_across_nodes && !assert_ksm_pages_count(0))
445 goto err_out;
446
447 numa_free(numa1_map_ptr, page_size);
448 numa_free(numa2_map_ptr, page_size);
449 printf("OK\n");
450 return KSFT_PASS;
451
452 err_out:
453 numa_free(numa1_map_ptr, page_size);
454 numa_free(numa2_map_ptr, page_size);
455 printf("Not OK\n");
456 return KSFT_FAIL;
457 }
458
allocate_transhuge(void * ptr,int pagemap_fd)459 int64_t allocate_transhuge(void *ptr, int pagemap_fd)
460 {
461 uint64_t ent[2];
462
463 /* drop pmd */
464 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
465 MAP_FIXED | MAP_ANONYMOUS |
466 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
467 errx(2, "mmap transhuge");
468
469 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
470 err(2, "MADV_HUGEPAGE");
471
472 /* allocate transparent huge page */
473 *(volatile void **)ptr = ptr;
474
475 if (pread(pagemap_fd, ent, sizeof(ent),
476 (uintptr_t)ptr >> (PAGE_SHIFT - 3)) != sizeof(ent))
477 err(2, "read pagemap");
478
479 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
480 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
481 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - PAGE_SHIFT)) - 1)))
482 return PAGEMAP_PFN(ent[0]);
483
484 return -1;
485 }
486
ksm_merge_hugepages_time(int mapping,int prot,int timeout,size_t map_size)487 static int ksm_merge_hugepages_time(int mapping, int prot, int timeout, size_t map_size)
488 {
489 void *map_ptr, *map_ptr_orig;
490 struct timespec start_time, end_time;
491 unsigned long scan_time_ns;
492 int pagemap_fd, n_normal_pages, n_huge_pages;
493
494 map_size *= MB;
495 size_t len = map_size;
496
497 len -= len % HPAGE_SIZE;
498 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
499 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
500 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE;
501
502 if (map_ptr_orig == MAP_FAILED)
503 err(2, "initial mmap");
504
505 if (madvise(map_ptr, len + HPAGE_SIZE, MADV_HUGEPAGE))
506 err(2, "MADV_HUGEPAGE");
507
508 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
509 if (pagemap_fd < 0)
510 err(2, "open pagemap");
511
512 n_normal_pages = 0;
513 n_huge_pages = 0;
514 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) {
515 if (allocate_transhuge(p, pagemap_fd) < 0)
516 n_normal_pages++;
517 else
518 n_huge_pages++;
519 }
520 printf("Number of normal pages: %d\n", n_normal_pages);
521 printf("Number of huge pages: %d\n", n_huge_pages);
522
523 memset(map_ptr, '*', len);
524
525 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
526 perror("clock_gettime");
527 goto err_out;
528 }
529 if (ksm_merge_pages(map_ptr, map_size, start_time, timeout))
530 goto err_out;
531 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
532 perror("clock_gettime");
533 goto err_out;
534 }
535
536 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
537 (end_time.tv_nsec - start_time.tv_nsec);
538
539 printf("Total size: %lu MiB\n", map_size / MB);
540 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
541 scan_time_ns % NSEC_PER_SEC);
542 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /
543 ((double)scan_time_ns / NSEC_PER_SEC));
544
545 munmap(map_ptr_orig, len + HPAGE_SIZE);
546 return KSFT_PASS;
547
548 err_out:
549 printf("Not OK\n");
550 munmap(map_ptr_orig, len + HPAGE_SIZE);
551 return KSFT_FAIL;
552 }
553
ksm_merge_time(int mapping,int prot,int timeout,size_t map_size)554 static int ksm_merge_time(int mapping, int prot, int timeout, size_t map_size)
555 {
556 void *map_ptr;
557 struct timespec start_time, end_time;
558 unsigned long scan_time_ns;
559
560 map_size *= MB;
561
562 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size);
563 if (!map_ptr)
564 return KSFT_FAIL;
565
566 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
567 perror("clock_gettime");
568 goto err_out;
569 }
570 if (ksm_merge_pages(map_ptr, map_size, start_time, timeout))
571 goto err_out;
572 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
573 perror("clock_gettime");
574 goto err_out;
575 }
576
577 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
578 (end_time.tv_nsec - start_time.tv_nsec);
579
580 printf("Total size: %lu MiB\n", map_size / MB);
581 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC,
582 scan_time_ns % NSEC_PER_SEC);
583 printf("Average speed: %.3f MiB/s\n", (map_size / MB) /
584 ((double)scan_time_ns / NSEC_PER_SEC));
585
586 munmap(map_ptr, map_size);
587 return KSFT_PASS;
588
589 err_out:
590 printf("Not OK\n");
591 munmap(map_ptr, map_size);
592 return KSFT_FAIL;
593 }
594
ksm_cow_time(int mapping,int prot,int timeout,size_t page_size)595 static int ksm_cow_time(int mapping, int prot, int timeout, size_t page_size)
596 {
597 void *map_ptr;
598 struct timespec start_time, end_time;
599 unsigned long cow_time_ns;
600
601 /* page_count must be less than 2*page_size */
602 size_t page_count = 4000;
603
604 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count);
605 if (!map_ptr)
606 return KSFT_FAIL;
607
608 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
609 perror("clock_gettime");
610 return KSFT_FAIL;
611 }
612 for (size_t i = 0; i < page_count - 1; i = i + 2)
613 memset(map_ptr + page_size * i, '-', 1);
614 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
615 perror("clock_gettime");
616 return KSFT_FAIL;
617 }
618
619 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
620 (end_time.tv_nsec - start_time.tv_nsec);
621
622 printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB);
623 printf("Not merged pages:\n");
624 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
625 cow_time_ns % NSEC_PER_SEC);
626 printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) /
627 ((double)cow_time_ns / NSEC_PER_SEC));
628
629 /* Create 2000 pairs of duplicate pages */
630 for (size_t i = 0; i < page_count - 1; i = i + 2) {
631 memset(map_ptr + page_size * i, '+', i / 2 + 1);
632 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1);
633 }
634 if (ksm_merge_pages(map_ptr, page_size * page_count, start_time, timeout))
635 goto err_out;
636
637 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
638 perror("clock_gettime");
639 goto err_out;
640 }
641 for (size_t i = 0; i < page_count - 1; i = i + 2)
642 memset(map_ptr + page_size * i, '-', 1);
643 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) {
644 perror("clock_gettime");
645 goto err_out;
646 }
647
648 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC +
649 (end_time.tv_nsec - start_time.tv_nsec);
650
651 printf("Merged pages:\n");
652 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC,
653 cow_time_ns % NSEC_PER_SEC);
654 printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) /
655 ((double)cow_time_ns / NSEC_PER_SEC));
656
657 munmap(map_ptr, page_size * page_count);
658 return KSFT_PASS;
659
660 err_out:
661 printf("Not OK\n");
662 munmap(map_ptr, page_size * page_count);
663 return KSFT_FAIL;
664 }
665
main(int argc,char * argv[])666 int main(int argc, char *argv[])
667 {
668 int ret, opt;
669 int prot = 0;
670 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT;
671 long page_count = KSM_PAGE_COUNT_DEFAULT;
672 size_t page_size = sysconf(_SC_PAGESIZE);
673 struct ksm_sysfs ksm_sysfs_old;
674 int test_name = CHECK_KSM_MERGE;
675 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT;
676 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT;
677 long size_MB = 0;
678
679 while ((opt = getopt(argc, argv, "ha:p:l:z:m:s:MUZNPCH")) != -1) {
680 switch (opt) {
681 case 'a':
682 prot = str_to_prot(optarg);
683 break;
684 case 'p':
685 page_count = atol(optarg);
686 if (page_count <= 0) {
687 printf("The number of pages must be greater than 0\n");
688 return KSFT_FAIL;
689 }
690 break;
691 case 'l':
692 ksm_scan_limit_sec = atoi(optarg);
693 if (ksm_scan_limit_sec <= 0) {
694 printf("Timeout value must be greater than 0\n");
695 return KSFT_FAIL;
696 }
697 break;
698 case 'h':
699 print_help();
700 break;
701 case 'z':
702 if (strcmp(optarg, "0") == 0)
703 use_zero_pages = 0;
704 else
705 use_zero_pages = 1;
706 break;
707 case 'm':
708 if (strcmp(optarg, "0") == 0)
709 merge_across_nodes = 0;
710 else
711 merge_across_nodes = 1;
712 break;
713 case 's':
714 size_MB = atoi(optarg);
715 if (size_MB <= 0) {
716 printf("Size must be greater than 0\n");
717 return KSFT_FAIL;
718 }
719 case 'M':
720 break;
721 case 'U':
722 test_name = CHECK_KSM_UNMERGE;
723 break;
724 case 'Z':
725 test_name = CHECK_KSM_ZERO_PAGE_MERGE;
726 break;
727 case 'N':
728 test_name = CHECK_KSM_NUMA_MERGE;
729 break;
730 case 'P':
731 test_name = KSM_MERGE_TIME;
732 break;
733 case 'H':
734 test_name = KSM_MERGE_TIME_HUGE_PAGES;
735 break;
736 case 'C':
737 test_name = KSM_COW_TIME;
738 break;
739 default:
740 return KSFT_FAIL;
741 }
742 }
743
744 if (prot == 0)
745 prot = str_to_prot(KSM_PROT_STR_DEFAULT);
746
747 if (access(KSM_SYSFS_PATH, F_OK)) {
748 printf("Config KSM not enabled\n");
749 return KSFT_SKIP;
750 }
751
752 if (ksm_save_def(&ksm_sysfs_old)) {
753 printf("Cannot save default tunables\n");
754 return KSFT_FAIL;
755 }
756
757 if (ksm_write_sysfs(KSM_FP("run"), 2) ||
758 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
759 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
760 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
761 return KSFT_FAIL;
762
763 switch (test_name) {
764 case CHECK_KSM_MERGE:
765 ret = check_ksm_merge(MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
766 ksm_scan_limit_sec, page_size);
767 break;
768 case CHECK_KSM_UNMERGE:
769 ret = check_ksm_unmerge(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
770 page_size);
771 break;
772 case CHECK_KSM_ZERO_PAGE_MERGE:
773 ret = check_ksm_zero_page_merge(MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count,
774 ksm_scan_limit_sec, use_zero_pages, page_size);
775 break;
776 case CHECK_KSM_NUMA_MERGE:
777 ret = check_ksm_numa_merge(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
778 merge_across_nodes, page_size);
779 break;
780 case KSM_MERGE_TIME:
781 if (size_MB == 0) {
782 printf("Option '-s' is required.\n");
783 return KSFT_FAIL;
784 }
785 ret = ksm_merge_time(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
786 size_MB);
787 break;
788 case KSM_MERGE_TIME_HUGE_PAGES:
789 if (size_MB == 0) {
790 printf("Option '-s' is required.\n");
791 return KSFT_FAIL;
792 }
793 ret = ksm_merge_hugepages_time(MAP_PRIVATE | MAP_ANONYMOUS, prot,
794 ksm_scan_limit_sec, size_MB);
795 break;
796 case KSM_COW_TIME:
797 ret = ksm_cow_time(MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec,
798 page_size);
799 break;
800 }
801
802 if (ksm_restore(&ksm_sysfs_old)) {
803 printf("Cannot restore default tunables\n");
804 return KSFT_FAIL;
805 }
806
807 return ret;
808 }
809