1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * KSM functional tests
4 *
5 * Copyright 2022, Red Hat, Inc.
6 *
7 * Author(s): David Hildenbrand <david@redhat.com>
8 */
9 #define _GNU_SOURCE
10 #include <stdlib.h>
11 #include <string.h>
12 #include <stdbool.h>
13 #include <stdint.h>
14 #include <unistd.h>
15 #include <errno.h>
16 #include <fcntl.h>
17 #include <sys/mman.h>
18 #include <sys/syscall.h>
19 #include <sys/ioctl.h>
20 #include <linux/userfaultfd.h>
21
22 #include "../kselftest.h"
23 #include "vm_util.h"
24
25 #define KiB 1024u
26 #define MiB (1024 * KiB)
27
28 static int ksm_fd;
29 static int ksm_full_scans_fd;
30 static int pagemap_fd;
31 static size_t pagesize;
32
range_maps_duplicates(char * addr,unsigned long size)33 static bool range_maps_duplicates(char *addr, unsigned long size)
34 {
35 unsigned long offs_a, offs_b, pfn_a, pfn_b;
36
37 /*
38 * There is no easy way to check if there are KSM pages mapped into
39 * this range. We only check that the range does not map the same PFN
40 * twice by comparing each pair of mapped pages.
41 */
42 for (offs_a = 0; offs_a < size; offs_a += pagesize) {
43 pfn_a = pagemap_get_pfn(pagemap_fd, addr + offs_a);
44 /* Page not present or PFN not exposed by the kernel. */
45 if (pfn_a == -1ul || !pfn_a)
46 continue;
47
48 for (offs_b = offs_a + pagesize; offs_b < size;
49 offs_b += pagesize) {
50 pfn_b = pagemap_get_pfn(pagemap_fd, addr + offs_b);
51 if (pfn_b == -1ul || !pfn_b)
52 continue;
53 if (pfn_a == pfn_b)
54 return true;
55 }
56 }
57 return false;
58 }
59
ksm_get_full_scans(void)60 static long ksm_get_full_scans(void)
61 {
62 char buf[10];
63 ssize_t ret;
64
65 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0);
66 if (ret <= 0)
67 return -errno;
68 buf[ret] = 0;
69
70 return strtol(buf, NULL, 10);
71 }
72
ksm_merge(void)73 static int ksm_merge(void)
74 {
75 long start_scans, end_scans;
76
77 /* Wait for two full scans such that any possible merging happened. */
78 start_scans = ksm_get_full_scans();
79 if (start_scans < 0)
80 return start_scans;
81 if (write(ksm_fd, "1", 1) != 1)
82 return -errno;
83 do {
84 end_scans = ksm_get_full_scans();
85 if (end_scans < 0)
86 return end_scans;
87 } while (end_scans < start_scans + 2);
88
89 return 0;
90 }
91
mmap_and_merge_range(char val,unsigned long size)92 static char *mmap_and_merge_range(char val, unsigned long size)
93 {
94 char *map;
95
96 map = mmap(NULL, size, PROT_READ|PROT_WRITE,
97 MAP_PRIVATE|MAP_ANON, -1, 0);
98 if (map == MAP_FAILED) {
99 ksft_test_result_fail("mmap() failed\n");
100 return MAP_FAILED;
101 }
102
103 /* Don't use THP. Ignore if THP are not around on a kernel. */
104 if (madvise(map, size, MADV_NOHUGEPAGE) && errno != EINVAL) {
105 ksft_test_result_fail("MADV_NOHUGEPAGE failed\n");
106 goto unmap;
107 }
108
109 /* Make sure each page contains the same values to merge them. */
110 memset(map, val, size);
111 if (madvise(map, size, MADV_MERGEABLE)) {
112 ksft_test_result_fail("MADV_MERGEABLE failed\n");
113 goto unmap;
114 }
115
116 /* Run KSM to trigger merging and wait. */
117 if (ksm_merge()) {
118 ksft_test_result_fail("Running KSM failed\n");
119 goto unmap;
120 }
121 return map;
122 unmap:
123 munmap(map, size);
124 return MAP_FAILED;
125 }
126
test_unmerge(void)127 static void test_unmerge(void)
128 {
129 const unsigned int size = 2 * MiB;
130 char *map;
131
132 ksft_print_msg("[RUN] %s\n", __func__);
133
134 map = mmap_and_merge_range(0xcf, size);
135 if (map == MAP_FAILED)
136 return;
137
138 if (madvise(map, size, MADV_UNMERGEABLE)) {
139 ksft_test_result_fail("MADV_UNMERGEABLE failed\n");
140 goto unmap;
141 }
142
143 ksft_test_result(!range_maps_duplicates(map, size),
144 "Pages were unmerged\n");
145 unmap:
146 munmap(map, size);
147 }
148
test_unmerge_discarded(void)149 static void test_unmerge_discarded(void)
150 {
151 const unsigned int size = 2 * MiB;
152 char *map;
153
154 ksft_print_msg("[RUN] %s\n", __func__);
155
156 map = mmap_and_merge_range(0xcf, size);
157 if (map == MAP_FAILED)
158 return;
159
160 /* Discard half of all mapped pages so we have pte_none() entries. */
161 if (madvise(map, size / 2, MADV_DONTNEED)) {
162 ksft_test_result_fail("MADV_DONTNEED failed\n");
163 goto unmap;
164 }
165
166 if (madvise(map, size, MADV_UNMERGEABLE)) {
167 ksft_test_result_fail("MADV_UNMERGEABLE failed\n");
168 goto unmap;
169 }
170
171 ksft_test_result(!range_maps_duplicates(map, size),
172 "Pages were unmerged\n");
173 unmap:
174 munmap(map, size);
175 }
176
177 #ifdef __NR_userfaultfd
test_unmerge_uffd_wp(void)178 static void test_unmerge_uffd_wp(void)
179 {
180 struct uffdio_writeprotect uffd_writeprotect;
181 struct uffdio_register uffdio_register;
182 const unsigned int size = 2 * MiB;
183 struct uffdio_api uffdio_api;
184 char *map;
185 int uffd;
186
187 ksft_print_msg("[RUN] %s\n", __func__);
188
189 map = mmap_and_merge_range(0xcf, size);
190 if (map == MAP_FAILED)
191 return;
192
193 /* See if UFFD is around. */
194 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
195 if (uffd < 0) {
196 ksft_test_result_skip("__NR_userfaultfd failed\n");
197 goto unmap;
198 }
199
200 /* See if UFFD-WP is around. */
201 uffdio_api.api = UFFD_API;
202 uffdio_api.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP;
203 if (ioctl(uffd, UFFDIO_API, &uffdio_api) < 0) {
204 ksft_test_result_fail("UFFDIO_API failed\n");
205 goto close_uffd;
206 }
207 if (!(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) {
208 ksft_test_result_skip("UFFD_FEATURE_PAGEFAULT_FLAG_WP not available\n");
209 goto close_uffd;
210 }
211
212 /* Register UFFD-WP, no need for an actual handler. */
213 uffdio_register.range.start = (unsigned long) map;
214 uffdio_register.range.len = size;
215 uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
216 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) < 0) {
217 ksft_test_result_fail("UFFDIO_REGISTER_MODE_WP failed\n");
218 goto close_uffd;
219 }
220
221 /* Write-protect the range using UFFD-WP. */
222 uffd_writeprotect.range.start = (unsigned long) map;
223 uffd_writeprotect.range.len = size;
224 uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_WP;
225 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) {
226 ksft_test_result_fail("UFFDIO_WRITEPROTECT failed\n");
227 goto close_uffd;
228 }
229
230 if (madvise(map, size, MADV_UNMERGEABLE)) {
231 ksft_test_result_fail("MADV_UNMERGEABLE failed\n");
232 goto close_uffd;
233 }
234
235 ksft_test_result(!range_maps_duplicates(map, size),
236 "Pages were unmerged\n");
237 close_uffd:
238 close(uffd);
239 unmap:
240 munmap(map, size);
241 }
242 #endif
243
main(int argc,char ** argv)244 int main(int argc, char **argv)
245 {
246 unsigned int tests = 2;
247 int err;
248
249 #ifdef __NR_userfaultfd
250 tests++;
251 #endif
252
253 ksft_print_header();
254 ksft_set_plan(tests);
255
256 pagesize = getpagesize();
257
258 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR);
259 if (ksm_fd < 0)
260 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/run\") failed\n");
261 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY);
262 if (ksm_full_scans_fd < 0)
263 ksft_exit_skip("open(\"/sys/kernel/mm/ksm/full_scans\") failed\n");
264 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
265 if (pagemap_fd < 0)
266 ksft_exit_skip("open(\"/proc/self/pagemap\") failed\n");
267
268 test_unmerge();
269 test_unmerge_discarded();
270 #ifdef __NR_userfaultfd
271 test_unmerge_uffd_wp();
272 #endif
273
274 err = ksft_get_fail_cnt();
275 if (err)
276 ksft_exit_fail_msg("%d out of %d tests failed\n",
277 err, ksft_test_num());
278 return ksft_exit_pass();
279 }
280