1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 /*
4 * This header provides generic wrappers for memory access instrumentation that
5 * the compiler cannot emit for: KASAN, KCSAN, KMSAN.
6 */
7 #ifndef _LINUX_INSTRUMENTED_H
8 #define _LINUX_INSTRUMENTED_H
9
10 #include <linux/compiler.h>
11 #include <linux/kasan-checks.h>
12 #include <linux/kcsan-checks.h>
13 #include <linux/kmsan-checks.h>
14 #include <linux/types.h>
15
16 /**
17 * instrument_read - instrument regular read access
18 *
19 * Instrument a regular read access. The instrumentation should be inserted
20 * before the actual read happens.
21 *
22 * @ptr address of access
23 * @size size of access
24 */
instrument_read(const volatile void * v,size_t size)25 static __always_inline void instrument_read(const volatile void *v, size_t size)
26 {
27 kasan_check_read(v, size);
28 kcsan_check_read(v, size);
29 }
30
31 /**
32 * instrument_write - instrument regular write access
33 *
34 * Instrument a regular write access. The instrumentation should be inserted
35 * before the actual write happens.
36 *
37 * @ptr address of access
38 * @size size of access
39 */
instrument_write(const volatile void * v,size_t size)40 static __always_inline void instrument_write(const volatile void *v, size_t size)
41 {
42 kasan_check_write(v, size);
43 kcsan_check_write(v, size);
44 }
45
46 /**
47 * instrument_read_write - instrument regular read-write access
48 *
49 * Instrument a regular write access. The instrumentation should be inserted
50 * before the actual write happens.
51 *
52 * @ptr address of access
53 * @size size of access
54 */
instrument_read_write(const volatile void * v,size_t size)55 static __always_inline void instrument_read_write(const volatile void *v, size_t size)
56 {
57 kasan_check_write(v, size);
58 kcsan_check_read_write(v, size);
59 }
60
61 /**
62 * instrument_atomic_read - instrument atomic read access
63 *
64 * Instrument an atomic read access. The instrumentation should be inserted
65 * before the actual read happens.
66 *
67 * @ptr address of access
68 * @size size of access
69 */
instrument_atomic_read(const volatile void * v,size_t size)70 static __always_inline void instrument_atomic_read(const volatile void *v, size_t size)
71 {
72 kasan_check_read(v, size);
73 kcsan_check_atomic_read(v, size);
74 }
75
76 /**
77 * instrument_atomic_write - instrument atomic write access
78 *
79 * Instrument an atomic write access. The instrumentation should be inserted
80 * before the actual write happens.
81 *
82 * @ptr address of access
83 * @size size of access
84 */
instrument_atomic_write(const volatile void * v,size_t size)85 static __always_inline void instrument_atomic_write(const volatile void *v, size_t size)
86 {
87 kasan_check_write(v, size);
88 kcsan_check_atomic_write(v, size);
89 }
90
91 /**
92 * instrument_atomic_read_write - instrument atomic read-write access
93 *
94 * Instrument an atomic read-write access. The instrumentation should be
95 * inserted before the actual write happens.
96 *
97 * @ptr address of access
98 * @size size of access
99 */
instrument_atomic_read_write(const volatile void * v,size_t size)100 static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size)
101 {
102 kasan_check_write(v, size);
103 kcsan_check_atomic_read_write(v, size);
104 }
105
106 /**
107 * instrument_copy_to_user - instrument reads of copy_to_user
108 *
109 * Instrument reads from kernel memory, that are due to copy_to_user (and
110 * variants). The instrumentation must be inserted before the accesses.
111 *
112 * @to destination address
113 * @from source address
114 * @n number of bytes to copy
115 */
116 static __always_inline void
instrument_copy_to_user(void __user * to,const void * from,unsigned long n)117 instrument_copy_to_user(void __user *to, const void *from, unsigned long n)
118 {
119 kasan_check_read(from, n);
120 kcsan_check_read(from, n);
121 kmsan_copy_to_user(to, from, n, 0);
122 }
123
124 /**
125 * instrument_copy_from_user_before - add instrumentation before copy_from_user
126 *
127 * Instrument writes to kernel memory, that are due to copy_from_user (and
128 * variants). The instrumentation should be inserted before the accesses.
129 *
130 * @to destination address
131 * @from source address
132 * @n number of bytes to copy
133 */
134 static __always_inline void
instrument_copy_from_user_before(const void * to,const void __user * from,unsigned long n)135 instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n)
136 {
137 kasan_check_write(to, n);
138 kcsan_check_write(to, n);
139 }
140
141 /**
142 * instrument_copy_from_user_after - add instrumentation after copy_from_user
143 *
144 * Instrument writes to kernel memory, that are due to copy_from_user (and
145 * variants). The instrumentation should be inserted after the accesses.
146 *
147 * @to destination address
148 * @from source address
149 * @n number of bytes to copy
150 * @left number of bytes not copied (as returned by copy_from_user)
151 */
152 static __always_inline void
instrument_copy_from_user_after(const void * to,const void __user * from,unsigned long n,unsigned long left)153 instrument_copy_from_user_after(const void *to, const void __user *from,
154 unsigned long n, unsigned long left)
155 {
156 kmsan_unpoison_memory(to, n - left);
157 }
158
159 /**
160 * instrument_get_user() - add instrumentation to get_user()-like macros
161 *
162 * get_user() and friends are fragile, so it may depend on the implementation
163 * whether the instrumentation happens before or after the data is copied from
164 * the userspace.
165 *
166 * @to destination variable, may not be address-taken
167 */
168 #define instrument_get_user(to) \
169 ({ \
170 u64 __tmp = (u64)(to); \
171 kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \
172 to = __tmp; \
173 })
174
175
176 /**
177 * instrument_put_user() - add instrumentation to put_user()-like macros
178 *
179 * put_user() and friends are fragile, so it may depend on the implementation
180 * whether the instrumentation happens before or after the data is copied from
181 * the userspace.
182 *
183 * @from source address
184 * @ptr userspace pointer to copy to
185 * @size number of bytes to copy
186 */
187 #define instrument_put_user(from, ptr, size) \
188 ({ \
189 kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \
190 })
191
192 #endif /* _LINUX_INSTRUMENTED_H */
193