1 /******************************************************************************
2 * arch/x86/mm/mm-locks.h
3 *
4 * Spinlocks used by the code in arch/x86/mm.
5 *
6 * Copyright (c) 2011 Citrix Systems, inc.
7 * Copyright (c) 2007 Advanced Micro Devices (Wei Huang)
8 * Copyright (c) 2006-2007 XenSource Inc.
9 * Copyright (c) 2006 Michael A Fetterman
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; If not, see <http://www.gnu.org/licenses/>.
23 */
24
25 #ifndef _MM_LOCKS_H
26 #define _MM_LOCKS_H
27
28 #include <asm/mem_sharing.h>
29
30 /* Per-CPU variable for enforcing the lock ordering */
31 DECLARE_PER_CPU(int, mm_lock_level);
32 #define __get_lock_level() (this_cpu(mm_lock_level))
33
34 DECLARE_PERCPU_RWLOCK_GLOBAL(p2m_percpu_rwlock);
35
mm_lock_init(mm_lock_t * l)36 static inline void mm_lock_init(mm_lock_t *l)
37 {
38 spin_lock_init(&l->lock);
39 l->locker = -1;
40 l->locker_function = "nobody";
41 l->unlock_level = 0;
42 }
43
mm_locked_by_me(mm_lock_t * l)44 static inline int mm_locked_by_me(mm_lock_t *l)
45 {
46 return (l->lock.recurse_cpu == current->processor);
47 }
48
49 /*
50 * If you see this crash, the numbers printed are order levels defined
51 * in this file.
52 */
53 #define __check_lock_level(l) \
54 do { \
55 if ( unlikely(__get_lock_level() > (l)) ) \
56 { \
57 printk("mm locking order violation: %i > %i\n", \
58 __get_lock_level(), (l)); \
59 BUG(); \
60 } \
61 } while(0)
62
63 #define __set_lock_level(l) \
64 do { \
65 __get_lock_level() = (l); \
66 } while(0)
67
_mm_lock(mm_lock_t * l,const char * func,int level,int rec)68 static inline void _mm_lock(mm_lock_t *l, const char *func, int level, int rec)
69 {
70 if ( !((mm_locked_by_me(l)) && rec) )
71 __check_lock_level(level);
72 spin_lock_recursive(&l->lock);
73 if ( l->lock.recurse_cnt == 1 )
74 {
75 l->locker_function = func;
76 l->unlock_level = __get_lock_level();
77 }
78 else if ( (unlikely(!rec)) )
79 panic("mm lock already held by %s", l->locker_function);
80 __set_lock_level(level);
81 }
82
_mm_enforce_order_lock_pre(int level)83 static inline void _mm_enforce_order_lock_pre(int level)
84 {
85 __check_lock_level(level);
86 }
87
_mm_enforce_order_lock_post(int level,int * unlock_level,unsigned short * recurse_count)88 static inline void _mm_enforce_order_lock_post(int level, int *unlock_level,
89 unsigned short *recurse_count)
90 {
91 if ( recurse_count )
92 {
93 if ( (*recurse_count)++ == 0 )
94 {
95 *unlock_level = __get_lock_level();
96 }
97 } else {
98 *unlock_level = __get_lock_level();
99 }
100 __set_lock_level(level);
101 }
102
103
mm_rwlock_init(mm_rwlock_t * l)104 static inline void mm_rwlock_init(mm_rwlock_t *l)
105 {
106 percpu_rwlock_resource_init(&l->lock, p2m_percpu_rwlock);
107 l->locker = -1;
108 l->locker_function = "nobody";
109 l->unlock_level = 0;
110 }
111
mm_write_locked_by_me(mm_rwlock_t * l)112 static inline int mm_write_locked_by_me(mm_rwlock_t *l)
113 {
114 return (l->locker == get_processor_id());
115 }
116
_mm_write_lock(mm_rwlock_t * l,const char * func,int level)117 static inline void _mm_write_lock(mm_rwlock_t *l, const char *func, int level)
118 {
119 if ( !mm_write_locked_by_me(l) )
120 {
121 __check_lock_level(level);
122 percpu_write_lock(p2m_percpu_rwlock, &l->lock);
123 l->locker = get_processor_id();
124 l->locker_function = func;
125 l->unlock_level = __get_lock_level();
126 __set_lock_level(level);
127 }
128 l->recurse_count++;
129 }
130
mm_write_unlock(mm_rwlock_t * l)131 static inline void mm_write_unlock(mm_rwlock_t *l)
132 {
133 if ( --(l->recurse_count) != 0 )
134 return;
135 l->locker = -1;
136 l->locker_function = "nobody";
137 __set_lock_level(l->unlock_level);
138 percpu_write_unlock(p2m_percpu_rwlock, &l->lock);
139 }
140
_mm_read_lock(mm_rwlock_t * l,int level)141 static inline void _mm_read_lock(mm_rwlock_t *l, int level)
142 {
143 __check_lock_level(level);
144 percpu_read_lock(p2m_percpu_rwlock, &l->lock);
145 /* There's nowhere to store the per-CPU unlock level so we can't
146 * set the lock level. */
147 }
148
mm_read_unlock(mm_rwlock_t * l)149 static inline void mm_read_unlock(mm_rwlock_t *l)
150 {
151 percpu_read_unlock(p2m_percpu_rwlock, &l->lock);
152 }
153
154 /* This wrapper uses the line number to express the locking order below */
155 #define declare_mm_lock(name) \
156 static inline void mm_lock_##name(mm_lock_t *l, const char *func, int rec)\
157 { _mm_lock(l, func, MM_LOCK_ORDER_##name, rec); }
158 #define declare_mm_rwlock(name) \
159 static inline void mm_write_lock_##name(mm_rwlock_t *l, const char *func) \
160 { _mm_write_lock(l, func, MM_LOCK_ORDER_##name); } \
161 static inline void mm_read_lock_##name(mm_rwlock_t *l) \
162 { _mm_read_lock(l, MM_LOCK_ORDER_##name); }
163 /* These capture the name of the calling function */
164 #define mm_lock(name, l) mm_lock_##name(l, __func__, 0)
165 #define mm_lock_recursive(name, l) mm_lock_##name(l, __func__, 1)
166 #define mm_write_lock(name, l) mm_write_lock_##name(l, __func__)
167 #define mm_read_lock(name, l) mm_read_lock_##name(l)
168
169 /* This wrapper is intended for "external" locks which do not use
170 * the mm_lock_t types. Such locks inside the mm code are also subject
171 * to ordering constraints. */
172 #define declare_mm_order_constraint(name) \
173 static inline void mm_enforce_order_lock_pre_##name(void) \
174 { _mm_enforce_order_lock_pre(MM_LOCK_ORDER_##name); } \
175 static inline void mm_enforce_order_lock_post_##name( \
176 int *unlock_level, unsigned short *recurse_count) \
177 { _mm_enforce_order_lock_post(MM_LOCK_ORDER_##name, unlock_level, recurse_count); } \
178
mm_unlock(mm_lock_t * l)179 static inline void mm_unlock(mm_lock_t *l)
180 {
181 if ( l->lock.recurse_cnt == 1 )
182 {
183 l->locker_function = "nobody";
184 __set_lock_level(l->unlock_level);
185 }
186 spin_unlock_recursive(&l->lock);
187 }
188
mm_enforce_order_unlock(int unlock_level,unsigned short * recurse_count)189 static inline void mm_enforce_order_unlock(int unlock_level,
190 unsigned short *recurse_count)
191 {
192 if ( recurse_count )
193 {
194 BUG_ON(*recurse_count == 0);
195 if ( (*recurse_count)-- == 1 )
196 {
197 __set_lock_level(unlock_level);
198 }
199 } else {
200 __set_lock_level(unlock_level);
201 }
202 }
203
204 /************************************************************************
205 * *
206 * To avoid deadlocks, these locks _MUST_ be taken in the order listed *
207 * below. The locking functions will enforce this. *
208 * *
209 ************************************************************************/
210
211 /* Nested P2M lock (per-domain)
212 *
213 * A per-domain lock that protects the mapping from nested-CR3 to
214 * nested-p2m. In particular it covers:
215 * - the array of nested-p2m tables, and all LRU activity therein; and
216 * - setting the "cr3" field of any p2m table to a non-P2M_BASE_EAADR value.
217 * (i.e. assigning a p2m table to be the shadow of that cr3 */
218
219 #define MM_LOCK_ORDER_nestedp2m 8
220 declare_mm_lock(nestedp2m)
221 #define nestedp2m_lock(d) mm_lock(nestedp2m, &(d)->arch.nested_p2m_lock)
222 #define nestedp2m_unlock(d) mm_unlock(&(d)->arch.nested_p2m_lock)
223
224 /* P2M lock (per-non-alt-p2m-table)
225 *
226 * This protects all queries and updates to the p2m table.
227 * Queries may be made under the read lock but all modifications
228 * need the main (write) lock.
229 *
230 * The write lock is recursive as it is common for a code path to look
231 * up a gfn and later mutate it.
232 *
233 * Note that this lock shares its implementation with the altp2m
234 * lock (not the altp2m list lock), so the implementation
235 * is found there.
236 *
237 * Changes made to the host p2m when in altp2m mode are propagated to the
238 * altp2ms synchronously in ept_set_entry(). At that point, we will hold
239 * the host p2m lock; propagating this change involves grabbing the
240 * altp2m_list lock, and the locks of the individual alternate p2ms. In
241 * order to allow us to maintain locking order discipline, we split the p2m
242 * lock into p2m (for host p2ms) and altp2m (for alternate p2ms), putting
243 * the altp2mlist lock in the middle.
244 */
245
246 #define MM_LOCK_ORDER_p2m 16
247 declare_mm_rwlock(p2m);
248
249 /* Sharing per page lock
250 *
251 * This is an external lock, not represented by an mm_lock_t. The memory
252 * sharing lock uses it to protect addition and removal of (gfn,domain)
253 * tuples to a shared page. We enforce order here against the p2m lock,
254 * which is taken after the page_lock to change the gfn's p2m entry.
255 *
256 * The lock is recursive because during share we lock two pages. */
257
258 #define MM_LOCK_ORDER_per_page_sharing 24
259 declare_mm_order_constraint(per_page_sharing)
260 #define page_sharing_mm_pre_lock() mm_enforce_order_lock_pre_per_page_sharing()
261 #define page_sharing_mm_post_lock(l, r) \
262 mm_enforce_order_lock_post_per_page_sharing((l), (r))
263 #define page_sharing_mm_unlock(l, r) mm_enforce_order_unlock((l), (r))
264
265 /* Alternate P2M list lock (per-domain)
266 *
267 * A per-domain lock that protects the list of alternate p2m's.
268 * Any operation that walks the list needs to acquire this lock.
269 * Additionally, before destroying an alternate p2m all VCPU's
270 * in the target domain must be paused.
271 */
272
273 #define MM_LOCK_ORDER_altp2mlist 32
274 declare_mm_lock(altp2mlist)
275 #define altp2m_list_lock(d) mm_lock(altp2mlist, &(d)->arch.altp2m_list_lock)
276 #define altp2m_list_unlock(d) mm_unlock(&(d)->arch.altp2m_list_lock)
277
278 /* P2M lock (per-altp2m-table)
279 *
280 * This protects all queries and updates to the p2m table.
281 * Queries may be made under the read lock but all modifications
282 * need the main (write) lock.
283 *
284 * The write lock is recursive as it is common for a code path to look
285 * up a gfn and later mutate it.
286 */
287
288 #define MM_LOCK_ORDER_altp2m 40
289 declare_mm_rwlock(altp2m);
290 #define p2m_lock(p) \
291 do { \
292 if ( p2m_is_altp2m(p) ) \
293 mm_write_lock(altp2m, &(p)->lock); \
294 else \
295 mm_write_lock(p2m, &(p)->lock); \
296 (p)->defer_flush++; \
297 } while (0)
298 #define p2m_unlock(p) \
299 do { \
300 if ( --(p)->defer_flush == 0 ) \
301 p2m_unlock_and_tlb_flush(p); \
302 else \
303 mm_write_unlock(&(p)->lock); \
304 } while (0)
305 #define gfn_lock(p,g,o) p2m_lock(p)
306 #define gfn_unlock(p,g,o) p2m_unlock(p)
307 #define p2m_read_lock(p) mm_read_lock(p2m, &(p)->lock)
308 #define p2m_read_unlock(p) mm_read_unlock(&(p)->lock)
309 #define p2m_locked_by_me(p) mm_write_locked_by_me(&(p)->lock)
310 #define gfn_locked_by_me(p,g) p2m_locked_by_me(p)
311
312 /* PoD lock (per-p2m-table)
313 *
314 * Protects private PoD data structs: entry and cache
315 * counts, page lists, sweep parameters. */
316
317 #define MM_LOCK_ORDER_pod 48
318 declare_mm_lock(pod)
319 #define pod_lock(p) mm_lock(pod, &(p)->pod.lock)
320 #define pod_unlock(p) mm_unlock(&(p)->pod.lock)
321 #define pod_locked_by_me(p) mm_locked_by_me(&(p)->pod.lock)
322
323 /* Page alloc lock (per-domain)
324 *
325 * This is an external lock, not represented by an mm_lock_t. However,
326 * pod code uses it in conjunction with the p2m lock, and expecting
327 * the ordering which we enforce here.
328 * The lock is not recursive. */
329
330 #define MM_LOCK_ORDER_page_alloc 56
331 declare_mm_order_constraint(page_alloc)
332 #define page_alloc_mm_pre_lock() mm_enforce_order_lock_pre_page_alloc()
333 #define page_alloc_mm_post_lock(l) mm_enforce_order_lock_post_page_alloc(&(l), NULL)
334 #define page_alloc_mm_unlock(l) mm_enforce_order_unlock((l), NULL)
335
336 /* Paging lock (per-domain)
337 *
338 * For shadow pagetables, this lock protects
339 * - all changes to shadow page table pages
340 * - the shadow hash table
341 * - the shadow page allocator
342 * - all changes to guest page table pages
343 * - all changes to the page_info->tlbflush_timestamp
344 * - the page_info->count fields on shadow pages
345 *
346 * For HAP, it protects the NPT/EPT tables and mode changes.
347 *
348 * It also protects the log-dirty bitmap from concurrent accesses (and
349 * teardowns, etc). */
350
351 #define MM_LOCK_ORDER_paging 64
352 declare_mm_lock(paging)
353 #define paging_lock(d) mm_lock(paging, &(d)->arch.paging.lock)
354 #define paging_lock_recursive(d) \
355 mm_lock_recursive(paging, &(d)->arch.paging.lock)
356 #define paging_unlock(d) mm_unlock(&(d)->arch.paging.lock)
357 #define paging_locked_by_me(d) mm_locked_by_me(&(d)->arch.paging.lock)
358
359 #endif /* _MM_LOCKS_H */
360