1 // Copyright (C) 2002-2019 Free Software Foundation, Inc.
2 //
3 // This file is part of GCC.
4 //
5 // GCC is free software; you can redistribute it and/or modify
6 // it under the terms of the GNU General Public License as published by
7 // the Free Software Foundation; either version 3, or (at your option)
8 // any later version.
9
10 // GCC is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 // GNU General Public License for more details.
14
15 // Under Section 7 of GPL version 3, you are granted additional
16 // permissions described in the GCC Runtime Library Exception, version
17 // 3.1, as published by the Free Software Foundation.
18
19 // You should have received a copy of the GNU General Public License and
20 // a copy of the GCC Runtime Library Exception along with this program;
21 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
22 // <http://www.gnu.org/licenses/>.
23
24 // Written by Mark Mitchell, CodeSourcery LLC, <mark@codesourcery.com>
25 // Thread support written by Jason Merrill, Red Hat Inc. <jason@redhat.com>
26
27 #include <bits/c++config.h>
28 #include <cxxabi.h>
29 #include <exception>
30 #include <new>
31 #include <ext/atomicity.h>
32 #include <ext/concurrence.h>
33 #include <bits/atomic_lockfree_defines.h>
34 #if defined(__GTHREADS) && defined(__GTHREAD_HAS_COND) \
35 && (ATOMIC_INT_LOCK_FREE > 1) && defined(_GLIBCXX_HAVE_LINUX_FUTEX)
36 # include <climits>
37 # include <syscall.h>
38 # include <unistd.h>
39 # define _GLIBCXX_USE_FUTEX
40 # define _GLIBCXX_FUTEX_WAIT 0
41 # define _GLIBCXX_FUTEX_WAKE 1
42 #endif
43
44 // The IA64/generic ABI uses the first byte of the guard variable.
45 // The ARM EABI uses the least significant bit.
46
47 // Thread-safe static local initialization support.
48 #ifdef __GTHREADS
49 # ifndef _GLIBCXX_USE_FUTEX
50 namespace
51 {
52 // A single mutex controlling all static initializations.
53 static __gnu_cxx::__recursive_mutex* static_mutex;
54
55 typedef char fake_recursive_mutex[sizeof(__gnu_cxx::__recursive_mutex)]
56 __attribute__ ((aligned(__alignof__(__gnu_cxx::__recursive_mutex))));
57 fake_recursive_mutex fake_mutex;
58
init()59 static void init()
60 { static_mutex = new (&fake_mutex) __gnu_cxx::__recursive_mutex(); }
61
62 __gnu_cxx::__recursive_mutex&
get_static_mutex()63 get_static_mutex()
64 {
65 static __gthread_once_t once = __GTHREAD_ONCE_INIT;
66 __gthread_once(&once, init);
67 return *static_mutex;
68 }
69
70 // Simple wrapper for exception safety.
71 struct mutex_wrapper
72 {
73 bool unlock;
mutex_wrapper__anon2631c22f0111::mutex_wrapper74 mutex_wrapper() : unlock(true)
75 { get_static_mutex().lock(); }
76
~mutex_wrapper__anon2631c22f0111::mutex_wrapper77 ~mutex_wrapper()
78 {
79 if (unlock)
80 static_mutex->unlock();
81 }
82 };
83 }
84 # endif
85
86 # if defined(__GTHREAD_HAS_COND) && !defined(_GLIBCXX_USE_FUTEX)
87 namespace
88 {
89 // A single condition variable controlling all static initializations.
90 static __gnu_cxx::__cond* static_cond;
91
92 // using a fake type to avoid initializing a static class.
93 typedef char fake_cond_t[sizeof(__gnu_cxx::__cond)]
94 __attribute__ ((aligned(__alignof__(__gnu_cxx::__cond))));
95 fake_cond_t fake_cond;
96
init_static_cond()97 static void init_static_cond()
98 { static_cond = new (&fake_cond) __gnu_cxx::__cond(); }
99
100 __gnu_cxx::__cond&
get_static_cond()101 get_static_cond()
102 {
103 static __gthread_once_t once = __GTHREAD_ONCE_INIT;
104 __gthread_once(&once, init_static_cond);
105 return *static_cond;
106 }
107 }
108 # endif
109
110 # ifndef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
111
112 // Test the guard variable with a memory load with
113 // acquire semantics.
114
115 inline bool
__test_and_acquire(__cxxabiv1::__guard * g)116 __test_and_acquire (__cxxabiv1::__guard *g)
117 {
118 unsigned char __c;
119 unsigned char *__p = reinterpret_cast<unsigned char *>(g);
120 __atomic_load (__p, &__c, __ATOMIC_ACQUIRE);
121 (void) __p;
122 return _GLIBCXX_GUARD_TEST(&__c);
123 }
124 # define _GLIBCXX_GUARD_TEST_AND_ACQUIRE(G) __test_and_acquire (G)
125 # endif
126
127 # ifndef _GLIBCXX_GUARD_SET_AND_RELEASE
128
129 // Set the guard variable to 1 with memory order release semantics.
130
131 inline void
__set_and_release(__cxxabiv1::__guard * g)132 __set_and_release (__cxxabiv1::__guard *g)
133 {
134 unsigned char *__p = reinterpret_cast<unsigned char *>(g);
135 unsigned char val = 1;
136 __atomic_store (__p, &val, __ATOMIC_RELEASE);
137 (void) __p;
138 }
139 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) __set_and_release (G)
140 # endif
141
142 #else /* !__GTHREADS */
143
144 # undef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
145 # undef _GLIBCXX_GUARD_SET_AND_RELEASE
146 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) _GLIBCXX_GUARD_SET (G)
147
148 #endif /* __GTHREADS */
149
150 //
151 // Here are C++ run-time routines for guarded initialization of static
152 // variables. There are 4 scenarios under which these routines are called:
153 //
154 // 1. Threads not supported (__GTHREADS not defined)
155 // 2. Threads are supported but not enabled at run-time.
156 // 3. Threads enabled at run-time but __gthreads_* are not fully POSIX.
157 // 4. Threads enabled at run-time and __gthreads_* support all POSIX threads
158 // primitives we need here.
159 //
160 // The old code supported scenarios 1-3 but was broken since it used a global
161 // mutex for all threads and had the mutex locked during the whole duration of
162 // initialization of a guarded static variable. The following created a
163 // dead-lock with the old code.
164 //
165 // Thread 1 acquires the global mutex.
166 // Thread 1 starts initializing static variable.
167 // Thread 1 creates thread 2 during initialization.
168 // Thread 2 attempts to acquire mutex to initialize another variable.
169 // Thread 2 blocks since thread 1 is locking the mutex.
170 // Thread 1 waits for result from thread 2 and also blocks. A deadlock.
171 //
172 // The new code here can handle this situation and thus is more robust. However,
173 // we need to use the POSIX thread condition variable, which is not supported
174 // in all platforms, notably older versions of Microsoft Windows. The gthr*.h
175 // headers define a symbol __GTHREAD_HAS_COND for platforms that support POSIX
176 // like condition variables. For platforms that do not support condition
177 // variables, we need to fall back to the old code.
178
179 // If _GLIBCXX_USE_FUTEX, no global mutex or condition variable is used,
180 // only atomic operations are used together with futex syscall.
181 // Valid values of the first integer in guard are:
182 // 0 No thread encountered the guarded init
183 // yet or it has been aborted.
184 // _GLIBCXX_GUARD_BIT The guarded static var has been successfully
185 // initialized.
186 // _GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
187 // and no other thread is waiting for its
188 // initialization.
189 // (_GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
190 // | _GLIBCXX_GUARD_WAITING_BIT) and some other threads are waiting until
191 // it is initialized.
192
193 namespace __cxxabiv1
194 {
195 #ifdef _GLIBCXX_USE_FUTEX
196 namespace
197 {
__guard_test_bit(const int __byte,const int __val)198 static inline int __guard_test_bit (const int __byte, const int __val)
199 {
200 union { int __i; char __c[sizeof (int)]; } __u = { 0 };
201 __u.__c[__byte] = __val;
202 return __u.__i;
203 }
204 }
205 #endif
206
207 static inline int
init_in_progress_flag(__guard * g)208 init_in_progress_flag(__guard* g)
209 { return ((char *)g)[1]; }
210
211 static inline void
set_init_in_progress_flag(__guard * g,int v)212 set_init_in_progress_flag(__guard* g, int v)
213 { ((char *)g)[1] = v; }
214
215 static inline void
throw_recursive_init_exception()216 throw_recursive_init_exception()
217 {
218 #if __cpp_exceptions
219 throw __gnu_cxx::recursive_init_error();
220 #else
221 // Use __builtin_trap so we don't require abort().
222 __builtin_trap();
223 #endif
224 }
225
226 // acquire() is a helper function used to acquire guard if thread support is
227 // not compiled in or is compiled in but not enabled at run-time.
228 static int
acquire(__guard * g)229 acquire(__guard *g)
230 {
231 // Quit if the object is already initialized.
232 if (_GLIBCXX_GUARD_TEST(g))
233 return 0;
234
235 if (init_in_progress_flag(g))
236 throw_recursive_init_exception();
237
238 set_init_in_progress_flag(g, 1);
239 return 1;
240 }
241
242 extern "C"
__cxa_guard_acquire(__guard * g)243 int __cxa_guard_acquire (__guard *g)
244 {
245 #ifdef __GTHREADS
246 // If the target can reorder loads, we need to insert a read memory
247 // barrier so that accesses to the guarded variable happen after the
248 // guard test.
249 if (_GLIBCXX_GUARD_TEST_AND_ACQUIRE (g))
250 return 0;
251
252 # ifdef _GLIBCXX_USE_FUTEX
253 // If __atomic_* and futex syscall are supported, don't use any global
254 // mutex.
255 if (__gthread_active_p ())
256 {
257 int *gi = (int *) (void *) g;
258 const int guard_bit = _GLIBCXX_GUARD_BIT;
259 const int pending_bit = _GLIBCXX_GUARD_PENDING_BIT;
260 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
261
262 while (1)
263 {
264 int expected(0);
265 if (__atomic_compare_exchange_n(gi, &expected, pending_bit, false,
266 __ATOMIC_ACQ_REL,
267 __ATOMIC_ACQUIRE))
268 {
269 // This thread should do the initialization.
270 return 1;
271 }
272
273 if (expected == guard_bit)
274 {
275 // Already initialized.
276 return 0;
277 }
278
279 if (expected == pending_bit)
280 {
281 // Use acquire here.
282 int newv = expected | waiting_bit;
283 if (!__atomic_compare_exchange_n(gi, &expected, newv, false,
284 __ATOMIC_ACQ_REL,
285 __ATOMIC_ACQUIRE))
286 {
287 if (expected == guard_bit)
288 {
289 // Make a thread that failed to set the
290 // waiting bit exit the function earlier,
291 // if it detects that another thread has
292 // successfully finished initialising.
293 return 0;
294 }
295 if (expected == 0)
296 continue;
297 }
298
299 expected = newv;
300 }
301
302 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAIT, expected, 0);
303 }
304 }
305 # else
306 if (__gthread_active_p ())
307 {
308 mutex_wrapper mw;
309
310 while (1) // When this loop is executing, mutex is locked.
311 {
312 # ifdef __GTHREAD_HAS_COND
313 // The static is already initialized.
314 if (_GLIBCXX_GUARD_TEST(g))
315 return 0; // The mutex will be unlocked via wrapper
316
317 if (init_in_progress_flag(g))
318 {
319 // The guarded static is currently being initialized by
320 // another thread, so we release mutex and wait for the
321 // condition variable. We will lock the mutex again after
322 // this.
323 get_static_cond().wait_recursive(&get_static_mutex());
324 }
325 else
326 {
327 set_init_in_progress_flag(g, 1);
328 return 1; // The mutex will be unlocked via wrapper.
329 }
330 # else
331 // This provides compatibility with older systems not supporting
332 // POSIX like condition variables.
333 if (acquire(g))
334 {
335 mw.unlock = false;
336 return 1; // The mutex still locked.
337 }
338 return 0; // The mutex will be unlocked via wrapper.
339 # endif
340 }
341 }
342 # endif
343 #endif
344
345 return acquire (g);
346 }
347
348 extern "C"
__cxa_guard_abort(__guard * g)349 void __cxa_guard_abort (__guard *g) throw ()
350 {
351 #ifdef _GLIBCXX_USE_FUTEX
352 // If __atomic_* and futex syscall are supported, don't use any global
353 // mutex.
354 if (__gthread_active_p ())
355 {
356 int *gi = (int *) (void *) g;
357 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
358 int old = __atomic_exchange_n (gi, 0, __ATOMIC_ACQ_REL);
359
360 if ((old & waiting_bit) != 0)
361 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
362 return;
363 }
364 #elif defined(__GTHREAD_HAS_COND)
365 if (__gthread_active_p())
366 {
367 mutex_wrapper mw;
368
369 set_init_in_progress_flag(g, 0);
370
371 // If we abort, we still need to wake up all other threads waiting for
372 // the condition variable.
373 get_static_cond().broadcast();
374 return;
375 }
376 #endif
377
378 set_init_in_progress_flag(g, 0);
379 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
380 // This provides compatibility with older systems not supporting POSIX like
381 // condition variables.
382 if (__gthread_active_p ())
383 static_mutex->unlock();
384 #endif
385 }
386
387 extern "C"
__cxa_guard_release(__guard * g)388 void __cxa_guard_release (__guard *g) throw ()
389 {
390 #ifdef _GLIBCXX_USE_FUTEX
391 // If __atomic_* and futex syscall are supported, don't use any global
392 // mutex.
393 if (__gthread_active_p ())
394 {
395 int *gi = (int *) (void *) g;
396 const int guard_bit = _GLIBCXX_GUARD_BIT;
397 const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
398 int old = __atomic_exchange_n (gi, guard_bit, __ATOMIC_ACQ_REL);
399
400 if ((old & waiting_bit) != 0)
401 syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
402 return;
403 }
404 #elif defined(__GTHREAD_HAS_COND)
405 if (__gthread_active_p())
406 {
407 mutex_wrapper mw;
408
409 set_init_in_progress_flag(g, 0);
410 _GLIBCXX_GUARD_SET_AND_RELEASE(g);
411
412 get_static_cond().broadcast();
413 return;
414 }
415 #endif
416
417 set_init_in_progress_flag(g, 0);
418 _GLIBCXX_GUARD_SET_AND_RELEASE (g);
419
420 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
421 // This provides compatibility with older systems not supporting POSIX like
422 // condition variables.
423 if (__gthread_active_p())
424 static_mutex->unlock();
425 #endif
426 }
427 }
428