1 /* Copyright (C) 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc. 2 3 The GNU C Library is free software; you can redistribute it and/or 4 modify it under the terms of the GNU Lesser General Public 5 License as published by the Free Software Foundation; either 6 version 2.1 of the License, or (at your option) any later version. 7 8 The GNU C Library is distributed in the hope that it will be useful, 9 but WITHOUT ANY WARRANTY; without even the implied warranty of 10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 Lesser General Public License for more details. 12 13 You should have received a copy of the GNU Lesser General Public 14 License along with the GNU C Library; if not, see 15 <http://www.gnu.org/licenses/>. */ 16 17 #ifndef _LOWLEVELLOCK_H 18 #define _LOWLEVELLOCK_H 1 19 20 #define FUTEX_WAIT 0 21 #define FUTEX_WAKE 1 22 #define FUTEX_REQUEUE 3 23 #define FUTEX_CMP_REQUEUE 4 24 #define FUTEX_WAKE_OP 5 25 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) 26 #define FUTEX_LOCK_PI 6 27 #define FUTEX_UNLOCK_PI 7 28 #define FUTEX_TRYLOCK_PI 8 29 #define FUTEX_WAIT_BITSET 9 30 #define FUTEX_WAKE_BITSET 10 31 #define FUTEX_PRIVATE_FLAG 128 32 #define FUTEX_CLOCK_REALTIME 256 33 34 #define FUTEX_BITSET_MATCH_ANY 0xffffffff 35 36 #ifndef __ASSEMBLER__ 37 38 #include <time.h> 39 #include <sys/param.h> 40 #include <bits/pthreadtypes.h> 41 #include <atomic.h> 42 #include <sysdep.h> 43 #include <bits/kernel-features.h> 44 45 #if defined(__UCLIBC_USE_TIME64__) 46 #include "internal/time64_helpers.h" 47 #endif 48 49 /* Values for 'private' parameter of locking macros. Yes, the 50 definition seems to be backwards. But it is not. The bit will be 51 reversed before passing to the system call. */ 52 #define LLL_PRIVATE 0 53 #define LLL_SHARED FUTEX_PRIVATE_FLAG 54 55 56 #if !defined NOT_IN_libc || defined IS_IN_rtld 57 /* In libc.so or ld.so all futexes are private. */ 58 # ifdef __ASSUME_PRIVATE_FUTEX 59 # define __lll_private_flag(fl, private) \ 60 ((fl) | FUTEX_PRIVATE_FLAG) 61 # else 62 # define __lll_private_flag(fl, private) \ 63 ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) 64 # endif 65 #else 66 # ifdef __ASSUME_PRIVATE_FUTEX 67 # define __lll_private_flag(fl, private) \ 68 (((fl) | FUTEX_PRIVATE_FLAG) ^ (private)) 69 # else 70 # define __lll_private_flag(fl, private) \ 71 (__builtin_constant_p (private) \ 72 ? ((private) == 0 \ 73 ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) \ 74 : (fl)) \ 75 : ((fl) | (((private) ^ FUTEX_PRIVATE_FLAG) \ 76 & THREAD_GETMEM (THREAD_SELF, header.private_futex)))) 77 # endif 78 #endif 79 80 #define lll_futex_wait(futexp, val, private) \ 81 lll_futex_timed_wait(futexp, val, NULL, private) 82 83 #if defined(__UCLIBC_USE_TIME64__) && defined(__NR_futex_time64) 84 85 #define lll_futex_timed_wait(futexp, val, timespec, private) \ 86 ({ \ 87 INTERNAL_SYSCALL_DECL (__err); \ 88 long int __ret; \ 89 __ret = INTERNAL_SYSCALL (futex_time64, __err, 4, (futexp), \ 90 __lll_private_flag (FUTEX_WAIT, private), \ 91 (val), (TO_TS64_P(timespec))); \ 92 __ret; \ 93 }) 94 95 #define lll_futex_wake(futexp, nr, private) \ 96 ({ \ 97 INTERNAL_SYSCALL_DECL (__err); \ 98 long int __ret; \ 99 __ret = INTERNAL_SYSCALL (futex_time64, __err, 4, (futexp), \ 100 __lll_private_flag (FUTEX_WAKE, private), \ 101 (nr), 0); \ 102 __ret; \ 103 }) 104 105 #else 106 107 #define lll_futex_timed_wait(futexp, val, timespec, private) \ 108 ({ \ 109 INTERNAL_SYSCALL_DECL (__err); \ 110 long int __ret; \ 111 __ret = INTERNAL_SYSCALL (futex, __err, 4, (futexp), \ 112 __lll_private_flag (FUTEX_WAIT, private), \ 113 (val), (timespec)); \ 114 __ret; \ 115 }) 116 117 #define lll_futex_wake(futexp, nr, private) \ 118 ({ \ 119 INTERNAL_SYSCALL_DECL (__err); \ 120 long int __ret; \ 121 __ret = INTERNAL_SYSCALL (futex, __err, 4, (futexp), \ 122 __lll_private_flag (FUTEX_WAKE, private), \ 123 (nr), 0); \ 124 __ret; \ 125 }) 126 127 #endif 128 129 #define lll_robust_dead(futexv, private) \ 130 do \ 131 { \ 132 int *__futexp = &(futexv); \ 133 atomic_or (__futexp, FUTEX_OWNER_DIED); \ 134 lll_futex_wake (__futexp, 1, private); \ 135 } \ 136 while (0) 137 138 /* Returns non-zero if error happened, zero if success. */ 139 140 #if defined(__UCLIBC_USE_TIME64__) && defined(__NR_futex_time64) 141 142 #define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \ 143 ({ \ 144 INTERNAL_SYSCALL_DECL (__err); \ 145 long int __ret; \ 146 __ret = INTERNAL_SYSCALL (futex_time64, __err, 6, (futexp), \ 147 __lll_private_flag (FUTEX_CMP_REQUEUE, private),\ 148 (nr_wake), (nr_move), (mutex), (val)); \ 149 INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ 150 }) 151 152 153 /* Returns non-zero if error happened, zero if success. */ 154 #define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \ 155 ({ \ 156 INTERNAL_SYSCALL_DECL (__err); \ 157 long int __ret; \ 158 __ret = INTERNAL_SYSCALL (futex_time64, __err, 6, (futexp), \ 159 __lll_private_flag (FUTEX_WAKE_OP, private), \ 160 (nr_wake), (nr_wake2), (futexp2), \ 161 FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ 162 INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ 163 }) 164 165 #else 166 167 168 #define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val, private) \ 169 ({ \ 170 INTERNAL_SYSCALL_DECL (__err); \ 171 long int __ret; \ 172 __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ 173 __lll_private_flag (FUTEX_CMP_REQUEUE, private),\ 174 (nr_wake), (nr_move), (mutex), (val)); \ 175 INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ 176 }) 177 178 179 /* Returns non-zero if error happened, zero if success. */ 180 #define lll_futex_wake_unlock(futexp, nr_wake, nr_wake2, futexp2, private) \ 181 ({ \ 182 INTERNAL_SYSCALL_DECL (__err); \ 183 long int __ret; \ 184 __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \ 185 __lll_private_flag (FUTEX_WAKE_OP, private), \ 186 (nr_wake), (nr_wake2), (futexp2), \ 187 FUTEX_OP_CLEAR_WAKE_IF_GT_ONE); \ 188 INTERNAL_SYSCALL_ERROR_P (__ret, __err); \ 189 }) 190 191 #endif 192 193 194 #define lll_trylock(lock) \ 195 atomic_compare_and_exchange_val_acq(&(lock), 1, 0) 196 197 #define lll_cond_trylock(lock) \ 198 atomic_compare_and_exchange_val_acq(&(lock), 2, 0) 199 200 #define __lll_robust_trylock(futex, id) \ 201 (atomic_compare_and_exchange_val_acq (futex, id, 0) != 0) 202 #define lll_robust_trylock(lock, id) \ 203 __lll_robust_trylock (&(lock), id) 204 205 extern void __lll_lock_wait_private (int *futex) attribute_hidden; 206 extern void __lll_lock_wait (int *futex, int private) attribute_hidden; 207 extern int __lll_robust_lock_wait (int *futex, int private) attribute_hidden; 208 209 #define __lll_lock(futex, private) \ 210 ((void) ({ \ 211 int *__futex = (futex); \ 212 if (__builtin_expect (atomic_compare_and_exchange_val_acq (__futex, \ 213 1, 0), 0)) \ 214 { \ 215 if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ 216 __lll_lock_wait_private (__futex); \ 217 else \ 218 __lll_lock_wait (__futex, private); \ 219 } \ 220 })) 221 #define lll_lock(futex, private) __lll_lock (&(futex), private) 222 223 224 #define __lll_robust_lock(futex, id, private) \ 225 ({ \ 226 int *__futex = (futex); \ 227 int __val = 0; \ 228 \ 229 if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, id, \ 230 0), 0)) \ 231 __val = __lll_robust_lock_wait (__futex, private); \ 232 __val; \ 233 }) 234 #define lll_robust_lock(futex, id, private) \ 235 __lll_robust_lock (&(futex), id, private) 236 237 238 #define __lll_cond_lock(futex, private) \ 239 ((void) ({ \ 240 int *__futex = (futex); \ 241 if (__builtin_expect (atomic_exchange_acq (__futex, 2), 0)) \ 242 __lll_lock_wait (__futex, private); \ 243 })) 244 #define lll_cond_lock(futex, private) __lll_cond_lock (&(futex), private) 245 246 247 #define lll_robust_cond_lock(futex, id, private) \ 248 __lll_robust_lock (&(futex), (id) | FUTEX_WAITERS, private) 249 250 251 extern int __lll_timedlock_wait (int *futex, const struct timespec *, 252 int private) attribute_hidden; 253 extern int __lll_robust_timedlock_wait (int *futex, const struct timespec *, 254 int private) attribute_hidden; 255 256 #define __lll_timedlock(futex, abstime, private) \ 257 ({ \ 258 int *__futex = (futex); \ 259 int __val = 0; \ 260 \ 261 if (__builtin_expect (atomic_exchange_acq (__futex, 1), 0)) \ 262 __val = __lll_timedlock_wait (__futex, abstime, private); \ 263 __val; \ 264 }) 265 #define lll_timedlock(futex, abstime, private) \ 266 __lll_timedlock (&(futex), abstime, private) 267 268 269 #define __lll_robust_timedlock(futex, abstime, id, private) \ 270 ({ \ 271 int *__futex = (futex); \ 272 int __val = 0; \ 273 \ 274 if (__builtin_expect (atomic_compare_and_exchange_bool_acq (__futex, id, \ 275 0), 0)) \ 276 __val = __lll_robust_timedlock_wait (__futex, abstime, private); \ 277 __val; \ 278 }) 279 #define lll_robust_timedlock(futex, abstime, id, private) \ 280 __lll_robust_timedlock (&(futex), abstime, id, private) 281 282 283 #define __lll_unlock(futex, private) \ 284 (void) \ 285 ({ int *__futex = (futex); \ 286 int __oldval = atomic_exchange_rel (__futex, 0); \ 287 if (__builtin_expect (__oldval > 1, 0)) \ 288 lll_futex_wake (__futex, 1, private); \ 289 }) 290 #define lll_unlock(futex, private) __lll_unlock(&(futex), private) 291 292 293 #define __lll_robust_unlock(futex, private) \ 294 (void) \ 295 ({ int *__futex = (futex); \ 296 int __oldval = atomic_exchange_rel (__futex, 0); \ 297 if (__builtin_expect (__oldval & FUTEX_WAITERS, 0)) \ 298 lll_futex_wake (__futex, 1, private); \ 299 }) 300 #define lll_robust_unlock(futex, private) \ 301 __lll_robust_unlock(&(futex), private) 302 303 304 #define lll_islocked(futex) \ 305 (futex != 0) 306 307 308 /* Our internal lock implementation is identical to the binary-compatible 309 mutex implementation. */ 310 311 /* Initializers for lock. */ 312 #define LLL_LOCK_INITIALIZER (0) 313 #define LLL_LOCK_INITIALIZER_LOCKED (1) 314 315 /* The states of a lock are: 316 0 - untaken 317 1 - taken by one user 318 >1 - taken by more users */ 319 320 /* The kernel notifies a process which uses CLONE_CLEARTID via futex 321 wakeup when the clone terminates. The memory location contains the 322 thread ID while the clone is running and is reset to zero 323 afterwards. */ 324 #define lll_wait_tid(tid) \ 325 do { \ 326 __typeof (tid) __tid; \ 327 while ((__tid = (tid)) != 0) \ 328 lll_futex_wait (&(tid), __tid, LLL_SHARED);\ 329 } while (0) 330 331 extern int __lll_timedwait_tid (int *, const struct timespec *) 332 attribute_hidden; 333 334 #define lll_timedwait_tid(tid, abstime) \ 335 ({ \ 336 int __res = 0; \ 337 if ((tid) != 0) \ 338 __res = __lll_timedwait_tid (&(tid), (abstime)); \ 339 __res; \ 340 }) 341 342 #endif /* __ASSEMBLER__ */ 343 344 #endif /* lowlevellock.h */ 345