1 /* Copyright (C) 2002-2004, 2006-2008, 2009 Free Software Foundation, Inc. 2 This file is part of the GNU C Library. 3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. 4 5 The GNU C Library is free software; you can redistribute it and/or 6 modify it under the terms of the GNU Lesser General Public 7 License as published by the Free Software Foundation; either 8 version 2.1 of the License, or (at your option) any later version. 9 10 The GNU C Library is distributed in the hope that it will be useful, 11 but WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 Lesser General Public License for more details. 14 15 You should have received a copy of the GNU Lesser General Public 16 License along with the GNU C Library; if not, see 17 <http://www.gnu.org/licenses/>. */ 18 19 #ifndef _LOWLEVELLOCK_H 20 #define _LOWLEVELLOCK_H 1 21 22 #ifndef __ASSEMBLER__ 23 # include <time.h> 24 # include <sys/param.h> 25 # include <bits/pthreadtypes.h> 26 # include <bits/kernel-features.h> 27 # include <tcb-offsets.h> 28 29 # ifndef LOCK_INSTR 30 # ifdef UP 31 # define LOCK_INSTR /* nothing */ 32 # else 33 # define LOCK_INSTR "lock;" 34 # endif 35 # endif 36 #else 37 # ifndef LOCK 38 # ifdef UP 39 # define LOCK 40 # else 41 # define LOCK lock 42 # endif 43 # endif 44 #endif 45 46 #define FUTEX_WAIT 0 47 #define FUTEX_WAKE 1 48 #define FUTEX_CMP_REQUEUE 4 49 #define FUTEX_WAKE_OP 5 50 #define FUTEX_LOCK_PI 6 51 #define FUTEX_UNLOCK_PI 7 52 #define FUTEX_TRYLOCK_PI 8 53 #define FUTEX_WAIT_BITSET 9 54 #define FUTEX_WAKE_BITSET 10 55 #define FUTEX_WAIT_REQUEUE_PI 11 56 #define FUTEX_CMP_REQUEUE_PI 12 57 #define FUTEX_PRIVATE_FLAG 128 58 #define FUTEX_CLOCK_REALTIME 256 59 60 #define FUTEX_BITSET_MATCH_ANY 0xffffffff 61 62 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) 63 64 /* Values for 'private' parameter of locking macros. Yes, the 65 definition seems to be backwards. But it is not. The bit will be 66 reversed before passing to the system call. */ 67 #define LLL_PRIVATE 0 68 #define LLL_SHARED FUTEX_PRIVATE_FLAG 69 70 #ifndef __ASSEMBLER__ 71 72 #if !defined NOT_IN_libc || defined IS_IN_rtld 73 /* In libc.so or ld.so all futexes are private. */ 74 # ifdef __ASSUME_PRIVATE_FUTEX 75 # define __lll_private_flag(fl, private) \ 76 ((fl) | FUTEX_PRIVATE_FLAG) 77 # else 78 # define __lll_private_flag(fl, private) \ 79 ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) 80 # endif 81 #else 82 # ifdef __ASSUME_PRIVATE_FUTEX 83 # define __lll_private_flag(fl, private) \ 84 (((fl) | FUTEX_PRIVATE_FLAG) ^ (private)) 85 # else 86 # define __lll_private_flag(fl, private) \ 87 (__builtin_constant_p (private) \ 88 ? ((private) == 0 \ 89 ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) \ 90 : (fl)) \ 91 : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG); \ 92 __asm__ ("andl %%fs:%P1, %0" : "+r" (__fl) \ 93 : "i" (offsetof (struct pthread, header.private_futex))); \ 94 __fl | (fl); })) 95 # endif 96 #endif 97 98 /* Initializer for lock. */ 99 #define LLL_LOCK_INITIALIZER (0) 100 #define LLL_LOCK_INITIALIZER_LOCKED (1) 101 #define LLL_LOCK_INITIALIZER_WAITERS (2) 102 103 /* Delay in spinlock loop. */ 104 #define BUSY_WAIT_NOP __asm__ ("rep; nop") 105 106 107 #define LLL_STUB_UNWIND_INFO_START \ 108 ".section .eh_frame,\"a\",@progbits\n" \ 109 "7:\t" ".long 9f-8f # Length of Common Information Entry\n" \ 110 "8:\t" ".long 0x0 # CIE Identifier Tag\n\t" \ 111 ".byte 0x1 # CIE Version\n\t" \ 112 ".ascii \"zR\\0\" # CIE Augmentation\n\t" \ 113 ".uleb128 0x1 # CIE Code Alignment Factor\n\t" \ 114 ".sleb128 -8 # CIE Data Alignment Factor\n\t" \ 115 ".byte 0x10 # CIE RA Column\n\t" \ 116 ".uleb128 0x1 # Augmentation size\n\t" \ 117 ".byte 0x1b # FDE Encoding (pcrel sdata4)\n\t" \ 118 ".byte 0x12 # DW_CFA_def_cfa_sf\n\t" \ 119 ".uleb128 0x7\n\t" \ 120 ".sleb128 16\n\t" \ 121 ".align 8\n" \ 122 "9:\t" ".long 23f-10f # FDE Length\n" \ 123 "10:\t" ".long 10b-7b # FDE CIE offset\n\t" \ 124 ".long 1b-. # FDE initial location\n\t" \ 125 ".long 6b-1b # FDE address range\n\t" \ 126 ".uleb128 0x0 # Augmentation size\n\t" \ 127 ".byte 0x16 # DW_CFA_val_expression\n\t" \ 128 ".uleb128 0x10\n\t" \ 129 ".uleb128 12f-11f\n" \ 130 "11:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ 131 ".sleb128 4b-1b\n" 132 #define LLL_STUB_UNWIND_INFO_END \ 133 ".byte 0x16 # DW_CFA_val_expression\n\t" \ 134 ".uleb128 0x10\n\t" \ 135 ".uleb128 14f-13f\n" \ 136 "13:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ 137 ".sleb128 4b-2b\n" \ 138 "14:\t" ".byte 0x40 + (3b-2b) # DW_CFA_advance_loc\n\t" \ 139 ".byte 0x0e # DW_CFA_def_cfa_offset\n\t" \ 140 ".uleb128 0\n\t" \ 141 ".byte 0x16 # DW_CFA_val_expression\n\t" \ 142 ".uleb128 0x10\n\t" \ 143 ".uleb128 16f-15f\n" \ 144 "15:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ 145 ".sleb128 4b-3b\n" \ 146 "16:\t" ".byte 0x40 + (4b-3b-1) # DW_CFA_advance_loc\n\t" \ 147 ".byte 0x0e # DW_CFA_def_cfa_offset\n\t" \ 148 ".uleb128 128\n\t" \ 149 ".byte 0x16 # DW_CFA_val_expression\n\t" \ 150 ".uleb128 0x10\n\t" \ 151 ".uleb128 20f-17f\n" \ 152 "17:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ 153 ".sleb128 19f-18f\n\t" \ 154 ".byte 0x0d # DW_OP_const4s\n" \ 155 "18:\t" ".4byte 4b-.\n\t" \ 156 ".byte 0x1c # DW_OP_minus\n\t" \ 157 ".byte 0x0d # DW_OP_const4s\n" \ 158 "19:\t" ".4byte 24f-.\n\t" \ 159 ".byte 0x22 # DW_OP_plus\n" \ 160 "20:\t" ".byte 0x40 + (5b-4b+1) # DW_CFA_advance_loc\n\t" \ 161 ".byte 0x13 # DW_CFA_def_cfa_offset_sf\n\t" \ 162 ".sleb128 16\n\t" \ 163 ".byte 0x16 # DW_CFA_val_expression\n\t" \ 164 ".uleb128 0x10\n\t" \ 165 ".uleb128 22f-21f\n" \ 166 "21:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ 167 ".sleb128 4b-5b\n" \ 168 "22:\t" ".align 8\n" \ 169 "23:\t" ".previous\n" 170 171 /* Unwind info for 172 1: leaq ..., %rdi 173 2: subq $128, %rsp 174 3: callq ... 175 4: addq $128, %rsp 176 5: jmp 24f 177 6: 178 snippet. */ 179 #define LLL_STUB_UNWIND_INFO_5 \ 180 LLL_STUB_UNWIND_INFO_START \ 181 "12:\t" ".byte 0x40 + (2b-1b) # DW_CFA_advance_loc\n\t" \ 182 LLL_STUB_UNWIND_INFO_END 183 184 /* Unwind info for 185 1: leaq ..., %rdi 186 0: movq ..., %rdx 187 2: subq $128, %rsp 188 3: callq ... 189 4: addq $128, %rsp 190 5: jmp 24f 191 6: 192 snippet. */ 193 #define LLL_STUB_UNWIND_INFO_6 \ 194 LLL_STUB_UNWIND_INFO_START \ 195 "12:\t" ".byte 0x40 + (0b-1b) # DW_CFA_advance_loc\n\t" \ 196 ".byte 0x16 # DW_CFA_val_expression\n\t" \ 197 ".uleb128 0x10\n\t" \ 198 ".uleb128 26f-25f\n" \ 199 "25:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ 200 ".sleb128 4b-0b\n" \ 201 "26:\t" ".byte 0x40 + (2b-0b) # DW_CFA_advance_loc\n\t" \ 202 LLL_STUB_UNWIND_INFO_END 203 204 205 #define lll_futex_wait(futex, val, private) \ 206 lll_futex_timed_wait(futex, val, NULL, private) 207 208 209 #define lll_futex_timed_wait(futex, val, timeout, private) \ 210 ({ \ 211 register const struct timespec *__to __asm__ ("r10") = timeout; \ 212 int __status; \ 213 register __typeof (val) _val __asm__ ("edx") = (val); \ 214 __asm__ __volatile__ ("syscall" \ 215 : "=a" (__status) \ 216 : "0" (SYS_futex), "D" (futex), \ 217 "S" (__lll_private_flag (FUTEX_WAIT, private)), \ 218 "d" (_val), "r" (__to) \ 219 : "memory", "cc", "r11", "cx"); \ 220 __status; \ 221 }) 222 223 224 #define lll_futex_wake(futex, nr, private) \ 225 do { \ 226 int __ignore; \ 227 register __typeof (nr) _nr __asm__ ("edx") = (nr); \ 228 __asm__ __volatile__ ("syscall" \ 229 : "=a" (__ignore) \ 230 : "0" (SYS_futex), "D" (futex), \ 231 "S" (__lll_private_flag (FUTEX_WAKE, private)), \ 232 "d" (_nr) \ 233 : "memory", "cc", "r10", "r11", "cx"); \ 234 } while (0) 235 236 237 /* NB: in the lll_trylock macro we simply return the value in %eax 238 after the cmpxchg instruction. In case the operation succeded this 239 value is zero. In case the operation failed, the cmpxchg instruction 240 has loaded the current value of the memory work which is guaranteed 241 to be nonzero. */ 242 #if defined NOT_IN_libc || defined UP 243 # define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1" 244 #else 245 # define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ 246 "je 0f\n\t" \ 247 "lock; cmpxchgl %2, %1\n\t" \ 248 "jmp 1f\n\t" \ 249 "0:\tcmpxchgl %2, %1\n\t" \ 250 "1:" 251 #endif 252 253 #define lll_trylock(futex) \ 254 ({ int ret; \ 255 __asm__ __volatile__ (__lll_trylock_asm \ 256 : "=a" (ret), "=m" (futex) \ 257 : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \ 258 "0" (LLL_LOCK_INITIALIZER) \ 259 : "memory"); \ 260 ret; }) 261 262 #define lll_robust_trylock(futex, id) \ 263 ({ int ret; \ 264 __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %2, %1" \ 265 : "=a" (ret), "=m" (futex) \ 266 : "r" (id), "m" (futex), "0" (LLL_LOCK_INITIALIZER) \ 267 : "memory"); \ 268 ret; }) 269 270 #define lll_cond_trylock(futex) \ 271 ({ int ret; \ 272 __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %2, %1" \ 273 : "=a" (ret), "=m" (futex) \ 274 : "r" (LLL_LOCK_INITIALIZER_WAITERS), \ 275 "m" (futex), "0" (LLL_LOCK_INITIALIZER) \ 276 : "memory"); \ 277 ret; }) 278 279 #if defined NOT_IN_libc || defined UP 280 # define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t" \ 281 "jnz 1f\n\t" 282 #else 283 # define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ 284 "je 0f\n\t" \ 285 "lock; cmpxchgl %4, %2\n\t" \ 286 "jnz 1f\n\t" \ 287 "jmp 24f\n" \ 288 "0:\tcmpxchgl %4, %2\n\t" \ 289 "jnz 1f\n\t" 290 #endif 291 292 #define lll_lock(futex, private) \ 293 (void) \ 294 ({ int ignore1, ignore2, ignore3; \ 295 if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ 296 __asm__ __volatile__ (__lll_lock_asm_start \ 297 ".subsection 1\n\t" \ 298 ".type _L_lock_%=, @function\n" \ 299 "_L_lock_%=:\n" \ 300 "1:\tleaq %2, %%rdi\n" \ 301 "2:\tsubq $128, %%rsp\n" \ 302 "3:\tcallq __lll_lock_wait_private\n" \ 303 "4:\taddq $128, %%rsp\n" \ 304 "5:\tjmp 24f\n" \ 305 "6:\t.size _L_lock_%=, 6b-1b\n\t" \ 306 ".previous\n" \ 307 LLL_STUB_UNWIND_INFO_5 \ 308 "24:" \ 309 : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \ 310 "=a" (ignore3) \ 311 : "0" (1), "m" (futex), "3" (0) \ 312 : "cx", "r11", "cc", "memory"); \ 313 else \ 314 __asm__ __volatile__ (__lll_lock_asm_start \ 315 ".subsection 1\n\t" \ 316 ".type _L_lock_%=, @function\n" \ 317 "_L_lock_%=:\n" \ 318 "1:\tleaq %2, %%rdi\n" \ 319 "2:\tsubq $128, %%rsp\n" \ 320 "3:\tcallq __lll_lock_wait\n" \ 321 "4:\taddq $128, %%rsp\n" \ 322 "5:\tjmp 24f\n" \ 323 "6:\t.size _L_lock_%=, 6b-1b\n\t" \ 324 ".previous\n" \ 325 LLL_STUB_UNWIND_INFO_5 \ 326 "24:" \ 327 : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ 328 "=a" (ignore3) \ 329 : "1" (1), "m" (futex), "3" (0), "0" (private) \ 330 : "cx", "r11", "cc", "memory"); \ 331 }) \ 332 333 #define lll_robust_lock(futex, id, private) \ 334 ({ int __ret, ignore1, ignore2; \ 335 __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %4, %2\n\t" \ 336 "jnz 1f\n\t" \ 337 ".subsection 1\n\t" \ 338 ".type _L_robust_lock_%=, @function\n" \ 339 "_L_robust_lock_%=:\n" \ 340 "1:\tleaq %2, %%rdi\n" \ 341 "2:\tsubq $128, %%rsp\n" \ 342 "3:\tcallq __lll_robust_lock_wait\n" \ 343 "4:\taddq $128, %%rsp\n" \ 344 "5:\tjmp 24f\n" \ 345 "6:\t.size _L_robust_lock_%=, 6b-1b\n\t" \ 346 ".previous\n" \ 347 LLL_STUB_UNWIND_INFO_5 \ 348 "24:" \ 349 : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ 350 "=a" (__ret) \ 351 : "1" (id), "m" (futex), "3" (0), "0" (private) \ 352 : "cx", "r11", "cc", "memory"); \ 353 __ret; }) 354 355 #define lll_cond_lock(futex, private) \ 356 (void) \ 357 ({ int ignore1, ignore2, ignore3; \ 358 __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %4, %2\n\t" \ 359 "jnz 1f\n\t" \ 360 ".subsection 1\n\t" \ 361 ".type _L_cond_lock_%=, @function\n" \ 362 "_L_cond_lock_%=:\n" \ 363 "1:\tleaq %2, %%rdi\n" \ 364 "2:\tsubq $128, %%rsp\n" \ 365 "3:\tcallq __lll_lock_wait\n" \ 366 "4:\taddq $128, %%rsp\n" \ 367 "5:\tjmp 24f\n" \ 368 "6:\t.size _L_cond_lock_%=, 6b-1b\n\t" \ 369 ".previous\n" \ 370 LLL_STUB_UNWIND_INFO_5 \ 371 "24:" \ 372 : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ 373 "=a" (ignore3) \ 374 : "1" (2), "m" (futex), "3" (0), "0" (private) \ 375 : "cx", "r11", "cc", "memory"); \ 376 }) 377 378 #define lll_robust_cond_lock(futex, id, private) \ 379 ({ int __ret, ignore1, ignore2; \ 380 __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %4, %2\n\t" \ 381 "jnz 1f\n\t" \ 382 ".subsection 1\n\t" \ 383 ".type _L_robust_cond_lock_%=, @function\n" \ 384 "_L_robust_cond_lock_%=:\n" \ 385 "1:\tleaq %2, %%rdi\n" \ 386 "2:\tsubq $128, %%rsp\n" \ 387 "3:\tcallq __lll_robust_lock_wait\n" \ 388 "4:\taddq $128, %%rsp\n" \ 389 "5:\tjmp 24f\n" \ 390 "6:\t.size _L_robust_cond_lock_%=, 6b-1b\n\t" \ 391 ".previous\n" \ 392 LLL_STUB_UNWIND_INFO_5 \ 393 "24:" \ 394 : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ 395 "=a" (__ret) \ 396 : "1" (id | FUTEX_WAITERS), "m" (futex), "3" (0), \ 397 "0" (private) \ 398 : "cx", "r11", "cc", "memory"); \ 399 __ret; }) 400 401 #define lll_timedlock(futex, timeout, private) \ 402 ({ int __ret, ignore1, ignore2, ignore3; \ 403 __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %1, %4\n\t" \ 404 "jnz 1f\n\t" \ 405 ".subsection 1\n\t" \ 406 ".type _L_timedlock_%=, @function\n" \ 407 "_L_timedlock_%=:\n" \ 408 "1:\tleaq %4, %%rdi\n" \ 409 "0:\tmovq %8, %%rdx\n" \ 410 "2:\tsubq $128, %%rsp\n" \ 411 "3:\tcallq __lll_timedlock_wait\n" \ 412 "4:\taddq $128, %%rsp\n" \ 413 "5:\tjmp 24f\n" \ 414 "6:\t.size _L_timedlock_%=, 6b-1b\n\t" \ 415 ".previous\n" \ 416 LLL_STUB_UNWIND_INFO_6 \ 417 "24:" \ 418 : "=a" (__ret), "=D" (ignore1), "=S" (ignore2), \ 419 "=&d" (ignore3), "=m" (futex) \ 420 : "0" (0), "1" (1), "m" (futex), "m" (timeout), \ 421 "2" (private) \ 422 : "memory", "cx", "cc", "r10", "r11"); \ 423 __ret; }) 424 425 #define lll_robust_timedlock(futex, timeout, id, private) \ 426 ({ int __ret, ignore1, ignore2, ignore3; \ 427 __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %1, %4\n\t" \ 428 "jnz 1f\n\t" \ 429 ".subsection 1\n\t" \ 430 ".type _L_robust_timedlock_%=, @function\n" \ 431 "_L_robust_timedlock_%=:\n" \ 432 "1:\tleaq %4, %%rdi\n" \ 433 "0:\tmovq %8, %%rdx\n" \ 434 "2:\tsubq $128, %%rsp\n" \ 435 "3:\tcallq __lll_robust_timedlock_wait\n" \ 436 "4:\taddq $128, %%rsp\n" \ 437 "5:\tjmp 24f\n" \ 438 "6:\t.size _L_robust_timedlock_%=, 6b-1b\n\t" \ 439 ".previous\n" \ 440 LLL_STUB_UNWIND_INFO_6 \ 441 "24:" \ 442 : "=a" (__ret), "=D" (ignore1), "=S" (ignore2), \ 443 "=&d" (ignore3), "=m" (futex) \ 444 : "0" (0), "1" (id), "m" (futex), "m" (timeout), \ 445 "2" (private) \ 446 : "memory", "cx", "cc", "r10", "r11"); \ 447 __ret; }) 448 449 #if defined NOT_IN_libc || defined UP 450 # define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t" \ 451 "jne 1f\n\t" 452 #else 453 # define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ 454 "je 0f\n\t" \ 455 "lock; decl %0\n\t" \ 456 "jne 1f\n\t" \ 457 "jmp 24f\n\t" \ 458 "0:\tdecl %0\n\t" \ 459 "jne 1f\n\t" 460 #endif 461 462 #define lll_unlock(futex, private) \ 463 (void) \ 464 ({ int ignore; \ 465 if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ 466 __asm__ __volatile__ (__lll_unlock_asm_start \ 467 ".subsection 1\n\t" \ 468 ".type _L_unlock_%=, @function\n" \ 469 "_L_unlock_%=:\n" \ 470 "1:\tleaq %0, %%rdi\n" \ 471 "2:\tsubq $128, %%rsp\n" \ 472 "3:\tcallq __lll_unlock_wake_private\n" \ 473 "4:\taddq $128, %%rsp\n" \ 474 "5:\tjmp 24f\n" \ 475 "6:\t.size _L_unlock_%=, 6b-1b\n\t" \ 476 ".previous\n" \ 477 LLL_STUB_UNWIND_INFO_5 \ 478 "24:" \ 479 : "=m" (futex), "=&D" (ignore) \ 480 : "m" (futex) \ 481 : "ax", "cx", "r11", "cc", "memory"); \ 482 else \ 483 __asm__ __volatile__ (__lll_unlock_asm_start \ 484 ".subsection 1\n\t" \ 485 ".type _L_unlock_%=, @function\n" \ 486 "_L_unlock_%=:\n" \ 487 "1:\tleaq %0, %%rdi\n" \ 488 "2:\tsubq $128, %%rsp\n" \ 489 "3:\tcallq __lll_unlock_wake\n" \ 490 "4:\taddq $128, %%rsp\n" \ 491 "5:\tjmp 24f\n" \ 492 "6:\t.size _L_unlock_%=, 6b-1b\n\t" \ 493 ".previous\n" \ 494 LLL_STUB_UNWIND_INFO_5 \ 495 "24:" \ 496 : "=m" (futex), "=&D" (ignore) \ 497 : "m" (futex), "S" (private) \ 498 : "ax", "cx", "r11", "cc", "memory"); \ 499 }) 500 501 #define lll_robust_unlock(futex, private) \ 502 do \ 503 { \ 504 int ignore; \ 505 __asm__ __volatile__ (LOCK_INSTR "andl %2, %0\n\t" \ 506 "jne 1f\n\t" \ 507 ".subsection 1\n\t" \ 508 ".type _L_robust_unlock_%=, @function\n" \ 509 "_L_robust_unlock_%=:\n" \ 510 "1:\tleaq %0, %%rdi\n" \ 511 "2:\tsubq $128, %%rsp\n" \ 512 "3:\tcallq __lll_unlock_wake\n" \ 513 "4:\taddq $128, %%rsp\n" \ 514 "5:\tjmp 24f\n" \ 515 "6:\t.size _L_robust_unlock_%=, 6b-1b\n\t" \ 516 ".previous\n" \ 517 LLL_STUB_UNWIND_INFO_5 \ 518 "24:" \ 519 : "=m" (futex), "=&D" (ignore) \ 520 : "i" (FUTEX_WAITERS), "m" (futex), \ 521 "S" (private) \ 522 : "ax", "cx", "r11", "cc", "memory"); \ 523 } \ 524 while (0) 525 526 #define lll_robust_dead(futex, private) \ 527 do \ 528 { \ 529 int ignore; \ 530 __asm__ __volatile__ (LOCK_INSTR "orl %3, (%2)\n\t" \ 531 "syscall" \ 532 : "=m" (futex), "=a" (ignore) \ 533 : "D" (&(futex)), "i" (FUTEX_OWNER_DIED), \ 534 "S" (__lll_private_flag (FUTEX_WAKE, private)), \ 535 "1" (__NR_futex), "d" (1) \ 536 : "cx", "r11", "cc", "memory"); \ 537 } \ 538 while (0) 539 540 /* Returns non-zero if error happened, zero if success. */ 541 #define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \ 542 ({ int __res; \ 543 register int __nr_move __asm__ ("r10") = nr_move; \ 544 register void *__mutex __asm__ ("r8") = mutex; \ 545 register int __val __asm__ ("r9") = val; \ 546 __asm__ __volatile__ ("syscall" \ 547 : "=a" (__res) \ 548 : "0" (__NR_futex), "D" ((void *) ftx), \ 549 "S" (__lll_private_flag (FUTEX_CMP_REQUEUE, \ 550 private)), "d" (nr_wake), \ 551 "r" (__nr_move), "r" (__mutex), "r" (__val) \ 552 : "cx", "r11", "cc", "memory"); \ 553 __res < 0; }) 554 555 #define lll_islocked(futex) \ 556 (futex != LLL_LOCK_INITIALIZER) 557 558 559 /* The kernel notifies a process with uses CLONE_CLEARTID via futex 560 wakeup when the clone terminates. The memory location contains the 561 thread ID while the clone is running and is reset to zero 562 afterwards. 563 564 The macro parameter must not have any side effect. */ 565 #define lll_wait_tid(tid) \ 566 do { \ 567 int __ignore; \ 568 register __typeof (tid) _tid __asm__ ("edx") = (tid); \ 569 if (_tid != 0) \ 570 __asm__ __volatile__ ("xorq %%r10, %%r10\n\t" \ 571 "1:\tmovq %2, %%rax\n\t" \ 572 "syscall\n\t" \ 573 "cmpl $0, (%%rdi)\n\t" \ 574 "jne 1b" \ 575 : "=&a" (__ignore) \ 576 : "S" (FUTEX_WAIT), "i" (SYS_futex), "D" (&tid), \ 577 "d" (_tid) \ 578 : "memory", "cc", "r10", "r11", "cx"); \ 579 } while (0) 580 581 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime) 582 attribute_hidden; 583 #define lll_timedwait_tid(tid, abstime) \ 584 ({ \ 585 int __ret = 0; \ 586 if (tid != 0) \ 587 { \ 588 if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) \ 589 __ret = EINVAL; \ 590 else \ 591 __ret = __lll_timedwait_tid (&tid, abstime); \ 592 } \ 593 __ret; }) 594 595 #endif /* !__ASSEMBLER__ */ 596 597 #endif /* lowlevellock.h */ 598