1 /* Internal macros for atomic operations for GNU C Library. 2 Copyright (C) 2002-2015 Free Software Foundation, Inc. 3 This file is part of the GNU C Library. 4 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. 5 6 The GNU C Library is free software; you can redistribute it and/or 7 modify it under the terms of the GNU Lesser General Public 8 License as published by the Free Software Foundation; either 9 version 2.1 of the License, or (at your option) any later version. 10 11 The GNU C Library is distributed in the hope that it will be useful, 12 but WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 Lesser General Public License for more details. 15 16 You should have received a copy of the GNU Lesser General Public 17 License along with the GNU C Library; if not, see 18 <http://www.gnu.org/licenses/>. */ 19 20 #ifndef _ATOMIC_H 21 #define _ATOMIC_H 1 22 23 /* This header defines three types of macros: 24 25 - atomic arithmetic and logic operation on memory. They all 26 have the prefix "atomic_". 27 28 - conditionally atomic operations of the same kinds. These 29 always behave identical but can be faster when atomicity 30 is not really needed since only one thread has access to 31 the memory location. In that case the code is slower in 32 the multi-thread case. The interfaces have the prefix 33 "catomic_". 34 35 - support functions like barriers. They also have the prefix 36 "atomic_". 37 38 Architectures must provide a few lowlevel macros (the compare 39 and exchange definitions). All others are optional. They 40 should only be provided if the architecture has specific 41 support for the operation. 42 43 As <atomic.h> macros are usually heavily nested and often use local 44 variables to make sure side-effects are evaluated properly, use for 45 macro local variables a per-macro unique prefix. This file uses 46 __atgN_ prefix where N is different in each macro. */ 47 48 #include <stdlib.h> 49 50 #include <bits/atomic.h> 51 52 /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the 53 bit width of *MEM. The calling macro puts parens around MEM 54 and following args. */ 55 #define __atomic_val_bysize(pre, post, mem, ...) \ 56 ({ \ 57 __typeof ((__typeof (*(mem))) *(mem)) __atg1_result; \ 58 if (sizeof (*mem) == 1) \ 59 __atg1_result = (__typeof ((__typeof (*(mem))) *(mem))) pre##_8_##post (mem, __VA_ARGS__); \ 60 else if (sizeof (*mem) == 2) \ 61 __atg1_result = (__typeof ((__typeof (*(mem))) *(mem))) pre##_16_##post (mem, __VA_ARGS__); \ 62 else if (sizeof (*mem) == 4) \ 63 __atg1_result = (__typeof ((__typeof (*(mem))) *(mem))) pre##_32_##post (mem, __VA_ARGS__); \ 64 else if (sizeof (*mem) == 8) \ 65 __atg1_result = (__typeof ((__typeof (*(mem))) *(mem))) pre##_64_##post (mem, __VA_ARGS__); \ 66 else \ 67 abort (); \ 68 __atg1_result; \ 69 }) 70 #define __atomic_bool_bysize(pre, post, mem, ...) \ 71 ({ \ 72 int __atg2_result; \ 73 if (sizeof (*mem) == 1) \ 74 __atg2_result = (int) pre##_8_##post (mem, __VA_ARGS__); \ 75 else if (sizeof (*mem) == 2) \ 76 __atg2_result = (int) pre##_16_##post (mem, __VA_ARGS__); \ 77 else if (sizeof (*mem) == 4) \ 78 __atg2_result = (int) pre##_32_##post (mem, __VA_ARGS__); \ 79 else if (sizeof (*mem) == 8) \ 80 __atg2_result = (int) pre##_64_##post (mem, __VA_ARGS__); \ 81 else \ 82 abort (); \ 83 __atg2_result; \ 84 }) 85 86 87 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. 88 Return the old *MEM value. */ 89 #if !defined atomic_compare_and_exchange_val_acq \ 90 && defined __arch_compare_and_exchange_val_32_acq 91 # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \ 92 __atomic_val_bysize (__arch_compare_and_exchange_val,acq, \ 93 mem, newval, oldval) 94 #endif 95 96 97 #ifndef catomic_compare_and_exchange_val_acq 98 # ifdef __arch_c_compare_and_exchange_val_32_acq 99 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \ 100 __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \ 101 mem, newval, oldval) 102 # else 103 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \ 104 atomic_compare_and_exchange_val_acq (mem, newval, oldval) 105 # endif 106 #endif 107 108 109 #ifndef catomic_compare_and_exchange_val_rel 110 # ifndef atomic_compare_and_exchange_val_rel 111 # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \ 112 catomic_compare_and_exchange_val_acq (mem, newval, oldval) 113 # else 114 # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \ 115 atomic_compare_and_exchange_val_rel (mem, newval, oldval) 116 # endif 117 #endif 118 119 120 #ifndef atomic_compare_and_exchange_val_rel 121 # define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \ 122 atomic_compare_and_exchange_val_acq (mem, newval, oldval) 123 #endif 124 125 126 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. 127 Return zero if *MEM was changed or non-zero if no exchange happened. */ 128 #ifndef atomic_compare_and_exchange_bool_acq 129 # ifdef __arch_compare_and_exchange_bool_32_acq 130 # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 131 __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq, \ 132 mem, newval, oldval) 133 # else 134 # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 135 ({ /* Cannot use __oldval here, because macros later in this file might \ 136 call this macro with __oldval argument. */ \ 137 __typeof (oldval) __atg3_old = (oldval); \ 138 atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old) \ 139 != __atg3_old; \ 140 }) 141 # endif 142 #endif 143 144 145 #ifndef catomic_compare_and_exchange_bool_acq 146 # ifdef __arch_c_compare_and_exchange_bool_32_acq 147 # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 148 __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq, \ 149 mem, newval, oldval) 150 # else 151 # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 152 ({ /* Cannot use __oldval here, because macros later in this file might \ 153 call this macro with __oldval argument. */ \ 154 __typeof (oldval) __atg4_old = (oldval); \ 155 catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old) \ 156 != __atg4_old; \ 157 }) 158 # endif 159 #endif 160 161 162 #ifndef catomic_compare_and_exchange_bool_rel 163 # ifndef atomic_compare_and_exchange_bool_rel 164 # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \ 165 catomic_compare_and_exchange_bool_acq (mem, newval, oldval) 166 # else 167 # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \ 168 atomic_compare_and_exchange_bool_rel (mem, newval, oldval) 169 # endif 170 #endif 171 172 173 #ifndef atomic_compare_and_exchange_bool_rel 174 # define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \ 175 atomic_compare_and_exchange_bool_acq (mem, newval, oldval) 176 #endif 177 178 179 /* Store NEWVALUE in *MEM and return the old value. */ 180 #ifndef atomic_exchange_acq 181 # define atomic_exchange_acq(mem, newvalue) \ 182 ({ __typeof (*(mem)) __atg5_oldval; \ 183 __typeof (mem) __atg5_memp = (mem); \ 184 __typeof (*(mem)) __atg5_value = (newvalue); \ 185 \ 186 do \ 187 __atg5_oldval = *__atg5_memp; \ 188 while (__builtin_expect \ 189 (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \ 190 __atg5_oldval), 0)); \ 191 \ 192 __atg5_oldval; }) 193 #endif 194 195 #ifndef atomic_exchange_rel 196 # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue) 197 #endif 198 199 200 /* Add VALUE to *MEM and return the old value of *MEM. */ 201 #ifndef atomic_exchange_and_add_acq 202 # ifdef atomic_exchange_and_add 203 # define atomic_exchange_and_add_acq(mem, value) \ 204 atomic_exchange_and_add (mem, value) 205 # else 206 # define atomic_exchange_and_add_acq(mem, value) \ 207 ({ __typeof (*(mem)) __atg6_oldval; \ 208 __typeof (mem) __atg6_memp = (mem); \ 209 __typeof (*(mem)) __atg6_value = (value); \ 210 \ 211 do \ 212 __atg6_oldval = *__atg6_memp; \ 213 while (__builtin_expect \ 214 (atomic_compare_and_exchange_bool_acq (__atg6_memp, \ 215 __atg6_oldval \ 216 + __atg6_value, \ 217 __atg6_oldval), 0)); \ 218 \ 219 __atg6_oldval; }) 220 # endif 221 #endif 222 223 #ifndef atomic_exchange_and_add_rel 224 # define atomic_exchange_and_add_rel(mem, value) \ 225 atomic_exchange_and_add_acq(mem, value) 226 #endif 227 228 #ifndef atomic_exchange_and_add 229 # define atomic_exchange_and_add(mem, value) \ 230 atomic_exchange_and_add_acq(mem, value) 231 #endif 232 233 #ifndef catomic_exchange_and_add 234 # define catomic_exchange_and_add(mem, value) \ 235 ({ __typeof (*(mem)) __atg7_oldv; \ 236 __typeof (mem) __atg7_memp = (mem); \ 237 __typeof (*(mem)) __atg7_value = (value); \ 238 \ 239 do \ 240 __atg7_oldv = *__atg7_memp; \ 241 while (__builtin_expect \ 242 (catomic_compare_and_exchange_bool_acq (__atg7_memp, \ 243 __atg7_oldv \ 244 + __atg7_value, \ 245 __atg7_oldv), 0)); \ 246 \ 247 __atg7_oldv; }) 248 #endif 249 250 251 #ifndef atomic_max 252 # define atomic_max(mem, value) \ 253 do { \ 254 __typeof (*(mem)) __atg8_oldval; \ 255 __typeof (mem) __atg8_memp = (mem); \ 256 __typeof (*(mem)) __atg8_value = (value); \ 257 do { \ 258 __atg8_oldval = *__atg8_memp; \ 259 if (__atg8_oldval >= __atg8_value) \ 260 break; \ 261 } while (__builtin_expect \ 262 (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\ 263 __atg8_oldval), 0)); \ 264 } while (0) 265 #endif 266 267 268 #ifndef catomic_max 269 # define catomic_max(mem, value) \ 270 do { \ 271 __typeof (*(mem)) __atg9_oldv; \ 272 __typeof (mem) __atg9_memp = (mem); \ 273 __typeof (*(mem)) __atg9_value = (value); \ 274 do { \ 275 __atg9_oldv = *__atg9_memp; \ 276 if (__atg9_oldv >= __atg9_value) \ 277 break; \ 278 } while (__builtin_expect \ 279 (catomic_compare_and_exchange_bool_acq (__atg9_memp, \ 280 __atg9_value, \ 281 __atg9_oldv), 0)); \ 282 } while (0) 283 #endif 284 285 286 #ifndef atomic_min 287 # define atomic_min(mem, value) \ 288 do { \ 289 __typeof (*(mem)) __atg10_oldval; \ 290 __typeof (mem) __atg10_memp = (mem); \ 291 __typeof (*(mem)) __atg10_value = (value); \ 292 do { \ 293 __atg10_oldval = *__atg10_memp; \ 294 if (__atg10_oldval <= __atg10_value) \ 295 break; \ 296 } while (__builtin_expect \ 297 (atomic_compare_and_exchange_bool_acq (__atg10_memp, \ 298 __atg10_value, \ 299 __atg10_oldval), 0)); \ 300 } while (0) 301 #endif 302 303 304 #ifndef atomic_add 305 # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value)) 306 #endif 307 308 309 #ifndef catomic_add 310 # define catomic_add(mem, value) \ 311 (void) catomic_exchange_and_add ((mem), (value)) 312 #endif 313 314 315 #ifndef atomic_increment 316 # define atomic_increment(mem) atomic_add ((mem), 1) 317 #endif 318 319 320 #ifndef catomic_increment 321 # define catomic_increment(mem) catomic_add ((mem), 1) 322 #endif 323 324 325 #ifndef atomic_increment_val 326 # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1) 327 #endif 328 329 330 #ifndef catomic_increment_val 331 # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1) 332 #endif 333 334 335 /* Add one to *MEM and return true iff it's now zero. */ 336 #ifndef atomic_increment_and_test 337 # define atomic_increment_and_test(mem) \ 338 (atomic_exchange_and_add ((mem), 1) + 1 == 0) 339 #endif 340 341 342 #ifndef atomic_decrement 343 # define atomic_decrement(mem) atomic_add ((mem), -1) 344 #endif 345 346 347 #ifndef catomic_decrement 348 # define catomic_decrement(mem) catomic_add ((mem), -1) 349 #endif 350 351 352 #ifndef atomic_decrement_val 353 # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1) 354 #endif 355 356 357 #ifndef catomic_decrement_val 358 # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1) 359 #endif 360 361 362 /* Subtract 1 from *MEM and return true iff it's now zero. */ 363 #ifndef atomic_decrement_and_test 364 # define atomic_decrement_and_test(mem) \ 365 (atomic_exchange_and_add ((mem), -1) == 1) 366 #endif 367 368 369 /* Decrement *MEM if it is > 0, and return the old value. */ 370 #ifndef atomic_decrement_if_positive 371 # define atomic_decrement_if_positive(mem) \ 372 ({ __typeof (*(mem)) __atg11_oldval; \ 373 __typeof (mem) __atg11_memp = (mem); \ 374 \ 375 do \ 376 { \ 377 __atg11_oldval = *__atg11_memp; \ 378 if (__builtin_expect (__atg11_oldval <= 0, 0)) \ 379 break; \ 380 } \ 381 while (__builtin_expect \ 382 (atomic_compare_and_exchange_bool_acq (__atg11_memp, \ 383 __atg11_oldval - 1, \ 384 __atg11_oldval), 0)); \ 385 __atg11_oldval; }) 386 #endif 387 388 389 #ifndef atomic_add_negative 390 # define atomic_add_negative(mem, value) \ 391 ({ __typeof (value) __atg12_value = (value); \ 392 atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; }) 393 #endif 394 395 396 #ifndef atomic_add_zero 397 # define atomic_add_zero(mem, value) \ 398 ({ __typeof (value) __atg13_value = (value); \ 399 atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; }) 400 #endif 401 402 403 #ifndef atomic_bit_set 404 # define atomic_bit_set(mem, bit) \ 405 (void) atomic_bit_test_set(mem, bit) 406 #endif 407 408 409 #ifndef atomic_bit_test_set 410 # define atomic_bit_test_set(mem, bit) \ 411 ({ __typeof (*(mem)) __atg14_old; \ 412 __typeof (mem) __atg14_memp = (mem); \ 413 __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit)); \ 414 \ 415 do \ 416 __atg14_old = (*__atg14_memp); \ 417 while (__builtin_expect \ 418 (atomic_compare_and_exchange_bool_acq (__atg14_memp, \ 419 __atg14_old | __atg14_mask,\ 420 __atg14_old), 0)); \ 421 \ 422 __atg14_old & __atg14_mask; }) 423 #endif 424 425 /* Atomically *mem &= mask. */ 426 #ifndef atomic_and 427 # define atomic_and(mem, mask) \ 428 do { \ 429 __typeof (*(mem)) __atg15_old; \ 430 __typeof (mem) __atg15_memp = (mem); \ 431 __typeof (*(mem)) __atg15_mask = (mask); \ 432 \ 433 do \ 434 __atg15_old = (*__atg15_memp); \ 435 while (__builtin_expect \ 436 (atomic_compare_and_exchange_bool_acq (__atg15_memp, \ 437 __atg15_old & __atg15_mask, \ 438 __atg15_old), 0)); \ 439 } while (0) 440 #endif 441 442 #ifndef catomic_and 443 # define catomic_and(mem, mask) \ 444 do { \ 445 __typeof (*(mem)) __atg20_old; \ 446 __typeof (mem) __atg20_memp = (mem); \ 447 __typeof (*(mem)) __atg20_mask = (mask); \ 448 \ 449 do \ 450 __atg20_old = (*__atg20_memp); \ 451 while (__builtin_expect \ 452 (catomic_compare_and_exchange_bool_acq (__atg20_memp, \ 453 __atg20_old & __atg20_mask,\ 454 __atg20_old), 0)); \ 455 } while (0) 456 #endif 457 458 /* Atomically *mem &= mask and return the old value of *mem. */ 459 #ifndef atomic_and_val 460 # define atomic_and_val(mem, mask) \ 461 ({ __typeof (*(mem)) __atg16_old; \ 462 __typeof (mem) __atg16_memp = (mem); \ 463 __typeof (*(mem)) __atg16_mask = (mask); \ 464 \ 465 do \ 466 __atg16_old = (*__atg16_memp); \ 467 while (__builtin_expect \ 468 (atomic_compare_and_exchange_bool_acq (__atg16_memp, \ 469 __atg16_old & __atg16_mask,\ 470 __atg16_old), 0)); \ 471 \ 472 __atg16_old; }) 473 #endif 474 475 /* Atomically *mem |= mask and return the old value of *mem. */ 476 #ifndef atomic_or 477 # define atomic_or(mem, mask) \ 478 do { \ 479 __typeof (*(mem)) __atg17_old; \ 480 __typeof (mem) __atg17_memp = (mem); \ 481 __typeof (*(mem)) __atg17_mask = (mask); \ 482 \ 483 do \ 484 __atg17_old = (*__atg17_memp); \ 485 while (__builtin_expect \ 486 (atomic_compare_and_exchange_bool_acq (__atg17_memp, \ 487 __atg17_old | __atg17_mask, \ 488 __atg17_old), 0)); \ 489 } while (0) 490 #endif 491 492 #ifndef catomic_or 493 # define catomic_or(mem, mask) \ 494 do { \ 495 __typeof (*(mem)) __atg18_old; \ 496 __typeof (mem) __atg18_memp = (mem); \ 497 __typeof (*(mem)) __atg18_mask = (mask); \ 498 \ 499 do \ 500 __atg18_old = (*__atg18_memp); \ 501 while (__builtin_expect \ 502 (catomic_compare_and_exchange_bool_acq (__atg18_memp, \ 503 __atg18_old | __atg18_mask,\ 504 __atg18_old), 0)); \ 505 } while (0) 506 #endif 507 508 /* Atomically *mem |= mask and return the old value of *mem. */ 509 #ifndef atomic_or_val 510 # define atomic_or_val(mem, mask) \ 511 ({ __typeof (*(mem)) __atg19_old; \ 512 __typeof (mem) __atg19_memp = (mem); \ 513 __typeof (*(mem)) __atg19_mask = (mask); \ 514 \ 515 do \ 516 __atg19_old = (*__atg19_memp); \ 517 while (__builtin_expect \ 518 (atomic_compare_and_exchange_bool_acq (__atg19_memp, \ 519 __atg19_old | __atg19_mask,\ 520 __atg19_old), 0)); \ 521 \ 522 __atg19_old; }) 523 #endif 524 525 #ifndef atomic_full_barrier 526 # define atomic_full_barrier() __asm__ ("" ::: "memory") 527 #endif 528 529 530 #ifndef atomic_read_barrier 531 # define atomic_read_barrier() atomic_full_barrier () 532 #endif 533 534 535 #ifndef atomic_write_barrier 536 # define atomic_write_barrier() atomic_full_barrier () 537 #endif 538 539 540 #ifndef atomic_forced_read 541 # define atomic_forced_read(x) \ 542 ({ __typeof (x) __x; __asm__ ("" : "=r" (__x) : "0" (x)); __x; }) 543 #endif 544 545 /* The following functions are a subset of the atomic operations provided by 546 C11. Usually, a function named atomic_OP_MO(args) is equivalent to C11's 547 atomic_OP_explicit(args, memory_order_MO); exceptions noted below. */ 548 549 /* Each arch can request to use compiler built-ins for C11 atomics. If it 550 does, all atomics will be based on these. */ 551 #if defined USE_ATOMIC_COMPILER_BUILTINS 552 553 /* We require 32b atomic operations; some archs also support 64b atomic 554 operations. */ 555 void __atomic_link_error (void); 556 # if defined(__HAVE_64B_ATOMICS) && __HAVE_64B_ATOMICS 557 # define __atomic_check_size(mem) \ 558 if ((sizeof (*mem) != 4) && (sizeof (*mem) != 8)) \ 559 __atomic_link_error (); 560 # else 561 # define __atomic_check_size(mem) \ 562 if (sizeof (*mem) != 4) \ 563 __atomic_link_error (); 564 # endif 565 566 # define atomic_thread_fence_acquire() \ 567 __atomic_thread_fence (__ATOMIC_ACQUIRE) 568 # define atomic_thread_fence_release() \ 569 __atomic_thread_fence (__ATOMIC_RELEASE) 570 # define atomic_thread_fence_seq_cst() \ 571 __atomic_thread_fence (__ATOMIC_SEQ_CST) 572 573 # define atomic_load_relaxed(mem) \ 574 ({ __atomic_check_size((mem)); __atomic_load_n ((mem), __ATOMIC_RELAXED); }) 575 # define atomic_load_acquire(mem) \ 576 ({ __atomic_check_size((mem)); __atomic_load_n ((mem), __ATOMIC_ACQUIRE); }) 577 578 # define atomic_store_relaxed(mem, val) \ 579 do { \ 580 __atomic_check_size((mem)); \ 581 __atomic_store_n ((mem), (val), __ATOMIC_RELAXED); \ 582 } while (0) 583 # define atomic_store_release(mem, val) \ 584 do { \ 585 __atomic_check_size((mem)); \ 586 __atomic_store_n ((mem), (val), __ATOMIC_RELEASE); \ 587 } while (0) 588 589 /* On failure, this CAS has memory_order_relaxed semantics. */ 590 # define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \ 591 ({ __atomic_check_size((mem)); \ 592 __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \ 593 __ATOMIC_RELAXED, __ATOMIC_RELAXED); }) 594 # define atomic_compare_exchange_weak_acquire(mem, expected, desired) \ 595 ({ __atomic_check_size((mem)); \ 596 __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \ 597 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); }) 598 # define atomic_compare_exchange_weak_release(mem, expected, desired) \ 599 ({ __atomic_check_size((mem)); \ 600 __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \ 601 __ATOMIC_RELEASE, __ATOMIC_RELAXED); }) 602 603 # define atomic_exchange_acquire(mem, desired) \ 604 ({ __atomic_check_size((mem)); \ 605 __atomic_exchange_n ((mem), (desired), __ATOMIC_ACQUIRE); }) 606 # define atomic_exchange_release(mem, desired) \ 607 ({ __atomic_check_size((mem)); \ 608 __atomic_exchange_n ((mem), (desired), __ATOMIC_RELEASE); }) 609 610 # define atomic_fetch_add_relaxed(mem, operand) \ 611 ({ __atomic_check_size((mem)); \ 612 __atomic_fetch_add ((mem), (operand), __ATOMIC_RELAXED); }) 613 # define atomic_fetch_add_acquire(mem, operand) \ 614 ({ __atomic_check_size((mem)); \ 615 __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQUIRE); }) 616 # define atomic_fetch_add_release(mem, operand) \ 617 ({ __atomic_check_size((mem)); \ 618 __atomic_fetch_add ((mem), (operand), __ATOMIC_RELEASE); }) 619 # define atomic_fetch_add_acq_rel(mem, operand) \ 620 ({ __atomic_check_size((mem)); \ 621 __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQ_REL); }) 622 623 # define atomic_fetch_and_acquire(mem, operand) \ 624 ({ __atomic_check_size((mem)); \ 625 __atomic_fetch_and ((mem), (operand), __ATOMIC_ACQUIRE); }) 626 627 # define atomic_fetch_or_relaxed(mem, operand) \ 628 ({ __atomic_check_size((mem)); \ 629 __atomic_fetch_or ((mem), (operand), __ATOMIC_RELAXED); }) 630 # define atomic_fetch_or_acquire(mem, operand) \ 631 ({ __atomic_check_size((mem)); \ 632 __atomic_fetch_or ((mem), (operand), __ATOMIC_ACQUIRE); }) 633 634 #else /* !USE_ATOMIC_COMPILER_BUILTINS */ 635 636 /* By default, we assume that read, write, and full barriers are equivalent 637 to acquire, release, and seq_cst barriers. Archs for which this does not 638 hold have to provide custom definitions of the fences. */ 639 # ifndef atomic_thread_fence_acquire 640 # define atomic_thread_fence_acquire() atomic_read_barrier () 641 # endif 642 # ifndef atomic_thread_fence_release 643 # define atomic_thread_fence_release() atomic_write_barrier () 644 # endif 645 # ifndef atomic_thread_fence_seq_cst 646 # define atomic_thread_fence_seq_cst() atomic_full_barrier () 647 # endif 648 649 # ifndef atomic_load_relaxed 650 # define atomic_load_relaxed(mem) \ 651 ({ __typeof (*(mem)) __atg100_val; \ 652 __asm__ ("" : "=r" (__atg100_val) : "0" (*(mem))); \ 653 __atg100_val; }) 654 # endif 655 # ifndef atomic_load_acquire 656 # define atomic_load_acquire(mem) \ 657 ({ __typeof (*(mem)) __atg101_val = atomic_load_relaxed (mem); \ 658 atomic_thread_fence_acquire (); \ 659 __atg101_val; }) 660 # endif 661 662 # ifndef atomic_store_relaxed 663 /* XXX Use inline asm here? */ 664 # define atomic_store_relaxed(mem, val) do { *(mem) = (val); } while (0) 665 # endif 666 # ifndef atomic_store_release 667 # define atomic_store_release(mem, val) \ 668 do { \ 669 atomic_thread_fence_release (); \ 670 atomic_store_relaxed ((mem), (val)); \ 671 } while (0) 672 # endif 673 674 /* On failure, this CAS has memory_order_relaxed semantics. */ 675 /* XXX This potentially has one branch more than necessary, but archs 676 currently do not define a CAS that returns both the previous value and 677 the success flag. */ 678 # ifndef atomic_compare_exchange_weak_acquire 679 # define atomic_compare_exchange_weak_acquire(mem, expected, desired) \ 680 ({ __typeof (*(expected)) __atg102_expected = *(expected); \ 681 *(expected) = \ 682 atomic_compare_and_exchange_val_acq ((mem), (desired), *(expected)); \ 683 *(expected) == __atg102_expected; }) 684 # endif 685 # ifndef atomic_compare_exchange_weak_relaxed 686 /* XXX Fall back to CAS with acquire MO because archs do not define a weaker 687 CAS. */ 688 # define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \ 689 atomic_compare_exchange_weak_acquire ((mem), (expected), (desired)) 690 # endif 691 # ifndef atomic_compare_exchange_weak_release 692 # define atomic_compare_exchange_weak_release(mem, expected, desired) \ 693 ({ __typeof (*(expected)) __atg103_expected = *(expected); \ 694 *(expected) = \ 695 atomic_compare_and_exchange_val_rel ((mem), (desired), *(expected)); \ 696 *(expected) == __atg103_expected; }) 697 # endif 698 699 # ifndef atomic_exchange_acquire 700 # define atomic_exchange_acquire(mem, val) \ 701 atomic_exchange_acq ((mem), (val)) 702 # endif 703 # ifndef atomic_exchange_release 704 # define atomic_exchange_release(mem, val) \ 705 atomic_exchange_rel ((mem), (val)) 706 # endif 707 708 # ifndef atomic_fetch_add_acquire 709 # define atomic_fetch_add_acquire(mem, operand) \ 710 atomic_exchange_and_add_acq ((mem), (operand)) 711 # endif 712 # ifndef atomic_fetch_add_relaxed 713 /* XXX Fall back to acquire MO because the MO semantics of 714 atomic_exchange_and_add are not documented; the generic version falls back 715 to atomic_exchange_and_add_acq if atomic_exchange_and_add is not defined, 716 and vice versa. */ 717 # define atomic_fetch_add_relaxed(mem, operand) \ 718 atomic_fetch_add_acquire ((mem), (operand)) 719 # endif 720 # ifndef atomic_fetch_add_release 721 # define atomic_fetch_add_release(mem, operand) \ 722 atomic_exchange_and_add_rel ((mem), (operand)) 723 # endif 724 # ifndef atomic_fetch_add_acq_rel 725 # define atomic_fetch_add_acq_rel(mem, operand) \ 726 ({ atomic_thread_fence_release (); \ 727 atomic_exchange_and_add_acq ((mem), (operand)); }) 728 # endif 729 730 /* XXX The default for atomic_and_val has acquire semantics, but this is not 731 documented. */ 732 # ifndef atomic_fetch_and_acquire 733 # define atomic_fetch_and_acquire(mem, operand) \ 734 atomic_and_val ((mem), (operand)) 735 # endif 736 737 /* XXX The default for atomic_or_val has acquire semantics, but this is not 738 documented. */ 739 # ifndef atomic_fetch_or_acquire 740 # define atomic_fetch_or_acquire(mem, operand) \ 741 atomic_or_val ((mem), (operand)) 742 # endif 743 /* XXX Fall back to acquire MO because archs do not define a weaker 744 atomic_or_val. */ 745 # ifndef atomic_fetch_or_relaxed 746 # define atomic_fetch_or_relaxed(mem, operand) \ 747 atomic_fetch_or_acquire ((mem), (operand)) 748 # endif 749 750 #endif /* !USE_ATOMIC_COMPILER_BUILTINS */ 751 752 753 #ifndef atomic_delay 754 # define atomic_delay() do { /* nothing */ } while (0) 755 #endif 756 757 #endif /* atomic.h */ 758