1 /* Internal macros for atomic operations for GNU C Library. 2 Copyright (C) 2002-2006, 2009 Free Software Foundation, Inc. 3 This file is part of the GNU C Library. 4 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. 5 6 The GNU C Library is free software; you can redistribute it and/or 7 modify it under the terms of the GNU Lesser General Public 8 License as published by the Free Software Foundation; either 9 version 2.1 of the License, or (at your option) any later version. 10 11 The GNU C Library is distributed in the hope that it will be useful, 12 but WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 Lesser General Public License for more details. 15 16 You should have received a copy of the GNU Lesser General Public 17 License along with the GNU C Library; if not, see 18 <http://www.gnu.org/licenses/>. */ 19 20 #ifndef _ATOMIC_H 21 #define _ATOMIC_H 1 22 23 /* This header defines three types of macros: 24 25 - atomic arithmetic and logic operation on memory. They all 26 have the prefix "atomic_". 27 28 - conditionally atomic operations of the same kinds. These 29 always behave identical but can be faster when atomicity 30 is not really needed since only one thread has access to 31 the memory location. In that case the code is slower in 32 the multi-thread case. The interfaces have the prefix 33 "catomic_". 34 35 - support functions like barriers. They also have the preifx 36 "atomic_". 37 38 Architectures must provide a few lowlevel macros (the compare 39 and exchange definitions). All others are optional. They 40 should only be provided if the architecture has specific 41 support for the operation. 42 43 As <atomic.h> macros are usually heavily nested and often use local 44 variables to make sure side-effects are evaluated properly, use for 45 macro local variables a per-macro unique prefix. This file uses 46 __atgN_ prefix where N is different in each macro. */ 47 48 #include <stdlib.h> 49 50 #include <bits/atomic.h> 51 52 /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the 53 bit width of *MEM. The calling macro puts parens around MEM 54 and following args. */ 55 #define __atomic_val_bysize(pre, post, mem, ...) \ 56 ({ \ 57 __typeof (*mem) __atg1_result; \ 58 if (sizeof (*mem) == 1) \ 59 __atg1_result = pre##_8_##post (mem, __VA_ARGS__); \ 60 else if (sizeof (*mem) == 2) \ 61 __atg1_result = pre##_16_##post (mem, __VA_ARGS__); \ 62 else if (sizeof (*mem) == 4) \ 63 __atg1_result = pre##_32_##post (mem, __VA_ARGS__); \ 64 else if (sizeof (*mem) == 8) \ 65 __atg1_result = pre##_64_##post (mem, __VA_ARGS__); \ 66 else \ 67 abort (); \ 68 __atg1_result; \ 69 }) 70 #define __atomic_bool_bysize(pre, post, mem, ...) \ 71 ({ \ 72 int __atg2_result; \ 73 if (sizeof (*mem) == 1) \ 74 __atg2_result = pre##_8_##post (mem, __VA_ARGS__); \ 75 else if (sizeof (*mem) == 2) \ 76 __atg2_result = pre##_16_##post (mem, __VA_ARGS__); \ 77 else if (sizeof (*mem) == 4) \ 78 __atg2_result = pre##_32_##post (mem, __VA_ARGS__); \ 79 else if (sizeof (*mem) == 8) \ 80 __atg2_result = pre##_64_##post (mem, __VA_ARGS__); \ 81 else \ 82 abort (); \ 83 __atg2_result; \ 84 }) 85 86 87 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. 88 Return the old *MEM value. */ 89 #if !defined atomic_compare_and_exchange_val_acq \ 90 && defined __arch_compare_and_exchange_val_32_acq 91 # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \ 92 __atomic_val_bysize (__arch_compare_and_exchange_val,acq, \ 93 mem, newval, oldval) 94 #endif 95 96 97 #ifndef catomic_compare_and_exchange_val_acq 98 # ifdef __arch_c_compare_and_exchange_val_32_acq 99 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \ 100 __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \ 101 mem, newval, oldval) 102 # else 103 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \ 104 atomic_compare_and_exchange_val_acq (mem, newval, oldval) 105 # endif 106 #endif 107 108 109 #ifndef catomic_compare_and_exchange_val_rel 110 # ifndef atomic_compare_and_exchange_val_rel 111 # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \ 112 catomic_compare_and_exchange_val_acq (mem, newval, oldval) 113 # else 114 # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \ 115 atomic_compare_and_exchange_val_rel (mem, newval, oldval) 116 # endif 117 #endif 118 119 120 #ifndef atomic_compare_and_exchange_val_rel 121 # define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \ 122 atomic_compare_and_exchange_val_acq (mem, newval, oldval) 123 #endif 124 125 126 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. 127 Return zero if *MEM was changed or non-zero if no exchange happened. */ 128 #ifndef atomic_compare_and_exchange_bool_acq 129 # ifdef __arch_compare_and_exchange_bool_32_acq 130 # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 131 __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq, \ 132 mem, newval, oldval) 133 # else 134 # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 135 ({ /* Cannot use __oldval here, because macros later in this file might \ 136 call this macro with __oldval argument. */ \ 137 __typeof (oldval) __atg3_old = (oldval); \ 138 atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old) \ 139 != __atg3_old; \ 140 }) 141 # endif 142 #endif 143 144 145 #ifndef catomic_compare_and_exchange_bool_acq 146 # ifdef __arch_c_compare_and_exchange_bool_32_acq 147 # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 148 __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq, \ 149 mem, newval, oldval) 150 # else 151 # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ 152 ({ /* Cannot use __oldval here, because macros later in this file might \ 153 call this macro with __oldval argument. */ \ 154 __typeof (oldval) __atg4_old = (oldval); \ 155 catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old) \ 156 != __atg4_old; \ 157 }) 158 # endif 159 #endif 160 161 162 #ifndef catomic_compare_and_exchange_bool_rel 163 # ifndef atomic_compare_and_exchange_bool_rel 164 # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \ 165 catomic_compare_and_exchange_bool_acq (mem, newval, oldval) 166 # else 167 # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \ 168 atomic_compare_and_exchange_bool_rel (mem, newval, oldval) 169 # endif 170 #endif 171 172 173 #ifndef atomic_compare_and_exchange_bool_rel 174 # define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \ 175 atomic_compare_and_exchange_bool_acq (mem, newval, oldval) 176 #endif 177 178 179 /* Store NEWVALUE in *MEM and return the old value. */ 180 #ifndef atomic_exchange_acq 181 # define atomic_exchange_acq(mem, newvalue) \ 182 ({ __typeof (*(mem)) __atg5_oldval; \ 183 __typeof (mem) __atg5_memp = (mem); \ 184 __typeof (*(mem)) __atg5_value = (newvalue); \ 185 \ 186 do \ 187 __atg5_oldval = *__atg5_memp; \ 188 while (__builtin_expect \ 189 (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \ 190 __atg5_oldval), 0)); \ 191 \ 192 __atg5_oldval; }) 193 #endif 194 195 #ifndef atomic_exchange_rel 196 # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue) 197 #endif 198 199 200 /* Add VALUE to *MEM and return the old value of *MEM. */ 201 #ifndef atomic_exchange_and_add 202 # define atomic_exchange_and_add(mem, value) \ 203 ({ __typeof (*(mem)) __atg6_oldval; \ 204 __typeof (mem) __atg6_memp = (mem); \ 205 __typeof (*(mem)) __atg6_value = (value); \ 206 \ 207 do \ 208 __atg6_oldval = *__atg6_memp; \ 209 while (__builtin_expect \ 210 (atomic_compare_and_exchange_bool_acq (__atg6_memp, \ 211 __atg6_oldval \ 212 + __atg6_value, \ 213 __atg6_oldval), 0)); \ 214 \ 215 __atg6_oldval; }) 216 #endif 217 218 219 #ifndef catomic_exchange_and_add 220 # define catomic_exchange_and_add(mem, value) \ 221 ({ __typeof (*(mem)) __atg7_oldv; \ 222 __typeof (mem) __atg7_memp = (mem); \ 223 __typeof (*(mem)) __atg7_value = (value); \ 224 \ 225 do \ 226 __atg7_oldv = *__atg7_memp; \ 227 while (__builtin_expect \ 228 (catomic_compare_and_exchange_bool_acq (__atg7_memp, \ 229 __atg7_oldv \ 230 + __atg7_value, \ 231 __atg7_oldv), 0)); \ 232 \ 233 __atg7_oldv; }) 234 #endif 235 236 237 #ifndef atomic_max 238 # define atomic_max(mem, value) \ 239 do { \ 240 __typeof (*(mem)) __atg8_oldval; \ 241 __typeof (mem) __atg8_memp = (mem); \ 242 __typeof (*(mem)) __atg8_value = (value); \ 243 do { \ 244 __atg8_oldval = *__atg8_memp; \ 245 if (__atg8_oldval >= __atg8_value) \ 246 break; \ 247 } while (__builtin_expect \ 248 (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\ 249 __atg8_oldval), 0)); \ 250 } while (0) 251 #endif 252 253 254 #ifndef catomic_max 255 # define catomic_max(mem, value) \ 256 do { \ 257 __typeof (*(mem)) __atg9_oldv; \ 258 __typeof (mem) __atg9_memp = (mem); \ 259 __typeof (*(mem)) __atg9_value = (value); \ 260 do { \ 261 __atg9_oldv = *__atg9_memp; \ 262 if (__atg9_oldv >= __atg9_value) \ 263 break; \ 264 } while (__builtin_expect \ 265 (catomic_compare_and_exchange_bool_acq (__atg9_memp, \ 266 __atg9_value, \ 267 __atg9_oldv), 0)); \ 268 } while (0) 269 #endif 270 271 272 #ifndef atomic_min 273 # define atomic_min(mem, value) \ 274 do { \ 275 __typeof (*(mem)) __atg10_oldval; \ 276 __typeof (mem) __atg10_memp = (mem); \ 277 __typeof (*(mem)) __atg10_value = (value); \ 278 do { \ 279 __atg10_oldval = *__atg10_memp; \ 280 if (__atg10_oldval <= __atg10_value) \ 281 break; \ 282 } while (__builtin_expect \ 283 (atomic_compare_and_exchange_bool_acq (__atg10_memp, \ 284 __atg10_value, \ 285 __atg10_oldval), 0)); \ 286 } while (0) 287 #endif 288 289 290 #ifndef atomic_add 291 # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value)) 292 #endif 293 294 295 #ifndef catomic_add 296 # define catomic_add(mem, value) \ 297 (void) catomic_exchange_and_add ((mem), (value)) 298 #endif 299 300 301 #ifndef atomic_increment 302 # define atomic_increment(mem) atomic_add ((mem), 1) 303 #endif 304 305 306 #ifndef catomic_increment 307 # define catomic_increment(mem) catomic_add ((mem), 1) 308 #endif 309 310 311 #ifndef atomic_increment_val 312 # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1) 313 #endif 314 315 316 #ifndef catomic_increment_val 317 # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1) 318 #endif 319 320 321 /* Add one to *MEM and return true iff it's now zero. */ 322 #ifndef atomic_increment_and_test 323 # define atomic_increment_and_test(mem) \ 324 (atomic_exchange_and_add ((mem), 1) + 1 == 0) 325 #endif 326 327 328 #ifndef atomic_decrement 329 # define atomic_decrement(mem) atomic_add ((mem), -1) 330 #endif 331 332 333 #ifndef catomic_decrement 334 # define catomic_decrement(mem) catomic_add ((mem), -1) 335 #endif 336 337 338 #ifndef atomic_decrement_val 339 # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1) 340 #endif 341 342 343 #ifndef catomic_decrement_val 344 # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1) 345 #endif 346 347 348 /* Subtract 1 from *MEM and return true iff it's now zero. */ 349 #ifndef atomic_decrement_and_test 350 # define atomic_decrement_and_test(mem) \ 351 (atomic_exchange_and_add ((mem), -1) == 1) 352 #endif 353 354 355 /* Decrement *MEM if it is > 0, and return the old value. */ 356 #ifndef atomic_decrement_if_positive 357 # define atomic_decrement_if_positive(mem) \ 358 ({ __typeof (*(mem)) __atg11_oldval; \ 359 __typeof (mem) __atg11_memp = (mem); \ 360 \ 361 do \ 362 { \ 363 __atg11_oldval = *__atg11_memp; \ 364 if (__builtin_expect (__atg11_oldval <= 0, 0)) \ 365 break; \ 366 } \ 367 while (__builtin_expect \ 368 (atomic_compare_and_exchange_bool_acq (__atg11_memp, \ 369 __atg11_oldval - 1, \ 370 __atg11_oldval), 0)); \ 371 __atg11_oldval; }) 372 #endif 373 374 375 #ifndef atomic_add_negative 376 # define atomic_add_negative(mem, value) \ 377 ({ __typeof (value) __atg12_value = (value); \ 378 atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; }) 379 #endif 380 381 382 #ifndef atomic_add_zero 383 # define atomic_add_zero(mem, value) \ 384 ({ __typeof (value) __atg13_value = (value); \ 385 atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; }) 386 #endif 387 388 389 #ifndef atomic_bit_set 390 # define atomic_bit_set(mem, bit) \ 391 (void) atomic_bit_test_set(mem, bit) 392 #endif 393 394 395 #ifndef atomic_bit_test_set 396 # define atomic_bit_test_set(mem, bit) \ 397 ({ __typeof (*(mem)) __atg14_old; \ 398 __typeof (mem) __atg14_memp = (mem); \ 399 __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit)); \ 400 \ 401 do \ 402 __atg14_old = (*__atg14_memp); \ 403 while (__builtin_expect \ 404 (atomic_compare_and_exchange_bool_acq (__atg14_memp, \ 405 __atg14_old | __atg14_mask,\ 406 __atg14_old), 0)); \ 407 \ 408 __atg14_old & __atg14_mask; }) 409 #endif 410 411 /* Atomically *mem &= mask. */ 412 #ifndef atomic_and 413 # define atomic_and(mem, mask) \ 414 do { \ 415 __typeof (*(mem)) __atg15_old; \ 416 __typeof (mem) __atg15_memp = (mem); \ 417 __typeof (*(mem)) __atg15_mask = (mask); \ 418 \ 419 do \ 420 __atg15_old = (*__atg15_memp); \ 421 while (__builtin_expect \ 422 (atomic_compare_and_exchange_bool_acq (__atg15_memp, \ 423 __atg15_old & __atg15_mask, \ 424 __atg15_old), 0)); \ 425 } while (0) 426 #endif 427 428 #ifndef catomic_and 429 # define catomic_and(mem, mask) \ 430 do { \ 431 __typeof (*(mem)) __atg20_old; \ 432 __typeof (mem) __atg20_memp = (mem); \ 433 __typeof (*(mem)) __atg20_mask = (mask); \ 434 \ 435 do \ 436 __atg20_old = (*__atg20_memp); \ 437 while (__builtin_expect \ 438 (catomic_compare_and_exchange_bool_acq (__atg20_memp, \ 439 __atg20_old & __atg20_mask,\ 440 __atg20_old), 0)); \ 441 } while (0) 442 #endif 443 444 /* Atomically *mem &= mask and return the old value of *mem. */ 445 #ifndef atomic_and_val 446 # define atomic_and_val(mem, mask) \ 447 ({ __typeof (*(mem)) __atg16_old; \ 448 __typeof (mem) __atg16_memp = (mem); \ 449 __typeof (*(mem)) __atg16_mask = (mask); \ 450 \ 451 do \ 452 __atg16_old = (*__atg16_memp); \ 453 while (__builtin_expect \ 454 (atomic_compare_and_exchange_bool_acq (__atg16_memp, \ 455 __atg16_old & __atg16_mask,\ 456 __atg16_old), 0)); \ 457 \ 458 __atg16_old; }) 459 #endif 460 461 /* Atomically *mem |= mask and return the old value of *mem. */ 462 #ifndef atomic_or 463 # define atomic_or(mem, mask) \ 464 do { \ 465 __typeof (*(mem)) __atg17_old; \ 466 __typeof (mem) __atg17_memp = (mem); \ 467 __typeof (*(mem)) __atg17_mask = (mask); \ 468 \ 469 do \ 470 __atg17_old = (*__atg17_memp); \ 471 while (__builtin_expect \ 472 (atomic_compare_and_exchange_bool_acq (__atg17_memp, \ 473 __atg17_old | __atg17_mask, \ 474 __atg17_old), 0)); \ 475 } while (0) 476 #endif 477 478 #ifndef catomic_or 479 # define catomic_or(mem, mask) \ 480 do { \ 481 __typeof (*(mem)) __atg18_old; \ 482 __typeof (mem) __atg18_memp = (mem); \ 483 __typeof (*(mem)) __atg18_mask = (mask); \ 484 \ 485 do \ 486 __atg18_old = (*__atg18_memp); \ 487 while (__builtin_expect \ 488 (catomic_compare_and_exchange_bool_acq (__atg18_memp, \ 489 __atg18_old | __atg18_mask,\ 490 __atg18_old), 0)); \ 491 } while (0) 492 #endif 493 494 /* Atomically *mem |= mask and return the old value of *mem. */ 495 #ifndef atomic_or_val 496 # define atomic_or_val(mem, mask) \ 497 ({ __typeof (*(mem)) __atg19_old; \ 498 __typeof (mem) __atg19_memp = (mem); \ 499 __typeof (*(mem)) __atg19_mask = (mask); \ 500 \ 501 do \ 502 __atg19_old = (*__atg19_memp); \ 503 while (__builtin_expect \ 504 (atomic_compare_and_exchange_bool_acq (__atg19_memp, \ 505 __atg19_old | __atg19_mask,\ 506 __atg19_old), 0)); \ 507 \ 508 __atg19_old; }) 509 #endif 510 511 #ifndef atomic_full_barrier 512 # define atomic_full_barrier() __asm__ ("" ::: "memory") 513 #endif 514 515 516 #ifndef atomic_read_barrier 517 # define atomic_read_barrier() atomic_full_barrier () 518 #endif 519 520 521 #ifndef atomic_write_barrier 522 # define atomic_write_barrier() atomic_full_barrier () 523 #endif 524 525 526 #ifndef atomic_forced_read 527 # define atomic_forced_read(x) \ 528 ({ __typeof (x) __x; __asm__ ("" : "=r" (__x) : "0" (x)); __x; }) 529 #endif 530 531 532 #ifndef atomic_delay 533 # define atomic_delay() do { /* nothing */ } while (0) 534 #endif 535 536 #endif /* atomic.h */ 537