1 /* Low-level functions for atomic operations. Mips version. 2 Copyright (C) 2005 Free Software Foundation, Inc. 3 This file is part of the GNU C Library. 4 5 The GNU C Library is free software; you can redistribute it and/or 6 modify it under the terms of the GNU Lesser General Public 7 License as published by the Free Software Foundation; either 8 version 2.1 of the License, or (at your option) any later version. 9 10 The GNU C Library is distributed in the hope that it will be useful, 11 but WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 Lesser General Public License for more details. 14 15 You should have received a copy of the GNU Lesser General Public 16 License along with the GNU C Library; if not, see 17 <http://www.gnu.org/licenses/>. */ 18 19 #ifndef _MIPS_BITS_ATOMIC_H 20 #define _MIPS_BITS_ATOMIC_H 1 21 22 #include <inttypes.h> 23 #include <sgidefs.h> 24 25 typedef int32_t atomic32_t; 26 typedef uint32_t uatomic32_t; 27 typedef int_fast32_t atomic_fast32_t; 28 typedef uint_fast32_t uatomic_fast32_t; 29 30 typedef int64_t atomic64_t; 31 typedef uint64_t uatomic64_t; 32 typedef int_fast64_t atomic_fast64_t; 33 typedef uint_fast64_t uatomic_fast64_t; 34 35 typedef intptr_t atomicptr_t; 36 typedef uintptr_t uatomicptr_t; 37 typedef intmax_t atomic_max_t; 38 typedef uintmax_t uatomic_max_t; 39 40 #if _MIPS_SIM == _ABIO32 && __mips < 2 41 #define MIPS_PUSH_MIPS2 ".set mips2\n\t" 42 #else 43 #define MIPS_PUSH_MIPS2 44 #endif 45 46 /* See the comments in <sys/asm.h> about the use of the sync instruction. */ 47 #ifndef MIPS_SYNC 48 # define MIPS_SYNC sync 49 #endif 50 51 /* Certain revisions of the R10000 Processor need an LL/SC Workaround 52 enabled. Revisions before 3.0 misbehave on atomic operations, and 53 Revs 2.6 and lower deadlock after several seconds due to other errata. 54 55 To quote the R10K Errata: 56 Workaround: The basic idea is to inhibit the four instructions 57 from simultaneously becoming active in R10000. Padding all 58 ll/sc sequences with nops or changing the looping branch in the 59 routines to a branch likely (which is always predicted taken 60 by R10000) will work. The nops should go after the loop, and the 61 number of them should be 28. This number could be decremented for 62 each additional instruction in the ll/sc loop such as the lock 63 modifier(s) between the ll and sc, the looping branch and its 64 delay slot. For typical short routines with one ll/sc loop, any 65 instructions after the loop could also count as a decrement. The 66 nop workaround pollutes the cache more but would be a few cycles 67 faster if all the code is in the cache and the looping branch 68 is predicted not taken. */ 69 70 71 #ifdef _MIPS_ARCH_R10000 72 #define R10K_BEQZ_INSN "beqzl" 73 #else 74 #define R10K_BEQZ_INSN "beqz" 75 #endif 76 77 #define MIPS_SYNC_STR_2(X) #X 78 #define MIPS_SYNC_STR_1(X) MIPS_SYNC_STR_2(X) 79 #define MIPS_SYNC_STR MIPS_SYNC_STR_1(MIPS_SYNC) 80 81 /* Compare and exchange. For all of the "xxx" routines, we expect a 82 "__prev" and a "__cmp" variable to be provided by the enclosing scope, 83 in which values are returned. */ 84 85 #define __arch_compare_and_exchange_xxx_8_int(mem, newval, oldval, rel, acq) \ 86 (abort (), __prev = 0, __cmp = 0) 87 88 #define __arch_compare_and_exchange_xxx_16_int(mem, newval, oldval, rel, acq) \ 89 (abort (), __prev = 0, __cmp = 0) 90 91 #define __arch_compare_and_exchange_xxx_32_int(mem, newval, oldval, rel, acq) \ 92 __asm__ __volatile__ ( \ 93 ".set push\n\t" \ 94 MIPS_PUSH_MIPS2 \ 95 rel "\n" \ 96 "1:\t" \ 97 "ll %0,%5\n\t" \ 98 "move %1,$0\n\t" \ 99 "bne %0,%3,2f\n\t" \ 100 "move %1,%4\n\t" \ 101 "sc %1,%2\n\t" \ 102 R10K_BEQZ_INSN" %1,1b\n" \ 103 acq "\n\t" \ 104 ".set pop\n" \ 105 "2:\n\t" \ 106 : "=&r" (__prev), "=&r" (__cmp), "=m" (*mem) \ 107 : "r" (oldval), "r" (newval), "m" (*mem) \ 108 : "memory") 109 110 #if _MIPS_SIM == _ABIO32 111 /* We can't do an atomic 64-bit operation in O32. */ 112 #define __arch_compare_and_exchange_xxx_64_int(mem, newval, oldval, rel, acq) \ 113 (abort (), __prev = 0, __cmp = 0) 114 #else 115 #define __arch_compare_and_exchange_xxx_64_int(mem, newval, oldval, rel, acq) \ 116 __asm__ __volatile__ ("\n" \ 117 ".set push\n\t" \ 118 MIPS_PUSH_MIPS2 \ 119 rel "\n" \ 120 "1:\t" \ 121 "lld %0,%5\n\t" \ 122 "move %1,$0\n\t" \ 123 "bne %0,%3,2f\n\t" \ 124 "move %1,%4\n\t" \ 125 "scd %1,%2\n\t" \ 126 R10K_BEQZ_INSN" %1,1b\n" \ 127 acq "\n\t" \ 128 ".set pop\n" \ 129 "2:\n\t" \ 130 : "=&r" (__prev), "=&r" (__cmp), "=m" (*mem) \ 131 : "r" (oldval), "r" (newval), "m" (*mem) \ 132 : "memory") 133 #endif 134 135 /* For all "bool" routines, we return FALSE if exchange succesful. */ 136 137 #define __arch_compare_and_exchange_bool_8_int(mem, new, old, rel, acq) \ 138 ({ __typeof (*mem) __prev attribute_unused; int __cmp; \ 139 __arch_compare_and_exchange_xxx_8_int(mem, new, old, rel, acq); \ 140 !__cmp; }) 141 142 #define __arch_compare_and_exchange_bool_16_int(mem, new, old, rel, acq) \ 143 ({ __typeof (*mem) __prev attribute_unused; int __cmp; \ 144 __arch_compare_and_exchange_xxx_16_int(mem, new, old, rel, acq); \ 145 !__cmp; }) 146 147 #define __arch_compare_and_exchange_bool_32_int(mem, new, old, rel, acq) \ 148 ({ __typeof (*mem) __prev attribute_unused; int __cmp; \ 149 __arch_compare_and_exchange_xxx_32_int(mem, new, old, rel, acq); \ 150 !__cmp; }) 151 152 #define __arch_compare_and_exchange_bool_64_int(mem, new, old, rel, acq) \ 153 ({ __typeof (*mem) __prev attribute_unused; int __cmp; \ 154 __arch_compare_and_exchange_xxx_64_int(mem, new, old, rel, acq); \ 155 !__cmp; }) 156 157 /* For all "val" routines, return the old value whether exchange 158 successful or not. */ 159 160 #define __arch_compare_and_exchange_val_8_int(mem, new, old, rel, acq) \ 161 ({ __typeof (*mem) __prev attribute_unused; int __cmp attribute_unused; \ 162 __arch_compare_and_exchange_xxx_8_int(mem, new, old, rel, acq); \ 163 (__typeof (*mem))__prev; }) 164 165 #define __arch_compare_and_exchange_val_16_int(mem, new, old, rel, acq) \ 166 ({ __typeof (*mem) __prev attribute_unused; int __cmp attribute_unused; \ 167 __arch_compare_and_exchange_xxx_16_int(mem, new, old, rel, acq); \ 168 (__typeof (*mem))__prev; }) 169 170 #define __arch_compare_and_exchange_val_32_int(mem, new, old, rel, acq) \ 171 ({ __typeof (*mem) __prev attribute_unused; int __cmp attribute_unused; \ 172 __arch_compare_and_exchange_xxx_32_int(mem, new, old, rel, acq); \ 173 (__typeof (*mem))__prev; }) 174 175 #define __arch_compare_and_exchange_val_64_int(mem, new, old, rel, acq) \ 176 ({ __typeof (*mem) __prev attribute_unused; int __cmp attribute_unused; \ 177 __arch_compare_and_exchange_xxx_64_int(mem, new, old, rel, acq); \ 178 (__typeof (*mem))__prev; }) 179 180 /* Compare and exchange with "acquire" semantics, ie barrier after. */ 181 182 #define atomic_compare_and_exchange_bool_acq(mem, new, old) \ 183 __atomic_bool_bysize (__arch_compare_and_exchange_bool, int, \ 184 mem, new, old, "", MIPS_SYNC_STR) 185 186 #define atomic_compare_and_exchange_val_acq(mem, new, old) \ 187 __atomic_val_bysize (__arch_compare_and_exchange_val, int, \ 188 mem, new, old, "", MIPS_SYNC_STR) 189 190 /* Compare and exchange with "release" semantics, ie barrier before. */ 191 192 #define atomic_compare_and_exchange_bool_rel(mem, new, old) \ 193 __atomic_bool_bysize (__arch_compare_and_exchange_bool, int, \ 194 mem, new, old, MIPS_SYNC_STR, "") 195 196 #define atomic_compare_and_exchange_val_rel(mem, new, old) \ 197 __atomic_val_bysize (__arch_compare_and_exchange_val, int, \ 198 mem, new, old, MIPS_SYNC_STR, "") 199 200 201 202 /* Atomic exchange (without compare). */ 203 204 #define __arch_exchange_xxx_8_int(mem, newval, rel, acq) \ 205 (abort (), 0) 206 207 #define __arch_exchange_xxx_16_int(mem, newval, rel, acq) \ 208 (abort (), 0) 209 210 #define __arch_exchange_xxx_32_int(mem, newval, rel, acq) \ 211 ({ __typeof (*mem) __prev; int __cmp; \ 212 __asm__ __volatile__ ("\n" \ 213 ".set push\n\t" \ 214 MIPS_PUSH_MIPS2 \ 215 rel "\n" \ 216 "1:\t" \ 217 "ll %0,%4\n\t" \ 218 "move %1,%3\n\t" \ 219 "sc %1,%2\n\t" \ 220 R10K_BEQZ_INSN" %1,1b\n" \ 221 acq "\n\t" \ 222 ".set pop\n" \ 223 "2:\n\t" \ 224 : "=&r" (__prev), "=&r" (__cmp), "=m" (*mem) \ 225 : "r" (newval), "m" (*mem) \ 226 : "memory"); \ 227 __prev; }) 228 229 #if _MIPS_SIM == _ABIO32 230 /* We can't do an atomic 64-bit operation in O32. */ 231 #define __arch_exchange_xxx_64_int(mem, newval, rel, acq) \ 232 (abort (), 0) 233 #else 234 #define __arch_exchange_xxx_64_int(mem, newval, rel, acq) \ 235 ({ __typeof (*mem) __prev; int __cmp; \ 236 __asm__ __volatile__ ("\n" \ 237 ".set push\n\t" \ 238 MIPS_PUSH_MIPS2 \ 239 rel "\n" \ 240 "1:\n" \ 241 "lld %0,%4\n\t" \ 242 "move %1,%3\n\t" \ 243 "scd %1,%2\n\t" \ 244 R10K_BEQZ_INSN" %1,1b\n" \ 245 acq "\n\t" \ 246 ".set pop\n" \ 247 "2:\n\t" \ 248 : "=&r" (__prev), "=&r" (__cmp), "=m" (*mem) \ 249 : "r" (newval), "m" (*mem) \ 250 : "memory"); \ 251 __prev; }) 252 #endif 253 254 #define atomic_exchange_acq(mem, value) \ 255 __atomic_val_bysize (__arch_exchange_xxx, int, mem, value, "", MIPS_SYNC_STR) 256 257 #define atomic_exchange_rel(mem, value) \ 258 __atomic_val_bysize (__arch_exchange_xxx, int, mem, value, MIPS_SYNC_STR, "") 259 260 261 /* Atomically add value and return the previous (unincremented) value. */ 262 263 #define __arch_exchange_and_add_8_int(mem, newval, rel, acq) \ 264 (abort (), (__typeof(*mem)) 0) 265 266 #define __arch_exchange_and_add_16_int(mem, newval, rel, acq) \ 267 (abort (), (__typeof(*mem)) 0) 268 269 #define __arch_exchange_and_add_32_int(mem, value, rel, acq) \ 270 ({ __typeof (*mem) __prev; int __cmp; \ 271 __asm__ __volatile__ ("\n" \ 272 ".set push\n\t" \ 273 MIPS_PUSH_MIPS2 \ 274 rel "\n" \ 275 "1:\t" \ 276 "ll %0,%4\n\t" \ 277 "addu %1,%0,%3\n\t" \ 278 "sc %1,%2\n\t" \ 279 R10K_BEQZ_INSN" %1,1b\n" \ 280 acq "\n\t" \ 281 ".set pop\n" \ 282 "2:\n\t" \ 283 : "=&r" (__prev), "=&r" (__cmp), "=m" (*mem) \ 284 : "r" (value), "m" (*mem) \ 285 : "memory"); \ 286 __prev; }) 287 288 #if _MIPS_SIM == _ABIO32 289 /* We can't do an atomic 64-bit operation in O32. */ 290 #define __arch_exchange_and_add_64_int(mem, value, rel, acq) \ 291 (abort (), (__typeof(*mem)) 0) 292 #else 293 #define __arch_exchange_and_add_64_int(mem, value, rel, acq) \ 294 ({ __typeof (*mem) __prev; int __cmp; \ 295 __asm__ __volatile__ ( \ 296 ".set push\n\t" \ 297 MIPS_PUSH_MIPS2 \ 298 rel "\n" \ 299 "1:\t" \ 300 "lld %0,%4\n\t" \ 301 "daddu %1,%0,%3\n\t" \ 302 "scd %1,%2\n\t" \ 303 R10K_BEQZ_INSN" %1,1b\n" \ 304 acq "\n\t" \ 305 ".set pop\n" \ 306 "2:\n\t" \ 307 : "=&r" (__prev), "=&r" (__cmp), "=m" (*mem) \ 308 : "r" (value), "m" (*mem) \ 309 : "memory"); \ 310 __prev; }) 311 #endif 312 313 /* ??? Barrier semantics for atomic_exchange_and_add appear to be 314 undefined. Use full barrier for now, as that's safe. */ 315 #define atomic_exchange_and_add(mem, value) \ 316 __atomic_val_bysize (__arch_exchange_and_add, int, mem, value, \ 317 MIPS_SYNC_STR, MIPS_SYNC_STR) 318 319 /* TODO: More atomic operations could be implemented efficiently; only the 320 basic requirements are done. */ 321 322 #define atomic_full_barrier() \ 323 __asm__ __volatile__ (".set push\n\t" \ 324 MIPS_PUSH_MIPS2 \ 325 MIPS_SYNC_STR "\n\t" \ 326 ".set pop" : : : "memory") 327 328 #endif /* bits/atomic.h */ 329