1 /* SPDX-License-Identifier: BSL-1.0 */ 2 /* Copyright (c) 2017 Arm Limited. All rights reserved. 3 4 Boost Software License - Version 1.0 - August 17th, 2003 5 6 Permission is hereby granted, free of charge, to any person or organization 7 obtaining a copy of the software and accompanying documentation covered by 8 this license (the "Software") to use, reproduce, display, distribute, 9 execute, and transmit the Software, and to prepare derivative works of the 10 Software, and to permit third-parties to whom the Software is furnished to do 11 so, all subject to the following: 12 13 The copyright notices in the Software and this entire statement, including 14 the above license grant, this restriction and the following disclaimer, must 15 be included in all copies of the Software, in whole or in part, and all 16 derivative works of the Software, unless such copies or derivative works are 17 solely in the form of machine-executable object code generated by a source 18 language processor. 19 20 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 23 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR 24 ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 25 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 26 DEALINGS IN THE SOFTWARE. */ 27 28 29 #ifdef __HAVE_LOAD_NO_SPECULATE 30 #define load_no_speculate(__ptr, __low, __high) \ 31 (__extension__ ({ \ 32 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 33 __builtin_load_no_speculate (__ptr_once, __low, __high, \ 34 0, __ptr_once); \ 35 })) 36 37 #define load_no_speculate_fail(__ptr, __low, __high, __failval) \ 38 (__extension__ ({ \ 39 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 40 __builtin_load_no_speculate (__ptr_once, __low, __high, \ 41 __failval, __ptr_once); \ 42 })) 43 44 #define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \ 45 (__builtin_load_no_speculate (__ptr, __low, __high, __failval, __cmpptr)) 46 47 #else 48 49 #ifdef __GNUC__ 50 #define __UNUSED __attribute__((unused)) 51 #else 52 #define __UNUSED 53 #endif 54 55 #ifdef __aarch64__ 56 57 #define __load_no_speculate1(__ptr, __low, __high, __failval, \ 58 __cmpptr, __w, __sz) \ 59 (__extension__ ({ \ 60 __typeof__ (0 + (*(__ptr))) __nln_val; \ 61 /* This typecasting is required to ensure correct handling of upper \ 62 bits of failval, to ensure a clean return from the CSEL below. */ \ 63 __typeof__(*(__ptr)) __fv \ 64 = (__typeof__(*(__ptr)))(unsigned long long) (__failval); \ 65 /* If __high is explicitly NULL, we must not emit the \ 66 upper-bound comparison. We need to cast __high to an \ 67 unsigned long long before handing it to __builtin_constant_p to \ 68 ensure that clang/llvm correctly detects NULL as a constant if it \ 69 is defined as (void*) 0. */ \ 70 if (__builtin_constant_p ((unsigned long long)__high) \ 71 && __high == ((void *)0)) \ 72 { \ 73 __asm__ volatile ( \ 74 "cmp\t%[__c], %[__l]\n\t" \ 75 "bcc\t.ns%=\n\t" \ 76 "ldr" __sz "\t%" __w "[__v], %[__p]\n" \ 77 ".ns%=:\n\t" \ 78 "csel\t%" __w "[__v], %" __w "[__v], %" __w "[__f], cs\n\t" \ 79 "hint\t#0x14 // CSDB" \ 80 /* The value we have loaded, or failval if the condition check \ 81 fails. */ \ 82 : [__v] "=&r" (__nln_val) \ 83 /* The pointer we wish to use for comparisons, and the low and \ 84 high bounds to use in that comparison. Note that this need \ 85 not be the same as the pointer from which we will load. */ \ 86 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 87 /* The memory location from which we will load. */ \ 88 [__p] "m" (*(__ptr)), \ 89 /* The value to return if the condition check fails. */ \ 90 [__f] "rZ" (__fv) \ 91 /* We always clobber the condition codes. */ \ 92 : "cc"); \ 93 } \ 94 else \ 95 { \ 96 __asm__ volatile ( \ 97 "cmp\t%[__c], %[__l]\n\t" \ 98 "ccmp\t%[__c], %[__h], 2, cs\n\t" \ 99 "bcs\t.ns%=\n\t" \ 100 "ldr" __sz "\t%" __w "[__v], %[__p]\n" \ 101 ".ns%=:\n\t" \ 102 "csel\t%" __w "[__v], %" __w "[__v], %" __w "[__f], cc\n\t" \ 103 "hint\t#0x14 // CSDB" \ 104 /* The value we have loaded, or failval if the condition check \ 105 fails. */ \ 106 : [__v] "=&r" (__nln_val) \ 107 /* The pointer we wish to use for comparisons, and the low and \ 108 high bounds to use in that comparison. Note that this need \ 109 not be the same as the pointer from which we will load. */ \ 110 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 111 [__h] "r" (__high), \ 112 /* The memory location from which we will load. */ \ 113 [__p] "m" (*(__ptr)), \ 114 /* The value to return if the condition check fails. */ \ 115 [__f] "rZ" (__fv) \ 116 /* We always clobber the condition codes. */ \ 117 : "cc"); \ 118 } \ 119 (__typeof__ (*(__ptr))) __nln_val; \ 120 })) 121 122 #define __load_no_speculate(__ptr, __low, __high, __failval, __cmpptr) \ 123 (__extension__ ({ \ 124 __typeof__ (0 + *(__ptr)) __nl_val; \ 125 \ 126 switch (sizeof(*(__ptr))) { \ 127 case 1: \ 128 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 129 __failval, __cmpptr, "w", "b"); \ 130 break; \ 131 case 2: \ 132 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 133 __failval, __cmpptr, "w", "h"); \ 134 break; \ 135 case 4: \ 136 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 137 __failval, __cmpptr, "w", ""); \ 138 break; \ 139 case 8: \ 140 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 141 __failval, __cmpptr, "x", ""); \ 142 break; \ 143 default: \ 144 { \ 145 char __static_assert_no_speculate_load_size_too_big \ 146 [sizeof (__nl_val) > 8 ? -1 : 1] __UNUSED; \ 147 break; \ 148 } \ 149 } \ 150 \ 151 (__typeof__ (*(__ptr))) __nl_val; \ 152 })) 153 154 #define load_no_speculate(__ptr, __low, __high) \ 155 (__extension__ ({ \ 156 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 157 __load_no_speculate (__ptr_once, __low, __high, 0, __ptr_once); \ 158 })) 159 160 #define load_no_speculate_fail(__ptr, __low, __high, __failval) \ 161 (__extension__ ({ \ 162 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 163 __load_no_speculate (__ptr_once, __low, __high, \ 164 __failval, __ptr_once); \ 165 })) 166 167 #define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \ 168 (__load_no_speculate (__ptr, __low, __high, __failval, __cmpptr)) 169 170 /* AArch32 support for ARM and Thumb-2. Thumb-1 is not supported. */ 171 #elif defined (__ARM_32BIT_STATE) && (defined (__thumb2__) || !defined (__thumb__)) 172 #ifdef __thumb2__ 173 /* Thumb2 case. */ 174 175 #define __load_no_speculate1(__ptr, __low, __high, __failval, \ 176 __cmpptr, __sz) \ 177 (__extension__ ({ \ 178 __typeof__ (0 + *(__ptr)) __nln_val; \ 179 __typeof__(*(__ptr)) __fv \ 180 = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ 181 /* If __high is explicitly NULL, we must not emit the \ 182 upper-bound comparison. We need to cast __high to an \ 183 unsigned long before handing it to __builtin_constant_p to \ 184 ensure that clang/llvm correctly detects NULL as a constant if it \ 185 is defined as (void*) 0. */ \ 186 if (__builtin_constant_p ((unsigned long)__high) \ 187 && __high == ((void *)0)) \ 188 { \ 189 __asm__ volatile ( \ 190 ".syntax unified\n\t" \ 191 "cmp\t%[__c], %[__l]\n\t" \ 192 "bcc\t.ns%=\n\t" \ 193 "ldr" __sz "\t%[__v], %[__p]\n" \ 194 ".ns%=:\n\t" \ 195 "it\tcc\n\t" \ 196 "movcc\t%[__v], %[__f]\n\t" \ 197 ".inst.n 0xf3af\t@ CSDB\n\t" \ 198 ".inst.n 0x8014\t@ CSDB" \ 199 /* The value we have loaded, or failval if the condition check \ 200 fails. */ \ 201 : [__v] "=&l" (__nln_val) \ 202 /* The pointer we wish to use for comparisons, and the low and \ 203 high bounds to use in that comparison. Note that this need \ 204 not be the same as the pointer from which we will load. */ \ 205 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 206 /* The memory location from which we will load. */ \ 207 [__p] "m" (*(__ptr)), \ 208 /* The value to return if the condition check fails. */ \ 209 [__f] "r" (__fv) \ 210 /* We always clobber the condition codes. */ \ 211 : "cc"); \ 212 } \ 213 else \ 214 { \ 215 __asm__ volatile ( \ 216 ".syntax unified\n\t" \ 217 "cmp\t%[__c], %[__l]\n\t" \ 218 "it\tcs\n\t" \ 219 "cmpcs\t%[__h], %[__c]\n\t" \ 220 "bls\t.ns%=\n\t" \ 221 "ldr" __sz "\t%[__v], %[__p]\n" \ 222 ".ns%=:\n\t" \ 223 "it\tls\n\t" \ 224 "movls\t%[__v], %[__f]\n\t" \ 225 ".inst.n 0xf3af\t@ CSDB\n\t" \ 226 ".inst.n 0x8014\t@ CSDB" \ 227 /* The value we have loaded, or failval if the condition check \ 228 fails. */ \ 229 : [__v] "=&l" (__nln_val) \ 230 /* The pointer we wish to use for comparisons, and the low and \ 231 high bounds to use in that comparison. Note that this need \ 232 not be the same as the pointer from which we will load. */ \ 233 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 234 [__h] "r" (__high), \ 235 /* The memory location from which we will load. */ \ 236 [__p] "m" (*(__ptr)), \ 237 /* The value to return if the condition check fails. */ \ 238 [__f] "r" (__fv) \ 239 /* We always clobber the condition codes. */ \ 240 : "cc"); \ 241 } \ 242 (__typeof__ (*(__ptr))) __nln_val; \ 243 })) \ 244 \ 245 /* Double-word version. */ 246 #define __load_no_speculate2(__ptr, __low, __high, __failval, \ 247 __cmpptr) \ 248 (__extension__ ({ \ 249 __typeof__ (0 + *(__ptr)) __nln_val; \ 250 __typeof__(*(__ptr)) __fv \ 251 = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ 252 /* If __high is explicitly NULL, we must not emit the \ 253 upper-bound comparison. We need to cast __high to an \ 254 unsigned long before handing it to __builtin_constant_p to \ 255 ensure that clang/llvm correctly detects NULL as a constant if it \ 256 is defined as (void*) 0. */ \ 257 if (__builtin_constant_p ((unsigned long)__high) \ 258 && __high == ((void *)0)) \ 259 { \ 260 __asm__ volatile ( \ 261 ".syntax unified\n\t" \ 262 "cmp\t%[__c], %[__l]\n\t" \ 263 "bcc\t.ns%=\n\t" \ 264 "ldr\t%Q[__v], [%[__p]]\n\t" \ 265 "ldr\t%R[__v], [%[__p], #4]\n" \ 266 ".ns%=:\n\t" \ 267 "it\tcc\n\t" \ 268 "movcc\t%Q[__v], %Q[__f]\n\t" \ 269 "it\tcc\n\t" \ 270 "movcc\t%R[__v], %R[__f]\n\t" \ 271 ".inst.n 0xf3af\t@ CSDB\n\t" \ 272 ".inst.n 0x8014\t@ CSDB" \ 273 /* The value we have loaded, or failval if the condition check \ 274 fails. */ \ 275 : [__v] "=&l" (__nln_val) \ 276 /* The pointer we wish to use for comparisons, and the low and \ 277 high bounds to use in that comparison. Note that this need \ 278 not be the same as the pointer from which we will load. */ \ 279 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 280 /* The memory location from which we will load. */ \ 281 [__p] "r" (__ptr), \ 282 /* The value to return if the condition check fails. */ \ 283 [__f] "r" (__fv) \ 284 /* We always clobber the condition codes. */ \ 285 : "cc"); \ 286 } \ 287 else \ 288 { \ 289 __asm__ volatile ( \ 290 ".syntax unified\n\t" \ 291 "cmp\t%[__c], %[__l]\n\t" \ 292 "it\tcs\n\t" \ 293 "cmpcs\t%[__h], %[__c]\n\t" \ 294 "bls\t.ns%=\n\t" \ 295 "ldr\t%Q[__v], [%[__p]]\n\t" \ 296 "ldr\t%R[__v], [%[__p], #4]\n" \ 297 ".ns%=:\n\t" \ 298 "it\tls\n\t" \ 299 "movls\t%Q[__v], %Q[__f]\n\t" \ 300 "it\tls\n\t" \ 301 "movls\t%R[__v], %R[__f]\n\t" \ 302 ".inst.n 0xf3af\t@ CSDB\n\t" \ 303 ".inst.n 0x8014\t@ CSDB" \ 304 /* The value we have loaded, or failval if the condition check \ 305 fails. */ \ 306 : [__v] "=&l" (__nln_val) \ 307 /* The pointer we wish to use for comparisons, and the low and \ 308 high bounds to use in that comparison. Note that this need \ 309 not be the same as the pointer from which we will load. */ \ 310 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 311 [__h] "r" (__high), \ 312 /* The memory location from which we will load. */ \ 313 [__p] "r" (__ptr), \ 314 /* The value to return if the condition check fails. */ \ 315 [__f] "r" (__fv) \ 316 /* We always clobber the condition codes. */ \ 317 : "cc"); \ 318 } \ 319 (__typeof__ (*(__ptr))) __nln_val; \ 320 })) 321 322 #else 323 /* ARM case. */ 324 325 #define __load_no_speculate1(__ptr, __low, __high, __failval, \ 326 __cmpptr, __sz) \ 327 (__extension__ ({ \ 328 __typeof__ (0 + *(__ptr)) __nln_val; \ 329 __typeof__(*(__ptr)) __fv \ 330 = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ 331 /* If __high is explicitly NULL, we must not emit the \ 332 upper-bound comparison. We need to cast __high to an \ 333 unsigned long before handing it to __builtin_constant_p to \ 334 ensure that clang/llvm correctly detects NULL as a constant if it \ 335 is defined as (void*) 0. */ \ 336 if (__builtin_constant_p ((unsigned long)__high) \ 337 && __high == ((void *)0)) \ 338 { \ 339 __asm__ volatile ( \ 340 ".syntax unified\n\t" \ 341 "cmp\t%[__c], %[__l]\n\t" \ 342 "ldr" __sz "cs\t%[__v], %[__p]\n\t" \ 343 "movcc\t%[__v], %[__f]\n\t" \ 344 ".inst 0xe320f014\t@ CSDB" \ 345 /* The value we have loaded, or failval if the condition check \ 346 fails. */ \ 347 : [__v] "=&r" (__nln_val) \ 348 /* The pointer we wish to use for comparisons, and the low and \ 349 high bounds to use in that comparison. Note that this need \ 350 not be the same as the pointer from which we will load. */ \ 351 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 352 /* The memory location from which we will load. */ \ 353 [__p] "m" (*(__ptr)), \ 354 /* The value to return if the condition check fails. */ \ 355 [__f] "rKI" (__fv) \ 356 /* We always clobber the condition codes. */ \ 357 : "cc"); \ 358 } \ 359 else \ 360 { \ 361 __asm__ volatile ( \ 362 ".syntax unified\n\t" \ 363 "cmp\t%[__c], %[__l]\n\t" \ 364 "cmpcs\t%[__h], %[__c]\n\t" \ 365 "ldr" __sz "hi\t%[__v], %[__p]\n\t" \ 366 "movls\t%[__v], %[__f]\n\t" \ 367 ".inst 0xe320f014\t@ CSDB" \ 368 /* The value we have loaded, or failval if the condition check \ 369 fails. */ \ 370 : [__v] "=&r" (__nln_val) \ 371 /* The pointer we wish to use for comparisons, and the low and \ 372 high bounds to use in that comparison. Note that this need \ 373 not be the same as the pointer from which we will load. */ \ 374 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 375 [__h] "r" (__high), \ 376 /* The memory location from which we will load. */ \ 377 [__p] "m" (*(__ptr)), \ 378 /* The value to return if the condition check fails. */ \ 379 [__f] "rKI" (__fv) \ 380 /* We always clobber the condition codes. */ \ 381 : "cc"); \ 382 } \ 383 (__typeof__ (*(__ptr))) __nln_val; \ 384 })) 385 386 /* Double-word version. */ 387 #define __load_no_speculate2(__ptr, __low, __high, __failval, \ 388 __cmpptr) \ 389 (__extension__ ({ \ 390 __typeof__ (0 + *(__ptr)) __nln_val; \ 391 __typeof__(*(__ptr)) __fv \ 392 = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ 393 /* If __high is explicitly NULL, we must not emit the \ 394 upper-bound comparison. We need to cast __high to an \ 395 unsigned long before handing it to __builtin_constant_p to \ 396 ensure that clang/llvm correctly detects NULL as a constant if it \ 397 is defined as (void*) 0. */ \ 398 if (__builtin_constant_p ((unsigned long)__high) \ 399 && __high == ((void *)0)) \ 400 { \ 401 __asm__ volatile ( \ 402 ".syntax unified\n\t" \ 403 "cmp\t%[__c], %[__l]\n\t" \ 404 "ldrcs\t%Q[__v], [%[__p]]\n\t" \ 405 "ldrcs\t%R[__v], [%[__p], #4]\n\t" \ 406 "movcc\t%Q[__v], %Q[__f]\n\t" \ 407 "movcc\t%R[__v], %R[__f]\n\t" \ 408 ".inst 0xe320f014\t@ CSDB" \ 409 /* The value we have loaded, or failval if the condition check \ 410 fails. */ \ 411 : [__v] "=&r" (__nln_val) \ 412 /* The pointer we wish to use for comparisons, and the low and \ 413 high bounds to use in that comparison. Note that this need \ 414 not be the same as the pointer from which we will load. */ \ 415 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 416 /* The memory location from which we will load. */ \ 417 [__p] "r" (__ptr), \ 418 /* The value to return if the condition check fails. */ \ 419 [__f] "r" (__fv) \ 420 /* We always clobber the condition codes. */ \ 421 : "cc"); \ 422 } \ 423 else \ 424 { \ 425 __asm__ volatile ( \ 426 ".syntax unified\n\t" \ 427 "cmp\t%[__c], %[__l]\n\t" \ 428 "cmpcs\t%[__h], %[__c]\n\t" \ 429 "ldrhi\t%Q[__v], [%[__p]]\n\t" \ 430 "ldrhi\t%R[__v], [%[__p], #4]\n\t" \ 431 "movls\t%Q[__v], %Q[__f]\n\t" \ 432 "movls\t%R[__v], %R[__f]\n\t" \ 433 ".inst 0xe320f014\t@ CSDB" \ 434 /* The value we have loaded, or failval if the condition check \ 435 fails. */ \ 436 : [__v] "=&r" (__nln_val) \ 437 /* The pointer we wish to use for comparisons, and the low and \ 438 high bounds to use in that comparison. Note that this need \ 439 not be the same as the pointer from which we will load. */ \ 440 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 441 [__h] "r" (__high), \ 442 /* The memory location from which we will load. */ \ 443 [__p] "r" (__ptr), \ 444 /* The value to return if the condition check fails. */ \ 445 [__f] "r" (__fv) \ 446 /* We always clobber the condition codes. */ \ 447 : "cc"); \ 448 } \ 449 (__typeof__ (*(__ptr))) __nln_val; \ 450 })) 451 452 #endif // __thumb2__ 453 454 /* Common to ARM and Thumb2. */ 455 456 #define __load_no_speculate(__ptr, __low, __high, __failval, __cmpptr) \ 457 (__extension__ ({ \ 458 __typeof__ (0 + *(__ptr)) __nl_val; \ 459 \ 460 switch (sizeof(*(__ptr))) { \ 461 case 1: \ 462 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 463 __failval, __cmpptr, "b"); \ 464 break; \ 465 case 2: \ 466 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 467 __failval, __cmpptr, "h"); \ 468 break; \ 469 case 4: \ 470 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 471 __failval, __cmpptr, ""); \ 472 break; \ 473 case 8: \ 474 __nl_val = __load_no_speculate2 (__ptr, __low, __high, \ 475 __failval, __cmpptr); \ 476 break; \ 477 default: \ 478 { \ 479 char __static_assert_no_speculate_load_size_too_big \ 480 [sizeof (__nl_val) > 8 ? -1 : 1] __UNUSED; \ 481 break; \ 482 } \ 483 } \ 484 \ 485 (__typeof__ (*(__ptr))) __nl_val; \ 486 })) 487 488 #define load_no_speculate(__ptr, __low, __high) \ 489 (__extension__ ({ \ 490 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 491 __load_no_speculate (__ptr_once, __low, __high, 0, __ptr_once); \ 492 })) 493 494 #define load_no_speculate_fail(__ptr, __low, __high, __failval) \ 495 (__extension__ ({ \ 496 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 497 __load_no_speculate (__ptr_once, __low, __high, \ 498 __failval, __ptr_once); \ 499 })) 500 501 #define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \ 502 (__load_no_speculate (__ptr, __low, __high, __failval, __cmpptr)) 503 504 #else 505 #error "No fallback provided for load_no_speculate" 506 #endif 507 508 #endif 509