1 #ifndef JEMALLOC_INTERNAL_TSD_TYPES_H 2 #define JEMALLOC_INTERNAL_TSD_TYPES_H 3 4 /* Maximum number of malloc_tsd users with cleanup functions. */ 5 #define MALLOC_TSD_CLEANUPS_MAX 2 6 7 typedef bool (*malloc_tsd_cleanup_t)(void); 8 9 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 10 !defined(_WIN32)) 11 typedef struct tsd_init_block_s tsd_init_block_t; 12 typedef struct tsd_init_head_s tsd_init_head_t; 13 #endif 14 15 typedef struct tsd_s tsd_t; 16 typedef struct tsdn_s tsdn_t; 17 18 #define TSDN_NULL ((tsdn_t *)0) 19 20 typedef enum { 21 tsd_state_uninitialized, 22 tsd_state_nominal, 23 tsd_state_purgatory, 24 tsd_state_reincarnated 25 } tsd_state_t; 26 27 /* 28 * TLS/TSD-agnostic macro-based implementation of thread-specific data. There 29 * are five macros that support (at least) three use cases: file-private, 30 * library-private, and library-private inlined. Following is an example 31 * library-private tsd variable: 32 * 33 * In example.h: 34 * typedef struct { 35 * int x; 36 * int y; 37 * } example_t; 38 * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) 39 * malloc_tsd_types(example_, example_t) 40 * malloc_tsd_protos(, example_, example_t) 41 * malloc_tsd_externs(example_, example_t) 42 * In example.c: 43 * malloc_tsd_data(, example_, example_t, EX_INITIALIZER) 44 * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER, 45 * example_tsd_cleanup) 46 * 47 * The result is a set of generated functions, e.g.: 48 * 49 * bool example_tsd_boot(void) {...} 50 * bool example_tsd_booted_get(void) {...} 51 * example_t *example_tsd_get(bool init) {...} 52 * void example_tsd_set(example_t *val) {...} 53 * 54 * Note that all of the functions deal in terms of (a_type *) rather than 55 * (a_type) so that it is possible to support non-pointer types (unlike 56 * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is 57 * cast to (void *). This means that the cleanup function needs to cast the 58 * function argument to (a_type *), then dereference the resulting pointer to 59 * access fields, e.g. 60 * 61 * void 62 * example_tsd_cleanup(void *arg) 63 * { 64 * example_t *example = (example_t *)arg; 65 * 66 * example->x = 42; 67 * [...] 68 * if ([want the cleanup function to be called again]) 69 * example_tsd_set(example); 70 * } 71 * 72 * If example_tsd_set() is called within example_tsd_cleanup(), it will be 73 * called again. This is similar to how pthreads TSD destruction works, except 74 * that pthreads only calls the cleanup function again if the value was set to 75 * non-NULL. 76 */ 77 78 /* malloc_tsd_types(). */ 79 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 80 #define malloc_tsd_types(a_name, a_type) 81 #elif (defined(JEMALLOC_TLS)) 82 #define malloc_tsd_types(a_name, a_type) 83 #elif (defined(_WIN32)) 84 #define malloc_tsd_types(a_name, a_type) \ 85 typedef struct { \ 86 bool initialized; \ 87 a_type val; \ 88 } a_name##tsd_wrapper_t; 89 #else 90 #define malloc_tsd_types(a_name, a_type) \ 91 typedef struct { \ 92 bool initialized; \ 93 a_type val; \ 94 } a_name##tsd_wrapper_t; 95 #endif 96 97 /* malloc_tsd_protos(). */ 98 #define malloc_tsd_protos(a_attr, a_name, a_type) \ 99 a_attr bool \ 100 a_name##tsd_boot0(void); \ 101 a_attr void \ 102 a_name##tsd_boot1(void); \ 103 a_attr bool \ 104 a_name##tsd_boot(void); \ 105 a_attr bool \ 106 a_name##tsd_booted_get(void); \ 107 a_attr a_type * \ 108 a_name##tsd_get(bool init); \ 109 a_attr void \ 110 a_name##tsd_set(a_type *val); 111 112 /* malloc_tsd_externs(). */ 113 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 114 #define malloc_tsd_externs(a_name, a_type) \ 115 extern __thread a_type a_name##tsd_tls; \ 116 extern __thread bool a_name##tsd_initialized; \ 117 extern bool a_name##tsd_booted; 118 #elif (defined(JEMALLOC_TLS)) 119 #define malloc_tsd_externs(a_name, a_type) \ 120 extern __thread a_type a_name##tsd_tls; \ 121 extern pthread_key_t a_name##tsd_tsd; \ 122 extern bool a_name##tsd_booted; 123 #elif (defined(_WIN32)) 124 #define malloc_tsd_externs(a_name, a_type) \ 125 extern DWORD a_name##tsd_tsd; \ 126 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 127 extern bool a_name##tsd_booted; 128 #else 129 #define malloc_tsd_externs(a_name, a_type) \ 130 extern pthread_key_t a_name##tsd_tsd; \ 131 extern tsd_init_head_t a_name##tsd_init_head; \ 132 extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ 133 extern bool a_name##tsd_booted; 134 #endif 135 136 /* malloc_tsd_data(). */ 137 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 138 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 139 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 140 a_name##tsd_tls = a_initializer; \ 141 a_attr __thread bool JEMALLOC_TLS_MODEL \ 142 a_name##tsd_initialized = false; \ 143 a_attr bool a_name##tsd_booted = false; 144 #elif (defined(JEMALLOC_TLS)) 145 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 146 a_attr __thread a_type JEMALLOC_TLS_MODEL \ 147 a_name##tsd_tls = a_initializer; \ 148 a_attr pthread_key_t a_name##tsd_tsd; \ 149 a_attr bool a_name##tsd_booted = false; 150 #elif (defined(_WIN32)) 151 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 152 a_attr DWORD a_name##tsd_tsd; \ 153 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 154 false, \ 155 a_initializer \ 156 }; \ 157 a_attr bool a_name##tsd_booted = false; 158 #else 159 #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ 160 a_attr pthread_key_t a_name##tsd_tsd; \ 161 a_attr tsd_init_head_t a_name##tsd_init_head = { \ 162 ql_head_initializer(blocks), \ 163 MALLOC_MUTEX_INITIALIZER \ 164 }; \ 165 a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ 166 false, \ 167 a_initializer \ 168 }; \ 169 a_attr bool a_name##tsd_booted = false; 170 #endif 171 172 /* malloc_tsd_funcs(). */ 173 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 174 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 175 a_cleanup) \ 176 /* Initialization/cleanup. */ \ 177 a_attr bool \ 178 a_name##tsd_cleanup_wrapper(void) \ 179 { \ 180 if (a_name##tsd_initialized) { \ 181 a_name##tsd_initialized = false; \ 182 a_cleanup(&a_name##tsd_tls); \ 183 } \ 184 return (a_name##tsd_initialized); \ 185 } \ 186 a_attr bool \ 187 a_name##tsd_boot0(void) \ 188 { \ 189 if (a_cleanup != malloc_tsd_no_cleanup) { \ 190 malloc_tsd_cleanup_register( \ 191 &a_name##tsd_cleanup_wrapper); \ 192 } \ 193 a_name##tsd_booted = true; \ 194 return (false); \ 195 } \ 196 a_attr void \ 197 a_name##tsd_boot1(void) \ 198 { \ 199 /* Do nothing. */ \ 200 } \ 201 a_attr bool \ 202 a_name##tsd_boot(void) \ 203 { \ 204 return (a_name##tsd_boot0()); \ 205 } \ 206 a_attr bool \ 207 a_name##tsd_booted_get(void) \ 208 { \ 209 return (a_name##tsd_booted); \ 210 } \ 211 a_attr bool \ 212 a_name##tsd_get_allocates(void) \ 213 { \ 214 return (false); \ 215 } \ 216 /* Get/set. */ \ 217 a_attr a_type * \ 218 a_name##tsd_get(bool init) \ 219 { \ 220 assert(a_name##tsd_booted); \ 221 return (&a_name##tsd_tls); \ 222 } \ 223 a_attr void \ 224 a_name##tsd_set(a_type *val) \ 225 { \ 226 assert(a_name##tsd_booted); \ 227 if (likely(&a_name##tsd_tls != val)) \ 228 a_name##tsd_tls = (*val); \ 229 if (a_cleanup != malloc_tsd_no_cleanup) \ 230 a_name##tsd_initialized = true; \ 231 } 232 #elif (defined(JEMALLOC_TLS)) 233 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 234 a_cleanup) \ 235 /* Initialization/cleanup. */ \ 236 a_attr bool \ 237 a_name##tsd_boot0(void) \ 238 { \ 239 if (a_cleanup != malloc_tsd_no_cleanup) { \ 240 if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \ 241 0) \ 242 return (true); \ 243 } \ 244 a_name##tsd_booted = true; \ 245 return (false); \ 246 } \ 247 a_attr void \ 248 a_name##tsd_boot1(void) \ 249 { \ 250 /* Do nothing. */ \ 251 } \ 252 a_attr bool \ 253 a_name##tsd_boot(void) \ 254 { \ 255 return (a_name##tsd_boot0()); \ 256 } \ 257 a_attr bool \ 258 a_name##tsd_booted_get(void) \ 259 { \ 260 return (a_name##tsd_booted); \ 261 } \ 262 a_attr bool \ 263 a_name##tsd_get_allocates(void) \ 264 { \ 265 return (false); \ 266 } \ 267 /* Get/set. */ \ 268 a_attr a_type * \ 269 a_name##tsd_get(bool init) \ 270 { \ 271 assert(a_name##tsd_booted); \ 272 return (&a_name##tsd_tls); \ 273 } \ 274 a_attr void \ 275 a_name##tsd_set(a_type *val) \ 276 { \ 277 assert(a_name##tsd_booted); \ 278 if (likely(&a_name##tsd_tls != val)) \ 279 a_name##tsd_tls = (*val); \ 280 if (a_cleanup != malloc_tsd_no_cleanup) { \ 281 if (pthread_setspecific(a_name##tsd_tsd, \ 282 (void *)(&a_name##tsd_tls))) { \ 283 malloc_write("<jemalloc>: Error" \ 284 " setting TSD for "#a_name"\n"); \ 285 if (opt_abort) \ 286 abort(); \ 287 } \ 288 } \ 289 } 290 #elif (defined(_WIN32)) 291 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 292 a_cleanup) \ 293 /* Initialization/cleanup. */ \ 294 a_attr bool \ 295 a_name##tsd_cleanup_wrapper(void) \ 296 { \ 297 DWORD error = GetLastError(); \ 298 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 299 TlsGetValue(a_name##tsd_tsd); \ 300 SetLastError(error); \ 301 \ 302 if (wrapper == NULL) \ 303 return (false); \ 304 if (a_cleanup != malloc_tsd_no_cleanup && \ 305 wrapper->initialized) { \ 306 wrapper->initialized = false; \ 307 a_cleanup(&wrapper->val); \ 308 if (wrapper->initialized) { \ 309 /* Trigger another cleanup round. */ \ 310 return (true); \ 311 } \ 312 } \ 313 malloc_tsd_dalloc(wrapper); \ 314 return (false); \ 315 } \ 316 a_attr void \ 317 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 318 { \ 319 if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \ 320 malloc_write("<jemalloc>: Error setting" \ 321 " TSD for "#a_name"\n"); \ 322 abort(); \ 323 } \ 324 } \ 325 a_attr a_name##tsd_wrapper_t * \ 326 a_name##tsd_wrapper_get(bool init) \ 327 { \ 328 DWORD error = GetLastError(); \ 329 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 330 TlsGetValue(a_name##tsd_tsd); \ 331 SetLastError(error); \ 332 \ 333 if (init && unlikely(wrapper == NULL)) { \ 334 wrapper = (a_name##tsd_wrapper_t *) \ 335 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 336 if (wrapper == NULL) { \ 337 malloc_write("<jemalloc>: Error allocating" \ 338 " TSD for "#a_name"\n"); \ 339 abort(); \ 340 } else { \ 341 wrapper->initialized = false; \ 342 wrapper->val = a_initializer; \ 343 } \ 344 a_name##tsd_wrapper_set(wrapper); \ 345 } \ 346 return (wrapper); \ 347 } \ 348 a_attr bool \ 349 a_name##tsd_boot0(void) \ 350 { \ 351 a_name##tsd_tsd = TlsAlloc(); \ 352 if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \ 353 return (true); \ 354 if (a_cleanup != malloc_tsd_no_cleanup) { \ 355 malloc_tsd_cleanup_register( \ 356 &a_name##tsd_cleanup_wrapper); \ 357 } \ 358 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 359 a_name##tsd_booted = true; \ 360 return (false); \ 361 } \ 362 a_attr void \ 363 a_name##tsd_boot1(void) \ 364 { \ 365 a_name##tsd_wrapper_t *wrapper; \ 366 wrapper = (a_name##tsd_wrapper_t *) \ 367 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 368 if (wrapper == NULL) { \ 369 malloc_write("<jemalloc>: Error allocating" \ 370 " TSD for "#a_name"\n"); \ 371 abort(); \ 372 } \ 373 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 374 sizeof(a_name##tsd_wrapper_t)); \ 375 a_name##tsd_wrapper_set(wrapper); \ 376 } \ 377 a_attr bool \ 378 a_name##tsd_boot(void) \ 379 { \ 380 if (a_name##tsd_boot0()) \ 381 return (true); \ 382 a_name##tsd_boot1(); \ 383 return (false); \ 384 } \ 385 a_attr bool \ 386 a_name##tsd_booted_get(void) \ 387 { \ 388 return (a_name##tsd_booted); \ 389 } \ 390 a_attr bool \ 391 a_name##tsd_get_allocates(void) \ 392 { \ 393 return (true); \ 394 } \ 395 /* Get/set. */ \ 396 a_attr a_type * \ 397 a_name##tsd_get(bool init) \ 398 { \ 399 a_name##tsd_wrapper_t *wrapper; \ 400 \ 401 assert(a_name##tsd_booted); \ 402 wrapper = a_name##tsd_wrapper_get(init); \ 403 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \ 404 return (NULL); \ 405 return (&wrapper->val); \ 406 } \ 407 a_attr void \ 408 a_name##tsd_set(a_type *val) \ 409 { \ 410 a_name##tsd_wrapper_t *wrapper; \ 411 \ 412 assert(a_name##tsd_booted); \ 413 wrapper = a_name##tsd_wrapper_get(true); \ 414 if (likely(&wrapper->val != val)) \ 415 wrapper->val = *(val); \ 416 if (a_cleanup != malloc_tsd_no_cleanup) \ 417 wrapper->initialized = true; \ 418 } 419 #else 420 #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ 421 a_cleanup) \ 422 /* Initialization/cleanup. */ \ 423 a_attr void \ 424 a_name##tsd_cleanup_wrapper(void *arg) \ 425 { \ 426 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \ 427 \ 428 if (a_cleanup != malloc_tsd_no_cleanup && \ 429 wrapper->initialized) { \ 430 wrapper->initialized = false; \ 431 a_cleanup(&wrapper->val); \ 432 if (wrapper->initialized) { \ 433 /* Trigger another cleanup round. */ \ 434 if (pthread_setspecific(a_name##tsd_tsd, \ 435 (void *)wrapper)) { \ 436 malloc_write("<jemalloc>: Error" \ 437 " setting TSD for "#a_name"\n"); \ 438 if (opt_abort) \ 439 abort(); \ 440 } \ 441 return; \ 442 } \ 443 } \ 444 malloc_tsd_dalloc(wrapper); \ 445 } \ 446 a_attr void \ 447 a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ 448 { \ 449 if (pthread_setspecific(a_name##tsd_tsd, \ 450 (void *)wrapper)) { \ 451 malloc_write("<jemalloc>: Error setting" \ 452 " TSD for "#a_name"\n"); \ 453 abort(); \ 454 } \ 455 } \ 456 a_attr a_name##tsd_wrapper_t * \ 457 a_name##tsd_wrapper_get(bool init) \ 458 { \ 459 a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ 460 pthread_getspecific(a_name##tsd_tsd); \ 461 \ 462 if (init && unlikely(wrapper == NULL)) { \ 463 tsd_init_block_t block; \ 464 wrapper = (a_name##tsd_wrapper_t *) \ 465 tsd_init_check_recursion(&a_name##tsd_init_head, \ 466 &block); \ 467 if (wrapper) \ 468 return (wrapper); \ 469 wrapper = (a_name##tsd_wrapper_t *) \ 470 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 471 block.data = (void *)wrapper; \ 472 if (wrapper == NULL) { \ 473 malloc_write("<jemalloc>: Error allocating" \ 474 " TSD for "#a_name"\n"); \ 475 abort(); \ 476 } else { \ 477 wrapper->initialized = false; \ 478 wrapper->val = a_initializer; \ 479 } \ 480 a_name##tsd_wrapper_set(wrapper); \ 481 tsd_init_finish(&a_name##tsd_init_head, &block); \ 482 } \ 483 return (wrapper); \ 484 } \ 485 a_attr bool \ 486 a_name##tsd_boot0(void) \ 487 { \ 488 if (pthread_key_create(&a_name##tsd_tsd, \ 489 a_name##tsd_cleanup_wrapper) != 0) \ 490 return (true); \ 491 a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ 492 a_name##tsd_booted = true; \ 493 return (false); \ 494 } \ 495 a_attr void \ 496 a_name##tsd_boot1(void) \ 497 { \ 498 a_name##tsd_wrapper_t *wrapper; \ 499 wrapper = (a_name##tsd_wrapper_t *) \ 500 malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ 501 if (wrapper == NULL) { \ 502 malloc_write("<jemalloc>: Error allocating" \ 503 " TSD for "#a_name"\n"); \ 504 abort(); \ 505 } \ 506 memcpy(wrapper, &a_name##tsd_boot_wrapper, \ 507 sizeof(a_name##tsd_wrapper_t)); \ 508 a_name##tsd_wrapper_set(wrapper); \ 509 } \ 510 a_attr bool \ 511 a_name##tsd_boot(void) \ 512 { \ 513 if (a_name##tsd_boot0()) \ 514 return (true); \ 515 a_name##tsd_boot1(); \ 516 return (false); \ 517 } \ 518 a_attr bool \ 519 a_name##tsd_booted_get(void) \ 520 { \ 521 return (a_name##tsd_booted); \ 522 } \ 523 a_attr bool \ 524 a_name##tsd_get_allocates(void) \ 525 { \ 526 return (true); \ 527 } \ 528 /* Get/set. */ \ 529 a_attr a_type * \ 530 a_name##tsd_get(bool init) \ 531 { \ 532 a_name##tsd_wrapper_t *wrapper; \ 533 \ 534 assert(a_name##tsd_booted); \ 535 wrapper = a_name##tsd_wrapper_get(init); \ 536 if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \ 537 return (NULL); \ 538 return (&wrapper->val); \ 539 } \ 540 a_attr void \ 541 a_name##tsd_set(a_type *val) \ 542 { \ 543 a_name##tsd_wrapper_t *wrapper; \ 544 \ 545 assert(a_name##tsd_booted); \ 546 wrapper = a_name##tsd_wrapper_get(true); \ 547 if (likely(&wrapper->val != val)) \ 548 wrapper->val = *(val); \ 549 if (a_cleanup != malloc_tsd_no_cleanup) \ 550 wrapper->initialized = true; \ 551 } 552 #endif 553 554 #endif /* JEMALLOC_INTERNAL_TSD_TYPES_H */ 555