Lines Matching refs:tsd
432 arena_bind(tsd_t *tsd, unsigned ind, bool internal) in arena_bind() argument
436 if (!tsd_nominal(tsd)) in arena_bind()
439 arena = arena_get(tsd_tsdn(tsd), ind, false); in arena_bind()
443 tsd_iarena_set(tsd, arena); in arena_bind()
445 tsd_arena_set(tsd, arena); in arena_bind()
449 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) in arena_migrate() argument
453 oldarena = arena_get(tsd_tsdn(tsd), oldind, false); in arena_migrate()
454 newarena = arena_get(tsd_tsdn(tsd), newind, false); in arena_migrate()
457 tsd_arena_set(tsd, newarena); in arena_migrate()
461 arena_unbind(tsd_t *tsd, unsigned ind, bool internal) in arena_unbind() argument
465 arena = arena_get(tsd_tsdn(tsd), ind, false); in arena_unbind()
468 tsd_iarena_set(tsd, NULL); in arena_unbind()
470 tsd_arena_set(tsd, NULL); in arena_unbind()
474 arena_tdata_get_hard(tsd_t *tsd, unsigned ind) in arena_tdata_get_hard() argument
477 arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); in arena_tdata_get_hard()
479 unsigned narenas_tdata = tsd_narenas_tdata_get(tsd); in arena_tdata_get_hard()
491 tsd_arenas_tdata_set(tsd, arenas_tdata); in arena_tdata_get_hard()
492 tsd_narenas_tdata_set(tsd, narenas_tdata); in arena_tdata_get_hard()
500 bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd); in arena_tdata_get_hard()
503 if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) { in arena_tdata_get_hard()
513 assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp); in arena_tdata_get_hard()
514 tsd_arenas_tdata_set(tsd, arenas_tdata); in arena_tdata_get_hard()
515 tsd_narenas_tdata_set(tsd, narenas_tdata); in arena_tdata_get_hard()
551 arena_choose_hard(tsd_t *tsd, bool internal) in arena_choose_hard() argument
570 malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock); in arena_choose_hard()
571 assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL); in arena_choose_hard()
573 if (arena_get(tsd_tsdn(tsd), i, false) != NULL) { in arena_choose_hard()
580 tsd_tsdn(tsd), i, false), !!j) < in arena_choose_hard()
582 tsd_tsdn(tsd), choose[j], false), in arena_choose_hard()
601 if (arena_nthreads_get(arena_get(tsd_tsdn(tsd), in arena_choose_hard()
609 ret = arena_get(tsd_tsdn(tsd), in arena_choose_hard()
617 arena = arena_init_locked(tsd_tsdn(tsd), in arena_choose_hard()
621 malloc_mutex_unlock(tsd_tsdn(tsd), in arena_choose_hard()
628 arena_bind(tsd, choose[j], !!j); in arena_choose_hard()
630 malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock); in arena_choose_hard()
632 ret = arena_get(tsd_tsdn(tsd), 0, false); in arena_choose_hard()
633 arena_bind(tsd, 0, false); in arena_choose_hard()
634 arena_bind(tsd, 0, true); in arena_choose_hard()
641 iarena_cleanup(tsd_t *tsd) in iarena_cleanup() argument
645 iarena = tsd_iarena_get(tsd); in iarena_cleanup()
647 arena_unbind(tsd, arena_ind_get(iarena), true); in iarena_cleanup()
651 arena_cleanup(tsd_t *tsd) in arena_cleanup() argument
655 arena = tsd_arena_get(tsd); in arena_cleanup()
657 arena_unbind(tsd, arena_ind_get(arena), false); in arena_cleanup()
661 arenas_tdata_cleanup(tsd_t *tsd) in arenas_tdata_cleanup() argument
666 *tsd_arenas_tdata_bypassp_get(tsd) = true; in arenas_tdata_cleanup()
668 arenas_tdata = tsd_arenas_tdata_get(tsd); in arenas_tdata_cleanup()
670 tsd_arenas_tdata_set(tsd, NULL); in arenas_tdata_cleanup()
1309 tsd_t *tsd; in malloc_init_hard() local
1328 tsd = malloc_tsd_boot0(); in malloc_init_hard()
1329 if (tsd == NULL) in malloc_init_hard()
1333 malloc_mutex_lock(tsd_tsdn(tsd), &init_lock); in malloc_init_hard()
1335 if (config_prof && prof_boot2(tsd)) { in malloc_init_hard()
1336 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); in malloc_init_hard()
1340 if (malloc_init_hard_finish(tsd_tsdn(tsd))) { in malloc_init_hard()
1341 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); in malloc_init_hard()
1345 malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); in malloc_init_hard()
1444 imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, in imalloc_no_sample() argument
1451 tcache = tcache_get(tsd, true); in imalloc_no_sample()
1455 tcache = tcaches_get(tsd, dopts->tcache_ind); in imalloc_no_sample()
1467 arena = arena_get(tsd_tsdn(tsd), dopts->arena_ind, true); in imalloc_no_sample()
1471 return ipalloct(tsd_tsdn(tsd), usize, dopts->alignment, in imalloc_no_sample()
1475 return iallocztm(tsd_tsdn(tsd), size, ind, dopts->zero, tcache, false, in imalloc_no_sample()
1480 imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, in imalloc_sample() argument
1496 ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize, in imalloc_sample()
1501 arena_prof_promote(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), ret), in imalloc_sample()
1504 ret = imalloc_no_sample(sopts, dopts, tsd, usize, usize, ind); in imalloc_sample()
1552 tsd_t *tsd = NULL; in imalloc_body() local
1616 tsd = tsd_fetch(); in imalloc_body()
1617 witness_assert_lockless(tsd_tsdn(tsd)); in imalloc_body()
1627 tsd, usize, prof_active_get_unlocked(), true); in imalloc_body()
1630 sopts, dopts, tsd, usize, usize, ind); in imalloc_body()
1637 sopts, dopts, tsd, usize, ind); in imalloc_body()
1643 prof_alloc_rollback(tsd, tctx, true); in imalloc_body()
1647 prof_malloc(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), allocation), in imalloc_body()
1657 allocation = imalloc_no_sample(sopts, dopts, tsd, usize, usize, in imalloc_body()
1672 assert(usize == isalloc(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), in imalloc_body()
1674 *tsd_thread_allocatedp_get(tsd) += usize; in imalloc_body()
1681 witness_assert_lockless(tsd_tsdn(tsd)); in imalloc_body()
1699 witness_assert_lockless(tsd_tsdn(tsd)); in imalloc_body()
1730 witness_assert_lockless(tsd_tsdn(tsd)); in imalloc_body()
1867 irealloc_prof_sample(tsd_t *tsd, extent_t *extent, void *old_ptr, in irealloc_prof_sample() argument
1875 p = iralloc(tsd, extent, old_ptr, old_usize, LARGE_MINCLASS, 0, in irealloc_prof_sample()
1879 arena_prof_promote(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), p), p, in irealloc_prof_sample()
1882 p = iralloc(tsd, extent, old_ptr, old_usize, usize, 0, false); in irealloc_prof_sample()
1888 irealloc_prof(tsd_t *tsd, extent_t *old_extent, void *old_ptr, size_t old_usize, in irealloc_prof() argument
1897 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_extent, old_ptr); in irealloc_prof()
1898 tctx = prof_alloc_prep(tsd, usize, prof_active, true); in irealloc_prof()
1900 p = irealloc_prof_sample(tsd, old_extent, old_ptr, old_usize, in irealloc_prof()
1903 p = iralloc(tsd, old_extent, old_ptr, old_usize, usize, 0, in irealloc_prof()
1907 prof_alloc_rollback(tsd, tctx, true); in irealloc_prof()
1910 extent = (p == old_ptr) ? old_extent : iealloc(tsd_tsdn(tsd), p); in irealloc_prof()
1911 prof_realloc(tsd, extent, p, usize, tctx, prof_active, true, old_extent, in irealloc_prof()
1918 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) in ifree() argument
1923 witness_assert_lockless(tsd_tsdn(tsd)); in ifree()
1928 extent = iealloc(tsd_tsdn(tsd), ptr); in ifree()
1930 usize = isalloc(tsd_tsdn(tsd), extent, ptr); in ifree()
1931 prof_free(tsd, extent, ptr, usize); in ifree()
1933 usize = isalloc(tsd_tsdn(tsd), extent, ptr); in ifree()
1935 *tsd_thread_deallocatedp_get(tsd) += usize; in ifree()
1938 idalloctm(tsd_tsdn(tsd), extent, ptr, tcache, false, false); in ifree()
1940 idalloctm(tsd_tsdn(tsd), extent, ptr, tcache, false, true); in ifree()
1944 isfree(tsd_t *tsd, extent_t *extent, void *ptr, size_t usize, tcache_t *tcache, in isfree() argument
1947 witness_assert_lockless(tsd_tsdn(tsd)); in isfree()
1953 prof_free(tsd, extent, ptr, usize); in isfree()
1955 *tsd_thread_deallocatedp_get(tsd) += usize; in isfree()
1958 isdalloct(tsd_tsdn(tsd), extent, ptr, usize, tcache, false); in isfree()
1960 isdalloct(tsd_tsdn(tsd), extent, ptr, usize, tcache, true); in isfree()
1975 tsd_t *tsd; in je_realloc() local
1979 tsd = tsd_fetch(); in je_realloc()
1980 ifree(tsd, ptr, tcache_get(tsd, false), true); in je_realloc()
1987 tsd_t *tsd; in je_realloc() local
1991 tsd = tsd_fetch(); in je_realloc()
1993 witness_assert_lockless(tsd_tsdn(tsd)); in je_realloc()
1995 extent = iealloc(tsd_tsdn(tsd), ptr); in je_realloc()
1996 old_usize = isalloc(tsd_tsdn(tsd), extent, ptr); in je_realloc()
2000 NULL : irealloc_prof(tsd, extent, ptr, old_usize, in je_realloc()
2005 ret = iralloc(tsd, extent, ptr, old_usize, size, 0, in je_realloc()
2008 tsdn = tsd_tsdn(tsd); in je_realloc()
2023 tsd_t *tsd; in je_realloc() local
2026 tsd = tsdn_tsd(tsdn); in je_realloc()
2027 *tsd_thread_allocatedp_get(tsd) += usize; in je_realloc()
2028 *tsd_thread_deallocatedp_get(tsd) += old_usize; in je_realloc()
2040 tsd_t *tsd = tsd_fetch(); in je_free() local
2041 witness_assert_lockless(tsd_tsdn(tsd)); in je_free()
2043 ifree(tsd, ptr, tcache_get(tsd, false), false); in je_free()
2045 ifree(tsd, ptr, tcache_get(tsd, false), true); in je_free()
2046 witness_assert_lockless(tsd_tsdn(tsd)); in je_free()
2249 irallocx_prof(tsd_t *tsd, extent_t *old_extent, void *old_ptr, size_t old_usize, in irallocx_prof() argument
2259 old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_extent, old_ptr); in irallocx_prof()
2260 tctx = prof_alloc_prep(tsd, *usize, prof_active, false); in irallocx_prof()
2262 p = irallocx_prof_sample(tsd_tsdn(tsd), old_extent, old_ptr, in irallocx_prof()
2265 p = iralloct(tsd_tsdn(tsd), old_extent, old_ptr, old_usize, in irallocx_prof()
2269 prof_alloc_rollback(tsd, tctx, false); in irallocx_prof()
2283 *usize = isalloc(tsd_tsdn(tsd), extent, p); in irallocx_prof()
2285 extent = iealloc(tsd_tsdn(tsd), p); in irallocx_prof()
2286 prof_realloc(tsd, extent, p, *usize, tctx, prof_active, false, in irallocx_prof()
2298 tsd_t *tsd; in je_rallocx() local
2310 tsd = tsd_fetch(); in je_rallocx()
2311 witness_assert_lockless(tsd_tsdn(tsd)); in je_rallocx()
2312 extent = iealloc(tsd_tsdn(tsd), ptr); in je_rallocx()
2316 arena = arena_get(tsd_tsdn(tsd), arena_ind, true); in je_rallocx()
2326 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); in je_rallocx()
2328 tcache = tcache_get(tsd, true); in je_rallocx()
2330 old_usize = isalloc(tsd_tsdn(tsd), extent, ptr); in je_rallocx()
2336 p = irallocx_prof(tsd, extent, ptr, old_usize, size, alignment, in je_rallocx()
2341 p = iralloct(tsd_tsdn(tsd), extent, ptr, old_usize, size, in je_rallocx()
2346 usize = isalloc(tsd_tsdn(tsd), iealloc(tsd_tsdn(tsd), in je_rallocx()
2353 *tsd_thread_allocatedp_get(tsd) += usize; in je_rallocx()
2354 *tsd_thread_deallocatedp_get(tsd) += old_usize; in je_rallocx()
2357 witness_assert_lockless(tsd_tsdn(tsd)); in je_rallocx()
2365 witness_assert_lockless(tsd_tsdn(tsd)); in je_rallocx()
2398 ixallocx_prof(tsd_t *tsd, extent_t *extent, void *ptr, size_t old_usize, in ixallocx_prof() argument
2406 old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, ptr); in ixallocx_prof()
2428 tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); in ixallocx_prof()
2431 usize = ixallocx_prof_sample(tsd_tsdn(tsd), extent, ptr, in ixallocx_prof()
2434 usize = ixallocx_helper(tsd_tsdn(tsd), extent, ptr, old_usize, in ixallocx_prof()
2438 prof_alloc_rollback(tsd, tctx, false); in ixallocx_prof()
2441 prof_realloc(tsd, extent, ptr, usize, tctx, prof_active, false, extent, in ixallocx_prof()
2450 tsd_t *tsd; in je_xallocx() local
2460 tsd = tsd_fetch(); in je_xallocx()
2461 witness_assert_lockless(tsd_tsdn(tsd)); in je_xallocx()
2462 extent = iealloc(tsd_tsdn(tsd), ptr); in je_xallocx()
2464 old_usize = isalloc(tsd_tsdn(tsd), extent, ptr); in je_xallocx()
2483 usize = ixallocx_prof(tsd, extent, ptr, old_usize, size, extra, in je_xallocx()
2486 usize = ixallocx_helper(tsd_tsdn(tsd), extent, ptr, old_usize, in je_xallocx()
2493 *tsd_thread_allocatedp_get(tsd) += usize; in je_xallocx()
2494 *tsd_thread_deallocatedp_get(tsd) += old_usize; in je_xallocx()
2498 witness_assert_lockless(tsd_tsdn(tsd)); in je_xallocx()
2526 tsd_t *tsd; in je_dallocx() local
2532 tsd = tsd_fetch(); in je_dallocx()
2533 witness_assert_lockless(tsd_tsdn(tsd)); in je_dallocx()
2538 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); in je_dallocx()
2540 tcache = tcache_get(tsd, false); in je_dallocx()
2544 ifree(tsd, ptr, tcache, false); in je_dallocx()
2546 ifree(tsd, ptr, tcache, true); in je_dallocx()
2547 witness_assert_lockless(tsd_tsdn(tsd)); in je_dallocx()
2568 tsd_t *tsd; in je_sdallocx() local
2575 tsd = tsd_fetch(); in je_sdallocx()
2576 extent = iealloc(tsd_tsdn(tsd), ptr); in je_sdallocx()
2577 usize = inallocx(tsd_tsdn(tsd), size, flags); in je_sdallocx()
2578 assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr)); in je_sdallocx()
2580 witness_assert_lockless(tsd_tsdn(tsd)); in je_sdallocx()
2585 tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); in je_sdallocx()
2587 tcache = tcache_get(tsd, false); in je_sdallocx()
2591 isfree(tsd, extent, ptr, usize, tcache, false); in je_sdallocx()
2593 isfree(tsd, extent, ptr, usize, tcache, true); in je_sdallocx()
2594 witness_assert_lockless(tsd_tsdn(tsd)); in je_sdallocx()
2625 tsd_t *tsd; in je_mallctl() local
2630 tsd = tsd_fetch(); in je_mallctl()
2631 witness_assert_lockless(tsd_tsdn(tsd)); in je_mallctl()
2632 ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); in je_mallctl()
2633 witness_assert_lockless(tsd_tsdn(tsd)); in je_mallctl()
2658 tsd_t *tsd; in je_mallctlbymib() local
2663 tsd = tsd_fetch(); in je_mallctlbymib()
2664 witness_assert_lockless(tsd_tsdn(tsd)); in je_mallctlbymib()
2665 ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); in je_mallctlbymib()
2666 witness_assert_lockless(tsd_tsdn(tsd)); in je_mallctlbymib()
2743 tsd_t *tsd; in jemalloc_prefork() local
2753 tsd = tsd_fetch(); in jemalloc_prefork()
2757 witness_prefork(tsd); in jemalloc_prefork()
2759 ctl_prefork(tsd_tsdn(tsd)); in jemalloc_prefork()
2760 malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock); in jemalloc_prefork()
2761 prof_prefork0(tsd_tsdn(tsd)); in jemalloc_prefork()
2764 if ((arena = arena_get(tsd_tsdn(tsd), j, false)) != in jemalloc_prefork()
2768 arena_prefork0(tsd_tsdn(tsd), arena); in jemalloc_prefork()
2771 arena_prefork1(tsd_tsdn(tsd), arena); in jemalloc_prefork()
2774 arena_prefork2(tsd_tsdn(tsd), arena); in jemalloc_prefork()
2782 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) in jemalloc_prefork()
2783 arena_prefork3(tsd_tsdn(tsd), arena); in jemalloc_prefork()
2785 prof_prefork1(tsd_tsdn(tsd)); in jemalloc_prefork()
2796 tsd_t *tsd; in jemalloc_postfork_parent() local
2805 tsd = tsd_fetch(); in jemalloc_postfork_parent()
2807 witness_postfork_parent(tsd); in jemalloc_postfork_parent()
2812 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) in jemalloc_postfork_parent()
2813 arena_postfork_parent(tsd_tsdn(tsd), arena); in jemalloc_postfork_parent()
2815 prof_postfork_parent(tsd_tsdn(tsd)); in jemalloc_postfork_parent()
2816 malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock); in jemalloc_postfork_parent()
2817 ctl_postfork_parent(tsd_tsdn(tsd)); in jemalloc_postfork_parent()
2823 tsd_t *tsd; in jemalloc_postfork_child() local
2828 tsd = tsd_fetch(); in jemalloc_postfork_child()
2830 witness_postfork_child(tsd); in jemalloc_postfork_child()
2835 if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) in jemalloc_postfork_child()
2836 arena_postfork_child(tsd_tsdn(tsd), arena); in jemalloc_postfork_child()
2838 prof_postfork_child(tsd_tsdn(tsd)); in jemalloc_postfork_child()
2839 malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock); in jemalloc_postfork_child()
2840 ctl_postfork_child(tsd_tsdn(tsd)); in jemalloc_postfork_child()