Lines Matching refs:tsdn
71 static void extent_record(tsdn_t *tsdn, arena_t *arena,
78 extent_alloc(tsdn_t *tsdn, arena_t *arena) in extent_alloc() argument
82 malloc_mutex_lock(tsdn, &arena->extent_cache_mtx); in extent_alloc()
85 malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); in extent_alloc()
86 return (base_alloc(tsdn, arena->base, sizeof(extent_t), in extent_alloc()
90 malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); in extent_alloc()
95 extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) in extent_dalloc() argument
97 malloc_mutex_lock(tsdn, &arena->extent_cache_mtx); in extent_dalloc()
100 malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); in extent_dalloc()
197 extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1], in extent_heaps_insert() argument
203 malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx); in extent_heaps_insert()
209 extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1], in extent_heaps_remove() argument
215 malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx); in extent_heaps_remove()
221 extent_rtree_acquire(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, in extent_rtree_acquire() argument
225 *r_elm_a = rtree_elm_acquire(tsdn, &extents_rtree, rtree_ctx, in extent_rtree_acquire()
232 *r_elm_b = rtree_elm_acquire(tsdn, &extents_rtree, rtree_ctx, in extent_rtree_acquire()
236 rtree_elm_release(tsdn, &extents_rtree, *r_elm_a); in extent_rtree_acquire()
247 extent_rtree_write_acquired(tsdn_t *tsdn, rtree_elm_t *elm_a, in extent_rtree_write_acquired() argument
250 rtree_elm_write_acquired(tsdn, &extents_rtree, elm_a, extent); in extent_rtree_write_acquired()
252 rtree_elm_write_acquired(tsdn, &extents_rtree, elm_b, extent); in extent_rtree_write_acquired()
256 extent_rtree_release(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b) in extent_rtree_release() argument
258 rtree_elm_release(tsdn, &extents_rtree, elm_a); in extent_rtree_release()
260 rtree_elm_release(tsdn, &extents_rtree, elm_b); in extent_rtree_release()
264 extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, in extent_interior_register() argument
272 rtree_write(tsdn, &extents_rtree, rtree_ctx, in extent_interior_register()
279 extent_gprof_add(tsdn_t *tsdn, const extent_t *extent) in extent_gprof_add() argument
295 prof_gdump(tsdn); in extent_gprof_add()
300 extent_gprof_sub(tsdn_t *tsdn, const extent_t *extent) in extent_gprof_sub() argument
312 extent_register(tsdn_t *tsdn, const extent_t *extent) in extent_register() argument
315 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); in extent_register()
318 if (extent_rtree_acquire(tsdn, rtree_ctx, extent, false, true, &elm_a, in extent_register()
321 extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent); in extent_register()
323 extent_interior_register(tsdn, rtree_ctx, extent); in extent_register()
324 extent_rtree_release(tsdn, elm_a, elm_b); in extent_register()
327 extent_gprof_add(tsdn, extent); in extent_register()
333 extent_reregister(tsdn_t *tsdn, const extent_t *extent) in extent_reregister() argument
335 bool err = extent_register(tsdn, extent); in extent_reregister()
340 extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, in extent_interior_deregister() argument
348 rtree_clear(tsdn, &extents_rtree, rtree_ctx, in extent_interior_deregister()
355 extent_deregister(tsdn_t *tsdn, extent_t *extent) in extent_deregister() argument
358 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); in extent_deregister()
361 extent_rtree_acquire(tsdn, rtree_ctx, extent, true, false, &elm_a, in extent_deregister()
363 extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL); in extent_deregister()
365 extent_interior_deregister(tsdn, rtree_ctx, extent); in extent_deregister()
368 extent_rtree_release(tsdn, elm_a, elm_b); in extent_deregister()
371 extent_gprof_sub(tsdn, extent); in extent_deregister()
379 extent_first_best_fit(tsdn_t *tsdn, arena_t *arena, in extent_first_best_fit() argument
384 malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); in extent_first_best_fit()
397 extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extent_leak() argument
405 if (extent_purge_lazy_wrapper(tsdn, arena, r_extent_hooks, in extent_leak()
407 extent_purge_forced_wrapper(tsdn, arena, r_extent_hooks, in extent_leak()
411 extent_dalloc(tsdn, arena, extent); in extent_leak()
415 extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extent_recycle() argument
422 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); in extent_recycle()
426 malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); in extent_recycle()
447 prev = extent_lookup(tsdn, (void *)((uintptr_t)new_addr - PAGE), in extent_recycle()
458 malloc_mutex_lock(tsdn, &arena->extents_mtx); in extent_recycle()
463 elm = rtree_elm_acquire(tsdn, &extents_rtree, rtree_ctx, in extent_recycle()
466 extent = rtree_elm_read_acquired(tsdn, &extents_rtree, in extent_recycle()
476 rtree_elm_release(tsdn, &extents_rtree, elm); in extent_recycle()
480 extent = extent_first_best_fit(tsdn, arena, extent_heaps, in extent_recycle()
485 malloc_mutex_unlock(tsdn, &arena->extents_mtx); in extent_recycle()
488 extent_heaps_remove(tsdn, extent_heaps, extent); in extent_recycle()
489 arena_extent_cache_maybe_remove(tsdn, arena, extent, cache); in extent_recycle()
504 extent = extent_split_wrapper(tsdn, arena, r_extent_hooks, in extent_recycle()
508 extent_deregister(tsdn, lead); in extent_recycle()
509 extent_leak(tsdn, arena, r_extent_hooks, cache, lead); in extent_recycle()
511 malloc_mutex_unlock(tsdn, &arena->extents_mtx); in extent_recycle()
514 extent_heaps_insert(tsdn, extent_heaps, lead); in extent_recycle()
515 arena_extent_cache_maybe_insert(tsdn, arena, lead, cache); in extent_recycle()
520 extent_t *trail = extent_split_wrapper(tsdn, arena, in extent_recycle()
523 extent_deregister(tsdn, extent); in extent_recycle()
524 extent_leak(tsdn, arena, r_extent_hooks, cache, in extent_recycle()
527 malloc_mutex_unlock(tsdn, &arena->extents_mtx); in extent_recycle()
530 extent_heaps_insert(tsdn, extent_heaps, trail); in extent_recycle()
531 arena_extent_cache_maybe_insert(tsdn, arena, trail, cache); in extent_recycle()
541 if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent, in extent_recycle()
544 malloc_mutex_unlock(tsdn, &arena->extents_mtx); in extent_recycle()
545 extent_record(tsdn, arena, r_extent_hooks, extent_heaps, in extent_recycle()
553 extent_addr_randomize(tsdn, extent, alignment); in extent_recycle()
557 extent_interior_register(tsdn, rtree_ctx, extent); in extent_recycle()
561 malloc_mutex_unlock(tsdn, &arena->extents_mtx); in extent_recycle()
586 extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, in extent_alloc_core() argument
596 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, in extent_alloc_core()
605 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero, in extent_alloc_core()
614 extent_alloc_cache_impl(tsdn_t *tsdn, arena_t *arena, in extent_alloc_cache_impl() argument
623 extent = extent_recycle(tsdn, arena, r_extent_hooks, in extent_alloc_cache_impl()
630 extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena, in extent_alloc_cache_locked() argument
634 malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); in extent_alloc_cache_locked()
636 return (extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, true, in extent_alloc_cache_locked()
641 extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, in extent_alloc_cache() argument
645 return (extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, false, in extent_alloc_cache()
650 extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr, in extent_alloc_default_impl() argument
655 ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero, in extent_alloc_default_impl()
664 tsdn_t *tsdn; in extent_alloc_default() local
669 tsdn = tsdn_fetch(); in extent_alloc_default()
670 arena = arena_get(tsdn, arena_ind, false); in extent_alloc_default()
677 return (extent_alloc_default_impl(tsdn, arena, new_addr, size, in extent_alloc_default()
682 extent_retain(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extent_retain() argument
687 extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained, in extent_retain()
697 extent_grow_retained(tsdn_t *tsdn, arena_t *arena, in extent_grow_retained() argument
720 extent = extent_alloc(tsdn, arena); in extent_grow_retained()
725 ptr = extent_alloc_core(tsdn, arena, new_addr, alloc_size, PAGE, in extent_grow_retained()
729 if (ptr == NULL || extent_register(tsdn, extent)) { in extent_grow_retained()
730 extent_dalloc(tsdn, arena, extent); in extent_grow_retained()
752 extent = extent_split_wrapper(tsdn, arena, r_extent_hooks, lead, in extent_grow_retained()
755 extent_deregister(tsdn, lead); in extent_grow_retained()
756 extent_leak(tsdn, arena, r_extent_hooks, false, lead); in extent_grow_retained()
759 extent_retain(tsdn, arena, r_extent_hooks, lead); in extent_grow_retained()
764 extent_t *trail = extent_split_wrapper(tsdn, arena, in extent_grow_retained()
767 extent_deregister(tsdn, extent); in extent_grow_retained()
768 extent_leak(tsdn, arena, r_extent_hooks, false, extent); in extent_grow_retained()
771 extent_retain(tsdn, arena, r_extent_hooks, trail); in extent_grow_retained()
781 if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent, in extent_grow_retained()
783 extent_retain(tsdn, arena, r_extent_hooks, extent); in extent_grow_retained()
791 extent_gprof_add(tsdn, extent); in extent_grow_retained()
794 extent_addr_randomize(tsdn, extent, alignment); in extent_grow_retained()
797 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, in extent_grow_retained()
801 extent_interior_register(tsdn, rtree_ctx, extent); in extent_grow_retained()
811 extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, in extent_alloc_retained() argument
820 extent = extent_recycle(tsdn, arena, r_extent_hooks, in extent_alloc_retained()
829 extent_gprof_add(tsdn, extent); in extent_alloc_retained()
832 extent = extent_grow_retained(tsdn, arena, r_extent_hooks, in extent_alloc_retained()
840 extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, in extent_alloc_wrapper_hard() argument
849 extent = extent_alloc(tsdn, arena); in extent_alloc_wrapper_hard()
854 addr = extent_alloc_default_impl(tsdn, arena, new_addr, size, in extent_alloc_wrapper_hard()
861 extent_dalloc(tsdn, arena, extent); in extent_alloc_wrapper_hard()
867 extent_addr_randomize(tsdn, extent, alignment); in extent_alloc_wrapper_hard()
868 if (extent_register(tsdn, extent)) { in extent_alloc_wrapper_hard()
869 extent_leak(tsdn, arena, r_extent_hooks, false, extent); in extent_alloc_wrapper_hard()
877 extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_alloc_wrapper() argument
885 extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, new_addr, in extent_alloc_wrapper()
888 extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks, in extent_alloc_wrapper()
911 extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, in extent_try_coalesce() argument
918 extent_heaps_remove(tsdn, extent_heaps, a); in extent_try_coalesce()
919 extent_heaps_remove(tsdn, extent_heaps, b); in extent_try_coalesce()
921 arena_extent_cache_maybe_remove(tsdn, extent_arena_get(a), a, cache); in extent_try_coalesce()
922 arena_extent_cache_maybe_remove(tsdn, extent_arena_get(b), b, cache); in extent_try_coalesce()
924 if (extent_merge_wrapper(tsdn, arena, r_extent_hooks, a, b)) { in extent_try_coalesce()
925 extent_heaps_insert(tsdn, extent_heaps, a); in extent_try_coalesce()
926 extent_heaps_insert(tsdn, extent_heaps, b); in extent_try_coalesce()
927 arena_extent_cache_maybe_insert(tsdn, extent_arena_get(a), a, in extent_try_coalesce()
929 arena_extent_cache_maybe_insert(tsdn, extent_arena_get(b), b, in extent_try_coalesce()
934 extent_heaps_insert(tsdn, extent_heaps, a); in extent_try_coalesce()
935 arena_extent_cache_maybe_insert(tsdn, extent_arena_get(a), a, cache); in extent_try_coalesce()
939 extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, in extent_record() argument
944 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); in extent_record()
948 malloc_mutex_lock(tsdn, &arena->extents_mtx); in extent_record()
955 extent_interior_deregister(tsdn, rtree_ctx, extent); in extent_record()
959 assert(extent_lookup(tsdn, extent_base_get(extent), true) == extent); in extent_record()
960 extent_heaps_insert(tsdn, extent_heaps, extent); in extent_record()
961 arena_extent_cache_maybe_insert(tsdn, arena, extent, cache); in extent_record()
964 next = rtree_read(tsdn, &extents_rtree, rtree_ctx, in extent_record()
967 extent_try_coalesce(tsdn, arena, r_extent_hooks, extent, next, in extent_record()
972 prev = rtree_read(tsdn, &extents_rtree, rtree_ctx, in extent_record()
975 extent_try_coalesce(tsdn, arena, r_extent_hooks, prev, extent, in extent_record()
979 malloc_mutex_unlock(tsdn, &arena->extents_mtx); in extent_record()
983 extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) in extent_dalloc_gap() argument
987 if (extent_register(tsdn, extent)) { in extent_dalloc_gap()
988 extent_leak(tsdn, arena, &extent_hooks, false, extent); in extent_dalloc_gap()
991 extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); in extent_dalloc_gap()
995 extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, in extent_dalloc_cache() argument
1004 extent_record(tsdn, arena, r_extent_hooks, arena->extents_cached, true, in extent_dalloc_cache()
1026 extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, in extent_dalloc_wrapper_try() argument
1041 extent_deregister(tsdn, extent); in extent_dalloc_wrapper_try()
1054 extent_dalloc(tsdn, arena, extent); in extent_dalloc_wrapper_try()
1060 extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_dalloc_wrapper() argument
1065 if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) in extent_dalloc_wrapper()
1068 extent_reregister(tsdn, extent); in extent_dalloc_wrapper()
1072 else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent, in extent_dalloc_wrapper()
1092 extent_gprof_sub(tsdn, extent); in extent_dalloc_wrapper()
1094 extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained, in extent_dalloc_wrapper()
1109 extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_commit_wrapper() argument
1134 extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_decommit_wrapper() argument
1167 extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_purge_lazy_wrapper() argument
1195 extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_purge_forced_wrapper() argument
1220 extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_split_wrapper() argument
1226 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); in extent_split_wrapper()
1236 trail = extent_alloc(tsdn, arena); in extent_split_wrapper()
1248 if (extent_rtree_acquire(tsdn, rtree_ctx, &lead, false, true, in extent_split_wrapper()
1257 if (extent_rtree_acquire(tsdn, rtree_ctx, trail, false, true, in extent_split_wrapper()
1269 extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent); in extent_split_wrapper()
1270 extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail); in extent_split_wrapper()
1272 extent_rtree_release(tsdn, lead_elm_a, lead_elm_b); in extent_split_wrapper()
1273 extent_rtree_release(tsdn, trail_elm_a, trail_elm_b); in extent_split_wrapper()
1277 extent_rtree_release(tsdn, trail_elm_a, trail_elm_b); in extent_split_wrapper()
1279 extent_rtree_release(tsdn, lead_elm_a, lead_elm_b); in extent_split_wrapper()
1281 extent_dalloc(tsdn, arena, trail); in extent_split_wrapper()
1309 extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, in extent_merge_wrapper() argument
1314 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); in extent_merge_wrapper()
1341 extent_rtree_acquire(tsdn, rtree_ctx, a, true, false, &a_elm_a, in extent_merge_wrapper()
1343 extent_rtree_acquire(tsdn, rtree_ctx, b, true, false, &b_elm_a, in extent_merge_wrapper()
1347 rtree_elm_write_acquired(tsdn, &extents_rtree, a_elm_b, NULL); in extent_merge_wrapper()
1348 rtree_elm_release(tsdn, &extents_rtree, a_elm_b); in extent_merge_wrapper()
1351 rtree_elm_write_acquired(tsdn, &extents_rtree, b_elm_a, NULL); in extent_merge_wrapper()
1352 rtree_elm_release(tsdn, &extents_rtree, b_elm_a); in extent_merge_wrapper()
1362 extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a); in extent_merge_wrapper()
1363 extent_rtree_release(tsdn, a_elm_a, b_elm_b); in extent_merge_wrapper()
1365 extent_dalloc(tsdn, extent_arena_get(b), b); in extent_merge_wrapper()