Lines Matching refs:size_bits
167 static void _trace_alloc(rt_page_t page, void *caller, size_t size_bits) in _trace_alloc() argument
172 page->trace_size = size_bits; in _trace_alloc()
190 void _report(rt_page_t page, size_t size_bits, char *msg) in _report() argument
198 static void _trace_free(rt_page_t page, void *caller, size_t size_bits) in _trace_free() argument
205 _report(page, size_bits, "free after free"); in _trace_free()
208 else if (page->trace_size != size_bits) in _trace_free()
210 rt_kprintf("free with size bits %lx\n", size_bits); in _trace_free()
211 _report(page, size_bits, "incompatible size bits parameter"); in _trace_free()
305 static rt_bool_t _pages_are_member(rt_ubase_t page_va, size_t size_bits) in _pages_are_member() argument
309 size_t frame_end = page_va + (1 << size_bits); in _pages_are_member()
331 static rt_err_t _unpoisoned_pages(char *head, rt_uint32_t size_bits) in _unpoisoned_pages() argument
338 int pages_count = 1 << size_bits; in _unpoisoned_pages()
348 … rt_kprintf("%s: Pages[%p, %d] is already in used by others!\n", __func__, head, size_bits); in _unpoisoned_pages()
362 static rt_err_t _poisoned_pages(char *head, rt_uint32_t size_bits) in _poisoned_pages() argument
369 int pages_count = 1 << size_bits; in _poisoned_pages()
379 rt_kprintf("%s: Pages[%p, %d] is freed before!\n", __func__, head, size_bits); in _poisoned_pages()
463 rt_uint32_t size_bits) in _buddy_get() argument
467 RT_ASSERT(size_bits < RT_PAGE_MAX_ORDER - 1); in _buddy_get()
470 addr ^= (1UL << (size_bits + ARCH_PAGE_SHIFT)); in _buddy_get()
475 rt_uint32_t size_bits) in _get_pgls_head_by_page() argument
480 if (size_bits < AFFID_BLK_BITS) in _get_pgls_head_by_page()
482 index = PAGE_TO_AFFID(page) >> size_bits; in _get_pgls_head_by_page()
483 RT_ASSERT(index < AFFID_NUMOF_ID_IN_SET(size_bits)); in _get_pgls_head_by_page()
485 RT_ASSERT(PGLS_IS_AFF_MAP(agr_pgls[size_bits])); in _get_pgls_head_by_page()
486 pgls_head = &PGLS_GET_AFF_MAP(agr_pgls[size_bits])[index]; in _get_pgls_head_by_page()
490 RT_ASSERT(!PGLS_IS_AFF_MAP(agr_pgls[size_bits])); in _get_pgls_head_by_page()
491 pgls_head = &agr_pgls[size_bits].page_list; in _get_pgls_head_by_page()
498 rt_uint32_t size_bits) in _get_pgls_head() argument
503 if (size_bits < AFFID_BLK_BITS) in _get_pgls_head()
505 index = affid >> size_bits; in _get_pgls_head()
506 RT_ASSERT(index < AFFID_NUMOF_ID_IN_SET(size_bits)); in _get_pgls_head()
508 RT_ASSERT(PGLS_IS_AFF_MAP(agr_pgls[size_bits])); in _get_pgls_head()
509 pgls_head = &PGLS_GET_AFF_MAP(agr_pgls[size_bits])[index]; in _get_pgls_head()
513 RT_ASSERT(!PGLS_IS_AFF_MAP(agr_pgls[size_bits])); in _get_pgls_head()
514 pgls_head = &agr_pgls[size_bits].page_list; in _get_pgls_head()
522 p->size_bits = ARCH_ADDRESS_WIDTH_BITS; in _page_alloc()
527 rt_uint32_t size_bits) in _page_remove() argument
543 RT_ASSERT(p->size_bits == size_bits); in _page_remove()
548 rt_uint32_t size_bits) in _page_insert() argument
557 p->size_bits = size_bits; in _page_insert()
560 static void _pages_ref_inc(struct rt_page *p, rt_uint32_t size_bits) in _pages_ref_inc() argument
567 idx = idx & ~((1UL << size_bits) - 1); in _pages_ref_inc()
574 static int _pages_ref_get(struct rt_page *p, rt_uint32_t size_bits) in _pages_ref_get() argument
581 idx = idx & ~((1UL << size_bits) - 1); in _pages_ref_get()
588 char *frame_va, rt_uint32_t size_bits) in _pages_free() argument
590 rt_uint32_t level = size_bits; in _pages_free()
597 RT_ASSERT(p->size_bits == ARCH_ADDRESS_WIDTH_BITS); in _pages_free()
598 RT_ASSERT(size_bits < RT_PAGE_MAX_ORDER); in _pages_free()
600 RT_ASSERT(_pages_are_member((rt_ubase_t)frame_va, size_bits)); in _pages_free()
609 _poisoned_pages(frame_va, size_bits); in _pages_free()
615 if (buddy && buddy->size_bits == level) in _pages_free()
634 pgls_agr_t agr_pgls[], rt_uint32_t size_bits, int affid, in __pages_alloc() argument
636 rt_uint32_t size_bits), in __pages_alloc() argument
638 rt_uint32_t size_bits), in __pages_alloc()
641 rt_page_t *pgls_head = _get_pgls_head(agr_pgls, affid, size_bits); in __pages_alloc()
646 page_remove(pgls_head, p, size_bits); in __pages_alloc()
654 for (level = size_bits + 1; level < RT_PAGE_MAX_ORDER; level++) in __pages_alloc()
676 while (level > size_bits) in __pages_alloc()
701 rt_uint32_t size_bits, int affid) in _pages_alloc() argument
703 return __pages_alloc(page_list, size_bits, affid, _page_remove, in _pages_alloc()
708 rt_uint32_t size_bits) in _early_page_remove() argument
727 RT_ASSERT(page_cont->size_bits == size_bits); in _early_page_remove()
728 page_cont->size_bits = ARCH_ADDRESS_WIDTH_BITS; in _early_page_remove()
735 page_cont->size_bits = ARCH_ADDRESS_WIDTH_BITS; in _early_page_alloc()
740 rt_uint32_t size_bits) in _early_page_insert() argument
754 page_cont->size_bits = size_bits; in _early_page_insert()
758 rt_uint32_t size_bits, int affid) in _early_pages_alloc() argument
760 return __pages_alloc(page_list, size_bits, affid, _early_page_remove, in _early_pages_alloc()
779 int rt_page_ref_get(void *addr, rt_uint32_t size_bits) in rt_page_ref_get() argument
787 ref = _pages_ref_get(p, size_bits); in rt_page_ref_get()
792 void rt_page_ref_inc(void *addr, rt_uint32_t size_bits) in rt_page_ref_inc() argument
799 _pages_ref_inc(p, size_bits); in rt_page_ref_inc()
804 rt_uint32_t size_bits, int affid);
825 rt_inline void *_do_pages_alloc(rt_uint32_t size_bits, size_t flags, int affid) in _do_pages_alloc() argument
833 p = pages_alloc_handler(page_list, size_bits, affid); in _do_pages_alloc()
836 _freed_nr -= 1 << size_bits; in _do_pages_alloc()
846 p = pages_alloc_handler(page_list, size_bits, affid); in _do_pages_alloc()
849 _freed_nr -= 1 << size_bits; in _do_pages_alloc()
850 _freed_nr_hi -= 1 << size_bits; in _do_pages_alloc()
862 TRACE_ALLOC(p, size_bits); in _do_pages_alloc()
867 _unpoisoned_pages(alloc_buf, size_bits); in _do_pages_alloc()
875 rt_inline int _get_balanced_id(rt_uint32_t size_bits) in _get_balanced_id() argument
878 return (last_alloc + (1u << size_bits)) & AFFID_BITS_MASK(size_bits); in _get_balanced_id()
881 static void *_do_pages_alloc_noaff(rt_uint32_t size_bits, size_t flags) in _do_pages_alloc_noaff() argument
885 if (size_bits < AFFID_BLK_BITS) in _do_pages_alloc_noaff()
887 int try_affid = _get_balanced_id(size_bits); in _do_pages_alloc_noaff()
888 size_t numof_id = AFFID_NUMOF_ID_IN_SET(size_bits); in _do_pages_alloc_noaff()
891 for (size_t i = 0; i < numof_id; i++, try_affid += 1 << size_bits) in _do_pages_alloc_noaff()
893 rc = _do_pages_alloc(size_bits, flags, try_affid & valid_affid_mask); in _do_pages_alloc_noaff()
902 rc = _do_pages_alloc(size_bits, flags, 0); in _do_pages_alloc_noaff()
912 void *rt_pages_alloc(rt_uint32_t size_bits) in rt_pages_alloc() argument
914 return _do_pages_alloc_noaff(size_bits, 0); in rt_pages_alloc()
917 void *rt_pages_alloc_ext(rt_uint32_t size_bits, size_t flags) in rt_pages_alloc_ext() argument
919 return _do_pages_alloc_noaff(size_bits, flags); in rt_pages_alloc_ext()
922 void *rt_pages_alloc_tagged(rt_uint32_t size_bits, long affid, size_t flags) in rt_pages_alloc_tagged() argument
926 current = _do_pages_alloc(size_bits, flags, affid); in rt_pages_alloc_tagged()
935 int rt_pages_free(void *addr, rt_uint32_t size_bits) in rt_pages_free() argument
946 real_free = _pages_free(page_list, p, addr, size_bits); in rt_pages_free()
949 _freed_nr += 1 << size_bits; in rt_pages_free()
952 _freed_nr_hi += 1 << size_bits; in rt_pages_free()
954 TRACE_FREE(p, size_bits); in rt_pages_free()
1057 iter->size_bits = ARCH_ADDRESS_WIDTH_BITS; in _invalid_uninstalled_shadow()
1062 void (*insert)(rt_page_t *ppg, rt_page_t page, rt_uint32_t size_bits)) in _install_page() argument
1093 int size_bits; in _install_page() local
1096 size_bits = in _install_page()
1099 if (align_bits < size_bits) in _install_page()
1101 size_bits = align_bits; in _install_page()
1103 if (size_bits > max_order) in _install_page()
1105 size_bits = max_order; in _install_page()
1109 p->size_bits = ARCH_ADDRESS_WIDTH_BITS; in _install_page()
1116 _page_nr_hi += 1 << (size_bits - ARCH_PAGE_SHIFT); in _install_page()
1117 _freed_nr_hi += 1 << (size_bits - ARCH_PAGE_SHIFT); in _install_page()
1120 page_order = size_bits - ARCH_PAGE_SHIFT; in _install_page()
1123 region.start += (1UL << size_bits); in _install_page()
1226 iter->size_bits = ARCH_ADDRESS_WIDTH_BITS; in rt_page_init()