Lines Matching refs:mm
17 static struct drm_buddy_block *drm_block_alloc(struct drm_buddy *mm, in drm_block_alloc() argument
38 static void drm_block_free(struct drm_buddy *mm, in drm_block_free() argument
44 static void list_insert_sorted(struct drm_buddy *mm, in list_insert_sorted() argument
50 head = &mm->free_list[drm_buddy_block_order(block)]; in list_insert_sorted()
81 static void mark_free(struct drm_buddy *mm, in mark_free() argument
87 list_insert_sorted(mm, block); in mark_free()
123 static unsigned int __drm_buddy_free(struct drm_buddy *mm, in __drm_buddy_free() argument
153 mm->clear_avail -= drm_buddy_block_size(mm, buddy); in __drm_buddy_free()
155 drm_block_free(mm, block); in __drm_buddy_free()
156 drm_block_free(mm, buddy); in __drm_buddy_free()
162 mark_free(mm, block); in __drm_buddy_free()
167 static int __force_merge(struct drm_buddy *mm, in __force_merge() argument
178 if (min_order > mm->max_order) in __force_merge()
184 list_for_each_entry_safe_reverse(block, prev, &mm->free_list[i], link) { in __force_merge()
192 block_end = block_start + drm_buddy_block_size(mm, block) - 1; in __force_merge()
214 mm->clear_avail -= drm_buddy_block_size(mm, block); in __force_merge()
216 order = __drm_buddy_free(mm, block, true); in __force_merge()
237 int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) in drm_buddy_init() argument
253 mm->size = size; in drm_buddy_init()
254 mm->avail = size; in drm_buddy_init()
255 mm->clear_avail = 0; in drm_buddy_init()
256 mm->chunk_size = chunk_size; in drm_buddy_init()
257 mm->max_order = ilog2(size) - ilog2(chunk_size); in drm_buddy_init()
259 BUG_ON(mm->max_order > DRM_BUDDY_MAX_ORDER); in drm_buddy_init()
261 mm->free_list = kmalloc_array(mm->max_order + 1, in drm_buddy_init()
264 if (!mm->free_list) in drm_buddy_init()
267 for (i = 0; i <= mm->max_order; ++i) in drm_buddy_init()
268 INIT_LIST_HEAD(&mm->free_list[i]); in drm_buddy_init()
270 mm->n_roots = hweight64(size); in drm_buddy_init()
272 mm->roots = kmalloc_array(mm->n_roots, in drm_buddy_init()
275 if (!mm->roots) in drm_buddy_init()
293 root = drm_block_alloc(mm, NULL, order, offset); in drm_buddy_init()
297 mark_free(mm, root); in drm_buddy_init()
299 BUG_ON(i > mm->max_order); in drm_buddy_init()
300 BUG_ON(drm_buddy_block_size(mm, root) < chunk_size); in drm_buddy_init()
302 mm->roots[i] = root; in drm_buddy_init()
313 drm_block_free(mm, mm->roots[i]); in drm_buddy_init()
314 kfree(mm->roots); in drm_buddy_init()
316 kfree(mm->free_list); in drm_buddy_init()
328 void drm_buddy_fini(struct drm_buddy *mm) in drm_buddy_fini() argument
334 size = mm->size; in drm_buddy_fini()
336 for (i = 0; i < mm->n_roots; ++i) { in drm_buddy_fini()
337 order = ilog2(size) - ilog2(mm->chunk_size); in drm_buddy_fini()
338 start = drm_buddy_block_offset(mm->roots[i]); in drm_buddy_fini()
339 __force_merge(mm, start, start + size, order); in drm_buddy_fini()
341 if (WARN_ON(!drm_buddy_block_is_free(mm->roots[i]))) in drm_buddy_fini()
344 drm_block_free(mm, mm->roots[i]); in drm_buddy_fini()
346 root_size = mm->chunk_size << order; in drm_buddy_fini()
350 WARN_ON(mm->avail != mm->size); in drm_buddy_fini()
352 kfree(mm->roots); in drm_buddy_fini()
353 kfree(mm->free_list); in drm_buddy_fini()
357 static int split_block(struct drm_buddy *mm, in split_block() argument
366 block->left = drm_block_alloc(mm, block, block_order, offset); in split_block()
370 block->right = drm_block_alloc(mm, block, block_order, in split_block()
371 offset + (mm->chunk_size << block_order)); in split_block()
373 drm_block_free(mm, block->left); in split_block()
377 mark_free(mm, block->left); in split_block()
378 mark_free(mm, block->right); in split_block()
417 void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear) in drm_buddy_reset_clear() argument
423 size = mm->size; in drm_buddy_reset_clear()
424 for (i = 0; i < mm->n_roots; ++i) { in drm_buddy_reset_clear()
425 order = ilog2(size) - ilog2(mm->chunk_size); in drm_buddy_reset_clear()
426 start = drm_buddy_block_offset(mm->roots[i]); in drm_buddy_reset_clear()
427 __force_merge(mm, start, start + size, order); in drm_buddy_reset_clear()
429 root_size = mm->chunk_size << order; in drm_buddy_reset_clear()
433 for (i = 0; i <= mm->max_order; ++i) { in drm_buddy_reset_clear()
436 list_for_each_entry_reverse(block, &mm->free_list[i], link) { in drm_buddy_reset_clear()
440 mm->clear_avail += drm_buddy_block_size(mm, block); in drm_buddy_reset_clear()
443 mm->clear_avail -= drm_buddy_block_size(mm, block); in drm_buddy_reset_clear()
457 void drm_buddy_free_block(struct drm_buddy *mm, in drm_buddy_free_block() argument
461 mm->avail += drm_buddy_block_size(mm, block); in drm_buddy_free_block()
463 mm->clear_avail += drm_buddy_block_size(mm, block); in drm_buddy_free_block()
465 __drm_buddy_free(mm, block, false); in drm_buddy_free_block()
469 static void __drm_buddy_free_list(struct drm_buddy *mm, in __drm_buddy_free_list() argument
483 drm_buddy_free_block(mm, block); in __drm_buddy_free_list()
489 static void drm_buddy_free_list_internal(struct drm_buddy *mm, in drm_buddy_free_list_internal() argument
497 __drm_buddy_free_list(mm, objects, false, false); in drm_buddy_free_list_internal()
507 void drm_buddy_free_list(struct drm_buddy *mm, in drm_buddy_free_list() argument
513 __drm_buddy_free_list(mm, objects, mark_clear, !mark_clear); in drm_buddy_free_list()
525 __alloc_range_bias(struct drm_buddy *mm, in __alloc_range_bias() argument
531 u64 req_size = mm->chunk_size << order; in __alloc_range_bias()
540 for (i = 0; i < mm->n_roots; ++i) in __alloc_range_bias()
541 list_add_tail(&mm->roots[i]->tmp_link, &dfs); in __alloc_range_bias()
559 block_end = block_start + drm_buddy_block_size(mm, block) - 1; in __alloc_range_bias()
591 err = split_block(mm, block); in __alloc_range_bias()
612 __drm_buddy_free(mm, block, false); in __alloc_range_bias()
617 __drm_buddy_alloc_range_bias(struct drm_buddy *mm, in __drm_buddy_alloc_range_bias() argument
625 block = __alloc_range_bias(mm, start, end, order, in __drm_buddy_alloc_range_bias()
628 return __alloc_range_bias(mm, start, end, order, in __drm_buddy_alloc_range_bias()
635 get_maxblock(struct drm_buddy *mm, unsigned int order, in get_maxblock() argument
641 for (i = order; i <= mm->max_order; ++i) { in get_maxblock()
644 list_for_each_entry_reverse(tmp_block, &mm->free_list[i], link) { in get_maxblock()
670 alloc_from_freelist(struct drm_buddy *mm, in alloc_from_freelist() argument
679 block = get_maxblock(mm, order, flags); in alloc_from_freelist()
684 for (tmp = order; tmp <= mm->max_order; ++tmp) { in alloc_from_freelist()
687 list_for_each_entry_reverse(tmp_block, &mm->free_list[tmp], link) { in alloc_from_freelist()
702 for (tmp = order; tmp <= mm->max_order; ++tmp) { in alloc_from_freelist()
703 if (!list_empty(&mm->free_list[tmp])) { in alloc_from_freelist()
704 block = list_last_entry(&mm->free_list[tmp], in alloc_from_freelist()
719 err = split_block(mm, block); in alloc_from_freelist()
730 __drm_buddy_free(mm, block, false); in alloc_from_freelist()
734 static int __alloc_range(struct drm_buddy *mm, in __alloc_range() argument
762 block_end = block_start + drm_buddy_block_size(mm, block) - 1; in __alloc_range()
775 total_allocated += drm_buddy_block_size(mm, block); in __alloc_range()
776 mm->avail -= drm_buddy_block_size(mm, block); in __alloc_range()
778 mm->clear_avail -= drm_buddy_block_size(mm, block); in __alloc_range()
781 } else if (!mm->clear_avail) { in __alloc_range()
788 err = split_block(mm, block); in __alloc_range()
816 __drm_buddy_free(mm, block, false); in __alloc_range()
823 drm_buddy_free_list_internal(mm, &allocated); in __alloc_range()
829 static int __drm_buddy_alloc_range(struct drm_buddy *mm, in __drm_buddy_alloc_range() argument
838 for (i = 0; i < mm->n_roots; ++i) in __drm_buddy_alloc_range()
839 list_add_tail(&mm->roots[i]->tmp_link, &dfs); in __drm_buddy_alloc_range()
841 return __alloc_range(mm, &dfs, start, size, in __drm_buddy_alloc_range()
845 static int __alloc_contig_try_harder(struct drm_buddy *mm, in __alloc_contig_try_harder() argument
860 pages = modify_size >> ilog2(mm->chunk_size); in __alloc_contig_try_harder()
865 list = &mm->free_list[order]; in __alloc_contig_try_harder()
872 err = __drm_buddy_alloc_range(mm, rhs_offset, size, in __alloc_contig_try_harder()
883 err = __drm_buddy_alloc_range(mm, lhs_offset, lhs_size, in __alloc_contig_try_harder()
889 drm_buddy_free_list_internal(mm, blocks); in __alloc_contig_try_harder()
893 drm_buddy_free_list_internal(mm, blocks); in __alloc_contig_try_harder()
918 int drm_buddy_block_trim(struct drm_buddy *mm, in drm_buddy_block_trim() argument
938 block_end = block_start + drm_buddy_block_size(mm, block); in drm_buddy_block_trim()
943 if (new_size > drm_buddy_block_size(mm, block)) in drm_buddy_block_trim()
946 if (!new_size || !IS_ALIGNED(new_size, mm->chunk_size)) in drm_buddy_block_trim()
949 if (new_size == drm_buddy_block_size(mm, block)) in drm_buddy_block_trim()
959 if (!IS_ALIGNED(new_start, mm->chunk_size)) in drm_buddy_block_trim()
967 mark_free(mm, block); in drm_buddy_block_trim()
968 mm->avail += drm_buddy_block_size(mm, block); in drm_buddy_block_trim()
970 mm->clear_avail += drm_buddy_block_size(mm, block); in drm_buddy_block_trim()
977 err = __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL); in drm_buddy_block_trim()
980 mm->avail -= drm_buddy_block_size(mm, block); in drm_buddy_block_trim()
982 mm->clear_avail -= drm_buddy_block_size(mm, block); in drm_buddy_block_trim()
992 __drm_buddy_alloc_blocks(struct drm_buddy *mm, in __drm_buddy_alloc_blocks() argument
999 return __drm_buddy_alloc_range_bias(mm, start, end, in __drm_buddy_alloc_blocks()
1003 return alloc_from_freelist(mm, order, flags); in __drm_buddy_alloc_blocks()
1026 int drm_buddy_alloc_blocks(struct drm_buddy *mm, in drm_buddy_alloc_blocks() argument
1039 if (size < mm->chunk_size) in drm_buddy_alloc_blocks()
1042 if (min_block_size < mm->chunk_size) in drm_buddy_alloc_blocks()
1048 if (!IS_ALIGNED(start | end | size, mm->chunk_size)) in drm_buddy_alloc_blocks()
1051 if (end > mm->size) in drm_buddy_alloc_blocks()
1054 if (range_overflows(start, size, mm->size)) in drm_buddy_alloc_blocks()
1062 return __drm_buddy_alloc_range(mm, start, size, NULL, blocks); in drm_buddy_alloc_blocks()
1077 pages = size >> ilog2(mm->chunk_size); in drm_buddy_alloc_blocks()
1079 min_order = ilog2(min_block_size) - ilog2(mm->chunk_size); in drm_buddy_alloc_blocks()
1083 BUG_ON(order > mm->max_order); in drm_buddy_alloc_blocks()
1087 block = __drm_buddy_alloc_blocks(mm, start, in drm_buddy_alloc_blocks()
1096 if (mm->clear_avail && in drm_buddy_alloc_blocks()
1097 !__force_merge(mm, start, end, min_order)) { in drm_buddy_alloc_blocks()
1098 block = __drm_buddy_alloc_blocks(mm, start, in drm_buddy_alloc_blocks()
1114 return __alloc_contig_try_harder(mm, in drm_buddy_alloc_blocks()
1124 mm->avail -= drm_buddy_block_size(mm, block); in drm_buddy_alloc_blocks()
1126 mm->clear_avail -= drm_buddy_block_size(mm, block); in drm_buddy_alloc_blocks()
1150 trim_size = drm_buddy_block_size(mm, block) - in drm_buddy_alloc_blocks()
1154 drm_buddy_block_trim(mm, in drm_buddy_alloc_blocks()
1167 drm_buddy_free_list_internal(mm, &allocated); in drm_buddy_alloc_blocks()
1179 void drm_buddy_block_print(struct drm_buddy *mm, in drm_buddy_block_print() argument
1184 u64 size = drm_buddy_block_size(mm, block); in drm_buddy_block_print()
1196 void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p) in drm_buddy_print() argument
1201 mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20); in drm_buddy_print()
1203 for (order = mm->max_order; order >= 0; order--) { in drm_buddy_print()
1207 list_for_each_entry(block, &mm->free_list[order], link) { in drm_buddy_print()
1214 free = count * (mm->chunk_size << order); in drm_buddy_print()