Lines Matching refs:object
293 struct kmemleak_object *object) in hex_dump_object() argument
295 const u8 *ptr = (const u8 *)object->pointer; in hex_dump_object()
298 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in hex_dump_object()
301 if (object->flags & OBJECT_PERCPU) in hex_dump_object()
302 ptr = (const u8 *)this_cpu_ptr((void __percpu *)object->pointer); in hex_dump_object()
305 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE); in hex_dump_object()
307 if (object->flags & OBJECT_PERCPU) in hex_dump_object()
326 static bool color_white(const struct kmemleak_object *object) in color_white() argument
328 return object->count != KMEMLEAK_BLACK && in color_white()
329 object->count < object->min_count; in color_white()
332 static bool color_gray(const struct kmemleak_object *object) in color_gray() argument
334 return object->min_count != KMEMLEAK_BLACK && in color_gray()
335 object->count >= object->min_count; in color_gray()
343 static bool unreferenced_object(struct kmemleak_object *object) in unreferenced_object() argument
345 return (color_white(object) && object->flags & OBJECT_ALLOCATED) && in unreferenced_object()
346 time_before_eq(object->jiffies + jiffies_min_age, in unreferenced_object()
350 static const char *__object_type_str(struct kmemleak_object *object) in __object_type_str() argument
352 if (object->flags & OBJECT_PHYS) in __object_type_str()
354 if (object->flags & OBJECT_PERCPU) in __object_type_str()
364 struct kmemleak_object *object) in print_unreferenced() argument
370 nr_entries = stack_depot_fetch(object->trace_handle, &entries); in print_unreferenced()
372 __object_type_str(object), in print_unreferenced()
373 object->pointer, object->size); in print_unreferenced()
375 object->comm, object->pid, object->jiffies); in print_unreferenced()
376 hex_dump_object(seq, object); in print_unreferenced()
377 warn_or_seq_printf(seq, " backtrace (crc %x):\n", object->checksum); in print_unreferenced()
390 static void dump_object_info(struct kmemleak_object *object) in dump_object_info() argument
393 __object_type_str(object), object->pointer, object->size); in dump_object_info()
395 object->comm, object->pid, object->jiffies); in dump_object_info()
396 pr_notice(" min_count = %d\n", object->min_count); in dump_object_info()
397 pr_notice(" count = %d\n", object->count); in dump_object_info()
398 pr_notice(" flags = 0x%x\n", object->flags); in dump_object_info()
399 pr_notice(" checksum = %u\n", object->checksum); in dump_object_info()
401 if (object->trace_handle) in dump_object_info()
402 stack_depot_print(object->trace_handle); in dump_object_info()
427 struct kmemleak_object *object; in __lookup_object() local
430 object = rb_entry(rb, struct kmemleak_object, rb_node); in __lookup_object()
431 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in __lookup_object()
434 rb = object->rb_node.rb_left; in __lookup_object()
435 else if (untagged_objp + object->size <= untagged_ptr) in __lookup_object()
436 rb = object->rb_node.rb_right; in __lookup_object()
438 return object; in __lookup_object()
442 dump_object_info(object); in __lookup_object()
461 static int get_object(struct kmemleak_object *object) in get_object() argument
463 return atomic_inc_not_zero(&object->use_count); in get_object()
472 struct kmemleak_object *object; in mem_pool_alloc() local
477 object = kmem_cache_alloc_noprof(object_cache, in mem_pool_alloc()
479 if (object) in mem_pool_alloc()
480 return object; in mem_pool_alloc()
485 object = list_first_entry_or_null(&mem_pool_free_list, in mem_pool_alloc()
486 typeof(*object), object_list); in mem_pool_alloc()
487 if (object) in mem_pool_alloc()
488 list_del(&object->object_list); in mem_pool_alloc()
490 object = &mem_pool[--mem_pool_free_count]; in mem_pool_alloc()
497 return object; in mem_pool_alloc()
503 static void mem_pool_free(struct kmemleak_object *object) in mem_pool_free() argument
507 if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) { in mem_pool_free()
508 kmem_cache_free(object_cache, object); in mem_pool_free()
514 list_add(&object->object_list, &mem_pool_free_list); in mem_pool_free()
525 struct kmemleak_object *object = in free_object_rcu() local
532 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) { in free_object_rcu()
536 mem_pool_free(object); in free_object_rcu()
546 static void put_object(struct kmemleak_object *object) in put_object() argument
548 if (!atomic_dec_and_test(&object->use_count)) in put_object()
552 WARN_ON(object->flags & OBJECT_ALLOCATED); in put_object()
560 call_rcu(&object->rcu, free_object_rcu); in put_object()
562 free_object_rcu(&object->rcu); in put_object()
572 struct kmemleak_object *object; in __find_and_get_object() local
576 object = __lookup_object(ptr, alias, objflags); in __find_and_get_object()
580 if (object && !get_object(object)) in __find_and_get_object()
581 object = NULL; in __find_and_get_object()
584 return object; in __find_and_get_object()
597 static void __remove_object(struct kmemleak_object *object) in __remove_object() argument
599 rb_erase(&object->rb_node, object_tree(object->flags)); in __remove_object()
600 if (!(object->del_state & DELSTATE_NO_DELETE)) in __remove_object()
601 list_del_rcu(&object->object_list); in __remove_object()
602 object->del_state |= DELSTATE_REMOVED; in __remove_object()
609 struct kmemleak_object *object; in __find_and_remove_object() local
611 object = __lookup_object(ptr, alias, objflags); in __find_and_remove_object()
612 if (object) in __find_and_remove_object()
613 __remove_object(object); in __find_and_remove_object()
615 return object; in __find_and_remove_object()
627 struct kmemleak_object *object; in find_and_remove_object() local
630 object = __find_and_remove_object(ptr, alias, objflags); in find_and_remove_object()
633 return object; in find_and_remove_object()
657 struct kmemleak_object *object; in __alloc_object() local
659 object = mem_pool_alloc(gfp); in __alloc_object()
660 if (!object) { in __alloc_object()
666 INIT_LIST_HEAD(&object->object_list); in __alloc_object()
667 INIT_LIST_HEAD(&object->gray_list); in __alloc_object()
668 INIT_HLIST_HEAD(&object->area_list); in __alloc_object()
669 raw_spin_lock_init(&object->lock); in __alloc_object()
670 atomic_set(&object->use_count, 1); in __alloc_object()
671 object->excess_ref = 0; in __alloc_object()
672 object->count = 0; /* white color initially */ in __alloc_object()
673 object->checksum = 0; in __alloc_object()
674 object->del_state = 0; in __alloc_object()
678 object->pid = 0; in __alloc_object()
679 strscpy(object->comm, "hardirq"); in __alloc_object()
681 object->pid = 0; in __alloc_object()
682 strscpy(object->comm, "softirq"); in __alloc_object()
684 object->pid = current->pid; in __alloc_object()
691 strscpy(object->comm, current->comm); in __alloc_object()
695 object->trace_handle = set_track_prepare(); in __alloc_object()
697 return object; in __alloc_object()
700 static int __link_object(struct kmemleak_object *object, unsigned long ptr, in __link_object() argument
709 object->flags = OBJECT_ALLOCATED | objflags; in __link_object()
710 object->pointer = ptr; in __link_object()
711 object->size = kfence_ksize((void *)ptr) ?: size; in __link_object()
712 object->min_count = min_count; in __link_object()
713 object->jiffies = jiffies; in __link_object()
749 rb_link_node(&object->rb_node, rb_parent, link); in __link_object()
750 rb_insert_color(&object->rb_node, object_tree(objflags)); in __link_object()
751 list_add_tail_rcu(&object->object_list, &object_list); in __link_object()
763 struct kmemleak_object *object; in __create_object() local
767 object = __alloc_object(gfp); in __create_object()
768 if (!object) in __create_object()
772 ret = __link_object(object, ptr, size, min_count, objflags); in __create_object()
775 mem_pool_free(object); in __create_object()
802 static void __delete_object(struct kmemleak_object *object) in __delete_object() argument
806 WARN_ON(!(object->flags & OBJECT_ALLOCATED)); in __delete_object()
807 WARN_ON(atomic_read(&object->use_count) < 1); in __delete_object()
813 raw_spin_lock_irqsave(&object->lock, flags); in __delete_object()
814 object->flags &= ~OBJECT_ALLOCATED; in __delete_object()
815 raw_spin_unlock_irqrestore(&object->lock, flags); in __delete_object()
816 put_object(object); in __delete_object()
825 struct kmemleak_object *object; in delete_object_full() local
827 object = find_and_remove_object(ptr, 0, objflags); in delete_object_full()
828 if (!object) { in delete_object_full()
835 __delete_object(object); in delete_object_full()
846 struct kmemleak_object *object, *object_l, *object_r; in delete_object_part() local
858 object = __find_and_remove_object(ptr, 1, objflags); in delete_object_part()
859 if (!object) { in delete_object_part()
872 start = object->pointer; in delete_object_part()
873 end = object->pointer + object->size; in delete_object_part()
876 object->min_count, objflags)) in delete_object_part()
880 object->min_count, objflags)) in delete_object_part()
885 if (object) in delete_object_part()
886 __delete_object(object); in delete_object_part()
895 static void __paint_it(struct kmemleak_object *object, int color) in __paint_it() argument
897 object->min_count = color; in __paint_it()
899 object->flags |= OBJECT_NO_SCAN; in __paint_it()
902 static void paint_it(struct kmemleak_object *object, int color) in paint_it() argument
906 raw_spin_lock_irqsave(&object->lock, flags); in paint_it()
907 __paint_it(object, color); in paint_it()
908 raw_spin_unlock_irqrestore(&object->lock, flags); in paint_it()
913 struct kmemleak_object *object; in paint_ptr() local
915 object = __find_and_get_object(ptr, 0, objflags); in paint_ptr()
916 if (!object) { in paint_ptr()
923 paint_it(object, color); in paint_ptr()
924 put_object(object); in paint_ptr()
952 struct kmemleak_object *object; in reset_checksum() local
954 object = find_and_get_object(ptr, 0); in reset_checksum()
955 if (!object) { in reset_checksum()
961 raw_spin_lock_irqsave(&object->lock, flags); in reset_checksum()
962 object->checksum = 0; in reset_checksum()
963 raw_spin_unlock_irqrestore(&object->lock, flags); in reset_checksum()
964 put_object(object); in reset_checksum()
974 struct kmemleak_object *object; in add_scan_area() local
979 object = find_and_get_object(ptr, 1); in add_scan_area()
980 if (!object) { in add_scan_area()
987 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in add_scan_area()
993 raw_spin_lock_irqsave(&object->lock, flags); in add_scan_area()
997 object->flags |= OBJECT_FULL_SCAN; in add_scan_area()
1001 size = untagged_objp + object->size - untagged_ptr; in add_scan_area()
1002 } else if (untagged_ptr + size > untagged_objp + object->size) { in add_scan_area()
1004 dump_object_info(object); in add_scan_area()
1013 hlist_add_head(&area->node, &object->area_list); in add_scan_area()
1015 raw_spin_unlock_irqrestore(&object->lock, flags); in add_scan_area()
1016 put_object(object); in add_scan_area()
1028 struct kmemleak_object *object; in object_set_excess_ref() local
1030 object = find_and_get_object(ptr, 0); in object_set_excess_ref()
1031 if (!object) { in object_set_excess_ref()
1037 raw_spin_lock_irqsave(&object->lock, flags); in object_set_excess_ref()
1038 object->excess_ref = excess_ref; in object_set_excess_ref()
1039 raw_spin_unlock_irqrestore(&object->lock, flags); in object_set_excess_ref()
1040 put_object(object); in object_set_excess_ref()
1051 struct kmemleak_object *object; in object_no_scan() local
1053 object = find_and_get_object(ptr, 0); in object_no_scan()
1054 if (!object) { in object_no_scan()
1059 raw_spin_lock_irqsave(&object->lock, flags); in object_no_scan()
1060 object->flags |= OBJECT_NO_SCAN; in object_no_scan()
1061 raw_spin_unlock_irqrestore(&object->lock, flags); in object_no_scan()
1062 put_object(object); in object_no_scan()
1192 struct kmemleak_object *object; in kmemleak_update_trace() local
1201 object = find_and_get_object((unsigned long)ptr, 1); in kmemleak_update_trace()
1202 if (!object) { in kmemleak_update_trace()
1211 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_update_trace()
1212 object->trace_handle = trace_handle; in kmemleak_update_trace()
1213 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_update_trace()
1215 put_object(object); in kmemleak_update_trace()
1375 static bool update_checksum(struct kmemleak_object *object) in update_checksum() argument
1377 u32 old_csum = object->checksum; in update_checksum()
1379 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in update_checksum()
1384 if (object->flags & OBJECT_PERCPU) { in update_checksum()
1387 object->checksum = 0; in update_checksum()
1389 void *ptr = per_cpu_ptr((void __percpu *)object->pointer, cpu); in update_checksum()
1391 object->checksum ^= crc32(0, kasan_reset_tag((void *)ptr), object->size); in update_checksum()
1394 object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size); in update_checksum()
1399 return object->checksum != old_csum; in update_checksum()
1405 static void update_refs(struct kmemleak_object *object) in update_refs() argument
1407 if (!color_white(object)) { in update_refs()
1418 object->count++; in update_refs()
1419 if (color_gray(object)) { in update_refs()
1421 WARN_ON(!get_object(object)); in update_refs()
1422 list_add_tail(&object->gray_list, &gray_list); in update_refs()
1429 struct kmemleak_object *object; in pointer_update_refs() local
1448 object = __lookup_object(pointer, 1, objflags); in pointer_update_refs()
1449 if (!object) in pointer_update_refs()
1451 if (object == scanned) in pointer_update_refs()
1460 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in pointer_update_refs()
1462 if (color_gray(object)) { in pointer_update_refs()
1463 excess_ref = object->excess_ref; in pointer_update_refs()
1467 update_refs(object); in pointer_update_refs()
1469 raw_spin_unlock(&object->lock); in pointer_update_refs()
1472 object = lookup_object(excess_ref, 0); in pointer_update_refs()
1473 if (!object) in pointer_update_refs()
1475 if (object == scanned) in pointer_update_refs()
1478 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in pointer_update_refs()
1479 update_refs(object); in pointer_update_refs()
1480 raw_spin_unlock(&object->lock); in pointer_update_refs()
1555 static void scan_object(struct kmemleak_object *object) in scan_object() argument
1564 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1565 if (object->flags & OBJECT_NO_SCAN) in scan_object()
1567 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1571 if (object->flags & OBJECT_PERCPU) { in scan_object()
1575 void *start = per_cpu_ptr((void __percpu *)object->pointer, cpu); in scan_object()
1576 void *end = start + object->size; in scan_object()
1578 scan_block(start, end, object); in scan_object()
1580 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1582 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1583 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1586 } else if (hlist_empty(&object->area_list) || in scan_object()
1587 object->flags & OBJECT_FULL_SCAN) { in scan_object()
1588 void *start = object->flags & OBJECT_PHYS ? in scan_object()
1589 __va((phys_addr_t)object->pointer) : in scan_object()
1590 (void *)object->pointer; in scan_object()
1591 void *end = start + object->size; in scan_object()
1596 scan_block(start, next, object); in scan_object()
1602 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1604 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1605 } while (object->flags & OBJECT_ALLOCATED); in scan_object()
1607 hlist_for_each_entry(area, &object->area_list, node) in scan_object()
1610 object); in scan_object()
1613 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1622 struct kmemleak_object *object, *tmp; in scan_gray_list() local
1629 object = list_entry(gray_list.next, typeof(*object), gray_list); in scan_gray_list()
1630 while (&object->gray_list != &gray_list) { in scan_gray_list()
1635 scan_object(object); in scan_gray_list()
1637 tmp = list_entry(object->gray_list.next, typeof(*object), in scan_gray_list()
1641 list_del(&object->gray_list); in scan_gray_list()
1642 put_object(object); in scan_gray_list()
1644 object = tmp; in scan_gray_list()
1654 static void kmemleak_cond_resched(struct kmemleak_object *object) in kmemleak_cond_resched() argument
1656 if (!get_object(object)) in kmemleak_cond_resched()
1660 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1662 object->del_state |= DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1670 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1671 list_del_rcu(&object->object_list); in kmemleak_cond_resched()
1672 object->del_state &= ~DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1675 put_object(object); in kmemleak_cond_resched()
1685 struct kmemleak_object *object; in kmemleak_scan() local
1694 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_scan()
1695 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1701 if (atomic_read(&object->use_count) > 1) { in kmemleak_scan()
1703 atomic_read(&object->use_count)); in kmemleak_scan()
1704 dump_object_info(object); in kmemleak_scan()
1709 if ((object->flags & OBJECT_PHYS) && in kmemleak_scan()
1710 !(object->flags & OBJECT_NO_SCAN)) { in kmemleak_scan()
1711 unsigned long phys = object->pointer; in kmemleak_scan()
1714 PHYS_PFN(phys + object->size) > max_low_pfn) in kmemleak_scan()
1715 __paint_it(object, KMEMLEAK_BLACK); in kmemleak_scan()
1719 object->count = 0; in kmemleak_scan()
1720 if (color_gray(object) && get_object(object)) in kmemleak_scan()
1721 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1723 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1726 kmemleak_cond_resched(object); in kmemleak_scan()
1794 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_scan()
1796 kmemleak_cond_resched(object); in kmemleak_scan()
1803 if (!color_white(object)) in kmemleak_scan()
1805 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1806 if (color_white(object) && (object->flags & OBJECT_ALLOCATED) in kmemleak_scan()
1807 && update_checksum(object) && get_object(object)) { in kmemleak_scan()
1809 object->count = object->min_count; in kmemleak_scan()
1810 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1812 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1831 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_scan()
1833 kmemleak_cond_resched(object); in kmemleak_scan()
1840 if (!color_white(object)) in kmemleak_scan()
1842 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1843 if (unreferenced_object(object) && in kmemleak_scan()
1844 !(object->flags & OBJECT_REPORTED)) { in kmemleak_scan()
1845 object->flags |= OBJECT_REPORTED; in kmemleak_scan()
1848 print_unreferenced(NULL, object); in kmemleak_scan()
1852 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1936 struct kmemleak_object *object; in kmemleak_seq_start() local
1945 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_seq_start()
1948 if (get_object(object)) in kmemleak_seq_start()
1951 object = NULL; in kmemleak_seq_start()
1953 return object; in kmemleak_seq_start()
2001 struct kmemleak_object *object = v; in kmemleak_seq_show() local
2004 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_seq_show()
2005 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object)) in kmemleak_seq_show()
2006 print_unreferenced(seq, object); in kmemleak_seq_show()
2007 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_seq_show()
2026 struct kmemleak_object *object; in __dump_str_object_info() local
2028 object = __find_and_get_object(addr, 1, objflags); in __dump_str_object_info()
2029 if (!object) in __dump_str_object_info()
2032 raw_spin_lock_irqsave(&object->lock, flags); in __dump_str_object_info()
2033 dump_object_info(object); in __dump_str_object_info()
2034 raw_spin_unlock_irqrestore(&object->lock, flags); in __dump_str_object_info()
2036 put_object(object); in __dump_str_object_info()
2069 struct kmemleak_object *object; in kmemleak_clear() local
2072 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_clear()
2073 raw_spin_lock_irq(&object->lock); in kmemleak_clear()
2074 if ((object->flags & OBJECT_REPORTED) && in kmemleak_clear()
2075 unreferenced_object(object)) in kmemleak_clear()
2076 __paint_it(object, KMEMLEAK_GREY); in kmemleak_clear()
2077 raw_spin_unlock_irq(&object->lock); in kmemleak_clear()
2186 struct kmemleak_object *object, *tmp; in __kmemleak_do_cleanup() local
2193 list_for_each_entry_safe(object, tmp, &object_list, object_list) { in __kmemleak_do_cleanup()
2194 __remove_object(object); in __kmemleak_do_cleanup()
2195 __delete_object(object); in __kmemleak_do_cleanup()