Lines Matching refs:bo
58 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); in amdgpu_bo_destroy() local
60 amdgpu_bo_kunmap(bo); in amdgpu_bo_destroy()
62 if (bo->tbo.base.import_attach) in amdgpu_bo_destroy()
63 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); in amdgpu_bo_destroy()
64 drm_gem_object_release(&bo->tbo.base); in amdgpu_bo_destroy()
65 amdgpu_bo_unref(&bo->parent); in amdgpu_bo_destroy()
66 kvfree(bo); in amdgpu_bo_destroy()
71 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); in amdgpu_bo_user_destroy() local
74 ubo = to_amdgpu_bo_user(bo); in amdgpu_bo_user_destroy()
82 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); in amdgpu_bo_vm_destroy() local
85 vmbo = to_amdgpu_bo_vm(bo); in amdgpu_bo_vm_destroy()
106 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) in amdgpu_bo_is_amdgpu_bo() argument
108 if (bo->destroy == &amdgpu_bo_destroy || in amdgpu_bo_is_amdgpu_bo()
109 bo->destroy == &amdgpu_bo_user_destroy || in amdgpu_bo_is_amdgpu_bo()
110 bo->destroy == &amdgpu_bo_vm_destroy) in amdgpu_bo_is_amdgpu_bo()
419 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, in amdgpu_bo_free_kernel() argument
422 if (*bo == NULL) in amdgpu_bo_free_kernel()
425 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); in amdgpu_bo_free_kernel()
427 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { in amdgpu_bo_free_kernel()
429 amdgpu_bo_kunmap(*bo); in amdgpu_bo_free_kernel()
431 amdgpu_bo_unpin(*bo); in amdgpu_bo_free_kernel()
432 amdgpu_bo_unreserve(*bo); in amdgpu_bo_free_kernel()
434 amdgpu_bo_unref(bo); in amdgpu_bo_free_kernel()
536 struct amdgpu_bo *bo; in amdgpu_bo_create() local
562 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); in amdgpu_bo_create()
563 if (bo == NULL) in amdgpu_bo_create()
565 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); in amdgpu_bo_create()
566 bo->vm_bo = NULL; in amdgpu_bo_create()
567 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : in amdgpu_bo_create()
569 bo->allowed_domains = bo->preferred_domains; in amdgpu_bo_create()
572 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) in amdgpu_bo_create()
573 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; in amdgpu_bo_create()
575 bo->flags = bp->flags; in amdgpu_bo_create()
577 if (!amdgpu_bo_support_uswc(bo->flags)) in amdgpu_bo_create()
578 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; in amdgpu_bo_create()
581 bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; in amdgpu_bo_create()
583 bo->tbo.bdev = &adev->mman.bdev; in amdgpu_bo_create()
586 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); in amdgpu_bo_create()
588 amdgpu_bo_placement_from_domain(bo, bp->domain); in amdgpu_bo_create()
590 bo->tbo.priority = 1; in amdgpu_bo_create()
595 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, in amdgpu_bo_create()
596 &bo->placement, page_align, &ctx, NULL, in amdgpu_bo_create()
602 bo->tbo.resource->mem_type == TTM_PL_VRAM && in amdgpu_bo_create()
603 bo->tbo.resource->start < adev->gmc.visible_vram_size >> PAGE_SHIFT) in amdgpu_bo_create()
610 bo->tbo.resource->mem_type == TTM_PL_VRAM) { in amdgpu_bo_create()
613 r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence); in amdgpu_bo_create()
617 dma_resv_add_fence(bo->tbo.base.resv, fence, in amdgpu_bo_create()
622 amdgpu_bo_unreserve(bo); in amdgpu_bo_create()
623 *bo_ptr = bo; in amdgpu_bo_create()
625 trace_amdgpu_bo_create(bo); in amdgpu_bo_create()
629 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; in amdgpu_bo_create()
635 dma_resv_unlock(bo->tbo.base.resv); in amdgpu_bo_create()
636 amdgpu_bo_unref(&bo); in amdgpu_bo_create()
714 struct amdgpu_device *adev = amdgpu_ttm_adev(vmbo->bo.tbo.bdev); in amdgpu_bo_add_to_shadow_list()
760 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) in amdgpu_bo_kmap() argument
765 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) in amdgpu_bo_kmap()
768 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, in amdgpu_bo_kmap()
773 kptr = amdgpu_bo_kptr(bo); in amdgpu_bo_kmap()
780 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); in amdgpu_bo_kmap()
785 *ptr = amdgpu_bo_kptr(bo); in amdgpu_bo_kmap()
799 void *amdgpu_bo_kptr(struct amdgpu_bo *bo) in amdgpu_bo_kptr() argument
803 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); in amdgpu_bo_kptr()
812 void amdgpu_bo_kunmap(struct amdgpu_bo *bo) in amdgpu_bo_kunmap() argument
814 if (bo->kmap.bo) in amdgpu_bo_kunmap()
815 ttm_bo_kunmap(&bo->kmap); in amdgpu_bo_kunmap()
827 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) in amdgpu_bo_ref() argument
829 if (bo == NULL) in amdgpu_bo_ref()
832 ttm_bo_get(&bo->tbo); in amdgpu_bo_ref()
833 return bo; in amdgpu_bo_ref()
842 void amdgpu_bo_unref(struct amdgpu_bo **bo) in amdgpu_bo_unref() argument
846 if ((*bo) == NULL) in amdgpu_bo_unref()
849 tbo = &((*bo)->tbo); in amdgpu_bo_unref()
851 *bo = NULL; in amdgpu_bo_unref()
876 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, in amdgpu_bo_pin_restricted() argument
879 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_bo_pin_restricted()
883 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) in amdgpu_bo_pin_restricted()
890 if (bo->preferred_domains & domain) in amdgpu_bo_pin_restricted()
891 domain = bo->preferred_domains & domain; in amdgpu_bo_pin_restricted()
894 if (bo->tbo.base.import_attach) { in amdgpu_bo_pin_restricted()
901 if (bo->tbo.pin_count) { in amdgpu_bo_pin_restricted()
902 uint32_t mem_type = bo->tbo.resource->mem_type; in amdgpu_bo_pin_restricted()
903 uint32_t mem_flags = bo->tbo.resource->placement; in amdgpu_bo_pin_restricted()
909 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && in amdgpu_bo_pin_restricted()
913 ttm_bo_pin(&bo->tbo); in amdgpu_bo_pin_restricted()
919 (amdgpu_bo_gpu_offset(bo) - domain_start)); in amdgpu_bo_pin_restricted()
930 if (bo->tbo.base.import_attach) in amdgpu_bo_pin_restricted()
931 dma_buf_pin(bo->tbo.base.import_attach); in amdgpu_bo_pin_restricted()
934 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) in amdgpu_bo_pin_restricted()
935 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; in amdgpu_bo_pin_restricted()
936 amdgpu_bo_placement_from_domain(bo, domain); in amdgpu_bo_pin_restricted()
937 for (i = 0; i < bo->placement.num_placement; i++) { in amdgpu_bo_pin_restricted()
943 if (fpfn > bo->placements[i].fpfn) in amdgpu_bo_pin_restricted()
944 bo->placements[i].fpfn = fpfn; in amdgpu_bo_pin_restricted()
945 if (!bo->placements[i].lpfn || in amdgpu_bo_pin_restricted()
946 (lpfn && lpfn < bo->placements[i].lpfn)) in amdgpu_bo_pin_restricted()
947 bo->placements[i].lpfn = lpfn; in amdgpu_bo_pin_restricted()
950 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in amdgpu_bo_pin_restricted()
952 dev_err(adev->dev, "%p pin failed\n", bo); in amdgpu_bo_pin_restricted()
956 ttm_bo_pin(&bo->tbo); in amdgpu_bo_pin_restricted()
958 domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); in amdgpu_bo_pin_restricted()
960 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); in amdgpu_bo_pin_restricted()
961 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), in amdgpu_bo_pin_restricted()
964 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); in amdgpu_bo_pin_restricted()
983 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) in amdgpu_bo_pin() argument
985 bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; in amdgpu_bo_pin()
986 return amdgpu_bo_pin_restricted(bo, domain, 0, 0); in amdgpu_bo_pin()
999 void amdgpu_bo_unpin(struct amdgpu_bo *bo) in amdgpu_bo_unpin() argument
1001 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_bo_unpin()
1003 ttm_bo_unpin(&bo->tbo); in amdgpu_bo_unpin()
1004 if (bo->tbo.pin_count) in amdgpu_bo_unpin()
1007 if (bo->tbo.base.import_attach) in amdgpu_bo_unpin()
1008 dma_buf_unpin(bo->tbo.base.import_attach); in amdgpu_bo_unpin()
1010 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { in amdgpu_bo_unpin()
1011 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); in amdgpu_bo_unpin()
1012 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), in amdgpu_bo_unpin()
1014 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { in amdgpu_bo_unpin()
1015 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); in amdgpu_bo_unpin()
1103 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) in amdgpu_bo_set_tiling_flags() argument
1105 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_bo_set_tiling_flags()
1108 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); in amdgpu_bo_set_tiling_flags()
1113 ubo = to_amdgpu_bo_user(bo); in amdgpu_bo_set_tiling_flags()
1126 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) in amdgpu_bo_get_tiling_flags() argument
1130 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); in amdgpu_bo_get_tiling_flags()
1131 dma_resv_assert_held(bo->tbo.base.resv); in amdgpu_bo_get_tiling_flags()
1132 ubo = to_amdgpu_bo_user(bo); in amdgpu_bo_get_tiling_flags()
1151 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata, in amdgpu_bo_set_metadata() argument
1157 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); in amdgpu_bo_set_metadata()
1158 ubo = to_amdgpu_bo_user(bo); in amdgpu_bo_set_metadata()
1198 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, in amdgpu_bo_get_metadata() argument
1207 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); in amdgpu_bo_get_metadata()
1208 ubo = to_amdgpu_bo_user(bo); in amdgpu_bo_get_metadata()
1236 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, in amdgpu_bo_move_notify() argument
1240 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_bo_move_notify()
1242 struct ttm_resource *old_mem = bo->resource; in amdgpu_bo_move_notify()
1244 if (!amdgpu_bo_is_amdgpu_bo(bo)) in amdgpu_bo_move_notify()
1247 abo = ttm_to_amdgpu_bo(bo); in amdgpu_bo_move_notify()
1253 bo->resource->mem_type != TTM_PL_SYSTEM) in amdgpu_bo_move_notify()
1268 void amdgpu_bo_get_memory(struct amdgpu_bo *bo, uint64_t *vram_mem, in amdgpu_bo_get_memory() argument
1273 domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); in amdgpu_bo_get_memory()
1276 *vram_mem += amdgpu_bo_size(bo); in amdgpu_bo_get_memory()
1279 *gtt_mem += amdgpu_bo_size(bo); in amdgpu_bo_get_memory()
1283 *cpu_mem += amdgpu_bo_size(bo); in amdgpu_bo_get_memory()
1295 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) in amdgpu_bo_release_notify() argument
1297 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_bo_release_notify()
1302 if (!amdgpu_bo_is_amdgpu_bo(bo)) in amdgpu_bo_release_notify()
1305 abo = ttm_to_amdgpu_bo(bo); in amdgpu_bo_release_notify()
1311 WARN_ON_ONCE(bo->type == ttm_bo_type_kernel in amdgpu_bo_release_notify()
1312 && bo->base.resv != &bo->base._resv); in amdgpu_bo_release_notify()
1313 if (bo->base.resv == &bo->base._resv) in amdgpu_bo_release_notify()
1316 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || in amdgpu_bo_release_notify()
1321 if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv))) in amdgpu_bo_release_notify()
1324 r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence); in amdgpu_bo_release_notify()
1330 dma_resv_unlock(bo->base.resv); in amdgpu_bo_release_notify()
1344 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) in amdgpu_bo_fault_reserve_notify() argument
1346 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); in amdgpu_bo_fault_reserve_notify()
1348 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); in amdgpu_bo_fault_reserve_notify()
1355 if (bo->resource->mem_type != TTM_PL_VRAM) in amdgpu_bo_fault_reserve_notify()
1358 offset = bo->resource->start << PAGE_SHIFT; in amdgpu_bo_fault_reserve_notify()
1359 if ((offset + bo->base.size) <= adev->gmc.visible_vram_size) in amdgpu_bo_fault_reserve_notify()
1375 r = ttm_bo_validate(bo, &abo->placement, &ctx); in amdgpu_bo_fault_reserve_notify()
1381 offset = bo->resource->start << PAGE_SHIFT; in amdgpu_bo_fault_reserve_notify()
1383 if (bo->resource->mem_type == TTM_PL_VRAM && in amdgpu_bo_fault_reserve_notify()
1384 (offset + bo->base.size) > adev->gmc.visible_vram_size) in amdgpu_bo_fault_reserve_notify()
1387 ttm_bo_move_to_lru_tail_unlocked(bo); in amdgpu_bo_fault_reserve_notify()
1399 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, in amdgpu_bo_fence() argument
1402 struct dma_resv *resv = bo->tbo.base.resv; in amdgpu_bo_fence()
1454 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) in amdgpu_bo_sync_wait() argument
1456 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_bo_sync_wait()
1458 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, in amdgpu_bo_sync_wait()
1472 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) in amdgpu_bo_gpu_offset() argument
1474 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); in amdgpu_bo_gpu_offset()
1475 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && in amdgpu_bo_gpu_offset()
1476 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); in amdgpu_bo_gpu_offset()
1477 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); in amdgpu_bo_gpu_offset()
1478 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && in amdgpu_bo_gpu_offset()
1479 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); in amdgpu_bo_gpu_offset()
1481 return amdgpu_bo_gpu_offset_no_check(bo); in amdgpu_bo_gpu_offset()
1491 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) in amdgpu_bo_gpu_offset_no_check() argument
1493 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_bo_gpu_offset_no_check()
1496 offset = (bo->tbo.resource->start << PAGE_SHIFT) + in amdgpu_bo_gpu_offset_no_check()
1497 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); in amdgpu_bo_gpu_offset_no_check()
1523 #define amdgpu_bo_print_flag(m, bo, flag) \ argument
1525 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \
1542 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) in amdgpu_bo_print_info() argument
1551 domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); in amdgpu_bo_print_info()
1565 size = amdgpu_bo_size(bo); in amdgpu_bo_print_info()
1569 pin_count = READ_ONCE(bo->tbo.pin_count); in amdgpu_bo_print_info()
1573 dma_buf = READ_ONCE(bo->tbo.base.dma_buf); in amdgpu_bo_print_info()
1574 attachment = READ_ONCE(bo->tbo.base.import_attach); in amdgpu_bo_print_info()
1581 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); in amdgpu_bo_print_info()
1582 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); in amdgpu_bo_print_info()
1583 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); in amdgpu_bo_print_info()
1584 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); in amdgpu_bo_print_info()
1585 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); in amdgpu_bo_print_info()
1586 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); in amdgpu_bo_print_info()
1587 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); in amdgpu_bo_print_info()