Lines Matching refs:adev
53 static int gmc_v11_0_ecc_interrupt_state(struct amdgpu_device *adev, in gmc_v11_0_ecc_interrupt_state() argument
62 gmc_v11_0_vm_fault_interrupt_state(struct amdgpu_device *adev, in gmc_v11_0_vm_fault_interrupt_state() argument
69 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), false); in gmc_v11_0_vm_fault_interrupt_state()
76 if (!adev->in_s0ix && (adev->in_runpm || adev->in_suspend || in gmc_v11_0_vm_fault_interrupt_state()
77 amdgpu_in_reset(adev))) in gmc_v11_0_vm_fault_interrupt_state()
78 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), false); in gmc_v11_0_vm_fault_interrupt_state()
82 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), true); in gmc_v11_0_vm_fault_interrupt_state()
89 if (!adev->in_s0ix) in gmc_v11_0_vm_fault_interrupt_state()
90 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), true); in gmc_v11_0_vm_fault_interrupt_state()
99 static int gmc_v11_0_process_interrupt(struct amdgpu_device *adev, in gmc_v11_0_process_interrupt() argument
105 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub_index]; in gmc_v11_0_process_interrupt()
112 if (!amdgpu_sriov_vf(adev)) { in gmc_v11_0_process_interrupt()
124 amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, in gmc_v11_0_process_interrupt()
131 dev_err(adev->dev, in gmc_v11_0_process_interrupt()
135 task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid); in gmc_v11_0_process_interrupt()
137 amdgpu_vm_print_task_info(adev, task_info); in gmc_v11_0_process_interrupt()
141 dev_err(adev->dev, " in page starting at address 0x%016llx from client %d\n", in gmc_v11_0_process_interrupt()
148 hub->vmhub_funcs->print_l2_protection_fault_status(adev, status); in gmc_v11_0_process_interrupt()
164 static void gmc_v11_0_set_irq_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_irq_funcs() argument
166 adev->gmc.vm_fault.num_types = 1; in gmc_v11_0_set_irq_funcs()
167 adev->gmc.vm_fault.funcs = &gmc_v11_0_irq_funcs; in gmc_v11_0_set_irq_funcs()
169 if (!amdgpu_sriov_vf(adev)) { in gmc_v11_0_set_irq_funcs()
170 adev->gmc.ecc_irq.num_types = 1; in gmc_v11_0_set_irq_funcs()
171 adev->gmc.ecc_irq.funcs = &gmc_v11_0_ecc_funcs; in gmc_v11_0_set_irq_funcs()
182 static bool gmc_v11_0_use_invalidate_semaphore(struct amdgpu_device *adev, in gmc_v11_0_use_invalidate_semaphore() argument
186 (!amdgpu_sriov_vf(adev))); in gmc_v11_0_use_invalidate_semaphore()
190 struct amdgpu_device *adev, in gmc_v11_0_get_vmid_pasid_mapping_info() argument
208 static void gmc_v11_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, in gmc_v11_0_flush_gpu_tlb() argument
211 bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(adev, vmhub); in gmc_v11_0_flush_gpu_tlb()
212 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; in gmc_v11_0_flush_gpu_tlb()
221 if ((vmhub == AMDGPU_GFXHUB(0)) && !adev->gfx.is_poweron) in gmc_v11_0_flush_gpu_tlb()
229 amdgpu_device_flush_hdp(adev, NULL); in gmc_v11_0_flush_gpu_tlb()
234 if ((adev->gfx.kiq[0].ring.sched.ready || adev->mes.ring[0].sched.ready) && in gmc_v11_0_flush_gpu_tlb()
235 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) { in gmc_v11_0_flush_gpu_tlb()
236 amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req, in gmc_v11_0_flush_gpu_tlb()
244 spin_lock(&adev->gmc.invalidate_lock); in gmc_v11_0_flush_gpu_tlb()
254 for (i = 0; i < adev->usec_timeout; i++) { in gmc_v11_0_flush_gpu_tlb()
262 if (i >= adev->usec_timeout) in gmc_v11_0_flush_gpu_tlb()
269 for (i = 0; i < adev->usec_timeout; i++) { in gmc_v11_0_flush_gpu_tlb()
285 !amdgpu_sriov_vf(adev)) { in gmc_v11_0_flush_gpu_tlb()
295 spin_unlock(&adev->gmc.invalidate_lock); in gmc_v11_0_flush_gpu_tlb()
297 if (i >= adev->usec_timeout) in gmc_v11_0_flush_gpu_tlb()
298 dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n"); in gmc_v11_0_flush_gpu_tlb()
312 static void gmc_v11_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, in gmc_v11_0_flush_gpu_tlb_pasid() argument
322 valid = gmc_v11_0_get_vmid_pasid_mapping_info(adev, vmid, in gmc_v11_0_flush_gpu_tlb_pasid()
328 for_each_set_bit(i, adev->vmhubs_mask, in gmc_v11_0_flush_gpu_tlb_pasid()
330 gmc_v11_0_flush_gpu_tlb(adev, vmid, i, in gmc_v11_0_flush_gpu_tlb_pasid()
333 gmc_v11_0_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(0), in gmc_v11_0_flush_gpu_tlb_pasid()
342 bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(ring->adev, ring->vm_hub); in gmc_v11_0_emit_flush_gpu_tlb()
343 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; in gmc_v11_0_emit_flush_gpu_tlb()
390 struct amdgpu_device *adev = ring->adev; in gmc_v11_0_emit_pasid_mapping() local
433 static uint64_t gmc_v11_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) in gmc_v11_0_map_mtype() argument
451 static void gmc_v11_0_get_vm_pde(struct amdgpu_device *adev, int level, in gmc_v11_0_get_vm_pde() argument
455 *addr = amdgpu_gmc_vram_mc2pa(adev, *addr); in gmc_v11_0_get_vm_pde()
458 if (!adev->gmc.translate_further) in gmc_v11_0_get_vm_pde()
474 static void gmc_v11_0_get_vm_pte(struct amdgpu_device *adev, in gmc_v11_0_get_vm_pte() argument
503 static unsigned int gmc_v11_0_get_vbios_fb_size(struct amdgpu_device *adev) in gmc_v11_0_get_vbios_fb_size() argument
536 static void gmc_v11_0_set_gmc_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_gmc_funcs() argument
538 adev->gmc.gmc_funcs = &gmc_v11_0_gmc_funcs; in gmc_v11_0_set_gmc_funcs()
541 static void gmc_v11_0_set_umc_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_umc_funcs() argument
543 switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) { in gmc_v11_0_set_umc_funcs()
545 adev->umc.channel_inst_num = UMC_V8_10_CHANNEL_INSTANCE_NUM; in gmc_v11_0_set_umc_funcs()
546 adev->umc.umc_inst_num = UMC_V8_10_UMC_INSTANCE_NUM; in gmc_v11_0_set_umc_funcs()
547 adev->umc.max_ras_err_cnt_per_query = UMC_V8_10_TOTAL_CHANNEL_NUM(adev); in gmc_v11_0_set_umc_funcs()
548 adev->umc.channel_offs = UMC_V8_10_PER_CHANNEL_OFFSET; in gmc_v11_0_set_umc_funcs()
549 adev->umc.retire_unit = UMC_V8_10_NA_COL_2BITS_POWER_OF_2_NUM; in gmc_v11_0_set_umc_funcs()
550 if (adev->umc.node_inst_num == 4) in gmc_v11_0_set_umc_funcs()
551 adev->umc.channel_idx_tbl = &umc_v8_10_channel_idx_tbl_ext0[0][0][0]; in gmc_v11_0_set_umc_funcs()
553 adev->umc.channel_idx_tbl = &umc_v8_10_channel_idx_tbl[0][0][0]; in gmc_v11_0_set_umc_funcs()
554 adev->umc.ras = &umc_v8_10_ras; in gmc_v11_0_set_umc_funcs()
564 static void gmc_v11_0_set_mmhub_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_mmhub_funcs() argument
566 switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { in gmc_v11_0_set_mmhub_funcs()
568 adev->mmhub.funcs = &mmhub_v3_0_1_funcs; in gmc_v11_0_set_mmhub_funcs()
571 adev->mmhub.funcs = &mmhub_v3_0_2_funcs; in gmc_v11_0_set_mmhub_funcs()
576 adev->mmhub.funcs = &mmhub_v3_3_funcs; in gmc_v11_0_set_mmhub_funcs()
579 adev->mmhub.funcs = &mmhub_v3_0_funcs; in gmc_v11_0_set_mmhub_funcs()
584 static void gmc_v11_0_set_gfxhub_funcs(struct amdgpu_device *adev) in gmc_v11_0_set_gfxhub_funcs() argument
586 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { in gmc_v11_0_set_gfxhub_funcs()
588 adev->gfxhub.funcs = &gfxhub_v3_0_3_funcs; in gmc_v11_0_set_gfxhub_funcs()
594 adev->gfxhub.funcs = &gfxhub_v11_5_0_funcs; in gmc_v11_0_set_gfxhub_funcs()
597 adev->gfxhub.funcs = &gfxhub_v3_0_funcs; in gmc_v11_0_set_gfxhub_funcs()
604 struct amdgpu_device *adev = ip_block->adev; in gmc_v11_0_early_init() local
606 gmc_v11_0_set_gfxhub_funcs(adev); in gmc_v11_0_early_init()
607 gmc_v11_0_set_mmhub_funcs(adev); in gmc_v11_0_early_init()
608 gmc_v11_0_set_gmc_funcs(adev); in gmc_v11_0_early_init()
609 gmc_v11_0_set_irq_funcs(adev); in gmc_v11_0_early_init()
610 gmc_v11_0_set_umc_funcs(adev); in gmc_v11_0_early_init()
612 adev->gmc.shared_aperture_start = 0x2000000000000000ULL; in gmc_v11_0_early_init()
613 adev->gmc.shared_aperture_end = in gmc_v11_0_early_init()
614 adev->gmc.shared_aperture_start + (4ULL << 30) - 1; in gmc_v11_0_early_init()
615 adev->gmc.private_aperture_start = 0x1000000000000000ULL; in gmc_v11_0_early_init()
616 adev->gmc.private_aperture_end = in gmc_v11_0_early_init()
617 adev->gmc.private_aperture_start + (4ULL << 30) - 1; in gmc_v11_0_early_init()
618 adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF; in gmc_v11_0_early_init()
625 struct amdgpu_device *adev = ip_block->adev; in gmc_v11_0_late_init() local
628 r = amdgpu_gmc_allocate_vm_inv_eng(adev); in gmc_v11_0_late_init()
632 r = amdgpu_gmc_ras_late_init(adev); in gmc_v11_0_late_init()
636 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); in gmc_v11_0_late_init()
639 static void gmc_v11_0_vram_gtt_location(struct amdgpu_device *adev, in gmc_v11_0_vram_gtt_location() argument
644 base = adev->mmhub.funcs->get_fb_location(adev); in gmc_v11_0_vram_gtt_location()
646 amdgpu_gmc_set_agp_default(adev, mc); in gmc_v11_0_vram_gtt_location()
647 amdgpu_gmc_vram_location(adev, &adev->gmc, base); in gmc_v11_0_vram_gtt_location()
648 amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_HIGH); in gmc_v11_0_vram_gtt_location()
649 if (!amdgpu_sriov_vf(adev) && in gmc_v11_0_vram_gtt_location()
650 (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(11, 5, 0)) && in gmc_v11_0_vram_gtt_location()
652 amdgpu_gmc_agp_location(adev, mc); in gmc_v11_0_vram_gtt_location()
655 if (amdgpu_sriov_vf(adev)) in gmc_v11_0_vram_gtt_location()
656 adev->vm_manager.vram_base_offset = 0; in gmc_v11_0_vram_gtt_location()
658 adev->vm_manager.vram_base_offset = adev->mmhub.funcs->get_mc_fb_offset(adev); in gmc_v11_0_vram_gtt_location()
670 static int gmc_v11_0_mc_init(struct amdgpu_device *adev) in gmc_v11_0_mc_init() argument
675 adev->gmc.mc_vram_size = in gmc_v11_0_mc_init()
676 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; in gmc_v11_0_mc_init()
677 adev->gmc.real_vram_size = adev->gmc.mc_vram_size; in gmc_v11_0_mc_init()
679 if (!(adev->flags & AMD_IS_APU)) { in gmc_v11_0_mc_init()
680 r = amdgpu_device_resize_fb_bar(adev); in gmc_v11_0_mc_init()
684 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); in gmc_v11_0_mc_init()
685 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); in gmc_v11_0_mc_init()
688 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) { in gmc_v11_0_mc_init()
689 adev->gmc.aper_base = adev->mmhub.funcs->get_mc_fb_offset(adev); in gmc_v11_0_mc_init()
690 adev->gmc.aper_size = adev->gmc.real_vram_size; in gmc_v11_0_mc_init()
694 adev->gmc.visible_vram_size = adev->gmc.aper_size; in gmc_v11_0_mc_init()
695 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) in gmc_v11_0_mc_init()
696 adev->gmc.visible_vram_size = adev->gmc.real_vram_size; in gmc_v11_0_mc_init()
700 adev->gmc.gart_size = 512ULL << 20; in gmc_v11_0_mc_init()
702 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; in gmc_v11_0_mc_init()
704 gmc_v11_0_vram_gtt_location(adev, &adev->gmc); in gmc_v11_0_mc_init()
709 static int gmc_v11_0_gart_init(struct amdgpu_device *adev) in gmc_v11_0_gart_init() argument
713 if (adev->gart.bo) { in gmc_v11_0_gart_init()
719 r = amdgpu_gart_init(adev); in gmc_v11_0_gart_init()
723 adev->gart.table_size = adev->gart.num_gpu_pages * 8; in gmc_v11_0_gart_init()
724 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_UC) | in gmc_v11_0_gart_init()
727 return amdgpu_gart_table_vram_alloc(adev); in gmc_v11_0_gart_init()
733 struct amdgpu_device *adev = ip_block->adev; in gmc_v11_0_sw_init() local
735 adev->mmhub.funcs->init(adev); in gmc_v11_0_sw_init()
737 adev->gfxhub.funcs->init(adev); in gmc_v11_0_sw_init()
739 spin_lock_init(&adev->gmc.invalidate_lock); in gmc_v11_0_sw_init()
741 r = amdgpu_atomfirmware_get_vram_info(adev, in gmc_v11_0_sw_init()
743 adev->gmc.vram_width = vram_width; in gmc_v11_0_sw_init()
745 adev->gmc.vram_type = vram_type; in gmc_v11_0_sw_init()
746 adev->gmc.vram_vendor = vram_vendor; in gmc_v11_0_sw_init()
752 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { in gmc_v11_0_sw_init()
754 adev->gmc.mall_size *= 2; in gmc_v11_0_sw_init()
760 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { in gmc_v11_0_sw_init()
770 set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask); in gmc_v11_0_sw_init()
771 set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask); in gmc_v11_0_sw_init()
777 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); in gmc_v11_0_sw_init()
784 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_VMC, in gmc_v11_0_sw_init()
786 &adev->gmc.vm_fault); in gmc_v11_0_sw_init()
791 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX, in gmc_v11_0_sw_init()
793 &adev->gmc.vm_fault); in gmc_v11_0_sw_init()
797 if (!amdgpu_sriov_vf(adev)) { in gmc_v11_0_sw_init()
799 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_DF, 0, in gmc_v11_0_sw_init()
800 &adev->gmc.ecc_irq); in gmc_v11_0_sw_init()
809 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ in gmc_v11_0_sw_init()
811 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); in gmc_v11_0_sw_init()
813 dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n"); in gmc_v11_0_sw_init()
817 adev->need_swiotlb = drm_need_swiotlb(44); in gmc_v11_0_sw_init()
819 r = gmc_v11_0_mc_init(adev); in gmc_v11_0_sw_init()
823 amdgpu_gmc_get_vbios_allocations(adev); in gmc_v11_0_sw_init()
826 r = amdgpu_bo_init(adev); in gmc_v11_0_sw_init()
830 r = gmc_v11_0_gart_init(adev); in gmc_v11_0_sw_init()
840 adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8; in gmc_v11_0_sw_init()
842 amdgpu_vm_manager_init(adev); in gmc_v11_0_sw_init()
844 r = amdgpu_gmc_ras_sw_init(adev); in gmc_v11_0_sw_init()
858 static void gmc_v11_0_gart_fini(struct amdgpu_device *adev) in gmc_v11_0_gart_fini() argument
860 amdgpu_gart_table_vram_free(adev); in gmc_v11_0_gart_fini()
865 struct amdgpu_device *adev = ip_block->adev; in gmc_v11_0_sw_fini() local
867 amdgpu_vm_manager_fini(adev); in gmc_v11_0_sw_fini()
868 gmc_v11_0_gart_fini(adev); in gmc_v11_0_sw_fini()
869 amdgpu_gem_force_release(adev); in gmc_v11_0_sw_fini()
870 amdgpu_bo_fini(adev); in gmc_v11_0_sw_fini()
875 static void gmc_v11_0_init_golden_registers(struct amdgpu_device *adev) in gmc_v11_0_init_golden_registers() argument
877 if (amdgpu_sriov_vf(adev)) { in gmc_v11_0_init_golden_registers()
878 struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)]; in gmc_v11_0_init_golden_registers()
890 static int gmc_v11_0_gart_enable(struct amdgpu_device *adev) in gmc_v11_0_gart_enable() argument
895 if (adev->gart.bo == NULL) { in gmc_v11_0_gart_enable()
896 dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); in gmc_v11_0_gart_enable()
900 amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr); in gmc_v11_0_gart_enable()
902 r = adev->mmhub.funcs->gart_enable(adev); in gmc_v11_0_gart_enable()
907 amdgpu_device_flush_hdp(adev, NULL); in gmc_v11_0_gart_enable()
912 adev->mmhub.funcs->set_fault_enable_default(adev, value); in gmc_v11_0_gart_enable()
913 gmc_v11_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB0(0), 0); in gmc_v11_0_gart_enable()
916 (unsigned int)(adev->gmc.gart_size >> 20), in gmc_v11_0_gart_enable()
917 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); in gmc_v11_0_gart_enable()
924 struct amdgpu_device *adev = ip_block->adev; in gmc_v11_0_hw_init() local
927 adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode; in gmc_v11_0_hw_init()
930 gmc_v11_0_init_golden_registers(adev); in gmc_v11_0_hw_init()
932 r = gmc_v11_0_gart_enable(adev); in gmc_v11_0_hw_init()
936 if (adev->umc.funcs && adev->umc.funcs->init_registers) in gmc_v11_0_hw_init()
937 adev->umc.funcs->init_registers(adev); in gmc_v11_0_hw_init()
949 static void gmc_v11_0_gart_disable(struct amdgpu_device *adev) in gmc_v11_0_gart_disable() argument
951 adev->mmhub.funcs->gart_disable(adev); in gmc_v11_0_gart_disable()
956 struct amdgpu_device *adev = ip_block->adev; in gmc_v11_0_hw_fini() local
958 if (amdgpu_sriov_vf(adev)) { in gmc_v11_0_hw_fini()
964 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); in gmc_v11_0_hw_fini()
966 if (adev->gmc.ecc_irq.funcs && in gmc_v11_0_hw_fini()
967 amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC)) in gmc_v11_0_hw_fini()
968 amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); in gmc_v11_0_hw_fini()
970 gmc_v11_0_gart_disable(adev); in gmc_v11_0_hw_fini()
990 amdgpu_vmid_reset_all(ip_block->adev); in gmc_v11_0_resume()
1011 struct amdgpu_device *adev = ip_block->adev; in gmc_v11_0_set_clockgating_state() local
1013 r = adev->mmhub.funcs->set_clockgating(adev, state); in gmc_v11_0_set_clockgating_state()
1017 return athub_v3_0_set_clockgating(adev, state); in gmc_v11_0_set_clockgating_state()
1022 struct amdgpu_device *adev = ip_block->adev; in gmc_v11_0_get_clockgating_state() local
1024 adev->mmhub.funcs->get_clockgating(adev, flags); in gmc_v11_0_get_clockgating_state()
1026 athub_v3_0_get_clockgating(adev, flags); in gmc_v11_0_get_clockgating_state()