Lines Matching refs:gfx
891 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; in gfx_v9_0_set_kiq_pm4_funcs()
1081 amdgpu_ucode_release(&adev->gfx.pfp_fw); in gfx_v9_0_free_microcode()
1082 amdgpu_ucode_release(&adev->gfx.me_fw); in gfx_v9_0_free_microcode()
1083 amdgpu_ucode_release(&adev->gfx.ce_fw); in gfx_v9_0_free_microcode()
1084 amdgpu_ucode_release(&adev->gfx.rlc_fw); in gfx_v9_0_free_microcode()
1085 amdgpu_ucode_release(&adev->gfx.mec_fw); in gfx_v9_0_free_microcode()
1086 amdgpu_ucode_release(&adev->gfx.mec2_fw); in gfx_v9_0_free_microcode()
1088 kfree(adev->gfx.rlc.register_list_format); in gfx_v9_0_free_microcode()
1093 adev->gfx.me_fw_write_wait = false; in gfx_v9_0_check_fw_write_wait()
1094 adev->gfx.mec_fw_write_wait = false; in gfx_v9_0_check_fw_write_wait()
1097 ((adev->gfx.mec_fw_version < 0x000001a5) || in gfx_v9_0_check_fw_write_wait()
1098 (adev->gfx.mec_feature_version < 46) || in gfx_v9_0_check_fw_write_wait()
1099 (adev->gfx.pfp_fw_version < 0x000000b7) || in gfx_v9_0_check_fw_write_wait()
1100 (adev->gfx.pfp_feature_version < 46))) in gfx_v9_0_check_fw_write_wait()
1105 if ((adev->gfx.me_fw_version >= 0x0000009c) && in gfx_v9_0_check_fw_write_wait()
1106 (adev->gfx.me_feature_version >= 42) && in gfx_v9_0_check_fw_write_wait()
1107 (adev->gfx.pfp_fw_version >= 0x000000b1) && in gfx_v9_0_check_fw_write_wait()
1108 (adev->gfx.pfp_feature_version >= 42)) in gfx_v9_0_check_fw_write_wait()
1109 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1111 if ((adev->gfx.mec_fw_version >= 0x00000193) && in gfx_v9_0_check_fw_write_wait()
1112 (adev->gfx.mec_feature_version >= 42)) in gfx_v9_0_check_fw_write_wait()
1113 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1116 if ((adev->gfx.me_fw_version >= 0x0000009c) && in gfx_v9_0_check_fw_write_wait()
1117 (adev->gfx.me_feature_version >= 44) && in gfx_v9_0_check_fw_write_wait()
1118 (adev->gfx.pfp_fw_version >= 0x000000b2) && in gfx_v9_0_check_fw_write_wait()
1119 (adev->gfx.pfp_feature_version >= 44)) in gfx_v9_0_check_fw_write_wait()
1120 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1122 if ((adev->gfx.mec_fw_version >= 0x00000196) && in gfx_v9_0_check_fw_write_wait()
1123 (adev->gfx.mec_feature_version >= 44)) in gfx_v9_0_check_fw_write_wait()
1124 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1127 if ((adev->gfx.me_fw_version >= 0x0000009c) && in gfx_v9_0_check_fw_write_wait()
1128 (adev->gfx.me_feature_version >= 44) && in gfx_v9_0_check_fw_write_wait()
1129 (adev->gfx.pfp_fw_version >= 0x000000b2) && in gfx_v9_0_check_fw_write_wait()
1130 (adev->gfx.pfp_feature_version >= 44)) in gfx_v9_0_check_fw_write_wait()
1131 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1133 if ((adev->gfx.mec_fw_version >= 0x00000197) && in gfx_v9_0_check_fw_write_wait()
1134 (adev->gfx.mec_feature_version >= 44)) in gfx_v9_0_check_fw_write_wait()
1135 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1139 if ((adev->gfx.me_fw_version >= 0x0000009c) && in gfx_v9_0_check_fw_write_wait()
1140 (adev->gfx.me_feature_version >= 42) && in gfx_v9_0_check_fw_write_wait()
1141 (adev->gfx.pfp_fw_version >= 0x000000b1) && in gfx_v9_0_check_fw_write_wait()
1142 (adev->gfx.pfp_feature_version >= 42)) in gfx_v9_0_check_fw_write_wait()
1143 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1145 if ((adev->gfx.mec_fw_version >= 0x00000192) && in gfx_v9_0_check_fw_write_wait()
1146 (adev->gfx.mec_feature_version >= 42)) in gfx_v9_0_check_fw_write_wait()
1147 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1150 adev->gfx.me_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1151 adev->gfx.mec_fw_write_wait = true; in gfx_v9_0_check_fw_write_wait()
1204 (adev->gfx.me_fw_version >= 0x000000a5) && in check_if_enlarge_doorbell_range()
1205 (adev->gfx.me_feature_version >= 52)) in check_if_enlarge_doorbell_range()
1226 adev->gfx.rlc_fw_version < 531) || in gfx_v9_0_check_if_need_gfxoff()
1227 (adev->gfx.rlc_feature_version < 1) || in gfx_v9_0_check_if_need_gfxoff()
1228 !adev->gfx.rlc.is_rlc_v2_1)) in gfx_v9_0_check_if_need_gfxoff()
1254 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, fw_name); in gfx_v9_0_init_cp_gfx_microcode()
1260 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, fw_name); in gfx_v9_0_init_cp_gfx_microcode()
1266 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, fw_name); in gfx_v9_0_init_cp_gfx_microcode()
1273 amdgpu_ucode_release(&adev->gfx.pfp_fw); in gfx_v9_0_init_cp_gfx_microcode()
1274 amdgpu_ucode_release(&adev->gfx.me_fw); in gfx_v9_0_init_cp_gfx_microcode()
1275 amdgpu_ucode_release(&adev->gfx.ce_fw); in gfx_v9_0_init_cp_gfx_microcode()
1310 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, fw_name); in gfx_v9_0_init_rlc_microcode()
1313 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; in gfx_v9_0_init_rlc_microcode()
1320 amdgpu_ucode_release(&adev->gfx.rlc_fw); in gfx_v9_0_init_rlc_microcode()
1346 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, fw_name); in gfx_v9_0_init_cp_compute_microcode()
1359 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, fw_name); in gfx_v9_0_init_cp_compute_microcode()
1365 amdgpu_ucode_release(&adev->gfx.mec2_fw); in gfx_v9_0_init_cp_compute_microcode()
1368 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version; in gfx_v9_0_init_cp_compute_microcode()
1369 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; in gfx_v9_0_init_cp_compute_microcode()
1377 amdgpu_ucode_release(&adev->gfx.mec_fw); in gfx_v9_0_init_cp_compute_microcode()
1390 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_init_microcode()
1442 if (adev->gfx.rlc.cs_data == NULL) in gfx_v9_0_get_csb_buffer()
1454 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { in gfx_v9_0_get_csb_buffer()
1478 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; in gfx_v9_0_init_always_on_cu_mask()
1492 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v9_0_init_always_on_cu_mask()
1493 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v9_0_init_always_on_cu_mask()
1499 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { in gfx_v9_0_init_always_on_cu_mask()
1635 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl; in gfx_v9_0_init_rlcg_reg_access_ctrl()
1643 adev->gfx.rlc.rlcg_reg_access_supported = true; in gfx_v9_0_init_rlcg_reg_access_ctrl()
1651 adev->gfx.rlc.cs_data = gfx9_cs_data; in gfx_v9_0_rlc_init()
1653 cs_data = adev->gfx.rlc.cs_data; in gfx_v9_0_rlc_init()
1664 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ in gfx_v9_0_rlc_init()
1683 if (adev->gfx.rlc.funcs->update_spm_vmid) in gfx_v9_0_rlc_init()
1684 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); in gfx_v9_0_rlc_init()
1691 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); in gfx_v9_0_mec_fini()
1692 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); in gfx_v9_0_mec_fini()
1706 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); in gfx_v9_0_mec_init()
1710 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; in gfx_v9_0_mec_init()
1715 &adev->gfx.mec.hpd_eop_obj, in gfx_v9_0_mec_init()
1716 &adev->gfx.mec.hpd_eop_gpu_addr, in gfx_v9_0_mec_init()
1726 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); in gfx_v9_0_mec_init()
1727 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); in gfx_v9_0_mec_init()
1730 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; in gfx_v9_0_mec_init()
1733 (adev->gfx.mec_fw->data + in gfx_v9_0_mec_init()
1739 &adev->gfx.mec.mec_fw_obj, in gfx_v9_0_mec_init()
1740 &adev->gfx.mec.mec_fw_gpu_addr, in gfx_v9_0_mec_init()
1750 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); in gfx_v9_0_mec_init()
1751 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); in gfx_v9_0_mec_init()
1855 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
1856 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
1857 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
1858 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
1859 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
1863 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
1864 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
1865 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
1866 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
1867 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
1872 adev->gfx.ras = &gfx_v9_0_ras; in gfx_v9_0_gpu_early_init()
1873 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
1874 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
1875 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
1876 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
1877 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
1888 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
1889 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
1890 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
1891 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
1892 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
1899 adev->gfx.ras = &gfx_v9_4_ras; in gfx_v9_0_gpu_early_init()
1900 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
1901 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
1902 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
1903 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
1904 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
1910 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
1911 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
1912 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
1913 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; in gfx_v9_0_gpu_early_init()
1914 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
1920 adev->gfx.ras = &gfx_v9_4_2_ras; in gfx_v9_0_gpu_early_init()
1921 adev->gfx.config.max_hw_contexts = 8; in gfx_v9_0_gpu_early_init()
1922 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; in gfx_v9_0_gpu_early_init()
1923 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; in gfx_v9_0_gpu_early_init()
1924 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; in gfx_v9_0_gpu_early_init()
1925 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; in gfx_v9_0_gpu_early_init()
1939 adev->gfx.config.gb_addr_config = gb_addr_config; in gfx_v9_0_gpu_early_init()
1941 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << in gfx_v9_0_gpu_early_init()
1943 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
1947 adev->gfx.config.max_tile_pipes = in gfx_v9_0_gpu_early_init()
1948 adev->gfx.config.gb_addr_config_fields.num_pipes; in gfx_v9_0_gpu_early_init()
1950 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << in gfx_v9_0_gpu_early_init()
1952 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
1955 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << in gfx_v9_0_gpu_early_init()
1957 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
1960 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << in gfx_v9_0_gpu_early_init()
1962 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
1965 adev->gfx.config.gb_addr_config_fields.num_se = 1 << in gfx_v9_0_gpu_early_init()
1967 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
1970 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + in gfx_v9_0_gpu_early_init()
1972 adev->gfx.config.gb_addr_config, in gfx_v9_0_gpu_early_init()
1983 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; in gfx_v9_0_compute_ring_init()
1986 ring = &adev->gfx.compute_ring[ring_id]; in gfx_v9_0_compute_ring_init()
1996 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr in gfx_v9_0_compute_ring_init()
2001 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) in gfx_v9_0_compute_ring_init()
2006 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, in gfx_v9_0_compute_ring_init()
2027 adev->gfx.mec.num_mec = 2; in gfx_v9_0_sw_init()
2030 adev->gfx.mec.num_mec = 1; in gfx_v9_0_sw_init()
2034 adev->gfx.mec.num_pipe_per_mec = 4; in gfx_v9_0_sw_init()
2035 adev->gfx.mec.num_queue_per_pipe = 8; in gfx_v9_0_sw_init()
2038 …_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); in gfx_v9_0_sw_init()
2044 &adev->gfx.priv_reg_irq); in gfx_v9_0_sw_init()
2050 &adev->gfx.priv_inst_irq); in gfx_v9_0_sw_init()
2056 &adev->gfx.cp_ecc_error_irq); in gfx_v9_0_sw_init()
2062 &adev->gfx.cp_ecc_error_irq); in gfx_v9_0_sw_init()
2066 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; in gfx_v9_0_sw_init()
2068 if (adev->gfx.rlc.funcs) { in gfx_v9_0_sw_init()
2069 if (adev->gfx.rlc.funcs->init) { in gfx_v9_0_sw_init()
2070 r = adev->gfx.rlc.funcs->init(adev); in gfx_v9_0_sw_init()
2085 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { in gfx_v9_0_sw_init()
2086 ring = &adev->gfx.gfx_ring[i]; in gfx_v9_0_sw_init()
2097 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, in gfx_v9_0_sw_init()
2105 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_sw_init()
2107 ring = &adev->gfx.sw_gfx_ring[i]; in gfx_v9_0_sw_init()
2114 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, in gfx_v9_0_sw_init()
2123 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0], in gfx_v9_0_sw_init()
2130 r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer, in gfx_v9_0_sw_init()
2131 &adev->gfx.sw_gfx_ring[i]); in gfx_v9_0_sw_init()
2141 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { in gfx_v9_0_sw_init()
2142 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { in gfx_v9_0_sw_init()
2143 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { in gfx_v9_0_sw_init()
2164 kiq = &adev->gfx.kiq; in gfx_v9_0_sw_init()
2174 adev->gfx.ce_ram_size = 0x8000; in gfx_v9_0_sw_init()
2194 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_sw_fini()
2196 amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]); in gfx_v9_0_sw_fini()
2197 amdgpu_ring_mux_fini(&adev->gfx.muxer); in gfx_v9_0_sw_fini()
2200 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v9_0_sw_fini()
2201 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); in gfx_v9_0_sw_fini()
2202 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v9_0_sw_fini()
2203 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); in gfx_v9_0_sw_fini()
2206 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); in gfx_v9_0_sw_fini()
2210 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, in gfx_v9_0_sw_fini()
2211 &adev->gfx.rlc.clear_state_gpu_addr, in gfx_v9_0_sw_fini()
2212 (void **)&adev->gfx.rlc.cs_ptr); in gfx_v9_0_sw_fini()
2214 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, in gfx_v9_0_sw_fini()
2215 &adev->gfx.rlc.cp_table_gpu_addr, in gfx_v9_0_sw_fini()
2216 (void **)&adev->gfx.rlc.cp_table_ptr); in gfx_v9_0_sw_fini()
2262 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / in gfx_v9_0_get_rb_active_bitmap()
2263 adev->gfx.config.max_sh_per_se); in gfx_v9_0_get_rb_active_bitmap()
2273 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / in gfx_v9_0_setup_rb()
2274 adev->gfx.config.max_sh_per_se; in gfx_v9_0_setup_rb()
2277 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v9_0_setup_rb()
2278 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v9_0_setup_rb()
2281 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * in gfx_v9_0_setup_rb()
2288 adev->gfx.config.backend_enable_mask = active_rbs; in gfx_v9_0_setup_rb()
2289 adev->gfx.config.num_rbs = hweight32(active_rbs); in gfx_v9_0_setup_rb()
2374 if (adev->gfx.num_gfx_rings) in gfx_v9_0_constants_init()
2376 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); in gfx_v9_0_constants_init()
2377 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); in gfx_v9_0_constants_init()
2420 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v9_0_wait_for_rlc_serdes()
2421 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v9_0_wait_for_rlc_serdes()
2464 if(adev->gfx.num_gfx_rings) in gfx_v9_0_enable_gui_idle_interrupt()
2472 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); in gfx_v9_0_init_csb()
2475 adev->gfx.rlc.clear_state_gpu_addr >> 32); in gfx_v9_0_init_csb()
2477 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); in gfx_v9_0_init_csb()
2479 adev->gfx.rlc.clear_state_size); in gfx_v9_0_init_csb()
2532 kmemdup(adev->gfx.rlc.register_list_format, in gfx_v9_1_init_rlc_save_restore_list()
2533 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); in gfx_v9_1_init_rlc_save_restore_list()
2540 adev->gfx.rlc.reg_list_format_direct_reg_list_length, in gfx_v9_1_init_rlc_save_restore_list()
2541 adev->gfx.rlc.reg_list_format_size_bytes >> 2, in gfx_v9_1_init_rlc_save_restore_list()
2556 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) in gfx_v9_1_init_rlc_save_restore_list()
2558 adev->gfx.rlc.register_restore[i]); in gfx_v9_1_init_rlc_save_restore_list()
2562 adev->gfx.rlc.reg_list_format_start); in gfx_v9_1_init_rlc_save_restore_list()
2565 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) in gfx_v9_1_init_rlc_save_restore_list()
2570 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { in gfx_v9_1_init_rlc_save_restore_list()
2592 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; in gfx_v9_1_init_rlc_save_restore_list()
2595 adev->gfx.rlc.reg_restore_list_size); in gfx_v9_1_init_rlc_save_restore_list()
2600 adev->gfx.rlc.starting_offsets_start); in gfx_v9_1_init_rlc_save_restore_list()
2801 if (adev->gfx.rlc.is_rlc_v2_1) { in gfx_v9_0_init_pg()
2815 adev->gfx.rlc.cp_table_gpu_addr >> 8); in gfx_v9_0_init_pg()
2855 rlc_ucode_ver, adev->gfx.rlc_fw_version); in gfx_v9_0_rlc_start()
2873 if (!adev->gfx.rlc_fw) in gfx_v9_0_rlc_load_microcode()
2876 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; in gfx_v9_0_rlc_load_microcode()
2879 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + in gfx_v9_0_rlc_load_microcode()
2887 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); in gfx_v9_0_rlc_load_microcode()
2901 adev->gfx.rlc.funcs->stop(adev); in gfx_v9_0_rlc_resume()
2933 adev->gfx.rlc.funcs->start(adev); in gfx_v9_0_rlc_resume()
2957 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) in gfx_v9_0_cp_gfx_load_microcode()
2961 adev->gfx.pfp_fw->data; in gfx_v9_0_cp_gfx_load_microcode()
2963 adev->gfx.ce_fw->data; in gfx_v9_0_cp_gfx_load_microcode()
2965 adev->gfx.me_fw->data; in gfx_v9_0_cp_gfx_load_microcode()
2975 (adev->gfx.pfp_fw->data + in gfx_v9_0_cp_gfx_load_microcode()
2981 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); in gfx_v9_0_cp_gfx_load_microcode()
2985 (adev->gfx.ce_fw->data + in gfx_v9_0_cp_gfx_load_microcode()
2991 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); in gfx_v9_0_cp_gfx_load_microcode()
2995 (adev->gfx.me_fw->data + in gfx_v9_0_cp_gfx_load_microcode()
3001 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); in gfx_v9_0_cp_gfx_load_microcode()
3008 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; in gfx_v9_0_cp_gfx_start()
3014 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); in gfx_v9_0_cp_gfx_start()
3082 ring = &adev->gfx.gfx_ring[0]; in gfx_v9_0_cp_gfx_resume()
3145 adev->gfx.kiq.ring.sched.ready = false; in gfx_v9_0_cp_compute_enable()
3157 if (!adev->gfx.mec_fw) in gfx_v9_0_cp_compute_load_microcode()
3162 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; in gfx_v9_0_cp_compute_load_microcode()
3166 (adev->gfx.mec_fw->data + in gfx_v9_0_cp_compute_load_microcode()
3174 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); in gfx_v9_0_cp_compute_load_microcode()
3176 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); in gfx_v9_0_cp_compute_load_microcode()
3186 adev->gfx.mec_fw_version); in gfx_v9_0_cp_compute_load_microcode()
3519 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; in gfx_v9_0_kiq_init_queue()
3522 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v9_0_kiq_init_queue()
3523 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); in gfx_v9_0_kiq_init_queue()
3547 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v9_0_kiq_init_queue()
3548 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); in gfx_v9_0_kiq_init_queue()
3558 int mqd_idx = ring - &adev->gfx.compute_ring[0]; in gfx_v9_0_kcq_init_queue()
3564 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; in gfx_v9_0_kcq_init_queue()
3577 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v9_0_kcq_init_queue()
3578 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); in gfx_v9_0_kcq_init_queue()
3581 if (adev->gfx.mec.mqd_backup[mqd_idx]) in gfx_v9_0_kcq_init_queue()
3582 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); in gfx_v9_0_kcq_init_queue()
3600 ring = &adev->gfx.kiq.ring; in gfx_v9_0_kiq_resume()
3625 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v9_0_kcq_resume()
3626 ring = &adev->gfx.compute_ring[i]; in gfx_v9_0_kcq_resume()
3656 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_cp_resume()
3672 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_cp_resume()
3682 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_cp_resume()
3683 ring = &adev->gfx.gfx_ring[0]; in gfx_v9_0_cp_resume()
3689 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v9_0_cp_resume()
3690 ring = &adev->gfx.compute_ring[i]; in gfx_v9_0_cp_resume()
3719 if (adev->gfx.num_gfx_rings) in gfx_v9_0_cp_enable()
3736 r = adev->gfx.rlc.funcs->resume(adev); in gfx_v9_0_hw_init()
3754 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); in gfx_v9_0_hw_fini()
3755 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v9_0_hw_fini()
3756 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v9_0_hw_fini()
3779 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, in gfx_v9_0_hw_fini()
3780 adev->gfx.kiq.ring.pipe, in gfx_v9_0_hw_fini()
3781 adev->gfx.kiq.ring.queue, 0); in gfx_v9_0_hw_fini()
3782 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); in gfx_v9_0_hw_fini()
3796 adev->gfx.rlc.funcs->stop(adev); in gfx_v9_0_hw_fini()
3868 adev->gfx.rlc.funcs->stop(adev); in gfx_v9_0_soft_reset()
3870 if (adev->gfx.num_gfx_rings) in gfx_v9_0_soft_reset()
3903 struct amdgpu_kiq *kiq = &adev->gfx.kiq; in gfx_v9_0_kiq_read_clock()
3993 mutex_lock(&adev->gfx.gpu_clock_mutex); in gfx_v9_0_get_gpu_clock_counter()
4001 mutex_unlock(&adev->gfx.gpu_clock_mutex); in gfx_v9_0_get_gpu_clock_counter()
4275 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; in gfx_v9_0_do_edc_gds_workarounds()
4322 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; in gfx_v9_0_do_edc_gpr_workarounds()
4329 int compute_dim_x = adev->gfx.config.max_shader_engines * in gfx_v9_0_do_edc_gpr_workarounds()
4330 adev->gfx.config.max_cu_per_sh * in gfx_v9_0_do_edc_gpr_workarounds()
4331 adev->gfx.config.max_sh_per_se; in gfx_v9_0_do_edc_gpr_workarounds()
4333 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; in gfx_v9_0_do_edc_gpr_workarounds()
4496 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; in gfx_v9_0_early_init()
4500 adev->gfx.num_gfx_rings = 0; in gfx_v9_0_early_init()
4502 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; in gfx_v9_0_early_init()
4503 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), in gfx_v9_0_early_init()
4544 if (adev->gfx.ras && in gfx_v9_0_ecc_late_init()
4545 adev->gfx.ras->enable_watchdog_timer) in gfx_v9_0_ecc_late_init()
4546 adev->gfx.ras->enable_watchdog_timer(adev); in gfx_v9_0_ecc_late_init()
4556 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); in gfx_v9_0_late_init()
4560 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); in gfx_v9_0_late_init()
4725 if (!adev->gfx.num_gfx_rings) in gfx_v9_0_update_3d_clock_gating()
5313 gfx[0].gfx_meta_data) + in gfx_v9_0_ring_emit_ce_meta()
5345 struct amdgpu_kiq *kiq = &adev->gfx.kiq; in gfx_v9_0_ring_preempt_ib()
5409 gfx[0].gfx_meta_data) + in gfx_v9_0_ring_emit_de_meta()
5417 gfx[0].gds_backup) + in gfx_v9_0_ring_emit_de_meta()
5570 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; in gfx_v9_0_ring_emit_reg_write_reg_wait()
5791 if (adev->gfx.num_gfx_rings && in gfx_v9_0_eop_irq()
5792 !amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) { in gfx_v9_0_eop_irq()
5795 amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]); in gfx_v9_0_eop_irq()
5800 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v9_0_eop_irq()
5801 ring = &adev->gfx.compute_ring[i]; in gfx_v9_0_eop_irq()
5826 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); in gfx_v9_0_fault()
5830 for (i = 0; i < adev->gfx.num_compute_rings; i++) { in gfx_v9_0_fault()
5831 ring = &adev->gfx.compute_ring[i]; in gfx_v9_0_fault()
6722 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { in gfx_v9_0_emit_wave_limit()
6928 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; in gfx_v9_0_set_ring_funcs()
6930 for (i = 0; i < adev->gfx.num_gfx_rings; i++) in gfx_v9_0_set_ring_funcs()
6931 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; in gfx_v9_0_set_ring_funcs()
6933 if (adev->gfx.num_gfx_rings) { in gfx_v9_0_set_ring_funcs()
6935 adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx; in gfx_v9_0_set_ring_funcs()
6938 for (i = 0; i < adev->gfx.num_compute_rings; i++) in gfx_v9_0_set_ring_funcs()
6939 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; in gfx_v9_0_set_ring_funcs()
6965 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; in gfx_v9_0_set_irq_funcs()
6966 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; in gfx_v9_0_set_irq_funcs()
6968 adev->gfx.priv_reg_irq.num_types = 1; in gfx_v9_0_set_irq_funcs()
6969 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; in gfx_v9_0_set_irq_funcs()
6971 adev->gfx.priv_inst_irq.num_types = 1; in gfx_v9_0_set_irq_funcs()
6972 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; in gfx_v9_0_set_irq_funcs()
6974 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ in gfx_v9_0_set_irq_funcs()
6975 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; in gfx_v9_0_set_irq_funcs()
6989 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; in gfx_v9_0_set_rlc_funcs()
7077 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); in gfx_v9_0_get_cu_active_bitmap()
7095 if (adev->gfx.config.max_shader_engines * in gfx_v9_0_get_cu_info()
7096 adev->gfx.config.max_sh_per_se > 16) in gfx_v9_0_get_cu_info()
7100 adev->gfx.config.max_shader_engines, in gfx_v9_0_get_cu_info()
7101 adev->gfx.config.max_sh_per_se); in gfx_v9_0_get_cu_info()
7104 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { in gfx_v9_0_get_cu_info()
7105 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { in gfx_v9_0_get_cu_info()
7111 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); in gfx_v9_0_get_cu_info()
7128 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { in gfx_v9_0_get_cu_info()
7130 if (counter < adev->gfx.config.max_cu_per_sh) in gfx_v9_0_get_cu_info()