Lines Matching refs:dev_priv
118 static int i915_workqueues_init(struct drm_i915_private *dev_priv) in i915_workqueues_init() argument
134 dev_priv->wq = alloc_ordered_workqueue("i915", 0); in i915_workqueues_init()
135 if (dev_priv->wq == NULL) in i915_workqueues_init()
144 dev_priv->unordered_wq = alloc_workqueue("i915-unordered", 0, 0); in i915_workqueues_init()
145 if (dev_priv->unordered_wq == NULL) in i915_workqueues_init()
151 destroy_workqueue(dev_priv->wq); in i915_workqueues_init()
153 drm_err(&dev_priv->drm, "Failed to allocate workqueues.\n"); in i915_workqueues_init()
158 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv) in i915_workqueues_cleanup() argument
160 destroy_workqueue(dev_priv->unordered_wq); in i915_workqueues_cleanup()
161 destroy_workqueue(dev_priv->wq); in i915_workqueues_cleanup()
174 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) in intel_detect_preproduction_hw() argument
178 pre |= IS_HASWELL_EARLY_SDV(dev_priv); in intel_detect_preproduction_hw()
179 pre |= IS_SKYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x6; in intel_detect_preproduction_hw()
180 pre |= IS_BROXTON(dev_priv) && INTEL_REVID(dev_priv) < 0xA; in intel_detect_preproduction_hw()
181 pre |= IS_KABYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1; in intel_detect_preproduction_hw()
182 pre |= IS_GEMINILAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x3; in intel_detect_preproduction_hw()
183 pre |= IS_ICELAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x7; in intel_detect_preproduction_hw()
184 pre |= IS_TIGERLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1; in intel_detect_preproduction_hw()
185 pre |= IS_DG1(dev_priv) && INTEL_REVID(dev_priv) < 0x1; in intel_detect_preproduction_hw()
186 pre |= IS_DG2_G10(dev_priv) && INTEL_REVID(dev_priv) < 0x8; in intel_detect_preproduction_hw()
187 pre |= IS_DG2_G11(dev_priv) && INTEL_REVID(dev_priv) < 0x5; in intel_detect_preproduction_hw()
188 pre |= IS_DG2_G12(dev_priv) && INTEL_REVID(dev_priv) < 0x1; in intel_detect_preproduction_hw()
191 drm_err(&dev_priv->drm, "This is a pre-production stepping. " in intel_detect_preproduction_hw()
218 static int i915_driver_early_probe(struct drm_i915_private *dev_priv) in i915_driver_early_probe() argument
220 struct intel_display *display = dev_priv->display; in i915_driver_early_probe()
223 if (i915_inject_probe_failure(dev_priv)) in i915_driver_early_probe()
226 intel_device_info_runtime_init_early(dev_priv); in i915_driver_early_probe()
228 intel_step_init(dev_priv); in i915_driver_early_probe()
230 intel_uncore_mmio_debug_init_early(dev_priv); in i915_driver_early_probe()
232 spin_lock_init(&dev_priv->gpu_error.lock); in i915_driver_early_probe()
235 vlv_iosf_sb_init(dev_priv); in i915_driver_early_probe()
236 mutex_init(&dev_priv->sb_lock); in i915_driver_early_probe()
238 i915_memcpy_init_early(dev_priv); in i915_driver_early_probe()
239 intel_runtime_pm_init_early(&dev_priv->runtime_pm); in i915_driver_early_probe()
241 ret = i915_workqueues_init(dev_priv); in i915_driver_early_probe()
245 ret = vlv_suspend_init(dev_priv); in i915_driver_early_probe()
249 ret = intel_region_ttm_device_init(dev_priv); in i915_driver_early_probe()
253 ret = intel_root_gt_init_early(dev_priv); in i915_driver_early_probe()
257 i915_gem_init_early(dev_priv); in i915_driver_early_probe()
259 intel_irq_init(dev_priv); in i915_driver_early_probe()
261 intel_clock_gating_hooks_init(dev_priv); in i915_driver_early_probe()
263 intel_detect_preproduction_hw(dev_priv); in i915_driver_early_probe()
268 intel_region_ttm_device_fini(dev_priv); in i915_driver_early_probe()
270 vlv_suspend_cleanup(dev_priv); in i915_driver_early_probe()
272 i915_workqueues_cleanup(dev_priv); in i915_driver_early_probe()
281 static void i915_driver_late_release(struct drm_i915_private *dev_priv) in i915_driver_late_release() argument
283 struct intel_display *display = dev_priv->display; in i915_driver_late_release()
285 intel_irq_fini(dev_priv); in i915_driver_late_release()
287 i915_gem_cleanup_early(dev_priv); in i915_driver_late_release()
288 intel_gt_driver_late_release_all(dev_priv); in i915_driver_late_release()
289 intel_region_ttm_device_fini(dev_priv); in i915_driver_late_release()
290 vlv_suspend_cleanup(dev_priv); in i915_driver_late_release()
291 i915_workqueues_cleanup(dev_priv); in i915_driver_late_release()
293 mutex_destroy(&dev_priv->sb_lock); in i915_driver_late_release()
294 vlv_iosf_sb_fini(dev_priv); in i915_driver_late_release()
297 i915_params_free(&dev_priv->params); in i915_driver_late_release()
311 static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv) in i915_driver_mmio_probe() argument
313 struct intel_display *display = dev_priv->display; in i915_driver_mmio_probe()
317 if (i915_inject_probe_failure(dev_priv)) in i915_driver_mmio_probe()
320 ret = intel_gmch_bridge_setup(dev_priv); in i915_driver_mmio_probe()
324 for_each_gt(gt, dev_priv, i) { in i915_driver_mmio_probe()
329 ret = drmm_add_action_or_reset(&dev_priv->drm, in i915_driver_mmio_probe()
337 intel_gmch_bar_setup(dev_priv); in i915_driver_mmio_probe()
338 intel_device_info_runtime_init(dev_priv); in i915_driver_mmio_probe()
341 for_each_gt(gt, dev_priv, i) { in i915_driver_mmio_probe()
348 sanitize_gpu(dev_priv); in i915_driver_mmio_probe()
353 intel_gmch_bar_teardown(dev_priv); in i915_driver_mmio_probe()
362 static void i915_driver_mmio_release(struct drm_i915_private *dev_priv) in i915_driver_mmio_release() argument
364 intel_gmch_bar_teardown(dev_priv); in i915_driver_mmio_release()
458 static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) in i915_driver_hw_probe() argument
460 struct intel_display *display = dev_priv->display; in i915_driver_hw_probe()
461 struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); in i915_driver_hw_probe()
464 if (i915_inject_probe_failure(dev_priv)) in i915_driver_hw_probe()
467 if (HAS_PPGTT(dev_priv)) { in i915_driver_hw_probe()
468 if (intel_vgpu_active(dev_priv) && in i915_driver_hw_probe()
469 !intel_vgpu_has_full_ppgtt(dev_priv)) { in i915_driver_hw_probe()
470 drm_err(&dev_priv->drm, in i915_driver_hw_probe()
476 if (HAS_EXECLISTS(dev_priv)) { in i915_driver_hw_probe()
482 if (intel_vgpu_active(dev_priv) && in i915_driver_hw_probe()
483 !intel_vgpu_has_hwsp_emulation(dev_priv)) { in i915_driver_hw_probe()
484 drm_err(&dev_priv->drm, in i915_driver_hw_probe()
491 intel_dram_edram_detect(dev_priv); in i915_driver_hw_probe()
493 ret = i915_set_dma_info(dev_priv); in i915_driver_hw_probe()
497 ret = i915_perf_init(dev_priv); in i915_driver_hw_probe()
501 ret = i915_ggtt_probe_hw(dev_priv); in i915_driver_hw_probe()
505 ret = aperture_remove_conflicting_pci_devices(pdev, dev_priv->drm.driver->name); in i915_driver_hw_probe()
509 ret = i915_ggtt_init_hw(dev_priv); in i915_driver_hw_probe()
517 ret = intel_gt_tiles_init(dev_priv); in i915_driver_hw_probe()
521 ret = intel_memory_regions_hw_probe(dev_priv); in i915_driver_hw_probe()
525 ret = i915_ggtt_enable_hw(dev_priv); in i915_driver_hw_probe()
527 drm_err(&dev_priv->drm, "failed to enable GGTT\n"); in i915_driver_hw_probe()
552 if (GRAPHICS_VER(dev_priv) >= 5) { in i915_driver_hw_probe()
554 drm_dbg(&dev_priv->drm, "can't enable MSI"); in i915_driver_hw_probe()
557 ret = intel_gvt_init(dev_priv); in i915_driver_hw_probe()
563 ret = i915_pcode_init(dev_priv); in i915_driver_hw_probe()
571 ret = intel_dram_detect(dev_priv); in i915_driver_hw_probe()
585 intel_memory_regions_driver_release(dev_priv); in i915_driver_hw_probe()
587 i915_ggtt_driver_release(dev_priv); in i915_driver_hw_probe()
588 i915_gem_drain_freed_objects(dev_priv); in i915_driver_hw_probe()
589 i915_ggtt_driver_late_release(dev_priv); in i915_driver_hw_probe()
591 i915_perf_fini(dev_priv); in i915_driver_hw_probe()
599 static void i915_driver_hw_remove(struct drm_i915_private *dev_priv) in i915_driver_hw_remove() argument
601 struct intel_display *display = dev_priv->display; in i915_driver_hw_remove()
602 struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); in i915_driver_hw_remove()
604 i915_perf_fini(dev_priv); in i915_driver_hw_remove()
619 static int i915_driver_register(struct drm_i915_private *dev_priv) in i915_driver_register() argument
621 struct intel_display *display = dev_priv->display; in i915_driver_register()
626 i915_gem_driver_register(dev_priv); in i915_driver_register()
627 i915_pmu_register(dev_priv); in i915_driver_register()
629 intel_vgpu_register(dev_priv); in i915_driver_register()
632 ret = drm_dev_register(&dev_priv->drm, 0); in i915_driver_register()
634 i915_probe_error(dev_priv, in i915_driver_register()
636 drm_dev_unregister(&dev_priv->drm); in i915_driver_register()
637 i915_pmu_unregister(dev_priv); in i915_driver_register()
638 i915_gem_driver_unregister(dev_priv); in i915_driver_register()
642 i915_debugfs_register(dev_priv); in i915_driver_register()
643 i915_setup_sysfs(dev_priv); in i915_driver_register()
646 i915_perf_register(dev_priv); in i915_driver_register()
648 for_each_gt(gt, dev_priv, i) in i915_driver_register()
651 intel_pxp_debugfs_register(dev_priv->pxp); in i915_driver_register()
653 i915_hwmon_register(dev_priv); in i915_driver_register()
658 intel_runtime_pm_enable(&dev_priv->runtime_pm); in i915_driver_register()
660 if (i915_switcheroo_register(dev_priv)) in i915_driver_register()
661 drm_err(&dev_priv->drm, "Failed to register vga switcheroo!\n"); in i915_driver_register()
670 static void i915_driver_unregister(struct drm_i915_private *dev_priv) in i915_driver_unregister() argument
672 struct intel_display *display = dev_priv->display; in i915_driver_unregister()
676 i915_switcheroo_unregister(dev_priv); in i915_driver_unregister()
678 intel_runtime_pm_disable(&dev_priv->runtime_pm); in i915_driver_unregister()
683 intel_pxp_fini(dev_priv); in i915_driver_unregister()
685 for_each_gt(gt, dev_priv, i) in i915_driver_unregister()
688 i915_hwmon_unregister(dev_priv); in i915_driver_unregister()
690 i915_perf_unregister(dev_priv); in i915_driver_unregister()
691 i915_pmu_unregister(dev_priv); in i915_driver_unregister()
693 i915_teardown_sysfs(dev_priv); in i915_driver_unregister()
694 drm_dev_unplug(&dev_priv->drm); in i915_driver_unregister()
696 i915_gem_driver_unregister(dev_priv); in i915_driver_unregister()
706 static void i915_welcome_messages(struct drm_i915_private *dev_priv) in i915_welcome_messages() argument
709 struct drm_printer p = drm_dbg_printer(&dev_priv->drm, DRM_UT_DRIVER, in i915_welcome_messages()
715 INTEL_DEVID(dev_priv), in i915_welcome_messages()
716 INTEL_REVID(dev_priv), in i915_welcome_messages()
717 intel_platform_name(INTEL_INFO(dev_priv)->platform), in i915_welcome_messages()
718 intel_subplatform(RUNTIME_INFO(dev_priv), in i915_welcome_messages()
719 INTEL_INFO(dev_priv)->platform), in i915_welcome_messages()
720 GRAPHICS_VER(dev_priv)); in i915_welcome_messages()
722 intel_device_info_print(INTEL_INFO(dev_priv), in i915_welcome_messages()
723 RUNTIME_INFO(dev_priv), &p); in i915_welcome_messages()
724 i915_print_iommu_status(dev_priv, &p); in i915_welcome_messages()
725 for_each_gt(gt, dev_priv, i) in i915_welcome_messages()
730 drm_info(&dev_priv->drm, "DRM_I915_DEBUG enabled\n"); in i915_welcome_messages()
732 drm_info(&dev_priv->drm, "DRM_I915_DEBUG_GEM enabled\n"); in i915_welcome_messages()
734 drm_info(&dev_priv->drm, in i915_welcome_messages()
921 struct drm_i915_private *dev_priv = to_i915(dev); in i915_driver_release() local
922 struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; in i915_driver_release()
925 if (!dev_priv->do_release) in i915_driver_release()
930 i915_gem_driver_release(dev_priv); in i915_driver_release()
932 intel_memory_regions_driver_release(dev_priv); in i915_driver_release()
933 i915_ggtt_driver_release(dev_priv); in i915_driver_release()
934 i915_gem_drain_freed_objects(dev_priv); in i915_driver_release()
935 i915_ggtt_driver_late_release(dev_priv); in i915_driver_release()
937 i915_driver_mmio_release(dev_priv); in i915_driver_release()
943 i915_driver_late_release(dev_priv); in i915_driver_release()
1019 static bool suspend_to_idle(struct drm_i915_private *dev_priv) in suspend_to_idle() argument
1052 struct drm_i915_private *dev_priv = to_i915(dev); in i915_drm_suspend() local
1053 struct intel_display *display = dev_priv->display; in i915_drm_suspend()
1054 struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); in i915_drm_suspend()
1057 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); in i915_drm_suspend()
1063 if (HAS_DISPLAY(dev_priv)) { in i915_drm_suspend()
1072 intel_irq_suspend(dev_priv); in i915_drm_suspend()
1075 if (HAS_DISPLAY(dev_priv)) in i915_drm_suspend()
1082 i915_ggtt_suspend(to_gt(dev_priv)->ggtt); in i915_drm_suspend()
1086 opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold; in i915_drm_suspend()
1089 dev_priv->suspend_count++; in i915_drm_suspend()
1093 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); in i915_drm_suspend()
1095 i915_gem_drain_freed_objects(dev_priv); in i915_drm_suspend()
1102 struct drm_i915_private *dev_priv = to_i915(dev); in i915_drm_suspend_late() local
1103 struct intel_display *display = dev_priv->display; in i915_drm_suspend_late()
1104 struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); in i915_drm_suspend_late()
1105 struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; in i915_drm_suspend_late()
1108 bool s2idle = !hibernation && suspend_to_idle(dev_priv); in i915_drm_suspend_late()
1112 intel_pxp_suspend(dev_priv->pxp); in i915_drm_suspend_late()
1114 i915_gem_suspend_late(dev_priv); in i915_drm_suspend_late()
1116 for_each_gt(gt, dev_priv, i) in i915_drm_suspend_late()
1121 ret = vlv_suspend_complete(dev_priv); in i915_drm_suspend_late()
1123 drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret); in i915_drm_suspend_late()
1142 if (!(hibernation && GRAPHICS_VER(dev_priv) < 6)) in i915_drm_suspend_late()
1147 if (!dev_priv->uncore.user_forcewake_count) in i915_drm_suspend_late()
1174 struct drm_i915_private *dev_priv = to_i915(dev); in i915_drm_resume() local
1175 struct intel_display *display = dev_priv->display; in i915_drm_resume()
1179 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); in i915_drm_resume()
1181 ret = i915_pcode_init(dev_priv); in i915_drm_resume()
1185 sanitize_gpu(dev_priv); in i915_drm_resume()
1187 ret = i915_ggtt_enable_hw(dev_priv); in i915_drm_resume()
1189 drm_err(&dev_priv->drm, "failed to re-enable GGTT\n"); in i915_drm_resume()
1191 i915_ggtt_resume(to_gt(dev_priv)->ggtt); in i915_drm_resume()
1193 for_each_gt(gt, dev_priv, i) in i915_drm_resume()
1220 intel_irq_resume(dev_priv); in i915_drm_resume()
1222 if (HAS_DISPLAY(dev_priv)) in i915_drm_resume()
1225 i915_gem_resume(dev_priv); in i915_drm_resume()
1229 intel_clock_gating_init(dev_priv); in i915_drm_resume()
1231 if (HAS_DISPLAY(dev_priv)) in i915_drm_resume()
1238 if (HAS_DISPLAY(dev_priv)) { in i915_drm_resume()
1250 intel_gvt_resume(dev_priv); in i915_drm_resume()
1252 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); in i915_drm_resume()
1259 struct drm_i915_private *dev_priv = to_i915(dev); in i915_drm_resume_early() local
1260 struct intel_display *display = dev_priv->display; in i915_drm_resume_early()
1261 struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); in i915_drm_resume_early()
1287 drm_err(&dev_priv->drm, in i915_drm_resume_early()
1310 disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); in i915_drm_resume_early()
1312 ret = vlv_resume_prepare(dev_priv, false); in i915_drm_resume_early()
1314 drm_err(&dev_priv->drm, in i915_drm_resume_early()
1317 for_each_gt(gt, dev_priv, i) in i915_drm_resume_early()
1322 enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); in i915_drm_resume_early()
1491 struct drm_i915_private *dev_priv = kdev_to_i915(kdev); in intel_runtime_suspend() local
1492 struct intel_display *display = dev_priv->display; in intel_runtime_suspend()
1493 struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; in intel_runtime_suspend()
1494 struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); in intel_runtime_suspend()
1499 if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv))) in intel_runtime_suspend()
1502 drm_dbg(&dev_priv->drm, "Suspending device\n"); in intel_runtime_suspend()
1510 i915_gem_runtime_suspend(dev_priv); in intel_runtime_suspend()
1512 intel_pxp_runtime_suspend(dev_priv->pxp); in intel_runtime_suspend()
1514 for_each_gt(gt, dev_priv, i) in intel_runtime_suspend()
1517 intel_irq_suspend(dev_priv); in intel_runtime_suspend()
1519 for_each_gt(gt, dev_priv, i) in intel_runtime_suspend()
1524 ret = vlv_suspend_complete(dev_priv); in intel_runtime_suspend()
1526 drm_err(&dev_priv->drm, in intel_runtime_suspend()
1528 intel_uncore_runtime_resume(&dev_priv->uncore); in intel_runtime_suspend()
1530 intel_irq_resume(dev_priv); in intel_runtime_suspend()
1532 for_each_gt(gt, dev_priv, i) in intel_runtime_suspend()
1543 if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore)) in intel_runtime_suspend()
1544 drm_err(&dev_priv->drm, in intel_runtime_suspend()
1560 if (IS_BROADWELL(dev_priv)) { in intel_runtime_suspend()
1579 assert_forcewakes_inactive(&dev_priv->uncore); in intel_runtime_suspend()
1581 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) in intel_runtime_suspend()
1584 drm_dbg(&dev_priv->drm, "Device suspended\n"); in intel_runtime_suspend()
1590 struct drm_i915_private *dev_priv = kdev_to_i915(kdev); in intel_runtime_resume() local
1591 struct intel_display *display = dev_priv->display; in intel_runtime_resume()
1592 struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; in intel_runtime_resume()
1593 struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); in intel_runtime_resume()
1598 if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv))) in intel_runtime_resume()
1601 drm_dbg(&dev_priv->drm, "Resuming device\n"); in intel_runtime_resume()
1603 drm_WARN_ON_ONCE(&dev_priv->drm, atomic_read(&rpm->wakeref_count)); in intel_runtime_resume()
1612 if (intel_uncore_unclaimed_mmio(&dev_priv->uncore)) in intel_runtime_resume()
1613 drm_dbg(&dev_priv->drm, in intel_runtime_resume()
1618 ret = vlv_resume_prepare(dev_priv, true); in intel_runtime_resume()
1620 for_each_gt(gt, dev_priv, i) in intel_runtime_resume()
1623 intel_irq_resume(dev_priv); in intel_runtime_resume()
1629 for_each_gt(gt, dev_priv, i) in intel_runtime_resume()
1632 intel_pxp_runtime_resume(dev_priv->pxp); in intel_runtime_resume()
1639 if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) { in intel_runtime_resume()
1649 drm_err(&dev_priv->drm, in intel_runtime_resume()
1652 drm_dbg(&dev_priv->drm, "Device resumed\n"); in intel_runtime_resume()