1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6 #include <linux/highmem.h>
7 #include <linux/moduleparam.h>
8 #include <linux/pci.h>
9 #include <linux/pm_runtime.h>
10 #include <linux/reboot.h>
11
12 #include "vpu_boot_api.h"
13 #include "ivpu_drv.h"
14 #include "ivpu_hw.h"
15 #include "ivpu_fw.h"
16 #include "ivpu_ipc.h"
17 #include "ivpu_job.h"
18 #include "ivpu_mmu.h"
19 #include "ivpu_pm.h"
20
21 static bool ivpu_disable_recovery;
22 module_param_named_unsafe(disable_recovery, ivpu_disable_recovery, bool, 0644);
23 MODULE_PARM_DESC(disable_recovery, "Disables recovery when VPU hang is detected");
24
25 #define PM_RESCHEDULE_LIMIT 5
26
ivpu_pm_prepare_cold_boot(struct ivpu_device * vdev)27 static void ivpu_pm_prepare_cold_boot(struct ivpu_device *vdev)
28 {
29 struct ivpu_fw_info *fw = vdev->fw;
30
31 ivpu_cmdq_reset_all_contexts(vdev);
32 ivpu_ipc_reset(vdev);
33 ivpu_fw_load(vdev);
34 fw->entry_point = fw->cold_boot_entry_point;
35 }
36
ivpu_pm_prepare_warm_boot(struct ivpu_device * vdev)37 static void ivpu_pm_prepare_warm_boot(struct ivpu_device *vdev)
38 {
39 struct ivpu_fw_info *fw = vdev->fw;
40 struct vpu_boot_params *bp = fw->mem->kvaddr;
41
42 if (!bp->save_restore_ret_address) {
43 ivpu_pm_prepare_cold_boot(vdev);
44 return;
45 }
46
47 ivpu_dbg(vdev, FW_BOOT, "Save/restore entry point %llx", bp->save_restore_ret_address);
48 fw->entry_point = bp->save_restore_ret_address;
49 }
50
ivpu_suspend(struct ivpu_device * vdev)51 static int ivpu_suspend(struct ivpu_device *vdev)
52 {
53 int ret;
54
55 ret = ivpu_shutdown(vdev);
56 if (ret) {
57 ivpu_err(vdev, "Failed to shutdown VPU: %d\n", ret);
58 return ret;
59 }
60
61 return ret;
62 }
63
ivpu_resume(struct ivpu_device * vdev)64 static int ivpu_resume(struct ivpu_device *vdev)
65 {
66 int ret;
67
68 retry:
69 ret = ivpu_hw_power_up(vdev);
70 if (ret) {
71 ivpu_err(vdev, "Failed to power up HW: %d\n", ret);
72 return ret;
73 }
74
75 ret = ivpu_mmu_enable(vdev);
76 if (ret) {
77 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);
78 ivpu_hw_power_down(vdev);
79 return ret;
80 }
81
82 ret = ivpu_boot(vdev);
83 if (ret) {
84 ivpu_mmu_disable(vdev);
85 ivpu_hw_power_down(vdev);
86 if (!ivpu_fw_is_cold_boot(vdev)) {
87 ivpu_warn(vdev, "Failed to resume the FW: %d. Retrying cold boot..\n", ret);
88 ivpu_pm_prepare_cold_boot(vdev);
89 goto retry;
90 } else {
91 ivpu_err(vdev, "Failed to resume the FW: %d\n", ret);
92 }
93 }
94
95 return ret;
96 }
97
ivpu_pm_recovery_work(struct work_struct * work)98 static void ivpu_pm_recovery_work(struct work_struct *work)
99 {
100 struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, recovery_work);
101 struct ivpu_device *vdev = pm->vdev;
102 char *evt[2] = {"IVPU_PM_EVENT=IVPU_RECOVER", NULL};
103 int ret;
104
105 ret = pci_reset_function(to_pci_dev(vdev->drm.dev));
106 if (ret)
107 ivpu_err(vdev, "Failed to reset VPU: %d\n", ret);
108
109 kobject_uevent_env(&vdev->drm.dev->kobj, KOBJ_CHANGE, evt);
110 }
111
ivpu_pm_schedule_recovery(struct ivpu_device * vdev)112 void ivpu_pm_schedule_recovery(struct ivpu_device *vdev)
113 {
114 struct ivpu_pm_info *pm = vdev->pm;
115
116 if (ivpu_disable_recovery) {
117 ivpu_err(vdev, "Recovery not available when disable_recovery param is set\n");
118 return;
119 }
120
121 if (ivpu_is_fpga(vdev)) {
122 ivpu_err(vdev, "Recovery not available on FPGA\n");
123 return;
124 }
125
126 /* Schedule recovery if it's not in progress */
127 if (atomic_cmpxchg(&pm->in_reset, 0, 1) == 0) {
128 ivpu_hw_irq_disable(vdev);
129 queue_work(system_long_wq, &pm->recovery_work);
130 }
131 }
132
ivpu_pm_suspend_cb(struct device * dev)133 int ivpu_pm_suspend_cb(struct device *dev)
134 {
135 struct drm_device *drm = dev_get_drvdata(dev);
136 struct ivpu_device *vdev = to_ivpu_device(drm);
137 int ret;
138
139 ivpu_dbg(vdev, PM, "Suspend..\n");
140
141 ret = ivpu_suspend(vdev);
142 if (ret && vdev->pm->suspend_reschedule_counter) {
143 ivpu_dbg(vdev, PM, "Failed to enter idle, rescheduling suspend, retries left %d\n",
144 vdev->pm->suspend_reschedule_counter);
145 pm_schedule_suspend(dev, vdev->timeout.reschedule_suspend);
146 vdev->pm->suspend_reschedule_counter--;
147 return -EBUSY;
148 } else if (!vdev->pm->suspend_reschedule_counter) {
149 ivpu_warn(vdev, "Failed to enter idle, force suspend\n");
150 ivpu_pm_prepare_cold_boot(vdev);
151 } else {
152 ivpu_pm_prepare_warm_boot(vdev);
153 }
154
155 vdev->pm->suspend_reschedule_counter = PM_RESCHEDULE_LIMIT;
156
157 pci_save_state(to_pci_dev(dev));
158 pci_set_power_state(to_pci_dev(dev), PCI_D3hot);
159
160 ivpu_dbg(vdev, PM, "Suspend done.\n");
161
162 return ret;
163 }
164
ivpu_pm_resume_cb(struct device * dev)165 int ivpu_pm_resume_cb(struct device *dev)
166 {
167 struct drm_device *drm = dev_get_drvdata(dev);
168 struct ivpu_device *vdev = to_ivpu_device(drm);
169 int ret;
170
171 ivpu_dbg(vdev, PM, "Resume..\n");
172
173 pci_set_power_state(to_pci_dev(dev), PCI_D0);
174 pci_restore_state(to_pci_dev(dev));
175
176 ret = ivpu_resume(vdev);
177 if (ret)
178 ivpu_err(vdev, "Failed to resume: %d\n", ret);
179
180 ivpu_dbg(vdev, PM, "Resume done.\n");
181
182 return ret;
183 }
184
ivpu_pm_runtime_suspend_cb(struct device * dev)185 int ivpu_pm_runtime_suspend_cb(struct device *dev)
186 {
187 struct drm_device *drm = dev_get_drvdata(dev);
188 struct ivpu_device *vdev = to_ivpu_device(drm);
189 int ret;
190
191 ivpu_dbg(vdev, PM, "Runtime suspend..\n");
192
193 if (!ivpu_hw_is_idle(vdev) && vdev->pm->suspend_reschedule_counter) {
194 ivpu_dbg(vdev, PM, "Failed to enter idle, rescheduling suspend, retries left %d\n",
195 vdev->pm->suspend_reschedule_counter);
196 pm_schedule_suspend(dev, vdev->timeout.reschedule_suspend);
197 vdev->pm->suspend_reschedule_counter--;
198 return -EAGAIN;
199 }
200
201 ret = ivpu_suspend(vdev);
202 if (ret)
203 ivpu_err(vdev, "Failed to set suspend VPU: %d\n", ret);
204
205 if (!vdev->pm->suspend_reschedule_counter) {
206 ivpu_warn(vdev, "VPU failed to enter idle, force suspended.\n");
207 ivpu_pm_prepare_cold_boot(vdev);
208 } else {
209 ivpu_pm_prepare_warm_boot(vdev);
210 }
211
212 vdev->pm->suspend_reschedule_counter = PM_RESCHEDULE_LIMIT;
213
214 ivpu_dbg(vdev, PM, "Runtime suspend done.\n");
215
216 return 0;
217 }
218
ivpu_pm_runtime_resume_cb(struct device * dev)219 int ivpu_pm_runtime_resume_cb(struct device *dev)
220 {
221 struct drm_device *drm = dev_get_drvdata(dev);
222 struct ivpu_device *vdev = to_ivpu_device(drm);
223 int ret;
224
225 ivpu_dbg(vdev, PM, "Runtime resume..\n");
226
227 ret = ivpu_resume(vdev);
228 if (ret)
229 ivpu_err(vdev, "Failed to set RESUME state: %d\n", ret);
230
231 ivpu_dbg(vdev, PM, "Runtime resume done.\n");
232
233 return ret;
234 }
235
ivpu_rpm_get(struct ivpu_device * vdev)236 int ivpu_rpm_get(struct ivpu_device *vdev)
237 {
238 int ret;
239
240 ivpu_dbg(vdev, RPM, "rpm_get count %d\n", atomic_read(&vdev->drm.dev->power.usage_count));
241
242 ret = pm_runtime_resume_and_get(vdev->drm.dev);
243 if (!drm_WARN_ON(&vdev->drm, ret < 0))
244 vdev->pm->suspend_reschedule_counter = PM_RESCHEDULE_LIMIT;
245
246 return ret;
247 }
248
ivpu_rpm_put(struct ivpu_device * vdev)249 void ivpu_rpm_put(struct ivpu_device *vdev)
250 {
251 ivpu_dbg(vdev, RPM, "rpm_put count %d\n", atomic_read(&vdev->drm.dev->power.usage_count));
252
253 pm_runtime_mark_last_busy(vdev->drm.dev);
254 pm_runtime_put_autosuspend(vdev->drm.dev);
255 }
256
ivpu_pm_reset_prepare_cb(struct pci_dev * pdev)257 void ivpu_pm_reset_prepare_cb(struct pci_dev *pdev)
258 {
259 struct ivpu_device *vdev = pci_get_drvdata(pdev);
260
261 pm_runtime_get_sync(vdev->drm.dev);
262
263 ivpu_dbg(vdev, PM, "Pre-reset..\n");
264 atomic_set(&vdev->pm->in_reset, 1);
265 ivpu_shutdown(vdev);
266 ivpu_pm_prepare_cold_boot(vdev);
267 ivpu_jobs_abort_all(vdev);
268 ivpu_dbg(vdev, PM, "Pre-reset done.\n");
269 }
270
ivpu_pm_reset_done_cb(struct pci_dev * pdev)271 void ivpu_pm_reset_done_cb(struct pci_dev *pdev)
272 {
273 struct ivpu_device *vdev = pci_get_drvdata(pdev);
274 int ret;
275
276 ivpu_dbg(vdev, PM, "Post-reset..\n");
277 ret = ivpu_resume(vdev);
278 if (ret)
279 ivpu_err(vdev, "Failed to set RESUME state: %d\n", ret);
280 atomic_set(&vdev->pm->in_reset, 0);
281 ivpu_dbg(vdev, PM, "Post-reset done.\n");
282
283 pm_runtime_put_autosuspend(vdev->drm.dev);
284 }
285
ivpu_pm_init(struct ivpu_device * vdev)286 int ivpu_pm_init(struct ivpu_device *vdev)
287 {
288 struct device *dev = vdev->drm.dev;
289 struct ivpu_pm_info *pm = vdev->pm;
290
291 pm->vdev = vdev;
292 pm->suspend_reschedule_counter = PM_RESCHEDULE_LIMIT;
293
294 atomic_set(&pm->in_reset, 0);
295 INIT_WORK(&pm->recovery_work, ivpu_pm_recovery_work);
296
297 pm_runtime_use_autosuspend(dev);
298
299 if (ivpu_disable_recovery)
300 pm_runtime_set_autosuspend_delay(dev, -1);
301 else if (ivpu_is_silicon(vdev))
302 pm_runtime_set_autosuspend_delay(dev, 100);
303 else
304 pm_runtime_set_autosuspend_delay(dev, 60000);
305
306 return 0;
307 }
308
ivpu_pm_enable(struct ivpu_device * vdev)309 void ivpu_pm_enable(struct ivpu_device *vdev)
310 {
311 struct device *dev = vdev->drm.dev;
312
313 pm_runtime_set_active(dev);
314 pm_runtime_allow(dev);
315 pm_runtime_mark_last_busy(dev);
316 pm_runtime_put_autosuspend(dev);
317
318 ivpu_dbg(vdev, RPM, "Enable RPM count %d\n", atomic_read(&dev->power.usage_count));
319 }
320
ivpu_pm_disable(struct ivpu_device * vdev)321 void ivpu_pm_disable(struct ivpu_device *vdev)
322 {
323 struct device *dev = vdev->drm.dev;
324
325 ivpu_dbg(vdev, RPM, "Disable RPM count %d\n", atomic_read(&dev->power.usage_count));
326
327 pm_runtime_get_noresume(vdev->drm.dev);
328 pm_runtime_forbid(vdev->drm.dev);
329 }
330