Lines Matching refs:pfdev
88 static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num) in panfrost_fence_create() argument
91 struct panfrost_job_slot *js = pfdev->js; in panfrost_fence_create()
97 fence->dev = pfdev->ddev; in panfrost_fence_create()
119 (job->pfdev->features.nr_core_groups == 2)) in panfrost_job_get_slot()
121 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987)) in panfrost_job_get_slot()
128 static void panfrost_job_write_affinity(struct panfrost_device *pfdev, in panfrost_job_write_affinity() argument
139 affinity = pfdev->features.shader_present; in panfrost_job_write_affinity()
141 job_write(pfdev, JS_AFFINITY_NEXT_LO(js), lower_32_bits(affinity)); in panfrost_job_write_affinity()
142 job_write(pfdev, JS_AFFINITY_NEXT_HI(js), upper_32_bits(affinity)); in panfrost_job_write_affinity()
150 if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) in panfrost_get_job_chain_flag()
157 panfrost_dequeue_job(struct panfrost_device *pfdev, int slot) in panfrost_dequeue_job() argument
159 struct panfrost_job *job = pfdev->jobs[slot][0]; in panfrost_dequeue_job()
162 pfdev->jobs[slot][0] = pfdev->jobs[slot][1]; in panfrost_dequeue_job()
163 pfdev->jobs[slot][1] = NULL; in panfrost_dequeue_job()
169 panfrost_enqueue_job(struct panfrost_device *pfdev, int slot, in panfrost_enqueue_job() argument
175 if (!pfdev->jobs[slot][0]) { in panfrost_enqueue_job()
176 pfdev->jobs[slot][0] = job; in panfrost_enqueue_job()
180 WARN_ON(pfdev->jobs[slot][1]); in panfrost_enqueue_job()
181 pfdev->jobs[slot][1] = job; in panfrost_enqueue_job()
183 panfrost_get_job_chain_flag(pfdev->jobs[slot][0])); in panfrost_enqueue_job()
189 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_hw_submit() local
195 panfrost_devfreq_record_busy(&pfdev->pfdevfreq); in panfrost_job_hw_submit()
197 ret = pm_runtime_get_sync(pfdev->dev); in panfrost_job_hw_submit()
201 if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) { in panfrost_job_hw_submit()
205 cfg = panfrost_mmu_as_get(pfdev, job->mmu); in panfrost_job_hw_submit()
207 job_write(pfdev, JS_HEAD_NEXT_LO(js), lower_32_bits(jc_head)); in panfrost_job_hw_submit()
208 job_write(pfdev, JS_HEAD_NEXT_HI(js), upper_32_bits(jc_head)); in panfrost_job_hw_submit()
210 panfrost_job_write_affinity(pfdev, job->requirements, js); in panfrost_job_hw_submit()
219 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION)) in panfrost_job_hw_submit()
222 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649)) in panfrost_job_hw_submit()
225 job_write(pfdev, JS_CONFIG_NEXT(js), cfg); in panfrost_job_hw_submit()
227 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION)) in panfrost_job_hw_submit()
228 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id); in panfrost_job_hw_submit()
232 spin_lock(&pfdev->js->job_lock); in panfrost_job_hw_submit()
233 subslot = panfrost_enqueue_job(pfdev, js, job); in panfrost_job_hw_submit()
235 if (!atomic_read(&pfdev->reset.pending)) { in panfrost_job_hw_submit()
236 job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START); in panfrost_job_hw_submit()
237 dev_dbg(pfdev->dev, in panfrost_job_hw_submit()
241 spin_unlock(&pfdev->js->job_lock); in panfrost_job_hw_submit()
277 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_push() local
286 mutex_lock(&pfdev->sched_lock); in panfrost_job_push()
294 mutex_unlock(&pfdev->sched_lock); in panfrost_job_push()
302 mutex_unlock(&pfdev->sched_lock); in panfrost_job_push()
360 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_run() local
373 fence = panfrost_fence_create(pfdev, slot); in panfrost_job_run()
386 void panfrost_job_enable_interrupts(struct panfrost_device *pfdev) in panfrost_job_enable_interrupts() argument
395 job_write(pfdev, JOB_INT_CLEAR, irq_mask); in panfrost_job_enable_interrupts()
396 job_write(pfdev, JOB_INT_MASK, irq_mask); in panfrost_job_enable_interrupts()
399 static void panfrost_job_handle_err(struct panfrost_device *pfdev, in panfrost_job_handle_err() argument
403 u32 js_status = job_read(pfdev, JS_STATUS(js)); in panfrost_job_handle_err()
408 dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x", in panfrost_job_handle_err()
410 job_read(pfdev, JS_HEAD_LO(js)), in panfrost_job_handle_err()
411 job_read(pfdev, JS_TAIL_LO(js))); in panfrost_job_handle_err()
413 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x", in panfrost_job_handle_err()
415 job_read(pfdev, JS_HEAD_LO(js)), in panfrost_job_handle_err()
416 job_read(pfdev, JS_TAIL_LO(js))); in panfrost_job_handle_err()
421 job->jc = job_read(pfdev, JS_TAIL_LO(js)) | in panfrost_job_handle_err()
422 ((u64)job_read(pfdev, JS_TAIL_HI(js)) << 32); in panfrost_job_handle_err()
439 panfrost_mmu_as_put(pfdev, job->mmu); in panfrost_job_handle_err()
440 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_job_handle_err()
445 pm_runtime_put_autosuspend(pfdev->dev); in panfrost_job_handle_err()
447 if (panfrost_exception_needs_reset(pfdev, js_status)) { in panfrost_job_handle_err()
448 atomic_set(&pfdev->reset.pending, 1); in panfrost_job_handle_err()
449 drm_sched_fault(&pfdev->js->queue[js].sched); in panfrost_job_handle_err()
453 static void panfrost_job_handle_done(struct panfrost_device *pfdev, in panfrost_job_handle_done() argument
460 panfrost_mmu_as_put(pfdev, job->mmu); in panfrost_job_handle_done()
461 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_job_handle_done()
464 pm_runtime_put_autosuspend(pfdev->dev); in panfrost_job_handle_done()
467 static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status) in panfrost_job_handle_irq() argument
484 done[j][1] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
486 done[j][0] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
494 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP); in panfrost_job_handle_irq()
495 failed[j] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
511 job_write(pfdev, JOB_INT_CLEAR, status); in panfrost_job_handle_irq()
513 js_state |= job_read(pfdev, JOB_INT_JS_STATE) & js_state_mask; in panfrost_job_handle_irq()
515 status = job_read(pfdev, JOB_INT_RAWSTAT); in panfrost_job_handle_irq()
524 panfrost_job_handle_err(pfdev, failed[j], j); in panfrost_job_handle_irq()
525 } else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) { in panfrost_job_handle_irq()
536 done[j][0] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
538 done[j][1] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
542 panfrost_job_handle_done(pfdev, done[j][i]); in panfrost_job_handle_irq()
552 if (!failed[j] || !pfdev->jobs[j][0]) in panfrost_job_handle_irq()
555 if (pfdev->jobs[j][0]->jc == 0) { in panfrost_job_handle_irq()
557 struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
560 panfrost_job_handle_done(pfdev, canceled); in panfrost_job_handle_irq()
561 } else if (!atomic_read(&pfdev->reset.pending)) { in panfrost_job_handle_irq()
563 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START); in panfrost_job_handle_irq()
568 static void panfrost_job_handle_irqs(struct panfrost_device *pfdev) in panfrost_job_handle_irqs() argument
570 u32 status = job_read(pfdev, JOB_INT_RAWSTAT); in panfrost_job_handle_irqs()
573 pm_runtime_mark_last_busy(pfdev->dev); in panfrost_job_handle_irqs()
575 spin_lock(&pfdev->js->job_lock); in panfrost_job_handle_irqs()
576 panfrost_job_handle_irq(pfdev, status); in panfrost_job_handle_irqs()
577 spin_unlock(&pfdev->js->job_lock); in panfrost_job_handle_irqs()
578 status = job_read(pfdev, JOB_INT_RAWSTAT); in panfrost_job_handle_irqs()
582 static u32 panfrost_active_slots(struct panfrost_device *pfdev, in panfrost_active_slots() argument
590 rawstat = job_read(pfdev, JOB_INT_RAWSTAT); in panfrost_active_slots()
604 panfrost_reset(struct panfrost_device *pfdev, in panfrost_reset() argument
612 if (!atomic_read(&pfdev->reset.pending)) in panfrost_reset()
629 drm_sched_stop(&pfdev->js->queue[i].sched, bad); in panfrost_reset()
639 job_write(pfdev, JOB_INT_MASK, 0); in panfrost_reset()
640 synchronize_irq(pfdev->js->irq); in panfrost_reset()
644 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP); in panfrost_reset()
645 job_write(pfdev, JS_COMMAND(i), JS_COMMAND_SOFT_STOP); in panfrost_reset()
649 ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state, in panfrost_reset()
650 !panfrost_active_slots(pfdev, &js_state_mask, js_state), in panfrost_reset()
654 dev_err(pfdev->dev, "Soft-stop failed\n"); in panfrost_reset()
657 panfrost_job_handle_irqs(pfdev); in panfrost_reset()
664 spin_lock(&pfdev->js->job_lock); in panfrost_reset()
666 for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) { in panfrost_reset()
667 pm_runtime_put_noidle(pfdev->dev); in panfrost_reset()
668 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_reset()
671 memset(pfdev->jobs, 0, sizeof(pfdev->jobs)); in panfrost_reset()
672 spin_unlock(&pfdev->js->job_lock); in panfrost_reset()
675 panfrost_device_reset(pfdev); in panfrost_reset()
680 job_write(pfdev, JOB_INT_MASK, 0); in panfrost_reset()
683 atomic_set(&pfdev->reset.pending, 0); in panfrost_reset()
694 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched); in panfrost_reset()
699 drm_sched_start(&pfdev->js->queue[i].sched, true); in panfrost_reset()
702 job_write(pfdev, JOB_INT_MASK, in panfrost_reset()
713 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_timedout() local
723 …dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sch… in panfrost_job_timedout()
725 job_read(pfdev, JS_CONFIG(js)), in panfrost_job_timedout()
726 job_read(pfdev, JS_STATUS(js)), in panfrost_job_timedout()
727 job_read(pfdev, JS_HEAD_LO(js)), in panfrost_job_timedout()
728 job_read(pfdev, JS_TAIL_LO(js)), in panfrost_job_timedout()
733 atomic_set(&pfdev->reset.pending, 1); in panfrost_job_timedout()
734 panfrost_reset(pfdev, sched_job); in panfrost_job_timedout()
741 struct panfrost_device *pfdev; in panfrost_reset_work() local
743 pfdev = container_of(work, struct panfrost_device, reset.work); in panfrost_reset_work()
744 panfrost_reset(pfdev, NULL); in panfrost_reset_work()
755 struct panfrost_device *pfdev = data; in panfrost_job_irq_handler_thread() local
757 panfrost_job_handle_irqs(pfdev); in panfrost_job_irq_handler_thread()
758 job_write(pfdev, JOB_INT_MASK, in panfrost_job_irq_handler_thread()
766 struct panfrost_device *pfdev = data; in panfrost_job_irq_handler() local
767 u32 status = job_read(pfdev, JOB_INT_STAT); in panfrost_job_irq_handler()
772 job_write(pfdev, JOB_INT_MASK, 0); in panfrost_job_irq_handler()
776 int panfrost_job_init(struct panfrost_device *pfdev) in panfrost_job_init() argument
786 if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) in panfrost_job_init()
789 pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL); in panfrost_job_init()
793 INIT_WORK(&pfdev->reset.work, panfrost_reset_work); in panfrost_job_init()
796 js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job"); in panfrost_job_init()
800 ret = devm_request_threaded_irq(pfdev->dev, js->irq, in panfrost_job_init()
804 pfdev); in panfrost_job_init()
806 dev_err(pfdev->dev, "failed to request job irq"); in panfrost_job_init()
810 pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0); in panfrost_job_init()
811 if (!pfdev->reset.wq) in panfrost_job_init()
821 pfdev->reset.wq, in panfrost_job_init()
822 NULL, "pan_js", pfdev->dev); in panfrost_job_init()
824 dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret); in panfrost_job_init()
829 panfrost_job_enable_interrupts(pfdev); in panfrost_job_init()
837 destroy_workqueue(pfdev->reset.wq); in panfrost_job_init()
841 void panfrost_job_fini(struct panfrost_device *pfdev) in panfrost_job_fini() argument
843 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_fini()
846 job_write(pfdev, JOB_INT_MASK, 0); in panfrost_job_fini()
852 cancel_work_sync(&pfdev->reset.work); in panfrost_job_fini()
853 destroy_workqueue(pfdev->reset.wq); in panfrost_job_fini()
858 struct panfrost_device *pfdev = panfrost_priv->pfdev; in panfrost_job_open() local
859 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_open()
876 struct panfrost_device *pfdev = panfrost_priv->pfdev; in panfrost_job_close() local
883 spin_lock(&pfdev->js->job_lock); in panfrost_job_close()
888 for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) { in panfrost_job_close()
889 struct panfrost_job *job = pfdev->jobs[i][j]; in panfrost_job_close()
897 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP); in panfrost_job_close()
904 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { in panfrost_job_close()
912 job_write(pfdev, JS_COMMAND(i), cmd); in panfrost_job_close()
915 spin_unlock(&pfdev->js->job_lock); in panfrost_job_close()
918 int panfrost_job_is_idle(struct panfrost_device *pfdev) in panfrost_job_is_idle() argument
920 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_is_idle()