Lines Matching refs:vdev
233 static void ivpu_mmu_config_check(struct ivpu_device *vdev) in ivpu_mmu_config_check() argument
238 if (ivpu_is_simics(vdev)) in ivpu_mmu_config_check()
245 ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref); in ivpu_mmu_config_check()
249 ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF); in ivpu_mmu_config_check()
253 ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF); in ivpu_mmu_config_check()
255 if (ivpu_is_simics(vdev)) in ivpu_mmu_config_check()
257 else if (ivpu_is_fpga(vdev)) in ivpu_mmu_config_check()
264 ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref); in ivpu_mmu_config_check()
267 static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev) in ivpu_mmu_cdtab_alloc() argument
269 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_cdtab_alloc()
273 cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL); in ivpu_mmu_cdtab_alloc()
277 ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size); in ivpu_mmu_cdtab_alloc()
282 static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev) in ivpu_mmu_strtab_alloc() argument
284 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_strtab_alloc()
288 strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL); in ivpu_mmu_strtab_alloc()
296 ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n", in ivpu_mmu_strtab_alloc()
302 static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev) in ivpu_mmu_cmdq_alloc() argument
304 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_cmdq_alloc()
307 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL); in ivpu_mmu_cmdq_alloc()
315 ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n", in ivpu_mmu_cmdq_alloc()
321 static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev) in ivpu_mmu_evtq_alloc() argument
323 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_evtq_alloc()
326 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL); in ivpu_mmu_evtq_alloc()
334 ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n", in ivpu_mmu_evtq_alloc()
340 static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev) in ivpu_mmu_structs_alloc() argument
344 ret = ivpu_mmu_cdtab_alloc(vdev); in ivpu_mmu_structs_alloc()
346 ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret); in ivpu_mmu_structs_alloc()
350 ret = ivpu_mmu_strtab_alloc(vdev); in ivpu_mmu_structs_alloc()
352 ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret); in ivpu_mmu_structs_alloc()
356 ret = ivpu_mmu_cmdq_alloc(vdev); in ivpu_mmu_structs_alloc()
358 ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret); in ivpu_mmu_structs_alloc()
362 ret = ivpu_mmu_evtq_alloc(vdev); in ivpu_mmu_structs_alloc()
364 ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret); in ivpu_mmu_structs_alloc()
369 static int ivpu_mmu_reg_write(struct ivpu_device *vdev, u32 reg, u32 val) in ivpu_mmu_reg_write() argument
379 ivpu_err(vdev, "Failed to write register 0x%x\n", reg); in ivpu_mmu_reg_write()
384 static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev) in ivpu_mmu_irqs_setup() argument
389 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, 0); in ivpu_mmu_irqs_setup()
393 return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, irq_ctrl); in ivpu_mmu_irqs_setup()
396 static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev) in ivpu_mmu_cmdq_wait_for_cons() argument
398 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq; in ivpu_mmu_cmdq_wait_for_cons()
404 static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1) in ivpu_mmu_cmdq_cmd_write() argument
406 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; in ivpu_mmu_cmdq_cmd_write()
411 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name); in ivpu_mmu_cmdq_cmd_write()
419 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1); in ivpu_mmu_cmdq_cmd_write()
424 static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev) in ivpu_mmu_cmdq_sync() argument
426 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; in ivpu_mmu_cmdq_sync()
435 ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0); in ivpu_mmu_cmdq_sync()
442 ret = ivpu_mmu_cmdq_wait_for_cons(vdev); in ivpu_mmu_cmdq_sync()
444 ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret); in ivpu_mmu_cmdq_sync()
449 static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev) in ivpu_mmu_cmdq_write_cfgi_all() argument
454 return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1); in ivpu_mmu_cmdq_write_cfgi_all()
457 static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid) in ivpu_mmu_cmdq_write_tlbi_nh_asid() argument
462 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0); in ivpu_mmu_cmdq_write_tlbi_nh_asid()
465 static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev) in ivpu_mmu_cmdq_write_tlbi_nsnh_all() argument
469 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0); in ivpu_mmu_cmdq_write_tlbi_nsnh_all()
472 static int ivpu_mmu_reset(struct ivpu_device *vdev) in ivpu_mmu_reset() argument
474 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_reset()
488 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, 0); in ivpu_mmu_reset()
508 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val); in ivpu_mmu_reset()
512 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); in ivpu_mmu_reset()
516 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); in ivpu_mmu_reset()
520 ret = ivpu_mmu_cmdq_sync(vdev); in ivpu_mmu_reset()
529 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val); in ivpu_mmu_reset()
534 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val); in ivpu_mmu_reset()
538 ret = ivpu_mmu_irqs_setup(vdev); in ivpu_mmu_reset()
543 return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val); in ivpu_mmu_reset()
546 static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid) in ivpu_mmu_strtab_link_cd() argument
548 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_strtab_link_cd()
576 ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]); in ivpu_mmu_strtab_link_cd()
579 static int ivpu_mmu_strtab_init(struct ivpu_device *vdev) in ivpu_mmu_strtab_init() argument
581 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0); in ivpu_mmu_strtab_init()
582 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3); in ivpu_mmu_strtab_init()
587 int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid) in ivpu_mmu_invalidate_tlb() argument
589 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_invalidate_tlb()
601 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid); in ivpu_mmu_invalidate_tlb()
605 ret = ivpu_mmu_cmdq_sync(vdev); in ivpu_mmu_invalidate_tlb()
611 static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma) in ivpu_mmu_cd_add() argument
613 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_cd_add()
655 ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n", in ivpu_mmu_cd_add()
667 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); in ivpu_mmu_cd_add()
671 ret = ivpu_mmu_cmdq_sync(vdev); in ivpu_mmu_cd_add()
677 static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev) in ivpu_mmu_cd_add_gbl() argument
681 ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma); in ivpu_mmu_cd_add_gbl()
683 ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret); in ivpu_mmu_cd_add_gbl()
688 static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma) in ivpu_mmu_cd_add_user() argument
693 ivpu_err(vdev, "Invalid SSID: %u\n", ssid); in ivpu_mmu_cd_add_user()
697 ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma); in ivpu_mmu_cd_add_user()
699 ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret); in ivpu_mmu_cd_add_user()
704 int ivpu_mmu_init(struct ivpu_device *vdev) in ivpu_mmu_init() argument
706 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_init()
709 ivpu_dbg(vdev, MMU, "Init..\n"); in ivpu_mmu_init()
711 drmm_mutex_init(&vdev->drm, &mmu->lock); in ivpu_mmu_init()
712 ivpu_mmu_config_check(vdev); in ivpu_mmu_init()
714 ret = ivpu_mmu_structs_alloc(vdev); in ivpu_mmu_init()
718 ret = ivpu_mmu_strtab_init(vdev); in ivpu_mmu_init()
720 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); in ivpu_mmu_init()
724 ret = ivpu_mmu_cd_add_gbl(vdev); in ivpu_mmu_init()
726 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); in ivpu_mmu_init()
730 ret = ivpu_mmu_enable(vdev); in ivpu_mmu_init()
732 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret); in ivpu_mmu_init()
736 ivpu_dbg(vdev, MMU, "Init done\n"); in ivpu_mmu_init()
741 int ivpu_mmu_enable(struct ivpu_device *vdev) in ivpu_mmu_enable() argument
743 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_enable()
750 ret = ivpu_mmu_reset(vdev); in ivpu_mmu_enable()
752 ivpu_err(vdev, "Failed to reset MMU: %d\n", ret); in ivpu_mmu_enable()
756 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); in ivpu_mmu_enable()
760 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); in ivpu_mmu_enable()
764 ret = ivpu_mmu_cmdq_sync(vdev); in ivpu_mmu_enable()
777 void ivpu_mmu_disable(struct ivpu_device *vdev) in ivpu_mmu_disable() argument
779 struct ivpu_mmu_info *mmu = vdev->mmu; in ivpu_mmu_disable()
786 static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event) in ivpu_mmu_dump_event() argument
794 …ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch… in ivpu_mmu_dump_event()
798 static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev) in ivpu_mmu_get_event() argument
800 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq; in ivpu_mmu_get_event()
816 void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev) in ivpu_mmu_irq_evtq_handler() argument
822 ivpu_dbg(vdev, IRQ, "MMU event queue\n"); in ivpu_mmu_irq_evtq_handler()
824 while ((event = ivpu_mmu_get_event(vdev)) != NULL) { in ivpu_mmu_irq_evtq_handler()
825 ivpu_mmu_dump_event(vdev, event); in ivpu_mmu_irq_evtq_handler()
831 ivpu_mmu_user_context_mark_invalid(vdev, ssid); in ivpu_mmu_irq_evtq_handler()
835 ivpu_pm_schedule_recovery(vdev); in ivpu_mmu_irq_evtq_handler()
838 void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev) in ivpu_mmu_irq_gerr_handler() argument
842 ivpu_dbg(vdev, IRQ, "MMU error\n"); in ivpu_mmu_irq_gerr_handler()
852 ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n"); in ivpu_mmu_irq_gerr_handler()
855 ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n"); in ivpu_mmu_irq_gerr_handler()
858 ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n"); in ivpu_mmu_irq_gerr_handler()
861 ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n"); in ivpu_mmu_irq_gerr_handler()
864 ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n"); in ivpu_mmu_irq_gerr_handler()
867 ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n"); in ivpu_mmu_irq_gerr_handler()
870 ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n"); in ivpu_mmu_irq_gerr_handler()
875 int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable) in ivpu_mmu_set_pgtable() argument
877 return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma); in ivpu_mmu_set_pgtable()
880 void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid) in ivpu_mmu_clear_pgtable() argument
882 ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */ in ivpu_mmu_clear_pgtable()