Lines Matching refs:smmu
72 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) in arm_smmu_rpm_get() argument
74 if (pm_runtime_enabled(smmu->dev)) in arm_smmu_rpm_get()
75 return pm_runtime_resume_and_get(smmu->dev); in arm_smmu_rpm_get()
80 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) in arm_smmu_rpm_put() argument
82 if (pm_runtime_enabled(smmu->dev)) in arm_smmu_rpm_put()
83 pm_runtime_put_autosuspend(smmu->dev); in arm_smmu_rpm_put()
131 struct arm_smmu_device **smmu) in arm_smmu_register_legacy_master() argument
174 *smmu = dev_get_drvdata(smmu_dev); in arm_smmu_register_legacy_master()
182 struct arm_smmu_device **smmu) in arm_smmu_register_legacy_master() argument
194 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, in __arm_smmu_tlb_sync() argument
200 if (smmu->impl && unlikely(smmu->impl->tlb_sync)) in __arm_smmu_tlb_sync()
201 return smmu->impl->tlb_sync(smmu, page, sync, status); in __arm_smmu_tlb_sync()
203 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); in __arm_smmu_tlb_sync()
206 reg = arm_smmu_readl(smmu, page, status); in __arm_smmu_tlb_sync()
213 dev_err_ratelimited(smmu->dev, in __arm_smmu_tlb_sync()
217 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) in arm_smmu_tlb_sync_global() argument
221 spin_lock_irqsave(&smmu->global_sync_lock, flags); in arm_smmu_tlb_sync_global()
222 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, in arm_smmu_tlb_sync_global()
224 spin_unlock_irqrestore(&smmu->global_sync_lock, flags); in arm_smmu_tlb_sync_global()
229 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_sync_context() local
233 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), in arm_smmu_tlb_sync_context()
246 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx, in arm_smmu_tlb_inv_context_s1()
254 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_inv_context_s2() local
258 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); in arm_smmu_tlb_inv_context_s2()
259 arm_smmu_tlb_sync_global(smmu); in arm_smmu_tlb_inv_context_s2()
266 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_inv_range_s1() local
270 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) in arm_smmu_tlb_inv_range_s1()
277 arm_smmu_cb_write(smmu, idx, reg, iova); in arm_smmu_tlb_inv_range_s1()
284 arm_smmu_cb_writeq(smmu, idx, reg, iova); in arm_smmu_tlb_inv_range_s1()
294 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_inv_range_s2() local
297 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) in arm_smmu_tlb_inv_range_s2()
303 arm_smmu_cb_writeq(smmu, idx, reg, iova); in arm_smmu_tlb_inv_range_s2()
305 arm_smmu_cb_write(smmu, idx, reg, iova); in arm_smmu_tlb_inv_range_s2()
366 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_add_page_s2_v1() local
368 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) in arm_smmu_tlb_add_page_s2_v1()
371 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); in arm_smmu_tlb_add_page_s2_v1()
398 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_context_fault() local
402 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); in arm_smmu_context_fault()
406 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); in arm_smmu_context_fault()
407 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); in arm_smmu_context_fault()
408 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); in arm_smmu_context_fault()
414 dev_err_ratelimited(smmu->dev, in arm_smmu_context_fault()
418 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); in arm_smmu_context_fault()
425 struct arm_smmu_device *smmu = dev; in arm_smmu_global_fault() local
429 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); in arm_smmu_global_fault()
430 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0); in arm_smmu_global_fault()
431 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1); in arm_smmu_global_fault()
432 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2); in arm_smmu_global_fault()
440 dev_err(smmu->dev, in arm_smmu_global_fault()
444 dev_err(smmu->dev, in arm_smmu_global_fault()
446 dev_err(smmu->dev, in arm_smmu_global_fault()
451 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr); in arm_smmu_global_fault()
459 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; in arm_smmu_init_context_bank()
512 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) in arm_smmu_write_context_bank() argument
516 struct arm_smmu_cb *cb = &smmu->cbs[idx]; in arm_smmu_write_context_bank()
521 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0); in arm_smmu_write_context_bank()
528 if (smmu->version > ARM_SMMU_V1) { in arm_smmu_write_context_bank()
534 if (smmu->features & ARM_SMMU_FEAT_VMID16) in arm_smmu_write_context_bank()
537 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg); in arm_smmu_write_context_bank()
542 if (smmu->version < ARM_SMMU_V2) in arm_smmu_write_context_bank()
554 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { in arm_smmu_write_context_bank()
558 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg); in arm_smmu_write_context_bank()
565 if (stage1 && smmu->version > ARM_SMMU_V1) in arm_smmu_write_context_bank()
566 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]); in arm_smmu_write_context_bank()
567 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]); in arm_smmu_write_context_bank()
571 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid); in arm_smmu_write_context_bank()
572 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); in arm_smmu_write_context_bank()
573 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]); in arm_smmu_write_context_bank()
575 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); in arm_smmu_write_context_bank()
577 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1, in arm_smmu_write_context_bank()
583 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]); in arm_smmu_write_context_bank()
584 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]); in arm_smmu_write_context_bank()
595 if (smmu->impl && smmu->impl->write_sctlr) in arm_smmu_write_context_bank()
596 smmu->impl->write_sctlr(smmu, idx, reg); in arm_smmu_write_context_bank()
598 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); in arm_smmu_write_context_bank()
602 struct arm_smmu_device *smmu, in arm_smmu_alloc_context_bank() argument
605 if (smmu->impl && smmu->impl->alloc_context_bank) in arm_smmu_alloc_context_bank()
606 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start); in arm_smmu_alloc_context_bank()
608 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); in arm_smmu_alloc_context_bank()
612 struct arm_smmu_device *smmu, in arm_smmu_init_domain_context() argument
625 if (smmu_domain->smmu) in arm_smmu_init_domain_context()
630 smmu_domain->smmu = smmu; in arm_smmu_init_domain_context()
652 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) in arm_smmu_init_domain_context()
654 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) in arm_smmu_init_domain_context()
665 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) in arm_smmu_init_domain_context()
669 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && in arm_smmu_init_domain_context()
673 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | in arm_smmu_init_domain_context()
686 start = smmu->num_s2_context_banks; in arm_smmu_init_domain_context()
687 ias = smmu->va_size; in arm_smmu_init_domain_context()
688 oas = smmu->ipa_size; in arm_smmu_init_domain_context()
710 ias = smmu->ipa_size; in arm_smmu_init_domain_context()
711 oas = smmu->pa_size; in arm_smmu_init_domain_context()
719 if (smmu->version == ARM_SMMU_V2) in arm_smmu_init_domain_context()
729 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start); in arm_smmu_init_domain_context()
734 smmu_domain->smmu = smmu; in arm_smmu_init_domain_context()
737 if (smmu->version < ARM_SMMU_V2) { in arm_smmu_init_domain_context()
738 cfg->irptndx = atomic_inc_return(&smmu->irptndx); in arm_smmu_init_domain_context()
739 cfg->irptndx %= smmu->num_context_irqs; in arm_smmu_init_domain_context()
750 .pgsize_bitmap = smmu->pgsize_bitmap, in arm_smmu_init_domain_context()
753 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK, in arm_smmu_init_domain_context()
755 .iommu_dev = smmu->dev, in arm_smmu_init_domain_context()
758 if (smmu->impl && smmu->impl->init_context) { in arm_smmu_init_domain_context()
759 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev); in arm_smmu_init_domain_context()
787 arm_smmu_write_context_bank(smmu, cfg->cbndx); in arm_smmu_init_domain_context()
793 irq = smmu->irqs[cfg->irptndx]; in arm_smmu_init_domain_context()
795 if (smmu->impl && smmu->impl->context_fault) in arm_smmu_init_domain_context()
796 context_fault = smmu->impl->context_fault; in arm_smmu_init_domain_context()
800 ret = devm_request_irq(smmu->dev, irq, context_fault, in arm_smmu_init_domain_context()
803 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", in arm_smmu_init_domain_context()
815 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); in arm_smmu_init_domain_context()
816 smmu_domain->smmu = NULL; in arm_smmu_init_domain_context()
825 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_destroy_domain_context() local
829 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY) in arm_smmu_destroy_domain_context()
832 ret = arm_smmu_rpm_get(smmu); in arm_smmu_destroy_domain_context()
840 smmu->cbs[cfg->cbndx].cfg = NULL; in arm_smmu_destroy_domain_context()
841 arm_smmu_write_context_bank(smmu, cfg->cbndx); in arm_smmu_destroy_domain_context()
844 irq = smmu->irqs[cfg->irptndx]; in arm_smmu_destroy_domain_context()
845 devm_free_irq(smmu->dev, irq, domain); in arm_smmu_destroy_domain_context()
849 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); in arm_smmu_destroy_domain_context()
851 arm_smmu_rpm_put(smmu); in arm_smmu_destroy_domain_context()
890 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) in arm_smmu_write_smr() argument
892 struct arm_smmu_smr *smr = smmu->smrs + idx; in arm_smmu_write_smr()
896 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) in arm_smmu_write_smr()
898 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg); in arm_smmu_write_smr()
901 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) in arm_smmu_write_s2cr() argument
903 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; in arm_smmu_write_s2cr()
906 if (smmu->impl && smmu->impl->write_s2cr) { in arm_smmu_write_s2cr()
907 smmu->impl->write_s2cr(smmu, idx); in arm_smmu_write_s2cr()
915 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && in arm_smmu_write_s2cr()
916 smmu->smrs[idx].valid) in arm_smmu_write_s2cr()
918 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg); in arm_smmu_write_s2cr()
921 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) in arm_smmu_write_sme() argument
923 arm_smmu_write_s2cr(smmu, idx); in arm_smmu_write_sme()
924 if (smmu->smrs) in arm_smmu_write_sme()
925 arm_smmu_write_smr(smmu, idx); in arm_smmu_write_sme()
932 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) in arm_smmu_test_smr_masks() argument
937 if (!smmu->smrs) in arm_smmu_test_smr_masks()
947 for (i = 0; i < smmu->num_mapping_groups; i++) in arm_smmu_test_smr_masks()
948 if (!smmu->smrs[i].valid) in arm_smmu_test_smr_masks()
957 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask); in arm_smmu_test_smr_masks()
958 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); in arm_smmu_test_smr_masks()
959 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); in arm_smmu_test_smr_masks()
960 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr); in arm_smmu_test_smr_masks()
962 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask); in arm_smmu_test_smr_masks()
963 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); in arm_smmu_test_smr_masks()
964 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); in arm_smmu_test_smr_masks()
965 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); in arm_smmu_test_smr_masks()
968 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) in arm_smmu_find_sme() argument
970 struct arm_smmu_smr *smrs = smmu->smrs; in arm_smmu_find_sme()
978 for (i = 0; i < smmu->num_mapping_groups; ++i) { in arm_smmu_find_sme()
1010 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) in arm_smmu_free_sme() argument
1012 if (--smmu->s2crs[idx].count) in arm_smmu_free_sme()
1015 smmu->s2crs[idx] = s2cr_init_val; in arm_smmu_free_sme()
1016 if (smmu->smrs) in arm_smmu_free_sme()
1017 smmu->smrs[idx].valid = false; in arm_smmu_free_sme()
1026 struct arm_smmu_device *smmu = cfg->smmu; in arm_smmu_master_alloc_smes() local
1027 struct arm_smmu_smr *smrs = smmu->smrs; in arm_smmu_master_alloc_smes()
1030 mutex_lock(&smmu->stream_map_mutex); in arm_smmu_master_alloc_smes()
1041 ret = arm_smmu_find_sme(smmu, sid, mask); in arm_smmu_master_alloc_smes()
1046 if (smrs && smmu->s2crs[idx].count == 0) { in arm_smmu_master_alloc_smes()
1051 smmu->s2crs[idx].count++; in arm_smmu_master_alloc_smes()
1057 arm_smmu_write_sme(smmu, idx); in arm_smmu_master_alloc_smes()
1059 mutex_unlock(&smmu->stream_map_mutex); in arm_smmu_master_alloc_smes()
1064 arm_smmu_free_sme(smmu, cfg->smendx[i]); in arm_smmu_master_alloc_smes()
1067 mutex_unlock(&smmu->stream_map_mutex); in arm_smmu_master_alloc_smes()
1074 struct arm_smmu_device *smmu = cfg->smmu; in arm_smmu_master_free_smes() local
1077 mutex_lock(&smmu->stream_map_mutex); in arm_smmu_master_free_smes()
1079 if (arm_smmu_free_sme(smmu, idx)) in arm_smmu_master_free_smes()
1080 arm_smmu_write_sme(smmu, idx); in arm_smmu_master_free_smes()
1083 mutex_unlock(&smmu->stream_map_mutex); in arm_smmu_master_free_smes()
1090 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_add_master() local
1091 struct arm_smmu_s2cr *s2cr = smmu->s2crs; in arm_smmu_domain_add_master()
1108 arm_smmu_write_s2cr(smmu, idx); in arm_smmu_domain_add_master()
1118 struct arm_smmu_device *smmu; in arm_smmu_attach_dev() local
1137 smmu = cfg->smmu; in arm_smmu_attach_dev()
1139 ret = arm_smmu_rpm_get(smmu); in arm_smmu_attach_dev()
1144 ret = arm_smmu_init_domain_context(domain, smmu, dev); in arm_smmu_attach_dev()
1152 if (smmu_domain->smmu != smmu) { in arm_smmu_attach_dev()
1171 pm_runtime_set_autosuspend_delay(smmu->dev, 20); in arm_smmu_attach_dev()
1172 pm_runtime_use_autosuspend(smmu->dev); in arm_smmu_attach_dev()
1175 arm_smmu_rpm_put(smmu); in arm_smmu_attach_dev()
1184 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; in arm_smmu_map_pages() local
1190 arm_smmu_rpm_get(smmu); in arm_smmu_map_pages()
1192 arm_smmu_rpm_put(smmu); in arm_smmu_map_pages()
1202 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; in arm_smmu_unmap_pages() local
1208 arm_smmu_rpm_get(smmu); in arm_smmu_unmap_pages()
1210 arm_smmu_rpm_put(smmu); in arm_smmu_unmap_pages()
1218 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_flush_iotlb_all() local
1221 arm_smmu_rpm_get(smmu); in arm_smmu_flush_iotlb_all()
1223 arm_smmu_rpm_put(smmu); in arm_smmu_flush_iotlb_all()
1231 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_iotlb_sync() local
1233 if (!smmu) in arm_smmu_iotlb_sync()
1236 arm_smmu_rpm_get(smmu); in arm_smmu_iotlb_sync()
1237 if (smmu->version == ARM_SMMU_V2 || in arm_smmu_iotlb_sync()
1241 arm_smmu_tlb_sync_global(smmu); in arm_smmu_iotlb_sync()
1242 arm_smmu_rpm_put(smmu); in arm_smmu_iotlb_sync()
1249 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_iova_to_phys_hard() local
1252 struct device *dev = smmu->dev; in arm_smmu_iova_to_phys_hard()
1260 ret = arm_smmu_rpm_get(smmu); in arm_smmu_iova_to_phys_hard()
1267 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va); in arm_smmu_iova_to_phys_hard()
1269 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va); in arm_smmu_iova_to_phys_hard()
1271 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR; in arm_smmu_iova_to_phys_hard()
1278 arm_smmu_rpm_put(smmu); in arm_smmu_iova_to_phys_hard()
1282 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR); in arm_smmu_iova_to_phys_hard()
1292 arm_smmu_rpm_put(smmu); in arm_smmu_iova_to_phys_hard()
1306 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && in arm_smmu_iova_to_phys()
1325 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK || in arm_smmu_capable()
1345 struct arm_smmu_device *smmu = NULL; in arm_smmu_probe_device() local
1351 ret = arm_smmu_register_legacy_master(dev, &smmu); in arm_smmu_probe_device()
1362 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); in arm_smmu_probe_device()
1372 if (sid & ~smmu->streamid_mask) { in arm_smmu_probe_device()
1374 sid, smmu->streamid_mask); in arm_smmu_probe_device()
1377 if (mask & ~smmu->smr_mask_mask) { in arm_smmu_probe_device()
1379 mask, smmu->smr_mask_mask); in arm_smmu_probe_device()
1390 cfg->smmu = smmu; in arm_smmu_probe_device()
1395 ret = arm_smmu_rpm_get(smmu); in arm_smmu_probe_device()
1400 arm_smmu_rpm_put(smmu); in arm_smmu_probe_device()
1405 device_link_add(dev, smmu->dev, in arm_smmu_probe_device()
1408 return &smmu->iommu; in arm_smmu_probe_device()
1423 ret = arm_smmu_rpm_get(cfg->smmu); in arm_smmu_release_device()
1429 arm_smmu_rpm_put(cfg->smmu); in arm_smmu_release_device()
1438 struct arm_smmu_device *smmu; in arm_smmu_probe_finalize() local
1441 smmu = cfg->smmu; in arm_smmu_probe_finalize()
1443 if (smmu->impl && smmu->impl->probe_finalize) in arm_smmu_probe_finalize()
1444 smmu->impl->probe_finalize(smmu, dev); in arm_smmu_probe_finalize()
1451 struct arm_smmu_device *smmu = cfg->smmu; in arm_smmu_device_group() local
1455 mutex_lock(&smmu->stream_map_mutex); in arm_smmu_device_group()
1457 if (group && smmu->s2crs[idx].group && in arm_smmu_device_group()
1458 group != smmu->s2crs[idx].group) { in arm_smmu_device_group()
1459 mutex_unlock(&smmu->stream_map_mutex); in arm_smmu_device_group()
1463 group = smmu->s2crs[idx].group; in arm_smmu_device_group()
1467 mutex_unlock(&smmu->stream_map_mutex); in arm_smmu_device_group()
1481 smmu->s2crs[idx].group = group; in arm_smmu_device_group()
1483 mutex_unlock(&smmu->stream_map_mutex); in arm_smmu_device_group()
1493 if (smmu_domain->smmu) in arm_smmu_enable_nesting()
1509 if (smmu_domain->smmu) in arm_smmu_set_pgtable_quirks()
1552 const struct arm_smmu_impl *impl = cfg->smmu->impl; in arm_smmu_def_domain_type()
1588 static void arm_smmu_device_reset(struct arm_smmu_device *smmu) in arm_smmu_device_reset() argument
1594 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); in arm_smmu_device_reset()
1595 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg); in arm_smmu_device_reset()
1601 for (i = 0; i < smmu->num_mapping_groups; ++i) in arm_smmu_device_reset()
1602 arm_smmu_write_sme(smmu, i); in arm_smmu_device_reset()
1605 for (i = 0; i < smmu->num_context_banks; ++i) { in arm_smmu_device_reset()
1606 arm_smmu_write_context_bank(smmu, i); in arm_smmu_device_reset()
1607 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT); in arm_smmu_device_reset()
1611 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL); in arm_smmu_device_reset()
1612 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL); in arm_smmu_device_reset()
1614 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); in arm_smmu_device_reset()
1636 if (smmu->features & ARM_SMMU_FEAT_VMID16) in arm_smmu_device_reset()
1639 if (smmu->features & ARM_SMMU_FEAT_EXIDS) in arm_smmu_device_reset()
1642 if (smmu->impl && smmu->impl->reset) in arm_smmu_device_reset()
1643 smmu->impl->reset(smmu); in arm_smmu_device_reset()
1646 arm_smmu_tlb_sync_global(smmu); in arm_smmu_device_reset()
1647 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); in arm_smmu_device_reset()
1669 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) in arm_smmu_device_cfg_probe() argument
1673 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; in arm_smmu_device_cfg_probe()
1676 dev_notice(smmu->dev, "probing hardware configuration...\n"); in arm_smmu_device_cfg_probe()
1677 dev_notice(smmu->dev, "SMMUv%d with:\n", in arm_smmu_device_cfg_probe()
1678 smmu->version == ARM_SMMU_V2 ? 2 : 1); in arm_smmu_device_cfg_probe()
1681 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0); in arm_smmu_device_cfg_probe()
1690 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; in arm_smmu_device_cfg_probe()
1691 dev_notice(smmu->dev, "\tstage 1 translation\n"); in arm_smmu_device_cfg_probe()
1695 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; in arm_smmu_device_cfg_probe()
1696 dev_notice(smmu->dev, "\tstage 2 translation\n"); in arm_smmu_device_cfg_probe()
1700 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; in arm_smmu_device_cfg_probe()
1701 dev_notice(smmu->dev, "\tnested translation\n"); in arm_smmu_device_cfg_probe()
1704 if (!(smmu->features & in arm_smmu_device_cfg_probe()
1706 dev_err(smmu->dev, "\tno translation support!\n"); in arm_smmu_device_cfg_probe()
1711 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) { in arm_smmu_device_cfg_probe()
1712 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; in arm_smmu_device_cfg_probe()
1713 dev_notice(smmu->dev, "\taddress translation ops\n"); in arm_smmu_device_cfg_probe()
1724 dev_notice(smmu->dev, "\t%scoherent table walk\n", in arm_smmu_device_cfg_probe()
1727 dev_notice(smmu->dev, in arm_smmu_device_cfg_probe()
1731 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) { in arm_smmu_device_cfg_probe()
1732 smmu->features |= ARM_SMMU_FEAT_EXIDS; in arm_smmu_device_cfg_probe()
1737 smmu->streamid_mask = size - 1; in arm_smmu_device_cfg_probe()
1739 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; in arm_smmu_device_cfg_probe()
1742 dev_err(smmu->dev, in arm_smmu_device_cfg_probe()
1748 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), in arm_smmu_device_cfg_probe()
1750 if (!smmu->smrs) in arm_smmu_device_cfg_probe()
1753 dev_notice(smmu->dev, in arm_smmu_device_cfg_probe()
1757 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), in arm_smmu_device_cfg_probe()
1759 if (!smmu->s2crs) in arm_smmu_device_cfg_probe()
1762 smmu->s2crs[i] = s2cr_init_val; in arm_smmu_device_cfg_probe()
1764 smmu->num_mapping_groups = size; in arm_smmu_device_cfg_probe()
1765 mutex_init(&smmu->stream_map_mutex); in arm_smmu_device_cfg_probe()
1766 spin_lock_init(&smmu->global_sync_lock); in arm_smmu_device_cfg_probe()
1768 if (smmu->version < ARM_SMMU_V2 || in arm_smmu_device_cfg_probe()
1770 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; in arm_smmu_device_cfg_probe()
1772 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; in arm_smmu_device_cfg_probe()
1776 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1); in arm_smmu_device_cfg_probe()
1777 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12; in arm_smmu_device_cfg_probe()
1781 if (smmu->numpage != 2 * size << smmu->pgshift) in arm_smmu_device_cfg_probe()
1782 dev_warn(smmu->dev, in arm_smmu_device_cfg_probe()
1784 2 * size << smmu->pgshift, smmu->numpage); in arm_smmu_device_cfg_probe()
1786 smmu->numpage = size; in arm_smmu_device_cfg_probe()
1788 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id); in arm_smmu_device_cfg_probe()
1789 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id); in arm_smmu_device_cfg_probe()
1790 if (smmu->num_s2_context_banks > smmu->num_context_banks) { in arm_smmu_device_cfg_probe()
1791 dev_err(smmu->dev, "impossible number of S2 context banks!\n"); in arm_smmu_device_cfg_probe()
1794 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", in arm_smmu_device_cfg_probe()
1795 smmu->num_context_banks, smmu->num_s2_context_banks); in arm_smmu_device_cfg_probe()
1796 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks, in arm_smmu_device_cfg_probe()
1797 sizeof(*smmu->cbs), GFP_KERNEL); in arm_smmu_device_cfg_probe()
1798 if (!smmu->cbs) in arm_smmu_device_cfg_probe()
1802 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2); in arm_smmu_device_cfg_probe()
1804 smmu->ipa_size = size; in arm_smmu_device_cfg_probe()
1808 smmu->pa_size = size; in arm_smmu_device_cfg_probe()
1811 smmu->features |= ARM_SMMU_FEAT_VMID16; in arm_smmu_device_cfg_probe()
1818 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) in arm_smmu_device_cfg_probe()
1819 dev_warn(smmu->dev, in arm_smmu_device_cfg_probe()
1822 if (smmu->version < ARM_SMMU_V2) { in arm_smmu_device_cfg_probe()
1823 smmu->va_size = smmu->ipa_size; in arm_smmu_device_cfg_probe()
1824 if (smmu->version == ARM_SMMU_V1_64K) in arm_smmu_device_cfg_probe()
1825 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; in arm_smmu_device_cfg_probe()
1828 smmu->va_size = arm_smmu_id_size_to_bits(size); in arm_smmu_device_cfg_probe()
1830 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; in arm_smmu_device_cfg_probe()
1832 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; in arm_smmu_device_cfg_probe()
1834 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; in arm_smmu_device_cfg_probe()
1837 if (smmu->impl && smmu->impl->cfg_probe) { in arm_smmu_device_cfg_probe()
1838 ret = smmu->impl->cfg_probe(smmu); in arm_smmu_device_cfg_probe()
1844 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) in arm_smmu_device_cfg_probe()
1845 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; in arm_smmu_device_cfg_probe()
1846 if (smmu->features & in arm_smmu_device_cfg_probe()
1848 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; in arm_smmu_device_cfg_probe()
1849 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) in arm_smmu_device_cfg_probe()
1850 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; in arm_smmu_device_cfg_probe()
1851 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) in arm_smmu_device_cfg_probe()
1852 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; in arm_smmu_device_cfg_probe()
1855 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; in arm_smmu_device_cfg_probe()
1857 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; in arm_smmu_device_cfg_probe()
1858 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", in arm_smmu_device_cfg_probe()
1859 smmu->pgsize_bitmap); in arm_smmu_device_cfg_probe()
1862 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) in arm_smmu_device_cfg_probe()
1863 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", in arm_smmu_device_cfg_probe()
1864 smmu->va_size, smmu->ipa_size); in arm_smmu_device_cfg_probe()
1866 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) in arm_smmu_device_cfg_probe()
1867 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", in arm_smmu_device_cfg_probe()
1868 smmu->ipa_size, smmu->pa_size); in arm_smmu_device_cfg_probe()
1902 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) in acpi_smmu_get_data() argument
1909 smmu->version = ARM_SMMU_V1; in acpi_smmu_get_data()
1910 smmu->model = GENERIC_SMMU; in acpi_smmu_get_data()
1913 smmu->version = ARM_SMMU_V1_64K; in acpi_smmu_get_data()
1914 smmu->model = GENERIC_SMMU; in acpi_smmu_get_data()
1917 smmu->version = ARM_SMMU_V2; in acpi_smmu_get_data()
1918 smmu->model = GENERIC_SMMU; in acpi_smmu_get_data()
1921 smmu->version = ARM_SMMU_V2; in acpi_smmu_get_data()
1922 smmu->model = ARM_MMU500; in acpi_smmu_get_data()
1925 smmu->version = ARM_SMMU_V2; in acpi_smmu_get_data()
1926 smmu->model = CAVIUM_SMMUV2; in acpi_smmu_get_data()
1935 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, in arm_smmu_device_acpi_probe() argument
1938 struct device *dev = smmu->dev; in arm_smmu_device_acpi_probe()
1947 ret = acpi_smmu_get_data(iort_smmu->model, smmu); in arm_smmu_device_acpi_probe()
1956 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; in arm_smmu_device_acpi_probe()
1961 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, in arm_smmu_device_acpi_probe() argument
1968 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu, in arm_smmu_device_dt_probe() argument
1972 struct device *dev = smmu->dev; in arm_smmu_device_dt_probe()
1981 smmu->version = data->version; in arm_smmu_device_dt_probe()
1982 smmu->model = data->model; in arm_smmu_device_dt_probe()
1999 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; in arm_smmu_device_dt_probe()
2004 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu) in arm_smmu_rmr_install_bypass_smr() argument
2012 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); in arm_smmu_rmr_install_bypass_smr()
2020 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); in arm_smmu_rmr_install_bypass_smr()
2022 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); in arm_smmu_rmr_install_bypass_smr()
2030 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0); in arm_smmu_rmr_install_bypass_smr()
2034 if (smmu->s2crs[idx].count == 0) { in arm_smmu_rmr_install_bypass_smr()
2035 smmu->smrs[idx].id = rmr->sids[i]; in arm_smmu_rmr_install_bypass_smr()
2036 smmu->smrs[idx].mask = 0; in arm_smmu_rmr_install_bypass_smr()
2037 smmu->smrs[idx].valid = true; in arm_smmu_rmr_install_bypass_smr()
2039 smmu->s2crs[idx].count++; in arm_smmu_rmr_install_bypass_smr()
2040 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS; in arm_smmu_rmr_install_bypass_smr()
2041 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT; in arm_smmu_rmr_install_bypass_smr()
2047 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt, in arm_smmu_rmr_install_bypass_smr()
2049 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); in arm_smmu_rmr_install_bypass_smr()
2055 struct arm_smmu_device *smmu; in arm_smmu_device_probe() local
2061 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); in arm_smmu_device_probe()
2062 if (!smmu) { in arm_smmu_device_probe()
2066 smmu->dev = dev; in arm_smmu_device_probe()
2069 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs); in arm_smmu_device_probe()
2071 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs); in arm_smmu_device_probe()
2075 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); in arm_smmu_device_probe()
2076 if (IS_ERR(smmu->base)) in arm_smmu_device_probe()
2077 return PTR_ERR(smmu->base); in arm_smmu_device_probe()
2078 smmu->ioaddr = res->start; in arm_smmu_device_probe()
2084 smmu->numpage = resource_size(res); in arm_smmu_device_probe()
2086 smmu = arm_smmu_impl_init(smmu); in arm_smmu_device_probe()
2087 if (IS_ERR(smmu)) in arm_smmu_device_probe()
2088 return PTR_ERR(smmu); in arm_smmu_device_probe()
2092 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs; in arm_smmu_device_probe()
2093 if (smmu->num_context_irqs <= 0) in arm_smmu_device_probe()
2098 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs, in arm_smmu_device_probe()
2099 sizeof(*smmu->irqs), GFP_KERNEL); in arm_smmu_device_probe()
2100 if (!smmu->irqs) in arm_smmu_device_probe()
2102 smmu->num_context_irqs); in arm_smmu_device_probe()
2104 for (i = 0; i < smmu->num_context_irqs; i++) { in arm_smmu_device_probe()
2109 smmu->irqs[i] = irq; in arm_smmu_device_probe()
2112 err = devm_clk_bulk_get_all(dev, &smmu->clks); in arm_smmu_device_probe()
2117 smmu->num_clks = err; in arm_smmu_device_probe()
2119 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); in arm_smmu_device_probe()
2123 err = arm_smmu_device_cfg_probe(smmu); in arm_smmu_device_probe()
2127 if (smmu->version == ARM_SMMU_V2) { in arm_smmu_device_probe()
2128 if (smmu->num_context_banks > smmu->num_context_irqs) { in arm_smmu_device_probe()
2131 smmu->num_context_irqs, smmu->num_context_banks); in arm_smmu_device_probe()
2136 smmu->num_context_irqs = smmu->num_context_banks; in arm_smmu_device_probe()
2139 if (smmu->impl && smmu->impl->global_fault) in arm_smmu_device_probe()
2140 global_fault = smmu->impl->global_fault; in arm_smmu_device_probe()
2151 "arm-smmu global fault", smmu); in arm_smmu_device_probe()
2158 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, in arm_smmu_device_probe()
2159 "smmu.%pa", &smmu->ioaddr); in arm_smmu_device_probe()
2165 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); in arm_smmu_device_probe()
2168 iommu_device_sysfs_remove(&smmu->iommu); in arm_smmu_device_probe()
2172 platform_set_drvdata(pdev, smmu); in arm_smmu_device_probe()
2175 arm_smmu_rmr_install_bypass_smr(smmu); in arm_smmu_device_probe()
2177 arm_smmu_device_reset(smmu); in arm_smmu_device_probe()
2178 arm_smmu_test_smr_masks(smmu); in arm_smmu_device_probe()
2196 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); in arm_smmu_device_shutdown() local
2198 if (!smmu) in arm_smmu_device_shutdown()
2201 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) in arm_smmu_device_shutdown()
2204 arm_smmu_rpm_get(smmu); in arm_smmu_device_shutdown()
2206 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD); in arm_smmu_device_shutdown()
2207 arm_smmu_rpm_put(smmu); in arm_smmu_device_shutdown()
2209 if (pm_runtime_enabled(smmu->dev)) in arm_smmu_device_shutdown()
2210 pm_runtime_force_suspend(smmu->dev); in arm_smmu_device_shutdown()
2212 clk_bulk_disable(smmu->num_clks, smmu->clks); in arm_smmu_device_shutdown()
2214 clk_bulk_unprepare(smmu->num_clks, smmu->clks); in arm_smmu_device_shutdown()
2219 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); in arm_smmu_device_remove() local
2221 if (!smmu) in arm_smmu_device_remove()
2224 iommu_device_unregister(&smmu->iommu); in arm_smmu_device_remove()
2225 iommu_device_sysfs_remove(&smmu->iommu); in arm_smmu_device_remove()
2234 struct arm_smmu_device *smmu = dev_get_drvdata(dev); in arm_smmu_runtime_resume() local
2237 ret = clk_bulk_enable(smmu->num_clks, smmu->clks); in arm_smmu_runtime_resume()
2241 arm_smmu_device_reset(smmu); in arm_smmu_runtime_resume()
2248 struct arm_smmu_device *smmu = dev_get_drvdata(dev); in arm_smmu_runtime_suspend() local
2250 clk_bulk_disable(smmu->num_clks, smmu->clks); in arm_smmu_runtime_suspend()
2258 struct arm_smmu_device *smmu = dev_get_drvdata(dev); in arm_smmu_pm_resume() local
2260 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks); in arm_smmu_pm_resume()
2269 clk_bulk_unprepare(smmu->num_clks, smmu->clks); in arm_smmu_pm_resume()
2277 struct arm_smmu_device *smmu = dev_get_drvdata(dev); in arm_smmu_pm_suspend() local
2287 clk_bulk_unprepare(smmu->num_clks, smmu->clks); in arm_smmu_pm_suspend()