Lines Matching refs:iommu
218 static inline bool context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn) in context_copied() argument
220 if (!iommu->copied_tables) in context_copied()
223 return test_bit(((long)bus << 8) | devfn, iommu->copied_tables); in context_copied()
227 set_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn) in set_context_copied() argument
229 set_bit(((long)bus << 8) | devfn, iommu->copied_tables); in set_context_copied()
233 clear_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn) in clear_context_copied() argument
235 clear_bit(((long)bus << 8) | devfn, iommu->copied_tables); in clear_context_copied()
268 struct intel_iommu *iommu; /* the corresponding iommu */ member
299 static bool translation_pre_enabled(struct intel_iommu *iommu) in translation_pre_enabled() argument
301 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED); in translation_pre_enabled()
304 static void clear_translation_pre_enabled(struct intel_iommu *iommu) in clear_translation_pre_enabled() argument
306 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED; in clear_translation_pre_enabled()
309 static void init_translation_status(struct intel_iommu *iommu) in init_translation_status() argument
313 gsts = readl(iommu->reg + DMAR_GSTS_REG); in init_translation_status()
315 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED; in init_translation_status()
399 static unsigned long __iommu_calculate_sagaw(struct intel_iommu *iommu) in __iommu_calculate_sagaw() argument
403 fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0); in __iommu_calculate_sagaw()
404 sl_sagaw = cap_sagaw(iommu->cap); in __iommu_calculate_sagaw()
407 if (!sm_supported(iommu) || !ecap_flts(iommu->ecap)) in __iommu_calculate_sagaw()
411 if (!ecap_slts(iommu->ecap)) in __iommu_calculate_sagaw()
417 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw) in __iommu_calculate_agaw() argument
422 sagaw = __iommu_calculate_sagaw(iommu); in __iommu_calculate_agaw()
434 int iommu_calculate_max_sagaw(struct intel_iommu *iommu) in iommu_calculate_max_sagaw() argument
436 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH); in iommu_calculate_max_sagaw()
444 int iommu_calculate_agaw(struct intel_iommu *iommu) in iommu_calculate_agaw() argument
446 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH); in iommu_calculate_agaw()
449 static inline bool iommu_paging_structure_coherency(struct intel_iommu *iommu) in iommu_paging_structure_coherency() argument
451 return sm_supported(iommu) ? in iommu_paging_structure_coherency()
452 ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap); in iommu_paging_structure_coherency()
459 struct intel_iommu *iommu; in domain_update_iommu_coherency() local
466 if (!iommu_paging_structure_coherency(info->iommu)) { in domain_update_iommu_coherency()
476 for_each_active_iommu(iommu, drhd) { in domain_update_iommu_coherency()
477 if (!iommu_paging_structure_coherency(iommu)) { in domain_update_iommu_coherency()
489 struct intel_iommu *iommu; in domain_update_iommu_superpage() local
497 for_each_active_iommu(iommu, drhd) { in domain_update_iommu_superpage()
498 if (iommu != skip) { in domain_update_iommu_superpage()
500 if (!cap_fl1gp_support(iommu->cap)) in domain_update_iommu_superpage()
503 mask &= cap_super_page_val(iommu->cap); in domain_update_iommu_superpage()
586 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, in iommu_context_addr() argument
589 struct root_entry *root = &iommu->root_entry[bus]; in iommu_context_addr()
597 if (!alloc && context_copied(iommu, bus, devfn)) in iommu_context_addr()
601 if (sm_supported(iommu)) { in iommu_context_addr()
615 context = alloc_pgtable_page(iommu->node, GFP_ATOMIC); in iommu_context_addr()
619 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE); in iommu_context_addr()
622 __iommu_flush_cache(iommu, entry, sizeof(*entry)); in iommu_context_addr()
684 static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev) in iommu_is_dummy() argument
686 if (!iommu || iommu->drhd->ignored) in iommu_is_dummy()
705 struct intel_iommu *iommu; in device_to_iommu() local
727 for_each_iommu(iommu, drhd) { in device_to_iommu()
761 iommu = NULL; in device_to_iommu()
763 if (iommu_is_dummy(iommu, dev)) in device_to_iommu()
764 iommu = NULL; in device_to_iommu()
768 return iommu; in device_to_iommu()
778 static void free_context_table(struct intel_iommu *iommu) in free_context_table() argument
783 if (!iommu->root_entry) in free_context_table()
787 context = iommu_context_addr(iommu, i, 0, 0); in free_context_table()
791 if (!sm_supported(iommu)) in free_context_table()
794 context = iommu_context_addr(iommu, i, 0x80, 0); in free_context_table()
799 free_pgtable_page(iommu->root_entry); in free_context_table()
800 iommu->root_entry = NULL; in free_context_table()
804 static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn, in pgtable_walk() argument
828 void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, in dmar_fault_dump_ptes() argument
840 pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr); in dmar_fault_dump_ptes()
843 rt_entry = &iommu->root_entry[bus]; in dmar_fault_dump_ptes()
849 if (sm_supported(iommu)) in dmar_fault_dump_ptes()
856 ctx_entry = iommu_context_addr(iommu, bus, devfn, 0); in dmar_fault_dump_ptes()
866 if (!sm_supported(iommu)) { in dmar_fault_dump_ptes()
906 pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level); in dmar_fault_dump_ptes()
1186 static int iommu_alloc_root_entry(struct intel_iommu *iommu) in iommu_alloc_root_entry() argument
1190 root = (struct root_entry *)alloc_pgtable_page(iommu->node, GFP_ATOMIC); in iommu_alloc_root_entry()
1193 iommu->name); in iommu_alloc_root_entry()
1197 __iommu_flush_cache(iommu, root, ROOT_SIZE); in iommu_alloc_root_entry()
1198 iommu->root_entry = root; in iommu_alloc_root_entry()
1203 static void iommu_set_root_entry(struct intel_iommu *iommu) in iommu_set_root_entry() argument
1209 addr = virt_to_phys(iommu->root_entry); in iommu_set_root_entry()
1210 if (sm_supported(iommu)) in iommu_set_root_entry()
1213 raw_spin_lock_irqsave(&iommu->register_lock, flag); in iommu_set_root_entry()
1214 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr); in iommu_set_root_entry()
1216 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG); in iommu_set_root_entry()
1219 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, in iommu_set_root_entry()
1222 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in iommu_set_root_entry()
1228 if (cap_esrtps(iommu->cap)) in iommu_set_root_entry()
1231 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); in iommu_set_root_entry()
1232 if (sm_supported(iommu)) in iommu_set_root_entry()
1233 qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0); in iommu_set_root_entry()
1234 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); in iommu_set_root_entry()
1237 void iommu_flush_write_buffer(struct intel_iommu *iommu) in iommu_flush_write_buffer() argument
1242 if (!rwbf_quirk && !cap_rwbf(iommu->cap)) in iommu_flush_write_buffer()
1245 raw_spin_lock_irqsave(&iommu->register_lock, flag); in iommu_flush_write_buffer()
1246 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG); in iommu_flush_write_buffer()
1249 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, in iommu_flush_write_buffer()
1252 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in iommu_flush_write_buffer()
1256 static void __iommu_flush_context(struct intel_iommu *iommu, in __iommu_flush_context() argument
1279 raw_spin_lock_irqsave(&iommu->register_lock, flag); in __iommu_flush_context()
1280 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val); in __iommu_flush_context()
1283 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG, in __iommu_flush_context()
1286 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in __iommu_flush_context()
1290 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, in __iommu_flush_iotlb() argument
1293 int tlb_offset = ecap_iotlb_offset(iommu->ecap); in __iommu_flush_iotlb()
1319 if (cap_read_drain(iommu->cap)) in __iommu_flush_iotlb()
1322 if (cap_write_drain(iommu->cap)) in __iommu_flush_iotlb()
1325 raw_spin_lock_irqsave(&iommu->register_lock, flag); in __iommu_flush_iotlb()
1328 dmar_writeq(iommu->reg + tlb_offset, val_iva); in __iommu_flush_iotlb()
1329 dmar_writeq(iommu->reg + tlb_offset + 8, val); in __iommu_flush_iotlb()
1332 IOMMU_WAIT_OP(iommu, tlb_offset + 8, in __iommu_flush_iotlb()
1335 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in __iommu_flush_iotlb()
1348 struct intel_iommu *iommu, u8 bus, u8 devfn) in domain_lookup_dev_info() argument
1355 if (info->iommu == iommu && info->bus == bus && in domain_lookup_dev_info()
1414 if (!ecap_dit(info->iommu->ecap)) in iommu_enable_pci_caps()
1481 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid, in __iommu_flush_dev_iotlb()
1501 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, in iommu_flush_iotlb_psi() argument
1509 u16 did = domain_id_iommu(domain, iommu); in iommu_flush_iotlb_psi()
1517 qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, pages, ih); in iommu_flush_iotlb_psi()
1544 if (!cap_pgsel_inv(iommu->cap) || in iommu_flush_iotlb_psi()
1545 mask > cap_max_amask_val(iommu->cap)) in iommu_flush_iotlb_psi()
1546 iommu->flush.flush_iotlb(iommu, did, 0, 0, in iommu_flush_iotlb_psi()
1549 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask, in iommu_flush_iotlb_psi()
1557 if (!cap_caching_mode(iommu->cap) || !map) in iommu_flush_iotlb_psi()
1562 static inline void __mapping_notify_one(struct intel_iommu *iommu, in __mapping_notify_one() argument
1570 if (cap_caching_mode(iommu->cap) && !domain->use_first_level) in __mapping_notify_one()
1571 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1); in __mapping_notify_one()
1573 iommu_flush_write_buffer(iommu); in __mapping_notify_one()
1583 struct intel_iommu *iommu = info->iommu; in intel_flush_iotlb_all() local
1584 u16 did = domain_id_iommu(dmar_domain, iommu); in intel_flush_iotlb_all()
1587 qi_flush_piotlb(iommu, did, PASID_RID2PASID, 0, -1, 0); in intel_flush_iotlb_all()
1589 iommu->flush.flush_iotlb(iommu, did, 0, 0, in intel_flush_iotlb_all()
1592 if (!cap_caching_mode(iommu->cap)) in intel_flush_iotlb_all()
1597 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) in iommu_disable_protect_mem_regions() argument
1602 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap)) in iommu_disable_protect_mem_regions()
1605 raw_spin_lock_irqsave(&iommu->register_lock, flags); in iommu_disable_protect_mem_regions()
1606 pmen = readl(iommu->reg + DMAR_PMEN_REG); in iommu_disable_protect_mem_regions()
1608 writel(pmen, iommu->reg + DMAR_PMEN_REG); in iommu_disable_protect_mem_regions()
1611 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG, in iommu_disable_protect_mem_regions()
1614 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in iommu_disable_protect_mem_regions()
1617 static void iommu_enable_translation(struct intel_iommu *iommu) in iommu_enable_translation() argument
1622 raw_spin_lock_irqsave(&iommu->register_lock, flags); in iommu_enable_translation()
1623 iommu->gcmd |= DMA_GCMD_TE; in iommu_enable_translation()
1624 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); in iommu_enable_translation()
1627 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, in iommu_enable_translation()
1630 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in iommu_enable_translation()
1633 static void iommu_disable_translation(struct intel_iommu *iommu) in iommu_disable_translation() argument
1638 if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated && in iommu_disable_translation()
1639 (cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap))) in iommu_disable_translation()
1642 raw_spin_lock_irqsave(&iommu->register_lock, flag); in iommu_disable_translation()
1643 iommu->gcmd &= ~DMA_GCMD_TE; in iommu_disable_translation()
1644 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); in iommu_disable_translation()
1647 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, in iommu_disable_translation()
1650 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in iommu_disable_translation()
1653 static int iommu_init_domains(struct intel_iommu *iommu) in iommu_init_domains() argument
1657 ndomains = cap_ndoms(iommu->cap); in iommu_init_domains()
1659 iommu->name, ndomains); in iommu_init_domains()
1661 spin_lock_init(&iommu->lock); in iommu_init_domains()
1663 iommu->domain_ids = bitmap_zalloc(ndomains, GFP_KERNEL); in iommu_init_domains()
1664 if (!iommu->domain_ids) in iommu_init_domains()
1673 set_bit(0, iommu->domain_ids); in iommu_init_domains()
1682 if (sm_supported(iommu)) in iommu_init_domains()
1683 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids); in iommu_init_domains()
1688 static void disable_dmar_iommu(struct intel_iommu *iommu) in disable_dmar_iommu() argument
1690 if (!iommu->domain_ids) in disable_dmar_iommu()
1697 if (WARN_ON(bitmap_weight(iommu->domain_ids, cap_ndoms(iommu->cap)) in disable_dmar_iommu()
1701 if (iommu->gcmd & DMA_GCMD_TE) in disable_dmar_iommu()
1702 iommu_disable_translation(iommu); in disable_dmar_iommu()
1705 static void free_dmar_iommu(struct intel_iommu *iommu) in free_dmar_iommu() argument
1707 if (iommu->domain_ids) { in free_dmar_iommu()
1708 bitmap_free(iommu->domain_ids); in free_dmar_iommu()
1709 iommu->domain_ids = NULL; in free_dmar_iommu()
1712 if (iommu->copied_tables) { in free_dmar_iommu()
1713 bitmap_free(iommu->copied_tables); in free_dmar_iommu()
1714 iommu->copied_tables = NULL; in free_dmar_iommu()
1718 free_context_table(iommu); in free_dmar_iommu()
1721 if (pasid_supported(iommu)) { in free_dmar_iommu()
1722 if (ecap_prs(iommu->ecap)) in free_dmar_iommu()
1723 intel_svm_finish_prq(iommu); in free_dmar_iommu()
1725 if (vccap_pasid(iommu->vccap)) in free_dmar_iommu()
1726 ioasid_unregister_allocator(&iommu->pasid_allocator); in free_dmar_iommu()
1769 struct intel_iommu *iommu) in domain_attach_iommu() argument
1779 spin_lock(&iommu->lock); in domain_attach_iommu()
1780 curr = xa_load(&domain->iommu_array, iommu->seq_id); in domain_attach_iommu()
1783 spin_unlock(&iommu->lock); in domain_attach_iommu()
1788 ndomains = cap_ndoms(iommu->cap); in domain_attach_iommu()
1789 num = find_first_zero_bit(iommu->domain_ids, ndomains); in domain_attach_iommu()
1791 pr_err("%s: No free domain ids\n", iommu->name); in domain_attach_iommu()
1795 set_bit(num, iommu->domain_ids); in domain_attach_iommu()
1798 info->iommu = iommu; in domain_attach_iommu()
1799 curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id, in domain_attach_iommu()
1807 spin_unlock(&iommu->lock); in domain_attach_iommu()
1811 clear_bit(info->did, iommu->domain_ids); in domain_attach_iommu()
1813 spin_unlock(&iommu->lock); in domain_attach_iommu()
1819 struct intel_iommu *iommu) in domain_detach_iommu() argument
1823 spin_lock(&iommu->lock); in domain_detach_iommu()
1824 info = xa_load(&domain->iommu_array, iommu->seq_id); in domain_detach_iommu()
1826 clear_bit(info->did, iommu->domain_ids); in domain_detach_iommu()
1827 xa_erase(&domain->iommu_array, iommu->seq_id); in domain_detach_iommu()
1832 spin_unlock(&iommu->lock); in domain_detach_iommu()
1914 struct intel_iommu *iommu, in domain_context_mapping_one() argument
1919 domain_lookup_dev_info(domain, iommu, bus, devfn); in domain_context_mapping_one()
1920 u16 did = domain_id_iommu(domain, iommu); in domain_context_mapping_one()
1935 spin_lock(&iommu->lock); in domain_context_mapping_one()
1937 context = iommu_context_addr(iommu, bus, devfn, 1); in domain_context_mapping_one()
1942 if (context_present(context) && !context_copied(iommu, bus, devfn)) in domain_context_mapping_one()
1954 if (context_copied(iommu, bus, devfn)) { in domain_context_mapping_one()
1957 if (did_old < cap_ndoms(iommu->cap)) { in domain_context_mapping_one()
1958 iommu->flush.flush_context(iommu, did_old, in domain_context_mapping_one()
1962 iommu->flush.flush_iotlb(iommu, did_old, 0, 0, in domain_context_mapping_one()
1966 clear_context_copied(iommu, bus, devfn); in domain_context_mapping_one()
1971 if (sm_supported(iommu)) { in domain_context_mapping_one()
2005 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { in domain_context_mapping_one()
2025 context_set_address_width(context, iommu->msagaw); in domain_context_mapping_one()
2033 if (!ecap_coherent(iommu->ecap)) in domain_context_mapping_one()
2042 if (cap_caching_mode(iommu->cap)) { in domain_context_mapping_one()
2043 iommu->flush.flush_context(iommu, 0, in domain_context_mapping_one()
2047 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); in domain_context_mapping_one()
2049 iommu_flush_write_buffer(iommu); in domain_context_mapping_one()
2055 spin_unlock(&iommu->lock); in domain_context_mapping_one()
2062 struct intel_iommu *iommu; member
2071 return domain_context_mapping_one(data->domain, data->iommu, in domain_context_mapping_cb()
2081 struct intel_iommu *iommu; in domain_context_mapping() local
2084 iommu = device_to_iommu(dev, &bus, &devfn); in domain_context_mapping()
2085 if (!iommu) in domain_context_mapping()
2091 return domain_context_mapping_one(domain, iommu, table, in domain_context_mapping()
2095 data.iommu = iommu; in domain_context_mapping()
2163 iommu_flush_iotlb_psi(info->iommu, domain, in switch_to_super_page()
2277 struct intel_iommu *iommu = info->iommu; in domain_context_clear_one() local
2281 if (!iommu) in domain_context_clear_one()
2284 spin_lock(&iommu->lock); in domain_context_clear_one()
2285 context = iommu_context_addr(iommu, bus, devfn, 0); in domain_context_clear_one()
2287 spin_unlock(&iommu->lock); in domain_context_clear_one()
2291 if (sm_supported(iommu)) { in domain_context_clear_one()
2295 did_old = domain_id_iommu(info->domain, iommu); in domain_context_clear_one()
2301 __iommu_flush_cache(iommu, context, sizeof(*context)); in domain_context_clear_one()
2302 spin_unlock(&iommu->lock); in domain_context_clear_one()
2303 iommu->flush.flush_context(iommu, in domain_context_clear_one()
2309 if (sm_supported(iommu)) in domain_context_clear_one()
2310 qi_flush_pasid_cache(iommu, did_old, QI_PC_ALL_PASIDS, 0); in domain_context_clear_one()
2312 iommu->flush.flush_iotlb(iommu, in domain_context_clear_one()
2321 static int domain_setup_first_level(struct intel_iommu *iommu, in domain_setup_first_level() argument
2334 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { in domain_setup_first_level()
2352 return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid, in domain_setup_first_level()
2353 domain_id_iommu(domain, iommu), in domain_setup_first_level()
2441 struct intel_iommu *iommu; in dmar_domain_attach_device() local
2446 iommu = device_to_iommu(dev, &bus, &devfn); in dmar_domain_attach_device()
2447 if (!iommu) in dmar_domain_attach_device()
2450 ret = domain_attach_iommu(domain, iommu); in dmar_domain_attach_device()
2459 if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) { in dmar_domain_attach_device()
2462 ret = intel_pasid_setup_pass_through(iommu, domain, in dmar_domain_attach_device()
2465 ret = domain_setup_first_level(iommu, domain, dev, in dmar_domain_attach_device()
2468 ret = intel_pasid_setup_second_level(iommu, domain, in dmar_domain_attach_device()
2596 static void intel_iommu_init_qi(struct intel_iommu *iommu) in intel_iommu_init_qi() argument
2604 if (!iommu->qi) { in intel_iommu_init_qi()
2608 dmar_fault(-1, iommu); in intel_iommu_init_qi()
2613 dmar_disable_qi(iommu); in intel_iommu_init_qi()
2616 if (dmar_enable_qi(iommu)) { in intel_iommu_init_qi()
2620 iommu->flush.flush_context = __iommu_flush_context; in intel_iommu_init_qi()
2621 iommu->flush.flush_iotlb = __iommu_flush_iotlb; in intel_iommu_init_qi()
2623 iommu->name); in intel_iommu_init_qi()
2625 iommu->flush.flush_context = qi_flush_context; in intel_iommu_init_qi()
2626 iommu->flush.flush_iotlb = qi_flush_iotlb; in intel_iommu_init_qi()
2627 pr_info("%s: Using Queued invalidation\n", iommu->name); in intel_iommu_init_qi()
2631 static int copy_context_table(struct intel_iommu *iommu, in copy_context_table() argument
2653 __iommu_flush_cache(iommu, new_ce, in copy_context_table()
2683 new_ce = alloc_pgtable_page(iommu->node, GFP_KERNEL); in copy_context_table()
2697 if (did >= 0 && did < cap_ndoms(iommu->cap)) in copy_context_table()
2698 set_bit(did, iommu->domain_ids); in copy_context_table()
2700 set_context_copied(iommu, bus, devfn); in copy_context_table()
2706 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE); in copy_context_table()
2715 static int copy_translation_tables(struct intel_iommu *iommu) in copy_translation_tables() argument
2725 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG); in copy_translation_tables()
2727 new_ext = !!sm_supported(iommu); in copy_translation_tables()
2738 iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL); in copy_translation_tables()
2739 if (!iommu->copied_tables) in copy_translation_tables()
2758 ret = copy_context_table(iommu, &old_rt[bus], in copy_translation_tables()
2762 iommu->name, bus); in copy_translation_tables()
2767 spin_lock(&iommu->lock); in copy_translation_tables()
2776 iommu->root_entry[bus].lo = val; in copy_translation_tables()
2783 iommu->root_entry[bus].hi = val; in copy_translation_tables()
2786 spin_unlock(&iommu->lock); in copy_translation_tables()
2790 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE); in copy_translation_tables()
2803 struct intel_iommu *iommu = data; in intel_vcmd_ioasid_alloc() local
2806 if (!iommu) in intel_vcmd_ioasid_alloc()
2816 if (vcmd_alloc_pasid(iommu, &ioasid)) in intel_vcmd_ioasid_alloc()
2824 struct intel_iommu *iommu = data; in intel_vcmd_ioasid_free() local
2826 if (!iommu) in intel_vcmd_ioasid_free()
2836 vcmd_free_pasid(iommu, ioasid); in intel_vcmd_ioasid_free()
2839 static void register_pasid_allocator(struct intel_iommu *iommu) in register_pasid_allocator() argument
2845 if (!cap_caching_mode(iommu->cap)) in register_pasid_allocator()
2848 if (!sm_supported(iommu)) { in register_pasid_allocator()
2860 if (!vccap_pasid(iommu->vccap)) in register_pasid_allocator()
2864 iommu->pasid_allocator.alloc = intel_vcmd_ioasid_alloc; in register_pasid_allocator()
2865 iommu->pasid_allocator.free = intel_vcmd_ioasid_free; in register_pasid_allocator()
2866 iommu->pasid_allocator.pdata = (void *)iommu; in register_pasid_allocator()
2867 if (ioasid_register_allocator(&iommu->pasid_allocator)) { in register_pasid_allocator()
2882 struct intel_iommu *iommu; in init_dmars() local
2889 for_each_iommu(iommu, drhd) { in init_dmars()
2891 iommu_disable_translation(iommu); in init_dmars()
2900 if (pasid_supported(iommu)) { in init_dmars()
2901 u32 temp = 2 << ecap_pss(iommu->ecap); in init_dmars()
2907 intel_iommu_init_qi(iommu); in init_dmars()
2909 ret = iommu_init_domains(iommu); in init_dmars()
2913 init_translation_status(iommu); in init_dmars()
2915 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) { in init_dmars()
2916 iommu_disable_translation(iommu); in init_dmars()
2917 clear_translation_pre_enabled(iommu); in init_dmars()
2919 iommu->name); in init_dmars()
2927 ret = iommu_alloc_root_entry(iommu); in init_dmars()
2931 if (translation_pre_enabled(iommu)) { in init_dmars()
2934 ret = copy_translation_tables(iommu); in init_dmars()
2946 iommu->name); in init_dmars()
2947 iommu_disable_translation(iommu); in init_dmars()
2948 clear_translation_pre_enabled(iommu); in init_dmars()
2951 iommu->name); in init_dmars()
2955 if (!ecap_pass_through(iommu->ecap)) in init_dmars()
2957 intel_svm_check(iommu); in init_dmars()
2965 for_each_active_iommu(iommu, drhd) { in init_dmars()
2966 iommu_flush_write_buffer(iommu); in init_dmars()
2968 register_pasid_allocator(iommu); in init_dmars()
2970 iommu_set_root_entry(iommu); in init_dmars()
2993 for_each_iommu(iommu, drhd) { in init_dmars()
3000 iommu_disable_protect_mem_regions(iommu); in init_dmars()
3004 iommu_flush_write_buffer(iommu); in init_dmars()
3007 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) { in init_dmars()
3013 ret = intel_svm_enable_prq(iommu); in init_dmars()
3019 ret = dmar_set_interrupt(iommu); in init_dmars()
3027 for_each_active_iommu(iommu, drhd) { in init_dmars()
3028 disable_dmar_iommu(iommu); in init_dmars()
3029 free_dmar_iommu(iommu); in init_dmars()
3079 struct intel_iommu *iommu = NULL; in init_iommu_hw() local
3081 for_each_active_iommu(iommu, drhd) in init_iommu_hw()
3082 if (iommu->qi) in init_iommu_hw()
3083 dmar_reenable_qi(iommu); in init_iommu_hw()
3085 for_each_iommu(iommu, drhd) { in init_iommu_hw()
3092 iommu_disable_protect_mem_regions(iommu); in init_iommu_hw()
3096 iommu_flush_write_buffer(iommu); in init_iommu_hw()
3097 iommu_set_root_entry(iommu); in init_iommu_hw()
3098 iommu_enable_translation(iommu); in init_iommu_hw()
3099 iommu_disable_protect_mem_regions(iommu); in init_iommu_hw()
3108 struct intel_iommu *iommu; in iommu_flush_all() local
3110 for_each_active_iommu(iommu, drhd) { in iommu_flush_all()
3111 iommu->flush.flush_context(iommu, 0, 0, 0, in iommu_flush_all()
3113 iommu->flush.flush_iotlb(iommu, 0, 0, 0, in iommu_flush_all()
3121 struct intel_iommu *iommu = NULL; in iommu_suspend() local
3124 for_each_active_iommu(iommu, drhd) { in iommu_suspend()
3125 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32), in iommu_suspend()
3127 if (!iommu->iommu_state) in iommu_suspend()
3133 for_each_active_iommu(iommu, drhd) { in iommu_suspend()
3134 iommu_disable_translation(iommu); in iommu_suspend()
3136 raw_spin_lock_irqsave(&iommu->register_lock, flag); in iommu_suspend()
3138 iommu->iommu_state[SR_DMAR_FECTL_REG] = in iommu_suspend()
3139 readl(iommu->reg + DMAR_FECTL_REG); in iommu_suspend()
3140 iommu->iommu_state[SR_DMAR_FEDATA_REG] = in iommu_suspend()
3141 readl(iommu->reg + DMAR_FEDATA_REG); in iommu_suspend()
3142 iommu->iommu_state[SR_DMAR_FEADDR_REG] = in iommu_suspend()
3143 readl(iommu->reg + DMAR_FEADDR_REG); in iommu_suspend()
3144 iommu->iommu_state[SR_DMAR_FEUADDR_REG] = in iommu_suspend()
3145 readl(iommu->reg + DMAR_FEUADDR_REG); in iommu_suspend()
3147 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in iommu_suspend()
3152 for_each_active_iommu(iommu, drhd) in iommu_suspend()
3153 kfree(iommu->iommu_state); in iommu_suspend()
3161 struct intel_iommu *iommu = NULL; in iommu_resume() local
3172 for_each_active_iommu(iommu, drhd) { in iommu_resume()
3174 raw_spin_lock_irqsave(&iommu->register_lock, flag); in iommu_resume()
3176 writel(iommu->iommu_state[SR_DMAR_FECTL_REG], in iommu_resume()
3177 iommu->reg + DMAR_FECTL_REG); in iommu_resume()
3178 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG], in iommu_resume()
3179 iommu->reg + DMAR_FEDATA_REG); in iommu_resume()
3180 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG], in iommu_resume()
3181 iommu->reg + DMAR_FEADDR_REG); in iommu_resume()
3182 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG], in iommu_resume()
3183 iommu->reg + DMAR_FEUADDR_REG); in iommu_resume()
3185 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in iommu_resume()
3188 for_each_active_iommu(iommu, drhd) in iommu_resume()
3189 kfree(iommu->iommu_state); in iommu_resume()
3414 struct intel_iommu *iommu = dmaru->iommu; in intel_iommu_add() local
3416 ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_DMAR, iommu); in intel_iommu_add()
3420 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) { in intel_iommu_add()
3422 iommu->name); in intel_iommu_add()
3426 sp = domain_update_iommu_superpage(NULL, iommu) - 1; in intel_iommu_add()
3427 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) { in intel_iommu_add()
3429 iommu->name); in intel_iommu_add()
3436 if (iommu->gcmd & DMA_GCMD_TE) in intel_iommu_add()
3437 iommu_disable_translation(iommu); in intel_iommu_add()
3439 ret = iommu_init_domains(iommu); in intel_iommu_add()
3441 ret = iommu_alloc_root_entry(iommu); in intel_iommu_add()
3445 intel_svm_check(iommu); in intel_iommu_add()
3452 iommu_disable_protect_mem_regions(iommu); in intel_iommu_add()
3456 intel_iommu_init_qi(iommu); in intel_iommu_add()
3457 iommu_flush_write_buffer(iommu); in intel_iommu_add()
3460 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) { in intel_iommu_add()
3461 ret = intel_svm_enable_prq(iommu); in intel_iommu_add()
3466 ret = dmar_set_interrupt(iommu); in intel_iommu_add()
3470 iommu_set_root_entry(iommu); in intel_iommu_add()
3471 iommu_enable_translation(iommu); in intel_iommu_add()
3473 iommu_disable_protect_mem_regions(iommu); in intel_iommu_add()
3477 disable_dmar_iommu(iommu); in intel_iommu_add()
3479 free_dmar_iommu(iommu); in intel_iommu_add()
3486 struct intel_iommu *iommu = dmaru->iommu; in dmar_iommu_hotplug() local
3490 if (iommu == NULL) in dmar_iommu_hotplug()
3496 disable_dmar_iommu(iommu); in dmar_iommu_hotplug()
3497 free_dmar_iommu(iommu); in dmar_iommu_hotplug()
3550 static int dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu) in dmar_ats_supported() argument
3570 return !(satcu->atc_required && !sm_supported(iommu)); in dmar_ats_supported()
3698 struct intel_iommu *iommu; in intel_iommu_memory_notifier() local
3704 for_each_active_iommu(iommu, drhd) in intel_iommu_memory_notifier()
3705 iommu_flush_iotlb_psi(iommu, si_domain, in intel_iommu_memory_notifier()
3724 struct intel_iommu *iommu = NULL; in intel_disable_iommus() local
3727 for_each_iommu(iommu, drhd) in intel_disable_iommus()
3728 iommu_disable_translation(iommu); in intel_disable_iommus()
3734 struct intel_iommu *iommu = NULL; in intel_iommu_shutdown() local
3742 for_each_iommu(iommu, drhd) in intel_iommu_shutdown()
3743 iommu_disable_protect_mem_regions(iommu); in intel_iommu_shutdown()
3755 return container_of(iommu_dev, struct intel_iommu, iommu); in dev_to_intel_iommu()
3761 struct intel_iommu *iommu = dev_to_intel_iommu(dev); in version_show() local
3762 u32 ver = readl(iommu->reg + DMAR_VER_REG); in version_show()
3771 struct intel_iommu *iommu = dev_to_intel_iommu(dev); in address_show() local
3772 return sprintf(buf, "%llx\n", iommu->reg_phys); in address_show()
3779 struct intel_iommu *iommu = dev_to_intel_iommu(dev); in cap_show() local
3780 return sprintf(buf, "%llx\n", iommu->cap); in cap_show()
3787 struct intel_iommu *iommu = dev_to_intel_iommu(dev); in ecap_show() local
3788 return sprintf(buf, "%llx\n", iommu->ecap); in ecap_show()
3795 struct intel_iommu *iommu = dev_to_intel_iommu(dev); in domains_supported_show() local
3796 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap)); in domains_supported_show()
3803 struct intel_iommu *iommu = dev_to_intel_iommu(dev); in domains_used_show() local
3804 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids, in domains_used_show()
3805 cap_ndoms(iommu->cap))); in domains_used_show()
3867 struct intel_iommu *iommu __maybe_unused; in probe_acpi_namespace_devices()
3871 for_each_active_iommu(iommu, drhd) { in probe_acpi_namespace_devices()
3923 struct intel_iommu *iommu; in intel_iommu_init() local
3968 for_each_iommu(iommu, drhd) in intel_iommu_init()
3969 iommu_disable_protect_mem_regions(iommu); in intel_iommu_init()
4004 for_each_active_iommu(iommu, drhd) { in intel_iommu_init()
4012 if (cap_caching_mode(iommu->cap) && in intel_iommu_init()
4017 iommu_device_sysfs_add(&iommu->iommu, NULL, in intel_iommu_init()
4019 "%s", iommu->name); in intel_iommu_init()
4020 iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL); in intel_iommu_init()
4022 iommu_pmu_register(iommu); in intel_iommu_init()
4034 for_each_iommu(iommu, drhd) { in intel_iommu_init()
4035 if (!drhd->ignored && !translation_pre_enabled(iommu)) in intel_iommu_init()
4036 iommu_enable_translation(iommu); in intel_iommu_init()
4038 iommu_disable_protect_mem_regions(iommu); in intel_iommu_init()
4070 if (!info->iommu || !info->dev || !dev_is_pci(info->dev)) in domain_context_clear()
4081 struct intel_iommu *iommu = info->iommu; in dmar_remove_one_dev_info() local
4085 if (dev_is_pci(info->dev) && sm_supported(iommu)) in dmar_remove_one_dev_info()
4086 intel_pasid_tear_down_entry(iommu, info->dev, in dmar_remove_one_dev_info()
4097 domain_detach_iommu(domain, iommu); in dmar_remove_one_dev_info()
4109 struct intel_iommu *iommu = info->iommu; in device_block_translation() local
4114 if (sm_supported(iommu)) in device_block_translation()
4115 intel_pasid_tear_down_entry(iommu, dev, in device_block_translation()
4128 domain_detach_iommu(info->domain, iommu); in device_block_translation()
4217 struct intel_iommu *iommu; in prepare_domain_attach_device() local
4220 iommu = device_to_iommu(dev, NULL, NULL); in prepare_domain_attach_device()
4221 if (!iommu) in prepare_domain_attach_device()
4224 if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap)) in prepare_domain_attach_device()
4228 addr_width = agaw_to_width(iommu->agaw); in prepare_domain_attach_device()
4229 if (addr_width > cap_mgaw(iommu->cap)) in prepare_domain_attach_device()
4230 addr_width = cap_mgaw(iommu->cap); in prepare_domain_attach_device()
4239 while (iommu->agaw < dmar_domain->agaw) { in prepare_domain_attach_device()
4393 iommu_flush_iotlb_psi(info->iommu, dmar_domain, in intel_iommu_tlb_sync()
4425 if (!ecap_sc_support(info->iommu->ecap)) { in domain_support_force_snooping()
4449 intel_pasid_setup_page_snoop_control(info->iommu, info->dev, in domain_set_force_snooping()
4484 return ecap_sc_support(info->iommu->ecap); in intel_iommu_capable()
4494 struct intel_iommu *iommu; in intel_iommu_probe_device() local
4498 iommu = device_to_iommu(dev, &bus, &devfn); in intel_iommu_probe_device()
4499 if (!iommu || !iommu->iommu.ops) in intel_iommu_probe_device()
4513 info->segment = iommu->segment; in intel_iommu_probe_device()
4517 info->iommu = iommu; in intel_iommu_probe_device()
4519 if (ecap_dev_iotlb_support(iommu->ecap) && in intel_iommu_probe_device()
4521 dmar_ats_supported(pdev, iommu)) { in intel_iommu_probe_device()
4525 if (sm_supported(iommu)) { in intel_iommu_probe_device()
4526 if (pasid_supported(iommu)) { in intel_iommu_probe_device()
4533 if (info->ats_supported && ecap_prs(iommu->ecap) && in intel_iommu_probe_device()
4541 if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) { in intel_iommu_probe_device()
4551 return &iommu->iommu; in intel_iommu_probe_device()
4640 struct intel_iommu *iommu; in intel_iommu_enable_sva() local
4646 iommu = info->iommu; in intel_iommu_enable_sva()
4647 if (!iommu) in intel_iommu_enable_sva()
4650 if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE)) in intel_iommu_enable_sva()
4656 ret = iopf_queue_add_device(iommu->iopf_queue, dev); in intel_iommu_enable_sva()
4662 iopf_queue_remove_device(iommu->iopf_queue, dev); in intel_iommu_enable_sva()
4670 struct intel_iommu *iommu = info->iommu; in intel_iommu_disable_sva() local
4677 ret = iopf_queue_remove_device(iommu->iopf_queue, dev); in intel_iommu_disable_sva()
4728 return translation_pre_enabled(info->iommu) && !info->domain; in intel_iommu_is_attach_deferred()
4758 __mapping_notify_one(info->iommu, dmar_domain, pfn, pages); in intel_iommu_iotlb_sync_map()
4763 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); in intel_iommu_remove_dev_pasid() local
4780 intel_pasid_tear_down_entry(iommu, dev, pasid, false); in intel_iommu_remove_dev_pasid()
5039 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid, in quirk_extra_dev_tlb_flush()
5042 qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid, in quirk_extra_dev_tlb_flush()
5061 int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob) in ecmd_submit_sync() argument
5067 if (!cap_ecmds(iommu->cap)) in ecmd_submit_sync()
5070 raw_spin_lock_irqsave(&iommu->register_lock, flags); in ecmd_submit_sync()
5072 res = dmar_readq(iommu->reg + DMAR_ECRSP_REG); in ecmd_submit_sync()
5085 dmar_writeq(iommu->reg + DMAR_ECEO_REG, ob); in ecmd_submit_sync()
5086 dmar_writeq(iommu->reg + DMAR_ECMD_REG, ecmd | (oa << DMA_ECMD_OA_SHIFT)); in ecmd_submit_sync()
5088 IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, dmar_readq, in ecmd_submit_sync()
5098 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in ecmd_submit_sync()