Lines Matching refs:cfg
194 struct io_pgtable_cfg *cfg) in __arm_lpae_alloc_pages() argument
196 struct device *dev = cfg->iommu_dev; in __arm_lpae_alloc_pages()
209 if (!cfg->coherent_walk) { in __arm_lpae_alloc_pages()
233 struct io_pgtable_cfg *cfg) in __arm_lpae_free_pages() argument
235 if (!cfg->coherent_walk) in __arm_lpae_free_pages()
236 dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), in __arm_lpae_free_pages()
242 struct io_pgtable_cfg *cfg) in __arm_lpae_sync_pte() argument
244 dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), in __arm_lpae_sync_pte()
248 static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg) in __arm_lpae_clear_pte() argument
253 if (!cfg->coherent_walk) in __arm_lpae_clear_pte()
254 __arm_lpae_sync_pte(ptep, 1, cfg); in __arm_lpae_clear_pte()
267 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_lpae_init_pte() local
279 if (!cfg->coherent_walk) in __arm_lpae_init_pte()
280 __arm_lpae_sync_pte(ptep, num_entries, cfg); in __arm_lpae_init_pte()
318 struct io_pgtable_cfg *cfg) in arm_lpae_install_table() argument
323 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_lpae_install_table()
335 if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC)) in arm_lpae_install_table()
339 __arm_lpae_sync_pte(ptep, 1, cfg); in arm_lpae_install_table()
354 struct io_pgtable_cfg *cfg = &data->iop.cfg; in __arm_lpae_map() local
379 cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg); in __arm_lpae_map()
383 pte = arm_lpae_install_table(cptep, ptep, 0, cfg); in __arm_lpae_map()
385 __arm_lpae_free_pages(cptep, tblsz, cfg); in __arm_lpae_map()
386 } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { in __arm_lpae_map()
387 __arm_lpae_sync_pte(ptep, 1, cfg); in __arm_lpae_map()
467 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) in arm_lpae_prot_to_pte()
481 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_map_pages() local
485 long iaext = (s64)iova >> cfg->ias; in arm_lpae_map_pages()
487 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) in arm_lpae_map_pages()
490 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) in arm_lpae_map_pages()
492 if (WARN_ON(iaext || paddr >> cfg->oas)) in arm_lpae_map_pages()
546 __arm_lpae_free_pages(start, table_size, &data->iop.cfg); in __arm_lpae_free_pgtable()
563 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_split_blk_unmap() local
574 tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg); in arm_lpae_split_blk_unmap()
595 pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg); in arm_lpae_split_blk_unmap()
597 __arm_lpae_free_pages(tablep, tablesz, cfg); in arm_lpae_split_blk_unmap()
646 __arm_lpae_clear_pte(ptep, &iop->cfg); in __arm_lpae_unmap()
681 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_unmap_pages() local
683 long iaext = (s64)iova >> cfg->ias; in arm_lpae_unmap_pages()
685 if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) in arm_lpae_unmap_pages()
688 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) in arm_lpae_unmap_pages()
739 static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) in arm_lpae_restrict_pgsizes() argument
751 if (cfg->pgsize_bitmap & PAGE_SIZE) in arm_lpae_restrict_pgsizes()
753 else if (cfg->pgsize_bitmap & ~PAGE_MASK) in arm_lpae_restrict_pgsizes()
754 granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); in arm_lpae_restrict_pgsizes()
755 else if (cfg->pgsize_bitmap & PAGE_MASK) in arm_lpae_restrict_pgsizes()
756 granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); in arm_lpae_restrict_pgsizes()
770 if (cfg->oas > 48) in arm_lpae_restrict_pgsizes()
777 cfg->pgsize_bitmap &= page_sizes; in arm_lpae_restrict_pgsizes()
778 cfg->ias = min(cfg->ias, max_addr_bits); in arm_lpae_restrict_pgsizes()
779 cfg->oas = min(cfg->oas, max_addr_bits); in arm_lpae_restrict_pgsizes()
783 arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) in arm_lpae_alloc_pgtable() argument
788 arm_lpae_restrict_pgsizes(cfg); in arm_lpae_alloc_pgtable()
790 if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) in arm_lpae_alloc_pgtable()
793 if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) in arm_lpae_alloc_pgtable()
796 if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) in arm_lpae_alloc_pgtable()
803 pg_shift = __ffs(cfg->pgsize_bitmap); in arm_lpae_alloc_pgtable()
806 va_bits = cfg->ias - pg_shift; in arm_lpae_alloc_pgtable()
825 arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) in arm_64_lpae_alloc_pgtable_s1() argument
829 typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; in arm_64_lpae_alloc_pgtable_s1()
832 if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | in arm_64_lpae_alloc_pgtable_s1()
837 data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s1()
842 if (cfg->coherent_walk) { in arm_64_lpae_alloc_pgtable_s1()
846 if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA) in arm_64_lpae_alloc_pgtable_s1()
851 if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) in arm_64_lpae_alloc_pgtable_s1()
857 tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1; in arm_64_lpae_alloc_pgtable_s1()
870 switch (cfg->oas) { in arm_64_lpae_alloc_pgtable_s1()
896 tcr->tsz = 64ULL - cfg->ias; in arm_64_lpae_alloc_pgtable_s1()
908 cfg->arm_lpae_s1_cfg.mair = reg; in arm_64_lpae_alloc_pgtable_s1()
912 GFP_KERNEL, cfg); in arm_64_lpae_alloc_pgtable_s1()
920 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s1()
929 arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) in arm_64_lpae_alloc_pgtable_s2() argument
933 typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; in arm_64_lpae_alloc_pgtable_s2()
936 if (cfg->quirks) in arm_64_lpae_alloc_pgtable_s2()
939 data = arm_lpae_alloc_pgtable(cfg); in arm_64_lpae_alloc_pgtable_s2()
958 if (cfg->coherent_walk) { in arm_64_lpae_alloc_pgtable_s2()
983 switch (cfg->oas) { in arm_64_lpae_alloc_pgtable_s2()
1009 vtcr->tsz = 64ULL - cfg->ias; in arm_64_lpae_alloc_pgtable_s2()
1014 GFP_KERNEL, cfg); in arm_64_lpae_alloc_pgtable_s2()
1022 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); in arm_64_lpae_alloc_pgtable_s2()
1031 arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) in arm_32_lpae_alloc_pgtable_s1() argument
1033 if (cfg->ias > 32 || cfg->oas > 40) in arm_32_lpae_alloc_pgtable_s1()
1036 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_32_lpae_alloc_pgtable_s1()
1037 return arm_64_lpae_alloc_pgtable_s1(cfg, cookie); in arm_32_lpae_alloc_pgtable_s1()
1041 arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) in arm_32_lpae_alloc_pgtable_s2() argument
1043 if (cfg->ias > 40 || cfg->oas > 40) in arm_32_lpae_alloc_pgtable_s2()
1046 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_32_lpae_alloc_pgtable_s2()
1047 return arm_64_lpae_alloc_pgtable_s2(cfg, cookie); in arm_32_lpae_alloc_pgtable_s2()
1051 arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) in arm_mali_lpae_alloc_pgtable() argument
1056 if (cfg->quirks) in arm_mali_lpae_alloc_pgtable()
1059 if (cfg->ias > 48 || cfg->oas > 40) in arm_mali_lpae_alloc_pgtable()
1062 cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); in arm_mali_lpae_alloc_pgtable()
1064 data = arm_lpae_alloc_pgtable(cfg); in arm_mali_lpae_alloc_pgtable()
1080 cfg->arm_mali_lpae_cfg.memattr = in arm_mali_lpae_alloc_pgtable()
1089 cfg); in arm_mali_lpae_alloc_pgtable()
1096 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) | in arm_mali_lpae_alloc_pgtable()
1099 if (cfg->coherent_walk) in arm_mali_lpae_alloc_pgtable()
1100 cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER; in arm_mali_lpae_alloc_pgtable()
1110 apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) in apple_dart_alloc_pgtable() argument
1115 if (cfg->oas > 36) in apple_dart_alloc_pgtable()
1118 data = arm_lpae_alloc_pgtable(cfg); in apple_dart_alloc_pgtable()
1136 cfg->apple_dart_cfg.n_ttbrs = 1 << data->pgd_bits; in apple_dart_alloc_pgtable()
1140 cfg); in apple_dart_alloc_pgtable()
1144 for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) in apple_dart_alloc_pgtable()
1145 cfg->apple_dart_cfg.ttbr[i] = in apple_dart_alloc_pgtable()
1217 struct io_pgtable_cfg *cfg = &data->iop.cfg; in arm_lpae_dump_ops() local
1220 cfg->pgsize_bitmap, cfg->ias); in arm_lpae_dump_ops()
1233 static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) in arm_lpae_run_tests() argument
1248 cfg_cookie = cfg; in arm_lpae_run_tests()
1249 ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg); in arm_lpae_run_tests()
1272 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { in arm_lpae_run_tests()
1293 size = 1UL << __ffs(cfg->pgsize_bitmap); in arm_lpae_run_tests()
1306 for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) { in arm_lpae_run_tests()
1345 struct io_pgtable_cfg cfg = { in arm_lpae_do_selftests() local
1353 cfg.pgsize_bitmap = pgsize[i]; in arm_lpae_do_selftests()
1354 cfg.ias = ias[j]; in arm_lpae_do_selftests()
1357 if (arm_lpae_run_tests(&cfg)) in arm_lpae_do_selftests()