| /drivers/iommu/intel/ |
| A D | pasid.c | 279 clflush_cache_range(pte, sizeof(*pte)); in intel_pasid_tear_down_entry()
302 clflush_cache_range(pte, sizeof(*pte)); in pasid_flush_caches()
326 clflush_cache_range(pte, sizeof(*pte)); in intel_pasid_flush_present()
396 if (!pte) { in intel_pasid_setup_first_level()
438 if (!pte) { in intel_pasid_replace_first_level()
507 if (!pte) { in intel_pasid_setup_second_level()
556 if (!pte) { in intel_pasid_replace_second_level()
590 if (!pte) { in intel_pasid_setup_dirty_tracking()
621 clflush_cache_range(pte, sizeof(*pte)); in intel_pasid_setup_dirty_tracking()
671 if (!pte) { in intel_pasid_setup_pass_through()
[all …]
|
| A D | iommu.c | 754 domain_flush_cache(domain, pte, sizeof(*pte)); in pfn_to_dma_pte()
824 pte++; in dma_pte_clear_range()
839 pte = &pte[pfn_level_offset(pfn, level)]; in dma_pte_free_level()
845 if (!dma_pte_present(pte) || dma_pte_superpage(pte)) in dma_pte_free_level()
864 domain_flush_cache(domain, pte, sizeof(*pte)); in dma_pte_free_level()
914 pte++; in dma_pte_list_pagetables()
926 pte = &pte[pfn_level_offset(pfn, level)]; in dma_pte_clear_level()
1592 pte++; in switch_to_super_page()
1694 pte++; in __domain_mapping()
3679 if (pte && dma_pte_present(pte)) in intel_iommu_iova_to_phys()
[all …]
|
| A D | pasid.h | 70 static inline bool pasid_pte_is_present(struct pasid_entry *pte) in pasid_pte_is_present() argument
72 return READ_ONCE(pte->val[0]) & PASID_PTE_PRESENT; in pasid_pte_is_present()
76 static inline bool pasid_pte_is_fault_disabled(struct pasid_entry *pte) in pasid_pte_is_fault_disabled() argument
78 return READ_ONCE(pte->val[0]) & PASID_PTE_FPD; in pasid_pte_is_fault_disabled()
82 static inline u16 pasid_pte_get_pgtt(struct pasid_entry *pte) in pasid_pte_get_pgtt() argument
84 return (u16)((READ_ONCE(pte->val[0]) >> 6) & 0x7); in pasid_pte_get_pgtt()
|
| A D | iommu.h | 872 static inline void dma_clear_pte(struct dma_pte *pte) in dma_clear_pte() argument
874 pte->val = 0; in dma_clear_pte()
877 static inline u64 dma_pte_addr(struct dma_pte *pte) in dma_pte_addr() argument
880 return pte->val & VTD_PAGE_MASK; in dma_pte_addr()
887 static inline bool dma_pte_present(struct dma_pte *pte) in dma_pte_present() argument
889 return (pte->val & 3) != 0; in dma_pte_present()
896 return (pte->val & DMA_SL_PTE_DIRTY) != 0; in dma_sl_pte_test_and_clear_dirty()
899 (unsigned long *)&pte->val); in dma_sl_pte_test_and_clear_dirty()
904 return (pte->val & DMA_PTE_LARGE_PAGE); in dma_pte_superpage()
909 return IS_ALIGNED((unsigned long)pte, VTD_PAGE_SIZE); in first_pte_in_page()
[all …]
|
| /drivers/iommu/amd/ |
| A D | io_pgtable_v2.c | 66 u64 pte; in set_pte_attr() local
79 return pte; in set_pte_attr()
149 cmpxchg64(pte, *pte, 0ULL); in v2_alloc_pte()
171 pte = &pte[PM_LEVEL_INDEX(level, iova)]; in v2_alloc_pte()
180 cmpxchg64(pte, *pte, 0ULL); in v2_alloc_pte()
197 u64 *pte; in fetch_pte() local
211 pte = get_pgtable_pte(*pte); in fetch_pte()
212 pte = &pte[PM_LEVEL_INDEX(level - 1, iova)]; in fetch_pte()
238 u64 *pte; in iommu_v2_map_pages() local
294 u64 *pte; in iommu_v2_unmap_pages() local
[all …]
|
| A D | io_pgtable.c | 115 u64 *pte; in increase_address_space() local
118 if (!pte) in increase_address_space()
137 pte = NULL; in increase_address_space()
237 pte = &pte[PM_LEVEL_INDEX(level, address)]; in alloc_pte()
252 u64 *pte; in fetch_pte() local
281 pte = IOMMU_PTE_PAGE(*pte); in fetch_pte()
282 pte = &pte[PM_LEVEL_INDEX(level, address)]; in fetch_pte()
291 pte = first_pte_l7(pte, page_size, NULL); in fetch_pte()
349 free_clear_pte(&pte[i], pte[i], &freelist); in iommu_v1_map_pages()
406 u64 *pte; in iommu_v1_unmap_pages() local
[all …]
|
| A D | amd_iommu_types.h | 348 #define PM_PTE_LEVEL(pte) (((pte) >> 9) & 0x7ULL) argument
387 #define PTE_PAGE_SIZE(pte) \ argument
388 (1ULL << (1 + ffz(((pte) | 0xfffULL))))
440 #define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_PR) argument
441 #define IOMMU_PTE_DIRTY(pte) ((pte) & IOMMU_PTE_HD) argument
442 #define IOMMU_PTE_PAGE(pte) (iommu_phys_to_virt((pte) & IOMMU_PAGE_MASK)) argument
443 #define IOMMU_PTE_MODE(pte) (((pte) >> 9) & 0x07) argument
|
| /drivers/iommu/ |
| A D | io-pgtable-arm-v7s.c | 83 #define ARM_V7S_PTE_IS_VALID(pte) (((pte) & 0x3) != 0) argument
192 return pte; in to_mtk_iopte()
203 return pte; in paddr_to_iopte()
329 ptep[i] = pte; in __arm_v7s_set_pte()
362 return pte; in arm_v7s_prot_to_pte()
378 return pte; in arm_v7s_pte_to_cont()
399 arm_v7s_iopte pte; in arm_v7s_init_pte() local
423 pte = arm_v7s_pte_to_cont(pte, lvl); in arm_v7s_init_pte()
482 if (!pte) { in __arm_v7s_map()
488 if (pte) in __arm_v7s_map()
[all …]
|
| A D | io-pgtable-dart.c | 66 #define iopte_deref(pte, d) __va(iopte_to_paddr(pte, d)) argument
83 dart_iopte pte; in paddr_to_iopte() local
92 return pte; in paddr_to_iopte()
192 if (!pte) in dart_get_l2()
202 dart_iopte pte = 0; in dart_prot_to_pte() local
220 return pte; in dart_prot_to_pte()
250 if (!pte) { in dart_map_pages()
256 if (pte) in dart_map_pages()
310 if (WARN_ON(!pte)) in dart_unmap_pages()
343 if (pte) { in dart_iova_to_phys()
[all …]
|
| A D | io-pgtable-arm.c | 148 #define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) argument
193 return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; in paddr_to_iopte()
456 if (!pte) { in __arm_lpae_map()
462 if (pte) in __arm_lpae_map()
468 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) { in __arm_lpae_map()
470 } else if (pte) { in __arm_lpae_map()
548 return pte; in arm_lpae_prot_to_pte()
607 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt)) in __arm_lpae_free_pgtable()
640 if (!pte) { in __arm_lpae_unmap()
653 if (!pte) { in __arm_lpae_unmap()
[all …]
|
| A D | rockchip-iommu.c | 266 return pte & RK_PTE_PAGE_VALID; in rk_pte_is_page_valid()
302 return pte & ~RK_PTE_PAGE_VALID; in rk_mk_pte_invalid()
545 u32 pte = 0; in log_iova() local
565 pte = *pte_addr; in log_iova()
567 if (!rk_pte_is_page_valid(pte)) in log_iova()
657 u32 dte, pte; in rk_iommu_iova_to_phys() local
669 if (!rk_pte_is_page_valid(pte)) in rk_iommu_iova_to_phys()
765 u32 pte = pte_addr[pte_count]; in rk_iommu_unmap_iova() local
766 if (!rk_pte_is_page_valid(pte)) in rk_iommu_unmap_iova()
788 u32 pte = pte_addr[pte_count]; in rk_iommu_map_iova() local
[all …]
|
| A D | tegra-smmu.c | 672 *pte = val; in tegra_smmu_set_pte()
729 u32 *pte; in __tegra_smmu_map() local
735 pte = as_get_pte(as, iova, &pte_dma, pt); in __tegra_smmu_map()
736 if (!pte) in __tegra_smmu_map()
740 if (*pte == 0) in __tegra_smmu_map()
751 tegra_smmu_set_pte(as, iova, pte, pte_dma, in __tegra_smmu_map()
763 u32 *pte; in __tegra_smmu_unmap() local
766 if (!pte || !*pte) in __tegra_smmu_unmap()
812 u32 *pte; in tegra_smmu_iova_to_phys() local
815 if (!pte || !*pte) in tegra_smmu_iova_to_phys()
[all …]
|
| /drivers/staging/media/atomisp/pci/mmu/ |
| A D | isp_mmu.c | 60 *(pt_virt + idx) = pte; in atomisp_set_pte()
69 unsigned int pte) in isp_pte_to_pgaddr() argument
163 pte); in mmu_unmap_l2_pte_error()
175 pte); in mmu_unmap_l1_pte_error()
194 unsigned int pte; in mmu_l2_map() local
208 if (ISP_PTE_VALID(mmu, pte)) { in mmu_l2_map()
210 l2_pt, idx, ptr, pte, phys); in mmu_l2_map()
358 unsigned int pte; in mmu_l2_unmap() local
371 if (!ISP_PTE_VALID(mmu, pte)) in mmu_l2_unmap()
531 unsigned int pte; in isp_mmu_exit() local
[all …]
|
| A D | sh_mmu_mrfld.c | 27 unsigned int pte) in sh_pte_to_phys() argument
31 return (phys_addr_t)((pte & ~mask) << ISP_PAGE_OFFSET); in sh_pte_to_phys()
37 unsigned int pte = sh_phys_to_pte(mmu, phys); in sh_get_pd_base() local
39 return HOST_ADDRESS(pte); in sh_get_pd_base()
|
| /drivers/gpu/drm/i915/gt/ |
| A D | intel_ggtt.c | 277 return pte; in mtl_ggtt_pte_encode()
289 return pte; in gen8_ggtt_pte_encode()
341 const gen8_pte_t pte) in gen8_ggtt_bind_ptes() argument
443 writeq(pte, addr); in gen8_set_pte()
481 gen8_pte_t pte; in gen8_ggtt_insert_page_bind() local
1365 return pte; in snb_pte_encode()
1388 return pte; in ivb_pte_encode()
1403 return pte; in byt_pte_encode()
1415 return pte; in hsw_pte_encode()
1435 return pte; in iris_pte_encode()
[all …]
|
| A D | gen8_ppgtt.c | 39 pte &= ~GEN8_PAGE_RW; in gen8_pte_encode()
48 pte |= PPAT_UNCACHED; in gen8_pte_encode()
51 pte |= PPAT_DISPLAY_ELLC; in gen8_pte_encode()
54 pte |= PPAT_CACHED; in gen8_pte_encode()
58 return pte; in gen8_pte_encode()
68 pte &= ~GEN8_PAGE_RW; in gen12_pte_encode()
71 pte |= GEN12_PPGTT_PTE_LM; in gen12_pte_encode()
85 return pte; in gen12_pte_encode()
286 GEM_BUG_ON(pte % 16); in __gen8_ppgtt_clear()
288 pte /= 16; in __gen8_ppgtt_clear()
[all …]
|
| /drivers/iommu/riscv/ |
| A D | iommu.c | 1083 #define _io_pte_present(pte) ((pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) argument
1084 #define _io_pte_leaf(pte) ((pte) & _PAGE_LEAF) argument
1085 #define _io_pte_none(pte) ((pte) == 0) argument
1089 unsigned long pte, in riscv_iommu_pte_free() argument
1095 if (!_io_pte_present(pte) || _io_pte_leaf(pte)) in riscv_iommu_pte_free()
1103 if (!_io_pte_none(pte) && cmpxchg_relaxed(ptr + i, pte, 0) == pte) in riscv_iommu_pte_free()
1118 unsigned long pte, old; in riscv_iommu_pte_alloc() local
1140 if (_io_pte_present(pte) && _io_pte_leaf(pte)) in riscv_iommu_pte_alloc()
1151 old = pte; in riscv_iommu_pte_alloc()
1168 unsigned long pte; in riscv_iommu_pte_fetch() local
[all …]
|
| /drivers/gpu/drm/gma500/ |
| A D | gtt.c | 83 u32 pte; in psb_gtt_insert_pages() local
93 pte = psb_gtt_mask_pte(page_to_pfn(pages[i]), PSB_MMU_CACHED_MEMORY); in psb_gtt_insert_pages()
94 iowrite32(pte, gtt_slot); in psb_gtt_insert_pages()
108 u32 pte; in psb_gtt_remove_pages() local
114 pte = psb_gtt_mask_pte(page_to_pfn(pdev->scratch_page), PSB_MMU_CACHED_MEMORY); in psb_gtt_remove_pages()
120 iowrite32(pte, gtt_slot); in psb_gtt_remove_pages()
174 uint32_t pte; in psb_gtt_clear() local
177 pte = psb_gtt_mask_pte(pfn_base, PSB_MMU_CACHED_MEMORY); in psb_gtt_clear()
180 iowrite32(pte, pdev->gtt_map + i); in psb_gtt_clear()
|
| /drivers/gpu/drm/xe/ |
| A D | xe_ggtt.c | 71 u64 pte = XE_PAGE_PRESENT; in xelp_ggtt_pte_flags() local
74 pte |= XE_GGTT_PTE_DM; in xelp_ggtt_pte_flags()
76 return pte; in xelp_ggtt_pte_flags()
82 u64 pte; in xelpg_ggtt_pte_flags() local
89 pte |= XELPG_GGTT_PTE_PAT0; in xelpg_ggtt_pte_flags()
92 pte |= XELPG_GGTT_PTE_PAT1; in xelpg_ggtt_pte_flags()
94 return pte; in xelpg_ggtt_pte_flags()
136 xe_ggtt_set_pte(ggtt, addr, pte); in xe_ggtt_set_pte_and_flush()
689 u64 start, pte, end; in xe_ggtt_map_bo() local
705 pte | xe_res_dma(&cur)); in xe_ggtt_map_bo()
[all …]
|
| /drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ |
| A D | vmm.c | 78 u32 pte[NVKM_VMM_LEVELS_MAX]; member
146 u32 pdei = it->pte[it->lvl + 1]; in nvkm_vmm_unref_pdes()
211 pgt->pte[lpti] -= pten; in nvkm_vmm_unref_sptes()
310 pgt->pte[lpti] += pten; in nvkm_vmm_ref_sptes()
401 memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes); in nvkm_vmm_sparse_unref_ptes()
535 const u32 ptei = it.pte[0]; in nvkm_vmm_iter()
575 it.pte[it.lvl] += ptes; in nvkm_vmm_iter()
579 it.pte[it.lvl++] = 0; in nvkm_vmm_iter()
580 it.pte[it.lvl]++; in nvkm_vmm_iter()
592 addr = it.pte[it.max--]; in nvkm_vmm_iter()
[all …]
|
| /drivers/gpu/drm/i915/ |
| A D | i915_mm.c | 51 static int remap_sg(pte_t *pte, unsigned long addr, void *data) in remap_sg() argument
59 set_pte_at(r->mm, addr, pte, in remap_sg()
73 static int remap_pfn(pte_t *pte, unsigned long addr, void *data) in remap_pfn() argument
78 set_pte_at(r->mm, addr, pte, pte_mkspecial(pfn_pte(r->pfn, r->prot))); in remap_pfn()
|
| /drivers/staging/media/ipu3/ |
| A D | ipu3-mmu.c | 29 #define IPU3_PTE2ADDR(pte) ((phys_addr_t)(pte) << IPU3_PAGE_SHIFT) argument
124 int pte; in imgu_mmu_alloc_page_table() local
130 for (pte = 0; pte < IPU3_PT_PTES; pte++) in imgu_mmu_alloc_page_table()
131 pt[pte] = pteval; in imgu_mmu_alloc_page_table()
|
| /drivers/staging/media/atomisp/include/mmu/ |
| A D | isp_mmu.h | 95 unsigned int pte);
112 #define ISP_PTE_VALID(mmu, pte) \ argument
113 ((pte) & ISP_PTE_VALID_MASK(mmu))
|
| /drivers/gpu/drm/xe/display/ |
| A D | xe_fb_pin.c | 27 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); in write_dpt_rotated() local
39 iosys_map_wr(map, *dpt_ofs, u64, pte | addr); in write_dpt_rotated()
60 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); in write_dpt_remapped() local
67 iosys_map_wr(map, *dpt_ofs, u64, pte | addr); in write_dpt_remapped()
131 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); in __xe_pin_fb_vma_dpt() local
137 iosys_map_wr(&dpt->vmap, x * 8, u64, pte | addr); in __xe_pin_fb_vma_dpt()
178 u64 pte = ggtt->pt_ops->pte_encode_flags(bo, xe->pat.idx[XE_CACHE_NONE]); in write_ggtt_rotated() local
186 ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte | addr); in write_ggtt_rotated()
|
| /drivers/gpu/drm/imagination/ |
| A D | pvr_vm_mips.c | 183 u32 pte; in pvr_vm_mips_map() local
192 pte = ((dma_addr >> ROGUE_MIPSFW_LOG2_PAGE_SIZE_4K) in pvr_vm_mips_map()
194 pte |= pte_flags; in pvr_vm_mips_map()
196 WRITE_ONCE(mips_data->pt[pfn], pte); in pvr_vm_mips_map()
|