1 // SPDX-License-Identifier: GPL-2.0+
2 /* Copyright (C) 2017-2018 Broadcom */
3 
4 /**
5  * DOC: Broadcom V3D MMU
6  *
7  * The V3D 3.x hardware (compared to VC4) now includes an MMU.  It has
8  * a single level of page tables for the V3D's 4GB address space to
9  * map to AXI bus addresses, thus it could need up to 4MB of
10  * physically contiguous memory to store the PTEs.
11  *
12  * Because the 4MB of contiguous memory for page tables is precious,
13  * and switching between them is expensive, we load all BOs into the
14  * same 4GB address space.
15  *
16  * To protect clients from each other, we should use the GMP to
17  * quickly mask out (at 128kb granularity) what pages are available to
18  * each client.  This is not yet implemented.
19  */
20 
21 #include "v3d_drv.h"
22 #include "v3d_regs.h"
23 
24 #define V3D_MMU_PAGE_SHIFT 12
25 
26 /* Note: All PTEs for the 1MB superpage must be filled with the
27  * superpage bit set.
28  */
29 #define V3D_PTE_SUPERPAGE BIT(31)
30 #define V3D_PTE_WRITEABLE BIT(29)
31 #define V3D_PTE_VALID BIT(28)
32 
v3d_mmu_flush_all(struct v3d_dev * v3d)33 static int v3d_mmu_flush_all(struct v3d_dev *v3d)
34 {
35 	int ret;
36 
37 	/* Make sure that another flush isn't already running when we
38 	 * start this one.
39 	 */
40 	ret = wait_for(!(V3D_READ(V3D_MMU_CTL) &
41 			 V3D_MMU_CTL_TLB_CLEARING), 100);
42 	if (ret)
43 		dev_err(v3d->drm.dev, "TLB clear wait idle pre-wait failed\n");
44 
45 	V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL) |
46 		  V3D_MMU_CTL_TLB_CLEAR);
47 
48 	V3D_WRITE(V3D_MMUC_CONTROL,
49 		  V3D_MMUC_CONTROL_FLUSH |
50 		  V3D_MMUC_CONTROL_ENABLE);
51 
52 	ret = wait_for(!(V3D_READ(V3D_MMU_CTL) &
53 			 V3D_MMU_CTL_TLB_CLEARING), 100);
54 	if (ret) {
55 		dev_err(v3d->drm.dev, "TLB clear wait idle failed\n");
56 		return ret;
57 	}
58 
59 	ret = wait_for(!(V3D_READ(V3D_MMUC_CONTROL) &
60 			 V3D_MMUC_CONTROL_FLUSHING), 100);
61 	if (ret)
62 		dev_err(v3d->drm.dev, "MMUC flush wait idle failed\n");
63 
64 	return ret;
65 }
66 
v3d_mmu_set_page_table(struct v3d_dev * v3d)67 int v3d_mmu_set_page_table(struct v3d_dev *v3d)
68 {
69 	V3D_WRITE(V3D_MMU_PT_PA_BASE, v3d->pt_paddr >> V3D_MMU_PAGE_SHIFT);
70 	V3D_WRITE(V3D_MMU_CTL,
71 		  V3D_MMU_CTL_ENABLE |
72 		  V3D_MMU_CTL_PT_INVALID_ENABLE |
73 		  V3D_MMU_CTL_PT_INVALID_ABORT |
74 		  V3D_MMU_CTL_PT_INVALID_INT |
75 		  V3D_MMU_CTL_WRITE_VIOLATION_ABORT |
76 		  V3D_MMU_CTL_WRITE_VIOLATION_INT |
77 		  V3D_MMU_CTL_CAP_EXCEEDED_ABORT |
78 		  V3D_MMU_CTL_CAP_EXCEEDED_INT);
79 	V3D_WRITE(V3D_MMU_ILLEGAL_ADDR,
80 		  (v3d->mmu_scratch_paddr >> V3D_MMU_PAGE_SHIFT) |
81 		  V3D_MMU_ILLEGAL_ADDR_ENABLE);
82 	V3D_WRITE(V3D_MMUC_CONTROL, V3D_MMUC_CONTROL_ENABLE);
83 
84 	return v3d_mmu_flush_all(v3d);
85 }
86 
v3d_mmu_insert_ptes(struct v3d_bo * bo)87 void v3d_mmu_insert_ptes(struct v3d_bo *bo)
88 {
89 	struct drm_gem_shmem_object *shmem_obj = &bo->base;
90 	struct v3d_dev *v3d = to_v3d_dev(shmem_obj->base.dev);
91 	u32 page = bo->node.start;
92 	u32 page_prot = V3D_PTE_WRITEABLE | V3D_PTE_VALID;
93 	struct sg_dma_page_iter dma_iter;
94 
95 	for_each_sgtable_dma_page(shmem_obj->sgt, &dma_iter, 0) {
96 		dma_addr_t dma_addr = sg_page_iter_dma_address(&dma_iter);
97 		u32 page_address = dma_addr >> V3D_MMU_PAGE_SHIFT;
98 		u32 pte = page_prot | page_address;
99 		u32 i;
100 
101 		BUG_ON(page_address + (PAGE_SIZE >> V3D_MMU_PAGE_SHIFT) >=
102 		       BIT(24));
103 		for (i = 0; i < PAGE_SIZE >> V3D_MMU_PAGE_SHIFT; i++)
104 			v3d->pt[page++] = pte + i;
105 	}
106 
107 	WARN_ON_ONCE(page - bo->node.start !=
108 		     shmem_obj->base.size >> V3D_MMU_PAGE_SHIFT);
109 
110 	if (v3d_mmu_flush_all(v3d))
111 		dev_err(v3d->drm.dev, "MMU flush timeout\n");
112 }
113 
v3d_mmu_remove_ptes(struct v3d_bo * bo)114 void v3d_mmu_remove_ptes(struct v3d_bo *bo)
115 {
116 	struct v3d_dev *v3d = to_v3d_dev(bo->base.base.dev);
117 	u32 npages = bo->base.base.size >> V3D_MMU_PAGE_SHIFT;
118 	u32 page;
119 
120 	for (page = bo->node.start; page < bo->node.start + npages; page++)
121 		v3d->pt[page] = 0;
122 
123 	if (v3d_mmu_flush_all(v3d))
124 		dev_err(v3d->drm.dev, "MMU flush timeout\n");
125 }
126