1 /* SPDX-License-Identifier: MIT
2 *
3 * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
4 */
5 #include "vmm.h"
6
7 #include <subdev/fb.h>
8
9 #include <nvhw/drf.h>
10 #include <nvhw/ref/gh100/dev_mmu.h>
11
12 static inline void
gh100_vmm_pgt_pte(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map,u64 addr)13 gh100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes,
14 struct nvkm_vmm_map *map, u64 addr)
15 {
16 u64 data = addr | map->type;
17
18 while (ptes--) {
19 VMM_WO064(pt, vmm, ptei++ * NV_MMU_VER3_PTE__SIZE, data);
20 data += map->next;
21 }
22 }
23
24 static void
gh100_vmm_pgt_sgl(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)25 gh100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes,
26 struct nvkm_vmm_map *map)
27 {
28 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gh100_vmm_pgt_pte);
29 }
30
31 static void
gh100_vmm_pgt_dma(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)32 gh100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes,
33 struct nvkm_vmm_map *map)
34 {
35 if (map->page->shift == PAGE_SHIFT) {
36 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
37
38 nvkm_kmap(pt->memory);
39 while (ptes--) {
40 const u64 data = *map->dma++ | map->type;
41
42 VMM_WO064(pt, vmm, ptei++ * NV_MMU_VER3_PTE__SIZE, data);
43 }
44 nvkm_done(pt->memory);
45 return;
46 }
47
48 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gh100_vmm_pgt_pte);
49 }
50
51 static void
gh100_vmm_pgt_mem(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)52 gh100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes,
53 struct nvkm_vmm_map *map)
54 {
55 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gh100_vmm_pgt_pte);
56 }
57
58 static void
gh100_vmm_pgt_sparse(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes)59 gh100_vmm_pgt_sparse(struct nvkm_vmm *vmm,
60 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
61 {
62 const u64 data = NVDEF(NV_MMU, VER3_PTE, PCF, SPARSE);
63
64 VMM_FO064(pt, vmm, ptei * NV_MMU_VER3_PTE__SIZE, data, ptes);
65 }
66
67 static const struct nvkm_vmm_desc_func
68 gh100_vmm_desc_spt = {
69 .unmap = gf100_vmm_pgt_unmap,
70 .sparse = gh100_vmm_pgt_sparse,
71 .mem = gh100_vmm_pgt_mem,
72 .dma = gh100_vmm_pgt_dma,
73 .sgl = gh100_vmm_pgt_sgl,
74 };
75
76 static void
gh100_vmm_lpt_invalid(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes)77 gh100_vmm_lpt_invalid(struct nvkm_vmm *vmm,
78 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
79 {
80 const u64 data = NVDEF(NV_MMU, VER3_PTE, PCF, NO_VALID_4KB_PAGE);
81
82 VMM_FO064(pt, vmm, ptei * NV_MMU_VER3_PTE__SIZE, data, ptes);
83 }
84
85 static const struct nvkm_vmm_desc_func
86 gh100_vmm_desc_lpt = {
87 .invalid = gh100_vmm_lpt_invalid,
88 .unmap = gf100_vmm_pgt_unmap,
89 .sparse = gh100_vmm_pgt_sparse,
90 .mem = gh100_vmm_pgt_mem,
91 };
92
93 static inline void
gh100_vmm_pd0_pte(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map,u64 addr)94 gh100_vmm_pd0_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
95 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
96 {
97 u64 data = addr | map->type;
98
99 while (ptes--) {
100 VMM_WO128(pt, vmm, ptei++ * NV_MMU_VER3_DUAL_PDE__SIZE, data, 0ULL);
101 data += map->next;
102 }
103 }
104
105 static void
gh100_vmm_pd0_mem(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)106 gh100_vmm_pd0_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
107 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
108 {
109 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gh100_vmm_pd0_pte);
110 }
111
112 static inline bool
gh100_vmm_pde(struct nvkm_mmu_pt * pt,u64 * data)113 gh100_vmm_pde(struct nvkm_mmu_pt *pt, u64 *data)
114 {
115 switch (nvkm_memory_target(pt->memory)) {
116 case NVKM_MEM_TARGET_VRAM:
117 *data |= NVDEF(NV_MMU, VER3_PDE, APERTURE, VIDEO_MEMORY);
118 *data |= NVDEF(NV_MMU, VER3_PDE, PCF, VALID_CACHED_ATS_NOT_ALLOWED);
119 break;
120 case NVKM_MEM_TARGET_HOST:
121 *data |= NVDEF(NV_MMU, VER3_PDE, APERTURE, SYSTEM_COHERENT_MEMORY);
122 *data |= NVDEF(NV_MMU, VER3_PDE, PCF, VALID_UNCACHED_ATS_ALLOWED);
123 break;
124 case NVKM_MEM_TARGET_NCOH:
125 *data |= NVDEF(NV_MMU, VER3_PDE, APERTURE, SYSTEM_NON_COHERENT_MEMORY);
126 *data |= NVDEF(NV_MMU, VER3_PDE, PCF, VALID_CACHED_ATS_ALLOWED);
127 break;
128 default:
129 WARN_ON(1);
130 return false;
131 }
132
133 *data |= pt->addr;
134 return true;
135 }
136
137 static void
gh100_vmm_pd0_pde(struct nvkm_vmm * vmm,struct nvkm_vmm_pt * pgd,u32 pdei)138 gh100_vmm_pd0_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
139 {
140 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
141 struct nvkm_mmu_pt *pd = pgd->pt[0];
142 u64 data[2] = {};
143
144 if (pgt->pt[0] && !gh100_vmm_pde(pgt->pt[0], &data[0]))
145 return;
146 if (pgt->pt[1] && !gh100_vmm_pde(pgt->pt[1], &data[1]))
147 return;
148
149 nvkm_kmap(pd->memory);
150 VMM_WO128(pd, vmm, pdei * NV_MMU_VER3_DUAL_PDE__SIZE, data[0], data[1]);
151 nvkm_done(pd->memory);
152 }
153
154 static void
gh100_vmm_pd0_sparse(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 pdei,u32 pdes)155 gh100_vmm_pd0_sparse(struct nvkm_vmm *vmm,
156 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes)
157 {
158 const u64 data = NVDEF(NV_MMU, VER3_DUAL_PDE, PCF_BIG, SPARSE_ATS_ALLOWED);
159
160 VMM_FO128(pt, vmm, pdei * NV_MMU_VER3_DUAL_PDE__SIZE, data, 0ULL, pdes);
161 }
162
163 static void
gh100_vmm_pd0_unmap(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 pdei,u32 pdes)164 gh100_vmm_pd0_unmap(struct nvkm_vmm *vmm,
165 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes)
166 {
167 VMM_FO128(pt, vmm, pdei * NV_MMU_VER3_DUAL_PDE__SIZE, 0ULL, 0ULL, pdes);
168 }
169
170 static const struct nvkm_vmm_desc_func
171 gh100_vmm_desc_pd0 = {
172 .unmap = gh100_vmm_pd0_unmap,
173 .sparse = gh100_vmm_pd0_sparse,
174 .pde = gh100_vmm_pd0_pde,
175 .mem = gh100_vmm_pd0_mem,
176 };
177
178 static void
gh100_vmm_pd1_pde(struct nvkm_vmm * vmm,struct nvkm_vmm_pt * pgd,u32 pdei)179 gh100_vmm_pd1_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
180 {
181 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
182 struct nvkm_mmu_pt *pd = pgd->pt[0];
183 u64 data = 0;
184
185 if (!gh100_vmm_pde(pgt->pt[0], &data))
186 return;
187
188 nvkm_kmap(pd->memory);
189 VMM_WO064(pd, vmm, pdei * NV_MMU_VER3_PDE__SIZE, data);
190 nvkm_done(pd->memory);
191 }
192
193 static const struct nvkm_vmm_desc_func
194 gh100_vmm_desc_pd1 = {
195 .unmap = gf100_vmm_pgt_unmap,
196 .sparse = gh100_vmm_pgt_sparse,
197 .pde = gh100_vmm_pd1_pde,
198 };
199
200 static const struct nvkm_vmm_desc
201 gh100_vmm_desc_16[] = {
202 { LPT, 5, 8, 0x0100, &gh100_vmm_desc_lpt },
203 { PGD, 8, 16, 0x1000, &gh100_vmm_desc_pd0 },
204 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 },
205 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 },
206 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 },
207 { PGD, 1, 8, 0x1000, &gh100_vmm_desc_pd1 },
208 {}
209 };
210
211 static const struct nvkm_vmm_desc
212 gh100_vmm_desc_12[] = {
213 { SPT, 9, 8, 0x1000, &gh100_vmm_desc_spt },
214 { PGD, 8, 16, 0x1000, &gh100_vmm_desc_pd0 },
215 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 },
216 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 },
217 { PGD, 9, 8, 0x1000, &gh100_vmm_desc_pd1 },
218 { PGD, 1, 8, 0x1000, &gh100_vmm_desc_pd1 },
219 {}
220 };
221
222 static int
gh100_vmm_valid(struct nvkm_vmm * vmm,bool ro,bool priv,u8 kind,u8 comp,struct nvkm_vmm_map * map)223 gh100_vmm_valid(struct nvkm_vmm *vmm, bool ro, bool priv, u8 kind, u8 comp,
224 struct nvkm_vmm_map *map)
225 {
226 const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
227 const bool vol = target == NVKM_MEM_TARGET_HOST;
228 const struct nvkm_vmm_page *page = map->page;
229 u8 kind_inv, pcf;
230 int kindn, aper;
231 const u8 *kindm;
232
233 map->next = 1ULL << page->shift;
234 map->type = 0;
235
236 aper = vmm->func->aper(target);
237 if (WARN_ON(aper < 0))
238 return aper;
239
240 kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv);
241 if (kind >= kindn || kindm[kind] == kind_inv) {
242 VMM_DEBUG(vmm, "kind %02x", kind);
243 return -EINVAL;
244 }
245
246 if (priv) {
247 if (ro) {
248 if (vol)
249 pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_UNCACHED_ACD;
250 else
251 pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RO_ATOMIC_CACHED_ACD;
252 } else {
253 if (vol)
254 pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_UNCACHED_ACD;
255 else
256 pcf = NV_MMU_VER3_PTE_PCF_PRIVILEGE_RW_ATOMIC_CACHED_ACD;
257 }
258 } else {
259 if (ro) {
260 if (vol)
261 pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_UNCACHED_ACD;
262 else
263 pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RO_ATOMIC_CACHED_ACD;
264 } else {
265 if (vol)
266 pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_UNCACHED_ACD;
267 else
268 pcf = NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_CACHED_ACD;
269 }
270 }
271
272 map->type |= NVDEF(NV_MMU, VER3_PTE, VALID, TRUE);
273 map->type |= NVVAL(NV_MMU, VER3_PTE, APERTURE, aper);
274 map->type |= NVVAL(NV_MMU, VER3_PTE, PCF, pcf);
275 map->type |= NVVAL(NV_MMU, VER3_PTE, KIND, kind);
276 return 0;
277 }
278
279 static const struct nvkm_vmm_func
280 gh100_vmm = {
281 .join = gv100_vmm_join,
282 .part = gf100_vmm_part,
283 .aper = gf100_vmm_aper,
284 .valid = gp100_vmm_valid,
285 .valid2 = gh100_vmm_valid,
286 .flush = tu102_vmm_flush,
287 .page = {
288 { 56, &gh100_vmm_desc_16[5], NVKM_VMM_PAGE_Sxxx },
289 { 47, &gh100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx },
290 { 38, &gh100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx },
291 { 29, &gh100_vmm_desc_16[2], NVKM_VMM_PAGE_SVxC },
292 { 21, &gh100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC },
293 { 16, &gh100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC },
294 { 12, &gh100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx },
295 {}
296 }
297 };
298
299 int
gh100_vmm_new(struct nvkm_mmu * mmu,bool managed,u64 addr,u64 size,void * argv,u32 argc,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)300 gh100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
301 void *argv, u32 argc, struct lock_class_key *key,
302 const char *name, struct nvkm_vmm **pvmm)
303 {
304 return gp100_vmm_new_(&gh100_vmm, mmu, managed, addr, size,
305 argv, argc, key, name, pvmm);
306 }
307