1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include <linux/device.h>
8 #include <linux/sched/task.h>
9 #include <linux/io-64-nonatomic-lo-hi.h>
10 #include <linux/cdev.h>
11 #include <linux/fs.h>
12 #include <linux/poll.h>
13 #include <linux/iommu.h>
14 #include <uapi/linux/idxd.h>
15 #include "registers.h"
16 #include "idxd.h"
17
18 struct idxd_cdev_context {
19 const char *name;
20 dev_t devt;
21 struct ida minor_ida;
22 };
23
24 /*
25 * ictx is an array based off of accelerator types. enum idxd_type
26 * is used as index
27 */
28 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
29 { .name = "dsa" },
30 { .name = "iax" }
31 };
32
33 struct idxd_user_context {
34 struct idxd_wq *wq;
35 struct task_struct *task;
36 unsigned int pasid;
37 unsigned int flags;
38 struct iommu_sva *sva;
39 };
40
idxd_cdev_dev_release(struct device * dev)41 static void idxd_cdev_dev_release(struct device *dev)
42 {
43 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
44 struct idxd_cdev_context *cdev_ctx;
45 struct idxd_wq *wq = idxd_cdev->wq;
46
47 cdev_ctx = &ictx[wq->idxd->data->type];
48 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
49 kfree(idxd_cdev);
50 }
51
52 static struct device_type idxd_cdev_device_type = {
53 .name = "idxd_cdev",
54 .release = idxd_cdev_dev_release,
55 };
56
inode_idxd_cdev(struct inode * inode)57 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
58 {
59 struct cdev *cdev = inode->i_cdev;
60
61 return container_of(cdev, struct idxd_cdev, cdev);
62 }
63
inode_wq(struct inode * inode)64 static inline struct idxd_wq *inode_wq(struct inode *inode)
65 {
66 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
67
68 return idxd_cdev->wq;
69 }
70
idxd_cdev_open(struct inode * inode,struct file * filp)71 static int idxd_cdev_open(struct inode *inode, struct file *filp)
72 {
73 struct idxd_user_context *ctx;
74 struct idxd_device *idxd;
75 struct idxd_wq *wq;
76 struct device *dev;
77 int rc = 0;
78 struct iommu_sva *sva;
79 unsigned int pasid;
80
81 wq = inode_wq(inode);
82 idxd = wq->idxd;
83 dev = &idxd->pdev->dev;
84
85 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
86
87 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
88 if (!ctx)
89 return -ENOMEM;
90
91 mutex_lock(&wq->wq_lock);
92
93 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
94 rc = -EBUSY;
95 goto failed;
96 }
97
98 ctx->wq = wq;
99 filp->private_data = ctx;
100
101 if (device_user_pasid_enabled(idxd)) {
102 sva = iommu_sva_bind_device(dev, current->mm);
103 if (IS_ERR(sva)) {
104 rc = PTR_ERR(sva);
105 dev_err(dev, "pasid allocation failed: %d\n", rc);
106 goto failed;
107 }
108
109 pasid = iommu_sva_get_pasid(sva);
110 if (pasid == IOMMU_PASID_INVALID) {
111 iommu_sva_unbind_device(sva);
112 rc = -EINVAL;
113 goto failed;
114 }
115
116 ctx->sva = sva;
117 ctx->pasid = pasid;
118
119 if (wq_dedicated(wq)) {
120 rc = idxd_wq_set_pasid(wq, pasid);
121 if (rc < 0) {
122 iommu_sva_unbind_device(sva);
123 dev_err(dev, "wq set pasid failed: %d\n", rc);
124 goto failed;
125 }
126 }
127 }
128
129 idxd_wq_get(wq);
130 mutex_unlock(&wq->wq_lock);
131 return 0;
132
133 failed:
134 mutex_unlock(&wq->wq_lock);
135 kfree(ctx);
136 return rc;
137 }
138
idxd_cdev_release(struct inode * node,struct file * filep)139 static int idxd_cdev_release(struct inode *node, struct file *filep)
140 {
141 struct idxd_user_context *ctx = filep->private_data;
142 struct idxd_wq *wq = ctx->wq;
143 struct idxd_device *idxd = wq->idxd;
144 struct device *dev = &idxd->pdev->dev;
145 int rc;
146
147 dev_dbg(dev, "%s called\n", __func__);
148 filep->private_data = NULL;
149
150 /* Wait for in-flight operations to complete. */
151 if (wq_shared(wq)) {
152 idxd_device_drain_pasid(idxd, ctx->pasid);
153 } else {
154 if (device_user_pasid_enabled(idxd)) {
155 /* The wq disable in the disable pasid function will drain the wq */
156 rc = idxd_wq_disable_pasid(wq);
157 if (rc < 0)
158 dev_err(dev, "wq disable pasid failed.\n");
159 } else {
160 idxd_wq_drain(wq);
161 }
162 }
163
164 if (ctx->sva)
165 iommu_sva_unbind_device(ctx->sva);
166 kfree(ctx);
167 mutex_lock(&wq->wq_lock);
168 idxd_wq_put(wq);
169 mutex_unlock(&wq->wq_lock);
170 return 0;
171 }
172
check_vma(struct idxd_wq * wq,struct vm_area_struct * vma,const char * func)173 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
174 const char *func)
175 {
176 struct device *dev = &wq->idxd->pdev->dev;
177
178 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
179 dev_info_ratelimited(dev,
180 "%s: %s: mapping too large: %lu\n",
181 current->comm, func,
182 vma->vm_end - vma->vm_start);
183 return -EINVAL;
184 }
185
186 return 0;
187 }
188
idxd_cdev_mmap(struct file * filp,struct vm_area_struct * vma)189 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
190 {
191 struct idxd_user_context *ctx = filp->private_data;
192 struct idxd_wq *wq = ctx->wq;
193 struct idxd_device *idxd = wq->idxd;
194 struct pci_dev *pdev = idxd->pdev;
195 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
196 unsigned long pfn;
197 int rc;
198
199 dev_dbg(&pdev->dev, "%s called\n", __func__);
200 rc = check_vma(wq, vma, __func__);
201 if (rc < 0)
202 return rc;
203
204 vm_flags_set(vma, VM_DONTCOPY);
205 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
206 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
207 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
208 vma->vm_private_data = ctx;
209
210 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
211 vma->vm_page_prot);
212 }
213
idxd_cdev_poll(struct file * filp,struct poll_table_struct * wait)214 static __poll_t idxd_cdev_poll(struct file *filp,
215 struct poll_table_struct *wait)
216 {
217 struct idxd_user_context *ctx = filp->private_data;
218 struct idxd_wq *wq = ctx->wq;
219 struct idxd_device *idxd = wq->idxd;
220 __poll_t out = 0;
221
222 poll_wait(filp, &wq->err_queue, wait);
223 spin_lock(&idxd->dev_lock);
224 if (idxd->sw_err.valid)
225 out = EPOLLIN | EPOLLRDNORM;
226 spin_unlock(&idxd->dev_lock);
227
228 return out;
229 }
230
231 static const struct file_operations idxd_cdev_fops = {
232 .owner = THIS_MODULE,
233 .open = idxd_cdev_open,
234 .release = idxd_cdev_release,
235 .mmap = idxd_cdev_mmap,
236 .poll = idxd_cdev_poll,
237 };
238
idxd_cdev_get_major(struct idxd_device * idxd)239 int idxd_cdev_get_major(struct idxd_device *idxd)
240 {
241 return MAJOR(ictx[idxd->data->type].devt);
242 }
243
idxd_wq_add_cdev(struct idxd_wq * wq)244 int idxd_wq_add_cdev(struct idxd_wq *wq)
245 {
246 struct idxd_device *idxd = wq->idxd;
247 struct idxd_cdev *idxd_cdev;
248 struct cdev *cdev;
249 struct device *dev;
250 struct idxd_cdev_context *cdev_ctx;
251 int rc, minor;
252
253 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
254 if (!idxd_cdev)
255 return -ENOMEM;
256
257 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
258 idxd_cdev->wq = wq;
259 cdev = &idxd_cdev->cdev;
260 dev = cdev_dev(idxd_cdev);
261 cdev_ctx = &ictx[wq->idxd->data->type];
262 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
263 if (minor < 0) {
264 kfree(idxd_cdev);
265 return minor;
266 }
267 idxd_cdev->minor = minor;
268
269 device_initialize(dev);
270 dev->parent = wq_confdev(wq);
271 dev->bus = &dsa_bus_type;
272 dev->type = &idxd_cdev_device_type;
273 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
274
275 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
276 if (rc < 0)
277 goto err;
278
279 wq->idxd_cdev = idxd_cdev;
280 cdev_init(cdev, &idxd_cdev_fops);
281 rc = cdev_device_add(cdev, dev);
282 if (rc) {
283 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
284 goto err;
285 }
286
287 return 0;
288
289 err:
290 put_device(dev);
291 wq->idxd_cdev = NULL;
292 return rc;
293 }
294
idxd_wq_del_cdev(struct idxd_wq * wq)295 void idxd_wq_del_cdev(struct idxd_wq *wq)
296 {
297 struct idxd_cdev *idxd_cdev;
298
299 idxd_cdev = wq->idxd_cdev;
300 wq->idxd_cdev = NULL;
301 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
302 put_device(cdev_dev(idxd_cdev));
303 }
304
idxd_user_drv_probe(struct idxd_dev * idxd_dev)305 static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
306 {
307 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
308 struct idxd_device *idxd = wq->idxd;
309 int rc;
310
311 if (idxd->state != IDXD_DEV_ENABLED)
312 return -ENXIO;
313
314 /*
315 * User type WQ is enabled only when SVA is enabled for two reasons:
316 * - If no IOMMU or IOMMU Passthrough without SVA, userspace
317 * can directly access physical address through the WQ.
318 * - The IDXD cdev driver does not provide any ways to pin
319 * user pages and translate the address from user VA to IOVA or
320 * PA without IOMMU SVA. Therefore the application has no way
321 * to instruct the device to perform DMA function. This makes
322 * the cdev not usable for normal application usage.
323 */
324 if (!device_user_pasid_enabled(idxd)) {
325 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU;
326 dev_dbg(&idxd->pdev->dev,
327 "User type WQ cannot be enabled without SVA.\n");
328
329 return -EOPNOTSUPP;
330 }
331
332 mutex_lock(&wq->wq_lock);
333 wq->type = IDXD_WQT_USER;
334 rc = drv_enable_wq(wq);
335 if (rc < 0)
336 goto err;
337
338 rc = idxd_wq_add_cdev(wq);
339 if (rc < 0) {
340 idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
341 goto err_cdev;
342 }
343
344 idxd->cmd_status = 0;
345 mutex_unlock(&wq->wq_lock);
346 return 0;
347
348 err_cdev:
349 drv_disable_wq(wq);
350 err:
351 wq->type = IDXD_WQT_NONE;
352 mutex_unlock(&wq->wq_lock);
353 return rc;
354 }
355
idxd_user_drv_remove(struct idxd_dev * idxd_dev)356 static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
357 {
358 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
359
360 mutex_lock(&wq->wq_lock);
361 idxd_wq_del_cdev(wq);
362 drv_disable_wq(wq);
363 wq->type = IDXD_WQT_NONE;
364 mutex_unlock(&wq->wq_lock);
365 }
366
367 static enum idxd_dev_type dev_types[] = {
368 IDXD_DEV_WQ,
369 IDXD_DEV_NONE,
370 };
371
372 struct idxd_device_driver idxd_user_drv = {
373 .probe = idxd_user_drv_probe,
374 .remove = idxd_user_drv_remove,
375 .name = "user",
376 .type = dev_types,
377 };
378 EXPORT_SYMBOL_GPL(idxd_user_drv);
379
idxd_cdev_register(void)380 int idxd_cdev_register(void)
381 {
382 int rc, i;
383
384 for (i = 0; i < IDXD_TYPE_MAX; i++) {
385 ida_init(&ictx[i].minor_ida);
386 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
387 ictx[i].name);
388 if (rc)
389 goto err_free_chrdev_region;
390 }
391
392 return 0;
393
394 err_free_chrdev_region:
395 for (i--; i >= 0; i--)
396 unregister_chrdev_region(ictx[i].devt, MINORMASK);
397
398 return rc;
399 }
400
idxd_cdev_remove(void)401 void idxd_cdev_remove(void)
402 {
403 int i;
404
405 for (i = 0; i < IDXD_TYPE_MAX; i++) {
406 unregister_chrdev_region(ictx[i].devt, MINORMASK);
407 ida_destroy(&ictx[i].minor_ida);
408 }
409 }
410