1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
3 */
4 #ifndef __IOMMUFD_PRIVATE_H
5 #define __IOMMUFD_PRIVATE_H
6
7 #include <linux/rwsem.h>
8 #include <linux/xarray.h>
9 #include <linux/refcount.h>
10 #include <linux/uaccess.h>
11
12 struct iommu_domain;
13 struct iommu_group;
14 struct iommu_option;
15
16 struct iommufd_ctx {
17 struct file *file;
18 struct xarray objects;
19
20 u8 account_mode;
21 /* Compatibility with VFIO no iommu */
22 u8 no_iommu_mode;
23 struct iommufd_ioas *vfio_ioas;
24 };
25
26 /*
27 * The IOVA to PFN map. The map automatically copies the PFNs into multiple
28 * domains and permits sharing of PFNs between io_pagetable instances. This
29 * supports both a design where IOAS's are 1:1 with a domain (eg because the
30 * domain is HW customized), or where the IOAS is 1:N with multiple generic
31 * domains. The io_pagetable holds an interval tree of iopt_areas which point
32 * to shared iopt_pages which hold the pfns mapped to the page table.
33 *
34 * The locking order is domains_rwsem -> iova_rwsem -> pages::mutex
35 */
36 struct io_pagetable {
37 struct rw_semaphore domains_rwsem;
38 struct xarray domains;
39 struct xarray access_list;
40 unsigned int next_domain_id;
41
42 struct rw_semaphore iova_rwsem;
43 struct rb_root_cached area_itree;
44 /* IOVA that cannot become reserved, struct iopt_allowed */
45 struct rb_root_cached allowed_itree;
46 /* IOVA that cannot be allocated, struct iopt_reserved */
47 struct rb_root_cached reserved_itree;
48 u8 disable_large_pages;
49 unsigned long iova_alignment;
50 };
51
52 void iopt_init_table(struct io_pagetable *iopt);
53 void iopt_destroy_table(struct io_pagetable *iopt);
54 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova,
55 unsigned long length, struct list_head *pages_list);
56 void iopt_free_pages_list(struct list_head *pages_list);
57 enum {
58 IOPT_ALLOC_IOVA = 1 << 0,
59 };
60 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt,
61 unsigned long *iova, void __user *uptr,
62 unsigned long length, int iommu_prot,
63 unsigned int flags);
64 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list,
65 unsigned long length, unsigned long *dst_iova,
66 int iommu_prot, unsigned int flags);
67 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
68 unsigned long length, unsigned long *unmapped);
69 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped);
70
71 void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
72 unsigned long length);
73 int iopt_table_add_domain(struct io_pagetable *iopt,
74 struct iommu_domain *domain);
75 void iopt_table_remove_domain(struct io_pagetable *iopt,
76 struct iommu_domain *domain);
77 int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt,
78 struct device *device,
79 struct iommu_group *group,
80 phys_addr_t *sw_msi_start);
81 int iopt_set_allow_iova(struct io_pagetable *iopt,
82 struct rb_root_cached *allowed_iova);
83 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,
84 unsigned long last, void *owner);
85 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner);
86 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas,
87 size_t num_iovas);
88 void iopt_enable_large_pages(struct io_pagetable *iopt);
89 int iopt_disable_large_pages(struct io_pagetable *iopt);
90
91 struct iommufd_ucmd {
92 struct iommufd_ctx *ictx;
93 void __user *ubuffer;
94 u32 user_size;
95 void *cmd;
96 };
97
98 int iommufd_vfio_ioctl(struct iommufd_ctx *ictx, unsigned int cmd,
99 unsigned long arg);
100
101 /* Copy the response in ucmd->cmd back to userspace. */
iommufd_ucmd_respond(struct iommufd_ucmd * ucmd,size_t cmd_len)102 static inline int iommufd_ucmd_respond(struct iommufd_ucmd *ucmd,
103 size_t cmd_len)
104 {
105 if (copy_to_user(ucmd->ubuffer, ucmd->cmd,
106 min_t(size_t, ucmd->user_size, cmd_len)))
107 return -EFAULT;
108 return 0;
109 }
110
111 enum iommufd_object_type {
112 IOMMUFD_OBJ_NONE,
113 IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE,
114 IOMMUFD_OBJ_DEVICE,
115 IOMMUFD_OBJ_HW_PAGETABLE,
116 IOMMUFD_OBJ_IOAS,
117 IOMMUFD_OBJ_ACCESS,
118 #ifdef CONFIG_IOMMUFD_TEST
119 IOMMUFD_OBJ_SELFTEST,
120 #endif
121 };
122
123 /* Base struct for all objects with a userspace ID handle. */
124 struct iommufd_object {
125 struct rw_semaphore destroy_rwsem;
126 refcount_t users;
127 enum iommufd_object_type type;
128 unsigned int id;
129 };
130
iommufd_lock_obj(struct iommufd_object * obj)131 static inline bool iommufd_lock_obj(struct iommufd_object *obj)
132 {
133 if (!down_read_trylock(&obj->destroy_rwsem))
134 return false;
135 if (!refcount_inc_not_zero(&obj->users)) {
136 up_read(&obj->destroy_rwsem);
137 return false;
138 }
139 return true;
140 }
141
142 struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
143 enum iommufd_object_type type);
iommufd_put_object(struct iommufd_object * obj)144 static inline void iommufd_put_object(struct iommufd_object *obj)
145 {
146 refcount_dec(&obj->users);
147 up_read(&obj->destroy_rwsem);
148 }
149
150 /**
151 * iommufd_ref_to_users() - Switch from destroy_rwsem to users refcount
152 * protection
153 * @obj - Object to release
154 *
155 * Objects have two refcount protections (destroy_rwsem and the refcount_t
156 * users). Holding either of these will prevent the object from being destroyed.
157 *
158 * Depending on the use case, one protection or the other is appropriate. In
159 * most cases references are being protected by the destroy_rwsem. This allows
160 * orderly destruction of the object because iommufd_object_destroy_user() will
161 * wait for it to become unlocked. However, as a rwsem, it cannot be held across
162 * a system call return. So cases that have longer term needs must switch
163 * to the weaker users refcount_t.
164 *
165 * With users protection iommufd_object_destroy_user() will return false,
166 * refusing to destroy the object, causing -EBUSY to userspace.
167 */
iommufd_ref_to_users(struct iommufd_object * obj)168 static inline void iommufd_ref_to_users(struct iommufd_object *obj)
169 {
170 up_read(&obj->destroy_rwsem);
171 /* iommufd_lock_obj() obtains users as well */
172 }
173 void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj);
174 void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx,
175 struct iommufd_object *obj);
176 void iommufd_object_finalize(struct iommufd_ctx *ictx,
177 struct iommufd_object *obj);
178 bool iommufd_object_destroy_user(struct iommufd_ctx *ictx,
179 struct iommufd_object *obj);
180 struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx,
181 size_t size,
182 enum iommufd_object_type type);
183
184 #define iommufd_object_alloc(ictx, ptr, type) \
185 container_of(_iommufd_object_alloc( \
186 ictx, \
187 sizeof(*(ptr)) + BUILD_BUG_ON_ZERO( \
188 offsetof(typeof(*(ptr)), \
189 obj) != 0), \
190 type), \
191 typeof(*(ptr)), obj)
192
193 /*
194 * The IO Address Space (IOAS) pagetable is a virtual page table backed by the
195 * io_pagetable object. It is a user controlled mapping of IOVA -> PFNs. The
196 * mapping is copied into all of the associated domains and made available to
197 * in-kernel users.
198 *
199 * Every iommu_domain that is created is wrapped in a iommufd_hw_pagetable
200 * object. When we go to attach a device to an IOAS we need to get an
201 * iommu_domain and wrapping iommufd_hw_pagetable for it.
202 *
203 * An iommu_domain & iommfd_hw_pagetable will be automatically selected
204 * for a device based on the hwpt_list. If no suitable iommu_domain
205 * is found a new iommu_domain will be created.
206 */
207 struct iommufd_ioas {
208 struct iommufd_object obj;
209 struct io_pagetable iopt;
210 struct mutex mutex;
211 struct list_head hwpt_list;
212 };
213
iommufd_get_ioas(struct iommufd_ucmd * ucmd,u32 id)214 static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ucmd *ucmd,
215 u32 id)
216 {
217 return container_of(iommufd_get_object(ucmd->ictx, id,
218 IOMMUFD_OBJ_IOAS),
219 struct iommufd_ioas, obj);
220 }
221
222 struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx);
223 int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd);
224 void iommufd_ioas_destroy(struct iommufd_object *obj);
225 int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd);
226 int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd);
227 int iommufd_ioas_map(struct iommufd_ucmd *ucmd);
228 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd);
229 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd);
230 int iommufd_ioas_option(struct iommufd_ucmd *ucmd);
231 int iommufd_option_rlimit_mode(struct iommu_option *cmd,
232 struct iommufd_ctx *ictx);
233
234 int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd);
235
236 /*
237 * A HW pagetable is called an iommu_domain inside the kernel. This user object
238 * allows directly creating and inspecting the domains. Domains that have kernel
239 * owned page tables will be associated with an iommufd_ioas that provides the
240 * IOVA to PFN map.
241 */
242 struct iommufd_hw_pagetable {
243 struct iommufd_object obj;
244 struct iommufd_ioas *ioas;
245 struct iommu_domain *domain;
246 bool auto_domain : 1;
247 bool enforce_cache_coherency : 1;
248 bool msi_cookie : 1;
249 /* Head at iommufd_ioas::hwpt_list */
250 struct list_head hwpt_item;
251 struct mutex devices_lock;
252 struct list_head devices;
253 };
254
255 struct iommufd_hw_pagetable *
256 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
257 struct device *dev);
258 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj);
259
260 void iommufd_device_destroy(struct iommufd_object *obj);
261
262 struct iommufd_access {
263 struct iommufd_object obj;
264 struct iommufd_ctx *ictx;
265 struct iommufd_ioas *ioas;
266 const struct iommufd_access_ops *ops;
267 void *data;
268 unsigned long iova_alignment;
269 u32 iopt_access_list_id;
270 };
271
272 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access);
273 void iopt_remove_access(struct io_pagetable *iopt,
274 struct iommufd_access *access);
275 void iommufd_access_destroy_object(struct iommufd_object *obj);
276
277 #ifdef CONFIG_IOMMUFD_TEST
278 struct iommufd_hw_pagetable *
279 iommufd_device_selftest_attach(struct iommufd_ctx *ictx,
280 struct iommufd_ioas *ioas,
281 struct device *mock_dev);
282 void iommufd_device_selftest_detach(struct iommufd_ctx *ictx,
283 struct iommufd_hw_pagetable *hwpt);
284 int iommufd_test(struct iommufd_ucmd *ucmd);
285 void iommufd_selftest_destroy(struct iommufd_object *obj);
286 extern size_t iommufd_test_memory_limit;
287 void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
288 unsigned int ioas_id, u64 *iova, u32 *flags);
289 bool iommufd_should_fail(void);
290 void __init iommufd_test_init(void);
291 void iommufd_test_exit(void);
292 #else
iommufd_test_syz_conv_iova_id(struct iommufd_ucmd * ucmd,unsigned int ioas_id,u64 * iova,u32 * flags)293 static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd,
294 unsigned int ioas_id,
295 u64 *iova, u32 *flags)
296 {
297 }
iommufd_should_fail(void)298 static inline bool iommufd_should_fail(void)
299 {
300 return false;
301 }
iommufd_test_init(void)302 static inline void __init iommufd_test_init(void)
303 {
304 }
iommufd_test_exit(void)305 static inline void iommufd_test_exit(void)
306 {
307 }
308 #endif
309 #endif
310