1 /*
2  * Copyright 2020 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Christian König
23  */
24 
25 #include <linux/iosys-map.h>
26 #include <linux/io-mapping.h>
27 #include <linux/scatterlist.h>
28 
29 #include <drm/ttm/ttm_bo.h>
30 #include <drm/ttm/ttm_placement.h>
31 #include <drm/ttm/ttm_resource.h>
32 
33 /**
34  * ttm_lru_bulk_move_init - initialize a bulk move structure
35  * @bulk: the structure to init
36  *
37  * For now just memset the structure to zero.
38  */
ttm_lru_bulk_move_init(struct ttm_lru_bulk_move * bulk)39 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
40 {
41 	memset(bulk, 0, sizeof(*bulk));
42 }
43 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
44 
45 /**
46  * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
47  *
48  * @bulk: bulk move structure
49  *
50  * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
51  * resource order never changes. Should be called with &ttm_device.lru_lock held.
52  */
ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move * bulk)53 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
54 {
55 	unsigned i, j;
56 
57 	for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
58 		for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
59 			struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
60 			struct ttm_resource_manager *man;
61 
62 			if (!pos->first)
63 				continue;
64 
65 			lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
66 			dma_resv_assert_held(pos->first->bo->base.resv);
67 			dma_resv_assert_held(pos->last->bo->base.resv);
68 
69 			man = ttm_manager_type(pos->first->bo->bdev, i);
70 			list_bulk_move_tail(&man->lru[j], &pos->first->lru,
71 					    &pos->last->lru);
72 		}
73 	}
74 }
75 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
76 
77 /* Return the bulk move pos object for this resource */
78 static struct ttm_lru_bulk_move_pos *
ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)79 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
80 {
81 	return &bulk->pos[res->mem_type][res->bo->priority];
82 }
83 
84 /* Move the resource to the tail of the bulk move range */
ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos * pos,struct ttm_resource * res)85 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
86 				       struct ttm_resource *res)
87 {
88 	if (pos->last != res) {
89 		list_move(&res->lru, &pos->last->lru);
90 		pos->last = res;
91 	}
92 }
93 
94 /* Add the resource to a bulk_move cursor */
ttm_lru_bulk_move_add(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)95 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
96 				  struct ttm_resource *res)
97 {
98 	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
99 
100 	if (!pos->first) {
101 		pos->first = res;
102 		pos->last = res;
103 	} else {
104 		ttm_lru_bulk_move_pos_tail(pos, res);
105 	}
106 }
107 
108 /* Remove the resource from a bulk_move range */
ttm_lru_bulk_move_del(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)109 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
110 				  struct ttm_resource *res)
111 {
112 	struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
113 
114 	if (unlikely(pos->first == res && pos->last == res)) {
115 		pos->first = NULL;
116 		pos->last = NULL;
117 	} else if (pos->first == res) {
118 		pos->first = list_next_entry(res, lru);
119 	} else if (pos->last == res) {
120 		pos->last = list_prev_entry(res, lru);
121 	} else {
122 		list_move(&res->lru, &pos->last->lru);
123 	}
124 }
125 
126 /* Add the resource to a bulk move if the BO is configured for it */
ttm_resource_add_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)127 void ttm_resource_add_bulk_move(struct ttm_resource *res,
128 				struct ttm_buffer_object *bo)
129 {
130 	if (bo->bulk_move && !bo->pin_count)
131 		ttm_lru_bulk_move_add(bo->bulk_move, res);
132 }
133 
134 /* Remove the resource from a bulk move if the BO is configured for it */
ttm_resource_del_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)135 void ttm_resource_del_bulk_move(struct ttm_resource *res,
136 				struct ttm_buffer_object *bo)
137 {
138 	if (bo->bulk_move && !bo->pin_count)
139 		ttm_lru_bulk_move_del(bo->bulk_move, res);
140 }
141 
142 /* Move a resource to the LRU or bulk tail */
ttm_resource_move_to_lru_tail(struct ttm_resource * res)143 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
144 {
145 	struct ttm_buffer_object *bo = res->bo;
146 	struct ttm_device *bdev = bo->bdev;
147 
148 	lockdep_assert_held(&bo->bdev->lru_lock);
149 
150 	if (bo->pin_count) {
151 		list_move_tail(&res->lru, &bdev->pinned);
152 
153 	} else	if (bo->bulk_move) {
154 		struct ttm_lru_bulk_move_pos *pos =
155 			ttm_lru_bulk_move_pos(bo->bulk_move, res);
156 
157 		ttm_lru_bulk_move_pos_tail(pos, res);
158 	} else {
159 		struct ttm_resource_manager *man;
160 
161 		man = ttm_manager_type(bdev, res->mem_type);
162 		list_move_tail(&res->lru, &man->lru[bo->priority]);
163 	}
164 }
165 
166 /**
167  * ttm_resource_init - resource object constructure
168  * @bo: buffer object this resources is allocated for
169  * @place: placement of the resource
170  * @res: the resource object to inistilize
171  *
172  * Initialize a new resource object. Counterpart of ttm_resource_fini().
173  */
ttm_resource_init(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource * res)174 void ttm_resource_init(struct ttm_buffer_object *bo,
175                        const struct ttm_place *place,
176                        struct ttm_resource *res)
177 {
178 	struct ttm_resource_manager *man;
179 
180 	res->start = 0;
181 	res->size = bo->base.size;
182 	res->mem_type = place->mem_type;
183 	res->placement = place->flags;
184 	res->bus.addr = NULL;
185 	res->bus.offset = 0;
186 	res->bus.is_iomem = false;
187 	res->bus.caching = ttm_cached;
188 	res->bo = bo;
189 
190 	man = ttm_manager_type(bo->bdev, place->mem_type);
191 	spin_lock(&bo->bdev->lru_lock);
192 	if (bo->pin_count)
193 		list_add_tail(&res->lru, &bo->bdev->pinned);
194 	else
195 		list_add_tail(&res->lru, &man->lru[bo->priority]);
196 	man->usage += res->size;
197 	spin_unlock(&bo->bdev->lru_lock);
198 }
199 EXPORT_SYMBOL(ttm_resource_init);
200 
201 /**
202  * ttm_resource_fini - resource destructor
203  * @man: the resource manager this resource belongs to
204  * @res: the resource to clean up
205  *
206  * Should be used by resource manager backends to clean up the TTM resource
207  * objects before freeing the underlying structure. Makes sure the resource is
208  * removed from the LRU before destruction.
209  * Counterpart of ttm_resource_init().
210  */
ttm_resource_fini(struct ttm_resource_manager * man,struct ttm_resource * res)211 void ttm_resource_fini(struct ttm_resource_manager *man,
212 		       struct ttm_resource *res)
213 {
214 	struct ttm_device *bdev = man->bdev;
215 
216 	spin_lock(&bdev->lru_lock);
217 	list_del_init(&res->lru);
218 	man->usage -= res->size;
219 	spin_unlock(&bdev->lru_lock);
220 }
221 EXPORT_SYMBOL(ttm_resource_fini);
222 
ttm_resource_alloc(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource ** res_ptr)223 int ttm_resource_alloc(struct ttm_buffer_object *bo,
224 		       const struct ttm_place *place,
225 		       struct ttm_resource **res_ptr)
226 {
227 	struct ttm_resource_manager *man =
228 		ttm_manager_type(bo->bdev, place->mem_type);
229 	int ret;
230 
231 	ret = man->func->alloc(man, bo, place, res_ptr);
232 	if (ret)
233 		return ret;
234 
235 	spin_lock(&bo->bdev->lru_lock);
236 	ttm_resource_add_bulk_move(*res_ptr, bo);
237 	spin_unlock(&bo->bdev->lru_lock);
238 	return 0;
239 }
240 
ttm_resource_free(struct ttm_buffer_object * bo,struct ttm_resource ** res)241 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
242 {
243 	struct ttm_resource_manager *man;
244 
245 	if (!*res)
246 		return;
247 
248 	spin_lock(&bo->bdev->lru_lock);
249 	ttm_resource_del_bulk_move(*res, bo);
250 	spin_unlock(&bo->bdev->lru_lock);
251 	man = ttm_manager_type(bo->bdev, (*res)->mem_type);
252 	man->func->free(man, *res);
253 	*res = NULL;
254 }
255 EXPORT_SYMBOL(ttm_resource_free);
256 
257 /**
258  * ttm_resource_intersects - test for intersection
259  *
260  * @bdev: TTM device structure
261  * @res: The resource to test
262  * @place: The placement to test
263  * @size: How many bytes the new allocation needs.
264  *
265  * Test if @res intersects with @place and @size. Used for testing if evictions
266  * are valueable or not.
267  *
268  * Returns true if the res placement intersects with @place and @size.
269  */
ttm_resource_intersects(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)270 bool ttm_resource_intersects(struct ttm_device *bdev,
271 			     struct ttm_resource *res,
272 			     const struct ttm_place *place,
273 			     size_t size)
274 {
275 	struct ttm_resource_manager *man;
276 
277 	if (!res)
278 		return false;
279 
280 	man = ttm_manager_type(bdev, res->mem_type);
281 	if (!place || !man->func->intersects)
282 		return true;
283 
284 	return man->func->intersects(man, res, place, size);
285 }
286 
287 /**
288  * ttm_resource_compatible - test for compatibility
289  *
290  * @bdev: TTM device structure
291  * @res: The resource to test
292  * @place: The placement to test
293  * @size: How many bytes the new allocation needs.
294  *
295  * Test if @res compatible with @place and @size.
296  *
297  * Returns true if the res placement compatible with @place and @size.
298  */
ttm_resource_compatible(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)299 bool ttm_resource_compatible(struct ttm_device *bdev,
300 			     struct ttm_resource *res,
301 			     const struct ttm_place *place,
302 			     size_t size)
303 {
304 	struct ttm_resource_manager *man;
305 
306 	if (!res || !place)
307 		return false;
308 
309 	man = ttm_manager_type(bdev, res->mem_type);
310 	if (!man->func->compatible)
311 		return true;
312 
313 	return man->func->compatible(man, res, place, size);
314 }
315 
ttm_resource_places_compat(struct ttm_resource * res,const struct ttm_place * places,unsigned num_placement)316 static bool ttm_resource_places_compat(struct ttm_resource *res,
317 				       const struct ttm_place *places,
318 				       unsigned num_placement)
319 {
320 	struct ttm_buffer_object *bo = res->bo;
321 	struct ttm_device *bdev = bo->bdev;
322 	unsigned i;
323 
324 	if (res->placement & TTM_PL_FLAG_TEMPORARY)
325 		return false;
326 
327 	for (i = 0; i < num_placement; i++) {
328 		const struct ttm_place *heap = &places[i];
329 
330 		if (!ttm_resource_compatible(bdev, res, heap, bo->base.size))
331 			continue;
332 
333 		if ((res->mem_type == heap->mem_type) &&
334 		    (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
335 		     (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
336 			return true;
337 	}
338 	return false;
339 }
340 
341 /**
342  * ttm_resource_compat - check if resource is compatible with placement
343  *
344  * @res: the resource to check
345  * @placement: the placement to check against
346  *
347  * Returns true if the placement is compatible.
348  */
ttm_resource_compat(struct ttm_resource * res,struct ttm_placement * placement)349 bool ttm_resource_compat(struct ttm_resource *res,
350 			 struct ttm_placement *placement)
351 {
352 	if (ttm_resource_places_compat(res, placement->placement,
353 				       placement->num_placement))
354 		return true;
355 
356 	if ((placement->busy_placement != placement->placement ||
357 	     placement->num_busy_placement > placement->num_placement) &&
358 	    ttm_resource_places_compat(res, placement->busy_placement,
359 				       placement->num_busy_placement))
360 		return true;
361 
362 	return false;
363 }
364 EXPORT_SYMBOL(ttm_resource_compat);
365 
ttm_resource_set_bo(struct ttm_resource * res,struct ttm_buffer_object * bo)366 void ttm_resource_set_bo(struct ttm_resource *res,
367 			 struct ttm_buffer_object *bo)
368 {
369 	spin_lock(&bo->bdev->lru_lock);
370 	res->bo = bo;
371 	spin_unlock(&bo->bdev->lru_lock);
372 }
373 
374 /**
375  * ttm_resource_manager_init
376  *
377  * @man: memory manager object to init
378  * @bdev: ttm device this manager belongs to
379  * @size: size of managed resources in arbitrary units
380  *
381  * Initialise core parts of a manager object.
382  */
ttm_resource_manager_init(struct ttm_resource_manager * man,struct ttm_device * bdev,uint64_t size)383 void ttm_resource_manager_init(struct ttm_resource_manager *man,
384 			       struct ttm_device *bdev,
385 			       uint64_t size)
386 {
387 	unsigned i;
388 
389 	spin_lock_init(&man->move_lock);
390 	man->bdev = bdev;
391 	man->size = size;
392 	man->usage = 0;
393 
394 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
395 		INIT_LIST_HEAD(&man->lru[i]);
396 	man->move = NULL;
397 }
398 EXPORT_SYMBOL(ttm_resource_manager_init);
399 
400 /*
401  * ttm_resource_manager_evict_all
402  *
403  * @bdev - device to use
404  * @man - manager to use
405  *
406  * Evict all the objects out of a memory manager until it is empty.
407  * Part of memory manager cleanup sequence.
408  */
ttm_resource_manager_evict_all(struct ttm_device * bdev,struct ttm_resource_manager * man)409 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
410 				   struct ttm_resource_manager *man)
411 {
412 	struct ttm_operation_ctx ctx = {
413 		.interruptible = false,
414 		.no_wait_gpu = false,
415 		.force_alloc = true
416 	};
417 	struct dma_fence *fence;
418 	int ret;
419 	unsigned i;
420 
421 	/*
422 	 * Can't use standard list traversal since we're unlocking.
423 	 */
424 
425 	spin_lock(&bdev->lru_lock);
426 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
427 		while (!list_empty(&man->lru[i])) {
428 			spin_unlock(&bdev->lru_lock);
429 			ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
430 						  NULL);
431 			if (ret)
432 				return ret;
433 			spin_lock(&bdev->lru_lock);
434 		}
435 	}
436 	spin_unlock(&bdev->lru_lock);
437 
438 	spin_lock(&man->move_lock);
439 	fence = dma_fence_get(man->move);
440 	spin_unlock(&man->move_lock);
441 
442 	if (fence) {
443 		ret = dma_fence_wait(fence, false);
444 		dma_fence_put(fence);
445 		if (ret)
446 			return ret;
447 	}
448 
449 	return 0;
450 }
451 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
452 
453 /**
454  * ttm_resource_manager_usage
455  *
456  * @man: A memory manager object.
457  *
458  * Return how many resources are currently used.
459  */
ttm_resource_manager_usage(struct ttm_resource_manager * man)460 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
461 {
462 	uint64_t usage;
463 
464 	spin_lock(&man->bdev->lru_lock);
465 	usage = man->usage;
466 	spin_unlock(&man->bdev->lru_lock);
467 	return usage;
468 }
469 EXPORT_SYMBOL(ttm_resource_manager_usage);
470 
471 /**
472  * ttm_resource_manager_debug
473  *
474  * @man: manager type to dump.
475  * @p: printer to use for debug.
476  */
ttm_resource_manager_debug(struct ttm_resource_manager * man,struct drm_printer * p)477 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
478 				struct drm_printer *p)
479 {
480 	drm_printf(p, "  use_type: %d\n", man->use_type);
481 	drm_printf(p, "  use_tt: %d\n", man->use_tt);
482 	drm_printf(p, "  size: %llu\n", man->size);
483 	drm_printf(p, "  usage: %llu\n", ttm_resource_manager_usage(man));
484 	if (man->func->debug)
485 		man->func->debug(man, p);
486 }
487 EXPORT_SYMBOL(ttm_resource_manager_debug);
488 
489 /**
490  * ttm_resource_manager_first
491  *
492  * @man: resource manager to iterate over
493  * @cursor: cursor to record the position
494  *
495  * Returns the first resource from the resource manager.
496  */
497 struct ttm_resource *
ttm_resource_manager_first(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor)498 ttm_resource_manager_first(struct ttm_resource_manager *man,
499 			   struct ttm_resource_cursor *cursor)
500 {
501 	struct ttm_resource *res;
502 
503 	lockdep_assert_held(&man->bdev->lru_lock);
504 
505 	for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
506 	     ++cursor->priority)
507 		list_for_each_entry(res, &man->lru[cursor->priority], lru)
508 			return res;
509 
510 	return NULL;
511 }
512 
513 /**
514  * ttm_resource_manager_next
515  *
516  * @man: resource manager to iterate over
517  * @cursor: cursor to record the position
518  * @res: the current resource pointer
519  *
520  * Returns the next resource from the resource manager.
521  */
522 struct ttm_resource *
ttm_resource_manager_next(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor,struct ttm_resource * res)523 ttm_resource_manager_next(struct ttm_resource_manager *man,
524 			  struct ttm_resource_cursor *cursor,
525 			  struct ttm_resource *res)
526 {
527 	lockdep_assert_held(&man->bdev->lru_lock);
528 
529 	list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
530 		return res;
531 
532 	for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
533 	     ++cursor->priority)
534 		list_for_each_entry(res, &man->lru[cursor->priority], lru)
535 			return res;
536 
537 	return NULL;
538 }
539 
ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)540 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
541 					  struct iosys_map *dmap,
542 					  pgoff_t i)
543 {
544 	struct ttm_kmap_iter_iomap *iter_io =
545 		container_of(iter, typeof(*iter_io), base);
546 	void __iomem *addr;
547 
548 retry:
549 	while (i >= iter_io->cache.end) {
550 		iter_io->cache.sg = iter_io->cache.sg ?
551 			sg_next(iter_io->cache.sg) : iter_io->st->sgl;
552 		iter_io->cache.i = iter_io->cache.end;
553 		iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
554 			PAGE_SHIFT;
555 		iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
556 			iter_io->start;
557 	}
558 
559 	if (i < iter_io->cache.i) {
560 		iter_io->cache.end = 0;
561 		iter_io->cache.sg = NULL;
562 		goto retry;
563 	}
564 
565 	addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
566 				       (((resource_size_t)i - iter_io->cache.i)
567 					<< PAGE_SHIFT));
568 	iosys_map_set_vaddr_iomem(dmap, addr);
569 }
570 
ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter * iter,struct iosys_map * map)571 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
572 					    struct iosys_map *map)
573 {
574 	io_mapping_unmap_local(map->vaddr_iomem);
575 }
576 
577 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
578 	.map_local =  ttm_kmap_iter_iomap_map_local,
579 	.unmap_local = ttm_kmap_iter_iomap_unmap_local,
580 	.maps_tt = false,
581 };
582 
583 /**
584  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
585  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
586  * @iomap: The struct io_mapping representing the underlying linear io_memory.
587  * @st: sg_table into @iomap, representing the memory of the struct
588  * ttm_resource.
589  * @start: Offset that needs to be subtracted from @st to make
590  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
591  *
592  * Return: Pointer to the embedded struct ttm_kmap_iter.
593  */
594 struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap * iter_io,struct io_mapping * iomap,struct sg_table * st,resource_size_t start)595 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
596 			 struct io_mapping *iomap,
597 			 struct sg_table *st,
598 			 resource_size_t start)
599 {
600 	iter_io->base.ops = &ttm_kmap_iter_io_ops;
601 	iter_io->iomap = iomap;
602 	iter_io->st = st;
603 	iter_io->start = start;
604 	memset(&iter_io->cache, 0, sizeof(iter_io->cache));
605 
606 	return &iter_io->base;
607 }
608 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
609 
610 /**
611  * DOC: Linear io iterator
612  *
613  * This code should die in the not too near future. Best would be if we could
614  * make io-mapping use memremap for all io memory, and have memremap
615  * implement a kmap_local functionality. We could then strip a huge amount of
616  * code. These linear io iterators are implemented to mimic old functionality,
617  * and they don't use kmap_local semantics at all internally. Rather ioremap or
618  * friends, and at least on 32-bit they add global TLB flushes and points
619  * of failure.
620  */
621 
ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)622 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
623 					      struct iosys_map *dmap,
624 					      pgoff_t i)
625 {
626 	struct ttm_kmap_iter_linear_io *iter_io =
627 		container_of(iter, typeof(*iter_io), base);
628 
629 	*dmap = iter_io->dmap;
630 	iosys_map_incr(dmap, i * PAGE_SIZE);
631 }
632 
633 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
634 	.map_local =  ttm_kmap_iter_linear_io_map_local,
635 	.maps_tt = false,
636 };
637 
638 /**
639  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
640  * @iter_io: The iterator to initialize
641  * @bdev: The TTM device
642  * @mem: The ttm resource representing the iomap.
643  *
644  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
645  * pointing at a linear chunk of io memory.
646  *
647  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
648  * failure.
649  */
650 struct ttm_kmap_iter *
ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)651 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
652 			     struct ttm_device *bdev,
653 			     struct ttm_resource *mem)
654 {
655 	int ret;
656 
657 	ret = ttm_mem_io_reserve(bdev, mem);
658 	if (ret)
659 		goto out_err;
660 	if (!mem->bus.is_iomem) {
661 		ret = -EINVAL;
662 		goto out_io_free;
663 	}
664 
665 	if (mem->bus.addr) {
666 		iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
667 		iter_io->needs_unmap = false;
668 	} else {
669 		iter_io->needs_unmap = true;
670 		memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
671 		if (mem->bus.caching == ttm_write_combined)
672 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
673 						  ioremap_wc(mem->bus.offset,
674 							     mem->size));
675 		else if (mem->bus.caching == ttm_cached)
676 			iosys_map_set_vaddr(&iter_io->dmap,
677 					    memremap(mem->bus.offset, mem->size,
678 						     MEMREMAP_WB |
679 						     MEMREMAP_WT |
680 						     MEMREMAP_WC));
681 
682 		/* If uncached requested or if mapping cached or wc failed */
683 		if (iosys_map_is_null(&iter_io->dmap))
684 			iosys_map_set_vaddr_iomem(&iter_io->dmap,
685 						  ioremap(mem->bus.offset,
686 							  mem->size));
687 
688 		if (iosys_map_is_null(&iter_io->dmap)) {
689 			ret = -ENOMEM;
690 			goto out_io_free;
691 		}
692 	}
693 
694 	iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
695 	return &iter_io->base;
696 
697 out_io_free:
698 	ttm_mem_io_free(bdev, mem);
699 out_err:
700 	return ERR_PTR(ret);
701 }
702 
703 /**
704  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
705  * @iter_io: The iterator to initialize
706  * @bdev: The TTM device
707  * @mem: The ttm resource representing the iomap.
708  *
709  * This function is for internal TTM use only. It cleans up a memcpy kmap
710  * iterator initialized by ttm_kmap_iter_linear_io_init.
711  */
712 void
ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)713 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
714 			     struct ttm_device *bdev,
715 			     struct ttm_resource *mem)
716 {
717 	if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
718 		if (iter_io->dmap.is_iomem)
719 			iounmap(iter_io->dmap.vaddr_iomem);
720 		else
721 			memunmap(iter_io->dmap.vaddr);
722 	}
723 
724 	ttm_mem_io_free(bdev, mem);
725 }
726 
727 #if defined(CONFIG_DEBUG_FS)
728 
ttm_resource_manager_show(struct seq_file * m,void * unused)729 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
730 {
731 	struct ttm_resource_manager *man =
732 		(struct ttm_resource_manager *)m->private;
733 	struct drm_printer p = drm_seq_file_printer(m);
734 	ttm_resource_manager_debug(man, &p);
735 	return 0;
736 }
737 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
738 
739 #endif
740 
741 /**
742  * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
743  * resource manager.
744  * @man: The TTM resource manager for which the debugfs stats file be creates
745  * @parent: debugfs directory in which the file will reside
746  * @name: The filename to create.
747  *
748  * This function setups up a debugfs file that can be used to look
749  * at debug statistics of the specified ttm_resource_manager.
750  */
ttm_resource_manager_create_debugfs(struct ttm_resource_manager * man,struct dentry * parent,const char * name)751 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
752 					 struct dentry * parent,
753 					 const char *name)
754 {
755 #if defined(CONFIG_DEBUG_FS)
756 	debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
757 #endif
758 }
759 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
760