1 /*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25 #include <linux/iosys-map.h>
26 #include <linux/io-mapping.h>
27 #include <linux/scatterlist.h>
28
29 #include <drm/ttm/ttm_bo.h>
30 #include <drm/ttm/ttm_placement.h>
31 #include <drm/ttm/ttm_resource.h>
32
33 /**
34 * ttm_lru_bulk_move_init - initialize a bulk move structure
35 * @bulk: the structure to init
36 *
37 * For now just memset the structure to zero.
38 */
ttm_lru_bulk_move_init(struct ttm_lru_bulk_move * bulk)39 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
40 {
41 memset(bulk, 0, sizeof(*bulk));
42 }
43 EXPORT_SYMBOL(ttm_lru_bulk_move_init);
44
45 /**
46 * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
47 *
48 * @bulk: bulk move structure
49 *
50 * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
51 * resource order never changes. Should be called with &ttm_device.lru_lock held.
52 */
ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move * bulk)53 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
54 {
55 unsigned i, j;
56
57 for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
58 for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
59 struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
60 struct ttm_resource_manager *man;
61
62 if (!pos->first)
63 continue;
64
65 lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
66 dma_resv_assert_held(pos->first->bo->base.resv);
67 dma_resv_assert_held(pos->last->bo->base.resv);
68
69 man = ttm_manager_type(pos->first->bo->bdev, i);
70 list_bulk_move_tail(&man->lru[j], &pos->first->lru,
71 &pos->last->lru);
72 }
73 }
74 }
75 EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
76
77 /* Return the bulk move pos object for this resource */
78 static struct ttm_lru_bulk_move_pos *
ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)79 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
80 {
81 return &bulk->pos[res->mem_type][res->bo->priority];
82 }
83
84 /* Move the resource to the tail of the bulk move range */
ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos * pos,struct ttm_resource * res)85 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
86 struct ttm_resource *res)
87 {
88 if (pos->last != res) {
89 list_move(&res->lru, &pos->last->lru);
90 pos->last = res;
91 }
92 }
93
94 /* Add the resource to a bulk_move cursor */
ttm_lru_bulk_move_add(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)95 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
96 struct ttm_resource *res)
97 {
98 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
99
100 if (!pos->first) {
101 pos->first = res;
102 pos->last = res;
103 } else {
104 ttm_lru_bulk_move_pos_tail(pos, res);
105 }
106 }
107
108 /* Remove the resource from a bulk_move range */
ttm_lru_bulk_move_del(struct ttm_lru_bulk_move * bulk,struct ttm_resource * res)109 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
110 struct ttm_resource *res)
111 {
112 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
113
114 if (unlikely(pos->first == res && pos->last == res)) {
115 pos->first = NULL;
116 pos->last = NULL;
117 } else if (pos->first == res) {
118 pos->first = list_next_entry(res, lru);
119 } else if (pos->last == res) {
120 pos->last = list_prev_entry(res, lru);
121 } else {
122 list_move(&res->lru, &pos->last->lru);
123 }
124 }
125
126 /* Add the resource to a bulk move if the BO is configured for it */
ttm_resource_add_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)127 void ttm_resource_add_bulk_move(struct ttm_resource *res,
128 struct ttm_buffer_object *bo)
129 {
130 if (bo->bulk_move && !bo->pin_count)
131 ttm_lru_bulk_move_add(bo->bulk_move, res);
132 }
133
134 /* Remove the resource from a bulk move if the BO is configured for it */
ttm_resource_del_bulk_move(struct ttm_resource * res,struct ttm_buffer_object * bo)135 void ttm_resource_del_bulk_move(struct ttm_resource *res,
136 struct ttm_buffer_object *bo)
137 {
138 if (bo->bulk_move && !bo->pin_count)
139 ttm_lru_bulk_move_del(bo->bulk_move, res);
140 }
141
142 /* Move a resource to the LRU or bulk tail */
ttm_resource_move_to_lru_tail(struct ttm_resource * res)143 void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
144 {
145 struct ttm_buffer_object *bo = res->bo;
146 struct ttm_device *bdev = bo->bdev;
147
148 lockdep_assert_held(&bo->bdev->lru_lock);
149
150 if (bo->pin_count) {
151 list_move_tail(&res->lru, &bdev->pinned);
152
153 } else if (bo->bulk_move) {
154 struct ttm_lru_bulk_move_pos *pos =
155 ttm_lru_bulk_move_pos(bo->bulk_move, res);
156
157 ttm_lru_bulk_move_pos_tail(pos, res);
158 } else {
159 struct ttm_resource_manager *man;
160
161 man = ttm_manager_type(bdev, res->mem_type);
162 list_move_tail(&res->lru, &man->lru[bo->priority]);
163 }
164 }
165
166 /**
167 * ttm_resource_init - resource object constructure
168 * @bo: buffer object this resources is allocated for
169 * @place: placement of the resource
170 * @res: the resource object to inistilize
171 *
172 * Initialize a new resource object. Counterpart of ttm_resource_fini().
173 */
ttm_resource_init(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource * res)174 void ttm_resource_init(struct ttm_buffer_object *bo,
175 const struct ttm_place *place,
176 struct ttm_resource *res)
177 {
178 struct ttm_resource_manager *man;
179
180 res->start = 0;
181 res->size = bo->base.size;
182 res->mem_type = place->mem_type;
183 res->placement = place->flags;
184 res->bus.addr = NULL;
185 res->bus.offset = 0;
186 res->bus.is_iomem = false;
187 res->bus.caching = ttm_cached;
188 res->bo = bo;
189
190 man = ttm_manager_type(bo->bdev, place->mem_type);
191 spin_lock(&bo->bdev->lru_lock);
192 if (bo->pin_count)
193 list_add_tail(&res->lru, &bo->bdev->pinned);
194 else
195 list_add_tail(&res->lru, &man->lru[bo->priority]);
196 man->usage += res->size;
197 spin_unlock(&bo->bdev->lru_lock);
198 }
199 EXPORT_SYMBOL(ttm_resource_init);
200
201 /**
202 * ttm_resource_fini - resource destructor
203 * @man: the resource manager this resource belongs to
204 * @res: the resource to clean up
205 *
206 * Should be used by resource manager backends to clean up the TTM resource
207 * objects before freeing the underlying structure. Makes sure the resource is
208 * removed from the LRU before destruction.
209 * Counterpart of ttm_resource_init().
210 */
ttm_resource_fini(struct ttm_resource_manager * man,struct ttm_resource * res)211 void ttm_resource_fini(struct ttm_resource_manager *man,
212 struct ttm_resource *res)
213 {
214 struct ttm_device *bdev = man->bdev;
215
216 spin_lock(&bdev->lru_lock);
217 list_del_init(&res->lru);
218 man->usage -= res->size;
219 spin_unlock(&bdev->lru_lock);
220 }
221 EXPORT_SYMBOL(ttm_resource_fini);
222
ttm_resource_alloc(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource ** res_ptr)223 int ttm_resource_alloc(struct ttm_buffer_object *bo,
224 const struct ttm_place *place,
225 struct ttm_resource **res_ptr)
226 {
227 struct ttm_resource_manager *man =
228 ttm_manager_type(bo->bdev, place->mem_type);
229 int ret;
230
231 ret = man->func->alloc(man, bo, place, res_ptr);
232 if (ret)
233 return ret;
234
235 spin_lock(&bo->bdev->lru_lock);
236 ttm_resource_add_bulk_move(*res_ptr, bo);
237 spin_unlock(&bo->bdev->lru_lock);
238 return 0;
239 }
240
ttm_resource_free(struct ttm_buffer_object * bo,struct ttm_resource ** res)241 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
242 {
243 struct ttm_resource_manager *man;
244
245 if (!*res)
246 return;
247
248 spin_lock(&bo->bdev->lru_lock);
249 ttm_resource_del_bulk_move(*res, bo);
250 spin_unlock(&bo->bdev->lru_lock);
251 man = ttm_manager_type(bo->bdev, (*res)->mem_type);
252 man->func->free(man, *res);
253 *res = NULL;
254 }
255 EXPORT_SYMBOL(ttm_resource_free);
256
257 /**
258 * ttm_resource_intersects - test for intersection
259 *
260 * @bdev: TTM device structure
261 * @res: The resource to test
262 * @place: The placement to test
263 * @size: How many bytes the new allocation needs.
264 *
265 * Test if @res intersects with @place and @size. Used for testing if evictions
266 * are valueable or not.
267 *
268 * Returns true if the res placement intersects with @place and @size.
269 */
ttm_resource_intersects(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)270 bool ttm_resource_intersects(struct ttm_device *bdev,
271 struct ttm_resource *res,
272 const struct ttm_place *place,
273 size_t size)
274 {
275 struct ttm_resource_manager *man;
276
277 if (!res)
278 return false;
279
280 man = ttm_manager_type(bdev, res->mem_type);
281 if (!place || !man->func->intersects)
282 return true;
283
284 return man->func->intersects(man, res, place, size);
285 }
286
287 /**
288 * ttm_resource_compatible - test for compatibility
289 *
290 * @bdev: TTM device structure
291 * @res: The resource to test
292 * @place: The placement to test
293 * @size: How many bytes the new allocation needs.
294 *
295 * Test if @res compatible with @place and @size.
296 *
297 * Returns true if the res placement compatible with @place and @size.
298 */
ttm_resource_compatible(struct ttm_device * bdev,struct ttm_resource * res,const struct ttm_place * place,size_t size)299 bool ttm_resource_compatible(struct ttm_device *bdev,
300 struct ttm_resource *res,
301 const struct ttm_place *place,
302 size_t size)
303 {
304 struct ttm_resource_manager *man;
305
306 if (!res || !place)
307 return false;
308
309 man = ttm_manager_type(bdev, res->mem_type);
310 if (!man->func->compatible)
311 return true;
312
313 return man->func->compatible(man, res, place, size);
314 }
315
ttm_resource_places_compat(struct ttm_resource * res,const struct ttm_place * places,unsigned num_placement)316 static bool ttm_resource_places_compat(struct ttm_resource *res,
317 const struct ttm_place *places,
318 unsigned num_placement)
319 {
320 struct ttm_buffer_object *bo = res->bo;
321 struct ttm_device *bdev = bo->bdev;
322 unsigned i;
323
324 if (res->placement & TTM_PL_FLAG_TEMPORARY)
325 return false;
326
327 for (i = 0; i < num_placement; i++) {
328 const struct ttm_place *heap = &places[i];
329
330 if (!ttm_resource_compatible(bdev, res, heap, bo->base.size))
331 continue;
332
333 if ((res->mem_type == heap->mem_type) &&
334 (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
335 (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
336 return true;
337 }
338 return false;
339 }
340
341 /**
342 * ttm_resource_compat - check if resource is compatible with placement
343 *
344 * @res: the resource to check
345 * @placement: the placement to check against
346 *
347 * Returns true if the placement is compatible.
348 */
ttm_resource_compat(struct ttm_resource * res,struct ttm_placement * placement)349 bool ttm_resource_compat(struct ttm_resource *res,
350 struct ttm_placement *placement)
351 {
352 if (ttm_resource_places_compat(res, placement->placement,
353 placement->num_placement))
354 return true;
355
356 if ((placement->busy_placement != placement->placement ||
357 placement->num_busy_placement > placement->num_placement) &&
358 ttm_resource_places_compat(res, placement->busy_placement,
359 placement->num_busy_placement))
360 return true;
361
362 return false;
363 }
364 EXPORT_SYMBOL(ttm_resource_compat);
365
ttm_resource_set_bo(struct ttm_resource * res,struct ttm_buffer_object * bo)366 void ttm_resource_set_bo(struct ttm_resource *res,
367 struct ttm_buffer_object *bo)
368 {
369 spin_lock(&bo->bdev->lru_lock);
370 res->bo = bo;
371 spin_unlock(&bo->bdev->lru_lock);
372 }
373
374 /**
375 * ttm_resource_manager_init
376 *
377 * @man: memory manager object to init
378 * @bdev: ttm device this manager belongs to
379 * @size: size of managed resources in arbitrary units
380 *
381 * Initialise core parts of a manager object.
382 */
ttm_resource_manager_init(struct ttm_resource_manager * man,struct ttm_device * bdev,uint64_t size)383 void ttm_resource_manager_init(struct ttm_resource_manager *man,
384 struct ttm_device *bdev,
385 uint64_t size)
386 {
387 unsigned i;
388
389 spin_lock_init(&man->move_lock);
390 man->bdev = bdev;
391 man->size = size;
392 man->usage = 0;
393
394 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
395 INIT_LIST_HEAD(&man->lru[i]);
396 man->move = NULL;
397 }
398 EXPORT_SYMBOL(ttm_resource_manager_init);
399
400 /*
401 * ttm_resource_manager_evict_all
402 *
403 * @bdev - device to use
404 * @man - manager to use
405 *
406 * Evict all the objects out of a memory manager until it is empty.
407 * Part of memory manager cleanup sequence.
408 */
ttm_resource_manager_evict_all(struct ttm_device * bdev,struct ttm_resource_manager * man)409 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
410 struct ttm_resource_manager *man)
411 {
412 struct ttm_operation_ctx ctx = {
413 .interruptible = false,
414 .no_wait_gpu = false,
415 .force_alloc = true
416 };
417 struct dma_fence *fence;
418 int ret;
419 unsigned i;
420
421 /*
422 * Can't use standard list traversal since we're unlocking.
423 */
424
425 spin_lock(&bdev->lru_lock);
426 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
427 while (!list_empty(&man->lru[i])) {
428 spin_unlock(&bdev->lru_lock);
429 ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
430 NULL);
431 if (ret)
432 return ret;
433 spin_lock(&bdev->lru_lock);
434 }
435 }
436 spin_unlock(&bdev->lru_lock);
437
438 spin_lock(&man->move_lock);
439 fence = dma_fence_get(man->move);
440 spin_unlock(&man->move_lock);
441
442 if (fence) {
443 ret = dma_fence_wait(fence, false);
444 dma_fence_put(fence);
445 if (ret)
446 return ret;
447 }
448
449 return 0;
450 }
451 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
452
453 /**
454 * ttm_resource_manager_usage
455 *
456 * @man: A memory manager object.
457 *
458 * Return how many resources are currently used.
459 */
ttm_resource_manager_usage(struct ttm_resource_manager * man)460 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
461 {
462 uint64_t usage;
463
464 spin_lock(&man->bdev->lru_lock);
465 usage = man->usage;
466 spin_unlock(&man->bdev->lru_lock);
467 return usage;
468 }
469 EXPORT_SYMBOL(ttm_resource_manager_usage);
470
471 /**
472 * ttm_resource_manager_debug
473 *
474 * @man: manager type to dump.
475 * @p: printer to use for debug.
476 */
ttm_resource_manager_debug(struct ttm_resource_manager * man,struct drm_printer * p)477 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
478 struct drm_printer *p)
479 {
480 drm_printf(p, " use_type: %d\n", man->use_type);
481 drm_printf(p, " use_tt: %d\n", man->use_tt);
482 drm_printf(p, " size: %llu\n", man->size);
483 drm_printf(p, " usage: %llu\n", ttm_resource_manager_usage(man));
484 if (man->func->debug)
485 man->func->debug(man, p);
486 }
487 EXPORT_SYMBOL(ttm_resource_manager_debug);
488
489 /**
490 * ttm_resource_manager_first
491 *
492 * @man: resource manager to iterate over
493 * @cursor: cursor to record the position
494 *
495 * Returns the first resource from the resource manager.
496 */
497 struct ttm_resource *
ttm_resource_manager_first(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor)498 ttm_resource_manager_first(struct ttm_resource_manager *man,
499 struct ttm_resource_cursor *cursor)
500 {
501 struct ttm_resource *res;
502
503 lockdep_assert_held(&man->bdev->lru_lock);
504
505 for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
506 ++cursor->priority)
507 list_for_each_entry(res, &man->lru[cursor->priority], lru)
508 return res;
509
510 return NULL;
511 }
512
513 /**
514 * ttm_resource_manager_next
515 *
516 * @man: resource manager to iterate over
517 * @cursor: cursor to record the position
518 * @res: the current resource pointer
519 *
520 * Returns the next resource from the resource manager.
521 */
522 struct ttm_resource *
ttm_resource_manager_next(struct ttm_resource_manager * man,struct ttm_resource_cursor * cursor,struct ttm_resource * res)523 ttm_resource_manager_next(struct ttm_resource_manager *man,
524 struct ttm_resource_cursor *cursor,
525 struct ttm_resource *res)
526 {
527 lockdep_assert_held(&man->bdev->lru_lock);
528
529 list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
530 return res;
531
532 for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
533 ++cursor->priority)
534 list_for_each_entry(res, &man->lru[cursor->priority], lru)
535 return res;
536
537 return NULL;
538 }
539
ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)540 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
541 struct iosys_map *dmap,
542 pgoff_t i)
543 {
544 struct ttm_kmap_iter_iomap *iter_io =
545 container_of(iter, typeof(*iter_io), base);
546 void __iomem *addr;
547
548 retry:
549 while (i >= iter_io->cache.end) {
550 iter_io->cache.sg = iter_io->cache.sg ?
551 sg_next(iter_io->cache.sg) : iter_io->st->sgl;
552 iter_io->cache.i = iter_io->cache.end;
553 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
554 PAGE_SHIFT;
555 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
556 iter_io->start;
557 }
558
559 if (i < iter_io->cache.i) {
560 iter_io->cache.end = 0;
561 iter_io->cache.sg = NULL;
562 goto retry;
563 }
564
565 addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
566 (((resource_size_t)i - iter_io->cache.i)
567 << PAGE_SHIFT));
568 iosys_map_set_vaddr_iomem(dmap, addr);
569 }
570
ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter * iter,struct iosys_map * map)571 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
572 struct iosys_map *map)
573 {
574 io_mapping_unmap_local(map->vaddr_iomem);
575 }
576
577 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
578 .map_local = ttm_kmap_iter_iomap_map_local,
579 .unmap_local = ttm_kmap_iter_iomap_unmap_local,
580 .maps_tt = false,
581 };
582
583 /**
584 * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
585 * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
586 * @iomap: The struct io_mapping representing the underlying linear io_memory.
587 * @st: sg_table into @iomap, representing the memory of the struct
588 * ttm_resource.
589 * @start: Offset that needs to be subtracted from @st to make
590 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
591 *
592 * Return: Pointer to the embedded struct ttm_kmap_iter.
593 */
594 struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap * iter_io,struct io_mapping * iomap,struct sg_table * st,resource_size_t start)595 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
596 struct io_mapping *iomap,
597 struct sg_table *st,
598 resource_size_t start)
599 {
600 iter_io->base.ops = &ttm_kmap_iter_io_ops;
601 iter_io->iomap = iomap;
602 iter_io->st = st;
603 iter_io->start = start;
604 memset(&iter_io->cache, 0, sizeof(iter_io->cache));
605
606 return &iter_io->base;
607 }
608 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
609
610 /**
611 * DOC: Linear io iterator
612 *
613 * This code should die in the not too near future. Best would be if we could
614 * make io-mapping use memremap for all io memory, and have memremap
615 * implement a kmap_local functionality. We could then strip a huge amount of
616 * code. These linear io iterators are implemented to mimic old functionality,
617 * and they don't use kmap_local semantics at all internally. Rather ioremap or
618 * friends, and at least on 32-bit they add global TLB flushes and points
619 * of failure.
620 */
621
ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter * iter,struct iosys_map * dmap,pgoff_t i)622 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
623 struct iosys_map *dmap,
624 pgoff_t i)
625 {
626 struct ttm_kmap_iter_linear_io *iter_io =
627 container_of(iter, typeof(*iter_io), base);
628
629 *dmap = iter_io->dmap;
630 iosys_map_incr(dmap, i * PAGE_SIZE);
631 }
632
633 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
634 .map_local = ttm_kmap_iter_linear_io_map_local,
635 .maps_tt = false,
636 };
637
638 /**
639 * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
640 * @iter_io: The iterator to initialize
641 * @bdev: The TTM device
642 * @mem: The ttm resource representing the iomap.
643 *
644 * This function is for internal TTM use only. It sets up a memcpy kmap iterator
645 * pointing at a linear chunk of io memory.
646 *
647 * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
648 * failure.
649 */
650 struct ttm_kmap_iter *
ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)651 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
652 struct ttm_device *bdev,
653 struct ttm_resource *mem)
654 {
655 int ret;
656
657 ret = ttm_mem_io_reserve(bdev, mem);
658 if (ret)
659 goto out_err;
660 if (!mem->bus.is_iomem) {
661 ret = -EINVAL;
662 goto out_io_free;
663 }
664
665 if (mem->bus.addr) {
666 iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
667 iter_io->needs_unmap = false;
668 } else {
669 iter_io->needs_unmap = true;
670 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
671 if (mem->bus.caching == ttm_write_combined)
672 iosys_map_set_vaddr_iomem(&iter_io->dmap,
673 ioremap_wc(mem->bus.offset,
674 mem->size));
675 else if (mem->bus.caching == ttm_cached)
676 iosys_map_set_vaddr(&iter_io->dmap,
677 memremap(mem->bus.offset, mem->size,
678 MEMREMAP_WB |
679 MEMREMAP_WT |
680 MEMREMAP_WC));
681
682 /* If uncached requested or if mapping cached or wc failed */
683 if (iosys_map_is_null(&iter_io->dmap))
684 iosys_map_set_vaddr_iomem(&iter_io->dmap,
685 ioremap(mem->bus.offset,
686 mem->size));
687
688 if (iosys_map_is_null(&iter_io->dmap)) {
689 ret = -ENOMEM;
690 goto out_io_free;
691 }
692 }
693
694 iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
695 return &iter_io->base;
696
697 out_io_free:
698 ttm_mem_io_free(bdev, mem);
699 out_err:
700 return ERR_PTR(ret);
701 }
702
703 /**
704 * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
705 * @iter_io: The iterator to initialize
706 * @bdev: The TTM device
707 * @mem: The ttm resource representing the iomap.
708 *
709 * This function is for internal TTM use only. It cleans up a memcpy kmap
710 * iterator initialized by ttm_kmap_iter_linear_io_init.
711 */
712 void
ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)713 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
714 struct ttm_device *bdev,
715 struct ttm_resource *mem)
716 {
717 if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
718 if (iter_io->dmap.is_iomem)
719 iounmap(iter_io->dmap.vaddr_iomem);
720 else
721 memunmap(iter_io->dmap.vaddr);
722 }
723
724 ttm_mem_io_free(bdev, mem);
725 }
726
727 #if defined(CONFIG_DEBUG_FS)
728
ttm_resource_manager_show(struct seq_file * m,void * unused)729 static int ttm_resource_manager_show(struct seq_file *m, void *unused)
730 {
731 struct ttm_resource_manager *man =
732 (struct ttm_resource_manager *)m->private;
733 struct drm_printer p = drm_seq_file_printer(m);
734 ttm_resource_manager_debug(man, &p);
735 return 0;
736 }
737 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
738
739 #endif
740
741 /**
742 * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
743 * resource manager.
744 * @man: The TTM resource manager for which the debugfs stats file be creates
745 * @parent: debugfs directory in which the file will reside
746 * @name: The filename to create.
747 *
748 * This function setups up a debugfs file that can be used to look
749 * at debug statistics of the specified ttm_resource_manager.
750 */
ttm_resource_manager_create_debugfs(struct ttm_resource_manager * man,struct dentry * parent,const char * name)751 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
752 struct dentry * parent,
753 const char *name)
754 {
755 #if defined(CONFIG_DEBUG_FS)
756 debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
757 #endif
758 }
759 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
760