1/*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25#include <linux/debugfs.h>
26#include <linux/export.h>
27#include <linux/io-mapping.h>
28#include <linux/iosys-map.h>
29#include <linux/scatterlist.h>
30#include <linux/cgroup_dmem.h>
31
32#include <drm/ttm/ttm_bo.h>
33#include <drm/ttm/ttm_placement.h>
34#include <drm/ttm/ttm_resource.h>
35#include <drm/ttm/ttm_tt.h>
36
37#include <drm/drm_print.h>
38#include <drm/drm_util.h>
39
40/* Detach the cursor from the bulk move list*/
41static void
42ttm_resource_cursor_clear_bulk(struct ttm_resource_cursor *cursor)
43{
44 lockdep_assert_held(&cursor->man->bdev->lru_lock);
45
46 cursor->bulk = NULL;
47 list_del_init(entry: &cursor->bulk_link);
48}
49
50/* Move the cursor to the end of the bulk move list it's in */
51static void ttm_resource_cursor_move_bulk_tail(struct ttm_lru_bulk_move *bulk,
52 struct ttm_resource_cursor *cursor)
53{
54 struct ttm_lru_bulk_move_pos *pos;
55
56 lockdep_assert_held(&cursor->man->bdev->lru_lock);
57
58 if (WARN_ON_ONCE(bulk != cursor->bulk)) {
59 list_del_init(entry: &cursor->bulk_link);
60 return;
61 }
62
63 pos = &bulk->pos[cursor->mem_type][cursor->priority];
64 if (pos->last)
65 list_move(list: &cursor->hitch.link, head: &pos->last->lru.link);
66 ttm_resource_cursor_clear_bulk(cursor);
67}
68
69/* Move all cursors attached to a bulk move to its end */
70static void ttm_bulk_move_adjust_cursors(struct ttm_lru_bulk_move *bulk)
71{
72 struct ttm_resource_cursor *cursor, *next;
73
74 list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
75 ttm_resource_cursor_move_bulk_tail(bulk, cursor);
76}
77
78/* Remove a cursor from an empty bulk move list */
79static void ttm_bulk_move_drop_cursors(struct ttm_lru_bulk_move *bulk)
80{
81 struct ttm_resource_cursor *cursor, *next;
82
83 list_for_each_entry_safe(cursor, next, &bulk->cursor_list, bulk_link)
84 ttm_resource_cursor_clear_bulk(cursor);
85}
86
87/**
88 * ttm_resource_cursor_init() - Initialize a struct ttm_resource_cursor
89 * @cursor: The cursor to initialize.
90 * @man: The resource manager.
91 *
92 * Initialize the cursor before using it for iteration.
93 */
94void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor,
95 struct ttm_resource_manager *man)
96{
97 cursor->priority = 0;
98 cursor->man = man;
99 ttm_lru_item_init(item: &cursor->hitch, type: TTM_LRU_HITCH);
100 INIT_LIST_HEAD(list: &cursor->bulk_link);
101 INIT_LIST_HEAD(list: &cursor->hitch.link);
102}
103
104/**
105 * ttm_resource_cursor_fini() - Finalize the LRU list cursor usage
106 * @cursor: The struct ttm_resource_cursor to finalize.
107 *
108 * The function pulls the LRU list cursor off any lists it was previusly
109 * attached to. Needs to be called with the LRU lock held. The function
110 * can be called multiple times after eachother.
111 */
112void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor)
113{
114 lockdep_assert_held(&cursor->man->bdev->lru_lock);
115 list_del_init(entry: &cursor->hitch.link);
116 ttm_resource_cursor_clear_bulk(cursor);
117}
118
119/**
120 * ttm_lru_bulk_move_init - initialize a bulk move structure
121 * @bulk: the structure to init
122 *
123 * For now just memset the structure to zero.
124 */
125void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
126{
127 memset(bulk, 0, sizeof(*bulk));
128 INIT_LIST_HEAD(list: &bulk->cursor_list);
129}
130EXPORT_SYMBOL(ttm_lru_bulk_move_init);
131
132/**
133 * ttm_lru_bulk_move_fini - finalize a bulk move structure
134 * @bdev: The struct ttm_device
135 * @bulk: the structure to finalize
136 *
137 * Sanity checks that bulk moves don't have any
138 * resources left and hence no cursors attached.
139 */
140void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
141 struct ttm_lru_bulk_move *bulk)
142{
143 spin_lock(lock: &bdev->lru_lock);
144 ttm_bulk_move_drop_cursors(bulk);
145 spin_unlock(lock: &bdev->lru_lock);
146}
147EXPORT_SYMBOL(ttm_lru_bulk_move_fini);
148
149/**
150 * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
151 *
152 * @bulk: bulk move structure
153 *
154 * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
155 * resource order never changes. Should be called with &ttm_device.lru_lock held.
156 */
157void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
158{
159 unsigned i, j;
160
161 ttm_bulk_move_adjust_cursors(bulk);
162 for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
163 for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
164 struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
165 struct ttm_resource_manager *man;
166
167 if (!pos->first)
168 continue;
169
170 lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
171 dma_resv_assert_held(pos->first->bo->base.resv);
172 dma_resv_assert_held(pos->last->bo->base.resv);
173
174 man = ttm_manager_type(bdev: pos->first->bo->bdev, mem_type: i);
175 list_bulk_move_tail(head: &man->lru[j], first: &pos->first->lru.link,
176 last: &pos->last->lru.link);
177 }
178 }
179}
180EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
181
182/* Return the bulk move pos object for this resource */
183static struct ttm_lru_bulk_move_pos *
184ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
185{
186 return &bulk->pos[res->mem_type][res->bo->priority];
187}
188
189/* Return the previous resource on the list (skip over non-resource list items) */
190static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur)
191{
192 struct ttm_lru_item *lru = &cur->lru;
193
194 do {
195 lru = list_prev_entry(lru, link);
196 } while (!ttm_lru_item_is_res(item: lru));
197
198 return ttm_lru_item_to_res(item: lru);
199}
200
201/* Return the next resource on the list (skip over non-resource list items) */
202static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur)
203{
204 struct ttm_lru_item *lru = &cur->lru;
205
206 do {
207 lru = list_next_entry(lru, link);
208 } while (!ttm_lru_item_is_res(item: lru));
209
210 return ttm_lru_item_to_res(item: lru);
211}
212
213/* Move the resource to the tail of the bulk move range */
214static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
215 struct ttm_resource *res)
216{
217 if (pos->last != res) {
218 if (pos->first == res)
219 pos->first = ttm_lru_next_res(cur: res);
220 list_move(list: &res->lru.link, head: &pos->last->lru.link);
221 pos->last = res;
222 }
223}
224
225/* Add the resource to a bulk_move cursor */
226static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
227 struct ttm_resource *res)
228{
229 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
230
231 if (!pos->first) {
232 pos->first = res;
233 pos->last = res;
234 } else {
235 WARN_ON(pos->first->bo->base.resv != res->bo->base.resv);
236 ttm_lru_bulk_move_pos_tail(pos, res);
237 }
238}
239
240/* Remove the resource from a bulk_move range */
241static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
242 struct ttm_resource *res)
243{
244 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
245
246 if (unlikely(WARN_ON(!pos->first || !pos->last) ||
247 (pos->first == res && pos->last == res))) {
248 pos->first = NULL;
249 pos->last = NULL;
250 } else if (pos->first == res) {
251 pos->first = ttm_lru_next_res(cur: res);
252 } else if (pos->last == res) {
253 pos->last = ttm_lru_prev_res(cur: res);
254 } else {
255 list_move(list: &res->lru.link, head: &pos->last->lru.link);
256 }
257}
258
259static bool ttm_resource_is_swapped(struct ttm_resource *res, struct ttm_buffer_object *bo)
260{
261 /*
262 * Take care when creating a new resource for a bo, that it is not considered
263 * swapped if it's not the current resource for the bo and is thus logically
264 * associated with the ttm_tt. Think a VRAM resource created to move a
265 * swapped-out bo to VRAM.
266 */
267 if (bo->resource != res || !bo->ttm)
268 return false;
269
270 dma_resv_assert_held(bo->base.resv);
271 return ttm_tt_is_swapped(tt: bo->ttm);
272}
273
274static bool ttm_resource_unevictable(struct ttm_resource *res, struct ttm_buffer_object *bo)
275{
276 return bo->pin_count || ttm_resource_is_swapped(res, bo);
277}
278
279/* Add the resource to a bulk move if the BO is configured for it */
280void ttm_resource_add_bulk_move(struct ttm_resource *res,
281 struct ttm_buffer_object *bo)
282{
283 if (bo->bulk_move && !ttm_resource_unevictable(res, bo))
284 ttm_lru_bulk_move_add(bulk: bo->bulk_move, res);
285}
286
287/* Remove the resource from a bulk move if the BO is configured for it */
288void ttm_resource_del_bulk_move(struct ttm_resource *res,
289 struct ttm_buffer_object *bo)
290{
291 if (bo->bulk_move && !ttm_resource_unevictable(res, bo))
292 ttm_lru_bulk_move_del(bulk: bo->bulk_move, res);
293}
294
295/* Move a resource to the LRU or bulk tail */
296void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
297{
298 struct ttm_buffer_object *bo = res->bo;
299 struct ttm_device *bdev = bo->bdev;
300
301 lockdep_assert_held(&bo->bdev->lru_lock);
302
303 if (ttm_resource_unevictable(res, bo)) {
304 list_move_tail(list: &res->lru.link, head: &bdev->unevictable);
305
306 } else if (bo->bulk_move) {
307 struct ttm_lru_bulk_move_pos *pos =
308 ttm_lru_bulk_move_pos(bulk: bo->bulk_move, res);
309
310 ttm_lru_bulk_move_pos_tail(pos, res);
311 } else {
312 struct ttm_resource_manager *man;
313
314 man = ttm_manager_type(bdev, mem_type: res->mem_type);
315 list_move_tail(list: &res->lru.link, head: &man->lru[bo->priority]);
316 }
317}
318
319/**
320 * ttm_resource_init - resource object constructure
321 * @bo: buffer object this resources is allocated for
322 * @place: placement of the resource
323 * @res: the resource object to inistilize
324 *
325 * Initialize a new resource object. Counterpart of ttm_resource_fini().
326 */
327void ttm_resource_init(struct ttm_buffer_object *bo,
328 const struct ttm_place *place,
329 struct ttm_resource *res)
330{
331 struct ttm_resource_manager *man;
332
333 res->start = 0;
334 res->size = bo->base.size;
335 res->mem_type = place->mem_type;
336 res->placement = place->flags;
337 res->bus.addr = NULL;
338 res->bus.offset = 0;
339 res->bus.is_iomem = false;
340 res->bus.caching = ttm_cached;
341 res->bo = bo;
342
343 man = ttm_manager_type(bdev: bo->bdev, mem_type: place->mem_type);
344 spin_lock(lock: &bo->bdev->lru_lock);
345 if (ttm_resource_unevictable(res, bo))
346 list_add_tail(new: &res->lru.link, head: &bo->bdev->unevictable);
347 else
348 list_add_tail(new: &res->lru.link, head: &man->lru[bo->priority]);
349 man->usage += res->size;
350 spin_unlock(lock: &bo->bdev->lru_lock);
351}
352EXPORT_SYMBOL(ttm_resource_init);
353
354/**
355 * ttm_resource_fini - resource destructor
356 * @man: the resource manager this resource belongs to
357 * @res: the resource to clean up
358 *
359 * Should be used by resource manager backends to clean up the TTM resource
360 * objects before freeing the underlying structure. Makes sure the resource is
361 * removed from the LRU before destruction.
362 * Counterpart of ttm_resource_init().
363 */
364void ttm_resource_fini(struct ttm_resource_manager *man,
365 struct ttm_resource *res)
366{
367 struct ttm_device *bdev = man->bdev;
368
369 spin_lock(lock: &bdev->lru_lock);
370 list_del_init(entry: &res->lru.link);
371 man->usage -= res->size;
372 spin_unlock(lock: &bdev->lru_lock);
373}
374EXPORT_SYMBOL(ttm_resource_fini);
375
376int ttm_resource_alloc(struct ttm_buffer_object *bo,
377 const struct ttm_place *place,
378 struct ttm_resource **res_ptr,
379 struct dmem_cgroup_pool_state **ret_limit_pool)
380{
381 struct ttm_resource_manager *man =
382 ttm_manager_type(bdev: bo->bdev, mem_type: place->mem_type);
383 struct dmem_cgroup_pool_state *pool = NULL;
384 int ret;
385
386 if (man->cg) {
387 ret = dmem_cgroup_try_charge(region: man->cg, size: bo->base.size, ret_pool: &pool, ret_limit_pool);
388 if (ret)
389 return ret;
390 }
391
392 ret = man->func->alloc(man, bo, place, res_ptr);
393 if (ret) {
394 if (pool)
395 dmem_cgroup_uncharge(pool, size: bo->base.size);
396 return ret;
397 }
398
399 (*res_ptr)->css = pool;
400
401 spin_lock(lock: &bo->bdev->lru_lock);
402 ttm_resource_add_bulk_move(res: *res_ptr, bo);
403 spin_unlock(lock: &bo->bdev->lru_lock);
404 return 0;
405}
406EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc);
407
408void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
409{
410 struct ttm_resource_manager *man;
411 struct dmem_cgroup_pool_state *pool;
412
413 if (!*res)
414 return;
415
416 spin_lock(lock: &bo->bdev->lru_lock);
417 ttm_resource_del_bulk_move(res: *res, bo);
418 spin_unlock(lock: &bo->bdev->lru_lock);
419
420 pool = (*res)->css;
421 man = ttm_manager_type(bdev: bo->bdev, mem_type: (*res)->mem_type);
422 man->func->free(man, *res);
423 *res = NULL;
424 if (man->cg)
425 dmem_cgroup_uncharge(pool, size: bo->base.size);
426}
427EXPORT_SYMBOL(ttm_resource_free);
428
429/**
430 * ttm_resource_intersects - test for intersection
431 *
432 * @bdev: TTM device structure
433 * @res: The resource to test
434 * @place: The placement to test
435 * @size: How many bytes the new allocation needs.
436 *
437 * Test if @res intersects with @place and @size. Used for testing if evictions
438 * are valueable or not.
439 *
440 * Returns true if the res placement intersects with @place and @size.
441 */
442bool ttm_resource_intersects(struct ttm_device *bdev,
443 struct ttm_resource *res,
444 const struct ttm_place *place,
445 size_t size)
446{
447 struct ttm_resource_manager *man;
448
449 if (!res)
450 return false;
451
452 man = ttm_manager_type(bdev, mem_type: res->mem_type);
453 if (!place || !man->func->intersects)
454 return true;
455
456 return man->func->intersects(man, res, place, size);
457}
458
459/**
460 * ttm_resource_compatible - check if resource is compatible with placement
461 *
462 * @res: the resource to check
463 * @placement: the placement to check against
464 * @evicting: true if the caller is doing evictions
465 *
466 * Returns true if the placement is compatible.
467 */
468bool ttm_resource_compatible(struct ttm_resource *res,
469 struct ttm_placement *placement,
470 bool evicting)
471{
472 struct ttm_buffer_object *bo = res->bo;
473 struct ttm_device *bdev = bo->bdev;
474 unsigned i;
475
476 if (res->placement & TTM_PL_FLAG_TEMPORARY)
477 return false;
478
479 for (i = 0; i < placement->num_placement; i++) {
480 const struct ttm_place *place = &placement->placement[i];
481 struct ttm_resource_manager *man;
482
483 if (res->mem_type != place->mem_type)
484 continue;
485
486 if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED :
487 TTM_PL_FLAG_FALLBACK))
488 continue;
489
490 if (place->flags & TTM_PL_FLAG_CONTIGUOUS &&
491 !(res->placement & TTM_PL_FLAG_CONTIGUOUS))
492 continue;
493
494 man = ttm_manager_type(bdev, mem_type: res->mem_type);
495 if (man->func->compatible &&
496 !man->func->compatible(man, res, place, bo->base.size))
497 continue;
498
499 return true;
500 }
501 return false;
502}
503
504void ttm_resource_set_bo(struct ttm_resource *res,
505 struct ttm_buffer_object *bo)
506{
507 spin_lock(lock: &bo->bdev->lru_lock);
508 res->bo = bo;
509 spin_unlock(lock: &bo->bdev->lru_lock);
510}
511
512/**
513 * ttm_resource_manager_init
514 *
515 * @man: memory manager object to init
516 * @bdev: ttm device this manager belongs to
517 * @size: size of managed resources in arbitrary units
518 *
519 * Initialise core parts of a manager object.
520 */
521void ttm_resource_manager_init(struct ttm_resource_manager *man,
522 struct ttm_device *bdev,
523 uint64_t size)
524{
525 unsigned i;
526
527 man->bdev = bdev;
528 man->size = size;
529 man->usage = 0;
530
531 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
532 INIT_LIST_HEAD(list: &man->lru[i]);
533 spin_lock_init(&man->eviction_lock);
534 for (i = 0; i < TTM_NUM_MOVE_FENCES; i++)
535 man->eviction_fences[i] = NULL;
536}
537EXPORT_SYMBOL(ttm_resource_manager_init);
538
539/*
540 * ttm_resource_manager_evict_all
541 *
542 * @bdev - device to use
543 * @man - manager to use
544 *
545 * Evict all the objects out of a memory manager until it is empty.
546 * Part of memory manager cleanup sequence.
547 */
548int ttm_resource_manager_evict_all(struct ttm_device *bdev,
549 struct ttm_resource_manager *man)
550{
551 struct ttm_operation_ctx ctx = {
552 .interruptible = false,
553 .no_wait_gpu = false,
554 };
555 struct dma_fence *fence;
556 int ret, i;
557
558 do {
559 ret = ttm_bo_evict_first(bdev, man, ctx: &ctx);
560 cond_resched();
561 } while (!ret);
562
563 if (ret && ret != -ENOENT)
564 return ret;
565
566 ret = 0;
567
568 spin_lock(lock: &man->eviction_lock);
569 for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) {
570 fence = man->eviction_fences[i];
571 if (fence && !dma_fence_is_signaled(fence)) {
572 dma_fence_get(fence);
573 spin_unlock(lock: &man->eviction_lock);
574 ret = dma_fence_wait(fence, intr: false);
575 dma_fence_put(fence);
576 if (ret)
577 return ret;
578 spin_lock(lock: &man->eviction_lock);
579 }
580 }
581 spin_unlock(lock: &man->eviction_lock);
582
583 return ret;
584}
585EXPORT_SYMBOL(ttm_resource_manager_evict_all);
586
587/**
588 * ttm_resource_manager_usage
589 *
590 * @man: A memory manager object.
591 *
592 * Return how many resources are currently used.
593 */
594uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
595{
596 uint64_t usage;
597
598 if (WARN_ON_ONCE(!man->bdev))
599 return 0;
600
601 spin_lock(lock: &man->bdev->lru_lock);
602 usage = man->usage;
603 spin_unlock(lock: &man->bdev->lru_lock);
604 return usage;
605}
606EXPORT_SYMBOL(ttm_resource_manager_usage);
607
608/**
609 * ttm_resource_manager_debug
610 *
611 * @man: manager type to dump.
612 * @p: printer to use for debug.
613 */
614void ttm_resource_manager_debug(struct ttm_resource_manager *man,
615 struct drm_printer *p)
616{
617 drm_printf(p, f: " use_type: %d\n", man->use_type);
618 drm_printf(p, f: " use_tt: %d\n", man->use_tt);
619 drm_printf(p, f: " size: %llu\n", man->size);
620 drm_printf(p, f: " usage: %llu\n", ttm_resource_manager_usage(man));
621 if (man->func->debug)
622 man->func->debug(man, p);
623}
624EXPORT_SYMBOL(ttm_resource_manager_debug);
625
626static void
627ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor,
628 struct ttm_lru_item *next_lru)
629{
630 struct ttm_resource *next = ttm_lru_item_to_res(item: next_lru);
631 struct ttm_lru_bulk_move *bulk = NULL;
632 struct ttm_buffer_object *bo = next->bo;
633
634 lockdep_assert_held(&cursor->man->bdev->lru_lock);
635 bulk = bo->bulk_move;
636
637 if (cursor->bulk != bulk) {
638 if (bulk) {
639 list_move_tail(list: &cursor->bulk_link, head: &bulk->cursor_list);
640 cursor->mem_type = next->mem_type;
641 } else {
642 list_del_init(entry: &cursor->bulk_link);
643 }
644 cursor->bulk = bulk;
645 }
646}
647
648/**
649 * ttm_resource_manager_first() - Start iterating over the resources
650 * of a resource manager
651 * @cursor: cursor to record the position
652 *
653 * Initializes the cursor and starts iterating. When done iterating,
654 * the caller must explicitly call ttm_resource_cursor_fini().
655 *
656 * Return: The first resource from the resource manager.
657 */
658struct ttm_resource *
659ttm_resource_manager_first(struct ttm_resource_cursor *cursor)
660{
661 struct ttm_resource_manager *man = cursor->man;
662
663 if (WARN_ON_ONCE(!man))
664 return NULL;
665
666 lockdep_assert_held(&man->bdev->lru_lock);
667
668 list_move(list: &cursor->hitch.link, head: &man->lru[cursor->priority]);
669 return ttm_resource_manager_next(cursor);
670}
671
672/**
673 * ttm_resource_manager_next() - Continue iterating over the resource manager
674 * resources
675 * @cursor: cursor to record the position
676 *
677 * Return: the next resource from the resource manager.
678 */
679struct ttm_resource *
680ttm_resource_manager_next(struct ttm_resource_cursor *cursor)
681{
682 struct ttm_resource_manager *man = cursor->man;
683 struct ttm_lru_item *lru;
684
685 lockdep_assert_held(&man->bdev->lru_lock);
686
687 for (;;) {
688 lru = &cursor->hitch;
689 list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) {
690 if (ttm_lru_item_is_res(item: lru)) {
691 ttm_resource_cursor_check_bulk(cursor, next_lru: lru);
692 list_move(list: &cursor->hitch.link, head: &lru->link);
693 return ttm_lru_item_to_res(item: lru);
694 }
695 }
696
697 if (++cursor->priority >= TTM_MAX_BO_PRIORITY)
698 break;
699
700 list_move(list: &cursor->hitch.link, head: &man->lru[cursor->priority]);
701 ttm_resource_cursor_clear_bulk(cursor);
702 }
703
704 return NULL;
705}
706
707/**
708 * ttm_lru_first_res_or_null() - Return the first resource on an lru list
709 * @head: The list head of the lru list.
710 *
711 * Return: Pointer to the first resource on the lru list or NULL if
712 * there is none.
713 */
714struct ttm_resource *ttm_lru_first_res_or_null(struct list_head *head)
715{
716 struct ttm_lru_item *lru;
717
718 list_for_each_entry(lru, head, link) {
719 if (ttm_lru_item_is_res(item: lru))
720 return ttm_lru_item_to_res(item: lru);
721 }
722
723 return NULL;
724}
725
726static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
727 struct iosys_map *dmap,
728 pgoff_t i)
729{
730 struct ttm_kmap_iter_iomap *iter_io =
731 container_of(iter, typeof(*iter_io), base);
732 void __iomem *addr;
733
734retry:
735 while (i >= iter_io->cache.end) {
736 iter_io->cache.sg = iter_io->cache.sg ?
737 sg_next(sg: iter_io->cache.sg) : iter_io->st->sgl;
738 iter_io->cache.i = iter_io->cache.end;
739 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
740 PAGE_SHIFT;
741 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
742 iter_io->start;
743 }
744
745 if (i < iter_io->cache.i) {
746 iter_io->cache.end = 0;
747 iter_io->cache.sg = NULL;
748 goto retry;
749 }
750
751 addr = io_mapping_map_local_wc(mapping: iter_io->iomap, offset: iter_io->cache.offs +
752 (((resource_size_t)i - iter_io->cache.i)
753 << PAGE_SHIFT));
754 iosys_map_set_vaddr_iomem(map: dmap, vaddr_iomem: addr);
755}
756
757static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
758 struct iosys_map *map)
759{
760 io_mapping_unmap_local(vaddr: map->vaddr_iomem);
761}
762
763static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
764 .map_local = ttm_kmap_iter_iomap_map_local,
765 .unmap_local = ttm_kmap_iter_iomap_unmap_local,
766 .maps_tt = false,
767};
768
769/**
770 * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
771 * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
772 * @iomap: The struct io_mapping representing the underlying linear io_memory.
773 * @st: sg_table into @iomap, representing the memory of the struct
774 * ttm_resource.
775 * @start: Offset that needs to be subtracted from @st to make
776 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
777 *
778 * Return: Pointer to the embedded struct ttm_kmap_iter.
779 */
780struct ttm_kmap_iter *
781ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
782 struct io_mapping *iomap,
783 struct sg_table *st,
784 resource_size_t start)
785{
786 iter_io->base.ops = &ttm_kmap_iter_io_ops;
787 iter_io->iomap = iomap;
788 iter_io->st = st;
789 iter_io->start = start;
790 memset(&iter_io->cache, 0, sizeof(iter_io->cache));
791
792 return &iter_io->base;
793}
794EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
795
796/**
797 * DOC: Linear io iterator
798 *
799 * This code should die in the not too near future. Best would be if we could
800 * make io-mapping use memremap for all io memory, and have memremap
801 * implement a kmap_local functionality. We could then strip a huge amount of
802 * code. These linear io iterators are implemented to mimic old functionality,
803 * and they don't use kmap_local semantics at all internally. Rather ioremap or
804 * friends, and at least on 32-bit they add global TLB flushes and points
805 * of failure.
806 */
807
808static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
809 struct iosys_map *dmap,
810 pgoff_t i)
811{
812 struct ttm_kmap_iter_linear_io *iter_io =
813 container_of(iter, typeof(*iter_io), base);
814
815 *dmap = iter_io->dmap;
816 iosys_map_incr(map: dmap, incr: i * PAGE_SIZE);
817}
818
819static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
820 .map_local = ttm_kmap_iter_linear_io_map_local,
821 .maps_tt = false,
822};
823
824/**
825 * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
826 * @iter_io: The iterator to initialize
827 * @bdev: The TTM device
828 * @mem: The ttm resource representing the iomap.
829 *
830 * This function is for internal TTM use only. It sets up a memcpy kmap iterator
831 * pointing at a linear chunk of io memory.
832 *
833 * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
834 * failure.
835 */
836struct ttm_kmap_iter *
837ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
838 struct ttm_device *bdev,
839 struct ttm_resource *mem)
840{
841 int ret;
842
843 ret = ttm_mem_io_reserve(bdev, mem);
844 if (ret)
845 goto out_err;
846 if (!mem->bus.is_iomem) {
847 ret = -EINVAL;
848 goto out_io_free;
849 }
850
851 if (mem->bus.addr) {
852 iosys_map_set_vaddr(map: &iter_io->dmap, vaddr: mem->bus.addr);
853 iter_io->needs_unmap = false;
854 } else {
855 iter_io->needs_unmap = true;
856 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
857 if (mem->bus.caching == ttm_write_combined)
858 iosys_map_set_vaddr_iomem(map: &iter_io->dmap,
859 ioremap_wc(offset: mem->bus.offset,
860 size: mem->size));
861 else if (mem->bus.caching == ttm_cached)
862 iosys_map_set_vaddr(map: &iter_io->dmap,
863 vaddr: memremap(offset: mem->bus.offset, size: mem->size,
864 flags: MEMREMAP_WB |
865 MEMREMAP_WT |
866 MEMREMAP_WC));
867
868 /* If uncached requested or if mapping cached or wc failed */
869 if (iosys_map_is_null(map: &iter_io->dmap))
870 iosys_map_set_vaddr_iomem(map: &iter_io->dmap,
871 ioremap(offset: mem->bus.offset,
872 size: mem->size));
873
874 if (iosys_map_is_null(map: &iter_io->dmap)) {
875 ret = -ENOMEM;
876 goto out_io_free;
877 }
878 }
879
880 iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
881 return &iter_io->base;
882
883out_io_free:
884 ttm_mem_io_free(bdev, mem);
885out_err:
886 return ERR_PTR(error: ret);
887}
888
889/**
890 * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
891 * @iter_io: The iterator to initialize
892 * @bdev: The TTM device
893 * @mem: The ttm resource representing the iomap.
894 *
895 * This function is for internal TTM use only. It cleans up a memcpy kmap
896 * iterator initialized by ttm_kmap_iter_linear_io_init.
897 */
898void
899ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
900 struct ttm_device *bdev,
901 struct ttm_resource *mem)
902{
903 if (iter_io->needs_unmap && iosys_map_is_set(map: &iter_io->dmap)) {
904 if (iter_io->dmap.is_iomem)
905 iounmap(addr: iter_io->dmap.vaddr_iomem);
906 else
907 memunmap(addr: iter_io->dmap.vaddr);
908 }
909
910 ttm_mem_io_free(bdev, mem);
911}
912
913#if defined(CONFIG_DEBUG_FS)
914
915static int ttm_resource_manager_show(struct seq_file *m, void *unused)
916{
917 struct ttm_resource_manager *man =
918 (struct ttm_resource_manager *)m->private;
919 struct drm_printer p = drm_seq_file_printer(f: m);
920 ttm_resource_manager_debug(man, &p);
921 return 0;
922}
923DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
924
925#endif
926
927/**
928 * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
929 * resource manager.
930 * @man: The TTM resource manager for which the debugfs stats file be creates
931 * @parent: debugfs directory in which the file will reside
932 * @name: The filename to create.
933 *
934 * This function setups up a debugfs file that can be used to look
935 * at debug statistics of the specified ttm_resource_manager.
936 */
937void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
938 struct dentry * parent,
939 const char *name)
940{
941#if defined(CONFIG_DEBUG_FS)
942 debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
943#endif
944}
945EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
946

source code of linux/drivers/gpu/drm/ttm/ttm_resource.c