1/*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25#ifndef _TTM_RESOURCE_H_
26#define _TTM_RESOURCE_H_
27
28#include <linux/types.h>
29#include <linux/list.h>
30#include <linux/mutex.h>
31#include <linux/iosys-map.h>
32#include <linux/dma-fence.h>
33
34#include <drm/ttm/ttm_caching.h>
35#include <drm/ttm/ttm_kmap_iter.h>
36
37#define TTM_MAX_BO_PRIORITY 4U
38#define TTM_NUM_MEM_TYPES 9
39
40struct dentry;
41struct dmem_cgroup_device;
42struct drm_printer;
43struct ttm_device;
44struct ttm_resource_manager;
45struct ttm_resource;
46struct ttm_place;
47struct ttm_buffer_object;
48struct ttm_placement;
49struct iosys_map;
50struct io_mapping;
51struct sg_table;
52struct scatterlist;
53
54/**
55 * define TTM_NUM_MOVE_FENCES - How many entities can be used for evictions
56 *
57 * Pipelined evictions can be spread on multiple entities. This
58 * is the max number of entities that can be used by the driver
59 * for that purpose.
60 */
61#define TTM_NUM_MOVE_FENCES 8
62
63/**
64 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses
65 */
66enum ttm_lru_item_type {
67 /** @TTM_LRU_RESOURCE: The resource subclass */
68 TTM_LRU_RESOURCE,
69 /** @TTM_LRU_HITCH: The iterator hitch subclass */
70 TTM_LRU_HITCH
71};
72
73/**
74 * struct ttm_lru_item - The TTM lru list node base class
75 * @link: The list link
76 * @type: The subclass type
77 */
78struct ttm_lru_item {
79 struct list_head link;
80 enum ttm_lru_item_type type;
81};
82
83/**
84 * ttm_lru_item_init() - initialize a struct ttm_lru_item
85 * @item: The item to initialize
86 * @type: The subclass type
87 */
88static inline void ttm_lru_item_init(struct ttm_lru_item *item,
89 enum ttm_lru_item_type type)
90{
91 item->type = type;
92 INIT_LIST_HEAD(list: &item->link);
93}
94
95static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item)
96{
97 return item->type == TTM_LRU_RESOURCE;
98}
99
100struct ttm_resource_manager_func {
101 /**
102 * struct ttm_resource_manager_func member alloc
103 *
104 * @man: Pointer to a memory type manager.
105 * @bo: Pointer to the buffer object we're allocating space for.
106 * @place: Placement details.
107 * @res: Resulting pointer to the ttm_resource.
108 *
109 * This function should allocate space in the memory type managed
110 * by @man. Placement details if applicable are given by @place. If
111 * successful, a filled in ttm_resource object should be returned in
112 * @res. @res::start should be set to a value identifying the beginning
113 * of the range allocated, and the function should return zero.
114 * If the manager can't fulfill the request -ENOSPC should be returned.
115 * If a system error occurred, preventing the request to be fulfilled,
116 * the function should return a negative error code.
117 *
118 * This function may not be called from within atomic context and needs
119 * to take care of its own locking to protect any data structures
120 * managing the space.
121 */
122 int (*alloc)(struct ttm_resource_manager *man,
123 struct ttm_buffer_object *bo,
124 const struct ttm_place *place,
125 struct ttm_resource **res);
126
127 /**
128 * struct ttm_resource_manager_func member free
129 *
130 * @man: Pointer to a memory type manager.
131 * @res: Pointer to a struct ttm_resource to be freed.
132 *
133 * This function frees memory type resources previously allocated.
134 * May not be called from within atomic context.
135 */
136 void (*free)(struct ttm_resource_manager *man,
137 struct ttm_resource *res);
138
139 /**
140 * struct ttm_resource_manager_func member intersects
141 *
142 * @man: Pointer to a memory type manager.
143 * @res: Pointer to a struct ttm_resource to be checked.
144 * @place: Placement to check against.
145 * @size: Size of the check.
146 *
147 * Test if @res intersects with @place + @size. Used to judge if
148 * evictions are valueable or not.
149 */
150 bool (*intersects)(struct ttm_resource_manager *man,
151 struct ttm_resource *res,
152 const struct ttm_place *place,
153 size_t size);
154
155 /**
156 * struct ttm_resource_manager_func member compatible
157 *
158 * @man: Pointer to a memory type manager.
159 * @res: Pointer to a struct ttm_resource to be checked.
160 * @place: Placement to check against.
161 * @size: Size of the check.
162 *
163 * Test if @res compatible with @place + @size. Used to check of
164 * the need to move the backing store or not.
165 */
166 bool (*compatible)(struct ttm_resource_manager *man,
167 struct ttm_resource *res,
168 const struct ttm_place *place,
169 size_t size);
170
171 /**
172 * struct ttm_resource_manager_func member debug
173 *
174 * @man: Pointer to a memory type manager.
175 * @printer: Prefix to be used in printout to identify the caller.
176 *
177 * This function is called to print out the state of the memory
178 * type manager to aid debugging of out-of-memory conditions.
179 * It may not be called from within atomic context.
180 */
181 void (*debug)(struct ttm_resource_manager *man,
182 struct drm_printer *printer);
183};
184
185/**
186 * struct ttm_resource_manager
187 *
188 * @use_type: The memory type is enabled.
189 * @use_tt: If a TT object should be used for the backing store.
190 * @size: Size of the managed region.
191 * @bdev: ttm device this manager belongs to
192 * @func: structure pointer implementing the range manager. See above
193 * @eviction_lock: lock for eviction fences
194 * @eviction_fences: The fences of the last pipelined move operation.
195 * @lru: The lru list for this memory type.
196 *
197 * This structure is used to identify and manage memory types for a device.
198 */
199struct ttm_resource_manager {
200 /*
201 * No protection. Constant from start.
202 */
203 bool use_type;
204 bool use_tt;
205 struct ttm_device *bdev;
206 uint64_t size;
207 const struct ttm_resource_manager_func *func;
208
209 /* This is very similar to a dma_resv object, but locking rules make
210 * it difficult to use one in this context.
211 */
212 spinlock_t eviction_lock;
213 struct dma_fence *eviction_fences[TTM_NUM_MOVE_FENCES];
214
215 /*
216 * Protected by the bdev->lru_lock.
217 */
218 struct list_head lru[TTM_MAX_BO_PRIORITY];
219
220 /**
221 * @usage: How much of the resources are used, protected by the
222 * bdev->lru_lock.
223 */
224 uint64_t usage;
225
226 /**
227 * @cg: &dmem_cgroup_region used for memory accounting, if not NULL.
228 */
229 struct dmem_cgroup_region *cg;
230};
231
232/**
233 * struct ttm_bus_placement
234 *
235 * @addr: mapped virtual address
236 * @offset: physical addr
237 * @is_iomem: is this io memory ?
238 * @caching: See enum ttm_caching
239 *
240 * Structure indicating the bus placement of an object.
241 */
242struct ttm_bus_placement {
243 void *addr;
244 phys_addr_t offset;
245 bool is_iomem;
246 enum ttm_caching caching;
247};
248
249/**
250 * struct ttm_resource
251 *
252 * @start: Start of the allocation.
253 * @size: Actual size of resource in bytes.
254 * @mem_type: Resource type of the allocation.
255 * @placement: Placement flags.
256 * @bus: Placement on io bus accessible to the CPU
257 * @bo: weak reference to the BO, protected by ttm_device::lru_lock
258 * @css: cgroup state this resource is charged to
259 *
260 * Structure indicating the placement and space resources used by a
261 * buffer object.
262 */
263struct ttm_resource {
264 unsigned long start;
265 size_t size;
266 uint32_t mem_type;
267 uint32_t placement;
268 struct ttm_bus_placement bus;
269 struct ttm_buffer_object *bo;
270
271 struct dmem_cgroup_pool_state *css;
272
273 /**
274 * @lru: Least recently used list, see &ttm_resource_manager.lru
275 */
276 struct ttm_lru_item lru;
277};
278
279/**
280 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource
281 * @item: The struct ttm_lru_item to downcast
282 *
283 * Return: Pointer to the embedding struct ttm_resource
284 */
285static inline struct ttm_resource *
286ttm_lru_item_to_res(struct ttm_lru_item *item)
287{
288 return container_of(item, struct ttm_resource, lru);
289}
290
291/**
292 * struct ttm_lru_bulk_move_pos
293 *
294 * @first: first res in the bulk move range
295 * @last: last res in the bulk move range
296 *
297 * Range of resources for a lru bulk move.
298 */
299struct ttm_lru_bulk_move_pos {
300 struct ttm_resource *first;
301 struct ttm_resource *last;
302};
303
304/**
305 * struct ttm_lru_bulk_move
306 * @pos: first/last lru entry for resources in the each domain/priority
307 * @cursor_list: The list of cursors currently traversing any of
308 * the sublists of @pos. Protected by the ttm device's lru_lock.
309 *
310 * Container for the current bulk move state. Should be used with
311 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().
312 * All BOs in a bulk_move structure need to share the same reservation object to
313 * ensure that the bulk as a whole is locked for eviction even if only one BO of
314 * the bulk is evicted.
315 */
316struct ttm_lru_bulk_move {
317 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];
318 struct list_head cursor_list;
319};
320
321/**
322 * struct ttm_resource_cursor
323 * @man: The resource manager currently being iterated over
324 * @hitch: A hitch list node inserted before the next resource
325 * to iterate over.
326 * @bulk_link: A list link for the list of cursors traversing the
327 * bulk sublist of @bulk. Protected by the ttm device's lru_lock.
328 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is
329 * inserted to. NULL if none. Never dereference this pointer since
330 * the struct ttm_lru_bulk_move object pointed to might have been
331 * freed. The pointer is only for comparison.
332 * @mem_type: The memory type of the LRU list being traversed.
333 * This field is valid iff @bulk != NULL.
334 * @priority: the current priority
335 *
336 * Cursor to iterate over the resources in a manager.
337 */
338struct ttm_resource_cursor {
339 struct ttm_resource_manager *man;
340 struct ttm_lru_item hitch;
341 struct list_head bulk_link;
342 struct ttm_lru_bulk_move *bulk;
343 unsigned int mem_type;
344 unsigned int priority;
345};
346
347void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor,
348 struct ttm_resource_manager *man);
349
350void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor);
351
352/**
353 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
354 * struct sg_table backed struct ttm_resource.
355 * @base: Embedded struct ttm_kmap_iter providing the usage interface.
356 * @iomap: struct io_mapping representing the underlying linear io_memory.
357 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
358 * @start: Offset that needs to be subtracted from @st to make
359 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
360 * @cache: Scatterlist traversal cache for fast lookups.
361 * @cache.sg: Pointer to the currently cached scatterlist segment.
362 * @cache.i: First index of @sg. PAGE_SIZE granularity.
363 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
364 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
365 */
366struct ttm_kmap_iter_iomap {
367 struct ttm_kmap_iter base;
368 struct io_mapping *iomap;
369 struct sg_table *st;
370 resource_size_t start;
371 struct {
372 struct scatterlist *sg;
373 pgoff_t i;
374 pgoff_t end;
375 pgoff_t offs;
376 } cache;
377};
378
379/**
380 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
381 * @base: The base iterator
382 * @dmap: Points to the starting address of the region
383 * @needs_unmap: Whether we need to unmap on fini
384 */
385struct ttm_kmap_iter_linear_io {
386 struct ttm_kmap_iter base;
387 struct iosys_map dmap;
388 bool needs_unmap;
389};
390
391/**
392 * ttm_resource_manager_set_used
393 *
394 * @man: A memory manager object.
395 * @used: usage state to set.
396 *
397 * Set the manager in use flag. If disabled the manager is no longer
398 * used for object placement.
399 */
400static inline void
401ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used)
402{
403 int i;
404
405 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++)
406 WARN_ON(!list_empty(&man->lru[i]));
407 man->use_type = used;
408}
409
410/**
411 * ttm_resource_manager_used
412 *
413 * @man: Manager to get used state for
414 *
415 * Get the in use flag for a manager.
416 * Returns:
417 * true is used, false if not.
418 */
419static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man)
420{
421 return man->use_type;
422}
423
424/**
425 * ttm_resource_manager_cleanup
426 *
427 * @man: A memory manager object.
428 *
429 * Cleanup the move fences from the memory manager object.
430 */
431static inline void
432ttm_resource_manager_cleanup(struct ttm_resource_manager *man)
433{
434 int i;
435
436 for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) {
437 dma_fence_put(fence: man->eviction_fences[i]);
438 man->eviction_fences[i] = NULL;
439 }
440}
441
442void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);
443void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);
444void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
445 struct ttm_lru_bulk_move *bulk);
446
447void ttm_resource_add_bulk_move(struct ttm_resource *res,
448 struct ttm_buffer_object *bo);
449void ttm_resource_del_bulk_move(struct ttm_resource *res,
450 struct ttm_buffer_object *bo);
451void ttm_resource_move_to_lru_tail(struct ttm_resource *res);
452
453void ttm_resource_init(struct ttm_buffer_object *bo,
454 const struct ttm_place *place,
455 struct ttm_resource *res);
456void ttm_resource_fini(struct ttm_resource_manager *man,
457 struct ttm_resource *res);
458
459int ttm_resource_alloc(struct ttm_buffer_object *bo,
460 const struct ttm_place *place,
461 struct ttm_resource **res,
462 struct dmem_cgroup_pool_state **ret_limit_pool);
463void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);
464bool ttm_resource_intersects(struct ttm_device *bdev,
465 struct ttm_resource *res,
466 const struct ttm_place *place,
467 size_t size);
468bool ttm_resource_compatible(struct ttm_resource *res,
469 struct ttm_placement *placement,
470 bool evicting);
471void ttm_resource_set_bo(struct ttm_resource *res,
472 struct ttm_buffer_object *bo);
473
474void ttm_resource_manager_init(struct ttm_resource_manager *man,
475 struct ttm_device *bdev,
476 uint64_t size);
477
478int ttm_resource_manager_evict_all(struct ttm_device *bdev,
479 struct ttm_resource_manager *man);
480
481uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man);
482void ttm_resource_manager_debug(struct ttm_resource_manager *man,
483 struct drm_printer *p);
484
485struct ttm_resource *
486ttm_resource_manager_first(struct ttm_resource_cursor *cursor);
487struct ttm_resource *
488ttm_resource_manager_next(struct ttm_resource_cursor *cursor);
489
490struct ttm_resource *
491ttm_lru_first_res_or_null(struct list_head *head);
492
493/**
494 * ttm_resource_manager_for_each_res - iterate over all resources
495 * @cursor: struct ttm_resource_cursor for the current position
496 * @res: the current resource
497 *
498 * Iterate over all the evictable resources in a resource manager.
499 */
500#define ttm_resource_manager_for_each_res(cursor, res) \
501 for (res = ttm_resource_manager_first(cursor); res; \
502 res = ttm_resource_manager_next(cursor))
503
504struct ttm_kmap_iter *
505ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
506 struct io_mapping *iomap,
507 struct sg_table *st,
508 resource_size_t start);
509
510struct ttm_kmap_iter_linear_io;
511
512struct ttm_kmap_iter *
513ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
514 struct ttm_device *bdev,
515 struct ttm_resource *mem);
516
517void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
518 struct ttm_device *bdev,
519 struct ttm_resource *mem);
520
521void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
522 struct dentry * parent,
523 const char *name);
524#endif
525

source code of linux/include/drm/ttm/ttm_resource.h