1/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
2
3#ifndef __DRM_GPUVM_H__
4#define __DRM_GPUVM_H__
5
6/*
7 * Copyright (c) 2022 Red Hat.
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
15 *
16 * The above copyright notice and this permission notice shall be included in
17 * all copies or substantial portions of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
23 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
24 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
25 * OTHER DEALINGS IN THE SOFTWARE.
26 */
27
28#include <linux/dma-resv.h>
29#include <linux/list.h>
30#include <linux/llist.h>
31#include <linux/rbtree.h>
32#include <linux/types.h>
33
34#include <drm/drm_device.h>
35#include <drm/drm_gem.h>
36#include <drm/drm_exec.h>
37
38struct drm_gpuvm;
39struct drm_gpuvm_bo;
40struct drm_gpuvm_ops;
41
42/**
43 * enum drm_gpuva_flags - flags for struct drm_gpuva
44 */
45enum drm_gpuva_flags {
46 /**
47 * @DRM_GPUVA_INVALIDATED:
48 *
49 * Flag indicating that the &drm_gpuva's backing GEM is invalidated.
50 */
51 DRM_GPUVA_INVALIDATED = (1 << 0),
52
53 /**
54 * @DRM_GPUVA_SPARSE:
55 *
56 * Flag indicating that the &drm_gpuva is a sparse mapping.
57 */
58 DRM_GPUVA_SPARSE = (1 << 1),
59
60 /**
61 * @DRM_GPUVA_USERBITS: user defined bits
62 */
63 DRM_GPUVA_USERBITS = (1 << 2),
64};
65
66/**
67 * struct drm_gpuva - structure to track a GPU VA mapping
68 *
69 * This structure represents a GPU VA mapping and is associated with a
70 * &drm_gpuvm.
71 *
72 * Typically, this structure is embedded in bigger driver structures.
73 */
74struct drm_gpuva {
75 /**
76 * @vm: the &drm_gpuvm this object is associated with
77 */
78 struct drm_gpuvm *vm;
79
80 /**
81 * @vm_bo: the &drm_gpuvm_bo abstraction for the mapped
82 * &drm_gem_object
83 */
84 struct drm_gpuvm_bo *vm_bo;
85
86 /**
87 * @flags: the &drm_gpuva_flags for this mapping
88 */
89 enum drm_gpuva_flags flags;
90
91 /**
92 * @va: structure containing the address and range of the &drm_gpuva
93 */
94 struct {
95 /**
96 * @va.addr: the start address
97 */
98 u64 addr;
99
100 /*
101 * @range: the range
102 */
103 u64 range;
104 } va;
105
106 /**
107 * @gem: structure containing the &drm_gem_object and its offset
108 */
109 struct {
110 /**
111 * @gem.offset: the offset within the &drm_gem_object
112 */
113 u64 offset;
114
115 /**
116 * @gem.obj: the mapped &drm_gem_object
117 */
118 struct drm_gem_object *obj;
119
120 /**
121 * @gem.entry: the &list_head to attach this object to a &drm_gpuvm_bo
122 */
123 struct list_head entry;
124 } gem;
125
126 /**
127 * @rb: structure containing data to store &drm_gpuvas in a rb-tree
128 */
129 struct {
130 /**
131 * @rb.node: the rb-tree node
132 */
133 struct rb_node node;
134
135 /**
136 * @rb.entry: The &list_head to additionally connect &drm_gpuvas
137 * in the same order they appear in the interval tree. This is
138 * useful to keep iterating &drm_gpuvas from a start node found
139 * through the rb-tree while doing modifications on the rb-tree
140 * itself.
141 */
142 struct list_head entry;
143
144 /**
145 * @rb.__subtree_last: needed by the interval tree, holding last-in-subtree
146 */
147 u64 __subtree_last;
148 } rb;
149};
150
151int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va);
152void drm_gpuva_remove(struct drm_gpuva *va);
153
154void drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo);
155void drm_gpuva_unlink(struct drm_gpuva *va);
156void drm_gpuva_unlink_defer(struct drm_gpuva *va);
157
158struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm,
159 u64 addr, u64 range);
160struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuvm *gpuvm,
161 u64 addr, u64 range);
162struct drm_gpuva *drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start);
163struct drm_gpuva *drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end);
164
165/**
166 * drm_gpuva_invalidate() - sets whether the backing GEM of this &drm_gpuva is
167 * invalidated
168 * @va: the &drm_gpuva to set the invalidate flag for
169 * @invalidate: indicates whether the &drm_gpuva is invalidated
170 */
171static inline void drm_gpuva_invalidate(struct drm_gpuva *va, bool invalidate)
172{
173 if (invalidate)
174 va->flags |= DRM_GPUVA_INVALIDATED;
175 else
176 va->flags &= ~DRM_GPUVA_INVALIDATED;
177}
178
179/**
180 * drm_gpuva_invalidated() - indicates whether the backing BO of this &drm_gpuva
181 * is invalidated
182 * @va: the &drm_gpuva to check
183 *
184 * Returns: %true if the GPU VA is invalidated, %false otherwise
185 */
186static inline bool drm_gpuva_invalidated(struct drm_gpuva *va)
187{
188 return va->flags & DRM_GPUVA_INVALIDATED;
189}
190
191/**
192 * enum drm_gpuvm_flags - flags for struct drm_gpuvm
193 */
194enum drm_gpuvm_flags {
195 /**
196 * @DRM_GPUVM_RESV_PROTECTED: GPUVM is protected externally by the
197 * GPUVM's &dma_resv lock
198 */
199 DRM_GPUVM_RESV_PROTECTED = BIT(0),
200
201 /**
202 * @DRM_GPUVM_IMMEDIATE_MODE: use the locking scheme for GEMs designed
203 * for modifying the GPUVM during the fence signalling path
204 *
205 * When set, gpuva.lock is used to protect gpuva.list in all GEM
206 * objects associated with this GPUVM. Otherwise, the GEMs dma-resv is
207 * used.
208 */
209 DRM_GPUVM_IMMEDIATE_MODE = BIT(1),
210
211 /**
212 * @DRM_GPUVM_USERBITS: user defined bits
213 */
214 DRM_GPUVM_USERBITS = BIT(2),
215};
216
217/**
218 * struct drm_gpuvm - DRM GPU VA Manager
219 *
220 * The DRM GPU VA Manager keeps track of a GPU's virtual address space by using
221 * &maple_tree structures. Typically, this structure is embedded in bigger
222 * driver structures.
223 *
224 * Drivers can pass addresses and ranges in an arbitrary unit, e.g. bytes or
225 * pages.
226 *
227 * There should be one manager instance per GPU virtual address space.
228 */
229struct drm_gpuvm {
230 /**
231 * @name: the name of the DRM GPU VA space
232 */
233 const char *name;
234
235 /**
236 * @flags: the &drm_gpuvm_flags of this GPUVM
237 */
238 enum drm_gpuvm_flags flags;
239
240 /**
241 * @drm: the &drm_device this VM lives in
242 */
243 struct drm_device *drm;
244
245 /**
246 * @mm_start: start of the VA space
247 */
248 u64 mm_start;
249
250 /**
251 * @mm_range: length of the VA space
252 */
253 u64 mm_range;
254
255 /**
256 * @rb: structures to track &drm_gpuva entries
257 */
258 struct {
259 /**
260 * @rb.tree: the rb-tree to track GPU VA mappings
261 */
262 struct rb_root_cached tree;
263
264 /**
265 * @rb.list: the &list_head to track GPU VA mappings
266 */
267 struct list_head list;
268 } rb;
269
270 /**
271 * @kref: reference count of this object
272 */
273 struct kref kref;
274
275 /**
276 * @kernel_alloc_node:
277 *
278 * &drm_gpuva representing the address space cutout reserved for
279 * the kernel
280 */
281 struct drm_gpuva kernel_alloc_node;
282
283 /**
284 * @ops: &drm_gpuvm_ops providing the split/merge steps to drivers
285 */
286 const struct drm_gpuvm_ops *ops;
287
288 /**
289 * @r_obj: Resv GEM object; representing the GPUVM's common &dma_resv.
290 */
291 struct drm_gem_object *r_obj;
292
293 /**
294 * @extobj: structure holding the extobj list
295 */
296 struct {
297 /**
298 * @extobj.list: &list_head storing &drm_gpuvm_bos serving as
299 * external object
300 */
301 struct list_head list;
302
303 /**
304 * @extobj.local_list: pointer to the local list temporarily
305 * storing entries from the external object list
306 */
307 struct list_head *local_list;
308
309 /**
310 * @extobj.lock: spinlock to protect the extobj list
311 */
312 spinlock_t lock;
313 } extobj;
314
315 /**
316 * @evict: structure holding the evict list and evict list lock
317 */
318 struct {
319 /**
320 * @evict.list: &list_head storing &drm_gpuvm_bos currently
321 * being evicted
322 */
323 struct list_head list;
324
325 /**
326 * @evict.local_list: pointer to the local list temporarily
327 * storing entries from the evicted object list
328 */
329 struct list_head *local_list;
330
331 /**
332 * @evict.lock: spinlock to protect the evict list
333 */
334 spinlock_t lock;
335 } evict;
336
337 /**
338 * @bo_defer: structure holding vm_bos that need to be destroyed
339 */
340 struct llist_head bo_defer;
341};
342
343void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
344 enum drm_gpuvm_flags flags,
345 struct drm_device *drm,
346 struct drm_gem_object *r_obj,
347 u64 start_offset, u64 range,
348 u64 reserve_offset, u64 reserve_range,
349 const struct drm_gpuvm_ops *ops);
350
351/**
352 * drm_gpuvm_get() - acquire a struct drm_gpuvm reference
353 * @gpuvm: the &drm_gpuvm to acquire the reference of
354 *
355 * This function acquires an additional reference to @gpuvm. It is illegal to
356 * call this without already holding a reference. No locks required.
357 *
358 * Returns: the &struct drm_gpuvm pointer
359 */
360static inline struct drm_gpuvm *
361drm_gpuvm_get(struct drm_gpuvm *gpuvm)
362{
363 kref_get(kref: &gpuvm->kref);
364
365 return gpuvm;
366}
367
368void drm_gpuvm_put(struct drm_gpuvm *gpuvm);
369
370bool drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range);
371bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range);
372
373struct drm_gem_object *
374drm_gpuvm_resv_object_alloc(struct drm_device *drm);
375
376/**
377 * drm_gpuvm_resv_protected() - indicates whether &DRM_GPUVM_RESV_PROTECTED is
378 * set
379 * @gpuvm: the &drm_gpuvm
380 *
381 * Returns: true if &DRM_GPUVM_RESV_PROTECTED is set, false otherwise.
382 */
383static inline bool
384drm_gpuvm_resv_protected(struct drm_gpuvm *gpuvm)
385{
386 return gpuvm->flags & DRM_GPUVM_RESV_PROTECTED;
387}
388
389/**
390 * drm_gpuvm_immediate_mode() - indicates whether &DRM_GPUVM_IMMEDIATE_MODE is
391 * set
392 * @gpuvm: the &drm_gpuvm
393 *
394 * Returns: true if &DRM_GPUVM_IMMEDIATE_MODE is set, false otherwise.
395 */
396static inline bool
397drm_gpuvm_immediate_mode(struct drm_gpuvm *gpuvm)
398{
399 return gpuvm->flags & DRM_GPUVM_IMMEDIATE_MODE;
400}
401
402/**
403 * drm_gpuvm_resv() - returns the &drm_gpuvm's &dma_resv
404 * @gpuvm__: the &drm_gpuvm
405 *
406 * Returns: a pointer to the &drm_gpuvm's shared &dma_resv
407 */
408#define drm_gpuvm_resv(gpuvm__) ((gpuvm__)->r_obj->resv)
409
410/**
411 * drm_gpuvm_resv_obj() - returns the &drm_gem_object holding the &drm_gpuvm's
412 * &dma_resv
413 * @gpuvm__: the &drm_gpuvm
414 *
415 * Returns: a pointer to the &drm_gem_object holding the &drm_gpuvm's shared
416 * &dma_resv
417 */
418#define drm_gpuvm_resv_obj(gpuvm__) ((gpuvm__)->r_obj)
419
420#define drm_gpuvm_resv_held(gpuvm__) \
421 dma_resv_held(drm_gpuvm_resv(gpuvm__))
422
423#define drm_gpuvm_resv_assert_held(gpuvm__) \
424 dma_resv_assert_held(drm_gpuvm_resv(gpuvm__))
425
426#define drm_gpuvm_resv_held(gpuvm__) \
427 dma_resv_held(drm_gpuvm_resv(gpuvm__))
428
429#define drm_gpuvm_resv_assert_held(gpuvm__) \
430 dma_resv_assert_held(drm_gpuvm_resv(gpuvm__))
431
432/**
433 * drm_gpuvm_is_extobj() - indicates whether the given &drm_gem_object is an
434 * external object
435 * @gpuvm: the &drm_gpuvm to check
436 * @obj: the &drm_gem_object to check
437 *
438 * Returns: true if the &drm_gem_object &dma_resv differs from the
439 * &drm_gpuvms &dma_resv, false otherwise
440 */
441static inline bool
442drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm,
443 struct drm_gem_object *obj)
444{
445 return obj && obj->resv != drm_gpuvm_resv(gpuvm);
446}
447
448static inline struct drm_gpuva *
449__drm_gpuva_next(struct drm_gpuva *va)
450{
451 if (va && !list_is_last(list: &va->rb.entry, head: &va->vm->rb.list))
452 return list_next_entry(va, rb.entry);
453
454 return NULL;
455}
456
457/**
458 * drm_gpuvm_for_each_va_range() - iterate over a range of &drm_gpuvas
459 * @va__: &drm_gpuva structure to assign to in each iteration step
460 * @gpuvm__: &drm_gpuvm to walk over
461 * @start__: starting offset, the first gpuva will overlap this
462 * @end__: ending offset, the last gpuva will start before this (but may
463 * overlap)
464 *
465 * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie
466 * between @start__ and @end__. It is implemented similarly to list_for_each(),
467 * but is using the &drm_gpuvm's internal interval tree to accelerate
468 * the search for the starting &drm_gpuva, and hence isn't safe against removal
469 * of elements. It assumes that @end__ is within (or is the upper limit of) the
470 * &drm_gpuvm. This iterator does not skip over the &drm_gpuvm's
471 * @kernel_alloc_node.
472 */
473#define drm_gpuvm_for_each_va_range(va__, gpuvm__, start__, end__) \
474 for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)); \
475 va__ && (va__->va.addr < (end__)); \
476 va__ = __drm_gpuva_next(va__))
477
478/**
479 * drm_gpuvm_for_each_va_range_safe() - safely iterate over a range of
480 * &drm_gpuvas
481 * @va__: &drm_gpuva to assign to in each iteration step
482 * @next__: another &drm_gpuva to use as temporary storage
483 * @gpuvm__: &drm_gpuvm to walk over
484 * @start__: starting offset, the first gpuva will overlap this
485 * @end__: ending offset, the last gpuva will start before this (but may
486 * overlap)
487 *
488 * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie
489 * between @start__ and @end__. It is implemented similarly to
490 * list_for_each_safe(), but is using the &drm_gpuvm's internal interval
491 * tree to accelerate the search for the starting &drm_gpuva, and hence is safe
492 * against removal of elements. It assumes that @end__ is within (or is the
493 * upper limit of) the &drm_gpuvm. This iterator does not skip over the
494 * &drm_gpuvm's @kernel_alloc_node.
495 */
496#define drm_gpuvm_for_each_va_range_safe(va__, next__, gpuvm__, start__, end__) \
497 for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)), \
498 next__ = __drm_gpuva_next(va__); \
499 va__ && (va__->va.addr < (end__)); \
500 va__ = next__, next__ = __drm_gpuva_next(va__))
501
502/**
503 * drm_gpuvm_for_each_va() - iterate over all &drm_gpuvas
504 * @va__: &drm_gpuva to assign to in each iteration step
505 * @gpuvm__: &drm_gpuvm to walk over
506 *
507 * This iterator walks over all &drm_gpuva structures associated with the given
508 * &drm_gpuvm.
509 */
510#define drm_gpuvm_for_each_va(va__, gpuvm__) \
511 list_for_each_entry(va__, &(gpuvm__)->rb.list, rb.entry)
512
513/**
514 * drm_gpuvm_for_each_va_safe() - safely iterate over all &drm_gpuvas
515 * @va__: &drm_gpuva to assign to in each iteration step
516 * @next__: another &drm_gpuva to use as temporary storage
517 * @gpuvm__: &drm_gpuvm to walk over
518 *
519 * This iterator walks over all &drm_gpuva structures associated with the given
520 * &drm_gpuvm. It is implemented with list_for_each_entry_safe(), and
521 * hence safe against the removal of elements.
522 */
523#define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \
524 list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry)
525
526/**
527 * struct drm_gpuvm_exec - &drm_gpuvm abstraction of &drm_exec
528 *
529 * This structure should be created on the stack as &drm_exec should be.
530 *
531 * Optionally, @extra can be set in order to lock additional &drm_gem_objects.
532 */
533struct drm_gpuvm_exec {
534 /**
535 * @exec: the &drm_exec structure
536 */
537 struct drm_exec exec;
538
539 /**
540 * @flags: the flags for the struct drm_exec
541 */
542 u32 flags;
543
544 /**
545 * @vm: the &drm_gpuvm to lock its DMA reservations
546 */
547 struct drm_gpuvm *vm;
548
549 /**
550 * @num_fences: the number of fences to reserve for the &dma_resv of the
551 * locked &drm_gem_objects
552 */
553 unsigned int num_fences;
554
555 /**
556 * @extra: Callback and corresponding private data for the driver to
557 * lock arbitrary additional &drm_gem_objects.
558 */
559 struct {
560 /**
561 * @extra.fn: The driver callback to lock additional
562 * &drm_gem_objects.
563 */
564 int (*fn)(struct drm_gpuvm_exec *vm_exec);
565
566 /**
567 * @extra.priv: driver private data for the @fn callback
568 */
569 void *priv;
570 } extra;
571};
572
573int drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm,
574 struct drm_exec *exec,
575 unsigned int num_fences);
576
577int drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
578 struct drm_exec *exec,
579 unsigned int num_fences);
580
581int drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm,
582 struct drm_exec *exec,
583 u64 addr, u64 range,
584 unsigned int num_fences);
585
586int drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec);
587
588int drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec,
589 struct drm_gem_object **objs,
590 unsigned int num_objs);
591
592int drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec,
593 u64 addr, u64 range);
594
595/**
596 * drm_gpuvm_exec_unlock() - lock all dma-resv of all assoiciated BOs
597 * @vm_exec: the &drm_gpuvm_exec wrapper
598 *
599 * Releases all dma-resv locks of all &drm_gem_objects previously acquired
600 * through drm_gpuvm_exec_lock() or its variants.
601 *
602 * Returns: 0 on success, negative error code on failure.
603 */
604static inline void
605drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec)
606{
607 drm_exec_fini(exec: &vm_exec->exec);
608}
609
610int drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec);
611void drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,
612 struct drm_exec *exec,
613 struct dma_fence *fence,
614 enum dma_resv_usage private_usage,
615 enum dma_resv_usage extobj_usage);
616
617/**
618 * drm_gpuvm_exec_resv_add_fence() - add fence to private and all extobj
619 * @vm_exec: the &drm_gpuvm_exec wrapper
620 * @fence: fence to add
621 * @private_usage: private dma-resv usage
622 * @extobj_usage: extobj dma-resv usage
623 *
624 * See drm_gpuvm_resv_add_fence().
625 */
626static inline void
627drm_gpuvm_exec_resv_add_fence(struct drm_gpuvm_exec *vm_exec,
628 struct dma_fence *fence,
629 enum dma_resv_usage private_usage,
630 enum dma_resv_usage extobj_usage)
631{
632 drm_gpuvm_resv_add_fence(gpuvm: vm_exec->vm, exec: &vm_exec->exec, fence,
633 private_usage, extobj_usage);
634}
635
636/**
637 * drm_gpuvm_exec_validate() - validate all BOs marked as evicted
638 * @vm_exec: the &drm_gpuvm_exec wrapper
639 *
640 * See drm_gpuvm_validate().
641 *
642 * Returns: 0 on success, negative error code on failure.
643 */
644static inline int
645drm_gpuvm_exec_validate(struct drm_gpuvm_exec *vm_exec)
646{
647 return drm_gpuvm_validate(gpuvm: vm_exec->vm, exec: &vm_exec->exec);
648}
649
650/**
651 * struct drm_gpuvm_bo - structure representing a &drm_gpuvm and
652 * &drm_gem_object combination
653 *
654 * This structure is an abstraction representing a &drm_gpuvm and
655 * &drm_gem_object combination. It serves as an indirection to accelerate
656 * iterating all &drm_gpuvas within a &drm_gpuvm backed by the same
657 * &drm_gem_object.
658 *
659 * Furthermore it is used cache evicted GEM objects for a certain GPU-VM to
660 * accelerate validation.
661 *
662 * Typically, drivers want to create an instance of a struct drm_gpuvm_bo once
663 * a GEM object is mapped first in a GPU-VM and release the instance once the
664 * last mapping of the GEM object in this GPU-VM is unmapped.
665 */
666struct drm_gpuvm_bo {
667 /**
668 * @vm: The &drm_gpuvm the @obj is mapped in. This is a reference
669 * counted pointer.
670 */
671 struct drm_gpuvm *vm;
672
673 /**
674 * @obj: The &drm_gem_object being mapped in @vm. This is a reference
675 * counted pointer.
676 */
677 struct drm_gem_object *obj;
678
679 /**
680 * @evicted: Indicates whether the &drm_gem_object is evicted; field
681 * protected by the &drm_gem_object's dma-resv lock.
682 */
683 bool evicted;
684
685 /**
686 * @kref: The reference count for this &drm_gpuvm_bo.
687 */
688 struct kref kref;
689
690 /**
691 * @list: Structure containing all &list_heads.
692 */
693 struct {
694 /**
695 * @list.gpuva: The list of linked &drm_gpuvas.
696 *
697 * It is safe to access entries from this list as long as the
698 * GEM's gpuva lock is held. See also struct drm_gem_object.
699 */
700 struct list_head gpuva;
701
702 /**
703 * @list.entry: Structure containing all &list_heads serving as
704 * entry.
705 */
706 struct {
707 /**
708 * @list.entry.gem: List entry to attach to the
709 * &drm_gem_objects gpuva list.
710 */
711 struct list_head gem;
712
713 /**
714 * @list.entry.evict: List entry to attach to the
715 * &drm_gpuvms extobj list.
716 */
717 struct list_head extobj;
718
719 /**
720 * @list.entry.evict: List entry to attach to the
721 * &drm_gpuvms evict list.
722 */
723 struct list_head evict;
724
725 /**
726 * @list.entry.bo_defer: List entry to attach to
727 * the &drm_gpuvms bo_defer list.
728 */
729 struct llist_node bo_defer;
730 } entry;
731 } list;
732};
733
734struct drm_gpuvm_bo *
735drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,
736 struct drm_gem_object *obj);
737
738struct drm_gpuvm_bo *
739drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm,
740 struct drm_gem_object *obj);
741struct drm_gpuvm_bo *
742drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *vm_bo);
743
744/**
745 * drm_gpuvm_bo_get() - acquire a struct drm_gpuvm_bo reference
746 * @vm_bo: the &drm_gpuvm_bo to acquire the reference of
747 *
748 * This function acquires an additional reference to @vm_bo. It is illegal to
749 * call this without already holding a reference. No locks required.
750 *
751 * Returns: the &struct vm_bo pointer
752 */
753static inline struct drm_gpuvm_bo *
754drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo)
755{
756 kref_get(kref: &vm_bo->kref);
757 return vm_bo;
758}
759
760bool drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo);
761
762bool drm_gpuvm_bo_put_deferred(struct drm_gpuvm_bo *vm_bo);
763void drm_gpuvm_bo_deferred_cleanup(struct drm_gpuvm *gpuvm);
764
765struct drm_gpuvm_bo *
766drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
767 struct drm_gem_object *obj);
768
769void drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict);
770
771/**
772 * drm_gpuvm_bo_gem_evict() - add/remove all &drm_gpuvm_bo's in the list
773 * to/from the &drm_gpuvms evicted list
774 * @obj: the &drm_gem_object
775 * @evict: indicates whether @obj is evicted
776 *
777 * See drm_gpuvm_bo_evict().
778 */
779static inline void
780drm_gpuvm_bo_gem_evict(struct drm_gem_object *obj, bool evict)
781{
782 struct drm_gpuvm_bo *vm_bo;
783
784 drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
785 drm_gem_gpuva_assert_lock_held(vm_bo->vm, obj);
786 drm_gpuvm_bo_evict(vm_bo, evict);
787 }
788}
789
790void drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo);
791
792/**
793 * drm_gpuvm_bo_for_each_va() - iterator to walk over a list of &drm_gpuva
794 * @va__: &drm_gpuva structure to assign to in each iteration step
795 * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with
796 *
797 * This iterator walks over all &drm_gpuva structures associated with the
798 * &drm_gpuvm_bo.
799 *
800 * The caller must hold the GEM's gpuva lock.
801 */
802#define drm_gpuvm_bo_for_each_va(va__, vm_bo__) \
803 list_for_each_entry(va__, &(vm_bo)->list.gpuva, gem.entry)
804
805/**
806 * drm_gpuvm_bo_for_each_va_safe() - iterator to safely walk over a list of
807 * &drm_gpuva
808 * @va__: &drm_gpuva structure to assign to in each iteration step
809 * @next__: &next &drm_gpuva to store the next step
810 * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with
811 *
812 * This iterator walks over all &drm_gpuva structures associated with the
813 * &drm_gpuvm_bo. It is implemented with list_for_each_entry_safe(), hence
814 * it is save against removal of elements.
815 *
816 * The caller must hold the GEM's gpuva lock.
817 */
818#define drm_gpuvm_bo_for_each_va_safe(va__, next__, vm_bo__) \
819 list_for_each_entry_safe(va__, next__, &(vm_bo)->list.gpuva, gem.entry)
820
821/**
822 * enum drm_gpuva_op_type - GPU VA operation type
823 *
824 * Operations to alter the GPU VA mappings tracked by the &drm_gpuvm.
825 */
826enum drm_gpuva_op_type {
827 /**
828 * @DRM_GPUVA_OP_MAP: the map op type
829 */
830 DRM_GPUVA_OP_MAP,
831
832 /**
833 * @DRM_GPUVA_OP_REMAP: the remap op type
834 */
835 DRM_GPUVA_OP_REMAP,
836
837 /**
838 * @DRM_GPUVA_OP_UNMAP: the unmap op type
839 */
840 DRM_GPUVA_OP_UNMAP,
841
842 /**
843 * @DRM_GPUVA_OP_PREFETCH: the prefetch op type
844 */
845 DRM_GPUVA_OP_PREFETCH,
846
847 /**
848 * @DRM_GPUVA_OP_DRIVER: the driver defined op type
849 */
850 DRM_GPUVA_OP_DRIVER,
851};
852
853/**
854 * struct drm_gpuva_op_map - GPU VA map operation
855 *
856 * This structure represents a single map operation generated by the
857 * DRM GPU VA manager.
858 */
859struct drm_gpuva_op_map {
860 /**
861 * @va: structure containing address and range of a map
862 * operation
863 */
864 struct {
865 /**
866 * @va.addr: the base address of the new mapping
867 */
868 u64 addr;
869
870 /**
871 * @va.range: the range of the new mapping
872 */
873 u64 range;
874 } va;
875
876 /**
877 * @gem: structure containing the &drm_gem_object and its offset
878 */
879 struct {
880 /**
881 * @gem.offset: the offset within the &drm_gem_object
882 */
883 u64 offset;
884
885 /**
886 * @gem.obj: the &drm_gem_object to map
887 */
888 struct drm_gem_object *obj;
889 } gem;
890};
891
892/**
893 * struct drm_gpuva_op_unmap - GPU VA unmap operation
894 *
895 * This structure represents a single unmap operation generated by the
896 * DRM GPU VA manager.
897 */
898struct drm_gpuva_op_unmap {
899 /**
900 * @va: the &drm_gpuva to unmap
901 */
902 struct drm_gpuva *va;
903
904 /**
905 * @keep:
906 *
907 * Indicates whether this &drm_gpuva is physically contiguous with the
908 * original mapping request.
909 *
910 * Optionally, if &keep is set, drivers may keep the actual page table
911 * mappings for this &drm_gpuva, adding the missing page table entries
912 * only and update the &drm_gpuvm accordingly.
913 */
914 bool keep;
915};
916
917/**
918 * struct drm_gpuva_op_remap - GPU VA remap operation
919 *
920 * This represents a single remap operation generated by the DRM GPU VA manager.
921 *
922 * A remap operation is generated when an existing GPU VA mmapping is split up
923 * by inserting a new GPU VA mapping or by partially unmapping existent
924 * mapping(s), hence it consists of a maximum of two map and one unmap
925 * operation.
926 *
927 * The @unmap operation takes care of removing the original existing mapping.
928 * @prev is used to remap the preceding part, @next the subsequent part.
929 *
930 * If either a new mapping's start address is aligned with the start address
931 * of the old mapping or the new mapping's end address is aligned with the
932 * end address of the old mapping, either @prev or @next is NULL.
933 *
934 * Note, the reason for a dedicated remap operation, rather than arbitrary
935 * unmap and map operations, is to give drivers the chance of extracting driver
936 * specific data for creating the new mappings from the unmap operations's
937 * &drm_gpuva structure which typically is embedded in larger driver specific
938 * structures.
939 */
940struct drm_gpuva_op_remap {
941 /**
942 * @prev: the preceding part of a split mapping
943 */
944 struct drm_gpuva_op_map *prev;
945
946 /**
947 * @next: the subsequent part of a split mapping
948 */
949 struct drm_gpuva_op_map *next;
950
951 /**
952 * @unmap: the unmap operation for the original existing mapping
953 */
954 struct drm_gpuva_op_unmap *unmap;
955};
956
957/**
958 * struct drm_gpuva_op_prefetch - GPU VA prefetch operation
959 *
960 * This structure represents a single prefetch operation generated by the
961 * DRM GPU VA manager.
962 */
963struct drm_gpuva_op_prefetch {
964 /**
965 * @va: the &drm_gpuva to prefetch
966 */
967 struct drm_gpuva *va;
968};
969
970/**
971 * struct drm_gpuva_op - GPU VA operation
972 *
973 * This structure represents a single generic operation.
974 *
975 * The particular type of the operation is defined by @op.
976 */
977struct drm_gpuva_op {
978 /**
979 * @entry:
980 *
981 * The &list_head used to distribute instances of this struct within
982 * &drm_gpuva_ops.
983 */
984 struct list_head entry;
985
986 /**
987 * @op: the type of the operation
988 */
989 enum drm_gpuva_op_type op;
990
991 union {
992 /**
993 * @map: the map operation
994 */
995 struct drm_gpuva_op_map map;
996
997 /**
998 * @remap: the remap operation
999 */
1000 struct drm_gpuva_op_remap remap;
1001
1002 /**
1003 * @unmap: the unmap operation
1004 */
1005 struct drm_gpuva_op_unmap unmap;
1006
1007 /**
1008 * @prefetch: the prefetch operation
1009 */
1010 struct drm_gpuva_op_prefetch prefetch;
1011 };
1012};
1013
1014/**
1015 * struct drm_gpuva_ops - wraps a list of &drm_gpuva_op
1016 */
1017struct drm_gpuva_ops {
1018 /**
1019 * @list: the &list_head
1020 */
1021 struct list_head list;
1022};
1023
1024/**
1025 * drm_gpuva_for_each_op() - iterator to walk over &drm_gpuva_ops
1026 * @op: &drm_gpuva_op to assign in each iteration step
1027 * @ops: &drm_gpuva_ops to walk
1028 *
1029 * This iterator walks over all ops within a given list of operations.
1030 */
1031#define drm_gpuva_for_each_op(op, ops) list_for_each_entry(op, &(ops)->list, entry)
1032
1033/**
1034 * drm_gpuva_for_each_op_safe() - iterator to safely walk over &drm_gpuva_ops
1035 * @op: &drm_gpuva_op to assign in each iteration step
1036 * @next: &next &drm_gpuva_op to store the next step
1037 * @ops: &drm_gpuva_ops to walk
1038 *
1039 * This iterator walks over all ops within a given list of operations. It is
1040 * implemented with list_for_each_safe(), so save against removal of elements.
1041 */
1042#define drm_gpuva_for_each_op_safe(op, next, ops) \
1043 list_for_each_entry_safe(op, next, &(ops)->list, entry)
1044
1045/**
1046 * drm_gpuva_for_each_op_from_reverse() - iterate backwards from the given point
1047 * @op: &drm_gpuva_op to assign in each iteration step
1048 * @ops: &drm_gpuva_ops to walk
1049 *
1050 * This iterator walks over all ops within a given list of operations beginning
1051 * from the given operation in reverse order.
1052 */
1053#define drm_gpuva_for_each_op_from_reverse(op, ops) \
1054 list_for_each_entry_from_reverse(op, &(ops)->list, entry)
1055
1056/**
1057 * drm_gpuva_for_each_op_reverse - iterator to walk over &drm_gpuva_ops in reverse
1058 * @op: &drm_gpuva_op to assign in each iteration step
1059 * @ops: &drm_gpuva_ops to walk
1060 *
1061 * This iterator walks over all ops within a given list of operations in reverse
1062 */
1063#define drm_gpuva_for_each_op_reverse(op, ops) \
1064 list_for_each_entry_reverse(op, &(ops)->list, entry)
1065
1066/**
1067 * drm_gpuva_first_op() - returns the first &drm_gpuva_op from &drm_gpuva_ops
1068 * @ops: the &drm_gpuva_ops to get the fist &drm_gpuva_op from
1069 */
1070#define drm_gpuva_first_op(ops) \
1071 list_first_entry(&(ops)->list, struct drm_gpuva_op, entry)
1072
1073/**
1074 * drm_gpuva_last_op() - returns the last &drm_gpuva_op from &drm_gpuva_ops
1075 * @ops: the &drm_gpuva_ops to get the last &drm_gpuva_op from
1076 */
1077#define drm_gpuva_last_op(ops) \
1078 list_last_entry(&(ops)->list, struct drm_gpuva_op, entry)
1079
1080/**
1081 * drm_gpuva_prev_op() - previous &drm_gpuva_op in the list
1082 * @op: the current &drm_gpuva_op
1083 */
1084#define drm_gpuva_prev_op(op) list_prev_entry(op, entry)
1085
1086/**
1087 * drm_gpuva_next_op() - next &drm_gpuva_op in the list
1088 * @op: the current &drm_gpuva_op
1089 */
1090#define drm_gpuva_next_op(op) list_next_entry(op, entry)
1091
1092/**
1093 * struct drm_gpuvm_map_req - arguments passed to drm_gpuvm_sm_map[_ops_create]()
1094 */
1095struct drm_gpuvm_map_req {
1096 /**
1097 * @map: struct drm_gpuva_op_map
1098 */
1099 struct drm_gpuva_op_map map;
1100};
1101
1102struct drm_gpuva_ops *
1103drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,
1104 const struct drm_gpuvm_map_req *req);
1105struct drm_gpuva_ops *
1106drm_gpuvm_madvise_ops_create(struct drm_gpuvm *gpuvm,
1107 const struct drm_gpuvm_map_req *req);
1108
1109struct drm_gpuva_ops *
1110drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm,
1111 u64 addr, u64 range);
1112
1113struct drm_gpuva_ops *
1114drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,
1115 u64 addr, u64 range);
1116
1117struct drm_gpuva_ops *
1118drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo);
1119
1120void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm,
1121 struct drm_gpuva_ops *ops);
1122
1123static inline void drm_gpuva_init_from_op(struct drm_gpuva *va,
1124 struct drm_gpuva_op_map *op)
1125{
1126 va->va.addr = op->va.addr;
1127 va->va.range = op->va.range;
1128 va->gem.obj = op->gem.obj;
1129 va->gem.offset = op->gem.offset;
1130}
1131
1132/**
1133 * struct drm_gpuvm_ops - callbacks for split/merge steps
1134 *
1135 * This structure defines the callbacks used by &drm_gpuvm_sm_map and
1136 * &drm_gpuvm_sm_unmap to provide the split/merge steps for map and unmap
1137 * operations to drivers.
1138 */
1139struct drm_gpuvm_ops {
1140 /**
1141 * @vm_free: called when the last reference of a struct drm_gpuvm is
1142 * dropped
1143 *
1144 * This callback is mandatory.
1145 */
1146 void (*vm_free)(struct drm_gpuvm *gpuvm);
1147
1148 /**
1149 * @op_alloc: called when the &drm_gpuvm allocates
1150 * a struct drm_gpuva_op
1151 *
1152 * Some drivers may want to embed struct drm_gpuva_op into driver
1153 * specific structures. By implementing this callback drivers can
1154 * allocate memory accordingly.
1155 *
1156 * This callback is optional.
1157 */
1158 struct drm_gpuva_op *(*op_alloc)(void);
1159
1160 /**
1161 * @op_free: called when the &drm_gpuvm frees a
1162 * struct drm_gpuva_op
1163 *
1164 * Some drivers may want to embed struct drm_gpuva_op into driver
1165 * specific structures. By implementing this callback drivers can
1166 * free the previously allocated memory accordingly.
1167 *
1168 * This callback is optional.
1169 */
1170 void (*op_free)(struct drm_gpuva_op *op);
1171
1172 /**
1173 * @vm_bo_alloc: called when the &drm_gpuvm allocates
1174 * a struct drm_gpuvm_bo
1175 *
1176 * Some drivers may want to embed struct drm_gpuvm_bo into driver
1177 * specific structures. By implementing this callback drivers can
1178 * allocate memory accordingly.
1179 *
1180 * This callback is optional.
1181 */
1182 struct drm_gpuvm_bo *(*vm_bo_alloc)(void);
1183
1184 /**
1185 * @vm_bo_free: called when the &drm_gpuvm frees a
1186 * struct drm_gpuvm_bo
1187 *
1188 * Some drivers may want to embed struct drm_gpuvm_bo into driver
1189 * specific structures. By implementing this callback drivers can
1190 * free the previously allocated memory accordingly.
1191 *
1192 * This callback is optional.
1193 */
1194 void (*vm_bo_free)(struct drm_gpuvm_bo *vm_bo);
1195
1196 /**
1197 * @vm_bo_validate: called from drm_gpuvm_validate()
1198 *
1199 * Drivers receive this callback for every evicted &drm_gem_object being
1200 * mapped in the corresponding &drm_gpuvm.
1201 *
1202 * Typically, drivers would call their driver specific variant of
1203 * ttm_bo_validate() from within this callback.
1204 */
1205 int (*vm_bo_validate)(struct drm_gpuvm_bo *vm_bo,
1206 struct drm_exec *exec);
1207
1208 /**
1209 * @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the
1210 * mapping once all previous steps were completed
1211 *
1212 * The &priv pointer matches the one the driver passed to
1213 * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively.
1214 *
1215 * Can be NULL if &drm_gpuvm_sm_map is used.
1216 */
1217 int (*sm_step_map)(struct drm_gpuva_op *op, void *priv);
1218
1219 /**
1220 * @sm_step_remap: called from &drm_gpuvm_sm_map and
1221 * &drm_gpuvm_sm_unmap to split up an existent mapping
1222 *
1223 * This callback is called when existent mapping needs to be split up.
1224 * This is the case when either a newly requested mapping overlaps or
1225 * is enclosed by an existent mapping or a partial unmap of an existent
1226 * mapping is requested.
1227 *
1228 * The &priv pointer matches the one the driver passed to
1229 * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively.
1230 *
1231 * Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is
1232 * used.
1233 */
1234 int (*sm_step_remap)(struct drm_gpuva_op *op, void *priv);
1235
1236 /**
1237 * @sm_step_unmap: called from &drm_gpuvm_sm_map and
1238 * &drm_gpuvm_sm_unmap to unmap an existing mapping
1239 *
1240 * This callback is called when existing mapping needs to be unmapped.
1241 * This is the case when either a newly requested mapping encloses an
1242 * existing mapping or an unmap of an existing mapping is requested.
1243 *
1244 * The &priv pointer matches the one the driver passed to
1245 * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively.
1246 *
1247 * Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is
1248 * used.
1249 */
1250 int (*sm_step_unmap)(struct drm_gpuva_op *op, void *priv);
1251};
1252
1253int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv,
1254 const struct drm_gpuvm_map_req *req);
1255
1256int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv,
1257 u64 addr, u64 range);
1258
1259int drm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm,
1260 struct drm_exec *exec, unsigned int num_fences,
1261 struct drm_gpuvm_map_req *req);
1262
1263int drm_gpuvm_sm_unmap_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec,
1264 u64 req_addr, u64 req_range);
1265
1266void drm_gpuva_map(struct drm_gpuvm *gpuvm,
1267 struct drm_gpuva *va,
1268 struct drm_gpuva_op_map *op);
1269
1270void drm_gpuva_remap(struct drm_gpuva *prev,
1271 struct drm_gpuva *next,
1272 struct drm_gpuva_op_remap *op);
1273
1274void drm_gpuva_unmap(struct drm_gpuva_op_unmap *op);
1275
1276/**
1277 * drm_gpuva_op_remap_to_unmap_range() - Helper to get the start and range of
1278 * the unmap stage of a remap op.
1279 * @op: Remap op.
1280 * @start_addr: Output pointer for the start of the required unmap.
1281 * @range: Output pointer for the length of the required unmap.
1282 *
1283 * The given start address and range will be set such that they represent the
1284 * range of the address space that was previously covered by the mapping being
1285 * re-mapped, but is now empty.
1286 */
1287static inline void
1288drm_gpuva_op_remap_to_unmap_range(const struct drm_gpuva_op_remap *op,
1289 u64 *start_addr, u64 *range)
1290{
1291 const u64 va_start = op->prev ?
1292 op->prev->va.addr + op->prev->va.range :
1293 op->unmap->va->va.addr;
1294 const u64 va_end = op->next ?
1295 op->next->va.addr :
1296 op->unmap->va->va.addr + op->unmap->va->va.range;
1297
1298 if (start_addr)
1299 *start_addr = va_start;
1300 if (range)
1301 *range = va_end - va_start;
1302}
1303
1304#endif /* __DRM_GPUVM_H__ */
1305

source code of linux/include/drm/drm_gpuvm.h