1/* SPDX-License-Identifier: MIT */
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#ifndef _XE_VM_TYPES_H_
7#define _XE_VM_TYPES_H_
8
9#include <drm/drm_gpusvm.h>
10#include <drm/drm_gpuvm.h>
11
12#include <linux/dma-resv.h>
13#include <linux/kref.h>
14#include <linux/mmu_notifier.h>
15#include <linux/scatterlist.h>
16
17#include "xe_device_types.h"
18#include "xe_pt_types.h"
19#include "xe_range_fence.h"
20#include "xe_userptr.h"
21
22struct xe_bo;
23struct xe_svm_range;
24struct xe_sync_entry;
25struct xe_user_fence;
26struct xe_vm;
27struct xe_vm_pgtable_update_op;
28
29#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
30#define TEST_VM_OPS_ERROR
31#define FORCE_OP_ERROR BIT(31)
32
33#define FORCE_OP_ERROR_LOCK 0
34#define FORCE_OP_ERROR_PREPARE 1
35#define FORCE_OP_ERROR_RUN 2
36#define FORCE_OP_ERROR_COUNT 3
37#endif
38
39#define XE_VMA_READ_ONLY DRM_GPUVA_USERBITS
40#define XE_VMA_DESTROYED (DRM_GPUVA_USERBITS << 1)
41#define XE_VMA_ATOMIC_PTE_BIT (DRM_GPUVA_USERBITS << 2)
42#define XE_VMA_PTE_4K (DRM_GPUVA_USERBITS << 3)
43#define XE_VMA_PTE_2M (DRM_GPUVA_USERBITS << 4)
44#define XE_VMA_PTE_1G (DRM_GPUVA_USERBITS << 5)
45#define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6)
46#define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7)
47#define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8)
48#define XE_VMA_SYSTEM_ALLOCATOR (DRM_GPUVA_USERBITS << 9)
49#define XE_VMA_MADV_AUTORESET (DRM_GPUVA_USERBITS << 10)
50
51/**
52 * struct xe_vma_mem_attr - memory attributes associated with vma
53 */
54struct xe_vma_mem_attr {
55 /** @preferred_loc: preferred memory_location */
56 struct {
57 /** @preferred_loc.migration_policy: Pages migration policy */
58 u32 migration_policy;
59
60 /**
61 * @preferred_loc.devmem_fd: used for determining pagemap_fd
62 * requested by user DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM and
63 * DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE mean system memory or
64 * closest device memory respectively.
65 */
66 u32 devmem_fd;
67 } preferred_loc;
68
69 /**
70 * @atomic_access: The atomic access type for the vma
71 * See %DRM_XE_VMA_ATOMIC_UNDEFINED, %DRM_XE_VMA_ATOMIC_DEVICE,
72 * %DRM_XE_VMA_ATOMIC_GLOBAL, and %DRM_XE_VMA_ATOMIC_CPU for possible
73 * values. These are defined in uapi/drm/xe_drm.h.
74 */
75 u32 atomic_access;
76
77 /**
78 * @default_pat_index: The pat index for VMA set during first bind by user.
79 */
80 u16 default_pat_index;
81
82 /**
83 * @pat_index: The pat index to use when encoding the PTEs for this vma.
84 * same as default_pat_index unless overwritten by madvise.
85 */
86 u16 pat_index;
87};
88
89struct xe_vma {
90 /** @gpuva: Base GPUVA object */
91 struct drm_gpuva gpuva;
92
93 /**
94 * @combined_links: links into lists which are mutually exclusive.
95 * Locking: vm lock in write mode OR vm lock in read mode and the vm's
96 * resv.
97 */
98 union {
99 /** @rebind: link into VM if this VMA needs rebinding. */
100 struct list_head rebind;
101 /** @destroy: link to contested list when VM is being closed. */
102 struct list_head destroy;
103 } combined_links;
104
105 union {
106 /** @destroy_cb: callback to destroy VMA when unbind job is done */
107 struct dma_fence_cb destroy_cb;
108 /** @destroy_work: worker to destroy this BO */
109 struct work_struct destroy_work;
110 };
111
112 /**
113 * @tile_invalidated: Tile mask of binding are invalidated for this VMA.
114 * protected by BO's resv and for userptrs, vm->svm.gpusvm.notifier_lock in
115 * write mode for writing or vm->svm.gpusvm.notifier_lock in read mode and
116 * the vm->resv. For stable reading, BO's resv or userptr
117 * vm->svm.gpusvm.notifier_lock in read mode is required. Can be
118 * opportunistically read with READ_ONCE outside of locks.
119 */
120 u8 tile_invalidated;
121
122 /** @tile_mask: Tile mask of where to create binding for this VMA */
123 u8 tile_mask;
124
125 /**
126 * @tile_present: Tile mask of binding are present for this VMA.
127 * protected by vm->lock, vm->resv and for userptrs,
128 * vm->svm.gpusvm.notifier_lock for writing. Needs either for reading,
129 * but if reading is done under the vm->lock only, it needs to be held
130 * in write mode.
131 */
132 u8 tile_present;
133
134 /** @tile_staged: bind is staged for this VMA */
135 u8 tile_staged;
136
137 /**
138 * @skip_invalidation: Used in madvise to avoid invalidation
139 * if mem attributes doesn't change
140 */
141 bool skip_invalidation;
142
143 /**
144 * @ufence: The user fence that was provided with MAP.
145 * Needs to be signalled before UNMAP can be processed.
146 */
147 struct xe_user_fence *ufence;
148
149 /**
150 * @attr: The attributes of vma which determines the migration policy
151 * and encoding of the PTEs for this vma.
152 */
153 struct xe_vma_mem_attr attr;
154};
155
156/**
157 * struct xe_userptr_vma - A userptr vma subclass
158 * @vma: The vma.
159 * @userptr: Additional userptr information.
160 */
161struct xe_userptr_vma {
162 struct xe_vma vma;
163 struct xe_userptr userptr;
164};
165
166struct xe_device;
167
168struct xe_vm {
169 /** @gpuvm: base GPUVM used to track VMAs */
170 struct drm_gpuvm gpuvm;
171
172 /** @svm: Shared virtual memory state */
173 struct {
174 /** @svm.gpusvm: base GPUSVM used to track fault allocations */
175 struct drm_gpusvm gpusvm;
176 /**
177 * @svm.garbage_collector: Garbage collector which is used unmap
178 * SVM range's GPU bindings and destroy the ranges.
179 */
180 struct {
181 /** @svm.garbage_collector.lock: Protect's range list */
182 spinlock_t lock;
183 /**
184 * @svm.garbage_collector.range_list: List of SVM ranges
185 * in the garbage collector.
186 */
187 struct list_head range_list;
188 /**
189 * @svm.garbage_collector.work: Worker which the
190 * garbage collector runs on.
191 */
192 struct work_struct work;
193 } garbage_collector;
194 } svm;
195
196 struct xe_device *xe;
197
198 /* exec queue used for (un)binding vma's */
199 struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE];
200
201 /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */
202 struct ttm_lru_bulk_move lru_bulk_move;
203
204 u64 size;
205
206 struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE];
207 struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL];
208
209 /**
210 * @flags: flags for this VM, statically setup a creation time aside
211 * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely
212 */
213#define XE_VM_FLAG_64K BIT(0)
214#define XE_VM_FLAG_LR_MODE BIT(1)
215#define XE_VM_FLAG_MIGRATION BIT(2)
216#define XE_VM_FLAG_SCRATCH_PAGE BIT(3)
217#define XE_VM_FLAG_FAULT_MODE BIT(4)
218#define XE_VM_FLAG_BANNED BIT(5)
219#define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags)
220#define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id)
221#define XE_VM_FLAG_GSC BIT(8)
222 unsigned long flags;
223
224 /**
225 * @lock: outer most lock, protects objects of anything attached to this
226 * VM
227 */
228 struct rw_semaphore lock;
229 /**
230 * @snap_mutex: Mutex used to guard insertions and removals from gpuva,
231 * so we can take a snapshot safely from devcoredump.
232 */
233 struct mutex snap_mutex;
234
235 /**
236 * @rebind_list: list of VMAs that need rebinding. Protected by the
237 * vm->lock in write mode, OR (the vm->lock in read mode and the
238 * vm resv).
239 */
240 struct list_head rebind_list;
241
242 /**
243 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling
244 * from an irq context can be last put and the destroy needs to be able
245 * to sleep.
246 */
247 struct work_struct destroy_work;
248
249 /**
250 * @rftree: range fence tree to track updates to page table structure.
251 * Used to implement conflict tracking between independent bind engines.
252 */
253 struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE];
254
255 const struct xe_pt_ops *pt_ops;
256
257 /** @userptr: user pointer state */
258 struct xe_userptr_vm userptr;
259
260 /** @preempt: preempt state */
261 struct {
262 /**
263 * @min_run_period_ms: The minimum run period before preempting
264 * an engine again
265 */
266 unsigned int min_run_period_ms;
267 /** @exec_queues: list of exec queues attached to this VM */
268 struct list_head exec_queues;
269 /** @num_exec_queues: number exec queues attached to this VM */
270 int num_exec_queues;
271 /**
272 * @rebind_deactivated: Whether rebind has been temporarily deactivated
273 * due to no work available. Protected by the vm resv.
274 */
275 bool rebind_deactivated;
276 /**
277 * @rebind_work: worker to rebind invalidated userptrs / evicted
278 * BOs
279 */
280 struct work_struct rebind_work;
281 /**
282 * @preempt.pm_activate_link: Link to list of rebind workers to be
283 * kicked on resume.
284 */
285 struct list_head pm_activate_link;
286 } preempt;
287
288 /** @um: unified memory state */
289 struct {
290 /** @asid: address space ID, unique to each VM */
291 u32 asid;
292 /**
293 * @last_fault_vma: Last fault VMA, used for fast lookup when we
294 * get a flood of faults to the same VMA
295 */
296 struct xe_vma *last_fault_vma;
297 } usm;
298
299 /** @error_capture: allow to track errors */
300 struct {
301 /** @capture_once: capture only one error per VM */
302 bool capture_once;
303 } error_capture;
304
305 /**
306 * @validation: Validation data only valid with the vm resv held.
307 * Note: This is really task state of the task holding the vm resv,
308 * and moving forward we should
309 * come up with a better way of passing this down the call-
310 * chain.
311 */
312 struct {
313 /**
314 * @validation.validating: The task that is currently making bos resident.
315 * for this vm.
316 * Protected by the VM's resv for writing. Opportunistic reading can be done
317 * using READ_ONCE. Note: This is a workaround for the
318 * TTM eviction_valuable() callback not being passed a struct
319 * ttm_operation_context(). Future work might want to address this.
320 */
321 struct task_struct *validating;
322 /**
323 * @validation.exec The drm_exec context used when locking the vm resv.
324 * Protected by the vm's resv.
325 */
326 struct drm_exec *_exec;
327 } validation;
328
329 /**
330 * @tlb_flush_seqno: Required TLB flush seqno for the next exec.
331 * protected by the vm resv.
332 */
333 u64 tlb_flush_seqno;
334 /** @batch_invalidate_tlb: Always invalidate TLB before batch start */
335 bool batch_invalidate_tlb;
336 /** @xef: Xe file handle for tracking this VM's drm client */
337 struct xe_file *xef;
338};
339
340/** struct xe_vma_op_map - VMA map operation */
341struct xe_vma_op_map {
342 /** @vma: VMA to map */
343 struct xe_vma *vma;
344 unsigned int vma_flags;
345 /** @immediate: Immediate bind */
346 bool immediate;
347 /** @read_only: Read only */
348 bool invalidate_on_bind;
349 /** @pat_index: The pat index to use for this operation. */
350 u16 pat_index;
351};
352
353/** struct xe_vma_op_remap - VMA remap operation */
354struct xe_vma_op_remap {
355 /** @prev: VMA preceding part of a split mapping */
356 struct xe_vma *prev;
357 /** @next: VMA subsequent part of a split mapping */
358 struct xe_vma *next;
359 /** @start: start of the VMA unmap */
360 u64 start;
361 /** @range: range of the VMA unmap */
362 u64 range;
363 /** @skip_prev: skip prev rebind */
364 bool skip_prev;
365 /** @skip_next: skip next rebind */
366 bool skip_next;
367 /** @unmap_done: unmap operation in done */
368 bool unmap_done;
369};
370
371/** struct xe_vma_op_prefetch - VMA prefetch operation */
372struct xe_vma_op_prefetch {
373 /** @region: memory region to prefetch to */
374 u32 region;
375};
376
377/** struct xe_vma_op_map_range - VMA map range operation */
378struct xe_vma_op_map_range {
379 /** @vma: VMA to map (system allocator VMA) */
380 struct xe_vma *vma;
381 /** @range: SVM range to map */
382 struct xe_svm_range *range;
383};
384
385/** struct xe_vma_op_unmap_range - VMA unmap range operation */
386struct xe_vma_op_unmap_range {
387 /** @range: SVM range to unmap */
388 struct xe_svm_range *range;
389};
390
391/** struct xe_vma_op_prefetch_range - VMA prefetch range operation */
392struct xe_vma_op_prefetch_range {
393 /** @range: xarray for SVM ranges data */
394 struct xarray range;
395 /** @ranges_count: number of svm ranges to map */
396 u32 ranges_count;
397 /**
398 * @tile: Pointer to the tile structure containing memory to prefetch.
399 * NULL if prefetch requested region is smem
400 */
401 struct xe_tile *tile;
402};
403
404/** enum xe_vma_op_flags - flags for VMA operation */
405enum xe_vma_op_flags {
406 /** @XE_VMA_OP_COMMITTED: VMA operation committed */
407 XE_VMA_OP_COMMITTED = BIT(0),
408 /** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */
409 XE_VMA_OP_PREV_COMMITTED = BIT(1),
410 /** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */
411 XE_VMA_OP_NEXT_COMMITTED = BIT(2),
412};
413
414/** enum xe_vma_subop - VMA sub-operation */
415enum xe_vma_subop {
416 /** @XE_VMA_SUBOP_MAP_RANGE: Map range */
417 XE_VMA_SUBOP_MAP_RANGE,
418 /** @XE_VMA_SUBOP_UNMAP_RANGE: Unmap range */
419 XE_VMA_SUBOP_UNMAP_RANGE,
420};
421
422/** struct xe_vma_op - VMA operation */
423struct xe_vma_op {
424 /** @base: GPUVA base operation */
425 struct drm_gpuva_op base;
426 /** @link: async operation link */
427 struct list_head link;
428 /** @flags: operation flags */
429 enum xe_vma_op_flags flags;
430 /** @subop: user defined sub-operation */
431 enum xe_vma_subop subop;
432 /** @tile_mask: Tile mask for operation */
433 u8 tile_mask;
434
435 union {
436 /** @map: VMA map operation specific data */
437 struct xe_vma_op_map map;
438 /** @remap: VMA remap operation specific data */
439 struct xe_vma_op_remap remap;
440 /** @prefetch: VMA prefetch operation specific data */
441 struct xe_vma_op_prefetch prefetch;
442 /** @map_range: VMA map range operation specific data */
443 struct xe_vma_op_map_range map_range;
444 /** @unmap_range: VMA unmap range operation specific data */
445 struct xe_vma_op_unmap_range unmap_range;
446 /** @prefetch_range: VMA prefetch range operation specific data */
447 struct xe_vma_op_prefetch_range prefetch_range;
448 };
449};
450
451/** struct xe_vma_ops - VMA operations */
452struct xe_vma_ops {
453 /** @list: list of VMA operations */
454 struct list_head list;
455 /** @vm: VM */
456 struct xe_vm *vm;
457 /** @q: exec queue for VMA operations */
458 struct xe_exec_queue *q;
459 /** @syncs: syncs these operation */
460 struct xe_sync_entry *syncs;
461 /** @num_syncs: number of syncs */
462 u32 num_syncs;
463 /** @pt_update_ops: page table update operations */
464 struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
465 /** @flag: signify the properties within xe_vma_ops*/
466#define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0)
467#define XE_VMA_OPS_FLAG_MADVISE BIT(1)
468#define XE_VMA_OPS_ARRAY_OF_BINDS BIT(2)
469#define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT BIT(3)
470 u32 flags;
471#ifdef TEST_VM_OPS_ERROR
472 /** @inject_error: inject error to test error handling */
473 bool inject_error;
474#endif
475};
476
477#endif
478

source code of linux/drivers/gpu/drm/xe/xe_vm_types.h