1	// SPDX-License-Identifier: MIT
2	/*
3	* Copyright © 2022 Intel Corporation
4	*/
5
6	#include "xe_pt.h"
7
8	#include "regs/xe_gtt_defs.h"
9	#include "xe_bo.h"
10	#include "xe_device.h"
11	#include "xe_drm_client.h"
12	#include "xe_exec_queue.h"
13	#include "xe_gt.h"
14	#include "xe_migrate.h"
15	#include "xe_pt_types.h"
16	#include "xe_pt_walk.h"
17	#include "xe_res_cursor.h"
18	#include "xe_sched_job.h"
19	#include "xe_svm.h"
20	#include "xe_sync.h"
21	#include "xe_tlb_inval_job.h"
22	#include "xe_trace.h"
23	#include "xe_ttm_stolen_mgr.h"
24	#include "xe_userptr.h"
25	#include "xe_vm.h"
26
27	struct xe_pt_dir {
28	struct xe_pt pt;
29	/** @children: Array of page-table child nodes */
30	struct xe_ptw *children[XE_PDES];
31	/** @staging: Array of page-table staging nodes */
32	struct xe_ptw *staging[XE_PDES];
33	};
34
35	#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
36	#define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr))
37	#define xe_pt_addr(__xe_pt) ((__xe_pt)->addr)
38	#else
39	#define xe_pt_set_addr(__xe_pt, __addr)
40	#define xe_pt_addr(__xe_pt) 0ull
41	#endif
42
43	static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48};
44	static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48};
45
46	#define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1)
47
48	static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt)
49	{
50	return container_of(pt, struct xe_pt_dir, pt);
51	}
52
53	static struct xe_pt *
54	xe_pt_entry_staging(struct xe_pt_dir *pt_dir, unsigned int index)
55	{
56	return container_of(pt_dir->staging[index], struct xe_pt, base);
57	}
58
59	static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,
60	unsigned int level)
61	{
62	struct xe_device *xe = tile_to_xe(tile);
63	u16 pat_index = xe->pat.idx[XE_CACHE_WB];
64	u8 id = tile->id;
65
66	if (!xe_vm_has_scratch(vm))
67	return 0;
68
69	if (level > MAX_HUGEPTE_LEVEL)
70	return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo,
71	0);
72
73	return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) |
74	XE_PTE_NULL;
75	}
76
77	static void xe_pt_free(struct xe_pt *pt)
78	{
79	if (pt->level)
80	kfree(objp: as_xe_pt_dir(pt));
81	else
82	kfree(objp: pt);
83	}
84
85	/**
86	* xe_pt_create() - Create a page-table.
87	* @vm: The vm to create for.
88	* @tile: The tile to create for.
89	* @level: The page-table level.
90	* @exec: The drm_exec object used to lock the vm.
91	*
92	* Allocate and initialize a single struct xe_pt metadata structure. Also
93	* create the corresponding page-table bo, but don't initialize it. If the
94	* level is grater than zero, then it's assumed to be a directory page-
95	* table and the directory structure is also allocated and initialized to
96	* NULL pointers.
97	*
98	* Return: A valid struct xe_pt pointer on success, Pointer error code on
99	* error.
100	*/
101	struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
102	unsigned int level, struct drm_exec *exec)
103	{
104	struct xe_pt *pt;
105	struct xe_bo *bo;
106	u32 bo_flags;
107	int err;
108
109	if (level) {
110	struct xe_pt_dir *dir = kzalloc(sizeof(*dir), GFP_KERNEL);
111
112	pt = (dir) ? &dir->pt : NULL;
113	} else {
114	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
115	}
116	if (!pt)
117	return ERR_PTR(error: -ENOMEM);
118
119	bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) |
120	XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE |
121	XE_BO_FLAG_NO_RESV_EVICT | XE_BO_FLAG_PAGETABLE;
122	if (vm->xef) /* userspace */
123	bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM;
124
125	pt->level = level;
126
127	drm_WARN_ON(&vm->xe->drm, IS_ERR_OR_NULL(exec));
128	bo = xe_bo_create_pin_map(xe: vm->xe, tile, vm, SZ_4K,
129	type: ttm_bo_type_kernel,
130	flags: bo_flags, exec);
131	if (IS_ERR(ptr: bo)) {
132	err = PTR_ERR(ptr: bo);
133	goto err_kfree;
134	}
135	pt->bo = bo;
136	pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL;
137	pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL;
138
139	if (vm->xef)
140	xe_drm_client_add_bo(client: vm->xef->client, bo: pt->bo);
141	xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL);
142
143	return pt;
144
145	err_kfree:
146	xe_pt_free(pt);
147	return ERR_PTR(error: err);
148	}
149	ALLOW_ERROR_INJECTION(xe_pt_create, ERRNO);
150
151	/**
152	* xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero
153	* entries.
154	* @tile: The tile the scratch pagetable of which to use.
155	* @vm: The vm we populate for.
156	* @pt: The pagetable the bo of which to initialize.
157	*
158	* Populate the page-table bo of @pt with entries pointing into the tile's
159	* scratch page-table tree if any. Otherwise populate with zeros.
160	*/
161	void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,
162	struct xe_pt *pt)
163	{
164	struct iosys_map *map = &pt->bo->vmap;
165	u64 empty;
166	int i;
167
168	if (!xe_vm_has_scratch(vm)) {
169	/*
170	* FIXME: Some memory is allocated already allocated to zero?
171	* Find out which memory that is and avoid this memset...
172	*/
173	xe_map_memset(xe: vm->xe, dst: map, offset: 0, value: 0, SZ_4K);
174	} else {
175	empty = __xe_pt_empty_pte(tile, vm, level: pt->level);
176	for (i = 0; i < XE_PDES; i++)
177	xe_pt_write(vm->xe, map, i, empty);
178	}
179	}
180
181	/**
182	* xe_pt_shift() - Return the ilog2 value of the size of the address range of
183	* a page-table at a certain level.
184	* @level: The level.
185	*
186	* Return: The ilog2 value of the size of the address range of a page-table
187	* at level @level.
188	*/
189	unsigned int xe_pt_shift(unsigned int level)
190	{
191	return XE_PTE_SHIFT + XE_PDE_SHIFT * level;
192	}
193
194	/**
195	* xe_pt_destroy() - Destroy a page-table tree.
196	* @pt: The root of the page-table tree to destroy.
197	* @flags: vm flags. Currently unused.
198	* @deferred: List head of lockless list for deferred putting. NULL for
199	* immediate putting.
200	*
201	* Puts the page-table bo, recursively calls xe_pt_destroy on all children
202	* and finally frees @pt. TODO: Can we remove the @flags argument?
203	*/
204	void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
205	{
206	int i;
207
208	if (!pt)
209	return;
210
211	XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list));
212	xe_bo_unpin(bo: pt->bo);
213	xe_bo_put_deferred(bo: pt->bo, deferred);
214
215	if (pt->level > 0 && pt->num_live) {
216	struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
217
218	for (i = 0; i < XE_PDES; i++) {
219	if (xe_pt_entry_staging(pt_dir, index: i))
220	xe_pt_destroy(pt: xe_pt_entry_staging(pt_dir, index: i), flags,
221	deferred);
222	}
223	}
224	xe_pt_free(pt);
225	}
226
227	/**
228	* xe_pt_clear() - Clear a page-table.
229	* @xe: xe device.
230	* @pt: The page-table.
231	*
232	* Clears page-table by setting to zero.
233	*/
234	void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt)
235	{
236	struct iosys_map *map = &pt->bo->vmap;
237
238	xe_map_memset(xe, dst: map, offset: 0, value: 0, SZ_4K);
239	}
240
241	/**
242	* DOC: Pagetable building
243	*
244	* Below we use the term "page-table" for both page-directories, containing
245	* pointers to lower level page-directories or page-tables, and level 0
246	* page-tables that contain only page-table-entries pointing to memory pages.
247	*
248	* When inserting an address range in an already existing page-table tree
249	* there will typically be a set of page-tables that are shared with other
250	* address ranges, and a set that are private to this address range.
251	* The set of shared page-tables can be at most two per level,
252	* and those can't be updated immediately because the entries of those
253	* page-tables may still be in use by the gpu for other mappings. Therefore
254	* when inserting entries into those, we instead stage those insertions by
255	* adding insertion data into struct xe_vm_pgtable_update structures. This
256	* data, (subtrees for the cpu and page-table-entries for the gpu) is then
257	* added in a separate commit step. CPU-data is committed while still under the
258	* vm lock, the object lock and for userptr, the notifier lock in read mode.
259	* The GPU async data is committed either by the GPU or CPU after fulfilling
260	* relevant dependencies.
261	* For non-shared page-tables (and, in fact, for shared ones that aren't
262	* existing at the time of staging), we add the data in-place without the
263	* special update structures. This private part of the page-table tree will
264	* remain disconnected from the vm page-table tree until data is committed to
265	* the shared page tables of the vm tree in the commit phase.
266	*/
267
268	struct xe_pt_update {
269	/** @update: The update structure we're building for this parent. */
270	struct xe_vm_pgtable_update *update;
271	/** @parent: The parent. Used to detect a parent change. */
272	struct xe_pt *parent;
273	/** @preexisting: Whether the parent was pre-existing or allocated */
274	bool preexisting;
275	};
276
277	/**
278	* struct xe_pt_stage_bind_walk - Walk state for the stage_bind walk.
279	*/
280	struct xe_pt_stage_bind_walk {
281	/** @base: The base class. */
282	struct xe_pt_walk base;
283
284	/* Input parameters for the walk */
285	/** @vm: The vm we're building for. */
286	struct xe_vm *vm;
287	/** @tile: The tile we're building for. */
288	struct xe_tile *tile;
289	/** @default_vram_pte: PTE flag only template for VRAM. No address is associated */
290	u64 default_vram_pte;
291	/** @default_system_pte: PTE flag only template for System. No address is associated */
292	u64 default_system_pte;
293	/** @dma_offset: DMA offset to add to the PTE. */
294	u64 dma_offset;
295	/**
296	* @needs_64K: This address range enforces 64K alignment and
297	* granularity on VRAM.
298	*/
299	bool needs_64K;
300	/** @clear_pt: clear page table entries during the bind walk */
301	bool clear_pt;
302	/**
303	* @vma: VMA being mapped
304	*/
305	struct xe_vma *vma;
306
307	/* Also input, but is updated during the walk*/
308	/** @curs: The DMA address cursor. */
309	struct xe_res_cursor *curs;
310	/** @va_curs_start: The Virtual address corresponding to @curs->start */
311	u64 va_curs_start;
312
313	/* Output */
314	/** @wupd: Walk output data for page-table updates. */
315	struct xe_walk_update {
316	/** @wupd.entries: Caller provided storage. */
317	struct xe_vm_pgtable_update *entries;
318	/** @wupd.num_used_entries: Number of update @entries used. */
319	unsigned int num_used_entries;
320	/** @wupd.updates: Tracks the update entry at a given level */
321	struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1];
322	} wupd;
323
324	/* Walk state */
325	/**
326	* @l0_end_addr: The end address of the current l0 leaf. Used for
327	* 64K granularity detection.
328	*/
329	u64 l0_end_addr;
330	/** @addr_64K: The start address of the current 64K chunk. */
331	u64 addr_64K;
332	/** @found_64K: Whether @add_64K actually points to a 64K chunk. */
333	bool found_64K;
334	};
335
336	static int
337	xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
338	pgoff_t offset, bool alloc_entries)
339	{
340	struct xe_pt_update *upd = &wupd->updates[parent->level];
341	struct xe_vm_pgtable_update *entry;
342
343	/*
344	* For *each level*, we could only have one active
345	* struct xt_pt_update at any one time. Once we move on to a
346	* new parent and page-directory, the old one is complete, and
347	* updates are either already stored in the build tree or in
348	* @wupd->entries
349	*/
350	if (likely(upd->parent == parent))
351	return 0;
352
353	upd->parent = parent;
354	upd->preexisting = true;
355
356	if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1)
357	return -EINVAL;
358
359	entry = wupd->entries + wupd->num_used_entries++;
360	upd->update = entry;
361	entry->ofs = offset;
362	entry->pt_bo = parent->bo;
363	entry->pt = parent;
364	entry->flags = 0;
365	entry->qwords = 0;
366	entry->pt_bo->update_index = -1;
367
368	if (alloc_entries) {
369	entry->pt_entries = kmalloc_array(XE_PDES,
370	sizeof(*entry->pt_entries),
371	GFP_KERNEL);
372	if (!entry->pt_entries)
373	return -ENOMEM;
374	}
375
376	return 0;
377	}
378
379	/*
380	* NOTE: This is a very frequently called function so we allow ourselves
381	* to annotate (using branch prediction hints) the fastpath of updating a
382	* non-pre-existing pagetable with leaf ptes.
383	*/
384	static int
385	xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent,
386	pgoff_t offset, struct xe_pt *xe_child, u64 pte)
387	{
388	struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level];
389	struct xe_pt_update *child_upd = xe_child ?
390	&xe_walk->wupd.updates[xe_child->level] : NULL;
391	int ret;
392
393	ret = xe_pt_new_shared(wupd: &xe_walk->wupd, parent, offset, alloc_entries: true);
394	if (unlikely(ret))
395	return ret;
396
397	/*
398	* Register this new pagetable so that it won't be recognized as
399	* a shared pagetable by a subsequent insertion.
400	*/
401	if (unlikely(child_upd)) {
402	child_upd->update = NULL;
403	child_upd->parent = xe_child;
404	child_upd->preexisting = false;
405	}
406
407	if (likely(!upd->preexisting)) {
408	/* Continue building a non-connected subtree. */
409	struct iosys_map *map = &parent->bo->vmap;
410
411	if (unlikely(xe_child)) {
412	parent->base.children[offset] = &xe_child->base;
413	parent->base.staging[offset] = &xe_child->base;
414	}
415
416	xe_pt_write(xe_walk->vm->xe, map, offset, pte);
417	parent->num_live++;
418	} else {
419	/* Shared pt. Stage update. */
420	unsigned int idx;
421	struct xe_vm_pgtable_update *entry = upd->update;
422
423	idx = offset - entry->ofs;
424	entry->pt_entries[idx].pt = xe_child;
425	entry->pt_entries[idx].pte = pte;
426	entry->qwords++;
427	}
428
429	return 0;
430	}
431
432	static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level,
433	struct xe_pt_stage_bind_walk *xe_walk)
434	{
435	u64 size, dma;
436
437	if (level > MAX_HUGEPTE_LEVEL)
438	return false;
439
440	/* Does the virtual range requested cover a huge pte? */
441	if (!xe_pt_covers(addr, end: next, level, walk: &xe_walk->base))
442	return false;
443
444	/* Does the DMA segment cover the whole pte? */
445	if (next - xe_walk->va_curs_start > xe_walk->curs->size)
446	return false;
447
448	/* null VMA's do not have dma addresses */
449	if (xe_vma_is_null(vma: xe_walk->vma))
450	return true;
451
452	/* if we are clearing page table, no dma addresses*/
453	if (xe_walk->clear_pt)
454	return true;
455
456	/* Is the DMA address huge PTE size aligned? */
457	size = next - addr;
458	dma = addr - xe_walk->va_curs_start + xe_res_dma(cur: xe_walk->curs);
459
460	return IS_ALIGNED(dma, size);
461	}
462
463	/*
464	* Scan the requested mapping to check whether it can be done entirely
465	* with 64K PTEs.
466	*/
467	static bool
468	xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
469	{
470	struct xe_res_cursor curs = *xe_walk->curs;
471
472	if (!IS_ALIGNED(addr, SZ_64K))
473	return false;
474
475	if (next > xe_walk->l0_end_addr)
476	return false;
477
478	/* null VMA's do not have dma addresses */
479	if (xe_vma_is_null(vma: xe_walk->vma))
480	return true;
481
482	xe_res_next(cur: &curs, size: addr - xe_walk->va_curs_start);
483	for (; addr < next; addr += SZ_64K) {
484	if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K)
485	return false;
486
487	xe_res_next(cur: &curs, SZ_64K);
488	}
489
490	return addr == next;
491	}
492
493	/*
494	* For non-compact "normal" 4K level-0 pagetables, we want to try to group
495	* addresses together in 64K-contigous regions to add a 64K TLB hint for the
496	* device to the PTE.
497	* This function determines whether the address is part of such a
498	* segment. For VRAM in normal pagetables, this is strictly necessary on
499	* some devices.
500	*/
501	static bool
502	xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
503	{
504	/* Address is within an already found 64k region */
505	if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K)
506	return true;
507
508	xe_walk->found_64K = xe_pt_scan_64K(addr, next: addr + SZ_64K, xe_walk);
509	xe_walk->addr_64K = addr;
510
511	return xe_walk->found_64K;
512	}
513
514	static int
515	xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset,
516	unsigned int level, u64 addr, u64 next,
517	struct xe_ptw **child,
518	enum page_walk_action *action,
519	struct xe_pt_walk *walk)
520	{
521	struct xe_pt_stage_bind_walk *xe_walk =
522	container_of(walk, typeof(*xe_walk), base);
523	u16 pat_index = xe_walk->vma->attr.pat_index;
524	struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base);
525	struct xe_vm *vm = xe_walk->vm;
526	struct xe_pt *xe_child;
527	bool covers;
528	int ret = 0;
529	u64 pte;
530
531	/* Is this a leaf entry ?*/
532	if (level == 0 || xe_pt_hugepte_possible(addr, next, level, xe_walk)) {
533	struct xe_res_cursor *curs = xe_walk->curs;
534	bool is_null = xe_vma_is_null(vma: xe_walk->vma);
535	bool is_vram = is_null ? false : xe_res_is_vram(cur: curs);
536
537	XE_WARN_ON(xe_walk->va_curs_start != addr);
538
539	if (xe_walk->clear_pt) {
540	pte = 0;
541	} else {
542	pte = vm->pt_ops->pte_encode_vma(is_null ? 0 :
543	xe_res_dma(cur: curs) +
544	xe_walk->dma_offset,
545	xe_walk->vma,
546	pat_index, level);
547	if (!is_null)
548	pte |= is_vram ? xe_walk->default_vram_pte :
549	xe_walk->default_system_pte;
550
551	/*
552	* Set the XE_PTE_PS64 hint if possible, otherwise if
553	* this device *requires* 64K PTE size for VRAM, fail.
554	*/
555	if (level == 0 && !xe_parent->is_compact) {
556	if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) {
557	xe_walk->vma->gpuva.flags |=
558	XE_VMA_PTE_64K;
559	pte |= XE_PTE_PS64;
560	} else if (XE_WARN_ON(xe_walk->needs_64K &&
561	is_vram)) {
562	return -EINVAL;
563	}
564	}
565	}
566
567	ret = xe_pt_insert_entry(xe_walk, parent: xe_parent, offset, NULL, pte);
568	if (unlikely(ret))
569	return ret;
570
571	if (!is_null && !xe_walk->clear_pt)
572	xe_res_next(cur: curs, size: next - addr);
573	xe_walk->va_curs_start = next;
574	xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level);
575	*action = ACTION_CONTINUE;
576
577	return ret;
578	}
579
580	/*
581	* Descending to lower level. Determine if we need to allocate a
582	* new page table or -directory, which we do if there is no
583	* previous one or there is one we can completely replace.
584	*/
585	if (level == 1) {
586	walk->shifts = xe_normal_pt_shifts;
587	xe_walk->l0_end_addr = next;
588	}
589
590	covers = xe_pt_covers(addr, end: next, level, walk: &xe_walk->base);
591	if (covers || !*child) {
592	u64 flags = 0;
593
594	xe_child = xe_pt_create(vm: xe_walk->vm, tile: xe_walk->tile, level: level - 1,
595	exec: xe_vm_validation_exec(vm));
596	if (IS_ERR(ptr: xe_child))
597	return PTR_ERR(ptr: xe_child);
598
599	xe_pt_set_addr(xe_child,
600	round_down(addr, 1ull << walk->shifts[level]));
601
602	if (!covers)
603	xe_pt_populate_empty(tile: xe_walk->tile, vm: xe_walk->vm, pt: xe_child);
604
605	*child = &xe_child->base;
606
607	/*
608	* Prefer the compact pagetable layout for L0 if possible. Only
609	* possible if VMA covers entire 2MB region as compact 64k and
610	* 4k pages cannot be mixed within a 2MB region.
611	* TODO: Suballocate the pt bo to avoid wasting a lot of
612	* memory.
613	*/
614	if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 &&
615	covers && xe_pt_scan_64K(addr, next, xe_walk)) {
616	walk->shifts = xe_compact_pt_shifts;
617	xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT;
618	flags |= XE_PDE_64K;
619	xe_child->is_compact = true;
620	}
621
622	pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0) | flags;
623	ret = xe_pt_insert_entry(xe_walk, parent: xe_parent, offset, xe_child,
624	pte);
625	}
626
627	*action = ACTION_SUBTREE;
628	return ret;
629	}
630
631	static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = {
632	.pt_entry = xe_pt_stage_bind_entry,
633	};
634
635	/*
636	* Default atomic expectations for different allocation scenarios are as follows:
637	*
638	* 1. Traditional API: When the VM is not in LR mode:
639	* - Device atomics are expected to function with all allocations.
640	*
641	* 2. Compute/SVM API: When the VM is in LR mode:
642	* - Device atomics are the default behavior when the bo is placed in a single region.
643	* - In all other cases device atomics will be disabled with AE=0 until an application
644	* request differently using a ioctl like madvise.
645	*/
646	static bool xe_atomic_for_vram(struct xe_vm *vm, struct xe_vma *vma)
647	{
648	if (vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
649	return false;
650
651	return true;
652	}
653
654	static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_vma *vma)
655	{
656	struct xe_device *xe = vm->xe;
657	struct xe_bo *bo = xe_vma_bo(vma);
658
659	if (!xe->info.has_device_atomics_on_smem ||
660	vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
661	return false;
662
663	if (vma->attr.atomic_access == DRM_XE_ATOMIC_DEVICE)
664	return true;
665
666	/*
667	* If a SMEM+LMEM allocation is backed by SMEM, a device
668	* atomics will cause a gpu page fault and which then
669	* gets migrated to LMEM, bind such allocations with
670	* device atomics enabled.
671	*/
672	return (!IS_DGFX(xe) || (!xe_vm_in_lr_mode(vm) ||
673	(bo && xe_bo_has_single_placement(bo))));
674	}
675
676	/**
677	* xe_pt_stage_bind() - Build a disconnected page-table tree for a given address
678	* range.
679	* @tile: The tile we're building for.
680	* @vma: The vma indicating the address range.
681	* @range: The range indicating the address range.
682	* @entries: Storage for the update entries used for connecting the tree to
683	* the main tree at commit time.
684	* @num_entries: On output contains the number of @entries used.
685	* @clear_pt: Clear the page table entries.
686	*
687	* This function builds a disconnected page-table tree for a given address
688	* range. The tree is connected to the main vm tree for the gpu using
689	* xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind().
690	* The function builds xe_vm_pgtable_update structures for already existing
691	* shared page-tables, and non-existing shared and non-shared page-tables
692	* are built and populated directly.
693	*
694	* Return 0 on success, negative error code on error.
695	*/
696	static int
697	xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
698	struct xe_svm_range *range,
699	struct xe_vm_pgtable_update *entries,
700	u32 *num_entries, bool clear_pt)
701	{
702	struct xe_device *xe = tile_to_xe(tile);
703	struct xe_bo *bo = xe_vma_bo(vma);
704	struct xe_res_cursor curs;
705	struct xe_vm *vm = xe_vma_vm(vma);
706	struct xe_pt_stage_bind_walk xe_walk = {
707	.base = {
708	.ops = &xe_pt_stage_bind_ops,
709	.shifts = xe_normal_pt_shifts,
710	.max_level = XE_PT_HIGHEST_LEVEL,
711	.staging = true,
712	},
713	.vm = vm,
714	.tile = tile,
715	.curs = &curs,
716	.va_curs_start = range ? xe_svm_range_start(range) :
717	xe_vma_start(vma),
718	.vma = vma,
719	.wupd.entries = entries,
720	.clear_pt = clear_pt,
721	};
722	struct xe_pt *pt = vm->pt_root[tile->id];
723	int ret;
724
725	if (range) {
726	/* Move this entire thing to xe_svm.c? */
727	xe_svm_notifier_lock(vm);
728	if (!xe_svm_range_pages_valid(range)) {
729	xe_svm_range_debug(range, operation: "BIND PREPARE - RETRY");
730	xe_svm_notifier_unlock(vm);
731	return -EAGAIN;
732	}
733	if (xe_svm_range_has_dma_mapping(range)) {
734	xe_res_first_dma(dma_addr: range->base.pages.dma_addr, start: 0,
735	size: xe_svm_range_size(range),
736	cur: &curs);
737	xe_svm_range_debug(range, operation: "BIND PREPARE - MIXED");
738	} else {
739	xe_assert(xe, false);
740	}
741	/*
742	* Note, when unlocking the resource cursor dma addresses may become
743	* stale, but the bind will be aborted anyway at commit time.
744	*/
745	xe_svm_notifier_unlock(vm);
746	}
747
748	xe_walk.needs_64K = (vm->flags & XE_VM_FLAG_64K);
749	if (clear_pt)
750	goto walk_pt;
751
752	if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) {
753	xe_walk.default_vram_pte = xe_atomic_for_vram(vm, vma) ? XE_USM_PPGTT_PTE_AE : 0;
754	xe_walk.default_system_pte = xe_atomic_for_system(vm, vma) ?
755	XE_USM_PPGTT_PTE_AE : 0;
756	}
757
758	xe_walk.default_vram_pte |= XE_PPGTT_PTE_DM;
759	xe_walk.dma_offset = bo ? vram_region_gpu_offset(res: bo->ttm.resource) : 0;
760	if (!range)
761	xe_bo_assert_held(bo);
762
763	if (!xe_vma_is_null(vma) && !range) {
764	if (xe_vma_is_userptr(vma))
765	xe_res_first_dma(dma_addr: to_userptr_vma(vma)->userptr.pages.dma_addr, start: 0,
766	size: xe_vma_size(vma), cur: &curs);
767	else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
768	xe_res_first(res: bo->ttm.resource, start: xe_vma_bo_offset(vma),
769	size: xe_vma_size(vma), cur: &curs);
770	else
771	xe_res_first_sg(sg: xe_bo_sg(bo), start: xe_vma_bo_offset(vma),
772	size: xe_vma_size(vma), cur: &curs);
773	} else if (!range) {
774	curs.size = xe_vma_size(vma);
775	}
776
777	walk_pt:
778	ret = xe_pt_walk_range(parent: &pt->base, level: pt->level,
779	addr: range ? xe_svm_range_start(range) : xe_vma_start(vma),
780	end: range ? xe_svm_range_end(range) : xe_vma_end(vma),
781	walk: &xe_walk.base);
782
783	*num_entries = xe_walk.wupd.num_used_entries;
784	return ret;
785	}
786
787	/**
788	* xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a
789	* shared pagetable.
790	* @addr: The start address within the non-shared pagetable.
791	* @end: The end address within the non-shared pagetable.
792	* @level: The level of the non-shared pagetable.
793	* @walk: Walk info. The function adjusts the walk action.
794	* @action: next action to perform (see enum page_walk_action)
795	* @offset: Ignored on input, First non-shared entry on output.
796	* @end_offset: Ignored on input, Last non-shared entry + 1 on output.
797	*
798	* A non-shared page-table has some entries that belong to the address range
799	* and others that don't. This function determines the entries that belong
800	* fully to the address range. Depending on level, some entries may
801	* partially belong to the address range (that can't happen at level 0).
802	* The function detects that and adjust those offsets to not include those
803	* partial entries. Iff it does detect partial entries, we know that there must
804	* be shared page tables also at lower levels, so it adjusts the walk action
805	* accordingly.
806	*
807	* Return: true if there were non-shared entries, false otherwise.
808	*/
809	static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level,
810	struct xe_pt_walk *walk,
811	enum page_walk_action *action,
812	pgoff_t *offset, pgoff_t *end_offset)
813	{
814	u64 size = 1ull << walk->shifts[level];
815
816	*offset = xe_pt_offset(addr, level, walk);
817	*end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset;
818
819	if (!level)
820	return true;
821
822	/*
823	* If addr or next are not size aligned, there are shared pts at lower
824	* level, so in that case traverse down the subtree
825	*/
826	*action = ACTION_CONTINUE;
827	if (!IS_ALIGNED(addr, size)) {
828	*action = ACTION_SUBTREE;
829	(*offset)++;
830	}
831
832	if (!IS_ALIGNED(end, size)) {
833	*action = ACTION_SUBTREE;
834	(*end_offset)--;
835	}
836
837	return *end_offset > *offset;
838	}
839
840	struct xe_pt_zap_ptes_walk {
841	/** @base: The walk base-class */
842	struct xe_pt_walk base;
843
844	/* Input parameters for the walk */
845	/** @tile: The tile we're building for */
846	struct xe_tile *tile;
847
848	/* Output */
849	/** @needs_invalidate: Whether we need to invalidate TLB*/
850	bool needs_invalidate;
851	};
852
853	static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset,
854	unsigned int level, u64 addr, u64 next,
855	struct xe_ptw **child,
856	enum page_walk_action *action,
857	struct xe_pt_walk *walk)
858	{
859	struct xe_pt_zap_ptes_walk *xe_walk =
860	container_of(walk, typeof(*xe_walk), base);
861	struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
862	pgoff_t end_offset;
863
864	XE_WARN_ON(!*child);
865	XE_WARN_ON(!level);
866
867	/*
868	* Note that we're called from an entry callback, and we're dealing
869	* with the child of that entry rather than the parent, so need to
870	* adjust level down.
871	*/
872	if (xe_pt_nonshared_offsets(addr, end: next, level: --level, walk, action, offset: &offset,
873	end_offset: &end_offset)) {
874	xe_map_memset(tile_to_xe(xe_walk->tile), dst: &xe_child->bo->vmap,
875	offset: offset * sizeof(u64), value: 0,
876	len: (end_offset - offset) * sizeof(u64));
877	xe_walk->needs_invalidate = true;
878	}
879
880	return 0;
881	}
882
883	static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = {
884	.pt_entry = xe_pt_zap_ptes_entry,
885	};
886
887	/**
888	* xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range
889	* @tile: The tile we're zapping for.
890	* @vma: GPU VMA detailing address range.
891	*
892	* Eviction and Userptr invalidation needs to be able to zap the
893	* gpu ptes of a given address range in pagefaulting mode.
894	* In order to be able to do that, that function needs access to the shared
895	* page-table entrieaso it can either clear the leaf PTEs or
896	* clear the pointers to lower-level page-tables. The caller is required
897	* to hold the necessary locks to ensure neither the page-table connectivity
898	* nor the page-table entries of the range is updated from under us.
899	*
900	* Return: Whether ptes were actually updated and a TLB invalidation is
901	* required.
902	*/
903	bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
904	{
905	struct xe_pt_zap_ptes_walk xe_walk = {
906	.base = {
907	.ops = &xe_pt_zap_ptes_ops,
908	.shifts = xe_normal_pt_shifts,
909	.max_level = XE_PT_HIGHEST_LEVEL,
910	},
911	.tile = tile,
912	};
913	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
914	u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated);
915
916	if (xe_vma_bo(vma))
917	xe_bo_assert_held(bo: xe_vma_bo(vma));
918	else if (xe_vma_is_userptr(vma))
919	lockdep_assert_held(&xe_vma_vm(vma)->svm.gpusvm.notifier_lock);
920
921	if (!(pt_mask & BIT(tile->id)))
922	return false;
923
924	(void)xe_pt_walk_shared(parent: &pt->base, level: pt->level, addr: xe_vma_start(vma),
925	end: xe_vma_end(vma), walk: &xe_walk.base);
926
927	return xe_walk.needs_invalidate;
928	}
929
930	/**
931	* xe_pt_zap_ptes_range() - Zap (zero) gpu ptes of a SVM range
932	* @tile: The tile we're zapping for.
933	* @vm: The VM we're zapping for.
934	* @range: The SVM range we're zapping for.
935	*
936	* SVM invalidation needs to be able to zap the gpu ptes of a given address
937	* range. In order to be able to do that, that function needs access to the
938	* shared page-table entries so it can either clear the leaf PTEs or
939	* clear the pointers to lower-level page-tables. The caller is required
940	* to hold the SVM notifier lock.
941	*
942	* Return: Whether ptes were actually updated and a TLB invalidation is
943	* required.
944	*/
945	bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm,
946	struct xe_svm_range *range)
947	{
948	struct xe_pt_zap_ptes_walk xe_walk = {
949	.base = {
950	.ops = &xe_pt_zap_ptes_ops,
951	.shifts = xe_normal_pt_shifts,
952	.max_level = XE_PT_HIGHEST_LEVEL,
953	},
954	.tile = tile,
955	};
956	struct xe_pt *pt = vm->pt_root[tile->id];
957	u8 pt_mask = (range->tile_present & ~range->tile_invalidated);
958
959	/*
960	* Locking rules:
961	*
962	* - notifier_lock (write): full protection against page table changes
963	* and MMU notifier invalidations.
964	*
965	* - notifier_lock (read) + vm_lock (write): combined protection against
966	* invalidations and concurrent page table modifications. (e.g., madvise)
967	*
968	*/
969	lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) ||
970	(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) &&
971	lockdep_is_held_type(&vm->lock, 0)));
972
973	if (!(pt_mask & BIT(tile->id)))
974	return false;
975
976	(void)xe_pt_walk_shared(parent: &pt->base, level: pt->level, addr: xe_svm_range_start(range),
977	end: xe_svm_range_end(range), walk: &xe_walk.base);
978
979	return xe_walk.needs_invalidate;
980	}
981
982	static void
983	xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,
984	struct iosys_map *map, void *data,
985	u32 qword_ofs, u32 num_qwords,
986	const struct xe_vm_pgtable_update *update)
987	{
988	struct xe_pt_entry *ptes = update->pt_entries;
989	u64 *ptr = data;
990	u32 i;
991
992	for (i = 0; i < num_qwords; i++) {
993	if (map)
994	xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
995	sizeof(u64), u64, ptes[i].pte);
996	else
997	ptr[i] = ptes[i].pte;
998	}
999	}
1000
1001	static void xe_pt_cancel_bind(struct xe_vma *vma,
1002	struct xe_vm_pgtable_update *entries,
1003	u32 num_entries)
1004	{
1005	u32 i, j;
1006
1007	for (i = 0; i < num_entries; i++) {
1008	struct xe_pt *pt = entries[i].pt;
1009
1010	if (!pt)
1011	continue;
1012
1013	if (pt->level) {
1014	for (j = 0; j < entries[i].qwords; j++)
1015	xe_pt_destroy(pt: entries[i].pt_entries[j].pt,
1016	flags: xe_vma_vm(vma)->flags, NULL);
1017	}
1018
1019	kfree(objp: entries[i].pt_entries);
1020	entries[i].pt_entries = NULL;
1021	entries[i].qwords = 0;
1022	}
1023	}
1024
1025	#define XE_INVALID_VMA ((struct xe_vma *)(0xdeaddeadull))
1026
1027	static void xe_pt_commit_prepare_locks_assert(struct xe_vma *vma)
1028	{
1029	struct xe_vm *vm;
1030
1031	if (vma == XE_INVALID_VMA)
1032	return;
1033
1034	vm = xe_vma_vm(vma);
1035	lockdep_assert_held(&vm->lock);
1036
1037	if (!xe_vma_has_no_bo(vma))
1038	dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);
1039
1040	xe_vm_assert_held(vm);
1041	}
1042
1043	static void xe_pt_commit_locks_assert(struct xe_vma *vma)
1044	{
1045	struct xe_vm *vm;
1046
1047	if (vma == XE_INVALID_VMA)
1048	return;
1049
1050	vm = xe_vma_vm(vma);
1051	xe_pt_commit_prepare_locks_assert(vma);
1052
1053	if (xe_vma_is_userptr(vma))
1054	xe_svm_assert_held_read(vm);
1055	}
1056
1057	static void xe_pt_commit(struct xe_vma *vma,
1058	struct xe_vm_pgtable_update *entries,
1059	u32 num_entries, struct llist_head *deferred)
1060	{
1061	u32 i, j;
1062
1063	xe_pt_commit_locks_assert(vma);
1064
1065	for (i = 0; i < num_entries; i++) {
1066	struct xe_pt *pt = entries[i].pt;
1067	struct xe_pt_dir *pt_dir;
1068
1069	if (!pt->level)
1070	continue;
1071
1072	pt_dir = as_xe_pt_dir(pt);
1073	for (j = 0; j < entries[i].qwords; j++) {
1074	struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
1075	int j_ = j + entries[i].ofs;
1076
1077	pt_dir->children[j_] = pt_dir->staging[j_];
1078	xe_pt_destroy(pt: oldpte, flags: (vma == XE_INVALID_VMA) ? 0 :
1079	xe_vma_vm(vma)->flags, deferred);
1080	}
1081	}
1082	}
1083
1084	static void xe_pt_abort_bind(struct xe_vma *vma,
1085	struct xe_vm_pgtable_update *entries,
1086	u32 num_entries, bool rebind)
1087	{
1088	int i, j;
1089
1090	xe_pt_commit_prepare_locks_assert(vma);
1091
1092	for (i = num_entries - 1; i >= 0; --i) {
1093	struct xe_pt *pt = entries[i].pt;
1094	struct xe_pt_dir *pt_dir;
1095
1096	if (!rebind)
1097	pt->num_live -= entries[i].qwords;
1098
1099	if (!pt->level)
1100	continue;
1101
1102	pt_dir = as_xe_pt_dir(pt);
1103	for (j = 0; j < entries[i].qwords; j++) {
1104	u32 j_ = j + entries[i].ofs;
1105	struct xe_pt *newpte = xe_pt_entry_staging(pt_dir, index: j_);
1106	struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
1107
1108	pt_dir->staging[j_] = oldpte ? &oldpte->base : 0;
1109	xe_pt_destroy(pt: newpte, flags: xe_vma_vm(vma)->flags, NULL);
1110	}
1111	}
1112	}
1113
1114	static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
1115	struct xe_vm_pgtable_update *entries,
1116	u32 num_entries, bool rebind)
1117	{
1118	u32 i, j;
1119
1120	xe_pt_commit_prepare_locks_assert(vma);
1121
1122	for (i = 0; i < num_entries; i++) {
1123	struct xe_pt *pt = entries[i].pt;
1124	struct xe_pt_dir *pt_dir;
1125
1126	if (!rebind)
1127	pt->num_live += entries[i].qwords;
1128
1129	if (!pt->level)
1130	continue;
1131
1132	pt_dir = as_xe_pt_dir(pt);
1133	for (j = 0; j < entries[i].qwords; j++) {
1134	u32 j_ = j + entries[i].ofs;
1135	struct xe_pt *newpte = entries[i].pt_entries[j].pt;
1136	struct xe_pt *oldpte = NULL;
1137
1138	if (xe_pt_entry_staging(pt_dir, index: j_))
1139	oldpte = xe_pt_entry_staging(pt_dir, index: j_);
1140
1141	pt_dir->staging[j_] = &newpte->base;
1142	entries[i].pt_entries[j].pt = oldpte;
1143	}
1144	}
1145	}
1146
1147	static void xe_pt_free_bind(struct xe_vm_pgtable_update *entries,
1148	u32 num_entries)
1149	{
1150	u32 i;
1151
1152	for (i = 0; i < num_entries; i++)
1153	kfree(objp: entries[i].pt_entries);
1154	}
1155
1156	static int
1157	xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
1158	struct xe_svm_range *range,
1159	struct xe_vm_pgtable_update *entries,
1160	u32 *num_entries, bool invalidate_on_bind)
1161	{
1162	int err;
1163
1164	*num_entries = 0;
1165	err = xe_pt_stage_bind(tile, vma, range, entries, num_entries,
1166	clear_pt: invalidate_on_bind);
1167	if (!err)
1168	xe_tile_assert(tile, *num_entries);
1169
1170	return err;
1171	}
1172
1173	static void xe_vm_dbg_print_entries(struct xe_device *xe,
1174	const struct xe_vm_pgtable_update *entries,
1175	unsigned int num_entries, bool bind)
1176	#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM))
1177	{
1178	unsigned int i;
1179
1180	vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind",
1181	num_entries);
1182	for (i = 0; i < num_entries; i++) {
1183	const struct xe_vm_pgtable_update *entry = &entries[i];
1184	struct xe_pt *xe_pt = entry->pt;
1185	u64 page_size = 1ull << xe_pt_shift(level: xe_pt->level);
1186	u64 end;
1187	u64 start;
1188
1189	xe_assert(xe, !entry->pt->is_compact);
1190	start = entry->ofs * page_size;
1191	end = start + page_size * entry->qwords;
1192	vm_dbg(&xe->drm,
1193	"\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n",
1194	i, xe_pt->level, entry->ofs, entry->qwords,
1195	xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0);
1196	}
1197	}
1198	#else
1199	{}
1200	#endif
1201
1202	static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs)
1203	{
1204	int i;
1205
1206	for (i = 0; i < num_syncs; i++) {
1207	struct dma_fence *fence = syncs[i].fence;
1208
1209	if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1210	&fence->flags))
1211	return false;
1212	}
1213
1214	return true;
1215	}
1216
1217	static int job_test_add_deps(struct xe_sched_job *job,
1218	struct dma_resv *resv,
1219	enum dma_resv_usage usage)
1220	{
1221	if (!job) {
1222	if (!dma_resv_test_signaled(obj: resv, usage))
1223	return -ETIME;
1224
1225	return 0;
1226	}
1227
1228	return xe_sched_job_add_deps(job, resv, usage);
1229	}
1230
1231	static int vma_add_deps(struct xe_vma *vma, struct xe_sched_job *job)
1232	{
1233	struct xe_bo *bo = xe_vma_bo(vma);
1234
1235	xe_bo_assert_held(bo);
1236
1237	if (bo && !bo->vm)
1238	return job_test_add_deps(job, resv: bo->ttm.base.resv,
1239	usage: DMA_RESV_USAGE_KERNEL);
1240
1241	return 0;
1242	}
1243
1244	static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op,
1245	struct xe_sched_job *job)
1246	{
1247	int err = 0;
1248
1249	/*
1250	* No need to check for is_cpu_addr_mirror here as vma_add_deps is a
1251	* NOP if VMA is_cpu_addr_mirror
1252	*/
1253
1254	switch (op->base.op) {
1255	case DRM_GPUVA_OP_MAP:
1256	if (!op->map.immediate && xe_vm_in_fault_mode(vm))
1257	break;
1258
1259	err = vma_add_deps(vma: op->map.vma, job);
1260	break;
1261	case DRM_GPUVA_OP_REMAP:
1262	if (op->remap.prev)
1263	err = vma_add_deps(vma: op->remap.prev, job);
1264	if (!err && op->remap.next)
1265	err = vma_add_deps(vma: op->remap.next, job);
1266	break;
1267	case DRM_GPUVA_OP_UNMAP:
1268	break;
1269	case DRM_GPUVA_OP_PREFETCH:
1270	err = vma_add_deps(vma: gpuva_to_vma(gpuva: op->base.prefetch.va), job);
1271	break;
1272	case DRM_GPUVA_OP_DRIVER:
1273	break;
1274	default:
1275	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
1276	}
1277
1278	return err;
1279	}
1280
1281	static int xe_pt_vm_dependencies(struct xe_sched_job *job,
1282	struct xe_tlb_inval_job *ijob,
1283	struct xe_tlb_inval_job *mjob,
1284	struct xe_vm *vm,
1285	struct xe_vma_ops *vops,
1286	struct xe_vm_pgtable_update_ops *pt_update_ops,
1287	struct xe_range_fence_tree *rftree)
1288	{
1289	struct xe_range_fence *rtfence;
1290	struct dma_fence *fence;
1291	struct xe_vma_op *op;
1292	int err = 0, i;
1293
1294	xe_vm_assert_held(vm);
1295
1296	if (!job && !no_in_syncs(syncs: vops->syncs, num_syncs: vops->num_syncs))
1297	return -ETIME;
1298
1299	if (!job && !xe_exec_queue_is_idle(q: pt_update_ops->q))
1300	return -ETIME;
1301
1302	if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) {
1303	err = job_test_add_deps(job, resv: xe_vm_resv(vm),
1304	usage: pt_update_ops->wait_vm_bookkeep ?
1305	DMA_RESV_USAGE_BOOKKEEP :
1306	DMA_RESV_USAGE_KERNEL);
1307	if (err)
1308	return err;
1309	}
1310
1311	rtfence = xe_range_fence_tree_first(tree: rftree, start: pt_update_ops->start,
1312	last: pt_update_ops->last);
1313	while (rtfence) {
1314	fence = rtfence->fence;
1315
1316	if (!dma_fence_is_signaled(fence)) {
1317	/*
1318	* Is this a CPU update? GPU is busy updating, so return
1319	* an error
1320	*/
1321	if (!job)
1322	return -ETIME;
1323
1324	dma_fence_get(fence);
1325	err = drm_sched_job_add_dependency(job: &job->drm, fence);
1326	if (err)
1327	return err;
1328	}
1329
1330	rtfence = xe_range_fence_tree_next(rfence: rtfence,
1331	start: pt_update_ops->start,
1332	last: pt_update_ops->last);
1333	}
1334
1335	list_for_each_entry(op, &vops->list, link) {
1336	err = op_add_deps(vm, op, job);
1337	if (err)
1338	return err;
1339	}
1340
1341	for (i = 0; job && !err && i < vops->num_syncs; i++)
1342	err = xe_sync_entry_add_deps(sync: &vops->syncs[i], job);
1343
1344	if (job) {
1345	if (ijob) {
1346	err = xe_tlb_inval_job_alloc_dep(job: ijob);
1347	if (err)
1348	return err;
1349	}
1350
1351	if (mjob) {
1352	err = xe_tlb_inval_job_alloc_dep(job: mjob);
1353	if (err)
1354	return err;
1355	}
1356	}
1357
1358	return err;
1359	}
1360
1361	static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update)
1362	{
1363	struct xe_vma_ops *vops = pt_update->vops;
1364	struct xe_vm *vm = vops->vm;
1365	struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id];
1366	struct xe_vm_pgtable_update_ops *pt_update_ops =
1367	&vops->pt_update_ops[pt_update->tile_id];
1368
1369	return xe_pt_vm_dependencies(job: pt_update->job, ijob: pt_update->ijob,
1370	mjob: pt_update->mjob, vm, vops: pt_update->vops,
1371	pt_update_ops, rftree);
1372	}
1373
1374	#if IS_ENABLED(CONFIG_DRM_GPUSVM)
1375	#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT
1376
1377	static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
1378	{
1379	u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2;
1380	static u32 count;
1381
1382	if (count++ % divisor == divisor - 1) {
1383	uvma->userptr.divisor = divisor << 1;
1384	return true;
1385	}
1386
1387	return false;
1388	}
1389
1390	#else
1391
1392	static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
1393	{
1394	return false;
1395	}
1396
1397	#endif
1398
1399	static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,
1400	struct xe_vm_pgtable_update_ops *pt_update)
1401	{
1402	struct xe_userptr_vma *uvma;
1403	unsigned long notifier_seq;
1404
1405	xe_svm_assert_held_read(vm);
1406
1407	if (!xe_vma_is_userptr(vma))
1408	return 0;
1409
1410	uvma = to_userptr_vma(vma);
1411	if (xe_pt_userptr_inject_eagain(uvma))
1412	xe_vma_userptr_force_invalidate(uvma);
1413
1414	notifier_seq = uvma->userptr.pages.notifier_seq;
1415
1416	if (!mmu_interval_read_retry(interval_sub: &uvma->userptr.notifier,
1417	seq: notifier_seq))
1418	return 0;
1419
1420	if (xe_vm_in_fault_mode(vm))
1421	return -EAGAIN;
1422
1423	/*
1424	* Just continue the operation since exec or rebind worker
1425	* will take care of rebinding.
1426	*/
1427	return 0;
1428	}
1429
1430	static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op,
1431	struct xe_vm_pgtable_update_ops *pt_update)
1432	{
1433	int err = 0;
1434
1435	xe_svm_assert_held_read(vm);
1436
1437	switch (op->base.op) {
1438	case DRM_GPUVA_OP_MAP:
1439	if (!op->map.immediate && xe_vm_in_fault_mode(vm))
1440	break;
1441
1442	err = vma_check_userptr(vm, vma: op->map.vma, pt_update);
1443	break;
1444	case DRM_GPUVA_OP_REMAP:
1445	if (op->remap.prev)
1446	err = vma_check_userptr(vm, vma: op->remap.prev, pt_update);
1447	if (!err && op->remap.next)
1448	err = vma_check_userptr(vm, vma: op->remap.next, pt_update);
1449	break;
1450	case DRM_GPUVA_OP_UNMAP:
1451	break;
1452	case DRM_GPUVA_OP_PREFETCH:
1453	if (xe_vma_is_cpu_addr_mirror(vma: gpuva_to_vma(gpuva: op->base.prefetch.va))) {
1454	struct xe_svm_range *range = op->map_range.range;
1455	unsigned long i;
1456
1457	xe_assert(vm->xe,
1458	xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.prefetch.va)));
1459	xa_for_each(&op->prefetch_range.range, i, range) {
1460	xe_svm_range_debug(range, operation: "PRE-COMMIT");
1461
1462	if (!xe_svm_range_pages_valid(range)) {
1463	xe_svm_range_debug(range, operation: "PRE-COMMIT - RETRY");
1464	return -ENODATA;
1465	}
1466	}
1467	} else {
1468	err = vma_check_userptr(vm, vma: gpuva_to_vma(gpuva: op->base.prefetch.va), pt_update);
1469	}
1470	break;
1471	#if IS_ENABLED(CONFIG_DRM_XE_GPUSVM)
1472	case DRM_GPUVA_OP_DRIVER:
1473	if (op->subop == XE_VMA_SUBOP_MAP_RANGE) {
1474	struct xe_svm_range *range = op->map_range.range;
1475
1476	xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma));
1477
1478	xe_svm_range_debug(range, operation: "PRE-COMMIT");
1479
1480	if (!xe_svm_range_pages_valid(range)) {
1481	xe_svm_range_debug(range, operation: "PRE-COMMIT - RETRY");
1482	return -EAGAIN;
1483	}
1484	}
1485	break;
1486	#endif
1487	default:
1488	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
1489	}
1490
1491	return err;
1492	}
1493
1494	static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
1495	{
1496	struct xe_vm *vm = pt_update->vops->vm;
1497	struct xe_vma_ops *vops = pt_update->vops;
1498	struct xe_vm_pgtable_update_ops *pt_update_ops =
1499	&vops->pt_update_ops[pt_update->tile_id];
1500	struct xe_vma_op *op;
1501	int err;
1502
1503	err = xe_pt_pre_commit(pt_update);
1504	if (err)
1505	return err;
1506
1507	xe_svm_notifier_lock(vm);
1508
1509	list_for_each_entry(op, &vops->list, link) {
1510	err = op_check_svm_userptr(vm, op, pt_update: pt_update_ops);
1511	if (err) {
1512	xe_svm_notifier_unlock(vm);
1513	break;
1514	}
1515	}
1516
1517	return err;
1518	}
1519	#endif
1520
1521	struct xe_pt_stage_unbind_walk {
1522	/** @base: The pagewalk base-class. */
1523	struct xe_pt_walk base;
1524
1525	/* Input parameters for the walk */
1526	/** @tile: The tile we're unbinding from. */
1527	struct xe_tile *tile;
1528
1529	/**
1530	* @modified_start: Walk range start, modified to include any
1531	* shared pagetables that we're the only user of and can thus
1532	* treat as private.
1533	*/
1534	u64 modified_start;
1535	/** @modified_end: Walk range start, modified like @modified_start. */
1536	u64 modified_end;
1537
1538	/* Output */
1539	/* @wupd: Structure to track the page-table updates we're building */
1540	struct xe_walk_update wupd;
1541	};
1542
1543	/*
1544	* Check whether this range is the only one populating this pagetable,
1545	* and in that case, update the walk range checks so that higher levels don't
1546	* view us as a shared pagetable.
1547	*/
1548	static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level,
1549	const struct xe_pt *child,
1550	enum page_walk_action *action,
1551	struct xe_pt_walk *walk)
1552	{
1553	struct xe_pt_stage_unbind_walk *xe_walk =
1554	container_of(walk, typeof(*xe_walk), base);
1555	unsigned int shift = walk->shifts[level];
1556	u64 size = 1ull << shift;
1557
1558	if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) &&
1559	((next - addr) >> shift) == child->num_live) {
1560	u64 size = 1ull << walk->shifts[level + 1];
1561
1562	*action = ACTION_CONTINUE;
1563
1564	if (xe_walk->modified_start >= addr)
1565	xe_walk->modified_start = round_down(addr, size);
1566	if (xe_walk->modified_end <= next)
1567	xe_walk->modified_end = round_up(next, size);
1568
1569	return true;
1570	}
1571
1572	return false;
1573	}
1574
1575	static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
1576	unsigned int level, u64 addr, u64 next,
1577	struct xe_ptw **child,
1578	enum page_walk_action *action,
1579	struct xe_pt_walk *walk)
1580	{
1581	struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
1582
1583	XE_WARN_ON(!*child);
1584	XE_WARN_ON(!level);
1585
1586	xe_pt_check_kill(addr, next, level: level - 1, child: xe_child, action, walk);
1587
1588	return 0;
1589	}
1590
1591	static int
1592	xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset,
1593	unsigned int level, u64 addr, u64 next,
1594	struct xe_ptw **child,
1595	enum page_walk_action *action,
1596	struct xe_pt_walk *walk)
1597	{
1598	struct xe_pt_stage_unbind_walk *xe_walk =
1599	container_of(walk, typeof(*xe_walk), base);
1600	struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
1601	pgoff_t end_offset;
1602	u64 size = 1ull << walk->shifts[--level];
1603	int err;
1604
1605	if (!IS_ALIGNED(addr, size))
1606	addr = xe_walk->modified_start;
1607	if (!IS_ALIGNED(next, size))
1608	next = xe_walk->modified_end;
1609
1610	/* Parent == *child is the root pt. Don't kill it. */
1611	if (parent != *child &&
1612	xe_pt_check_kill(addr, next, level, child: xe_child, action, walk))
1613	return 0;
1614
1615	if (!xe_pt_nonshared_offsets(addr, end: next, level, walk, action, offset: &offset,
1616	end_offset: &end_offset))
1617	return 0;
1618
1619	err = xe_pt_new_shared(wupd: &xe_walk->wupd, parent: xe_child, offset, alloc_entries: true);
1620	if (err)
1621	return err;
1622
1623	xe_walk->wupd.updates[level].update->qwords = end_offset - offset;
1624
1625	return 0;
1626	}
1627
1628	static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = {
1629	.pt_entry = xe_pt_stage_unbind_entry,
1630	.pt_post_descend = xe_pt_stage_unbind_post_descend,
1631	};
1632
1633	/**
1634	* xe_pt_stage_unbind() - Build page-table update structures for an unbind
1635	* operation
1636	* @tile: The tile we're unbinding for.
1637	* @vm: The vm
1638	* @vma: The vma we're unbinding.
1639	* @range: The range we're unbinding.
1640	* @entries: Caller-provided storage for the update structures.
1641	*
1642	* Builds page-table update structures for an unbind operation. The function
1643	* will attempt to remove all page-tables that we're the only user
1644	* of, and for that to work, the unbind operation must be committed in the
1645	* same critical section that blocks racing binds to the same page-table tree.
1646	*
1647	* Return: The number of entries used.
1648	*/
1649	static unsigned int xe_pt_stage_unbind(struct xe_tile *tile,
1650	struct xe_vm *vm,
1651	struct xe_vma *vma,
1652	struct xe_svm_range *range,
1653	struct xe_vm_pgtable_update *entries)
1654	{
1655	u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma);
1656	u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma);
1657	struct xe_pt_stage_unbind_walk xe_walk = {
1658	.base = {
1659	.ops = &xe_pt_stage_unbind_ops,
1660	.shifts = xe_normal_pt_shifts,
1661	.max_level = XE_PT_HIGHEST_LEVEL,
1662	.staging = true,
1663	},
1664	.tile = tile,
1665	.modified_start = start,
1666	.modified_end = end,
1667	.wupd.entries = entries,
1668	};
1669	struct xe_pt *pt = vm->pt_root[tile->id];
1670
1671	(void)xe_pt_walk_shared(parent: &pt->base, level: pt->level, addr: start, end,
1672	walk: &xe_walk.base);
1673
1674	return xe_walk.wupd.num_used_entries;
1675	}
1676
1677	static void
1678	xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
1679	struct xe_tile *tile, struct iosys_map *map,
1680	void *ptr, u32 qword_ofs, u32 num_qwords,
1681	const struct xe_vm_pgtable_update *update)
1682	{
1683	struct xe_vm *vm = pt_update->vops->vm;
1684	u64 empty = __xe_pt_empty_pte(tile, vm, level: update->pt->level);
1685	int i;
1686
1687	if (map && map->is_iomem)
1688	for (i = 0; i < num_qwords; ++i)
1689	xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
1690	sizeof(u64), u64, empty);
1691	else if (map)
1692	memset64(s: map->vaddr + qword_ofs * sizeof(u64), v: empty,
1693	n: num_qwords);
1694	else
1695	memset64(s: ptr, v: empty, n: num_qwords);
1696	}
1697
1698	static void xe_pt_abort_unbind(struct xe_vma *vma,
1699	struct xe_vm_pgtable_update *entries,
1700	u32 num_entries)
1701	{
1702	int i, j;
1703
1704	xe_pt_commit_prepare_locks_assert(vma);
1705
1706	for (i = num_entries - 1; i >= 0; --i) {
1707	struct xe_vm_pgtable_update *entry = &entries[i];
1708	struct xe_pt *pt = entry->pt;
1709	struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
1710
1711	pt->num_live += entry->qwords;
1712
1713	if (!pt->level)
1714	continue;
1715
1716	for (j = entry->ofs; j < entry->ofs + entry->qwords; j++)
1717	pt_dir->staging[j] =
1718	entries[i].pt_entries[j - entry->ofs].pt ?
1719	&entries[i].pt_entries[j - entry->ofs].pt->base : NULL;
1720	}
1721	}
1722
1723	static void
1724	xe_pt_commit_prepare_unbind(struct xe_vma *vma,
1725	struct xe_vm_pgtable_update *entries,
1726	u32 num_entries)
1727	{
1728	int i, j;
1729
1730	xe_pt_commit_prepare_locks_assert(vma);
1731
1732	for (i = 0; i < num_entries; ++i) {
1733	struct xe_vm_pgtable_update *entry = &entries[i];
1734	struct xe_pt *pt = entry->pt;
1735	struct xe_pt_dir *pt_dir;
1736
1737	pt->num_live -= entry->qwords;
1738	if (!pt->level)
1739	continue;
1740
1741	pt_dir = as_xe_pt_dir(pt);
1742	for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) {
1743	entry->pt_entries[j - entry->ofs].pt =
1744	xe_pt_entry_staging(pt_dir, index: j);
1745	pt_dir->staging[j] = NULL;
1746	}
1747	}
1748	}
1749
1750	static void
1751	xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops,
1752	u64 start, u64 end)
1753	{
1754	u64 last;
1755	u32 current_op = pt_update_ops->current_op;
1756	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1757	int i, level = 0;
1758
1759	for (i = 0; i < pt_op->num_entries; i++) {
1760	const struct xe_vm_pgtable_update *entry = &pt_op->entries[i];
1761
1762	if (entry->pt->level > level)
1763	level = entry->pt->level;
1764	}
1765
1766	/* Greedy (non-optimal) calculation but simple */
1767	start = ALIGN_DOWN(start, 0x1ull << xe_pt_shift(level));
1768	last = ALIGN(end, 0x1ull << xe_pt_shift(level)) - 1;
1769
1770	if (start < pt_update_ops->start)
1771	pt_update_ops->start = start;
1772	if (last > pt_update_ops->last)
1773	pt_update_ops->last = last;
1774	}
1775
1776	static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma)
1777	{
1778	int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0;
1779
1780	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
1781	return dma_resv_reserve_fences(obj: xe_vma_bo(vma)->ttm.base.resv,
1782	num_fences: xe->info.tile_count << shift);
1783
1784	return 0;
1785	}
1786
1787	static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
1788	struct xe_vm_pgtable_update_ops *pt_update_ops,
1789	struct xe_vma *vma, bool invalidate_on_bind)
1790	{
1791	u32 current_op = pt_update_ops->current_op;
1792	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1793	int err;
1794
1795	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
1796	xe_bo_assert_held(bo: xe_vma_bo(vma));
1797
1798	vm_dbg(&xe_vma_vm(vma)->xe->drm,
1799	"Preparing bind, with range [%llx...%llx)\n",
1800	xe_vma_start(vma), xe_vma_end(vma) - 1);
1801
1802	pt_op->vma = NULL;
1803	pt_op->bind = true;
1804	pt_op->rebind = BIT(tile->id) & vma->tile_present;
1805
1806	err = vma_reserve_fences(tile_to_xe(tile), vma);
1807	if (err)
1808	return err;
1809
1810	err = xe_pt_prepare_bind(tile, vma, NULL, entries: pt_op->entries,
1811	num_entries: &pt_op->num_entries, invalidate_on_bind);
1812	if (!err) {
1813	xe_tile_assert(tile, pt_op->num_entries <=
1814	ARRAY_SIZE(pt_op->entries));
1815	xe_vm_dbg_print_entries(tile_to_xe(tile), entries: pt_op->entries,
1816	num_entries: pt_op->num_entries, bind: true);
1817
1818	xe_pt_update_ops_rfence_interval(pt_update_ops,
1819	start: xe_vma_start(vma),
1820	end: xe_vma_end(vma));
1821	++pt_update_ops->current_op;
1822	pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
1823
1824	/*
1825	* If rebind, we have to invalidate TLB on !LR vms to invalidate
1826	* cached PTEs point to freed memory. On LR vms this is done
1827	* automatically when the context is re-enabled by the rebind worker,
1828	* or in fault mode it was invalidated on PTE zapping.
1829	*
1830	* If !rebind, and scratch enabled VMs, there is a chance the scratch
1831	* PTE is already cached in the TLB so it needs to be invalidated.
1832	* On !LR VMs this is done in the ring ops preceding a batch, but on
1833	* LR, in particular on user-space batch buffer chaining, it needs to
1834	* be done here.
1835	*/
1836	if ((!pt_op->rebind && xe_vm_has_scratch(vm) &&
1837	xe_vm_in_lr_mode(vm)))
1838	pt_update_ops->needs_invalidation = true;
1839	else if (pt_op->rebind && !xe_vm_in_lr_mode(vm))
1840	/* We bump also if batch_invalidate_tlb is true */
1841	vm->tlb_flush_seqno++;
1842
1843	vma->tile_staged |= BIT(tile->id);
1844	pt_op->vma = vma;
1845	xe_pt_commit_prepare_bind(vma, entries: pt_op->entries,
1846	num_entries: pt_op->num_entries, rebind: pt_op->rebind);
1847	} else {
1848	xe_pt_cancel_bind(vma, entries: pt_op->entries, num_entries: pt_op->num_entries);
1849	}
1850
1851	return err;
1852	}
1853
1854	static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile,
1855	struct xe_vm_pgtable_update_ops *pt_update_ops,
1856	struct xe_vma *vma, struct xe_svm_range *range)
1857	{
1858	u32 current_op = pt_update_ops->current_op;
1859	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1860	int err;
1861
1862	xe_tile_assert(tile, xe_vma_is_cpu_addr_mirror(vma));
1863
1864	vm_dbg(&xe_vma_vm(vma)->xe->drm,
1865	"Preparing bind, with range [%lx...%lx)\n",
1866	xe_svm_range_start(range), xe_svm_range_end(range) - 1);
1867
1868	pt_op->vma = NULL;
1869	pt_op->bind = true;
1870	pt_op->rebind = BIT(tile->id) & range->tile_present;
1871
1872	err = xe_pt_prepare_bind(tile, vma, range, entries: pt_op->entries,
1873	num_entries: &pt_op->num_entries, invalidate_on_bind: false);
1874	if (!err) {
1875	xe_tile_assert(tile, pt_op->num_entries <=
1876	ARRAY_SIZE(pt_op->entries));
1877	xe_vm_dbg_print_entries(tile_to_xe(tile), entries: pt_op->entries,
1878	num_entries: pt_op->num_entries, bind: true);
1879
1880	xe_pt_update_ops_rfence_interval(pt_update_ops,
1881	start: xe_svm_range_start(range),
1882	end: xe_svm_range_end(range));
1883	++pt_update_ops->current_op;
1884	pt_update_ops->needs_svm_lock = true;
1885
1886	pt_op->vma = vma;
1887	xe_pt_commit_prepare_bind(vma, entries: pt_op->entries,
1888	num_entries: pt_op->num_entries, rebind: pt_op->rebind);
1889	} else {
1890	xe_pt_cancel_bind(vma, entries: pt_op->entries, num_entries: pt_op->num_entries);
1891	}
1892
1893	return err;
1894	}
1895
1896	static int unbind_op_prepare(struct xe_tile *tile,
1897	struct xe_vm_pgtable_update_ops *pt_update_ops,
1898	struct xe_vma *vma)
1899	{
1900	u32 current_op = pt_update_ops->current_op;
1901	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1902	int err;
1903
1904	if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))
1905	return 0;
1906
1907	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
1908	xe_bo_assert_held(bo: xe_vma_bo(vma));
1909
1910	vm_dbg(&xe_vma_vm(vma)->xe->drm,
1911	"Preparing unbind, with range [%llx...%llx)\n",
1912	xe_vma_start(vma), xe_vma_end(vma) - 1);
1913
1914	pt_op->vma = vma;
1915	pt_op->bind = false;
1916	pt_op->rebind = false;
1917
1918	err = vma_reserve_fences(tile_to_xe(tile), vma);
1919	if (err)
1920	return err;
1921
1922	pt_op->num_entries = xe_pt_stage_unbind(tile, vm: xe_vma_vm(vma),
1923	vma, NULL, entries: pt_op->entries);
1924
1925	xe_vm_dbg_print_entries(tile_to_xe(tile), entries: pt_op->entries,
1926	num_entries: pt_op->num_entries, bind: false);
1927	xe_pt_update_ops_rfence_interval(pt_update_ops, start: xe_vma_start(vma),
1928	end: xe_vma_end(vma));
1929	++pt_update_ops->current_op;
1930	pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
1931	pt_update_ops->needs_invalidation = true;
1932
1933	xe_pt_commit_prepare_unbind(vma, entries: pt_op->entries, num_entries: pt_op->num_entries);
1934
1935	return 0;
1936	}
1937
1938	static bool
1939	xe_pt_op_check_range_skip_invalidation(struct xe_vm_pgtable_update_op *pt_op,
1940	struct xe_svm_range *range)
1941	{
1942	struct xe_vm_pgtable_update *update = pt_op->entries;
1943
1944	XE_WARN_ON(!pt_op->num_entries);
1945
1946	/*
1947	* We can't skip the invalidation if we are removing PTEs that span more
1948	* than the range, do some checks to ensure we are removing PTEs that
1949	* are invalid.
1950	*/
1951
1952	if (pt_op->num_entries > 1)
1953	return false;
1954
1955	if (update->pt->level == 0)
1956	return true;
1957
1958	if (update->pt->level == 1)
1959	return xe_svm_range_size(range) >= SZ_2M;
1960
1961	return false;
1962	}
1963
1964	static int unbind_range_prepare(struct xe_vm *vm,
1965	struct xe_tile *tile,
1966	struct xe_vm_pgtable_update_ops *pt_update_ops,
1967	struct xe_svm_range *range)
1968	{
1969	u32 current_op = pt_update_ops->current_op;
1970	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
1971
1972	if (!(range->tile_present & BIT(tile->id)))
1973	return 0;
1974
1975	vm_dbg(&vm->xe->drm,
1976	"Preparing unbind, with range [%lx...%lx)\n",
1977	xe_svm_range_start(range), xe_svm_range_end(range) - 1);
1978
1979	pt_op->vma = XE_INVALID_VMA;
1980	pt_op->bind = false;
1981	pt_op->rebind = false;
1982
1983	pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range,
1984	entries: pt_op->entries);
1985
1986	xe_vm_dbg_print_entries(tile_to_xe(tile), entries: pt_op->entries,
1987	num_entries: pt_op->num_entries, bind: false);
1988	xe_pt_update_ops_rfence_interval(pt_update_ops, start: xe_svm_range_start(range),
1989	end: xe_svm_range_end(range));
1990	++pt_update_ops->current_op;
1991	pt_update_ops->needs_svm_lock = true;
1992	pt_update_ops->needs_invalidation |= xe_vm_has_scratch(vm) ||
1993	xe_vm_has_valid_gpu_mapping(tile, range->tile_present,
1994	range->tile_invalidated) ||
1995	!xe_pt_op_check_range_skip_invalidation(pt_op, range);
1996
1997	xe_pt_commit_prepare_unbind(XE_INVALID_VMA, entries: pt_op->entries,
1998	num_entries: pt_op->num_entries);
1999
2000	return 0;
2001	}
2002
2003	static int op_prepare(struct xe_vm *vm,
2004	struct xe_tile *tile,
2005	struct xe_vm_pgtable_update_ops *pt_update_ops,
2006	struct xe_vma_op *op)
2007	{
2008	int err = 0;
2009
2010	xe_vm_assert_held(vm);
2011
2012	switch (op->base.op) {
2013	case DRM_GPUVA_OP_MAP:
2014	if ((!op->map.immediate && xe_vm_in_fault_mode(vm) &&
2015	!op->map.invalidate_on_bind) ||
2016	(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR))
2017	break;
2018
2019	err = bind_op_prepare(vm, tile, pt_update_ops, vma: op->map.vma,
2020	invalidate_on_bind: op->map.invalidate_on_bind);
2021	pt_update_ops->wait_vm_kernel = true;
2022	break;
2023	case DRM_GPUVA_OP_REMAP:
2024	{
2025	struct xe_vma *old = gpuva_to_vma(gpuva: op->base.remap.unmap->va);
2026
2027	if (xe_vma_is_cpu_addr_mirror(vma: old))
2028	break;
2029
2030	err = unbind_op_prepare(tile, pt_update_ops, vma: old);
2031
2032	if (!err && op->remap.prev) {
2033	err = bind_op_prepare(vm, tile, pt_update_ops,
2034	vma: op->remap.prev, invalidate_on_bind: false);
2035	pt_update_ops->wait_vm_bookkeep = true;
2036	}
2037	if (!err && op->remap.next) {
2038	err = bind_op_prepare(vm, tile, pt_update_ops,
2039	vma: op->remap.next, invalidate_on_bind: false);
2040	pt_update_ops->wait_vm_bookkeep = true;
2041	}
2042	break;
2043	}
2044	case DRM_GPUVA_OP_UNMAP:
2045	{
2046	struct xe_vma *vma = gpuva_to_vma(gpuva: op->base.unmap.va);
2047
2048	if (xe_vma_is_cpu_addr_mirror(vma))
2049	break;
2050
2051	err = unbind_op_prepare(tile, pt_update_ops, vma);
2052	break;
2053	}
2054	case DRM_GPUVA_OP_PREFETCH:
2055	{
2056	struct xe_vma *vma = gpuva_to_vma(gpuva: op->base.prefetch.va);
2057
2058	if (xe_vma_is_cpu_addr_mirror(vma)) {
2059	struct xe_svm_range *range;
2060	unsigned long i;
2061
2062	xa_for_each(&op->prefetch_range.range, i, range) {
2063	err = bind_range_prepare(vm, tile, pt_update_ops,
2064	vma, range);
2065	if (err)
2066	return err;
2067	}
2068	} else {
2069	err = bind_op_prepare(vm, tile, pt_update_ops, vma, invalidate_on_bind: false);
2070	pt_update_ops->wait_vm_kernel = true;
2071	}
2072	break;
2073	}
2074	case DRM_GPUVA_OP_DRIVER:
2075	if (op->subop == XE_VMA_SUBOP_MAP_RANGE) {
2076	xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma));
2077
2078	err = bind_range_prepare(vm, tile, pt_update_ops,
2079	vma: op->map_range.vma,
2080	range: op->map_range.range);
2081	} else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) {
2082	err = unbind_range_prepare(vm, tile, pt_update_ops,
2083	range: op->unmap_range.range);
2084	}
2085	break;
2086	default:
2087	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2088	}
2089
2090	return err;
2091	}
2092
2093	static void
2094	xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
2095	{
2096	init_llist_head(list: &pt_update_ops->deferred);
2097	pt_update_ops->start = ~0x0ull;
2098	pt_update_ops->last = 0x0ull;
2099	}
2100
2101	/**
2102	* xe_pt_update_ops_prepare() - Prepare PT update operations
2103	* @tile: Tile of PT update operations
2104	* @vops: VMA operationa
2105	*
2106	* Prepare PT update operations which includes updating internal PT state,
2107	* allocate memory for page tables, populate page table being pruned in, and
2108	* create PT update operations for leaf insertion / removal.
2109	*
2110	* Return: 0 on success, negative error code on error.
2111	*/
2112	int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)
2113	{
2114	struct xe_vm_pgtable_update_ops *pt_update_ops =
2115	&vops->pt_update_ops[tile->id];
2116	struct xe_vma_op *op;
2117	int shift = tile->media_gt ? 1 : 0;
2118	int err;
2119
2120	lockdep_assert_held(&vops->vm->lock);
2121	xe_vm_assert_held(vops->vm);
2122
2123	xe_pt_update_ops_init(pt_update_ops);
2124
2125	err = dma_resv_reserve_fences(obj: xe_vm_resv(vm: vops->vm),
2126	tile_to_xe(tile)->info.tile_count << shift);
2127	if (err)
2128	return err;
2129
2130	list_for_each_entry(op, &vops->list, link) {
2131	err = op_prepare(vm: vops->vm, tile, pt_update_ops, op);
2132
2133	if (err)
2134	return err;
2135	}
2136
2137	xe_tile_assert(tile, pt_update_ops->current_op <=
2138	pt_update_ops->num_ops);
2139
2140	#ifdef TEST_VM_OPS_ERROR
2141	if (vops->inject_error &&
2142	vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE)
2143	return -ENOSPC;
2144	#endif
2145
2146	return 0;
2147	}
2148	ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO);
2149
2150	static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
2151	struct xe_vm_pgtable_update_ops *pt_update_ops,
2152	struct xe_vma *vma, struct dma_fence *fence,
2153	struct dma_fence *fence2, bool invalidate_on_bind)
2154	{
2155	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
2156
2157	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
2158	dma_resv_add_fence(obj: xe_vma_bo(vma)->ttm.base.resv, fence,
2159	usage: pt_update_ops->wait_vm_bookkeep ?
2160	DMA_RESV_USAGE_KERNEL :
2161	DMA_RESV_USAGE_BOOKKEEP);
2162	if (fence2)
2163	dma_resv_add_fence(obj: xe_vma_bo(vma)->ttm.base.resv, fence: fence2,
2164	usage: pt_update_ops->wait_vm_bookkeep ?
2165	DMA_RESV_USAGE_KERNEL :
2166	DMA_RESV_USAGE_BOOKKEEP);
2167	}
2168	/* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
2169	WRITE_ONCE(vma->tile_present, vma->tile_present | BIT(tile->id));
2170	if (invalidate_on_bind)
2171	WRITE_ONCE(vma->tile_invalidated,
2172	vma->tile_invalidated | BIT(tile->id));
2173	else
2174	WRITE_ONCE(vma->tile_invalidated,
2175	vma->tile_invalidated & ~BIT(tile->id));
2176	vma->tile_staged &= ~BIT(tile->id);
2177	if (xe_vma_is_userptr(vma)) {
2178	xe_svm_assert_held_read(vm);
2179	to_userptr_vma(vma)->userptr.initial_bind = true;
2180	}
2181
2182	/*
2183	* Kick rebind worker if this bind triggers preempt fences and not in
2184	* the rebind worker
2185	*/
2186	if (pt_update_ops->wait_vm_bookkeep &&
2187	xe_vm_in_preempt_fence_mode(vm) &&
2188	!current->mm)
2189	xe_vm_queue_rebind_worker(vm);
2190	}
2191
2192	static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
2193	struct xe_vm_pgtable_update_ops *pt_update_ops,
2194	struct xe_vma *vma, struct dma_fence *fence,
2195	struct dma_fence *fence2)
2196	{
2197	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
2198
2199	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
2200	dma_resv_add_fence(obj: xe_vma_bo(vma)->ttm.base.resv, fence,
2201	usage: pt_update_ops->wait_vm_bookkeep ?
2202	DMA_RESV_USAGE_KERNEL :
2203	DMA_RESV_USAGE_BOOKKEEP);
2204	if (fence2)
2205	dma_resv_add_fence(obj: xe_vma_bo(vma)->ttm.base.resv, fence: fence2,
2206	usage: pt_update_ops->wait_vm_bookkeep ?
2207	DMA_RESV_USAGE_KERNEL :
2208	DMA_RESV_USAGE_BOOKKEEP);
2209	}
2210	vma->tile_present &= ~BIT(tile->id);
2211	if (!vma->tile_present) {
2212	list_del_init(entry: &vma->combined_links.rebind);
2213	if (xe_vma_is_userptr(vma)) {
2214	xe_svm_assert_held_read(vm);
2215
2216	spin_lock(lock: &vm->userptr.invalidated_lock);
2217	list_del_init(entry: &to_userptr_vma(vma)->userptr.invalidate_link);
2218	spin_unlock(lock: &vm->userptr.invalidated_lock);
2219	}
2220	}
2221	}
2222
2223	static void range_present_and_invalidated_tile(struct xe_vm *vm,
2224	struct xe_svm_range *range,
2225	u8 tile_id)
2226	{
2227	/* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
2228
2229	lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);
2230
2231	WRITE_ONCE(range->tile_present, range->tile_present | BIT(tile_id));
2232	WRITE_ONCE(range->tile_invalidated, range->tile_invalidated & ~BIT(tile_id));
2233	}
2234
2235	static void op_commit(struct xe_vm *vm,
2236	struct xe_tile *tile,
2237	struct xe_vm_pgtable_update_ops *pt_update_ops,
2238	struct xe_vma_op *op, struct dma_fence *fence,
2239	struct dma_fence *fence2)
2240	{
2241	xe_vm_assert_held(vm);
2242
2243	switch (op->base.op) {
2244	case DRM_GPUVA_OP_MAP:
2245	if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) ||
2246	(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR))
2247	break;
2248
2249	bind_op_commit(vm, tile, pt_update_ops, vma: op->map.vma, fence,
2250	fence2, invalidate_on_bind: op->map.invalidate_on_bind);
2251	break;
2252	case DRM_GPUVA_OP_REMAP:
2253	{
2254	struct xe_vma *old = gpuva_to_vma(gpuva: op->base.remap.unmap->va);
2255
2256	if (xe_vma_is_cpu_addr_mirror(vma: old))
2257	break;
2258
2259	unbind_op_commit(vm, tile, pt_update_ops, vma: old, fence, fence2);
2260
2261	if (op->remap.prev)
2262	bind_op_commit(vm, tile, pt_update_ops, vma: op->remap.prev,
2263	fence, fence2, invalidate_on_bind: false);
2264	if (op->remap.next)
2265	bind_op_commit(vm, tile, pt_update_ops, vma: op->remap.next,
2266	fence, fence2, invalidate_on_bind: false);
2267	break;
2268	}
2269	case DRM_GPUVA_OP_UNMAP:
2270	{
2271	struct xe_vma *vma = gpuva_to_vma(gpuva: op->base.unmap.va);
2272
2273	if (!xe_vma_is_cpu_addr_mirror(vma))
2274	unbind_op_commit(vm, tile, pt_update_ops, vma, fence,
2275	fence2);
2276	break;
2277	}
2278	case DRM_GPUVA_OP_PREFETCH:
2279	{
2280	struct xe_vma *vma = gpuva_to_vma(gpuva: op->base.prefetch.va);
2281
2282	if (xe_vma_is_cpu_addr_mirror(vma)) {
2283	struct xe_svm_range *range = NULL;
2284	unsigned long i;
2285
2286	xa_for_each(&op->prefetch_range.range, i, range)
2287	range_present_and_invalidated_tile(vm, range, tile_id: tile->id);
2288	} else {
2289	bind_op_commit(vm, tile, pt_update_ops, vma, fence,
2290	fence2, invalidate_on_bind: false);
2291	}
2292	break;
2293	}
2294	case DRM_GPUVA_OP_DRIVER:
2295	{
2296	/* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
2297	if (op->subop == XE_VMA_SUBOP_MAP_RANGE)
2298	range_present_and_invalidated_tile(vm, range: op->map_range.range, tile_id: tile->id);
2299	else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE)
2300	WRITE_ONCE(op->unmap_range.range->tile_present,
2301	op->unmap_range.range->tile_present &
2302	~BIT(tile->id));
2303
2304	break;
2305	}
2306	default:
2307	drm_warn(&vm->xe->drm, "NOT POSSIBLE");
2308	}
2309	}
2310
2311	static const struct xe_migrate_pt_update_ops migrate_ops = {
2312	.populate = xe_vm_populate_pgtable,
2313	.clear = xe_migrate_clear_pgtable_callback,
2314	.pre_commit = xe_pt_pre_commit,
2315	};
2316
2317	#if IS_ENABLED(CONFIG_DRM_GPUSVM)
2318	static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops = {
2319	.populate = xe_vm_populate_pgtable,
2320	.clear = xe_migrate_clear_pgtable_callback,
2321	.pre_commit = xe_pt_svm_userptr_pre_commit,
2322	};
2323	#else
2324	static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops;
2325	#endif
2326
2327	static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q,
2328	struct xe_gt *gt)
2329	{
2330	if (xe_gt_is_media_type(gt))
2331	return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT].dep_scheduler;
2332
2333	return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT].dep_scheduler;
2334	}
2335
2336	/**
2337	* xe_pt_update_ops_run() - Run PT update operations
2338	* @tile: Tile of PT update operations
2339	* @vops: VMA operationa
2340	*
2341	* Run PT update operations which includes committing internal PT state changes,
2342	* creating job for PT update operations for leaf insertion / removal, and
2343	* installing job fence in various places.
2344	*
2345	* Return: fence on success, negative ERR_PTR on error.
2346	*/
2347	struct dma_fence *
2348	xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
2349	{
2350	struct xe_vm *vm = vops->vm;
2351	struct xe_vm_pgtable_update_ops *pt_update_ops =
2352	&vops->pt_update_ops[tile->id];
2353	struct xe_exec_queue *q = pt_update_ops->q;
2354	struct dma_fence *fence, *ifence = NULL, *mfence = NULL;
2355	struct xe_tlb_inval_job *ijob = NULL, *mjob = NULL;
2356	struct xe_range_fence *rfence;
2357	struct xe_vma_op *op;
2358	int err = 0, i;
2359	struct xe_migrate_pt_update update = {
2360	.ops = pt_update_ops->needs_svm_lock ?
2361	&svm_userptr_migrate_ops :
2362	&migrate_ops,
2363	.vops = vops,
2364	.tile_id = tile->id,
2365	};
2366
2367	lockdep_assert_held(&vm->lock);
2368	xe_vm_assert_held(vm);
2369
2370	if (!pt_update_ops->current_op) {
2371	xe_tile_assert(tile, xe_vm_in_fault_mode(vm));
2372
2373	return dma_fence_get_stub();
2374	}
2375
2376	#ifdef TEST_VM_OPS_ERROR
2377	if (vops->inject_error &&
2378	vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN)
2379	return ERR_PTR(-ENOSPC);
2380	#endif
2381
2382	if (pt_update_ops->needs_invalidation) {
2383	struct xe_dep_scheduler *dep_scheduler =
2384	to_dep_scheduler(q, gt: tile->primary_gt);
2385
2386	ijob = xe_tlb_inval_job_create(q, tlb_inval: &tile->primary_gt->tlb_inval,
2387	dep_scheduler, vm,
2388	start: pt_update_ops->start,
2389	end: pt_update_ops->last,
2390	XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
2391	if (IS_ERR(ptr: ijob)) {
2392	err = PTR_ERR(ptr: ijob);
2393	goto kill_vm_tile1;
2394	}
2395	update.ijob = ijob;
2396
2397	if (tile->media_gt) {
2398	dep_scheduler = to_dep_scheduler(q, gt: tile->media_gt);
2399
2400	mjob = xe_tlb_inval_job_create(q,
2401	tlb_inval: &tile->media_gt->tlb_inval,
2402	dep_scheduler, vm,
2403	start: pt_update_ops->start,
2404	end: pt_update_ops->last,
2405	XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT);
2406	if (IS_ERR(ptr: mjob)) {
2407	err = PTR_ERR(ptr: mjob);
2408	goto free_ijob;
2409	}
2410	update.mjob = mjob;
2411	}
2412	}
2413
2414	rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
2415	if (!rfence) {
2416	err = -ENOMEM;
2417	goto free_ijob;
2418	}
2419
2420	fence = xe_migrate_update_pgtables(m: tile->migrate, pt_update: &update);
2421	if (IS_ERR(ptr: fence)) {
2422	err = PTR_ERR(ptr: fence);
2423	goto free_rfence;
2424	}
2425
2426	/* Point of no return - VM killed if failure after this */
2427	for (i = 0; i < pt_update_ops->current_op; ++i) {
2428	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
2429
2430	xe_pt_commit(vma: pt_op->vma, entries: pt_op->entries,
2431	num_entries: pt_op->num_entries, deferred: &pt_update_ops->deferred);
2432	pt_op->vma = NULL; /* skip in xe_pt_update_ops_abort */
2433	}
2434
2435	if (xe_range_fence_insert(tree: &vm->rftree[tile->id], rfence,
2436	ops: &xe_range_fence_kfree_ops,
2437	start: pt_update_ops->start,
2438	end: pt_update_ops->last, fence))
2439	dma_fence_wait(fence, intr: false);
2440
2441	if (ijob)
2442	ifence = xe_tlb_inval_job_push(job: ijob, m: tile->migrate, fence);
2443	if (mjob)
2444	mfence = xe_tlb_inval_job_push(job: mjob, m: tile->migrate, fence);
2445
2446	if (!mjob && !ijob) {
2447	dma_resv_add_fence(obj: xe_vm_resv(vm), fence,
2448	usage: pt_update_ops->wait_vm_bookkeep ?
2449	DMA_RESV_USAGE_KERNEL :
2450	DMA_RESV_USAGE_BOOKKEEP);
2451
2452	list_for_each_entry(op, &vops->list, link)
2453	op_commit(vm: vops->vm, tile, pt_update_ops, op, fence, NULL);
2454	} else if (ijob && !mjob) {
2455	dma_resv_add_fence(obj: xe_vm_resv(vm), fence: ifence,
2456	usage: pt_update_ops->wait_vm_bookkeep ?
2457	DMA_RESV_USAGE_KERNEL :
2458	DMA_RESV_USAGE_BOOKKEEP);
2459
2460	list_for_each_entry(op, &vops->list, link)
2461	op_commit(vm: vops->vm, tile, pt_update_ops, op, fence: ifence, NULL);
2462	} else {
2463	dma_resv_add_fence(obj: xe_vm_resv(vm), fence: ifence,
2464	usage: pt_update_ops->wait_vm_bookkeep ?
2465	DMA_RESV_USAGE_KERNEL :
2466	DMA_RESV_USAGE_BOOKKEEP);
2467
2468	dma_resv_add_fence(obj: xe_vm_resv(vm), fence: mfence,
2469	usage: pt_update_ops->wait_vm_bookkeep ?
2470	DMA_RESV_USAGE_KERNEL :
2471	DMA_RESV_USAGE_BOOKKEEP);
2472
2473	list_for_each_entry(op, &vops->list, link)
2474	op_commit(vm: vops->vm, tile, pt_update_ops, op, fence: ifence,
2475	fence2: mfence);
2476	}
2477
2478	if (pt_update_ops->needs_svm_lock)
2479	xe_svm_notifier_unlock(vm);
2480
2481	/*
2482	* The last fence is only used for zero bind queue idling; migrate
2483	* queues are not exposed to user space.
2484	*/
2485	if (!(q->flags & EXEC_QUEUE_FLAG_MIGRATE))
2486	xe_exec_queue_last_fence_set(e: q, vm, fence);
2487
2488	xe_tlb_inval_job_put(job: mjob);
2489	xe_tlb_inval_job_put(job: ijob);
2490	dma_fence_put(fence: ifence);
2491	dma_fence_put(fence: mfence);
2492
2493	return fence;
2494
2495	free_rfence:
2496	kfree(objp: rfence);
2497	free_ijob:
2498	xe_tlb_inval_job_put(job: mjob);
2499	xe_tlb_inval_job_put(job: ijob);
2500	kill_vm_tile1:
2501	if (err != -EAGAIN && err != -ENODATA && tile->id)
2502	xe_vm_kill(vm: vops->vm, unlocked: false);
2503
2504	return ERR_PTR(error: err);
2505	}
2506	ALLOW_ERROR_INJECTION(xe_pt_update_ops_run, ERRNO);
2507
2508	/**
2509	* xe_pt_update_ops_fini() - Finish PT update operations
2510	* @tile: Tile of PT update operations
2511	* @vops: VMA operations
2512	*
2513	* Finish PT update operations by committing to destroy page table memory
2514	*/
2515	void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
2516	{
2517	struct xe_vm_pgtable_update_ops *pt_update_ops =
2518	&vops->pt_update_ops[tile->id];
2519	int i;
2520
2521	lockdep_assert_held(&vops->vm->lock);
2522	xe_vm_assert_held(vops->vm);
2523
2524	for (i = 0; i < pt_update_ops->current_op; ++i) {
2525	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
2526
2527	xe_pt_free_bind(entries: pt_op->entries, num_entries: pt_op->num_entries);
2528	}
2529	xe_bo_put_commit(deferred: &vops->pt_update_ops[tile->id].deferred);
2530	}
2531
2532	/**
2533	* xe_pt_update_ops_abort() - Abort PT update operations
2534	* @tile: Tile of PT update operations
2535	* @vops: VMA operationa
2536	*
2537	* Abort PT update operations by unwinding internal PT state
2538	*/
2539	void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
2540	{
2541	struct xe_vm_pgtable_update_ops *pt_update_ops =
2542	&vops->pt_update_ops[tile->id];
2543	int i;
2544
2545	lockdep_assert_held(&vops->vm->lock);
2546	xe_vm_assert_held(vops->vm);
2547
2548	for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
2549	struct xe_vm_pgtable_update_op *pt_op =
2550	&pt_update_ops->ops[i];
2551
2552	if (!pt_op->vma || i >= pt_update_ops->current_op)
2553	continue;
2554
2555	if (pt_op->bind)
2556	xe_pt_abort_bind(vma: pt_op->vma, entries: pt_op->entries,
2557	num_entries: pt_op->num_entries,
2558	rebind: pt_op->rebind);
2559	else
2560	xe_pt_abort_unbind(vma: pt_op->vma, entries: pt_op->entries,
2561	num_entries: pt_op->num_entries);
2562	}
2563
2564	xe_pt_update_ops_fini(tile, vops);
2565	}
2566