1	/*
2	* Copyright 2008 Advanced Micro Devices, Inc.
3	* Copyright 2008 Red Hat Inc.
4	* Copyright 2009 Jerome Glisse.
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a
7	* copy of this software and associated documentation files (the "Software"),
8	* to deal in the Software without restriction, including without limitation
9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
10	* and/or sell copies of the Software, and to permit persons to whom the
11	* Software is furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22	* OTHER DEALINGS IN THE SOFTWARE.
23	*
24	* Authors: Dave Airlie
25	* Alex Deucher
26	* Jerome Glisse
27	*/
28
29	#include <linux/dma-fence-array.h>
30	#include <linux/interval_tree_generic.h>
31	#include <linux/idr.h>
32	#include <linux/dma-buf.h>
33
34	#include <drm/amdgpu_drm.h>
35	#include <drm/drm_drv.h>
36	#include <drm/ttm/ttm_tt.h>
37	#include <drm/drm_exec.h>
38	#include "amdgpu.h"
39	#include "amdgpu_vm.h"
40	#include "amdgpu_trace.h"
41	#include "amdgpu_amdkfd.h"
42	#include "amdgpu_gmc.h"
43	#include "amdgpu_xgmi.h"
44	#include "amdgpu_dma_buf.h"
45	#include "amdgpu_res_cursor.h"
46	#include "kfd_svm.h"
47
48	/**
49	* DOC: GPUVM
50	*
51	* GPUVM is the MMU functionality provided on the GPU.
52	* GPUVM is similar to the legacy GART on older asics, however
53	* rather than there being a single global GART table
54	* for the entire GPU, there can be multiple GPUVM page tables active
55	* at any given time. The GPUVM page tables can contain a mix
56	* VRAM pages and system pages (both memory and MMIO) and system pages
57	* can be mapped as snooped (cached system pages) or unsnooped
58	* (uncached system pages).
59	*
60	* Each active GPUVM has an ID associated with it and there is a page table
61	* linked with each VMID. When executing a command buffer,
62	* the kernel tells the engine what VMID to use for that command
63	* buffer. VMIDs are allocated dynamically as commands are submitted.
64	* The userspace drivers maintain their own address space and the kernel
65	* sets up their pages tables accordingly when they submit their
66	* command buffers and a VMID is assigned.
67	* The hardware supports up to 16 active GPUVMs at any given time.
68	*
69	* Each GPUVM is represented by a 1-2 or 1-5 level page table, depending
70	* on the ASIC family. GPUVM supports RWX attributes on each page as well
71	* as other features such as encryption and caching attributes.
72	*
73	* VMID 0 is special. It is the GPUVM used for the kernel driver. In
74	* addition to an aperture managed by a page table, VMID 0 also has
75	* several other apertures. There is an aperture for direct access to VRAM
76	* and there is a legacy AGP aperture which just forwards accesses directly
77	* to the matching system physical addresses (or IOVAs when an IOMMU is
78	* present). These apertures provide direct access to these memories without
79	* incurring the overhead of a page table. VMID 0 is used by the kernel
80	* driver for tasks like memory management.
81	*
82	* GPU clients (i.e., engines on the GPU) use GPUVM VMIDs to access memory.
83	* For user applications, each application can have their own unique GPUVM
84	* address space. The application manages the address space and the kernel
85	* driver manages the GPUVM page tables for each process. If an GPU client
86	* accesses an invalid page, it will generate a GPU page fault, similar to
87	* accessing an invalid page on a CPU.
88	*/
89
90	#define START(node) ((node)->start)
91	#define LAST(node) ((node)->last)
92
93	INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
94	START, LAST, static, amdgpu_vm_it)
95
96	#undef START
97	#undef LAST
98
99	/**
100	* struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
101	*/
102	struct amdgpu_prt_cb {
103
104	/**
105	* @adev: amdgpu device
106	*/
107	struct amdgpu_device *adev;
108
109	/**
110	* @cb: callback
111	*/
112	struct dma_fence_cb cb;
113	};
114
115	/**
116	* struct amdgpu_vm_tlb_seq_struct - Helper to increment the TLB flush sequence
117	*/
118	struct amdgpu_vm_tlb_seq_struct {
119	/**
120	* @vm: pointer to the amdgpu_vm structure to set the fence sequence on
121	*/
122	struct amdgpu_vm *vm;
123
124	/**
125	* @cb: callback
126	*/
127	struct dma_fence_cb cb;
128	};
129
130	/**
131	* amdgpu_vm_assert_locked - check if VM is correctly locked
132	* @vm: the VM which schould be tested
133	*
134	* Asserts that the VM root PD is locked.
135	*/
136	static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm)
137	{
138	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
139	}
140
141	/**
142	* amdgpu_vm_bo_evicted - vm_bo is evicted
143	*
144	* @vm_bo: vm_bo which is evicted
145	*
146	* State for PDs/PTs and per VM BOs which are not at the location they should
147	* be.
148	*/
149	static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
150	{
151	struct amdgpu_vm *vm = vm_bo->vm;
152	struct amdgpu_bo *bo = vm_bo->bo;
153
154	vm_bo->moved = true;
155	amdgpu_vm_assert_locked(vm);
156	spin_lock(lock: &vm_bo->vm->status_lock);
157	if (bo->tbo.type == ttm_bo_type_kernel)
158	list_move(list: &vm_bo->vm_status, head: &vm->evicted);
159	else
160	list_move_tail(list: &vm_bo->vm_status, head: &vm->evicted);
161	spin_unlock(lock: &vm_bo->vm->status_lock);
162	}
163	/**
164	* amdgpu_vm_bo_moved - vm_bo is moved
165	*
166	* @vm_bo: vm_bo which is moved
167	*
168	* State for per VM BOs which are moved, but that change is not yet reflected
169	* in the page tables.
170	*/
171	static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
172	{
173	amdgpu_vm_assert_locked(vm: vm_bo->vm);
174	spin_lock(lock: &vm_bo->vm->status_lock);
175	list_move(list: &vm_bo->vm_status, head: &vm_bo->vm->moved);
176	spin_unlock(lock: &vm_bo->vm->status_lock);
177	}
178
179	/**
180	* amdgpu_vm_bo_idle - vm_bo is idle
181	*
182	* @vm_bo: vm_bo which is now idle
183	*
184	* State for PDs/PTs and per VM BOs which have gone through the state machine
185	* and are now idle.
186	*/
187	static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
188	{
189	amdgpu_vm_assert_locked(vm: vm_bo->vm);
190	spin_lock(lock: &vm_bo->vm->status_lock);
191	list_move(list: &vm_bo->vm_status, head: &vm_bo->vm->idle);
192	spin_unlock(lock: &vm_bo->vm->status_lock);
193	vm_bo->moved = false;
194	}
195
196	/**
197	* amdgpu_vm_bo_invalidated - vm_bo is invalidated
198	*
199	* @vm_bo: vm_bo which is now invalidated
200	*
201	* State for normal BOs which are invalidated and that change not yet reflected
202	* in the PTs.
203	*/
204	static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
205	{
206	spin_lock(lock: &vm_bo->vm->status_lock);
207	list_move(list: &vm_bo->vm_status, head: &vm_bo->vm->invalidated);
208	spin_unlock(lock: &vm_bo->vm->status_lock);
209	}
210
211	/**
212	* amdgpu_vm_bo_evicted_user - vm_bo is evicted
213	*
214	* @vm_bo: vm_bo which is evicted
215	*
216	* State for BOs used by user mode queues which are not at the location they
217	* should be.
218	*/
219	static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
220	{
221	vm_bo->moved = true;
222	spin_lock(lock: &vm_bo->vm->status_lock);
223	list_move(list: &vm_bo->vm_status, head: &vm_bo->vm->evicted_user);
224	spin_unlock(lock: &vm_bo->vm->status_lock);
225	}
226
227	/**
228	* amdgpu_vm_bo_relocated - vm_bo is reloacted
229	*
230	* @vm_bo: vm_bo which is relocated
231	*
232	* State for PDs/PTs which needs to update their parent PD.
233	* For the root PD, just move to idle state.
234	*/
235	static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
236	{
237	amdgpu_vm_assert_locked(vm: vm_bo->vm);
238	if (vm_bo->bo->parent) {
239	spin_lock(lock: &vm_bo->vm->status_lock);
240	list_move(list: &vm_bo->vm_status, head: &vm_bo->vm->relocated);
241	spin_unlock(lock: &vm_bo->vm->status_lock);
242	} else {
243	amdgpu_vm_bo_idle(vm_bo);
244	}
245	}
246
247	/**
248	* amdgpu_vm_bo_done - vm_bo is done
249	*
250	* @vm_bo: vm_bo which is now done
251	*
252	* State for normal BOs which are invalidated and that change has been updated
253	* in the PTs.
254	*/
255	static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
256	{
257	amdgpu_vm_assert_locked(vm: vm_bo->vm);
258	spin_lock(lock: &vm_bo->vm->status_lock);
259	list_move(list: &vm_bo->vm_status, head: &vm_bo->vm->done);
260	spin_unlock(lock: &vm_bo->vm->status_lock);
261	}
262
263	/**
264	* amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
265	* @vm: the VM which state machine to reset
266	*
267	* Move all vm_bo object in the VM into a state where they will be updated
268	* again during validation.
269	*/
270	static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
271	{
272	struct amdgpu_vm_bo_base *vm_bo, *tmp;
273
274	amdgpu_vm_assert_locked(vm);
275
276	spin_lock(lock: &vm->status_lock);
277	list_splice_init(list: &vm->done, head: &vm->invalidated);
278	list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
279	vm_bo->moved = true;
280
281	list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
282	struct amdgpu_bo *bo = vm_bo->bo;
283
284	vm_bo->moved = true;
285	if (!bo || bo->tbo.type != ttm_bo_type_kernel)
286	list_move(list: &vm_bo->vm_status, head: &vm_bo->vm->moved);
287	else if (bo->parent)
288	list_move(list: &vm_bo->vm_status, head: &vm_bo->vm->relocated);
289	}
290	spin_unlock(lock: &vm->status_lock);
291	}
292
293	/**
294	* amdgpu_vm_update_shared - helper to update shared memory stat
295	* @base: base structure for tracking BO usage in a VM
296	*
297	* Takes the vm status_lock and updates the shared memory stat. If the basic
298	* stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
299	* as well.
300	*/
301	static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
302	{
303	struct amdgpu_vm *vm = base->vm;
304	struct amdgpu_bo *bo = base->bo;
305	uint64_t size = amdgpu_bo_size(bo);
306	uint32_t bo_memtype = amdgpu_bo_mem_stats_placement(bo);
307	bool shared;
308
309	dma_resv_assert_held(bo->tbo.base.resv);
310	spin_lock(lock: &vm->status_lock);
311	shared = drm_gem_object_is_shared_for_memory_stats(obj: &bo->tbo.base);
312	if (base->shared != shared) {
313	base->shared = shared;
314	if (shared) {
315	vm->stats[bo_memtype].drm.shared += size;
316	vm->stats[bo_memtype].drm.private -= size;
317	} else {
318	vm->stats[bo_memtype].drm.shared -= size;
319	vm->stats[bo_memtype].drm.private += size;
320	}
321	}
322	spin_unlock(lock: &vm->status_lock);
323	}
324
325	/**
326	* amdgpu_vm_bo_update_shared - callback when bo gets shared/unshared
327	* @bo: amdgpu buffer object
328	*
329	* Update the per VM stats for all the vm if needed from private to shared or
330	* vice versa.
331	*/
332	void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
333	{
334	struct amdgpu_vm_bo_base *base;
335
336	for (base = bo->vm_bo; base; base = base->next)
337	amdgpu_vm_update_shared(base);
338	}
339
340	/**
341	* amdgpu_vm_update_stats_locked - helper to update normal memory stat
342	* @base: base structure for tracking BO usage in a VM
343	* @res: the ttm_resource to use for the purpose of accounting, may or may not
344	* be bo->tbo.resource
345	* @sign: if we should add (+1) or subtract (-1) from the stat
346	*
347	* Caller need to have the vm status_lock held. Useful for when multiple update
348	* need to happen at the same time.
349	*/
350	static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
351	struct ttm_resource *res, int sign)
352	{
353	struct amdgpu_vm *vm = base->vm;
354	struct amdgpu_bo *bo = base->bo;
355	int64_t size = sign * amdgpu_bo_size(bo);
356	uint32_t bo_memtype = amdgpu_bo_mem_stats_placement(bo);
357
358	/* For drm-total- and drm-shared-, BO are accounted by their preferred
359	* placement, see also amdgpu_bo_mem_stats_placement.
360	*/
361	if (base->shared)
362	vm->stats[bo_memtype].drm.shared += size;
363	else
364	vm->stats[bo_memtype].drm.private += size;
365
366	if (res && res->mem_type < __AMDGPU_PL_NUM) {
367	uint32_t res_memtype = res->mem_type;
368
369	vm->stats[res_memtype].drm.resident += size;
370	/* BO only count as purgeable if it is resident,
371	* since otherwise there's nothing to purge.
372	*/
373	if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
374	vm->stats[res_memtype].drm.purgeable += size;
375	if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type: res_memtype)))
376	vm->stats[bo_memtype].evicted += size;
377	}
378	}
379
380	/**
381	* amdgpu_vm_update_stats - helper to update normal memory stat
382	* @base: base structure for tracking BO usage in a VM
383	* @res: the ttm_resource to use for the purpose of accounting, may or may not
384	* be bo->tbo.resource
385	* @sign: if we should add (+1) or subtract (-1) from the stat
386	*
387	* Updates the basic memory stat when bo is added/deleted/moved.
388	*/
389	void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
390	struct ttm_resource *res, int sign)
391	{
392	struct amdgpu_vm *vm = base->vm;
393
394	spin_lock(lock: &vm->status_lock);
395	amdgpu_vm_update_stats_locked(base, res, sign);
396	spin_unlock(lock: &vm->status_lock);
397	}
398
399	/**
400	* amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
401	*
402	* @base: base structure for tracking BO usage in a VM
403	* @vm: vm to which bo is to be added
404	* @bo: amdgpu buffer object
405	*
406	* Initialize a bo_va_base structure and add it to the appropriate lists
407	*
408	*/
409	void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
410	struct amdgpu_vm *vm, struct amdgpu_bo *bo)
411	{
412	base->vm = vm;
413	base->bo = bo;
414	base->next = NULL;
415	INIT_LIST_HEAD(list: &base->vm_status);
416
417	if (!bo)
418	return;
419	base->next = bo->vm_bo;
420	bo->vm_bo = base;
421
422	spin_lock(lock: &vm->status_lock);
423	base->shared = drm_gem_object_is_shared_for_memory_stats(obj: &bo->tbo.base);
424	amdgpu_vm_update_stats_locked(base, res: bo->tbo.resource, sign: +1);
425	spin_unlock(lock: &vm->status_lock);
426
427	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
428	return;
429
430	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
431
432	ttm_bo_set_bulk_move(bo: &bo->tbo, bulk: &vm->lru_bulk_move);
433	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
434	amdgpu_vm_bo_relocated(vm_bo: base);
435	else
436	amdgpu_vm_bo_idle(vm_bo: base);
437
438	if (bo->preferred_domains &
439	amdgpu_mem_type_to_domain(mem_type: bo->tbo.resource->mem_type))
440	return;
441
442	/*
443	* we checked all the prerequisites, but it looks like this per vm bo
444	* is currently evicted. add the bo to the evicted list to make sure it
445	* is validated on next vm use to avoid fault.
446	* */
447	amdgpu_vm_bo_evicted(vm_bo: base);
448	}
449
450	/**
451	* amdgpu_vm_lock_pd - lock PD in drm_exec
452	*
453	* @vm: vm providing the BOs
454	* @exec: drm execution context
455	* @num_fences: number of extra fences to reserve
456	*
457	* Lock the VM root PD in the DRM execution context.
458	*/
459	int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
460	unsigned int num_fences)
461	{
462	/* We need at least two fences for the VM PD/PT updates */
463	return drm_exec_prepare_obj(exec, obj: &vm->root.bo->tbo.base,
464	num_fences: 2 + num_fences);
465	}
466
467	/**
468	* amdgpu_vm_lock_done_list - lock all BOs on the done list
469	* @vm: vm providing the BOs
470	* @exec: drm execution context
471	* @num_fences: number of extra fences to reserve
472	*
473	* Lock the BOs on the done list in the DRM execution context.
474	*/
475	int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
476	unsigned int num_fences)
477	{
478	struct list_head *prev = &vm->done;
479	struct amdgpu_bo_va *bo_va;
480	struct amdgpu_bo *bo;
481	int ret;
482
483	/* We can only trust prev->next while holding the lock */
484	spin_lock(lock: &vm->status_lock);
485	while (!list_is_head(list: prev->next, head: &vm->done)) {
486	bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
487
488	bo = bo_va->base.bo;
489	if (bo) {
490	amdgpu_bo_ref(bo);
491	spin_unlock(lock: &vm->status_lock);
492
493	ret = drm_exec_prepare_obj(exec, obj: &bo->tbo.base, num_fences: 1);
494	amdgpu_bo_unref(bo: &bo);
495	if (unlikely(ret))
496	return ret;
497
498	spin_lock(lock: &vm->status_lock);
499	}
500	prev = prev->next;
501	}
502	spin_unlock(lock: &vm->status_lock);
503
504	return 0;
505	}
506
507	/**
508	* amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
509	*
510	* @adev: amdgpu device pointer
511	* @vm: vm providing the BOs
512	*
513	* Move all BOs to the end of LRU and remember their positions to put them
514	* together.
515	*/
516	void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
517	struct amdgpu_vm *vm)
518	{
519	spin_lock(lock: &adev->mman.bdev.lru_lock);
520	ttm_lru_bulk_move_tail(bulk: &vm->lru_bulk_move);
521	spin_unlock(lock: &adev->mman.bdev.lru_lock);
522	}
523
524	/* Create scheduler entities for page table updates */
525	static int amdgpu_vm_init_entities(struct amdgpu_device *adev,
526	struct amdgpu_vm *vm)
527	{
528	int r;
529
530	r = drm_sched_entity_init(entity: &vm->immediate, priority: DRM_SCHED_PRIORITY_NORMAL,
531	sched_list: adev->vm_manager.vm_pte_scheds,
532	num_sched_list: adev->vm_manager.vm_pte_num_scheds, NULL);
533	if (r)
534	goto error;
535
536	return drm_sched_entity_init(entity: &vm->delayed, priority: DRM_SCHED_PRIORITY_NORMAL,
537	sched_list: adev->vm_manager.vm_pte_scheds,
538	num_sched_list: adev->vm_manager.vm_pte_num_scheds, NULL);
539
540	error:
541	drm_sched_entity_destroy(entity: &vm->immediate);
542	return r;
543	}
544
545	/* Destroy the entities for page table updates again */
546	static void amdgpu_vm_fini_entities(struct amdgpu_vm *vm)
547	{
548	drm_sched_entity_destroy(entity: &vm->immediate);
549	drm_sched_entity_destroy(entity: &vm->delayed);
550	}
551
552	/**
553	* amdgpu_vm_generation - return the page table re-generation counter
554	* @adev: the amdgpu_device
555	* @vm: optional VM to check, might be NULL
556	*
557	* Returns a page table re-generation token to allow checking if submissions
558	* are still valid to use this VM. The VM parameter might be NULL in which case
559	* just the VRAM lost counter will be used.
560	*/
561	uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm)
562	{
563	uint64_t result = (u64)atomic_read(v: &adev->vram_lost_counter) << 32;
564
565	if (!vm)
566	return result;
567
568	result += lower_32_bits(vm->generation);
569	/* Add one if the page tables will be re-generated on next CS */
570	if (drm_sched_entity_error(entity: &vm->delayed))
571	++result;
572
573	return result;
574	}
575
576	/**
577	* amdgpu_vm_validate - validate evicted BOs tracked in the VM
578	*
579	* @adev: amdgpu device pointer
580	* @vm: vm providing the BOs
581	* @ticket: optional reservation ticket used to reserve the VM
582	* @validate: callback to do the validation
583	* @param: parameter for the validation callback
584	*
585	* Validate the page table BOs and per-VM BOs on command submission if
586	* necessary. If a ticket is given, also try to validate evicted user queue
587	* BOs. They must already be reserved with the given ticket.
588	*
589	* Returns:
590	* Validation result.
591	*/
592	int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
593	struct ww_acquire_ctx *ticket,
594	int (*validate)(void *p, struct amdgpu_bo *bo),
595	void *param)
596	{
597	uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
598	struct amdgpu_vm_bo_base *bo_base;
599	struct amdgpu_bo *bo;
600	int r;
601
602	if (vm->generation != new_vm_generation) {
603	vm->generation = new_vm_generation;
604	amdgpu_vm_bo_reset_state_machine(vm);
605	amdgpu_vm_fini_entities(vm);
606	r = amdgpu_vm_init_entities(adev, vm);
607	if (r)
608	return r;
609	}
610
611	spin_lock(lock: &vm->status_lock);
612	while (!list_empty(head: &vm->evicted)) {
613	bo_base = list_first_entry(&vm->evicted,
614	struct amdgpu_vm_bo_base,
615	vm_status);
616	spin_unlock(lock: &vm->status_lock);
617
618	bo = bo_base->bo;
619
620	r = validate(param, bo);
621	if (r)
622	return r;
623
624	if (bo->tbo.type != ttm_bo_type_kernel) {
625	amdgpu_vm_bo_moved(vm_bo: bo_base);
626	} else {
627	vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
628	amdgpu_vm_bo_relocated(vm_bo: bo_base);
629	}
630	spin_lock(lock: &vm->status_lock);
631	}
632	while (ticket && !list_empty(head: &vm->evicted_user)) {
633	bo_base = list_first_entry(&vm->evicted_user,
634	struct amdgpu_vm_bo_base,
635	vm_status);
636	spin_unlock(lock: &vm->status_lock);
637
638	bo = bo_base->bo;
639	dma_resv_assert_held(bo->tbo.base.resv);
640
641	r = validate(param, bo);
642	if (r)
643	return r;
644
645	amdgpu_vm_bo_invalidated(vm_bo: bo_base);
646
647	spin_lock(lock: &vm->status_lock);
648	}
649	spin_unlock(lock: &vm->status_lock);
650
651	amdgpu_vm_eviction_lock(vm);
652	vm->evicting = false;
653	amdgpu_vm_eviction_unlock(vm);
654
655	return 0;
656	}
657
658	/**
659	* amdgpu_vm_ready - check VM is ready for updates
660	*
661	* @vm: VM to check
662	*
663	* Check if all VM PDs/PTs are ready for updates
664	*
665	* Returns:
666	* True if VM is not evicting and all VM entities are not stopped
667	*/
668	bool amdgpu_vm_ready(struct amdgpu_vm *vm)
669	{
670	bool ret;
671
672	amdgpu_vm_assert_locked(vm);
673
674	amdgpu_vm_eviction_lock(vm);
675	ret = !vm->evicting;
676	amdgpu_vm_eviction_unlock(vm);
677
678	spin_lock(lock: &vm->status_lock);
679	ret &= list_empty(head: &vm->evicted);
680	spin_unlock(lock: &vm->status_lock);
681
682	spin_lock(lock: &vm->immediate.lock);
683	ret &= !vm->immediate.stopped;
684	spin_unlock(lock: &vm->immediate.lock);
685
686	spin_lock(lock: &vm->delayed.lock);
687	ret &= !vm->delayed.stopped;
688	spin_unlock(lock: &vm->delayed.lock);
689
690	return ret;
691	}
692
693	/**
694	* amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
695	*
696	* @adev: amdgpu_device pointer
697	*/
698	void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
699	{
700	const struct amdgpu_ip_block *ip_block;
701	bool has_compute_vm_bug;
702	struct amdgpu_ring *ring;
703	int i;
704
705	has_compute_vm_bug = false;
706
707	ip_block = amdgpu_device_ip_get_ip_block(adev, type: AMD_IP_BLOCK_TYPE_GFX);
708	if (ip_block) {
709	/* Compute has a VM bug for GFX version < 7.
710	Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
711	if (ip_block->version->major <= 7)
712	has_compute_vm_bug = true;
713	else if (ip_block->version->major == 8)
714	if (adev->gfx.mec_fw_version < 673)
715	has_compute_vm_bug = true;
716	}
717
718	for (i = 0; i < adev->num_rings; i++) {
719	ring = adev->rings[i];
720	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
721	/* only compute rings */
722	ring->has_compute_vm_bug = has_compute_vm_bug;
723	else
724	ring->has_compute_vm_bug = false;
725	}
726	}
727
728	/**
729	* amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
730	*
731	* @ring: ring on which the job will be submitted
732	* @job: job to submit
733	*
734	* Returns:
735	* True if sync is needed.
736	*/
737	bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
738	struct amdgpu_job *job)
739	{
740	struct amdgpu_device *adev = ring->adev;
741	unsigned vmhub = ring->vm_hub;
742	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
743
744	if (job->vmid == 0)
745	return false;
746
747	if (job->vm_needs_flush || ring->has_compute_vm_bug)
748	return true;
749
750	if (ring->funcs->emit_gds_switch && job->gds_switch_needed)
751	return true;
752
753	if (amdgpu_vmid_had_gpu_reset(adev, id: &id_mgr->ids[job->vmid]))
754	return true;
755
756	return false;
757	}
758
759	/**
760	* amdgpu_vm_flush - hardware flush the vm
761	*
762	* @ring: ring to use for flush
763	* @job: related job
764	* @need_pipe_sync: is pipe sync needed
765	*
766	* Emit a VM flush when it is necessary.
767	*
768	* Returns:
769	* 0 on success, errno otherwise.
770	*/
771	int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
772	bool need_pipe_sync)
773	{
774	struct amdgpu_device *adev = ring->adev;
775	struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id];
776	unsigned vmhub = ring->vm_hub;
777	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
778	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
779	bool spm_update_needed = job->spm_update_needed;
780	bool gds_switch_needed = ring->funcs->emit_gds_switch &&
781	job->gds_switch_needed;
782	bool vm_flush_needed = job->vm_needs_flush;
783	bool cleaner_shader_needed = false;
784	bool pasid_mapping_needed = false;
785	struct dma_fence *fence = NULL;
786	unsigned int patch;
787	int r;
788
789	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
790	gds_switch_needed = true;
791	vm_flush_needed = true;
792	pasid_mapping_needed = true;
793	spm_update_needed = true;
794	}
795
796	mutex_lock(&id_mgr->lock);
797	if (id->pasid != job->pasid || !id->pasid_mapping ||
798	!dma_fence_is_signaled(fence: id->pasid_mapping))
799	pasid_mapping_needed = true;
800	mutex_unlock(lock: &id_mgr->lock);
801
802	gds_switch_needed &= !!ring->funcs->emit_gds_switch;
803	vm_flush_needed &= !!ring->funcs->emit_vm_flush &&
804	job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
805	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
806	ring->funcs->emit_wreg;
807
808	cleaner_shader_needed = job->run_cleaner_shader &&
809	adev->gfx.enable_cleaner_shader &&
810	ring->funcs->emit_cleaner_shader && job->base.s_fence &&
811	&job->base.s_fence->scheduled == isolation->spearhead;
812
813	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync &&
814	!cleaner_shader_needed)
815	return 0;
816
817	amdgpu_ring_ib_begin(ring);
818	if (ring->funcs->init_cond_exec)
819	patch = amdgpu_ring_init_cond_exec(ring,
820	ring->cond_exe_gpu_addr);
821
822	if (need_pipe_sync)
823	amdgpu_ring_emit_pipeline_sync(ring);
824
825	if (cleaner_shader_needed)
826	ring->funcs->emit_cleaner_shader(ring);
827
828	if (vm_flush_needed) {
829	trace_amdgpu_vm_flush(ring, vmid: job->vmid, pd_addr: job->vm_pd_addr);
830	amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
831	}
832
833	if (pasid_mapping_needed)
834	amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
835
836	if (spm_update_needed && adev->gfx.rlc.funcs->update_spm_vmid)
837	adev->gfx.rlc.funcs->update_spm_vmid(adev, ring, job->vmid);
838
839	if (ring->funcs->emit_gds_switch &&
840	gds_switch_needed) {
841	amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
842	job->gds_size, job->gws_base,
843	job->gws_size, job->oa_base,
844	job->oa_size);
845	}
846
847	if (vm_flush_needed || pasid_mapping_needed || cleaner_shader_needed) {
848	r = amdgpu_fence_emit(ring, af: job->hw_vm_fence, flags: 0);
849	if (r)
850	return r;
851	fence = &job->hw_vm_fence->base;
852	/* get a ref for the job */
853	dma_fence_get(fence);
854	}
855
856	if (vm_flush_needed) {
857	mutex_lock(&id_mgr->lock);
858	dma_fence_put(fence: id->last_flush);
859	id->last_flush = dma_fence_get(fence);
860	id->current_gpu_reset_count =
861	atomic_read(v: &adev->gpu_reset_counter);
862	mutex_unlock(lock: &id_mgr->lock);
863	}
864
865	if (pasid_mapping_needed) {
866	mutex_lock(&id_mgr->lock);
867	id->pasid = job->pasid;
868	dma_fence_put(fence: id->pasid_mapping);
869	id->pasid_mapping = dma_fence_get(fence);
870	mutex_unlock(lock: &id_mgr->lock);
871	}
872
873	/*
874	* Make sure that all other submissions wait for the cleaner shader to
875	* finish before we push them to the HW.
876	*/
877	if (cleaner_shader_needed) {
878	trace_amdgpu_cleaner_shader(ring, fence);
879	mutex_lock(&adev->enforce_isolation_mutex);
880	dma_fence_put(fence: isolation->spearhead);
881	isolation->spearhead = dma_fence_get(fence);
882	mutex_unlock(lock: &adev->enforce_isolation_mutex);
883	}
884	dma_fence_put(fence);
885
886	amdgpu_ring_patch_cond_exec(ring, offset: patch);
887
888	/* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
889	if (ring->funcs->emit_switch_buffer) {
890	amdgpu_ring_emit_switch_buffer(ring);
891	amdgpu_ring_emit_switch_buffer(ring);
892	}
893
894	amdgpu_ring_ib_end(ring);
895	return 0;
896	}
897
898	/**
899	* amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
900	*
901	* @vm: requested vm
902	* @bo: requested buffer object
903	*
904	* Find @bo inside the requested vm.
905	* Search inside the @bos vm list for the requested vm
906	* Returns the found bo_va or NULL if none is found
907	*
908	* Object has to be reserved!
909	*
910	* Returns:
911	* Found bo_va or NULL.
912	*/
913	struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
914	struct amdgpu_bo *bo)
915	{
916	struct amdgpu_vm_bo_base *base;
917
918	for (base = bo->vm_bo; base; base = base->next) {
919	if (base->vm != vm)
920	continue;
921
922	return container_of(base, struct amdgpu_bo_va, base);
923	}
924	return NULL;
925	}
926
927	/**
928	* amdgpu_vm_map_gart - Resolve gart mapping of addr
929	*
930	* @pages_addr: optional DMA address to use for lookup
931	* @addr: the unmapped addr
932	*
933	* Look up the physical address of the page that the pte resolves
934	* to.
935	*
936	* Returns:
937	* The pointer for the page table entry.
938	*/
939	uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
940	{
941	uint64_t result;
942
943	/* page table offset */
944	result = pages_addr[addr >> PAGE_SHIFT];
945
946	/* in case cpu page size != gpu page size*/
947	result |= addr & (~PAGE_MASK);
948
949	result &= 0xFFFFFFFFFFFFF000ULL;
950
951	return result;
952	}
953
954	/**
955	* amdgpu_vm_update_pdes - make sure that all directories are valid
956	*
957	* @adev: amdgpu_device pointer
958	* @vm: requested vm
959	* @immediate: submit immediately to the paging queue
960	*
961	* Makes sure all directories are up to date.
962	*
963	* Returns:
964	* 0 for success, error for failure.
965	*/
966	int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
967	struct amdgpu_vm *vm, bool immediate)
968	{
969	struct amdgpu_vm_update_params params;
970	struct amdgpu_vm_bo_base *entry;
971	bool flush_tlb_needed = false;
972	LIST_HEAD(relocated);
973	int r, idx;
974
975	amdgpu_vm_assert_locked(vm);
976
977	spin_lock(lock: &vm->status_lock);
978	list_splice_init(list: &vm->relocated, head: &relocated);
979	spin_unlock(lock: &vm->status_lock);
980
981	if (list_empty(head: &relocated))
982	return 0;
983
984	if (!drm_dev_enter(dev: adev_to_drm(adev), idx: &idx))
985	return -ENODEV;
986
987	memset(&params, 0, sizeof(params));
988	params.adev = adev;
989	params.vm = vm;
990	params.immediate = immediate;
991
992	r = vm->update_funcs->prepare(&params, NULL,
993	AMDGPU_KERNEL_JOB_ID_VM_UPDATE_PDES);
994	if (r)
995	goto error;
996
997	list_for_each_entry(entry, &relocated, vm_status) {
998	/* vm_flush_needed after updating moved PDEs */
999	flush_tlb_needed |= entry->moved;
1000
1001	r = amdgpu_vm_pde_update(params: &params, entry);
1002	if (r)
1003	goto error;
1004	}
1005
1006	r = vm->update_funcs->commit(&params, &vm->last_update);
1007	if (r)
1008	goto error;
1009
1010	if (flush_tlb_needed)
1011	atomic64_inc(v: &vm->tlb_seq);
1012
1013	while (!list_empty(head: &relocated)) {
1014	entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
1015	vm_status);
1016	amdgpu_vm_bo_idle(vm_bo: entry);
1017	}
1018
1019	error:
1020	drm_dev_exit(idx);
1021	return r;
1022	}
1023
1024	/**
1025	* amdgpu_vm_tlb_seq_cb - make sure to increment tlb sequence
1026	* @fence: unused
1027	* @cb: the callback structure
1028	*
1029	* Increments the tlb sequence to make sure that future CS execute a VM flush.
1030	*/
1031	static void amdgpu_vm_tlb_seq_cb(struct dma_fence *fence,
1032	struct dma_fence_cb *cb)
1033	{
1034	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
1035
1036	tlb_cb = container_of(cb, typeof(*tlb_cb), cb);
1037	atomic64_inc(v: &tlb_cb->vm->tlb_seq);
1038	kfree(objp: tlb_cb);
1039	}
1040
1041	/**
1042	* amdgpu_vm_tlb_flush - prepare TLB flush
1043	*
1044	* @params: parameters for update
1045	* @fence: input fence to sync TLB flush with
1046	* @tlb_cb: the callback structure
1047	*
1048	* Increments the tlb sequence to make sure that future CS execute a VM flush.
1049	*/
1050	static void
1051	amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params,
1052	struct dma_fence **fence,
1053	struct amdgpu_vm_tlb_seq_struct *tlb_cb)
1054	{
1055	struct amdgpu_vm *vm = params->vm;
1056
1057	tlb_cb->vm = vm;
1058	if (!fence || !*fence) {
1059	amdgpu_vm_tlb_seq_cb(NULL, cb: &tlb_cb->cb);
1060	return;
1061	}
1062
1063	if (!dma_fence_add_callback(fence: *fence, cb: &tlb_cb->cb,
1064	func: amdgpu_vm_tlb_seq_cb)) {
1065	dma_fence_put(fence: vm->last_tlb_flush);
1066	vm->last_tlb_flush = dma_fence_get(fence: *fence);
1067	} else {
1068	amdgpu_vm_tlb_seq_cb(NULL, cb: &tlb_cb->cb);
1069	}
1070
1071	/* Prepare a TLB flush fence to be attached to PTs */
1072	if (!params->unlocked) {
1073	amdgpu_vm_tlb_fence_create(adev: params->adev, vm, fence);
1074
1075	/* Makes sure no PD/PT is freed before the flush */
1076	dma_resv_add_fence(obj: vm->root.bo->tbo.base.resv, fence: *fence,
1077	usage: DMA_RESV_USAGE_BOOKKEEP);
1078	}
1079	}
1080
1081	/**
1082	* amdgpu_vm_update_range - update a range in the vm page table
1083	*
1084	* @adev: amdgpu_device pointer to use for commands
1085	* @vm: the VM to update the range
1086	* @immediate: immediate submission in a page fault
1087	* @unlocked: unlocked invalidation during MM callback
1088	* @flush_tlb: trigger tlb invalidation after update completed
1089	* @allow_override: change MTYPE for local NUMA nodes
1090	* @sync: fences we need to sync to
1091	* @start: start of mapped range
1092	* @last: last mapped entry
1093	* @flags: flags for the entries
1094	* @offset: offset into nodes and pages_addr
1095	* @vram_base: base for vram mappings
1096	* @res: ttm_resource to map
1097	* @pages_addr: DMA addresses to use for mapping
1098	* @fence: optional resulting fence
1099	*
1100	* Fill in the page table entries between @start and @last.
1101	*
1102	* Returns:
1103	* 0 for success, negative erro code for failure.
1104	*/
1105	int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
1106	bool immediate, bool unlocked, bool flush_tlb,
1107	bool allow_override, struct amdgpu_sync *sync,
1108	uint64_t start, uint64_t last, uint64_t flags,
1109	uint64_t offset, uint64_t vram_base,
1110	struct ttm_resource *res, dma_addr_t *pages_addr,
1111	struct dma_fence **fence)
1112	{
1113	struct amdgpu_vm_tlb_seq_struct *tlb_cb;
1114	struct amdgpu_vm_update_params params;
1115	struct amdgpu_res_cursor cursor;
1116	int r, idx;
1117
1118	if (!drm_dev_enter(dev: adev_to_drm(adev), idx: &idx))
1119	return -ENODEV;
1120
1121	tlb_cb = kmalloc(sizeof(*tlb_cb), GFP_KERNEL);
1122	if (!tlb_cb) {
1123	drm_dev_exit(idx);
1124	return -ENOMEM;
1125	}
1126
1127	/* Vega20+XGMI where PTEs get inadvertently cached in L2 texture cache,
1128	* heavy-weight flush TLB unconditionally.
1129	*/
1130	flush_tlb |= adev->gmc.xgmi.num_physical_nodes &&
1131	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 0);
1132
1133	/*
1134	* On GFX8 and older any 8 PTE block with a valid bit set enters the TLB
1135	*/
1136	flush_tlb |= amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) < IP_VERSION(9, 0, 0);
1137
1138	memset(&params, 0, sizeof(params));
1139	params.adev = adev;
1140	params.vm = vm;
1141	params.immediate = immediate;
1142	params.pages_addr = pages_addr;
1143	params.unlocked = unlocked;
1144	params.needs_flush = flush_tlb;
1145	params.allow_override = allow_override;
1146	INIT_LIST_HEAD(list: &params.tlb_flush_waitlist);
1147
1148	amdgpu_vm_eviction_lock(vm);
1149	if (vm->evicting) {
1150	r = -EBUSY;
1151	goto error_free;
1152	}
1153
1154	if (!unlocked && !dma_fence_is_signaled(fence: vm->last_unlocked)) {
1155	struct dma_fence *tmp = dma_fence_get_stub();
1156
1157	amdgpu_bo_fence(bo: vm->root.bo, fence: vm->last_unlocked, shared: true);
1158	swap(vm->last_unlocked, tmp);
1159	dma_fence_put(fence: tmp);
1160	}
1161
1162	r = vm->update_funcs->prepare(&params, sync,
1163	AMDGPU_KERNEL_JOB_ID_VM_UPDATE_RANGE);
1164	if (r)
1165	goto error_free;
1166
1167	amdgpu_res_first(res: pages_addr ? NULL : res, start: offset,
1168	size: (last - start + 1) * AMDGPU_GPU_PAGE_SIZE, cur: &cursor);
1169	while (cursor.remaining) {
1170	uint64_t tmp, num_entries, addr;
1171
1172	num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;
1173	if (pages_addr) {
1174	bool contiguous = true;
1175
1176	if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {
1177	uint64_t pfn = cursor.start >> PAGE_SHIFT;
1178	uint64_t count;
1179
1180	contiguous = pages_addr[pfn + 1] ==
1181	pages_addr[pfn] + PAGE_SIZE;
1182
1183	tmp = num_entries /
1184	AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1185	for (count = 2; count < tmp; ++count) {
1186	uint64_t idx = pfn + count;
1187
1188	if (contiguous != (pages_addr[idx] ==
1189	pages_addr[idx - 1] + PAGE_SIZE))
1190	break;
1191	}
1192	if (!contiguous)
1193	count--;
1194	num_entries = count *
1195	AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1196	}
1197
1198	if (!contiguous) {
1199	addr = cursor.start;
1200	params.pages_addr = pages_addr;
1201	} else {
1202	addr = pages_addr[cursor.start >> PAGE_SHIFT];
1203	params.pages_addr = NULL;
1204	}
1205
1206	} else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT_FLAG(adev))) {
1207	addr = vram_base + cursor.start;
1208	} else {
1209	addr = 0;
1210	}
1211
1212	tmp = start + num_entries;
1213	r = amdgpu_vm_ptes_update(params: &params, start, end: tmp, dst: addr, flags);
1214	if (r)
1215	goto error_free;
1216
1217	amdgpu_res_next(cur: &cursor, size: num_entries * AMDGPU_GPU_PAGE_SIZE);
1218	start = tmp;
1219	}
1220
1221	r = vm->update_funcs->commit(&params, fence);
1222	if (r)
1223	goto error_free;
1224
1225	if (params.needs_flush) {
1226	amdgpu_vm_tlb_flush(params: &params, fence, tlb_cb);
1227	tlb_cb = NULL;
1228	}
1229
1230	amdgpu_vm_pt_free_list(adev, params: &params);
1231
1232	error_free:
1233	kfree(objp: tlb_cb);
1234	amdgpu_vm_eviction_unlock(vm);
1235	drm_dev_exit(idx);
1236	return r;
1237	}
1238
1239	void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
1240	struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
1241	{
1242	spin_lock(lock: &vm->status_lock);
1243	memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
1244	spin_unlock(lock: &vm->status_lock);
1245	}
1246
1247	/**
1248	* amdgpu_vm_bo_update - update all BO mappings in the vm page table
1249	*
1250	* @adev: amdgpu_device pointer
1251	* @bo_va: requested BO and VM object
1252	* @clear: if true clear the entries
1253	*
1254	* Fill in the page table entries for @bo_va.
1255	*
1256	* Returns:
1257	* 0 for success, -EINVAL for failure.
1258	*/
1259	int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
1260	bool clear)
1261	{
1262	struct amdgpu_bo *bo = bo_va->base.bo;
1263	struct amdgpu_vm *vm = bo_va->base.vm;
1264	struct amdgpu_bo_va_mapping *mapping;
1265	struct dma_fence **last_update;
1266	dma_addr_t *pages_addr = NULL;
1267	struct ttm_resource *mem;
1268	struct amdgpu_sync sync;
1269	bool flush_tlb = clear;
1270	uint64_t vram_base;
1271	uint64_t flags;
1272	bool uncached;
1273	int r;
1274
1275	amdgpu_sync_create(sync: &sync);
1276	if (clear) {
1277	mem = NULL;
1278
1279	/* Implicitly sync to command submissions in the same VM before
1280	* unmapping.
1281	*/
1282	r = amdgpu_sync_resv(adev, sync: &sync, resv: vm->root.bo->tbo.base.resv,
1283	mode: AMDGPU_SYNC_EQ_OWNER, owner: vm);
1284	if (r)
1285	goto error_free;
1286	if (bo) {
1287	r = amdgpu_sync_kfd(sync: &sync, resv: bo->tbo.base.resv);
1288	if (r)
1289	goto error_free;
1290	}
1291	} else if (!bo) {
1292	mem = NULL;
1293
1294	/* PRT map operations don't need to sync to anything. */
1295
1296	} else {
1297	struct drm_gem_object *obj = &bo->tbo.base;
1298
1299	if (drm_gem_is_imported(obj) && bo_va->is_xgmi) {
1300	struct dma_buf *dma_buf = obj->import_attach->dmabuf;
1301	struct drm_gem_object *gobj = dma_buf->priv;
1302	struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
1303
1304	if (abo->tbo.resource &&
1305	abo->tbo.resource->mem_type == TTM_PL_VRAM)
1306	bo = gem_to_amdgpu_bo(gobj);
1307	}
1308	mem = bo->tbo.resource;
1309	if (mem && (mem->mem_type == TTM_PL_TT ||
1310	mem->mem_type == AMDGPU_PL_PREEMPT))
1311	pages_addr = bo->tbo.ttm->dma_address;
1312
1313	/* Implicitly sync to moving fences before mapping anything */
1314	r = amdgpu_sync_resv(adev, sync: &sync, resv: bo->tbo.base.resv,
1315	mode: AMDGPU_SYNC_EXPLICIT, owner: vm);
1316	if (r)
1317	goto error_free;
1318	}
1319
1320	if (bo) {
1321	struct amdgpu_device *bo_adev;
1322
1323	flags = amdgpu_ttm_tt_pte_flags(adev, ttm: bo->tbo.ttm, mem);
1324
1325	if (amdgpu_bo_encrypted(bo))
1326	flags |= AMDGPU_PTE_TMZ;
1327
1328	bo_adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
1329	vram_base = bo_adev->vm_manager.vram_base_offset;
1330	uncached = (bo->flags & AMDGPU_GEM_CREATE_UNCACHED) != 0;
1331	} else {
1332	flags = 0x0;
1333	vram_base = 0;
1334	uncached = false;
1335	}
1336
1337	if (clear || amdgpu_vm_is_bo_always_valid(vm, bo))
1338	last_update = &vm->last_update;
1339	else
1340	last_update = &bo_va->last_pt_update;
1341
1342	if (!clear && bo_va->base.moved) {
1343	flush_tlb = true;
1344	list_splice_init(list: &bo_va->valids, head: &bo_va->invalids);
1345
1346	} else if (bo_va->cleared != clear) {
1347	list_splice_init(list: &bo_va->valids, head: &bo_va->invalids);
1348	}
1349
1350	list_for_each_entry(mapping, &bo_va->invalids, list) {
1351	uint64_t update_flags = flags;
1352
1353	/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1354	* but in case of something, we filter the flags in first place
1355	*/
1356	if (!(mapping->flags & AMDGPU_VM_PAGE_READABLE))
1357	update_flags &= ~AMDGPU_PTE_READABLE;
1358	if (!(mapping->flags & AMDGPU_VM_PAGE_WRITEABLE))
1359	update_flags &= ~AMDGPU_PTE_WRITEABLE;
1360
1361	/* Apply ASIC specific mapping flags */
1362	amdgpu_gmc_get_vm_pte(adev, vm, bo, mapping->flags,
1363	&update_flags);
1364
1365	trace_amdgpu_vm_bo_update(mapping);
1366
1367	r = amdgpu_vm_update_range(adev, vm, immediate: false, unlocked: false, flush_tlb,
1368	allow_override: !uncached, sync: &sync, start: mapping->start,
1369	last: mapping->last, flags: update_flags,
1370	offset: mapping->offset, vram_base, res: mem,
1371	pages_addr, fence: last_update);
1372	if (r)
1373	goto error_free;
1374	}
1375
1376	/* If the BO is not in its preferred location add it back to
1377	* the evicted list so that it gets validated again on the
1378	* next command submission.
1379	*/
1380	if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
1381	if (bo->tbo.resource &&
1382	!(bo->preferred_domains &
1383	amdgpu_mem_type_to_domain(mem_type: bo->tbo.resource->mem_type)))
1384	amdgpu_vm_bo_evicted(vm_bo: &bo_va->base);
1385	else
1386	amdgpu_vm_bo_idle(vm_bo: &bo_va->base);
1387	} else {
1388	amdgpu_vm_bo_done(vm_bo: &bo_va->base);
1389	}
1390
1391	list_splice_init(list: &bo_va->invalids, head: &bo_va->valids);
1392	bo_va->cleared = clear;
1393	bo_va->base.moved = false;
1394
1395	if (trace_amdgpu_vm_bo_mapping_enabled()) {
1396	list_for_each_entry(mapping, &bo_va->valids, list)
1397	trace_amdgpu_vm_bo_mapping(mapping);
1398	}
1399
1400	error_free:
1401	amdgpu_sync_free(sync: &sync);
1402	return r;
1403	}
1404
1405	/**
1406	* amdgpu_vm_update_prt_state - update the global PRT state
1407	*
1408	* @adev: amdgpu_device pointer
1409	*/
1410	static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1411	{
1412	unsigned long flags;
1413	bool enable;
1414
1415	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1416	enable = !!atomic_read(v: &adev->vm_manager.num_prt_users);
1417	adev->gmc.gmc_funcs->set_prt(adev, enable);
1418	spin_unlock_irqrestore(lock: &adev->vm_manager.prt_lock, flags);
1419	}
1420
1421	/**
1422	* amdgpu_vm_prt_get - add a PRT user
1423	*
1424	* @adev: amdgpu_device pointer
1425	*/
1426	static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1427	{
1428	if (!adev->gmc.gmc_funcs->set_prt)
1429	return;
1430
1431	if (atomic_inc_return(v: &adev->vm_manager.num_prt_users) == 1)
1432	amdgpu_vm_update_prt_state(adev);
1433	}
1434
1435	/**
1436	* amdgpu_vm_prt_put - drop a PRT user
1437	*
1438	* @adev: amdgpu_device pointer
1439	*/
1440	static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1441	{
1442	if (atomic_dec_return(v: &adev->vm_manager.num_prt_users) == 0)
1443	amdgpu_vm_update_prt_state(adev);
1444	}
1445
1446	/**
1447	* amdgpu_vm_prt_cb - callback for updating the PRT status
1448	*
1449	* @fence: fence for the callback
1450	* @_cb: the callback function
1451	*/
1452	static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1453	{
1454	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1455
1456	amdgpu_vm_prt_put(adev: cb->adev);
1457	kfree(objp: cb);
1458	}
1459
1460	/**
1461	* amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1462	*
1463	* @adev: amdgpu_device pointer
1464	* @fence: fence for the callback
1465	*/
1466	static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1467	struct dma_fence *fence)
1468	{
1469	struct amdgpu_prt_cb *cb;
1470
1471	if (!adev->gmc.gmc_funcs->set_prt)
1472	return;
1473
1474	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1475	if (!cb) {
1476	/* Last resort when we are OOM */
1477	if (fence)
1478	dma_fence_wait(fence, intr: false);
1479
1480	amdgpu_vm_prt_put(adev);
1481	} else {
1482	cb->adev = adev;
1483	if (!fence || dma_fence_add_callback(fence, cb: &cb->cb,
1484	func: amdgpu_vm_prt_cb))
1485	amdgpu_vm_prt_cb(fence, cb: &cb->cb);
1486	}
1487	}
1488
1489	/**
1490	* amdgpu_vm_free_mapping - free a mapping
1491	*
1492	* @adev: amdgpu_device pointer
1493	* @vm: requested vm
1494	* @mapping: mapping to be freed
1495	* @fence: fence of the unmap operation
1496	*
1497	* Free a mapping and make sure we decrease the PRT usage count if applicable.
1498	*/
1499	static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1500	struct amdgpu_vm *vm,
1501	struct amdgpu_bo_va_mapping *mapping,
1502	struct dma_fence *fence)
1503	{
1504	if (mapping->flags & AMDGPU_VM_PAGE_PRT)
1505	amdgpu_vm_add_prt_cb(adev, fence);
1506	kfree(objp: mapping);
1507	}
1508
1509	/**
1510	* amdgpu_vm_prt_fini - finish all prt mappings
1511	*
1512	* @adev: amdgpu_device pointer
1513	* @vm: requested vm
1514	*
1515	* Register a cleanup callback to disable PRT support after VM dies.
1516	*/
1517	static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1518	{
1519	struct dma_resv *resv = vm->root.bo->tbo.base.resv;
1520	struct dma_resv_iter cursor;
1521	struct dma_fence *fence;
1522
1523	dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) {
1524	/* Add a callback for each fence in the reservation object */
1525	amdgpu_vm_prt_get(adev);
1526	amdgpu_vm_add_prt_cb(adev, fence);
1527	}
1528	}
1529
1530	/**
1531	* amdgpu_vm_clear_freed - clear freed BOs in the PT
1532	*
1533	* @adev: amdgpu_device pointer
1534	* @vm: requested vm
1535	* @fence: optional resulting fence (unchanged if no work needed to be done
1536	* or if an error occurred)
1537	*
1538	* Make sure all freed BOs are cleared in the PT.
1539	* PTs have to be reserved and mutex must be locked!
1540	*
1541	* Returns:
1542	* 0 for success.
1543	*
1544	*/
1545	int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1546	struct amdgpu_vm *vm,
1547	struct dma_fence **fence)
1548	{
1549	struct amdgpu_bo_va_mapping *mapping;
1550	struct dma_fence *f = NULL;
1551	struct amdgpu_sync sync;
1552	int r;
1553
1554
1555	/*
1556	* Implicitly sync to command submissions in the same VM before
1557	* unmapping.
1558	*/
1559	amdgpu_sync_create(sync: &sync);
1560	r = amdgpu_sync_resv(adev, sync: &sync, resv: vm->root.bo->tbo.base.resv,
1561	mode: AMDGPU_SYNC_EQ_OWNER, owner: vm);
1562	if (r)
1563	goto error_free;
1564
1565	while (!list_empty(head: &vm->freed)) {
1566	mapping = list_first_entry(&vm->freed,
1567	struct amdgpu_bo_va_mapping, list);
1568	list_del(entry: &mapping->list);
1569
1570	r = amdgpu_vm_update_range(adev, vm, immediate: false, unlocked: false, flush_tlb: true, allow_override: false,
1571	sync: &sync, start: mapping->start, last: mapping->last,
1572	flags: 0, offset: 0, vram_base: 0, NULL, NULL, fence: &f);
1573	amdgpu_vm_free_mapping(adev, vm, mapping, fence: f);
1574	if (r) {
1575	dma_fence_put(fence: f);
1576	goto error_free;
1577	}
1578	}
1579
1580	if (fence && f) {
1581	dma_fence_put(fence: *fence);
1582	*fence = f;
1583	} else {
1584	dma_fence_put(fence: f);
1585	}
1586
1587	error_free:
1588	amdgpu_sync_free(sync: &sync);
1589	return r;
1590
1591	}
1592
1593	/**
1594	* amdgpu_vm_handle_moved - handle moved BOs in the PT
1595	*
1596	* @adev: amdgpu_device pointer
1597	* @vm: requested vm
1598	* @ticket: optional reservation ticket used to reserve the VM
1599	*
1600	* Make sure all BOs which are moved are updated in the PTs.
1601	*
1602	* Returns:
1603	* 0 for success.
1604	*
1605	* PTs have to be reserved!
1606	*/
1607	int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1608	struct amdgpu_vm *vm,
1609	struct ww_acquire_ctx *ticket)
1610	{
1611	struct amdgpu_bo_va *bo_va;
1612	struct dma_resv *resv;
1613	bool clear, unlock;
1614	int r;
1615
1616	spin_lock(lock: &vm->status_lock);
1617	while (!list_empty(head: &vm->moved)) {
1618	bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
1619	base.vm_status);
1620	spin_unlock(lock: &vm->status_lock);
1621
1622	/* Per VM BOs never need to bo cleared in the page tables */
1623	r = amdgpu_vm_bo_update(adev, bo_va, clear: false);
1624	if (r)
1625	return r;
1626	spin_lock(lock: &vm->status_lock);
1627	}
1628
1629	while (!list_empty(head: &vm->invalidated)) {
1630	bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
1631	base.vm_status);
1632	resv = bo_va->base.bo->tbo.base.resv;
1633	spin_unlock(lock: &vm->status_lock);
1634
1635	/* Try to reserve the BO to avoid clearing its ptes */
1636	if (!adev->debug_vm && dma_resv_trylock(obj: resv)) {
1637	clear = false;
1638	unlock = true;
1639	/* The caller is already holding the reservation lock */
1640	} else if (ticket && dma_resv_locking_ctx(obj: resv) == ticket) {
1641	clear = false;
1642	unlock = false;
1643	/* Somebody else is using the BO right now */
1644	} else {
1645	clear = true;
1646	unlock = false;
1647	}
1648
1649	r = amdgpu_vm_bo_update(adev, bo_va, clear);
1650
1651	if (unlock)
1652	dma_resv_unlock(obj: resv);
1653	if (r)
1654	return r;
1655
1656	/* Remember evicted DMABuf imports in compute VMs for later
1657	* validation
1658	*/
1659	if (vm->is_compute_context &&
1660	drm_gem_is_imported(obj: &bo_va->base.bo->tbo.base) &&
1661	(!bo_va->base.bo->tbo.resource ||
1662	bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
1663	amdgpu_vm_bo_evicted_user(vm_bo: &bo_va->base);
1664
1665	spin_lock(lock: &vm->status_lock);
1666	}
1667	spin_unlock(lock: &vm->status_lock);
1668
1669	return 0;
1670	}
1671
1672	/**
1673	* amdgpu_vm_flush_compute_tlb - Flush TLB on compute VM
1674	*
1675	* @adev: amdgpu_device pointer
1676	* @vm: requested vm
1677	* @flush_type: flush type
1678	* @xcc_mask: mask of XCCs that belong to the compute partition in need of a TLB flush.
1679	*
1680	* Flush TLB if needed for a compute VM.
1681	*
1682	* Returns:
1683	* 0 for success.
1684	*/
1685	int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
1686	struct amdgpu_vm *vm,
1687	uint32_t flush_type,
1688	uint32_t xcc_mask)
1689	{
1690	uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
1691	bool all_hub = false;
1692	int xcc = 0, r = 0;
1693
1694	WARN_ON_ONCE(!vm->is_compute_context);
1695
1696	/*
1697	* It can be that we race and lose here, but that is extremely unlikely
1698	* and the worst thing which could happen is that we flush the changes
1699	* into the TLB once more which is harmless.
1700	*/
1701	if (atomic64_xchg(v: &vm->kfd_last_flushed_seq, new: tlb_seq) == tlb_seq)
1702	return 0;
1703
1704	if (adev->family == AMDGPU_FAMILY_AI ||
1705	adev->family == AMDGPU_FAMILY_RV)
1706	all_hub = true;
1707
1708	for_each_inst(xcc, xcc_mask) {
1709	r = amdgpu_gmc_flush_gpu_tlb_pasid(adev, pasid: vm->pasid, flush_type,
1710	all_hub, inst: xcc);
1711	if (r)
1712	break;
1713	}
1714	return r;
1715	}
1716
1717	/**
1718	* amdgpu_vm_bo_add - add a bo to a specific vm
1719	*
1720	* @adev: amdgpu_device pointer
1721	* @vm: requested vm
1722	* @bo: amdgpu buffer object
1723	*
1724	* Add @bo into the requested vm.
1725	* Add @bo to the list of bos associated with the vm
1726	*
1727	* Returns:
1728	* Newly added bo_va or NULL for failure
1729	*
1730	* Object has to be reserved!
1731	*/
1732	struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1733	struct amdgpu_vm *vm,
1734	struct amdgpu_bo *bo)
1735	{
1736	struct amdgpu_bo_va *bo_va;
1737
1738	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1739	if (bo_va == NULL) {
1740	return NULL;
1741	}
1742	amdgpu_vm_bo_base_init(base: &bo_va->base, vm, bo);
1743
1744	bo_va->ref_count = 1;
1745	bo_va->last_pt_update = dma_fence_get_stub();
1746	INIT_LIST_HEAD(list: &bo_va->valids);
1747	INIT_LIST_HEAD(list: &bo_va->invalids);
1748
1749	if (!bo)
1750	return bo_va;
1751
1752	dma_resv_assert_held(bo->tbo.base.resv);
1753	if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) {
1754	bo_va->is_xgmi = true;
1755	/* Power up XGMI if it can be potentially used */
1756	amdgpu_xgmi_set_pstate(adev, pstate: AMDGPU_XGMI_PSTATE_MAX_VEGA20);
1757	}
1758
1759	return bo_va;
1760	}
1761
1762
1763	/**
1764	* amdgpu_vm_bo_insert_map - insert a new mapping
1765	*
1766	* @adev: amdgpu_device pointer
1767	* @bo_va: bo_va to store the address
1768	* @mapping: the mapping to insert
1769	*
1770	* Insert a new mapping into all structures.
1771	*/
1772	static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
1773	struct amdgpu_bo_va *bo_va,
1774	struct amdgpu_bo_va_mapping *mapping)
1775	{
1776	struct amdgpu_vm *vm = bo_va->base.vm;
1777	struct amdgpu_bo *bo = bo_va->base.bo;
1778
1779	mapping->bo_va = bo_va;
1780	list_add(new: &mapping->list, head: &bo_va->invalids);
1781	amdgpu_vm_it_insert(node: mapping, root: &vm->va);
1782
1783	if (mapping->flags & AMDGPU_VM_PAGE_PRT)
1784	amdgpu_vm_prt_get(adev);
1785
1786	if (amdgpu_vm_is_bo_always_valid(vm, bo) && !bo_va->base.moved)
1787	amdgpu_vm_bo_moved(vm_bo: &bo_va->base);
1788
1789	trace_amdgpu_vm_bo_map(bo_va, mapping);
1790	}
1791
1792	/* Validate operation parameters to prevent potential abuse */
1793	static int amdgpu_vm_verify_parameters(struct amdgpu_device *adev,
1794	struct amdgpu_bo *bo,
1795	uint64_t saddr,
1796	uint64_t offset,
1797	uint64_t size)
1798	{
1799	uint64_t tmp, lpfn;
1800
1801	if (saddr & AMDGPU_GPU_PAGE_MASK
1802	|| offset & AMDGPU_GPU_PAGE_MASK
1803	|| size & AMDGPU_GPU_PAGE_MASK)
1804	return -EINVAL;
1805
1806	if (check_add_overflow(saddr, size, &tmp)
1807	|| check_add_overflow(offset, size, &tmp)
1808	|| size == 0 /* which also leads to end < begin */)
1809	return -EINVAL;
1810
1811	/* make sure object fit at this offset */
1812	if (bo && offset + size > amdgpu_bo_size(bo))
1813	return -EINVAL;
1814
1815	/* Ensure last pfn not exceed max_pfn */
1816	lpfn = (saddr + size - 1) >> AMDGPU_GPU_PAGE_SHIFT;
1817	if (lpfn >= adev->vm_manager.max_pfn)
1818	return -EINVAL;
1819
1820	return 0;
1821	}
1822
1823	/**
1824	* amdgpu_vm_bo_map - map bo inside a vm
1825	*
1826	* @adev: amdgpu_device pointer
1827	* @bo_va: bo_va to store the address
1828	* @saddr: where to map the BO
1829	* @offset: requested offset in the BO
1830	* @size: BO size in bytes
1831	* @flags: attributes of pages (read/write/valid/etc.)
1832	*
1833	* Add a mapping of the BO at the specefied addr into the VM.
1834	*
1835	* Returns:
1836	* 0 for success, error for failure.
1837	*
1838	* Object has to be reserved and unreserved outside!
1839	*/
1840	int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1841	struct amdgpu_bo_va *bo_va,
1842	uint64_t saddr, uint64_t offset,
1843	uint64_t size, uint32_t flags)
1844	{
1845	struct amdgpu_bo_va_mapping *mapping, *tmp;
1846	struct amdgpu_bo *bo = bo_va->base.bo;
1847	struct amdgpu_vm *vm = bo_va->base.vm;
1848	uint64_t eaddr;
1849	int r;
1850
1851	r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
1852	if (r)
1853	return r;
1854
1855	saddr /= AMDGPU_GPU_PAGE_SIZE;
1856	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
1857
1858	tmp = amdgpu_vm_it_iter_first(root: &vm->va, start: saddr, last: eaddr);
1859	if (tmp) {
1860	/* bo and tmp overlap, invalid addr */
1861	dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1862	"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1863	tmp->start, tmp->last + 1);
1864	return -EINVAL;
1865	}
1866
1867	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1868	if (!mapping)
1869	return -ENOMEM;
1870
1871	mapping->start = saddr;
1872	mapping->last = eaddr;
1873	mapping->offset = offset;
1874	mapping->flags = flags;
1875
1876	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1877
1878	return 0;
1879	}
1880
1881	/**
1882	* amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
1883	*
1884	* @adev: amdgpu_device pointer
1885	* @bo_va: bo_va to store the address
1886	* @saddr: where to map the BO
1887	* @offset: requested offset in the BO
1888	* @size: BO size in bytes
1889	* @flags: attributes of pages (read/write/valid/etc.)
1890	*
1891	* Add a mapping of the BO at the specefied addr into the VM. Replace existing
1892	* mappings as we do so.
1893	*
1894	* Returns:
1895	* 0 for success, error for failure.
1896	*
1897	* Object has to be reserved and unreserved outside!
1898	*/
1899	int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
1900	struct amdgpu_bo_va *bo_va,
1901	uint64_t saddr, uint64_t offset,
1902	uint64_t size, uint32_t flags)
1903	{
1904	struct amdgpu_bo_va_mapping *mapping;
1905	struct amdgpu_bo *bo = bo_va->base.bo;
1906	uint64_t eaddr;
1907	int r;
1908
1909	r = amdgpu_vm_verify_parameters(adev, bo, saddr, offset, size);
1910	if (r)
1911	return r;
1912
1913	/* Allocate all the needed memory */
1914	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1915	if (!mapping)
1916	return -ENOMEM;
1917
1918	r = amdgpu_vm_bo_clear_mappings(adev, vm: bo_va->base.vm, saddr, size);
1919	if (r) {
1920	kfree(objp: mapping);
1921	return r;
1922	}
1923
1924	saddr /= AMDGPU_GPU_PAGE_SIZE;
1925	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
1926
1927	mapping->start = saddr;
1928	mapping->last = eaddr;
1929	mapping->offset = offset;
1930	mapping->flags = flags;
1931
1932	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1933
1934	return 0;
1935	}
1936
1937	/**
1938	* amdgpu_vm_bo_unmap - remove bo mapping from vm
1939	*
1940	* @adev: amdgpu_device pointer
1941	* @bo_va: bo_va to remove the address from
1942	* @saddr: where to the BO is mapped
1943	*
1944	* Remove a mapping of the BO at the specefied addr from the VM.
1945	*
1946	* Returns:
1947	* 0 for success, error for failure.
1948	*
1949	* Object has to be reserved and unreserved outside!
1950	*/
1951	int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1952	struct amdgpu_bo_va *bo_va,
1953	uint64_t saddr)
1954	{
1955	struct amdgpu_bo_va_mapping *mapping;
1956	struct amdgpu_vm *vm = bo_va->base.vm;
1957	bool valid = true;
1958	int r;
1959
1960	saddr /= AMDGPU_GPU_PAGE_SIZE;
1961
1962	list_for_each_entry(mapping, &bo_va->valids, list) {
1963	if (mapping->start == saddr)
1964	break;
1965	}
1966
1967	if (&mapping->list == &bo_va->valids) {
1968	valid = false;
1969
1970	list_for_each_entry(mapping, &bo_va->invalids, list) {
1971	if (mapping->start == saddr)
1972	break;
1973	}
1974
1975	if (&mapping->list == &bo_va->invalids)
1976	return -ENOENT;
1977	}
1978
1979	/* It's unlikely to happen that the mapping userq hasn't been idled
1980	* during user requests GEM unmap IOCTL except for forcing the unmap
1981	* from user space.
1982	*/
1983	if (unlikely(atomic_read(&bo_va->userq_va_mapped) > 0)) {
1984	r = amdgpu_userq_gem_va_unmap_validate(adev, mapping, saddr);
1985	if (unlikely(r == -EBUSY))
1986	dev_warn_once(adev->dev,
1987	"Attempt to unmap an active userq buffer\n");
1988	}
1989
1990	list_del(entry: &mapping->list);
1991	amdgpu_vm_it_remove(node: mapping, root: &vm->va);
1992	mapping->bo_va = NULL;
1993	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1994
1995	if (valid)
1996	list_add(new: &mapping->list, head: &vm->freed);
1997	else
1998	amdgpu_vm_free_mapping(adev, vm, mapping,
1999	fence: bo_va->last_pt_update);
2000
2001	return 0;
2002	}
2003
2004	/**
2005	* amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2006	*
2007	* @adev: amdgpu_device pointer
2008	* @vm: VM structure to use
2009	* @saddr: start of the range
2010	* @size: size of the range
2011	*
2012	* Remove all mappings in a range, split them as appropriate.
2013	*
2014	* Returns:
2015	* 0 for success, error for failure.
2016	*/
2017	int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2018	struct amdgpu_vm *vm,
2019	uint64_t saddr, uint64_t size)
2020	{
2021	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2022	LIST_HEAD(removed);
2023	uint64_t eaddr;
2024	int r;
2025
2026	r = amdgpu_vm_verify_parameters(adev, NULL, saddr, offset: 0, size);
2027	if (r)
2028	return r;
2029
2030	saddr /= AMDGPU_GPU_PAGE_SIZE;
2031	eaddr = saddr + (size - 1) / AMDGPU_GPU_PAGE_SIZE;
2032
2033	/* Allocate all the needed memory */
2034	before = kzalloc(sizeof(*before), GFP_KERNEL);
2035	if (!before)
2036	return -ENOMEM;
2037	INIT_LIST_HEAD(list: &before->list);
2038
2039	after = kzalloc(sizeof(*after), GFP_KERNEL);
2040	if (!after) {
2041	kfree(objp: before);
2042	return -ENOMEM;
2043	}
2044	INIT_LIST_HEAD(list: &after->list);
2045
2046	/* Now gather all removed mappings */
2047	tmp = amdgpu_vm_it_iter_first(root: &vm->va, start: saddr, last: eaddr);
2048	while (tmp) {
2049	/* Remember mapping split at the start */
2050	if (tmp->start < saddr) {
2051	before->start = tmp->start;
2052	before->last = saddr - 1;
2053	before->offset = tmp->offset;
2054	before->flags = tmp->flags;
2055	before->bo_va = tmp->bo_va;
2056	list_add(new: &before->list, head: &tmp->bo_va->invalids);
2057	}
2058
2059	/* Remember mapping split at the end */
2060	if (tmp->last > eaddr) {
2061	after->start = eaddr + 1;
2062	after->last = tmp->last;
2063	after->offset = tmp->offset;
2064	after->offset += (after->start - tmp->start) << PAGE_SHIFT;
2065	after->flags = tmp->flags;
2066	after->bo_va = tmp->bo_va;
2067	list_add(new: &after->list, head: &tmp->bo_va->invalids);
2068	}
2069
2070	list_del(entry: &tmp->list);
2071	list_add(new: &tmp->list, head: &removed);
2072
2073	tmp = amdgpu_vm_it_iter_next(node: tmp, start: saddr, last: eaddr);
2074	}
2075
2076	/* And free them up */
2077	list_for_each_entry_safe(tmp, next, &removed, list) {
2078	amdgpu_vm_it_remove(node: tmp, root: &vm->va);
2079	list_del(entry: &tmp->list);
2080
2081	if (tmp->start < saddr)
2082	tmp->start = saddr;
2083	if (tmp->last > eaddr)
2084	tmp->last = eaddr;
2085
2086	tmp->bo_va = NULL;
2087	list_add(new: &tmp->list, head: &vm->freed);
2088	trace_amdgpu_vm_bo_unmap(NULL, mapping: tmp);
2089	}
2090
2091	/* Insert partial mapping before the range */
2092	if (!list_empty(head: &before->list)) {
2093	struct amdgpu_bo *bo = before->bo_va->base.bo;
2094
2095	amdgpu_vm_it_insert(node: before, root: &vm->va);
2096	if (before->flags & AMDGPU_VM_PAGE_PRT)
2097	amdgpu_vm_prt_get(adev);
2098
2099	if (amdgpu_vm_is_bo_always_valid(vm, bo) &&
2100	!before->bo_va->base.moved)
2101	amdgpu_vm_bo_moved(vm_bo: &before->bo_va->base);
2102	} else {
2103	kfree(objp: before);
2104	}
2105
2106	/* Insert partial mapping after the range */
2107	if (!list_empty(head: &after->list)) {
2108	struct amdgpu_bo *bo = after->bo_va->base.bo;
2109
2110	amdgpu_vm_it_insert(node: after, root: &vm->va);
2111	if (after->flags & AMDGPU_VM_PAGE_PRT)
2112	amdgpu_vm_prt_get(adev);
2113
2114	if (amdgpu_vm_is_bo_always_valid(vm, bo) &&
2115	!after->bo_va->base.moved)
2116	amdgpu_vm_bo_moved(vm_bo: &after->bo_va->base);
2117	} else {
2118	kfree(objp: after);
2119	}
2120
2121	return 0;
2122	}
2123
2124	/**
2125	* amdgpu_vm_bo_lookup_mapping - find mapping by address
2126	*
2127	* @vm: the requested VM
2128	* @addr: the address
2129	*
2130	* Find a mapping by it's address.
2131	*
2132	* Returns:
2133	* The amdgpu_bo_va_mapping matching for addr or NULL
2134	*
2135	*/
2136	struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2137	uint64_t addr)
2138	{
2139	return amdgpu_vm_it_iter_first(root: &vm->va, start: addr, last: addr);
2140	}
2141
2142	/**
2143	* amdgpu_vm_bo_trace_cs - trace all reserved mappings
2144	*
2145	* @vm: the requested vm
2146	* @ticket: CS ticket
2147	*
2148	* Trace all mappings of BOs reserved during a command submission.
2149	*/
2150	void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
2151	{
2152	struct amdgpu_bo_va_mapping *mapping;
2153
2154	if (!trace_amdgpu_vm_bo_cs_enabled())
2155	return;
2156
2157	for (mapping = amdgpu_vm_it_iter_first(root: &vm->va, start: 0, U64_MAX); mapping;
2158	mapping = amdgpu_vm_it_iter_next(node: mapping, start: 0, U64_MAX)) {
2159	if (mapping->bo_va && mapping->bo_va->base.bo) {
2160	struct amdgpu_bo *bo;
2161
2162	bo = mapping->bo_va->base.bo;
2163	if (dma_resv_locking_ctx(obj: bo->tbo.base.resv) !=
2164	ticket)
2165	continue;
2166	}
2167
2168	trace_amdgpu_vm_bo_cs(mapping);
2169	}
2170	}
2171
2172	/**
2173	* amdgpu_vm_bo_del - remove a bo from a specific vm
2174	*
2175	* @adev: amdgpu_device pointer
2176	* @bo_va: requested bo_va
2177	*
2178	* Remove @bo_va->bo from the requested vm.
2179	*
2180	* Object have to be reserved!
2181	*/
2182	void amdgpu_vm_bo_del(struct amdgpu_device *adev,
2183	struct amdgpu_bo_va *bo_va)
2184	{
2185	struct amdgpu_bo_va_mapping *mapping, *next;
2186	struct amdgpu_bo *bo = bo_va->base.bo;
2187	struct amdgpu_vm *vm = bo_va->base.vm;
2188	struct amdgpu_vm_bo_base **base;
2189
2190	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
2191
2192	if (bo) {
2193	dma_resv_assert_held(bo->tbo.base.resv);
2194	if (amdgpu_vm_is_bo_always_valid(vm, bo))
2195	ttm_bo_set_bulk_move(bo: &bo->tbo, NULL);
2196
2197	for (base = &bo_va->base.bo->vm_bo; *base;
2198	base = &(*base)->next) {
2199	if (*base != &bo_va->base)
2200	continue;
2201
2202	amdgpu_vm_update_stats(base: *base, res: bo->tbo.resource, sign: -1);
2203	*base = bo_va->base.next;
2204	break;
2205	}
2206	}
2207
2208	spin_lock(lock: &vm->status_lock);
2209	list_del(entry: &bo_va->base.vm_status);
2210	spin_unlock(lock: &vm->status_lock);
2211
2212	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2213	list_del(entry: &mapping->list);
2214	amdgpu_vm_it_remove(node: mapping, root: &vm->va);
2215	mapping->bo_va = NULL;
2216	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2217	list_add(new: &mapping->list, head: &vm->freed);
2218	}
2219	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2220	list_del(entry: &mapping->list);
2221	amdgpu_vm_it_remove(node: mapping, root: &vm->va);
2222	amdgpu_vm_free_mapping(adev, vm, mapping,
2223	fence: bo_va->last_pt_update);
2224	}
2225
2226	dma_fence_put(fence: bo_va->last_pt_update);
2227
2228	if (bo && bo_va->is_xgmi)
2229	amdgpu_xgmi_set_pstate(adev, pstate: AMDGPU_XGMI_PSTATE_MIN);
2230
2231	kfree(objp: bo_va);
2232	}
2233
2234	/**
2235	* amdgpu_vm_evictable - check if we can evict a VM
2236	*
2237	* @bo: A page table of the VM.
2238	*
2239	* Check if it is possible to evict a VM.
2240	*/
2241	bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
2242	{
2243	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
2244
2245	/* Page tables of a destroyed VM can go away immediately */
2246	if (!bo_base || !bo_base->vm)
2247	return true;
2248
2249	/* Don't evict VM page tables while they are busy */
2250	if (!dma_resv_test_signaled(obj: bo->tbo.base.resv, usage: DMA_RESV_USAGE_BOOKKEEP))
2251	return false;
2252
2253	/* Try to block ongoing updates */
2254	if (!amdgpu_vm_eviction_trylock(vm: bo_base->vm))
2255	return false;
2256
2257	/* Don't evict VM page tables while they are updated */
2258	if (!dma_fence_is_signaled(fence: bo_base->vm->last_unlocked)) {
2259	amdgpu_vm_eviction_unlock(vm: bo_base->vm);
2260	return false;
2261	}
2262
2263	bo_base->vm->evicting = true;
2264	amdgpu_vm_eviction_unlock(vm: bo_base->vm);
2265	return true;
2266	}
2267
2268	/**
2269	* amdgpu_vm_bo_invalidate - mark the bo as invalid
2270	*
2271	* @bo: amdgpu buffer object
2272	* @evicted: is the BO evicted
2273	*
2274	* Mark @bo as invalid.
2275	*/
2276	void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted)
2277	{
2278	struct amdgpu_vm_bo_base *bo_base;
2279
2280	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2281	struct amdgpu_vm *vm = bo_base->vm;
2282
2283	if (evicted && amdgpu_vm_is_bo_always_valid(vm, bo)) {
2284	amdgpu_vm_bo_evicted(vm_bo: bo_base);
2285	continue;
2286	}
2287
2288	if (bo_base->moved)
2289	continue;
2290	bo_base->moved = true;
2291
2292	if (bo->tbo.type == ttm_bo_type_kernel)
2293	amdgpu_vm_bo_relocated(vm_bo: bo_base);
2294	else if (amdgpu_vm_is_bo_always_valid(vm, bo))
2295	amdgpu_vm_bo_moved(vm_bo: bo_base);
2296	else
2297	amdgpu_vm_bo_invalidated(vm_bo: bo_base);
2298	}
2299	}
2300
2301	/**
2302	* amdgpu_vm_bo_move - handle BO move
2303	*
2304	* @bo: amdgpu buffer object
2305	* @new_mem: the new placement of the BO move
2306	* @evicted: is the BO evicted
2307	*
2308	* Update the memory stats for the new placement and mark @bo as invalid.
2309	*/
2310	void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
2311	bool evicted)
2312	{
2313	struct amdgpu_vm_bo_base *bo_base;
2314
2315	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2316	struct amdgpu_vm *vm = bo_base->vm;
2317
2318	spin_lock(lock: &vm->status_lock);
2319	amdgpu_vm_update_stats_locked(base: bo_base, res: bo->tbo.resource, sign: -1);
2320	amdgpu_vm_update_stats_locked(base: bo_base, res: new_mem, sign: +1);
2321	spin_unlock(lock: &vm->status_lock);
2322	}
2323
2324	amdgpu_vm_bo_invalidate(bo, evicted);
2325	}
2326
2327	/**
2328	* amdgpu_vm_get_block_size - calculate VM page table size as power of two
2329	*
2330	* @vm_size: VM size
2331	*
2332	* Returns:
2333	* VM page table as power of two
2334	*/
2335	static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2336	{
2337	/* Total bits covered by PD + PTs */
2338	unsigned bits = ilog2(vm_size) + 18;
2339
2340	/* Make sure the PD is 4K in size up to 8GB address space.
2341	Above that split equal between PD and PTs */
2342	if (vm_size <= 8)
2343	return (bits - 9);
2344	else
2345	return ((bits + 3) / 2);
2346	}
2347
2348	/**
2349	* amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2350	*
2351	* @adev: amdgpu_device pointer
2352	* @min_vm_size: the minimum vm size in GB if it's set auto
2353	* @fragment_size_default: Default PTE fragment size
2354	* @max_level: max VMPT level
2355	* @max_bits: max address space size in bits
2356	*
2357	*/
2358	void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
2359	uint32_t fragment_size_default, unsigned max_level,
2360	unsigned max_bits)
2361	{
2362	unsigned int max_size = 1 << (max_bits - 30);
2363	unsigned int vm_size;
2364	uint64_t tmp;
2365
2366	/* adjust vm size first */
2367	if (amdgpu_vm_size != -1) {
2368	vm_size = amdgpu_vm_size;
2369	if (vm_size > max_size) {
2370	dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2371	amdgpu_vm_size, max_size);
2372	vm_size = max_size;
2373	}
2374	} else {
2375	struct sysinfo si;
2376	unsigned int phys_ram_gb;
2377
2378	/* Optimal VM size depends on the amount of physical
2379	* RAM available. Underlying requirements and
2380	* assumptions:
2381	*
2382	* - Need to map system memory and VRAM from all GPUs
2383	* - VRAM from other GPUs not known here
2384	* - Assume VRAM <= system memory
2385	* - On GFX8 and older, VM space can be segmented for
2386	* different MTYPEs
2387	* - Need to allow room for fragmentation, guard pages etc.
2388	*
2389	* This adds up to a rough guess of system memory x3.
2390	* Round up to power of two to maximize the available
2391	* VM size with the given page table size.
2392	*/
2393	si_meminfo(val: &si);
2394	phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
2395	(1 << 30) - 1) >> 30;
2396	vm_size = roundup_pow_of_two(
2397	clamp(phys_ram_gb * 3, min_vm_size, max_size));
2398	}
2399
2400	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2401
2402	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2403	if (amdgpu_vm_block_size != -1)
2404	tmp >>= amdgpu_vm_block_size - 9;
2405	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2406	adev->vm_manager.num_level = min_t(unsigned int, max_level, tmp);
2407	switch (adev->vm_manager.num_level) {
2408	case 3:
2409	adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2410	break;
2411	case 2:
2412	adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2413	break;
2414	case 1:
2415	adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2416	break;
2417	default:
2418	dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2419	}
2420	/* block size depends on vm size and hw setup*/
2421	if (amdgpu_vm_block_size != -1)
2422	adev->vm_manager.block_size =
2423	min((unsigned)amdgpu_vm_block_size, max_bits
2424	- AMDGPU_GPU_PAGE_SHIFT
2425	- 9 * adev->vm_manager.num_level);
2426	else if (adev->vm_manager.num_level > 1)
2427	adev->vm_manager.block_size = 9;
2428	else
2429	adev->vm_manager.block_size = amdgpu_vm_get_block_size(vm_size: tmp);
2430
2431	if (amdgpu_vm_fragment_size == -1)
2432	adev->vm_manager.fragment_size = fragment_size_default;
2433	else
2434	adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2435
2436	dev_info(
2437	adev->dev,
2438	"vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2439	vm_size, adev->vm_manager.num_level + 1,
2440	adev->vm_manager.block_size, adev->vm_manager.fragment_size);
2441	}
2442
2443	/**
2444	* amdgpu_vm_wait_idle - wait for the VM to become idle
2445	*
2446	* @vm: VM object to wait for
2447	* @timeout: timeout to wait for VM to become idle
2448	*/
2449	long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2450	{
2451	timeout = drm_sched_entity_flush(entity: &vm->immediate, timeout);
2452	if (timeout <= 0)
2453	return timeout;
2454
2455	return drm_sched_entity_flush(entity: &vm->delayed, timeout);
2456	}
2457
2458	static void amdgpu_vm_destroy_task_info(struct kref *kref)
2459	{
2460	struct amdgpu_task_info *ti = container_of(kref, struct amdgpu_task_info, refcount);
2461
2462	kfree(objp: ti);
2463	}
2464
2465	static inline struct amdgpu_vm *
2466	amdgpu_vm_get_vm_from_pasid(struct amdgpu_device *adev, u32 pasid)
2467	{
2468	struct amdgpu_vm *vm;
2469	unsigned long flags;
2470
2471	xa_lock_irqsave(&adev->vm_manager.pasids, flags);
2472	vm = xa_load(&adev->vm_manager.pasids, index: pasid);
2473	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
2474
2475	return vm;
2476	}
2477
2478	/**
2479	* amdgpu_vm_put_task_info - reference down the vm task_info ptr
2480	*
2481	* @task_info: task_info struct under discussion.
2482	*
2483	* frees the vm task_info ptr at the last put
2484	*/
2485	void amdgpu_vm_put_task_info(struct amdgpu_task_info *task_info)
2486	{
2487	if (task_info)
2488	kref_put(kref: &task_info->refcount, release: amdgpu_vm_destroy_task_info);
2489	}
2490
2491	/**
2492	* amdgpu_vm_get_task_info_vm - Extracts task info for a vm.
2493	*
2494	* @vm: VM to get info from
2495	*
2496	* Returns the reference counted task_info structure, which must be
2497	* referenced down with amdgpu_vm_put_task_info.
2498	*/
2499	struct amdgpu_task_info *
2500	amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm)
2501	{
2502	struct amdgpu_task_info *ti = NULL;
2503
2504	if (vm) {
2505	ti = vm->task_info;
2506	kref_get(kref: &vm->task_info->refcount);
2507	}
2508
2509	return ti;
2510	}
2511
2512	/**
2513	* amdgpu_vm_get_task_info_pasid - Extracts task info for a PASID.
2514	*
2515	* @adev: drm device pointer
2516	* @pasid: PASID identifier for VM
2517	*
2518	* Returns the reference counted task_info structure, which must be
2519	* referenced down with amdgpu_vm_put_task_info.
2520	*/
2521	struct amdgpu_task_info *
2522	amdgpu_vm_get_task_info_pasid(struct amdgpu_device *adev, u32 pasid)
2523	{
2524	return amdgpu_vm_get_task_info_vm(
2525	vm: amdgpu_vm_get_vm_from_pasid(adev, pasid));
2526	}
2527
2528	static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm)
2529	{
2530	vm->task_info = kzalloc(sizeof(struct amdgpu_task_info), GFP_KERNEL);
2531	if (!vm->task_info)
2532	return -ENOMEM;
2533
2534	kref_init(kref: &vm->task_info->refcount);
2535	return 0;
2536	}
2537
2538	/**
2539	* amdgpu_vm_set_task_info - Sets VMs task info.
2540	*
2541	* @vm: vm for which to set the info
2542	*/
2543	void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
2544	{
2545	if (!vm->task_info)
2546	return;
2547
2548	if (vm->task_info->task.pid == current->pid)
2549	return;
2550
2551	vm->task_info->task.pid = current->pid;
2552	get_task_comm(vm->task_info->task.comm, current);
2553
2554	if (current->group_leader->mm != current->mm)
2555	return;
2556
2557	vm->task_info->tgid = current->group_leader->pid;
2558	get_task_comm(vm->task_info->process_name, current->group_leader);
2559	}
2560
2561	/**
2562	* amdgpu_vm_init - initialize a vm instance
2563	*
2564	* @adev: amdgpu_device pointer
2565	* @vm: requested vm
2566	* @xcp_id: GPU partition selection id
2567	* @pasid: the pasid the VM is using on this GPU
2568	*
2569	* Init @vm fields.
2570	*
2571	* Returns:
2572	* 0 for success, error for failure.
2573	*/
2574	int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2575	int32_t xcp_id, uint32_t pasid)
2576	{
2577	struct amdgpu_bo *root_bo;
2578	struct amdgpu_bo_vm *root;
2579	int r, i;
2580
2581	vm->va = RB_ROOT_CACHED;
2582	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2583	vm->reserved_vmid[i] = NULL;
2584	INIT_LIST_HEAD(list: &vm->evicted);
2585	INIT_LIST_HEAD(list: &vm->evicted_user);
2586	INIT_LIST_HEAD(list: &vm->relocated);
2587	INIT_LIST_HEAD(list: &vm->moved);
2588	INIT_LIST_HEAD(list: &vm->idle);
2589	INIT_LIST_HEAD(list: &vm->invalidated);
2590	spin_lock_init(&vm->status_lock);
2591	INIT_LIST_HEAD(list: &vm->freed);
2592	INIT_LIST_HEAD(list: &vm->done);
2593	INIT_KFIFO(vm->faults);
2594
2595	r = amdgpu_vm_init_entities(adev, vm);
2596	if (r)
2597	return r;
2598
2599	ttm_lru_bulk_move_init(bulk: &vm->lru_bulk_move);
2600
2601	vm->is_compute_context = false;
2602
2603	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2604	AMDGPU_VM_USE_CPU_FOR_GFX);
2605
2606	dev_dbg(adev->dev, "VM update mode is %s\n",
2607	vm->use_cpu_for_update ? "CPU" : "SDMA");
2608	WARN_ONCE((vm->use_cpu_for_update &&
2609	!amdgpu_gmc_vram_full_visible(&adev->gmc)),
2610	"CPU update of VM recommended only for large BAR system\n");
2611
2612	if (vm->use_cpu_for_update)
2613	vm->update_funcs = &amdgpu_vm_cpu_funcs;
2614	else
2615	vm->update_funcs = &amdgpu_vm_sdma_funcs;
2616
2617	vm->last_update = dma_fence_get_stub();
2618	vm->last_unlocked = dma_fence_get_stub();
2619	vm->last_tlb_flush = dma_fence_get_stub();
2620	vm->generation = amdgpu_vm_generation(adev, NULL);
2621
2622	mutex_init(&vm->eviction_lock);
2623	vm->evicting = false;
2624	vm->tlb_fence_context = dma_fence_context_alloc(num: 1);
2625
2626	r = amdgpu_vm_pt_create(adev, vm, level: adev->vm_manager.root_level,
2627	immediate: false, vmbo: &root, xcp_id);
2628	if (r)
2629	goto error_free_delayed;
2630
2631	root_bo = amdgpu_bo_ref(bo: &root->bo);
2632	r = amdgpu_bo_reserve(bo: root_bo, no_intr: true);
2633	if (r) {
2634	amdgpu_bo_unref(bo: &root_bo);
2635	goto error_free_delayed;
2636	}
2637
2638	amdgpu_vm_bo_base_init(base: &vm->root, vm, bo: root_bo);
2639	r = dma_resv_reserve_fences(obj: root_bo->tbo.base.resv, num_fences: 1);
2640	if (r)
2641	goto error_free_root;
2642
2643	r = amdgpu_vm_pt_clear(adev, vm, vmbo: root, immediate: false);
2644	if (r)
2645	goto error_free_root;
2646
2647	r = amdgpu_vm_create_task_info(vm);
2648	if (r)
2649	dev_dbg(adev->dev, "Failed to create task info for VM\n");
2650
2651	/* Store new PASID in XArray (if non-zero) */
2652	if (pasid != 0) {
2653	r = xa_err(entry: xa_store_irq(xa: &adev->vm_manager.pasids, index: pasid, entry: vm, GFP_KERNEL));
2654	if (r < 0)
2655	goto error_free_root;
2656
2657	vm->pasid = pasid;
2658	}
2659
2660	amdgpu_bo_unreserve(bo: vm->root.bo);
2661	amdgpu_bo_unref(bo: &root_bo);
2662
2663	return 0;
2664
2665	error_free_root:
2666	/* If PASID was partially set, erase it from XArray before failing */
2667	if (vm->pasid != 0) {
2668	xa_erase_irq(xa: &adev->vm_manager.pasids, index: vm->pasid);
2669	vm->pasid = 0;
2670	}
2671	amdgpu_vm_pt_free_root(adev, vm);
2672	amdgpu_bo_unreserve(bo: vm->root.bo);
2673	amdgpu_bo_unref(bo: &root_bo);
2674
2675	error_free_delayed:
2676	dma_fence_put(fence: vm->last_tlb_flush);
2677	dma_fence_put(fence: vm->last_unlocked);
2678	ttm_lru_bulk_move_fini(bdev: &adev->mman.bdev, bulk: &vm->lru_bulk_move);
2679	amdgpu_vm_fini_entities(vm);
2680
2681	return r;
2682	}
2683
2684	/**
2685	* amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2686	*
2687	* @adev: amdgpu_device pointer
2688	* @vm: requested vm
2689	*
2690	* This only works on GFX VMs that don't have any BOs added and no
2691	* page tables allocated yet.
2692	*
2693	* Changes the following VM parameters:
2694	* - use_cpu_for_update
2695	* - pte_supports_ats
2696	*
2697	* Reinitializes the page directory to reflect the changed ATS
2698	* setting.
2699	*
2700	* Returns:
2701	* 0 for success, -errno for errors.
2702	*/
2703	int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2704	{
2705	int r;
2706
2707	r = amdgpu_bo_reserve(bo: vm->root.bo, no_intr: true);
2708	if (r)
2709	return r;
2710
2711	/* Update VM state */
2712	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2713	AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2714	dev_dbg(adev->dev, "VM update mode is %s\n",
2715	vm->use_cpu_for_update ? "CPU" : "SDMA");
2716	WARN_ONCE((vm->use_cpu_for_update &&
2717	!amdgpu_gmc_vram_full_visible(&adev->gmc)),
2718	"CPU update of VM recommended only for large BAR system\n");
2719
2720	if (vm->use_cpu_for_update) {
2721	/* Sync with last SDMA update/clear before switching to CPU */
2722	r = amdgpu_bo_sync_wait(bo: vm->root.bo,
2723	AMDGPU_FENCE_OWNER_UNDEFINED, intr: true);
2724	if (r)
2725	goto unreserve_bo;
2726
2727	vm->update_funcs = &amdgpu_vm_cpu_funcs;
2728	r = amdgpu_vm_pt_map_tables(adev, vm);
2729	if (r)
2730	goto unreserve_bo;
2731
2732	} else {
2733	vm->update_funcs = &amdgpu_vm_sdma_funcs;
2734	}
2735
2736	dma_fence_put(fence: vm->last_update);
2737	vm->last_update = dma_fence_get_stub();
2738	vm->is_compute_context = true;
2739
2740	unreserve_bo:
2741	amdgpu_bo_unreserve(bo: vm->root.bo);
2742	return r;
2743	}
2744
2745	static int amdgpu_vm_stats_is_zero(struct amdgpu_vm *vm)
2746	{
2747	for (int i = 0; i < __AMDGPU_PL_NUM; ++i) {
2748	if (!(drm_memory_stats_is_zero(stats: &vm->stats[i].drm) &&
2749	vm->stats[i].evicted == 0))
2750	return false;
2751	}
2752	return true;
2753	}
2754
2755	/**
2756	* amdgpu_vm_fini - tear down a vm instance
2757	*
2758	* @adev: amdgpu_device pointer
2759	* @vm: requested vm
2760	*
2761	* Tear down @vm.
2762	* Unbind the VM and remove all bos from the vm bo list
2763	*/
2764	void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2765	{
2766	struct amdgpu_bo_va_mapping *mapping, *tmp;
2767	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2768	struct amdgpu_bo *root;
2769	unsigned long flags;
2770	int i;
2771
2772	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2773
2774	root = amdgpu_bo_ref(bo: vm->root.bo);
2775	amdgpu_bo_reserve(bo: root, no_intr: true);
2776	/* Remove PASID mapping before destroying VM */
2777	if (vm->pasid != 0) {
2778	xa_erase_irq(xa: &adev->vm_manager.pasids, index: vm->pasid);
2779	vm->pasid = 0;
2780	}
2781	dma_fence_wait(fence: vm->last_unlocked, intr: false);
2782	dma_fence_put(fence: vm->last_unlocked);
2783	dma_fence_wait(fence: vm->last_tlb_flush, intr: false);
2784	/* Make sure that all fence callbacks have completed */
2785	spin_lock_irqsave(vm->last_tlb_flush->lock, flags);
2786	spin_unlock_irqrestore(lock: vm->last_tlb_flush->lock, flags);
2787	dma_fence_put(fence: vm->last_tlb_flush);
2788
2789	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2790	if (mapping->flags & AMDGPU_VM_PAGE_PRT && prt_fini_needed) {
2791	amdgpu_vm_prt_fini(adev, vm);
2792	prt_fini_needed = false;
2793	}
2794
2795	list_del(entry: &mapping->list);
2796	amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2797	}
2798
2799	amdgpu_vm_pt_free_root(adev, vm);
2800	amdgpu_bo_unreserve(bo: root);
2801	amdgpu_bo_unref(bo: &root);
2802	WARN_ON(vm->root.bo);
2803
2804	amdgpu_vm_fini_entities(vm);
2805
2806	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2807	dev_err(adev->dev, "still active bo inside vm\n");
2808	}
2809	rbtree_postorder_for_each_entry_safe(mapping, tmp,
2810	&vm->va.rb_root, rb) {
2811	/* Don't remove the mapping here, we don't want to trigger a
2812	* rebalance and the tree is about to be destroyed anyway.
2813	*/
2814	list_del(entry: &mapping->list);
2815	kfree(objp: mapping);
2816	}
2817
2818	dma_fence_put(fence: vm->last_update);
2819
2820	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) {
2821	amdgpu_vmid_free_reserved(adev, vm, vmhub: i);
2822	}
2823
2824	ttm_lru_bulk_move_fini(bdev: &adev->mman.bdev, bulk: &vm->lru_bulk_move);
2825
2826	if (!amdgpu_vm_stats_is_zero(vm)) {
2827	struct amdgpu_task_info *ti = vm->task_info;
2828
2829	dev_warn(adev->dev,
2830	"VM memory stats for proc %s(%d) task %s(%d) is non-zero when fini\n",
2831	ti->process_name, ti->task.pid, ti->task.comm, ti->tgid);
2832	}
2833
2834	amdgpu_vm_put_task_info(task_info: vm->task_info);
2835	}
2836
2837	/**
2838	* amdgpu_vm_manager_init - init the VM manager
2839	*
2840	* @adev: amdgpu_device pointer
2841	*
2842	* Initialize the VM manager structures
2843	*/
2844	void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2845	{
2846	/* Concurrent flushes are only possible starting with Vega10 and
2847	* are broken on Navi10 and Navi14.
2848	*/
2849	adev->vm_manager.concurrent_flush = !(adev->asic_type < CHIP_VEGA10 ||
2850	adev->asic_type == CHIP_NAVI10 ||
2851	adev->asic_type == CHIP_NAVI14);
2852	amdgpu_vmid_mgr_init(adev);
2853
2854	spin_lock_init(&adev->vm_manager.prt_lock);
2855	atomic_set(v: &adev->vm_manager.num_prt_users, i: 0);
2856
2857	/* If not overridden by the user, by default, only in large BAR systems
2858	* Compute VM tables will be updated by CPU
2859	*/
2860	#ifdef CONFIG_X86_64
2861	if (amdgpu_vm_update_mode == -1) {
2862	/* For asic with VF MMIO access protection
2863	* avoid using CPU for VM table updates
2864	*/
2865	if (amdgpu_gmc_vram_full_visible(gmc: &adev->gmc) &&
2866	!amdgpu_sriov_vf_mmio_access_protection(adev))
2867	adev->vm_manager.vm_update_mode =
2868	AMDGPU_VM_USE_CPU_FOR_COMPUTE;
2869	else
2870	adev->vm_manager.vm_update_mode = 0;
2871	} else
2872	adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
2873	#else
2874	adev->vm_manager.vm_update_mode = 0;
2875	#endif
2876
2877	xa_init_flags(xa: &adev->vm_manager.pasids, XA_FLAGS_LOCK_IRQ);
2878	}
2879
2880	/**
2881	* amdgpu_vm_manager_fini - cleanup VM manager
2882	*
2883	* @adev: amdgpu_device pointer
2884	*
2885	* Cleanup the VM manager and free resources.
2886	*/
2887	void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
2888	{
2889	WARN_ON(!xa_empty(&adev->vm_manager.pasids));
2890	xa_destroy(&adev->vm_manager.pasids);
2891
2892	amdgpu_vmid_mgr_fini(adev);
2893	}
2894
2895	/**
2896	* amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
2897	*
2898	* @dev: drm device pointer
2899	* @data: drm_amdgpu_vm
2900	* @filp: drm file pointer
2901	*
2902	* Returns:
2903	* 0 for success, -errno for errors.
2904	*/
2905	int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
2906	{
2907	union drm_amdgpu_vm *args = data;
2908	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
2909	struct amdgpu_fpriv *fpriv = filp->driver_priv;
2910	struct amdgpu_vm *vm = &fpriv->vm;
2911
2912	/* No valid flags defined yet */
2913	if (args->in.flags)
2914	return -EINVAL;
2915
2916	switch (args->in.op) {
2917	case AMDGPU_VM_OP_RESERVE_VMID:
2918	/* We only have requirement to reserve vmid from gfxhub */
2919	return amdgpu_vmid_alloc_reserved(adev, vm, AMDGPU_GFXHUB(0));
2920	case AMDGPU_VM_OP_UNRESERVE_VMID:
2921	amdgpu_vmid_free_reserved(adev, vm, AMDGPU_GFXHUB(0));
2922	break;
2923	default:
2924	return -EINVAL;
2925	}
2926
2927	return 0;
2928	}
2929
2930	/**
2931	* amdgpu_vm_handle_fault - graceful handling of VM faults.
2932	* @adev: amdgpu device pointer
2933	* @pasid: PASID of the VM
2934	* @ts: Timestamp of the fault
2935	* @vmid: VMID, only used for GFX 9.4.3.
2936	* @node_id: Node_id received in IH cookie. Only applicable for
2937	* GFX 9.4.3.
2938	* @addr: Address of the fault
2939	* @write_fault: true is write fault, false is read fault
2940	*
2941	* Try to gracefully handle a VM fault. Return true if the fault was handled and
2942	* shouldn't be reported any more.
2943	*/
2944	bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
2945	u32 vmid, u32 node_id, uint64_t addr, uint64_t ts,
2946	bool write_fault)
2947	{
2948	bool is_compute_context = false;
2949	struct amdgpu_bo *root;
2950	unsigned long irqflags;
2951	uint64_t value, flags;
2952	struct amdgpu_vm *vm;
2953	int r;
2954
2955	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2956	vm = xa_load(&adev->vm_manager.pasids, index: pasid);
2957	if (vm) {
2958	root = amdgpu_bo_ref(bo: vm->root.bo);
2959	is_compute_context = vm->is_compute_context;
2960	} else {
2961	root = NULL;
2962	}
2963	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2964
2965	if (!root)
2966	return false;
2967
2968	addr /= AMDGPU_GPU_PAGE_SIZE;
2969
2970	if (is_compute_context && !svm_range_restore_pages(adev, pasid, vmid,
2971	node_id, addr, ts, write_fault)) {
2972	amdgpu_bo_unref(bo: &root);
2973	return true;
2974	}
2975
2976	r = amdgpu_bo_reserve(bo: root, no_intr: true);
2977	if (r)
2978	goto error_unref;
2979
2980	/* Double check that the VM still exists */
2981	xa_lock_irqsave(&adev->vm_manager.pasids, irqflags);
2982	vm = xa_load(&adev->vm_manager.pasids, index: pasid);
2983	if (vm && vm->root.bo != root)
2984	vm = NULL;
2985	xa_unlock_irqrestore(&adev->vm_manager.pasids, irqflags);
2986	if (!vm)
2987	goto error_unlock;
2988
2989	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
2990	AMDGPU_PTE_SYSTEM;
2991
2992	if (is_compute_context) {
2993	/* Intentionally setting invalid PTE flag
2994	* combination to force a no-retry-fault
2995	*/
2996	flags = AMDGPU_VM_NORETRY_FLAGS;
2997	value = 0;
2998	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
2999	/* Redirect the access to the dummy page */
3000	value = adev->dummy_page_addr;
3001	flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
3002	AMDGPU_PTE_WRITEABLE;
3003
3004	} else {
3005	/* Let the hw retry silently on the PTE */
3006	value = 0;
3007	}
3008
3009	r = dma_resv_reserve_fences(obj: root->tbo.base.resv, num_fences: 1);
3010	if (r) {
3011	pr_debug("failed %d to reserve fence slot\n", r);
3012	goto error_unlock;
3013	}
3014
3015	r = amdgpu_vm_update_range(adev, vm, immediate: true, unlocked: false, flush_tlb: false, allow_override: false,
3016	NULL, start: addr, last: addr, flags, offset: value, vram_base: 0, NULL, NULL, NULL);
3017	if (r)
3018	goto error_unlock;
3019
3020	r = amdgpu_vm_update_pdes(adev, vm, immediate: true);
3021
3022	error_unlock:
3023	amdgpu_bo_unreserve(bo: root);
3024	if (r < 0)
3025	dev_err(adev->dev, "Can't handle page fault (%d)\n", r);
3026
3027	error_unref:
3028	amdgpu_bo_unref(bo: &root);
3029
3030	return false;
3031	}
3032
3033	#if defined(CONFIG_DEBUG_FS)
3034	/**
3035	* amdgpu_debugfs_vm_bo_info - print BO info for the VM
3036	*
3037	* @vm: Requested VM for printing BO info
3038	* @m: debugfs file
3039	*
3040	* Print BO information in debugfs file for the VM
3041	*/
3042	void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
3043	{
3044	struct amdgpu_bo_va *bo_va, *tmp;
3045	u64 total_idle = 0;
3046	u64 total_evicted = 0;
3047	u64 total_relocated = 0;
3048	u64 total_moved = 0;
3049	u64 total_invalidated = 0;
3050	u64 total_done = 0;
3051	unsigned int total_idle_objs = 0;
3052	unsigned int total_evicted_objs = 0;
3053	unsigned int total_relocated_objs = 0;
3054	unsigned int total_moved_objs = 0;
3055	unsigned int total_invalidated_objs = 0;
3056	unsigned int total_done_objs = 0;
3057	unsigned int id = 0;
3058
3059	amdgpu_vm_assert_locked(vm);
3060
3061	spin_lock(lock: &vm->status_lock);
3062	seq_puts(m, s: "\tIdle BOs:\n");
3063	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
3064	if (!bo_va->base.bo)
3065	continue;
3066	total_idle += amdgpu_bo_print_info(id: id++, bo: bo_va->base.bo, m);
3067	}
3068	total_idle_objs = id;
3069	id = 0;
3070
3071	seq_puts(m, s: "\tEvicted BOs:\n");
3072	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
3073	if (!bo_va->base.bo)
3074	continue;
3075	total_evicted += amdgpu_bo_print_info(id: id++, bo: bo_va->base.bo, m);
3076	}
3077	total_evicted_objs = id;
3078	id = 0;
3079
3080	seq_puts(m, s: "\tRelocated BOs:\n");
3081	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
3082	if (!bo_va->base.bo)
3083	continue;
3084	total_relocated += amdgpu_bo_print_info(id: id++, bo: bo_va->base.bo, m);
3085	}
3086	total_relocated_objs = id;
3087	id = 0;
3088
3089	seq_puts(m, s: "\tMoved BOs:\n");
3090	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
3091	if (!bo_va->base.bo)
3092	continue;
3093	total_moved += amdgpu_bo_print_info(id: id++, bo: bo_va->base.bo, m);
3094	}
3095	total_moved_objs = id;
3096	id = 0;
3097
3098	seq_puts(m, s: "\tInvalidated BOs:\n");
3099	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
3100	if (!bo_va->base.bo)
3101	continue;
3102	total_invalidated += amdgpu_bo_print_info(id: id++, bo: bo_va->base.bo, m);
3103	}
3104	total_invalidated_objs = id;
3105	id = 0;
3106
3107	seq_puts(m, s: "\tDone BOs:\n");
3108	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
3109	if (!bo_va->base.bo)
3110	continue;
3111	total_done += amdgpu_bo_print_info(id: id++, bo: bo_va->base.bo, m);
3112	}
3113	spin_unlock(lock: &vm->status_lock);
3114	total_done_objs = id;
3115
3116	seq_printf(m, fmt: "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle,
3117	total_idle_objs);
3118	seq_printf(m, fmt: "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted,
3119	total_evicted_objs);
3120	seq_printf(m, fmt: "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated,
3121	total_relocated_objs);
3122	seq_printf(m, fmt: "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved,
3123	total_moved_objs);
3124	seq_printf(m, fmt: "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
3125	total_invalidated_objs);
3126	seq_printf(m, fmt: "\tTotal done size: %12lld\tobjs:\t%d\n", total_done,
3127	total_done_objs);
3128	}
3129	#endif
3130
3131	/**
3132	* amdgpu_vm_update_fault_cache - update cached fault into.
3133	* @adev: amdgpu device pointer
3134	* @pasid: PASID of the VM
3135	* @addr: Address of the fault
3136	* @status: GPUVM fault status register
3137	* @vmhub: which vmhub got the fault
3138	*
3139	* Cache the fault info for later use by userspace in debugging.
3140	*/
3141	void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
3142	unsigned int pasid,
3143	uint64_t addr,
3144	uint32_t status,
3145	unsigned int vmhub)
3146	{
3147	struct amdgpu_vm *vm;
3148	unsigned long flags;
3149
3150	xa_lock_irqsave(&adev->vm_manager.pasids, flags);
3151
3152	vm = xa_load(&adev->vm_manager.pasids, index: pasid);
3153	/* Don't update the fault cache if status is 0. In the multiple
3154	* fault case, subsequent faults will return a 0 status which is
3155	* useless for userspace and replaces the useful fault status, so
3156	* only update if status is non-0.
3157	*/
3158	if (vm && status) {
3159	vm->fault_info.addr = addr;
3160	vm->fault_info.status = status;
3161	/*
3162	* Update the fault information globally for later usage
3163	* when vm could be stale or freed.
3164	*/
3165	adev->vm_manager.fault_info.addr = addr;
3166	adev->vm_manager.fault_info.vmhub = vmhub;
3167	adev->vm_manager.fault_info.status = status;
3168
3169	if (AMDGPU_IS_GFXHUB(vmhub)) {
3170	vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_GFX;
3171	vm->fault_info.vmhub |=
3172	(vmhub - AMDGPU_GFXHUB_START) << AMDGPU_VMHUB_IDX_SHIFT;
3173	} else if (AMDGPU_IS_MMHUB0(vmhub)) {
3174	vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM0;
3175	vm->fault_info.vmhub |=
3176	(vmhub - AMDGPU_MMHUB0_START) << AMDGPU_VMHUB_IDX_SHIFT;
3177	} else if (AMDGPU_IS_MMHUB1(vmhub)) {
3178	vm->fault_info.vmhub = AMDGPU_VMHUB_TYPE_MM1;
3179	vm->fault_info.vmhub |=
3180	(vmhub - AMDGPU_MMHUB1_START) << AMDGPU_VMHUB_IDX_SHIFT;
3181	} else {
3182	WARN_ONCE(1, "Invalid vmhub %u\n", vmhub);
3183	}
3184	}
3185	xa_unlock_irqrestore(&adev->vm_manager.pasids, flags);
3186	}
3187
3188	/**
3189	* amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
3190	*
3191	* @vm: VM to test against.
3192	* @bo: BO to be tested.
3193	*
3194	* Returns true if the BO shares the dma_resv object with the root PD and is
3195	* always guaranteed to be valid inside the VM.
3196	*/
3197	bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
3198	{
3199	return bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv;
3200	}
3201
3202	void amdgpu_vm_print_task_info(struct amdgpu_device *adev,
3203	struct amdgpu_task_info *task_info)
3204	{
3205	dev_err(adev->dev,
3206	" Process %s pid %d thread %s pid %d\n",
3207	task_info->process_name, task_info->tgid,
3208	task_info->task.comm, task_info->task.pid);
3209	}
3210