1	/*
2	* Copyright 2009 Jerome Glisse.
3	* All Rights Reserved.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the
7	* "Software"), to deal in the Software without restriction, including
8	* without limitation the rights to use, copy, modify, merge, publish,
9	* distribute, sub license, and/or sell copies of the Software, and to
10	* permit persons to whom the Software is furnished to do so, subject to
11	* the following conditions:
12	*
13	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19	* USE OR OTHER DEALINGS IN THE SOFTWARE.
20	*
21	* The above copyright notice and this permission notice (including the
22	* next paragraph) shall be included in all copies or substantial portions
23	* of the Software.
24	*
25	*/
26	/*
27	* Authors:
28	* Jerome Glisse <glisse@freedesktop.org>
29	* Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30	* Dave Airlie
31	*/
32
33	#include <linux/dma-mapping.h>
34	#include <linux/iommu.h>
35	#include <linux/pagemap.h>
36	#include <linux/sched/task.h>
37	#include <linux/sched/mm.h>
38	#include <linux/seq_file.h>
39	#include <linux/slab.h>
40	#include <linux/swap.h>
41	#include <linux/dma-buf.h>
42	#include <linux/sizes.h>
43	#include <linux/module.h>
44
45	#include <drm/drm_drv.h>
46	#include <drm/ttm/ttm_bo.h>
47	#include <drm/ttm/ttm_placement.h>
48	#include <drm/ttm/ttm_range_manager.h>
49	#include <drm/ttm/ttm_tt.h>
50
51	#include <drm/amdgpu_drm.h>
52
53	#include "amdgpu.h"
54	#include "amdgpu_object.h"
55	#include "amdgpu_trace.h"
56	#include "amdgpu_amdkfd.h"
57	#include "amdgpu_sdma.h"
58	#include "amdgpu_ras.h"
59	#include "amdgpu_hmm.h"
60	#include "amdgpu_atomfirmware.h"
61	#include "amdgpu_res_cursor.h"
62	#include "bif/bif_4_1_d.h"
63
64	MODULE_IMPORT_NS("DMA_BUF");
65
66	#define AMDGPU_TTM_VRAM_MAX_DW_READ ((size_t)128)
67
68	static int amdgpu_ttm_backend_bind(struct ttm_device *bdev,
69	struct ttm_tt *ttm,
70	struct ttm_resource *bo_mem);
71	static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev,
72	struct ttm_tt *ttm);
73
74	static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev,
75	unsigned int type,
76	uint64_t size_in_page)
77	{
78	return ttm_range_man_init(bdev: &adev->mman.bdev, type,
79	use_tt: false, p_size: size_in_page);
80	}
81
82	/**
83	* amdgpu_evict_flags - Compute placement flags
84	*
85	* @bo: The buffer object to evict
86	* @placement: Possible destination(s) for evicted BO
87	*
88	* Fill in placement data when ttm_bo_evict() is called
89	*/
90	static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
91	struct ttm_placement *placement)
92	{
93	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->bdev);
94	struct amdgpu_bo *abo;
95	static const struct ttm_place placements = {
96	.fpfn = 0,
97	.lpfn = 0,
98	.mem_type = TTM_PL_SYSTEM,
99	.flags = 0
100	};
101
102	/* Don't handle scatter gather BOs */
103	if (bo->type == ttm_bo_type_sg) {
104	placement->num_placement = 0;
105	return;
106	}
107
108	/* Object isn't an AMDGPU object so ignore */
109	if (!amdgpu_bo_is_amdgpu_bo(bo)) {
110	placement->placement = &placements;
111	placement->num_placement = 1;
112	return;
113	}
114
115	abo = ttm_to_amdgpu_bo(tbo: bo);
116	if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) {
117	placement->num_placement = 0;
118	return;
119	}
120
121	switch (bo->resource->mem_type) {
122	case AMDGPU_PL_GDS:
123	case AMDGPU_PL_GWS:
124	case AMDGPU_PL_OA:
125	case AMDGPU_PL_DOORBELL:
126	case AMDGPU_PL_MMIO_REMAP:
127	placement->num_placement = 0;
128	return;
129
130	case TTM_PL_VRAM:
131	if (!adev->mman.buffer_funcs_enabled) {
132	/* Move to system memory */
133	amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
134
135	} else if (!amdgpu_gmc_vram_full_visible(gmc: &adev->gmc) &&
136	!(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
137	amdgpu_res_cpu_visible(adev, res: bo->resource)) {
138
139	/* Try evicting to the CPU inaccessible part of VRAM
140	* first, but only set GTT as busy placement, so this
141	* BO will be evicted to GTT rather than causing other
142	* BOs to be evicted from VRAM
143	*/
144	amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
145	AMDGPU_GEM_DOMAIN_GTT |
146	AMDGPU_GEM_DOMAIN_CPU);
147	abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
148	abo->placements[0].lpfn = 0;
149	abo->placements[0].flags |= TTM_PL_FLAG_DESIRED;
150	} else {
151	/* Move to GTT memory */
152	amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT |
153	AMDGPU_GEM_DOMAIN_CPU);
154	}
155	break;
156	case TTM_PL_TT:
157	case AMDGPU_PL_PREEMPT:
158	default:
159	amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
160	break;
161	}
162	*placement = abo->placement;
163	}
164
165	/**
166	* amdgpu_ttm_map_buffer - Map memory into the GART windows
167	* @bo: buffer object to map
168	* @mem: memory object to map
169	* @mm_cur: range to map
170	* @window: which GART window to use
171	* @ring: DMA ring to use for the copy
172	* @tmz: if we should setup a TMZ enabled mapping
173	* @size: in number of bytes to map, out number of bytes mapped
174	* @addr: resulting address inside the MC address space
175	*
176	* Setup one of the GART windows to access a specific piece of memory or return
177	* the physical address for local memory.
178	*/
179	static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo,
180	struct ttm_resource *mem,
181	struct amdgpu_res_cursor *mm_cur,
182	unsigned int window, struct amdgpu_ring *ring,
183	bool tmz, uint64_t *size, uint64_t *addr)
184	{
185	struct amdgpu_device *adev = ring->adev;
186	unsigned int offset, num_pages, num_dw, num_bytes;
187	uint64_t src_addr, dst_addr;
188	struct amdgpu_job *job;
189	void *cpu_addr;
190	uint64_t flags;
191	int r;
192
193	BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
194	AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
195
196	if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT))
197	return -EINVAL;
198
199	/* Map only what can't be accessed directly */
200	if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) {
201	*addr = amdgpu_ttm_domain_start(adev, type: mem->mem_type) +
202	mm_cur->start;
203	return 0;
204	}
205
206
207	/*
208	* If start begins at an offset inside the page, then adjust the size
209	* and addr accordingly
210	*/
211	offset = mm_cur->start & ~PAGE_MASK;
212
213	num_pages = PFN_UP(*size + offset);
214	num_pages = min_t(uint32_t, num_pages, AMDGPU_GTT_MAX_TRANSFER_SIZE);
215
216	*size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset);
217
218	*addr = adev->gmc.gart_start;
219	*addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
220	AMDGPU_GPU_PAGE_SIZE;
221	*addr += offset;
222
223	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
224	num_bytes = num_pages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
225
226	r = amdgpu_job_alloc_with_ib(adev, entity: &adev->mman.high_pr,
227	AMDGPU_FENCE_OWNER_UNDEFINED,
228	size: num_dw * 4 + num_bytes,
229	pool_type: AMDGPU_IB_POOL_DELAYED, job: &job,
230	AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER);
231	if (r)
232	return r;
233
234	src_addr = num_dw * 4;
235	src_addr += job->ibs[0].gpu_addr;
236
237	dst_addr = amdgpu_bo_gpu_offset(bo: adev->gart.bo);
238	dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
239	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
240	dst_addr, num_bytes, 0);
241
242	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
243	WARN_ON(job->ibs[0].length_dw > num_dw);
244
245	flags = amdgpu_ttm_tt_pte_flags(adev, ttm: bo->ttm, mem);
246	if (tmz)
247	flags |= AMDGPU_PTE_TMZ;
248
249	cpu_addr = &job->ibs[0].ptr[num_dw];
250
251	if (mem->mem_type == TTM_PL_TT) {
252	dma_addr_t *dma_addr;
253
254	dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT];
255	amdgpu_gart_map(adev, offset: 0, pages: num_pages, dma_addr, flags, dst: cpu_addr);
256	} else {
257	u64 pa = mm_cur->start + adev->vm_manager.vram_base_offset;
258
259	amdgpu_gart_map_vram_range(adev, pa, start_page: 0, num_pages, flags, dst: cpu_addr);
260	}
261
262	dma_fence_put(fence: amdgpu_job_submit(job));
263	return 0;
264	}
265
266	/**
267	* amdgpu_ttm_copy_mem_to_mem - Helper function for copy
268	* @adev: amdgpu device
269	* @src: buffer/address where to read from
270	* @dst: buffer/address where to write to
271	* @size: number of bytes to copy
272	* @tmz: if a secure copy should be used
273	* @resv: resv object to sync to
274	* @f: Returns the last fence if multiple jobs are submitted.
275	*
276	* The function copies @size bytes from {src->mem + src->offset} to
277	* {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
278	* move and different for a BO to BO copy.
279	*
280	*/
281	__attribute__((nonnull))
282	static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
283	const struct amdgpu_copy_mem *src,
284	const struct amdgpu_copy_mem *dst,
285	uint64_t size, bool tmz,
286	struct dma_resv *resv,
287	struct dma_fence **f)
288	{
289	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
290	struct amdgpu_res_cursor src_mm, dst_mm;
291	struct dma_fence *fence = NULL;
292	int r = 0;
293	uint32_t copy_flags = 0;
294	struct amdgpu_bo *abo_src, *abo_dst;
295
296	if (!adev->mman.buffer_funcs_enabled) {
297	dev_err(adev->dev,
298	"Trying to move memory with ring turned off.\n");
299	return -EINVAL;
300	}
301
302	amdgpu_res_first(res: src->mem, start: src->offset, size, cur: &src_mm);
303	amdgpu_res_first(res: dst->mem, start: dst->offset, size, cur: &dst_mm);
304
305	mutex_lock(&adev->mman.gtt_window_lock);
306	while (src_mm.remaining) {
307	uint64_t from, to, cur_size, tiling_flags;
308	uint32_t num_type, data_format, max_com, write_compress_disable;
309	struct dma_fence *next;
310
311	/* Never copy more than 256MiB at once to avoid a timeout */
312	cur_size = min3(src_mm.size, dst_mm.size, 256ULL << 20);
313
314	/* Map src to window 0 and dst to window 1. */
315	r = amdgpu_ttm_map_buffer(bo: src->bo, mem: src->mem, mm_cur: &src_mm,
316	window: 0, ring, tmz, size: &cur_size, addr: &from);
317	if (r)
318	goto error;
319
320	r = amdgpu_ttm_map_buffer(bo: dst->bo, mem: dst->mem, mm_cur: &dst_mm,
321	window: 1, ring, tmz, size: &cur_size, addr: &to);
322	if (r)
323	goto error;
324
325	abo_src = ttm_to_amdgpu_bo(tbo: src->bo);
326	abo_dst = ttm_to_amdgpu_bo(tbo: dst->bo);
327	if (tmz)
328	copy_flags |= AMDGPU_COPY_FLAGS_TMZ;
329	if ((abo_src->flags & AMDGPU_GEM_CREATE_GFX12_DCC) &&
330	(abo_src->tbo.resource->mem_type == TTM_PL_VRAM))
331	copy_flags |= AMDGPU_COPY_FLAGS_READ_DECOMPRESSED;
332	if ((abo_dst->flags & AMDGPU_GEM_CREATE_GFX12_DCC) &&
333	(dst->mem->mem_type == TTM_PL_VRAM)) {
334	copy_flags |= AMDGPU_COPY_FLAGS_WRITE_COMPRESSED;
335	amdgpu_bo_get_tiling_flags(bo: abo_dst, tiling_flags: &tiling_flags);
336	max_com = AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_MAX_COMPRESSED_BLOCK);
337	num_type = AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_NUMBER_TYPE);
338	data_format = AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_DATA_FORMAT);
339	write_compress_disable =
340	AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_WRITE_COMPRESS_DISABLE);
341	copy_flags |= (AMDGPU_COPY_FLAGS_SET(MAX_COMPRESSED, max_com) |
342	AMDGPU_COPY_FLAGS_SET(NUMBER_TYPE, num_type) |
343	AMDGPU_COPY_FLAGS_SET(DATA_FORMAT, data_format) |
344	AMDGPU_COPY_FLAGS_SET(WRITE_COMPRESS_DISABLE,
345	write_compress_disable));
346	}
347
348	r = amdgpu_copy_buffer(ring, src_offset: from, dst_offset: to, byte_count: cur_size, resv,
349	fence: &next, direct_submit: false, vm_needs_flush: true, copy_flags);
350	if (r)
351	goto error;
352
353	dma_fence_put(fence);
354	fence = next;
355
356	amdgpu_res_next(cur: &src_mm, size: cur_size);
357	amdgpu_res_next(cur: &dst_mm, size: cur_size);
358	}
359	error:
360	mutex_unlock(lock: &adev->mman.gtt_window_lock);
361	*f = fence;
362	return r;
363	}
364
365	/*
366	* amdgpu_move_blit - Copy an entire buffer to another buffer
367	*
368	* This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
369	* help move buffers to and from VRAM.
370	*/
371	static int amdgpu_move_blit(struct ttm_buffer_object *bo,
372	bool evict,
373	struct ttm_resource *new_mem,
374	struct ttm_resource *old_mem)
375	{
376	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->bdev);
377	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo: bo);
378	struct amdgpu_copy_mem src, dst;
379	struct dma_fence *fence = NULL;
380	int r;
381
382	src.bo = bo;
383	dst.bo = bo;
384	src.mem = old_mem;
385	dst.mem = new_mem;
386	src.offset = 0;
387	dst.offset = 0;
388
389	r = amdgpu_ttm_copy_mem_to_mem(adev, src: &src, dst: &dst,
390	size: new_mem->size,
391	tmz: amdgpu_bo_encrypted(bo: abo),
392	resv: bo->base.resv, f: &fence);
393	if (r)
394	goto error;
395
396	/* clear the space being freed */
397	if (old_mem->mem_type == TTM_PL_VRAM &&
398	(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
399	struct dma_fence *wipe_fence = NULL;
400
401	r = amdgpu_fill_buffer(bo: abo, src_data: 0, NULL, fence: &wipe_fence,
402	delayed: false, AMDGPU_KERNEL_JOB_ID_MOVE_BLIT);
403	if (r) {
404	goto error;
405	} else if (wipe_fence) {
406	amdgpu_vram_mgr_set_cleared(res: bo->resource);
407	dma_fence_put(fence);
408	fence = wipe_fence;
409	}
410	}
411
412	/* Always block for VM page tables before committing the new location */
413	if (bo->type == ttm_bo_type_kernel)
414	r = ttm_bo_move_accel_cleanup(bo, fence, evict: true, pipeline: false, new_mem);
415	else
416	r = ttm_bo_move_accel_cleanup(bo, fence, evict, pipeline: true, new_mem);
417	dma_fence_put(fence);
418	return r;
419
420	error:
421	if (fence)
422	dma_fence_wait(fence, intr: false);
423	dma_fence_put(fence);
424	return r;
425	}
426
427	/**
428	* amdgpu_res_cpu_visible - Check that resource can be accessed by CPU
429	* @adev: amdgpu device
430	* @res: the resource to check
431	*
432	* Returns: true if the full resource is CPU visible, false otherwise.
433	*/
434	bool amdgpu_res_cpu_visible(struct amdgpu_device *adev,
435	struct ttm_resource *res)
436	{
437	struct amdgpu_res_cursor cursor;
438
439	if (!res)
440	return false;
441
442	if (res->mem_type == TTM_PL_SYSTEM || res->mem_type == TTM_PL_TT ||
443	res->mem_type == AMDGPU_PL_PREEMPT || res->mem_type == AMDGPU_PL_DOORBELL ||
444	res->mem_type == AMDGPU_PL_MMIO_REMAP)
445	return true;
446
447	if (res->mem_type != TTM_PL_VRAM)
448	return false;
449
450	amdgpu_res_first(res, start: 0, size: res->size, cur: &cursor);
451	while (cursor.remaining) {
452	if ((cursor.start + cursor.size) > adev->gmc.visible_vram_size)
453	return false;
454	amdgpu_res_next(cur: &cursor, size: cursor.size);
455	}
456
457	return true;
458	}
459
460	/*
461	* amdgpu_res_copyable - Check that memory can be accessed by ttm_bo_move_memcpy
462	*
463	* Called by amdgpu_bo_move()
464	*/
465	static bool amdgpu_res_copyable(struct amdgpu_device *adev,
466	struct ttm_resource *mem)
467	{
468	if (!amdgpu_res_cpu_visible(adev, res: mem))
469	return false;
470
471	/* ttm_resource_ioremap only supports contiguous memory */
472	if (mem->mem_type == TTM_PL_VRAM &&
473	!(mem->placement & TTM_PL_FLAG_CONTIGUOUS))
474	return false;
475
476	return true;
477	}
478
479	/*
480	* amdgpu_bo_move - Move a buffer object to a new memory location
481	*
482	* Called by ttm_bo_handle_move_mem()
483	*/
484	static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
485	struct ttm_operation_ctx *ctx,
486	struct ttm_resource *new_mem,
487	struct ttm_place *hop)
488	{
489	struct amdgpu_device *adev;
490	struct amdgpu_bo *abo;
491	struct ttm_resource *old_mem = bo->resource;
492	int r;
493
494	if (new_mem->mem_type == TTM_PL_TT ||
495	new_mem->mem_type == AMDGPU_PL_PREEMPT) {
496	r = amdgpu_ttm_backend_bind(bdev: bo->bdev, ttm: bo->ttm, bo_mem: new_mem);
497	if (r)
498	return r;
499	}
500
501	abo = ttm_to_amdgpu_bo(tbo: bo);
502	adev = amdgpu_ttm_adev(bdev: bo->bdev);
503
504	if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM &&
505	bo->ttm == NULL)) {
506	amdgpu_bo_move_notify(bo, evict, new_mem);
507	ttm_bo_move_null(bo, new_mem);
508	return 0;
509	}
510	if (old_mem->mem_type == TTM_PL_SYSTEM &&
511	(new_mem->mem_type == TTM_PL_TT ||
512	new_mem->mem_type == AMDGPU_PL_PREEMPT)) {
513	amdgpu_bo_move_notify(bo, evict, new_mem);
514	ttm_bo_move_null(bo, new_mem);
515	return 0;
516	}
517	if ((old_mem->mem_type == TTM_PL_TT ||
518	old_mem->mem_type == AMDGPU_PL_PREEMPT) &&
519	new_mem->mem_type == TTM_PL_SYSTEM) {
520	r = ttm_bo_wait_ctx(bo, ctx);
521	if (r)
522	return r;
523
524	amdgpu_ttm_backend_unbind(bdev: bo->bdev, ttm: bo->ttm);
525	amdgpu_bo_move_notify(bo, evict, new_mem);
526	ttm_resource_free(bo, res: &bo->resource);
527	ttm_bo_assign_mem(bo, new_mem);
528	return 0;
529	}
530
531	if (old_mem->mem_type == AMDGPU_PL_GDS ||
532	old_mem->mem_type == AMDGPU_PL_GWS ||
533	old_mem->mem_type == AMDGPU_PL_OA ||
534	old_mem->mem_type == AMDGPU_PL_DOORBELL ||
535	old_mem->mem_type == AMDGPU_PL_MMIO_REMAP ||
536	new_mem->mem_type == AMDGPU_PL_GDS ||
537	new_mem->mem_type == AMDGPU_PL_GWS ||
538	new_mem->mem_type == AMDGPU_PL_OA ||
539	new_mem->mem_type == AMDGPU_PL_DOORBELL ||
540	new_mem->mem_type == AMDGPU_PL_MMIO_REMAP) {
541	/* Nothing to save here */
542	amdgpu_bo_move_notify(bo, evict, new_mem);
543	ttm_bo_move_null(bo, new_mem);
544	return 0;
545	}
546
547	if (bo->type == ttm_bo_type_device &&
548	new_mem->mem_type == TTM_PL_VRAM &&
549	old_mem->mem_type != TTM_PL_VRAM) {
550	/* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
551	* accesses the BO after it's moved.
552	*/
553	abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
554	}
555
556	if (adev->mman.buffer_funcs_enabled &&
557	((old_mem->mem_type == TTM_PL_SYSTEM &&
558	new_mem->mem_type == TTM_PL_VRAM) ||
559	(old_mem->mem_type == TTM_PL_VRAM &&
560	new_mem->mem_type == TTM_PL_SYSTEM))) {
561	hop->fpfn = 0;
562	hop->lpfn = 0;
563	hop->mem_type = TTM_PL_TT;
564	hop->flags = TTM_PL_FLAG_TEMPORARY;
565	return -EMULTIHOP;
566	}
567
568	amdgpu_bo_move_notify(bo, evict, new_mem);
569	if (adev->mman.buffer_funcs_enabled)
570	r = amdgpu_move_blit(bo, evict, new_mem, old_mem);
571	else
572	r = -ENODEV;
573
574	if (r) {
575	/* Check that all memory is CPU accessible */
576	if (!amdgpu_res_copyable(adev, mem: old_mem) ||
577	!amdgpu_res_copyable(adev, mem: new_mem)) {
578	pr_err("Move buffer fallback to memcpy unavailable\n");
579	return r;
580	}
581
582	r = ttm_bo_move_memcpy(bo, ctx, new_mem);
583	if (r)
584	return r;
585	}
586
587	/* update statistics after the move */
588	if (evict)
589	atomic64_inc(v: &adev->num_evictions);
590	atomic64_add(i: bo->base.size, v: &adev->num_bytes_moved);
591	return 0;
592	}
593
594	/*
595	* amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
596	*
597	* Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
598	*/
599	static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev,
600	struct ttm_resource *mem)
601	{
602	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
603
604	switch (mem->mem_type) {
605	case TTM_PL_SYSTEM:
606	/* system memory */
607	return 0;
608	case TTM_PL_TT:
609	case AMDGPU_PL_PREEMPT:
610	break;
611	case TTM_PL_VRAM:
612	mem->bus.offset = mem->start << PAGE_SHIFT;
613
614	if (adev->mman.aper_base_kaddr &&
615	mem->placement & TTM_PL_FLAG_CONTIGUOUS)
616	mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
617	mem->bus.offset;
618
619	mem->bus.offset += adev->gmc.aper_base;
620	mem->bus.is_iomem = true;
621	break;
622	case AMDGPU_PL_DOORBELL:
623	mem->bus.offset = mem->start << PAGE_SHIFT;
624	mem->bus.offset += adev->doorbell.base;
625	mem->bus.is_iomem = true;
626	mem->bus.caching = ttm_uncached;
627	break;
628	case AMDGPU_PL_MMIO_REMAP:
629	mem->bus.offset = mem->start << PAGE_SHIFT;
630	mem->bus.offset += adev->rmmio_remap.bus_addr;
631	mem->bus.is_iomem = true;
632	mem->bus.caching = ttm_uncached;
633	break;
634	default:
635	return -EINVAL;
636	}
637	return 0;
638	}
639
640	static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
641	unsigned long page_offset)
642	{
643	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->bdev);
644	struct amdgpu_res_cursor cursor;
645
646	amdgpu_res_first(res: bo->resource, start: (u64)page_offset << PAGE_SHIFT, size: 0,
647	cur: &cursor);
648
649	if (bo->resource->mem_type == AMDGPU_PL_DOORBELL)
650	return ((uint64_t)(adev->doorbell.base + cursor.start)) >> PAGE_SHIFT;
651	else if (bo->resource->mem_type == AMDGPU_PL_MMIO_REMAP)
652	return ((uint64_t)(adev->rmmio_remap.bus_addr + cursor.start)) >> PAGE_SHIFT;
653
654	return (adev->gmc.aper_base + cursor.start) >> PAGE_SHIFT;
655	}
656
657	/**
658	* amdgpu_ttm_domain_start - Returns GPU start address
659	* @adev: amdgpu device object
660	* @type: type of the memory
661	*
662	* Returns:
663	* GPU start address of a memory domain
664	*/
665
666	uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type)
667	{
668	switch (type) {
669	case TTM_PL_TT:
670	return adev->gmc.gart_start;
671	case TTM_PL_VRAM:
672	return adev->gmc.vram_start;
673	}
674
675	return 0;
676	}
677
678	/*
679	* TTM backend functions.
680	*/
681	struct amdgpu_ttm_tt {
682	struct ttm_tt ttm;
683	struct drm_gem_object *gobj;
684	u64 offset;
685	uint64_t userptr;
686	struct task_struct *usertask;
687	uint32_t userflags;
688	bool bound;
689	int32_t pool_id;
690	};
691
692	#define ttm_to_amdgpu_ttm_tt(ptr) container_of(ptr, struct amdgpu_ttm_tt, ttm)
693
694	#ifdef CONFIG_DRM_AMDGPU_USERPTR
695	/*
696	* amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
697	* memory and start HMM tracking CPU page table update
698	*
699	* Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
700	* once afterwards to stop HMM tracking. Its the caller responsibility to ensure
701	* that range is a valid memory and it is freed too.
702	*/
703	int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
704	struct amdgpu_hmm_range *range)
705	{
706	struct ttm_tt *ttm = bo->tbo.ttm;
707	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
708	unsigned long start = gtt->userptr;
709	struct vm_area_struct *vma;
710	struct mm_struct *mm;
711	bool readonly;
712	int r = 0;
713
714	mm = bo->notifier.mm;
715	if (unlikely(!mm)) {
716	DRM_DEBUG_DRIVER("BO is not registered?\n");
717	return -EFAULT;
718	}
719
720	if (!mmget_not_zero(mm)) /* Happens during process shutdown */
721	return -ESRCH;
722
723	mmap_read_lock(mm);
724	vma = vma_lookup(mm, addr: start);
725	if (unlikely(!vma)) {
726	r = -EFAULT;
727	goto out_unlock;
728	}
729	if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
730	vma->vm_file)) {
731	r = -EPERM;
732	goto out_unlock;
733	}
734
735	readonly = amdgpu_ttm_tt_is_readonly(ttm);
736	r = amdgpu_hmm_range_get_pages(notifier: &bo->notifier, start, npages: ttm->num_pages,
737	readonly, NULL, range);
738	out_unlock:
739	mmap_read_unlock(mm);
740	if (r)
741	pr_debug("failed %d to get user pages 0x%lx\n", r, start);
742
743	mmput(mm);
744
745	return r;
746	}
747
748	#endif
749
750	/*
751	* amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
752	*
753	* Called by amdgpu_cs_list_validate(). This creates the page list
754	* that backs user memory and will ultimately be mapped into the device
755	* address space.
756	*/
757	void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct amdgpu_hmm_range *range)
758	{
759	unsigned long i;
760
761	for (i = 0; i < ttm->num_pages; ++i)
762	ttm->pages[i] = range ? hmm_pfn_to_page(hmm_pfn: range->hmm_range.hmm_pfns[i]) : NULL;
763	}
764
765	/*
766	* amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
767	*
768	* Called by amdgpu_ttm_backend_bind()
769	**/
770	static int amdgpu_ttm_tt_pin_userptr(struct ttm_device *bdev,
771	struct ttm_tt *ttm)
772	{
773	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
774	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
775	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
776	enum dma_data_direction direction = write ?
777	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
778	int r;
779
780	/* Allocate an SG array and squash pages into it */
781	r = sg_alloc_table_from_pages(sgt: ttm->sg, pages: ttm->pages, n_pages: ttm->num_pages, offset: 0,
782	size: (u64)ttm->num_pages << PAGE_SHIFT,
783	GFP_KERNEL);
784	if (r)
785	goto release_sg;
786
787	/* Map SG to device */
788	r = dma_map_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
789	if (r)
790	goto release_sg_table;
791
792	/* convert SG to linear array of pages and dma addresses */
793	drm_prime_sg_to_dma_addr_array(sgt: ttm->sg, addrs: gtt->ttm.dma_address,
794	max_pages: ttm->num_pages);
795
796	return 0;
797
798	release_sg_table:
799	sg_free_table(ttm->sg);
800	release_sg:
801	kfree(objp: ttm->sg);
802	ttm->sg = NULL;
803	return r;
804	}
805
806	/*
807	* amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
808	*/
809	static void amdgpu_ttm_tt_unpin_userptr(struct ttm_device *bdev,
810	struct ttm_tt *ttm)
811	{
812	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
813	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
814	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
815	enum dma_data_direction direction = write ?
816	DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
817
818	/* double check that we don't free the table twice */
819	if (!ttm->sg || !ttm->sg->sgl)
820	return;
821
822	/* unmap the pages mapped to the device */
823	dma_unmap_sgtable(dev: adev->dev, sgt: ttm->sg, dir: direction, attrs: 0);
824	sg_free_table(ttm->sg);
825	}
826
827	/*
828	* total_pages is constructed as MQD0+CtrlStack0 + MQD1+CtrlStack1 + ...
829	* MQDn+CtrlStackn where n is the number of XCCs per partition.
830	* pages_per_xcc is the size of one MQD+CtrlStack. The first page is MQD
831	* and uses memory type default, UC. The rest of pages_per_xcc are
832	* Ctrl stack and modify their memory type to NC.
833	*/
834	static void amdgpu_ttm_gart_bind_gfx9_mqd(struct amdgpu_device *adev,
835	struct ttm_tt *ttm, uint64_t flags)
836	{
837	struct amdgpu_ttm_tt *gtt = (void *)ttm;
838	uint64_t total_pages = ttm->num_pages;
839	int num_xcc = max(1U, adev->gfx.num_xcc_per_xcp);
840	uint64_t page_idx, pages_per_xcc;
841	int i;
842	uint64_t ctrl_flags = AMDGPU_PTE_MTYPE_VG10(flags, AMDGPU_MTYPE_NC);
843
844	pages_per_xcc = total_pages;
845	do_div(pages_per_xcc, num_xcc);
846
847	for (i = 0, page_idx = 0; i < num_xcc; i++, page_idx += pages_per_xcc) {
848	/* MQD page: use default flags */
849	amdgpu_gart_bind(adev,
850	offset: gtt->offset + (page_idx << PAGE_SHIFT),
851	pages: 1, dma_addr: &gtt->ttm.dma_address[page_idx], flags);
852	/*
853	* Ctrl pages - modify the memory type to NC (ctrl_flags) from
854	* the second page of the BO onward.
855	*/
856	amdgpu_gart_bind(adev,
857	offset: gtt->offset + ((page_idx + 1) << PAGE_SHIFT),
858	pages: pages_per_xcc - 1,
859	dma_addr: &gtt->ttm.dma_address[page_idx + 1],
860	flags: ctrl_flags);
861	}
862	}
863
864	static void amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
865	struct ttm_buffer_object *tbo,
866	uint64_t flags)
867	{
868	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
869	struct ttm_tt *ttm = tbo->ttm;
870	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
871
872	if (amdgpu_bo_encrypted(bo: abo))
873	flags |= AMDGPU_PTE_TMZ;
874
875	if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) {
876	amdgpu_ttm_gart_bind_gfx9_mqd(adev, ttm, flags);
877	} else {
878	amdgpu_gart_bind(adev, offset: gtt->offset, pages: ttm->num_pages,
879	dma_addr: gtt->ttm.dma_address, flags);
880	}
881	gtt->bound = true;
882	}
883
884	/*
885	* amdgpu_ttm_backend_bind - Bind GTT memory
886	*
887	* Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
888	* This handles binding GTT memory to the device address space.
889	*/
890	static int amdgpu_ttm_backend_bind(struct ttm_device *bdev,
891	struct ttm_tt *ttm,
892	struct ttm_resource *bo_mem)
893	{
894	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
895	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
896	uint64_t flags;
897	int r;
898
899	if (!bo_mem)
900	return -EINVAL;
901
902	if (gtt->bound)
903	return 0;
904
905	if (gtt->userptr) {
906	r = amdgpu_ttm_tt_pin_userptr(bdev, ttm);
907	if (r) {
908	dev_err(adev->dev, "failed to pin userptr\n");
909	return r;
910	}
911	} else if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) {
912	if (!ttm->sg) {
913	struct dma_buf_attachment *attach;
914	struct sg_table *sgt;
915
916	attach = gtt->gobj->import_attach;
917	sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
918	if (IS_ERR(ptr: sgt))
919	return PTR_ERR(ptr: sgt);
920
921	ttm->sg = sgt;
922	}
923
924	drm_prime_sg_to_dma_addr_array(sgt: ttm->sg, addrs: gtt->ttm.dma_address,
925	max_pages: ttm->num_pages);
926	}
927
928	if (!ttm->num_pages) {
929	WARN(1, "nothing to bind %u pages for mreg %p back %p!\n",
930	ttm->num_pages, bo_mem, ttm);
931	}
932
933	if (bo_mem->mem_type != TTM_PL_TT ||
934	!amdgpu_gtt_mgr_has_gart_addr(mem: bo_mem)) {
935	gtt->offset = AMDGPU_BO_INVALID_OFFSET;
936	return 0;
937	}
938
939	/* compute PTE flags relevant to this BO memory */
940	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem: bo_mem);
941
942	/* bind pages into GART page tables */
943	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
944	amdgpu_gart_bind(adev, offset: gtt->offset, pages: ttm->num_pages,
945	dma_addr: gtt->ttm.dma_address, flags);
946	gtt->bound = true;
947	return 0;
948	}
949
950	/*
951	* amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either
952	* through AGP or GART aperture.
953	*
954	* If bo is accessible through AGP aperture, then use AGP aperture
955	* to access bo; otherwise allocate logical space in GART aperture
956	* and map bo to GART aperture.
957	*/
958	int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
959	{
960	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->bdev);
961	struct ttm_operation_ctx ctx = { false, false };
962	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(bo->ttm);
963	struct ttm_placement placement;
964	struct ttm_place placements;
965	struct ttm_resource *tmp;
966	uint64_t addr, flags;
967	int r;
968
969	if (bo->resource->start != AMDGPU_BO_INVALID_OFFSET)
970	return 0;
971
972	addr = amdgpu_gmc_agp_addr(bo);
973	if (addr != AMDGPU_BO_INVALID_OFFSET)
974	return 0;
975
976	/* allocate GART space */
977	placement.num_placement = 1;
978	placement.placement = &placements;
979	placements.fpfn = 0;
980	placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
981	placements.mem_type = TTM_PL_TT;
982	placements.flags = bo->resource->placement;
983
984	r = ttm_bo_mem_space(bo, placement: &placement, mem: &tmp, ctx: &ctx);
985	if (unlikely(r))
986	return r;
987
988	/* compute PTE flags for this buffer object */
989	flags = amdgpu_ttm_tt_pte_flags(adev, ttm: bo->ttm, mem: tmp);
990
991	/* Bind pages */
992	gtt->offset = (u64)tmp->start << PAGE_SHIFT;
993	amdgpu_ttm_gart_bind(adev, tbo: bo, flags);
994	amdgpu_gart_invalidate_tlb(adev);
995	ttm_resource_free(bo, res: &bo->resource);
996	ttm_bo_assign_mem(bo, new_mem: tmp);
997
998	return 0;
999	}
1000
1001	/*
1002	* amdgpu_ttm_recover_gart - Rebind GTT pages
1003	*
1004	* Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
1005	* rebind GTT pages during a GPU reset.
1006	*/
1007	void amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1008	{
1009	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: tbo->bdev);
1010	uint64_t flags;
1011
1012	if (!tbo->ttm)
1013	return;
1014
1015	flags = amdgpu_ttm_tt_pte_flags(adev, ttm: tbo->ttm, mem: tbo->resource);
1016	amdgpu_ttm_gart_bind(adev, tbo, flags);
1017	}
1018
1019	/*
1020	* amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
1021	*
1022	* Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
1023	* ttm_tt_destroy().
1024	*/
1025	static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev,
1026	struct ttm_tt *ttm)
1027	{
1028	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
1029	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1030
1031	/* if the pages have userptr pinning then clear that first */
1032	if (gtt->userptr) {
1033	amdgpu_ttm_tt_unpin_userptr(bdev, ttm);
1034	} else if (ttm->sg && drm_gem_is_imported(obj: gtt->gobj)) {
1035	struct dma_buf_attachment *attach;
1036
1037	attach = gtt->gobj->import_attach;
1038	dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL);
1039	ttm->sg = NULL;
1040	}
1041
1042	if (!gtt->bound)
1043	return;
1044
1045	if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1046	return;
1047
1048	/* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
1049	amdgpu_gart_unbind(adev, offset: gtt->offset, pages: ttm->num_pages);
1050	gtt->bound = false;
1051	}
1052
1053	static void amdgpu_ttm_backend_destroy(struct ttm_device *bdev,
1054	struct ttm_tt *ttm)
1055	{
1056	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1057
1058	if (gtt->usertask)
1059	put_task_struct(t: gtt->usertask);
1060
1061	ttm_tt_fini(ttm: &gtt->ttm);
1062	kfree(objp: gtt);
1063	}
1064
1065	/**
1066	* amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
1067	*
1068	* @bo: The buffer object to create a GTT ttm_tt object around
1069	* @page_flags: Page flags to be added to the ttm_tt object
1070	*
1071	* Called by ttm_tt_create().
1072	*/
1073	static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
1074	uint32_t page_flags)
1075	{
1076	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->bdev);
1077	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo: bo);
1078	struct amdgpu_ttm_tt *gtt;
1079	enum ttm_caching caching;
1080
1081	gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
1082	if (!gtt)
1083	return NULL;
1084
1085	gtt->gobj = &bo->base;
1086	if (adev->gmc.mem_partitions && abo->xcp_id >= 0)
1087	gtt->pool_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
1088	else
1089	gtt->pool_id = abo->xcp_id;
1090
1091	if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
1092	caching = ttm_write_combined;
1093	else
1094	caching = ttm_cached;
1095
1096	/* allocate space for the uninitialized page entries */
1097	if (ttm_sg_tt_init(ttm_dma: &gtt->ttm, bo, page_flags, caching)) {
1098	kfree(objp: gtt);
1099	return NULL;
1100	}
1101	return &gtt->ttm;
1102	}
1103
1104	/*
1105	* amdgpu_ttm_tt_populate - Map GTT pages visible to the device
1106	*
1107	* Map the pages of a ttm_tt object to an address space visible
1108	* to the underlying device.
1109	*/
1110	static int amdgpu_ttm_tt_populate(struct ttm_device *bdev,
1111	struct ttm_tt *ttm,
1112	struct ttm_operation_ctx *ctx)
1113	{
1114	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
1115	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1116	struct ttm_pool *pool;
1117	pgoff_t i;
1118	int ret;
1119
1120	/* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
1121	if (gtt->userptr) {
1122	ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1123	if (!ttm->sg)
1124	return -ENOMEM;
1125	return 0;
1126	}
1127
1128	if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL)
1129	return 0;
1130
1131	if (adev->mman.ttm_pools && gtt->pool_id >= 0)
1132	pool = &adev->mman.ttm_pools[gtt->pool_id];
1133	else
1134	pool = &adev->mman.bdev.pool;
1135	ret = ttm_pool_alloc(pool, tt: ttm, ctx);
1136	if (ret)
1137	return ret;
1138
1139	for (i = 0; i < ttm->num_pages; ++i)
1140	ttm->pages[i]->mapping = bdev->dev_mapping;
1141
1142	return 0;
1143	}
1144
1145	/*
1146	* amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
1147	*
1148	* Unmaps pages of a ttm_tt object from the device address space and
1149	* unpopulates the page array backing it.
1150	*/
1151	static void amdgpu_ttm_tt_unpopulate(struct ttm_device *bdev,
1152	struct ttm_tt *ttm)
1153	{
1154	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1155	struct amdgpu_device *adev;
1156	struct ttm_pool *pool;
1157	pgoff_t i;
1158
1159	amdgpu_ttm_backend_unbind(bdev, ttm);
1160
1161	if (gtt->userptr) {
1162	amdgpu_ttm_tt_set_user_pages(ttm, NULL);
1163	kfree(objp: ttm->sg);
1164	ttm->sg = NULL;
1165	return;
1166	}
1167
1168	if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL)
1169	return;
1170
1171	for (i = 0; i < ttm->num_pages; ++i)
1172	ttm->pages[i]->mapping = NULL;
1173
1174	adev = amdgpu_ttm_adev(bdev);
1175
1176	if (adev->mman.ttm_pools && gtt->pool_id >= 0)
1177	pool = &adev->mman.ttm_pools[gtt->pool_id];
1178	else
1179	pool = &adev->mman.bdev.pool;
1180
1181	return ttm_pool_free(pool, tt: ttm);
1182	}
1183
1184	/**
1185	* amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
1186	* task
1187	*
1188	* @tbo: The ttm_buffer_object that contains the userptr
1189	* @user_addr: The returned value
1190	*/
1191	int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo,
1192	uint64_t *user_addr)
1193	{
1194	struct amdgpu_ttm_tt *gtt;
1195
1196	if (!tbo->ttm)
1197	return -EINVAL;
1198
1199	gtt = (void *)tbo->ttm;
1200	*user_addr = gtt->userptr;
1201	return 0;
1202	}
1203
1204	/**
1205	* amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
1206	* task
1207	*
1208	* @bo: The ttm_buffer_object to bind this userptr to
1209	* @addr: The address in the current tasks VM space to use
1210	* @flags: Requirements of userptr object.
1211	*
1212	* Called by amdgpu_gem_userptr_ioctl() and kfd_ioctl_alloc_memory_of_gpu() to
1213	* bind userptr pages to current task and by kfd_ioctl_acquire_vm() to
1214	* initialize GPU VM for a KFD process.
1215	*/
1216	int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo,
1217	uint64_t addr, uint32_t flags)
1218	{
1219	struct amdgpu_ttm_tt *gtt;
1220
1221	if (!bo->ttm) {
1222	/* TODO: We want a separate TTM object type for userptrs */
1223	bo->ttm = amdgpu_ttm_tt_create(bo, page_flags: 0);
1224	if (bo->ttm == NULL)
1225	return -ENOMEM;
1226	}
1227
1228	/* Set TTM_TT_FLAG_EXTERNAL before populate but after create. */
1229	bo->ttm->page_flags |= TTM_TT_FLAG_EXTERNAL;
1230
1231	gtt = ttm_to_amdgpu_ttm_tt(bo->ttm);
1232	gtt->userptr = addr;
1233	gtt->userflags = flags;
1234
1235	if (gtt->usertask)
1236	put_task_struct(t: gtt->usertask);
1237	gtt->usertask = current->group_leader;
1238	get_task_struct(t: gtt->usertask);
1239
1240	return 0;
1241	}
1242
1243	/*
1244	* amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
1245	*/
1246	struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
1247	{
1248	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1249
1250	if (gtt == NULL)
1251	return NULL;
1252
1253	if (gtt->usertask == NULL)
1254	return NULL;
1255
1256	return gtt->usertask->mm;
1257	}
1258
1259	/*
1260	* amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
1261	* address range for the current task.
1262	*
1263	*/
1264	bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1265	unsigned long end, unsigned long *userptr)
1266	{
1267	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1268	unsigned long size;
1269
1270	if (gtt == NULL || !gtt->userptr)
1271	return false;
1272
1273	/* Return false if no part of the ttm_tt object lies within
1274	* the range
1275	*/
1276	size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE;
1277	if (gtt->userptr > end || gtt->userptr + size <= start)
1278	return false;
1279
1280	if (userptr)
1281	*userptr = gtt->userptr;
1282	return true;
1283	}
1284
1285	/*
1286	* amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
1287	*/
1288	bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
1289	{
1290	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1291
1292	if (gtt == NULL || !gtt->userptr)
1293	return false;
1294
1295	return true;
1296	}
1297
1298	/*
1299	* amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
1300	*/
1301	bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
1302	{
1303	struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
1304
1305	if (gtt == NULL)
1306	return false;
1307
1308	return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1309	}
1310
1311	/**
1312	* amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
1313	*
1314	* @ttm: The ttm_tt object to compute the flags for
1315	* @mem: The memory registry backing this ttm_tt object
1316	*
1317	* Figure out the flags to use for a VM PDE (Page Directory Entry).
1318	*/
1319	uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem)
1320	{
1321	uint64_t flags = 0;
1322
1323	if (mem && mem->mem_type != TTM_PL_SYSTEM)
1324	flags |= AMDGPU_PTE_VALID;
1325
1326	if (mem && (mem->mem_type == TTM_PL_TT ||
1327	mem->mem_type == AMDGPU_PL_DOORBELL ||
1328	mem->mem_type == AMDGPU_PL_PREEMPT ||
1329	mem->mem_type == AMDGPU_PL_MMIO_REMAP)) {
1330	flags |= AMDGPU_PTE_SYSTEM;
1331
1332	if (ttm && ttm->caching == ttm_cached)
1333	flags |= AMDGPU_PTE_SNOOPED;
1334	}
1335
1336	if (mem && mem->mem_type == TTM_PL_VRAM &&
1337	mem->bus.caching == ttm_cached)
1338	flags |= AMDGPU_PTE_SNOOPED;
1339
1340	return flags;
1341	}
1342
1343	/**
1344	* amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
1345	*
1346	* @adev: amdgpu_device pointer
1347	* @ttm: The ttm_tt object to compute the flags for
1348	* @mem: The memory registry backing this ttm_tt object
1349	*
1350	* Figure out the flags to use for a VM PTE (Page Table Entry).
1351	*/
1352	uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1353	struct ttm_resource *mem)
1354	{
1355	uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
1356
1357	flags |= adev->gart.gart_pte_flags;
1358	flags |= AMDGPU_PTE_READABLE;
1359
1360	if (!amdgpu_ttm_tt_is_readonly(ttm))
1361	flags |= AMDGPU_PTE_WRITEABLE;
1362
1363	return flags;
1364	}
1365
1366	/*
1367	* amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
1368	* object.
1369	*
1370	* Return true if eviction is sensible. Called by ttm_mem_evict_first() on
1371	* behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
1372	* it can find space for a new object and by ttm_bo_force_list_clean() which is
1373	* used to clean out a memory space.
1374	*/
1375	static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1376	const struct ttm_place *place)
1377	{
1378	struct dma_resv_iter resv_cursor;
1379	struct dma_fence *f;
1380
1381	if (!amdgpu_bo_is_amdgpu_bo(bo))
1382	return ttm_bo_eviction_valuable(bo, place);
1383
1384	/* Swapout? */
1385	if (bo->resource->mem_type == TTM_PL_SYSTEM)
1386	return true;
1387
1388	if (bo->type == ttm_bo_type_kernel &&
1389	!amdgpu_vm_evictable(bo: ttm_to_amdgpu_bo(tbo: bo)))
1390	return false;
1391
1392	/* If bo is a KFD BO, check if the bo belongs to the current process.
1393	* If true, then return false as any KFD process needs all its BOs to
1394	* be resident to run successfully
1395	*/
1396	dma_resv_for_each_fence(&resv_cursor, bo->base.resv,
1397	DMA_RESV_USAGE_BOOKKEEP, f) {
1398	if (amdkfd_fence_check_mm(f, current->mm) &&
1399	!(place->flags & TTM_PL_FLAG_CONTIGUOUS))
1400	return false;
1401	}
1402
1403	/* Preemptible BOs don't own system resources managed by the
1404	* driver (pages, VRAM, GART space). They point to resources
1405	* owned by someone else (e.g. pageable memory in user mode
1406	* or a DMABuf). They are used in a preemptible context so we
1407	* can guarantee no deadlocks and good QoS in case of MMU
1408	* notifiers or DMABuf move notifiers from the resource owner.
1409	*/
1410	if (bo->resource->mem_type == AMDGPU_PL_PREEMPT)
1411	return false;
1412
1413	if (bo->resource->mem_type == TTM_PL_TT &&
1414	amdgpu_bo_encrypted(bo: ttm_to_amdgpu_bo(tbo: bo)))
1415	return false;
1416
1417	return ttm_bo_eviction_valuable(bo, place);
1418	}
1419
1420	static void amdgpu_ttm_vram_mm_access(struct amdgpu_device *adev, loff_t pos,
1421	void *buf, size_t size, bool write)
1422	{
1423	while (size) {
1424	uint64_t aligned_pos = ALIGN_DOWN(pos, 4);
1425	uint64_t bytes = 4 - (pos & 0x3);
1426	uint32_t shift = (pos & 0x3) * 8;
1427	uint32_t mask = 0xffffffff << shift;
1428	uint32_t value = 0;
1429
1430	if (size < bytes) {
1431	mask &= 0xffffffff >> (bytes - size) * 8;
1432	bytes = size;
1433	}
1434
1435	if (mask != 0xffffffff) {
1436	amdgpu_device_mm_access(adev, pos: aligned_pos, buf: &value, size: 4, write: false);
1437	if (write) {
1438	value &= ~mask;
1439	value |= (*(uint32_t *)buf << shift) & mask;
1440	amdgpu_device_mm_access(adev, pos: aligned_pos, buf: &value, size: 4, write: true);
1441	} else {
1442	value = (value & mask) >> shift;
1443	memcpy(buf, &value, bytes);
1444	}
1445	} else {
1446	amdgpu_device_mm_access(adev, pos: aligned_pos, buf, size: 4, write);
1447	}
1448
1449	pos += bytes;
1450	buf += bytes;
1451	size -= bytes;
1452	}
1453	}
1454
1455	static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
1456	unsigned long offset, void *buf,
1457	int len, int write)
1458	{
1459	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo: bo);
1460	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: abo->tbo.bdev);
1461	struct amdgpu_res_cursor src_mm;
1462	struct amdgpu_job *job;
1463	struct dma_fence *fence;
1464	uint64_t src_addr, dst_addr;
1465	unsigned int num_dw;
1466	int r, idx;
1467
1468	if (len != PAGE_SIZE)
1469	return -EINVAL;
1470
1471	if (!adev->mman.sdma_access_ptr)
1472	return -EACCES;
1473
1474	if (!drm_dev_enter(dev: adev_to_drm(adev), idx: &idx))
1475	return -ENODEV;
1476
1477	if (write)
1478	memcpy(adev->mman.sdma_access_ptr, buf, len);
1479
1480	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
1481	r = amdgpu_job_alloc_with_ib(adev, entity: &adev->mman.high_pr,
1482	AMDGPU_FENCE_OWNER_UNDEFINED,
1483	size: num_dw * 4, pool_type: AMDGPU_IB_POOL_DELAYED,
1484	job: &job,
1485	AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA);
1486	if (r)
1487	goto out;
1488
1489	mutex_lock(&adev->mman.gtt_window_lock);
1490	amdgpu_res_first(res: abo->tbo.resource, start: offset, size: len, cur: &src_mm);
1491	src_addr = amdgpu_ttm_domain_start(adev, type: bo->resource->mem_type) +
1492	src_mm.start;
1493	dst_addr = amdgpu_bo_gpu_offset(bo: adev->mman.sdma_access_bo);
1494	if (write)
1495	swap(src_addr, dst_addr);
1496
1497	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr,
1498	PAGE_SIZE, 0);
1499
1500	amdgpu_ring_pad_ib(adev->mman.buffer_funcs_ring, &job->ibs[0]);
1501	WARN_ON(job->ibs[0].length_dw > num_dw);
1502
1503	fence = amdgpu_job_submit(job);
1504	mutex_unlock(lock: &adev->mman.gtt_window_lock);
1505
1506	if (!dma_fence_wait_timeout(fence, intr: false, timeout: adev->sdma_timeout))
1507	r = -ETIMEDOUT;
1508	dma_fence_put(fence);
1509
1510	if (!(r || write))
1511	memcpy(buf, adev->mman.sdma_access_ptr, len);
1512	out:
1513	drm_dev_exit(idx);
1514	return r;
1515	}
1516
1517	/**
1518	* amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
1519	*
1520	* @bo: The buffer object to read/write
1521	* @offset: Offset into buffer object
1522	* @buf: Secondary buffer to write/read from
1523	* @len: Length in bytes of access
1524	* @write: true if writing
1525	*
1526	* This is used to access VRAM that backs a buffer object via MMIO
1527	* access for debugging purposes.
1528	*/
1529	static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
1530	unsigned long offset, void *buf, int len,
1531	int write)
1532	{
1533	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo: bo);
1534	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: abo->tbo.bdev);
1535	struct amdgpu_res_cursor cursor;
1536	int ret = 0;
1537
1538	if (bo->resource->mem_type != TTM_PL_VRAM)
1539	return -EIO;
1540
1541	if (amdgpu_device_has_timeouts_enabled(adev) &&
1542	!amdgpu_ttm_access_memory_sdma(bo, offset, buf, len, write))
1543	return len;
1544
1545	amdgpu_res_first(res: bo->resource, start: offset, size: len, cur: &cursor);
1546	while (cursor.remaining) {
1547	size_t count, size = cursor.size;
1548	loff_t pos = cursor.start;
1549
1550	count = amdgpu_device_aper_access(adev, pos, buf, size, write);
1551	size -= count;
1552	if (size) {
1553	/* using MM to access rest vram and handle un-aligned address */
1554	pos += count;
1555	buf += count;
1556	amdgpu_ttm_vram_mm_access(adev, pos, buf, size, write);
1557	}
1558
1559	ret += cursor.size;
1560	buf += cursor.size;
1561	amdgpu_res_next(cur: &cursor, size: cursor.size);
1562	}
1563
1564	return ret;
1565	}
1566
1567	static void
1568	amdgpu_bo_delete_mem_notify(struct ttm_buffer_object *bo)
1569	{
1570	amdgpu_bo_move_notify(bo, evict: false, NULL);
1571	}
1572
1573	static struct ttm_device_funcs amdgpu_bo_driver = {
1574	.ttm_tt_create = &amdgpu_ttm_tt_create,
1575	.ttm_tt_populate = &amdgpu_ttm_tt_populate,
1576	.ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
1577	.ttm_tt_destroy = &amdgpu_ttm_backend_destroy,
1578	.eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
1579	.evict_flags = &amdgpu_evict_flags,
1580	.move = &amdgpu_bo_move,
1581	.delete_mem_notify = &amdgpu_bo_delete_mem_notify,
1582	.release_notify = &amdgpu_bo_release_notify,
1583	.io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1584	.io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1585	.access_memory = &amdgpu_ttm_access_memory,
1586	};
1587
1588	/*
1589	* Firmware Reservation functions
1590	*/
1591	/**
1592	* amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
1593	*
1594	* @adev: amdgpu_device pointer
1595	*
1596	* free fw reserved vram if it has been reserved.
1597	*/
1598	static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
1599	{
1600	amdgpu_bo_free_kernel(bo: &adev->mman.fw_vram_usage_reserved_bo,
1601	NULL, cpu_addr: &adev->mman.fw_vram_usage_va);
1602	}
1603
1604	/*
1605	* Driver Reservation functions
1606	*/
1607	/**
1608	* amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram
1609	*
1610	* @adev: amdgpu_device pointer
1611	*
1612	* free drv reserved vram if it has been reserved.
1613	*/
1614	static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev)
1615	{
1616	amdgpu_bo_free_kernel(bo: &adev->mman.drv_vram_usage_reserved_bo,
1617	NULL,
1618	cpu_addr: &adev->mman.drv_vram_usage_va);
1619	}
1620
1621	/**
1622	* amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
1623	*
1624	* @adev: amdgpu_device pointer
1625	*
1626	* create bo vram reservation from fw.
1627	*/
1628	static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
1629	{
1630	uint64_t vram_size = adev->gmc.visible_vram_size;
1631
1632	adev->mman.fw_vram_usage_va = NULL;
1633	adev->mman.fw_vram_usage_reserved_bo = NULL;
1634
1635	if (adev->mman.fw_vram_usage_size == 0 ||
1636	adev->mman.fw_vram_usage_size > vram_size)
1637	return 0;
1638
1639	return amdgpu_bo_create_kernel_at(adev,
1640	offset: adev->mman.fw_vram_usage_start_offset,
1641	size: adev->mman.fw_vram_usage_size,
1642	bo_ptr: &adev->mman.fw_vram_usage_reserved_bo,
1643	cpu_addr: &adev->mman.fw_vram_usage_va);
1644	}
1645
1646	/**
1647	* amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver
1648	*
1649	* @adev: amdgpu_device pointer
1650	*
1651	* create bo vram reservation from drv.
1652	*/
1653	static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
1654	{
1655	u64 vram_size = adev->gmc.visible_vram_size;
1656
1657	adev->mman.drv_vram_usage_va = NULL;
1658	adev->mman.drv_vram_usage_reserved_bo = NULL;
1659
1660	if (adev->mman.drv_vram_usage_size == 0 ||
1661	adev->mman.drv_vram_usage_size > vram_size)
1662	return 0;
1663
1664	return amdgpu_bo_create_kernel_at(adev,
1665	offset: adev->mman.drv_vram_usage_start_offset,
1666	size: adev->mman.drv_vram_usage_size,
1667	bo_ptr: &adev->mman.drv_vram_usage_reserved_bo,
1668	cpu_addr: &adev->mman.drv_vram_usage_va);
1669	}
1670
1671	/*
1672	* Memoy training reservation functions
1673	*/
1674
1675	/**
1676	* amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
1677	*
1678	* @adev: amdgpu_device pointer
1679	*
1680	* free memory training reserved vram if it has been reserved.
1681	*/
1682	static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device *adev)
1683	{
1684	struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1685
1686	ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
1687	amdgpu_bo_free_kernel(bo: &ctx->c2p_bo, NULL, NULL);
1688	ctx->c2p_bo = NULL;
1689
1690	return 0;
1691	}
1692
1693	static void amdgpu_ttm_training_data_block_init(struct amdgpu_device *adev,
1694	uint32_t reserve_size)
1695	{
1696	struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1697
1698	memset(ctx, 0, sizeof(*ctx));
1699
1700	ctx->c2p_train_data_offset =
1701	ALIGN((adev->gmc.mc_vram_size - reserve_size - SZ_1M), SZ_1M);
1702	ctx->p2c_train_data_offset =
1703	(adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET);
1704	ctx->train_data_size =
1705	GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES;
1706
1707	DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
1708	ctx->train_data_size,
1709	ctx->p2c_train_data_offset,
1710	ctx->c2p_train_data_offset);
1711	}
1712
1713	/*
1714	* reserve TMR memory at the top of VRAM which holds
1715	* IP Discovery data and is protected by PSP.
1716	*/
1717	static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev)
1718	{
1719	struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
1720	bool mem_train_support = false;
1721	uint32_t reserve_size = 0;
1722	int ret;
1723
1724	if (adev->bios && !amdgpu_sriov_vf(adev)) {
1725	if (amdgpu_atomfirmware_mem_training_supported(adev))
1726	mem_train_support = true;
1727	else
1728	DRM_DEBUG("memory training does not support!\n");
1729	}
1730
1731	/*
1732	* Query reserved tmr size through atom firmwareinfo for Sienna_Cichlid and onwards for all
1733	* the use cases (IP discovery/G6 memory training/profiling/diagnostic data.etc)
1734	*
1735	* Otherwise, fallback to legacy approach to check and reserve tmr block for ip
1736	* discovery data and G6 memory training data respectively
1737	*/
1738	if (adev->bios)
1739	reserve_size =
1740	amdgpu_atomfirmware_get_fw_reserved_fb_size(adev);
1741
1742	if (!adev->bios &&
1743	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3) ||
1744	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 4) ||
1745	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 5, 0)))
1746	reserve_size = max(reserve_size, (uint32_t)280 << 20);
1747	else if (!reserve_size)
1748	reserve_size = DISCOVERY_TMR_OFFSET;
1749
1750	if (mem_train_support) {
1751	/* reserve vram for mem train according to TMR location */
1752	amdgpu_ttm_training_data_block_init(adev, reserve_size);
1753	ret = amdgpu_bo_create_kernel_at(adev,
1754	offset: ctx->c2p_train_data_offset,
1755	size: ctx->train_data_size,
1756	bo_ptr: &ctx->c2p_bo,
1757	NULL);
1758	if (ret) {
1759	dev_err(adev->dev, "alloc c2p_bo failed(%d)!\n", ret);
1760	amdgpu_ttm_training_reserve_vram_fini(adev);
1761	return ret;
1762	}
1763	ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS;
1764	}
1765
1766	ret = amdgpu_bo_create_kernel_at(
1767	adev, offset: adev->gmc.real_vram_size - reserve_size, size: reserve_size,
1768	bo_ptr: &adev->mman.fw_reserved_memory, NULL);
1769	if (ret) {
1770	dev_err(adev->dev, "alloc tmr failed(%d)!\n", ret);
1771	amdgpu_bo_free_kernel(bo: &adev->mman.fw_reserved_memory, NULL,
1772	NULL);
1773	return ret;
1774	}
1775
1776	return 0;
1777	}
1778
1779	static int amdgpu_ttm_pools_init(struct amdgpu_device *adev)
1780	{
1781	int i;
1782
1783	if (!adev->gmc.is_app_apu || !adev->gmc.num_mem_partitions)
1784	return 0;
1785
1786	adev->mman.ttm_pools = kcalloc(adev->gmc.num_mem_partitions,
1787	sizeof(*adev->mman.ttm_pools),
1788	GFP_KERNEL);
1789	if (!adev->mman.ttm_pools)
1790	return -ENOMEM;
1791
1792	for (i = 0; i < adev->gmc.num_mem_partitions; i++) {
1793	ttm_pool_init(pool: &adev->mman.ttm_pools[i], dev: adev->dev,
1794	nid: adev->gmc.mem_partitions[i].numa.node,
1795	TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M)));
1796	}
1797	return 0;
1798	}
1799
1800	static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev)
1801	{
1802	int i;
1803
1804	if (!adev->gmc.is_app_apu || !adev->mman.ttm_pools)
1805	return;
1806
1807	for (i = 0; i < adev->gmc.num_mem_partitions; i++)
1808	ttm_pool_fini(pool: &adev->mman.ttm_pools[i]);
1809
1810	kfree(objp: adev->mman.ttm_pools);
1811	adev->mman.ttm_pools = NULL;
1812	}
1813
1814	/**
1815	* amdgpu_ttm_mmio_remap_bo_init - Allocate the singleton 4K MMIO_REMAP BO
1816	* @adev: amdgpu device
1817	*
1818	* Allocates a one-page (4K) GEM BO in AMDGPU_GEM_DOMAIN_MMIO_REMAP when the
1819	* hardware exposes a remap base (adev->rmmio_remap.bus_addr) and the host
1820	* PAGE_SIZE is <= AMDGPU_GPU_PAGE_SIZE (4K). The BO is created as a regular
1821	* GEM object (amdgpu_bo_create).
1822	*
1823	* Return:
1824	* * 0 on success or intentional skip (feature not present/unsupported)
1825	* * negative errno on allocation failure
1826	*/
1827	static int amdgpu_ttm_mmio_remap_bo_init(struct amdgpu_device *adev)
1828	{
1829	struct amdgpu_bo_param bp;
1830	int r;
1831
1832	/* Skip if HW doesn't expose remap, or if PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE (4K). */
1833	if (!adev->rmmio_remap.bus_addr || PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE)
1834	return 0;
1835
1836	memset(&bp, 0, sizeof(bp));
1837
1838	/* Create exactly one GEM BO in the MMIO_REMAP domain. */
1839	bp.type = ttm_bo_type_device; /* userspace-mappable GEM */
1840	bp.size = AMDGPU_GPU_PAGE_SIZE; /* 4K */
1841	bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
1842	bp.domain = AMDGPU_GEM_DOMAIN_MMIO_REMAP;
1843	bp.flags = 0;
1844	bp.resv = NULL;
1845	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
1846
1847	r = amdgpu_bo_create(adev, bp: &bp, bo_ptr: &adev->rmmio_remap.bo);
1848	if (r)
1849	return r;
1850
1851	return 0;
1852	}
1853
1854	/**
1855	* amdgpu_ttm_mmio_remap_bo_fini - Free the singleton MMIO_REMAP BO
1856	* @adev: amdgpu device
1857	*
1858	* Frees the kernel-owned MMIO_REMAP BO if it was allocated by
1859	* amdgpu_ttm_mmio_remap_bo_init().
1860	*/
1861	static void amdgpu_ttm_mmio_remap_bo_fini(struct amdgpu_device *adev)
1862	{
1863	amdgpu_bo_unref(bo: &adev->rmmio_remap.bo);
1864	adev->rmmio_remap.bo = NULL;
1865	}
1866
1867	/*
1868	* amdgpu_ttm_init - Init the memory management (ttm) as well as various
1869	* gtt/vram related fields.
1870	*
1871	* This initializes all of the memory space pools that the TTM layer
1872	* will need such as the GTT space (system memory mapped to the device),
1873	* VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
1874	* can be mapped per VMID.
1875	*/
1876	int amdgpu_ttm_init(struct amdgpu_device *adev)
1877	{
1878	uint64_t gtt_size;
1879	int r;
1880
1881	mutex_init(&adev->mman.gtt_window_lock);
1882
1883	dma_set_max_seg_size(dev: adev->dev, UINT_MAX);
1884	/* No others user of address space so set it to 0 */
1885	r = ttm_device_init(bdev: &adev->mman.bdev, funcs: &amdgpu_bo_driver, dev: adev->dev,
1886	mapping: adev_to_drm(adev)->anon_inode->i_mapping,
1887	vma_manager: adev_to_drm(adev)->vma_offset_manager,
1888	alloc_flags: (adev->need_swiotlb ?
1889	TTM_ALLOCATION_POOL_USE_DMA_ALLOC : 0) |
1890	(dma_addressing_limited(dev: adev->dev) ?
1891	TTM_ALLOCATION_POOL_USE_DMA32 : 0) |
1892	TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M)));
1893	if (r) {
1894	dev_err(adev->dev,
1895	"failed initializing buffer object driver(%d).\n", r);
1896	return r;
1897	}
1898
1899	r = amdgpu_ttm_pools_init(adev);
1900	if (r) {
1901	dev_err(adev->dev, "failed to init ttm pools(%d).\n", r);
1902	return r;
1903	}
1904	adev->mman.initialized = true;
1905
1906	if (!adev->gmc.is_app_apu) {
1907	/* Initialize VRAM pool with all of VRAM divided into pages */
1908	r = amdgpu_vram_mgr_init(adev);
1909	if (r) {
1910	dev_err(adev->dev, "Failed initializing VRAM heap.\n");
1911	return r;
1912	}
1913	}
1914
1915	/* Change the size here instead of the init above so only lpfn is affected */
1916	amdgpu_ttm_set_buffer_funcs_status(adev, enable: false);
1917	#ifdef CONFIG_64BIT
1918	#ifdef CONFIG_X86
1919	if (adev->gmc.xgmi.connected_to_cpu)
1920	adev->mman.aper_base_kaddr = ioremap_cache(offset: adev->gmc.aper_base,
1921	size: adev->gmc.visible_vram_size);
1922
1923	else if (adev->gmc.is_app_apu)
1924	DRM_DEBUG_DRIVER(
1925	"No need to ioremap when real vram size is 0\n");
1926	else
1927	#endif
1928	adev->mman.aper_base_kaddr = ioremap_wc(offset: adev->gmc.aper_base,
1929	size: adev->gmc.visible_vram_size);
1930	#endif
1931
1932	/*
1933	*The reserved vram for firmware must be pinned to the specified
1934	*place on the VRAM, so reserve it early.
1935	*/
1936	r = amdgpu_ttm_fw_reserve_vram_init(adev);
1937	if (r)
1938	return r;
1939
1940	/*
1941	* The reserved VRAM for the driver must be pinned to a specific
1942	* location in VRAM, so reserve it early.
1943	*/
1944	r = amdgpu_ttm_drv_reserve_vram_init(adev);
1945	if (r)
1946	return r;
1947
1948	/*
1949	* only NAVI10 and later ASICs support IP discovery.
1950	* If IP discovery is enabled, a block of memory should be
1951	* reserved for it.
1952	*/
1953	if (adev->discovery.reserve_tmr) {
1954	r = amdgpu_ttm_reserve_tmr(adev);
1955	if (r)
1956	return r;
1957	}
1958
1959	/* allocate memory as required for VGA
1960	* This is used for VGA emulation and pre-OS scanout buffers to
1961	* avoid display artifacts while transitioning between pre-OS
1962	* and driver.
1963	*/
1964	if (!adev->gmc.is_app_apu) {
1965	r = amdgpu_bo_create_kernel_at(adev, offset: 0,
1966	size: adev->mman.stolen_vga_size,
1967	bo_ptr: &adev->mman.stolen_vga_memory,
1968	NULL);
1969	if (r)
1970	return r;
1971
1972	r = amdgpu_bo_create_kernel_at(adev, offset: adev->mman.stolen_vga_size,
1973	size: adev->mman.stolen_extended_size,
1974	bo_ptr: &adev->mman.stolen_extended_memory,
1975	NULL);
1976
1977	if (r)
1978	return r;
1979
1980	r = amdgpu_bo_create_kernel_at(adev,
1981	offset: adev->mman.stolen_reserved_offset,
1982	size: adev->mman.stolen_reserved_size,
1983	bo_ptr: &adev->mman.stolen_reserved_memory,
1984	NULL);
1985	if (r)
1986	return r;
1987	} else {
1988	DRM_DEBUG_DRIVER("Skipped stolen memory reservation\n");
1989	}
1990
1991	dev_info(adev->dev, "amdgpu: %uM of VRAM memory ready\n",
1992	(unsigned int)(adev->gmc.real_vram_size / (1024 * 1024)));
1993
1994	/* Compute GTT size, either based on TTM limit
1995	* or whatever the user passed on module init.
1996	*/
1997	gtt_size = ttm_tt_pages_limit() << PAGE_SHIFT;
1998	if (amdgpu_gtt_size != -1) {
1999	uint64_t configured_size = (uint64_t)amdgpu_gtt_size << 20;
2000
2001	drm_warn(&adev->ddev,
2002	"Configuring gttsize via module parameter is deprecated, please use ttm.pages_limit\n");
2003	if (gtt_size != configured_size)
2004	drm_warn(&adev->ddev,
2005	"GTT size has been set as %llu but TTM size has been set as %llu, this is unusual\n",
2006	configured_size, gtt_size);
2007
2008	gtt_size = configured_size;
2009	}
2010
2011	/* Initialize GTT memory pool */
2012	r = amdgpu_gtt_mgr_init(adev, gtt_size);
2013	if (r) {
2014	dev_err(adev->dev, "Failed initializing GTT heap.\n");
2015	return r;
2016	}
2017	dev_info(adev->dev, "amdgpu: %uM of GTT memory ready.\n",
2018	(unsigned int)(gtt_size / (1024 * 1024)));
2019
2020	if (adev->flags & AMD_IS_APU) {
2021	if (adev->gmc.real_vram_size < gtt_size)
2022	adev->apu_prefer_gtt = true;
2023	}
2024
2025	/* Initialize doorbell pool on PCI BAR */
2026	r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_DOORBELL, size_in_page: adev->doorbell.size / PAGE_SIZE);
2027	if (r) {
2028	dev_err(adev->dev, "Failed initializing doorbell heap.\n");
2029	return r;
2030	}
2031
2032	/* Create a boorbell page for kernel usages */
2033	r = amdgpu_doorbell_create_kernel_doorbells(adev);
2034	if (r) {
2035	dev_err(adev->dev, "Failed to initialize kernel doorbells.\n");
2036	return r;
2037	}
2038
2039	/* Initialize MMIO-remap pool (single page 4K) */
2040	r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_MMIO_REMAP, size_in_page: 1);
2041	if (r) {
2042	dev_err(adev->dev, "Failed initializing MMIO-remap heap.\n");
2043	return r;
2044	}
2045
2046	/* Allocate the singleton MMIO_REMAP BO (4K) if supported */
2047	r = amdgpu_ttm_mmio_remap_bo_init(adev);
2048	if (r)
2049	return r;
2050
2051	/* Initialize preemptible memory pool */
2052	r = amdgpu_preempt_mgr_init(adev);
2053	if (r) {
2054	dev_err(adev->dev, "Failed initializing PREEMPT heap.\n");
2055	return r;
2056	}
2057
2058	/* Initialize various on-chip memory pools */
2059	r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, size_in_page: adev->gds.gds_size);
2060	if (r) {
2061	dev_err(adev->dev, "Failed initializing GDS heap.\n");
2062	return r;
2063	}
2064
2065	r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, size_in_page: adev->gds.gws_size);
2066	if (r) {
2067	dev_err(adev->dev, "Failed initializing gws heap.\n");
2068	return r;
2069	}
2070
2071	r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, size_in_page: adev->gds.oa_size);
2072	if (r) {
2073	dev_err(adev->dev, "Failed initializing oa heap.\n");
2074	return r;
2075	}
2076	if (amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
2077	AMDGPU_GEM_DOMAIN_GTT,
2078	bo_ptr: &adev->mman.sdma_access_bo, NULL,
2079	cpu_addr: &adev->mman.sdma_access_ptr))
2080	DRM_WARN("Debug VRAM access will use slowpath MM access\n");
2081
2082	return 0;
2083	}
2084
2085	/*
2086	* amdgpu_ttm_fini - De-initialize the TTM memory pools
2087	*/
2088	void amdgpu_ttm_fini(struct amdgpu_device *adev)
2089	{
2090	int idx;
2091
2092	if (!adev->mman.initialized)
2093	return;
2094
2095	amdgpu_ttm_pools_fini(adev);
2096
2097	amdgpu_ttm_training_reserve_vram_fini(adev);
2098	/* return the stolen vga memory back to VRAM */
2099	if (!adev->gmc.is_app_apu) {
2100	amdgpu_bo_free_kernel(bo: &adev->mman.stolen_vga_memory, NULL, NULL);
2101	amdgpu_bo_free_kernel(bo: &adev->mman.stolen_extended_memory, NULL, NULL);
2102	/* return the FW reserved memory back to VRAM */
2103	amdgpu_bo_free_kernel(bo: &adev->mman.fw_reserved_memory, NULL,
2104	NULL);
2105	amdgpu_bo_free_kernel(bo: &adev->mman.fw_reserved_memory_extend, NULL,
2106	NULL);
2107	if (adev->mman.stolen_reserved_size)
2108	amdgpu_bo_free_kernel(bo: &adev->mman.stolen_reserved_memory,
2109	NULL, NULL);
2110	}
2111	amdgpu_bo_free_kernel(bo: &adev->mman.sdma_access_bo, NULL,
2112	cpu_addr: &adev->mman.sdma_access_ptr);
2113
2114	amdgpu_ttm_mmio_remap_bo_fini(adev);
2115	amdgpu_ttm_fw_reserve_vram_fini(adev);
2116	amdgpu_ttm_drv_reserve_vram_fini(adev);
2117
2118	if (drm_dev_enter(dev: adev_to_drm(adev), idx: &idx)) {
2119
2120	if (adev->mman.aper_base_kaddr)
2121	iounmap(addr: adev->mman.aper_base_kaddr);
2122	adev->mman.aper_base_kaddr = NULL;
2123
2124	drm_dev_exit(idx);
2125	}
2126
2127	if (!adev->gmc.is_app_apu)
2128	amdgpu_vram_mgr_fini(adev);
2129	amdgpu_gtt_mgr_fini(adev);
2130	amdgpu_preempt_mgr_fini(adev);
2131	amdgpu_doorbell_fini(adev);
2132
2133	ttm_range_man_fini(bdev: &adev->mman.bdev, AMDGPU_PL_GDS);
2134	ttm_range_man_fini(bdev: &adev->mman.bdev, AMDGPU_PL_GWS);
2135	ttm_range_man_fini(bdev: &adev->mman.bdev, AMDGPU_PL_OA);
2136	ttm_range_man_fini(bdev: &adev->mman.bdev, AMDGPU_PL_DOORBELL);
2137	ttm_range_man_fini(bdev: &adev->mman.bdev, AMDGPU_PL_MMIO_REMAP);
2138	ttm_device_fini(bdev: &adev->mman.bdev);
2139	adev->mman.initialized = false;
2140	dev_info(adev->dev, "amdgpu: ttm finalized\n");
2141	}
2142
2143	/**
2144	* amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
2145	*
2146	* @adev: amdgpu_device pointer
2147	* @enable: true when we can use buffer functions.
2148	*
2149	* Enable/disable use of buffer functions during suspend/resume. This should
2150	* only be called at bootup or when userspace isn't running.
2151	*/
2152	void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
2153	{
2154	struct ttm_resource_manager *man = ttm_manager_type(bdev: &adev->mman.bdev, TTM_PL_VRAM);
2155	uint64_t size;
2156	int r;
2157
2158	if (!adev->mman.initialized || amdgpu_in_reset(adev) ||
2159	adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu)
2160	return;
2161
2162	if (enable) {
2163	struct amdgpu_ring *ring;
2164	struct drm_gpu_scheduler *sched;
2165
2166	ring = adev->mman.buffer_funcs_ring;
2167	sched = &ring->sched;
2168	r = drm_sched_entity_init(entity: &adev->mman.high_pr,
2169	priority: DRM_SCHED_PRIORITY_KERNEL, sched_list: &sched,
2170	num_sched_list: 1, NULL);
2171	if (r) {
2172	dev_err(adev->dev,
2173	"Failed setting up TTM BO move entity (%d)\n",
2174	r);
2175	return;
2176	}
2177
2178	r = drm_sched_entity_init(entity: &adev->mman.low_pr,
2179	priority: DRM_SCHED_PRIORITY_NORMAL, sched_list: &sched,
2180	num_sched_list: 1, NULL);
2181	if (r) {
2182	dev_err(adev->dev,
2183	"Failed setting up TTM BO move entity (%d)\n",
2184	r);
2185	goto error_free_entity;
2186	}
2187	} else {
2188	drm_sched_entity_destroy(entity: &adev->mman.high_pr);
2189	drm_sched_entity_destroy(entity: &adev->mman.low_pr);
2190	/* Drop all the old fences since re-creating the scheduler entities
2191	* will allocate new contexts.
2192	*/
2193	ttm_resource_manager_cleanup(man);
2194	}
2195
2196	/* this just adjusts TTM size idea, which sets lpfn to the correct value */
2197	if (enable)
2198	size = adev->gmc.real_vram_size;
2199	else
2200	size = adev->gmc.visible_vram_size;
2201	man->size = size;
2202	adev->mman.buffer_funcs_enabled = enable;
2203
2204	return;
2205
2206	error_free_entity:
2207	drm_sched_entity_destroy(entity: &adev->mman.high_pr);
2208	}
2209
2210	static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
2211	bool direct_submit,
2212	unsigned int num_dw,
2213	struct dma_resv *resv,
2214	bool vm_needs_flush,
2215	struct amdgpu_job **job,
2216	bool delayed, u64 k_job_id)
2217	{
2218	enum amdgpu_ib_pool_type pool = direct_submit ?
2219	AMDGPU_IB_POOL_DIRECT :
2220	AMDGPU_IB_POOL_DELAYED;
2221	int r;
2222	struct drm_sched_entity *entity = delayed ? &adev->mman.low_pr :
2223	&adev->mman.high_pr;
2224	r = amdgpu_job_alloc_with_ib(adev, entity,
2225	AMDGPU_FENCE_OWNER_UNDEFINED,
2226	size: num_dw * 4, pool_type: pool, job, k_job_id);
2227	if (r)
2228	return r;
2229
2230	if (vm_needs_flush) {
2231	(*job)->vm_pd_addr = amdgpu_gmc_pd_addr(bo: adev->gmc.pdb0_bo ?
2232	adev->gmc.pdb0_bo :
2233	adev->gart.bo);
2234	(*job)->vm_needs_flush = true;
2235	}
2236	if (!resv)
2237	return 0;
2238
2239	return drm_sched_job_add_resv_dependencies(job: &(*job)->base, resv,
2240	usage: DMA_RESV_USAGE_BOOKKEEP);
2241	}
2242
2243	int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
2244	uint64_t dst_offset, uint32_t byte_count,
2245	struct dma_resv *resv,
2246	struct dma_fence **fence, bool direct_submit,
2247	bool vm_needs_flush, uint32_t copy_flags)
2248	{
2249	struct amdgpu_device *adev = ring->adev;
2250	unsigned int num_loops, num_dw;
2251	struct amdgpu_job *job;
2252	uint32_t max_bytes;
2253	unsigned int i;
2254	int r;
2255
2256	if (!direct_submit && !ring->sched.ready) {
2257	dev_err(adev->dev,
2258	"Trying to move memory with ring turned off.\n");
2259	return -EINVAL;
2260	}
2261
2262	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
2263	num_loops = DIV_ROUND_UP(byte_count, max_bytes);
2264	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);
2265	r = amdgpu_ttm_prepare_job(adev, direct_submit, num_dw,
2266	resv, vm_needs_flush, job: &job, delayed: false,
2267	AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER);
2268	if (r)
2269	return r;
2270
2271	for (i = 0; i < num_loops; i++) {
2272	uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2273
2274	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
2275	dst_offset, cur_size_in_bytes, copy_flags);
2276	src_offset += cur_size_in_bytes;
2277	dst_offset += cur_size_in_bytes;
2278	byte_count -= cur_size_in_bytes;
2279	}
2280
2281	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2282	WARN_ON(job->ibs[0].length_dw > num_dw);
2283	if (direct_submit)
2284	r = amdgpu_job_submit_direct(job, ring, fence);
2285	else
2286	*fence = amdgpu_job_submit(job);
2287	if (r)
2288	goto error_free;
2289
2290	return r;
2291
2292	error_free:
2293	amdgpu_job_free(job);
2294	dev_err(adev->dev, "Error scheduling IBs (%d)\n", r);
2295	return r;
2296	}
2297
2298	static int amdgpu_ttm_fill_mem(struct amdgpu_ring *ring, uint32_t src_data,
2299	uint64_t dst_addr, uint32_t byte_count,
2300	struct dma_resv *resv,
2301	struct dma_fence **fence,
2302	bool vm_needs_flush, bool delayed,
2303	u64 k_job_id)
2304	{
2305	struct amdgpu_device *adev = ring->adev;
2306	unsigned int num_loops, num_dw;
2307	struct amdgpu_job *job;
2308	uint32_t max_bytes;
2309	unsigned int i;
2310	int r;
2311
2312	max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2313	num_loops = DIV_ROUND_UP_ULL(byte_count, max_bytes);
2314	num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8);
2315	r = amdgpu_ttm_prepare_job(adev, direct_submit: false, num_dw, resv, vm_needs_flush,
2316	job: &job, delayed, k_job_id);
2317	if (r)
2318	return r;
2319
2320	for (i = 0; i < num_loops; i++) {
2321	uint32_t cur_size = min(byte_count, max_bytes);
2322
2323	amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr,
2324	cur_size);
2325
2326	dst_addr += cur_size;
2327	byte_count -= cur_size;
2328	}
2329
2330	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2331	WARN_ON(job->ibs[0].length_dw > num_dw);
2332	*fence = amdgpu_job_submit(job);
2333	return 0;
2334	}
2335
2336	/**
2337	* amdgpu_ttm_clear_buffer - clear memory buffers
2338	* @bo: amdgpu buffer object
2339	* @resv: reservation object
2340	* @fence: dma_fence associated with the operation
2341	*
2342	* Clear the memory buffer resource.
2343	*
2344	* Returns:
2345	* 0 for success or a negative error code on failure.
2346	*/
2347	int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
2348	struct dma_resv *resv,
2349	struct dma_fence **fence)
2350	{
2351	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
2352	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2353	struct amdgpu_res_cursor cursor;
2354	u64 addr;
2355	int r = 0;
2356
2357	if (!adev->mman.buffer_funcs_enabled)
2358	return -EINVAL;
2359
2360	if (!fence)
2361	return -EINVAL;
2362
2363	*fence = dma_fence_get_stub();
2364
2365	amdgpu_res_first(res: bo->tbo.resource, start: 0, size: amdgpu_bo_size(bo), cur: &cursor);
2366
2367	mutex_lock(&adev->mman.gtt_window_lock);
2368	while (cursor.remaining) {
2369	struct dma_fence *next = NULL;
2370	u64 size;
2371
2372	if (amdgpu_res_cleared(cur: &cursor)) {
2373	amdgpu_res_next(cur: &cursor, size: cursor.size);
2374	continue;
2375	}
2376
2377	/* Never clear more than 256MiB at once to avoid timeouts */
2378	size = min(cursor.size, 256ULL << 20);
2379
2380	r = amdgpu_ttm_map_buffer(bo: &bo->tbo, mem: bo->tbo.resource, mm_cur: &cursor,
2381	window: 1, ring, tmz: false, size: &size, addr: &addr);
2382	if (r)
2383	goto err;
2384
2385	r = amdgpu_ttm_fill_mem(ring, src_data: 0, dst_addr: addr, byte_count: size, resv,
2386	fence: &next, vm_needs_flush: true, delayed: true,
2387	AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
2388	if (r)
2389	goto err;
2390
2391	dma_fence_put(fence: *fence);
2392	*fence = next;
2393
2394	amdgpu_res_next(cur: &cursor, size);
2395	}
2396	err:
2397	mutex_unlock(lock: &adev->mman.gtt_window_lock);
2398
2399	return r;
2400	}
2401
2402	int amdgpu_fill_buffer(struct amdgpu_bo *bo,
2403	uint32_t src_data,
2404	struct dma_resv *resv,
2405	struct dma_fence **f,
2406	bool delayed,
2407	u64 k_job_id)
2408	{
2409	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
2410	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2411	struct dma_fence *fence = NULL;
2412	struct amdgpu_res_cursor dst;
2413	int r;
2414
2415	if (!adev->mman.buffer_funcs_enabled) {
2416	dev_err(adev->dev,
2417	"Trying to clear memory with ring turned off.\n");
2418	return -EINVAL;
2419	}
2420
2421	amdgpu_res_first(res: bo->tbo.resource, start: 0, size: amdgpu_bo_size(bo), cur: &dst);
2422
2423	mutex_lock(&adev->mman.gtt_window_lock);
2424	while (dst.remaining) {
2425	struct dma_fence *next;
2426	uint64_t cur_size, to;
2427
2428	/* Never fill more than 256MiB at once to avoid timeouts */
2429	cur_size = min(dst.size, 256ULL << 20);
2430
2431	r = amdgpu_ttm_map_buffer(bo: &bo->tbo, mem: bo->tbo.resource, mm_cur: &dst,
2432	window: 1, ring, tmz: false, size: &cur_size, addr: &to);
2433	if (r)
2434	goto error;
2435
2436	r = amdgpu_ttm_fill_mem(ring, src_data, dst_addr: to, byte_count: cur_size, resv,
2437	fence: &next, vm_needs_flush: true, delayed, k_job_id);
2438	if (r)
2439	goto error;
2440
2441	dma_fence_put(fence);
2442	fence = next;
2443
2444	amdgpu_res_next(cur: &dst, size: cur_size);
2445	}
2446	error:
2447	mutex_unlock(lock: &adev->mman.gtt_window_lock);
2448	if (f)
2449	*f = dma_fence_get(fence);
2450	dma_fence_put(fence);
2451	return r;
2452	}
2453
2454	/**
2455	* amdgpu_ttm_evict_resources - evict memory buffers
2456	* @adev: amdgpu device object
2457	* @mem_type: evicted BO's memory type
2458	*
2459	* Evicts all @mem_type buffers on the lru list of the memory type.
2460	*
2461	* Returns:
2462	* 0 for success or a negative error code on failure.
2463	*/
2464	int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type)
2465	{
2466	struct ttm_resource_manager *man;
2467
2468	switch (mem_type) {
2469	case TTM_PL_VRAM:
2470	case TTM_PL_TT:
2471	case AMDGPU_PL_GWS:
2472	case AMDGPU_PL_GDS:
2473	case AMDGPU_PL_OA:
2474	man = ttm_manager_type(bdev: &adev->mman.bdev, mem_type);
2475	break;
2476	default:
2477	dev_err(adev->dev, "Trying to evict invalid memory type\n");
2478	return -EINVAL;
2479	}
2480
2481	return ttm_resource_manager_evict_all(bdev: &adev->mman.bdev, man);
2482	}
2483
2484	#if defined(CONFIG_DEBUG_FS)
2485
2486	static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused)
2487	{
2488	struct amdgpu_device *adev = m->private;
2489
2490	return ttm_pool_debugfs(pool: &adev->mman.bdev.pool, m);
2491	}
2492
2493	DEFINE_SHOW_ATTRIBUTE(amdgpu_ttm_page_pool);
2494
2495	/*
2496	* amdgpu_ttm_vram_read - Linear read access to VRAM
2497	*
2498	* Accesses VRAM via MMIO for debugging purposes.
2499	*/
2500	static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
2501	size_t size, loff_t *pos)
2502	{
2503	struct amdgpu_device *adev = file_inode(f)->i_private;
2504	ssize_t result = 0;
2505
2506	if (size & 0x3 || *pos & 0x3)
2507	return -EINVAL;
2508
2509	if (*pos >= adev->gmc.mc_vram_size)
2510	return -ENXIO;
2511
2512	size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos));
2513	while (size) {
2514	size_t bytes = min(size, AMDGPU_TTM_VRAM_MAX_DW_READ * 4);
2515	uint32_t value[AMDGPU_TTM_VRAM_MAX_DW_READ];
2516
2517	amdgpu_device_vram_access(adev, pos: *pos, buf: value, size: bytes, write: false);
2518	if (copy_to_user(to: buf, from: value, n: bytes))
2519	return -EFAULT;
2520
2521	result += bytes;
2522	buf += bytes;
2523	*pos += bytes;
2524	size -= bytes;
2525	}
2526
2527	return result;
2528	}
2529
2530	/*
2531	* amdgpu_ttm_vram_write - Linear write access to VRAM
2532	*
2533	* Accesses VRAM via MMIO for debugging purposes.
2534	*/
2535	static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
2536	size_t size, loff_t *pos)
2537	{
2538	struct amdgpu_device *adev = file_inode(f)->i_private;
2539	ssize_t result = 0;
2540	int r;
2541
2542	if (size & 0x3 || *pos & 0x3)
2543	return -EINVAL;
2544
2545	if (*pos >= adev->gmc.mc_vram_size)
2546	return -ENXIO;
2547
2548	while (size) {
2549	uint32_t value;
2550
2551	if (*pos >= adev->gmc.mc_vram_size)
2552	return result;
2553
2554	r = get_user(value, (uint32_t *)buf);
2555	if (r)
2556	return r;
2557
2558	amdgpu_device_mm_access(adev, pos: *pos, buf: &value, size: 4, write: true);
2559
2560	result += 4;
2561	buf += 4;
2562	*pos += 4;
2563	size -= 4;
2564	}
2565
2566	return result;
2567	}
2568
2569	static const struct file_operations amdgpu_ttm_vram_fops = {
2570	.owner = THIS_MODULE,
2571	.read = amdgpu_ttm_vram_read,
2572	.write = amdgpu_ttm_vram_write,
2573	.llseek = default_llseek,
2574	};
2575
2576	/*
2577	* amdgpu_iomem_read - Virtual read access to GPU mapped memory
2578	*
2579	* This function is used to read memory that has been mapped to the
2580	* GPU and the known addresses are not physical addresses but instead
2581	* bus addresses (e.g., what you'd put in an IB or ring buffer).
2582	*/
2583	static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
2584	size_t size, loff_t *pos)
2585	{
2586	struct amdgpu_device *adev = file_inode(f)->i_private;
2587	struct iommu_domain *dom;
2588	ssize_t result = 0;
2589	int r;
2590
2591	/* retrieve the IOMMU domain if any for this device */
2592	dom = iommu_get_domain_for_dev(dev: adev->dev);
2593
2594	while (size) {
2595	phys_addr_t addr = *pos & PAGE_MASK;
2596	loff_t off = *pos & ~PAGE_MASK;
2597	size_t bytes = PAGE_SIZE - off;
2598	unsigned long pfn;
2599	struct page *p;
2600	void *ptr;
2601
2602	bytes = min(bytes, size);
2603
2604	/* Translate the bus address to a physical address. If
2605	* the domain is NULL it means there is no IOMMU active
2606	* and the address translation is the identity
2607	*/
2608	addr = dom ? iommu_iova_to_phys(domain: dom, iova: addr) : addr;
2609
2610	pfn = addr >> PAGE_SHIFT;
2611	if (!pfn_valid(pfn))
2612	return -EPERM;
2613
2614	p = pfn_to_page(pfn);
2615	if (p->mapping != adev->mman.bdev.dev_mapping)
2616	return -EPERM;
2617
2618	ptr = kmap_local_page(page: p);
2619	r = copy_to_user(to: buf, from: ptr + off, n: bytes);
2620	kunmap_local(ptr);
2621	if (r)
2622	return -EFAULT;
2623
2624	size -= bytes;
2625	*pos += bytes;
2626	result += bytes;
2627	}
2628
2629	return result;
2630	}
2631
2632	/*
2633	* amdgpu_iomem_write - Virtual write access to GPU mapped memory
2634	*
2635	* This function is used to write memory that has been mapped to the
2636	* GPU and the known addresses are not physical addresses but instead
2637	* bus addresses (e.g., what you'd put in an IB or ring buffer).
2638	*/
2639	static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
2640	size_t size, loff_t *pos)
2641	{
2642	struct amdgpu_device *adev = file_inode(f)->i_private;
2643	struct iommu_domain *dom;
2644	ssize_t result = 0;
2645	int r;
2646
2647	dom = iommu_get_domain_for_dev(dev: adev->dev);
2648
2649	while (size) {
2650	phys_addr_t addr = *pos & PAGE_MASK;
2651	loff_t off = *pos & ~PAGE_MASK;
2652	size_t bytes = PAGE_SIZE - off;
2653	unsigned long pfn;
2654	struct page *p;
2655	void *ptr;
2656
2657	bytes = min(bytes, size);
2658
2659	addr = dom ? iommu_iova_to_phys(domain: dom, iova: addr) : addr;
2660
2661	pfn = addr >> PAGE_SHIFT;
2662	if (!pfn_valid(pfn))
2663	return -EPERM;
2664
2665	p = pfn_to_page(pfn);
2666	if (p->mapping != adev->mman.bdev.dev_mapping)
2667	return -EPERM;
2668
2669	ptr = kmap_local_page(page: p);
2670	r = copy_from_user(to: ptr + off, from: buf, n: bytes);
2671	kunmap_local(ptr);
2672	if (r)
2673	return -EFAULT;
2674
2675	size -= bytes;
2676	*pos += bytes;
2677	result += bytes;
2678	}
2679
2680	return result;
2681	}
2682
2683	static const struct file_operations amdgpu_ttm_iomem_fops = {
2684	.owner = THIS_MODULE,
2685	.read = amdgpu_iomem_read,
2686	.write = amdgpu_iomem_write,
2687	.llseek = default_llseek
2688	};
2689
2690	#endif
2691
2692	void amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
2693	{
2694	#if defined(CONFIG_DEBUG_FS)
2695	struct drm_minor *minor = adev_to_drm(adev)->primary;
2696	struct dentry *root = minor->debugfs_root;
2697
2698	debugfs_create_file_size(name: "amdgpu_vram", mode: 0444, parent: root, data: adev,
2699	fops: &amdgpu_ttm_vram_fops, file_size: adev->gmc.mc_vram_size);
2700	debugfs_create_file("amdgpu_iomem", 0444, root, adev,
2701	&amdgpu_ttm_iomem_fops);
2702	debugfs_create_file("ttm_page_pool", 0444, root, adev,
2703	&amdgpu_ttm_page_pool_fops);
2704	ttm_resource_manager_create_debugfs(man: ttm_manager_type(bdev: &adev->mman.bdev,
2705	TTM_PL_VRAM),
2706	parent: root, name: "amdgpu_vram_mm");
2707	ttm_resource_manager_create_debugfs(man: ttm_manager_type(bdev: &adev->mman.bdev,
2708	TTM_PL_TT),
2709	parent: root, name: "amdgpu_gtt_mm");
2710	ttm_resource_manager_create_debugfs(man: ttm_manager_type(bdev: &adev->mman.bdev,
2711	AMDGPU_PL_GDS),
2712	parent: root, name: "amdgpu_gds_mm");
2713	ttm_resource_manager_create_debugfs(man: ttm_manager_type(bdev: &adev->mman.bdev,
2714	AMDGPU_PL_GWS),
2715	parent: root, name: "amdgpu_gws_mm");
2716	ttm_resource_manager_create_debugfs(man: ttm_manager_type(bdev: &adev->mman.bdev,
2717	AMDGPU_PL_OA),
2718	parent: root, name: "amdgpu_oa_mm");
2719
2720	#endif
2721	}
2722