1	/*
2	* Copyright 2009 Jerome Glisse.
3	* All Rights Reserved.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the
7	* "Software"), to deal in the Software without restriction, including
8	* without limitation the rights to use, copy, modify, merge, publish,
9	* distribute, sub license, and/or sell copies of the Software, and to
10	* permit persons to whom the Software is furnished to do so, subject to
11	* the following conditions:
12	*
13	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19	* USE OR OTHER DEALINGS IN THE SOFTWARE.
20	*
21	* The above copyright notice and this permission notice (including the
22	* next paragraph) shall be included in all copies or substantial portions
23	* of the Software.
24	*
25	*/
26	/*
27	* Authors:
28	* Jerome Glisse <glisse@freedesktop.org>
29	* Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30	* Dave Airlie
31	*/
32	#include <linux/list.h>
33	#include <linux/slab.h>
34	#include <linux/dma-buf.h>
35	#include <linux/export.h>
36
37	#include <drm/drm_drv.h>
38	#include <drm/amdgpu_drm.h>
39	#include <drm/drm_cache.h>
40	#include "amdgpu.h"
41	#include "amdgpu_trace.h"
42	#include "amdgpu_amdkfd.h"
43	#include "amdgpu_vram_mgr.h"
44	#include "amdgpu_vm.h"
45	#include "amdgpu_dma_buf.h"
46
47	/**
48	* DOC: amdgpu_object
49	*
50	* This defines the interfaces to operate on an &amdgpu_bo buffer object which
51	* represents memory used by driver (VRAM, system memory, etc.). The driver
52	* provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
53	* to create/destroy/set buffer object which are then managed by the kernel TTM
54	* memory manager.
55	* The interfaces are also used internally by kernel clients, including gfx,
56	* uvd, etc. for kernel managed allocations used by the GPU.
57	*
58	*/
59
60	static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
61	{
62	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
63
64	amdgpu_bo_kunmap(bo);
65
66	if (drm_gem_is_imported(obj: &bo->tbo.base))
67	drm_prime_gem_destroy(obj: &bo->tbo.base, sg: bo->tbo.sg);
68	drm_gem_object_release(obj: &bo->tbo.base);
69	amdgpu_bo_unref(bo: &bo->parent);
70	kvfree(addr: bo);
71	}
72
73	static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
74	{
75	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
76	struct amdgpu_bo_user *ubo;
77
78	ubo = to_amdgpu_bo_user(bo);
79	kfree(objp: ubo->metadata);
80	amdgpu_bo_destroy(tbo);
81	}
82
83	/**
84	* amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
85	* @bo: buffer object to be checked
86	*
87	* Uses destroy function associated with the object to determine if this is
88	* an &amdgpu_bo.
89	*
90	* Returns:
91	* true if the object belongs to &amdgpu_bo, false if not.
92	*/
93	bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
94	{
95	if (bo->destroy == &amdgpu_bo_destroy ||
96	bo->destroy == &amdgpu_bo_user_destroy)
97	return true;
98
99	return false;
100	}
101
102	/**
103	* amdgpu_bo_placement_from_domain - set buffer's placement
104	* @abo: &amdgpu_bo buffer object whose placement is to be set
105	* @domain: requested domain
106	*
107	* Sets buffer's placement according to requested domain and the buffer's
108	* flags.
109	*/
110	void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
111	{
112	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: abo->tbo.bdev);
113	struct ttm_placement *placement = &abo->placement;
114	struct ttm_place *places = abo->placements;
115	u64 flags = abo->flags;
116	u32 c = 0;
117
118	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
119	unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
120	int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
121
122	if (adev->gmc.mem_partitions && mem_id >= 0) {
123	places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn;
124	/*
125	* memory partition range lpfn is inclusive start + size - 1
126	* TTM place lpfn is exclusive start + size
127	*/
128	places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1;
129	} else {
130	places[c].fpfn = 0;
131	places[c].lpfn = 0;
132	}
133	places[c].mem_type = TTM_PL_VRAM;
134	places[c].flags = 0;
135
136	if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
137	places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn);
138	else
139	places[c].flags |= TTM_PL_FLAG_TOPDOWN;
140
141	if (abo->tbo.type == ttm_bo_type_kernel &&
142	flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
143	places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
144
145	c++;
146	}
147
148	if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) {
149	places[c].fpfn = 0;
150	places[c].lpfn = 0;
151	places[c].mem_type = AMDGPU_PL_DOORBELL;
152	places[c].flags = 0;
153	c++;
154	}
155
156	if (domain & AMDGPU_GEM_DOMAIN_MMIO_REMAP) {
157	places[c].fpfn = 0;
158	places[c].lpfn = 0;
159	places[c].mem_type = AMDGPU_PL_MMIO_REMAP;
160	places[c].flags = 0;
161	c++;
162	}
163
164	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
165	places[c].fpfn = 0;
166	places[c].lpfn = 0;
167	places[c].mem_type =
168	abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
169	AMDGPU_PL_PREEMPT : TTM_PL_TT;
170	places[c].flags = 0;
171	/*
172	* When GTT is just an alternative to VRAM make sure that we
173	* only use it as fallback and still try to fill up VRAM first.
174	*/
175	if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) &&
176	domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)
177	places[c].flags |= TTM_PL_FLAG_FALLBACK;
178	c++;
179	}
180
181	if (domain & AMDGPU_GEM_DOMAIN_CPU) {
182	places[c].fpfn = 0;
183	places[c].lpfn = 0;
184	places[c].mem_type = TTM_PL_SYSTEM;
185	places[c].flags = 0;
186	c++;
187	}
188
189	if (domain & AMDGPU_GEM_DOMAIN_GDS) {
190	places[c].fpfn = 0;
191	places[c].lpfn = 0;
192	places[c].mem_type = AMDGPU_PL_GDS;
193	places[c].flags = 0;
194	c++;
195	}
196
197	if (domain & AMDGPU_GEM_DOMAIN_GWS) {
198	places[c].fpfn = 0;
199	places[c].lpfn = 0;
200	places[c].mem_type = AMDGPU_PL_GWS;
201	places[c].flags = 0;
202	c++;
203	}
204
205	if (domain & AMDGPU_GEM_DOMAIN_OA) {
206	places[c].fpfn = 0;
207	places[c].lpfn = 0;
208	places[c].mem_type = AMDGPU_PL_OA;
209	places[c].flags = 0;
210	c++;
211	}
212
213	if (!c) {
214	places[c].fpfn = 0;
215	places[c].lpfn = 0;
216	places[c].mem_type = TTM_PL_SYSTEM;
217	places[c].flags = 0;
218	c++;
219	}
220
221	BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS);
222
223	placement->num_placement = c;
224	placement->placement = places;
225	}
226
227	/**
228	* amdgpu_bo_create_reserved - create reserved BO for kernel use
229	*
230	* @adev: amdgpu device object
231	* @size: size for the new BO
232	* @align: alignment for the new BO
233	* @domain: where to place it
234	* @bo_ptr: used to initialize BOs in structures
235	* @gpu_addr: GPU addr of the pinned BO
236	* @cpu_addr: optional CPU address mapping
237	*
238	* Allocates and pins a BO for kernel internal use, and returns it still
239	* reserved.
240	*
241	* Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
242	*
243	* Returns:
244	* 0 on success, negative error code otherwise.
245	*/
246	int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
247	unsigned long size, int align,
248	u32 domain, struct amdgpu_bo **bo_ptr,
249	u64 *gpu_addr, void **cpu_addr)
250	{
251	struct amdgpu_bo_param bp;
252	bool free = false;
253	int r;
254
255	if (!size) {
256	amdgpu_bo_unref(bo: bo_ptr);
257	return 0;
258	}
259
260	memset(&bp, 0, sizeof(bp));
261	bp.size = size;
262	bp.byte_align = align;
263	bp.domain = domain;
264	bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
265	: AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
266	bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
267	bp.type = ttm_bo_type_kernel;
268	bp.resv = NULL;
269	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
270
271	if (!*bo_ptr) {
272	r = amdgpu_bo_create(adev, bp: &bp, bo_ptr);
273	if (r) {
274	dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
275	r);
276	return r;
277	}
278	free = true;
279	}
280
281	r = amdgpu_bo_reserve(bo: *bo_ptr, no_intr: false);
282	if (r) {
283	dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
284	goto error_free;
285	}
286
287	r = amdgpu_bo_pin(bo: *bo_ptr, domain);
288	if (r) {
289	dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
290	goto error_unreserve;
291	}
292
293	r = amdgpu_ttm_alloc_gart(bo: &(*bo_ptr)->tbo);
294	if (r) {
295	dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
296	goto error_unpin;
297	}
298
299	if (gpu_addr)
300	*gpu_addr = amdgpu_bo_gpu_offset(bo: *bo_ptr);
301
302	if (cpu_addr) {
303	r = amdgpu_bo_kmap(bo: *bo_ptr, ptr: cpu_addr);
304	if (r) {
305	dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
306	goto error_unpin;
307	}
308	}
309
310	return 0;
311
312	error_unpin:
313	amdgpu_bo_unpin(bo: *bo_ptr);
314	error_unreserve:
315	amdgpu_bo_unreserve(bo: *bo_ptr);
316
317	error_free:
318	if (free)
319	amdgpu_bo_unref(bo: bo_ptr);
320
321	return r;
322	}
323
324	/**
325	* amdgpu_bo_create_kernel - create BO for kernel use
326	*
327	* @adev: amdgpu device object
328	* @size: size for the new BO
329	* @align: alignment for the new BO
330	* @domain: where to place it
331	* @bo_ptr: used to initialize BOs in structures
332	* @gpu_addr: GPU addr of the pinned BO
333	* @cpu_addr: optional CPU address mapping
334	*
335	* Allocates and pins a BO for kernel internal use.
336	*
337	* This function is exported to allow the V4L2 isp device
338	* external to drm device to create and access the kernel BO.
339	*
340	* Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
341	*
342	* Returns:
343	* 0 on success, negative error code otherwise.
344	*/
345	int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
346	unsigned long size, int align,
347	u32 domain, struct amdgpu_bo **bo_ptr,
348	u64 *gpu_addr, void **cpu_addr)
349	{
350	int r;
351
352	r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
353	gpu_addr, cpu_addr);
354
355	if (r)
356	return r;
357
358	if (*bo_ptr)
359	amdgpu_bo_unreserve(bo: *bo_ptr);
360
361	return 0;
362	}
363
364	/**
365	* amdgpu_bo_create_isp_user - create user BO for isp
366	*
367	* @adev: amdgpu device object
368	* @dma_buf: DMABUF handle for isp buffer
369	* @domain: where to place it
370	* @bo: used to initialize BOs in structures
371	* @gpu_addr: GPU addr of the pinned BO
372	*
373	* Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does
374	* GART alloc to generate gpu_addr for BO to make it accessible through the
375	* GART aperture for ISP HW.
376	*
377	* This function is exported to allow the V4L2 isp device external to drm device
378	* to create and access the isp user BO.
379	*
380	* Returns:
381	* 0 on success, negative error code otherwise.
382	*/
383	int amdgpu_bo_create_isp_user(struct amdgpu_device *adev,
384	struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo,
385	u64 *gpu_addr)
386
387	{
388	struct drm_gem_object *gem_obj;
389	int r;
390
391	gem_obj = amdgpu_gem_prime_import(dev: &adev->ddev, dma_buf);
392	*bo = gem_to_amdgpu_bo(gem_obj);
393	if (!(*bo)) {
394	dev_err(adev->dev, "failed to get valid isp user bo\n");
395	return -EINVAL;
396	}
397
398	r = amdgpu_bo_reserve(bo: *bo, no_intr: false);
399	if (r) {
400	dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r);
401	return r;
402	}
403
404	r = amdgpu_bo_pin(bo: *bo, domain);
405	if (r) {
406	dev_err(adev->dev, "(%d) isp user bo pin failed\n", r);
407	goto error_unreserve;
408	}
409
410	r = amdgpu_ttm_alloc_gart(bo: &(*bo)->tbo);
411	if (r) {
412	dev_err(adev->dev, "%p bind failed\n", *bo);
413	goto error_unpin;
414	}
415
416	if (!WARN_ON(!gpu_addr))
417	*gpu_addr = amdgpu_bo_gpu_offset(bo: *bo);
418
419	amdgpu_bo_unreserve(bo: *bo);
420
421	return 0;
422
423	error_unpin:
424	amdgpu_bo_unpin(bo: *bo);
425	error_unreserve:
426	amdgpu_bo_unreserve(bo: *bo);
427	amdgpu_bo_unref(bo);
428
429	return r;
430	}
431
432	/**
433	* amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
434	*
435	* @adev: amdgpu device object
436	* @offset: offset of the BO
437	* @size: size of the BO
438	* @bo_ptr: used to initialize BOs in structures
439	* @cpu_addr: optional CPU address mapping
440	*
441	* Creates a kernel BO at a specific offset in VRAM.
442	*
443	* Returns:
444	* 0 on success, negative error code otherwise.
445	*/
446	int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
447	uint64_t offset, uint64_t size,
448	struct amdgpu_bo **bo_ptr, void **cpu_addr)
449	{
450	struct ttm_operation_ctx ctx = { false, false };
451	unsigned int i;
452	int r;
453
454	offset &= PAGE_MASK;
455	size = ALIGN(size, PAGE_SIZE);
456
457	r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE,
458	AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL,
459	cpu_addr);
460	if (r)
461	return r;
462
463	if ((*bo_ptr) == NULL)
464	return 0;
465
466	/*
467	* Remove the original mem node and create a new one at the request
468	* position.
469	*/
470	if (cpu_addr)
471	amdgpu_bo_kunmap(bo: *bo_ptr);
472
473	ttm_resource_free(bo: &(*bo_ptr)->tbo, res: &(*bo_ptr)->tbo.resource);
474
475	for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
476	(*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
477	(*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
478	}
479	r = ttm_bo_mem_space(bo: &(*bo_ptr)->tbo, placement: &(*bo_ptr)->placement,
480	mem: &(*bo_ptr)->tbo.resource, ctx: &ctx);
481	if (r)
482	goto error;
483
484	if (cpu_addr) {
485	r = amdgpu_bo_kmap(bo: *bo_ptr, ptr: cpu_addr);
486	if (r)
487	goto error;
488	}
489
490	amdgpu_bo_unreserve(bo: *bo_ptr);
491	return 0;
492
493	error:
494	amdgpu_bo_unreserve(bo: *bo_ptr);
495	amdgpu_bo_unref(bo: bo_ptr);
496	return r;
497	}
498
499	/**
500	* amdgpu_bo_free_kernel - free BO for kernel use
501	*
502	* @bo: amdgpu BO to free
503	* @gpu_addr: pointer to where the BO's GPU memory space address was stored
504	* @cpu_addr: pointer to where the BO's CPU memory space address was stored
505	*
506	* unmaps and unpin a BO for kernel internal use.
507	*
508	* This function is exported to allow the V4L2 isp device
509	* external to drm device to free the kernel BO.
510	*/
511	void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
512	void **cpu_addr)
513	{
514	if (*bo == NULL)
515	return;
516
517	WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend);
518
519	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
520	if (cpu_addr)
521	amdgpu_bo_kunmap(bo: *bo);
522
523	amdgpu_bo_unpin(bo: *bo);
524	amdgpu_bo_unreserve(bo: *bo);
525	}
526	amdgpu_bo_unref(bo);
527
528	if (gpu_addr)
529	*gpu_addr = 0;
530
531	if (cpu_addr)
532	*cpu_addr = NULL;
533	}
534
535	/**
536	* amdgpu_bo_free_isp_user - free BO for isp use
537	*
538	* @bo: amdgpu isp user BO to free
539	*
540	* unpin and unref BO for isp internal use.
541	*
542	* This function is exported to allow the V4L2 isp device
543	* external to drm device to free the isp user BO.
544	*/
545	void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo)
546	{
547	if (bo == NULL)
548	return;
549
550	if (amdgpu_bo_reserve(bo, no_intr: true) == 0) {
551	amdgpu_bo_unpin(bo);
552	amdgpu_bo_unreserve(bo);
553	}
554	amdgpu_bo_unref(bo: &bo);
555	}
556
557	/* Validate bo size is bit bigger than the request domain */
558	static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
559	unsigned long size, u32 domain)
560	{
561	struct ttm_resource_manager *man = NULL;
562
563	/*
564	* If GTT is part of requested domains the check must succeed to
565	* allow fall back to GTT.
566	*/
567	if (domain & AMDGPU_GEM_DOMAIN_GTT)
568	man = ttm_manager_type(bdev: &adev->mman.bdev, TTM_PL_TT);
569	else if (domain & AMDGPU_GEM_DOMAIN_VRAM)
570	man = ttm_manager_type(bdev: &adev->mman.bdev, TTM_PL_VRAM);
571	else
572	return true;
573
574	if (!man) {
575	if (domain & AMDGPU_GEM_DOMAIN_GTT)
576	WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized");
577	return false;
578	}
579
580	/* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */
581	if (size < man->size)
582	return true;
583
584	DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size);
585	return false;
586	}
587
588	bool amdgpu_bo_support_uswc(u64 bo_flags)
589	{
590
591	#ifdef CONFIG_X86_32
592	/* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
593	* See https://bugs.freedesktop.org/show_bug.cgi?id=84627
594	*/
595	return false;
596	#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
597	/* Don't try to enable write-combining when it can't work, or things
598	* may be slow
599	* See https://bugs.freedesktop.org/show_bug.cgi?id=88758
600	*/
601
602	#ifndef CONFIG_COMPILE_TEST
603	#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
604	thanks to write-combining
605	#endif
606
607	if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
608	DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
609	"better performance thanks to write-combining\n");
610	return false;
611	#else
612	/* For architectures that don't support WC memory,
613	* mask out the WC flag from the BO
614	*/
615	if (!drm_arch_can_wc_memory())
616	return false;
617
618	return true;
619	#endif
620	}
621
622	/**
623	* amdgpu_bo_create - create an &amdgpu_bo buffer object
624	* @adev: amdgpu device object
625	* @bp: parameters to be used for the buffer object
626	* @bo_ptr: pointer to the buffer object pointer
627	*
628	* Creates an &amdgpu_bo buffer object.
629	*
630	* Returns:
631	* 0 for success or a negative error code on failure.
632	*/
633	int amdgpu_bo_create(struct amdgpu_device *adev,
634	struct amdgpu_bo_param *bp,
635	struct amdgpu_bo **bo_ptr)
636	{
637	struct ttm_operation_ctx ctx = {
638	.interruptible = (bp->type != ttm_bo_type_kernel),
639	.no_wait_gpu = bp->no_wait_gpu,
640	/* We opt to avoid OOM on system pages allocations */
641	.gfp_retry_mayfail = true,
642	.allow_res_evict = bp->type != ttm_bo_type_kernel,
643	.resv = bp->resv
644	};
645	struct amdgpu_bo *bo;
646	unsigned long page_align, size = bp->size;
647	int r;
648
649	/* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
650	if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
651	/* GWS and OA don't need any alignment. */
652	page_align = bp->byte_align;
653	size <<= PAGE_SHIFT;
654
655	} else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) {
656	/* Both size and alignment must be a multiple of 4. */
657	page_align = ALIGN(bp->byte_align, 4);
658	size = ALIGN(size, 4) << PAGE_SHIFT;
659	} else {
660	/* Memory should be aligned at least to a page size. */
661	page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
662	size = ALIGN(size, PAGE_SIZE);
663	}
664
665	if (!amdgpu_bo_validate_size(adev, size, domain: bp->domain))
666	return -ENOMEM;
667
668	BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo));
669
670	*bo_ptr = NULL;
671	bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);
672	if (bo == NULL)
673	return -ENOMEM;
674	drm_gem_private_object_init(dev: adev_to_drm(adev), obj: &bo->tbo.base, size);
675	bo->tbo.base.funcs = &amdgpu_gem_object_funcs;
676	bo->vm_bo = NULL;
677	bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
678	bp->domain;
679	bo->allowed_domains = bo->preferred_domains;
680	if (bp->type != ttm_bo_type_kernel &&
681	!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
682	bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
683	bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
684
685	bo->flags = bp->flags;
686
687	if (adev->gmc.mem_partitions)
688	/* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */
689	bo->xcp_id = bp->xcp_id_plus1 - 1;
690	else
691	/* For GPUs without spatial partitioning */
692	bo->xcp_id = 0;
693
694	if (!amdgpu_bo_support_uswc(bo_flags: bo->flags))
695	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
696
697	bo->tbo.bdev = &adev->mman.bdev;
698	if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
699	AMDGPU_GEM_DOMAIN_GDS))
700	amdgpu_bo_placement_from_domain(abo: bo, AMDGPU_GEM_DOMAIN_CPU);
701	else
702	amdgpu_bo_placement_from_domain(abo: bo, domain: bp->domain);
703	if (bp->type == ttm_bo_type_kernel)
704	bo->tbo.priority = 2;
705	else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE))
706	bo->tbo.priority = 1;
707
708	if (!bp->destroy)
709	bp->destroy = &amdgpu_bo_destroy;
710
711	r = ttm_bo_init_reserved(bdev: &adev->mman.bdev, bo: &bo->tbo, type: bp->type,
712	placement: &bo->placement, alignment: page_align, ctx: &ctx, NULL,
713	resv: bp->resv, destroy: bp->destroy);
714	if (unlikely(r != 0))
715	return r;
716
717	if (!amdgpu_gmc_vram_full_visible(gmc: &adev->gmc) &&
718	amdgpu_res_cpu_visible(adev, res: bo->tbo.resource))
719	amdgpu_cs_report_moved_bytes(adev, num_bytes: ctx.bytes_moved,
720	num_vis_bytes: ctx.bytes_moved);
721	else
722	amdgpu_cs_report_moved_bytes(adev, num_bytes: ctx.bytes_moved, num_vis_bytes: 0);
723
724	if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
725	bo->tbo.resource->mem_type == TTM_PL_VRAM) {
726	struct dma_fence *fence;
727
728	r = amdgpu_ttm_clear_buffer(bo, resv: bo->tbo.base.resv, fence: &fence);
729	if (unlikely(r))
730	goto fail_unreserve;
731
732	dma_resv_add_fence(obj: bo->tbo.base.resv, fence,
733	usage: DMA_RESV_USAGE_KERNEL);
734	dma_fence_put(fence);
735	}
736	if (!bp->resv)
737	amdgpu_bo_unreserve(bo);
738	*bo_ptr = bo;
739
740	trace_amdgpu_bo_create(bo);
741
742	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
743	if (bp->type == ttm_bo_type_device)
744	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
745
746	return 0;
747
748	fail_unreserve:
749	if (!bp->resv)
750	dma_resv_unlock(obj: bo->tbo.base.resv);
751	amdgpu_bo_unref(bo: &bo);
752	return r;
753	}
754
755	/**
756	* amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
757	* @adev: amdgpu device object
758	* @bp: parameters to be used for the buffer object
759	* @ubo_ptr: pointer to the buffer object pointer
760	*
761	* Create a BO to be used by user application;
762	*
763	* Returns:
764	* 0 for success or a negative error code on failure.
765	*/
766
767	int amdgpu_bo_create_user(struct amdgpu_device *adev,
768	struct amdgpu_bo_param *bp,
769	struct amdgpu_bo_user **ubo_ptr)
770	{
771	struct amdgpu_bo *bo_ptr;
772	int r;
773
774	bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
775	bp->destroy = &amdgpu_bo_user_destroy;
776	r = amdgpu_bo_create(adev, bp, bo_ptr: &bo_ptr);
777	if (r)
778	return r;
779
780	*ubo_ptr = to_amdgpu_bo_user(bo_ptr);
781	return r;
782	}
783
784	/**
785	* amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
786	* @adev: amdgpu device object
787	* @bp: parameters to be used for the buffer object
788	* @vmbo_ptr: pointer to the buffer object pointer
789	*
790	* Create a BO to be for GPUVM.
791	*
792	* Returns:
793	* 0 for success or a negative error code on failure.
794	*/
795
796	int amdgpu_bo_create_vm(struct amdgpu_device *adev,
797	struct amdgpu_bo_param *bp,
798	struct amdgpu_bo_vm **vmbo_ptr)
799	{
800	struct amdgpu_bo *bo_ptr;
801	int r;
802
803	/* bo_ptr_size will be determined by the caller and it depends on
804	* num of amdgpu_vm_pt entries.
805	*/
806	BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
807	r = amdgpu_bo_create(adev, bp, bo_ptr: &bo_ptr);
808	if (r)
809	return r;
810
811	*vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
812	return r;
813	}
814
815	/**
816	* amdgpu_bo_kmap - map an &amdgpu_bo buffer object
817	* @bo: &amdgpu_bo buffer object to be mapped
818	* @ptr: kernel virtual address to be returned
819	*
820	* Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
821	* amdgpu_bo_kptr() to get the kernel virtual address.
822	*
823	* Returns:
824	* 0 for success or a negative error code on failure.
825	*/
826	int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
827	{
828	void *kptr;
829	long r;
830
831	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
832	return -EPERM;
833
834	r = dma_resv_wait_timeout(obj: bo->tbo.base.resv, usage: DMA_RESV_USAGE_KERNEL,
835	intr: false, MAX_SCHEDULE_TIMEOUT);
836	if (r < 0)
837	return r;
838
839	kptr = amdgpu_bo_kptr(bo);
840	if (kptr) {
841	if (ptr)
842	*ptr = kptr;
843	return 0;
844	}
845
846	r = ttm_bo_kmap(bo: &bo->tbo, start_page: 0, PFN_UP(bo->tbo.base.size), map: &bo->kmap);
847	if (r)
848	return r;
849
850	if (ptr)
851	*ptr = amdgpu_bo_kptr(bo);
852
853	return 0;
854	}
855
856	/**
857	* amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
858	* @bo: &amdgpu_bo buffer object
859	*
860	* Calls ttm_kmap_obj_virtual() to get the kernel virtual address
861	*
862	* Returns:
863	* the virtual address of a buffer object area.
864	*/
865	void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
866	{
867	bool is_iomem;
868
869	return ttm_kmap_obj_virtual(map: &bo->kmap, is_iomem: &is_iomem);
870	}
871
872	/**
873	* amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
874	* @bo: &amdgpu_bo buffer object to be unmapped
875	*
876	* Unmaps a kernel map set up by amdgpu_bo_kmap().
877	*/
878	void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
879	{
880	if (bo->kmap.bo)
881	ttm_bo_kunmap(map: &bo->kmap);
882	}
883
884	/**
885	* amdgpu_bo_ref - reference an &amdgpu_bo buffer object
886	* @bo: &amdgpu_bo buffer object
887	*
888	* References the contained &ttm_buffer_object.
889	*
890	* Returns:
891	* a refcounted pointer to the &amdgpu_bo buffer object.
892	*/
893	struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
894	{
895	if (bo == NULL)
896	return NULL;
897
898	drm_gem_object_get(obj: &bo->tbo.base);
899	return bo;
900	}
901
902	/**
903	* amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
904	* @bo: &amdgpu_bo buffer object
905	*
906	* Unreferences the contained &ttm_buffer_object and clear the pointer
907	*/
908	void amdgpu_bo_unref(struct amdgpu_bo **bo)
909	{
910	if ((*bo) == NULL)
911	return;
912
913	drm_gem_object_put(obj: &(*bo)->tbo.base);
914	*bo = NULL;
915	}
916
917	/**
918	* amdgpu_bo_pin - pin an &amdgpu_bo buffer object
919	* @bo: &amdgpu_bo buffer object to be pinned
920	* @domain: domain to be pinned to
921	*
922	* Pins the buffer object according to requested domain. If the memory is
923	* unbound gart memory, binds the pages into gart table. Adjusts pin_count and
924	* pin_size accordingly.
925	*
926	* Pinning means to lock pages in memory along with keeping them at a fixed
927	* offset. It is required when a buffer can not be moved, for example, when
928	* a display buffer is being scanned out.
929	*
930	* Returns:
931	* 0 for success or a negative error code on failure.
932	*/
933	int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
934	{
935	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
936	struct ttm_operation_ctx ctx = { false, false };
937	int r, i;
938
939	if (amdgpu_ttm_tt_get_usermm(ttm: bo->tbo.ttm))
940	return -EPERM;
941
942	/* Check domain to be pinned to against preferred domains */
943	if (bo->preferred_domains & domain)
944	domain = bo->preferred_domains & domain;
945
946	/* A shared bo cannot be migrated to VRAM */
947	if (drm_gem_is_imported(obj: &bo->tbo.base)) {
948	if (domain & AMDGPU_GEM_DOMAIN_GTT)
949	domain = AMDGPU_GEM_DOMAIN_GTT;
950	else
951	return -EINVAL;
952	}
953
954	if (bo->tbo.pin_count) {
955	uint32_t mem_type = bo->tbo.resource->mem_type;
956	uint32_t mem_flags = bo->tbo.resource->placement;
957
958	if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
959	return -EINVAL;
960
961	if ((mem_type == TTM_PL_VRAM) &&
962	(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&
963	!(mem_flags & TTM_PL_FLAG_CONTIGUOUS))
964	return -EINVAL;
965
966	ttm_bo_pin(bo: &bo->tbo);
967	return 0;
968	}
969
970	/* This assumes only APU display buffers are pinned with (VRAM|GTT).
971	* See function amdgpu_display_supported_domains()
972	*/
973	domain = amdgpu_bo_get_preferred_domain(adev, domain);
974
975	if (drm_gem_is_imported(obj: &bo->tbo.base))
976	dma_buf_pin(attach: bo->tbo.base.import_attach);
977
978	/* force to pin into visible video ram */
979	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
980	bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
981	amdgpu_bo_placement_from_domain(abo: bo, domain);
982	for (i = 0; i < bo->placement.num_placement; i++) {
983	if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS &&
984	bo->placements[i].mem_type == TTM_PL_VRAM)
985	bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;
986	}
987
988	r = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
989	if (unlikely(r)) {
990	dev_err(adev->dev, "%p pin failed\n", bo);
991	goto error;
992	}
993
994	ttm_bo_pin(bo: &bo->tbo);
995
996	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
997	atomic64_add(i: amdgpu_bo_size(bo), v: &adev->vram_pin_size);
998	atomic64_add(i: amdgpu_vram_mgr_bo_visible_size(bo),
999	v: &adev->visible_pin_size);
1000	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1001	atomic64_add(i: amdgpu_bo_size(bo), v: &adev->gart_pin_size);
1002	}
1003
1004	error:
1005	return r;
1006	}
1007
1008	/**
1009	* amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
1010	* @bo: &amdgpu_bo buffer object to be unpinned
1011	*
1012	* Decreases the pin_count, and clears the flags if pin_count reaches 0.
1013	* Changes placement and pin size accordingly.
1014	*
1015	* Returns:
1016	* 0 for success or a negative error code on failure.
1017	*/
1018	void amdgpu_bo_unpin(struct amdgpu_bo *bo)
1019	{
1020	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
1021
1022	ttm_bo_unpin(bo: &bo->tbo);
1023	if (bo->tbo.pin_count)
1024	return;
1025
1026	if (drm_gem_is_imported(obj: &bo->tbo.base))
1027	dma_buf_unpin(attach: bo->tbo.base.import_attach);
1028
1029	if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1030	atomic64_sub(i: amdgpu_bo_size(bo), v: &adev->vram_pin_size);
1031	atomic64_sub(i: amdgpu_vram_mgr_bo_visible_size(bo),
1032	v: &adev->visible_pin_size);
1033	} else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1034	atomic64_sub(i: amdgpu_bo_size(bo), v: &adev->gart_pin_size);
1035	}
1036
1037	}
1038
1039	static const char * const amdgpu_vram_names[] = {
1040	"UNKNOWN",
1041	"GDDR1",
1042	"DDR2",
1043	"GDDR3",
1044	"GDDR4",
1045	"GDDR5",
1046	"HBM",
1047	"DDR3",
1048	"DDR4",
1049	"GDDR6",
1050	"DDR5",
1051	"LPDDR4",
1052	"LPDDR5",
1053	"HBM3E"
1054	};
1055
1056	/**
1057	* amdgpu_bo_init - initialize memory manager
1058	* @adev: amdgpu device object
1059	*
1060	* Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1061	*
1062	* Returns:
1063	* 0 for success or a negative error code on failure.
1064	*/
1065	int amdgpu_bo_init(struct amdgpu_device *adev)
1066	{
1067	/* On A+A platform, VRAM can be mapped as WB */
1068	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1069	/* reserve PAT memory space to WC for VRAM */
1070	int r = arch_io_reserve_memtype_wc(start: adev->gmc.aper_base,
1071	size: adev->gmc.aper_size);
1072
1073	if (r) {
1074	DRM_ERROR("Unable to set WC memtype for the aperture base\n");
1075	return r;
1076	}
1077
1078	/* Add an MTRR for the VRAM */
1079	adev->gmc.vram_mtrr = arch_phys_wc_add(base: adev->gmc.aper_base,
1080	size: adev->gmc.aper_size);
1081	}
1082
1083	DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
1084	adev->gmc.mc_vram_size >> 20,
1085	(unsigned long long)adev->gmc.aper_size >> 20);
1086	DRM_INFO("RAM width %dbits %s\n",
1087	adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
1088	return amdgpu_ttm_init(adev);
1089	}
1090
1091	/**
1092	* amdgpu_bo_fini - tear down memory manager
1093	* @adev: amdgpu device object
1094	*
1095	* Reverses amdgpu_bo_init() to tear down memory manager.
1096	*/
1097	void amdgpu_bo_fini(struct amdgpu_device *adev)
1098	{
1099	int idx;
1100
1101	amdgpu_ttm_fini(adev);
1102
1103	if (drm_dev_enter(dev: adev_to_drm(adev), idx: &idx)) {
1104	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1105	arch_phys_wc_del(handle: adev->gmc.vram_mtrr);
1106	arch_io_free_memtype_wc(start: adev->gmc.aper_base, size: adev->gmc.aper_size);
1107	}
1108	drm_dev_exit(idx);
1109	}
1110	}
1111
1112	/**
1113	* amdgpu_bo_set_tiling_flags - set tiling flags
1114	* @bo: &amdgpu_bo buffer object
1115	* @tiling_flags: new flags
1116	*
1117	* Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1118	* kernel driver to set the tiling flags on a buffer.
1119	*
1120	* Returns:
1121	* 0 for success or a negative error code on failure.
1122	*/
1123	int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1124	{
1125	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
1126	struct amdgpu_bo_user *ubo;
1127
1128	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1129	if (adev->family <= AMDGPU_FAMILY_CZ &&
1130	AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1131	return -EINVAL;
1132
1133	ubo = to_amdgpu_bo_user(bo);
1134	ubo->tiling_flags = tiling_flags;
1135	return 0;
1136	}
1137
1138	/**
1139	* amdgpu_bo_get_tiling_flags - get tiling flags
1140	* @bo: &amdgpu_bo buffer object
1141	* @tiling_flags: returned flags
1142	*
1143	* Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1144	* set the tiling flags on a buffer.
1145	*/
1146	void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1147	{
1148	struct amdgpu_bo_user *ubo;
1149
1150	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1151	dma_resv_assert_held(bo->tbo.base.resv);
1152	ubo = to_amdgpu_bo_user(bo);
1153
1154	if (tiling_flags)
1155	*tiling_flags = ubo->tiling_flags;
1156	}
1157
1158	/**
1159	* amdgpu_bo_set_metadata - set metadata
1160	* @bo: &amdgpu_bo buffer object
1161	* @metadata: new metadata
1162	* @metadata_size: size of the new metadata
1163	* @flags: flags of the new metadata
1164	*
1165	* Sets buffer object's metadata, its size and flags.
1166	* Used via GEM ioctl.
1167	*
1168	* Returns:
1169	* 0 for success or a negative error code on failure.
1170	*/
1171	int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata,
1172	u32 metadata_size, uint64_t flags)
1173	{
1174	struct amdgpu_bo_user *ubo;
1175	void *buffer;
1176
1177	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1178	ubo = to_amdgpu_bo_user(bo);
1179	if (!metadata_size) {
1180	if (ubo->metadata_size) {
1181	kfree(objp: ubo->metadata);
1182	ubo->metadata = NULL;
1183	ubo->metadata_size = 0;
1184	}
1185	return 0;
1186	}
1187
1188	if (metadata == NULL)
1189	return -EINVAL;
1190
1191	buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1192	if (buffer == NULL)
1193	return -ENOMEM;
1194
1195	kfree(objp: ubo->metadata);
1196	ubo->metadata_flags = flags;
1197	ubo->metadata = buffer;
1198	ubo->metadata_size = metadata_size;
1199
1200	return 0;
1201	}
1202
1203	/**
1204	* amdgpu_bo_get_metadata - get metadata
1205	* @bo: &amdgpu_bo buffer object
1206	* @buffer: returned metadata
1207	* @buffer_size: size of the buffer
1208	* @metadata_size: size of the returned metadata
1209	* @flags: flags of the returned metadata
1210	*
1211	* Gets buffer object's metadata, its size and flags. buffer_size shall not be
1212	* less than metadata_size.
1213	* Used via GEM ioctl.
1214	*
1215	* Returns:
1216	* 0 for success or a negative error code on failure.
1217	*/
1218	int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1219	size_t buffer_size, uint32_t *metadata_size,
1220	uint64_t *flags)
1221	{
1222	struct amdgpu_bo_user *ubo;
1223
1224	if (!buffer && !metadata_size)
1225	return -EINVAL;
1226
1227	BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1228	ubo = to_amdgpu_bo_user(bo);
1229	if (metadata_size)
1230	*metadata_size = ubo->metadata_size;
1231
1232	if (buffer) {
1233	if (buffer_size < ubo->metadata_size)
1234	return -EINVAL;
1235
1236	if (ubo->metadata_size)
1237	memcpy(buffer, ubo->metadata, ubo->metadata_size);
1238	}
1239
1240	if (flags)
1241	*flags = ubo->metadata_flags;
1242
1243	return 0;
1244	}
1245
1246	/**
1247	* amdgpu_bo_move_notify - notification about a memory move
1248	* @bo: pointer to a buffer object
1249	* @evict: if this move is evicting the buffer from the graphics address space
1250	* @new_mem: new resource for backing the BO
1251	*
1252	* Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1253	* bookkeeping.
1254	* TTM driver callback which is called when ttm moves a buffer.
1255	*/
1256	void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1257	bool evict,
1258	struct ttm_resource *new_mem)
1259	{
1260	struct ttm_resource *old_mem = bo->resource;
1261	struct amdgpu_bo *abo;
1262
1263	if (!amdgpu_bo_is_amdgpu_bo(bo))
1264	return;
1265
1266	abo = ttm_to_amdgpu_bo(tbo: bo);
1267	amdgpu_vm_bo_move(bo: abo, new_mem, evicted: evict);
1268
1269	amdgpu_bo_kunmap(bo: abo);
1270
1271	if (abo->tbo.base.dma_buf && !drm_gem_is_imported(obj: &abo->tbo.base) &&
1272	old_mem && old_mem->mem_type != TTM_PL_SYSTEM)
1273	dma_buf_move_notify(dma_buf: abo->tbo.base.dma_buf);
1274
1275	/* move_notify is called before move happens */
1276	trace_amdgpu_bo_move(bo: abo, new_placement: new_mem ? new_mem->mem_type : -1,
1277	old_placement: old_mem ? old_mem->mem_type : -1);
1278	}
1279
1280	/**
1281	* amdgpu_bo_release_notify - notification about a BO being released
1282	* @bo: pointer to a buffer object
1283	*
1284	* Wipes VRAM buffers whose contents should not be leaked before the
1285	* memory is released.
1286	*/
1287	void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
1288	{
1289	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->bdev);
1290	struct dma_fence *fence = NULL;
1291	struct amdgpu_bo *abo;
1292	int r;
1293
1294	if (!amdgpu_bo_is_amdgpu_bo(bo))
1295	return;
1296
1297	abo = ttm_to_amdgpu_bo(tbo: bo);
1298
1299	WARN_ON(abo->vm_bo);
1300
1301	if (abo->kfd_bo)
1302	amdgpu_amdkfd_release_notify(bo: abo);
1303
1304	/*
1305	* We lock the private dma_resv object here and since the BO is about to
1306	* be released nobody else should have a pointer to it.
1307	* So when this locking here fails something is wrong with the reference
1308	* counting.
1309	*/
1310	if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv)))
1311	return;
1312
1313	amdgpu_amdkfd_remove_all_eviction_fences(bo: abo);
1314
1315	if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM ||
1316	!(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) ||
1317	adev->in_suspend || drm_dev_is_unplugged(dev: adev_to_drm(adev)))
1318	goto out;
1319
1320	r = dma_resv_reserve_fences(obj: &bo->base._resv, num_fences: 1);
1321	if (r)
1322	goto out;
1323
1324	r = amdgpu_fill_buffer(bo: abo, src_data: 0, resv: &bo->base._resv, fence: &fence, delayed: true,
1325	AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
1326	if (WARN_ON(r))
1327	goto out;
1328
1329	amdgpu_vram_mgr_set_cleared(res: bo->resource);
1330	dma_resv_add_fence(obj: &bo->base._resv, fence, usage: DMA_RESV_USAGE_KERNEL);
1331	dma_fence_put(fence);
1332
1333	out:
1334	dma_resv_unlock(obj: &bo->base._resv);
1335	}
1336
1337	/**
1338	* amdgpu_bo_fault_reserve_notify - notification about a memory fault
1339	* @bo: pointer to a buffer object
1340	*
1341	* Notifies the driver we are taking a fault on this BO and have reserved it,
1342	* also performs bookkeeping.
1343	* TTM driver callback for dealing with vm faults.
1344	*
1345	* Returns:
1346	* 0 for success or a negative error code on failure.
1347	*/
1348	vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1349	{
1350	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->bdev);
1351	struct ttm_operation_ctx ctx = { false, false };
1352	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo: bo);
1353	int r;
1354
1355	/* Remember that this BO was accessed by the CPU */
1356	abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1357
1358	if (amdgpu_res_cpu_visible(adev, res: bo->resource))
1359	return 0;
1360
1361	/* Can't move a pinned BO to visible VRAM */
1362	if (abo->tbo.pin_count > 0)
1363	return VM_FAULT_SIGBUS;
1364
1365	/* hurrah the memory is not visible ! */
1366	atomic64_inc(v: &adev->num_vram_cpu_page_faults);
1367	amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1368	AMDGPU_GEM_DOMAIN_GTT);
1369
1370	/* Avoid costly evictions; only set GTT as a busy placement */
1371	abo->placements[0].flags |= TTM_PL_FLAG_DESIRED;
1372
1373	r = ttm_bo_validate(bo, placement: &abo->placement, ctx: &ctx);
1374	if (unlikely(r == -EBUSY || r == -ERESTARTSYS))
1375	return VM_FAULT_NOPAGE;
1376	else if (unlikely(r))
1377	return VM_FAULT_SIGBUS;
1378
1379	/* this should never happen */
1380	if (bo->resource->mem_type == TTM_PL_VRAM &&
1381	!amdgpu_res_cpu_visible(adev, res: bo->resource))
1382	return VM_FAULT_SIGBUS;
1383
1384	ttm_bo_move_to_lru_tail_unlocked(bo);
1385	return 0;
1386	}
1387
1388	/**
1389	* amdgpu_bo_fence - add fence to buffer object
1390	*
1391	* @bo: buffer object in question
1392	* @fence: fence to add
1393	* @shared: true if fence should be added shared
1394	*
1395	*/
1396	void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1397	bool shared)
1398	{
1399	struct dma_resv *resv = bo->tbo.base.resv;
1400	int r;
1401
1402	r = dma_resv_reserve_fences(obj: resv, num_fences: 1);
1403	if (r) {
1404	/* As last resort on OOM we block for the fence */
1405	dma_fence_wait(fence, intr: false);
1406	return;
1407	}
1408
1409	dma_resv_add_fence(obj: resv, fence, usage: shared ? DMA_RESV_USAGE_READ :
1410	DMA_RESV_USAGE_WRITE);
1411	}
1412
1413	/**
1414	* amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1415	*
1416	* @adev: amdgpu device pointer
1417	* @resv: reservation object to sync to
1418	* @sync_mode: synchronization mode
1419	* @owner: fence owner
1420	* @intr: Whether the wait is interruptible
1421	*
1422	* Extract the fences from the reservation object and waits for them to finish.
1423	*
1424	* Returns:
1425	* 0 on success, errno otherwise.
1426	*/
1427	int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
1428	enum amdgpu_sync_mode sync_mode, void *owner,
1429	bool intr)
1430	{
1431	struct amdgpu_sync sync;
1432	int r;
1433
1434	amdgpu_sync_create(sync: &sync);
1435	amdgpu_sync_resv(adev, sync: &sync, resv, mode: sync_mode, owner);
1436	r = amdgpu_sync_wait(sync: &sync, intr);
1437	amdgpu_sync_free(sync: &sync);
1438	return r;
1439	}
1440
1441	/**
1442	* amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1443	* @bo: buffer object to wait for
1444	* @owner: fence owner
1445	* @intr: Whether the wait is interruptible
1446	*
1447	* Wrapper to wait for fences in a BO.
1448	* Returns:
1449	* 0 on success, errno otherwise.
1450	*/
1451	int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)
1452	{
1453	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
1454
1455	return amdgpu_bo_sync_wait_resv(adev, resv: bo->tbo.base.resv,
1456	sync_mode: AMDGPU_SYNC_NE_OWNER, owner, intr);
1457	}
1458
1459	/**
1460	* amdgpu_bo_gpu_offset - return GPU offset of bo
1461	* @bo: amdgpu object for which we query the offset
1462	*
1463	* Note: object should either be pinned or reserved when calling this
1464	* function, it might be useful to add check for this for debugging.
1465	*
1466	* Returns:
1467	* current GPU offset of the object.
1468	*/
1469	u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1470	{
1471	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);
1472	WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&
1473	!bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);
1474	WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);
1475	WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&
1476	!(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1477
1478	return amdgpu_bo_gpu_offset_no_check(bo);
1479	}
1480
1481	/**
1482	* amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo
1483	* @bo: amdgpu VRAM buffer object for which we query the offset
1484	*
1485	* Returns:
1486	* current FB aperture GPU offset of the object.
1487	*/
1488	u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo)
1489	{
1490	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
1491	uint64_t offset, fb_base;
1492
1493	WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM);
1494
1495	fb_base = adev->gmc.fb_start;
1496	fb_base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1497	offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base;
1498	return amdgpu_gmc_sign_extend(addr: offset);
1499	}
1500
1501	/**
1502	* amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1503	* @bo: amdgpu object for which we query the offset
1504	*
1505	* Returns:
1506	* current GPU offset of the object without raising warnings.
1507	*/
1508	u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)
1509	{
1510	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
1511	uint64_t offset = AMDGPU_BO_INVALID_OFFSET;
1512
1513	if (bo->tbo.resource->mem_type == TTM_PL_TT)
1514	offset = amdgpu_gmc_agp_addr(bo: &bo->tbo);
1515
1516	if (offset == AMDGPU_BO_INVALID_OFFSET)
1517	offset = (bo->tbo.resource->start << PAGE_SHIFT) +
1518	amdgpu_ttm_domain_start(adev, type: bo->tbo.resource->mem_type);
1519
1520	return amdgpu_gmc_sign_extend(addr: offset);
1521	}
1522
1523	/**
1524	* amdgpu_bo_mem_stats_placement - bo placement for memory accounting
1525	* @bo: the buffer object we should look at
1526	*
1527	* BO can have multiple preferred placements, to avoid double counting we want
1528	* to file it under a single placement for memory stats.
1529	* Luckily, if we take the highest set bit in preferred_domains the result is
1530	* quite sensible.
1531	*
1532	* Returns:
1533	* Which of the placements should the BO be accounted under.
1534	*/
1535	uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo)
1536	{
1537	uint32_t domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK;
1538
1539	if (!domain)
1540	return TTM_PL_SYSTEM;
1541
1542	switch (rounddown_pow_of_two(domain)) {
1543	case AMDGPU_GEM_DOMAIN_CPU:
1544	return TTM_PL_SYSTEM;
1545	case AMDGPU_GEM_DOMAIN_GTT:
1546	return TTM_PL_TT;
1547	case AMDGPU_GEM_DOMAIN_VRAM:
1548	return TTM_PL_VRAM;
1549	case AMDGPU_GEM_DOMAIN_GDS:
1550	return AMDGPU_PL_GDS;
1551	case AMDGPU_GEM_DOMAIN_GWS:
1552	return AMDGPU_PL_GWS;
1553	case AMDGPU_GEM_DOMAIN_OA:
1554	return AMDGPU_PL_OA;
1555	case AMDGPU_GEM_DOMAIN_DOORBELL:
1556	return AMDGPU_PL_DOORBELL;
1557	case AMDGPU_GEM_DOMAIN_MMIO_REMAP:
1558	return AMDGPU_PL_MMIO_REMAP;
1559	default:
1560	return TTM_PL_SYSTEM;
1561	}
1562	}
1563
1564	/**
1565	* amdgpu_bo_get_preferred_domain - get preferred domain
1566	* @adev: amdgpu device object
1567	* @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1568	*
1569	* Returns:
1570	* Which of the allowed domains is preferred for allocating the BO.
1571	*/
1572	uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
1573	uint32_t domain)
1574	{
1575	if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) &&
1576	((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) {
1577	domain = AMDGPU_GEM_DOMAIN_VRAM;
1578	if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1579	domain = AMDGPU_GEM_DOMAIN_GTT;
1580	}
1581	return domain;
1582	}
1583
1584	#if defined(CONFIG_DEBUG_FS)
1585	#define amdgpu_bo_print_flag(m, bo, flag) \
1586	do { \
1587	if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \
1588	seq_printf((m), " " #flag); \
1589	} \
1590	} while (0)
1591
1592	/**
1593	* amdgpu_bo_print_info - print BO info in debugfs file
1594	*
1595	* @id: Index or Id of the BO
1596	* @bo: Requested BO for printing info
1597	* @m: debugfs file
1598	*
1599	* Print BO information in debugfs file
1600	*
1601	* Returns:
1602	* Size of the BO in bytes.
1603	*/
1604	u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
1605	{
1606	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
1607	struct dma_buf_attachment *attachment;
1608	struct dma_buf *dma_buf;
1609	const char *placement;
1610	unsigned int pin_count;
1611	u64 size;
1612
1613	if (dma_resv_trylock(obj: bo->tbo.base.resv)) {
1614	if (!bo->tbo.resource) {
1615	placement = "NONE";
1616	} else {
1617	switch (bo->tbo.resource->mem_type) {
1618	case TTM_PL_VRAM:
1619	if (amdgpu_res_cpu_visible(adev, res: bo->tbo.resource))
1620	placement = "VRAM VISIBLE";
1621	else
1622	placement = "VRAM";
1623	break;
1624	case TTM_PL_TT:
1625	placement = "GTT";
1626	break;
1627	case AMDGPU_PL_GDS:
1628	placement = "GDS";
1629	break;
1630	case AMDGPU_PL_GWS:
1631	placement = "GWS";
1632	break;
1633	case AMDGPU_PL_OA:
1634	placement = "OA";
1635	break;
1636	case AMDGPU_PL_PREEMPT:
1637	placement = "PREEMPTIBLE";
1638	break;
1639	case AMDGPU_PL_DOORBELL:
1640	placement = "DOORBELL";
1641	break;
1642	case AMDGPU_PL_MMIO_REMAP:
1643	placement = "MMIO REMAP";
1644	break;
1645	case TTM_PL_SYSTEM:
1646	default:
1647	placement = "CPU";
1648	break;
1649	}
1650	}
1651	dma_resv_unlock(obj: bo->tbo.base.resv);
1652	} else {
1653	placement = "UNKNOWN";
1654	}
1655
1656	size = amdgpu_bo_size(bo);
1657	seq_printf(m, fmt: "\t\t0x%08x: %12lld byte %s",
1658	id, size, placement);
1659
1660	pin_count = READ_ONCE(bo->tbo.pin_count);
1661	if (pin_count)
1662	seq_printf(m, fmt: " pin count %d", pin_count);
1663
1664	dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
1665	attachment = READ_ONCE(bo->tbo.base.import_attach);
1666
1667	if (attachment)
1668	seq_printf(m, fmt: " imported from ino:%lu", file_inode(f: dma_buf->file)->i_ino);
1669	else if (dma_buf)
1670	seq_printf(m, fmt: " exported as ino:%lu", file_inode(f: dma_buf->file)->i_ino);
1671
1672	amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
1673	amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1674	amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1675	amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1676	amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1677	amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1678	amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);
1679	/* Add the gem obj resv fence dump*/
1680	if (dma_resv_trylock(obj: bo->tbo.base.resv)) {
1681	dma_resv_describe(obj: bo->tbo.base.resv, seq: m);
1682	dma_resv_unlock(obj: bo->tbo.base.resv);
1683	}
1684	seq_puts(m, s: "\n");
1685
1686	return size;
1687	}
1688	#endif
1689