1	/*
2	* Copyright 2018 Advanced Micro Devices, Inc.
3	* All Rights Reserved.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the
7	* "Software"), to deal in the Software without restriction, including
8	* without limitation the rights to use, copy, modify, merge, publish,
9	* distribute, sub license, and/or sell copies of the Software, and to
10	* permit persons to whom the Software is furnished to do so, subject to
11	* the following conditions:
12	*
13	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19	* USE OR OTHER DEALINGS IN THE SOFTWARE.
20	*
21	* The above copyright notice and this permission notice (including the
22	* next paragraph) shall be included in all copies or substantial portions
23	* of the Software.
24	*
25	*/
26
27	#include <linux/io-64-nonatomic-lo-hi.h>
28	#ifdef CONFIG_X86
29	#include <asm/hypervisor.h>
30	#endif
31
32	#include "amdgpu.h"
33	#include "amdgpu_gmc.h"
34	#include "amdgpu_ras.h"
35	#include "amdgpu_reset.h"
36	#include "amdgpu_xgmi.h"
37
38	#include <drm/drm_drv.h>
39	#include <drm/ttm/ttm_tt.h>
40
41	static const u64 four_gb = 0x100000000ULL;
42
43	bool amdgpu_gmc_is_pdb0_enabled(struct amdgpu_device *adev)
44	{
45	return adev->gmc.xgmi.connected_to_cpu || amdgpu_virt_xgmi_migrate_enabled(adev);
46	}
47
48	/**
49	* amdgpu_gmc_pdb0_alloc - allocate vram for pdb0
50	*
51	* @adev: amdgpu_device pointer
52	*
53	* Allocate video memory for pdb0 and map it for CPU access
54	* Returns 0 for success, error for failure.
55	*/
56	int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev)
57	{
58	int r;
59	struct amdgpu_bo_param bp;
60	u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes;
61	uint32_t pde0_page_shift = adev->gmc.vmid0_page_table_block_size + 21;
62	uint32_t npdes = (vram_size + (1ULL << pde0_page_shift) - 1) >> pde0_page_shift;
63
64	memset(&bp, 0, sizeof(bp));
65	bp.size = PAGE_ALIGN((npdes + 1) * 8);
66	bp.byte_align = PAGE_SIZE;
67	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
68	bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
69	AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
70	bp.type = ttm_bo_type_kernel;
71	bp.resv = NULL;
72	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
73
74	r = amdgpu_bo_create(adev, bp: &bp, bo_ptr: &adev->gmc.pdb0_bo);
75	if (r)
76	return r;
77
78	r = amdgpu_bo_reserve(bo: adev->gmc.pdb0_bo, no_intr: false);
79	if (unlikely(r != 0))
80	goto bo_reserve_failure;
81
82	r = amdgpu_bo_pin(bo: adev->gmc.pdb0_bo, AMDGPU_GEM_DOMAIN_VRAM);
83	if (r)
84	goto bo_pin_failure;
85	r = amdgpu_bo_kmap(bo: adev->gmc.pdb0_bo, ptr: &adev->gmc.ptr_pdb0);
86	if (r)
87	goto bo_kmap_failure;
88
89	amdgpu_bo_unreserve(bo: adev->gmc.pdb0_bo);
90	return 0;
91
92	bo_kmap_failure:
93	amdgpu_bo_unpin(bo: adev->gmc.pdb0_bo);
94	bo_pin_failure:
95	amdgpu_bo_unreserve(bo: adev->gmc.pdb0_bo);
96	bo_reserve_failure:
97	amdgpu_bo_unref(bo: &adev->gmc.pdb0_bo);
98	return r;
99	}
100
101	/**
102	* amdgpu_gmc_get_pde_for_bo - get the PDE for a BO
103	*
104	* @bo: the BO to get the PDE for
105	* @level: the level in the PD hirarchy
106	* @addr: resulting addr
107	* @flags: resulting flags
108	*
109	* Get the address and flags to be used for a PDE (Page Directory Entry).
110	*/
111	void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level,
112	uint64_t *addr, uint64_t *flags)
113	{
114	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
115
116	switch (bo->tbo.resource->mem_type) {
117	case TTM_PL_TT:
118	*addr = bo->tbo.ttm->dma_address[0];
119	break;
120	case TTM_PL_VRAM:
121	*addr = amdgpu_bo_gpu_offset(bo);
122	break;
123	default:
124	*addr = 0;
125	break;
126	}
127	*flags = amdgpu_ttm_tt_pde_flags(ttm: bo->tbo.ttm, mem: bo->tbo.resource);
128	amdgpu_gmc_get_vm_pde(adev, level, addr, flags);
129	}
130
131	/*
132	* amdgpu_gmc_pd_addr - return the address of the root directory
133	*/
134	uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo)
135	{
136	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
137	uint64_t pd_addr;
138
139	/* TODO: move that into ASIC specific code */
140	if (adev->asic_type >= CHIP_VEGA10) {
141	uint64_t flags = AMDGPU_PTE_VALID;
142
143	amdgpu_gmc_get_pde_for_bo(bo, level: -1, addr: &pd_addr, flags: &flags);
144	pd_addr |= flags;
145	} else {
146	pd_addr = amdgpu_bo_gpu_offset(bo);
147	}
148	return pd_addr;
149	}
150
151	/**
152	* amdgpu_gmc_set_pte_pde - update the page tables using CPU
153	*
154	* @adev: amdgpu_device pointer
155	* @cpu_pt_addr: cpu address of the page table
156	* @gpu_page_idx: entry in the page table to update
157	* @addr: dst addr to write into pte/pde
158	* @flags: access flags
159	*
160	* Update the page tables using CPU.
161	*/
162	int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
163	uint32_t gpu_page_idx, uint64_t addr,
164	uint64_t flags)
165	{
166	void __iomem *ptr = (void *)cpu_pt_addr;
167	uint64_t value;
168
169	/*
170	* The following is for PTE only. GART does not have PDEs.
171	*/
172	value = addr & 0x0000FFFFFFFFF000ULL;
173	value |= flags;
174	writeq(val: value, addr: ptr + (gpu_page_idx * 8));
175
176	return 0;
177	}
178
179	/**
180	* amdgpu_gmc_agp_addr - return the address in the AGP address space
181	*
182	* @bo: TTM BO which needs the address, must be in GTT domain
183	*
184	* Tries to figure out how to access the BO through the AGP aperture. Returns
185	* AMDGPU_BO_INVALID_OFFSET if that is not possible.
186	*/
187	uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo)
188	{
189	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev: bo->bdev);
190
191	if (!bo->ttm)
192	return AMDGPU_BO_INVALID_OFFSET;
193
194	if (bo->ttm->num_pages != 1 || bo->ttm->caching == ttm_cached)
195	return AMDGPU_BO_INVALID_OFFSET;
196
197	if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size)
198	return AMDGPU_BO_INVALID_OFFSET;
199
200	return adev->gmc.agp_start + bo->ttm->dma_address[0];
201	}
202
203	/**
204	* amdgpu_gmc_vram_location - try to find VRAM location
205	*
206	* @adev: amdgpu device structure holding all necessary information
207	* @mc: memory controller structure holding memory information
208	* @base: base address at which to put VRAM
209	*
210	* Function will try to place VRAM at base address provided
211	* as parameter.
212	*/
213	void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,
214	u64 base)
215	{
216	uint64_t vis_limit = (uint64_t)amdgpu_vis_vram_limit << 20;
217	uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
218
219	mc->vram_start = base;
220	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
221	if (limit < mc->real_vram_size)
222	mc->real_vram_size = limit;
223
224	if (vis_limit && vis_limit < mc->visible_vram_size)
225	mc->visible_vram_size = vis_limit;
226
227	if (mc->real_vram_size < mc->visible_vram_size)
228	mc->visible_vram_size = mc->real_vram_size;
229
230	if (mc->xgmi.num_physical_nodes == 0) {
231	mc->fb_start = mc->vram_start;
232	mc->fb_end = mc->vram_end;
233	}
234	dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
235	mc->mc_vram_size >> 20, mc->vram_start,
236	mc->vram_end, mc->real_vram_size >> 20);
237	}
238
239	/** amdgpu_gmc_sysvm_location - place vram and gart in sysvm aperture
240	*
241	* @adev: amdgpu device structure holding all necessary information
242	* @mc: memory controller structure holding memory information
243	*
244	* This function is only used if use GART for FB translation. In such
245	* case, we use sysvm aperture (vmid0 page tables) for both vram
246	* and gart (aka system memory) access.
247	*
248	* GPUVM (and our organization of vmid0 page tables) require sysvm
249	* aperture to be placed at a location aligned with 8 times of native
250	* page size. For example, if vm_context0_cntl.page_table_block_size
251	* is 12, then native page size is 8G (2M*2^12), sysvm should start
252	* with a 64G aligned address. For simplicity, we just put sysvm at
253	* address 0. So vram start at address 0 and gart is right after vram.
254	*/
255	void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
256	{
257	u64 hive_vram_start = 0;
258	u64 hive_vram_end = mc->xgmi.node_segment_size * mc->xgmi.num_physical_nodes - 1;
259	mc->vram_start = mc->xgmi.node_segment_size * mc->xgmi.physical_node_id;
260	mc->vram_end = mc->vram_start + mc->xgmi.node_segment_size - 1;
261	/* node_segment_size may not 4GB aligned on SRIOV, align up is needed. */
262	mc->gart_start = ALIGN(hive_vram_end + 1, four_gb);
263	mc->gart_end = mc->gart_start + mc->gart_size - 1;
264	if (amdgpu_virt_xgmi_migrate_enabled(adev)) {
265	/* set mc->vram_start to 0 to switch the returned GPU address of
266	* amdgpu_bo_create_reserved() from FB aperture to GART aperture.
267	*/
268	mc->vram_start = 0;
269	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
270	mc->visible_vram_size = min(mc->visible_vram_size, mc->real_vram_size);
271	} else {
272	mc->fb_start = hive_vram_start;
273	mc->fb_end = hive_vram_end;
274	}
275	dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
276	mc->mc_vram_size >> 20, mc->vram_start,
277	mc->vram_end, mc->real_vram_size >> 20);
278	dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n",
279	mc->gart_size >> 20, mc->gart_start, mc->gart_end);
280	}
281
282	/**
283	* amdgpu_gmc_gart_location - try to find GART location
284	*
285	* @adev: amdgpu device structure holding all necessary information
286	* @mc: memory controller structure holding memory information
287	* @gart_placement: GART placement policy with respect to VRAM
288	*
289	* Function will try to place GART before or after VRAM.
290	* If GART size is bigger than space left then we ajust GART size.
291	* Thus function will never fails.
292	*/
293	void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,
294	enum amdgpu_gart_placement gart_placement)
295	{
296	u64 size_af, size_bf;
297	/*To avoid the hole, limit the max mc address to AMDGPU_GMC_HOLE_START*/
298	u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1);
299
300	/* VCE doesn't like it when BOs cross a 4GB segment, so align
301	* the GART base on a 4GB boundary as well.
302	*/
303	size_bf = mc->fb_start;
304	size_af = max_mc_address + 1 - ALIGN(mc->fb_end + 1, four_gb);
305
306	if (mc->gart_size > max(size_bf, size_af)) {
307	dev_warn(adev->dev, "limiting GART\n");
308	mc->gart_size = max(size_bf, size_af);
309	}
310
311	switch (gart_placement) {
312	case AMDGPU_GART_PLACEMENT_HIGH:
313	mc->gart_start = max_mc_address - mc->gart_size + 1;
314	break;
315	case AMDGPU_GART_PLACEMENT_LOW:
316	mc->gart_start = 0;
317	break;
318	case AMDGPU_GART_PLACEMENT_BEST_FIT:
319	default:
320	if ((size_bf >= mc->gart_size && size_bf < size_af) ||
321	(size_af < mc->gart_size))
322	mc->gart_start = 0;
323	else
324	mc->gart_start = max_mc_address - mc->gart_size + 1;
325	break;
326	}
327
328	mc->gart_start &= ~(four_gb - 1);
329	mc->gart_end = mc->gart_start + mc->gart_size - 1;
330	dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n",
331	mc->gart_size >> 20, mc->gart_start, mc->gart_end);
332	}
333
334	/**
335	* amdgpu_gmc_agp_location - try to find AGP location
336	* @adev: amdgpu device structure holding all necessary information
337	* @mc: memory controller structure holding memory information
338	*
339	* Function will place try to find a place for the AGP BAR in the MC address
340	* space.
341	*
342	* AGP BAR will be assigned the largest available hole in the address space.
343	* Should be called after VRAM and GART locations are setup.
344	*/
345	void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
346	{
347	const uint64_t sixteen_gb = 1ULL << 34;
348	const uint64_t sixteen_gb_mask = ~(sixteen_gb - 1);
349	u64 size_af, size_bf;
350
351	if (mc->fb_start > mc->gart_start) {
352	size_bf = (mc->fb_start & sixteen_gb_mask) -
353	ALIGN(mc->gart_end + 1, sixteen_gb);
354	size_af = mc->mc_mask + 1 - ALIGN(mc->fb_end + 1, sixteen_gb);
355	} else {
356	size_bf = mc->fb_start & sixteen_gb_mask;
357	size_af = (mc->gart_start & sixteen_gb_mask) -
358	ALIGN(mc->fb_end + 1, sixteen_gb);
359	}
360
361	if (size_bf > size_af) {
362	mc->agp_start = (mc->fb_start - size_bf) & sixteen_gb_mask;
363	mc->agp_size = size_bf;
364	} else {
365	mc->agp_start = ALIGN(mc->fb_end + 1, sixteen_gb);
366	mc->agp_size = size_af;
367	}
368
369	mc->agp_end = mc->agp_start + mc->agp_size - 1;
370	dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n",
371	mc->agp_size >> 20, mc->agp_start, mc->agp_end);
372	}
373
374	/**
375	* amdgpu_gmc_set_agp_default - Set the default AGP aperture value.
376	* @adev: amdgpu device structure holding all necessary information
377	* @mc: memory controller structure holding memory information
378	*
379	* To disable the AGP aperture, you need to set the start to a larger
380	* value than the end. This function sets the default value which
381	* can then be overridden using amdgpu_gmc_agp_location() if you want
382	* to enable the AGP aperture on a specific chip.
383	*
384	*/
385	void amdgpu_gmc_set_agp_default(struct amdgpu_device *adev,
386	struct amdgpu_gmc *mc)
387	{
388	mc->agp_start = 0xffffffffffff;
389	mc->agp_end = 0;
390	mc->agp_size = 0;
391	}
392
393	/**
394	* amdgpu_gmc_fault_key - get hask key from vm fault address and pasid
395	*
396	* @addr: 48 bit physical address, page aligned (36 significant bits)
397	* @pasid: 16 bit process address space identifier
398	*/
399	static inline uint64_t amdgpu_gmc_fault_key(uint64_t addr, uint16_t pasid)
400	{
401	return addr << 4 | pasid;
402	}
403
404	/**
405	* amdgpu_gmc_filter_faults - filter VM faults
406	*
407	* @adev: amdgpu device structure
408	* @ih: interrupt ring that the fault received from
409	* @addr: address of the VM fault
410	* @pasid: PASID of the process causing the fault
411	* @timestamp: timestamp of the fault
412	*
413	* Returns:
414	* True if the fault was filtered and should not be processed further.
415	* False if the fault is a new one and needs to be handled.
416	*/
417	bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev,
418	struct amdgpu_ih_ring *ih, uint64_t addr,
419	uint16_t pasid, uint64_t timestamp)
420	{
421	struct amdgpu_gmc *gmc = &adev->gmc;
422	uint64_t stamp, key = amdgpu_gmc_fault_key(addr, pasid);
423	struct amdgpu_gmc_fault *fault;
424	uint32_t hash;
425
426	/* Stale retry fault if timestamp goes backward */
427	if (amdgpu_ih_ts_after(timestamp, ih->processed_timestamp))
428	return true;
429
430	/* If we don't have space left in the ring buffer return immediately */
431	stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) -
432	AMDGPU_GMC_FAULT_TIMEOUT;
433	if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp)
434	return true;
435
436	/* Try to find the fault in the hash */
437	hash = hash_64(val: key, AMDGPU_GMC_FAULT_HASH_ORDER);
438	fault = &gmc->fault_ring[gmc->fault_hash[hash].idx];
439	while (fault->timestamp >= stamp) {
440	uint64_t tmp;
441
442	if (atomic64_read(v: &fault->key) == key) {
443	/*
444	* if we get a fault which is already present in
445	* the fault_ring and the timestamp of
446	* the fault is after the expired timestamp,
447	* then this is a new fault that needs to be added
448	* into the fault ring.
449	*/
450	if (fault->timestamp_expiry != 0 &&
451	amdgpu_ih_ts_after(fault->timestamp_expiry,
452	timestamp))
453	break;
454	else
455	return true;
456	}
457
458	tmp = fault->timestamp;
459	fault = &gmc->fault_ring[fault->next];
460
461	/* Check if the entry was reused */
462	if (fault->timestamp >= tmp)
463	break;
464	}
465
466	/* Add the fault to the ring */
467	fault = &gmc->fault_ring[gmc->last_fault];
468	atomic64_set(v: &fault->key, i: key);
469	fault->timestamp = timestamp;
470
471	/* And update the hash */
472	fault->next = gmc->fault_hash[hash].idx;
473	gmc->fault_hash[hash].idx = gmc->last_fault++;
474	return false;
475	}
476
477	/**
478	* amdgpu_gmc_filter_faults_remove - remove address from VM faults filter
479	*
480	* @adev: amdgpu device structure
481	* @addr: address of the VM fault
482	* @pasid: PASID of the process causing the fault
483	*
484	* Remove the address from fault filter, then future vm fault on this address
485	* will pass to retry fault handler to recover.
486	*/
487	void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
488	uint16_t pasid)
489	{
490	struct amdgpu_gmc *gmc = &adev->gmc;
491	uint64_t key = amdgpu_gmc_fault_key(addr, pasid);
492	struct amdgpu_ih_ring *ih;
493	struct amdgpu_gmc_fault *fault;
494	uint32_t last_wptr;
495	uint64_t last_ts;
496	uint32_t hash;
497	uint64_t tmp;
498
499	if (adev->irq.retry_cam_enabled)
500	return;
501	else if (adev->irq.ih1.ring_size)
502	ih = &adev->irq.ih1;
503	else if (adev->irq.ih_soft.enabled)
504	ih = &adev->irq.ih_soft;
505	else
506	return;
507
508	/* Get the WPTR of the last entry in IH ring */
509	last_wptr = amdgpu_ih_get_wptr(adev, ih);
510	/* Order wptr with ring data. */
511	rmb();
512	/* Get the timetamp of the last entry in IH ring */
513	last_ts = amdgpu_ih_decode_iv_ts(adev, ih, last_wptr, -1);
514
515	hash = hash_64(val: key, AMDGPU_GMC_FAULT_HASH_ORDER);
516	fault = &gmc->fault_ring[gmc->fault_hash[hash].idx];
517	do {
518	if (atomic64_read(v: &fault->key) == key) {
519	/*
520	* Update the timestamp when this fault
521	* expired.
522	*/
523	fault->timestamp_expiry = last_ts;
524	break;
525	}
526
527	tmp = fault->timestamp;
528	fault = &gmc->fault_ring[fault->next];
529	} while (fault->timestamp < tmp);
530	}
531
532	int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev)
533	{
534	int r;
535
536	/* umc ras block */
537	r = amdgpu_umc_ras_sw_init(adev);
538	if (r)
539	return r;
540
541	/* mmhub ras block */
542	r = amdgpu_mmhub_ras_sw_init(adev);
543	if (r)
544	return r;
545
546	/* hdp ras block */
547	r = amdgpu_hdp_ras_sw_init(adev);
548	if (r)
549	return r;
550
551	/* mca.x ras block */
552	r = amdgpu_mca_mp0_ras_sw_init(adev);
553	if (r)
554	return r;
555
556	r = amdgpu_mca_mp1_ras_sw_init(adev);
557	if (r)
558	return r;
559
560	r = amdgpu_mca_mpio_ras_sw_init(adev);
561	if (r)
562	return r;
563
564	/* xgmi ras block */
565	r = amdgpu_xgmi_ras_sw_init(adev);
566	if (r)
567	return r;
568
569	return 0;
570	}
571
572	int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev)
573	{
574	return 0;
575	}
576
577	void amdgpu_gmc_ras_fini(struct amdgpu_device *adev)
578	{
579
580	}
581
582	/*
583	* The latest engine allocation on gfx9/10 is:
584	* Engine 2, 3: firmware
585	* Engine 0, 1, 4~16: amdgpu ring,
586	* subject to change when ring number changes
587	* Engine 17: Gart flushes
588	*/
589	#define AMDGPU_VMHUB_INV_ENG_BITMAP 0x1FFF3
590
591	int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)
592	{
593	struct amdgpu_ring *ring;
594	unsigned vm_inv_engs[AMDGPU_MAX_VMHUBS] = {0};
595	unsigned i;
596	unsigned vmhub, inv_eng;
597	struct amdgpu_ring *shared_ring;
598
599	/* init the vm inv eng for all vmhubs */
600	for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
601	vm_inv_engs[i] = AMDGPU_VMHUB_INV_ENG_BITMAP;
602	/* reserve engine 5 for firmware */
603	if (adev->enable_mes)
604	vm_inv_engs[i] &= ~(1 << 5);
605	/* reserve engine 6 for uni mes */
606	if (adev->enable_uni_mes)
607	vm_inv_engs[i] &= ~(1 << 6);
608	/* reserve mmhub engine 3 for firmware */
609	if (adev->enable_umsch_mm)
610	vm_inv_engs[i] &= ~(1 << 3);
611	}
612
613	for (i = 0; i < adev->num_rings; ++i) {
614	ring = adev->rings[i];
615	vmhub = ring->vm_hub;
616
617	if (ring == &adev->mes.ring[0] ||
618	ring == &adev->mes.ring[1] ||
619	ring == &adev->umsch_mm.ring ||
620	ring == &adev->cper.ring_buf)
621	continue;
622
623	/* Skip if the ring is a shared ring */
624	if (amdgpu_sdma_is_shared_inv_eng(adev, ring))
625	continue;
626
627	inv_eng = ffs(vm_inv_engs[vmhub]);
628	if (!inv_eng) {
629	dev_err(adev->dev, "no VM inv eng for ring %s\n",
630	ring->name);
631	return -EINVAL;
632	}
633
634	ring->vm_inv_eng = inv_eng - 1;
635	vm_inv_engs[vmhub] &= ~(1 << ring->vm_inv_eng);
636
637	dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n",
638	ring->name, ring->vm_inv_eng, ring->vm_hub);
639	/* SDMA has a special packet which allows it to use the same
640	* invalidation engine for all the rings in one instance.
641	* Therefore, we do not allocate a separate VM invalidation engine
642	* for SDMA page rings. Instead, they share the VM invalidation
643	* engine with the SDMA gfx ring. This change ensures efficient
644	* resource management and avoids the issue of insufficient VM
645	* invalidation engines.
646	*/
647	shared_ring = amdgpu_sdma_get_shared_ring(adev, ring);
648	if (shared_ring) {
649	shared_ring->vm_inv_eng = ring->vm_inv_eng;
650	dev_info(adev->dev, "ring %s shares VM invalidation engine %u with ring %s on hub %u\n",
651	ring->name, ring->vm_inv_eng, shared_ring->name, ring->vm_hub);
652	continue;
653	}
654	}
655
656	return 0;
657	}
658
659	void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
660	uint32_t vmhub, uint32_t flush_type)
661	{
662	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
663	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
664	struct dma_fence *fence;
665	struct amdgpu_job *job;
666	int r;
667
668	if (!hub->sdma_invalidation_workaround || vmid ||
669	!adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready ||
670	!ring->sched.ready) {
671	/*
672	* A GPU reset should flush all TLBs anyway, so no need to do
673	* this while one is ongoing.
674	*/
675	if (!down_read_trylock(sem: &adev->reset_domain->sem))
676	return;
677
678	if (adev->gmc.flush_tlb_needs_extra_type_2)
679	adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid,
680	vmhub, 2);
681
682	if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2)
683	adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid,
684	vmhub, 0);
685
686	adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub,
687	flush_type);
688	up_read(sem: &adev->reset_domain->sem);
689	return;
690	}
691
692	/* The SDMA on Navi 1x has a bug which can theoretically result in memory
693	* corruption if an invalidation happens at the same time as an VA
694	* translation. Avoid this by doing the invalidation from the SDMA
695	* itself at least for GART.
696	*/
697	mutex_lock(&adev->mman.gtt_window_lock);
698	r = amdgpu_job_alloc_with_ib(adev: ring->adev, entity: &adev->mman.high_pr,
699	AMDGPU_FENCE_OWNER_UNDEFINED,
700	size: 16 * 4, pool_type: AMDGPU_IB_POOL_IMMEDIATE,
701	job: &job, AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB);
702	if (r)
703	goto error_alloc;
704
705	job->vm_pd_addr = amdgpu_gmc_pd_addr(bo: adev->gart.bo);
706	job->vm_needs_flush = true;
707	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
708	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
709	fence = amdgpu_job_submit(job);
710	mutex_unlock(lock: &adev->mman.gtt_window_lock);
711
712	dma_fence_wait(fence, intr: false);
713	dma_fence_put(fence);
714
715	return;
716
717	error_alloc:
718	mutex_unlock(lock: &adev->mman.gtt_window_lock);
719	dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r);
720	}
721
722	int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid,
723	uint32_t flush_type, bool all_hub,
724	uint32_t inst)
725	{
726	struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring;
727	struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst];
728	unsigned int ndw;
729	int r, cnt = 0;
730	uint32_t seq;
731
732	/*
733	* A GPU reset should flush all TLBs anyway, so no need to do
734	* this while one is ongoing.
735	*/
736	if (!down_read_trylock(sem: &adev->reset_domain->sem))
737	return 0;
738
739	if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) {
740
741	if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) {
742	r = 0;
743	goto error_unlock_reset;
744	}
745
746	if (adev->gmc.flush_tlb_needs_extra_type_2)
747	adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
748	2, all_hub,
749	inst);
750
751	if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2)
752	adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
753	0, all_hub,
754	inst);
755
756	adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid,
757	flush_type, all_hub,
758	inst);
759	r = 0;
760	} else {
761	/* 2 dwords flush + 8 dwords fence */
762	ndw = kiq->pmf->invalidate_tlbs_size + 8;
763
764	if (adev->gmc.flush_tlb_needs_extra_type_2)
765	ndw += kiq->pmf->invalidate_tlbs_size;
766
767	if (adev->gmc.flush_tlb_needs_extra_type_0)
768	ndw += kiq->pmf->invalidate_tlbs_size;
769
770	spin_lock(lock: &adev->gfx.kiq[inst].ring_lock);
771	r = amdgpu_ring_alloc(ring, ndw);
772	if (r) {
773	spin_unlock(lock: &adev->gfx.kiq[inst].ring_lock);
774	goto error_unlock_reset;
775	}
776	if (adev->gmc.flush_tlb_needs_extra_type_2)
777	kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub);
778
779	if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0)
780	kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub);
781
782	kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub);
783	r = amdgpu_fence_emit_polling(ring, s: &seq, MAX_KIQ_REG_WAIT);
784	if (r) {
785	amdgpu_ring_undo(ring);
786	spin_unlock(lock: &adev->gfx.kiq[inst].ring_lock);
787	goto error_unlock_reset;
788	}
789
790	amdgpu_ring_commit(ring);
791	spin_unlock(lock: &adev->gfx.kiq[inst].ring_lock);
792
793	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
794
795	might_sleep();
796	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
797	!amdgpu_reset_pending(domain: adev->reset_domain)) {
798	msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
799	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
800	}
801
802	if (cnt > MAX_KIQ_REG_TRY) {
803	dev_err(adev->dev, "timeout waiting for kiq fence\n");
804	r = -ETIME;
805	} else
806	r = 0;
807	}
808
809	error_unlock_reset:
810	up_read(sem: &adev->reset_domain->sem);
811	return r;
812	}
813
814	void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,
815	uint32_t reg0, uint32_t reg1,
816	uint32_t ref, uint32_t mask,
817	uint32_t xcc_inst)
818	{
819	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_inst];
820	struct amdgpu_ring *ring = &kiq->ring;
821	signed long r, cnt = 0;
822	unsigned long flags;
823	uint32_t seq;
824
825	if (adev->mes.ring[0].sched.ready) {
826	amdgpu_mes_reg_write_reg_wait(adev, reg0, reg1,
827	ref, mask);
828	return;
829	}
830
831	spin_lock_irqsave(&kiq->ring_lock, flags);
832	amdgpu_ring_alloc(ring, ndw: 32);
833	amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1,
834	ref, mask);
835	r = amdgpu_fence_emit_polling(ring, s: &seq, MAX_KIQ_REG_WAIT);
836	if (r)
837	goto failed_undo;
838
839	amdgpu_ring_commit(ring);
840	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
841
842	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
843
844	/* don't wait anymore for IRQ context */
845	if (r < 1 && in_interrupt())
846	goto failed_kiq;
847
848	might_sleep();
849	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY &&
850	!amdgpu_reset_pending(domain: adev->reset_domain)) {
851
852	msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
853	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
854	}
855
856	if (cnt > MAX_KIQ_REG_TRY)
857	goto failed_kiq;
858
859	return;
860
861	failed_undo:
862	amdgpu_ring_undo(ring);
863	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
864	failed_kiq:
865	dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1);
866	}
867
868	/**
869	* amdgpu_gmc_tmz_set -- check and set if a device supports TMZ
870	* @adev: amdgpu_device pointer
871	*
872	* Check and set if an the device @adev supports Trusted Memory
873	* Zones (TMZ).
874	*/
875	void amdgpu_gmc_tmz_set(struct amdgpu_device *adev)
876	{
877	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
878	/* RAVEN */
879	case IP_VERSION(9, 2, 2):
880	case IP_VERSION(9, 1, 0):
881	/* RENOIR looks like RAVEN */
882	case IP_VERSION(9, 3, 0):
883	/* GC 10.3.7 */
884	case IP_VERSION(10, 3, 7):
885	/* GC 11.0.1 */
886	case IP_VERSION(11, 0, 1):
887	if (amdgpu_tmz == 0) {
888	adev->gmc.tmz_enabled = false;
889	dev_info(adev->dev,
890	"Trusted Memory Zone (TMZ) feature disabled (cmd line)\n");
891	} else {
892	adev->gmc.tmz_enabled = true;
893	dev_info(adev->dev,
894	"Trusted Memory Zone (TMZ) feature enabled\n");
895	}
896	break;
897	case IP_VERSION(10, 1, 10):
898	case IP_VERSION(10, 1, 1):
899	case IP_VERSION(10, 1, 2):
900	case IP_VERSION(10, 1, 3):
901	case IP_VERSION(10, 3, 0):
902	case IP_VERSION(10, 3, 2):
903	case IP_VERSION(10, 3, 4):
904	case IP_VERSION(10, 3, 5):
905	case IP_VERSION(10, 3, 6):
906	/* VANGOGH */
907	case IP_VERSION(10, 3, 1):
908	/* YELLOW_CARP*/
909	case IP_VERSION(10, 3, 3):
910	case IP_VERSION(11, 0, 4):
911	case IP_VERSION(11, 5, 0):
912	case IP_VERSION(11, 5, 1):
913	case IP_VERSION(11, 5, 2):
914	case IP_VERSION(11, 5, 3):
915	/* Don't enable it by default yet.
916	*/
917	if (amdgpu_tmz < 1) {
918	adev->gmc.tmz_enabled = false;
919	dev_info(adev->dev,
920	"Trusted Memory Zone (TMZ) feature disabled as experimental (default)\n");
921	} else {
922	adev->gmc.tmz_enabled = true;
923	dev_info(adev->dev,
924	"Trusted Memory Zone (TMZ) feature enabled as experimental (cmd line)\n");
925	}
926	break;
927	default:
928	adev->gmc.tmz_enabled = false;
929	dev_info(adev->dev,
930	"Trusted Memory Zone (TMZ) feature not supported\n");
931	break;
932	}
933	}
934
935	/**
936	* amdgpu_gmc_noretry_set -- set per asic noretry defaults
937	* @adev: amdgpu_device pointer
938	*
939	* Set a per asic default for the no-retry parameter.
940	*
941	*/
942	void amdgpu_gmc_noretry_set(struct amdgpu_device *adev)
943	{
944	struct amdgpu_gmc *gmc = &adev->gmc;
945	uint32_t gc_ver = amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0);
946	bool noretry_default = (gc_ver == IP_VERSION(9, 0, 1) ||
947	gc_ver == IP_VERSION(9, 4, 0) ||
948	gc_ver == IP_VERSION(9, 4, 1) ||
949	gc_ver == IP_VERSION(9, 4, 2) ||
950	gc_ver == IP_VERSION(9, 4, 3) ||
951	gc_ver == IP_VERSION(9, 4, 4) ||
952	gc_ver == IP_VERSION(9, 5, 0) ||
953	gc_ver >= IP_VERSION(10, 3, 0));
954
955	if (!amdgpu_sriov_xnack_support(adev))
956	gmc->noretry = 1;
957	else
958	gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry;
959	}
960
961	void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type,
962	bool enable)
963	{
964	struct amdgpu_vmhub *hub;
965	u32 tmp, reg, i;
966
967	hub = &adev->vmhub[hub_type];
968	for (i = 0; i < 16; i++) {
969	reg = hub->vm_context0_cntl + hub->ctx_distance * i;
970
971	tmp = (hub_type == AMDGPU_GFXHUB(0)) ?
972	RREG32_SOC15_IP(GC, reg) :
973	RREG32_SOC15_IP(MMHUB, reg);
974
975	if (enable)
976	tmp |= hub->vm_cntx_cntl_vm_fault;
977	else
978	tmp &= ~hub->vm_cntx_cntl_vm_fault;
979
980	(hub_type == AMDGPU_GFXHUB(0)) ?
981	WREG32_SOC15_IP(GC, reg, tmp) :
982	WREG32_SOC15_IP(MMHUB, reg, tmp);
983	}
984	}
985
986	void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev)
987	{
988	unsigned size;
989
990	/*
991	* Some ASICs need to reserve a region of video memory to avoid access
992	* from driver
993	*/
994	adev->mman.stolen_reserved_offset = 0;
995	adev->mman.stolen_reserved_size = 0;
996
997	/*
998	* TODO:
999	* Currently there is a bug where some memory client outside
1000	* of the driver writes to first 8M of VRAM on S3 resume,
1001	* this overrides GART which by default gets placed in first 8M and
1002	* causes VM_FAULTS once GTT is accessed.
1003	* Keep the stolen memory reservation until the while this is not solved.
1004	*/
1005	switch (adev->asic_type) {
1006	case CHIP_VEGA10:
1007	adev->mman.keep_stolen_vga_memory = true;
1008	/*
1009	* VEGA10 SRIOV VF with MS_HYPERV host needs some firmware reserved area.
1010	*/
1011	#ifdef CONFIG_X86
1012	if (amdgpu_sriov_vf(adev) && hypervisor_is_type(type: X86_HYPER_MS_HYPERV)) {
1013	adev->mman.stolen_reserved_offset = 0x500000;
1014	adev->mman.stolen_reserved_size = 0x200000;
1015	}
1016	#endif
1017	break;
1018	case CHIP_RAVEN:
1019	case CHIP_RENOIR:
1020	adev->mman.keep_stolen_vga_memory = true;
1021	break;
1022	default:
1023	adev->mman.keep_stolen_vga_memory = false;
1024	break;
1025	}
1026
1027	if (amdgpu_sriov_vf(adev) ||
1028	!amdgpu_device_has_display_hardware(adev)) {
1029	size = 0;
1030	} else {
1031	size = amdgpu_gmc_get_vbios_fb_size(adev);
1032
1033	if (adev->mman.keep_stolen_vga_memory)
1034	size = max(size, (unsigned)AMDGPU_VBIOS_VGA_ALLOCATION);
1035	}
1036
1037	/* set to 0 if the pre-OS buffer uses up most of vram */
1038	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
1039	size = 0;
1040
1041	if (size > AMDGPU_VBIOS_VGA_ALLOCATION) {
1042	adev->mman.stolen_vga_size = AMDGPU_VBIOS_VGA_ALLOCATION;
1043	adev->mman.stolen_extended_size = size - adev->mman.stolen_vga_size;
1044	} else {
1045	adev->mman.stolen_vga_size = size;
1046	adev->mman.stolen_extended_size = 0;
1047	}
1048	}
1049
1050	/**
1051	* amdgpu_gmc_init_pdb0 - initialize PDB0
1052	*
1053	* @adev: amdgpu_device pointer
1054	*
1055	* This function is only used when GART page table is used
1056	* for FB address translatioin. In such a case, we construct
1057	* a 2-level system VM page table: PDB0->PTB, to cover both
1058	* VRAM of the hive and system memory.
1059	*
1060	* PDB0 is static, initialized once on driver initialization.
1061	* The first n entries of PDB0 are used as PTE by setting
1062	* P bit to 1, pointing to VRAM. The n+1'th entry points
1063	* to a big PTB covering system memory.
1064	*
1065	*/
1066	void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev)
1067	{
1068	int i;
1069	uint64_t flags = adev->gart.gart_pte_flags; //TODO it is UC. explore NC/RW?
1070	/* Each PDE0 (used as PTE) covers (2^vmid0_page_table_block_size)*2M
1071	*/
1072	u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes;
1073	u64 pde0_page_size = (1ULL<<adev->gmc.vmid0_page_table_block_size)<<21;
1074	u64 vram_addr, vram_end;
1075	u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, bo: adev->gart.bo);
1076	int idx;
1077
1078	if (!drm_dev_enter(dev: adev_to_drm(adev), idx: &idx))
1079	return;
1080
1081	flags |= AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE;
1082	flags |= AMDGPU_PTE_WRITEABLE;
1083	flags |= AMDGPU_PTE_SNOOPED;
1084	flags |= AMDGPU_PTE_FRAG((adev->gmc.vmid0_page_table_block_size + 9*1));
1085	flags |= AMDGPU_PDE_PTE_FLAG(adev);
1086
1087	vram_addr = adev->vm_manager.vram_base_offset;
1088	if (!amdgpu_virt_xgmi_migrate_enabled(adev))
1089	vram_addr -= adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1090	vram_end = vram_addr + vram_size;
1091
1092	/* The first n PDE0 entries are used as PTE,
1093	* pointing to vram
1094	*/
1095	for (i = 0; vram_addr < vram_end; i++, vram_addr += pde0_page_size)
1096	amdgpu_gmc_set_pte_pde(adev, cpu_pt_addr: adev->gmc.ptr_pdb0, gpu_page_idx: i, addr: vram_addr, flags);
1097
1098	/* The n+1'th PDE0 entry points to a huge
1099	* PTB who has more than 512 entries each
1100	* pointing to a 4K system page
1101	*/
1102	flags = AMDGPU_PTE_VALID;
1103	flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PDE_BFS_FLAG(adev, 0);
1104	/* Requires gart_ptb_gpu_pa to be 4K aligned */
1105	amdgpu_gmc_set_pte_pde(adev, cpu_pt_addr: adev->gmc.ptr_pdb0, gpu_page_idx: i, addr: gart_ptb_gpu_pa, flags);
1106	drm_dev_exit(idx);
1107	}
1108
1109	/**
1110	* amdgpu_gmc_vram_mc2pa - calculate vram buffer's physical address from MC
1111	* address
1112	*
1113	* @adev: amdgpu_device pointer
1114	* @mc_addr: MC address of buffer
1115	*/
1116	uint64_t amdgpu_gmc_vram_mc2pa(struct amdgpu_device *adev, uint64_t mc_addr)
1117	{
1118	return mc_addr - adev->gmc.vram_start + adev->vm_manager.vram_base_offset;
1119	}
1120
1121	/**
1122	* amdgpu_gmc_vram_pa - calculate vram buffer object's physical address from
1123	* GPU's view
1124	*
1125	* @adev: amdgpu_device pointer
1126	* @bo: amdgpu buffer object
1127	*/
1128	uint64_t amdgpu_gmc_vram_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo)
1129	{
1130	return amdgpu_gmc_vram_mc2pa(adev, mc_addr: amdgpu_bo_gpu_offset(bo));
1131	}
1132
1133	int amdgpu_gmc_vram_checking(struct amdgpu_device *adev)
1134	{
1135	struct amdgpu_bo *vram_bo = NULL;
1136	uint64_t vram_gpu = 0;
1137	void *vram_ptr = NULL;
1138
1139	int ret, size = 0x100000;
1140	uint8_t cptr[10];
1141
1142	ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
1143	AMDGPU_GEM_DOMAIN_VRAM,
1144	bo_ptr: &vram_bo,
1145	gpu_addr: &vram_gpu,
1146	cpu_addr: &vram_ptr);
1147	if (ret)
1148	return ret;
1149
1150	memset(vram_ptr, 0x86, size);
1151	memset(cptr, 0x86, 10);
1152
1153	/**
1154	* Check the start, the mid, and the end of the memory if the content of
1155	* each byte is the pattern "0x86". If yes, we suppose the vram bo is
1156	* workable.
1157	*
1158	* Note: If check the each byte of whole 1M bo, it will cost too many
1159	* seconds, so here, we just pick up three parts for emulation.
1160	*/
1161	ret = memcmp(p: vram_ptr, q: cptr, size: 10);
1162	if (ret) {
1163	ret = -EIO;
1164	goto release_buffer;
1165	}
1166
1167	ret = memcmp(p: vram_ptr + (size / 2), q: cptr, size: 10);
1168	if (ret) {
1169	ret = -EIO;
1170	goto release_buffer;
1171	}
1172
1173	ret = memcmp(p: vram_ptr + size - 10, q: cptr, size: 10);
1174	if (ret) {
1175	ret = -EIO;
1176	goto release_buffer;
1177	}
1178
1179	release_buffer:
1180	amdgpu_bo_free_kernel(bo: &vram_bo, gpu_addr: &vram_gpu,
1181	cpu_addr: &vram_ptr);
1182
1183	return ret;
1184	}
1185
1186	static const char *nps_desc[] = {
1187	[AMDGPU_NPS1_PARTITION_MODE] = "NPS1",
1188	[AMDGPU_NPS2_PARTITION_MODE] = "NPS2",
1189	[AMDGPU_NPS3_PARTITION_MODE] = "NPS3",
1190	[AMDGPU_NPS4_PARTITION_MODE] = "NPS4",
1191	[AMDGPU_NPS6_PARTITION_MODE] = "NPS6",
1192	[AMDGPU_NPS8_PARTITION_MODE] = "NPS8",
1193	};
1194
1195	static ssize_t available_memory_partition_show(struct device *dev,
1196	struct device_attribute *addr,
1197	char *buf)
1198	{
1199	struct drm_device *ddev = dev_get_drvdata(dev);
1200	struct amdgpu_device *adev = drm_to_adev(ddev);
1201	int size = 0, mode;
1202	char *sep = "";
1203
1204	for_each_inst(mode, adev->gmc.supported_nps_modes) {
1205	size += sysfs_emit_at(buf, at: size, fmt: "%s%s", sep, nps_desc[mode]);
1206	sep = ", ";
1207	}
1208	size += sysfs_emit_at(buf, at: size, fmt: "\n");
1209
1210	return size;
1211	}
1212
1213	static ssize_t current_memory_partition_store(struct device *dev,
1214	struct device_attribute *attr,
1215	const char *buf, size_t count)
1216	{
1217	struct drm_device *ddev = dev_get_drvdata(dev);
1218	struct amdgpu_device *adev = drm_to_adev(ddev);
1219	enum amdgpu_memory_partition mode;
1220	struct amdgpu_hive_info *hive;
1221	int i;
1222
1223	mode = UNKNOWN_MEMORY_PARTITION_MODE;
1224	for_each_inst(i, adev->gmc.supported_nps_modes) {
1225	if (!strncasecmp(s1: nps_desc[i], s2: buf, strlen(nps_desc[i]))) {
1226	mode = i;
1227	break;
1228	}
1229	}
1230
1231	if (mode == UNKNOWN_MEMORY_PARTITION_MODE)
1232	return -EINVAL;
1233
1234	if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) {
1235	dev_info(
1236	adev->dev,
1237	"requested NPS mode is same as current NPS mode, skipping\n");
1238	return count;
1239	}
1240
1241	/* If device is part of hive, all devices in the hive should request the
1242	* same mode. Hence store the requested mode in hive.
1243	*/
1244	hive = amdgpu_get_xgmi_hive(adev);
1245	if (hive) {
1246	atomic_set(v: &hive->requested_nps_mode, i: mode);
1247	amdgpu_put_xgmi_hive(hive);
1248	} else {
1249	adev->gmc.requested_nps_mode = mode;
1250	}
1251
1252	dev_info(
1253	adev->dev,
1254	"NPS mode change requested, please remove and reload the driver\n");
1255
1256	return count;
1257	}
1258
1259	static ssize_t current_memory_partition_show(
1260	struct device *dev, struct device_attribute *addr, char *buf)
1261	{
1262	struct drm_device *ddev = dev_get_drvdata(dev);
1263	struct amdgpu_device *adev = drm_to_adev(ddev);
1264	enum amdgpu_memory_partition mode;
1265
1266	/* Only minimal precaution taken to reject requests while in reset */
1267	if (amdgpu_in_reset(adev))
1268	return -EPERM;
1269
1270	mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
1271	if ((mode >= ARRAY_SIZE(nps_desc)) ||
1272	(BIT(mode) & AMDGPU_ALL_NPS_MASK) != BIT(mode))
1273	return sysfs_emit(buf, fmt: "UNKNOWN\n");
1274
1275	return sysfs_emit(buf, fmt: "%s\n", nps_desc[mode]);
1276	}
1277
1278	static DEVICE_ATTR_RW(current_memory_partition);
1279	static DEVICE_ATTR_RO(available_memory_partition);
1280
1281	int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev)
1282	{
1283	bool nps_switch_support;
1284	int r = 0;
1285
1286	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
1287	return 0;
1288
1289	nps_switch_support = (hweight32(adev->gmc.supported_nps_modes &
1290	AMDGPU_ALL_NPS_MASK) > 1);
1291	if (!nps_switch_support)
1292	dev_attr_current_memory_partition.attr.mode &=
1293	~(S_IWUSR | S_IWGRP | S_IWOTH);
1294	else
1295	r = device_create_file(device: adev->dev,
1296	entry: &dev_attr_available_memory_partition);
1297
1298	if (r)
1299	return r;
1300
1301	return device_create_file(device: adev->dev,
1302	entry: &dev_attr_current_memory_partition);
1303	}
1304
1305	void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev)
1306	{
1307	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
1308	return;
1309
1310	device_remove_file(dev: adev->dev, attr: &dev_attr_current_memory_partition);
1311	device_remove_file(dev: adev->dev, attr: &dev_attr_available_memory_partition);
1312	}
1313
1314	int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev,
1315	struct amdgpu_mem_partition_info *mem_ranges,
1316	uint8_t *exp_ranges)
1317	{
1318	struct amdgpu_gmc_memrange *ranges;
1319	int range_cnt, ret, i, j;
1320	uint32_t nps_type;
1321	bool refresh;
1322
1323	if (!mem_ranges || !exp_ranges)
1324	return -EINVAL;
1325
1326	refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) &&
1327	(adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS);
1328	ret = amdgpu_discovery_get_nps_info(adev, nps_type: &nps_type, ranges: &ranges,
1329	range_cnt: &range_cnt, refresh);
1330
1331	if (ret)
1332	return ret;
1333
1334	/* TODO: For now, expect ranges and partition count to be the same.
1335	* Adjust if there are holes expected in any NPS domain.
1336	*/
1337	if (*exp_ranges && (range_cnt != *exp_ranges)) {
1338	dev_warn(
1339	adev->dev,
1340	"NPS config mismatch - expected ranges: %d discovery - nps mode: %d, nps ranges: %d",
1341	*exp_ranges, nps_type, range_cnt);
1342	ret = -EINVAL;
1343	goto err;
1344	}
1345
1346	for (i = 0; i < range_cnt; ++i) {
1347	if (ranges[i].base_address >= ranges[i].limit_address) {
1348	dev_warn(
1349	adev->dev,
1350	"Invalid NPS range - nps mode: %d, range[%d]: base: %llx limit: %llx",
1351	nps_type, i, ranges[i].base_address,
1352	ranges[i].limit_address);
1353	ret = -EINVAL;
1354	goto err;
1355	}
1356
1357	/* Check for overlaps, not expecting any now */
1358	for (j = i - 1; j >= 0; j--) {
1359	if (max(ranges[j].base_address,
1360	ranges[i].base_address) <=
1361	min(ranges[j].limit_address,
1362	ranges[i].limit_address)) {
1363	dev_warn(
1364	adev->dev,
1365	"overlapping ranges detected [ %llx - %llx ] | [%llx - %llx]",
1366	ranges[j].base_address,
1367	ranges[j].limit_address,
1368	ranges[i].base_address,
1369	ranges[i].limit_address);
1370	ret = -EINVAL;
1371	goto err;
1372	}
1373	}
1374
1375	mem_ranges[i].range.fpfn =
1376	(ranges[i].base_address -
1377	adev->vm_manager.vram_base_offset) >>
1378	AMDGPU_GPU_PAGE_SHIFT;
1379	mem_ranges[i].range.lpfn =
1380	(ranges[i].limit_address -
1381	adev->vm_manager.vram_base_offset) >>
1382	AMDGPU_GPU_PAGE_SHIFT;
1383	mem_ranges[i].size =
1384	ranges[i].limit_address - ranges[i].base_address + 1;
1385	}
1386
1387	if (!*exp_ranges)
1388	*exp_ranges = range_cnt;
1389	err:
1390	kfree(objp: ranges);
1391
1392	return ret;
1393	}
1394
1395	int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev,
1396	int nps_mode)
1397	{
1398	/* Not supported on VF devices and APUs */
1399	if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
1400	return -EOPNOTSUPP;
1401
1402	if (!adev->psp.funcs) {
1403	dev_err(adev->dev,
1404	"PSP interface not available for nps mode change request");
1405	return -EINVAL;
1406	}
1407
1408	return psp_memory_partition(psp: &adev->psp, mode: nps_mode);
1409	}
1410
1411	static inline bool amdgpu_gmc_need_nps_switch_req(struct amdgpu_device *adev,
1412	int req_nps_mode,
1413	int cur_nps_mode)
1414	{
1415	return (((BIT(req_nps_mode) & adev->gmc.supported_nps_modes) ==
1416	BIT(req_nps_mode)) &&
1417	req_nps_mode != cur_nps_mode);
1418	}
1419
1420	void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev)
1421	{
1422	int req_nps_mode, cur_nps_mode, r;
1423	struct amdgpu_hive_info *hive;
1424
1425	if (amdgpu_sriov_vf(adev) || !adev->gmc.supported_nps_modes ||
1426	!adev->gmc.gmc_funcs->request_mem_partition_mode)
1427	return;
1428
1429	cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
1430	hive = amdgpu_get_xgmi_hive(adev);
1431	if (hive) {
1432	req_nps_mode = atomic_read(v: &hive->requested_nps_mode);
1433	if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode,
1434	cur_nps_mode)) {
1435	amdgpu_put_xgmi_hive(hive);
1436	return;
1437	}
1438	r = amdgpu_xgmi_request_nps_change(adev, hive, req_nps_mode);
1439	amdgpu_put_xgmi_hive(hive);
1440	goto out;
1441	}
1442
1443	req_nps_mode = adev->gmc.requested_nps_mode;
1444	if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, cur_nps_mode))
1445	return;
1446
1447	/* even if this fails, we should let driver unload w/o blocking */
1448	r = adev->gmc.gmc_funcs->request_mem_partition_mode(adev, req_nps_mode);
1449	out:
1450	if (r)
1451	dev_err(adev->dev, "NPS mode change request failed\n");
1452	else
1453	dev_info(
1454	adev->dev,
1455	"NPS mode change request done, reload driver to complete the change\n");
1456	}
1457
1458	bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev)
1459	{
1460	if (adev->gmc.gmc_funcs->need_reset_on_init)
1461	return adev->gmc.gmc_funcs->need_reset_on_init(adev);
1462
1463	return false;
1464	}
1465
1466	enum amdgpu_memory_partition
1467	amdgpu_gmc_get_vf_memory_partition(struct amdgpu_device *adev)
1468	{
1469	switch (adev->gmc.num_mem_partitions) {
1470	case 0:
1471	return UNKNOWN_MEMORY_PARTITION_MODE;
1472	case 1:
1473	return AMDGPU_NPS1_PARTITION_MODE;
1474	case 2:
1475	return AMDGPU_NPS2_PARTITION_MODE;
1476	case 4:
1477	return AMDGPU_NPS4_PARTITION_MODE;
1478	case 8:
1479	return AMDGPU_NPS8_PARTITION_MODE;
1480	default:
1481	return AMDGPU_NPS1_PARTITION_MODE;
1482	}
1483	}
1484
1485	enum amdgpu_memory_partition
1486	amdgpu_gmc_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes)
1487	{
1488	enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE;
1489
1490	if (adev->nbio.funcs &&
1491	adev->nbio.funcs->get_memory_partition_mode)
1492	mode = adev->nbio.funcs->get_memory_partition_mode(adev,
1493	supp_modes);
1494	else
1495	dev_warn(adev->dev, "memory partition mode query is not supported\n");
1496
1497	return mode;
1498	}
1499
1500	enum amdgpu_memory_partition
1501	amdgpu_gmc_query_memory_partition(struct amdgpu_device *adev)
1502	{
1503	if (amdgpu_sriov_vf(adev))
1504	return amdgpu_gmc_get_vf_memory_partition(adev);
1505	else
1506	return amdgpu_gmc_get_memory_partition(adev, NULL);
1507	}
1508
1509	static bool amdgpu_gmc_validate_partition_info(struct amdgpu_device *adev)
1510	{
1511	enum amdgpu_memory_partition mode;
1512	u32 supp_modes;
1513	bool valid;
1514
1515	mode = amdgpu_gmc_get_memory_partition(adev, supp_modes: &supp_modes);
1516
1517	/* Mode detected by hardware not present in supported modes */
1518	if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) &&
1519	!(BIT(mode - 1) & supp_modes))
1520	return false;
1521
1522	switch (mode) {
1523	case UNKNOWN_MEMORY_PARTITION_MODE:
1524	case AMDGPU_NPS1_PARTITION_MODE:
1525	valid = (adev->gmc.num_mem_partitions == 1);
1526	break;
1527	case AMDGPU_NPS2_PARTITION_MODE:
1528	valid = (adev->gmc.num_mem_partitions == 2);
1529	break;
1530	case AMDGPU_NPS4_PARTITION_MODE:
1531	valid = (adev->gmc.num_mem_partitions == 3 ||
1532	adev->gmc.num_mem_partitions == 4);
1533	break;
1534	case AMDGPU_NPS8_PARTITION_MODE:
1535	valid = (adev->gmc.num_mem_partitions == 8);
1536	break;
1537	default:
1538	valid = false;
1539	}
1540
1541	return valid;
1542	}
1543
1544	static bool amdgpu_gmc_is_node_present(int *node_ids, int num_ids, int nid)
1545	{
1546	int i;
1547
1548	/* Check if node with id 'nid' is present in 'node_ids' array */
1549	for (i = 0; i < num_ids; ++i)
1550	if (node_ids[i] == nid)
1551	return true;
1552
1553	return false;
1554	}
1555
1556	static void
1557	amdgpu_gmc_init_acpi_mem_ranges(struct amdgpu_device *adev,
1558	struct amdgpu_mem_partition_info *mem_ranges)
1559	{
1560	struct amdgpu_numa_info numa_info;
1561	int node_ids[AMDGPU_MAX_MEM_RANGES];
1562	int num_ranges = 0, ret;
1563	int num_xcc, xcc_id;
1564	uint32_t xcc_mask;
1565
1566	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1567	xcc_mask = (1U << num_xcc) - 1;
1568
1569	for_each_inst(xcc_id, xcc_mask) {
1570	ret = amdgpu_acpi_get_mem_info(adev, xcc_id, numa_info: &numa_info);
1571	if (ret)
1572	continue;
1573
1574	if (numa_info.nid == NUMA_NO_NODE) {
1575	mem_ranges[0].size = numa_info.size;
1576	mem_ranges[0].numa.node = numa_info.nid;
1577	num_ranges = 1;
1578	break;
1579	}
1580
1581	if (amdgpu_gmc_is_node_present(node_ids, num_ids: num_ranges,
1582	nid: numa_info.nid))
1583	continue;
1584
1585	node_ids[num_ranges] = numa_info.nid;
1586	mem_ranges[num_ranges].numa.node = numa_info.nid;
1587	mem_ranges[num_ranges].size = numa_info.size;
1588	++num_ranges;
1589	}
1590
1591	adev->gmc.num_mem_partitions = num_ranges;
1592	}
1593
1594	void amdgpu_gmc_init_sw_mem_ranges(struct amdgpu_device *adev,
1595	struct amdgpu_mem_partition_info *mem_ranges)
1596	{
1597	enum amdgpu_memory_partition mode;
1598	u32 start_addr = 0, size;
1599	int i, r, l;
1600
1601	mode = amdgpu_gmc_query_memory_partition(adev);
1602
1603	switch (mode) {
1604	case UNKNOWN_MEMORY_PARTITION_MODE:
1605	adev->gmc.num_mem_partitions = 0;
1606	break;
1607	case AMDGPU_NPS1_PARTITION_MODE:
1608	adev->gmc.num_mem_partitions = 1;
1609	break;
1610	case AMDGPU_NPS2_PARTITION_MODE:
1611	adev->gmc.num_mem_partitions = 2;
1612	break;
1613	case AMDGPU_NPS4_PARTITION_MODE:
1614	if (adev->flags & AMD_IS_APU)
1615	adev->gmc.num_mem_partitions = 3;
1616	else
1617	adev->gmc.num_mem_partitions = 4;
1618	break;
1619	case AMDGPU_NPS8_PARTITION_MODE:
1620	adev->gmc.num_mem_partitions = 8;
1621	break;
1622	default:
1623	adev->gmc.num_mem_partitions = 1;
1624	break;
1625	}
1626
1627	/* Use NPS range info, if populated */
1628	r = amdgpu_gmc_get_nps_memranges(adev, mem_ranges,
1629	exp_ranges: &adev->gmc.num_mem_partitions);
1630	if (!r) {
1631	l = 0;
1632	for (i = 1; i < adev->gmc.num_mem_partitions; ++i) {
1633	if (mem_ranges[i].range.lpfn >
1634	mem_ranges[i - 1].range.lpfn)
1635	l = i;
1636	}
1637
1638	} else {
1639	if (!adev->gmc.num_mem_partitions) {
1640	dev_warn(adev->dev,
1641	"Not able to detect NPS mode, fall back to NPS1\n");
1642	adev->gmc.num_mem_partitions = 1;
1643	}
1644	/* Fallback to sw based calculation */
1645	size = (adev->gmc.real_vram_size + SZ_16M) >> AMDGPU_GPU_PAGE_SHIFT;
1646	size /= adev->gmc.num_mem_partitions;
1647
1648	for (i = 0; i < adev->gmc.num_mem_partitions; ++i) {
1649	mem_ranges[i].range.fpfn = start_addr;
1650	mem_ranges[i].size =
1651	((u64)size << AMDGPU_GPU_PAGE_SHIFT);
1652	mem_ranges[i].range.lpfn = start_addr + size - 1;
1653	start_addr += size;
1654	}
1655
1656	l = adev->gmc.num_mem_partitions - 1;
1657	}
1658
1659	/* Adjust the last one */
1660	mem_ranges[l].range.lpfn =
1661	(adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1;
1662	mem_ranges[l].size =
1663	adev->gmc.real_vram_size -
1664	((u64)mem_ranges[l].range.fpfn << AMDGPU_GPU_PAGE_SHIFT);
1665	}
1666
1667	int amdgpu_gmc_init_mem_ranges(struct amdgpu_device *adev)
1668	{
1669	bool valid;
1670
1671	adev->gmc.mem_partitions = kcalloc(AMDGPU_MAX_MEM_RANGES,
1672	sizeof(struct amdgpu_mem_partition_info),
1673	GFP_KERNEL);
1674	if (!adev->gmc.mem_partitions)
1675	return -ENOMEM;
1676
1677	if (adev->gmc.is_app_apu)
1678	amdgpu_gmc_init_acpi_mem_ranges(adev, mem_ranges: adev->gmc.mem_partitions);
1679	else
1680	amdgpu_gmc_init_sw_mem_ranges(adev, mem_ranges: adev->gmc.mem_partitions);
1681
1682	if (amdgpu_sriov_vf(adev))
1683	valid = true;
1684	else
1685	valid = amdgpu_gmc_validate_partition_info(adev);
1686	if (!valid) {
1687	/* TODO: handle invalid case */
1688	dev_warn(adev->dev,
1689	"Mem ranges not matching with hardware config\n");
1690	}
1691
1692	return 0;
1693	}
1694