1	/*
2	* Copyright 2016 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/firmware.h>
25	#include <linux/pci.h>
26
27	#include <drm/drm_cache.h>
28
29	#include "amdgpu.h"
30	#include "gmc_v9_0.h"
31	#include "amdgpu_atomfirmware.h"
32	#include "amdgpu_gem.h"
33
34	#include "gc/gc_9_0_sh_mask.h"
35	#include "dce/dce_12_0_offset.h"
36	#include "dce/dce_12_0_sh_mask.h"
37	#include "vega10_enum.h"
38	#include "mmhub/mmhub_1_0_offset.h"
39	#include "athub/athub_1_0_sh_mask.h"
40	#include "athub/athub_1_0_offset.h"
41	#include "oss/osssys_4_0_offset.h"
42
43	#include "soc15.h"
44	#include "soc15d.h"
45	#include "soc15_common.h"
46	#include "umc/umc_6_0_sh_mask.h"
47
48	#include "gfxhub_v1_0.h"
49	#include "mmhub_v1_0.h"
50	#include "athub_v1_0.h"
51	#include "gfxhub_v1_1.h"
52	#include "gfxhub_v1_2.h"
53	#include "mmhub_v9_4.h"
54	#include "mmhub_v1_7.h"
55	#include "mmhub_v1_8.h"
56	#include "umc_v6_1.h"
57	#include "umc_v6_0.h"
58	#include "umc_v6_7.h"
59	#include "umc_v12_0.h"
60	#include "hdp_v4_0.h"
61	#include "mca_v3_0.h"
62
63	#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
64
65	#include "amdgpu_ras.h"
66	#include "amdgpu_xgmi.h"
67
68	/* add these here since we already include dce12 headers and these are for DCN */
69	#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d
70	#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
71	#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0
72	#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10
73	#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL
74	#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L
75	#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d
76	#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2
77
78	#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2 0x05ea
79	#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX 2
80
81	static const char * const gfxhub_client_ids[] = {
82	"CB",
83	"DB",
84	"IA",
85	"WD",
86	"CPF",
87	"CPC",
88	"CPG",
89	"RLC",
90	"TCP",
91	"SQC (inst)",
92	"SQC (data)",
93	"SQG",
94	"PA",
95	};
96
97	static const char *mmhub_client_ids_raven[][2] = {
98	[0][0] = "MP1",
99	[1][0] = "MP0",
100	[2][0] = "VCN",
101	[3][0] = "VCNU",
102	[4][0] = "HDP",
103	[5][0] = "DCE",
104	[13][0] = "UTCL2",
105	[19][0] = "TLS",
106	[26][0] = "OSS",
107	[27][0] = "SDMA0",
108	[0][1] = "MP1",
109	[1][1] = "MP0",
110	[2][1] = "VCN",
111	[3][1] = "VCNU",
112	[4][1] = "HDP",
113	[5][1] = "XDP",
114	[6][1] = "DBGU0",
115	[7][1] = "DCE",
116	[8][1] = "DCEDWB0",
117	[9][1] = "DCEDWB1",
118	[26][1] = "OSS",
119	[27][1] = "SDMA0",
120	};
121
122	static const char *mmhub_client_ids_renoir[][2] = {
123	[0][0] = "MP1",
124	[1][0] = "MP0",
125	[2][0] = "HDP",
126	[4][0] = "DCEDMC",
127	[5][0] = "DCEVGA",
128	[13][0] = "UTCL2",
129	[19][0] = "TLS",
130	[26][0] = "OSS",
131	[27][0] = "SDMA0",
132	[28][0] = "VCN",
133	[29][0] = "VCNU",
134	[30][0] = "JPEG",
135	[0][1] = "MP1",
136	[1][1] = "MP0",
137	[2][1] = "HDP",
138	[3][1] = "XDP",
139	[6][1] = "DBGU0",
140	[7][1] = "DCEDMC",
141	[8][1] = "DCEVGA",
142	[9][1] = "DCEDWB",
143	[26][1] = "OSS",
144	[27][1] = "SDMA0",
145	[28][1] = "VCN",
146	[29][1] = "VCNU",
147	[30][1] = "JPEG",
148	};
149
150	static const char *mmhub_client_ids_vega10[][2] = {
151	[0][0] = "MP0",
152	[1][0] = "UVD",
153	[2][0] = "UVDU",
154	[3][0] = "HDP",
155	[13][0] = "UTCL2",
156	[14][0] = "OSS",
157	[15][0] = "SDMA1",
158	[32+0][0] = "VCE0",
159	[32+1][0] = "VCE0U",
160	[32+2][0] = "XDMA",
161	[32+3][0] = "DCE",
162	[32+4][0] = "MP1",
163	[32+14][0] = "SDMA0",
164	[0][1] = "MP0",
165	[1][1] = "UVD",
166	[2][1] = "UVDU",
167	[3][1] = "DBGU0",
168	[4][1] = "HDP",
169	[5][1] = "XDP",
170	[14][1] = "OSS",
171	[15][1] = "SDMA0",
172	[32+0][1] = "VCE0",
173	[32+1][1] = "VCE0U",
174	[32+2][1] = "XDMA",
175	[32+3][1] = "DCE",
176	[32+4][1] = "DCEDWB",
177	[32+5][1] = "MP1",
178	[32+6][1] = "DBGU1",
179	[32+14][1] = "SDMA1",
180	};
181
182	static const char *mmhub_client_ids_vega12[][2] = {
183	[0][0] = "MP0",
184	[1][0] = "VCE0",
185	[2][0] = "VCE0U",
186	[3][0] = "HDP",
187	[13][0] = "UTCL2",
188	[14][0] = "OSS",
189	[15][0] = "SDMA1",
190	[32+0][0] = "DCE",
191	[32+1][0] = "XDMA",
192	[32+2][0] = "UVD",
193	[32+3][0] = "UVDU",
194	[32+4][0] = "MP1",
195	[32+15][0] = "SDMA0",
196	[0][1] = "MP0",
197	[1][1] = "VCE0",
198	[2][1] = "VCE0U",
199	[3][1] = "DBGU0",
200	[4][1] = "HDP",
201	[5][1] = "XDP",
202	[14][1] = "OSS",
203	[15][1] = "SDMA0",
204	[32+0][1] = "DCE",
205	[32+1][1] = "DCEDWB",
206	[32+2][1] = "XDMA",
207	[32+3][1] = "UVD",
208	[32+4][1] = "UVDU",
209	[32+5][1] = "MP1",
210	[32+6][1] = "DBGU1",
211	[32+15][1] = "SDMA1",
212	};
213
214	static const char *mmhub_client_ids_vega20[][2] = {
215	[0][0] = "XDMA",
216	[1][0] = "DCE",
217	[2][0] = "VCE0",
218	[3][0] = "VCE0U",
219	[4][0] = "UVD",
220	[5][0] = "UVD1U",
221	[13][0] = "OSS",
222	[14][0] = "HDP",
223	[15][0] = "SDMA0",
224	[32+0][0] = "UVD",
225	[32+1][0] = "UVDU",
226	[32+2][0] = "MP1",
227	[32+3][0] = "MP0",
228	[32+12][0] = "UTCL2",
229	[32+14][0] = "SDMA1",
230	[0][1] = "XDMA",
231	[1][1] = "DCE",
232	[2][1] = "DCEDWB",
233	[3][1] = "VCE0",
234	[4][1] = "VCE0U",
235	[5][1] = "UVD1",
236	[6][1] = "UVD1U",
237	[7][1] = "DBGU0",
238	[8][1] = "XDP",
239	[13][1] = "OSS",
240	[14][1] = "HDP",
241	[15][1] = "SDMA0",
242	[32+0][1] = "UVD",
243	[32+1][1] = "UVDU",
244	[32+2][1] = "DBGU1",
245	[32+3][1] = "MP1",
246	[32+4][1] = "MP0",
247	[32+14][1] = "SDMA1",
248	};
249
250	static const char *mmhub_client_ids_arcturus[][2] = {
251	[0][0] = "DBGU1",
252	[1][0] = "XDP",
253	[2][0] = "MP1",
254	[14][0] = "HDP",
255	[171][0] = "JPEG",
256	[172][0] = "VCN",
257	[173][0] = "VCNU",
258	[203][0] = "JPEG1",
259	[204][0] = "VCN1",
260	[205][0] = "VCN1U",
261	[256][0] = "SDMA0",
262	[257][0] = "SDMA1",
263	[258][0] = "SDMA2",
264	[259][0] = "SDMA3",
265	[260][0] = "SDMA4",
266	[261][0] = "SDMA5",
267	[262][0] = "SDMA6",
268	[263][0] = "SDMA7",
269	[384][0] = "OSS",
270	[0][1] = "DBGU1",
271	[1][1] = "XDP",
272	[2][1] = "MP1",
273	[14][1] = "HDP",
274	[171][1] = "JPEG",
275	[172][1] = "VCN",
276	[173][1] = "VCNU",
277	[203][1] = "JPEG1",
278	[204][1] = "VCN1",
279	[205][1] = "VCN1U",
280	[256][1] = "SDMA0",
281	[257][1] = "SDMA1",
282	[258][1] = "SDMA2",
283	[259][1] = "SDMA3",
284	[260][1] = "SDMA4",
285	[261][1] = "SDMA5",
286	[262][1] = "SDMA6",
287	[263][1] = "SDMA7",
288	[384][1] = "OSS",
289	};
290
291	static const char *mmhub_client_ids_aldebaran[][2] = {
292	[2][0] = "MP1",
293	[3][0] = "MP0",
294	[32+1][0] = "DBGU_IO0",
295	[32+2][0] = "DBGU_IO2",
296	[32+4][0] = "MPIO",
297	[96+11][0] = "JPEG0",
298	[96+12][0] = "VCN0",
299	[96+13][0] = "VCNU0",
300	[128+11][0] = "JPEG1",
301	[128+12][0] = "VCN1",
302	[128+13][0] = "VCNU1",
303	[160+1][0] = "XDP",
304	[160+14][0] = "HDP",
305	[256+0][0] = "SDMA0",
306	[256+1][0] = "SDMA1",
307	[256+2][0] = "SDMA2",
308	[256+3][0] = "SDMA3",
309	[256+4][0] = "SDMA4",
310	[384+0][0] = "OSS",
311	[2][1] = "MP1",
312	[3][1] = "MP0",
313	[32+1][1] = "DBGU_IO0",
314	[32+2][1] = "DBGU_IO2",
315	[32+4][1] = "MPIO",
316	[96+11][1] = "JPEG0",
317	[96+12][1] = "VCN0",
318	[96+13][1] = "VCNU0",
319	[128+11][1] = "JPEG1",
320	[128+12][1] = "VCN1",
321	[128+13][1] = "VCNU1",
322	[160+1][1] = "XDP",
323	[160+14][1] = "HDP",
324	[256+0][1] = "SDMA0",
325	[256+1][1] = "SDMA1",
326	[256+2][1] = "SDMA2",
327	[256+3][1] = "SDMA3",
328	[256+4][1] = "SDMA4",
329	[384+0][1] = "OSS",
330	};
331
332	static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = {
333	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
334	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
335	};
336
337	static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = {
338	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
339	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
340	};
341
342	static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
343	(0x000143c0 + 0x00000000),
344	(0x000143c0 + 0x00000800),
345	(0x000143c0 + 0x00001000),
346	(0x000143c0 + 0x00001800),
347	(0x000543c0 + 0x00000000),
348	(0x000543c0 + 0x00000800),
349	(0x000543c0 + 0x00001000),
350	(0x000543c0 + 0x00001800),
351	(0x000943c0 + 0x00000000),
352	(0x000943c0 + 0x00000800),
353	(0x000943c0 + 0x00001000),
354	(0x000943c0 + 0x00001800),
355	(0x000d43c0 + 0x00000000),
356	(0x000d43c0 + 0x00000800),
357	(0x000d43c0 + 0x00001000),
358	(0x000d43c0 + 0x00001800),
359	(0x001143c0 + 0x00000000),
360	(0x001143c0 + 0x00000800),
361	(0x001143c0 + 0x00001000),
362	(0x001143c0 + 0x00001800),
363	(0x001543c0 + 0x00000000),
364	(0x001543c0 + 0x00000800),
365	(0x001543c0 + 0x00001000),
366	(0x001543c0 + 0x00001800),
367	(0x001943c0 + 0x00000000),
368	(0x001943c0 + 0x00000800),
369	(0x001943c0 + 0x00001000),
370	(0x001943c0 + 0x00001800),
371	(0x001d43c0 + 0x00000000),
372	(0x001d43c0 + 0x00000800),
373	(0x001d43c0 + 0x00001000),
374	(0x001d43c0 + 0x00001800),
375	};
376
377	static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
378	(0x000143e0 + 0x00000000),
379	(0x000143e0 + 0x00000800),
380	(0x000143e0 + 0x00001000),
381	(0x000143e0 + 0x00001800),
382	(0x000543e0 + 0x00000000),
383	(0x000543e0 + 0x00000800),
384	(0x000543e0 + 0x00001000),
385	(0x000543e0 + 0x00001800),
386	(0x000943e0 + 0x00000000),
387	(0x000943e0 + 0x00000800),
388	(0x000943e0 + 0x00001000),
389	(0x000943e0 + 0x00001800),
390	(0x000d43e0 + 0x00000000),
391	(0x000d43e0 + 0x00000800),
392	(0x000d43e0 + 0x00001000),
393	(0x000d43e0 + 0x00001800),
394	(0x001143e0 + 0x00000000),
395	(0x001143e0 + 0x00000800),
396	(0x001143e0 + 0x00001000),
397	(0x001143e0 + 0x00001800),
398	(0x001543e0 + 0x00000000),
399	(0x001543e0 + 0x00000800),
400	(0x001543e0 + 0x00001000),
401	(0x001543e0 + 0x00001800),
402	(0x001943e0 + 0x00000000),
403	(0x001943e0 + 0x00000800),
404	(0x001943e0 + 0x00001000),
405	(0x001943e0 + 0x00001800),
406	(0x001d43e0 + 0x00000000),
407	(0x001d43e0 + 0x00000800),
408	(0x001d43e0 + 0x00001000),
409	(0x001d43e0 + 0x00001800),
410	};
411
412	static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
413	struct amdgpu_irq_src *src,
414	unsigned int type,
415	enum amdgpu_interrupt_state state)
416	{
417	u32 bits, i, tmp, reg;
418
419	/* Devices newer then VEGA10/12 shall have these programming
420	* sequences performed by PSP BL
421	*/
422	if (adev->asic_type >= CHIP_VEGA20)
423	return 0;
424
425	bits = 0x7f;
426
427	switch (state) {
428	case AMDGPU_IRQ_STATE_DISABLE:
429	for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
430	reg = ecc_umc_mcumc_ctrl_addrs[i];
431	tmp = RREG32(reg);
432	tmp &= ~bits;
433	WREG32(reg, tmp);
434	}
435	for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
436	reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
437	tmp = RREG32(reg);
438	tmp &= ~bits;
439	WREG32(reg, tmp);
440	}
441	break;
442	case AMDGPU_IRQ_STATE_ENABLE:
443	for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
444	reg = ecc_umc_mcumc_ctrl_addrs[i];
445	tmp = RREG32(reg);
446	tmp |= bits;
447	WREG32(reg, tmp);
448	}
449	for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
450	reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
451	tmp = RREG32(reg);
452	tmp |= bits;
453	WREG32(reg, tmp);
454	}
455	break;
456	default:
457	break;
458	}
459
460	return 0;
461	}
462
463	static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
464	struct amdgpu_irq_src *src,
465	unsigned int type,
466	enum amdgpu_interrupt_state state)
467	{
468	struct amdgpu_vmhub *hub;
469	u32 tmp, reg, bits, i, j;
470
471	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
472	VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
473	VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
474	VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
475	VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
476	VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
477	VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
478
479	switch (state) {
480	case AMDGPU_IRQ_STATE_DISABLE:
481	for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
482	hub = &adev->vmhub[j];
483	for (i = 0; i < 16; i++) {
484	reg = hub->vm_context0_cntl + i;
485
486	/* This works because this interrupt is only
487	* enabled at init/resume and disabled in
488	* fini/suspend, so the overall state doesn't
489	* change over the course of suspend/resume.
490	*/
491	if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
492	continue;
493
494	if (j >= AMDGPU_MMHUB0(0))
495	tmp = RREG32_SOC15_IP(MMHUB, reg);
496	else
497	tmp = RREG32_XCC(reg, j);
498
499	tmp &= ~bits;
500
501	if (j >= AMDGPU_MMHUB0(0))
502	WREG32_SOC15_IP(MMHUB, reg, tmp);
503	else
504	WREG32_XCC(reg, tmp, j);
505	}
506	}
507	break;
508	case AMDGPU_IRQ_STATE_ENABLE:
509	for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
510	hub = &adev->vmhub[j];
511	for (i = 0; i < 16; i++) {
512	reg = hub->vm_context0_cntl + i;
513
514	/* This works because this interrupt is only
515	* enabled at init/resume and disabled in
516	* fini/suspend, so the overall state doesn't
517	* change over the course of suspend/resume.
518	*/
519	if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
520	continue;
521
522	if (j >= AMDGPU_MMHUB0(0))
523	tmp = RREG32_SOC15_IP(MMHUB, reg);
524	else
525	tmp = RREG32_XCC(reg, j);
526
527	tmp |= bits;
528
529	if (j >= AMDGPU_MMHUB0(0))
530	WREG32_SOC15_IP(MMHUB, reg, tmp);
531	else
532	WREG32_XCC(reg, tmp, j);
533	}
534	}
535	break;
536	default:
537	break;
538	}
539
540	return 0;
541	}
542
543	static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
544	struct amdgpu_irq_src *source,
545	struct amdgpu_iv_entry *entry)
546	{
547	bool retry_fault = !!(entry->src_data[1] &
548	AMDGPU_GMC9_FAULT_SOURCE_DATA_RETRY);
549	bool write_fault = !!(entry->src_data[1] &
550	AMDGPU_GMC9_FAULT_SOURCE_DATA_WRITE);
551	uint32_t status = 0, cid = 0, rw = 0, fed = 0;
552	struct amdgpu_task_info *task_info;
553	struct amdgpu_vmhub *hub;
554	const char *mmhub_cid;
555	const char *hub_name;
556	unsigned int vmhub;
557	u64 addr;
558	uint32_t cam_index = 0;
559	int ret, xcc_id = 0;
560	uint32_t node_id;
561
562	node_id = entry->node_id;
563
564	addr = (u64)entry->src_data[0] << 12;
565	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
566
567	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
568	hub_name = "mmhub0";
569	vmhub = AMDGPU_MMHUB0(node_id / 4);
570	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
571	hub_name = "mmhub1";
572	vmhub = AMDGPU_MMHUB1(0);
573	} else {
574	hub_name = "gfxhub0";
575	if (adev->gfx.funcs->ih_node_to_logical_xcc) {
576	xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
577	node_id);
578	if (xcc_id < 0)
579	xcc_id = 0;
580	}
581	vmhub = xcc_id;
582	}
583	hub = &adev->vmhub[vmhub];
584
585	if (retry_fault) {
586	if (adev->irq.retry_cam_enabled) {
587	/* Delegate it to a different ring if the hardware hasn't
588	* already done it.
589	*/
590	if (entry->ih == &adev->irq.ih) {
591	amdgpu_irq_delegate(adev, entry, num_dw: 8);
592	return 1;
593	}
594
595	cam_index = entry->src_data[2] & 0x3ff;
596
597	ret = amdgpu_vm_handle_fault(adev, pasid: entry->pasid, vmid: entry->vmid, node_id,
598	addr, ts: entry->timestamp, write_fault);
599	WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
600	if (ret)
601	return 1;
602	} else {
603	/* Process it onyl if it's the first fault for this address */
604	if (entry->ih != &adev->irq.ih_soft &&
605	amdgpu_gmc_filter_faults(adev, ih: entry->ih, addr, pasid: entry->pasid,
606	timestamp: entry->timestamp))
607	return 1;
608
609	/* Delegate it to a different ring if the hardware hasn't
610	* already done it.
611	*/
612	if (entry->ih == &adev->irq.ih) {
613	amdgpu_irq_delegate(adev, entry, num_dw: 8);
614	return 1;
615	}
616
617	/* Try to handle the recoverable page faults by filling page
618	* tables
619	*/
620	if (amdgpu_vm_handle_fault(adev, pasid: entry->pasid, vmid: entry->vmid, node_id,
621	addr, ts: entry->timestamp, write_fault))
622	return 1;
623	}
624	}
625
626	if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault))
627	return 1;
628
629	if (!printk_ratelimit())
630	return 0;
631
632	dev_err(adev->dev,
633	"[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name,
634	retry_fault ? "retry" : "no-retry",
635	entry->src_id, entry->ring_id, entry->vmid, entry->pasid);
636
637	task_info = amdgpu_vm_get_task_info_pasid(adev, pasid: entry->pasid);
638	if (task_info) {
639	amdgpu_vm_print_task_info(adev, task_info);
640	amdgpu_vm_put_task_info(task_info);
641	}
642
643	dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n",
644	addr, entry->client_id,
645	soc15_ih_clientid_name[entry->client_id]);
646
647	if (amdgpu_is_multi_aid(adev))
648	dev_err(adev->dev, " cookie node_id %d fault from die %s%d%s\n",
649	node_id, node_id % 4 == 3 ? "RSV" : "AID", node_id / 4,
650	node_id % 4 == 1 ? ".XCD0" : node_id % 4 == 2 ? ".XCD1" : "");
651
652	if (amdgpu_sriov_vf(adev))
653	return 0;
654
655	/*
656	* Issue a dummy read to wait for the status register to
657	* be updated to avoid reading an incorrect value due to
658	* the new fast GRBM interface.
659	*/
660	if ((entry->vmid_src == AMDGPU_GFXHUB(0)) &&
661	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) < IP_VERSION(9, 4, 2)))
662	RREG32(hub->vm_l2_pro_fault_status);
663
664	status = RREG32(hub->vm_l2_pro_fault_status);
665	cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
666	rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
667	fed = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
668
669	/* for fed error, kfd will handle it, return directly */
670	if (fed && amdgpu_ras_is_poison_mode_supported(adev) &&
671	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >= IP_VERSION(9, 4, 2)))
672	return 0;
673
674	/* Only print L2 fault status if the status register could be read and
675	* contains useful information
676	*/
677	if (!status)
678	return 0;
679
680	if (!amdgpu_sriov_vf(adev))
681	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
682
683	amdgpu_vm_update_fault_cache(adev, pasid: entry->pasid, addr, status, vmhub);
684
685	dev_err(adev->dev,
686	"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
687	status);
688	if (entry->vmid_src == AMDGPU_GFXHUB(0)) {
689	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
690	cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
691	gfxhub_client_ids[cid],
692	cid);
693	} else {
694	switch (amdgpu_ip_version(adev, ip: MMHUB_HWIP, inst: 0)) {
695	case IP_VERSION(9, 0, 0):
696	mmhub_cid = mmhub_client_ids_vega10[cid][rw];
697	break;
698	case IP_VERSION(9, 3, 0):
699	mmhub_cid = mmhub_client_ids_vega12[cid][rw];
700	break;
701	case IP_VERSION(9, 4, 0):
702	mmhub_cid = mmhub_client_ids_vega20[cid][rw];
703	break;
704	case IP_VERSION(9, 4, 1):
705	mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
706	break;
707	case IP_VERSION(9, 1, 0):
708	case IP_VERSION(9, 2, 0):
709	mmhub_cid = mmhub_client_ids_raven[cid][rw];
710	break;
711	case IP_VERSION(1, 5, 0):
712	case IP_VERSION(2, 4, 0):
713	mmhub_cid = mmhub_client_ids_renoir[cid][rw];
714	break;
715	case IP_VERSION(1, 8, 0):
716	case IP_VERSION(9, 4, 2):
717	mmhub_cid = mmhub_client_ids_aldebaran[cid][rw];
718	break;
719	default:
720	mmhub_cid = NULL;
721	break;
722	}
723	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
724	mmhub_cid ? mmhub_cid : "unknown", cid);
725	}
726	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
727	REG_GET_FIELD(status,
728	VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
729	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
730	REG_GET_FIELD(status,
731	VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
732	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
733	REG_GET_FIELD(status,
734	VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
735	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
736	REG_GET_FIELD(status,
737	VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
738	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
739	return 0;
740	}
741
742	static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
743	.set = gmc_v9_0_vm_fault_interrupt_state,
744	.process = gmc_v9_0_process_interrupt,
745	};
746
747
748	static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
749	.set = gmc_v9_0_ecc_interrupt_state,
750	.process = amdgpu_umc_process_ecc_irq,
751	};
752
753	static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
754	{
755	adev->gmc.vm_fault.num_types = 1;
756	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
757
758	if (!amdgpu_sriov_vf(adev) &&
759	!adev->gmc.xgmi.connected_to_cpu &&
760	!adev->gmc.is_app_apu) {
761	adev->gmc.ecc_irq.num_types = 1;
762	adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
763	}
764	}
765
766	static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
767	uint32_t flush_type)
768	{
769	u32 req = 0;
770
771	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
772	PER_VMID_INVALIDATE_REQ, 1 << vmid);
773	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
774	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
775	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
776	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
777	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
778	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
779	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
780	CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
781
782	return req;
783	}
784
785	/**
786	* gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
787	*
788	* @adev: amdgpu_device pointer
789	* @vmhub: vmhub type
790	*
791	*/
792	static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
793	uint32_t vmhub)
794	{
795	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2) ||
796	amdgpu_is_multi_aid(adev))
797	return false;
798
799	return ((vmhub == AMDGPU_MMHUB0(0) ||
800	vmhub == AMDGPU_MMHUB1(0)) &&
801	(!amdgpu_sriov_vf(adev)) &&
802	(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
803	(adev->apu_flags & AMD_APU_IS_PICASSO))));
804	}
805
806	static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
807	uint8_t vmid, uint16_t *p_pasid)
808	{
809	uint32_t value;
810
811	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
812	+ vmid);
813	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
814
815	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
816	}
817
818	/*
819	* GART
820	* VMID 0 is the physical GPU addresses as used by the kernel.
821	* VMIDs 1-15 are used for userspace clients and are handled
822	* by the amdgpu vm/hsa code.
823	*/
824
825	/**
826	* gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
827	*
828	* @adev: amdgpu_device pointer
829	* @vmid: vm instance to flush
830	* @vmhub: which hub to flush
831	* @flush_type: the flush type
832	*
833	* Flush the TLB for the requested page table using certain type.
834	*/
835	static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
836	uint32_t vmhub, uint32_t flush_type)
837	{
838	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
839	u32 j, inv_req, tmp, sem, req, ack, inst;
840	const unsigned int eng = 17;
841	struct amdgpu_vmhub *hub;
842
843	BUG_ON(vmhub >= AMDGPU_MAX_VMHUBS);
844
845	hub = &adev->vmhub[vmhub];
846	inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
847	sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng;
848	req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
849	ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
850
851	if (vmhub >= AMDGPU_MMHUB0(0))
852	inst = 0;
853	else
854	inst = vmhub;
855
856	/* This is necessary for SRIOV as well as for GFXOFF to function
857	* properly under bare metal
858	*/
859	if (adev->gfx.kiq[inst].ring.sched.ready &&
860	(amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
861	uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
862	uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
863
864	amdgpu_gmc_fw_reg_write_reg_wait(adev, reg0: req, reg1: ack, ref: inv_req,
865	mask: 1 << vmid, xcc_inst: inst);
866	return;
867	}
868
869	/* This path is needed before KIQ/MES/GFXOFF are set up */
870	spin_lock(lock: &adev->gmc.invalidate_lock);
871
872	/*
873	* It may lose gpuvm invalidate acknowldege state across power-gating
874	* off cycle, add semaphore acquire before invalidation and semaphore
875	* release after invalidation to avoid entering power gated state
876	* to WA the Issue
877	*/
878
879	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
880	if (use_semaphore) {
881	for (j = 0; j < adev->usec_timeout; j++) {
882	/* a read return value of 1 means semaphore acquire */
883	if (vmhub >= AMDGPU_MMHUB0(0))
884	tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, sem, GET_INST(GC, inst));
885	else
886	tmp = RREG32_SOC15_IP_NO_KIQ(GC, sem, GET_INST(GC, inst));
887	if (tmp & 0x1)
888	break;
889	udelay(usec: 1);
890	}
891
892	if (j >= adev->usec_timeout)
893	DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
894	}
895
896	if (vmhub >= AMDGPU_MMHUB0(0))
897	WREG32_SOC15_IP_NO_KIQ(MMHUB, req, inv_req, GET_INST(GC, inst));
898	else
899	WREG32_SOC15_IP_NO_KIQ(GC, req, inv_req, GET_INST(GC, inst));
900
901	/*
902	* Issue a dummy read to wait for the ACK register to
903	* be cleared to avoid a false ACK due to the new fast
904	* GRBM interface.
905	*/
906	if ((vmhub == AMDGPU_GFXHUB(0)) &&
907	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) < IP_VERSION(9, 4, 2)))
908	RREG32_NO_KIQ(req);
909
910	for (j = 0; j < adev->usec_timeout; j++) {
911	if (vmhub >= AMDGPU_MMHUB0(0))
912	tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, ack, GET_INST(GC, inst));
913	else
914	tmp = RREG32_SOC15_IP_NO_KIQ(GC, ack, GET_INST(GC, inst));
915	if (tmp & (1 << vmid))
916	break;
917	udelay(usec: 1);
918	}
919
920	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
921	if (use_semaphore) {
922	/*
923	* add semaphore release after invalidation,
924	* write with 0 means semaphore release
925	*/
926	if (vmhub >= AMDGPU_MMHUB0(0))
927	WREG32_SOC15_IP_NO_KIQ(MMHUB, sem, 0, GET_INST(GC, inst));
928	else
929	WREG32_SOC15_IP_NO_KIQ(GC, sem, 0, GET_INST(GC, inst));
930	}
931
932	spin_unlock(lock: &adev->gmc.invalidate_lock);
933
934	if (j < adev->usec_timeout)
935	return;
936
937	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
938	}
939
940	/**
941	* gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
942	*
943	* @adev: amdgpu_device pointer
944	* @pasid: pasid to be flush
945	* @flush_type: the flush type
946	* @all_hub: flush all hubs
947	* @inst: is used to select which instance of KIQ to use for the invalidation
948	*
949	* Flush the TLB for the requested pasid.
950	*/
951	static void gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
952	uint16_t pasid, uint32_t flush_type,
953	bool all_hub, uint32_t inst)
954	{
955	uint16_t queried;
956	int i, vmid;
957
958	for (vmid = 1; vmid < 16; vmid++) {
959	bool valid;
960
961	valid = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
962	p_pasid: &queried);
963	if (!valid || queried != pasid)
964	continue;
965
966	if (all_hub) {
967	for_each_set_bit(i, adev->vmhubs_mask,
968	AMDGPU_MAX_VMHUBS)
969	gmc_v9_0_flush_gpu_tlb(adev, vmid, vmhub: i,
970	flush_type);
971	} else {
972	gmc_v9_0_flush_gpu_tlb(adev, vmid,
973	AMDGPU_GFXHUB(0),
974	flush_type);
975	}
976	}
977	}
978
979	static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
980	unsigned int vmid, uint64_t pd_addr)
981	{
982	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev: ring->adev, vmhub: ring->vm_hub);
983	struct amdgpu_device *adev = ring->adev;
984	struct amdgpu_vmhub *hub = &adev->vmhub[ring->vm_hub];
985	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, flush_type: 0);
986	unsigned int eng = ring->vm_inv_eng;
987
988	/*
989	* It may lose gpuvm invalidate acknowldege state across power-gating
990	* off cycle, add semaphore acquire before invalidation and semaphore
991	* release after invalidation to avoid entering power gated state
992	* to WA the Issue
993	*/
994
995	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
996	if (use_semaphore)
997	/* a read return value of 1 means semaphore acuqire */
998	amdgpu_ring_emit_reg_wait(ring,
999	hub->vm_inv_eng0_sem +
1000	hub->eng_distance * eng, 0x1, 0x1);
1001
1002	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
1003	(hub->ctx_addr_distance * vmid),
1004	lower_32_bits(pd_addr));
1005
1006	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
1007	(hub->ctx_addr_distance * vmid),
1008	upper_32_bits(pd_addr));
1009
1010	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
1011	hub->eng_distance * eng,
1012	hub->vm_inv_eng0_ack +
1013	hub->eng_distance * eng,
1014	req, 1 << vmid);
1015
1016	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
1017	if (use_semaphore)
1018	/*
1019	* add semaphore release after invalidation,
1020	* write with 0 means semaphore release
1021	*/
1022	amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
1023	hub->eng_distance * eng, 0);
1024
1025	return pd_addr;
1026	}
1027
1028	static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,
1029	unsigned int pasid)
1030	{
1031	struct amdgpu_device *adev = ring->adev;
1032	uint32_t reg;
1033
1034	/* Do nothing because there's no lut register for mmhub1. */
1035	if (ring->vm_hub == AMDGPU_MMHUB1(0))
1036	return;
1037
1038	if (ring->vm_hub == AMDGPU_GFXHUB(0))
1039	reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
1040	else
1041	reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
1042
1043	amdgpu_ring_emit_wreg(ring, reg, pasid);
1044	}
1045
1046	/*
1047	* PTE format on VEGA 10:
1048	* 63:59 reserved
1049	* 58:57 mtype
1050	* 56 F
1051	* 55 L
1052	* 54 P
1053	* 53 SW
1054	* 52 T
1055	* 50:48 reserved
1056	* 47:12 4k physical page base address
1057	* 11:7 fragment
1058	* 6 write
1059	* 5 read
1060	* 4 exe
1061	* 3 Z
1062	* 2 snooped
1063	* 1 system
1064	* 0 valid
1065	*
1066	* PDE format on VEGA 10:
1067	* 63:59 block fragment size
1068	* 58:55 reserved
1069	* 54 P
1070	* 53:48 reserved
1071	* 47:6 physical base address of PD or PTE
1072	* 5:3 reserved
1073	* 2 C
1074	* 1 system
1075	* 0 valid
1076	*/
1077
1078	static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1079	uint64_t *addr, uint64_t *flags)
1080	{
1081	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1082	*addr = amdgpu_gmc_vram_mc2pa(adev, mc_addr: *addr);
1083	BUG_ON(*addr & 0xFFFF00000000003FULL);
1084
1085	if (!adev->gmc.translate_further)
1086	return;
1087
1088	if (level == AMDGPU_VM_PDB1) {
1089	/* Set the block fragment size */
1090	if (!(*flags & AMDGPU_PDE_PTE))
1091	*flags |= AMDGPU_PDE_BFS(0x9);
1092
1093	} else if (level == AMDGPU_VM_PDB0) {
1094	if (*flags & AMDGPU_PDE_PTE) {
1095	*flags &= ~AMDGPU_PDE_PTE;
1096	if (!(*flags & AMDGPU_PTE_VALID))
1097	*addr |= 1 << PAGE_SHIFT;
1098	} else {
1099	*flags |= AMDGPU_PTE_TF;
1100	}
1101	}
1102	}
1103
1104	static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
1105	struct amdgpu_vm *vm,
1106	struct amdgpu_bo *bo,
1107	uint32_t vm_flags,
1108	uint64_t *flags)
1109	{
1110	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bdev: bo->tbo.bdev);
1111	bool is_vram = bo->tbo.resource &&
1112	bo->tbo.resource->mem_type == TTM_PL_VRAM;
1113	bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
1114	AMDGPU_GEM_CREATE_EXT_COHERENT);
1115	bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT;
1116	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
1117	unsigned int mtype_local, mtype;
1118	uint32_t gc_ip_version = amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0);
1119	bool snoop = false;
1120	bool is_local;
1121
1122	dma_resv_assert_held(bo->tbo.base.resv);
1123
1124	switch (gc_ip_version) {
1125	case IP_VERSION(9, 4, 1):
1126	case IP_VERSION(9, 4, 2):
1127	if (is_vram) {
1128	if (bo_adev == adev) {
1129	if (uncached)
1130	mtype = MTYPE_UC;
1131	else if (coherent)
1132	mtype = MTYPE_CC;
1133	else
1134	mtype = MTYPE_RW;
1135	/* FIXME: is this still needed? Or does
1136	* amdgpu_ttm_tt_pde_flags already handle this?
1137	*/
1138	if (gc_ip_version == IP_VERSION(9, 4, 2) &&
1139	adev->gmc.xgmi.connected_to_cpu)
1140	snoop = true;
1141	} else {
1142	if (uncached || coherent)
1143	mtype = MTYPE_UC;
1144	else
1145	mtype = MTYPE_NC;
1146	if (amdgpu_xgmi_same_hive(adev, bo_adev))
1147	snoop = true;
1148	}
1149	} else {
1150	if (uncached || coherent)
1151	mtype = MTYPE_UC;
1152	else
1153	mtype = MTYPE_NC;
1154	/* FIXME: is this still needed? Or does
1155	* amdgpu_ttm_tt_pde_flags already handle this?
1156	*/
1157	snoop = true;
1158	}
1159	break;
1160	case IP_VERSION(9, 4, 3):
1161	case IP_VERSION(9, 4, 4):
1162	case IP_VERSION(9, 5, 0):
1163	/* Only local VRAM BOs or system memory on non-NUMA APUs
1164	* can be assumed to be local in their entirety. Choose
1165	* MTYPE_NC as safe fallback for all system memory BOs on
1166	* NUMA systems. Their MTYPE can be overridden per-page in
1167	* gmc_v9_0_override_vm_pte_flags.
1168	*/
1169	mtype_local = MTYPE_RW;
1170	if (amdgpu_mtype_local == 1) {
1171	DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
1172	mtype_local = MTYPE_NC;
1173	} else if (amdgpu_mtype_local == 2) {
1174	DRM_INFO_ONCE("Using MTYPE_CC for local memory\n");
1175	mtype_local = MTYPE_CC;
1176	} else {
1177	DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
1178	}
1179	is_local = (!is_vram && (adev->flags & AMD_IS_APU) &&
1180	num_possible_nodes() <= 1) ||
1181	(is_vram && adev == bo_adev &&
1182	KFD_XCP_MEM_ID(adev, bo->xcp_id) == vm->mem_id);
1183	snoop = true;
1184	if (uncached) {
1185	mtype = MTYPE_UC;
1186	} else if (ext_coherent) {
1187	mtype = is_local ? MTYPE_CC : MTYPE_UC;
1188	} else if (adev->flags & AMD_IS_APU) {
1189	mtype = is_local ? mtype_local : MTYPE_NC;
1190	} else {
1191	/* dGPU */
1192	if (is_local)
1193	mtype = mtype_local;
1194	else if (gc_ip_version < IP_VERSION(9, 5, 0) && !is_vram)
1195	mtype = MTYPE_UC;
1196	else
1197	mtype = MTYPE_NC;
1198	}
1199
1200	break;
1201	default:
1202	if (uncached || coherent)
1203	mtype = MTYPE_UC;
1204	else
1205	mtype = MTYPE_NC;
1206
1207	/* FIXME: is this still needed? Or does
1208	* amdgpu_ttm_tt_pde_flags already handle this?
1209	*/
1210	if (!is_vram)
1211	snoop = true;
1212	}
1213
1214	if (mtype != MTYPE_NC)
1215	*flags = AMDGPU_PTE_MTYPE_VG10(*flags, mtype);
1216
1217	*flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
1218	}
1219
1220	static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1221	struct amdgpu_vm *vm,
1222	struct amdgpu_bo *bo,
1223	uint32_t vm_flags,
1224	uint64_t *flags)
1225	{
1226	if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
1227	*flags |= AMDGPU_PTE_EXECUTABLE;
1228	else
1229	*flags &= ~AMDGPU_PTE_EXECUTABLE;
1230
1231	switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
1232	case AMDGPU_VM_MTYPE_DEFAULT:
1233	case AMDGPU_VM_MTYPE_NC:
1234	default:
1235	*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_NC);
1236	break;
1237	case AMDGPU_VM_MTYPE_WC:
1238	*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_WC);
1239	break;
1240	case AMDGPU_VM_MTYPE_RW:
1241	*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_RW);
1242	break;
1243	case AMDGPU_VM_MTYPE_CC:
1244	*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_CC);
1245	break;
1246	case AMDGPU_VM_MTYPE_UC:
1247	*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_UC);
1248	break;
1249	}
1250
1251	if (vm_flags & AMDGPU_VM_PAGE_PRT) {
1252	*flags |= AMDGPU_PTE_PRT;
1253	*flags &= ~AMDGPU_PTE_VALID;
1254	}
1255
1256	if ((*flags & AMDGPU_PTE_VALID) && bo)
1257	gmc_v9_0_get_coherence_flags(adev, vm, bo, vm_flags, flags);
1258	}
1259
1260	static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
1261	struct amdgpu_vm *vm,
1262	uint64_t addr, uint64_t *flags)
1263	{
1264	int local_node, nid;
1265
1266	/* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system
1267	* memory can use more efficient MTYPEs.
1268	*/
1269	if (!(adev->flags & AMD_IS_APU) ||
1270	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) != IP_VERSION(9, 4, 3))
1271	return;
1272
1273	/* Only direct-mapped memory allows us to determine the NUMA node from
1274	* the DMA address.
1275	*/
1276	if (!adev->ram_is_direct_mapped) {
1277	dev_dbg_ratelimited(adev->dev, "RAM is not direct mapped\n");
1278	return;
1279	}
1280
1281	/* MTYPE_NC is the same default and can be overridden.
1282	* MTYPE_UC will be present if the memory is extended-coherent
1283	* and can also be overridden.
1284	*/
1285	if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1286	AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_NC) &&
1287	(*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1288	AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_UC)) {
1289	dev_dbg_ratelimited(adev->dev, "MTYPE is not NC or UC\n");
1290	return;
1291	}
1292
1293	/* FIXME: Only supported on native mode for now. For carve-out, the
1294	* NUMA affinity of the GPU/VM needs to come from the PCI info because
1295	* memory partitions are not associated with different NUMA nodes.
1296	*/
1297	if (adev->gmc.is_app_apu && vm->mem_id >= 0) {
1298	local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
1299	} else {
1300	dev_dbg_ratelimited(adev->dev, "Only native mode APU is supported.\n");
1301	return;
1302	}
1303
1304	/* Only handle real RAM. Mappings of PCIe resources don't have struct
1305	* page or NUMA nodes.
1306	*/
1307	if (!page_is_ram(pfn: addr >> PAGE_SHIFT)) {
1308	dev_dbg_ratelimited(adev->dev, "Page is not RAM.\n");
1309	return;
1310	}
1311	nid = pfn_to_nid(addr >> PAGE_SHIFT);
1312	dev_dbg_ratelimited(adev->dev, "vm->mem_id=%d, local_node=%d, nid=%d\n",
1313	vm->mem_id, local_node, nid);
1314	if (nid == local_node) {
1315	uint64_t old_flags = *flags;
1316	if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) ==
1317	AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_NC)) {
1318	unsigned int mtype_local = MTYPE_RW;
1319
1320	if (amdgpu_mtype_local == 1)
1321	mtype_local = MTYPE_NC;
1322	else if (amdgpu_mtype_local == 2)
1323	mtype_local = MTYPE_CC;
1324
1325	*flags = AMDGPU_PTE_MTYPE_VG10(*flags, mtype_local);
1326	} else {
1327	/* MTYPE_UC case */
1328	*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_CC);
1329	}
1330
1331	dev_dbg_ratelimited(adev->dev, "flags updated from %llx to %llx\n",
1332	old_flags, *flags);
1333	}
1334	}
1335
1336	static unsigned int gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1337	{
1338	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1339	unsigned int size;
1340
1341	/* TODO move to DC so GMC doesn't need to hard-code DCN registers */
1342
1343	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1344	size = AMDGPU_VBIOS_VGA_ALLOCATION;
1345	} else {
1346	u32 viewport;
1347
1348	switch (amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0)) {
1349	case IP_VERSION(1, 0, 0):
1350	case IP_VERSION(1, 0, 1):
1351	viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1352	size = (REG_GET_FIELD(viewport,
1353	HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1354	REG_GET_FIELD(viewport,
1355	HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1356	4);
1357	break;
1358	case IP_VERSION(2, 1, 0):
1359	viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2);
1360	size = (REG_GET_FIELD(viewport,
1361	HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1362	REG_GET_FIELD(viewport,
1363	HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1364	4);
1365	break;
1366	default:
1367	viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1368	size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1369	REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1370	4);
1371	break;
1372	}
1373	}
1374
1375	return size;
1376	}
1377
1378	static bool gmc_v9_0_need_reset_on_init(struct amdgpu_device *adev)
1379	{
1380	if (adev->nbio.funcs && adev->nbio.funcs->is_nps_switch_requested &&
1381	adev->nbio.funcs->is_nps_switch_requested(adev)) {
1382	adev->gmc.reset_flags |= AMDGPU_GMC_INIT_RESET_NPS;
1383	return true;
1384	}
1385
1386	return false;
1387	}
1388
1389	static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1390	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1391	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1392	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1393	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1394	.get_vm_pde = gmc_v9_0_get_vm_pde,
1395	.get_vm_pte = gmc_v9_0_get_vm_pte,
1396	.override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,
1397	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1398	.query_mem_partition_mode = &amdgpu_gmc_query_memory_partition,
1399	.request_mem_partition_mode = &amdgpu_gmc_request_memory_partition,
1400	.need_reset_on_init = &gmc_v9_0_need_reset_on_init,
1401	};
1402
1403	static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1404	{
1405	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1406	}
1407
1408	static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1409	{
1410	switch (amdgpu_ip_version(adev, ip: UMC_HWIP, inst: 0)) {
1411	case IP_VERSION(6, 0, 0):
1412	adev->umc.funcs = &umc_v6_0_funcs;
1413	break;
1414	case IP_VERSION(6, 1, 1):
1415	adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1416	adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1417	adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1418	adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1419	adev->umc.retire_unit = 1;
1420	adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1421	adev->umc.ras = &umc_v6_1_ras;
1422	break;
1423	case IP_VERSION(6, 1, 2):
1424	adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1425	adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1426	adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1427	adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1428	adev->umc.retire_unit = 1;
1429	adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1430	adev->umc.ras = &umc_v6_1_ras;
1431	break;
1432	case IP_VERSION(6, 7, 0):
1433	adev->umc.max_ras_err_cnt_per_query =
1434	UMC_V6_7_TOTAL_CHANNEL_NUM * UMC_V6_7_BAD_PAGE_NUM_PER_CHANNEL;
1435	adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM;
1436	adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM;
1437	adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET;
1438	adev->umc.retire_unit = (UMC_V6_7_NA_MAP_PA_NUM * 2);
1439	if (!adev->gmc.xgmi.connected_to_cpu)
1440	adev->umc.ras = &umc_v6_7_ras;
1441	if (1 & adev->smuio.funcs->get_die_id(adev))
1442	adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0];
1443	else
1444	adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0];
1445	break;
1446	case IP_VERSION(12, 0, 0):
1447	case IP_VERSION(12, 5, 0):
1448	adev->umc.max_ras_err_cnt_per_query =
1449	UMC_V12_0_TOTAL_CHANNEL_NUM(adev) * UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL;
1450	adev->umc.channel_inst_num = UMC_V12_0_CHANNEL_INSTANCE_NUM;
1451	adev->umc.umc_inst_num = UMC_V12_0_UMC_INSTANCE_NUM;
1452	adev->umc.node_inst_num /= UMC_V12_0_UMC_INSTANCE_NUM;
1453	adev->umc.channel_offs = UMC_V12_0_PER_CHANNEL_OFFSET;
1454	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu)
1455	adev->umc.ras = &umc_v12_0_ras;
1456	break;
1457	default:
1458	break;
1459	}
1460	}
1461
1462	static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1463	{
1464	switch (amdgpu_ip_version(adev, ip: MMHUB_HWIP, inst: 0)) {
1465	case IP_VERSION(9, 4, 1):
1466	adev->mmhub.funcs = &mmhub_v9_4_funcs;
1467	break;
1468	case IP_VERSION(9, 4, 2):
1469	adev->mmhub.funcs = &mmhub_v1_7_funcs;
1470	break;
1471	case IP_VERSION(1, 8, 0):
1472	case IP_VERSION(1, 8, 1):
1473	adev->mmhub.funcs = &mmhub_v1_8_funcs;
1474	break;
1475	default:
1476	adev->mmhub.funcs = &mmhub_v1_0_funcs;
1477	break;
1478	}
1479	}
1480
1481	static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
1482	{
1483	switch (amdgpu_ip_version(adev, ip: MMHUB_HWIP, inst: 0)) {
1484	case IP_VERSION(9, 4, 0):
1485	adev->mmhub.ras = &mmhub_v1_0_ras;
1486	break;
1487	case IP_VERSION(9, 4, 1):
1488	adev->mmhub.ras = &mmhub_v9_4_ras;
1489	break;
1490	case IP_VERSION(9, 4, 2):
1491	adev->mmhub.ras = &mmhub_v1_7_ras;
1492	break;
1493	case IP_VERSION(1, 8, 0):
1494	case IP_VERSION(1, 8, 1):
1495	adev->mmhub.ras = &mmhub_v1_8_ras;
1496	break;
1497	default:
1498	/* mmhub ras is not available */
1499	break;
1500	}
1501	}
1502
1503	static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1504	{
1505	if (amdgpu_is_multi_aid(adev))
1506	adev->gfxhub.funcs = &gfxhub_v1_2_funcs;
1507	else
1508	adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
1509	}
1510
1511	static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
1512	{
1513	adev->hdp.ras = &hdp_v4_0_ras;
1514	}
1515
1516	static void gmc_v9_0_set_mca_ras_funcs(struct amdgpu_device *adev)
1517	{
1518	struct amdgpu_mca *mca = &adev->mca;
1519
1520	/* is UMC the right IP to check for MCA? Maybe DF? */
1521	switch (amdgpu_ip_version(adev, ip: UMC_HWIP, inst: 0)) {
1522	case IP_VERSION(6, 7, 0):
1523	if (!adev->gmc.xgmi.connected_to_cpu) {
1524	mca->mp0.ras = &mca_v3_0_mp0_ras;
1525	mca->mp1.ras = &mca_v3_0_mp1_ras;
1526	mca->mpio.ras = &mca_v3_0_mpio_ras;
1527	}
1528	break;
1529	default:
1530	break;
1531	}
1532	}
1533
1534	static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev)
1535	{
1536	if (!adev->gmc.xgmi.connected_to_cpu)
1537	adev->gmc.xgmi.ras = &xgmi_ras;
1538	}
1539
1540	static void gmc_v9_0_init_nps_details(struct amdgpu_device *adev)
1541	{
1542	enum amdgpu_memory_partition mode;
1543	uint32_t supp_modes;
1544	int i;
1545
1546	adev->gmc.supported_nps_modes = 0;
1547
1548	if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
1549	return;
1550
1551	mode = amdgpu_gmc_get_memory_partition(adev, supp_modes: &supp_modes);
1552
1553	/* Mode detected by hardware and supported modes available */
1554	if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && supp_modes) {
1555	while ((i = ffs(supp_modes))) {
1556	if (AMDGPU_ALL_NPS_MASK & BIT(i))
1557	adev->gmc.supported_nps_modes |= BIT(i);
1558	supp_modes &= supp_modes - 1;
1559	}
1560	} else {
1561	/*TODO: Check PSP version also which supports NPS switch. Otherwise keep
1562	* supported modes as 0.
1563	*/
1564	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1565	case IP_VERSION(9, 4, 3):
1566	case IP_VERSION(9, 4, 4):
1567	adev->gmc.supported_nps_modes =
1568	BIT(AMDGPU_NPS1_PARTITION_MODE) |
1569	BIT(AMDGPU_NPS4_PARTITION_MODE);
1570	break;
1571	default:
1572	break;
1573	}
1574	}
1575	}
1576
1577	static int gmc_v9_0_early_init(struct amdgpu_ip_block *ip_block)
1578	{
1579	struct amdgpu_device *adev = ip_block->adev;
1580
1581	/*
1582	* 9.4.0, 9.4.1 and 9.4.3 don't have XGMI defined
1583	* in their IP discovery tables
1584	*/
1585	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 0) ||
1586	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1) ||
1587	amdgpu_is_multi_aid(adev))
1588	adev->gmc.xgmi.supported = true;
1589
1590	if (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0) == IP_VERSION(6, 1, 0)) {
1591	adev->gmc.xgmi.supported = true;
1592	adev->gmc.xgmi.connected_to_cpu =
1593	adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
1594	}
1595
1596	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3)) {
1597	enum amdgpu_pkg_type pkg_type =
1598	adev->smuio.funcs->get_pkg_type(adev);
1599	/* On GFXIP 9.4.3. APU, there is no physical VRAM domain present
1600	* and the APU, can be in used two possible modes:
1601	* - carveout mode
1602	* - native APU mode
1603	* "is_app_apu" can be used to identify the APU in the native
1604	* mode.
1605	*/
1606	adev->gmc.is_app_apu = (pkg_type == AMDGPU_PKG_TYPE_APU &&
1607	!pci_resource_len(adev->pdev, 0));
1608	}
1609
1610	gmc_v9_0_set_gmc_funcs(adev);
1611	gmc_v9_0_set_irq_funcs(adev);
1612	gmc_v9_0_set_umc_funcs(adev);
1613	gmc_v9_0_set_mmhub_funcs(adev);
1614	gmc_v9_0_set_mmhub_ras_funcs(adev);
1615	gmc_v9_0_set_gfxhub_funcs(adev);
1616	gmc_v9_0_set_hdp_ras_funcs(adev);
1617	gmc_v9_0_set_mca_ras_funcs(adev);
1618	gmc_v9_0_set_xgmi_ras_funcs(adev);
1619
1620	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1621	adev->gmc.shared_aperture_end =
1622	adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1623	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1624	adev->gmc.private_aperture_end =
1625	adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1626	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
1627
1628	return 0;
1629	}
1630
1631	static int gmc_v9_0_late_init(struct amdgpu_ip_block *ip_block)
1632	{
1633	struct amdgpu_device *adev = ip_block->adev;
1634	int r;
1635
1636	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1637	if (r)
1638	return r;
1639
1640	/*
1641	* Workaround performance drop issue with VBIOS enables partial
1642	* writes, while disables HBM ECC for vega10.
1643	*/
1644	if (!amdgpu_sriov_vf(adev) &&
1645	(amdgpu_ip_version(adev, ip: UMC_HWIP, inst: 0) == IP_VERSION(6, 0, 0))) {
1646	if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1647	if (adev->df.funcs &&
1648	adev->df.funcs->enable_ecc_force_par_wr_rmw)
1649	adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1650	}
1651	}
1652
1653	if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1654	amdgpu_ras_reset_error_count(adev, block: AMDGPU_RAS_BLOCK__MMHUB);
1655	amdgpu_ras_reset_error_count(adev, block: AMDGPU_RAS_BLOCK__HDP);
1656	}
1657
1658	r = amdgpu_gmc_ras_late_init(adev);
1659	if (r)
1660	return r;
1661
1662	return amdgpu_irq_get(adev, src: &adev->gmc.vm_fault, type: 0);
1663	}
1664
1665	static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1666	struct amdgpu_gmc *mc)
1667	{
1668	u64 base = adev->mmhub.funcs->get_fb_location(adev);
1669
1670	amdgpu_gmc_set_agp_default(adev, mc);
1671
1672	/* add the xgmi offset of the physical node */
1673	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1674	if (amdgpu_gmc_is_pdb0_enabled(adev)) {
1675	amdgpu_gmc_sysvm_location(adev, mc);
1676	} else {
1677	amdgpu_gmc_vram_location(adev, mc, base);
1678	amdgpu_gmc_gart_location(adev, mc, gart_placement: AMDGPU_GART_PLACEMENT_BEST_FIT);
1679	if (!amdgpu_sriov_vf(adev) && (amdgpu_agp == 1))
1680	amdgpu_gmc_agp_location(adev, mc);
1681	}
1682	/* base offset of vram pages */
1683	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1684
1685	/* XXX: add the xgmi offset of the physical node? */
1686	adev->vm_manager.vram_base_offset +=
1687	adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1688	}
1689
1690	/**
1691	* gmc_v9_0_mc_init - initialize the memory controller driver params
1692	*
1693	* @adev: amdgpu_device pointer
1694	*
1695	* Look up the amount of vram, vram width, and decide how to place
1696	* vram and gart within the GPU's physical address space.
1697	* Returns 0 for success.
1698	*/
1699	static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1700	{
1701	int r;
1702
1703	/* size in MB on si */
1704	if (!adev->gmc.is_app_apu) {
1705	adev->gmc.mc_vram_size =
1706	adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1707	} else {
1708	DRM_DEBUG("Set mc_vram_size = 0 for APP APU\n");
1709	adev->gmc.mc_vram_size = 0;
1710	}
1711	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1712
1713	if (!(adev->flags & AMD_IS_APU) &&
1714	!adev->gmc.xgmi.connected_to_cpu) {
1715	r = amdgpu_device_resize_fb_bar(adev);
1716	if (r)
1717	return r;
1718	}
1719	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1720	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1721
1722	#ifdef CONFIG_X86_64
1723	/*
1724	* AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi
1725	* interface can use VRAM through here as it appears system reserved
1726	* memory in host address space.
1727	*
1728	* For APUs, VRAM is just the stolen system memory and can be accessed
1729	* directly.
1730	*
1731	* Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR.
1732	*/
1733
1734	/* check whether both host-gpu and gpu-gpu xgmi links exist */
1735	if ((!amdgpu_sriov_vf(adev) &&
1736	(adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) ||
1737	(adev->gmc.xgmi.supported &&
1738	adev->gmc.xgmi.connected_to_cpu)) {
1739	adev->gmc.aper_base =
1740	adev->gfxhub.funcs->get_mc_fb_offset(adev) +
1741	adev->gmc.xgmi.physical_node_id *
1742	adev->gmc.xgmi.node_segment_size;
1743	adev->gmc.aper_size = adev->gmc.real_vram_size;
1744	}
1745
1746	#endif
1747	adev->gmc.visible_vram_size = adev->gmc.aper_size;
1748
1749	/* set the gart size */
1750	if (amdgpu_gart_size == -1) {
1751	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1752	case IP_VERSION(9, 0, 1): /* all engines support GPUVM */
1753	case IP_VERSION(9, 2, 1): /* all engines support GPUVM */
1754	case IP_VERSION(9, 4, 0):
1755	case IP_VERSION(9, 4, 1):
1756	case IP_VERSION(9, 4, 2):
1757	case IP_VERSION(9, 4, 3):
1758	case IP_VERSION(9, 4, 4):
1759	case IP_VERSION(9, 5, 0):
1760	default:
1761	adev->gmc.gart_size = 512ULL << 20;
1762	break;
1763	case IP_VERSION(9, 1, 0): /* DCE SG support */
1764	case IP_VERSION(9, 2, 2): /* DCE SG support */
1765	case IP_VERSION(9, 3, 0):
1766	adev->gmc.gart_size = 1024ULL << 20;
1767	break;
1768	}
1769	} else {
1770	adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1771	}
1772
1773	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
1774
1775	gmc_v9_0_vram_gtt_location(adev, mc: &adev->gmc);
1776
1777	return 0;
1778	}
1779
1780	static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1781	{
1782	int r;
1783
1784	if (adev->gart.bo) {
1785	WARN(1, "VEGA10 PCIE GART already initialized\n");
1786	return 0;
1787	}
1788
1789	if (amdgpu_gmc_is_pdb0_enabled(adev)) {
1790	adev->gmc.vmid0_page_table_depth = 1;
1791	adev->gmc.vmid0_page_table_block_size = 12;
1792	} else {
1793	adev->gmc.vmid0_page_table_depth = 0;
1794	adev->gmc.vmid0_page_table_block_size = 0;
1795	}
1796
1797	/* Initialize common gart structure */
1798	r = amdgpu_gart_init(adev);
1799	if (r)
1800	return r;
1801	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1802	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_UC) |
1803	AMDGPU_PTE_EXECUTABLE;
1804
1805	if (!adev->gmc.real_vram_size) {
1806	dev_info(adev->dev, "Put GART in system memory for APU\n");
1807	r = amdgpu_gart_table_ram_alloc(adev);
1808	if (r)
1809	dev_err(adev->dev, "Failed to allocate GART in system memory\n");
1810	} else {
1811	r = amdgpu_gart_table_vram_alloc(adev);
1812	if (r)
1813	return r;
1814
1815	if (amdgpu_gmc_is_pdb0_enabled(adev))
1816	r = amdgpu_gmc_pdb0_alloc(adev);
1817	}
1818
1819	return r;
1820	}
1821
1822	/**
1823	* gmc_v9_0_save_registers - saves regs
1824	*
1825	* @adev: amdgpu_device pointer
1826	*
1827	* This saves potential register values that should be
1828	* restored upon resume
1829	*/
1830	static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1831	{
1832	if ((amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) == IP_VERSION(1, 0, 0)) ||
1833	(amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) == IP_VERSION(1, 0, 1)))
1834	adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1835	}
1836
1837	static void gmc_v9_4_3_init_vram_info(struct amdgpu_device *adev)
1838	{
1839	static const u32 regBIF_BIOS_SCRATCH_4 = 0x50;
1840	u32 vram_info;
1841
1842	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
1843	adev->gmc.vram_width = 128 * 64;
1844
1845	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 5, 0))
1846	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM3E;
1847
1848	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 4) &&
1849	adev->rev_id == 0x3)
1850	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM3E;
1851
1852	if (!(adev->flags & AMD_IS_APU) && !amdgpu_sriov_vf(adev)) {
1853	vram_info = RREG32(regBIF_BIOS_SCRATCH_4);
1854	adev->gmc.vram_vendor = vram_info & 0xF;
1855	}
1856	}
1857
1858	static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
1859	{
1860	int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
1861	struct amdgpu_device *adev = ip_block->adev;
1862	unsigned long inst_mask = adev->aid_mask;
1863
1864	adev->gfxhub.funcs->init(adev);
1865
1866	adev->mmhub.funcs->init(adev);
1867
1868	spin_lock_init(&adev->gmc.invalidate_lock);
1869
1870	if (amdgpu_is_multi_aid(adev)) {
1871	gmc_v9_4_3_init_vram_info(adev);
1872	} else if (!adev->bios) {
1873	if (adev->flags & AMD_IS_APU) {
1874	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4;
1875	adev->gmc.vram_width = 64 * 64;
1876	} else {
1877	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
1878	adev->gmc.vram_width = 128 * 64;
1879	}
1880	} else {
1881	r = amdgpu_atomfirmware_get_vram_info(adev,
1882	vram_width: &vram_width, vram_type: &vram_type, vram_vendor: &vram_vendor);
1883	if (amdgpu_sriov_vf(adev))
1884	/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1885	* and DF related registers is not readable, seems hardcord is the
1886	* only way to set the correct vram_width
1887	*/
1888	adev->gmc.vram_width = 2048;
1889	else if (amdgpu_emu_mode != 1)
1890	adev->gmc.vram_width = vram_width;
1891
1892	if (!adev->gmc.vram_width) {
1893	int chansize, numchan;
1894
1895	/* hbm memory channel size */
1896	if (adev->flags & AMD_IS_APU)
1897	chansize = 64;
1898	else
1899	chansize = 128;
1900	if (adev->df.funcs &&
1901	adev->df.funcs->get_hbm_channel_number) {
1902	numchan = adev->df.funcs->get_hbm_channel_number(adev);
1903	adev->gmc.vram_width = numchan * chansize;
1904	}
1905	}
1906
1907	adev->gmc.vram_type = vram_type;
1908	adev->gmc.vram_vendor = vram_vendor;
1909	}
1910	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1911	case IP_VERSION(9, 1, 0):
1912	case IP_VERSION(9, 2, 2):
1913	set_bit(AMDGPU_GFXHUB(0), addr: adev->vmhubs_mask);
1914	set_bit(AMDGPU_MMHUB0(0), addr: adev->vmhubs_mask);
1915
1916	if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
1917	amdgpu_vm_adjust_size(adev, min_vm_size: 256 * 1024, fragment_size_default: 9, max_level: 3, max_bits: 48);
1918	} else {
1919	/* vm_size is 128TB + 512GB for legacy 3-level page support */
1920	amdgpu_vm_adjust_size(adev, min_vm_size: 128 * 1024 + 512, fragment_size_default: 9, max_level: 2, max_bits: 48);
1921	adev->gmc.translate_further =
1922	adev->vm_manager.num_level > 1;
1923	}
1924	break;
1925	case IP_VERSION(9, 0, 1):
1926	case IP_VERSION(9, 2, 1):
1927	case IP_VERSION(9, 4, 0):
1928	case IP_VERSION(9, 3, 0):
1929	case IP_VERSION(9, 4, 2):
1930	set_bit(AMDGPU_GFXHUB(0), addr: adev->vmhubs_mask);
1931	set_bit(AMDGPU_MMHUB0(0), addr: adev->vmhubs_mask);
1932
1933	/*
1934	* To fulfill 4-level page support,
1935	* vm size is 256TB (48bit), maximum size of Vega10,
1936	* block size 512 (9bit)
1937	*/
1938
1939	amdgpu_vm_adjust_size(adev, min_vm_size: 256 * 1024, fragment_size_default: 9, max_level: 3, max_bits: 48);
1940	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
1941	adev->gmc.translate_further = adev->vm_manager.num_level > 1;
1942	break;
1943	case IP_VERSION(9, 4, 1):
1944	set_bit(AMDGPU_GFXHUB(0), addr: adev->vmhubs_mask);
1945	set_bit(AMDGPU_MMHUB0(0), addr: adev->vmhubs_mask);
1946	set_bit(AMDGPU_MMHUB1(0), addr: adev->vmhubs_mask);
1947
1948	/* Keep the vm size same with Vega20 */
1949	amdgpu_vm_adjust_size(adev, min_vm_size: 256 * 1024, fragment_size_default: 9, max_level: 3, max_bits: 48);
1950	adev->gmc.translate_further = adev->vm_manager.num_level > 1;
1951	break;
1952	case IP_VERSION(9, 4, 3):
1953	case IP_VERSION(9, 4, 4):
1954	case IP_VERSION(9, 5, 0):
1955	bitmap_set(map: adev->vmhubs_mask, AMDGPU_GFXHUB(0),
1956	NUM_XCC(adev->gfx.xcc_mask));
1957
1958	inst_mask <<= AMDGPU_MMHUB0(0);
1959	bitmap_or(dst: adev->vmhubs_mask, src1: adev->vmhubs_mask, src2: &inst_mask, nbits: 32);
1960
1961	amdgpu_vm_adjust_size(adev, min_vm_size: 256 * 1024, fragment_size_default: 9, max_level: 3, max_bits: 48);
1962	adev->gmc.translate_further = adev->vm_manager.num_level > 1;
1963	break;
1964	default:
1965	break;
1966	}
1967
1968	/* This interrupt is VMC page fault.*/
1969	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
1970	source: &adev->gmc.vm_fault);
1971	if (r)
1972	return r;
1973
1974	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1)) {
1975	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
1976	source: &adev->gmc.vm_fault);
1977	if (r)
1978	return r;
1979	}
1980
1981	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
1982	source: &adev->gmc.vm_fault);
1983
1984	if (r)
1985	return r;
1986
1987	if (!amdgpu_sriov_vf(adev) &&
1988	!adev->gmc.xgmi.connected_to_cpu &&
1989	!adev->gmc.is_app_apu) {
1990	/* interrupt sent to DF. */
1991	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_DF, src_id: 0,
1992	source: &adev->gmc.ecc_irq);
1993	if (r)
1994	return r;
1995	}
1996
1997	/* Set the internal MC address mask
1998	* This is the max address of the GPU's
1999	* internal address space.
2000	*/
2001	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
2002
2003	dma_addr_bits = amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >=
2004	IP_VERSION(9, 4, 2) ?
2005	48 :
2006	44;
2007	r = dma_set_mask_and_coherent(dev: adev->dev, DMA_BIT_MASK(dma_addr_bits));
2008	if (r) {
2009	dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
2010	return r;
2011	}
2012	adev->need_swiotlb = drm_need_swiotlb(dma_bits: dma_addr_bits);
2013
2014	r = gmc_v9_0_mc_init(adev);
2015	if (r)
2016	return r;
2017
2018	amdgpu_gmc_get_vbios_allocations(adev);
2019
2020	if (amdgpu_is_multi_aid(adev)) {
2021	r = amdgpu_gmc_init_mem_ranges(adev);
2022	if (r)
2023	return r;
2024	}
2025
2026	/* Memory manager */
2027	r = amdgpu_bo_init(adev);
2028	if (r)
2029	return r;
2030
2031	r = gmc_v9_0_gart_init(adev);
2032	if (r)
2033	return r;
2034
2035	gmc_v9_0_init_nps_details(adev);
2036	/*
2037	* number of VMs
2038	* VMID 0 is reserved for System
2039	* amdgpu graphics/compute will use VMIDs 1..n-1
2040	* amdkfd will use VMIDs n..15
2041	*
2042	* The first KFD VMID is 8 for GPUs with graphics, 3 for
2043	* compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
2044	* for video processing.
2045	*/
2046	adev->vm_manager.first_kfd_vmid =
2047	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 1) ||
2048	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2) ||
2049	amdgpu_is_multi_aid(adev)) ?
2050	3 :
2051	8;
2052
2053	amdgpu_vm_manager_init(adev);
2054
2055	gmc_v9_0_save_registers(adev);
2056
2057	r = amdgpu_gmc_ras_sw_init(adev);
2058	if (r)
2059	return r;
2060
2061	if (amdgpu_is_multi_aid(adev))
2062	amdgpu_gmc_sysfs_init(adev);
2063
2064	return 0;
2065	}
2066
2067	static int gmc_v9_0_sw_fini(struct amdgpu_ip_block *ip_block)
2068	{
2069	struct amdgpu_device *adev = ip_block->adev;
2070
2071	if (amdgpu_is_multi_aid(adev))
2072	amdgpu_gmc_sysfs_fini(adev);
2073
2074	amdgpu_gmc_ras_fini(adev);
2075	amdgpu_gem_force_release(adev);
2076	amdgpu_vm_manager_fini(adev);
2077	if (!adev->gmc.real_vram_size) {
2078	dev_info(adev->dev, "Put GART in system memory for APU free\n");
2079	amdgpu_gart_table_ram_free(adev);
2080	} else {
2081	amdgpu_gart_table_vram_free(adev);
2082	}
2083	amdgpu_bo_free_kernel(bo: &adev->gmc.pdb0_bo, NULL, cpu_addr: &adev->gmc.ptr_pdb0);
2084	amdgpu_bo_fini(adev);
2085
2086	adev->gmc.num_mem_partitions = 0;
2087	kfree(objp: adev->gmc.mem_partitions);
2088
2089	return 0;
2090	}
2091
2092	static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
2093	{
2094	switch (amdgpu_ip_version(adev, ip: MMHUB_HWIP, inst: 0)) {
2095	case IP_VERSION(9, 0, 0):
2096	if (amdgpu_sriov_vf(adev))
2097	break;
2098	fallthrough;
2099	case IP_VERSION(9, 4, 0):
2100	soc15_program_register_sequence(adev,
2101	registers: golden_settings_mmhub_1_0_0,
2102	ARRAY_SIZE(golden_settings_mmhub_1_0_0));
2103	soc15_program_register_sequence(adev,
2104	registers: golden_settings_athub_1_0_0,
2105	ARRAY_SIZE(golden_settings_athub_1_0_0));
2106	break;
2107	case IP_VERSION(9, 1, 0):
2108	case IP_VERSION(9, 2, 0):
2109	/* TODO for renoir */
2110	soc15_program_register_sequence(adev,
2111	registers: golden_settings_athub_1_0_0,
2112	ARRAY_SIZE(golden_settings_athub_1_0_0));
2113	break;
2114	default:
2115	break;
2116	}
2117	}
2118
2119	/**
2120	* gmc_v9_0_restore_registers - restores regs
2121	*
2122	* @adev: amdgpu_device pointer
2123	*
2124	* This restores register values, saved at suspend.
2125	*/
2126	void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
2127	{
2128	if ((amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) == IP_VERSION(1, 0, 0)) ||
2129	(amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) == IP_VERSION(1, 0, 1))) {
2130	WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
2131	WARN_ON(adev->gmc.sdpif_register !=
2132	RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
2133	}
2134	}
2135
2136	/**
2137	* gmc_v9_0_gart_enable - gart enable
2138	*
2139	* @adev: amdgpu_device pointer
2140	*/
2141	static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
2142	{
2143	int r;
2144
2145	if (amdgpu_gmc_is_pdb0_enabled(adev))
2146	amdgpu_gmc_init_pdb0(adev);
2147
2148	if (adev->gart.bo == NULL) {
2149	dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
2150	return -EINVAL;
2151	}
2152
2153	amdgpu_gtt_mgr_recover(mgr: &adev->mman.gtt_mgr);
2154
2155	if (!adev->in_s0ix) {
2156	r = adev->gfxhub.funcs->gart_enable(adev);
2157	if (r)
2158	return r;
2159	}
2160
2161	r = adev->mmhub.funcs->gart_enable(adev);
2162	if (r)
2163	return r;
2164
2165	DRM_INFO("PCIE GART of %uM enabled.\n",
2166	(unsigned int)(adev->gmc.gart_size >> 20));
2167	if (adev->gmc.pdb0_bo)
2168	DRM_INFO("PDB0 located at 0x%016llX\n",
2169	(unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo));
2170	DRM_INFO("PTB located at 0x%016llX\n",
2171	(unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
2172
2173	return 0;
2174	}
2175
2176	static int gmc_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
2177	{
2178	struct amdgpu_device *adev = ip_block->adev;
2179	bool value;
2180	int i, r;
2181
2182	adev->gmc.flush_pasid_uses_kiq = true;
2183
2184	/* Vega20+XGMI caches PTEs in TC and TLB. Add a heavy-weight TLB flush
2185	* (type 2), which flushes both. Due to a race condition with
2186	* concurrent memory accesses using the same TLB cache line, we still
2187	* need a second TLB flush after this.
2188	*/
2189	adev->gmc.flush_tlb_needs_extra_type_2 =
2190	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 0) &&
2191	adev->gmc.xgmi.num_physical_nodes;
2192
2193	/* The sequence of these two function calls matters.*/
2194	gmc_v9_0_init_golden_registers(adev);
2195
2196	if (adev->mode_info.num_crtc) {
2197	/* Lockout access through VGA aperture*/
2198	WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
2199	/* disable VGA render */
2200	WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
2201	}
2202
2203	if (adev->mmhub.funcs->update_power_gating)
2204	adev->mmhub.funcs->update_power_gating(adev, true);
2205
2206	adev->hdp.funcs->init_registers(adev);
2207
2208	/* After HDP is initialized, flush HDP.*/
2209	amdgpu_device_flush_hdp(adev, NULL);
2210
2211	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
2212	value = false;
2213	else
2214	value = true;
2215
2216	if (!amdgpu_sriov_vf(adev)) {
2217	if (!adev->in_s0ix)
2218	adev->gfxhub.funcs->set_fault_enable_default(adev, value);
2219	adev->mmhub.funcs->set_fault_enable_default(adev, value);
2220	}
2221	for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
2222	if (adev->in_s0ix && (i == AMDGPU_GFXHUB(0)))
2223	continue;
2224	gmc_v9_0_flush_gpu_tlb(adev, vmid: 0, vmhub: i, flush_type: 0);
2225	}
2226
2227	if (adev->umc.funcs && adev->umc.funcs->init_registers)
2228	adev->umc.funcs->init_registers(adev);
2229
2230	r = gmc_v9_0_gart_enable(adev);
2231	if (r)
2232	return r;
2233
2234	if (amdgpu_emu_mode == 1)
2235	return amdgpu_gmc_vram_checking(adev);
2236
2237	return 0;
2238	}
2239
2240	/**
2241	* gmc_v9_0_gart_disable - gart disable
2242	*
2243	* @adev: amdgpu_device pointer
2244	*
2245	* This disables all VM page table.
2246	*/
2247	static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
2248	{
2249	if (!adev->in_s0ix)
2250	adev->gfxhub.funcs->gart_disable(adev);
2251	adev->mmhub.funcs->gart_disable(adev);
2252	}
2253
2254	static int gmc_v9_0_hw_fini(struct amdgpu_ip_block *ip_block)
2255	{
2256	struct amdgpu_device *adev = ip_block->adev;
2257
2258	gmc_v9_0_gart_disable(adev);
2259
2260	if (amdgpu_sriov_vf(adev)) {
2261	/* full access mode, so don't touch any GMC register */
2262	DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
2263	return 0;
2264	}
2265
2266	/*
2267	* Pair the operations did in gmc_v9_0_hw_init and thus maintain
2268	* a correct cached state for GMC. Otherwise, the "gate" again
2269	* operation on S3 resuming will fail due to wrong cached state.
2270	*/
2271	if (adev->mmhub.funcs->update_power_gating)
2272	adev->mmhub.funcs->update_power_gating(adev, false);
2273
2274	/*
2275	* For minimal init, late_init is not called, hence VM fault/RAS irqs
2276	* are not enabled.
2277	*/
2278	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
2279	amdgpu_irq_put(adev, src: &adev->gmc.vm_fault, type: 0);
2280
2281	if (adev->gmc.ecc_irq.funcs &&
2282	amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__UMC))
2283	amdgpu_irq_put(adev, src: &adev->gmc.ecc_irq, type: 0);
2284	}
2285
2286	return 0;
2287	}
2288
2289	static int gmc_v9_0_suspend(struct amdgpu_ip_block *ip_block)
2290	{
2291	return gmc_v9_0_hw_fini(ip_block);
2292	}
2293
2294	static int gmc_v9_0_resume(struct amdgpu_ip_block *ip_block)
2295	{
2296	struct amdgpu_device *adev = ip_block->adev;
2297	int r;
2298
2299	/* If a reset is done for NPS mode switch, read the memory range
2300	* information again.
2301	*/
2302	if (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS) {
2303	amdgpu_gmc_init_sw_mem_ranges(adev, mem_ranges: adev->gmc.mem_partitions);
2304	adev->gmc.reset_flags &= ~AMDGPU_GMC_INIT_RESET_NPS;
2305	}
2306
2307	r = gmc_v9_0_hw_init(ip_block);
2308	if (r)
2309	return r;
2310
2311	amdgpu_vmid_reset_all(adev: ip_block->adev);
2312
2313	return 0;
2314	}
2315
2316	static bool gmc_v9_0_is_idle(struct amdgpu_ip_block *ip_block)
2317	{
2318	/* MC is always ready in GMC v9.*/
2319	return true;
2320	}
2321
2322	static int gmc_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
2323	{
2324	/* There is no need to wait for MC idle in GMC v9.*/
2325	return 0;
2326	}
2327
2328	static int gmc_v9_0_soft_reset(struct amdgpu_ip_block *ip_block)
2329	{
2330	/* XXX for emulation.*/
2331	return 0;
2332	}
2333
2334	static int gmc_v9_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2335	enum amd_clockgating_state state)
2336	{
2337	struct amdgpu_device *adev = ip_block->adev;
2338
2339	adev->mmhub.funcs->set_clockgating(adev, state);
2340
2341	athub_v1_0_set_clockgating(adev, state);
2342
2343	return 0;
2344	}
2345
2346	static void gmc_v9_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
2347	{
2348	struct amdgpu_device *adev = ip_block->adev;
2349
2350	adev->mmhub.funcs->get_clockgating(adev, flags);
2351
2352	athub_v1_0_get_clockgating(adev, flags);
2353	}
2354
2355	static int gmc_v9_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
2356	enum amd_powergating_state state)
2357	{
2358	return 0;
2359	}
2360
2361	const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
2362	.name = "gmc_v9_0",
2363	.early_init = gmc_v9_0_early_init,
2364	.late_init = gmc_v9_0_late_init,
2365	.sw_init = gmc_v9_0_sw_init,
2366	.sw_fini = gmc_v9_0_sw_fini,
2367	.hw_init = gmc_v9_0_hw_init,
2368	.hw_fini = gmc_v9_0_hw_fini,
2369	.suspend = gmc_v9_0_suspend,
2370	.resume = gmc_v9_0_resume,
2371	.is_idle = gmc_v9_0_is_idle,
2372	.wait_for_idle = gmc_v9_0_wait_for_idle,
2373	.soft_reset = gmc_v9_0_soft_reset,
2374	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
2375	.set_powergating_state = gmc_v9_0_set_powergating_state,
2376	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
2377	};
2378
2379	const struct amdgpu_ip_block_version gmc_v9_0_ip_block = {
2380	.type = AMD_IP_BLOCK_TYPE_GMC,
2381	.major = 9,
2382	.minor = 0,
2383	.rev = 0,
2384	.funcs = &gmc_v9_0_ip_funcs,
2385	};
2386