1	/*
2	* Copyright 2022 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23	#include <linux/firmware.h>
24
25	#include "amdgpu.h"
26	#include "amdgpu_gfx.h"
27	#include "soc15.h"
28	#include "soc15d.h"
29	#include "soc15_common.h"
30	#include "vega10_enum.h"
31
32	#include "v9_structs.h"
33
34	#include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
35
36	#include "gc/gc_9_4_3_offset.h"
37	#include "gc/gc_9_4_3_sh_mask.h"
38
39	#include "gfx_v9_4_3.h"
40	#include "gfx_v9_4_3_cleaner_shader.h"
41	#include "amdgpu_xcp.h"
42	#include "amdgpu_aca.h"
43
44	MODULE_FIRMWARE("amdgpu/gc_9_4_3_mec.bin");
45	MODULE_FIRMWARE("amdgpu/gc_9_4_4_mec.bin");
46	MODULE_FIRMWARE("amdgpu/gc_9_5_0_mec.bin");
47	MODULE_FIRMWARE("amdgpu/gc_9_4_3_rlc.bin");
48	MODULE_FIRMWARE("amdgpu/gc_9_4_4_rlc.bin");
49	MODULE_FIRMWARE("amdgpu/gc_9_5_0_rlc.bin");
50	MODULE_FIRMWARE("amdgpu/gc_9_4_3_sjt_mec.bin");
51	MODULE_FIRMWARE("amdgpu/gc_9_4_4_sjt_mec.bin");
52
53	#define GFX9_MEC_HPD_SIZE 4096
54	#define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
55
56	#define GOLDEN_GB_ADDR_CONFIG 0x2a114042
57	#define CP_HQD_PERSISTENT_STATE_DEFAULT 0xbe05301
58
59	#define XCC_REG_RANGE_0_LOW 0x2000 /* XCC gfxdec0 lower Bound */
60	#define XCC_REG_RANGE_0_HIGH 0x3400 /* XCC gfxdec0 upper Bound */
61	#define XCC_REG_RANGE_1_LOW 0xA000 /* XCC gfxdec1 lower Bound */
62	#define XCC_REG_RANGE_1_HIGH 0x10000 /* XCC gfxdec1 upper Bound */
63
64	#define NORMALIZE_XCC_REG_OFFSET(offset) \
65	(offset & 0xFFFF)
66
67	static const struct amdgpu_hwip_reg_entry gc_reg_list_9_4_3[] = {
68	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
69	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
70	SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
71	SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
72	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
73	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
74	SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
75	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
76	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
77	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
78	SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
79	SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
80	SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
81	SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
82	SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT),
83	SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT),
84	SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS),
85	SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
86	SOC15_REG_ENTRY_STR(GC, 0, regSQC_DCACHE_UTCL1_STATUS),
87	SOC15_REG_ENTRY_STR(GC, 0, regSQC_ICACHE_UTCL1_STATUS),
88	SOC15_REG_ENTRY_STR(GC, 0, regSQ_UTCL1_STATUS),
89	SOC15_REG_ENTRY_STR(GC, 0, regTCP_UTCL1_STATUS),
90	SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
91	SOC15_REG_ENTRY_STR(GC, 0, regVM_L2_PROTECTION_FAULT_CNTL),
92	SOC15_REG_ENTRY_STR(GC, 0, regVM_L2_PROTECTION_FAULT_STATUS),
93	SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
94	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
95	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR),
96	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC2_INSTR_PNTR),
97	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
98	SOC15_REG_ENTRY_STR(GC, 0, regRLC_STAT),
99	SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_COMMAND),
100	SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_MESSAGE),
101	SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_ARGUMENT_1),
102	SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_ARGUMENT_2),
103	SOC15_REG_ENTRY_STR(GC, 0, regSMU_RLC_RESPONSE),
104	SOC15_REG_ENTRY_STR(GC, 0, regRLC_SAFE_MODE),
105	SOC15_REG_ENTRY_STR(GC, 0, regRLC_SMU_SAFE_MODE),
106	SOC15_REG_ENTRY_STR(GC, 0, regRLC_INT_STAT),
107	SOC15_REG_ENTRY_STR(GC, 0, regRLC_GPM_GENERAL_6),
108	/* SE status registers */
109	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
110	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
111	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2),
112	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3)
113	};
114
115	static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9_4_3[] = {
116	/* compute queue registers */
117	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
118	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ACTIVE),
119	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
120	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
121	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
122	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
123	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
124	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
125	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
126	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
127	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
128	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
129	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
130	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
131	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
132	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
133	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
134	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
135	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
136	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
137	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
138	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
139	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
140	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
141	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
142	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
143	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
144	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
145	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
146	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
147	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
148	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
149	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
150	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
151	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
152	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
153	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GFX_STATUS),
154	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
155	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
156	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
157	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
158	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
159	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
160	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
161	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
162	};
163
164	struct amdgpu_gfx_ras gfx_v9_4_3_ras;
165
166	static void gfx_v9_4_3_set_ring_funcs(struct amdgpu_device *adev);
167	static void gfx_v9_4_3_set_irq_funcs(struct amdgpu_device *adev);
168	static void gfx_v9_4_3_set_gds_init(struct amdgpu_device *adev);
169	static void gfx_v9_4_3_set_rlc_funcs(struct amdgpu_device *adev);
170	static int gfx_v9_4_3_get_cu_info(struct amdgpu_device *adev,
171	struct amdgpu_cu_info *cu_info);
172	static void gfx_v9_4_3_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
173	static void gfx_v9_4_3_xcc_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
174
175	static void gfx_v9_4_3_kiq_set_resources(struct amdgpu_ring *kiq_ring,
176	uint64_t queue_mask)
177	{
178	struct amdgpu_device *adev = kiq_ring->adev;
179	u64 shader_mc_addr;
180
181	/* Cleaner shader MC address */
182	shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8;
183
184	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
185	amdgpu_ring_write(ring: kiq_ring,
186	PACKET3_SET_RESOURCES_VMID_MASK(0) |
187	/* vmid_mask:0* queue_type:0 (KIQ) */
188	PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
189	amdgpu_ring_write(ring: kiq_ring,
190	lower_32_bits(queue_mask)); /* queue mask lo */
191	amdgpu_ring_write(ring: kiq_ring,
192	upper_32_bits(queue_mask)); /* queue mask hi */
193	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */
194	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */
195	amdgpu_ring_write(ring: kiq_ring, v: 0); /* oac mask */
196	amdgpu_ring_write(ring: kiq_ring, v: 0); /* gds heap base:0, gds heap size:0 */
197	}
198
199	static void gfx_v9_4_3_kiq_map_queues(struct amdgpu_ring *kiq_ring,
200	struct amdgpu_ring *ring)
201	{
202	struct amdgpu_device *adev = kiq_ring->adev;
203	uint64_t mqd_addr = amdgpu_bo_gpu_offset(bo: ring->mqd_obj);
204	uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
205	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
206
207	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
208	/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
209	amdgpu_ring_write(ring: kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
210	PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
211	PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
212	PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
213	PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
214	PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
215	/*queue_type: normal compute queue */
216	PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
217	/* alloc format: all_on_one_pipe */
218	PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
219	PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
220	/* num_queues: must be 1 */
221	PACKET3_MAP_QUEUES_NUM_QUEUES(1));
222	amdgpu_ring_write(ring: kiq_ring,
223	PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
224	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(mqd_addr));
225	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(mqd_addr));
226	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(wptr_addr));
227	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(wptr_addr));
228	}
229
230	static void gfx_v9_4_3_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
231	struct amdgpu_ring *ring,
232	enum amdgpu_unmap_queues_action action,
233	u64 gpu_addr, u64 seq)
234	{
235	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
236
237	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
238	amdgpu_ring_write(ring: kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
239	PACKET3_UNMAP_QUEUES_ACTION(action) |
240	PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
241	PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
242	PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
243	amdgpu_ring_write(ring: kiq_ring,
244	PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
245
246	if (action == PREEMPT_QUEUES_NO_UNMAP) {
247	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(gpu_addr));
248	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(gpu_addr));
249	amdgpu_ring_write(ring: kiq_ring, v: seq);
250	} else {
251	amdgpu_ring_write(ring: kiq_ring, v: 0);
252	amdgpu_ring_write(ring: kiq_ring, v: 0);
253	amdgpu_ring_write(ring: kiq_ring, v: 0);
254	}
255	}
256
257	static void gfx_v9_4_3_kiq_query_status(struct amdgpu_ring *kiq_ring,
258	struct amdgpu_ring *ring,
259	u64 addr,
260	u64 seq)
261	{
262	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
263
264	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
265	amdgpu_ring_write(ring: kiq_ring,
266	PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
267	PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
268	PACKET3_QUERY_STATUS_COMMAND(2));
269	/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
270	amdgpu_ring_write(ring: kiq_ring,
271	PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
272	PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
273	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(addr));
274	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(addr));
275	amdgpu_ring_write(ring: kiq_ring, lower_32_bits(seq));
276	amdgpu_ring_write(ring: kiq_ring, upper_32_bits(seq));
277	}
278
279	static void gfx_v9_4_3_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
280	uint16_t pasid, uint32_t flush_type,
281	bool all_hub)
282	{
283	amdgpu_ring_write(ring: kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
284	amdgpu_ring_write(ring: kiq_ring,
285	PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
286	PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
287	PACKET3_INVALIDATE_TLBS_PASID(pasid) |
288	PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
289	}
290
291	static void gfx_v9_4_3_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type,
292	uint32_t me_id, uint32_t pipe_id, uint32_t queue_id,
293	uint32_t xcc_id, uint32_t vmid)
294	{
295	struct amdgpu_device *adev = kiq_ring->adev;
296	unsigned i;
297
298	/* enter save mode */
299	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
300	mutex_lock(&adev->srbm_mutex);
301	soc15_grbm_select(adev, me: me_id, pipe: pipe_id, queue: queue_id, vmid: 0, xcc_id);
302
303	if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
304	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 0x2);
305	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_COMPUTE_QUEUE_RESET, 0x1);
306	/* wait till dequeue take effects */
307	for (i = 0; i < adev->usec_timeout; i++) {
308	if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
309	break;
310	udelay(usec: 1);
311	}
312	if (i >= adev->usec_timeout)
313	dev_err(adev->dev, "fail to wait on hqd deactive\n");
314	} else {
315	dev_err(adev->dev, "reset queue_type(%d) not supported\n\n", queue_type);
316	}
317
318	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
319	mutex_unlock(lock: &adev->srbm_mutex);
320	/* exit safe mode */
321	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
322	}
323
324	static const struct kiq_pm4_funcs gfx_v9_4_3_kiq_pm4_funcs = {
325	.kiq_set_resources = gfx_v9_4_3_kiq_set_resources,
326	.kiq_map_queues = gfx_v9_4_3_kiq_map_queues,
327	.kiq_unmap_queues = gfx_v9_4_3_kiq_unmap_queues,
328	.kiq_query_status = gfx_v9_4_3_kiq_query_status,
329	.kiq_invalidate_tlbs = gfx_v9_4_3_kiq_invalidate_tlbs,
330	.kiq_reset_hw_queue = gfx_v9_4_3_kiq_reset_hw_queue,
331	.set_resources_size = 8,
332	.map_queues_size = 7,
333	.unmap_queues_size = 6,
334	.query_status_size = 7,
335	.invalidate_tlbs_size = 2,
336	};
337
338	static void gfx_v9_4_3_set_kiq_pm4_funcs(struct amdgpu_device *adev)
339	{
340	int i, num_xcc;
341
342	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
343	for (i = 0; i < num_xcc; i++)
344	adev->gfx.kiq[i].pmf = &gfx_v9_4_3_kiq_pm4_funcs;
345	}
346
347	static void gfx_v9_4_3_init_golden_registers(struct amdgpu_device *adev)
348	{
349	int i, num_xcc, dev_inst;
350
351	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
352	for (i = 0; i < num_xcc; i++) {
353	dev_inst = GET_INST(GC, i);
354
355	WREG32_SOC15(GC, dev_inst, regGB_ADDR_CONFIG,
356	GOLDEN_GB_ADDR_CONFIG);
357	WREG32_FIELD15_PREREG(GC, dev_inst, TCP_UTCL1_CNTL2, SPARE, 0x1);
358	}
359	}
360
361	static uint32_t gfx_v9_4_3_normalize_xcc_reg_offset(uint32_t reg)
362	{
363	uint32_t normalized_reg = NORMALIZE_XCC_REG_OFFSET(reg);
364
365	/* If it is an XCC reg, normalize the reg to keep
366	lower 16 bits in local xcc */
367
368	if (((normalized_reg >= XCC_REG_RANGE_0_LOW) && (normalized_reg < XCC_REG_RANGE_0_HIGH)) ||
369	((normalized_reg >= XCC_REG_RANGE_1_LOW) && (normalized_reg < XCC_REG_RANGE_1_HIGH)))
370	return normalized_reg;
371	else
372	return reg;
373	}
374
375	static void gfx_v9_4_3_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
376	bool wc, uint32_t reg, uint32_t val)
377	{
378	reg = gfx_v9_4_3_normalize_xcc_reg_offset(reg);
379	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
380	amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
381	WRITE_DATA_DST_SEL(0) |
382	(wc ? WR_CONFIRM : 0));
383	amdgpu_ring_write(ring, v: reg);
384	amdgpu_ring_write(ring, v: 0);
385	amdgpu_ring_write(ring, v: val);
386	}
387
388	static void gfx_v9_4_3_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
389	int mem_space, int opt, uint32_t addr0,
390	uint32_t addr1, uint32_t ref, uint32_t mask,
391	uint32_t inv)
392	{
393	/* Only do the normalization on regspace */
394	if (mem_space == 0) {
395	addr0 = gfx_v9_4_3_normalize_xcc_reg_offset(reg: addr0);
396	addr1 = gfx_v9_4_3_normalize_xcc_reg_offset(reg: addr1);
397	}
398
399	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
400	amdgpu_ring_write(ring,
401	/* memory (1) or register (0) */
402	v: (WAIT_REG_MEM_MEM_SPACE(mem_space) |
403	WAIT_REG_MEM_OPERATION(opt) | /* wait */
404	WAIT_REG_MEM_FUNCTION(3) | /* equal */
405	WAIT_REG_MEM_ENGINE(eng_sel)));
406
407	if (mem_space)
408	BUG_ON(addr0 & 0x3); /* Dword align */
409	amdgpu_ring_write(ring, v: addr0);
410	amdgpu_ring_write(ring, v: addr1);
411	amdgpu_ring_write(ring, v: ref);
412	amdgpu_ring_write(ring, v: mask);
413	amdgpu_ring_write(ring, v: inv); /* poll interval */
414	}
415
416	static int gfx_v9_4_3_ring_test_ring(struct amdgpu_ring *ring)
417	{
418	uint32_t scratch_reg0_offset, xcc_offset;
419	struct amdgpu_device *adev = ring->adev;
420	uint32_t tmp = 0;
421	unsigned i;
422	int r;
423
424	/* Use register offset which is local to XCC in the packet */
425	xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
426	scratch_reg0_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, ring->xcc_id), regSCRATCH_REG0);
427	WREG32(scratch_reg0_offset, 0xCAFEDEAD);
428	tmp = RREG32(scratch_reg0_offset);
429
430	r = amdgpu_ring_alloc(ring, ndw: 3);
431	if (r)
432	return r;
433
434	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
435	amdgpu_ring_write(ring, v: xcc_offset - PACKET3_SET_UCONFIG_REG_START);
436	amdgpu_ring_write(ring, v: 0xDEADBEEF);
437	amdgpu_ring_commit(ring);
438
439	for (i = 0; i < adev->usec_timeout; i++) {
440	tmp = RREG32(scratch_reg0_offset);
441	if (tmp == 0xDEADBEEF)
442	break;
443	udelay(usec: 1);
444	}
445
446	if (i >= adev->usec_timeout)
447	r = -ETIMEDOUT;
448	return r;
449	}
450
451	static int gfx_v9_4_3_ring_test_ib(struct amdgpu_ring *ring, long timeout)
452	{
453	struct amdgpu_device *adev = ring->adev;
454	struct amdgpu_ib ib;
455	struct dma_fence *f = NULL;
456
457	unsigned index;
458	uint64_t gpu_addr;
459	uint32_t tmp;
460	long r;
461
462	r = amdgpu_device_wb_get(adev, wb: &index);
463	if (r)
464	return r;
465
466	gpu_addr = adev->wb.gpu_addr + (index * 4);
467	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
468	memset(&ib, 0, sizeof(ib));
469
470	r = amdgpu_ib_get(adev, NULL, size: 20, pool: AMDGPU_IB_POOL_DIRECT, ib: &ib);
471	if (r)
472	goto err1;
473
474	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
475	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
476	ib.ptr[2] = lower_32_bits(gpu_addr);
477	ib.ptr[3] = upper_32_bits(gpu_addr);
478	ib.ptr[4] = 0xDEADBEEF;
479	ib.length_dw = 5;
480
481	r = amdgpu_ib_schedule(ring, num_ibs: 1, ibs: &ib, NULL, f: &f);
482	if (r)
483	goto err2;
484
485	r = dma_fence_wait_timeout(f, intr: false, timeout);
486	if (r == 0) {
487	r = -ETIMEDOUT;
488	goto err2;
489	} else if (r < 0) {
490	goto err2;
491	}
492
493	tmp = adev->wb.wb[index];
494	if (tmp == 0xDEADBEEF)
495	r = 0;
496	else
497	r = -EINVAL;
498
499	err2:
500	amdgpu_ib_free(ib: &ib, NULL);
501	dma_fence_put(fence: f);
502	err1:
503	amdgpu_device_wb_free(adev, wb: index);
504	return r;
505	}
506
507
508	/* This value might differs per partition */
509	static uint64_t gfx_v9_4_3_get_gpu_clock_counter(struct amdgpu_device *adev)
510	{
511	uint64_t clock;
512
513	mutex_lock(&adev->gfx.gpu_clock_mutex);
514	WREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
515	clock = (uint64_t)RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_GPU_CLOCK_COUNT_LSB) |
516	((uint64_t)RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
517	mutex_unlock(lock: &adev->gfx.gpu_clock_mutex);
518
519	return clock;
520	}
521
522	static void gfx_v9_4_3_free_microcode(struct amdgpu_device *adev)
523	{
524	amdgpu_ucode_release(fw: &adev->gfx.pfp_fw);
525	amdgpu_ucode_release(fw: &adev->gfx.me_fw);
526	amdgpu_ucode_release(fw: &adev->gfx.ce_fw);
527	amdgpu_ucode_release(fw: &adev->gfx.rlc_fw);
528	amdgpu_ucode_release(fw: &adev->gfx.mec_fw);
529	amdgpu_ucode_release(fw: &adev->gfx.mec2_fw);
530
531	kfree(objp: adev->gfx.rlc.register_list_format);
532	}
533
534	static int gfx_v9_4_3_init_rlc_microcode(struct amdgpu_device *adev,
535	const char *chip_name)
536	{
537	int err;
538	const struct rlc_firmware_header_v2_0 *rlc_hdr;
539	uint16_t version_major;
540	uint16_t version_minor;
541
542
543	err = amdgpu_ucode_request(adev, fw: &adev->gfx.rlc_fw,
544	required: AMDGPU_UCODE_REQUIRED,
545	fmt: "amdgpu/%s_rlc.bin", chip_name);
546	if (err)
547	goto out;
548	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
549
550	version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
551	version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
552	err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
553	out:
554	if (err)
555	amdgpu_ucode_release(fw: &adev->gfx.rlc_fw);
556
557	return err;
558	}
559
560	static int gfx_v9_4_3_init_cp_compute_microcode(struct amdgpu_device *adev,
561	const char *chip_name)
562	{
563	int err;
564
565	if (amdgpu_sriov_vf(adev)) {
566	err = amdgpu_ucode_request(adev, fw: &adev->gfx.mec_fw,
567	required: AMDGPU_UCODE_REQUIRED,
568	fmt: "amdgpu/%s_sjt_mec.bin", chip_name);
569
570	if (err)
571	err = amdgpu_ucode_request(adev, fw: &adev->gfx.mec_fw,
572	required: AMDGPU_UCODE_REQUIRED,
573	fmt: "amdgpu/%s_mec.bin", chip_name);
574	} else
575	err = amdgpu_ucode_request(adev, fw: &adev->gfx.mec_fw,
576	required: AMDGPU_UCODE_REQUIRED,
577	fmt: "amdgpu/%s_mec.bin", chip_name);
578	if (err)
579	goto out;
580	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC1);
581	amdgpu_gfx_cp_init_microcode(adev, ucode_id: AMDGPU_UCODE_ID_CP_MEC1_JT);
582
583	adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version;
584	adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
585
586	out:
587	if (err)
588	amdgpu_ucode_release(fw: &adev->gfx.mec_fw);
589	return err;
590	}
591
592	static int gfx_v9_4_3_init_microcode(struct amdgpu_device *adev)
593	{
594	char ucode_prefix[15];
595	int r;
596
597	amdgpu_ucode_ip_version_decode(adev, block_type: GC_HWIP, ucode_prefix, len: sizeof(ucode_prefix));
598
599	r = gfx_v9_4_3_init_rlc_microcode(adev, chip_name: ucode_prefix);
600	if (r)
601	return r;
602
603	r = gfx_v9_4_3_init_cp_compute_microcode(adev, chip_name: ucode_prefix);
604	if (r)
605	return r;
606
607	return r;
608	}
609
610	static void gfx_v9_4_3_mec_fini(struct amdgpu_device *adev)
611	{
612	amdgpu_bo_free_kernel(bo: &adev->gfx.mec.hpd_eop_obj, NULL, NULL);
613	amdgpu_bo_free_kernel(bo: &adev->gfx.mec.mec_fw_obj, NULL, NULL);
614	}
615
616	static int gfx_v9_4_3_mec_init(struct amdgpu_device *adev)
617	{
618	int r, i, num_xcc;
619	u32 *hpd;
620	const __le32 *fw_data;
621	unsigned fw_size;
622	u32 *fw;
623	size_t mec_hpd_size;
624
625	const struct gfx_firmware_header_v1_0 *mec_hdr;
626
627	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
628	for (i = 0; i < num_xcc; i++)
629	bitmap_zero(dst: adev->gfx.mec_bitmap[i].queue_bitmap,
630	AMDGPU_MAX_COMPUTE_QUEUES);
631
632	/* take ownership of the relevant compute queues */
633	amdgpu_gfx_compute_queue_acquire(adev);
634	mec_hpd_size =
635	adev->gfx.num_compute_rings * num_xcc * GFX9_MEC_HPD_SIZE;
636	if (mec_hpd_size) {
637	r = amdgpu_bo_create_reserved(adev, size: mec_hpd_size, PAGE_SIZE,
638	AMDGPU_GEM_DOMAIN_VRAM |
639	AMDGPU_GEM_DOMAIN_GTT,
640	bo_ptr: &adev->gfx.mec.hpd_eop_obj,
641	gpu_addr: &adev->gfx.mec.hpd_eop_gpu_addr,
642	cpu_addr: (void **)&hpd);
643	if (r) {
644	dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
645	gfx_v9_4_3_mec_fini(adev);
646	return r;
647	}
648
649	if (amdgpu_emu_mode == 1) {
650	for (i = 0; i < mec_hpd_size / 4; i++) {
651	memset((void *)(hpd + i), 0, 4);
652	if (i % 50 == 0)
653	msleep(msecs: 1);
654	}
655	} else {
656	memset(hpd, 0, mec_hpd_size);
657	}
658
659	amdgpu_bo_kunmap(bo: adev->gfx.mec.hpd_eop_obj);
660	amdgpu_bo_unreserve(bo: adev->gfx.mec.hpd_eop_obj);
661	}
662
663	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
664
665	fw_data = (const __le32 *)
666	(adev->gfx.mec_fw->data +
667	le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
668	fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
669
670	r = amdgpu_bo_create_reserved(adev, size: mec_hdr->header.ucode_size_bytes,
671	PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
672	bo_ptr: &adev->gfx.mec.mec_fw_obj,
673	gpu_addr: &adev->gfx.mec.mec_fw_gpu_addr,
674	cpu_addr: (void **)&fw);
675	if (r) {
676	dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
677	gfx_v9_4_3_mec_fini(adev);
678	return r;
679	}
680
681	memcpy(fw, fw_data, fw_size);
682
683	amdgpu_bo_kunmap(bo: adev->gfx.mec.mec_fw_obj);
684	amdgpu_bo_unreserve(bo: adev->gfx.mec.mec_fw_obj);
685
686	return 0;
687	}
688
689	static void gfx_v9_4_3_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num,
690	u32 sh_num, u32 instance, int xcc_id)
691	{
692	u32 data;
693
694	if (instance == 0xffffffff)
695	data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
696	INSTANCE_BROADCAST_WRITES, 1);
697	else
698	data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
699	INSTANCE_INDEX, instance);
700
701	if (se_num == 0xffffffff)
702	data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
703	SE_BROADCAST_WRITES, 1);
704	else
705	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
706
707	if (sh_num == 0xffffffff)
708	data = REG_SET_FIELD(data, GRBM_GFX_INDEX,
709	SH_BROADCAST_WRITES, 1);
710	else
711	data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
712
713	WREG32_SOC15_RLC_SHADOW_EX(reg, GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX, data);
714	}
715
716	static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t address)
717	{
718	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
719	(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
720	(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
721	(address << SQ_IND_INDEX__INDEX__SHIFT) |
722	(SQ_IND_INDEX__FORCE_READ_MASK));
723	return RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
724	}
725
726	static void wave_read_regs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
727	uint32_t wave, uint32_t thread,
728	uint32_t regno, uint32_t num, uint32_t *out)
729	{
730	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX,
731	(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
732	(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
733	(regno << SQ_IND_INDEX__INDEX__SHIFT) |
734	(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
735	(SQ_IND_INDEX__FORCE_READ_MASK) |
736	(SQ_IND_INDEX__AUTO_INCR_MASK));
737	while (num--)
738	*(out++) = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA);
739	}
740
741	static void gfx_v9_4_3_read_wave_data(struct amdgpu_device *adev,
742	uint32_t xcc_id, uint32_t simd, uint32_t wave,
743	uint32_t *dst, int *no_fields)
744	{
745	/* type 1 wave data */
746	dst[(*no_fields)++] = 1;
747	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_STATUS);
748	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_PC_LO);
749	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_PC_HI);
750	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_EXEC_LO);
751	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_EXEC_HI);
752	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_HW_ID);
753	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_INST_DW0);
754	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_INST_DW1);
755	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_GPR_ALLOC);
756	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_LDS_ALLOC);
757	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_TRAPSTS);
758	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_IB_STS);
759	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_IB_DBG0);
760	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_M0);
761	dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, simd, wave, ixSQ_WAVE_MODE);
762	}
763
764	static void gfx_v9_4_3_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
765	uint32_t wave, uint32_t start,
766	uint32_t size, uint32_t *dst)
767	{
768	wave_read_regs(adev, xcc_id, simd, wave, thread: 0,
769	regno: start + SQIND_WAVE_SGPRS_OFFSET, num: size, out: dst);
770	}
771
772	static void gfx_v9_4_3_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
773	uint32_t wave, uint32_t thread,
774	uint32_t start, uint32_t size,
775	uint32_t *dst)
776	{
777	wave_read_regs(adev, xcc_id, simd, wave, thread,
778	regno: start + SQIND_WAVE_VGPRS_OFFSET, num: size, out: dst);
779	}
780
781	static void gfx_v9_4_3_select_me_pipe_q(struct amdgpu_device *adev,
782	u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
783	{
784	soc15_grbm_select(adev, me, pipe, queue: q, vmid: vm, GET_INST(GC, xcc_id));
785	}
786
787	static int gfx_v9_4_3_get_xccs_per_xcp(struct amdgpu_device *adev)
788	{
789	u32 xcp_ctl;
790
791	/* Value is expected to be the same on all, fetch from first instance */
792	xcp_ctl = RREG32_SOC15(GC, GET_INST(GC, 0), regCP_HYP_XCP_CTL);
793
794	return REG_GET_FIELD(xcp_ctl, CP_HYP_XCP_CTL, NUM_XCC_IN_XCP);
795	}
796
797	static int gfx_v9_4_3_switch_compute_partition(struct amdgpu_device *adev,
798	int num_xccs_per_xcp)
799	{
800	int ret, i, num_xcc;
801	u32 tmp = 0;
802
803	if (adev->psp.funcs) {
804	ret = psp_spatial_partition(psp: &adev->psp,
805	NUM_XCC(adev->gfx.xcc_mask) /
806	num_xccs_per_xcp);
807	if (ret)
808	return ret;
809	} else {
810	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
811
812	for (i = 0; i < num_xcc; i++) {
813	tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, NUM_XCC_IN_XCP,
814	num_xccs_per_xcp);
815	tmp = REG_SET_FIELD(tmp, CP_HYP_XCP_CTL, VIRTUAL_XCC_ID,
816	i % num_xccs_per_xcp);
817	WREG32_SOC15(GC, GET_INST(GC, i), regCP_HYP_XCP_CTL,
818	tmp);
819	}
820	ret = 0;
821	}
822
823	adev->gfx.num_xcc_per_xcp = num_xccs_per_xcp;
824
825	return ret;
826	}
827
828	static int gfx_v9_4_3_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)
829	{
830	int xcc;
831
832	xcc = hweight8(adev->gfx.xcc_mask & GENMASK(ih_node / 2, 0));
833	if (!xcc) {
834	dev_err(adev->dev, "Couldn't find xcc mapping from IH node");
835	return -EINVAL;
836	}
837
838	return xcc - 1;
839	}
840
841	static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = {
842	.get_gpu_clock_counter = &gfx_v9_4_3_get_gpu_clock_counter,
843	.select_se_sh = &gfx_v9_4_3_xcc_select_se_sh,
844	.read_wave_data = &gfx_v9_4_3_read_wave_data,
845	.read_wave_sgprs = &gfx_v9_4_3_read_wave_sgprs,
846	.read_wave_vgprs = &gfx_v9_4_3_read_wave_vgprs,
847	.select_me_pipe_q = &gfx_v9_4_3_select_me_pipe_q,
848	.switch_partition_mode = &gfx_v9_4_3_switch_compute_partition,
849	.ih_node_to_logical_xcc = &gfx_v9_4_3_ih_to_xcc_inst,
850	.get_xccs_per_xcp = &gfx_v9_4_3_get_xccs_per_xcp,
851	};
852
853	static int gfx_v9_4_3_aca_bank_parser(struct aca_handle *handle,
854	struct aca_bank *bank, enum aca_smu_type type,
855	void *data)
856	{
857	struct aca_bank_info info;
858	u64 misc0;
859	u32 instlo;
860	int ret;
861
862	ret = aca_bank_info_decode(bank, info: &info);
863	if (ret)
864	return ret;
865
866	/* NOTE: overwrite info.die_id with xcd id for gfx */
867	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
868	instlo &= GENMASK(31, 1);
869	info.die_id = instlo == mmSMNAID_XCD0_MCA_SMU ? 0 : 1;
870
871	misc0 = bank->regs[ACA_REG_IDX_MISC0];
872
873	switch (type) {
874	case ACA_SMU_TYPE_UE:
875	bank->aca_err_type = ACA_ERROR_TYPE_UE;
876	ret = aca_error_cache_log_bank_error(handle, info: &info, type: bank->aca_err_type, count: 1ULL);
877	break;
878	case ACA_SMU_TYPE_CE:
879	bank->aca_err_type = ACA_ERROR_TYPE_CE;
880	ret = aca_error_cache_log_bank_error(handle, info: &info, type: bank->aca_err_type,
881	ACA_REG__MISC0__ERRCNT(misc0));
882	break;
883	default:
884	return -EINVAL;
885	}
886
887	return ret;
888	}
889
890	static bool gfx_v9_4_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
891	enum aca_smu_type type, void *data)
892	{
893	u32 instlo;
894
895	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
896	instlo &= GENMASK(31, 1);
897	switch (instlo) {
898	case mmSMNAID_XCD0_MCA_SMU:
899	case mmSMNAID_XCD1_MCA_SMU:
900	case mmSMNXCD_XCD0_MCA_SMU:
901	return true;
902	default:
903	break;
904	}
905
906	return false;
907	}
908
909	static const struct aca_bank_ops gfx_v9_4_3_aca_bank_ops = {
910	.aca_bank_parser = gfx_v9_4_3_aca_bank_parser,
911	.aca_bank_is_valid = gfx_v9_4_3_aca_bank_is_valid,
912	};
913
914	static const struct aca_info gfx_v9_4_3_aca_info = {
915	.hwip = ACA_HWIP_TYPE_SMU,
916	.mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
917	.bank_ops = &gfx_v9_4_3_aca_bank_ops,
918	};
919
920	static int gfx_v9_4_3_gpu_early_init(struct amdgpu_device *adev)
921	{
922	adev->gfx.funcs = &gfx_v9_4_3_gfx_funcs;
923	adev->gfx.ras = &gfx_v9_4_3_ras;
924
925	adev->gfx.config.max_hw_contexts = 8;
926	adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
927	adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
928	adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
929	adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
930	adev->gfx.config.gb_addr_config = GOLDEN_GB_ADDR_CONFIG;
931
932	adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
933	REG_GET_FIELD(
934	adev->gfx.config.gb_addr_config,
935	GB_ADDR_CONFIG,
936	NUM_PIPES);
937
938	adev->gfx.config.max_tile_pipes =
939	adev->gfx.config.gb_addr_config_fields.num_pipes;
940
941	adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
942	REG_GET_FIELD(
943	adev->gfx.config.gb_addr_config,
944	GB_ADDR_CONFIG,
945	NUM_BANKS);
946	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
947	REG_GET_FIELD(
948	adev->gfx.config.gb_addr_config,
949	GB_ADDR_CONFIG,
950	MAX_COMPRESSED_FRAGS);
951	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
952	REG_GET_FIELD(
953	adev->gfx.config.gb_addr_config,
954	GB_ADDR_CONFIG,
955	NUM_RB_PER_SE);
956	adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
957	REG_GET_FIELD(
958	adev->gfx.config.gb_addr_config,
959	GB_ADDR_CONFIG,
960	NUM_SHADER_ENGINES);
961	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
962	REG_GET_FIELD(
963	adev->gfx.config.gb_addr_config,
964	GB_ADDR_CONFIG,
965	PIPE_INTERLEAVE_SIZE));
966
967	return 0;
968	}
969
970	static int gfx_v9_4_3_compute_ring_init(struct amdgpu_device *adev, int ring_id,
971	int xcc_id, int mec, int pipe, int queue)
972	{
973	unsigned irq_type;
974	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
975	unsigned int hw_prio;
976	uint32_t xcc_doorbell_start;
977
978	ring = &adev->gfx.compute_ring[xcc_id * adev->gfx.num_compute_rings +
979	ring_id];
980
981	/* mec0 is me1 */
982	ring->xcc_id = xcc_id;
983	ring->me = mec + 1;
984	ring->pipe = pipe;
985	ring->queue = queue;
986
987	ring->ring_obj = NULL;
988	ring->use_doorbell = true;
989	xcc_doorbell_start = adev->doorbell_index.mec_ring0 +
990	xcc_id * adev->doorbell_index.xcc_doorbell_range;
991	ring->doorbell_index = (xcc_doorbell_start + ring_id) << 1;
992	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr +
993	(ring_id + xcc_id * adev->gfx.num_compute_rings) *
994	GFX9_MEC_HPD_SIZE;
995	ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
996	sprintf(buf: ring->name, fmt: "comp_%d.%d.%d.%d",
997	ring->xcc_id, ring->me, ring->pipe, ring->queue);
998
999	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1000	+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1001	+ ring->pipe;
1002	hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1003	AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL;
1004	/* type-2 packets are deprecated on MEC, use type-3 instead */
1005	return amdgpu_ring_init(adev, ring, max_dw: 1024, irq_src: &adev->gfx.eop_irq, irq_type,
1006	hw_prio, NULL);
1007	}
1008
1009	static void gfx_v9_4_3_alloc_ip_dump(struct amdgpu_device *adev)
1010	{
1011	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9_4_3);
1012	uint32_t *ptr, num_xcc, inst;
1013
1014	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1015
1016	ptr = kcalloc(reg_count * num_xcc, sizeof(uint32_t), GFP_KERNEL);
1017	if (!ptr) {
1018	DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
1019	adev->gfx.ip_dump_core = NULL;
1020	} else {
1021	adev->gfx.ip_dump_core = ptr;
1022	}
1023
1024	/* Allocate memory for compute queue registers for all the instances */
1025	reg_count = ARRAY_SIZE(gc_cp_reg_list_9_4_3);
1026	inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
1027	adev->gfx.mec.num_queue_per_pipe;
1028
1029	ptr = kcalloc(reg_count * inst * num_xcc, sizeof(uint32_t), GFP_KERNEL);
1030	if (!ptr) {
1031	DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
1032	adev->gfx.ip_dump_compute_queues = NULL;
1033	} else {
1034	adev->gfx.ip_dump_compute_queues = ptr;
1035	}
1036	}
1037
1038	static int gfx_v9_4_3_sw_init(struct amdgpu_ip_block *ip_block)
1039	{
1040	int i, j, k, r, ring_id, xcc_id, num_xcc;
1041	struct amdgpu_device *adev = ip_block->adev;
1042
1043	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1044	case IP_VERSION(9, 4, 3):
1045	case IP_VERSION(9, 4, 4):
1046	adev->gfx.cleaner_shader_ptr = gfx_9_4_3_cleaner_shader_hex;
1047	adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_3_cleaner_shader_hex);
1048	if (adev->gfx.mec_fw_version >= 153) {
1049	adev->gfx.enable_cleaner_shader = true;
1050	r = amdgpu_gfx_cleaner_shader_sw_init(adev, cleaner_shader_size: adev->gfx.cleaner_shader_size);
1051	if (r) {
1052	adev->gfx.enable_cleaner_shader = false;
1053	dev_err(adev->dev, "Failed to initialize cleaner shader\n");
1054	}
1055	}
1056	break;
1057	default:
1058	adev->gfx.enable_cleaner_shader = false;
1059	break;
1060	}
1061
1062	adev->gfx.mec.num_mec = 2;
1063	adev->gfx.mec.num_pipe_per_mec = 4;
1064	adev->gfx.mec.num_queue_per_pipe = 8;
1065
1066	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1067
1068	/* EOP Event */
1069	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, source: &adev->gfx.eop_irq);
1070	if (r)
1071	return r;
1072
1073	/* Bad opcode Event */
1074	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP,
1075	GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR,
1076	source: &adev->gfx.bad_op_irq);
1077	if (r)
1078	return r;
1079
1080	/* Privileged reg */
1081	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
1082	source: &adev->gfx.priv_reg_irq);
1083	if (r)
1084	return r;
1085
1086	/* Privileged inst */
1087	r = amdgpu_irq_add_id(adev, client_id: SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
1088	source: &adev->gfx.priv_inst_irq);
1089	if (r)
1090	return r;
1091
1092	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1093
1094	r = adev->gfx.rlc.funcs->init(adev);
1095	if (r) {
1096	DRM_ERROR("Failed to init rlc BOs!\n");
1097	return r;
1098	}
1099
1100	r = gfx_v9_4_3_mec_init(adev);
1101	if (r) {
1102	DRM_ERROR("Failed to init MEC BOs!\n");
1103	return r;
1104	}
1105
1106	/* set up the compute queues - allocate horizontally across pipes */
1107	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1108	ring_id = 0;
1109	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1110	for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1111	for (k = 0; k < adev->gfx.mec.num_pipe_per_mec;
1112	k++) {
1113	if (!amdgpu_gfx_is_mec_queue_enabled(
1114	adev, xcc_id, mec: i, pipe: k, queue: j))
1115	continue;
1116
1117	r = gfx_v9_4_3_compute_ring_init(adev,
1118	ring_id,
1119	xcc_id,
1120	mec: i, pipe: k, queue: j);
1121	if (r)
1122	return r;
1123
1124	ring_id++;
1125	}
1126	}
1127	}
1128
1129	r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, xcc_id);
1130	if (r) {
1131	DRM_ERROR("Failed to init KIQ BOs!\n");
1132	return r;
1133	}
1134
1135	r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
1136	if (r)
1137	return r;
1138
1139	/* create MQD for all compute queues as wel as KIQ for SRIOV case */
1140	r = amdgpu_gfx_mqd_sw_init(adev,
1141	mqd_size: sizeof(struct v9_mqd_allocation), xcc_id);
1142	if (r)
1143	return r;
1144	}
1145
1146	adev->gfx.compute_supported_reset =
1147	amdgpu_get_soft_full_reset_mask(ring: &adev->gfx.compute_ring[0]);
1148	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1149	case IP_VERSION(9, 4, 3):
1150	case IP_VERSION(9, 4, 4):
1151	if ((adev->gfx.mec_fw_version >= 155) &&
1152	!amdgpu_sriov_vf(adev) &&
1153	!adev->debug_disable_gpu_ring_reset) {
1154	adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1155	adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE;
1156	}
1157	break;
1158	case IP_VERSION(9, 5, 0):
1159	if ((adev->gfx.mec_fw_version >= 21) &&
1160	!amdgpu_sriov_vf(adev) &&
1161	!adev->debug_disable_gpu_ring_reset) {
1162	adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
1163	adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_PIPE;
1164	}
1165	break;
1166	default:
1167	break;
1168	}
1169	r = gfx_v9_4_3_gpu_early_init(adev);
1170	if (r)
1171	return r;
1172
1173	r = amdgpu_gfx_ras_sw_init(adev);
1174	if (r)
1175	return r;
1176
1177	r = amdgpu_gfx_sysfs_init(adev);
1178	if (r)
1179	return r;
1180
1181	gfx_v9_4_3_alloc_ip_dump(adev);
1182
1183	return 0;
1184	}
1185
1186	static int gfx_v9_4_3_sw_fini(struct amdgpu_ip_block *ip_block)
1187	{
1188	int i, num_xcc;
1189	struct amdgpu_device *adev = ip_block->adev;
1190
1191	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1192	for (i = 0; i < adev->gfx.num_compute_rings * num_xcc; i++)
1193	amdgpu_ring_fini(ring: &adev->gfx.compute_ring[i]);
1194
1195	for (i = 0; i < num_xcc; i++) {
1196	amdgpu_gfx_mqd_sw_fini(adev, xcc_id: i);
1197	amdgpu_gfx_kiq_free_ring(ring: &adev->gfx.kiq[i].ring);
1198	amdgpu_gfx_kiq_fini(adev, xcc_id: i);
1199	}
1200
1201	amdgpu_gfx_cleaner_shader_sw_fini(adev);
1202
1203	gfx_v9_4_3_mec_fini(adev);
1204	amdgpu_bo_unref(bo: &adev->gfx.rlc.clear_state_obj);
1205	gfx_v9_4_3_free_microcode(adev);
1206	amdgpu_gfx_sysfs_fini(adev);
1207
1208	kfree(objp: adev->gfx.ip_dump_core);
1209	kfree(objp: adev->gfx.ip_dump_compute_queues);
1210
1211	return 0;
1212	}
1213
1214	#define DEFAULT_SH_MEM_BASES (0x6000)
1215	static void gfx_v9_4_3_xcc_init_compute_vmid(struct amdgpu_device *adev,
1216	int xcc_id)
1217	{
1218	int i;
1219	uint32_t sh_mem_config;
1220	uint32_t sh_mem_bases;
1221	uint32_t data;
1222
1223	/*
1224	* Configure apertures:
1225	* LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
1226	* Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
1227	* GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
1228	*/
1229	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
1230
1231	sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
1232	SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
1233	SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
1234
1235	mutex_lock(&adev->srbm_mutex);
1236	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1237	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: i, GET_INST(GC, xcc_id));
1238	/* CP and shaders */
1239	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSH_MEM_CONFIG, sh_mem_config);
1240	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, sh_mem_bases);
1241
1242	/* Enable trap for each kfd vmid. */
1243	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL);
1244	data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1);
1245	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL, data);
1246	}
1247	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, GET_INST(GC, xcc_id));
1248	mutex_unlock(lock: &adev->srbm_mutex);
1249
1250	/*
1251	* Initialize all compute VMIDs to have no GDS, GWS, or OA
1252	* access. These should be enabled by FW for target VMIDs.
1253	*/
1254	for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
1255	WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_BASE, 2 * i, 0);
1256	WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_SIZE, 2 * i, 0);
1257	WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_GWS_VMID0, i, 0);
1258	WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_OA_VMID0, i, 0);
1259	}
1260	}
1261
1262	static void gfx_v9_4_3_xcc_init_gds_vmid(struct amdgpu_device *adev, int xcc_id)
1263	{
1264	int vmid;
1265
1266	/*
1267	* Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
1268	* access. Compute VMIDs should be enabled by FW for target VMIDs,
1269	* the driver can enable them for graphics. VMID0 should maintain
1270	* access so that HWS firmware can save/restore entries.
1271	*/
1272	for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
1273	WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_BASE, 2 * vmid, 0);
1274	WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_VMID0_SIZE, 2 * vmid, 0);
1275	WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_GWS_VMID0, vmid, 0);
1276	WREG32_SOC15_OFFSET(GC, GET_INST(GC, xcc_id), regGDS_OA_VMID0, vmid, 0);
1277	}
1278	}
1279
1280	/* For ASICs that needs xnack chain and MEC version supports, set SG_CONFIG1
1281	* DISABLE_XNACK_CHECK_IN_RETRY_DISABLE bit and inform KFD to set xnack_chain
1282	* bit in SET_RESOURCES
1283	*/
1284	static void gfx_v9_4_3_xcc_init_sq(struct amdgpu_device *adev, int xcc_id)
1285	{
1286	uint32_t data;
1287
1288	if (!(adev->gmc.xnack_flags & AMDGPU_GMC_XNACK_FLAG_CHAIN))
1289	return;
1290
1291	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_CONFIG1);
1292	data = REG_SET_FIELD(data, SQ_CONFIG1, DISABLE_XNACK_CHECK_IN_RETRY_DISABLE, 1);
1293	WREG32_SOC15(GC, xcc_id, regSQ_CONFIG1, data);
1294	}
1295
1296	static void gfx_v9_4_3_xcc_constants_init(struct amdgpu_device *adev,
1297	int xcc_id)
1298	{
1299	u32 tmp;
1300	int i;
1301
1302	/* XXX SH_MEM regs */
1303	/* where to put LDS, scratch, GPUVM in FSA64 space */
1304	mutex_lock(&adev->srbm_mutex);
1305	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
1306	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: i, GET_INST(GC, xcc_id));
1307	/* CP and shaders */
1308	if (i == 0) {
1309	tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
1310	SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1311	tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
1312	!!adev->gmc.noretry);
1313	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
1314	regSH_MEM_CONFIG, tmp);
1315	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
1316	regSH_MEM_BASES, 0);
1317	} else {
1318	tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
1319	SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1320	tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
1321	!!adev->gmc.noretry);
1322	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
1323	regSH_MEM_CONFIG, tmp);
1324	tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1325	(adev->gmc.private_aperture_start >>
1326	48));
1327	tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1328	(adev->gmc.shared_aperture_start >>
1329	48));
1330	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id),
1331	regSH_MEM_BASES, tmp);
1332	}
1333	}
1334	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, GET_INST(GC, 0));
1335
1336	mutex_unlock(lock: &adev->srbm_mutex);
1337
1338	gfx_v9_4_3_xcc_init_compute_vmid(adev, xcc_id);
1339	gfx_v9_4_3_xcc_init_gds_vmid(adev, xcc_id);
1340	gfx_v9_4_3_xcc_init_sq(adev, xcc_id);
1341	}
1342
1343	static void gfx_v9_4_3_constants_init(struct amdgpu_device *adev)
1344	{
1345	int i, num_xcc;
1346
1347	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1348
1349	gfx_v9_4_3_get_cu_info(adev, cu_info: &adev->gfx.cu_info);
1350	adev->gfx.config.db_debug2 =
1351	RREG32_SOC15(GC, GET_INST(GC, 0), regDB_DEBUG2);
1352
1353	switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1354	/* ToDo: GC 9.4.4 */
1355	case IP_VERSION(9, 4, 3):
1356	if (adev->gfx.mec_fw_version >= 184 &&
1357	(amdgpu_sriov_reg_access_sq_config(adev) ||
1358	!amdgpu_sriov_vf(adev)))
1359	adev->gmc.xnack_flags |= AMDGPU_GMC_XNACK_FLAG_CHAIN;
1360	break;
1361	case IP_VERSION(9, 5, 0):
1362	if (adev->gfx.mec_fw_version >= 23)
1363	adev->gmc.xnack_flags |= AMDGPU_GMC_XNACK_FLAG_CHAIN;
1364	break;
1365	default:
1366	break;
1367	}
1368
1369	for (i = 0; i < num_xcc; i++)
1370	gfx_v9_4_3_xcc_constants_init(adev, xcc_id: i);
1371	}
1372
1373	static void
1374	gfx_v9_4_3_xcc_enable_save_restore_machine(struct amdgpu_device *adev,
1375	int xcc_id)
1376	{
1377	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_SRM_CNTL, SRM_ENABLE, 1);
1378	}
1379
1380	static void gfx_v9_4_3_xcc_init_pg(struct amdgpu_device *adev, int xcc_id)
1381	{
1382	/*
1383	* Rlc save restore list is workable since v2_1.
1384	*/
1385	gfx_v9_4_3_xcc_enable_save_restore_machine(adev, xcc_id);
1386	}
1387
1388	static void gfx_v9_4_3_xcc_disable_gpa_mode(struct amdgpu_device *adev, int xcc_id)
1389	{
1390	uint32_t data;
1391
1392	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG);
1393	data |= CPC_PSP_DEBUG__UTCL2IUGPAOVERRIDE_MASK;
1394	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG, data);
1395	}
1396
1397	static bool gfx_v9_4_3_is_rlc_enabled(struct amdgpu_device *adev)
1398	{
1399	uint32_t rlc_setting;
1400
1401	/* if RLC is not enabled, do nothing */
1402	rlc_setting = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CNTL);
1403	if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
1404	return false;
1405
1406	return true;
1407	}
1408
1409	static void gfx_v9_4_3_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
1410	{
1411	uint32_t data;
1412	unsigned i;
1413
1414	data = RLC_SAFE_MODE__CMD_MASK;
1415	data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
1416	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE, data);
1417
1418	/* wait for RLC_SAFE_MODE */
1419	for (i = 0; i < adev->usec_timeout; i++) {
1420	if (!REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
1421	break;
1422	udelay(usec: 1);
1423	}
1424	}
1425
1426	static void gfx_v9_4_3_xcc_unset_safe_mode(struct amdgpu_device *adev,
1427	int xcc_id)
1428	{
1429	uint32_t data;
1430
1431	data = RLC_SAFE_MODE__CMD_MASK;
1432	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE, data);
1433	}
1434
1435	static void gfx_v9_4_3_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
1436	{
1437	int xcc_id, num_xcc;
1438	struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1439
1440	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1441	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1442	reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[GET_INST(GC, xcc_id)];
1443	reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG0);
1444	reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG1);
1445	reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG2);
1446	reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regSCRATCH_REG3);
1447	reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_CNTL);
1448	reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX);
1449	reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPARE_INT);
1450	}
1451	adev->gfx.rlc.rlcg_reg_access_supported = true;
1452	}
1453
1454	static int gfx_v9_4_3_rlc_init(struct amdgpu_device *adev)
1455	{
1456	/* init spm vmid with 0xf */
1457	if (adev->gfx.rlc.funcs->update_spm_vmid)
1458	adev->gfx.rlc.funcs->update_spm_vmid(adev, NULL, 0xf);
1459
1460	return 0;
1461	}
1462
1463	static void gfx_v9_4_3_xcc_wait_for_rlc_serdes(struct amdgpu_device *adev,
1464	int xcc_id)
1465	{
1466	u32 i, j, k;
1467	u32 mask;
1468
1469	mutex_lock(&adev->grbm_idx_mutex);
1470	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1471	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1472	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: i, sh_num: j, instance: 0xffffffff,
1473	xcc_id);
1474	for (k = 0; k < adev->usec_timeout; k++) {
1475	if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SERDES_CU_MASTER_BUSY) == 0)
1476	break;
1477	udelay(usec: 1);
1478	}
1479	if (k == adev->usec_timeout) {
1480	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: 0xffffffff,
1481	sh_num: 0xffffffff,
1482	instance: 0xffffffff, xcc_id);
1483	mutex_unlock(lock: &adev->grbm_idx_mutex);
1484	DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
1485	i, j);
1486	return;
1487	}
1488	}
1489	}
1490	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: 0xffffffff, sh_num: 0xffffffff, instance: 0xffffffff,
1491	xcc_id);
1492	mutex_unlock(lock: &adev->grbm_idx_mutex);
1493
1494	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
1495	RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
1496	RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
1497	RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
1498	for (k = 0; k < adev->usec_timeout; k++) {
1499	if ((RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
1500	break;
1501	udelay(usec: 1);
1502	}
1503	}
1504
1505	static void gfx_v9_4_3_xcc_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1506	bool enable, int xcc_id)
1507	{
1508	u32 tmp;
1509
1510	/* These interrupts should be enabled to drive DS clock */
1511
1512	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0);
1513
1514	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
1515	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
1516	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
1517
1518	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0, tmp);
1519	}
1520
1521	static void gfx_v9_4_3_xcc_rlc_stop(struct amdgpu_device *adev, int xcc_id)
1522	{
1523	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_CNTL,
1524	RLC_ENABLE_F32, 0);
1525	gfx_v9_4_3_xcc_enable_gui_idle_interrupt(adev, enable: false, xcc_id);
1526	gfx_v9_4_3_xcc_wait_for_rlc_serdes(adev, xcc_id);
1527	}
1528
1529	static void gfx_v9_4_3_rlc_stop(struct amdgpu_device *adev)
1530	{
1531	int i, num_xcc;
1532
1533	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1534	for (i = 0; i < num_xcc; i++)
1535	gfx_v9_4_3_xcc_rlc_stop(adev, xcc_id: i);
1536	}
1537
1538	static void gfx_v9_4_3_xcc_rlc_reset(struct amdgpu_device *adev, int xcc_id)
1539	{
1540	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), GRBM_SOFT_RESET,
1541	SOFT_RESET_RLC, 1);
1542	udelay(usec: 50);
1543	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), GRBM_SOFT_RESET,
1544	SOFT_RESET_RLC, 0);
1545	udelay(usec: 50);
1546	}
1547
1548	static void gfx_v9_4_3_rlc_reset(struct amdgpu_device *adev)
1549	{
1550	int i, num_xcc;
1551
1552	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1553	for (i = 0; i < num_xcc; i++)
1554	gfx_v9_4_3_xcc_rlc_reset(adev, xcc_id: i);
1555	}
1556
1557	static void gfx_v9_4_3_xcc_rlc_start(struct amdgpu_device *adev, int xcc_id)
1558	{
1559	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_CNTL,
1560	RLC_ENABLE_F32, 1);
1561	udelay(usec: 50);
1562
1563	/* carrizo do enable cp interrupt after cp inited */
1564	if (!(adev->flags & AMD_IS_APU)) {
1565	gfx_v9_4_3_xcc_enable_gui_idle_interrupt(adev, enable: true, xcc_id);
1566	udelay(usec: 50);
1567	}
1568	}
1569
1570	static void gfx_v9_4_3_rlc_start(struct amdgpu_device *adev)
1571	{
1572	#ifdef AMDGPU_RLC_DEBUG_RETRY
1573	u32 rlc_ucode_ver;
1574	#endif
1575	int i, num_xcc;
1576
1577	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1578	for (i = 0; i < num_xcc; i++) {
1579	gfx_v9_4_3_xcc_rlc_start(adev, xcc_id: i);
1580	#ifdef AMDGPU_RLC_DEBUG_RETRY
1581	/* RLC_GPM_GENERAL_6 : RLC Ucode version */
1582	rlc_ucode_ver = RREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_GENERAL_6);
1583	if (rlc_ucode_ver == 0x108) {
1584	dev_info(adev->dev,
1585	"Using rlc debug ucode. regRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
1586	rlc_ucode_ver, adev->gfx.rlc_fw_version);
1587	/* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
1588	* default is 0x9C4 to create a 100us interval */
1589	WREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_TIMER_INT_3, 0x9C4);
1590	/* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
1591	* to disable the page fault retry interrupts, default is
1592	* 0x100 (256) */
1593	WREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_GENERAL_12, 0x100);
1594	}
1595	#endif
1596	}
1597	}
1598
1599	static int gfx_v9_4_3_xcc_rlc_load_microcode(struct amdgpu_device *adev,
1600	int xcc_id)
1601	{
1602	const struct rlc_firmware_header_v2_0 *hdr;
1603	const __le32 *fw_data;
1604	unsigned i, fw_size;
1605
1606	if (!adev->gfx.rlc_fw)
1607	return -EINVAL;
1608
1609	hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1610	amdgpu_ucode_print_rlc_hdr(hdr: &hdr->header);
1611
1612	fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1613	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1614	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1615
1616	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_ADDR,
1617	RLCG_UCODE_LOADING_START_ADDRESS);
1618	for (i = 0; i < fw_size; i++) {
1619	if (amdgpu_emu_mode == 1 && i % 100 == 0) {
1620	dev_info(adev->dev, "Write RLC ucode data %u DWs\n", i);
1621	msleep(msecs: 1);
1622	}
1623	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
1624	}
1625	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
1626
1627	return 0;
1628	}
1629
1630	static int gfx_v9_4_3_xcc_rlc_resume(struct amdgpu_device *adev, int xcc_id)
1631	{
1632	int r;
1633
1634	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
1635	gfx_v9_4_3_xcc_rlc_stop(adev, xcc_id);
1636	/* legacy rlc firmware loading */
1637	r = gfx_v9_4_3_xcc_rlc_load_microcode(adev, xcc_id);
1638	if (r)
1639	return r;
1640	gfx_v9_4_3_xcc_rlc_start(adev, xcc_id);
1641	}
1642
1643	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
1644	/* disable CG */
1645	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, 0);
1646	gfx_v9_4_3_xcc_init_pg(adev, xcc_id);
1647	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
1648
1649	return 0;
1650	}
1651
1652	static int gfx_v9_4_3_rlc_resume(struct amdgpu_device *adev)
1653	{
1654	int r, i, num_xcc;
1655
1656	if (amdgpu_sriov_vf(adev))
1657	return 0;
1658
1659	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1660	for (i = 0; i < num_xcc; i++) {
1661	r = gfx_v9_4_3_xcc_rlc_resume(adev, xcc_id: i);
1662	if (r)
1663	return r;
1664	}
1665
1666	return 0;
1667	}
1668
1669	static void gfx_v9_4_3_update_spm_vmid(struct amdgpu_device *adev, struct amdgpu_ring *ring,
1670	unsigned vmid)
1671	{
1672	u32 reg, pre_data, data;
1673
1674	reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL);
1675	if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev))
1676	pre_data = RREG32_NO_KIQ(reg);
1677	else
1678	pre_data = RREG32(reg);
1679
1680	data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK);
1681	data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
1682
1683	if (pre_data != data) {
1684	if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) {
1685	WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL, data);
1686	} else
1687	WREG32_SOC15(GC, GET_INST(GC, 0), regRLC_SPM_MC_CNTL, data);
1688	}
1689	}
1690
1691	static const struct soc15_reg_rlcg rlcg_access_gc_9_4_3[] = {
1692	{SOC15_REG_ENTRY(GC, 0, regGRBM_GFX_INDEX)},
1693	{SOC15_REG_ENTRY(GC, 0, regSQ_IND_INDEX)},
1694	};
1695
1696	static bool gfx_v9_4_3_check_rlcg_range(struct amdgpu_device *adev,
1697	uint32_t offset,
1698	struct soc15_reg_rlcg *entries, int arr_size)
1699	{
1700	int i, inst;
1701	uint32_t reg;
1702
1703	if (!entries)
1704	return false;
1705
1706	for (i = 0; i < arr_size; i++) {
1707	const struct soc15_reg_rlcg *entry;
1708
1709	entry = &entries[i];
1710	inst = adev->ip_map.logical_to_dev_inst ?
1711	adev->ip_map.logical_to_dev_inst(
1712	adev, entry->hwip, entry->instance) :
1713	entry->instance;
1714	reg = adev->reg_offset[entry->hwip][inst][entry->segment] +
1715	entry->reg;
1716	if (offset == reg)
1717	return true;
1718	}
1719
1720	return false;
1721	}
1722
1723	static bool gfx_v9_4_3_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
1724	{
1725	return gfx_v9_4_3_check_rlcg_range(adev, offset,
1726	entries: (void *)rlcg_access_gc_9_4_3,
1727	ARRAY_SIZE(rlcg_access_gc_9_4_3));
1728	}
1729
1730	static void gfx_v9_4_3_xcc_cp_compute_enable(struct amdgpu_device *adev,
1731	bool enable, int xcc_id)
1732	{
1733	if (enable) {
1734	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MEC_CNTL, 0);
1735	} else {
1736	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MEC_CNTL,
1737	(CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK |
1738	CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK |
1739	CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK |
1740	CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK |
1741	CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK |
1742	CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK |
1743	CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK |
1744	CP_MEC_CNTL__MEC_ME1_HALT_MASK |
1745	CP_MEC_CNTL__MEC_ME2_HALT_MASK));
1746	adev->gfx.kiq[xcc_id].ring.sched.ready = false;
1747	}
1748	udelay(usec: 50);
1749	}
1750
1751	static int gfx_v9_4_3_xcc_cp_compute_load_microcode(struct amdgpu_device *adev,
1752	int xcc_id)
1753	{
1754	const struct gfx_firmware_header_v1_0 *mec_hdr;
1755	const __le32 *fw_data;
1756	unsigned i;
1757	u32 tmp;
1758	u32 mec_ucode_addr_offset;
1759	u32 mec_ucode_data_offset;
1760
1761	if (!adev->gfx.mec_fw)
1762	return -EINVAL;
1763
1764	gfx_v9_4_3_xcc_cp_compute_enable(adev, enable: false, xcc_id);
1765
1766	mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1767	amdgpu_ucode_print_gfx_hdr(hdr: &mec_hdr->header);
1768
1769	fw_data = (const __le32 *)
1770	(adev->gfx.mec_fw->data +
1771	le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1772	tmp = 0;
1773	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
1774	tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
1775	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL, tmp);
1776
1777	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_LO,
1778	adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
1779	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_HI,
1780	upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
1781
1782	mec_ucode_addr_offset =
1783	SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_MEC_ME1_UCODE_ADDR);
1784	mec_ucode_data_offset =
1785	SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_MEC_ME1_UCODE_DATA);
1786
1787	/* MEC1 */
1788	WREG32(mec_ucode_addr_offset, mec_hdr->jt_offset);
1789	for (i = 0; i < mec_hdr->jt_size; i++)
1790	WREG32(mec_ucode_data_offset,
1791	le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
1792
1793	WREG32(mec_ucode_addr_offset, adev->gfx.mec_fw_version);
1794	/* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
1795
1796	return 0;
1797	}
1798
1799	/* KIQ functions */
1800	static void gfx_v9_4_3_xcc_kiq_setting(struct amdgpu_ring *ring, int xcc_id)
1801	{
1802	uint32_t tmp;
1803	struct amdgpu_device *adev = ring->adev;
1804
1805	/* tell RLC which is KIQ queue */
1806	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
1807	tmp &= 0xffffff00;
1808	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1809	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp | 0x80);
1810	}
1811
1812	static void gfx_v9_4_3_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
1813	{
1814	struct amdgpu_device *adev = ring->adev;
1815
1816	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
1817	if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
1818	mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
1819	mqd->cp_hqd_queue_priority =
1820	AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
1821	}
1822	}
1823	}
1824
1825	static int gfx_v9_4_3_xcc_mqd_init(struct amdgpu_ring *ring, int xcc_id)
1826	{
1827	struct amdgpu_device *adev = ring->adev;
1828	struct v9_mqd *mqd = ring->mqd_ptr;
1829	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
1830	uint32_t tmp;
1831
1832	mqd->header = 0xC0310800;
1833	mqd->compute_pipelinestat_enable = 0x00000001;
1834	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
1835	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
1836	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
1837	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
1838	mqd->compute_misc_reserved = 0x00000003;
1839
1840	mqd->dynamic_cu_mask_addr_lo =
1841	lower_32_bits(ring->mqd_gpu_addr
1842	+ offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
1843	mqd->dynamic_cu_mask_addr_hi =
1844	upper_32_bits(ring->mqd_gpu_addr
1845	+ offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
1846
1847	eop_base_addr = ring->eop_gpu_addr >> 8;
1848	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
1849	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
1850
1851	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1852	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_CONTROL);
1853	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
1854	(order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
1855
1856	mqd->cp_hqd_eop_control = tmp;
1857
1858	/* enable doorbell? */
1859	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL);
1860
1861	if (ring->use_doorbell) {
1862	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1863	DOORBELL_OFFSET, ring->doorbell_index);
1864	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1865	DOORBELL_EN, 1);
1866	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1867	DOORBELL_SOURCE, 0);
1868	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1869	DOORBELL_HIT, 0);
1870	if (amdgpu_sriov_multi_vf_mode(adev))
1871	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1872	DOORBELL_MODE, 1);
1873	} else {
1874	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
1875	DOORBELL_EN, 0);
1876	}
1877
1878	mqd->cp_hqd_pq_doorbell_control = tmp;
1879
1880	/* disable the queue if it's active */
1881	ring->wptr = 0;
1882	mqd->cp_hqd_dequeue_request = 0;
1883	mqd->cp_hqd_pq_rptr = 0;
1884	mqd->cp_hqd_pq_wptr_lo = 0;
1885	mqd->cp_hqd_pq_wptr_hi = 0;
1886
1887	/* set the pointer to the MQD */
1888	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
1889	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
1890
1891	/* set MQD vmid to 0 */
1892	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL);
1893	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
1894	mqd->cp_mqd_control = tmp;
1895
1896	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
1897	hqd_gpu_addr = ring->gpu_addr >> 8;
1898	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
1899	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
1900
1901	/* set up the HQD, this is similar to CP_RB0_CNTL */
1902	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL);
1903	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
1904	(order_base_2(ring->ring_size / 4) - 1));
1905	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
1906	((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
1907	#ifdef __BIG_ENDIAN
1908	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
1909	#endif
1910	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
1911	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
1912	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
1913	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
1914	mqd->cp_hqd_pq_control = tmp;
1915
1916	/* set the wb address whether it's enabled or not */
1917	wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
1918	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
1919	mqd->cp_hqd_pq_rptr_report_addr_hi =
1920	upper_32_bits(wb_gpu_addr) & 0xffff;
1921
1922	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
1923	wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
1924	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
1925	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
1926
1927	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
1928	ring->wptr = 0;
1929	mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR);
1930
1931	/* set the vmid for the queue */
1932	mqd->cp_hqd_vmid = 0;
1933
1934	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE);
1935	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
1936	mqd->cp_hqd_persistent_state = tmp;
1937
1938	/* set MIN_IB_AVAIL_SIZE */
1939	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_IB_CONTROL);
1940	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
1941	mqd->cp_hqd_ib_control = tmp;
1942
1943	/* set static priority for a queue/ring */
1944	gfx_v9_4_3_mqd_set_priority(ring, mqd);
1945	mqd->cp_hqd_quantum = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_QUANTUM);
1946
1947	/* map_queues packet doesn't need activate the queue,
1948	* so only kiq need set this field.
1949	*/
1950	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
1951	mqd->cp_hqd_active = 1;
1952
1953	return 0;
1954	}
1955
1956	static int gfx_v9_4_3_xcc_kiq_init_register(struct amdgpu_ring *ring,
1957	int xcc_id)
1958	{
1959	struct amdgpu_device *adev = ring->adev;
1960	struct v9_mqd *mqd = ring->mqd_ptr;
1961	int j;
1962
1963	/* disable wptr polling */
1964	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_WPTR_POLL_CNTL, EN, 0);
1965
1966	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR,
1967	mqd->cp_hqd_eop_base_addr_lo);
1968	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR_HI,
1969	mqd->cp_hqd_eop_base_addr_hi);
1970
1971	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
1972	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_CONTROL,
1973	mqd->cp_hqd_eop_control);
1974
1975	/* enable doorbell? */
1976	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
1977	mqd->cp_hqd_pq_doorbell_control);
1978
1979	/* disable the queue if it's active */
1980	if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
1981	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
1982	for (j = 0; j < adev->usec_timeout; j++) {
1983	if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
1984	break;
1985	udelay(usec: 1);
1986	}
1987	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST,
1988	mqd->cp_hqd_dequeue_request);
1989	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR,
1990	mqd->cp_hqd_pq_rptr);
1991	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
1992	mqd->cp_hqd_pq_wptr_lo);
1993	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
1994	mqd->cp_hqd_pq_wptr_hi);
1995	}
1996
1997	/* set the pointer to the MQD */
1998	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR,
1999	mqd->cp_mqd_base_addr_lo);
2000	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR_HI,
2001	mqd->cp_mqd_base_addr_hi);
2002
2003	/* set MQD vmid to 0 */
2004	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL,
2005	mqd->cp_mqd_control);
2006
2007	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2008	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE,
2009	mqd->cp_hqd_pq_base_lo);
2010	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE_HI,
2011	mqd->cp_hqd_pq_base_hi);
2012
2013	/* set up the HQD, this is similar to CP_RB0_CNTL */
2014	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL,
2015	mqd->cp_hqd_pq_control);
2016
2017	/* set the wb address whether it's enabled or not */
2018	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR,
2019	mqd->cp_hqd_pq_rptr_report_addr_lo);
2020	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
2021	mqd->cp_hqd_pq_rptr_report_addr_hi);
2022
2023	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2024	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR,
2025	mqd->cp_hqd_pq_wptr_poll_addr_lo);
2026	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
2027	mqd->cp_hqd_pq_wptr_poll_addr_hi);
2028
2029	/* enable the doorbell if requested */
2030	if (ring->use_doorbell) {
2031	WREG32_SOC15(
2032	GC, GET_INST(GC, xcc_id),
2033	regCP_MEC_DOORBELL_RANGE_LOWER,
2034	((adev->doorbell_index.kiq +
2035	xcc_id * adev->doorbell_index.xcc_doorbell_range) *
2036	2) << 2);
2037	WREG32_SOC15(
2038	GC, GET_INST(GC, xcc_id),
2039	regCP_MEC_DOORBELL_RANGE_UPPER,
2040	((adev->doorbell_index.userqueue_end +
2041	xcc_id * adev->doorbell_index.xcc_doorbell_range) *
2042	2) << 2);
2043	}
2044
2045	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
2046	mqd->cp_hqd_pq_doorbell_control);
2047
2048	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2049	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
2050	mqd->cp_hqd_pq_wptr_lo);
2051	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
2052	mqd->cp_hqd_pq_wptr_hi);
2053
2054	/* set the vmid for the queue */
2055	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID, mqd->cp_hqd_vmid);
2056
2057	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE,
2058	mqd->cp_hqd_persistent_state);
2059
2060	/* activate the queue */
2061	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE,
2062	mqd->cp_hqd_active);
2063
2064	if (ring->use_doorbell)
2065	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_STATUS, DOORBELL_ENABLE, 1);
2066
2067	return 0;
2068	}
2069
2070	static int gfx_v9_4_3_xcc_q_fini_register(struct amdgpu_ring *ring,
2071	int xcc_id)
2072	{
2073	struct amdgpu_device *adev = ring->adev;
2074	int j;
2075
2076	/* disable the queue if it's active */
2077	if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
2078
2079	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
2080
2081	for (j = 0; j < adev->usec_timeout; j++) {
2082	if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
2083	break;
2084	udelay(usec: 1);
2085	}
2086
2087	if (j == AMDGPU_MAX_USEC_TIMEOUT) {
2088	DRM_DEBUG("%s dequeue request failed.\n", ring->name);
2089
2090	/* Manual disable if dequeue request times out */
2091	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, 0);
2092	}
2093
2094	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST,
2095	0);
2096	}
2097
2098	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_IQ_TIMER, 0);
2099	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_IB_CONTROL, 0);
2100	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE, CP_HQD_PERSISTENT_STATE_DEFAULT);
2101	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
2102	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, 0);
2103	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR, 0);
2104	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI, 0);
2105	WREG32_SOC15_RLC(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO, 0);
2106
2107	return 0;
2108	}
2109
2110	static int gfx_v9_4_3_xcc_kiq_init_queue(struct amdgpu_ring *ring, int xcc_id)
2111	{
2112	struct amdgpu_device *adev = ring->adev;
2113	struct v9_mqd *mqd = ring->mqd_ptr;
2114	struct v9_mqd *tmp_mqd;
2115
2116	gfx_v9_4_3_xcc_kiq_setting(ring, xcc_id);
2117
2118	/* GPU could be in bad state during probe, driver trigger the reset
2119	* after load the SMU, in this case , the mqd is not be initialized.
2120	* driver need to re-init the mqd.
2121	* check mqd->cp_hqd_pq_control since this value should not be 0
2122	*/
2123	tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[xcc_id].mqd_backup;
2124	if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control) {
2125	/* for GPU_RESET case , reset MQD to a clean status */
2126	if (adev->gfx.kiq[xcc_id].mqd_backup)
2127	memcpy(mqd, adev->gfx.kiq[xcc_id].mqd_backup, sizeof(struct v9_mqd_allocation));
2128
2129	/* reset ring buffer */
2130	ring->wptr = 0;
2131	amdgpu_ring_clear_ring(ring);
2132	mutex_lock(&adev->srbm_mutex);
2133	soc15_grbm_select(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, vmid: 0, GET_INST(GC, xcc_id));
2134	gfx_v9_4_3_xcc_kiq_init_register(ring, xcc_id);
2135	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, GET_INST(GC, xcc_id));
2136	mutex_unlock(lock: &adev->srbm_mutex);
2137	} else {
2138	memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
2139	((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
2140	((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
2141	mutex_lock(&adev->srbm_mutex);
2142	if (amdgpu_sriov_vf(adev) && adev->in_suspend)
2143	amdgpu_ring_clear_ring(ring);
2144	soc15_grbm_select(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, vmid: 0, GET_INST(GC, xcc_id));
2145	gfx_v9_4_3_xcc_mqd_init(ring, xcc_id);
2146	gfx_v9_4_3_xcc_kiq_init_register(ring, xcc_id);
2147	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, GET_INST(GC, xcc_id));
2148	mutex_unlock(lock: &adev->srbm_mutex);
2149
2150	if (adev->gfx.kiq[xcc_id].mqd_backup)
2151	memcpy(adev->gfx.kiq[xcc_id].mqd_backup, mqd, sizeof(struct v9_mqd_allocation));
2152	}
2153
2154	return 0;
2155	}
2156
2157	static void gfx_v9_4_3_xcc_kcq_init_queue(struct amdgpu_ring *ring, int xcc_id,
2158	bool restore)
2159	{
2160	struct amdgpu_device *adev = ring->adev;
2161	struct v9_mqd *mqd = ring->mqd_ptr;
2162	int mqd_idx = ring - &adev->gfx.compute_ring[0];
2163	struct v9_mqd *tmp_mqd;
2164
2165	/* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control
2166	* is not be initialized before
2167	*/
2168	tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx];
2169
2170	if (!restore && (!tmp_mqd->cp_hqd_pq_control ||
2171	(!amdgpu_in_reset(adev) && !adev->in_suspend))) {
2172	memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
2173	((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
2174	((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
2175	mutex_lock(&adev->srbm_mutex);
2176	soc15_grbm_select(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, vmid: 0, GET_INST(GC, xcc_id));
2177	gfx_v9_4_3_xcc_mqd_init(ring, xcc_id);
2178	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, GET_INST(GC, xcc_id));
2179	mutex_unlock(lock: &adev->srbm_mutex);
2180
2181	if (adev->gfx.mec.mqd_backup[mqd_idx])
2182	memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
2183	} else {
2184	/* restore MQD to a clean status */
2185	if (adev->gfx.mec.mqd_backup[mqd_idx])
2186	memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
2187	/* reset ring buffer */
2188	ring->wptr = 0;
2189	atomic64_set(v: (atomic64_t *)&adev->wb.wb[ring->wptr_offs], i: 0);
2190	amdgpu_ring_clear_ring(ring);
2191	}
2192	}
2193
2194	static int gfx_v9_4_3_xcc_kcq_fini_register(struct amdgpu_device *adev, int xcc_id)
2195	{
2196	struct amdgpu_ring *ring;
2197	int j;
2198
2199	for (j = 0; j < adev->gfx.num_compute_rings; j++) {
2200	ring = &adev->gfx.compute_ring[j + xcc_id * adev->gfx.num_compute_rings];
2201	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
2202	mutex_lock(&adev->srbm_mutex);
2203	soc15_grbm_select(adev, me: ring->me,
2204	pipe: ring->pipe,
2205	queue: ring->queue, vmid: 0, GET_INST(GC, xcc_id));
2206	gfx_v9_4_3_xcc_q_fini_register(ring, xcc_id);
2207	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, GET_INST(GC, xcc_id));
2208	mutex_unlock(lock: &adev->srbm_mutex);
2209	}
2210	}
2211
2212	return 0;
2213	}
2214
2215	static int gfx_v9_4_3_xcc_kiq_resume(struct amdgpu_device *adev, int xcc_id)
2216	{
2217	gfx_v9_4_3_xcc_kiq_init_queue(ring: &adev->gfx.kiq[xcc_id].ring, xcc_id);
2218	return 0;
2219	}
2220
2221	static int gfx_v9_4_3_xcc_kcq_resume(struct amdgpu_device *adev, int xcc_id)
2222	{
2223	struct amdgpu_ring *ring;
2224	int i;
2225
2226	gfx_v9_4_3_xcc_cp_compute_enable(adev, enable: true, xcc_id);
2227
2228	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2229	ring = &adev->gfx.compute_ring[i + xcc_id *
2230	adev->gfx.num_compute_rings];
2231
2232	gfx_v9_4_3_xcc_kcq_init_queue(ring, xcc_id, restore: false);
2233	}
2234
2235	return amdgpu_gfx_enable_kcq(adev, xcc_id);
2236	}
2237
2238	static int gfx_v9_4_3_xcc_cp_resume(struct amdgpu_device *adev, int xcc_id)
2239	{
2240	struct amdgpu_ring *ring;
2241	int r, j;
2242
2243	gfx_v9_4_3_xcc_enable_gui_idle_interrupt(adev, enable: false, xcc_id);
2244
2245	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
2246	gfx_v9_4_3_xcc_disable_gpa_mode(adev, xcc_id);
2247
2248	r = gfx_v9_4_3_xcc_cp_compute_load_microcode(adev, xcc_id);
2249	if (r)
2250	return r;
2251	} else {
2252	gfx_v9_4_3_xcc_cp_compute_enable(adev, enable: false, xcc_id);
2253	}
2254
2255	r = gfx_v9_4_3_xcc_kiq_resume(adev, xcc_id);
2256	if (r)
2257	return r;
2258
2259	r = gfx_v9_4_3_xcc_kcq_resume(adev, xcc_id);
2260	if (r)
2261	return r;
2262
2263	for (j = 0; j < adev->gfx.num_compute_rings; j++) {
2264	ring = &adev->gfx.compute_ring
2265	[j + xcc_id * adev->gfx.num_compute_rings];
2266	r = amdgpu_ring_test_helper(ring);
2267	if (r)
2268	return r;
2269	}
2270
2271	gfx_v9_4_3_xcc_enable_gui_idle_interrupt(adev, enable: true, xcc_id);
2272
2273	return 0;
2274	}
2275
2276	static int gfx_v9_4_3_cp_resume(struct amdgpu_device *adev)
2277	{
2278	int r = 0, i, num_xcc, num_xcp, num_xcc_per_xcp;
2279
2280	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2281	if (amdgpu_sriov_vf(adev)) {
2282	enum amdgpu_gfx_partition mode;
2283
2284	mode = amdgpu_xcp_query_partition_mode(xcp_mgr: adev->xcp_mgr,
2285	AMDGPU_XCP_FL_NONE);
2286	if (mode == AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
2287	return -EINVAL;
2288	num_xcc_per_xcp = gfx_v9_4_3_get_xccs_per_xcp(adev);
2289	adev->gfx.num_xcc_per_xcp = num_xcc_per_xcp;
2290	num_xcp = num_xcc / num_xcc_per_xcp;
2291	r = amdgpu_xcp_init(xcp_mgr: adev->xcp_mgr, num_xcps: num_xcp, mode);
2292
2293	} else {
2294	if (adev->in_suspend)
2295	amdgpu_xcp_restore_partition_mode(xcp_mgr: adev->xcp_mgr);
2296	else if (amdgpu_xcp_query_partition_mode(xcp_mgr: adev->xcp_mgr,
2297	AMDGPU_XCP_FL_NONE) ==
2298	AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE)
2299	r = amdgpu_xcp_switch_partition_mode(
2300	xcp_mgr: adev->xcp_mgr, mode: amdgpu_user_partt_mode);
2301	}
2302	if (r)
2303	return r;
2304
2305	for (i = 0; i < num_xcc; i++) {
2306	r = gfx_v9_4_3_xcc_cp_resume(adev, xcc_id: i);
2307	if (r)
2308	return r;
2309	}
2310
2311	return 0;
2312	}
2313
2314	static void gfx_v9_4_3_xcc_fini(struct amdgpu_device *adev, int xcc_id)
2315	{
2316	if (amdgpu_gfx_disable_kcq(adev, xcc_id))
2317	DRM_ERROR("XCD %d KCQ disable failed\n", xcc_id);
2318
2319	if (amdgpu_sriov_vf(adev)) {
2320	/* must disable polling for SRIOV when hw finished, otherwise
2321	* CPC engine may still keep fetching WB address which is already
2322	* invalid after sw finished and trigger DMAR reading error in
2323	* hypervisor side.
2324	*/
2325	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_WPTR_POLL_CNTL, EN, 0);
2326	return;
2327	}
2328
2329	/* Use deinitialize sequence from CAIL when unbinding device
2330	* from driver, otherwise KIQ is hanging when binding back
2331	*/
2332	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
2333	mutex_lock(&adev->srbm_mutex);
2334	soc15_grbm_select(adev, me: adev->gfx.kiq[xcc_id].ring.me,
2335	pipe: adev->gfx.kiq[xcc_id].ring.pipe,
2336	queue: adev->gfx.kiq[xcc_id].ring.queue, vmid: 0,
2337	GET_INST(GC, xcc_id));
2338	gfx_v9_4_3_xcc_q_fini_register(ring: &adev->gfx.kiq[xcc_id].ring,
2339	xcc_id);
2340	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, GET_INST(GC, xcc_id));
2341	mutex_unlock(lock: &adev->srbm_mutex);
2342	}
2343
2344	gfx_v9_4_3_xcc_kcq_fini_register(adev, xcc_id);
2345	gfx_v9_4_3_xcc_cp_compute_enable(adev, enable: false, xcc_id);
2346	}
2347
2348	static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block)
2349	{
2350	int r;
2351	struct amdgpu_device *adev = ip_block->adev;
2352
2353	amdgpu_gfx_cleaner_shader_init(adev, cleaner_shader_size: adev->gfx.cleaner_shader_size,
2354	cleaner_shader_ptr: adev->gfx.cleaner_shader_ptr);
2355
2356	if (!amdgpu_sriov_vf(adev))
2357	gfx_v9_4_3_init_golden_registers(adev);
2358
2359	gfx_v9_4_3_constants_init(adev);
2360
2361	r = adev->gfx.rlc.funcs->resume(adev);
2362	if (r)
2363	return r;
2364
2365	r = gfx_v9_4_3_cp_resume(adev);
2366	if (r)
2367	return r;
2368
2369	return r;
2370	}
2371
2372	static int gfx_v9_4_3_hw_fini(struct amdgpu_ip_block *ip_block)
2373	{
2374	struct amdgpu_device *adev = ip_block->adev;
2375	int i, num_xcc;
2376
2377	amdgpu_irq_put(adev, src: &adev->gfx.priv_reg_irq, type: 0);
2378	amdgpu_irq_put(adev, src: &adev->gfx.priv_inst_irq, type: 0);
2379	amdgpu_irq_put(adev, src: &adev->gfx.bad_op_irq, type: 0);
2380
2381	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2382	for (i = 0; i < num_xcc; i++) {
2383	gfx_v9_4_3_xcc_fini(adev, xcc_id: i);
2384	}
2385
2386	return 0;
2387	}
2388
2389	static int gfx_v9_4_3_suspend(struct amdgpu_ip_block *ip_block)
2390	{
2391	return gfx_v9_4_3_hw_fini(ip_block);
2392	}
2393
2394	static int gfx_v9_4_3_resume(struct amdgpu_ip_block *ip_block)
2395	{
2396	return gfx_v9_4_3_hw_init(ip_block);
2397	}
2398
2399	static bool gfx_v9_4_3_is_idle(struct amdgpu_ip_block *ip_block)
2400	{
2401	struct amdgpu_device *adev = ip_block->adev;
2402	int i, num_xcc;
2403
2404	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2405	for (i = 0; i < num_xcc; i++) {
2406	if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), regGRBM_STATUS),
2407	GRBM_STATUS, GUI_ACTIVE))
2408	return false;
2409	}
2410	return true;
2411	}
2412
2413	static int gfx_v9_4_3_wait_for_idle(struct amdgpu_ip_block *ip_block)
2414	{
2415	unsigned i;
2416	struct amdgpu_device *adev = ip_block->adev;
2417
2418	for (i = 0; i < adev->usec_timeout; i++) {
2419	if (gfx_v9_4_3_is_idle(ip_block))
2420	return 0;
2421	udelay(usec: 1);
2422	}
2423	return -ETIMEDOUT;
2424	}
2425
2426	static int gfx_v9_4_3_soft_reset(struct amdgpu_ip_block *ip_block)
2427	{
2428	u32 grbm_soft_reset = 0;
2429	u32 tmp;
2430	struct amdgpu_device *adev = ip_block->adev;
2431
2432	/* GRBM_STATUS */
2433	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_STATUS);
2434	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
2435	GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
2436	GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
2437	GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
2438	GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
2439	GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
2440	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2441	GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
2442	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2443	GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
2444	}
2445
2446	if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
2447	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2448	GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
2449	}
2450
2451	/* GRBM_STATUS2 */
2452	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_STATUS2);
2453	if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
2454	grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
2455	GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2456
2457
2458	if (grbm_soft_reset) {
2459	/* stop the rlc */
2460	adev->gfx.rlc.funcs->stop(adev);
2461
2462	/* Disable MEC parsing/prefetching */
2463	gfx_v9_4_3_xcc_cp_compute_enable(adev, enable: false, xcc_id: 0);
2464
2465	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2466	tmp |= grbm_soft_reset;
2467	dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
2468	WREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET, tmp);
2469	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2470
2471	udelay(usec: 50);
2472
2473	tmp &= ~grbm_soft_reset;
2474	WREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET, tmp);
2475	tmp = RREG32_SOC15(GC, GET_INST(GC, 0), regGRBM_SOFT_RESET);
2476
2477	/* Wait a little for things to settle down */
2478	udelay(usec: 50);
2479	}
2480	return 0;
2481	}
2482
2483	static void gfx_v9_4_3_ring_emit_gds_switch(struct amdgpu_ring *ring,
2484	uint32_t vmid,
2485	uint32_t gds_base, uint32_t gds_size,
2486	uint32_t gws_base, uint32_t gws_size,
2487	uint32_t oa_base, uint32_t oa_size)
2488	{
2489	struct amdgpu_device *adev = ring->adev;
2490
2491	/* GDS Base */
2492	gfx_v9_4_3_write_data_to_reg(ring, eng_sel: 0, wc: false,
2493	SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regGDS_VMID0_BASE) + 2 * vmid,
2494	val: gds_base);
2495
2496	/* GDS Size */
2497	gfx_v9_4_3_write_data_to_reg(ring, eng_sel: 0, wc: false,
2498	SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regGDS_VMID0_SIZE) + 2 * vmid,
2499	val: gds_size);
2500
2501	/* GWS */
2502	gfx_v9_4_3_write_data_to_reg(ring, eng_sel: 0, wc: false,
2503	SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regGDS_GWS_VMID0) + vmid,
2504	val: gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
2505
2506	/* OA */
2507	gfx_v9_4_3_write_data_to_reg(ring, eng_sel: 0, wc: false,
2508	SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regGDS_OA_VMID0) + vmid,
2509	val: (1 << (oa_size + oa_base)) - (1 << oa_base));
2510	}
2511
2512	static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block)
2513	{
2514	struct amdgpu_device *adev = ip_block->adev;
2515
2516	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
2517	AMDGPU_MAX_COMPUTE_RINGS);
2518	gfx_v9_4_3_set_kiq_pm4_funcs(adev);
2519	gfx_v9_4_3_set_ring_funcs(adev);
2520	gfx_v9_4_3_set_irq_funcs(adev);
2521	gfx_v9_4_3_set_gds_init(adev);
2522	gfx_v9_4_3_set_rlc_funcs(adev);
2523
2524	/* init rlcg reg access ctrl */
2525	gfx_v9_4_3_init_rlcg_reg_access_ctrl(adev);
2526
2527	return gfx_v9_4_3_init_microcode(adev);
2528	}
2529
2530	static int gfx_v9_4_3_late_init(struct amdgpu_ip_block *ip_block)
2531	{
2532	struct amdgpu_device *adev = ip_block->adev;
2533	int r;
2534
2535	r = amdgpu_irq_get(adev, src: &adev->gfx.priv_reg_irq, type: 0);
2536	if (r)
2537	return r;
2538
2539	r = amdgpu_irq_get(adev, src: &adev->gfx.priv_inst_irq, type: 0);
2540	if (r)
2541	return r;
2542
2543	r = amdgpu_irq_get(adev, src: &adev->gfx.bad_op_irq, type: 0);
2544	if (r)
2545	return r;
2546
2547	if (adev->gfx.ras &&
2548	adev->gfx.ras->enable_watchdog_timer)
2549	adev->gfx.ras->enable_watchdog_timer(adev);
2550
2551	return 0;
2552	}
2553
2554	static void gfx_v9_4_3_xcc_update_sram_fgcg(struct amdgpu_device *adev,
2555	bool enable, int xcc_id)
2556	{
2557	uint32_t def, data;
2558
2559	if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG))
2560	return;
2561
2562	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
2563	regRLC_CGTT_MGCG_OVERRIDE);
2564
2565	if (enable)
2566	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
2567	else
2568	data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK;
2569
2570	if (def != data)
2571	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2572	regRLC_CGTT_MGCG_OVERRIDE, data);
2573
2574	}
2575
2576	static void gfx_v9_4_3_xcc_update_repeater_fgcg(struct amdgpu_device *adev,
2577	bool enable, int xcc_id)
2578	{
2579	uint32_t def, data;
2580
2581	if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG))
2582	return;
2583
2584	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
2585	regRLC_CGTT_MGCG_OVERRIDE);
2586
2587	if (enable)
2588	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REP_FGCG_OVERRIDE_MASK;
2589	else
2590	data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REP_FGCG_OVERRIDE_MASK;
2591
2592	if (def != data)
2593	WREG32_SOC15(GC, GET_INST(GC, xcc_id),
2594	regRLC_CGTT_MGCG_OVERRIDE, data);
2595	}
2596
2597	static void
2598	gfx_v9_4_3_xcc_update_medium_grain_clock_gating(struct amdgpu_device *adev,
2599	bool enable, int xcc_id)
2600	{
2601	uint32_t data, def;
2602
2603	/* It is disabled by HW by default */
2604	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
2605	/* 1 - RLC_CGTT_MGCG_OVERRIDE */
2606	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
2607
2608	data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2609	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
2610	RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
2611	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
2612
2613	if (def != data)
2614	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2615
2616	/* MGLS is a global flag to control all MGLS in GFX */
2617	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
2618	/* 2 - RLC memory Light sleep */
2619	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
2620	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_MEM_SLP_CNTL);
2621	data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
2622	if (def != data)
2623	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_MEM_SLP_CNTL, data);
2624	}
2625	/* 3 - CP memory Light sleep */
2626	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
2627	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEM_SLP_CNTL);
2628	data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2629	if (def != data)
2630	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEM_SLP_CNTL, data);
2631	}
2632	}
2633	} else {
2634	/* 1 - MGCG_OVERRIDE */
2635	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
2636
2637	data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
2638	RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
2639	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
2640	RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
2641
2642	if (def != data)
2643	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2644
2645	/* 2 - disable MGLS in RLC */
2646	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_MEM_SLP_CNTL);
2647	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
2648	data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
2649	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_MEM_SLP_CNTL, data);
2650	}
2651
2652	/* 3 - disable MGLS in CP */
2653	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEM_SLP_CNTL);
2654	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
2655	data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
2656	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEM_SLP_CNTL, data);
2657	}
2658	}
2659
2660	}
2661
2662	static void
2663	gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
2664	bool enable, int xcc_id)
2665	{
2666	uint32_t def, data;
2667
2668	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
2669
2670	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
2671	/* unset CGCG override */
2672	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
2673	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
2674	data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
2675	else
2676	data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
2677	/* update CGCG and CGLS override bits */
2678	if (def != data)
2679	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
2680
2681	/* CGCG Hysteresis: 400us */
2682	def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
2683
2684	data = (0x2710
2685	<< RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
2686	RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
2687	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
2688	data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
2689	RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
2690	if (def != data)
2691	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);
2692
2693	/* set IDLE_POLL_COUNT(0x33450100)*/
2694	def = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL);
2695	data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
2696	(0x3345 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2697	if (def != data)
2698	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL, data);
2699	} else {
2700	def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL);
2701	/* reset CGCG/CGLS bits */
2702	data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
2703	/* disable cgcg and cgls in FSM */
2704	if (def != data)
2705	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data);
2706	}
2707
2708	}
2709
2710	static int gfx_v9_4_3_xcc_update_gfx_clock_gating(struct amdgpu_device *adev,
2711	bool enable, int xcc_id)
2712	{
2713	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id);
2714
2715	if (enable) {
2716	/* FGCG */
2717	gfx_v9_4_3_xcc_update_sram_fgcg(adev, enable, xcc_id);
2718	gfx_v9_4_3_xcc_update_repeater_fgcg(adev, enable, xcc_id);
2719
2720	/* CGCG/CGLS should be enabled after MGCG/MGLS
2721	* === MGCG + MGLS ===
2722	*/
2723	gfx_v9_4_3_xcc_update_medium_grain_clock_gating(adev, enable,
2724	xcc_id);
2725	/* === CGCG + CGLS === */
2726	gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(adev, enable,
2727	xcc_id);
2728	} else {
2729	/* CGCG/CGLS should be disabled before MGCG/MGLS
2730	* === CGCG + CGLS ===
2731	*/
2732	gfx_v9_4_3_xcc_update_coarse_grain_clock_gating(adev, enable,
2733	xcc_id);
2734	/* === MGCG + MGLS === */
2735	gfx_v9_4_3_xcc_update_medium_grain_clock_gating(adev, enable,
2736	xcc_id);
2737
2738	/* FGCG */
2739	gfx_v9_4_3_xcc_update_sram_fgcg(adev, enable, xcc_id);
2740	gfx_v9_4_3_xcc_update_repeater_fgcg(adev, enable, xcc_id);
2741	}
2742
2743	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id);
2744
2745	return 0;
2746	}
2747
2748	static const struct amdgpu_rlc_funcs gfx_v9_4_3_rlc_funcs = {
2749	.is_rlc_enabled = gfx_v9_4_3_is_rlc_enabled,
2750	.set_safe_mode = gfx_v9_4_3_xcc_set_safe_mode,
2751	.unset_safe_mode = gfx_v9_4_3_xcc_unset_safe_mode,
2752	.init = gfx_v9_4_3_rlc_init,
2753	.resume = gfx_v9_4_3_rlc_resume,
2754	.stop = gfx_v9_4_3_rlc_stop,
2755	.reset = gfx_v9_4_3_rlc_reset,
2756	.start = gfx_v9_4_3_rlc_start,
2757	.update_spm_vmid = gfx_v9_4_3_update_spm_vmid,
2758	.is_rlcg_access_range = gfx_v9_4_3_is_rlcg_access_range,
2759	};
2760
2761	static int gfx_v9_4_3_set_powergating_state(struct amdgpu_ip_block *ip_block,
2762	enum amd_powergating_state state)
2763	{
2764	return 0;
2765	}
2766
2767	static int gfx_v9_4_3_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2768	enum amd_clockgating_state state)
2769	{
2770	struct amdgpu_device *adev = ip_block->adev;
2771	int i, num_xcc;
2772
2773	if (amdgpu_sriov_vf(adev))
2774	return 0;
2775
2776	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
2777	for (i = 0; i < num_xcc; i++)
2778	gfx_v9_4_3_xcc_update_gfx_clock_gating(
2779	adev, enable: state == AMD_CG_STATE_GATE, xcc_id: i);
2780
2781	return 0;
2782	}
2783
2784	static void gfx_v9_4_3_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
2785	{
2786	struct amdgpu_device *adev = ip_block->adev;
2787	int data;
2788
2789	if (amdgpu_sriov_vf(adev))
2790	*flags = 0;
2791
2792	/* AMD_CG_SUPPORT_GFX_MGCG */
2793	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_CGTT_MGCG_OVERRIDE));
2794	if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
2795	*flags |= AMD_CG_SUPPORT_GFX_MGCG;
2796
2797	/* AMD_CG_SUPPORT_GFX_CGCG */
2798	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_CGCG_CGLS_CTRL));
2799	if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
2800	*flags |= AMD_CG_SUPPORT_GFX_CGCG;
2801
2802	/* AMD_CG_SUPPORT_GFX_CGLS */
2803	if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
2804	*flags |= AMD_CG_SUPPORT_GFX_CGLS;
2805
2806	/* AMD_CG_SUPPORT_GFX_RLC_LS */
2807	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regRLC_MEM_SLP_CNTL));
2808	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
2809	*flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
2810
2811	/* AMD_CG_SUPPORT_GFX_CP_LS */
2812	data = RREG32_KIQ(SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCP_MEM_SLP_CNTL));
2813	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
2814	*flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
2815	}
2816
2817	static void gfx_v9_4_3_ring_emit_hdp_flush(struct amdgpu_ring *ring)
2818	{
2819	struct amdgpu_device *adev = ring->adev;
2820	u32 ref_and_mask, reg_mem_engine;
2821	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
2822
2823	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
2824	switch (ring->me) {
2825	case 1:
2826	ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
2827	break;
2828	case 2:
2829	ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
2830	break;
2831	default:
2832	return;
2833	}
2834	reg_mem_engine = 0;
2835	} else {
2836	ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
2837	reg_mem_engine = 1; /* pfp */
2838	}
2839
2840	gfx_v9_4_3_wait_reg_mem(ring, eng_sel: reg_mem_engine, mem_space: 0, opt: 1,
2841	addr0: adev->nbio.funcs->get_hdp_flush_req_offset(adev),
2842	addr1: adev->nbio.funcs->get_hdp_flush_done_offset(adev),
2843	ref: ref_and_mask, mask: ref_and_mask, inv: 0x20);
2844	}
2845
2846	static void gfx_v9_4_3_ring_emit_ib_compute(struct amdgpu_ring *ring,
2847	struct amdgpu_job *job,
2848	struct amdgpu_ib *ib,
2849	uint32_t flags)
2850	{
2851	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
2852	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
2853
2854	/* Currently, there is a high possibility to get wave ID mismatch
2855	* between ME and GDS, leading to a hw deadlock, because ME generates
2856	* different wave IDs than the GDS expects. This situation happens
2857	* randomly when at least 5 compute pipes use GDS ordered append.
2858	* The wave IDs generated by ME are also wrong after suspend/resume.
2859	* Those are probably bugs somewhere else in the kernel driver.
2860	*
2861	* Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
2862	* GDS to 0 for this ring (me/pipe).
2863	*/
2864	if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
2865	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2866	amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID);
2867	amdgpu_ring_write(ring, v: ring->adev->gds.gds_compute_max_wave_id);
2868	}
2869
2870	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2871	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
2872	amdgpu_ring_write(ring,
2873	#ifdef __BIG_ENDIAN
2874	(2 << 0) |
2875	#endif
2876	lower_32_bits(ib->gpu_addr));
2877	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
2878	amdgpu_ring_write(ring, v: control);
2879	}
2880
2881	static void gfx_v9_4_3_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
2882	u64 seq, unsigned flags)
2883	{
2884	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
2885	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
2886	bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
2887
2888	/* RELEASE_MEM - flush caches, send int */
2889	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
2890	amdgpu_ring_write(ring, v: ((writeback ? (EOP_TC_WB_ACTION_EN |
2891	EOP_TC_NC_ACTION_EN) :
2892	(EOP_TCL1_ACTION_EN |
2893	EOP_TC_ACTION_EN |
2894	EOP_TC_WB_ACTION_EN |
2895	EOP_TC_MD_ACTION_EN)) |
2896	EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2897	EVENT_INDEX(5)));
2898	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
2899
2900	/*
2901	* the address should be Qword aligned if 64bit write, Dword
2902	* aligned if only send 32bit data low (discard data high)
2903	*/
2904	if (write64bit)
2905	BUG_ON(addr & 0x7);
2906	else
2907	BUG_ON(addr & 0x3);
2908	amdgpu_ring_write(ring, lower_32_bits(addr));
2909	amdgpu_ring_write(ring, upper_32_bits(addr));
2910	amdgpu_ring_write(ring, lower_32_bits(seq));
2911	amdgpu_ring_write(ring, upper_32_bits(seq));
2912	amdgpu_ring_write(ring, v: 0);
2913	}
2914
2915	static void gfx_v9_4_3_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
2916	{
2917	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
2918	uint32_t seq = ring->fence_drv.sync_seq;
2919	uint64_t addr = ring->fence_drv.gpu_addr;
2920
2921	gfx_v9_4_3_wait_reg_mem(ring, eng_sel: usepfp, mem_space: 1, opt: 0,
2922	lower_32_bits(addr), upper_32_bits(addr),
2923	ref: seq, mask: 0xffffffff, inv: 4);
2924	}
2925
2926	static void gfx_v9_4_3_ring_emit_vm_flush(struct amdgpu_ring *ring,
2927	unsigned vmid, uint64_t pd_addr)
2928	{
2929	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
2930	}
2931
2932	static u64 gfx_v9_4_3_ring_get_rptr_compute(struct amdgpu_ring *ring)
2933	{
2934	return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
2935	}
2936
2937	static u64 gfx_v9_4_3_ring_get_wptr_compute(struct amdgpu_ring *ring)
2938	{
2939	u64 wptr;
2940
2941	/* XXX check if swapping is necessary on BE */
2942	if (ring->use_doorbell)
2943	wptr = atomic64_read(v: (atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
2944	else
2945	BUG();
2946	return wptr;
2947	}
2948
2949	static void gfx_v9_4_3_ring_set_wptr_compute(struct amdgpu_ring *ring)
2950	{
2951	struct amdgpu_device *adev = ring->adev;
2952
2953	/* XXX check if swapping is necessary on BE */
2954	if (ring->use_doorbell) {
2955	atomic64_set(v: (atomic64_t *)&adev->wb.wb[ring->wptr_offs], i: ring->wptr);
2956	WDOORBELL64(ring->doorbell_index, ring->wptr);
2957	} else {
2958	BUG(); /* only DOORBELL method supported on gfx9 now */
2959	}
2960	}
2961
2962	static void gfx_v9_4_3_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
2963	u64 seq, unsigned int flags)
2964	{
2965	struct amdgpu_device *adev = ring->adev;
2966
2967	/* we only allocate 32bit for each seq wb address */
2968	BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
2969
2970	/* write fence seq to the "addr" */
2971	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2972	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(0) |
2973	WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
2974	amdgpu_ring_write(ring, lower_32_bits(addr));
2975	amdgpu_ring_write(ring, upper_32_bits(addr));
2976	amdgpu_ring_write(ring, lower_32_bits(seq));
2977
2978	if (flags & AMDGPU_FENCE_FLAG_INT) {
2979	/* set register to trigger INT */
2980	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2981	amdgpu_ring_write(ring, v: (WRITE_DATA_ENGINE_SEL(0) |
2982	WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
2983	amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS));
2984	amdgpu_ring_write(ring, v: 0);
2985	amdgpu_ring_write(ring, v: 0x20000000); /* src_id is 178 */
2986	}
2987	}
2988
2989	static void gfx_v9_4_3_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
2990	uint32_t reg_val_offs)
2991	{
2992	struct amdgpu_device *adev = ring->adev;
2993
2994	reg = gfx_v9_4_3_normalize_xcc_reg_offset(reg);
2995
2996	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
2997	amdgpu_ring_write(ring, v: 0 | /* src: register*/
2998	(5 << 8) | /* dst: memory */
2999	(1 << 20)); /* write confirm */
3000	amdgpu_ring_write(ring, v: reg);
3001	amdgpu_ring_write(ring, v: 0);
3002	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
3003	reg_val_offs * 4));
3004	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
3005	reg_val_offs * 4));
3006	}
3007
3008	static void gfx_v9_4_3_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
3009	uint32_t val)
3010	{
3011	uint32_t cmd = 0;
3012
3013	reg = gfx_v9_4_3_normalize_xcc_reg_offset(reg);
3014
3015	switch (ring->funcs->type) {
3016	case AMDGPU_RING_TYPE_GFX:
3017	cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
3018	break;
3019	case AMDGPU_RING_TYPE_KIQ:
3020	cmd = (1 << 16); /* no inc addr */
3021	break;
3022	default:
3023	cmd = WR_CONFIRM;
3024	break;
3025	}
3026	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3027	amdgpu_ring_write(ring, v: cmd);
3028	amdgpu_ring_write(ring, v: reg);
3029	amdgpu_ring_write(ring, v: 0);
3030	amdgpu_ring_write(ring, v: val);
3031	}
3032
3033	static void gfx_v9_4_3_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
3034	uint32_t val, uint32_t mask)
3035	{
3036	gfx_v9_4_3_wait_reg_mem(ring, eng_sel: 0, mem_space: 0, opt: 0, addr0: reg, addr1: 0, ref: val, mask, inv: 0x20);
3037	}
3038
3039	static void gfx_v9_4_3_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
3040	uint32_t reg0, uint32_t reg1,
3041	uint32_t ref, uint32_t mask)
3042	{
3043	amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, val0: reg1,
3044	reg1: ref, val1: mask);
3045	}
3046
3047	static void gfx_v9_4_3_ring_soft_recovery(struct amdgpu_ring *ring,
3048	unsigned vmid)
3049	{
3050	struct amdgpu_device *adev = ring->adev;
3051	uint32_t value = 0;
3052
3053	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
3054	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
3055	value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
3056	value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
3057	amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id: ring->xcc_id);
3058	WREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regSQ_CMD, value);
3059	amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id: ring->xcc_id);
3060	}
3061
3062	static void gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3063	struct amdgpu_device *adev, int me, int pipe,
3064	enum amdgpu_interrupt_state state, int xcc_id)
3065	{
3066	u32 mec_int_cntl, mec_int_cntl_reg;
3067
3068	/*
3069	* amdgpu controls only the first MEC. That's why this function only
3070	* handles the setting of interrupts for this specific MEC. All other
3071	* pipes' interrupts are set by amdkfd.
3072	*/
3073
3074	if (me == 1) {
3075	switch (pipe) {
3076	case 0:
3077	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE0_INT_CNTL);
3078	break;
3079	case 1:
3080	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE1_INT_CNTL);
3081	break;
3082	case 2:
3083	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE2_INT_CNTL);
3084	break;
3085	case 3:
3086	mec_int_cntl_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE3_INT_CNTL);
3087	break;
3088	default:
3089	DRM_DEBUG("invalid pipe %d\n", pipe);
3090	return;
3091	}
3092	} else {
3093	DRM_DEBUG("invalid me %d\n", me);
3094	return;
3095	}
3096
3097	switch (state) {
3098	case AMDGPU_IRQ_STATE_DISABLE:
3099	mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
3100	mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3101	TIME_STAMP_INT_ENABLE, 0);
3102	WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
3103	break;
3104	case AMDGPU_IRQ_STATE_ENABLE:
3105	mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id);
3106	mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3107	TIME_STAMP_INT_ENABLE, 1);
3108	WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id);
3109	break;
3110	default:
3111	break;
3112	}
3113	}
3114
3115	static u32 gfx_v9_4_3_get_cpc_int_cntl(struct amdgpu_device *adev,
3116	int xcc_id, int me, int pipe)
3117	{
3118	/*
3119	* amdgpu controls only the first MEC. That's why this function only
3120	* handles the setting of interrupts for this specific MEC. All other
3121	* pipes' interrupts are set by amdkfd.
3122	*/
3123	if (me != 1)
3124	return 0;
3125
3126	switch (pipe) {
3127	case 0:
3128	return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE0_INT_CNTL);
3129	case 1:
3130	return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE1_INT_CNTL);
3131	case 2:
3132	return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE2_INT_CNTL);
3133	case 3:
3134	return SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regCP_ME1_PIPE3_INT_CNTL);
3135	default:
3136	return 0;
3137	}
3138	}
3139
3140	static int gfx_v9_4_3_set_priv_reg_fault_state(struct amdgpu_device *adev,
3141	struct amdgpu_irq_src *source,
3142	unsigned type,
3143	enum amdgpu_interrupt_state state)
3144	{
3145	u32 mec_int_cntl_reg, mec_int_cntl;
3146	int i, j, k, num_xcc;
3147
3148	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3149	switch (state) {
3150	case AMDGPU_IRQ_STATE_DISABLE:
3151	case AMDGPU_IRQ_STATE_ENABLE:
3152	for (i = 0; i < num_xcc; i++) {
3153	WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
3154	PRIV_REG_INT_ENABLE,
3155	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3156	for (j = 0; j < adev->gfx.mec.num_mec; j++) {
3157	for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
3158	/* MECs start at 1 */
3159	mec_int_cntl_reg = gfx_v9_4_3_get_cpc_int_cntl(adev, xcc_id: i, me: j + 1, pipe: k);
3160
3161	if (mec_int_cntl_reg) {
3162	mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, i);
3163	mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3164	PRIV_REG_INT_ENABLE,
3165	state == AMDGPU_IRQ_STATE_ENABLE ?
3166	1 : 0);
3167	WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, i);
3168	}
3169	}
3170	}
3171	}
3172	break;
3173	default:
3174	break;
3175	}
3176
3177	return 0;
3178	}
3179
3180	static int gfx_v9_4_3_set_bad_op_fault_state(struct amdgpu_device *adev,
3181	struct amdgpu_irq_src *source,
3182	unsigned type,
3183	enum amdgpu_interrupt_state state)
3184	{
3185	u32 mec_int_cntl_reg, mec_int_cntl;
3186	int i, j, k, num_xcc;
3187
3188	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3189	switch (state) {
3190	case AMDGPU_IRQ_STATE_DISABLE:
3191	case AMDGPU_IRQ_STATE_ENABLE:
3192	for (i = 0; i < num_xcc; i++) {
3193	WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
3194	OPCODE_ERROR_INT_ENABLE,
3195	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3196	for (j = 0; j < adev->gfx.mec.num_mec; j++) {
3197	for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
3198	/* MECs start at 1 */
3199	mec_int_cntl_reg = gfx_v9_4_3_get_cpc_int_cntl(adev, xcc_id: i, me: j + 1, pipe: k);
3200
3201	if (mec_int_cntl_reg) {
3202	mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, i);
3203	mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
3204	OPCODE_ERROR_INT_ENABLE,
3205	state == AMDGPU_IRQ_STATE_ENABLE ?
3206	1 : 0);
3207	WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, i);
3208	}
3209	}
3210	}
3211	}
3212	break;
3213	default:
3214	break;
3215	}
3216
3217	return 0;
3218	}
3219
3220	static int gfx_v9_4_3_set_priv_inst_fault_state(struct amdgpu_device *adev,
3221	struct amdgpu_irq_src *source,
3222	unsigned type,
3223	enum amdgpu_interrupt_state state)
3224	{
3225	int i, num_xcc;
3226
3227	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3228	switch (state) {
3229	case AMDGPU_IRQ_STATE_DISABLE:
3230	case AMDGPU_IRQ_STATE_ENABLE:
3231	for (i = 0; i < num_xcc; i++)
3232	WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0,
3233	PRIV_INSTR_INT_ENABLE,
3234	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
3235	break;
3236	default:
3237	break;
3238	}
3239
3240	return 0;
3241	}
3242
3243	static int gfx_v9_4_3_set_eop_interrupt_state(struct amdgpu_device *adev,
3244	struct amdgpu_irq_src *src,
3245	unsigned type,
3246	enum amdgpu_interrupt_state state)
3247	{
3248	int i, num_xcc;
3249
3250	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
3251	for (i = 0; i < num_xcc; i++) {
3252	switch (type) {
3253	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
3254	gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3255	adev, me: 1, pipe: 0, state, xcc_id: i);
3256	break;
3257	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
3258	gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3259	adev, me: 1, pipe: 1, state, xcc_id: i);
3260	break;
3261	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
3262	gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3263	adev, me: 1, pipe: 2, state, xcc_id: i);
3264	break;
3265	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
3266	gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3267	adev, me: 1, pipe: 3, state, xcc_id: i);
3268	break;
3269	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
3270	gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3271	adev, me: 2, pipe: 0, state, xcc_id: i);
3272	break;
3273	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
3274	gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3275	adev, me: 2, pipe: 1, state, xcc_id: i);
3276	break;
3277	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
3278	gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3279	adev, me: 2, pipe: 2, state, xcc_id: i);
3280	break;
3281	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
3282	gfx_v9_4_3_xcc_set_compute_eop_interrupt_state(
3283	adev, me: 2, pipe: 3, state, xcc_id: i);
3284	break;
3285	default:
3286	break;
3287	}
3288	}
3289
3290	return 0;
3291	}
3292
3293	static int gfx_v9_4_3_eop_irq(struct amdgpu_device *adev,
3294	struct amdgpu_irq_src *source,
3295	struct amdgpu_iv_entry *entry)
3296	{
3297	int i, xcc_id;
3298	u8 me_id, pipe_id, queue_id;
3299	struct amdgpu_ring *ring;
3300
3301	DRM_DEBUG("IH: CP EOP\n");
3302	me_id = (entry->ring_id & 0x0c) >> 2;
3303	pipe_id = (entry->ring_id & 0x03) >> 0;
3304	queue_id = (entry->ring_id & 0x70) >> 4;
3305
3306	xcc_id = gfx_v9_4_3_ih_to_xcc_inst(adev, ih_node: entry->node_id);
3307
3308	if (xcc_id == -EINVAL)
3309	return -EINVAL;
3310
3311	switch (me_id) {
3312	case 0:
3313	case 1:
3314	case 2:
3315	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3316	ring = &adev->gfx.compute_ring
3317	[i +
3318	xcc_id * adev->gfx.num_compute_rings];
3319	/* Per-queue interrupt is supported for MEC starting from VI.
3320	* The interrupt can only be enabled/disabled per pipe instead of per queue.
3321	*/
3322
3323	if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
3324	amdgpu_fence_process(ring);
3325	}
3326	break;
3327	}
3328	return 0;
3329	}
3330
3331	static void gfx_v9_4_3_fault(struct amdgpu_device *adev,
3332	struct amdgpu_iv_entry *entry)
3333	{
3334	u8 me_id, pipe_id, queue_id;
3335	struct amdgpu_ring *ring;
3336	int i, xcc_id;
3337
3338	me_id = (entry->ring_id & 0x0c) >> 2;
3339	pipe_id = (entry->ring_id & 0x03) >> 0;
3340	queue_id = (entry->ring_id & 0x70) >> 4;
3341
3342	xcc_id = gfx_v9_4_3_ih_to_xcc_inst(adev, ih_node: entry->node_id);
3343
3344	if (xcc_id == -EINVAL)
3345	return;
3346
3347	switch (me_id) {
3348	case 0:
3349	case 1:
3350	case 2:
3351	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3352	ring = &adev->gfx.compute_ring
3353	[i +
3354	xcc_id * adev->gfx.num_compute_rings];
3355	if (ring->me == me_id && ring->pipe == pipe_id &&
3356	ring->queue == queue_id)
3357	drm_sched_fault(sched: &ring->sched);
3358	}
3359	break;
3360	}
3361	}
3362
3363	static int gfx_v9_4_3_priv_reg_irq(struct amdgpu_device *adev,
3364	struct amdgpu_irq_src *source,
3365	struct amdgpu_iv_entry *entry)
3366	{
3367	DRM_ERROR("Illegal register access in command stream\n");
3368	gfx_v9_4_3_fault(adev, entry);
3369	return 0;
3370	}
3371
3372	static int gfx_v9_4_3_bad_op_irq(struct amdgpu_device *adev,
3373	struct amdgpu_irq_src *source,
3374	struct amdgpu_iv_entry *entry)
3375	{
3376	DRM_ERROR("Illegal opcode in command stream\n");
3377	gfx_v9_4_3_fault(adev, entry);
3378	return 0;
3379	}
3380
3381	static int gfx_v9_4_3_priv_inst_irq(struct amdgpu_device *adev,
3382	struct amdgpu_irq_src *source,
3383	struct amdgpu_iv_entry *entry)
3384	{
3385	DRM_ERROR("Illegal instruction in command stream\n");
3386	gfx_v9_4_3_fault(adev, entry);
3387	return 0;
3388	}
3389
3390	static void gfx_v9_4_3_emit_mem_sync(struct amdgpu_ring *ring)
3391	{
3392	const unsigned int cp_coher_cntl =
3393	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) |
3394	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) |
3395	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) |
3396	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) |
3397	PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1);
3398
3399	/* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */
3400	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
3401	amdgpu_ring_write(ring, v: cp_coher_cntl); /* CP_COHER_CNTL */
3402	amdgpu_ring_write(ring, v: 0xffffffff); /* CP_COHER_SIZE */
3403	amdgpu_ring_write(ring, v: 0xffffff); /* CP_COHER_SIZE_HI */
3404	amdgpu_ring_write(ring, v: 0); /* CP_COHER_BASE */
3405	amdgpu_ring_write(ring, v: 0); /* CP_COHER_BASE_HI */
3406	amdgpu_ring_write(ring, v: 0x0000000A); /* POLL_INTERVAL */
3407	}
3408
3409	static void gfx_v9_4_3_emit_wave_limit_cs(struct amdgpu_ring *ring,
3410	uint32_t pipe, bool enable)
3411	{
3412	struct amdgpu_device *adev = ring->adev;
3413	uint32_t val;
3414	uint32_t wcl_cs_reg;
3415
3416	/* regSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
3417	val = enable ? 0x1 : 0x7f;
3418
3419	switch (pipe) {
3420	case 0:
3421	wcl_cs_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_CS0);
3422	break;
3423	case 1:
3424	wcl_cs_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_CS1);
3425	break;
3426	case 2:
3427	wcl_cs_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_CS2);
3428	break;
3429	case 3:
3430	wcl_cs_reg = SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_CS3);
3431	break;
3432	default:
3433	DRM_DEBUG("invalid pipe %d\n", pipe);
3434	return;
3435	}
3436
3437	amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
3438
3439	}
3440	static void gfx_v9_4_3_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
3441	{
3442	struct amdgpu_device *adev = ring->adev;
3443	uint32_t val;
3444	int i;
3445
3446	/* regSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
3447	* number of gfx waves. Setting 5 bit will make sure gfx only gets
3448	* around 25% of gpu resources.
3449	*/
3450	val = enable ? 0x1f : 0x07ffffff;
3451	amdgpu_ring_emit_wreg(ring,
3452	SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regSPI_WCL_PIPE_PERCENT_GFX),
3453	val);
3454
3455	/* Restrict waves for normal/low priority compute queues as well
3456	* to get best QoS for high priority compute jobs.
3457	*
3458	* amdgpu controls only 1st ME(0-3 CS pipes).
3459	*/
3460	for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
3461	if (i != ring->pipe)
3462	gfx_v9_4_3_emit_wave_limit_cs(ring, pipe: i, enable);
3463
3464	}
3465	}
3466
3467	static int gfx_v9_4_3_unmap_done(struct amdgpu_device *adev, uint32_t me,
3468	uint32_t pipe, uint32_t queue,
3469	uint32_t xcc_id)
3470	{
3471	int i, r;
3472	/* make sure dequeue is complete*/
3473	gfx_v9_4_3_xcc_set_safe_mode(adev, xcc_id);
3474	mutex_lock(&adev->srbm_mutex);
3475	soc15_grbm_select(adev, me, pipe, queue, vmid: 0, GET_INST(GC, xcc_id));
3476	for (i = 0; i < adev->usec_timeout; i++) {
3477	if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
3478	break;
3479	udelay(usec: 1);
3480	}
3481	if (i >= adev->usec_timeout)
3482	r = -ETIMEDOUT;
3483	else
3484	r = 0;
3485	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, GET_INST(GC, xcc_id));
3486	mutex_unlock(lock: &adev->srbm_mutex);
3487	gfx_v9_4_3_xcc_unset_safe_mode(adev, xcc_id);
3488
3489	return r;
3490
3491	}
3492
3493	static bool gfx_v9_4_3_pipe_reset_support(struct amdgpu_device *adev)
3494	{
3495	if (!!(adev->gfx.compute_supported_reset & AMDGPU_RESET_TYPE_PER_PIPE))
3496	return true;
3497	else
3498	dev_warn_once(adev->dev, "Please use the latest MEC version to see whether support pipe reset\n");
3499
3500	return false;
3501	}
3502
3503	static int gfx_v9_4_3_reset_hw_pipe(struct amdgpu_ring *ring)
3504	{
3505	struct amdgpu_device *adev = ring->adev;
3506	uint32_t reset_pipe, clean_pipe;
3507	int r;
3508
3509	if (!gfx_v9_4_3_pipe_reset_support(adev))
3510	return -EINVAL;
3511
3512	gfx_v9_4_3_xcc_set_safe_mode(adev, xcc_id: ring->xcc_id);
3513	mutex_lock(&adev->srbm_mutex);
3514
3515	reset_pipe = RREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regCP_MEC_CNTL);
3516	clean_pipe = reset_pipe;
3517
3518	if (ring->me == 1) {
3519	switch (ring->pipe) {
3520	case 0:
3521	reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3522	MEC_ME1_PIPE0_RESET, 1);
3523	break;
3524	case 1:
3525	reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3526	MEC_ME1_PIPE1_RESET, 1);
3527	break;
3528	case 2:
3529	reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3530	MEC_ME1_PIPE2_RESET, 1);
3531	break;
3532	case 3:
3533	reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3534	MEC_ME1_PIPE3_RESET, 1);
3535	break;
3536	default:
3537	break;
3538	}
3539	} else {
3540	if (ring->pipe)
3541	reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3542	MEC_ME2_PIPE1_RESET, 1);
3543	else
3544	reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
3545	MEC_ME2_PIPE0_RESET, 1);
3546	}
3547
3548	WREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regCP_MEC_CNTL, reset_pipe);
3549	WREG32_SOC15(GC, GET_INST(GC, ring->xcc_id), regCP_MEC_CNTL, clean_pipe);
3550	mutex_unlock(lock: &adev->srbm_mutex);
3551	gfx_v9_4_3_xcc_unset_safe_mode(adev, xcc_id: ring->xcc_id);
3552
3553	r = gfx_v9_4_3_unmap_done(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, xcc_id: ring->xcc_id);
3554	return r;
3555	}
3556
3557	static int gfx_v9_4_3_reset_kcq(struct amdgpu_ring *ring,
3558	unsigned int vmid,
3559	struct amdgpu_fence *timedout_fence)
3560	{
3561	struct amdgpu_device *adev = ring->adev;
3562	struct amdgpu_kiq *kiq = &adev->gfx.kiq[ring->xcc_id];
3563	struct amdgpu_ring *kiq_ring = &kiq->ring;
3564	int reset_mode = AMDGPU_RESET_TYPE_PER_QUEUE;
3565	unsigned long flags;
3566	int r;
3567
3568	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
3569	return -EINVAL;
3570
3571	amdgpu_ring_reset_helper_begin(ring, guilty_fence: timedout_fence);
3572
3573	spin_lock_irqsave(&kiq->ring_lock, flags);
3574
3575	if (amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->unmap_queues_size)) {
3576	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
3577	return -ENOMEM;
3578	}
3579
3580	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES,
3581	0, 0);
3582	amdgpu_ring_commit(ring: kiq_ring);
3583
3584	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
3585
3586	r = amdgpu_ring_test_ring(kiq_ring);
3587	if (r) {
3588	dev_err(adev->dev, "kiq ring test failed after ring: %s queue reset\n",
3589	ring->name);
3590	goto pipe_reset;
3591	}
3592
3593	r = gfx_v9_4_3_unmap_done(adev, me: ring->me, pipe: ring->pipe, queue: ring->queue, xcc_id: ring->xcc_id);
3594	if (r)
3595	dev_err(adev->dev, "fail to wait on hqd deactive and will try pipe reset\n");
3596
3597	pipe_reset:
3598	if (r) {
3599	if (!(adev->gfx.compute_supported_reset & AMDGPU_RESET_TYPE_PER_PIPE))
3600	return -EOPNOTSUPP;
3601	r = gfx_v9_4_3_reset_hw_pipe(ring);
3602	reset_mode = AMDGPU_RESET_TYPE_PER_PIPE;
3603	dev_info(adev->dev, "ring: %s pipe reset :%s\n", ring->name,
3604	r ? "failed" : "successfully");
3605	if (r)
3606	return r;
3607	}
3608
3609	gfx_v9_4_3_xcc_kcq_init_queue(ring, xcc_id: ring->xcc_id, restore: true);
3610
3611	spin_lock_irqsave(&kiq->ring_lock, flags);
3612	r = amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->map_queues_size);
3613	if (r) {
3614	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
3615	return -ENOMEM;
3616	}
3617	kiq->pmf->kiq_map_queues(kiq_ring, ring);
3618	amdgpu_ring_commit(ring: kiq_ring);
3619	r = amdgpu_ring_test_ring(kiq_ring);
3620	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
3621	if (r) {
3622	if (reset_mode == AMDGPU_RESET_TYPE_PER_QUEUE)
3623	goto pipe_reset;
3624
3625	dev_err(adev->dev, "fail to remap queue\n");
3626	return r;
3627	}
3628
3629	if (reset_mode == AMDGPU_RESET_TYPE_PER_QUEUE) {
3630	r = amdgpu_ring_test_ring(ring);
3631	if (r)
3632	goto pipe_reset;
3633	}
3634
3635
3636	return amdgpu_ring_reset_helper_end(ring, guilty_fence: timedout_fence);
3637	}
3638
3639	enum amdgpu_gfx_cp_ras_mem_id {
3640	AMDGPU_GFX_CP_MEM1 = 1,
3641	AMDGPU_GFX_CP_MEM2,
3642	AMDGPU_GFX_CP_MEM3,
3643	AMDGPU_GFX_CP_MEM4,
3644	AMDGPU_GFX_CP_MEM5,
3645	};
3646
3647	enum amdgpu_gfx_gcea_ras_mem_id {
3648	AMDGPU_GFX_GCEA_IOWR_CMDMEM = 4,
3649	AMDGPU_GFX_GCEA_IORD_CMDMEM,
3650	AMDGPU_GFX_GCEA_GMIWR_CMDMEM,
3651	AMDGPU_GFX_GCEA_GMIRD_CMDMEM,
3652	AMDGPU_GFX_GCEA_DRAMWR_CMDMEM,
3653	AMDGPU_GFX_GCEA_DRAMRD_CMDMEM,
3654	AMDGPU_GFX_GCEA_MAM_DMEM0,
3655	AMDGPU_GFX_GCEA_MAM_DMEM1,
3656	AMDGPU_GFX_GCEA_MAM_DMEM2,
3657	AMDGPU_GFX_GCEA_MAM_DMEM3,
3658	AMDGPU_GFX_GCEA_MAM_AMEM0,
3659	AMDGPU_GFX_GCEA_MAM_AMEM1,
3660	AMDGPU_GFX_GCEA_MAM_AMEM2,
3661	AMDGPU_GFX_GCEA_MAM_AMEM3,
3662	AMDGPU_GFX_GCEA_MAM_AFLUSH_BUFFER,
3663	AMDGPU_GFX_GCEA_WRET_TAGMEM,
3664	AMDGPU_GFX_GCEA_RRET_TAGMEM,
3665	AMDGPU_GFX_GCEA_IOWR_DATAMEM,
3666	AMDGPU_GFX_GCEA_GMIWR_DATAMEM,
3667	AMDGPU_GFX_GCEA_DRAM_DATAMEM,
3668	};
3669
3670	enum amdgpu_gfx_gc_cane_ras_mem_id {
3671	AMDGPU_GFX_GC_CANE_MEM0 = 0,
3672	};
3673
3674	enum amdgpu_gfx_gcutcl2_ras_mem_id {
3675	AMDGPU_GFX_GCUTCL2_MEM2P512X95 = 160,
3676	};
3677
3678	enum amdgpu_gfx_gds_ras_mem_id {
3679	AMDGPU_GFX_GDS_MEM0 = 0,
3680	};
3681
3682	enum amdgpu_gfx_lds_ras_mem_id {
3683	AMDGPU_GFX_LDS_BANK0 = 0,
3684	AMDGPU_GFX_LDS_BANK1,
3685	AMDGPU_GFX_LDS_BANK2,
3686	AMDGPU_GFX_LDS_BANK3,
3687	AMDGPU_GFX_LDS_BANK4,
3688	AMDGPU_GFX_LDS_BANK5,
3689	AMDGPU_GFX_LDS_BANK6,
3690	AMDGPU_GFX_LDS_BANK7,
3691	AMDGPU_GFX_LDS_BANK8,
3692	AMDGPU_GFX_LDS_BANK9,
3693	AMDGPU_GFX_LDS_BANK10,
3694	AMDGPU_GFX_LDS_BANK11,
3695	AMDGPU_GFX_LDS_BANK12,
3696	AMDGPU_GFX_LDS_BANK13,
3697	AMDGPU_GFX_LDS_BANK14,
3698	AMDGPU_GFX_LDS_BANK15,
3699	AMDGPU_GFX_LDS_BANK16,
3700	AMDGPU_GFX_LDS_BANK17,
3701	AMDGPU_GFX_LDS_BANK18,
3702	AMDGPU_GFX_LDS_BANK19,
3703	AMDGPU_GFX_LDS_BANK20,
3704	AMDGPU_GFX_LDS_BANK21,
3705	AMDGPU_GFX_LDS_BANK22,
3706	AMDGPU_GFX_LDS_BANK23,
3707	AMDGPU_GFX_LDS_BANK24,
3708	AMDGPU_GFX_LDS_BANK25,
3709	AMDGPU_GFX_LDS_BANK26,
3710	AMDGPU_GFX_LDS_BANK27,
3711	AMDGPU_GFX_LDS_BANK28,
3712	AMDGPU_GFX_LDS_BANK29,
3713	AMDGPU_GFX_LDS_BANK30,
3714	AMDGPU_GFX_LDS_BANK31,
3715	AMDGPU_GFX_LDS_SP_BUFFER_A,
3716	AMDGPU_GFX_LDS_SP_BUFFER_B,
3717	};
3718
3719	enum amdgpu_gfx_rlc_ras_mem_id {
3720	AMDGPU_GFX_RLC_GPMF32 = 1,
3721	AMDGPU_GFX_RLC_RLCVF32,
3722	AMDGPU_GFX_RLC_SCRATCH,
3723	AMDGPU_GFX_RLC_SRM_ARAM,
3724	AMDGPU_GFX_RLC_SRM_DRAM,
3725	AMDGPU_GFX_RLC_TCTAG,
3726	AMDGPU_GFX_RLC_SPM_SE,
3727	AMDGPU_GFX_RLC_SPM_GRBMT,
3728	};
3729
3730	enum amdgpu_gfx_sp_ras_mem_id {
3731	AMDGPU_GFX_SP_SIMDID0 = 0,
3732	};
3733
3734	enum amdgpu_gfx_spi_ras_mem_id {
3735	AMDGPU_GFX_SPI_MEM0 = 0,
3736	AMDGPU_GFX_SPI_MEM1,
3737	AMDGPU_GFX_SPI_MEM2,
3738	AMDGPU_GFX_SPI_MEM3,
3739	};
3740
3741	enum amdgpu_gfx_sqc_ras_mem_id {
3742	AMDGPU_GFX_SQC_INST_CACHE_A = 100,
3743	AMDGPU_GFX_SQC_INST_CACHE_B = 101,
3744	AMDGPU_GFX_SQC_INST_CACHE_TAG_A = 102,
3745	AMDGPU_GFX_SQC_INST_CACHE_TAG_B = 103,
3746	AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_A = 104,
3747	AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_B = 105,
3748	AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_A = 106,
3749	AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_B = 107,
3750	AMDGPU_GFX_SQC_DATA_CACHE_A = 200,
3751	AMDGPU_GFX_SQC_DATA_CACHE_B = 201,
3752	AMDGPU_GFX_SQC_DATA_CACHE_TAG_A = 202,
3753	AMDGPU_GFX_SQC_DATA_CACHE_TAG_B = 203,
3754	AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_A = 204,
3755	AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_B = 205,
3756	AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_A = 206,
3757	AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_B = 207,
3758	AMDGPU_GFX_SQC_DIRTY_BIT_A = 208,
3759	AMDGPU_GFX_SQC_DIRTY_BIT_B = 209,
3760	AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU0 = 210,
3761	AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU1 = 211,
3762	AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_A = 212,
3763	AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_B = 213,
3764	AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_INST_CACHE = 108,
3765	};
3766
3767	enum amdgpu_gfx_sq_ras_mem_id {
3768	AMDGPU_GFX_SQ_SGPR_MEM0 = 0,
3769	AMDGPU_GFX_SQ_SGPR_MEM1,
3770	AMDGPU_GFX_SQ_SGPR_MEM2,
3771	AMDGPU_GFX_SQ_SGPR_MEM3,
3772	};
3773
3774	enum amdgpu_gfx_ta_ras_mem_id {
3775	AMDGPU_GFX_TA_FS_AFIFO_RAM_LO = 1,
3776	AMDGPU_GFX_TA_FS_AFIFO_RAM_HI,
3777	AMDGPU_GFX_TA_FS_CFIFO_RAM,
3778	AMDGPU_GFX_TA_FSX_LFIFO,
3779	AMDGPU_GFX_TA_FS_DFIFO_RAM,
3780	};
3781
3782	enum amdgpu_gfx_tcc_ras_mem_id {
3783	AMDGPU_GFX_TCC_MEM1 = 1,
3784	};
3785
3786	enum amdgpu_gfx_tca_ras_mem_id {
3787	AMDGPU_GFX_TCA_MEM1 = 1,
3788	};
3789
3790	enum amdgpu_gfx_tci_ras_mem_id {
3791	AMDGPU_GFX_TCIW_MEM = 1,
3792	};
3793
3794	enum amdgpu_gfx_tcp_ras_mem_id {
3795	AMDGPU_GFX_TCP_LFIFO0 = 1,
3796	AMDGPU_GFX_TCP_SET0BANK0_RAM,
3797	AMDGPU_GFX_TCP_SET0BANK1_RAM,
3798	AMDGPU_GFX_TCP_SET0BANK2_RAM,
3799	AMDGPU_GFX_TCP_SET0BANK3_RAM,
3800	AMDGPU_GFX_TCP_SET1BANK0_RAM,
3801	AMDGPU_GFX_TCP_SET1BANK1_RAM,
3802	AMDGPU_GFX_TCP_SET1BANK2_RAM,
3803	AMDGPU_GFX_TCP_SET1BANK3_RAM,
3804	AMDGPU_GFX_TCP_SET2BANK0_RAM,
3805	AMDGPU_GFX_TCP_SET2BANK1_RAM,
3806	AMDGPU_GFX_TCP_SET2BANK2_RAM,
3807	AMDGPU_GFX_TCP_SET2BANK3_RAM,
3808	AMDGPU_GFX_TCP_SET3BANK0_RAM,
3809	AMDGPU_GFX_TCP_SET3BANK1_RAM,
3810	AMDGPU_GFX_TCP_SET3BANK2_RAM,
3811	AMDGPU_GFX_TCP_SET3BANK3_RAM,
3812	AMDGPU_GFX_TCP_VM_FIFO,
3813	AMDGPU_GFX_TCP_DB_TAGRAM0,
3814	AMDGPU_GFX_TCP_DB_TAGRAM1,
3815	AMDGPU_GFX_TCP_DB_TAGRAM2,
3816	AMDGPU_GFX_TCP_DB_TAGRAM3,
3817	AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE0,
3818	AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE1,
3819	AMDGPU_GFX_TCP_CMD_FIFO,
3820	};
3821
3822	enum amdgpu_gfx_td_ras_mem_id {
3823	AMDGPU_GFX_TD_UTD_CS_FIFO_MEM = 1,
3824	AMDGPU_GFX_TD_UTD_SS_FIFO_LO_MEM,
3825	AMDGPU_GFX_TD_UTD_SS_FIFO_HI_MEM,
3826	};
3827
3828	enum amdgpu_gfx_tcx_ras_mem_id {
3829	AMDGPU_GFX_TCX_FIFOD0 = 0,
3830	AMDGPU_GFX_TCX_FIFOD1,
3831	AMDGPU_GFX_TCX_FIFOD2,
3832	AMDGPU_GFX_TCX_FIFOD3,
3833	AMDGPU_GFX_TCX_FIFOD4,
3834	AMDGPU_GFX_TCX_FIFOD5,
3835	AMDGPU_GFX_TCX_FIFOD6,
3836	AMDGPU_GFX_TCX_FIFOD7,
3837	AMDGPU_GFX_TCX_FIFOB0,
3838	AMDGPU_GFX_TCX_FIFOB1,
3839	AMDGPU_GFX_TCX_FIFOB2,
3840	AMDGPU_GFX_TCX_FIFOB3,
3841	AMDGPU_GFX_TCX_FIFOB4,
3842	AMDGPU_GFX_TCX_FIFOB5,
3843	AMDGPU_GFX_TCX_FIFOB6,
3844	AMDGPU_GFX_TCX_FIFOB7,
3845	AMDGPU_GFX_TCX_FIFOA0,
3846	AMDGPU_GFX_TCX_FIFOA1,
3847	AMDGPU_GFX_TCX_FIFOA2,
3848	AMDGPU_GFX_TCX_FIFOA3,
3849	AMDGPU_GFX_TCX_FIFOA4,
3850	AMDGPU_GFX_TCX_FIFOA5,
3851	AMDGPU_GFX_TCX_FIFOA6,
3852	AMDGPU_GFX_TCX_FIFOA7,
3853	AMDGPU_GFX_TCX_CFIFO0,
3854	AMDGPU_GFX_TCX_CFIFO1,
3855	AMDGPU_GFX_TCX_CFIFO2,
3856	AMDGPU_GFX_TCX_CFIFO3,
3857	AMDGPU_GFX_TCX_CFIFO4,
3858	AMDGPU_GFX_TCX_CFIFO5,
3859	AMDGPU_GFX_TCX_CFIFO6,
3860	AMDGPU_GFX_TCX_CFIFO7,
3861	AMDGPU_GFX_TCX_FIFO_ACKB0,
3862	AMDGPU_GFX_TCX_FIFO_ACKB1,
3863	AMDGPU_GFX_TCX_FIFO_ACKB2,
3864	AMDGPU_GFX_TCX_FIFO_ACKB3,
3865	AMDGPU_GFX_TCX_FIFO_ACKB4,
3866	AMDGPU_GFX_TCX_FIFO_ACKB5,
3867	AMDGPU_GFX_TCX_FIFO_ACKB6,
3868	AMDGPU_GFX_TCX_FIFO_ACKB7,
3869	AMDGPU_GFX_TCX_FIFO_ACKD0,
3870	AMDGPU_GFX_TCX_FIFO_ACKD1,
3871	AMDGPU_GFX_TCX_FIFO_ACKD2,
3872	AMDGPU_GFX_TCX_FIFO_ACKD3,
3873	AMDGPU_GFX_TCX_FIFO_ACKD4,
3874	AMDGPU_GFX_TCX_FIFO_ACKD5,
3875	AMDGPU_GFX_TCX_FIFO_ACKD6,
3876	AMDGPU_GFX_TCX_FIFO_ACKD7,
3877	AMDGPU_GFX_TCX_DST_FIFOA0,
3878	AMDGPU_GFX_TCX_DST_FIFOA1,
3879	AMDGPU_GFX_TCX_DST_FIFOA2,
3880	AMDGPU_GFX_TCX_DST_FIFOA3,
3881	AMDGPU_GFX_TCX_DST_FIFOA4,
3882	AMDGPU_GFX_TCX_DST_FIFOA5,
3883	AMDGPU_GFX_TCX_DST_FIFOA6,
3884	AMDGPU_GFX_TCX_DST_FIFOA7,
3885	AMDGPU_GFX_TCX_DST_FIFOB0,
3886	AMDGPU_GFX_TCX_DST_FIFOB1,
3887	AMDGPU_GFX_TCX_DST_FIFOB2,
3888	AMDGPU_GFX_TCX_DST_FIFOB3,
3889	AMDGPU_GFX_TCX_DST_FIFOB4,
3890	AMDGPU_GFX_TCX_DST_FIFOB5,
3891	AMDGPU_GFX_TCX_DST_FIFOB6,
3892	AMDGPU_GFX_TCX_DST_FIFOB7,
3893	AMDGPU_GFX_TCX_DST_FIFOD0,
3894	AMDGPU_GFX_TCX_DST_FIFOD1,
3895	AMDGPU_GFX_TCX_DST_FIFOD2,
3896	AMDGPU_GFX_TCX_DST_FIFOD3,
3897	AMDGPU_GFX_TCX_DST_FIFOD4,
3898	AMDGPU_GFX_TCX_DST_FIFOD5,
3899	AMDGPU_GFX_TCX_DST_FIFOD6,
3900	AMDGPU_GFX_TCX_DST_FIFOD7,
3901	AMDGPU_GFX_TCX_DST_FIFO_ACKB0,
3902	AMDGPU_GFX_TCX_DST_FIFO_ACKB1,
3903	AMDGPU_GFX_TCX_DST_FIFO_ACKB2,
3904	AMDGPU_GFX_TCX_DST_FIFO_ACKB3,
3905	AMDGPU_GFX_TCX_DST_FIFO_ACKB4,
3906	AMDGPU_GFX_TCX_DST_FIFO_ACKB5,
3907	AMDGPU_GFX_TCX_DST_FIFO_ACKB6,
3908	AMDGPU_GFX_TCX_DST_FIFO_ACKB7,
3909	AMDGPU_GFX_TCX_DST_FIFO_ACKD0,
3910	AMDGPU_GFX_TCX_DST_FIFO_ACKD1,
3911	AMDGPU_GFX_TCX_DST_FIFO_ACKD2,
3912	AMDGPU_GFX_TCX_DST_FIFO_ACKD3,
3913	AMDGPU_GFX_TCX_DST_FIFO_ACKD4,
3914	AMDGPU_GFX_TCX_DST_FIFO_ACKD5,
3915	AMDGPU_GFX_TCX_DST_FIFO_ACKD6,
3916	AMDGPU_GFX_TCX_DST_FIFO_ACKD7,
3917	};
3918
3919	enum amdgpu_gfx_atc_l2_ras_mem_id {
3920	AMDGPU_GFX_ATC_L2_MEM0 = 0,
3921	};
3922
3923	enum amdgpu_gfx_utcl2_ras_mem_id {
3924	AMDGPU_GFX_UTCL2_MEM0 = 0,
3925	};
3926
3927	enum amdgpu_gfx_vml2_ras_mem_id {
3928	AMDGPU_GFX_VML2_MEM0 = 0,
3929	};
3930
3931	enum amdgpu_gfx_vml2_walker_ras_mem_id {
3932	AMDGPU_GFX_VML2_WALKER_MEM0 = 0,
3933	};
3934
3935	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_cp_mem_list[] = {
3936	{AMDGPU_GFX_CP_MEM1, "CP_MEM1"},
3937	{AMDGPU_GFX_CP_MEM2, "CP_MEM2"},
3938	{AMDGPU_GFX_CP_MEM3, "CP_MEM3"},
3939	{AMDGPU_GFX_CP_MEM4, "CP_MEM4"},
3940	{AMDGPU_GFX_CP_MEM5, "CP_MEM5"},
3941	};
3942
3943	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gcea_mem_list[] = {
3944	{AMDGPU_GFX_GCEA_IOWR_CMDMEM, "GCEA_IOWR_CMDMEM"},
3945	{AMDGPU_GFX_GCEA_IORD_CMDMEM, "GCEA_IORD_CMDMEM"},
3946	{AMDGPU_GFX_GCEA_GMIWR_CMDMEM, "GCEA_GMIWR_CMDMEM"},
3947	{AMDGPU_GFX_GCEA_GMIRD_CMDMEM, "GCEA_GMIRD_CMDMEM"},
3948	{AMDGPU_GFX_GCEA_DRAMWR_CMDMEM, "GCEA_DRAMWR_CMDMEM"},
3949	{AMDGPU_GFX_GCEA_DRAMRD_CMDMEM, "GCEA_DRAMRD_CMDMEM"},
3950	{AMDGPU_GFX_GCEA_MAM_DMEM0, "GCEA_MAM_DMEM0"},
3951	{AMDGPU_GFX_GCEA_MAM_DMEM1, "GCEA_MAM_DMEM1"},
3952	{AMDGPU_GFX_GCEA_MAM_DMEM2, "GCEA_MAM_DMEM2"},
3953	{AMDGPU_GFX_GCEA_MAM_DMEM3, "GCEA_MAM_DMEM3"},
3954	{AMDGPU_GFX_GCEA_MAM_AMEM0, "GCEA_MAM_AMEM0"},
3955	{AMDGPU_GFX_GCEA_MAM_AMEM1, "GCEA_MAM_AMEM1"},
3956	{AMDGPU_GFX_GCEA_MAM_AMEM2, "GCEA_MAM_AMEM2"},
3957	{AMDGPU_GFX_GCEA_MAM_AMEM3, "GCEA_MAM_AMEM3"},
3958	{AMDGPU_GFX_GCEA_MAM_AFLUSH_BUFFER, "GCEA_MAM_AFLUSH_BUFFER"},
3959	{AMDGPU_GFX_GCEA_WRET_TAGMEM, "GCEA_WRET_TAGMEM"},
3960	{AMDGPU_GFX_GCEA_RRET_TAGMEM, "GCEA_RRET_TAGMEM"},
3961	{AMDGPU_GFX_GCEA_IOWR_DATAMEM, "GCEA_IOWR_DATAMEM"},
3962	{AMDGPU_GFX_GCEA_GMIWR_DATAMEM, "GCEA_GMIWR_DATAMEM"},
3963	{AMDGPU_GFX_GCEA_DRAM_DATAMEM, "GCEA_DRAM_DATAMEM"},
3964	};
3965
3966	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gc_cane_mem_list[] = {
3967	{AMDGPU_GFX_GC_CANE_MEM0, "GC_CANE_MEM0"},
3968	};
3969
3970	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gcutcl2_mem_list[] = {
3971	{AMDGPU_GFX_GCUTCL2_MEM2P512X95, "GCUTCL2_MEM2P512X95"},
3972	};
3973
3974	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gds_mem_list[] = {
3975	{AMDGPU_GFX_GDS_MEM0, "GDS_MEM"},
3976	};
3977
3978	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_lds_mem_list[] = {
3979	{AMDGPU_GFX_LDS_BANK0, "LDS_BANK0"},
3980	{AMDGPU_GFX_LDS_BANK1, "LDS_BANK1"},
3981	{AMDGPU_GFX_LDS_BANK2, "LDS_BANK2"},
3982	{AMDGPU_GFX_LDS_BANK3, "LDS_BANK3"},
3983	{AMDGPU_GFX_LDS_BANK4, "LDS_BANK4"},
3984	{AMDGPU_GFX_LDS_BANK5, "LDS_BANK5"},
3985	{AMDGPU_GFX_LDS_BANK6, "LDS_BANK6"},
3986	{AMDGPU_GFX_LDS_BANK7, "LDS_BANK7"},
3987	{AMDGPU_GFX_LDS_BANK8, "LDS_BANK8"},
3988	{AMDGPU_GFX_LDS_BANK9, "LDS_BANK9"},
3989	{AMDGPU_GFX_LDS_BANK10, "LDS_BANK10"},
3990	{AMDGPU_GFX_LDS_BANK11, "LDS_BANK11"},
3991	{AMDGPU_GFX_LDS_BANK12, "LDS_BANK12"},
3992	{AMDGPU_GFX_LDS_BANK13, "LDS_BANK13"},
3993	{AMDGPU_GFX_LDS_BANK14, "LDS_BANK14"},
3994	{AMDGPU_GFX_LDS_BANK15, "LDS_BANK15"},
3995	{AMDGPU_GFX_LDS_BANK16, "LDS_BANK16"},
3996	{AMDGPU_GFX_LDS_BANK17, "LDS_BANK17"},
3997	{AMDGPU_GFX_LDS_BANK18, "LDS_BANK18"},
3998	{AMDGPU_GFX_LDS_BANK19, "LDS_BANK19"},
3999	{AMDGPU_GFX_LDS_BANK20, "LDS_BANK20"},
4000	{AMDGPU_GFX_LDS_BANK21, "LDS_BANK21"},
4001	{AMDGPU_GFX_LDS_BANK22, "LDS_BANK22"},
4002	{AMDGPU_GFX_LDS_BANK23, "LDS_BANK23"},
4003	{AMDGPU_GFX_LDS_BANK24, "LDS_BANK24"},
4004	{AMDGPU_GFX_LDS_BANK25, "LDS_BANK25"},
4005	{AMDGPU_GFX_LDS_BANK26, "LDS_BANK26"},
4006	{AMDGPU_GFX_LDS_BANK27, "LDS_BANK27"},
4007	{AMDGPU_GFX_LDS_BANK28, "LDS_BANK28"},
4008	{AMDGPU_GFX_LDS_BANK29, "LDS_BANK29"},
4009	{AMDGPU_GFX_LDS_BANK30, "LDS_BANK30"},
4010	{AMDGPU_GFX_LDS_BANK31, "LDS_BANK31"},
4011	{AMDGPU_GFX_LDS_SP_BUFFER_A, "LDS_SP_BUFFER_A"},
4012	{AMDGPU_GFX_LDS_SP_BUFFER_B, "LDS_SP_BUFFER_B"},
4013	};
4014
4015	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_rlc_mem_list[] = {
4016	{AMDGPU_GFX_RLC_GPMF32, "RLC_GPMF32"},
4017	{AMDGPU_GFX_RLC_RLCVF32, "RLC_RLCVF32"},
4018	{AMDGPU_GFX_RLC_SCRATCH, "RLC_SCRATCH"},
4019	{AMDGPU_GFX_RLC_SRM_ARAM, "RLC_SRM_ARAM"},
4020	{AMDGPU_GFX_RLC_SRM_DRAM, "RLC_SRM_DRAM"},
4021	{AMDGPU_GFX_RLC_TCTAG, "RLC_TCTAG"},
4022	{AMDGPU_GFX_RLC_SPM_SE, "RLC_SPM_SE"},
4023	{AMDGPU_GFX_RLC_SPM_GRBMT, "RLC_SPM_GRBMT"},
4024	};
4025
4026	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_sp_mem_list[] = {
4027	{AMDGPU_GFX_SP_SIMDID0, "SP_SIMDID0"},
4028	};
4029
4030	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_spi_mem_list[] = {
4031	{AMDGPU_GFX_SPI_MEM0, "SPI_MEM0"},
4032	{AMDGPU_GFX_SPI_MEM1, "SPI_MEM1"},
4033	{AMDGPU_GFX_SPI_MEM2, "SPI_MEM2"},
4034	{AMDGPU_GFX_SPI_MEM3, "SPI_MEM3"},
4035	};
4036
4037	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_sqc_mem_list[] = {
4038	{AMDGPU_GFX_SQC_INST_CACHE_A, "SQC_INST_CACHE_A"},
4039	{AMDGPU_GFX_SQC_INST_CACHE_B, "SQC_INST_CACHE_B"},
4040	{AMDGPU_GFX_SQC_INST_CACHE_TAG_A, "SQC_INST_CACHE_TAG_A"},
4041	{AMDGPU_GFX_SQC_INST_CACHE_TAG_B, "SQC_INST_CACHE_TAG_B"},
4042	{AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_A, "SQC_INST_CACHE_MISS_FIFO_A"},
4043	{AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_B, "SQC_INST_CACHE_MISS_FIFO_B"},
4044	{AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_A, "SQC_INST_CACHE_GATCL1_MISS_FIFO_A"},
4045	{AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_B, "SQC_INST_CACHE_GATCL1_MISS_FIFO_B"},
4046	{AMDGPU_GFX_SQC_DATA_CACHE_A, "SQC_DATA_CACHE_A"},
4047	{AMDGPU_GFX_SQC_DATA_CACHE_B, "SQC_DATA_CACHE_B"},
4048	{AMDGPU_GFX_SQC_DATA_CACHE_TAG_A, "SQC_DATA_CACHE_TAG_A"},
4049	{AMDGPU_GFX_SQC_DATA_CACHE_TAG_B, "SQC_DATA_CACHE_TAG_B"},
4050	{AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_A, "SQC_DATA_CACHE_MISS_FIFO_A"},
4051	{AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_B, "SQC_DATA_CACHE_MISS_FIFO_B"},
4052	{AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_A, "SQC_DATA_CACHE_HIT_FIFO_A"},
4053	{AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_B, "SQC_DATA_CACHE_HIT_FIFO_B"},
4054	{AMDGPU_GFX_SQC_DIRTY_BIT_A, "SQC_DIRTY_BIT_A"},
4055	{AMDGPU_GFX_SQC_DIRTY_BIT_B, "SQC_DIRTY_BIT_B"},
4056	{AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU0, "SQC_WRITE_DATA_BUFFER_CU0"},
4057	{AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU1, "SQC_WRITE_DATA_BUFFER_CU1"},
4058	{AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_A, "SQC_UTCL1_MISS_LFIFO_DATA_CACHE_A"},
4059	{AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_B, "SQC_UTCL1_MISS_LFIFO_DATA_CACHE_B"},
4060	{AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_INST_CACHE, "SQC_UTCL1_MISS_LFIFO_INST_CACHE"},
4061	};
4062
4063	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_sq_mem_list[] = {
4064	{AMDGPU_GFX_SQ_SGPR_MEM0, "SQ_SGPR_MEM0"},
4065	{AMDGPU_GFX_SQ_SGPR_MEM1, "SQ_SGPR_MEM1"},
4066	{AMDGPU_GFX_SQ_SGPR_MEM2, "SQ_SGPR_MEM2"},
4067	{AMDGPU_GFX_SQ_SGPR_MEM3, "SQ_SGPR_MEM3"},
4068	};
4069
4070	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_ta_mem_list[] = {
4071	{AMDGPU_GFX_TA_FS_AFIFO_RAM_LO, "TA_FS_AFIFO_RAM_LO"},
4072	{AMDGPU_GFX_TA_FS_AFIFO_RAM_HI, "TA_FS_AFIFO_RAM_HI"},
4073	{AMDGPU_GFX_TA_FS_CFIFO_RAM, "TA_FS_CFIFO_RAM"},
4074	{AMDGPU_GFX_TA_FSX_LFIFO, "TA_FSX_LFIFO"},
4075	{AMDGPU_GFX_TA_FS_DFIFO_RAM, "TA_FS_DFIFO_RAM"},
4076	};
4077
4078	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tcc_mem_list[] = {
4079	{AMDGPU_GFX_TCC_MEM1, "TCC_MEM1"},
4080	};
4081
4082	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tca_mem_list[] = {
4083	{AMDGPU_GFX_TCA_MEM1, "TCA_MEM1"},
4084	};
4085
4086	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tci_mem_list[] = {
4087	{AMDGPU_GFX_TCIW_MEM, "TCIW_MEM"},
4088	};
4089
4090	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tcp_mem_list[] = {
4091	{AMDGPU_GFX_TCP_LFIFO0, "TCP_LFIFO0"},
4092	{AMDGPU_GFX_TCP_SET0BANK0_RAM, "TCP_SET0BANK0_RAM"},
4093	{AMDGPU_GFX_TCP_SET0BANK1_RAM, "TCP_SET0BANK1_RAM"},
4094	{AMDGPU_GFX_TCP_SET0BANK2_RAM, "TCP_SET0BANK2_RAM"},
4095	{AMDGPU_GFX_TCP_SET0BANK3_RAM, "TCP_SET0BANK3_RAM"},
4096	{AMDGPU_GFX_TCP_SET1BANK0_RAM, "TCP_SET1BANK0_RAM"},
4097	{AMDGPU_GFX_TCP_SET1BANK1_RAM, "TCP_SET1BANK1_RAM"},
4098	{AMDGPU_GFX_TCP_SET1BANK2_RAM, "TCP_SET1BANK2_RAM"},
4099	{AMDGPU_GFX_TCP_SET1BANK3_RAM, "TCP_SET1BANK3_RAM"},
4100	{AMDGPU_GFX_TCP_SET2BANK0_RAM, "TCP_SET2BANK0_RAM"},
4101	{AMDGPU_GFX_TCP_SET2BANK1_RAM, "TCP_SET2BANK1_RAM"},
4102	{AMDGPU_GFX_TCP_SET2BANK2_RAM, "TCP_SET2BANK2_RAM"},
4103	{AMDGPU_GFX_TCP_SET2BANK3_RAM, "TCP_SET2BANK3_RAM"},
4104	{AMDGPU_GFX_TCP_SET3BANK0_RAM, "TCP_SET3BANK0_RAM"},
4105	{AMDGPU_GFX_TCP_SET3BANK1_RAM, "TCP_SET3BANK1_RAM"},
4106	{AMDGPU_GFX_TCP_SET3BANK2_RAM, "TCP_SET3BANK2_RAM"},
4107	{AMDGPU_GFX_TCP_SET3BANK3_RAM, "TCP_SET3BANK3_RAM"},
4108	{AMDGPU_GFX_TCP_VM_FIFO, "TCP_VM_FIFO"},
4109	{AMDGPU_GFX_TCP_DB_TAGRAM0, "TCP_DB_TAGRAM0"},
4110	{AMDGPU_GFX_TCP_DB_TAGRAM1, "TCP_DB_TAGRAM1"},
4111	{AMDGPU_GFX_TCP_DB_TAGRAM2, "TCP_DB_TAGRAM2"},
4112	{AMDGPU_GFX_TCP_DB_TAGRAM3, "TCP_DB_TAGRAM3"},
4113	{AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE0, "TCP_UTCL1_LFIFO_PROBE0"},
4114	{AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE1, "TCP_UTCL1_LFIFO_PROBE1"},
4115	{AMDGPU_GFX_TCP_CMD_FIFO, "TCP_CMD_FIFO"},
4116	};
4117
4118	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_td_mem_list[] = {
4119	{AMDGPU_GFX_TD_UTD_CS_FIFO_MEM, "TD_UTD_CS_FIFO_MEM"},
4120	{AMDGPU_GFX_TD_UTD_SS_FIFO_LO_MEM, "TD_UTD_SS_FIFO_LO_MEM"},
4121	{AMDGPU_GFX_TD_UTD_SS_FIFO_HI_MEM, "TD_UTD_SS_FIFO_HI_MEM"},
4122	};
4123
4124	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tcx_mem_list[] = {
4125	{AMDGPU_GFX_TCX_FIFOD0, "TCX_FIFOD0"},
4126	{AMDGPU_GFX_TCX_FIFOD1, "TCX_FIFOD1"},
4127	{AMDGPU_GFX_TCX_FIFOD2, "TCX_FIFOD2"},
4128	{AMDGPU_GFX_TCX_FIFOD3, "TCX_FIFOD3"},
4129	{AMDGPU_GFX_TCX_FIFOD4, "TCX_FIFOD4"},
4130	{AMDGPU_GFX_TCX_FIFOD5, "TCX_FIFOD5"},
4131	{AMDGPU_GFX_TCX_FIFOD6, "TCX_FIFOD6"},
4132	{AMDGPU_GFX_TCX_FIFOD7, "TCX_FIFOD7"},
4133	{AMDGPU_GFX_TCX_FIFOB0, "TCX_FIFOB0"},
4134	{AMDGPU_GFX_TCX_FIFOB1, "TCX_FIFOB1"},
4135	{AMDGPU_GFX_TCX_FIFOB2, "TCX_FIFOB2"},
4136	{AMDGPU_GFX_TCX_FIFOB3, "TCX_FIFOB3"},
4137	{AMDGPU_GFX_TCX_FIFOB4, "TCX_FIFOB4"},
4138	{AMDGPU_GFX_TCX_FIFOB5, "TCX_FIFOB5"},
4139	{AMDGPU_GFX_TCX_FIFOB6, "TCX_FIFOB6"},
4140	{AMDGPU_GFX_TCX_FIFOB7, "TCX_FIFOB7"},
4141	{AMDGPU_GFX_TCX_FIFOA0, "TCX_FIFOA0"},
4142	{AMDGPU_GFX_TCX_FIFOA1, "TCX_FIFOA1"},
4143	{AMDGPU_GFX_TCX_FIFOA2, "TCX_FIFOA2"},
4144	{AMDGPU_GFX_TCX_FIFOA3, "TCX_FIFOA3"},
4145	{AMDGPU_GFX_TCX_FIFOA4, "TCX_FIFOA4"},
4146	{AMDGPU_GFX_TCX_FIFOA5, "TCX_FIFOA5"},
4147	{AMDGPU_GFX_TCX_FIFOA6, "TCX_FIFOA6"},
4148	{AMDGPU_GFX_TCX_FIFOA7, "TCX_FIFOA7"},
4149	{AMDGPU_GFX_TCX_CFIFO0, "TCX_CFIFO0"},
4150	{AMDGPU_GFX_TCX_CFIFO1, "TCX_CFIFO1"},
4151	{AMDGPU_GFX_TCX_CFIFO2, "TCX_CFIFO2"},
4152	{AMDGPU_GFX_TCX_CFIFO3, "TCX_CFIFO3"},
4153	{AMDGPU_GFX_TCX_CFIFO4, "TCX_CFIFO4"},
4154	{AMDGPU_GFX_TCX_CFIFO5, "TCX_CFIFO5"},
4155	{AMDGPU_GFX_TCX_CFIFO6, "TCX_CFIFO6"},
4156	{AMDGPU_GFX_TCX_CFIFO7, "TCX_CFIFO7"},
4157	{AMDGPU_GFX_TCX_FIFO_ACKB0, "TCX_FIFO_ACKB0"},
4158	{AMDGPU_GFX_TCX_FIFO_ACKB1, "TCX_FIFO_ACKB1"},
4159	{AMDGPU_GFX_TCX_FIFO_ACKB2, "TCX_FIFO_ACKB2"},
4160	{AMDGPU_GFX_TCX_FIFO_ACKB3, "TCX_FIFO_ACKB3"},
4161	{AMDGPU_GFX_TCX_FIFO_ACKB4, "TCX_FIFO_ACKB4"},
4162	{AMDGPU_GFX_TCX_FIFO_ACKB5, "TCX_FIFO_ACKB5"},
4163	{AMDGPU_GFX_TCX_FIFO_ACKB6, "TCX_FIFO_ACKB6"},
4164	{AMDGPU_GFX_TCX_FIFO_ACKB7, "TCX_FIFO_ACKB7"},
4165	{AMDGPU_GFX_TCX_FIFO_ACKD0, "TCX_FIFO_ACKD0"},
4166	{AMDGPU_GFX_TCX_FIFO_ACKD1, "TCX_FIFO_ACKD1"},
4167	{AMDGPU_GFX_TCX_FIFO_ACKD2, "TCX_FIFO_ACKD2"},
4168	{AMDGPU_GFX_TCX_FIFO_ACKD3, "TCX_FIFO_ACKD3"},
4169	{AMDGPU_GFX_TCX_FIFO_ACKD4, "TCX_FIFO_ACKD4"},
4170	{AMDGPU_GFX_TCX_FIFO_ACKD5, "TCX_FIFO_ACKD5"},
4171	{AMDGPU_GFX_TCX_FIFO_ACKD6, "TCX_FIFO_ACKD6"},
4172	{AMDGPU_GFX_TCX_FIFO_ACKD7, "TCX_FIFO_ACKD7"},
4173	{AMDGPU_GFX_TCX_DST_FIFOA0, "TCX_DST_FIFOA0"},
4174	{AMDGPU_GFX_TCX_DST_FIFOA1, "TCX_DST_FIFOA1"},
4175	{AMDGPU_GFX_TCX_DST_FIFOA2, "TCX_DST_FIFOA2"},
4176	{AMDGPU_GFX_TCX_DST_FIFOA3, "TCX_DST_FIFOA3"},
4177	{AMDGPU_GFX_TCX_DST_FIFOA4, "TCX_DST_FIFOA4"},
4178	{AMDGPU_GFX_TCX_DST_FIFOA5, "TCX_DST_FIFOA5"},
4179	{AMDGPU_GFX_TCX_DST_FIFOA6, "TCX_DST_FIFOA6"},
4180	{AMDGPU_GFX_TCX_DST_FIFOA7, "TCX_DST_FIFOA7"},
4181	{AMDGPU_GFX_TCX_DST_FIFOB0, "TCX_DST_FIFOB0"},
4182	{AMDGPU_GFX_TCX_DST_FIFOB1, "TCX_DST_FIFOB1"},
4183	{AMDGPU_GFX_TCX_DST_FIFOB2, "TCX_DST_FIFOB2"},
4184	{AMDGPU_GFX_TCX_DST_FIFOB3, "TCX_DST_FIFOB3"},
4185	{AMDGPU_GFX_TCX_DST_FIFOB4, "TCX_DST_FIFOB4"},
4186	{AMDGPU_GFX_TCX_DST_FIFOB5, "TCX_DST_FIFOB5"},
4187	{AMDGPU_GFX_TCX_DST_FIFOB6, "TCX_DST_FIFOB6"},
4188	{AMDGPU_GFX_TCX_DST_FIFOB7, "TCX_DST_FIFOB7"},
4189	{AMDGPU_GFX_TCX_DST_FIFOD0, "TCX_DST_FIFOD0"},
4190	{AMDGPU_GFX_TCX_DST_FIFOD1, "TCX_DST_FIFOD1"},
4191	{AMDGPU_GFX_TCX_DST_FIFOD2, "TCX_DST_FIFOD2"},
4192	{AMDGPU_GFX_TCX_DST_FIFOD3, "TCX_DST_FIFOD3"},
4193	{AMDGPU_GFX_TCX_DST_FIFOD4, "TCX_DST_FIFOD4"},
4194	{AMDGPU_GFX_TCX_DST_FIFOD5, "TCX_DST_FIFOD5"},
4195	{AMDGPU_GFX_TCX_DST_FIFOD6, "TCX_DST_FIFOD6"},
4196	{AMDGPU_GFX_TCX_DST_FIFOD7, "TCX_DST_FIFOD7"},
4197	{AMDGPU_GFX_TCX_DST_FIFO_ACKB0, "TCX_DST_FIFO_ACKB0"},
4198	{AMDGPU_GFX_TCX_DST_FIFO_ACKB1, "TCX_DST_FIFO_ACKB1"},
4199	{AMDGPU_GFX_TCX_DST_FIFO_ACKB2, "TCX_DST_FIFO_ACKB2"},
4200	{AMDGPU_GFX_TCX_DST_FIFO_ACKB3, "TCX_DST_FIFO_ACKB3"},
4201	{AMDGPU_GFX_TCX_DST_FIFO_ACKB4, "TCX_DST_FIFO_ACKB4"},
4202	{AMDGPU_GFX_TCX_DST_FIFO_ACKB5, "TCX_DST_FIFO_ACKB5"},
4203	{AMDGPU_GFX_TCX_DST_FIFO_ACKB6, "TCX_DST_FIFO_ACKB6"},
4204	{AMDGPU_GFX_TCX_DST_FIFO_ACKB7, "TCX_DST_FIFO_ACKB7"},
4205	{AMDGPU_GFX_TCX_DST_FIFO_ACKD0, "TCX_DST_FIFO_ACKD0"},
4206	{AMDGPU_GFX_TCX_DST_FIFO_ACKD1, "TCX_DST_FIFO_ACKD1"},
4207	{AMDGPU_GFX_TCX_DST_FIFO_ACKD2, "TCX_DST_FIFO_ACKD2"},
4208	{AMDGPU_GFX_TCX_DST_FIFO_ACKD3, "TCX_DST_FIFO_ACKD3"},
4209	{AMDGPU_GFX_TCX_DST_FIFO_ACKD4, "TCX_DST_FIFO_ACKD4"},
4210	{AMDGPU_GFX_TCX_DST_FIFO_ACKD5, "TCX_DST_FIFO_ACKD5"},
4211	{AMDGPU_GFX_TCX_DST_FIFO_ACKD6, "TCX_DST_FIFO_ACKD6"},
4212	{AMDGPU_GFX_TCX_DST_FIFO_ACKD7, "TCX_DST_FIFO_ACKD7"},
4213	};
4214
4215	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_atc_l2_mem_list[] = {
4216	{AMDGPU_GFX_ATC_L2_MEM, "ATC_L2_MEM"},
4217	};
4218
4219	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_utcl2_mem_list[] = {
4220	{AMDGPU_GFX_UTCL2_MEM, "UTCL2_MEM"},
4221	};
4222
4223	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_vml2_mem_list[] = {
4224	{AMDGPU_GFX_VML2_MEM, "VML2_MEM"},
4225	};
4226
4227	static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_vml2_walker_mem_list[] = {
4228	{AMDGPU_GFX_VML2_WALKER_MEM, "VML2_WALKER_MEM"},
4229	};
4230
4231	static const struct amdgpu_gfx_ras_mem_id_entry gfx_v9_4_3_ras_mem_list_array[AMDGPU_GFX_MEM_TYPE_NUM] = {
4232	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_cp_mem_list)
4233	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gcea_mem_list)
4234	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gc_cane_mem_list)
4235	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gcutcl2_mem_list)
4236	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gds_mem_list)
4237	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_lds_mem_list)
4238	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_rlc_mem_list)
4239	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_sp_mem_list)
4240	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_spi_mem_list)
4241	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_sqc_mem_list)
4242	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_sq_mem_list)
4243	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_ta_mem_list)
4244	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tcc_mem_list)
4245	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tca_mem_list)
4246	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tci_mem_list)
4247	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tcp_mem_list)
4248	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_td_mem_list)
4249	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tcx_mem_list)
4250	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_atc_l2_mem_list)
4251	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_utcl2_mem_list)
4252	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_vml2_mem_list)
4253	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_vml2_walker_mem_list)
4254	};
4255
4256	static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ce_reg_list[] = {
4257	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regRLC_CE_ERR_STATUS_LOW, regRLC_CE_ERR_STATUS_HIGH),
4258	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "RLC"},
4259	AMDGPU_GFX_RLC_MEM, 1},
4260	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPC_CE_ERR_STATUS_LO, regCPC_CE_ERR_STATUS_HI),
4261	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPC"},
4262	AMDGPU_GFX_CP_MEM, 1},
4263	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPF_CE_ERR_STATUS_LO, regCPF_CE_ERR_STATUS_HI),
4264	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPF"},
4265	AMDGPU_GFX_CP_MEM, 1},
4266	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPG_CE_ERR_STATUS_LO, regCPG_CE_ERR_STATUS_HI),
4267	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPG"},
4268	AMDGPU_GFX_CP_MEM, 1},
4269	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGDS_CE_ERR_STATUS_LO, regGDS_CE_ERR_STATUS_HI),
4270	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GDS"},
4271	AMDGPU_GFX_GDS_MEM, 1},
4272	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGC_CANE_CE_ERR_STATUS_LO, regGC_CANE_CE_ERR_STATUS_HI),
4273	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CANE"},
4274	AMDGPU_GFX_GC_CANE_MEM, 1},
4275	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_CE_ERR_STATUS_LO, regSPI_CE_ERR_STATUS_HI),
4276	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"},
4277	AMDGPU_GFX_SPI_MEM, 1},
4278	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_CE_ERR_STATUS_LO, regSP0_CE_ERR_STATUS_HI),
4279	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"},
4280	AMDGPU_GFX_SP_MEM, 4},
4281	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_CE_ERR_STATUS_LO, regSP1_CE_ERR_STATUS_HI),
4282	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"},
4283	AMDGPU_GFX_SP_MEM, 4},
4284	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_CE_ERR_STATUS_LO, regSQ_CE_ERR_STATUS_HI),
4285	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"},
4286	AMDGPU_GFX_SQ_MEM, 4},
4287	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_CE_EDC_LO, regSQC_CE_EDC_HI),
4288	5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"},
4289	AMDGPU_GFX_SQC_MEM, 4},
4290	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_CE_ERR_STATUS_LO, regTCX_CE_ERR_STATUS_HI),
4291	2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"},
4292	AMDGPU_GFX_TCX_MEM, 1},
4293	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCC_CE_ERR_STATUS_LO, regTCC_CE_ERR_STATUS_HI),
4294	16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCC"},
4295	AMDGPU_GFX_TCC_MEM, 1},
4296	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_CE_EDC_LO, regTA_CE_EDC_HI),
4297	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"},
4298	AMDGPU_GFX_TA_MEM, 4},
4299	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_CE_EDC_LO_REG, regTCI_CE_EDC_HI_REG),
4300	27, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"},
4301	AMDGPU_GFX_TCI_MEM, 1},
4302	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_CE_EDC_LO_REG, regTCP_CE_EDC_HI_REG),
4303	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"},
4304	AMDGPU_GFX_TCP_MEM, 4},
4305	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_CE_EDC_LO, regTD_CE_EDC_HI),
4306	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"},
4307	AMDGPU_GFX_TD_MEM, 4},
4308	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGCEA_CE_ERR_STATUS_LO, regGCEA_CE_ERR_STATUS_HI),
4309	16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GCEA"},
4310	AMDGPU_GFX_GCEA_MEM, 1},
4311	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_CE_ERR_STATUS_LO, regLDS_CE_ERR_STATUS_HI),
4312	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"},
4313	AMDGPU_GFX_LDS_MEM, 4},
4314	};
4315
4316	static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ue_reg_list[] = {
4317	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regRLC_UE_ERR_STATUS_LOW, regRLC_UE_ERR_STATUS_HIGH),
4318	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "RLC"},
4319	AMDGPU_GFX_RLC_MEM, 1},
4320	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPC_UE_ERR_STATUS_LO, regCPC_UE_ERR_STATUS_HI),
4321	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPC"},
4322	AMDGPU_GFX_CP_MEM, 1},
4323	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPF_UE_ERR_STATUS_LO, regCPF_UE_ERR_STATUS_HI),
4324	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPF"},
4325	AMDGPU_GFX_CP_MEM, 1},
4326	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPG_UE_ERR_STATUS_LO, regCPG_UE_ERR_STATUS_HI),
4327	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPG"},
4328	AMDGPU_GFX_CP_MEM, 1},
4329	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGDS_UE_ERR_STATUS_LO, regGDS_UE_ERR_STATUS_HI),
4330	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GDS"},
4331	AMDGPU_GFX_GDS_MEM, 1},
4332	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGC_CANE_UE_ERR_STATUS_LO, regGC_CANE_UE_ERR_STATUS_HI),
4333	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CANE"},
4334	AMDGPU_GFX_GC_CANE_MEM, 1},
4335	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_UE_ERR_STATUS_LO, regSPI_UE_ERR_STATUS_HI),
4336	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"},
4337	AMDGPU_GFX_SPI_MEM, 1},
4338	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_UE_ERR_STATUS_LO, regSP0_UE_ERR_STATUS_HI),
4339	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"},
4340	AMDGPU_GFX_SP_MEM, 4},
4341	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_UE_ERR_STATUS_LO, regSP1_UE_ERR_STATUS_HI),
4342	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"},
4343	AMDGPU_GFX_SP_MEM, 4},
4344	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_UE_ERR_STATUS_LO, regSQ_UE_ERR_STATUS_HI),
4345	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"},
4346	AMDGPU_GFX_SQ_MEM, 4},
4347	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_UE_EDC_LO, regSQC_UE_EDC_HI),
4348	5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"},
4349	AMDGPU_GFX_SQC_MEM, 4},
4350	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_UE_ERR_STATUS_LO, regTCX_UE_ERR_STATUS_HI),
4351	2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"},
4352	AMDGPU_GFX_TCX_MEM, 1},
4353	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCC_UE_ERR_STATUS_LO, regTCC_UE_ERR_STATUS_HI),
4354	16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCC"},
4355	AMDGPU_GFX_TCC_MEM, 1},
4356	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_UE_EDC_LO, regTA_UE_EDC_HI),
4357	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"},
4358	AMDGPU_GFX_TA_MEM, 4},
4359	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_UE_EDC_LO_REG, regTCI_UE_EDC_HI_REG),
4360	27, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"},
4361	AMDGPU_GFX_TCI_MEM, 1},
4362	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_UE_EDC_LO_REG, regTCP_UE_EDC_HI_REG),
4363	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"},
4364	AMDGPU_GFX_TCP_MEM, 4},
4365	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_UE_EDC_LO, regTD_UE_EDC_HI),
4366	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"},
4367	AMDGPU_GFX_TD_MEM, 4},
4368	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCA_UE_ERR_STATUS_LO, regTCA_UE_ERR_STATUS_HI),
4369	2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCA"},
4370	AMDGPU_GFX_TCA_MEM, 1},
4371	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGCEA_UE_ERR_STATUS_LO, regGCEA_UE_ERR_STATUS_HI),
4372	16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GCEA"},
4373	AMDGPU_GFX_GCEA_MEM, 1},
4374	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_UE_ERR_STATUS_LO, regLDS_UE_ERR_STATUS_HI),
4375	10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"},
4376	AMDGPU_GFX_LDS_MEM, 4},
4377	};
4378
4379	static void gfx_v9_4_3_inst_query_ras_err_count(struct amdgpu_device *adev,
4380	void *ras_error_status, int xcc_id)
4381	{
4382	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
4383	unsigned long ce_count = 0, ue_count = 0;
4384	uint32_t i, j, k;
4385
4386	/* NOTE: convert xcc_id to physical XCD ID (XCD0 or XCD1) */
4387	struct amdgpu_smuio_mcm_config_info mcm_info = {
4388	.socket_id = adev->smuio.funcs->get_socket_id(adev),
4389	.die_id = xcc_id & 0x01 ? 1 : 0,
4390	};
4391
4392	mutex_lock(&adev->grbm_idx_mutex);
4393
4394	for (i = 0; i < ARRAY_SIZE(gfx_v9_4_3_ce_reg_list); i++) {
4395	for (j = 0; j < gfx_v9_4_3_ce_reg_list[i].se_num; j++) {
4396	for (k = 0; k < gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst; k++) {
4397	/* no need to select if instance number is 1 */
4398	if (gfx_v9_4_3_ce_reg_list[i].se_num > 1 ||
4399	gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst > 1)
4400	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: j, sh_num: 0, instance: k, xcc_id);
4401
4402	amdgpu_ras_inst_query_ras_error_count(adev,
4403	reg_list: &(gfx_v9_4_3_ce_reg_list[i].reg_entry),
4404	reg_list_size: 1,
4405	mem_list: gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ce_reg_list[i].mem_id_type].mem_id_ent,
4406	mem_list_size: gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ce_reg_list[i].mem_id_type].size,
4407	GET_INST(GC, xcc_id),
4408	err_type: AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE,
4409	err_count: &ce_count);
4410
4411	amdgpu_ras_inst_query_ras_error_count(adev,
4412	reg_list: &(gfx_v9_4_3_ue_reg_list[i].reg_entry),
4413	reg_list_size: 1,
4414	mem_list: gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].mem_id_ent,
4415	mem_list_size: gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].size,
4416	GET_INST(GC, xcc_id),
4417	err_type: AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
4418	err_count: &ue_count);
4419	}
4420	}
4421	}
4422
4423	/* handle extra register entries of UE */
4424	for (; i < ARRAY_SIZE(gfx_v9_4_3_ue_reg_list); i++) {
4425	for (j = 0; j < gfx_v9_4_3_ue_reg_list[i].se_num; j++) {
4426	for (k = 0; k < gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst; k++) {
4427	/* no need to select if instance number is 1 */
4428	if (gfx_v9_4_3_ue_reg_list[i].se_num > 1 ||
4429	gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst > 1)
4430	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: j, sh_num: 0, instance: k, xcc_id);
4431
4432	amdgpu_ras_inst_query_ras_error_count(adev,
4433	reg_list: &(gfx_v9_4_3_ue_reg_list[i].reg_entry),
4434	reg_list_size: 1,
4435	mem_list: gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].mem_id_ent,
4436	mem_list_size: gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].size,
4437	GET_INST(GC, xcc_id),
4438	err_type: AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
4439	err_count: &ue_count);
4440	}
4441	}
4442	}
4443
4444	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: 0xffffffff, sh_num: 0xffffffff, instance: 0xffffffff,
4445	xcc_id);
4446	mutex_unlock(lock: &adev->grbm_idx_mutex);
4447
4448	/* the caller should make sure initialize value of
4449	* err_data->ue_count and err_data->ce_count
4450	*/
4451	amdgpu_ras_error_statistic_ue_count(err_data, mcm_info: &mcm_info, count: ue_count);
4452	amdgpu_ras_error_statistic_ce_count(err_data, mcm_info: &mcm_info, count: ce_count);
4453	}
4454
4455	static void gfx_v9_4_3_inst_reset_ras_err_count(struct amdgpu_device *adev,
4456	void *ras_error_status, int xcc_id)
4457	{
4458	uint32_t i, j, k;
4459
4460	mutex_lock(&adev->grbm_idx_mutex);
4461
4462	for (i = 0; i < ARRAY_SIZE(gfx_v9_4_3_ce_reg_list); i++) {
4463	for (j = 0; j < gfx_v9_4_3_ce_reg_list[i].se_num; j++) {
4464	for (k = 0; k < gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst; k++) {
4465	/* no need to select if instance number is 1 */
4466	if (gfx_v9_4_3_ce_reg_list[i].se_num > 1 ||
4467	gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst > 1)
4468	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: j, sh_num: 0, instance: k, xcc_id);
4469
4470	amdgpu_ras_inst_reset_ras_error_count(adev,
4471	reg_list: &(gfx_v9_4_3_ce_reg_list[i].reg_entry),
4472	reg_list_size: 1,
4473	GET_INST(GC, xcc_id));
4474
4475	amdgpu_ras_inst_reset_ras_error_count(adev,
4476	reg_list: &(gfx_v9_4_3_ue_reg_list[i].reg_entry),
4477	reg_list_size: 1,
4478	GET_INST(GC, xcc_id));
4479	}
4480	}
4481	}
4482
4483	/* handle extra register entries of UE */
4484	for (; i < ARRAY_SIZE(gfx_v9_4_3_ue_reg_list); i++) {
4485	for (j = 0; j < gfx_v9_4_3_ue_reg_list[i].se_num; j++) {
4486	for (k = 0; k < gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst; k++) {
4487	/* no need to select if instance number is 1 */
4488	if (gfx_v9_4_3_ue_reg_list[i].se_num > 1 ||
4489	gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst > 1)
4490	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: j, sh_num: 0, instance: k, xcc_id);
4491
4492	amdgpu_ras_inst_reset_ras_error_count(adev,
4493	reg_list: &(gfx_v9_4_3_ue_reg_list[i].reg_entry),
4494	reg_list_size: 1,
4495	GET_INST(GC, xcc_id));
4496	}
4497	}
4498	}
4499
4500	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: 0xffffffff, sh_num: 0xffffffff, instance: 0xffffffff,
4501	xcc_id);
4502	mutex_unlock(lock: &adev->grbm_idx_mutex);
4503	}
4504
4505	static void gfx_v9_4_3_inst_enable_watchdog_timer(struct amdgpu_device *adev,
4506	void *ras_error_status, int xcc_id)
4507	{
4508	uint32_t i;
4509	uint32_t data;
4510
4511	if (amdgpu_sriov_vf(adev))
4512	return;
4513
4514	data = RREG32_SOC15(GC, GET_INST(GC, 0), regSQ_TIMEOUT_CONFIG);
4515	data = REG_SET_FIELD(data, SQ_TIMEOUT_CONFIG, TIMEOUT_FATAL_DISABLE,
4516	amdgpu_watchdog_timer.timeout_fatal_disable ? 1 : 0);
4517
4518	if (amdgpu_watchdog_timer.timeout_fatal_disable &&
4519	(amdgpu_watchdog_timer.period < 1 ||
4520	amdgpu_watchdog_timer.period > 0x23)) {
4521	dev_warn(adev->dev, "Watchdog period range is 1 to 0x23\n");
4522	amdgpu_watchdog_timer.period = 0x23;
4523	}
4524	data = REG_SET_FIELD(data, SQ_TIMEOUT_CONFIG, PERIOD_SEL,
4525	amdgpu_watchdog_timer.period);
4526
4527	mutex_lock(&adev->grbm_idx_mutex);
4528	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
4529	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: i, sh_num: 0xffffffff, instance: 0xffffffff, xcc_id);
4530	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_TIMEOUT_CONFIG, data);
4531	}
4532	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: 0xffffffff, sh_num: 0xffffffff, instance: 0xffffffff,
4533	xcc_id);
4534	mutex_unlock(lock: &adev->grbm_idx_mutex);
4535	}
4536
4537	static void gfx_v9_4_3_query_ras_error_count(struct amdgpu_device *adev,
4538	void *ras_error_status)
4539	{
4540	amdgpu_gfx_ras_error_func(adev, ras_error_status,
4541	func: gfx_v9_4_3_inst_query_ras_err_count);
4542	}
4543
4544	static void gfx_v9_4_3_reset_ras_error_count(struct amdgpu_device *adev)
4545	{
4546	amdgpu_gfx_ras_error_func(adev, NULL, func: gfx_v9_4_3_inst_reset_ras_err_count);
4547	}
4548
4549	static void gfx_v9_4_3_enable_watchdog_timer(struct amdgpu_device *adev)
4550	{
4551	amdgpu_gfx_ras_error_func(adev, NULL, func: gfx_v9_4_3_inst_enable_watchdog_timer);
4552	}
4553
4554	static void gfx_v9_4_3_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop)
4555	{
4556	/* Header itself is a NOP packet */
4557	if (num_nop == 1) {
4558	amdgpu_ring_write(ring, v: ring->funcs->nop);
4559	return;
4560	}
4561
4562	/* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/
4563	amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe)));
4564
4565	/* Header is at index 0, followed by num_nops - 1 NOP packet's */
4566	amdgpu_ring_insert_nop(ring, count: num_nop - 1);
4567	}
4568
4569	static void gfx_v9_4_3_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
4570	{
4571	struct amdgpu_device *adev = ip_block->adev;
4572	uint32_t i, j, k;
4573	uint32_t xcc_id, xcc_offset, inst_offset;
4574	uint32_t num_xcc, reg, num_inst;
4575	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9_4_3);
4576
4577	if (!adev->gfx.ip_dump_core)
4578	return;
4579
4580	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4581	drm_printf(p, f: "Number of Instances:%d\n", num_xcc);
4582	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
4583	xcc_offset = xcc_id * reg_count;
4584	drm_printf(p, f: "\nInstance id:%d\n", xcc_id);
4585	for (i = 0; i < reg_count; i++)
4586	drm_printf(p, f: "%-50s \t 0x%08x\n",
4587	gc_reg_list_9_4_3[i].reg_name,
4588	adev->gfx.ip_dump_core[xcc_offset + i]);
4589	}
4590
4591	/* print compute queue registers for all instances */
4592	if (!adev->gfx.ip_dump_compute_queues)
4593	return;
4594
4595	num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
4596	adev->gfx.mec.num_queue_per_pipe;
4597
4598	reg_count = ARRAY_SIZE(gc_cp_reg_list_9_4_3);
4599	drm_printf(p, f: "\nnum_xcc: %d num_mec: %d num_pipe: %d num_queue: %d\n",
4600	num_xcc,
4601	adev->gfx.mec.num_mec,
4602	adev->gfx.mec.num_pipe_per_mec,
4603	adev->gfx.mec.num_queue_per_pipe);
4604
4605	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
4606	xcc_offset = xcc_id * reg_count * num_inst;
4607	inst_offset = 0;
4608	for (i = 0; i < adev->gfx.mec.num_mec; i++) {
4609	for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
4610	for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
4611	drm_printf(p,
4612	f: "\nxcc:%d mec:%d, pipe:%d, queue:%d\n",
4613	xcc_id, i, j, k);
4614	for (reg = 0; reg < reg_count; reg++) {
4615	if (i && gc_cp_reg_list_9_4_3[reg].reg_offset ==
4616	regCP_MEC_ME1_HEADER_DUMP)
4617	drm_printf(p,
4618	f: "%-50s \t 0x%08x\n",
4619	"regCP_MEC_ME2_HEADER_DUMP",
4620	adev->gfx.ip_dump_compute_queues
4621	[xcc_offset + inst_offset +
4622	reg]);
4623	else
4624	drm_printf(p,
4625	f: "%-50s \t 0x%08x\n",
4626	gc_cp_reg_list_9_4_3[reg].reg_name,
4627	adev->gfx.ip_dump_compute_queues
4628	[xcc_offset + inst_offset +
4629	reg]);
4630	}
4631	inst_offset += reg_count;
4632	}
4633	}
4634	}
4635	}
4636	}
4637
4638	static void gfx_v9_4_3_ip_dump(struct amdgpu_ip_block *ip_block)
4639	{
4640	struct amdgpu_device *adev = ip_block->adev;
4641	uint32_t i, j, k;
4642	uint32_t num_xcc, reg, num_inst;
4643	uint32_t xcc_id, xcc_offset, inst_offset;
4644	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9_4_3);
4645
4646	if (!adev->gfx.ip_dump_core)
4647	return;
4648
4649	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4650
4651	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
4652	xcc_offset = xcc_id * reg_count;
4653	for (i = 0; i < reg_count; i++)
4654	adev->gfx.ip_dump_core[xcc_offset + i] =
4655	RREG32(SOC15_REG_ENTRY_OFFSET_INST(gc_reg_list_9_4_3[i],
4656	GET_INST(GC, xcc_id)));
4657	}
4658
4659	/* dump compute queue registers for all instances */
4660	if (!adev->gfx.ip_dump_compute_queues)
4661	return;
4662
4663	num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
4664	adev->gfx.mec.num_queue_per_pipe;
4665	reg_count = ARRAY_SIZE(gc_cp_reg_list_9_4_3);
4666	mutex_lock(&adev->srbm_mutex);
4667	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
4668	xcc_offset = xcc_id * reg_count * num_inst;
4669	inst_offset = 0;
4670	for (i = 0; i < adev->gfx.mec.num_mec; i++) {
4671	for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
4672	for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
4673	/* ME0 is for GFX so start from 1 for CP */
4674	soc15_grbm_select(adev, me: 1 + i, pipe: j, queue: k, vmid: 0,
4675	GET_INST(GC, xcc_id));
4676
4677	for (reg = 0; reg < reg_count; reg++) {
4678	if (i && gc_cp_reg_list_9_4_3[reg].reg_offset ==
4679	regCP_MEC_ME1_HEADER_DUMP)
4680	adev->gfx.ip_dump_compute_queues
4681	[xcc_offset +
4682	inst_offset + reg] =
4683	RREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id),
4684	regCP_MEC_ME2_HEADER_DUMP));
4685	else
4686	adev->gfx.ip_dump_compute_queues
4687	[xcc_offset +
4688	inst_offset + reg] =
4689	RREG32(SOC15_REG_ENTRY_OFFSET_INST(
4690	gc_cp_reg_list_9_4_3[reg],
4691	GET_INST(GC, xcc_id)));
4692	}
4693	inst_offset += reg_count;
4694	}
4695	}
4696	}
4697	}
4698	soc15_grbm_select(adev, me: 0, pipe: 0, queue: 0, vmid: 0, xcc_id: 0);
4699	mutex_unlock(lock: &adev->srbm_mutex);
4700	}
4701
4702	static void gfx_v9_4_3_ring_emit_cleaner_shader(struct amdgpu_ring *ring)
4703	{
4704	/* Emit the cleaner shader */
4705	amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0));
4706	amdgpu_ring_write(ring, v: 0); /* RESERVED field, programmed to zero */
4707	}
4708
4709	static const struct amd_ip_funcs gfx_v9_4_3_ip_funcs = {
4710	.name = "gfx_v9_4_3",
4711	.early_init = gfx_v9_4_3_early_init,
4712	.late_init = gfx_v9_4_3_late_init,
4713	.sw_init = gfx_v9_4_3_sw_init,
4714	.sw_fini = gfx_v9_4_3_sw_fini,
4715	.hw_init = gfx_v9_4_3_hw_init,
4716	.hw_fini = gfx_v9_4_3_hw_fini,
4717	.suspend = gfx_v9_4_3_suspend,
4718	.resume = gfx_v9_4_3_resume,
4719	.is_idle = gfx_v9_4_3_is_idle,
4720	.wait_for_idle = gfx_v9_4_3_wait_for_idle,
4721	.soft_reset = gfx_v9_4_3_soft_reset,
4722	.set_clockgating_state = gfx_v9_4_3_set_clockgating_state,
4723	.set_powergating_state = gfx_v9_4_3_set_powergating_state,
4724	.get_clockgating_state = gfx_v9_4_3_get_clockgating_state,
4725	.dump_ip_state = gfx_v9_4_3_ip_dump,
4726	.print_ip_state = gfx_v9_4_3_ip_print,
4727	};
4728
4729	static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_compute = {
4730	.type = AMDGPU_RING_TYPE_COMPUTE,
4731	.align_mask = 0xff,
4732	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
4733	.support_64bit_ptrs = true,
4734	.get_rptr = gfx_v9_4_3_ring_get_rptr_compute,
4735	.get_wptr = gfx_v9_4_3_ring_get_wptr_compute,
4736	.set_wptr = gfx_v9_4_3_ring_set_wptr_compute,
4737	.emit_frame_size =
4738	20 + /* gfx_v9_4_3_ring_emit_gds_switch */
4739	7 + /* gfx_v9_4_3_ring_emit_hdp_flush */
4740	5 + /* hdp invalidate */
4741	7 + /* gfx_v9_4_3_ring_emit_pipeline_sync */
4742	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4743	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4744	2 + /* gfx_v9_4_3_ring_emit_vm_flush */
4745	8 + 8 + 8 + /* gfx_v9_4_3_ring_emit_fence x3 for user fence, vm fence */
4746	7 + /* gfx_v9_4_3_emit_mem_sync */
4747	5 + /* gfx_v9_4_3_emit_wave_limit for updating regSPI_WCL_PIPE_PERCENT_GFX register */
4748	15 + /* for updating 3 regSPI_WCL_PIPE_PERCENT_CS registers */
4749	2, /* gfx_v9_4_3_ring_emit_cleaner_shader */
4750	.emit_ib_size = 7, /* gfx_v9_4_3_ring_emit_ib_compute */
4751	.emit_ib = gfx_v9_4_3_ring_emit_ib_compute,
4752	.emit_fence = gfx_v9_4_3_ring_emit_fence,
4753	.emit_pipeline_sync = gfx_v9_4_3_ring_emit_pipeline_sync,
4754	.emit_vm_flush = gfx_v9_4_3_ring_emit_vm_flush,
4755	.emit_gds_switch = gfx_v9_4_3_ring_emit_gds_switch,
4756	.emit_hdp_flush = gfx_v9_4_3_ring_emit_hdp_flush,
4757	.test_ring = gfx_v9_4_3_ring_test_ring,
4758	.test_ib = gfx_v9_4_3_ring_test_ib,
4759	.insert_nop = gfx_v9_4_3_ring_insert_nop,
4760	.pad_ib = amdgpu_ring_generic_pad_ib,
4761	.emit_wreg = gfx_v9_4_3_ring_emit_wreg,
4762	.emit_reg_wait = gfx_v9_4_3_ring_emit_reg_wait,
4763	.emit_reg_write_reg_wait = gfx_v9_4_3_ring_emit_reg_write_reg_wait,
4764	.soft_recovery = gfx_v9_4_3_ring_soft_recovery,
4765	.emit_mem_sync = gfx_v9_4_3_emit_mem_sync,
4766	.emit_wave_limit = gfx_v9_4_3_emit_wave_limit,
4767	.reset = gfx_v9_4_3_reset_kcq,
4768	.emit_cleaner_shader = gfx_v9_4_3_ring_emit_cleaner_shader,
4769	.begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use,
4770	.end_use = amdgpu_gfx_enforce_isolation_ring_end_use,
4771	};
4772
4773	static const struct amdgpu_ring_funcs gfx_v9_4_3_ring_funcs_kiq = {
4774	.type = AMDGPU_RING_TYPE_KIQ,
4775	.align_mask = 0xff,
4776	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
4777	.support_64bit_ptrs = true,
4778	.get_rptr = gfx_v9_4_3_ring_get_rptr_compute,
4779	.get_wptr = gfx_v9_4_3_ring_get_wptr_compute,
4780	.set_wptr = gfx_v9_4_3_ring_set_wptr_compute,
4781	.emit_frame_size =
4782	20 + /* gfx_v9_4_3_ring_emit_gds_switch */
4783	7 + /* gfx_v9_4_3_ring_emit_hdp_flush */
4784	5 + /* hdp invalidate */
4785	7 + /* gfx_v9_4_3_ring_emit_pipeline_sync */
4786	SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
4787	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
4788	2 + /* gfx_v9_4_3_ring_emit_vm_flush */
4789	8 + 8 + 8, /* gfx_v9_4_3_ring_emit_fence_kiq x3 for user fence, vm fence */
4790	.emit_ib_size = 7, /* gfx_v9_4_3_ring_emit_ib_compute */
4791	.emit_fence = gfx_v9_4_3_ring_emit_fence_kiq,
4792	.test_ring = gfx_v9_4_3_ring_test_ring,
4793	.insert_nop = amdgpu_ring_insert_nop,
4794	.pad_ib = amdgpu_ring_generic_pad_ib,
4795	.emit_rreg = gfx_v9_4_3_ring_emit_rreg,
4796	.emit_wreg = gfx_v9_4_3_ring_emit_wreg,
4797	.emit_reg_wait = gfx_v9_4_3_ring_emit_reg_wait,
4798	.emit_reg_write_reg_wait = gfx_v9_4_3_ring_emit_reg_write_reg_wait,
4799	.emit_hdp_flush = gfx_v9_4_3_ring_emit_hdp_flush,
4800	};
4801
4802	static void gfx_v9_4_3_set_ring_funcs(struct amdgpu_device *adev)
4803	{
4804	int i, j, num_xcc;
4805
4806	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
4807	for (i = 0; i < num_xcc; i++) {
4808	adev->gfx.kiq[i].ring.funcs = &gfx_v9_4_3_ring_funcs_kiq;
4809
4810	for (j = 0; j < adev->gfx.num_compute_rings; j++)
4811	adev->gfx.compute_ring[j + i * adev->gfx.num_compute_rings].funcs
4812	= &gfx_v9_4_3_ring_funcs_compute;
4813	}
4814	}
4815
4816	static const struct amdgpu_irq_src_funcs gfx_v9_4_3_eop_irq_funcs = {
4817	.set = gfx_v9_4_3_set_eop_interrupt_state,
4818	.process = gfx_v9_4_3_eop_irq,
4819	};
4820
4821	static const struct amdgpu_irq_src_funcs gfx_v9_4_3_priv_reg_irq_funcs = {
4822	.set = gfx_v9_4_3_set_priv_reg_fault_state,
4823	.process = gfx_v9_4_3_priv_reg_irq,
4824	};
4825
4826	static const struct amdgpu_irq_src_funcs gfx_v9_4_3_bad_op_irq_funcs = {
4827	.set = gfx_v9_4_3_set_bad_op_fault_state,
4828	.process = gfx_v9_4_3_bad_op_irq,
4829	};
4830
4831	static const struct amdgpu_irq_src_funcs gfx_v9_4_3_priv_inst_irq_funcs = {
4832	.set = gfx_v9_4_3_set_priv_inst_fault_state,
4833	.process = gfx_v9_4_3_priv_inst_irq,
4834	};
4835
4836	static void gfx_v9_4_3_set_irq_funcs(struct amdgpu_device *adev)
4837	{
4838	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
4839	adev->gfx.eop_irq.funcs = &gfx_v9_4_3_eop_irq_funcs;
4840
4841	adev->gfx.priv_reg_irq.num_types = 1;
4842	adev->gfx.priv_reg_irq.funcs = &gfx_v9_4_3_priv_reg_irq_funcs;
4843
4844	adev->gfx.bad_op_irq.num_types = 1;
4845	adev->gfx.bad_op_irq.funcs = &gfx_v9_4_3_bad_op_irq_funcs;
4846
4847	adev->gfx.priv_inst_irq.num_types = 1;
4848	adev->gfx.priv_inst_irq.funcs = &gfx_v9_4_3_priv_inst_irq_funcs;
4849	}
4850
4851	static void gfx_v9_4_3_set_rlc_funcs(struct amdgpu_device *adev)
4852	{
4853	adev->gfx.rlc.funcs = &gfx_v9_4_3_rlc_funcs;
4854	}
4855
4856
4857	static void gfx_v9_4_3_set_gds_init(struct amdgpu_device *adev)
4858	{
4859	/* 9.4.3 variants removed all the GDS internal memory,
4860	* only support GWS opcode in kernel, like barrier
4861	* semaphore.etc */
4862
4863	/* init asic gds info */
4864	adev->gds.gds_size = 0;
4865	adev->gds.gds_compute_max_wave_id = 0;
4866	adev->gds.gws_size = 64;
4867	adev->gds.oa_size = 16;
4868	}
4869
4870	static void gfx_v9_4_3_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
4871	u32 bitmap, int xcc_id)
4872	{
4873	u32 data;
4874
4875	if (!bitmap)
4876	return;
4877
4878	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
4879	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
4880
4881	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG, data);
4882	}
4883
4884	static u32 gfx_v9_4_3_get_cu_active_bitmap(struct amdgpu_device *adev, int xcc_id)
4885	{
4886	u32 data, mask;
4887
4888	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SHADER_ARRAY_CONFIG);
4889	data |= RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG);
4890
4891	data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
4892	data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
4893
4894	mask = amdgpu_gfx_create_bitmask(bit_width: adev->gfx.config.max_cu_per_sh);
4895
4896	return (~data) & mask;
4897	}
4898
4899	static int gfx_v9_4_3_get_cu_info(struct amdgpu_device *adev,
4900	struct amdgpu_cu_info *cu_info)
4901	{
4902	int i, j, k, prev_counter, counter, xcc_id, active_cu_number = 0;
4903	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0, tmp;
4904	unsigned disable_masks[4 * 4];
4905	bool is_symmetric_cus;
4906
4907	if (!adev || !cu_info)
4908	return -EINVAL;
4909
4910	/*
4911	* 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
4912	*/
4913	if (adev->gfx.config.max_shader_engines *
4914	adev->gfx.config.max_sh_per_se > 16)
4915	return -EINVAL;
4916
4917	amdgpu_gfx_parse_disable_cu(mask: disable_masks,
4918	max_se: adev->gfx.config.max_shader_engines,
4919	max_sh: adev->gfx.config.max_sh_per_se);
4920
4921	mutex_lock(&adev->grbm_idx_mutex);
4922	for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
4923	is_symmetric_cus = true;
4924	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
4925	for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
4926	mask = 1;
4927	ao_bitmap = 0;
4928	counter = 0;
4929	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: i, sh_num: j, instance: 0xffffffff, xcc_id);
4930	gfx_v9_4_3_set_user_cu_inactive_bitmap(
4931	adev,
4932	bitmap: disable_masks[i * adev->gfx.config.max_sh_per_se + j],
4933	xcc_id);
4934	bitmap = gfx_v9_4_3_get_cu_active_bitmap(adev, xcc_id);
4935
4936	cu_info->bitmap[xcc_id][i][j] = bitmap;
4937
4938	for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
4939	if (bitmap & mask) {
4940	if (counter < adev->gfx.config.max_cu_per_sh)
4941	ao_bitmap |= mask;
4942	counter++;
4943	}
4944	mask <<= 1;
4945	}
4946	active_cu_number += counter;
4947	if (i < 2 && j < 2)
4948	ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
4949	cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
4950	}
4951	if (i && is_symmetric_cus && prev_counter != counter)
4952	is_symmetric_cus = false;
4953	prev_counter = counter;
4954	}
4955	if (is_symmetric_cus) {
4956	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_DEBUG);
4957	tmp = REG_SET_FIELD(tmp, CP_CPC_DEBUG, CPC_HARVESTING_RELAUNCH_DISABLE, 1);
4958	tmp = REG_SET_FIELD(tmp, CP_CPC_DEBUG, CPC_HARVESTING_DISPATCH_DISABLE, 1);
4959	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_DEBUG, tmp);
4960	}
4961	gfx_v9_4_3_xcc_select_se_sh(adev, se_num: 0xffffffff, sh_num: 0xffffffff, instance: 0xffffffff,
4962	xcc_id);
4963	}
4964	mutex_unlock(lock: &adev->grbm_idx_mutex);
4965
4966	cu_info->number = active_cu_number;
4967	cu_info->ao_cu_mask = ao_cu_mask;
4968	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
4969
4970	return 0;
4971	}
4972
4973	const struct amdgpu_ip_block_version gfx_v9_4_3_ip_block = {
4974	.type = AMD_IP_BLOCK_TYPE_GFX,
4975	.major = 9,
4976	.minor = 4,
4977	.rev = 3,
4978	.funcs = &gfx_v9_4_3_ip_funcs,
4979	};
4980
4981	static int gfx_v9_4_3_xcp_resume(void *handle, uint32_t inst_mask)
4982	{
4983	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4984	uint32_t tmp_mask;
4985	int i, r;
4986
4987	/* TODO : Initialize golden regs */
4988	/* gfx_v9_4_3_init_golden_registers(adev); */
4989
4990	tmp_mask = inst_mask;
4991	for_each_inst(i, tmp_mask)
4992	gfx_v9_4_3_xcc_constants_init(adev, xcc_id: i);
4993
4994	if (!amdgpu_sriov_vf(adev)) {
4995	tmp_mask = inst_mask;
4996	for_each_inst(i, tmp_mask) {
4997	r = gfx_v9_4_3_xcc_rlc_resume(adev, xcc_id: i);
4998	if (r)
4999	return r;
5000	}
5001	}
5002
5003	tmp_mask = inst_mask;
5004	for_each_inst(i, tmp_mask) {
5005	r = gfx_v9_4_3_xcc_cp_resume(adev, xcc_id: i);
5006	if (r)
5007	return r;
5008	}
5009
5010	return 0;
5011	}
5012
5013	static int gfx_v9_4_3_xcp_suspend(void *handle, uint32_t inst_mask)
5014	{
5015	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5016	int i;
5017
5018	for_each_inst(i, inst_mask)
5019	gfx_v9_4_3_xcc_fini(adev, xcc_id: i);
5020
5021	return 0;
5022	}
5023
5024	struct amdgpu_xcp_ip_funcs gfx_v9_4_3_xcp_funcs = {
5025	.suspend = &gfx_v9_4_3_xcp_suspend,
5026	.resume = &gfx_v9_4_3_xcp_resume
5027	};
5028
5029	struct amdgpu_ras_block_hw_ops gfx_v9_4_3_ras_ops = {
5030	.query_ras_error_count = &gfx_v9_4_3_query_ras_error_count,
5031	.reset_ras_error_count = &gfx_v9_4_3_reset_ras_error_count,
5032	};
5033
5034	static int gfx_v9_4_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
5035	{
5036	int r;
5037
5038	r = amdgpu_ras_block_late_init(adev, ras_block);
5039	if (r)
5040	return r;
5041
5042	r = amdgpu_ras_bind_aca(adev, blk: AMDGPU_RAS_BLOCK__GFX,
5043	aca_info: &gfx_v9_4_3_aca_info,
5044	NULL);
5045	if (r)
5046	goto late_fini;
5047
5048	return 0;
5049
5050	late_fini:
5051	amdgpu_ras_block_late_fini(adev, ras_block);
5052
5053	return r;
5054	}
5055
5056	struct amdgpu_gfx_ras gfx_v9_4_3_ras = {
5057	.ras_block = {
5058	.hw_ops = &gfx_v9_4_3_ras_ops,
5059	.ras_late_init = &gfx_v9_4_3_ras_late_init,
5060	},
5061	.enable_watchdog_timer = &gfx_v9_4_3_enable_watchdog_timer,
5062	};
5063