1	/*
2	* Copyright 2021 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/firmware.h>
25	#include "amdgpu.h"
26	#include "amdgpu_vcn.h"
27	#include "amdgpu_pm.h"
28	#include "amdgpu_cs.h"
29	#include "soc15.h"
30	#include "soc15d.h"
31	#include "soc15_hw_ip.h"
32	#include "vcn_v2_0.h"
33	#include "mmsch_v4_0.h"
34	#include "vcn_v4_0.h"
35
36	#include "vcn/vcn_4_0_0_offset.h"
37	#include "vcn/vcn_4_0_0_sh_mask.h"
38	#include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
39
40	#include <drm/drm_drv.h>
41
42	#define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL
43	#define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX
44	#define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA
45	#define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX
46
47	#define VCN_VID_SOC_ADDRESS_2_0 0x1fb00
48	#define VCN1_VID_SOC_ADDRESS_3_0 0x48300
49	#define VCN1_AON_SOC_ADDRESS_3_0 0x48000
50
51	#define VCN_HARVEST_MMSCH 0
52
53	#define RDECODE_MSG_CREATE 0x00000000
54	#define RDECODE_MESSAGE_CREATE 0x00000001
55
56	static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0[] = {
57	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS),
58	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS),
59	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID),
60	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2),
61	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0),
62	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1),
63	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD),
64	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI),
65	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO),
66	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2),
67	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2),
68	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3),
69	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3),
70	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4),
71	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4),
72	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR),
73	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR),
74	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2),
75	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2),
76	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3),
77	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3),
78	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4),
79	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4),
80	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE),
81	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2),
82	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3),
83	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4),
84	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG),
85	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS),
86	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL),
87	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA),
88	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK),
89	SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE)
90	};
91
92	static int amdgpu_ih_clientid_vcns[] = {
93	SOC15_IH_CLIENTID_VCN,
94	SOC15_IH_CLIENTID_VCN1
95	};
96
97	static int vcn_v4_0_start_sriov(struct amdgpu_device *adev);
98	static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev);
99	static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev);
100	static int vcn_v4_0_set_pg_state(struct amdgpu_vcn_inst *vinst,
101	enum amd_powergating_state state);
102	static int vcn_v4_0_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
103	struct dpg_pause_state *new_state);
104	static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring);
105	static void vcn_v4_0_set_ras_funcs(struct amdgpu_device *adev);
106
107	/**
108	* vcn_v4_0_early_init - set function pointers and load microcode
109	*
110	* @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
111	*
112	* Set ring and irq function pointers
113	* Load microcode from filesystem
114	*/
115	static int vcn_v4_0_early_init(struct amdgpu_ip_block *ip_block)
116	{
117	struct amdgpu_device *adev = ip_block->adev;
118	int i, r;
119
120	if (amdgpu_sriov_vf(adev)) {
121	adev->vcn.harvest_config = VCN_HARVEST_MMSCH;
122	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
123	if (amdgpu_vcn_is_disabled_vcn(adev, type: VCN_ENCODE_RING, vcn_instance: i)) {
124	adev->vcn.harvest_config |= 1 << i;
125	dev_info(adev->dev, "VCN%d is disabled by hypervisor\n", i);
126	}
127	}
128	}
129
130	for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
131	/* re-use enc ring as unified ring */
132	adev->vcn.inst[i].num_enc_rings = 1;
133
134	vcn_v4_0_set_unified_ring_funcs(adev);
135	vcn_v4_0_set_irq_funcs(adev);
136	vcn_v4_0_set_ras_funcs(adev);
137
138	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
139	adev->vcn.inst[i].set_pg_state = vcn_v4_0_set_pg_state;
140
141	r = amdgpu_vcn_early_init(adev, i);
142	if (r)
143	return r;
144	}
145
146	return 0;
147	}
148
149	static int vcn_v4_0_fw_shared_init(struct amdgpu_device *adev, int inst_idx)
150	{
151	struct amdgpu_vcn4_fw_shared *fw_shared;
152
153	fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
154	fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
155	fw_shared->sq.is_enabled = 1;
156
157	fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG);
158	fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ?
159	AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU;
160
161	if (amdgpu_ip_version(adev, ip: VCN_HWIP, inst: 0) ==
162	IP_VERSION(4, 0, 2)) {
163	fw_shared->present_flag_0 |= AMDGPU_FW_SHARED_FLAG_0_DRM_KEY_INJECT;
164	fw_shared->drm_key_wa.method =
165	AMDGPU_DRM_KEY_INJECT_WORKAROUND_VCNFW_ASD_HANDSHAKING;
166	}
167
168	if (amdgpu_vcnfw_log)
169	amdgpu_vcn_fwlog_init(vcn: &adev->vcn.inst[inst_idx]);
170
171	return 0;
172	}
173
174	/**
175	* vcn_v4_0_sw_init - sw init for VCN block
176	*
177	* @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
178	*
179	* Load firmware and sw initialization
180	*/
181	static int vcn_v4_0_sw_init(struct amdgpu_ip_block *ip_block)
182	{
183	struct amdgpu_ring *ring;
184	struct amdgpu_device *adev = ip_block->adev;
185	int i, r;
186
187	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
188	if (adev->vcn.harvest_config & (1 << i))
189	continue;
190
191	r = amdgpu_vcn_sw_init(adev, i);
192	if (r)
193	return r;
194
195	amdgpu_vcn_setup_ucode(adev, i);
196
197	r = amdgpu_vcn_resume(adev, i);
198	if (r)
199	return r;
200
201	/* Init instance 0 sched_score to 1, so it's scheduled after other instances */
202	if (i == 0)
203	atomic_set(v: &adev->vcn.inst[i].sched_score, i: 1);
204	else
205	atomic_set(v: &adev->vcn.inst[i].sched_score, i: 0);
206
207	/* VCN UNIFIED TRAP */
208	r = amdgpu_irq_add_id(adev, client_id: amdgpu_ih_clientid_vcns[i],
209	VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, source: &adev->vcn.inst[i].irq);
210	if (r)
211	return r;
212
213	/* VCN POISON TRAP */
214	r = amdgpu_irq_add_id(adev, client_id: amdgpu_ih_clientid_vcns[i],
215	VCN_4_0__SRCID_UVD_POISON, source: &adev->vcn.inst[i].ras_poison_irq);
216	if (r)
217	return r;
218
219	ring = &adev->vcn.inst[i].ring_enc[0];
220	ring->use_doorbell = true;
221	if (amdgpu_sriov_vf(adev))
222	ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + i *
223	(adev->vcn.inst[i].num_enc_rings + 1) + 1;
224	else
225	ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
226	ring->vm_hub = AMDGPU_MMHUB0(0);
227	sprintf(buf: ring->name, fmt: "vcn_unified_%d", i);
228
229	r = amdgpu_ring_init(adev, ring, max_dw: 512, irq_src: &adev->vcn.inst[i].irq, irq_type: 0,
230	hw_prio: AMDGPU_RING_PRIO_0, sched_score: &adev->vcn.inst[i].sched_score);
231	if (r)
232	return r;
233
234	vcn_v4_0_fw_shared_init(adev, inst_idx: i);
235
236	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
237	adev->vcn.inst[i].pause_dpg_mode = vcn_v4_0_pause_dpg_mode;
238	}
239
240	adev->vcn.supported_reset =
241	amdgpu_get_soft_full_reset_mask(ring: &adev->vcn.inst[0].ring_enc[0]);
242	if (!amdgpu_sriov_vf(adev))
243	adev->vcn.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
244
245	if (amdgpu_sriov_vf(adev)) {
246	r = amdgpu_virt_alloc_mm_table(adev);
247	if (r)
248	return r;
249	}
250
251
252	r = amdgpu_vcn_ras_sw_init(adev);
253	if (r)
254	return r;
255
256	r = amdgpu_vcn_reg_dump_init(adev, reg: vcn_reg_list_4_0, ARRAY_SIZE(vcn_reg_list_4_0));
257	if (r)
258	return r;
259
260	r = amdgpu_vcn_sysfs_reset_mask_init(adev);
261	if (r)
262	return r;
263
264	return 0;
265	}
266
267	/**
268	* vcn_v4_0_sw_fini - sw fini for VCN block
269	*
270	* @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
271	*
272	* VCN suspend and free up sw allocation
273	*/
274	static int vcn_v4_0_sw_fini(struct amdgpu_ip_block *ip_block)
275	{
276	struct amdgpu_device *adev = ip_block->adev;
277	int i, r, idx;
278
279	if (drm_dev_enter(dev: adev_to_drm(adev), idx: &idx)) {
280	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
281	struct amdgpu_vcn4_fw_shared *fw_shared;
282
283	if (adev->vcn.harvest_config & (1 << i))
284	continue;
285
286	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
287	fw_shared->present_flag_0 = 0;
288	fw_shared->sq.is_enabled = 0;
289	}
290
291	drm_dev_exit(idx);
292	}
293
294	if (amdgpu_sriov_vf(adev))
295	amdgpu_virt_free_mm_table(adev);
296
297	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
298	r = amdgpu_vcn_suspend(adev, i);
299	if (r)
300	return r;
301	}
302
303	amdgpu_vcn_sysfs_reset_mask_fini(adev);
304
305	for (i = 0; i < adev->vcn.num_vcn_inst; i++)
306	amdgpu_vcn_sw_fini(adev, i);
307
308	return 0;
309	}
310
311	/**
312	* vcn_v4_0_hw_init - start and test VCN block
313	*
314	* @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
315	*
316	* Initialize the hardware, boot up the VCPU and do some testing
317	*/
318	static int vcn_v4_0_hw_init(struct amdgpu_ip_block *ip_block)
319	{
320	struct amdgpu_device *adev = ip_block->adev;
321	struct amdgpu_ring *ring;
322	int i, r;
323
324	if (amdgpu_sriov_vf(adev)) {
325	r = vcn_v4_0_start_sriov(adev);
326	if (r)
327	return r;
328
329	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
330	if (adev->vcn.harvest_config & (1 << i))
331	continue;
332
333	ring = &adev->vcn.inst[i].ring_enc[0];
334	ring->wptr = 0;
335	ring->wptr_old = 0;
336	vcn_v4_0_unified_ring_set_wptr(ring);
337	ring->sched.ready = true;
338	}
339	} else {
340	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
341	if (adev->vcn.harvest_config & (1 << i))
342	continue;
343
344	ring = &adev->vcn.inst[i].ring_enc[0];
345
346	adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
347	((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
348
349	r = amdgpu_ring_test_helper(ring);
350	if (r)
351	return r;
352	}
353	}
354
355	return 0;
356	}
357
358	/**
359	* vcn_v4_0_hw_fini - stop the hardware block
360	*
361	* @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
362	*
363	* Stop the VCN block, mark ring as not ready any more
364	*/
365	static int vcn_v4_0_hw_fini(struct amdgpu_ip_block *ip_block)
366	{
367	struct amdgpu_device *adev = ip_block->adev;
368	int i;
369
370	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
371	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
372
373	if (adev->vcn.harvest_config & (1 << i))
374	continue;
375
376	cancel_delayed_work_sync(dwork: &vinst->idle_work);
377
378	if (!amdgpu_sriov_vf(adev)) {
379	if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
380	(vinst->cur_state != AMD_PG_STATE_GATE &&
381	RREG32_SOC15(VCN, i, regUVD_STATUS))) {
382	vinst->set_pg_state(vinst, AMD_PG_STATE_GATE);
383	}
384	}
385	if (amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__VCN))
386	amdgpu_irq_put(adev, src: &vinst->ras_poison_irq, type: 0);
387	}
388
389	return 0;
390	}
391
392	/**
393	* vcn_v4_0_suspend - suspend VCN block
394	*
395	* @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
396	*
397	* HW fini and suspend VCN block
398	*/
399	static int vcn_v4_0_suspend(struct amdgpu_ip_block *ip_block)
400	{
401	struct amdgpu_device *adev = ip_block->adev;
402	int r, i;
403
404	r = vcn_v4_0_hw_fini(ip_block);
405	if (r)
406	return r;
407
408	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
409	r = amdgpu_vcn_suspend(adev: ip_block->adev, i);
410	if (r)
411	return r;
412	}
413
414	return 0;
415	}
416
417	/**
418	* vcn_v4_0_resume - resume VCN block
419	*
420	* @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
421	*
422	* Resume firmware and hw init VCN block
423	*/
424	static int vcn_v4_0_resume(struct amdgpu_ip_block *ip_block)
425	{
426	struct amdgpu_device *adev = ip_block->adev;
427	int r, i;
428
429	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
430	r = amdgpu_vcn_resume(adev: ip_block->adev, i);
431	if (r)
432	return r;
433	}
434
435	r = vcn_v4_0_hw_init(ip_block);
436
437	return r;
438	}
439
440	/**
441	* vcn_v4_0_mc_resume - memory controller programming
442	*
443	* @vinst: VCN instance
444	*
445	* Let the VCN memory controller know it's offsets
446	*/
447	static void vcn_v4_0_mc_resume(struct amdgpu_vcn_inst *vinst)
448	{
449	struct amdgpu_device *adev = vinst->adev;
450	int inst = vinst->inst;
451	uint32_t offset, size;
452	const struct common_firmware_header *hdr;
453
454	hdr = (const struct common_firmware_header *)adev->vcn.inst[inst].fw->data;
455	size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
456
457	/* cache window 0: fw */
458	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
459	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
460	(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
461	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
462	(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
463	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
464	offset = 0;
465	} else {
466	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
467	lower_32_bits(adev->vcn.inst[inst].gpu_addr));
468	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
469	upper_32_bits(adev->vcn.inst[inst].gpu_addr));
470	offset = size;
471	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
472	}
473	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
474
475	/* cache window 1: stack */
476	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
477	lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
478	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
479	upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
480	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
481	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
482
483	/* cache window 2: context */
484	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
485	lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
486	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
487	upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
488	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
489	WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
490
491	/* non-cache window */
492	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
493	lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
494	WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
495	upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
496	WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
497	WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
498	AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
499	}
500
501	/**
502	* vcn_v4_0_mc_resume_dpg_mode - memory controller programming for dpg mode
503	*
504	* @vinst: VCN instance
505	* @indirect: indirectly write sram
506	*
507	* Let the VCN memory controller know it's offsets with dpg mode
508	*/
509	static void vcn_v4_0_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst,
510	bool indirect)
511	{
512	struct amdgpu_device *adev = vinst->adev;
513	int inst_idx = vinst->inst;
514	uint32_t offset, size;
515	const struct common_firmware_header *hdr;
516	hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data;
517	size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
518
519	/* cache window 0: fw */
520	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
521	if (!indirect) {
522	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
523	VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
524	(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
525	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
526	VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
527	(adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
528	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
529	VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
530	} else {
531	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
532	VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
533	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
534	VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
535	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
536	VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
537	}
538	offset = 0;
539	} else {
540	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
541	VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
542	lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
543	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
544	VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
545	upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
546	offset = size;
547	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
548	VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
549	AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
550	}
551
552	if (!indirect)
553	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
554	VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
555	else
556	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
557	VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
558
559	/* cache window 1: stack */
560	if (!indirect) {
561	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
562	VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
563	lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
564	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
565	VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
566	upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
567	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
568	VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
569	} else {
570	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
571	VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
572	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
573	VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
574	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
575	VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
576	}
577	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
578	VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
579
580	/* cache window 2: context */
581	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
582	VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
583	lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
584	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
585	VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
586	upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
587	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
588	VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
589	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
590	VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
591
592	/* non-cache window */
593	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
594	VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
595	lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
596	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
597	VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
598	upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
599	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
600	VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
601	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
602	VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
603	AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
604
605	/* VCN global tiling registers */
606	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
607	VCN, inst_idx, regUVD_GFX10_ADDR_CONFIG),
608	adev->gfx.config.gb_addr_config, 0, indirect);
609	}
610
611	/**
612	* vcn_v4_0_disable_static_power_gating - disable VCN static power gating
613	*
614	* @vinst: VCN instance
615	*
616	* Disable static power gating for VCN block
617	*/
618	static void vcn_v4_0_disable_static_power_gating(struct amdgpu_vcn_inst *vinst)
619	{
620	struct amdgpu_device *adev = vinst->adev;
621	int inst = vinst->inst;
622	uint32_t data = 0;
623
624	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
625	data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
626	| 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
627	| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
628	| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
629	| 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
630	| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
631	| 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
632	| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
633	| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
634	| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
635	| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
636	| 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
637	| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
638	| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
639
640	WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
641	SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS,
642	UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
643	} else {
644	uint32_t value;
645
646	value = (inst) ? 0x2200800 : 0;
647	data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
648	| 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
649	| 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
650	| 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
651	| 1 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
652	| 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
653	| 1 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
654	| 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
655	| 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
656	| 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
657	| 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
658	| 1 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
659	| 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
660	| 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
661
662	WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
663	SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, value, 0x3F3FFFFF);
664	}
665
666	data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
667	data &= ~0x103;
668	if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
669	data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
670	UVD_POWER_STATUS__UVD_PG_EN_MASK;
671
672	WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
673
674	return;
675	}
676
677	/**
678	* vcn_v4_0_enable_static_power_gating - enable VCN static power gating
679	*
680	* @vinst: VCN instance
681	*
682	* Enable static power gating for VCN block
683	*/
684	static void vcn_v4_0_enable_static_power_gating(struct amdgpu_vcn_inst *vinst)
685	{
686	struct amdgpu_device *adev = vinst->adev;
687	int inst = vinst->inst;
688	uint32_t data;
689
690	if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
691	/* Before power off, this indicator has to be turned on */
692	data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
693	data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
694	data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
695	WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
696
697	data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
698	| 2 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
699	| 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
700	| 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
701	| 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
702	| 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
703	| 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
704	| 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
705	| 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
706	| 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
707	| 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
708	| 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
709	| 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
710	| 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
711	WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
712
713	data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
714	| 2 << UVD_PGFSM_STATUS__UVDS_PWR_STATUS__SHIFT
715	| 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
716	| 2 << UVD_PGFSM_STATUS__UVDTC_PWR_STATUS__SHIFT
717	| 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
718	| 2 << UVD_PGFSM_STATUS__UVDTA_PWR_STATUS__SHIFT
719	| 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
720	| 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
721	| 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
722	| 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
723	| 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
724	| 2 << UVD_PGFSM_STATUS__UVDTB_PWR_STATUS__SHIFT
725	| 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
726	| 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
727	SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
728	}
729
730	return;
731	}
732
733	/**
734	* vcn_v4_0_disable_clock_gating - disable VCN clock gating
735	*
736	* @vinst: VCN instance
737	*
738	* Disable clock gating for VCN block
739	*/
740	static void vcn_v4_0_disable_clock_gating(struct amdgpu_vcn_inst *vinst)
741	{
742	struct amdgpu_device *adev = vinst->adev;
743	int inst = vinst->inst;
744	uint32_t data;
745
746	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
747	return;
748
749	/* VCN disable CGC */
750	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
751	data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
752	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
753	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
754	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
755
756	data = RREG32_SOC15(VCN, inst, regUVD_CGC_GATE);
757	data &= ~(UVD_CGC_GATE__SYS_MASK
758	| UVD_CGC_GATE__UDEC_MASK
759	| UVD_CGC_GATE__MPEG2_MASK
760	| UVD_CGC_GATE__REGS_MASK
761	| UVD_CGC_GATE__RBC_MASK
762	| UVD_CGC_GATE__LMI_MC_MASK
763	| UVD_CGC_GATE__LMI_UMC_MASK
764	| UVD_CGC_GATE__IDCT_MASK
765	| UVD_CGC_GATE__MPRD_MASK
766	| UVD_CGC_GATE__MPC_MASK
767	| UVD_CGC_GATE__LBSI_MASK
768	| UVD_CGC_GATE__LRBBM_MASK
769	| UVD_CGC_GATE__UDEC_RE_MASK
770	| UVD_CGC_GATE__UDEC_CM_MASK
771	| UVD_CGC_GATE__UDEC_IT_MASK
772	| UVD_CGC_GATE__UDEC_DB_MASK
773	| UVD_CGC_GATE__UDEC_MP_MASK
774	| UVD_CGC_GATE__WCB_MASK
775	| UVD_CGC_GATE__VCPU_MASK
776	| UVD_CGC_GATE__MMSCH_MASK);
777
778	WREG32_SOC15(VCN, inst, regUVD_CGC_GATE, data);
779	SOC15_WAIT_ON_RREG(VCN, inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF);
780
781	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
782	data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
783	| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
784	| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
785	| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
786	| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
787	| UVD_CGC_CTRL__SYS_MODE_MASK
788	| UVD_CGC_CTRL__UDEC_MODE_MASK
789	| UVD_CGC_CTRL__MPEG2_MODE_MASK
790	| UVD_CGC_CTRL__REGS_MODE_MASK
791	| UVD_CGC_CTRL__RBC_MODE_MASK
792	| UVD_CGC_CTRL__LMI_MC_MODE_MASK
793	| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
794	| UVD_CGC_CTRL__IDCT_MODE_MASK
795	| UVD_CGC_CTRL__MPRD_MODE_MASK
796	| UVD_CGC_CTRL__MPC_MODE_MASK
797	| UVD_CGC_CTRL__LBSI_MODE_MASK
798	| UVD_CGC_CTRL__LRBBM_MODE_MASK
799	| UVD_CGC_CTRL__WCB_MODE_MASK
800	| UVD_CGC_CTRL__VCPU_MODE_MASK
801	| UVD_CGC_CTRL__MMSCH_MODE_MASK);
802	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
803
804	data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE);
805	data |= (UVD_SUVD_CGC_GATE__SRE_MASK
806	| UVD_SUVD_CGC_GATE__SIT_MASK
807	| UVD_SUVD_CGC_GATE__SMP_MASK
808	| UVD_SUVD_CGC_GATE__SCM_MASK
809	| UVD_SUVD_CGC_GATE__SDB_MASK
810	| UVD_SUVD_CGC_GATE__SRE_H264_MASK
811	| UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
812	| UVD_SUVD_CGC_GATE__SIT_H264_MASK
813	| UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
814	| UVD_SUVD_CGC_GATE__SCM_H264_MASK
815	| UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
816	| UVD_SUVD_CGC_GATE__SDB_H264_MASK
817	| UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
818	| UVD_SUVD_CGC_GATE__SCLR_MASK
819	| UVD_SUVD_CGC_GATE__UVD_SC_MASK
820	| UVD_SUVD_CGC_GATE__ENT_MASK
821	| UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
822	| UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
823	| UVD_SUVD_CGC_GATE__SITE_MASK
824	| UVD_SUVD_CGC_GATE__SRE_VP9_MASK
825	| UVD_SUVD_CGC_GATE__SCM_VP9_MASK
826	| UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
827	| UVD_SUVD_CGC_GATE__SDB_VP9_MASK
828	| UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
829	WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE, data);
830
831	data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
832	data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
833	| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
834	| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
835	| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
836	| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
837	| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
838	| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
839	| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
840	| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
841	| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
842	WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
843	}
844
845	/**
846	* vcn_v4_0_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
847	*
848	* @vinst: VCN instance
849	* @sram_sel: sram select
850	* @indirect: indirectly write sram
851	*
852	* Disable clock gating for VCN block with dpg mode
853	*/
854	static void vcn_v4_0_disable_clock_gating_dpg_mode(struct amdgpu_vcn_inst *vinst,
855	uint8_t sram_sel,
856	uint8_t indirect)
857	{
858	struct amdgpu_device *adev = vinst->adev;
859	int inst_idx = vinst->inst;
860	uint32_t reg_data = 0;
861
862	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
863	return;
864
865	/* enable sw clock gating control */
866	reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
867	reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
868	reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
869	reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
870	UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
871	UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
872	UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
873	UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
874	UVD_CGC_CTRL__SYS_MODE_MASK |
875	UVD_CGC_CTRL__UDEC_MODE_MASK |
876	UVD_CGC_CTRL__MPEG2_MODE_MASK |
877	UVD_CGC_CTRL__REGS_MODE_MASK |
878	UVD_CGC_CTRL__RBC_MODE_MASK |
879	UVD_CGC_CTRL__LMI_MC_MODE_MASK |
880	UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
881	UVD_CGC_CTRL__IDCT_MODE_MASK |
882	UVD_CGC_CTRL__MPRD_MODE_MASK |
883	UVD_CGC_CTRL__MPC_MODE_MASK |
884	UVD_CGC_CTRL__LBSI_MODE_MASK |
885	UVD_CGC_CTRL__LRBBM_MODE_MASK |
886	UVD_CGC_CTRL__WCB_MODE_MASK |
887	UVD_CGC_CTRL__VCPU_MODE_MASK);
888	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
889	VCN, inst_idx, regUVD_CGC_CTRL), reg_data, sram_sel, indirect);
890
891	/* turn off clock gating */
892	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
893	VCN, inst_idx, regUVD_CGC_GATE), 0, sram_sel, indirect);
894
895	/* turn on SUVD clock gating */
896	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
897	VCN, inst_idx, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
898
899	/* turn on sw mode in UVD_SUVD_CGC_CTRL */
900	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
901	VCN, inst_idx, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
902	}
903
904	/**
905	* vcn_v4_0_enable_clock_gating - enable VCN clock gating
906	*
907	* @vinst: VCN instance
908	*
909	* Enable clock gating for VCN block
910	*/
911	static void vcn_v4_0_enable_clock_gating(struct amdgpu_vcn_inst *vinst)
912	{
913	struct amdgpu_device *adev = vinst->adev;
914	int inst = vinst->inst;
915	uint32_t data;
916
917	if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
918	return;
919
920	/* enable VCN CGC */
921	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
922	data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
923	data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
924	data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
925	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
926
927	data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
928	data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
929	| UVD_CGC_CTRL__UDEC_CM_MODE_MASK
930	| UVD_CGC_CTRL__UDEC_IT_MODE_MASK
931	| UVD_CGC_CTRL__UDEC_DB_MODE_MASK
932	| UVD_CGC_CTRL__UDEC_MP_MODE_MASK
933	| UVD_CGC_CTRL__SYS_MODE_MASK
934	| UVD_CGC_CTRL__UDEC_MODE_MASK
935	| UVD_CGC_CTRL__MPEG2_MODE_MASK
936	| UVD_CGC_CTRL__REGS_MODE_MASK
937	| UVD_CGC_CTRL__RBC_MODE_MASK
938	| UVD_CGC_CTRL__LMI_MC_MODE_MASK
939	| UVD_CGC_CTRL__LMI_UMC_MODE_MASK
940	| UVD_CGC_CTRL__IDCT_MODE_MASK
941	| UVD_CGC_CTRL__MPRD_MODE_MASK
942	| UVD_CGC_CTRL__MPC_MODE_MASK
943	| UVD_CGC_CTRL__LBSI_MODE_MASK
944	| UVD_CGC_CTRL__LRBBM_MODE_MASK
945	| UVD_CGC_CTRL__WCB_MODE_MASK
946	| UVD_CGC_CTRL__VCPU_MODE_MASK
947	| UVD_CGC_CTRL__MMSCH_MODE_MASK);
948	WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
949
950	data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
951	data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
952	| UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
953	| UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
954	| UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
955	| UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
956	| UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
957	| UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
958	| UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
959	| UVD_SUVD_CGC_CTRL__IME_MODE_MASK
960	| UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
961	WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
962	}
963
964	static void vcn_v4_0_enable_ras(struct amdgpu_vcn_inst *vinst,
965	bool indirect)
966	{
967	struct amdgpu_device *adev = vinst->adev;
968	int inst_idx = vinst->inst;
969	uint32_t tmp;
970
971	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__VCN))
972	return;
973
974	tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK |
975	VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK |
976	VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK |
977	VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK;
978	WREG32_SOC15_DPG_MODE(inst_idx,
979	SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL),
980	tmp, 0, indirect);
981
982	tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK;
983	WREG32_SOC15_DPG_MODE(inst_idx,
984	SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN),
985	tmp, 0, indirect);
986	}
987
988	/**
989	* vcn_v4_0_start_dpg_mode - VCN start with dpg mode
990	*
991	* @vinst: VCN instance
992	* @indirect: indirectly write sram
993	*
994	* Start VCN block with dpg mode
995	*/
996	static int vcn_v4_0_start_dpg_mode(struct amdgpu_vcn_inst *vinst, bool indirect)
997	{
998	struct amdgpu_device *adev = vinst->adev;
999	int inst_idx = vinst->inst;
1000	struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1001	struct amdgpu_ring *ring;
1002	uint32_t tmp;
1003	int ret;
1004
1005	/* disable register anti-hang mechanism */
1006	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
1007	~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1008	/* enable dynamic power gating mode */
1009	tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
1010	tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
1011	tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
1012	WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
1013
1014	if (indirect)
1015	adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
1016
1017	/* enable clock gating */
1018	vcn_v4_0_disable_clock_gating_dpg_mode(vinst, sram_sel: 0, indirect);
1019
1020	/* enable VCPU clock */
1021	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1022	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
1023	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1024	VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
1025
1026	/* disable master interupt */
1027	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1028	VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
1029
1030	/* setup regUVD_LMI_CTRL */
1031	tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1032	UVD_LMI_CTRL__REQ_MODE_MASK |
1033	UVD_LMI_CTRL__CRC_RESET_MASK |
1034	UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1035	UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1036	UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
1037	(8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
1038	0x00100000L);
1039	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1040	VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
1041
1042	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1043	VCN, inst_idx, regUVD_MPC_CNTL),
1044	0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
1045
1046	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1047	VCN, inst_idx, regUVD_MPC_SET_MUXA0),
1048	((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1049	(0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1050	(0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1051	(0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
1052
1053	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1054	VCN, inst_idx, regUVD_MPC_SET_MUXB0),
1055	((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1056	(0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1057	(0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1058	(0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
1059
1060	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1061	VCN, inst_idx, regUVD_MPC_SET_MUX),
1062	((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1063	(0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1064	(0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1065
1066	vcn_v4_0_mc_resume_dpg_mode(vinst, indirect);
1067
1068	tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1069	tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1070	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1071	VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
1072
1073	/* enable LMI MC and UMC channels */
1074	tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
1075	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1076	VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
1077
1078	vcn_v4_0_enable_ras(vinst, indirect);
1079
1080	/* enable master interrupt */
1081	WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1082	VCN, inst_idx, regUVD_MASTINT_EN),
1083	UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1084
1085
1086	if (indirect) {
1087	ret = amdgpu_vcn_psp_update_sram(adev, inst_idx, ucode_id: 0);
1088	if (ret) {
1089	dev_err(adev->dev, "vcn sram load failed %d\n", ret);
1090	return ret;
1091	}
1092	}
1093
1094	ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1095
1096	WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
1097	WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1098	WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
1099
1100	tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
1101	tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1102	WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
1103	fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1104	WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);
1105	WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);
1106
1107	tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
1108	WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
1109	ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1110
1111	tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
1112	tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1113	WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
1114	fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1115
1116	WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
1117	ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1118	VCN_RB1_DB_CTRL__EN_MASK);
1119
1120	/* Keeping one read-back to ensure all register writes are done,
1121	* otherwise it may introduce race conditions.
1122	*/
1123	RREG32_SOC15(VCN, inst_idx, regUVD_STATUS);
1124
1125	return 0;
1126	}
1127
1128
1129	/**
1130	* vcn_v4_0_start - VCN start
1131	*
1132	* @vinst: VCN instance
1133	*
1134	* Start VCN block
1135	*/
1136	static int vcn_v4_0_start(struct amdgpu_vcn_inst *vinst)
1137	{
1138	struct amdgpu_device *adev = vinst->adev;
1139	int i = vinst->inst;
1140	struct amdgpu_vcn4_fw_shared *fw_shared;
1141	struct amdgpu_ring *ring;
1142	uint32_t tmp;
1143	int j, k, r;
1144
1145	if (adev->vcn.harvest_config & (1 << i))
1146	return 0;
1147
1148	if (adev->pm.dpm_enabled)
1149	amdgpu_dpm_enable_vcn(adev, enable: true, inst: i);
1150
1151	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1152
1153	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1154	return vcn_v4_0_start_dpg_mode(vinst, indirect: adev->vcn.inst[i].indirect_sram);
1155
1156	/* disable VCN power gating */
1157	vcn_v4_0_disable_static_power_gating(vinst);
1158
1159	/* set VCN status busy */
1160	tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1161	WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
1162
1163	/*SW clock gating */
1164	vcn_v4_0_disable_clock_gating(vinst);
1165
1166	/* enable VCPU clock */
1167	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1168	UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1169
1170	/* disable master interrupt */
1171	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
1172	~UVD_MASTINT_EN__VCPU_EN_MASK);
1173
1174	/* enable LMI MC and UMC channels */
1175	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
1176	~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1177
1178	tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1179	tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1180	tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1181	WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1182
1183	/* setup regUVD_LMI_CTRL */
1184	tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
1185	WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
1186	UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1187	UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1188	UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1189	UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1190
1191	/* setup regUVD_MPC_CNTL */
1192	tmp = RREG32_SOC15(VCN, i, regUVD_MPC_CNTL);
1193	tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1194	tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1195	WREG32_SOC15(VCN, i, regUVD_MPC_CNTL, tmp);
1196
1197	/* setup UVD_MPC_SET_MUXA0 */
1198	WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXA0,
1199	((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1200	(0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1201	(0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1202	(0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1203
1204	/* setup UVD_MPC_SET_MUXB0 */
1205	WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXB0,
1206	((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1207	(0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1208	(0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1209	(0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1210
1211	/* setup UVD_MPC_SET_MUX */
1212	WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUX,
1213	((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1214	(0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1215	(0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1216
1217	vcn_v4_0_mc_resume(vinst);
1218
1219	/* VCN global tiling registers */
1220	WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
1221	adev->gfx.config.gb_addr_config);
1222
1223	/* unblock VCPU register access */
1224	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
1225	~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1226
1227	/* release VCPU reset to boot */
1228	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1229	~UVD_VCPU_CNTL__BLK_RST_MASK);
1230
1231	for (j = 0; j < 10; ++j) {
1232	uint32_t status;
1233
1234	for (k = 0; k < 100; ++k) {
1235	status = RREG32_SOC15(VCN, i, regUVD_STATUS);
1236	if (status & 2)
1237	break;
1238	mdelay(10);
1239	if (amdgpu_emu_mode == 1)
1240	msleep(msecs: 1);
1241	}
1242
1243	if (amdgpu_emu_mode == 1) {
1244	r = -1;
1245	if (status & 2) {
1246	r = 0;
1247	break;
1248	}
1249	} else {
1250	r = 0;
1251	if (status & 2)
1252	break;
1253
1254	dev_err(adev->dev, "VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);
1255	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1256	UVD_VCPU_CNTL__BLK_RST_MASK,
1257	~UVD_VCPU_CNTL__BLK_RST_MASK);
1258	mdelay(10);
1259	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1260	~UVD_VCPU_CNTL__BLK_RST_MASK);
1261
1262	mdelay(10);
1263	r = -1;
1264	}
1265	}
1266
1267	if (r) {
1268	dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
1269	return r;
1270	}
1271
1272	/* enable master interrupt */
1273	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
1274	UVD_MASTINT_EN__VCPU_EN_MASK,
1275	~UVD_MASTINT_EN__VCPU_EN_MASK);
1276
1277	/* clear the busy bit of VCN_STATUS */
1278	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
1279	~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1280
1281	ring = &adev->vcn.inst[i].ring_enc[0];
1282	WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
1283	ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1284	VCN_RB1_DB_CTRL__EN_MASK);
1285
1286	WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
1287	WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1288	WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
1289
1290	tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1291	tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1292	WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1293	fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1294	WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);
1295	WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);
1296
1297	tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
1298	WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
1299	ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
1300
1301	tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1302	tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1303	WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1304	fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1305
1306	/* Keeping one read-back to ensure all register writes are done,
1307	* otherwise it may introduce race conditions.
1308	*/
1309	RREG32_SOC15(VCN, i, regUVD_STATUS);
1310
1311	return 0;
1312	}
1313
1314	static int vcn_v4_0_init_ring_metadata(struct amdgpu_device *adev, uint32_t vcn_inst, struct amdgpu_ring *ring_enc)
1315	{
1316	struct amdgpu_vcn_rb_metadata *rb_metadata = NULL;
1317	uint8_t *rb_ptr = (uint8_t *)ring_enc->ring;
1318
1319	rb_ptr += ring_enc->ring_size;
1320	rb_metadata = (struct amdgpu_vcn_rb_metadata *)rb_ptr;
1321
1322	memset(rb_metadata, 0, sizeof(struct amdgpu_vcn_rb_metadata));
1323	rb_metadata->size = sizeof(struct amdgpu_vcn_rb_metadata);
1324	rb_metadata->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
1325	rb_metadata->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_DECOUPLE_FLAG);
1326	rb_metadata->version = 1;
1327	rb_metadata->ring_id = vcn_inst & 0xFF;
1328
1329	return 0;
1330	}
1331
1332	static int vcn_v4_0_start_sriov(struct amdgpu_device *adev)
1333	{
1334	int i;
1335	struct amdgpu_ring *ring_enc;
1336	uint64_t cache_addr;
1337	uint64_t rb_enc_addr;
1338	uint64_t ctx_addr;
1339	uint32_t param, resp, expected;
1340	uint32_t offset, cache_size;
1341	uint32_t tmp, timeout;
1342
1343	struct amdgpu_mm_table *table = &adev->virt.mm_table;
1344	uint32_t *table_loc;
1345	uint32_t table_size;
1346	uint32_t size, size_dw;
1347	uint32_t init_status;
1348	uint32_t enabled_vcn;
1349
1350	struct mmsch_v4_0_cmd_direct_write
1351	direct_wt = { {0} };
1352	struct mmsch_v4_0_cmd_direct_read_modify_write
1353	direct_rd_mod_wt = { {0} };
1354	struct mmsch_v4_0_cmd_end end = { {0} };
1355	struct mmsch_v4_0_init_header header;
1356
1357	struct amdgpu_vcn4_fw_shared *fw_shared;
1358	struct amdgpu_fw_shared_rb_setup *rb_setup;
1359
1360	direct_wt.cmd_header.command_type =
1361	MMSCH_COMMAND__DIRECT_REG_WRITE;
1362	direct_rd_mod_wt.cmd_header.command_type =
1363	MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1364	end.cmd_header.command_type =
1365	MMSCH_COMMAND__END;
1366
1367	header.version = MMSCH_VERSION;
1368	header.total_size = sizeof(struct mmsch_v4_0_init_header) >> 2;
1369	for (i = 0; i < MMSCH_V4_0_VCN_INSTANCES; i++) {
1370	header.inst[i].init_status = 0;
1371	header.inst[i].table_offset = 0;
1372	header.inst[i].table_size = 0;
1373	}
1374
1375	table_loc = (uint32_t *)table->cpu_addr;
1376	table_loc += header.total_size;
1377	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1378	if (adev->vcn.harvest_config & (1 << i))
1379	continue;
1380
1381	// Must re/init fw_shared at beginning
1382	vcn_v4_0_fw_shared_init(adev, inst_idx: i);
1383
1384	table_size = 0;
1385
1386	MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1387	regUVD_STATUS),
1388	~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1389
1390	cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4);
1391
1392	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1393	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1394	regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1395	adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1396	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1397	regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1398	adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1399	offset = 0;
1400	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1401	regUVD_VCPU_CACHE_OFFSET0),
1402	0);
1403	} else {
1404	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1405	regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1406	lower_32_bits(adev->vcn.inst[i].gpu_addr));
1407	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1408	regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1409	upper_32_bits(adev->vcn.inst[i].gpu_addr));
1410	offset = cache_size;
1411	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1412	regUVD_VCPU_CACHE_OFFSET0),
1413	AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1414	}
1415
1416	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1417	regUVD_VCPU_CACHE_SIZE0),
1418	cache_size);
1419
1420	cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1421	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1422	regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1423	lower_32_bits(cache_addr));
1424	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1425	regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1426	upper_32_bits(cache_addr));
1427	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1428	regUVD_VCPU_CACHE_OFFSET1),
1429	0);
1430	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1431	regUVD_VCPU_CACHE_SIZE1),
1432	AMDGPU_VCN_STACK_SIZE);
1433
1434	cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1435	AMDGPU_VCN_STACK_SIZE;
1436	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1437	regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1438	lower_32_bits(cache_addr));
1439	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1440	regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1441	upper_32_bits(cache_addr));
1442	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1443	regUVD_VCPU_CACHE_OFFSET2),
1444	0);
1445	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1446	regUVD_VCPU_CACHE_SIZE2),
1447	AMDGPU_VCN_CONTEXT_SIZE);
1448
1449	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1450	rb_setup = &fw_shared->rb_setup;
1451
1452	ring_enc = &adev->vcn.inst[i].ring_enc[0];
1453	ring_enc->wptr = 0;
1454	rb_enc_addr = ring_enc->gpu_addr;
1455
1456	rb_setup->is_rb_enabled_flags |= RB_ENABLED;
1457	fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
1458
1459	if (amdgpu_sriov_is_vcn_rb_decouple(adev)) {
1460	vcn_v4_0_init_ring_metadata(adev, vcn_inst: i, ring_enc);
1461
1462	memset((void *)&rb_setup->rb_info, 0, sizeof(struct amdgpu_vcn_rb_setup_info) * MAX_NUM_VCN_RB_SETUP);
1463	if (!(adev->vcn.harvest_config & (1 << 0))) {
1464	rb_setup->rb_info[0].rb_addr_lo = lower_32_bits(adev->vcn.inst[0].ring_enc[0].gpu_addr);
1465	rb_setup->rb_info[0].rb_addr_hi = upper_32_bits(adev->vcn.inst[0].ring_enc[0].gpu_addr);
1466	rb_setup->rb_info[0].rb_size = adev->vcn.inst[0].ring_enc[0].ring_size / 4;
1467	}
1468	if (!(adev->vcn.harvest_config & (1 << 1))) {
1469	rb_setup->rb_info[2].rb_addr_lo = lower_32_bits(adev->vcn.inst[1].ring_enc[0].gpu_addr);
1470	rb_setup->rb_info[2].rb_addr_hi = upper_32_bits(adev->vcn.inst[1].ring_enc[0].gpu_addr);
1471	rb_setup->rb_info[2].rb_size = adev->vcn.inst[1].ring_enc[0].ring_size / 4;
1472	}
1473	fw_shared->decouple.is_enabled = 1;
1474	fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_DECOUPLE_FLAG);
1475	} else {
1476	rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr);
1477	rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr);
1478	rb_setup->rb_size = ring_enc->ring_size / 4;
1479	}
1480
1481	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1482	regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
1483	lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1484	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1485	regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
1486	upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1487	MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1488	regUVD_VCPU_NONCACHE_SIZE0),
1489	AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
1490
1491	/* add end packet */
1492	MMSCH_V4_0_INSERT_END();
1493
1494	/* refine header */
1495	header.inst[i].init_status = 0;
1496	header.inst[i].table_offset = header.total_size;
1497	header.inst[i].table_size = table_size;
1498	header.total_size += table_size;
1499	}
1500
1501	/* Update init table header in memory */
1502	size = sizeof(struct mmsch_v4_0_init_header);
1503	table_loc = (uint32_t *)table->cpu_addr;
1504	memcpy((void *)table_loc, &header, size);
1505
1506	/* message MMSCH (in VCN[0]) to initialize this client
1507	* 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1508	* of memory descriptor location
1509	*/
1510	ctx_addr = table->gpu_addr;
1511	WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1512	WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1513
1514	/* 2, update vmid of descriptor */
1515	tmp = RREG32_SOC15(VCN, 0, regMMSCH_VF_VMID);
1516	tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1517	/* use domain0 for MM scheduler */
1518	tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1519	WREG32_SOC15(VCN, 0, regMMSCH_VF_VMID, tmp);
1520
1521	/* 3, notify mmsch about the size of this descriptor */
1522	size = header.total_size;
1523	WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE, size);
1524
1525	/* 4, set resp to zero */
1526	WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP, 0);
1527
1528	/* 5, kick off the initialization and wait until
1529	* MMSCH_VF_MAILBOX_RESP becomes non-zero
1530	*/
1531	param = 0x00000001;
1532	WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_HOST, param);
1533	tmp = 0;
1534	timeout = 1000;
1535	resp = 0;
1536	expected = MMSCH_VF_MAILBOX_RESP__OK;
1537	while (resp != expected) {
1538	resp = RREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP);
1539	if (resp != 0)
1540	break;
1541
1542	udelay(usec: 10);
1543	tmp = tmp + 10;
1544	if (tmp >= timeout) {
1545	DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1546	" waiting for regMMSCH_VF_MAILBOX_RESP "\
1547	"(expected=0x%08x, readback=0x%08x)\n",
1548	tmp, expected, resp);
1549	return -EBUSY;
1550	}
1551	}
1552	enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, type: VCN_DECODE_RING, vcn_instance: 0) ? 1 : 0;
1553	init_status = ((struct mmsch_v4_0_init_header *)(table_loc))->inst[enabled_vcn].init_status;
1554	if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE
1555	&& init_status != MMSCH_VF_ENGINE_STATUS__PASS)
1556	DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\
1557	"status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status);
1558
1559	return 0;
1560	}
1561
1562	/**
1563	* vcn_v4_0_stop_dpg_mode - VCN stop with dpg mode
1564	*
1565	* @vinst: VCN instance
1566	*
1567	* Stop VCN block with dpg mode
1568	*/
1569	static void vcn_v4_0_stop_dpg_mode(struct amdgpu_vcn_inst *vinst)
1570	{
1571	struct amdgpu_device *adev = vinst->adev;
1572	int inst_idx = vinst->inst;
1573	struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1574	uint32_t tmp;
1575
1576	vcn_v4_0_pause_dpg_mode(vinst, new_state: &state);
1577	/* Wait for power status to be 1 */
1578	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1579	UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1580
1581	/* wait for read ptr to be equal to write ptr */
1582	tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1583	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1584
1585	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1586	UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1587
1588	/* disable dynamic power gating mode */
1589	WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
1590	~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1591
1592	/* Keeping one read-back to ensure all register writes are done,
1593	* otherwise it may introduce race conditions.
1594	*/
1595	RREG32_SOC15(VCN, inst_idx, regUVD_STATUS);
1596	}
1597
1598	/**
1599	* vcn_v4_0_stop - VCN stop
1600	*
1601	* @vinst: VCN instance
1602	*
1603	* Stop VCN block
1604	*/
1605	static int vcn_v4_0_stop(struct amdgpu_vcn_inst *vinst)
1606	{
1607	struct amdgpu_device *adev = vinst->adev;
1608	int i = vinst->inst;
1609	struct amdgpu_vcn4_fw_shared *fw_shared;
1610	uint32_t tmp;
1611	int r = 0;
1612
1613	if (adev->vcn.harvest_config & (1 << i))
1614	return 0;
1615
1616	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1617	fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
1618
1619	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1620	vcn_v4_0_stop_dpg_mode(vinst);
1621	goto done;
1622	}
1623
1624	/* wait for vcn idle */
1625	r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1626	if (r)
1627	goto done;
1628
1629	tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1630	UVD_LMI_STATUS__READ_CLEAN_MASK |
1631	UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1632	UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1633	r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1634	if (r)
1635	goto done;
1636
1637	/* disable LMI UMC channel */
1638	tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
1639	tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1640	WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
1641	tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1642	UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1643	r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1644	if (r)
1645	goto done;
1646
1647	/* block VCPU register access */
1648	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
1649	UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1650	~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1651
1652	/* reset VCPU */
1653	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1654	UVD_VCPU_CNTL__BLK_RST_MASK,
1655	~UVD_VCPU_CNTL__BLK_RST_MASK);
1656
1657	/* disable VCPU clock */
1658	WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1659	~(UVD_VCPU_CNTL__CLK_EN_MASK));
1660
1661	/* apply soft reset */
1662	tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1663	tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1664	WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1665	tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1666	tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1667	WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1668
1669	/* clear status */
1670	WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
1671
1672	/* apply HW clock gating */
1673	vcn_v4_0_enable_clock_gating(vinst);
1674
1675	/* enable VCN power gating */
1676	vcn_v4_0_enable_static_power_gating(vinst);
1677
1678	/* Keeping one read-back to ensure all register writes are done,
1679	* otherwise it may introduce race conditions.
1680	*/
1681	RREG32_SOC15(VCN, i, regUVD_STATUS);
1682
1683	done:
1684	if (adev->pm.dpm_enabled)
1685	amdgpu_dpm_enable_vcn(adev, enable: false, inst: i);
1686
1687	return 0;
1688	}
1689
1690	/**
1691	* vcn_v4_0_pause_dpg_mode - VCN pause with dpg mode
1692	*
1693	* @vinst: VCN instance
1694	* @new_state: pause state
1695	*
1696	* Pause dpg mode for VCN block
1697	*/
1698	static int vcn_v4_0_pause_dpg_mode(struct amdgpu_vcn_inst *vinst,
1699	struct dpg_pause_state *new_state)
1700	{
1701	struct amdgpu_device *adev = vinst->adev;
1702	int inst_idx = vinst->inst;
1703	uint32_t reg_data = 0;
1704	int ret_code;
1705
1706	/* pause/unpause if state is changed */
1707	if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1708	DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
1709	adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1710	reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
1711	(~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1712
1713	if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1714	ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
1715	UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1716
1717	if (!ret_code) {
1718	/* pause DPG */
1719	reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1720	WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1721
1722	/* wait for ACK */
1723	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1724	UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1725	UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1726
1727	SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS,
1728	UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1729	}
1730	} else {
1731	/* unpause dpg, no need to wait */
1732	reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1733	WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1734	}
1735	adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1736	}
1737
1738	return 0;
1739	}
1740
1741	/**
1742	* vcn_v4_0_unified_ring_get_rptr - get unified read pointer
1743	*
1744	* @ring: amdgpu_ring pointer
1745	*
1746	* Returns the current hardware unified read pointer
1747	*/
1748	static uint64_t vcn_v4_0_unified_ring_get_rptr(struct amdgpu_ring *ring)
1749	{
1750	struct amdgpu_device *adev = ring->adev;
1751
1752	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1753	DRM_ERROR("wrong ring id is identified in %s", __func__);
1754
1755	return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
1756	}
1757
1758	/**
1759	* vcn_v4_0_unified_ring_get_wptr - get unified write pointer
1760	*
1761	* @ring: amdgpu_ring pointer
1762	*
1763	* Returns the current hardware unified write pointer
1764	*/
1765	static uint64_t vcn_v4_0_unified_ring_get_wptr(struct amdgpu_ring *ring)
1766	{
1767	struct amdgpu_device *adev = ring->adev;
1768
1769	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1770	DRM_ERROR("wrong ring id is identified in %s", __func__);
1771
1772	if (ring->use_doorbell)
1773	return *ring->wptr_cpu_addr;
1774	else
1775	return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
1776	}
1777
1778	/**
1779	* vcn_v4_0_unified_ring_set_wptr - set enc write pointer
1780	*
1781	* @ring: amdgpu_ring pointer
1782	*
1783	* Commits the enc write pointer to the hardware
1784	*/
1785	static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring)
1786	{
1787	struct amdgpu_device *adev = ring->adev;
1788
1789	if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1790	DRM_ERROR("wrong ring id is identified in %s", __func__);
1791
1792	if (ring->use_doorbell) {
1793	*ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1794	WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1795	} else {
1796	WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
1797	}
1798	}
1799
1800	static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p,
1801	struct amdgpu_job *job)
1802	{
1803	struct drm_gpu_scheduler **scheds;
1804	struct dma_fence *fence;
1805
1806	/* if VCN0 is harvested, we can't support AV1 */
1807	if (p->adev->vcn.harvest_config & AMDGPU_VCN_HARVEST_VCN0)
1808	return -EINVAL;
1809
1810	/* wait for all jobs to finish before switching to instance 0 */
1811	fence = amdgpu_ctx_get_fence(ctx: p->ctx, entity: job->base.entity, seq: ~0ull);
1812	if (fence) {
1813	dma_fence_wait(fence, intr: false);
1814	dma_fence_put(fence);
1815	}
1816
1817	scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC]
1818	[AMDGPU_RING_PRIO_0].sched;
1819	drm_sched_entity_modify_sched(entity: job->base.entity, sched_list: scheds, num_sched_list: 1);
1820	return 0;
1821	}
1822
1823	static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
1824	uint64_t addr)
1825	{
1826	struct ttm_operation_ctx ctx = { false, false };
1827	struct amdgpu_bo_va_mapping *map;
1828	uint32_t *msg, num_buffers;
1829	struct amdgpu_bo *bo;
1830	uint64_t start, end;
1831	unsigned int i;
1832	void *ptr;
1833	int r;
1834
1835	addr &= AMDGPU_GMC_HOLE_MASK;
1836	r = amdgpu_cs_find_mapping(parser: p, addr, bo: &bo, mapping: &map);
1837	if (r) {
1838	DRM_ERROR("Can't find BO for addr 0x%08llx\n", addr);
1839	return r;
1840	}
1841
1842	start = map->start * AMDGPU_GPU_PAGE_SIZE;
1843	end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1844	if (addr & 0x7) {
1845	DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1846	return -EINVAL;
1847	}
1848
1849	bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1850	amdgpu_bo_placement_from_domain(abo: bo, domain: bo->allowed_domains);
1851	r = ttm_bo_validate(bo: &bo->tbo, placement: &bo->placement, ctx: &ctx);
1852	if (r) {
1853	DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1854	return r;
1855	}
1856
1857	r = amdgpu_bo_kmap(bo, ptr: &ptr);
1858	if (r) {
1859	DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1860	return r;
1861	}
1862
1863	msg = ptr + addr - start;
1864
1865	/* Check length */
1866	if (msg[1] > end - addr) {
1867	r = -EINVAL;
1868	goto out;
1869	}
1870
1871	if (msg[3] != RDECODE_MSG_CREATE)
1872	goto out;
1873
1874	num_buffers = msg[2];
1875	for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1876	uint32_t offset, size, *create;
1877
1878	if (msg[0] != RDECODE_MESSAGE_CREATE)
1879	continue;
1880
1881	offset = msg[1];
1882	size = msg[2];
1883
1884	if (offset + size > end) {
1885	r = -EINVAL;
1886	goto out;
1887	}
1888
1889	create = ptr + addr + offset - start;
1890
1891	/* H264, HEVC and VP9 can run on any instance */
1892	if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1893	continue;
1894
1895	r = vcn_v4_0_limit_sched(p, job);
1896	if (r)
1897	goto out;
1898	}
1899
1900	out:
1901	amdgpu_bo_kunmap(bo);
1902	return r;
1903	}
1904
1905	#define RADEON_VCN_ENGINE_TYPE_ENCODE (0x00000002)
1906	#define RADEON_VCN_ENGINE_TYPE_DECODE (0x00000003)
1907	#define RADEON_VCN_ENGINE_INFO (0x30000001)
1908	#define RENCODE_ENCODE_STANDARD_AV1 2
1909	#define RENCODE_IB_PARAM_SESSION_INIT 0x00000003
1910
1911	/* return the offset in ib if id is found, -1 otherwise */
1912	static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start)
1913	{
1914	int i;
1915
1916	for (i = start; i < ib->length_dw && ib->ptr[i] >= 8; i += ib->ptr[i] / 4) {
1917	if (ib->ptr[i + 1] == id)
1918	return i;
1919	}
1920	return -1;
1921	}
1922
1923	static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1924	struct amdgpu_job *job,
1925	struct amdgpu_ib *ib)
1926	{
1927	struct amdgpu_ring *ring = amdgpu_job_ring(job);
1928	struct amdgpu_vcn_decode_buffer *decode_buffer;
1929	uint64_t addr;
1930	uint32_t val;
1931	int idx = 0, sidx;
1932
1933	/* The first instance can decode anything */
1934	if (!ring->me)
1935	return 0;
1936
1937	while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, start: idx)) >= 0) {
1938	val = amdgpu_ib_get_value(ib, idx: idx + 2); /* RADEON_VCN_ENGINE_TYPE */
1939	if (val == RADEON_VCN_ENGINE_TYPE_DECODE) {
1940	decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[idx + 6];
1941
1942	if (!(decode_buffer->valid_buf_flag & 0x1))
1943	return 0;
1944
1945	addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 |
1946	decode_buffer->msg_buffer_address_lo;
1947	return vcn_v4_0_dec_msg(p, job, addr);
1948	} else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) {
1949	sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, start: idx);
1950	if (sidx >= 0 && ib->ptr[sidx + 2] == RENCODE_ENCODE_STANDARD_AV1)
1951	return vcn_v4_0_limit_sched(p, job);
1952	}
1953	idx += ib->ptr[idx] / 4;
1954	}
1955	return 0;
1956	}
1957
1958	static int vcn_v4_0_ring_reset(struct amdgpu_ring *ring,
1959	unsigned int vmid,
1960	struct amdgpu_fence *timedout_fence)
1961	{
1962	struct amdgpu_device *adev = ring->adev;
1963	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
1964	int r;
1965
1966	amdgpu_ring_reset_helper_begin(ring, guilty_fence: timedout_fence);
1967	r = vcn_v4_0_stop(vinst);
1968	if (r)
1969	return r;
1970	r = vcn_v4_0_start(vinst);
1971	if (r)
1972	return r;
1973	return amdgpu_ring_reset_helper_end(ring, guilty_fence: timedout_fence);
1974	}
1975
1976	static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
1977	.type = AMDGPU_RING_TYPE_VCN_ENC,
1978	.align_mask = 0x3f,
1979	.nop = VCN_ENC_CMD_NO_OP,
1980	.extra_bytes = sizeof(struct amdgpu_vcn_rb_metadata),
1981	.get_rptr = vcn_v4_0_unified_ring_get_rptr,
1982	.get_wptr = vcn_v4_0_unified_ring_get_wptr,
1983	.set_wptr = vcn_v4_0_unified_ring_set_wptr,
1984	.patch_cs_in_place = vcn_v4_0_ring_patch_cs_in_place,
1985	.emit_frame_size =
1986	SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1987	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1988	4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1989	5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1990	1, /* vcn_v2_0_enc_ring_insert_end */
1991	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1992	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
1993	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
1994	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1995	.test_ring = amdgpu_vcn_enc_ring_test_ring,
1996	.test_ib = amdgpu_vcn_unified_ring_test_ib,
1997	.insert_nop = amdgpu_ring_insert_nop,
1998	.insert_end = vcn_v2_0_enc_ring_insert_end,
1999	.pad_ib = amdgpu_ring_generic_pad_ib,
2000	.begin_use = amdgpu_vcn_ring_begin_use,
2001	.end_use = amdgpu_vcn_ring_end_use,
2002	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2003	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2004	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2005	.reset = vcn_v4_0_ring_reset,
2006	};
2007
2008	/**
2009	* vcn_v4_0_set_unified_ring_funcs - set unified ring functions
2010	*
2011	* @adev: amdgpu_device pointer
2012	*
2013	* Set unified ring functions
2014	*/
2015	static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev)
2016	{
2017	int i;
2018
2019	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2020	if (adev->vcn.harvest_config & (1 << i))
2021	continue;
2022
2023	if (amdgpu_ip_version(adev, ip: VCN_HWIP, inst: 0) == IP_VERSION(4, 0, 2))
2024	vcn_v4_0_unified_ring_vm_funcs.secure_submission_supported = true;
2025
2026	adev->vcn.inst[i].ring_enc[0].funcs =
2027	(const struct amdgpu_ring_funcs *)&vcn_v4_0_unified_ring_vm_funcs;
2028	adev->vcn.inst[i].ring_enc[0].me = i;
2029	}
2030	}
2031
2032	/**
2033	* vcn_v4_0_is_idle - check VCN block is idle
2034	*
2035	* @ip_block: Pointer to the amdgpu_ip_block structure
2036	*
2037	* Check whether VCN block is idle
2038	*/
2039	static bool vcn_v4_0_is_idle(struct amdgpu_ip_block *ip_block)
2040	{
2041	struct amdgpu_device *adev = ip_block->adev;
2042	int i, ret = 1;
2043
2044	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2045	if (adev->vcn.harvest_config & (1 << i))
2046	continue;
2047
2048	ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
2049	}
2050
2051	return ret;
2052	}
2053
2054	/**
2055	* vcn_v4_0_wait_for_idle - wait for VCN block idle
2056	*
2057	* @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
2058	*
2059	* Wait for VCN block idle
2060	*/
2061	static int vcn_v4_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
2062	{
2063	struct amdgpu_device *adev = ip_block->adev;
2064	int i, ret = 0;
2065
2066	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2067	if (adev->vcn.harvest_config & (1 << i))
2068	continue;
2069
2070	ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
2071	UVD_STATUS__IDLE);
2072	if (ret)
2073	return ret;
2074	}
2075
2076	return ret;
2077	}
2078
2079	/**
2080	* vcn_v4_0_set_clockgating_state - set VCN block clockgating state
2081	*
2082	* @ip_block: amdgpu_ip_block pointer
2083	* @state: clock gating state
2084	*
2085	* Set VCN block clockgating state
2086	*/
2087	static int vcn_v4_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2088	enum amd_clockgating_state state)
2089	{
2090	struct amdgpu_device *adev = ip_block->adev;
2091	bool enable = state == AMD_CG_STATE_GATE;
2092	int i;
2093
2094	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2095	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i];
2096
2097	if (adev->vcn.harvest_config & (1 << i))
2098	continue;
2099
2100	if (enable) {
2101	if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
2102	return -EBUSY;
2103	vcn_v4_0_enable_clock_gating(vinst);
2104	} else {
2105	vcn_v4_0_disable_clock_gating(vinst);
2106	}
2107	}
2108
2109	return 0;
2110	}
2111
2112	static int vcn_v4_0_set_pg_state(struct amdgpu_vcn_inst *vinst,
2113	enum amd_powergating_state state)
2114	{
2115	struct amdgpu_device *adev = vinst->adev;
2116	int ret = 0;
2117
2118	/* for SRIOV, guest should not control VCN Power-gating
2119	* MMSCH FW should control Power-gating and clock-gating
2120	* guest should avoid touching CGC and PG
2121	*/
2122	if (amdgpu_sriov_vf(adev)) {
2123	vinst->cur_state = AMD_PG_STATE_UNGATE;
2124	return 0;
2125	}
2126
2127	if (state == vinst->cur_state)
2128	return 0;
2129
2130	if (state == AMD_PG_STATE_GATE)
2131	ret = vcn_v4_0_stop(vinst);
2132	else
2133	ret = vcn_v4_0_start(vinst);
2134
2135	if (!ret)
2136	vinst->cur_state = state;
2137
2138	return ret;
2139	}
2140
2141	/**
2142	* vcn_v4_0_set_ras_interrupt_state - set VCN block RAS interrupt state
2143	*
2144	* @adev: amdgpu_device pointer
2145	* @source: interrupt sources
2146	* @type: interrupt types
2147	* @state: interrupt states
2148	*
2149	* Set VCN block RAS interrupt state
2150	*/
2151	static int vcn_v4_0_set_ras_interrupt_state(struct amdgpu_device *adev,
2152	struct amdgpu_irq_src *source,
2153	unsigned int type,
2154	enum amdgpu_interrupt_state state)
2155	{
2156	return 0;
2157	}
2158
2159	/**
2160	* vcn_v4_0_process_interrupt - process VCN block interrupt
2161	*
2162	* @adev: amdgpu_device pointer
2163	* @source: interrupt sources
2164	* @entry: interrupt entry from clients and sources
2165	*
2166	* Process VCN block interrupt
2167	*/
2168	static int vcn_v4_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
2169	struct amdgpu_iv_entry *entry)
2170	{
2171	uint32_t ip_instance;
2172
2173	if (amdgpu_sriov_is_vcn_rb_decouple(adev)) {
2174	ip_instance = entry->ring_id;
2175	} else {
2176	switch (entry->client_id) {
2177	case SOC15_IH_CLIENTID_VCN:
2178	ip_instance = 0;
2179	break;
2180	case SOC15_IH_CLIENTID_VCN1:
2181	ip_instance = 1;
2182	break;
2183	default:
2184	DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2185	return 0;
2186	}
2187	}
2188
2189	DRM_DEBUG("IH: VCN TRAP\n");
2190
2191	switch (entry->src_id) {
2192	case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2193	amdgpu_fence_process(ring: &adev->vcn.inst[ip_instance].ring_enc[0]);
2194	break;
2195	default:
2196	DRM_ERROR("Unhandled interrupt: %d %d\n",
2197	entry->src_id, entry->src_data[0]);
2198	break;
2199	}
2200
2201	return 0;
2202	}
2203
2204	static const struct amdgpu_irq_src_funcs vcn_v4_0_irq_funcs = {
2205	.process = vcn_v4_0_process_interrupt,
2206	};
2207
2208	static const struct amdgpu_irq_src_funcs vcn_v4_0_ras_irq_funcs = {
2209	.set = vcn_v4_0_set_ras_interrupt_state,
2210	.process = amdgpu_vcn_process_poison_irq,
2211	};
2212
2213	/**
2214	* vcn_v4_0_set_irq_funcs - set VCN block interrupt irq functions
2215	*
2216	* @adev: amdgpu_device pointer
2217	*
2218	* Set VCN block interrupt irq functions
2219	*/
2220	static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev)
2221	{
2222	int i;
2223
2224	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2225	if (adev->vcn.harvest_config & (1 << i))
2226	continue;
2227
2228	adev->vcn.inst[i].irq.num_types = adev->vcn.inst[i].num_enc_rings + 1;
2229	adev->vcn.inst[i].irq.funcs = &vcn_v4_0_irq_funcs;
2230
2231	adev->vcn.inst[i].ras_poison_irq.num_types = adev->vcn.inst[i].num_enc_rings + 1;
2232	adev->vcn.inst[i].ras_poison_irq.funcs = &vcn_v4_0_ras_irq_funcs;
2233	}
2234	}
2235
2236	static const struct amd_ip_funcs vcn_v4_0_ip_funcs = {
2237	.name = "vcn_v4_0",
2238	.early_init = vcn_v4_0_early_init,
2239	.sw_init = vcn_v4_0_sw_init,
2240	.sw_fini = vcn_v4_0_sw_fini,
2241	.hw_init = vcn_v4_0_hw_init,
2242	.hw_fini = vcn_v4_0_hw_fini,
2243	.suspend = vcn_v4_0_suspend,
2244	.resume = vcn_v4_0_resume,
2245	.is_idle = vcn_v4_0_is_idle,
2246	.wait_for_idle = vcn_v4_0_wait_for_idle,
2247	.set_clockgating_state = vcn_v4_0_set_clockgating_state,
2248	.set_powergating_state = vcn_set_powergating_state,
2249	.dump_ip_state = amdgpu_vcn_dump_ip_state,
2250	.print_ip_state = amdgpu_vcn_print_ip_state,
2251	};
2252
2253	const struct amdgpu_ip_block_version vcn_v4_0_ip_block = {
2254	.type = AMD_IP_BLOCK_TYPE_VCN,
2255	.major = 4,
2256	.minor = 0,
2257	.rev = 0,
2258	.funcs = &vcn_v4_0_ip_funcs,
2259	};
2260
2261	static uint32_t vcn_v4_0_query_poison_by_instance(struct amdgpu_device *adev,
2262	uint32_t instance, uint32_t sub_block)
2263	{
2264	uint32_t poison_stat = 0, reg_value = 0;
2265
2266	switch (sub_block) {
2267	case AMDGPU_VCN_V4_0_VCPU_VCODEC:
2268	reg_value = RREG32_SOC15(VCN, instance, regUVD_RAS_VCPU_VCODEC_STATUS);
2269	poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);
2270	break;
2271	default:
2272	break;
2273	}
2274
2275	if (poison_stat)
2276	dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",
2277	instance, sub_block);
2278
2279	return poison_stat;
2280	}
2281
2282	static bool vcn_v4_0_query_ras_poison_status(struct amdgpu_device *adev)
2283	{
2284	uint32_t inst, sub;
2285	uint32_t poison_stat = 0;
2286
2287	for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)
2288	for (sub = 0; sub < AMDGPU_VCN_V4_0_MAX_SUB_BLOCK; sub++)
2289	poison_stat +=
2290	vcn_v4_0_query_poison_by_instance(adev, instance: inst, sub_block: sub);
2291
2292	return !!poison_stat;
2293	}
2294
2295	const struct amdgpu_ras_block_hw_ops vcn_v4_0_ras_hw_ops = {
2296	.query_poison_status = vcn_v4_0_query_ras_poison_status,
2297	};
2298
2299	static struct amdgpu_vcn_ras vcn_v4_0_ras = {
2300	.ras_block = {
2301	.hw_ops = &vcn_v4_0_ras_hw_ops,
2302	.ras_late_init = amdgpu_vcn_ras_late_init,
2303	},
2304	};
2305
2306	static void vcn_v4_0_set_ras_funcs(struct amdgpu_device *adev)
2307	{
2308	switch (amdgpu_ip_version(adev, ip: VCN_HWIP, inst: 0)) {
2309	case IP_VERSION(4, 0, 0):
2310	adev->vcn.ras = &vcn_v4_0_ras;
2311	break;
2312	default:
2313	break;
2314	}
2315	}
2316