1	/*
2	* Copyright 2016 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/delay.h>
25	#include <linux/firmware.h>
26	#include <linux/module.h>
27	#include <linux/pci.h>
28
29	#include "amdgpu.h"
30	#include "amdgpu_ucode.h"
31	#include "amdgpu_trace.h"
32
33	#include "sdma0/sdma0_4_2_offset.h"
34	#include "sdma0/sdma0_4_2_sh_mask.h"
35	#include "sdma1/sdma1_4_2_offset.h"
36	#include "sdma1/sdma1_4_2_sh_mask.h"
37	#include "sdma2/sdma2_4_2_2_offset.h"
38	#include "sdma2/sdma2_4_2_2_sh_mask.h"
39	#include "sdma3/sdma3_4_2_2_offset.h"
40	#include "sdma3/sdma3_4_2_2_sh_mask.h"
41	#include "sdma4/sdma4_4_2_2_offset.h"
42	#include "sdma4/sdma4_4_2_2_sh_mask.h"
43	#include "sdma5/sdma5_4_2_2_offset.h"
44	#include "sdma5/sdma5_4_2_2_sh_mask.h"
45	#include "sdma6/sdma6_4_2_2_offset.h"
46	#include "sdma6/sdma6_4_2_2_sh_mask.h"
47	#include "sdma7/sdma7_4_2_2_offset.h"
48	#include "sdma7/sdma7_4_2_2_sh_mask.h"
49	#include "sdma0/sdma0_4_1_default.h"
50
51	#include "soc15_common.h"
52	#include "soc15.h"
53	#include "vega10_sdma_pkt_open.h"
54
55	#include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"
56	#include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"
57
58	#include "amdgpu_ras.h"
59	#include "sdma_v4_4.h"
60
61	MODULE_FIRMWARE("amdgpu/vega10_sdma.bin");
62	MODULE_FIRMWARE("amdgpu/vega10_sdma1.bin");
63	MODULE_FIRMWARE("amdgpu/vega12_sdma.bin");
64	MODULE_FIRMWARE("amdgpu/vega12_sdma1.bin");
65	MODULE_FIRMWARE("amdgpu/vega20_sdma.bin");
66	MODULE_FIRMWARE("amdgpu/vega20_sdma1.bin");
67	MODULE_FIRMWARE("amdgpu/raven_sdma.bin");
68	MODULE_FIRMWARE("amdgpu/picasso_sdma.bin");
69	MODULE_FIRMWARE("amdgpu/raven2_sdma.bin");
70	MODULE_FIRMWARE("amdgpu/arcturus_sdma.bin");
71	MODULE_FIRMWARE("amdgpu/renoir_sdma.bin");
72	MODULE_FIRMWARE("amdgpu/green_sardine_sdma.bin");
73	MODULE_FIRMWARE("amdgpu/aldebaran_sdma.bin");
74
75	static const struct amdgpu_hwip_reg_entry sdma_reg_list_4_0[] = {
76	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS_REG),
77	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS1_REG),
78	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS2_REG),
79	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_STATUS3_REG),
80	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UCODE_CHECKSUM),
81	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH_HI),
82	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RB_RPTR_FETCH),
83	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_STATUS),
84	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_STATUS),
85	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK0),
86	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_RD_XNACK1),
87	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK0),
88	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_UTCL1_WR_XNACK1),
89	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_CNTL),
90	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR),
91	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_RPTR_HI),
92	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR),
93	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_RB_WPTR_HI),
94	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_OFFSET),
95	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_LO),
96	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_BASE_HI),
97	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_CNTL),
98	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_RPTR),
99	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_IB_SUB_REMAIN),
100	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_GFX_DUMMY_REG),
101	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_CNTL),
102	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR),
103	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_RPTR_HI),
104	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR),
105	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_RB_WPTR_HI),
106	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_OFFSET),
107	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_LO),
108	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_IB_BASE_HI),
109	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_PAGE_DUMMY_REG),
110	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_CNTL),
111	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR),
112	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_RPTR_HI),
113	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR),
114	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_RB_WPTR_HI),
115	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_OFFSET),
116	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_LO),
117	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_IB_BASE_HI),
118	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_RLC0_DUMMY_REG),
119	SOC15_REG_ENTRY_STR(GC, 0, mmSDMA0_VM_CNTL)
120	};
121
122	#define SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK 0x000000F8L
123	#define SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK 0xFC000000L
124
125	#define WREG32_SDMA(instance, offset, value) \
126	WREG32(sdma_v4_0_get_reg_offset(adev, (instance), (offset)), value)
127	#define RREG32_SDMA(instance, offset) \
128	RREG32(sdma_v4_0_get_reg_offset(adev, (instance), (offset)))
129
130	static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev);
131	static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev);
132	static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev);
133	static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev);
134	static void sdma_v4_0_set_ras_funcs(struct amdgpu_device *adev);
135
136	static const struct soc15_reg_golden golden_settings_sdma_4[] = {
137	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07),
138	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xff000ff0, 0x3f000100),
139	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_IB_CNTL, 0x800f0100, 0x00000100),
140	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
141	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_IB_CNTL, 0x800f0100, 0x00000100),
142	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
143	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0x003ff006, 0x0003c000),
144	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL, 0x800f0100, 0x00000100),
145	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
146	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL, 0x800f0100, 0x00000100),
147	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
148	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0),
149	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xfc000000, 0x00000000),
150	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CLK_CTRL, 0xffffffff, 0x3f000100),
151	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_IB_CNTL, 0x800f0100, 0x00000100),
152	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
153	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_IB_CNTL, 0x800f0100, 0x00000100),
154	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
155	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_POWER_CNTL, 0x003ff000, 0x0003c000),
156	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_IB_CNTL, 0x800f0100, 0x00000100),
157	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
158	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_IB_CNTL, 0x800f0100, 0x00000100),
159	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
160	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_PAGE, 0x000003ff, 0x000003c0),
161	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_WATERMK, 0xfc000000, 0x00000000)
162	};
163
164	static const struct soc15_reg_golden golden_settings_sdma_vg10[] = {
165	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),
166	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),
167	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
168	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831d07),
169	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),
170	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),
171	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
172	};
173
174	static const struct soc15_reg_golden golden_settings_sdma_vg12[] = {
175	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00104001),
176	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104001),
177	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
178	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831d07),
179	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0018773f, 0x00104001),
180	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104001),
181	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
182	};
183
184	static const struct soc15_reg_golden golden_settings_sdma_4_1[] = {
185	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07),
186	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100),
187	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100),
188	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
189	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0xfc3fffff, 0x40000051),
190	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100),
191	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
192	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100),
193	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
194	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003e0),
195	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xfc000000, 0x00000000)
196	};
197
198	static const struct soc15_reg_golden golden_settings_sdma0_4_2_init[] = {
199	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000),
200	};
201
202	static const struct soc15_reg_golden golden_settings_sdma0_4_2[] =
203	{
204	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
205	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100),
206	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
207	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
208	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
209	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
210	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
211	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
212	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RD_BURST_CNTL, 0x0000000f, 0x00000003),
213	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
214	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000),
215	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
216	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
217	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC2_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
218	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
219	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC3_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
220	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
221	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC4_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
222	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
223	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC5_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
224	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
225	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC6_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
226	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
227	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC7_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
228	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
229	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0),
230	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
231	};
232
233	static const struct soc15_reg_golden golden_settings_sdma1_4_2[] = {
234	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
235	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CLK_CTRL, 0xffffffff, 0x3f000100),
236	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
237	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
238	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
239	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
240	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
241	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
242	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RD_BURST_CNTL, 0x0000000f, 0x00000003),
243	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
244	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff0, 0x00403000),
245	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
246	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
247	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC2_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
248	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC2_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
249	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC3_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
250	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC3_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
251	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC4_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
252	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC4_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
253	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC5_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
254	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC5_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
255	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC6_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
256	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC6_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
257	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC7_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
258	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
259	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_PAGE, 0x000003ff, 0x000003c0),
260	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
261	};
262
263	static const struct soc15_reg_golden golden_settings_sdma_rv1[] =
264	{
265	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00000002),
266	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00000002)
267	};
268
269	static const struct soc15_reg_golden golden_settings_sdma_rv2[] =
270	{
271	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00003001),
272	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00003001)
273	};
274
275	static const struct soc15_reg_golden golden_settings_sdma_arct[] =
276	{
277	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
278	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
279	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
280	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
281	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
282	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
283	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
284	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
285	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
286	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
287	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
288	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
289	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
290	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
291	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
292	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
293	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
294	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
295	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
296	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
297	SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
298	SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
299	SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
300	SOC15_REG_GOLDEN_VALUE(SDMA5, 0, mmSDMA5_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
301	SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
302	SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
303	SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
304	SOC15_REG_GOLDEN_VALUE(SDMA6, 0, mmSDMA6_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
305	SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
306	SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_GB_ADDR_CONFIG, 0x0000773f, 0x00004002),
307	SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_GB_ADDR_CONFIG_READ, 0x0000773f, 0x00004002),
308	SOC15_REG_GOLDEN_VALUE(SDMA7, 0, mmSDMA7_UTCL1_TIMEOUT, 0xffffffff, 0x00010001)
309	};
310
311	static const struct soc15_reg_golden golden_settings_sdma_aldebaran[] = {
312	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),
313	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),
314	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
315	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),
316	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),
317	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
318	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),
319	SOC15_REG_GOLDEN_VALUE(SDMA2, 0, mmSDMA2_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),
320	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA2_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
321	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),
322	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),
323	SOC15_REG_GOLDEN_VALUE(SDMA3, 0, mmSDMA3_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
324	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),
325	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),
326	SOC15_REG_GOLDEN_VALUE(SDMA4, 0, mmSDMA4_UTCL1_TIMEOUT, 0xffffffff, 0x00010001),
327	};
328
329	static const struct soc15_reg_golden golden_settings_sdma_4_3[] = {
330	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
331	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100),
332	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00000002),
333	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00000002),
334	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
335	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0x003fff07, 0x40000051),
336	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
337	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
338	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003e0),
339	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xfc000000, 0x03fbe1fe)
340	};
341
342	static const struct soc15_ras_field_entry sdma_v4_0_ras_fields[] = {
343	{ "SDMA_UCODE_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
344	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_UCODE_BUF_SED),
345	0, 0,
346	},
347	{ "SDMA_RB_CMD_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
348	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_RB_CMD_BUF_SED),
349	0, 0,
350	},
351	{ "SDMA_IB_CMD_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
352	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_IB_CMD_BUF_SED),
353	0, 0,
354	},
355	{ "SDMA_UTCL1_RD_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
356	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_UTCL1_RD_FIFO_SED),
357	0, 0,
358	},
359	{ "SDMA_UTCL1_RDBST_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
360	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_UTCL1_RDBST_FIFO_SED),
361	0, 0,
362	},
363	{ "SDMA_DATA_LUT_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
364	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_DATA_LUT_FIFO_SED),
365	0, 0,
366	},
367	{ "SDMA_MBANK_DATA_BUF0_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
368	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF0_SED),
369	0, 0,
370	},
371	{ "SDMA_MBANK_DATA_BUF1_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
372	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF1_SED),
373	0, 0,
374	},
375	{ "SDMA_MBANK_DATA_BUF2_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
376	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF2_SED),
377	0, 0,
378	},
379	{ "SDMA_MBANK_DATA_BUF3_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
380	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF3_SED),
381	0, 0,
382	},
383	{ "SDMA_MBANK_DATA_BUF4_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
384	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF4_SED),
385	0, 0,
386	},
387	{ "SDMA_MBANK_DATA_BUF5_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
388	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF5_SED),
389	0, 0,
390	},
391	{ "SDMA_MBANK_DATA_BUF6_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
392	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF6_SED),
393	0, 0,
394	},
395	{ "SDMA_MBANK_DATA_BUF7_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
396	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF7_SED),
397	0, 0,
398	},
399	{ "SDMA_MBANK_DATA_BUF8_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
400	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF8_SED),
401	0, 0,
402	},
403	{ "SDMA_MBANK_DATA_BUF9_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
404	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF9_SED),
405	0, 0,
406	},
407	{ "SDMA_MBANK_DATA_BUF10_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
408	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF10_SED),
409	0, 0,
410	},
411	{ "SDMA_MBANK_DATA_BUF11_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
412	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF11_SED),
413	0, 0,
414	},
415	{ "SDMA_MBANK_DATA_BUF12_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
416	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF12_SED),
417	0, 0,
418	},
419	{ "SDMA_MBANK_DATA_BUF13_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
420	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF13_SED),
421	0, 0,
422	},
423	{ "SDMA_MBANK_DATA_BUF14_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
424	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF14_SED),
425	0, 0,
426	},
427	{ "SDMA_MBANK_DATA_BUF15_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
428	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MBANK_DATA_BUF15_SED),
429	0, 0,
430	},
431	{ "SDMA_SPLIT_DAT_BUF_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
432	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_SPLIT_DAT_BUF_SED),
433	0, 0,
434	},
435	{ "SDMA_MC_WR_ADDR_FIFO_SED", SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_EDC_COUNTER),
436	SOC15_REG_FIELD(SDMA0_EDC_COUNTER, SDMA_MC_WR_ADDR_FIFO_SED),
437	0, 0,
438	},
439	};
440
441	static u32 sdma_v4_0_get_reg_offset(struct amdgpu_device *adev,
442	u32 instance, u32 offset)
443	{
444	switch (instance) {
445	case 0:
446	return (adev->reg_offset[SDMA0_HWIP][0][0] + offset);
447	case 1:
448	return (adev->reg_offset[SDMA1_HWIP][0][0] + offset);
449	case 2:
450	return (adev->reg_offset[SDMA2_HWIP][0][1] + offset);
451	case 3:
452	return (adev->reg_offset[SDMA3_HWIP][0][1] + offset);
453	case 4:
454	return (adev->reg_offset[SDMA4_HWIP][0][1] + offset);
455	case 5:
456	return (adev->reg_offset[SDMA5_HWIP][0][1] + offset);
457	case 6:
458	return (adev->reg_offset[SDMA6_HWIP][0][1] + offset);
459	case 7:
460	return (adev->reg_offset[SDMA7_HWIP][0][1] + offset);
461	default:
462	break;
463	}
464	return 0;
465	}
466
467	static unsigned sdma_v4_0_seq_to_irq_id(int seq_num)
468	{
469	switch (seq_num) {
470	case 0:
471	return SOC15_IH_CLIENTID_SDMA0;
472	case 1:
473	return SOC15_IH_CLIENTID_SDMA1;
474	case 2:
475	return SOC15_IH_CLIENTID_SDMA2;
476	case 3:
477	return SOC15_IH_CLIENTID_SDMA3;
478	case 4:
479	return SOC15_IH_CLIENTID_SDMA4;
480	case 5:
481	return SOC15_IH_CLIENTID_SDMA5;
482	case 6:
483	return SOC15_IH_CLIENTID_SDMA6;
484	case 7:
485	return SOC15_IH_CLIENTID_SDMA7;
486	default:
487	break;
488	}
489	return -EINVAL;
490	}
491
492	static int sdma_v4_0_irq_id_to_seq(unsigned client_id)
493	{
494	switch (client_id) {
495	case SOC15_IH_CLIENTID_SDMA0:
496	return 0;
497	case SOC15_IH_CLIENTID_SDMA1:
498	return 1;
499	case SOC15_IH_CLIENTID_SDMA2:
500	return 2;
501	case SOC15_IH_CLIENTID_SDMA3:
502	return 3;
503	case SOC15_IH_CLIENTID_SDMA4:
504	return 4;
505	case SOC15_IH_CLIENTID_SDMA5:
506	return 5;
507	case SOC15_IH_CLIENTID_SDMA6:
508	return 6;
509	case SOC15_IH_CLIENTID_SDMA7:
510	return 7;
511	default:
512	break;
513	}
514	return -EINVAL;
515	}
516
517	static void sdma_v4_0_init_golden_registers(struct amdgpu_device *adev)
518	{
519	switch (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0)) {
520	case IP_VERSION(4, 0, 0):
521	soc15_program_register_sequence(adev,
522	registers: golden_settings_sdma_4,
523	ARRAY_SIZE(golden_settings_sdma_4));
524	soc15_program_register_sequence(adev,
525	registers: golden_settings_sdma_vg10,
526	ARRAY_SIZE(golden_settings_sdma_vg10));
527	break;
528	case IP_VERSION(4, 0, 1):
529	soc15_program_register_sequence(adev,
530	registers: golden_settings_sdma_4,
531	ARRAY_SIZE(golden_settings_sdma_4));
532	soc15_program_register_sequence(adev,
533	registers: golden_settings_sdma_vg12,
534	ARRAY_SIZE(golden_settings_sdma_vg12));
535	break;
536	case IP_VERSION(4, 2, 0):
537	soc15_program_register_sequence(adev,
538	registers: golden_settings_sdma0_4_2_init,
539	ARRAY_SIZE(golden_settings_sdma0_4_2_init));
540	soc15_program_register_sequence(adev,
541	registers: golden_settings_sdma0_4_2,
542	ARRAY_SIZE(golden_settings_sdma0_4_2));
543	soc15_program_register_sequence(adev,
544	registers: golden_settings_sdma1_4_2,
545	ARRAY_SIZE(golden_settings_sdma1_4_2));
546	break;
547	case IP_VERSION(4, 2, 2):
548	soc15_program_register_sequence(adev,
549	registers: golden_settings_sdma_arct,
550	ARRAY_SIZE(golden_settings_sdma_arct));
551	break;
552	case IP_VERSION(4, 4, 0):
553	soc15_program_register_sequence(adev,
554	registers: golden_settings_sdma_aldebaran,
555	ARRAY_SIZE(golden_settings_sdma_aldebaran));
556	break;
557	case IP_VERSION(4, 1, 0):
558	case IP_VERSION(4, 1, 1):
559	soc15_program_register_sequence(adev,
560	registers: golden_settings_sdma_4_1,
561	ARRAY_SIZE(golden_settings_sdma_4_1));
562	if (adev->apu_flags & AMD_APU_IS_RAVEN2)
563	soc15_program_register_sequence(adev,
564	registers: golden_settings_sdma_rv2,
565	ARRAY_SIZE(golden_settings_sdma_rv2));
566	else
567	soc15_program_register_sequence(adev,
568	registers: golden_settings_sdma_rv1,
569	ARRAY_SIZE(golden_settings_sdma_rv1));
570	break;
571	case IP_VERSION(4, 1, 2):
572	soc15_program_register_sequence(adev,
573	registers: golden_settings_sdma_4_3,
574	ARRAY_SIZE(golden_settings_sdma_4_3));
575	break;
576	default:
577	break;
578	}
579	}
580
581	static void sdma_v4_0_setup_ulv(struct amdgpu_device *adev)
582	{
583	int i;
584
585	/*
586	* The only chips with SDMAv4 and ULV are VG10 and VG20.
587	* Server SKUs take a different hysteresis setting from other SKUs.
588	*/
589	switch (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0)) {
590	case IP_VERSION(4, 0, 0):
591	if (adev->pdev->device == 0x6860)
592	break;
593	return;
594	case IP_VERSION(4, 2, 0):
595	if (adev->pdev->device == 0x66a1)
596	break;
597	return;
598	default:
599	return;
600	}
601
602	for (i = 0; i < adev->sdma.num_instances; i++) {
603	uint32_t temp;
604
605	temp = RREG32_SDMA(i, mmSDMA0_ULV_CNTL);
606	temp = REG_SET_FIELD(temp, SDMA0_ULV_CNTL, HYSTERESIS, 0x0);
607	WREG32_SDMA(i, mmSDMA0_ULV_CNTL, temp);
608	}
609	}
610
611	/**
612	* sdma_v4_0_init_microcode - load ucode images from disk
613	*
614	* @adev: amdgpu_device pointer
615	*
616	* Use the firmware interface to load the ucode images into
617	* the driver (not loaded into hw).
618	* Returns 0 on success, error on failure.
619	*/
620
621	// emulation only, won't work on real chip
622	// vega10 real chip need to use PSP to load firmware
623	static int sdma_v4_0_init_microcode(struct amdgpu_device *adev)
624	{
625	int ret, i;
626
627	for (i = 0; i < adev->sdma.num_instances; i++) {
628	if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) ==
629	IP_VERSION(4, 2, 2) ||
630	amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) ==
631	IP_VERSION(4, 4, 0)) {
632	/* Acturus & Aldebaran will leverage the same FW memory
633	for every SDMA instance */
634	ret = amdgpu_sdma_init_microcode(adev, instance: 0, duplicate: true);
635	break;
636	} else {
637	ret = amdgpu_sdma_init_microcode(adev, instance: i, duplicate: false);
638	if (ret)
639	return ret;
640	}
641	}
642
643	return ret;
644	}
645
646	/**
647	* sdma_v4_0_ring_get_rptr - get the current read pointer
648	*
649	* @ring: amdgpu ring pointer
650	*
651	* Get the current rptr from the hardware (VEGA10+).
652	*/
653	static uint64_t sdma_v4_0_ring_get_rptr(struct amdgpu_ring *ring)
654	{
655	u64 *rptr;
656
657	/* XXX check if swapping is necessary on BE */
658	rptr = ((u64 *)ring->rptr_cpu_addr);
659
660	DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
661	return ((*rptr) >> 2);
662	}
663
664	/**
665	* sdma_v4_0_ring_get_wptr - get the current write pointer
666	*
667	* @ring: amdgpu ring pointer
668	*
669	* Get the current wptr from the hardware (VEGA10+).
670	*/
671	static uint64_t sdma_v4_0_ring_get_wptr(struct amdgpu_ring *ring)
672	{
673	struct amdgpu_device *adev = ring->adev;
674	u64 wptr;
675
676	if (ring->use_doorbell) {
677	/* XXX check if swapping is necessary on BE */
678	wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
679	DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
680	} else {
681	wptr = RREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR_HI);
682	wptr = wptr << 32;
683	wptr |= RREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR);
684	DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n",
685	ring->me, wptr);
686	}
687
688	return wptr >> 2;
689	}
690
691	/**
692	* sdma_v4_0_ring_set_wptr - commit the write pointer
693	*
694	* @ring: amdgpu ring pointer
695	*
696	* Write the wptr back to the hardware (VEGA10+).
697	*/
698	static void sdma_v4_0_ring_set_wptr(struct amdgpu_ring *ring)
699	{
700	struct amdgpu_device *adev = ring->adev;
701
702	DRM_DEBUG("Setting write pointer\n");
703	if (ring->use_doorbell) {
704	u64 *wb = (u64 *)ring->wptr_cpu_addr;
705
706	DRM_DEBUG("Using doorbell -- "
707	"wptr_offs == 0x%08x "
708	"lower_32_bits(ring->wptr << 2) == 0x%08x "
709	"upper_32_bits(ring->wptr << 2) == 0x%08x\n",
710	ring->wptr_offs,
711	lower_32_bits(ring->wptr << 2),
712	upper_32_bits(ring->wptr << 2));
713	/* XXX check if swapping is necessary on BE */
714	WRITE_ONCE(*wb, (ring->wptr << 2));
715	DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
716	ring->doorbell_index, ring->wptr << 2);
717	WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
718	} else {
719	DRM_DEBUG("Not using doorbell -- "
720	"mmSDMA%i_GFX_RB_WPTR == 0x%08x "
721	"mmSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
722	ring->me,
723	lower_32_bits(ring->wptr << 2),
724	ring->me,
725	upper_32_bits(ring->wptr << 2));
726	WREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR,
727	lower_32_bits(ring->wptr << 2));
728	WREG32_SDMA(ring->me, mmSDMA0_GFX_RB_WPTR_HI,
729	upper_32_bits(ring->wptr << 2));
730	}
731	}
732
733	/**
734	* sdma_v4_0_page_ring_get_wptr - get the current write pointer
735	*
736	* @ring: amdgpu ring pointer
737	*
738	* Get the current wptr from the hardware (VEGA10+).
739	*/
740	static uint64_t sdma_v4_0_page_ring_get_wptr(struct amdgpu_ring *ring)
741	{
742	struct amdgpu_device *adev = ring->adev;
743	u64 wptr;
744
745	if (ring->use_doorbell) {
746	/* XXX check if swapping is necessary on BE */
747	wptr = READ_ONCE(*((u64 *)ring->wptr_cpu_addr));
748	} else {
749	wptr = RREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR_HI);
750	wptr = wptr << 32;
751	wptr |= RREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR);
752	}
753
754	return wptr >> 2;
755	}
756
757	/**
758	* sdma_v4_0_page_ring_set_wptr - commit the write pointer
759	*
760	* @ring: amdgpu ring pointer
761	*
762	* Write the wptr back to the hardware (VEGA10+).
763	*/
764	static void sdma_v4_0_page_ring_set_wptr(struct amdgpu_ring *ring)
765	{
766	struct amdgpu_device *adev = ring->adev;
767
768	if (ring->use_doorbell) {
769	u64 *wb = (u64 *)ring->wptr_cpu_addr;
770
771	/* XXX check if swapping is necessary on BE */
772	WRITE_ONCE(*wb, (ring->wptr << 2));
773	WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
774	} else {
775	uint64_t wptr = ring->wptr << 2;
776
777	WREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR,
778	lower_32_bits(wptr));
779	WREG32_SDMA(ring->me, mmSDMA0_PAGE_RB_WPTR_HI,
780	upper_32_bits(wptr));
781	}
782	}
783
784	static void sdma_v4_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
785	{
786	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
787	int i;
788
789	for (i = 0; i < count; i++)
790	if (sdma && sdma->burst_nop && (i == 0))
791	amdgpu_ring_write(ring, v: ring->funcs->nop |
792	SDMA_PKT_NOP_HEADER_COUNT(count - 1));
793	else
794	amdgpu_ring_write(ring, v: ring->funcs->nop);
795	}
796
797	/**
798	* sdma_v4_0_ring_emit_ib - Schedule an IB on the DMA engine
799	*
800	* @ring: amdgpu ring pointer
801	* @job: job to retrieve vmid from
802	* @ib: IB object to schedule
803	* @flags: unused
804	*
805	* Schedule an IB in the DMA ring (VEGA10).
806	*/
807	static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
808	struct amdgpu_job *job,
809	struct amdgpu_ib *ib,
810	uint32_t flags)
811	{
812	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
813
814	/* IB packet must end on a 8 DW boundary */
815	sdma_v4_0_ring_insert_nop(ring, count: (2 - lower_32_bits(ring->wptr)) & 7);
816
817	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
818	SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
819	/* base must be 32 byte aligned */
820	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
821	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
822	amdgpu_ring_write(ring, v: ib->length_dw);
823	amdgpu_ring_write(ring, v: 0);
824	amdgpu_ring_write(ring, v: 0);
825
826	}
827
828	static void sdma_v4_0_wait_reg_mem(struct amdgpu_ring *ring,
829	int mem_space, int hdp,
830	uint32_t addr0, uint32_t addr1,
831	uint32_t ref, uint32_t mask,
832	uint32_t inv)
833	{
834	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
835	SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp) |
836	SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space) |
837	SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
838	if (mem_space) {
839	/* memory */
840	amdgpu_ring_write(ring, v: addr0);
841	amdgpu_ring_write(ring, v: addr1);
842	} else {
843	/* registers */
844	amdgpu_ring_write(ring, v: addr0 << 2);
845	amdgpu_ring_write(ring, v: addr1 << 2);
846	}
847	amdgpu_ring_write(ring, v: ref); /* reference */
848	amdgpu_ring_write(ring, v: mask); /* mask */
849	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
850	SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)); /* retry count, poll interval */
851	}
852
853	/**
854	* sdma_v4_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
855	*
856	* @ring: amdgpu ring pointer
857	*
858	* Emit an hdp flush packet on the requested DMA ring.
859	*/
860	static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
861	{
862	struct amdgpu_device *adev = ring->adev;
863	u32 ref_and_mask = 0;
864	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
865
866	ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
867
868	sdma_v4_0_wait_reg_mem(ring, mem_space: 0, hdp: 1,
869	addr0: adev->nbio.funcs->get_hdp_flush_done_offset(adev),
870	addr1: adev->nbio.funcs->get_hdp_flush_req_offset(adev),
871	ref: ref_and_mask, mask: ref_and_mask, inv: 10);
872	}
873
874	/**
875	* sdma_v4_0_ring_emit_fence - emit a fence on the DMA ring
876	*
877	* @ring: amdgpu ring pointer
878	* @addr: address
879	* @seq: sequence number
880	* @flags: fence related flags
881	*
882	* Add a DMA fence packet to the ring to write
883	* the fence seq number and DMA trap packet to generate
884	* an interrupt if needed (VEGA10).
885	*/
886	static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
887	unsigned flags)
888	{
889	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
890	/* write the fence */
891	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
892	/* zero in first two bits */
893	BUG_ON(addr & 0x3);
894	amdgpu_ring_write(ring, lower_32_bits(addr));
895	amdgpu_ring_write(ring, upper_32_bits(addr));
896	amdgpu_ring_write(ring, lower_32_bits(seq));
897
898	/* optionally write high bits as well */
899	if (write64bit) {
900	addr += 4;
901	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
902	/* zero in first two bits */
903	BUG_ON(addr & 0x3);
904	amdgpu_ring_write(ring, lower_32_bits(addr));
905	amdgpu_ring_write(ring, upper_32_bits(addr));
906	amdgpu_ring_write(ring, upper_32_bits(seq));
907	}
908
909	/* generate an interrupt */
910	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
911	amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
912	}
913
914
915	/**
916	* sdma_v4_0_gfx_enable - enable the gfx async dma engines
917	*
918	* @adev: amdgpu_device pointer
919	* @enable: enable SDMA RB/IB
920	* control the gfx async dma ring buffers (VEGA10).
921	*/
922	static void sdma_v4_0_gfx_enable(struct amdgpu_device *adev, bool enable)
923	{
924	u32 rb_cntl, ib_cntl;
925	int i;
926
927	for (i = 0; i < adev->sdma.num_instances; i++) {
928	rb_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL);
929	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, enable ? 1 : 0);
930	WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl);
931	ib_cntl = RREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL);
932	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, enable ? 1 : 0);
933	WREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL, ib_cntl);
934	}
935	}
936
937	/**
938	* sdma_v4_0_rlc_stop - stop the compute async dma engines
939	*
940	* @adev: amdgpu_device pointer
941	*
942	* Stop the compute async dma queues (VEGA10).
943	*/
944	static void sdma_v4_0_rlc_stop(struct amdgpu_device *adev)
945	{
946	/* XXX todo */
947	}
948
949	/**
950	* sdma_v4_0_page_stop - stop the page async dma engines
951	*
952	* @adev: amdgpu_device pointer
953	*
954	* Stop the page async dma ring buffers (VEGA10).
955	*/
956	static void sdma_v4_0_page_stop(struct amdgpu_device *adev)
957	{
958	u32 rb_cntl, ib_cntl;
959	int i;
960
961	for (i = 0; i < adev->sdma.num_instances; i++) {
962	rb_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL);
963	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL,
964	RB_ENABLE, 0);
965	WREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL, rb_cntl);
966	ib_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL);
967	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_PAGE_IB_CNTL,
968	IB_ENABLE, 0);
969	WREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL, ib_cntl);
970	}
971	}
972
973	/**
974	* sdma_v4_0_ctx_switch_enable - stop the async dma engines context switch
975	*
976	* @adev: amdgpu_device pointer
977	* @enable: enable/disable the DMA MEs context switch.
978	*
979	* Halt or unhalt the async dma engines context switch (VEGA10).
980	*/
981	static void sdma_v4_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
982	{
983	u32 f32_cntl, phase_quantum = 0;
984	int i;
985
986	if (amdgpu_sdma_phase_quantum) {
987	unsigned value = amdgpu_sdma_phase_quantum;
988	unsigned unit = 0;
989
990	while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
991	SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {
992	value = (value + 1) >> 1;
993	unit++;
994	}
995	if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
996	SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {
997	value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
998	SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);
999	unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
1000	SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);
1001	WARN_ONCE(1,
1002	"clamping sdma_phase_quantum to %uK clock cycles\n",
1003	value << unit);
1004	}
1005	phase_quantum =
1006	value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |
1007	unit << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;
1008	}
1009
1010	for (i = 0; i < adev->sdma.num_instances; i++) {
1011	f32_cntl = RREG32_SDMA(i, mmSDMA0_CNTL);
1012	f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
1013	AUTO_CTXSW_ENABLE, enable ? 1 : 0);
1014	if (enable && amdgpu_sdma_phase_quantum) {
1015	WREG32_SDMA(i, mmSDMA0_PHASE0_QUANTUM, phase_quantum);
1016	WREG32_SDMA(i, mmSDMA0_PHASE1_QUANTUM, phase_quantum);
1017	WREG32_SDMA(i, mmSDMA0_PHASE2_QUANTUM, phase_quantum);
1018	}
1019	WREG32_SDMA(i, mmSDMA0_CNTL, f32_cntl);
1020
1021	/*
1022	* Enable SDMA utilization. Its only supported on
1023	* Arcturus for the moment and firmware version 14
1024	* and above.
1025	*/
1026	if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) ==
1027	IP_VERSION(4, 2, 2) &&
1028	adev->sdma.instance[i].fw_version >= 14)
1029	WREG32_SDMA(i, mmSDMA0_PUB_DUMMY_REG2, enable);
1030	/* Extend page fault timeout to avoid interrupt storm */
1031	WREG32_SDMA(i, mmSDMA0_UTCL1_TIMEOUT, 0x00800080);
1032	}
1033
1034	}
1035
1036	/**
1037	* sdma_v4_0_enable - stop the async dma engines
1038	*
1039	* @adev: amdgpu_device pointer
1040	* @enable: enable/disable the DMA MEs.
1041	*
1042	* Halt or unhalt the async dma engines (VEGA10).
1043	*/
1044	static void sdma_v4_0_enable(struct amdgpu_device *adev, bool enable)
1045	{
1046	u32 f32_cntl;
1047	int i;
1048
1049	if (!enable) {
1050	sdma_v4_0_gfx_enable(adev, enable);
1051	sdma_v4_0_rlc_stop(adev);
1052	if (adev->sdma.has_page_queue)
1053	sdma_v4_0_page_stop(adev);
1054	}
1055
1056	for (i = 0; i < adev->sdma.num_instances; i++) {
1057	f32_cntl = RREG32_SDMA(i, mmSDMA0_F32_CNTL);
1058	f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1);
1059	WREG32_SDMA(i, mmSDMA0_F32_CNTL, f32_cntl);
1060	}
1061	}
1062
1063	/*
1064	* sdma_v4_0_rb_cntl - get parameters for rb_cntl
1065	*/
1066	static uint32_t sdma_v4_0_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl)
1067	{
1068	/* Set ring buffer size in dwords */
1069	uint32_t rb_bufsz = order_base_2(ring->ring_size / 4);
1070
1071	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
1072	#ifdef __BIG_ENDIAN
1073	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
1074	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,
1075	RPTR_WRITEBACK_SWAP_ENABLE, 1);
1076	#endif
1077	return rb_cntl;
1078	}
1079
1080	/**
1081	* sdma_v4_0_gfx_resume - setup and start the async dma engines
1082	*
1083	* @adev: amdgpu_device pointer
1084	* @i: instance to resume
1085	*
1086	* Set up the gfx DMA ring buffers and enable them (VEGA10).
1087	* Returns 0 for success, error for failure.
1088	*/
1089	static void sdma_v4_0_gfx_resume(struct amdgpu_device *adev, unsigned int i)
1090	{
1091	struct amdgpu_ring *ring = &adev->sdma.instance[i].ring;
1092	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
1093	u32 doorbell;
1094	u32 doorbell_offset;
1095	u64 wptr_gpu_addr;
1096
1097	rb_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL);
1098	rb_cntl = sdma_v4_0_rb_cntl(ring, rb_cntl);
1099	WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl);
1100
1101	/* Initialize the ring buffer's read and write pointers */
1102	WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR, 0);
1103	WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR_HI, 0);
1104	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR, 0);
1105	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_HI, 0);
1106
1107	/* set the wb address whether it's enabled or not */
1108	WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR_ADDR_HI,
1109	upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
1110	WREG32_SDMA(i, mmSDMA0_GFX_RB_RPTR_ADDR_LO,
1111	lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
1112
1113	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,
1114	RPTR_WRITEBACK_ENABLE, 1);
1115
1116	WREG32_SDMA(i, mmSDMA0_GFX_RB_BASE, ring->gpu_addr >> 8);
1117	WREG32_SDMA(i, mmSDMA0_GFX_RB_BASE_HI, ring->gpu_addr >> 40);
1118
1119	ring->wptr = 0;
1120
1121	/* before programing wptr to a less value, need set minor_ptr_update first */
1122	WREG32_SDMA(i, mmSDMA0_GFX_MINOR_PTR_UPDATE, 1);
1123
1124	doorbell = RREG32_SDMA(i, mmSDMA0_GFX_DOORBELL);
1125	doorbell_offset = RREG32_SDMA(i, mmSDMA0_GFX_DOORBELL_OFFSET);
1126
1127	doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE,
1128	ring->use_doorbell);
1129	doorbell_offset = REG_SET_FIELD(doorbell_offset,
1130	SDMA0_GFX_DOORBELL_OFFSET,
1131	OFFSET, ring->doorbell_index);
1132	WREG32_SDMA(i, mmSDMA0_GFX_DOORBELL, doorbell);
1133	WREG32_SDMA(i, mmSDMA0_GFX_DOORBELL_OFFSET, doorbell_offset);
1134
1135	sdma_v4_0_ring_set_wptr(ring);
1136
1137	/* set minor_ptr_update to 0 after wptr programed */
1138	WREG32_SDMA(i, mmSDMA0_GFX_MINOR_PTR_UPDATE, 0);
1139
1140	/* setup the wptr shadow polling */
1141	wptr_gpu_addr = ring->wptr_gpu_addr;
1142	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO,
1143	lower_32_bits(wptr_gpu_addr));
1144	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI,
1145	upper_32_bits(wptr_gpu_addr));
1146	wptr_poll_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL);
1147	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
1148	SDMA0_GFX_RB_WPTR_POLL_CNTL,
1149	F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
1150	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
1151
1152	/* enable DMA RB */
1153	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);
1154	WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl);
1155
1156	ib_cntl = RREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL);
1157	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1);
1158	#ifdef __BIG_ENDIAN
1159	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
1160	#endif
1161	/* enable DMA IBs */
1162	WREG32_SDMA(i, mmSDMA0_GFX_IB_CNTL, ib_cntl);
1163	}
1164
1165	/**
1166	* sdma_v4_0_page_resume - setup and start the async dma engines
1167	*
1168	* @adev: amdgpu_device pointer
1169	* @i: instance to resume
1170	*
1171	* Set up the page DMA ring buffers and enable them (VEGA10).
1172	* Returns 0 for success, error for failure.
1173	*/
1174	static void sdma_v4_0_page_resume(struct amdgpu_device *adev, unsigned int i)
1175	{
1176	struct amdgpu_ring *ring = &adev->sdma.instance[i].page;
1177	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
1178	u32 doorbell;
1179	u32 doorbell_offset;
1180	u64 wptr_gpu_addr;
1181
1182	rb_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL);
1183	rb_cntl = sdma_v4_0_rb_cntl(ring, rb_cntl);
1184	WREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL, rb_cntl);
1185
1186	/* Initialize the ring buffer's read and write pointers */
1187	WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR, 0);
1188	WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR_HI, 0);
1189	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR, 0);
1190	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_HI, 0);
1191
1192	/* set the wb address whether it's enabled or not */
1193	WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR_ADDR_HI,
1194	upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
1195	WREG32_SDMA(i, mmSDMA0_PAGE_RB_RPTR_ADDR_LO,
1196	lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
1197
1198	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL,
1199	RPTR_WRITEBACK_ENABLE, 1);
1200
1201	WREG32_SDMA(i, mmSDMA0_PAGE_RB_BASE, ring->gpu_addr >> 8);
1202	WREG32_SDMA(i, mmSDMA0_PAGE_RB_BASE_HI, ring->gpu_addr >> 40);
1203
1204	ring->wptr = 0;
1205
1206	/* before programing wptr to a less value, need set minor_ptr_update first */
1207	WREG32_SDMA(i, mmSDMA0_PAGE_MINOR_PTR_UPDATE, 1);
1208
1209	doorbell = RREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL);
1210	doorbell_offset = RREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL_OFFSET);
1211
1212	doorbell = REG_SET_FIELD(doorbell, SDMA0_PAGE_DOORBELL, ENABLE,
1213	ring->use_doorbell);
1214	doorbell_offset = REG_SET_FIELD(doorbell_offset,
1215	SDMA0_PAGE_DOORBELL_OFFSET,
1216	OFFSET, ring->doorbell_index);
1217	WREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL, doorbell);
1218	WREG32_SDMA(i, mmSDMA0_PAGE_DOORBELL_OFFSET, doorbell_offset);
1219
1220	/* paging queue doorbell range is setup at sdma_v4_0_gfx_resume */
1221	sdma_v4_0_page_ring_set_wptr(ring);
1222
1223	/* set minor_ptr_update to 0 after wptr programed */
1224	WREG32_SDMA(i, mmSDMA0_PAGE_MINOR_PTR_UPDATE, 0);
1225
1226	/* setup the wptr shadow polling */
1227	wptr_gpu_addr = ring->wptr_gpu_addr;
1228	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_LO,
1229	lower_32_bits(wptr_gpu_addr));
1230	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_HI,
1231	upper_32_bits(wptr_gpu_addr));
1232	wptr_poll_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL);
1233	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
1234	SDMA0_PAGE_RB_WPTR_POLL_CNTL,
1235	F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
1236	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
1237
1238	/* enable DMA RB */
1239	rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL, RB_ENABLE, 1);
1240	WREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL, rb_cntl);
1241
1242	ib_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL);
1243	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_PAGE_IB_CNTL, IB_ENABLE, 1);
1244	#ifdef __BIG_ENDIAN
1245	ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_PAGE_IB_CNTL, IB_SWAP_ENABLE, 1);
1246	#endif
1247	/* enable DMA IBs */
1248	WREG32_SDMA(i, mmSDMA0_PAGE_IB_CNTL, ib_cntl);
1249	}
1250
1251	static void
1252	sdma_v4_1_update_power_gating(struct amdgpu_device *adev, bool enable)
1253	{
1254	uint32_t def, data;
1255
1256	if (enable && (adev->pg_flags & AMD_PG_SUPPORT_SDMA)) {
1257	/* enable idle interrupt */
1258	def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL));
1259	data |= SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK;
1260
1261	if (data != def)
1262	WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL), data);
1263	} else {
1264	/* disable idle interrupt */
1265	def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL));
1266	data &= ~SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK;
1267	if (data != def)
1268	WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL), data);
1269	}
1270	}
1271
1272	static void sdma_v4_1_init_power_gating(struct amdgpu_device *adev)
1273	{
1274	uint32_t def, data;
1275
1276	/* Enable HW based PG. */
1277	def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL));
1278	data |= SDMA0_POWER_CNTL__PG_CNTL_ENABLE_MASK;
1279	if (data != def)
1280	WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), data);
1281
1282	/* enable interrupt */
1283	def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL));
1284	data |= SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK;
1285	if (data != def)
1286	WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CNTL), data);
1287
1288	/* Configure hold time to filter in-valid power on/off request. Use default right now */
1289	def = data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL));
1290	data &= ~SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK;
1291	data |= (mmSDMA0_POWER_CNTL_DEFAULT & SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK);
1292	/* Configure switch time for hysteresis purpose. Use default right now */
1293	data &= ~SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK;
1294	data |= (mmSDMA0_POWER_CNTL_DEFAULT & SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK);
1295	if(data != def)
1296	WREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), data);
1297	}
1298
1299	static void sdma_v4_0_init_pg(struct amdgpu_device *adev)
1300	{
1301	if (!(adev->pg_flags & AMD_PG_SUPPORT_SDMA))
1302	return;
1303
1304	switch (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0)) {
1305	case IP_VERSION(4, 1, 0):
1306	case IP_VERSION(4, 1, 1):
1307	case IP_VERSION(4, 1, 2):
1308	sdma_v4_1_init_power_gating(adev);
1309	sdma_v4_1_update_power_gating(adev, enable: true);
1310	break;
1311	default:
1312	break;
1313	}
1314	}
1315
1316	/**
1317	* sdma_v4_0_rlc_resume - setup and start the async dma engines
1318	*
1319	* @adev: amdgpu_device pointer
1320	*
1321	* Set up the compute DMA queues and enable them (VEGA10).
1322	* Returns 0 for success, error for failure.
1323	*/
1324	static int sdma_v4_0_rlc_resume(struct amdgpu_device *adev)
1325	{
1326	sdma_v4_0_init_pg(adev);
1327
1328	return 0;
1329	}
1330
1331	/**
1332	* sdma_v4_0_load_microcode - load the sDMA ME ucode
1333	*
1334	* @adev: amdgpu_device pointer
1335	*
1336	* Loads the sDMA0/1 ucode.
1337	* Returns 0 for success, -EINVAL if the ucode is not available.
1338	*/
1339	static int sdma_v4_0_load_microcode(struct amdgpu_device *adev)
1340	{
1341	const struct sdma_firmware_header_v1_0 *hdr;
1342	const __le32 *fw_data;
1343	u32 fw_size;
1344	int i, j;
1345
1346	/* halt the MEs */
1347	sdma_v4_0_enable(adev, enable: false);
1348
1349	for (i = 0; i < adev->sdma.num_instances; i++) {
1350	if (!adev->sdma.instance[i].fw)
1351	return -EINVAL;
1352
1353	hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
1354	amdgpu_ucode_print_sdma_hdr(hdr: &hdr->header);
1355	fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1356
1357	fw_data = (const __le32 *)
1358	(adev->sdma.instance[i].fw->data +
1359	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1360
1361	WREG32_SDMA(i, mmSDMA0_UCODE_ADDR, 0);
1362
1363	for (j = 0; j < fw_size; j++)
1364	WREG32_SDMA(i, mmSDMA0_UCODE_DATA,
1365	le32_to_cpup(fw_data++));
1366
1367	WREG32_SDMA(i, mmSDMA0_UCODE_ADDR,
1368	adev->sdma.instance[i].fw_version);
1369	}
1370
1371	return 0;
1372	}
1373
1374	/**
1375	* sdma_v4_0_start - setup and start the async dma engines
1376	*
1377	* @adev: amdgpu_device pointer
1378	*
1379	* Set up the DMA engines and enable them (VEGA10).
1380	* Returns 0 for success, error for failure.
1381	*/
1382	static int sdma_v4_0_start(struct amdgpu_device *adev)
1383	{
1384	struct amdgpu_ring *ring;
1385	int i, r = 0;
1386
1387	if (amdgpu_sriov_vf(adev)) {
1388	sdma_v4_0_ctx_switch_enable(adev, enable: false);
1389	sdma_v4_0_enable(adev, enable: false);
1390	} else {
1391
1392	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
1393	r = sdma_v4_0_load_microcode(adev);
1394	if (r)
1395	return r;
1396	}
1397
1398	/* unhalt the MEs */
1399	sdma_v4_0_enable(adev, enable: true);
1400	/* enable sdma ring preemption */
1401	sdma_v4_0_ctx_switch_enable(adev, enable: true);
1402	}
1403
1404	/* start the gfx rings and rlc compute queues */
1405	for (i = 0; i < adev->sdma.num_instances; i++) {
1406	uint32_t temp;
1407
1408	WREG32_SDMA(i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL, 0);
1409	sdma_v4_0_gfx_resume(adev, i);
1410	if (adev->sdma.has_page_queue)
1411	sdma_v4_0_page_resume(adev, i);
1412
1413	/* set utc l1 enable flag always to 1 */
1414	temp = RREG32_SDMA(i, mmSDMA0_CNTL);
1415	temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1);
1416	WREG32_SDMA(i, mmSDMA0_CNTL, temp);
1417
1418	if (!amdgpu_sriov_vf(adev)) {
1419	/* unhalt engine */
1420	temp = RREG32_SDMA(i, mmSDMA0_F32_CNTL);
1421	temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0);
1422	WREG32_SDMA(i, mmSDMA0_F32_CNTL, temp);
1423	}
1424	}
1425
1426	if (amdgpu_sriov_vf(adev)) {
1427	sdma_v4_0_ctx_switch_enable(adev, enable: true);
1428	sdma_v4_0_enable(adev, enable: true);
1429	} else {
1430	r = sdma_v4_0_rlc_resume(adev);
1431	if (r)
1432	return r;
1433	}
1434
1435	for (i = 0; i < adev->sdma.num_instances; i++) {
1436	ring = &adev->sdma.instance[i].ring;
1437
1438	r = amdgpu_ring_test_helper(ring);
1439	if (r)
1440	return r;
1441
1442	if (adev->sdma.has_page_queue) {
1443	struct amdgpu_ring *page = &adev->sdma.instance[i].page;
1444
1445	r = amdgpu_ring_test_helper(ring: page);
1446	if (r)
1447	return r;
1448	}
1449	}
1450
1451	return r;
1452	}
1453
1454	/**
1455	* sdma_v4_0_ring_test_ring - simple async dma engine test
1456	*
1457	* @ring: amdgpu_ring structure holding ring information
1458	*
1459	* Test the DMA engine by writing using it to write an
1460	* value to memory. (VEGA10).
1461	* Returns 0 for success, error for failure.
1462	*/
1463	static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
1464	{
1465	struct amdgpu_device *adev = ring->adev;
1466	unsigned i;
1467	unsigned index;
1468	int r;
1469	u32 tmp;
1470	u64 gpu_addr;
1471
1472	r = amdgpu_device_wb_get(adev, wb: &index);
1473	if (r)
1474	return r;
1475
1476	gpu_addr = adev->wb.gpu_addr + (index * 4);
1477	tmp = 0xCAFEDEAD;
1478	adev->wb.wb[index] = cpu_to_le32(tmp);
1479
1480	r = amdgpu_ring_alloc(ring, ndw: 5);
1481	if (r)
1482	goto error_free_wb;
1483
1484	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1485	SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
1486	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
1487	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
1488	amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
1489	amdgpu_ring_write(ring, v: 0xDEADBEEF);
1490	amdgpu_ring_commit(ring);
1491
1492	for (i = 0; i < adev->usec_timeout; i++) {
1493	tmp = le32_to_cpu(adev->wb.wb[index]);
1494	if (tmp == 0xDEADBEEF)
1495	break;
1496	udelay(usec: 1);
1497	}
1498
1499	if (i >= adev->usec_timeout)
1500	r = -ETIMEDOUT;
1501
1502	error_free_wb:
1503	amdgpu_device_wb_free(adev, wb: index);
1504	return r;
1505	}
1506
1507	/**
1508	* sdma_v4_0_ring_test_ib - test an IB on the DMA engine
1509	*
1510	* @ring: amdgpu_ring structure holding ring information
1511	* @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
1512	*
1513	* Test a simple IB in the DMA ring (VEGA10).
1514	* Returns 0 on success, error on failure.
1515	*/
1516	static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1517	{
1518	struct amdgpu_device *adev = ring->adev;
1519	struct amdgpu_ib ib;
1520	struct dma_fence *f = NULL;
1521	unsigned index;
1522	long r;
1523	u32 tmp = 0;
1524	u64 gpu_addr;
1525
1526	r = amdgpu_device_wb_get(adev, wb: &index);
1527	if (r)
1528	return r;
1529
1530	gpu_addr = adev->wb.gpu_addr + (index * 4);
1531	tmp = 0xCAFEDEAD;
1532	adev->wb.wb[index] = cpu_to_le32(tmp);
1533	memset(&ib, 0, sizeof(ib));
1534	r = amdgpu_ib_get(adev, NULL, size: 256,
1535	pool: AMDGPU_IB_POOL_DIRECT, ib: &ib);
1536	if (r)
1537	goto err0;
1538
1539	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1540	SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1541	ib.ptr[1] = lower_32_bits(gpu_addr);
1542	ib.ptr[2] = upper_32_bits(gpu_addr);
1543	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1544	ib.ptr[4] = 0xDEADBEEF;
1545	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1546	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1547	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1548	ib.length_dw = 8;
1549
1550	r = amdgpu_ib_schedule(ring, num_ibs: 1, ibs: &ib, NULL, f: &f);
1551	if (r)
1552	goto err1;
1553
1554	r = dma_fence_wait_timeout(f, intr: false, timeout);
1555	if (r == 0) {
1556	r = -ETIMEDOUT;
1557	goto err1;
1558	} else if (r < 0) {
1559	goto err1;
1560	}
1561	tmp = le32_to_cpu(adev->wb.wb[index]);
1562	if (tmp == 0xDEADBEEF)
1563	r = 0;
1564	else
1565	r = -EINVAL;
1566
1567	err1:
1568	amdgpu_ib_free(ib: &ib, NULL);
1569	dma_fence_put(fence: f);
1570	err0:
1571	amdgpu_device_wb_free(adev, wb: index);
1572	return r;
1573	}
1574
1575
1576	/**
1577	* sdma_v4_0_vm_copy_pte - update PTEs by copying them from the GART
1578	*
1579	* @ib: indirect buffer to fill with commands
1580	* @pe: addr of the page entry
1581	* @src: src addr to copy from
1582	* @count: number of page entries to update
1583	*
1584	* Update PTEs by copying them from the GART using sDMA (VEGA10).
1585	*/
1586	static void sdma_v4_0_vm_copy_pte(struct amdgpu_ib *ib,
1587	uint64_t pe, uint64_t src,
1588	unsigned count)
1589	{
1590	unsigned bytes = count * 8;
1591
1592	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1593	SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1594	ib->ptr[ib->length_dw++] = bytes - 1;
1595	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1596	ib->ptr[ib->length_dw++] = lower_32_bits(src);
1597	ib->ptr[ib->length_dw++] = upper_32_bits(src);
1598	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1599	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1600
1601	}
1602
1603	/**
1604	* sdma_v4_0_vm_write_pte - update PTEs by writing them manually
1605	*
1606	* @ib: indirect buffer to fill with commands
1607	* @pe: addr of the page entry
1608	* @value: dst addr to write into pe
1609	* @count: number of page entries to update
1610	* @incr: increase next addr by incr bytes
1611	*
1612	* Update PTEs by writing them manually using sDMA (VEGA10).
1613	*/
1614	static void sdma_v4_0_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1615	uint64_t value, unsigned count,
1616	uint32_t incr)
1617	{
1618	unsigned ndw = count * 2;
1619
1620	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1621	SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1622	ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1623	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1624	ib->ptr[ib->length_dw++] = ndw - 1;
1625	for (; ndw > 0; ndw -= 2) {
1626	ib->ptr[ib->length_dw++] = lower_32_bits(value);
1627	ib->ptr[ib->length_dw++] = upper_32_bits(value);
1628	value += incr;
1629	}
1630	}
1631
1632	/**
1633	* sdma_v4_0_vm_set_pte_pde - update the page tables using sDMA
1634	*
1635	* @ib: indirect buffer to fill with commands
1636	* @pe: addr of the page entry
1637	* @addr: dst addr to write into pe
1638	* @count: number of page entries to update
1639	* @incr: increase next addr by incr bytes
1640	* @flags: access flags
1641	*
1642	* Update the page tables using sDMA (VEGA10).
1643	*/
1644	static void sdma_v4_0_vm_set_pte_pde(struct amdgpu_ib *ib,
1645	uint64_t pe,
1646	uint64_t addr, unsigned count,
1647	uint32_t incr, uint64_t flags)
1648	{
1649	/* for physically contiguous pages (vram) */
1650	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
1651	ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1652	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1653	ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1654	ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1655	ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1656	ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1657	ib->ptr[ib->length_dw++] = incr; /* increment size */
1658	ib->ptr[ib->length_dw++] = 0;
1659	ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1660	}
1661
1662	/**
1663	* sdma_v4_0_ring_pad_ib - pad the IB to the required number of dw
1664	*
1665	* @ring: amdgpu_ring structure holding ring information
1666	* @ib: indirect buffer to fill with padding
1667	*/
1668	static void sdma_v4_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1669	{
1670	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1671	u32 pad_count;
1672	int i;
1673
1674	pad_count = (-ib->length_dw) & 7;
1675	for (i = 0; i < pad_count; i++)
1676	if (sdma && sdma->burst_nop && (i == 0))
1677	ib->ptr[ib->length_dw++] =
1678	SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
1679	SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1680	else
1681	ib->ptr[ib->length_dw++] =
1682	SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
1683	}
1684
1685
1686	/**
1687	* sdma_v4_0_ring_emit_pipeline_sync - sync the pipeline
1688	*
1689	* @ring: amdgpu_ring pointer
1690	*
1691	* Make sure all previous operations are completed (CIK).
1692	*/
1693	static void sdma_v4_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1694	{
1695	uint32_t seq = ring->fence_drv.sync_seq;
1696	uint64_t addr = ring->fence_drv.gpu_addr;
1697
1698	/* wait for idle */
1699	sdma_v4_0_wait_reg_mem(ring, mem_space: 1, hdp: 0,
1700	addr0: addr & 0xfffffffc,
1701	upper_32_bits(addr) & 0xffffffff,
1702	ref: seq, mask: 0xffffffff, inv: 4);
1703	}
1704
1705
1706	/**
1707	* sdma_v4_0_ring_emit_vm_flush - vm flush using sDMA
1708	*
1709	* @ring: amdgpu_ring pointer
1710	* @vmid: vmid number to use
1711	* @pd_addr: address
1712	*
1713	* Update the page table base and flush the VM TLB
1714	* using sDMA (VEGA10).
1715	*/
1716	static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
1717	unsigned vmid, uint64_t pd_addr)
1718	{
1719	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1720	}
1721
1722	static void sdma_v4_0_ring_emit_wreg(struct amdgpu_ring *ring,
1723	uint32_t reg, uint32_t val)
1724	{
1725	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1726	SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1727	amdgpu_ring_write(ring, v: reg);
1728	amdgpu_ring_write(ring, v: val);
1729	}
1730
1731	static void sdma_v4_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1732	uint32_t val, uint32_t mask)
1733	{
1734	sdma_v4_0_wait_reg_mem(ring, mem_space: 0, hdp: 0, addr0: reg, addr1: 0, ref: val, mask, inv: 10);
1735	}
1736
1737	static bool sdma_v4_0_fw_support_paging_queue(struct amdgpu_device *adev)
1738	{
1739	uint fw_version = adev->sdma.instance[0].fw_version;
1740
1741	switch (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0)) {
1742	case IP_VERSION(4, 0, 0):
1743	return fw_version >= 430;
1744	case IP_VERSION(4, 0, 1):
1745	/*return fw_version >= 31;*/
1746	return false;
1747	case IP_VERSION(4, 2, 0):
1748	return fw_version >= 123;
1749	default:
1750	return false;
1751	}
1752	}
1753
1754	static int sdma_v4_0_early_init(struct amdgpu_ip_block *ip_block)
1755	{
1756	struct amdgpu_device *adev = ip_block->adev;
1757	int r;
1758
1759	r = sdma_v4_0_init_microcode(adev);
1760	if (r)
1761	return r;
1762
1763	/* TODO: Page queue breaks driver reload under SRIOV */
1764	if ((amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) == IP_VERSION(4, 0, 0)) &&
1765	amdgpu_sriov_vf((adev)))
1766	adev->sdma.has_page_queue = false;
1767	else if (sdma_v4_0_fw_support_paging_queue(adev))
1768	adev->sdma.has_page_queue = true;
1769
1770	sdma_v4_0_set_ring_funcs(adev);
1771	sdma_v4_0_set_buffer_funcs(adev);
1772	sdma_v4_0_set_vm_pte_funcs(adev);
1773	sdma_v4_0_set_irq_funcs(adev);
1774	sdma_v4_0_set_ras_funcs(adev);
1775
1776	return 0;
1777	}
1778
1779	static int sdma_v4_0_process_ras_data_cb(struct amdgpu_device *adev,
1780	void *err_data,
1781	struct amdgpu_iv_entry *entry);
1782
1783	static int sdma_v4_0_late_init(struct amdgpu_ip_block *ip_block)
1784	{
1785	struct amdgpu_device *adev = ip_block->adev;
1786
1787	sdma_v4_0_setup_ulv(adev);
1788
1789	if (!amdgpu_persistent_edc_harvesting_supported(adev))
1790	amdgpu_ras_reset_error_count(adev, block: AMDGPU_RAS_BLOCK__SDMA);
1791
1792	return 0;
1793	}
1794
1795	static int sdma_v4_0_sw_init(struct amdgpu_ip_block *ip_block)
1796	{
1797	struct amdgpu_ring *ring;
1798	int r, i;
1799	struct amdgpu_device *adev = ip_block->adev;
1800	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_0);
1801	uint32_t *ptr;
1802
1803	/* SDMA trap event */
1804	for (i = 0; i < adev->sdma.num_instances; i++) {
1805	r = amdgpu_irq_add_id(adev, client_id: sdma_v4_0_seq_to_irq_id(seq_num: i),
1806	SDMA0_4_0__SRCID__SDMA_TRAP,
1807	source: &adev->sdma.trap_irq);
1808	if (r)
1809	return r;
1810	}
1811
1812	/* SDMA SRAM ECC event */
1813	for (i = 0; i < adev->sdma.num_instances; i++) {
1814	r = amdgpu_irq_add_id(adev, client_id: sdma_v4_0_seq_to_irq_id(seq_num: i),
1815	SDMA0_4_0__SRCID__SDMA_SRAM_ECC,
1816	source: &adev->sdma.ecc_irq);
1817	if (r)
1818	return r;
1819	}
1820
1821	/* SDMA VM_HOLE/DOORBELL_INV/POLL_TIMEOUT/SRBM_WRITE_PROTECTION event*/
1822	for (i = 0; i < adev->sdma.num_instances; i++) {
1823	r = amdgpu_irq_add_id(adev, client_id: sdma_v4_0_seq_to_irq_id(seq_num: i),
1824	SDMA0_4_0__SRCID__SDMA_VM_HOLE,
1825	source: &adev->sdma.vm_hole_irq);
1826	if (r)
1827	return r;
1828
1829	r = amdgpu_irq_add_id(adev, client_id: sdma_v4_0_seq_to_irq_id(seq_num: i),
1830	SDMA0_4_0__SRCID__SDMA_DOORBELL_INVALID,
1831	source: &adev->sdma.doorbell_invalid_irq);
1832	if (r)
1833	return r;
1834
1835	r = amdgpu_irq_add_id(adev, client_id: sdma_v4_0_seq_to_irq_id(seq_num: i),
1836	SDMA0_4_0__SRCID__SDMA_POLL_TIMEOUT,
1837	source: &adev->sdma.pool_timeout_irq);
1838	if (r)
1839	return r;
1840
1841	r = amdgpu_irq_add_id(adev, client_id: sdma_v4_0_seq_to_irq_id(seq_num: i),
1842	SDMA0_4_0__SRCID__SDMA_SRBMWRITE,
1843	source: &adev->sdma.srbm_write_irq);
1844	if (r)
1845	return r;
1846	}
1847
1848	for (i = 0; i < adev->sdma.num_instances; i++) {
1849	ring = &adev->sdma.instance[i].ring;
1850	ring->ring_obj = NULL;
1851	ring->use_doorbell = true;
1852
1853	DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1854	ring->use_doorbell?"true":"false");
1855
1856	/* doorbell size is 2 dwords, get DWORD offset */
1857	ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
1858
1859	/*
1860	* On Arcturus, SDMA instance 5~7 has a different vmhub
1861	* type(AMDGPU_MMHUB1).
1862	*/
1863	if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) ==
1864	IP_VERSION(4, 2, 2) &&
1865	i >= 5)
1866	ring->vm_hub = AMDGPU_MMHUB1(0);
1867	else
1868	ring->vm_hub = AMDGPU_MMHUB0(0);
1869
1870	sprintf(buf: ring->name, fmt: "sdma%d", i);
1871	r = amdgpu_ring_init(adev, ring, max_dw: 1024, irq_src: &adev->sdma.trap_irq,
1872	irq_type: AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1873	hw_prio: AMDGPU_RING_PRIO_DEFAULT, NULL);
1874	if (r)
1875	return r;
1876
1877	if (adev->sdma.has_page_queue) {
1878	ring = &adev->sdma.instance[i].page;
1879	ring->ring_obj = NULL;
1880	ring->use_doorbell = true;
1881
1882	/* paging queue use same doorbell index/routing as gfx queue
1883	* with 0x400 (4096 dwords) offset on second doorbell page
1884	*/
1885	if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) >=
1886	IP_VERSION(4, 0, 0) &&
1887	amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) <
1888	IP_VERSION(4, 2, 0)) {
1889	ring->doorbell_index =
1890	adev->doorbell_index.sdma_engine[i] << 1;
1891	ring->doorbell_index += 0x400;
1892	} else {
1893	/* From vega20, the sdma_doorbell_range in 1st
1894	* doorbell page is reserved for page queue.
1895	*/
1896	ring->doorbell_index =
1897	(adev->doorbell_index.sdma_engine[i] + 1) << 1;
1898	}
1899
1900	if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) ==
1901	IP_VERSION(4, 2, 2) &&
1902	i >= 5)
1903	ring->vm_hub = AMDGPU_MMHUB1(0);
1904	else
1905	ring->vm_hub = AMDGPU_MMHUB0(0);
1906
1907	sprintf(buf: ring->name, fmt: "page%d", i);
1908	r = amdgpu_ring_init(adev, ring, max_dw: 1024,
1909	irq_src: &adev->sdma.trap_irq,
1910	irq_type: AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1911	hw_prio: AMDGPU_RING_PRIO_DEFAULT, NULL);
1912	if (r)
1913	return r;
1914	}
1915	}
1916
1917	if (amdgpu_sdma_ras_sw_init(adev)) {
1918	dev_err(adev->dev, "Failed to initialize sdma ras block!\n");
1919	return -EINVAL;
1920	}
1921
1922	/* Allocate memory for SDMA IP Dump buffer */
1923	ptr = kcalloc(adev->sdma.num_instances * reg_count, sizeof(uint32_t), GFP_KERNEL);
1924	if (ptr)
1925	adev->sdma.ip_dump = ptr;
1926	else
1927	DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n");
1928
1929	return r;
1930	}
1931
1932	static int sdma_v4_0_sw_fini(struct amdgpu_ip_block *ip_block)
1933	{
1934	struct amdgpu_device *adev = ip_block->adev;
1935	int i;
1936
1937	for (i = 0; i < adev->sdma.num_instances; i++) {
1938	amdgpu_ring_fini(ring: &adev->sdma.instance[i].ring);
1939	if (adev->sdma.has_page_queue)
1940	amdgpu_ring_fini(ring: &adev->sdma.instance[i].page);
1941	}
1942
1943	if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) == IP_VERSION(4, 2, 2) ||
1944	amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) == IP_VERSION(4, 4, 0))
1945	amdgpu_sdma_destroy_inst_ctx(adev, duplicate: true);
1946	else
1947	amdgpu_sdma_destroy_inst_ctx(adev, duplicate: false);
1948
1949	kfree(objp: adev->sdma.ip_dump);
1950
1951	return 0;
1952	}
1953
1954	static int sdma_v4_0_hw_init(struct amdgpu_ip_block *ip_block)
1955	{
1956	struct amdgpu_device *adev = ip_block->adev;
1957
1958	if (adev->flags & AMD_IS_APU)
1959	amdgpu_dpm_set_powergating_by_smu(adev, block_type: AMD_IP_BLOCK_TYPE_SDMA, gate: false, inst: 0);
1960
1961	if (!amdgpu_sriov_vf(adev))
1962	sdma_v4_0_init_golden_registers(adev);
1963
1964	return sdma_v4_0_start(adev);
1965	}
1966
1967	static int sdma_v4_0_hw_fini(struct amdgpu_ip_block *ip_block)
1968	{
1969	struct amdgpu_device *adev = ip_block->adev;
1970	int i;
1971
1972	if (amdgpu_sriov_vf(adev))
1973	return 0;
1974
1975	if (amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__SDMA)) {
1976	for (i = 0; i < adev->sdma.num_instances; i++) {
1977	amdgpu_irq_put(adev, src: &adev->sdma.ecc_irq,
1978	type: AMDGPU_SDMA_IRQ_INSTANCE0 + i);
1979	}
1980	}
1981
1982	sdma_v4_0_ctx_switch_enable(adev, enable: false);
1983	sdma_v4_0_enable(adev, enable: false);
1984
1985	if (adev->flags & AMD_IS_APU)
1986	amdgpu_dpm_set_powergating_by_smu(adev, block_type: AMD_IP_BLOCK_TYPE_SDMA, gate: true, inst: 0);
1987
1988	return 0;
1989	}
1990
1991	static int sdma_v4_0_suspend(struct amdgpu_ip_block *ip_block)
1992	{
1993	struct amdgpu_device *adev = ip_block->adev;
1994
1995	/* SMU saves SDMA state for us */
1996	if (adev->in_s0ix) {
1997	sdma_v4_0_gfx_enable(adev, enable: false);
1998	return 0;
1999	}
2000
2001	return sdma_v4_0_hw_fini(ip_block);
2002	}
2003
2004	static int sdma_v4_0_resume(struct amdgpu_ip_block *ip_block)
2005	{
2006	struct amdgpu_device *adev = ip_block->adev;
2007
2008	/* SMU restores SDMA state for us */
2009	if (adev->in_s0ix) {
2010	sdma_v4_0_enable(adev, enable: true);
2011	sdma_v4_0_gfx_enable(adev, enable: true);
2012	return 0;
2013	}
2014
2015	return sdma_v4_0_hw_init(ip_block);
2016	}
2017
2018	static bool sdma_v4_0_is_idle(struct amdgpu_ip_block *ip_block)
2019	{
2020	struct amdgpu_device *adev = ip_block->adev;
2021	u32 i;
2022
2023	for (i = 0; i < adev->sdma.num_instances; i++) {
2024	u32 tmp = RREG32_SDMA(i, mmSDMA0_STATUS_REG);
2025
2026	if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
2027	return false;
2028	}
2029
2030	return true;
2031	}
2032
2033	static int sdma_v4_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
2034	{
2035	unsigned i, j;
2036	u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];
2037	struct amdgpu_device *adev = ip_block->adev;
2038
2039	for (i = 0; i < adev->usec_timeout; i++) {
2040	for (j = 0; j < adev->sdma.num_instances; j++) {
2041	sdma[j] = RREG32_SDMA(j, mmSDMA0_STATUS_REG);
2042	if (!(sdma[j] & SDMA0_STATUS_REG__IDLE_MASK))
2043	break;
2044	}
2045	if (j == adev->sdma.num_instances)
2046	return 0;
2047	udelay(usec: 1);
2048	}
2049	return -ETIMEDOUT;
2050	}
2051
2052	static int sdma_v4_0_soft_reset(struct amdgpu_ip_block *ip_block)
2053	{
2054	/* todo */
2055
2056	return 0;
2057	}
2058
2059	static int sdma_v4_0_set_trap_irq_state(struct amdgpu_device *adev,
2060	struct amdgpu_irq_src *source,
2061	unsigned type,
2062	enum amdgpu_interrupt_state state)
2063	{
2064	u32 sdma_cntl;
2065
2066	sdma_cntl = RREG32_SDMA(type, mmSDMA0_CNTL);
2067	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
2068	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
2069	WREG32_SDMA(type, mmSDMA0_CNTL, sdma_cntl);
2070
2071	return 0;
2072	}
2073
2074	static int sdma_v4_0_process_trap_irq(struct amdgpu_device *adev,
2075	struct amdgpu_irq_src *source,
2076	struct amdgpu_iv_entry *entry)
2077	{
2078	int instance;
2079
2080	DRM_DEBUG("IH: SDMA trap\n");
2081	instance = sdma_v4_0_irq_id_to_seq(client_id: entry->client_id);
2082	if (instance < 0)
2083	return instance;
2084
2085	switch (entry->ring_id) {
2086	case 0:
2087	amdgpu_fence_process(ring: &adev->sdma.instance[instance].ring);
2088	break;
2089	case 1:
2090	if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) ==
2091	IP_VERSION(4, 2, 0))
2092	amdgpu_fence_process(ring: &adev->sdma.instance[instance].page);
2093	break;
2094	case 2:
2095	/* XXX compute */
2096	break;
2097	case 3:
2098	if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) !=
2099	IP_VERSION(4, 2, 0))
2100	amdgpu_fence_process(ring: &adev->sdma.instance[instance].page);
2101	break;
2102	}
2103	return 0;
2104	}
2105
2106	static int sdma_v4_0_process_ras_data_cb(struct amdgpu_device *adev,
2107	void *err_data,
2108	struct amdgpu_iv_entry *entry)
2109	{
2110	int instance;
2111
2112	/* When “Full RAS” is enabled, the per-IP interrupt sources should
2113	* be disabled and the driver should only look for the aggregated
2114	* interrupt via sync flood
2115	*/
2116	if (amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX))
2117	goto out;
2118
2119	instance = sdma_v4_0_irq_id_to_seq(client_id: entry->client_id);
2120	if (instance < 0)
2121	goto out;
2122
2123	amdgpu_sdma_process_ras_data_cb(adev, err_data, entry);
2124
2125	out:
2126	return AMDGPU_RAS_SUCCESS;
2127	}
2128
2129	static int sdma_v4_0_process_illegal_inst_irq(struct amdgpu_device *adev,
2130	struct amdgpu_irq_src *source,
2131	struct amdgpu_iv_entry *entry)
2132	{
2133	int instance;
2134
2135	DRM_ERROR("Illegal instruction in SDMA command stream\n");
2136
2137	instance = sdma_v4_0_irq_id_to_seq(client_id: entry->client_id);
2138	if (instance < 0)
2139	return 0;
2140
2141	switch (entry->ring_id) {
2142	case 0:
2143	drm_sched_fault(sched: &adev->sdma.instance[instance].ring.sched);
2144	break;
2145	}
2146	return 0;
2147	}
2148
2149	static int sdma_v4_0_set_ecc_irq_state(struct amdgpu_device *adev,
2150	struct amdgpu_irq_src *source,
2151	unsigned type,
2152	enum amdgpu_interrupt_state state)
2153	{
2154	u32 sdma_edc_config;
2155
2156	sdma_edc_config = RREG32_SDMA(type, mmSDMA0_EDC_CONFIG);
2157	sdma_edc_config = REG_SET_FIELD(sdma_edc_config, SDMA0_EDC_CONFIG, ECC_INT_ENABLE,
2158	state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
2159	WREG32_SDMA(type, mmSDMA0_EDC_CONFIG, sdma_edc_config);
2160
2161	return 0;
2162	}
2163
2164	static int sdma_v4_0_print_iv_entry(struct amdgpu_device *adev,
2165	struct amdgpu_iv_entry *entry)
2166	{
2167	int instance;
2168	struct amdgpu_task_info *task_info;
2169	u64 addr;
2170
2171	instance = sdma_v4_0_irq_id_to_seq(client_id: entry->client_id);
2172	if (instance < 0 || instance >= adev->sdma.num_instances) {
2173	dev_err(adev->dev, "sdma instance invalid %d\n", instance);
2174	return -EINVAL;
2175	}
2176
2177	addr = (u64)entry->src_data[0] << 12;
2178	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
2179
2180	dev_dbg_ratelimited(adev->dev,
2181	"[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u pasid:%u\n",
2182	instance, addr, entry->src_id, entry->ring_id, entry->vmid,
2183	entry->pasid);
2184
2185	task_info = amdgpu_vm_get_task_info_pasid(adev, pasid: entry->pasid);
2186	if (task_info) {
2187	dev_dbg_ratelimited(adev->dev,
2188	" for process %s pid %d thread %s pid %d\n",
2189	task_info->process_name, task_info->tgid,
2190	task_info->task.comm, task_info->task.pid);
2191	amdgpu_vm_put_task_info(task_info);
2192	}
2193
2194	return 0;
2195	}
2196
2197	static int sdma_v4_0_process_vm_hole_irq(struct amdgpu_device *adev,
2198	struct amdgpu_irq_src *source,
2199	struct amdgpu_iv_entry *entry)
2200	{
2201	dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");
2202	sdma_v4_0_print_iv_entry(adev, entry);
2203	return 0;
2204	}
2205
2206	static int sdma_v4_0_process_doorbell_invalid_irq(struct amdgpu_device *adev,
2207	struct amdgpu_irq_src *source,
2208	struct amdgpu_iv_entry *entry)
2209	{
2210	dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
2211	sdma_v4_0_print_iv_entry(adev, entry);
2212	return 0;
2213	}
2214
2215	static int sdma_v4_0_process_pool_timeout_irq(struct amdgpu_device *adev,
2216	struct amdgpu_irq_src *source,
2217	struct amdgpu_iv_entry *entry)
2218	{
2219	dev_dbg_ratelimited(adev->dev,
2220	"Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");
2221	sdma_v4_0_print_iv_entry(adev, entry);
2222	return 0;
2223	}
2224
2225	static int sdma_v4_0_process_srbm_write_irq(struct amdgpu_device *adev,
2226	struct amdgpu_irq_src *source,
2227	struct amdgpu_iv_entry *entry)
2228	{
2229	dev_dbg_ratelimited(adev->dev,
2230	"SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");
2231	sdma_v4_0_print_iv_entry(adev, entry);
2232	return 0;
2233	}
2234
2235	static void sdma_v4_0_update_medium_grain_clock_gating(
2236	struct amdgpu_device *adev,
2237	bool enable)
2238	{
2239	uint32_t data, def;
2240	int i;
2241
2242	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
2243	for (i = 0; i < adev->sdma.num_instances; i++) {
2244	def = data = RREG32_SDMA(i, mmSDMA0_CLK_CTRL);
2245	data &= ~(SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK |
2246	SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK |
2247	SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK |
2248	SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
2249	SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
2250	SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
2251	SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
2252	SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK);
2253	if (def != data)
2254	WREG32_SDMA(i, mmSDMA0_CLK_CTRL, data);
2255	}
2256	} else {
2257	for (i = 0; i < adev->sdma.num_instances; i++) {
2258	def = data = RREG32_SDMA(i, mmSDMA0_CLK_CTRL);
2259	data |= (SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK |
2260	SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK |
2261	SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK |
2262	SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK |
2263	SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK |
2264	SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK |
2265	SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK |
2266	SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK);
2267	if (def != data)
2268	WREG32_SDMA(i, mmSDMA0_CLK_CTRL, data);
2269	}
2270	}
2271	}
2272
2273
2274	static void sdma_v4_0_update_medium_grain_light_sleep(
2275	struct amdgpu_device *adev,
2276	bool enable)
2277	{
2278	uint32_t data, def;
2279	int i;
2280
2281	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {
2282	for (i = 0; i < adev->sdma.num_instances; i++) {
2283	/* 1-not override: enable sdma mem light sleep */
2284	def = data = RREG32_SDMA(0, mmSDMA0_POWER_CNTL);
2285	data |= SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
2286	if (def != data)
2287	WREG32_SDMA(0, mmSDMA0_POWER_CNTL, data);
2288	}
2289	} else {
2290	for (i = 0; i < adev->sdma.num_instances; i++) {
2291	/* 0-override:disable sdma mem light sleep */
2292	def = data = RREG32_SDMA(0, mmSDMA0_POWER_CNTL);
2293	data &= ~SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
2294	if (def != data)
2295	WREG32_SDMA(0, mmSDMA0_POWER_CNTL, data);
2296	}
2297	}
2298	}
2299
2300	static int sdma_v4_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2301	enum amd_clockgating_state state)
2302	{
2303	struct amdgpu_device *adev = ip_block->adev;
2304
2305	if (amdgpu_sriov_vf(adev))
2306	return 0;
2307
2308	sdma_v4_0_update_medium_grain_clock_gating(adev,
2309	enable: state == AMD_CG_STATE_GATE);
2310	sdma_v4_0_update_medium_grain_light_sleep(adev,
2311	enable: state == AMD_CG_STATE_GATE);
2312	return 0;
2313	}
2314
2315	static int sdma_v4_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
2316	enum amd_powergating_state state)
2317	{
2318	struct amdgpu_device *adev = ip_block->adev;
2319
2320	switch (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0)) {
2321	case IP_VERSION(4, 1, 0):
2322	case IP_VERSION(4, 1, 1):
2323	case IP_VERSION(4, 1, 2):
2324	sdma_v4_1_update_power_gating(adev,
2325	enable: state == AMD_PG_STATE_GATE);
2326	break;
2327	default:
2328	break;
2329	}
2330
2331	return 0;
2332	}
2333
2334	static void sdma_v4_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
2335	{
2336	struct amdgpu_device *adev = ip_block->adev;
2337	int data;
2338
2339	if (amdgpu_sriov_vf(adev))
2340	*flags = 0;
2341
2342	/* AMD_CG_SUPPORT_SDMA_MGCG */
2343	data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL));
2344	if (!(data & SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK))
2345	*flags |= AMD_CG_SUPPORT_SDMA_MGCG;
2346
2347	/* AMD_CG_SUPPORT_SDMA_LS */
2348	data = RREG32(SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL));
2349	if (data & SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
2350	*flags |= AMD_CG_SUPPORT_SDMA_LS;
2351	}
2352
2353	static void sdma_v4_0_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
2354	{
2355	struct amdgpu_device *adev = ip_block->adev;
2356	int i, j;
2357	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_0);
2358	uint32_t instance_offset;
2359
2360	if (!adev->sdma.ip_dump)
2361	return;
2362
2363	drm_printf(p, f: "num_instances:%d\n", adev->sdma.num_instances);
2364	for (i = 0; i < adev->sdma.num_instances; i++) {
2365	instance_offset = i * reg_count;
2366	drm_printf(p, f: "\nInstance:%d\n", i);
2367
2368	for (j = 0; j < reg_count; j++)
2369	drm_printf(p, f: "%-50s \t 0x%08x\n", sdma_reg_list_4_0[j].reg_name,
2370	adev->sdma.ip_dump[instance_offset + j]);
2371	}
2372	}
2373
2374	static void sdma_v4_0_dump_ip_state(struct amdgpu_ip_block *ip_block)
2375	{
2376	struct amdgpu_device *adev = ip_block->adev;
2377	int i, j;
2378	uint32_t instance_offset;
2379	uint32_t reg_count = ARRAY_SIZE(sdma_reg_list_4_0);
2380
2381	if (!adev->sdma.ip_dump)
2382	return;
2383
2384	for (i = 0; i < adev->sdma.num_instances; i++) {
2385	instance_offset = i * reg_count;
2386	for (j = 0; j < reg_count; j++)
2387	adev->sdma.ip_dump[instance_offset + j] =
2388	RREG32(sdma_v4_0_get_reg_offset(adev, i,
2389	sdma_reg_list_4_0[j].reg_offset));
2390	}
2391	}
2392
2393	const struct amd_ip_funcs sdma_v4_0_ip_funcs = {
2394	.name = "sdma_v4_0",
2395	.early_init = sdma_v4_0_early_init,
2396	.late_init = sdma_v4_0_late_init,
2397	.sw_init = sdma_v4_0_sw_init,
2398	.sw_fini = sdma_v4_0_sw_fini,
2399	.hw_init = sdma_v4_0_hw_init,
2400	.hw_fini = sdma_v4_0_hw_fini,
2401	.suspend = sdma_v4_0_suspend,
2402	.resume = sdma_v4_0_resume,
2403	.is_idle = sdma_v4_0_is_idle,
2404	.wait_for_idle = sdma_v4_0_wait_for_idle,
2405	.soft_reset = sdma_v4_0_soft_reset,
2406	.set_clockgating_state = sdma_v4_0_set_clockgating_state,
2407	.set_powergating_state = sdma_v4_0_set_powergating_state,
2408	.get_clockgating_state = sdma_v4_0_get_clockgating_state,
2409	.dump_ip_state = sdma_v4_0_dump_ip_state,
2410	.print_ip_state = sdma_v4_0_print_ip_state,
2411	};
2412
2413	static const struct amdgpu_ring_funcs sdma_v4_0_ring_funcs = {
2414	.type = AMDGPU_RING_TYPE_SDMA,
2415	.align_mask = 0xff,
2416	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
2417	.support_64bit_ptrs = true,
2418	.secure_submission_supported = true,
2419	.get_rptr = sdma_v4_0_ring_get_rptr,
2420	.get_wptr = sdma_v4_0_ring_get_wptr,
2421	.set_wptr = sdma_v4_0_ring_set_wptr,
2422	.emit_frame_size =
2423	6 + /* sdma_v4_0_ring_emit_hdp_flush */
2424	3 + /* hdp invalidate */
2425	6 + /* sdma_v4_0_ring_emit_pipeline_sync */
2426	/* sdma_v4_0_ring_emit_vm_flush */
2427	SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2428	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
2429	10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */
2430	.emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */
2431	.emit_ib = sdma_v4_0_ring_emit_ib,
2432	.emit_fence = sdma_v4_0_ring_emit_fence,
2433	.emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync,
2434	.emit_vm_flush = sdma_v4_0_ring_emit_vm_flush,
2435	.emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush,
2436	.test_ring = sdma_v4_0_ring_test_ring,
2437	.test_ib = sdma_v4_0_ring_test_ib,
2438	.insert_nop = sdma_v4_0_ring_insert_nop,
2439	.pad_ib = sdma_v4_0_ring_pad_ib,
2440	.emit_wreg = sdma_v4_0_ring_emit_wreg,
2441	.emit_reg_wait = sdma_v4_0_ring_emit_reg_wait,
2442	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2443	};
2444
2445	static const struct amdgpu_ring_funcs sdma_v4_0_page_ring_funcs = {
2446	.type = AMDGPU_RING_TYPE_SDMA,
2447	.align_mask = 0xff,
2448	.nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
2449	.support_64bit_ptrs = true,
2450	.secure_submission_supported = true,
2451	.get_rptr = sdma_v4_0_ring_get_rptr,
2452	.get_wptr = sdma_v4_0_page_ring_get_wptr,
2453	.set_wptr = sdma_v4_0_page_ring_set_wptr,
2454	.emit_frame_size =
2455	6 + /* sdma_v4_0_ring_emit_hdp_flush */
2456	3 + /* hdp invalidate */
2457	6 + /* sdma_v4_0_ring_emit_pipeline_sync */
2458	/* sdma_v4_0_ring_emit_vm_flush */
2459	SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2460	SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
2461	10 + 10 + 10, /* sdma_v4_0_ring_emit_fence x3 for user fence, vm fence */
2462	.emit_ib_size = 7 + 6, /* sdma_v4_0_ring_emit_ib */
2463	.emit_ib = sdma_v4_0_ring_emit_ib,
2464	.emit_fence = sdma_v4_0_ring_emit_fence,
2465	.emit_pipeline_sync = sdma_v4_0_ring_emit_pipeline_sync,
2466	.emit_vm_flush = sdma_v4_0_ring_emit_vm_flush,
2467	.emit_hdp_flush = sdma_v4_0_ring_emit_hdp_flush,
2468	.test_ring = sdma_v4_0_ring_test_ring,
2469	.test_ib = sdma_v4_0_ring_test_ib,
2470	.insert_nop = sdma_v4_0_ring_insert_nop,
2471	.pad_ib = sdma_v4_0_ring_pad_ib,
2472	.emit_wreg = sdma_v4_0_ring_emit_wreg,
2473	.emit_reg_wait = sdma_v4_0_ring_emit_reg_wait,
2474	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2475	};
2476
2477	static void sdma_v4_0_set_ring_funcs(struct amdgpu_device *adev)
2478	{
2479	int i;
2480
2481	for (i = 0; i < adev->sdma.num_instances; i++) {
2482	adev->sdma.instance[i].ring.funcs = &sdma_v4_0_ring_funcs;
2483	adev->sdma.instance[i].ring.me = i;
2484	if (adev->sdma.has_page_queue) {
2485	adev->sdma.instance[i].page.funcs =
2486	&sdma_v4_0_page_ring_funcs;
2487	adev->sdma.instance[i].page.me = i;
2488	}
2489	}
2490	}
2491
2492	static const struct amdgpu_irq_src_funcs sdma_v4_0_trap_irq_funcs = {
2493	.set = sdma_v4_0_set_trap_irq_state,
2494	.process = sdma_v4_0_process_trap_irq,
2495	};
2496
2497	static const struct amdgpu_irq_src_funcs sdma_v4_0_illegal_inst_irq_funcs = {
2498	.process = sdma_v4_0_process_illegal_inst_irq,
2499	};
2500
2501	static const struct amdgpu_irq_src_funcs sdma_v4_0_ecc_irq_funcs = {
2502	.set = sdma_v4_0_set_ecc_irq_state,
2503	.process = amdgpu_sdma_process_ecc_irq,
2504	};
2505
2506	static const struct amdgpu_irq_src_funcs sdma_v4_0_vm_hole_irq_funcs = {
2507	.process = sdma_v4_0_process_vm_hole_irq,
2508	};
2509
2510	static const struct amdgpu_irq_src_funcs sdma_v4_0_doorbell_invalid_irq_funcs = {
2511	.process = sdma_v4_0_process_doorbell_invalid_irq,
2512	};
2513
2514	static const struct amdgpu_irq_src_funcs sdma_v4_0_pool_timeout_irq_funcs = {
2515	.process = sdma_v4_0_process_pool_timeout_irq,
2516	};
2517
2518	static const struct amdgpu_irq_src_funcs sdma_v4_0_srbm_write_irq_funcs = {
2519	.process = sdma_v4_0_process_srbm_write_irq,
2520	};
2521
2522	static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev)
2523	{
2524	adev->sdma.trap_irq.num_types = adev->sdma.num_instances;
2525	adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;
2526	/*For Arcturus and Aldebaran, add another 4 irq handler*/
2527	switch (adev->sdma.num_instances) {
2528	case 5:
2529	case 8:
2530	adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;
2531	adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;
2532	adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;
2533	adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;
2534	break;
2535	default:
2536	break;
2537	}
2538	adev->sdma.trap_irq.funcs = &sdma_v4_0_trap_irq_funcs;
2539	adev->sdma.illegal_inst_irq.funcs = &sdma_v4_0_illegal_inst_irq_funcs;
2540	adev->sdma.ecc_irq.funcs = &sdma_v4_0_ecc_irq_funcs;
2541	adev->sdma.vm_hole_irq.funcs = &sdma_v4_0_vm_hole_irq_funcs;
2542	adev->sdma.doorbell_invalid_irq.funcs = &sdma_v4_0_doorbell_invalid_irq_funcs;
2543	adev->sdma.pool_timeout_irq.funcs = &sdma_v4_0_pool_timeout_irq_funcs;
2544	adev->sdma.srbm_write_irq.funcs = &sdma_v4_0_srbm_write_irq_funcs;
2545	}
2546
2547	/**
2548	* sdma_v4_0_emit_copy_buffer - copy buffer using the sDMA engine
2549	*
2550	* @ib: indirect buffer to copy to
2551	* @src_offset: src GPU address
2552	* @dst_offset: dst GPU address
2553	* @byte_count: number of bytes to xfer
2554	* @copy_flags: copy flags for the buffers
2555	*
2556	* Copy GPU buffers using the DMA engine (VEGA10/12).
2557	* Used by the amdgpu ttm implementation to move pages if
2558	* registered as the asic copy callback.
2559	*/
2560	static void sdma_v4_0_emit_copy_buffer(struct amdgpu_ib *ib,
2561	uint64_t src_offset,
2562	uint64_t dst_offset,
2563	uint32_t byte_count,
2564	uint32_t copy_flags)
2565	{
2566	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
2567	SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
2568	SDMA_PKT_COPY_LINEAR_HEADER_TMZ((copy_flags & AMDGPU_COPY_FLAGS_TMZ) ? 1 : 0);
2569	ib->ptr[ib->length_dw++] = byte_count - 1;
2570	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
2571	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
2572	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
2573	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2574	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2575	}
2576
2577	/**
2578	* sdma_v4_0_emit_fill_buffer - fill buffer using the sDMA engine
2579	*
2580	* @ib: indirect buffer to copy to
2581	* @src_data: value to write to buffer
2582	* @dst_offset: dst GPU address
2583	* @byte_count: number of bytes to xfer
2584	*
2585	* Fill GPU buffers using the DMA engine (VEGA10/12).
2586	*/
2587	static void sdma_v4_0_emit_fill_buffer(struct amdgpu_ib *ib,
2588	uint32_t src_data,
2589	uint64_t dst_offset,
2590	uint32_t byte_count)
2591	{
2592	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
2593	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2594	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2595	ib->ptr[ib->length_dw++] = src_data;
2596	ib->ptr[ib->length_dw++] = byte_count - 1;
2597	}
2598
2599	static const struct amdgpu_buffer_funcs sdma_v4_0_buffer_funcs = {
2600	.copy_max_bytes = 0x400000,
2601	.copy_num_dw = 7,
2602	.emit_copy_buffer = sdma_v4_0_emit_copy_buffer,
2603
2604	.fill_max_bytes = 0x400000,
2605	.fill_num_dw = 5,
2606	.emit_fill_buffer = sdma_v4_0_emit_fill_buffer,
2607	};
2608
2609	static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev)
2610	{
2611	adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs;
2612	if (adev->sdma.has_page_queue)
2613	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
2614	else
2615	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
2616	}
2617
2618	static const struct amdgpu_vm_pte_funcs sdma_v4_0_vm_pte_funcs = {
2619	.copy_pte_num_dw = 7,
2620	.copy_pte = sdma_v4_0_vm_copy_pte,
2621
2622	.write_pte = sdma_v4_0_vm_write_pte,
2623	.set_pte_pde = sdma_v4_0_vm_set_pte_pde,
2624	};
2625
2626	static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev)
2627	{
2628	struct drm_gpu_scheduler *sched;
2629	unsigned i;
2630
2631	adev->vm_manager.vm_pte_funcs = &sdma_v4_0_vm_pte_funcs;
2632	for (i = 0; i < adev->sdma.num_instances; i++) {
2633	if (adev->sdma.has_page_queue)
2634	sched = &adev->sdma.instance[i].page.sched;
2635	else
2636	sched = &adev->sdma.instance[i].ring.sched;
2637	adev->vm_manager.vm_pte_scheds[i] = sched;
2638	}
2639	adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
2640	}
2641
2642	static void sdma_v4_0_get_ras_error_count(uint32_t value,
2643	uint32_t instance,
2644	uint32_t *sec_count)
2645	{
2646	uint32_t i;
2647	uint32_t sec_cnt;
2648
2649	/* double bits error (multiple bits) error detection is not supported */
2650	for (i = 0; i < ARRAY_SIZE(sdma_v4_0_ras_fields); i++) {
2651	/* the SDMA_EDC_COUNTER register in each sdma instance
2652	* shares the same sed shift_mask
2653	* */
2654	sec_cnt = (value &
2655	sdma_v4_0_ras_fields[i].sec_count_mask) >>
2656	sdma_v4_0_ras_fields[i].sec_count_shift;
2657	if (sec_cnt) {
2658	DRM_INFO("Detected %s in SDMA%d, SED %d\n",
2659	sdma_v4_0_ras_fields[i].name,
2660	instance, sec_cnt);
2661	*sec_count += sec_cnt;
2662	}
2663	}
2664	}
2665
2666	static int sdma_v4_0_query_ras_error_count_by_instance(struct amdgpu_device *adev,
2667	uint32_t instance, void *ras_error_status)
2668	{
2669	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
2670	uint32_t sec_count = 0;
2671	uint32_t reg_value = 0;
2672
2673	reg_value = RREG32_SDMA(instance, mmSDMA0_EDC_COUNTER);
2674	/* double bit error is not supported */
2675	if (reg_value)
2676	sdma_v4_0_get_ras_error_count(value: reg_value,
2677	instance, sec_count: &sec_count);
2678	/* err_data->ce_count should be initialized to 0
2679	* before calling into this function */
2680	err_data->ce_count += sec_count;
2681	/* double bit error is not supported
2682	* set ue count to 0 */
2683	err_data->ue_count = 0;
2684
2685	return 0;
2686	};
2687
2688	static void sdma_v4_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status)
2689	{
2690	int i = 0;
2691
2692	for (i = 0; i < adev->sdma.num_instances; i++) {
2693	if (sdma_v4_0_query_ras_error_count_by_instance(adev, instance: i, ras_error_status)) {
2694	dev_err(adev->dev, "Query ras error count failed in SDMA%d\n", i);
2695	return;
2696	}
2697	}
2698	}
2699
2700	static void sdma_v4_0_reset_ras_error_count(struct amdgpu_device *adev)
2701	{
2702	int i;
2703
2704	/* read back edc counter registers to clear the counters */
2705	if (amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__SDMA)) {
2706	for (i = 0; i < adev->sdma.num_instances; i++)
2707	RREG32_SDMA(i, mmSDMA0_EDC_COUNTER);
2708	}
2709	}
2710
2711	const struct amdgpu_ras_block_hw_ops sdma_v4_0_ras_hw_ops = {
2712	.query_ras_error_count = sdma_v4_0_query_ras_error_count,
2713	.reset_ras_error_count = sdma_v4_0_reset_ras_error_count,
2714	};
2715
2716	static struct amdgpu_sdma_ras sdma_v4_0_ras = {
2717	.ras_block = {
2718	.hw_ops = &sdma_v4_0_ras_hw_ops,
2719	.ras_cb = sdma_v4_0_process_ras_data_cb,
2720	},
2721	};
2722
2723	static void sdma_v4_0_set_ras_funcs(struct amdgpu_device *adev)
2724	{
2725	switch (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0)) {
2726	case IP_VERSION(4, 2, 0):
2727	case IP_VERSION(4, 2, 2):
2728	adev->sdma.ras = &sdma_v4_0_ras;
2729	break;
2730	case IP_VERSION(4, 4, 0):
2731	adev->sdma.ras = &sdma_v4_4_ras;
2732	break;
2733	default:
2734	break;
2735	}
2736	}
2737
2738	const struct amdgpu_ip_block_version sdma_v4_0_ip_block = {
2739	.type = AMD_IP_BLOCK_TYPE_SDMA,
2740	.major = 4,
2741	.minor = 0,
2742	.rev = 0,
2743	.funcs = &sdma_v4_0_ip_funcs,
2744	};
2745