1	/*
2	* Copyright 2015 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/firmware.h>
25	#include <linux/slab.h>
26	#include <linux/module.h>
27	#include <linux/pci.h>
28
29	#include <drm/amdgpu_drm.h>
30
31	#include "amdgpu.h"
32	#include "amdgpu_atombios.h"
33	#include "amdgpu_ih.h"
34	#include "amdgpu_uvd.h"
35	#include "amdgpu_vce.h"
36	#include "atom.h"
37	#include "amd_pcie.h"
38
39	#include "si_dpm.h"
40	#include "sid.h"
41	#include "si_ih.h"
42	#include "gfx_v6_0.h"
43	#include "gmc_v6_0.h"
44	#include "si_dma.h"
45	#include "dce_v6_0.h"
46	#include "si.h"
47	#include "uvd_v3_1.h"
48	#include "vce_v1_0.h"
49
50	#include "uvd/uvd_4_0_d.h"
51
52	#include "smu/smu_6_0_d.h"
53	#include "smu/smu_6_0_sh_mask.h"
54
55	#include "gca/gfx_6_0_d.h"
56	#include "gca/gfx_6_0_sh_mask.h"
57
58	#include "oss/oss_1_0_d.h"
59	#include "oss/oss_1_0_sh_mask.h"
60
61	#include "gmc/gmc_6_0_d.h"
62	#include"gmc/gmc_6_0_sh_mask.h"
63
64	#include "dce/dce_6_0_d.h"
65	#include "dce/dce_6_0_sh_mask.h"
66
67	#include "bif/bif_3_0_d.h"
68	#include "bif/bif_3_0_sh_mask.h"
69	#include "si_enums.h"
70
71	#include "amdgpu_dm.h"
72	#include "amdgpu_vkms.h"
73
74	static const u32 tahiti_golden_registers[] =
75	{
76	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
77	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
78	mmDB_DEBUG, 0xffffffff, 0x00000000,
79	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
80	mmDB_DEBUG3, 0x0002021c, 0x00020200,
81	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
82	0x340c, 0x000000c0, 0x00800040,
83	0x360c, 0x000000c0, 0x00800040,
84	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
85	mmFBC_MISC, 0x00200000, 0x50100000,
86	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
87	mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
88	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
89	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
90	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
91	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
92	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
93	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
94	0x000c, 0xffffffff, 0x0040,
95	0x000d, 0x00000040, 0x00004040,
96	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
97	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
98	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
99	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
100	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
101	mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
102	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
103	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
104	mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
105	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
106	mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
107	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
108	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
109	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
110	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
111	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
112	};
113
114	static const u32 tahiti_golden_registers2[] =
115	{
116	mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
117	};
118
119	static const u32 tahiti_golden_rlc_registers[] =
120	{
121	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
122	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
123	0x311f, 0xffffffff, 0x10104040,
124	0x3122, 0xffffffff, 0x0100000a,
125	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
126	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
127	mmUVD_CGC_GATE, 0x00000008, 0x00000000,
128	};
129
130	static const u32 pitcairn_golden_registers[] =
131	{
132	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
133	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
134	mmDB_DEBUG, 0xffffffff, 0x00000000,
135	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
136	mmDB_DEBUG3, 0x0002021c, 0x00020200,
137	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
138	0x340c, 0x000300c0, 0x00800040,
139	0x360c, 0x000300c0, 0x00800040,
140	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
141	mmFBC_MISC, 0x00200000, 0x50100000,
142	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
143	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
144	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
145	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
146	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
147	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
148	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
149	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
150	0x000c, 0xffffffff, 0x0040,
151	0x000d, 0x00000040, 0x00004040,
152	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
153	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
154	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
155	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
156	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
157	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
158	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
159	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
160	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
161	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
162	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
163	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
164	};
165
166	static const u32 pitcairn_golden_rlc_registers[] =
167	{
168	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
169	mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
170	0x311f, 0xffffffff, 0x10102020,
171	0x3122, 0xffffffff, 0x01000020,
172	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
173	mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
174	};
175
176	static const u32 verde_pg_init[] =
177	{
178	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
179	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
180	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
181	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
182	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
183	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
184	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
185	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
186	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
187	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
188	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
189	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
190	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
191	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
192	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
193	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
194	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
195	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
196	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
197	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
198	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
199	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
200	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
201	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
202	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
203	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
204	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
205	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
206	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
207	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
208	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
209	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
210	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
211	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
212	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
213	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
214	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
215	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
216	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
217	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
218	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
219	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
220	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
221	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
222	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
223	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
224	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
225	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
226	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
227	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
228	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
229	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
230	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
231	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
232	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
233	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
234	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
235	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
236	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
237	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
238	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
239	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
240	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
241	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
242	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
243	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
244	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
245	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
246	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
247	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
248	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
249	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
250	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
251	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
252	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
253	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
254	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
255	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
256	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
257	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
258	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
259	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
260	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
261	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
262	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
263	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
264	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
265	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
266	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
267	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
268	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
269	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
270	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
271	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
272	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
273	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
274	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
275	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
276	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
277	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
278	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
279	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
280	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
281	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
282	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
283	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
284	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
285	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
286	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
287	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
288	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
289	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
290	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
291	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
292	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
293	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
294	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
295	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
296	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
297	mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
298	mmGMCON_MISC2, 0xfc00, 0x2000,
299	mmGMCON_MISC3, 0xffffffff, 0xfc0,
300	mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
301	};
302
303	static const u32 verde_golden_rlc_registers[] =
304	{
305	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
306	mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
307	0x311f, 0xffffffff, 0x10808020,
308	0x3122, 0xffffffff, 0x00800008,
309	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
310	mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
311	};
312
313	static const u32 verde_golden_registers[] =
314	{
315	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
316	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
317	mmDB_DEBUG, 0xffffffff, 0x00000000,
318	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
319	mmDB_DEBUG3, 0x0002021c, 0x00020200,
320	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
321	0x340c, 0x000300c0, 0x00800040,
322	0x360c, 0x000300c0, 0x00800040,
323	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
324	mmFBC_MISC, 0x00200000, 0x50100000,
325	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
326	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
327	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
328	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
329	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
330	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
331	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
332	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
333	0x000c, 0xffffffff, 0x0040,
334	0x000d, 0x00000040, 0x00004040,
335	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
336	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
337	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
338	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
339	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
340	mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
341	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
342	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
343	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
344	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
345	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
346	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
347	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
348	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
349	};
350
351	static const u32 oland_golden_registers[] =
352	{
353	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
354	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
355	mmDB_DEBUG, 0xffffffff, 0x00000000,
356	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
357	mmDB_DEBUG3, 0x0002021c, 0x00020200,
358	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
359	0x340c, 0x000300c0, 0x00800040,
360	0x360c, 0x000300c0, 0x00800040,
361	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
362	mmFBC_MISC, 0x00200000, 0x50100000,
363	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
364	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
365	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
366	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
367	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
368	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
369	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
370	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
371	0x000c, 0xffffffff, 0x0040,
372	0x000d, 0x00000040, 0x00004040,
373	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
374	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
375	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
376	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
377	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
378	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
379	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
380	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
381	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
382	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
383	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
384	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
385
386	};
387
388	static const u32 oland_golden_rlc_registers[] =
389	{
390	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
391	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
392	0x311f, 0xffffffff, 0x10104040,
393	0x3122, 0xffffffff, 0x0100000a,
394	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
395	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
396	};
397
398	static const u32 hainan_golden_registers[] =
399	{
400	0x17bc, 0x00000030, 0x00000011,
401	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
402	mmDB_DEBUG, 0xffffffff, 0x00000000,
403	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
404	mmDB_DEBUG3, 0x0002021c, 0x00020200,
405	0x031e, 0x00000080, 0x00000000,
406	0x3430, 0xff000fff, 0x00000100,
407	0x340c, 0x000300c0, 0x00800040,
408	0x3630, 0xff000fff, 0x00000100,
409	0x360c, 0x000300c0, 0x00800040,
410	0x16ec, 0x000000f0, 0x00000070,
411	0x16f0, 0x00200000, 0x50100000,
412	0x1c0c, 0x31000311, 0x00000011,
413	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
414	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
415	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
416	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
417	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
418	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
419	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
420	0x000c, 0xffffffff, 0x0040,
421	0x000d, 0x00000040, 0x00004040,
422	mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
423	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
424	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
425	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
426	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
427	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
428	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
429	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
430	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
431	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
432	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
433	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
434	};
435
436	static const u32 hainan_golden_registers2[] =
437	{
438	mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
439	};
440
441	static const u32 tahiti_mgcg_cgcg_init[] =
442	{
443	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
444	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
445	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
446	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
447	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
448	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
449	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
450	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
451	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
452	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
453	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
454	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
455	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
456	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
457	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
458	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
459	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
460	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
461	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
462	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
463	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
464	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
465	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
466	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
467	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
468	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
469	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
470	0x2458, 0xffffffff, 0x00010000,
471	0x2459, 0xffffffff, 0x00030002,
472	0x245a, 0xffffffff, 0x00040007,
473	0x245b, 0xffffffff, 0x00060005,
474	0x245c, 0xffffffff, 0x00090008,
475	0x245d, 0xffffffff, 0x00020001,
476	0x245e, 0xffffffff, 0x00040003,
477	0x245f, 0xffffffff, 0x00000007,
478	0x2460, 0xffffffff, 0x00060005,
479	0x2461, 0xffffffff, 0x00090008,
480	0x2462, 0xffffffff, 0x00030002,
481	0x2463, 0xffffffff, 0x00050004,
482	0x2464, 0xffffffff, 0x00000008,
483	0x2465, 0xffffffff, 0x00070006,
484	0x2466, 0xffffffff, 0x000a0009,
485	0x2467, 0xffffffff, 0x00040003,
486	0x2468, 0xffffffff, 0x00060005,
487	0x2469, 0xffffffff, 0x00000009,
488	0x246a, 0xffffffff, 0x00080007,
489	0x246b, 0xffffffff, 0x000b000a,
490	0x246c, 0xffffffff, 0x00050004,
491	0x246d, 0xffffffff, 0x00070006,
492	0x246e, 0xffffffff, 0x0008000b,
493	0x246f, 0xffffffff, 0x000a0009,
494	0x2470, 0xffffffff, 0x000d000c,
495	0x2471, 0xffffffff, 0x00060005,
496	0x2472, 0xffffffff, 0x00080007,
497	0x2473, 0xffffffff, 0x0000000b,
498	0x2474, 0xffffffff, 0x000a0009,
499	0x2475, 0xffffffff, 0x000d000c,
500	0x2476, 0xffffffff, 0x00070006,
501	0x2477, 0xffffffff, 0x00090008,
502	0x2478, 0xffffffff, 0x0000000c,
503	0x2479, 0xffffffff, 0x000b000a,
504	0x247a, 0xffffffff, 0x000e000d,
505	0x247b, 0xffffffff, 0x00080007,
506	0x247c, 0xffffffff, 0x000a0009,
507	0x247d, 0xffffffff, 0x0000000d,
508	0x247e, 0xffffffff, 0x000c000b,
509	0x247f, 0xffffffff, 0x000f000e,
510	0x2480, 0xffffffff, 0x00090008,
511	0x2481, 0xffffffff, 0x000b000a,
512	0x2482, 0xffffffff, 0x000c000f,
513	0x2483, 0xffffffff, 0x000e000d,
514	0x2484, 0xffffffff, 0x00110010,
515	0x2485, 0xffffffff, 0x000a0009,
516	0x2486, 0xffffffff, 0x000c000b,
517	0x2487, 0xffffffff, 0x0000000f,
518	0x2488, 0xffffffff, 0x000e000d,
519	0x2489, 0xffffffff, 0x00110010,
520	0x248a, 0xffffffff, 0x000b000a,
521	0x248b, 0xffffffff, 0x000d000c,
522	0x248c, 0xffffffff, 0x00000010,
523	0x248d, 0xffffffff, 0x000f000e,
524	0x248e, 0xffffffff, 0x00120011,
525	0x248f, 0xffffffff, 0x000c000b,
526	0x2490, 0xffffffff, 0x000e000d,
527	0x2491, 0xffffffff, 0x00000011,
528	0x2492, 0xffffffff, 0x0010000f,
529	0x2493, 0xffffffff, 0x00130012,
530	0x2494, 0xffffffff, 0x000d000c,
531	0x2495, 0xffffffff, 0x000f000e,
532	0x2496, 0xffffffff, 0x00100013,
533	0x2497, 0xffffffff, 0x00120011,
534	0x2498, 0xffffffff, 0x00150014,
535	0x2499, 0xffffffff, 0x000e000d,
536	0x249a, 0xffffffff, 0x0010000f,
537	0x249b, 0xffffffff, 0x00000013,
538	0x249c, 0xffffffff, 0x00120011,
539	0x249d, 0xffffffff, 0x00150014,
540	0x249e, 0xffffffff, 0x000f000e,
541	0x249f, 0xffffffff, 0x00110010,
542	0x24a0, 0xffffffff, 0x00000014,
543	0x24a1, 0xffffffff, 0x00130012,
544	0x24a2, 0xffffffff, 0x00160015,
545	0x24a3, 0xffffffff, 0x0010000f,
546	0x24a4, 0xffffffff, 0x00120011,
547	0x24a5, 0xffffffff, 0x00000015,
548	0x24a6, 0xffffffff, 0x00140013,
549	0x24a7, 0xffffffff, 0x00170016,
550	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
551	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
552	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
553	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
554	0x000c, 0xffffffff, 0x0000001c,
555	0x000d, 0x000f0000, 0x000f0000,
556	0x0583, 0xffffffff, 0x00000100,
557	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
558	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
559	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
560	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
561	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
562	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
563	0x157a, 0x00000001, 0x00000001,
564	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
565	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
566	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
567	0x3430, 0xfffffff0, 0x00000100,
568	0x3630, 0xfffffff0, 0x00000100,
569	};
570	static const u32 pitcairn_mgcg_cgcg_init[] =
571	{
572	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
573	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
574	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
575	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
576	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
577	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
578	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
579	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
580	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
581	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
582	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
583	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
584	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
585	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
586	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
587	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
588	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
589	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
590	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
591	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
592	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
593	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
594	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
595	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
596	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
597	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
598	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
599	0x2458, 0xffffffff, 0x00010000,
600	0x2459, 0xffffffff, 0x00030002,
601	0x245a, 0xffffffff, 0x00040007,
602	0x245b, 0xffffffff, 0x00060005,
603	0x245c, 0xffffffff, 0x00090008,
604	0x245d, 0xffffffff, 0x00020001,
605	0x245e, 0xffffffff, 0x00040003,
606	0x245f, 0xffffffff, 0x00000007,
607	0x2460, 0xffffffff, 0x00060005,
608	0x2461, 0xffffffff, 0x00090008,
609	0x2462, 0xffffffff, 0x00030002,
610	0x2463, 0xffffffff, 0x00050004,
611	0x2464, 0xffffffff, 0x00000008,
612	0x2465, 0xffffffff, 0x00070006,
613	0x2466, 0xffffffff, 0x000a0009,
614	0x2467, 0xffffffff, 0x00040003,
615	0x2468, 0xffffffff, 0x00060005,
616	0x2469, 0xffffffff, 0x00000009,
617	0x246a, 0xffffffff, 0x00080007,
618	0x246b, 0xffffffff, 0x000b000a,
619	0x246c, 0xffffffff, 0x00050004,
620	0x246d, 0xffffffff, 0x00070006,
621	0x246e, 0xffffffff, 0x0008000b,
622	0x246f, 0xffffffff, 0x000a0009,
623	0x2470, 0xffffffff, 0x000d000c,
624	0x2480, 0xffffffff, 0x00090008,
625	0x2481, 0xffffffff, 0x000b000a,
626	0x2482, 0xffffffff, 0x000c000f,
627	0x2483, 0xffffffff, 0x000e000d,
628	0x2484, 0xffffffff, 0x00110010,
629	0x2485, 0xffffffff, 0x000a0009,
630	0x2486, 0xffffffff, 0x000c000b,
631	0x2487, 0xffffffff, 0x0000000f,
632	0x2488, 0xffffffff, 0x000e000d,
633	0x2489, 0xffffffff, 0x00110010,
634	0x248a, 0xffffffff, 0x000b000a,
635	0x248b, 0xffffffff, 0x000d000c,
636	0x248c, 0xffffffff, 0x00000010,
637	0x248d, 0xffffffff, 0x000f000e,
638	0x248e, 0xffffffff, 0x00120011,
639	0x248f, 0xffffffff, 0x000c000b,
640	0x2490, 0xffffffff, 0x000e000d,
641	0x2491, 0xffffffff, 0x00000011,
642	0x2492, 0xffffffff, 0x0010000f,
643	0x2493, 0xffffffff, 0x00130012,
644	0x2494, 0xffffffff, 0x000d000c,
645	0x2495, 0xffffffff, 0x000f000e,
646	0x2496, 0xffffffff, 0x00100013,
647	0x2497, 0xffffffff, 0x00120011,
648	0x2498, 0xffffffff, 0x00150014,
649	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
650	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
651	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
652	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
653	0x000c, 0xffffffff, 0x0000001c,
654	0x000d, 0x000f0000, 0x000f0000,
655	0x0583, 0xffffffff, 0x00000100,
656	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
657	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
658	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
659	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
660	0x157a, 0x00000001, 0x00000001,
661	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
662	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
663	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
664	0x3430, 0xfffffff0, 0x00000100,
665	0x3630, 0xfffffff0, 0x00000100,
666	};
667
668	static const u32 verde_mgcg_cgcg_init[] =
669	{
670	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
671	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
672	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
673	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
674	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
675	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
676	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
677	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
678	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
679	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
680	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
681	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
682	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
683	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
684	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
685	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
686	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
687	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
688	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
689	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
690	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
691	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
692	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
693	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
694	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
695	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
696	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
697	0x2458, 0xffffffff, 0x00010000,
698	0x2459, 0xffffffff, 0x00030002,
699	0x245a, 0xffffffff, 0x00040007,
700	0x245b, 0xffffffff, 0x00060005,
701	0x245c, 0xffffffff, 0x00090008,
702	0x245d, 0xffffffff, 0x00020001,
703	0x245e, 0xffffffff, 0x00040003,
704	0x245f, 0xffffffff, 0x00000007,
705	0x2460, 0xffffffff, 0x00060005,
706	0x2461, 0xffffffff, 0x00090008,
707	0x2462, 0xffffffff, 0x00030002,
708	0x2463, 0xffffffff, 0x00050004,
709	0x2464, 0xffffffff, 0x00000008,
710	0x2465, 0xffffffff, 0x00070006,
711	0x2466, 0xffffffff, 0x000a0009,
712	0x2467, 0xffffffff, 0x00040003,
713	0x2468, 0xffffffff, 0x00060005,
714	0x2469, 0xffffffff, 0x00000009,
715	0x246a, 0xffffffff, 0x00080007,
716	0x246b, 0xffffffff, 0x000b000a,
717	0x246c, 0xffffffff, 0x00050004,
718	0x246d, 0xffffffff, 0x00070006,
719	0x246e, 0xffffffff, 0x0008000b,
720	0x246f, 0xffffffff, 0x000a0009,
721	0x2470, 0xffffffff, 0x000d000c,
722	0x2480, 0xffffffff, 0x00090008,
723	0x2481, 0xffffffff, 0x000b000a,
724	0x2482, 0xffffffff, 0x000c000f,
725	0x2483, 0xffffffff, 0x000e000d,
726	0x2484, 0xffffffff, 0x00110010,
727	0x2485, 0xffffffff, 0x000a0009,
728	0x2486, 0xffffffff, 0x000c000b,
729	0x2487, 0xffffffff, 0x0000000f,
730	0x2488, 0xffffffff, 0x000e000d,
731	0x2489, 0xffffffff, 0x00110010,
732	0x248a, 0xffffffff, 0x000b000a,
733	0x248b, 0xffffffff, 0x000d000c,
734	0x248c, 0xffffffff, 0x00000010,
735	0x248d, 0xffffffff, 0x000f000e,
736	0x248e, 0xffffffff, 0x00120011,
737	0x248f, 0xffffffff, 0x000c000b,
738	0x2490, 0xffffffff, 0x000e000d,
739	0x2491, 0xffffffff, 0x00000011,
740	0x2492, 0xffffffff, 0x0010000f,
741	0x2493, 0xffffffff, 0x00130012,
742	0x2494, 0xffffffff, 0x000d000c,
743	0x2495, 0xffffffff, 0x000f000e,
744	0x2496, 0xffffffff, 0x00100013,
745	0x2497, 0xffffffff, 0x00120011,
746	0x2498, 0xffffffff, 0x00150014,
747	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
748	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
749	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
750	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
751	0x000c, 0xffffffff, 0x0000001c,
752	0x000d, 0x000f0000, 0x000f0000,
753	0x0583, 0xffffffff, 0x00000100,
754	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
755	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
756	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
757	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
758	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
759	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
760	0x157a, 0x00000001, 0x00000001,
761	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
762	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
763	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
764	0x3430, 0xfffffff0, 0x00000100,
765	0x3630, 0xfffffff0, 0x00000100,
766	};
767
768	static const u32 oland_mgcg_cgcg_init[] =
769	{
770	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
771	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
772	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
773	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
774	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
775	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
776	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
777	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
778	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
779	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
780	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
781	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
782	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
783	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
784	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
785	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
786	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
787	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
788	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
789	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
790	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
791	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
792	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
793	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
794	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
795	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
796	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
797	0x2458, 0xffffffff, 0x00010000,
798	0x2459, 0xffffffff, 0x00030002,
799	0x245a, 0xffffffff, 0x00040007,
800	0x245b, 0xffffffff, 0x00060005,
801	0x245c, 0xffffffff, 0x00090008,
802	0x245d, 0xffffffff, 0x00020001,
803	0x245e, 0xffffffff, 0x00040003,
804	0x245f, 0xffffffff, 0x00000007,
805	0x2460, 0xffffffff, 0x00060005,
806	0x2461, 0xffffffff, 0x00090008,
807	0x2462, 0xffffffff, 0x00030002,
808	0x2463, 0xffffffff, 0x00050004,
809	0x2464, 0xffffffff, 0x00000008,
810	0x2465, 0xffffffff, 0x00070006,
811	0x2466, 0xffffffff, 0x000a0009,
812	0x2467, 0xffffffff, 0x00040003,
813	0x2468, 0xffffffff, 0x00060005,
814	0x2469, 0xffffffff, 0x00000009,
815	0x246a, 0xffffffff, 0x00080007,
816	0x246b, 0xffffffff, 0x000b000a,
817	0x246c, 0xffffffff, 0x00050004,
818	0x246d, 0xffffffff, 0x00070006,
819	0x246e, 0xffffffff, 0x0008000b,
820	0x246f, 0xffffffff, 0x000a0009,
821	0x2470, 0xffffffff, 0x000d000c,
822	0x2471, 0xffffffff, 0x00060005,
823	0x2472, 0xffffffff, 0x00080007,
824	0x2473, 0xffffffff, 0x0000000b,
825	0x2474, 0xffffffff, 0x000a0009,
826	0x2475, 0xffffffff, 0x000d000c,
827	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
828	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
829	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
830	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
831	0x000c, 0xffffffff, 0x0000001c,
832	0x000d, 0x000f0000, 0x000f0000,
833	0x0583, 0xffffffff, 0x00000100,
834	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
835	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
836	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
837	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
838	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
839	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
840	0x157a, 0x00000001, 0x00000001,
841	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
842	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
843	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
844	0x3430, 0xfffffff0, 0x00000100,
845	0x3630, 0xfffffff0, 0x00000100,
846	};
847
848	static const u32 hainan_mgcg_cgcg_init[] =
849	{
850	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
851	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
852	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
853	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
854	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
855	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
856	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
857	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
858	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
859	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
860	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
861	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
862	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
863	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
864	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
865	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
866	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
867	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
868	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
869	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
870	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
871	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
872	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
873	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
874	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
875	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
876	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
877	0x2458, 0xffffffff, 0x00010000,
878	0x2459, 0xffffffff, 0x00030002,
879	0x245a, 0xffffffff, 0x00040007,
880	0x245b, 0xffffffff, 0x00060005,
881	0x245c, 0xffffffff, 0x00090008,
882	0x245d, 0xffffffff, 0x00020001,
883	0x245e, 0xffffffff, 0x00040003,
884	0x245f, 0xffffffff, 0x00000007,
885	0x2460, 0xffffffff, 0x00060005,
886	0x2461, 0xffffffff, 0x00090008,
887	0x2462, 0xffffffff, 0x00030002,
888	0x2463, 0xffffffff, 0x00050004,
889	0x2464, 0xffffffff, 0x00000008,
890	0x2465, 0xffffffff, 0x00070006,
891	0x2466, 0xffffffff, 0x000a0009,
892	0x2467, 0xffffffff, 0x00040003,
893	0x2468, 0xffffffff, 0x00060005,
894	0x2469, 0xffffffff, 0x00000009,
895	0x246a, 0xffffffff, 0x00080007,
896	0x246b, 0xffffffff, 0x000b000a,
897	0x246c, 0xffffffff, 0x00050004,
898	0x246d, 0xffffffff, 0x00070006,
899	0x246e, 0xffffffff, 0x0008000b,
900	0x246f, 0xffffffff, 0x000a0009,
901	0x2470, 0xffffffff, 0x000d000c,
902	0x2471, 0xffffffff, 0x00060005,
903	0x2472, 0xffffffff, 0x00080007,
904	0x2473, 0xffffffff, 0x0000000b,
905	0x2474, 0xffffffff, 0x000a0009,
906	0x2475, 0xffffffff, 0x000d000c,
907	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
908	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
909	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
910	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
911	0x000c, 0xffffffff, 0x0000001c,
912	0x000d, 0x000f0000, 0x000f0000,
913	0x0583, 0xffffffff, 0x00000100,
914	0x0409, 0xffffffff, 0x00000100,
915	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
916	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
917	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
918	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
919	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
920	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
921	0x3430, 0xfffffff0, 0x00000100,
922	0x3630, 0xfffffff0, 0x00000100,
923	};
924
925	/* tahiti, pitcairn, verde */
926	static const struct amdgpu_video_codec_info tahiti_video_codecs_encode_array[] =
927	{
928	{
929	.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
930	.max_width = 2048,
931	.max_height = 1152,
932	.max_pixels_per_frame = 2048 * 1152,
933	.max_level = 0,
934	},
935	};
936
937	static const struct amdgpu_video_codecs tahiti_video_codecs_encode =
938	{
939	.codec_count = ARRAY_SIZE(tahiti_video_codecs_encode_array),
940	.codec_array = tahiti_video_codecs_encode_array,
941	};
942
943	/* oland and hainan don't support encode */
944	static const struct amdgpu_video_codecs hainan_video_codecs_encode =
945	{
946	.codec_count = 0,
947	.codec_array = NULL,
948	};
949
950	/* tahiti, pitcairn, verde, oland */
951	static const struct amdgpu_video_codec_info tahiti_video_codecs_decode_array[] =
952	{
953	{
954	.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
955	.max_width = 2048,
956	.max_height = 1152,
957	.max_pixels_per_frame = 2048 * 1152,
958	.max_level = 3,
959	},
960	{
961	.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
962	.max_width = 2048,
963	.max_height = 1152,
964	.max_pixels_per_frame = 2048 * 1152,
965	.max_level = 5,
966	},
967	{
968	.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
969	.max_width = 2048,
970	.max_height = 1152,
971	.max_pixels_per_frame = 2048 * 1152,
972	.max_level = 41,
973	},
974	{
975	.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
976	.max_width = 2048,
977	.max_height = 1152,
978	.max_pixels_per_frame = 2048 * 1152,
979	.max_level = 4,
980	},
981	};
982
983	static const struct amdgpu_video_codecs tahiti_video_codecs_decode =
984	{
985	.codec_count = ARRAY_SIZE(tahiti_video_codecs_decode_array),
986	.codec_array = tahiti_video_codecs_decode_array,
987	};
988
989	/* hainan doesn't support decode */
990	static const struct amdgpu_video_codecs hainan_video_codecs_decode =
991	{
992	.codec_count = 0,
993	.codec_array = NULL,
994	};
995
996	static int si_query_video_codecs(struct amdgpu_device *adev, bool encode,
997	const struct amdgpu_video_codecs **codecs)
998	{
999	switch (adev->asic_type) {
1000	case CHIP_VERDE:
1001	case CHIP_TAHITI:
1002	case CHIP_PITCAIRN:
1003	if (encode)
1004	*codecs = &tahiti_video_codecs_encode;
1005	else
1006	*codecs = &tahiti_video_codecs_decode;
1007	return 0;
1008	case CHIP_OLAND:
1009	if (encode)
1010	*codecs = &hainan_video_codecs_encode;
1011	else
1012	*codecs = &tahiti_video_codecs_decode;
1013	return 0;
1014	case CHIP_HAINAN:
1015	if (encode)
1016	*codecs = &hainan_video_codecs_encode;
1017	else
1018	*codecs = &hainan_video_codecs_decode;
1019	return 0;
1020	default:
1021	return -EINVAL;
1022	}
1023	}
1024
1025	static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
1026	{
1027	unsigned long flags;
1028	u32 r;
1029
1030	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1031	WREG32(AMDGPU_PCIE_INDEX, reg);
1032	(void)RREG32(AMDGPU_PCIE_INDEX);
1033	r = RREG32(AMDGPU_PCIE_DATA);
1034	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
1035	return r;
1036	}
1037
1038	static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1039	{
1040	unsigned long flags;
1041
1042	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1043	WREG32(AMDGPU_PCIE_INDEX, reg);
1044	(void)RREG32(AMDGPU_PCIE_INDEX);
1045	WREG32(AMDGPU_PCIE_DATA, v);
1046	(void)RREG32(AMDGPU_PCIE_DATA);
1047	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
1048	}
1049
1050	static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
1051	{
1052	unsigned long flags;
1053	u32 r;
1054
1055	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1056	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1057	(void)RREG32(PCIE_PORT_INDEX);
1058	r = RREG32(PCIE_PORT_DATA);
1059	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
1060	return r;
1061	}
1062
1063	static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1064	{
1065	unsigned long flags;
1066
1067	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1068	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1069	(void)RREG32(PCIE_PORT_INDEX);
1070	WREG32(PCIE_PORT_DATA, (v));
1071	(void)RREG32(PCIE_PORT_DATA);
1072	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
1073	}
1074
1075	static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
1076	{
1077	unsigned long flags;
1078	u32 r;
1079
1080	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1081	WREG32(mmSMC_IND_INDEX_0, (reg));
1082	r = RREG32(mmSMC_IND_DATA_0);
1083	spin_unlock_irqrestore(lock: &adev->smc_idx_lock, flags);
1084	return r;
1085	}
1086
1087	static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1088	{
1089	unsigned long flags;
1090
1091	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1092	WREG32(mmSMC_IND_INDEX_0, (reg));
1093	WREG32(mmSMC_IND_DATA_0, (v));
1094	spin_unlock_irqrestore(lock: &adev->smc_idx_lock, flags);
1095	}
1096
1097	static u32 si_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
1098	{
1099	unsigned long flags;
1100	u32 r;
1101
1102	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
1103	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
1104	r = RREG32(mmUVD_CTX_DATA);
1105	spin_unlock_irqrestore(lock: &adev->uvd_ctx_idx_lock, flags);
1106	return r;
1107	}
1108
1109	static void si_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1110	{
1111	unsigned long flags;
1112
1113	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
1114	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
1115	WREG32(mmUVD_CTX_DATA, (v));
1116	spin_unlock_irqrestore(lock: &adev->uvd_ctx_idx_lock, flags);
1117	}
1118
1119	static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
1120	{mmGRBM_STATUS},
1121	{mmGRBM_STATUS2},
1122	{mmGRBM_STATUS_SE0},
1123	{mmGRBM_STATUS_SE1},
1124	{mmSRBM_STATUS},
1125	{mmSRBM_STATUS2},
1126	{mmDMA_STATUS_REG + DMA0_REGISTER_OFFSET},
1127	{mmDMA_STATUS_REG + DMA1_REGISTER_OFFSET},
1128	{mmCP_STAT},
1129	{mmCP_STALLED_STAT1},
1130	{mmCP_STALLED_STAT2},
1131	{mmCP_STALLED_STAT3},
1132	{mmGB_ADDR_CONFIG},
1133	{mmMC_ARB_RAMCFG},
1134	{mmGB_TILE_MODE0},
1135	{mmGB_TILE_MODE1},
1136	{mmGB_TILE_MODE2},
1137	{mmGB_TILE_MODE3},
1138	{mmGB_TILE_MODE4},
1139	{mmGB_TILE_MODE5},
1140	{mmGB_TILE_MODE6},
1141	{mmGB_TILE_MODE7},
1142	{mmGB_TILE_MODE8},
1143	{mmGB_TILE_MODE9},
1144	{mmGB_TILE_MODE10},
1145	{mmGB_TILE_MODE11},
1146	{mmGB_TILE_MODE12},
1147	{mmGB_TILE_MODE13},
1148	{mmGB_TILE_MODE14},
1149	{mmGB_TILE_MODE15},
1150	{mmGB_TILE_MODE16},
1151	{mmGB_TILE_MODE17},
1152	{mmGB_TILE_MODE18},
1153	{mmGB_TILE_MODE19},
1154	{mmGB_TILE_MODE20},
1155	{mmGB_TILE_MODE21},
1156	{mmGB_TILE_MODE22},
1157	{mmGB_TILE_MODE23},
1158	{mmGB_TILE_MODE24},
1159	{mmGB_TILE_MODE25},
1160	{mmGB_TILE_MODE26},
1161	{mmGB_TILE_MODE27},
1162	{mmGB_TILE_MODE28},
1163	{mmGB_TILE_MODE29},
1164	{mmGB_TILE_MODE30},
1165	{mmGB_TILE_MODE31},
1166	{mmCC_RB_BACKEND_DISABLE, true},
1167	{mmGC_USER_RB_BACKEND_DISABLE, true},
1168	{mmPA_SC_RASTER_CONFIG, true},
1169	};
1170
1171	static uint32_t si_get_register_value(struct amdgpu_device *adev,
1172	bool indexed, u32 se_num,
1173	u32 sh_num, u32 reg_offset)
1174	{
1175	if (indexed) {
1176	uint32_t val;
1177	unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1178	unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1179
1180	switch (reg_offset) {
1181	case mmCC_RB_BACKEND_DISABLE:
1182	return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1183	case mmGC_USER_RB_BACKEND_DISABLE:
1184	return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1185	case mmPA_SC_RASTER_CONFIG:
1186	return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1187	}
1188
1189	mutex_lock(&adev->grbm_idx_mutex);
1190	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1191	amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1192
1193	val = RREG32(reg_offset);
1194
1195	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1196	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1197	mutex_unlock(lock: &adev->grbm_idx_mutex);
1198	return val;
1199	} else {
1200	unsigned idx;
1201
1202	switch (reg_offset) {
1203	case mmGB_ADDR_CONFIG:
1204	return adev->gfx.config.gb_addr_config;
1205	case mmMC_ARB_RAMCFG:
1206	return adev->gfx.config.mc_arb_ramcfg;
1207	case mmGB_TILE_MODE0:
1208	case mmGB_TILE_MODE1:
1209	case mmGB_TILE_MODE2:
1210	case mmGB_TILE_MODE3:
1211	case mmGB_TILE_MODE4:
1212	case mmGB_TILE_MODE5:
1213	case mmGB_TILE_MODE6:
1214	case mmGB_TILE_MODE7:
1215	case mmGB_TILE_MODE8:
1216	case mmGB_TILE_MODE9:
1217	case mmGB_TILE_MODE10:
1218	case mmGB_TILE_MODE11:
1219	case mmGB_TILE_MODE12:
1220	case mmGB_TILE_MODE13:
1221	case mmGB_TILE_MODE14:
1222	case mmGB_TILE_MODE15:
1223	case mmGB_TILE_MODE16:
1224	case mmGB_TILE_MODE17:
1225	case mmGB_TILE_MODE18:
1226	case mmGB_TILE_MODE19:
1227	case mmGB_TILE_MODE20:
1228	case mmGB_TILE_MODE21:
1229	case mmGB_TILE_MODE22:
1230	case mmGB_TILE_MODE23:
1231	case mmGB_TILE_MODE24:
1232	case mmGB_TILE_MODE25:
1233	case mmGB_TILE_MODE26:
1234	case mmGB_TILE_MODE27:
1235	case mmGB_TILE_MODE28:
1236	case mmGB_TILE_MODE29:
1237	case mmGB_TILE_MODE30:
1238	case mmGB_TILE_MODE31:
1239	idx = (reg_offset - mmGB_TILE_MODE0);
1240	return adev->gfx.config.tile_mode_array[idx];
1241	default:
1242	return RREG32(reg_offset);
1243	}
1244	}
1245	}
1246	static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1247	u32 sh_num, u32 reg_offset, u32 *value)
1248	{
1249	uint32_t i;
1250
1251	*value = 0;
1252	for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1253	bool indexed = si_allowed_read_registers[i].grbm_indexed;
1254
1255	if (reg_offset != si_allowed_read_registers[i].reg_offset)
1256	continue;
1257
1258	*value = si_get_register_value(adev, indexed, se_num, sh_num,
1259	reg_offset);
1260	return 0;
1261	}
1262	return -EINVAL;
1263	}
1264
1265	static bool si_read_disabled_bios(struct amdgpu_device *adev)
1266	{
1267	u32 bus_cntl;
1268	u32 d1vga_control = 0;
1269	u32 d2vga_control = 0;
1270	u32 vga_render_control = 0;
1271	u32 rom_cntl;
1272	bool r;
1273
1274	bus_cntl = RREG32(mmBUS_CNTL);
1275	if (adev->mode_info.num_crtc) {
1276	d1vga_control = RREG32(mmD1VGA_CONTROL);
1277	d2vga_control = RREG32(mmD2VGA_CONTROL);
1278	vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
1279	}
1280	rom_cntl = RREG32(R600_ROM_CNTL);
1281
1282	/* enable the rom */
1283	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
1284	if (adev->mode_info.num_crtc) {
1285	/* Disable VGA mode */
1286	WREG32(mmD1VGA_CONTROL,
1287	(d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
1288	D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
1289	WREG32(mmD2VGA_CONTROL,
1290	(d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
1291	D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
1292	WREG32(mmVGA_RENDER_CONTROL,
1293	(vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
1294	}
1295	WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1296
1297	r = amdgpu_read_bios(adev);
1298
1299	/* restore regs */
1300	WREG32(mmBUS_CNTL, bus_cntl);
1301	if (adev->mode_info.num_crtc) {
1302	WREG32(mmD1VGA_CONTROL, d1vga_control);
1303	WREG32(mmD2VGA_CONTROL, d2vga_control);
1304	WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1305	}
1306	WREG32(R600_ROM_CNTL, rom_cntl);
1307	return r;
1308	}
1309
1310	#define mmROM_INDEX 0x2A
1311	#define mmROM_DATA 0x2B
1312
1313	static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1314	u8 *bios, u32 length_bytes)
1315	{
1316	u32 *dw_ptr;
1317	u32 i, length_dw;
1318
1319	if (bios == NULL)
1320	return false;
1321	if (length_bytes == 0)
1322	return false;
1323	/* APU vbios image is part of sbios image */
1324	if (adev->flags & AMD_IS_APU)
1325	return false;
1326
1327	dw_ptr = (u32 *)bios;
1328	length_dw = ALIGN(length_bytes, 4) / 4;
1329	/* set rom index to 0 */
1330	WREG32(mmROM_INDEX, 0);
1331	for (i = 0; i < length_dw; i++)
1332	dw_ptr[i] = RREG32(mmROM_DATA);
1333
1334	return true;
1335	}
1336
1337	static void si_set_clk_bypass_mode(struct amdgpu_device *adev)
1338	{
1339	u32 tmp, i;
1340
1341	tmp = RREG32(mmCG_SPLL_FUNC_CNTL);
1342	tmp |= CG_SPLL_FUNC_CNTL__SPLL_BYPASS_EN_MASK;
1343	WREG32(mmCG_SPLL_FUNC_CNTL, tmp);
1344
1345	tmp = RREG32(mmCG_SPLL_FUNC_CNTL_2);
1346	tmp |= CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG_MASK;
1347	WREG32(mmCG_SPLL_FUNC_CNTL_2, tmp);
1348
1349	for (i = 0; i < adev->usec_timeout; i++) {
1350	if (RREG32(mmCG_SPLL_STATUS) & CG_SPLL_STATUS__SPLL_CHG_STATUS_MASK)
1351	break;
1352	udelay(usec: 1);
1353	}
1354
1355	tmp = RREG32(mmCG_SPLL_FUNC_CNTL_2);
1356	tmp &= ~(CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG_MASK |
1357	CG_SPLL_FUNC_CNTL_2__SCLK_MUX_UPDATE_MASK);
1358	WREG32(mmCG_SPLL_FUNC_CNTL_2, tmp);
1359
1360	tmp = RREG32(MPLL_CNTL_MODE);
1361	tmp &= ~MPLL_MCLK_SEL;
1362	WREG32(MPLL_CNTL_MODE, tmp);
1363	}
1364
1365	static void si_spll_powerdown(struct amdgpu_device *adev)
1366	{
1367	u32 tmp;
1368
1369	tmp = RREG32(mmSPLL_CNTL_MODE);
1370	tmp |= SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL_MASK;
1371	WREG32(mmSPLL_CNTL_MODE, tmp);
1372
1373	tmp = RREG32(mmCG_SPLL_FUNC_CNTL);
1374	tmp |= CG_SPLL_FUNC_CNTL__SPLL_RESET_MASK;
1375	WREG32(mmCG_SPLL_FUNC_CNTL, tmp);
1376
1377	tmp = RREG32(mmCG_SPLL_FUNC_CNTL);
1378	tmp |= CG_SPLL_FUNC_CNTL__SPLL_SLEEP_MASK;
1379	WREG32(mmCG_SPLL_FUNC_CNTL, tmp);
1380
1381	tmp = RREG32(mmSPLL_CNTL_MODE);
1382	tmp &= ~SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL_MASK;
1383	WREG32(mmSPLL_CNTL_MODE, tmp);
1384	}
1385
1386	static int si_gpu_pci_config_reset(struct amdgpu_device *adev)
1387	{
1388	u32 i;
1389	int r = -EINVAL;
1390
1391	amdgpu_atombios_scratch_regs_engine_hung(adev, hung: true);
1392
1393	/* set mclk/sclk to bypass */
1394	si_set_clk_bypass_mode(adev);
1395	/* powerdown spll */
1396	si_spll_powerdown(adev);
1397	/* disable BM */
1398	pci_clear_master(dev: adev->pdev);
1399	/* reset */
1400	amdgpu_device_pci_config_reset(adev);
1401
1402	udelay(usec: 100);
1403
1404	/* wait for asic to come out of reset */
1405	for (i = 0; i < adev->usec_timeout; i++) {
1406	if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1407	/* enable BM */
1408	pci_set_master(dev: adev->pdev);
1409	adev->has_hw_reset = true;
1410	r = 0;
1411	break;
1412	}
1413	udelay(usec: 1);
1414	}
1415	amdgpu_atombios_scratch_regs_engine_hung(adev, hung: false);
1416
1417	return r;
1418	}
1419
1420	static int si_asic_supports_baco(struct amdgpu_device *adev)
1421	{
1422	return 0;
1423	}
1424
1425	static enum amd_reset_method
1426	si_asic_reset_method(struct amdgpu_device *adev)
1427	{
1428	if (amdgpu_reset_method == AMD_RESET_METHOD_PCI)
1429	return amdgpu_reset_method;
1430	else if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY &&
1431	amdgpu_reset_method != -1)
1432	dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n",
1433	amdgpu_reset_method);
1434
1435	return AMD_RESET_METHOD_LEGACY;
1436	}
1437
1438	static int si_asic_reset(struct amdgpu_device *adev)
1439	{
1440	int r;
1441
1442	switch (si_asic_reset_method(adev)) {
1443	case AMD_RESET_METHOD_PCI:
1444	dev_info(adev->dev, "PCI reset\n");
1445	r = amdgpu_device_pci_reset(adev);
1446	break;
1447	default:
1448	dev_info(adev->dev, "PCI CONFIG reset\n");
1449	r = si_gpu_pci_config_reset(adev);
1450	break;
1451	}
1452
1453	return r;
1454	}
1455
1456	static u32 si_get_config_memsize(struct amdgpu_device *adev)
1457	{
1458	return RREG32(mmCONFIG_MEMSIZE);
1459	}
1460
1461	static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1462	{
1463	uint32_t temp;
1464
1465	temp = RREG32(mmCONFIG_CNTL);
1466	if (!state) {
1467	temp &= ~(1<<0);
1468	temp |= (1<<1);
1469	} else {
1470	temp &= ~(1<<1);
1471	}
1472	WREG32(mmCONFIG_CNTL, temp);
1473	}
1474
1475	static u32 si_get_xclk(struct amdgpu_device *adev)
1476	{
1477	u32 reference_clock = adev->clock.spll.reference_freq;
1478	u32 tmp;
1479
1480	tmp = RREG32(mmCG_CLKPIN_CNTL_2);
1481	if (tmp & CG_CLKPIN_CNTL_2__MUX_TCLK_TO_XCLK_MASK)
1482	return TCLK;
1483
1484	tmp = RREG32(mmCG_CLKPIN_CNTL);
1485	if (tmp & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
1486	return reference_clock / 4;
1487
1488	return reference_clock;
1489	}
1490
1491	static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1492	{
1493	if (!ring || !ring->funcs->emit_wreg) {
1494	WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1495	RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1496	} else {
1497	amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1498	}
1499	}
1500
1501	static void si_invalidate_hdp(struct amdgpu_device *adev,
1502	struct amdgpu_ring *ring)
1503	{
1504	if (!ring || !ring->funcs->emit_wreg) {
1505	WREG32(mmHDP_DEBUG0, 1);
1506	RREG32(mmHDP_DEBUG0);
1507	} else {
1508	amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1509	}
1510	}
1511
1512	static bool si_need_full_reset(struct amdgpu_device *adev)
1513	{
1514	/* change this when we support soft reset */
1515	return true;
1516	}
1517
1518	static bool si_need_reset_on_init(struct amdgpu_device *adev)
1519	{
1520	return false;
1521	}
1522
1523	static int si_get_pcie_lanes(struct amdgpu_device *adev)
1524	{
1525	u32 link_width_cntl;
1526
1527	if (adev->flags & AMD_IS_APU)
1528	return 0;
1529
1530	link_width_cntl = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
1531
1532	switch ((link_width_cntl & PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT) {
1533	case LC_LINK_WIDTH_X1:
1534	return 1;
1535	case LC_LINK_WIDTH_X2:
1536	return 2;
1537	case LC_LINK_WIDTH_X4:
1538	return 4;
1539	case LC_LINK_WIDTH_X8:
1540	return 8;
1541	case LC_LINK_WIDTH_X0:
1542	case LC_LINK_WIDTH_X16:
1543	default:
1544	return 16;
1545	}
1546	}
1547
1548	static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes)
1549	{
1550	u32 link_width_cntl, mask;
1551
1552	if (adev->flags & AMD_IS_APU)
1553	return;
1554
1555	switch (lanes) {
1556	case 0:
1557	mask = LC_LINK_WIDTH_X0;
1558	break;
1559	case 1:
1560	mask = LC_LINK_WIDTH_X1;
1561	break;
1562	case 2:
1563	mask = LC_LINK_WIDTH_X2;
1564	break;
1565	case 4:
1566	mask = LC_LINK_WIDTH_X4;
1567	break;
1568	case 8:
1569	mask = LC_LINK_WIDTH_X8;
1570	break;
1571	case 16:
1572	mask = LC_LINK_WIDTH_X16;
1573	break;
1574	default:
1575	DRM_ERROR("invalid pcie lane request: %d\n", lanes);
1576	return;
1577	}
1578
1579	link_width_cntl = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
1580	link_width_cntl &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK;
1581	link_width_cntl |= mask << PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT;
1582	link_width_cntl |= (PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK |
1583	PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_ARC_MISSING_ESCAPE_MASK);
1584
1585	WREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1586	}
1587
1588	static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1589	uint64_t *count1)
1590	{
1591	uint32_t perfctr = 0;
1592	uint64_t cnt0_of, cnt1_of;
1593	int tmp;
1594
1595	/* This reports 0 on APUs, so return to avoid writing/reading registers
1596	* that may or may not be different from their GPU counterparts
1597	*/
1598	if (adev->flags & AMD_IS_APU)
1599	return;
1600
1601	/* Set the 2 events that we wish to watch, defined above */
1602	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1603	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1604	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1605
1606	/* Write to enable desired perf counters */
1607	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1608	/* Zero out and enable the perf counters
1609	* Write 0x5:
1610	* Bit 0 = Start all counters(1)
1611	* Bit 2 = Global counter reset enable(1)
1612	*/
1613	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1614
1615	msleep(msecs: 1000);
1616
1617	/* Load the shadow and disable the perf counters
1618	* Write 0x2:
1619	* Bit 0 = Stop counters(0)
1620	* Bit 1 = Load the shadow counters(1)
1621	*/
1622	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1623
1624	/* Read register values to get any >32bit overflow */
1625	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1626	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1627	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1628
1629	/* Get the values and add the overflow */
1630	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1631	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1632	}
1633
1634	static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)
1635	{
1636	uint64_t nak_r, nak_g;
1637
1638	/* Get the number of NAKs received and generated */
1639	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1640	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1641
1642	/* Add the total number of NAKs, i.e the number of replays */
1643	return (nak_r + nak_g);
1644	}
1645
1646	static int si_uvd_send_upll_ctlreq(struct amdgpu_device *adev,
1647	unsigned cg_upll_func_cntl)
1648	{
1649	unsigned i;
1650
1651	/* Make sure UPLL_CTLREQ is deasserted */
1652	WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1653
1654	mdelay(10);
1655
1656	/* Assert UPLL_CTLREQ */
1657	WREG32_P(cg_upll_func_cntl, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1658
1659	/* Wait for CTLACK and CTLACK2 to get asserted */
1660	for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1661	uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1662
1663	if ((RREG32(cg_upll_func_cntl) & mask) == mask)
1664	break;
1665	mdelay(10);
1666	}
1667
1668	/* Deassert UPLL_CTLREQ */
1669	WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1670
1671	if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1672	DRM_ERROR("Timeout setting UVD clocks!\n");
1673	return -ETIMEDOUT;
1674	}
1675
1676	return 0;
1677	}
1678
1679	static unsigned si_uvd_calc_upll_post_div(unsigned vco_freq,
1680	unsigned target_freq,
1681	unsigned pd_min,
1682	unsigned pd_even)
1683	{
1684	unsigned post_div = vco_freq / target_freq;
1685
1686	/* Adjust to post divider minimum value */
1687	if (post_div < pd_min)
1688	post_div = pd_min;
1689
1690	/* We alway need a frequency less than or equal the target */
1691	if ((vco_freq / post_div) > target_freq)
1692	post_div += 1;
1693
1694	/* Post dividers above a certain value must be even */
1695	if (post_div > pd_even && post_div % 2)
1696	post_div += 1;
1697
1698	return post_div;
1699	}
1700
1701	/**
1702	* si_calc_upll_dividers - calc UPLL clock dividers
1703	*
1704	* @adev: amdgpu_device pointer
1705	* @vclk: wanted VCLK
1706	* @dclk: wanted DCLK
1707	* @vco_min: minimum VCO frequency
1708	* @vco_max: maximum VCO frequency
1709	* @fb_factor: factor to multiply vco freq with
1710	* @fb_mask: limit and bitmask for feedback divider
1711	* @pd_min: post divider minimum
1712	* @pd_max: post divider maximum
1713	* @pd_even: post divider must be even above this value
1714	* @optimal_fb_div: resulting feedback divider
1715	* @optimal_vclk_div: resulting vclk post divider
1716	* @optimal_dclk_div: resulting dclk post divider
1717	*
1718	* Calculate dividers for UVDs UPLL (except APUs).
1719	* Returns zero on success; -EINVAL on error.
1720	*/
1721	static int si_calc_upll_dividers(struct amdgpu_device *adev,
1722	unsigned vclk, unsigned dclk,
1723	unsigned vco_min, unsigned vco_max,
1724	unsigned fb_factor, unsigned fb_mask,
1725	unsigned pd_min, unsigned pd_max,
1726	unsigned pd_even,
1727	unsigned *optimal_fb_div,
1728	unsigned *optimal_vclk_div,
1729	unsigned *optimal_dclk_div)
1730	{
1731	unsigned vco_freq, ref_freq = adev->clock.spll.reference_freq;
1732
1733	/* Start off with something large */
1734	unsigned optimal_score = ~0;
1735
1736	/* Loop through vco from low to high */
1737	vco_min = max(max(vco_min, vclk), dclk);
1738	for (vco_freq = vco_min; vco_freq <= vco_max; vco_freq += 100) {
1739	uint64_t fb_div = (uint64_t)vco_freq * fb_factor;
1740	unsigned vclk_div, dclk_div, score;
1741
1742	do_div(fb_div, ref_freq);
1743
1744	/* fb div out of range ? */
1745	if (fb_div > fb_mask)
1746	break; /* It can oly get worse */
1747
1748	fb_div &= fb_mask;
1749
1750	/* Calc vclk divider with current vco freq */
1751	vclk_div = si_uvd_calc_upll_post_div(vco_freq, target_freq: vclk,
1752	pd_min, pd_even);
1753	if (vclk_div > pd_max)
1754	break; /* vco is too big, it has to stop */
1755
1756	/* Calc dclk divider with current vco freq */
1757	dclk_div = si_uvd_calc_upll_post_div(vco_freq, target_freq: dclk,
1758	pd_min, pd_even);
1759	if (dclk_div > pd_max)
1760	break; /* vco is too big, it has to stop */
1761
1762	/* Calc score with current vco freq */
1763	score = vclk - (vco_freq / vclk_div) + dclk - (vco_freq / dclk_div);
1764
1765	/* Determine if this vco setting is better than current optimal settings */
1766	if (score < optimal_score) {
1767	*optimal_fb_div = fb_div;
1768	*optimal_vclk_div = vclk_div;
1769	*optimal_dclk_div = dclk_div;
1770	optimal_score = score;
1771	if (optimal_score == 0)
1772	break; /* It can't get better than this */
1773	}
1774	}
1775
1776	/* Did we found a valid setup ? */
1777	if (optimal_score == ~0)
1778	return -EINVAL;
1779
1780	return 0;
1781	}
1782
1783	static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1784	{
1785	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1786	int r;
1787
1788	/* Bypass vclk and dclk with bclk */
1789	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1790	VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1791	~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1792
1793	/* Put PLL in bypass mode */
1794	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1795
1796	if (!vclk || !dclk) {
1797	/* Keep the Bypass mode */
1798	return 0;
1799	}
1800
1801	r = si_calc_upll_dividers(adev, vclk, dclk, vco_min: 125000, vco_max: 250000,
1802	fb_factor: 16384, fb_mask: 0x03FFFFFF, pd_min: 0, pd_max: 128, pd_even: 5,
1803	optimal_fb_div: &fb_div, optimal_vclk_div: &vclk_div, optimal_dclk_div: &dclk_div);
1804	if (r)
1805	return r;
1806
1807	/* Set RESET_ANTI_MUX to 0 */
1808	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1809
1810	/* Set VCO_MODE to 1 */
1811	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1812
1813	/* Disable sleep mode */
1814	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1815
1816	/* Deassert UPLL_RESET */
1817	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1818
1819	mdelay(1);
1820
1821	r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1822	if (r)
1823	return r;
1824
1825	/* Assert UPLL_RESET again */
1826	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1827
1828	/* Disable spread spectrum. */
1829	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1830
1831	/* Set feedback divider */
1832	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1833
1834	/* Set ref divider to 0 */
1835	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1836
1837	if (fb_div < 307200)
1838	WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1839	else
1840	WREG32_P(CG_UPLL_FUNC_CNTL_4,
1841	UPLL_SPARE_ISPARE9,
1842	~UPLL_SPARE_ISPARE9);
1843
1844	/* Set PDIV_A and PDIV_B */
1845	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1846	UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1847	~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1848
1849	/* Give the PLL some time to settle */
1850	mdelay(15);
1851
1852	/* Deassert PLL_RESET */
1853	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1854
1855	mdelay(15);
1856
1857	/* Switch from bypass mode to normal mode */
1858	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1859
1860	r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1861	if (r)
1862	return r;
1863
1864	/* Switch VCLK and DCLK selection */
1865	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1866	VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1867	~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1868
1869	mdelay(100);
1870
1871	return 0;
1872	}
1873
1874	static int si_vce_send_vcepll_ctlreq(struct amdgpu_device *adev)
1875	{
1876	unsigned i;
1877
1878	/* Make sure VCEPLL_CTLREQ is deasserted */
1879	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1880
1881	mdelay(10);
1882
1883	/* Assert UPLL_CTLREQ */
1884	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1885
1886	/* Wait for CTLACK and CTLACK2 to get asserted */
1887	for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1888	uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1889
1890	if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
1891	break;
1892	mdelay(10);
1893	}
1894
1895	/* Deassert UPLL_CTLREQ */
1896	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1897
1898	if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1899	DRM_ERROR("Timeout setting VCE clocks!\n");
1900	return -ETIMEDOUT;
1901	}
1902
1903	return 0;
1904	}
1905
1906	static int si_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1907	{
1908	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
1909	int r;
1910
1911	/* Bypass evclk and ecclk with bclk */
1912	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1913	EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
1914	~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
1915
1916	/* Put PLL in bypass mode */
1917	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
1918	~VCEPLL_BYPASS_EN_MASK);
1919
1920	if (!evclk || !ecclk) {
1921	/*
1922	* On some chips, the PLL takes way too long to get out of
1923	* sleep mode, causing a timeout waiting on CTLACK/CTLACK2.
1924	* Leave the PLL running in bypass mode.
1925	*/
1926	if (adev->pdev->device == 0x6780)
1927	return 0;
1928
1929	/* Keep the Bypass mode, put PLL to sleep */
1930	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1931	~VCEPLL_SLEEP_MASK);
1932	return 0;
1933	}
1934
1935	r = si_calc_upll_dividers(adev, vclk: evclk, dclk: ecclk, vco_min: 125000, vco_max: 250000,
1936	fb_factor: 16384, fb_mask: 0x03FFFFFF, pd_min: 0, pd_max: 128, pd_even: 5,
1937	optimal_fb_div: &fb_div, optimal_vclk_div: &evclk_div, optimal_dclk_div: &ecclk_div);
1938	if (r)
1939	return r;
1940
1941	/* Set RESET_ANTI_MUX to 0 */
1942	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1943
1944	/* Set VCO_MODE to 1 */
1945	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
1946	~VCEPLL_VCO_MODE_MASK);
1947
1948	/* Toggle VCEPLL_SLEEP to 1 then back to 0 */
1949	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1950	~VCEPLL_SLEEP_MASK);
1951	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
1952
1953	/* Deassert VCEPLL_RESET */
1954	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1955
1956	mdelay(1);
1957
1958	r = si_vce_send_vcepll_ctlreq(adev);
1959	if (r)
1960	return r;
1961
1962	/* Assert VCEPLL_RESET again */
1963	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
1964
1965	/* Disable spread spectrum. */
1966	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1967
1968	/* Set feedback divider */
1969	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3,
1970	VCEPLL_FB_DIV(fb_div),
1971	~VCEPLL_FB_DIV_MASK);
1972
1973	/* Set ref divider to 0 */
1974	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
1975
1976	/* Set PDIV_A and PDIV_B */
1977	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1978	VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
1979	~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
1980
1981	/* Give the PLL some time to settle */
1982	mdelay(15);
1983
1984	/* Deassert PLL_RESET */
1985	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1986
1987	mdelay(15);
1988
1989	/* Switch from bypass mode to normal mode */
1990	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
1991
1992	r = si_vce_send_vcepll_ctlreq(adev);
1993	if (r)
1994	return r;
1995
1996	/* Switch VCLK and DCLK selection */
1997	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1998	EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
1999	~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
2000
2001	mdelay(100);
2002
2003	return 0;
2004	}
2005
2006	static void si_pre_asic_init(struct amdgpu_device *adev)
2007	{
2008	}
2009
2010	static const struct amdgpu_asic_funcs si_asic_funcs =
2011	{
2012	.read_disabled_bios = &si_read_disabled_bios,
2013	.read_bios_from_rom = &si_read_bios_from_rom,
2014	.read_register = &si_read_register,
2015	.reset = &si_asic_reset,
2016	.reset_method = &si_asic_reset_method,
2017	.set_vga_state = &si_vga_set_state,
2018	.get_xclk = &si_get_xclk,
2019	.set_uvd_clocks = &si_set_uvd_clocks,
2020	.set_vce_clocks = &si_set_vce_clocks,
2021	.get_pcie_lanes = &si_get_pcie_lanes,
2022	.set_pcie_lanes = &si_set_pcie_lanes,
2023	.get_config_memsize = &si_get_config_memsize,
2024	.flush_hdp = &si_flush_hdp,
2025	.invalidate_hdp = &si_invalidate_hdp,
2026	.need_full_reset = &si_need_full_reset,
2027	.get_pcie_usage = &si_get_pcie_usage,
2028	.need_reset_on_init = &si_need_reset_on_init,
2029	.get_pcie_replay_count = &si_get_pcie_replay_count,
2030	.supports_baco = &si_asic_supports_baco,
2031	.pre_asic_init = &si_pre_asic_init,
2032	.query_video_codecs = &si_query_video_codecs,
2033	};
2034
2035	static uint32_t si_get_rev_id(struct amdgpu_device *adev)
2036	{
2037	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
2038	>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
2039	}
2040
2041	static int si_common_early_init(struct amdgpu_ip_block *ip_block)
2042	{
2043	struct amdgpu_device *adev = ip_block->adev;
2044
2045	adev->smc_rreg = &si_smc_rreg;
2046	adev->smc_wreg = &si_smc_wreg;
2047	adev->pcie_rreg = &si_pcie_rreg;
2048	adev->pcie_wreg = &si_pcie_wreg;
2049	adev->pciep_rreg = &si_pciep_rreg;
2050	adev->pciep_wreg = &si_pciep_wreg;
2051	adev->uvd_ctx_rreg = si_uvd_ctx_rreg;
2052	adev->uvd_ctx_wreg = si_uvd_ctx_wreg;
2053	adev->didt_rreg = NULL;
2054	adev->didt_wreg = NULL;
2055
2056	adev->asic_funcs = &si_asic_funcs;
2057
2058	adev->rev_id = si_get_rev_id(adev);
2059	adev->external_rev_id = 0xFF;
2060	switch (adev->asic_type) {
2061	case CHIP_TAHITI:
2062	adev->cg_flags =
2063	AMD_CG_SUPPORT_GFX_MGCG |
2064	AMD_CG_SUPPORT_GFX_MGLS |
2065	/*AMD_CG_SUPPORT_GFX_CGCG |*/
2066	AMD_CG_SUPPORT_GFX_CGLS |
2067	AMD_CG_SUPPORT_GFX_CGTS |
2068	AMD_CG_SUPPORT_GFX_CP_LS |
2069	AMD_CG_SUPPORT_MC_MGCG |
2070	AMD_CG_SUPPORT_SDMA_MGCG |
2071	AMD_CG_SUPPORT_BIF_LS |
2072	AMD_CG_SUPPORT_VCE_MGCG |
2073	AMD_CG_SUPPORT_UVD_MGCG |
2074	AMD_CG_SUPPORT_HDP_LS |
2075	AMD_CG_SUPPORT_HDP_MGCG;
2076	adev->pg_flags = 0;
2077	adev->external_rev_id = (adev->rev_id == 0) ? 1 :
2078	(adev->rev_id == 1) ? 5 : 6;
2079	break;
2080	case CHIP_PITCAIRN:
2081	adev->cg_flags =
2082	AMD_CG_SUPPORT_GFX_MGCG |
2083	AMD_CG_SUPPORT_GFX_MGLS |
2084	/*AMD_CG_SUPPORT_GFX_CGCG |*/
2085	AMD_CG_SUPPORT_GFX_CGLS |
2086	AMD_CG_SUPPORT_GFX_CGTS |
2087	AMD_CG_SUPPORT_GFX_CP_LS |
2088	AMD_CG_SUPPORT_GFX_RLC_LS |
2089	AMD_CG_SUPPORT_MC_LS |
2090	AMD_CG_SUPPORT_MC_MGCG |
2091	AMD_CG_SUPPORT_SDMA_MGCG |
2092	AMD_CG_SUPPORT_BIF_LS |
2093	AMD_CG_SUPPORT_VCE_MGCG |
2094	AMD_CG_SUPPORT_UVD_MGCG |
2095	AMD_CG_SUPPORT_HDP_LS |
2096	AMD_CG_SUPPORT_HDP_MGCG;
2097	adev->pg_flags = 0;
2098	adev->external_rev_id = adev->rev_id + 20;
2099	break;
2100
2101	case CHIP_VERDE:
2102	adev->cg_flags =
2103	AMD_CG_SUPPORT_GFX_MGCG |
2104	AMD_CG_SUPPORT_GFX_MGLS |
2105	AMD_CG_SUPPORT_GFX_CGLS |
2106	AMD_CG_SUPPORT_GFX_CGTS |
2107	AMD_CG_SUPPORT_GFX_CGTS_LS |
2108	AMD_CG_SUPPORT_GFX_CP_LS |
2109	AMD_CG_SUPPORT_MC_LS |
2110	AMD_CG_SUPPORT_MC_MGCG |
2111	AMD_CG_SUPPORT_SDMA_MGCG |
2112	AMD_CG_SUPPORT_SDMA_LS |
2113	AMD_CG_SUPPORT_BIF_LS |
2114	AMD_CG_SUPPORT_VCE_MGCG |
2115	AMD_CG_SUPPORT_UVD_MGCG |
2116	AMD_CG_SUPPORT_HDP_LS |
2117	AMD_CG_SUPPORT_HDP_MGCG;
2118	adev->pg_flags = 0;
2119	//???
2120	adev->external_rev_id = adev->rev_id + 40;
2121	break;
2122	case CHIP_OLAND:
2123	adev->cg_flags =
2124	AMD_CG_SUPPORT_GFX_MGCG |
2125	AMD_CG_SUPPORT_GFX_MGLS |
2126	/*AMD_CG_SUPPORT_GFX_CGCG |*/
2127	AMD_CG_SUPPORT_GFX_CGLS |
2128	AMD_CG_SUPPORT_GFX_CGTS |
2129	AMD_CG_SUPPORT_GFX_CP_LS |
2130	AMD_CG_SUPPORT_GFX_RLC_LS |
2131	AMD_CG_SUPPORT_MC_LS |
2132	AMD_CG_SUPPORT_MC_MGCG |
2133	AMD_CG_SUPPORT_SDMA_MGCG |
2134	AMD_CG_SUPPORT_BIF_LS |
2135	AMD_CG_SUPPORT_UVD_MGCG |
2136	AMD_CG_SUPPORT_HDP_LS |
2137	AMD_CG_SUPPORT_HDP_MGCG;
2138	adev->pg_flags = 0;
2139	adev->external_rev_id = 60;
2140	break;
2141	case CHIP_HAINAN:
2142	adev->cg_flags =
2143	AMD_CG_SUPPORT_GFX_MGCG |
2144	AMD_CG_SUPPORT_GFX_MGLS |
2145	/*AMD_CG_SUPPORT_GFX_CGCG |*/
2146	AMD_CG_SUPPORT_GFX_CGLS |
2147	AMD_CG_SUPPORT_GFX_CGTS |
2148	AMD_CG_SUPPORT_GFX_CP_LS |
2149	AMD_CG_SUPPORT_GFX_RLC_LS |
2150	AMD_CG_SUPPORT_MC_LS |
2151	AMD_CG_SUPPORT_MC_MGCG |
2152	AMD_CG_SUPPORT_SDMA_MGCG |
2153	AMD_CG_SUPPORT_BIF_LS |
2154	AMD_CG_SUPPORT_HDP_LS |
2155	AMD_CG_SUPPORT_HDP_MGCG;
2156	adev->pg_flags = 0;
2157	adev->external_rev_id = 70;
2158	break;
2159
2160	default:
2161	return -EINVAL;
2162	}
2163
2164	return 0;
2165	}
2166
2167	static void si_init_golden_registers(struct amdgpu_device *adev)
2168	{
2169	switch (adev->asic_type) {
2170	case CHIP_TAHITI:
2171	amdgpu_device_program_register_sequence(adev,
2172	registers: tahiti_golden_registers,
2173	ARRAY_SIZE(tahiti_golden_registers));
2174	amdgpu_device_program_register_sequence(adev,
2175	registers: tahiti_golden_rlc_registers,
2176	ARRAY_SIZE(tahiti_golden_rlc_registers));
2177	amdgpu_device_program_register_sequence(adev,
2178	registers: tahiti_mgcg_cgcg_init,
2179	ARRAY_SIZE(tahiti_mgcg_cgcg_init));
2180	amdgpu_device_program_register_sequence(adev,
2181	registers: tahiti_golden_registers2,
2182	ARRAY_SIZE(tahiti_golden_registers2));
2183	break;
2184	case CHIP_PITCAIRN:
2185	amdgpu_device_program_register_sequence(adev,
2186	registers: pitcairn_golden_registers,
2187	ARRAY_SIZE(pitcairn_golden_registers));
2188	amdgpu_device_program_register_sequence(adev,
2189	registers: pitcairn_golden_rlc_registers,
2190	ARRAY_SIZE(pitcairn_golden_rlc_registers));
2191	amdgpu_device_program_register_sequence(adev,
2192	registers: pitcairn_mgcg_cgcg_init,
2193	ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
2194	break;
2195	case CHIP_VERDE:
2196	amdgpu_device_program_register_sequence(adev,
2197	registers: verde_golden_registers,
2198	ARRAY_SIZE(verde_golden_registers));
2199	amdgpu_device_program_register_sequence(adev,
2200	registers: verde_golden_rlc_registers,
2201	ARRAY_SIZE(verde_golden_rlc_registers));
2202	amdgpu_device_program_register_sequence(adev,
2203	registers: verde_mgcg_cgcg_init,
2204	ARRAY_SIZE(verde_mgcg_cgcg_init));
2205	amdgpu_device_program_register_sequence(adev,
2206	registers: verde_pg_init,
2207	ARRAY_SIZE(verde_pg_init));
2208	break;
2209	case CHIP_OLAND:
2210	amdgpu_device_program_register_sequence(adev,
2211	registers: oland_golden_registers,
2212	ARRAY_SIZE(oland_golden_registers));
2213	amdgpu_device_program_register_sequence(adev,
2214	registers: oland_golden_rlc_registers,
2215	ARRAY_SIZE(oland_golden_rlc_registers));
2216	amdgpu_device_program_register_sequence(adev,
2217	registers: oland_mgcg_cgcg_init,
2218	ARRAY_SIZE(oland_mgcg_cgcg_init));
2219	break;
2220	case CHIP_HAINAN:
2221	amdgpu_device_program_register_sequence(adev,
2222	registers: hainan_golden_registers,
2223	ARRAY_SIZE(hainan_golden_registers));
2224	amdgpu_device_program_register_sequence(adev,
2225	registers: hainan_golden_registers2,
2226	ARRAY_SIZE(hainan_golden_registers2));
2227	amdgpu_device_program_register_sequence(adev,
2228	registers: hainan_mgcg_cgcg_init,
2229	ARRAY_SIZE(hainan_mgcg_cgcg_init));
2230	break;
2231
2232
2233	default:
2234	BUG();
2235	}
2236	}
2237
2238	static void si_pcie_gen3_enable(struct amdgpu_device *adev)
2239	{
2240	struct pci_dev *root = adev->pdev->bus->self;
2241	u32 speed_cntl, current_data_rate;
2242	int i;
2243	u16 tmp16;
2244
2245	if (pci_is_root_bus(pbus: adev->pdev->bus))
2246	return;
2247
2248	if (amdgpu_pcie_gen2 == 0)
2249	return;
2250
2251	if (adev->flags & AMD_IS_APU)
2252	return;
2253
2254	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2255	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
2256	return;
2257
2258	speed_cntl = RREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL);
2259	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
2260	PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
2261	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2262	if (current_data_rate == 2) {
2263	DRM_INFO("PCIE gen 3 link speeds already enabled\n");
2264	return;
2265	}
2266	DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
2267	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
2268	if (current_data_rate == 1) {
2269	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
2270	return;
2271	}
2272	DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
2273	}
2274
2275	if (!pci_is_pcie(dev: root) || !pci_is_pcie(dev: adev->pdev))
2276	return;
2277
2278	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2279	if (current_data_rate != 2) {
2280	u16 bridge_cfg, gpu_cfg;
2281	u16 bridge_cfg2, gpu_cfg2;
2282	u32 max_lw, current_lw, tmp;
2283
2284	pcie_capability_set_word(dev: root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
2285	pcie_capability_set_word(dev: adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
2286
2287	tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
2288	max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >> PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
2289	current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK) >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
2290
2291	if (current_lw < max_lw) {
2292	tmp = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
2293	if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
2294	tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK | PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
2295	tmp |= (max_lw << PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
2296	tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK | PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK | PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
2297	WREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
2298	}
2299	}
2300
2301	for (i = 0; i < 10; i++) {
2302	pcie_capability_read_word(dev: adev->pdev,
2303	PCI_EXP_DEVSTA,
2304	val: &tmp16);
2305	if (tmp16 & PCI_EXP_DEVSTA_TRPND)
2306	break;
2307
2308	pcie_capability_read_word(dev: root, PCI_EXP_LNKCTL,
2309	val: &bridge_cfg);
2310	pcie_capability_read_word(dev: adev->pdev,
2311	PCI_EXP_LNKCTL,
2312	val: &gpu_cfg);
2313
2314	pcie_capability_read_word(dev: root, PCI_EXP_LNKCTL2,
2315	val: &bridge_cfg2);
2316	pcie_capability_read_word(dev: adev->pdev,
2317	PCI_EXP_LNKCTL2,
2318	val: &gpu_cfg2);
2319
2320	tmp = RREG32_PCIE_PORT(ixPCIE_LC_CNTL4);
2321	tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
2322	WREG32_PCIE_PORT(ixPCIE_LC_CNTL4, tmp);
2323
2324	tmp = RREG32_PCIE_PORT(ixPCIE_LC_CNTL4);
2325	tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
2326	WREG32_PCIE_PORT(ixPCIE_LC_CNTL4, tmp);
2327
2328	mdelay(100);
2329
2330	pcie_capability_clear_and_set_word(dev: root, PCI_EXP_LNKCTL,
2331	PCI_EXP_LNKCTL_HAWD,
2332	set: bridge_cfg &
2333	PCI_EXP_LNKCTL_HAWD);
2334	pcie_capability_clear_and_set_word(dev: adev->pdev, PCI_EXP_LNKCTL,
2335	PCI_EXP_LNKCTL_HAWD,
2336	set: gpu_cfg &
2337	PCI_EXP_LNKCTL_HAWD);
2338
2339	pcie_capability_clear_and_set_word(dev: root, PCI_EXP_LNKCTL2,
2340	PCI_EXP_LNKCTL2_ENTER_COMP |
2341	PCI_EXP_LNKCTL2_TX_MARGIN,
2342	set: bridge_cfg2 &
2343	(PCI_EXP_LNKCTL2_ENTER_COMP |
2344	PCI_EXP_LNKCTL2_TX_MARGIN));
2345	pcie_capability_clear_and_set_word(dev: adev->pdev, PCI_EXP_LNKCTL2,
2346	PCI_EXP_LNKCTL2_ENTER_COMP |
2347	PCI_EXP_LNKCTL2_TX_MARGIN,
2348	set: gpu_cfg2 &
2349	(PCI_EXP_LNKCTL2_ENTER_COMP |
2350	PCI_EXP_LNKCTL2_TX_MARGIN));
2351
2352	tmp = RREG32_PCIE_PORT(ixPCIE_LC_CNTL4);
2353	tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
2354	WREG32_PCIE_PORT(ixPCIE_LC_CNTL4, tmp);
2355	}
2356	}
2357	}
2358
2359	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK | PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
2360	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
2361	WREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL, speed_cntl);
2362
2363	tmp16 = 0;
2364	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
2365	tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
2366	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
2367	tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
2368	else
2369	tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
2370	pcie_capability_clear_and_set_word(dev: adev->pdev, PCI_EXP_LNKCTL2,
2371	PCI_EXP_LNKCTL2_TLS, set: tmp16);
2372
2373	speed_cntl = RREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL);
2374	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
2375	WREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL, speed_cntl);
2376
2377	for (i = 0; i < adev->usec_timeout; i++) {
2378	speed_cntl = RREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL);
2379	if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
2380	break;
2381	udelay(usec: 1);
2382	}
2383	}
2384
2385	static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
2386	{
2387	unsigned long flags;
2388	u32 r;
2389
2390	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2391	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2392	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
2393	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
2394	return r;
2395	}
2396
2397	static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2398	{
2399	unsigned long flags;
2400
2401	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2402	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2403	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
2404	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
2405	}
2406
2407	static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
2408	{
2409	unsigned long flags;
2410	u32 r;
2411
2412	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2413	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2414	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
2415	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
2416	return r;
2417	}
2418
2419	static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2420	{
2421	unsigned long flags;
2422
2423	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
2424	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2425	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
2426	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
2427	}
2428	static void si_program_aspm(struct amdgpu_device *adev)
2429	{
2430	u32 data, orig;
2431	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
2432	bool disable_clkreq = false;
2433
2434	if (!amdgpu_device_should_use_aspm(adev))
2435	return;
2436
2437	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_N_FTS_CNTL);
2438	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
2439	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) | PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
2440	if (orig != data)
2441	WREG32_PCIE_PORT(ixPCIE_LC_N_FTS_CNTL, data);
2442
2443	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL3);
2444	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
2445	if (orig != data)
2446	WREG32_PCIE_PORT(ixPCIE_LC_CNTL3, data);
2447
2448	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
2449	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
2450	if (orig != data)
2451	WREG32_PCIE(ixPCIE_P_CNTL, data);
2452
2453	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL);
2454	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK | PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
2455	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
2456	if (!disable_l0s)
2457	data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
2458
2459	if (!disable_l1) {
2460	data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
2461	data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
2462	if (orig != data)
2463	WREG32_PCIE_PORT(ixPCIE_LC_CNTL, data);
2464
2465	if (!disable_plloff_in_l1) {
2466	bool clk_req_support;
2467
2468	orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_0);
2469	data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK | PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
2470	data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) | (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
2471	if (orig != data)
2472	si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_0, v: data);
2473
2474	orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_1);
2475	data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK | PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
2476	data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) | (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
2477	if (orig != data)
2478	si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_1, v: data);
2479
2480	orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_0);
2481	data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK | PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
2482	data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) | (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
2483	if (orig != data)
2484	si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_0, v: data);
2485
2486	orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_1);
2487	data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK | PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
2488	data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) | (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
2489	if (orig != data)
2490	si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_1, v: data);
2491
2492	if ((adev->asic_type != CHIP_OLAND) && (adev->asic_type != CHIP_HAINAN)) {
2493	orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_0);
2494	data &= ~PB0_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0_MASK;
2495	if (orig != data)
2496	si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_0, v: data);
2497
2498	orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_1);
2499	data &= ~PB0_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1_MASK;
2500	if (orig != data)
2501	si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_1, v: data);
2502
2503	orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_2);
2504	data &= ~PB0_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2_MASK;
2505	if (orig != data)
2506	si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_2, v: data);
2507
2508	orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_3);
2509	data &= ~PB0_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3_MASK;
2510	if (orig != data)
2511	si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_3, v: data);
2512
2513	orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_0);
2514	data &= ~PB1_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0_MASK;
2515	if (orig != data)
2516	si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_0, v: data);
2517
2518	orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_1);
2519	data &= ~PB1_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1_MASK;
2520	if (orig != data)
2521	si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_1, v: data);
2522
2523	orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_2);
2524	data &= ~PB1_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2_MASK;
2525	if (orig != data)
2526	si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_2, v: data);
2527
2528	orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_3);
2529	data &= ~PB1_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3_MASK;
2530	if (orig != data)
2531	si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_3, v: data);
2532	}
2533	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
2534	data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
2535	data |= (3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
2536	if (orig != data)
2537	WREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL, data);
2538
2539	orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_CNTL);
2540	data &= ~PB0_PIF_CNTL__LS2_EXIT_TIME_MASK;
2541	if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2542	data |= (5 << PB0_PIF_CNTL__LS2_EXIT_TIME__SHIFT);
2543	if (orig != data)
2544	si_pif_phy0_wreg(adev,ixPB0_PIF_CNTL, v: data);
2545
2546	orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_CNTL);
2547	data &= ~PB1_PIF_CNTL__LS2_EXIT_TIME_MASK;
2548	if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2549	data |= (5 << PB1_PIF_CNTL__LS2_EXIT_TIME__SHIFT);
2550	if (orig != data)
2551	si_pif_phy1_wreg(adev,ixPB1_PIF_CNTL, v: data);
2552
2553	if (!disable_clkreq &&
2554	!pci_is_root_bus(pbus: adev->pdev->bus)) {
2555	struct pci_dev *root = adev->pdev->bus->self;
2556	u32 lnkcap;
2557
2558	clk_req_support = false;
2559	pcie_capability_read_dword(dev: root, PCI_EXP_LNKCAP, val: &lnkcap);
2560	if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
2561	clk_req_support = true;
2562	} else {
2563	clk_req_support = false;
2564	}
2565
2566	if (clk_req_support) {
2567	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL2);
2568	data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
2569	if (orig != data)
2570	WREG32_PCIE_PORT(ixPCIE_LC_CNTL2, data);
2571
2572	orig = data = RREG32(mmTHM_CLK_CNTL);
2573	data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK | THM_CLK_CNTL__TMON_CLK_SEL_MASK);
2574	data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) | (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
2575	if (orig != data)
2576	WREG32(mmTHM_CLK_CNTL, data);
2577
2578	orig = data = RREG32(mmMISC_CLK_CNTL);
2579	data &= ~(MISC_CLK_CNTL__DEEP_SLEEP_CLK_SEL_MASK | MISC_CLK_CNTL__ZCLK_SEL_MASK);
2580	data |= (1 << MISC_CLK_CNTL__DEEP_SLEEP_CLK_SEL__SHIFT) | (1 << MISC_CLK_CNTL__ZCLK_SEL__SHIFT);
2581	if (orig != data)
2582	WREG32(mmMISC_CLK_CNTL, data);
2583
2584	orig = data = RREG32(mmCG_CLKPIN_CNTL);
2585	data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
2586	if (orig != data)
2587	WREG32(mmCG_CLKPIN_CNTL, data);
2588
2589	orig = data = RREG32(mmCG_CLKPIN_CNTL_2);
2590	data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
2591	if (orig != data)
2592	WREG32(mmCG_CLKPIN_CNTL_2, data);
2593
2594	orig = data = RREG32(mmMPLL_BYPASSCLK_SEL);
2595	data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
2596	data |= 4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT;
2597	if (orig != data)
2598	WREG32(mmMPLL_BYPASSCLK_SEL, data);
2599
2600	orig = data = RREG32(mmSPLL_CNTL_MODE);
2601	data &= ~SPLL_CNTL_MODE__SPLL_REFCLK_SEL_MASK;
2602	if (orig != data)
2603	WREG32(mmSPLL_CNTL_MODE, data);
2604	}
2605	}
2606	} else {
2607	if (orig != data)
2608	WREG32_PCIE_PORT(ixPCIE_LC_CNTL, data);
2609	}
2610
2611	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
2612	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK | PCIE_CNTL2__MST_MEM_LS_EN_MASK | PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
2613	if (orig != data)
2614	WREG32_PCIE(ixPCIE_CNTL2, data);
2615
2616	if (!disable_l0s) {
2617	data = RREG32_PCIE_PORT(ixPCIE_LC_N_FTS_CNTL);
2618	if((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) == PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
2619	data = RREG32_PCIE(ixPCIE_LC_STATUS1);
2620	if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) && (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
2621	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL);
2622	data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
2623	if (orig != data)
2624	WREG32_PCIE_PORT(ixPCIE_LC_CNTL, data);
2625	}
2626	}
2627	}
2628	}
2629
2630	static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
2631	{
2632	int readrq;
2633	u16 v;
2634
2635	readrq = pcie_get_readrq(dev: adev->pdev);
2636	v = ffs(readrq) - 8;
2637	if ((v == 0) || (v == 6) || (v == 7))
2638	pcie_set_readrq(dev: adev->pdev, rq: 512);
2639	}
2640
2641	static int si_common_hw_init(struct amdgpu_ip_block *ip_block)
2642	{
2643	struct amdgpu_device *adev = ip_block->adev;
2644
2645	si_fix_pci_max_read_req_size(adev);
2646	si_init_golden_registers(adev);
2647	si_pcie_gen3_enable(adev);
2648	si_program_aspm(adev);
2649
2650	return 0;
2651	}
2652
2653	static int si_common_hw_fini(struct amdgpu_ip_block *ip_block)
2654	{
2655	return 0;
2656	}
2657
2658	static int si_common_resume(struct amdgpu_ip_block *ip_block)
2659	{
2660	return si_common_hw_init(ip_block);
2661	}
2662
2663	static bool si_common_is_idle(struct amdgpu_ip_block *ip_block)
2664	{
2665	return true;
2666	}
2667
2668	static int si_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2669	enum amd_clockgating_state state)
2670	{
2671	return 0;
2672	}
2673
2674	static int si_common_set_powergating_state(struct amdgpu_ip_block *ip_block,
2675	enum amd_powergating_state state)
2676	{
2677	return 0;
2678	}
2679
2680	static const struct amd_ip_funcs si_common_ip_funcs = {
2681	.name = "si_common",
2682	.early_init = si_common_early_init,
2683	.hw_init = si_common_hw_init,
2684	.hw_fini = si_common_hw_fini,
2685	.resume = si_common_resume,
2686	.is_idle = si_common_is_idle,
2687	.set_clockgating_state = si_common_set_clockgating_state,
2688	.set_powergating_state = si_common_set_powergating_state,
2689	};
2690
2691	static const struct amdgpu_ip_block_version si_common_ip_block =
2692	{
2693	.type = AMD_IP_BLOCK_TYPE_COMMON,
2694	.major = 1,
2695	.minor = 0,
2696	.rev = 0,
2697	.funcs = &si_common_ip_funcs,
2698	};
2699
2700	int si_set_ip_blocks(struct amdgpu_device *adev)
2701	{
2702	switch (adev->asic_type) {
2703	case CHIP_VERDE:
2704	case CHIP_TAHITI:
2705	case CHIP_PITCAIRN:
2706	amdgpu_device_ip_block_add(adev, ip_block_version: &si_common_ip_block);
2707	amdgpu_device_ip_block_add(adev, ip_block_version: &gmc_v6_0_ip_block);
2708	amdgpu_device_ip_block_add(adev, ip_block_version: &si_ih_ip_block);
2709	amdgpu_device_ip_block_add(adev, ip_block_version: &gfx_v6_0_ip_block);
2710	amdgpu_device_ip_block_add(adev, ip_block_version: &si_dma_ip_block);
2711	amdgpu_device_ip_block_add(adev, ip_block_version: &si_smu_ip_block);
2712	if (adev->enable_virtual_display)
2713	amdgpu_device_ip_block_add(adev, ip_block_version: &amdgpu_vkms_ip_block);
2714	#if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)
2715	else if (amdgpu_device_has_dc_support(adev))
2716	amdgpu_device_ip_block_add(adev, ip_block_version: &dm_ip_block);
2717	#endif
2718	else
2719	amdgpu_device_ip_block_add(adev, ip_block_version: &dce_v6_0_ip_block);
2720	amdgpu_device_ip_block_add(adev, ip_block_version: &uvd_v3_1_ip_block);
2721	amdgpu_device_ip_block_add(adev, ip_block_version: &vce_v1_0_ip_block);
2722	break;
2723	case CHIP_OLAND:
2724	amdgpu_device_ip_block_add(adev, ip_block_version: &si_common_ip_block);
2725	amdgpu_device_ip_block_add(adev, ip_block_version: &gmc_v6_0_ip_block);
2726	amdgpu_device_ip_block_add(adev, ip_block_version: &si_ih_ip_block);
2727	amdgpu_device_ip_block_add(adev, ip_block_version: &gfx_v6_0_ip_block);
2728	amdgpu_device_ip_block_add(adev, ip_block_version: &si_dma_ip_block);
2729	amdgpu_device_ip_block_add(adev, ip_block_version: &si_smu_ip_block);
2730	if (adev->enable_virtual_display)
2731	amdgpu_device_ip_block_add(adev, ip_block_version: &amdgpu_vkms_ip_block);
2732	#if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)
2733	else if (amdgpu_device_has_dc_support(adev))
2734	amdgpu_device_ip_block_add(adev, ip_block_version: &dm_ip_block);
2735	#endif
2736	else
2737	amdgpu_device_ip_block_add(adev, ip_block_version: &dce_v6_4_ip_block);
2738	amdgpu_device_ip_block_add(adev, ip_block_version: &uvd_v3_1_ip_block);
2739	break;
2740	case CHIP_HAINAN:
2741	amdgpu_device_ip_block_add(adev, ip_block_version: &si_common_ip_block);
2742	amdgpu_device_ip_block_add(adev, ip_block_version: &gmc_v6_0_ip_block);
2743	amdgpu_device_ip_block_add(adev, ip_block_version: &si_ih_ip_block);
2744	amdgpu_device_ip_block_add(adev, ip_block_version: &gfx_v6_0_ip_block);
2745	amdgpu_device_ip_block_add(adev, ip_block_version: &si_dma_ip_block);
2746	amdgpu_device_ip_block_add(adev, ip_block_version: &si_smu_ip_block);
2747	if (adev->enable_virtual_display)
2748	amdgpu_device_ip_block_add(adev, ip_block_version: &amdgpu_vkms_ip_block);
2749	break;
2750	default:
2751	BUG();
2752	}
2753	return 0;
2754	}
2755
2756