1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24#include <linux/list.h>
25#include "amdgpu.h"
26#include "amdgpu_xgmi.h"
27#include "amdgpu_ras.h"
28#include "soc15.h"
29#include "df/df_3_6_offset.h"
30#include "xgmi/xgmi_4_0_0_smn.h"
31#include "xgmi/xgmi_4_0_0_sh_mask.h"
32#include "xgmi/xgmi_6_1_0_sh_mask.h"
33#include "wafl/wafl2_4_0_0_smn.h"
34#include "wafl/wafl2_4_0_0_sh_mask.h"
35
36#include "amdgpu_reset.h"
37
38#define smnPCS_XGMI3X16_PCS_ERROR_STATUS 0x11a0020c
39#define smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK 0x11a00218
40#define smnPCS_GOPX1_PCS_ERROR_STATUS 0x12200210
41#define smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK 0x12200218
42
43#define XGMI_STATE_DISABLE 0xD1
44#define XGMI_STATE_LS0 0x81
45#define XGMI_LINK_ACTIVE 1
46#define XGMI_LINK_INACTIVE 0
47
48static DEFINE_MUTEX(xgmi_mutex);
49
50#define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4
51
52static LIST_HEAD(xgmi_hive_list);
53
54static const int xgmi_pcs_err_status_reg_vg20[] = {
55 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
56 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
57};
58
59static const int wafl_pcs_err_status_reg_vg20[] = {
60 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
61 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
62};
63
64static const int xgmi_pcs_err_status_reg_arct[] = {
65 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
66 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
67 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x500000,
68 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x600000,
69 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x700000,
70 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x800000,
71};
72
73/* same as vg20*/
74static const int wafl_pcs_err_status_reg_arct[] = {
75 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
76 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
77};
78
79static const int xgmi3x16_pcs_err_status_reg_aldebaran[] = {
80 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
81 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000,
82 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x200000,
83 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x300000,
84 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x400000,
85 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x500000,
86 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x600000,
87 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x700000
88};
89
90static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
91 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
92 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000,
93 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x200000,
94 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x300000,
95 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x400000,
96 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x500000,
97 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x600000,
98 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x700000
99};
100
101static const int walf_pcs_err_status_reg_aldebaran[] = {
102 smnPCS_GOPX1_PCS_ERROR_STATUS,
103 smnPCS_GOPX1_PCS_ERROR_STATUS + 0x100000
104};
105
106static const int walf_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
107 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK,
108 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
109};
110
111static const int xgmi3x16_pcs_err_status_reg_v6_4[] = {
112 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
113 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000
114};
115
116static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[] = {
117 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
118 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
119};
120
121static const u64 xgmi_v6_4_0_mca_base_array[] = {
122 0x11a09200,
123 0x11b09200,
124};
125
126static const char *xgmi_v6_4_0_ras_error_code_ext[32] = {
127 [0x00] = "XGMI PCS DataLossErr",
128 [0x01] = "XGMI PCS TrainingErr",
129 [0x02] = "XGMI PCS FlowCtrlAckErr",
130 [0x03] = "XGMI PCS RxFifoUnderflowErr",
131 [0x04] = "XGMI PCS RxFifoOverflowErr",
132 [0x05] = "XGMI PCS CRCErr",
133 [0x06] = "XGMI PCS BERExceededErr",
134 [0x07] = "XGMI PCS TxMetaDataErr",
135 [0x08] = "XGMI PCS ReplayBufParityErr",
136 [0x09] = "XGMI PCS DataParityErr",
137 [0x0a] = "XGMI PCS ReplayFifoOverflowErr",
138 [0x0b] = "XGMI PCS ReplayFifoUnderflowErr",
139 [0x0c] = "XGMI PCS ElasticFifoOverflowErr",
140 [0x0d] = "XGMI PCS DeskewErr",
141 [0x0e] = "XGMI PCS FlowCtrlCRCErr",
142 [0x0f] = "XGMI PCS DataStartupLimitErr",
143 [0x10] = "XGMI PCS FCInitTimeoutErr",
144 [0x11] = "XGMI PCS RecoveryTimeoutErr",
145 [0x12] = "XGMI PCS ReadySerialTimeoutErr",
146 [0x13] = "XGMI PCS ReadySerialAttemptErr",
147 [0x14] = "XGMI PCS RecoveryAttemptErr",
148 [0x15] = "XGMI PCS RecoveryRelockAttemptErr",
149 [0x16] = "XGMI PCS ReplayAttemptErr",
150 [0x17] = "XGMI PCS SyncHdrErr",
151 [0x18] = "XGMI PCS TxReplayTimeoutErr",
152 [0x19] = "XGMI PCS RxReplayTimeoutErr",
153 [0x1a] = "XGMI PCS LinkSubTxTimeoutErr",
154 [0x1b] = "XGMI PCS LinkSubRxTimeoutErr",
155 [0x1c] = "XGMI PCS RxCMDPktErr",
156};
157
158static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = {
159 {"XGMI PCS DataLossErr",
160 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)},
161 {"XGMI PCS TrainingErr",
162 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TrainingErr)},
163 {"XGMI PCS CRCErr",
164 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, CRCErr)},
165 {"XGMI PCS BERExceededErr",
166 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, BERExceededErr)},
167 {"XGMI PCS TxMetaDataErr",
168 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TxMetaDataErr)},
169 {"XGMI PCS ReplayBufParityErr",
170 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayBufParityErr)},
171 {"XGMI PCS DataParityErr",
172 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataParityErr)},
173 {"XGMI PCS ReplayFifoOverflowErr",
174 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
175 {"XGMI PCS ReplayFifoUnderflowErr",
176 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
177 {"XGMI PCS ElasticFifoOverflowErr",
178 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
179 {"XGMI PCS DeskewErr",
180 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DeskewErr)},
181 {"XGMI PCS DataStartupLimitErr",
182 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataStartupLimitErr)},
183 {"XGMI PCS FCInitTimeoutErr",
184 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
185 {"XGMI PCS RecoveryTimeoutErr",
186 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
187 {"XGMI PCS ReadySerialTimeoutErr",
188 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
189 {"XGMI PCS ReadySerialAttemptErr",
190 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
191 {"XGMI PCS RecoveryAttemptErr",
192 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
193 {"XGMI PCS RecoveryRelockAttemptErr",
194 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
195};
196
197static const struct amdgpu_pcs_ras_field wafl_pcs_ras_fields[] = {
198 {"WAFL PCS DataLossErr",
199 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataLossErr)},
200 {"WAFL PCS TrainingErr",
201 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TrainingErr)},
202 {"WAFL PCS CRCErr",
203 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, CRCErr)},
204 {"WAFL PCS BERExceededErr",
205 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, BERExceededErr)},
206 {"WAFL PCS TxMetaDataErr",
207 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TxMetaDataErr)},
208 {"WAFL PCS ReplayBufParityErr",
209 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayBufParityErr)},
210 {"WAFL PCS DataParityErr",
211 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataParityErr)},
212 {"WAFL PCS ReplayFifoOverflowErr",
213 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
214 {"WAFL PCS ReplayFifoUnderflowErr",
215 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
216 {"WAFL PCS ElasticFifoOverflowErr",
217 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
218 {"WAFL PCS DeskewErr",
219 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DeskewErr)},
220 {"WAFL PCS DataStartupLimitErr",
221 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataStartupLimitErr)},
222 {"WAFL PCS FCInitTimeoutErr",
223 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, FCInitTimeoutErr)},
224 {"WAFL PCS RecoveryTimeoutErr",
225 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
226 {"WAFL PCS ReadySerialTimeoutErr",
227 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
228 {"WAFL PCS ReadySerialAttemptErr",
229 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
230 {"WAFL PCS RecoveryAttemptErr",
231 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryAttemptErr)},
232 {"WAFL PCS RecoveryRelockAttemptErr",
233 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
234};
235
236static const struct amdgpu_pcs_ras_field xgmi3x16_pcs_ras_fields[] = {
237 {"XGMI3X16 PCS DataLossErr",
238 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataLossErr)},
239 {"XGMI3X16 PCS TrainingErr",
240 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TrainingErr)},
241 {"XGMI3X16 PCS FlowCtrlAckErr",
242 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlAckErr)},
243 {"XGMI3X16 PCS RxFifoUnderflowErr",
244 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoUnderflowErr)},
245 {"XGMI3X16 PCS RxFifoOverflowErr",
246 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoOverflowErr)},
247 {"XGMI3X16 PCS CRCErr",
248 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, CRCErr)},
249 {"XGMI3X16 PCS BERExceededErr",
250 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, BERExceededErr)},
251 {"XGMI3X16 PCS TxVcidDataErr",
252 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxVcidDataErr)},
253 {"XGMI3X16 PCS ReplayBufParityErr",
254 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayBufParityErr)},
255 {"XGMI3X16 PCS DataParityErr",
256 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataParityErr)},
257 {"XGMI3X16 PCS ReplayFifoOverflowErr",
258 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
259 {"XGMI3X16 PCS ReplayFifoUnderflowErr",
260 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
261 {"XGMI3X16 PCS ElasticFifoOverflowErr",
262 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
263 {"XGMI3X16 PCS DeskewErr",
264 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DeskewErr)},
265 {"XGMI3X16 PCS FlowCtrlCRCErr",
266 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlCRCErr)},
267 {"XGMI3X16 PCS DataStartupLimitErr",
268 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataStartupLimitErr)},
269 {"XGMI3X16 PCS FCInitTimeoutErr",
270 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
271 {"XGMI3X16 PCS RecoveryTimeoutErr",
272 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
273 {"XGMI3X16 PCS ReadySerialTimeoutErr",
274 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
275 {"XGMI3X16 PCS ReadySerialAttemptErr",
276 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
277 {"XGMI3X16 PCS RecoveryAttemptErr",
278 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
279 {"XGMI3X16 PCS RecoveryRelockAttemptErr",
280 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
281 {"XGMI3X16 PCS ReplayAttemptErr",
282 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayAttemptErr)},
283 {"XGMI3X16 PCS SyncHdrErr",
284 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, SyncHdrErr)},
285 {"XGMI3X16 PCS TxReplayTimeoutErr",
286 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxReplayTimeoutErr)},
287 {"XGMI3X16 PCS RxReplayTimeoutErr",
288 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxReplayTimeoutErr)},
289 {"XGMI3X16 PCS LinkSubTxTimeoutErr",
290 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubTxTimeoutErr)},
291 {"XGMI3X16 PCS LinkSubRxTimeoutErr",
292 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubRxTimeoutErr)},
293 {"XGMI3X16 PCS RxCMDPktErr",
294 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxCMDPktErr)},
295};
296
297int amdgpu_xgmi_get_ext_link(struct amdgpu_device *adev, int link_num)
298{
299 int link_map_6_4_x[8] = { 0, 3, 1, 2, 7, 6, 4, 5 };
300
301 if (adev->gmc.xgmi.num_physical_nodes <= 1)
302 return -EINVAL;
303
304 switch (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0)) {
305 case IP_VERSION(6, 4, 0):
306 case IP_VERSION(6, 4, 1):
307 if (link_num < ARRAY_SIZE(link_map_6_4_x))
308 return link_map_6_4_x[link_num];
309 break;
310 default:
311 return -EINVAL;
312 }
313
314 return -EINVAL;
315}
316
317static u32 xgmi_v6_4_get_link_status(struct amdgpu_device *adev, int global_link_num)
318{
319 const u32 smn_xgmi_6_4_pcs_state_hist1[2] = { 0x11a00070, 0x11b00070 };
320 const u32 smn_xgmi_6_4_1_pcs_state_hist1[2] = { 0x12100070,
321 0x11b00070 };
322 u32 i, n;
323 u64 addr;
324
325 switch (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0)) {
326 case IP_VERSION(6, 4, 0):
327 n = ARRAY_SIZE(smn_xgmi_6_4_pcs_state_hist1);
328 addr = smn_xgmi_6_4_pcs_state_hist1[global_link_num % n];
329 break;
330 case IP_VERSION(6, 4, 1):
331 n = ARRAY_SIZE(smn_xgmi_6_4_1_pcs_state_hist1);
332 addr = smn_xgmi_6_4_1_pcs_state_hist1[global_link_num % n];
333 break;
334 default:
335 return U32_MAX;
336 }
337
338 i = global_link_num / n;
339
340 if (!(adev->aid_mask & BIT(i)))
341 return U32_MAX;
342
343 addr += adev->asic_funcs->encode_ext_smn_addressing(i);
344
345 return RREG32_PCIE_EXT(addr);
346}
347
348int amdgpu_get_xgmi_link_status(struct amdgpu_device *adev, int global_link_num)
349{
350 u32 xgmi_state_reg_val;
351
352 if (adev->gmc.xgmi.num_physical_nodes <= 1)
353 return -EINVAL;
354
355 switch (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0)) {
356 case IP_VERSION(6, 4, 0):
357 case IP_VERSION(6, 4, 1):
358 xgmi_state_reg_val = xgmi_v6_4_get_link_status(adev, global_link_num);
359 break;
360 default:
361 return -EOPNOTSUPP;
362 }
363
364 if ((xgmi_state_reg_val & 0xFF) == XGMI_STATE_DISABLE)
365 return -ENOLINK;
366
367 if ((xgmi_state_reg_val & 0xFF) == XGMI_STATE_LS0)
368 return XGMI_LINK_ACTIVE;
369
370 return XGMI_LINK_INACTIVE;
371}
372
373/**
374 * DOC: AMDGPU XGMI Support
375 *
376 * XGMI is a high speed interconnect that joins multiple GPU cards
377 * into a homogeneous memory space that is organized by a collective
378 * hive ID and individual node IDs, both of which are 64-bit numbers.
379 *
380 * The file xgmi_device_id contains the unique per GPU device ID and
381 * is stored in the /sys/class/drm/card${cardno}/device/ directory.
382 *
383 * Inside the device directory a sub-directory 'xgmi_hive_info' is
384 * created which contains the hive ID and the list of nodes.
385 *
386 * The hive ID is stored in:
387 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
388 *
389 * The node information is stored in numbered directories:
390 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
391 *
392 * Each device has their own xgmi_hive_info direction with a mirror
393 * set of node sub-directories.
394 *
395 * The XGMI memory space is built by contiguously adding the power of
396 * two padded VRAM space from each node to each other.
397 *
398 */
399
400static struct attribute amdgpu_xgmi_hive_id = {
401 .name = "xgmi_hive_id",
402 .mode = S_IRUGO
403};
404
405static struct attribute *amdgpu_xgmi_hive_attrs[] = {
406 &amdgpu_xgmi_hive_id,
407 NULL
408};
409ATTRIBUTE_GROUPS(amdgpu_xgmi_hive);
410
411static ssize_t amdgpu_xgmi_show_attrs(struct kobject *kobj,
412 struct attribute *attr, char *buf)
413{
414 struct amdgpu_hive_info *hive = container_of(
415 kobj, struct amdgpu_hive_info, kobj);
416
417 if (attr == &amdgpu_xgmi_hive_id)
418 return snprintf(buf, PAGE_SIZE, fmt: "%llu\n", hive->hive_id);
419
420 return 0;
421}
422
423static void amdgpu_xgmi_hive_release(struct kobject *kobj)
424{
425 struct amdgpu_hive_info *hive = container_of(
426 kobj, struct amdgpu_hive_info, kobj);
427
428 amdgpu_reset_put_reset_domain(domain: hive->reset_domain);
429 hive->reset_domain = NULL;
430
431 mutex_destroy(lock: &hive->hive_lock);
432 kfree(objp: hive);
433}
434
435static const struct sysfs_ops amdgpu_xgmi_hive_ops = {
436 .show = amdgpu_xgmi_show_attrs,
437};
438
439static const struct kobj_type amdgpu_xgmi_hive_type = {
440 .release = amdgpu_xgmi_hive_release,
441 .sysfs_ops = &amdgpu_xgmi_hive_ops,
442 .default_groups = amdgpu_xgmi_hive_groups,
443};
444
445static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
446 struct device_attribute *attr,
447 char *buf)
448{
449 struct drm_device *ddev = dev_get_drvdata(dev);
450 struct amdgpu_device *adev = drm_to_adev(ddev);
451
452 return sysfs_emit(buf, fmt: "%llu\n", adev->gmc.xgmi.node_id);
453
454}
455
456static ssize_t amdgpu_xgmi_show_physical_id(struct device *dev,
457 struct device_attribute *attr,
458 char *buf)
459{
460 struct drm_device *ddev = dev_get_drvdata(dev);
461 struct amdgpu_device *adev = drm_to_adev(ddev);
462
463 return sysfs_emit(buf, fmt: "%u\n", adev->gmc.xgmi.physical_node_id);
464
465}
466
467static ssize_t amdgpu_xgmi_show_num_hops(struct device *dev,
468 struct device_attribute *attr,
469 char *buf)
470{
471 struct drm_device *ddev = dev_get_drvdata(dev);
472 struct amdgpu_device *adev = drm_to_adev(ddev);
473 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
474 int i;
475
476 for (i = 0; i < top->num_nodes; i++)
477 sprintf(buf: buf + 3 * i, fmt: "%02x ", top->nodes[i].num_hops);
478
479 return sysfs_emit(buf, fmt: "%s\n", buf);
480}
481
482static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,
483 struct device_attribute *attr,
484 char *buf)
485{
486 struct drm_device *ddev = dev_get_drvdata(dev);
487 struct amdgpu_device *adev = drm_to_adev(ddev);
488 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
489 int i;
490
491 for (i = 0; i < top->num_nodes; i++)
492 sprintf(buf: buf + 3 * i, fmt: "%02x ", top->nodes[i].num_links);
493
494 return sysfs_emit(buf, fmt: "%s\n", buf);
495}
496
497static ssize_t amdgpu_xgmi_show_connected_port_num(struct device *dev,
498 struct device_attribute *attr,
499 char *buf)
500{
501 struct drm_device *ddev = dev_get_drvdata(dev);
502 struct amdgpu_device *adev = drm_to_adev(ddev);
503 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
504 int i, j, size = 0;
505 int current_node;
506 /*
507 * get the node id in the sysfs for the current socket and show
508 * it in the port num info output in the sysfs for easy reading.
509 * it is NOT the one retrieved from xgmi ta.
510 */
511 for (i = 0; i < top->num_nodes; i++) {
512 if (top->nodes[i].node_id == adev->gmc.xgmi.node_id) {
513 current_node = i;
514 break;
515 }
516 }
517
518 if (i == top->num_nodes)
519 return -EINVAL;
520
521 for (i = 0; i < top->num_nodes; i++) {
522 for (j = 0; j < top->nodes[i].num_links; j++)
523 /* node id in sysfs starts from 1 rather than 0 so +1 here */
524 size += sysfs_emit_at(buf, at: size, fmt: "%02x:%02x -> %02x:%02x\n", current_node + 1,
525 top->nodes[i].port_num[j].src_xgmi_port_num, i + 1,
526 top->nodes[i].port_num[j].dst_xgmi_port_num);
527 }
528
529 return size;
530}
531
532#define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801)
533static ssize_t amdgpu_xgmi_show_error(struct device *dev,
534 struct device_attribute *attr,
535 char *buf)
536{
537 struct drm_device *ddev = dev_get_drvdata(dev);
538 struct amdgpu_device *adev = drm_to_adev(ddev);
539 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;
540 uint64_t fica_out;
541 unsigned int error_count = 0;
542
543 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);
544 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);
545
546 if ((!adev->df.funcs) ||
547 (!adev->df.funcs->get_fica) ||
548 (!adev->df.funcs->set_fica))
549 return -EINVAL;
550
551 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_ctl_in);
552 if (fica_out != 0x1f)
553 pr_err("xGMI error counters not enabled!\n");
554
555 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_status_in);
556
557 if ((fica_out & 0xffff) == 2)
558 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);
559
560 adev->df.funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);
561
562 return sysfs_emit(buf, fmt: "%u\n", error_count);
563}
564
565
566static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
567static DEVICE_ATTR(xgmi_physical_id, 0444, amdgpu_xgmi_show_physical_id, NULL);
568static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
569static DEVICE_ATTR(xgmi_num_hops, S_IRUGO, amdgpu_xgmi_show_num_hops, NULL);
570static DEVICE_ATTR(xgmi_num_links, S_IRUGO, amdgpu_xgmi_show_num_links, NULL);
571static DEVICE_ATTR(xgmi_port_num, S_IRUGO, amdgpu_xgmi_show_connected_port_num, NULL);
572
573static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
574 struct amdgpu_hive_info *hive)
575{
576 int ret = 0;
577 char node[10] = { 0 };
578
579 /* Create xgmi device id file */
580 ret = device_create_file(device: adev->dev, entry: &dev_attr_xgmi_device_id);
581 if (ret) {
582 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
583 return ret;
584 }
585
586 ret = device_create_file(device: adev->dev, entry: &dev_attr_xgmi_physical_id);
587 if (ret) {
588 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_physical_id\n");
589 return ret;
590 }
591
592 /* Create xgmi error file */
593 ret = device_create_file(device: adev->dev, entry: &dev_attr_xgmi_error);
594 if (ret)
595 pr_err("failed to create xgmi_error\n");
596
597 /* Create xgmi num hops file */
598 ret = device_create_file(device: adev->dev, entry: &dev_attr_xgmi_num_hops);
599 if (ret)
600 pr_err("failed to create xgmi_num_hops\n");
601
602 /* Create xgmi num links file */
603 ret = device_create_file(device: adev->dev, entry: &dev_attr_xgmi_num_links);
604 if (ret)
605 pr_err("failed to create xgmi_num_links\n");
606
607 /* Create xgmi port num file if supported */
608 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
609 ret = device_create_file(device: adev->dev, entry: &dev_attr_xgmi_port_num);
610 if (ret)
611 dev_err(adev->dev, "failed to create xgmi_port_num\n");
612 }
613
614 /* Create sysfs link to hive info folder on the first device */
615 if (hive->kobj.parent != (&adev->dev->kobj)) {
616 ret = sysfs_create_link(kobj: &adev->dev->kobj, target: &hive->kobj,
617 name: "xgmi_hive_info");
618 if (ret) {
619 dev_err(adev->dev, "XGMI: Failed to create link to hive info");
620 goto remove_file;
621 }
622 }
623
624 sprintf(buf: node, fmt: "node%d", atomic_read(v: &hive->number_devices));
625 /* Create sysfs link form the hive folder to yourself */
626 ret = sysfs_create_link(kobj: &hive->kobj, target: &adev->dev->kobj, name: node);
627 if (ret) {
628 dev_err(adev->dev, "XGMI: Failed to create link from hive info");
629 goto remove_link;
630 }
631
632 goto success;
633
634
635remove_link:
636 sysfs_remove_link(kobj: &adev->dev->kobj, name: adev_to_drm(adev)->unique);
637
638remove_file:
639 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_device_id);
640 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_physical_id);
641 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_error);
642 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_num_hops);
643 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_num_links);
644 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
645 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_port_num);
646
647success:
648 return ret;
649}
650
651static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
652 struct amdgpu_hive_info *hive)
653{
654 char node[10];
655 memset(node, 0, sizeof(node));
656
657 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_device_id);
658 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_physical_id);
659 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_error);
660 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_num_hops);
661 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_num_links);
662 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
663 device_remove_file(dev: adev->dev, attr: &dev_attr_xgmi_port_num);
664
665 if (hive->kobj.parent != (&adev->dev->kobj))
666 sysfs_remove_link(kobj: &adev->dev->kobj,name: "xgmi_hive_info");
667
668 sprintf(buf: node, fmt: "node%d", atomic_read(v: &hive->number_devices));
669 sysfs_remove_link(kobj: &hive->kobj, name: node);
670
671}
672
673
674
675struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
676{
677 struct amdgpu_hive_info *hive = NULL;
678 int ret;
679
680 if (!adev->gmc.xgmi.hive_id)
681 return NULL;
682
683 if (adev->hive) {
684 kobject_get(kobj: &adev->hive->kobj);
685 return adev->hive;
686 }
687
688 mutex_lock(&xgmi_mutex);
689
690 list_for_each_entry(hive, &xgmi_hive_list, node) {
691 if (hive->hive_id == adev->gmc.xgmi.hive_id)
692 goto pro_end;
693 }
694
695 hive = kzalloc(sizeof(*hive), GFP_KERNEL);
696 if (!hive) {
697 dev_err(adev->dev, "XGMI: allocation failed\n");
698 ret = -ENOMEM;
699 hive = NULL;
700 goto pro_end;
701 }
702
703 /* initialize new hive if not exist */
704 ret = kobject_init_and_add(kobj: &hive->kobj,
705 ktype: &amdgpu_xgmi_hive_type,
706 parent: &adev->dev->kobj,
707 fmt: "%s", "xgmi_hive_info");
708 if (ret) {
709 dev_err(adev->dev, "XGMI: failed initializing kobject for xgmi hive\n");
710 kobject_put(kobj: &hive->kobj);
711 hive = NULL;
712 goto pro_end;
713 }
714
715 /**
716 * Only init hive->reset_domain for none SRIOV configuration. For SRIOV,
717 * Host driver decide how to reset the GPU either through FLR or chain reset.
718 * Guest side will get individual notifications from the host for the FLR
719 * if necessary.
720 */
721 if (!amdgpu_sriov_vf(adev)) {
722 /**
723 * Avoid recreating reset domain when hive is reconstructed for the case
724 * of reset the devices in the XGMI hive during probe for passthrough GPU
725 * See https://www.spinics.net/lists/amd-gfx/msg58836.html
726 */
727 if (adev->reset_domain->type != XGMI_HIVE) {
728 hive->reset_domain =
729 amdgpu_reset_create_reset_domain(type: XGMI_HIVE, wq_name: "amdgpu-reset-hive");
730 if (!hive->reset_domain) {
731 dev_err(adev->dev, "XGMI: failed initializing reset domain for xgmi hive\n");
732 ret = -ENOMEM;
733 kobject_put(kobj: &hive->kobj);
734 hive = NULL;
735 goto pro_end;
736 }
737 } else {
738 amdgpu_reset_get_reset_domain(domain: adev->reset_domain);
739 hive->reset_domain = adev->reset_domain;
740 }
741 }
742
743 hive->hive_id = adev->gmc.xgmi.hive_id;
744 INIT_LIST_HEAD(list: &hive->device_list);
745 INIT_LIST_HEAD(list: &hive->node);
746 mutex_init(&hive->hive_lock);
747 atomic_set(v: &hive->number_devices, i: 0);
748 task_barrier_init(tb: &hive->tb);
749 hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN;
750 hive->hi_req_gpu = NULL;
751 atomic_set(v: &hive->requested_nps_mode, i: UNKNOWN_MEMORY_PARTITION_MODE);
752
753 /*
754 * hive pstate on boot is high in vega20 so we have to go to low
755 * pstate on after boot.
756 */
757 hive->hi_req_count = AMDGPU_MAX_XGMI_DEVICE_PER_HIVE;
758 list_add_tail(new: &hive->node, head: &xgmi_hive_list);
759
760pro_end:
761 if (hive)
762 kobject_get(kobj: &hive->kobj);
763 mutex_unlock(lock: &xgmi_mutex);
764 return hive;
765}
766
767void amdgpu_put_xgmi_hive(struct amdgpu_hive_info *hive)
768{
769 if (hive)
770 kobject_put(kobj: &hive->kobj);
771}
772
773int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
774{
775 int ret = 0;
776 struct amdgpu_hive_info *hive;
777 struct amdgpu_device *request_adev;
778 bool is_hi_req = pstate == AMDGPU_XGMI_PSTATE_MAX_VEGA20;
779 bool init_low;
780
781 hive = amdgpu_get_xgmi_hive(adev);
782 if (!hive)
783 return 0;
784
785 request_adev = hive->hi_req_gpu ? hive->hi_req_gpu : adev;
786 init_low = hive->pstate == AMDGPU_XGMI_PSTATE_UNKNOWN;
787 amdgpu_put_xgmi_hive(hive);
788 /* fw bug so temporarily disable pstate switching */
789 return 0;
790
791 if (!hive || adev->asic_type != CHIP_VEGA20)
792 return 0;
793
794 mutex_lock(&hive->hive_lock);
795
796 if (is_hi_req)
797 hive->hi_req_count++;
798 else
799 hive->hi_req_count--;
800
801 /*
802 * Vega20 only needs single peer to request pstate high for the hive to
803 * go high but all peers must request pstate low for the hive to go low
804 */
805 if (hive->pstate == pstate ||
806 (!is_hi_req && hive->hi_req_count && !init_low))
807 goto out;
808
809 dev_dbg(request_adev->dev, "Set xgmi pstate %d.\n", pstate);
810
811 ret = amdgpu_dpm_set_xgmi_pstate(adev: request_adev, pstate);
812 if (ret) {
813 dev_err(request_adev->dev,
814 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
815 request_adev->gmc.xgmi.node_id,
816 request_adev->gmc.xgmi.hive_id, ret);
817 goto out;
818 }
819
820 if (init_low)
821 hive->pstate = hive->hi_req_count ?
822 hive->pstate : AMDGPU_XGMI_PSTATE_MIN;
823 else {
824 hive->pstate = pstate;
825 hive->hi_req_gpu = pstate != AMDGPU_XGMI_PSTATE_MIN ?
826 adev : NULL;
827 }
828out:
829 mutex_unlock(lock: &hive->hive_lock);
830 return ret;
831}
832
833int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
834{
835 int ret;
836
837 if (amdgpu_sriov_vf(adev))
838 return 0;
839
840 /* Each psp need to set the latest topology */
841 ret = psp_xgmi_set_topology_info(psp: &adev->psp,
842 number_devices: atomic_read(v: &hive->number_devices),
843 topology: &adev->psp.xgmi_context.top_info);
844 if (ret)
845 dev_err(adev->dev,
846 "XGMI: Set topology failure on device %llx, hive %llx, ret %d",
847 adev->gmc.xgmi.node_id,
848 adev->gmc.xgmi.hive_id, ret);
849
850 return ret;
851}
852
853
854/*
855 * NOTE psp_xgmi_node_info.num_hops layout is as follows:
856 * num_hops[7:6] = link type (0 = xGMI2, 1 = xGMI3, 2/3 = reserved)
857 * num_hops[5:3] = reserved
858 * num_hops[2:0] = number of hops
859 */
860int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
861 struct amdgpu_device *peer_adev)
862{
863 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
864 uint8_t num_hops_mask = 0x7;
865 int i;
866
867 if (!adev->gmc.xgmi.supported)
868 return 0;
869
870 for (i = 0 ; i < top->num_nodes; ++i)
871 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
872 return top->nodes[i].num_hops & num_hops_mask;
873
874 dev_err(adev->dev, "Failed to get xgmi hops count for peer %d.\n",
875 peer_adev->gmc.xgmi.physical_node_id);
876
877 return 0;
878}
879
880int amdgpu_xgmi_get_bandwidth(struct amdgpu_device *adev, struct amdgpu_device *peer_adev,
881 enum amdgpu_xgmi_bw_mode bw_mode, enum amdgpu_xgmi_bw_unit bw_unit,
882 uint32_t *min_bw, uint32_t *max_bw)
883{
884 bool peer_mode = bw_mode == AMDGPU_XGMI_BW_MODE_PER_PEER;
885 int unit_scale = bw_unit == AMDGPU_XGMI_BW_UNIT_MBYTES ? 1000 : 1;
886 int num_lanes = adev->gmc.xgmi.max_width;
887 int speed = adev->gmc.xgmi.max_speed;
888 int num_links = !peer_mode ? 1 : -1;
889
890 if (!(min_bw && max_bw))
891 return -EINVAL;
892
893 *min_bw = 0;
894 *max_bw = 0;
895
896 if (!adev->gmc.xgmi.supported)
897 return -ENODATA;
898
899 if (peer_mode && !peer_adev)
900 return -EINVAL;
901
902 if (peer_mode) {
903 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
904 int i;
905
906 for (i = 0 ; i < top->num_nodes; ++i) {
907 if (top->nodes[i].node_id != peer_adev->gmc.xgmi.node_id)
908 continue;
909
910 num_links = top->nodes[i].num_links;
911 break;
912 }
913 }
914
915 if (num_links == -1) {
916 dev_err(adev->dev, "Failed to get number of xgmi links for peer %d.\n",
917 peer_adev->gmc.xgmi.physical_node_id);
918 } else if (num_links) {
919 int per_link_bw = (speed * num_lanes * unit_scale)/BITS_PER_BYTE;
920
921 *min_bw = per_link_bw;
922 *max_bw = num_links * per_link_bw;
923 }
924
925 return 0;
926}
927
928bool amdgpu_xgmi_get_is_sharing_enabled(struct amdgpu_device *adev,
929 struct amdgpu_device *peer_adev)
930{
931 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
932 int i;
933
934 /* Sharing should always be enabled for non-SRIOV. */
935 if (!amdgpu_sriov_vf(adev))
936 return true;
937
938 for (i = 0 ; i < top->num_nodes; ++i)
939 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
940 return !!top->nodes[i].is_sharing_enabled;
941
942 return false;
943}
944
945/*
946 * Devices that support extended data require the entire hive to initialize with
947 * the shared memory buffer flag set.
948 *
949 * Hive locks and conditions apply - see amdgpu_xgmi_add_device
950 */
951static int amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info *hive,
952 bool set_extended_data)
953{
954 struct amdgpu_device *tmp_adev;
955 int ret;
956
957 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
958 ret = psp_xgmi_initialize(psp: &tmp_adev->psp, set_extended_data, load_ta: false);
959 if (ret) {
960 dev_err(tmp_adev->dev,
961 "XGMI: Failed to initialize xgmi session for data partition %i\n",
962 set_extended_data);
963 return ret;
964 }
965
966 }
967
968 return 0;
969}
970
971int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
972{
973 struct psp_xgmi_topology_info *top_info;
974 struct amdgpu_hive_info *hive;
975 struct amdgpu_xgmi *entry;
976 struct amdgpu_device *tmp_adev = NULL;
977
978 int count = 0, ret = 0;
979
980 if (!adev->gmc.xgmi.supported)
981 return 0;
982
983 if (amdgpu_device_ip_get_ip_block(adev, type: AMD_IP_BLOCK_TYPE_PSP)) {
984 ret = psp_xgmi_initialize(psp: &adev->psp, set_extended_data: false, load_ta: true);
985 if (ret) {
986 dev_err(adev->dev,
987 "XGMI: Failed to initialize xgmi session\n");
988 return ret;
989 }
990
991 ret = psp_xgmi_get_hive_id(psp: &adev->psp, hive_id: &adev->gmc.xgmi.hive_id);
992 if (ret) {
993 dev_err(adev->dev,
994 "XGMI: Failed to get hive id\n");
995 return ret;
996 }
997
998 ret = psp_xgmi_get_node_id(psp: &adev->psp, node_id: &adev->gmc.xgmi.node_id);
999 if (ret) {
1000 dev_err(adev->dev,
1001 "XGMI: Failed to get node id\n");
1002 return ret;
1003 }
1004 } else {
1005 adev->gmc.xgmi.hive_id = 16;
1006 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;
1007 }
1008
1009 hive = amdgpu_get_xgmi_hive(adev);
1010 if (!hive) {
1011 ret = -EINVAL;
1012 dev_err(adev->dev,
1013 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",
1014 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);
1015 goto exit;
1016 }
1017 mutex_lock(&hive->hive_lock);
1018
1019 top_info = &adev->psp.xgmi_context.top_info;
1020
1021 list_add_tail(new: &adev->gmc.xgmi.head, head: &hive->device_list);
1022 list_for_each_entry(entry, &hive->device_list, head)
1023 top_info->nodes[count++].node_id = entry->node_id;
1024 top_info->num_nodes = count;
1025 atomic_set(v: &hive->number_devices, i: count);
1026
1027 task_barrier_add_task(tb: &hive->tb);
1028
1029 if (amdgpu_device_ip_get_ip_block(adev, type: AMD_IP_BLOCK_TYPE_PSP)) {
1030 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1031 /* update node list for other device in the hive */
1032 if (tmp_adev != adev) {
1033 top_info = &tmp_adev->psp.xgmi_context.top_info;
1034 top_info->nodes[count - 1].node_id =
1035 adev->gmc.xgmi.node_id;
1036 top_info->num_nodes = count;
1037 }
1038 ret = amdgpu_xgmi_update_topology(hive, adev: tmp_adev);
1039 if (ret)
1040 goto exit_unlock;
1041 }
1042
1043 if (amdgpu_sriov_vf(adev) &&
1044 adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
1045 /* only get topology for VF being init if it can support full duplex */
1046 ret = psp_xgmi_get_topology_info(psp: &adev->psp, number_devices: count,
1047 topology: &adev->psp.xgmi_context.top_info, get_extended_data: false);
1048 if (ret) {
1049 dev_err(adev->dev,
1050 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
1051 adev->gmc.xgmi.node_id,
1052 adev->gmc.xgmi.hive_id, ret);
1053 /* To do: continue with some node failed or disable the whole hive*/
1054 goto exit_unlock;
1055 }
1056 } else {
1057 /* get latest topology info for each device from psp */
1058 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1059 ret = psp_xgmi_get_topology_info(psp: &tmp_adev->psp, number_devices: count,
1060 topology: &tmp_adev->psp.xgmi_context.top_info, get_extended_data: false);
1061 if (ret) {
1062 dev_err(tmp_adev->dev,
1063 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
1064 tmp_adev->gmc.xgmi.node_id,
1065 tmp_adev->gmc.xgmi.hive_id, ret);
1066 /* To do : continue with some node failed or disable the whole hive */
1067 goto exit_unlock;
1068 }
1069 }
1070 }
1071
1072 /* get topology again for hives that support extended data */
1073 if (adev->psp.xgmi_context.supports_extended_data) {
1074
1075 /* initialize the hive to get extended data. */
1076 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, set_extended_data: true);
1077 if (ret)
1078 goto exit_unlock;
1079
1080 /* get the extended data. */
1081 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1082 ret = psp_xgmi_get_topology_info(psp: &tmp_adev->psp, number_devices: count,
1083 topology: &tmp_adev->psp.xgmi_context.top_info, get_extended_data: true);
1084 if (ret) {
1085 dev_err(tmp_adev->dev,
1086 "XGMI: Get topology for extended data failure on device %llx, hive %llx, ret %d",
1087 tmp_adev->gmc.xgmi.node_id,
1088 tmp_adev->gmc.xgmi.hive_id, ret);
1089 goto exit_unlock;
1090 }
1091 }
1092
1093 /* initialize the hive to get non-extended data for the next round. */
1094 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, set_extended_data: false);
1095 if (ret)
1096 goto exit_unlock;
1097
1098 }
1099 }
1100
1101 if (!ret)
1102 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
1103
1104exit_unlock:
1105 mutex_unlock(lock: &hive->hive_lock);
1106exit:
1107 if (!ret) {
1108 adev->hive = hive;
1109 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
1110 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
1111 } else {
1112 amdgpu_put_xgmi_hive(hive);
1113 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
1114 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
1115 ret);
1116 }
1117
1118 return ret;
1119}
1120
1121int amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
1122{
1123 struct amdgpu_hive_info *hive = adev->hive;
1124
1125 if (!adev->gmc.xgmi.supported)
1126 return -EINVAL;
1127
1128 if (!hive)
1129 return -EINVAL;
1130
1131 mutex_lock(&hive->hive_lock);
1132 task_barrier_rem_task(tb: &hive->tb);
1133 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
1134 if (hive->hi_req_gpu == adev)
1135 hive->hi_req_gpu = NULL;
1136 list_del(entry: &adev->gmc.xgmi.head);
1137 mutex_unlock(lock: &hive->hive_lock);
1138
1139 amdgpu_put_xgmi_hive(hive);
1140 adev->hive = NULL;
1141
1142 if (atomic_dec_return(v: &hive->number_devices) == 0) {
1143 /* Remove the hive from global hive list */
1144 mutex_lock(&xgmi_mutex);
1145 list_del(entry: &hive->node);
1146 mutex_unlock(lock: &xgmi_mutex);
1147
1148 amdgpu_put_xgmi_hive(hive);
1149 }
1150
1151 return 0;
1152}
1153
1154static int xgmi_v6_4_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
1155 enum aca_smu_type type, void *data)
1156{
1157 struct amdgpu_device *adev = handle->adev;
1158 struct aca_bank_info info;
1159 const char *error_str;
1160 u64 status, count;
1161 int ret, ext_error_code;
1162
1163 ret = aca_bank_info_decode(bank, info: &info);
1164 if (ret)
1165 return ret;
1166
1167 status = bank->regs[ACA_REG_IDX_STATUS];
1168 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1169
1170 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1171 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1172 if (error_str)
1173 dev_info(adev->dev, "%s detected\n", error_str);
1174
1175 count = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]);
1176
1177 switch (type) {
1178 case ACA_SMU_TYPE_UE:
1179 if (ext_error_code != 0 && ext_error_code != 9)
1180 count = 0ULL;
1181
1182 bank->aca_err_type = ACA_ERROR_TYPE_UE;
1183 ret = aca_error_cache_log_bank_error(handle, info: &info, type: ACA_ERROR_TYPE_UE, count);
1184 break;
1185 case ACA_SMU_TYPE_CE:
1186 count = ext_error_code == 6 ? count : 0ULL;
1187 bank->aca_err_type = ACA_ERROR_TYPE_CE;
1188 ret = aca_error_cache_log_bank_error(handle, info: &info, type: bank->aca_err_type, count);
1189 break;
1190 default:
1191 return -EINVAL;
1192 }
1193
1194 return ret;
1195}
1196
1197static const struct aca_bank_ops xgmi_v6_4_0_aca_bank_ops = {
1198 .aca_bank_parser = xgmi_v6_4_0_aca_bank_parser,
1199};
1200
1201static const struct aca_info xgmi_v6_4_0_aca_info = {
1202 .hwip = ACA_HWIP_TYPE_PCS_XGMI,
1203 .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
1204 .bank_ops = &xgmi_v6_4_0_aca_bank_ops,
1205};
1206
1207static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1208{
1209 int r;
1210
1211 if (!adev->gmc.xgmi.supported ||
1212 adev->gmc.xgmi.num_physical_nodes == 0)
1213 return 0;
1214
1215 amdgpu_ras_reset_error_count(adev, block: AMDGPU_RAS_BLOCK__XGMI_WAFL);
1216
1217 r = amdgpu_ras_block_late_init(adev, ras_block);
1218 if (r)
1219 return r;
1220
1221 switch (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0)) {
1222 case IP_VERSION(6, 4, 0):
1223 case IP_VERSION(6, 4, 1):
1224 r = amdgpu_ras_bind_aca(adev, blk: AMDGPU_RAS_BLOCK__XGMI_WAFL,
1225 aca_info: &xgmi_v6_4_0_aca_info, NULL);
1226 if (r)
1227 goto late_fini;
1228 break;
1229 default:
1230 break;
1231 }
1232
1233 return 0;
1234
1235late_fini:
1236 amdgpu_ras_block_late_fini(adev, ras_block);
1237
1238 return r;
1239}
1240
1241uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
1242 uint64_t addr)
1243{
1244 struct amdgpu_xgmi *xgmi = &adev->gmc.xgmi;
1245 return (addr + xgmi->physical_node_id * xgmi->node_segment_size);
1246}
1247
1248static void pcs_clear_status(struct amdgpu_device *adev, uint32_t pcs_status_reg)
1249{
1250 WREG32_PCIE(pcs_status_reg, 0xFFFFFFFF);
1251 WREG32_PCIE(pcs_status_reg, 0);
1252}
1253
1254static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev)
1255{
1256 uint32_t i;
1257
1258 switch (adev->asic_type) {
1259 case CHIP_ARCTURUS:
1260 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++)
1261 pcs_clear_status(adev,
1262 pcs_status_reg: xgmi_pcs_err_status_reg_arct[i]);
1263 break;
1264 case CHIP_VEGA20:
1265 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++)
1266 pcs_clear_status(adev,
1267 pcs_status_reg: xgmi_pcs_err_status_reg_vg20[i]);
1268 break;
1269 case CHIP_ALDEBARAN:
1270 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++)
1271 pcs_clear_status(adev,
1272 pcs_status_reg: xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1273 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++)
1274 pcs_clear_status(adev,
1275 pcs_status_reg: walf_pcs_err_status_reg_aldebaran[i]);
1276 break;
1277 default:
1278 break;
1279 }
1280
1281 switch (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0)) {
1282 case IP_VERSION(6, 4, 0):
1283 case IP_VERSION(6, 4, 1):
1284 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++)
1285 pcs_clear_status(adev,
1286 pcs_status_reg: xgmi3x16_pcs_err_status_reg_v6_4[i]);
1287 break;
1288 default:
1289 break;
1290 }
1291}
1292
1293static void __xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst, u64 mca_base)
1294{
1295 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1296}
1297
1298static void xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst)
1299{
1300 int i;
1301
1302 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1303 __xgmi_v6_4_0_reset_error_count(adev, xgmi_inst, mca_base: xgmi_v6_4_0_mca_base_array[i]);
1304}
1305
1306static void xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device *adev)
1307{
1308 int i;
1309
1310 for_each_inst(i, adev->aid_mask)
1311 xgmi_v6_4_0_reset_error_count(adev, xgmi_inst: i);
1312}
1313
1314static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev)
1315{
1316 switch (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0)) {
1317 case IP_VERSION(6, 4, 0):
1318 case IP_VERSION(6, 4, 1):
1319 xgmi_v6_4_0_reset_ras_error_count(adev);
1320 break;
1321 default:
1322 amdgpu_xgmi_legacy_reset_ras_error_count(adev);
1323 break;
1324 }
1325}
1326
1327static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,
1328 uint32_t value,
1329 uint32_t mask_value,
1330 uint32_t *ue_count,
1331 uint32_t *ce_count,
1332 bool is_xgmi_pcs,
1333 bool check_mask)
1334{
1335 int i;
1336 int ue_cnt = 0;
1337 const struct amdgpu_pcs_ras_field *pcs_ras_fields = NULL;
1338 uint32_t field_array_size = 0;
1339
1340 if (is_xgmi_pcs) {
1341 if (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0) ==
1342 IP_VERSION(6, 1, 0) ||
1343 amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0) ==
1344 IP_VERSION(6, 4, 0) ||
1345 amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0) ==
1346 IP_VERSION(6, 4, 1)) {
1347 pcs_ras_fields = &xgmi3x16_pcs_ras_fields[0];
1348 field_array_size = ARRAY_SIZE(xgmi3x16_pcs_ras_fields);
1349 } else {
1350 pcs_ras_fields = &xgmi_pcs_ras_fields[0];
1351 field_array_size = ARRAY_SIZE(xgmi_pcs_ras_fields);
1352 }
1353 } else {
1354 pcs_ras_fields = &wafl_pcs_ras_fields[0];
1355 field_array_size = ARRAY_SIZE(wafl_pcs_ras_fields);
1356 }
1357
1358 if (check_mask)
1359 value = value & ~mask_value;
1360
1361 /* query xgmi/walf pcs error status,
1362 * only ue is supported */
1363 for (i = 0; value && i < field_array_size; i++) {
1364 ue_cnt = (value &
1365 pcs_ras_fields[i].pcs_err_mask) >>
1366 pcs_ras_fields[i].pcs_err_shift;
1367 if (ue_cnt) {
1368 dev_info(adev->dev, "%s detected\n",
1369 pcs_ras_fields[i].err_name);
1370 *ue_count += ue_cnt;
1371 }
1372
1373 /* reset bit value if the bit is checked */
1374 value &= ~(pcs_ras_fields[i].pcs_err_mask);
1375 }
1376
1377 return 0;
1378}
1379
1380static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev,
1381 void *ras_error_status)
1382{
1383 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1384 int i, supported = 1;
1385 uint32_t data, mask_data = 0;
1386 uint32_t ue_cnt = 0, ce_cnt = 0;
1387
1388 if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__XGMI_WAFL))
1389 return ;
1390
1391 err_data->ue_count = 0;
1392 err_data->ce_count = 0;
1393
1394 switch (adev->asic_type) {
1395 case CHIP_ARCTURUS:
1396 /* check xgmi pcs error */
1397 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++) {
1398 data = RREG32_PCIE(xgmi_pcs_err_status_reg_arct[i]);
1399 if (data)
1400 amdgpu_xgmi_query_pcs_error_status(adev, value: data,
1401 mask_value: mask_data, ue_count: &ue_cnt, ce_count: &ce_cnt, is_xgmi_pcs: true, check_mask: false);
1402 }
1403 /* check wafl pcs error */
1404 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_arct); i++) {
1405 data = RREG32_PCIE(wafl_pcs_err_status_reg_arct[i]);
1406 if (data)
1407 amdgpu_xgmi_query_pcs_error_status(adev, value: data,
1408 mask_value: mask_data, ue_count: &ue_cnt, ce_count: &ce_cnt, is_xgmi_pcs: false, check_mask: false);
1409 }
1410 break;
1411 case CHIP_VEGA20:
1412 /* check xgmi pcs error */
1413 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++) {
1414 data = RREG32_PCIE(xgmi_pcs_err_status_reg_vg20[i]);
1415 if (data)
1416 amdgpu_xgmi_query_pcs_error_status(adev, value: data,
1417 mask_value: mask_data, ue_count: &ue_cnt, ce_count: &ce_cnt, is_xgmi_pcs: true, check_mask: false);
1418 }
1419 /* check wafl pcs error */
1420 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_vg20); i++) {
1421 data = RREG32_PCIE(wafl_pcs_err_status_reg_vg20[i]);
1422 if (data)
1423 amdgpu_xgmi_query_pcs_error_status(adev, value: data,
1424 mask_value: mask_data, ue_count: &ue_cnt, ce_count: &ce_cnt, is_xgmi_pcs: false, check_mask: false);
1425 }
1426 break;
1427 case CHIP_ALDEBARAN:
1428 /* check xgmi3x16 pcs error */
1429 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++) {
1430 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1431 mask_data =
1432 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1433 if (data)
1434 amdgpu_xgmi_query_pcs_error_status(adev, value: data,
1435 mask_value: mask_data, ue_count: &ue_cnt, ce_count: &ce_cnt, is_xgmi_pcs: true, check_mask: true);
1436 }
1437 /* check wafl pcs error */
1438 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++) {
1439 data = RREG32_PCIE(walf_pcs_err_status_reg_aldebaran[i]);
1440 mask_data =
1441 RREG32_PCIE(walf_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1442 if (data)
1443 amdgpu_xgmi_query_pcs_error_status(adev, value: data,
1444 mask_value: mask_data, ue_count: &ue_cnt, ce_count: &ce_cnt, is_xgmi_pcs: false, check_mask: true);
1445 }
1446 break;
1447 default:
1448 supported = 0;
1449 break;
1450 }
1451
1452 switch (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0)) {
1453 case IP_VERSION(6, 4, 0):
1454 case IP_VERSION(6, 4, 1):
1455 /* check xgmi3x16 pcs error */
1456 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++) {
1457 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_v6_4[i]);
1458 mask_data =
1459 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[i]);
1460 if (data)
1461 amdgpu_xgmi_query_pcs_error_status(adev, value: data,
1462 mask_value: mask_data, ue_count: &ue_cnt, ce_count: &ce_cnt, is_xgmi_pcs: true, check_mask: true);
1463 }
1464 break;
1465 default:
1466 if (!supported)
1467 dev_warn(adev->dev, "XGMI RAS error query not supported");
1468 break;
1469 }
1470
1471 amdgpu_ras_reset_error_count(adev, block: AMDGPU_RAS_BLOCK__XGMI_WAFL);
1472
1473 err_data->ue_count += ue_cnt;
1474 err_data->ce_count += ce_cnt;
1475}
1476
1477static enum aca_error_type xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device *adev, u64 status)
1478{
1479 const char *error_str;
1480 int ext_error_code;
1481
1482 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1483
1484 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1485 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1486 if (error_str)
1487 dev_info(adev->dev, "%s detected\n", error_str);
1488
1489 switch (ext_error_code) {
1490 case 0:
1491 return ACA_ERROR_TYPE_UE;
1492 case 6:
1493 return ACA_ERROR_TYPE_CE;
1494 default:
1495 return -EINVAL;
1496 }
1497
1498 return -EINVAL;
1499}
1500
1501static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct amdgpu_smuio_mcm_config_info *mcm_info,
1502 u64 mca_base, struct ras_err_data *err_data)
1503{
1504 int xgmi_inst = mcm_info->die_id;
1505 u64 status = 0;
1506
1507 status = RREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS);
1508 if (!ACA_REG__STATUS__VAL(status))
1509 return;
1510
1511 switch (xgmi_v6_4_0_pcs_mca_get_error_type(adev, status)) {
1512 case ACA_ERROR_TYPE_UE:
1513 amdgpu_ras_error_statistic_ue_count(err_data, mcm_info, count: 1ULL);
1514 break;
1515 case ACA_ERROR_TYPE_CE:
1516 amdgpu_ras_error_statistic_ce_count(err_data, mcm_info, count: 1ULL);
1517 break;
1518 default:
1519 break;
1520 }
1521
1522 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1523}
1524
1525static void xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, int xgmi_inst, struct ras_err_data *err_data)
1526{
1527 struct amdgpu_smuio_mcm_config_info mcm_info = {
1528 .socket_id = adev->smuio.funcs->get_socket_id(adev),
1529 .die_id = xgmi_inst,
1530 };
1531 int i;
1532
1533 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1534 __xgmi_v6_4_0_query_error_count(adev, mcm_info: &mcm_info, mca_base: xgmi_v6_4_0_mca_base_array[i], err_data);
1535}
1536
1537static void xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status)
1538{
1539 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1540 int i;
1541
1542 for_each_inst(i, adev->aid_mask)
1543 xgmi_v6_4_0_query_error_count(adev, xgmi_inst: i, err_data);
1544}
1545
1546static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev,
1547 void *ras_error_status)
1548{
1549 switch (amdgpu_ip_version(adev, ip: XGMI_HWIP, inst: 0)) {
1550 case IP_VERSION(6, 4, 0):
1551 case IP_VERSION(6, 4, 1):
1552 xgmi_v6_4_0_query_ras_error_count(adev, ras_error_status);
1553 break;
1554 default:
1555 amdgpu_xgmi_legacy_query_ras_error_count(adev, ras_error_status);
1556 break;
1557 }
1558}
1559
1560/* Trigger XGMI/WAFL error */
1561static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev,
1562 void *inject_if, uint32_t instance_mask)
1563{
1564 int ret1, ret2;
1565 struct ta_ras_trigger_error_input *block_info =
1566 (struct ta_ras_trigger_error_input *)inject_if;
1567
1568 if (amdgpu_dpm_set_df_cstate(adev, cstate: DF_CSTATE_DISALLOW))
1569 dev_warn(adev->dev, "Failed to disallow df cstate");
1570
1571 ret1 = amdgpu_dpm_set_pm_policy(adev, policy_type: PP_PM_POLICY_XGMI_PLPD, policy_level: XGMI_PLPD_DISALLOW);
1572 if (ret1 && ret1 != -EOPNOTSUPP)
1573 dev_warn(adev->dev, "Failed to disallow XGMI power down");
1574
1575 ret2 = psp_ras_trigger_error(psp: &adev->psp, info: block_info, instance_mask);
1576
1577 if (amdgpu_ras_intr_triggered())
1578 return ret2;
1579
1580 ret1 = amdgpu_dpm_set_pm_policy(adev, policy_type: PP_PM_POLICY_XGMI_PLPD, policy_level: XGMI_PLPD_DEFAULT);
1581 if (ret1 && ret1 != -EOPNOTSUPP)
1582 dev_warn(adev->dev, "Failed to allow XGMI power down");
1583
1584 if (amdgpu_dpm_set_df_cstate(adev, cstate: DF_CSTATE_ALLOW))
1585 dev_warn(adev->dev, "Failed to allow df cstate");
1586
1587 return ret2;
1588}
1589
1590struct amdgpu_ras_block_hw_ops xgmi_ras_hw_ops = {
1591 .query_ras_error_count = amdgpu_xgmi_query_ras_error_count,
1592 .reset_ras_error_count = amdgpu_xgmi_reset_ras_error_count,
1593 .ras_error_inject = amdgpu_ras_error_inject_xgmi,
1594};
1595
1596struct amdgpu_xgmi_ras xgmi_ras = {
1597 .ras_block = {
1598 .hw_ops = &xgmi_ras_hw_ops,
1599 .ras_late_init = amdgpu_xgmi_ras_late_init,
1600 },
1601};
1602
1603int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev)
1604{
1605 int err;
1606 struct amdgpu_xgmi_ras *ras;
1607
1608 if (!adev->gmc.xgmi.ras)
1609 return 0;
1610
1611 ras = adev->gmc.xgmi.ras;
1612 err = amdgpu_ras_register_ras_block(adev, ras_block_obj: &ras->ras_block);
1613 if (err) {
1614 dev_err(adev->dev, "Failed to register xgmi_wafl_pcs ras block!\n");
1615 return err;
1616 }
1617
1618 strcpy(p: ras->ras_block.ras_comm.name, q: "xgmi_wafl");
1619 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__XGMI_WAFL;
1620 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
1621 adev->gmc.xgmi.ras_if = &ras->ras_block.ras_comm;
1622
1623 return 0;
1624}
1625
1626static void amdgpu_xgmi_reset_on_init_work(struct work_struct *work)
1627{
1628 struct amdgpu_hive_info *hive =
1629 container_of(work, struct amdgpu_hive_info, reset_on_init_work);
1630 struct amdgpu_reset_context reset_context;
1631 struct amdgpu_device *tmp_adev;
1632 struct list_head device_list;
1633 int r;
1634
1635 mutex_lock(&hive->hive_lock);
1636
1637 INIT_LIST_HEAD(list: &device_list);
1638 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
1639 list_add_tail(new: &tmp_adev->reset_list, head: &device_list);
1640
1641 tmp_adev = list_first_entry(&device_list, struct amdgpu_device,
1642 reset_list);
1643 amdgpu_device_lock_reset_domain(reset_domain: tmp_adev->reset_domain);
1644
1645 reset_context.method = AMD_RESET_METHOD_ON_INIT;
1646 reset_context.reset_req_dev = tmp_adev;
1647 reset_context.hive = hive;
1648 reset_context.reset_device_list = &device_list;
1649 set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context.flags);
1650 set_bit(nr: AMDGPU_SKIP_COREDUMP, addr: &reset_context.flags);
1651
1652 amdgpu_reset_do_xgmi_reset_on_init(reset_context: &reset_context);
1653 mutex_unlock(lock: &hive->hive_lock);
1654 amdgpu_device_unlock_reset_domain(reset_domain: tmp_adev->reset_domain);
1655
1656 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1657 r = amdgpu_ras_init_badpage_info(adev: tmp_adev);
1658 if (r && r != -EHWPOISON)
1659 dev_err(tmp_adev->dev,
1660 "error during bad page data initialization");
1661 }
1662}
1663
1664static void amdgpu_xgmi_schedule_reset_on_init(struct amdgpu_hive_info *hive)
1665{
1666 INIT_WORK(&hive->reset_on_init_work, amdgpu_xgmi_reset_on_init_work);
1667 amdgpu_reset_domain_schedule(domain: hive->reset_domain,
1668 work: &hive->reset_on_init_work);
1669}
1670
1671int amdgpu_xgmi_reset_on_init(struct amdgpu_device *adev)
1672{
1673 struct amdgpu_hive_info *hive;
1674 bool reset_scheduled;
1675 int num_devs;
1676
1677 hive = amdgpu_get_xgmi_hive(adev);
1678 if (!hive)
1679 return -EINVAL;
1680
1681 mutex_lock(&hive->hive_lock);
1682 num_devs = atomic_read(v: &hive->number_devices);
1683 reset_scheduled = false;
1684 if (num_devs == adev->gmc.xgmi.num_physical_nodes) {
1685 amdgpu_xgmi_schedule_reset_on_init(hive);
1686 reset_scheduled = true;
1687 }
1688
1689 mutex_unlock(lock: &hive->hive_lock);
1690 amdgpu_put_xgmi_hive(hive);
1691
1692 if (reset_scheduled)
1693 flush_work(work: &hive->reset_on_init_work);
1694
1695 return 0;
1696}
1697
1698int amdgpu_xgmi_request_nps_change(struct amdgpu_device *adev,
1699 struct amdgpu_hive_info *hive,
1700 int req_nps_mode)
1701{
1702 struct amdgpu_device *tmp_adev;
1703 int cur_nps_mode, r;
1704
1705 /* This is expected to be called only during unload of driver. The
1706 * request needs to be placed only once for all devices in the hive. If
1707 * one of them fail, revert the request for previous successful devices.
1708 * After placing the request, make hive mode as UNKNOWN so that other
1709 * devices don't request anymore.
1710 */
1711 mutex_lock(&hive->hive_lock);
1712 if (atomic_read(v: &hive->requested_nps_mode) ==
1713 UNKNOWN_MEMORY_PARTITION_MODE) {
1714 dev_dbg(adev->dev, "Unexpected entry for hive NPS change");
1715 mutex_unlock(lock: &hive->hive_lock);
1716 return 0;
1717 }
1718 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1719 r = adev->gmc.gmc_funcs->request_mem_partition_mode(
1720 tmp_adev, req_nps_mode);
1721 if (r)
1722 break;
1723 }
1724 if (r) {
1725 /* Request back current mode if one of the requests failed */
1726 cur_nps_mode =
1727 adev->gmc.gmc_funcs->query_mem_partition_mode(tmp_adev);
1728 list_for_each_entry_continue_reverse(
1729 tmp_adev, &hive->device_list, gmc.xgmi.head)
1730 adev->gmc.gmc_funcs->request_mem_partition_mode(
1731 tmp_adev, cur_nps_mode);
1732 }
1733 /* Set to UNKNOWN so that other devices don't request anymore */
1734 atomic_set(v: &hive->requested_nps_mode, i: UNKNOWN_MEMORY_PARTITION_MODE);
1735 mutex_unlock(lock: &hive->hive_lock);
1736
1737 return r;
1738}
1739
1740bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
1741 struct amdgpu_device *bo_adev)
1742{
1743 return (amdgpu_use_xgmi_p2p && adev != bo_adev &&
1744 adev->gmc.xgmi.hive_id &&
1745 adev->gmc.xgmi.hive_id == bo_adev->gmc.xgmi.hive_id);
1746}
1747
1748void amdgpu_xgmi_early_init(struct amdgpu_device *adev)
1749{
1750 if (!adev->gmc.xgmi.supported)
1751 return;
1752
1753 switch (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0)) {
1754 case IP_VERSION(9, 4, 0):
1755 case IP_VERSION(9, 4, 1):
1756 case IP_VERSION(9, 4, 2):
1757 /* 25 GT/s */
1758 adev->gmc.xgmi.max_speed = 25;
1759 adev->gmc.xgmi.max_width = 16;
1760 break;
1761 case IP_VERSION(9, 4, 3):
1762 case IP_VERSION(9, 4, 4):
1763 case IP_VERSION(9, 5, 0):
1764 /* 32 GT/s */
1765 adev->gmc.xgmi.max_speed = 32;
1766 adev->gmc.xgmi.max_width = 16;
1767 break;
1768 default:
1769 break;
1770 }
1771}
1772
1773void amgpu_xgmi_set_max_speed_width(struct amdgpu_device *adev,
1774 uint16_t max_speed, uint8_t max_width)
1775{
1776 adev->gmc.xgmi.max_speed = max_speed;
1777 adev->gmc.xgmi.max_width = max_width;
1778}
1779

source code of linux/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c