1	/*
2	* Copyright 2018 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*
23	*/
24	#include <linux/debugfs.h>
25	#include <linux/list.h>
26	#include <linux/module.h>
27	#include <linux/uaccess.h>
28	#include <linux/reboot.h>
29	#include <linux/syscalls.h>
30	#include <linux/pm_runtime.h>
31	#include <linux/list_sort.h>
32
33	#include "amdgpu.h"
34	#include "amdgpu_ras.h"
35	#include "amdgpu_atomfirmware.h"
36	#include "amdgpu_xgmi.h"
37	#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
38	#include "nbio_v4_3.h"
39	#include "nbif_v6_3_1.h"
40	#include "nbio_v7_9.h"
41	#include "atom.h"
42	#include "amdgpu_reset.h"
43	#include "amdgpu_psp.h"
44	#include "amdgpu_ras_mgr.h"
45
46	#ifdef CONFIG_X86_MCE_AMD
47	#include <asm/mce.h>
48
49	static bool notifier_registered;
50	#endif
51	static const char *RAS_FS_NAME = "ras";
52
53	const char *ras_error_string[] = {
54	"none",
55	"parity",
56	"single_correctable",
57	"multi_uncorrectable",
58	"poison",
59	};
60
61	const char *ras_block_string[] = {
62	"umc",
63	"sdma",
64	"gfx",
65	"mmhub",
66	"athub",
67	"pcie_bif",
68	"hdp",
69	"xgmi_wafl",
70	"df",
71	"smn",
72	"sem",
73	"mp0",
74	"mp1",
75	"fuse",
76	"mca",
77	"vcn",
78	"jpeg",
79	"ih",
80	"mpio",
81	"mmsch",
82	};
83
84	const char *ras_mca_block_string[] = {
85	"mca_mp0",
86	"mca_mp1",
87	"mca_mpio",
88	"mca_iohc",
89	};
90
91	struct amdgpu_ras_block_list {
92	/* ras block link */
93	struct list_head node;
94
95	struct amdgpu_ras_block_object *ras_obj;
96	};
97
98	const char *get_ras_block_str(struct ras_common_if *ras_block)
99	{
100	if (!ras_block)
101	return "NULL";
102
103	if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT ||
104	ras_block->block >= ARRAY_SIZE(ras_block_string))
105	return "OUT OF RANGE";
106
107	if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
108	return ras_mca_block_string[ras_block->sub_block_index];
109
110	return ras_block_string[ras_block->block];
111	}
112
113	#define ras_block_str(_BLOCK_) \
114	(((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
115
116	#define ras_err_str(i) (ras_error_string[ffs(i)])
117
118	#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
119
120	/* inject address is 52 bits */
121	#define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52)
122
123	/* typical ECC bad page rate is 1 bad page per 100MB VRAM */
124	#define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL)
125
126	#define MAX_UMC_POISON_POLLING_TIME_ASYNC 10
127
128	#define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms
129
130	#define MAX_FLUSH_RETIRE_DWORK_TIMES 100
131
132	#define BYPASS_ALLOCATED_ADDRESS 0x0
133	#define BYPASS_INITIALIZATION_ADDRESS 0x1
134
135	enum amdgpu_ras_retire_page_reservation {
136	AMDGPU_RAS_RETIRE_PAGE_RESERVED,
137	AMDGPU_RAS_RETIRE_PAGE_PENDING,
138	AMDGPU_RAS_RETIRE_PAGE_FAULT,
139	};
140
141	atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
142
143	static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
144	uint64_t addr);
145	static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
146	uint64_t addr);
147
148	static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev);
149	static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev);
150
151	#ifdef CONFIG_X86_MCE_AMD
152	static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
153	static void
154	amdgpu_unregister_bad_pages_mca_notifier(struct amdgpu_device *adev);
155	struct mce_notifier_adev_list {
156	struct amdgpu_device *devs[MAX_GPU_INSTANCE];
157	int num_gpu;
158	};
159	static struct mce_notifier_adev_list mce_adev_list;
160	#endif
161
162	void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
163	{
164	if (adev && amdgpu_ras_get_context(adev))
165	amdgpu_ras_get_context(adev)->error_query_ready = ready;
166	}
167
168	static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
169	{
170	if (adev && amdgpu_ras_get_context(adev))
171	return amdgpu_ras_get_context(adev)->error_query_ready;
172
173	return false;
174	}
175
176	static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
177	{
178	struct ras_err_data err_data;
179	struct eeprom_table_record err_rec;
180	int ret;
181
182	ret = amdgpu_ras_check_bad_page(adev, addr: address);
183	if (ret == -EINVAL) {
184	dev_warn(adev->dev,
185	"RAS WARN: input address 0x%llx is invalid.\n",
186	address);
187	return -EINVAL;
188	} else if (ret == 1) {
189	dev_warn(adev->dev,
190	"RAS WARN: 0x%llx has already been marked as bad page!\n",
191	address);
192	return 0;
193	}
194
195	ret = amdgpu_ras_error_data_init(err_data: &err_data);
196	if (ret)
197	return ret;
198
199	memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
200	err_data.err_addr = &err_rec;
201	amdgpu_umc_fill_error_record(err_data: &err_data, err_addr: address, retired_page: address, channel_index: 0, umc_inst: 0);
202
203	if (amdgpu_bad_page_threshold != 0) {
204	amdgpu_ras_add_bad_pages(adev, bps: err_data.err_addr,
205	pages: err_data.err_addr_cnt, from_rom: false);
206	amdgpu_ras_save_bad_pages(adev, NULL);
207	}
208
209	amdgpu_ras_error_data_fini(err_data: &err_data);
210
211	dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
212	dev_warn(adev->dev, "Clear EEPROM:\n");
213	dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
214
215	return 0;
216	}
217
218	static int amdgpu_check_address_validity(struct amdgpu_device *adev,
219	uint64_t address, uint64_t flags)
220	{
221	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
222	struct amdgpu_vram_block_info blk_info;
223	uint64_t page_pfns[32] = {0};
224	int i, ret, count;
225	bool hit = false;
226
227	if (amdgpu_ip_version(adev, ip: UMC_HWIP, inst: 0) < IP_VERSION(12, 0, 0))
228	return 0;
229
230	if (amdgpu_sriov_vf(adev)) {
231	if (amdgpu_virt_check_vf_critical_region(adev, addr: address, hit: &hit))
232	return -EPERM;
233	return hit ? -EACCES : 0;
234	}
235
236	if ((address >= adev->gmc.mc_vram_size) ||
237	(address >= RAS_UMC_INJECT_ADDR_LIMIT))
238	return -EFAULT;
239
240	count = amdgpu_umc_lookup_bad_pages_in_a_row(adev,
241	pa_addr: address, pfns: page_pfns, ARRAY_SIZE(page_pfns));
242	if (count <= 0)
243	return -EPERM;
244
245	for (i = 0; i < count; i++) {
246	memset(&blk_info, 0, sizeof(blk_info));
247	ret = amdgpu_vram_mgr_query_address_block_info(mgr: &adev->mman.vram_mgr,
248	address: page_pfns[i] << AMDGPU_GPU_PAGE_SHIFT, info: &blk_info);
249	if (!ret) {
250	/* The input address that needs to be checked is allocated by
251	* current calling process, so it is necessary to exclude
252	* the calling process.
253	*/
254	if ((flags == BYPASS_ALLOCATED_ADDRESS) &&
255	((blk_info.task.pid != task_pid_nr(current)) ||
256	strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN)))
257	return -EACCES;
258	else if ((flags == BYPASS_INITIALIZATION_ADDRESS) &&
259	(blk_info.task.pid == con->init_task_pid) &&
260	!strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN))
261	return -EACCES;
262	}
263	}
264
265	return 0;
266	}
267
268	static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
269	size_t size, loff_t *pos)
270	{
271	struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
272	struct ras_query_if info = {
273	.head = obj->head,
274	};
275	ssize_t s;
276	char val[128];
277
278	if (amdgpu_ras_query_error_status(adev: obj->adev, info: &info))
279	return -EINVAL;
280
281	/* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
282	if (amdgpu_ip_version(adev: obj->adev, ip: MP0_HWIP, inst: 0) != IP_VERSION(11, 0, 2) &&
283	amdgpu_ip_version(adev: obj->adev, ip: MP0_HWIP, inst: 0) != IP_VERSION(11, 0, 4)) {
284	if (amdgpu_ras_reset_error_status(adev: obj->adev, block: info.head.block))
285	dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
286	}
287
288	s = snprintf(buf: val, size: sizeof(val), fmt: "%s: %lu\n%s: %lu\n",
289	"ue", info.ue_count,
290	"ce", info.ce_count);
291	if (*pos >= s)
292	return 0;
293
294	s -= *pos;
295	s = min_t(u64, s, size);
296
297
298	if (copy_to_user(to: buf, from: &val[*pos], n: s))
299	return -EINVAL;
300
301	*pos += s;
302
303	return s;
304	}
305
306	static const struct file_operations amdgpu_ras_debugfs_ops = {
307	.owner = THIS_MODULE,
308	.read = amdgpu_ras_debugfs_read,
309	.write = NULL,
310	.llseek = default_llseek
311	};
312
313	static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
314	{
315	int i;
316
317	for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
318	*block_id = i;
319	if (strcmp(name, ras_block_string[i]) == 0)
320	return 0;
321	}
322	return -EINVAL;
323	}
324
325	static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
326	const char __user *buf, size_t size,
327	loff_t *pos, struct ras_debug_if *data)
328	{
329	ssize_t s = min_t(u64, 64, size);
330	char str[65];
331	char block_name[33];
332	char err[9] = "ue";
333	int op = -1;
334	int block_id;
335	uint32_t sub_block;
336	u64 address, value;
337	/* default value is 0 if the mask is not set by user */
338	u32 instance_mask = 0;
339
340	if (*pos)
341	return -EINVAL;
342	*pos = size;
343
344	memset(str, 0, sizeof(str));
345	memset(data, 0, sizeof(*data));
346
347	if (copy_from_user(to: str, from: buf, n: s))
348	return -EINVAL;
349
350	if (sscanf(str, "disable %32s", block_name) == 1)
351	op = 0;
352	else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
353	op = 1;
354	else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
355	op = 2;
356	else if (strstr(str, "retire_page") != NULL)
357	op = 3;
358	else if (strstr(str, "check_address") != NULL)
359	op = 4;
360	else if (str[0] && str[1] && str[2] && str[3])
361	/* ascii string, but commands are not matched. */
362	return -EINVAL;
363
364	if (op != -1) {
365	if (op == 3) {
366	if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
367	sscanf(str, "%*s %llu", &address) != 1)
368	return -EINVAL;
369
370	data->op = op;
371	data->inject.address = address;
372
373	return 0;
374	} else if (op == 4) {
375	if (sscanf(str, "%*s 0x%llx 0x%llx", &address, &value) != 2 &&
376	sscanf(str, "%*s %llu %llu", &address, &value) != 2)
377	return -EINVAL;
378
379	data->op = op;
380	data->inject.address = address;
381	data->inject.value = value;
382	return 0;
383	}
384
385	if (amdgpu_ras_find_block_id_by_name(name: block_name, block_id: &block_id))
386	return -EINVAL;
387
388	data->head.block = block_id;
389	/* only ue, ce and poison errors are supported */
390	if (!memcmp(p: "ue", q: err, size: 2))
391	data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
392	else if (!memcmp(p: "ce", q: err, size: 2))
393	data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
394	else if (!memcmp(p: "poison", q: err, size: 6))
395	data->head.type = AMDGPU_RAS_ERROR__POISON;
396	else
397	return -EINVAL;
398
399	data->op = op;
400
401	if (op == 2) {
402	if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x",
403	&sub_block, &address, &value, &instance_mask) != 4 &&
404	sscanf(str, "%*s %*s %*s %u %llu %llu %u",
405	&sub_block, &address, &value, &instance_mask) != 4 &&
406	sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
407	&sub_block, &address, &value) != 3 &&
408	sscanf(str, "%*s %*s %*s %u %llu %llu",
409	&sub_block, &address, &value) != 3)
410	return -EINVAL;
411	data->head.sub_block_index = sub_block;
412	data->inject.address = address;
413	data->inject.value = value;
414	data->inject.instance_mask = instance_mask;
415	}
416	} else {
417	if (size < sizeof(*data))
418	return -EINVAL;
419
420	if (copy_from_user(to: data, from: buf, n: sizeof(*data)))
421	return -EINVAL;
422	}
423
424	return 0;
425	}
426
427	static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev,
428	struct ras_debug_if *data)
429	{
430	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
431	uint32_t mask, inst_mask = data->inject.instance_mask;
432
433	/* no need to set instance mask if there is only one instance */
434	if (num_xcc <= 1 && inst_mask) {
435	data->inject.instance_mask = 0;
436	dev_dbg(adev->dev,
437	"RAS inject mask(0x%x) isn't supported and force it to 0.\n",
438	inst_mask);
439
440	return;
441	}
442
443	switch (data->head.block) {
444	case AMDGPU_RAS_BLOCK__GFX:
445	mask = GENMASK(num_xcc - 1, 0);
446	break;
447	case AMDGPU_RAS_BLOCK__SDMA:
448	mask = GENMASK(adev->sdma.num_instances - 1, 0);
449	break;
450	case AMDGPU_RAS_BLOCK__VCN:
451	case AMDGPU_RAS_BLOCK__JPEG:
452	mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0);
453	break;
454	default:
455	mask = inst_mask;
456	break;
457	}
458
459	/* remove invalid bits in instance mask */
460	data->inject.instance_mask &= mask;
461	if (inst_mask != data->inject.instance_mask)
462	dev_dbg(adev->dev,
463	"Adjust RAS inject mask 0x%x to 0x%x\n",
464	inst_mask, data->inject.instance_mask);
465	}
466
467	/**
468	* DOC: AMDGPU RAS debugfs control interface
469	*
470	* The control interface accepts struct ras_debug_if which has two members.
471	*
472	* First member: ras_debug_if::head or ras_debug_if::inject.
473	*
474	* head is used to indicate which IP block will be under control.
475	*
476	* head has four members, they are block, type, sub_block_index, name.
477	* block: which IP will be under control.
478	* type: what kind of error will be enabled/disabled/injected.
479	* sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
480	* name: the name of IP.
481	*
482	* inject has three more members than head, they are address, value and mask.
483	* As their names indicate, inject operation will write the
484	* value to the address.
485	*
486	* The second member: struct ras_debug_if::op.
487	* It has three kinds of operations.
488	*
489	* - 0: disable RAS on the block. Take ::head as its data.
490	* - 1: enable RAS on the block. Take ::head as its data.
491	* - 2: inject errors on the block. Take ::inject as its data.
492	*
493	* How to use the interface?
494	*
495	* In a program
496	*
497	* Copy the struct ras_debug_if in your code and initialize it.
498	* Write the struct to the control interface.
499	*
500	* From shell
501	*
502	* .. code-block:: bash
503	*
504	* echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
505	* echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
506	* echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
507	*
508	* Where N, is the card which you want to affect.
509	*
510	* "disable" requires only the block.
511	* "enable" requires the block and error type.
512	* "inject" requires the block, error type, address, and value.
513	*
514	* The block is one of: umc, sdma, gfx, etc.
515	* see ras_block_string[] for details
516	*
517	* The error type is one of: ue, ce and poison where,
518	* ue is multi-uncorrectable
519	* ce is single-correctable
520	* poison is poison
521	*
522	* The sub-block is a the sub-block index, pass 0 if there is no sub-block.
523	* The address and value are hexadecimal numbers, leading 0x is optional.
524	* The mask means instance mask, is optional, default value is 0x1.
525	*
526	* For instance,
527	*
528	* .. code-block:: bash
529	*
530	* echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
531	* echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl
532	* echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
533	*
534	* How to check the result of the operation?
535	*
536	* To check disable/enable, see "ras" features at,
537	* /sys/class/drm/card[0/1/2...]/device/ras/features
538	*
539	* To check inject, see the corresponding error count at,
540	* /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
541	*
542	* .. note::
543	* Operations are only allowed on blocks which are supported.
544	* Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
545	* to see which blocks support RAS on a particular asic.
546	*
547	*/
548	static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
549	const char __user *buf,
550	size_t size, loff_t *pos)
551	{
552	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
553	struct ras_debug_if data;
554	int ret = 0;
555
556	if (!amdgpu_ras_get_error_query_ready(adev)) {
557	dev_warn(adev->dev, "RAS WARN: error injection "
558	"currently inaccessible\n");
559	return size;
560	}
561
562	ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, data: &data);
563	if (ret)
564	return ret;
565
566	if (data.op == 3) {
567	ret = amdgpu_reserve_page_direct(adev, address: data.inject.address);
568	if (!ret)
569	return size;
570	else
571	return ret;
572	} else if (data.op == 4) {
573	ret = amdgpu_check_address_validity(adev, address: data.inject.address, flags: data.inject.value);
574	return ret ? ret : size;
575	}
576
577	if (!amdgpu_ras_is_supported(adev, block: data.head.block))
578	return -EINVAL;
579
580	switch (data.op) {
581	case 0:
582	ret = amdgpu_ras_feature_enable(adev, head: &data.head, enable: 0);
583	break;
584	case 1:
585	ret = amdgpu_ras_feature_enable(adev, head: &data.head, enable: 1);
586	break;
587	case 2:
588	/* umc ce/ue error injection for a bad page is not allowed */
589	if (data.head.block == AMDGPU_RAS_BLOCK__UMC)
590	ret = amdgpu_ras_check_bad_page(adev, addr: data.inject.address);
591	if (ret == -EINVAL) {
592	dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.",
593	data.inject.address);
594	break;
595	} else if (ret == 1) {
596	dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has already been marked as bad!\n",
597	data.inject.address);
598	break;
599	}
600
601	amdgpu_ras_instance_mask_check(adev, data: &data);
602
603	/* data.inject.address is offset instead of absolute gpu address */
604	ret = amdgpu_ras_error_inject(adev, info: &data.inject);
605	break;
606	default:
607	ret = -EINVAL;
608	break;
609	}
610
611	if (ret)
612	return ret;
613
614	return size;
615	}
616
617	static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev);
618
619	/**
620	* DOC: AMDGPU RAS debugfs EEPROM table reset interface
621	*
622	* Some boards contain an EEPROM which is used to persistently store a list of
623	* bad pages which experiences ECC errors in vram. This interface provides
624	* a way to reset the EEPROM, e.g., after testing error injection.
625	*
626	* Usage:
627	*
628	* .. code-block:: bash
629	*
630	* echo 1 > ../ras/ras_eeprom_reset
631	*
632	* will reset EEPROM table to 0 entries.
633	*
634	*/
635	static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
636	const char __user *buf,
637	size_t size, loff_t *pos)
638	{
639	struct amdgpu_device *adev =
640	(struct amdgpu_device *)file_inode(f)->i_private;
641	int ret;
642
643	if (amdgpu_uniras_enabled(adev)) {
644	ret = amdgpu_uniras_clear_badpages_info(adev);
645	return ret ? ret : size;
646	}
647
648	ret = amdgpu_ras_eeprom_reset_table(
649	control: &(amdgpu_ras_get_context(adev)->eeprom_control));
650
651	if (!ret) {
652	/* Something was written to EEPROM.
653	*/
654	amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
655	return size;
656	} else {
657	return ret;
658	}
659	}
660
661	static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
662	.owner = THIS_MODULE,
663	.read = NULL,
664	.write = amdgpu_ras_debugfs_ctrl_write,
665	.llseek = default_llseek
666	};
667
668	static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
669	.owner = THIS_MODULE,
670	.read = NULL,
671	.write = amdgpu_ras_debugfs_eeprom_write,
672	.llseek = default_llseek
673	};
674
675	/**
676	* DOC: AMDGPU RAS sysfs Error Count Interface
677	*
678	* It allows the user to read the error count for each IP block on the gpu through
679	* /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
680	*
681	* It outputs the multiple lines which report the uncorrected (ue) and corrected
682	* (ce) error counts.
683	*
684	* The format of one line is below,
685	*
686	* [ce|ue]: count
687	*
688	* Example:
689	*
690	* .. code-block:: bash
691	*
692	* ue: 0
693	* ce: 1
694	*
695	*/
696	static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
697	struct device_attribute *attr, char *buf)
698	{
699	struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
700	struct ras_query_if info = {
701	.head = obj->head,
702	};
703
704	if (!amdgpu_ras_get_error_query_ready(adev: obj->adev))
705	return sysfs_emit(buf, fmt: "Query currently inaccessible\n");
706
707	if (amdgpu_ras_query_error_status(adev: obj->adev, info: &info))
708	return -EINVAL;
709
710	if (amdgpu_ip_version(adev: obj->adev, ip: MP0_HWIP, inst: 0) != IP_VERSION(11, 0, 2) &&
711	amdgpu_ip_version(adev: obj->adev, ip: MP0_HWIP, inst: 0) != IP_VERSION(11, 0, 4)) {
712	if (amdgpu_ras_reset_error_status(adev: obj->adev, block: info.head.block))
713	dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
714	}
715
716	if (info.head.block == AMDGPU_RAS_BLOCK__UMC)
717	return sysfs_emit(buf, fmt: "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
718	"ce", info.ce_count, "de", info.de_count);
719	else
720	return sysfs_emit(buf, fmt: "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
721	"ce", info.ce_count);
722	}
723
724	/* obj begin */
725
726	#define get_obj(obj) do { (obj)->use++; } while (0)
727	#define alive_obj(obj) ((obj)->use)
728
729	static inline void put_obj(struct ras_manager *obj)
730	{
731	if (obj && (--obj->use == 0)) {
732	list_del(entry: &obj->node);
733	amdgpu_ras_error_data_fini(err_data: &obj->err_data);
734	}
735
736	if (obj && (obj->use < 0))
737	DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
738	}
739
740	/* make one obj and return it. */
741	static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
742	struct ras_common_if *head)
743	{
744	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
745	struct ras_manager *obj;
746
747	if (!adev->ras_enabled || !con)
748	return NULL;
749
750	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
751	return NULL;
752
753	if (head->block == AMDGPU_RAS_BLOCK__MCA) {
754	if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
755	return NULL;
756
757	obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
758	} else
759	obj = &con->objs[head->block];
760
761	/* already exist. return obj? */
762	if (alive_obj(obj))
763	return NULL;
764
765	if (amdgpu_ras_error_data_init(err_data: &obj->err_data))
766	return NULL;
767
768	obj->head = *head;
769	obj->adev = adev;
770	list_add(new: &obj->node, head: &con->head);
771	get_obj(obj);
772
773	return obj;
774	}
775
776	/* return an obj equal to head, or the first when head is NULL */
777	struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
778	struct ras_common_if *head)
779	{
780	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
781	struct ras_manager *obj;
782	int i;
783
784	if (!adev->ras_enabled || !con)
785	return NULL;
786
787	if (head) {
788	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
789	return NULL;
790
791	if (head->block == AMDGPU_RAS_BLOCK__MCA) {
792	if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
793	return NULL;
794
795	obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
796	} else
797	obj = &con->objs[head->block];
798
799	if (alive_obj(obj))
800	return obj;
801	} else {
802	for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
803	obj = &con->objs[i];
804	if (alive_obj(obj))
805	return obj;
806	}
807	}
808
809	return NULL;
810	}
811	/* obj end */
812
813	/* feature ctl begin */
814	static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
815	struct ras_common_if *head)
816	{
817	return adev->ras_hw_enabled & BIT(head->block);
818	}
819
820	static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
821	struct ras_common_if *head)
822	{
823	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
824
825	return con->features & BIT(head->block);
826	}
827
828	/*
829	* if obj is not created, then create one.
830	* set feature enable flag.
831	*/
832	static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
833	struct ras_common_if *head, int enable)
834	{
835	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
836	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
837
838	/* If hardware does not support ras, then do not create obj.
839	* But if hardware support ras, we can create the obj.
840	* Ras framework checks con->hw_supported to see if it need do
841	* corresponding initialization.
842	* IP checks con->support to see if it need disable ras.
843	*/
844	if (!amdgpu_ras_is_feature_allowed(adev, head))
845	return 0;
846
847	if (enable) {
848	if (!obj) {
849	obj = amdgpu_ras_create_obj(adev, head);
850	if (!obj)
851	return -EINVAL;
852	} else {
853	/* In case we create obj somewhere else */
854	get_obj(obj);
855	}
856	con->features |= BIT(head->block);
857	} else {
858	if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
859	con->features &= ~BIT(head->block);
860	put_obj(obj);
861	}
862	}
863
864	return 0;
865	}
866
867	/* wrapper of psp_ras_enable_features */
868	int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
869	struct ras_common_if *head, bool enable)
870	{
871	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
872	union ta_ras_cmd_input *info;
873	int ret;
874
875	if (!con)
876	return -EINVAL;
877
878	/* For non-gfx ip, do not enable ras feature if it is not allowed */
879	/* For gfx ip, regardless of feature support status, */
880	/* Force issue enable or disable ras feature commands */
881	if (head->block != AMDGPU_RAS_BLOCK__GFX &&
882	!amdgpu_ras_is_feature_allowed(adev, head))
883	return 0;
884
885	/* Only enable gfx ras feature from host side */
886	if (head->block == AMDGPU_RAS_BLOCK__GFX &&
887	!amdgpu_sriov_vf(adev) &&
888	!amdgpu_ras_intr_triggered()) {
889	info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
890	if (!info)
891	return -ENOMEM;
892
893	if (!enable) {
894	info->disable_features = (struct ta_ras_disable_features_input) {
895	.block_id = amdgpu_ras_block_to_ta(block: head->block),
896	.error_type = amdgpu_ras_error_to_ta(error: head->type),
897	};
898	} else {
899	info->enable_features = (struct ta_ras_enable_features_input) {
900	.block_id = amdgpu_ras_block_to_ta(block: head->block),
901	.error_type = amdgpu_ras_error_to_ta(error: head->type),
902	};
903	}
904
905	ret = psp_ras_enable_features(psp: &adev->psp, info, enable);
906	if (ret) {
907	dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
908	enable ? "enable":"disable",
909	get_ras_block_str(head),
910	amdgpu_ras_is_poison_mode_supported(adev), ret);
911	kfree(objp: info);
912	return ret;
913	}
914
915	kfree(objp: info);
916	}
917
918	/* setup the obj */
919	__amdgpu_ras_feature_enable(adev, head, enable);
920
921	return 0;
922	}
923
924	/* Only used in device probe stage and called only once. */
925	int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
926	struct ras_common_if *head, bool enable)
927	{
928	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
929	int ret;
930
931	if (!con)
932	return -EINVAL;
933
934	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
935	if (enable) {
936	/* There is no harm to issue a ras TA cmd regardless of
937	* the currecnt ras state.
938	* If current state == target state, it will do nothing
939	* But sometimes it requests driver to reset and repost
940	* with error code -EAGAIN.
941	*/
942	ret = amdgpu_ras_feature_enable(adev, head, enable: 1);
943	/* With old ras TA, we might fail to enable ras.
944	* Log it and just setup the object.
945	* TODO need remove this WA in the future.
946	*/
947	if (ret == -EINVAL) {
948	ret = __amdgpu_ras_feature_enable(adev, head, enable: 1);
949	if (!ret)
950	dev_info(adev->dev,
951	"RAS INFO: %s setup object\n",
952	get_ras_block_str(head));
953	}
954	} else {
955	/* setup the object then issue a ras TA disable cmd.*/
956	ret = __amdgpu_ras_feature_enable(adev, head, enable: 1);
957	if (ret)
958	return ret;
959
960	/* gfx block ras disable cmd must send to ras-ta */
961	if (head->block == AMDGPU_RAS_BLOCK__GFX)
962	con->features |= BIT(head->block);
963
964	ret = amdgpu_ras_feature_enable(adev, head, enable: 0);
965
966	/* clean gfx block ras features flag */
967	if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
968	con->features &= ~BIT(head->block);
969	}
970	} else
971	ret = amdgpu_ras_feature_enable(adev, head, enable);
972
973	return ret;
974	}
975
976	static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
977	bool bypass)
978	{
979	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
980	struct ras_manager *obj, *tmp;
981
982	list_for_each_entry_safe(obj, tmp, &con->head, node) {
983	/* bypass psp.
984	* aka just release the obj and corresponding flags
985	*/
986	if (bypass) {
987	if (__amdgpu_ras_feature_enable(adev, head: &obj->head, enable: 0))
988	break;
989	} else {
990	if (amdgpu_ras_feature_enable(adev, head: &obj->head, enable: 0))
991	break;
992	}
993	}
994
995	return con->features;
996	}
997
998	static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
999	bool bypass)
1000	{
1001	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1002	int i;
1003	const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
1004
1005	for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
1006	struct ras_common_if head = {
1007	.block = i,
1008	.type = default_ras_type,
1009	.sub_block_index = 0,
1010	};
1011
1012	if (i == AMDGPU_RAS_BLOCK__MCA)
1013	continue;
1014
1015	if (bypass) {
1016	/*
1017	* bypass psp. vbios enable ras for us.
1018	* so just create the obj
1019	*/
1020	if (__amdgpu_ras_feature_enable(adev, head: &head, enable: 1))
1021	break;
1022	} else {
1023	if (amdgpu_ras_feature_enable(adev, head: &head, enable: 1))
1024	break;
1025	}
1026	}
1027
1028	for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
1029	struct ras_common_if head = {
1030	.block = AMDGPU_RAS_BLOCK__MCA,
1031	.type = default_ras_type,
1032	.sub_block_index = i,
1033	};
1034
1035	if (bypass) {
1036	/*
1037	* bypass psp. vbios enable ras for us.
1038	* so just create the obj
1039	*/
1040	if (__amdgpu_ras_feature_enable(adev, head: &head, enable: 1))
1041	break;
1042	} else {
1043	if (amdgpu_ras_feature_enable(adev, head: &head, enable: 1))
1044	break;
1045	}
1046	}
1047
1048	return con->features;
1049	}
1050	/* feature ctl end */
1051
1052	static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
1053	enum amdgpu_ras_block block)
1054	{
1055	if (!block_obj)
1056	return -EINVAL;
1057
1058	if (block_obj->ras_comm.block == block)
1059	return 0;
1060
1061	return -EINVAL;
1062	}
1063
1064	static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
1065	enum amdgpu_ras_block block, uint32_t sub_block_index)
1066	{
1067	struct amdgpu_ras_block_list *node, *tmp;
1068	struct amdgpu_ras_block_object *obj;
1069
1070	if (block >= AMDGPU_RAS_BLOCK__LAST)
1071	return NULL;
1072
1073	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
1074	if (!node->ras_obj) {
1075	dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
1076	continue;
1077	}
1078
1079	obj = node->ras_obj;
1080	if (obj->ras_block_match) {
1081	if (obj->ras_block_match(obj, block, sub_block_index) == 0)
1082	return obj;
1083	} else {
1084	if (amdgpu_ras_block_match_default(block_obj: obj, block) == 0)
1085	return obj;
1086	}
1087	}
1088
1089	return NULL;
1090	}
1091
1092	static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
1093	{
1094	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
1095	int ret = 0;
1096
1097	/*
1098	* choosing right query method according to
1099	* whether smu support query error information
1100	*/
1101	ret = amdgpu_dpm_get_ecc_info(adev, umc_ecc: (void *)&(ras->umc_ecc));
1102	if (ret == -EOPNOTSUPP) {
1103	if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1104	adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
1105	adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
1106
1107	/* umc query_ras_error_address is also responsible for clearing
1108	* error status
1109	*/
1110	if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1111	adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
1112	adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
1113	} else if (!ret) {
1114	if (adev->umc.ras &&
1115	adev->umc.ras->ecc_info_query_ras_error_count)
1116	adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
1117
1118	if (adev->umc.ras &&
1119	adev->umc.ras->ecc_info_query_ras_error_address)
1120	adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
1121	}
1122	}
1123
1124	static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
1125	struct ras_manager *ras_mgr,
1126	struct ras_err_data *err_data,
1127	struct ras_query_context *qctx,
1128	const char *blk_name,
1129	bool is_ue,
1130	bool is_de)
1131	{
1132	struct amdgpu_smuio_mcm_config_info *mcm_info;
1133	struct ras_err_node *err_node;
1134	struct ras_err_info *err_info;
1135	u64 event_id = qctx->evid.event_id;
1136
1137	if (is_ue) {
1138	for_each_ras_error(err_node, err_data) {
1139	err_info = &err_node->err_info;
1140	mcm_info = &err_info->mcm_info;
1141	if (err_info->ue_count) {
1142	RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1143	"%lld new uncorrectable hardware errors detected in %s block\n",
1144	mcm_info->socket_id,
1145	mcm_info->die_id,
1146	err_info->ue_count,
1147	blk_name);
1148	}
1149	}
1150
1151	for_each_ras_error(err_node, &ras_mgr->err_data) {
1152	err_info = &err_node->err_info;
1153	mcm_info = &err_info->mcm_info;
1154	RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1155	"%lld uncorrectable hardware errors detected in total in %s block\n",
1156	mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name);
1157	}
1158
1159	} else {
1160	if (is_de) {
1161	for_each_ras_error(err_node, err_data) {
1162	err_info = &err_node->err_info;
1163	mcm_info = &err_info->mcm_info;
1164	if (err_info->de_count) {
1165	RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1166	"%lld new deferred hardware errors detected in %s block\n",
1167	mcm_info->socket_id,
1168	mcm_info->die_id,
1169	err_info->de_count,
1170	blk_name);
1171	}
1172	}
1173
1174	for_each_ras_error(err_node, &ras_mgr->err_data) {
1175	err_info = &err_node->err_info;
1176	mcm_info = &err_info->mcm_info;
1177	RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1178	"%lld deferred hardware errors detected in total in %s block\n",
1179	mcm_info->socket_id, mcm_info->die_id,
1180	err_info->de_count, blk_name);
1181	}
1182	} else {
1183	if (adev->debug_disable_ce_logs)
1184	return;
1185
1186	for_each_ras_error(err_node, err_data) {
1187	err_info = &err_node->err_info;
1188	mcm_info = &err_info->mcm_info;
1189	if (err_info->ce_count) {
1190	RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1191	"%lld new correctable hardware errors detected in %s block\n",
1192	mcm_info->socket_id,
1193	mcm_info->die_id,
1194	err_info->ce_count,
1195	blk_name);
1196	}
1197	}
1198
1199	for_each_ras_error(err_node, &ras_mgr->err_data) {
1200	err_info = &err_node->err_info;
1201	mcm_info = &err_info->mcm_info;
1202	RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1203	"%lld correctable hardware errors detected in total in %s block\n",
1204	mcm_info->socket_id, mcm_info->die_id,
1205	err_info->ce_count, blk_name);
1206	}
1207	}
1208	}
1209	}
1210
1211	static inline bool err_data_has_source_info(struct ras_err_data *data)
1212	{
1213	return !list_empty(head: &data->err_node_list);
1214	}
1215
1216	static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
1217	struct ras_query_if *query_if,
1218	struct ras_err_data *err_data,
1219	struct ras_query_context *qctx)
1220	{
1221	struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, head: &query_if->head);
1222	const char *blk_name = get_ras_block_str(ras_block: &query_if->head);
1223	u64 event_id = qctx->evid.event_id;
1224
1225	if (err_data->ce_count) {
1226	if (err_data_has_source_info(data: err_data)) {
1227	amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1228	blk_name, is_ue: false, is_de: false);
1229	} else if (!adev->aid_mask &&
1230	adev->smuio.funcs &&
1231	adev->smuio.funcs->get_socket_id &&
1232	adev->smuio.funcs->get_die_id) {
1233	RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1234	"%ld correctable hardware errors "
1235	"detected in %s block\n",
1236	adev->smuio.funcs->get_socket_id(adev),
1237	adev->smuio.funcs->get_die_id(adev),
1238	ras_mgr->err_data.ce_count,
1239	blk_name);
1240	} else {
1241	RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors "
1242	"detected in %s block\n",
1243	ras_mgr->err_data.ce_count,
1244	blk_name);
1245	}
1246	}
1247
1248	if (err_data->ue_count) {
1249	if (err_data_has_source_info(data: err_data)) {
1250	amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1251	blk_name, is_ue: true, is_de: false);
1252	} else if (!adev->aid_mask &&
1253	adev->smuio.funcs &&
1254	adev->smuio.funcs->get_socket_id &&
1255	adev->smuio.funcs->get_die_id) {
1256	RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1257	"%ld uncorrectable hardware errors "
1258	"detected in %s block\n",
1259	adev->smuio.funcs->get_socket_id(adev),
1260	adev->smuio.funcs->get_die_id(adev),
1261	ras_mgr->err_data.ue_count,
1262	blk_name);
1263	} else {
1264	RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors "
1265	"detected in %s block\n",
1266	ras_mgr->err_data.ue_count,
1267	blk_name);
1268	}
1269	}
1270
1271	if (err_data->de_count) {
1272	if (err_data_has_source_info(data: err_data)) {
1273	amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1274	blk_name, is_ue: false, is_de: true);
1275	} else if (!adev->aid_mask &&
1276	adev->smuio.funcs &&
1277	adev->smuio.funcs->get_socket_id &&
1278	adev->smuio.funcs->get_die_id) {
1279	RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1280	"%ld deferred hardware errors "
1281	"detected in %s block\n",
1282	adev->smuio.funcs->get_socket_id(adev),
1283	adev->smuio.funcs->get_die_id(adev),
1284	ras_mgr->err_data.de_count,
1285	blk_name);
1286	} else {
1287	RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors "
1288	"detected in %s block\n",
1289	ras_mgr->err_data.de_count,
1290	blk_name);
1291	}
1292	}
1293	}
1294
1295	static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev,
1296	struct ras_query_if *query_if,
1297	struct ras_err_data *err_data,
1298	struct ras_query_context *qctx)
1299	{
1300	unsigned long new_ue, new_ce, new_de;
1301	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head: &query_if->head);
1302	const char *blk_name = get_ras_block_str(ras_block: &query_if->head);
1303	u64 event_id = qctx->evid.event_id;
1304
1305	new_ce = err_data->ce_count - obj->err_data.ce_count;
1306	new_ue = err_data->ue_count - obj->err_data.ue_count;
1307	new_de = err_data->de_count - obj->err_data.de_count;
1308
1309	if (new_ce) {
1310	RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors "
1311	"detected in %s block\n",
1312	new_ce,
1313	blk_name);
1314	}
1315
1316	if (new_ue) {
1317	RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors "
1318	"detected in %s block\n",
1319	new_ue,
1320	blk_name);
1321	}
1322
1323	if (new_de) {
1324	RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors "
1325	"detected in %s block\n",
1326	new_de,
1327	blk_name);
1328	}
1329	}
1330
1331	static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data)
1332	{
1333	struct ras_err_node *err_node;
1334	struct ras_err_info *err_info;
1335
1336	if (err_data_has_source_info(data: err_data)) {
1337	for_each_ras_error(err_node, err_data) {
1338	err_info = &err_node->err_info;
1339	amdgpu_ras_error_statistic_de_count(err_data: &obj->err_data,
1340	mcm_info: &err_info->mcm_info, count: err_info->de_count);
1341	amdgpu_ras_error_statistic_ce_count(err_data: &obj->err_data,
1342	mcm_info: &err_info->mcm_info, count: err_info->ce_count);
1343	amdgpu_ras_error_statistic_ue_count(err_data: &obj->err_data,
1344	mcm_info: &err_info->mcm_info, count: err_info->ue_count);
1345	}
1346	} else {
1347	/* for legacy asic path which doesn't has error source info */
1348	obj->err_data.ue_count += err_data->ue_count;
1349	obj->err_data.ce_count += err_data->ce_count;
1350	obj->err_data.de_count += err_data->de_count;
1351	}
1352	}
1353
1354	static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj,
1355	struct ras_err_data *err_data)
1356	{
1357	/* Host reports absolute counts */
1358	obj->err_data.ue_count = err_data->ue_count;
1359	obj->err_data.ce_count = err_data->ce_count;
1360	obj->err_data.de_count = err_data->de_count;
1361	}
1362
1363	static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1364	{
1365	struct ras_common_if head;
1366
1367	memset(&head, 0, sizeof(head));
1368	head.block = blk;
1369
1370	return amdgpu_ras_find_obj(adev, head: &head);
1371	}
1372
1373	int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1374	const struct aca_info *aca_info, void *data)
1375	{
1376	struct ras_manager *obj;
1377
1378	/* in resume phase, no need to create aca fs node */
1379	if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
1380	return 0;
1381
1382	obj = get_ras_manager(adev, blk);
1383	if (!obj)
1384	return -EINVAL;
1385
1386	return amdgpu_aca_add_handle(adev, handle: &obj->aca_handle, ras_block_str(blk), aca_info, data);
1387	}
1388
1389	int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1390	{
1391	struct ras_manager *obj;
1392
1393	obj = get_ras_manager(adev, blk);
1394	if (!obj)
1395	return -EINVAL;
1396
1397	amdgpu_aca_remove_handle(handle: &obj->aca_handle);
1398
1399	return 0;
1400	}
1401
1402	static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1403	enum aca_error_type type, struct ras_err_data *err_data,
1404	struct ras_query_context *qctx)
1405	{
1406	struct ras_manager *obj;
1407
1408	obj = get_ras_manager(adev, blk);
1409	if (!obj)
1410	return -EINVAL;
1411
1412	return amdgpu_aca_get_error_data(adev, handle: &obj->aca_handle, type, err_data, qctx);
1413	}
1414
1415	ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
1416	struct aca_handle *handle, char *buf, void *data)
1417	{
1418	struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle);
1419	struct ras_query_if info = {
1420	.head = obj->head,
1421	};
1422
1423	if (!amdgpu_ras_get_error_query_ready(adev: obj->adev))
1424	return sysfs_emit(buf, fmt: "Query currently inaccessible\n");
1425
1426	if (amdgpu_ras_query_error_status(adev: obj->adev, info: &info))
1427	return -EINVAL;
1428
1429	return sysfs_emit(buf, fmt: "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
1430	"ce", info.ce_count, "de", info.de_count);
1431	}
1432
1433	static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
1434	struct ras_query_if *info,
1435	struct ras_err_data *err_data,
1436	struct ras_query_context *qctx,
1437	unsigned int error_query_mode)
1438	{
1439	enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT;
1440	struct amdgpu_ras_block_object *block_obj = NULL;
1441	int ret;
1442
1443	if (blk == AMDGPU_RAS_BLOCK_COUNT)
1444	return -EINVAL;
1445
1446	if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY)
1447	return -EINVAL;
1448
1449	if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1450	return amdgpu_virt_req_ras_err_count(adev, block: blk, err_data);
1451	} else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
1452	if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
1453	amdgpu_ras_get_ecc_info(adev, err_data);
1454	} else {
1455	block_obj = amdgpu_ras_get_ras_block(adev, block: info->head.block, sub_block_index: 0);
1456	if (!block_obj || !block_obj->hw_ops) {
1457	dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1458	get_ras_block_str(&info->head));
1459	return -EINVAL;
1460	}
1461
1462	if (block_obj->hw_ops->query_ras_error_count)
1463	block_obj->hw_ops->query_ras_error_count(adev, err_data);
1464
1465	if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
1466	(info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
1467	(info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
1468	if (block_obj->hw_ops->query_ras_error_status)
1469	block_obj->hw_ops->query_ras_error_status(adev);
1470	}
1471	}
1472	} else {
1473	if (amdgpu_aca_is_enabled(adev)) {
1474	ret = amdgpu_aca_log_ras_error_data(adev, blk, type: ACA_ERROR_TYPE_UE, err_data, qctx);
1475	if (ret)
1476	return ret;
1477
1478	ret = amdgpu_aca_log_ras_error_data(adev, blk, type: ACA_ERROR_TYPE_CE, err_data, qctx);
1479	if (ret)
1480	return ret;
1481
1482	ret = amdgpu_aca_log_ras_error_data(adev, blk, type: ACA_ERROR_TYPE_DEFERRED, err_data, qctx);
1483	if (ret)
1484	return ret;
1485	} else {
1486	/* FIXME: add code to check return value later */
1487	amdgpu_mca_smu_log_ras_error(adev, blk, type: AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx);
1488	amdgpu_mca_smu_log_ras_error(adev, blk, type: AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx);
1489	}
1490	}
1491
1492	return 0;
1493	}
1494
1495	/* query/inject/cure begin */
1496	static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev,
1497	struct ras_query_if *info,
1498	enum ras_event_type type)
1499	{
1500	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head: &info->head);
1501	struct ras_err_data err_data;
1502	struct ras_query_context qctx;
1503	unsigned int error_query_mode;
1504	int ret;
1505
1506	if (!obj)
1507	return -EINVAL;
1508
1509	ret = amdgpu_ras_error_data_init(err_data: &err_data);
1510	if (ret)
1511	return ret;
1512
1513	if (!amdgpu_ras_get_error_query_mode(adev, mode: &error_query_mode))
1514	return -EINVAL;
1515
1516	memset(&qctx, 0, sizeof(qctx));
1517	qctx.evid.type = type;
1518	qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type);
1519
1520	if (!down_read_trylock(sem: &adev->reset_domain->sem)) {
1521	ret = -EIO;
1522	goto out_fini_err_data;
1523	}
1524
1525	ret = amdgpu_ras_query_error_status_helper(adev, info,
1526	err_data: &err_data,
1527	qctx: &qctx,
1528	error_query_mode);
1529	up_read(sem: &adev->reset_domain->sem);
1530	if (ret)
1531	goto out_fini_err_data;
1532
1533	if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1534	amdgpu_rasmgr_error_data_statistic_update(obj, err_data: &err_data);
1535	amdgpu_ras_error_generate_report(adev, query_if: info, err_data: &err_data, qctx: &qctx);
1536	} else {
1537	/* Host provides absolute error counts. First generate the report
1538	* using the previous VF internal count against new host count.
1539	* Then Update VF internal count.
1540	*/
1541	amdgpu_ras_virt_error_generate_report(adev, query_if: info, err_data: &err_data, qctx: &qctx);
1542	amdgpu_ras_mgr_virt_error_data_statistics_update(obj, err_data: &err_data);
1543	}
1544
1545	info->ue_count = obj->err_data.ue_count;
1546	info->ce_count = obj->err_data.ce_count;
1547	info->de_count = obj->err_data.de_count;
1548
1549	out_fini_err_data:
1550	amdgpu_ras_error_data_fini(err_data: &err_data);
1551
1552	return ret;
1553	}
1554
1555	static int amdgpu_uniras_clear_badpages_info(struct amdgpu_device *adev)
1556	{
1557	struct ras_cmd_dev_handle req = {0};
1558	int ret;
1559
1560	ret = amdgpu_ras_mgr_handle_ras_cmd(adev, cmd_id: RAS_CMD__CLEAR_BAD_PAGE_INFO,
1561	input: &req, input_size: sizeof(req), NULL, out_size: 0);
1562	if (ret) {
1563	dev_err(adev->dev, "Failed to clear bad pages info, ret: %d\n", ret);
1564	return ret;
1565	}
1566
1567	return 0;
1568	}
1569
1570	static int amdgpu_uniras_query_block_ecc(struct amdgpu_device *adev,
1571	struct ras_query_if *info)
1572	{
1573	struct ras_cmd_block_ecc_info_req req = {0};
1574	struct ras_cmd_block_ecc_info_rsp rsp = {0};
1575	int ret;
1576
1577	if (!info)
1578	return -EINVAL;
1579
1580	req.block_id = info->head.block;
1581	req.subblock_id = info->head.sub_block_index;
1582
1583	ret = amdgpu_ras_mgr_handle_ras_cmd(adev, cmd_id: RAS_CMD__GET_BLOCK_ECC_STATUS,
1584	input: &req, input_size: sizeof(req), output: &rsp, out_size: sizeof(rsp));
1585	if (!ret) {
1586	info->ce_count = rsp.ce_count;
1587	info->ue_count = rsp.ue_count;
1588	info->de_count = rsp.de_count;
1589	}
1590
1591	return ret;
1592	}
1593
1594	int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info)
1595	{
1596	if (amdgpu_uniras_enabled(adev))
1597	return amdgpu_uniras_query_block_ecc(adev, info);
1598	else
1599	return amdgpu_ras_query_error_status_with_event(adev, info, type: RAS_EVENT_TYPE_INVALID);
1600	}
1601
1602	int amdgpu_ras_reset_error_count(struct amdgpu_device *adev,
1603	enum amdgpu_ras_block block)
1604	{
1605	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, sub_block_index: 0);
1606	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
1607	const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
1608
1609	if (!block_obj || !block_obj->hw_ops) {
1610	dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1611	ras_block_str(block));
1612	return -EOPNOTSUPP;
1613	}
1614
1615	if (!amdgpu_ras_is_supported(adev, block) ||
1616	!amdgpu_ras_get_aca_debug_mode(adev))
1617	return -EOPNOTSUPP;
1618
1619	if (amdgpu_sriov_vf(adev))
1620	return -EOPNOTSUPP;
1621
1622	/* skip ras error reset in gpu reset */
1623	if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) &&
1624	((smu_funcs && smu_funcs->set_debug_mode) ||
1625	(mca_funcs && mca_funcs->mca_set_debug_mode)))
1626	return -EOPNOTSUPP;
1627
1628	if (block_obj->hw_ops->reset_ras_error_count)
1629	block_obj->hw_ops->reset_ras_error_count(adev);
1630
1631	return 0;
1632	}
1633
1634	int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1635	enum amdgpu_ras_block block)
1636	{
1637	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, sub_block_index: 0);
1638
1639	if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP)
1640	return 0;
1641
1642	if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1643	(block == AMDGPU_RAS_BLOCK__MMHUB)) {
1644	if (block_obj->hw_ops->reset_ras_error_status)
1645	block_obj->hw_ops->reset_ras_error_status(adev);
1646	}
1647
1648	return 0;
1649	}
1650
1651	static int amdgpu_uniras_error_inject(struct amdgpu_device *adev,
1652	struct ras_inject_if *info)
1653	{
1654	struct ras_cmd_inject_error_req inject_req;
1655	struct ras_cmd_inject_error_rsp rsp;
1656
1657	if (!info)
1658	return -EINVAL;
1659
1660	memset(&inject_req, 0, sizeof(inject_req));
1661	inject_req.block_id = info->head.block;
1662	inject_req.subblock_id = info->head.sub_block_index;
1663	inject_req.address = info->address;
1664	inject_req.error_type = info->head.type;
1665	inject_req.instance_mask = info->instance_mask;
1666	inject_req.method = info->value;
1667
1668	return amdgpu_ras_mgr_handle_ras_cmd(adev, cmd_id: RAS_CMD__INJECT_ERROR,
1669	input: &inject_req, input_size: sizeof(inject_req), output: &rsp, out_size: sizeof(rsp));
1670	}
1671
1672	/* wrapper of psp_ras_trigger_error */
1673	int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1674	struct ras_inject_if *info)
1675	{
1676	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head: &info->head);
1677	struct ta_ras_trigger_error_input block_info = {
1678	.block_id = amdgpu_ras_block_to_ta(block: info->head.block),
1679	.inject_error_type = amdgpu_ras_error_to_ta(error: info->head.type),
1680	.sub_block_index = info->head.sub_block_index,
1681	.address = info->address,
1682	.value = info->value,
1683	};
1684	int ret = -EINVAL;
1685	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1686	block: info->head.block,
1687	sub_block_index: info->head.sub_block_index);
1688
1689	if (amdgpu_uniras_enabled(adev))
1690	return amdgpu_uniras_error_inject(adev, info);
1691
1692	/* inject on guest isn't allowed, return success directly */
1693	if (amdgpu_sriov_vf(adev))
1694	return 0;
1695
1696	if (!obj)
1697	return -EINVAL;
1698
1699	if (!block_obj || !block_obj->hw_ops) {
1700	dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1701	get_ras_block_str(&info->head));
1702	return -EINVAL;
1703	}
1704
1705	/* Calculate XGMI relative offset */
1706	if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1707	info->head.block != AMDGPU_RAS_BLOCK__GFX) {
1708	block_info.address =
1709	amdgpu_xgmi_get_relative_phy_addr(adev,
1710	addr: block_info.address);
1711	}
1712
1713	if (block_obj->hw_ops->ras_error_inject) {
1714	if (info->head.block == AMDGPU_RAS_BLOCK__GFX)
1715	ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask);
1716	else /* Special ras_error_inject is defined (e.g: xgmi) */
1717	ret = block_obj->hw_ops->ras_error_inject(adev, &block_info,
1718	info->instance_mask);
1719	} else {
1720	/* default path */
1721	ret = psp_ras_trigger_error(psp: &adev->psp, info: &block_info, instance_mask: info->instance_mask);
1722	}
1723
1724	if (ret)
1725	dev_err(adev->dev, "ras inject %s failed %d\n",
1726	get_ras_block_str(&info->head), ret);
1727
1728	return ret;
1729	}
1730
1731	/**
1732	* amdgpu_ras_query_error_count_helper -- Get error counter for specific IP
1733	* @adev: pointer to AMD GPU device
1734	* @ce_count: pointer to an integer to be set to the count of correctible errors.
1735	* @ue_count: pointer to an integer to be set to the count of uncorrectible errors.
1736	* @query_info: pointer to ras_query_if
1737	*
1738	* Return 0 for query success or do nothing, otherwise return an error
1739	* on failures
1740	*/
1741	static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev,
1742	unsigned long *ce_count,
1743	unsigned long *ue_count,
1744	struct ras_query_if *query_info)
1745	{
1746	int ret;
1747
1748	if (!query_info)
1749	/* do nothing if query_info is not specified */
1750	return 0;
1751
1752	ret = amdgpu_ras_query_error_status(adev, info: query_info);
1753	if (ret)
1754	return ret;
1755
1756	*ce_count += query_info->ce_count;
1757	*ue_count += query_info->ue_count;
1758
1759	/* some hardware/IP supports read to clear
1760	* no need to explictly reset the err status after the query call */
1761	if (amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0) != IP_VERSION(11, 0, 2) &&
1762	amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0) != IP_VERSION(11, 0, 4)) {
1763	if (amdgpu_ras_reset_error_status(adev, block: query_info->head.block))
1764	dev_warn(adev->dev,
1765	"Failed to reset error counter and error status\n");
1766	}
1767
1768	return 0;
1769	}
1770
1771	/**
1772	* amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP
1773	* @adev: pointer to AMD GPU device
1774	* @ce_count: pointer to an integer to be set to the count of correctible errors.
1775	* @ue_count: pointer to an integer to be set to the count of uncorrectible
1776	* errors.
1777	* @query_info: pointer to ras_query_if if the query request is only for
1778	* specific ip block; if info is NULL, then the qurey request is for
1779	* all the ip blocks that support query ras error counters/status
1780	*
1781	* If set, @ce_count or @ue_count, count and return the corresponding
1782	* error counts in those integer pointers. Return 0 if the device
1783	* supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1784	*/
1785	int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1786	unsigned long *ce_count,
1787	unsigned long *ue_count,
1788	struct ras_query_if *query_info)
1789	{
1790	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1791	struct ras_manager *obj;
1792	unsigned long ce, ue;
1793	int ret;
1794
1795	if (!adev->ras_enabled || !con)
1796	return -EOPNOTSUPP;
1797
1798	/* Don't count since no reporting.
1799	*/
1800	if (!ce_count && !ue_count)
1801	return 0;
1802
1803	ce = 0;
1804	ue = 0;
1805	if (!query_info) {
1806	/* query all the ip blocks that support ras query interface */
1807	list_for_each_entry(obj, &con->head, node) {
1808	struct ras_query_if info = {
1809	.head = obj->head,
1810	};
1811
1812	ret = amdgpu_ras_query_error_count_helper(adev, ce_count: &ce, ue_count: &ue, query_info: &info);
1813	}
1814	} else {
1815	/* query specific ip block */
1816	ret = amdgpu_ras_query_error_count_helper(adev, ce_count: &ce, ue_count: &ue, query_info);
1817	}
1818
1819	if (ret)
1820	return ret;
1821
1822	if (ce_count)
1823	*ce_count = ce;
1824
1825	if (ue_count)
1826	*ue_count = ue;
1827
1828	return 0;
1829	}
1830	/* query/inject/cure end */
1831
1832
1833	/* sysfs begin */
1834
1835	static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1836	struct ras_badpage *bps, uint32_t count, uint32_t start);
1837	static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev,
1838	struct ras_badpage *bps, uint32_t count, uint32_t start);
1839
1840	static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1841	{
1842	switch (flags) {
1843	case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1844	return "R";
1845	case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1846	return "P";
1847	case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1848	default:
1849	return "F";
1850	}
1851	}
1852
1853	/**
1854	* DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1855	*
1856	* It allows user to read the bad pages of vram on the gpu through
1857	* /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1858	*
1859	* It outputs multiple lines, and each line stands for one gpu page.
1860	*
1861	* The format of one line is below,
1862	* gpu pfn : gpu page size : flags
1863	*
1864	* gpu pfn and gpu page size are printed in hex format.
1865	* flags can be one of below character,
1866	*
1867	* R: reserved, this gpu page is reserved and not able to use.
1868	*
1869	* P: pending for reserve, this gpu page is marked as bad, will be reserved
1870	* in next window of page_reserve.
1871	*
1872	* F: unable to reserve. this gpu page can't be reserved due to some reasons.
1873	*
1874	* Examples:
1875	*
1876	* .. code-block:: bash
1877	*
1878	* 0x00000001 : 0x00001000 : R
1879	* 0x00000002 : 0x00001000 : P
1880	*
1881	*/
1882
1883	static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1884	struct kobject *kobj, const struct bin_attribute *attr,
1885	char *buf, loff_t ppos, size_t count)
1886	{
1887	struct amdgpu_ras *con =
1888	container_of(attr, struct amdgpu_ras, badpages_attr);
1889	struct amdgpu_device *adev = con->adev;
1890	const unsigned int element_size =
1891	sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1892	unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1893	unsigned int end = div64_ul(ppos + count - 1, element_size);
1894	ssize_t s = 0;
1895	struct ras_badpage *bps = NULL;
1896	int bps_count = 0, i, status;
1897	uint64_t address;
1898
1899	memset(buf, 0, count);
1900
1901	bps_count = end - start;
1902	bps = kmalloc_array(bps_count, sizeof(*bps), GFP_KERNEL);
1903	if (!bps)
1904	return 0;
1905
1906	memset(bps, 0, sizeof(*bps) * bps_count);
1907
1908	if (amdgpu_uniras_enabled(adev))
1909	bps_count = amdgpu_uniras_badpages_read(adev, bps, count: bps_count, start);
1910	else
1911	bps_count = amdgpu_ras_badpages_read(adev, bps, count: bps_count, start);
1912
1913	if (bps_count <= 0) {
1914	kfree(objp: bps);
1915	return 0;
1916	}
1917
1918	for (i = 0; i < bps_count; i++) {
1919	address = ((uint64_t)bps[i].bp) << AMDGPU_GPU_PAGE_SHIFT;
1920	if (amdgpu_ras_check_critical_address(adev, addr: address))
1921	continue;
1922
1923	bps[i].size = AMDGPU_GPU_PAGE_SIZE;
1924
1925	status = amdgpu_vram_mgr_query_page_status(mgr: &adev->mman.vram_mgr,
1926	start: address);
1927	if (status == -EBUSY)
1928	bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
1929	else if (status == -ENOENT)
1930	bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
1931	else
1932	bps[i].flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED;
1933
1934	s += scnprintf(buf: &buf[s], size: element_size + 1,
1935	fmt: "0x%08x : 0x%08x : %1s\n",
1936	bps[i].bp,
1937	bps[i].size,
1938	amdgpu_ras_badpage_flags_str(flags: bps[i].flags));
1939	}
1940
1941	kfree(objp: bps);
1942
1943	return s;
1944	}
1945
1946	static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1947	struct device_attribute *attr, char *buf)
1948	{
1949	struct amdgpu_ras *con =
1950	container_of(attr, struct amdgpu_ras, features_attr);
1951
1952	return sysfs_emit(buf, fmt: "feature mask: 0x%x\n", con->features);
1953	}
1954
1955	static bool amdgpu_ras_get_version_info(struct amdgpu_device *adev, u32 *major,
1956	u32 *minor, u32 *rev)
1957	{
1958	int i;
1959
1960	if (!adev || !major || !minor || !rev || !amdgpu_uniras_enabled(adev))
1961	return false;
1962
1963	for (i = 0; i < adev->num_ip_blocks; i++) {
1964	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_RAS) {
1965	*major = adev->ip_blocks[i].version->major;
1966	*minor = adev->ip_blocks[i].version->minor;
1967	*rev = adev->ip_blocks[i].version->rev;
1968	return true;
1969	}
1970	}
1971
1972	return false;
1973	}
1974
1975	static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev,
1976	struct device_attribute *attr, char *buf)
1977	{
1978	struct amdgpu_ras *con =
1979	container_of(attr, struct amdgpu_ras, version_attr);
1980	u32 major, minor, rev;
1981	ssize_t size = 0;
1982
1983	size += sysfs_emit_at(buf, at: size, fmt: "table version: 0x%x\n",
1984	con->eeprom_control.tbl_hdr.version);
1985
1986	if (amdgpu_ras_get_version_info(adev: con->adev, major: &major, minor: &minor, rev: &rev))
1987	size += sysfs_emit_at(buf, at: size, fmt: "ras version: %u.%u.%u\n",
1988	major, minor, rev);
1989
1990	return size;
1991	}
1992
1993	static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev,
1994	struct device_attribute *attr, char *buf)
1995	{
1996	struct amdgpu_ras *con =
1997	container_of(attr, struct amdgpu_ras, schema_attr);
1998	return sysfs_emit(buf, fmt: "schema: 0x%x\n", con->schema);
1999	}
2000
2001	static struct {
2002	enum ras_event_type type;
2003	const char *name;
2004	} dump_event[] = {
2005	{RAS_EVENT_TYPE_FATAL, "Fatal Error"},
2006	{RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"},
2007	{RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"},
2008	};
2009
2010	static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev,
2011	struct device_attribute *attr, char *buf)
2012	{
2013	struct amdgpu_ras *con =
2014	container_of(attr, struct amdgpu_ras, event_state_attr);
2015	struct ras_event_manager *event_mgr = con->event_mgr;
2016	struct ras_event_state *event_state;
2017	int i, size = 0;
2018
2019	if (!event_mgr)
2020	return -EINVAL;
2021
2022	size += sysfs_emit_at(buf, at: size, fmt: "current seqno: %llu\n", atomic64_read(v: &event_mgr->seqno));
2023	for (i = 0; i < ARRAY_SIZE(dump_event); i++) {
2024	event_state = &event_mgr->event_state[dump_event[i].type];
2025	size += sysfs_emit_at(buf, at: size, fmt: "%s: count:%llu, last_seqno:%llu\n",
2026	dump_event[i].name,
2027	atomic64_read(v: &event_state->count),
2028	event_state->last_seqno);
2029	}
2030
2031	return (ssize_t)size;
2032	}
2033
2034	static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
2035	{
2036	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2037
2038	if (adev->dev->kobj.sd)
2039	sysfs_remove_file_from_group(kobj: &adev->dev->kobj,
2040	attr: &con->badpages_attr.attr,
2041	group: RAS_FS_NAME);
2042	}
2043
2044	static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev)
2045	{
2046	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2047	struct attribute *attrs[] = {
2048	&con->features_attr.attr,
2049	&con->version_attr.attr,
2050	&con->schema_attr.attr,
2051	&con->event_state_attr.attr,
2052	NULL
2053	};
2054	struct attribute_group group = {
2055	.name = RAS_FS_NAME,
2056	.attrs = attrs,
2057	};
2058
2059	if (adev->dev->kobj.sd)
2060	sysfs_remove_group(kobj: &adev->dev->kobj, grp: &group);
2061
2062	return 0;
2063	}
2064
2065	int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
2066	struct ras_common_if *head)
2067	{
2068	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2069
2070	if (amdgpu_aca_is_enabled(adev))
2071	return 0;
2072
2073	if (!obj || obj->attr_inuse)
2074	return -EINVAL;
2075
2076	if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, block: head->block))
2077	return 0;
2078
2079	get_obj(obj);
2080
2081	snprintf(buf: obj->fs_data.sysfs_name, size: sizeof(obj->fs_data.sysfs_name),
2082	fmt: "%s_err_count", head->name);
2083
2084	obj->sysfs_attr = (struct device_attribute){
2085	.attr = {
2086	.name = obj->fs_data.sysfs_name,
2087	.mode = S_IRUGO,
2088	},
2089	.show = amdgpu_ras_sysfs_read,
2090	};
2091	sysfs_attr_init(&obj->sysfs_attr.attr);
2092
2093	if (sysfs_add_file_to_group(kobj: &adev->dev->kobj,
2094	attr: &obj->sysfs_attr.attr,
2095	group: RAS_FS_NAME)) {
2096	put_obj(obj);
2097	return -EINVAL;
2098	}
2099
2100	obj->attr_inuse = 1;
2101
2102	return 0;
2103	}
2104
2105	int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
2106	struct ras_common_if *head)
2107	{
2108	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2109
2110	if (amdgpu_aca_is_enabled(adev))
2111	return 0;
2112
2113	if (!obj || !obj->attr_inuse)
2114	return -EINVAL;
2115
2116	if (adev->dev->kobj.sd)
2117	sysfs_remove_file_from_group(kobj: &adev->dev->kobj,
2118	attr: &obj->sysfs_attr.attr,
2119	group: RAS_FS_NAME);
2120	obj->attr_inuse = 0;
2121	put_obj(obj);
2122
2123	return 0;
2124	}
2125
2126	static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
2127	{
2128	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2129	struct ras_manager *obj, *tmp;
2130
2131	list_for_each_entry_safe(obj, tmp, &con->head, node) {
2132	amdgpu_ras_sysfs_remove(adev, head: &obj->head);
2133	}
2134
2135	if (amdgpu_bad_page_threshold != 0)
2136	amdgpu_ras_sysfs_remove_bad_page_node(adev);
2137
2138	amdgpu_ras_sysfs_remove_dev_attr_node(adev);
2139
2140	return 0;
2141	}
2142	/* sysfs end */
2143
2144	/**
2145	* DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
2146	*
2147	* Normally when there is an uncorrectable error, the driver will reset
2148	* the GPU to recover. However, in the event of an unrecoverable error,
2149	* the driver provides an interface to reboot the system automatically
2150	* in that event.
2151	*
2152	* The following file in debugfs provides that interface:
2153	* /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
2154	*
2155	* Usage:
2156	*
2157	* .. code-block:: bash
2158	*
2159	* echo true > .../ras/auto_reboot
2160	*
2161	*/
2162	/* debugfs begin */
2163	static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
2164	{
2165	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2166	struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control;
2167	struct drm_minor *minor = adev_to_drm(adev)->primary;
2168	struct dentry *dir;
2169
2170	dir = debugfs_create_dir(name: RAS_FS_NAME, parent: minor->debugfs_root);
2171	debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
2172	&amdgpu_ras_debugfs_ctrl_ops);
2173	debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
2174	&amdgpu_ras_debugfs_eeprom_ops);
2175	debugfs_create_u32(name: "bad_page_cnt_threshold", mode: 0444, parent: dir,
2176	value: &con->bad_page_cnt_threshold);
2177	debugfs_create_u32(name: "ras_num_recs", mode: 0444, parent: dir, value: &eeprom->ras_num_recs);
2178	debugfs_create_x32(name: "ras_hw_enabled", mode: 0444, parent: dir, value: &adev->ras_hw_enabled);
2179	debugfs_create_x32(name: "ras_enabled", mode: 0444, parent: dir, value: &adev->ras_enabled);
2180	debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
2181	&amdgpu_ras_debugfs_eeprom_size_ops);
2182	con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
2183	S_IRUGO, dir, adev,
2184	&amdgpu_ras_debugfs_eeprom_table_ops);
2185	amdgpu_ras_debugfs_set_ret_size(control: &con->eeprom_control);
2186
2187	/*
2188	* After one uncorrectable error happens, usually GPU recovery will
2189	* be scheduled. But due to the known problem in GPU recovery failing
2190	* to bring GPU back, below interface provides one direct way to
2191	* user to reboot system automatically in such case within
2192	* ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
2193	* will never be called.
2194	*/
2195	debugfs_create_bool(name: "auto_reboot", S_IWUGO | S_IRUGO, parent: dir, value: &con->reboot);
2196
2197	/*
2198	* User could set this not to clean up hardware's error count register
2199	* of RAS IPs during ras recovery.
2200	*/
2201	debugfs_create_bool(name: "disable_ras_err_cnt_harvest", mode: 0644, parent: dir,
2202	value: &con->disable_ras_err_cnt_harvest);
2203	return dir;
2204	}
2205
2206	static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
2207	struct ras_fs_if *head,
2208	struct dentry *dir)
2209	{
2210	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head: &head->head);
2211
2212	if (!obj || !dir)
2213	return;
2214
2215	get_obj(obj);
2216
2217	memcpy(obj->fs_data.debugfs_name,
2218	head->debugfs_name,
2219	sizeof(obj->fs_data.debugfs_name));
2220
2221	debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
2222	obj, &amdgpu_ras_debugfs_ops);
2223	}
2224
2225	static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev)
2226	{
2227	bool ret;
2228
2229	switch (amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0)) {
2230	case IP_VERSION(13, 0, 6):
2231	case IP_VERSION(13, 0, 12):
2232	case IP_VERSION(13, 0, 14):
2233	ret = true;
2234	break;
2235	default:
2236	ret = false;
2237	break;
2238	}
2239
2240	return ret;
2241	}
2242
2243	void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
2244	{
2245	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2246	struct dentry *dir;
2247	struct ras_manager *obj;
2248	struct ras_fs_if fs_info;
2249
2250	/*
2251	* it won't be called in resume path, no need to check
2252	* suspend and gpu reset status
2253	*/
2254	if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
2255	return;
2256
2257	dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
2258
2259	list_for_each_entry(obj, &con->head, node) {
2260	if (amdgpu_ras_is_supported(adev, block: obj->head.block) &&
2261	(obj->attr_inuse == 1)) {
2262	sprintf(buf: fs_info.debugfs_name, fmt: "%s_err_inject",
2263	get_ras_block_str(ras_block: &obj->head));
2264	fs_info.head = obj->head;
2265	amdgpu_ras_debugfs_create(adev, head: &fs_info, dir);
2266	}
2267	}
2268
2269	if (amdgpu_ras_aca_is_supported(adev)) {
2270	if (amdgpu_aca_is_enabled(adev))
2271	amdgpu_aca_smu_debugfs_init(adev, root: dir);
2272	else
2273	amdgpu_mca_smu_debugfs_init(adev, root: dir);
2274	}
2275	}
2276
2277	/* debugfs end */
2278
2279	/* ras fs */
2280	static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
2281	amdgpu_ras_sysfs_badpages_read, NULL, 0);
2282	static DEVICE_ATTR(features, S_IRUGO,
2283	amdgpu_ras_sysfs_features_read, NULL);
2284	static DEVICE_ATTR(version, 0444,
2285	amdgpu_ras_sysfs_version_show, NULL);
2286	static DEVICE_ATTR(schema, 0444,
2287	amdgpu_ras_sysfs_schema_show, NULL);
2288	static DEVICE_ATTR(event_state, 0444,
2289	amdgpu_ras_sysfs_event_state_show, NULL);
2290	static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
2291	{
2292	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2293	struct attribute_group group = {
2294	.name = RAS_FS_NAME,
2295	};
2296	struct attribute *attrs[] = {
2297	&con->features_attr.attr,
2298	&con->version_attr.attr,
2299	&con->schema_attr.attr,
2300	&con->event_state_attr.attr,
2301	NULL
2302	};
2303	const struct bin_attribute *bin_attrs[] = {
2304	NULL,
2305	NULL,
2306	};
2307	int r;
2308
2309	group.attrs = attrs;
2310
2311	/* add features entry */
2312	con->features_attr = dev_attr_features;
2313	sysfs_attr_init(attrs[0]);
2314
2315	/* add version entry */
2316	con->version_attr = dev_attr_version;
2317	sysfs_attr_init(attrs[1]);
2318
2319	/* add schema entry */
2320	con->schema_attr = dev_attr_schema;
2321	sysfs_attr_init(attrs[2]);
2322
2323	/* add event_state entry */
2324	con->event_state_attr = dev_attr_event_state;
2325	sysfs_attr_init(attrs[3]);
2326
2327	if (amdgpu_bad_page_threshold != 0) {
2328	/* add bad_page_features entry */
2329	con->badpages_attr = bin_attr_gpu_vram_bad_pages;
2330	sysfs_bin_attr_init(&con->badpages_attr);
2331	bin_attrs[0] = &con->badpages_attr;
2332	group.bin_attrs = bin_attrs;
2333	}
2334
2335	r = sysfs_create_group(kobj: &adev->dev->kobj, grp: &group);
2336	if (r)
2337	dev_err(adev->dev, "Failed to create RAS sysfs group!");
2338
2339	return 0;
2340	}
2341
2342	static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
2343	{
2344	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2345	struct ras_manager *con_obj, *ip_obj, *tmp;
2346
2347	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2348	list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
2349	ip_obj = amdgpu_ras_find_obj(adev, head: &con_obj->head);
2350	if (ip_obj)
2351	put_obj(obj: ip_obj);
2352	}
2353	}
2354
2355	amdgpu_ras_sysfs_remove_all(adev);
2356	return 0;
2357	}
2358	/* ras fs end */
2359
2360	/* ih begin */
2361
2362	/* For the hardware that cannot enable bif ring for both ras_controller_irq
2363	* and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
2364	* register to check whether the interrupt is triggered or not, and properly
2365	* ack the interrupt if it is there
2366	*/
2367	void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
2368	{
2369	/* Fatal error events are handled on host side */
2370	if (amdgpu_sriov_vf(adev))
2371	return;
2372	/*
2373	* If the current interrupt is caused by a non-fatal RAS error, skip
2374	* check for fatal error. For fatal errors, FED status of all devices
2375	* in XGMI hive gets set when the first device gets fatal error
2376	* interrupt. The error gets propagated to other devices as well, so
2377	* make sure to ack the interrupt regardless of FED status.
2378	*/
2379	if (!amdgpu_ras_get_fed_status(adev) &&
2380	amdgpu_ras_is_err_state(adev, block: AMDGPU_RAS_BLOCK__ANY))
2381	return;
2382
2383	if (amdgpu_uniras_enabled(adev)) {
2384	amdgpu_ras_mgr_handle_fatal_interrupt(adev, NULL);
2385	return;
2386	}
2387
2388	if (adev->nbio.ras &&
2389	adev->nbio.ras->handle_ras_controller_intr_no_bifring)
2390	adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
2391
2392	if (adev->nbio.ras &&
2393	adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
2394	adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
2395	}
2396
2397	static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
2398	struct amdgpu_iv_entry *entry)
2399	{
2400	bool poison_stat = false;
2401	struct amdgpu_device *adev = obj->adev;
2402	struct amdgpu_ras_block_object *block_obj =
2403	amdgpu_ras_get_ras_block(adev, block: obj->head.block, sub_block_index: 0);
2404	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2405	enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION;
2406	u64 event_id;
2407	int ret;
2408
2409	if (!block_obj || !con)
2410	return;
2411
2412	ret = amdgpu_ras_mark_ras_event(adev, type);
2413	if (ret)
2414	return;
2415
2416	amdgpu_ras_set_err_poison(adev, block: block_obj->ras_comm.block);
2417	/* both query_poison_status and handle_poison_consumption are optional,
2418	* but at least one of them should be implemented if we need poison
2419	* consumption handler
2420	*/
2421	if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) {
2422	poison_stat = block_obj->hw_ops->query_poison_status(adev);
2423	if (!poison_stat) {
2424	/* Not poison consumption interrupt, no need to handle it */
2425	dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
2426	block_obj->ras_comm.name);
2427
2428	return;
2429	}
2430	}
2431
2432	amdgpu_umc_poison_handler(adev, block: obj->head.block, reset: 0);
2433
2434	if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
2435	poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
2436
2437	/* gpu reset is fallback for failed and default cases.
2438	* For RMA case, amdgpu_umc_poison_handler will handle gpu reset.
2439	*/
2440	if (poison_stat && !amdgpu_ras_is_rma(adev)) {
2441	event_id = amdgpu_ras_acquire_event_id(adev, type);
2442	RAS_EVENT_LOG(adev, event_id,
2443	"GPU reset for %s RAS poison consumption is issued!\n",
2444	block_obj->ras_comm.name);
2445	amdgpu_ras_reset_gpu(adev);
2446	}
2447
2448	if (!poison_stat)
2449	amdgpu_gfx_poison_consumption_handler(adev, entry);
2450	}
2451
2452	static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
2453	struct amdgpu_iv_entry *entry)
2454	{
2455	struct amdgpu_device *adev = obj->adev;
2456	enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
2457	u64 event_id;
2458	int ret;
2459
2460	ret = amdgpu_ras_mark_ras_event(adev, type);
2461	if (ret)
2462	return;
2463
2464	event_id = amdgpu_ras_acquire_event_id(adev, type);
2465	RAS_EVENT_LOG(adev, event_id, "Poison is created\n");
2466
2467	if (amdgpu_ip_version(adev: obj->adev, ip: UMC_HWIP, inst: 0) >= IP_VERSION(12, 0, 0)) {
2468	struct amdgpu_ras *con = amdgpu_ras_get_context(adev: obj->adev);
2469
2470	atomic_inc(v: &con->page_retirement_req_cnt);
2471	atomic_inc(v: &con->poison_creation_count);
2472
2473	wake_up(&con->page_retirement_wq);
2474	}
2475	}
2476
2477	static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
2478	struct amdgpu_iv_entry *entry)
2479	{
2480	struct ras_ih_data *data = &obj->ih_data;
2481	struct ras_err_data err_data;
2482	int ret;
2483
2484	if (!data->cb)
2485	return;
2486
2487	ret = amdgpu_ras_error_data_init(err_data: &err_data);
2488	if (ret)
2489	return;
2490
2491	/* Let IP handle its data, maybe we need get the output
2492	* from the callback to update the error type/count, etc
2493	*/
2494	amdgpu_ras_set_fed(adev: obj->adev, status: true);
2495	ret = data->cb(obj->adev, &err_data, entry);
2496	/* ue will trigger an interrupt, and in that case
2497	* we need do a reset to recovery the whole system.
2498	* But leave IP do that recovery, here we just dispatch
2499	* the error.
2500	*/
2501	if (ret == AMDGPU_RAS_SUCCESS) {
2502	/* these counts could be left as 0 if
2503	* some blocks do not count error number
2504	*/
2505	obj->err_data.ue_count += err_data.ue_count;
2506	obj->err_data.ce_count += err_data.ce_count;
2507	obj->err_data.de_count += err_data.de_count;
2508	}
2509
2510	amdgpu_ras_error_data_fini(err_data: &err_data);
2511	}
2512
2513	static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
2514	{
2515	struct ras_ih_data *data = &obj->ih_data;
2516	struct amdgpu_iv_entry entry;
2517
2518	while (data->rptr != data->wptr) {
2519	rmb();
2520	memcpy(&entry, &data->ring[data->rptr],
2521	data->element_size);
2522
2523	wmb();
2524	data->rptr = (data->aligned_element_size +
2525	data->rptr) % data->ring_size;
2526
2527	if (amdgpu_ras_is_poison_mode_supported(adev: obj->adev)) {
2528	if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2529	amdgpu_ras_interrupt_poison_creation_handler(obj, entry: &entry);
2530	else
2531	amdgpu_ras_interrupt_poison_consumption_handler(obj, entry: &entry);
2532	} else {
2533	if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2534	amdgpu_ras_interrupt_umc_handler(obj, entry: &entry);
2535	else
2536	dev_warn(obj->adev->dev,
2537	"No RAS interrupt handler for non-UMC block with poison disabled.\n");
2538	}
2539	}
2540	}
2541
2542	static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
2543	{
2544	struct ras_ih_data *data =
2545	container_of(work, struct ras_ih_data, ih_work);
2546	struct ras_manager *obj =
2547	container_of(data, struct ras_manager, ih_data);
2548
2549	amdgpu_ras_interrupt_handler(obj);
2550	}
2551
2552	int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
2553	struct ras_dispatch_if *info)
2554	{
2555	struct ras_manager *obj;
2556	struct ras_ih_data *data;
2557
2558	if (amdgpu_uniras_enabled(adev)) {
2559	struct ras_ih_info ih_info;
2560
2561	memset(&ih_info, 0, sizeof(ih_info));
2562	ih_info.block = info->head.block;
2563	memcpy(&ih_info.iv_entry, info->entry, sizeof(struct amdgpu_iv_entry));
2564
2565	return amdgpu_ras_mgr_handle_controller_interrupt(adev, data: &ih_info);
2566	}
2567
2568	obj = amdgpu_ras_find_obj(adev, head: &info->head);
2569	if (!obj)
2570	return -EINVAL;
2571
2572	data = &obj->ih_data;
2573
2574	if (data->inuse == 0)
2575	return 0;
2576
2577	/* Might be overflow... */
2578	memcpy(&data->ring[data->wptr], info->entry,
2579	data->element_size);
2580
2581	wmb();
2582	data->wptr = (data->aligned_element_size +
2583	data->wptr) % data->ring_size;
2584
2585	schedule_work(work: &data->ih_work);
2586
2587	return 0;
2588	}
2589
2590	int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
2591	struct ras_common_if *head)
2592	{
2593	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2594	struct ras_ih_data *data;
2595
2596	if (!obj)
2597	return -EINVAL;
2598
2599	data = &obj->ih_data;
2600	if (data->inuse == 0)
2601	return 0;
2602
2603	cancel_work_sync(work: &data->ih_work);
2604
2605	kfree(objp: data->ring);
2606	memset(data, 0, sizeof(*data));
2607	put_obj(obj);
2608
2609	return 0;
2610	}
2611
2612	int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
2613	struct ras_common_if *head)
2614	{
2615	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2616	struct ras_ih_data *data;
2617	struct amdgpu_ras_block_object *ras_obj;
2618
2619	if (!obj) {
2620	/* in case we registe the IH before enable ras feature */
2621	obj = amdgpu_ras_create_obj(adev, head);
2622	if (!obj)
2623	return -EINVAL;
2624	} else
2625	get_obj(obj);
2626
2627	ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
2628
2629	data = &obj->ih_data;
2630	/* add the callback.etc */
2631	*data = (struct ras_ih_data) {
2632	.inuse = 0,
2633	.cb = ras_obj->ras_cb,
2634	.element_size = sizeof(struct amdgpu_iv_entry),
2635	.rptr = 0,
2636	.wptr = 0,
2637	};
2638
2639	INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
2640
2641	data->aligned_element_size = ALIGN(data->element_size, 8);
2642	/* the ring can store 64 iv entries. */
2643	data->ring_size = 64 * data->aligned_element_size;
2644	data->ring = kmalloc(data->ring_size, GFP_KERNEL);
2645	if (!data->ring) {
2646	put_obj(obj);
2647	return -ENOMEM;
2648	}
2649
2650	/* IH is ready */
2651	data->inuse = 1;
2652
2653	return 0;
2654	}
2655
2656	static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
2657	{
2658	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2659	struct ras_manager *obj, *tmp;
2660
2661	list_for_each_entry_safe(obj, tmp, &con->head, node) {
2662	amdgpu_ras_interrupt_remove_handler(adev, head: &obj->head);
2663	}
2664
2665	return 0;
2666	}
2667	/* ih end */
2668
2669	/* traversal all IPs except NBIO to query error counter */
2670	static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type)
2671	{
2672	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2673	struct ras_manager *obj;
2674
2675	if (!adev->ras_enabled || !con)
2676	return;
2677
2678	list_for_each_entry(obj, &con->head, node) {
2679	struct ras_query_if info = {
2680	.head = obj->head,
2681	};
2682
2683	/*
2684	* PCIE_BIF IP has one different isr by ras controller
2685	* interrupt, the specific ras counter query will be
2686	* done in that isr. So skip such block from common
2687	* sync flood interrupt isr calling.
2688	*/
2689	if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
2690	continue;
2691
2692	/*
2693	* this is a workaround for aldebaran, skip send msg to
2694	* smu to get ecc_info table due to smu handle get ecc
2695	* info table failed temporarily.
2696	* should be removed until smu fix handle ecc_info table.
2697	*/
2698	if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
2699	(amdgpu_ip_version(adev, ip: MP1_HWIP, inst: 0) ==
2700	IP_VERSION(13, 0, 2)))
2701	continue;
2702
2703	amdgpu_ras_query_error_status_with_event(adev, info: &info, type);
2704
2705	if (amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0) !=
2706	IP_VERSION(11, 0, 2) &&
2707	amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0) !=
2708	IP_VERSION(11, 0, 4) &&
2709	amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0) !=
2710	IP_VERSION(13, 0, 0)) {
2711	if (amdgpu_ras_reset_error_status(adev, block: info.head.block))
2712	dev_warn(adev->dev, "Failed to reset error counter and error status");
2713	}
2714	}
2715	}
2716
2717	/* Parse RdRspStatus and WrRspStatus */
2718	static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
2719	struct ras_query_if *info)
2720	{
2721	struct amdgpu_ras_block_object *block_obj;
2722	/*
2723	* Only two block need to query read/write
2724	* RspStatus at current state
2725	*/
2726	if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
2727	(info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
2728	return;
2729
2730	block_obj = amdgpu_ras_get_ras_block(adev,
2731	block: info->head.block,
2732	sub_block_index: info->head.sub_block_index);
2733
2734	if (!block_obj || !block_obj->hw_ops) {
2735	dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
2736	get_ras_block_str(&info->head));
2737	return;
2738	}
2739
2740	if (block_obj->hw_ops->query_ras_error_status)
2741	block_obj->hw_ops->query_ras_error_status(adev);
2742
2743	}
2744
2745	static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
2746	{
2747	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2748	struct ras_manager *obj;
2749
2750	if (!adev->ras_enabled || !con)
2751	return;
2752
2753	list_for_each_entry(obj, &con->head, node) {
2754	struct ras_query_if info = {
2755	.head = obj->head,
2756	};
2757
2758	amdgpu_ras_error_status_query(adev, info: &info);
2759	}
2760	}
2761
2762	static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
2763	struct ras_badpage *bps, uint32_t count, uint32_t start)
2764	{
2765	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2766	struct ras_err_handler_data *data;
2767	int r = 0;
2768	uint32_t i;
2769
2770	if (!con || !con->eh_data || !bps || !count)
2771	return -EINVAL;
2772
2773	mutex_lock(&con->recovery_lock);
2774	data = con->eh_data;
2775	if (start < data->count) {
2776	for (i = start; i < data->count; i++) {
2777	if (!data->bps[i].ts)
2778	continue;
2779
2780	bps[r].bp = data->bps[i].retired_page;
2781	r++;
2782	if (r >= count)
2783	break;
2784	}
2785	}
2786	mutex_unlock(lock: &con->recovery_lock);
2787
2788	return r;
2789	}
2790
2791	static int amdgpu_uniras_badpages_read(struct amdgpu_device *adev,
2792	struct ras_badpage *bps, uint32_t count, uint32_t start)
2793	{
2794	struct ras_cmd_bad_pages_info_req cmd_input;
2795	struct ras_cmd_bad_pages_info_rsp *output;
2796	uint32_t group, start_group, end_group;
2797	uint32_t pos, pos_in_group;
2798	int r = 0, i;
2799
2800	if (!bps || !count)
2801	return -EINVAL;
2802
2803	output = kmalloc(sizeof(*output), GFP_KERNEL);
2804	if (!output)
2805	return -ENOMEM;
2806
2807	memset(&cmd_input, 0, sizeof(cmd_input));
2808
2809	start_group = start / RAS_CMD_MAX_BAD_PAGES_PER_GROUP;
2810	end_group = (start + count + RAS_CMD_MAX_BAD_PAGES_PER_GROUP - 1) /
2811	RAS_CMD_MAX_BAD_PAGES_PER_GROUP;
2812
2813	pos = start;
2814	for (group = start_group; group < end_group; group++) {
2815	memset(output, 0, sizeof(*output));
2816	cmd_input.group_index = group;
2817	if (amdgpu_ras_mgr_handle_ras_cmd(adev, cmd_id: RAS_CMD__GET_BAD_PAGES,
2818	input: &cmd_input, input_size: sizeof(cmd_input), output, out_size: sizeof(*output)))
2819	goto out;
2820
2821	if (pos >= output->bp_total_cnt)
2822	goto out;
2823
2824	pos_in_group = pos - group * RAS_CMD_MAX_BAD_PAGES_PER_GROUP;
2825	for (i = pos_in_group; i < output->bp_in_group; i++, pos++) {
2826	if (!output->records[i].ts)
2827	continue;
2828
2829	bps[r].bp = output->records[i].retired_page;
2830	r++;
2831	if (r >= count)
2832	goto out;
2833	}
2834	}
2835
2836	out:
2837	kfree(objp: output);
2838	return r;
2839	}
2840
2841	static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev,
2842	struct amdgpu_hive_info *hive, bool status)
2843	{
2844	struct amdgpu_device *tmp_adev;
2845
2846	if (hive) {
2847	list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
2848	amdgpu_ras_set_fed(adev: tmp_adev, status);
2849	} else {
2850	amdgpu_ras_set_fed(adev, status);
2851	}
2852	}
2853
2854	bool amdgpu_ras_in_recovery(struct amdgpu_device *adev)
2855	{
2856	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2857	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2858	int hive_ras_recovery = 0;
2859
2860	if (hive) {
2861	hive_ras_recovery = atomic_read(v: &hive->ras_recovery);
2862	amdgpu_put_xgmi_hive(hive);
2863	}
2864
2865	if (ras && (atomic_read(v: &ras->in_recovery) || hive_ras_recovery))
2866	return true;
2867
2868	return false;
2869	}
2870
2871	static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev)
2872	{
2873	if (amdgpu_ras_intr_triggered())
2874	return RAS_EVENT_TYPE_FATAL;
2875	else
2876	return RAS_EVENT_TYPE_POISON_CONSUMPTION;
2877	}
2878
2879	static void amdgpu_ras_do_recovery(struct work_struct *work)
2880	{
2881	struct amdgpu_ras *ras =
2882	container_of(work, struct amdgpu_ras, recovery_work);
2883	struct amdgpu_device *remote_adev = NULL;
2884	struct amdgpu_device *adev = ras->adev;
2885	struct list_head device_list, *device_list_handle = NULL;
2886	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2887	unsigned int error_query_mode;
2888	enum ras_event_type type;
2889
2890	if (hive) {
2891	atomic_set(v: &hive->ras_recovery, i: 1);
2892
2893	/* If any device which is part of the hive received RAS fatal
2894	* error interrupt, set fatal error status on all. This
2895	* condition will need a recovery, and flag will be cleared
2896	* as part of recovery.
2897	*/
2898	list_for_each_entry(remote_adev, &hive->device_list,
2899	gmc.xgmi.head)
2900	if (amdgpu_ras_get_fed_status(adev: remote_adev)) {
2901	amdgpu_ras_set_fed_all(adev, hive, status: true);
2902	break;
2903	}
2904	}
2905	if (!ras->disable_ras_err_cnt_harvest) {
2906
2907	/* Build list of devices to query RAS related errors */
2908	if (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
2909	device_list_handle = &hive->device_list;
2910	} else {
2911	INIT_LIST_HEAD(list: &device_list);
2912	list_add_tail(new: &adev->gmc.xgmi.head, head: &device_list);
2913	device_list_handle = &device_list;
2914	}
2915
2916	if (amdgpu_ras_get_error_query_mode(adev, mode: &error_query_mode)) {
2917	if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY) {
2918	/* wait 500ms to ensure pmfw polling mca bank info done */
2919	msleep(msecs: 500);
2920	}
2921	}
2922
2923	type = amdgpu_ras_get_fatal_error_event(adev);
2924	list_for_each_entry(remote_adev,
2925	device_list_handle, gmc.xgmi.head) {
2926	if (amdgpu_uniras_enabled(adev: remote_adev)) {
2927	amdgpu_ras_mgr_update_ras_ecc(adev: remote_adev);
2928	} else {
2929	amdgpu_ras_query_err_status(adev: remote_adev);
2930	amdgpu_ras_log_on_err_counter(adev: remote_adev, type);
2931	}
2932	}
2933
2934	}
2935
2936	if (amdgpu_device_should_recover_gpu(adev: ras->adev)) {
2937	struct amdgpu_reset_context reset_context;
2938	memset(&reset_context, 0, sizeof(reset_context));
2939
2940	reset_context.method = AMD_RESET_METHOD_NONE;
2941	reset_context.reset_req_dev = adev;
2942	reset_context.src = AMDGPU_RESET_SRC_RAS;
2943	set_bit(nr: AMDGPU_SKIP_COREDUMP, addr: &reset_context.flags);
2944
2945	/* Perform full reset in fatal error mode */
2946	if (!amdgpu_ras_is_poison_mode_supported(adev: ras->adev))
2947	set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context.flags);
2948	else {
2949	clear_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context.flags);
2950
2951	if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) {
2952	ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET;
2953	reset_context.method = AMD_RESET_METHOD_MODE2;
2954	}
2955
2956	/* Fatal error occurs in poison mode, mode1 reset is used to
2957	* recover gpu.
2958	*/
2959	if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) {
2960	ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2961	set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context.flags);
2962
2963	psp_fatal_error_recovery_quirk(&adev->psp);
2964	}
2965	}
2966
2967	amdgpu_device_gpu_recover(adev: ras->adev, NULL, reset_context: &reset_context);
2968	}
2969	atomic_set(v: &ras->in_recovery, i: 0);
2970	if (hive) {
2971	atomic_set(v: &hive->ras_recovery, i: 0);
2972	amdgpu_put_xgmi_hive(hive);
2973	}
2974	}
2975
2976	/* alloc/realloc bps array */
2977	static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
2978	struct ras_err_handler_data *data, int pages)
2979	{
2980	unsigned int old_space = data->count + data->space_left;
2981	unsigned int new_space = old_space + pages;
2982	unsigned int align_space = ALIGN(new_space, 512);
2983	void *bps = kmalloc_array(align_space, sizeof(*data->bps), GFP_KERNEL);
2984
2985	if (!bps) {
2986	return -ENOMEM;
2987	}
2988
2989	if (data->bps) {
2990	memcpy(bps, data->bps,
2991	data->count * sizeof(*data->bps));
2992	kfree(objp: data->bps);
2993	}
2994
2995	data->bps = bps;
2996	data->space_left += align_space - old_space;
2997	return 0;
2998	}
2999
3000	static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev,
3001	struct eeprom_table_record *bps,
3002	struct ras_err_data *err_data)
3003	{
3004	struct ta_ras_query_address_input addr_in;
3005	uint32_t socket = 0;
3006	int ret = 0;
3007
3008	if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
3009	socket = adev->smuio.funcs->get_socket_id(adev);
3010
3011	/* reinit err_data */
3012	err_data->err_addr_cnt = 0;
3013	err_data->err_addr_len = adev->umc.retire_unit;
3014
3015	memset(&addr_in, 0, sizeof(addr_in));
3016	addr_in.ma.err_addr = bps->address;
3017	addr_in.ma.socket_id = socket;
3018	addr_in.ma.ch_inst = bps->mem_channel;
3019	if (!amdgpu_ras_smu_eeprom_supported(adev)) {
3020	/* tell RAS TA the node instance is not used */
3021	addr_in.ma.node_inst = TA_RAS_INV_NODE;
3022	} else {
3023	addr_in.ma.umc_inst = bps->mcumc_id;
3024	addr_in.ma.node_inst = bps->cu;
3025	}
3026
3027	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
3028	ret = adev->umc.ras->convert_ras_err_addr(adev, err_data,
3029	&addr_in, NULL, false);
3030
3031	return ret;
3032	}
3033
3034	static int amdgpu_ras_mca2pa(struct amdgpu_device *adev,
3035	struct eeprom_table_record *bps,
3036	struct ras_err_data *err_data)
3037	{
3038	struct ta_ras_query_address_input addr_in;
3039	uint32_t die_id, socket = 0;
3040
3041	if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
3042	socket = adev->smuio.funcs->get_socket_id(adev);
3043
3044	/* although die id is gotten from PA in nps1 mode, the id is
3045	* fitable for any nps mode
3046	*/
3047	if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa)
3048	die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address,
3049	bps->retired_page << AMDGPU_GPU_PAGE_SHIFT);
3050	else
3051	return -EINVAL;
3052
3053	/* reinit err_data */
3054	err_data->err_addr_cnt = 0;
3055	err_data->err_addr_len = adev->umc.retire_unit;
3056
3057	memset(&addr_in, 0, sizeof(addr_in));
3058	addr_in.ma.err_addr = bps->address;
3059	addr_in.ma.ch_inst = bps->mem_channel;
3060	addr_in.ma.umc_inst = bps->mcumc_id;
3061	addr_in.ma.node_inst = die_id;
3062	addr_in.ma.socket_id = socket;
3063
3064	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
3065	return adev->umc.ras->convert_ras_err_addr(adev, err_data,
3066	&addr_in, NULL, false);
3067	else
3068	return -EINVAL;
3069	}
3070
3071	static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev,
3072	struct eeprom_table_record *bps, int count)
3073	{
3074	int j;
3075	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3076	struct ras_err_handler_data *data = con->eh_data;
3077
3078	for (j = 0; j < count; j++) {
3079	if (amdgpu_ras_check_bad_page_unlock(con,
3080	addr: bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT)) {
3081	data->count++;
3082	data->space_left--;
3083	continue;
3084	}
3085
3086	if (!data->space_left &&
3087	amdgpu_ras_realloc_eh_data_space(adev, data, pages: 256)) {
3088	return -ENOMEM;
3089	}
3090
3091	amdgpu_ras_reserve_page(adev, pfn: bps[j].retired_page);
3092
3093	memcpy(&data->bps[data->count], &(bps[j]),
3094	sizeof(struct eeprom_table_record));
3095	data->count++;
3096	data->space_left--;
3097	con->bad_page_num++;
3098	}
3099
3100	return 0;
3101	}
3102
3103	static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev,
3104	struct eeprom_table_record *bps, struct ras_err_data *err_data,
3105	enum amdgpu_memory_partition nps)
3106	{
3107	int i = 0;
3108	enum amdgpu_memory_partition save_nps;
3109
3110	save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
3111
3112	/*old asics just have pa in eeprom*/
3113	if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {
3114	memcpy(err_data->err_addr, bps,
3115	sizeof(struct eeprom_table_record) * adev->umc.retire_unit);
3116	goto out;
3117	}
3118
3119	for (i = 0; i < adev->umc.retire_unit; i++)
3120	bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
3121
3122	if (save_nps) {
3123	if (save_nps == nps) {
3124	if (amdgpu_umc_pages_in_a_row(adev, err_data,
3125	pa_addr: bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT))
3126	return -EINVAL;
3127	for (i = 0; i < adev->umc.retire_unit; i++) {
3128	err_data->err_addr[i].address = bps[0].address;
3129	err_data->err_addr[i].mem_channel = bps[0].mem_channel;
3130	err_data->err_addr[i].bank = bps[0].bank;
3131	err_data->err_addr[i].err_type = bps[0].err_type;
3132	err_data->err_addr[i].mcumc_id = bps[0].mcumc_id;
3133	}
3134	} else {
3135	if (amdgpu_ras_mca2pa_by_idx(adev, bps: &bps[0], err_data))
3136	return -EINVAL;
3137	}
3138	} else {
3139	if (bps[0].address == 0) {
3140	/* for specific old eeprom data, mca address is not stored,
3141	* calc it from pa
3142	*/
3143	if (amdgpu_umc_pa2mca(adev, pa: bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT,
3144	mca: &(bps[0].address), nps: AMDGPU_NPS1_PARTITION_MODE))
3145	return -EINVAL;
3146	}
3147
3148	if (amdgpu_ras_mca2pa(adev, bps: &bps[0], err_data)) {
3149	if (nps == AMDGPU_NPS1_PARTITION_MODE)
3150	memcpy(err_data->err_addr, bps,
3151	sizeof(struct eeprom_table_record) * adev->umc.retire_unit);
3152	else
3153	return -EOPNOTSUPP;
3154	}
3155	}
3156
3157	out:
3158	return __amdgpu_ras_restore_bad_pages(adev, bps: err_data->err_addr, count: adev->umc.retire_unit);
3159	}
3160
3161	static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev,
3162	struct eeprom_table_record *bps, struct ras_err_data *err_data,
3163	enum amdgpu_memory_partition nps)
3164	{
3165	int i = 0;
3166	enum amdgpu_memory_partition save_nps;
3167
3168	if (!amdgpu_ras_smu_eeprom_supported(adev)) {
3169	save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
3170	bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
3171	} else {
3172	/* if pmfw manages eeprom, save_nps is not stored on eeprom,
3173	* we should always convert mca address into physical address,
3174	* make save_nps different from nps
3175	*/
3176	save_nps = nps + 1;
3177	}
3178
3179	if (save_nps == nps) {
3180	if (amdgpu_umc_pages_in_a_row(adev, err_data,
3181	pa_addr: bps->retired_page << AMDGPU_GPU_PAGE_SHIFT))
3182	return -EINVAL;
3183	for (i = 0; i < adev->umc.retire_unit; i++) {
3184	err_data->err_addr[i].address = bps->address;
3185	err_data->err_addr[i].mem_channel = bps->mem_channel;
3186	err_data->err_addr[i].bank = bps->bank;
3187	err_data->err_addr[i].err_type = bps->err_type;
3188	err_data->err_addr[i].mcumc_id = bps->mcumc_id;
3189	}
3190	} else {
3191	if (bps->address) {
3192	if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data))
3193	return -EINVAL;
3194	} else {
3195	/* for specific old eeprom data, mca address is not stored,
3196	* calc it from pa
3197	*/
3198	if (amdgpu_umc_pa2mca(adev, pa: bps->retired_page << AMDGPU_GPU_PAGE_SHIFT,
3199	mca: &(bps->address), nps: AMDGPU_NPS1_PARTITION_MODE))
3200	return -EINVAL;
3201
3202	if (amdgpu_ras_mca2pa(adev, bps, err_data))
3203	return -EOPNOTSUPP;
3204	}
3205	}
3206
3207	return __amdgpu_ras_restore_bad_pages(adev, bps: err_data->err_addr,
3208	count: adev->umc.retire_unit);
3209	}
3210
3211	/* it deal with vram only. */
3212	int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
3213	struct eeprom_table_record *bps, int pages, bool from_rom)
3214	{
3215	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3216	struct ras_err_data err_data;
3217	struct amdgpu_ras_eeprom_control *control =
3218	&adev->psp.ras_context.ras->eeprom_control;
3219	enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE;
3220	int ret = 0;
3221	uint32_t i = 0;
3222
3223	if (!con || !con->eh_data || !bps || pages <= 0)
3224	return 0;
3225
3226	if (from_rom) {
3227	err_data.err_addr =
3228	kcalloc(adev->umc.retire_unit,
3229	sizeof(struct eeprom_table_record), GFP_KERNEL);
3230	if (!err_data.err_addr) {
3231	dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n");
3232	return -ENOMEM;
3233	}
3234
3235	if (adev->gmc.gmc_funcs->query_mem_partition_mode)
3236	nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
3237	}
3238
3239	mutex_lock(&con->recovery_lock);
3240
3241	if (from_rom) {
3242	/* there is no pa recs in V3, so skip pa recs processing */
3243	if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) &&
3244	!amdgpu_ras_smu_eeprom_supported(adev)) {
3245	for (i = 0; i < pages; i++) {
3246	if (control->ras_num_recs - i >= adev->umc.retire_unit) {
3247	if ((bps[i].address == bps[i + 1].address) &&
3248	(bps[i].mem_channel == bps[i + 1].mem_channel)) {
3249	/* deal with retire_unit records a time */
3250	ret = __amdgpu_ras_convert_rec_array_from_rom(adev,
3251	bps: &bps[i], err_data: &err_data, nps);
3252	if (ret)
3253	con->bad_page_num -= adev->umc.retire_unit;
3254	i += (adev->umc.retire_unit - 1);
3255	} else {
3256	break;
3257	}
3258	} else {
3259	break;
3260	}
3261	}
3262	}
3263	for (; i < pages; i++) {
3264	ret = __amdgpu_ras_convert_rec_from_rom(adev,
3265	bps: &bps[i], err_data: &err_data, nps);
3266	if (ret)
3267	con->bad_page_num -= adev->umc.retire_unit;
3268	}
3269
3270	con->eh_data->count_saved = con->eh_data->count;
3271	} else {
3272	ret = __amdgpu_ras_restore_bad_pages(adev, bps, count: pages);
3273	}
3274
3275	if (from_rom)
3276	kfree(objp: err_data.err_addr);
3277	mutex_unlock(lock: &con->recovery_lock);
3278
3279	return ret;
3280	}
3281
3282	/*
3283	* write error record array to eeprom, the function should be
3284	* protected by recovery_lock
3285	* new_cnt: new added UE count, excluding reserved bad pages, can be NULL
3286	*/
3287	int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
3288	unsigned long *new_cnt)
3289	{
3290	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3291	struct ras_err_handler_data *data;
3292	struct amdgpu_ras_eeprom_control *control;
3293	int save_count, unit_num, i;
3294
3295	if (!con || !con->eh_data) {
3296	if (new_cnt)
3297	*new_cnt = 0;
3298
3299	return 0;
3300	}
3301
3302	if (!con->eeprom_control.is_eeprom_valid) {
3303	dev_warn(adev->dev,
3304	"Failed to save EEPROM table data because of EEPROM data corruption!");
3305	if (new_cnt)
3306	*new_cnt = 0;
3307
3308	return 0;
3309	}
3310
3311	mutex_lock(&con->recovery_lock);
3312	control = &con->eeprom_control;
3313	data = con->eh_data;
3314	if (amdgpu_ras_smu_eeprom_supported(adev))
3315	unit_num = control->ras_num_recs -
3316	control->ras_num_recs_old;
3317	else
3318	unit_num = data->count / adev->umc.retire_unit -
3319	control->ras_num_recs;
3320
3321	save_count = con->bad_page_num - control->ras_num_bad_pages;
3322	mutex_unlock(lock: &con->recovery_lock);
3323
3324	if (new_cnt)
3325	*new_cnt = unit_num;
3326
3327	/* only new entries are saved */
3328	if (unit_num && save_count) {
3329	/*old asics only save pa to eeprom like before*/
3330	if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {
3331	if (amdgpu_ras_eeprom_append(control,
3332	records: &data->bps[data->count_saved], num: unit_num)) {
3333	dev_err(adev->dev, "Failed to save EEPROM table data!");
3334	return -EIO;
3335	}
3336	} else {
3337	for (i = 0; i < unit_num; i++) {
3338	if (amdgpu_ras_eeprom_append(control,
3339	records: &data->bps[data->count_saved +
3340	i * adev->umc.retire_unit], num: 1)) {
3341	dev_err(adev->dev, "Failed to save EEPROM table data!");
3342	return -EIO;
3343	}
3344	}
3345	}
3346
3347	dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
3348	data->count_saved = data->count;
3349	}
3350
3351	return 0;
3352	}
3353
3354	/*
3355	* read error record array in eeprom and reserve enough space for
3356	* storing new bad pages
3357	*/
3358	static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
3359	{
3360	struct amdgpu_ras_eeprom_control *control =
3361	&adev->psp.ras_context.ras->eeprom_control;
3362	struct eeprom_table_record *bps;
3363	int ret, i = 0;
3364
3365	/* no bad page record, skip eeprom access */
3366	if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
3367	return 0;
3368
3369	bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
3370	if (!bps)
3371	return -ENOMEM;
3372
3373	ret = amdgpu_ras_eeprom_read(control, records: bps, num: control->ras_num_recs);
3374	if (ret) {
3375	dev_err(adev->dev, "Failed to load EEPROM table records!");
3376	} else {
3377	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
3378	/*In V3, there is no pa recs, and some cases(when address==0) may be parsed
3379	as pa recs, so add verion check to avoid it.
3380	*/
3381	if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) &&
3382	!amdgpu_ras_smu_eeprom_supported(adev)) {
3383	for (i = 0; i < control->ras_num_recs; i++) {
3384	if ((control->ras_num_recs - i) >= adev->umc.retire_unit) {
3385	if ((bps[i].address == bps[i + 1].address) &&
3386	(bps[i].mem_channel == bps[i + 1].mem_channel)) {
3387	control->ras_num_pa_recs += adev->umc.retire_unit;
3388	i += (adev->umc.retire_unit - 1);
3389	} else {
3390	control->ras_num_mca_recs +=
3391	(control->ras_num_recs - i);
3392	break;
3393	}
3394	} else {
3395	control->ras_num_mca_recs += (control->ras_num_recs - i);
3396	break;
3397	}
3398	}
3399	} else {
3400	control->ras_num_mca_recs = control->ras_num_recs;
3401	}
3402	}
3403
3404	ret = amdgpu_ras_add_bad_pages(adev, bps, pages: control->ras_num_recs, from_rom: true);
3405	if (ret)
3406	goto out;
3407
3408	ret = amdgpu_ras_eeprom_check(control);
3409	if (ret)
3410	goto out;
3411
3412	/* HW not usable */
3413	if (amdgpu_ras_is_rma(adev))
3414	ret = -EHWPOISON;
3415	}
3416
3417	out:
3418	kfree(objp: bps);
3419	return ret;
3420	}
3421
3422	static int amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
3423	uint64_t addr)
3424	{
3425	struct ras_err_handler_data *data = con->eh_data;
3426	struct amdgpu_device *adev = con->adev;
3427	int i;
3428
3429	if ((addr >= adev->gmc.mc_vram_size &&
3430	adev->gmc.mc_vram_size) ||
3431	(addr >= RAS_UMC_INJECT_ADDR_LIMIT))
3432	return -EINVAL;
3433
3434	addr >>= AMDGPU_GPU_PAGE_SHIFT;
3435	for (i = 0; i < data->count; i++)
3436	if (addr == data->bps[i].retired_page)
3437	return 1;
3438
3439	return 0;
3440	}
3441
3442	/*
3443	* check if an address belongs to bad page
3444	*
3445	* Note: this check is only for umc block
3446	*/
3447	static int amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
3448	uint64_t addr)
3449	{
3450	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3451	int ret = 0;
3452
3453	if (!con || !con->eh_data)
3454	return ret;
3455
3456	mutex_lock(&con->recovery_lock);
3457	ret = amdgpu_ras_check_bad_page_unlock(con, addr);
3458	mutex_unlock(lock: &con->recovery_lock);
3459	return ret;
3460	}
3461
3462	static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
3463	uint32_t max_count)
3464	{
3465	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3466
3467	/*
3468	* amdgpu_bad_page_threshold is used to config
3469	* the threshold for the number of bad pages.
3470	* -1: Threshold is set to default value
3471	* Driver will issue a warning message when threshold is reached
3472	* and continue runtime services.
3473	* 0: Disable bad page retirement
3474	* Driver will not retire bad pages
3475	* which is intended for debugging purpose.
3476	* -2: Threshold is determined by a formula
3477	* that assumes 1 bad page per 100M of local memory.
3478	* Driver will continue runtime services when threhold is reached.
3479	* 0 < threshold < max number of bad page records in EEPROM,
3480	* A user-defined threshold is set
3481	* Driver will halt runtime services when this custom threshold is reached.
3482	*/
3483	if (amdgpu_bad_page_threshold == -2) {
3484	u64 val = adev->gmc.mc_vram_size;
3485
3486	do_div(val, RAS_BAD_PAGE_COVER);
3487	con->bad_page_cnt_threshold = min(lower_32_bits(val),
3488	max_count);
3489	} else if (amdgpu_bad_page_threshold == -1) {
3490	con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4;
3491	} else {
3492	con->bad_page_cnt_threshold = min_t(int, max_count,
3493	amdgpu_bad_page_threshold);
3494	}
3495	}
3496
3497	int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
3498	enum amdgpu_ras_block block, uint16_t pasid,
3499	pasid_notify pasid_fn, void *data, uint32_t reset)
3500	{
3501	int ret = 0;
3502	struct ras_poison_msg poison_msg;
3503	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3504
3505	memset(&poison_msg, 0, sizeof(poison_msg));
3506	poison_msg.block = block;
3507	poison_msg.pasid = pasid;
3508	poison_msg.reset = reset;
3509	poison_msg.pasid_fn = pasid_fn;
3510	poison_msg.data = data;
3511
3512	ret = kfifo_put(&con->poison_fifo, poison_msg);
3513	if (!ret) {
3514	dev_err(adev->dev, "Poison message fifo is full!\n");
3515	return -ENOSPC;
3516	}
3517
3518	return 0;
3519	}
3520
3521	static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev,
3522	struct ras_poison_msg *poison_msg)
3523	{
3524	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3525
3526	return kfifo_get(&con->poison_fifo, poison_msg);
3527	}
3528
3529	static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log)
3530	{
3531	mutex_init(&ecc_log->lock);
3532
3533	INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL);
3534	ecc_log->de_queried_count = 0;
3535	ecc_log->consumption_q_count = 0;
3536	}
3537
3538	static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log)
3539	{
3540	struct radix_tree_iter iter;
3541	void __rcu **slot;
3542	struct ras_ecc_err *ecc_err;
3543
3544	mutex_lock(&ecc_log->lock);
3545	radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) {
3546	ecc_err = radix_tree_deref_slot(slot);
3547	kfree(objp: ecc_err->err_pages.pfn);
3548	kfree(objp: ecc_err);
3549	radix_tree_iter_delete(&ecc_log->de_page_tree, iter: &iter, slot);
3550	}
3551	mutex_unlock(lock: &ecc_log->lock);
3552
3553	mutex_destroy(lock: &ecc_log->lock);
3554	ecc_log->de_queried_count = 0;
3555	ecc_log->consumption_q_count = 0;
3556	}
3557
3558	static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con,
3559	uint32_t delayed_ms)
3560	{
3561	int ret;
3562
3563	mutex_lock(&con->umc_ecc_log.lock);
3564	ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree,
3565	UMC_ECC_NEW_DETECTED_TAG);
3566	mutex_unlock(lock: &con->umc_ecc_log.lock);
3567
3568	if (ret)
3569	schedule_delayed_work(dwork: &con->page_retirement_dwork,
3570	delay: msecs_to_jiffies(m: delayed_ms));
3571
3572	return ret ? true : false;
3573	}
3574
3575	static void amdgpu_ras_do_page_retirement(struct work_struct *work)
3576	{
3577	struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3578	page_retirement_dwork.work);
3579	struct amdgpu_device *adev = con->adev;
3580	struct ras_err_data err_data;
3581
3582	/* If gpu reset is ongoing, delay retiring the bad pages */
3583	if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) {
3584	amdgpu_ras_schedule_retirement_dwork(con,
3585	AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3);
3586	return;
3587	}
3588
3589	amdgpu_ras_error_data_init(err_data: &err_data);
3590
3591	amdgpu_umc_handle_bad_pages(adev, ras_error_status: &err_data);
3592
3593	amdgpu_ras_error_data_fini(err_data: &err_data);
3594
3595	amdgpu_ras_schedule_retirement_dwork(con,
3596	AMDGPU_RAS_RETIRE_PAGE_INTERVAL);
3597	}
3598
3599	static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev,
3600	uint32_t poison_creation_count)
3601	{
3602	int ret = 0;
3603	struct ras_ecc_log_info *ecc_log;
3604	struct ras_query_if info;
3605	u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC;
3606	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3607	u64 de_queried_count;
3608	u64 consumption_q_count;
3609	enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
3610
3611	memset(&info, 0, sizeof(info));
3612	info.head.block = AMDGPU_RAS_BLOCK__UMC;
3613
3614	ecc_log = &ras->umc_ecc_log;
3615	ecc_log->de_queried_count = 0;
3616	ecc_log->consumption_q_count = 0;
3617
3618	do {
3619	ret = amdgpu_ras_query_error_status_with_event(adev, info: &info, type);
3620	if (ret)
3621	return ret;
3622
3623	de_queried_count = ecc_log->de_queried_count;
3624	consumption_q_count = ecc_log->consumption_q_count;
3625
3626	if (de_queried_count && consumption_q_count)
3627	break;
3628
3629	msleep(msecs: 100);
3630	} while (--timeout);
3631
3632	if (de_queried_count)
3633	schedule_delayed_work(dwork: &ras->page_retirement_dwork, delay: 0);
3634
3635	if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(v: &ras->rma_in_recovery, old: 0, new: 1) == 0)
3636	amdgpu_ras_reset_gpu(adev);
3637
3638	return 0;
3639	}
3640
3641	static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev)
3642	{
3643	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3644	struct ras_poison_msg msg;
3645	int ret;
3646
3647	do {
3648	ret = kfifo_get(&con->poison_fifo, &msg);
3649	} while (ret);
3650	}
3651
3652	static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev,
3653	uint32_t msg_count, uint32_t *gpu_reset)
3654	{
3655	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3656	uint32_t reset_flags = 0, reset = 0;
3657	struct ras_poison_msg msg;
3658	int ret, i;
3659
3660	kgd2kfd_set_sram_ecc_flag(kfd: adev->kfd.dev);
3661
3662	for (i = 0; i < msg_count; i++) {
3663	ret = amdgpu_ras_get_poison_req(adev, poison_msg: &msg);
3664	if (!ret)
3665	continue;
3666
3667	if (msg.pasid_fn)
3668	msg.pasid_fn(adev, msg.pasid, msg.data);
3669
3670	reset_flags |= msg.reset;
3671	}
3672
3673	/*
3674	* Try to ensure poison creation handler is completed first
3675	* to set rma if bad page exceed threshold.
3676	*/
3677	flush_delayed_work(dwork: &con->page_retirement_dwork);
3678
3679	/* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */
3680	if (reset_flags && !amdgpu_ras_is_rma(adev)) {
3681	if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET)
3682	reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET;
3683	else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET)
3684	reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
3685	else
3686	reset = reset_flags;
3687
3688	con->gpu_reset_flags |= reset;
3689	amdgpu_ras_reset_gpu(adev);
3690
3691	*gpu_reset = reset;
3692
3693	/* Wait for gpu recovery to complete */
3694	flush_work(work: &con->recovery_work);
3695	}
3696
3697	return 0;
3698	}
3699
3700	static int amdgpu_ras_page_retirement_thread(void *param)
3701	{
3702	struct amdgpu_device *adev = (struct amdgpu_device *)param;
3703	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3704	uint32_t poison_creation_count, msg_count;
3705	uint32_t gpu_reset;
3706	int ret;
3707
3708	while (!kthread_should_stop()) {
3709
3710	wait_event_interruptible(con->page_retirement_wq,
3711	kthread_should_stop() ||
3712	atomic_read(&con->page_retirement_req_cnt));
3713
3714	if (kthread_should_stop())
3715	break;
3716
3717	mutex_lock(&con->poison_lock);
3718	gpu_reset = 0;
3719
3720	do {
3721	poison_creation_count = atomic_read(v: &con->poison_creation_count);
3722	ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count);
3723	if (ret == -EIO)
3724	break;
3725
3726	if (poison_creation_count) {
3727	atomic_sub(i: poison_creation_count, v: &con->poison_creation_count);
3728	atomic_sub(i: poison_creation_count, v: &con->page_retirement_req_cnt);
3729	}
3730	} while (atomic_read(v: &con->poison_creation_count) &&
3731	!atomic_read(v: &con->poison_consumption_count));
3732
3733	if (ret != -EIO) {
3734	msg_count = kfifo_len(&con->poison_fifo);
3735	if (msg_count) {
3736	ret = amdgpu_ras_poison_consumption_handler(adev,
3737	msg_count, gpu_reset: &gpu_reset);
3738	if ((ret != -EIO) &&
3739	(gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET))
3740	atomic_sub(i: msg_count, v: &con->page_retirement_req_cnt);
3741	}
3742	}
3743
3744	if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) {
3745	/* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */
3746	/* Clear poison creation request */
3747	atomic_set(v: &con->poison_creation_count, i: 0);
3748	atomic_set(v: &con->poison_consumption_count, i: 0);
3749
3750	/* Clear poison fifo */
3751	amdgpu_ras_clear_poison_fifo(adev);
3752
3753	/* Clear all poison requests */
3754	atomic_set(v: &con->page_retirement_req_cnt, i: 0);
3755
3756	if (ret == -EIO) {
3757	/* Wait for mode-1 reset to complete */
3758	down_read(sem: &adev->reset_domain->sem);
3759	up_read(sem: &adev->reset_domain->sem);
3760	}
3761
3762	/* Wake up work to save bad pages to eeprom */
3763	schedule_delayed_work(dwork: &con->page_retirement_dwork, delay: 0);
3764	} else if (gpu_reset) {
3765	/* gpu just completed mode-2 reset or other reset */
3766	/* Clear poison consumption messages cached in fifo */
3767	msg_count = kfifo_len(&con->poison_fifo);
3768	if (msg_count) {
3769	amdgpu_ras_clear_poison_fifo(adev);
3770	atomic_sub(i: msg_count, v: &con->page_retirement_req_cnt);
3771	}
3772
3773	atomic_set(v: &con->poison_consumption_count, i: 0);
3774
3775	/* Wake up work to save bad pages to eeprom */
3776	schedule_delayed_work(dwork: &con->page_retirement_dwork, delay: 0);
3777	}
3778	mutex_unlock(lock: &con->poison_lock);
3779	}
3780
3781	return 0;
3782	}
3783
3784	int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
3785	{
3786	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3787	struct amdgpu_ras_eeprom_control *control;
3788	int ret;
3789
3790	if (!con || amdgpu_sriov_vf(adev))
3791	return 0;
3792
3793	if (amdgpu_uniras_enabled(adev))
3794	return 0;
3795
3796	control = &con->eeprom_control;
3797	con->ras_smu_drv = amdgpu_dpm_get_ras_smu_driver(adev);
3798
3799	ret = amdgpu_ras_eeprom_init(control);
3800	control->is_eeprom_valid = !ret;
3801
3802	if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr)
3803	control->ras_num_pa_recs = control->ras_num_recs;
3804
3805	if (adev->umc.ras &&
3806	adev->umc.ras->get_retire_flip_bits)
3807	adev->umc.ras->get_retire_flip_bits(adev);
3808
3809	if (control->ras_num_recs && control->is_eeprom_valid) {
3810	ret = amdgpu_ras_load_bad_pages(adev);
3811	if (ret) {
3812	control->is_eeprom_valid = false;
3813	return 0;
3814	}
3815
3816	amdgpu_dpm_send_hbm_bad_pages_num(
3817	adev, size: control->ras_num_bad_pages);
3818
3819	if (con->update_channel_flag == true) {
3820	amdgpu_dpm_send_hbm_bad_channel_flag(
3821	adev, size: control->bad_channel_bitmap);
3822	con->update_channel_flag = false;
3823	}
3824
3825	/* The format action is only applied to new ASICs */
3826	if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 &&
3827	control->tbl_hdr.version < RAS_TABLE_VER_V3)
3828	if (!amdgpu_ras_eeprom_reset_table(control))
3829	if (amdgpu_ras_save_bad_pages(adev, NULL))
3830	dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n");
3831	}
3832
3833	return 0;
3834	}
3835
3836	int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info)
3837	{
3838	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3839	struct ras_err_handler_data **data;
3840	u32 max_eeprom_records_count = 0;
3841	int ret;
3842
3843	if (!con || amdgpu_sriov_vf(adev))
3844	return 0;
3845
3846	/* Allow access to RAS EEPROM via debugfs, when the ASIC
3847	* supports RAS and debugfs is enabled, but when
3848	* adev->ras_enabled is unset, i.e. when "ras_enable"
3849	* module parameter is set to 0.
3850	*/
3851	con->adev = adev;
3852
3853	if (!adev->ras_enabled)
3854	return 0;
3855
3856	data = &con->eh_data;
3857	*data = kzalloc(sizeof(**data), GFP_KERNEL);
3858	if (!*data) {
3859	ret = -ENOMEM;
3860	goto out;
3861	}
3862
3863	mutex_init(&con->recovery_lock);
3864	mutex_init(&con->poison_lock);
3865	INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
3866	atomic_set(v: &con->in_recovery, i: 0);
3867	atomic_set(v: &con->rma_in_recovery, i: 0);
3868	con->eeprom_control.bad_channel_bitmap = 0;
3869
3870	max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(control: &con->eeprom_control);
3871	amdgpu_ras_validate_threshold(adev, max_count: max_eeprom_records_count);
3872
3873	if (init_bp_info) {
3874	ret = amdgpu_ras_init_badpage_info(adev);
3875	if (ret)
3876	goto free;
3877	}
3878
3879	mutex_init(&con->page_rsv_lock);
3880	INIT_KFIFO(con->poison_fifo);
3881	mutex_init(&con->page_retirement_lock);
3882	init_waitqueue_head(&con->page_retirement_wq);
3883	atomic_set(v: &con->page_retirement_req_cnt, i: 0);
3884	atomic_set(v: &con->poison_creation_count, i: 0);
3885	atomic_set(v: &con->poison_consumption_count, i: 0);
3886	con->page_retirement_thread =
3887	kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement");
3888	if (IS_ERR(ptr: con->page_retirement_thread)) {
3889	con->page_retirement_thread = NULL;
3890	dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n");
3891	}
3892
3893	INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement);
3894	amdgpu_ras_ecc_log_init(ecc_log: &con->umc_ecc_log);
3895	#ifdef CONFIG_X86_MCE_AMD
3896	if ((adev->asic_type == CHIP_ALDEBARAN) &&
3897	(adev->gmc.xgmi.connected_to_cpu))
3898	amdgpu_register_bad_pages_mca_notifier(adev);
3899	#endif
3900	return 0;
3901
3902	free:
3903	kfree(objp: (*data)->bps);
3904	kfree(objp: *data);
3905	con->eh_data = NULL;
3906	out:
3907	dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
3908
3909	/*
3910	* Except error threshold exceeding case, other failure cases in this
3911	* function would not fail amdgpu driver init.
3912	*/
3913	if (!amdgpu_ras_is_rma(adev))
3914	ret = 0;
3915	else
3916	ret = -EINVAL;
3917
3918	return ret;
3919	}
3920
3921	static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
3922	{
3923	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3924	struct ras_err_handler_data *data = con->eh_data;
3925	int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES;
3926	bool ret;
3927
3928	/* recovery_init failed to init it, fini is useless */
3929	if (!data)
3930	return 0;
3931
3932	/* Save all cached bad pages to eeprom */
3933	do {
3934	flush_delayed_work(dwork: &con->page_retirement_dwork);
3935	ret = amdgpu_ras_schedule_retirement_dwork(con, delayed_ms: 0);
3936	} while (ret && max_flush_timeout--);
3937
3938	if (con->page_retirement_thread)
3939	kthread_stop(k: con->page_retirement_thread);
3940
3941	atomic_set(v: &con->page_retirement_req_cnt, i: 0);
3942	atomic_set(v: &con->poison_creation_count, i: 0);
3943
3944	mutex_destroy(lock: &con->page_rsv_lock);
3945
3946	cancel_work_sync(work: &con->recovery_work);
3947
3948	cancel_delayed_work_sync(dwork: &con->page_retirement_dwork);
3949
3950	amdgpu_ras_ecc_log_fini(ecc_log: &con->umc_ecc_log);
3951
3952	mutex_lock(&con->recovery_lock);
3953	con->eh_data = NULL;
3954	kfree(objp: data->bps);
3955	kfree(objp: data);
3956	mutex_unlock(lock: &con->recovery_lock);
3957
3958	amdgpu_ras_critical_region_init(adev);
3959	#ifdef CONFIG_X86_MCE_AMD
3960	amdgpu_unregister_bad_pages_mca_notifier(adev);
3961	#endif
3962	return 0;
3963	}
3964	/* recovery end */
3965
3966	static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
3967	{
3968	if (amdgpu_sriov_vf(adev)) {
3969	switch (amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0)) {
3970	case IP_VERSION(13, 0, 2):
3971	case IP_VERSION(13, 0, 6):
3972	case IP_VERSION(13, 0, 12):
3973	case IP_VERSION(13, 0, 14):
3974	return true;
3975	default:
3976	return false;
3977	}
3978	}
3979
3980	if (adev->asic_type == CHIP_IP_DISCOVERY) {
3981	switch (amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0)) {
3982	case IP_VERSION(13, 0, 0):
3983	case IP_VERSION(13, 0, 6):
3984	case IP_VERSION(13, 0, 10):
3985	case IP_VERSION(13, 0, 12):
3986	case IP_VERSION(13, 0, 14):
3987	case IP_VERSION(14, 0, 3):
3988	return true;
3989	default:
3990	return false;
3991	}
3992	}
3993
3994	return adev->asic_type == CHIP_VEGA10 ||
3995	adev->asic_type == CHIP_VEGA20 ||
3996	adev->asic_type == CHIP_ARCTURUS ||
3997	adev->asic_type == CHIP_ALDEBARAN ||
3998	adev->asic_type == CHIP_SIENNA_CICHLID;
3999	}
4000
4001	/*
4002	* this is workaround for vega20 workstation sku,
4003	* force enable gfx ras, ignore vbios gfx ras flag
4004	* due to GC EDC can not write
4005	*/
4006	static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
4007	{
4008	struct atom_context *ctx = adev->mode_info.atom_context;
4009
4010	if (!ctx)
4011	return;
4012
4013	if (strnstr(ctx->vbios_pn, "D16406",
4014	sizeof(ctx->vbios_pn)) ||
4015	strnstr(ctx->vbios_pn, "D36002",
4016	sizeof(ctx->vbios_pn)))
4017	adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
4018	}
4019
4020	/* Query ras capablity via atomfirmware interface */
4021	static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev)
4022	{
4023	/* mem_ecc cap */
4024	if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
4025	dev_info(adev->dev, "MEM ECC is active.\n");
4026	adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
4027	1 << AMDGPU_RAS_BLOCK__DF);
4028	} else {
4029	dev_info(adev->dev, "MEM ECC is not presented.\n");
4030	}
4031
4032	/* sram_ecc cap */
4033	if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
4034	dev_info(adev->dev, "SRAM ECC is active.\n");
4035	if (!amdgpu_sriov_vf(adev))
4036	adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
4037	1 << AMDGPU_RAS_BLOCK__DF);
4038	else
4039	adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
4040	1 << AMDGPU_RAS_BLOCK__SDMA |
4041	1 << AMDGPU_RAS_BLOCK__GFX);
4042
4043	/*
4044	* VCN/JPEG RAS can be supported on both bare metal and
4045	* SRIOV environment
4046	*/
4047	if (amdgpu_ip_version(adev, ip: VCN_HWIP, inst: 0) == IP_VERSION(2, 6, 0) ||
4048	amdgpu_ip_version(adev, ip: VCN_HWIP, inst: 0) == IP_VERSION(4, 0, 0) ||
4049	amdgpu_ip_version(adev, ip: VCN_HWIP, inst: 0) == IP_VERSION(4, 0, 3) ||
4050	amdgpu_ip_version(adev, ip: VCN_HWIP, inst: 0) == IP_VERSION(5, 0, 1))
4051	adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
4052	1 << AMDGPU_RAS_BLOCK__JPEG);
4053	else
4054	adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
4055	1 << AMDGPU_RAS_BLOCK__JPEG);
4056
4057	/*
4058	* XGMI RAS is not supported if xgmi num physical nodes
4059	* is zero
4060	*/
4061	if (!adev->gmc.xgmi.num_physical_nodes)
4062	adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
4063	} else {
4064	dev_info(adev->dev, "SRAM ECC is not presented.\n");
4065	}
4066	}
4067
4068	/* Query poison mode from umc/df IP callbacks */
4069	static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev)
4070	{
4071	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4072	bool df_poison, umc_poison;
4073
4074	/* poison setting is useless on SRIOV guest */
4075	if (amdgpu_sriov_vf(adev) || !con)
4076	return;
4077
4078	/* Init poison supported flag, the default value is false */
4079	if (adev->gmc.xgmi.connected_to_cpu ||
4080	adev->gmc.is_app_apu) {
4081	/* enabled by default when GPU is connected to CPU */
4082	con->poison_supported = true;
4083	} else if (adev->df.funcs &&
4084	adev->df.funcs->query_ras_poison_mode &&
4085	adev->umc.ras &&
4086	adev->umc.ras->query_ras_poison_mode) {
4087	df_poison =
4088	adev->df.funcs->query_ras_poison_mode(adev);
4089	umc_poison =
4090	adev->umc.ras->query_ras_poison_mode(adev);
4091
4092	/* Only poison is set in both DF and UMC, we can support it */
4093	if (df_poison && umc_poison)
4094	con->poison_supported = true;
4095	else if (df_poison != umc_poison)
4096	dev_warn(adev->dev,
4097	"Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
4098	df_poison, umc_poison);
4099	}
4100	}
4101
4102	/*
4103	* check hardware's ras ability which will be saved in hw_supported.
4104	* if hardware does not support ras, we can skip some ras initializtion and
4105	* forbid some ras operations from IP.
4106	* if software itself, say boot parameter, limit the ras ability. We still
4107	* need allow IP do some limited operations, like disable. In such case,
4108	* we have to initialize ras as normal. but need check if operation is
4109	* allowed or not in each function.
4110	*/
4111	static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
4112	{
4113	adev->ras_hw_enabled = adev->ras_enabled = 0;
4114
4115	if (!amdgpu_ras_asic_supported(adev))
4116	return;
4117
4118	if (amdgpu_sriov_vf(adev)) {
4119	if (amdgpu_virt_get_ras_capability(adev))
4120	goto init_ras_enabled_flag;
4121	}
4122
4123	/* query ras capability from psp */
4124	if (amdgpu_psp_get_ras_capability(psp: &adev->psp))
4125	goto init_ras_enabled_flag;
4126
4127	/* query ras capablity from bios */
4128	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4129	amdgpu_ras_query_ras_capablity_from_vbios(adev);
4130	} else {
4131	/* driver only manages a few IP blocks RAS feature
4132	* when GPU is connected cpu through XGMI */
4133	adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
4134	1 << AMDGPU_RAS_BLOCK__SDMA |
4135	1 << AMDGPU_RAS_BLOCK__MMHUB);
4136	}
4137
4138	/* apply asic specific settings (vega20 only for now) */
4139	amdgpu_ras_get_quirks(adev);
4140
4141	/* query poison mode from umc/df ip callback */
4142	amdgpu_ras_query_poison_mode(adev);
4143
4144	init_ras_enabled_flag:
4145	/* hw_supported needs to be aligned with RAS block mask. */
4146	adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
4147
4148	adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
4149	adev->ras_hw_enabled & amdgpu_ras_mask;
4150
4151	/* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */
4152	if (!amdgpu_sriov_vf(adev)) {
4153	adev->aca.is_enabled =
4154	(amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0) == IP_VERSION(13, 0, 6) ||
4155	amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0) == IP_VERSION(13, 0, 12) ||
4156	amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0) == IP_VERSION(13, 0, 14));
4157	}
4158
4159	/* bad page feature is not applicable to specific app platform */
4160	if (adev->gmc.is_app_apu &&
4161	amdgpu_ip_version(adev, ip: UMC_HWIP, inst: 0) == IP_VERSION(12, 0, 0))
4162	amdgpu_bad_page_threshold = 0;
4163	}
4164
4165	static void amdgpu_ras_counte_dw(struct work_struct *work)
4166	{
4167	struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
4168	ras_counte_delay_work.work);
4169	struct amdgpu_device *adev = con->adev;
4170	struct drm_device *dev = adev_to_drm(adev);
4171	unsigned long ce_count, ue_count;
4172	int res;
4173
4174	res = pm_runtime_get_sync(dev: dev->dev);
4175	if (res < 0)
4176	goto Out;
4177
4178	/* Cache new values.
4179	*/
4180	if (amdgpu_ras_query_error_count(adev, ce_count: &ce_count, ue_count: &ue_count, NULL) == 0) {
4181	atomic_set(v: &con->ras_ce_count, i: ce_count);
4182	atomic_set(v: &con->ras_ue_count, i: ue_count);
4183	}
4184
4185	Out:
4186	pm_runtime_put_autosuspend(dev: dev->dev);
4187	}
4188
4189	static int amdgpu_get_ras_schema(struct amdgpu_device *adev)
4190	{
4191	return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 |
4192	AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE |
4193	AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE |
4194	AMDGPU_RAS_ERROR__PARITY;
4195	}
4196
4197	static void ras_event_mgr_init(struct ras_event_manager *mgr)
4198	{
4199	struct ras_event_state *event_state;
4200	int i;
4201
4202	memset(mgr, 0, sizeof(*mgr));
4203	atomic64_set(v: &mgr->seqno, i: 0);
4204
4205	for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) {
4206	event_state = &mgr->event_state[i];
4207	event_state->last_seqno = RAS_EVENT_INVALID_ID;
4208	atomic64_set(v: &event_state->count, i: 0);
4209	}
4210	}
4211
4212	static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev)
4213	{
4214	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4215	struct amdgpu_hive_info *hive;
4216
4217	if (!ras)
4218	return;
4219
4220	hive = amdgpu_get_xgmi_hive(adev);
4221	ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr;
4222
4223	/* init event manager with node 0 on xgmi system */
4224	if (!amdgpu_reset_in_recovery(adev)) {
4225	if (!hive || adev->gmc.xgmi.node_id == 0)
4226	ras_event_mgr_init(mgr: ras->event_mgr);
4227	}
4228
4229	if (hive)
4230	amdgpu_put_xgmi_hive(hive);
4231	}
4232
4233	static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev)
4234	{
4235	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4236
4237	if (!con || (adev->flags & AMD_IS_APU))
4238	return;
4239
4240	switch (amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0)) {
4241	case IP_VERSION(13, 0, 2):
4242	case IP_VERSION(13, 0, 6):
4243	case IP_VERSION(13, 0, 12):
4244	con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT;
4245	break;
4246	case IP_VERSION(13, 0, 14):
4247	con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1);
4248	break;
4249	default:
4250	break;
4251	}
4252	}
4253
4254	int amdgpu_ras_init(struct amdgpu_device *adev)
4255	{
4256	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4257	int r;
4258
4259	if (con)
4260	return 0;
4261
4262	con = kzalloc(sizeof(*con) +
4263	sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
4264	sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
4265	GFP_KERNEL);
4266	if (!con)
4267	return -ENOMEM;
4268
4269	con->adev = adev;
4270	INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
4271	atomic_set(v: &con->ras_ce_count, i: 0);
4272	atomic_set(v: &con->ras_ue_count, i: 0);
4273
4274	con->objs = (struct ras_manager *)(con + 1);
4275
4276	amdgpu_ras_set_context(adev, ras_con: con);
4277
4278	amdgpu_ras_check_supported(adev);
4279
4280	if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
4281	/* set gfx block ras context feature for VEGA20 Gaming
4282	* send ras disable cmd to ras ta during ras late init.
4283	*/
4284	if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
4285	con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
4286
4287	return 0;
4288	}
4289
4290	r = 0;
4291	goto release_con;
4292	}
4293
4294	con->update_channel_flag = false;
4295	con->features = 0;
4296	con->schema = 0;
4297	INIT_LIST_HEAD(list: &con->head);
4298	/* Might need get this flag from vbios. */
4299	con->flags = RAS_DEFAULT_FLAGS;
4300
4301	/* initialize nbio ras function ahead of any other
4302	* ras functions so hardware fatal error interrupt
4303	* can be enabled as early as possible */
4304	switch (amdgpu_ip_version(adev, ip: NBIO_HWIP, inst: 0)) {
4305	case IP_VERSION(7, 4, 0):
4306	case IP_VERSION(7, 4, 1):
4307	case IP_VERSION(7, 4, 4):
4308	if (!adev->gmc.xgmi.connected_to_cpu)
4309	adev->nbio.ras = &nbio_v7_4_ras;
4310	break;
4311	case IP_VERSION(4, 3, 0):
4312	if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
4313	/* unlike other generation of nbio ras,
4314	* nbio v4_3 only support fatal error interrupt
4315	* to inform software that DF is freezed due to
4316	* system fatal error event. driver should not
4317	* enable nbio ras in such case. Instead,
4318	* check DF RAS */
4319	adev->nbio.ras = &nbio_v4_3_ras;
4320	break;
4321	case IP_VERSION(6, 3, 1):
4322	if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
4323	/* unlike other generation of nbio ras,
4324	* nbif v6_3_1 only support fatal error interrupt
4325	* to inform software that DF is freezed due to
4326	* system fatal error event. driver should not
4327	* enable nbio ras in such case. Instead,
4328	* check DF RAS
4329	*/
4330	adev->nbio.ras = &nbif_v6_3_1_ras;
4331	break;
4332	case IP_VERSION(7, 9, 0):
4333	case IP_VERSION(7, 9, 1):
4334	if (!adev->gmc.is_app_apu)
4335	adev->nbio.ras = &nbio_v7_9_ras;
4336	break;
4337	default:
4338	/* nbio ras is not available */
4339	break;
4340	}
4341
4342	/* nbio ras block needs to be enabled ahead of other ras blocks
4343	* to handle fatal error */
4344	r = amdgpu_nbio_ras_sw_init(adev);
4345	if (r)
4346	return r;
4347
4348	if (adev->nbio.ras &&
4349	adev->nbio.ras->init_ras_controller_interrupt) {
4350	r = adev->nbio.ras->init_ras_controller_interrupt(adev);
4351	if (r)
4352	goto release_con;
4353	}
4354
4355	if (adev->nbio.ras &&
4356	adev->nbio.ras->init_ras_err_event_athub_interrupt) {
4357	r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
4358	if (r)
4359	goto release_con;
4360	}
4361
4362	/* Packed socket_id to ras feature mask bits[31:29] */
4363	if (adev->smuio.funcs &&
4364	adev->smuio.funcs->get_socket_id)
4365	con->features |= ((adev->smuio.funcs->get_socket_id(adev)) <<
4366	AMDGPU_RAS_FEATURES_SOCKETID_SHIFT);
4367
4368	/* Get RAS schema for particular SOC */
4369	con->schema = amdgpu_get_ras_schema(adev);
4370
4371	amdgpu_ras_init_reserved_vram_size(adev);
4372
4373	if (amdgpu_ras_fs_init(adev)) {
4374	r = -EINVAL;
4375	goto release_con;
4376	}
4377
4378	if (amdgpu_ras_aca_is_supported(adev)) {
4379	if (amdgpu_aca_is_enabled(adev))
4380	r = amdgpu_aca_init(adev);
4381	else
4382	r = amdgpu_mca_init(adev);
4383	if (r)
4384	goto release_con;
4385	}
4386
4387	con->init_task_pid = task_pid_nr(current);
4388	get_task_comm(con->init_task_comm, current);
4389
4390	mutex_init(&con->critical_region_lock);
4391	INIT_LIST_HEAD(list: &con->critical_region_head);
4392
4393	dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
4394	"hardware ability[%x] ras_mask[%x]\n",
4395	adev->ras_hw_enabled, adev->ras_enabled);
4396
4397	return 0;
4398	release_con:
4399	amdgpu_ras_set_context(adev, NULL);
4400	kfree(objp: con);
4401
4402	return r;
4403	}
4404
4405	int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
4406	{
4407	if (adev->gmc.xgmi.connected_to_cpu ||
4408	adev->gmc.is_app_apu)
4409	return 1;
4410	return 0;
4411	}
4412
4413	static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
4414	struct ras_common_if *ras_block)
4415	{
4416	struct ras_query_if info = {
4417	.head = *ras_block,
4418	};
4419
4420	if (!amdgpu_persistent_edc_harvesting_supported(adev))
4421	return 0;
4422
4423	if (amdgpu_ras_query_error_status(adev, info: &info) != 0)
4424	DRM_WARN("RAS init harvest failure");
4425
4426	if (amdgpu_ras_reset_error_status(adev, block: ras_block->block) != 0)
4427	DRM_WARN("RAS init harvest reset failure");
4428
4429	return 0;
4430	}
4431
4432	bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
4433	{
4434	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4435
4436	if (!con)
4437	return false;
4438
4439	return con->poison_supported;
4440	}
4441
4442	/* helper function to handle common stuff in ip late init phase */
4443	int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
4444	struct ras_common_if *ras_block)
4445	{
4446	struct amdgpu_ras_block_object *ras_obj = NULL;
4447	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4448	struct ras_query_if *query_info;
4449	unsigned long ue_count, ce_count;
4450	int r;
4451
4452	/* disable RAS feature per IP block if it is not supported */
4453	if (!amdgpu_ras_is_supported(adev, block: ras_block->block)) {
4454	amdgpu_ras_feature_enable_on_boot(adev, head: ras_block, enable: 0);
4455	return 0;
4456	}
4457
4458	r = amdgpu_ras_feature_enable_on_boot(adev, head: ras_block, enable: 1);
4459	if (r) {
4460	if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) {
4461	/* in resume phase, if fail to enable ras,
4462	* clean up all ras fs nodes, and disable ras */
4463	goto cleanup;
4464	} else
4465	return r;
4466	}
4467
4468	/* check for errors on warm reset edc persisant supported ASIC */
4469	amdgpu_persistent_edc_harvesting(adev, ras_block);
4470
4471	/* in resume phase, no need to create ras fs node */
4472	if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
4473	return 0;
4474
4475	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4476	if (ras_obj->ras_cb || (ras_obj->hw_ops &&
4477	(ras_obj->hw_ops->query_poison_status ||
4478	ras_obj->hw_ops->handle_poison_consumption))) {
4479	r = amdgpu_ras_interrupt_add_handler(adev, head: ras_block);
4480	if (r)
4481	goto cleanup;
4482	}
4483
4484	if (ras_obj->hw_ops &&
4485	(ras_obj->hw_ops->query_ras_error_count ||
4486	ras_obj->hw_ops->query_ras_error_status)) {
4487	r = amdgpu_ras_sysfs_create(adev, head: ras_block);
4488	if (r)
4489	goto interrupt;
4490
4491	/* Those are the cached values at init.
4492	*/
4493	query_info = kzalloc(sizeof(*query_info), GFP_KERNEL);
4494	if (!query_info)
4495	return -ENOMEM;
4496	memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if));
4497
4498	if (amdgpu_ras_query_error_count(adev, ce_count: &ce_count, ue_count: &ue_count, query_info) == 0) {
4499	atomic_set(v: &con->ras_ce_count, i: ce_count);
4500	atomic_set(v: &con->ras_ue_count, i: ue_count);
4501	}
4502
4503	kfree(objp: query_info);
4504	}
4505
4506	return 0;
4507
4508	interrupt:
4509	if (ras_obj->ras_cb)
4510	amdgpu_ras_interrupt_remove_handler(adev, head: ras_block);
4511	cleanup:
4512	amdgpu_ras_feature_enable(adev, head: ras_block, enable: 0);
4513	return r;
4514	}
4515
4516	static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
4517	struct ras_common_if *ras_block)
4518	{
4519	return amdgpu_ras_block_late_init(adev, ras_block);
4520	}
4521
4522	/* helper function to remove ras fs node and interrupt handler */
4523	void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
4524	struct ras_common_if *ras_block)
4525	{
4526	struct amdgpu_ras_block_object *ras_obj;
4527	if (!ras_block)
4528	return;
4529
4530	amdgpu_ras_sysfs_remove(adev, head: ras_block);
4531
4532	ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4533	if (ras_obj->ras_cb)
4534	amdgpu_ras_interrupt_remove_handler(adev, head: ras_block);
4535	}
4536
4537	static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
4538	struct ras_common_if *ras_block)
4539	{
4540	return amdgpu_ras_block_late_fini(adev, ras_block);
4541	}
4542
4543	/* do some init work after IP late init as dependence.
4544	* and it runs in resume/gpu reset/booting up cases.
4545	*/
4546	void amdgpu_ras_resume(struct amdgpu_device *adev)
4547	{
4548	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4549	struct ras_manager *obj, *tmp;
4550
4551	if (!adev->ras_enabled || !con) {
4552	/* clean ras context for VEGA20 Gaming after send ras disable cmd */
4553	amdgpu_release_ras_context(adev);
4554
4555	return;
4556	}
4557
4558	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
4559	/* Set up all other IPs which are not implemented. There is a
4560	* tricky thing that IP's actual ras error type should be
4561	* MULTI_UNCORRECTABLE, but as driver does not handle it, so
4562	* ERROR_NONE make sense anyway.
4563	*/
4564	amdgpu_ras_enable_all_features(adev, bypass: 1);
4565
4566	/* We enable ras on all hw_supported block, but as boot
4567	* parameter might disable some of them and one or more IP has
4568	* not implemented yet. So we disable them on behalf.
4569	*/
4570	list_for_each_entry_safe(obj, tmp, &con->head, node) {
4571	if (!amdgpu_ras_is_supported(adev, block: obj->head.block)) {
4572	amdgpu_ras_feature_enable(adev, head: &obj->head, enable: 0);
4573	/* there should be no any reference. */
4574	WARN_ON(alive_obj(obj));
4575	}
4576	}
4577	}
4578	}
4579
4580	void amdgpu_ras_suspend(struct amdgpu_device *adev)
4581	{
4582	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4583
4584	if (!adev->ras_enabled || !con)
4585	return;
4586
4587	amdgpu_ras_disable_all_features(adev, bypass: 0);
4588	/* Make sure all ras objects are disabled. */
4589	if (AMDGPU_RAS_GET_FEATURES(con->features))
4590	amdgpu_ras_disable_all_features(adev, bypass: 1);
4591	}
4592
4593	int amdgpu_ras_late_init(struct amdgpu_device *adev)
4594	{
4595	struct amdgpu_ras_block_list *node, *tmp;
4596	struct amdgpu_ras_block_object *obj;
4597	int r;
4598
4599	amdgpu_ras_event_mgr_init(adev);
4600
4601	if (amdgpu_ras_aca_is_supported(adev)) {
4602	if (amdgpu_reset_in_recovery(adev)) {
4603	if (amdgpu_aca_is_enabled(adev))
4604	r = amdgpu_aca_reset(adev);
4605	else
4606	r = amdgpu_mca_reset(adev);
4607	if (r)
4608	return r;
4609	}
4610
4611	if (!amdgpu_sriov_vf(adev)) {
4612	if (amdgpu_aca_is_enabled(adev))
4613	amdgpu_ras_set_aca_debug_mode(adev, enable: false);
4614	else
4615	amdgpu_ras_set_mca_debug_mode(adev, enable: false);
4616	}
4617	}
4618
4619	/* Guest side doesn't need init ras feature */
4620	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev))
4621	return 0;
4622
4623	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
4624	obj = node->ras_obj;
4625	if (!obj) {
4626	dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
4627	continue;
4628	}
4629
4630	if (!amdgpu_ras_is_supported(adev, block: obj->ras_comm.block))
4631	continue;
4632
4633	if (obj->ras_late_init) {
4634	r = obj->ras_late_init(adev, &obj->ras_comm);
4635	if (r) {
4636	dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
4637	obj->ras_comm.name, r);
4638	return r;
4639	}
4640	} else
4641	amdgpu_ras_block_late_init_default(adev, ras_block: &obj->ras_comm);
4642	}
4643
4644	return 0;
4645	}
4646
4647	/* do some fini work before IP fini as dependence */
4648	int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
4649	{
4650	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4651
4652	if (!adev->ras_enabled || !con)
4653	return 0;
4654
4655
4656	/* Need disable ras on all IPs here before ip [hw/sw]fini */
4657	if (AMDGPU_RAS_GET_FEATURES(con->features))
4658	amdgpu_ras_disable_all_features(adev, bypass: 0);
4659	amdgpu_ras_recovery_fini(adev);
4660	return 0;
4661	}
4662
4663	int amdgpu_ras_fini(struct amdgpu_device *adev)
4664	{
4665	struct amdgpu_ras_block_list *ras_node, *tmp;
4666	struct amdgpu_ras_block_object *obj = NULL;
4667	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4668
4669	if (!adev->ras_enabled || !con)
4670	return 0;
4671
4672	amdgpu_ras_critical_region_fini(adev);
4673	mutex_destroy(lock: &con->critical_region_lock);
4674
4675	list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
4676	if (ras_node->ras_obj) {
4677	obj = ras_node->ras_obj;
4678	if (amdgpu_ras_is_supported(adev, block: obj->ras_comm.block) &&
4679	obj->ras_fini)
4680	obj->ras_fini(adev, &obj->ras_comm);
4681	else
4682	amdgpu_ras_block_late_fini_default(adev, ras_block: &obj->ras_comm);
4683	}
4684
4685	/* Clear ras blocks from ras_list and free ras block list node */
4686	list_del(entry: &ras_node->node);
4687	kfree(objp: ras_node);
4688	}
4689
4690	amdgpu_ras_fs_fini(adev);
4691	amdgpu_ras_interrupt_remove_all(adev);
4692
4693	if (amdgpu_ras_aca_is_supported(adev)) {
4694	if (amdgpu_aca_is_enabled(adev))
4695	amdgpu_aca_fini(adev);
4696	else
4697	amdgpu_mca_fini(adev);
4698	}
4699
4700	WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared");
4701
4702	if (AMDGPU_RAS_GET_FEATURES(con->features))
4703	amdgpu_ras_disable_all_features(adev, bypass: 0);
4704
4705	cancel_delayed_work_sync(dwork: &con->ras_counte_delay_work);
4706
4707	amdgpu_ras_set_context(adev, NULL);
4708	kfree(objp: con);
4709
4710	return 0;
4711	}
4712
4713	bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev)
4714	{
4715	struct amdgpu_ras *ras;
4716
4717	ras = amdgpu_ras_get_context(adev);
4718	if (!ras)
4719	return false;
4720
4721	return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4722	}
4723
4724	void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status)
4725	{
4726	struct amdgpu_ras *ras;
4727
4728	ras = amdgpu_ras_get_context(adev);
4729	if (ras) {
4730	if (status)
4731	set_bit(nr: AMDGPU_RAS_BLOCK__LAST, addr: &ras->ras_err_state);
4732	else
4733	clear_bit(nr: AMDGPU_RAS_BLOCK__LAST, addr: &ras->ras_err_state);
4734	}
4735	}
4736
4737	void amdgpu_ras_clear_err_state(struct amdgpu_device *adev)
4738	{
4739	struct amdgpu_ras *ras;
4740
4741	ras = amdgpu_ras_get_context(adev);
4742	if (ras) {
4743	ras->ras_err_state = 0;
4744	ras->gpu_reset_flags = 0;
4745	}
4746	}
4747
4748	void amdgpu_ras_set_err_poison(struct amdgpu_device *adev,
4749	enum amdgpu_ras_block block)
4750	{
4751	struct amdgpu_ras *ras;
4752
4753	ras = amdgpu_ras_get_context(adev);
4754	if (ras)
4755	set_bit(nr: block, addr: &ras->ras_err_state);
4756	}
4757
4758	bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block)
4759	{
4760	struct amdgpu_ras *ras;
4761
4762	ras = amdgpu_ras_get_context(adev);
4763	if (ras) {
4764	if (block == AMDGPU_RAS_BLOCK__ANY)
4765	return (ras->ras_err_state != 0);
4766	else
4767	return test_bit(block, &ras->ras_err_state) ||
4768	test_bit(AMDGPU_RAS_BLOCK__LAST,
4769	&ras->ras_err_state);
4770	}
4771
4772	return false;
4773	}
4774
4775	static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev)
4776	{
4777	struct amdgpu_ras *ras;
4778
4779	ras = amdgpu_ras_get_context(adev);
4780	if (!ras)
4781	return NULL;
4782
4783	return ras->event_mgr;
4784	}
4785
4786	int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type,
4787	const void *caller)
4788	{
4789	struct ras_event_manager *event_mgr;
4790	struct ras_event_state *event_state;
4791	int ret = 0;
4792
4793	if (amdgpu_uniras_enabled(adev))
4794	return 0;
4795
4796	if (type >= RAS_EVENT_TYPE_COUNT) {
4797	ret = -EINVAL;
4798	goto out;
4799	}
4800
4801	event_mgr = __get_ras_event_mgr(adev);
4802	if (!event_mgr) {
4803	ret = -EINVAL;
4804	goto out;
4805	}
4806
4807	event_state = &event_mgr->event_state[type];
4808	event_state->last_seqno = atomic64_inc_return(v: &event_mgr->seqno);
4809	atomic64_inc(v: &event_state->count);
4810
4811	out:
4812	if (ret && caller)
4813	dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n",
4814	(int)type, caller, ret);
4815
4816	return ret;
4817	}
4818
4819	u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type)
4820	{
4821	struct ras_event_manager *event_mgr;
4822	u64 id;
4823
4824	if (type >= RAS_EVENT_TYPE_COUNT)
4825	return RAS_EVENT_INVALID_ID;
4826
4827	switch (type) {
4828	case RAS_EVENT_TYPE_FATAL:
4829	case RAS_EVENT_TYPE_POISON_CREATION:
4830	case RAS_EVENT_TYPE_POISON_CONSUMPTION:
4831	event_mgr = __get_ras_event_mgr(adev);
4832	if (!event_mgr)
4833	return RAS_EVENT_INVALID_ID;
4834
4835	id = event_mgr->event_state[type].last_seqno;
4836	break;
4837	case RAS_EVENT_TYPE_INVALID:
4838	default:
4839	id = RAS_EVENT_INVALID_ID;
4840	break;
4841	}
4842
4843	return id;
4844	}
4845
4846	int amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
4847	{
4848	if (atomic_cmpxchg(v: &amdgpu_ras_in_intr, old: 0, new: 1) == 0) {
4849	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4850	enum ras_event_type type = RAS_EVENT_TYPE_FATAL;
4851	u64 event_id = RAS_EVENT_INVALID_ID;
4852
4853	if (amdgpu_uniras_enabled(adev))
4854	return 0;
4855
4856	if (!amdgpu_ras_mark_ras_event(adev, type))
4857	event_id = amdgpu_ras_acquire_event_id(adev, type);
4858
4859	RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error"
4860	"(ERREVENT_ATHUB_INTERRUPT) detected!\n");
4861
4862	amdgpu_ras_set_fed(adev, status: true);
4863	ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4864	amdgpu_ras_reset_gpu(adev);
4865	}
4866
4867	return -EBUSY;
4868	}
4869
4870	bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
4871	{
4872	if (adev->asic_type == CHIP_VEGA20 &&
4873	adev->pm.fw_version <= 0x283400) {
4874	return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
4875	amdgpu_ras_intr_triggered();
4876	}
4877
4878	return false;
4879	}
4880
4881	void amdgpu_release_ras_context(struct amdgpu_device *adev)
4882	{
4883	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4884
4885	if (!con)
4886	return;
4887
4888	if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
4889	con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
4890	amdgpu_ras_set_context(adev, NULL);
4891	kfree(objp: con);
4892	}
4893	}
4894
4895	#ifdef CONFIG_X86_MCE_AMD
4896	static struct amdgpu_device *find_adev(uint32_t node_id)
4897	{
4898	int i;
4899	struct amdgpu_device *adev = NULL;
4900
4901	for (i = 0; i < mce_adev_list.num_gpu; i++) {
4902	adev = mce_adev_list.devs[i];
4903
4904	if (adev && adev->gmc.xgmi.connected_to_cpu &&
4905	adev->gmc.xgmi.physical_node_id == node_id)
4906	break;
4907	adev = NULL;
4908	}
4909
4910	return adev;
4911	}
4912
4913	#define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF)
4914	#define GET_UMC_INST(m) (((m) >> 21) & 0x7)
4915	#define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
4916	#define GPU_ID_OFFSET 8
4917
4918	static int amdgpu_bad_page_notifier(struct notifier_block *nb,
4919	unsigned long val, void *data)
4920	{
4921	struct mce *m = (struct mce *)data;
4922	struct amdgpu_device *adev = NULL;
4923	uint32_t gpu_id = 0;
4924	uint32_t umc_inst = 0, ch_inst = 0;
4925
4926	/*
4927	* If the error was generated in UMC_V2, which belongs to GPU UMCs,
4928	* and error occurred in DramECC (Extended error code = 0) then only
4929	* process the error, else bail out.
4930	*/
4931	if (!m || !((smca_get_bank_type(cpu: m->extcpu, bank: m->bank) == SMCA_UMC_V2) &&
4932	(XEC(m->status, 0x3f) == 0x0)))
4933	return NOTIFY_DONE;
4934
4935	/*
4936	* If it is correctable error, return.
4937	*/
4938	if (mce_is_correctable(m))
4939	return NOTIFY_OK;
4940
4941	/*
4942	* GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
4943	*/
4944	gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
4945
4946	adev = find_adev(node_id: gpu_id);
4947	if (!adev) {
4948	DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
4949	gpu_id);
4950	return NOTIFY_DONE;
4951	}
4952
4953	/*
4954	* If it is uncorrectable error, then find out UMC instance and
4955	* channel index.
4956	*/
4957	umc_inst = GET_UMC_INST(m->ipid);
4958	ch_inst = GET_CHAN_INDEX(m->ipid);
4959
4960	dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
4961	umc_inst, ch_inst);
4962
4963	if (!amdgpu_umc_page_retirement_mca(adev, err_addr: m->addr, ch_inst, umc_inst))
4964	return NOTIFY_OK;
4965	else
4966	return NOTIFY_DONE;
4967	}
4968
4969	static struct notifier_block amdgpu_bad_page_nb = {
4970	.notifier_call = amdgpu_bad_page_notifier,
4971	.priority = MCE_PRIO_UC,
4972	};
4973
4974	static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
4975	{
4976	/*
4977	* Add the adev to the mce_adev_list.
4978	* During mode2 reset, amdgpu device is temporarily
4979	* removed from the mgpu_info list which can cause
4980	* page retirement to fail.
4981	* Use this list instead of mgpu_info to find the amdgpu
4982	* device on which the UMC error was reported.
4983	*/
4984	mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
4985
4986	/*
4987	* Register the x86 notifier only once
4988	* with MCE subsystem.
4989	*/
4990	if (notifier_registered == false) {
4991	mce_register_decode_chain(nb: &amdgpu_bad_page_nb);
4992	notifier_registered = true;
4993	}
4994	}
4995	static void amdgpu_unregister_bad_pages_mca_notifier(struct amdgpu_device *adev)
4996	{
4997	int i, j;
4998
4999	if (!notifier_registered && !mce_adev_list.num_gpu)
5000	return;
5001	for (i = 0, j = 0; i < mce_adev_list.num_gpu; i++) {
5002	if (mce_adev_list.devs[i] == adev)
5003	mce_adev_list.devs[i] = NULL;
5004	if (!mce_adev_list.devs[i])
5005	++j;
5006	}
5007
5008	if (j == mce_adev_list.num_gpu) {
5009	mce_adev_list.num_gpu = 0;
5010	/* Unregister x86 notifier with MCE subsystem. */
5011	if (notifier_registered) {
5012	mce_unregister_decode_chain(nb: &amdgpu_bad_page_nb);
5013	notifier_registered = false;
5014	}
5015	}
5016	}
5017	#endif
5018
5019	struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
5020	{
5021	if (!adev)
5022	return NULL;
5023
5024	return adev->psp.ras_context.ras;
5025	}
5026
5027	int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
5028	{
5029	if (!adev)
5030	return -EINVAL;
5031
5032	adev->psp.ras_context.ras = ras_con;
5033	return 0;
5034	}
5035
5036	/* check if ras is supported on block, say, sdma, gfx */
5037	int amdgpu_ras_is_supported(struct amdgpu_device *adev,
5038	unsigned int block)
5039	{
5040	int ret = 0;
5041	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5042
5043	if (block >= AMDGPU_RAS_BLOCK_COUNT)
5044	return 0;
5045
5046	ret = ras && (adev->ras_enabled & (1 << block));
5047
5048	/* For the special asic with mem ecc enabled but sram ecc
5049	* not enabled, even if the ras block is not supported on
5050	* .ras_enabled, if the asic supports poison mode and the
5051	* ras block has ras configuration, it can be considered
5052	* that the ras block supports ras function.
5053	*/
5054	if (!ret &&
5055	(block == AMDGPU_RAS_BLOCK__GFX ||
5056	block == AMDGPU_RAS_BLOCK__SDMA ||
5057	block == AMDGPU_RAS_BLOCK__VCN ||
5058	block == AMDGPU_RAS_BLOCK__JPEG) &&
5059	(amdgpu_ras_mask & (1 << block)) &&
5060	amdgpu_ras_is_poison_mode_supported(adev) &&
5061	amdgpu_ras_get_ras_block(adev, block, sub_block_index: 0))
5062	ret = 1;
5063
5064	return ret;
5065	}
5066
5067	int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
5068	{
5069	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5070
5071	/* mode1 is the only selection for RMA status */
5072	if (amdgpu_ras_is_rma(adev)) {
5073	ras->gpu_reset_flags = 0;
5074	ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
5075	}
5076
5077	if (atomic_cmpxchg(v: &ras->in_recovery, old: 0, new: 1) == 0) {
5078	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
5079	int hive_ras_recovery = 0;
5080
5081	if (hive) {
5082	hive_ras_recovery = atomic_read(v: &hive->ras_recovery);
5083	amdgpu_put_xgmi_hive(hive);
5084	}
5085	/* In the case of multiple GPUs, after a GPU has started
5086	* resetting all GPUs on hive, other GPUs do not need to
5087	* trigger GPU reset again.
5088	*/
5089	if (!hive_ras_recovery)
5090	amdgpu_reset_domain_schedule(domain: ras->adev->reset_domain, work: &ras->recovery_work);
5091	else
5092	atomic_set(v: &ras->in_recovery, i: 0);
5093	} else {
5094	flush_work(work: &ras->recovery_work);
5095	amdgpu_reset_domain_schedule(domain: ras->adev->reset_domain, work: &ras->recovery_work);
5096	}
5097
5098	return 0;
5099	}
5100
5101	int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable)
5102	{
5103	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5104	int ret = 0;
5105
5106	if (con) {
5107	ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
5108	if (!ret)
5109	con->is_aca_debug_mode = enable;
5110	}
5111
5112	return ret;
5113	}
5114
5115	int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable)
5116	{
5117	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5118	int ret = 0;
5119
5120	if (con) {
5121	if (amdgpu_aca_is_enabled(adev))
5122	ret = amdgpu_aca_smu_set_debug_mode(adev, en: enable);
5123	else
5124	ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
5125	if (!ret)
5126	con->is_aca_debug_mode = enable;
5127	}
5128
5129	return ret;
5130	}
5131
5132	bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev)
5133	{
5134	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5135	const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
5136	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
5137
5138	if (!con)
5139	return false;
5140
5141	if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) ||
5142	(!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode))
5143	return con->is_aca_debug_mode;
5144	else
5145	return true;
5146	}
5147
5148	bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
5149	unsigned int *error_query_mode)
5150	{
5151	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5152	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
5153	const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
5154
5155	if (!con) {
5156	*error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY;
5157	return false;
5158	}
5159
5160	if (amdgpu_sriov_vf(adev)) {
5161	*error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY;
5162	} else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) {
5163	*error_query_mode =
5164	(con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY;
5165	} else {
5166	*error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY;
5167	}
5168
5169	return true;
5170	}
5171
5172	/* Register each ip ras block into amdgpu ras */
5173	int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
5174	struct amdgpu_ras_block_object *ras_block_obj)
5175	{
5176	struct amdgpu_ras_block_list *ras_node;
5177	if (!adev || !ras_block_obj)
5178	return -EINVAL;
5179
5180	ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
5181	if (!ras_node)
5182	return -ENOMEM;
5183
5184	INIT_LIST_HEAD(list: &ras_node->node);
5185	ras_node->ras_obj = ras_block_obj;
5186	list_add_tail(new: &ras_node->node, head: &adev->ras_list);
5187
5188	return 0;
5189	}
5190
5191	void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name)
5192	{
5193	if (!err_type_name)
5194	return;
5195
5196	switch (err_type) {
5197	case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
5198	sprintf(buf: err_type_name, fmt: "correctable");
5199	break;
5200	case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
5201	sprintf(buf: err_type_name, fmt: "uncorrectable");
5202	break;
5203	default:
5204	sprintf(buf: err_type_name, fmt: "unknown");
5205	break;
5206	}
5207	}
5208
5209	bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev,
5210	const struct amdgpu_ras_err_status_reg_entry *reg_entry,
5211	uint32_t instance,
5212	uint32_t *memory_id)
5213	{
5214	uint32_t err_status_lo_data, err_status_lo_offset;
5215
5216	if (!reg_entry)
5217	return false;
5218
5219	err_status_lo_offset =
5220	AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
5221	reg_entry->seg_lo, reg_entry->reg_lo);
5222	err_status_lo_data = RREG32(err_status_lo_offset);
5223
5224	if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) &&
5225	!REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG))
5226	return false;
5227
5228	*memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID);
5229
5230	return true;
5231	}
5232
5233	bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev,
5234	const struct amdgpu_ras_err_status_reg_entry *reg_entry,
5235	uint32_t instance,
5236	unsigned long *err_cnt)
5237	{
5238	uint32_t err_status_hi_data, err_status_hi_offset;
5239
5240	if (!reg_entry)
5241	return false;
5242
5243	err_status_hi_offset =
5244	AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
5245	reg_entry->seg_hi, reg_entry->reg_hi);
5246	err_status_hi_data = RREG32(err_status_hi_offset);
5247
5248	if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) &&
5249	!REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG))
5250	/* keep the check here in case we need to refer to the result later */
5251	dev_dbg(adev->dev, "Invalid err_info field\n");
5252
5253	/* read err count */
5254	*err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT);
5255
5256	return true;
5257	}
5258
5259	void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev,
5260	const struct amdgpu_ras_err_status_reg_entry *reg_list,
5261	uint32_t reg_list_size,
5262	const struct amdgpu_ras_memory_id_entry *mem_list,
5263	uint32_t mem_list_size,
5264	uint32_t instance,
5265	uint32_t err_type,
5266	unsigned long *err_count)
5267	{
5268	uint32_t memory_id;
5269	unsigned long err_cnt;
5270	char err_type_name[16];
5271	uint32_t i, j;
5272
5273	for (i = 0; i < reg_list_size; i++) {
5274	/* query memory_id from err_status_lo */
5275	if (!amdgpu_ras_inst_get_memory_id_field(adev, reg_entry: &reg_list[i],
5276	instance, memory_id: &memory_id))
5277	continue;
5278
5279	/* query err_cnt from err_status_hi */
5280	if (!amdgpu_ras_inst_get_err_cnt_field(adev, reg_entry: &reg_list[i],
5281	instance, err_cnt: &err_cnt) ||
5282	!err_cnt)
5283	continue;
5284
5285	*err_count += err_cnt;
5286
5287	/* log the errors */
5288	amdgpu_ras_get_error_type_name(err_type, err_type_name);
5289	if (!mem_list) {
5290	/* memory_list is not supported */
5291	dev_info(adev->dev,
5292	"%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n",
5293	err_cnt, err_type_name,
5294	reg_list[i].block_name,
5295	instance, memory_id);
5296	} else {
5297	for (j = 0; j < mem_list_size; j++) {
5298	if (memory_id == mem_list[j].memory_id) {
5299	dev_info(adev->dev,
5300	"%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n",
5301	err_cnt, err_type_name,
5302	reg_list[i].block_name,
5303	instance, mem_list[j].name);
5304	break;
5305	}
5306	}
5307	}
5308	}
5309	}
5310
5311	void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev,
5312	const struct amdgpu_ras_err_status_reg_entry *reg_list,
5313	uint32_t reg_list_size,
5314	uint32_t instance)
5315	{
5316	uint32_t err_status_lo_offset, err_status_hi_offset;
5317	uint32_t i;
5318
5319	for (i = 0; i < reg_list_size; i++) {
5320	err_status_lo_offset =
5321	AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
5322	reg_list[i].seg_lo, reg_list[i].reg_lo);
5323	err_status_hi_offset =
5324	AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
5325	reg_list[i].seg_hi, reg_list[i].reg_hi);
5326	WREG32(err_status_lo_offset, 0);
5327	WREG32(err_status_hi_offset, 0);
5328	}
5329	}
5330
5331	int amdgpu_ras_error_data_init(struct ras_err_data *err_data)
5332	{
5333	memset(err_data, 0, sizeof(*err_data));
5334
5335	INIT_LIST_HEAD(list: &err_data->err_node_list);
5336
5337	return 0;
5338	}
5339
5340	static void amdgpu_ras_error_node_release(struct ras_err_node *err_node)
5341	{
5342	if (!err_node)
5343	return;
5344
5345	list_del(entry: &err_node->node);
5346	kvfree(addr: err_node);
5347	}
5348
5349	void amdgpu_ras_error_data_fini(struct ras_err_data *err_data)
5350	{
5351	struct ras_err_node *err_node, *tmp;
5352
5353	list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node)
5354	amdgpu_ras_error_node_release(err_node);
5355	}
5356
5357	static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data,
5358	struct amdgpu_smuio_mcm_config_info *mcm_info)
5359	{
5360	struct ras_err_node *err_node;
5361	struct amdgpu_smuio_mcm_config_info *ref_id;
5362
5363	if (!err_data || !mcm_info)
5364	return NULL;
5365
5366	for_each_ras_error(err_node, err_data) {
5367	ref_id = &err_node->err_info.mcm_info;
5368
5369	if (mcm_info->socket_id == ref_id->socket_id &&
5370	mcm_info->die_id == ref_id->die_id)
5371	return err_node;
5372	}
5373
5374	return NULL;
5375	}
5376
5377	static struct ras_err_node *amdgpu_ras_error_node_new(void)
5378	{
5379	struct ras_err_node *err_node;
5380
5381	err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL);
5382	if (!err_node)
5383	return NULL;
5384
5385	INIT_LIST_HEAD(list: &err_node->node);
5386
5387	return err_node;
5388	}
5389
5390	static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b)
5391	{
5392	struct ras_err_node *nodea = container_of(a, struct ras_err_node, node);
5393	struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node);
5394	struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info;
5395	struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info;
5396
5397	if (unlikely(infoa->socket_id != infob->socket_id))
5398	return infoa->socket_id - infob->socket_id;
5399	else
5400	return infoa->die_id - infob->die_id;
5401
5402	return 0;
5403	}
5404
5405	static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data,
5406	struct amdgpu_smuio_mcm_config_info *mcm_info)
5407	{
5408	struct ras_err_node *err_node;
5409
5410	err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info);
5411	if (err_node)
5412	return &err_node->err_info;
5413
5414	err_node = amdgpu_ras_error_node_new();
5415	if (!err_node)
5416	return NULL;
5417
5418	memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info));
5419
5420	err_data->err_list_count++;
5421	list_add_tail(new: &err_node->node, head: &err_data->err_node_list);
5422	list_sort(NULL, head: &err_data->err_node_list, cmp: ras_err_info_cmp);
5423
5424	return &err_node->err_info;
5425	}
5426
5427	int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data,
5428	struct amdgpu_smuio_mcm_config_info *mcm_info,
5429	u64 count)
5430	{
5431	struct ras_err_info *err_info;
5432
5433	if (!err_data || !mcm_info)
5434	return -EINVAL;
5435
5436	if (!count)
5437	return 0;
5438
5439	err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5440	if (!err_info)
5441	return -EINVAL;
5442
5443	err_info->ue_count += count;
5444	err_data->ue_count += count;
5445
5446	return 0;
5447	}
5448
5449	int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
5450	struct amdgpu_smuio_mcm_config_info *mcm_info,
5451	u64 count)
5452	{
5453	struct ras_err_info *err_info;
5454
5455	if (!err_data || !mcm_info)
5456	return -EINVAL;
5457
5458	if (!count)
5459	return 0;
5460
5461	err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5462	if (!err_info)
5463	return -EINVAL;
5464
5465	err_info->ce_count += count;
5466	err_data->ce_count += count;
5467
5468	return 0;
5469	}
5470
5471	int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
5472	struct amdgpu_smuio_mcm_config_info *mcm_info,
5473	u64 count)
5474	{
5475	struct ras_err_info *err_info;
5476
5477	if (!err_data || !mcm_info)
5478	return -EINVAL;
5479
5480	if (!count)
5481	return 0;
5482
5483	err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5484	if (!err_info)
5485	return -EINVAL;
5486
5487	err_info->de_count += count;
5488	err_data->de_count += count;
5489
5490	return 0;
5491	}
5492
5493	#define mmMP0_SMN_C2PMSG_92 0x1609C
5494	#define mmMP0_SMN_C2PMSG_126 0x160BE
5495	static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev,
5496	u32 instance)
5497	{
5498	u32 socket_id, aid_id, hbm_id;
5499	u32 fw_status;
5500	u32 boot_error;
5501	u64 reg_addr;
5502
5503	/* The pattern for smn addressing in other SOC could be different from
5504	* the one for aqua_vanjaram. We should revisit the code if the pattern
5505	* is changed. In such case, replace the aqua_vanjaram implementation
5506	* with more common helper */
5507	reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5508	aqua_vanjaram_encode_ext_smn_addressing(ext_id: instance);
5509	fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5510
5511	reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) +
5512	aqua_vanjaram_encode_ext_smn_addressing(ext_id: instance);
5513	boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5514
5515	socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error);
5516	aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error);
5517	hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1);
5518
5519	if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error))
5520	dev_info(adev->dev,
5521	"socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n",
5522	socket_id, aid_id, hbm_id, fw_status);
5523
5524	if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error))
5525	dev_info(adev->dev,
5526	"socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n",
5527	socket_id, aid_id, fw_status);
5528
5529	if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error))
5530	dev_info(adev->dev,
5531	"socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n",
5532	socket_id, aid_id, fw_status);
5533
5534	if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error))
5535	dev_info(adev->dev,
5536	"socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n",
5537	socket_id, aid_id, fw_status);
5538
5539	if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error))
5540	dev_info(adev->dev,
5541	"socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n",
5542	socket_id, aid_id, fw_status);
5543
5544	if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error))
5545	dev_info(adev->dev,
5546	"socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n",
5547	socket_id, aid_id, fw_status);
5548
5549	if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error))
5550	dev_info(adev->dev,
5551	"socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n",
5552	socket_id, aid_id, hbm_id, fw_status);
5553
5554	if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error))
5555	dev_info(adev->dev,
5556	"socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n",
5557	socket_id, aid_id, hbm_id, fw_status);
5558
5559	if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error))
5560	dev_info(adev->dev,
5561	"socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n",
5562	socket_id, aid_id, fw_status);
5563
5564	if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error))
5565	dev_info(adev->dev,
5566	"socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n",
5567	socket_id, aid_id, fw_status);
5568	}
5569
5570	static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev,
5571	u32 instance)
5572	{
5573	u64 reg_addr;
5574	u32 reg_data;
5575	int retry_loop;
5576
5577	reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5578	aqua_vanjaram_encode_ext_smn_addressing(ext_id: instance);
5579
5580	for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) {
5581	reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5582	if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS)
5583	return false;
5584	else
5585	msleep(msecs: 1);
5586	}
5587
5588	return true;
5589	}
5590
5591	void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances)
5592	{
5593	u32 i;
5594
5595	for (i = 0; i < num_instances; i++) {
5596	if (amdgpu_ras_boot_error_detected(adev, instance: i))
5597	amdgpu_ras_boot_time_error_reporting(adev, instance: i);
5598	}
5599	}
5600
5601	int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn)
5602	{
5603	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5604	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
5605	uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT;
5606	int ret = 0;
5607
5608	if (amdgpu_ras_check_critical_address(adev, addr: start))
5609	return 0;
5610
5611	mutex_lock(&con->page_rsv_lock);
5612	ret = amdgpu_vram_mgr_query_page_status(mgr, start);
5613	if (ret == -ENOENT)
5614	ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE);
5615	mutex_unlock(lock: &con->page_rsv_lock);
5616
5617	return ret;
5618	}
5619
5620	void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id,
5621	const char *fmt, ...)
5622	{
5623	struct va_format vaf;
5624	va_list args;
5625
5626	va_start(args, fmt);
5627	vaf.fmt = fmt;
5628	vaf.va = &args;
5629
5630	if (RAS_EVENT_ID_IS_VALID(event_id))
5631	dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf);
5632	else
5633	dev_printk(KERN_INFO, adev->dev, "%pV", &vaf);
5634
5635	va_end(args);
5636	}
5637
5638	bool amdgpu_ras_is_rma(struct amdgpu_device *adev)
5639	{
5640	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5641
5642	if (amdgpu_uniras_enabled(adev))
5643	return amdgpu_ras_mgr_is_rma(adev);
5644
5645	if (!con)
5646	return false;
5647
5648	return con->is_rma;
5649	}
5650
5651	int amdgpu_ras_add_critical_region(struct amdgpu_device *adev,
5652	struct amdgpu_bo *bo)
5653	{
5654	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5655	struct amdgpu_vram_mgr_resource *vres;
5656	struct ras_critical_region *region;
5657	struct drm_buddy_block *block;
5658	int ret = 0;
5659
5660	if (!bo || !bo->tbo.resource)
5661	return -EINVAL;
5662
5663	vres = to_amdgpu_vram_mgr_resource(res: bo->tbo.resource);
5664
5665	mutex_lock(&con->critical_region_lock);
5666
5667	/* Check if the bo had been recorded */
5668	list_for_each_entry(region, &con->critical_region_head, node)
5669	if (region->bo == bo)
5670	goto out;
5671
5672	/* Record new critical amdgpu bo */
5673	list_for_each_entry(block, &vres->blocks, link) {
5674	region = kzalloc(sizeof(*region), GFP_KERNEL);
5675	if (!region) {
5676	ret = -ENOMEM;
5677	goto out;
5678	}
5679	region->bo = bo;
5680	region->start = amdgpu_vram_mgr_block_start(block);
5681	region->size = amdgpu_vram_mgr_block_size(block);
5682	list_add_tail(new: &region->node, head: &con->critical_region_head);
5683	}
5684
5685	out:
5686	mutex_unlock(lock: &con->critical_region_lock);
5687
5688	return ret;
5689	}
5690
5691	static void amdgpu_ras_critical_region_init(struct amdgpu_device *adev)
5692	{
5693	amdgpu_ras_add_critical_region(adev, bo: adev->mman.fw_reserved_memory);
5694	}
5695
5696	static void amdgpu_ras_critical_region_fini(struct amdgpu_device *adev)
5697	{
5698	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5699	struct ras_critical_region *region, *tmp;
5700
5701	mutex_lock(&con->critical_region_lock);
5702	list_for_each_entry_safe(region, tmp, &con->critical_region_head, node) {
5703	list_del(entry: &region->node);
5704	kfree(objp: region);
5705	}
5706	mutex_unlock(lock: &con->critical_region_lock);
5707	}
5708
5709	bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr)
5710	{
5711	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5712	struct ras_critical_region *region;
5713	bool ret = false;
5714
5715	mutex_lock(&con->critical_region_lock);
5716	list_for_each_entry(region, &con->critical_region_head, node) {
5717	if ((region->start <= addr) &&
5718	(addr < (region->start + region->size))) {
5719	ret = true;
5720	break;
5721	}
5722	}
5723	mutex_unlock(lock: &con->critical_region_lock);
5724
5725	return ret;
5726	}
5727
5728	void amdgpu_ras_pre_reset(struct amdgpu_device *adev,
5729	struct list_head *device_list)
5730	{
5731	struct amdgpu_device *tmp_adev = NULL;
5732
5733	list_for_each_entry(tmp_adev, device_list, reset_list) {
5734	if (amdgpu_uniras_enabled(adev: tmp_adev))
5735	amdgpu_ras_mgr_pre_reset(adev: tmp_adev);
5736	}
5737	}
5738
5739	void amdgpu_ras_post_reset(struct amdgpu_device *adev,
5740	struct list_head *device_list)
5741	{
5742	struct amdgpu_device *tmp_adev = NULL;
5743
5744	list_for_each_entry(tmp_adev, device_list, reset_list) {
5745	if (amdgpu_uniras_enabled(adev: tmp_adev))
5746	amdgpu_ras_mgr_post_reset(adev: tmp_adev);
5747	}
5748	}
5749