1	/*
2	* Copyright 2014 Advanced Micro Devices, Inc.
3	* Copyright 2008 Red Hat Inc.
4	* Copyright 2009 Jerome Glisse.
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a
7	* copy of this software and associated documentation files (the "Software"),
8	* to deal in the Software without restriction, including without limitation
9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
10	* and/or sell copies of the Software, and to permit persons to whom the
11	* Software is furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22	* OTHER DEALINGS IN THE SOFTWARE.
23	*
24	*/
25
26	#include <linux/firmware.h>
27	#include <linux/pm_runtime.h>
28
29	#include "amdgpu.h"
30	#include "amdgpu_gfx.h"
31	#include "amdgpu_rlc.h"
32	#include "amdgpu_ras.h"
33	#include "amdgpu_reset.h"
34	#include "amdgpu_xcp.h"
35	#include "amdgpu_xgmi.h"
36	#include "amdgpu_mes.h"
37	#include "nvd.h"
38
39	/* delay 0.1 second to enable gfx off feature */
40	#define GFX_OFF_DELAY_ENABLE msecs_to_jiffies(100)
41
42	#define GFX_OFF_NO_DELAY 0
43
44	/*
45	* GPU GFX IP block helpers function.
46	*/
47
48	int amdgpu_gfx_mec_queue_to_bit(struct amdgpu_device *adev, int mec,
49	int pipe, int queue)
50	{
51	int bit = 0;
52
53	bit += mec * adev->gfx.mec.num_pipe_per_mec
54	* adev->gfx.mec.num_queue_per_pipe;
55	bit += pipe * adev->gfx.mec.num_queue_per_pipe;
56	bit += queue;
57
58	return bit;
59	}
60
61	void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit,
62	int *mec, int *pipe, int *queue)
63	{
64	*queue = bit % adev->gfx.mec.num_queue_per_pipe;
65	*pipe = (bit / adev->gfx.mec.num_queue_per_pipe)
66	% adev->gfx.mec.num_pipe_per_mec;
67	*mec = (bit / adev->gfx.mec.num_queue_per_pipe)
68	/ adev->gfx.mec.num_pipe_per_mec;
69
70	}
71
72	bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev,
73	int xcc_id, int mec, int pipe, int queue)
74	{
75	return test_bit(amdgpu_gfx_mec_queue_to_bit(adev, mec, pipe, queue),
76	adev->gfx.mec_bitmap[xcc_id].queue_bitmap);
77	}
78
79	static int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev,
80	int me, int pipe, int queue)
81	{
82	int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */
83	int bit = 0;
84
85	bit += me * adev->gfx.me.num_pipe_per_me
86	* num_queue_per_pipe;
87	bit += pipe * num_queue_per_pipe;
88	bit += queue;
89
90	return bit;
91	}
92
93	bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev,
94	int me, int pipe, int queue)
95	{
96	return test_bit(amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue),
97	adev->gfx.me.queue_bitmap);
98	}
99
100	/**
101	* amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter
102	*
103	* @mask: array in which the per-shader array disable masks will be stored
104	* @max_se: number of SEs
105	* @max_sh: number of SHs
106	*
107	* The bitmask of CUs to be disabled in the shader array determined by se and
108	* sh is stored in mask[se * max_sh + sh].
109	*/
110	void amdgpu_gfx_parse_disable_cu(unsigned int *mask, unsigned int max_se, unsigned int max_sh)
111	{
112	unsigned int se, sh, cu;
113	const char *p;
114
115	memset(mask, 0, sizeof(*mask) * max_se * max_sh);
116
117	if (!amdgpu_disable_cu || !*amdgpu_disable_cu)
118	return;
119
120	p = amdgpu_disable_cu;
121	for (;;) {
122	char *next;
123	int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu);
124
125	if (ret < 3) {
126	DRM_ERROR("amdgpu: could not parse disable_cu\n");
127	return;
128	}
129
130	if (se < max_se && sh < max_sh && cu < 16) {
131	DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu);
132	mask[se * max_sh + sh] |= 1u << cu;
133	} else {
134	DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n",
135	se, sh, cu);
136	}
137
138	next = strchr(p, ',');
139	if (!next)
140	break;
141	p = next + 1;
142	}
143	}
144
145	static bool amdgpu_gfx_is_graphics_multipipe_capable(struct amdgpu_device *adev)
146	{
147	return amdgpu_async_gfx_ring && adev->gfx.me.num_pipe_per_me > 1;
148	}
149
150	static bool amdgpu_gfx_is_compute_multipipe_capable(struct amdgpu_device *adev)
151	{
152	if (amdgpu_compute_multipipe != -1) {
153	dev_info(adev->dev, "amdgpu: forcing compute pipe policy %d\n",
154	amdgpu_compute_multipipe);
155	return amdgpu_compute_multipipe == 1;
156	}
157
158	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) > IP_VERSION(9, 0, 0))
159	return true;
160
161	/* FIXME: spreading the queues across pipes causes perf regressions
162	* on POLARIS11 compute workloads */
163	if (adev->asic_type == CHIP_POLARIS11)
164	return false;
165
166	return adev->gfx.mec.num_mec > 1;
167	}
168
169	bool amdgpu_gfx_is_high_priority_graphics_queue(struct amdgpu_device *adev,
170	struct amdgpu_ring *ring)
171	{
172	int queue = ring->queue;
173	int pipe = ring->pipe;
174
175	/* Policy: use pipe1 queue0 as high priority graphics queue if we
176	* have more than one gfx pipe.
177	*/
178	if (amdgpu_gfx_is_graphics_multipipe_capable(adev) &&
179	adev->gfx.num_gfx_rings > 1 && pipe == 1 && queue == 0) {
180	int me = ring->me;
181	int bit;
182
183	bit = amdgpu_gfx_me_queue_to_bit(adev, me, pipe, queue);
184	if (ring == &adev->gfx.gfx_ring[bit])
185	return true;
186	}
187
188	return false;
189	}
190
191	bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev,
192	struct amdgpu_ring *ring)
193	{
194	/* Policy: use 1st queue as high priority compute queue if we
195	* have more than one compute queue.
196	*/
197	if (adev->gfx.num_compute_rings > 1 &&
198	ring == &adev->gfx.compute_ring[0])
199	return true;
200
201	return false;
202	}
203
204	void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
205	{
206	int i, j, queue, pipe;
207	bool multipipe_policy = amdgpu_gfx_is_compute_multipipe_capable(adev);
208	int max_queues_per_mec = min(adev->gfx.mec.num_pipe_per_mec *
209	adev->gfx.mec.num_queue_per_pipe,
210	adev->gfx.num_compute_rings);
211	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
212
213	if (multipipe_policy) {
214	/* policy: make queues evenly cross all pipes on MEC1 only
215	* for multiple xcc, just use the original policy for simplicity */
216	for (j = 0; j < num_xcc; j++) {
217	for (i = 0; i < max_queues_per_mec; i++) {
218	pipe = i % adev->gfx.mec.num_pipe_per_mec;
219	queue = (i / adev->gfx.mec.num_pipe_per_mec) %
220	adev->gfx.mec.num_queue_per_pipe;
221
222	set_bit(nr: pipe * adev->gfx.mec.num_queue_per_pipe + queue,
223	addr: adev->gfx.mec_bitmap[j].queue_bitmap);
224	}
225	}
226	} else {
227	/* policy: amdgpu owns all queues in the given pipe */
228	for (j = 0; j < num_xcc; j++) {
229	for (i = 0; i < max_queues_per_mec; ++i)
230	set_bit(nr: i, addr: adev->gfx.mec_bitmap[j].queue_bitmap);
231	}
232	}
233
234	for (j = 0; j < num_xcc; j++) {
235	dev_dbg(adev->dev, "mec queue bitmap weight=%d\n",
236	bitmap_weight(adev->gfx.mec_bitmap[j].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES));
237	}
238	}
239
240	void amdgpu_gfx_graphics_queue_acquire(struct amdgpu_device *adev)
241	{
242	int i, queue, pipe;
243	bool multipipe_policy = amdgpu_gfx_is_graphics_multipipe_capable(adev);
244	int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */
245	int max_queues_per_me = adev->gfx.me.num_pipe_per_me * num_queue_per_pipe;
246
247	if (multipipe_policy) {
248	/* policy: amdgpu owns the first queue per pipe at this stage
249	* will extend to mulitple queues per pipe later */
250	for (i = 0; i < max_queues_per_me; i++) {
251	pipe = i % adev->gfx.me.num_pipe_per_me;
252	queue = (i / adev->gfx.me.num_pipe_per_me) %
253	num_queue_per_pipe;
254
255	set_bit(nr: pipe * num_queue_per_pipe + queue,
256	addr: adev->gfx.me.queue_bitmap);
257	}
258	} else {
259	for (i = 0; i < max_queues_per_me; ++i)
260	set_bit(nr: i, addr: adev->gfx.me.queue_bitmap);
261	}
262
263	/* update the number of active graphics rings */
264	if (adev->gfx.num_gfx_rings)
265	adev->gfx.num_gfx_rings =
266	bitmap_weight(src: adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
267	}
268
269	static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
270	struct amdgpu_ring *ring, int xcc_id)
271	{
272	int queue_bit;
273	int mec, pipe, queue;
274
275	queue_bit = adev->gfx.mec.num_mec
276	* adev->gfx.mec.num_pipe_per_mec
277	* adev->gfx.mec.num_queue_per_pipe;
278
279	while (--queue_bit >= 0) {
280	if (test_bit(queue_bit, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))
281	continue;
282
283	amdgpu_queue_mask_bit_to_mec_queue(adev, bit: queue_bit, mec: &mec, pipe: &pipe, queue: &queue);
284
285	/*
286	* 1. Using pipes 2/3 from MEC 2 seems cause problems.
287	* 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN
288	* only can be issued on queue 0.
289	*/
290	if ((mec == 1 && pipe > 1) || queue != 0)
291	continue;
292
293	ring->me = mec + 1;
294	ring->pipe = pipe;
295	ring->queue = queue;
296
297	return 0;
298	}
299
300	dev_err(adev->dev, "Failed to find a queue for KIQ\n");
301	return -EINVAL;
302	}
303
304	int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev, int xcc_id)
305	{
306	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
307	struct amdgpu_irq_src *irq = &kiq->irq;
308	struct amdgpu_ring *ring = &kiq->ring;
309	int r = 0;
310
311	spin_lock_init(&kiq->ring_lock);
312
313	ring->adev = NULL;
314	ring->ring_obj = NULL;
315	ring->use_doorbell = true;
316	ring->xcc_id = xcc_id;
317	ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
318	ring->doorbell_index =
319	(adev->doorbell_index.kiq +
320	xcc_id * adev->doorbell_index.xcc_doorbell_range)
321	<< 1;
322
323	r = amdgpu_gfx_kiq_acquire(adev, ring, xcc_id);
324	if (r)
325	return r;
326
327	ring->eop_gpu_addr = kiq->eop_gpu_addr;
328	ring->no_scheduler = true;
329	snprintf(buf: ring->name, size: sizeof(ring->name), fmt: "kiq_%hhu.%hhu.%hhu.%hhu",
330	(unsigned char)xcc_id, (unsigned char)ring->me,
331	(unsigned char)ring->pipe, (unsigned char)ring->queue);
332	r = amdgpu_ring_init(adev, ring, max_dw: 1024, irq_src: irq, irq_type: AMDGPU_CP_KIQ_IRQ_DRIVER0,
333	hw_prio: AMDGPU_RING_PRIO_DEFAULT, NULL);
334	if (r)
335	dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r);
336
337	return r;
338	}
339
340	void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring)
341	{
342	amdgpu_ring_fini(ring);
343	}
344
345	void amdgpu_gfx_kiq_fini(struct amdgpu_device *adev, int xcc_id)
346	{
347	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
348
349	amdgpu_bo_free_kernel(bo: &kiq->eop_obj, gpu_addr: &kiq->eop_gpu_addr, NULL);
350	}
351
352	int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
353	unsigned int hpd_size, int xcc_id)
354	{
355	int r;
356	u32 *hpd;
357	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
358
359	r = amdgpu_bo_create_kernel(adev, size: hpd_size, PAGE_SIZE,
360	AMDGPU_GEM_DOMAIN_GTT, bo_ptr: &kiq->eop_obj,
361	gpu_addr: &kiq->eop_gpu_addr, cpu_addr: (void **)&hpd);
362	if (r) {
363	dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r);
364	return r;
365	}
366
367	memset(hpd, 0, hpd_size);
368
369	r = amdgpu_bo_reserve(bo: kiq->eop_obj, no_intr: true);
370	if (unlikely(r != 0))
371	dev_warn(adev->dev, "(%d) reserve kiq eop bo failed\n", r);
372	amdgpu_bo_kunmap(bo: kiq->eop_obj);
373	amdgpu_bo_unreserve(bo: kiq->eop_obj);
374
375	return 0;
376	}
377
378	/* create MQD for each compute/gfx queue */
379	int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
380	unsigned int mqd_size, int xcc_id)
381	{
382	int r, i, j;
383	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
384	struct amdgpu_ring *ring = &kiq->ring;
385	u32 domain = AMDGPU_GEM_DOMAIN_GTT;
386
387	#if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
388	/* Only enable on gfx10 and 11 for now to avoid changing behavior on older chips */
389	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >= IP_VERSION(10, 0, 0))
390	domain |= AMDGPU_GEM_DOMAIN_VRAM;
391	#endif
392
393	/* create MQD for KIQ */
394	if (!adev->enable_mes_kiq && !ring->mqd_obj) {
395	/* originaly the KIQ MQD is put in GTT domain, but for SRIOV VRAM domain is a must
396	* otherwise hypervisor trigger SAVE_VF fail after driver unloaded which mean MQD
397	* deallocated and gart_unbind, to strict diverage we decide to use VRAM domain for
398	* KIQ MQD no matter SRIOV or Bare-metal
399	*/
400	r = amdgpu_bo_create_kernel(adev, size: mqd_size, PAGE_SIZE,
401	AMDGPU_GEM_DOMAIN_VRAM |
402	AMDGPU_GEM_DOMAIN_GTT,
403	bo_ptr: &ring->mqd_obj,
404	gpu_addr: &ring->mqd_gpu_addr,
405	cpu_addr: &ring->mqd_ptr);
406	if (r) {
407	dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
408	return r;
409	}
410
411	/* prepare MQD backup */
412	kiq->mqd_backup = kzalloc(mqd_size, GFP_KERNEL);
413	if (!kiq->mqd_backup) {
414	dev_warn(adev->dev,
415	"no memory to create MQD backup for ring %s\n", ring->name);
416	return -ENOMEM;
417	}
418	}
419
420	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
421	/* create MQD for each KGQ */
422	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
423	ring = &adev->gfx.gfx_ring[i];
424	if (!ring->mqd_obj) {
425	r = amdgpu_bo_create_kernel(adev, size: mqd_size, PAGE_SIZE,
426	domain, bo_ptr: &ring->mqd_obj,
427	gpu_addr: &ring->mqd_gpu_addr, cpu_addr: &ring->mqd_ptr);
428	if (r) {
429	dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
430	return r;
431	}
432
433	ring->mqd_size = mqd_size;
434	/* prepare MQD backup */
435	adev->gfx.me.mqd_backup[i] = kzalloc(mqd_size, GFP_KERNEL);
436	if (!adev->gfx.me.mqd_backup[i]) {
437	dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
438	return -ENOMEM;
439	}
440	}
441	}
442	}
443
444	/* create MQD for each KCQ */
445	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
446	j = i + xcc_id * adev->gfx.num_compute_rings;
447	ring = &adev->gfx.compute_ring[j];
448	if (!ring->mqd_obj) {
449	r = amdgpu_bo_create_kernel(adev, size: mqd_size, PAGE_SIZE,
450	domain, bo_ptr: &ring->mqd_obj,
451	gpu_addr: &ring->mqd_gpu_addr, cpu_addr: &ring->mqd_ptr);
452	if (r) {
453	dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
454	return r;
455	}
456
457	ring->mqd_size = mqd_size;
458	/* prepare MQD backup */
459	adev->gfx.mec.mqd_backup[j] = kzalloc(mqd_size, GFP_KERNEL);
460	if (!adev->gfx.mec.mqd_backup[j]) {
461	dev_warn(adev->dev, "no memory to create MQD backup for ring %s\n", ring->name);
462	return -ENOMEM;
463	}
464	}
465	}
466
467	return 0;
468	}
469
470	void amdgpu_gfx_mqd_sw_fini(struct amdgpu_device *adev, int xcc_id)
471	{
472	struct amdgpu_ring *ring = NULL;
473	int i, j;
474	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
475
476	if (adev->asic_type >= CHIP_NAVI10 && amdgpu_async_gfx_ring) {
477	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
478	ring = &adev->gfx.gfx_ring[i];
479	kfree(objp: adev->gfx.me.mqd_backup[i]);
480	amdgpu_bo_free_kernel(bo: &ring->mqd_obj,
481	gpu_addr: &ring->mqd_gpu_addr,
482	cpu_addr: &ring->mqd_ptr);
483	}
484	}
485
486	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
487	j = i + xcc_id * adev->gfx.num_compute_rings;
488	ring = &adev->gfx.compute_ring[j];
489	kfree(objp: adev->gfx.mec.mqd_backup[j]);
490	amdgpu_bo_free_kernel(bo: &ring->mqd_obj,
491	gpu_addr: &ring->mqd_gpu_addr,
492	cpu_addr: &ring->mqd_ptr);
493	}
494
495	ring = &kiq->ring;
496	kfree(objp: kiq->mqd_backup);
497	amdgpu_bo_free_kernel(bo: &ring->mqd_obj,
498	gpu_addr: &ring->mqd_gpu_addr,
499	cpu_addr: &ring->mqd_ptr);
500	}
501
502	int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev, int xcc_id)
503	{
504	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
505	struct amdgpu_ring *kiq_ring = &kiq->ring;
506	int i, r = 0;
507	int j;
508
509	if (adev->enable_mes) {
510	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
511	j = i + xcc_id * adev->gfx.num_compute_rings;
512	amdgpu_mes_unmap_legacy_queue(adev,
513	ring: &adev->gfx.compute_ring[j],
514	action: RESET_QUEUES, gpu_addr: 0, seq: 0);
515	}
516	return 0;
517	}
518
519	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
520	return -EINVAL;
521
522	if (!kiq_ring->sched.ready || amdgpu_in_reset(adev))
523	return 0;
524
525	spin_lock(lock: &kiq->ring_lock);
526	if (amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->unmap_queues_size *
527	adev->gfx.num_compute_rings)) {
528	spin_unlock(lock: &kiq->ring_lock);
529	return -ENOMEM;
530	}
531
532	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
533	j = i + xcc_id * adev->gfx.num_compute_rings;
534	kiq->pmf->kiq_unmap_queues(kiq_ring,
535	&adev->gfx.compute_ring[j],
536	RESET_QUEUES, 0, 0);
537	}
538	/* Submit unmap queue packet */
539	amdgpu_ring_commit(ring: kiq_ring);
540	/*
541	* Ring test will do a basic scratch register change check. Just run
542	* this to ensure that unmap queues that is submitted before got
543	* processed successfully before returning.
544	*/
545	r = amdgpu_ring_test_helper(ring: kiq_ring);
546
547	spin_unlock(lock: &kiq->ring_lock);
548
549	return r;
550	}
551
552	int amdgpu_gfx_disable_kgq(struct amdgpu_device *adev, int xcc_id)
553	{
554	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
555	struct amdgpu_ring *kiq_ring = &kiq->ring;
556	int i, r = 0;
557	int j;
558
559	if (adev->enable_mes) {
560	if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
561	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
562	j = i + xcc_id * adev->gfx.num_gfx_rings;
563	amdgpu_mes_unmap_legacy_queue(adev,
564	ring: &adev->gfx.gfx_ring[j],
565	action: PREEMPT_QUEUES, gpu_addr: 0, seq: 0);
566	}
567	}
568	return 0;
569	}
570
571	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
572	return -EINVAL;
573
574	if (!adev->gfx.kiq[0].ring.sched.ready || amdgpu_in_reset(adev))
575	return 0;
576
577	if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
578	spin_lock(lock: &kiq->ring_lock);
579	if (amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->unmap_queues_size *
580	adev->gfx.num_gfx_rings)) {
581	spin_unlock(lock: &kiq->ring_lock);
582	return -ENOMEM;
583	}
584
585	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
586	j = i + xcc_id * adev->gfx.num_gfx_rings;
587	kiq->pmf->kiq_unmap_queues(kiq_ring,
588	&adev->gfx.gfx_ring[j],
589	PREEMPT_QUEUES, 0, 0);
590	}
591	/* Submit unmap queue packet */
592	amdgpu_ring_commit(ring: kiq_ring);
593
594	/*
595	* Ring test will do a basic scratch register change check.
596	* Just run this to ensure that unmap queues that is submitted
597	* before got processed successfully before returning.
598	*/
599	r = amdgpu_ring_test_helper(ring: kiq_ring);
600	spin_unlock(lock: &kiq->ring_lock);
601	}
602
603	return r;
604	}
605
606	int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
607	int queue_bit)
608	{
609	int mec, pipe, queue;
610	int set_resource_bit = 0;
611
612	amdgpu_queue_mask_bit_to_mec_queue(adev, bit: queue_bit, mec: &mec, pipe: &pipe, queue: &queue);
613
614	set_resource_bit = mec * 4 * 8 + pipe * 8 + queue;
615
616	return set_resource_bit;
617	}
618
619	static int amdgpu_gfx_mes_enable_kcq(struct amdgpu_device *adev, int xcc_id)
620	{
621	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
622	struct amdgpu_ring *kiq_ring = &kiq->ring;
623	uint64_t queue_mask = ~0ULL;
624	int r, i, j;
625
626	amdgpu_device_flush_hdp(adev, NULL);
627
628	if (!adev->enable_uni_mes) {
629	spin_lock(lock: &kiq->ring_lock);
630	r = amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->set_resources_size);
631	if (r) {
632	dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
633	spin_unlock(lock: &kiq->ring_lock);
634	return r;
635	}
636
637	kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
638	r = amdgpu_ring_test_helper(ring: kiq_ring);
639	spin_unlock(lock: &kiq->ring_lock);
640	if (r)
641	dev_err(adev->dev, "KIQ failed to set resources\n");
642	}
643
644	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
645	j = i + xcc_id * adev->gfx.num_compute_rings;
646	r = amdgpu_mes_map_legacy_queue(adev,
647	ring: &adev->gfx.compute_ring[j]);
648	if (r) {
649	dev_err(adev->dev, "failed to map compute queue\n");
650	return r;
651	}
652	}
653
654	return 0;
655	}
656
657	int amdgpu_gfx_enable_kcq(struct amdgpu_device *adev, int xcc_id)
658	{
659	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
660	struct amdgpu_ring *kiq_ring = &kiq->ring;
661	uint64_t queue_mask = 0;
662	int r, i, j;
663
664	if (adev->mes.enable_legacy_queue_map)
665	return amdgpu_gfx_mes_enable_kcq(adev, xcc_id);
666
667	if (!kiq->pmf || !kiq->pmf->kiq_map_queues || !kiq->pmf->kiq_set_resources)
668	return -EINVAL;
669
670	for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
671	if (!test_bit(i, adev->gfx.mec_bitmap[xcc_id].queue_bitmap))
672	continue;
673
674	/* This situation may be hit in the future if a new HW
675	* generation exposes more than 64 queues. If so, the
676	* definition of queue_mask needs updating */
677	if (WARN_ON(i > (sizeof(queue_mask)*8))) {
678	dev_err(adev->dev, "Invalid KCQ enabled: %d\n", i);
679	break;
680	}
681
682	queue_mask |= (1ull << amdgpu_queue_mask_bit_to_set_resource_bit(adev, queue_bit: i));
683	}
684
685	amdgpu_device_flush_hdp(adev, NULL);
686
687	dev_info(adev->dev, "kiq ring mec %d pipe %d q %d\n", kiq_ring->me,
688	kiq_ring->pipe, kiq_ring->queue);
689
690	spin_lock(lock: &kiq->ring_lock);
691	r = amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->map_queues_size *
692	adev->gfx.num_compute_rings +
693	kiq->pmf->set_resources_size);
694	if (r) {
695	dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
696	spin_unlock(lock: &kiq->ring_lock);
697	return r;
698	}
699
700	kiq->pmf->kiq_set_resources(kiq_ring, queue_mask);
701	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
702	j = i + xcc_id * adev->gfx.num_compute_rings;
703	kiq->pmf->kiq_map_queues(kiq_ring,
704	&adev->gfx.compute_ring[j]);
705	}
706	/* Submit map queue packet */
707	amdgpu_ring_commit(ring: kiq_ring);
708	/*
709	* Ring test will do a basic scratch register change check. Just run
710	* this to ensure that map queues that is submitted before got
711	* processed successfully before returning.
712	*/
713	r = amdgpu_ring_test_helper(ring: kiq_ring);
714	spin_unlock(lock: &kiq->ring_lock);
715	if (r)
716	dev_err(adev->dev, "KCQ enable failed\n");
717
718	return r;
719	}
720
721	int amdgpu_gfx_enable_kgq(struct amdgpu_device *adev, int xcc_id)
722	{
723	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
724	struct amdgpu_ring *kiq_ring = &kiq->ring;
725	int r, i, j;
726
727	if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
728	return -EINVAL;
729
730	amdgpu_device_flush_hdp(adev, NULL);
731
732	if (adev->mes.enable_legacy_queue_map) {
733	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
734	j = i + xcc_id * adev->gfx.num_gfx_rings;
735	r = amdgpu_mes_map_legacy_queue(adev,
736	ring: &adev->gfx.gfx_ring[j]);
737	if (r) {
738	dev_err(adev->dev, "failed to map gfx queue\n");
739	return r;
740	}
741	}
742
743	return 0;
744	}
745
746	spin_lock(lock: &kiq->ring_lock);
747	/* No need to map kcq on the slave */
748	if (amdgpu_gfx_is_master_xcc(adev, xcc_id)) {
749	r = amdgpu_ring_alloc(ring: kiq_ring, ndw: kiq->pmf->map_queues_size *
750	adev->gfx.num_gfx_rings);
751	if (r) {
752	dev_err(adev->dev, "Failed to lock KIQ (%d).\n", r);
753	spin_unlock(lock: &kiq->ring_lock);
754	return r;
755	}
756
757	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
758	j = i + xcc_id * adev->gfx.num_gfx_rings;
759	kiq->pmf->kiq_map_queues(kiq_ring,
760	&adev->gfx.gfx_ring[j]);
761	}
762	}
763	/* Submit map queue packet */
764	amdgpu_ring_commit(ring: kiq_ring);
765	/*
766	* Ring test will do a basic scratch register change check. Just run
767	* this to ensure that map queues that is submitted before got
768	* processed successfully before returning.
769	*/
770	r = amdgpu_ring_test_helper(ring: kiq_ring);
771	spin_unlock(lock: &kiq->ring_lock);
772	if (r)
773	dev_err(adev->dev, "KGQ enable failed\n");
774
775	return r;
776	}
777
778	static void amdgpu_gfx_do_off_ctrl(struct amdgpu_device *adev, bool enable,
779	bool no_delay)
780	{
781	unsigned long delay = GFX_OFF_DELAY_ENABLE;
782
783	if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
784	return;
785
786	mutex_lock(&adev->gfx.gfx_off_mutex);
787
788	if (enable) {
789	/* If the count is already 0, it means there's an imbalance bug somewhere.
790	* Note that the bug may be in a different caller than the one which triggers the
791	* WARN_ON_ONCE.
792	*/
793	if (WARN_ON_ONCE(adev->gfx.gfx_off_req_count == 0))
794	goto unlock;
795
796	adev->gfx.gfx_off_req_count--;
797
798	if (adev->gfx.gfx_off_req_count == 0 &&
799	!adev->gfx.gfx_off_state) {
800	/* If going to s2idle, no need to wait */
801	if (no_delay) {
802	if (!amdgpu_dpm_set_powergating_by_smu(adev,
803	block_type: AMD_IP_BLOCK_TYPE_GFX, gate: true, inst: 0))
804	adev->gfx.gfx_off_state = true;
805	} else {
806	schedule_delayed_work(dwork: &adev->gfx.gfx_off_delay_work,
807	delay);
808	}
809	}
810	} else {
811	if (adev->gfx.gfx_off_req_count == 0) {
812	cancel_delayed_work_sync(dwork: &adev->gfx.gfx_off_delay_work);
813
814	if (adev->gfx.gfx_off_state &&
815	!amdgpu_dpm_set_powergating_by_smu(adev, block_type: AMD_IP_BLOCK_TYPE_GFX, gate: false, inst: 0)) {
816	adev->gfx.gfx_off_state = false;
817
818	if (adev->gfx.funcs->init_spm_golden) {
819	dev_dbg(adev->dev,
820	"GFXOFF is disabled, re-init SPM golden settings\n");
821	amdgpu_gfx_init_spm_golden(adev);
822	}
823	}
824	}
825
826	adev->gfx.gfx_off_req_count++;
827	}
828
829	unlock:
830	mutex_unlock(lock: &adev->gfx.gfx_off_mutex);
831	}
832
833	/* amdgpu_gfx_off_ctrl - Handle gfx off feature enable/disable
834	*
835	* @adev: amdgpu_device pointer
836	* @bool enable true: enable gfx off feature, false: disable gfx off feature
837	*
838	* 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled.
839	* 2. other client can send request to disable gfx off feature, the request should be honored.
840	* 3. other client can cancel their request of disable gfx off feature
841	* 4. other client should not send request to enable gfx off feature before disable gfx off feature.
842	*
843	* gfx off allow will be delayed by GFX_OFF_DELAY_ENABLE ms.
844	*/
845	void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable)
846	{
847	/* If going to s2idle, no need to wait */
848	bool no_delay = adev->in_s0ix ? true : false;
849
850	amdgpu_gfx_do_off_ctrl(adev, enable, no_delay);
851	}
852
853	/* amdgpu_gfx_off_ctrl_immediate - Handle gfx off feature enable/disable
854	*
855	* @adev: amdgpu_device pointer
856	* @bool enable true: enable gfx off feature, false: disable gfx off feature
857	*
858	* 1. gfx off feature will be enabled by gfx ip after gfx cg pg enabled.
859	* 2. other client can send request to disable gfx off feature, the request should be honored.
860	* 3. other client can cancel their request of disable gfx off feature
861	* 4. other client should not send request to enable gfx off feature before disable gfx off feature.
862	*
863	* gfx off allow will be issued immediately.
864	*/
865	void amdgpu_gfx_off_ctrl_immediate(struct amdgpu_device *adev, bool enable)
866	{
867	amdgpu_gfx_do_off_ctrl(adev, enable, no_delay: true);
868	}
869
870	int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value)
871	{
872	int r = 0;
873
874	mutex_lock(&adev->gfx.gfx_off_mutex);
875
876	r = amdgpu_dpm_set_residency_gfxoff(adev, value);
877
878	mutex_unlock(lock: &adev->gfx.gfx_off_mutex);
879
880	return r;
881	}
882
883	int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *value)
884	{
885	int r = 0;
886
887	mutex_lock(&adev->gfx.gfx_off_mutex);
888
889	r = amdgpu_dpm_get_residency_gfxoff(adev, value);
890
891	mutex_unlock(lock: &adev->gfx.gfx_off_mutex);
892
893	return r;
894	}
895
896	int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value)
897	{
898	int r = 0;
899
900	mutex_lock(&adev->gfx.gfx_off_mutex);
901
902	r = amdgpu_dpm_get_entrycount_gfxoff(adev, value);
903
904	mutex_unlock(lock: &adev->gfx.gfx_off_mutex);
905
906	return r;
907	}
908
909	int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value)
910	{
911
912	int r = 0;
913
914	mutex_lock(&adev->gfx.gfx_off_mutex);
915
916	r = amdgpu_dpm_get_status_gfxoff(adev, value);
917
918	mutex_unlock(lock: &adev->gfx.gfx_off_mutex);
919
920	return r;
921	}
922
923	int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
924	{
925	int r;
926
927	if (amdgpu_ras_is_supported(adev, block: ras_block->block)) {
928	if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
929	r = amdgpu_ras_reset_error_status(adev, block: AMDGPU_RAS_BLOCK__GFX);
930	if (r)
931	return r;
932	}
933
934	r = amdgpu_ras_block_late_init(adev, ras_block);
935	if (r)
936	return r;
937
938	if (amdgpu_sriov_vf(adev))
939	return r;
940
941	if (adev->gfx.cp_ecc_error_irq.funcs) {
942	r = amdgpu_irq_get(adev, src: &adev->gfx.cp_ecc_error_irq, type: 0);
943	if (r)
944	goto late_fini;
945	}
946	} else {
947	amdgpu_ras_feature_enable_on_boot(adev, head: ras_block, enable: 0);
948	}
949
950	return 0;
951	late_fini:
952	amdgpu_ras_block_late_fini(adev, ras_block);
953	return r;
954	}
955
956	int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev)
957	{
958	int err = 0;
959	struct amdgpu_gfx_ras *ras = NULL;
960
961	/* adev->gfx.ras is NULL, which means gfx does not
962	* support ras function, then do nothing here.
963	*/
964	if (!adev->gfx.ras)
965	return 0;
966
967	ras = adev->gfx.ras;
968
969	err = amdgpu_ras_register_ras_block(adev, ras_block_obj: &ras->ras_block);
970	if (err) {
971	dev_err(adev->dev, "Failed to register gfx ras block!\n");
972	return err;
973	}
974
975	strcpy(p: ras->ras_block.ras_comm.name, q: "gfx");
976	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__GFX;
977	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
978	adev->gfx.ras_if = &ras->ras_block.ras_comm;
979
980	/* If not define special ras_late_init function, use gfx default ras_late_init */
981	if (!ras->ras_block.ras_late_init)
982	ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init;
983
984	/* If not defined special ras_cb function, use default ras_cb */
985	if (!ras->ras_block.ras_cb)
986	ras->ras_block.ras_cb = amdgpu_gfx_process_ras_data_cb;
987
988	return 0;
989	}
990
991	int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev,
992	struct amdgpu_iv_entry *entry)
993	{
994	if (adev->gfx.ras && adev->gfx.ras->poison_consumption_handler)
995	return adev->gfx.ras->poison_consumption_handler(adev, entry);
996
997	return 0;
998	}
999
1000	int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev,
1001	void *err_data,
1002	struct amdgpu_iv_entry *entry)
1003	{
1004	/* TODO ue will trigger an interrupt.
1005	*
1006	* When “Full RAS” is enabled, the per-IP interrupt sources should
1007	* be disabled and the driver should only look for the aggregated
1008	* interrupt via sync flood
1009	*/
1010	if (!amdgpu_ras_is_supported(adev, block: AMDGPU_RAS_BLOCK__GFX)) {
1011	kgd2kfd_set_sram_ecc_flag(kfd: adev->kfd.dev);
1012	if (adev->gfx.ras && adev->gfx.ras->ras_block.hw_ops &&
1013	adev->gfx.ras->ras_block.hw_ops->query_ras_error_count)
1014	adev->gfx.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
1015	amdgpu_ras_reset_gpu(adev);
1016	}
1017	return AMDGPU_RAS_SUCCESS;
1018	}
1019
1020	int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev,
1021	struct amdgpu_irq_src *source,
1022	struct amdgpu_iv_entry *entry)
1023	{
1024	struct ras_common_if *ras_if = adev->gfx.ras_if;
1025	struct ras_dispatch_if ih_data = {
1026	.entry = entry,
1027	};
1028
1029	if (!ras_if)
1030	return 0;
1031
1032	ih_data.head = *ras_if;
1033
1034	dev_err(adev->dev, "CP ECC ERROR IRQ\n");
1035	amdgpu_ras_interrupt_dispatch(adev, info: &ih_data);
1036	return 0;
1037	}
1038
1039	void amdgpu_gfx_ras_error_func(struct amdgpu_device *adev,
1040	void *ras_error_status,
1041	void (*func)(struct amdgpu_device *adev, void *ras_error_status,
1042	int xcc_id))
1043	{
1044	int i;
1045	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
1046	uint32_t xcc_mask = GENMASK(num_xcc - 1, 0);
1047	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1048
1049	if (err_data) {
1050	err_data->ue_count = 0;
1051	err_data->ce_count = 0;
1052	}
1053
1054	for_each_inst(i, xcc_mask)
1055	func(adev, ras_error_status, i);
1056	}
1057
1058	uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_id)
1059	{
1060	signed long r, cnt = 0;
1061	unsigned long flags;
1062	uint32_t seq, reg_val_offs = 0, value = 0;
1063	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
1064	struct amdgpu_ring *ring = &kiq->ring;
1065
1066	if (amdgpu_device_skip_hw_access(adev))
1067	return 0;
1068
1069	if (adev->mes.ring[0].sched.ready)
1070	return amdgpu_mes_rreg(adev, reg);
1071
1072	BUG_ON(!ring->funcs->emit_rreg);
1073
1074	spin_lock_irqsave(&kiq->ring_lock, flags);
1075	if (amdgpu_device_wb_get(adev, wb: &reg_val_offs)) {
1076	pr_err("critical bug! too many kiq readers\n");
1077	goto failed_unlock;
1078	}
1079	r = amdgpu_ring_alloc(ring, ndw: 32);
1080	if (r)
1081	goto failed_unlock;
1082
1083	amdgpu_ring_emit_rreg(ring, reg, reg_val_offs);
1084	r = amdgpu_fence_emit_polling(ring, s: &seq, MAX_KIQ_REG_WAIT);
1085	if (r)
1086	goto failed_undo;
1087
1088	amdgpu_ring_commit(ring);
1089	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
1090
1091	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
1092
1093	/* don't wait anymore for gpu reset case because this way may
1094	* block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1095	* is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1096	* never return if we keep waiting in virt_kiq_rreg, which cause
1097	* gpu_recover() hang there.
1098	*
1099	* also don't wait anymore for IRQ context
1100	* */
1101	if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1102	goto failed_kiq_read;
1103
1104	might_sleep();
1105	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1106	if (amdgpu_in_reset(adev))
1107	goto failed_kiq_read;
1108
1109	msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1110	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
1111	}
1112
1113	if (cnt > MAX_KIQ_REG_TRY)
1114	goto failed_kiq_read;
1115
1116	mb();
1117	value = adev->wb.wb[reg_val_offs];
1118	amdgpu_device_wb_free(adev, wb: reg_val_offs);
1119	return value;
1120
1121	failed_undo:
1122	amdgpu_ring_undo(ring);
1123	failed_unlock:
1124	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
1125	failed_kiq_read:
1126	if (reg_val_offs)
1127	amdgpu_device_wb_free(adev, wb: reg_val_offs);
1128	dev_err(adev->dev, "failed to read reg:%x\n", reg);
1129	return ~0;
1130	}
1131
1132	void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint32_t xcc_id)
1133	{
1134	signed long r, cnt = 0;
1135	unsigned long flags;
1136	uint32_t seq;
1137	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
1138	struct amdgpu_ring *ring = &kiq->ring;
1139
1140	BUG_ON(!ring->funcs->emit_wreg);
1141
1142	if (amdgpu_device_skip_hw_access(adev))
1143	return;
1144
1145	if (adev->mes.ring[0].sched.ready) {
1146	amdgpu_mes_wreg(adev, reg, val: v);
1147	return;
1148	}
1149
1150	spin_lock_irqsave(&kiq->ring_lock, flags);
1151	r = amdgpu_ring_alloc(ring, ndw: 32);
1152	if (r)
1153	goto failed_unlock;
1154
1155	amdgpu_ring_emit_wreg(ring, reg, v);
1156	r = amdgpu_fence_emit_polling(ring, s: &seq, MAX_KIQ_REG_WAIT);
1157	if (r)
1158	goto failed_undo;
1159
1160	amdgpu_ring_commit(ring);
1161	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
1162
1163	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
1164
1165	/* don't wait anymore for gpu reset case because this way may
1166	* block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1167	* is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1168	* never return if we keep waiting in virt_kiq_rreg, which cause
1169	* gpu_recover() hang there.
1170	*
1171	* also don't wait anymore for IRQ context
1172	* */
1173	if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1174	goto failed_kiq_write;
1175
1176	might_sleep();
1177	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1178	if (amdgpu_in_reset(adev))
1179	goto failed_kiq_write;
1180
1181	msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1182	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
1183	}
1184
1185	if (cnt > MAX_KIQ_REG_TRY)
1186	goto failed_kiq_write;
1187
1188	return;
1189
1190	failed_undo:
1191	amdgpu_ring_undo(ring);
1192	failed_unlock:
1193	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
1194	failed_kiq_write:
1195	dev_err(adev->dev, "failed to write reg:%x\n", reg);
1196	}
1197
1198	int amdgpu_kiq_hdp_flush(struct amdgpu_device *adev)
1199	{
1200	signed long r, cnt = 0;
1201	unsigned long flags;
1202	uint32_t seq;
1203	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
1204	struct amdgpu_ring *ring = &kiq->ring;
1205
1206	if (amdgpu_device_skip_hw_access(adev))
1207	return 0;
1208
1209	if (adev->enable_mes_kiq && adev->mes.ring[0].sched.ready)
1210	return amdgpu_mes_hdp_flush(adev);
1211
1212	if (!ring->funcs->emit_hdp_flush) {
1213	return -EOPNOTSUPP;
1214	}
1215
1216	spin_lock_irqsave(&kiq->ring_lock, flags);
1217	r = amdgpu_ring_alloc(ring, ndw: 32);
1218	if (r)
1219	goto failed_unlock;
1220
1221	amdgpu_ring_emit_hdp_flush(ring);
1222	r = amdgpu_fence_emit_polling(ring, s: &seq, MAX_KIQ_REG_WAIT);
1223	if (r)
1224	goto failed_undo;
1225
1226	amdgpu_ring_commit(ring);
1227	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
1228
1229	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
1230
1231	/* don't wait anymore for gpu reset case because this way may
1232	* block gpu_recover() routine forever, e.g. this virt_kiq_rreg
1233	* is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
1234	* never return if we keep waiting in virt_kiq_rreg, which cause
1235	* gpu_recover() hang there.
1236	*
1237	* also don't wait anymore for IRQ context
1238	* */
1239	if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt()))
1240	goto failed_kiq_hdp_flush;
1241
1242	might_sleep();
1243	while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
1244	if (amdgpu_in_reset(adev))
1245	goto failed_kiq_hdp_flush;
1246
1247	msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
1248	r = amdgpu_fence_wait_polling(ring, wait_seq: seq, MAX_KIQ_REG_WAIT);
1249	}
1250
1251	if (cnt > MAX_KIQ_REG_TRY) {
1252	dev_err(adev->dev, "failed to flush HDP via KIQ timeout\n");
1253	return -ETIMEDOUT;
1254	}
1255
1256	return 0;
1257
1258	failed_undo:
1259	amdgpu_ring_undo(ring);
1260	failed_unlock:
1261	spin_unlock_irqrestore(lock: &kiq->ring_lock, flags);
1262	failed_kiq_hdp_flush:
1263	dev_err(adev->dev, "failed to flush HDP via KIQ\n");
1264	return r < 0 ? r : -EIO;
1265	}
1266
1267	int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev)
1268	{
1269	if (amdgpu_num_kcq == -1) {
1270	return 8;
1271	} else if (amdgpu_num_kcq > 8 || amdgpu_num_kcq < 0) {
1272	dev_warn(adev->dev, "set kernel compute queue number to 8 due to invalid parameter provided by user\n");
1273	return 8;
1274	}
1275	return amdgpu_num_kcq;
1276	}
1277
1278	void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev,
1279	uint32_t ucode_id)
1280	{
1281	const struct gfx_firmware_header_v1_0 *cp_hdr;
1282	const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0;
1283	struct amdgpu_firmware_info *info = NULL;
1284	const struct firmware *ucode_fw;
1285	unsigned int fw_size;
1286
1287	switch (ucode_id) {
1288	case AMDGPU_UCODE_ID_CP_PFP:
1289	cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1290	adev->gfx.pfp_fw->data;
1291	adev->gfx.pfp_fw_version =
1292	le32_to_cpu(cp_hdr->header.ucode_version);
1293	adev->gfx.pfp_feature_version =
1294	le32_to_cpu(cp_hdr->ucode_feature_version);
1295	ucode_fw = adev->gfx.pfp_fw;
1296	fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1297	break;
1298	case AMDGPU_UCODE_ID_CP_RS64_PFP:
1299	cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1300	adev->gfx.pfp_fw->data;
1301	adev->gfx.pfp_fw_version =
1302	le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1303	adev->gfx.pfp_feature_version =
1304	le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1305	ucode_fw = adev->gfx.pfp_fw;
1306	fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1307	break;
1308	case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
1309	case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
1310	cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1311	adev->gfx.pfp_fw->data;
1312	ucode_fw = adev->gfx.pfp_fw;
1313	fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1314	break;
1315	case AMDGPU_UCODE_ID_CP_ME:
1316	cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1317	adev->gfx.me_fw->data;
1318	adev->gfx.me_fw_version =
1319	le32_to_cpu(cp_hdr->header.ucode_version);
1320	adev->gfx.me_feature_version =
1321	le32_to_cpu(cp_hdr->ucode_feature_version);
1322	ucode_fw = adev->gfx.me_fw;
1323	fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1324	break;
1325	case AMDGPU_UCODE_ID_CP_RS64_ME:
1326	cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1327	adev->gfx.me_fw->data;
1328	adev->gfx.me_fw_version =
1329	le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1330	adev->gfx.me_feature_version =
1331	le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1332	ucode_fw = adev->gfx.me_fw;
1333	fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1334	break;
1335	case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
1336	case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
1337	cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1338	adev->gfx.me_fw->data;
1339	ucode_fw = adev->gfx.me_fw;
1340	fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1341	break;
1342	case AMDGPU_UCODE_ID_CP_CE:
1343	cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1344	adev->gfx.ce_fw->data;
1345	adev->gfx.ce_fw_version =
1346	le32_to_cpu(cp_hdr->header.ucode_version);
1347	adev->gfx.ce_feature_version =
1348	le32_to_cpu(cp_hdr->ucode_feature_version);
1349	ucode_fw = adev->gfx.ce_fw;
1350	fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
1351	break;
1352	case AMDGPU_UCODE_ID_CP_MEC1:
1353	cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1354	adev->gfx.mec_fw->data;
1355	adev->gfx.mec_fw_version =
1356	le32_to_cpu(cp_hdr->header.ucode_version);
1357	adev->gfx.mec_feature_version =
1358	le32_to_cpu(cp_hdr->ucode_feature_version);
1359	ucode_fw = adev->gfx.mec_fw;
1360	fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1361	le32_to_cpu(cp_hdr->jt_size) * 4;
1362	break;
1363	case AMDGPU_UCODE_ID_CP_MEC1_JT:
1364	cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1365	adev->gfx.mec_fw->data;
1366	ucode_fw = adev->gfx.mec_fw;
1367	fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
1368	break;
1369	case AMDGPU_UCODE_ID_CP_MEC2:
1370	cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1371	adev->gfx.mec2_fw->data;
1372	adev->gfx.mec2_fw_version =
1373	le32_to_cpu(cp_hdr->header.ucode_version);
1374	adev->gfx.mec2_feature_version =
1375	le32_to_cpu(cp_hdr->ucode_feature_version);
1376	ucode_fw = adev->gfx.mec2_fw;
1377	fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
1378	le32_to_cpu(cp_hdr->jt_size) * 4;
1379	break;
1380	case AMDGPU_UCODE_ID_CP_MEC2_JT:
1381	cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1382	adev->gfx.mec2_fw->data;
1383	ucode_fw = adev->gfx.mec2_fw;
1384	fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
1385	break;
1386	case AMDGPU_UCODE_ID_CP_RS64_MEC:
1387	cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1388	adev->gfx.mec_fw->data;
1389	adev->gfx.mec_fw_version =
1390	le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
1391	adev->gfx.mec_feature_version =
1392	le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
1393	ucode_fw = adev->gfx.mec_fw;
1394	fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
1395	break;
1396	case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
1397	case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
1398	case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
1399	case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
1400	cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
1401	adev->gfx.mec_fw->data;
1402	ucode_fw = adev->gfx.mec_fw;
1403	fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
1404	break;
1405	default:
1406	dev_err(adev->dev, "Invalid ucode id %u\n", ucode_id);
1407	return;
1408	}
1409
1410	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1411	info = &adev->firmware.ucode[ucode_id];
1412	info->ucode_id = ucode_id;
1413	info->fw = ucode_fw;
1414	adev->firmware.fw_size += ALIGN(fw_size, PAGE_SIZE);
1415	}
1416	}
1417
1418	bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id)
1419	{
1420	return !(xcc_id % (adev->gfx.num_xcc_per_xcp ?
1421	adev->gfx.num_xcc_per_xcp : 1));
1422	}
1423
1424	static ssize_t amdgpu_gfx_get_current_compute_partition(struct device *dev,
1425	struct device_attribute *addr,
1426	char *buf)
1427	{
1428	struct drm_device *ddev = dev_get_drvdata(dev);
1429	struct amdgpu_device *adev = drm_to_adev(ddev);
1430	int mode;
1431
1432	/* Only minimal precaution taken to reject requests while in reset.*/
1433	if (amdgpu_in_reset(adev))
1434	return -EPERM;
1435
1436	mode = amdgpu_xcp_query_partition_mode(xcp_mgr: adev->xcp_mgr,
1437	AMDGPU_XCP_FL_NONE);
1438
1439	return sysfs_emit(buf, fmt: "%s\n", amdgpu_gfx_compute_mode_desc(mode));
1440	}
1441
1442	static ssize_t amdgpu_gfx_set_compute_partition(struct device *dev,
1443	struct device_attribute *addr,
1444	const char *buf, size_t count)
1445	{
1446	struct drm_device *ddev = dev_get_drvdata(dev);
1447	struct amdgpu_device *adev = drm_to_adev(ddev);
1448	enum amdgpu_gfx_partition mode;
1449	int ret = 0, num_xcc;
1450
1451	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1452	if (num_xcc % 2 != 0)
1453	return -EINVAL;
1454
1455	if (!strncasecmp(s1: "SPX", s2: buf, strlen("SPX"))) {
1456	mode = AMDGPU_SPX_PARTITION_MODE;
1457	} else if (!strncasecmp(s1: "DPX", s2: buf, strlen("DPX"))) {
1458	/*
1459	* DPX mode needs AIDs to be in multiple of 2.
1460	* Each AID connects 2 XCCs.
1461	*/
1462	if (num_xcc%4)
1463	return -EINVAL;
1464	mode = AMDGPU_DPX_PARTITION_MODE;
1465	} else if (!strncasecmp(s1: "TPX", s2: buf, strlen("TPX"))) {
1466	if (num_xcc != 6)
1467	return -EINVAL;
1468	mode = AMDGPU_TPX_PARTITION_MODE;
1469	} else if (!strncasecmp(s1: "QPX", s2: buf, strlen("QPX"))) {
1470	if (num_xcc != 8)
1471	return -EINVAL;
1472	mode = AMDGPU_QPX_PARTITION_MODE;
1473	} else if (!strncasecmp(s1: "CPX", s2: buf, strlen("CPX"))) {
1474	mode = AMDGPU_CPX_PARTITION_MODE;
1475	} else {
1476	return -EINVAL;
1477	}
1478
1479	/* Don't allow a switch while under reset */
1480	if (!down_read_trylock(sem: &adev->reset_domain->sem))
1481	return -EPERM;
1482
1483	ret = amdgpu_xcp_switch_partition_mode(xcp_mgr: adev->xcp_mgr, mode);
1484
1485	up_read(sem: &adev->reset_domain->sem);
1486
1487	if (ret)
1488	return ret;
1489
1490	return count;
1491	}
1492
1493	static const char *xcp_desc[] = {
1494	[AMDGPU_SPX_PARTITION_MODE] = "SPX",
1495	[AMDGPU_DPX_PARTITION_MODE] = "DPX",
1496	[AMDGPU_TPX_PARTITION_MODE] = "TPX",
1497	[AMDGPU_QPX_PARTITION_MODE] = "QPX",
1498	[AMDGPU_CPX_PARTITION_MODE] = "CPX",
1499	};
1500
1501	static ssize_t amdgpu_gfx_get_available_compute_partition(struct device *dev,
1502	struct device_attribute *addr,
1503	char *buf)
1504	{
1505	struct drm_device *ddev = dev_get_drvdata(dev);
1506	struct amdgpu_device *adev = drm_to_adev(ddev);
1507	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1508	int size = 0, mode;
1509	char *sep = "";
1510
1511	if (!xcp_mgr || !xcp_mgr->avail_xcp_modes)
1512	return sysfs_emit(buf, fmt: "Not supported\n");
1513
1514	for_each_inst(mode, xcp_mgr->avail_xcp_modes) {
1515	size += sysfs_emit_at(buf, at: size, fmt: "%s%s", sep, xcp_desc[mode]);
1516	sep = ", ";
1517	}
1518
1519	size += sysfs_emit_at(buf, at: size, fmt: "\n");
1520
1521	return size;
1522	}
1523
1524	static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring)
1525	{
1526	struct amdgpu_device *adev = ring->adev;
1527	struct drm_gpu_scheduler *sched = &ring->sched;
1528	struct drm_sched_entity entity;
1529	static atomic_t counter;
1530	struct dma_fence *f;
1531	struct amdgpu_job *job;
1532	struct amdgpu_ib *ib;
1533	void *owner;
1534	int i, r;
1535
1536	/* Initialize the scheduler entity */
1537	r = drm_sched_entity_init(entity: &entity, priority: DRM_SCHED_PRIORITY_NORMAL,
1538	sched_list: &sched, num_sched_list: 1, NULL);
1539	if (r) {
1540	dev_err(adev->dev, "Failed setting up GFX kernel entity.\n");
1541	goto err;
1542	}
1543
1544	/*
1545	* Use some unique dummy value as the owner to make sure we execute
1546	* the cleaner shader on each submission. The value just need to change
1547	* for each submission and is otherwise meaningless.
1548	*/
1549	owner = (void *)(unsigned long)atomic_inc_return(v: &counter);
1550
1551	r = amdgpu_job_alloc_with_ib(adev: ring->adev, entity: &entity, owner,
1552	size: 64, pool_type: 0, job: &job,
1553	AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER);
1554	if (r)
1555	goto err;
1556
1557	job->enforce_isolation = true;
1558	/* always run the cleaner shader */
1559	job->run_cleaner_shader = true;
1560
1561	ib = &job->ibs[0];
1562	for (i = 0; i <= ring->funcs->align_mask; ++i)
1563	ib->ptr[i] = ring->funcs->nop;
1564	ib->length_dw = ring->funcs->align_mask + 1;
1565
1566	f = amdgpu_job_submit(job);
1567
1568	r = dma_fence_wait(fence: f, intr: false);
1569	if (r)
1570	goto err;
1571
1572	dma_fence_put(fence: f);
1573
1574	/* Clean up the scheduler entity */
1575	drm_sched_entity_destroy(entity: &entity);
1576	return 0;
1577
1578	err:
1579	return r;
1580	}
1581
1582	static int amdgpu_gfx_run_cleaner_shader(struct amdgpu_device *adev, int xcp_id)
1583	{
1584	int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1585	struct amdgpu_ring *ring;
1586	int num_xcc_to_clear;
1587	int i, r, xcc_id;
1588
1589	if (adev->gfx.num_xcc_per_xcp)
1590	num_xcc_to_clear = adev->gfx.num_xcc_per_xcp;
1591	else
1592	num_xcc_to_clear = 1;
1593
1594	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
1595	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
1596	ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings];
1597	if ((ring->xcp_id == xcp_id) && ring->sched.ready) {
1598	r = amdgpu_gfx_run_cleaner_shader_job(ring);
1599	if (r)
1600	return r;
1601	num_xcc_to_clear--;
1602	break;
1603	}
1604	}
1605	}
1606
1607	if (num_xcc_to_clear)
1608	return -ENOENT;
1609
1610	return 0;
1611	}
1612
1613	/**
1614	* amdgpu_gfx_set_run_cleaner_shader - Execute the AMDGPU GFX Cleaner Shader
1615	* @dev: The device structure
1616	* @attr: The device attribute structure
1617	* @buf: The buffer containing the input data
1618	* @count: The size of the input data
1619	*
1620	* Provides the sysfs interface to manually run a cleaner shader, which is
1621	* used to clear the GPU state between different tasks. Writing a value to the
1622	* 'run_cleaner_shader' sysfs file triggers the cleaner shader execution.
1623	* The value written corresponds to the partition index on multi-partition
1624	* devices. On single-partition devices, the value should be '0'.
1625	*
1626	* The cleaner shader clears the Local Data Store (LDS) and General Purpose
1627	* Registers (GPRs) to ensure data isolation between GPU workloads.
1628	*
1629	* Return: The number of bytes written to the sysfs file.
1630	*/
1631	static ssize_t amdgpu_gfx_set_run_cleaner_shader(struct device *dev,
1632	struct device_attribute *attr,
1633	const char *buf,
1634	size_t count)
1635	{
1636	struct drm_device *ddev = dev_get_drvdata(dev);
1637	struct amdgpu_device *adev = drm_to_adev(ddev);
1638	int ret;
1639	long value;
1640
1641	if (amdgpu_in_reset(adev))
1642	return -EPERM;
1643	if (adev->in_suspend && !adev->in_runpm)
1644	return -EPERM;
1645
1646	if (adev->gfx.disable_kq)
1647	return -EPERM;
1648
1649	ret = kstrtol(s: buf, base: 0, res: &value);
1650
1651	if (ret)
1652	return -EINVAL;
1653
1654	if (value < 0)
1655	return -EINVAL;
1656
1657	if (adev->xcp_mgr) {
1658	if (value >= adev->xcp_mgr->num_xcps)
1659	return -EINVAL;
1660	} else {
1661	if (value > 1)
1662	return -EINVAL;
1663	}
1664
1665	ret = pm_runtime_get_sync(dev: ddev->dev);
1666	if (ret < 0) {
1667	pm_runtime_put_autosuspend(dev: ddev->dev);
1668	return ret;
1669	}
1670
1671	ret = amdgpu_gfx_run_cleaner_shader(adev, xcp_id: value);
1672
1673	pm_runtime_put_autosuspend(dev: ddev->dev);
1674
1675	if (ret)
1676	return ret;
1677
1678	return count;
1679	}
1680
1681	/**
1682	* amdgpu_gfx_get_enforce_isolation - Query AMDGPU GFX Enforce Isolation Settings
1683	* @dev: The device structure
1684	* @attr: The device attribute structure
1685	* @buf: The buffer to store the output data
1686	*
1687	* Provides the sysfs read interface to get the current settings of the 'enforce_isolation'
1688	* feature for each GPU partition. Reading from the 'enforce_isolation'
1689	* sysfs file returns the isolation settings for all partitions, where '0'
1690	* indicates disabled, '1' indicates enabled, and '2' indicates enabled in legacy mode,
1691	* and '3' indicates enabled without cleaner shader.
1692	*
1693	* Return: The number of bytes read from the sysfs file.
1694	*/
1695	static ssize_t amdgpu_gfx_get_enforce_isolation(struct device *dev,
1696	struct device_attribute *attr,
1697	char *buf)
1698	{
1699	struct drm_device *ddev = dev_get_drvdata(dev);
1700	struct amdgpu_device *adev = drm_to_adev(ddev);
1701	int i;
1702	ssize_t size = 0;
1703
1704	if (adev->xcp_mgr) {
1705	for (i = 0; i < adev->xcp_mgr->num_xcps; i++) {
1706	size += sysfs_emit_at(buf, at: size, fmt: "%u", adev->enforce_isolation[i]);
1707	if (i < (adev->xcp_mgr->num_xcps - 1))
1708	size += sysfs_emit_at(buf, at: size, fmt: " ");
1709	}
1710	buf[size++] = '\n';
1711	} else {
1712	size = sysfs_emit_at(buf, at: 0, fmt: "%u\n", adev->enforce_isolation[0]);
1713	}
1714
1715	return size;
1716	}
1717
1718	/**
1719	* amdgpu_gfx_set_enforce_isolation - Control AMDGPU GFX Enforce Isolation
1720	* @dev: The device structure
1721	* @attr: The device attribute structure
1722	* @buf: The buffer containing the input data
1723	* @count: The size of the input data
1724	*
1725	* This function allows control over the 'enforce_isolation' feature, which
1726	* serializes access to the graphics engine. Writing '0' to disable, '1' to
1727	* enable isolation with cleaner shader, '2' to enable legacy isolation without
1728	* cleaner shader, or '3' to enable process isolation without submitting the
1729	* cleaner shader to the 'enforce_isolation' sysfs file sets the isolation mode
1730	* for each partition. The input should specify the setting for all
1731	* partitions.
1732	*
1733	* Return: The number of bytes written to the sysfs file.
1734	*/
1735	static ssize_t amdgpu_gfx_set_enforce_isolation(struct device *dev,
1736	struct device_attribute *attr,
1737	const char *buf, size_t count)
1738	{
1739	struct drm_device *ddev = dev_get_drvdata(dev);
1740	struct amdgpu_device *adev = drm_to_adev(ddev);
1741	long partition_values[MAX_XCP] = {0};
1742	int ret, i, num_partitions;
1743	const char *input_buf = buf;
1744
1745	for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) {
1746	ret = sscanf(input_buf, "%ld", &partition_values[i]);
1747	if (ret <= 0)
1748	break;
1749
1750	/* Move the pointer to the next value in the string */
1751	input_buf = strchr(input_buf, ' ');
1752	if (input_buf) {
1753	input_buf++;
1754	} else {
1755	i++;
1756	break;
1757	}
1758	}
1759	num_partitions = i;
1760
1761	if (adev->xcp_mgr && num_partitions != adev->xcp_mgr->num_xcps)
1762	return -EINVAL;
1763
1764	if (!adev->xcp_mgr && num_partitions != 1)
1765	return -EINVAL;
1766
1767	for (i = 0; i < num_partitions; i++) {
1768	if (partition_values[i] != 0 &&
1769	partition_values[i] != 1 &&
1770	partition_values[i] != 2 &&
1771	partition_values[i] != 3)
1772	return -EINVAL;
1773	}
1774
1775	mutex_lock(&adev->enforce_isolation_mutex);
1776	for (i = 0; i < num_partitions; i++) {
1777	switch (partition_values[i]) {
1778	case 0:
1779	default:
1780	adev->enforce_isolation[i] = AMDGPU_ENFORCE_ISOLATION_DISABLE;
1781	break;
1782	case 1:
1783	adev->enforce_isolation[i] =
1784	AMDGPU_ENFORCE_ISOLATION_ENABLE;
1785	break;
1786	case 2:
1787	adev->enforce_isolation[i] =
1788	AMDGPU_ENFORCE_ISOLATION_ENABLE_LEGACY;
1789	break;
1790	case 3:
1791	adev->enforce_isolation[i] =
1792	AMDGPU_ENFORCE_ISOLATION_NO_CLEANER_SHADER;
1793	break;
1794	}
1795	}
1796	mutex_unlock(lock: &adev->enforce_isolation_mutex);
1797
1798	amdgpu_mes_update_enforce_isolation(adev);
1799
1800	return count;
1801	}
1802
1803	static ssize_t amdgpu_gfx_get_gfx_reset_mask(struct device *dev,
1804	struct device_attribute *attr,
1805	char *buf)
1806	{
1807	struct drm_device *ddev = dev_get_drvdata(dev);
1808	struct amdgpu_device *adev = drm_to_adev(ddev);
1809
1810	if (!adev)
1811	return -ENODEV;
1812
1813	return amdgpu_show_reset_mask(buf, supported_reset: adev->gfx.gfx_supported_reset);
1814	}
1815
1816	static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev,
1817	struct device_attribute *attr,
1818	char *buf)
1819	{
1820	struct drm_device *ddev = dev_get_drvdata(dev);
1821	struct amdgpu_device *adev = drm_to_adev(ddev);
1822
1823	if (!adev)
1824	return -ENODEV;
1825
1826	return amdgpu_show_reset_mask(buf, supported_reset: adev->gfx.compute_supported_reset);
1827	}
1828
1829	static DEVICE_ATTR(run_cleaner_shader, 0200,
1830	NULL, amdgpu_gfx_set_run_cleaner_shader);
1831
1832	static DEVICE_ATTR(enforce_isolation, 0644,
1833	amdgpu_gfx_get_enforce_isolation,
1834	amdgpu_gfx_set_enforce_isolation);
1835
1836	static DEVICE_ATTR(current_compute_partition, 0644,
1837	amdgpu_gfx_get_current_compute_partition,
1838	amdgpu_gfx_set_compute_partition);
1839
1840	static DEVICE_ATTR(available_compute_partition, 0444,
1841	amdgpu_gfx_get_available_compute_partition, NULL);
1842	static DEVICE_ATTR(gfx_reset_mask, 0444,
1843	amdgpu_gfx_get_gfx_reset_mask, NULL);
1844
1845	static DEVICE_ATTR(compute_reset_mask, 0444,
1846	amdgpu_gfx_get_compute_reset_mask, NULL);
1847
1848	static int amdgpu_gfx_sysfs_xcp_init(struct amdgpu_device *adev)
1849	{
1850	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1851	bool xcp_switch_supported;
1852	int r;
1853
1854	if (!xcp_mgr)
1855	return 0;
1856
1857	xcp_switch_supported =
1858	(xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode);
1859
1860	if (!xcp_switch_supported)
1861	dev_attr_current_compute_partition.attr.mode &=
1862	~(S_IWUSR | S_IWGRP | S_IWOTH);
1863
1864	r = device_create_file(device: adev->dev, entry: &dev_attr_current_compute_partition);
1865	if (r)
1866	return r;
1867
1868	if (xcp_switch_supported)
1869	r = device_create_file(device: adev->dev,
1870	entry: &dev_attr_available_compute_partition);
1871
1872	return r;
1873	}
1874
1875	static void amdgpu_gfx_sysfs_xcp_fini(struct amdgpu_device *adev)
1876	{
1877	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
1878	bool xcp_switch_supported;
1879
1880	if (!xcp_mgr)
1881	return;
1882
1883	xcp_switch_supported =
1884	(xcp_mgr->funcs && xcp_mgr->funcs->switch_partition_mode);
1885	device_remove_file(dev: adev->dev, attr: &dev_attr_current_compute_partition);
1886
1887	if (xcp_switch_supported)
1888	device_remove_file(dev: adev->dev,
1889	attr: &dev_attr_available_compute_partition);
1890	}
1891
1892	static int amdgpu_gfx_sysfs_isolation_shader_init(struct amdgpu_device *adev)
1893	{
1894	int r;
1895
1896	r = device_create_file(device: adev->dev, entry: &dev_attr_enforce_isolation);
1897	if (r)
1898	return r;
1899	if (adev->gfx.enable_cleaner_shader)
1900	r = device_create_file(device: adev->dev, entry: &dev_attr_run_cleaner_shader);
1901
1902	return r;
1903	}
1904
1905	static void amdgpu_gfx_sysfs_isolation_shader_fini(struct amdgpu_device *adev)
1906	{
1907	device_remove_file(dev: adev->dev, attr: &dev_attr_enforce_isolation);
1908	if (adev->gfx.enable_cleaner_shader)
1909	device_remove_file(dev: adev->dev, attr: &dev_attr_run_cleaner_shader);
1910	}
1911
1912	static int amdgpu_gfx_sysfs_reset_mask_init(struct amdgpu_device *adev)
1913	{
1914	int r = 0;
1915
1916	if (!amdgpu_gpu_recovery)
1917	return r;
1918
1919	if (adev->gfx.num_gfx_rings) {
1920	r = device_create_file(device: adev->dev, entry: &dev_attr_gfx_reset_mask);
1921	if (r)
1922	return r;
1923	}
1924
1925	if (adev->gfx.num_compute_rings) {
1926	r = device_create_file(device: adev->dev, entry: &dev_attr_compute_reset_mask);
1927	if (r)
1928	return r;
1929	}
1930
1931	return r;
1932	}
1933
1934	static void amdgpu_gfx_sysfs_reset_mask_fini(struct amdgpu_device *adev)
1935	{
1936	if (!amdgpu_gpu_recovery)
1937	return;
1938
1939	if (adev->gfx.num_gfx_rings)
1940	device_remove_file(dev: adev->dev, attr: &dev_attr_gfx_reset_mask);
1941
1942	if (adev->gfx.num_compute_rings)
1943	device_remove_file(dev: adev->dev, attr: &dev_attr_compute_reset_mask);
1944	}
1945
1946	int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev)
1947	{
1948	int r;
1949
1950	r = amdgpu_gfx_sysfs_xcp_init(adev);
1951	if (r) {
1952	dev_err(adev->dev, "failed to create xcp sysfs files");
1953	return r;
1954	}
1955
1956	r = amdgpu_gfx_sysfs_isolation_shader_init(adev);
1957	if (r)
1958	dev_err(adev->dev, "failed to create isolation sysfs files");
1959
1960	r = amdgpu_gfx_sysfs_reset_mask_init(adev);
1961	if (r)
1962	dev_err(adev->dev, "failed to create reset mask sysfs files");
1963
1964	return r;
1965	}
1966
1967	void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev)
1968	{
1969	if (adev->dev->kobj.sd) {
1970	amdgpu_gfx_sysfs_xcp_fini(adev);
1971	amdgpu_gfx_sysfs_isolation_shader_fini(adev);
1972	amdgpu_gfx_sysfs_reset_mask_fini(adev);
1973	}
1974	}
1975
1976	int amdgpu_gfx_cleaner_shader_sw_init(struct amdgpu_device *adev,
1977	unsigned int cleaner_shader_size)
1978	{
1979	if (!adev->gfx.enable_cleaner_shader)
1980	return -EOPNOTSUPP;
1981
1982	return amdgpu_bo_create_kernel(adev, size: cleaner_shader_size, PAGE_SIZE,
1983	AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT,
1984	bo_ptr: &adev->gfx.cleaner_shader_obj,
1985	gpu_addr: &adev->gfx.cleaner_shader_gpu_addr,
1986	cpu_addr: (void **)&adev->gfx.cleaner_shader_cpu_ptr);
1987	}
1988
1989	void amdgpu_gfx_cleaner_shader_sw_fini(struct amdgpu_device *adev)
1990	{
1991	if (!adev->gfx.enable_cleaner_shader)
1992	return;
1993
1994	amdgpu_bo_free_kernel(bo: &adev->gfx.cleaner_shader_obj,
1995	gpu_addr: &adev->gfx.cleaner_shader_gpu_addr,
1996	cpu_addr: (void **)&adev->gfx.cleaner_shader_cpu_ptr);
1997	}
1998
1999	void amdgpu_gfx_cleaner_shader_init(struct amdgpu_device *adev,
2000	unsigned int cleaner_shader_size,
2001	const void *cleaner_shader_ptr)
2002	{
2003	if (!adev->gfx.enable_cleaner_shader)
2004	return;
2005
2006	if (adev->gfx.cleaner_shader_cpu_ptr && cleaner_shader_ptr)
2007	memcpy_toio(adev->gfx.cleaner_shader_cpu_ptr, cleaner_shader_ptr,
2008	cleaner_shader_size);
2009	}
2010
2011	/**
2012	* amdgpu_gfx_kfd_sch_ctrl - Control the KFD scheduler from the KGD (Graphics Driver)
2013	* @adev: amdgpu_device pointer
2014	* @idx: Index of the scheduler to control
2015	* @enable: Whether to enable or disable the KFD scheduler
2016	*
2017	* This function is used to control the KFD (Kernel Fusion Driver) scheduler
2018	* from the KGD. It is part of the cleaner shader feature. This function plays
2019	* a key role in enforcing process isolation on the GPU.
2020	*
2021	* The function uses a reference count mechanism (kfd_sch_req_count) to keep
2022	* track of the number of requests to enable the KFD scheduler. When a request
2023	* to enable the KFD scheduler is made, the reference count is decremented.
2024	* When the reference count reaches zero, a delayed work is scheduled to
2025	* enforce isolation after a delay of GFX_SLICE_PERIOD.
2026	*
2027	* When a request to disable the KFD scheduler is made, the function first
2028	* checks if the reference count is zero. If it is, it cancels the delayed work
2029	* for enforcing isolation and checks if the KFD scheduler is active. If the
2030	* KFD scheduler is active, it sends a request to stop the KFD scheduler and
2031	* sets the KFD scheduler state to inactive. Then, it increments the reference
2032	* count.
2033	*
2034	* The function is synchronized using the kfd_sch_mutex to ensure that the KFD
2035	* scheduler state and reference count are updated atomically.
2036	*
2037	* Note: If the reference count is already zero when a request to enable the
2038	* KFD scheduler is made, it means there's an imbalance bug somewhere. The
2039	* function triggers a warning in this case.
2040	*/
2041	static void amdgpu_gfx_kfd_sch_ctrl(struct amdgpu_device *adev, u32 idx,
2042	bool enable)
2043	{
2044	mutex_lock(&adev->gfx.userq_sch_mutex);
2045
2046	if (enable) {
2047	/* If the count is already 0, it means there's an imbalance bug somewhere.
2048	* Note that the bug may be in a different caller than the one which triggers the
2049	* WARN_ON_ONCE.
2050	*/
2051	if (WARN_ON_ONCE(adev->gfx.userq_sch_req_count[idx] == 0)) {
2052	dev_err(adev->dev, "Attempted to enable KFD scheduler when reference count is already zero\n");
2053	goto unlock;
2054	}
2055
2056	adev->gfx.userq_sch_req_count[idx]--;
2057
2058	if (adev->gfx.userq_sch_req_count[idx] == 0 &&
2059	adev->gfx.userq_sch_inactive[idx]) {
2060	schedule_delayed_work(dwork: &adev->gfx.enforce_isolation[idx].work,
2061	delay: msecs_to_jiffies(m: adev->gfx.enforce_isolation_time[idx]));
2062	}
2063	} else {
2064	if (adev->gfx.userq_sch_req_count[idx] == 0) {
2065	cancel_delayed_work_sync(dwork: &adev->gfx.enforce_isolation[idx].work);
2066	if (!adev->gfx.userq_sch_inactive[idx]) {
2067	amdgpu_userq_stop_sched_for_enforce_isolation(adev, idx);
2068	if (adev->kfd.init_complete)
2069	amdgpu_amdkfd_stop_sched(adev, node_id: idx);
2070	adev->gfx.userq_sch_inactive[idx] = true;
2071	}
2072	}
2073
2074	adev->gfx.userq_sch_req_count[idx]++;
2075	}
2076
2077	unlock:
2078	mutex_unlock(lock: &adev->gfx.userq_sch_mutex);
2079	}
2080
2081	/**
2082	* amdgpu_gfx_enforce_isolation_handler - work handler for enforcing shader isolation
2083	*
2084	* @work: work_struct.
2085	*
2086	* This function is the work handler for enforcing shader isolation on AMD GPUs.
2087	* It counts the number of emitted fences for each GFX and compute ring. If there
2088	* are any fences, it schedules the `enforce_isolation_work` to be run after a
2089	* delay of `GFX_SLICE_PERIOD`. If there are no fences, it signals the Kernel Fusion
2090	* Driver (KFD) to resume the runqueue. The function is synchronized using the
2091	* `enforce_isolation_mutex`.
2092	*/
2093	void amdgpu_gfx_enforce_isolation_handler(struct work_struct *work)
2094	{
2095	struct amdgpu_isolation_work *isolation_work =
2096	container_of(work, struct amdgpu_isolation_work, work.work);
2097	struct amdgpu_device *adev = isolation_work->adev;
2098	u32 i, idx, fences = 0;
2099
2100	if (isolation_work->xcp_id == AMDGPU_XCP_NO_PARTITION)
2101	idx = 0;
2102	else
2103	idx = isolation_work->xcp_id;
2104
2105	if (idx >= MAX_XCP)
2106	return;
2107
2108	mutex_lock(&adev->enforce_isolation_mutex);
2109	for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i) {
2110	if (isolation_work->xcp_id == adev->gfx.gfx_ring[i].xcp_id)
2111	fences += amdgpu_fence_count_emitted(ring: &adev->gfx.gfx_ring[i]);
2112	}
2113	for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i) {
2114	if (isolation_work->xcp_id == adev->gfx.compute_ring[i].xcp_id)
2115	fences += amdgpu_fence_count_emitted(ring: &adev->gfx.compute_ring[i]);
2116	}
2117	if (fences) {
2118	/* we've already had our timeslice, so let's wrap this up */
2119	schedule_delayed_work(dwork: &adev->gfx.enforce_isolation[idx].work,
2120	delay: msecs_to_jiffies(m: 1));
2121	} else {
2122	/* Tell KFD to resume the runqueue */
2123	WARN_ON_ONCE(!adev->gfx.userq_sch_inactive[idx]);
2124	WARN_ON_ONCE(adev->gfx.userq_sch_req_count[idx]);
2125
2126	amdgpu_userq_start_sched_for_enforce_isolation(adev, idx);
2127	if (adev->kfd.init_complete)
2128	amdgpu_amdkfd_start_sched(adev, node_id: idx);
2129	adev->gfx.userq_sch_inactive[idx] = false;
2130	}
2131	mutex_unlock(lock: &adev->enforce_isolation_mutex);
2132	}
2133
2134	/**
2135	* amdgpu_gfx_enforce_isolation_wait_for_kfd - Manage KFD wait period for process isolation
2136	* @adev: amdgpu_device pointer
2137	* @idx: Index of the GPU partition
2138	*
2139	* When kernel submissions come in, the jobs are given a time slice and once
2140	* that time slice is up, if there are KFD user queues active, kernel
2141	* submissions are blocked until KFD has had its time slice. Once the KFD time
2142	* slice is up, KFD user queues are preempted and kernel submissions are
2143	* unblocked and allowed to run again.
2144	*/
2145	static void
2146	amdgpu_gfx_enforce_isolation_wait_for_kfd(struct amdgpu_device *adev,
2147	u32 idx)
2148	{
2149	unsigned long cjiffies;
2150	bool wait = false;
2151
2152	mutex_lock(&adev->enforce_isolation_mutex);
2153	if (adev->enforce_isolation[idx] == AMDGPU_ENFORCE_ISOLATION_ENABLE) {
2154	/* set the initial values if nothing is set */
2155	if (!adev->gfx.enforce_isolation_jiffies[idx]) {
2156	adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2157	adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2158	}
2159	/* Make sure KFD gets a chance to run */
2160	if (amdgpu_amdkfd_compute_active(adev, node_id: idx)) {
2161	cjiffies = jiffies;
2162	if (time_after(cjiffies, adev->gfx.enforce_isolation_jiffies[idx])) {
2163	cjiffies -= adev->gfx.enforce_isolation_jiffies[idx];
2164	if ((jiffies_to_msecs(j: cjiffies) >= GFX_SLICE_PERIOD_MS)) {
2165	/* if our time is up, let KGD work drain before scheduling more */
2166	wait = true;
2167	/* reset the timer period */
2168	adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2169	} else {
2170	/* set the timer period to what's left in our time slice */
2171	adev->gfx.enforce_isolation_time[idx] =
2172	GFX_SLICE_PERIOD_MS - jiffies_to_msecs(j: cjiffies);
2173	}
2174	} else {
2175	/* if jiffies wrap around we will just wait a little longer */
2176	adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2177	}
2178	} else {
2179	/* if there is no KFD work, then set the full slice period */
2180	adev->gfx.enforce_isolation_jiffies[idx] = jiffies;
2181	adev->gfx.enforce_isolation_time[idx] = GFX_SLICE_PERIOD_MS;
2182	}
2183	}
2184	mutex_unlock(lock: &adev->enforce_isolation_mutex);
2185
2186	if (wait)
2187	msleep(GFX_SLICE_PERIOD_MS);
2188	}
2189
2190	/**
2191	* amdgpu_gfx_enforce_isolation_ring_begin_use - Begin use of a ring with enforced isolation
2192	* @ring: Pointer to the amdgpu_ring structure
2193	*
2194	* Ring begin_use helper implementation for gfx which serializes access to the
2195	* gfx IP between kernel submission IOCTLs and KFD user queues when isolation
2196	* enforcement is enabled. The kernel submission IOCTLs and KFD user queues
2197	* each get a time slice when both are active.
2198	*/
2199	void amdgpu_gfx_enforce_isolation_ring_begin_use(struct amdgpu_ring *ring)
2200	{
2201	struct amdgpu_device *adev = ring->adev;
2202	u32 idx;
2203	bool sched_work = false;
2204
2205	if (!adev->gfx.enable_cleaner_shader)
2206	return;
2207
2208	if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION)
2209	idx = 0;
2210	else
2211	idx = ring->xcp_id;
2212
2213	if (idx >= MAX_XCP)
2214	return;
2215
2216	/* Don't submit more work until KFD has had some time */
2217	amdgpu_gfx_enforce_isolation_wait_for_kfd(adev, idx);
2218
2219	mutex_lock(&adev->enforce_isolation_mutex);
2220	if (adev->enforce_isolation[idx] == AMDGPU_ENFORCE_ISOLATION_ENABLE) {
2221	if (adev->kfd.init_complete)
2222	sched_work = true;
2223	}
2224	mutex_unlock(lock: &adev->enforce_isolation_mutex);
2225
2226	if (sched_work)
2227	amdgpu_gfx_kfd_sch_ctrl(adev, idx, enable: false);
2228	}
2229
2230	/**
2231	* amdgpu_gfx_enforce_isolation_ring_end_use - End use of a ring with enforced isolation
2232	* @ring: Pointer to the amdgpu_ring structure
2233	*
2234	* Ring end_use helper implementation for gfx which serializes access to the
2235	* gfx IP between kernel submission IOCTLs and KFD user queues when isolation
2236	* enforcement is enabled. The kernel submission IOCTLs and KFD user queues
2237	* each get a time slice when both are active.
2238	*/
2239	void amdgpu_gfx_enforce_isolation_ring_end_use(struct amdgpu_ring *ring)
2240	{
2241	struct amdgpu_device *adev = ring->adev;
2242	u32 idx;
2243	bool sched_work = false;
2244
2245	if (!adev->gfx.enable_cleaner_shader)
2246	return;
2247
2248	if (ring->xcp_id == AMDGPU_XCP_NO_PARTITION)
2249	idx = 0;
2250	else
2251	idx = ring->xcp_id;
2252
2253	if (idx >= MAX_XCP)
2254	return;
2255
2256	mutex_lock(&adev->enforce_isolation_mutex);
2257	if (adev->enforce_isolation[idx] == AMDGPU_ENFORCE_ISOLATION_ENABLE) {
2258	if (adev->kfd.init_complete)
2259	sched_work = true;
2260	}
2261	mutex_unlock(lock: &adev->enforce_isolation_mutex);
2262
2263	if (sched_work)
2264	amdgpu_gfx_kfd_sch_ctrl(adev, idx, enable: true);
2265	}
2266
2267	void amdgpu_gfx_profile_idle_work_handler(struct work_struct *work)
2268	{
2269	struct amdgpu_device *adev =
2270	container_of(work, struct amdgpu_device, gfx.idle_work.work);
2271	enum PP_SMC_POWER_PROFILE profile;
2272	u32 i, fences = 0;
2273	int r;
2274
2275	if (adev->gfx.num_gfx_rings)
2276	profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D;
2277	else
2278	profile = PP_SMC_POWER_PROFILE_COMPUTE;
2279
2280	for (i = 0; i < AMDGPU_MAX_GFX_RINGS; ++i)
2281	fences += amdgpu_fence_count_emitted(ring: &adev->gfx.gfx_ring[i]);
2282	for (i = 0; i < (AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES); ++i)
2283	fences += amdgpu_fence_count_emitted(ring: &adev->gfx.compute_ring[i]);
2284	if (!fences && !atomic_read(v: &adev->gfx.total_submission_cnt)) {
2285	mutex_lock(&adev->gfx.workload_profile_mutex);
2286	if (adev->gfx.workload_profile_active) {
2287	r = amdgpu_dpm_switch_power_profile(adev, type: profile, en: false);
2288	if (r)
2289	dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r,
2290	profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ?
2291	"fullscreen 3D" : "compute");
2292	adev->gfx.workload_profile_active = false;
2293	}
2294	mutex_unlock(lock: &adev->gfx.workload_profile_mutex);
2295	} else {
2296	schedule_delayed_work(dwork: &adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT);
2297	}
2298	}
2299
2300	void amdgpu_gfx_profile_ring_begin_use(struct amdgpu_ring *ring)
2301	{
2302	struct amdgpu_device *adev = ring->adev;
2303	enum PP_SMC_POWER_PROFILE profile;
2304	int r;
2305
2306	if (amdgpu_dpm_is_overdrive_enabled(adev))
2307	return;
2308
2309	if (adev->gfx.num_gfx_rings)
2310	profile = PP_SMC_POWER_PROFILE_FULLSCREEN3D;
2311	else
2312	profile = PP_SMC_POWER_PROFILE_COMPUTE;
2313
2314	atomic_inc(v: &adev->gfx.total_submission_cnt);
2315
2316	cancel_delayed_work_sync(dwork: &adev->gfx.idle_work);
2317
2318	/* We can safely return early here because we've cancelled the
2319	* the delayed work so there is no one else to set it to false
2320	* and we don't care if someone else sets it to true.
2321	*/
2322	if (adev->gfx.workload_profile_active)
2323	return;
2324
2325	mutex_lock(&adev->gfx.workload_profile_mutex);
2326	if (!adev->gfx.workload_profile_active) {
2327	r = amdgpu_dpm_switch_power_profile(adev, type: profile, en: true);
2328	if (r)
2329	dev_warn(adev->dev, "(%d) failed to disable %s power profile mode\n", r,
2330	profile == PP_SMC_POWER_PROFILE_FULLSCREEN3D ?
2331	"fullscreen 3D" : "compute");
2332	adev->gfx.workload_profile_active = true;
2333	}
2334	mutex_unlock(lock: &adev->gfx.workload_profile_mutex);
2335	}
2336
2337	void amdgpu_gfx_profile_ring_end_use(struct amdgpu_ring *ring)
2338	{
2339	struct amdgpu_device *adev = ring->adev;
2340
2341	if (amdgpu_dpm_is_overdrive_enabled(adev))
2342	return;
2343
2344	atomic_dec(v: &ring->adev->gfx.total_submission_cnt);
2345
2346	schedule_delayed_work(dwork: &ring->adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT);
2347	}
2348
2349	/**
2350	* amdgpu_gfx_csb_preamble_start - Set CSB preamble start
2351	*
2352	* @buffer: This is an output variable that gets the PACKET3 preamble setup.
2353	*
2354	* Return:
2355	* return the latest index.
2356	*/
2357	u32 amdgpu_gfx_csb_preamble_start(u32 *buffer)
2358	{
2359	u32 count = 0;
2360
2361	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2362	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2363
2364	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2365	buffer[count++] = cpu_to_le32(0x80000000);
2366	buffer[count++] = cpu_to_le32(0x80000000);
2367
2368	return count;
2369	}
2370
2371	/**
2372	* amdgpu_gfx_csb_data_parser - Parser CS data
2373	*
2374	* @adev: amdgpu_device pointer used to get the CS data and other gfx info.
2375	* @buffer: This is an output variable that gets the PACKET3 preamble end.
2376	* @count: Index to start set the preemble end.
2377	*
2378	* Return:
2379	* return the latest index.
2380	*/
2381	u32 amdgpu_gfx_csb_data_parser(struct amdgpu_device *adev, u32 *buffer, u32 count)
2382	{
2383	const struct cs_section_def *sect = NULL;
2384	const struct cs_extent_def *ext = NULL;
2385	u32 i;
2386
2387	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
2388	for (ext = sect->section; ext->extent != NULL; ++ext) {
2389	if (sect->id == SECT_CONTEXT) {
2390	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
2391	buffer[count++] = cpu_to_le32(ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
2392
2393	for (i = 0; i < ext->reg_count; i++)
2394	buffer[count++] = cpu_to_le32(ext->extent[i]);
2395	}
2396	}
2397	}
2398
2399	return count;
2400	}
2401
2402	/**
2403	* amdgpu_gfx_csb_preamble_end - Set CSB preamble end
2404	*
2405	* @buffer: This is an output variable that gets the PACKET3 preamble end.
2406	* @count: Index to start set the preemble end.
2407	*/
2408	void amdgpu_gfx_csb_preamble_end(u32 *buffer, u32 count)
2409	{
2410	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2411	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
2412
2413	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
2414	buffer[count++] = cpu_to_le32(0);
2415	}
2416
2417	/*
2418	* debugfs for to enable/disable gfx job submission to specific core.
2419	*/
2420	#if defined(CONFIG_DEBUG_FS)
2421	static int amdgpu_debugfs_gfx_sched_mask_set(void *data, u64 val)
2422	{
2423	struct amdgpu_device *adev = (struct amdgpu_device *)data;
2424	u32 i;
2425	u64 mask = 0;
2426	struct amdgpu_ring *ring;
2427
2428	if (!adev)
2429	return -ENODEV;
2430
2431	mask = (1ULL << adev->gfx.num_gfx_rings) - 1;
2432	if ((val & mask) == 0)
2433	return -EINVAL;
2434
2435	for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
2436	ring = &adev->gfx.gfx_ring[i];
2437	if (val & (1 << i))
2438	ring->sched.ready = true;
2439	else
2440	ring->sched.ready = false;
2441	}
2442	/* publish sched.ready flag update effective immediately across smp */
2443	smp_rmb();
2444	return 0;
2445	}
2446
2447	static int amdgpu_debugfs_gfx_sched_mask_get(void *data, u64 *val)
2448	{
2449	struct amdgpu_device *adev = (struct amdgpu_device *)data;
2450	u32 i;
2451	u64 mask = 0;
2452	struct amdgpu_ring *ring;
2453
2454	if (!adev)
2455	return -ENODEV;
2456	for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
2457	ring = &adev->gfx.gfx_ring[i];
2458	if (ring->sched.ready)
2459	mask |= 1ULL << i;
2460	}
2461
2462	*val = mask;
2463	return 0;
2464	}
2465
2466	DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gfx_sched_mask_fops,
2467	amdgpu_debugfs_gfx_sched_mask_get,
2468	amdgpu_debugfs_gfx_sched_mask_set, "%llx\n");
2469
2470	#endif
2471
2472	void amdgpu_debugfs_gfx_sched_mask_init(struct amdgpu_device *adev)
2473	{
2474	#if defined(CONFIG_DEBUG_FS)
2475	struct drm_minor *minor = adev_to_drm(adev)->primary;
2476	struct dentry *root = minor->debugfs_root;
2477	char name[32];
2478
2479	if (!(adev->gfx.num_gfx_rings > 1))
2480	return;
2481	sprintf(buf: name, fmt: "amdgpu_gfx_sched_mask");
2482	debugfs_create_file(name, 0600, root, adev,
2483	&amdgpu_debugfs_gfx_sched_mask_fops);
2484	#endif
2485	}
2486
2487	/*
2488	* debugfs for to enable/disable compute job submission to specific core.
2489	*/
2490	#if defined(CONFIG_DEBUG_FS)
2491	static int amdgpu_debugfs_compute_sched_mask_set(void *data, u64 val)
2492	{
2493	struct amdgpu_device *adev = (struct amdgpu_device *)data;
2494	u32 i;
2495	u64 mask = 0;
2496	struct amdgpu_ring *ring;
2497
2498	if (!adev)
2499	return -ENODEV;
2500
2501	mask = (1ULL << adev->gfx.num_compute_rings) - 1;
2502	if ((val & mask) == 0)
2503	return -EINVAL;
2504
2505	for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
2506	ring = &adev->gfx.compute_ring[i];
2507	if (val & (1 << i))
2508	ring->sched.ready = true;
2509	else
2510	ring->sched.ready = false;
2511	}
2512
2513	/* publish sched.ready flag update effective immediately across smp */
2514	smp_rmb();
2515	return 0;
2516	}
2517
2518	static int amdgpu_debugfs_compute_sched_mask_get(void *data, u64 *val)
2519	{
2520	struct amdgpu_device *adev = (struct amdgpu_device *)data;
2521	u32 i;
2522	u64 mask = 0;
2523	struct amdgpu_ring *ring;
2524
2525	if (!adev)
2526	return -ENODEV;
2527	for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
2528	ring = &adev->gfx.compute_ring[i];
2529	if (ring->sched.ready)
2530	mask |= 1ULL << i;
2531	}
2532
2533	*val = mask;
2534	return 0;
2535	}
2536
2537	DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_compute_sched_mask_fops,
2538	amdgpu_debugfs_compute_sched_mask_get,
2539	amdgpu_debugfs_compute_sched_mask_set, "%llx\n");
2540
2541	#endif
2542
2543	void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev)
2544	{
2545	#if defined(CONFIG_DEBUG_FS)
2546	struct drm_minor *minor = adev_to_drm(adev)->primary;
2547	struct dentry *root = minor->debugfs_root;
2548	char name[32];
2549
2550	if (!(adev->gfx.num_compute_rings > 1))
2551	return;
2552	sprintf(buf: name, fmt: "amdgpu_compute_sched_mask");
2553	debugfs_create_file(name, 0600, root, adev,
2554	&amdgpu_debugfs_compute_sched_mask_fops);
2555	#endif
2556	}
2557
2558