1	// SPDX-License-Identifier: GPL-2.0 OR MIT
2	/*
3	* Copyright 2014-2022 Advanced Micro Devices, Inc.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*
23	*/
24
25	#include <linux/ratelimit.h>
26	#include <linux/printk.h>
27	#include <linux/slab.h>
28	#include <linux/list.h>
29	#include <linux/types.h>
30	#include <linux/bitops.h>
31	#include <linux/sched.h>
32	#include "kfd_priv.h"
33	#include "kfd_device_queue_manager.h"
34	#include "kfd_mqd_manager.h"
35	#include "cik_regs.h"
36	#include "kfd_kernel_queue.h"
37	#include "amdgpu_amdkfd.h"
38	#include "amdgpu_reset.h"
39	#include "amdgpu_sdma.h"
40	#include "mes_v11_api_def.h"
41	#include "kfd_debug.h"
42
43	/* Size of the per-pipe EOP queue */
44	#define CIK_HPD_EOP_BYTES_LOG2 11
45	#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
46	/* See unmap_queues_cpsch() */
47	#define USE_DEFAULT_GRACE_PERIOD 0xffffffff
48
49	static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
50	u32 pasid, unsigned int vmid);
51
52	static int execute_queues_cpsch(struct device_queue_manager *dqm,
53	enum kfd_unmap_queues_filter filter,
54	uint32_t filter_param,
55	uint32_t grace_period);
56	static int unmap_queues_cpsch(struct device_queue_manager *dqm,
57	enum kfd_unmap_queues_filter filter,
58	uint32_t filter_param,
59	uint32_t grace_period,
60	bool reset);
61
62	static int map_queues_cpsch(struct device_queue_manager *dqm);
63
64	static void deallocate_sdma_queue(struct device_queue_manager *dqm,
65	struct queue *q);
66
67	static inline void deallocate_hqd(struct device_queue_manager *dqm,
68	struct queue *q);
69	static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
70	static int allocate_sdma_queue(struct device_queue_manager *dqm,
71	struct queue *q, const uint32_t *restore_sdma_id);
72
73	static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
74
75	static inline
76	enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
77	{
78	if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
79	return KFD_MQD_TYPE_SDMA;
80	return KFD_MQD_TYPE_CP;
81	}
82
83	static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
84	{
85	int i;
86	int pipe_offset = (mec * dqm->dev->kfd->shared_resources.num_pipe_per_mec
87	+ pipe) * dqm->dev->kfd->shared_resources.num_queue_per_pipe;
88
89	/* queue is available for KFD usage if bit is 1 */
90	for (i = 0; i < dqm->dev->kfd->shared_resources.num_queue_per_pipe; ++i)
91	if (test_bit(pipe_offset + i,
92	dqm->dev->kfd->shared_resources.cp_queue_bitmap))
93	return true;
94	return false;
95	}
96
97	unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
98	{
99	return bitmap_weight(src: dqm->dev->kfd->shared_resources.cp_queue_bitmap,
100	AMDGPU_MAX_QUEUES);
101	}
102
103	unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
104	{
105	return dqm->dev->kfd->shared_resources.num_queue_per_pipe;
106	}
107
108	unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
109	{
110	return dqm->dev->kfd->shared_resources.num_pipe_per_mec;
111	}
112
113	static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
114	{
115	return kfd_get_num_sdma_engines(kdev: dqm->dev) +
116	kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev);
117	}
118
119	unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
120	{
121	return kfd_get_num_sdma_engines(kdev: dqm->dev) *
122	dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
123	}
124
125	unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
126	{
127	return kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev) *
128	dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
129	}
130
131	static void init_sdma_bitmaps(struct device_queue_manager *dqm)
132	{
133	bitmap_zero(dst: dqm->sdma_bitmap, KFD_MAX_SDMA_QUEUES);
134	bitmap_set(map: dqm->sdma_bitmap, start: 0, nbits: get_num_sdma_queues(dqm));
135
136	bitmap_zero(dst: dqm->xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
137	bitmap_set(map: dqm->xgmi_sdma_bitmap, start: 0, nbits: get_num_xgmi_sdma_queues(dqm));
138
139	/* Mask out the reserved queues */
140	bitmap_andnot(dst: dqm->sdma_bitmap, src1: dqm->sdma_bitmap,
141	src2: dqm->dev->kfd->device_info.reserved_sdma_queues_bitmap,
142	KFD_MAX_SDMA_QUEUES);
143	}
144
145	void program_sh_mem_settings(struct device_queue_manager *dqm,
146	struct qcm_process_device *qpd)
147	{
148	uint32_t xcc_mask = dqm->dev->xcc_mask;
149	int xcc_id;
150
151	for_each_inst(xcc_id, xcc_mask)
152	dqm->dev->kfd2kgd->program_sh_mem_settings(
153	dqm->dev->adev, qpd->vmid, qpd->sh_mem_config,
154	qpd->sh_mem_ape1_base, qpd->sh_mem_ape1_limit,
155	qpd->sh_mem_bases, xcc_id);
156	}
157
158	static void kfd_hws_hang(struct device_queue_manager *dqm)
159	{
160	struct device_process_node *cur;
161	struct qcm_process_device *qpd;
162	struct queue *q;
163
164	/* Mark all device queues as reset. */
165	list_for_each_entry(cur, &dqm->queues, list) {
166	qpd = cur->qpd;
167	list_for_each_entry(q, &qpd->queues_list, list) {
168	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
169
170	pdd->has_reset_queue = true;
171	}
172	}
173
174	/*
175	* Issue a GPU reset if HWS is unresponsive
176	*/
177	amdgpu_amdkfd_gpu_reset(adev: dqm->dev->adev);
178	}
179
180	static int convert_to_mes_queue_type(int queue_type)
181	{
182	int mes_queue_type;
183
184	switch (queue_type) {
185	case KFD_QUEUE_TYPE_COMPUTE:
186	mes_queue_type = MES_QUEUE_TYPE_COMPUTE;
187	break;
188	case KFD_QUEUE_TYPE_SDMA:
189	mes_queue_type = MES_QUEUE_TYPE_SDMA;
190	break;
191	default:
192	WARN(1, "Invalid queue type %d", queue_type);
193	mes_queue_type = -EINVAL;
194	break;
195	}
196
197	return mes_queue_type;
198	}
199
200	static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
201	struct qcm_process_device *qpd)
202	{
203	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
204	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
205	struct mes_add_queue_input queue_input;
206	int r, queue_type;
207	uint64_t wptr_addr_off;
208
209	if (!dqm->sched_running || dqm->sched_halt)
210	return 0;
211	if (!down_read_trylock(sem: &adev->reset_domain->sem))
212	return -EIO;
213
214	memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
215	queue_input.process_id = pdd->pasid;
216	queue_input.page_table_base_addr = qpd->page_table_base;
217	queue_input.process_va_start = 0;
218	queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
219	/* MES unit for quantum is 100ns */
220	queue_input.process_quantum = KFD_MES_PROCESS_QUANTUM; /* Equivalent to 10ms. */
221	queue_input.process_context_addr = pdd->proc_ctx_gpu_addr;
222	queue_input.gang_quantum = KFD_MES_GANG_QUANTUM; /* Equivalent to 1ms */
223	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
224	queue_input.inprocess_gang_priority = q->properties.priority;
225	queue_input.gang_global_priority_level =
226	AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
227	queue_input.doorbell_offset = q->properties.doorbell_off;
228	queue_input.mqd_addr = q->gart_mqd_addr;
229	queue_input.wptr_addr = (uint64_t)q->properties.write_ptr;
230
231	wptr_addr_off = (uint64_t)q->properties.write_ptr & (PAGE_SIZE - 1);
232	queue_input.wptr_mc_addr = amdgpu_bo_gpu_offset(bo: q->properties.wptr_bo) + wptr_addr_off;
233
234	queue_input.is_kfd_process = 1;
235	queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);
236	queue_input.queue_size = q->properties.queue_size >> 2;
237
238	queue_input.paging = false;
239	queue_input.tba_addr = qpd->tba_addr;
240	queue_input.tma_addr = qpd->tma_addr;
241	queue_input.trap_en = !kfd_dbg_has_cwsr_workaround(dev: q->device);
242	queue_input.skip_process_ctx_clear =
243	qpd->pqm->process->runtime_info.runtime_state == DEBUG_RUNTIME_STATE_ENABLED &&
244	(qpd->pqm->process->debug_trap_enabled ||
245	kfd_dbg_has_ttmps_always_setup(dev: q->device));
246
247	queue_type = convert_to_mes_queue_type(queue_type: q->properties.type);
248	if (queue_type < 0) {
249	dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n",
250	q->properties.type);
251	up_read(sem: &adev->reset_domain->sem);
252	return -EINVAL;
253	}
254	queue_input.queue_type = (uint32_t)queue_type;
255
256	queue_input.exclusively_scheduled = q->properties.is_gws;
257
258	amdgpu_mes_lock(mes: &adev->mes);
259	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
260	amdgpu_mes_unlock(mes: &adev->mes);
261	up_read(sem: &adev->reset_domain->sem);
262	if (r) {
263	dev_err(adev->dev, "failed to add hardware queue to MES, doorbell=0x%x\n",
264	q->properties.doorbell_off);
265	dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
266	kfd_hws_hang(dqm);
267	}
268
269	return r;
270	}
271
272	static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
273	struct qcm_process_device *qpd)
274	{
275	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
276	int r;
277	struct mes_remove_queue_input queue_input;
278
279	if (!dqm->sched_running || dqm->sched_halt)
280	return 0;
281	if (!down_read_trylock(sem: &adev->reset_domain->sem))
282	return -EIO;
283
284	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
285	queue_input.doorbell_offset = q->properties.doorbell_off;
286	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
287
288	amdgpu_mes_lock(mes: &adev->mes);
289	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
290	amdgpu_mes_unlock(mes: &adev->mes);
291	up_read(sem: &adev->reset_domain->sem);
292
293	if (r) {
294	dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n",
295	q->properties.doorbell_off);
296	dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
297	kfd_hws_hang(dqm);
298	}
299
300	return r;
301	}
302
303	static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
304	{
305	struct device_process_node *cur;
306	struct device *dev = dqm->dev->adev->dev;
307	struct qcm_process_device *qpd;
308	struct queue *q;
309	int retval = 0;
310
311	list_for_each_entry(cur, &dqm->queues, list) {
312	qpd = cur->qpd;
313	list_for_each_entry(q, &qpd->queues_list, list) {
314	if (q->properties.is_active) {
315	retval = remove_queue_mes(dqm, q, qpd);
316	if (retval) {
317	dev_err(dev, "%s: Failed to remove queue %d for dev %d",
318	__func__,
319	q->properties.queue_id,
320	dqm->dev->id);
321	return retval;
322	}
323	}
324	}
325	}
326
327	return retval;
328	}
329
330	static int add_all_kfd_queues_mes(struct device_queue_manager *dqm)
331	{
332	struct device_process_node *cur;
333	struct device *dev = dqm->dev->adev->dev;
334	struct qcm_process_device *qpd;
335	struct queue *q;
336	int retval = 0;
337
338	list_for_each_entry(cur, &dqm->queues, list) {
339	qpd = cur->qpd;
340	list_for_each_entry(q, &qpd->queues_list, list) {
341	if (!q->properties.is_active)
342	continue;
343	retval = add_queue_mes(dqm, q, qpd);
344	if (retval) {
345	dev_err(dev, "%s: Failed to add queue %d for dev %d",
346	__func__,
347	q->properties.queue_id,
348	dqm->dev->id);
349	return retval;
350	}
351	}
352	}
353
354	return retval;
355	}
356
357	static int suspend_all_queues_mes(struct device_queue_manager *dqm)
358	{
359	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
360	int r = 0;
361
362	if (!down_read_trylock(sem: &adev->reset_domain->sem))
363	return -EIO;
364
365	r = amdgpu_mes_suspend(adev);
366	up_read(sem: &adev->reset_domain->sem);
367
368	if (r) {
369	dev_err(adev->dev, "failed to suspend gangs from MES\n");
370	dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
371	kfd_hws_hang(dqm);
372	}
373
374	return r;
375	}
376
377	static int resume_all_queues_mes(struct device_queue_manager *dqm)
378	{
379	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
380	int r = 0;
381
382	if (!down_read_trylock(sem: &adev->reset_domain->sem))
383	return -EIO;
384
385	r = amdgpu_mes_resume(adev);
386	up_read(sem: &adev->reset_domain->sem);
387
388	if (r) {
389	dev_err(adev->dev, "failed to resume gangs from MES\n");
390	dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
391	kfd_hws_hang(dqm);
392	}
393
394	return r;
395	}
396
397	static void increment_queue_count(struct device_queue_manager *dqm,
398	struct qcm_process_device *qpd,
399	struct queue *q)
400	{
401	dqm->active_queue_count++;
402	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
403	q->properties.type == KFD_QUEUE_TYPE_DIQ)
404	dqm->active_cp_queue_count++;
405
406	if (q->properties.is_gws) {
407	dqm->gws_queue_count++;
408	qpd->mapped_gws_queue = true;
409	}
410	}
411
412	static void decrement_queue_count(struct device_queue_manager *dqm,
413	struct qcm_process_device *qpd,
414	struct queue *q)
415	{
416	dqm->active_queue_count--;
417	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
418	q->properties.type == KFD_QUEUE_TYPE_DIQ)
419	dqm->active_cp_queue_count--;
420
421	if (q->properties.is_gws) {
422	dqm->gws_queue_count--;
423	qpd->mapped_gws_queue = false;
424	}
425	}
426
427	/*
428	* Allocate a doorbell ID to this queue.
429	* If doorbell_id is passed in, make sure requested ID is valid then allocate it.
430	*/
431	static int allocate_doorbell(struct qcm_process_device *qpd,
432	struct queue *q,
433	uint32_t const *restore_id)
434	{
435	struct kfd_node *dev = qpd->dqm->dev;
436
437	if (!KFD_IS_SOC15(dev)) {
438	/* On pre-SOC15 chips we need to use the queue ID to
439	* preserve the user mode ABI.
440	*/
441
442	if (restore_id && *restore_id != q->properties.queue_id)
443	return -EINVAL;
444
445	q->doorbell_id = q->properties.queue_id;
446	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
447	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
448	/* For SDMA queues on SOC15 with 8-byte doorbell, use static
449	* doorbell assignments based on the engine and queue id.
450	* The doobell index distance between RLC (2*i) and (2*i+1)
451	* for a SDMA engine is 512.
452	*/
453
454	uint32_t *idx_offset = dev->kfd->shared_resources.sdma_doorbell_idx;
455
456	/*
457	* q->properties.sdma_engine_id corresponds to the virtual
458	* sdma engine number. However, for doorbell allocation,
459	* we need the physical sdma engine id in order to get the
460	* correct doorbell offset.
461	*/
462	uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id *
463	get_num_all_sdma_engines(dqm: qpd->dqm) +
464	q->properties.sdma_engine_id]
465	+ (q->properties.sdma_queue_id & 1)
466	* KFD_QUEUE_DOORBELL_MIRROR_OFFSET
467	+ (q->properties.sdma_queue_id >> 1);
468
469	if (restore_id && *restore_id != valid_id)
470	return -EINVAL;
471	q->doorbell_id = valid_id;
472	} else {
473	/* For CP queues on SOC15 */
474	if (restore_id) {
475	/* make sure that ID is free */
476	if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))
477	return -EINVAL;
478
479	q->doorbell_id = *restore_id;
480	} else {
481	/* or reserve a free doorbell ID */
482	unsigned int found;
483
484	found = find_first_zero_bit(addr: qpd->doorbell_bitmap,
485	KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
486	if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
487	pr_debug("No doorbells available");
488	return -EBUSY;
489	}
490	set_bit(nr: found, addr: qpd->doorbell_bitmap);
491	q->doorbell_id = found;
492	}
493	}
494
495	q->properties.doorbell_off = amdgpu_doorbell_index_on_bar(adev: dev->adev,
496	db_bo: qpd->proc_doorbells,
497	doorbell_index: q->doorbell_id,
498	db_size: dev->kfd->device_info.doorbell_size);
499	return 0;
500	}
501
502	static void deallocate_doorbell(struct qcm_process_device *qpd,
503	struct queue *q)
504	{
505	unsigned int old;
506	struct kfd_node *dev = qpd->dqm->dev;
507
508	if (!KFD_IS_SOC15(dev) ||
509	q->properties.type == KFD_QUEUE_TYPE_SDMA ||
510	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
511	return;
512
513	old = test_and_clear_bit(nr: q->doorbell_id, addr: qpd->doorbell_bitmap);
514	WARN_ON(!old);
515	}
516
517	static void program_trap_handler_settings(struct device_queue_manager *dqm,
518	struct qcm_process_device *qpd)
519	{
520	uint32_t xcc_mask = dqm->dev->xcc_mask;
521	int xcc_id;
522
523	if (dqm->dev->kfd2kgd->program_trap_handler_settings)
524	for_each_inst(xcc_id, xcc_mask)
525	dqm->dev->kfd2kgd->program_trap_handler_settings(
526	dqm->dev->adev, qpd->vmid, qpd->tba_addr,
527	qpd->tma_addr, xcc_id);
528	}
529
530	static int allocate_vmid(struct device_queue_manager *dqm,
531	struct qcm_process_device *qpd,
532	struct queue *q)
533	{
534	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
535	struct device *dev = dqm->dev->adev->dev;
536	int allocated_vmid = -1, i;
537
538	for (i = dqm->dev->vm_info.first_vmid_kfd;
539	i <= dqm->dev->vm_info.last_vmid_kfd; i++) {
540	if (!dqm->vmid_pasid[i]) {
541	allocated_vmid = i;
542	break;
543	}
544	}
545
546	if (allocated_vmid < 0) {
547	dev_err(dev, "no more vmid to allocate\n");
548	return -ENOSPC;
549	}
550
551	pr_debug("vmid allocated: %d\n", allocated_vmid);
552
553	dqm->vmid_pasid[allocated_vmid] = pdd->pasid;
554
555	set_pasid_vmid_mapping(dqm, pasid: pdd->pasid, vmid: allocated_vmid);
556
557	qpd->vmid = allocated_vmid;
558	q->properties.vmid = allocated_vmid;
559
560	program_sh_mem_settings(dqm, qpd);
561
562	if (KFD_IS_SOC15(dqm->dev) && dqm->dev->kfd->cwsr_enabled)
563	program_trap_handler_settings(dqm, qpd);
564
565	/* qpd->page_table_base is set earlier when register_process()
566	* is called, i.e. when the first queue is created.
567	*/
568	dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev,
569	qpd->vmid,
570	qpd->page_table_base);
571	/* invalidate the VM context after pasid and vmid mapping is set up */
572	kfd_flush_tlb(qpd_to_pdd(qpd), type: TLB_FLUSH_LEGACY);
573
574	if (dqm->dev->kfd2kgd->set_scratch_backing_va)
575	dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
576	qpd->sh_hidden_private_base, qpd->vmid);
577
578	return 0;
579	}
580
581	static int flush_texture_cache_nocpsch(struct kfd_node *kdev,
582	struct qcm_process_device *qpd)
583	{
584	const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf;
585	int ret;
586
587	if (!qpd->ib_kaddr)
588	return -ENOMEM;
589
590	ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
591	if (ret)
592	return ret;
593
594	return amdgpu_amdkfd_submit_ib(adev: kdev->adev, engine: KGD_ENGINE_MEC1, vmid: qpd->vmid,
595	gpu_addr: qpd->ib_base, ib_cmd: (uint32_t *)qpd->ib_kaddr,
596	ib_len: pmf->release_mem_size / sizeof(uint32_t));
597	}
598
599	static void deallocate_vmid(struct device_queue_manager *dqm,
600	struct qcm_process_device *qpd,
601	struct queue *q)
602	{
603	struct device *dev = dqm->dev->adev->dev;
604
605	/* On GFX v7, CP doesn't flush TC at dequeue */
606	if (q->device->adev->asic_type == CHIP_HAWAII)
607	if (flush_texture_cache_nocpsch(kdev: q->device, qpd))
608	dev_err(dev, "Failed to flush TC\n");
609
610	kfd_flush_tlb(qpd_to_pdd(qpd), type: TLB_FLUSH_LEGACY);
611
612	/* Release the vmid mapping */
613	set_pasid_vmid_mapping(dqm, pasid: 0, vmid: qpd->vmid);
614	dqm->vmid_pasid[qpd->vmid] = 0;
615
616	qpd->vmid = 0;
617	q->properties.vmid = 0;
618	}
619
620	static int create_queue_nocpsch(struct device_queue_manager *dqm,
621	struct queue *q,
622	struct qcm_process_device *qpd,
623	const struct kfd_criu_queue_priv_data *qd,
624	const void *restore_mqd, const void *restore_ctl_stack)
625	{
626	struct mqd_manager *mqd_mgr;
627	int retval;
628
629	dqm_lock(dqm);
630
631	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
632	pr_warn("Can't create new usermode queue because %d queues were already created\n",
633	dqm->total_queue_count);
634	retval = -EPERM;
635	goto out_unlock;
636	}
637
638	if (list_empty(head: &qpd->queues_list)) {
639	retval = allocate_vmid(dqm, qpd, q);
640	if (retval)
641	goto out_unlock;
642	}
643	q->properties.vmid = qpd->vmid;
644	/*
645	* Eviction state logic: mark all queues as evicted, even ones
646	* not currently active. Restoring inactive queues later only
647	* updates the is_evicted flag but is a no-op otherwise.
648	*/
649	q->properties.is_evicted = !!qpd->evicted;
650
651	q->properties.tba_addr = qpd->tba_addr;
652	q->properties.tma_addr = qpd->tma_addr;
653
654	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
655	type: q->properties.type)];
656	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
657	retval = allocate_hqd(dqm, q);
658	if (retval)
659	goto deallocate_vmid;
660	pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
661	q->pipe, q->queue);
662	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
663	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
664	retval = allocate_sdma_queue(dqm, q, restore_sdma_id: qd ? &qd->sdma_id : NULL);
665	if (retval)
666	goto deallocate_vmid;
667	dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
668	}
669
670	retval = allocate_doorbell(qpd, q, restore_id: qd ? &qd->doorbell_id : NULL);
671	if (retval)
672	goto out_deallocate_hqd;
673
674	/* Temporarily release dqm lock to avoid a circular lock dependency */
675	dqm_unlock(dqm);
676	q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
677	dqm_lock(dqm);
678
679	if (!q->mqd_mem_obj) {
680	retval = -ENOMEM;
681	goto out_deallocate_doorbell;
682	}
683
684	if (qd)
685	mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
686	&q->properties, restore_mqd, restore_ctl_stack,
687	qd->ctl_stack_size);
688	else
689	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
690	&q->gart_mqd_addr, &q->properties);
691
692	if (q->properties.is_active) {
693	if (!dqm->sched_running) {
694	WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
695	goto add_queue_to_list;
696	}
697
698	if (WARN(q->process->mm != current->mm,
699	"should only run in user thread"))
700	retval = -EFAULT;
701	else
702	retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
703	q->queue, &q->properties, current->mm);
704	if (retval)
705	goto out_free_mqd;
706	}
707
708	add_queue_to_list:
709	list_add(new: &q->list, head: &qpd->queues_list);
710	qpd->queue_count++;
711	if (q->properties.is_active)
712	increment_queue_count(dqm, qpd, q);
713
714	/*
715	* Unconditionally increment this counter, regardless of the queue's
716	* type or whether the queue is active.
717	*/
718	dqm->total_queue_count++;
719	pr_debug("Total of %d queues are accountable so far\n",
720	dqm->total_queue_count);
721	goto out_unlock;
722
723	out_free_mqd:
724	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
725	out_deallocate_doorbell:
726	deallocate_doorbell(qpd, q);
727	out_deallocate_hqd:
728	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
729	deallocate_hqd(dqm, q);
730	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
731	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
732	deallocate_sdma_queue(dqm, q);
733	deallocate_vmid:
734	if (list_empty(head: &qpd->queues_list))
735	deallocate_vmid(dqm, qpd, q);
736	out_unlock:
737	dqm_unlock(dqm);
738	return retval;
739	}
740
741	static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
742	{
743	bool set;
744	int pipe, bit, i;
745
746	set = false;
747
748	for (pipe = dqm->next_pipe_to_allocate, i = 0;
749	i < get_pipes_per_mec(dqm);
750	pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
751
752	if (!is_pipe_enabled(dqm, mec: 0, pipe))
753	continue;
754
755	if (dqm->allocated_queues[pipe] != 0) {
756	bit = ffs(dqm->allocated_queues[pipe]) - 1;
757	dqm->allocated_queues[pipe] &= ~(1 << bit);
758	q->pipe = pipe;
759	q->queue = bit;
760	set = true;
761	break;
762	}
763	}
764
765	if (!set)
766	return -EBUSY;
767
768	pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
769	/* horizontal hqd allocation */
770	dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
771
772	return 0;
773	}
774
775	static inline void deallocate_hqd(struct device_queue_manager *dqm,
776	struct queue *q)
777	{
778	dqm->allocated_queues[q->pipe] |= (1 << q->queue);
779	}
780
781	#define SQ_IND_CMD_CMD_KILL 0x00000003
782	#define SQ_IND_CMD_MODE_BROADCAST 0x00000001
783
784	static int dbgdev_wave_reset_wavefronts(struct kfd_node *dev, struct kfd_process *p)
785	{
786	int status = 0;
787	unsigned int vmid;
788	uint16_t queried_pasid;
789	union SQ_CMD_BITS reg_sq_cmd;
790	union GRBM_GFX_INDEX_BITS reg_gfx_index;
791	struct kfd_process_device *pdd;
792	int first_vmid_to_scan = dev->vm_info.first_vmid_kfd;
793	int last_vmid_to_scan = dev->vm_info.last_vmid_kfd;
794	uint32_t xcc_mask = dev->xcc_mask;
795	int xcc_id;
796
797	reg_sq_cmd.u32All = 0;
798	reg_gfx_index.u32All = 0;
799
800	pr_debug("Killing all process wavefronts\n");
801
802	if (!dev->kfd2kgd->get_atc_vmid_pasid_mapping_info) {
803	dev_err(dev->adev->dev, "no vmid pasid mapping supported\n");
804	return -EOPNOTSUPP;
805	}
806
807	/* taking the VMID for that process on the safe way using PDD */
808	pdd = kfd_get_process_device_data(dev, p);
809	if (!pdd)
810	return -EFAULT;
811
812	/* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
813	* ATC_VMID15_PASID_MAPPING
814	* to check which VMID the current process is mapped to.
815	*/
816
817	for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
818	status = dev->kfd2kgd->get_atc_vmid_pasid_mapping_info
819	(dev->adev, vmid, &queried_pasid);
820
821	if (status && queried_pasid == pdd->pasid) {
822	pr_debug("Killing wave fronts of vmid %d and process pid %d\n",
823	vmid, p->lead_thread->pid);
824	break;
825	}
826	}
827
828	if (vmid > last_vmid_to_scan) {
829	dev_err(dev->adev->dev, "Didn't find vmid for process pid %d\n",
830	p->lead_thread->pid);
831	return -EFAULT;
832	}
833
834	reg_gfx_index.bits.sh_broadcast_writes = 1;
835	reg_gfx_index.bits.se_broadcast_writes = 1;
836	reg_gfx_index.bits.instance_broadcast_writes = 1;
837	reg_sq_cmd.bits.mode = SQ_IND_CMD_MODE_BROADCAST;
838	reg_sq_cmd.bits.cmd = SQ_IND_CMD_CMD_KILL;
839	reg_sq_cmd.bits.vm_id = vmid;
840
841	for_each_inst(xcc_id, xcc_mask)
842	dev->kfd2kgd->wave_control_execute(
843	dev->adev, reg_gfx_index.u32All,
844	reg_sq_cmd.u32All, xcc_id);
845
846	return 0;
847	}
848
849	/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
850	* to avoid asynchronized access
851	*/
852	static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
853	struct qcm_process_device *qpd,
854	struct queue *q)
855	{
856	int retval;
857	struct mqd_manager *mqd_mgr;
858
859	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
860	type: q->properties.type)];
861
862	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
863	deallocate_hqd(dqm, q);
864	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
865	deallocate_sdma_queue(dqm, q);
866	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
867	deallocate_sdma_queue(dqm, q);
868	else {
869	pr_debug("q->properties.type %d is invalid\n",
870	q->properties.type);
871	return -EINVAL;
872	}
873	dqm->total_queue_count--;
874
875	deallocate_doorbell(qpd, q);
876
877	if (!dqm->sched_running) {
878	WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
879	return 0;
880	}
881
882	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
883	KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
884	KFD_UNMAP_LATENCY_MS,
885	q->pipe, q->queue);
886	if (retval == -ETIME)
887	qpd->reset_wavefronts = true;
888
889	list_del(entry: &q->list);
890	if (list_empty(head: &qpd->queues_list)) {
891	if (qpd->reset_wavefronts) {
892	pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
893	dqm->dev);
894	/* dbgdev_wave_reset_wavefronts has to be called before
895	* deallocate_vmid(), i.e. when vmid is still in use.
896	*/
897	dbgdev_wave_reset_wavefronts(dev: dqm->dev,
898	p: qpd->pqm->process);
899	qpd->reset_wavefronts = false;
900	}
901
902	deallocate_vmid(dqm, qpd, q);
903	}
904	qpd->queue_count--;
905	if (q->properties.is_active)
906	decrement_queue_count(dqm, qpd, q);
907
908	return retval;
909	}
910
911	static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
912	struct qcm_process_device *qpd,
913	struct queue *q)
914	{
915	int retval;
916	uint64_t sdma_val = 0;
917	struct device *dev = dqm->dev->adev->dev;
918	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
919	struct mqd_manager *mqd_mgr =
920	dqm->mqd_mgrs[get_mqd_type_from_queue_type(type: q->properties.type)];
921
922	/* Get the SDMA queue stats */
923	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
924	(q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
925	retval = read_sdma_queue_counter(q_rptr: (uint64_t __user *)q->properties.read_ptr,
926	val: &sdma_val);
927	if (retval)
928	dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
929	q->properties.queue_id);
930	}
931
932	dqm_lock(dqm);
933	retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
934	if (!retval)
935	pdd->sdma_past_activity_counter += sdma_val;
936	dqm_unlock(dqm);
937
938	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
939
940	return retval;
941	}
942
943	static int update_queue(struct device_queue_manager *dqm, struct queue *q,
944	struct mqd_update_info *minfo)
945	{
946	int retval = 0;
947	struct device *dev = dqm->dev->adev->dev;
948	struct mqd_manager *mqd_mgr;
949	struct kfd_process_device *pdd;
950	bool prev_active = false;
951
952	dqm_lock(dqm);
953	pdd = kfd_get_process_device_data(dev: q->device, p: q->process);
954	if (!pdd) {
955	retval = -ENODEV;
956	goto out_unlock;
957	}
958	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
959	type: q->properties.type)];
960
961	/* Save previous activity state for counters */
962	prev_active = q->properties.is_active;
963
964	/* Make sure the queue is unmapped before updating the MQD */
965	if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
966	if (!dqm->dev->kfd->shared_resources.enable_mes)
967	retval = unmap_queues_cpsch(dqm,
968	filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0, USE_DEFAULT_GRACE_PERIOD, reset: false);
969	else if (prev_active)
970	retval = remove_queue_mes(dqm, q, qpd: &pdd->qpd);
971
972	/* queue is reset so inaccessable */
973	if (pdd->has_reset_queue) {
974	retval = -EACCES;
975	goto out_unlock;
976	}
977
978	if (retval) {
979	dev_err(dev, "unmap queue failed\n");
980	goto out_unlock;
981	}
982	} else if (prev_active &&
983	(q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
984	q->properties.type == KFD_QUEUE_TYPE_SDMA ||
985	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
986
987	if (!dqm->sched_running) {
988	WARN_ONCE(1, "Update non-HWS queue while stopped\n");
989	goto out_unlock;
990	}
991
992	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
993	(dqm->dev->kfd->cwsr_enabled ?
994	KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
995	KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
996	KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
997	if (retval) {
998	dev_err(dev, "destroy mqd failed\n");
999	goto out_unlock;
1000	}
1001	}
1002
1003	mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo);
1004
1005	/*
1006	* check active state vs. the previous state and modify
1007	* counter accordingly. map_queues_cpsch uses the
1008	* dqm->active_queue_count to determine whether a new runlist must be
1009	* uploaded.
1010	*/
1011	if (q->properties.is_active && !prev_active) {
1012	increment_queue_count(dqm, qpd: &pdd->qpd, q);
1013	} else if (!q->properties.is_active && prev_active) {
1014	decrement_queue_count(dqm, qpd: &pdd->qpd, q);
1015	} else if (q->gws && !q->properties.is_gws) {
1016	if (q->properties.is_active) {
1017	dqm->gws_queue_count++;
1018	pdd->qpd.mapped_gws_queue = true;
1019	}
1020	q->properties.is_gws = true;
1021	} else if (!q->gws && q->properties.is_gws) {
1022	if (q->properties.is_active) {
1023	dqm->gws_queue_count--;
1024	pdd->qpd.mapped_gws_queue = false;
1025	}
1026	q->properties.is_gws = false;
1027	}
1028
1029	if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
1030	if (!dqm->dev->kfd->shared_resources.enable_mes)
1031	retval = map_queues_cpsch(dqm);
1032	else if (q->properties.is_active)
1033	retval = add_queue_mes(dqm, q, qpd: &pdd->qpd);
1034	} else if (q->properties.is_active &&
1035	(q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1036	q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1037	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1038	if (WARN(q->process->mm != current->mm,
1039	"should only run in user thread"))
1040	retval = -EFAULT;
1041	else
1042	retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
1043	q->pipe, q->queue,
1044	&q->properties, current->mm);
1045	}
1046
1047	out_unlock:
1048	dqm_unlock(dqm);
1049	return retval;
1050	}
1051
1052	/* suspend_single_queue does not lock the dqm like the
1053	* evict_process_queues_cpsch or evict_process_queues_nocpsch. You should
1054	* lock the dqm before calling, and unlock after calling.
1055	*
1056	* The reason we don't lock the dqm is because this function may be
1057	* called on multiple queues in a loop, so rather than locking/unlocking
1058	* multiple times, we will just keep the dqm locked for all of the calls.
1059	*/
1060	static int suspend_single_queue(struct device_queue_manager *dqm,
1061	struct kfd_process_device *pdd,
1062	struct queue *q)
1063	{
1064	bool is_new;
1065
1066	if (q->properties.is_suspended)
1067	return 0;
1068
1069	pr_debug("Suspending process pid %d queue [%i]\n",
1070	pdd->process->lead_thread->pid,
1071	q->properties.queue_id);
1072
1073	is_new = q->properties.exception_status & KFD_EC_MASK(EC_QUEUE_NEW);
1074
1075	if (is_new || q->properties.is_being_destroyed) {
1076	pr_debug("Suspend: skip %s queue id %i\n",
1077	is_new ? "new" : "destroyed",
1078	q->properties.queue_id);
1079	return -EBUSY;
1080	}
1081
1082	q->properties.is_suspended = true;
1083	if (q->properties.is_active) {
1084	if (dqm->dev->kfd->shared_resources.enable_mes) {
1085	int r = remove_queue_mes(dqm, q, qpd: &pdd->qpd);
1086
1087	if (r)
1088	return r;
1089	}
1090
1091	decrement_queue_count(dqm, qpd: &pdd->qpd, q);
1092	q->properties.is_active = false;
1093	}
1094
1095	return 0;
1096	}
1097
1098	/* resume_single_queue does not lock the dqm like the functions
1099	* restore_process_queues_cpsch or restore_process_queues_nocpsch. You should
1100	* lock the dqm before calling, and unlock after calling.
1101	*
1102	* The reason we don't lock the dqm is because this function may be
1103	* called on multiple queues in a loop, so rather than locking/unlocking
1104	* multiple times, we will just keep the dqm locked for all of the calls.
1105	*/
1106	static int resume_single_queue(struct device_queue_manager *dqm,
1107	struct qcm_process_device *qpd,
1108	struct queue *q)
1109	{
1110	struct kfd_process_device *pdd;
1111
1112	if (!q->properties.is_suspended)
1113	return 0;
1114
1115	pdd = qpd_to_pdd(qpd);
1116
1117	pr_debug("Restoring from suspend process pid %d queue [%i]\n",
1118	pdd->process->lead_thread->pid,
1119	q->properties.queue_id);
1120
1121	q->properties.is_suspended = false;
1122
1123	if (QUEUE_IS_ACTIVE(q->properties)) {
1124	if (dqm->dev->kfd->shared_resources.enable_mes) {
1125	int r = add_queue_mes(dqm, q, qpd: &pdd->qpd);
1126
1127	if (r)
1128	return r;
1129	}
1130
1131	q->properties.is_active = true;
1132	increment_queue_count(dqm, qpd, q);
1133	}
1134
1135	return 0;
1136	}
1137
1138	static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
1139	struct qcm_process_device *qpd)
1140	{
1141	struct queue *q;
1142	struct mqd_manager *mqd_mgr;
1143	struct kfd_process_device *pdd;
1144	int retval, ret = 0;
1145
1146	dqm_lock(dqm);
1147	if (qpd->evicted++ > 0) /* already evicted, do nothing */
1148	goto out;
1149
1150	pdd = qpd_to_pdd(qpd);
1151	pr_debug_ratelimited("Evicting process pid %d queues\n",
1152	pdd->process->lead_thread->pid);
1153
1154	pdd->last_evict_timestamp = get_jiffies_64();
1155	/* Mark all queues as evicted. Deactivate all active queues on
1156	* the qpd.
1157	*/
1158	list_for_each_entry(q, &qpd->queues_list, list) {
1159	q->properties.is_evicted = true;
1160	if (!q->properties.is_active)
1161	continue;
1162
1163	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1164	type: q->properties.type)];
1165	q->properties.is_active = false;
1166	decrement_queue_count(dqm, qpd, q);
1167
1168	if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
1169	continue;
1170
1171	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
1172	(dqm->dev->kfd->cwsr_enabled ?
1173	KFD_PREEMPT_TYPE_WAVEFRONT_SAVE :
1174	KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
1175	KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
1176	if (retval && !ret)
1177	/* Return the first error, but keep going to
1178	* maintain a consistent eviction state
1179	*/
1180	ret = retval;
1181	}
1182
1183	out:
1184	dqm_unlock(dqm);
1185	return ret;
1186	}
1187
1188	static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
1189	struct qcm_process_device *qpd)
1190	{
1191	struct queue *q;
1192	struct device *dev = dqm->dev->adev->dev;
1193	struct kfd_process_device *pdd;
1194	int retval = 0;
1195
1196	dqm_lock(dqm);
1197	if (qpd->evicted++ > 0) /* already evicted, do nothing */
1198	goto out;
1199
1200	pdd = qpd_to_pdd(qpd);
1201
1202	/* The debugger creates processes that temporarily have not acquired
1203	* all VMs for all devices and has no VMs itself.
1204	* Skip queue eviction on process eviction.
1205	*/
1206	if (!pdd->drm_priv)
1207	goto out;
1208
1209	pr_debug_ratelimited("Evicting process pid %d queues\n",
1210	pdd->process->lead_thread->pid);
1211
1212	if (dqm->dev->kfd->shared_resources.enable_mes)
1213	pdd->last_evict_timestamp = get_jiffies_64();
1214
1215	/* Mark all queues as evicted. Deactivate all active queues on
1216	* the qpd.
1217	*/
1218	list_for_each_entry(q, &qpd->queues_list, list) {
1219	q->properties.is_evicted = true;
1220	if (!q->properties.is_active)
1221	continue;
1222
1223	q->properties.is_active = false;
1224	decrement_queue_count(dqm, qpd, q);
1225
1226	if (dqm->dev->kfd->shared_resources.enable_mes) {
1227	retval = remove_queue_mes(dqm, q, qpd);
1228	if (retval) {
1229	dev_err(dev, "Failed to evict queue %d\n",
1230	q->properties.queue_id);
1231	goto out;
1232	}
1233	}
1234	}
1235
1236	if (!dqm->dev->kfd->shared_resources.enable_mes) {
1237	pdd->last_evict_timestamp = get_jiffies_64();
1238	retval = execute_queues_cpsch(dqm,
1239	filter: qpd->is_debug ?
1240	KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
1241	KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0,
1242	USE_DEFAULT_GRACE_PERIOD);
1243	}
1244
1245	out:
1246	dqm_unlock(dqm);
1247	return retval;
1248	}
1249
1250	static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
1251	struct qcm_process_device *qpd)
1252	{
1253	struct mm_struct *mm = NULL;
1254	struct queue *q;
1255	struct mqd_manager *mqd_mgr;
1256	struct kfd_process_device *pdd;
1257	uint64_t pd_base;
1258	uint64_t eviction_duration;
1259	int retval, ret = 0;
1260
1261	pdd = qpd_to_pdd(qpd);
1262	/* Retrieve PD base */
1263	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(drm_priv: pdd->drm_priv);
1264
1265	dqm_lock(dqm);
1266	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1267	goto out;
1268	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1269	qpd->evicted--;
1270	goto out;
1271	}
1272
1273	pr_debug_ratelimited("Restoring process pid %d queues\n",
1274	pdd->process->lead_thread->pid);
1275
1276	/* Update PD Base in QPD */
1277	qpd->page_table_base = pd_base;
1278	pr_debug("Updated PD address to 0x%llx\n", pd_base);
1279
1280	if (!list_empty(head: &qpd->queues_list)) {
1281	dqm->dev->kfd2kgd->set_vm_context_page_table_base(
1282	dqm->dev->adev,
1283	qpd->vmid,
1284	qpd->page_table_base);
1285	kfd_flush_tlb(pdd, type: TLB_FLUSH_LEGACY);
1286	}
1287
1288	/* Take a safe reference to the mm_struct, which may otherwise
1289	* disappear even while the kfd_process is still referenced.
1290	*/
1291	mm = get_task_mm(task: pdd->process->lead_thread);
1292	if (!mm) {
1293	ret = -EFAULT;
1294	goto out;
1295	}
1296
1297	/* Remove the eviction flags. Activate queues that are not
1298	* inactive for other reasons.
1299	*/
1300	list_for_each_entry(q, &qpd->queues_list, list) {
1301	q->properties.is_evicted = false;
1302	if (!QUEUE_IS_ACTIVE(q->properties))
1303	continue;
1304
1305	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1306	type: q->properties.type)];
1307	q->properties.is_active = true;
1308	increment_queue_count(dqm, qpd, q);
1309
1310	if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
1311	continue;
1312
1313	retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
1314	q->queue, &q->properties, mm);
1315	if (retval && !ret)
1316	/* Return the first error, but keep going to
1317	* maintain a consistent eviction state
1318	*/
1319	ret = retval;
1320	}
1321	qpd->evicted = 0;
1322	eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1323	atomic64_add(i: eviction_duration, v: &pdd->evict_duration_counter);
1324	out:
1325	if (mm)
1326	mmput(mm);
1327	dqm_unlock(dqm);
1328	return ret;
1329	}
1330
1331	static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
1332	struct qcm_process_device *qpd)
1333	{
1334	struct queue *q;
1335	struct device *dev = dqm->dev->adev->dev;
1336	struct kfd_process_device *pdd;
1337	uint64_t eviction_duration;
1338	int retval = 0;
1339
1340	pdd = qpd_to_pdd(qpd);
1341
1342	dqm_lock(dqm);
1343	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
1344	goto out;
1345	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
1346	qpd->evicted--;
1347	goto out;
1348	}
1349
1350	/* The debugger creates processes that temporarily have not acquired
1351	* all VMs for all devices and has no VMs itself.
1352	* Skip queue restore on process restore.
1353	*/
1354	if (!pdd->drm_priv)
1355	goto vm_not_acquired;
1356
1357	pr_debug_ratelimited("Restoring process pid %d queues\n",
1358	pdd->process->lead_thread->pid);
1359
1360	/* Update PD Base in QPD */
1361	qpd->page_table_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(drm_priv: pdd->drm_priv);
1362	pr_debug("Updated PD address to 0x%llx\n", qpd->page_table_base);
1363
1364	/* activate all active queues on the qpd */
1365	list_for_each_entry(q, &qpd->queues_list, list) {
1366	q->properties.is_evicted = false;
1367	if (!QUEUE_IS_ACTIVE(q->properties))
1368	continue;
1369
1370	q->properties.is_active = true;
1371	increment_queue_count(dqm, qpd: &pdd->qpd, q);
1372
1373	if (dqm->dev->kfd->shared_resources.enable_mes) {
1374	retval = add_queue_mes(dqm, q, qpd);
1375	if (retval) {
1376	dev_err(dev, "Failed to restore queue %d\n",
1377	q->properties.queue_id);
1378	goto out;
1379	}
1380	}
1381	}
1382	if (!dqm->dev->kfd->shared_resources.enable_mes)
1383	retval = execute_queues_cpsch(dqm,
1384	filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
1385	eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
1386	atomic64_add(i: eviction_duration, v: &pdd->evict_duration_counter);
1387	vm_not_acquired:
1388	qpd->evicted = 0;
1389	out:
1390	dqm_unlock(dqm);
1391	return retval;
1392	}
1393
1394	static int register_process(struct device_queue_manager *dqm,
1395	struct qcm_process_device *qpd)
1396	{
1397	struct device_process_node *n;
1398	struct kfd_process_device *pdd;
1399	uint64_t pd_base;
1400	int retval;
1401
1402	n = kzalloc(sizeof(*n), GFP_KERNEL);
1403	if (!n)
1404	return -ENOMEM;
1405
1406	n->qpd = qpd;
1407
1408	pdd = qpd_to_pdd(qpd);
1409	/* Retrieve PD base */
1410	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(drm_priv: pdd->drm_priv);
1411
1412	dqm_lock(dqm);
1413	list_add(new: &n->list, head: &dqm->queues);
1414
1415	/* Update PD Base in QPD */
1416	qpd->page_table_base = pd_base;
1417	pr_debug("Updated PD address to 0x%llx\n", pd_base);
1418
1419	retval = dqm->asic_ops.update_qpd(dqm, qpd);
1420
1421	dqm->processes_count++;
1422
1423	dqm_unlock(dqm);
1424
1425	/* Outside the DQM lock because under the DQM lock we can't do
1426	* reclaim or take other locks that others hold while reclaiming.
1427	*/
1428	kfd_inc_compute_active(dev: dqm->dev);
1429
1430	return retval;
1431	}
1432
1433	static int unregister_process(struct device_queue_manager *dqm,
1434	struct qcm_process_device *qpd)
1435	{
1436	int retval;
1437	struct device_process_node *cur, *next;
1438
1439	pr_debug("qpd->queues_list is %s\n",
1440	list_empty(&qpd->queues_list) ? "empty" : "not empty");
1441
1442	retval = 0;
1443	dqm_lock(dqm);
1444
1445	list_for_each_entry_safe(cur, next, &dqm->queues, list) {
1446	if (qpd == cur->qpd) {
1447	list_del(entry: &cur->list);
1448	kfree(objp: cur);
1449	dqm->processes_count--;
1450	goto out;
1451	}
1452	}
1453	/* qpd not found in dqm list */
1454	retval = 1;
1455	out:
1456	dqm_unlock(dqm);
1457
1458	/* Outside the DQM lock because under the DQM lock we can't do
1459	* reclaim or take other locks that others hold while reclaiming.
1460	*/
1461	if (!retval)
1462	kfd_dec_compute_active(dev: dqm->dev);
1463
1464	return retval;
1465	}
1466
1467	static int
1468	set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
1469	unsigned int vmid)
1470	{
1471	uint32_t xcc_mask = dqm->dev->xcc_mask;
1472	int xcc_id, ret;
1473
1474	for_each_inst(xcc_id, xcc_mask) {
1475	ret = dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
1476	dqm->dev->adev, pasid, vmid, xcc_id);
1477	if (ret)
1478	break;
1479	}
1480
1481	return ret;
1482	}
1483
1484	static void init_interrupts(struct device_queue_manager *dqm)
1485	{
1486	uint32_t xcc_mask = dqm->dev->xcc_mask;
1487	unsigned int i, xcc_id;
1488
1489	for_each_inst(xcc_id, xcc_mask) {
1490	for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++) {
1491	if (is_pipe_enabled(dqm, mec: 0, pipe: i)) {
1492	dqm->dev->kfd2kgd->init_interrupts(
1493	dqm->dev->adev, i, xcc_id);
1494	}
1495	}
1496	}
1497	}
1498
1499	static int initialize_nocpsch(struct device_queue_manager *dqm)
1500	{
1501	int pipe, queue;
1502
1503	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1504
1505	dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
1506	sizeof(unsigned int), GFP_KERNEL);
1507	if (!dqm->allocated_queues)
1508	return -ENOMEM;
1509
1510	mutex_init(&dqm->lock_hidden);
1511	INIT_LIST_HEAD(list: &dqm->queues);
1512	dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;
1513	dqm->active_cp_queue_count = 0;
1514	dqm->gws_queue_count = 0;
1515
1516	for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1517	int pipe_offset = pipe * get_queues_per_pipe(dqm);
1518
1519	for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
1520	if (test_bit(pipe_offset + queue,
1521	dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1522	dqm->allocated_queues[pipe] |= 1 << queue;
1523	}
1524
1525	memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
1526
1527	init_sdma_bitmaps(dqm);
1528
1529	return 0;
1530	}
1531
1532	static void uninitialize(struct device_queue_manager *dqm)
1533	{
1534	int i;
1535
1536	WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);
1537
1538	kfree(objp: dqm->allocated_queues);
1539	for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
1540	kfree(objp: dqm->mqd_mgrs[i]);
1541	mutex_destroy(lock: &dqm->lock_hidden);
1542	}
1543
1544	static int start_nocpsch(struct device_queue_manager *dqm)
1545	{
1546	int r = 0;
1547
1548	pr_info("SW scheduler is used");
1549	init_interrupts(dqm);
1550
1551	if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1552	r = pm_init(pm: &dqm->packet_mgr, dqm);
1553	if (!r)
1554	dqm->sched_running = true;
1555
1556	return r;
1557	}
1558
1559	static int stop_nocpsch(struct device_queue_manager *dqm)
1560	{
1561	dqm_lock(dqm);
1562	if (!dqm->sched_running) {
1563	dqm_unlock(dqm);
1564	return 0;
1565	}
1566
1567	if (dqm->dev->adev->asic_type == CHIP_HAWAII)
1568	pm_uninit(pm: &dqm->packet_mgr);
1569	dqm->sched_running = false;
1570	dqm_unlock(dqm);
1571
1572	return 0;
1573	}
1574
1575	static int allocate_sdma_queue(struct device_queue_manager *dqm,
1576	struct queue *q, const uint32_t *restore_sdma_id)
1577	{
1578	struct device *dev = dqm->dev->adev->dev;
1579	int bit;
1580
1581	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1582	if (bitmap_empty(src: dqm->sdma_bitmap, nbits: get_num_sdma_queues(dqm))) {
1583	dev_warn(dev, "No more SDMA queue to allocate (%d total queues)\n",
1584	get_num_sdma_queues(dqm));
1585	return -ENOMEM;
1586	}
1587
1588	if (restore_sdma_id) {
1589	/* Re-use existing sdma_id */
1590	if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {
1591	dev_err(dev, "SDMA queue already in use\n");
1592	return -EBUSY;
1593	}
1594	clear_bit(nr: *restore_sdma_id, addr: dqm->sdma_bitmap);
1595	q->sdma_id = *restore_sdma_id;
1596	} else {
1597	/* Find first available sdma_id */
1598	bit = find_first_bit(addr: dqm->sdma_bitmap,
1599	size: get_num_sdma_queues(dqm));
1600	clear_bit(nr: bit, addr: dqm->sdma_bitmap);
1601	q->sdma_id = bit;
1602	}
1603
1604	q->properties.sdma_engine_id =
1605	q->sdma_id % kfd_get_num_sdma_engines(kdev: dqm->dev);
1606	q->properties.sdma_queue_id = q->sdma_id /
1607	kfd_get_num_sdma_engines(kdev: dqm->dev);
1608	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1609	if (bitmap_empty(src: dqm->xgmi_sdma_bitmap, nbits: get_num_xgmi_sdma_queues(dqm))) {
1610	dev_warn(dev, "No more XGMI SDMA queue to allocate (%d total queues)\n",
1611	get_num_xgmi_sdma_queues(dqm));
1612	return -ENOMEM;
1613	}
1614	if (restore_sdma_id) {
1615	/* Re-use existing sdma_id */
1616	if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {
1617	dev_err(dev, "SDMA queue already in use\n");
1618	return -EBUSY;
1619	}
1620	clear_bit(nr: *restore_sdma_id, addr: dqm->xgmi_sdma_bitmap);
1621	q->sdma_id = *restore_sdma_id;
1622	} else {
1623	bit = find_first_bit(addr: dqm->xgmi_sdma_bitmap,
1624	size: get_num_xgmi_sdma_queues(dqm));
1625	clear_bit(nr: bit, addr: dqm->xgmi_sdma_bitmap);
1626	q->sdma_id = bit;
1627	}
1628	/* sdma_engine_id is sdma id including
1629	* both PCIe-optimized SDMAs and XGMI-
1630	* optimized SDMAs. The calculation below
1631	* assumes the first N engines are always
1632	* PCIe-optimized ones
1633	*/
1634	q->properties.sdma_engine_id =
1635	kfd_get_num_sdma_engines(kdev: dqm->dev) +
1636	q->sdma_id % kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev);
1637	q->properties.sdma_queue_id = q->sdma_id /
1638	kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev);
1639	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
1640	int i, num_queues, num_engines, eng_offset = 0, start_engine;
1641	bool free_bit_found = false, is_xgmi = false;
1642
1643	if (q->properties.sdma_engine_id < kfd_get_num_sdma_engines(kdev: dqm->dev)) {
1644	num_queues = get_num_sdma_queues(dqm);
1645	num_engines = kfd_get_num_sdma_engines(kdev: dqm->dev);
1646	q->properties.type = KFD_QUEUE_TYPE_SDMA;
1647	} else {
1648	num_queues = get_num_xgmi_sdma_queues(dqm);
1649	num_engines = kfd_get_num_xgmi_sdma_engines(kdev: dqm->dev);
1650	eng_offset = kfd_get_num_sdma_engines(kdev: dqm->dev);
1651	q->properties.type = KFD_QUEUE_TYPE_SDMA_XGMI;
1652	is_xgmi = true;
1653	}
1654
1655	/* Scan available bit based on target engine ID. */
1656	start_engine = q->properties.sdma_engine_id - eng_offset;
1657	for (i = start_engine; i < num_queues; i += num_engines) {
1658
1659	if (!test_bit(i, is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap))
1660	continue;
1661
1662	clear_bit(nr: i, addr: is_xgmi ? dqm->xgmi_sdma_bitmap : dqm->sdma_bitmap);
1663	q->sdma_id = i;
1664	q->properties.sdma_queue_id = q->sdma_id / num_engines;
1665	free_bit_found = true;
1666	break;
1667	}
1668
1669	if (!free_bit_found) {
1670	dev_warn(dev, "No more SDMA queue to allocate for target ID %i (%d total queues)\n",
1671	q->properties.sdma_engine_id, num_queues);
1672	return -ENOMEM;
1673	}
1674	}
1675
1676	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
1677	pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
1678
1679	return 0;
1680	}
1681
1682	static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1683	struct queue *q)
1684	{
1685	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1686	if (q->sdma_id >= get_num_sdma_queues(dqm))
1687	return;
1688	set_bit(nr: q->sdma_id, addr: dqm->sdma_bitmap);
1689	} else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1690	if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
1691	return;
1692	set_bit(nr: q->sdma_id, addr: dqm->xgmi_sdma_bitmap);
1693	}
1694	}
1695
1696	/*
1697	* Device Queue Manager implementation for cp scheduler
1698	*/
1699
1700	static int set_sched_resources(struct device_queue_manager *dqm)
1701	{
1702	int i, mec;
1703	struct scheduling_resources res;
1704	struct device *dev = dqm->dev->adev->dev;
1705
1706	res.vmid_mask = dqm->dev->compute_vmid_bitmap;
1707
1708	res.queue_mask = 0;
1709	for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
1710	mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
1711	/ dqm->dev->kfd->shared_resources.num_pipe_per_mec;
1712
1713	if (!test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
1714	continue;
1715
1716	/* only acquire queues from the first MEC */
1717	if (mec > 0)
1718	continue;
1719
1720	/* This situation may be hit in the future if a new HW
1721	* generation exposes more than 64 queues. If so, the
1722	* definition of res.queue_mask needs updating
1723	*/
1724	if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
1725	dev_err(dev, "Invalid queue enabled by amdgpu: %d\n", i);
1726	break;
1727	}
1728
1729	res.queue_mask |= 1ull
1730	<< amdgpu_queue_mask_bit_to_set_resource_bit(
1731	adev: dqm->dev->adev, queue_bit: i);
1732	}
1733	res.gws_mask = ~0ull;
1734	res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
1735
1736	pr_debug("Scheduling resources:\n"
1737	"vmid mask: 0x%8X\n"
1738	"queue mask: 0x%8llX\n",
1739	res.vmid_mask, res.queue_mask);
1740
1741	return pm_send_set_resources(pm: &dqm->packet_mgr, res: &res);
1742	}
1743
1744	static int initialize_cpsch(struct device_queue_manager *dqm)
1745	{
1746	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1747
1748	mutex_init(&dqm->lock_hidden);
1749	INIT_LIST_HEAD(list: &dqm->queues);
1750	dqm->active_queue_count = dqm->processes_count = 0;
1751	dqm->active_cp_queue_count = 0;
1752	dqm->gws_queue_count = 0;
1753	dqm->active_runlist = false;
1754	dqm->trap_debug_vmid = 0;
1755
1756	init_sdma_bitmaps(dqm);
1757
1758	update_dqm_wait_times(dqm);
1759	return 0;
1760	}
1761
1762	/* halt_cpsch:
1763	* Unmap queues so the schedule doesn't continue remaining jobs in the queue.
1764	* Then set dqm->sched_halt so queues don't map to runlist until unhalt_cpsch
1765	* is called.
1766	*/
1767	static int halt_cpsch(struct device_queue_manager *dqm)
1768	{
1769	int ret = 0;
1770
1771	dqm_lock(dqm);
1772	if (!dqm->sched_running) {
1773	dqm_unlock(dqm);
1774	return 0;
1775	}
1776
1777	WARN_ONCE(dqm->sched_halt, "Scheduling is already on halt\n");
1778
1779	if (!dqm->is_hws_hang) {
1780	if (!dqm->dev->kfd->shared_resources.enable_mes)
1781	ret = unmap_queues_cpsch(dqm,
1782	filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0,
1783	USE_DEFAULT_GRACE_PERIOD, reset: false);
1784	else
1785	ret = remove_all_kfd_queues_mes(dqm);
1786	}
1787	dqm->sched_halt = true;
1788	dqm_unlock(dqm);
1789
1790	return ret;
1791	}
1792
1793	/* unhalt_cpsch
1794	* Unset dqm->sched_halt and map queues back to runlist
1795	*/
1796	static int unhalt_cpsch(struct device_queue_manager *dqm)
1797	{
1798	int ret = 0;
1799
1800	dqm_lock(dqm);
1801	if (!dqm->sched_running || !dqm->sched_halt) {
1802	WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n");
1803	dqm_unlock(dqm);
1804	return 0;
1805	}
1806	dqm->sched_halt = false;
1807	if (!dqm->dev->kfd->shared_resources.enable_mes)
1808	ret = execute_queues_cpsch(dqm,
1809	filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
1810	filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
1811	else
1812	ret = add_all_kfd_queues_mes(dqm);
1813
1814	dqm_unlock(dqm);
1815
1816	return ret;
1817	}
1818
1819	static int start_cpsch(struct device_queue_manager *dqm)
1820	{
1821	struct device *dev = dqm->dev->adev->dev;
1822	int retval, num_hw_queue_slots;
1823
1824	retval = 0;
1825
1826	dqm_lock(dqm);
1827
1828	if (!dqm->dev->kfd->shared_resources.enable_mes) {
1829	retval = pm_init(pm: &dqm->packet_mgr, dqm);
1830	if (retval)
1831	goto fail_packet_manager_init;
1832
1833	retval = set_sched_resources(dqm);
1834	if (retval)
1835	goto fail_set_sched_resources;
1836	}
1837	pr_debug("Allocating fence memory\n");
1838
1839	/* allocate fence memory on the gart */
1840	retval = kfd_gtt_sa_allocate(node: dqm->dev, size: sizeof(*dqm->fence_addr),
1841	mem_obj: &dqm->fence_mem);
1842
1843	if (retval)
1844	goto fail_allocate_vidmem;
1845
1846	dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr;
1847	dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
1848
1849	init_interrupts(dqm);
1850
1851	/* clear hang status when driver try to start the hw scheduler */
1852	dqm->sched_running = true;
1853
1854	if (!dqm->dev->kfd->shared_resources.enable_mes) {
1855	if (pm_config_dequeue_wait_counts(pm: &dqm->packet_mgr,
1856	cmd: KFD_DEQUEUE_WAIT_INIT, wait_counts_config: 0 /* unused */))
1857	dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
1858	execute_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
1859	}
1860
1861	/* setup per-queue reset detection buffer */
1862	num_hw_queue_slots = dqm->dev->kfd->shared_resources.num_queue_per_pipe *
1863	dqm->dev->kfd->shared_resources.num_pipe_per_mec *
1864	NUM_XCC(dqm->dev->xcc_mask);
1865
1866	dqm->detect_hang_info_size = num_hw_queue_slots * sizeof(struct dqm_detect_hang_info);
1867	dqm->detect_hang_info = kzalloc(dqm->detect_hang_info_size, GFP_KERNEL);
1868
1869	if (!dqm->detect_hang_info) {
1870	retval = -ENOMEM;
1871	goto fail_detect_hang_buffer;
1872	}
1873
1874	dqm_unlock(dqm);
1875
1876	return 0;
1877	fail_detect_hang_buffer:
1878	kfd_gtt_sa_free(node: dqm->dev, mem_obj: dqm->fence_mem);
1879	fail_allocate_vidmem:
1880	fail_set_sched_resources:
1881	if (!dqm->dev->kfd->shared_resources.enable_mes)
1882	pm_uninit(pm: &dqm->packet_mgr);
1883	fail_packet_manager_init:
1884	dqm_unlock(dqm);
1885	return retval;
1886	}
1887
1888	static int stop_cpsch(struct device_queue_manager *dqm)
1889	{
1890	int ret = 0;
1891
1892	dqm_lock(dqm);
1893	if (!dqm->sched_running) {
1894	dqm_unlock(dqm);
1895	return 0;
1896	}
1897
1898	if (!dqm->dev->kfd->shared_resources.enable_mes)
1899	ret = unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
1900	filter_param: 0, USE_DEFAULT_GRACE_PERIOD, reset: false);
1901	else
1902	ret = remove_all_kfd_queues_mes(dqm);
1903
1904	dqm->sched_running = false;
1905
1906	if (!dqm->dev->kfd->shared_resources.enable_mes)
1907	pm_release_ib(pm: &dqm->packet_mgr);
1908
1909	kfd_gtt_sa_free(node: dqm->dev, mem_obj: dqm->fence_mem);
1910	if (!dqm->dev->kfd->shared_resources.enable_mes)
1911	pm_uninit(pm: &dqm->packet_mgr);
1912	kfree(objp: dqm->detect_hang_info);
1913	dqm->detect_hang_info = NULL;
1914	dqm_unlock(dqm);
1915
1916	return ret;
1917	}
1918
1919	static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1920	struct kernel_queue *kq,
1921	struct qcm_process_device *qpd)
1922	{
1923	dqm_lock(dqm);
1924	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1925	pr_warn("Can't create new kernel queue because %d queues were already created\n",
1926	dqm->total_queue_count);
1927	dqm_unlock(dqm);
1928	return -EPERM;
1929	}
1930
1931	/*
1932	* Unconditionally increment this counter, regardless of the queue's
1933	* type or whether the queue is active.
1934	*/
1935	dqm->total_queue_count++;
1936	pr_debug("Total of %d queues are accountable so far\n",
1937	dqm->total_queue_count);
1938
1939	list_add(new: &kq->list, head: &qpd->priv_queue_list);
1940	increment_queue_count(dqm, qpd, q: kq->queue);
1941	qpd->is_debug = true;
1942	execute_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0,
1943	USE_DEFAULT_GRACE_PERIOD);
1944	dqm_unlock(dqm);
1945
1946	return 0;
1947	}
1948
1949	static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1950	struct kernel_queue *kq,
1951	struct qcm_process_device *qpd)
1952	{
1953	dqm_lock(dqm);
1954	list_del(entry: &kq->list);
1955	decrement_queue_count(dqm, qpd, q: kq->queue);
1956	qpd->is_debug = false;
1957	execute_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0,
1958	USE_DEFAULT_GRACE_PERIOD);
1959	/*
1960	* Unconditionally decrement this counter, regardless of the queue's
1961	* type.
1962	*/
1963	dqm->total_queue_count--;
1964	pr_debug("Total of %d queues are accountable so far\n",
1965	dqm->total_queue_count);
1966	dqm_unlock(dqm);
1967	}
1968
1969	static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
1970	struct qcm_process_device *qpd,
1971	const struct kfd_criu_queue_priv_data *qd,
1972	const void *restore_mqd, const void *restore_ctl_stack)
1973	{
1974	int retval;
1975	struct mqd_manager *mqd_mgr;
1976
1977	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1978	pr_warn("Can't create new usermode queue because %d queues were already created\n",
1979	dqm->total_queue_count);
1980	retval = -EPERM;
1981	goto out;
1982	}
1983
1984	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1985	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI ||
1986	q->properties.type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) {
1987	dqm_lock(dqm);
1988	retval = allocate_sdma_queue(dqm, q, restore_sdma_id: qd ? &qd->sdma_id : NULL);
1989	dqm_unlock(dqm);
1990	if (retval)
1991	goto out;
1992	}
1993
1994	retval = allocate_doorbell(qpd, q, restore_id: qd ? &qd->doorbell_id : NULL);
1995	if (retval)
1996	goto out_deallocate_sdma_queue;
1997
1998	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1999	type: q->properties.type)];
2000
2001	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2002	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
2003	dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
2004	q->properties.tba_addr = qpd->tba_addr;
2005	q->properties.tma_addr = qpd->tma_addr;
2006	q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
2007	if (!q->mqd_mem_obj) {
2008	retval = -ENOMEM;
2009	goto out_deallocate_doorbell;
2010	}
2011
2012	dqm_lock(dqm);
2013	/*
2014	* Eviction state logic: mark all queues as evicted, even ones
2015	* not currently active. Restoring inactive queues later only
2016	* updates the is_evicted flag but is a no-op otherwise.
2017	*/
2018	q->properties.is_evicted = !!qpd->evicted;
2019	q->properties.is_dbg_wa = qpd->pqm->process->debug_trap_enabled &&
2020	kfd_dbg_has_cwsr_workaround(dev: q->device);
2021
2022	if (qd)
2023	mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,
2024	&q->properties, restore_mqd, restore_ctl_stack,
2025	qd->ctl_stack_size);
2026	else
2027	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
2028	&q->gart_mqd_addr, &q->properties);
2029
2030	list_add(new: &q->list, head: &qpd->queues_list);
2031	qpd->queue_count++;
2032
2033	if (q->properties.is_active) {
2034	increment_queue_count(dqm, qpd, q);
2035
2036	if (!dqm->dev->kfd->shared_resources.enable_mes)
2037	retval = execute_queues_cpsch(dqm,
2038	filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
2039	else
2040	retval = add_queue_mes(dqm, q, qpd);
2041	if (retval)
2042	goto cleanup_queue;
2043	}
2044
2045	/*
2046	* Unconditionally increment this counter, regardless of the queue's
2047	* type or whether the queue is active.
2048	*/
2049	dqm->total_queue_count++;
2050
2051	pr_debug("Total of %d queues are accountable so far\n",
2052	dqm->total_queue_count);
2053
2054	dqm_unlock(dqm);
2055	return retval;
2056
2057	cleanup_queue:
2058	qpd->queue_count--;
2059	list_del(entry: &q->list);
2060	if (q->properties.is_active)
2061	decrement_queue_count(dqm, qpd, q);
2062	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2063	dqm_unlock(dqm);
2064	out_deallocate_doorbell:
2065	deallocate_doorbell(qpd, q);
2066	out_deallocate_sdma_queue:
2067	if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2068	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
2069	dqm_lock(dqm);
2070	deallocate_sdma_queue(dqm, q);
2071	dqm_unlock(dqm);
2072	}
2073	out:
2074	return retval;
2075	}
2076
2077	int amdkfd_fence_wait_timeout(struct device_queue_manager *dqm,
2078	uint64_t fence_value,
2079	unsigned int timeout_ms)
2080	{
2081	unsigned long end_jiffies = msecs_to_jiffies(m: timeout_ms) + jiffies;
2082	struct device *dev = dqm->dev->adev->dev;
2083	uint64_t *fence_addr = dqm->fence_addr;
2084
2085	while (*fence_addr != fence_value) {
2086	/* Fatal err detected, this response won't come */
2087	if (amdgpu_amdkfd_is_fed(adev: dqm->dev->adev) ||
2088	amdgpu_in_reset(adev: dqm->dev->adev))
2089	return -EIO;
2090
2091	if (time_after(jiffies, end_jiffies)) {
2092	dev_err(dev, "qcm fence wait loop timeout expired\n");
2093	/* In HWS case, this is used to halt the driver thread
2094	* in order not to mess up CP states before doing
2095	* scandumps for FW debugging.
2096	*/
2097	while (halt_if_hws_hang)
2098	schedule();
2099
2100	return -ETIME;
2101	}
2102	schedule();
2103	}
2104
2105	return 0;
2106	}
2107
2108	/* dqm->lock mutex has to be locked before calling this function */
2109	static int map_queues_cpsch(struct device_queue_manager *dqm)
2110	{
2111	struct device *dev = dqm->dev->adev->dev;
2112	int retval;
2113
2114	if (!dqm->sched_running || dqm->sched_halt)
2115	return 0;
2116	if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)
2117	return 0;
2118	if (dqm->active_runlist)
2119	return 0;
2120
2121	retval = pm_send_runlist(pm: &dqm->packet_mgr, dqm_queues: &dqm->queues);
2122	pr_debug("%s sent runlist\n", __func__);
2123	if (retval) {
2124	dev_err(dev, "failed to execute runlist\n");
2125	return retval;
2126	}
2127	dqm->active_runlist = true;
2128
2129	return retval;
2130	}
2131
2132	static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
2133	struct qcm_process_device *qpd)
2134	{
2135	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2136
2137	dev_err(dqm->dev->adev->dev, "queue id 0x%0x at pasid %d is reset\n",
2138	q->properties.queue_id, pdd->process->lead_thread->pid);
2139
2140	pdd->has_reset_queue = true;
2141	if (q->properties.is_active) {
2142	q->properties.is_active = false;
2143	decrement_queue_count(dqm, qpd, q);
2144	}
2145	}
2146
2147	static int detect_queue_hang(struct device_queue_manager *dqm)
2148	{
2149	int i;
2150
2151	/* detect should be used only in dqm locked queue reset */
2152	if (WARN_ON(dqm->detect_hang_count > 0))
2153	return 0;
2154
2155	memset(dqm->detect_hang_info, 0, dqm->detect_hang_info_size);
2156
2157	for (i = 0; i < AMDGPU_MAX_QUEUES; ++i) {
2158	uint32_t mec, pipe, queue;
2159	int xcc_id;
2160
2161	mec = (i / dqm->dev->kfd->shared_resources.num_queue_per_pipe)
2162	/ dqm->dev->kfd->shared_resources.num_pipe_per_mec;
2163
2164	if (mec || !test_bit(i, dqm->dev->kfd->shared_resources.cp_queue_bitmap))
2165	continue;
2166
2167	amdgpu_queue_mask_bit_to_mec_queue(adev: dqm->dev->adev, bit: i, mec: &mec, pipe: &pipe, queue: &queue);
2168
2169	for_each_inst(xcc_id, dqm->dev->xcc_mask) {
2170	uint64_t queue_addr = dqm->dev->kfd2kgd->hqd_get_pq_addr(
2171	dqm->dev->adev, pipe, queue, xcc_id);
2172	struct dqm_detect_hang_info hang_info;
2173
2174	if (!queue_addr)
2175	continue;
2176
2177	hang_info.pipe_id = pipe;
2178	hang_info.queue_id = queue;
2179	hang_info.xcc_id = xcc_id;
2180	hang_info.queue_address = queue_addr;
2181
2182	dqm->detect_hang_info[dqm->detect_hang_count] = hang_info;
2183	dqm->detect_hang_count++;
2184	}
2185	}
2186
2187	return dqm->detect_hang_count;
2188	}
2189
2190	static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uint64_t queue_address)
2191	{
2192	struct device_process_node *cur;
2193	struct qcm_process_device *qpd;
2194	struct queue *q;
2195
2196	list_for_each_entry(cur, &dqm->queues, list) {
2197	qpd = cur->qpd;
2198	list_for_each_entry(q, &qpd->queues_list, list) {
2199	if (queue_address == q->properties.queue_address)
2200	return q;
2201	}
2202	}
2203
2204	return NULL;
2205	}
2206
2207	static int reset_hung_queues(struct device_queue_manager *dqm)
2208	{
2209	int r = 0, reset_count = 0, i;
2210
2211	if (!dqm->detect_hang_info || dqm->is_hws_hang)
2212	return -EIO;
2213
2214	/* assume dqm locked. */
2215	if (!detect_queue_hang(dqm))
2216	return -ENOTRECOVERABLE;
2217
2218	for (i = 0; i < dqm->detect_hang_count; i++) {
2219	struct dqm_detect_hang_info hang_info = dqm->detect_hang_info[i];
2220	struct queue *q = find_queue_by_address(dqm, queue_address: hang_info.queue_address);
2221	struct kfd_process_device *pdd;
2222	uint64_t queue_addr = 0;
2223
2224	if (!q) {
2225	r = -ENOTRECOVERABLE;
2226	goto reset_fail;
2227	}
2228
2229	pdd = kfd_get_process_device_data(dev: dqm->dev, p: q->process);
2230	if (!pdd) {
2231	r = -ENOTRECOVERABLE;
2232	goto reset_fail;
2233	}
2234
2235	queue_addr = dqm->dev->kfd2kgd->hqd_reset(dqm->dev->adev,
2236	hang_info.pipe_id, hang_info.queue_id, hang_info.xcc_id,
2237	KFD_UNMAP_LATENCY_MS);
2238
2239	/* either reset failed or we reset an unexpected queue. */
2240	if (queue_addr != q->properties.queue_address) {
2241	r = -ENOTRECOVERABLE;
2242	goto reset_fail;
2243	}
2244
2245	set_queue_as_reset(dqm, q, qpd: &pdd->qpd);
2246	reset_count++;
2247	}
2248
2249	if (reset_count == dqm->detect_hang_count)
2250	kfd_signal_reset_event(dev: dqm->dev);
2251	else
2252	r = -ENOTRECOVERABLE;
2253
2254	reset_fail:
2255	dqm->detect_hang_count = 0;
2256
2257	return r;
2258	}
2259
2260	static bool sdma_has_hang(struct device_queue_manager *dqm)
2261	{
2262	int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2263	int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2264	int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2265	int i, j;
2266
2267	for (i = engine_start; i < engine_end; i++) {
2268	for (j = 0; j < num_queues_per_eng; j++) {
2269	if (!dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j))
2270	continue;
2271
2272	return true;
2273	}
2274	}
2275
2276	return false;
2277	}
2278
2279	static bool set_sdma_queue_as_reset(struct device_queue_manager *dqm,
2280	uint32_t doorbell_off)
2281	{
2282	struct device_process_node *cur;
2283	struct qcm_process_device *qpd;
2284	struct queue *q;
2285
2286	list_for_each_entry(cur, &dqm->queues, list) {
2287	qpd = cur->qpd;
2288	list_for_each_entry(q, &qpd->queues_list, list) {
2289	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA ||
2290	q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) &&
2291	q->properties.doorbell_off == doorbell_off) {
2292	set_queue_as_reset(dqm, q, qpd);
2293	return true;
2294	}
2295	}
2296	}
2297
2298	return false;
2299	}
2300
2301	static int reset_hung_queues_sdma(struct device_queue_manager *dqm)
2302	{
2303	int engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
2304	int engine_end = engine_start + get_num_all_sdma_engines(dqm);
2305	int num_queues_per_eng = dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
2306	int r = 0, i, j;
2307
2308	if (dqm->is_hws_hang)
2309	return -EIO;
2310
2311	/* Scan for hung HW queues and reset engine. */
2312	dqm->detect_hang_count = 0;
2313	for (i = engine_start; i < engine_end; i++) {
2314	for (j = 0; j < num_queues_per_eng; j++) {
2315	uint32_t doorbell_off =
2316	dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j);
2317
2318	if (!doorbell_off)
2319	continue;
2320
2321	/* Reset engine and check. */
2322	if (amdgpu_sdma_reset_engine(adev: dqm->dev->adev, instance_id: i, caller_handles_kernel_queues: false) ||
2323	dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) ||
2324	!set_sdma_queue_as_reset(dqm, doorbell_off)) {
2325	r = -ENOTRECOVERABLE;
2326	goto reset_fail;
2327	}
2328
2329	/* Should only expect one queue active per engine */
2330	dqm->detect_hang_count++;
2331	break;
2332	}
2333	}
2334
2335	/* Signal process reset */
2336	if (dqm->detect_hang_count)
2337	kfd_signal_reset_event(dev: dqm->dev);
2338	else
2339	r = -ENOTRECOVERABLE;
2340
2341	reset_fail:
2342	dqm->detect_hang_count = 0;
2343
2344	return r;
2345	}
2346
2347	static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma)
2348	{
2349	struct amdgpu_device *adev = dqm->dev->adev;
2350
2351	while (halt_if_hws_hang)
2352	schedule();
2353
2354	if (adev->debug_disable_gpu_ring_reset) {
2355	dev_info_once(adev->dev,
2356	"%s queue hung, but ring reset disabled",
2357	is_sdma ? "sdma" : "compute");
2358
2359	return -EPERM;
2360	}
2361	if (!amdgpu_gpu_recovery)
2362	return -ENOTRECOVERABLE;
2363
2364	return is_sdma ? reset_hung_queues_sdma(dqm) : reset_hung_queues(dqm);
2365	}
2366
2367	/* dqm->lock mutex has to be locked before calling this function
2368	*
2369	* @grace_period: If USE_DEFAULT_GRACE_PERIOD then default wait time
2370	* for context switch latency. Lower values are used by debugger
2371	* since context switching are triggered at high frequency.
2372	* This is configured by setting CP_IQ_WAIT_TIME2.SCH_WAVE
2373	*
2374	*/
2375	static int unmap_queues_cpsch(struct device_queue_manager *dqm,
2376	enum kfd_unmap_queues_filter filter,
2377	uint32_t filter_param,
2378	uint32_t grace_period,
2379	bool reset)
2380	{
2381	struct device *dev = dqm->dev->adev->dev;
2382	struct mqd_manager *mqd_mgr;
2383	int retval;
2384
2385	if (!dqm->sched_running)
2386	return 0;
2387	if (!dqm->active_runlist)
2388	return 0;
2389	if (!down_read_trylock(sem: &dqm->dev->adev->reset_domain->sem))
2390	return -EIO;
2391
2392	if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2393	retval = pm_config_dequeue_wait_counts(pm: &dqm->packet_mgr,
2394	cmd: KFD_DEQUEUE_WAIT_SET_SCH_WAVE, wait_counts_config: grace_period);
2395	if (retval)
2396	goto out;
2397	}
2398
2399	retval = pm_send_unmap_queue(pm: &dqm->packet_mgr, mode: filter, filter_param, reset);
2400	if (retval)
2401	goto out;
2402
2403	*dqm->fence_addr = KFD_FENCE_INIT;
2404	mb();
2405	pm_send_query_status(pm: &dqm->packet_mgr, fence_address: dqm->fence_gpu_addr,
2406	KFD_FENCE_COMPLETED);
2407	/* should be timed out */
2408	retval = amdkfd_fence_wait_timeout(dqm, KFD_FENCE_COMPLETED,
2409	timeout_ms: queue_preemption_timeout_ms);
2410	if (retval) {
2411	dev_err(dev, "The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
2412	kfd_hws_hang(dqm);
2413	goto out;
2414	}
2415
2416	/* In the current MEC firmware implementation, if compute queue
2417	* doesn't response to the preemption request in time, HIQ will
2418	* abandon the unmap request without returning any timeout error
2419	* to driver. Instead, MEC firmware will log the doorbell of the
2420	* unresponding compute queue to HIQ.MQD.queue_doorbell_id fields.
2421	* To make sure the queue unmap was successful, driver need to
2422	* check those fields
2423	*/
2424	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
2425	if (mqd_mgr->check_preemption_failed(mqd_mgr, dqm->packet_mgr.priv_queue->queue->mqd) &&
2426	reset_queues_on_hws_hang(dqm, is_sdma: false))
2427	goto reset_fail;
2428
2429	/* Check for SDMA hang and attempt SDMA reset */
2430	if (sdma_has_hang(dqm) && reset_queues_on_hws_hang(dqm, is_sdma: true))
2431	goto reset_fail;
2432
2433	/* We need to reset the grace period value for this device */
2434	if (grace_period != USE_DEFAULT_GRACE_PERIOD) {
2435	if (pm_config_dequeue_wait_counts(pm: &dqm->packet_mgr,
2436	cmd: KFD_DEQUEUE_WAIT_RESET, wait_counts_config: 0 /* unused */))
2437	dev_err(dev, "Failed to reset grace period\n");
2438	}
2439
2440	pm_release_ib(pm: &dqm->packet_mgr);
2441	dqm->active_runlist = false;
2442	out:
2443	up_read(sem: &dqm->dev->adev->reset_domain->sem);
2444	return retval;
2445
2446	reset_fail:
2447	dqm->is_hws_hang = true;
2448	kfd_hws_hang(dqm);
2449	up_read(sem: &dqm->dev->adev->reset_domain->sem);
2450	return -ETIME;
2451	}
2452
2453	/* only for compute queue */
2454	static int reset_queues_cpsch(struct device_queue_manager *dqm, uint16_t pasid)
2455	{
2456	int retval;
2457
2458	dqm_lock(dqm);
2459
2460	retval = unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_BY_PASID,
2461	filter_param: pasid, USE_DEFAULT_GRACE_PERIOD, reset: true);
2462
2463	dqm_unlock(dqm);
2464	return retval;
2465	}
2466
2467	/* dqm->lock mutex has to be locked before calling this function */
2468	static int execute_queues_cpsch(struct device_queue_manager *dqm,
2469	enum kfd_unmap_queues_filter filter,
2470	uint32_t filter_param,
2471	uint32_t grace_period)
2472	{
2473	int retval;
2474
2475	if (!down_read_trylock(sem: &dqm->dev->adev->reset_domain->sem))
2476	return -EIO;
2477	retval = unmap_queues_cpsch(dqm, filter, filter_param, grace_period, reset: false);
2478	if (!retval)
2479	retval = map_queues_cpsch(dqm);
2480	up_read(sem: &dqm->dev->adev->reset_domain->sem);
2481	return retval;
2482	}
2483
2484	static int wait_on_destroy_queue(struct device_queue_manager *dqm,
2485	struct queue *q)
2486	{
2487	struct kfd_process_device *pdd = kfd_get_process_device_data(dev: q->device,
2488	p: q->process);
2489	int ret = 0;
2490
2491	if (WARN_ON(!pdd))
2492	return ret;
2493
2494	if (pdd->qpd.is_debug)
2495	return ret;
2496
2497	q->properties.is_being_destroyed = true;
2498
2499	if (pdd->process->debug_trap_enabled && q->properties.is_suspended) {
2500	dqm_unlock(dqm);
2501	mutex_unlock(lock: &q->process->mutex);
2502	ret = wait_event_interruptible(dqm->destroy_wait,
2503	!q->properties.is_suspended);
2504
2505	mutex_lock(&q->process->mutex);
2506	dqm_lock(dqm);
2507	}
2508
2509	return ret;
2510	}
2511
2512	static int destroy_queue_cpsch(struct device_queue_manager *dqm,
2513	struct qcm_process_device *qpd,
2514	struct queue *q)
2515	{
2516	int retval;
2517	struct mqd_manager *mqd_mgr;
2518	uint64_t sdma_val = 0;
2519	struct kfd_process_device *pdd = qpd_to_pdd(qpd);
2520	struct device *dev = dqm->dev->adev->dev;
2521
2522	/* Get the SDMA queue stats */
2523	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2524	(q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2525	retval = read_sdma_queue_counter(q_rptr: (uint64_t __user *)q->properties.read_ptr,
2526	val: &sdma_val);
2527	if (retval)
2528	dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
2529	q->properties.queue_id);
2530	}
2531
2532	/* remove queue from list to prevent rescheduling after preemption */
2533	dqm_lock(dqm);
2534
2535	retval = wait_on_destroy_queue(dqm, q);
2536
2537	if (retval) {
2538	dqm_unlock(dqm);
2539	return retval;
2540	}
2541
2542	if (qpd->is_debug) {
2543	/*
2544	* error, currently we do not allow to destroy a queue
2545	* of a currently debugged process
2546	*/
2547	retval = -EBUSY;
2548	goto failed_try_destroy_debugged_queue;
2549
2550	}
2551
2552	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2553	type: q->properties.type)];
2554
2555	deallocate_doorbell(qpd, q);
2556
2557	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
2558	(q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2559	deallocate_sdma_queue(dqm, q);
2560	pdd->sdma_past_activity_counter += sdma_val;
2561	}
2562
2563	if (q->properties.is_active) {
2564	decrement_queue_count(dqm, qpd, q);
2565	q->properties.is_active = false;
2566	if (!dqm->dev->kfd->shared_resources.enable_mes) {
2567	retval = execute_queues_cpsch(dqm,
2568	filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0,
2569	USE_DEFAULT_GRACE_PERIOD);
2570	if (retval == -ETIME)
2571	qpd->reset_wavefronts = true;
2572	} else {
2573	retval = remove_queue_mes(dqm, q, qpd);
2574	}
2575	}
2576	list_del(entry: &q->list);
2577	qpd->queue_count--;
2578
2579	/*
2580	* Unconditionally decrement this counter, regardless of the queue's
2581	* type
2582	*/
2583	dqm->total_queue_count--;
2584	pr_debug("Total of %d queues are accountable so far\n",
2585	dqm->total_queue_count);
2586
2587	dqm_unlock(dqm);
2588
2589	/*
2590	* Do free_mqd and raise delete event after dqm_unlock(dqm) to avoid
2591	* circular locking
2592	*/
2593	kfd_dbg_ev_raise(KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE),
2594	process: qpd->pqm->process, dev: q->device,
2595	source_id: -1, use_worker: false, NULL, exception_data_size: 0);
2596
2597	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2598
2599	return retval;
2600
2601	failed_try_destroy_debugged_queue:
2602
2603	dqm_unlock(dqm);
2604	return retval;
2605	}
2606
2607	static bool set_cache_memory_policy(struct device_queue_manager *dqm,
2608	struct qcm_process_device *qpd,
2609	enum cache_policy default_policy,
2610	enum cache_policy alternate_policy,
2611	void __user *alternate_aperture_base,
2612	uint64_t alternate_aperture_size,
2613	u32 misc_process_properties)
2614	{
2615	bool retval = true;
2616
2617	if (!dqm->asic_ops.set_cache_memory_policy)
2618	return retval;
2619
2620	dqm_lock(dqm);
2621
2622	retval = dqm->asic_ops.set_cache_memory_policy(
2623	dqm,
2624	qpd,
2625	default_policy,
2626	alternate_policy,
2627	alternate_aperture_base,
2628	alternate_aperture_size,
2629	misc_process_properties);
2630
2631	if (retval)
2632	goto out;
2633
2634	if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
2635	program_sh_mem_settings(dqm, qpd);
2636
2637	pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
2638	qpd->sh_mem_config, qpd->sh_mem_ape1_base,
2639	qpd->sh_mem_ape1_limit);
2640
2641	out:
2642	dqm_unlock(dqm);
2643	return retval;
2644	}
2645
2646	static int process_termination_nocpsch(struct device_queue_manager *dqm,
2647	struct qcm_process_device *qpd)
2648	{
2649	struct queue *q;
2650	struct device_process_node *cur, *next_dpn;
2651	int retval = 0;
2652	bool found = false;
2653
2654	dqm_lock(dqm);
2655
2656	/* Clear all user mode queues */
2657	while (!list_empty(head: &qpd->queues_list)) {
2658	struct mqd_manager *mqd_mgr;
2659	int ret;
2660
2661	q = list_first_entry(&qpd->queues_list, struct queue, list);
2662	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2663	type: q->properties.type)];
2664	ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
2665	if (ret)
2666	retval = ret;
2667	dqm_unlock(dqm);
2668	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2669	dqm_lock(dqm);
2670	}
2671
2672	/* Unregister process */
2673	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2674	if (qpd == cur->qpd) {
2675	list_del(entry: &cur->list);
2676	kfree(objp: cur);
2677	dqm->processes_count--;
2678	found = true;
2679	break;
2680	}
2681	}
2682
2683	dqm_unlock(dqm);
2684
2685	/* Outside the DQM lock because under the DQM lock we can't do
2686	* reclaim or take other locks that others hold while reclaiming.
2687	*/
2688	if (found)
2689	kfd_dec_compute_active(dev: dqm->dev);
2690
2691	return retval;
2692	}
2693
2694	static int get_wave_state(struct device_queue_manager *dqm,
2695	struct queue *q,
2696	void __user *ctl_stack,
2697	u32 *ctl_stack_used_size,
2698	u32 *save_area_used_size)
2699	{
2700	struct mqd_manager *mqd_mgr;
2701
2702	dqm_lock(dqm);
2703
2704	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
2705
2706	if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
2707	q->properties.is_active || !q->device->kfd->cwsr_enabled ||
2708	!mqd_mgr->get_wave_state) {
2709	dqm_unlock(dqm);
2710	return -EINVAL;
2711	}
2712
2713	dqm_unlock(dqm);
2714
2715	/*
2716	* get_wave_state is outside the dqm lock to prevent circular locking
2717	* and the queue should be protected against destruction by the process
2718	* lock.
2719	*/
2720	return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, &q->properties,
2721	ctl_stack, ctl_stack_used_size, save_area_used_size);
2722	}
2723
2724	static void get_queue_checkpoint_info(struct device_queue_manager *dqm,
2725	const struct queue *q,
2726	u32 *mqd_size,
2727	u32 *ctl_stack_size)
2728	{
2729	struct mqd_manager *mqd_mgr;
2730	enum KFD_MQD_TYPE mqd_type =
2731	get_mqd_type_from_queue_type(type: q->properties.type);
2732
2733	dqm_lock(dqm);
2734	mqd_mgr = dqm->mqd_mgrs[mqd_type];
2735	*mqd_size = mqd_mgr->mqd_size * NUM_XCC(mqd_mgr->dev->xcc_mask);
2736	*ctl_stack_size = 0;
2737
2738	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info)
2739	mqd_mgr->get_checkpoint_info(mqd_mgr, q->mqd, ctl_stack_size);
2740
2741	dqm_unlock(dqm);
2742	}
2743
2744	static int checkpoint_mqd(struct device_queue_manager *dqm,
2745	const struct queue *q,
2746	void *mqd,
2747	void *ctl_stack)
2748	{
2749	struct mqd_manager *mqd_mgr;
2750	int r = 0;
2751	enum KFD_MQD_TYPE mqd_type =
2752	get_mqd_type_from_queue_type(type: q->properties.type);
2753
2754	dqm_lock(dqm);
2755
2756	if (q->properties.is_active || !q->device->kfd->cwsr_enabled) {
2757	r = -EINVAL;
2758	goto dqm_unlock;
2759	}
2760
2761	mqd_mgr = dqm->mqd_mgrs[mqd_type];
2762	if (!mqd_mgr->checkpoint_mqd) {
2763	r = -EOPNOTSUPP;
2764	goto dqm_unlock;
2765	}
2766
2767	mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd, ctl_stack);
2768
2769	dqm_unlock:
2770	dqm_unlock(dqm);
2771	return r;
2772	}
2773
2774	static int process_termination_cpsch(struct device_queue_manager *dqm,
2775	struct qcm_process_device *qpd)
2776	{
2777	int retval;
2778	struct queue *q;
2779	struct device *dev = dqm->dev->adev->dev;
2780	struct kernel_queue *kq, *kq_next;
2781	struct mqd_manager *mqd_mgr;
2782	struct device_process_node *cur, *next_dpn;
2783	enum kfd_unmap_queues_filter filter =
2784	KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
2785	bool found = false;
2786
2787	retval = 0;
2788
2789	dqm_lock(dqm);
2790
2791	/* Clean all kernel queues */
2792	list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
2793	list_del(entry: &kq->list);
2794	decrement_queue_count(dqm, qpd, q: kq->queue);
2795	qpd->is_debug = false;
2796	dqm->total_queue_count--;
2797	filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
2798	}
2799
2800	/* Clear all user mode queues */
2801	list_for_each_entry(q, &qpd->queues_list, list) {
2802	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
2803	deallocate_sdma_queue(dqm, q);
2804	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
2805	deallocate_sdma_queue(dqm, q);
2806
2807	if (q->properties.is_active) {
2808	decrement_queue_count(dqm, qpd, q);
2809
2810	if (dqm->dev->kfd->shared_resources.enable_mes) {
2811	retval = remove_queue_mes(dqm, q, qpd);
2812	if (retval)
2813	dev_err(dev, "Failed to remove queue %d\n",
2814	q->properties.queue_id);
2815	}
2816	}
2817
2818	dqm->total_queue_count--;
2819	}
2820
2821	/* Unregister process */
2822	list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
2823	if (qpd == cur->qpd) {
2824	list_del(entry: &cur->list);
2825	kfree(objp: cur);
2826	dqm->processes_count--;
2827	found = true;
2828	break;
2829	}
2830	}
2831
2832	if (!dqm->dev->kfd->shared_resources.enable_mes)
2833	retval = execute_queues_cpsch(dqm, filter, filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
2834
2835	if ((retval || qpd->reset_wavefronts) &&
2836	down_read_trylock(sem: &dqm->dev->adev->reset_domain->sem)) {
2837	pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
2838	dbgdev_wave_reset_wavefronts(dev: dqm->dev, p: qpd->pqm->process);
2839	qpd->reset_wavefronts = false;
2840	up_read(sem: &dqm->dev->adev->reset_domain->sem);
2841	}
2842
2843	/* Lastly, free mqd resources.
2844	* Do free_mqd() after dqm_unlock to avoid circular locking.
2845	*/
2846	while (!list_empty(head: &qpd->queues_list)) {
2847	q = list_first_entry(&qpd->queues_list, struct queue, list);
2848	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
2849	type: q->properties.type)];
2850	list_del(entry: &q->list);
2851	qpd->queue_count--;
2852	dqm_unlock(dqm);
2853	mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
2854	dqm_lock(dqm);
2855	}
2856	dqm_unlock(dqm);
2857
2858	/* Outside the DQM lock because under the DQM lock we can't do
2859	* reclaim or take other locks that others hold while reclaiming.
2860	*/
2861	if (found)
2862	kfd_dec_compute_active(dev: dqm->dev);
2863
2864	return retval;
2865	}
2866
2867	static int init_mqd_managers(struct device_queue_manager *dqm)
2868	{
2869	int i, j;
2870	struct device *dev = dqm->dev->adev->dev;
2871	struct mqd_manager *mqd_mgr;
2872
2873	for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
2874	mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
2875	if (!mqd_mgr) {
2876	dev_err(dev, "mqd manager [%d] initialization failed\n", i);
2877	goto out_free;
2878	}
2879	dqm->mqd_mgrs[i] = mqd_mgr;
2880	}
2881
2882	return 0;
2883
2884	out_free:
2885	for (j = 0; j < i; j++) {
2886	kfree(objp: dqm->mqd_mgrs[j]);
2887	dqm->mqd_mgrs[j] = NULL;
2888	}
2889
2890	return -ENOMEM;
2891	}
2892
2893	/* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
2894	static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
2895	{
2896	int retval;
2897	struct kfd_node *dev = dqm->dev;
2898	struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
2899	uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
2900	get_num_all_sdma_engines(dqm) *
2901	dev->kfd->device_info.num_sdma_queues_per_engine +
2902	(dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size *
2903	NUM_XCC(dqm->dev->xcc_mask));
2904
2905	retval = amdgpu_amdkfd_alloc_gtt_mem(adev: dev->adev, size,
2906	mem_obj: &(mem_obj->gtt_mem), gpu_addr: &(mem_obj->gpu_addr),
2907	cpu_ptr: (void *)&(mem_obj->cpu_ptr), mqd_gfx9: false);
2908
2909	return retval;
2910	}
2911
2912	static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
2913	struct kfd_mem_obj *mqd)
2914	{
2915	WARN(!mqd, "No hiq sdma mqd trunk to free");
2916
2917	amdgpu_amdkfd_free_gtt_mem(adev: dev->adev, mem_obj: &mqd->gtt_mem);
2918	}
2919
2920	struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
2921	{
2922	struct device_queue_manager *dqm;
2923
2924	pr_debug("Loading device queue manager\n");
2925
2926	dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
2927	if (!dqm)
2928	return NULL;
2929
2930	switch (dev->adev->asic_type) {
2931	/* HWS is not available on Hawaii. */
2932	case CHIP_HAWAII:
2933	/* HWS depends on CWSR for timely dequeue. CWSR is not
2934	* available on Tonga.
2935	*
2936	* FIXME: This argument also applies to Kaveri.
2937	*/
2938	case CHIP_TONGA:
2939	dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
2940	break;
2941	default:
2942	dqm->sched_policy = sched_policy;
2943	break;
2944	}
2945
2946	dqm->dev = dev;
2947	switch (dqm->sched_policy) {
2948	case KFD_SCHED_POLICY_HWS:
2949	case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
2950	/* initialize dqm for cp scheduling */
2951	dqm->ops.create_queue = create_queue_cpsch;
2952	dqm->ops.initialize = initialize_cpsch;
2953	dqm->ops.start = start_cpsch;
2954	dqm->ops.stop = stop_cpsch;
2955	dqm->ops.halt = halt_cpsch;
2956	dqm->ops.unhalt = unhalt_cpsch;
2957	dqm->ops.destroy_queue = destroy_queue_cpsch;
2958	dqm->ops.update_queue = update_queue;
2959	dqm->ops.register_process = register_process;
2960	dqm->ops.unregister_process = unregister_process;
2961	dqm->ops.uninitialize = uninitialize;
2962	dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
2963	dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
2964	dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2965	dqm->ops.process_termination = process_termination_cpsch;
2966	dqm->ops.evict_process_queues = evict_process_queues_cpsch;
2967	dqm->ops.restore_process_queues = restore_process_queues_cpsch;
2968	dqm->ops.get_wave_state = get_wave_state;
2969	dqm->ops.reset_queues = reset_queues_cpsch;
2970	dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2971	dqm->ops.checkpoint_mqd = checkpoint_mqd;
2972	break;
2973	case KFD_SCHED_POLICY_NO_HWS:
2974	/* initialize dqm for no cp scheduling */
2975	dqm->ops.start = start_nocpsch;
2976	dqm->ops.stop = stop_nocpsch;
2977	dqm->ops.create_queue = create_queue_nocpsch;
2978	dqm->ops.destroy_queue = destroy_queue_nocpsch;
2979	dqm->ops.update_queue = update_queue;
2980	dqm->ops.register_process = register_process;
2981	dqm->ops.unregister_process = unregister_process;
2982	dqm->ops.initialize = initialize_nocpsch;
2983	dqm->ops.uninitialize = uninitialize;
2984	dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
2985	dqm->ops.process_termination = process_termination_nocpsch;
2986	dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
2987	dqm->ops.restore_process_queues =
2988	restore_process_queues_nocpsch;
2989	dqm->ops.get_wave_state = get_wave_state;
2990	dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
2991	dqm->ops.checkpoint_mqd = checkpoint_mqd;
2992	break;
2993	default:
2994	dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy);
2995	goto out_free;
2996	}
2997
2998	switch (dev->adev->asic_type) {
2999	case CHIP_KAVERI:
3000	case CHIP_HAWAII:
3001	device_queue_manager_init_cik(asic_ops: &dqm->asic_ops);
3002	break;
3003
3004	case CHIP_CARRIZO:
3005	case CHIP_TONGA:
3006	case CHIP_FIJI:
3007	case CHIP_POLARIS10:
3008	case CHIP_POLARIS11:
3009	case CHIP_POLARIS12:
3010	case CHIP_VEGAM:
3011	device_queue_manager_init_vi(asic_ops: &dqm->asic_ops);
3012	break;
3013
3014	default:
3015	if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 0, 0))
3016	device_queue_manager_init_v12(asic_ops: &dqm->asic_ops);
3017	else if (KFD_GC_VERSION(dev) >= IP_VERSION(11, 0, 0))
3018	device_queue_manager_init_v11(asic_ops: &dqm->asic_ops);
3019	else if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
3020	device_queue_manager_init_v10(asic_ops: &dqm->asic_ops);
3021	else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
3022	device_queue_manager_init_v9(asic_ops: &dqm->asic_ops);
3023	else {
3024	WARN(1, "Unexpected ASIC family %u",
3025	dev->adev->asic_type);
3026	goto out_free;
3027	}
3028	}
3029
3030	if (init_mqd_managers(dqm))
3031	goto out_free;
3032
3033	if (!dev->kfd->shared_resources.enable_mes && allocate_hiq_sdma_mqd(dqm)) {
3034	dev_err(dev->adev->dev, "Failed to allocate hiq sdma mqd trunk buffer\n");
3035	goto out_free;
3036	}
3037
3038	if (!dqm->ops.initialize(dqm)) {
3039	init_waitqueue_head(&dqm->destroy_wait);
3040	return dqm;
3041	}
3042
3043	if (!dev->kfd->shared_resources.enable_mes)
3044	deallocate_hiq_sdma_mqd(dev, mqd: &dqm->hiq_sdma_mqd);
3045
3046	out_free:
3047	kfree(objp: dqm);
3048	return NULL;
3049	}
3050
3051	void device_queue_manager_uninit(struct device_queue_manager *dqm)
3052	{
3053	dqm->ops.stop(dqm);
3054	dqm->ops.uninitialize(dqm);
3055	if (!dqm->dev->kfd->shared_resources.enable_mes)
3056	deallocate_hiq_sdma_mqd(dev: dqm->dev, mqd: &dqm->hiq_sdma_mqd);
3057	kfree(objp: dqm);
3058	}
3059
3060	int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id)
3061	{
3062	struct kfd_process_device *pdd = NULL;
3063	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, pdd: &pdd);
3064	struct device_queue_manager *dqm = knode->dqm;
3065	struct device *dev = dqm->dev->adev->dev;
3066	struct qcm_process_device *qpd;
3067	struct queue *q = NULL;
3068	int ret = 0;
3069
3070	if (!pdd)
3071	return -EINVAL;
3072
3073	dqm_lock(dqm);
3074
3075	if (pdd) {
3076	qpd = &pdd->qpd;
3077
3078	list_for_each_entry(q, &qpd->queues_list, list) {
3079	if (q->doorbell_id == doorbell_id && q->properties.is_active) {
3080	ret = suspend_all_queues_mes(dqm);
3081	if (ret) {
3082	dev_err(dev, "Suspending all queues failed");
3083	goto out;
3084	}
3085
3086	q->properties.is_evicted = true;
3087	q->properties.is_active = false;
3088	decrement_queue_count(dqm, qpd, q);
3089
3090	ret = remove_queue_mes(dqm, q, qpd);
3091	if (ret) {
3092	dev_err(dev, "Removing bad queue failed");
3093	goto out;
3094	}
3095
3096	ret = resume_all_queues_mes(dqm);
3097	if (ret)
3098	dev_err(dev, "Resuming all queues failed");
3099
3100	break;
3101	}
3102	}
3103	}
3104
3105	out:
3106	dqm_unlock(dqm);
3107	kfd_unref_process(p);
3108	return ret;
3109	}
3110
3111	int kfd_evict_process_device(struct kfd_process_device *pdd)
3112	{
3113	struct device_queue_manager *dqm;
3114	struct kfd_process *p;
3115
3116	p = pdd->process;
3117	dqm = pdd->dev->dqm;
3118
3119	WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
3120
3121	return dqm->ops.evict_process_queues(dqm, &pdd->qpd);
3122	}
3123
3124	int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
3125	struct qcm_process_device *qpd)
3126	{
3127	int r;
3128	struct device *dev = dqm->dev->adev->dev;
3129	int updated_vmid_mask;
3130
3131	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3132	dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3133	return -EINVAL;
3134	}
3135
3136	dqm_lock(dqm);
3137
3138	if (dqm->trap_debug_vmid != 0) {
3139	dev_err(dev, "Trap debug id already reserved\n");
3140	r = -EBUSY;
3141	goto out_unlock;
3142	}
3143
3144	r = unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0,
3145	USE_DEFAULT_GRACE_PERIOD, reset: false);
3146	if (r)
3147	goto out_unlock;
3148
3149	updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3150	updated_vmid_mask &= ~(1 << dqm->dev->vm_info.last_vmid_kfd);
3151
3152	dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3153	dqm->trap_debug_vmid = dqm->dev->vm_info.last_vmid_kfd;
3154	r = set_sched_resources(dqm);
3155	if (r)
3156	goto out_unlock;
3157
3158	r = map_queues_cpsch(dqm);
3159	if (r)
3160	goto out_unlock;
3161
3162	pr_debug("Reserved VMID for trap debug: %i\n", dqm->trap_debug_vmid);
3163
3164	out_unlock:
3165	dqm_unlock(dqm);
3166	return r;
3167	}
3168
3169	/*
3170	* Releases vmid for the trap debugger
3171	*/
3172	int release_debug_trap_vmid(struct device_queue_manager *dqm,
3173	struct qcm_process_device *qpd)
3174	{
3175	struct device *dev = dqm->dev->adev->dev;
3176	int r;
3177	int updated_vmid_mask;
3178	uint32_t trap_debug_vmid;
3179
3180	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3181	dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3182	return -EINVAL;
3183	}
3184
3185	dqm_lock(dqm);
3186	trap_debug_vmid = dqm->trap_debug_vmid;
3187	if (dqm->trap_debug_vmid == 0) {
3188	dev_err(dev, "Trap debug id is not reserved\n");
3189	r = -EINVAL;
3190	goto out_unlock;
3191	}
3192
3193	r = unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0,
3194	USE_DEFAULT_GRACE_PERIOD, reset: false);
3195	if (r)
3196	goto out_unlock;
3197
3198	updated_vmid_mask = dqm->dev->kfd->shared_resources.compute_vmid_bitmap;
3199	updated_vmid_mask |= (1 << dqm->dev->vm_info.last_vmid_kfd);
3200
3201	dqm->dev->kfd->shared_resources.compute_vmid_bitmap = updated_vmid_mask;
3202	dqm->trap_debug_vmid = 0;
3203	r = set_sched_resources(dqm);
3204	if (r)
3205	goto out_unlock;
3206
3207	r = map_queues_cpsch(dqm);
3208	if (r)
3209	goto out_unlock;
3210
3211	pr_debug("Released VMID for trap debug: %i\n", trap_debug_vmid);
3212
3213	out_unlock:
3214	dqm_unlock(dqm);
3215	return r;
3216	}
3217
3218	#define QUEUE_NOT_FOUND -1
3219	/* invalidate queue operation in array */
3220	static void q_array_invalidate(uint32_t num_queues, uint32_t *queue_ids)
3221	{
3222	int i;
3223
3224	for (i = 0; i < num_queues; i++)
3225	queue_ids[i] |= KFD_DBG_QUEUE_INVALID_MASK;
3226	}
3227
3228	/* find queue index in array */
3229	static int q_array_get_index(unsigned int queue_id,
3230	uint32_t num_queues,
3231	uint32_t *queue_ids)
3232	{
3233	int i;
3234
3235	for (i = 0; i < num_queues; i++)
3236	if (queue_id == (queue_ids[i] & ~KFD_DBG_QUEUE_INVALID_MASK))
3237	return i;
3238
3239	return QUEUE_NOT_FOUND;
3240	}
3241
3242	struct copy_context_work_handler_workarea {
3243	struct work_struct copy_context_work;
3244	struct kfd_process *p;
3245	};
3246
3247	static void copy_context_work_handler(struct work_struct *work)
3248	{
3249	struct copy_context_work_handler_workarea *workarea;
3250	struct mqd_manager *mqd_mgr;
3251	struct queue *q;
3252	struct mm_struct *mm;
3253	struct kfd_process *p;
3254	uint32_t tmp_ctl_stack_used_size, tmp_save_area_used_size;
3255	int i;
3256
3257	workarea = container_of(work,
3258	struct copy_context_work_handler_workarea,
3259	copy_context_work);
3260
3261	p = workarea->p;
3262	mm = get_task_mm(task: p->lead_thread);
3263
3264	if (!mm)
3265	return;
3266
3267	kthread_use_mm(mm);
3268	for (i = 0; i < p->n_pdds; i++) {
3269	struct kfd_process_device *pdd = p->pdds[i];
3270	struct device_queue_manager *dqm = pdd->dev->dqm;
3271	struct qcm_process_device *qpd = &pdd->qpd;
3272
3273	list_for_each_entry(q, &qpd->queues_list, list) {
3274	if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE)
3275	continue;
3276
3277	mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
3278
3279	/* We ignore the return value from get_wave_state
3280	* because
3281	* i) right now, it always returns 0, and
3282	* ii) if we hit an error, we would continue to the
3283	* next queue anyway.
3284	*/
3285	mqd_mgr->get_wave_state(mqd_mgr,
3286	q->mqd,
3287	&q->properties,
3288	(void __user *) q->properties.ctx_save_restore_area_address,
3289	&tmp_ctl_stack_used_size,
3290	&tmp_save_area_used_size);
3291	}
3292	}
3293	kthread_unuse_mm(mm);
3294	mmput(mm);
3295	}
3296
3297	static uint32_t *get_queue_ids(uint32_t num_queues, uint32_t *usr_queue_id_array)
3298	{
3299	size_t array_size = num_queues * sizeof(uint32_t);
3300
3301	if (!usr_queue_id_array)
3302	return NULL;
3303
3304	return memdup_user(usr_queue_id_array, array_size);
3305	}
3306
3307	int resume_queues(struct kfd_process *p,
3308	uint32_t num_queues,
3309	uint32_t *usr_queue_id_array)
3310	{
3311	uint32_t *queue_ids = NULL;
3312	int total_resumed = 0;
3313	int i;
3314
3315	if (usr_queue_id_array) {
3316	queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3317
3318	if (IS_ERR(ptr: queue_ids))
3319	return PTR_ERR(ptr: queue_ids);
3320
3321	/* mask all queues as invalid. unmask per successful request */
3322	q_array_invalidate(num_queues, queue_ids);
3323	}
3324
3325	for (i = 0; i < p->n_pdds; i++) {
3326	struct kfd_process_device *pdd = p->pdds[i];
3327	struct device_queue_manager *dqm = pdd->dev->dqm;
3328	struct device *dev = dqm->dev->adev->dev;
3329	struct qcm_process_device *qpd = &pdd->qpd;
3330	struct queue *q;
3331	int r, per_device_resumed = 0;
3332
3333	dqm_lock(dqm);
3334
3335	/* unmask queues that resume or already resumed as valid */
3336	list_for_each_entry(q, &qpd->queues_list, list) {
3337	int q_idx = QUEUE_NOT_FOUND;
3338
3339	if (queue_ids)
3340	q_idx = q_array_get_index(
3341	queue_id: q->properties.queue_id,
3342	num_queues,
3343	queue_ids);
3344
3345	if (!queue_ids || q_idx != QUEUE_NOT_FOUND) {
3346	int err = resume_single_queue(dqm, qpd: &pdd->qpd, q);
3347
3348	if (queue_ids) {
3349	if (!err) {
3350	queue_ids[q_idx] &=
3351	~KFD_DBG_QUEUE_INVALID_MASK;
3352	} else {
3353	queue_ids[q_idx] |=
3354	KFD_DBG_QUEUE_ERROR_MASK;
3355	break;
3356	}
3357	}
3358
3359	if (dqm->dev->kfd->shared_resources.enable_mes) {
3360	wake_up_all(&dqm->destroy_wait);
3361	if (!err)
3362	total_resumed++;
3363	} else {
3364	per_device_resumed++;
3365	}
3366	}
3367	}
3368
3369	if (!per_device_resumed) {
3370	dqm_unlock(dqm);
3371	continue;
3372	}
3373
3374	r = execute_queues_cpsch(dqm,
3375	filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
3376	filter_param: 0,
3377	USE_DEFAULT_GRACE_PERIOD);
3378	if (r) {
3379	dev_err(dev, "Failed to resume process queues\n");
3380	if (queue_ids) {
3381	list_for_each_entry(q, &qpd->queues_list, list) {
3382	int q_idx = q_array_get_index(
3383	queue_id: q->properties.queue_id,
3384	num_queues,
3385	queue_ids);
3386
3387	/* mask queue as error on resume fail */
3388	if (q_idx != QUEUE_NOT_FOUND)
3389	queue_ids[q_idx] |=
3390	KFD_DBG_QUEUE_ERROR_MASK;
3391	}
3392	}
3393	} else {
3394	wake_up_all(&dqm->destroy_wait);
3395	total_resumed += per_device_resumed;
3396	}
3397
3398	dqm_unlock(dqm);
3399	}
3400
3401	if (queue_ids) {
3402	if (copy_to_user(to: (void __user *)usr_queue_id_array, from: queue_ids,
3403	n: num_queues * sizeof(uint32_t)))
3404	pr_err("copy_to_user failed on queue resume\n");
3405
3406	kfree(objp: queue_ids);
3407	}
3408
3409	return total_resumed;
3410	}
3411
3412	int suspend_queues(struct kfd_process *p,
3413	uint32_t num_queues,
3414	uint32_t grace_period,
3415	uint64_t exception_clear_mask,
3416	uint32_t *usr_queue_id_array)
3417	{
3418	uint32_t *queue_ids = get_queue_ids(num_queues, usr_queue_id_array);
3419	int total_suspended = 0;
3420	int i;
3421
3422	if (IS_ERR(ptr: queue_ids))
3423	return PTR_ERR(ptr: queue_ids);
3424
3425	/* mask all queues as invalid. umask on successful request */
3426	q_array_invalidate(num_queues, queue_ids);
3427
3428	for (i = 0; i < p->n_pdds; i++) {
3429	struct kfd_process_device *pdd = p->pdds[i];
3430	struct device_queue_manager *dqm = pdd->dev->dqm;
3431	struct device *dev = dqm->dev->adev->dev;
3432	struct qcm_process_device *qpd = &pdd->qpd;
3433	struct queue *q;
3434	int r, per_device_suspended = 0;
3435
3436	mutex_lock(&p->event_mutex);
3437	dqm_lock(dqm);
3438
3439	/* unmask queues that suspend or already suspended */
3440	list_for_each_entry(q, &qpd->queues_list, list) {
3441	int q_idx = q_array_get_index(queue_id: q->properties.queue_id,
3442	num_queues,
3443	queue_ids);
3444
3445	if (q_idx != QUEUE_NOT_FOUND) {
3446	int err = suspend_single_queue(dqm, pdd, q);
3447	bool is_mes = dqm->dev->kfd->shared_resources.enable_mes;
3448
3449	if (!err) {
3450	queue_ids[q_idx] &= ~KFD_DBG_QUEUE_INVALID_MASK;
3451	if (exception_clear_mask && is_mes)
3452	q->properties.exception_status &=
3453	~exception_clear_mask;
3454
3455	if (is_mes)
3456	total_suspended++;
3457	else
3458	per_device_suspended++;
3459	} else if (err != -EBUSY) {
3460	r = err;
3461	queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3462	break;
3463	}
3464	}
3465	}
3466
3467	if (!per_device_suspended) {
3468	dqm_unlock(dqm);
3469	mutex_unlock(lock: &p->event_mutex);
3470	if (total_suspended)
3471	amdgpu_amdkfd_debug_mem_fence(adev: dqm->dev->adev);
3472	continue;
3473	}
3474
3475	r = execute_queues_cpsch(dqm,
3476	filter: KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, filter_param: 0,
3477	grace_period);
3478
3479	if (r)
3480	dev_err(dev, "Failed to suspend process queues.\n");
3481	else
3482	total_suspended += per_device_suspended;
3483
3484	list_for_each_entry(q, &qpd->queues_list, list) {
3485	int q_idx = q_array_get_index(queue_id: q->properties.queue_id,
3486	num_queues, queue_ids);
3487
3488	if (q_idx == QUEUE_NOT_FOUND)
3489	continue;
3490
3491	/* mask queue as error on suspend fail */
3492	if (r)
3493	queue_ids[q_idx] |= KFD_DBG_QUEUE_ERROR_MASK;
3494	else if (exception_clear_mask)
3495	q->properties.exception_status &=
3496	~exception_clear_mask;
3497	}
3498
3499	dqm_unlock(dqm);
3500	mutex_unlock(lock: &p->event_mutex);
3501	amdgpu_device_flush_hdp(adev: dqm->dev->adev, NULL);
3502	}
3503
3504	if (total_suspended) {
3505	struct copy_context_work_handler_workarea copy_context_worker;
3506
3507	INIT_WORK_ONSTACK(
3508	&copy_context_worker.copy_context_work,
3509	copy_context_work_handler);
3510
3511	copy_context_worker.p = p;
3512
3513	schedule_work(work: &copy_context_worker.copy_context_work);
3514
3515
3516	flush_work(work: &copy_context_worker.copy_context_work);
3517	destroy_work_on_stack(work: &copy_context_worker.copy_context_work);
3518	}
3519
3520	if (copy_to_user(to: (void __user *)usr_queue_id_array, from: queue_ids,
3521	n: num_queues * sizeof(uint32_t)))
3522	pr_err("copy_to_user failed on queue suspend\n");
3523
3524	kfree(objp: queue_ids);
3525
3526	return total_suspended;
3527	}
3528
3529	static uint32_t set_queue_type_for_user(struct queue_properties *q_props)
3530	{
3531	switch (q_props->type) {
3532	case KFD_QUEUE_TYPE_COMPUTE:
3533	return q_props->format == KFD_QUEUE_FORMAT_PM4
3534	? KFD_IOC_QUEUE_TYPE_COMPUTE
3535	: KFD_IOC_QUEUE_TYPE_COMPUTE_AQL;
3536	case KFD_QUEUE_TYPE_SDMA:
3537	return KFD_IOC_QUEUE_TYPE_SDMA;
3538	case KFD_QUEUE_TYPE_SDMA_XGMI:
3539	return KFD_IOC_QUEUE_TYPE_SDMA_XGMI;
3540	default:
3541	WARN_ONCE(true, "queue type not recognized!");
3542	return 0xffffffff;
3543	};
3544	}
3545
3546	void set_queue_snapshot_entry(struct queue *q,
3547	uint64_t exception_clear_mask,
3548	struct kfd_queue_snapshot_entry *qss_entry)
3549	{
3550	qss_entry->ring_base_address = q->properties.queue_address;
3551	qss_entry->write_pointer_address = (uint64_t)q->properties.write_ptr;
3552	qss_entry->read_pointer_address = (uint64_t)q->properties.read_ptr;
3553	qss_entry->ctx_save_restore_address =
3554	q->properties.ctx_save_restore_area_address;
3555	qss_entry->ctx_save_restore_area_size =
3556	q->properties.ctx_save_restore_area_size;
3557	qss_entry->exception_status = q->properties.exception_status;
3558	qss_entry->queue_id = q->properties.queue_id;
3559	qss_entry->gpu_id = q->device->id;
3560	qss_entry->ring_size = (uint32_t)q->properties.queue_size;
3561	qss_entry->queue_type = set_queue_type_for_user(&q->properties);
3562	q->properties.exception_status &= ~exception_clear_mask;
3563	}
3564
3565	int debug_lock_and_unmap(struct device_queue_manager *dqm)
3566	{
3567	struct device *dev = dqm->dev->adev->dev;
3568	int r;
3569
3570	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3571	dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3572	return -EINVAL;
3573	}
3574
3575	if (!kfd_dbg_is_per_vmid_supported(dev: dqm->dev))
3576	return 0;
3577
3578	dqm_lock(dqm);
3579
3580	r = unmap_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, filter_param: 0, grace_period: 0, reset: false);
3581	if (r)
3582	dqm_unlock(dqm);
3583
3584	return r;
3585	}
3586
3587	int debug_map_and_unlock(struct device_queue_manager *dqm)
3588	{
3589	struct device *dev = dqm->dev->adev->dev;
3590	int r;
3591
3592	if (dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
3593	dev_err(dev, "Unsupported on sched_policy: %i\n", dqm->sched_policy);
3594	return -EINVAL;
3595	}
3596
3597	if (!kfd_dbg_is_per_vmid_supported(dev: dqm->dev))
3598	return 0;
3599
3600	r = map_queues_cpsch(dqm);
3601
3602	dqm_unlock(dqm);
3603
3604	return r;
3605	}
3606
3607	int debug_refresh_runlist(struct device_queue_manager *dqm)
3608	{
3609	int r = debug_lock_and_unmap(dqm);
3610
3611	if (r)
3612	return r;
3613
3614	return debug_map_and_unlock(dqm);
3615	}
3616
3617	bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,
3618	struct qcm_process_device *qpd,
3619	int doorbell_off, u32 *queue_format)
3620	{
3621	struct queue *q;
3622	bool r = false;
3623
3624	if (!queue_format)
3625	return r;
3626
3627	dqm_lock(dqm);
3628
3629	list_for_each_entry(q, &qpd->queues_list, list) {
3630	if (q->properties.doorbell_off == doorbell_off) {
3631	*queue_format = q->properties.format;
3632	r = true;
3633	goto out;
3634	}
3635	}
3636
3637	out:
3638	dqm_unlock(dqm);
3639	return r;
3640	}
3641	#if defined(CONFIG_DEBUG_FS)
3642
3643	static void seq_reg_dump(struct seq_file *m,
3644	uint32_t (*dump)[2], uint32_t n_regs)
3645	{
3646	uint32_t i, count;
3647
3648	for (i = 0, count = 0; i < n_regs; i++) {
3649	if (count == 0 ||
3650	dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
3651	seq_printf(m, fmt: "%s %08x: %08x",
3652	i ? "\n" : "",
3653	dump[i][0], dump[i][1]);
3654	count = 7;
3655	} else {
3656	seq_printf(m, fmt: " %08x", dump[i][1]);
3657	count--;
3658	}
3659	}
3660
3661	seq_puts(m, s: "\n");
3662	}
3663
3664	int dqm_debugfs_hqds(struct seq_file *m, void *data)
3665	{
3666	struct device_queue_manager *dqm = data;
3667	uint32_t xcc_mask = dqm->dev->xcc_mask;
3668	uint32_t (*dump)[2], n_regs;
3669	int pipe, queue;
3670	int r = 0, xcc_id;
3671	uint32_t sdma_engine_start;
3672
3673	if (!dqm->sched_running) {
3674	seq_puts(m, s: " Device is stopped\n");
3675	return 0;
3676	}
3677
3678	for_each_inst(xcc_id, xcc_mask) {
3679	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3680	KFD_CIK_HIQ_PIPE,
3681	KFD_CIK_HIQ_QUEUE, &dump,
3682	&n_regs, xcc_id);
3683	if (!r) {
3684	seq_printf(
3685	m,
3686	fmt: " Inst %d, HIQ on MEC %d Pipe %d Queue %d\n",
3687	xcc_id,
3688	KFD_CIK_HIQ_PIPE / get_pipes_per_mec(dqm) + 1,
3689	KFD_CIK_HIQ_PIPE % get_pipes_per_mec(dqm),
3690	KFD_CIK_HIQ_QUEUE);
3691	seq_reg_dump(m, dump, n_regs);
3692
3693	kfree(objp: dump);
3694	}
3695
3696	for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
3697	int pipe_offset = pipe * get_queues_per_pipe(dqm);
3698
3699	for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
3700	if (!test_bit(pipe_offset + queue,
3701	dqm->dev->kfd->shared_resources.cp_queue_bitmap))
3702	continue;
3703
3704	r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
3705	pipe, queue,
3706	&dump, &n_regs,
3707	xcc_id);
3708	if (r)
3709	break;
3710
3711	seq_printf(m,
3712	fmt: " Inst %d, CP Pipe %d, Queue %d\n",
3713	xcc_id, pipe, queue);
3714	seq_reg_dump(m, dump, n_regs);
3715
3716	kfree(objp: dump);
3717	}
3718	}
3719	}
3720
3721	sdma_engine_start = dqm->dev->node_id * get_num_all_sdma_engines(dqm);
3722	for (pipe = sdma_engine_start;
3723	pipe < (sdma_engine_start + get_num_all_sdma_engines(dqm));
3724	pipe++) {
3725	for (queue = 0;
3726	queue < dqm->dev->kfd->device_info.num_sdma_queues_per_engine;
3727	queue++) {
3728	r = dqm->dev->kfd2kgd->hqd_sdma_dump(
3729	dqm->dev->adev, pipe, queue, &dump, &n_regs);
3730	if (r)
3731	break;
3732
3733	seq_printf(m, fmt: " SDMA Engine %d, RLC %d\n",
3734	pipe, queue);
3735	seq_reg_dump(m, dump, n_regs);
3736
3737	kfree(objp: dump);
3738	}
3739	}
3740
3741	return r;
3742	}
3743
3744	int dqm_debugfs_hang_hws(struct device_queue_manager *dqm)
3745	{
3746	int r = 0;
3747
3748	dqm_lock(dqm);
3749	r = pm_debugfs_hang_hws(pm: &dqm->packet_mgr);
3750	if (r) {
3751	dqm_unlock(dqm);
3752	return r;
3753	}
3754	dqm->active_runlist = true;
3755	r = execute_queues_cpsch(dqm, filter: KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
3756	filter_param: 0, USE_DEFAULT_GRACE_PERIOD);
3757	dqm_unlock(dqm);
3758
3759	return r;
3760	}
3761
3762	#endif
3763