1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 * Christian König
28 */
29#include <linux/seq_file.h>
30#include <linux/slab.h>
31#include <linux/uaccess.h>
32#include <linux/debugfs.h>
33
34#include <drm/amdgpu_drm.h>
35#include "amdgpu.h"
36#include "amdgpu_ras_mgr.h"
37#include "atom.h"
38
39/*
40 * Rings
41 * Most engines on the GPU are fed via ring buffers. Ring
42 * buffers are areas of GPU accessible memory that the host
43 * writes commands into and the GPU reads commands out of.
44 * There is a rptr (read pointer) that determines where the
45 * GPU is currently reading, and a wptr (write pointer)
46 * which determines where the host has written. When the
47 * pointers are equal, the ring is idle. When the host
48 * writes commands to the ring buffer, it increments the
49 * wptr. The GPU then starts fetching commands and executes
50 * them until the pointers are equal again.
51 */
52
53/**
54 * amdgpu_ring_max_ibs - Return max IBs that fit in a single submission.
55 *
56 * @type: ring type for which to return the limit.
57 */
58unsigned int amdgpu_ring_max_ibs(enum amdgpu_ring_type type)
59{
60 switch (type) {
61 case AMDGPU_RING_TYPE_GFX:
62 /* Need to keep at least 192 on GFX7+ for old radv. */
63 return 192;
64 case AMDGPU_RING_TYPE_COMPUTE:
65 return 125;
66 case AMDGPU_RING_TYPE_VCN_JPEG:
67 return 16;
68 default:
69 return 49;
70 }
71}
72
73/**
74 * amdgpu_ring_alloc - allocate space on the ring buffer
75 *
76 * @ring: amdgpu_ring structure holding ring information
77 * @ndw: number of dwords to allocate in the ring buffer
78 *
79 * Allocate @ndw dwords in the ring buffer (all asics).
80 * Returns 0 on success, error on failure.
81 */
82int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned int ndw)
83{
84 /* Align requested size with padding so unlock_commit can
85 * pad safely */
86 ndw = (ndw + ring->funcs->align_mask) & ~ring->funcs->align_mask;
87
88 /* Make sure we aren't trying to allocate more space
89 * than the maximum for one submission
90 */
91 if (WARN_ON_ONCE(ndw > ring->max_dw))
92 return -ENOMEM;
93
94 ring->count_dw = ndw;
95 ring->wptr_old = ring->wptr;
96
97 if (ring->funcs->begin_use)
98 ring->funcs->begin_use(ring);
99
100 return 0;
101}
102
103/**
104 * amdgpu_ring_alloc_reemit - allocate space on the ring buffer for reemit
105 *
106 * @ring: amdgpu_ring structure holding ring information
107 * @ndw: number of dwords to allocate in the ring buffer
108 *
109 * Allocate @ndw dwords in the ring buffer (all asics).
110 * doesn't check the max_dw limit as we may be reemitting
111 * several submissions.
112 */
113static void amdgpu_ring_alloc_reemit(struct amdgpu_ring *ring, unsigned int ndw)
114{
115 /* Align requested size with padding so unlock_commit can
116 * pad safely */
117 ndw = (ndw + ring->funcs->align_mask) & ~ring->funcs->align_mask;
118
119 ring->count_dw = ndw;
120 ring->wptr_old = ring->wptr;
121
122 if (ring->funcs->begin_use)
123 ring->funcs->begin_use(ring);
124}
125
126/** amdgpu_ring_insert_nop - insert NOP packets
127 *
128 * @ring: amdgpu_ring structure holding ring information
129 * @count: the number of NOP packets to insert
130 *
131 * This is the generic insert_nop function for rings except SDMA
132 */
133void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
134{
135 uint32_t occupied, chunk1, chunk2;
136
137 occupied = ring->wptr & ring->buf_mask;
138 chunk1 = ring->buf_mask + 1 - occupied;
139 chunk1 = (chunk1 >= count) ? count : chunk1;
140 chunk2 = count - chunk1;
141
142 if (chunk1)
143 memset32(s: &ring->ring[occupied], v: ring->funcs->nop, n: chunk1);
144
145 if (chunk2)
146 memset32(s: ring->ring, v: ring->funcs->nop, n: chunk2);
147
148 ring->wptr += count;
149 ring->wptr &= ring->ptr_mask;
150 ring->count_dw -= count;
151}
152
153/**
154 * amdgpu_ring_generic_pad_ib - pad IB with NOP packets
155 *
156 * @ring: amdgpu_ring structure holding ring information
157 * @ib: IB to add NOP packets to
158 *
159 * This is the generic pad_ib function for rings except SDMA
160 */
161void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
162{
163 u32 align_mask = ring->funcs->align_mask;
164 u32 count = ib->length_dw & align_mask;
165
166 if (count) {
167 count = align_mask + 1 - count;
168
169 memset32(s: &ib->ptr[ib->length_dw], v: ring->funcs->nop, n: count);
170
171 ib->length_dw += count;
172 }
173}
174
175/**
176 * amdgpu_ring_commit - tell the GPU to execute the new
177 * commands on the ring buffer
178 *
179 * @ring: amdgpu_ring structure holding ring information
180 *
181 * Update the wptr (write pointer) to tell the GPU to
182 * execute new commands on the ring buffer (all asics).
183 */
184void amdgpu_ring_commit(struct amdgpu_ring *ring)
185{
186 uint32_t count;
187
188 if (ring->count_dw < 0)
189 DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
190
191 /* We pad to match fetch size */
192 count = ring->funcs->align_mask + 1 -
193 (ring->wptr & ring->funcs->align_mask);
194 count &= ring->funcs->align_mask;
195
196 if (count != 0)
197 ring->funcs->insert_nop(ring, count);
198
199 mb();
200 amdgpu_ring_set_wptr(ring);
201
202 if (ring->funcs->end_use)
203 ring->funcs->end_use(ring);
204}
205
206/**
207 * amdgpu_ring_undo - reset the wptr
208 *
209 * @ring: amdgpu_ring structure holding ring information
210 *
211 * Reset the driver's copy of the wptr (all asics).
212 */
213void amdgpu_ring_undo(struct amdgpu_ring *ring)
214{
215 ring->wptr = ring->wptr_old;
216
217 if (ring->funcs->end_use)
218 ring->funcs->end_use(ring);
219}
220
221#define amdgpu_ring_get_gpu_addr(ring, offset) \
222 (ring->adev->wb.gpu_addr + offset * 4)
223
224#define amdgpu_ring_get_cpu_addr(ring, offset) \
225 (&ring->adev->wb.wb[offset])
226
227/**
228 * amdgpu_ring_init - init driver ring struct.
229 *
230 * @adev: amdgpu_device pointer
231 * @ring: amdgpu_ring structure holding ring information
232 * @max_dw: maximum number of dw for ring alloc
233 * @irq_src: interrupt source to use for this ring
234 * @irq_type: interrupt type to use for this ring
235 * @hw_prio: ring priority (NORMAL/HIGH)
236 * @sched_score: optional score atomic shared with other schedulers
237 *
238 * Initialize the driver information for the selected ring (all asics).
239 * Returns 0 on success, error on failure.
240 */
241int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
242 unsigned int max_dw, struct amdgpu_irq_src *irq_src,
243 unsigned int irq_type, unsigned int hw_prio,
244 atomic_t *sched_score)
245{
246 int r;
247 int sched_hw_submission = amdgpu_sched_hw_submission;
248 u32 *num_sched;
249 u32 hw_ip;
250 unsigned int max_ibs_dw;
251
252 /* Set the hw submission limit higher for KIQ because
253 * it's used for a number of gfx/compute tasks by both
254 * KFD and KGD which may have outstanding fences and
255 * it doesn't really use the gpu scheduler anyway;
256 * KIQ tasks get submitted directly to the ring.
257 */
258 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
259 sched_hw_submission = max(sched_hw_submission, 256);
260 if (ring->funcs->type == AMDGPU_RING_TYPE_MES)
261 sched_hw_submission = 8;
262 else if (ring == &adev->sdma.instance[0].page)
263 sched_hw_submission = 256;
264
265 if (ring->adev == NULL) {
266 if (adev->num_rings >= AMDGPU_MAX_RINGS)
267 return -EINVAL;
268
269 ring->adev = adev;
270 ring->num_hw_submission = sched_hw_submission;
271 ring->sched_score = sched_score;
272 ring->vmid_wait = dma_fence_get_stub();
273
274 ring->idx = adev->num_rings++;
275 adev->rings[ring->idx] = ring;
276
277 r = amdgpu_fence_driver_init_ring(ring);
278 if (r)
279 return r;
280 }
281
282 r = amdgpu_device_wb_get(adev, wb: &ring->rptr_offs);
283 if (r) {
284 dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r);
285 return r;
286 }
287
288 r = amdgpu_device_wb_get(adev, wb: &ring->wptr_offs);
289 if (r) {
290 dev_err(adev->dev, "(%d) ring wptr_offs wb alloc failed\n", r);
291 return r;
292 }
293
294 r = amdgpu_device_wb_get(adev, wb: &ring->fence_offs);
295 if (r) {
296 dev_err(adev->dev, "(%d) ring fence_offs wb alloc failed\n", r);
297 return r;
298 }
299
300 r = amdgpu_device_wb_get(adev, wb: &ring->trail_fence_offs);
301 if (r) {
302 dev_err(adev->dev, "(%d) ring trail_fence_offs wb alloc failed\n", r);
303 return r;
304 }
305
306 r = amdgpu_device_wb_get(adev, wb: &ring->cond_exe_offs);
307 if (r) {
308 dev_err(adev->dev, "(%d) ring cond_exec_polling wb alloc failed\n", r);
309 return r;
310 }
311
312 ring->fence_gpu_addr =
313 amdgpu_ring_get_gpu_addr(ring, ring->fence_offs);
314 ring->fence_cpu_addr =
315 amdgpu_ring_get_cpu_addr(ring, ring->fence_offs);
316
317 ring->rptr_gpu_addr =
318 amdgpu_ring_get_gpu_addr(ring, ring->rptr_offs);
319 ring->rptr_cpu_addr =
320 amdgpu_ring_get_cpu_addr(ring, ring->rptr_offs);
321
322 ring->wptr_gpu_addr =
323 amdgpu_ring_get_gpu_addr(ring, ring->wptr_offs);
324 ring->wptr_cpu_addr =
325 amdgpu_ring_get_cpu_addr(ring, ring->wptr_offs);
326
327 ring->trail_fence_gpu_addr =
328 amdgpu_ring_get_gpu_addr(ring, ring->trail_fence_offs);
329 ring->trail_fence_cpu_addr =
330 amdgpu_ring_get_cpu_addr(ring, ring->trail_fence_offs);
331
332 ring->cond_exe_gpu_addr =
333 amdgpu_ring_get_gpu_addr(ring, ring->cond_exe_offs);
334 ring->cond_exe_cpu_addr =
335 amdgpu_ring_get_cpu_addr(ring, ring->cond_exe_offs);
336
337 /* always set cond_exec_polling to CONTINUE */
338 *ring->cond_exe_cpu_addr = 1;
339
340 if (ring->funcs->type != AMDGPU_RING_TYPE_CPER) {
341 r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type);
342 if (r) {
343 dev_err(adev->dev, "failed initializing fences (%d).\n", r);
344 return r;
345 }
346
347 max_ibs_dw = ring->funcs->emit_frame_size +
348 amdgpu_ring_max_ibs(type: ring->funcs->type) * ring->funcs->emit_ib_size;
349 max_ibs_dw = (max_ibs_dw + ring->funcs->align_mask) & ~ring->funcs->align_mask;
350
351 if (WARN_ON(max_ibs_dw > max_dw))
352 max_dw = max_ibs_dw;
353
354 ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission);
355 } else {
356 ring->ring_size = roundup_pow_of_two(max_dw * 4);
357 ring->count_dw = (ring->ring_size - 4) >> 2;
358 /* ring buffer is empty now */
359 ring->wptr = *ring->rptr_cpu_addr = 0;
360 }
361
362 ring->buf_mask = (ring->ring_size / 4) - 1;
363 ring->ptr_mask = ring->funcs->support_64bit_ptrs ?
364 0xffffffffffffffff : ring->buf_mask;
365 /* Initialize cached_rptr to 0 */
366 ring->cached_rptr = 0;
367
368 if (!ring->ring_backup) {
369 ring->ring_backup = kvzalloc(ring->ring_size, GFP_KERNEL);
370 if (!ring->ring_backup)
371 return -ENOMEM;
372 }
373
374 /* Allocate ring buffer */
375 if (ring->ring_obj == NULL) {
376 r = amdgpu_bo_create_kernel(adev, size: ring->ring_size + ring->funcs->extra_bytes,
377 PAGE_SIZE,
378 AMDGPU_GEM_DOMAIN_GTT,
379 bo_ptr: &ring->ring_obj,
380 gpu_addr: &ring->gpu_addr,
381 cpu_addr: (void **)&ring->ring);
382 if (r) {
383 dev_err(adev->dev, "(%d) ring create failed\n", r);
384 kvfree(addr: ring->ring_backup);
385 return r;
386 }
387 amdgpu_ring_clear_ring(ring);
388 }
389
390 ring->max_dw = max_dw;
391 ring->hw_prio = hw_prio;
392
393 if (!ring->no_scheduler && ring->funcs->type < AMDGPU_HW_IP_NUM) {
394 hw_ip = ring->funcs->type;
395 num_sched = &adev->gpu_sched[hw_ip][hw_prio].num_scheds;
396 adev->gpu_sched[hw_ip][hw_prio].sched[(*num_sched)++] =
397 &ring->sched;
398 }
399
400 return 0;
401}
402
403/**
404 * amdgpu_ring_fini - tear down the driver ring struct.
405 *
406 * @ring: amdgpu_ring structure holding ring information
407 *
408 * Tear down the driver information for the selected ring (all asics).
409 */
410void amdgpu_ring_fini(struct amdgpu_ring *ring)
411{
412
413 /* Not to finish a ring which is not initialized */
414 if (!(ring->adev) || !(ring->adev->rings[ring->idx]))
415 return;
416
417 ring->sched.ready = false;
418
419 amdgpu_device_wb_free(adev: ring->adev, wb: ring->rptr_offs);
420 amdgpu_device_wb_free(adev: ring->adev, wb: ring->wptr_offs);
421
422 amdgpu_device_wb_free(adev: ring->adev, wb: ring->cond_exe_offs);
423 amdgpu_device_wb_free(adev: ring->adev, wb: ring->fence_offs);
424
425 amdgpu_bo_free_kernel(bo: &ring->ring_obj,
426 gpu_addr: &ring->gpu_addr,
427 cpu_addr: (void **)&ring->ring);
428 kvfree(addr: ring->ring_backup);
429 ring->ring_backup = NULL;
430
431 dma_fence_put(fence: ring->vmid_wait);
432 ring->vmid_wait = NULL;
433 ring->me = 0;
434}
435
436/**
437 * amdgpu_ring_emit_reg_write_reg_wait_helper - ring helper
438 *
439 * @ring: ring to write to
440 * @reg0: register to write
441 * @reg1: register to wait on
442 * @ref: reference value to write/wait on
443 * @mask: mask to wait on
444 *
445 * Helper for rings that don't support write and wait in a
446 * single oneshot packet.
447 */
448void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring,
449 uint32_t reg0, uint32_t reg1,
450 uint32_t ref, uint32_t mask)
451{
452 amdgpu_ring_emit_wreg(ring, reg0, ref);
453 amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask);
454}
455
456/**
457 * amdgpu_ring_soft_recovery - try to soft recover a ring lockup
458 *
459 * @ring: ring to try the recovery on
460 * @vmid: VMID we try to get going again
461 * @fence: timedout fence
462 *
463 * Tries to get a ring proceeding again when it is stuck.
464 */
465bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid,
466 struct dma_fence *fence)
467{
468 unsigned long flags;
469 ktime_t deadline;
470 bool ret;
471
472 deadline = ktime_add_us(kt: ktime_get(), usec: 10000);
473
474 if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence)
475 return false;
476
477 spin_lock_irqsave(fence->lock, flags);
478 if (!dma_fence_is_signaled_locked(fence))
479 dma_fence_set_error(fence, error: -ENODATA);
480 spin_unlock_irqrestore(lock: fence->lock, flags);
481
482 while (!dma_fence_is_signaled(fence) &&
483 ktime_to_ns(ktime_sub(deadline, ktime_get())) > 0)
484 ring->funcs->soft_recovery(ring, vmid);
485
486 ret = dma_fence_is_signaled(fence);
487 /* increment the counter only if soft reset worked */
488 if (ret)
489 atomic_inc(v: &ring->adev->gpu_reset_counter);
490
491 return ret;
492}
493
494/*
495 * Debugfs info
496 */
497#if defined(CONFIG_DEBUG_FS)
498
499static ssize_t amdgpu_ras_cper_debugfs_read(struct file *f, char __user *buf,
500 size_t size, loff_t *offset)
501{
502 const uint8_t ring_header_size = 12;
503 struct amdgpu_ring *ring = file_inode(f)->i_private;
504 struct ras_cmd_cper_snapshot_req *snapshot_req __free(kfree) =
505 kzalloc(sizeof(struct ras_cmd_cper_snapshot_req), GFP_KERNEL);
506 struct ras_cmd_cper_snapshot_rsp *snapshot_rsp __free(kfree) =
507 kzalloc(sizeof(struct ras_cmd_cper_snapshot_rsp), GFP_KERNEL);
508 struct ras_cmd_cper_record_req *record_req __free(kfree) =
509 kzalloc(sizeof(struct ras_cmd_cper_record_req), GFP_KERNEL);
510 struct ras_cmd_cper_record_rsp *record_rsp __free(kfree) =
511 kzalloc(sizeof(struct ras_cmd_cper_record_rsp), GFP_KERNEL);
512 uint8_t *ring_header __free(kfree) =
513 kzalloc(ring_header_size, GFP_KERNEL);
514 uint32_t total_cper_num;
515 uint64_t start_cper_id;
516 int r;
517
518 if (!snapshot_req || !snapshot_rsp || !record_req || !record_rsp ||
519 !ring_header)
520 return -ENOMEM;
521
522 if (!(*offset)) {
523 /* Need at least 12 bytes for the header on the first read */
524 if (size < ring_header_size)
525 return -EINVAL;
526
527 if (copy_to_user(to: buf, from: ring_header, n: ring_header_size))
528 return -EFAULT;
529 buf += ring_header_size;
530 size -= ring_header_size;
531 }
532
533 r = amdgpu_ras_mgr_handle_ras_cmd(adev: ring->adev,
534 cmd_id: RAS_CMD__GET_CPER_SNAPSHOT,
535 input: snapshot_req, input_size: sizeof(struct ras_cmd_cper_snapshot_req),
536 output: snapshot_rsp, out_size: sizeof(struct ras_cmd_cper_snapshot_rsp));
537 if (r || !snapshot_rsp->total_cper_num)
538 return r;
539
540 start_cper_id = snapshot_rsp->start_cper_id;
541 total_cper_num = snapshot_rsp->total_cper_num;
542
543 record_req->buf_ptr = (uint64_t)(uintptr_t)buf;
544 record_req->buf_size = size;
545 record_req->cper_start_id = start_cper_id + *offset;
546 record_req->cper_num = total_cper_num;
547 r = amdgpu_ras_mgr_handle_ras_cmd(adev: ring->adev, cmd_id: RAS_CMD__GET_CPER_RECORD,
548 input: record_req, input_size: sizeof(struct ras_cmd_cper_record_req),
549 output: record_rsp, out_size: sizeof(struct ras_cmd_cper_record_rsp));
550 if (r)
551 return r;
552
553 r = *offset ? record_rsp->real_data_size : record_rsp->real_data_size + ring_header_size;
554 (*offset) += record_rsp->real_cper_num;
555
556 return r;
557}
558
559/* Layout of file is 12 bytes consisting of
560 * - rptr
561 * - wptr
562 * - driver's copy of wptr
563 *
564 * followed by n-words of ring data
565 */
566static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf,
567 size_t size, loff_t *pos)
568{
569 struct amdgpu_ring *ring = file_inode(f)->i_private;
570 uint32_t value, result, early[3];
571 uint64_t p;
572 loff_t i;
573 int r;
574
575 if (ring->funcs->type == AMDGPU_RING_TYPE_CPER && amdgpu_uniras_enabled(adev: ring->adev))
576 return amdgpu_ras_cper_debugfs_read(f, buf, size, offset: pos);
577
578 if (*pos & 3 || size & 3)
579 return -EINVAL;
580
581 result = 0;
582
583 if (*pos < 12) {
584 if (ring->funcs->type == AMDGPU_RING_TYPE_CPER)
585 mutex_lock(&ring->adev->cper.ring_lock);
586
587 early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask;
588 early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask;
589 early[2] = ring->wptr & ring->buf_mask;
590 for (i = *pos / 4; i < 3 && size; i++) {
591 r = put_user(early[i], (uint32_t *)buf);
592 if (r) {
593 result = r;
594 goto out;
595 }
596 buf += 4;
597 result += 4;
598 size -= 4;
599 *pos += 4;
600 }
601 }
602
603 if (ring->funcs->type != AMDGPU_RING_TYPE_CPER) {
604 while (size) {
605 if (*pos >= (ring->ring_size + 12))
606 return result;
607
608 value = ring->ring[(*pos - 12)/4];
609 r = put_user(value, (uint32_t *)buf);
610 if (r)
611 return r;
612 buf += 4;
613 result += 4;
614 size -= 4;
615 *pos += 4;
616 }
617 } else {
618 p = early[0];
619 if (early[0] <= early[1])
620 size = (early[1] - early[0]);
621 else
622 size = ring->ring_size - (early[0] - early[1]);
623
624 while (size) {
625 if (p == early[1])
626 goto out;
627
628 value = ring->ring[p];
629 r = put_user(value, (uint32_t *)buf);
630 if (r) {
631 result = r;
632 goto out;
633 }
634
635 buf += 4;
636 result += 4;
637 size--;
638 p++;
639 p &= ring->ptr_mask;
640 }
641 }
642
643out:
644 if (ring->funcs->type == AMDGPU_RING_TYPE_CPER)
645 mutex_unlock(lock: &ring->adev->cper.ring_lock);
646
647 return result;
648}
649
650static ssize_t amdgpu_debugfs_virt_ring_read(struct file *f, char __user *buf,
651 size_t size, loff_t *pos)
652{
653 struct amdgpu_ring *ring = file_inode(f)->i_private;
654
655 if (*pos & 3 || size & 3)
656 return -EINVAL;
657
658 if (ring->funcs->type == AMDGPU_RING_TYPE_CPER)
659 amdgpu_virt_req_ras_cper_dump(adev: ring->adev, force_update: false);
660
661 return amdgpu_debugfs_ring_read(f, buf, size, pos);
662}
663
664static const struct file_operations amdgpu_debugfs_ring_fops = {
665 .owner = THIS_MODULE,
666 .read = amdgpu_debugfs_ring_read,
667 .llseek = default_llseek
668};
669
670static const struct file_operations amdgpu_debugfs_virt_ring_fops = {
671 .owner = THIS_MODULE,
672 .read = amdgpu_debugfs_virt_ring_read,
673 .llseek = default_llseek
674};
675
676static ssize_t amdgpu_debugfs_mqd_read(struct file *f, char __user *buf,
677 size_t size, loff_t *pos)
678{
679 struct amdgpu_ring *ring = file_inode(f)->i_private;
680 ssize_t bytes = min_t(ssize_t, ring->mqd_size - *pos, size);
681 void *from = ((u8 *)ring->mqd_ptr) + *pos;
682
683 if (*pos > ring->mqd_size)
684 return 0;
685
686 if (copy_to_user(to: buf, from, n: bytes))
687 return -EFAULT;
688
689 *pos += bytes;
690 return bytes;
691}
692
693static const struct file_operations amdgpu_debugfs_mqd_fops = {
694 .owner = THIS_MODULE,
695 .read = amdgpu_debugfs_mqd_read,
696 .llseek = default_llseek
697};
698
699static int amdgpu_debugfs_ring_error(void *data, u64 val)
700{
701 struct amdgpu_ring *ring = data;
702
703 amdgpu_fence_driver_set_error(ring, error: val);
704 return 0;
705}
706
707DEFINE_DEBUGFS_ATTRIBUTE_SIGNED(amdgpu_debugfs_error_fops, NULL,
708 amdgpu_debugfs_ring_error, "%lld\n");
709
710#endif
711
712void amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
713 struct amdgpu_ring *ring)
714{
715#if defined(CONFIG_DEBUG_FS)
716 struct drm_minor *minor = adev_to_drm(adev)->primary;
717 struct dentry *root = minor->debugfs_root;
718 char name[32];
719
720 sprintf(buf: name, fmt: "amdgpu_ring_%s", ring->name);
721 if (amdgpu_sriov_vf(adev))
722 debugfs_create_file_size(name, S_IFREG | 0444, parent: root, data: ring,
723 fops: &amdgpu_debugfs_virt_ring_fops,
724 file_size: ring->ring_size + 12);
725 else
726 debugfs_create_file_size(name, S_IFREG | 0444, parent: root, data: ring,
727 fops: &amdgpu_debugfs_ring_fops,
728 file_size: ring->ring_size + 12);
729
730 if (ring->mqd_obj) {
731 sprintf(buf: name, fmt: "amdgpu_mqd_%s", ring->name);
732 debugfs_create_file_size(name, S_IFREG | 0444, parent: root, data: ring,
733 fops: &amdgpu_debugfs_mqd_fops,
734 file_size: ring->mqd_size);
735 }
736
737 sprintf(buf: name, fmt: "amdgpu_error_%s", ring->name);
738 debugfs_create_file(name, 0200, root, ring,
739 &amdgpu_debugfs_error_fops);
740
741#endif
742}
743
744/**
745 * amdgpu_ring_test_helper - tests ring and set sched readiness status
746 *
747 * @ring: ring to try the recovery on
748 *
749 * Tests ring and set sched readiness status
750 *
751 * Returns 0 on success, error on failure.
752 */
753int amdgpu_ring_test_helper(struct amdgpu_ring *ring)
754{
755 struct amdgpu_device *adev = ring->adev;
756 int r;
757
758 r = amdgpu_ring_test_ring(ring);
759 if (r)
760 DRM_DEV_ERROR(adev->dev, "ring %s test failed (%d)\n",
761 ring->name, r);
762 else
763 DRM_DEV_DEBUG(adev->dev, "ring test on %s succeeded\n",
764 ring->name);
765
766 ring->sched.ready = !r;
767
768 return r;
769}
770
771static void amdgpu_ring_to_mqd_prop(struct amdgpu_ring *ring,
772 struct amdgpu_mqd_prop *prop)
773{
774 struct amdgpu_device *adev = ring->adev;
775 bool is_high_prio_compute = ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE &&
776 amdgpu_gfx_is_high_priority_compute_queue(adev, ring);
777 bool is_high_prio_gfx = ring->funcs->type == AMDGPU_RING_TYPE_GFX &&
778 amdgpu_gfx_is_high_priority_graphics_queue(adev, ring);
779
780 memset(prop, 0, sizeof(*prop));
781
782 prop->mqd_gpu_addr = ring->mqd_gpu_addr;
783 prop->hqd_base_gpu_addr = ring->gpu_addr;
784 prop->rptr_gpu_addr = ring->rptr_gpu_addr;
785 prop->wptr_gpu_addr = ring->wptr_gpu_addr;
786 prop->queue_size = ring->ring_size;
787 prop->eop_gpu_addr = ring->eop_gpu_addr;
788 prop->use_doorbell = ring->use_doorbell;
789 prop->doorbell_index = ring->doorbell_index;
790 prop->kernel_queue = true;
791
792 /* map_queues packet doesn't need activate the queue,
793 * so only kiq need set this field.
794 */
795 prop->hqd_active = ring->funcs->type == AMDGPU_RING_TYPE_KIQ;
796
797 prop->allow_tunneling = is_high_prio_compute;
798 if (is_high_prio_compute || is_high_prio_gfx) {
799 prop->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
800 prop->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
801 }
802}
803
804int amdgpu_ring_init_mqd(struct amdgpu_ring *ring)
805{
806 struct amdgpu_device *adev = ring->adev;
807 struct amdgpu_mqd *mqd_mgr;
808 struct amdgpu_mqd_prop prop;
809
810 amdgpu_ring_to_mqd_prop(ring, prop: &prop);
811
812 ring->wptr = 0;
813
814 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
815 mqd_mgr = &adev->mqds[AMDGPU_HW_IP_COMPUTE];
816 else
817 mqd_mgr = &adev->mqds[ring->funcs->type];
818
819 return mqd_mgr->init_mqd(adev, ring->mqd_ptr, &prop);
820}
821
822void amdgpu_ring_ib_begin(struct amdgpu_ring *ring)
823{
824 if (ring->is_sw_ring)
825 amdgpu_sw_ring_ib_begin(ring);
826}
827
828void amdgpu_ring_ib_end(struct amdgpu_ring *ring)
829{
830 if (ring->is_sw_ring)
831 amdgpu_sw_ring_ib_end(ring);
832}
833
834void amdgpu_ring_ib_on_emit_cntl(struct amdgpu_ring *ring)
835{
836 if (ring->is_sw_ring)
837 amdgpu_sw_ring_ib_mark_offset(ring, type: AMDGPU_MUX_OFFSET_TYPE_CONTROL);
838}
839
840void amdgpu_ring_ib_on_emit_ce(struct amdgpu_ring *ring)
841{
842 if (ring->is_sw_ring)
843 amdgpu_sw_ring_ib_mark_offset(ring, type: AMDGPU_MUX_OFFSET_TYPE_CE);
844}
845
846void amdgpu_ring_ib_on_emit_de(struct amdgpu_ring *ring)
847{
848 if (ring->is_sw_ring)
849 amdgpu_sw_ring_ib_mark_offset(ring, type: AMDGPU_MUX_OFFSET_TYPE_DE);
850}
851
852bool amdgpu_ring_sched_ready(struct amdgpu_ring *ring)
853{
854 if (!ring)
855 return false;
856
857 if (ring->no_scheduler || !drm_sched_wqueue_ready(sched: &ring->sched))
858 return false;
859
860 return true;
861}
862
863void amdgpu_ring_reset_helper_begin(struct amdgpu_ring *ring,
864 struct amdgpu_fence *guilty_fence)
865{
866 /* Stop the scheduler to prevent anybody else from touching the ring buffer. */
867 drm_sched_wqueue_stop(sched: &ring->sched);
868 /* back up the non-guilty commands */
869 amdgpu_ring_backup_unprocessed_commands(ring, guilty_fence);
870}
871
872int amdgpu_ring_reset_helper_end(struct amdgpu_ring *ring,
873 struct amdgpu_fence *guilty_fence)
874{
875 unsigned int i;
876 int r;
877
878 /* verify that the ring is functional */
879 r = amdgpu_ring_test_ring(ring);
880 if (r)
881 return r;
882
883 /* signal the guilty fence and set an error on all fences from the context */
884 if (guilty_fence)
885 amdgpu_fence_driver_guilty_force_completion(af: guilty_fence);
886 /* Re-emit the non-guilty commands */
887 if (ring->ring_backup_entries_to_copy) {
888 amdgpu_ring_alloc_reemit(ring, ndw: ring->ring_backup_entries_to_copy);
889 for (i = 0; i < ring->ring_backup_entries_to_copy; i++)
890 amdgpu_ring_write(ring, v: ring->ring_backup[i]);
891 amdgpu_ring_commit(ring);
892 }
893 /* Start the scheduler again */
894 drm_sched_wqueue_start(sched: &ring->sched);
895 return 0;
896}
897
898bool amdgpu_ring_is_reset_type_supported(struct amdgpu_ring *ring,
899 u32 reset_type)
900{
901 switch (ring->funcs->type) {
902 case AMDGPU_RING_TYPE_GFX:
903 if (ring->adev->gfx.gfx_supported_reset & reset_type)
904 return true;
905 break;
906 case AMDGPU_RING_TYPE_COMPUTE:
907 if (ring->adev->gfx.compute_supported_reset & reset_type)
908 return true;
909 break;
910 case AMDGPU_RING_TYPE_SDMA:
911 if (ring->adev->sdma.supported_reset & reset_type)
912 return true;
913 break;
914 case AMDGPU_RING_TYPE_VCN_DEC:
915 case AMDGPU_RING_TYPE_VCN_ENC:
916 if (ring->adev->vcn.supported_reset & reset_type)
917 return true;
918 break;
919 case AMDGPU_RING_TYPE_VCN_JPEG:
920 if (ring->adev->jpeg.supported_reset & reset_type)
921 return true;
922 break;
923 default:
924 break;
925 }
926 return false;
927}
928

source code of linux/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c