1/*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26/*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Dave Airlie
30 */
31#include <linux/seq_file.h>
32#include <linux/atomic.h>
33#include <linux/wait.h>
34#include <linux/kref.h>
35#include <linux/slab.h>
36#include <linux/firmware.h>
37#include <linux/pm_runtime.h>
38
39#include <drm/drm_drv.h>
40#include "amdgpu.h"
41#include "amdgpu_trace.h"
42#include "amdgpu_reset.h"
43
44/*
45 * Cast helper
46 */
47static const struct dma_fence_ops amdgpu_fence_ops;
48static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f)
49{
50 struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base);
51
52 return __f;
53}
54
55/**
56 * amdgpu_fence_write - write a fence value
57 *
58 * @ring: ring the fence is associated with
59 * @seq: sequence number to write
60 *
61 * Writes a fence value to memory (all asics).
62 */
63static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
64{
65 struct amdgpu_fence_driver *drv = &ring->fence_drv;
66
67 if (drv->cpu_addr)
68 *drv->cpu_addr = cpu_to_le32(seq);
69}
70
71/**
72 * amdgpu_fence_read - read a fence value
73 *
74 * @ring: ring the fence is associated with
75 *
76 * Reads a fence value from memory (all asics).
77 * Returns the value of the fence read from memory.
78 */
79static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
80{
81 struct amdgpu_fence_driver *drv = &ring->fence_drv;
82 u32 seq = 0;
83
84 if (drv->cpu_addr)
85 seq = le32_to_cpu(*drv->cpu_addr);
86 else
87 seq = atomic_read(v: &drv->last_seq);
88
89 return seq;
90}
91
92static void amdgpu_fence_save_fence_wptr_start(struct amdgpu_fence *af)
93{
94 af->fence_wptr_start = af->ring->wptr;
95}
96
97static void amdgpu_fence_save_fence_wptr_end(struct amdgpu_fence *af)
98{
99 af->fence_wptr_end = af->ring->wptr;
100}
101
102/**
103 * amdgpu_fence_emit - emit a fence on the requested ring
104 *
105 * @ring: ring the fence is associated with
106 * @af: amdgpu fence input
107 * @flags: flags to pass into the subordinate .emit_fence() call
108 *
109 * Emits a fence command on the requested ring (all asics).
110 * Returns 0 on success, -ENOMEM on failure.
111 */
112int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,
113 unsigned int flags)
114{
115 struct amdgpu_device *adev = ring->adev;
116 struct dma_fence *fence;
117 struct dma_fence __rcu **ptr;
118 uint32_t seq;
119 int r;
120
121 fence = &af->base;
122 af->ring = ring;
123
124 seq = ++ring->fence_drv.sync_seq;
125 dma_fence_init(fence, ops: &amdgpu_fence_ops,
126 lock: &ring->fence_drv.lock,
127 context: adev->fence_context + ring->idx, seqno: seq);
128
129 amdgpu_fence_save_fence_wptr_start(af);
130 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
131 seq, flags | AMDGPU_FENCE_FLAG_INT);
132 amdgpu_fence_save_fence_wptr_end(af);
133 amdgpu_fence_save_wptr(af);
134 pm_runtime_get_noresume(dev: adev_to_drm(adev)->dev);
135 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
136 if (unlikely(rcu_dereference_protected(*ptr, 1))) {
137 struct dma_fence *old;
138
139 rcu_read_lock();
140 old = dma_fence_get_rcu_safe(fencep: ptr);
141 rcu_read_unlock();
142
143 if (old) {
144 r = dma_fence_wait(fence: old, intr: false);
145 dma_fence_put(fence: old);
146 if (r)
147 return r;
148 }
149 }
150
151 to_amdgpu_fence(f: fence)->start_timestamp = ktime_get();
152
153 /* This function can't be called concurrently anyway, otherwise
154 * emitting the fence would mess up the hardware ring buffer.
155 */
156 rcu_assign_pointer(*ptr, dma_fence_get(fence));
157
158 return 0;
159}
160
161/**
162 * amdgpu_fence_emit_polling - emit a fence on the requeste ring
163 *
164 * @ring: ring the fence is associated with
165 * @s: resulting sequence number
166 * @timeout: the timeout for waiting in usecs
167 *
168 * Emits a fence command on the requested ring (all asics).
169 * Used For polling fence.
170 * Returns 0 on success, -ENOMEM on failure.
171 */
172int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
173 uint32_t timeout)
174{
175 uint32_t seq;
176 signed long r;
177
178 if (!s)
179 return -EINVAL;
180
181 seq = ++ring->fence_drv.sync_seq;
182 r = amdgpu_fence_wait_polling(ring,
183 wait_seq: seq - ring->fence_drv.num_fences_mask,
184 timeout);
185 if (r < 1)
186 return -ETIMEDOUT;
187
188 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
189 seq, 0);
190
191 *s = seq;
192
193 return 0;
194}
195
196/**
197 * amdgpu_fence_schedule_fallback - schedule fallback check
198 *
199 * @ring: pointer to struct amdgpu_ring
200 *
201 * Start a timer as fallback to our interrupts.
202 */
203static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
204{
205 mod_timer(timer: &ring->fence_drv.fallback_timer,
206 expires: jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);
207}
208
209/**
210 * amdgpu_fence_process - check for fence activity
211 *
212 * @ring: pointer to struct amdgpu_ring
213 *
214 * Checks the current fence value and calculates the last
215 * signalled fence value. Wakes the fence queue if the
216 * sequence number has increased.
217 *
218 * Returns true if fence was processed
219 */
220bool amdgpu_fence_process(struct amdgpu_ring *ring)
221{
222 struct amdgpu_fence_driver *drv = &ring->fence_drv;
223 struct amdgpu_device *adev = ring->adev;
224 uint32_t seq, last_seq;
225
226 do {
227 last_seq = atomic_read(v: &ring->fence_drv.last_seq);
228 seq = amdgpu_fence_read(ring);
229
230 } while (atomic_cmpxchg(v: &drv->last_seq, old: last_seq, new: seq) != last_seq);
231
232 if (timer_delete(timer: &ring->fence_drv.fallback_timer) &&
233 seq != ring->fence_drv.sync_seq)
234 amdgpu_fence_schedule_fallback(ring);
235
236 if (unlikely(seq == last_seq))
237 return false;
238
239 last_seq &= drv->num_fences_mask;
240 seq &= drv->num_fences_mask;
241
242 do {
243 struct dma_fence *fence, **ptr;
244 struct amdgpu_fence *am_fence;
245
246 ++last_seq;
247 last_seq &= drv->num_fences_mask;
248 ptr = &drv->fences[last_seq];
249
250 /* There is always exactly one thread signaling this fence slot */
251 fence = rcu_dereference_protected(*ptr, 1);
252 RCU_INIT_POINTER(*ptr, NULL);
253
254 if (!fence)
255 continue;
256
257 /* Save the wptr in the fence driver so we know what the last processed
258 * wptr was. This is required for re-emitting the ring state for
259 * queues that are reset but are not guilty and thus have no guilty fence.
260 */
261 am_fence = container_of(fence, struct amdgpu_fence, base);
262 drv->signalled_wptr = am_fence->wptr;
263 dma_fence_signal(fence);
264 dma_fence_put(fence);
265 pm_runtime_put_autosuspend(dev: adev_to_drm(adev)->dev);
266 } while (last_seq != seq);
267
268 return true;
269}
270
271/**
272 * amdgpu_fence_fallback - fallback for hardware interrupts
273 *
274 * @t: timer context used to obtain the pointer to ring structure
275 *
276 * Checks for fence activity.
277 */
278static void amdgpu_fence_fallback(struct timer_list *t)
279{
280 struct amdgpu_ring *ring = timer_container_of(ring, t,
281 fence_drv.fallback_timer);
282
283 if (amdgpu_fence_process(ring))
284 dev_warn(ring->adev->dev,
285 "Fence fallback timer expired on ring %s\n",
286 ring->name);
287}
288
289/**
290 * amdgpu_fence_wait_empty - wait for all fences to signal
291 *
292 * @ring: ring index the fence is associated with
293 *
294 * Wait for all fences on the requested ring to signal (all asics).
295 * Returns 0 if the fences have passed, error for all other cases.
296 */
297int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
298{
299 uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);
300 struct dma_fence *fence, **ptr;
301 int r;
302
303 if (!seq)
304 return 0;
305
306 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
307 rcu_read_lock();
308 fence = rcu_dereference(*ptr);
309 if (!fence || !dma_fence_get_rcu(fence)) {
310 rcu_read_unlock();
311 return 0;
312 }
313 rcu_read_unlock();
314
315 r = dma_fence_wait(fence, intr: false);
316 dma_fence_put(fence);
317 return r;
318}
319
320/**
321 * amdgpu_fence_wait_polling - busy wait for givn sequence number
322 *
323 * @ring: ring index the fence is associated with
324 * @wait_seq: sequence number to wait
325 * @timeout: the timeout for waiting in usecs
326 *
327 * Wait for all fences on the requested ring to signal (all asics).
328 * Returns left time if no timeout, 0 or minus if timeout.
329 */
330signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
331 uint32_t wait_seq,
332 signed long timeout)
333{
334
335 while ((int32_t)(wait_seq - amdgpu_fence_read(ring)) > 0 && timeout > 0) {
336 udelay(usec: 2);
337 timeout -= 2;
338 }
339 return timeout > 0 ? timeout : 0;
340}
341/**
342 * amdgpu_fence_count_emitted - get the count of emitted fences
343 *
344 * @ring: ring the fence is associated with
345 *
346 * Get the number of fences emitted on the requested ring (all asics).
347 * Returns the number of emitted fences on the ring. Used by the
348 * dynpm code to ring track activity.
349 */
350unsigned int amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
351{
352 uint64_t emitted;
353
354 /* We are not protected by ring lock when reading the last sequence
355 * but it's ok to report slightly wrong fence count here.
356 */
357 emitted = 0x100000000ull;
358 emitted -= atomic_read(v: &ring->fence_drv.last_seq);
359 emitted += READ_ONCE(ring->fence_drv.sync_seq);
360 return lower_32_bits(emitted);
361}
362
363/**
364 * amdgpu_fence_last_unsignaled_time_us - the time fence emitted until now
365 * @ring: ring the fence is associated with
366 *
367 * Find the earliest fence unsignaled until now, calculate the time delta
368 * between the time fence emitted and now.
369 */
370u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring)
371{
372 struct amdgpu_fence_driver *drv = &ring->fence_drv;
373 struct dma_fence *fence;
374 uint32_t last_seq, sync_seq;
375
376 last_seq = atomic_read(v: &ring->fence_drv.last_seq);
377 sync_seq = READ_ONCE(ring->fence_drv.sync_seq);
378 if (last_seq == sync_seq)
379 return 0;
380
381 ++last_seq;
382 last_seq &= drv->num_fences_mask;
383 fence = drv->fences[last_seq];
384 if (!fence)
385 return 0;
386
387 return ktime_us_delta(later: ktime_get(),
388 earlier: to_amdgpu_fence(f: fence)->start_timestamp);
389}
390
391/**
392 * amdgpu_fence_update_start_timestamp - update the timestamp of the fence
393 * @ring: ring the fence is associated with
394 * @seq: the fence seq number to update.
395 * @timestamp: the start timestamp to update.
396 *
397 * The function called at the time the fence and related ib is about to
398 * resubmit to gpu in MCBP scenario. Thus we do not consider race condition
399 * with amdgpu_fence_process to modify the same fence.
400 */
401void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp)
402{
403 struct amdgpu_fence_driver *drv = &ring->fence_drv;
404 struct dma_fence *fence;
405
406 seq &= drv->num_fences_mask;
407 fence = drv->fences[seq];
408 if (!fence)
409 return;
410
411 to_amdgpu_fence(f: fence)->start_timestamp = timestamp;
412}
413
414/**
415 * amdgpu_fence_driver_start_ring - make the fence driver
416 * ready for use on the requested ring.
417 *
418 * @ring: ring to start the fence driver on
419 * @irq_src: interrupt source to use for this ring
420 * @irq_type: interrupt type to use for this ring
421 *
422 * Make the fence driver ready for processing (all asics).
423 * Not all asics have all rings, so each asic will only
424 * start the fence driver on the rings it has.
425 * Returns 0 for success, errors for failure.
426 */
427int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
428 struct amdgpu_irq_src *irq_src,
429 unsigned int irq_type)
430{
431 struct amdgpu_device *adev = ring->adev;
432 uint64_t index;
433
434 if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {
435 ring->fence_drv.cpu_addr = ring->fence_cpu_addr;
436 ring->fence_drv.gpu_addr = ring->fence_gpu_addr;
437 } else {
438 /* put fence directly behind firmware */
439 index = ALIGN(adev->uvd.fw->size, 8);
440 ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;
441 ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;
442 }
443 amdgpu_fence_write(ring, seq: atomic_read(v: &ring->fence_drv.last_seq));
444
445 ring->fence_drv.irq_src = irq_src;
446 ring->fence_drv.irq_type = irq_type;
447 ring->fence_drv.initialized = true;
448
449 DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n",
450 ring->name, ring->fence_drv.gpu_addr);
451 return 0;
452}
453
454/**
455 * amdgpu_fence_driver_init_ring - init the fence driver
456 * for the requested ring.
457 *
458 * @ring: ring to init the fence driver on
459 *
460 * Init the fence driver for the requested ring (all asics).
461 * Helper function for amdgpu_fence_driver_init().
462 */
463int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
464{
465 struct amdgpu_device *adev = ring->adev;
466
467 if (!adev)
468 return -EINVAL;
469
470 if (!is_power_of_2(n: ring->num_hw_submission))
471 return -EINVAL;
472
473 ring->fence_drv.cpu_addr = NULL;
474 ring->fence_drv.gpu_addr = 0;
475 ring->fence_drv.sync_seq = 0;
476 atomic_set(v: &ring->fence_drv.last_seq, i: 0);
477 ring->fence_drv.initialized = false;
478
479 timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);
480
481 ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1;
482 spin_lock_init(&ring->fence_drv.lock);
483 ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *),
484 GFP_KERNEL);
485
486 if (!ring->fence_drv.fences)
487 return -ENOMEM;
488
489 return 0;
490}
491
492/**
493 * amdgpu_fence_driver_sw_init - init the fence driver
494 * for all possible rings.
495 *
496 * @adev: amdgpu device pointer
497 *
498 * Init the fence driver for all possible rings (all asics).
499 * Not all asics have all rings, so each asic will only
500 * start the fence driver on the rings it has using
501 * amdgpu_fence_driver_start_ring().
502 * Returns 0 for success.
503 */
504int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev)
505{
506 return 0;
507}
508
509/**
510 * amdgpu_fence_need_ring_interrupt_restore - helper function to check whether
511 * fence driver interrupts need to be restored.
512 *
513 * @ring: ring that to be checked
514 *
515 * Interrupts for rings that belong to GFX IP don't need to be restored
516 * when the target power state is s0ix.
517 *
518 * Return true if need to restore interrupts, false otherwise.
519 */
520static bool amdgpu_fence_need_ring_interrupt_restore(struct amdgpu_ring *ring)
521{
522 struct amdgpu_device *adev = ring->adev;
523 bool is_gfx_power_domain = false;
524
525 switch (ring->funcs->type) {
526 case AMDGPU_RING_TYPE_SDMA:
527 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
528 if (amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) >=
529 IP_VERSION(5, 0, 0))
530 is_gfx_power_domain = true;
531 break;
532 case AMDGPU_RING_TYPE_GFX:
533 case AMDGPU_RING_TYPE_COMPUTE:
534 case AMDGPU_RING_TYPE_KIQ:
535 case AMDGPU_RING_TYPE_MES:
536 is_gfx_power_domain = true;
537 break;
538 default:
539 break;
540 }
541
542 return !(adev->in_s0ix && is_gfx_power_domain);
543}
544
545/**
546 * amdgpu_fence_driver_hw_fini - tear down the fence driver
547 * for all possible rings.
548 *
549 * @adev: amdgpu device pointer
550 *
551 * Tear down the fence driver for all possible rings (all asics).
552 */
553void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev)
554{
555 int i, r;
556
557 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
558 struct amdgpu_ring *ring = adev->rings[i];
559
560 if (!ring || !ring->fence_drv.initialized)
561 continue;
562
563 /* You can't wait for HW to signal if it's gone */
564 if (!drm_dev_is_unplugged(dev: adev_to_drm(adev)))
565 r = amdgpu_fence_wait_empty(ring);
566 else
567 r = -ENODEV;
568 /* no need to trigger GPU reset as we are unloading */
569 if (r)
570 amdgpu_fence_driver_force_completion(ring);
571
572 if (!drm_dev_is_unplugged(dev: adev_to_drm(adev)) &&
573 ring->fence_drv.irq_src &&
574 amdgpu_fence_need_ring_interrupt_restore(ring))
575 amdgpu_irq_put(adev, src: ring->fence_drv.irq_src,
576 type: ring->fence_drv.irq_type);
577
578 timer_delete_sync(timer: &ring->fence_drv.fallback_timer);
579 }
580}
581
582/* Will either stop and flush handlers for amdgpu interrupt or reanble it */
583void amdgpu_fence_driver_isr_toggle(struct amdgpu_device *adev, bool stop)
584{
585 int i;
586
587 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
588 struct amdgpu_ring *ring = adev->rings[i];
589
590 if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src)
591 continue;
592
593 if (stop)
594 disable_irq(irq: adev->irq.irq);
595 else
596 enable_irq(irq: adev->irq.irq);
597 }
598}
599
600void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev)
601{
602 unsigned int i, j;
603
604 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
605 struct amdgpu_ring *ring = adev->rings[i];
606
607 if (!ring || !ring->fence_drv.initialized)
608 continue;
609
610 /*
611 * Notice we check for sched.ops since there's some
612 * override on the meaning of sched.ready by amdgpu.
613 * The natural check would be sched.ready, which is
614 * set as drm_sched_init() finishes...
615 */
616 if (ring->sched.ops)
617 drm_sched_fini(sched: &ring->sched);
618
619 for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
620 dma_fence_put(fence: ring->fence_drv.fences[j]);
621 kfree(objp: ring->fence_drv.fences);
622 ring->fence_drv.fences = NULL;
623 ring->fence_drv.initialized = false;
624 }
625}
626
627/**
628 * amdgpu_fence_driver_hw_init - enable the fence driver
629 * for all possible rings.
630 *
631 * @adev: amdgpu device pointer
632 *
633 * Enable the fence driver for all possible rings (all asics).
634 * Not all asics have all rings, so each asic will only
635 * start the fence driver on the rings it has using
636 * amdgpu_fence_driver_start_ring().
637 * Returns 0 for success.
638 */
639void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev)
640{
641 int i;
642
643 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
644 struct amdgpu_ring *ring = adev->rings[i];
645
646 if (!ring || !ring->fence_drv.initialized)
647 continue;
648
649 /* enable the interrupt */
650 if (ring->fence_drv.irq_src &&
651 amdgpu_fence_need_ring_interrupt_restore(ring))
652 amdgpu_irq_get(adev, src: ring->fence_drv.irq_src,
653 type: ring->fence_drv.irq_type);
654 }
655}
656
657/**
658 * amdgpu_fence_driver_set_error - set error code on fences
659 * @ring: the ring which contains the fences
660 * @error: the error code to set
661 *
662 * Set an error code to all the fences pending on the ring.
663 */
664void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error)
665{
666 struct amdgpu_fence_driver *drv = &ring->fence_drv;
667 unsigned long flags;
668
669 spin_lock_irqsave(&drv->lock, flags);
670 for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) {
671 struct dma_fence *fence;
672
673 fence = rcu_dereference_protected(drv->fences[i],
674 lockdep_is_held(&drv->lock));
675 if (fence && !dma_fence_is_signaled_locked(fence))
676 dma_fence_set_error(fence, error);
677 }
678 spin_unlock_irqrestore(lock: &drv->lock, flags);
679}
680
681/**
682 * amdgpu_fence_driver_force_completion - force signal latest fence of ring
683 *
684 * @ring: fence of the ring to signal
685 *
686 */
687void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
688{
689 amdgpu_fence_driver_set_error(ring, error: -ECANCELED);
690 amdgpu_fence_write(ring, seq: ring->fence_drv.sync_seq);
691 amdgpu_fence_process(ring);
692}
693
694
695/*
696 * Kernel queue reset handling
697 *
698 * The driver can reset individual queues for most engines, but those queues
699 * may contain work from multiple contexts. Resetting the queue will reset
700 * lose all of that state. In order to minimize the collateral damage, the
701 * driver will save the ring contents which are not associated with the guilty
702 * context prior to resetting the queue. After resetting the queue the queue
703 * contents from the other contexts is re-emitted to the rings so that it can
704 * be processed by the engine. To handle this, we save the queue's write
705 * pointer (wptr) in the fences associated with each context. If we get a
706 * queue timeout, we can then use the wptrs from the fences to determine
707 * which data needs to be saved out of the queue's ring buffer.
708 */
709
710/**
711 * amdgpu_fence_driver_guilty_force_completion - force signal of specified sequence
712 *
713 * @af: fence of the ring to signal
714 *
715 */
716void amdgpu_fence_driver_guilty_force_completion(struct amdgpu_fence *af)
717{
718 struct dma_fence *unprocessed;
719 struct dma_fence __rcu **ptr;
720 struct amdgpu_fence *fence;
721 struct amdgpu_ring *ring = af->ring;
722 unsigned long flags;
723 u32 seq, last_seq;
724 bool reemitted = false;
725
726 last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask;
727 seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;
728
729 /* mark all fences from the guilty context with an error */
730 spin_lock_irqsave(&ring->fence_drv.lock, flags);
731 do {
732 last_seq++;
733 last_seq &= ring->fence_drv.num_fences_mask;
734
735 ptr = &ring->fence_drv.fences[last_seq];
736 rcu_read_lock();
737 unprocessed = rcu_dereference(*ptr);
738
739 if (unprocessed && !dma_fence_is_signaled_locked(fence: unprocessed)) {
740 fence = container_of(unprocessed, struct amdgpu_fence, base);
741
742 if (fence->reemitted > 1)
743 reemitted = true;
744 else if (fence == af)
745 dma_fence_set_error(fence: &fence->base, error: -ETIME);
746 else if (fence->context == af->context)
747 dma_fence_set_error(fence: &fence->base, error: -ECANCELED);
748 }
749 rcu_read_unlock();
750 } while (last_seq != seq);
751 spin_unlock_irqrestore(lock: &ring->fence_drv.lock, flags);
752
753 if (reemitted) {
754 /* if we've already reemitted once then just cancel everything */
755 amdgpu_fence_driver_force_completion(ring: af->ring);
756 af->ring->ring_backup_entries_to_copy = 0;
757 }
758}
759
760void amdgpu_fence_save_wptr(struct amdgpu_fence *af)
761{
762 af->wptr = af->ring->wptr;
763}
764
765static void amdgpu_ring_backup_unprocessed_command(struct amdgpu_ring *ring,
766 u64 start_wptr, u64 end_wptr)
767{
768 unsigned int first_idx = start_wptr & ring->buf_mask;
769 unsigned int last_idx = end_wptr & ring->buf_mask;
770 unsigned int i;
771
772 /* Backup the contents of the ring buffer. */
773 for (i = first_idx; i != last_idx; ++i, i &= ring->buf_mask)
774 ring->ring_backup[ring->ring_backup_entries_to_copy++] = ring->ring[i];
775}
776
777void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
778 struct amdgpu_fence *guilty_fence)
779{
780 struct dma_fence *unprocessed;
781 struct dma_fence __rcu **ptr;
782 struct amdgpu_fence *fence;
783 u64 wptr;
784 u32 seq, last_seq;
785
786 last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask;
787 seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;
788 wptr = ring->fence_drv.signalled_wptr;
789 ring->ring_backup_entries_to_copy = 0;
790
791 do {
792 last_seq++;
793 last_seq &= ring->fence_drv.num_fences_mask;
794
795 ptr = &ring->fence_drv.fences[last_seq];
796 rcu_read_lock();
797 unprocessed = rcu_dereference(*ptr);
798
799 if (unprocessed && !dma_fence_is_signaled(fence: unprocessed)) {
800 fence = container_of(unprocessed, struct amdgpu_fence, base);
801
802 /* save everything if the ring is not guilty, otherwise
803 * just save the content from other contexts.
804 */
805 if (!fence->reemitted &&
806 (!guilty_fence || (fence->context != guilty_fence->context))) {
807 amdgpu_ring_backup_unprocessed_command(ring, start_wptr: wptr,
808 end_wptr: fence->wptr);
809 } else if (!fence->reemitted) {
810 /* always save the fence */
811 amdgpu_ring_backup_unprocessed_command(ring,
812 start_wptr: fence->fence_wptr_start,
813 end_wptr: fence->fence_wptr_end);
814 }
815 wptr = fence->wptr;
816 fence->reemitted++;
817 }
818 rcu_read_unlock();
819 } while (last_seq != seq);
820}
821
822/*
823 * Common fence implementation
824 */
825
826static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
827{
828 return "amdgpu";
829}
830
831static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
832{
833 return (const char *)to_amdgpu_fence(f)->ring->name;
834}
835
836/**
837 * amdgpu_fence_enable_signaling - enable signalling on fence
838 * @f: fence
839 *
840 * This function is called with fence_queue lock held, and adds a callback
841 * to fence_queue that checks if this fence is signaled, and if so it
842 * signals the fence and removes itself.
843 */
844static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
845{
846 if (!timer_pending(timer: &to_amdgpu_fence(f)->ring->fence_drv.fallback_timer))
847 amdgpu_fence_schedule_fallback(ring: to_amdgpu_fence(f)->ring);
848
849 return true;
850}
851
852/**
853 * amdgpu_fence_free - free up the fence memory
854 *
855 * @rcu: RCU callback head
856 *
857 * Free up the fence memory after the RCU grace period.
858 */
859static void amdgpu_fence_free(struct rcu_head *rcu)
860{
861 struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
862
863 /* free fence_slab if it's separated fence*/
864 kfree(objp: to_amdgpu_fence(f));
865}
866
867/**
868 * amdgpu_fence_release - callback that fence can be freed
869 *
870 * @f: fence
871 *
872 * This function is called when the reference count becomes zero.
873 * It just RCU schedules freeing up the fence.
874 */
875static void amdgpu_fence_release(struct dma_fence *f)
876{
877 call_rcu(head: &f->rcu, func: amdgpu_fence_free);
878}
879
880static const struct dma_fence_ops amdgpu_fence_ops = {
881 .get_driver_name = amdgpu_fence_get_driver_name,
882 .get_timeline_name = amdgpu_fence_get_timeline_name,
883 .enable_signaling = amdgpu_fence_enable_signaling,
884 .release = amdgpu_fence_release,
885};
886
887/*
888 * Fence debugfs
889 */
890#if defined(CONFIG_DEBUG_FS)
891static int amdgpu_debugfs_fence_info_show(struct seq_file *m, void *unused)
892{
893 struct amdgpu_device *adev = m->private;
894 int i;
895
896 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
897 struct amdgpu_ring *ring = adev->rings[i];
898
899 if (!ring || !ring->fence_drv.initialized)
900 continue;
901
902 amdgpu_fence_process(ring);
903
904 seq_printf(m, fmt: "--- ring %d (%s) ---\n", i, ring->name);
905 seq_printf(m, fmt: "Last signaled fence 0x%08x\n",
906 atomic_read(v: &ring->fence_drv.last_seq));
907 seq_printf(m, fmt: "Last emitted 0x%08x\n",
908 ring->fence_drv.sync_seq);
909
910 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||
911 ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {
912 seq_printf(m, fmt: "Last signaled trailing fence 0x%08x\n",
913 le32_to_cpu(*ring->trail_fence_cpu_addr));
914 seq_printf(m, fmt: "Last emitted 0x%08x\n",
915 ring->trail_seq);
916 }
917
918 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
919 continue;
920
921 /* set in CP_VMID_PREEMPT and preemption occurred */
922 seq_printf(m, fmt: "Last preempted 0x%08x\n",
923 le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
924 /* set in CP_VMID_RESET and reset occurred */
925 seq_printf(m, fmt: "Last reset 0x%08x\n",
926 le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));
927 /* Both preemption and reset occurred */
928 seq_printf(m, fmt: "Last both 0x%08x\n",
929 le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));
930 }
931 return 0;
932}
933
934/*
935 * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
936 *
937 * Manually trigger a gpu reset at the next fence wait.
938 */
939static int gpu_recover_get(void *data, u64 *val)
940{
941 struct amdgpu_device *adev = (struct amdgpu_device *)data;
942 struct drm_device *dev = adev_to_drm(adev);
943 int r;
944
945 r = pm_runtime_get_sync(dev: dev->dev);
946 if (r < 0) {
947 pm_runtime_put_autosuspend(dev: dev->dev);
948 return 0;
949 }
950
951 if (amdgpu_reset_domain_schedule(domain: adev->reset_domain, work: &adev->reset_work))
952 flush_work(work: &adev->reset_work);
953
954 *val = atomic_read(v: &adev->reset_domain->reset_res);
955
956 pm_runtime_put_autosuspend(dev: dev->dev);
957
958 return 0;
959}
960
961DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_fence_info);
962DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, NULL,
963 "%lld\n");
964
965static void amdgpu_debugfs_reset_work(struct work_struct *work)
966{
967 struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
968 reset_work);
969
970 struct amdgpu_reset_context reset_context;
971
972 memset(&reset_context, 0, sizeof(reset_context));
973
974 reset_context.method = AMD_RESET_METHOD_NONE;
975 reset_context.reset_req_dev = adev;
976 reset_context.src = AMDGPU_RESET_SRC_USER;
977 set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context.flags);
978 set_bit(nr: AMDGPU_SKIP_COREDUMP, addr: &reset_context.flags);
979
980 amdgpu_device_gpu_recover(adev, NULL, reset_context: &reset_context);
981}
982
983#endif
984
985void amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
986{
987#if defined(CONFIG_DEBUG_FS)
988 struct drm_minor *minor = adev_to_drm(adev)->primary;
989 struct dentry *root = minor->debugfs_root;
990
991 debugfs_create_file("amdgpu_fence_info", 0444, root, adev,
992 &amdgpu_debugfs_fence_info_fops);
993
994 if (!amdgpu_sriov_vf(adev)) {
995
996 INIT_WORK(&adev->reset_work, amdgpu_debugfs_reset_work);
997 debugfs_create_file("amdgpu_gpu_recover", 0444, root, adev,
998 &amdgpu_debugfs_gpu_recover_fops);
999 }
1000#endif
1001}
1002
1003

source code of linux/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c