1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6
7#include <linux/kref.h>
8#include <linux/string_helpers.h>
9
10#include <drm/drm_print.h>
11
12#include "gem/i915_gem_pm.h"
13#include "gt/intel_gt.h"
14
15#include "i915_selftest.h"
16
17#include "igt_flush_test.h"
18#include "lib_sw_fence.h"
19
20struct live_active {
21 struct i915_active base;
22 struct kref ref;
23 bool retired;
24};
25
26static void __live_get(struct live_active *active)
27{
28 kref_get(kref: &active->ref);
29}
30
31static void __live_free(struct live_active *active)
32{
33 i915_active_fini(ref: &active->base);
34 kfree(objp: active);
35}
36
37static void __live_release(struct kref *ref)
38{
39 struct live_active *active = container_of(ref, typeof(*active), ref);
40
41 __live_free(active);
42}
43
44static void __live_put(struct live_active *active)
45{
46 kref_put(kref: &active->ref, release: __live_release);
47}
48
49static int __live_active(struct i915_active *base)
50{
51 struct live_active *active = container_of(base, typeof(*active), base);
52
53 __live_get(active);
54 return 0;
55}
56
57static void __live_retire(struct i915_active *base)
58{
59 struct live_active *active = container_of(base, typeof(*active), base);
60
61 active->retired = true;
62 __live_put(active);
63}
64
65static struct live_active *__live_alloc(struct drm_i915_private *i915)
66{
67 struct live_active *active;
68
69 active = kzalloc(sizeof(*active), GFP_KERNEL);
70 if (!active)
71 return NULL;
72
73 kref_init(kref: &active->ref);
74 i915_active_init(&active->base, __live_active, __live_retire, 0);
75
76 return active;
77}
78
79static struct live_active *
80__live_active_setup(struct drm_i915_private *i915)
81{
82 struct intel_engine_cs *engine;
83 struct i915_sw_fence *submit;
84 struct live_active *active;
85 unsigned int count = 0;
86 int err = 0;
87
88 active = __live_alloc(i915);
89 if (!active)
90 return ERR_PTR(error: -ENOMEM);
91
92 submit = heap_fence_create(GFP_KERNEL);
93 if (!submit) {
94 kfree(objp: active);
95 return ERR_PTR(error: -ENOMEM);
96 }
97
98 err = i915_active_acquire(ref: &active->base);
99 if (err)
100 goto out;
101
102 for_each_uabi_engine(engine, i915) {
103 struct i915_request *rq;
104
105 rq = intel_engine_create_kernel_request(engine);
106 if (IS_ERR(ptr: rq)) {
107 err = PTR_ERR(ptr: rq);
108 break;
109 }
110
111 err = i915_sw_fence_await_sw_fence_gfp(fence: &rq->submit,
112 after: submit,
113 GFP_KERNEL);
114 if (err >= 0)
115 err = i915_active_add_request(ref: &active->base, rq);
116 i915_request_add(rq);
117 if (err) {
118 pr_err("Failed to track active ref!\n");
119 break;
120 }
121
122 count++;
123 }
124
125 i915_active_release(ref: &active->base);
126 if (READ_ONCE(active->retired) && count) {
127 pr_err("i915_active retired before submission!\n");
128 err = -EINVAL;
129 }
130 if (atomic_read(v: &active->base.count) != count) {
131 pr_err("i915_active not tracking all requests, found %d, expected %d\n",
132 atomic_read(&active->base.count), count);
133 err = -EINVAL;
134 }
135
136out:
137 i915_sw_fence_commit(fence: submit);
138 heap_fence_put(fence: submit);
139 if (err) {
140 __live_put(active);
141 active = ERR_PTR(error: err);
142 }
143
144 return active;
145}
146
147static int live_active_wait(void *arg)
148{
149 struct drm_i915_private *i915 = arg;
150 struct live_active *active;
151 int err = 0;
152
153 /* Check that we get a callback when requests retire upon waiting */
154
155 active = __live_active_setup(i915);
156 if (IS_ERR(ptr: active))
157 return PTR_ERR(ptr: active);
158
159 __i915_active_wait(ref: &active->base, TASK_UNINTERRUPTIBLE);
160 if (!READ_ONCE(active->retired)) {
161 struct drm_printer p = drm_err_printer(drm: &i915->drm, prefix: __func__);
162
163 drm_printf(p: &p, f: "i915_active not retired after waiting!\n");
164 i915_active_print(ref: &active->base, m: &p);
165
166 err = -EINVAL;
167 }
168
169 __live_put(active);
170
171 if (igt_flush_test(i915))
172 err = -EIO;
173
174 return err;
175}
176
177static int live_active_retire(void *arg)
178{
179 struct drm_i915_private *i915 = arg;
180 struct live_active *active;
181 int err = 0;
182
183 /* Check that we get a callback when requests are indirectly retired */
184
185 active = __live_active_setup(i915);
186 if (IS_ERR(ptr: active))
187 return PTR_ERR(ptr: active);
188
189 /* waits for & retires all requests */
190 if (igt_flush_test(i915))
191 err = -EIO;
192
193 if (!READ_ONCE(active->retired)) {
194 struct drm_printer p = drm_err_printer(drm: &i915->drm, prefix: __func__);
195
196 drm_printf(p: &p, f: "i915_active not retired after flushing!\n");
197 i915_active_print(ref: &active->base, m: &p);
198
199 err = -EINVAL;
200 }
201
202 __live_put(active);
203
204 return err;
205}
206
207static int live_active_barrier(void *arg)
208{
209 struct drm_i915_private *i915 = arg;
210 struct intel_engine_cs *engine;
211 struct live_active *active;
212 int err = 0;
213
214 /* Check that we get a callback when requests retire upon waiting */
215
216 active = __live_alloc(i915);
217 if (!active)
218 return -ENOMEM;
219
220 err = i915_active_acquire(ref: &active->base);
221 if (err)
222 goto out;
223
224 for_each_uabi_engine(engine, i915) {
225 err = i915_active_acquire_preallocate_barrier(ref: &active->base,
226 engine);
227 if (err)
228 break;
229
230 i915_active_acquire_barrier(ref: &active->base);
231 }
232
233 i915_active_release(ref: &active->base);
234 if (err)
235 goto out;
236
237 __i915_active_wait(ref: &active->base, TASK_UNINTERRUPTIBLE);
238 if (!READ_ONCE(active->retired)) {
239 pr_err("i915_active not retired after flushing barriers!\n");
240 err = -EINVAL;
241 }
242
243out:
244 __live_put(active);
245
246 if (igt_flush_test(i915))
247 err = -EIO;
248
249 return err;
250}
251
252int i915_active_live_selftests(struct drm_i915_private *i915)
253{
254 static const struct i915_subtest tests[] = {
255 SUBTEST(live_active_wait),
256 SUBTEST(live_active_retire),
257 SUBTEST(live_active_barrier),
258 };
259
260 if (intel_gt_is_wedged(gt: to_gt(i915)))
261 return 0;
262
263 return i915_subtests(tests, i915);
264}
265
266static struct intel_engine_cs *node_to_barrier(struct active_node *it)
267{
268 struct intel_engine_cs *engine;
269
270 if (!is_barrier(active: &it->base))
271 return NULL;
272
273 engine = __barrier_to_engine(node: it);
274 smp_rmb(); /* serialise with add_active_barriers */
275 if (!is_barrier(active: &it->base))
276 return NULL;
277
278 return engine;
279}
280
281void i915_active_print(struct i915_active *ref, struct drm_printer *m)
282{
283 drm_printf(p: m, f: "active %ps:%ps\n", ref->active, ref->retire);
284 drm_printf(p: m, f: "\tcount: %d\n", atomic_read(v: &ref->count));
285 drm_printf(p: m, f: "\tpreallocated barriers? %s\n",
286 str_yes_no(v: !llist_empty(head: &ref->preallocated_barriers)));
287
288 if (i915_active_acquire_if_busy(ref)) {
289 struct active_node *it, *n;
290
291 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) {
292 struct intel_engine_cs *engine;
293
294 engine = node_to_barrier(it);
295 if (engine) {
296 drm_printf(p: m, f: "\tbarrier: %s\n", engine->name);
297 continue;
298 }
299
300 if (i915_active_fence_isset(active: &it->base)) {
301 drm_printf(p: m,
302 f: "\ttimeline: %llx\n", it->timeline);
303 continue;
304 }
305 }
306
307 i915_active_release(ref);
308 }
309}
310
311static void spin_unlock_wait(spinlock_t *lock)
312{
313 spin_lock_irq(lock);
314 spin_unlock_irq(lock);
315}
316
317static void active_flush(struct i915_active *ref,
318 struct i915_active_fence *active)
319{
320 struct dma_fence *fence;
321
322 fence = xchg(__active_fence_slot(active), NULL);
323 if (!fence)
324 return;
325
326 spin_lock_irq(lock: fence->lock);
327 __list_del_entry(entry: &active->cb.node);
328 spin_unlock_irq(lock: fence->lock); /* serialise with fence->cb_list */
329 atomic_dec(v: &ref->count);
330
331 GEM_BUG_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
332}
333
334void i915_active_unlock_wait(struct i915_active *ref)
335{
336 if (i915_active_acquire_if_busy(ref)) {
337 struct active_node *it, *n;
338
339 /* Wait for all active callbacks */
340 rcu_read_lock();
341 active_flush(ref, active: &ref->excl);
342 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node)
343 active_flush(ref, active: &it->base);
344 rcu_read_unlock();
345
346 i915_active_release(ref);
347 }
348
349 /* And wait for the retire callback */
350 spin_unlock_wait(lock: &ref->tree_lock);
351
352 /* ... which may have been on a thread instead */
353 flush_work(work: &ref->work);
354}
355

source code of linux/drivers/gpu/drm/i915/selftests/i915_active.c