1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020-2021 Intel Corporation
4 */
5
6#include "gt/intel_migrate.h"
7#include "gt/intel_gpu_commands.h"
8#include "gem/i915_gem_ttm_move.h"
9
10#include "i915_deps.h"
11
12#include "selftests/igt_reset.h"
13#include "selftests/igt_spinner.h"
14
15static int igt_fill_check_buffer(struct drm_i915_gem_object *obj,
16 struct intel_gt *gt,
17 bool fill)
18{
19 unsigned int i, count = obj->base.size / sizeof(u32);
20 enum i915_map_type map_type =
21 intel_gt_coherent_map_type(gt, obj, always_coherent: false);
22 u32 *cur;
23 int err = 0;
24
25 assert_object_held(obj);
26 cur = i915_gem_object_pin_map(obj, type: map_type);
27 if (IS_ERR(ptr: cur))
28 return PTR_ERR(ptr: cur);
29
30 if (fill)
31 for (i = 0; i < count; ++i)
32 *cur++ = i;
33 else
34 for (i = 0; i < count; ++i)
35 if (*cur++ != i) {
36 pr_err("Object content mismatch at location %d of %d\n", i, count);
37 err = -EINVAL;
38 break;
39 }
40
41 i915_gem_object_unpin_map(obj);
42
43 return err;
44}
45
46static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src,
47 enum intel_region_id dst)
48{
49 struct drm_i915_private *i915 = gt->i915;
50 struct intel_memory_region *src_mr = i915->mm.regions[src];
51 struct intel_memory_region *dst_mr = i915->mm.regions[dst];
52 struct drm_i915_gem_object *obj;
53 struct i915_gem_ww_ctx ww;
54 int err = 0;
55
56 GEM_BUG_ON(!src_mr);
57 GEM_BUG_ON(!dst_mr);
58
59 /* Switch object backing-store on create */
60 obj = i915_gem_object_create_region(mem: src_mr, size: dst_mr->min_page_size, page_size: 0, flags: 0);
61 if (IS_ERR(ptr: obj))
62 return PTR_ERR(ptr: obj);
63
64 for_i915_gem_ww(&ww, err, true) {
65 err = i915_gem_object_lock(obj, ww: &ww);
66 if (err)
67 continue;
68
69 err = igt_fill_check_buffer(obj, gt, fill: true);
70 if (err)
71 continue;
72
73 err = i915_gem_object_migrate(obj, ww: &ww, id: dst);
74 if (err)
75 continue;
76
77 err = i915_gem_object_pin_pages(obj);
78 if (err)
79 continue;
80
81 if (i915_gem_object_can_migrate(obj, id: src))
82 err = -EINVAL;
83
84 i915_gem_object_unpin_pages(obj);
85 err = i915_gem_object_wait_migration(obj, flags: true);
86 if (err)
87 continue;
88
89 err = igt_fill_check_buffer(obj, gt, fill: false);
90 }
91 i915_gem_object_put(obj);
92
93 return err;
94}
95
96static int igt_smem_create_migrate(void *arg)
97{
98 return igt_create_migrate(gt: arg, src: INTEL_REGION_LMEM_0, dst: INTEL_REGION_SMEM);
99}
100
101static int igt_lmem_create_migrate(void *arg)
102{
103 return igt_create_migrate(gt: arg, src: INTEL_REGION_SMEM, dst: INTEL_REGION_LMEM_0);
104}
105
106static int igt_same_create_migrate(void *arg)
107{
108 return igt_create_migrate(gt: arg, src: INTEL_REGION_LMEM_0, dst: INTEL_REGION_LMEM_0);
109}
110
111static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww,
112 struct drm_i915_gem_object *obj,
113 struct i915_vma *vma,
114 bool silent_migrate)
115{
116 int err;
117
118 err = i915_gem_object_lock(obj, ww);
119 if (err)
120 return err;
121
122 if (vma) {
123 err = i915_vma_pin_ww(vma, ww, size: obj->base.size, alignment: 0,
124 flags: 0UL | PIN_OFFSET_FIXED |
125 PIN_USER);
126 if (err) {
127 if (err != -EINTR && err != ERESTARTSYS &&
128 err != -EDEADLK)
129 pr_err("Failed to pin vma.\n");
130 return err;
131 }
132
133 i915_vma_unpin(vma);
134 }
135
136 /*
137 * Migration will implicitly unbind (asynchronously) any bound
138 * vmas.
139 */
140 if (i915_gem_object_is_lmem(obj)) {
141 err = i915_gem_object_migrate(obj, ww, id: INTEL_REGION_SMEM);
142 if (err) {
143 if (!silent_migrate)
144 pr_err("Object failed migration to smem\n");
145 if (err)
146 return err;
147 }
148
149 if (i915_gem_object_is_lmem(obj)) {
150 pr_err("object still backed by lmem\n");
151 err = -EINVAL;
152 }
153
154 if (!i915_gem_object_has_struct_page(obj)) {
155 pr_err("object not backed by struct page\n");
156 err = -EINVAL;
157 }
158
159 } else {
160 err = i915_gem_object_migrate(obj, ww, id: INTEL_REGION_LMEM_0);
161 if (err) {
162 if (!silent_migrate)
163 pr_err("Object failed migration to lmem\n");
164 if (err)
165 return err;
166 }
167
168 if (i915_gem_object_has_struct_page(obj)) {
169 pr_err("object still backed by struct page\n");
170 err = -EINVAL;
171 }
172
173 if (!i915_gem_object_is_lmem(obj)) {
174 pr_err("object not backed by lmem\n");
175 err = -EINVAL;
176 }
177 }
178
179 return err;
180}
181
182static int __igt_lmem_pages_migrate(struct intel_gt *gt,
183 struct i915_address_space *vm,
184 struct i915_deps *deps,
185 struct igt_spinner *spin,
186 struct dma_fence *spin_fence,
187 bool borked_migrate)
188{
189 struct drm_i915_private *i915 = gt->i915;
190 struct drm_i915_gem_object *obj;
191 struct i915_vma *vma = NULL;
192 struct i915_gem_ww_ctx ww;
193 struct i915_request *rq;
194 int err;
195 int i;
196
197 /* From LMEM to shmem and back again */
198
199 obj = i915_gem_object_create_lmem(i915, SZ_2M, flags: 0);
200 if (IS_ERR(ptr: obj))
201 return PTR_ERR(ptr: obj);
202
203 if (vm) {
204 vma = i915_vma_instance(obj, vm, NULL);
205 if (IS_ERR(ptr: vma)) {
206 err = PTR_ERR(ptr: vma);
207 goto out_put;
208 }
209 }
210
211 /* Initial GPU fill, sync, CPU initialization. */
212 for_i915_gem_ww(&ww, err, true) {
213 err = i915_gem_object_lock(obj, ww: &ww);
214 if (err)
215 continue;
216
217 err = ____i915_gem_object_get_pages(obj);
218 if (err)
219 continue;
220
221 err = intel_migrate_clear(m: &gt->migrate, ww: &ww, deps,
222 sg: obj->mm.pages->sgl, pat_index: obj->pat_index,
223 is_lmem: i915_gem_object_is_lmem(obj),
224 value: 0xdeadbeaf, out: &rq);
225 if (rq) {
226 err = dma_resv_reserve_fences(obj: obj->base.resv, num_fences: 1);
227 if (!err)
228 dma_resv_add_fence(obj: obj->base.resv, fence: &rq->fence,
229 usage: DMA_RESV_USAGE_KERNEL);
230 i915_request_put(rq);
231 }
232 if (err)
233 continue;
234
235 if (!vma) {
236 err = igt_fill_check_buffer(obj, gt, fill: true);
237 if (err)
238 continue;
239 }
240 }
241 if (err)
242 goto out_put;
243
244 /*
245 * Migrate to and from smem without explicitly syncing.
246 * Finalize with data in smem for fast readout.
247 */
248 for (i = 1; i <= 5; ++i) {
249 for_i915_gem_ww(&ww, err, true)
250 err = lmem_pages_migrate_one(ww: &ww, obj, vma,
251 silent_migrate: borked_migrate);
252 if (err)
253 goto out_put;
254 }
255
256 err = i915_gem_object_lock_interruptible(obj, NULL);
257 if (err)
258 goto out_put;
259
260 if (spin) {
261 if (dma_fence_is_signaled(fence: spin_fence)) {
262 pr_err("Spinner was terminated by hangcheck.\n");
263 err = -EBUSY;
264 goto out_unlock;
265 }
266 igt_spinner_end(spin);
267 }
268
269 /* Finally sync migration and check content. */
270 err = i915_gem_object_wait_migration(obj, flags: true);
271 if (err)
272 goto out_unlock;
273
274 if (vma) {
275 err = i915_vma_wait_for_bind(vma);
276 if (err)
277 goto out_unlock;
278 } else {
279 err = igt_fill_check_buffer(obj, gt, fill: false);
280 }
281
282out_unlock:
283 i915_gem_object_unlock(obj);
284out_put:
285 i915_gem_object_put(obj);
286
287 return err;
288}
289
290static int igt_lmem_pages_failsafe_migrate(void *arg)
291{
292 int fail_gpu, fail_alloc, ban_memcpy, ret;
293 struct intel_gt *gt = arg;
294
295 for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
296 for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) {
297 for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
298 pr_info("Simulated failure modes: gpu: %d, alloc:%d, ban_memcpy: %d\n",
299 fail_gpu, fail_alloc, ban_memcpy);
300 i915_ttm_migrate_set_ban_memcpy(ban: ban_memcpy);
301 i915_ttm_migrate_set_failure_modes(gpu_migration: fail_gpu,
302 work_allocation: fail_alloc);
303 ret = __igt_lmem_pages_migrate(gt, NULL, NULL,
304 NULL, NULL,
305 borked_migrate: ban_memcpy &&
306 fail_gpu);
307
308 if (ban_memcpy && fail_gpu) {
309 struct intel_gt *__gt;
310 unsigned int id;
311
312 if (ret != -EIO) {
313 pr_err("expected -EIO, got (%d)\n", ret);
314 ret = -EINVAL;
315 } else {
316 ret = 0;
317 }
318
319 for_each_gt(__gt, gt->i915, id) {
320 intel_wakeref_t wakeref;
321 bool wedged;
322
323 mutex_lock(&__gt->reset.mutex);
324 wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
325 mutex_unlock(lock: &__gt->reset.mutex);
326
327 if (fail_gpu && !fail_alloc) {
328 if (!wedged) {
329 pr_err("gt(%u) not wedged\n", id);
330 ret = -EINVAL;
331 continue;
332 }
333 } else if (wedged) {
334 pr_err("gt(%u) incorrectly wedged\n", id);
335 ret = -EINVAL;
336 } else {
337 continue;
338 }
339
340 wakeref = intel_runtime_pm_get(rpm: __gt->uncore->rpm);
341 igt_global_reset_lock(gt: __gt);
342 intel_gt_reset(gt: __gt, ALL_ENGINES, NULL);
343 igt_global_reset_unlock(gt: __gt);
344 intel_runtime_pm_put(rpm: __gt->uncore->rpm, wref: wakeref);
345 }
346 if (ret)
347 goto out_err;
348 }
349 }
350 }
351 }
352
353out_err:
354 i915_ttm_migrate_set_failure_modes(gpu_migration: false, work_allocation: false);
355 i915_ttm_migrate_set_ban_memcpy(ban: false);
356 return ret;
357}
358
359/*
360 * This subtest tests that unbinding at migration is indeed performed
361 * async. We launch a spinner and a number of migrations depending on
362 * that spinner to have terminated. Before each migration we bind a
363 * vma, which should then be async unbound by the migration operation.
364 * If we are able to schedule migrations without blocking while the
365 * spinner is still running, those unbinds are indeed async and non-
366 * blocking.
367 *
368 * Note that each async bind operation is awaiting the previous migration
369 * due to the moving fence resulting from the migration.
370 */
371static int igt_async_migrate(struct intel_gt *gt)
372{
373 struct intel_engine_cs *engine;
374 enum intel_engine_id id;
375 struct i915_ppgtt *ppgtt;
376 struct igt_spinner spin;
377 int err;
378
379 ppgtt = i915_ppgtt_create(gt, lmem_pt_obj_flags: 0);
380 if (IS_ERR(ptr: ppgtt))
381 return PTR_ERR(ptr: ppgtt);
382
383 if (igt_spinner_init(spin: &spin, gt)) {
384 err = -ENOMEM;
385 goto out_spin;
386 }
387
388 for_each_engine(engine, gt, id) {
389 struct ttm_operation_ctx ctx = {
390 .interruptible = true
391 };
392 struct dma_fence *spin_fence;
393 struct intel_context *ce;
394 struct i915_request *rq;
395 struct i915_deps deps;
396
397 ce = intel_context_create(engine);
398 if (IS_ERR(ptr: ce)) {
399 err = PTR_ERR(ptr: ce);
400 goto out_ce;
401 }
402
403 /*
404 * Use MI_NOOP, making the spinner non-preemptible. If there
405 * is a code path where we fail async operation due to the
406 * running spinner, we will block and fail to end the
407 * spinner resulting in a deadlock. But with a non-
408 * preemptible spinner, hangcheck will terminate the spinner
409 * for us, and we will later detect that and fail the test.
410 */
411 rq = igt_spinner_create_request(spin: &spin, ce, MI_NOOP);
412 intel_context_put(ce);
413 if (IS_ERR(ptr: rq)) {
414 err = PTR_ERR(ptr: rq);
415 goto out_ce;
416 }
417
418 i915_deps_init(deps: &deps, GFP_KERNEL);
419 err = i915_deps_add_dependency(deps: &deps, fence: &rq->fence, ctx: &ctx);
420 spin_fence = dma_fence_get(fence: &rq->fence);
421 i915_request_add(rq);
422 if (err)
423 goto out_ce;
424
425 err = __igt_lmem_pages_migrate(gt, vm: &ppgtt->vm, deps: &deps, spin: &spin,
426 spin_fence, borked_migrate: false);
427 i915_deps_fini(deps: &deps);
428 dma_fence_put(fence: spin_fence);
429 if (err)
430 goto out_ce;
431 }
432
433out_ce:
434 igt_spinner_fini(spin: &spin);
435out_spin:
436 i915_vm_put(vm: &ppgtt->vm);
437
438 return err;
439}
440
441/*
442 * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while
443 * arming the migration error check and block async migration. This
444 * will cause us to deadlock and hangcheck will terminate the spinner
445 * causing the test to fail.
446 */
447#define ASYNC_FAIL_ALLOC 1
448static int igt_lmem_async_migrate(void *arg)
449{
450 int fail_gpu, fail_alloc, ban_memcpy, ret;
451 struct intel_gt *gt = arg;
452
453 for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) {
454 for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) {
455 for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) {
456 pr_info("Simulated failure modes: gpu: %d, alloc: %d, ban_memcpy: %d\n",
457 fail_gpu, fail_alloc, ban_memcpy);
458 i915_ttm_migrate_set_ban_memcpy(ban: ban_memcpy);
459 i915_ttm_migrate_set_failure_modes(gpu_migration: fail_gpu,
460 work_allocation: fail_alloc);
461 ret = igt_async_migrate(gt);
462
463 if (fail_gpu && ban_memcpy) {
464 struct intel_gt *__gt;
465 unsigned int id;
466
467 if (ret != -EIO) {
468 pr_err("expected -EIO, got (%d)\n", ret);
469 ret = -EINVAL;
470 } else {
471 ret = 0;
472 }
473
474 for_each_gt(__gt, gt->i915, id) {
475 intel_wakeref_t wakeref;
476 bool wedged;
477
478 mutex_lock(&__gt->reset.mutex);
479 wedged = test_bit(I915_WEDGED, &__gt->reset.flags);
480 mutex_unlock(lock: &__gt->reset.mutex);
481
482 if (fail_gpu && !fail_alloc) {
483 if (!wedged) {
484 pr_err("gt(%u) not wedged\n", id);
485 ret = -EINVAL;
486 continue;
487 }
488 } else if (wedged) {
489 pr_err("gt(%u) incorrectly wedged\n", id);
490 ret = -EINVAL;
491 } else {
492 continue;
493 }
494
495 wakeref = intel_runtime_pm_get(rpm: __gt->uncore->rpm);
496 igt_global_reset_lock(gt: __gt);
497 intel_gt_reset(gt: __gt, ALL_ENGINES, NULL);
498 igt_global_reset_unlock(gt: __gt);
499 intel_runtime_pm_put(rpm: __gt->uncore->rpm, wref: wakeref);
500 }
501 }
502 if (ret)
503 goto out_err;
504 }
505 }
506 }
507
508out_err:
509 i915_ttm_migrate_set_failure_modes(gpu_migration: false, work_allocation: false);
510 i915_ttm_migrate_set_ban_memcpy(ban: false);
511 return ret;
512}
513
514int i915_gem_migrate_live_selftests(struct drm_i915_private *i915)
515{
516 static const struct i915_subtest tests[] = {
517 SUBTEST(igt_smem_create_migrate),
518 SUBTEST(igt_lmem_create_migrate),
519 SUBTEST(igt_same_create_migrate),
520 SUBTEST(igt_lmem_pages_failsafe_migrate),
521 SUBTEST(igt_lmem_async_migrate),
522 };
523
524 if (!HAS_LMEM(i915))
525 return 0;
526
527 return intel_gt_live_subtests(tests, to_gt(i915));
528}
529

source code of linux/drivers/gpu/drm/i915/gem/selftests/i915_gem_migrate.c