1// SPDX-License-Identifier: GPL-2.0 AND MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include <kunit/test.h>
7#include <kunit/visibility.h>
8
9#include <linux/iosys-map.h>
10#include <linux/math64.h>
11#include <linux/prandom.h>
12#include <linux/swap.h>
13
14#include <uapi/linux/sysinfo.h>
15
16#include "tests/xe_kunit_helpers.h"
17#include "tests/xe_pci_test.h"
18#include "tests/xe_test.h"
19
20#include "xe_bo_evict.h"
21#include "xe_pci.h"
22#include "xe_pm.h"
23
24static int ccs_test_migrate(struct xe_tile *tile, struct xe_bo *bo,
25 bool clear, u64 get_val, u64 assign_val,
26 struct kunit *test, struct drm_exec *exec)
27{
28 struct dma_fence *fence;
29 struct ttm_tt *ttm;
30 struct page *page;
31 pgoff_t ccs_page;
32 long timeout;
33 u64 *cpu_map;
34 int ret;
35 u32 offset;
36
37 /* Move bo to VRAM if not already there. */
38 ret = xe_bo_validate(bo, NULL, false, exec);
39 if (ret) {
40 KUNIT_FAIL(test, "Failed to validate bo.\n");
41 return ret;
42 }
43
44 /* Optionally clear bo *and* CCS data in VRAM. */
45 if (clear) {
46 fence = xe_migrate_clear(tile->migrate, bo, bo->ttm.resource,
47 XE_MIGRATE_CLEAR_FLAG_FULL);
48 if (IS_ERR(ptr: fence)) {
49 KUNIT_FAIL(test, "Failed to submit bo clear.\n");
50 return PTR_ERR(ptr: fence);
51 }
52
53 if (dma_fence_wait_timeout(fence, false, 5 * HZ) <= 0) {
54 dma_fence_put(fence);
55 KUNIT_FAIL(test, "Timeout while clearing bo.\n");
56 return -ETIME;
57 }
58
59 dma_fence_put(fence);
60 }
61
62 /* Evict to system. CCS data should be copied. */
63 ret = xe_bo_evict(bo, exec);
64 if (ret) {
65 KUNIT_FAIL(test, "Failed to evict bo.\n");
66 return ret;
67 }
68
69 /* Sync all migration blits */
70 timeout = dma_resv_wait_timeout(bo->ttm.base.resv,
71 DMA_RESV_USAGE_KERNEL,
72 true,
73 5 * HZ);
74 if (timeout <= 0) {
75 KUNIT_FAIL(test, "Failed to sync bo eviction.\n");
76 return -ETIME;
77 }
78
79 /*
80 * Bo with CCS data is now in system memory. Verify backing store
81 * and data integrity. Then assign for the next testing round while
82 * we still have a CPU map.
83 */
84 ttm = bo->ttm.ttm;
85 if (!ttm || !ttm_tt_is_populated(ttm)) {
86 KUNIT_FAIL(test, "Bo was not in expected placement.\n");
87 return -EINVAL;
88 }
89
90 ccs_page = xe_bo_ccs_pages_start(bo) >> PAGE_SHIFT;
91 if (ccs_page >= ttm->num_pages) {
92 KUNIT_FAIL(test, "No TTM CCS pages present.\n");
93 return -EINVAL;
94 }
95
96 page = ttm->pages[ccs_page];
97 cpu_map = kmap_local_page(page);
98
99 /* Check first CCS value */
100 if (cpu_map[0] != get_val) {
101 KUNIT_FAIL(test,
102 "Expected CCS readout 0x%016llx, got 0x%016llx.\n",
103 (unsigned long long)get_val,
104 (unsigned long long)cpu_map[0]);
105 ret = -EINVAL;
106 }
107
108 /* Check last CCS value, or at least last value in page. */
109 offset = xe_device_ccs_bytes(tile_to_xe(tile), xe_bo_size(bo));
110 offset = min_t(u32, offset, PAGE_SIZE) / sizeof(u64) - 1;
111 if (cpu_map[offset] != get_val) {
112 KUNIT_FAIL(test,
113 "Expected CCS readout 0x%016llx, got 0x%016llx.\n",
114 (unsigned long long)get_val,
115 (unsigned long long)cpu_map[offset]);
116 ret = -EINVAL;
117 }
118
119 cpu_map[0] = assign_val;
120 cpu_map[offset] = assign_val;
121 kunmap_local(cpu_map);
122
123 return ret;
124}
125
126static void ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile,
127 struct kunit *test)
128{
129 struct xe_bo *bo;
130
131 int ret;
132
133 /* TODO: Sanity check */
134 unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
135 struct drm_exec *exec = XE_VALIDATION_OPT_OUT;
136
137 if (IS_DGFX(xe))
138 kunit_info(test, "Testing vram id %u\n", tile->id);
139 else
140 kunit_info(test, "Testing system memory\n");
141
142 bo = xe_bo_create_user(xe, NULL, SZ_1M, DRM_XE_GEM_CPU_CACHING_WC,
143 bo_flags, exec);
144 if (IS_ERR(ptr: bo)) {
145 KUNIT_FAIL(test, "Failed to create bo.\n");
146 return;
147 }
148
149 xe_bo_lock(bo, false);
150
151 kunit_info(test, "Verifying that CCS data is cleared on creation.\n");
152 ret = ccs_test_migrate(tile, bo, clear: false, get_val: 0ULL, assign_val: 0xdeadbeefdeadbeefULL,
153 test, exec);
154 if (ret)
155 goto out_unlock;
156
157 kunit_info(test, "Verifying that CCS data survives migration.\n");
158 ret = ccs_test_migrate(tile, bo, clear: false, get_val: 0xdeadbeefdeadbeefULL,
159 assign_val: 0xdeadbeefdeadbeefULL, test, exec);
160 if (ret)
161 goto out_unlock;
162
163 kunit_info(test, "Verifying that CCS data can be properly cleared.\n");
164 ret = ccs_test_migrate(tile, bo, clear: true, get_val: 0ULL, assign_val: 0ULL, test, exec);
165
166out_unlock:
167 xe_bo_unlock(bo);
168 xe_bo_put(bo);
169}
170
171static int ccs_test_run_device(struct xe_device *xe)
172{
173 struct kunit *test = kunit_get_current_test();
174 struct xe_tile *tile;
175 int id;
176
177 if (!xe_device_has_flat_ccs(xe)) {
178 kunit_skip(test, "non-flat-ccs device\n");
179 return 0;
180 }
181
182 /* For xe2+ dgfx, we don't handle ccs metadata */
183 if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)) {
184 kunit_skip(test, "xe2+ dgfx device\n");
185 return 0;
186 }
187
188 xe_pm_runtime_get(xe);
189
190 for_each_tile(tile, xe, id) {
191 /* For igfx run only for primary tile */
192 if (!IS_DGFX(xe) && id > 0)
193 continue;
194 ccs_test_run_tile(xe, tile, test);
195 }
196
197 xe_pm_runtime_put(xe);
198
199 return 0;
200}
201
202static void xe_ccs_migrate_kunit(struct kunit *test)
203{
204 struct xe_device *xe = test->priv;
205
206 ccs_test_run_device(xe);
207}
208
209static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struct kunit *test)
210{
211 struct xe_bo *bo, *external;
212 unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
213 struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->migrate);
214 struct drm_exec *exec = XE_VALIDATION_OPT_OUT;
215 struct xe_gt *__gt;
216 int err, i, id;
217
218 kunit_info(test, "Testing device %s vram id %u\n",
219 dev_name(xe->drm.dev), tile->id);
220
221 for (i = 0; i < 2; ++i) {
222 xe_vm_lock(vm, false);
223 bo = xe_bo_create_user(xe, vm, 0x10000,
224 DRM_XE_GEM_CPU_CACHING_WC,
225 bo_flags, exec);
226 xe_vm_unlock(vm);
227 if (IS_ERR(ptr: bo)) {
228 KUNIT_FAIL(test, "bo create err=%pe\n", bo);
229 break;
230 }
231
232 external = xe_bo_create_user(xe, NULL, 0x10000,
233 DRM_XE_GEM_CPU_CACHING_WC,
234 bo_flags, NULL);
235 if (IS_ERR(ptr: external)) {
236 KUNIT_FAIL(test, "external bo create err=%pe\n", external);
237 goto cleanup_bo;
238 }
239
240 xe_bo_lock(external, false);
241 err = xe_bo_pin_external(external, false, exec);
242 xe_bo_unlock(external);
243 if (err) {
244 KUNIT_FAIL(test, "external bo pin err=%pe\n",
245 ERR_PTR(err));
246 goto cleanup_external;
247 }
248
249 err = xe_bo_evict_all(xe);
250 if (err) {
251 KUNIT_FAIL(test, "evict err=%pe\n", ERR_PTR(err));
252 goto cleanup_all;
253 }
254
255 for_each_gt(__gt, xe, id)
256 xe_gt_sanitize(__gt);
257 err = xe_bo_restore_early(xe);
258 /*
259 * Snapshotting the CTB and copying back a potentially old
260 * version seems risky, depending on what might have been
261 * inflight. Also it seems snapshotting the ADS object and
262 * copying back results in serious breakage. Normally when
263 * calling xe_bo_restore_kernel() we always fully restart the
264 * GT, which re-intializes such things. We could potentially
265 * skip saving and restoring such objects in xe_bo_evict_all()
266 * however seems quite fragile not to also restart the GT. Try
267 * to do that here by triggering a GT reset.
268 */
269 for_each_gt(__gt, xe, id)
270 xe_gt_reset(__gt);
271
272 if (err) {
273 KUNIT_FAIL(test, "restore kernel err=%pe\n",
274 ERR_PTR(err));
275 goto cleanup_all;
276 }
277
278 err = xe_bo_restore_late(xe);
279 if (err) {
280 KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err));
281 goto cleanup_all;
282 }
283
284 if (!xe_bo_is_vram(external)) {
285 KUNIT_FAIL(test, "external bo is not vram\n");
286 err = -EPROTO;
287 goto cleanup_all;
288 }
289
290 if (xe_bo_is_vram(bo)) {
291 KUNIT_FAIL(test, "bo is vram\n");
292 err = -EPROTO;
293 goto cleanup_all;
294 }
295
296 if (i) {
297 down_read(sem: &vm->lock);
298 xe_vm_lock(vm, false);
299 err = xe_bo_validate(bo, bo->vm, false, exec);
300 xe_vm_unlock(vm);
301 up_read(sem: &vm->lock);
302 if (err) {
303 KUNIT_FAIL(test, "bo valid err=%pe\n",
304 ERR_PTR(err));
305 goto cleanup_all;
306 }
307 xe_bo_lock(external, false);
308 err = xe_bo_validate(external, NULL, false, exec);
309 xe_bo_unlock(external);
310 if (err) {
311 KUNIT_FAIL(test, "external bo valid err=%pe\n",
312 ERR_PTR(err));
313 goto cleanup_all;
314 }
315 }
316
317 xe_bo_lock(external, false);
318 xe_bo_unpin_external(external);
319 xe_bo_unlock(external);
320
321 xe_bo_put(external);
322
323 xe_bo_lock(bo, false);
324 __xe_bo_unset_bulk_move(bo);
325 xe_bo_unlock(bo);
326 xe_bo_put(bo);
327 continue;
328
329cleanup_all:
330 xe_bo_lock(external, false);
331 xe_bo_unpin_external(external);
332 xe_bo_unlock(external);
333cleanup_external:
334 xe_bo_put(external);
335cleanup_bo:
336 xe_bo_lock(bo, false);
337 __xe_bo_unset_bulk_move(bo);
338 xe_bo_unlock(bo);
339 xe_bo_put(bo);
340 break;
341 }
342
343 xe_vm_put(vm);
344
345 return 0;
346}
347
348static int evict_test_run_device(struct xe_device *xe)
349{
350 struct kunit *test = kunit_get_current_test();
351 struct xe_tile *tile;
352 int id;
353
354 if (!IS_DGFX(xe)) {
355 kunit_skip(test, "non-discrete device\n");
356 return 0;
357 }
358
359 xe_pm_runtime_get(xe);
360
361 for_each_tile(tile, xe, id)
362 evict_test_run_tile(xe, tile, test);
363
364 xe_pm_runtime_put(xe);
365
366 return 0;
367}
368
369static void xe_bo_evict_kunit(struct kunit *test)
370{
371 struct xe_device *xe = test->priv;
372
373 evict_test_run_device(xe);
374}
375
376struct xe_bo_link {
377 struct list_head link;
378 struct xe_bo *bo;
379 u32 val;
380};
381
382#define XE_BO_SHRINK_SIZE ((unsigned long)SZ_64M)
383
384static int shrink_test_fill_random(struct xe_bo *bo, struct rnd_state *state,
385 struct xe_bo_link *link)
386{
387 struct iosys_map map;
388 int ret = ttm_bo_vmap(&bo->ttm, &map);
389 size_t __maybe_unused i;
390
391 if (ret)
392 return ret;
393
394 for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) {
395 u32 val = prandom_u32_state(state);
396
397 iosys_map_wr(&map, i, u32, val);
398 if (i == 0)
399 link->val = val;
400 }
401
402 ttm_bo_vunmap(&bo->ttm, &map);
403 return 0;
404}
405
406static bool shrink_test_verify(struct kunit *test, struct xe_bo *bo,
407 unsigned int bo_nr, struct rnd_state *state,
408 struct xe_bo_link *link)
409{
410 struct iosys_map map;
411 int ret = ttm_bo_vmap(&bo->ttm, &map);
412 size_t i;
413 bool failed = false;
414
415 if (ret) {
416 KUNIT_FAIL(test, "Error mapping bo %u for content check.\n", bo_nr);
417 return true;
418 }
419
420 for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) {
421 u32 val = prandom_u32_state(state);
422
423 if (iosys_map_rd(&map, i, u32) != val) {
424 KUNIT_FAIL(test, "Content not preserved, bo %u offset 0x%016llx",
425 bo_nr, (unsigned long long)i);
426 kunit_info(test, "Failed value is 0x%08x, recorded 0x%08x\n",
427 (unsigned int)iosys_map_rd(&map, i, u32), val);
428 if (i == 0 && val != link->val)
429 kunit_info(test, "Looks like PRNG is out of sync.\n");
430 failed = true;
431 break;
432 }
433 }
434
435 ttm_bo_vunmap(&bo->ttm, &map);
436
437 return failed;
438}
439
440/*
441 * Try to create system bos corresponding to twice the amount
442 * of available system memory to test shrinker functionality.
443 * If no swap space is available to accommodate the
444 * memory overcommit, mark bos purgeable.
445 */
446static int shrink_test_run_device(struct xe_device *xe)
447{
448 struct kunit *test = kunit_get_current_test();
449 LIST_HEAD(bos);
450 struct xe_bo_link *link, *next;
451 struct sysinfo si;
452 u64 ram, ram_and_swap, purgeable = 0, alloced, to_alloc, limit;
453 unsigned int interrupted = 0, successful = 0, count = 0;
454 struct rnd_state prng;
455 u64 rand_seed;
456 bool failed = false;
457
458 rand_seed = get_random_u64();
459 prandom_seed_state(state: &prng, seed: rand_seed);
460 kunit_info(test, "Random seed is 0x%016llx.\n",
461 (unsigned long long)rand_seed);
462
463 /* Skip if execution time is expected to be too long. */
464
465 limit = SZ_32G;
466 /* IGFX with flat CCS needs to copy when swapping / shrinking */
467 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe))
468 limit = SZ_16G;
469
470 si_meminfo(val: &si);
471 ram = (size_t)si.freeram * si.mem_unit;
472 if (ram > limit) {
473 kunit_skip(test, "Too long expected execution time.\n");
474 return 0;
475 }
476 to_alloc = ram * 2;
477
478 ram_and_swap = ram + get_nr_swap_pages() * PAGE_SIZE;
479 if (to_alloc > ram_and_swap)
480 purgeable = to_alloc - ram_and_swap;
481 purgeable += div64_u64(dividend: purgeable, divisor: 5);
482
483 kunit_info(test, "Free ram is %lu bytes. Will allocate twice of that.\n",
484 (unsigned long)ram);
485 for (alloced = 0; alloced < to_alloc; alloced += XE_BO_SHRINK_SIZE) {
486 struct xe_bo *bo;
487 unsigned int mem_type;
488 struct xe_ttm_tt *xe_tt;
489
490 link = kzalloc(sizeof(*link), GFP_KERNEL);
491 if (!link) {
492 KUNIT_FAIL(test, "Unexpected link allocation failure\n");
493 failed = true;
494 break;
495 }
496
497 INIT_LIST_HEAD(list: &link->link);
498
499 /* We can create bos using WC caching here. But it is slower. */
500 bo = xe_bo_create_user(xe, NULL, XE_BO_SHRINK_SIZE,
501 DRM_XE_GEM_CPU_CACHING_WB,
502 XE_BO_FLAG_SYSTEM, NULL);
503 if (IS_ERR(ptr: bo)) {
504 if (bo != ERR_PTR(error: -ENOMEM) && bo != ERR_PTR(error: -ENOSPC) &&
505 bo != ERR_PTR(error: -EINTR) && bo != ERR_PTR(error: -ERESTARTSYS))
506 KUNIT_FAIL(test, "Error creating bo: %pe\n", bo);
507 kfree(objp: link);
508 failed = true;
509 break;
510 }
511 xe_bo_lock(bo, false);
512 xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm);
513
514 /*
515 * Allocate purgeable bos first, because if we do it the
516 * other way around, they may not be subject to swapping...
517 */
518 if (alloced < purgeable) {
519 xe_ttm_tt_account_subtract(xe, &xe_tt->ttm);
520 xe_tt->purgeable = true;
521 xe_ttm_tt_account_add(xe, &xe_tt->ttm);
522 bo->ttm.priority = 0;
523 spin_lock(lock: &bo->ttm.bdev->lru_lock);
524 ttm_bo_move_to_lru_tail(&bo->ttm);
525 spin_unlock(lock: &bo->ttm.bdev->lru_lock);
526 } else {
527 int ret = shrink_test_fill_random(bo, state: &prng, link);
528
529 if (ret) {
530 xe_bo_unlock(bo);
531 xe_bo_put(bo);
532 KUNIT_FAIL(test, "Error filling bo with random data: %pe\n",
533 ERR_PTR(ret));
534 kfree(objp: link);
535 failed = true;
536 break;
537 }
538 }
539
540 mem_type = bo->ttm.resource->mem_type;
541 xe_bo_unlock(bo);
542 link->bo = bo;
543 list_add_tail(new: &link->link, head: &bos);
544
545 if (mem_type != XE_PL_TT) {
546 KUNIT_FAIL(test, "Bo in incorrect memory type: %u\n",
547 bo->ttm.resource->mem_type);
548 failed = true;
549 }
550 cond_resched();
551 if (signal_pending(current))
552 break;
553 }
554
555 /*
556 * Read back and destroy bos. Reset the pseudo-random seed to get an
557 * identical pseudo-random number sequence for readback.
558 */
559 prandom_seed_state(state: &prng, seed: rand_seed);
560 list_for_each_entry_safe(link, next, &bos, link) {
561 static struct ttm_operation_ctx ctx = {.interruptible = true};
562 struct xe_bo *bo = link->bo;
563 struct xe_ttm_tt *xe_tt;
564 int ret;
565
566 count++;
567 if (!signal_pending(current) && !failed) {
568 bool purgeable, intr = false;
569
570 xe_bo_lock(bo, NULL);
571
572 /* xe_tt->purgeable is cleared on validate. */
573 xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm);
574 purgeable = xe_tt->purgeable;
575 do {
576 ret = ttm_bo_validate(&bo->ttm, &tt_placement, &ctx);
577 if (ret == -EINTR)
578 intr = true;
579 } while (ret == -EINTR && !signal_pending(current));
580 if (!ret && !purgeable)
581 failed = shrink_test_verify(test, bo, bo_nr: count, state: &prng, link);
582
583 xe_bo_unlock(bo);
584 if (ret) {
585 KUNIT_FAIL(test, "Validation failed: %pe\n",
586 ERR_PTR(ret));
587 failed = true;
588 } else if (intr) {
589 interrupted++;
590 } else {
591 successful++;
592 }
593 }
594 xe_bo_put(link->bo);
595 list_del(entry: &link->link);
596 kfree(objp: link);
597 }
598 kunit_info(test, "Readbacks interrupted: %u successful: %u\n",
599 interrupted, successful);
600
601 return 0;
602}
603
604static void xe_bo_shrink_kunit(struct kunit *test)
605{
606 struct xe_device *xe = test->priv;
607
608 shrink_test_run_device(xe);
609}
610
611static struct kunit_case xe_bo_tests[] = {
612 KUNIT_CASE_PARAM(xe_ccs_migrate_kunit, xe_pci_live_device_gen_param),
613 KUNIT_CASE_PARAM(xe_bo_evict_kunit, xe_pci_live_device_gen_param),
614 {}
615};
616
617VISIBLE_IF_KUNIT
618struct kunit_suite xe_bo_test_suite = {
619 .name = "xe_bo",
620 .test_cases = xe_bo_tests,
621 .init = xe_kunit_helper_xe_device_live_test_init,
622};
623EXPORT_SYMBOL_IF_KUNIT(xe_bo_test_suite);
624
625static struct kunit_case xe_bo_shrink_test[] = {
626 KUNIT_CASE_PARAM_ATTR(xe_bo_shrink_kunit, xe_pci_live_device_gen_param,
627 {.speed = KUNIT_SPEED_SLOW}),
628 {}
629};
630
631VISIBLE_IF_KUNIT
632struct kunit_suite xe_bo_shrink_test_suite = {
633 .name = "xe_bo_shrink",
634 .test_cases = xe_bo_shrink_test,
635 .init = xe_kunit_helper_xe_device_live_test_init,
636};
637EXPORT_SYMBOL_IF_KUNIT(xe_bo_shrink_test_suite);
638

source code of linux/drivers/gpu/drm/xe/tests/xe_bo.c