1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020-2022 Intel Corporation
4 */
5
6#include <kunit/test.h>
7#include <kunit/visibility.h>
8
9#include "tests/xe_kunit_helpers.h"
10#include "tests/xe_pci_test.h"
11
12#include "xe_pci.h"
13#include "xe_pm.h"
14
15static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence,
16 const char *str, struct kunit *test)
17{
18 long ret;
19
20 if (IS_ERR(ptr: fence)) {
21 KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str,
22 PTR_ERR(fence));
23 return true;
24 }
25 if (!fence)
26 return true;
27
28 ret = dma_fence_wait_timeout(fence, false, 5 * HZ);
29 if (ret <= 0) {
30 KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret);
31 return true;
32 }
33
34 return false;
35}
36
37static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe,
38 struct xe_bb *bb, u32 second_idx, const char *str,
39 struct kunit *test)
40{
41 u64 batch_base = xe_migrate_batch_base(m, xe->info.has_usm);
42 struct xe_sched_job *job = xe_bb_create_migration_job(m->q, bb,
43 batch_base,
44 second_idx);
45 struct dma_fence *fence;
46
47 if (IS_ERR(ptr: job)) {
48 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
49 PTR_ERR(job));
50 return PTR_ERR(ptr: job);
51 }
52
53 xe_sched_job_arm(job);
54 fence = dma_fence_get(&job->drm.s_fence->finished);
55 xe_sched_job_push(job);
56
57 if (sanity_fence_failed(xe, fence, str, test))
58 return -ETIMEDOUT;
59
60 dma_fence_put(fence);
61 kunit_info(test, "%s: Job completed\n", str);
62 return 0;
63}
64
65#define check(_retval, _expected, str, _test) \
66 do { if ((_retval) != (_expected)) { \
67 KUNIT_FAIL(_test, "Sanity check failed: " str \
68 " expected %llx, got %llx\n", \
69 (u64)(_expected), (u64)(_retval)); \
70 } } while (0)
71
72static void test_copy(struct xe_migrate *m, struct xe_bo *bo,
73 struct kunit *test, u32 region, struct drm_exec *exec)
74{
75 struct xe_device *xe = tile_to_xe(m->tile);
76 u64 retval, expected = 0;
77 bool big = xe_bo_size(bo) >= SZ_2M;
78 struct dma_fence *fence;
79 const char *str = big ? "Copying big bo" : "Copying small bo";
80 int err;
81
82 struct xe_bo *remote = xe_bo_create_locked(xe, m->tile, NULL,
83 xe_bo_size(bo),
84 ttm_bo_type_kernel,
85 region |
86 XE_BO_FLAG_NEEDS_CPU_ACCESS |
87 XE_BO_FLAG_PINNED,
88 exec);
89 if (IS_ERR(ptr: remote)) {
90 KUNIT_FAIL(test, "Failed to allocate remote bo for %s: %pe\n",
91 str, remote);
92 return;
93 }
94
95 err = xe_bo_validate(remote, NULL, false, exec);
96 if (err) {
97 KUNIT_FAIL(test, "Failed to validate system bo for %s: %i\n",
98 str, err);
99 goto out_unlock;
100 }
101
102 err = xe_bo_vmap(remote);
103 if (err) {
104 KUNIT_FAIL(test, "Failed to vmap system bo for %s: %i\n",
105 str, err);
106 goto out_unlock;
107 }
108
109 xe_map_memset(xe, &remote->vmap, 0, 0xd0, xe_bo_size(remote));
110 fence = xe_migrate_clear(m, remote, remote->ttm.resource,
111 XE_MIGRATE_CLEAR_FLAG_FULL);
112 if (!sanity_fence_failed(xe, fence, str: big ? "Clearing remote big bo" :
113 "Clearing remote small bo", test)) {
114 retval = xe_map_rd(xe, &remote->vmap, 0, u64);
115 check(retval, expected, "remote first offset should be cleared",
116 test);
117 retval = xe_map_rd(xe, &remote->vmap, xe_bo_size(remote) - 8, u64);
118 check(retval, expected, "remote last offset should be cleared",
119 test);
120 }
121 dma_fence_put(fence);
122
123 /* Try to copy 0xc0 from remote to vram with 2MB or 64KiB/4KiB pages */
124 xe_map_memset(xe, &remote->vmap, 0, 0xc0, xe_bo_size(remote));
125 xe_map_memset(xe, &bo->vmap, 0, 0xd0, xe_bo_size(bo));
126
127 expected = 0xc0c0c0c0c0c0c0c0;
128 fence = xe_migrate_copy(m, remote, bo, remote->ttm.resource,
129 bo->ttm.resource, false);
130 if (!sanity_fence_failed(xe, fence, str: big ? "Copying big bo remote -> vram" :
131 "Copying small bo remote -> vram", test)) {
132 retval = xe_map_rd(xe, &bo->vmap, 0, u64);
133 check(retval, expected,
134 "remote -> vram bo first offset should be copied", test);
135 retval = xe_map_rd(xe, &bo->vmap, xe_bo_size(bo) - 8, u64);
136 check(retval, expected,
137 "remote -> vram bo offset should be copied", test);
138 }
139 dma_fence_put(fence);
140
141 /* And other way around.. slightly hacky.. */
142 xe_map_memset(xe, &remote->vmap, 0, 0xd0, xe_bo_size(remote));
143 xe_map_memset(xe, &bo->vmap, 0, 0xc0, xe_bo_size(bo));
144
145 fence = xe_migrate_copy(m, bo, remote, bo->ttm.resource,
146 remote->ttm.resource, false);
147 if (!sanity_fence_failed(xe, fence, str: big ? "Copying big bo vram -> remote" :
148 "Copying small bo vram -> remote", test)) {
149 retval = xe_map_rd(xe, &remote->vmap, 0, u64);
150 check(retval, expected,
151 "vram -> remote bo first offset should be copied", test);
152 retval = xe_map_rd(xe, &remote->vmap, xe_bo_size(bo) - 8, u64);
153 check(retval, expected,
154 "vram -> remote bo last offset should be copied", test);
155 }
156 dma_fence_put(fence);
157
158 xe_bo_vunmap(remote);
159out_unlock:
160 xe_bo_unlock(remote);
161 xe_bo_put(remote);
162}
163
164static void test_copy_sysmem(struct xe_migrate *m, struct xe_bo *bo,
165 struct drm_exec *exec, struct kunit *test)
166{
167 test_copy(m, bo, test, region: XE_BO_FLAG_SYSTEM, exec);
168}
169
170static void test_copy_vram(struct xe_migrate *m, struct xe_bo *bo,
171 struct drm_exec *exec, struct kunit *test)
172{
173 u32 region;
174
175 if (bo->ttm.resource->mem_type == XE_PL_SYSTEM)
176 return;
177
178 if (bo->ttm.resource->mem_type == XE_PL_VRAM0)
179 region = XE_BO_FLAG_VRAM1;
180 else
181 region = XE_BO_FLAG_VRAM0;
182 test_copy(m, bo, test, region, exec);
183}
184
185static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test,
186 struct drm_exec *exec)
187{
188 struct xe_tile *tile = m->tile;
189 struct xe_device *xe = tile_to_xe(tile);
190 struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny;
191 struct xe_res_cursor src_it;
192 struct dma_fence *fence;
193 u64 retval, expected;
194 struct xe_bb *bb;
195 int err;
196 u8 id = tile->id;
197
198 err = xe_bo_vmap(bo);
199 if (err) {
200 KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n",
201 PTR_ERR(bo));
202 return;
203 }
204
205 big = xe_bo_create_pin_map(xe, tile, m->q->vm, SZ_4M,
206 ttm_bo_type_kernel,
207 XE_BO_FLAG_VRAM_IF_DGFX(tile),
208 exec);
209 if (IS_ERR(ptr: big)) {
210 KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big));
211 goto vunmap;
212 }
213
214 pt = xe_bo_create_pin_map(xe, tile, m->q->vm, XE_PAGE_SIZE,
215 ttm_bo_type_kernel,
216 XE_BO_FLAG_VRAM_IF_DGFX(tile),
217 exec);
218 if (IS_ERR(ptr: pt)) {
219 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
220 PTR_ERR(pt));
221 goto free_big;
222 }
223
224 tiny = xe_bo_create_pin_map(xe, tile, m->q->vm,
225 2 * SZ_4K,
226 ttm_bo_type_kernel,
227 XE_BO_FLAG_VRAM_IF_DGFX(tile),
228 exec);
229 if (IS_ERR(ptr: tiny)) {
230 KUNIT_FAIL(test, "Failed to allocate tiny fake pt: %li\n",
231 PTR_ERR(tiny));
232 goto free_pt;
233 }
234
235 bb = xe_bb_new(tile->primary_gt, 32, xe->info.has_usm);
236 if (IS_ERR(ptr: bb)) {
237 KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n",
238 PTR_ERR(bb));
239 goto free_tiny;
240 }
241
242 kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n",
243 (unsigned long)xe_bo_main_addr(m->q->vm->pt_root[id]->bo, XE_PAGE_SIZE),
244 (unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE));
245
246 /* First part of the test, are we updating our pagetable bo with a new entry? */
247 xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64,
248 0xdeaddeadbeefbeef);
249 expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe->pat.idx[XE_CACHE_WB], 0);
250 if (m->q->vm->flags & XE_VM_FLAG_64K)
251 expected |= XE_PTE_PS64;
252 if (xe_bo_is_vram(pt))
253 xe_res_first(pt->ttm.resource, 0, xe_bo_size(pt), &src_it);
254 else
255 xe_res_first_sg(xe_bo_sg(pt), 0, xe_bo_size(pt), &src_it);
256
257 emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt), false,
258 &src_it, XE_PAGE_SIZE, pt->ttm.resource);
259
260 run_sanity_job(m, xe, bb, second_idx: bb->len, str: "Writing PTE for our fake PT", test);
261
262 retval = xe_map_rd(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1),
263 u64);
264 check(retval, expected, "PTE entry write", test);
265
266 /* Now try to write data to our newly mapped 'pagetable', see if it succeeds */
267 bb->len = 0;
268 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
269 xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead);
270 expected = 0;
271
272 emit_clear(tile->primary_gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4,
273 IS_DGFX(xe));
274 run_sanity_job(m, xe, bb, second_idx: 1, str: "Writing to our newly mapped pagetable",
275 test);
276
277 retval = xe_map_rd(xe, &pt->vmap, 0, u32);
278 check(retval, expected, "Write to PT after adding PTE", test);
279
280 /* Sanity checks passed, try the full ones! */
281
282 /* Clear a small bo */
283 kunit_info(test, "Clearing small buffer object\n");
284 xe_map_memset(xe, &tiny->vmap, 0, 0x22, xe_bo_size(tiny));
285 expected = 0;
286 fence = xe_migrate_clear(m, tiny, tiny->ttm.resource,
287 XE_MIGRATE_CLEAR_FLAG_FULL);
288 if (sanity_fence_failed(xe, fence, str: "Clearing small bo", test))
289 goto out;
290
291 dma_fence_put(fence);
292 retval = xe_map_rd(xe, &tiny->vmap, 0, u32);
293 check(retval, expected, "Command clear small first value", test);
294 retval = xe_map_rd(xe, &tiny->vmap, xe_bo_size(tiny) - 4, u32);
295 check(retval, expected, "Command clear small last value", test);
296
297 kunit_info(test, "Copying small buffer object to system\n");
298 test_copy_sysmem(m, bo: tiny, exec, test);
299 if (xe->info.tile_count > 1) {
300 kunit_info(test, "Copying small buffer object to other vram\n");
301 test_copy_vram(m, bo: tiny, exec, test);
302 }
303
304 /* Clear a big bo */
305 kunit_info(test, "Clearing big buffer object\n");
306 xe_map_memset(xe, &big->vmap, 0, 0x11, xe_bo_size(big));
307 expected = 0;
308 fence = xe_migrate_clear(m, big, big->ttm.resource,
309 XE_MIGRATE_CLEAR_FLAG_FULL);
310 if (sanity_fence_failed(xe, fence, str: "Clearing big bo", test))
311 goto out;
312
313 dma_fence_put(fence);
314 retval = xe_map_rd(xe, &big->vmap, 0, u32);
315 check(retval, expected, "Command clear big first value", test);
316 retval = xe_map_rd(xe, &big->vmap, xe_bo_size(big) - 4, u32);
317 check(retval, expected, "Command clear big last value", test);
318
319 kunit_info(test, "Copying big buffer object to system\n");
320 test_copy_sysmem(m, bo: big, exec, test);
321 if (xe->info.tile_count > 1) {
322 kunit_info(test, "Copying big buffer object to other vram\n");
323 test_copy_vram(m, bo: big, exec, test);
324 }
325
326out:
327 xe_bb_free(bb, NULL);
328free_tiny:
329 xe_bo_unpin(tiny);
330 xe_bo_put(tiny);
331free_pt:
332 xe_bo_unpin(pt);
333 xe_bo_put(pt);
334free_big:
335 xe_bo_unpin(big);
336 xe_bo_put(big);
337vunmap:
338 xe_bo_vunmap(m->pt_bo);
339}
340
341static int migrate_test_run_device(struct xe_device *xe)
342{
343 struct kunit *test = kunit_get_current_test();
344 struct xe_tile *tile;
345 int id;
346
347 xe_pm_runtime_get(xe);
348
349 for_each_tile(tile, xe, id) {
350 struct xe_migrate *m = tile->migrate;
351 struct drm_exec *exec = XE_VALIDATION_OPT_OUT;
352
353 kunit_info(test, "Testing tile id %d.\n", id);
354 xe_vm_lock(m->q->vm, false);
355 xe_migrate_sanity_test(m, test, exec);
356 xe_vm_unlock(m->q->vm);
357 }
358
359 xe_pm_runtime_put(xe);
360
361 return 0;
362}
363
364static void xe_migrate_sanity_kunit(struct kunit *test)
365{
366 struct xe_device *xe = test->priv;
367
368 migrate_test_run_device(xe);
369}
370
371static struct dma_fence *blt_copy(struct xe_tile *tile,
372 struct xe_bo *src_bo, struct xe_bo *dst_bo,
373 bool copy_only_ccs, const char *str, struct kunit *test)
374{
375 struct xe_gt *gt = tile->primary_gt;
376 struct xe_migrate *m = tile->migrate;
377 struct xe_device *xe = gt_to_xe(gt);
378 struct dma_fence *fence = NULL;
379 u64 size = xe_bo_size(src_bo);
380 struct xe_res_cursor src_it, dst_it;
381 struct ttm_resource *src = src_bo->ttm.resource, *dst = dst_bo->ttm.resource;
382 u64 src_L0_ofs, dst_L0_ofs;
383 u32 src_L0_pt, dst_L0_pt;
384 u64 src_L0, dst_L0;
385 int err;
386 bool src_is_vram = mem_type_is_vram(src->mem_type);
387 bool dst_is_vram = mem_type_is_vram(dst->mem_type);
388
389 if (!src_is_vram)
390 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
391 else
392 xe_res_first(src, 0, size, &src_it);
393
394 if (!dst_is_vram)
395 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it);
396 else
397 xe_res_first(dst, 0, size, &dst_it);
398
399 while (size) {
400 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */
401 struct xe_sched_job *job;
402 struct xe_bb *bb;
403 u32 flush_flags = 0;
404 u32 update_idx;
405 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
406 u32 pte_flags;
407
408 src_L0 = xe_migrate_res_sizes(m, &src_it);
409 dst_L0 = xe_migrate_res_sizes(m, &dst_it);
410
411 src_L0 = min(src_L0, dst_L0);
412
413 pte_flags = src_is_vram ? (PTE_UPDATE_FLAG_IS_VRAM |
414 PTE_UPDATE_FLAG_IS_COMP_PTE) : 0;
415 batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0,
416 &src_L0_ofs, &src_L0_pt, 0, 0,
417 avail_pts);
418
419 pte_flags = dst_is_vram ? (PTE_UPDATE_FLAG_IS_VRAM |
420 PTE_UPDATE_FLAG_IS_COMP_PTE) : 0;
421 batch_size += pte_update_size(m, pte_flags, dst, &dst_it, &src_L0,
422 &dst_L0_ofs, &dst_L0_pt, 0,
423 avail_pts, avail_pts);
424
425 /* Add copy commands size here */
426 batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +
427 ((xe_device_has_flat_ccs(xe) && copy_only_ccs) ? EMIT_COPY_CCS_DW : 0);
428
429 bb = xe_bb_new(gt, batch_size, xe->info.has_usm);
430 if (IS_ERR(ptr: bb)) {
431 err = PTR_ERR(ptr: bb);
432 goto err_sync;
433 }
434
435 if (src_is_vram)
436 xe_res_next(&src_it, src_L0);
437 else
438 emit_pte(m, bb, src_L0_pt, src_is_vram, false,
439 &src_it, src_L0, src);
440
441 if (dst_is_vram)
442 xe_res_next(&dst_it, src_L0);
443 else
444 emit_pte(m, bb, dst_L0_pt, dst_is_vram, false,
445 &dst_it, src_L0, dst);
446
447 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
448 update_idx = bb->len;
449 if (!copy_only_ccs)
450 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);
451
452 if (copy_only_ccs)
453 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs,
454 src_is_vram, dst_L0_ofs,
455 dst_is_vram, src_L0, dst_L0_ofs,
456 copy_only_ccs);
457
458 job = xe_bb_create_migration_job(m->q, bb,
459 xe_migrate_batch_base(m, xe->info.has_usm),
460 update_idx);
461 if (IS_ERR(ptr: job)) {
462 err = PTR_ERR(ptr: job);
463 goto err;
464 }
465
466 xe_sched_job_add_migrate_flush(job, flush_flags);
467
468 mutex_lock(&m->job_mutex);
469 xe_sched_job_arm(job);
470 dma_fence_put(fence);
471 fence = dma_fence_get(&job->drm.s_fence->finished);
472 xe_sched_job_push(job);
473
474 dma_fence_put(m->fence);
475 m->fence = dma_fence_get(fence);
476
477 mutex_unlock(lock: &m->job_mutex);
478
479 xe_bb_free(bb, fence);
480 size -= src_L0;
481 continue;
482
483err:
484 xe_bb_free(bb, NULL);
485
486err_sync:
487 if (fence) {
488 dma_fence_wait(fence, false);
489 dma_fence_put(fence);
490 }
491 return ERR_PTR(error: err);
492 }
493
494 return fence;
495}
496
497static void test_migrate(struct xe_device *xe, struct xe_tile *tile,
498 struct xe_bo *sys_bo, struct xe_bo *vram_bo, struct xe_bo *ccs_bo,
499 struct drm_exec *exec, struct kunit *test)
500{
501 struct dma_fence *fence;
502 u64 expected, retval;
503 long timeout;
504 long ret;
505
506 expected = 0xd0d0d0d0d0d0d0d0;
507 xe_map_memset(xe, &sys_bo->vmap, 0, 0xd0, xe_bo_size(sys_bo));
508
509 fence = blt_copy(tile, src_bo: sys_bo, dst_bo: vram_bo, copy_only_ccs: false, str: "Blit copy from sysmem to vram", test);
510 if (!sanity_fence_failed(xe, fence, str: "Blit copy from sysmem to vram", test)) {
511 retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
512 if (retval == expected)
513 KUNIT_FAIL(test, "Sanity check failed: VRAM must have compressed value\n");
514 }
515 dma_fence_put(fence);
516
517 kunit_info(test, "Evict vram buffer object\n");
518 ret = xe_bo_evict(vram_bo, exec);
519 if (ret) {
520 KUNIT_FAIL(test, "Failed to evict bo.\n");
521 return;
522 }
523
524 ret = xe_bo_vmap(vram_bo);
525 if (ret) {
526 KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret);
527 return;
528 }
529
530 retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
531 check(retval, expected, "Clear evicted vram data first value", test);
532 retval = xe_map_rd(xe, &vram_bo->vmap, xe_bo_size(vram_bo) - 8, u64);
533 check(retval, expected, "Clear evicted vram data last value", test);
534
535 fence = blt_copy(tile, src_bo: vram_bo, dst_bo: ccs_bo,
536 copy_only_ccs: true, str: "Blit surf copy from vram to sysmem", test);
537 if (!sanity_fence_failed(xe, fence, str: "Clear ccs buffer data", test)) {
538 retval = xe_map_rd(xe, &ccs_bo->vmap, 0, u64);
539 check(retval, 0, "Clear ccs data first value", test);
540
541 retval = xe_map_rd(xe, &ccs_bo->vmap, xe_bo_size(ccs_bo) - 8, u64);
542 check(retval, 0, "Clear ccs data last value", test);
543 }
544 dma_fence_put(fence);
545
546 kunit_info(test, "Restore vram buffer object\n");
547 ret = xe_bo_validate(vram_bo, NULL, false, exec);
548 if (ret) {
549 KUNIT_FAIL(test, "Failed to validate vram bo for: %li\n", ret);
550 return;
551 }
552
553 /* Sync all migration blits */
554 timeout = dma_resv_wait_timeout(vram_bo->ttm.base.resv,
555 DMA_RESV_USAGE_KERNEL,
556 true,
557 5 * HZ);
558 if (timeout <= 0) {
559 KUNIT_FAIL(test, "Failed to sync bo eviction.\n");
560 return;
561 }
562
563 ret = xe_bo_vmap(vram_bo);
564 if (ret) {
565 KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret);
566 return;
567 }
568
569 retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
570 check(retval, expected, "Restored value must be equal to initial value", test);
571 retval = xe_map_rd(xe, &vram_bo->vmap, xe_bo_size(vram_bo) - 8, u64);
572 check(retval, expected, "Restored value must be equal to initial value", test);
573
574 fence = blt_copy(tile, src_bo: vram_bo, dst_bo: ccs_bo,
575 copy_only_ccs: true, str: "Blit surf copy from vram to sysmem", test);
576 if (!sanity_fence_failed(xe, fence, str: "Clear ccs buffer data", test)) {
577 retval = xe_map_rd(xe, &ccs_bo->vmap, 0, u64);
578 check(retval, 0, "Clear ccs data first value", test);
579 retval = xe_map_rd(xe, &ccs_bo->vmap, xe_bo_size(ccs_bo) - 8, u64);
580 check(retval, 0, "Clear ccs data last value", test);
581 }
582 dma_fence_put(fence);
583}
584
585static void test_clear(struct xe_device *xe, struct xe_tile *tile,
586 struct xe_bo *sys_bo, struct xe_bo *vram_bo, struct kunit *test)
587{
588 struct dma_fence *fence;
589 u64 expected, retval;
590
591 expected = 0xd0d0d0d0d0d0d0d0;
592 xe_map_memset(xe, &sys_bo->vmap, 0, 0xd0, xe_bo_size(sys_bo));
593
594 fence = blt_copy(tile, src_bo: sys_bo, dst_bo: vram_bo, copy_only_ccs: false, str: "Blit copy from sysmem to vram", test);
595 if (!sanity_fence_failed(xe, fence, str: "Blit copy from sysmem to vram", test)) {
596 retval = xe_map_rd(xe, &vram_bo->vmap, 0, u64);
597 if (retval == expected)
598 KUNIT_FAIL(test, "Sanity check failed: VRAM must have compressed value\n");
599 }
600 dma_fence_put(fence);
601
602 fence = blt_copy(tile, src_bo: vram_bo, dst_bo: sys_bo, copy_only_ccs: false, str: "Blit copy from vram to sysmem", test);
603 if (!sanity_fence_failed(xe, fence, str: "Blit copy from vram to sysmem", test)) {
604 retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64);
605 check(retval, expected, "Decompressed value must be equal to initial value", test);
606 retval = xe_map_rd(xe, &sys_bo->vmap, xe_bo_size(sys_bo) - 8, u64);
607 check(retval, expected, "Decompressed value must be equal to initial value", test);
608 }
609 dma_fence_put(fence);
610
611 kunit_info(test, "Clear vram buffer object\n");
612 expected = 0x0000000000000000;
613 fence = xe_migrate_clear(tile->migrate, vram_bo, vram_bo->ttm.resource,
614 XE_MIGRATE_CLEAR_FLAG_FULL);
615 if (sanity_fence_failed(xe, fence, str: "Clear vram_bo", test))
616 return;
617 dma_fence_put(fence);
618
619 fence = blt_copy(tile, src_bo: vram_bo, dst_bo: sys_bo,
620 copy_only_ccs: false, str: "Blit copy from vram to sysmem", test);
621 if (!sanity_fence_failed(xe, fence, str: "Clear main buffer data", test)) {
622 retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64);
623 check(retval, expected, "Clear main buffer first value", test);
624 retval = xe_map_rd(xe, &sys_bo->vmap, xe_bo_size(sys_bo) - 8, u64);
625 check(retval, expected, "Clear main buffer last value", test);
626 }
627 dma_fence_put(fence);
628
629 fence = blt_copy(tile, src_bo: vram_bo, dst_bo: sys_bo,
630 copy_only_ccs: true, str: "Blit surf copy from vram to sysmem", test);
631 if (!sanity_fence_failed(xe, fence, str: "Clear ccs buffer data", test)) {
632 retval = xe_map_rd(xe, &sys_bo->vmap, 0, u64);
633 check(retval, expected, "Clear ccs data first value", test);
634 retval = xe_map_rd(xe, &sys_bo->vmap, xe_bo_size(sys_bo) - 8, u64);
635 check(retval, expected, "Clear ccs data last value", test);
636 }
637 dma_fence_put(fence);
638}
639
640static void validate_ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile,
641 struct kunit *test)
642{
643 struct xe_bo *sys_bo, *vram_bo = NULL, *ccs_bo = NULL;
644 unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
645 struct drm_exec *exec;
646 long ret;
647
648 sys_bo = xe_bo_create_user(xe, NULL, SZ_4M,
649 DRM_XE_GEM_CPU_CACHING_WC,
650 XE_BO_FLAG_SYSTEM |
651 XE_BO_FLAG_NEEDS_CPU_ACCESS |
652 XE_BO_FLAG_PINNED, NULL);
653
654 if (IS_ERR(ptr: sys_bo)) {
655 KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
656 PTR_ERR(sys_bo));
657 return;
658 }
659
660 exec = XE_VALIDATION_OPT_OUT;
661 xe_bo_lock(sys_bo, false);
662 ret = xe_bo_validate(sys_bo, NULL, false, exec);
663 if (ret) {
664 KUNIT_FAIL(test, "Failed to validate system bo for: %li\n", ret);
665 goto free_sysbo;
666 }
667
668 ret = xe_bo_vmap(sys_bo);
669 if (ret) {
670 KUNIT_FAIL(test, "Failed to vmap system bo: %li\n", ret);
671 goto free_sysbo;
672 }
673 xe_bo_unlock(sys_bo);
674
675 ccs_bo = xe_bo_create_user(xe, NULL, SZ_4M,
676 DRM_XE_GEM_CPU_CACHING_WC,
677 bo_flags | XE_BO_FLAG_NEEDS_CPU_ACCESS |
678 XE_BO_FLAG_PINNED, NULL);
679
680 if (IS_ERR(ptr: ccs_bo)) {
681 KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
682 PTR_ERR(ccs_bo));
683 return;
684 }
685
686 xe_bo_lock(ccs_bo, false);
687 ret = xe_bo_validate(ccs_bo, NULL, false, exec);
688 if (ret) {
689 KUNIT_FAIL(test, "Failed to validate system bo for: %li\n", ret);
690 goto free_ccsbo;
691 }
692
693 ret = xe_bo_vmap(ccs_bo);
694 if (ret) {
695 KUNIT_FAIL(test, "Failed to vmap system bo: %li\n", ret);
696 goto free_ccsbo;
697 }
698 xe_bo_unlock(ccs_bo);
699
700 vram_bo = xe_bo_create_user(xe, NULL, SZ_4M,
701 DRM_XE_GEM_CPU_CACHING_WC,
702 bo_flags | XE_BO_FLAG_NEEDS_CPU_ACCESS |
703 XE_BO_FLAG_PINNED, NULL);
704 if (IS_ERR(ptr: vram_bo)) {
705 KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n",
706 PTR_ERR(vram_bo));
707 return;
708 }
709
710 xe_bo_lock(vram_bo, false);
711 ret = xe_bo_validate(vram_bo, NULL, false, exec);
712 if (ret) {
713 KUNIT_FAIL(test, "Failed to validate vram bo for: %li\n", ret);
714 goto free_vrambo;
715 }
716
717 ret = xe_bo_vmap(vram_bo);
718 if (ret) {
719 KUNIT_FAIL(test, "Failed to vmap vram bo: %li\n", ret);
720 goto free_vrambo;
721 }
722
723 test_clear(xe, tile, sys_bo, vram_bo, test);
724 test_migrate(xe, tile, sys_bo, vram_bo, ccs_bo, exec, test);
725 xe_bo_unlock(vram_bo);
726
727 xe_bo_lock(vram_bo, false);
728 xe_bo_vunmap(vram_bo);
729 xe_bo_unlock(vram_bo);
730
731 xe_bo_lock(ccs_bo, false);
732 xe_bo_vunmap(ccs_bo);
733 xe_bo_unlock(ccs_bo);
734
735 xe_bo_lock(sys_bo, false);
736 xe_bo_vunmap(sys_bo);
737 xe_bo_unlock(sys_bo);
738free_vrambo:
739 xe_bo_put(vram_bo);
740free_ccsbo:
741 xe_bo_put(ccs_bo);
742free_sysbo:
743 xe_bo_put(sys_bo);
744}
745
746static int validate_ccs_test_run_device(struct xe_device *xe)
747{
748 struct kunit *test = kunit_get_current_test();
749 struct xe_tile *tile;
750 int id;
751
752 if (!xe_device_has_flat_ccs(xe)) {
753 kunit_skip(test, "non-flat-ccs device\n");
754 return 0;
755 }
756
757 if (!(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe))) {
758 kunit_skip(test, "non-xe2 discrete device\n");
759 return 0;
760 }
761
762 xe_pm_runtime_get(xe);
763
764 for_each_tile(tile, xe, id)
765 validate_ccs_test_run_tile(xe, tile, test);
766
767 xe_pm_runtime_put(xe);
768
769 return 0;
770}
771
772static void xe_validate_ccs_kunit(struct kunit *test)
773{
774 struct xe_device *xe = test->priv;
775
776 validate_ccs_test_run_device(xe);
777}
778
779static struct kunit_case xe_migrate_tests[] = {
780 KUNIT_CASE_PARAM(xe_migrate_sanity_kunit, xe_pci_live_device_gen_param),
781 KUNIT_CASE_PARAM(xe_validate_ccs_kunit, xe_pci_live_device_gen_param),
782 {}
783};
784
785VISIBLE_IF_KUNIT
786struct kunit_suite xe_migrate_test_suite = {
787 .name = "xe_migrate",
788 .test_cases = xe_migrate_tests,
789 .init = xe_kunit_helper_xe_device_live_test_init,
790};
791EXPORT_SYMBOL_IF_KUNIT(xe_migrate_test_suite);
792

source code of linux/drivers/gpu/drm/xe/tests/xe_migrate.c