1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/prime_numbers.h>
26
27#include "gem/i915_gem_context.h"
28#include "gem/i915_gem_internal.h"
29#include "gem/selftests/mock_context.h"
30
31#include "i915_scatterlist.h"
32#include "i915_selftest.h"
33
34#include "mock_gem_device.h"
35#include "mock_gtt.h"
36
37static bool assert_vma(struct i915_vma *vma,
38 struct drm_i915_gem_object *obj,
39 struct i915_gem_context *ctx)
40{
41 bool ok = true;
42
43 if (vma->vm != ctx->vm) {
44 pr_err("VMA created with wrong VM\n");
45 ok = false;
46 }
47
48 if (vma->size != obj->base.size) {
49 pr_err("VMA created with wrong size, found %llu, expected %zu\n",
50 vma->size, obj->base.size);
51 ok = false;
52 }
53
54 if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) {
55 pr_err("VMA created with wrong type [%d]\n",
56 vma->gtt_view.type);
57 ok = false;
58 }
59
60 return ok;
61}
62
63static struct i915_vma *
64checked_vma_instance(struct drm_i915_gem_object *obj,
65 struct i915_address_space *vm,
66 const struct i915_gtt_view *view)
67{
68 struct i915_vma *vma;
69 bool ok = true;
70
71 vma = i915_vma_instance(obj, vm, view);
72 if (IS_ERR(ptr: vma))
73 return vma;
74
75 /* Manual checks, will be reinforced by i915_vma_compare! */
76 if (vma->vm != vm) {
77 pr_err("VMA's vm [%p] does not match request [%p]\n",
78 vma->vm, vm);
79 ok = false;
80 }
81
82 if (i915_is_ggtt(vm) != i915_vma_is_ggtt(vma)) {
83 pr_err("VMA ggtt status [%d] does not match parent [%d]\n",
84 i915_vma_is_ggtt(vma), i915_is_ggtt(vm));
85 ok = false;
86 }
87
88 if (i915_vma_compare(vma, vm, view)) {
89 pr_err("i915_vma_compare failed with create parameters!\n");
90 return ERR_PTR(error: -EINVAL);
91 }
92
93 if (i915_vma_compare(vma, vm: vma->vm,
94 view: i915_vma_is_ggtt(vma) ? &vma->gtt_view : NULL)) {
95 pr_err("i915_vma_compare failed with itself\n");
96 return ERR_PTR(error: -EINVAL);
97 }
98
99 if (!ok) {
100 pr_err("i915_vma_compare failed to detect the difference!\n");
101 return ERR_PTR(error: -EINVAL);
102 }
103
104 return vma;
105}
106
107static int create_vmas(struct drm_i915_private *i915,
108 struct list_head *objects,
109 struct list_head *contexts)
110{
111 struct drm_i915_gem_object *obj;
112 struct i915_gem_context *ctx;
113 int pinned;
114
115 list_for_each_entry(obj, objects, st_link) {
116 for (pinned = 0; pinned <= 1; pinned++) {
117 list_for_each_entry(ctx, contexts, link) {
118 struct i915_address_space *vm;
119 struct i915_vma *vma;
120 int err;
121
122 vm = i915_gem_context_get_eb_vm(ctx);
123 vma = checked_vma_instance(obj, vm, NULL);
124 i915_vm_put(vm);
125 if (IS_ERR(ptr: vma))
126 return PTR_ERR(ptr: vma);
127
128 if (!assert_vma(vma, obj, ctx)) {
129 pr_err("VMA lookup/create failed\n");
130 return -EINVAL;
131 }
132
133 if (!pinned) {
134 err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_USER);
135 if (err) {
136 pr_err("Failed to pin VMA\n");
137 return err;
138 }
139 } else {
140 i915_vma_unpin(vma);
141 }
142 }
143 }
144 }
145
146 return 0;
147}
148
149static int igt_vma_create(void *arg)
150{
151 struct i915_ggtt *ggtt = arg;
152 struct drm_i915_private *i915 = ggtt->vm.i915;
153 struct drm_i915_gem_object *obj, *on;
154 struct i915_gem_context *ctx, *cn;
155 unsigned long num_obj, num_ctx;
156 unsigned long no, nc;
157 IGT_TIMEOUT(end_time);
158 LIST_HEAD(contexts);
159 LIST_HEAD(objects);
160 int err = -ENOMEM;
161
162 /*
163 * Exercise creating many vma amongst many objections, checking the
164 * vma creation and lookup routines.
165 */
166
167 no = 0;
168 for_each_prime_number(num_obj, ULONG_MAX - 1) {
169 for (; no < num_obj; no++) {
170 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
171 if (IS_ERR(ptr: obj))
172 goto out;
173
174 list_add(new: &obj->st_link, head: &objects);
175 }
176
177 nc = 0;
178 for_each_prime_number(num_ctx, 2 * BITS_PER_LONG) {
179 for (; nc < num_ctx; nc++) {
180 ctx = mock_context(i915, name: "mock");
181 if (!ctx)
182 goto out;
183
184 list_move(list: &ctx->link, head: &contexts);
185 }
186
187 err = create_vmas(i915, objects: &objects, contexts: &contexts);
188 if (err)
189 goto out;
190
191 if (igt_timeout(end_time,
192 "%s timed out: after %lu objects in %lu contexts\n",
193 __func__, no, nc))
194 goto end;
195 }
196
197 list_for_each_entry_safe(ctx, cn, &contexts, link) {
198 list_del_init(entry: &ctx->link);
199 mock_context_close(ctx);
200 }
201
202 cond_resched();
203 }
204
205end:
206 /* Final pass to lookup all created contexts */
207 err = create_vmas(i915, objects: &objects, contexts: &contexts);
208out:
209 list_for_each_entry_safe(ctx, cn, &contexts, link) {
210 list_del_init(entry: &ctx->link);
211 mock_context_close(ctx);
212 }
213
214 list_for_each_entry_safe(obj, on, &objects, st_link)
215 i915_gem_object_put(obj);
216 return err;
217}
218
219struct pin_mode {
220 u64 size;
221 u64 flags;
222 bool (*assert)(const struct i915_vma *,
223 const struct pin_mode *mode,
224 int result);
225 const char *string;
226};
227
228static bool assert_pin_valid(const struct i915_vma *vma,
229 const struct pin_mode *mode,
230 int result)
231{
232 if (result)
233 return false;
234
235 if (i915_vma_misplaced(vma, size: mode->size, alignment: 0, flags: mode->flags))
236 return false;
237
238 return true;
239}
240
241__maybe_unused
242static bool assert_pin_enospc(const struct i915_vma *vma,
243 const struct pin_mode *mode,
244 int result)
245{
246 return result == -ENOSPC;
247}
248
249__maybe_unused
250static bool assert_pin_einval(const struct i915_vma *vma,
251 const struct pin_mode *mode,
252 int result)
253{
254 return result == -EINVAL;
255}
256
257static int igt_vma_pin1(void *arg)
258{
259 struct i915_ggtt *ggtt = arg;
260 const struct pin_mode modes[] = {
261#define VALID(sz, fl) { .size = (sz), .flags = (fl), .assert = assert_pin_valid, .string = #sz ", " #fl ", (valid) " }
262#define __INVALID(sz, fl, check, eval) { .size = (sz), .flags = (fl), .assert = (check), .string = #sz ", " #fl ", (invalid " #eval ")" }
263#define INVALID(sz, fl) __INVALID(sz, fl, assert_pin_einval, EINVAL)
264#define NOSPACE(sz, fl) __INVALID(sz, fl, assert_pin_enospc, ENOSPC)
265 VALID(0, PIN_GLOBAL),
266 VALID(0, PIN_GLOBAL | PIN_MAPPABLE),
267
268 VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 4096),
269 VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 8192),
270 VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
271 VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
272 VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)),
273
274 VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)),
275 INVALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | ggtt->mappable_end),
276 VALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)),
277 INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | ggtt->vm.total),
278 INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | round_down(U64_MAX, PAGE_SIZE)),
279
280 VALID(4096, PIN_GLOBAL),
281 VALID(8192, PIN_GLOBAL),
282 VALID(ggtt->mappable_end - 4096, PIN_GLOBAL | PIN_MAPPABLE),
283 VALID(ggtt->mappable_end, PIN_GLOBAL | PIN_MAPPABLE),
284 NOSPACE(ggtt->mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE),
285 VALID(ggtt->vm.total - 4096, PIN_GLOBAL),
286 VALID(ggtt->vm.total, PIN_GLOBAL),
287 NOSPACE(ggtt->vm.total + 4096, PIN_GLOBAL),
288 NOSPACE(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL),
289 INVALID(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)),
290 INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)),
291 INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (round_down(U64_MAX, PAGE_SIZE) - 4096)),
292
293 VALID(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
294
295#if !IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
296 /*
297 * Misusing BIAS is a programming error (it is not controllable
298 * from userspace) so when debugging is enabled, it explodes.
299 * However, the tests are still quite interesting for checking
300 * variable start, end and size.
301 */
302 NOSPACE(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | ggtt->mappable_end),
303 NOSPACE(0, PIN_GLOBAL | PIN_OFFSET_BIAS | ggtt->vm.total),
304 NOSPACE(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
305 NOSPACE(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)),
306#endif
307 { },
308#undef NOSPACE
309#undef INVALID
310#undef __INVALID
311#undef VALID
312 }, *m;
313 struct drm_i915_gem_object *obj;
314 struct i915_vma *vma;
315 int err = -EINVAL;
316
317 /*
318 * Exercise all the weird and wonderful i915_vma_pin requests,
319 * focusing on error handling of boundary conditions.
320 */
321
322 GEM_BUG_ON(!drm_mm_clean(&ggtt->vm.mm));
323
324 obj = i915_gem_object_create_internal(i915: ggtt->vm.i915, PAGE_SIZE);
325 if (IS_ERR(ptr: obj))
326 return PTR_ERR(ptr: obj);
327
328 vma = checked_vma_instance(obj, vm: &ggtt->vm, NULL);
329 if (IS_ERR(ptr: vma))
330 goto out;
331
332 for (m = modes; m->assert; m++) {
333 err = i915_vma_pin(vma, size: m->size, alignment: 0, flags: m->flags);
334 if (!m->assert(vma, m, err)) {
335 pr_err("%s to pin single page into GGTT with mode[%d:%s]: size=%llx flags=%llx, err=%d\n",
336 m->assert == assert_pin_valid ? "Failed" : "Unexpectedly succeeded",
337 (int)(m - modes), m->string, m->size, m->flags,
338 err);
339 if (!err)
340 i915_vma_unpin(vma);
341 err = -EINVAL;
342 goto out;
343 }
344
345 if (!err) {
346 i915_vma_unpin(vma);
347 err = i915_vma_unbind_unlocked(vma);
348 if (err) {
349 pr_err("Failed to unbind single page from GGTT, err=%d\n", err);
350 goto out;
351 }
352 }
353
354 cond_resched();
355 }
356
357 err = 0;
358out:
359 i915_gem_object_put(obj);
360 return err;
361}
362
363static unsigned long rotated_index(const struct intel_rotation_info *r,
364 unsigned int n,
365 unsigned int x,
366 unsigned int y)
367{
368 return (r->plane[n].src_stride * (r->plane[n].height - y - 1) +
369 r->plane[n].offset + x);
370}
371
372static struct scatterlist *
373assert_rotated(struct drm_i915_gem_object *obj,
374 const struct intel_rotation_info *r, unsigned int n,
375 struct scatterlist *sg)
376{
377 unsigned int x, y;
378
379 for (x = 0; x < r->plane[n].width; x++) {
380 unsigned int left;
381
382 for (y = 0; y < r->plane[n].height; y++) {
383 unsigned long src_idx;
384 dma_addr_t src;
385
386 if (!sg) {
387 pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
388 n, x, y);
389 return ERR_PTR(error: -EINVAL);
390 }
391
392 src_idx = rotated_index(r, n, x, y);
393 src = i915_gem_object_get_dma_address(obj, src_idx);
394
395 if (sg_dma_len(sg) != PAGE_SIZE) {
396 pr_err("Invalid sg.length, found %d, expected %lu for rotated page (%d, %d) [src index %lu]\n",
397 sg_dma_len(sg), PAGE_SIZE,
398 x, y, src_idx);
399 return ERR_PTR(error: -EINVAL);
400 }
401
402 if (sg_dma_address(sg) != src) {
403 pr_err("Invalid address for rotated page (%d, %d) [src index %lu]\n",
404 x, y, src_idx);
405 return ERR_PTR(error: -EINVAL);
406 }
407
408 sg = sg_next(sg);
409 }
410
411 left = (r->plane[n].dst_stride - y) * PAGE_SIZE;
412
413 if (!left)
414 continue;
415
416 if (!sg) {
417 pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
418 n, x, y);
419 return ERR_PTR(error: -EINVAL);
420 }
421
422 if (sg_dma_len(sg) != left) {
423 pr_err("Invalid sg.length, found %d, expected %u for rotated page (%d, %d)\n",
424 sg_dma_len(sg), left, x, y);
425 return ERR_PTR(error: -EINVAL);
426 }
427
428 if (sg_dma_address(sg) != 0) {
429 pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n",
430 &sg_dma_address(sg), x, y);
431 return ERR_PTR(error: -EINVAL);
432 }
433
434 sg = sg_next(sg);
435 }
436
437 return sg;
438}
439
440static unsigned long remapped_index(const struct intel_remapped_info *r,
441 unsigned int n,
442 unsigned int x,
443 unsigned int y)
444{
445 return (r->plane[n].src_stride * y +
446 r->plane[n].offset + x);
447}
448
449static struct scatterlist *
450assert_remapped(struct drm_i915_gem_object *obj,
451 const struct intel_remapped_info *r, unsigned int n,
452 struct scatterlist *sg)
453{
454 unsigned int x, y;
455 unsigned int left = 0;
456 unsigned int offset;
457
458 for (y = 0; y < r->plane[n].height; y++) {
459 for (x = 0; x < r->plane[n].width; x++) {
460 unsigned long src_idx;
461 dma_addr_t src;
462
463 if (!sg) {
464 pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
465 n, x, y);
466 return ERR_PTR(error: -EINVAL);
467 }
468 if (!left) {
469 offset = 0;
470 left = sg_dma_len(sg);
471 }
472
473 src_idx = remapped_index(r, n, x, y);
474 src = i915_gem_object_get_dma_address(obj, src_idx);
475
476 if (left < PAGE_SIZE || left & (PAGE_SIZE-1)) {
477 pr_err("Invalid sg.length, found %d, expected %lu for remapped page (%d, %d) [src index %lu]\n",
478 sg_dma_len(sg), PAGE_SIZE,
479 x, y, src_idx);
480 return ERR_PTR(error: -EINVAL);
481 }
482
483 if (sg_dma_address(sg) + offset != src) {
484 pr_err("Invalid address for remapped page (%d, %d) [src index %lu]\n",
485 x, y, src_idx);
486 return ERR_PTR(error: -EINVAL);
487 }
488
489 left -= PAGE_SIZE;
490 offset += PAGE_SIZE;
491
492
493 if (!left)
494 sg = sg_next(sg);
495 }
496
497 if (left) {
498 pr_err("Unexpected sg tail with %d size for remapped page (%d, %d)\n",
499 left,
500 x, y);
501 return ERR_PTR(error: -EINVAL);
502 }
503
504 left = (r->plane[n].dst_stride - r->plane[n].width) * PAGE_SIZE;
505
506 if (!left)
507 continue;
508
509 if (!sg) {
510 pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
511 n, x, y);
512 return ERR_PTR(error: -EINVAL);
513 }
514
515 if (sg_dma_len(sg) != left) {
516 pr_err("Invalid sg.length, found %u, expected %u for remapped page (%d, %d)\n",
517 sg_dma_len(sg), left,
518 x, y);
519 return ERR_PTR(error: -EINVAL);
520 }
521
522 if (sg_dma_address(sg) != 0) {
523 pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n",
524 &sg_dma_address(sg),
525 x, y);
526 return ERR_PTR(error: -EINVAL);
527 }
528
529 sg = sg_next(sg);
530 left = 0;
531 }
532
533 return sg;
534}
535
536static unsigned int remapped_size(enum i915_gtt_view_type view_type,
537 const struct intel_remapped_plane_info *a,
538 const struct intel_remapped_plane_info *b)
539{
540
541 if (view_type == I915_GTT_VIEW_ROTATED)
542 return a->dst_stride * a->width + b->dst_stride * b->width;
543 else
544 return a->dst_stride * a->height + b->dst_stride * b->height;
545}
546
547static int igt_vma_rotate_remap(void *arg)
548{
549 struct i915_ggtt *ggtt = arg;
550 struct i915_address_space *vm = &ggtt->vm;
551 struct drm_i915_gem_object *obj;
552 const struct intel_remapped_plane_info planes[] = {
553 { .width = 1, .height = 1, .src_stride = 1 },
554 { .width = 2, .height = 2, .src_stride = 2 },
555 { .width = 4, .height = 4, .src_stride = 4 },
556 { .width = 8, .height = 8, .src_stride = 8 },
557
558 { .width = 3, .height = 5, .src_stride = 3 },
559 { .width = 3, .height = 5, .src_stride = 4 },
560 { .width = 3, .height = 5, .src_stride = 5 },
561
562 { .width = 5, .height = 3, .src_stride = 5 },
563 { .width = 5, .height = 3, .src_stride = 7 },
564 { .width = 5, .height = 3, .src_stride = 9 },
565
566 { .width = 4, .height = 6, .src_stride = 6 },
567 { .width = 6, .height = 4, .src_stride = 6 },
568
569 { .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 },
570 { .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 },
571 { .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 },
572
573 { }
574 }, *a, *b;
575 enum i915_gtt_view_type types[] = {
576 I915_GTT_VIEW_ROTATED,
577 I915_GTT_VIEW_REMAPPED,
578 0,
579 }, *t;
580 const unsigned int max_pages = 64;
581 int err = -ENOMEM;
582
583 /*
584 * Create VMA for many different combinations of planes and check
585 * that the page layout within the rotated VMA match our expectations.
586 */
587
588 obj = i915_gem_object_create_internal(i915: vm->i915, size: max_pages * PAGE_SIZE);
589 if (IS_ERR(ptr: obj))
590 goto out;
591
592 for (t = types; *t; t++) {
593 for (a = planes; a->width; a++) {
594 for (b = planes + ARRAY_SIZE(planes); b-- != planes; ) {
595 struct i915_gtt_view view = {
596 .type = *t,
597 .remapped.plane[0] = *a,
598 .remapped.plane[1] = *b,
599 };
600 struct intel_remapped_plane_info *plane_info = view.remapped.plane;
601 unsigned int n, max_offset;
602
603 max_offset = max(plane_info[0].src_stride * plane_info[0].height,
604 plane_info[1].src_stride * plane_info[1].height);
605 GEM_BUG_ON(max_offset > max_pages);
606 max_offset = max_pages - max_offset;
607
608 if (!plane_info[0].dst_stride)
609 plane_info[0].dst_stride = view.type == I915_GTT_VIEW_ROTATED ?
610 plane_info[0].height :
611 plane_info[0].width;
612 if (!plane_info[1].dst_stride)
613 plane_info[1].dst_stride = view.type == I915_GTT_VIEW_ROTATED ?
614 plane_info[1].height :
615 plane_info[1].width;
616
617 for_each_prime_number_from(plane_info[0].offset, 0, max_offset) {
618 for_each_prime_number_from(plane_info[1].offset, 0, max_offset) {
619 struct scatterlist *sg;
620 struct i915_vma *vma;
621 unsigned int expected_pages;
622
623 vma = checked_vma_instance(obj, vm, view: &view);
624 if (IS_ERR(ptr: vma)) {
625 err = PTR_ERR(ptr: vma);
626 goto out_object;
627 }
628
629 err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_GLOBAL);
630 if (err) {
631 pr_err("Failed to pin VMA, err=%d\n", err);
632 goto out_object;
633 }
634
635 expected_pages = remapped_size(view_type: view.type, a: &plane_info[0], b: &plane_info[1]);
636
637 if (view.type == I915_GTT_VIEW_ROTATED &&
638 vma->size != expected_pages * PAGE_SIZE) {
639 pr_err("VMA is wrong size, expected %lu, found %llu\n",
640 PAGE_SIZE * expected_pages, vma->size);
641 err = -EINVAL;
642 goto out_object;
643 }
644
645 if (view.type == I915_GTT_VIEW_REMAPPED &&
646 vma->size > expected_pages * PAGE_SIZE) {
647 pr_err("VMA is wrong size, expected %lu, found %llu\n",
648 PAGE_SIZE * expected_pages, vma->size);
649 err = -EINVAL;
650 goto out_object;
651 }
652
653 if (vma->pages->nents > expected_pages) {
654 pr_err("sg table is wrong sizeo, expected %u, found %u nents\n",
655 expected_pages, vma->pages->nents);
656 err = -EINVAL;
657 goto out_object;
658 }
659
660 if (vma->node.size < vma->size) {
661 pr_err("VMA binding too small, expected %llu, found %llu\n",
662 vma->size, vma->node.size);
663 err = -EINVAL;
664 goto out_object;
665 }
666
667 if (vma->pages == obj->mm.pages) {
668 pr_err("VMA using unrotated object pages!\n");
669 err = -EINVAL;
670 goto out_object;
671 }
672
673 sg = vma->pages->sgl;
674 for (n = 0; n < ARRAY_SIZE(view.rotated.plane); n++) {
675 if (view.type == I915_GTT_VIEW_ROTATED)
676 sg = assert_rotated(obj, r: &view.rotated, n, sg);
677 else
678 sg = assert_remapped(obj, r: &view.remapped, n, sg);
679 if (IS_ERR(ptr: sg)) {
680 pr_err("Inconsistent %s VMA pages for plane %d: [(%d, %d, %d, %d, %d), (%d, %d, %d, %d, %d)]\n",
681 view.type == I915_GTT_VIEW_ROTATED ?
682 "rotated" : "remapped", n,
683 plane_info[0].width,
684 plane_info[0].height,
685 plane_info[0].src_stride,
686 plane_info[0].dst_stride,
687 plane_info[0].offset,
688 plane_info[1].width,
689 plane_info[1].height,
690 plane_info[1].src_stride,
691 plane_info[1].dst_stride,
692 plane_info[1].offset);
693 err = -EINVAL;
694 goto out_object;
695 }
696 }
697
698 i915_vma_unpin(vma);
699 err = i915_vma_unbind_unlocked(vma);
700 if (err) {
701 pr_err("Unbinding returned %i\n", err);
702 goto out_object;
703 }
704 cond_resched();
705 }
706 }
707 }
708 }
709 }
710
711out_object:
712 i915_gem_object_put(obj);
713out:
714 return err;
715}
716
717static bool assert_partial(struct drm_i915_gem_object *obj,
718 struct i915_vma *vma,
719 unsigned long offset,
720 unsigned long size)
721{
722 struct sgt_iter sgt;
723 dma_addr_t dma;
724
725 for_each_sgt_daddr(dma, sgt, vma->pages) {
726 dma_addr_t src;
727
728 if (!size) {
729 pr_err("Partial scattergather list too long\n");
730 return false;
731 }
732
733 src = i915_gem_object_get_dma_address(obj, offset);
734 if (src != dma) {
735 pr_err("DMA mismatch for partial page offset %lu\n",
736 offset);
737 return false;
738 }
739
740 offset++;
741 size--;
742 }
743
744 return true;
745}
746
747static bool assert_pin(struct i915_vma *vma,
748 struct i915_gtt_view *view,
749 u64 size,
750 const char *name)
751{
752 bool ok = true;
753
754 if (vma->size != size) {
755 pr_err("(%s) VMA is wrong size, expected %llu, found %llu\n",
756 name, size, vma->size);
757 ok = false;
758 }
759
760 if (vma->node.size < vma->size) {
761 pr_err("(%s) VMA binding too small, expected %llu, found %llu\n",
762 name, vma->size, vma->node.size);
763 ok = false;
764 }
765
766 if (view && view->type != I915_GTT_VIEW_NORMAL) {
767 if (memcmp(p: &vma->gtt_view, q: view, size: sizeof(*view))) {
768 pr_err("(%s) VMA mismatch upon creation!\n",
769 name);
770 ok = false;
771 }
772
773 if (vma->pages == vma->obj->mm.pages) {
774 pr_err("(%s) VMA using original object pages!\n",
775 name);
776 ok = false;
777 }
778 } else {
779 if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) {
780 pr_err("Not the normal ggtt view! Found %d\n",
781 vma->gtt_view.type);
782 ok = false;
783 }
784
785 if (vma->pages != vma->obj->mm.pages) {
786 pr_err("VMA not using object pages!\n");
787 ok = false;
788 }
789 }
790
791 return ok;
792}
793
794static int igt_vma_partial(void *arg)
795{
796 struct i915_ggtt *ggtt = arg;
797 struct i915_address_space *vm = &ggtt->vm;
798 const unsigned int npages = 1021; /* prime! */
799 struct drm_i915_gem_object *obj;
800 const struct phase {
801 const char *name;
802 } phases[] = {
803 { "create" },
804 { "lookup" },
805 { },
806 }, *p;
807 unsigned int sz, offset;
808 struct i915_vma *vma;
809 int err = -ENOMEM;
810
811 /*
812 * Create lots of different VMA for the object and check that
813 * we are returned the same VMA when we later request the same range.
814 */
815
816 obj = i915_gem_object_create_internal(i915: vm->i915, size: npages * PAGE_SIZE);
817 if (IS_ERR(ptr: obj))
818 goto out;
819
820 for (p = phases; p->name; p++) { /* exercise both create/lookup */
821 unsigned int count, nvma;
822
823 nvma = 0;
824 for_each_prime_number_from(sz, 1, npages) {
825 for_each_prime_number_from(offset, 0, npages - sz) {
826 struct i915_gtt_view view;
827
828 view.type = I915_GTT_VIEW_PARTIAL;
829 view.partial.offset = offset;
830 view.partial.size = sz;
831
832 if (sz == npages)
833 view.type = I915_GTT_VIEW_NORMAL;
834
835 vma = checked_vma_instance(obj, vm, view: &view);
836 if (IS_ERR(ptr: vma)) {
837 err = PTR_ERR(ptr: vma);
838 goto out_object;
839 }
840
841 err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_GLOBAL);
842 if (err)
843 goto out_object;
844
845 if (!assert_pin(vma, view: &view, size: sz*PAGE_SIZE, name: p->name)) {
846 pr_err("(%s) Inconsistent partial pinning for (offset=%d, size=%d)\n",
847 p->name, offset, sz);
848 err = -EINVAL;
849 goto out_object;
850 }
851
852 if (!assert_partial(obj, vma, offset, size: sz)) {
853 pr_err("(%s) Inconsistent partial pages for (offset=%d, size=%d)\n",
854 p->name, offset, sz);
855 err = -EINVAL;
856 goto out_object;
857 }
858
859 i915_vma_unpin(vma);
860 nvma++;
861 err = i915_vma_unbind_unlocked(vma);
862 if (err) {
863 pr_err("Unbinding returned %i\n", err);
864 goto out_object;
865 }
866
867 cond_resched();
868 }
869 }
870
871 count = 0;
872 list_for_each_entry(vma, &obj->vma.list, obj_link)
873 count++;
874 if (count != nvma) {
875 pr_err("(%s) All partial vma were not recorded on the obj->vma_list: found %u, expected %u\n",
876 p->name, count, nvma);
877 err = -EINVAL;
878 goto out_object;
879 }
880
881 /* Check that we did create the whole object mapping */
882 vma = checked_vma_instance(obj, vm, NULL);
883 if (IS_ERR(ptr: vma)) {
884 err = PTR_ERR(ptr: vma);
885 goto out_object;
886 }
887
888 err = i915_vma_pin(vma, size: 0, alignment: 0, PIN_GLOBAL);
889 if (err)
890 goto out_object;
891
892 if (!assert_pin(vma, NULL, size: obj->base.size, name: p->name)) {
893 pr_err("(%s) inconsistent full pin\n", p->name);
894 err = -EINVAL;
895 goto out_object;
896 }
897
898 i915_vma_unpin(vma);
899
900 err = i915_vma_unbind_unlocked(vma);
901 if (err) {
902 pr_err("Unbinding returned %i\n", err);
903 goto out_object;
904 }
905
906 count = 0;
907 list_for_each_entry(vma, &obj->vma.list, obj_link)
908 count++;
909 if (count != nvma) {
910 pr_err("(%s) allocated an extra full vma!\n", p->name);
911 err = -EINVAL;
912 goto out_object;
913 }
914 }
915
916out_object:
917 i915_gem_object_put(obj);
918out:
919 return err;
920}
921
922int i915_vma_mock_selftests(void)
923{
924 static const struct i915_subtest tests[] = {
925 SUBTEST(igt_vma_create),
926 SUBTEST(igt_vma_pin1),
927 SUBTEST(igt_vma_rotate_remap),
928 SUBTEST(igt_vma_partial),
929 };
930 struct drm_i915_private *i915;
931 struct intel_gt *gt;
932 int err;
933
934 i915 = mock_gem_device();
935 if (!i915)
936 return -ENOMEM;
937
938 /* allocate the ggtt */
939 err = intel_gt_assign_ggtt(gt: to_gt(i915));
940 if (err)
941 goto out_put;
942
943 gt = to_gt(i915);
944
945 mock_init_ggtt(gt);
946
947 err = i915_subtests(tests, gt->ggtt);
948
949 mock_device_flush(i915);
950 i915_gem_drain_freed_objects(i915);
951 mock_fini_ggtt(ggtt: gt->ggtt);
952
953out_put:
954 mock_destroy_device(i915);
955 return err;
956}
957
958static int igt_vma_remapped_gtt(void *arg)
959{
960 struct drm_i915_private *i915 = arg;
961 const struct intel_remapped_plane_info planes[] = {
962 { .width = 1, .height = 1, .src_stride = 1 },
963 { .width = 2, .height = 2, .src_stride = 2 },
964 { .width = 4, .height = 4, .src_stride = 4 },
965 { .width = 8, .height = 8, .src_stride = 8 },
966
967 { .width = 3, .height = 5, .src_stride = 3 },
968 { .width = 3, .height = 5, .src_stride = 4 },
969 { .width = 3, .height = 5, .src_stride = 5 },
970
971 { .width = 5, .height = 3, .src_stride = 5 },
972 { .width = 5, .height = 3, .src_stride = 7 },
973 { .width = 5, .height = 3, .src_stride = 9 },
974
975 { .width = 4, .height = 6, .src_stride = 6 },
976 { .width = 6, .height = 4, .src_stride = 6 },
977
978 { .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 },
979 { .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 },
980 { .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 },
981
982 { }
983 }, *p;
984 enum i915_gtt_view_type types[] = {
985 I915_GTT_VIEW_ROTATED,
986 I915_GTT_VIEW_REMAPPED,
987 0,
988 }, *t;
989 struct drm_i915_gem_object *obj;
990 intel_wakeref_t wakeref;
991 int err = 0;
992
993 if (!i915_ggtt_has_aperture(ggtt: to_gt(i915)->ggtt))
994 return 0;
995
996 obj = i915_gem_object_create_internal(i915, size: 10 * 10 * PAGE_SIZE);
997 if (IS_ERR(ptr: obj))
998 return PTR_ERR(ptr: obj);
999
1000 wakeref = intel_runtime_pm_get(rpm: &i915->runtime_pm);
1001
1002 for (t = types; *t; t++) {
1003 for (p = planes; p->width; p++) {
1004 struct i915_gtt_view view = {
1005 .type = *t,
1006 .rotated.plane[0] = *p,
1007 };
1008 struct intel_remapped_plane_info *plane_info = view.rotated.plane;
1009 struct i915_vma *vma;
1010 u32 __iomem *map;
1011 unsigned int x, y;
1012
1013 i915_gem_object_lock(obj, NULL);
1014 err = i915_gem_object_set_to_gtt_domain(obj, write: true);
1015 i915_gem_object_unlock(obj);
1016 if (err)
1017 goto out;
1018
1019 if (!plane_info[0].dst_stride)
1020 plane_info[0].dst_stride = *t == I915_GTT_VIEW_ROTATED ?
1021 p->height : p->width;
1022
1023 vma = i915_gem_object_ggtt_pin(obj, view: &view, size: 0, alignment: 0, PIN_MAPPABLE);
1024 if (IS_ERR(ptr: vma)) {
1025 err = PTR_ERR(ptr: vma);
1026 goto out;
1027 }
1028
1029 GEM_BUG_ON(vma->gtt_view.type != *t);
1030
1031 map = i915_vma_pin_iomap(vma);
1032 i915_vma_unpin(vma);
1033 if (IS_ERR(ptr: map)) {
1034 err = PTR_ERR(ptr: map);
1035 goto out;
1036 }
1037
1038 for (y = 0 ; y < plane_info[0].height; y++) {
1039 for (x = 0 ; x < plane_info[0].width; x++) {
1040 unsigned int offset;
1041 u32 val = y << 16 | x;
1042
1043 if (*t == I915_GTT_VIEW_ROTATED)
1044 offset = (x * plane_info[0].dst_stride + y) * PAGE_SIZE;
1045 else
1046 offset = (y * plane_info[0].dst_stride + x) * PAGE_SIZE;
1047
1048 iowrite32(val, &map[offset / sizeof(*map)]);
1049 }
1050 }
1051
1052 i915_vma_unpin_iomap(vma);
1053
1054 vma = i915_gem_object_ggtt_pin(obj, NULL, size: 0, alignment: 0, PIN_MAPPABLE);
1055 if (IS_ERR(ptr: vma)) {
1056 err = PTR_ERR(ptr: vma);
1057 goto out;
1058 }
1059
1060 GEM_BUG_ON(vma->gtt_view.type != I915_GTT_VIEW_NORMAL);
1061
1062 map = i915_vma_pin_iomap(vma);
1063 i915_vma_unpin(vma);
1064 if (IS_ERR(ptr: map)) {
1065 err = PTR_ERR(ptr: map);
1066 goto out;
1067 }
1068
1069 for (y = 0 ; y < plane_info[0].height; y++) {
1070 for (x = 0 ; x < plane_info[0].width; x++) {
1071 unsigned int offset, src_idx;
1072 u32 exp = y << 16 | x;
1073 u32 val;
1074
1075 if (*t == I915_GTT_VIEW_ROTATED)
1076 src_idx = rotated_index(r: &view.rotated, n: 0, x, y);
1077 else
1078 src_idx = remapped_index(r: &view.remapped, n: 0, x, y);
1079 offset = src_idx * PAGE_SIZE;
1080
1081 val = ioread32(&map[offset / sizeof(*map)]);
1082 if (val != exp) {
1083 pr_err("%s VMA write test failed, expected 0x%x, found 0x%x\n",
1084 *t == I915_GTT_VIEW_ROTATED ? "Rotated" : "Remapped",
1085 exp, val);
1086 i915_vma_unpin_iomap(vma);
1087 err = -EINVAL;
1088 goto out;
1089 }
1090 }
1091 }
1092 i915_vma_unpin_iomap(vma);
1093
1094 cond_resched();
1095 }
1096 }
1097
1098out:
1099 intel_runtime_pm_put(rpm: &i915->runtime_pm, wref: wakeref);
1100 i915_gem_object_put(obj);
1101
1102 return err;
1103}
1104
1105int i915_vma_live_selftests(struct drm_i915_private *i915)
1106{
1107 static const struct i915_subtest tests[] = {
1108 SUBTEST(igt_vma_remapped_gtt),
1109 };
1110
1111 return i915_live_subtests(tests, i915);
1112}
1113

source code of linux/drivers/gpu/drm/i915/selftests/i915_vma.c