1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2017 Intel Corporation
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 *
24 */
25
26#include <linux/highmem.h>
27#include <linux/sched/mm.h>
28
29#include <drm/drm_cache.h>
30#include <drm/drm_print.h>
31
32#include "display/intel_frontbuffer.h"
33#include "pxp/intel_pxp.h"
34
35#include "i915_drv.h"
36#include "i915_file_private.h"
37#include "i915_gem_clflush.h"
38#include "i915_gem_context.h"
39#include "i915_gem_dmabuf.h"
40#include "i915_gem_mman.h"
41#include "i915_gem_object.h"
42#include "i915_gem_object_frontbuffer.h"
43#include "i915_gem_ttm.h"
44#include "i915_memcpy.h"
45#include "i915_trace.h"
46
47static struct kmem_cache *slab_objects;
48
49static const struct drm_gem_object_funcs i915_gem_object_funcs;
50
51unsigned int i915_gem_get_pat_index(struct drm_i915_private *i915,
52 enum i915_cache_level level)
53{
54 if (drm_WARN_ON(&i915->drm, level >= I915_MAX_CACHE_LEVEL))
55 return 0;
56
57 return INTEL_INFO(i915)->cachelevel_to_pat[level];
58}
59
60bool i915_gem_object_has_cache_level(const struct drm_i915_gem_object *obj,
61 enum i915_cache_level lvl)
62{
63 /*
64 * In case the pat_index is set by user space, this kernel mode
65 * driver should leave the coherency to be managed by user space,
66 * simply return true here.
67 */
68 if (obj->pat_set_by_user)
69 return true;
70
71 /*
72 * Otherwise the pat_index should have been converted from cache_level
73 * so that the following comparison is valid.
74 */
75 return obj->pat_index == i915_gem_get_pat_index(obj_to_i915(obj), level: lvl);
76}
77
78struct drm_i915_gem_object *i915_gem_object_alloc(void)
79{
80 struct drm_i915_gem_object *obj;
81
82 obj = kmem_cache_zalloc(slab_objects, GFP_KERNEL);
83 if (!obj)
84 return NULL;
85 obj->base.funcs = &i915_gem_object_funcs;
86
87 return obj;
88}
89
90void i915_gem_object_free(struct drm_i915_gem_object *obj)
91{
92 return kmem_cache_free(s: slab_objects, objp: obj);
93}
94
95void i915_gem_object_init(struct drm_i915_gem_object *obj,
96 const struct drm_i915_gem_object_ops *ops,
97 struct lock_class_key *key, unsigned flags)
98{
99 /*
100 * A gem object is embedded both in a struct ttm_buffer_object :/ and
101 * in a drm_i915_gem_object. Make sure they are aliased.
102 */
103 BUILD_BUG_ON(offsetof(typeof(*obj), base) !=
104 offsetof(typeof(*obj), __do_not_access.base));
105
106 spin_lock_init(&obj->vma.lock);
107 INIT_LIST_HEAD(list: &obj->vma.list);
108
109 INIT_LIST_HEAD(list: &obj->mm.link);
110
111#ifdef CONFIG_PROC_FS
112 INIT_LIST_HEAD(list: &obj->client_link);
113#endif
114
115 INIT_LIST_HEAD(list: &obj->lut_list);
116 spin_lock_init(&obj->lut_lock);
117
118 spin_lock_init(&obj->mmo.lock);
119 obj->mmo.offsets = RB_ROOT;
120
121 init_rcu_head(head: &obj->rcu);
122
123 obj->ops = ops;
124 GEM_BUG_ON(flags & ~I915_BO_ALLOC_FLAGS);
125 obj->flags = flags;
126
127 obj->mm.madv = I915_MADV_WILLNEED;
128 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
129 mutex_init(&obj->mm.get_page.lock);
130 INIT_RADIX_TREE(&obj->mm.get_dma_page.radix, GFP_KERNEL | __GFP_NOWARN);
131 mutex_init(&obj->mm.get_dma_page.lock);
132}
133
134/**
135 * __i915_gem_object_fini - Clean up a GEM object initialization
136 * @obj: The gem object to cleanup
137 *
138 * This function cleans up gem object fields that are set up by
139 * drm_gem_private_object_init() and i915_gem_object_init().
140 * It's primarily intended as a helper for backends that need to
141 * clean up the gem object in separate steps.
142 */
143void __i915_gem_object_fini(struct drm_i915_gem_object *obj)
144{
145 mutex_destroy(lock: &obj->mm.get_page.lock);
146 mutex_destroy(lock: &obj->mm.get_dma_page.lock);
147 dma_resv_fini(obj: &obj->base._resv);
148}
149
150/**
151 * i915_gem_object_set_cache_coherency - Mark up the object's coherency levels
152 * for a given cache_level
153 * @obj: #drm_i915_gem_object
154 * @cache_level: cache level
155 */
156void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
157 unsigned int cache_level)
158{
159 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
160
161 obj->pat_index = i915_gem_get_pat_index(i915, level: cache_level);
162
163 if (cache_level != I915_CACHE_NONE)
164 obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ |
165 I915_BO_CACHE_COHERENT_FOR_WRITE);
166 else if (HAS_LLC(i915))
167 obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ;
168 else
169 obj->cache_coherent = 0;
170
171 obj->cache_dirty =
172 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) &&
173 !IS_DGFX(i915);
174}
175
176/**
177 * i915_gem_object_set_pat_index - set PAT index to be used in PTE encode
178 * @obj: #drm_i915_gem_object
179 * @pat_index: PAT index
180 *
181 * This is a clone of i915_gem_object_set_cache_coherency taking pat index
182 * instead of cache_level as its second argument.
183 */
184void i915_gem_object_set_pat_index(struct drm_i915_gem_object *obj,
185 unsigned int pat_index)
186{
187 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
188
189 if (obj->pat_index == pat_index)
190 return;
191
192 obj->pat_index = pat_index;
193
194 if (pat_index != i915_gem_get_pat_index(i915, level: I915_CACHE_NONE))
195 obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ |
196 I915_BO_CACHE_COHERENT_FOR_WRITE);
197 else if (HAS_LLC(i915))
198 obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ;
199 else
200 obj->cache_coherent = 0;
201
202 obj->cache_dirty =
203 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) &&
204 !IS_DGFX(i915);
205}
206
207bool i915_gem_object_can_bypass_llc(struct drm_i915_gem_object *obj)
208{
209 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
210
211 /*
212 * This is purely from a security perspective, so we simply don't care
213 * about non-userspace objects being able to bypass the LLC.
214 */
215 if (!(obj->flags & I915_BO_ALLOC_USER))
216 return false;
217
218 /*
219 * Always flush cache for UMD objects at creation time.
220 */
221 if (obj->pat_set_by_user)
222 return true;
223
224 /*
225 * EHL and JSL add the 'Bypass LLC' MOCS entry, which should make it
226 * possible for userspace to bypass the GTT caching bits set by the
227 * kernel, as per the given object cache_level. This is troublesome
228 * since the heavy flush we apply when first gathering the pages is
229 * skipped if the kernel thinks the object is coherent with the GPU. As
230 * a result it might be possible to bypass the cache and read the
231 * contents of the page directly, which could be stale data. If it's
232 * just a case of userspace shooting themselves in the foot then so be
233 * it, but since i915 takes the stance of always zeroing memory before
234 * handing it to userspace, we need to prevent this.
235 */
236 return (IS_JASPERLAKE(i915) || IS_ELKHARTLAKE(i915));
237}
238
239static void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
240{
241 struct drm_i915_gem_object *obj = to_intel_bo(gem);
242 struct drm_i915_file_private *fpriv = file->driver_priv;
243 struct i915_lut_handle bookmark = {};
244 struct i915_mmap_offset *mmo, *mn;
245 struct i915_lut_handle *lut, *ln;
246 LIST_HEAD(close);
247
248 spin_lock(lock: &obj->lut_lock);
249 list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {
250 struct i915_gem_context *ctx = lut->ctx;
251
252 if (ctx && ctx->file_priv == fpriv) {
253 i915_gem_context_get(ctx);
254 list_move(list: &lut->obj_link, head: &close);
255 }
256
257 /* Break long locks, and carefully continue on from this spot */
258 if (&ln->obj_link != &obj->lut_list) {
259 list_add_tail(new: &bookmark.obj_link, head: &ln->obj_link);
260 if (cond_resched_lock(&obj->lut_lock))
261 list_safe_reset_next(&bookmark, ln, obj_link);
262 __list_del_entry(entry: &bookmark.obj_link);
263 }
264 }
265 spin_unlock(lock: &obj->lut_lock);
266
267 spin_lock(lock: &obj->mmo.lock);
268 rbtree_postorder_for_each_entry_safe(mmo, mn, &obj->mmo.offsets, offset)
269 drm_vma_node_revoke(node: &mmo->vma_node, tag: file);
270 spin_unlock(lock: &obj->mmo.lock);
271
272 list_for_each_entry_safe(lut, ln, &close, obj_link) {
273 struct i915_gem_context *ctx = lut->ctx;
274 struct i915_vma *vma;
275
276 /*
277 * We allow the process to have multiple handles to the same
278 * vma, in the same fd namespace, by virtue of flink/open.
279 */
280
281 mutex_lock(&ctx->lut_mutex);
282 vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
283 if (vma) {
284 GEM_BUG_ON(vma->obj != obj);
285 GEM_BUG_ON(!atomic_read(&vma->open_count));
286 i915_vma_close(vma);
287 }
288 mutex_unlock(lock: &ctx->lut_mutex);
289
290 i915_gem_context_put(ctx: lut->ctx);
291 i915_lut_handle_free(lut);
292 i915_gem_object_put(obj);
293 }
294}
295
296void __i915_gem_free_object_rcu(struct rcu_head *head)
297{
298 struct drm_i915_gem_object *obj =
299 container_of(head, typeof(*obj), rcu);
300 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
301
302 /* We need to keep this alive for RCU read access from fdinfo. */
303 if (obj->mm.n_placements > 1)
304 kfree(objp: obj->mm.placements);
305
306 i915_gem_object_free(obj);
307
308 GEM_BUG_ON(!atomic_read(&i915->mm.free_count));
309 atomic_dec(v: &i915->mm.free_count);
310}
311
312static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj)
313{
314 /* Skip serialisation and waking the device if known to be not used. */
315
316 if (obj->userfault_count && !IS_DGFX(to_i915(obj->base.dev)))
317 i915_gem_object_release_mmap_gtt(obj);
318
319 if (!RB_EMPTY_ROOT(&obj->mmo.offsets)) {
320 struct i915_mmap_offset *mmo, *mn;
321
322 i915_gem_object_release_mmap_offset(obj);
323
324 rbtree_postorder_for_each_entry_safe(mmo, mn,
325 &obj->mmo.offsets,
326 offset) {
327 drm_vma_offset_remove(mgr: obj->base.dev->vma_offset_manager,
328 node: &mmo->vma_node);
329 kfree(objp: mmo);
330 }
331 obj->mmo.offsets = RB_ROOT;
332 }
333}
334
335/**
336 * __i915_gem_object_pages_fini - Clean up pages use of a gem object
337 * @obj: The gem object to clean up
338 *
339 * This function cleans up usage of the object mm.pages member. It
340 * is intended for backends that need to clean up a gem object in
341 * separate steps and needs to be called when the object is idle before
342 * the object's backing memory is freed.
343 */
344void __i915_gem_object_pages_fini(struct drm_i915_gem_object *obj)
345{
346 assert_object_held_shared(obj);
347
348 if (!list_empty(head: &obj->vma.list)) {
349 struct i915_vma *vma;
350
351 spin_lock(lock: &obj->vma.lock);
352 while ((vma = list_first_entry_or_null(&obj->vma.list,
353 struct i915_vma,
354 obj_link))) {
355 GEM_BUG_ON(vma->obj != obj);
356 spin_unlock(lock: &obj->vma.lock);
357
358 i915_vma_destroy(vma);
359
360 spin_lock(lock: &obj->vma.lock);
361 }
362 spin_unlock(lock: &obj->vma.lock);
363 }
364
365 __i915_gem_object_free_mmaps(obj);
366
367 atomic_set(v: &obj->mm.pages_pin_count, i: 0);
368
369 /*
370 * dma_buf_unmap_attachment() requires reservation to be
371 * locked. The imported GEM shouldn't share reservation lock
372 * and ttm_bo_cleanup_memtype_use() shouldn't be invoked for
373 * dma-buf, so it's safe to take the lock.
374 */
375 if (obj->base.import_attach)
376 i915_gem_object_lock(obj, NULL);
377
378 __i915_gem_object_put_pages(obj);
379
380 if (obj->base.import_attach)
381 i915_gem_object_unlock(obj);
382
383 GEM_BUG_ON(i915_gem_object_has_pages(obj));
384}
385
386void __i915_gem_free_object(struct drm_i915_gem_object *obj)
387{
388 trace_i915_gem_object_destroy(obj);
389
390 GEM_BUG_ON(!list_empty(&obj->lut_list));
391
392 bitmap_free(bitmap: obj->bit_17);
393
394 if (obj->base.import_attach)
395 drm_prime_gem_destroy(obj: &obj->base, NULL);
396
397 drm_gem_free_mmap_offset(obj: &obj->base);
398
399 if (obj->ops->release)
400 obj->ops->release(obj);
401
402 if (obj->shares_resv_from)
403 i915_vm_resv_put(vm: obj->shares_resv_from);
404
405 __i915_gem_object_fini(obj);
406}
407
408static void __i915_gem_free_objects(struct drm_i915_private *i915,
409 struct llist_node *freed)
410{
411 struct drm_i915_gem_object *obj, *on;
412
413 llist_for_each_entry_safe(obj, on, freed, freed) {
414 might_sleep();
415 if (obj->ops->delayed_free) {
416 obj->ops->delayed_free(obj);
417 continue;
418 }
419
420 __i915_gem_object_pages_fini(obj);
421 __i915_gem_free_object(obj);
422
423 /* But keep the pointer alive for RCU-protected lookups */
424 call_rcu(head: &obj->rcu, func: __i915_gem_free_object_rcu);
425 cond_resched();
426 }
427}
428
429void i915_gem_flush_free_objects(struct drm_i915_private *i915)
430{
431 struct llist_node *freed = llist_del_all(head: &i915->mm.free_list);
432
433 if (unlikely(freed))
434 __i915_gem_free_objects(i915, freed);
435}
436
437static void __i915_gem_free_work(struct work_struct *work)
438{
439 struct drm_i915_private *i915 =
440 container_of(work, struct drm_i915_private, mm.free_work);
441
442 i915_gem_flush_free_objects(i915);
443}
444
445static void i915_gem_free_object(struct drm_gem_object *gem_obj)
446{
447 struct drm_i915_gem_object *obj = to_intel_bo(gem: gem_obj);
448 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
449
450 GEM_BUG_ON(i915_gem_object_is_framebuffer(obj));
451
452 i915_drm_client_remove_object(obj);
453
454 /*
455 * Before we free the object, make sure any pure RCU-only
456 * read-side critical sections are complete, e.g.
457 * i915_gem_busy_ioctl(). For the corresponding synchronized
458 * lookup see i915_gem_object_lookup_rcu().
459 */
460 atomic_inc(v: &i915->mm.free_count);
461
462 /*
463 * Since we require blocking on drm_i915_gem_object->vma.lock to unbind
464 * the freed object from the GPU before releasing resources back to the
465 * system, we can not do that directly from the RCU callback (which may
466 * be a softirq context), but must instead then defer that work onto a
467 * kthread. We use the RCU callback rather than move the freed object
468 * directly onto the work queue so that we can mix between using the
469 * worker and performing frees directly from subsequent allocations for
470 * crude but effective memory throttling.
471 */
472
473 if (llist_add(new: &obj->freed, head: &i915->mm.free_list))
474 queue_work(wq: i915->wq, work: &i915->mm.free_work);
475}
476
477void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,
478 enum fb_op_origin origin)
479{
480 struct i915_frontbuffer *front;
481
482 front = i915_gem_object_frontbuffer_lookup(obj);
483 if (front) {
484 intel_frontbuffer_flush(front: &front->base, origin);
485 i915_gem_object_frontbuffer_put(front);
486 }
487}
488
489void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,
490 enum fb_op_origin origin)
491{
492 struct i915_frontbuffer *front;
493
494 front = i915_gem_object_frontbuffer_lookup(obj);
495 if (front) {
496 intel_frontbuffer_invalidate(front: &front->base, origin);
497 i915_gem_object_frontbuffer_put(front);
498 }
499}
500
501static void
502i915_gem_object_read_from_page_kmap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size)
503{
504 pgoff_t idx = offset >> PAGE_SHIFT;
505 void *src_ptr;
506
507 src_ptr = kmap_local_page(i915_gem_object_get_page(obj, idx))
508 + offset_in_page(offset);
509 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
510 drm_clflush_virt_range(addr: src_ptr, length: size);
511 memcpy(dst, src_ptr, size);
512
513 kunmap_local(src_ptr);
514}
515
516static void
517i915_gem_object_read_from_page_iomap(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size)
518{
519 pgoff_t idx = offset >> PAGE_SHIFT;
520 dma_addr_t dma = i915_gem_object_get_dma_address(obj, idx);
521 void __iomem *src_map;
522 void __iomem *src_ptr;
523
524 src_map = io_mapping_map_wc(mapping: &obj->mm.region->iomap,
525 offset: dma - obj->mm.region->region.start,
526 PAGE_SIZE);
527
528 src_ptr = src_map + offset_in_page(offset);
529 if (!i915_memcpy_from_wc(dst, src: (void __force *)src_ptr, len: size))
530 memcpy_fromio(dst, src_ptr, size);
531
532 io_mapping_unmap(vaddr: src_map);
533}
534
535static bool object_has_mappable_iomem(struct drm_i915_gem_object *obj)
536{
537 GEM_BUG_ON(!i915_gem_object_has_iomem(obj));
538
539 if (IS_DGFX(to_i915(obj->base.dev)))
540 return i915_ttm_resource_mappable(res: i915_gem_to_ttm(obj)->resource);
541
542 return true;
543}
544
545/**
546 * i915_gem_object_read_from_page - read data from the page of a GEM object
547 * @obj: GEM object to read from
548 * @offset: offset within the object
549 * @dst: buffer to store the read data
550 * @size: size to read
551 *
552 * Reads data from @obj at the specified offset. The requested region to read
553 * from can't cross a page boundary. The caller must ensure that @obj pages
554 * are pinned and that @obj is synced wrt. any related writes.
555 *
556 * Return: %0 on success or -ENODEV if the type of @obj's backing store is
557 * unsupported.
558 */
559int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, void *dst, int size)
560{
561 GEM_BUG_ON(overflows_type(offset >> PAGE_SHIFT, pgoff_t));
562 GEM_BUG_ON(offset >= obj->base.size);
563 GEM_BUG_ON(offset_in_page(offset) > PAGE_SIZE - size);
564 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
565
566 if (i915_gem_object_has_struct_page(obj))
567 i915_gem_object_read_from_page_kmap(obj, offset, dst, size);
568 else if (i915_gem_object_has_iomem(obj) && object_has_mappable_iomem(obj))
569 i915_gem_object_read_from_page_iomap(obj, offset, dst, size);
570 else
571 return -ENODEV;
572
573 return 0;
574}
575
576/**
577 * i915_gem_object_evictable - Whether object is likely evictable after unbind.
578 * @obj: The object to check
579 *
580 * This function checks whether the object is likely unvictable after unbind.
581 * If the object is not locked when checking, the result is only advisory.
582 * If the object is locked when checking, and the function returns true,
583 * then an eviction should indeed be possible. But since unlocked vma
584 * unpinning and unbinding is currently possible, the object can actually
585 * become evictable even if this function returns false.
586 *
587 * Return: true if the object may be evictable. False otherwise.
588 */
589bool i915_gem_object_evictable(struct drm_i915_gem_object *obj)
590{
591 struct i915_vma *vma;
592 int pin_count = atomic_read(v: &obj->mm.pages_pin_count);
593
594 if (!pin_count)
595 return true;
596
597 spin_lock(lock: &obj->vma.lock);
598 list_for_each_entry(vma, &obj->vma.list, obj_link) {
599 if (i915_vma_is_pinned(vma)) {
600 spin_unlock(lock: &obj->vma.lock);
601 return false;
602 }
603 if (atomic_read(v: &vma->pages_count))
604 pin_count--;
605 }
606 spin_unlock(lock: &obj->vma.lock);
607 GEM_WARN_ON(pin_count < 0);
608
609 return pin_count == 0;
610}
611
612/**
613 * i915_gem_object_migratable - Whether the object is migratable out of the
614 * current region.
615 * @obj: Pointer to the object.
616 *
617 * Return: Whether the object is allowed to be resident in other
618 * regions than the current while pages are present.
619 */
620bool i915_gem_object_migratable(struct drm_i915_gem_object *obj)
621{
622 struct intel_memory_region *mr = READ_ONCE(obj->mm.region);
623
624 if (!mr)
625 return false;
626
627 return obj->mm.n_placements > 1;
628}
629
630/**
631 * i915_gem_object_has_struct_page - Whether the object is page-backed
632 * @obj: The object to query.
633 *
634 * This function should only be called while the object is locked or pinned,
635 * otherwise the page backing may change under the caller.
636 *
637 * Return: True if page-backed, false otherwise.
638 */
639bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
640{
641#ifdef CONFIG_LOCKDEP
642 if (IS_DGFX(to_i915(obj->base.dev)) &&
643 i915_gem_object_evictable(obj: (void __force *)obj))
644 assert_object_held_shared(obj);
645#endif
646 return obj->mem_flags & I915_BO_FLAG_STRUCT_PAGE;
647}
648
649/**
650 * i915_gem_object_has_iomem - Whether the object is iomem-backed
651 * @obj: The object to query.
652 *
653 * This function should only be called while the object is locked or pinned,
654 * otherwise the iomem backing may change under the caller.
655 *
656 * Return: True if iomem-backed, false otherwise.
657 */
658bool i915_gem_object_has_iomem(const struct drm_i915_gem_object *obj)
659{
660#ifdef CONFIG_LOCKDEP
661 if (IS_DGFX(to_i915(obj->base.dev)) &&
662 i915_gem_object_evictable(obj: (void __force *)obj))
663 assert_object_held_shared(obj);
664#endif
665 return obj->mem_flags & I915_BO_FLAG_IOMEM;
666}
667
668/**
669 * i915_gem_object_can_migrate - Whether an object likely can be migrated
670 *
671 * @obj: The object to migrate
672 * @id: The region intended to migrate to
673 *
674 * Check whether the object backend supports migration to the
675 * given region. Note that pinning may affect the ability to migrate as
676 * returned by this function.
677 *
678 * This function is primarily intended as a helper for checking the
679 * possibility to migrate objects and might be slightly less permissive
680 * than i915_gem_object_migrate() when it comes to objects with the
681 * I915_BO_ALLOC_USER flag set.
682 *
683 * Return: true if migration is possible, false otherwise.
684 */
685bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj,
686 enum intel_region_id id)
687{
688 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
689 unsigned int num_allowed = obj->mm.n_placements;
690 struct intel_memory_region *mr;
691 unsigned int i;
692
693 GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN);
694 GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED);
695
696 mr = i915->mm.regions[id];
697 if (!mr)
698 return false;
699
700 if (!IS_ALIGNED(obj->base.size, mr->min_page_size))
701 return false;
702
703 if (obj->mm.region == mr)
704 return true;
705
706 if (!i915_gem_object_evictable(obj))
707 return false;
708
709 if (!obj->ops->migrate)
710 return false;
711
712 if (!(obj->flags & I915_BO_ALLOC_USER))
713 return true;
714
715 if (num_allowed == 0)
716 return false;
717
718 for (i = 0; i < num_allowed; ++i) {
719 if (mr == obj->mm.placements[i])
720 return true;
721 }
722
723 return false;
724}
725
726/**
727 * i915_gem_object_migrate - Migrate an object to the desired region id
728 * @obj: The object to migrate.
729 * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may
730 * not be successful in evicting other objects to make room for this object.
731 * @id: The region id to migrate to.
732 *
733 * Attempt to migrate the object to the desired memory region. The
734 * object backend must support migration and the object may not be
735 * pinned, (explicitly pinned pages or pinned vmas). The object must
736 * be locked.
737 * On successful completion, the object will have pages pointing to
738 * memory in the new region, but an async migration task may not have
739 * completed yet, and to accomplish that, i915_gem_object_wait_migration()
740 * must be called.
741 *
742 * Note: the @ww parameter is not used yet, but included to make sure
743 * callers put some effort into obtaining a valid ww ctx if one is
744 * available.
745 *
746 * Return: 0 on success. Negative error code on failure. In particular may
747 * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance
748 * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and
749 * -EBUSY if the object is pinned.
750 */
751int i915_gem_object_migrate(struct drm_i915_gem_object *obj,
752 struct i915_gem_ww_ctx *ww,
753 enum intel_region_id id)
754{
755 return __i915_gem_object_migrate(obj, ww, id, flags: obj->flags);
756}
757
758/**
759 * __i915_gem_object_migrate - Migrate an object to the desired region id, with
760 * control of the extra flags
761 * @obj: The object to migrate.
762 * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may
763 * not be successful in evicting other objects to make room for this object.
764 * @id: The region id to migrate to.
765 * @flags: The object flags. Normally just obj->flags.
766 *
767 * Attempt to migrate the object to the desired memory region. The
768 * object backend must support migration and the object may not be
769 * pinned, (explicitly pinned pages or pinned vmas). The object must
770 * be locked.
771 * On successful completion, the object will have pages pointing to
772 * memory in the new region, but an async migration task may not have
773 * completed yet, and to accomplish that, i915_gem_object_wait_migration()
774 * must be called.
775 *
776 * Note: the @ww parameter is not used yet, but included to make sure
777 * callers put some effort into obtaining a valid ww ctx if one is
778 * available.
779 *
780 * Return: 0 on success. Negative error code on failure. In particular may
781 * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance
782 * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and
783 * -EBUSY if the object is pinned.
784 */
785int __i915_gem_object_migrate(struct drm_i915_gem_object *obj,
786 struct i915_gem_ww_ctx *ww,
787 enum intel_region_id id,
788 unsigned int flags)
789{
790 struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
791 struct intel_memory_region *mr;
792
793 GEM_BUG_ON(id >= INTEL_REGION_UNKNOWN);
794 GEM_BUG_ON(obj->mm.madv != I915_MADV_WILLNEED);
795 assert_object_held(obj);
796
797 mr = i915->mm.regions[id];
798 GEM_BUG_ON(!mr);
799
800 if (!i915_gem_object_can_migrate(obj, id))
801 return -EINVAL;
802
803 if (!obj->ops->migrate) {
804 if (GEM_WARN_ON(obj->mm.region != mr))
805 return -EINVAL;
806 return 0;
807 }
808
809 return obj->ops->migrate(obj, mr, flags);
810}
811
812/**
813 * i915_gem_object_placement_possible - Check whether the object can be
814 * placed at certain memory type
815 * @obj: Pointer to the object
816 * @type: The memory type to check
817 *
818 * Return: True if the object can be placed in @type. False otherwise.
819 */
820bool i915_gem_object_placement_possible(struct drm_i915_gem_object *obj,
821 enum intel_memory_type type)
822{
823 unsigned int i;
824
825 if (!obj->mm.n_placements) {
826 switch (type) {
827 case INTEL_MEMORY_LOCAL:
828 return i915_gem_object_has_iomem(obj);
829 case INTEL_MEMORY_SYSTEM:
830 return i915_gem_object_has_pages(obj);
831 default:
832 /* Ignore stolen for now */
833 GEM_BUG_ON(1);
834 return false;
835 }
836 }
837
838 for (i = 0; i < obj->mm.n_placements; i++) {
839 if (obj->mm.placements[i]->type == type)
840 return true;
841 }
842
843 return false;
844}
845
846/**
847 * i915_gem_object_needs_ccs_pages - Check whether the object requires extra
848 * pages when placed in system-memory, in order to save and later restore the
849 * flat-CCS aux state when the object is moved between local-memory and
850 * system-memory
851 * @obj: Pointer to the object
852 *
853 * Return: True if the object needs extra ccs pages. False otherwise.
854 */
855bool i915_gem_object_needs_ccs_pages(struct drm_i915_gem_object *obj)
856{
857 bool lmem_placement = false;
858 int i;
859
860 if (!HAS_FLAT_CCS(to_i915(obj->base.dev)))
861 return false;
862
863 if (obj->flags & I915_BO_ALLOC_CCS_AUX)
864 return true;
865
866 for (i = 0; i < obj->mm.n_placements; i++) {
867 /* Compression is not allowed for the objects with smem placement */
868 if (obj->mm.placements[i]->type == INTEL_MEMORY_SYSTEM)
869 return false;
870 if (!lmem_placement &&
871 obj->mm.placements[i]->type == INTEL_MEMORY_LOCAL)
872 lmem_placement = true;
873 }
874
875 return lmem_placement;
876}
877
878static int i915_gem_vmap_object(struct drm_gem_object *gem_obj,
879 struct iosys_map *map)
880{
881 struct drm_i915_gem_object *obj = to_intel_bo(gem: gem_obj);
882 void *vaddr;
883
884 vaddr = i915_gem_object_pin_map(obj, type: I915_MAP_WB);
885 if (IS_ERR(ptr: vaddr))
886 return PTR_ERR(ptr: vaddr);
887
888 iosys_map_set_vaddr(map, vaddr);
889
890 return 0;
891}
892
893static void i915_gem_vunmap_object(struct drm_gem_object *gem_obj,
894 struct iosys_map *map)
895{
896 struct drm_i915_gem_object *obj = to_intel_bo(gem: gem_obj);
897
898 i915_gem_object_flush_map(obj);
899 i915_gem_object_unpin_map(obj);
900}
901
902void i915_gem_init__objects(struct drm_i915_private *i915)
903{
904 INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
905}
906
907void i915_objects_module_exit(void)
908{
909 kmem_cache_destroy(s: slab_objects);
910}
911
912int __init i915_objects_module_init(void)
913{
914 slab_objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
915 if (!slab_objects)
916 return -ENOMEM;
917
918 return 0;
919}
920
921static const struct drm_gem_object_funcs i915_gem_object_funcs = {
922 .free = i915_gem_free_object,
923 .close = i915_gem_close_object,
924 .export = i915_gem_prime_export,
925 .vmap = i915_gem_vmap_object,
926 .vunmap = i915_gem_vunmap_object,
927};
928
929/**
930 * i915_gem_object_get_moving_fence - Get the object's moving fence if any
931 * @obj: The object whose moving fence to get.
932 * @fence: The resulting fence
933 *
934 * A non-signaled moving fence means that there is an async operation
935 * pending on the object that needs to be waited on before setting up
936 * any GPU- or CPU PTEs to the object's pages.
937 *
938 * Return: Negative error code or 0 for success.
939 */
940int i915_gem_object_get_moving_fence(struct drm_i915_gem_object *obj,
941 struct dma_fence **fence)
942{
943 return dma_resv_get_singleton(obj: obj->base.resv, usage: DMA_RESV_USAGE_KERNEL,
944 fence);
945}
946
947/**
948 * i915_gem_object_wait_moving_fence - Wait for the object's moving fence if any
949 * @obj: The object whose moving fence to wait for.
950 * @intr: Whether to wait interruptible.
951 *
952 * If the moving fence signaled without an error, it is detached from the
953 * object and put.
954 *
955 * Return: 0 if successful, -ERESTARTSYS if the wait was interrupted,
956 * negative error code if the async operation represented by the
957 * moving fence failed.
958 */
959int i915_gem_object_wait_moving_fence(struct drm_i915_gem_object *obj,
960 bool intr)
961{
962 long ret;
963
964 assert_object_held(obj);
965
966 ret = dma_resv_wait_timeout(obj: obj->base. resv, usage: DMA_RESV_USAGE_KERNEL,
967 intr, MAX_SCHEDULE_TIMEOUT);
968 if (!ret)
969 ret = -ETIME;
970 else if (ret > 0 && i915_gem_object_has_unknown_state(obj))
971 ret = -EIO;
972
973 return ret < 0 ? ret : 0;
974}
975
976/*
977 * i915_gem_object_has_unknown_state - Return true if the object backing pages are
978 * in an unknown_state. This means that userspace must NEVER be allowed to touch
979 * the pages, with either the GPU or CPU.
980 *
981 * ONLY valid to be called after ensuring that all kernel fences have signalled
982 * (in particular the fence for moving/clearing the object).
983 */
984bool i915_gem_object_has_unknown_state(struct drm_i915_gem_object *obj)
985{
986 /*
987 * The below barrier pairs with the dma_fence_signal() in
988 * __memcpy_work(). We should only sample the unknown_state after all
989 * the kernel fences have signalled.
990 */
991 smp_rmb();
992 return obj->mm.unknown_state;
993}
994
995#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
996#include "selftests/huge_gem_object.c"
997#include "selftests/huge_pages.c"
998#include "selftests/i915_gem_migrate.c"
999#include "selftests/i915_gem_object.c"
1000#include "selftests/i915_gem_coherency.c"
1001#endif
1002

source code of linux/drivers/gpu/drm/i915/gem/i915_gem_object.c