1	/*
2	* Copyright © 2016 Intel Corporation
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice (including the next
12	* paragraph) shall be included in all copies or substantial portions of the
13	* Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21	* IN THE SOFTWARE.
22	*
23	*/
24
25	#include <linux/sched/mm.h>
26	#include <linux/dma-fence-array.h>
27
28	#include <drm/drm_gem.h>
29	#include <drm/drm_print.h>
30
31	#include "display/intel_fb.h"
32	#include "display/intel_frontbuffer.h"
33	#include "gem/i915_gem_lmem.h"
34	#include "gem/i915_gem_object_frontbuffer.h"
35	#include "gem/i915_gem_tiling.h"
36	#include "gt/intel_engine.h"
37	#include "gt/intel_engine_heartbeat.h"
38	#include "gt/intel_gt.h"
39	#include "gt/intel_gt_pm.h"
40	#include "gt/intel_gt_requests.h"
41	#include "gt/intel_tlb.h"
42
43	#include "i915_drv.h"
44	#include "i915_gem_evict.h"
45	#include "i915_sw_fence_work.h"
46	#include "i915_trace.h"
47	#include "i915_vma.h"
48	#include "i915_vma_resource.h"
49
50	static inline void assert_vma_held_evict(const struct i915_vma *vma)
51	{
52	/*
53	* We may be forced to unbind when the vm is dead, to clean it up.
54	* This is the only exception to the requirement of the object lock
55	* being held.
56	*/
57	if (kref_read(kref: &vma->vm->ref))
58	assert_object_held_shared(obj: vma->obj);
59	}
60
61	static struct kmem_cache *slab_vmas;
62
63	static struct i915_vma *i915_vma_alloc(void)
64	{
65	return kmem_cache_zalloc(slab_vmas, GFP_KERNEL);
66	}
67
68	static void i915_vma_free(struct i915_vma *vma)
69	{
70	return kmem_cache_free(s: slab_vmas, objp: vma);
71	}
72
73	#if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
74
75	#include <linux/stackdepot.h>
76
77	static void vma_print_allocator(struct i915_vma *vma, const char *reason)
78	{
79	char buf[512];
80
81	if (!vma->node.stack) {
82	drm_dbg(vma->obj->base.dev,
83	"vma.node [%08llx + %08llx] %s: unknown owner\n",
84	vma->node.start, vma->node.size, reason);
85	return;
86	}
87
88	stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0);
89	drm_dbg(vma->obj->base.dev,
90	"vma.node [%08llx + %08llx] %s: inserted at %s\n",
91	vma->node.start, vma->node.size, reason, buf);
92	}
93
94	#else
95
96	static void vma_print_allocator(struct i915_vma *vma, const char *reason)
97	{
98	}
99
100	#endif
101
102	static inline struct i915_vma *active_to_vma(struct i915_active *ref)
103	{
104	return container_of(ref, typeof(struct i915_vma), active);
105	}
106
107	static int __i915_vma_active(struct i915_active *ref)
108	{
109	struct i915_vma *vma = active_to_vma(ref);
110
111	if (!i915_vma_tryget(vma))
112	return -ENOENT;
113
114	/*
115	* Exclude global GTT VMA from holding a GT wakeref
116	* while active, otherwise GPU never goes idle.
117	*/
118	if (!i915_vma_is_ggtt(vma)) {
119	/*
120	* Since we and our _retire() counterpart can be
121	* called asynchronously, storing a wakeref tracking
122	* handle inside struct i915_vma is not safe, and
123	* there is no other good place for that. Hence,
124	* use untracked variants of intel_gt_pm_get/put().
125	*/
126	intel_gt_pm_get_untracked(gt: vma->vm->gt);
127	}
128
129	return 0;
130	}
131
132	static void __i915_vma_retire(struct i915_active *ref)
133	{
134	struct i915_vma *vma = active_to_vma(ref);
135
136	if (!i915_vma_is_ggtt(vma)) {
137	/*
138	* Since we can be called from atomic contexts,
139	* use an async variant of intel_gt_pm_put().
140	*/
141	intel_gt_pm_put_async_untracked(gt: vma->vm->gt);
142	}
143
144	i915_vma_put(vma);
145	}
146
147	static struct i915_vma *
148	vma_create(struct drm_i915_gem_object *obj,
149	struct i915_address_space *vm,
150	const struct i915_gtt_view *view)
151	{
152	struct i915_vma *pos = ERR_PTR(error: -E2BIG);
153	struct i915_vma *vma;
154	struct rb_node *rb, **p;
155	int err;
156
157	/* The aliasing_ppgtt should never be used directly! */
158	GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm);
159
160	vma = i915_vma_alloc();
161	if (vma == NULL)
162	return ERR_PTR(error: -ENOMEM);
163
164	vma->ops = &vm->vma_ops;
165	vma->obj = obj;
166	vma->size = obj->base.size;
167	vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
168
169	i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0);
170
171	/* Declare ourselves safe for use inside shrinkers */
172	if (IS_ENABLED(CONFIG_LOCKDEP)) {
173	fs_reclaim_acquire(GFP_KERNEL);
174	might_lock(&vma->active.mutex);
175	fs_reclaim_release(GFP_KERNEL);
176	}
177
178	INIT_LIST_HEAD(list: &vma->closed_link);
179	INIT_LIST_HEAD(list: &vma->obj_link);
180	RB_CLEAR_NODE(&vma->obj_node);
181
182	if (view && view->type != I915_GTT_VIEW_NORMAL) {
183	vma->gtt_view = *view;
184	if (view->type == I915_GTT_VIEW_PARTIAL) {
185	GEM_BUG_ON(range_overflows_t(u64,
186	view->partial.offset,
187	view->partial.size,
188	obj->base.size >> PAGE_SHIFT));
189	vma->size = view->partial.size;
190	vma->size <<= PAGE_SHIFT;
191	GEM_BUG_ON(vma->size > obj->base.size);
192	} else if (view->type == I915_GTT_VIEW_ROTATED) {
193	vma->size = intel_rotation_info_size(rot_info: &view->rotated);
194	vma->size <<= PAGE_SHIFT;
195	} else if (view->type == I915_GTT_VIEW_REMAPPED) {
196	vma->size = intel_remapped_info_size(rem_info: &view->remapped);
197	vma->size <<= PAGE_SHIFT;
198	}
199	}
200
201	if (unlikely(vma->size > vm->total))
202	goto err_vma;
203
204	GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE));
205
206	err = mutex_lock_interruptible(&vm->mutex);
207	if (err) {
208	pos = ERR_PTR(error: err);
209	goto err_vma;
210	}
211
212	vma->vm = vm;
213	list_add_tail(new: &vma->vm_link, head: &vm->unbound_list);
214
215	spin_lock(lock: &obj->vma.lock);
216	if (i915_is_ggtt(vm)) {
217	if (unlikely(overflows_type(vma->size, u32)))
218	goto err_unlock;
219
220	vma->fence_size = i915_gem_fence_size(i915: vm->i915, size: vma->size,
221	tiling: i915_gem_object_get_tiling(obj),
222	stride: i915_gem_object_get_stride(obj));
223	if (unlikely(vma->fence_size < vma->size || /* overflow */
224	vma->fence_size > vm->total))
225	goto err_unlock;
226
227	GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT));
228
229	vma->fence_alignment = i915_gem_fence_alignment(i915: vm->i915, size: vma->size,
230	tiling: i915_gem_object_get_tiling(obj),
231	stride: i915_gem_object_get_stride(obj));
232	GEM_BUG_ON(!is_power_of_2(vma->fence_alignment));
233
234	__set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));
235	}
236
237	rb = NULL;
238	p = &obj->vma.tree.rb_node;
239	while (*p) {
240	long cmp;
241
242	rb = *p;
243	pos = rb_entry(rb, struct i915_vma, obj_node);
244
245	/*
246	* If the view already exists in the tree, another thread
247	* already created a matching vma, so return the older instance
248	* and dispose of ours.
249	*/
250	cmp = i915_vma_compare(vma: pos, vm, view);
251	if (cmp < 0)
252	p = &rb->rb_right;
253	else if (cmp > 0)
254	p = &rb->rb_left;
255	else
256	goto err_unlock;
257	}
258	rb_link_node(node: &vma->obj_node, parent: rb, rb_link: p);
259	rb_insert_color(&vma->obj_node, &obj->vma.tree);
260
261	if (i915_vma_is_ggtt(vma))
262	/*
263	* We put the GGTT vma at the start of the vma-list, followed
264	* by the ppGGTT vma. This allows us to break early when
265	* iterating over only the GGTT vma for an object, see
266	* for_each_ggtt_vma()
267	*/
268	list_add(new: &vma->obj_link, head: &obj->vma.list);
269	else
270	list_add_tail(new: &vma->obj_link, head: &obj->vma.list);
271
272	spin_unlock(lock: &obj->vma.lock);
273	mutex_unlock(lock: &vm->mutex);
274
275	return vma;
276
277	err_unlock:
278	spin_unlock(lock: &obj->vma.lock);
279	list_del_init(entry: &vma->vm_link);
280	mutex_unlock(lock: &vm->mutex);
281	err_vma:
282	i915_vma_free(vma);
283	return pos;
284	}
285
286	static struct i915_vma *
287	i915_vma_lookup(struct drm_i915_gem_object *obj,
288	struct i915_address_space *vm,
289	const struct i915_gtt_view *view)
290	{
291	struct rb_node *rb;
292
293	rb = obj->vma.tree.rb_node;
294	while (rb) {
295	struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
296	long cmp;
297
298	cmp = i915_vma_compare(vma, vm, view);
299	if (cmp == 0)
300	return vma;
301
302	if (cmp < 0)
303	rb = rb->rb_right;
304	else
305	rb = rb->rb_left;
306	}
307
308	return NULL;
309	}
310
311	/**
312	* i915_vma_instance - return the singleton instance of the VMA
313	* @obj: parent &struct drm_i915_gem_object to be mapped
314	* @vm: address space in which the mapping is located
315	* @view: additional mapping requirements
316	*
317	* i915_vma_instance() looks up an existing VMA of the @obj in the @vm with
318	* the same @view characteristics. If a match is not found, one is created.
319	* Once created, the VMA is kept until either the object is freed, or the
320	* address space is closed.
321	*
322	* Returns the vma, or an error pointer.
323	*/
324	struct i915_vma *
325	i915_vma_instance(struct drm_i915_gem_object *obj,
326	struct i915_address_space *vm,
327	const struct i915_gtt_view *view)
328	{
329	struct i915_vma *vma;
330
331	GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm));
332	GEM_BUG_ON(!kref_read(&vm->ref));
333
334	spin_lock(lock: &obj->vma.lock);
335	vma = i915_vma_lookup(obj, vm, view);
336	spin_unlock(lock: &obj->vma.lock);
337
338	/* vma_create() will resolve the race if another creates the vma */
339	if (unlikely(!vma))
340	vma = vma_create(obj, vm, view);
341
342	GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view));
343	return vma;
344	}
345
346	struct i915_vma_work {
347	struct dma_fence_work base;
348	struct i915_address_space *vm;
349	struct i915_vm_pt_stash stash;
350	struct i915_vma_resource *vma_res;
351	struct drm_i915_gem_object *obj;
352	struct i915_sw_dma_fence_cb cb;
353	unsigned int pat_index;
354	unsigned int flags;
355	};
356
357	static void __vma_bind(struct dma_fence_work *work)
358	{
359	struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
360	struct i915_vma_resource *vma_res = vw->vma_res;
361
362	/*
363	* We are about the bind the object, which must mean we have already
364	* signaled the work to potentially clear/move the pages underneath. If
365	* something went wrong at that stage then the object should have
366	* unknown_state set, in which case we need to skip the bind.
367	*/
368	if (i915_gem_object_has_unknown_state(obj: vw->obj))
369	return;
370
371	vma_res->ops->bind_vma(vma_res->vm, &vw->stash,
372	vma_res, vw->pat_index, vw->flags);
373	}
374
375	static void __vma_release(struct dma_fence_work *work)
376	{
377	struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
378
379	if (vw->obj)
380	i915_gem_object_put(obj: vw->obj);
381
382	i915_vm_free_pt_stash(vm: vw->vm, stash: &vw->stash);
383	if (vw->vma_res)
384	i915_vma_resource_put(vma_res: vw->vma_res);
385	}
386
387	static const struct dma_fence_work_ops bind_ops = {
388	.name = "bind",
389	.work = __vma_bind,
390	.release = __vma_release,
391	};
392
393	struct i915_vma_work *i915_vma_work(void)
394	{
395	struct i915_vma_work *vw;
396
397	vw = kzalloc(sizeof(*vw), GFP_KERNEL);
398	if (!vw)
399	return NULL;
400
401	dma_fence_work_init(f: &vw->base, ops: &bind_ops);
402	vw->base.dma.error = -EAGAIN; /* disable the worker by default */
403
404	return vw;
405	}
406
407	int i915_vma_wait_for_bind(struct i915_vma *vma)
408	{
409	int err = 0;
410
411	if (rcu_access_pointer(vma->active.excl.fence)) {
412	struct dma_fence *fence;
413
414	rcu_read_lock();
415	fence = dma_fence_get_rcu_safe(fencep: &vma->active.excl.fence);
416	rcu_read_unlock();
417	if (fence) {
418	err = dma_fence_wait(fence, intr: true);
419	dma_fence_put(fence);
420	}
421	}
422
423	return err;
424	}
425
426	#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
427	static int i915_vma_verify_bind_complete(struct i915_vma *vma)
428	{
429	struct dma_fence *fence = i915_active_fence_get(&vma->active.excl);
430	int err;
431
432	if (!fence)
433	return 0;
434
435	if (dma_fence_is_signaled(fence))
436	err = fence->error;
437	else
438	err = -EBUSY;
439
440	dma_fence_put(fence);
441
442	return err;
443	}
444	#else
445	#define i915_vma_verify_bind_complete(_vma) 0
446	#endif
447
448	I915_SELFTEST_EXPORT void
449	i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res,
450	struct i915_vma *vma)
451	{
452	struct drm_i915_gem_object *obj = vma->obj;
453
454	i915_vma_resource_init(vma_res, vm: vma->vm, pages: vma->pages, page_sizes: &vma->page_sizes,
455	pages_rsgt: obj->mm.rsgt, readonly: i915_gem_object_is_readonly(obj),
456	lmem: i915_gem_object_is_lmem(obj), mr: obj->mm.region,
457	ops: vma->ops, private: vma->private, start: __i915_vma_offset(vma),
458	node_size: __i915_vma_size(vma), size: vma->size, guard: vma->guard);
459	}
460
461	/**
462	* i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
463	* @vma: VMA to map
464	* @pat_index: PAT index to set in PTE
465	* @flags: flags like global or local mapping
466	* @work: preallocated worker for allocating and binding the PTE
467	* @vma_res: pointer to a preallocated vma resource. The resource is either
468	* consumed or freed.
469	*
470	* DMA addresses are taken from the scatter-gather table of this object (or of
471	* this VMA in case of non-default GGTT views) and PTE entries set up.
472	* Note that DMA addresses are also the only part of the SG table we care about.
473	*/
474	int i915_vma_bind(struct i915_vma *vma,
475	unsigned int pat_index,
476	u32 flags,
477	struct i915_vma_work *work,
478	struct i915_vma_resource *vma_res)
479	{
480	u32 bind_flags;
481	u32 vma_flags;
482	int ret;
483
484	lockdep_assert_held(&vma->vm->mutex);
485	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
486	GEM_BUG_ON(vma->size > i915_vma_size(vma));
487
488	if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start,
489	vma->node.size,
490	vma->vm->total))) {
491	i915_vma_resource_free(vma_res);
492	return -ENODEV;
493	}
494
495	if (GEM_DEBUG_WARN_ON(!flags)) {
496	i915_vma_resource_free(vma_res);
497	return -EINVAL;
498	}
499
500	bind_flags = flags;
501	bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
502
503	vma_flags = atomic_read(v: &vma->flags);
504	vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
505
506	bind_flags &= ~vma_flags;
507	if (bind_flags == 0) {
508	i915_vma_resource_free(vma_res);
509	return 0;
510	}
511
512	GEM_BUG_ON(!atomic_read(&vma->pages_count));
513
514	/* Wait for or await async unbinds touching our range */
515	if (work && bind_flags & vma->vm->bind_async_flags)
516	ret = i915_vma_resource_bind_dep_await(vm: vma->vm,
517	sw_fence: &work->base.chain,
518	first: vma->node.start,
519	last: vma->node.size,
520	intr: true,
521	GFP_NOWAIT |
522	__GFP_RETRY_MAYFAIL |
523	__GFP_NOWARN);
524	else
525	ret = i915_vma_resource_bind_dep_sync(vm: vma->vm, first: vma->node.start,
526	last: vma->node.size, intr: true);
527	if (ret) {
528	i915_vma_resource_free(vma_res);
529	return ret;
530	}
531
532	if (vma->resource || !vma_res) {
533	/* Rebinding with an additional I915_VMA_*_BIND */
534	GEM_WARN_ON(!vma_flags);
535	i915_vma_resource_free(vma_res);
536	} else {
537	i915_vma_resource_init_from_vma(vma_res, vma);
538	vma->resource = vma_res;
539	}
540	trace_i915_vma_bind(vma, flags: bind_flags);
541	if (work && bind_flags & vma->vm->bind_async_flags) {
542	struct dma_fence *prev;
543
544	work->vma_res = i915_vma_resource_get(vma_res: vma->resource);
545	work->pat_index = pat_index;
546	work->flags = bind_flags;
547
548	/*
549	* Note we only want to chain up to the migration fence on
550	* the pages (not the object itself). As we don't track that,
551	* yet, we have to use the exclusive fence instead.
552	*
553	* Also note that we do not want to track the async vma as
554	* part of the obj->resv->excl_fence as it only affects
555	* execution and not content or object's backing store lifetime.
556	*/
557	prev = i915_active_set_exclusive(ref: &vma->active, f: &work->base.dma);
558	if (prev) {
559	__i915_sw_fence_await_dma_fence(fence: &work->base.chain,
560	dma: prev,
561	cb: &work->cb);
562	dma_fence_put(fence: prev);
563	}
564
565	work->base.dma.error = 0; /* enable the queue_work() */
566	work->obj = i915_gem_object_get(obj: vma->obj);
567	} else {
568	ret = i915_gem_object_wait_moving_fence(obj: vma->obj, intr: true);
569	if (ret) {
570	i915_vma_resource_free(vma_res: vma->resource);
571	vma->resource = NULL;
572
573	return ret;
574	}
575	vma->ops->bind_vma(vma->vm, NULL, vma->resource, pat_index,
576	bind_flags);
577	}
578
579	atomic_or(i: bind_flags, v: &vma->flags);
580	return 0;
581	}
582
583	void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
584	{
585	void __iomem *ptr;
586	int err;
587
588	if (WARN_ON_ONCE(vma->obj->flags & I915_BO_ALLOC_GPU_ONLY))
589	return IOMEM_ERR_PTR(-EINVAL);
590
591	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
592	GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND));
593	GEM_BUG_ON(i915_vma_verify_bind_complete(vma));
594
595	ptr = READ_ONCE(vma->iomap);
596	if (ptr == NULL) {
597	/*
598	* TODO: consider just using i915_gem_object_pin_map() for lmem
599	* instead, which already supports mapping non-contiguous chunks
600	* of pages, that way we can also drop the
601	* I915_BO_ALLOC_CONTIGUOUS when allocating the object.
602	*/
603	if (i915_gem_object_is_lmem(obj: vma->obj)) {
604	ptr = i915_gem_object_lmem_io_map(obj: vma->obj, n: 0,
605	size: vma->obj->base.size);
606	} else if (i915_vma_is_map_and_fenceable(vma)) {
607	ptr = io_mapping_map_wc(mapping: &i915_vm_to_ggtt(vm: vma->vm)->iomap,
608	offset: i915_vma_offset(vma),
609	size: i915_vma_size(vma));
610	} else {
611	ptr = (void __iomem *)
612	i915_gem_object_pin_map(obj: vma->obj, type: I915_MAP_WC);
613	if (IS_ERR(ptr)) {
614	err = PTR_ERR(ptr);
615	goto err;
616	}
617	ptr = page_pack_bits(ptr, 1);
618	}
619
620	if (ptr == NULL) {
621	err = -ENOMEM;
622	goto err;
623	}
624
625	if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))) {
626	if (page_unmask_bits(ptr))
627	__i915_gem_object_release_map(obj: vma->obj);
628	else
629	io_mapping_unmap(vaddr: ptr);
630	ptr = vma->iomap;
631	}
632	}
633
634	__i915_vma_pin(vma);
635
636	err = i915_vma_pin_fence(vma);
637	if (err)
638	goto err_unpin;
639
640	i915_vma_set_ggtt_write(vma);
641
642	/* NB Access through the GTT requires the device to be awake. */
643	return page_mask_bits(ptr);
644
645	err_unpin:
646	__i915_vma_unpin(vma);
647	err:
648	return IOMEM_ERR_PTR(err);
649	}
650
651	void i915_vma_flush_writes(struct i915_vma *vma)
652	{
653	if (i915_vma_unset_ggtt_write(vma))
654	intel_gt_flush_ggtt_writes(gt: vma->vm->gt);
655	}
656
657	void i915_vma_unpin_iomap(struct i915_vma *vma)
658	{
659	GEM_BUG_ON(vma->iomap == NULL);
660
661	/* XXX We keep the mapping until __i915_vma_unbind()/evict() */
662
663	i915_vma_flush_writes(vma);
664
665	i915_vma_unpin_fence(vma);
666	i915_vma_unpin(vma);
667	}
668
669	void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags)
670	{
671	struct i915_vma *vma;
672	struct drm_i915_gem_object *obj;
673
674	vma = fetch_and_zero(p_vma);
675	if (!vma)
676	return;
677
678	obj = vma->obj;
679	GEM_BUG_ON(!obj);
680
681	i915_vma_unpin(vma);
682
683	if (flags & I915_VMA_RELEASE_MAP)
684	i915_gem_object_unpin_map(obj);
685
686	i915_gem_object_put(obj);
687	}
688
689	bool i915_vma_misplaced(const struct i915_vma *vma,
690	u64 size, u64 alignment, u64 flags)
691	{
692	if (!drm_mm_node_allocated(node: &vma->node))
693	return false;
694
695	if (test_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma)))
696	return true;
697
698	if (i915_vma_size(vma) < size)
699	return true;
700
701	GEM_BUG_ON(alignment && !is_power_of_2(alignment));
702	if (alignment && !IS_ALIGNED(i915_vma_offset(vma), alignment))
703	return true;
704
705	if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
706	return true;
707
708	if (flags & PIN_OFFSET_BIAS &&
709	i915_vma_offset(vma) < (flags & PIN_OFFSET_MASK))
710	return true;
711
712	if (flags & PIN_OFFSET_FIXED &&
713	i915_vma_offset(vma) != (flags & PIN_OFFSET_MASK))
714	return true;
715
716	if (flags & PIN_OFFSET_GUARD &&
717	vma->guard < (flags & PIN_OFFSET_MASK))
718	return true;
719
720	return false;
721	}
722
723	void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
724	{
725	bool mappable, fenceable;
726
727	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
728	GEM_BUG_ON(!vma->fence_size);
729
730	fenceable = (i915_vma_size(vma) >= vma->fence_size &&
731	IS_ALIGNED(i915_vma_offset(vma), vma->fence_alignment));
732
733	mappable = i915_ggtt_offset(vma) + vma->fence_size <=
734	i915_vm_to_ggtt(vm: vma->vm)->mappable_end;
735
736	if (mappable && fenceable)
737	set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
738	else
739	clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
740	}
741
742	bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color)
743	{
744	struct drm_mm_node *node = &vma->node;
745	struct drm_mm_node *other;
746
747	/*
748	* On some machines we have to be careful when putting differing types
749	* of snoopable memory together to avoid the prefetcher crossing memory
750	* domains and dying. During vm initialisation, we decide whether or not
751	* these constraints apply and set the drm_mm.color_adjust
752	* appropriately.
753	*/
754	if (!i915_vm_has_cache_coloring(vm: vma->vm))
755	return true;
756
757	/* Only valid to be called on an already inserted vma */
758	GEM_BUG_ON(!drm_mm_node_allocated(node));
759	GEM_BUG_ON(list_empty(&node->node_list));
760
761	other = list_prev_entry(node, node_list);
762	if (i915_node_color_differs(node: other, color) &&
763	!drm_mm_hole_follows(node: other))
764	return false;
765
766	other = list_next_entry(node, node_list);
767	if (i915_node_color_differs(node: other, color) &&
768	!drm_mm_hole_follows(node))
769	return false;
770
771	return true;
772	}
773
774	/**
775	* i915_vma_insert - finds a slot for the vma in its address space
776	* @vma: the vma
777	* @ww: An optional struct i915_gem_ww_ctx
778	* @size: requested size in bytes (can be larger than the VMA)
779	* @alignment: required alignment
780	* @flags: mask of PIN_* flags to use
781	*
782	* First we try to allocate some free space that meets the requirements for
783	* the VMA. Failing that, if the flags permit, it will evict an old VMA,
784	* preferably the oldest idle entry to make room for the new VMA.
785	*
786	* Returns:
787	* 0 on success, negative error code otherwise.
788	*/
789	static int
790	i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
791	u64 size, u64 alignment, u64 flags)
792	{
793	unsigned long color, guard;
794	u64 start, end;
795	int ret;
796
797	GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
798	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
799	GEM_BUG_ON(hweight64(flags & (PIN_OFFSET_GUARD | PIN_OFFSET_FIXED | PIN_OFFSET_BIAS)) > 1);
800
801	size = max(size, vma->size);
802	alignment = max_t(typeof(alignment), alignment, vma->display_alignment);
803	if (flags & PIN_MAPPABLE) {
804	size = max_t(typeof(size), size, vma->fence_size);
805	alignment = max_t(typeof(alignment),
806	alignment, vma->fence_alignment);
807	}
808
809	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
810	GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
811	GEM_BUG_ON(!is_power_of_2(alignment));
812
813	guard = vma->guard; /* retain guard across rebinds */
814	if (flags & PIN_OFFSET_GUARD) {
815	GEM_BUG_ON(overflows_type(flags & PIN_OFFSET_MASK, u32));
816	guard = max_t(u32, guard, flags & PIN_OFFSET_MASK);
817	}
818	/*
819	* As we align the node upon insertion, but the hardware gets
820	* node.start + guard, the easiest way to make that work is
821	* to make the guard a multiple of the alignment size.
822	*/
823	guard = ALIGN(guard, alignment);
824
825	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
826	GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
827
828	end = vma->vm->total;
829	if (flags & PIN_MAPPABLE)
830	end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end);
831	if (flags & PIN_ZONE_4G)
832	end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
833	GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
834
835	alignment = max(alignment, i915_vm_obj_min_alignment(vma->vm, vma->obj));
836
837	/*
838	* If binding the object/GGTT view requires more space than the entire
839	* aperture has, reject it early before evicting everything in a vain
840	* attempt to find space.
841	*/
842	if (size > end - 2 * guard) {
843	drm_dbg(vma->obj->base.dev,
844	"Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",
845	size, flags & PIN_MAPPABLE ? "mappable" : "total", end);
846	return -ENOSPC;
847	}
848
849	color = 0;
850
851	if (i915_vm_has_cache_coloring(vm: vma->vm))
852	color = vma->obj->pat_index;
853
854	if (flags & PIN_OFFSET_FIXED) {
855	u64 offset = flags & PIN_OFFSET_MASK;
856	if (!IS_ALIGNED(offset, alignment) ||
857	range_overflows(offset, size, end))
858	return -EINVAL;
859	/*
860	* The caller knows not of the guard added by others and
861	* requests for the offset of the start of its buffer
862	* to be fixed, which may not be the same as the position
863	* of the vma->node due to the guard pages.
864	*/
865	if (offset < guard || offset + size > end - guard)
866	return -ENOSPC;
867
868	ret = i915_gem_gtt_reserve(vm: vma->vm, ww, node: &vma->node,
869	size: size + 2 * guard,
870	offset: offset - guard,
871	color, flags);
872	if (ret)
873	return ret;
874	} else {
875	size += 2 * guard;
876	/*
877	* We only support huge gtt pages through the 48b PPGTT,
878	* however we also don't want to force any alignment for
879	* objects which need to be tightly packed into the low 32bits.
880	*
881	* Note that we assume that GGTT are limited to 4GiB for the
882	* foreseeable future. See also i915_ggtt_offset().
883	*/
884	if (upper_32_bits(end - 1) &&
885	vma->page_sizes.sg > I915_GTT_PAGE_SIZE &&
886	!HAS_64K_PAGES(vma->vm->i915)) {
887	/*
888	* We can't mix 64K and 4K PTEs in the same page-table
889	* (2M block), and so to avoid the ugliness and
890	* complexity of coloring we opt for just aligning 64K
891	* objects to 2M.
892	*/
893	u64 page_alignment =
894	rounddown_pow_of_two(vma->page_sizes.sg |
895	I915_GTT_PAGE_SIZE_2M);
896
897	/*
898	* Check we don't expand for the limited Global GTT
899	* (mappable aperture is even more precious!). This
900	* also checks that we exclude the aliasing-ppgtt.
901	*/
902	GEM_BUG_ON(i915_vma_is_ggtt(vma));
903
904	alignment = max(alignment, page_alignment);
905
906	if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
907	size = round_up(size, I915_GTT_PAGE_SIZE_2M);
908	}
909
910	ret = i915_gem_gtt_insert(vm: vma->vm, ww, node: &vma->node,
911	size, alignment, color,
912	start, end, flags);
913	if (ret)
914	return ret;
915
916	GEM_BUG_ON(vma->node.start < start);
917	GEM_BUG_ON(vma->node.start + vma->node.size > end);
918	}
919	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
920	GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color));
921
922	list_move_tail(list: &vma->vm_link, head: &vma->vm->bound_list);
923	vma->guard = guard;
924
925	return 0;
926	}
927
928	static void
929	i915_vma_detach(struct i915_vma *vma)
930	{
931	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
932	GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
933
934	/*
935	* And finally now the object is completely decoupled from this
936	* vma, we can drop its hold on the backing storage and allow
937	* it to be reaped by the shrinker.
938	*/
939	list_move_tail(list: &vma->vm_link, head: &vma->vm->unbound_list);
940	}
941
942	static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
943	{
944	unsigned int bound;
945
946	bound = atomic_read(v: &vma->flags);
947
948	if (flags & PIN_VALIDATE) {
949	flags &= I915_VMA_BIND_MASK;
950
951	return (flags & bound) == flags;
952	}
953
954	/* with the lock mandatory for unbind, we don't race here */
955	flags &= I915_VMA_BIND_MASK;
956	do {
957	if (unlikely(flags & ~bound))
958	return false;
959
960	if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR)))
961	return false;
962
963	GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0);
964	} while (!atomic_try_cmpxchg(v: &vma->flags, old: &bound, new: bound + 1));
965
966	return true;
967	}
968
969	static struct scatterlist *
970	rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
971	unsigned int width, unsigned int height,
972	unsigned int src_stride, unsigned int dst_stride,
973	struct sg_table *st, struct scatterlist *sg)
974	{
975	unsigned int column, row;
976	pgoff_t src_idx;
977
978	for (column = 0; column < width; column++) {
979	unsigned int left;
980
981	src_idx = src_stride * (height - 1) + column + offset;
982	for (row = 0; row < height; row++) {
983	st->nents++;
984	/*
985	* We don't need the pages, but need to initialize
986	* the entries so the sg list can be happily traversed.
987	* The only thing we need are DMA addresses.
988	*/
989	sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, offset: 0);
990	sg_dma_address(sg) =
991	i915_gem_object_get_dma_address(obj, src_idx);
992	sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
993	sg = sg_next(sg);
994	src_idx -= src_stride;
995	}
996
997	left = (dst_stride - height) * I915_GTT_PAGE_SIZE;
998
999	if (!left)
1000	continue;
1001
1002	st->nents++;
1003
1004	/*
1005	* The DE ignores the PTEs for the padding tiles, the sg entry
1006	* here is just a convenience to indicate how many padding PTEs
1007	* to insert at this spot.
1008	*/
1009	sg_set_page(sg, NULL, len: left, offset: 0);
1010	sg_dma_address(sg) = 0;
1011	sg_dma_len(sg) = left;
1012	sg = sg_next(sg);
1013	}
1014
1015	return sg;
1016	}
1017
1018	static noinline struct sg_table *
1019	intel_rotate_pages(struct intel_rotation_info *rot_info,
1020	struct drm_i915_gem_object *obj)
1021	{
1022	unsigned int size = intel_rotation_info_size(rot_info);
1023	struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
1024	struct sg_table *st;
1025	struct scatterlist *sg;
1026	int ret = -ENOMEM;
1027	int i;
1028
1029	/* Allocate target SG list. */
1030	st = kmalloc(sizeof(*st), GFP_KERNEL);
1031	if (!st)
1032	goto err_st_alloc;
1033
1034	ret = sg_alloc_table(st, size, GFP_KERNEL);
1035	if (ret)
1036	goto err_sg_alloc;
1037
1038	st->nents = 0;
1039	sg = st->sgl;
1040
1041	for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
1042	sg = rotate_pages(obj, offset: rot_info->plane[i].offset,
1043	width: rot_info->plane[i].width, height: rot_info->plane[i].height,
1044	src_stride: rot_info->plane[i].src_stride,
1045	dst_stride: rot_info->plane[i].dst_stride,
1046	st, sg);
1047
1048	return st;
1049
1050	err_sg_alloc:
1051	kfree(objp: st);
1052	err_st_alloc:
1053
1054	drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1055	obj->base.size, rot_info->plane[0].width,
1056	rot_info->plane[0].height, size);
1057
1058	return ERR_PTR(error: ret);
1059	}
1060
1061	static struct scatterlist *
1062	add_padding_pages(unsigned int count,
1063	struct sg_table *st, struct scatterlist *sg)
1064	{
1065	st->nents++;
1066
1067	/*
1068	* The DE ignores the PTEs for the padding tiles, the sg entry
1069	* here is just a convenience to indicate how many padding PTEs
1070	* to insert at this spot.
1071	*/
1072	sg_set_page(sg, NULL, len: count * I915_GTT_PAGE_SIZE, offset: 0);
1073	sg_dma_address(sg) = 0;
1074	sg_dma_len(sg) = count * I915_GTT_PAGE_SIZE;
1075	sg = sg_next(sg);
1076
1077	return sg;
1078	}
1079
1080	static struct scatterlist *
1081	remap_tiled_color_plane_pages(struct drm_i915_gem_object *obj,
1082	unsigned long offset, unsigned int alignment_pad,
1083	unsigned int width, unsigned int height,
1084	unsigned int src_stride, unsigned int dst_stride,
1085	struct sg_table *st, struct scatterlist *sg,
1086	unsigned int *gtt_offset)
1087	{
1088	unsigned int row;
1089
1090	if (!width || !height)
1091	return sg;
1092
1093	if (alignment_pad)
1094	sg = add_padding_pages(count: alignment_pad, st, sg);
1095
1096	for (row = 0; row < height; row++) {
1097	unsigned int left = width * I915_GTT_PAGE_SIZE;
1098
1099	while (left) {
1100	dma_addr_t addr;
1101	unsigned int length;
1102
1103	/*
1104	* We don't need the pages, but need to initialize
1105	* the entries so the sg list can be happily traversed.
1106	* The only thing we need are DMA addresses.
1107	*/
1108
1109	addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
1110
1111	length = min(left, length);
1112
1113	st->nents++;
1114
1115	sg_set_page(sg, NULL, len: length, offset: 0);
1116	sg_dma_address(sg) = addr;
1117	sg_dma_len(sg) = length;
1118	sg = sg_next(sg);
1119
1120	offset += length / I915_GTT_PAGE_SIZE;
1121	left -= length;
1122	}
1123
1124	offset += src_stride - width;
1125
1126	left = (dst_stride - width) * I915_GTT_PAGE_SIZE;
1127
1128	if (!left)
1129	continue;
1130
1131	sg = add_padding_pages(count: left >> PAGE_SHIFT, st, sg);
1132	}
1133
1134	*gtt_offset += alignment_pad + dst_stride * height;
1135
1136	return sg;
1137	}
1138
1139	static struct scatterlist *
1140	remap_contiguous_pages(struct drm_i915_gem_object *obj,
1141	pgoff_t obj_offset,
1142	unsigned int count,
1143	struct sg_table *st, struct scatterlist *sg)
1144	{
1145	struct scatterlist *iter;
1146	unsigned int offset;
1147
1148	iter = i915_gem_object_get_sg_dma(obj, obj_offset, &offset);
1149	GEM_BUG_ON(!iter);
1150
1151	do {
1152	unsigned int len;
1153
1154	len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT),
1155	count << PAGE_SHIFT);
1156	sg_set_page(sg, NULL, len, offset: 0);
1157	sg_dma_address(sg) =
1158	sg_dma_address(iter) + (offset << PAGE_SHIFT);
1159	sg_dma_len(sg) = len;
1160
1161	st->nents++;
1162	count -= len >> PAGE_SHIFT;
1163	if (count == 0)
1164	return sg;
1165
1166	sg = __sg_next(sg);
1167	iter = __sg_next(sg: iter);
1168	offset = 0;
1169	} while (1);
1170	}
1171
1172	static struct scatterlist *
1173	remap_linear_color_plane_pages(struct drm_i915_gem_object *obj,
1174	pgoff_t obj_offset, unsigned int alignment_pad,
1175	unsigned int size,
1176	struct sg_table *st, struct scatterlist *sg,
1177	unsigned int *gtt_offset)
1178	{
1179	if (!size)
1180	return sg;
1181
1182	if (alignment_pad)
1183	sg = add_padding_pages(count: alignment_pad, st, sg);
1184
1185	sg = remap_contiguous_pages(obj, obj_offset, count: size, st, sg);
1186	sg = sg_next(sg);
1187
1188	*gtt_offset += alignment_pad + size;
1189
1190	return sg;
1191	}
1192
1193	static struct scatterlist *
1194	remap_color_plane_pages(const struct intel_remapped_info *rem_info,
1195	struct drm_i915_gem_object *obj,
1196	int color_plane,
1197	struct sg_table *st, struct scatterlist *sg,
1198	unsigned int *gtt_offset)
1199	{
1200	unsigned int alignment_pad = 0;
1201
1202	if (rem_info->plane_alignment)
1203	alignment_pad = ALIGN(*gtt_offset, rem_info->plane_alignment) - *gtt_offset;
1204
1205	if (rem_info->plane[color_plane].linear)
1206	sg = remap_linear_color_plane_pages(obj,
1207	obj_offset: rem_info->plane[color_plane].offset,
1208	alignment_pad,
1209	size: rem_info->plane[color_plane].size,
1210	st, sg,
1211	gtt_offset);
1212
1213	else
1214	sg = remap_tiled_color_plane_pages(obj,
1215	offset: rem_info->plane[color_plane].offset,
1216	alignment_pad,
1217	width: rem_info->plane[color_plane].width,
1218	height: rem_info->plane[color_plane].height,
1219	src_stride: rem_info->plane[color_plane].src_stride,
1220	dst_stride: rem_info->plane[color_plane].dst_stride,
1221	st, sg,
1222	gtt_offset);
1223
1224	return sg;
1225	}
1226
1227	static noinline struct sg_table *
1228	intel_remap_pages(struct intel_remapped_info *rem_info,
1229	struct drm_i915_gem_object *obj)
1230	{
1231	unsigned int size = intel_remapped_info_size(rem_info);
1232	struct drm_i915_private *i915 = to_i915(dev: obj->base.dev);
1233	struct sg_table *st;
1234	struct scatterlist *sg;
1235	unsigned int gtt_offset = 0;
1236	int ret = -ENOMEM;
1237	int i;
1238
1239	/* Allocate target SG list. */
1240	st = kmalloc(sizeof(*st), GFP_KERNEL);
1241	if (!st)
1242	goto err_st_alloc;
1243
1244	ret = sg_alloc_table(st, size, GFP_KERNEL);
1245	if (ret)
1246	goto err_sg_alloc;
1247
1248	st->nents = 0;
1249	sg = st->sgl;
1250
1251	for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
1252	sg = remap_color_plane_pages(rem_info, obj, color_plane: i, st, sg, gtt_offset: &gtt_offset);
1253
1254	i915_sg_trim(orig_st: st);
1255
1256	return st;
1257
1258	err_sg_alloc:
1259	kfree(objp: st);
1260	err_st_alloc:
1261
1262	drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1263	obj->base.size, rem_info->plane[0].width,
1264	rem_info->plane[0].height, size);
1265
1266	return ERR_PTR(error: ret);
1267	}
1268
1269	static noinline struct sg_table *
1270	intel_partial_pages(const struct i915_gtt_view *view,
1271	struct drm_i915_gem_object *obj)
1272	{
1273	struct sg_table *st;
1274	struct scatterlist *sg;
1275	unsigned int count = view->partial.size;
1276	int ret = -ENOMEM;
1277
1278	st = kmalloc(sizeof(*st), GFP_KERNEL);
1279	if (!st)
1280	goto err_st_alloc;
1281
1282	ret = sg_alloc_table(st, count, GFP_KERNEL);
1283	if (ret)
1284	goto err_sg_alloc;
1285
1286	st->nents = 0;
1287
1288	sg = remap_contiguous_pages(obj, obj_offset: view->partial.offset, count, st, sg: st->sgl);
1289
1290	sg_mark_end(sg);
1291	i915_sg_trim(orig_st: st); /* Drop any unused tail entries. */
1292
1293	return st;
1294
1295	err_sg_alloc:
1296	kfree(objp: st);
1297	err_st_alloc:
1298	return ERR_PTR(error: ret);
1299	}
1300
1301	static int
1302	__i915_vma_get_pages(struct i915_vma *vma)
1303	{
1304	struct sg_table *pages;
1305
1306	/*
1307	* The vma->pages are only valid within the lifespan of the borrowed
1308	* obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
1309	* must be the vma->pages. A simple rule is that vma->pages must only
1310	* be accessed when the obj->mm.pages are pinned.
1311	*/
1312	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
1313
1314	switch (vma->gtt_view.type) {
1315	default:
1316	GEM_BUG_ON(vma->gtt_view.type);
1317	fallthrough;
1318	case I915_GTT_VIEW_NORMAL:
1319	pages = vma->obj->mm.pages;
1320	break;
1321
1322	case I915_GTT_VIEW_ROTATED:
1323	pages =
1324	intel_rotate_pages(rot_info: &vma->gtt_view.rotated, obj: vma->obj);
1325	break;
1326
1327	case I915_GTT_VIEW_REMAPPED:
1328	pages =
1329	intel_remap_pages(rem_info: &vma->gtt_view.remapped, obj: vma->obj);
1330	break;
1331
1332	case I915_GTT_VIEW_PARTIAL:
1333	pages = intel_partial_pages(view: &vma->gtt_view, obj: vma->obj);
1334	break;
1335	}
1336
1337	if (IS_ERR(ptr: pages)) {
1338	drm_err(&vma->vm->i915->drm,
1339	"Failed to get pages for VMA view type %u (%ld)!\n",
1340	vma->gtt_view.type, PTR_ERR(pages));
1341	return PTR_ERR(ptr: pages);
1342	}
1343
1344	vma->pages = pages;
1345
1346	return 0;
1347	}
1348
1349	I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
1350	{
1351	int err;
1352
1353	if (atomic_add_unless(v: &vma->pages_count, a: 1, u: 0))
1354	return 0;
1355
1356	err = i915_gem_object_pin_pages(obj: vma->obj);
1357	if (err)
1358	return err;
1359
1360	err = __i915_vma_get_pages(vma);
1361	if (err)
1362	goto err_unpin;
1363
1364	vma->page_sizes = vma->obj->mm.page_sizes;
1365	atomic_inc(v: &vma->pages_count);
1366
1367	return 0;
1368
1369	err_unpin:
1370	__i915_gem_object_unpin_pages(obj: vma->obj);
1371
1372	return err;
1373	}
1374
1375	void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
1376	{
1377	struct intel_gt *gt;
1378	int id;
1379
1380	if (!tlb)
1381	return;
1382
1383	/*
1384	* Before we release the pages that were bound by this vma, we
1385	* must invalidate all the TLBs that may still have a reference
1386	* back to our physical address. It only needs to be done once,
1387	* so after updating the PTE to point away from the pages, record
1388	* the most recent TLB invalidation seqno, and if we have not yet
1389	* flushed the TLBs upon release, perform a full invalidation.
1390	*/
1391	for_each_gt(gt, vm->i915, id)
1392	WRITE_ONCE(tlb[id],
1393	intel_gt_next_invalidate_tlb_full(gt));
1394	}
1395
1396	static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
1397	{
1398	/* We allocate under vma_get_pages, so beware the shrinker */
1399	GEM_BUG_ON(atomic_read(&vma->pages_count) < count);
1400
1401	if (atomic_sub_return(i: count, v: &vma->pages_count) == 0) {
1402	if (vma->pages != vma->obj->mm.pages) {
1403	sg_free_table(vma->pages);
1404	kfree(objp: vma->pages);
1405	}
1406	vma->pages = NULL;
1407
1408	i915_gem_object_unpin_pages(obj: vma->obj);
1409	}
1410	}
1411
1412	I915_SELFTEST_EXPORT void i915_vma_put_pages(struct i915_vma *vma)
1413	{
1414	if (atomic_add_unless(v: &vma->pages_count, a: -1, u: 1))
1415	return;
1416
1417	__vma_put_pages(vma, count: 1);
1418	}
1419
1420	static void vma_unbind_pages(struct i915_vma *vma)
1421	{
1422	unsigned int count;
1423
1424	lockdep_assert_held(&vma->vm->mutex);
1425
1426	/* The upper portion of pages_count is the number of bindings */
1427	count = atomic_read(v: &vma->pages_count);
1428	count >>= I915_VMA_PAGES_BIAS;
1429	GEM_BUG_ON(!count);
1430
1431	__vma_put_pages(vma, count: count | count << I915_VMA_PAGES_BIAS);
1432	}
1433
1434	int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1435	u64 size, u64 alignment, u64 flags)
1436	{
1437	struct i915_vma_work *work = NULL;
1438	struct dma_fence *moving = NULL;
1439	struct i915_vma_resource *vma_res = NULL;
1440	intel_wakeref_t wakeref;
1441	unsigned int bound;
1442	int err;
1443
1444	assert_vma_held(vma);
1445	GEM_BUG_ON(!ww);
1446
1447	BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
1448	BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
1449
1450	GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL)));
1451
1452	/* First try and grab the pin without rebinding the vma */
1453	if (try_qad_pin(vma, flags))
1454	return 0;
1455
1456	err = i915_vma_get_pages(vma);
1457	if (err)
1458	return err;
1459
1460	/*
1461	* In case of a global GTT, we must hold a runtime-pm wakeref
1462	* while global PTEs are updated. In other cases, we hold
1463	* the rpm reference while the VMA is active. Since runtime
1464	* resume may require allocations, which are forbidden inside
1465	* vm->mutex, get the first rpm wakeref outside of the mutex.
1466	*/
1467	wakeref = intel_runtime_pm_get(rpm: &vma->vm->i915->runtime_pm);
1468
1469	if (flags & vma->vm->bind_async_flags) {
1470	/* lock VM */
1471	err = i915_vm_lock_objects(vm: vma->vm, ww);
1472	if (err)
1473	goto err_rpm;
1474
1475	work = i915_vma_work();
1476	if (!work) {
1477	err = -ENOMEM;
1478	goto err_rpm;
1479	}
1480
1481	work->vm = vma->vm;
1482
1483	err = i915_gem_object_get_moving_fence(obj: vma->obj, fence: &moving);
1484	if (err)
1485	goto err_rpm;
1486
1487	dma_fence_work_chain(f: &work->base, signal: moving);
1488
1489	/* Allocate enough page directories to used PTE */
1490	if (vma->vm->allocate_va_range) {
1491	err = i915_vm_alloc_pt_stash(vm: vma->vm,
1492	stash: &work->stash,
1493	size: vma->size);
1494	if (err)
1495	goto err_fence;
1496
1497	err = i915_vm_map_pt_stash(vm: vma->vm, stash: &work->stash);
1498	if (err)
1499	goto err_fence;
1500	}
1501	}
1502
1503	vma_res = i915_vma_resource_alloc();
1504	if (IS_ERR(ptr: vma_res)) {
1505	err = PTR_ERR(ptr: vma_res);
1506	goto err_fence;
1507	}
1508
1509	/*
1510	* Differentiate between user/kernel vma inside the aliasing-ppgtt.
1511	*
1512	* We conflate the Global GTT with the user's vma when using the
1513	* aliasing-ppgtt, but it is still vitally important to try and
1514	* keep the use cases distinct. For example, userptr objects are
1515	* not allowed inside the Global GTT as that will cause lock
1516	* inversions when we have to evict them the mmu_notifier callbacks -
1517	* but they are allowed to be part of the user ppGTT which can never
1518	* be mapped. As such we try to give the distinct users of the same
1519	* mutex, distinct lockclasses [equivalent to how we keep i915_ggtt
1520	* and i915_ppgtt separate].
1521	*
1522	* NB this may cause us to mask real lock inversions -- while the
1523	* code is safe today, lockdep may not be able to spot future
1524	* transgressions.
1525	*/
1526	err = mutex_lock_interruptible_nested(lock: &vma->vm->mutex,
1527	subclass: !(flags & PIN_GLOBAL));
1528	if (err)
1529	goto err_vma_res;
1530
1531	/* No more allocations allowed now we hold vm->mutex */
1532
1533	if (unlikely(i915_vma_is_closed(vma))) {
1534	err = -ENOENT;
1535	goto err_unlock;
1536	}
1537
1538	bound = atomic_read(v: &vma->flags);
1539	if (unlikely(bound & I915_VMA_ERROR)) {
1540	err = -ENOMEM;
1541	goto err_unlock;
1542	}
1543
1544	if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))) {
1545	err = -EAGAIN; /* pins are meant to be fairly temporary */
1546	goto err_unlock;
1547	}
1548
1549	if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) {
1550	if (!(flags & PIN_VALIDATE))
1551	__i915_vma_pin(vma);
1552	goto err_unlock;
1553	}
1554
1555	err = i915_active_acquire(ref: &vma->active);
1556	if (err)
1557	goto err_unlock;
1558
1559	if (!(bound & I915_VMA_BIND_MASK)) {
1560	err = i915_vma_insert(vma, ww, size, alignment, flags);
1561	if (err)
1562	goto err_active;
1563
1564	if (i915_is_ggtt(vma->vm))
1565	__i915_vma_set_map_and_fenceable(vma);
1566	}
1567
1568	GEM_BUG_ON(!vma->pages);
1569	err = i915_vma_bind(vma,
1570	pat_index: vma->obj->pat_index,
1571	flags, work, vma_res);
1572	vma_res = NULL;
1573	if (err)
1574	goto err_remove;
1575
1576	/* There should only be at most 2 active bindings (user, global) */
1577	GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound);
1578	atomic_add(I915_VMA_PAGES_ACTIVE, v: &vma->pages_count);
1579	list_move_tail(list: &vma->vm_link, head: &vma->vm->bound_list);
1580
1581	if (!(flags & PIN_VALIDATE)) {
1582	__i915_vma_pin(vma);
1583	GEM_BUG_ON(!i915_vma_is_pinned(vma));
1584	}
1585	GEM_BUG_ON(!i915_vma_is_bound(vma, flags));
1586	GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
1587
1588	err_remove:
1589	if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) {
1590	i915_vma_detach(vma);
1591	drm_mm_remove_node(node: &vma->node);
1592	}
1593	err_active:
1594	i915_active_release(ref: &vma->active);
1595	err_unlock:
1596	mutex_unlock(lock: &vma->vm->mutex);
1597	err_vma_res:
1598	i915_vma_resource_free(vma_res);
1599	err_fence:
1600	if (work) {
1601	/*
1602	* When pinning VMA to GGTT on CHV or BXT with VTD enabled,
1603	* commit VMA binding asynchronously to avoid risk of lock
1604	* inversion among reservation_ww locks held here and
1605	* cpu_hotplug_lock acquired from stop_machine(), which we
1606	* wrap around GGTT updates when running in those environments.
1607	*/
1608	if (i915_vma_is_ggtt(vma) &&
1609	intel_vm_no_concurrent_access_wa(i915: vma->vm->i915))
1610	dma_fence_work_commit(f: &work->base);
1611	else
1612	dma_fence_work_commit_imm(f: &work->base);
1613	}
1614	err_rpm:
1615	intel_runtime_pm_put(rpm: &vma->vm->i915->runtime_pm, wref: wakeref);
1616
1617	if (moving)
1618	dma_fence_put(fence: moving);
1619
1620	i915_vma_put_pages(vma);
1621	return err;
1622	}
1623
1624	int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
1625	{
1626	struct i915_gem_ww_ctx ww;
1627	int err;
1628
1629	i915_gem_ww_ctx_init(ctx: &ww, intr: true);
1630	retry:
1631	err = i915_gem_object_lock(obj: vma->obj, ww: &ww);
1632	if (!err)
1633	err = i915_vma_pin_ww(vma, ww: &ww, size, alignment, flags);
1634	if (err == -EDEADLK) {
1635	err = i915_gem_ww_ctx_backoff(ctx: &ww);
1636	if (!err)
1637	goto retry;
1638	}
1639	i915_gem_ww_ctx_fini(ctx: &ww);
1640
1641	return err;
1642	}
1643
1644	static void flush_idle_contexts(struct intel_gt *gt)
1645	{
1646	struct intel_engine_cs *engine;
1647	enum intel_engine_id id;
1648
1649	for_each_engine(engine, gt, id)
1650	intel_engine_flush_barriers(engine);
1651
1652	intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT);
1653	}
1654
1655	static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1656	u32 align, unsigned int flags)
1657	{
1658	struct i915_address_space *vm = vma->vm;
1659	struct intel_gt *gt;
1660	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
1661	int err;
1662
1663	do {
1664	err = i915_vma_pin_ww(vma, ww, size: 0, alignment: align, flags: flags | PIN_GLOBAL);
1665
1666	if (err != -ENOSPC) {
1667	if (!err) {
1668	err = i915_vma_wait_for_bind(vma);
1669	if (err)
1670	i915_vma_unpin(vma);
1671	}
1672	return err;
1673	}
1674
1675	/* Unlike i915_vma_pin, we don't take no for an answer! */
1676	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
1677	flush_idle_contexts(gt);
1678	if (mutex_lock_interruptible(&vm->mutex) == 0) {
1679	/*
1680	* We pass NULL ww here, as we don't want to unbind
1681	* locked objects when called from execbuf when pinning
1682	* is removed. This would probably regress badly.
1683	*/
1684	i915_gem_evict_vm(vm, NULL, NULL);
1685	mutex_unlock(lock: &vm->mutex);
1686	}
1687	} while (1);
1688	}
1689
1690	int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1691	u32 align, unsigned int flags)
1692	{
1693	struct i915_gem_ww_ctx _ww;
1694	int err;
1695
1696	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
1697
1698	if (ww)
1699	return __i915_ggtt_pin(vma, ww, align, flags);
1700
1701	lockdep_assert_not_held(&vma->obj->base.resv->lock.base);
1702
1703	for_i915_gem_ww(&_ww, err, true) {
1704	err = i915_gem_object_lock(obj: vma->obj, ww: &_ww);
1705	if (!err)
1706	err = __i915_ggtt_pin(vma, ww: &_ww, align, flags);
1707	}
1708
1709	return err;
1710	}
1711
1712	/**
1713	* i915_ggtt_clear_scanout - Clear scanout flag for all objects ggtt vmas
1714	* @obj: i915 GEM object
1715	* This function clears scanout flags for objects ggtt vmas. These flags are set
1716	* when object is pinned for display use and this function to clear them all is
1717	* targeted to be called by frontbuffer tracking code when the frontbuffer is
1718	* about to be released.
1719	*/
1720	void i915_ggtt_clear_scanout(struct drm_i915_gem_object *obj)
1721	{
1722	struct i915_vma *vma;
1723
1724	spin_lock(lock: &obj->vma.lock);
1725	for_each_ggtt_vma(vma, obj) {
1726	i915_vma_clear_scanout(vma);
1727	vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
1728	}
1729	spin_unlock(lock: &obj->vma.lock);
1730	}
1731
1732	static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
1733	{
1734	/*
1735	* We defer actually closing, unbinding and destroying the VMA until
1736	* the next idle point, or if the object is freed in the meantime. By
1737	* postponing the unbind, we allow for it to be resurrected by the
1738	* client, avoiding the work required to rebind the VMA. This is
1739	* advantageous for DRI, where the client/server pass objects
1740	* between themselves, temporarily opening a local VMA to the
1741	* object, and then closing it again. The same object is then reused
1742	* on the next frame (or two, depending on the depth of the swap queue)
1743	* causing us to rebind the VMA once more. This ends up being a lot
1744	* of wasted work for the steady state.
1745	*/
1746	GEM_BUG_ON(i915_vma_is_closed(vma));
1747	list_add(new: &vma->closed_link, head: &gt->closed_vma);
1748	}
1749
1750	void i915_vma_close(struct i915_vma *vma)
1751	{
1752	struct intel_gt *gt = vma->vm->gt;
1753	unsigned long flags;
1754
1755	if (i915_vma_is_ggtt(vma))
1756	return;
1757
1758	GEM_BUG_ON(!atomic_read(&vma->open_count));
1759	if (atomic_dec_and_lock_irqsave(&vma->open_count,
1760	&gt->closed_lock,
1761	flags)) {
1762	__vma_close(vma, gt);
1763	spin_unlock_irqrestore(lock: &gt->closed_lock, flags);
1764	}
1765	}
1766
1767	static void __i915_vma_remove_closed(struct i915_vma *vma)
1768	{
1769	list_del_init(entry: &vma->closed_link);
1770	}
1771
1772	void i915_vma_reopen(struct i915_vma *vma)
1773	{
1774	struct intel_gt *gt = vma->vm->gt;
1775
1776	spin_lock_irq(lock: &gt->closed_lock);
1777	if (i915_vma_is_closed(vma))
1778	__i915_vma_remove_closed(vma);
1779	spin_unlock_irq(lock: &gt->closed_lock);
1780	}
1781
1782	static void force_unbind(struct i915_vma *vma)
1783	{
1784	if (!drm_mm_node_allocated(node: &vma->node))
1785	return;
1786
1787	atomic_and(i: ~I915_VMA_PIN_MASK, v: &vma->flags);
1788	WARN_ON(__i915_vma_unbind(vma));
1789	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
1790	}
1791
1792	static void release_references(struct i915_vma *vma, struct intel_gt *gt,
1793	bool vm_ddestroy)
1794	{
1795	struct drm_i915_gem_object *obj = vma->obj;
1796
1797	GEM_BUG_ON(i915_vma_is_active(vma));
1798
1799	spin_lock(lock: &obj->vma.lock);
1800	list_del(entry: &vma->obj_link);
1801	if (!RB_EMPTY_NODE(&vma->obj_node))
1802	rb_erase(&vma->obj_node, &obj->vma.tree);
1803
1804	spin_unlock(lock: &obj->vma.lock);
1805
1806	spin_lock_irq(lock: &gt->closed_lock);
1807	__i915_vma_remove_closed(vma);
1808	spin_unlock_irq(lock: &gt->closed_lock);
1809
1810	if (vm_ddestroy)
1811	i915_vm_resv_put(vm: vma->vm);
1812
1813	i915_active_fini(ref: &vma->active);
1814	GEM_WARN_ON(vma->resource);
1815	i915_vma_free(vma);
1816	}
1817
1818	/*
1819	* i915_vma_destroy_locked - Remove all weak reference to the vma and put
1820	* the initial reference.
1821	*
1822	* This function should be called when it's decided the vma isn't needed
1823	* anymore. The caller must assure that it doesn't race with another lookup
1824	* plus destroy, typically by taking an appropriate reference.
1825	*
1826	* Current callsites are
1827	* - __i915_gem_object_pages_fini()
1828	* - __i915_vm_close() - Blocks the above function by taking a reference on
1829	* the object.
1830	* - __i915_vma_parked() - Blocks the above functions by taking a reference
1831	* on the vm and a reference on the object. Also takes the object lock so
1832	* destruction from __i915_vma_parked() can be blocked by holding the
1833	* object lock. Since the object lock is only allowed from within i915 with
1834	* an object refcount, holding the object lock also implicitly blocks the
1835	* vma freeing from __i915_gem_object_pages_fini().
1836	*
1837	* Because of locks taken during destruction, a vma is also guaranteed to
1838	* stay alive while the following locks are held if it was looked up while
1839	* holding one of the locks:
1840	* - vm->mutex
1841	* - obj->vma.lock
1842	* - gt->closed_lock
1843	*/
1844	void i915_vma_destroy_locked(struct i915_vma *vma)
1845	{
1846	lockdep_assert_held(&vma->vm->mutex);
1847
1848	force_unbind(vma);
1849	list_del_init(entry: &vma->vm_link);
1850	release_references(vma, gt: vma->vm->gt, vm_ddestroy: false);
1851	}
1852
1853	void i915_vma_destroy(struct i915_vma *vma)
1854	{
1855	struct intel_gt *gt;
1856	bool vm_ddestroy;
1857
1858	mutex_lock(&vma->vm->mutex);
1859	force_unbind(vma);
1860	list_del_init(entry: &vma->vm_link);
1861	vm_ddestroy = vma->vm_ddestroy;
1862	vma->vm_ddestroy = false;
1863
1864	/* vma->vm may be freed when releasing vma->vm->mutex. */
1865	gt = vma->vm->gt;
1866	mutex_unlock(lock: &vma->vm->mutex);
1867	release_references(vma, gt, vm_ddestroy);
1868	}
1869
1870	void i915_vma_parked(struct intel_gt *gt)
1871	{
1872	struct i915_vma *vma, *next;
1873	LIST_HEAD(closed);
1874
1875	spin_lock_irq(lock: &gt->closed_lock);
1876	list_for_each_entry_safe(vma, next, &gt->closed_vma, closed_link) {
1877	struct drm_i915_gem_object *obj = vma->obj;
1878	struct i915_address_space *vm = vma->vm;
1879
1880	/* XXX All to avoid keeping a reference on i915_vma itself */
1881
1882	if (!kref_get_unless_zero(kref: &obj->base.refcount))
1883	continue;
1884
1885	if (!i915_vm_tryget(vm)) {
1886	i915_gem_object_put(obj);
1887	continue;
1888	}
1889
1890	list_move(list: &vma->closed_link, head: &closed);
1891	}
1892	spin_unlock_irq(lock: &gt->closed_lock);
1893
1894	/* As the GT is held idle, no vma can be reopened as we destroy them */
1895	list_for_each_entry_safe(vma, next, &closed, closed_link) {
1896	struct drm_i915_gem_object *obj = vma->obj;
1897	struct i915_address_space *vm = vma->vm;
1898
1899	if (i915_gem_object_trylock(obj, NULL)) {
1900	INIT_LIST_HEAD(list: &vma->closed_link);
1901	i915_vma_destroy(vma);
1902	i915_gem_object_unlock(obj);
1903	} else {
1904	/* back you go.. */
1905	spin_lock_irq(lock: &gt->closed_lock);
1906	list_add(new: &vma->closed_link, head: &gt->closed_vma);
1907	spin_unlock_irq(lock: &gt->closed_lock);
1908	}
1909
1910	i915_gem_object_put(obj);
1911	i915_vm_put(vm);
1912	}
1913	}
1914
1915	static void __i915_vma_iounmap(struct i915_vma *vma)
1916	{
1917	GEM_BUG_ON(i915_vma_is_pinned(vma));
1918
1919	if (vma->iomap == NULL)
1920	return;
1921
1922	if (page_unmask_bits(vma->iomap))
1923	__i915_gem_object_release_map(obj: vma->obj);
1924	else
1925	io_mapping_unmap(vaddr: vma->iomap);
1926	vma->iomap = NULL;
1927	}
1928
1929	void i915_vma_revoke_mmap(struct i915_vma *vma)
1930	{
1931	struct drm_vma_offset_node *node;
1932	u64 vma_offset;
1933
1934	if (!i915_vma_has_userfault(vma))
1935	return;
1936
1937	GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
1938	GEM_BUG_ON(!vma->obj->userfault_count);
1939
1940	node = &vma->mmo->vma_node;
1941	vma_offset = vma->gtt_view.partial.offset << PAGE_SHIFT;
1942	unmap_mapping_range(mapping: vma->vm->i915->drm.anon_inode->i_mapping,
1943	holebegin: drm_vma_node_offset_addr(node) + vma_offset,
1944	holelen: vma->size,
1945	even_cows: 1);
1946
1947	i915_vma_unset_userfault(vma);
1948	if (!--vma->obj->userfault_count)
1949	list_del(entry: &vma->obj->userfault_link);
1950	}
1951
1952	static int
1953	__i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma)
1954	{
1955	return __i915_request_await_exclusive(rq, active: &vma->active);
1956	}
1957
1958	static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq)
1959	{
1960	int err;
1961
1962	/* Wait for the vma to be bound before we start! */
1963	err = __i915_request_await_bind(rq, vma);
1964	if (err)
1965	return err;
1966
1967	return i915_active_add_request(ref: &vma->active, rq);
1968	}
1969
1970	int _i915_vma_move_to_active(struct i915_vma *vma,
1971	struct i915_request *rq,
1972	struct dma_fence *fence,
1973	unsigned int flags)
1974	{
1975	struct drm_i915_gem_object *obj = vma->obj;
1976	int err;
1977
1978	assert_object_held(obj);
1979
1980	GEM_BUG_ON(!vma->pages);
1981
1982	if (!(flags & __EXEC_OBJECT_NO_REQUEST_AWAIT)) {
1983	err = i915_request_await_object(to: rq, obj: vma->obj, write: flags & EXEC_OBJECT_WRITE);
1984	if (unlikely(err))
1985	return err;
1986	}
1987	err = __i915_vma_move_to_active(vma, rq);
1988	if (unlikely(err))
1989	return err;
1990
1991	/*
1992	* Reserve fences slot early to prevent an allocation after preparing
1993	* the workload and associating fences with dma_resv.
1994	*/
1995	if (fence && !(flags & __EXEC_OBJECT_NO_RESERVE)) {
1996	struct dma_fence *curr;
1997	int idx;
1998
1999	dma_fence_array_for_each(curr, idx, fence)
2000	;
2001	err = dma_resv_reserve_fences(obj: vma->obj->base.resv, num_fences: idx);
2002	if (unlikely(err))
2003	return err;
2004	}
2005
2006	if (flags & EXEC_OBJECT_WRITE) {
2007	struct i915_frontbuffer *front;
2008
2009	front = i915_gem_object_frontbuffer_lookup(obj);
2010	if (unlikely(front)) {
2011	if (intel_frontbuffer_invalidate(front: &front->base, origin: ORIGIN_CS))
2012	i915_active_add_request(ref: &front->write, rq);
2013	i915_gem_object_frontbuffer_put(front);
2014	}
2015	}
2016
2017	if (fence) {
2018	struct dma_fence *curr;
2019	enum dma_resv_usage usage;
2020	int idx;
2021
2022	if (flags & EXEC_OBJECT_WRITE) {
2023	usage = DMA_RESV_USAGE_WRITE;
2024	obj->write_domain = I915_GEM_DOMAIN_RENDER;
2025	obj->read_domains = 0;
2026	} else {
2027	usage = DMA_RESV_USAGE_READ;
2028	obj->write_domain = 0;
2029	}
2030
2031	dma_fence_array_for_each(curr, idx, fence)
2032	dma_resv_add_fence(obj: vma->obj->base.resv, fence: curr, usage);
2033	}
2034
2035	if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence)
2036	i915_active_add_request(ref: &vma->fence->active, rq);
2037
2038	obj->read_domains |= I915_GEM_GPU_DOMAINS;
2039	obj->mm.dirty = true;
2040
2041	GEM_BUG_ON(!i915_vma_is_active(vma));
2042	return 0;
2043	}
2044
2045	struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
2046	{
2047	struct i915_vma_resource *vma_res = vma->resource;
2048	struct dma_fence *unbind_fence;
2049
2050	GEM_BUG_ON(i915_vma_is_pinned(vma));
2051	assert_vma_held_evict(vma);
2052
2053	if (i915_vma_is_map_and_fenceable(vma)) {
2054	/* Force a pagefault for domain tracking on next user access */
2055	i915_vma_revoke_mmap(vma);
2056
2057	/*
2058	* Check that we have flushed all writes through the GGTT
2059	* before the unbind, other due to non-strict nature of those
2060	* indirect writes they may end up referencing the GGTT PTE
2061	* after the unbind.
2062	*
2063	* Note that we may be concurrently poking at the GGTT_WRITE
2064	* bit from set-domain, as we mark all GGTT vma associated
2065	* with an object. We know this is for another vma, as we
2066	* are currently unbinding this one -- so if this vma will be
2067	* reused, it will be refaulted and have its dirty bit set
2068	* before the next write.
2069	*/
2070	i915_vma_flush_writes(vma);
2071
2072	/* release the fence reg _after_ flushing */
2073	i915_vma_revoke_fence(vma);
2074
2075	clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
2076	}
2077
2078	__i915_vma_iounmap(vma);
2079
2080	GEM_BUG_ON(vma->fence);
2081	GEM_BUG_ON(i915_vma_has_userfault(vma));
2082
2083	/* Object backend must be async capable. */
2084	GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt);
2085
2086	/* If vm is not open, unbind is a nop. */
2087	vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND) &&
2088	kref_read(kref: &vma->vm->ref);
2089	vma_res->skip_pte_rewrite = !kref_read(kref: &vma->vm->ref) ||
2090	vma->vm->skip_pte_rewrite;
2091	trace_i915_vma_unbind(vma);
2092
2093	if (async)
2094	unbind_fence = i915_vma_resource_unbind(vma_res,
2095	tlb: vma->obj->mm.tlb);
2096	else
2097	unbind_fence = i915_vma_resource_unbind(vma_res, NULL);
2098
2099	vma->resource = NULL;
2100
2101	atomic_and(i: ~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE),
2102	v: &vma->flags);
2103
2104	i915_vma_detach(vma);
2105
2106	if (!async) {
2107	if (unbind_fence) {
2108	dma_fence_wait(fence: unbind_fence, intr: false);
2109	dma_fence_put(fence: unbind_fence);
2110	unbind_fence = NULL;
2111	}
2112	vma_invalidate_tlb(vm: vma->vm, tlb: vma->obj->mm.tlb);
2113	}
2114
2115	/*
2116	* Binding itself may not have completed until the unbind fence signals,
2117	* so don't drop the pages until that happens, unless the resource is
2118	* async_capable.
2119	*/
2120
2121	vma_unbind_pages(vma);
2122	return unbind_fence;
2123	}
2124
2125	int __i915_vma_unbind(struct i915_vma *vma)
2126	{
2127	int ret;
2128
2129	lockdep_assert_held(&vma->vm->mutex);
2130	assert_vma_held_evict(vma);
2131
2132	if (!drm_mm_node_allocated(node: &vma->node))
2133	return 0;
2134
2135	if (i915_vma_is_pinned(vma)) {
2136	vma_print_allocator(vma, reason: "is pinned");
2137	return -EAGAIN;
2138	}
2139
2140	/*
2141	* After confirming that no one else is pinning this vma, wait for
2142	* any laggards who may have crept in during the wait (through
2143	* a residual pin skipping the vm->mutex) to complete.
2144	*/
2145	ret = i915_vma_sync(vma);
2146	if (ret)
2147	return ret;
2148
2149	GEM_BUG_ON(i915_vma_is_active(vma));
2150	__i915_vma_evict(vma, async: false);
2151
2152	drm_mm_remove_node(node: &vma->node); /* pairs with i915_vma_release() */
2153	return 0;
2154	}
2155
2156	static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma)
2157	{
2158	struct dma_fence *fence;
2159
2160	lockdep_assert_held(&vma->vm->mutex);
2161
2162	if (!drm_mm_node_allocated(node: &vma->node))
2163	return NULL;
2164
2165	if (i915_vma_is_pinned(vma) ||
2166	&vma->obj->mm.rsgt->table != vma->resource->bi.pages)
2167	return ERR_PTR(error: -EAGAIN);
2168
2169	/*
2170	* We probably need to replace this with awaiting the fences of the
2171	* object's dma_resv when the vma active goes away. When doing that
2172	* we need to be careful to not add the vma_resource unbind fence
2173	* immediately to the object's dma_resv, because then unbinding
2174	* the next vma from the object, in case there are many, will
2175	* actually await the unbinding of the previous vmas, which is
2176	* undesirable.
2177	*/
2178	if (i915_sw_fence_await_active(fence: &vma->resource->chain, ref: &vma->active,
2179	I915_ACTIVE_AWAIT_EXCL |
2180	I915_ACTIVE_AWAIT_ACTIVE) < 0) {
2181	return ERR_PTR(error: -EBUSY);
2182	}
2183
2184	fence = __i915_vma_evict(vma, async: true);
2185
2186	drm_mm_remove_node(node: &vma->node); /* pairs with i915_vma_release() */
2187
2188	return fence;
2189	}
2190
2191	int i915_vma_unbind(struct i915_vma *vma)
2192	{
2193	struct i915_address_space *vm = vma->vm;
2194	intel_wakeref_t wakeref = NULL;
2195	int err;
2196
2197	assert_object_held_shared(obj: vma->obj);
2198
2199	/* Optimistic wait before taking the mutex */
2200	err = i915_vma_sync(vma);
2201	if (err)
2202	return err;
2203
2204	if (!drm_mm_node_allocated(node: &vma->node))
2205	return 0;
2206
2207	if (i915_vma_is_pinned(vma)) {
2208	vma_print_allocator(vma, reason: "is pinned");
2209	return -EAGAIN;
2210	}
2211
2212	if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
2213	/* XXX not always required: nop_clear_range */
2214	wakeref = intel_runtime_pm_get(rpm: &vm->i915->runtime_pm);
2215
2216	err = mutex_lock_interruptible_nested(lock: &vma->vm->mutex, subclass: !wakeref);
2217	if (err)
2218	goto out_rpm;
2219
2220	err = __i915_vma_unbind(vma);
2221	mutex_unlock(lock: &vm->mutex);
2222
2223	out_rpm:
2224	if (wakeref)
2225	intel_runtime_pm_put(rpm: &vm->i915->runtime_pm, wref: wakeref);
2226	return err;
2227	}
2228
2229	int i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm)
2230	{
2231	struct drm_i915_gem_object *obj = vma->obj;
2232	struct i915_address_space *vm = vma->vm;
2233	intel_wakeref_t wakeref = NULL;
2234	struct dma_fence *fence;
2235	int err;
2236
2237	/*
2238	* We need the dma-resv lock since we add the
2239	* unbind fence to the dma-resv object.
2240	*/
2241	assert_object_held(obj);
2242
2243	if (!drm_mm_node_allocated(node: &vma->node))
2244	return 0;
2245
2246	if (i915_vma_is_pinned(vma)) {
2247	vma_print_allocator(vma, reason: "is pinned");
2248	return -EAGAIN;
2249	}
2250
2251	if (!obj->mm.rsgt)
2252	return -EBUSY;
2253
2254	err = dma_resv_reserve_fences(obj: obj->base.resv, num_fences: 2);
2255	if (err)
2256	return -EBUSY;
2257
2258	/*
2259	* It would be great if we could grab this wakeref from the
2260	* async unbind work if needed, but we can't because it uses
2261	* kmalloc and it's in the dma-fence signalling critical path.
2262	*/
2263	if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
2264	wakeref = intel_runtime_pm_get(rpm: &vm->i915->runtime_pm);
2265
2266	if (trylock_vm && !mutex_trylock(&vm->mutex)) {
2267	err = -EBUSY;
2268	goto out_rpm;
2269	} else if (!trylock_vm) {
2270	err = mutex_lock_interruptible_nested(lock: &vm->mutex, subclass: !wakeref);
2271	if (err)
2272	goto out_rpm;
2273	}
2274
2275	fence = __i915_vma_unbind_async(vma);
2276	mutex_unlock(lock: &vm->mutex);
2277	if (IS_ERR_OR_NULL(ptr: fence)) {
2278	err = PTR_ERR_OR_ZERO(ptr: fence);
2279	goto out_rpm;
2280	}
2281
2282	dma_resv_add_fence(obj: obj->base.resv, fence, usage: DMA_RESV_USAGE_READ);
2283	dma_fence_put(fence);
2284
2285	out_rpm:
2286	if (wakeref)
2287	intel_runtime_pm_put(rpm: &vm->i915->runtime_pm, wref: wakeref);
2288	return err;
2289	}
2290
2291	int i915_vma_unbind_unlocked(struct i915_vma *vma)
2292	{
2293	int err;
2294
2295	i915_gem_object_lock(obj: vma->obj, NULL);
2296	err = i915_vma_unbind(vma);
2297	i915_gem_object_unlock(obj: vma->obj);
2298
2299	return err;
2300	}
2301
2302	struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma)
2303	{
2304	i915_gem_object_make_unshrinkable(obj: vma->obj);
2305	return vma;
2306	}
2307
2308	void i915_vma_make_shrinkable(struct i915_vma *vma)
2309	{
2310	i915_gem_object_make_shrinkable(obj: vma->obj);
2311	}
2312
2313	void i915_vma_make_purgeable(struct i915_vma *vma)
2314	{
2315	i915_gem_object_make_purgeable(obj: vma->obj);
2316	}
2317
2318	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2319	#include "selftests/i915_vma.c"
2320	#endif
2321
2322	void i915_vma_module_exit(void)
2323	{
2324	kmem_cache_destroy(s: slab_vmas);
2325	}
2326
2327	int __init i915_vma_module_init(void)
2328	{
2329	slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
2330	if (!slab_vmas)
2331	return -ENOMEM;
2332
2333	return 0;
2334	}
2335