1	// SPDX-License-Identifier: GPL-2.0-only OR MIT
2	/*
3	* Copyright (c) 2022 Red Hat.
4	*
5	* Permission is hereby granted, free of charge, to any person obtaining a
6	* copy of this software and associated documentation files (the "Software"),
7	* to deal in the Software without restriction, including without limitation
8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
9	* and/or sell copies of the Software, and to permit persons to whom the
10	* Software is furnished to do so, subject to the following conditions:
11	*
12	* The above copyright notice and this permission notice shall be included in
13	* all copies or substantial portions of the Software.
14	*
15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21	* OTHER DEALINGS IN THE SOFTWARE.
22	*
23	* Authors:
24	* Danilo Krummrich <dakr@redhat.com>
25	*
26	*/
27
28	#include <drm/drm_gpuvm.h>
29	#include <drm/drm_print.h>
30
31	#include <linux/export.h>
32	#include <linux/interval_tree_generic.h>
33	#include <linux/mm.h>
34
35	/**
36	* DOC: Overview
37	*
38	* The DRM GPU VA Manager, represented by struct drm_gpuvm keeps track of a
39	* GPU's virtual address (VA) space and manages the corresponding virtual
40	* mappings represented by &drm_gpuva objects. It also keeps track of the
41	* mapping's backing &drm_gem_object buffers.
42	*
43	* &drm_gem_object buffers maintain a list of &drm_gpuva objects representing
44	* all existing GPU VA mappings using this &drm_gem_object as backing buffer.
45	*
46	* GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also
47	* keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'.
48	*
49	* The GPU VA manager internally uses a rb-tree to manage the
50	* &drm_gpuva mappings within a GPU's virtual address space.
51	*
52	* The &drm_gpuvm structure contains a special &drm_gpuva representing the
53	* portion of VA space reserved by the kernel. This node is initialized together
54	* with the GPU VA manager instance and removed when the GPU VA manager is
55	* destroyed.
56	*
57	* In a typical application drivers would embed struct drm_gpuvm and
58	* struct drm_gpuva within their own driver specific structures, there won't be
59	* any memory allocations of its own nor memory allocations of &drm_gpuva
60	* entries.
61	*
62	* The data structures needed to store &drm_gpuvas within the &drm_gpuvm are
63	* contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva
64	* entries from within dma-fence signalling critical sections it is enough to
65	* pre-allocate the &drm_gpuva structures.
66	*
67	* &drm_gem_objects which are private to a single VM can share a common
68	* &dma_resv in order to improve locking efficiency (e.g. with &drm_exec).
69	* For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in
70	* the following called 'resv object', which serves as the container of the
71	* GPUVM's shared &dma_resv. This resv object can be a driver specific
72	* &drm_gem_object, such as the &drm_gem_object containing the root page table,
73	* but it can also be a 'dummy' object, which can be allocated with
74	* drm_gpuvm_resv_object_alloc().
75	*
76	* In order to connect a struct drm_gpuva to its backing &drm_gem_object each
77	* &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each
78	* &drm_gpuvm_bo contains a list of &drm_gpuva structures.
79	*
80	* A &drm_gpuvm_bo is an abstraction that represents a combination of a
81	* &drm_gpuvm and a &drm_gem_object. Every such combination should be unique.
82	* This is ensured by the API through drm_gpuvm_bo_obtain() and
83	* drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding
84	* &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this
85	* particular combination. If not present, a new instance is created and linked
86	* to the &drm_gem_object.
87	*
88	* &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used
89	* as entry for the &drm_gpuvm's lists of external and evicted objects. Those
90	* lists are maintained in order to accelerate locking of dma-resv locks and
91	* validation of evicted objects bound in a &drm_gpuvm. For instance, all
92	* &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling
93	* drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in
94	* order to validate all evicted &drm_gem_objects. It is also possible to lock
95	* additional &drm_gem_objects by providing the corresponding parameters to
96	* drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making
97	* use of helper functions such as drm_gpuvm_prepare_range() or
98	* drm_gpuvm_prepare_objects().
99	*
100	* Every bound &drm_gem_object is treated as external object when its &dma_resv
101	* structure is different than the &drm_gpuvm's common &dma_resv structure.
102	*/
103
104	/**
105	* DOC: Split and Merge
106	*
107	* Besides its capability to manage and represent a GPU VA space, the
108	* GPU VA manager also provides functions to let the &drm_gpuvm calculate a
109	* sequence of operations to satisfy a given map or unmap request.
110	*
111	* Therefore the DRM GPU VA manager provides an algorithm implementing splitting
112	* and merging of existing GPU VA mappings with the ones that are requested to
113	* be mapped or unmapped. This feature is required by the Vulkan API to
114	* implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this
115	* as VM BIND.
116	*
117	* Drivers can call drm_gpuvm_sm_map() to receive a sequence of callbacks
118	* containing map, unmap and remap operations for a given newly requested
119	* mapping. The sequence of callbacks represents the set of operations to
120	* execute in order to integrate the new mapping cleanly into the current state
121	* of the GPU VA space.
122	*
123	* Depending on how the new GPU VA mapping intersects with the existing mappings
124	* of the GPU VA space the &drm_gpuvm_ops callbacks contain an arbitrary amount
125	* of unmap operations, a maximum of two remap operations and a single map
126	* operation. The caller might receive no callback at all if no operation is
127	* required, e.g. if the requested mapping already exists in the exact same way.
128	*
129	* The single map operation represents the original map operation requested by
130	* the caller.
131	*
132	* &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the
133	* &drm_gpuva to unmap is physically contiguous with the original mapping
134	* request. Optionally, if 'keep' is set, drivers may keep the actual page table
135	* entries for this &drm_gpuva, adding the missing page table entries only and
136	* update the &drm_gpuvm's view of things accordingly.
137	*
138	* Drivers may do the same optimization, namely delta page table updates, also
139	* for remap operations. This is possible since &drm_gpuva_op_remap consists of
140	* one unmap operation and one or two map operations, such that drivers can
141	* derive the page table update delta accordingly.
142	*
143	* Note that there can't be more than two existing mappings to split up, one at
144	* the beginning and one at the end of the new mapping, hence there is a
145	* maximum of two remap operations.
146	*
147	* Analogous to drm_gpuvm_sm_map() drm_gpuvm_sm_unmap() uses &drm_gpuvm_ops to
148	* call back into the driver in order to unmap a range of GPU VA space. The
149	* logic behind this function is way simpler though: For all existing mappings
150	* enclosed by the given range unmap operations are created. For mappings which
151	* are only partially located within the given range, remap operations are
152	* created such that those mappings are split up and re-mapped partially.
153	*
154	* As an alternative to drm_gpuvm_sm_map() and drm_gpuvm_sm_unmap(),
155	* drm_gpuvm_sm_map_ops_create() and drm_gpuvm_sm_unmap_ops_create() can be used
156	* to directly obtain an instance of struct drm_gpuva_ops containing a list of
157	* &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list
158	* contains the &drm_gpuva_ops analogous to the callbacks one would receive when
159	* calling drm_gpuvm_sm_map() or drm_gpuvm_sm_unmap(). While this way requires
160	* more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to
161	* iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory
162	* allocations are possible (e.g. to allocate GPU page tables) and once in the
163	* dma-fence signalling critical path.
164	*
165	* To update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert() and
166	* drm_gpuva_remove() may be used. These functions can safely be used from
167	* &drm_gpuvm_ops callbacks originating from drm_gpuvm_sm_map() or
168	* drm_gpuvm_sm_unmap(). However, it might be more convenient to use the
169	* provided helper functions drm_gpuva_map(), drm_gpuva_remap() and
170	* drm_gpuva_unmap() instead.
171	*
172	* The following diagram depicts the basic relationships of existing GPU VA
173	* mappings, a newly requested mapping and the resulting mappings as implemented
174	* by drm_gpuvm_sm_map() - it doesn't cover any arbitrary combinations of these.
175	*
176	* 1) Requested mapping is identical. Replace it, but indicate the backing PTEs
177	* could be kept.
178	*
179	* ::
180	*
181	* 0 a 1
182	* old: |-----------| (bo_offset=n)
183	*
184	* 0 a 1
185	* req: |-----------| (bo_offset=n)
186	*
187	* 0 a 1
188	* new: |-----------| (bo_offset=n)
189	*
190	*
191	* 2) Requested mapping is identical, except for the BO offset, hence replace
192	* the mapping.
193	*
194	* ::
195	*
196	* 0 a 1
197	* old: |-----------| (bo_offset=n)
198	*
199	* 0 a 1
200	* req: |-----------| (bo_offset=m)
201	*
202	* 0 a 1
203	* new: |-----------| (bo_offset=m)
204	*
205	*
206	* 3) Requested mapping is identical, except for the backing BO, hence replace
207	* the mapping.
208	*
209	* ::
210	*
211	* 0 a 1
212	* old: |-----------| (bo_offset=n)
213	*
214	* 0 b 1
215	* req: |-----------| (bo_offset=n)
216	*
217	* 0 b 1
218	* new: |-----------| (bo_offset=n)
219	*
220	*
221	* 4) Existent mapping is a left aligned subset of the requested one, hence
222	* replace the existing one.
223	*
224	* ::
225	*
226	* 0 a 1
227	* old: |-----| (bo_offset=n)
228	*
229	* 0 a 2
230	* req: |-----------| (bo_offset=n)
231	*
232	* 0 a 2
233	* new: |-----------| (bo_offset=n)
234	*
235	* .. note::
236	* We expect to see the same result for a request with a different BO
237	* and/or non-contiguous BO offset.
238	*
239	*
240	* 5) Requested mapping's range is a left aligned subset of the existing one,
241	* but backed by a different BO. Hence, map the requested mapping and split
242	* the existing one adjusting its BO offset.
243	*
244	* ::
245	*
246	* 0 a 2
247	* old: |-----------| (bo_offset=n)
248	*
249	* 0 b 1
250	* req: |-----| (bo_offset=n)
251	*
252	* 0 b 1 a' 2
253	* new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1)
254	*
255	* .. note::
256	* We expect to see the same result for a request with a different BO
257	* and/or non-contiguous BO offset.
258	*
259	*
260	* 6) Existent mapping is a superset of the requested mapping. Split it up, but
261	* indicate that the backing PTEs could be kept.
262	*
263	* ::
264	*
265	* 0 a 2
266	* old: |-----------| (bo_offset=n)
267	*
268	* 0 a 1
269	* req: |-----| (bo_offset=n)
270	*
271	* 0 a 1 a' 2
272	* new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1)
273	*
274	*
275	* 7) Requested mapping's range is a right aligned subset of the existing one,
276	* but backed by a different BO. Hence, map the requested mapping and split
277	* the existing one, without adjusting the BO offset.
278	*
279	* ::
280	*
281	* 0 a 2
282	* old: |-----------| (bo_offset=n)
283	*
284	* 1 b 2
285	* req: |-----| (bo_offset=m)
286	*
287	* 0 a 1 b 2
288	* new: |-----|-----| (a.bo_offset=n,b.bo_offset=m)
289	*
290	*
291	* 8) Existent mapping is a superset of the requested mapping. Split it up, but
292	* indicate that the backing PTEs could be kept.
293	*
294	* ::
295	*
296	* 0 a 2
297	* old: |-----------| (bo_offset=n)
298	*
299	* 1 a 2
300	* req: |-----| (bo_offset=n+1)
301	*
302	* 0 a' 1 a 2
303	* new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1)
304	*
305	*
306	* 9) Existent mapping is overlapped at the end by the requested mapping backed
307	* by a different BO. Hence, map the requested mapping and split up the
308	* existing one, without adjusting the BO offset.
309	*
310	* ::
311	*
312	* 0 a 2
313	* old: |-----------| (bo_offset=n)
314	*
315	* 1 b 3
316	* req: |-----------| (bo_offset=m)
317	*
318	* 0 a 1 b 3
319	* new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m)
320	*
321	*
322	* 10) Existent mapping is overlapped by the requested mapping, both having the
323	* same backing BO with a contiguous offset. Indicate the backing PTEs of
324	* the old mapping could be kept.
325	*
326	* ::
327	*
328	* 0 a 2
329	* old: |-----------| (bo_offset=n)
330	*
331	* 1 a 3
332	* req: |-----------| (bo_offset=n+1)
333	*
334	* 0 a' 1 a 3
335	* new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1)
336	*
337	*
338	* 11) Requested mapping's range is a centered subset of the existing one
339	* having a different backing BO. Hence, map the requested mapping and split
340	* up the existing one in two mappings, adjusting the BO offset of the right
341	* one accordingly.
342	*
343	* ::
344	*
345	* 0 a 3
346	* old: |-----------------| (bo_offset=n)
347	*
348	* 1 b 2
349	* req: |-----| (bo_offset=m)
350	*
351	* 0 a 1 b 2 a' 3
352	* new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2)
353	*
354	*
355	* 12) Requested mapping is a contiguous subset of the existing one. Split it
356	* up, but indicate that the backing PTEs could be kept.
357	*
358	* ::
359	*
360	* 0 a 3
361	* old: |-----------------| (bo_offset=n)
362	*
363	* 1 a 2
364	* req: |-----| (bo_offset=n+1)
365	*
366	* 0 a' 1 a 2 a'' 3
367	* old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2)
368	*
369	*
370	* 13) Existent mapping is a right aligned subset of the requested one, hence
371	* replace the existing one.
372	*
373	* ::
374	*
375	* 1 a 2
376	* old: |-----| (bo_offset=n+1)
377	*
378	* 0 a 2
379	* req: |-----------| (bo_offset=n)
380	*
381	* 0 a 2
382	* new: |-----------| (bo_offset=n)
383	*
384	* .. note::
385	* We expect to see the same result for a request with a different bo
386	* and/or non-contiguous bo_offset.
387	*
388	*
389	* 14) Existent mapping is a centered subset of the requested one, hence
390	* replace the existing one.
391	*
392	* ::
393	*
394	* 1 a 2
395	* old: |-----| (bo_offset=n+1)
396	*
397	* 0 a 3
398	* req: |----------------| (bo_offset=n)
399	*
400	* 0 a 3
401	* new: |----------------| (bo_offset=n)
402	*
403	* .. note::
404	* We expect to see the same result for a request with a different bo
405	* and/or non-contiguous bo_offset.
406	*
407	*
408	* 15) Existent mappings is overlapped at the beginning by the requested mapping
409	* backed by a different BO. Hence, map the requested mapping and split up
410	* the existing one, adjusting its BO offset accordingly.
411	*
412	* ::
413	*
414	* 1 a 3
415	* old: |-----------| (bo_offset=n)
416	*
417	* 0 b 2
418	* req: |-----------| (bo_offset=m)
419	*
420	* 0 b 2 a' 3
421	* new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2)
422	*/
423
424	/**
425	* DOC: Madvise Logic - Splitting and Traversal
426	*
427	* This logic handles GPU VA range updates by generating remap and map operations
428	* without performing unmaps or merging existing mappings.
429	*
430	* 1) The requested range lies entirely within a single drm_gpuva. The logic splits
431	* the existing mapping at the start and end boundaries and inserts a new map.
432	*
433	* ::
434	* a start end b
435	* pre: |-----------------------|
436	* drm_gpuva1
437	*
438	* a start end b
439	* new: |-----|=========|-------|
440	* remap map remap
441	*
442	* one REMAP and one MAP : Same behaviour as SPLIT and MERGE
443	*
444	* 2) The requested range spans multiple drm_gpuva regions. The logic traverses
445	* across boundaries, remapping the start and end segments, and inserting two
446	* map operations to cover the full range.
447	*
448	* :: a start b c end d
449	* pre: |------------------|--------------|------------------|
450	* drm_gpuva1 drm_gpuva2 drm_gpuva3
451	*
452	* a start b c end d
453	* new: |-------|==========|--------------|========|---------|
454	* remap1 map1 drm_gpuva2 map2 remap2
455	*
456	* two REMAPS and two MAPS
457	*
458	* 3) Either start or end lies within a drm_gpuva. A single remap and map operation
459	* are generated to update the affected portion.
460	*
461	*
462	* :: a/start b c end d
463	* pre: |------------------|--------------|------------------|
464	* drm_gpuva1 drm_gpuva2 drm_gpuva3
465	*
466	* a/start b c end d
467	* new: |------------------|--------------|========|---------|
468	* drm_gpuva1 drm_gpuva2 map1 remap1
469	*
470	* :: a start b c/end d
471	* pre: |------------------|--------------|------------------|
472	* drm_gpuva1 drm_gpuva2 drm_gpuva3
473	*
474	* a start b c/end d
475	* new: |-------|==========|--------------|------------------|
476	* remap1 map1 drm_gpuva2 drm_gpuva3
477	*
478	* one REMAP and one MAP
479	*
480	* 4) Both start and end align with existing drm_gpuva boundaries. No operations
481	* are needed as the range is already covered.
482	*
483	* 5) No existing drm_gpuvas. No operations.
484	*
485	* Unlike drm_gpuvm_sm_map_ops_create, this logic avoids unmaps and merging,
486	* focusing solely on remap and map operations for efficient traversal and update.
487	*/
488
489	/**
490	* DOC: Locking
491	*
492	* In terms of managing &drm_gpuva entries DRM GPUVM does not take care of
493	* locking itself, it is the drivers responsibility to take care about locking.
494	* Drivers might want to protect the following operations: inserting, removing
495	* and iterating &drm_gpuva objects as well as generating all kinds of
496	* operations, such as split / merge or prefetch.
497	*
498	* DRM GPUVM also does not take care of the locking of the backing
499	* &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by
500	* itself; drivers are responsible to enforce mutual exclusion using either the
501	* GEMs dma_resv lock or the GEMs gpuva.lock mutex.
502	*
503	* However, DRM GPUVM contains lockdep checks to ensure callers of its API hold
504	* the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed
505	* by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also
506	* drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put().
507	*
508	* The latter is required since on creation and destruction of a &drm_gpuvm_bo
509	* the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list.
510	* Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and
511	* &drm_gem_object must be able to observe previous creations and destructions
512	* of &drm_gpuvm_bos in order to keep instances unique.
513	*
514	* The &drm_gpuvm's lists for keeping track of external and evicted objects are
515	* protected against concurrent insertion / removal and iteration internally.
516	*
517	* However, drivers still need ensure to protect concurrent calls to functions
518	* iterating those lists, namely drm_gpuvm_prepare_objects() and
519	* drm_gpuvm_validate().
520	*
521	* Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate
522	* that the corresponding &dma_resv locks are held in order to protect the
523	* lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and
524	* the corresponding lockdep checks are enabled. This is an optimization for
525	* drivers which are capable of taking the corresponding &dma_resv locks and
526	* hence do not require internal locking.
527	*/
528
529	/**
530	* DOC: Examples
531	*
532	* This section gives two examples on how to let the DRM GPUVA Manager generate
533	* &drm_gpuva_op in order to satisfy a given map or unmap request and how to
534	* make use of them.
535	*
536	* The below code is strictly limited to illustrate the generic usage pattern.
537	* To maintain simplicity, it doesn't make use of any abstractions for common
538	* code, different (asynchronous) stages with fence signalling critical paths,
539	* any other helpers or error handling in terms of freeing memory and dropping
540	* previously taken locks.
541	*
542	* 1) Obtain a list of &drm_gpuva_op to create a new mapping::
543	*
544	* // Allocates a new &drm_gpuva.
545	* struct drm_gpuva * driver_gpuva_alloc(void);
546	*
547	* // Typically drivers would embed the &drm_gpuvm and &drm_gpuva
548	* // structure in individual driver structures and lock the dma-resv with
549	* // drm_exec or similar helpers.
550	* int driver_mapping_create(struct drm_gpuvm *gpuvm,
551	* u64 addr, u64 range,
552	* struct drm_gem_object *obj, u64 offset)
553	* {
554	* struct drm_gpuvm_map_req map_req = {
555	* .map.va.addr = addr,
556	* .map.va.range = range,
557	* .map.gem.obj = obj,
558	* .map.gem.offset = offset,
559	* };
560	* struct drm_gpuva_ops *ops;
561	* struct drm_gpuva_op *op
562	* struct drm_gpuvm_bo *vm_bo;
563	*
564	* driver_lock_va_space();
565	* ops = drm_gpuvm_sm_map_ops_create(gpuvm, &map_req);
566	* if (IS_ERR(ops))
567	* return PTR_ERR(ops);
568	*
569	* vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj);
570	* if (IS_ERR(vm_bo))
571	* return PTR_ERR(vm_bo);
572	*
573	* drm_gpuva_for_each_op(op, ops) {
574	* struct drm_gpuva *va;
575	*
576	* switch (op->op) {
577	* case DRM_GPUVA_OP_MAP:
578	* va = driver_gpuva_alloc();
579	* if (!va)
580	* ; // unwind previous VA space updates,
581	* // free memory and unlock
582	*
583	* driver_vm_map();
584	* drm_gpuva_map(gpuvm, va, &op->map);
585	* drm_gpuva_link(va, vm_bo);
586	*
587	* break;
588	* case DRM_GPUVA_OP_REMAP: {
589	* struct drm_gpuva *prev = NULL, *next = NULL;
590	*
591	* va = op->remap.unmap->va;
592	*
593	* if (op->remap.prev) {
594	* prev = driver_gpuva_alloc();
595	* if (!prev)
596	* ; // unwind previous VA space
597	* // updates, free memory and
598	* // unlock
599	* }
600	*
601	* if (op->remap.next) {
602	* next = driver_gpuva_alloc();
603	* if (!next)
604	* ; // unwind previous VA space
605	* // updates, free memory and
606	* // unlock
607	* }
608	*
609	* driver_vm_remap();
610	* drm_gpuva_remap(prev, next, &op->remap);
611	*
612	* if (prev)
613	* drm_gpuva_link(prev, va->vm_bo);
614	* if (next)
615	* drm_gpuva_link(next, va->vm_bo);
616	* drm_gpuva_unlink(va);
617	*
618	* break;
619	* }
620	* case DRM_GPUVA_OP_UNMAP:
621	* va = op->unmap->va;
622	*
623	* driver_vm_unmap();
624	* drm_gpuva_unlink(va);
625	* drm_gpuva_unmap(&op->unmap);
626	*
627	* break;
628	* default:
629	* break;
630	* }
631	* }
632	* drm_gpuvm_bo_put(vm_bo);
633	* driver_unlock_va_space();
634	*
635	* return 0;
636	* }
637	*
638	* 2) Receive a callback for each &drm_gpuva_op to create a new mapping::
639	*
640	* struct driver_context {
641	* struct drm_gpuvm *gpuvm;
642	* struct drm_gpuvm_bo *vm_bo;
643	* struct drm_gpuva *new_va;
644	* struct drm_gpuva *prev_va;
645	* struct drm_gpuva *next_va;
646	* };
647	*
648	* // ops to pass to drm_gpuvm_init()
649	* static const struct drm_gpuvm_ops driver_gpuvm_ops = {
650	* .sm_step_map = driver_gpuva_map,
651	* .sm_step_remap = driver_gpuva_remap,
652	* .sm_step_unmap = driver_gpuva_unmap,
653	* };
654	*
655	* // Typically drivers would embed the &drm_gpuvm and &drm_gpuva
656	* // structure in individual driver structures and lock the dma-resv with
657	* // drm_exec or similar helpers.
658	* int driver_mapping_create(struct drm_gpuvm *gpuvm,
659	* u64 addr, u64 range,
660	* struct drm_gem_object *obj, u64 offset)
661	* {
662	* struct driver_context ctx;
663	* struct drm_gpuvm_bo *vm_bo;
664	* struct drm_gpuva_ops *ops;
665	* struct drm_gpuva_op *op;
666	* int ret = 0;
667	*
668	* ctx.gpuvm = gpuvm;
669	*
670	* ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL);
671	* ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL);
672	* ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL);
673	* ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
674	* if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) {
675	* ret = -ENOMEM;
676	* goto out;
677	* }
678	*
679	* // Typically protected with a driver specific GEM gpuva lock
680	* // used in the fence signaling path for drm_gpuva_link() and
681	* // drm_gpuva_unlink(), hence pre-allocate.
682	* ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo);
683	*
684	* driver_lock_va_space();
685	* ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset);
686	* driver_unlock_va_space();
687	*
688	* out:
689	* drm_gpuvm_bo_put(ctx.vm_bo);
690	* kfree(ctx.new_va);
691	* kfree(ctx.prev_va);
692	* kfree(ctx.next_va);
693	* return ret;
694	* }
695	*
696	* int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx)
697	* {
698	* struct driver_context *ctx = __ctx;
699	*
700	* drm_gpuva_map(ctx->vm, ctx->new_va, &op->map);
701	*
702	* drm_gpuva_link(ctx->new_va, ctx->vm_bo);
703	*
704	* // prevent the new GPUVA from being freed in
705	* // driver_mapping_create()
706	* ctx->new_va = NULL;
707	*
708	* return 0;
709	* }
710	*
711	* int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx)
712	* {
713	* struct driver_context *ctx = __ctx;
714	* struct drm_gpuva *va = op->remap.unmap->va;
715	*
716	* drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap);
717	*
718	* if (op->remap.prev) {
719	* drm_gpuva_link(ctx->prev_va, va->vm_bo);
720	* ctx->prev_va = NULL;
721	* }
722	*
723	* if (op->remap.next) {
724	* drm_gpuva_link(ctx->next_va, va->vm_bo);
725	* ctx->next_va = NULL;
726	* }
727	*
728	* drm_gpuva_unlink(va);
729	* kfree(va);
730	*
731	* return 0;
732	* }
733	*
734	* int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx)
735	* {
736	* drm_gpuva_unlink(op->unmap.va);
737	* drm_gpuva_unmap(&op->unmap);
738	* kfree(op->unmap.va);
739	*
740	* return 0;
741	* }
742	*/
743
744	/**
745	* get_next_vm_bo_from_list() - get the next vm_bo element
746	* @__gpuvm: the &drm_gpuvm
747	* @__list_name: the name of the list we're iterating on
748	* @__local_list: a pointer to the local list used to store already iterated items
749	* @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list()
750	*
751	* This helper is here to provide lockless list iteration. Lockless as in, the
752	* iterator releases the lock immediately after picking the first element from
753	* the list, so list insertion and deletion can happen concurrently.
754	*
755	* Elements popped from the original list are kept in a local list, so removal
756	* and is_empty checks can still happen while we're iterating the list.
757	*/
758	#define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo) \
759	({ \
760	struct drm_gpuvm_bo *__vm_bo = NULL; \
761	\
762	drm_gpuvm_bo_put(__prev_vm_bo); \
763	\
764	spin_lock(&(__gpuvm)->__list_name.lock); \
765	if (!(__gpuvm)->__list_name.local_list) \
766	(__gpuvm)->__list_name.local_list = __local_list; \
767	else \
768	drm_WARN_ON((__gpuvm)->drm, \
769	(__gpuvm)->__list_name.local_list != __local_list); \
770	\
771	while (!list_empty(&(__gpuvm)->__list_name.list)) { \
772	__vm_bo = list_first_entry(&(__gpuvm)->__list_name.list, \
773	struct drm_gpuvm_bo, \
774	list.entry.__list_name); \
775	if (kref_get_unless_zero(&__vm_bo->kref)) { \
776	list_move_tail(&(__vm_bo)->list.entry.__list_name, \
777	__local_list); \
778	break; \
779	} else { \
780	list_del_init(&(__vm_bo)->list.entry.__list_name); \
781	__vm_bo = NULL; \
782	} \
783	} \
784	spin_unlock(&(__gpuvm)->__list_name.lock); \
785	\
786	__vm_bo; \
787	})
788
789	/**
790	* for_each_vm_bo_in_list() - internal vm_bo list iterator
791	* @__gpuvm: the &drm_gpuvm
792	* @__list_name: the name of the list we're iterating on
793	* @__local_list: a pointer to the local list used to store already iterated items
794	* @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step
795	*
796	* This helper is here to provide lockless list iteration. Lockless as in, the
797	* iterator releases the lock immediately after picking the first element from the
798	* list, hence list insertion and deletion can happen concurrently.
799	*
800	* It is not allowed to re-assign the vm_bo pointer from inside this loop.
801	*
802	* Typical use:
803	*
804	* struct drm_gpuvm_bo *vm_bo;
805	* LIST_HEAD(my_local_list);
806	*
807	* ret = 0;
808	* for_each_vm_bo_in_list(gpuvm, <list_name>, &my_local_list, vm_bo) {
809	* ret = do_something_with_vm_bo(..., vm_bo);
810	* if (ret)
811	* break;
812	* }
813	* // Drop ref in case we break out of the loop.
814	* drm_gpuvm_bo_put(vm_bo);
815	* restore_vm_bo_list(gpuvm, <list_name>, &my_local_list);
816	*
817	*
818	* Only used for internal list iterations, not meant to be exposed to the outside
819	* world.
820	*/
821	#define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo) \
822	for (__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \
823	__local_list, NULL); \
824	__vm_bo; \
825	__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \
826	__local_list, __vm_bo))
827
828	static void
829	__restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock,
830	struct list_head *list, struct list_head **local_list)
831	{
832	/* Merge back the two lists, moving local list elements to the
833	* head to preserve previous ordering, in case it matters.
834	*/
835	spin_lock(lock);
836	if (*local_list) {
837	list_splice(list: *local_list, head: list);
838	*local_list = NULL;
839	}
840	spin_unlock(lock);
841	}
842
843	/**
844	* restore_vm_bo_list() - move vm_bo elements back to their original list
845	* @__gpuvm: the &drm_gpuvm
846	* @__list_name: the name of the list we're iterating on
847	*
848	* When we're done iterating a vm_bo list, we should call restore_vm_bo_list()
849	* to restore the original state and let new iterations take place.
850	*/
851	#define restore_vm_bo_list(__gpuvm, __list_name) \
852	__restore_vm_bo_list((__gpuvm), &(__gpuvm)->__list_name.lock, \
853	&(__gpuvm)->__list_name.list, \
854	&(__gpuvm)->__list_name.local_list)
855
856	static void
857	cond_spin_lock(spinlock_t *lock, bool cond)
858	{
859	if (cond)
860	spin_lock(lock);
861	}
862
863	static void
864	cond_spin_unlock(spinlock_t *lock, bool cond)
865	{
866	if (cond)
867	spin_unlock(lock);
868	}
869
870	static void
871	__drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock,
872	struct list_head *entry, struct list_head *list)
873	{
874	cond_spin_lock(lock, cond: !!lock);
875	if (list_empty(head: entry))
876	list_add_tail(new: entry, head: list);
877	cond_spin_unlock(lock, cond: !!lock);
878	}
879
880	/**
881	* drm_gpuvm_bo_is_zombie() - check whether this vm_bo is scheduled for cleanup
882	* @vm_bo: the &drm_gpuvm_bo
883	*
884	* When a vm_bo is scheduled for cleanup using the bo_defer list, it is not
885	* immediately removed from the evict and extobj lists. Therefore, anyone
886	* iterating these lists should skip entries that are being destroyed.
887	*
888	* Checking the refcount without incrementing it is okay as long as the lock
889	* protecting the evict/extobj list is held for as long as you are using the
890	* vm_bo, because even if the refcount hits zero while you are using it, freeing
891	* the vm_bo requires taking the list's lock.
892	*
893	* Zombie entries can be observed on the evict and extobj lists regardless of
894	* whether DRM_GPUVM_RESV_PROTECTED is used, but they remain on the lists for a
895	* longer time when the resv lock is used because we can't take the resv lock
896	* during run_job() in immediate mode, meaning that they need to remain on the
897	* lists until drm_gpuvm_bo_deferred_cleanup() is called.
898	*/
899	static bool
900	drm_gpuvm_bo_is_zombie(struct drm_gpuvm_bo *vm_bo)
901	{
902	return !kref_read(kref: &vm_bo->kref);
903	}
904
905	/**
906	* drm_gpuvm_bo_list_add() - insert a vm_bo into the given list
907	* @__vm_bo: the &drm_gpuvm_bo
908	* @__list_name: the name of the list to insert into
909	* @__lock: whether to lock with the internal spinlock
910	*
911	* Inserts the given @__vm_bo into the list specified by @__list_name.
912	*/
913	#define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock) \
914	__drm_gpuvm_bo_list_add((__vm_bo)->vm, \
915	__lock ? &(__vm_bo)->vm->__list_name.lock : \
916	NULL, \
917	&(__vm_bo)->list.entry.__list_name, \
918	&(__vm_bo)->vm->__list_name.list)
919
920	static void
921	__drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock,
922	struct list_head *entry, bool init)
923	{
924	cond_spin_lock(lock, cond: !!lock);
925	if (init) {
926	if (!list_empty(head: entry))
927	list_del_init(entry);
928	} else {
929	list_del(entry);
930	}
931	cond_spin_unlock(lock, cond: !!lock);
932	}
933
934	/**
935	* drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list
936	* @__vm_bo: the &drm_gpuvm_bo
937	* @__list_name: the name of the list to insert into
938	* @__lock: whether to lock with the internal spinlock
939	*
940	* Removes the given @__vm_bo from the list specified by @__list_name.
941	*/
942	#define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock) \
943	__drm_gpuvm_bo_list_del((__vm_bo)->vm, \
944	__lock ? &(__vm_bo)->vm->__list_name.lock : \
945	NULL, \
946	&(__vm_bo)->list.entry.__list_name, \
947	true)
948
949	/**
950	* drm_gpuvm_bo_list_del() - remove a vm_bo from the given list
951	* @__vm_bo: the &drm_gpuvm_bo
952	* @__list_name: the name of the list to insert into
953	* @__lock: whether to lock with the internal spinlock
954	*
955	* Removes the given @__vm_bo from the list specified by @__list_name.
956	*/
957	#define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock) \
958	__drm_gpuvm_bo_list_del((__vm_bo)->vm, \
959	__lock ? &(__vm_bo)->vm->__list_name.lock : \
960	NULL, \
961	&(__vm_bo)->list.entry.__list_name, \
962	false)
963
964	#define to_drm_gpuva(__node) container_of((__node), struct drm_gpuva, rb.node)
965
966	#define GPUVA_START(node) ((node)->va.addr)
967	#define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)
968
969	/* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain
970	* about this.
971	*/
972	INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last,
973	GPUVA_START, GPUVA_LAST, static __maybe_unused,
974	drm_gpuva_it)
975
976	static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm,
977	struct drm_gpuva *va);
978	static void __drm_gpuva_remove(struct drm_gpuva *va);
979
980	static bool
981	drm_gpuvm_check_overflow(u64 addr, u64 range)
982	{
983	u64 end;
984
985	return check_add_overflow(addr, range, &end);
986	}
987
988	static bool
989	drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
990	{
991	return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range),
992	"GPUVA address limited to %zu bytes.\n", sizeof(addr));
993	}
994
995	static bool
996	drm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
997	{
998	u64 end = addr + range;
999	u64 mm_start = gpuvm->mm_start;
1000	u64 mm_end = mm_start + gpuvm->mm_range;
1001
1002	return addr >= mm_start && end <= mm_end;
1003	}
1004
1005	static bool
1006	drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
1007	{
1008	u64 end = addr + range;
1009	u64 kstart = gpuvm->kernel_alloc_node.va.addr;
1010	u64 krange = gpuvm->kernel_alloc_node.va.range;
1011	u64 kend = kstart + krange;
1012
1013	return krange && addr < kend && kstart < end;
1014	}
1015
1016	/**
1017	* drm_gpuvm_range_valid() - checks whether the given range is valid for the
1018	* given &drm_gpuvm
1019	* @gpuvm: the GPUVM to check the range for
1020	* @addr: the base address
1021	* @range: the range starting from the base address
1022	*
1023	* Checks whether the range is within the GPUVM's managed boundaries.
1024	*
1025	* Returns: true for a valid range, false otherwise
1026	*/
1027	bool
1028	drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,
1029	u64 addr, u64 range)
1030	{
1031	return !drm_gpuvm_check_overflow(addr, range) &&
1032	drm_gpuvm_in_mm_range(gpuvm, addr, range) &&
1033	!drm_gpuvm_in_kernel_node(gpuvm, addr, range);
1034	}
1035	EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid);
1036
1037	static void
1038	drm_gpuvm_gem_object_free(struct drm_gem_object *obj)
1039	{
1040	drm_gem_object_release(obj);
1041	kfree(objp: obj);
1042	}
1043
1044	static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = {
1045	.free = drm_gpuvm_gem_object_free,
1046	};
1047
1048	/**
1049	* drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object
1050	* @drm: the drivers &drm_device
1051	*
1052	* Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in
1053	* order to serve as root GEM object providing the &drm_resv shared across
1054	* &drm_gem_objects local to a single GPUVM.
1055	*
1056	* Returns: the &drm_gem_object on success, NULL on failure
1057	*/
1058	struct drm_gem_object *
1059	drm_gpuvm_resv_object_alloc(struct drm_device *drm)
1060	{
1061	struct drm_gem_object *obj;
1062
1063	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
1064	if (!obj)
1065	return NULL;
1066
1067	obj->funcs = &drm_gpuvm_object_funcs;
1068	drm_gem_private_object_init(dev: drm, obj, size: 0);
1069
1070	return obj;
1071	}
1072	EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);
1073
1074	/**
1075	* drm_gpuvm_init() - initialize a &drm_gpuvm
1076	* @gpuvm: pointer to the &drm_gpuvm to initialize
1077	* @name: the name of the GPU VA space
1078	* @flags: the &drm_gpuvm_flags for this GPUVM
1079	* @drm: the &drm_device this VM resides in
1080	* @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv
1081	* @start_offset: the start offset of the GPU VA space
1082	* @range: the size of the GPU VA space
1083	* @reserve_offset: the start of the kernel reserved GPU VA area
1084	* @reserve_range: the size of the kernel reserved GPU VA area
1085	* @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap
1086	*
1087	* The &drm_gpuvm must be initialized with this function before use.
1088	*
1089	* Note that @gpuvm must be cleared to 0 before calling this function. The given
1090	* &name is expected to be managed by the surrounding driver structures.
1091	*/
1092	void
1093	drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
1094	enum drm_gpuvm_flags flags,
1095	struct drm_device *drm,
1096	struct drm_gem_object *r_obj,
1097	u64 start_offset, u64 range,
1098	u64 reserve_offset, u64 reserve_range,
1099	const struct drm_gpuvm_ops *ops)
1100	{
1101	gpuvm->rb.tree = RB_ROOT_CACHED;
1102	INIT_LIST_HEAD(list: &gpuvm->rb.list);
1103
1104	INIT_LIST_HEAD(list: &gpuvm->extobj.list);
1105	spin_lock_init(&gpuvm->extobj.lock);
1106
1107	INIT_LIST_HEAD(list: &gpuvm->evict.list);
1108	spin_lock_init(&gpuvm->evict.lock);
1109
1110	init_llist_head(list: &gpuvm->bo_defer);
1111
1112	kref_init(kref: &gpuvm->kref);
1113
1114	gpuvm->name = name ? name : "unknown";
1115	gpuvm->flags = flags;
1116	gpuvm->ops = ops;
1117	gpuvm->drm = drm;
1118	gpuvm->r_obj = r_obj;
1119
1120	drm_gem_object_get(obj: r_obj);
1121
1122	drm_gpuvm_warn_check_overflow(gpuvm, addr: start_offset, range);
1123	gpuvm->mm_start = start_offset;
1124	gpuvm->mm_range = range;
1125
1126	memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva));
1127	if (reserve_range) {
1128	gpuvm->kernel_alloc_node.va.addr = reserve_offset;
1129	gpuvm->kernel_alloc_node.va.range = reserve_range;
1130
1131	if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset,
1132	reserve_range)))
1133	__drm_gpuva_insert(gpuvm, va: &gpuvm->kernel_alloc_node);
1134	}
1135	}
1136	EXPORT_SYMBOL_GPL(drm_gpuvm_init);
1137
1138	static void
1139	drm_gpuvm_fini(struct drm_gpuvm *gpuvm)
1140	{
1141	gpuvm->name = NULL;
1142
1143	if (gpuvm->kernel_alloc_node.va.range)
1144	__drm_gpuva_remove(va: &gpuvm->kernel_alloc_node);
1145
1146	drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root),
1147	"GPUVA tree is not empty, potentially leaking memory.\n");
1148
1149	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list),
1150	"Extobj list should be empty.\n");
1151	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list),
1152	"Evict list should be empty.\n");
1153	drm_WARN(gpuvm->drm, !llist_empty(&gpuvm->bo_defer),
1154	"VM BO cleanup list should be empty.\n");
1155
1156	drm_gem_object_put(obj: gpuvm->r_obj);
1157	}
1158
1159	static void
1160	drm_gpuvm_free(struct kref *kref)
1161	{
1162	struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref);
1163
1164	drm_gpuvm_fini(gpuvm);
1165
1166	if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free))
1167	return;
1168
1169	gpuvm->ops->vm_free(gpuvm);
1170	}
1171
1172	/**
1173	* drm_gpuvm_put() - drop a struct drm_gpuvm reference
1174	* @gpuvm: the &drm_gpuvm to release the reference of
1175	*
1176	* This releases a reference to @gpuvm.
1177	*
1178	* This function may be called from atomic context.
1179	*/
1180	void
1181	drm_gpuvm_put(struct drm_gpuvm *gpuvm)
1182	{
1183	if (gpuvm)
1184	kref_put(kref: &gpuvm->kref, release: drm_gpuvm_free);
1185	}
1186	EXPORT_SYMBOL_GPL(drm_gpuvm_put);
1187
1188	static int
1189	exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj,
1190	unsigned int num_fences)
1191	{
1192	return num_fences ? drm_exec_prepare_obj(exec, obj, num_fences) :
1193	drm_exec_lock_obj(exec, obj);
1194	}
1195
1196	/**
1197	* drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv
1198	* @gpuvm: the &drm_gpuvm
1199	* @exec: the &drm_exec context
1200	* @num_fences: the amount of &dma_fences to reserve
1201	*
1202	* Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object; if
1203	* @num_fences is zero drm_exec_lock_obj() is called instead.
1204	*
1205	* Using this function directly, it is the drivers responsibility to call
1206	* drm_exec_init() and drm_exec_fini() accordingly.
1207	*
1208	* Returns: 0 on success, negative error code on failure.
1209	*/
1210	int
1211	drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm,
1212	struct drm_exec *exec,
1213	unsigned int num_fences)
1214	{
1215	return exec_prepare_obj(exec, obj: gpuvm->r_obj, num_fences);
1216	}
1217	EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_vm);
1218
1219	static int
1220	__drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
1221	struct drm_exec *exec,
1222	unsigned int num_fences)
1223	{
1224	struct drm_gpuvm_bo *vm_bo;
1225	LIST_HEAD(extobjs);
1226	int ret = 0;
1227
1228	for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) {
1229	ret = exec_prepare_obj(exec, obj: vm_bo->obj, num_fences);
1230	if (ret)
1231	break;
1232	}
1233	/* Drop ref in case we break out of the loop. */
1234	drm_gpuvm_bo_put(vm_bo);
1235	restore_vm_bo_list(gpuvm, extobj);
1236
1237	return ret;
1238	}
1239
1240	static int
1241	drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm,
1242	struct drm_exec *exec,
1243	unsigned int num_fences)
1244	{
1245	struct drm_gpuvm_bo *vm_bo;
1246	int ret = 0;
1247
1248	drm_gpuvm_resv_assert_held(gpuvm);
1249	list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) {
1250	if (drm_gpuvm_bo_is_zombie(vm_bo))
1251	continue;
1252
1253	ret = exec_prepare_obj(exec, obj: vm_bo->obj, num_fences);
1254	if (ret)
1255	break;
1256
1257	if (vm_bo->evicted)
1258	drm_gpuvm_bo_list_add(vm_bo, evict, false);
1259	}
1260
1261	return ret;
1262	}
1263
1264	/**
1265	* drm_gpuvm_prepare_objects() - prepare all associated BOs
1266	* @gpuvm: the &drm_gpuvm
1267	* @exec: the &drm_exec locking context
1268	* @num_fences: the amount of &dma_fences to reserve
1269	*
1270	* Calls drm_exec_prepare_obj() for all &drm_gem_objects the given
1271	* &drm_gpuvm contains mappings of; if @num_fences is zero drm_exec_lock_obj()
1272	* is called instead.
1273	*
1274	* Using this function directly, it is the drivers responsibility to call
1275	* drm_exec_init() and drm_exec_fini() accordingly.
1276	*
1277	* Note: This function is safe against concurrent insertion and removal of
1278	* external objects, however it is not safe against concurrent usage itself.
1279	*
1280	* Drivers need to make sure to protect this case with either an outer VM lock
1281	* or by calling drm_gpuvm_prepare_vm() before this function within the
1282	* drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures
1283	* mutual exclusion.
1284	*
1285	* Returns: 0 on success, negative error code on failure.
1286	*/
1287	int
1288	drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
1289	struct drm_exec *exec,
1290	unsigned int num_fences)
1291	{
1292	if (drm_gpuvm_resv_protected(gpuvm))
1293	return drm_gpuvm_prepare_objects_locked(gpuvm, exec,
1294	num_fences);
1295	else
1296	return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
1297	}
1298	EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects);
1299
1300	/**
1301	* drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range
1302	* @gpuvm: the &drm_gpuvm
1303	* @exec: the &drm_exec locking context
1304	* @addr: the start address within the VA space
1305	* @range: the range to iterate within the VA space
1306	* @num_fences: the amount of &dma_fences to reserve
1307	*
1308	* Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr
1309	* and @addr + @range; if @num_fences is zero drm_exec_lock_obj() is called
1310	* instead.
1311	*
1312	* Returns: 0 on success, negative error code on failure.
1313	*/
1314	int
1315	drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec,
1316	u64 addr, u64 range, unsigned int num_fences)
1317	{
1318	struct drm_gpuva *va;
1319	u64 end = addr + range;
1320	int ret;
1321
1322	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {
1323	struct drm_gem_object *obj = va->gem.obj;
1324
1325	ret = exec_prepare_obj(exec, obj, num_fences);
1326	if (ret)
1327	return ret;
1328	}
1329
1330	return 0;
1331	}
1332	EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_range);
1333
1334	/**
1335	* drm_gpuvm_exec_lock() - lock all dma-resv of all associated BOs
1336	* @vm_exec: the &drm_gpuvm_exec wrapper
1337	*
1338	* Acquires all dma-resv locks of all &drm_gem_objects the given
1339	* &drm_gpuvm contains mappings of.
1340	*
1341	* Additionally, when calling this function with struct drm_gpuvm_exec::extra
1342	* being set the driver receives the given @fn callback to lock additional
1343	* dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers
1344	* would call drm_exec_prepare_obj() from within this callback.
1345	*
1346	* Returns: 0 on success, negative error code on failure.
1347	*/
1348	int
1349	drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec)
1350	{
1351	struct drm_gpuvm *gpuvm = vm_exec->vm;
1352	struct drm_exec *exec = &vm_exec->exec;
1353	unsigned int num_fences = vm_exec->num_fences;
1354	int ret;
1355
1356	drm_exec_init(exec, flags: vm_exec->flags, nr: 0);
1357
1358	drm_exec_until_all_locked(exec) {
1359	ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences);
1360	drm_exec_retry_on_contention(exec);
1361	if (ret)
1362	goto err;
1363
1364	ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
1365	drm_exec_retry_on_contention(exec);
1366	if (ret)
1367	goto err;
1368
1369	if (vm_exec->extra.fn) {
1370	ret = vm_exec->extra.fn(vm_exec);
1371	drm_exec_retry_on_contention(exec);
1372	if (ret)
1373	goto err;
1374	}
1375	}
1376
1377	return 0;
1378
1379	err:
1380	drm_exec_fini(exec);
1381	return ret;
1382	}
1383	EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock);
1384
1385	static int
1386	fn_lock_array(struct drm_gpuvm_exec *vm_exec)
1387	{
1388	struct {
1389	struct drm_gem_object **objs;
1390	unsigned int num_objs;
1391	} *args = vm_exec->extra.priv;
1392
1393	return drm_exec_prepare_array(exec: &vm_exec->exec, objects: args->objs,
1394	num_objects: args->num_objs, num_fences: vm_exec->num_fences);
1395	}
1396
1397	/**
1398	* drm_gpuvm_exec_lock_array() - lock all dma-resv of all associated BOs
1399	* @vm_exec: the &drm_gpuvm_exec wrapper
1400	* @objs: additional &drm_gem_objects to lock
1401	* @num_objs: the number of additional &drm_gem_objects to lock
1402	*
1403	* Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm
1404	* contains mappings of, plus the ones given through @objs.
1405	*
1406	* Returns: 0 on success, negative error code on failure.
1407	*/
1408	int
1409	drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec,
1410	struct drm_gem_object **objs,
1411	unsigned int num_objs)
1412	{
1413	struct {
1414	struct drm_gem_object **objs;
1415	unsigned int num_objs;
1416	} args;
1417
1418	args.objs = objs;
1419	args.num_objs = num_objs;
1420
1421	vm_exec->extra.fn = fn_lock_array;
1422	vm_exec->extra.priv = &args;
1423
1424	return drm_gpuvm_exec_lock(vm_exec);
1425	}
1426	EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_array);
1427
1428	/**
1429	* drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range
1430	* @vm_exec: the &drm_gpuvm_exec wrapper
1431	* @addr: the start address within the VA space
1432	* @range: the range to iterate within the VA space
1433	*
1434	* Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and
1435	* @addr + @range.
1436	*
1437	* Returns: 0 on success, negative error code on failure.
1438	*/
1439	int
1440	drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec,
1441	u64 addr, u64 range)
1442	{
1443	struct drm_gpuvm *gpuvm = vm_exec->vm;
1444	struct drm_exec *exec = &vm_exec->exec;
1445	int ret;
1446
1447	drm_exec_init(exec, flags: vm_exec->flags, nr: 0);
1448
1449	drm_exec_until_all_locked(exec) {
1450	ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range,
1451	vm_exec->num_fences);
1452	drm_exec_retry_on_contention(exec);
1453	if (ret)
1454	goto err;
1455	}
1456
1457	return ret;
1458
1459	err:
1460	drm_exec_fini(exec);
1461	return ret;
1462	}
1463	EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_range);
1464
1465	static int
1466	__drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
1467	{
1468	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1469	struct drm_gpuvm_bo *vm_bo;
1470	LIST_HEAD(evict);
1471	int ret = 0;
1472
1473	for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) {
1474	ret = ops->vm_bo_validate(vm_bo, exec);
1475	if (ret)
1476	break;
1477	}
1478	/* Drop ref in case we break out of the loop. */
1479	drm_gpuvm_bo_put(vm_bo);
1480	restore_vm_bo_list(gpuvm, evict);
1481
1482	return ret;
1483	}
1484
1485	static int
1486	drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
1487	{
1488	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1489	struct drm_gpuvm_bo *vm_bo, *next;
1490	int ret = 0;
1491
1492	drm_gpuvm_resv_assert_held(gpuvm);
1493
1494	list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list,
1495	list.entry.evict) {
1496	if (drm_gpuvm_bo_is_zombie(vm_bo))
1497	continue;
1498
1499	ret = ops->vm_bo_validate(vm_bo, exec);
1500	if (ret)
1501	break;
1502
1503	dma_resv_assert_held(vm_bo->obj->resv);
1504	if (!vm_bo->evicted)
1505	drm_gpuvm_bo_list_del_init(vm_bo, evict, false);
1506	}
1507
1508	return ret;
1509	}
1510
1511	/**
1512	* drm_gpuvm_validate() - validate all BOs marked as evicted
1513	* @gpuvm: the &drm_gpuvm to validate evicted BOs
1514	* @exec: the &drm_exec instance used for locking the GPUVM
1515	*
1516	* Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer
1517	* objects being mapped in the given &drm_gpuvm.
1518	*
1519	* Returns: 0 on success, negative error code on failure.
1520	*/
1521	int
1522	drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
1523	{
1524	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1525
1526	if (unlikely(!ops || !ops->vm_bo_validate))
1527	return -EOPNOTSUPP;
1528
1529	if (drm_gpuvm_resv_protected(gpuvm))
1530	return drm_gpuvm_validate_locked(gpuvm, exec);
1531	else
1532	return __drm_gpuvm_validate(gpuvm, exec);
1533	}
1534	EXPORT_SYMBOL_GPL(drm_gpuvm_validate);
1535
1536	/**
1537	* drm_gpuvm_resv_add_fence - add fence to private and all extobj
1538	* dma-resv
1539	* @gpuvm: the &drm_gpuvm to add a fence to
1540	* @exec: the &drm_exec locking context
1541	* @fence: fence to add
1542	* @private_usage: private dma-resv usage
1543	* @extobj_usage: extobj dma-resv usage
1544	*/
1545	void
1546	drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,
1547	struct drm_exec *exec,
1548	struct dma_fence *fence,
1549	enum dma_resv_usage private_usage,
1550	enum dma_resv_usage extobj_usage)
1551	{
1552	struct drm_gem_object *obj;
1553	unsigned long index;
1554
1555	drm_exec_for_each_locked_object(exec, index, obj) {
1556	dma_resv_assert_held(obj->resv);
1557	dma_resv_add_fence(obj: obj->resv, fence,
1558	usage: drm_gpuvm_is_extobj(gpuvm, obj) ?
1559	extobj_usage : private_usage);
1560	}
1561	}
1562	EXPORT_SYMBOL_GPL(drm_gpuvm_resv_add_fence);
1563
1564	/**
1565	* drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo
1566	* @gpuvm: The &drm_gpuvm the @obj is mapped in.
1567	* @obj: The &drm_gem_object being mapped in the @gpuvm.
1568	*
1569	* If provided by the driver, this function uses the &drm_gpuvm_ops
1570	* vm_bo_alloc() callback to allocate.
1571	*
1572	* Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
1573	*/
1574	struct drm_gpuvm_bo *
1575	drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,
1576	struct drm_gem_object *obj)
1577	{
1578	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1579	struct drm_gpuvm_bo *vm_bo;
1580
1581	if (ops && ops->vm_bo_alloc)
1582	vm_bo = ops->vm_bo_alloc();
1583	else
1584	vm_bo = kzalloc(sizeof(*vm_bo), GFP_KERNEL);
1585
1586	if (unlikely(!vm_bo))
1587	return NULL;
1588
1589	vm_bo->vm = drm_gpuvm_get(gpuvm);
1590	vm_bo->obj = obj;
1591	drm_gem_object_get(obj);
1592
1593	kref_init(kref: &vm_bo->kref);
1594	INIT_LIST_HEAD(list: &vm_bo->list.gpuva);
1595	INIT_LIST_HEAD(list: &vm_bo->list.entry.gem);
1596
1597	INIT_LIST_HEAD(list: &vm_bo->list.entry.extobj);
1598	INIT_LIST_HEAD(list: &vm_bo->list.entry.evict);
1599	init_llist_node(node: &vm_bo->list.entry.bo_defer);
1600
1601	return vm_bo;
1602	}
1603	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create);
1604
1605	/*
1606	* drm_gpuvm_bo_destroy_not_in_lists() - final part of drm_gpuvm_bo cleanup
1607	* @vm_bo: the &drm_gpuvm_bo to destroy
1608	*
1609	* It is illegal to call this method if the @vm_bo is present in the GEMs gpuva
1610	* list, the extobj list, or the evicted list.
1611	*
1612	* Note that this puts a refcount on the GEM object, which may destroy the GEM
1613	* object if the refcount reaches zero. It's illegal for this to happen if the
1614	* caller holds the GEMs gpuva mutex because it would free the mutex.
1615	*/
1616	static void
1617	drm_gpuvm_bo_destroy_not_in_lists(struct drm_gpuvm_bo *vm_bo)
1618	{
1619	struct drm_gpuvm *gpuvm = vm_bo->vm;
1620	const struct drm_gpuvm_ops *ops = gpuvm->ops;
1621	struct drm_gem_object *obj = vm_bo->obj;
1622
1623	if (ops && ops->vm_bo_free)
1624	ops->vm_bo_free(vm_bo);
1625	else
1626	kfree(objp: vm_bo);
1627
1628	drm_gpuvm_put(gpuvm);
1629	drm_gem_object_put(obj);
1630	}
1631
1632	static void
1633	drm_gpuvm_bo_destroy_not_in_lists_kref(struct kref *kref)
1634	{
1635	struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
1636	kref);
1637
1638	drm_gpuvm_bo_destroy_not_in_lists(vm_bo);
1639	}
1640
1641	static void
1642	drm_gpuvm_bo_destroy(struct kref *kref)
1643	{
1644	struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
1645	kref);
1646	struct drm_gpuvm *gpuvm = vm_bo->vm;
1647	bool lock = !drm_gpuvm_resv_protected(gpuvm);
1648
1649	if (!lock)
1650	drm_gpuvm_resv_assert_held(gpuvm);
1651
1652	drm_gpuvm_bo_list_del(vm_bo, extobj, lock);
1653	drm_gpuvm_bo_list_del(vm_bo, evict, lock);
1654
1655	drm_gem_gpuva_assert_lock_held(gpuvm, vm_bo->obj);
1656	list_del(entry: &vm_bo->list.entry.gem);
1657
1658	drm_gpuvm_bo_destroy_not_in_lists(vm_bo);
1659	}
1660
1661	/**
1662	* drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference
1663	* @vm_bo: the &drm_gpuvm_bo to release the reference of
1664	*
1665	* This releases a reference to @vm_bo.
1666	*
1667	* If the reference count drops to zero, the &gpuvm_bo is destroyed, which
1668	* includes removing it from the GEMs gpuva list. Hence, if a call to this
1669	* function can potentially let the reference count drop to zero the caller must
1670	* hold the lock that the GEM uses for its gpuva list (either the GEM's
1671	* dma-resv or gpuva.lock mutex).
1672	*
1673	* This function may only be called from non-atomic context.
1674	*
1675	* Returns: true if vm_bo was destroyed, false otherwise.
1676	*/
1677	bool
1678	drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo)
1679	{
1680	might_sleep();
1681
1682	if (vm_bo)
1683	return !!kref_put(kref: &vm_bo->kref, release: drm_gpuvm_bo_destroy);
1684
1685	return false;
1686	}
1687	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put);
1688
1689	/*
1690	* drm_gpuvm_bo_into_zombie() - called when the vm_bo becomes a zombie due to
1691	* deferred cleanup
1692	*
1693	* If deferred cleanup is used, then this must be called right after the vm_bo
1694	* refcount drops to zero. Must be called with GEM mutex held. After releasing
1695	* the GEM mutex, drm_gpuvm_bo_defer_zombie_cleanup() must be called.
1696	*/
1697	static void
1698	drm_gpuvm_bo_into_zombie(struct kref *kref)
1699	{
1700	struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
1701	kref);
1702
1703	if (!drm_gpuvm_resv_protected(gpuvm: vm_bo->vm)) {
1704	drm_gpuvm_bo_list_del(vm_bo, extobj, true);
1705	drm_gpuvm_bo_list_del(vm_bo, evict, true);
1706	}
1707
1708	list_del(entry: &vm_bo->list.entry.gem);
1709	}
1710
1711	/*
1712	* drm_gpuvm_bo_defer_zombie_cleanup() - adds a new zombie vm_bo to the
1713	* bo_defer list
1714	*
1715	* Called after drm_gpuvm_bo_into_zombie(). GEM mutex must not be held.
1716	*
1717	* It's important that the GEM stays alive for the duration in which we hold
1718	* the mutex, but the instant we add the vm_bo to bo_defer, another thread
1719	* might call drm_gpuvm_bo_deferred_cleanup() and put the GEM. Therefore, to
1720	* avoid kfreeing a mutex we are holding, the GEM mutex must be released
1721	* *before* calling this function.
1722	*/
1723	static void
1724	drm_gpuvm_bo_defer_zombie_cleanup(struct drm_gpuvm_bo *vm_bo)
1725	{
1726	llist_add(new: &vm_bo->list.entry.bo_defer, head: &vm_bo->vm->bo_defer);
1727	}
1728
1729	static void
1730	drm_gpuvm_bo_defer_free(struct kref *kref)
1731	{
1732	struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
1733	kref);
1734
1735	drm_gpuvm_bo_into_zombie(kref);
1736	mutex_unlock(lock: &vm_bo->obj->gpuva.lock);
1737	drm_gpuvm_bo_defer_zombie_cleanup(vm_bo);
1738	}
1739
1740	/**
1741	* drm_gpuvm_bo_put_deferred() - drop a struct drm_gpuvm_bo reference with
1742	* deferred cleanup
1743	* @vm_bo: the &drm_gpuvm_bo to release the reference of
1744	*
1745	* This releases a reference to @vm_bo.
1746	*
1747	* This might take and release the GEMs GPUVA lock. You should call
1748	* drm_gpuvm_bo_deferred_cleanup() later to complete the cleanup process.
1749	*
1750	* Returns: true if vm_bo is being destroyed, false otherwise.
1751	*/
1752	bool
1753	drm_gpuvm_bo_put_deferred(struct drm_gpuvm_bo *vm_bo)
1754	{
1755	if (!vm_bo)
1756	return false;
1757
1758	drm_WARN_ON(vm_bo->vm->drm, !drm_gpuvm_immediate_mode(vm_bo->vm));
1759
1760	return !!kref_put_mutex(kref: &vm_bo->kref,
1761	release: drm_gpuvm_bo_defer_free,
1762	mutex: &vm_bo->obj->gpuva.lock);
1763	}
1764	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put_deferred);
1765
1766	/**
1767	* drm_gpuvm_bo_deferred_cleanup() - clean up BOs in the deferred list
1768	* deferred cleanup
1769	* @gpuvm: the VM to clean up
1770	*
1771	* Cleans up &drm_gpuvm_bo instances in the deferred cleanup list.
1772	*/
1773	void
1774	drm_gpuvm_bo_deferred_cleanup(struct drm_gpuvm *gpuvm)
1775	{
1776	struct drm_gpuvm_bo *vm_bo;
1777	struct llist_node *bo_defer;
1778
1779	bo_defer = llist_del_all(head: &gpuvm->bo_defer);
1780	if (!bo_defer)
1781	return;
1782
1783	if (drm_gpuvm_resv_protected(gpuvm)) {
1784	dma_resv_lock(drm_gpuvm_resv(gpuvm), NULL);
1785	llist_for_each_entry(vm_bo, bo_defer, list.entry.bo_defer) {
1786	drm_gpuvm_bo_list_del(vm_bo, extobj, false);
1787	drm_gpuvm_bo_list_del(vm_bo, evict, false);
1788	}
1789	dma_resv_unlock(drm_gpuvm_resv(gpuvm));
1790	}
1791
1792	while (bo_defer) {
1793	vm_bo = llist_entry(bo_defer, struct drm_gpuvm_bo, list.entry.bo_defer);
1794	bo_defer = bo_defer->next;
1795	drm_gpuvm_bo_destroy_not_in_lists(vm_bo);
1796	}
1797	}
1798	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_deferred_cleanup);
1799
1800	static struct drm_gpuvm_bo *
1801	__drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
1802	struct drm_gem_object *obj)
1803	{
1804	struct drm_gpuvm_bo *vm_bo;
1805
1806	drm_gem_gpuva_assert_lock_held(gpuvm, obj);
1807	drm_gem_for_each_gpuvm_bo(vm_bo, obj)
1808	if (vm_bo->vm == gpuvm)
1809	return vm_bo;
1810
1811	return NULL;
1812	}
1813
1814	/**
1815	* drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given
1816	* &drm_gpuvm and &drm_gem_object
1817	* @gpuvm: The &drm_gpuvm the @obj is mapped in.
1818	* @obj: The &drm_gem_object being mapped in the @gpuvm.
1819	*
1820	* Find the &drm_gpuvm_bo representing the combination of the given
1821	* &drm_gpuvm and &drm_gem_object. If found, increases the reference
1822	* count of the &drm_gpuvm_bo accordingly.
1823	*
1824	* Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
1825	*/
1826	struct drm_gpuvm_bo *
1827	drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
1828	struct drm_gem_object *obj)
1829	{
1830	struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj);
1831
1832	return vm_bo ? drm_gpuvm_bo_get(vm_bo) : NULL;
1833	}
1834	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_find);
1835
1836	/**
1837	* drm_gpuvm_bo_obtain() - obtains an instance of the &drm_gpuvm_bo for the
1838	* given &drm_gpuvm and &drm_gem_object
1839	* @gpuvm: The &drm_gpuvm the @obj is mapped in.
1840	* @obj: The &drm_gem_object being mapped in the @gpuvm.
1841	*
1842	* Find the &drm_gpuvm_bo representing the combination of the given
1843	* &drm_gpuvm and &drm_gem_object. If found, increases the reference
1844	* count of the &drm_gpuvm_bo accordingly. If not found, allocates a new
1845	* &drm_gpuvm_bo.
1846	*
1847	* A new &drm_gpuvm_bo is added to the GEMs gpuva list.
1848	*
1849	* Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure
1850	*/
1851	struct drm_gpuvm_bo *
1852	drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm,
1853	struct drm_gem_object *obj)
1854	{
1855	struct drm_gpuvm_bo *vm_bo;
1856
1857	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
1858	if (vm_bo)
1859	return vm_bo;
1860
1861	vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
1862	if (!vm_bo)
1863	return ERR_PTR(error: -ENOMEM);
1864
1865	drm_gem_gpuva_assert_lock_held(gpuvm, obj);
1866	list_add_tail(new: &vm_bo->list.entry.gem, head: &obj->gpuva.list);
1867
1868	return vm_bo;
1869	}
1870	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain);
1871
1872	/**
1873	* drm_gpuvm_bo_obtain_prealloc() - obtains an instance of the &drm_gpuvm_bo
1874	* for the given &drm_gpuvm and &drm_gem_object
1875	* @__vm_bo: A pre-allocated struct drm_gpuvm_bo.
1876	*
1877	* Find the &drm_gpuvm_bo representing the combination of the given
1878	* &drm_gpuvm and &drm_gem_object. If found, increases the reference
1879	* count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference
1880	* count is decreased. If not found @__vm_bo is returned without further
1881	* increase of the reference count.
1882	*
1883	* The provided @__vm_bo must not already be in the gpuva, evict, or extobj
1884	* lists prior to calling this method.
1885	*
1886	* A new &drm_gpuvm_bo is added to the GEMs gpuva list.
1887	*
1888	* Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing
1889	* &drm_gpuvm_bo was found
1890	*/
1891	struct drm_gpuvm_bo *
1892	drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo)
1893	{
1894	struct drm_gpuvm *gpuvm = __vm_bo->vm;
1895	struct drm_gem_object *obj = __vm_bo->obj;
1896	struct drm_gpuvm_bo *vm_bo;
1897
1898	drm_WARN_ON(gpuvm->drm, !drm_gpuvm_immediate_mode(gpuvm));
1899
1900	mutex_lock(&obj->gpuva.lock);
1901	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
1902	if (vm_bo) {
1903	mutex_unlock(lock: &obj->gpuva.lock);
1904	kref_put(kref: &__vm_bo->kref, release: drm_gpuvm_bo_destroy_not_in_lists_kref);
1905	return vm_bo;
1906	}
1907
1908	drm_gem_gpuva_assert_lock_held(gpuvm, obj);
1909	list_add_tail(new: &__vm_bo->list.entry.gem, head: &obj->gpuva.list);
1910	mutex_unlock(lock: &obj->gpuva.lock);
1911
1912	return __vm_bo;
1913	}
1914	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc);
1915
1916	/**
1917	* drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's
1918	* extobj list
1919	* @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list.
1920	*
1921	* Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list
1922	* already and if the corresponding &drm_gem_object is an external object,
1923	* actually.
1924	*/
1925	void
1926	drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo)
1927	{
1928	struct drm_gpuvm *gpuvm = vm_bo->vm;
1929	bool lock = !drm_gpuvm_resv_protected(gpuvm);
1930
1931	if (!lock)
1932	drm_gpuvm_resv_assert_held(gpuvm);
1933
1934	if (drm_gpuvm_is_extobj(gpuvm, obj: vm_bo->obj))
1935	drm_gpuvm_bo_list_add(vm_bo, extobj, lock);
1936	}
1937	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_extobj_add);
1938
1939	/**
1940	* drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms
1941	* evicted list
1942	* @vm_bo: the &drm_gpuvm_bo to add or remove
1943	* @evict: indicates whether the object is evicted
1944	*
1945	* Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvm's evicted list.
1946	*/
1947	void
1948	drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict)
1949	{
1950	struct drm_gpuvm *gpuvm = vm_bo->vm;
1951	struct drm_gem_object *obj = vm_bo->obj;
1952	bool lock = !drm_gpuvm_resv_protected(gpuvm);
1953
1954	dma_resv_assert_held(obj->resv);
1955	vm_bo->evicted = evict;
1956
1957	/* Can't add external objects to the evicted list directly if not using
1958	* internal spinlocks, since in this case the evicted list is protected
1959	* with the VM's common dma-resv lock.
1960	*/
1961	if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock)
1962	return;
1963
1964	if (evict)
1965	drm_gpuvm_bo_list_add(vm_bo, evict, lock);
1966	else
1967	drm_gpuvm_bo_list_del_init(vm_bo, evict, lock);
1968	}
1969	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_evict);
1970
1971	static int
1972	__drm_gpuva_insert(struct drm_gpuvm *gpuvm,
1973	struct drm_gpuva *va)
1974	{
1975	struct rb_node *node;
1976	struct list_head *head;
1977
1978	if (drm_gpuva_it_iter_first(root: &gpuvm->rb.tree,
1979	GPUVA_START(va),
1980	GPUVA_LAST(va)))
1981	return -EEXIST;
1982
1983	va->vm = gpuvm;
1984
1985	drm_gpuva_it_insert(node: va, root: &gpuvm->rb.tree);
1986
1987	node = rb_prev(&va->rb.node);
1988	if (node)
1989	head = &(to_drm_gpuva(node))->rb.entry;
1990	else
1991	head = &gpuvm->rb.list;
1992
1993	list_add(new: &va->rb.entry, head);
1994
1995	return 0;
1996	}
1997
1998	/**
1999	* drm_gpuva_insert() - insert a &drm_gpuva
2000	* @gpuvm: the &drm_gpuvm to insert the &drm_gpuva in
2001	* @va: the &drm_gpuva to insert
2002	*
2003	* Insert a &drm_gpuva with a given address and range into a
2004	* &drm_gpuvm.
2005	*
2006	* It is safe to use this function using the safe versions of iterating the GPU
2007	* VA space, such as drm_gpuvm_for_each_va_safe() and
2008	* drm_gpuvm_for_each_va_range_safe().
2009	*
2010	* Returns: 0 on success, negative error code on failure.
2011	*/
2012	int
2013	drm_gpuva_insert(struct drm_gpuvm *gpuvm,
2014	struct drm_gpuva *va)
2015	{
2016	u64 addr = va->va.addr;
2017	u64 range = va->va.range;
2018	int ret;
2019
2020	if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range)))
2021	return -EINVAL;
2022
2023	ret = __drm_gpuva_insert(gpuvm, va);
2024	if (likely(!ret))
2025	/* Take a reference of the GPUVM for the successfully inserted
2026	* drm_gpuva. We can't take the reference in
2027	* __drm_gpuva_insert() itself, since we don't want to increse
2028	* the reference count for the GPUVM's kernel_alloc_node.
2029	*/
2030	drm_gpuvm_get(gpuvm);
2031
2032	return ret;
2033	}
2034	EXPORT_SYMBOL_GPL(drm_gpuva_insert);
2035
2036	static void
2037	__drm_gpuva_remove(struct drm_gpuva *va)
2038	{
2039	drm_gpuva_it_remove(node: va, root: &va->vm->rb.tree);
2040	list_del_init(entry: &va->rb.entry);
2041	}
2042
2043	/**
2044	* drm_gpuva_remove() - remove a &drm_gpuva
2045	* @va: the &drm_gpuva to remove
2046	*
2047	* This removes the given &va from the underlying tree.
2048	*
2049	* It is safe to use this function using the safe versions of iterating the GPU
2050	* VA space, such as drm_gpuvm_for_each_va_safe() and
2051	* drm_gpuvm_for_each_va_range_safe().
2052	*/
2053	void
2054	drm_gpuva_remove(struct drm_gpuva *va)
2055	{
2056	struct drm_gpuvm *gpuvm = va->vm;
2057
2058	if (unlikely(va == &gpuvm->kernel_alloc_node)) {
2059	drm_WARN(gpuvm->drm, 1,
2060	"Can't destroy kernel reserved node.\n");
2061	return;
2062	}
2063
2064	__drm_gpuva_remove(va);
2065	drm_gpuvm_put(va->vm);
2066	}
2067	EXPORT_SYMBOL_GPL(drm_gpuva_remove);
2068
2069	/**
2070	* drm_gpuva_link() - link a &drm_gpuva
2071	* @va: the &drm_gpuva to link
2072	* @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to
2073	*
2074	* This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the
2075	* &drm_gpuvm_bo to the &drm_gem_object it is associated with.
2076	*
2077	* For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional
2078	* reference of the latter is taken.
2079	*
2080	* This function expects the caller to protect the GEM's GPUVA list against
2081	* concurrent access using either the GEM's dma-resv or gpuva.lock mutex.
2082	*/
2083	void
2084	drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo)
2085	{
2086	struct drm_gem_object *obj = va->gem.obj;
2087	struct drm_gpuvm *gpuvm = va->vm;
2088
2089	if (unlikely(!obj))
2090	return;
2091
2092	drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj);
2093
2094	va->vm_bo = drm_gpuvm_bo_get(vm_bo);
2095
2096	drm_gem_gpuva_assert_lock_held(gpuvm, obj);
2097	list_add_tail(new: &va->gem.entry, head: &vm_bo->list.gpuva);
2098	}
2099	EXPORT_SYMBOL_GPL(drm_gpuva_link);
2100
2101	/**
2102	* drm_gpuva_unlink() - unlink a &drm_gpuva
2103	* @va: the &drm_gpuva to unlink
2104	*
2105	* This removes the given &va from the GPU VA list of the &drm_gem_object it is
2106	* associated with.
2107	*
2108	* This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and
2109	* the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case
2110	* this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo.
2111	*
2112	* For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of
2113	* the latter is dropped.
2114	*
2115	* This function expects the caller to protect the GEM's GPUVA list against
2116	* concurrent access using either the GEM's dma-resv or gpuva.lock mutex.
2117	*/
2118	void
2119	drm_gpuva_unlink(struct drm_gpuva *va)
2120	{
2121	struct drm_gem_object *obj = va->gem.obj;
2122	struct drm_gpuvm_bo *vm_bo = va->vm_bo;
2123
2124	if (unlikely(!obj))
2125	return;
2126
2127	drm_gem_gpuva_assert_lock_held(va->vm, obj);
2128	list_del_init(entry: &va->gem.entry);
2129
2130	va->vm_bo = NULL;
2131	drm_gpuvm_bo_put(vm_bo);
2132	}
2133	EXPORT_SYMBOL_GPL(drm_gpuva_unlink);
2134
2135	/**
2136	* drm_gpuva_unlink_defer() - unlink a &drm_gpuva with deferred vm_bo cleanup
2137	* @va: the &drm_gpuva to unlink
2138	*
2139	* Similar to drm_gpuva_unlink(), but uses drm_gpuvm_bo_put_deferred() and takes
2140	* the lock for the caller.
2141	*/
2142	void
2143	drm_gpuva_unlink_defer(struct drm_gpuva *va)
2144	{
2145	struct drm_gem_object *obj = va->gem.obj;
2146	struct drm_gpuvm_bo *vm_bo = va->vm_bo;
2147	bool should_defer_bo;
2148
2149	if (unlikely(!obj))
2150	return;
2151
2152	drm_WARN_ON(vm_bo->vm->drm, !drm_gpuvm_immediate_mode(vm_bo->vm));
2153
2154	mutex_lock(&obj->gpuva.lock);
2155	list_del_init(entry: &va->gem.entry);
2156
2157	/*
2158	* This is drm_gpuvm_bo_put_deferred() except we already hold the mutex.
2159	*/
2160	should_defer_bo = kref_put(kref: &vm_bo->kref, release: drm_gpuvm_bo_into_zombie);
2161	mutex_unlock(lock: &obj->gpuva.lock);
2162	if (should_defer_bo)
2163	drm_gpuvm_bo_defer_zombie_cleanup(vm_bo);
2164
2165	va->vm_bo = NULL;
2166	}
2167	EXPORT_SYMBOL_GPL(drm_gpuva_unlink_defer);
2168
2169	/**
2170	* drm_gpuva_find_first() - find the first &drm_gpuva in the given range
2171	* @gpuvm: the &drm_gpuvm to search in
2172	* @addr: the &drm_gpuvas address
2173	* @range: the &drm_gpuvas range
2174	*
2175	* Returns: the first &drm_gpuva within the given range
2176	*/
2177	struct drm_gpuva *
2178	drm_gpuva_find_first(struct drm_gpuvm *gpuvm,
2179	u64 addr, u64 range)
2180	{
2181	u64 last = addr + range - 1;
2182
2183	return drm_gpuva_it_iter_first(root: &gpuvm->rb.tree, start: addr, last);
2184	}
2185	EXPORT_SYMBOL_GPL(drm_gpuva_find_first);
2186
2187	/**
2188	* drm_gpuva_find() - find a &drm_gpuva
2189	* @gpuvm: the &drm_gpuvm to search in
2190	* @addr: the &drm_gpuvas address
2191	* @range: the &drm_gpuvas range
2192	*
2193	* Returns: the &drm_gpuva at a given &addr and with a given &range
2194	*/
2195	struct drm_gpuva *
2196	drm_gpuva_find(struct drm_gpuvm *gpuvm,
2197	u64 addr, u64 range)
2198	{
2199	struct drm_gpuva *va;
2200
2201	va = drm_gpuva_find_first(gpuvm, addr, range);
2202	if (!va)
2203	goto out;
2204
2205	if (va->va.addr != addr ||
2206	va->va.range != range)
2207	goto out;
2208
2209	return va;
2210
2211	out:
2212	return NULL;
2213	}
2214	EXPORT_SYMBOL_GPL(drm_gpuva_find);
2215
2216	/**
2217	* drm_gpuva_find_prev() - find the &drm_gpuva before the given address
2218	* @gpuvm: the &drm_gpuvm to search in
2219	* @start: the given GPU VA's start address
2220	*
2221	* Find the adjacent &drm_gpuva before the GPU VA with given &start address.
2222	*
2223	* Note that if there is any free space between the GPU VA mappings no mapping
2224	* is returned.
2225	*
2226	* Returns: a pointer to the found &drm_gpuva or NULL if none was found
2227	*/
2228	struct drm_gpuva *
2229	drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start)
2230	{
2231	if (!drm_gpuvm_range_valid(gpuvm, start - 1, 1))
2232	return NULL;
2233
2234	return drm_gpuva_it_iter_first(root: &gpuvm->rb.tree, start: start - 1, last: start);
2235	}
2236	EXPORT_SYMBOL_GPL(drm_gpuva_find_prev);
2237
2238	/**
2239	* drm_gpuva_find_next() - find the &drm_gpuva after the given address
2240	* @gpuvm: the &drm_gpuvm to search in
2241	* @end: the given GPU VA's end address
2242	*
2243	* Find the adjacent &drm_gpuva after the GPU VA with given &end address.
2244	*
2245	* Note that if there is any free space between the GPU VA mappings no mapping
2246	* is returned.
2247	*
2248	* Returns: a pointer to the found &drm_gpuva or NULL if none was found
2249	*/
2250	struct drm_gpuva *
2251	drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end)
2252	{
2253	if (!drm_gpuvm_range_valid(gpuvm, end, 1))
2254	return NULL;
2255
2256	return drm_gpuva_it_iter_first(root: &gpuvm->rb.tree, start: end, last: end + 1);
2257	}
2258	EXPORT_SYMBOL_GPL(drm_gpuva_find_next);
2259
2260	/**
2261	* drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space
2262	* is empty
2263	* @gpuvm: the &drm_gpuvm to check the range for
2264	* @addr: the start address of the range
2265	* @range: the range of the interval
2266	*
2267	* Returns: true if the interval is empty, false otherwise
2268	*/
2269	bool
2270	drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
2271	{
2272	return !drm_gpuva_find_first(gpuvm, addr, range);
2273	}
2274	EXPORT_SYMBOL_GPL(drm_gpuvm_interval_empty);
2275
2276	/**
2277	* drm_gpuva_map() - helper to insert a &drm_gpuva according to a
2278	* &drm_gpuva_op_map
2279	* @gpuvm: the &drm_gpuvm
2280	* @va: the &drm_gpuva to insert
2281	* @op: the &drm_gpuva_op_map to initialize @va with
2282	*
2283	* Initializes the @va from the @op and inserts it into the given @gpuvm.
2284	*/
2285	void
2286	drm_gpuva_map(struct drm_gpuvm *gpuvm,
2287	struct drm_gpuva *va,
2288	struct drm_gpuva_op_map *op)
2289	{
2290	drm_gpuva_init_from_op(va, op);
2291	drm_gpuva_insert(gpuvm, va);
2292	}
2293	EXPORT_SYMBOL_GPL(drm_gpuva_map);
2294
2295	/**
2296	* drm_gpuva_remap() - helper to remap a &drm_gpuva according to a
2297	* &drm_gpuva_op_remap
2298	* @prev: the &drm_gpuva to remap when keeping the start of a mapping
2299	* @next: the &drm_gpuva to remap when keeping the end of a mapping
2300	* @op: the &drm_gpuva_op_remap to initialize @prev and @next with
2301	*
2302	* Removes the currently mapped &drm_gpuva and remaps it using @prev and/or
2303	* @next.
2304	*/
2305	void
2306	drm_gpuva_remap(struct drm_gpuva *prev,
2307	struct drm_gpuva *next,
2308	struct drm_gpuva_op_remap *op)
2309	{
2310	struct drm_gpuva *va = op->unmap->va;
2311	struct drm_gpuvm *gpuvm = va->vm;
2312
2313	drm_gpuva_remove(va);
2314
2315	if (op->prev) {
2316	drm_gpuva_init_from_op(va: prev, op: op->prev);
2317	drm_gpuva_insert(gpuvm, prev);
2318	}
2319
2320	if (op->next) {
2321	drm_gpuva_init_from_op(va: next, op: op->next);
2322	drm_gpuva_insert(gpuvm, next);
2323	}
2324	}
2325	EXPORT_SYMBOL_GPL(drm_gpuva_remap);
2326
2327	/**
2328	* drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a
2329	* &drm_gpuva_op_unmap
2330	* @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove
2331	*
2332	* Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap.
2333	*/
2334	void
2335	drm_gpuva_unmap(struct drm_gpuva_op_unmap *op)
2336	{
2337	drm_gpuva_remove(op->va);
2338	}
2339	EXPORT_SYMBOL_GPL(drm_gpuva_unmap);
2340
2341	static int
2342	op_map_cb(const struct drm_gpuvm_ops *fn, void *priv,
2343	const struct drm_gpuvm_map_req *req)
2344	{
2345	struct drm_gpuva_op op = {};
2346
2347	if (!req)
2348	return 0;
2349
2350	op.op = DRM_GPUVA_OP_MAP;
2351	op.map.va.addr = req->map.va.addr;
2352	op.map.va.range = req->map.va.range;
2353	op.map.gem.obj = req->map.gem.obj;
2354	op.map.gem.offset = req->map.gem.offset;
2355
2356	return fn->sm_step_map(&op, priv);
2357	}
2358
2359	static int
2360	op_remap_cb(const struct drm_gpuvm_ops *fn, void *priv,
2361	struct drm_gpuva_op_map *prev,
2362	struct drm_gpuva_op_map *next,
2363	struct drm_gpuva_op_unmap *unmap)
2364	{
2365	struct drm_gpuva_op op = {};
2366	struct drm_gpuva_op_remap *r;
2367
2368	op.op = DRM_GPUVA_OP_REMAP;
2369	r = &op.remap;
2370	r->prev = prev;
2371	r->next = next;
2372	r->unmap = unmap;
2373
2374	return fn->sm_step_remap(&op, priv);
2375	}
2376
2377	static int
2378	op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv,
2379	struct drm_gpuva *va, bool merge, bool madvise)
2380	{
2381	struct drm_gpuva_op op = {};
2382
2383	if (madvise)
2384	return 0;
2385
2386	op.op = DRM_GPUVA_OP_UNMAP;
2387	op.unmap.va = va;
2388	op.unmap.keep = merge;
2389
2390	return fn->sm_step_unmap(&op, priv);
2391	}
2392
2393	static int
2394	__drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm,
2395	const struct drm_gpuvm_ops *ops, void *priv,
2396	const struct drm_gpuvm_map_req *req,
2397	bool madvise)
2398	{
2399	struct drm_gem_object *req_obj = req->map.gem.obj;
2400	const struct drm_gpuvm_map_req *op_map = madvise ? NULL : req;
2401	struct drm_gpuva *va, *next;
2402	u64 req_offset = req->map.gem.offset;
2403	u64 req_range = req->map.va.range;
2404	u64 req_addr = req->map.va.addr;
2405	u64 req_end = req_addr + req_range;
2406	int ret;
2407
2408	if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))
2409	return -EINVAL;
2410
2411	drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {
2412	struct drm_gem_object *obj = va->gem.obj;
2413	u64 offset = va->gem.offset;
2414	u64 addr = va->va.addr;
2415	u64 range = va->va.range;
2416	u64 end = addr + range;
2417	bool merge = !!va->gem.obj;
2418
2419	if (madvise && obj)
2420	continue;
2421
2422	if (addr == req_addr) {
2423	merge &= obj == req_obj &&
2424	offset == req_offset;
2425
2426	if (end == req_end) {
2427	ret = op_unmap_cb(fn: ops, priv, va, merge, madvise);
2428	if (ret)
2429	return ret;
2430	break;
2431	}
2432
2433	if (end < req_end) {
2434	ret = op_unmap_cb(fn: ops, priv, va, merge, madvise);
2435	if (ret)
2436	return ret;
2437	continue;
2438	}
2439
2440	if (end > req_end) {
2441	struct drm_gpuva_op_map n = {
2442	.va.addr = req_end,
2443	.va.range = range - req_range,
2444	.gem.obj = obj,
2445	.gem.offset = offset + req_range,
2446	};
2447	struct drm_gpuva_op_unmap u = {
2448	.va = va,
2449	.keep = merge,
2450	};
2451
2452	ret = op_remap_cb(fn: ops, priv, NULL, next: &n, unmap: &u);
2453	if (ret)
2454	return ret;
2455
2456	if (madvise)
2457	op_map = req;
2458	break;
2459	}
2460	} else if (addr < req_addr) {
2461	u64 ls_range = req_addr - addr;
2462	struct drm_gpuva_op_map p = {
2463	.va.addr = addr,
2464	.va.range = ls_range,
2465	.gem.obj = obj,
2466	.gem.offset = offset,
2467	};
2468	struct drm_gpuva_op_unmap u = { .va = va };
2469
2470	merge &= obj == req_obj &&
2471	offset + ls_range == req_offset;
2472	u.keep = merge;
2473
2474	if (end == req_end) {
2475	ret = op_remap_cb(fn: ops, priv, prev: &p, NULL, unmap: &u);
2476	if (ret)
2477	return ret;
2478
2479	if (madvise)
2480	op_map = req;
2481	break;
2482	}
2483
2484	if (end < req_end) {
2485	ret = op_remap_cb(fn: ops, priv, prev: &p, NULL, unmap: &u);
2486	if (ret)
2487	return ret;
2488
2489	if (madvise) {
2490	struct drm_gpuvm_map_req map_req = {
2491	.map.va.addr = req_addr,
2492	.map.va.range = end - req_addr,
2493	};
2494
2495	ret = op_map_cb(fn: ops, priv, req: &map_req);
2496	if (ret)
2497	return ret;
2498	}
2499
2500	continue;
2501	}
2502
2503	if (end > req_end) {
2504	struct drm_gpuva_op_map n = {
2505	.va.addr = req_end,
2506	.va.range = end - req_end,
2507	.gem.obj = obj,
2508	.gem.offset = offset + ls_range +
2509	req_range,
2510	};
2511
2512	ret = op_remap_cb(fn: ops, priv, prev: &p, next: &n, unmap: &u);
2513	if (ret)
2514	return ret;
2515
2516	if (madvise)
2517	op_map = req;
2518	break;
2519	}
2520	} else if (addr > req_addr) {
2521	merge &= obj == req_obj &&
2522	offset == req_offset +
2523	(addr - req_addr);
2524
2525	if (end == req_end) {
2526	ret = op_unmap_cb(fn: ops, priv, va, merge, madvise);
2527	if (ret)
2528	return ret;
2529
2530	break;
2531	}
2532
2533	if (end < req_end) {
2534	ret = op_unmap_cb(fn: ops, priv, va, merge, madvise);
2535	if (ret)
2536	return ret;
2537
2538	continue;
2539	}
2540
2541	if (end > req_end) {
2542	struct drm_gpuva_op_map n = {
2543	.va.addr = req_end,
2544	.va.range = end - req_end,
2545	.gem.obj = obj,
2546	.gem.offset = offset + req_end - addr,
2547	};
2548	struct drm_gpuva_op_unmap u = {
2549	.va = va,
2550	.keep = merge,
2551	};
2552
2553	ret = op_remap_cb(fn: ops, priv, NULL, next: &n, unmap: &u);
2554	if (ret)
2555	return ret;
2556
2557	if (madvise) {
2558	struct drm_gpuvm_map_req map_req = {
2559	.map.va.addr = addr,
2560	.map.va.range = req_end - addr,
2561	};
2562
2563	return op_map_cb(fn: ops, priv, req: &map_req);
2564	}
2565	break;
2566	}
2567	}
2568	}
2569	return op_map_cb(fn: ops, priv, req: op_map);
2570	}
2571
2572	static int
2573	__drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm,
2574	const struct drm_gpuvm_ops *ops, void *priv,
2575	u64 req_addr, u64 req_range)
2576	{
2577	struct drm_gpuva *va, *next;
2578	u64 req_end = req_addr + req_range;
2579	int ret;
2580
2581	if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))
2582	return -EINVAL;
2583
2584	drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {
2585	struct drm_gpuva_op_map prev = {}, next = {};
2586	bool prev_split = false, next_split = false;
2587	struct drm_gem_object *obj = va->gem.obj;
2588	u64 offset = va->gem.offset;
2589	u64 addr = va->va.addr;
2590	u64 range = va->va.range;
2591	u64 end = addr + range;
2592
2593	if (addr < req_addr) {
2594	prev.va.addr = addr;
2595	prev.va.range = req_addr - addr;
2596	prev.gem.obj = obj;
2597	prev.gem.offset = offset;
2598
2599	prev_split = true;
2600	}
2601
2602	if (end > req_end) {
2603	next.va.addr = req_end;
2604	next.va.range = end - req_end;
2605	next.gem.obj = obj;
2606	next.gem.offset = offset + (req_end - addr);
2607
2608	next_split = true;
2609	}
2610
2611	if (prev_split || next_split) {
2612	struct drm_gpuva_op_unmap unmap = { .va = va };
2613
2614	ret = op_remap_cb(fn: ops, priv,
2615	prev: prev_split ? &prev : NULL,
2616	next: next_split ? &next : NULL,
2617	unmap: &unmap);
2618	if (ret)
2619	return ret;
2620	} else {
2621	ret = op_unmap_cb(fn: ops, priv, va, merge: false, madvise: false);
2622	if (ret)
2623	return ret;
2624	}
2625	}
2626
2627	return 0;
2628	}
2629
2630	/**
2631	* drm_gpuvm_sm_map() - calls the &drm_gpuva_op split/merge steps
2632	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2633	* @priv: pointer to a driver private data structure
2634	* @req: ptr to struct drm_gpuvm_map_req
2635	*
2636	* This function iterates the given range of the GPU VA space. It utilizes the
2637	* &drm_gpuvm_ops to call back into the driver providing the split and merge
2638	* steps.
2639	*
2640	* Drivers may use these callbacks to update the GPU VA space right away within
2641	* the callback. In case the driver decides to copy and store the operations for
2642	* later processing neither this function nor &drm_gpuvm_sm_unmap is allowed to
2643	* be called before the &drm_gpuvm's view of the GPU VA space was
2644	* updated with the previous set of operations. To update the
2645	* &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
2646	* drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
2647	* used.
2648	*
2649	* A sequence of callbacks can contain map, unmap and remap operations, but
2650	* the sequence of callbacks might also be empty if no operation is required,
2651	* e.g. if the requested mapping already exists in the exact same way.
2652	*
2653	* There can be an arbitrary amount of unmap operations, a maximum of two remap
2654	* operations and a single map operation. The latter one represents the original
2655	* map operation requested by the caller.
2656	*
2657	* Returns: 0 on success or a negative error code
2658	*/
2659	int
2660	drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv,
2661	const struct drm_gpuvm_map_req *req)
2662	{
2663	const struct drm_gpuvm_ops *ops = gpuvm->ops;
2664
2665	if (unlikely(!(ops && ops->sm_step_map &&
2666	ops->sm_step_remap &&
2667	ops->sm_step_unmap)))
2668	return -EINVAL;
2669
2670	return __drm_gpuvm_sm_map(gpuvm, ops, priv, req, madvise: false);
2671	}
2672	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map);
2673
2674	/**
2675	* drm_gpuvm_sm_unmap() - calls the &drm_gpuva_ops to split on unmap
2676	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2677	* @priv: pointer to a driver private data structure
2678	* @req_addr: the start address of the range to unmap
2679	* @req_range: the range of the mappings to unmap
2680	*
2681	* This function iterates the given range of the GPU VA space. It utilizes the
2682	* &drm_gpuvm_ops to call back into the driver providing the operations to
2683	* unmap and, if required, split existing mappings.
2684	*
2685	* Drivers may use these callbacks to update the GPU VA space right away within
2686	* the callback. In case the driver decides to copy and store the operations for
2687	* later processing neither this function nor &drm_gpuvm_sm_map is allowed to be
2688	* called before the &drm_gpuvm's view of the GPU VA space was updated
2689	* with the previous set of operations. To update the &drm_gpuvm's view
2690	* of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or
2691	* drm_gpuva_destroy_unlocked() should be used.
2692	*
2693	* A sequence of callbacks can contain unmap and remap operations, depending on
2694	* whether there are actual overlapping mappings to split.
2695	*
2696	* There can be an arbitrary amount of unmap operations and a maximum of two
2697	* remap operations.
2698	*
2699	* Returns: 0 on success or a negative error code
2700	*/
2701	int
2702	drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv,
2703	u64 req_addr, u64 req_range)
2704	{
2705	const struct drm_gpuvm_ops *ops = gpuvm->ops;
2706
2707	if (unlikely(!(ops && ops->sm_step_remap &&
2708	ops->sm_step_unmap)))
2709	return -EINVAL;
2710
2711	return __drm_gpuvm_sm_unmap(gpuvm, ops, priv,
2712	req_addr, req_range);
2713	}
2714	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap);
2715
2716	static int
2717	drm_gpuva_sm_step_lock(struct drm_gpuva_op *op, void *priv)
2718	{
2719	struct drm_exec *exec = priv;
2720
2721	switch (op->op) {
2722	case DRM_GPUVA_OP_REMAP:
2723	if (op->remap.unmap->va->gem.obj)
2724	return drm_exec_lock_obj(exec, obj: op->remap.unmap->va->gem.obj);
2725	return 0;
2726	case DRM_GPUVA_OP_UNMAP:
2727	if (op->unmap.va->gem.obj)
2728	return drm_exec_lock_obj(exec, obj: op->unmap.va->gem.obj);
2729	return 0;
2730	default:
2731	return 0;
2732	}
2733	}
2734
2735	static const struct drm_gpuvm_ops lock_ops = {
2736	.sm_step_map = drm_gpuva_sm_step_lock,
2737	.sm_step_remap = drm_gpuva_sm_step_lock,
2738	.sm_step_unmap = drm_gpuva_sm_step_lock,
2739	};
2740
2741	/**
2742	* drm_gpuvm_sm_map_exec_lock() - locks the objects touched by a drm_gpuvm_sm_map()
2743	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2744	* @exec: the &drm_exec locking context
2745	* @num_fences: for newly mapped objects, the # of fences to reserve
2746	* @req: ptr to drm_gpuvm_map_req struct
2747	*
2748	* This function locks (drm_exec_lock_obj()) objects that will be unmapped/
2749	* remapped, and locks+prepares (drm_exec_prepare_object()) objects that
2750	* will be newly mapped.
2751	*
2752	* The expected usage is::
2753	*
2754	* vm_bind {
2755	* struct drm_exec exec;
2756	*
2757	* // IGNORE_DUPLICATES is required, INTERRUPTIBLE_WAIT is recommended:
2758	* drm_exec_init(&exec, IGNORE_DUPLICATES | INTERRUPTIBLE_WAIT, 0);
2759	*
2760	* drm_exec_until_all_locked (&exec) {
2761	* for_each_vm_bind_operation {
2762	* switch (op->op) {
2763	* case DRIVER_OP_UNMAP:
2764	* ret = drm_gpuvm_sm_unmap_exec_lock(gpuvm, &exec, op->addr, op->range);
2765	* break;
2766	* case DRIVER_OP_MAP:
2767	* ret = drm_gpuvm_sm_map_exec_lock(gpuvm, &exec, num_fences, &req);
2768	* break;
2769	* }
2770	*
2771	* drm_exec_retry_on_contention(&exec);
2772	* if (ret)
2773	* return ret;
2774	* }
2775	* }
2776	* }
2777	*
2778	* This enables all locking to be performed before the driver begins modifying
2779	* the VM. This is safe to do in the case of overlapping DRIVER_VM_BIND_OPs,
2780	* where an earlier op can alter the sequence of steps generated for a later
2781	* op, because the later altered step will involve the same GEM object(s)
2782	* already seen in the earlier locking step. For example:
2783	*
2784	* 1) An earlier driver DRIVER_OP_UNMAP op removes the need for a
2785	* DRM_GPUVA_OP_REMAP/UNMAP step. This is safe because we've already
2786	* locked the GEM object in the earlier DRIVER_OP_UNMAP op.
2787	*
2788	* 2) An earlier DRIVER_OP_MAP op overlaps with a later DRIVER_OP_MAP/UNMAP
2789	* op, introducing a DRM_GPUVA_OP_REMAP/UNMAP that wouldn't have been
2790	* required without the earlier DRIVER_OP_MAP. This is safe because we've
2791	* already locked the GEM object in the earlier DRIVER_OP_MAP step.
2792	*
2793	* Returns: 0 on success or a negative error code
2794	*/
2795	int
2796	drm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm,
2797	struct drm_exec *exec, unsigned int num_fences,
2798	struct drm_gpuvm_map_req *req)
2799	{
2800	struct drm_gem_object *req_obj = req->map.gem.obj;
2801
2802	if (req_obj) {
2803	int ret = drm_exec_prepare_obj(exec, obj: req_obj, num_fences);
2804	if (ret)
2805	return ret;
2806	}
2807
2808	return __drm_gpuvm_sm_map(gpuvm, ops: &lock_ops, priv: exec, req, madvise: false);
2809
2810	}
2811	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_exec_lock);
2812
2813	/**
2814	* drm_gpuvm_sm_unmap_exec_lock() - locks the objects touched by drm_gpuvm_sm_unmap()
2815	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2816	* @exec: the &drm_exec locking context
2817	* @req_addr: the start address of the range to unmap
2818	* @req_range: the range of the mappings to unmap
2819	*
2820	* This function locks (drm_exec_lock_obj()) objects that will be unmapped/
2821	* remapped by drm_gpuvm_sm_unmap().
2822	*
2823	* See drm_gpuvm_sm_map_exec_lock() for expected usage.
2824	*
2825	* Returns: 0 on success or a negative error code
2826	*/
2827	int
2828	drm_gpuvm_sm_unmap_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec,
2829	u64 req_addr, u64 req_range)
2830	{
2831	return __drm_gpuvm_sm_unmap(gpuvm, ops: &lock_ops, priv: exec,
2832	req_addr, req_range);
2833	}
2834	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_exec_lock);
2835
2836	static struct drm_gpuva_op *
2837	gpuva_op_alloc(struct drm_gpuvm *gpuvm)
2838	{
2839	const struct drm_gpuvm_ops *fn = gpuvm->ops;
2840	struct drm_gpuva_op *op;
2841
2842	if (fn && fn->op_alloc)
2843	op = fn->op_alloc();
2844	else
2845	op = kzalloc(sizeof(*op), GFP_KERNEL);
2846
2847	if (unlikely(!op))
2848	return NULL;
2849
2850	return op;
2851	}
2852
2853	static void
2854	gpuva_op_free(struct drm_gpuvm *gpuvm,
2855	struct drm_gpuva_op *op)
2856	{
2857	const struct drm_gpuvm_ops *fn = gpuvm->ops;
2858
2859	if (fn && fn->op_free)
2860	fn->op_free(op);
2861	else
2862	kfree(objp: op);
2863	}
2864
2865	static int
2866	drm_gpuva_sm_step(struct drm_gpuva_op *__op,
2867	void *priv)
2868	{
2869	struct {
2870	struct drm_gpuvm *vm;
2871	struct drm_gpuva_ops *ops;
2872	} *args = priv;
2873	struct drm_gpuvm *gpuvm = args->vm;
2874	struct drm_gpuva_ops *ops = args->ops;
2875	struct drm_gpuva_op *op;
2876
2877	op = gpuva_op_alloc(gpuvm);
2878	if (unlikely(!op))
2879	goto err;
2880
2881	memcpy(op, __op, sizeof(*op));
2882
2883	if (op->op == DRM_GPUVA_OP_REMAP) {
2884	struct drm_gpuva_op_remap *__r = &__op->remap;
2885	struct drm_gpuva_op_remap *r = &op->remap;
2886
2887	r->unmap = kmemdup(__r->unmap, sizeof(*r->unmap),
2888	GFP_KERNEL);
2889	if (unlikely(!r->unmap))
2890	goto err_free_op;
2891
2892	if (__r->prev) {
2893	r->prev = kmemdup(__r->prev, sizeof(*r->prev),
2894	GFP_KERNEL);
2895	if (unlikely(!r->prev))
2896	goto err_free_unmap;
2897	}
2898
2899	if (__r->next) {
2900	r->next = kmemdup(__r->next, sizeof(*r->next),
2901	GFP_KERNEL);
2902	if (unlikely(!r->next))
2903	goto err_free_prev;
2904	}
2905	}
2906
2907	list_add_tail(new: &op->entry, head: &ops->list);
2908
2909	return 0;
2910
2911	err_free_unmap:
2912	kfree(objp: op->remap.unmap);
2913	err_free_prev:
2914	kfree(objp: op->remap.prev);
2915	err_free_op:
2916	gpuva_op_free(gpuvm, op);
2917	err:
2918	return -ENOMEM;
2919	}
2920
2921	static const struct drm_gpuvm_ops gpuvm_list_ops = {
2922	.sm_step_map = drm_gpuva_sm_step,
2923	.sm_step_remap = drm_gpuva_sm_step,
2924	.sm_step_unmap = drm_gpuva_sm_step,
2925	};
2926
2927	static struct drm_gpuva_ops *
2928	__drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,
2929	const struct drm_gpuvm_map_req *req,
2930	bool madvise)
2931	{
2932	struct drm_gpuva_ops *ops;
2933	struct {
2934	struct drm_gpuvm *vm;
2935	struct drm_gpuva_ops *ops;
2936	} args;
2937	int ret;
2938
2939	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
2940	if (unlikely(!ops))
2941	return ERR_PTR(error: -ENOMEM);
2942
2943	INIT_LIST_HEAD(list: &ops->list);
2944
2945	args.vm = gpuvm;
2946	args.ops = ops;
2947
2948	ret = __drm_gpuvm_sm_map(gpuvm, ops: &gpuvm_list_ops, priv: &args, req, madvise);
2949	if (ret)
2950	goto err_free_ops;
2951
2952	return ops;
2953
2954	err_free_ops:
2955	drm_gpuva_ops_free(gpuvm, ops);
2956	return ERR_PTR(error: ret);
2957	}
2958
2959	/**
2960	* drm_gpuvm_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge
2961	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2962	* @req: map request arguments
2963	*
2964	* This function creates a list of operations to perform splitting and merging
2965	* of existing mapping(s) with the newly requested one.
2966	*
2967	* The list can be iterated with &drm_gpuva_for_each_op and must be processed
2968	* in the given order. It can contain map, unmap and remap operations, but it
2969	* also can be empty if no operation is required, e.g. if the requested mapping
2970	* already exists in the exact same way.
2971	*
2972	* There can be an arbitrary amount of unmap operations, a maximum of two remap
2973	* operations and a single map operation. The latter one represents the original
2974	* map operation requested by the caller.
2975	*
2976	* Note that before calling this function again with another mapping request it
2977	* is necessary to update the &drm_gpuvm's view of the GPU VA space. The
2978	* previously obtained operations must be either processed or abandoned. To
2979	* update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
2980	* drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
2981	* used.
2982	*
2983	* After the caller finished processing the returned &drm_gpuva_ops, they must
2984	* be freed with &drm_gpuva_ops_free.
2985	*
2986	* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
2987	*/
2988	struct drm_gpuva_ops *
2989	drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,
2990	const struct drm_gpuvm_map_req *req)
2991	{
2992	return __drm_gpuvm_sm_map_ops_create(gpuvm, req, madvise: false);
2993	}
2994	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_ops_create);
2995
2996	/**
2997	* drm_gpuvm_madvise_ops_create() - creates the &drm_gpuva_ops to split
2998	* @gpuvm: the &drm_gpuvm representing the GPU VA space
2999	* @req: map request arguments
3000	*
3001	* This function creates a list of operations to perform splitting
3002	* of existent mapping(s) at start or end, based on the request map.
3003	*
3004	* The list can be iterated with &drm_gpuva_for_each_op and must be processed
3005	* in the given order. It can contain map and remap operations, but it
3006	* also can be empty if no operation is required, e.g. if the requested mapping
3007	* already exists is the exact same way.
3008	*
3009	* There will be no unmap operations, a maximum of two remap operations and two
3010	* map operations. The two map operations correspond to: one from start to the
3011	* end of drm_gpuvaX, and another from the start of drm_gpuvaY to end.
3012	*
3013	* Note that before calling this function again with another mapping request it
3014	* is necessary to update the &drm_gpuvm's view of the GPU VA space. The
3015	* previously obtained operations must be either processed or abandoned. To
3016	* update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
3017	* drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
3018	* used.
3019	*
3020	* After the caller finished processing the returned &drm_gpuva_ops, they must
3021	* be freed with &drm_gpuva_ops_free.
3022	*
3023	* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
3024	*/
3025	struct drm_gpuva_ops *
3026	drm_gpuvm_madvise_ops_create(struct drm_gpuvm *gpuvm,
3027	const struct drm_gpuvm_map_req *req)
3028	{
3029	return __drm_gpuvm_sm_map_ops_create(gpuvm, req, madvise: true);
3030	}
3031	EXPORT_SYMBOL_GPL(drm_gpuvm_madvise_ops_create);
3032
3033	/**
3034	* drm_gpuvm_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on
3035	* unmap
3036	* @gpuvm: the &drm_gpuvm representing the GPU VA space
3037	* @req_addr: the start address of the range to unmap
3038	* @req_range: the range of the mappings to unmap
3039	*
3040	* This function creates a list of operations to perform unmapping and, if
3041	* required, splitting of the mappings overlapping the unmap range.
3042	*
3043	* The list can be iterated with &drm_gpuva_for_each_op and must be processed
3044	* in the given order. It can contain unmap and remap operations, depending on
3045	* whether there are actual overlapping mappings to split.
3046	*
3047	* There can be an arbitrary amount of unmap operations and a maximum of two
3048	* remap operations.
3049	*
3050	* Note that before calling this function again with another range to unmap it
3051	* is necessary to update the &drm_gpuvm's view of the GPU VA space. The
3052	* previously obtained operations must be processed or abandoned. To update the
3053	* &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
3054	* drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
3055	* used.
3056	*
3057	* After the caller finished processing the returned &drm_gpuva_ops, they must
3058	* be freed with &drm_gpuva_ops_free.
3059	*
3060	* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
3061	*/
3062	struct drm_gpuva_ops *
3063	drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm,
3064	u64 req_addr, u64 req_range)
3065	{
3066	struct drm_gpuva_ops *ops;
3067	struct {
3068	struct drm_gpuvm *vm;
3069	struct drm_gpuva_ops *ops;
3070	} args;
3071	int ret;
3072
3073	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
3074	if (unlikely(!ops))
3075	return ERR_PTR(error: -ENOMEM);
3076
3077	INIT_LIST_HEAD(list: &ops->list);
3078
3079	args.vm = gpuvm;
3080	args.ops = ops;
3081
3082	ret = __drm_gpuvm_sm_unmap(gpuvm, ops: &gpuvm_list_ops, priv: &args,
3083	req_addr, req_range);
3084	if (ret)
3085	goto err_free_ops;
3086
3087	return ops;
3088
3089	err_free_ops:
3090	drm_gpuva_ops_free(gpuvm, ops);
3091	return ERR_PTR(error: ret);
3092	}
3093	EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_ops_create);
3094
3095	/**
3096	* drm_gpuvm_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch
3097	* @gpuvm: the &drm_gpuvm representing the GPU VA space
3098	* @addr: the start address of the range to prefetch
3099	* @range: the range of the mappings to prefetch
3100	*
3101	* This function creates a list of operations to perform prefetching.
3102	*
3103	* The list can be iterated with &drm_gpuva_for_each_op and must be processed
3104	* in the given order. It can contain prefetch operations.
3105	*
3106	* There can be an arbitrary amount of prefetch operations.
3107	*
3108	* After the caller finished processing the returned &drm_gpuva_ops, they must
3109	* be freed with &drm_gpuva_ops_free.
3110	*
3111	* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
3112	*/
3113	struct drm_gpuva_ops *
3114	drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,
3115	u64 addr, u64 range)
3116	{
3117	struct drm_gpuva_ops *ops;
3118	struct drm_gpuva_op *op;
3119	struct drm_gpuva *va;
3120	u64 end = addr + range;
3121	int ret;
3122
3123	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
3124	if (!ops)
3125	return ERR_PTR(error: -ENOMEM);
3126
3127	INIT_LIST_HEAD(list: &ops->list);
3128
3129	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {
3130	op = gpuva_op_alloc(gpuvm);
3131	if (!op) {
3132	ret = -ENOMEM;
3133	goto err_free_ops;
3134	}
3135
3136	op->op = DRM_GPUVA_OP_PREFETCH;
3137	op->prefetch.va = va;
3138	list_add_tail(new: &op->entry, head: &ops->list);
3139	}
3140
3141	return ops;
3142
3143	err_free_ops:
3144	drm_gpuva_ops_free(gpuvm, ops);
3145	return ERR_PTR(error: ret);
3146	}
3147	EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create);
3148
3149	/**
3150	* drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM
3151	* @vm_bo: the &drm_gpuvm_bo abstraction
3152	*
3153	* This function creates a list of operations to perform unmapping for every
3154	* GPUVA attached to a GEM.
3155	*
3156	* The list can be iterated with &drm_gpuva_for_each_op and consists out of an
3157	* arbitrary amount of unmap operations.
3158	*
3159	* After the caller finished processing the returned &drm_gpuva_ops, they must
3160	* be freed with &drm_gpuva_ops_free.
3161	*
3162	* This function expects the caller to protect the GEM's GPUVA list against
3163	* concurrent access using either the GEM's dma-resv or gpuva.lock mutex.
3164	*
3165	* Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
3166	*/
3167	struct drm_gpuva_ops *
3168	drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo)
3169	{
3170	struct drm_gpuva_ops *ops;
3171	struct drm_gpuva_op *op;
3172	struct drm_gpuva *va;
3173	int ret;
3174
3175	drm_gem_gpuva_assert_lock_held(vm_bo->vm, vm_bo->obj);
3176
3177	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
3178	if (!ops)
3179	return ERR_PTR(error: -ENOMEM);
3180
3181	INIT_LIST_HEAD(list: &ops->list);
3182
3183	drm_gpuvm_bo_for_each_va(va, vm_bo) {
3184	op = gpuva_op_alloc(gpuvm: vm_bo->vm);
3185	if (!op) {
3186	ret = -ENOMEM;
3187	goto err_free_ops;
3188	}
3189
3190	op->op = DRM_GPUVA_OP_UNMAP;
3191	op->unmap.va = va;
3192	list_add_tail(new: &op->entry, head: &ops->list);
3193	}
3194
3195	return ops;
3196
3197	err_free_ops:
3198	drm_gpuva_ops_free(gpuvm: vm_bo->vm, ops);
3199	return ERR_PTR(error: ret);
3200	}
3201	EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap_ops_create);
3202
3203	/**
3204	* drm_gpuva_ops_free() - free the given &drm_gpuva_ops
3205	* @gpuvm: the &drm_gpuvm the ops were created for
3206	* @ops: the &drm_gpuva_ops to free
3207	*
3208	* Frees the given &drm_gpuva_ops structure including all the ops associated
3209	* with it.
3210	*/
3211	void
3212	drm_gpuva_ops_free(struct drm_gpuvm *gpuvm,
3213	struct drm_gpuva_ops *ops)
3214	{
3215	struct drm_gpuva_op *op, *next;
3216
3217	drm_gpuva_for_each_op_safe(op, next, ops) {
3218	list_del(entry: &op->entry);
3219
3220	if (op->op == DRM_GPUVA_OP_REMAP) {
3221	kfree(objp: op->remap.prev);
3222	kfree(objp: op->remap.next);
3223	kfree(objp: op->remap.unmap);
3224	}
3225
3226	gpuva_op_free(gpuvm, op);
3227	}
3228
3229	kfree(objp: ops);
3230	}
3231	EXPORT_SYMBOL_GPL(drm_gpuva_ops_free);
3232
3233	MODULE_DESCRIPTION("DRM GPUVM");
3234	MODULE_LICENSE("GPL");
3235