1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2025 Intel Corporation
4 */
5
6#include "xe_svm.h"
7#include "xe_userptr.h"
8
9#include <linux/mm.h>
10
11#include "xe_trace_bo.h"
12
13/**
14 * xe_vma_userptr_check_repin() - Advisory check for repin needed
15 * @uvma: The userptr vma
16 *
17 * Check if the userptr vma has been invalidated since last successful
18 * repin. The check is advisory only and can the function can be called
19 * without the vm->svm.gpusvm.notifier_lock held. There is no guarantee that the
20 * vma userptr will remain valid after a lockless check, so typically
21 * the call needs to be followed by a proper check under the notifier_lock.
22 *
23 * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended.
24 */
25int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma)
26{
27 return mmu_interval_check_retry(interval_sub: &uvma->userptr.notifier,
28 seq: uvma->userptr.pages.notifier_seq) ?
29 -EAGAIN : 0;
30}
31
32/**
33 * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs
34 * that need repinning.
35 * @vm: The VM.
36 *
37 * This function checks for whether the VM has userptrs that need repinning,
38 * and provides a release-type barrier on the svm.gpusvm.notifier_lock after
39 * checking.
40 *
41 * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are.
42 */
43int __xe_vm_userptr_needs_repin(struct xe_vm *vm)
44{
45 lockdep_assert_held_read(&vm->svm.gpusvm.notifier_lock);
46
47 return (list_empty(head: &vm->userptr.repin_list) &&
48 list_empty(head: &vm->userptr.invalidated)) ? 0 : -EAGAIN;
49}
50
51int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma)
52{
53 struct xe_vma *vma = &uvma->vma;
54 struct xe_vm *vm = xe_vma_vm(vma);
55 struct xe_device *xe = vm->xe;
56 struct drm_gpusvm_ctx ctx = {
57 .read_only = xe_vma_read_only(vma),
58 .device_private_page_owner = xe_svm_devm_owner(xe),
59 .allow_mixed = true,
60 };
61
62 lockdep_assert_held(&vm->lock);
63 xe_assert(xe, xe_vma_is_userptr(vma));
64
65 if (vma->gpuva.flags & XE_VMA_DESTROYED)
66 return 0;
67
68 return drm_gpusvm_get_pages(gpusvm: &vm->svm.gpusvm, svm_pages: &uvma->userptr.pages,
69 mm: uvma->userptr.notifier.mm,
70 notifier: &uvma->userptr.notifier,
71 pages_start: xe_vma_userptr(vma),
72 pages_end: xe_vma_userptr(vma) + xe_vma_size(vma),
73 ctx: &ctx);
74}
75
76static void __vma_userptr_invalidate(struct xe_vm *vm, struct xe_userptr_vma *uvma)
77{
78 struct xe_userptr *userptr = &uvma->userptr;
79 struct xe_vma *vma = &uvma->vma;
80 struct dma_resv_iter cursor;
81 struct dma_fence *fence;
82 struct drm_gpusvm_ctx ctx = {
83 .in_notifier = true,
84 .read_only = xe_vma_read_only(vma),
85 };
86 long err;
87
88 /*
89 * Tell exec and rebind worker they need to repin and rebind this
90 * userptr.
91 */
92 if (!xe_vm_in_fault_mode(vm) &&
93 !(vma->gpuva.flags & XE_VMA_DESTROYED)) {
94 spin_lock(lock: &vm->userptr.invalidated_lock);
95 list_move_tail(list: &userptr->invalidate_link,
96 head: &vm->userptr.invalidated);
97 spin_unlock(lock: &vm->userptr.invalidated_lock);
98 }
99
100 /*
101 * Preempt fences turn into schedule disables, pipeline these.
102 * Note that even in fault mode, we need to wait for binds and
103 * unbinds to complete, and those are attached as BOOKMARK fences
104 * to the vm.
105 */
106 dma_resv_iter_begin(cursor: &cursor, obj: xe_vm_resv(vm),
107 usage: DMA_RESV_USAGE_BOOKKEEP);
108 dma_resv_for_each_fence_unlocked(&cursor, fence)
109 dma_fence_enable_sw_signaling(fence);
110 dma_resv_iter_end(cursor: &cursor);
111
112 err = dma_resv_wait_timeout(obj: xe_vm_resv(vm),
113 usage: DMA_RESV_USAGE_BOOKKEEP,
114 intr: false, MAX_SCHEDULE_TIMEOUT);
115 XE_WARN_ON(err <= 0);
116
117 if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {
118 err = xe_vm_invalidate_vma(vma);
119 XE_WARN_ON(err);
120 }
121
122 drm_gpusvm_unmap_pages(gpusvm: &vm->svm.gpusvm, svm_pages: &uvma->userptr.pages,
123 npages: xe_vma_size(vma) >> PAGE_SHIFT, ctx: &ctx);
124}
125
126static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
127 const struct mmu_notifier_range *range,
128 unsigned long cur_seq)
129{
130 struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier);
131 struct xe_vma *vma = &uvma->vma;
132 struct xe_vm *vm = xe_vma_vm(vma);
133
134 xe_assert(vm->xe, xe_vma_is_userptr(vma));
135 trace_xe_vma_userptr_invalidate(vma);
136
137 if (!mmu_notifier_range_blockable(range))
138 return false;
139
140 vm_dbg(&xe_vma_vm(vma)->xe->drm,
141 "NOTIFIER: addr=0x%016llx, range=0x%016llx",
142 xe_vma_start(vma), xe_vma_size(vma));
143
144 down_write(sem: &vm->svm.gpusvm.notifier_lock);
145 mmu_interval_set_seq(interval_sub: mni, cur_seq);
146
147 __vma_userptr_invalidate(vm, uvma);
148 up_write(sem: &vm->svm.gpusvm.notifier_lock);
149 trace_xe_vma_userptr_invalidate_complete(vma);
150
151 return true;
152}
153
154static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = {
155 .invalidate = vma_userptr_invalidate,
156};
157
158#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
159/**
160 * xe_vma_userptr_force_invalidate() - force invalidate a userptr
161 * @uvma: The userptr vma to invalidate
162 *
163 * Perform a forced userptr invalidation for testing purposes.
164 */
165void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma)
166{
167 struct xe_vm *vm = xe_vma_vm(vma: &uvma->vma);
168
169 /* Protect against concurrent userptr pinning */
170 lockdep_assert_held(&vm->lock);
171 /* Protect against concurrent notifiers */
172 lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);
173 /*
174 * Protect against concurrent instances of this function and
175 * the critical exec sections
176 */
177 xe_vm_assert_held(vm);
178
179 if (!mmu_interval_read_retry(interval_sub: &uvma->userptr.notifier,
180 seq: uvma->userptr.pages.notifier_seq))
181 uvma->userptr.pages.notifier_seq -= 2;
182 __vma_userptr_invalidate(vm, uvma);
183}
184#endif
185
186int xe_vm_userptr_pin(struct xe_vm *vm)
187{
188 struct xe_userptr_vma *uvma, *next;
189 int err = 0;
190
191 xe_assert(vm->xe, !xe_vm_in_fault_mode(vm));
192 lockdep_assert_held_write(&vm->lock);
193
194 /* Collect invalidated userptrs */
195 spin_lock(lock: &vm->userptr.invalidated_lock);
196 xe_assert(vm->xe, list_empty(&vm->userptr.repin_list));
197 list_for_each_entry_safe(uvma, next, &vm->userptr.invalidated,
198 userptr.invalidate_link) {
199 list_del_init(entry: &uvma->userptr.invalidate_link);
200 list_add_tail(new: &uvma->userptr.repin_link,
201 head: &vm->userptr.repin_list);
202 }
203 spin_unlock(lock: &vm->userptr.invalidated_lock);
204
205 /* Pin and move to bind list */
206 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,
207 userptr.repin_link) {
208 err = xe_vma_userptr_pin_pages(uvma);
209 if (err == -EFAULT) {
210 list_del_init(entry: &uvma->userptr.repin_link);
211 /*
212 * We might have already done the pin once already, but
213 * then had to retry before the re-bind happened, due
214 * some other condition in the caller, but in the
215 * meantime the userptr got dinged by the notifier such
216 * that we need to revalidate here, but this time we hit
217 * the EFAULT. In such a case make sure we remove
218 * ourselves from the rebind list to avoid going down in
219 * flames.
220 */
221 if (!list_empty(head: &uvma->vma.combined_links.rebind))
222 list_del_init(entry: &uvma->vma.combined_links.rebind);
223
224 /* Wait for pending binds */
225 xe_vm_lock(vm, intr: false);
226 dma_resv_wait_timeout(obj: xe_vm_resv(vm),
227 usage: DMA_RESV_USAGE_BOOKKEEP,
228 intr: false, MAX_SCHEDULE_TIMEOUT);
229
230 down_read(sem: &vm->svm.gpusvm.notifier_lock);
231 err = xe_vm_invalidate_vma(vma: &uvma->vma);
232 up_read(sem: &vm->svm.gpusvm.notifier_lock);
233 xe_vm_unlock(vm);
234 if (err)
235 break;
236 } else {
237 if (err)
238 break;
239
240 list_del_init(entry: &uvma->userptr.repin_link);
241 list_move_tail(list: &uvma->vma.combined_links.rebind,
242 head: &vm->rebind_list);
243 }
244 }
245
246 if (err) {
247 down_write(sem: &vm->svm.gpusvm.notifier_lock);
248 spin_lock(lock: &vm->userptr.invalidated_lock);
249 list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,
250 userptr.repin_link) {
251 list_del_init(entry: &uvma->userptr.repin_link);
252 list_move_tail(list: &uvma->userptr.invalidate_link,
253 head: &vm->userptr.invalidated);
254 }
255 spin_unlock(lock: &vm->userptr.invalidated_lock);
256 up_write(sem: &vm->svm.gpusvm.notifier_lock);
257 }
258 return err;
259}
260
261/**
262 * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs
263 * that need repinning.
264 * @vm: The VM.
265 *
266 * This function does an advisory check for whether the VM has userptrs that
267 * need repinning.
268 *
269 * Return: 0 if there are no indications of userptrs needing repinning,
270 * -EAGAIN if there are.
271 */
272int xe_vm_userptr_check_repin(struct xe_vm *vm)
273{
274 return (list_empty_careful(head: &vm->userptr.repin_list) &&
275 list_empty_careful(head: &vm->userptr.invalidated)) ? 0 : -EAGAIN;
276}
277
278int xe_userptr_setup(struct xe_userptr_vma *uvma, unsigned long start,
279 unsigned long range)
280{
281 struct xe_userptr *userptr = &uvma->userptr;
282 int err;
283
284 INIT_LIST_HEAD(list: &userptr->invalidate_link);
285 INIT_LIST_HEAD(list: &userptr->repin_link);
286
287 err = mmu_interval_notifier_insert(interval_sub: &userptr->notifier, current->mm,
288 start, length: range,
289 ops: &vma_userptr_notifier_ops);
290 if (err)
291 return err;
292
293 userptr->pages.notifier_seq = LONG_MAX;
294
295 return 0;
296}
297
298void xe_userptr_remove(struct xe_userptr_vma *uvma)
299{
300 struct xe_vm *vm = xe_vma_vm(vma: &uvma->vma);
301 struct xe_userptr *userptr = &uvma->userptr;
302
303 drm_gpusvm_free_pages(gpusvm: &vm->svm.gpusvm, svm_pages: &uvma->userptr.pages,
304 npages: xe_vma_size(vma: &uvma->vma) >> PAGE_SHIFT);
305
306 /*
307 * Since userptr pages are not pinned, we can't remove
308 * the notifier until we're sure the GPU is not accessing
309 * them anymore
310 */
311 mmu_interval_notifier_remove(interval_sub: &userptr->notifier);
312}
313
314void xe_userptr_destroy(struct xe_userptr_vma *uvma)
315{
316 struct xe_vm *vm = xe_vma_vm(vma: &uvma->vma);
317
318 spin_lock(lock: &vm->userptr.invalidated_lock);
319 xe_assert(vm->xe, list_empty(&uvma->userptr.repin_link));
320 list_del(entry: &uvma->userptr.invalidate_link);
321 spin_unlock(lock: &vm->userptr.invalidated_lock);
322}
323

source code of linux/drivers/gpu/drm/xe/xe_userptr.c