1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/*
3 * Copyright © 2024 Intel Corporation
4 *
5 * Authors:
6 * Matthew Brost <matthew.brost@intel.com>
7 */
8
9#include <linux/dma-mapping.h>
10#include <linux/export.h>
11#include <linux/hmm.h>
12#include <linux/hugetlb_inline.h>
13#include <linux/memremap.h>
14#include <linux/mm_types.h>
15#include <linux/slab.h>
16
17#include <drm/drm_device.h>
18#include <drm/drm_gpusvm.h>
19#include <drm/drm_pagemap.h>
20#include <drm/drm_print.h>
21
22/**
23 * DOC: Overview
24 *
25 * GPU Shared Virtual Memory (GPU SVM) layer for the Direct Rendering Manager (DRM)
26 * is a component of the DRM framework designed to manage shared virtual memory
27 * between the CPU and GPU. It enables efficient data exchange and processing
28 * for GPU-accelerated applications by allowing memory sharing and
29 * synchronization between the CPU's and GPU's virtual address spaces.
30 *
31 * Key GPU SVM Components:
32 *
33 * - Notifiers:
34 * Used for tracking memory intervals and notifying the GPU of changes,
35 * notifiers are sized based on a GPU SVM initialization parameter, with a
36 * recommendation of 512M or larger. They maintain a Red-BlacK tree and a
37 * list of ranges that fall within the notifier interval. Notifiers are
38 * tracked within a GPU SVM Red-BlacK tree and list and are dynamically
39 * inserted or removed as ranges within the interval are created or
40 * destroyed.
41 * - Ranges:
42 * Represent memory ranges mapped in a DRM device and managed by GPU SVM.
43 * They are sized based on an array of chunk sizes, which is a GPU SVM
44 * initialization parameter, and the CPU address space. Upon GPU fault,
45 * the largest aligned chunk that fits within the faulting CPU address
46 * space is chosen for the range size. Ranges are expected to be
47 * dynamically allocated on GPU fault and removed on an MMU notifier UNMAP
48 * event. As mentioned above, ranges are tracked in a notifier's Red-Black
49 * tree.
50 *
51 * - Operations:
52 * Define the interface for driver-specific GPU SVM operations such as
53 * range allocation, notifier allocation, and invalidations.
54 *
55 * - Device Memory Allocations:
56 * Embedded structure containing enough information for GPU SVM to migrate
57 * to / from device memory.
58 *
59 * - Device Memory Operations:
60 * Define the interface for driver-specific device memory operations
61 * release memory, populate pfns, and copy to / from device memory.
62 *
63 * This layer provides interfaces for allocating, mapping, migrating, and
64 * releasing memory ranges between the CPU and GPU. It handles all core memory
65 * management interactions (DMA mapping, HMM, and migration) and provides
66 * driver-specific virtual functions (vfuncs). This infrastructure is sufficient
67 * to build the expected driver components for an SVM implementation as detailed
68 * below.
69 *
70 * Expected Driver Components:
71 *
72 * - GPU page fault handler:
73 * Used to create ranges and notifiers based on the fault address,
74 * optionally migrate the range to device memory, and create GPU bindings.
75 *
76 * - Garbage collector:
77 * Used to unmap and destroy GPU bindings for ranges. Ranges are expected
78 * to be added to the garbage collector upon a MMU_NOTIFY_UNMAP event in
79 * notifier callback.
80 *
81 * - Notifier callback:
82 * Used to invalidate and DMA unmap GPU bindings for ranges.
83 */
84
85/**
86 * DOC: Locking
87 *
88 * GPU SVM handles locking for core MM interactions, i.e., it locks/unlocks the
89 * mmap lock as needed.
90 *
91 * GPU SVM introduces a global notifier lock, which safeguards the notifier's
92 * range RB tree and list, as well as the range's DMA mappings and sequence
93 * number. GPU SVM manages all necessary locking and unlocking operations,
94 * except for the recheck range's pages being valid
95 * (drm_gpusvm_range_pages_valid) when the driver is committing GPU bindings.
96 * This lock corresponds to the ``driver->update`` lock mentioned in
97 * Documentation/mm/hmm.rst. Future revisions may transition from a GPU SVM
98 * global lock to a per-notifier lock if finer-grained locking is deemed
99 * necessary.
100 *
101 * In addition to the locking mentioned above, the driver should implement a
102 * lock to safeguard core GPU SVM function calls that modify state, such as
103 * drm_gpusvm_range_find_or_insert and drm_gpusvm_range_remove. This lock is
104 * denoted as 'driver_svm_lock' in code examples. Finer grained driver side
105 * locking should also be possible for concurrent GPU fault processing within a
106 * single GPU SVM. The 'driver_svm_lock' can be via drm_gpusvm_driver_set_lock
107 * to add annotations to GPU SVM.
108 */
109
110/**
111 * DOC: Partial Unmapping of Ranges
112 *
113 * Partial unmapping of ranges (e.g., 1M out of 2M is unmapped by CPU resulting
114 * in MMU_NOTIFY_UNMAP event) presents several challenges, with the main one
115 * being that a subset of the range still has CPU and GPU mappings. If the
116 * backing store for the range is in device memory, a subset of the backing
117 * store has references. One option would be to split the range and device
118 * memory backing store, but the implementation for this would be quite
119 * complicated. Given that partial unmappings are rare and driver-defined range
120 * sizes are relatively small, GPU SVM does not support splitting of ranges.
121 *
122 * With no support for range splitting, upon partial unmapping of a range, the
123 * driver is expected to invalidate and destroy the entire range. If the range
124 * has device memory as its backing, the driver is also expected to migrate any
125 * remaining pages back to RAM.
126 */
127
128/**
129 * DOC: Examples
130 *
131 * This section provides three examples of how to build the expected driver
132 * components: the GPU page fault handler, the garbage collector, and the
133 * notifier callback.
134 *
135 * The generic code provided does not include logic for complex migration
136 * policies, optimized invalidations, fined grained driver locking, or other
137 * potentially required driver locking (e.g., DMA-resv locks).
138 *
139 * 1) GPU page fault handler
140 *
141 * .. code-block:: c
142 *
143 * int driver_bind_range(struct drm_gpusvm *gpusvm, struct drm_gpusvm_range *range)
144 * {
145 * int err = 0;
146 *
147 * driver_alloc_and_setup_memory_for_bind(gpusvm, range);
148 *
149 * drm_gpusvm_notifier_lock(gpusvm);
150 * if (drm_gpusvm_range_pages_valid(range))
151 * driver_commit_bind(gpusvm, range);
152 * else
153 * err = -EAGAIN;
154 * drm_gpusvm_notifier_unlock(gpusvm);
155 *
156 * return err;
157 * }
158 *
159 * int driver_gpu_fault(struct drm_gpusvm *gpusvm, unsigned long fault_addr,
160 * unsigned long gpuva_start, unsigned long gpuva_end)
161 * {
162 * struct drm_gpusvm_ctx ctx = {};
163 * int err;
164 *
165 * driver_svm_lock();
166 * retry:
167 * // Always process UNMAPs first so view of GPU SVM ranges is current
168 * driver_garbage_collector(gpusvm);
169 *
170 * range = drm_gpusvm_range_find_or_insert(gpusvm, fault_addr,
171 * gpuva_start, gpuva_end,
172 * &ctx);
173 * if (IS_ERR(range)) {
174 * err = PTR_ERR(range);
175 * goto unlock;
176 * }
177 *
178 * if (driver_migration_policy(range)) {
179 * err = drm_pagemap_populate_mm(driver_choose_drm_pagemap(),
180 * gpuva_start, gpuva_end, gpusvm->mm,
181 * ctx->timeslice_ms);
182 * if (err) // CPU mappings may have changed
183 * goto retry;
184 * }
185 *
186 * err = drm_gpusvm_range_get_pages(gpusvm, range, &ctx);
187 * if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) { // CPU mappings changed
188 * if (err == -EOPNOTSUPP)
189 * drm_gpusvm_range_evict(gpusvm, range);
190 * goto retry;
191 * } else if (err) {
192 * goto unlock;
193 * }
194 *
195 * err = driver_bind_range(gpusvm, range);
196 * if (err == -EAGAIN) // CPU mappings changed
197 * goto retry
198 *
199 * unlock:
200 * driver_svm_unlock();
201 * return err;
202 * }
203 *
204 * 2) Garbage Collector
205 *
206 * .. code-block:: c
207 *
208 * void __driver_garbage_collector(struct drm_gpusvm *gpusvm,
209 * struct drm_gpusvm_range *range)
210 * {
211 * assert_driver_svm_locked(gpusvm);
212 *
213 * // Partial unmap, migrate any remaining device memory pages back to RAM
214 * if (range->flags.partial_unmap)
215 * drm_gpusvm_range_evict(gpusvm, range);
216 *
217 * driver_unbind_range(range);
218 * drm_gpusvm_range_remove(gpusvm, range);
219 * }
220 *
221 * void driver_garbage_collector(struct drm_gpusvm *gpusvm)
222 * {
223 * assert_driver_svm_locked(gpusvm);
224 *
225 * for_each_range_in_garbage_collector(gpusvm, range)
226 * __driver_garbage_collector(gpusvm, range);
227 * }
228 *
229 * 3) Notifier callback
230 *
231 * .. code-block:: c
232 *
233 * void driver_invalidation(struct drm_gpusvm *gpusvm,
234 * struct drm_gpusvm_notifier *notifier,
235 * const struct mmu_notifier_range *mmu_range)
236 * {
237 * struct drm_gpusvm_ctx ctx = { .in_notifier = true, };
238 * struct drm_gpusvm_range *range = NULL;
239 *
240 * driver_invalidate_device_pages(gpusvm, mmu_range->start, mmu_range->end);
241 *
242 * drm_gpusvm_for_each_range(range, notifier, mmu_range->start,
243 * mmu_range->end) {
244 * drm_gpusvm_range_unmap_pages(gpusvm, range, &ctx);
245 *
246 * if (mmu_range->event != MMU_NOTIFY_UNMAP)
247 * continue;
248 *
249 * drm_gpusvm_range_set_unmapped(range, mmu_range);
250 * driver_garbage_collector_add(gpusvm, range);
251 * }
252 * }
253 */
254
255/**
256 * npages_in_range() - Calculate the number of pages in a given range
257 * @start: The start address of the range
258 * @end: The end address of the range
259 *
260 * This macro calculates the number of pages in a given memory range,
261 * specified by the start and end addresses. It divides the difference
262 * between the end and start addresses by the page size (PAGE_SIZE) to
263 * determine the number of pages in the range.
264 *
265 * Return: The number of pages in the specified range.
266 */
267static unsigned long
268npages_in_range(unsigned long start, unsigned long end)
269{
270 return (end - start) >> PAGE_SHIFT;
271}
272
273/**
274 * drm_gpusvm_notifier_find() - Find GPU SVM notifier from GPU SVM
275 * @gpusvm: Pointer to the GPU SVM structure.
276 * @start: Start address of the notifier
277 * @end: End address of the notifier
278 *
279 * Return: A pointer to the drm_gpusvm_notifier if found or NULL
280 */
281struct drm_gpusvm_notifier *
282drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
283 unsigned long end)
284{
285 struct interval_tree_node *itree;
286
287 itree = interval_tree_iter_first(root: &gpusvm->root, start, last: end - 1);
288
289 if (itree)
290 return container_of(itree, struct drm_gpusvm_notifier, itree);
291 else
292 return NULL;
293}
294EXPORT_SYMBOL_GPL(drm_gpusvm_notifier_find);
295
296/**
297 * drm_gpusvm_range_find() - Find GPU SVM range from GPU SVM notifier
298 * @notifier: Pointer to the GPU SVM notifier structure.
299 * @start: Start address of the range
300 * @end: End address of the range
301 *
302 * Return: A pointer to the drm_gpusvm_range if found or NULL
303 */
304struct drm_gpusvm_range *
305drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
306 unsigned long end)
307{
308 struct interval_tree_node *itree;
309
310 itree = interval_tree_iter_first(root: &notifier->root, start, last: end - 1);
311
312 if (itree)
313 return container_of(itree, struct drm_gpusvm_range, itree);
314 else
315 return NULL;
316}
317EXPORT_SYMBOL_GPL(drm_gpusvm_range_find);
318
319/**
320 * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier.
321 * @mni: Pointer to the mmu_interval_notifier structure.
322 * @mmu_range: Pointer to the mmu_notifier_range structure.
323 * @cur_seq: Current sequence number.
324 *
325 * This function serves as a generic MMU notifier for GPU SVM. It sets the MMU
326 * notifier sequence number and calls the driver invalidate vfunc under
327 * gpusvm->notifier_lock.
328 *
329 * Return: true if the operation succeeds, false otherwise.
330 */
331static bool
332drm_gpusvm_notifier_invalidate(struct mmu_interval_notifier *mni,
333 const struct mmu_notifier_range *mmu_range,
334 unsigned long cur_seq)
335{
336 struct drm_gpusvm_notifier *notifier =
337 container_of(mni, typeof(*notifier), notifier);
338 struct drm_gpusvm *gpusvm = notifier->gpusvm;
339
340 if (!mmu_notifier_range_blockable(range: mmu_range))
341 return false;
342
343 down_write(sem: &gpusvm->notifier_lock);
344 mmu_interval_set_seq(interval_sub: mni, cur_seq);
345 gpusvm->ops->invalidate(gpusvm, notifier, mmu_range);
346 up_write(sem: &gpusvm->notifier_lock);
347
348 return true;
349}
350
351/*
352 * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM
353 */
354static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {
355 .invalidate = drm_gpusvm_notifier_invalidate,
356};
357
358/**
359 * drm_gpusvm_init() - Initialize the GPU SVM.
360 * @gpusvm: Pointer to the GPU SVM structure.
361 * @name: Name of the GPU SVM.
362 * @drm: Pointer to the DRM device structure.
363 * @mm: Pointer to the mm_struct for the address space.
364 * @mm_start: Start address of GPU SVM.
365 * @mm_range: Range of the GPU SVM.
366 * @notifier_size: Size of individual notifiers.
367 * @ops: Pointer to the operations structure for GPU SVM.
368 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation.
369 * Entries should be powers of 2 in descending order with last
370 * entry being SZ_4K.
371 * @num_chunks: Number of chunks.
372 *
373 * This function initializes the GPU SVM.
374 *
375 * Note: If only using the simple drm_gpusvm_pages API (get/unmap/free),
376 * then only @gpusvm, @name, and @drm are expected. However, the same base
377 * @gpusvm can also be used with both modes together in which case the full
378 * setup is needed, where the core drm_gpusvm_pages API will simply never use
379 * the other fields.
380 *
381 * Return: 0 on success, a negative error code on failure.
382 */
383int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
384 const char *name, struct drm_device *drm,
385 struct mm_struct *mm,
386 unsigned long mm_start, unsigned long mm_range,
387 unsigned long notifier_size,
388 const struct drm_gpusvm_ops *ops,
389 const unsigned long *chunk_sizes, int num_chunks)
390{
391 if (mm) {
392 if (!ops->invalidate || !num_chunks)
393 return -EINVAL;
394 mmgrab(mm);
395 } else {
396 /* No full SVM mode, only core drm_gpusvm_pages API. */
397 if (ops || num_chunks || mm_range || notifier_size)
398 return -EINVAL;
399 }
400
401 gpusvm->name = name;
402 gpusvm->drm = drm;
403 gpusvm->mm = mm;
404 gpusvm->mm_start = mm_start;
405 gpusvm->mm_range = mm_range;
406 gpusvm->notifier_size = notifier_size;
407 gpusvm->ops = ops;
408 gpusvm->chunk_sizes = chunk_sizes;
409 gpusvm->num_chunks = num_chunks;
410
411 gpusvm->root = RB_ROOT_CACHED;
412 INIT_LIST_HEAD(list: &gpusvm->notifier_list);
413
414 init_rwsem(&gpusvm->notifier_lock);
415
416 fs_reclaim_acquire(GFP_KERNEL);
417 might_lock(&gpusvm->notifier_lock);
418 fs_reclaim_release(GFP_KERNEL);
419
420#ifdef CONFIG_LOCKDEP
421 gpusvm->lock_dep_map = NULL;
422#endif
423
424 return 0;
425}
426EXPORT_SYMBOL_GPL(drm_gpusvm_init);
427
428/**
429 * to_drm_gpusvm_notifier() - retrieve the container struct for a given rbtree node
430 * @node: a pointer to the rbtree node embedded within a drm_gpusvm_notifier struct
431 *
432 * Return: A pointer to the containing drm_gpusvm_notifier structure.
433 */
434static struct drm_gpusvm_notifier *to_drm_gpusvm_notifier(struct rb_node *node)
435{
436 return container_of(node, struct drm_gpusvm_notifier, itree.rb);
437}
438
439/**
440 * drm_gpusvm_notifier_insert() - Insert GPU SVM notifier
441 * @gpusvm: Pointer to the GPU SVM structure
442 * @notifier: Pointer to the GPU SVM notifier structure
443 *
444 * This function inserts the GPU SVM notifier into the GPU SVM RB tree and list.
445 */
446static void drm_gpusvm_notifier_insert(struct drm_gpusvm *gpusvm,
447 struct drm_gpusvm_notifier *notifier)
448{
449 struct rb_node *node;
450 struct list_head *head;
451
452 interval_tree_insert(node: &notifier->itree, root: &gpusvm->root);
453
454 node = rb_prev(&notifier->itree.rb);
455 if (node)
456 head = &(to_drm_gpusvm_notifier(node))->entry;
457 else
458 head = &gpusvm->notifier_list;
459
460 list_add(new: &notifier->entry, head);
461}
462
463/**
464 * drm_gpusvm_notifier_remove() - Remove GPU SVM notifier
465 * @gpusvm: Pointer to the GPU SVM tructure
466 * @notifier: Pointer to the GPU SVM notifier structure
467 *
468 * This function removes the GPU SVM notifier from the GPU SVM RB tree and list.
469 */
470static void drm_gpusvm_notifier_remove(struct drm_gpusvm *gpusvm,
471 struct drm_gpusvm_notifier *notifier)
472{
473 interval_tree_remove(node: &notifier->itree, root: &gpusvm->root);
474 list_del(entry: &notifier->entry);
475}
476
477/**
478 * drm_gpusvm_fini() - Finalize the GPU SVM.
479 * @gpusvm: Pointer to the GPU SVM structure.
480 *
481 * This function finalizes the GPU SVM by cleaning up any remaining ranges and
482 * notifiers, and dropping a reference to struct MM.
483 */
484void drm_gpusvm_fini(struct drm_gpusvm *gpusvm)
485{
486 struct drm_gpusvm_notifier *notifier, *next;
487
488 drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) {
489 struct drm_gpusvm_range *range, *__next;
490
491 /*
492 * Remove notifier first to avoid racing with any invalidation
493 */
494 mmu_interval_notifier_remove(interval_sub: &notifier->notifier);
495 notifier->flags.removed = true;
496
497 drm_gpusvm_for_each_range_safe(range, __next, notifier, 0,
498 LONG_MAX)
499 drm_gpusvm_range_remove(gpusvm, range);
500 }
501
502 if (gpusvm->mm)
503 mmdrop(mm: gpusvm->mm);
504 WARN_ON(!RB_EMPTY_ROOT(&gpusvm->root.rb_root));
505}
506EXPORT_SYMBOL_GPL(drm_gpusvm_fini);
507
508/**
509 * drm_gpusvm_notifier_alloc() - Allocate GPU SVM notifier
510 * @gpusvm: Pointer to the GPU SVM structure
511 * @fault_addr: Fault address
512 *
513 * This function allocates and initializes the GPU SVM notifier structure.
514 *
515 * Return: Pointer to the allocated GPU SVM notifier on success, ERR_PTR() on failure.
516 */
517static struct drm_gpusvm_notifier *
518drm_gpusvm_notifier_alloc(struct drm_gpusvm *gpusvm, unsigned long fault_addr)
519{
520 struct drm_gpusvm_notifier *notifier;
521
522 if (gpusvm->ops->notifier_alloc)
523 notifier = gpusvm->ops->notifier_alloc();
524 else
525 notifier = kzalloc(sizeof(*notifier), GFP_KERNEL);
526
527 if (!notifier)
528 return ERR_PTR(error: -ENOMEM);
529
530 notifier->gpusvm = gpusvm;
531 notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size);
532 notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1;
533 INIT_LIST_HEAD(list: &notifier->entry);
534 notifier->root = RB_ROOT_CACHED;
535 INIT_LIST_HEAD(list: &notifier->range_list);
536
537 return notifier;
538}
539
540/**
541 * drm_gpusvm_notifier_free() - Free GPU SVM notifier
542 * @gpusvm: Pointer to the GPU SVM structure
543 * @notifier: Pointer to the GPU SVM notifier structure
544 *
545 * This function frees the GPU SVM notifier structure.
546 */
547static void drm_gpusvm_notifier_free(struct drm_gpusvm *gpusvm,
548 struct drm_gpusvm_notifier *notifier)
549{
550 WARN_ON(!RB_EMPTY_ROOT(&notifier->root.rb_root));
551
552 if (gpusvm->ops->notifier_free)
553 gpusvm->ops->notifier_free(notifier);
554 else
555 kfree(objp: notifier);
556}
557
558/**
559 * to_drm_gpusvm_range() - retrieve the container struct for a given rbtree node
560 * @node: a pointer to the rbtree node embedded within a drm_gpusvm_range struct
561 *
562 * Return: A pointer to the containing drm_gpusvm_range structure.
563 */
564static struct drm_gpusvm_range *to_drm_gpusvm_range(struct rb_node *node)
565{
566 return container_of(node, struct drm_gpusvm_range, itree.rb);
567}
568
569/**
570 * drm_gpusvm_range_insert() - Insert GPU SVM range
571 * @notifier: Pointer to the GPU SVM notifier structure
572 * @range: Pointer to the GPU SVM range structure
573 *
574 * This function inserts the GPU SVM range into the notifier RB tree and list.
575 */
576static void drm_gpusvm_range_insert(struct drm_gpusvm_notifier *notifier,
577 struct drm_gpusvm_range *range)
578{
579 struct rb_node *node;
580 struct list_head *head;
581
582 drm_gpusvm_notifier_lock(notifier->gpusvm);
583 interval_tree_insert(node: &range->itree, root: &notifier->root);
584
585 node = rb_prev(&range->itree.rb);
586 if (node)
587 head = &(to_drm_gpusvm_range(node))->entry;
588 else
589 head = &notifier->range_list;
590
591 list_add(new: &range->entry, head);
592 drm_gpusvm_notifier_unlock(notifier->gpusvm);
593}
594
595/**
596 * __drm_gpusvm_range_remove() - Remove GPU SVM range
597 * @notifier: Pointer to the GPU SVM notifier structure
598 * @range: Pointer to the GPU SVM range structure
599 *
600 * This macro removes the GPU SVM range from the notifier RB tree and list.
601 */
602static void __drm_gpusvm_range_remove(struct drm_gpusvm_notifier *notifier,
603 struct drm_gpusvm_range *range)
604{
605 interval_tree_remove(node: &range->itree, root: &notifier->root);
606 list_del(entry: &range->entry);
607}
608
609/**
610 * drm_gpusvm_range_alloc() - Allocate GPU SVM range
611 * @gpusvm: Pointer to the GPU SVM structure
612 * @notifier: Pointer to the GPU SVM notifier structure
613 * @fault_addr: Fault address
614 * @chunk_size: Chunk size
615 * @migrate_devmem: Flag indicating whether to migrate device memory
616 *
617 * This function allocates and initializes the GPU SVM range structure.
618 *
619 * Return: Pointer to the allocated GPU SVM range on success, ERR_PTR() on failure.
620 */
621static struct drm_gpusvm_range *
622drm_gpusvm_range_alloc(struct drm_gpusvm *gpusvm,
623 struct drm_gpusvm_notifier *notifier,
624 unsigned long fault_addr, unsigned long chunk_size,
625 bool migrate_devmem)
626{
627 struct drm_gpusvm_range *range;
628
629 if (gpusvm->ops->range_alloc)
630 range = gpusvm->ops->range_alloc(gpusvm);
631 else
632 range = kzalloc(sizeof(*range), GFP_KERNEL);
633
634 if (!range)
635 return ERR_PTR(error: -ENOMEM);
636
637 kref_init(kref: &range->refcount);
638 range->gpusvm = gpusvm;
639 range->notifier = notifier;
640 range->itree.start = ALIGN_DOWN(fault_addr, chunk_size);
641 range->itree.last = ALIGN(fault_addr + 1, chunk_size) - 1;
642 INIT_LIST_HEAD(list: &range->entry);
643 range->pages.notifier_seq = LONG_MAX;
644 range->pages.flags.migrate_devmem = migrate_devmem ? 1 : 0;
645
646 return range;
647}
648
649/**
650 * drm_gpusvm_hmm_pfn_to_order() - Get the largest CPU mapping order.
651 * @hmm_pfn: The current hmm_pfn.
652 * @hmm_pfn_index: Index of the @hmm_pfn within the pfn array.
653 * @npages: Number of pages within the pfn array i.e the hmm range size.
654 *
655 * To allow skipping PFNs with the same flags (like when they belong to
656 * the same huge PTE) when looping over the pfn array, take a given a hmm_pfn,
657 * and return the largest order that will fit inside the CPU PTE, but also
658 * crucially accounting for the original hmm range boundaries.
659 *
660 * Return: The largest order that will safely fit within the size of the hmm_pfn
661 * CPU PTE.
662 */
663static unsigned int drm_gpusvm_hmm_pfn_to_order(unsigned long hmm_pfn,
664 unsigned long hmm_pfn_index,
665 unsigned long npages)
666{
667 unsigned long size;
668
669 size = 1UL << hmm_pfn_to_map_order(hmm_pfn);
670 size -= (hmm_pfn & ~HMM_PFN_FLAGS) & (size - 1);
671 hmm_pfn_index += size;
672 if (hmm_pfn_index > npages)
673 size -= (hmm_pfn_index - npages);
674
675 return ilog2(size);
676}
677
678/**
679 * drm_gpusvm_check_pages() - Check pages
680 * @gpusvm: Pointer to the GPU SVM structure
681 * @notifier: Pointer to the GPU SVM notifier structure
682 * @start: Start address
683 * @end: End address
684 * @dev_private_owner: The device private page owner
685 *
686 * Check if pages between start and end have been faulted in on the CPU. Use to
687 * prevent migration of pages without CPU backing store.
688 *
689 * Return: True if pages have been faulted into CPU, False otherwise
690 */
691static bool drm_gpusvm_check_pages(struct drm_gpusvm *gpusvm,
692 struct drm_gpusvm_notifier *notifier,
693 unsigned long start, unsigned long end,
694 void *dev_private_owner)
695{
696 struct hmm_range hmm_range = {
697 .default_flags = 0,
698 .notifier = &notifier->notifier,
699 .start = start,
700 .end = end,
701 .dev_private_owner = dev_private_owner,
702 };
703 unsigned long timeout =
704 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
705 unsigned long *pfns;
706 unsigned long npages = npages_in_range(start, end);
707 int err, i;
708
709 mmap_assert_locked(mm: gpusvm->mm);
710
711 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
712 if (!pfns)
713 return false;
714
715 hmm_range.notifier_seq = mmu_interval_read_begin(interval_sub: &notifier->notifier);
716 hmm_range.hmm_pfns = pfns;
717
718 while (true) {
719 err = hmm_range_fault(range: &hmm_range);
720 if (err == -EBUSY) {
721 if (time_after(jiffies, timeout))
722 break;
723
724 hmm_range.notifier_seq =
725 mmu_interval_read_begin(interval_sub: &notifier->notifier);
726 continue;
727 }
728 break;
729 }
730 if (err)
731 goto err_free;
732
733 for (i = 0; i < npages;) {
734 if (!(pfns[i] & HMM_PFN_VALID)) {
735 err = -EFAULT;
736 goto err_free;
737 }
738 i += 0x1 << drm_gpusvm_hmm_pfn_to_order(hmm_pfn: pfns[i], hmm_pfn_index: i, npages);
739 }
740
741err_free:
742 kvfree(addr: pfns);
743 return err ? false : true;
744}
745
746/**
747 * drm_gpusvm_range_chunk_size() - Determine chunk size for GPU SVM range
748 * @gpusvm: Pointer to the GPU SVM structure
749 * @notifier: Pointer to the GPU SVM notifier structure
750 * @vas: Pointer to the virtual memory area structure
751 * @fault_addr: Fault address
752 * @gpuva_start: Start address of GPUVA which mirrors CPU
753 * @gpuva_end: End address of GPUVA which mirrors CPU
754 * @check_pages_threshold: Check CPU pages for present threshold
755 * @dev_private_owner: The device private page owner
756 *
757 * This function determines the chunk size for the GPU SVM range based on the
758 * fault address, GPU SVM chunk sizes, existing GPU SVM ranges, and the virtual
759 * memory area boundaries.
760 *
761 * Return: Chunk size on success, LONG_MAX on failure.
762 */
763static unsigned long
764drm_gpusvm_range_chunk_size(struct drm_gpusvm *gpusvm,
765 struct drm_gpusvm_notifier *notifier,
766 struct vm_area_struct *vas,
767 unsigned long fault_addr,
768 unsigned long gpuva_start,
769 unsigned long gpuva_end,
770 unsigned long check_pages_threshold,
771 void *dev_private_owner)
772{
773 unsigned long start, end;
774 int i = 0;
775
776retry:
777 for (; i < gpusvm->num_chunks; ++i) {
778 start = ALIGN_DOWN(fault_addr, gpusvm->chunk_sizes[i]);
779 end = ALIGN(fault_addr + 1, gpusvm->chunk_sizes[i]);
780
781 if (start >= vas->vm_start && end <= vas->vm_end &&
782 start >= drm_gpusvm_notifier_start(notifier) &&
783 end <= drm_gpusvm_notifier_end(notifier) &&
784 start >= gpuva_start && end <= gpuva_end)
785 break;
786 }
787
788 if (i == gpusvm->num_chunks)
789 return LONG_MAX;
790
791 /*
792 * If allocation more than page, ensure not to overlap with existing
793 * ranges.
794 */
795 if (end - start != SZ_4K) {
796 struct drm_gpusvm_range *range;
797
798 range = drm_gpusvm_range_find(notifier, start, end);
799 if (range) {
800 ++i;
801 goto retry;
802 }
803
804 /*
805 * XXX: Only create range on pages CPU has faulted in. Without
806 * this check, or prefault, on BMG 'xe_exec_system_allocator --r
807 * process-many-malloc' fails. In the failure case, each process
808 * mallocs 16k but the CPU VMA is ~128k which results in 64k SVM
809 * ranges. When migrating the SVM ranges, some processes fail in
810 * drm_pagemap_migrate_to_devmem with 'migrate.cpages != npages'
811 * and then upon drm_gpusvm_range_get_pages device pages from
812 * other processes are collected + faulted in which creates all
813 * sorts of problems. Unsure exactly how this happening, also
814 * problem goes away if 'xe_exec_system_allocator --r
815 * process-many-malloc' mallocs at least 64k at a time.
816 */
817 if (end - start <= check_pages_threshold &&
818 !drm_gpusvm_check_pages(gpusvm, notifier, start, end, dev_private_owner)) {
819 ++i;
820 goto retry;
821 }
822 }
823
824 return end - start;
825}
826
827#ifdef CONFIG_LOCKDEP
828/**
829 * drm_gpusvm_driver_lock_held() - Assert GPU SVM driver lock is held
830 * @gpusvm: Pointer to the GPU SVM structure.
831 *
832 * Ensure driver lock is held.
833 */
834static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm)
835{
836 if ((gpusvm)->lock_dep_map)
837 lockdep_assert(lock_is_held_type((gpusvm)->lock_dep_map, 0));
838}
839#else
840static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm)
841{
842}
843#endif
844
845/**
846 * drm_gpusvm_find_vma_start() - Find start address for first VMA in range
847 * @gpusvm: Pointer to the GPU SVM structure
848 * @start: The inclusive start user address.
849 * @end: The exclusive end user address.
850 *
851 * Returns: The start address of first VMA within the provided range,
852 * ULONG_MAX otherwise. Assumes start_addr < end_addr.
853 */
854unsigned long
855drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
856 unsigned long start,
857 unsigned long end)
858{
859 struct mm_struct *mm = gpusvm->mm;
860 struct vm_area_struct *vma;
861 unsigned long addr = ULONG_MAX;
862
863 if (!mmget_not_zero(mm))
864 return addr;
865
866 mmap_read_lock(mm);
867
868 vma = find_vma_intersection(mm, start_addr: start, end_addr: end);
869 if (vma)
870 addr = vma->vm_start;
871
872 mmap_read_unlock(mm);
873 mmput(mm);
874
875 return addr;
876}
877EXPORT_SYMBOL_GPL(drm_gpusvm_find_vma_start);
878
879/**
880 * drm_gpusvm_range_find_or_insert() - Find or insert GPU SVM range
881 * @gpusvm: Pointer to the GPU SVM structure
882 * @fault_addr: Fault address
883 * @gpuva_start: Start address of GPUVA which mirrors CPU
884 * @gpuva_end: End address of GPUVA which mirrors CPU
885 * @ctx: GPU SVM context
886 *
887 * This function finds or inserts a newly allocated a GPU SVM range based on the
888 * fault address. Caller must hold a lock to protect range lookup and insertion.
889 *
890 * Return: Pointer to the GPU SVM range on success, ERR_PTR() on failure.
891 */
892struct drm_gpusvm_range *
893drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
894 unsigned long fault_addr,
895 unsigned long gpuva_start,
896 unsigned long gpuva_end,
897 const struct drm_gpusvm_ctx *ctx)
898{
899 struct drm_gpusvm_notifier *notifier;
900 struct drm_gpusvm_range *range;
901 struct mm_struct *mm = gpusvm->mm;
902 struct vm_area_struct *vas;
903 bool notifier_alloc = false;
904 unsigned long chunk_size;
905 int err;
906 bool migrate_devmem;
907
908 drm_gpusvm_driver_lock_held(gpusvm);
909
910 if (fault_addr < gpusvm->mm_start ||
911 fault_addr > gpusvm->mm_start + gpusvm->mm_range)
912 return ERR_PTR(error: -EINVAL);
913
914 if (!mmget_not_zero(mm))
915 return ERR_PTR(error: -EFAULT);
916
917 notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr, fault_addr + 1);
918 if (!notifier) {
919 notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr);
920 if (IS_ERR(ptr: notifier)) {
921 err = PTR_ERR(ptr: notifier);
922 goto err_mmunlock;
923 }
924 notifier_alloc = true;
925 err = mmu_interval_notifier_insert(interval_sub: &notifier->notifier,
926 mm,
927 start: drm_gpusvm_notifier_start(notifier),
928 length: drm_gpusvm_notifier_size(notifier),
929 ops: &drm_gpusvm_notifier_ops);
930 if (err)
931 goto err_notifier;
932 }
933
934 mmap_read_lock(mm);
935
936 vas = vma_lookup(mm, addr: fault_addr);
937 if (!vas) {
938 err = -ENOENT;
939 goto err_notifier_remove;
940 }
941
942 if (!ctx->read_only && !(vas->vm_flags & VM_WRITE)) {
943 err = -EPERM;
944 goto err_notifier_remove;
945 }
946
947 range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1);
948 if (range)
949 goto out_mmunlock;
950 /*
951 * XXX: Short-circuiting migration based on migrate_vma_* current
952 * limitations. If/when migrate_vma_* add more support, this logic will
953 * have to change.
954 */
955 migrate_devmem = ctx->devmem_possible &&
956 vma_is_anonymous(vma: vas) && !is_vm_hugetlb_page(vma: vas);
957
958 chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas,
959 fault_addr, gpuva_start,
960 gpuva_end,
961 check_pages_threshold: ctx->check_pages_threshold,
962 dev_private_owner: ctx->device_private_page_owner);
963 if (chunk_size == LONG_MAX) {
964 err = -EINVAL;
965 goto err_notifier_remove;
966 }
967
968 range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size,
969 migrate_devmem);
970 if (IS_ERR(ptr: range)) {
971 err = PTR_ERR(ptr: range);
972 goto err_notifier_remove;
973 }
974
975 drm_gpusvm_range_insert(notifier, range);
976 if (notifier_alloc)
977 drm_gpusvm_notifier_insert(gpusvm, notifier);
978
979out_mmunlock:
980 mmap_read_unlock(mm);
981 mmput(mm);
982
983 return range;
984
985err_notifier_remove:
986 mmap_read_unlock(mm);
987 if (notifier_alloc)
988 mmu_interval_notifier_remove(interval_sub: &notifier->notifier);
989err_notifier:
990 if (notifier_alloc)
991 drm_gpusvm_notifier_free(gpusvm, notifier);
992err_mmunlock:
993 mmput(mm);
994 return ERR_PTR(error: err);
995}
996EXPORT_SYMBOL_GPL(drm_gpusvm_range_find_or_insert);
997
998/**
999 * __drm_gpusvm_unmap_pages() - Unmap pages associated with GPU SVM pages (internal)
1000 * @gpusvm: Pointer to the GPU SVM structure
1001 * @svm_pages: Pointer to the GPU SVM pages structure
1002 * @npages: Number of pages to unmap
1003 *
1004 * This function unmap pages associated with a GPU SVM pages struct. Assumes and
1005 * asserts correct locking is in place when called.
1006 */
1007static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
1008 struct drm_gpusvm_pages *svm_pages,
1009 unsigned long npages)
1010{
1011 struct drm_pagemap *dpagemap = svm_pages->dpagemap;
1012 struct device *dev = gpusvm->drm->dev;
1013 unsigned long i, j;
1014
1015 lockdep_assert_held(&gpusvm->notifier_lock);
1016
1017 if (svm_pages->flags.has_dma_mapping) {
1018 struct drm_gpusvm_pages_flags flags = {
1019 .__flags = svm_pages->flags.__flags,
1020 };
1021
1022 for (i = 0, j = 0; i < npages; j++) {
1023 struct drm_pagemap_addr *addr = &svm_pages->dma_addr[j];
1024
1025 if (addr->proto == DRM_INTERCONNECT_SYSTEM)
1026 dma_unmap_page(dev,
1027 addr->addr,
1028 PAGE_SIZE << addr->order,
1029 addr->dir);
1030 else if (dpagemap && dpagemap->ops->device_unmap)
1031 dpagemap->ops->device_unmap(dpagemap,
1032 dev, *addr);
1033 i += 1 << addr->order;
1034 }
1035
1036 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1037 flags.has_devmem_pages = false;
1038 flags.has_dma_mapping = false;
1039 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1040
1041 svm_pages->dpagemap = NULL;
1042 }
1043}
1044
1045/**
1046 * __drm_gpusvm_free_pages() - Free dma array associated with GPU SVM pages
1047 * @gpusvm: Pointer to the GPU SVM structure
1048 * @svm_pages: Pointer to the GPU SVM pages structure
1049 *
1050 * This function frees the dma address array associated with a GPU SVM range.
1051 */
1052static void __drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
1053 struct drm_gpusvm_pages *svm_pages)
1054{
1055 lockdep_assert_held(&gpusvm->notifier_lock);
1056
1057 if (svm_pages->dma_addr) {
1058 kvfree(addr: svm_pages->dma_addr);
1059 svm_pages->dma_addr = NULL;
1060 }
1061}
1062
1063/**
1064 * drm_gpusvm_free_pages() - Free dma-mapping associated with GPU SVM pages
1065 * struct
1066 * @gpusvm: Pointer to the GPU SVM structure
1067 * @svm_pages: Pointer to the GPU SVM pages structure
1068 * @npages: Number of mapped pages
1069 *
1070 * This function unmaps and frees the dma address array associated with a GPU
1071 * SVM pages struct.
1072 */
1073void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
1074 struct drm_gpusvm_pages *svm_pages,
1075 unsigned long npages)
1076{
1077 drm_gpusvm_notifier_lock(gpusvm);
1078 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages);
1079 __drm_gpusvm_free_pages(gpusvm, svm_pages);
1080 drm_gpusvm_notifier_unlock(gpusvm);
1081}
1082EXPORT_SYMBOL_GPL(drm_gpusvm_free_pages);
1083
1084/**
1085 * drm_gpusvm_range_remove() - Remove GPU SVM range
1086 * @gpusvm: Pointer to the GPU SVM structure
1087 * @range: Pointer to the GPU SVM range to be removed
1088 *
1089 * This function removes the specified GPU SVM range and also removes the parent
1090 * GPU SVM notifier if no more ranges remain in the notifier. The caller must
1091 * hold a lock to protect range and notifier removal.
1092 */
1093void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
1094 struct drm_gpusvm_range *range)
1095{
1096 unsigned long npages = npages_in_range(start: drm_gpusvm_range_start(range),
1097 end: drm_gpusvm_range_end(range));
1098 struct drm_gpusvm_notifier *notifier;
1099
1100 drm_gpusvm_driver_lock_held(gpusvm);
1101
1102 notifier = drm_gpusvm_notifier_find(gpusvm,
1103 drm_gpusvm_range_start(range),
1104 drm_gpusvm_range_start(range) + 1);
1105 if (WARN_ON_ONCE(!notifier))
1106 return;
1107
1108 drm_gpusvm_notifier_lock(gpusvm);
1109 __drm_gpusvm_unmap_pages(gpusvm, svm_pages: &range->pages, npages);
1110 __drm_gpusvm_free_pages(gpusvm, svm_pages: &range->pages);
1111 __drm_gpusvm_range_remove(notifier, range);
1112 drm_gpusvm_notifier_unlock(gpusvm);
1113
1114 drm_gpusvm_range_put(range);
1115
1116 if (RB_EMPTY_ROOT(&notifier->root.rb_root)) {
1117 if (!notifier->flags.removed)
1118 mmu_interval_notifier_remove(interval_sub: &notifier->notifier);
1119 drm_gpusvm_notifier_remove(gpusvm, notifier);
1120 drm_gpusvm_notifier_free(gpusvm, notifier);
1121 }
1122}
1123EXPORT_SYMBOL_GPL(drm_gpusvm_range_remove);
1124
1125/**
1126 * drm_gpusvm_range_get() - Get a reference to GPU SVM range
1127 * @range: Pointer to the GPU SVM range
1128 *
1129 * This function increments the reference count of the specified GPU SVM range.
1130 *
1131 * Return: Pointer to the GPU SVM range.
1132 */
1133struct drm_gpusvm_range *
1134drm_gpusvm_range_get(struct drm_gpusvm_range *range)
1135{
1136 kref_get(kref: &range->refcount);
1137
1138 return range;
1139}
1140EXPORT_SYMBOL_GPL(drm_gpusvm_range_get);
1141
1142/**
1143 * drm_gpusvm_range_destroy() - Destroy GPU SVM range
1144 * @refcount: Pointer to the reference counter embedded in the GPU SVM range
1145 *
1146 * This function destroys the specified GPU SVM range when its reference count
1147 * reaches zero. If a custom range-free function is provided, it is invoked to
1148 * free the range; otherwise, the range is deallocated using kfree().
1149 */
1150static void drm_gpusvm_range_destroy(struct kref *refcount)
1151{
1152 struct drm_gpusvm_range *range =
1153 container_of(refcount, struct drm_gpusvm_range, refcount);
1154 struct drm_gpusvm *gpusvm = range->gpusvm;
1155
1156 if (gpusvm->ops->range_free)
1157 gpusvm->ops->range_free(range);
1158 else
1159 kfree(objp: range);
1160}
1161
1162/**
1163 * drm_gpusvm_range_put() - Put a reference to GPU SVM range
1164 * @range: Pointer to the GPU SVM range
1165 *
1166 * This function decrements the reference count of the specified GPU SVM range
1167 * and frees it when the count reaches zero.
1168 */
1169void drm_gpusvm_range_put(struct drm_gpusvm_range *range)
1170{
1171 kref_put(kref: &range->refcount, release: drm_gpusvm_range_destroy);
1172}
1173EXPORT_SYMBOL_GPL(drm_gpusvm_range_put);
1174
1175/**
1176 * drm_gpusvm_pages_valid() - GPU SVM range pages valid
1177 * @gpusvm: Pointer to the GPU SVM structure
1178 * @svm_pages: Pointer to the GPU SVM pages structure
1179 *
1180 * This function determines if a GPU SVM range pages are valid. Expected be
1181 * called holding gpusvm->notifier_lock and as the last step before committing a
1182 * GPU binding. This is akin to a notifier seqno check in the HMM documentation
1183 * but due to wider notifiers (i.e., notifiers which span multiple ranges) this
1184 * function is required for finer grained checking (i.e., per range) if pages
1185 * are valid.
1186 *
1187 * Return: True if GPU SVM range has valid pages, False otherwise
1188 */
1189static bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
1190 struct drm_gpusvm_pages *svm_pages)
1191{
1192 lockdep_assert_held(&gpusvm->notifier_lock);
1193
1194 return svm_pages->flags.has_devmem_pages || svm_pages->flags.has_dma_mapping;
1195}
1196
1197/**
1198 * drm_gpusvm_range_pages_valid() - GPU SVM range pages valid
1199 * @gpusvm: Pointer to the GPU SVM structure
1200 * @range: Pointer to the GPU SVM range structure
1201 *
1202 * This function determines if a GPU SVM range pages are valid. Expected be
1203 * called holding gpusvm->notifier_lock and as the last step before committing a
1204 * GPU binding. This is akin to a notifier seqno check in the HMM documentation
1205 * but due to wider notifiers (i.e., notifiers which span multiple ranges) this
1206 * function is required for finer grained checking (i.e., per range) if pages
1207 * are valid.
1208 *
1209 * Return: True if GPU SVM range has valid pages, False otherwise
1210 */
1211bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm,
1212 struct drm_gpusvm_range *range)
1213{
1214 return drm_gpusvm_pages_valid(gpusvm, svm_pages: &range->pages);
1215}
1216EXPORT_SYMBOL_GPL(drm_gpusvm_range_pages_valid);
1217
1218/**
1219 * drm_gpusvm_range_pages_valid_unlocked() - GPU SVM range pages valid unlocked
1220 * @gpusvm: Pointer to the GPU SVM structure
1221 * @range: Pointer to the GPU SVM range structure
1222 *
1223 * This function determines if a GPU SVM range pages are valid. Expected be
1224 * called without holding gpusvm->notifier_lock.
1225 *
1226 * Return: True if GPU SVM range has valid pages, False otherwise
1227 */
1228static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm,
1229 struct drm_gpusvm_pages *svm_pages)
1230{
1231 bool pages_valid;
1232
1233 if (!svm_pages->dma_addr)
1234 return false;
1235
1236 drm_gpusvm_notifier_lock(gpusvm);
1237 pages_valid = drm_gpusvm_pages_valid(gpusvm, svm_pages);
1238 if (!pages_valid)
1239 __drm_gpusvm_free_pages(gpusvm, svm_pages);
1240 drm_gpusvm_notifier_unlock(gpusvm);
1241
1242 return pages_valid;
1243}
1244
1245/**
1246 * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct
1247 * @gpusvm: Pointer to the GPU SVM structure
1248 * @svm_pages: The SVM pages to populate. This will contain the dma-addresses
1249 * @mm: The mm corresponding to the CPU range
1250 * @notifier: The corresponding notifier for the given CPU range
1251 * @pages_start: Start CPU address for the pages
1252 * @pages_end: End CPU address for the pages (exclusive)
1253 * @ctx: GPU SVM context
1254 *
1255 * This function gets and maps pages for CPU range and ensures they are
1256 * mapped for DMA access.
1257 *
1258 * Return: 0 on success, negative error code on failure.
1259 */
1260int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
1261 struct drm_gpusvm_pages *svm_pages,
1262 struct mm_struct *mm,
1263 struct mmu_interval_notifier *notifier,
1264 unsigned long pages_start, unsigned long pages_end,
1265 const struct drm_gpusvm_ctx *ctx)
1266{
1267 struct hmm_range hmm_range = {
1268 .default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 :
1269 HMM_PFN_REQ_WRITE),
1270 .notifier = notifier,
1271 .start = pages_start,
1272 .end = pages_end,
1273 .dev_private_owner = ctx->device_private_page_owner,
1274 };
1275 void *zdd;
1276 unsigned long timeout =
1277 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1278 unsigned long i, j;
1279 unsigned long npages = npages_in_range(start: pages_start, end: pages_end);
1280 unsigned long num_dma_mapped;
1281 unsigned int order = 0;
1282 unsigned long *pfns;
1283 int err = 0;
1284 struct dev_pagemap *pagemap;
1285 struct drm_pagemap *dpagemap;
1286 struct drm_gpusvm_pages_flags flags;
1287 enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE :
1288 DMA_BIDIRECTIONAL;
1289
1290retry:
1291 hmm_range.notifier_seq = mmu_interval_read_begin(interval_sub: notifier);
1292 if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages))
1293 goto set_seqno;
1294
1295 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
1296 if (!pfns)
1297 return -ENOMEM;
1298
1299 if (!mmget_not_zero(mm)) {
1300 err = -EFAULT;
1301 goto err_free;
1302 }
1303
1304 hmm_range.hmm_pfns = pfns;
1305 while (true) {
1306 mmap_read_lock(mm);
1307 err = hmm_range_fault(range: &hmm_range);
1308 mmap_read_unlock(mm);
1309
1310 if (err == -EBUSY) {
1311 if (time_after(jiffies, timeout))
1312 break;
1313
1314 hmm_range.notifier_seq =
1315 mmu_interval_read_begin(interval_sub: notifier);
1316 continue;
1317 }
1318 break;
1319 }
1320 mmput(mm);
1321 if (err)
1322 goto err_free;
1323
1324map_pages:
1325 /*
1326 * Perform all dma mappings under the notifier lock to not
1327 * access freed pages. A notifier will either block on
1328 * the notifier lock or unmap dma.
1329 */
1330 drm_gpusvm_notifier_lock(gpusvm);
1331
1332 flags.__flags = svm_pages->flags.__flags;
1333 if (flags.unmapped) {
1334 drm_gpusvm_notifier_unlock(gpusvm);
1335 err = -EFAULT;
1336 goto err_free;
1337 }
1338
1339 if (mmu_interval_read_retry(interval_sub: notifier, seq: hmm_range.notifier_seq)) {
1340 drm_gpusvm_notifier_unlock(gpusvm);
1341 kvfree(addr: pfns);
1342 goto retry;
1343 }
1344
1345 if (!svm_pages->dma_addr) {
1346 /* Unlock and restart mapping to allocate memory. */
1347 drm_gpusvm_notifier_unlock(gpusvm);
1348 svm_pages->dma_addr =
1349 kvmalloc_array(npages, sizeof(*svm_pages->dma_addr), GFP_KERNEL);
1350 if (!svm_pages->dma_addr) {
1351 err = -ENOMEM;
1352 goto err_free;
1353 }
1354 goto map_pages;
1355 }
1356
1357 zdd = NULL;
1358 pagemap = NULL;
1359 num_dma_mapped = 0;
1360 for (i = 0, j = 0; i < npages; ++j) {
1361 struct page *page = hmm_pfn_to_page(hmm_pfn: pfns[i]);
1362
1363 order = drm_gpusvm_hmm_pfn_to_order(hmm_pfn: pfns[i], hmm_pfn_index: i, npages);
1364 if (is_device_private_page(page) ||
1365 is_device_coherent_page(page)) {
1366 if (!ctx->allow_mixed &&
1367 zdd != page->zone_device_data && i > 0) {
1368 err = -EOPNOTSUPP;
1369 goto err_unmap;
1370 }
1371 zdd = page->zone_device_data;
1372 if (pagemap != page_pgmap(page)) {
1373 if (i > 0) {
1374 err = -EOPNOTSUPP;
1375 goto err_unmap;
1376 }
1377
1378 pagemap = page_pgmap(page);
1379 dpagemap = drm_pagemap_page_to_dpagemap(page);
1380 if (drm_WARN_ON(gpusvm->drm, !dpagemap)) {
1381 /*
1382 * Raced. This is not supposed to happen
1383 * since hmm_range_fault() should've migrated
1384 * this page to system.
1385 */
1386 err = -EAGAIN;
1387 goto err_unmap;
1388 }
1389 }
1390 svm_pages->dma_addr[j] =
1391 dpagemap->ops->device_map(dpagemap,
1392 gpusvm->drm->dev,
1393 page, order,
1394 dma_dir);
1395 if (dma_mapping_error(dev: gpusvm->drm->dev,
1396 dma_addr: svm_pages->dma_addr[j].addr)) {
1397 err = -EFAULT;
1398 goto err_unmap;
1399 }
1400 } else {
1401 dma_addr_t addr;
1402
1403 if (is_zone_device_page(page) ||
1404 (pagemap && !ctx->allow_mixed)) {
1405 err = -EOPNOTSUPP;
1406 goto err_unmap;
1407 }
1408
1409 if (ctx->devmem_only) {
1410 err = -EFAULT;
1411 goto err_unmap;
1412 }
1413
1414 addr = dma_map_page(gpusvm->drm->dev,
1415 page, 0,
1416 PAGE_SIZE << order,
1417 dma_dir);
1418 if (dma_mapping_error(dev: gpusvm->drm->dev, dma_addr: addr)) {
1419 err = -EFAULT;
1420 goto err_unmap;
1421 }
1422
1423 svm_pages->dma_addr[j] = drm_pagemap_addr_encode
1424 (addr, proto: DRM_INTERCONNECT_SYSTEM, order,
1425 dir: dma_dir);
1426 }
1427 i += 1 << order;
1428 num_dma_mapped = i;
1429 flags.has_dma_mapping = true;
1430 }
1431
1432 if (pagemap) {
1433 flags.has_devmem_pages = true;
1434 svm_pages->dpagemap = dpagemap;
1435 }
1436
1437 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1438 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1439
1440 drm_gpusvm_notifier_unlock(gpusvm);
1441 kvfree(addr: pfns);
1442set_seqno:
1443 svm_pages->notifier_seq = hmm_range.notifier_seq;
1444
1445 return 0;
1446
1447err_unmap:
1448 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages: num_dma_mapped);
1449 drm_gpusvm_notifier_unlock(gpusvm);
1450err_free:
1451 kvfree(addr: pfns);
1452 if (err == -EAGAIN)
1453 goto retry;
1454 return err;
1455}
1456EXPORT_SYMBOL_GPL(drm_gpusvm_get_pages);
1457
1458/**
1459 * drm_gpusvm_range_get_pages() - Get pages for a GPU SVM range
1460 * @gpusvm: Pointer to the GPU SVM structure
1461 * @range: Pointer to the GPU SVM range structure
1462 * @ctx: GPU SVM context
1463 *
1464 * This function gets pages for a GPU SVM range and ensures they are mapped for
1465 * DMA access.
1466 *
1467 * Return: 0 on success, negative error code on failure.
1468 */
1469int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm,
1470 struct drm_gpusvm_range *range,
1471 const struct drm_gpusvm_ctx *ctx)
1472{
1473 return drm_gpusvm_get_pages(gpusvm, &range->pages, gpusvm->mm,
1474 &range->notifier->notifier,
1475 drm_gpusvm_range_start(range),
1476 drm_gpusvm_range_end(range), ctx);
1477}
1478EXPORT_SYMBOL_GPL(drm_gpusvm_range_get_pages);
1479
1480/**
1481 * drm_gpusvm_unmap_pages() - Unmap GPU svm pages
1482 * @gpusvm: Pointer to the GPU SVM structure
1483 * @svm_pages: Pointer to the GPU SVM pages structure
1484 * @npages: Number of pages in @svm_pages.
1485 * @ctx: GPU SVM context
1486 *
1487 * This function unmaps pages associated with a GPU SVM pages struct. If
1488 * @in_notifier is set, it is assumed that gpusvm->notifier_lock is held in
1489 * write mode; if it is clear, it acquires gpusvm->notifier_lock in read mode.
1490 * Must be called in the invalidate() callback of the corresponding notifier for
1491 * IOMMU security model.
1492 */
1493void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
1494 struct drm_gpusvm_pages *svm_pages,
1495 unsigned long npages,
1496 const struct drm_gpusvm_ctx *ctx)
1497{
1498 if (ctx->in_notifier)
1499 lockdep_assert_held_write(&gpusvm->notifier_lock);
1500 else
1501 drm_gpusvm_notifier_lock(gpusvm);
1502
1503 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages);
1504
1505 if (!ctx->in_notifier)
1506 drm_gpusvm_notifier_unlock(gpusvm);
1507}
1508EXPORT_SYMBOL_GPL(drm_gpusvm_unmap_pages);
1509
1510/**
1511 * drm_gpusvm_range_unmap_pages() - Unmap pages associated with a GPU SVM range
1512 * @gpusvm: Pointer to the GPU SVM structure
1513 * @range: Pointer to the GPU SVM range structure
1514 * @ctx: GPU SVM context
1515 *
1516 * This function unmaps pages associated with a GPU SVM range. If @in_notifier
1517 * is set, it is assumed that gpusvm->notifier_lock is held in write mode; if it
1518 * is clear, it acquires gpusvm->notifier_lock in read mode. Must be called on
1519 * each GPU SVM range attached to notifier in gpusvm->ops->invalidate for IOMMU
1520 * security model.
1521 */
1522void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm,
1523 struct drm_gpusvm_range *range,
1524 const struct drm_gpusvm_ctx *ctx)
1525{
1526 unsigned long npages = npages_in_range(start: drm_gpusvm_range_start(range),
1527 end: drm_gpusvm_range_end(range));
1528
1529 return drm_gpusvm_unmap_pages(gpusvm, &range->pages, npages, ctx);
1530}
1531EXPORT_SYMBOL_GPL(drm_gpusvm_range_unmap_pages);
1532
1533/**
1534 * drm_gpusvm_range_evict() - Evict GPU SVM range
1535 * @gpusvm: Pointer to the GPU SVM structure
1536 * @range: Pointer to the GPU SVM range to be removed
1537 *
1538 * This function evicts the specified GPU SVM range.
1539 *
1540 * Return: 0 on success, a negative error code on failure.
1541 */
1542int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
1543 struct drm_gpusvm_range *range)
1544{
1545 struct mmu_interval_notifier *notifier = &range->notifier->notifier;
1546 struct hmm_range hmm_range = {
1547 .default_flags = HMM_PFN_REQ_FAULT,
1548 .notifier = notifier,
1549 .start = drm_gpusvm_range_start(range),
1550 .end = drm_gpusvm_range_end(range),
1551 .dev_private_owner = NULL,
1552 };
1553 unsigned long timeout =
1554 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1555 unsigned long *pfns;
1556 unsigned long npages = npages_in_range(start: drm_gpusvm_range_start(range),
1557 end: drm_gpusvm_range_end(range));
1558 int err = 0;
1559 struct mm_struct *mm = gpusvm->mm;
1560
1561 if (!mmget_not_zero(mm))
1562 return -EFAULT;
1563
1564 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
1565 if (!pfns)
1566 return -ENOMEM;
1567
1568 hmm_range.hmm_pfns = pfns;
1569 while (!time_after(jiffies, timeout)) {
1570 hmm_range.notifier_seq = mmu_interval_read_begin(interval_sub: notifier);
1571 if (time_after(jiffies, timeout)) {
1572 err = -ETIME;
1573 break;
1574 }
1575
1576 mmap_read_lock(mm);
1577 err = hmm_range_fault(range: &hmm_range);
1578 mmap_read_unlock(mm);
1579 if (err != -EBUSY)
1580 break;
1581 }
1582
1583 kvfree(addr: pfns);
1584 mmput(mm);
1585
1586 return err;
1587}
1588EXPORT_SYMBOL_GPL(drm_gpusvm_range_evict);
1589
1590/**
1591 * drm_gpusvm_has_mapping() - Check if GPU SVM has mapping for the given address range
1592 * @gpusvm: Pointer to the GPU SVM structure.
1593 * @start: Start address
1594 * @end: End address
1595 *
1596 * Return: True if GPU SVM has mapping, False otherwise
1597 */
1598bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
1599 unsigned long end)
1600{
1601 struct drm_gpusvm_notifier *notifier;
1602
1603 drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) {
1604 struct drm_gpusvm_range *range = NULL;
1605
1606 drm_gpusvm_for_each_range(range, notifier, start, end)
1607 return true;
1608 }
1609
1610 return false;
1611}
1612EXPORT_SYMBOL_GPL(drm_gpusvm_has_mapping);
1613
1614/**
1615 * drm_gpusvm_range_set_unmapped() - Mark a GPU SVM range as unmapped
1616 * @range: Pointer to the GPU SVM range structure.
1617 * @mmu_range: Pointer to the MMU notifier range structure.
1618 *
1619 * This function marks a GPU SVM range as unmapped and sets the partial_unmap flag
1620 * if the range partially falls within the provided MMU notifier range.
1621 */
1622void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
1623 const struct mmu_notifier_range *mmu_range)
1624{
1625 lockdep_assert_held_write(&range->gpusvm->notifier_lock);
1626
1627 range->pages.flags.unmapped = true;
1628 if (drm_gpusvm_range_start(range) < mmu_range->start ||
1629 drm_gpusvm_range_end(range) > mmu_range->end)
1630 range->pages.flags.partial_unmap = true;
1631}
1632EXPORT_SYMBOL_GPL(drm_gpusvm_range_set_unmapped);
1633
1634MODULE_DESCRIPTION("DRM GPUSVM");
1635MODULE_LICENSE("GPL");
1636

source code of linux/drivers/gpu/drm/drm_gpusvm.c