1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/*
3 * Copyright 2020 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors: Christian König
24 */
25
26/* Pooling of allocated pages is necessary because changing the caching
27 * attributes on x86 of the linear mapping requires a costly cross CPU TLB
28 * invalidate for those addresses.
29 *
30 * Additional to that allocations from the DMA coherent API are pooled as well
31 * cause they are rather slow compared to alloc_pages+map.
32 */
33
34#include <linux/export.h>
35#include <linux/module.h>
36#include <linux/dma-mapping.h>
37#include <linux/debugfs.h>
38#include <linux/highmem.h>
39#include <linux/sched/mm.h>
40
41#ifdef CONFIG_X86
42#include <asm/set_memory.h>
43#endif
44
45#include <drm/ttm/ttm_backup.h>
46#include <drm/ttm/ttm_pool.h>
47#include <drm/ttm/ttm_tt.h>
48#include <drm/ttm/ttm_bo.h>
49
50#include "ttm_module.h"
51#include "ttm_pool_internal.h"
52
53#ifdef CONFIG_FAULT_INJECTION
54#include <linux/fault-inject.h>
55static DECLARE_FAULT_ATTR(backup_fault_inject);
56#else
57#define should_fail(...) false
58#endif
59
60/**
61 * struct ttm_pool_dma - Helper object for coherent DMA mappings
62 *
63 * @addr: original DMA address returned for the mapping
64 * @vaddr: original vaddr return for the mapping and order in the lower bits
65 */
66struct ttm_pool_dma {
67 dma_addr_t addr;
68 unsigned long vaddr;
69};
70
71/**
72 * struct ttm_pool_alloc_state - Current state of the tt page allocation process
73 * @pages: Pointer to the next tt page pointer to populate.
74 * @caching_divide: Pointer to the first page pointer whose page has a staged but
75 * not committed caching transition from write-back to @tt_caching.
76 * @dma_addr: Pointer to the next tt dma_address entry to populate if any.
77 * @remaining_pages: Remaining pages to populate.
78 * @tt_caching: The requested cpu-caching for the pages allocated.
79 */
80struct ttm_pool_alloc_state {
81 struct page **pages;
82 struct page **caching_divide;
83 dma_addr_t *dma_addr;
84 pgoff_t remaining_pages;
85 enum ttm_caching tt_caching;
86};
87
88/**
89 * struct ttm_pool_tt_restore - State representing restore from backup
90 * @pool: The pool used for page allocation while restoring.
91 * @snapshot_alloc: A snapshot of the most recent struct ttm_pool_alloc_state.
92 * @alloced_page: Pointer to the page most recently allocated from a pool or system.
93 * @first_dma: The dma address corresponding to @alloced_page if dma_mapping
94 * is requested.
95 * @alloced_pages: The number of allocated pages present in the struct ttm_tt
96 * page vector from this restore session.
97 * @restored_pages: The number of 4K pages restored for @alloced_page (which
98 * is typically a multi-order page).
99 * @page_caching: The struct ttm_tt requested caching
100 * @order: The order of @alloced_page.
101 *
102 * Recovery from backup might fail when we've recovered less than the
103 * full ttm_tt. In order not to loose any data (yet), keep information
104 * around that allows us to restart a failed ttm backup recovery.
105 */
106struct ttm_pool_tt_restore {
107 struct ttm_pool *pool;
108 struct ttm_pool_alloc_state snapshot_alloc;
109 struct page *alloced_page;
110 dma_addr_t first_dma;
111 pgoff_t alloced_pages;
112 pgoff_t restored_pages;
113 enum ttm_caching page_caching;
114 unsigned int order;
115};
116
117static unsigned long page_pool_size;
118
119MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool");
120module_param(page_pool_size, ulong, 0644);
121
122static atomic_long_t allocated_pages;
123
124static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS];
125static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS];
126
127static struct ttm_pool_type global_dma32_write_combined[NR_PAGE_ORDERS];
128static struct ttm_pool_type global_dma32_uncached[NR_PAGE_ORDERS];
129
130static spinlock_t shrinker_lock;
131static struct list_head shrinker_list;
132static struct shrinker *mm_shrinker;
133static DECLARE_RWSEM(pool_shrink_rwsem);
134
135/* Allocate pages of size 1 << order with the given gfp_flags */
136static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
137 unsigned int order)
138{
139 const unsigned int beneficial_order = ttm_pool_beneficial_order(pool);
140 unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS;
141 struct ttm_pool_dma *dma;
142 struct page *p;
143 void *vaddr;
144
145 /* Don't set the __GFP_COMP flag for higher order allocations.
146 * Mapping pages directly into an userspace process and calling
147 * put_page() on a TTM allocated page is illegal.
148 */
149 if (order)
150 gfp_flags |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN |
151 __GFP_THISNODE;
152
153 /*
154 * Do not add latency to the allocation path for allocations orders
155 * device tolds us do not bring them additional performance gains.
156 */
157 if (beneficial_order && order > beneficial_order)
158 gfp_flags &= ~__GFP_DIRECT_RECLAIM;
159
160 if (!ttm_pool_uses_dma_alloc(pool)) {
161 p = alloc_pages_node(pool->nid, gfp_flags, order);
162 if (p)
163 p->private = order;
164 return p;
165 }
166
167 dma = kmalloc(sizeof(*dma), GFP_KERNEL);
168 if (!dma)
169 return NULL;
170
171 if (order)
172 attr |= DMA_ATTR_NO_WARN;
173
174 vaddr = dma_alloc_attrs(dev: pool->dev, size: (1ULL << order) * PAGE_SIZE,
175 dma_handle: &dma->addr, flag: gfp_flags, attrs: attr);
176 if (!vaddr)
177 goto error_free;
178
179 /* TODO: This is an illegal abuse of the DMA API, but we need to rework
180 * TTM page fault handling and extend the DMA API to clean this up.
181 */
182 if (is_vmalloc_addr(x: vaddr))
183 p = vmalloc_to_page(addr: vaddr);
184 else
185 p = virt_to_page(vaddr);
186
187 dma->vaddr = (unsigned long)vaddr | order;
188 p->private = (unsigned long)dma;
189 return p;
190
191error_free:
192 kfree(objp: dma);
193 return NULL;
194}
195
196/* Reset the caching and pages of size 1 << order */
197static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching,
198 unsigned int order, struct page *p)
199{
200 unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS;
201 struct ttm_pool_dma *dma;
202 void *vaddr;
203
204#ifdef CONFIG_X86
205 /* We don't care that set_pages_wb is inefficient here. This is only
206 * used when we have to shrink and CPU overhead is irrelevant then.
207 */
208 if (caching != ttm_cached && !PageHighMem(page: p))
209 set_pages_wb(page: p, numpages: 1 << order);
210#endif
211
212 if (!pool || !ttm_pool_uses_dma_alloc(pool)) {
213 __free_pages(page: p, order);
214 return;
215 }
216
217 if (order)
218 attr |= DMA_ATTR_NO_WARN;
219
220 dma = (void *)p->private;
221 vaddr = (void *)(dma->vaddr & PAGE_MASK);
222 dma_free_attrs(dev: pool->dev, size: (1UL << order) * PAGE_SIZE, cpu_addr: vaddr, dma_handle: dma->addr,
223 attrs: attr);
224 kfree(objp: dma);
225}
226
227/* Apply any cpu-caching deferred during page allocation */
228static int ttm_pool_apply_caching(struct ttm_pool_alloc_state *alloc)
229{
230#ifdef CONFIG_X86
231 unsigned int num_pages = alloc->pages - alloc->caching_divide;
232
233 if (!num_pages)
234 return 0;
235
236 switch (alloc->tt_caching) {
237 case ttm_cached:
238 break;
239 case ttm_write_combined:
240 return set_pages_array_wc(pages: alloc->caching_divide, addrinarray: num_pages);
241 case ttm_uncached:
242 return set_pages_array_uc(pages: alloc->caching_divide, addrinarray: num_pages);
243 }
244#endif
245 alloc->caching_divide = alloc->pages;
246 return 0;
247}
248
249/* DMA Map pages of 1 << order size and return the resulting dma_address. */
250static int ttm_pool_map(struct ttm_pool *pool, unsigned int order,
251 struct page *p, dma_addr_t *dma_addr)
252{
253 dma_addr_t addr;
254
255 if (ttm_pool_uses_dma_alloc(pool)) {
256 struct ttm_pool_dma *dma = (void *)p->private;
257
258 addr = dma->addr;
259 } else {
260 size_t size = (1ULL << order) * PAGE_SIZE;
261
262 addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL);
263 if (dma_mapping_error(dev: pool->dev, dma_addr: addr))
264 return -EFAULT;
265 }
266
267 *dma_addr = addr;
268
269 return 0;
270}
271
272/* Unmap pages of 1 << order size */
273static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr,
274 unsigned int num_pages)
275{
276 /* Unmapped while freeing the page */
277 if (ttm_pool_uses_dma_alloc(pool))
278 return;
279
280 dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT,
281 DMA_BIDIRECTIONAL);
282}
283
284/* Give pages into a specific pool_type */
285static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p)
286{
287 unsigned int i, num_pages = 1 << pt->order;
288
289 for (i = 0; i < num_pages; ++i) {
290 if (PageHighMem(page: p))
291 clear_highpage(page: p + i);
292 else
293 clear_page(page_address(p + i));
294 }
295
296 spin_lock(lock: &pt->lock);
297 list_add(new: &p->lru, head: &pt->pages);
298 spin_unlock(lock: &pt->lock);
299 atomic_long_add(i: 1 << pt->order, v: &allocated_pages);
300}
301
302/* Take pages from a specific pool_type, return NULL when nothing available */
303static struct page *ttm_pool_type_take(struct ttm_pool_type *pt)
304{
305 struct page *p;
306
307 spin_lock(lock: &pt->lock);
308 p = list_first_entry_or_null(&pt->pages, typeof(*p), lru);
309 if (p) {
310 atomic_long_sub(i: 1 << pt->order, v: &allocated_pages);
311 list_del(entry: &p->lru);
312 }
313 spin_unlock(lock: &pt->lock);
314
315 return p;
316}
317
318/* Initialize and add a pool type to the global shrinker list */
319static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool,
320 enum ttm_caching caching, unsigned int order)
321{
322 pt->pool = pool;
323 pt->caching = caching;
324 pt->order = order;
325 spin_lock_init(&pt->lock);
326 INIT_LIST_HEAD(list: &pt->pages);
327
328 spin_lock(lock: &shrinker_lock);
329 list_add_tail(new: &pt->shrinker_list, head: &shrinker_list);
330 spin_unlock(lock: &shrinker_lock);
331}
332
333/* Remove a pool_type from the global shrinker list and free all pages */
334static void ttm_pool_type_fini(struct ttm_pool_type *pt)
335{
336 struct page *p;
337
338 spin_lock(lock: &shrinker_lock);
339 list_del(entry: &pt->shrinker_list);
340 spin_unlock(lock: &shrinker_lock);
341
342 while ((p = ttm_pool_type_take(pt)))
343 ttm_pool_free_page(pool: pt->pool, caching: pt->caching, order: pt->order, p);
344}
345
346/* Return the pool_type to use for the given caching and order */
347static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,
348 enum ttm_caching caching,
349 unsigned int order)
350{
351 if (ttm_pool_uses_dma_alloc(pool))
352 return &pool->caching[caching].orders[order];
353
354#ifdef CONFIG_X86
355 switch (caching) {
356 case ttm_write_combined:
357 if (pool->nid != NUMA_NO_NODE)
358 return &pool->caching[caching].orders[order];
359
360 if (ttm_pool_uses_dma32(pool))
361 return &global_dma32_write_combined[order];
362
363 return &global_write_combined[order];
364 case ttm_uncached:
365 if (pool->nid != NUMA_NO_NODE)
366 return &pool->caching[caching].orders[order];
367
368 if (ttm_pool_uses_dma32(pool))
369 return &global_dma32_uncached[order];
370
371 return &global_uncached[order];
372 default:
373 break;
374 }
375#endif
376
377 return NULL;
378}
379
380/* Free pages using the global shrinker list */
381static unsigned int ttm_pool_shrink(void)
382{
383 struct ttm_pool_type *pt;
384 unsigned int num_pages;
385 struct page *p;
386
387 down_read(sem: &pool_shrink_rwsem);
388 spin_lock(lock: &shrinker_lock);
389 pt = list_first_entry(&shrinker_list, typeof(*pt), shrinker_list);
390 list_move_tail(list: &pt->shrinker_list, head: &shrinker_list);
391 spin_unlock(lock: &shrinker_lock);
392
393 p = ttm_pool_type_take(pt);
394 if (p) {
395 ttm_pool_free_page(pool: pt->pool, caching: pt->caching, order: pt->order, p);
396 num_pages = 1 << pt->order;
397 } else {
398 num_pages = 0;
399 }
400 up_read(sem: &pool_shrink_rwsem);
401
402 return num_pages;
403}
404
405/* Return the allocation order based for a page */
406static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p)
407{
408 if (ttm_pool_uses_dma_alloc(pool)) {
409 struct ttm_pool_dma *dma = (void *)p->private;
410
411 return dma->vaddr & ~PAGE_MASK;
412 }
413
414 return p->private;
415}
416
417/*
418 * Split larger pages so that we can free each PAGE_SIZE page as soon
419 * as it has been backed up, in order to avoid memory pressure during
420 * reclaim.
421 */
422static void ttm_pool_split_for_swap(struct ttm_pool *pool, struct page *p)
423{
424 unsigned int order = ttm_pool_page_order(pool, p);
425 pgoff_t nr;
426
427 if (!order)
428 return;
429
430 split_page(page: p, order);
431 nr = 1UL << order;
432 while (nr--)
433 (p++)->private = 0;
434}
435
436/**
437 * DOC: Partial backup and restoration of a struct ttm_tt.
438 *
439 * Swapout using ttm_backup_backup_page() and swapin using
440 * ttm_backup_copy_page() may fail.
441 * The former most likely due to lack of swap-space or memory, the latter due
442 * to lack of memory or because of signal interruption during waits.
443 *
444 * Backup failure is easily handled by using a ttm_tt pages vector that holds
445 * both backup handles and page pointers. This has to be taken into account when
446 * restoring such a ttm_tt from backup, and when freeing it while backed up.
447 * When restoring, for simplicity, new pages are actually allocated from the
448 * pool and the contents of any old pages are copied in and then the old pages
449 * are released.
450 *
451 * For restoration failures, the struct ttm_pool_tt_restore holds sufficient state
452 * to be able to resume an interrupted restore, and that structure is freed once
453 * the restoration is complete. If the struct ttm_tt is destroyed while there
454 * is a valid struct ttm_pool_tt_restore attached, that is also properly taken
455 * care of.
456 */
457
458/* Is restore ongoing for the currently allocated page? */
459static bool ttm_pool_restore_valid(const struct ttm_pool_tt_restore *restore)
460{
461 return restore && restore->restored_pages < (1 << restore->order);
462}
463
464/* DMA unmap and free a multi-order page, either to the relevant pool or to system. */
465static pgoff_t ttm_pool_unmap_and_free(struct ttm_pool *pool, struct page *page,
466 const dma_addr_t *dma_addr, enum ttm_caching caching)
467{
468 struct ttm_pool_type *pt = NULL;
469 unsigned int order;
470 pgoff_t nr;
471
472 if (pool) {
473 order = ttm_pool_page_order(pool, p: page);
474 nr = (1UL << order);
475 if (dma_addr)
476 ttm_pool_unmap(pool, dma_addr: *dma_addr, num_pages: nr);
477
478 pt = ttm_pool_select_type(pool, caching, order);
479 } else {
480 order = page->private;
481 nr = (1UL << order);
482 }
483
484 if (pt)
485 ttm_pool_type_give(pt, p: page);
486 else
487 ttm_pool_free_page(pool, caching, order, p: page);
488
489 return nr;
490}
491
492/* Populate the page-array using the most recent allocated multi-order page. */
493static void ttm_pool_allocated_page_commit(struct page *allocated,
494 dma_addr_t first_dma,
495 struct ttm_pool_alloc_state *alloc,
496 pgoff_t nr)
497{
498 pgoff_t i;
499
500 for (i = 0; i < nr; ++i)
501 *alloc->pages++ = allocated++;
502
503 alloc->remaining_pages -= nr;
504
505 if (!alloc->dma_addr)
506 return;
507
508 for (i = 0; i < nr; ++i) {
509 *alloc->dma_addr++ = first_dma;
510 first_dma += PAGE_SIZE;
511 }
512}
513
514/*
515 * When restoring, restore backed-up content to the newly allocated page and
516 * if successful, populate the page-table and dma-address arrays.
517 */
518static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore,
519 struct file *backup,
520 const struct ttm_operation_ctx *ctx,
521 struct ttm_pool_alloc_state *alloc)
522
523{
524 pgoff_t i, nr = 1UL << restore->order;
525 struct page **first_page = alloc->pages;
526 struct page *p;
527 int ret = 0;
528
529 for (i = restore->restored_pages; i < nr; ++i) {
530 p = first_page[i];
531 if (ttm_backup_page_ptr_is_handle(page: p)) {
532 unsigned long handle = ttm_backup_page_ptr_to_handle(page: p);
533
534 if (IS_ENABLED(CONFIG_FAULT_INJECTION) && ctx->interruptible &&
535 should_fail(attr: &backup_fault_inject, size: 1)) {
536 ret = -EINTR;
537 break;
538 }
539
540 if (handle == 0) {
541 restore->restored_pages++;
542 continue;
543 }
544
545 ret = ttm_backup_copy_page(backup, dst: restore->alloced_page + i,
546 handle, intr: ctx->interruptible);
547 if (ret)
548 break;
549
550 ttm_backup_drop(backup, handle);
551 } else if (p) {
552 /*
553 * We could probably avoid splitting the old page
554 * using clever logic, but ATM we don't care, as
555 * we prioritize releasing memory ASAP. Note that
556 * here, the old retained page is always write-back
557 * cached.
558 */
559 ttm_pool_split_for_swap(pool: restore->pool, p);
560 copy_highpage(to: restore->alloced_page + i, from: p);
561 __free_pages(page: p, order: 0);
562 }
563
564 restore->restored_pages++;
565 first_page[i] = ttm_backup_handle_to_page_ptr(handle: 0);
566 }
567
568 if (ret) {
569 if (!restore->restored_pages) {
570 dma_addr_t *dma_addr = alloc->dma_addr ? &restore->first_dma : NULL;
571
572 ttm_pool_unmap_and_free(pool: restore->pool, page: restore->alloced_page,
573 dma_addr, caching: restore->page_caching);
574 restore->restored_pages = nr;
575 }
576 return ret;
577 }
578
579 ttm_pool_allocated_page_commit(allocated: restore->alloced_page, first_dma: restore->first_dma,
580 alloc, nr);
581 if (restore->page_caching == alloc->tt_caching || PageHighMem(page: restore->alloced_page))
582 alloc->caching_divide = alloc->pages;
583 restore->snapshot_alloc = *alloc;
584 restore->alloced_pages += nr;
585
586 return 0;
587}
588
589/* If restoring, save information needed for ttm_pool_restore_commit(). */
590static void
591ttm_pool_page_allocated_restore(struct ttm_pool *pool, unsigned int order,
592 struct page *p,
593 enum ttm_caching page_caching,
594 dma_addr_t first_dma,
595 struct ttm_pool_tt_restore *restore,
596 const struct ttm_pool_alloc_state *alloc)
597{
598 restore->pool = pool;
599 restore->order = order;
600 restore->restored_pages = 0;
601 restore->page_caching = page_caching;
602 restore->first_dma = first_dma;
603 restore->alloced_page = p;
604 restore->snapshot_alloc = *alloc;
605}
606
607/*
608 * Called when we got a page, either from a pool or newly allocated.
609 * if needed, dma map the page and populate the dma address array.
610 * Populate the page address array.
611 * If the caching is consistent, update any deferred caching. Otherwise
612 * stage this page for an upcoming deferred caching update.
613 */
614static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order,
615 struct page *p, enum ttm_caching page_caching,
616 struct ttm_pool_alloc_state *alloc,
617 struct ttm_pool_tt_restore *restore)
618{
619 bool caching_consistent;
620 dma_addr_t first_dma;
621 int r = 0;
622
623 caching_consistent = (page_caching == alloc->tt_caching) || PageHighMem(page: p);
624
625 if (caching_consistent) {
626 r = ttm_pool_apply_caching(alloc);
627 if (r)
628 return r;
629 }
630
631 if (alloc->dma_addr) {
632 r = ttm_pool_map(pool, order, p, dma_addr: &first_dma);
633 if (r)
634 return r;
635 }
636
637 if (restore) {
638 ttm_pool_page_allocated_restore(pool, order, p, page_caching,
639 first_dma, restore, alloc);
640 } else {
641 ttm_pool_allocated_page_commit(allocated: p, first_dma, alloc, nr: 1UL << order);
642
643 if (caching_consistent)
644 alloc->caching_divide = alloc->pages;
645 }
646
647 return 0;
648}
649
650/**
651 * ttm_pool_free_range() - Free a range of TTM pages
652 * @pool: The pool used for allocating.
653 * @tt: The struct ttm_tt holding the page pointers.
654 * @caching: The page caching mode used by the range.
655 * @start_page: index for first page to free.
656 * @end_page: index for last page to free + 1.
657 *
658 * During allocation the ttm_tt page-vector may be populated with ranges of
659 * pages with different attributes if allocation hit an error without being
660 * able to completely fulfill the allocation. This function can be used
661 * to free these individual ranges.
662 */
663static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt,
664 enum ttm_caching caching,
665 pgoff_t start_page, pgoff_t end_page)
666{
667 struct page **pages = &tt->pages[start_page];
668 struct file *backup = tt->backup;
669 pgoff_t i, nr;
670
671 for (i = start_page; i < end_page; i += nr, pages += nr) {
672 struct page *p = *pages;
673
674 nr = 1;
675 if (ttm_backup_page_ptr_is_handle(page: p)) {
676 unsigned long handle = ttm_backup_page_ptr_to_handle(page: p);
677
678 if (handle != 0)
679 ttm_backup_drop(backup, handle);
680 } else if (p) {
681 dma_addr_t *dma_addr = tt->dma_address ?
682 tt->dma_address + i : NULL;
683
684 nr = ttm_pool_unmap_and_free(pool, page: p, dma_addr, caching);
685 }
686 }
687}
688
689static void ttm_pool_alloc_state_init(const struct ttm_tt *tt,
690 struct ttm_pool_alloc_state *alloc)
691{
692 alloc->pages = tt->pages;
693 alloc->caching_divide = tt->pages;
694 alloc->dma_addr = tt->dma_address;
695 alloc->remaining_pages = tt->num_pages;
696 alloc->tt_caching = tt->caching;
697}
698
699/*
700 * Find a suitable allocation order based on highest desired order
701 * and number of remaining pages
702 */
703static unsigned int ttm_pool_alloc_find_order(unsigned int highest,
704 const struct ttm_pool_alloc_state *alloc)
705{
706 return min_t(unsigned int, highest, __fls(alloc->remaining_pages));
707}
708
709static int __ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
710 const struct ttm_operation_ctx *ctx,
711 struct ttm_pool_alloc_state *alloc,
712 struct ttm_pool_tt_restore *restore)
713{
714 enum ttm_caching page_caching;
715 gfp_t gfp_flags = GFP_USER;
716 pgoff_t caching_divide;
717 unsigned int order;
718 bool allow_pools;
719 struct page *p;
720 int r;
721
722 WARN_ON(!alloc->remaining_pages || ttm_tt_is_populated(tt));
723 WARN_ON(alloc->dma_addr && !pool->dev);
724
725 if (tt->page_flags & TTM_TT_FLAG_ZERO_ALLOC)
726 gfp_flags |= __GFP_ZERO;
727
728 if (ctx->gfp_retry_mayfail)
729 gfp_flags |= __GFP_RETRY_MAYFAIL;
730
731 if (ttm_pool_uses_dma32(pool))
732 gfp_flags |= GFP_DMA32;
733 else
734 gfp_flags |= GFP_HIGHUSER;
735
736 page_caching = tt->caching;
737 allow_pools = true;
738 for (order = ttm_pool_alloc_find_order(MAX_PAGE_ORDER, alloc);
739 alloc->remaining_pages;
740 order = ttm_pool_alloc_find_order(highest: order, alloc)) {
741 struct ttm_pool_type *pt;
742
743 /* First, try to allocate a page from a pool if one exists. */
744 p = NULL;
745 pt = ttm_pool_select_type(pool, caching: page_caching, order);
746 if (pt && allow_pools)
747 p = ttm_pool_type_take(pt);
748 /*
749 * If that fails or previously failed, allocate from system.
750 * Note that this also disallows additional pool allocations using
751 * write-back cached pools of the same order. Consider removing
752 * that behaviour.
753 */
754 if (!p) {
755 page_caching = ttm_cached;
756 allow_pools = false;
757 p = ttm_pool_alloc_page(pool, gfp_flags, order);
758 }
759 /* If that fails, lower the order if possible and retry. */
760 if (!p) {
761 if (order) {
762 --order;
763 page_caching = tt->caching;
764 allow_pools = true;
765 continue;
766 }
767 r = -ENOMEM;
768 goto error_free_all;
769 }
770 r = ttm_pool_page_allocated(pool, order, p, page_caching, alloc,
771 restore);
772 if (r)
773 goto error_free_page;
774
775 if (ttm_pool_restore_valid(restore)) {
776 r = ttm_pool_restore_commit(restore, backup: tt->backup, ctx, alloc);
777 if (r)
778 goto error_free_all;
779 }
780 }
781
782 r = ttm_pool_apply_caching(alloc);
783 if (r)
784 goto error_free_all;
785
786 kfree(objp: tt->restore);
787 tt->restore = NULL;
788
789 return 0;
790
791error_free_page:
792 ttm_pool_free_page(pool, caching: page_caching, order, p);
793
794error_free_all:
795 if (tt->restore)
796 return r;
797
798 caching_divide = alloc->caching_divide - tt->pages;
799 ttm_pool_free_range(pool, tt, caching: tt->caching, start_page: 0, end_page: caching_divide);
800 ttm_pool_free_range(pool, tt, caching: ttm_cached, start_page: caching_divide,
801 end_page: tt->num_pages - alloc->remaining_pages);
802
803 return r;
804}
805
806/**
807 * ttm_pool_alloc - Fill a ttm_tt object
808 *
809 * @pool: ttm_pool to use
810 * @tt: ttm_tt object to fill
811 * @ctx: operation context
812 *
813 * Fill the ttm_tt object with pages and also make sure to DMA map them when
814 * necessary.
815 *
816 * Returns: 0 on successe, negative error code otherwise.
817 */
818int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
819 struct ttm_operation_ctx *ctx)
820{
821 struct ttm_pool_alloc_state alloc;
822
823 if (WARN_ON(ttm_tt_is_backed_up(tt)))
824 return -EINVAL;
825
826 ttm_pool_alloc_state_init(tt, alloc: &alloc);
827
828 return __ttm_pool_alloc(pool, tt, ctx, alloc: &alloc, NULL);
829}
830EXPORT_SYMBOL(ttm_pool_alloc);
831
832/**
833 * ttm_pool_restore_and_alloc - Fill a ttm_tt, restoring previously backed-up
834 * content.
835 *
836 * @pool: ttm_pool to use
837 * @tt: ttm_tt object to fill
838 * @ctx: operation context
839 *
840 * Fill the ttm_tt object with pages and also make sure to DMA map them when
841 * necessary. Read in backed-up content.
842 *
843 * Returns: 0 on successe, negative error code otherwise.
844 */
845int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
846 const struct ttm_operation_ctx *ctx)
847{
848 struct ttm_pool_alloc_state alloc;
849
850 if (WARN_ON(!ttm_tt_is_backed_up(tt)))
851 return -EINVAL;
852
853 if (!tt->restore) {
854 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
855
856 ttm_pool_alloc_state_init(tt, alloc: &alloc);
857 if (ctx->gfp_retry_mayfail)
858 gfp |= __GFP_RETRY_MAYFAIL;
859
860 tt->restore = kzalloc(sizeof(*tt->restore), gfp);
861 if (!tt->restore)
862 return -ENOMEM;
863
864 tt->restore->snapshot_alloc = alloc;
865 tt->restore->pool = pool;
866 tt->restore->restored_pages = 1;
867 } else {
868 struct ttm_pool_tt_restore *restore = tt->restore;
869 int ret;
870
871 alloc = restore->snapshot_alloc;
872 if (ttm_pool_restore_valid(restore: tt->restore)) {
873 ret = ttm_pool_restore_commit(restore, backup: tt->backup, ctx, alloc: &alloc);
874 if (ret)
875 return ret;
876 }
877 if (!alloc.remaining_pages)
878 return 0;
879 }
880
881 return __ttm_pool_alloc(pool, tt, ctx, alloc: &alloc, restore: tt->restore);
882}
883
884/**
885 * ttm_pool_free - Free the backing pages from a ttm_tt object
886 *
887 * @pool: Pool to give pages back to.
888 * @tt: ttm_tt object to unpopulate
889 *
890 * Give the packing pages back to a pool or free them
891 */
892void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt)
893{
894 ttm_pool_free_range(pool, tt, caching: tt->caching, start_page: 0, end_page: tt->num_pages);
895
896 while (atomic_long_read(v: &allocated_pages) > page_pool_size)
897 ttm_pool_shrink();
898}
899EXPORT_SYMBOL(ttm_pool_free);
900
901/**
902 * ttm_pool_drop_backed_up() - Release content of a swapped-out struct ttm_tt
903 * @tt: The struct ttm_tt.
904 *
905 * Release handles with associated content or any remaining pages of
906 * a backed-up struct ttm_tt.
907 */
908void ttm_pool_drop_backed_up(struct ttm_tt *tt)
909{
910 struct ttm_pool_tt_restore *restore;
911 pgoff_t start_page = 0;
912
913 WARN_ON(!ttm_tt_is_backed_up(tt));
914
915 restore = tt->restore;
916
917 /*
918 * Unmap and free any uncommitted restore page.
919 * any tt page-array backup entries already read back has
920 * been cleared already
921 */
922 if (ttm_pool_restore_valid(restore)) {
923 dma_addr_t *dma_addr = tt->dma_address ? &restore->first_dma : NULL;
924
925 ttm_pool_unmap_and_free(pool: restore->pool, page: restore->alloced_page,
926 dma_addr, caching: restore->page_caching);
927 restore->restored_pages = 1UL << restore->order;
928 }
929
930 /*
931 * If a restore is ongoing, part of the tt pages may have a
932 * caching different than writeback.
933 */
934 if (restore) {
935 pgoff_t mid = restore->snapshot_alloc.caching_divide - tt->pages;
936
937 start_page = restore->alloced_pages;
938 WARN_ON(mid > start_page);
939 /* Pages that might be dma-mapped and non-cached */
940 ttm_pool_free_range(pool: restore->pool, tt, caching: tt->caching,
941 start_page: 0, end_page: mid);
942 /* Pages that might be dma-mapped but cached */
943 ttm_pool_free_range(pool: restore->pool, tt, caching: ttm_cached,
944 start_page: mid, end_page: restore->alloced_pages);
945 kfree(objp: restore);
946 tt->restore = NULL;
947 }
948
949 ttm_pool_free_range(NULL, tt, caching: ttm_cached, start_page, end_page: tt->num_pages);
950}
951
952/**
953 * ttm_pool_backup() - Back up or purge a struct ttm_tt
954 * @pool: The pool used when allocating the struct ttm_tt.
955 * @tt: The struct ttm_tt.
956 * @flags: Flags to govern the backup behaviour.
957 *
958 * Back up or purge a struct ttm_tt. If @purge is true, then
959 * all pages will be freed directly to the system rather than to the pool
960 * they were allocated from, making the function behave similarly to
961 * ttm_pool_free(). If @purge is false the pages will be backed up instead,
962 * exchanged for handles.
963 * A subsequent call to ttm_pool_restore_and_alloc() will then read back the content and
964 * a subsequent call to ttm_pool_drop_backed_up() will drop it.
965 * If backup of a page fails for whatever reason, @ttm will still be
966 * partially backed up, retaining those pages for which backup fails.
967 * In that case, this function can be retried, possibly after freeing up
968 * memory resources.
969 *
970 * Return: Number of pages actually backed up or freed, or negative
971 * error code on error.
972 */
973long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt,
974 const struct ttm_backup_flags *flags)
975{
976 struct file *backup = tt->backup;
977 struct page *page;
978 unsigned long handle;
979 gfp_t alloc_gfp;
980 gfp_t gfp;
981 int ret = 0;
982 pgoff_t shrunken = 0;
983 pgoff_t i, num_pages;
984
985 if (WARN_ON(ttm_tt_is_backed_up(tt)))
986 return -EINVAL;
987
988 if ((!ttm_backup_bytes_avail() && !flags->purge) ||
989 ttm_pool_uses_dma_alloc(pool) || ttm_tt_is_backed_up(tt))
990 return -EBUSY;
991
992#ifdef CONFIG_X86
993 /* Anything returned to the system needs to be cached. */
994 if (tt->caching != ttm_cached)
995 set_pages_array_wb(pages: tt->pages, addrinarray: tt->num_pages);
996#endif
997
998 if (tt->dma_address || flags->purge) {
999 for (i = 0; i < tt->num_pages; i += num_pages) {
1000 unsigned int order;
1001
1002 page = tt->pages[i];
1003 if (unlikely(!page)) {
1004 num_pages = 1;
1005 continue;
1006 }
1007
1008 order = ttm_pool_page_order(pool, p: page);
1009 num_pages = 1UL << order;
1010 if (tt->dma_address)
1011 ttm_pool_unmap(pool, dma_addr: tt->dma_address[i],
1012 num_pages);
1013 if (flags->purge) {
1014 shrunken += num_pages;
1015 page->private = 0;
1016 __free_pages(page, order);
1017 memset(tt->pages + i, 0,
1018 num_pages * sizeof(*tt->pages));
1019 }
1020 }
1021 }
1022
1023 if (flags->purge)
1024 return shrunken;
1025
1026 if (ttm_pool_uses_dma32(pool))
1027 gfp = GFP_DMA32;
1028 else
1029 gfp = GFP_HIGHUSER;
1030
1031 alloc_gfp = GFP_KERNEL | __GFP_HIGH | __GFP_NOWARN | __GFP_RETRY_MAYFAIL;
1032
1033 num_pages = tt->num_pages;
1034
1035 /* Pretend doing fault injection by shrinking only half of the pages. */
1036 if (IS_ENABLED(CONFIG_FAULT_INJECTION) && should_fail(attr: &backup_fault_inject, size: 1))
1037 num_pages = DIV_ROUND_UP(num_pages, 2);
1038
1039 for (i = 0; i < num_pages; ++i) {
1040 s64 shandle;
1041
1042 page = tt->pages[i];
1043 if (unlikely(!page))
1044 continue;
1045
1046 ttm_pool_split_for_swap(pool, p: page);
1047
1048 shandle = ttm_backup_backup_page(backup, page, writeback: flags->writeback, idx: i,
1049 page_gfp: gfp, alloc_gfp);
1050 if (shandle < 0) {
1051 /* We allow partially shrunken tts */
1052 ret = shandle;
1053 break;
1054 }
1055 handle = shandle;
1056 tt->pages[i] = ttm_backup_handle_to_page_ptr(handle);
1057 put_page(page);
1058 shrunken++;
1059 }
1060
1061 return shrunken ? shrunken : ret;
1062}
1063
1064/**
1065 * ttm_pool_init - Initialize a pool
1066 *
1067 * @pool: the pool to initialize
1068 * @dev: device for DMA allocations and mappings
1069 * @nid: NUMA node to use for allocations
1070 * @alloc_flags: TTM_ALLOCATION_POOL_* flags
1071 *
1072 * Initialize the pool and its pool types.
1073 */
1074void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
1075 int nid, unsigned int alloc_flags)
1076{
1077 unsigned int i, j;
1078
1079 WARN_ON(!dev && ttm_pool_uses_dma_alloc(pool));
1080
1081 pool->dev = dev;
1082 pool->nid = nid;
1083 pool->alloc_flags = alloc_flags;
1084
1085 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {
1086 for (j = 0; j < NR_PAGE_ORDERS; ++j) {
1087 struct ttm_pool_type *pt;
1088
1089 /* Initialize only pool types which are actually used */
1090 pt = ttm_pool_select_type(pool, caching: i, order: j);
1091 if (pt != &pool->caching[i].orders[j])
1092 continue;
1093
1094 ttm_pool_type_init(pt, pool, caching: i, order: j);
1095 }
1096 }
1097}
1098EXPORT_SYMBOL(ttm_pool_init);
1099
1100/**
1101 * ttm_pool_synchronize_shrinkers - Wait for all running shrinkers to complete.
1102 *
1103 * This is useful to guarantee that all shrinker invocations have seen an
1104 * update, before freeing memory, similar to rcu.
1105 */
1106static void ttm_pool_synchronize_shrinkers(void)
1107{
1108 down_write(sem: &pool_shrink_rwsem);
1109 up_write(sem: &pool_shrink_rwsem);
1110}
1111
1112/**
1113 * ttm_pool_fini - Cleanup a pool
1114 *
1115 * @pool: the pool to clean up
1116 *
1117 * Free all pages in the pool and unregister the types from the global
1118 * shrinker.
1119 */
1120void ttm_pool_fini(struct ttm_pool *pool)
1121{
1122 unsigned int i, j;
1123
1124 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {
1125 for (j = 0; j < NR_PAGE_ORDERS; ++j) {
1126 struct ttm_pool_type *pt;
1127
1128 pt = ttm_pool_select_type(pool, caching: i, order: j);
1129 if (pt != &pool->caching[i].orders[j])
1130 continue;
1131
1132 ttm_pool_type_fini(pt);
1133 }
1134 }
1135
1136 /* We removed the pool types from the LRU, but we need to also make sure
1137 * that no shrinker is concurrently freeing pages from the pool.
1138 */
1139 ttm_pool_synchronize_shrinkers();
1140}
1141EXPORT_SYMBOL(ttm_pool_fini);
1142
1143/* Free average pool number of pages. */
1144#define TTM_SHRINKER_BATCH ((1 << (MAX_PAGE_ORDER / 2)) * NR_PAGE_ORDERS)
1145
1146static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink,
1147 struct shrink_control *sc)
1148{
1149 unsigned long num_freed = 0;
1150
1151 do
1152 num_freed += ttm_pool_shrink();
1153 while (num_freed < sc->nr_to_scan &&
1154 atomic_long_read(v: &allocated_pages));
1155
1156 sc->nr_scanned = num_freed;
1157
1158 return num_freed ?: SHRINK_STOP;
1159}
1160
1161/* Return the number of pages available or SHRINK_EMPTY if we have none */
1162static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink,
1163 struct shrink_control *sc)
1164{
1165 unsigned long num_pages = atomic_long_read(v: &allocated_pages);
1166
1167 return num_pages ? num_pages : SHRINK_EMPTY;
1168}
1169
1170#ifdef CONFIG_DEBUG_FS
1171/* Count the number of pages available in a pool_type */
1172static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt)
1173{
1174 unsigned int count = 0;
1175 struct page *p;
1176
1177 spin_lock(lock: &pt->lock);
1178 /* Only used for debugfs, the overhead doesn't matter */
1179 list_for_each_entry(p, &pt->pages, lru)
1180 ++count;
1181 spin_unlock(lock: &pt->lock);
1182
1183 return count;
1184}
1185
1186/* Print a nice header for the order */
1187static void ttm_pool_debugfs_header(struct seq_file *m)
1188{
1189 unsigned int i;
1190
1191 seq_puts(m, s: "\t ");
1192 for (i = 0; i < NR_PAGE_ORDERS; ++i)
1193 seq_printf(m, fmt: " ---%2u---", i);
1194 seq_puts(m, s: "\n");
1195}
1196
1197/* Dump information about the different pool types */
1198static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt,
1199 struct seq_file *m)
1200{
1201 unsigned int i;
1202
1203 for (i = 0; i < NR_PAGE_ORDERS; ++i)
1204 seq_printf(m, fmt: " %8u", ttm_pool_type_count(pt: &pt[i]));
1205 seq_puts(m, s: "\n");
1206}
1207
1208/* Dump the total amount of allocated pages */
1209static void ttm_pool_debugfs_footer(struct seq_file *m)
1210{
1211 seq_printf(m, fmt: "\ntotal\t: %8lu of %8lu\n",
1212 atomic_long_read(v: &allocated_pages), page_pool_size);
1213}
1214
1215/* Dump the information for the global pools */
1216static int ttm_pool_debugfs_globals_show(struct seq_file *m, void *data)
1217{
1218 ttm_pool_debugfs_header(m);
1219
1220 spin_lock(lock: &shrinker_lock);
1221 seq_puts(m, s: "wc\t:");
1222 ttm_pool_debugfs_orders(pt: global_write_combined, m);
1223 seq_puts(m, s: "uc\t:");
1224 ttm_pool_debugfs_orders(pt: global_uncached, m);
1225 seq_puts(m, s: "wc 32\t:");
1226 ttm_pool_debugfs_orders(pt: global_dma32_write_combined, m);
1227 seq_puts(m, s: "uc 32\t:");
1228 ttm_pool_debugfs_orders(pt: global_dma32_uncached, m);
1229 spin_unlock(lock: &shrinker_lock);
1230
1231 ttm_pool_debugfs_footer(m);
1232
1233 return 0;
1234}
1235DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_globals);
1236
1237/**
1238 * ttm_pool_debugfs - Debugfs dump function for a pool
1239 *
1240 * @pool: the pool to dump the information for
1241 * @m: seq_file to dump to
1242 *
1243 * Make a debugfs dump with the per pool and global information.
1244 */
1245int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m)
1246{
1247 unsigned int i;
1248
1249 if (!ttm_pool_uses_dma_alloc(pool) && pool->nid == NUMA_NO_NODE) {
1250 seq_puts(m, s: "unused\n");
1251 return 0;
1252 }
1253
1254 ttm_pool_debugfs_header(m);
1255
1256 spin_lock(lock: &shrinker_lock);
1257 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) {
1258 if (!ttm_pool_select_type(pool, caching: i, order: 0))
1259 continue;
1260 if (ttm_pool_uses_dma_alloc(pool))
1261 seq_puts(m, s: "DMA ");
1262 else
1263 seq_printf(m, fmt: "N%d ", pool->nid);
1264 switch (i) {
1265 case ttm_cached:
1266 seq_puts(m, s: "\t:");
1267 break;
1268 case ttm_write_combined:
1269 seq_puts(m, s: "wc\t:");
1270 break;
1271 case ttm_uncached:
1272 seq_puts(m, s: "uc\t:");
1273 break;
1274 }
1275 ttm_pool_debugfs_orders(pt: pool->caching[i].orders, m);
1276 }
1277 spin_unlock(lock: &shrinker_lock);
1278
1279 ttm_pool_debugfs_footer(m);
1280 return 0;
1281}
1282EXPORT_SYMBOL(ttm_pool_debugfs);
1283
1284/* Test the shrinker functions and dump the result */
1285static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data)
1286{
1287 struct shrink_control sc = {
1288 .gfp_mask = GFP_NOFS,
1289 .nr_to_scan = TTM_SHRINKER_BATCH,
1290 };
1291 unsigned long count;
1292
1293 fs_reclaim_acquire(GFP_KERNEL);
1294 count = ttm_pool_shrinker_count(shrink: mm_shrinker, sc: &sc);
1295 seq_printf(m, fmt: "%lu/%lu\n", count,
1296 ttm_pool_shrinker_scan(shrink: mm_shrinker, sc: &sc));
1297 fs_reclaim_release(GFP_KERNEL);
1298
1299 return 0;
1300}
1301DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink);
1302
1303#endif
1304
1305/**
1306 * ttm_pool_mgr_init - Initialize globals
1307 *
1308 * @num_pages: default number of pages
1309 *
1310 * Initialize the global locks and lists for the MM shrinker.
1311 */
1312int ttm_pool_mgr_init(unsigned long num_pages)
1313{
1314 unsigned int i;
1315
1316 if (!page_pool_size)
1317 page_pool_size = num_pages;
1318
1319 spin_lock_init(&shrinker_lock);
1320 INIT_LIST_HEAD(list: &shrinker_list);
1321
1322 for (i = 0; i < NR_PAGE_ORDERS; ++i) {
1323 ttm_pool_type_init(pt: &global_write_combined[i], NULL,
1324 caching: ttm_write_combined, order: i);
1325 ttm_pool_type_init(pt: &global_uncached[i], NULL, caching: ttm_uncached, order: i);
1326
1327 ttm_pool_type_init(pt: &global_dma32_write_combined[i], NULL,
1328 caching: ttm_write_combined, order: i);
1329 ttm_pool_type_init(pt: &global_dma32_uncached[i], NULL,
1330 caching: ttm_uncached, order: i);
1331 }
1332
1333#ifdef CONFIG_DEBUG_FS
1334 debugfs_create_file("page_pool", 0444, ttm_debugfs_root, NULL,
1335 &ttm_pool_debugfs_globals_fops);
1336 debugfs_create_file("page_pool_shrink", 0400, ttm_debugfs_root, NULL,
1337 &ttm_pool_debugfs_shrink_fops);
1338#ifdef CONFIG_FAULT_INJECTION
1339 fault_create_debugfs_attr(name: "backup_fault_inject", parent: ttm_debugfs_root,
1340 attr: &backup_fault_inject);
1341#endif
1342#endif
1343
1344 mm_shrinker = shrinker_alloc(flags: 0, fmt: "drm-ttm_pool");
1345 if (!mm_shrinker)
1346 return -ENOMEM;
1347
1348 mm_shrinker->count_objects = ttm_pool_shrinker_count;
1349 mm_shrinker->scan_objects = ttm_pool_shrinker_scan;
1350 mm_shrinker->batch = TTM_SHRINKER_BATCH;
1351 mm_shrinker->seeks = 1;
1352
1353 shrinker_register(shrinker: mm_shrinker);
1354
1355 return 0;
1356}
1357
1358/**
1359 * ttm_pool_mgr_fini - Finalize globals
1360 *
1361 * Cleanup the global pools and unregister the MM shrinker.
1362 */
1363void ttm_pool_mgr_fini(void)
1364{
1365 unsigned int i;
1366
1367 for (i = 0; i < NR_PAGE_ORDERS; ++i) {
1368 ttm_pool_type_fini(pt: &global_write_combined[i]);
1369 ttm_pool_type_fini(pt: &global_uncached[i]);
1370
1371 ttm_pool_type_fini(pt: &global_dma32_write_combined[i]);
1372 ttm_pool_type_fini(pt: &global_dma32_uncached[i]);
1373 }
1374
1375 shrinker_free(shrinker: mm_shrinker);
1376 WARN_ON(!list_empty(&shrinker_list));
1377}
1378

source code of linux/drivers/gpu/drm/ttm/ttm_pool.c