1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2024 Intel Corporation
4 */
5
6#include <linux/shrinker.h>
7
8#include <drm/drm_managed.h>
9#include <drm/ttm/ttm_backup.h>
10#include <drm/ttm/ttm_bo.h>
11#include <drm/ttm/ttm_tt.h>
12
13#include "xe_bo.h"
14#include "xe_pm.h"
15#include "xe_shrinker.h"
16
17/**
18 * struct xe_shrinker - per-device shrinker
19 * @xe: Back pointer to the device.
20 * @lock: Lock protecting accounting.
21 * @shrinkable_pages: Number of pages that are currently shrinkable.
22 * @purgeable_pages: Number of pages that are currently purgeable.
23 * @shrink: Pointer to the mm shrinker.
24 * @pm_worker: Worker to wake up the device if required.
25 */
26struct xe_shrinker {
27 struct xe_device *xe;
28 rwlock_t lock;
29 long shrinkable_pages;
30 long purgeable_pages;
31 struct shrinker *shrink;
32 struct work_struct pm_worker;
33};
34
35static struct xe_shrinker *to_xe_shrinker(struct shrinker *shrink)
36{
37 return shrink->private_data;
38}
39
40/**
41 * xe_shrinker_mod_pages() - Modify shrinker page accounting
42 * @shrinker: Pointer to the struct xe_shrinker.
43 * @shrinkable: Shrinkable pages delta. May be negative.
44 * @purgeable: Purgeable page delta. May be negative.
45 *
46 * Modifies the shrinkable and purgeable pages accounting.
47 */
48void
49xe_shrinker_mod_pages(struct xe_shrinker *shrinker, long shrinkable, long purgeable)
50{
51 write_lock(&shrinker->lock);
52 shrinker->shrinkable_pages += shrinkable;
53 shrinker->purgeable_pages += purgeable;
54 write_unlock(&shrinker->lock);
55}
56
57static s64 __xe_shrinker_walk(struct xe_device *xe,
58 struct ttm_operation_ctx *ctx,
59 const struct xe_bo_shrink_flags flags,
60 unsigned long to_scan, unsigned long *scanned)
61{
62 unsigned int mem_type;
63 s64 freed = 0, lret;
64
65 for (mem_type = XE_PL_SYSTEM; mem_type <= XE_PL_TT; ++mem_type) {
66 struct ttm_resource_manager *man = ttm_manager_type(bdev: &xe->ttm, mem_type);
67 struct ttm_bo_lru_cursor curs;
68 struct ttm_buffer_object *ttm_bo;
69 struct ttm_lru_walk_arg arg = {
70 .ctx = ctx,
71 .trylock_only = true,
72 };
73
74 if (!man || !man->use_tt)
75 continue;
76
77 ttm_bo_lru_for_each_reserved_guarded(&curs, man, &arg, ttm_bo) {
78 if (!ttm_bo_shrink_suitable(bo: ttm_bo, ctx))
79 continue;
80
81 lret = xe_bo_shrink(ctx, bo: ttm_bo, flags, scanned);
82 if (lret < 0)
83 return lret;
84
85 freed += lret;
86 if (*scanned >= to_scan)
87 break;
88 }
89 /* Trylocks should never error, just fail. */
90 xe_assert(xe, !IS_ERR(ttm_bo));
91 }
92
93 return freed;
94}
95
96/*
97 * Try shrinking idle objects without writeback first, then if not sufficient,
98 * try also non-idle objects and finally if that's not sufficient either,
99 * add writeback. This avoids stalls and explicit writebacks with light or
100 * moderate memory pressure.
101 */
102static s64 xe_shrinker_walk(struct xe_device *xe,
103 struct ttm_operation_ctx *ctx,
104 const struct xe_bo_shrink_flags flags,
105 unsigned long to_scan, unsigned long *scanned)
106{
107 bool no_wait_gpu = true;
108 struct xe_bo_shrink_flags save_flags = flags;
109 s64 lret, freed;
110
111 swap(no_wait_gpu, ctx->no_wait_gpu);
112 save_flags.writeback = false;
113 lret = __xe_shrinker_walk(xe, ctx, flags: save_flags, to_scan, scanned);
114 swap(no_wait_gpu, ctx->no_wait_gpu);
115 if (lret < 0 || *scanned >= to_scan)
116 return lret;
117
118 freed = lret;
119 if (!ctx->no_wait_gpu) {
120 lret = __xe_shrinker_walk(xe, ctx, flags: save_flags, to_scan, scanned);
121 if (lret < 0)
122 return lret;
123 freed += lret;
124 if (*scanned >= to_scan)
125 return freed;
126 }
127
128 if (flags.writeback) {
129 lret = __xe_shrinker_walk(xe, ctx, flags, to_scan, scanned);
130 if (lret < 0)
131 return lret;
132 freed += lret;
133 }
134
135 return freed;
136}
137
138static unsigned long
139xe_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
140{
141 struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
142 unsigned long num_pages;
143 bool can_backup = !!(sc->gfp_mask & __GFP_FS);
144
145 num_pages = ttm_backup_bytes_avail() >> PAGE_SHIFT;
146 read_lock(&shrinker->lock);
147
148 if (can_backup)
149 num_pages = min_t(unsigned long, num_pages, shrinker->shrinkable_pages);
150 else
151 num_pages = 0;
152
153 num_pages += shrinker->purgeable_pages;
154 read_unlock(&shrinker->lock);
155
156 return num_pages ? num_pages : SHRINK_EMPTY;
157}
158
159/*
160 * Check if we need runtime pm, and if so try to grab a reference if
161 * already active. If grabbing a reference fails, queue a worker that
162 * does it for us outside of reclaim, but don't wait for it to complete.
163 * If bo shrinking needs an rpm reference and we don't have it (yet),
164 * that bo will be skipped anyway.
165 */
166static bool xe_shrinker_runtime_pm_get(struct xe_shrinker *shrinker, bool force,
167 unsigned long nr_to_scan, bool can_backup)
168{
169 struct xe_device *xe = shrinker->xe;
170
171 if (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe) ||
172 !ttm_backup_bytes_avail())
173 return false;
174
175 if (!force) {
176 read_lock(&shrinker->lock);
177 force = (nr_to_scan > shrinker->purgeable_pages && can_backup);
178 read_unlock(&shrinker->lock);
179 if (!force)
180 return false;
181 }
182
183 if (!xe_pm_runtime_get_if_active(xe)) {
184 if (xe_rpm_reclaim_safe(xe) && !ttm_bo_shrink_avoid_wait()) {
185 xe_pm_runtime_get(xe);
186 return true;
187 }
188 queue_work(wq: xe->unordered_wq, work: &shrinker->pm_worker);
189 return false;
190 }
191
192 return true;
193}
194
195static void xe_shrinker_runtime_pm_put(struct xe_shrinker *shrinker, bool runtime_pm)
196{
197 if (runtime_pm)
198 xe_pm_runtime_put(xe: shrinker->xe);
199}
200
201static unsigned long xe_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
202{
203 struct xe_shrinker *shrinker = to_xe_shrinker(shrink);
204 struct ttm_operation_ctx ctx = {
205 .interruptible = false,
206 .no_wait_gpu = ttm_bo_shrink_avoid_wait(),
207 };
208 unsigned long nr_to_scan, nr_scanned = 0, freed = 0;
209 struct xe_bo_shrink_flags shrink_flags = {
210 .purge = true,
211 /* Don't request writeback without __GFP_IO. */
212 .writeback = !ctx.no_wait_gpu && (sc->gfp_mask & __GFP_IO),
213 };
214 bool runtime_pm;
215 bool purgeable;
216 bool can_backup = !!(sc->gfp_mask & __GFP_FS);
217 s64 lret;
218
219 nr_to_scan = sc->nr_to_scan;
220
221 read_lock(&shrinker->lock);
222 purgeable = !!shrinker->purgeable_pages;
223 read_unlock(&shrinker->lock);
224
225 /* Might need runtime PM. Try to wake early if it looks like it. */
226 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, force: false, nr_to_scan, can_backup);
227
228 if (purgeable && nr_scanned < nr_to_scan) {
229 lret = xe_shrinker_walk(xe: shrinker->xe, ctx: &ctx, flags: shrink_flags,
230 to_scan: nr_to_scan, scanned: &nr_scanned);
231 if (lret >= 0)
232 freed += lret;
233 }
234
235 sc->nr_scanned = nr_scanned;
236 if (nr_scanned >= nr_to_scan || !can_backup)
237 goto out;
238
239 /* If we didn't wake before, try to do it now if needed. */
240 if (!runtime_pm)
241 runtime_pm = xe_shrinker_runtime_pm_get(shrinker, force: true, nr_to_scan: 0, can_backup);
242
243 shrink_flags.purge = false;
244
245 lret = xe_shrinker_walk(xe: shrinker->xe, ctx: &ctx, flags: shrink_flags,
246 to_scan: nr_to_scan, scanned: &nr_scanned);
247 if (lret >= 0)
248 freed += lret;
249
250 sc->nr_scanned = nr_scanned;
251out:
252 xe_shrinker_runtime_pm_put(shrinker, runtime_pm);
253 return nr_scanned ? freed : SHRINK_STOP;
254}
255
256/* Wake up the device for shrinking. */
257static void xe_shrinker_pm(struct work_struct *work)
258{
259 struct xe_shrinker *shrinker =
260 container_of(work, typeof(*shrinker), pm_worker);
261
262 xe_pm_runtime_get(xe: shrinker->xe);
263 xe_pm_runtime_put(xe: shrinker->xe);
264}
265
266static void xe_shrinker_fini(struct drm_device *drm, void *arg)
267{
268 struct xe_shrinker *shrinker = arg;
269
270 xe_assert(shrinker->xe, !shrinker->shrinkable_pages);
271 xe_assert(shrinker->xe, !shrinker->purgeable_pages);
272 shrinker_free(shrinker: shrinker->shrink);
273 flush_work(work: &shrinker->pm_worker);
274 kfree(objp: shrinker);
275}
276
277/**
278 * xe_shrinker_create() - Create an xe per-device shrinker
279 * @xe: Pointer to the xe device.
280 *
281 * Return: %0 on success. Negative error code on failure.
282 */
283int xe_shrinker_create(struct xe_device *xe)
284{
285 struct xe_shrinker *shrinker = kzalloc(sizeof(*shrinker), GFP_KERNEL);
286
287 if (!shrinker)
288 return -ENOMEM;
289
290 shrinker->shrink = shrinker_alloc(flags: 0, fmt: "drm-xe_gem:%s", xe->drm.unique);
291 if (!shrinker->shrink) {
292 kfree(objp: shrinker);
293 return -ENOMEM;
294 }
295
296 INIT_WORK(&shrinker->pm_worker, xe_shrinker_pm);
297 shrinker->xe = xe;
298 rwlock_init(&shrinker->lock);
299 shrinker->shrink->count_objects = xe_shrinker_count;
300 shrinker->shrink->scan_objects = xe_shrinker_scan;
301 shrinker->shrink->private_data = shrinker;
302 shrinker_register(shrinker: shrinker->shrink);
303 xe->mem.shrinker = shrinker;
304
305 return drmm_add_action_or_reset(&xe->drm, xe_shrinker_fini, shrinker);
306}
307

source code of linux/drivers/gpu/drm/xe/xe_shrinker.c