1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include <drm/drm_print.h>
7#include <drm/ttm/ttm_placement.h>
8#include <drm/ttm/ttm_tt.h>
9
10#include "i915_drv.h"
11#include "intel_memory_region.h"
12#include "intel_region_ttm.h"
13
14#include "gem/i915_gem_region.h"
15#include "gem/i915_gem_ttm.h"
16#include "gem/i915_gem_ttm_move.h"
17#include "gem/i915_gem_ttm_pm.h"
18
19/**
20 * i915_ttm_backup_free - Free any backup attached to this object
21 * @obj: The object whose backup is to be freed.
22 */
23void i915_ttm_backup_free(struct drm_i915_gem_object *obj)
24{
25 if (obj->ttm.backup) {
26 i915_gem_object_put(obj: obj->ttm.backup);
27 obj->ttm.backup = NULL;
28 }
29}
30
31/**
32 * struct i915_gem_ttm_pm_apply - Apply-to-region subclass for restore
33 * @base: The i915_gem_apply_to_region we derive from.
34 * @allow_gpu: Whether using the gpu blitter is allowed.
35 * @backup_pinned: On backup, backup also pinned objects.
36 */
37struct i915_gem_ttm_pm_apply {
38 struct i915_gem_apply_to_region base;
39 bool allow_gpu : 1;
40 bool backup_pinned : 1;
41};
42
43static int i915_ttm_backup(struct i915_gem_apply_to_region *apply,
44 struct drm_i915_gem_object *obj)
45{
46 struct i915_gem_ttm_pm_apply *pm_apply =
47 container_of(apply, typeof(*pm_apply), base);
48 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
49 struct ttm_buffer_object *backup_bo;
50 struct drm_i915_private *i915 =
51 container_of(bo->bdev, typeof(*i915), bdev);
52 struct drm_i915_gem_object *backup;
53 struct ttm_operation_ctx ctx = {};
54 unsigned int flags;
55 int err = 0;
56
57 if (!i915_ttm_cpu_maps_iomem(mem: bo->resource) || obj->ttm.backup)
58 return 0;
59
60 if (pm_apply->allow_gpu && i915_gem_object_evictable(obj))
61 return ttm_bo_validate(bo, placement: i915_ttm_sys_placement(), ctx: &ctx);
62
63 if (!pm_apply->backup_pinned ||
64 (pm_apply->allow_gpu && (obj->flags & I915_BO_ALLOC_PM_EARLY)))
65 return 0;
66
67 if (obj->flags & I915_BO_ALLOC_PM_VOLATILE)
68 return 0;
69
70 /*
71 * It seems that we might have some framebuffers still pinned at this
72 * stage, but for such objects we might also need to deal with the CCS
73 * aux state. Make sure we force the save/restore of the CCS state,
74 * otherwise we might observe display corruption, when returning from
75 * suspend.
76 */
77 flags = 0;
78 if (i915_gem_object_needs_ccs_pages(obj)) {
79 WARN_ON_ONCE(!i915_gem_object_is_framebuffer(obj));
80 WARN_ON_ONCE(!pm_apply->allow_gpu);
81
82 flags = I915_BO_ALLOC_CCS_AUX;
83 }
84 backup = i915_gem_object_create_region(mem: i915->mm.regions[INTEL_REGION_SMEM],
85 size: obj->base.size, page_size: 0, flags);
86 if (IS_ERR(ptr: backup))
87 return PTR_ERR(ptr: backup);
88
89 err = i915_gem_object_lock(obj: backup, ww: apply->ww);
90 if (err)
91 goto out_no_lock;
92
93 backup_bo = i915_gem_to_ttm(obj: backup);
94 err = ttm_bo_populate(bo: backup_bo, ctx: &ctx);
95 if (err)
96 goto out_no_populate;
97
98 err = i915_gem_obj_copy_ttm(dst: backup, src: obj, allow_accel: pm_apply->allow_gpu, intr: false);
99 if (err) {
100 drm_err(&i915->drm,
101 "Unable to copy from device to system memory, err:%pe\n",
102 ERR_PTR(err));
103 goto out_no_populate;
104 }
105 ttm_bo_wait_ctx(bo: backup_bo, ctx: &ctx);
106
107 obj->ttm.backup = backup;
108 return 0;
109
110out_no_populate:
111 i915_gem_ww_unlock_single(obj: backup);
112out_no_lock:
113 i915_gem_object_put(obj: backup);
114
115 return err;
116}
117
118static int i915_ttm_recover(struct i915_gem_apply_to_region *apply,
119 struct drm_i915_gem_object *obj)
120{
121 i915_ttm_backup_free(obj);
122 return 0;
123}
124
125/**
126 * i915_ttm_recover_region - Free the backup of all objects of a region
127 * @mr: The memory region
128 *
129 * Checks all objects of a region if there is backup attached and if so
130 * frees that backup. Typically this is called to recover after a partially
131 * performed backup.
132 */
133void i915_ttm_recover_region(struct intel_memory_region *mr)
134{
135 static const struct i915_gem_apply_to_region_ops recover_ops = {
136 .process_obj = i915_ttm_recover,
137 };
138 struct i915_gem_apply_to_region apply = {.ops = &recover_ops};
139 int ret;
140
141 ret = i915_gem_process_region(mr, apply: &apply);
142 GEM_WARN_ON(ret);
143}
144
145/**
146 * i915_ttm_backup_region - Back up all objects of a region to smem.
147 * @mr: The memory region
148 * @flags: TTM backup flags
149 *
150 * Loops over all objects of a region and either evicts them if they are
151 * evictable or backs them up using a backup object if they are pinned.
152 *
153 * Return: Zero on success. Negative error code on error.
154 */
155int i915_ttm_backup_region(struct intel_memory_region *mr, u32 flags)
156{
157 static const struct i915_gem_apply_to_region_ops backup_ops = {
158 .process_obj = i915_ttm_backup,
159 };
160 struct i915_gem_ttm_pm_apply pm_apply = {
161 .base = {.ops = &backup_ops},
162 .allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
163 .backup_pinned = flags & I915_TTM_BACKUP_PINNED,
164 };
165
166 return i915_gem_process_region(mr, apply: &pm_apply.base);
167}
168
169static int i915_ttm_restore(struct i915_gem_apply_to_region *apply,
170 struct drm_i915_gem_object *obj)
171{
172 struct i915_gem_ttm_pm_apply *pm_apply =
173 container_of(apply, typeof(*pm_apply), base);
174 struct drm_i915_gem_object *backup = obj->ttm.backup;
175 struct ttm_buffer_object *backup_bo = i915_gem_to_ttm(obj: backup);
176 struct ttm_operation_ctx ctx = {};
177 int err;
178
179 if (!backup)
180 return 0;
181
182 if (!pm_apply->allow_gpu && !(obj->flags & I915_BO_ALLOC_PM_EARLY))
183 return 0;
184
185 err = i915_gem_object_lock(obj: backup, ww: apply->ww);
186 if (err)
187 return err;
188
189 /* Content may have been swapped. */
190 if (!backup_bo->resource)
191 err = ttm_bo_validate(bo: backup_bo, placement: i915_ttm_sys_placement(), ctx: &ctx);
192 if (!err)
193 err = ttm_bo_populate(bo: backup_bo, ctx: &ctx);
194 if (!err) {
195 err = i915_gem_obj_copy_ttm(dst: obj, src: backup, allow_accel: pm_apply->allow_gpu,
196 intr: false);
197 GEM_WARN_ON(err);
198 ttm_bo_wait_ctx(bo: backup_bo, ctx: &ctx);
199
200 obj->ttm.backup = NULL;
201 err = 0;
202 }
203
204 i915_gem_ww_unlock_single(obj: backup);
205
206 if (!err)
207 i915_gem_object_put(obj: backup);
208
209 return err;
210}
211
212/**
213 * i915_ttm_restore_region - Restore backed-up objects of a region from smem.
214 * @mr: The memory region
215 * @flags: TTM backup flags
216 *
217 * Loops over all objects of a region and if they are backed-up, restores
218 * them from smem.
219 *
220 * Return: Zero on success. Negative error code on error.
221 */
222int i915_ttm_restore_region(struct intel_memory_region *mr, u32 flags)
223{
224 static const struct i915_gem_apply_to_region_ops restore_ops = {
225 .process_obj = i915_ttm_restore,
226 };
227 struct i915_gem_ttm_pm_apply pm_apply = {
228 .base = {.ops = &restore_ops},
229 .allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
230 };
231
232 return i915_gem_process_region(mr, apply: &pm_apply.base);
233}
234

source code of linux/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c